[clang-tools-extra] r363803 - [clangd] Collect tokens of main files when building the AST
Ilya Biryukov via cfe-commits
cfe-commits at lists.llvm.org
Wed Jun 19 07:03:20 PDT 2019
Author: ibiryukov
Date: Wed Jun 19 07:03:19 2019
New Revision: 363803
URL: http://llvm.org/viewvc/llvm-project?rev=363803&view=rev
Log:
[clangd] Collect tokens of main files when building the AST
Summary:
The first use of this is a code tweak to expand macro calls.
Will later be used to build syntax trees.
The memory overhead is small as we only store tokens of the main file.
Reviewers: sammccall
Reviewed By: sammccall
Subscribers: mgorny, MaskRay, jkorous, arphaman, kadircet, cfe-commits
Tags: #clang
Differential Revision: https://reviews.llvm.org/D62956
Modified:
clang-tools-extra/trunk/clangd/CMakeLists.txt
clang-tools-extra/trunk/clangd/ClangdUnit.cpp
clang-tools-extra/trunk/clangd/ClangdUnit.h
clang-tools-extra/trunk/clangd/tool/CMakeLists.txt
clang-tools-extra/trunk/clangd/unittests/CMakeLists.txt
clang-tools-extra/trunk/clangd/unittests/ClangdUnitTests.cpp
Modified: clang-tools-extra/trunk/clangd/CMakeLists.txt
URL: http://llvm.org/viewvc/llvm-project/clang-tools-extra/trunk/clangd/CMakeLists.txt?rev=363803&r1=363802&r2=363803&view=diff
==============================================================================
--- clang-tools-extra/trunk/clangd/CMakeLists.txt (original)
+++ clang-tools-extra/trunk/clangd/CMakeLists.txt Wed Jun 19 07:03:19 2019
@@ -128,6 +128,7 @@ add_clang_library(clangDaemon
clangToolingCore
clangToolingInclusions
clangToolingRefactoring
+ clangToolingSyntax
${LLVM_PTHREAD_LIB}
${CLANGD_ATOMIC_LIB}
)
Modified: clang-tools-extra/trunk/clangd/ClangdUnit.cpp
URL: http://llvm.org/viewvc/llvm-project/clang-tools-extra/trunk/clangd/ClangdUnit.cpp?rev=363803&r1=363802&r2=363803&view=diff
==============================================================================
--- clang-tools-extra/trunk/clangd/ClangdUnit.cpp (original)
+++ clang-tools-extra/trunk/clangd/ClangdUnit.cpp Wed Jun 19 07:03:19 2019
@@ -36,6 +36,7 @@
#include "clang/Serialization/ASTWriter.h"
#include "clang/Serialization/PCHContainerOperations.h"
#include "clang/Tooling/CompilationDatabase.h"
+#include "clang/Tooling/Syntax/Tokens.h"
#include "llvm/ADT/ArrayRef.h"
#include "llvm/ADT/STLExtras.h"
#include "llvm/ADT/SmallString.h"
@@ -418,6 +419,9 @@ ParsedAST::build(std::unique_ptr<Compile
collectIWYUHeaderMaps(&CanonIncludes);
Clang->getPreprocessor().addCommentHandler(IWYUHandler.get());
+ // Collect tokens of the main file.
+ syntax::TokenCollector Tokens(Clang->getPreprocessor());
+
if (!Action->Execute())
log("Execute() failed when building AST for {0}", MainInput.getFile());
@@ -446,8 +450,9 @@ ParsedAST::build(std::unique_ptr<Compile
if (Preamble)
Diags.insert(Diags.begin(), Preamble->Diags.begin(), Preamble->Diags.end());
return ParsedAST(std::move(Preamble), std::move(Clang), std::move(Action),
- std::move(ParsedDecls), std::move(Diags),
- std::move(Includes), std::move(CanonIncludes));
+ std::move(Tokens).consume(), std::move(ParsedDecls),
+ std::move(Diags), std::move(Includes),
+ std::move(CanonIncludes));
}
ParsedAST::ParsedAST(ParsedAST &&Other) = default;
@@ -540,11 +545,13 @@ PreambleData::PreambleData(PrecompiledPr
ParsedAST::ParsedAST(std::shared_ptr<const PreambleData> Preamble,
std::unique_ptr<CompilerInstance> Clang,
std::unique_ptr<FrontendAction> Action,
+ syntax::TokenBuffer Tokens,
std::vector<Decl *> LocalTopLevelDecls,
std::vector<Diag> Diags, IncludeStructure Includes,
CanonicalIncludes CanonIncludes)
: Preamble(std::move(Preamble)), Clang(std::move(Clang)),
- Action(std::move(Action)), Diags(std::move(Diags)),
+ Action(std::move(Action)), Tokens(std::move(Tokens)),
+ Diags(std::move(Diags)),
LocalTopLevelDecls(std::move(LocalTopLevelDecls)),
Includes(std::move(Includes)), CanonIncludes(std::move(CanonIncludes)) {
assert(this->Clang);
Modified: clang-tools-extra/trunk/clangd/ClangdUnit.h
URL: http://llvm.org/viewvc/llvm-project/clang-tools-extra/trunk/clangd/ClangdUnit.h?rev=363803&r1=363802&r2=363803&view=diff
==============================================================================
--- clang-tools-extra/trunk/clangd/ClangdUnit.h (original)
+++ clang-tools-extra/trunk/clangd/ClangdUnit.h Wed Jun 19 07:03:19 2019
@@ -24,6 +24,7 @@
#include "clang/Serialization/ASTBitCodes.h"
#include "clang/Tooling/CompilationDatabase.h"
#include "clang/Tooling/Core/Replacement.h"
+#include "clang/Tooling/Syntax/Tokens.h"
#include <memory>
#include <string>
#include <vector>
@@ -115,10 +116,14 @@ public:
const IncludeStructure &getIncludeStructure() const;
const CanonicalIncludes &getCanonicalIncludes() const;
+ /// Tokens recorded while parsing the main file.
+ /// (!) does not have tokens from the preamble.
+ const syntax::TokenBuffer &getTokens() const { return Tokens; }
+
private:
ParsedAST(std::shared_ptr<const PreambleData> Preamble,
std::unique_ptr<CompilerInstance> Clang,
- std::unique_ptr<FrontendAction> Action,
+ std::unique_ptr<FrontendAction> Action, syntax::TokenBuffer Tokens,
std::vector<Decl *> LocalTopLevelDecls, std::vector<Diag> Diags,
IncludeStructure Includes, CanonicalIncludes CanonIncludes);
@@ -132,6 +137,11 @@ private:
// FrontendAction.EndSourceFile).
std::unique_ptr<CompilerInstance> Clang;
std::unique_ptr<FrontendAction> Action;
+ /// Tokens recorded after the preamble finished.
+ /// - Includes all spelled tokens for the main file.
+ /// - Includes expanded tokens produced **after** preabmle.
+ /// - Does not have spelled or expanded tokens for files from preamble.
+ syntax::TokenBuffer Tokens;
// Data, stored after parsing.
std::vector<Diag> Diags;
Modified: clang-tools-extra/trunk/clangd/tool/CMakeLists.txt
URL: http://llvm.org/viewvc/llvm-project/clang-tools-extra/trunk/clangd/tool/CMakeLists.txt?rev=363803&r1=363802&r2=363803&view=diff
==============================================================================
--- clang-tools-extra/trunk/clangd/tool/CMakeLists.txt (original)
+++ clang-tools-extra/trunk/clangd/tool/CMakeLists.txt Wed Jun 19 07:03:19 2019
@@ -26,5 +26,6 @@ target_link_libraries(clangd
clangSema
clangTooling
clangToolingCore
+ clangToolingSyntax
${CLANGD_XPC_LIBS}
)
Modified: clang-tools-extra/trunk/clangd/unittests/CMakeLists.txt
URL: http://llvm.org/viewvc/llvm-project/clang-tools-extra/trunk/clangd/unittests/CMakeLists.txt?rev=363803&r1=363802&r2=363803&view=diff
==============================================================================
--- clang-tools-extra/trunk/clangd/unittests/CMakeLists.txt (original)
+++ clang-tools-extra/trunk/clangd/unittests/CMakeLists.txt Wed Jun 19 07:03:19 2019
@@ -87,6 +87,7 @@ target_link_libraries(ClangdTests
clangTooling
clangToolingCore
clangToolingInclusions
+ clangToolingSyntax
LLVMSupport
LLVMTestingSupport
)
Modified: clang-tools-extra/trunk/clangd/unittests/ClangdUnitTests.cpp
URL: http://llvm.org/viewvc/llvm-project/clang-tools-extra/trunk/clangd/unittests/ClangdUnitTests.cpp?rev=363803&r1=363802&r2=363803&view=diff
==============================================================================
--- clang-tools-extra/trunk/clangd/unittests/ClangdUnitTests.cpp (original)
+++ clang-tools-extra/trunk/clangd/unittests/ClangdUnitTests.cpp Wed Jun 19 07:03:19 2019
@@ -10,6 +10,8 @@
#include "ClangdUnit.h"
#include "SourceCode.h"
#include "TestTU.h"
+#include "clang/Basic/TokenKinds.h"
+#include "clang/Tooling/Syntax/Tokens.h"
#include "llvm/Support/ScopedPrinter.h"
#include "gmock/gmock.h"
#include "gtest/gtest.h"
@@ -81,6 +83,37 @@ TEST(ClangdUnitTest, TopLevelDecls) {
EXPECT_THAT(AST.getLocalTopLevelDecls(), ElementsAre(DeclNamed("main")));
}
+TEST(ClangdUnitTest, TokensAfterPreamble) {
+ TestTU TU;
+ TU.AdditionalFiles["foo.h"] = R"(
+ int foo();
+ )";
+ TU.Code = R"cpp(
+ #include "foo.h"
+ first_token;
+ void test() {
+ }
+ last_token
+)cpp";
+ auto AST = TU.build();
+ const syntax::TokenBuffer &T = AST.getTokens();
+ const auto &SM = AST.getSourceManager();
+
+ ASSERT_GT(T.expandedTokens().size(), 2u);
+ // Check first token after the preamble.
+ EXPECT_EQ(T.expandedTokens().front().text(SM), "first_token");
+ // Last token is always 'eof'.
+ EXPECT_EQ(T.expandedTokens().back().kind(), tok::eof);
+ // Check the token before 'eof'.
+ EXPECT_EQ(T.expandedTokens().drop_back().back().text(SM), "last_token");
+
+ // The spelled tokens for the main file should have everything.
+ auto Spelled = T.spelledTokens(SM.getMainFileID());
+ ASSERT_FALSE(Spelled.empty());
+ EXPECT_EQ(Spelled.front().kind(), tok::hash);
+ EXPECT_EQ(Spelled.back().text(SM), "last_token");
+}
+
} // namespace
} // namespace clangd
} // namespace clang
More information about the cfe-commits
mailing list