r330159 - Defer adding keywords to the identifier table until after the language options have been loaded from the AST file.
Aaron Ballman via cfe-commits
cfe-commits at lists.llvm.org
Mon Apr 16 14:07:08 PDT 2018
Author: aaronballman
Date: Mon Apr 16 14:07:08 2018
New Revision: 330159
URL: http://llvm.org/viewvc/llvm-project?rev=330159&view=rev
Log:
Defer adding keywords to the identifier table until after the language options have been loaded from the AST file.
This fixes issues with "class" being reported as an identifier in "enum class" because the construct is not present when using default language options.
Patch by Johann Klähn.
Modified:
cfe/trunk/include/clang/Basic/IdentifierTable.h
cfe/trunk/lib/Basic/IdentifierTable.cpp
cfe/trunk/lib/Lex/Preprocessor.cpp
cfe/trunk/unittests/libclang/LibclangTest.cpp
Modified: cfe/trunk/include/clang/Basic/IdentifierTable.h
URL: http://llvm.org/viewvc/llvm-project/cfe/trunk/include/clang/Basic/IdentifierTable.h?rev=330159&r1=330158&r2=330159&view=diff
==============================================================================
--- cfe/trunk/include/clang/Basic/IdentifierTable.h (original)
+++ cfe/trunk/include/clang/Basic/IdentifierTable.h Mon Apr 16 14:07:08 2018
@@ -467,10 +467,13 @@ class IdentifierTable {
IdentifierInfoLookup* ExternalLookup;
public:
+ /// \brief Create the identifier table.
+ explicit IdentifierTable(IdentifierInfoLookup *ExternalLookup = nullptr);
+
/// \brief Create the identifier table, populating it with info about the
/// language keywords for the language specified by \p LangOpts.
- IdentifierTable(const LangOptions &LangOpts,
- IdentifierInfoLookup* externalLookup = nullptr);
+ explicit IdentifierTable(const LangOptions &LangOpts,
+ IdentifierInfoLookup *ExternalLookup = nullptr);
/// \brief Set the external identifier lookup mechanism.
void setExternalIdentifierLookup(IdentifierInfoLookup *IILookup) {
@@ -558,6 +561,8 @@ public:
/// hashing is doing.
void PrintStats() const;
+ /// \brief Populate the identifier table with info about the language keywords
+ /// for the language specified by \p LangOpts.
void AddKeywords(const LangOptions &LangOpts);
};
Modified: cfe/trunk/lib/Basic/IdentifierTable.cpp
URL: http://llvm.org/viewvc/llvm-project/cfe/trunk/lib/Basic/IdentifierTable.cpp?rev=330159&r1=330158&r2=330159&view=diff
==============================================================================
--- cfe/trunk/lib/Basic/IdentifierTable.cpp (original)
+++ cfe/trunk/lib/Basic/IdentifierTable.cpp Mon Apr 16 14:07:08 2018
@@ -79,16 +79,16 @@ IdentifierIterator *IdentifierInfoLookup
return new EmptyLookupIterator();
}
+IdentifierTable::IdentifierTable(IdentifierInfoLookup *ExternalLookup)
+ : HashTable(8192), // Start with space for 8K identifiers.
+ ExternalLookup(ExternalLookup) {}
+
IdentifierTable::IdentifierTable(const LangOptions &LangOpts,
- IdentifierInfoLookup* externalLookup)
- : HashTable(8192), // Start with space for 8K identifiers.
- ExternalLookup(externalLookup) {
+ IdentifierInfoLookup *ExternalLookup)
+ : IdentifierTable(ExternalLookup) {
// Populate the identifier table with info about keywords for the current
// language.
AddKeywords(LangOpts);
-
- // Add the '_experimental_modules_import' contextual keyword.
- get("import").setModulesImport(true);
}
//===----------------------------------------------------------------------===//
@@ -237,6 +237,9 @@ void IdentifierTable::AddKeywords(const
if (LangOpts.DeclSpecKeyword)
AddKeyword("__declspec", tok::kw___declspec, KEYALL, LangOpts, *this);
+
+ // Add the '_experimental_modules_import' contextual keyword.
+ get("import").setModulesImport(true);
}
/// \brief Checks if the specified token kind represents a keyword in the
Modified: cfe/trunk/lib/Lex/Preprocessor.cpp
URL: http://llvm.org/viewvc/llvm-project/cfe/trunk/lib/Lex/Preprocessor.cpp?rev=330159&r1=330158&r2=330159&view=diff
==============================================================================
--- cfe/trunk/lib/Lex/Preprocessor.cpp (original)
+++ cfe/trunk/lib/Lex/Preprocessor.cpp Mon Apr 16 14:07:08 2018
@@ -85,12 +85,14 @@ Preprocessor::Preprocessor(std::shared_p
IdentifierInfoLookup *IILookup, bool OwnsHeaders,
TranslationUnitKind TUKind)
: PPOpts(std::move(PPOpts)), Diags(&diags), LangOpts(opts),
- FileMgr(Headers.getFileMgr()), SourceMgr(SM),
- PCMCache(PCMCache), ScratchBuf(new ScratchBuffer(SourceMgr)),
- HeaderInfo(Headers), TheModuleLoader(TheModuleLoader),
- ExternalSource(nullptr), Identifiers(opts, IILookup),
- PragmaHandlers(new PragmaNamespace(StringRef())), TUKind(TUKind),
- SkipMainFilePreamble(0, true),
+ FileMgr(Headers.getFileMgr()), SourceMgr(SM), PCMCache(PCMCache),
+ ScratchBuf(new ScratchBuffer(SourceMgr)), HeaderInfo(Headers),
+ TheModuleLoader(TheModuleLoader), ExternalSource(nullptr),
+ // As the language options may have not been loaded yet (when
+ // deserializing an ASTUnit), adding keywords to the identifier table is
+ // deferred to Preprocessor::Initialize().
+ Identifiers(IILookup), PragmaHandlers(new PragmaNamespace(StringRef())),
+ TUKind(TUKind), SkipMainFilePreamble(0, true),
CurSubmoduleState(&NullSubmoduleState) {
OwnsHeaderSearch = OwnsHeaders;
@@ -190,6 +192,9 @@ void Preprocessor::Initialize(const Targ
// Initialize information about built-ins.
BuiltinInfo.InitializeTarget(Target, AuxTarget);
HeaderInfo.setTarget(Target);
+
+ // Populate the identifier table with info about keywords for the current language.
+ Identifiers.AddKeywords(LangOpts);
}
void Preprocessor::InitializeForModelFile() {
Modified: cfe/trunk/unittests/libclang/LibclangTest.cpp
URL: http://llvm.org/viewvc/llvm-project/cfe/trunk/unittests/libclang/LibclangTest.cpp?rev=330159&r1=330158&r2=330159&view=diff
==============================================================================
--- cfe/trunk/unittests/libclang/LibclangTest.cpp (original)
+++ cfe/trunk/unittests/libclang/LibclangTest.cpp Mon Apr 16 14:07:08 2018
@@ -698,3 +698,67 @@ TEST_F(LibclangReparseTest, Preprocessor
clang_disposeSourceRangeList(Ranges);
}
}
+
+class LibclangSerializationTest : public LibclangParseTest {
+public:
+ bool SaveAndLoadTU(const std::string &Filename) {
+ unsigned options = clang_defaultSaveOptions(ClangTU);
+ if (clang_saveTranslationUnit(ClangTU, Filename.c_str(), options) !=
+ CXSaveError_None) {
+ DEBUG(llvm::dbgs() << "Saving failed\n");
+ return false;
+ }
+
+ clang_disposeTranslationUnit(ClangTU);
+
+ ClangTU = clang_createTranslationUnit(Index, Filename.c_str());
+
+ if (!ClangTU) {
+ DEBUG(llvm::dbgs() << "Loading failed\n");
+ return false;
+ }
+
+ return true;
+ }
+};
+
+TEST_F(LibclangSerializationTest, TokenKindsAreCorrectAfterLoading) {
+ // Ensure that "class" is recognized as a keyword token after serializing
+ // and reloading the AST, as it is not a keyword for the default LangOptions.
+ std::string HeaderName = "test.h";
+ WriteFile(HeaderName, "enum class Something {};");
+
+ const char *Argv[] = {"-xc++-header", "-std=c++11"};
+
+ ClangTU = clang_parseTranslationUnit(Index, HeaderName.c_str(), Argv,
+ sizeof(Argv) / sizeof(Argv[0]), nullptr,
+ 0, TUFlags);
+
+ auto CheckTokenKinds = [=]() {
+ CXSourceRange Range =
+ clang_getCursorExtent(clang_getTranslationUnitCursor(ClangTU));
+
+ CXToken *Tokens;
+ unsigned int NumTokens;
+ clang_tokenize(ClangTU, Range, &Tokens, &NumTokens);
+
+ ASSERT_EQ(6u, NumTokens);
+ EXPECT_EQ(CXToken_Keyword, clang_getTokenKind(Tokens[0]));
+ EXPECT_EQ(CXToken_Keyword, clang_getTokenKind(Tokens[1]));
+ EXPECT_EQ(CXToken_Identifier, clang_getTokenKind(Tokens[2]));
+ EXPECT_EQ(CXToken_Punctuation, clang_getTokenKind(Tokens[3]));
+ EXPECT_EQ(CXToken_Punctuation, clang_getTokenKind(Tokens[4]));
+ EXPECT_EQ(CXToken_Punctuation, clang_getTokenKind(Tokens[5]));
+
+ clang_disposeTokens(ClangTU, Tokens, NumTokens);
+ };
+
+ CheckTokenKinds();
+
+ std::string ASTName = "test.ast";
+ WriteFile(ASTName, "");
+
+ ASSERT_TRUE(SaveAndLoadTU(ASTName));
+
+ CheckTokenKinds();
+}
More information about the cfe-commits
mailing list