r361007 - [Lex] Allow to consume tokens while preprocessing
Ilya Biryukov via cfe-commits
cfe-commits at lists.llvm.org
Fri May 17 02:32:06 PDT 2019
Author: ibiryukov
Date: Fri May 17 02:32:05 2019
New Revision: 361007
URL: http://llvm.org/viewvc/llvm-project?rev=361007&view=rev
Log:
[Lex] Allow to consume tokens while preprocessing
Summary:
By adding a hook to consume all tokens produced by the preprocessor.
The intention of this change is to make it possible to consume the
expanded tokens without re-runnig the preprocessor with minimal changes
to the preprocessor and minimal performance penalty when preprocessing
without recording the tokens.
The added hook is very low-level and reconstructing the expanded token
stream requires more work in the client code, the actual algorithm to
collect the tokens using this hook can be found in the follow-up change.
Reviewers: rsmith
Reviewed By: rsmith
Subscribers: eraman, nemanjai, kbarton, jsji, riccibruno, cfe-commits
Tags: #clang
Differential Revision: https://reviews.llvm.org/D59885
Modified:
cfe/trunk/include/clang/Lex/Preprocessor.h
cfe/trunk/include/clang/Lex/Token.h
cfe/trunk/include/clang/Lex/TokenLexer.h
cfe/trunk/include/clang/Parse/Parser.h
cfe/trunk/lib/Frontend/PrintPreprocessedOutput.cpp
cfe/trunk/lib/Lex/MacroArgs.cpp
cfe/trunk/lib/Lex/PPCaching.cpp
cfe/trunk/lib/Lex/PPDirectives.cpp
cfe/trunk/lib/Lex/PPLexerChange.cpp
cfe/trunk/lib/Lex/PPMacroExpansion.cpp
cfe/trunk/lib/Lex/Pragma.cpp
cfe/trunk/lib/Lex/Preprocessor.cpp
cfe/trunk/lib/Lex/TokenLexer.cpp
cfe/trunk/lib/Parse/ParseCXXInlineMethods.cpp
cfe/trunk/lib/Parse/ParseDecl.cpp
cfe/trunk/lib/Parse/ParseDeclCXX.cpp
cfe/trunk/lib/Parse/ParseExpr.cpp
cfe/trunk/lib/Parse/ParseExprCXX.cpp
cfe/trunk/lib/Parse/ParseObjc.cpp
cfe/trunk/lib/Parse/ParseOpenMP.cpp
cfe/trunk/lib/Parse/ParsePragma.cpp
cfe/trunk/lib/Parse/ParseStmtAsm.cpp
cfe/trunk/lib/Parse/ParseTemplate.cpp
cfe/trunk/lib/Parse/Parser.cpp
cfe/trunk/lib/Rewrite/HTMLRewrite.cpp
Modified: cfe/trunk/include/clang/Lex/Preprocessor.h
URL: http://llvm.org/viewvc/llvm-project/cfe/trunk/include/clang/Lex/Preprocessor.h?rev=361007&r1=361006&r2=361007&view=diff
==============================================================================
--- cfe/trunk/include/clang/Lex/Preprocessor.h (original)
+++ cfe/trunk/include/clang/Lex/Preprocessor.h Fri May 17 02:32:05 2019
@@ -33,6 +33,7 @@
#include "llvm/ADT/ArrayRef.h"
#include "llvm/ADT/DenseMap.h"
#include "llvm/ADT/FoldingSet.h"
+#include "llvm/ADT/FunctionExtras.h"
#include "llvm/ADT/None.h"
#include "llvm/ADT/Optional.h"
#include "llvm/ADT/PointerUnion.h"
@@ -48,8 +49,8 @@
#include <cassert>
#include <cstddef>
#include <cstdint>
-#include <memory>
#include <map>
+#include <memory>
#include <string>
#include <utility>
#include <vector>
@@ -124,6 +125,7 @@ class Preprocessor {
friend class VAOptDefinitionContext;
friend class VariadicMacroScopeGuard;
+ llvm::unique_function<void(const clang::Token &)> OnToken;
std::shared_ptr<PreprocessorOptions> PPOpts;
DiagnosticsEngine *Diags;
LangOptions &LangOpts;
@@ -998,6 +1000,13 @@ public:
}
/// \}
+ /// Register a function that would be called on each token in the final
+ /// expanded token stream.
+ /// This also reports annotation tokens produced by the parser.
+ void setTokenWatcher(llvm::unique_function<void(const clang::Token &)> F) {
+ OnToken = std::move(F);
+ }
+
bool isMacroDefined(StringRef Id) {
return isMacroDefined(&Identifiers.get(Id));
}
@@ -1282,6 +1291,7 @@ public:
void EnterMacro(Token &Tok, SourceLocation ILEnd, MacroInfo *Macro,
MacroArgs *Args);
+private:
/// Add a "macro" context to the top of the include stack,
/// which will cause the lexer to start returning the specified tokens.
///
@@ -1293,18 +1303,24 @@ public:
/// of tokens has a permanent owner somewhere, so they do not need to be
/// copied. If it is true, it assumes the array of tokens is allocated with
/// \c new[] and the Preprocessor will delete[] it.
-private:
+ ///
+ /// If \p IsReinject the resulting tokens will have Token::IsReinjected flag
+ /// set, see the flag documentation for details.
void EnterTokenStream(const Token *Toks, unsigned NumToks,
- bool DisableMacroExpansion, bool OwnsTokens);
+ bool DisableMacroExpansion, bool OwnsTokens,
+ bool IsReinject);
public:
void EnterTokenStream(std::unique_ptr<Token[]> Toks, unsigned NumToks,
- bool DisableMacroExpansion) {
- EnterTokenStream(Toks.release(), NumToks, DisableMacroExpansion, true);
+ bool DisableMacroExpansion, bool IsReinject) {
+ EnterTokenStream(Toks.release(), NumToks, DisableMacroExpansion, true,
+ IsReinject);
}
- void EnterTokenStream(ArrayRef<Token> Toks, bool DisableMacroExpansion) {
- EnterTokenStream(Toks.data(), Toks.size(), DisableMacroExpansion, false);
+ void EnterTokenStream(ArrayRef<Token> Toks, bool DisableMacroExpansion,
+ bool IsReinject) {
+ EnterTokenStream(Toks.data(), Toks.size(), DisableMacroExpansion, false,
+ IsReinject);
}
/// Pop the current lexer/macro exp off the top of the lexer stack.
@@ -1449,15 +1465,18 @@ public:
///
/// If BackTrack() is called afterwards, the token will remain at the
/// insertion point.
- void EnterToken(const Token &Tok) {
+ /// If \p IsReinject is true, resulting token will have Token::IsReinjected
+ /// flag set. See the flag documentation for details.
+ void EnterToken(const Token &Tok, bool IsReinject) {
if (LexLevel) {
// It's not correct in general to enter caching lex mode while in the
// middle of a nested lexing action.
auto TokCopy = llvm::make_unique<Token[]>(1);
TokCopy[0] = Tok;
- EnterTokenStream(std::move(TokCopy), 1, true);
+ EnterTokenStream(std::move(TokCopy), 1, true, IsReinject);
} else {
EnterCachingLexMode();
+ assert(IsReinject && "new tokens in the middle of cached stream");
CachedTokens.insert(CachedTokens.begin()+CachedLexPos, Tok);
}
}
@@ -2136,7 +2155,7 @@ private:
//===--------------------------------------------------------------------===//
// Caching stuff.
- void CachingLex(Token &Result, bool &IsNewToken);
+ void CachingLex(Token &Result);
bool InCachingLexMode() const {
// If the Lexer pointers are 0 and IncludeMacroStack is empty, it means
Modified: cfe/trunk/include/clang/Lex/Token.h
URL: http://llvm.org/viewvc/llvm-project/cfe/trunk/include/clang/Lex/Token.h?rev=361007&r1=361006&r2=361007&view=diff
==============================================================================
--- cfe/trunk/include/clang/Lex/Token.h (original)
+++ cfe/trunk/include/clang/Lex/Token.h Fri May 17 02:32:05 2019
@@ -70,20 +70,23 @@ class Token {
public:
// Various flags set per token:
enum TokenFlags {
- StartOfLine = 0x01, // At start of line or only after whitespace
- // (considering the line after macro expansion).
- LeadingSpace = 0x02, // Whitespace exists before this token (considering
- // whitespace after macro expansion).
- DisableExpand = 0x04, // This identifier may never be macro expanded.
- NeedsCleaning = 0x08, // Contained an escaped newline or trigraph.
+ StartOfLine = 0x01, // At start of line or only after whitespace
+ // (considering the line after macro expansion).
+ LeadingSpace = 0x02, // Whitespace exists before this token (considering
+ // whitespace after macro expansion).
+ DisableExpand = 0x04, // This identifier may never be macro expanded.
+ NeedsCleaning = 0x08, // Contained an escaped newline or trigraph.
LeadingEmptyMacro = 0x10, // Empty macro exists before this token.
- HasUDSuffix = 0x20, // This string or character literal has a ud-suffix.
- HasUCN = 0x40, // This identifier contains a UCN.
- IgnoredComma = 0x80, // This comma is not a macro argument separator (MS).
+ HasUDSuffix = 0x20, // This string or character literal has a ud-suffix.
+ HasUCN = 0x40, // This identifier contains a UCN.
+ IgnoredComma = 0x80, // This comma is not a macro argument separator (MS).
StringifiedInMacro = 0x100, // This string or character literal is formed by
// macro stringizing or charizing operator.
CommaAfterElided = 0x200, // The comma following this token was elided (MS).
IsEditorPlaceholder = 0x400, // This identifier is a placeholder.
+ IsReinjected = 0x800, // A phase 4 token that was produced before and
+ // re-added, e.g. via EnterTokenStream. Annotation
+ // tokens are *not* reinjected.
};
tok::TokenKind getKind() const { return Kind; }
Modified: cfe/trunk/include/clang/Lex/TokenLexer.h
URL: http://llvm.org/viewvc/llvm-project/cfe/trunk/include/clang/Lex/TokenLexer.h?rev=361007&r1=361006&r2=361007&view=diff
==============================================================================
--- cfe/trunk/include/clang/Lex/TokenLexer.h (original)
+++ cfe/trunk/include/clang/Lex/TokenLexer.h Fri May 17 02:32:05 2019
@@ -96,6 +96,10 @@ class TokenLexer {
/// should not be subject to further macro expansion.
bool DisableMacroExpansion : 1;
+ /// When true, the produced tokens have Token::IsReinjected flag set.
+ /// See the flag documentation for details.
+ bool IsReinject : 1;
+
public:
/// Create a TokenLexer for the specified macro with the specified actual
/// arguments. Note that this ctor takes ownership of the ActualArgs pointer.
@@ -111,9 +115,9 @@ public:
/// specified, this takes ownership of the tokens and delete[]'s them when
/// the token lexer is empty.
TokenLexer(const Token *TokArray, unsigned NumToks, bool DisableExpansion,
- bool ownsTokens, Preprocessor &pp)
+ bool ownsTokens, bool isReinject, Preprocessor &pp)
: PP(pp), OwnsTokens(false) {
- Init(TokArray, NumToks, DisableExpansion, ownsTokens);
+ Init(TokArray, NumToks, DisableExpansion, ownsTokens, isReinject);
}
TokenLexer(const TokenLexer &) = delete;
@@ -132,8 +136,8 @@ public:
///
/// DisableExpansion is true when macro expansion of tokens lexed from this
/// stream should be disabled.
- void Init(const Token *TokArray, unsigned NumToks,
- bool DisableMacroExpansion, bool OwnsTokens);
+ void Init(const Token *TokArray, unsigned NumToks, bool DisableMacroExpansion,
+ bool OwnsTokens, bool IsReinject);
/// If the next token lexed will pop this macro off the
/// expansion stack, return 2. If the next unexpanded token is a '(', return
Modified: cfe/trunk/include/clang/Parse/Parser.h
URL: http://llvm.org/viewvc/llvm-project/cfe/trunk/include/clang/Parse/Parser.h?rev=361007&r1=361006&r2=361007&view=diff
==============================================================================
--- cfe/trunk/include/clang/Parse/Parser.h (original)
+++ cfe/trunk/include/clang/Parse/Parser.h Fri May 17 02:32:05 2019
@@ -254,7 +254,7 @@ class Parser : public CodeCompletionHand
Depth = Depth - AddedLevels + D;
AddedLevels = D;
}
-
+
unsigned getDepth() const { return Depth; }
unsigned getOriginalDepth() const { return Depth - AddedLevels; }
};
@@ -536,9 +536,9 @@ private:
/// token the current token.
void UnconsumeToken(Token &Consumed) {
Token Next = Tok;
- PP.EnterToken(Consumed);
+ PP.EnterToken(Consumed, /*IsReinject*/true);
PP.Lex(Tok);
- PP.EnterToken(Next);
+ PP.EnterToken(Next, /*IsReinject*/true);
}
SourceLocation ConsumeAnnotationToken() {
Modified: cfe/trunk/lib/Frontend/PrintPreprocessedOutput.cpp
URL: http://llvm.org/viewvc/llvm-project/cfe/trunk/lib/Frontend/PrintPreprocessedOutput.cpp?rev=361007&r1=361006&r2=361007&view=diff
==============================================================================
--- cfe/trunk/lib/Frontend/PrintPreprocessedOutput.cpp (original)
+++ cfe/trunk/lib/Frontend/PrintPreprocessedOutput.cpp Fri May 17 02:32:05 2019
@@ -678,7 +678,8 @@ struct UnknownPragmaHandler : public Pra
auto Toks = llvm::make_unique<Token[]>(1);
Toks[0] = PragmaTok;
PP.EnterTokenStream(std::move(Toks), /*NumToks=*/1,
- /*DisableMacroExpansion=*/false);
+ /*DisableMacroExpansion=*/false,
+ /*IsReinject=*/false);
PP.Lex(PragmaTok);
}
Token PrevToken;
Modified: cfe/trunk/lib/Lex/MacroArgs.cpp
URL: http://llvm.org/viewvc/llvm-project/cfe/trunk/lib/Lex/MacroArgs.cpp?rev=361007&r1=361006&r2=361007&view=diff
==============================================================================
--- cfe/trunk/lib/Lex/MacroArgs.cpp (original)
+++ cfe/trunk/lib/Lex/MacroArgs.cpp Fri May 17 02:32:05 2019
@@ -181,7 +181,7 @@ const std::vector<Token> &MacroArgs::get
// list. With this installed, we lex expanded tokens until we hit the EOF
// token at the end of the unexp list.
PP.EnterTokenStream(AT, NumToks, false /*disable expand*/,
- false /*owns tokens*/);
+ false /*owns tokens*/, false /*is reinject*/);
// Lex all of the macro-expanded tokens into Result.
do {
Modified: cfe/trunk/lib/Lex/PPCaching.cpp
URL: http://llvm.org/viewvc/llvm-project/cfe/trunk/lib/Lex/PPCaching.cpp?rev=361007&r1=361006&r2=361007&view=diff
==============================================================================
--- cfe/trunk/lib/Lex/PPCaching.cpp (original)
+++ cfe/trunk/lib/Lex/PPCaching.cpp Fri May 17 02:32:05 2019
@@ -45,7 +45,7 @@ void Preprocessor::Backtrack() {
recomputeCurLexerKind();
}
-void Preprocessor::CachingLex(Token &Result, bool &IsNewToken) {
+void Preprocessor::CachingLex(Token &Result) {
if (!InCachingLexMode())
return;
@@ -55,7 +55,7 @@ void Preprocessor::CachingLex(Token &Res
if (CachedLexPos < CachedTokens.size()) {
Result = CachedTokens[CachedLexPos++];
- IsNewToken = false;
+ Result.setFlag(Token::IsReinjected);
return;
}
Modified: cfe/trunk/lib/Lex/PPDirectives.cpp
URL: http://llvm.org/viewvc/llvm-project/cfe/trunk/lib/Lex/PPDirectives.cpp?rev=361007&r1=361006&r2=361007&view=diff
==============================================================================
--- cfe/trunk/lib/Lex/PPDirectives.cpp (original)
+++ cfe/trunk/lib/Lex/PPDirectives.cpp Fri May 17 02:32:05 2019
@@ -1035,7 +1035,7 @@ void Preprocessor::HandleDirective(Token
// Enter this token stream so that we re-lex the tokens. Make sure to
// enable macro expansion, in case the token after the # is an identifier
// that is expanded.
- EnterTokenStream(std::move(Toks), 2, false);
+ EnterTokenStream(std::move(Toks), 2, false, /*IsReinject*/false);
return;
}
@@ -1518,7 +1518,7 @@ void Preprocessor::EnterAnnotationToken(
Tok[0].setLocation(Range.getBegin());
Tok[0].setAnnotationEndLoc(Range.getEnd());
Tok[0].setAnnotationValue(AnnotationVal);
- EnterTokenStream(std::move(Tok), 1, true);
+ EnterTokenStream(std::move(Tok), 1, true, /*IsReinject*/ false);
}
/// Produce a diagnostic informing the user that a #include or similar
Modified: cfe/trunk/lib/Lex/PPLexerChange.cpp
URL: http://llvm.org/viewvc/llvm-project/cfe/trunk/lib/Lex/PPLexerChange.cpp?rev=361007&r1=361006&r2=361007&view=diff
==============================================================================
--- cfe/trunk/lib/Lex/PPLexerChange.cpp (original)
+++ cfe/trunk/lib/Lex/PPLexerChange.cpp Fri May 17 02:32:05 2019
@@ -154,10 +154,11 @@ void Preprocessor::EnterMacro(Token &Tok
/// must be freed.
///
void Preprocessor::EnterTokenStream(const Token *Toks, unsigned NumToks,
- bool DisableMacroExpansion,
- bool OwnsTokens) {
+ bool DisableMacroExpansion, bool OwnsTokens,
+ bool IsReinject) {
if (CurLexerKind == CLK_CachingLexer) {
if (CachedLexPos < CachedTokens.size()) {
+ assert(IsReinject && "new tokens in the middle of cached stream");
// We're entering tokens into the middle of our cached token stream. We
// can't represent that, so just insert the tokens into the buffer.
CachedTokens.insert(CachedTokens.begin() + CachedLexPos,
@@ -170,7 +171,8 @@ void Preprocessor::EnterTokenStream(cons
// New tokens are at the end of the cached token sequnece; insert the
// token stream underneath the caching lexer.
ExitCachingLexMode();
- EnterTokenStream(Toks, NumToks, DisableMacroExpansion, OwnsTokens);
+ EnterTokenStream(Toks, NumToks, DisableMacroExpansion, OwnsTokens,
+ IsReinject);
EnterCachingLexMode();
return;
}
@@ -179,10 +181,11 @@ void Preprocessor::EnterTokenStream(cons
std::unique_ptr<TokenLexer> TokLexer;
if (NumCachedTokenLexers == 0) {
TokLexer = llvm::make_unique<TokenLexer>(
- Toks, NumToks, DisableMacroExpansion, OwnsTokens, *this);
+ Toks, NumToks, DisableMacroExpansion, OwnsTokens, IsReinject, *this);
} else {
TokLexer = std::move(TokenLexerCache[--NumCachedTokenLexers]);
- TokLexer->Init(Toks, NumToks, DisableMacroExpansion, OwnsTokens);
+ TokLexer->Init(Toks, NumToks, DisableMacroExpansion, OwnsTokens,
+ IsReinject);
}
// Save our current state.
Modified: cfe/trunk/lib/Lex/PPMacroExpansion.cpp
URL: http://llvm.org/viewvc/llvm-project/cfe/trunk/lib/Lex/PPMacroExpansion.cpp?rev=361007&r1=361006&r2=361007&view=diff
==============================================================================
--- cfe/trunk/lib/Lex/PPMacroExpansion.cpp (original)
+++ cfe/trunk/lib/Lex/PPMacroExpansion.cpp Fri May 17 02:32:05 2019
@@ -806,7 +806,7 @@ MacroArgs *Preprocessor::ReadMacroCallAr
// Do not lose the EOF/EOD.
auto Toks = llvm::make_unique<Token[]>(1);
Toks[0] = Tok;
- EnterTokenStream(std::move(Toks), 1, true);
+ EnterTokenStream(std::move(Toks), 1, true, /*IsReinject*/ false);
break;
} else if (Tok.is(tok::r_paren)) {
// If we found the ) token, the macro arg list is done.
Modified: cfe/trunk/lib/Lex/Pragma.cpp
URL: http://llvm.org/viewvc/llvm-project/cfe/trunk/lib/Lex/Pragma.cpp?rev=361007&r1=361006&r2=361007&view=diff
==============================================================================
--- cfe/trunk/lib/Lex/Pragma.cpp (original)
+++ cfe/trunk/lib/Lex/Pragma.cpp Fri May 17 02:32:05 2019
@@ -189,7 +189,8 @@ void Preprocessor::Handle_Pragma(Token &
std::copy(Tokens.begin() + 1, Tokens.end(), Toks.get());
Toks[Tokens.size() - 1] = Tok;
Self.EnterTokenStream(std::move(Toks), Tokens.size(),
- /*DisableMacroExpansion*/ true);
+ /*DisableMacroExpansion*/ true,
+ /*IsReinject*/ true);
// ... and return the _Pragma token unchanged.
Tok = *Tokens.begin();
@@ -366,7 +367,8 @@ void Preprocessor::HandleMicrosoft__prag
std::copy(PragmaToks.begin(), PragmaToks.end(), TokArray);
// Push the tokens onto the stack.
- EnterTokenStream(TokArray, PragmaToks.size(), true, true);
+ EnterTokenStream(TokArray, PragmaToks.size(), true, true,
+ /*IsReinject*/ false);
// With everything set up, lex this as a #pragma directive.
HandlePragmaDirective(PragmaLoc, PIK___pragma);
@@ -1028,7 +1030,7 @@ struct PragmaDebugHandler : public Pragm
Crasher.startToken();
Crasher.setKind(tok::annot_pragma_parser_crash);
Crasher.setAnnotationRange(SourceRange(Tok.getLocation()));
- PP.EnterToken(Crasher);
+ PP.EnterToken(Crasher, /*IsReinject*/false);
} else if (II->isStr("dump")) {
Token Identifier;
PP.LexUnexpandedToken(Identifier);
@@ -1040,7 +1042,7 @@ struct PragmaDebugHandler : public Pragm
SourceRange(Tok.getLocation(), Identifier.getLocation()));
DumpAnnot.setAnnotationValue(DumpII);
PP.DiscardUntilEndOfDirective();
- PP.EnterToken(DumpAnnot);
+ PP.EnterToken(DumpAnnot, /*IsReinject*/false);
} else {
PP.Diag(Identifier, diag::warn_pragma_debug_missing_argument)
<< II->getName();
@@ -1123,7 +1125,8 @@ struct PragmaDebugHandler : public Pragm
Toks[0].setKind(tok::annot_pragma_captured);
Toks[0].setLocation(NameLoc);
- PP.EnterTokenStream(Toks, /*DisableMacroExpansion=*/true);
+ PP.EnterTokenStream(Toks, /*DisableMacroExpansion=*/true,
+ /*IsReinject=*/false);
}
// Disable MSVC warning about runtime stack overflow.
Modified: cfe/trunk/lib/Lex/Preprocessor.cpp
URL: http://llvm.org/viewvc/llvm-project/cfe/trunk/lib/Lex/Preprocessor.cpp?rev=361007&r1=361006&r2=361007&view=diff
==============================================================================
--- cfe/trunk/lib/Lex/Preprocessor.cpp (original)
+++ cfe/trunk/lib/Lex/Preprocessor.cpp Fri May 17 02:32:05 2019
@@ -635,8 +635,7 @@ void Preprocessor::SkipTokensWhileUsingP
CurTokenLexer->Lex(Tok);
break;
case CLK_CachingLexer:
- bool IsNewToken;
- CachingLex(Tok, IsNewToken);
+ CachingLex(Tok);
break;
case CLK_LexAfterModuleImport:
LexAfterModuleImport(Tok);
@@ -881,7 +880,6 @@ void Preprocessor::Lex(Token &Result) {
// We loop here until a lex function returns a token; this avoids recursion.
bool ReturnedToken;
- bool IsNewToken = true;
do {
switch (CurLexerKind) {
case CLK_Lexer:
@@ -891,7 +889,7 @@ void Preprocessor::Lex(Token &Result) {
ReturnedToken = CurTokenLexer->Lex(Result);
break;
case CLK_CachingLexer:
- CachingLex(Result, IsNewToken);
+ CachingLex(Result);
ReturnedToken = true;
break;
case CLK_LexAfterModuleImport:
@@ -911,7 +909,8 @@ void Preprocessor::Lex(Token &Result) {
// Update ImportSeqState to track our position within a C++20 import-seq
// if this token is being produced as a result of phase 4 of translation.
- if (getLangOpts().CPlusPlusModules && LexLevel == 1 && IsNewToken) {
+ if (getLangOpts().CPlusPlusModules && LexLevel == 1 &&
+ !Result.getFlag(Token::IsReinjected)) {
switch (Result.getKind()) {
case tok::l_paren: case tok::l_square: case tok::l_brace:
ImportSeqState.handleOpenBracket();
@@ -952,6 +951,8 @@ void Preprocessor::Lex(Token &Result) {
LastTokenWasAt = Result.is(tok::at);
--LexLevel;
+ if (OnToken && LexLevel == 0 && !Result.getFlag(Token::IsReinjected))
+ OnToken(Result);
}
/// Lex a header-name token (including one formed from header-name-tokens if
@@ -1131,7 +1132,7 @@ bool Preprocessor::LexAfterModuleImport(
auto ToksCopy = llvm::make_unique<Token[]>(Toks.size());
std::copy(Toks.begin(), Toks.end(), ToksCopy.get());
EnterTokenStream(std::move(ToksCopy), Toks.size(),
- /*DisableMacroExpansion*/ true);
+ /*DisableMacroExpansion*/ true, /*IsReinject*/ false);
};
// Check for a header-name.
Modified: cfe/trunk/lib/Lex/TokenLexer.cpp
URL: http://llvm.org/viewvc/llvm-project/cfe/trunk/lib/Lex/TokenLexer.cpp?rev=361007&r1=361006&r2=361007&view=diff
==============================================================================
--- cfe/trunk/lib/Lex/TokenLexer.cpp (original)
+++ cfe/trunk/lib/Lex/TokenLexer.cpp Fri May 17 02:32:05 2019
@@ -53,6 +53,7 @@ void TokenLexer::Init(Token &Tok, Source
Tokens = &*Macro->tokens_begin();
OwnsTokens = false;
DisableMacroExpansion = false;
+ IsReinject = false;
NumTokens = Macro->tokens_end()-Macro->tokens_begin();
MacroExpansionStart = SourceLocation();
@@ -91,7 +92,9 @@ void TokenLexer::Init(Token &Tok, Source
/// Create a TokenLexer for the specified token stream. This does not
/// take ownership of the specified token vector.
void TokenLexer::Init(const Token *TokArray, unsigned NumToks,
- bool disableMacroExpansion, bool ownsTokens) {
+ bool disableMacroExpansion, bool ownsTokens,
+ bool isReinject) {
+ assert(!isReinject || disableMacroExpansion);
// If the client is reusing a TokenLexer, make sure to free any memory
// associated with it.
destroy();
@@ -101,6 +104,7 @@ void TokenLexer::Init(const Token *TokAr
Tokens = TokArray;
OwnsTokens = ownsTokens;
DisableMacroExpansion = disableMacroExpansion;
+ IsReinject = isReinject;
NumTokens = NumToks;
CurTokenIdx = 0;
ExpandLocStart = ExpandLocEnd = SourceLocation();
@@ -647,6 +651,8 @@ bool TokenLexer::Lex(Token &Tok) {
// Get the next token to return.
Tok = Tokens[CurTokenIdx++];
+ if (IsReinject)
+ Tok.setFlag(Token::IsReinjected);
bool TokenIsFromPaste = false;
Modified: cfe/trunk/lib/Parse/ParseCXXInlineMethods.cpp
URL: http://llvm.org/viewvc/llvm-project/cfe/trunk/lib/Parse/ParseCXXInlineMethods.cpp?rev=361007&r1=361006&r2=361007&view=diff
==============================================================================
--- cfe/trunk/lib/Parse/ParseCXXInlineMethods.cpp (original)
+++ cfe/trunk/lib/Parse/ParseCXXInlineMethods.cpp Fri May 17 02:32:05 2019
@@ -323,7 +323,7 @@ void Parser::ParseLexedMethodDeclaration
// Parse the default argument from its saved token stream.
Toks->push_back(Tok); // So that the current token doesn't get lost
- PP.EnterTokenStream(*Toks, true);
+ PP.EnterTokenStream(*Toks, true, /*IsReinject*/ true);
// Consume the previously-pushed token.
ConsumeAnyToken();
@@ -396,7 +396,7 @@ void Parser::ParseLexedMethodDeclaration
// Parse the default argument from its saved token stream.
Toks->push_back(Tok); // So that the current token doesn't get lost
- PP.EnterTokenStream(*Toks, true);
+ PP.EnterTokenStream(*Toks, true, /*IsReinject*/true);
// Consume the previously-pushed token.
ConsumeAnyToken();
@@ -503,7 +503,7 @@ void Parser::ParseLexedMethodDef(LexedMe
// Append the current token at the end of the new token stream so that it
// doesn't get lost.
LM.Toks.push_back(Tok);
- PP.EnterTokenStream(LM.Toks, true);
+ PP.EnterTokenStream(LM.Toks, true, /*IsReinject*/true);
// Consume the previously pushed token.
ConsumeAnyToken(/*ConsumeCodeCompletionTok=*/true);
@@ -617,7 +617,7 @@ void Parser::ParseLexedMemberInitializer
// Append the current token at the end of the new token stream so that it
// doesn't get lost.
MI.Toks.push_back(Tok);
- PP.EnterTokenStream(MI.Toks, true);
+ PP.EnterTokenStream(MI.Toks, true, /*IsReinject*/true);
// Consume the previously pushed token.
ConsumeAnyToken(/*ConsumeCodeCompletionTok=*/true);
@@ -989,7 +989,8 @@ public:
auto Buffer = llvm::make_unique<Token[]>(Toks.size());
std::copy(Toks.begin() + 1, Toks.end(), Buffer.get());
Buffer[Toks.size() - 1] = Self.Tok;
- Self.PP.EnterTokenStream(std::move(Buffer), Toks.size(), true);
+ Self.PP.EnterTokenStream(std::move(Buffer), Toks.size(), true,
+ /*IsReinject*/ true);
Self.Tok = Toks.front();
}
Modified: cfe/trunk/lib/Parse/ParseDecl.cpp
URL: http://llvm.org/viewvc/llvm-project/cfe/trunk/lib/Parse/ParseDecl.cpp?rev=361007&r1=361006&r2=361007&view=diff
==============================================================================
--- cfe/trunk/lib/Parse/ParseDecl.cpp (original)
+++ cfe/trunk/lib/Parse/ParseDecl.cpp Fri May 17 02:32:05 2019
@@ -1470,7 +1470,7 @@ void Parser::ParseLexedAttribute(LatePar
// Append the current token at the end of the new token stream so that it
// doesn't get lost.
LA.Toks.push_back(Tok);
- PP.EnterTokenStream(LA.Toks, true);
+ PP.EnterTokenStream(LA.Toks, true, /*IsReinject=*/true);
// Consume the previously pushed token.
ConsumeAnyToken(/*ConsumeCodeCompletionTok=*/true);
@@ -4424,7 +4424,7 @@ void Parser::ParseEnumSpecifier(SourceLo
if (Tok.isNot(tok::semi)) {
// A semicolon was missing after this declaration. Diagnose and recover.
ExpectAndConsume(tok::semi, diag::err_expected_after, "enum");
- PP.EnterToken(Tok);
+ PP.EnterToken(Tok, /*IsReinject=*/true);
Tok.setKind(tok::semi);
}
} else {
@@ -4702,7 +4702,7 @@ void Parser::ParseEnumBody(SourceLocatio
// Push this token back into the preprocessor and change our current token
// to ';' so that the rest of the code recovers as though there were an
// ';' after the definition.
- PP.EnterToken(Tok);
+ PP.EnterToken(Tok, /*IsReinject=*/true);
Tok.setKind(tok::semi);
}
}
Modified: cfe/trunk/lib/Parse/ParseDeclCXX.cpp
URL: http://llvm.org/viewvc/llvm-project/cfe/trunk/lib/Parse/ParseDeclCXX.cpp?rev=361007&r1=361006&r2=361007&view=diff
==============================================================================
--- cfe/trunk/lib/Parse/ParseDeclCXX.cpp (original)
+++ cfe/trunk/lib/Parse/ParseDeclCXX.cpp Fri May 17 02:32:05 2019
@@ -1040,7 +1040,7 @@ void Parser::AnnotateExistingDecltypeSpe
if (PP.isBacktrackEnabled())
PP.RevertCachedTokens(1);
else
- PP.EnterToken(Tok);
+ PP.EnterToken(Tok, /*IsReinject*/true);
Tok.setKind(tok::annot_decltype);
setExprAnnotation(Tok,
@@ -1736,7 +1736,7 @@ void Parser::ParseClassSpecifier(tok::To
// A semicolon was missing after this declaration. Diagnose and recover.
ExpectAndConsume(tok::semi, diag::err_expected_after,
DeclSpec::getSpecifierName(TagType, PPol));
- PP.EnterToken(Tok);
+ PP.EnterToken(Tok, /*IsReinject*/true);
Tok.setKind(tok::semi);
}
} else
@@ -2013,7 +2013,7 @@ void Parser::ParseClassSpecifier(tok::To
// Push this token back into the preprocessor and change our current token
// to ';' so that the rest of the code recovers as though there were an
// ';' after the definition.
- PP.EnterToken(Tok);
+ PP.EnterToken(Tok, /*IsReinject=*/true);
Tok.setKind(tok::semi);
}
}
@@ -3280,7 +3280,7 @@ void Parser::ParseCXXMemberSpecification
if (SuggestFixIt) {
LBraceDiag << FixItHint::CreateInsertion(BraceLoc, " {");
// Try recovering from missing { after base-clause.
- PP.EnterToken(Tok);
+ PP.EnterToken(Tok, /*IsReinject*/true);
Tok.setKind(tok::l_brace);
} else {
if (TagDecl)
@@ -3376,12 +3376,12 @@ void Parser::DiagnoseUnexpectedNamespace
diag::note_missing_end_of_definition_before) << D;
// Push '};' onto the token stream to recover.
- PP.EnterToken(Tok);
+ PP.EnterToken(Tok, /*IsReinject*/ true);
Tok.startToken();
Tok.setLocation(PP.getLocForEndOfToken(PrevTokLocation));
Tok.setKind(tok::semi);
- PP.EnterToken(Tok);
+ PP.EnterToken(Tok, /*IsReinject*/ true);
Tok.setKind(tok::r_brace);
}
Modified: cfe/trunk/lib/Parse/ParseExpr.cpp
URL: http://llvm.org/viewvc/llvm-project/cfe/trunk/lib/Parse/ParseExpr.cpp?rev=361007&r1=361006&r2=361007&view=diff
==============================================================================
--- cfe/trunk/lib/Parse/ParseExpr.cpp (original)
+++ cfe/trunk/lib/Parse/ParseExpr.cpp Fri May 17 02:32:05 2019
@@ -303,7 +303,7 @@ Parser::ParseRHSOfBinaryExpression(ExprR
// We can't do this before consuming the comma, because
// isNotExpressionStart() looks at the token stream.
if (OpToken.is(tok::comma) && isNotExpressionStart()) {
- PP.EnterToken(Tok);
+ PP.EnterToken(Tok, /*IsReinject*/true);
Tok = OpToken;
return LHS;
}
@@ -313,7 +313,7 @@ Parser::ParseRHSOfBinaryExpression(ExprR
if (isFoldOperator(NextTokPrec) && Tok.is(tok::ellipsis)) {
// FIXME: We can't check this via lookahead before we consume the token
// because that tickles a lexer bug.
- PP.EnterToken(Tok);
+ PP.EnterToken(Tok, /*IsReinject*/true);
Tok = OpToken;
return LHS;
}
@@ -326,7 +326,7 @@ Parser::ParseRHSOfBinaryExpression(ExprR
if (getLangOpts().ObjC && getLangOpts().CPlusPlus &&
Tok.isOneOf(tok::colon, tok::r_square) &&
OpToken.getIdentifierInfo() != nullptr) {
- PP.EnterToken(Tok);
+ PP.EnterToken(Tok, /*IsReinject*/true);
Tok = OpToken;
return LHS;
}
Modified: cfe/trunk/lib/Parse/ParseExprCXX.cpp
URL: http://llvm.org/viewvc/llvm-project/cfe/trunk/lib/Parse/ParseExprCXX.cpp?rev=361007&r1=361006&r2=361007&view=diff
==============================================================================
--- cfe/trunk/lib/Parse/ParseExprCXX.cpp (original)
+++ cfe/trunk/lib/Parse/ParseExprCXX.cpp Fri May 17 02:32:05 2019
@@ -69,9 +69,9 @@ static void FixDigraph(Parser &P, Prepro
DigraphToken.setLength(1);
// Push new tokens back to token stream.
- PP.EnterToken(ColonToken);
+ PP.EnterToken(ColonToken, /*IsReinject*/ true);
if (!AtDigraph)
- PP.EnterToken(DigraphToken);
+ PP.EnterToken(DigraphToken, /*IsReinject*/ true);
}
// Check for '<::' which should be '< ::' instead of '[:' when following
@@ -434,7 +434,7 @@ bool Parser::ParseOptionalCXXScopeSpecif
Token ColonColon;
PP.Lex(ColonColon);
ColonColon.setKind(tok::colon);
- PP.EnterToken(ColonColon);
+ PP.EnterToken(ColonColon, /*IsReinject*/ true);
break;
}
}
@@ -460,8 +460,8 @@ bool Parser::ParseOptionalCXXScopeSpecif
// mistyped '::' instead of ':'.
if (CorrectionFlagPtr && IsCorrectedToColon) {
ColonColon.setKind(tok::colon);
- PP.EnterToken(Tok);
- PP.EnterToken(ColonColon);
+ PP.EnterToken(Tok, /*IsReinject*/ true);
+ PP.EnterToken(ColonColon, /*IsReinject*/ true);
Tok = Identifier;
break;
}
@@ -3300,7 +3300,8 @@ Parser::ParseCXXAmbiguousParenExpression
Toks.push_back(Tok);
// Re-enter the stored parenthesized tokens into the token stream, so we may
// parse them now.
- PP.EnterTokenStream(Toks, true /*DisableMacroExpansion*/);
+ PP.EnterTokenStream(Toks, /*DisableMacroExpansion*/ true,
+ /*IsReinject*/ true);
// Drop the current token and bring the first cached one. It's the same token
// as when we entered this function.
ConsumeAnyToken();
Modified: cfe/trunk/lib/Parse/ParseObjc.cpp
URL: http://llvm.org/viewvc/llvm-project/cfe/trunk/lib/Parse/ParseObjc.cpp?rev=361007&r1=361006&r2=361007&view=diff
==============================================================================
--- cfe/trunk/lib/Parse/ParseObjc.cpp (original)
+++ cfe/trunk/lib/Parse/ParseObjc.cpp Fri May 17 02:32:05 2019
@@ -1943,7 +1943,7 @@ void Parser::ParseObjCClassInstanceVaria
Tok.setLocation(Tok.getLocation().getLocWithOffset(-1));
Tok.setKind(tok::at);
Tok.setLength(1);
- PP.EnterToken(Tok);
+ PP.EnterToken(Tok, /*IsReinject*/true);
HelperActionsForIvarDeclarations(interfaceDecl, atLoc,
T, AllIvarDecls, true);
return;
@@ -3654,7 +3654,7 @@ void Parser::ParseLexedObjCMethodDefs(Le
// Append the current token at the end of the new token stream so that it
// doesn't get lost.
LM.Toks.push_back(Tok);
- PP.EnterTokenStream(LM.Toks, true);
+ PP.EnterTokenStream(LM.Toks, true, /*IsReinject*/true);
// Consume the previously pushed token.
ConsumeAnyToken(/*ConsumeCodeCompletionTok=*/true);
Modified: cfe/trunk/lib/Parse/ParseOpenMP.cpp
URL: http://llvm.org/viewvc/llvm-project/cfe/trunk/lib/Parse/ParseOpenMP.cpp?rev=361007&r1=361006&r2=361007&view=diff
==============================================================================
--- cfe/trunk/lib/Parse/ParseOpenMP.cpp (original)
+++ cfe/trunk/lib/Parse/ParseOpenMP.cpp Fri May 17 02:32:05 2019
@@ -747,8 +747,9 @@ static bool parseDeclareSimdClauses(
Parser::DeclGroupPtrTy
Parser::ParseOMPDeclareSimdClauses(Parser::DeclGroupPtrTy Ptr,
CachedTokens &Toks, SourceLocation Loc) {
- PP.EnterToken(Tok);
- PP.EnterTokenStream(Toks, /*DisableMacroExpansion=*/true);
+ PP.EnterToken(Tok, /*IsReinject*/ true);
+ PP.EnterTokenStream(Toks, /*DisableMacroExpansion=*/true,
+ /*IsReinject*/ true);
// Consume the previously pushed token.
ConsumeAnyToken(/*ConsumeCodeCompletionTok=*/true);
@@ -1319,7 +1320,7 @@ Parser::ParseOpenMPDeclarativeOrExecutab
FlushHasClause = true;
// Push copy of the current token back to stream to properly parse
// pseudo-clause OMPFlushClause.
- PP.EnterToken(Tok);
+ PP.EnterToken(Tok, /*IsReinject*/ true);
}
LLVM_FALLTHROUGH;
case OMPD_taskyield:
Modified: cfe/trunk/lib/Parse/ParsePragma.cpp
URL: http://llvm.org/viewvc/llvm-project/cfe/trunk/lib/Parse/ParsePragma.cpp?rev=361007&r1=361006&r2=361007&view=diff
==============================================================================
--- cfe/trunk/lib/Parse/ParsePragma.cpp (original)
+++ cfe/trunk/lib/Parse/ParsePragma.cpp Fri May 17 02:32:05 2019
@@ -117,7 +117,8 @@ struct PragmaSTDC_FENV_ACCESSHandler : p
Toks[0].setAnnotationEndLoc(Tok.getLocation());
Toks[0].setAnnotationValue(reinterpret_cast<void*>(
static_cast<uintptr_t>(OOS)));
- PP.EnterTokenStream(Toks, /*DisableMacroExpansion=*/true);
+ PP.EnterTokenStream(Toks, /*DisableMacroExpansion=*/true,
+ /*IsReinject=*/false);
}
};
@@ -739,7 +740,8 @@ void Parser::HandlePragmaMSPragma() {
// Grab the tokens out of the annotation and enter them into the stream.
auto TheTokens =
(std::pair<std::unique_ptr<Token[]>, size_t> *)Tok.getAnnotationValue();
- PP.EnterTokenStream(std::move(TheTokens->first), TheTokens->second, true);
+ PP.EnterTokenStream(std::move(TheTokens->first), TheTokens->second, true,
+ /*IsReinject=*/true);
SourceLocation PragmaLocation = ConsumeAnnotationToken();
assert(Tok.isAnyIdentifier());
StringRef PragmaName = Tok.getIdentifierInfo()->getName();
@@ -1111,7 +1113,8 @@ bool Parser::HandlePragmaLoopHint(LoopHi
Hint.StateLoc = IdentifierLoc::create(Actions.Context, StateLoc, StateInfo);
} else {
// Enter constant expression including eof terminator into token stream.
- PP.EnterTokenStream(Toks, /*DisableMacroExpansion=*/false);
+ PP.EnterTokenStream(Toks, /*DisableMacroExpansion=*/false,
+ /*IsReinject=*/false);
ConsumeAnnotationToken();
ExprResult R = ParseConstantExpression();
@@ -1414,7 +1417,8 @@ void Parser::HandlePragmaAttribute() {
return;
}
- PP.EnterTokenStream(Info->Tokens, /*DisableMacroExpansion=*/false);
+ PP.EnterTokenStream(Info->Tokens, /*DisableMacroExpansion=*/false,
+ /*IsReinject=*/false);
ConsumeAnnotationToken();
ParsedAttributes &Attrs = Info->Attributes;
@@ -1623,8 +1627,9 @@ void PragmaGCCVisibilityHandler::HandleP
Toks[0].setLocation(VisLoc);
Toks[0].setAnnotationEndLoc(EndLoc);
Toks[0].setAnnotationValue(
- const_cast<void*>(static_cast<const void*>(VisType)));
- PP.EnterTokenStream(std::move(Toks), 1, /*DisableMacroExpansion=*/true);
+ const_cast<void *>(static_cast<const void *>(VisType)));
+ PP.EnterTokenStream(std::move(Toks), 1, /*DisableMacroExpansion=*/true,
+ /*IsReinject=*/false);
}
// #pragma pack(...) comes in the following delicious flavors:
@@ -1737,7 +1742,8 @@ void PragmaPackHandler::HandlePragma(Pre
Toks[0].setLocation(PackLoc);
Toks[0].setAnnotationEndLoc(RParenLoc);
Toks[0].setAnnotationValue(static_cast<void*>(Info));
- PP.EnterTokenStream(Toks, /*DisableMacroExpansion=*/true);
+ PP.EnterTokenStream(Toks, /*DisableMacroExpansion=*/true,
+ /*IsReinject=*/false);
}
// #pragma ms_struct on
@@ -1780,7 +1786,8 @@ void PragmaMSStructHandler::HandlePragma
Toks[0].setAnnotationEndLoc(EndLoc);
Toks[0].setAnnotationValue(reinterpret_cast<void*>(
static_cast<uintptr_t>(Kind)));
- PP.EnterTokenStream(Toks, /*DisableMacroExpansion=*/true);
+ PP.EnterTokenStream(Toks, /*DisableMacroExpansion=*/true,
+ /*IsReinject=*/false);
}
// #pragma clang section bss="abc" data="" rodata="def" text=""
@@ -1893,7 +1900,8 @@ static void ParseAlignPragma(Preprocesso
Toks[0].setAnnotationEndLoc(EndLoc);
Toks[0].setAnnotationValue(reinterpret_cast<void*>(
static_cast<uintptr_t>(Kind)));
- PP.EnterTokenStream(Toks, /*DisableMacroExpansion=*/true);
+ PP.EnterTokenStream(Toks, /*DisableMacroExpansion=*/true,
+ /*IsReinject=*/false);
}
void PragmaAlignHandler::HandlePragma(Preprocessor &PP,
@@ -1985,7 +1993,8 @@ void PragmaUnusedHandler::HandlePragma(P
pragmaUnusedTok.setLocation(UnusedLoc);
idTok = Identifiers[i];
}
- PP.EnterTokenStream(Toks, /*DisableMacroExpansion=*/true);
+ PP.EnterTokenStream(Toks, /*DisableMacroExpansion=*/true,
+ /*IsReinject=*/false);
}
// #pragma weak identifier
@@ -2034,7 +2043,8 @@ void PragmaWeakHandler::HandlePragma(Pre
pragmaUnusedTok.setAnnotationEndLoc(AliasName.getLocation());
Toks[1] = WeakName;
Toks[2] = AliasName;
- PP.EnterTokenStream(Toks, /*DisableMacroExpansion=*/true);
+ PP.EnterTokenStream(Toks, /*DisableMacroExpansion=*/true,
+ /*IsReinject=*/false);
} else {
MutableArrayRef<Token> Toks(
PP.getPreprocessorAllocator().Allocate<Token>(2), 2);
@@ -2044,7 +2054,8 @@ void PragmaWeakHandler::HandlePragma(Pre
pragmaUnusedTok.setLocation(WeakLoc);
pragmaUnusedTok.setAnnotationEndLoc(WeakLoc);
Toks[1] = WeakName;
- PP.EnterTokenStream(Toks, /*DisableMacroExpansion=*/true);
+ PP.EnterTokenStream(Toks, /*DisableMacroExpansion=*/true,
+ /*IsReinject=*/false);
}
}
@@ -2089,7 +2100,8 @@ void PragmaRedefineExtnameHandler::Handl
pragmaRedefTok.setAnnotationEndLoc(AliasName.getLocation());
Toks[1] = RedefName;
Toks[2] = AliasName;
- PP.EnterTokenStream(Toks, /*DisableMacroExpansion=*/true);
+ PP.EnterTokenStream(Toks, /*DisableMacroExpansion=*/true,
+ /*IsReinject=*/false);
}
@@ -2109,7 +2121,8 @@ PragmaFPContractHandler::HandlePragma(Pr
Toks[0].setAnnotationEndLoc(Tok.getLocation());
Toks[0].setAnnotationValue(reinterpret_cast<void*>(
static_cast<uintptr_t>(OOS)));
- PP.EnterTokenStream(Toks, /*DisableMacroExpansion=*/true);
+ PP.EnterTokenStream(Toks, /*DisableMacroExpansion=*/true,
+ /*IsReinject=*/false);
}
void
@@ -2171,7 +2184,8 @@ PragmaOpenCLExtensionHandler::HandlePrag
Toks[0].setLocation(NameLoc);
Toks[0].setAnnotationValue(static_cast<void*>(Info));
Toks[0].setAnnotationEndLoc(StateLoc);
- PP.EnterTokenStream(Toks, /*DisableMacroExpansion=*/true);
+ PP.EnterTokenStream(Toks, /*DisableMacroExpansion=*/true,
+ /*IsReinject=*/false);
if (PP.getPPCallbacks())
PP.getPPCallbacks()->PragmaOpenCLExtension(NameLoc, Ext,
@@ -2230,7 +2244,7 @@ PragmaOpenMPHandler::HandlePragma(Prepro
auto Toks = llvm::make_unique<Token[]>(Pragma.size());
std::copy(Pragma.begin(), Pragma.end(), Toks.get());
PP.EnterTokenStream(std::move(Toks), Pragma.size(),
- /*DisableMacroExpansion=*/false);
+ /*DisableMacroExpansion=*/false, /*IsReinject=*/false);
}
/// Handle '#pragma pointers_to_members'
@@ -2328,7 +2342,7 @@ void PragmaMSPointersToMembers::HandlePr
AnnotTok.setAnnotationEndLoc(EndLoc);
AnnotTok.setAnnotationValue(
reinterpret_cast<void *>(static_cast<uintptr_t>(RepresentationMethod)));
- PP.EnterToken(AnnotTok);
+ PP.EnterToken(AnnotTok, /*IsReinject=*/true);
}
/// Handle '#pragma vtordisp'
@@ -2421,7 +2435,7 @@ void PragmaMSVtorDisp::HandlePragma(Prep
AnnotTok.setAnnotationEndLoc(EndLoc);
AnnotTok.setAnnotationValue(reinterpret_cast<void *>(
static_cast<uintptr_t>((Action << 16) | (Value & 0xFFFF))));
- PP.EnterToken(AnnotTok);
+ PP.EnterToken(AnnotTok, /*IsReinject=*/false);
}
/// Handle all MS pragmas. Simply forwards the tokens after inserting
@@ -2452,7 +2466,7 @@ void PragmaMSPragma::HandlePragma(Prepro
std::pair<std::unique_ptr<Token[]>, size_t>(std::move(TokenArray),
TokenVector.size());
AnnotTok.setAnnotationValue(Value);
- PP.EnterToken(AnnotTok);
+ PP.EnterToken(AnnotTok, /*IsReinject*/ false);
}
/// Handle the Microsoft \#pragma detect_mismatch extension.
@@ -2736,7 +2750,7 @@ void PragmaFPHandler::HandlePragma(Prepr
std::copy(TokenList.begin(), TokenList.end(), TokenArray.get());
PP.EnterTokenStream(std::move(TokenArray), TokenList.size(),
- /*DisableMacroExpansion=*/false);
+ /*DisableMacroExpansion=*/false, /*IsReinject=*/false);
}
void Parser::HandlePragmaFP() {
@@ -2919,7 +2933,7 @@ void PragmaLoopHintHandler::HandlePragma
std::copy(TokenList.begin(), TokenList.end(), TokenArray.get());
PP.EnterTokenStream(std::move(TokenArray), TokenList.size(),
- /*DisableMacroExpansion=*/false);
+ /*DisableMacroExpansion=*/false, /*IsReinject=*/false);
}
/// Handle the loop unroll optimization pragmas.
@@ -2994,7 +3008,7 @@ void PragmaUnrollHintHandler::HandlePrag
TokenArray[0].setAnnotationEndLoc(PragmaName.getLocation());
TokenArray[0].setAnnotationValue(static_cast<void *>(Info));
PP.EnterTokenStream(std::move(TokenArray), 1,
- /*DisableMacroExpansion=*/false);
+ /*DisableMacroExpansion=*/false, /*IsReinject=*/false);
}
/// Handle the Microsoft \#pragma intrinsic extension.
@@ -3266,5 +3280,5 @@ void PragmaAttributeHandler::HandlePragm
TokenArray[0].setAnnotationEndLoc(FirstToken.getLocation());
TokenArray[0].setAnnotationValue(static_cast<void *>(Info));
PP.EnterTokenStream(std::move(TokenArray), 1,
- /*DisableMacroExpansion=*/false);
+ /*DisableMacroExpansion=*/false, /*IsReinject=*/false);
}
Modified: cfe/trunk/lib/Parse/ParseStmtAsm.cpp
URL: http://llvm.org/viewvc/llvm-project/cfe/trunk/lib/Parse/ParseStmtAsm.cpp?rev=361007&r1=361006&r2=361007&view=diff
==============================================================================
--- cfe/trunk/lib/Parse/ParseStmtAsm.cpp (original)
+++ cfe/trunk/lib/Parse/ParseStmtAsm.cpp Fri May 17 02:32:05 2019
@@ -213,7 +213,8 @@ ExprResult Parser::ParseMSAsmIdentifier(
// Also copy the current token over.
LineToks.push_back(Tok);
- PP.EnterTokenStream(LineToks, /*DisableMacroExpansions*/ true);
+ PP.EnterTokenStream(LineToks, /*DisableMacroExpansions*/ true,
+ /*IsReinject*/ true);
// Clear the current token and advance to the first token in LineToks.
ConsumeAnyToken();
Modified: cfe/trunk/lib/Parse/ParseTemplate.cpp
URL: http://llvm.org/viewvc/llvm-project/cfe/trunk/lib/Parse/ParseTemplate.cpp?rev=361007&r1=361006&r2=361007&view=diff
==============================================================================
--- cfe/trunk/lib/Parse/ParseTemplate.cpp (original)
+++ cfe/trunk/lib/Parse/ParseTemplate.cpp Fri May 17 02:32:05 2019
@@ -919,7 +919,7 @@ bool Parser::ParseGreaterThanInTemplateL
PrevTokLocation = RAngleLoc;
} else {
PrevTokLocation = TokBeforeGreaterLoc;
- PP.EnterToken(Tok);
+ PP.EnterToken(Tok, /*IsReinject=*/true);
Tok = Greater;
}
@@ -1402,7 +1402,7 @@ void Parser::ParseLateTemplatedFuncDef(L
// Append the current token at the end of the new token stream so that it
// doesn't get lost.
LPT.Toks.push_back(Tok);
- PP.EnterTokenStream(LPT.Toks, true);
+ PP.EnterTokenStream(LPT.Toks, true, /*IsReinject*/true);
// Consume the previously pushed token.
ConsumeAnyToken(/*ConsumeCodeCompletionTok=*/true);
Modified: cfe/trunk/lib/Parse/Parser.cpp
URL: http://llvm.org/viewvc/llvm-project/cfe/trunk/lib/Parse/Parser.cpp?rev=361007&r1=361006&r2=361007&view=diff
==============================================================================
--- cfe/trunk/lib/Parse/Parser.cpp (original)
+++ cfe/trunk/lib/Parse/Parser.cpp Fri May 17 02:32:05 2019
@@ -1545,7 +1545,7 @@ void Parser::AnnotateScopeToken(CXXScope
if (PP.isBacktrackEnabled())
PP.RevertCachedTokens(1);
else
- PP.EnterToken(Tok);
+ PP.EnterToken(Tok, /*IsReinject=*/true);
Tok.setKind(tok::annot_cxxscope);
Tok.setAnnotationValue(Actions.SaveNestedNameSpecifierAnnotation(SS));
Tok.setAnnotationRange(SS.getRange());
@@ -1764,7 +1764,7 @@ bool Parser::TryAnnotateTypeOrScopeToken
Token TypedefToken;
PP.Lex(TypedefToken);
bool Result = TryAnnotateTypeOrScopeToken();
- PP.EnterToken(Tok);
+ PP.EnterToken(Tok, /*IsReinject=*/true);
Tok = TypedefToken;
if (!Result)
Diag(Tok.getLocation(), diag::warn_expected_qualified_after_typename);
Modified: cfe/trunk/lib/Rewrite/HTMLRewrite.cpp
URL: http://llvm.org/viewvc/llvm-project/cfe/trunk/lib/Rewrite/HTMLRewrite.cpp?rev=361007&r1=361006&r2=361007&view=diff
==============================================================================
--- cfe/trunk/lib/Rewrite/HTMLRewrite.cpp (original)
+++ cfe/trunk/lib/Rewrite/HTMLRewrite.cpp Fri May 17 02:32:05 2019
@@ -572,7 +572,7 @@ void html::HighlightMacros(Rewriter &R,
// Enter the tokens we just lexed. This will cause them to be macro expanded
// but won't enter sub-files (because we removed #'s).
- TmpPP.EnterTokenStream(TokenStream, false);
+ TmpPP.EnterTokenStream(TokenStream, false, /*IsReinject=*/false);
TokenConcatenation ConcatInfo(TmpPP);
More information about the cfe-commits
mailing list