r218333 - PR18793: If we try to EnterTokenStream when our current lexer is a caching

[Richard Smith]

Author: rsmith
Date: Tue Sep 23 16:05:52 2014
New Revision: 218333

URL: http://llvm.org/viewvc/llvm-project?rev=218333&view=rev
Log:
PR18793: If we try to EnterTokenStream when our current lexer is a caching
lexer, add the token buffer underneath the caching lexer where possible and
push the tokens directly into the caching lexer otherwise. We previously
put the lexer into a corrupted state where we could not guarantee to provide
the tokens in the right order and would sometimes assert.

Modified:
    cfe/trunk/include/clang/Lex/Preprocessor.h
    cfe/trunk/lib/Lex/PPLexerChange.cpp
    cfe/trunk/test/Parser/cxx-template-argument.cpp

Modified: cfe/trunk/include/clang/Lex/Preprocessor.h
URL: http://llvm.org/viewvc/llvm-project/cfe/trunk/include/clang/Lex/Preprocessor.h?rev=218333&r1=218332&r2=218333&view=diff
==============================================================================
--- cfe/trunk/include/clang/Lex/Preprocessor.h (original)
+++ cfe/trunk/include/clang/Lex/Preprocessor.h Tue Sep 23 16:05:52 2014
@@ -1361,6 +1361,7 @@ public:
 private:
 
   void PushIncludeMacroStack() {
+    assert(CurLexerKind != CLK_CachingLexer && "cannot push a caching lexer");
     IncludeMacroStack.push_back(IncludeStackInfo(
         CurLexerKind, CurSubmodule, std::move(CurLexer), std::move(CurPTHLexer),
         CurPPLexer, std::move(CurTokenLexer), CurDirLookup));

Modified: cfe/trunk/lib/Lex/PPLexerChange.cpp
URL: http://llvm.org/viewvc/llvm-project/cfe/trunk/lib/Lex/PPLexerChange.cpp?rev=218333&r1=218332&r2=218333&view=diff
==============================================================================
--- cfe/trunk/lib/Lex/PPLexerChange.cpp (original)
+++ cfe/trunk/lib/Lex/PPLexerChange.cpp Tue Sep 23 16:05:52 2014
@@ -190,6 +190,25 @@ void Preprocessor::EnterMacro(Token &Tok
 void Preprocessor::EnterTokenStream(const Token *Toks, unsigned NumToks,
                                     bool DisableMacroExpansion,
                                     bool OwnsTokens) {
+  if (CurLexerKind == CLK_CachingLexer) {
+    if (CachedLexPos < CachedTokens.size()) {
+      // We're entering tokens into the middle of our cached token stream. We
+      // can't represent that, so just insert the tokens into the buffer.
+      CachedTokens.insert(CachedTokens.begin() + CachedLexPos,
+                          Toks, Toks + NumToks);
+      if (OwnsTokens)
+        delete [] Toks;
+      return;
+    }
+
+    // New tokens are at the end of the cached token sequnece; insert the
+    // token stream underneath the caching lexer.
+    ExitCachingLexMode();
+    EnterTokenStream(Toks, NumToks, DisableMacroExpansion, OwnsTokens);
+    EnterCachingLexMode();
+    return;
+  }
+
   // Create a macro expander to expand from the specified token stream.
   std::unique_ptr<TokenLexer> TokLexer;
   if (NumCachedTokenLexers == 0) {

Modified: cfe/trunk/test/Parser/cxx-template-argument.cpp
URL: http://llvm.org/viewvc/llvm-project/cfe/trunk/test/Parser/cxx-template-argument.cpp?rev=218333&r1=218332&r2=218333&view=diff
==============================================================================
--- cfe/trunk/test/Parser/cxx-template-argument.cpp (original)
+++ cfe/trunk/test/Parser/cxx-template-argument.cpp Tue Sep 23 16:05:52 2014
@@ -106,3 +106,8 @@ namespace pr16225add {
   { };
 
 }
+
+namespace PR18793 {
+  template<typename T, T> struct S {};
+  template<typename T> int g(S<T, (T())> *);
+}