r224505 - Parse: Don't parse after the eof has been consumed

[David Majnemer]

Author: majnemer
Date: Thu Dec 18 03:57:31 2014
New Revision: 224505

URL: http://llvm.org/viewvc/llvm-project?rev=224505&view=rev
Log:
Parse: Don't parse after the eof has been consumed

ParseCXXNonStaticMemberInitializer stashes away all the tokens for the
initializer and an additional EOF token to denote where the initializer
ends.  However, it is possible for ParseLexedMemberInitializer to get
its hands on the "real" EOF token; since the two tokens are
indistinguishable, we end up consuming the EOF and descend into madness.

Instead, make it possible to tell which EOF token we are looking at.

This fixes PR21872.

Added:
    cfe/trunk/test/Parser/PR21872.cpp
Modified:
    cfe/trunk/include/clang/Lex/Token.h
    cfe/trunk/lib/Parse/ParseCXXInlineMethods.cpp

Modified: cfe/trunk/include/clang/Lex/Token.h
URL: http://llvm.org/viewvc/llvm-project/cfe/trunk/include/clang/Lex/Token.h?rev=224505&r1=224504&r2=224505&view=diff
==============================================================================
--- cfe/trunk/include/clang/Lex/Token.h (original)
+++ cfe/trunk/include/clang/Lex/Token.h Thu Dec 18 03:57:31 2014
@@ -23,6 +23,7 @@
 
 namespace clang {
 
+class Decl;
 class IdentifierInfo;
 
 /// Token - This structure provides full information about a lexed token.
@@ -58,6 +59,8 @@ class Token {
   ///    may be dirty (have trigraphs / escaped newlines).
   ///  Annotations (resolved type names, C++ scopes, etc): isAnnotation().
   ///    This is a pointer to sema-specific data for the annotation token.
+  ///  Eof:
+  //     This is a pointer to a Decl.
   ///  Other:
   ///    This is null.
   void *PtrData;
@@ -164,12 +167,23 @@ public:
     assert(!isAnnotation() &&
            "getIdentifierInfo() on an annotation token!");
     if (isLiteral()) return nullptr;
+    if (is(tok::eof)) return nullptr;
     return (IdentifierInfo*) PtrData;
   }
   void setIdentifierInfo(IdentifierInfo *II) {
     PtrData = (void*) II;
   }
 
+  const Decl *getDecl() const {
+    assert(is(tok::eof));
+    return reinterpret_cast<const Decl *>(PtrData);
+  }
+  void setDecl(const Decl *D) {
+    assert(is(tok::eof));
+    assert(!PtrData);
+    PtrData = const_cast<Decl *>(D);
+  }
+
   /// getRawIdentifier - For a raw identifier token (i.e., an identifier
   /// lexed in raw mode), returns a reference to the text substring in the
   /// buffer if known.

Modified: cfe/trunk/lib/Parse/ParseCXXInlineMethods.cpp
URL: http://llvm.org/viewvc/llvm-project/cfe/trunk/lib/Parse/ParseCXXInlineMethods.cpp?rev=224505&r1=224504&r2=224505&view=diff
==============================================================================
--- cfe/trunk/lib/Parse/ParseCXXInlineMethods.cpp (original)
+++ cfe/trunk/lib/Parse/ParseCXXInlineMethods.cpp Thu Dec 18 03:57:31 2014
@@ -218,6 +218,7 @@ void Parser::ParseCXXNonStaticMemberInit
   Eof.startToken();
   Eof.setKind(tok::eof);
   Eof.setLocation(Tok.getLocation());
+  Eof.setDecl(VarD);
   Toks.push_back(Eof);
 }
 
@@ -622,7 +623,9 @@ void Parser::ParseLexedMemberInitializer
     while (Tok.isNot(tok::eof))
       ConsumeAnyToken();
   }
-  ConsumeAnyToken();
+  // Make sure this is *our* artificial EOF token.
+  if (Tok.getDecl() == MI.Field)
+    ConsumeAnyToken();
 }
 
 /// ConsumeAndStoreUntil - Consume and store the token at the passed token

Added: cfe/trunk/test/Parser/PR21872.cpp
URL: http://llvm.org/viewvc/llvm-project/cfe/trunk/test/Parser/PR21872.cpp?rev=224505&view=auto
==============================================================================
--- cfe/trunk/test/Parser/PR21872.cpp (added)
+++ cfe/trunk/test/Parser/PR21872.cpp Thu Dec 18 03:57:31 2014
@@ -0,0 +1,4 @@
+// RUN: not %clang_cc1 -fsyntax-only %s
+template <typename T> struct S {
+    int k = ((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((
+int f;