[clang] [clang-repl] Address error recovery fixing infinite loop while parsing (PR #127569)

[Anutosh Bhat via cfe-commits]

anutosh491 wrote:

This happens because the while loop is running only taking `eof` into account and not `annot_repl_input_end` which is specific to clang-repl i'd assume .... hence it never terminates.

I need to add tests for the same. But that being said here is something concerning I think 

**So finding those infinite loops won't be that hard. Even after the above change, I see**
```
clang-repl> namespace myspace {
<<< inputs >>>:1:1: error: expected expression
<<< inputs >>>:1:1: error: expected expression
<<< inputs >>>:1:1: error: expected expression
<<< inputs >>>:1:1: error: expected expression
<<< inputs >>>:1:1: error: expected expression
<<< inputs >>>:1:1: error: expected expression
<<< inputs >>>:1:1: error: expected expression
<<< inputs >>>:1:1: error: expected expression
<<< inputs >>>:1:1: error: expected expression
<<< inputs >>>:1:1: error: expected expression
<<< inputs >>>:1:1: error: expected expression
<<< inputs >>>:1:1: error: expected expression
<<< inputs >>>:1:1: error: expected expression
<<< inputs >>>:1:1: error: expected expression
<<< inputs >>>:1:1: error: expected expression
<<< inputs >>>:1:1: error: expected expression
<<< inputs >>>:1:1: error: expected expression
<<< inputs >>>:1:1: error: expected expression
<<< inputs >>>:1:1: error: expected expression
fatal error: too many errors emitted, stopping now [-ferror-limit=]
```

This happens because there are quite some places apart from this too where `eof` is handled but `annot_repl_input_end` is not.
```
xeus-cpp-wasm-build) anutosh491 at Anutoshs-MacBook-Air llvm-project % grep -ir "Tok.isNot(tok::eof)"
./clang-tools-extra/clang-tidy/google/ExplicitConstructorCheck.cpp:  } while (Tok.isNot(tok::eof) && Tok.getLocation() < EndLoc);
./clang-tools-extra/clang-tidy/readability/NamespaceCommentCheck.cpp:  bool NeedLineBreak = NextTokenIsOnSameLine && Tok.isNot(tok::eof);
./clang/tools/clang-refactor/TestSupport.cpp:  for (Lex.LexFromRawLexer(Tok); Tok.isNot(tok::eof);
./clang/lib/Rewrite/HTMLRewrite.cpp:  while (Tok.isNot(tok::eof)) {
./clang/lib/Rewrite/HTMLRewrite.cpp:      while (!Tok.isAtStartOfLine() && Tok.isNot(tok::eof)) {
./clang/lib/Rewrite/HTMLRewrite.cpp:  while (Tok.isNot(tok::eof)) {
./clang/lib/Rewrite/TokenRewriter.cpp:  while (RawTok.isNot(tok::eof)) {
./clang/lib/Frontend/FrontendActions.cpp:  } while (Tok.isNot(tok::eof));
./clang/lib/Frontend/FrontendActions.cpp:  while (RawTok.isNot(tok::eof)) {
./clang/lib/Frontend/FrontendActions.cpp:  } while (Tok.isNot(tok::eof));
./clang/lib/Frontend/FrontendActions.cpp:  } while (Tok.isNot(tok::eof));
./clang/lib/Frontend/FrontendActions.cpp:      } while (Tok.isNot(tok::eof));
./clang/lib/Frontend/Rewrite/RewriteMacros.cpp:  } while (RawTok.isNot(tok::eof));
./clang/lib/Frontend/Rewrite/RewriteMacros.cpp:  while (RawTok.isNot(tok::eof) || PPTok.isNot(tok::eof)) {
./clang/lib/Frontend/Rewrite/RewriteMacros.cpp:      while (!RawTok.isAtStartOfLine() && RawTok.isNot(tok::eof))
./clang/lib/Frontend/Rewrite/InclusionRewriter.cpp:  } while (Tok.isNot(tok::eof));
./clang/lib/Frontend/VerifyDiagnosticConsumer.cpp:  while (Tok.isNot(tok::eof)) {
./clang/lib/Lex/PPDirectives.cpp:  if (Callbacks && (Tok.isNot(tok::eof) || !isRecordingPreamble()))
./clang/lib/Lex/PPDirectives.cpp:    assert(TmpTok.isNot(tok::eof) && "Didn't find end of -imacros!");
./clang/lib/Lex/Pragma.cpp:    if (Tok.isNot(tok::r_paren) && Tok.isNot(tok::eof))
./clang/lib/Lex/Pragma.cpp:           Tok.isNot(tok::eof))
./clang/lib/Lex/Pragma.cpp:  while (Tok.isNot(tok::eof)) {
./clang/lib/Lex/PPLexerChange.cpp:  while (Tok.isNot(tok::eod) && Tok.isNot(tok::eof))
./clang/lib/Parse/ParseExprCXX.cpp:    while (Tok.isNot(tok::eof))
./clang/lib/Parse/ParseDeclCXX.cpp:           Tok.isNot(tok::eof)) {
./clang/lib/Parse/ParseDeclCXX.cpp:         Tok.isNot(tok::eof)) {
./clang/lib/Parse/ParseDeclCXX.cpp:           Tok.isNot(tok::eof)) {
./clang/lib/Parse/ParseOpenMP.cpp:      while (Cnt && Tok.isNot(tok::eof)) {
./clang/lib/Parse/ParseObjc.cpp:      while (Tok.getLocation() != OrigLoc && Tok.isNot(tok::eof))
./clang/lib/Parse/ParseDecl.cpp:  while (Tok.isNot(tok::eof))
./clang/lib/Parse/ParseDecl.cpp:         Tok.isNot(tok::eof)) {
./clang/lib/Parse/ParseDecl.cpp:      (Tok.isNot(tok::eof) || Tok.getEofData() != TypeStr.data())) {
./clang/lib/Parse/ParseDecl.cpp:  while (Tok.isNot(tok::eof))
./clang/lib/Parse/ParseHLSL.cpp:  while (Tok.isNot(tok::r_brace) && Tok.isNot(tok::eof)) {
./clang/lib/Parse/ParseCXXInlineMethods.cpp:        if (Tok.isNot(tok::eof) || Tok.getEofData() != Param) {
./clang/lib/Parse/ParseCXXInlineMethods.cpp:      while (Tok.isNot(tok::eof))
./clang/lib/Parse/ParseCXXInlineMethods.cpp:    if (Tok.isNot(tok::eof) || Tok.getEofData() != LM.Method)
./clang/lib/Parse/ParseCXXInlineMethods.cpp:    while (Tok.isNot(tok::eof))
./clang/lib/Parse/ParseCXXInlineMethods.cpp:    while (Tok.isNot(tok::eof))
./clang/lib/Parse/ParseCXXInlineMethods.cpp:      while (Tok.isNot(tok::eof))
./clang/lib/Parse/ParseCXXInlineMethods.cpp:  while (Tok.isNot(tok::eof))
./clang/lib/Parse/ParseCXXInlineMethods.cpp:  if (Tok.isNot(tok::eof)) {
./clang/lib/Parse/ParseCXXInlineMethods.cpp:    while (Tok.isNot(tok::eof))
./clang/lib/Parse/ParseCXXInlineMethods.cpp:  while (Tok.isNot(tok::eof))
./clang/lib/Parse/ParseStmt.cpp:         Tok.isNot(tok::eof) && Tok.isNot(tok::annot_repl_input_end)) {
./clang/lib/Parse/ParsePragma.cpp:    while (Tok.isNot(tok::eof))
./clang/lib/Parse/ParsePragma.cpp:  if (Tok.isNot(tok::eof)) {
./clang/lib/Parse/ParsePragma.cpp:  if (Tok.isNot(tok::eof)) {
./clang/lib/Parse/ParsePragma.cpp:        while (Tok.isNot(tok::eof))
./clang/lib/Parse/ParsePragma.cpp:      if (Tok.isNot(tok::eof)) {
./clang/lib/Parse/ParsePragma.cpp:        while (Tok.isNot(tok::eof))
./clang/lib/Parse/ParsePragma.cpp:    if (Tok.isNot(tok::eof)) {
./clang/lib/Parse/ParsePragma.cpp:      while (Tok.isNot(tok::eof))
./clang/lib/Parse/ParsePragma.cpp:  if (Tok.isNot(tok::eof)) {
./clang/lib/Parse/ParsePragma.cpp:  while (Tok.isNot(tok::eod) && Tok.isNot(tok::eof)) {
./clang/lib/Parse/Parser.cpp:      while (Tok.isNot(tok::eof))
./clang/lib/AST/CommentParser.cpp:  while (Tok.isNot(tok::eof)) {
./clang/lib/AST/CommentBriefParser.cpp:  while (Tok.isNot(tok::eof)) {
```

I am guessing we can always come up with a use case where  we might be dealing with a while loop takes care of eof but not 
`annot_repl_input_end` !!

Does this mean, I should just brute through all of these and add the checks anywhere required ?


https://github.com/llvm/llvm-project/pull/127569