[clang] [clang-repl] Address error recovery fixing infinite loop while parsing (PR #127569)
Anutosh Bhat via cfe-commits
cfe-commits at lists.llvm.org
Sun Oct 18 21:54:37 PDT 2026
https://github.com/anutosh491 updated https://github.com/llvm/llvm-project/pull/127569
>From 8a6bc4b5dbd5ce9443616d29e6bde988c2a7bcae Mon Sep 17 00:00:00 2001
From: anutosh491 <andersonbhat491 at gmail.com>
Date: Mon, 19 Oct 2026 10:24:22 +0530
Subject: [PATCH] [clang-repl] Use EOF for incremental input
---
clang/include/clang/Basic/TokenKinds.def | 3 --
clang/include/clang/Parse/Parser.h | 3 +-
.../lib/Frontend/PrintPreprocessedOutput.cpp | 7 +--
clang/lib/Interpreter/IncrementalParser.cpp | 6 +--
clang/lib/Lex/PPLexerChange.cpp | 8 +--
clang/lib/Lex/Preprocessor.cpp | 15 ++++--
clang/lib/Parse/ParseCXXInlineMethods.cpp | 1 -
clang/lib/Parse/ParseDecl.cpp | 6 +--
clang/lib/Parse/ParseStmt.cpp | 6 ++-
clang/lib/Parse/Parser.cpp | 2 -
clang/lib/Serialization/ASTReader.cpp | 1 -
clang/lib/Serialization/ASTWriter.cpp | 1 -
.../Interpreter/repl-input-end-recovery.cpp | 50 +++++++++++++++++++
13 files changed, 75 insertions(+), 34 deletions(-)
create mode 100644 clang/test/Interpreter/repl-input-end-recovery.cpp
diff --git a/clang/include/clang/Basic/TokenKinds.def b/clang/include/clang/Basic/TokenKinds.def
index 528825ad68b8dc..802947e31cbc18 100644
--- a/clang/include/clang/Basic/TokenKinds.def
+++ b/clang/include/clang/Basic/TokenKinds.def
@@ -917,9 +917,6 @@ ANNOTATION(module_name)
// into the name of a header unit.
ANNOTATION(header_unit)
-// Annotation for end of input in clang-repl.
-ANNOTATION(repl_input_end)
-
// Annotation for #embed
ANNOTATION(embed)
diff --git a/clang/include/clang/Parse/Parser.h b/clang/include/clang/Parse/Parser.h
index 9a1abd4f3872de..7f91c7d4670ff1 100644
--- a/clang/include/clang/Parse/Parser.h
+++ b/clang/include/clang/Parse/Parser.h
@@ -829,8 +829,7 @@ class Parser : public CodeCompletionHandler {
bool isEofOrEom() {
tok::TokenKind Kind = Tok.getKind();
return Kind == tok::eof || Kind == tok::annot_module_begin ||
- Kind == tok::annot_module_end || Kind == tok::annot_module_include ||
- Kind == tok::annot_repl_input_end;
+ Kind == tok::annot_module_end || Kind == tok::annot_module_include;
}
static void setTypeAnnotation(Token &Tok, TypeResult T) {
diff --git a/clang/lib/Frontend/PrintPreprocessedOutput.cpp b/clang/lib/Frontend/PrintPreprocessedOutput.cpp
index 23da82769e2aeb..ab5ac8a4a69c6d 100644
--- a/clang/lib/Frontend/PrintPreprocessedOutput.cpp
+++ b/clang/lib/Frontend/PrintPreprocessedOutput.cpp
@@ -789,8 +789,7 @@ void PrintPPOutputPPCallbacks::HandleWhitespaceBeforeTok(const Token &Tok,
if (Tok.is(tok::eof) ||
(Tok.isAnnotation() && !Tok.is(tok::annot_header_unit) &&
!Tok.is(tok::annot_module_begin) && !Tok.is(tok::annot_module_end) &&
- !Tok.is(tok::annot_repl_input_end) && !Tok.is(tok::annot_embed) &&
- !Tok.is(tok::annot_module_name)))
+ !Tok.is(tok::annot_embed) && !Tok.is(tok::annot_module_name)))
return;
// EmittedDirectiveOnThisLine takes priority over RequireSameLine.
@@ -946,8 +945,6 @@ static void PrintPreprocessedTokens(Preprocessor &PP, Token &Tok,
// -traditional-cpp the lexer keeps /all/ whitespace, including comments.
PP.Lex(Tok);
continue;
- } else if (Tok.is(tok::annot_repl_input_end)) {
- // Fall through to exit the loop.
} else if (Tok.is(tok::eod)) {
// Don't print end of directive tokens, since they are typically newlines
// that mess up our line tracking. These come from unknown pre-processor
@@ -1084,7 +1081,7 @@ static void PrintPreprocessedTokens(Preprocessor &PP, Token &Tok,
Callbacks->setEmittedTokensOnThisLine();
IsStartOfLine = false;
- if (Tok.is(tok::eof) || Tok.is(tok::annot_repl_input_end))
+ if (Tok.is(tok::eof))
break;
PP.Lex(Tok);
diff --git a/clang/lib/Interpreter/IncrementalParser.cpp b/clang/lib/Interpreter/IncrementalParser.cpp
index 3cfabb5e5daa18..4d1ba3473d48ef 100644
--- a/clang/lib/Interpreter/IncrementalParser.cpp
+++ b/clang/lib/Interpreter/IncrementalParser.cpp
@@ -65,7 +65,7 @@ IncrementalParser::ParseOrWrapTopLevelDecl() {
C.addTranslationUnitDecl();
// Skip previous eof due to last incremental input.
- if (P->getCurToken().is(tok::annot_repl_input_end)) {
+ if (P->getCurToken().is(tok::eof)) {
P->ConsumeAnyToken();
// FIXME: Clang does not call ExitScope on finalizing the regular TU, we
// might want to do that around HandleEndOfTranslationUnit.
@@ -165,11 +165,11 @@ IncrementalParser::Parse(llvm::StringRef input) {
Token Tok;
do {
PP.Lex(Tok);
- } while (Tok.isNot(tok::annot_repl_input_end));
+ } while (Tok.isNot(tok::eof));
} else {
Token AssertTok;
PP.Lex(AssertTok);
- assert(AssertTok.is(tok::annot_repl_input_end) &&
+ assert(AssertTok.is(tok::eof) &&
"Lexer must be EOF when starting incremental parse!");
}
diff --git a/clang/lib/Lex/PPLexerChange.cpp b/clang/lib/Lex/PPLexerChange.cpp
index 552902e70f094b..995a44dd2904a1 100644
--- a/clang/lib/Lex/PPLexerChange.cpp
+++ b/clang/lib/Lex/PPLexerChange.cpp
@@ -532,13 +532,7 @@ bool Preprocessor::HandleEndOfFile(Token &Result, bool isEndOfMacro) {
Result.startToken();
CurLexer->BufferPtr = EndPos;
- if (getLangOpts().IncrementalExtensions) {
- CurLexer->FormTokenWithChars(Result, EndPos, tok::annot_repl_input_end);
- Result.setAnnotationEndLoc(Result.getLocation());
- Result.setAnnotationValue(nullptr);
- } else {
- CurLexer->FormTokenWithChars(Result, EndPos, tok::eof);
- }
+ CurLexer->FormTokenWithChars(Result, EndPos, tok::eof);
if (isCodeCompletionEnabled()) {
// Inserting the code-completion point increases the source buffer by 1,
diff --git a/clang/lib/Lex/Preprocessor.cpp b/clang/lib/Lex/Preprocessor.cpp
index 84907adc5d744a..a8717424f1d155 100644
--- a/clang/lib/Lex/Preprocessor.cpp
+++ b/clang/lib/Lex/Preprocessor.cpp
@@ -1020,12 +1020,22 @@ void Preprocessor::Lex(Token &Result) {
// This token is injected to represent the translation of '#include "a.h"'
// into "import a.h;". Mimic the notional ';'.
case tok::annot_module_include:
- case tok::annot_repl_input_end:
case tok::semi:
TrackGMFState.handleSemi();
StdCXXImportSeqState.handleSemi();
ModuleDeclState.handleSemi();
break;
+ case tok::eof:
+ if (getLangOpts().IncrementalExtensions) {
+ TrackGMFState.handleSemi();
+ StdCXXImportSeqState.handleSemi();
+ ModuleDeclState.handleSemi();
+ } else {
+ TrackGMFState.handleMisc();
+ StdCXXImportSeqState.handleMisc();
+ ModuleDeclState.handleMisc();
+ }
+ break;
case tok::header_name:
case tok::annot_header_unit:
StdCXXImportSeqState.handleHeaderName();
@@ -1098,8 +1108,7 @@ void Preprocessor::LexTokensUntilEOF(std::vector<Token> *Tokens) {
while (1) {
Token Tok;
Lex(Tok);
- if (Tok.isOneOf(tok::unknown, tok::eof, tok::eod,
- tok::annot_repl_input_end))
+ if (Tok.isOneOf(tok::unknown, tok::eof, tok::eod))
break;
if (Tokens != nullptr)
Tokens->push_back(Tok);
diff --git a/clang/lib/Parse/ParseCXXInlineMethods.cpp b/clang/lib/Parse/ParseCXXInlineMethods.cpp
index 6b478ec2e36f6d..22e15cbe5b260e 100644
--- a/clang/lib/Parse/ParseCXXInlineMethods.cpp
+++ b/clang/lib/Parse/ParseCXXInlineMethods.cpp
@@ -902,7 +902,6 @@ bool Parser::ConsumeAndStoreUntil(tok::TokenKind T1, tok::TokenKind T2,
case tok::annot_module_begin:
case tok::annot_module_end:
case tok::annot_module_include:
- case tok::annot_repl_input_end:
// Ran out of tokens.
return false;
diff --git a/clang/lib/Parse/ParseDecl.cpp b/clang/lib/Parse/ParseDecl.cpp
index 40249287184446..5b76b5d2ef274b 100644
--- a/clang/lib/Parse/ParseDecl.cpp
+++ b/clang/lib/Parse/ParseDecl.cpp
@@ -2155,7 +2155,6 @@ void Parser::SkipMalformedDecl() {
case tok::annot_module_begin:
case tok::annot_module_end:
case tok::annot_module_include:
- case tok::annot_repl_input_end:
return;
default:
@@ -5899,9 +5898,8 @@ Parser::DeclGroupPtrTy Parser::ParseTopLevelStmtDecl() {
TLSD->setStmt(R.get());
}
- if (Tok.is(tok::annot_repl_input_end) &&
- Tok.getAnnotationValue() != nullptr) {
- ConsumeAnnotationToken();
+ if (Tok.is(tok::eof) && Tok.getEofData()) {
+ ConsumeToken();
TLSD->setSemiMissing();
}
diff --git a/clang/lib/Parse/ParseStmt.cpp b/clang/lib/Parse/ParseStmt.cpp
index 14bea6a1a79487..de82003f109328 100644
--- a/clang/lib/Parse/ParseStmt.cpp
+++ b/clang/lib/Parse/ParseStmt.cpp
@@ -584,7 +584,9 @@ StmtResult Parser::ParseExprStatement(ParsedStmtContext StmtCtx) {
Token *CurTok = nullptr;
// If the semicolon is missing at the end of REPL input, we want to print
// the result. Note we shouldn't eat the token since the callback needs it.
- if (Tok.is(tok::annot_repl_input_end))
+ // Preserve the data on EOF tokens that delimit cached declarations.
+ if (Tok.is(tok::eof) && getLangOpts().IncrementalExtensions &&
+ !Tok.getEofData())
CurTok = &Tok;
else
// Otherwise, eat the semicolon.
@@ -592,7 +594,7 @@ StmtResult Parser::ParseExprStatement(ParsedStmtContext StmtCtx) {
StmtResult R = handleExprStmt(Expr, StmtCtx);
if (CurTok && !R.isInvalid())
- CurTok->setAnnotationValue(R.get());
+ CurTok->setEofData(R.get());
return R;
}
diff --git a/clang/lib/Parse/Parser.cpp b/clang/lib/Parse/Parser.cpp
index c5d6a17fbfcd34..f0f4c7195381e5 100644
--- a/clang/lib/Parse/Parser.cpp
+++ b/clang/lib/Parse/Parser.cpp
@@ -354,7 +354,6 @@ bool Parser::SkipUntil(ArrayRef<tok::TokenKind> Toks, SkipUntilFlags Flags) {
case tok::annot_module_begin:
case tok::annot_module_end:
case tok::annot_module_include:
- case tok::annot_repl_input_end:
// Stop before we change submodules. They generally indicate a "good"
// place to pick up parsing again (except in the special case where
// we're trying to skip to EOF).
@@ -690,7 +689,6 @@ bool Parser::ParseTopLevelDecl(DeclGroupPtrTy &Result,
return false;
case tok::eof:
- case tok::annot_repl_input_end:
// Check whether -fmax-tokens= was reached.
if (PP.getMaxTokens() != 0 && PP.getTokenCount() > PP.getMaxTokens()) {
PP.Diag(Tok.getLocation(), diag::warn_max_tokens_total)
diff --git a/clang/lib/Serialization/ASTReader.cpp b/clang/lib/Serialization/ASTReader.cpp
index e6f6be496e5e58..56f1aa8a7f932a 100644
--- a/clang/lib/Serialization/ASTReader.cpp
+++ b/clang/lib/Serialization/ASTReader.cpp
@@ -2194,7 +2194,6 @@ Token ASTReader::ReadToken(ModuleFile &M, const RecordDataImpl &Record,
case tok::annot_pragma_unused:
case tok::annot_pragma_openacc:
case tok::annot_pragma_openacc_end:
- case tok::annot_repl_input_end:
break;
default:
llvm_unreachable("missing deserialization code for annotation token");
diff --git a/clang/lib/Serialization/ASTWriter.cpp b/clang/lib/Serialization/ASTWriter.cpp
index 0afba1c84fd627..48735e32eb5f66 100644
--- a/clang/lib/Serialization/ASTWriter.cpp
+++ b/clang/lib/Serialization/ASTWriter.cpp
@@ -5432,7 +5432,6 @@ void ASTWriter::AddToken(const Token &Tok, RecordDataImpl &Record) {
case tok::annot_pragma_unused:
case tok::annot_pragma_openacc:
case tok::annot_pragma_openacc_end:
- case tok::annot_repl_input_end:
break;
default:
llvm_unreachable("missing serialization code for annotation token");
diff --git a/clang/test/Interpreter/repl-input-end-recovery.cpp b/clang/test/Interpreter/repl-input-end-recovery.cpp
new file mode 100644
index 00000000000000..0b9d94117d4edd
--- /dev/null
+++ b/clang/test/Interpreter/repl-input-end-recovery.cpp
@@ -0,0 +1,50 @@
+// REQUIRES: host-supports-jit
+// RUN: cat %s | clang-repl > %t.out 2> %t.err
+// RUN: FileCheck %s --check-prefix=OUT < %t.out
+// RUN: FileCheck %s --check-prefix=ERR --implicit-check-not='fatal error:' < %t.err
+// RUN: not clang-repl -Xcc -fms-extensions '__pragma(section("broken", )' \
+// RUN: 'extern "C" int puts(const char *); ::puts("recovered");' 2>&1 \
+// RUN: | FileCheck %s --check-prefix=MS
+
+// MS: error: missing terminating ')' character
+// MS: error: Parsing failed.
+// MS: {{^recovered$}}
+
+extern "C" int printf(const char *, ...);
+
+void foo() { int x = 5;
+// ERR: error: expected '}'
+// ERR: error: Parsing failed.
+
+int g1 = 0; void foo() { g1 = 5; } foo(); printf("g1 = %d\n", ::g1);
+// OUT: g1 = 5
+
+void (*test)() = [](){ if }
+// ERR: error: expected '(' after 'if'
+// ERR: error: Parsing failed.
+
+int g2 = 0; void (*test)() = [](){ if (1) g2 = 7; }; test(); printf("g2 = %d\n", ::g2);
+// OUT: g2 = 7
+
+namespace myspace {
+// ERR: error: expected '}'
+// ERR: error: Parsing failed.
+
+namespace myspace { int v = 11; } printf("v = %d\n", ::myspace::v);
+// OUT: v = 11
+
+extern "C" { int cvalue;
+// ERR: error: expected '}'
+// ERR: error: Parsing failed.
+
+extern "C" { int cvalue = 13; } printf("cvalue = %d\n", ::cvalue);
+// OUT: cvalue = 13
+
+struct X { using type = int };
+// ERR: error: expected ';' after alias declaration
+// ERR: error: Parsing failed.
+
+struct X { using type = int; }; X::type t = 3; printf("t = %d\n", ::t);
+// OUT: t = 3
+
+%quit
More information about the cfe-commits
mailing list