[clang] [clang-repl] Fix printing preprocessed tokens and macros (PR #104964)

[Jonas Hahnfeld via cfe-commits]

https://github.com/hahnjo created https://github.com/llvm/llvm-project/pull/104964

None

>From dc37b356fb9527c3b4cf6b31f55d2dd5067fc29d Mon Sep 17 00:00:00 2001
From: Jonas Hahnfeld <jonas.hahnfeld at cern.ch>
Date: Tue, 20 Aug 2024 16:25:15 +0200
Subject: [PATCH 1/2] [clang-repl] Fix printing preprocessed tokens

---
 clang/lib/Frontend/PrintPreprocessedOutput.cpp | 6 +++---
 clang/test/Interpreter/preprocessor.cpp        | 3 +++
 2 files changed, 6 insertions(+), 3 deletions(-)
 create mode 100644 clang/test/Interpreter/preprocessor.cpp

diff --git a/clang/lib/Frontend/PrintPreprocessedOutput.cpp b/clang/lib/Frontend/PrintPreprocessedOutput.cpp
index 135dca0e6a1775..1d53a6f0183df1 100644
--- a/clang/lib/Frontend/PrintPreprocessedOutput.cpp
+++ b/clang/lib/Frontend/PrintPreprocessedOutput.cpp
@@ -916,8 +916,7 @@ static void PrintPreprocessedTokens(Preprocessor &PP, Token &Tok,
       PP.Lex(Tok);
       continue;
     } else if (Tok.is(tok::annot_repl_input_end)) {
-      PP.Lex(Tok);
-      continue;
+      // Fall through to exit the loop.
     } else if (Tok.is(tok::eod)) {
       // Don't print end of directive tokens, since they are typically newlines
       // that mess up our line tracking. These come from unknown pre-processor
@@ -1025,7 +1024,8 @@ static void PrintPreprocessedTokens(Preprocessor &PP, Token &Tok,
     Callbacks->setEmittedTokensOnThisLine();
     IsStartOfLine = false;
 
-    if (Tok.is(tok::eof)) break;
+    if (Tok.is(tok::eof) || Tok.is(tok::annot_repl_input_end))
+      break;
 
     PP.Lex(Tok);
     // If lexing that token causes us to need to skip future tokens, do so now.
diff --git a/clang/test/Interpreter/preprocessor.cpp b/clang/test/Interpreter/preprocessor.cpp
new file mode 100644
index 00000000000000..fe40d5878f9e76
--- /dev/null
+++ b/clang/test/Interpreter/preprocessor.cpp
@@ -0,0 +1,3 @@
+// RUN: %clang_cc1 -fincremental-extensions -E %s
+// RUN: %clang_cc1 -fincremental-extensions -E -dD %s
+// RUN: %clang_cc1 -fincremental-extensions -E -dI %s

>From 31eaf0d22b0d1ca596f53183525ac0d2ccd9a032 Mon Sep 17 00:00:00 2001
From: Jonas Hahnfeld <jonas.hahnfeld at cern.ch>
Date: Tue, 20 Aug 2024 16:29:05 +0200
Subject: [PATCH 2/2] [clang-repl] Fix printing macros

---
 clang/lib/Frontend/PrintPreprocessedOutput.cpp | 4 +---
 clang/test/Interpreter/preprocessor.cpp        | 1 +
 2 files changed, 2 insertions(+), 3 deletions(-)

diff --git a/clang/lib/Frontend/PrintPreprocessedOutput.cpp b/clang/lib/Frontend/PrintPreprocessedOutput.cpp
index 1d53a6f0183df1..383d4356084916 100644
--- a/clang/lib/Frontend/PrintPreprocessedOutput.cpp
+++ b/clang/lib/Frontend/PrintPreprocessedOutput.cpp
@@ -1048,9 +1048,7 @@ static void DoPrintMacros(Preprocessor &PP, raw_ostream *OS) {
   // the macro table at the end.
   PP.EnterMainSourceFile();
 
-  Token Tok;
-  do PP.Lex(Tok);
-  while (Tok.isNot(tok::eof));
+  PP.LexTokensUntilEOF();
 
   SmallVector<id_macro_pair, 128> MacrosByID;
   for (Preprocessor::macro_iterator I = PP.macro_begin(), E = PP.macro_end();
diff --git a/clang/test/Interpreter/preprocessor.cpp b/clang/test/Interpreter/preprocessor.cpp
index fe40d5878f9e76..8239fd45e661b0 100644
--- a/clang/test/Interpreter/preprocessor.cpp
+++ b/clang/test/Interpreter/preprocessor.cpp
@@ -1,3 +1,4 @@
 // RUN: %clang_cc1 -fincremental-extensions -E %s
 // RUN: %clang_cc1 -fincremental-extensions -E -dD %s
 // RUN: %clang_cc1 -fincremental-extensions -E -dI %s
+// RUN: %clang_cc1 -fincremental-extensions -E -dM %s