[clang] Re-apply "[Parse] Split incremental-extensions" (PR #66446)

[via cfe-commits]

llvmbot wrote:


<!--LLVM PR SUMMARY COMMENT-->

@llvm/pr-subscribers-clang
            
<details>
<summary>Changes</summary>
Re-applies #65683 with a fix to always run
`Actions.ActOnEndOfTranslationUnit` regardless of incremental processing.
--
Full diff: https://github.com/llvm/llvm-project/pull/66446.diff

4 Files Affected:

- (modified) clang/include/clang/Lex/Preprocessor.h (+5-5) 
- (modified) clang/lib/Lex/PPLexerChange.cpp (+1-1) 
- (modified) clang/lib/Lex/Preprocessor.cpp (+4) 
- (modified) clang/lib/Parse/Parser.cpp (+6-1) 


<pre>
diff --git a/clang/include/clang/Lex/Preprocessor.h b/clang/include/clang/Lex/Preprocessor.h
index bc1d94a61508d8d..575d08b83fd3a02 100644
--- a/clang/include/clang/Lex/Preprocessor.h
+++ b/clang/include/clang/Lex/Preprocessor.h
@@ -277,6 +277,9 @@ class Preprocessor {
   /// Empty line handler.
   EmptylineHandler *Emptyline = nullptr;
 
+  /// True to avoid tearing down the lexer etc on EOF
+  bool IncrementalProcessing = false;
+
 public:
   /// The kind of translation unit we are processing.
   const TranslationUnitKind TUKind;
@@ -1910,14 +1913,11 @@ class Preprocessor {
   void recomputeCurLexerKind();
 
   /// Returns true if incremental processing is enabled
-  bool isIncrementalProcessingEnabled() const {
-    return getLangOpts().IncrementalExtensions;
-  }
+  bool isIncrementalProcessingEnabled() const { return IncrementalProcessing; }
 
   /// Enables the incremental processing
   void enableIncrementalProcessing(bool value = true) {
-    // FIXME: Drop this interface.
-    const_cast&lt;LangOptions &amp;&gt;(getLangOpts()).IncrementalExtensions = value;
+    IncrementalProcessing = value;
   }
 
   /// Specify the point at which code-completion will be performed.
diff --git a/clang/lib/Lex/PPLexerChange.cpp b/clang/lib/Lex/PPLexerChange.cpp
index ab005381adfaf2c..811a760420e0a2d 100644
--- a/clang/lib/Lex/PPLexerChange.cpp
+++ b/clang/lib/Lex/PPLexerChange.cpp
@@ -541,7 +541,7 @@ bool Preprocessor::HandleEndOfFile(Token &amp;Result, bool isEndOfMacro) {
   Result.startToken();
   CurLexer-&gt;BufferPtr = EndPos;
 
-  if (isIncrementalProcessingEnabled()) {
+  if (getLangOpts().IncrementalExtensions) {
     CurLexer-&gt;FormTokenWithChars(Result, EndPos, tok::annot_repl_input_end);
     Result.setAnnotationEndLoc(Result.getLocation());
     Result.setAnnotationValue(nullptr);
diff --git a/clang/lib/Lex/Preprocessor.cpp b/clang/lib/Lex/Preprocessor.cpp
index 8de78a13930ed62..f0381c18a8b6f77 100644
--- a/clang/lib/Lex/Preprocessor.cpp
+++ b/clang/lib/Lex/Preprocessor.cpp
@@ -146,6 +146,10 @@ Preprocessor::Preprocessor(std::shared_ptr&lt;PreprocessorOptions&gt; PPOpts,
     Ident_AbnormalTermination = nullptr;
   }
 
+  // Default incremental processing to -fincremental-extensions, clients can
+  // override with `enableIncrementalProcessing` if desired.
+  IncrementalProcessing = LangOpts.IncrementalExtensions;
+
   // If using a PCH where a #pragma hdrstop is expected, start skipping tokens.
   if (usingPCHWithPragmaHdrStop())
     SkippingUntilPragmaHdrStop = true;
diff --git a/clang/lib/Parse/Parser.cpp b/clang/lib/Parse/Parser.cpp
index 09215b8303ecf9c..dda1dbaa0c21aa9 100644
--- a/clang/lib/Parse/Parser.cpp
+++ b/clang/lib/Parse/Parser.cpp
@@ -615,6 +615,11 @@ bool Parser::ParseTopLevelDecl(DeclGroupPtrTy &amp;Result,
                                Sema::ModuleImportState &amp;ImportState) {
   DestroyTemplateIdAnnotationsRAIIObj CleanupRAII(*this);
 
+  // Skip over the EOF token, flagging end of previous input for incremental
+  // processing
+  if (PP.isIncrementalProcessingEnabled() &amp;&amp; Tok.is(tok::eof))
+    ConsumeToken();
+
   Result = nullptr;
   switch (Tok.getKind()) {
   case tok::annot_pragma_unused:
@@ -1038,7 +1043,7 @@ Parser::ParseExternalDeclaration(ParsedAttributes &amp;Attrs,
       ConsumeToken();
       return nullptr;
     }
-    if (PP.isIncrementalProcessingEnabled() &amp;&amp;
+    if (getLangOpts().IncrementalExtensions &amp;&amp;
         !isDeclarationStatement(/*DisambiguatingWithExpression=*/true))
       return ParseTopLevelStmtDecl();
 
</pre>
</details>


https://github.com/llvm/llvm-project/pull/66446