[flang-commits] [flang] 4299d9b - [flang] Accomodate historic preprocessing usage (#78868)

[via flang-commits]

Author: Peter Klausler
Date: 2024-01-25T16:00:57-08:00
New Revision: 4299d9b1af45843fd3230dc429fcf9cc19460303

URL: https://github.com/llvm/llvm-project/commit/4299d9b1af45843fd3230dc429fcf9cc19460303
DIFF: https://github.com/llvm/llvm-project/commit/4299d9b1af45843fd3230dc429fcf9cc19460303.diff

LOG: [flang] Accomodate historic preprocessing usage (#78868)

Some Fortran codes use line continuation as a form of token pasting; see
https://github.com/llvm/llvm-project/issues/78797. This works in
compilers that run a C-like preprocessor and then apply line
continuation to its output; f18 implements line continuation during
tokenization and preprocessing, but can still handle this case.

In the rare case when an identifier is split across two or more
continuation lines, this patch allows its parts to be distinct
preprocessing tokens for the purpose of macro replacemnt. They (or their
replacement texts) can be effectively rejoined later as a single
identifier when the cooked character stream is tokenized in parsing.

Fixes https://github.com/llvm/llvm-project/issues/78797.

Added: 
    flang/test/Preprocessing/pp133.F90

Modified: 
    flang/lib/Parser/prescan.cpp
    flang/lib/Parser/prescan.h

Removed: 
    


################################################################################
diff  --git a/flang/lib/Parser/prescan.cpp b/flang/lib/Parser/prescan.cpp
index 68d7d9f0c53c475..6bccfc3f9baea9f 100644
--- a/flang/lib/Parser/prescan.cpp
+++ b/flang/lib/Parser/prescan.cpp
@@ -438,7 +438,8 @@ void Prescanner::NextChar() {
 // character is reached; handles C-style comments in preprocessing
 // directives, Fortran ! comments, stuff after the right margin in
 // fixed form, and all forms of line continuation.
-void Prescanner::SkipToNextSignificantCharacter() {
+bool Prescanner::SkipToNextSignificantCharacter() {
+  auto anyContinuationLine{false};
   if (inPreprocessorDirective_) {
     SkipCComments();
   } else {
@@ -449,6 +450,7 @@ void Prescanner::SkipToNextSignificantCharacter() {
       mightNeedSpace = *at_ == '\n';
     }
     for (; Continuation(mightNeedSpace); mightNeedSpace = false) {
+      anyContinuationLine = true;
       ++continuationLines_;
       if (MustSkipToEndOfLine()) {
         SkipToEndOfLine();
@@ -458,6 +460,7 @@ void Prescanner::SkipToNextSignificantCharacter() {
       tabInCurrentLine_ = true;
     }
   }
+  return anyContinuationLine;
 }
 
 void Prescanner::SkipCComments() {
@@ -625,7 +628,23 @@ bool Prescanner::NextToken(TokenSequence &tokens) {
     }
     preventHollerith_ = false;
   } else if (IsLegalInIdentifier(*at_)) {
-    while (IsLegalInIdentifier(EmitCharAndAdvance(tokens, *at_))) {
+    int parts{1};
+    do {
+      EmitChar(tokens, *at_);
+      ++at_, ++column_;
+      if (SkipToNextSignificantCharacter() && IsLegalIdentifierStart(*at_)) {
+        tokens.CloseToken();
+        ++parts;
+      }
+    } while (IsLegalInIdentifier(*at_));
+    if (parts >= 3) {
+      // Subtlety: When an identifier is split across three or more continuation
+      // lines, its parts are kept as distinct pp-tokens so that macro
+      // operates on them independently.  This trick accommodates the historic
+      // practice of using line continuation for token pasting after
+      // replacement.
+    } else if (parts == 2) {
+      tokens.ReopenLastToken();
     }
     if (InFixedFormSource()) {
       SkipSpaces();

diff  --git a/flang/lib/Parser/prescan.h b/flang/lib/Parser/prescan.h
index 84e046c1b102f0a..7442b5d22633543 100644
--- a/flang/lib/Parser/prescan.h
+++ b/flang/lib/Parser/prescan.h
@@ -159,7 +159,8 @@ class Prescanner {
   void SkipToEndOfLine();
   bool MustSkipToEndOfLine() const;
   void NextChar();
-  void SkipToNextSignificantCharacter();
+  // True when input flowed to a continuation line
+  bool SkipToNextSignificantCharacter();
   void SkipCComments();
   void SkipSpaces();
   static const char *SkipWhiteSpace(const char *);

diff  --git a/flang/test/Preprocessing/pp133.F90 b/flang/test/Preprocessing/pp133.F90
new file mode 100644
index 000000000000000..01e7b010d426ece
--- /dev/null
+++ b/flang/test/Preprocessing/pp133.F90
@@ -0,0 +1,9 @@
+! RUN: %flang -E %s 2>&1 | FileCheck %s
+! CHECK: print *, ADC
+#define B D
+implicit none
+real ADC
+print *, A&
+  &B&
+  &C
+end