[flang-commits] [flang] 189c083 - [flang] Fix edge case with Hollerith

[peter klausler via flang-commits]

Author: peter klausler
Date: 2020-08-10T11:31:41-07:00
New Revision: 189c0833dfd654f08a8655ca765e11e8890a92ae

URL: https://github.com/llvm/llvm-project/commit/189c0833dfd654f08a8655ca765e11e8890a92ae
DIFF: https://github.com/llvm/llvm-project/commit/189c0833dfd654f08a8655ca765e11e8890a92ae.diff

LOG: [flang] Fix edge case with Hollerith

To prevent mistokenization of CHARACTER*2HXY as a Hollerith
literal constant while allowing it in DATA A/2*2HXY/, there's
a little state that tracks whether a / has been seen earlier
in the same statement.  But it was being reset on each line,
not statement, so Hollerith in a DATA statement continuation
line was incorrectly tokenized.  Fixed.

Differential Revision: https://reviews.llvm.org/D85571

Added: 
    flang/test/Preprocessing/hollerith.f

Modified: 
    flang/lib/Parser/prescan.cpp
    flang/lib/Parser/prescan.h

Removed: 
    


################################################################################
diff  --git a/flang/lib/Parser/prescan.cpp b/flang/lib/Parser/prescan.cpp
index face0d20e4fc..9e90f7f6228f 100644
--- a/flang/lib/Parser/prescan.cpp
+++ b/flang/lib/Parser/prescan.cpp
@@ -110,7 +110,7 @@ void Prescanner::Statement() {
   case LineClassification::Kind::CompilerDirective:
     directiveSentinel_ = line.sentinel;
     CHECK(InCompilerDirective());
-    BeginSourceLineAndAdvance();
+    BeginStatementAndAdvance();
     if (inFixedForm_) {
       CHECK(IsFixedFormCommentChar(*at_));
     } else {
@@ -144,7 +144,7 @@ void Prescanner::Statement() {
     }
     break;
   case LineClassification::Kind::Source:
-    BeginSourceLineAndAdvance();
+    BeginStatementAndAdvance();
     if (inFixedForm_) {
       LabelField(tokens);
     } else if (skipLeadingAmpersand_) {
@@ -226,7 +226,7 @@ void Prescanner::Statement() {
 TokenSequence Prescanner::TokenizePreprocessorDirective() {
   CHECK(nextLine_ < limit_ && !inPreprocessorDirective_);
   inPreprocessorDirective_ = true;
-  BeginSourceLineAndAdvance();
+  BeginStatementAndAdvance();
   TokenSequence tokens;
   while (NextToken(tokens)) {
   }
@@ -497,12 +497,8 @@ bool Prescanner::NextToken(TokenSequence &tokens) {
     } while (IsLegalInIdentifier(EmitCharAndAdvance(tokens, *at_)));
     if (*at_ == '\'' || *at_ == '"') {
       QuotedCharacterLiteral(tokens, start);
-      preventHollerith_ = false;
-    } else {
-      // Subtle: Don't misrecognize labeled DO statement label as Hollerith
-      // when the loop control variable starts with 'H'.
-      preventHollerith_ = true;
     }
+    preventHollerith_ = false;
   } else if (*at_ == '*') {
     if (EmitCharAndAdvance(tokens, '*') == '*') {
       EmitCharAndAdvance(tokens, '*');
@@ -510,7 +506,7 @@ bool Prescanner::NextToken(TokenSequence &tokens) {
       // Subtle ambiguity:
       //  CHARACTER*2H     declares H because *2 is a kind specifier
       //  DATAC/N*2H  /    is repeated Hollerith
-      preventHollerith_ = !slashInCurrentLine_;
+      preventHollerith_ = !slashInCurrentStatement_;
     }
   } else {
     char ch{*at_};
@@ -530,7 +526,7 @@ bool Prescanner::NextToken(TokenSequence &tokens) {
       // token comprises two characters
       EmitCharAndAdvance(tokens, nch);
     } else if (ch == '/') {
-      slashInCurrentLine_ = true;
+      slashInCurrentStatement_ = true;
     }
   }
   tokens.CloseToken();

diff  --git a/flang/lib/Parser/prescan.h b/flang/lib/Parser/prescan.h
index c4fdeae6e15e..595f8f701185 100644
--- a/flang/lib/Parser/prescan.h
+++ b/flang/lib/Parser/prescan.h
@@ -95,9 +95,6 @@ class Prescanner {
     at_ = at;
     column_ = 1;
     tabInCurrentLine_ = false;
-    slashInCurrentLine_ = false;
-    preventHollerith_ = false;
-    delimiterNesting_ = 0;
   }
 
   void BeginSourceLineAndAdvance() {
@@ -105,6 +102,13 @@ class Prescanner {
     NextLine();
   }
 
+  void BeginStatementAndAdvance() {
+    BeginSourceLineAndAdvance();
+    slashInCurrentStatement_ = false;
+    preventHollerith_ = false;
+    delimiterNesting_ = 0;
+  }
+
   Provenance GetProvenance(const char *sourceChar) const {
     return startProvenance_ + (sourceChar - start_);
   }
@@ -199,8 +203,8 @@ class Prescanner {
   const char *at_{nullptr}; // next character to process; < nextLine_
   int column_{1}; // card image column position of next character
   bool tabInCurrentLine_{false};
-  bool slashInCurrentLine_{false};
-  bool preventHollerith_{false};
+  bool slashInCurrentStatement_{false};
+  bool preventHollerith_{false}; // CHARACTER*4HIMOM not Hollerith
   bool inCharLiteral_{false};
   bool inPreprocessorDirective_{false};
 

diff  --git a/flang/test/Preprocessing/hollerith.f b/flang/test/Preprocessing/hollerith.f
new file mode 100644
index 000000000000..0c920e0678e1
--- /dev/null
+++ b/flang/test/Preprocessing/hollerith.f
@@ -0,0 +1,14 @@
+! RUN: %f18 -E %s 2>&1 | FileCheck %s
+! CHECK: character*1hi
+! CHECK: dataa/1*1h /
+! CHECK: datab/1*1h /
+! CHECK: do1h=1,2
+      CHARACTER*1H I
+      CHARACTER*1 A,B
+      INTEGER H
+      DATA A/1*1H /
+      DATA B/
+     +1*1H /
+      DO1H =1,2
+ 1    CONTINUE
+      END