[flang-commits] [flang] a0a1a4e - [flang] Improve error handling for bad characters in source

[peter klausler via flang-commits]

Author: peter klausler
Date: 2020-08-25T11:42:19-07:00
New Revision: a0a1a4e5c83db53c806c56011a8741b31ab598a4

URL: https://github.com/llvm/llvm-project/commit/a0a1a4e5c83db53c806c56011a8741b31ab598a4
DIFF: https://github.com/llvm/llvm-project/commit/a0a1a4e5c83db53c806c56011a8741b31ab598a4.diff

LOG: [flang] Improve error handling for bad characters in source

When an illegal character appears in Fortran source (after
preprocessing), catch and report it in the prescanning phase
rather than leaving it for the parser to cope with.

Differential Revision: https://reviews.llvm.org/D86553

Added: 
    

Modified: 
    flang/include/flang/Parser/characters.h
    flang/lib/Parser/prescan.cpp
    flang/lib/Parser/token-sequence.cpp
    flang/lib/Parser/token-sequence.h
    flang/test/Preprocessing/pp130.F90

Removed: 
    


################################################################################
diff  --git a/flang/include/flang/Parser/characters.h b/flang/include/flang/Parser/characters.h
index eefb524e5d87..120560625da2 100644
--- a/flang/include/flang/Parser/characters.h
+++ b/flang/include/flang/Parser/characters.h
@@ -151,6 +151,33 @@ inline constexpr std::optional<char> BackslashEscapeChar(char ch) {
   }
 }
 
+// Does not include spaces or line ending characters.
+inline constexpr bool IsValidFortranTokenCharacter(char ch) {
+  switch (ch) {
+  case '"':
+  case '%':
+  case '\'':
+  case '(':
+  case ')':
+  case '*':
+  case '+':
+  case ',':
+  case '-':
+  case '.':
+  case '/':
+  case ':':
+  case ';':
+  case '<':
+  case '=':
+  case '>':
+  case '[':
+  case ']':
+    return true;
+  default:
+    return IsLegalIdentifierStart(ch) || IsDecimalDigit(ch);
+  }
+}
+
 struct EncodedCharacter {
   static constexpr int maxEncodingBytes{6};
   char buffer[maxEncodingBytes];

diff  --git a/flang/lib/Parser/prescan.cpp b/flang/lib/Parser/prescan.cpp
index 7fdeaf00d185..5e6f13797646 100644
--- a/flang/lib/Parser/prescan.cpp
+++ b/flang/lib/Parser/prescan.cpp
@@ -184,7 +184,8 @@ void Prescanner::Statement() {
     case LineClassification::Kind::PreprocessorDirective:
       Say(preprocessed->GetProvenanceRange(),
           "Preprocessed line resembles a preprocessor directive"_en_US);
-      preprocessed->ToLowerCase().Emit(cooked_);
+      preprocessed->ToLowerCase().CheckBadFortranCharacters(messages_).Emit(
+          cooked_);
       break;
     case LineClassification::Kind::CompilerDirective:
       if (preprocessed->HasRedundantBlanks()) {
@@ -193,7 +194,9 @@ void Prescanner::Statement() {
       NormalizeCompilerDirectiveCommentMarker(*preprocessed);
       preprocessed->ToLowerCase();
       SourceFormChange(preprocessed->ToString());
-      preprocessed->ClipComment(true /* skip first ! */).Emit(cooked_);
+      preprocessed->ClipComment(true /* skip first ! */)
+          .CheckBadFortranCharacters(messages_)
+          .Emit(cooked_);
       break;
     case LineClassification::Kind::Source:
       if (inFixedForm_) {
@@ -205,7 +208,10 @@ void Prescanner::Statement() {
           preprocessed->RemoveRedundantBlanks();
         }
       }
-      preprocessed->ToLowerCase().ClipComment().Emit(cooked_);
+      preprocessed->ToLowerCase()
+          .ClipComment()
+          .CheckBadFortranCharacters(messages_)
+          .Emit(cooked_);
       break;
     }
   } else {
@@ -213,7 +219,7 @@ void Prescanner::Statement() {
     if (line.kind == LineClassification::Kind::CompilerDirective) {
       SourceFormChange(tokens.ToString());
     }
-    tokens.Emit(cooked_);
+    tokens.CheckBadFortranCharacters(messages_).Emit(cooked_);
   }
   if (omitNewline_) {
     omitNewline_ = false;

diff  --git a/flang/lib/Parser/token-sequence.cpp b/flang/lib/Parser/token-sequence.cpp
index 07c5b12e5f75..4797cb759a72 100644
--- a/flang/lib/Parser/token-sequence.cpp
+++ b/flang/lib/Parser/token-sequence.cpp
@@ -8,6 +8,7 @@
 
 #include "token-sequence.h"
 #include "flang/Parser/characters.h"
+#include "flang/Parser/message.h"
 #include "llvm/Support/raw_ostream.h"
 
 namespace Fortran::parser {
@@ -310,4 +311,25 @@ ProvenanceRange TokenSequence::GetIntervalProvenanceRange(
 ProvenanceRange TokenSequence::GetProvenanceRange() const {
   return GetIntervalProvenanceRange(0, start_.size());
 }
+
+const TokenSequence &TokenSequence::CheckBadFortranCharacters(
+    Messages &messages) const {
+  std::size_t tokens{SizeInTokens()};
+  for (std::size_t j{0}; j < tokens; ++j) {
+    CharBlock token{TokenAt(j)};
+    char ch{token.FirstNonBlank()};
+    if (ch != ' ' && !IsValidFortranTokenCharacter(ch)) {
+      if (ch == '!' && j == 0) {
+        // allow in !dir$
+      } else if (ch < ' ' || ch >= '\x7f') {
+        messages.Say(GetTokenProvenanceRange(j),
+            "bad character (0x%02x) in Fortran token"_err_en_US, ch & 0xff);
+      } else {
+        messages.Say(GetTokenProvenanceRange(j),
+            "bad character ('%c') in Fortran token"_err_en_US, ch);
+      }
+    }
+  }
+  return *this;
+}
 } // namespace Fortran::parser

diff  --git a/flang/lib/Parser/token-sequence.h b/flang/lib/Parser/token-sequence.h
index d98c0b955c5e..6a10ef1977d3 100644
--- a/flang/lib/Parser/token-sequence.h
+++ b/flang/lib/Parser/token-sequence.h
@@ -27,6 +27,8 @@ class raw_ostream;
 
 namespace Fortran::parser {
 
+class Messages;
+
 // Buffers a contiguous sequence of characters that has been partitioned into
 // a sequence of preprocessing tokens with provenances.
 class TokenSequence {
@@ -115,6 +117,7 @@ class TokenSequence {
   TokenSequence &RemoveBlanks(std::size_t firstChar = 0);
   TokenSequence &RemoveRedundantBlanks(std::size_t firstChar = 0);
   TokenSequence &ClipComment(bool skipFirst = false);
+  const TokenSequence &CheckBadFortranCharacters(Messages &) const;
   void Emit(CookedSource &) const;
   void Dump(llvm::raw_ostream &) const;
 

diff  --git a/flang/test/Preprocessing/pp130.F90 b/flang/test/Preprocessing/pp130.F90
index af4ad126e6fa..be1148807b8b 100644
--- a/flang/test/Preprocessing/pp130.F90
+++ b/flang/test/Preprocessing/pp130.F90
@@ -1,5 +1,5 @@
-! RUN: %f18 -E %s 2>&1 | FileCheck %s
-! CHECK: j = j + &
+! RUN: (%f18 -E %s 2>&1 || true) | FileCheck %s
+! CHECK: error: bad character ('&') in Fortran token
 ! #define KWM &, use for continuation w/o pasting (ifort and nag seem to continue #define)
 #define KWM &