[flang-commits] [clang-tools-extra] [clang] [libcxx] [llvm] [flang] [lldb] [compiler-rt] [lld] [YAMLParser] Unfold multi-line scalar values (PR #70898)

Scott Linder via flang-commits flang-commits at lists.llvm.org
Mon Nov 6 14:15:36 PST 2023


================
@@ -2030,187 +2030,219 @@ bool Node::failed() const {
 }
 
 StringRef ScalarNode::getValue(SmallVectorImpl<char> &Storage) const {
-  // TODO: Handle newlines properly. We need to remove leading whitespace.
-  if (Value[0] == '"') { // Double quoted.
-    // Pull off the leading and trailing "s.
-    StringRef UnquotedValue = Value.substr(1, Value.size() - 2);
-    // Search for characters that would require unescaping the value.
-    StringRef::size_type i = UnquotedValue.find_first_of("\\\r\n");
-    if (i != StringRef::npos)
-      return unescapeDoubleQuoted(UnquotedValue, i, Storage);
+  if (Value[0] == '"')
+    return getDoubleQuotedValue(Value, Storage);
+  if (Value[0] == '\'')
+    return getSingleQuotedValue(Value, Storage);
+  return getPlainValue(Value, Storage);
+}
+
+static StringRef
+parseScalarValue(StringRef UnquotedValue, SmallVectorImpl<char> &Storage,
+                 StringRef LookupChars,
+                 std::function<StringRef(StringRef, SmallVectorImpl<char> &)>
+                     UnescapeCallback) {
+  size_t I = UnquotedValue.find_first_of(LookupChars);
+  if (I == StringRef::npos)
     return UnquotedValue;
-  } else if (Value[0] == '\'') { // Single quoted.
-    // Pull off the leading and trailing 's.
-    StringRef UnquotedValue = Value.substr(1, Value.size() - 2);
-    StringRef::size_type i = UnquotedValue.find('\'');
-    if (i != StringRef::npos) {
-      // We're going to need Storage.
-      Storage.clear();
-      Storage.reserve(UnquotedValue.size());
-      for (; i != StringRef::npos; i = UnquotedValue.find('\'')) {
-        StringRef Valid(UnquotedValue.begin(), i);
-        llvm::append_range(Storage, Valid);
-        Storage.push_back('\'');
-        UnquotedValue = UnquotedValue.substr(i + 2);
-      }
-      llvm::append_range(Storage, UnquotedValue);
-      return StringRef(Storage.begin(), Storage.size());
-    }
-    return UnquotedValue;
-  }
-  // Plain.
-  // Trim whitespace ('b-char' and 's-white').
-  // NOTE: Alternatively we could change the scanner to not include whitespace
-  //       here in the first place.
-  return Value.rtrim("\x0A\x0D\x20\x09");
-}
 
-StringRef ScalarNode::unescapeDoubleQuoted( StringRef UnquotedValue
-                                          , StringRef::size_type i
-                                          , SmallVectorImpl<char> &Storage)
-                                          const {
-  // Use Storage to build proper value.
   Storage.clear();
   Storage.reserve(UnquotedValue.size());
-  for (; i != StringRef::npos; i = UnquotedValue.find_first_of("\\\r\n")) {
-    // Insert all previous chars into Storage.
-    StringRef Valid(UnquotedValue.begin(), i);
-    llvm::append_range(Storage, Valid);
-    // Chop off inserted chars.
-    UnquotedValue = UnquotedValue.substr(i);
-
-    assert(!UnquotedValue.empty() && "Can't be empty!");
-
-    // Parse escape or line break.
-    switch (UnquotedValue[0]) {
-    case '\r':
-    case '\n':
-      Storage.push_back('\n');
-      if (   UnquotedValue.size() > 1
-          && (UnquotedValue[1] == '\r' || UnquotedValue[1] == '\n'))
-        UnquotedValue = UnquotedValue.substr(1);
-      UnquotedValue = UnquotedValue.substr(1);
-      break;
-    default:
-      if (UnquotedValue.size() == 1) {
-        Token T;
-        T.Range = StringRef(UnquotedValue.begin(), 1);
-        setError("Unrecognized escape code", T);
-        return "";
-      }
-      UnquotedValue = UnquotedValue.substr(1);
-      switch (UnquotedValue[0]) {
-      default: {
-          Token T;
-          T.Range = StringRef(UnquotedValue.begin(), 1);
-          setError("Unrecognized escape code", T);
-          return "";
-        }
-      case '\r':
-      case '\n':
-        // Remove the new line.
-        if (   UnquotedValue.size() > 1
-            && (UnquotedValue[1] == '\r' || UnquotedValue[1] == '\n'))
-          UnquotedValue = UnquotedValue.substr(1);
-        // If this was just a single byte newline, it will get skipped
-        // below.
-        break;
-      case '0':
-        Storage.push_back(0x00);
-        break;
-      case 'a':
-        Storage.push_back(0x07);
-        break;
-      case 'b':
-        Storage.push_back(0x08);
-        break;
-      case 't':
-      case 0x09:
-        Storage.push_back(0x09);
-        break;
-      case 'n':
-        Storage.push_back(0x0A);
-        break;
-      case 'v':
-        Storage.push_back(0x0B);
-        break;
-      case 'f':
-        Storage.push_back(0x0C);
-        break;
-      case 'r':
-        Storage.push_back(0x0D);
-        break;
-      case 'e':
-        Storage.push_back(0x1B);
-        break;
+  char LastNewLineAddedAs = '\0';
+  for (; I != StringRef::npos; I = UnquotedValue.find_first_of(LookupChars)) {
+    if (UnquotedValue[I] != '\x0D' && UnquotedValue[I] != '\x0A') {
+      llvm::append_range(Storage, UnquotedValue.take_front(I));
+      UnquotedValue = UnescapeCallback(UnquotedValue.drop_front(I), Storage);
+      LastNewLineAddedAs = '\0';
+      continue;
+    }
+    if (size_t LastNonSWhite = UnquotedValue.find_last_not_of("\x20\x09", I);
+        LastNonSWhite != StringRef::npos) {
+      llvm::append_range(Storage, UnquotedValue.take_front(LastNonSWhite + 1));
+      Storage.push_back(' ');
+      LastNewLineAddedAs = ' ';
+    } else {
+      // Note: we can't just check if the last character in Storage is ' ',
+      // '\n', or something else; that would give a wrong result for double
+      // quoted values containing an escaped space character before a new-line
+      // character.
+      switch (LastNewLineAddedAs) {
       case ' ':
-        Storage.push_back(0x20);
-        break;
-      case '"':
-        Storage.push_back(0x22);
-        break;
-      case '/':
-        Storage.push_back(0x2F);
-        break;
-      case '\\':
-        Storage.push_back(0x5C);
-        break;
-      case 'N':
-        encodeUTF8(0x85, Storage);
+        assert(!Storage.empty() && Storage.back() == ' ');
+        Storage.back() = '\n';
+        LastNewLineAddedAs = '\n';
         break;
-      case '_':
-        encodeUTF8(0xA0, Storage);
-        break;
-      case 'L':
-        encodeUTF8(0x2028, Storage);
+      case '\n':
+        assert(!Storage.empty() && Storage.back() == '\n');
+        Storage.push_back('\n');
         break;
-      case 'P':
-        encodeUTF8(0x2029, Storage);
+      default:
+        Storage.push_back(' ');
+        LastNewLineAddedAs = ' ';
         break;
-      case 'x': {
-          if (UnquotedValue.size() < 3)
-            // TODO: Report error.
-            break;
-          unsigned int UnicodeScalarValue;
-          if (UnquotedValue.substr(1, 2).getAsInteger(16, UnicodeScalarValue))
-            // TODO: Report error.
-            UnicodeScalarValue = 0xFFFD;
-          encodeUTF8(UnicodeScalarValue, Storage);
-          UnquotedValue = UnquotedValue.substr(2);
-          break;
-        }
-      case 'u': {
-          if (UnquotedValue.size() < 5)
-            // TODO: Report error.
-            break;
-          unsigned int UnicodeScalarValue;
-          if (UnquotedValue.substr(1, 4).getAsInteger(16, UnicodeScalarValue))
-            // TODO: Report error.
-            UnicodeScalarValue = 0xFFFD;
-          encodeUTF8(UnicodeScalarValue, Storage);
-          UnquotedValue = UnquotedValue.substr(4);
-          break;
-        }
-      case 'U': {
-          if (UnquotedValue.size() < 9)
-            // TODO: Report error.
-            break;
-          unsigned int UnicodeScalarValue;
-          if (UnquotedValue.substr(1, 8).getAsInteger(16, UnicodeScalarValue))
-            // TODO: Report error.
-            UnicodeScalarValue = 0xFFFD;
-          encodeUTF8(UnicodeScalarValue, Storage);
-          UnquotedValue = UnquotedValue.substr(8);
-          break;
-        }
       }
-      UnquotedValue = UnquotedValue.substr(1);
     }
+    // Handle Windows-style EOL
+    if (UnquotedValue.substr(I, 2) == "\x0D\x0A")
+      I++;
+    UnquotedValue = UnquotedValue.drop_front(I + 1).ltrim("\x20\x09");
   }
   llvm::append_range(Storage, UnquotedValue);
   return StringRef(Storage.begin(), Storage.size());
 }
 
+StringRef
+ScalarNode::getDoubleQuotedValue(StringRef RawValue,
+                                 SmallVectorImpl<char> &Storage) const {
+  assert(RawValue.size() >= 2 && RawValue.front() == '"' &&
+         RawValue.back() == '"');
+  StringRef UnquotedValue = RawValue.substr(1, RawValue.size() - 2);
+
+  auto UnescapeFunc = [this](StringRef UnquotedValue,
+                             SmallVectorImpl<char> &Storage) {
+    assert(UnquotedValue.take_front(1) == "\\");
+    if (UnquotedValue.size() == 1) {
+      Token T;
+      T.Range = UnquotedValue;
+      this->setError("Unrecognized escape code", T);
----------------
slinder1 wrote:

There is no ambiguity with `this->` dropped, right? I think we generally avoid gratuitous mentions of `this`, but I can't actually find it in any docs

https://github.com/llvm/llvm-project/pull/70898


More information about the flang-commits mailing list