[PATCH] D41490: [YAML] Fix UTF-8 handling

[Francis Visoiu Mistrih via Phabricator via llvm-commits]

thegameg created this revision.
thegameg added reviewers: JDevlieghere, MatzeB, vladimir.plyashkun.

Previous YAML quoting patches broke UTF-8 printing in YAML: see https://reviews.llvm.org/D41290#961801.


https://reviews.llvm.org/D41490

Files:
  lib/Support/YAMLTraits.cpp
  unittests/Support/YAMLIOTest.cpp


Index: unittests/Support/YAMLIOTest.cpp
===================================================================
--- unittests/Support/YAMLIOTest.cpp
+++ unittests/Support/YAMLIOTest.cpp
@@ -2541,3 +2541,31 @@
   ostr.flush();
   EXPECT_EQ("'abc''fdf'", out);
 }
+
+TEST(YAMLIO, TestEscapedUTF8SingleQuoteInsideDoubleQuote) {
+  std::string Id = "parameter ''параметр'' is unused";
+
+  std::string out;
+  llvm::raw_string_ostream ostr(out);
+  Output xout(ostr, nullptr, 0);
+
+  llvm::yaml::EmptyContext Ctx;
+  yamlize(xout, Id, true, Ctx);
+
+  ostr.flush();
+  EXPECT_EQ("\"parameter ''параметр'' is unused\"", out);
+}
+
+TEST(YAMLIO, TestEscapedUTF8) {
+  std::string Id = "/*параметр*/";
+
+  std::string out;
+  llvm::raw_string_ostream ostr(out);
+  Output xout(ostr, nullptr, 0);
+
+  llvm::yaml::EmptyContext Ctx;
+  yamlize(xout, Id, true, Ctx);
+
+  ostr.flush();
+  EXPECT_EQ("\"/*параметр*/\"", out);
+}
Index: lib/Support/YAMLTraits.cpp
===================================================================
--- lib/Support/YAMLTraits.cpp
+++ lib/Support/YAMLTraits.cpp
@@ -657,7 +657,10 @@
       }
       i = j + 1;
     } else if (MustQuote == QuotingType::Double &&
-               !sys::unicode::isPrintable(S[j])) {
+               !sys::unicode::isPrintable(S[j]) && (S[j] & 0x80) == 0) {
+      // If we're double quoting non-printable characters, we prefer printing
+      // them as "\x" + their hex representation. Note that special casing is
+      // needed for UTF-8, where we want to print the non-escaped characters.
       output(StringRef(&Base[i], j - i)); // "flush"
       output(StringLiteral("\\x"));
 


-------------- next part --------------
A non-text attachment was scrubbed...
Name: D41490.127880.patch
Type: text/x-patch
Size: 1668 bytes
Desc: not available
URL: <http://lists.llvm.org/pipermail/llvm-commits/attachments/20171221/515efb1a/attachment.bin>