[PATCH] [Support] Add a Unicode conversion wrapper from UTF16 to UTF8

[Reid Kleckner]

Hi aaron.ballman,

This is to support parsing UTF16 response files in LLVM/lib/Option for
lld and clang.

http://llvm-reviews.chandlerc.com/D1138

Files:
  include/llvm/Support/ConvertUTF.h
  lib/Support/ConvertUTFWrapper.cpp
  unittests/Support/CMakeLists.txt
  unittests/Support/ConvertUTFTest.cpp

Index: include/llvm/Support/ConvertUTF.h
===================================================================
--- include/llvm/Support/ConvertUTF.h
+++ include/llvm/Support/ConvertUTF.h
@@ -112,6 +112,9 @@
 
 #define UNI_MAX_UTF8_BYTES_PER_CODE_POINT 4
 
+#define UNI_UTF16_BYTE_ORDER_MARK_NATIVE  0xFEFF
+#define UNI_UTF16_BYTE_ORDER_MARK_SWAPPED 0xFFFE
+
 typedef enum {
   conversionOK,           /* conversion successful */
   sourceExhausted,        /* partial character in source, but hit end */
@@ -219,6 +222,13 @@
     return sourceExhausted;
   return ConvertUTF8toUTF32(source, *source + size, &target, target + 1, flags);
 }
+
+/**
+ * Converts a stream of raw bytes assumed to be UTF16 into a UTF8 std::string.
+ */
+bool convertUTF16ToUTF8String(const char *Src, const char *SrcEnd,
+                              std::string &Out);
+
 } /* end namespace llvm */
 
 #endif
Index: lib/Support/ConvertUTFWrapper.cpp
===================================================================
--- lib/Support/ConvertUTFWrapper.cpp
+++ lib/Support/ConvertUTFWrapper.cpp
@@ -8,6 +8,9 @@
 //===----------------------------------------------------------------------===//
 
 #include "llvm/Support/ConvertUTF.h"
+#include "llvm/Support/SwapByteOrder.h"
+#include <string>
+#include <vector>
 
 namespace llvm {
 
@@ -72,5 +75,51 @@
   return true;
 }
 
+bool convertUTF16ToUTF8String(const char *SrcBytes, const char *SrcBytesEnd,
+                              std::string &Out) {
+  assert(Out.empty());
+
+  // Error out on an uneven byte count.
+  if ((SrcBytesEnd - SrcBytes) % 2)
+    return false;
+
+  const UTF16 *Src = reinterpret_cast<const UTF16 *>(SrcBytes);
+  const UTF16 *SrcEnd = reinterpret_cast<const UTF16 *>(SrcBytesEnd);
+
+  if (Src - SrcEnd == 0)
+    return true;
+
+  // Byteswap if necessary.
+  std::vector<UTF16> ByteSwapped;
+  if (Src[0] == UNI_UTF16_BYTE_ORDER_MARK_SWAPPED) {
+    ByteSwapped.insert(ByteSwapped.end(), Src, SrcEnd);
+    for (unsigned I = 0, E = ByteSwapped.size(); I != E; ++I)
+      ByteSwapped[I] = llvm::sys::SwapByteOrder_16(ByteSwapped[I]);
+    Src = &ByteSwapped[0];
+    SrcEnd = &ByteSwapped[ByteSwapped.size() - 1] + 1;
+  }
+
+  // Skip the BOM for conversion.
+  if (Src[0] == UNI_UTF16_BYTE_ORDER_MARK_NATIVE)
+    Src++;
+
+  // Just allocate enough space up front.  We'll shrink it later.
+  Out.resize((SrcEnd - Src) * UNI_MAX_UTF8_BYTES_PER_CODE_POINT);
+  UTF8 *Dst = reinterpret_cast<UTF8 *>(&Out[0]);
+  UTF8 *DstEnd = Dst + Out.size();
+
+  ConversionResult CR =
+      ConvertUTF16toUTF8(&Src, SrcEnd, &Dst, DstEnd, strictConversion);
+  assert(CR != targetExhausted);
+
+  if (CR != conversionOK) {
+    Out.clear();
+    return false;
+  }
+
+  Out.resize(reinterpret_cast<char *>(Dst) - &Out[0]);
+  return true;
+}
+
 } // end namespace llvm
 
Index: unittests/Support/CMakeLists.txt
===================================================================
--- unittests/Support/CMakeLists.txt
+++ unittests/Support/CMakeLists.txt
@@ -9,6 +9,7 @@
   ArrayRecyclerTest.cpp
   BlockFrequencyTest.cpp
   Casting.cpp
+  ConvertUTFTest.cpp
   CommandLineTest.cpp
   CompressionTest.cpp
   ConstantRangeTest.cpp
Index: unittests/Support/ConvertUTFTest.cpp
===================================================================
--- /dev/null
+++ unittests/Support/ConvertUTFTest.cpp
@@ -0,0 +1,51 @@
+//===- llvm/unittest/Support/ConvertUTFTest.cpp - ConvertUTF tests --------===//
+//
+//                     The LLVM Compiler Infrastructure
+//
+// This file is distributed under the University of Illinois Open Source
+// License. See LICENSE.TXT for details.
+//
+//===----------------------------------------------------------------------===//
+
+#include "llvm/Support/ConvertUTF.h"
+#include "gtest/gtest.h"
+#include <string>
+
+TEST(ConvertUTFTest, ConvertUTF16LittleEndianToUTF8String) {
+  // Src is the look of disapproval.
+  static const char Src[] = "\xff\xfe\xa0\x0c_\x00\xa0\x0c";
+  const char *SrcEnd = Src + sizeof(Src) - 1;
+  std::string Result;
+  bool Success = llvm::convertUTF16ToUTF8String(Src, SrcEnd, Result);
+  EXPECT_TRUE(Success);
+  std::string Expected("\xe0\xb2\xa0_\xe0\xb2\xa0");
+  EXPECT_EQ(Expected, Result);
+}
+
+TEST(ConvertUTFTest, ConvertUTF16BigEndianToUTF8String) {
+  // Src is the look of disapproval.
+  static const char Src[] = "\xfe\xff\x0c\xa0\x00_\x0c\xa0";
+  const char *SrcEnd = Src + sizeof(Src) - 1;
+  std::string Result;
+  bool Success = llvm::convertUTF16ToUTF8String(Src, SrcEnd, Result);
+  EXPECT_TRUE(Success);
+  std::string Expected("\xe0\xb2\xa0_\xe0\xb2\xa0");
+  EXPECT_EQ(Expected, Result);
+}
+
+TEST(ConvertUTFTest, OddLengthInput) {
+  static const char Src[6] = "xxxxx";
+  const char *SrcEnd = Src + 5;
+  std::string Result;
+  bool Success = llvm::convertUTF16ToUTF8String(Src, SrcEnd, Result);
+  EXPECT_FALSE(Success);
+}
+
+TEST(ConvertUTFTest, Empty) {
+  static const char Src[] = "";
+  const char *SrcEnd = Src;
+  std::string Result;
+  bool Success = llvm::convertUTF16ToUTF8String(Src, SrcEnd, Result);
+  EXPECT_TRUE(Success);
+  EXPECT_TRUE(Result.empty());
+}
-------------- next part --------------
A non-text attachment was scrubbed...
Name: D1138.1.patch
Type: text/x-patch
Size: 5115 bytes
Desc: not available
URL: <http://lists.llvm.org/pipermail/llvm-commits/attachments/20130715/0596da57/attachment.bin>