[libc-commits] [libc] [libc] Add UTF-16 support in CharacterConverter [1/6] (PR #230434)

[Michael Jones via libc-commits]

================
@@ -127,6 +194,57 @@ LIBC_INLINE int CharacterConverter::push(char8_t utf8_byte) {
   return EILSEQ;
 }
 
+LIBC_INLINE int CharacterConverter::push(char16_t utf16) {
+  // Checking for high surrogate if first push
+  if (isEmpty()) {
+    bool is_surrogate = (utf16 & 0xF800) == 0xD800;
+    if (is_surrogate) {
+      bool is_high_surrogate = (utf16 & 0x400) == 0;
+      if (!is_high_surrogate) {
+        // unpaired surrogate -> reset the state
+        // bytes_stored and total_bytes will always be 0 here
+        state->partial = static_cast<char32_t>(0);
+        return EILSEQ;
+      }
+      // high surrogate
+      // code point between 0x10000 and 0x10FFFF
+      state->total_bytes = 4;
+      state->bytes_stored = 2;
+      // 10 bits each from the high surrogate and the low surrogate, and the
+      // full code point is from 0x10000, so we add 0x40 to partial, which after
+      // shifting becomes 0x10000
+      state->partial = static_cast<char32_t>(utf16 & 0x3FF) + 0x40;
+    } else {
+      state->partial = static_cast<char32_t>(utf16);
+      // determine number of utf-8 bytes needed to represent this utf16 value
+      uint8_t i;
+      for (i = 0; i < 2; i++) {
+        if (state->partial <= MAX_VALUE_PER_UTF8_LEN[i])
+          break;
+      }
+      state->total_bytes = i + 1;
+      state->bytes_stored = i + 1;
+    }
+    return 0;
+  }
+  // Pushing when partially popping is a misuse in the library
+  if (isFull() || isPartiallyPopping()) {
+    clear();
+    return -1;
+  }
----------------
michaelrj-google wrote:

same as above, this should be a `LIBC_ASSERT`

https://github.com/llvm/llvm-project/pull/230434