[libc-commits] [libc] [libc] Implemented CharacterConverter push/pop for utf32->utf8 conversions (PR #143971)

Michael Jones via libc-commits libc-commits at lists.llvm.org
Thu Jun 12 15:28:05 PDT 2025


================
@@ -22,13 +23,134 @@ bool CharacterConverter::isComplete() {
   return state->bytes_processed == state->total_bytes;
 }
 
-int CharacterConverter::push(char8_t utf8_byte) {}
+int CharacterConverter::push(char32_t utf32) {
+  state->partial = utf32;
+  state->bytes_processed = 0;
+  state->total_bytes = 0;
 
-int CharacterConverter::push(char32_t utf32) {}
+  // determine number of utf-8 bytes needed to represent this utf32 value
+  char32_t ranges[] = {0x7f, 0x7ff, 0xffff, 0x10ffff};
+  const int num_ranges = 4;
+  for (uint8_t i = 0; i < num_ranges; i++) {
+    if (state->partial <= ranges[i]) {
+      state->total_bytes = i + 1;
+      break;
+    }
+  }
+  if (state->total_bytes == 0) {
+    return -1;
+  }
 
-utf_ret<char8_t> CharacterConverter::pop_utf8() {}
+  return 0;
+}
+
+utf_ret<char8_t> CharacterConverter::pop_utf8_seqlength1() {
+  utf_ret<char8_t> result;
+  result.error = 0;
+
+  // 0xxxxxxx
+  switch (state->bytes_processed) {
+  case 0:
+    result.out = (char8_t)(state->partial);
+    break;
+  default:
+    result.error = -1;
+    return result;
+  }
+
+  state->bytes_processed++;
+  return result;
+}
+
+utf_ret<char8_t> CharacterConverter::pop_utf8_seqlength2() {
+  utf_ret<char8_t> result;
+  result.error = 0;
+
+  // 110xxxxx 10xxxxxx
+  char32_t utf32 = state->partial;
+  switch (state->bytes_processed) {
+  case 0:
+    result.out = (char8_t)(0xC0 | (utf32 >> 6));
+    break;
+  case 1:
+    result.out = (char8_t)(0x80 | (utf32 & 0x3f));
+    break;
+  default:
+    result.error = -1;
+    return result;
+  }
+
+  state->bytes_processed++;
+  return result;
+}
+
+utf_ret<char8_t> CharacterConverter::pop_utf8_seqlength3() {
+  utf_ret<char8_t> result;
+  result.error = 0;
 
-utf_ret<char32_t> CharacterConverter::pop_utf32() {}
+  // 1110xxxx 10xxxxxx 10xxxxxx
+  char32_t utf32 = state->partial;
+  switch (state->bytes_processed) {
+  case 0:
+    result.out = (char8_t)(0xE0 | (utf32 >> 12));
+    break;
+  case 1:
+    result.out = (char8_t)(0x80 | ((utf32 >> 6) & 0x3f));
+    break;
+  case 2:
+    result.out = (char8_t)(0x80 | (utf32 & 0x3f));
+    break;
+  default:
+    result.error = -1;
+    return result;
+  }
+
+  state->bytes_processed++;
+  return result;
+}
+
+utf_ret<char8_t> CharacterConverter::pop_utf8_seqlength4() {
+  utf_ret<char8_t> result;
+  result.error = 0;
+
+  // 11110xxx 10xxxxxx 10xxxxxx 10xxxxxx
+  char32_t utf32 = state->partial;
+  switch (state->bytes_processed) {
+  case 0:
+    result.out = (char8_t)(0xF0 | (utf32 >> 18));
+    break;
+  case 1:
+    result.out = (char8_t)(0x80 | ((utf32 >> 12) & 0x3f));
+    break;
+  case 2:
+    result.out = (char8_t)(0x80 | ((utf32 >> 6) & 0x3f));
+    break;
+  case 3:
+    result.out = (char8_t)(0x80 | (utf32 & 0x3f));
+    break;
+  default:
+    result.error = -1;
+    return result;
+  }
+
+  state->bytes_processed++;
+  return result;
+}
----------------
michaelrj-google wrote:

I'm pretty sure it's feasible to collapse these all down into one function. It seems like for the first char you're always shifting `(total_bytes - 1) * 6` bits, then adding `{0, 2, 3, 4}` leading 1s. For all the others you're shifting `(3 - bytes_processed) * 6` bits then using the same mask.

https://github.com/llvm/llvm-project/pull/143971


More information about the libc-commits mailing list