[libc-commits] [libc] [libc] Add UTF-16 support in CharacterConverter [1/6] (PR #230434)
via libc-commits
libc-commits at lists.llvm.org
Wed Oct 14 02:02:51 PDT 2026
https://github.com/yangbowen updated https://github.com/llvm/llvm-project/pull/230434
>From 5693e91825a6e2b4f5b596406307d70ead6665df Mon Sep 17 00:00:00 2001
From: yangbowen <ybw1234567 at hotmail.com>
Date: Thu, 8 Oct 2026 20:23:01 +0000
Subject: [PATCH 1/4] [libc] Add UTF-16 support in CharacterConverter
---
libc/hdr/types/CMakeLists.txt | 11 +
libc/hdr/types/char16_t.h | 27 ++
libc/src/__support/wchar/CMakeLists.txt | 2 +
.../src/__support/wchar/character_converter.h | 183 +++++++++++++-
libc/test/src/__support/wchar/CMakeLists.txt | 26 ++
.../src/__support/wchar/utf16_to_8_test.cpp | 235 ++++++++++++++++++
.../src/__support/wchar/utf32_to_8_test.cpp | 26 +-
.../src/__support/wchar/utf8_to_16_test.cpp | 183 ++++++++++++++
.../src/__support/wchar/utf8_to_32_test.cpp | 139 +++++++++++
9 files changed, 816 insertions(+), 16 deletions(-)
create mode 100644 libc/hdr/types/char16_t.h
create mode 100644 libc/test/src/__support/wchar/utf16_to_8_test.cpp
create mode 100644 libc/test/src/__support/wchar/utf8_to_16_test.cpp
diff --git a/libc/hdr/types/CMakeLists.txt b/libc/hdr/types/CMakeLists.txt
index 92a2985b74be7..6ca38970729cf 100644
--- a/libc/hdr/types/CMakeLists.txt
+++ b/libc/hdr/types/CMakeLists.txt
@@ -9,6 +9,17 @@ add_proxy_header_library(
libc.include.uchar
)
+add_proxy_header_library(
+ char16_t
+ HDRS
+ char16_t.h
+ DEPENDS
+ libc.hdr.uchar_overlay
+ FULL_BUILD_DEPENDS
+ libc.include.llvm-libc-types.char16_t
+ libc.include.uchar
+)
+
add_proxy_header_library(
char32_t
HDRS
diff --git a/libc/hdr/types/char16_t.h b/libc/hdr/types/char16_t.h
new file mode 100644
index 0000000000000..a59d698243b09
--- /dev/null
+++ b/libc/hdr/types/char16_t.h
@@ -0,0 +1,27 @@
+//===-- Definition of char16_t.h ------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef LLVM_LIBC_HDR_TYPES_CHAR16_T_H
+#define LLVM_LIBC_HDR_TYPES_CHAR16_T_H
+
+#ifdef LIBC_FULL_BUILD
+
+#include "include/llvm-libc-types/char16_t.h"
+
+#else // overlay mode
+
+// MacOS doesn't provide uchar.h so we use the types provided by LLVM-libc.
+#ifdef __APPLE__
+#include "include/llvm-libc-types/char16_t.h"
+#else
+#include "hdr/uchar_overlay.h"
+#endif
+
+#endif // LIBC_FULL_BUILD
+
+#endif // LLVM_LIBC_HDR_TYPES_CHAR16_T_H
diff --git a/libc/src/__support/wchar/CMakeLists.txt b/libc/src/__support/wchar/CMakeLists.txt
index c4e7f0dbaa178..91fd788fd0762 100644
--- a/libc/src/__support/wchar/CMakeLists.txt
+++ b/libc/src/__support/wchar/CMakeLists.txt
@@ -29,11 +29,13 @@ add_header_library(
DEPENDS
libc.hdr.errno_macros
libc.hdr.types.char8_t
+ libc.hdr.types.char16_t
libc.hdr.types.char32_t
libc.hdr.types.size_t
libc.src.__support.common
libc.src.__support.CPP.type_traits
libc.src.__support.error_or
+ libc.src.__support.libc_assert
libc.src.__support.macros.config
libc.src.__support.macros.properties.types
libc.src.__support.math_extras
diff --git a/libc/src/__support/wchar/character_converter.h b/libc/src/__support/wchar/character_converter.h
index f754a602d2cce..f7ccf6f18b36c 100644
--- a/libc/src/__support/wchar/character_converter.h
+++ b/libc/src/__support/wchar/character_converter.h
@@ -10,6 +10,7 @@
#define LLVM_LIBC_SRC___SUPPORT_CHARACTER_CONVERTER_H
#include "hdr/errno_macros.h"
+#include "hdr/types/char16_t.h"
#include "hdr/types/char32_t.h"
#include "hdr/types/char8_t.h"
#include "hdr/types/size_t.h"
@@ -18,6 +19,7 @@
#include "src/__support/CPP/type_traits.h"
#include "src/__support/common.h"
#include "src/__support/error_or.h"
+#include "src/__support/libc_assert.h"
#include "src/__support/macros/properties/types.h"
#include "src/__support/math_extras.h"
#include "src/__support/wchar/mbstate.h"
@@ -57,18 +59,65 @@ class CharacterConverter {
LIBC_INLINE bool isEmpty() { return state->bytes_stored == 0; }
bool isValidState();
+ /*
+ * When converting between UTF-16 and UTF-8, the converter might be in a state
+ * of either partially-pushing or partially-popping. For example consider the
+ * following sequence:
+ * * push((char8_t)0xF0);
+ * * push((char8_t)0xA4);
+ * * [A](partial=0x24,bytes_stored=2,total_bytes=4)
+ * * push((char8_t)0xAD);
+ * * push((char8_t)0xA2);
+ * * (partial=0x24B62,bytes_stored=4,total_bytes=4)
+ * * pop_utf16(); -> 0xD852
+ * * [B](partial=0x24B62,bytes_stored=2,total_bytes=4)
+ * * pop_utf16(); -> 0xDF62
+ * At point [A] and point [B], the stored bytes would both be 2. But at [A] we
+ * must continue reading input, while at [B] we must write the output low
+ * surrogate and clear the state.
+ * Thus we must have some means to distinguish between pushing and popping.
+ * Notice that: When pushing, we do left-shifts on state->partial before each
+ * push. When popping, we keep state->partial at the complete code point.
+ * Since we disallow overlong encodings, the complete UTF-8 code point is
+ * always greater than
+ * * MAX_VALUE_PER_UTF8_LEN[state->total_bytes - 2]
+ * but at most
+ * * MAX_VALUE_PER_UTF8_LEN[state->total_bytes - 1]
+ * If we are partially through pushing, state->partial is at most
+ * * char32 >> 6
+ * thus at most
+ * * MAX_VALUE_PER_UTF8_LEN[state->total_bytes - 1] >> 6
+ * Observe that, an item in MAX_VALUE_PER_UTF8_LEN right-shifted by 6 is
+ * always less than the preceding item. Based on these facts, we can determine
+ * if we are partially through pushing or popping, by comparing state->partial
+ * with MAX_VALUE_PER_UTF8_LEN[state->total_bytes - 2]
+ * Also when converting UTF-16 -> UTF-8, in a partial state,
+ * * bytes_stored=2
+ * * total_bytes=4
+ * so the threshold 0xFFFF is also correct.
+ */
+ LIBC_INLINE bool isPartiallyPopping() {
+ if (isEmpty() || isFull())
+ return false;
+ // When neither empty nor full, state->total_bytes is at least 2
+ char32_t threshold = MAX_VALUE_PER_UTF8_LEN[state->total_bytes - 2];
+ return state->partial > threshold;
+ }
+
template <typename CharType> size_t sizeAs();
int push(char8_t utf8_byte);
+ int push(char16_t utf16);
int push(char32_t utf32);
ErrorOr<char8_t> pop_utf8();
+ ErrorOr<char16_t> pop_utf16();
ErrorOr<char32_t> pop_utf32();
-#if defined(LIBC_TYPES_WCHAR_T_IS_UTF32)
+#if defined(LIBC_TYPES_WCHAR_T_IS_UTF32) || defined(LIBC_TYPES_WCHAR_T_IS_UTF16)
int push(wchar_t wchar);
ErrorOr<wchar_t> pop_wchar();
-#endif // LIBC_TYPES_WCHAR_T_IS_UTF32
+#endif // LIBC_TYPES_WCHAR_T_IS_UTF32 || LIBC_TYPES_WCHAR_T_IS_UTF16
template <typename CharType> ErrorOr<CharType> pop();
};
@@ -112,6 +161,11 @@ LIBC_INLINE int CharacterConverter::push(char8_t utf8_byte) {
state->bytes_stored++;
return 0;
}
+ // Pushing when partially popping is a misuse in the library
+ if (isPartiallyPopping()) {
+ clear();
+ return -1;
+ }
// Any subsequent push
// Adding 6 more bits so need to left shift
if (num_ones == 1 && !isFull()) {
@@ -119,6 +173,19 @@ LIBC_INLINE int CharacterConverter::push(char8_t utf8_byte) {
state->partial = state->partial << ENCODED_BITS_PER_UTF8;
state->partial |= byte;
state->bytes_stored++;
+ if (isFull()) {
+ // Detect and reject overlong encodings
+ // Needed for isPartiallyPopping to work
+ if (state->partial <= MAX_VALUE_PER_UTF8_LEN[state->total_bytes - 2]) {
+ clear();
+ return EILSEQ;
+ }
+ // Also reject out-of-range code points
+ if (state->partial > MAX_VALUE_PER_UTF8_LEN[state->total_bytes - 1]) {
+ clear();
+ return EILSEQ;
+ }
+ }
return 0;
}
@@ -127,6 +194,57 @@ LIBC_INLINE int CharacterConverter::push(char8_t utf8_byte) {
return EILSEQ;
}
+LIBC_INLINE int CharacterConverter::push(char16_t utf16) {
+ // Checking for high surrogate if first push
+ if (isEmpty()) {
+ bool is_surrogate = (utf16 & 0xF800) == 0xD800;
+ if (is_surrogate) {
+ bool is_high_surrogate = (utf16 & 0x400) == 0;
+ if (!is_high_surrogate) {
+ // unpaired surrogate -> reset the state
+ // bytes_stored and total_bytes will always be 0 here
+ state->partial = static_cast<char32_t>(0);
+ return EILSEQ;
+ }
+ // high surrogate
+ // code point between 0x10000 and 0x10FFFF
+ state->total_bytes = 4;
+ state->bytes_stored = 2;
+ // 10 bits each from the high surrogate and the low surrogate, and the
+ // full code point is from 0x10000, so we add 0x40 to partial, which after
+ // shifting becomes 0x10000
+ state->partial = static_cast<char32_t>(utf16 & 0x3FF) + 0x40;
+ } else {
+ state->partial = static_cast<char32_t>(utf16);
+ // determine number of utf-8 bytes needed to represent this utf16 value
+ uint8_t i;
+ for (i = 0; i < 2; i++) {
+ if (state->partial <= MAX_VALUE_PER_UTF8_LEN[i])
+ break;
+ }
+ state->total_bytes = i + 1;
+ state->bytes_stored = i + 1;
+ }
+ return 0;
+ }
+ // Pushing when partially popping is a misuse in the library
+ if (isFull() || isPartiallyPopping()) {
+ clear();
+ return -1;
+ }
+ bool is_low_surrogate = (utf16 & 0xFC00) == 0xDC00;
+ if (!is_low_surrogate) {
+ // unpaired surrogate -> reset the state
+ clear();
+ return EILSEQ;
+ }
+ char32_t low = utf16 & 0x3FF;
+ state->partial <<= 10;
+ state->partial |= low;
+ state->bytes_stored += 2;
+ return 0;
+}
+
LIBC_INLINE int CharacterConverter::push(char32_t utf32) {
// we can't be partially through a conversion when pushing a utf32 value
if (!isEmpty())
@@ -160,8 +278,36 @@ LIBC_INLINE ErrorOr<char32_t> CharacterConverter::pop_utf32() {
return utf32;
}
+LIBC_INLINE ErrorOr<char16_t> CharacterConverter::pop_utf16() {
+ if (!isFull() && !isPartiallyPopping())
+ return Error(-1);
+ char32_t output;
+ if (state->total_bytes <= 3) {
+ // no surrogates needed
+ output = state->partial;
+ // disallow using UTF-8 or UTF-32 to encode unpaired surrogates
+ bool is_surrogate = (output & 0xF800) == 0xD800;
+ if (is_surrogate) {
+ clear();
+ return Error(EILSEQ);
+ }
+ clear();
+ } else {
+ if (isFull()) {
+ // high surrogate first
+ output = ((state->partial - 0x10000) >> 10) | 0xD800;
+ state->bytes_stored = 2;
+ } else {
+ // low surrogate
+ output = ((state->partial - 0x10000) & 0x3FF) | 0xDC00;
+ clear();
+ }
+ }
+ return static_cast<char16_t>(output);
+}
+
LIBC_INLINE ErrorOr<char8_t> CharacterConverter::pop_utf8() {
- if (isEmpty())
+ if (!isFull() && !isPartiallyPopping())
return Error(-1);
constexpr char8_t FIRST_BYTE_HEADERS[] = {0, 0xC0, 0xE0, 0xF0};
@@ -196,6 +342,10 @@ template <> LIBC_INLINE ErrorOr<char8_t> CharacterConverter::pop() {
return pop_utf8();
}
+template <> LIBC_INLINE ErrorOr<char16_t> CharacterConverter::pop() {
+ return pop_utf16();
+}
+
template <> LIBC_INLINE ErrorOr<char32_t> CharacterConverter::pop() {
return pop_utf32();
}
@@ -204,6 +354,10 @@ template <> LIBC_INLINE size_t CharacterConverter::sizeAs<char8_t>() {
return state->total_bytes;
}
+template <> LIBC_INLINE size_t CharacterConverter::sizeAs<char16_t>() {
+ return state->total_bytes > 3 ? 2 : 1;
+}
+
template <> LIBC_INLINE size_t CharacterConverter::sizeAs<char32_t>() {
return 1;
}
@@ -229,7 +383,28 @@ template <> LIBC_INLINE size_t CharacterConverter::sizeAs<wchar_t>() {
return sizeAs<char32_t>();
}
-#endif // LIBC_TYPES_WCHAR_T_IS_UTF32
+#elif defined(LIBC_TYPES_WCHAR_T_IS_UTF16)
+
+LIBC_INLINE int CharacterConverter::push(wchar_t wchar) {
+ return push(static_cast<char16_t>(wchar));
+}
+
+LIBC_INLINE ErrorOr<wchar_t> CharacterConverter::pop_wchar() {
+ ErrorOr<char16_t> Result = pop_utf16();
+ if (!Result)
+ return Error(Result.error());
+ return static_cast<wchar_t>(*Result);
+}
+
+template <> LIBC_INLINE ErrorOr<wchar_t> CharacterConverter::pop() {
+ return pop_wchar();
+}
+
+template <> LIBC_INLINE size_t CharacterConverter::sizeAs<wchar_t>() {
+ return sizeAs<char16_t>();
+}
+
+#endif // LIBC_TYPES_WCHAR_T_IS_UTF32 || LIBC_TYPES_WCHAR_T_IS_UTF16
} // namespace internal
} // namespace LIBC_NAMESPACE_DECL
diff --git a/libc/test/src/__support/wchar/CMakeLists.txt b/libc/test/src/__support/wchar/CMakeLists.txt
index 259e896ee835d..907950833dbcc 100644
--- a/libc/test/src/__support/wchar/CMakeLists.txt
+++ b/libc/test/src/__support/wchar/CMakeLists.txt
@@ -41,6 +41,32 @@ add_libc_test(
libc.src.__support.wchar.mbstate
)
+add_libc_test(
+ utf8_to_16_test
+ SUITE
+ libc-support-tests
+ SRCS
+ utf8_to_16_test.cpp
+ DEPENDS
+ libc.src.__support.error_or
+ libc.src.__support.macros.properties.types
+ libc.src.__support.wchar.character_converter
+ libc.src.__support.wchar.mbstate
+)
+
+add_libc_test(
+ utf16_to_8_test
+ SUITE
+ libc-support-tests
+ SRCS
+ utf16_to_8_test.cpp
+ DEPENDS
+ libc.src.__support.error_or
+ libc.src.__support.macros.properties.types
+ libc.src.__support.wchar.character_converter
+ libc.src.__support.wchar.mbstate
+)
+
add_libc_test(
wcsnrtombs_test
SUITE
diff --git a/libc/test/src/__support/wchar/utf16_to_8_test.cpp b/libc/test/src/__support/wchar/utf16_to_8_test.cpp
new file mode 100644
index 0000000000000..737f17595ce10
--- /dev/null
+++ b/libc/test/src/__support/wchar/utf16_to_8_test.cpp
@@ -0,0 +1,235 @@
+//===-- Unittests for the CharacterConverter class (utf16 -> 8) -----------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#include "src/__support/common.h"
+#include "src/__support/macros/properties/types.h"
+#include "src/__support/wchar/character_converter.h"
+#include "src/__support/wchar/mbstate.h"
+
+#include "test/UnitTest/Test.h"
+
+#if defined(LIBC_TYPES_WCHAR_T_IS_UTF16)
+using TestCharTypesUTF16 = LIBC_NAMESPACE::testing::TypeList<char16_t, wchar_t>;
+#else
+using TestCharTypesUTF16 = LIBC_NAMESPACE::testing::TypeList<char16_t>;
+#endif
+
+TYPED_TEST(LlvmLibcCharacterConverterUTF16To8Test, OneByte,
+ TestCharTypesUTF16) {
+ using CharType16 = ParamType;
+
+ LIBC_NAMESPACE::internal::mbstate state;
+ LIBC_NAMESPACE::internal::CharacterConverter cr(&state);
+ cr.clear();
+
+ // utf8 1-byte encodings are identical to their utf16 representations
+ CharType16 utf16_A = 0x41; // 'A'
+ cr.push(utf16_A);
+ ASSERT_TRUE(cr.isFull());
+ auto popped = cr.pop_utf8();
+ ASSERT_TRUE(popped.has_value());
+ ASSERT_EQ(static_cast<char>(popped.value()), 'A');
+ ASSERT_TRUE(cr.isEmpty());
+
+ CharType16 utf16_B = 0x42; // 'B'
+ cr.push(utf16_B);
+ ASSERT_TRUE(cr.isFull());
+ popped = cr.pop_utf8();
+ ASSERT_TRUE(popped.has_value());
+ ASSERT_EQ(static_cast<char>(popped.value()), 'B');
+ ASSERT_TRUE(cr.isEmpty());
+
+ // should error if we try to pop another utf8 byte out
+ popped = cr.pop_utf8();
+ ASSERT_FALSE(popped.has_value());
+}
+
+TYPED_TEST(LlvmLibcCharacterConverterUTF16To8Test, TwoByte,
+ TestCharTypesUTF16) {
+ using CharType16 = ParamType;
+
+ LIBC_NAMESPACE::internal::mbstate state;
+ LIBC_NAMESPACE::internal::CharacterConverter cr(&state);
+ cr.clear();
+
+ // testing utf16: 0xff -> utf8: 0xc3 0xbf
+ CharType16 utf16 = 0xff;
+ cr.push(utf16);
+ ASSERT_TRUE(cr.isFull());
+ auto popped = cr.pop_utf8();
+ ASSERT_TRUE(popped.has_value());
+ ASSERT_EQ(static_cast<int>(popped.value()), 0xc3);
+ ASSERT_TRUE(cr.isPartiallyPopping());
+ popped = cr.pop_utf8();
+ ASSERT_TRUE(popped.has_value());
+ ASSERT_EQ(static_cast<int>(popped.value()), 0xbf);
+ ASSERT_TRUE(cr.isEmpty());
+
+ // testing utf16: 0x58e -> utf8: 0xd6 0x8e
+ utf16 = 0x58e;
+ cr.push(utf16);
+ ASSERT_TRUE(cr.isFull());
+ popped = cr.pop_utf8();
+ ASSERT_TRUE(popped.has_value());
+ ASSERT_EQ(static_cast<int>(popped.value()), 0xd6);
+ ASSERT_TRUE(cr.isPartiallyPopping());
+ popped = cr.pop_utf8();
+ ASSERT_TRUE(popped.has_value());
+ ASSERT_EQ(static_cast<int>(popped.value()), 0x8e);
+ ASSERT_TRUE(cr.isEmpty());
+
+ // should error if we try to pop another utf8 byte out
+ popped = cr.pop_utf8();
+ ASSERT_FALSE(popped.has_value());
+}
+
+TYPED_TEST(LlvmLibcCharacterConverterUTF16To8Test, ThreeByte,
+ TestCharTypesUTF16) {
+ using CharType16 = ParamType;
+
+ LIBC_NAMESPACE::internal::mbstate state;
+ LIBC_NAMESPACE::internal::CharacterConverter cr(&state);
+ cr.clear();
+
+ // testing utf16: 0xac15 -> utf8: 0xea 0xb0 0x95
+ CharType16 utf16 = 0xac15;
+ cr.push(utf16);
+ ASSERT_TRUE(cr.isFull());
+ auto popped = cr.pop_utf8();
+ ASSERT_TRUE(popped.has_value());
+ ASSERT_EQ(static_cast<int>(popped.value()), 0xea);
+ ASSERT_TRUE(cr.isPartiallyPopping());
+ popped = cr.pop_utf8();
+ ASSERT_TRUE(popped.has_value());
+ ASSERT_EQ(static_cast<int>(popped.value()), 0xb0);
+ ASSERT_TRUE(cr.isPartiallyPopping());
+ popped = cr.pop_utf8();
+ ASSERT_TRUE(popped.has_value());
+ ASSERT_EQ(static_cast<int>(popped.value()), 0x95);
+ ASSERT_TRUE(cr.isEmpty());
+
+ // testing utf16: 0x267b -> utf8: 0xe2 0x99 0xbb
+ utf16 = 0x267b;
+ cr.push(utf16);
+ ASSERT_TRUE(cr.isFull());
+ popped = cr.pop_utf8();
+ ASSERT_TRUE(popped.has_value());
+ ASSERT_EQ(static_cast<int>(popped.value()), 0xe2);
+ ASSERT_TRUE(cr.isPartiallyPopping());
+ popped = cr.pop_utf8();
+ ASSERT_TRUE(popped.has_value());
+ ASSERT_EQ(static_cast<int>(popped.value()), 0x99);
+ ASSERT_TRUE(cr.isPartiallyPopping());
+ popped = cr.pop_utf8();
+ ASSERT_TRUE(popped.has_value());
+ ASSERT_EQ(static_cast<int>(popped.value()), 0xbb);
+ ASSERT_TRUE(cr.isEmpty());
+
+ // should error if we try to pop another utf8 byte out
+ popped = cr.pop_utf8();
+ ASSERT_FALSE(popped.has_value());
+}
+
+TYPED_TEST(LlvmLibcCharacterConverterUTF16To8Test, FourByte,
+ TestCharTypesUTF16) {
+ using CharType16 = ParamType;
+
+ LIBC_NAMESPACE::internal::mbstate state;
+ LIBC_NAMESPACE::internal::CharacterConverter cr(&state);
+ cr.clear();
+
+ // testing utf16: 0xd83e 0xdd21 -> utf8: 0xf0 0x9f 0xa4 0xa1
+ CharType16 utf16_h = 0xd83e;
+ cr.push(utf16_h);
+ ASSERT_FALSE(cr.isPartiallyPopping());
+ CharType16 utf16_l = 0xdd21;
+ cr.push(utf16_l);
+ ASSERT_TRUE(cr.isFull());
+ auto popped = cr.pop_utf8();
+ ASSERT_TRUE(popped.has_value());
+ ASSERT_EQ(static_cast<int>(popped.value()), 0xf0);
+ ASSERT_TRUE(cr.isPartiallyPopping());
+ popped = cr.pop_utf8();
+ ASSERT_TRUE(popped.has_value());
+ ASSERT_EQ(static_cast<int>(popped.value()), 0x9f);
+ ASSERT_TRUE(cr.isPartiallyPopping());
+ popped = cr.pop_utf8();
+ ASSERT_TRUE(popped.has_value());
+ ASSERT_EQ(static_cast<int>(popped.value()), 0xa4);
+ ASSERT_TRUE(cr.isPartiallyPopping());
+ popped = cr.pop_utf8();
+ ASSERT_TRUE(popped.has_value());
+ ASSERT_EQ(static_cast<int>(popped.value()), 0xa1);
+ ASSERT_TRUE(cr.isEmpty());
+
+ // testing utf16: 0xd808 0xdd21 -> utf8: 0xf0 0x92 0x84 0xa1
+ utf16_h = 0xd808;
+ cr.push(utf16_h);
+ ASSERT_FALSE(cr.isPartiallyPopping());
+ utf16_l = 0xdd21;
+ cr.push(utf16_l);
+ ASSERT_TRUE(cr.isFull());
+ popped = cr.pop_utf8();
+ ASSERT_TRUE(popped.has_value());
+ ASSERT_EQ(static_cast<int>(popped.value()), 0xf0);
+ ASSERT_TRUE(cr.isPartiallyPopping());
+ popped = cr.pop_utf8();
+ ASSERT_TRUE(popped.has_value());
+ ASSERT_EQ(static_cast<int>(popped.value()), 0x92);
+ ASSERT_TRUE(cr.isPartiallyPopping());
+ popped = cr.pop_utf8();
+ ASSERT_TRUE(popped.has_value());
+ ASSERT_EQ(static_cast<int>(popped.value()), 0x84);
+ ASSERT_TRUE(cr.isPartiallyPopping());
+ popped = cr.pop_utf8();
+ ASSERT_TRUE(popped.has_value());
+ ASSERT_EQ(static_cast<int>(popped.value()), 0xa1);
+ ASSERT_TRUE(cr.isEmpty());
+
+ // should error if we try to pop another utf8 byte out
+ popped = cr.pop_utf8();
+ ASSERT_FALSE(popped.has_value());
+}
+
+TYPED_TEST(LlvmLibcCharacterConverterUTF16To8Test, CantPushMidConversion,
+ TestCharTypesUTF16) {
+ using CharType16 = ParamType;
+
+ LIBC_NAMESPACE::internal::mbstate state;
+ LIBC_NAMESPACE::internal::CharacterConverter cr(&state);
+ cr.clear();
+
+ // testing utf16: 0xd808 0xdd21 -> utf8: 0xf0 0x92 0x84 0xa1
+ CharType16 utf16_h = 0xd808;
+ ASSERT_EQ(cr.push(utf16_h), 0);
+ CharType16 utf16_l = 0xdd21;
+ ASSERT_EQ(cr.push(utf16_l), 0);
+ auto popped = cr.pop_utf8();
+ ASSERT_TRUE(popped.has_value());
+
+ // can't push a utf16 without finishing popping the utf8 bytes out
+ int err = cr.push(utf16_l);
+ ASSERT_EQ(err, -1);
+}
+
+TYPED_TEST(LlvmLibcCharacterConverterUTF16To8Test, CantPopMidPushing,
+ TestCharTypesUTF16) {
+ using CharType16 = ParamType;
+
+ LIBC_NAMESPACE::internal::mbstate state;
+ LIBC_NAMESPACE::internal::CharacterConverter cr(&state);
+ cr.clear();
+
+ // testing utf16: 0xd808 0xdd21 -> utf8: 0xf0 0x92 0x84 0xa1
+ CharType16 utf16_h = 0xd808;
+ ASSERT_EQ(cr.push(utf16_h), 0);
+
+ // can't pop a utf8 without finishing pushing the utf16 code units
+ auto popped = cr.pop_utf8();
+ ASSERT_FALSE(popped.has_value());
+}
diff --git a/libc/test/src/__support/wchar/utf32_to_8_test.cpp b/libc/test/src/__support/wchar/utf32_to_8_test.cpp
index 52e10e1191175..bc911050cff35 100644
--- a/libc/test/src/__support/wchar/utf32_to_8_test.cpp
+++ b/libc/test/src/__support/wchar/utf32_to_8_test.cpp
@@ -64,7 +64,7 @@ TYPED_TEST(LlvmLibcCharacterConverterUTF32To8Test, TwoByte,
auto popped = cr.pop_utf8();
ASSERT_TRUE(popped.has_value());
ASSERT_EQ(static_cast<int>(popped.value()), 0xc3);
- ASSERT_TRUE(!cr.isEmpty());
+ ASSERT_TRUE(cr.isPartiallyPopping());
popped = cr.pop_utf8();
ASSERT_TRUE(popped.has_value());
ASSERT_EQ(static_cast<int>(popped.value()), 0xbf);
@@ -77,7 +77,7 @@ TYPED_TEST(LlvmLibcCharacterConverterUTF32To8Test, TwoByte,
popped = cr.pop_utf8();
ASSERT_TRUE(popped.has_value());
ASSERT_EQ(static_cast<int>(popped.value()), 0xd6);
- ASSERT_TRUE(!cr.isEmpty());
+ ASSERT_TRUE(cr.isPartiallyPopping());
popped = cr.pop_utf8();
ASSERT_TRUE(popped.has_value());
ASSERT_EQ(static_cast<int>(popped.value()), 0x8e);
@@ -103,11 +103,11 @@ TYPED_TEST(LlvmLibcCharacterConverterUTF32To8Test, ThreeByte,
auto popped = cr.pop_utf8();
ASSERT_TRUE(popped.has_value());
ASSERT_EQ(static_cast<int>(popped.value()), 0xea);
- ASSERT_TRUE(!cr.isEmpty());
+ ASSERT_TRUE(cr.isPartiallyPopping());
popped = cr.pop_utf8();
ASSERT_TRUE(popped.has_value());
ASSERT_EQ(static_cast<int>(popped.value()), 0xb0);
- ASSERT_TRUE(!cr.isEmpty());
+ ASSERT_TRUE(cr.isPartiallyPopping());
popped = cr.pop_utf8();
ASSERT_TRUE(popped.has_value());
ASSERT_EQ(static_cast<int>(popped.value()), 0x95);
@@ -120,11 +120,11 @@ TYPED_TEST(LlvmLibcCharacterConverterUTF32To8Test, ThreeByte,
popped = cr.pop_utf8();
ASSERT_TRUE(popped.has_value());
ASSERT_EQ(static_cast<int>(popped.value()), 0xe2);
- ASSERT_TRUE(!cr.isEmpty());
+ ASSERT_TRUE(cr.isPartiallyPopping());
popped = cr.pop_utf8();
ASSERT_TRUE(popped.has_value());
ASSERT_EQ(static_cast<int>(popped.value()), 0x99);
- ASSERT_TRUE(!cr.isEmpty());
+ ASSERT_TRUE(cr.isPartiallyPopping());
popped = cr.pop_utf8();
ASSERT_TRUE(popped.has_value());
ASSERT_EQ(static_cast<int>(popped.value()), 0xbb);
@@ -150,15 +150,15 @@ TYPED_TEST(LlvmLibcCharacterConverterUTF32To8Test, FourByte,
auto popped = cr.pop_utf8();
ASSERT_TRUE(popped.has_value());
ASSERT_EQ(static_cast<int>(popped.value()), 0xf0);
- ASSERT_TRUE(!cr.isEmpty());
+ ASSERT_TRUE(cr.isPartiallyPopping());
popped = cr.pop_utf8();
ASSERT_TRUE(popped.has_value());
ASSERT_EQ(static_cast<int>(popped.value()), 0x9f);
- ASSERT_TRUE(!cr.isEmpty());
+ ASSERT_TRUE(cr.isPartiallyPopping());
popped = cr.pop_utf8();
ASSERT_TRUE(popped.has_value());
ASSERT_EQ(static_cast<int>(popped.value()), 0xa4);
- ASSERT_TRUE(!cr.isEmpty());
+ ASSERT_TRUE(cr.isPartiallyPopping());
popped = cr.pop_utf8();
ASSERT_TRUE(popped.has_value());
ASSERT_EQ(static_cast<int>(popped.value()), 0xa1);
@@ -171,15 +171,15 @@ TYPED_TEST(LlvmLibcCharacterConverterUTF32To8Test, FourByte,
popped = cr.pop_utf8();
ASSERT_TRUE(popped.has_value());
ASSERT_EQ(static_cast<int>(popped.value()), 0xf0);
- ASSERT_TRUE(!cr.isEmpty());
+ ASSERT_TRUE(cr.isPartiallyPopping());
popped = cr.pop_utf8();
ASSERT_TRUE(popped.has_value());
ASSERT_EQ(static_cast<int>(popped.value()), 0x92);
- ASSERT_TRUE(!cr.isEmpty());
+ ASSERT_TRUE(cr.isPartiallyPopping());
popped = cr.pop_utf8();
ASSERT_TRUE(popped.has_value());
ASSERT_EQ(static_cast<int>(popped.value()), 0x84);
- ASSERT_TRUE(!cr.isEmpty());
+ ASSERT_TRUE(cr.isPartiallyPopping());
popped = cr.pop_utf8();
ASSERT_TRUE(popped.has_value());
ASSERT_EQ(static_cast<int>(popped.value()), 0xa1);
@@ -213,6 +213,8 @@ TEST(LlvmLibcCharacterConverterUTF32To8Test, InvalidState) {
LIBC_NAMESPACE::internal::mbstate s1;
LIBC_NAMESPACE::internal::CharacterConverter c1(&s1);
ASSERT_TRUE(c1.isValidState());
+ ASSERT_TRUE(c1.isEmpty());
+ ASSERT_FALSE(c1.isPartiallyPopping());
LIBC_NAMESPACE::internal::mbstate s2{0, 2, 0};
LIBC_NAMESPACE::internal::CharacterConverter c2(&s2);
diff --git a/libc/test/src/__support/wchar/utf8_to_16_test.cpp b/libc/test/src/__support/wchar/utf8_to_16_test.cpp
new file mode 100644
index 0000000000000..ed38ddb7596b8
--- /dev/null
+++ b/libc/test/src/__support/wchar/utf8_to_16_test.cpp
@@ -0,0 +1,183 @@
+//===-- Unittests for character_converter utf8->utf16 ---------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#include "hdr/errno_macros.h"
+#include "src/__support/error_or.h"
+#include "src/__support/macros/properties/types.h"
+#include "src/__support/wchar/character_converter.h"
+#include "src/__support/wchar/mbstate.h"
+#include "test/UnitTest/Test.h"
+
+#if defined(LIBC_TYPES_WCHAR_T_IS_UTF16)
+using TestCharTypesUTF16 = LIBC_NAMESPACE::testing::TypeList<char16_t, wchar_t>;
+#else
+using TestCharTypesUTF16 = LIBC_NAMESPACE::testing::TypeList<char16_t>;
+#endif
+
+TYPED_TEST(LlvmLibcCharacterConverterUTF8To16Test, OneByte,
+ TestCharTypesUTF16) {
+ using CharType16 = ParamType;
+
+ LIBC_NAMESPACE::internal::mbstate state;
+ state.bytes_stored = 0;
+ state.total_bytes = 0;
+ char ch = 'A';
+
+ LIBC_NAMESPACE::internal::CharacterConverter char_conv(&state);
+ int err = char_conv.push(static_cast<char8_t>(ch));
+ auto wch = char_conv.pop<CharType16>();
+
+ ASSERT_EQ(err, 0);
+ ASSERT_TRUE(wch.has_value());
+ ASSERT_EQ(static_cast<int>(wch.value()), 65);
+}
+
+TYPED_TEST(LlvmLibcCharacterConverterUTF8To16Test, TwoBytes,
+ TestCharTypesUTF16) {
+ using CharType16 = ParamType;
+
+ LIBC_NAMESPACE::internal::mbstate state;
+ state.bytes_stored = 0;
+ state.total_bytes = 0;
+ const char ch[2] = {static_cast<char>(0xC2),
+ static_cast<char>(0x8E)}; // car symbol
+
+ LIBC_NAMESPACE::internal::CharacterConverter char_conv(&state);
+ char_conv.push(static_cast<char8_t>(ch[0]));
+ ASSERT_FALSE(char_conv.isPartiallyPopping());
+ char_conv.push(static_cast<char8_t>(ch[1]));
+ auto wch = char_conv.pop<CharType16>();
+
+ ASSERT_TRUE(wch.has_value());
+ ASSERT_EQ(static_cast<int>(wch.value()), 142);
+}
+
+TYPED_TEST(LlvmLibcCharacterConverterUTF8To16Test, ThreeBytes,
+ TestCharTypesUTF16) {
+ using CharType16 = ParamType;
+
+ LIBC_NAMESPACE::internal::mbstate state;
+ state.bytes_stored = 0;
+ state.total_bytes = 0;
+ const char ch[3] = {static_cast<char>(0xE2), static_cast<char>(0x88),
+ static_cast<char>(0x91)}; // ∑ sigma symbol
+
+ LIBC_NAMESPACE::internal::CharacterConverter char_conv(&state);
+ char_conv.push(static_cast<char8_t>(ch[0]));
+ ASSERT_FALSE(char_conv.isPartiallyPopping());
+ char_conv.push(static_cast<char8_t>(ch[1]));
+ ASSERT_FALSE(char_conv.isPartiallyPopping());
+ char_conv.push(static_cast<char8_t>(ch[2]));
+ auto wch = char_conv.pop<CharType16>();
+
+ ASSERT_TRUE(wch.has_value());
+ ASSERT_EQ(static_cast<int>(wch.value()), 8721);
+}
+
+TYPED_TEST(LlvmLibcCharacterConverterUTF8To16Test, FourBytes,
+ TestCharTypesUTF16) {
+ using CharType16 = ParamType;
+
+ LIBC_NAMESPACE::internal::mbstate state;
+ state.bytes_stored = 0;
+ state.total_bytes = 0;
+ const char ch[4] = {static_cast<char>(0xF0), static_cast<char>(0x9F),
+ static_cast<char>(0xA4),
+ static_cast<char>(0xA1)}; // 🤡 clown emoji
+
+ LIBC_NAMESPACE::internal::CharacterConverter char_conv(&state);
+ char_conv.push(static_cast<char8_t>(ch[0]));
+ ASSERT_FALSE(char_conv.isPartiallyPopping());
+ char_conv.push(static_cast<char8_t>(ch[1]));
+ ASSERT_FALSE(char_conv.isPartiallyPopping());
+ char_conv.push(static_cast<char8_t>(ch[2]));
+ ASSERT_FALSE(char_conv.isPartiallyPopping());
+ char_conv.push(static_cast<char8_t>(ch[3]));
+ auto wch_h = char_conv.pop<CharType16>();
+ ASSERT_TRUE(char_conv.isPartiallyPopping());
+ auto wch_l = char_conv.pop<CharType16>();
+
+ ASSERT_TRUE(wch_h.has_value());
+ ASSERT_EQ(static_cast<int>(wch_h.value()), 55358);
+ ASSERT_TRUE(wch_l.has_value());
+ ASSERT_EQ(static_cast<int>(wch_l.value()), 56609);
+}
+
+TYPED_TEST(LlvmLibcCharacterConverterUTF8To16Test, ValidTwoByteWithExtraRead,
+ TestCharTypesUTF16) {
+ using CharType16 = ParamType;
+
+ LIBC_NAMESPACE::internal::mbstate state;
+ state.bytes_stored = 0;
+ state.total_bytes = 0;
+ const char ch[3] = {static_cast<char>(0xC2), static_cast<char>(0x8E),
+ static_cast<char>(0x80)};
+
+ LIBC_NAMESPACE::internal::CharacterConverter char_conv(&state);
+ int err = char_conv.push(static_cast<char8_t>(ch[0]));
+ ASSERT_EQ(err, 0);
+ err = char_conv.push(static_cast<char8_t>(ch[1]));
+ ASSERT_EQ(err, 0);
+ // Should produce an error on 3rd byte
+ err = char_conv.push(static_cast<char8_t>(ch[2]));
+ ASSERT_EQ(err, EILSEQ);
+
+ // Should produce an error since mbstate was reset
+ auto wch = char_conv.pop<CharType16>();
+ ASSERT_FALSE(wch.has_value());
+}
+
+TYPED_TEST(LlvmLibcCharacterConverterUTF8To16Test, TwoValidTwoBytes,
+ TestCharTypesUTF16) {
+ using CharType16 = ParamType;
+
+ LIBC_NAMESPACE::internal::mbstate state;
+ state.bytes_stored = 0;
+ state.total_bytes = 0;
+ const char ch[4] = {static_cast<char>(0xC2), static_cast<char>(0x8E),
+ static_cast<char>(0xC7), static_cast<char>(0x8C)};
+
+ LIBC_NAMESPACE::internal::CharacterConverter char_conv(&state);
+ int err = char_conv.push(static_cast<char8_t>(ch[0]));
+ ASSERT_EQ(err, 0);
+ err = char_conv.push(static_cast<char8_t>(ch[1]));
+ ASSERT_EQ(err, 0);
+ auto wch = char_conv.pop<CharType16>();
+ ASSERT_TRUE(wch.has_value());
+ ASSERT_EQ(static_cast<int>(wch.value()), 142);
+
+ // Second two byte character
+ err = char_conv.push(static_cast<char8_t>(ch[2]));
+ ASSERT_EQ(err, 0);
+ err = char_conv.push(static_cast<char8_t>(ch[3]));
+ ASSERT_EQ(err, 0);
+ wch = char_conv.pop<CharType16>();
+ ASSERT_TRUE(wch.has_value());
+ ASSERT_EQ(static_cast<int>(wch.value()), 460);
+}
+
+TYPED_TEST(LlvmLibcCharacterConverterUTF8To16Test, InvalidPop,
+ TestCharTypesUTF16) {
+ using CharType16 = ParamType;
+
+ LIBC_NAMESPACE::internal::mbstate state;
+ state.bytes_stored = 0;
+ state.total_bytes = 0;
+ LIBC_NAMESPACE::internal::CharacterConverter char_conv(&state);
+ const char ch[2] = {static_cast<char>(0xC2), static_cast<char>(0x8E)};
+ int err = char_conv.push(static_cast<char8_t>(ch[0]));
+ ASSERT_EQ(err, 0);
+ auto wch = char_conv.pop<CharType16>();
+ ASSERT_FALSE(
+ wch.has_value()); // Should fail since we have not read enough bytes
+ err = char_conv.push(static_cast<char8_t>(ch[1]));
+ ASSERT_EQ(err, 0);
+ wch = char_conv.pop<CharType16>();
+ ASSERT_TRUE(wch.has_value());
+ ASSERT_EQ(static_cast<int>(wch.value()), 142);
+}
diff --git a/libc/test/src/__support/wchar/utf8_to_32_test.cpp b/libc/test/src/__support/wchar/utf8_to_32_test.cpp
index db74cddafcbd7..8e72371ca7c06 100644
--- a/libc/test/src/__support/wchar/utf8_to_32_test.cpp
+++ b/libc/test/src/__support/wchar/utf8_to_32_test.cpp
@@ -49,6 +49,7 @@ TYPED_TEST(LlvmLibcCharacterConverterUTF8To32Test, TwoBytes,
LIBC_NAMESPACE::internal::CharacterConverter char_conv(&state);
char_conv.push(static_cast<char8_t>(ch[0]));
+ ASSERT_FALSE(char_conv.isPartiallyPopping());
char_conv.push(static_cast<char8_t>(ch[1]));
auto wch = char_conv.pop<CharType32>();
@@ -68,7 +69,9 @@ TYPED_TEST(LlvmLibcCharacterConverterUTF8To32Test, ThreeBytes,
LIBC_NAMESPACE::internal::CharacterConverter char_conv(&state);
char_conv.push(static_cast<char8_t>(ch[0]));
+ ASSERT_FALSE(char_conv.isPartiallyPopping());
char_conv.push(static_cast<char8_t>(ch[1]));
+ ASSERT_FALSE(char_conv.isPartiallyPopping());
char_conv.push(static_cast<char8_t>(ch[2]));
auto wch = char_conv.pop<CharType32>();
@@ -89,8 +92,11 @@ TYPED_TEST(LlvmLibcCharacterConverterUTF8To32Test, FourBytes,
LIBC_NAMESPACE::internal::CharacterConverter char_conv(&state);
char_conv.push(static_cast<char8_t>(ch[0]));
+ ASSERT_FALSE(char_conv.isPartiallyPopping());
char_conv.push(static_cast<char8_t>(ch[1]));
+ ASSERT_FALSE(char_conv.isPartiallyPopping());
char_conv.push(static_cast<char8_t>(ch[2]));
+ ASSERT_FALSE(char_conv.isPartiallyPopping());
char_conv.push(static_cast<char8_t>(ch[3]));
auto wch = char_conv.pop<CharType32>();
@@ -223,3 +229,136 @@ TYPED_TEST(LlvmLibcCharacterConverterUTF8To32Test, InvalidPop,
ASSERT_TRUE(wch.has_value());
ASSERT_EQ(static_cast<int>(wch.value()), 142);
}
+
+TYPED_TEST(LlvmLibcCharacterConverterUTF8To32Test, RejectOverlongTwoBytes,
+ TestCharTypesUTF32) {
+ using CharType32 = ParamType;
+
+ LIBC_NAMESPACE::internal::mbstate state;
+ state.bytes_stored = 0;
+ state.total_bytes = 0;
+ const char ch[4] = {static_cast<char>(0xC1), static_cast<char>(0xBF),
+ static_cast<char>(0xC2), static_cast<char>(0x80)};
+
+ LIBC_NAMESPACE::internal::CharacterConverter char_conv(&state);
+ // Overlong encoding should be rejected U+007F
+ int err = char_conv.push(static_cast<char8_t>(ch[0]));
+ ASSERT_EQ(err, 0);
+ err = char_conv.push(static_cast<char8_t>(ch[1]));
+ ASSERT_EQ(err, EILSEQ);
+
+ // Not overlong, don't reject U+0080
+ err = char_conv.push(static_cast<char8_t>(ch[2]));
+ ASSERT_EQ(err, 0);
+ err = char_conv.push(static_cast<char8_t>(ch[3]));
+ ASSERT_EQ(err, 0);
+ auto wch = char_conv.pop<CharType32>();
+ ASSERT_TRUE(wch.has_value());
+ ASSERT_EQ(static_cast<int>(wch.value()), 128);
+}
+
+TYPED_TEST(LlvmLibcCharacterConverterUTF8To32Test, RejectOverlongThreeBytes,
+ TestCharTypesUTF32) {
+ using CharType32 = ParamType;
+
+ LIBC_NAMESPACE::internal::mbstate state;
+ state.bytes_stored = 0;
+ state.total_bytes = 0;
+ const char ch[6] = {static_cast<char>(0xE0), static_cast<char>(0x9F),
+ static_cast<char>(0xBF), static_cast<char>(0xE0),
+ static_cast<char>(0xA0), static_cast<char>(0x80)};
+
+ LIBC_NAMESPACE::internal::CharacterConverter char_conv(&state);
+ // Overlong encoding should be rejected U+07FF
+ int err = char_conv.push(static_cast<char8_t>(ch[0]));
+ ASSERT_EQ(err, 0);
+ err = char_conv.push(static_cast<char8_t>(ch[1]));
+ ASSERT_EQ(err, 0);
+ err = char_conv.push(static_cast<char8_t>(ch[2]));
+ ASSERT_EQ(err, EILSEQ);
+
+ // Not overlong, don't reject U+0800
+ err = char_conv.push(static_cast<char8_t>(ch[3]));
+ ASSERT_EQ(err, 0);
+ err = char_conv.push(static_cast<char8_t>(ch[4]));
+ ASSERT_EQ(err, 0);
+ err = char_conv.push(static_cast<char8_t>(ch[5]));
+ ASSERT_EQ(err, 0);
+ auto wch = char_conv.pop<CharType32>();
+ ASSERT_TRUE(wch.has_value());
+ ASSERT_EQ(static_cast<int>(wch.value()), 2048);
+}
+
+TYPED_TEST(LlvmLibcCharacterConverterUTF8To32Test, RejectOverlongFourBytes,
+ TestCharTypesUTF32) {
+ using CharType32 = ParamType;
+
+ LIBC_NAMESPACE::internal::mbstate state;
+ state.bytes_stored = 0;
+ state.total_bytes = 0;
+ const char ch[8] = {static_cast<char>(0xF0), static_cast<char>(0x8F),
+ static_cast<char>(0xBF), static_cast<char>(0xBF),
+ static_cast<char>(0xF0), static_cast<char>(0x90),
+ static_cast<char>(0x80), static_cast<char>(0x80)};
+
+ LIBC_NAMESPACE::internal::CharacterConverter char_conv(&state);
+ // Overlong encoding should be rejected U+FFFF
+ int err = char_conv.push(static_cast<char8_t>(ch[0]));
+ ASSERT_EQ(err, 0);
+ err = char_conv.push(static_cast<char8_t>(ch[1]));
+ ASSERT_EQ(err, 0);
+ err = char_conv.push(static_cast<char8_t>(ch[2]));
+ ASSERT_EQ(err, 0);
+ err = char_conv.push(static_cast<char8_t>(ch[3]));
+ ASSERT_EQ(err, EILSEQ);
+
+ // Not overlong, don't reject U+10000
+ err = char_conv.push(static_cast<char8_t>(ch[4]));
+ ASSERT_EQ(err, 0);
+ err = char_conv.push(static_cast<char8_t>(ch[5]));
+ ASSERT_EQ(err, 0);
+ err = char_conv.push(static_cast<char8_t>(ch[6]));
+ ASSERT_EQ(err, 0);
+ err = char_conv.push(static_cast<char8_t>(ch[7]));
+ ASSERT_EQ(err, 0);
+ auto wch = char_conv.pop<CharType32>();
+ ASSERT_TRUE(wch.has_value());
+ ASSERT_EQ(static_cast<int>(wch.value()), 65536);
+}
+
+TYPED_TEST(LlvmLibcCharacterConverterUTF8To32Test, RejectOutOfRange,
+ TestCharTypesUTF32) {
+ using CharType32 = ParamType;
+
+ LIBC_NAMESPACE::internal::mbstate state;
+ state.bytes_stored = 0;
+ state.total_bytes = 0;
+ const char ch[8] = {static_cast<char>(0xF4), static_cast<char>(0x90),
+ static_cast<char>(0x80), static_cast<char>(0x80),
+ static_cast<char>(0xF4), static_cast<char>(0x8F),
+ static_cast<char>(0xBF), static_cast<char>(0xBF)};
+
+ LIBC_NAMESPACE::internal::CharacterConverter char_conv(&state);
+ // Out of range code point should be rejected U+110000
+ int err = char_conv.push(static_cast<char8_t>(ch[0]));
+ ASSERT_EQ(err, 0);
+ err = char_conv.push(static_cast<char8_t>(ch[1]));
+ ASSERT_EQ(err, 0);
+ err = char_conv.push(static_cast<char8_t>(ch[2]));
+ ASSERT_EQ(err, 0);
+ err = char_conv.push(static_cast<char8_t>(ch[3]));
+ ASSERT_EQ(err, EILSEQ);
+
+ // In range, don't reject U+10FFFF
+ err = char_conv.push(static_cast<char8_t>(ch[4]));
+ ASSERT_EQ(err, 0);
+ err = char_conv.push(static_cast<char8_t>(ch[5]));
+ ASSERT_EQ(err, 0);
+ err = char_conv.push(static_cast<char8_t>(ch[6]));
+ ASSERT_EQ(err, 0);
+ err = char_conv.push(static_cast<char8_t>(ch[7]));
+ ASSERT_EQ(err, 0);
+ auto wch = char_conv.pop<CharType32>();
+ ASSERT_TRUE(wch.has_value());
+ ASSERT_EQ(static_cast<int>(wch.value()), 1114111);
+}
>From 8336c8be163cbcd0bc46ee3bdd63c49b56129c27 Mon Sep 17 00:00:00 2001
From: yangbowen <ybw1234567 at hotmail.com>
Date: Wed, 14 Oct 2026 07:39:29 +0000
Subject: [PATCH 2/4] Eliminate magic numbers, reduce condition nesting, use
const variables
---
.../src/__support/wchar/character_converter.h | 74 +++++++++++--------
1 file changed, 43 insertions(+), 31 deletions(-)
diff --git a/libc/src/__support/wchar/character_converter.h b/libc/src/__support/wchar/character_converter.h
index f7ccf6f18b36c..0c8948e153340 100644
--- a/libc/src/__support/wchar/character_converter.h
+++ b/libc/src/__support/wchar/character_converter.h
@@ -44,6 +44,17 @@ class CharacterConverter {
0x10ffff};
static constexpr int MAX_UTF8_LENGTH = 4;
+ static constexpr char16_t UTF16_SURROGATE_START = 0xD800;
+ static constexpr char16_t UTF16_SURROGATE_END = 0xDFFF;
+ static constexpr char16_t UTF16_HIGH_SURROGATE_START = 0xD800;
+ static constexpr char16_t UTF16_HIGH_SURROGATE_END = 0xDBFF;
+ static constexpr char16_t UTF16_LOW_SURROGATE_START = 0xDC00;
+ static constexpr char16_t UTF16_LOW_SURROGATE_END = 0xDFFF;
+ static constexpr char32_t UTF16_SURROGATE_ENCODED_BASE = 0x10000;
+ static constexpr size_t ENCODED_BITS_PER_UTF16_SURROGATE = 10;
+ static constexpr uint16_t MASK_ENCODED_BITS_UTF16_SURROGATE =
+ mask_trailing_ones<uint16_t, ENCODED_BITS_PER_UTF16_SURROGATE>();
+
public:
explicit LIBC_INLINE CharacterConverter(mbstate *state_ptr)
: state(state_ptr) {}
@@ -100,7 +111,7 @@ class CharacterConverter {
if (isEmpty() || isFull())
return false;
// When neither empty nor full, state->total_bytes is at least 2
- char32_t threshold = MAX_VALUE_PER_UTF8_LEN[state->total_bytes - 2];
+ const char32_t threshold = MAX_VALUE_PER_UTF8_LEN[state->total_bytes - 2];
return state->partial > threshold;
}
@@ -134,7 +145,7 @@ LIBC_INLINE bool CharacterConverter::isValidState() {
}
LIBC_INLINE int CharacterConverter::push(char8_t utf8_byte) {
- uint8_t num_ones = static_cast<uint8_t>(cpp::countl_one(utf8_byte));
+ const uint8_t num_ones = static_cast<uint8_t>(cpp::countl_one(utf8_byte));
// Checking the first byte if first push
if (isEmpty()) {
// UTF-8 char has 1 byte total
@@ -146,7 +157,7 @@ LIBC_INLINE int CharacterConverter::push(char8_t utf8_byte) {
/* Since the format is 110xxxxx, 1110xxxx, and 11110xxx for 2, 3, and 4,
we will make the base mask with 7 ones and right shift it as necessary. */
constexpr size_t SIGNIFICANT_BITS = 7;
- char8_t base_mask =
+ constexpr char8_t base_mask =
static_cast<char8_t>(mask_trailing_ones<uint8_t, SIGNIFICANT_BITS>());
state->total_bytes = num_ones;
utf8_byte &= (base_mask >> num_ones);
@@ -162,14 +173,11 @@ LIBC_INLINE int CharacterConverter::push(char8_t utf8_byte) {
return 0;
}
// Pushing when partially popping is a misuse in the library
- if (isPartiallyPopping()) {
- clear();
- return -1;
- }
+ LIBC_ASSERT(!isPartiallyPopping());
// Any subsequent push
// Adding 6 more bits so need to left shift
if (num_ones == 1 && !isFull()) {
- char32_t byte = utf8_byte & MASK_ENCODED_BITS;
+ const char32_t byte = utf8_byte & MASK_ENCODED_BITS;
state->partial = state->partial << ENCODED_BITS_PER_UTF8;
state->partial |= byte;
state->bytes_stored++;
@@ -195,25 +203,28 @@ LIBC_INLINE int CharacterConverter::push(char8_t utf8_byte) {
}
LIBC_INLINE int CharacterConverter::push(char16_t utf16) {
+ const bool is_high_surrogate =
+ utf16 >= UTF16_HIGH_SURROGATE_START && utf16 <= UTF16_HIGH_SURROGATE_END;
+ const bool is_low_surrogate =
+ utf16 >= UTF16_LOW_SURROGATE_START && utf16 <= UTF16_LOW_SURROGATE_END;
// Checking for high surrogate if first push
if (isEmpty()) {
- bool is_surrogate = (utf16 & 0xF800) == 0xD800;
- if (is_surrogate) {
- bool is_high_surrogate = (utf16 & 0x400) == 0;
- if (!is_high_surrogate) {
- // unpaired surrogate -> reset the state
- // bytes_stored and total_bytes will always be 0 here
- state->partial = static_cast<char32_t>(0);
- return EILSEQ;
- }
+ if (is_low_surrogate) {
+ // unpaired surrogate -> reset the state
+ // bytes_stored and total_bytes will always be 0 here
+ state->partial = static_cast<char32_t>(0);
+ return EILSEQ;
+ }
+ if (is_high_surrogate) {
// high surrogate
// code point between 0x10000 and 0x10FFFF
state->total_bytes = 4;
state->bytes_stored = 2;
- // 10 bits each from the high surrogate and the low surrogate, and the
- // full code point is from 0x10000, so we add 0x40 to partial, which after
- // shifting becomes 0x10000
- state->partial = static_cast<char32_t>(utf16 & 0x3FF) + 0x40;
+ constexpr char32_t ENCODED_BASE_PRESHIFT =
+ UTF16_SURROGATE_ENCODED_BASE >> ENCODED_BITS_PER_UTF16_SURROGATE;
+ state->partial =
+ static_cast<char32_t>(utf16 & MASK_ENCODED_BITS_UTF16_SURROGATE) +
+ ENCODED_BASE_PRESHIFT;
} else {
state->partial = static_cast<char32_t>(utf16);
// determine number of utf-8 bytes needed to represent this utf16 value
@@ -228,17 +239,13 @@ LIBC_INLINE int CharacterConverter::push(char16_t utf16) {
return 0;
}
// Pushing when partially popping is a misuse in the library
- if (isFull() || isPartiallyPopping()) {
- clear();
- return -1;
- }
- bool is_low_surrogate = (utf16 & 0xFC00) == 0xDC00;
+ LIBC_ASSERT(!isFull() && !isPartiallyPopping());
if (!is_low_surrogate) {
// unpaired surrogate -> reset the state
clear();
return EILSEQ;
}
- char32_t low = utf16 & 0x3FF;
+ const char32_t low = utf16 & MASK_ENCODED_BITS_UTF16_SURROGATE;
state->partial <<= 10;
state->partial |= low;
state->bytes_stored += 2;
@@ -272,7 +279,7 @@ LIBC_INLINE ErrorOr<char32_t> CharacterConverter::pop_utf32() {
// whether enough bytes have been pushed
if (!isFull())
return Error(-1);
- char32_t utf32 = state->partial;
+ const char32_t utf32 = state->partial;
// reset if successful pop
clear();
return utf32;
@@ -286,7 +293,8 @@ LIBC_INLINE ErrorOr<char16_t> CharacterConverter::pop_utf16() {
// no surrogates needed
output = state->partial;
// disallow using UTF-8 or UTF-32 to encode unpaired surrogates
- bool is_surrogate = (output & 0xF800) == 0xD800;
+ const bool is_surrogate =
+ output >= UTF16_SURROGATE_START && output <= UTF16_SURROGATE_END;
if (is_surrogate) {
clear();
return Error(EILSEQ);
@@ -295,11 +303,15 @@ LIBC_INLINE ErrorOr<char16_t> CharacterConverter::pop_utf16() {
} else {
if (isFull()) {
// high surrogate first
- output = ((state->partial - 0x10000) >> 10) | 0xD800;
+ output = ((state->partial - UTF16_SURROGATE_ENCODED_BASE) >>
+ ENCODED_BITS_PER_UTF16_SURROGATE) |
+ UTF16_HIGH_SURROGATE_START;
state->bytes_stored = 2;
} else {
// low surrogate
- output = ((state->partial - 0x10000) & 0x3FF) | 0xDC00;
+ output = ((state->partial - UTF16_SURROGATE_ENCODED_BASE) &
+ MASK_ENCODED_BITS_UTF16_SURROGATE) |
+ UTF16_LOW_SURROGATE_START;
clear();
}
}
>From 4afed0f3fc1eded276bdc073377f892c8801f860 Mon Sep 17 00:00:00 2001
From: yangbowen <ybw1234567 at hotmail.com>
Date: Wed, 14 Oct 2026 07:48:35 +0000
Subject: [PATCH 3/4] Simplify loop
---
libc/src/__support/wchar/character_converter.h | 6 ++----
1 file changed, 2 insertions(+), 4 deletions(-)
diff --git a/libc/src/__support/wchar/character_converter.h b/libc/src/__support/wchar/character_converter.h
index 0c8948e153340..3e2c55ee9a356 100644
--- a/libc/src/__support/wchar/character_converter.h
+++ b/libc/src/__support/wchar/character_converter.h
@@ -229,10 +229,8 @@ LIBC_INLINE int CharacterConverter::push(char16_t utf16) {
state->partial = static_cast<char32_t>(utf16);
// determine number of utf-8 bytes needed to represent this utf16 value
uint8_t i;
- for (i = 0; i < 2; i++) {
- if (state->partial <= MAX_VALUE_PER_UTF8_LEN[i])
- break;
- }
+ for (i = 0; i < 2 && MAX_VALUE_PER_UTF8_LEN[i] < state->partial; i++)
+ ;
state->total_bytes = i + 1;
state->bytes_stored = i + 1;
}
>From 68eff96a3e4b13d9e4b6224575a5982cd73142ad Mon Sep 17 00:00:00 2001
From: yangbowen <ybw1234567 at hotmail.com>
Date: Wed, 14 Oct 2026 09:02:24 +0000
Subject: [PATCH 4/4] File headers
---
libc/hdr/types/char16_t.h | 7 ++++++-
libc/test/src/__support/wchar/utf16_to_8_test.cpp | 7 ++++++-
libc/test/src/__support/wchar/utf8_to_16_test.cpp | 7 ++++++-
3 files changed, 18 insertions(+), 3 deletions(-)
diff --git a/libc/hdr/types/char16_t.h b/libc/hdr/types/char16_t.h
index a59d698243b09..04b1318d0e571 100644
--- a/libc/hdr/types/char16_t.h
+++ b/libc/hdr/types/char16_t.h
@@ -1,10 +1,15 @@
-//===-- Definition of char16_t.h ------------------------------------------===//
+//===----------------------------------------------------------------------===//
//
// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
// See https://llvm.org/LICENSE.txt for license information.
// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
//
//===----------------------------------------------------------------------===//
+///
+/// \file
+/// Proxy header for char16_t.
+///
+//===----------------------------------------------------------------------===//
#ifndef LLVM_LIBC_HDR_TYPES_CHAR16_T_H
#define LLVM_LIBC_HDR_TYPES_CHAR16_T_H
diff --git a/libc/test/src/__support/wchar/utf16_to_8_test.cpp b/libc/test/src/__support/wchar/utf16_to_8_test.cpp
index 737f17595ce10..fbe7f5689e18a 100644
--- a/libc/test/src/__support/wchar/utf16_to_8_test.cpp
+++ b/libc/test/src/__support/wchar/utf16_to_8_test.cpp
@@ -1,10 +1,15 @@
-//===-- Unittests for the CharacterConverter class (utf16 -> 8) -----------===//
+//===----------------------------------------------------------------------===//
//
// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
// See https://llvm.org/LICENSE.txt for license information.
// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
//
//===----------------------------------------------------------------------===//
+///
+/// \file
+/// Unittests for CharacterConverter utf16->utf8
+///
+//===----------------------------------------------------------------------===//
#include "src/__support/common.h"
#include "src/__support/macros/properties/types.h"
diff --git a/libc/test/src/__support/wchar/utf8_to_16_test.cpp b/libc/test/src/__support/wchar/utf8_to_16_test.cpp
index ed38ddb7596b8..fc5b3312fd956 100644
--- a/libc/test/src/__support/wchar/utf8_to_16_test.cpp
+++ b/libc/test/src/__support/wchar/utf8_to_16_test.cpp
@@ -1,10 +1,15 @@
-//===-- Unittests for character_converter utf8->utf16 ---------------------===//
+//===----------------------------------------------------------------------===//
//
// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
// See https://llvm.org/LICENSE.txt for license information.
// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
//
//===----------------------------------------------------------------------===//
+///
+/// \file
+/// Unittests for CharacterConverter utf8->utf16
+///
+//===----------------------------------------------------------------------===//
#include "hdr/errno_macros.h"
#include "src/__support/error_or.h"
More information about the libc-commits
mailing list