[libc-commits] [libc] [libc] Add UTF-16 support in CharacterConverter (PR #230434)
via libc-commits
libc-commits at lists.llvm.org
Fri Oct 9 03:05:06 PDT 2026
llvmorg-github-actions[bot] wrote:
<!--LLVM PR SUMMARY COMMENT-->
@llvm/pr-subscribers-libc
Author: yangbowen
<details>
<summary>Changes</summary>
This pull request adds support of UTF-16 in `CharacterConverter`. It does not use the functionality yet. Follow-ups will add uses to the functionality that this PR implements.
Aside from `push(char16_t)` and `pop_utf16()` that implements the conversion, another function `isPartiallyPopping()` is added, which distinguishes if the current state is "partially pushing" or "partially popping". Since UTF-16 can have 2 code units for 1 code point, this reporting function will be crucial to the follow-up uses.
For `isPartiallyPopping()` to work, UTF-8 overlong encodings must be rejected. They pose a security hazard anyway. This PR adds new checks that reject UTF-8 overlong encodings. This PR also adds new checks that reject `>U+10FFFF` inputs.
Tests are added to cover the above-mentioned functionalities. This includes testing that `isPartiallyPopping()` works in UTF-8 -> UTF-32 conversions, which the function is designed to work.
---
Patch is 37.61 KiB, truncated to 20.00 KiB below, full version: https://github.com/llvm/llvm-project/pull/230434.diff
9 Files Affected:
- (modified) libc/hdr/types/CMakeLists.txt (+11)
- (added) libc/hdr/types/char16_t.h (+27)
- (modified) libc/src/__support/wchar/CMakeLists.txt (+2)
- (modified) libc/src/__support/wchar/character_converter.h (+179-4)
- (modified) libc/test/src/__support/wchar/CMakeLists.txt (+26)
- (added) libc/test/src/__support/wchar/utf16_to_8_test.cpp (+235)
- (modified) libc/test/src/__support/wchar/utf32_to_8_test.cpp (+14-12)
- (added) libc/test/src/__support/wchar/utf8_to_16_test.cpp (+183)
- (modified) libc/test/src/__support/wchar/utf8_to_32_test.cpp (+139)
``````````diff
diff --git a/libc/hdr/types/CMakeLists.txt b/libc/hdr/types/CMakeLists.txt
index 92a2985b74be72..6ca38970729cfd 100644
--- a/libc/hdr/types/CMakeLists.txt
+++ b/libc/hdr/types/CMakeLists.txt
@@ -9,6 +9,17 @@ add_proxy_header_library(
libc.include.uchar
)
+add_proxy_header_library(
+ char16_t
+ HDRS
+ char16_t.h
+ DEPENDS
+ libc.hdr.uchar_overlay
+ FULL_BUILD_DEPENDS
+ libc.include.llvm-libc-types.char16_t
+ libc.include.uchar
+)
+
add_proxy_header_library(
char32_t
HDRS
diff --git a/libc/hdr/types/char16_t.h b/libc/hdr/types/char16_t.h
new file mode 100644
index 00000000000000..a59d698243b090
--- /dev/null
+++ b/libc/hdr/types/char16_t.h
@@ -0,0 +1,27 @@
+//===-- Definition of char16_t.h ------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef LLVM_LIBC_HDR_TYPES_CHAR16_T_H
+#define LLVM_LIBC_HDR_TYPES_CHAR16_T_H
+
+#ifdef LIBC_FULL_BUILD
+
+#include "include/llvm-libc-types/char16_t.h"
+
+#else // overlay mode
+
+// MacOS doesn't provide uchar.h so we use the types provided by LLVM-libc.
+#ifdef __APPLE__
+#include "include/llvm-libc-types/char16_t.h"
+#else
+#include "hdr/uchar_overlay.h"
+#endif
+
+#endif // LIBC_FULL_BUILD
+
+#endif // LLVM_LIBC_HDR_TYPES_CHAR16_T_H
diff --git a/libc/src/__support/wchar/CMakeLists.txt b/libc/src/__support/wchar/CMakeLists.txt
index c4e7f0dbaa1786..91fd788fd07627 100644
--- a/libc/src/__support/wchar/CMakeLists.txt
+++ b/libc/src/__support/wchar/CMakeLists.txt
@@ -29,11 +29,13 @@ add_header_library(
DEPENDS
libc.hdr.errno_macros
libc.hdr.types.char8_t
+ libc.hdr.types.char16_t
libc.hdr.types.char32_t
libc.hdr.types.size_t
libc.src.__support.common
libc.src.__support.CPP.type_traits
libc.src.__support.error_or
+ libc.src.__support.libc_assert
libc.src.__support.macros.config
libc.src.__support.macros.properties.types
libc.src.__support.math_extras
diff --git a/libc/src/__support/wchar/character_converter.h b/libc/src/__support/wchar/character_converter.h
index f754a602d2cce0..f7ccf6f18b36cc 100644
--- a/libc/src/__support/wchar/character_converter.h
+++ b/libc/src/__support/wchar/character_converter.h
@@ -10,6 +10,7 @@
#define LLVM_LIBC_SRC___SUPPORT_CHARACTER_CONVERTER_H
#include "hdr/errno_macros.h"
+#include "hdr/types/char16_t.h"
#include "hdr/types/char32_t.h"
#include "hdr/types/char8_t.h"
#include "hdr/types/size_t.h"
@@ -18,6 +19,7 @@
#include "src/__support/CPP/type_traits.h"
#include "src/__support/common.h"
#include "src/__support/error_or.h"
+#include "src/__support/libc_assert.h"
#include "src/__support/macros/properties/types.h"
#include "src/__support/math_extras.h"
#include "src/__support/wchar/mbstate.h"
@@ -57,18 +59,65 @@ class CharacterConverter {
LIBC_INLINE bool isEmpty() { return state->bytes_stored == 0; }
bool isValidState();
+ /*
+ * When converting between UTF-16 and UTF-8, the converter might be in a state
+ * of either partially-pushing or partially-popping. For example consider the
+ * following sequence:
+ * * push((char8_t)0xF0);
+ * * push((char8_t)0xA4);
+ * * [A](partial=0x24,bytes_stored=2,total_bytes=4)
+ * * push((char8_t)0xAD);
+ * * push((char8_t)0xA2);
+ * * (partial=0x24B62,bytes_stored=4,total_bytes=4)
+ * * pop_utf16(); -> 0xD852
+ * * [B](partial=0x24B62,bytes_stored=2,total_bytes=4)
+ * * pop_utf16(); -> 0xDF62
+ * At point [A] and point [B], the stored bytes would both be 2. But at [A] we
+ * must continue reading input, while at [B] we must write the output low
+ * surrogate and clear the state.
+ * Thus we must have some means to distinguish between pushing and popping.
+ * Notice that: When pushing, we do left-shifts on state->partial before each
+ * push. When popping, we keep state->partial at the complete code point.
+ * Since we disallow overlong encodings, the complete UTF-8 code point is
+ * always greater than
+ * * MAX_VALUE_PER_UTF8_LEN[state->total_bytes - 2]
+ * but at most
+ * * MAX_VALUE_PER_UTF8_LEN[state->total_bytes - 1]
+ * If we are partially through pushing, state->partial is at most
+ * * char32 >> 6
+ * thus at most
+ * * MAX_VALUE_PER_UTF8_LEN[state->total_bytes - 1] >> 6
+ * Observe that, an item in MAX_VALUE_PER_UTF8_LEN right-shifted by 6 is
+ * always less than the preceding item. Based on these facts, we can determine
+ * if we are partially through pushing or popping, by comparing state->partial
+ * with MAX_VALUE_PER_UTF8_LEN[state->total_bytes - 2]
+ * Also when converting UTF-16 -> UTF-8, in a partial state,
+ * * bytes_stored=2
+ * * total_bytes=4
+ * so the threshold 0xFFFF is also correct.
+ */
+ LIBC_INLINE bool isPartiallyPopping() {
+ if (isEmpty() || isFull())
+ return false;
+ // When neither empty nor full, state->total_bytes is at least 2
+ char32_t threshold = MAX_VALUE_PER_UTF8_LEN[state->total_bytes - 2];
+ return state->partial > threshold;
+ }
+
template <typename CharType> size_t sizeAs();
int push(char8_t utf8_byte);
+ int push(char16_t utf16);
int push(char32_t utf32);
ErrorOr<char8_t> pop_utf8();
+ ErrorOr<char16_t> pop_utf16();
ErrorOr<char32_t> pop_utf32();
-#if defined(LIBC_TYPES_WCHAR_T_IS_UTF32)
+#if defined(LIBC_TYPES_WCHAR_T_IS_UTF32) || defined(LIBC_TYPES_WCHAR_T_IS_UTF16)
int push(wchar_t wchar);
ErrorOr<wchar_t> pop_wchar();
-#endif // LIBC_TYPES_WCHAR_T_IS_UTF32
+#endif // LIBC_TYPES_WCHAR_T_IS_UTF32 || LIBC_TYPES_WCHAR_T_IS_UTF16
template <typename CharType> ErrorOr<CharType> pop();
};
@@ -112,6 +161,11 @@ LIBC_INLINE int CharacterConverter::push(char8_t utf8_byte) {
state->bytes_stored++;
return 0;
}
+ // Pushing when partially popping is a misuse in the library
+ if (isPartiallyPopping()) {
+ clear();
+ return -1;
+ }
// Any subsequent push
// Adding 6 more bits so need to left shift
if (num_ones == 1 && !isFull()) {
@@ -119,6 +173,19 @@ LIBC_INLINE int CharacterConverter::push(char8_t utf8_byte) {
state->partial = state->partial << ENCODED_BITS_PER_UTF8;
state->partial |= byte;
state->bytes_stored++;
+ if (isFull()) {
+ // Detect and reject overlong encodings
+ // Needed for isPartiallyPopping to work
+ if (state->partial <= MAX_VALUE_PER_UTF8_LEN[state->total_bytes - 2]) {
+ clear();
+ return EILSEQ;
+ }
+ // Also reject out-of-range code points
+ if (state->partial > MAX_VALUE_PER_UTF8_LEN[state->total_bytes - 1]) {
+ clear();
+ return EILSEQ;
+ }
+ }
return 0;
}
@@ -127,6 +194,57 @@ LIBC_INLINE int CharacterConverter::push(char8_t utf8_byte) {
return EILSEQ;
}
+LIBC_INLINE int CharacterConverter::push(char16_t utf16) {
+ // Checking for high surrogate if first push
+ if (isEmpty()) {
+ bool is_surrogate = (utf16 & 0xF800) == 0xD800;
+ if (is_surrogate) {
+ bool is_high_surrogate = (utf16 & 0x400) == 0;
+ if (!is_high_surrogate) {
+ // unpaired surrogate -> reset the state
+ // bytes_stored and total_bytes will always be 0 here
+ state->partial = static_cast<char32_t>(0);
+ return EILSEQ;
+ }
+ // high surrogate
+ // code point between 0x10000 and 0x10FFFF
+ state->total_bytes = 4;
+ state->bytes_stored = 2;
+ // 10 bits each from the high surrogate and the low surrogate, and the
+ // full code point is from 0x10000, so we add 0x40 to partial, which after
+ // shifting becomes 0x10000
+ state->partial = static_cast<char32_t>(utf16 & 0x3FF) + 0x40;
+ } else {
+ state->partial = static_cast<char32_t>(utf16);
+ // determine number of utf-8 bytes needed to represent this utf16 value
+ uint8_t i;
+ for (i = 0; i < 2; i++) {
+ if (state->partial <= MAX_VALUE_PER_UTF8_LEN[i])
+ break;
+ }
+ state->total_bytes = i + 1;
+ state->bytes_stored = i + 1;
+ }
+ return 0;
+ }
+ // Pushing when partially popping is a misuse in the library
+ if (isFull() || isPartiallyPopping()) {
+ clear();
+ return -1;
+ }
+ bool is_low_surrogate = (utf16 & 0xFC00) == 0xDC00;
+ if (!is_low_surrogate) {
+ // unpaired surrogate -> reset the state
+ clear();
+ return EILSEQ;
+ }
+ char32_t low = utf16 & 0x3FF;
+ state->partial <<= 10;
+ state->partial |= low;
+ state->bytes_stored += 2;
+ return 0;
+}
+
LIBC_INLINE int CharacterConverter::push(char32_t utf32) {
// we can't be partially through a conversion when pushing a utf32 value
if (!isEmpty())
@@ -160,8 +278,36 @@ LIBC_INLINE ErrorOr<char32_t> CharacterConverter::pop_utf32() {
return utf32;
}
+LIBC_INLINE ErrorOr<char16_t> CharacterConverter::pop_utf16() {
+ if (!isFull() && !isPartiallyPopping())
+ return Error(-1);
+ char32_t output;
+ if (state->total_bytes <= 3) {
+ // no surrogates needed
+ output = state->partial;
+ // disallow using UTF-8 or UTF-32 to encode unpaired surrogates
+ bool is_surrogate = (output & 0xF800) == 0xD800;
+ if (is_surrogate) {
+ clear();
+ return Error(EILSEQ);
+ }
+ clear();
+ } else {
+ if (isFull()) {
+ // high surrogate first
+ output = ((state->partial - 0x10000) >> 10) | 0xD800;
+ state->bytes_stored = 2;
+ } else {
+ // low surrogate
+ output = ((state->partial - 0x10000) & 0x3FF) | 0xDC00;
+ clear();
+ }
+ }
+ return static_cast<char16_t>(output);
+}
+
LIBC_INLINE ErrorOr<char8_t> CharacterConverter::pop_utf8() {
- if (isEmpty())
+ if (!isFull() && !isPartiallyPopping())
return Error(-1);
constexpr char8_t FIRST_BYTE_HEADERS[] = {0, 0xC0, 0xE0, 0xF0};
@@ -196,6 +342,10 @@ template <> LIBC_INLINE ErrorOr<char8_t> CharacterConverter::pop() {
return pop_utf8();
}
+template <> LIBC_INLINE ErrorOr<char16_t> CharacterConverter::pop() {
+ return pop_utf16();
+}
+
template <> LIBC_INLINE ErrorOr<char32_t> CharacterConverter::pop() {
return pop_utf32();
}
@@ -204,6 +354,10 @@ template <> LIBC_INLINE size_t CharacterConverter::sizeAs<char8_t>() {
return state->total_bytes;
}
+template <> LIBC_INLINE size_t CharacterConverter::sizeAs<char16_t>() {
+ return state->total_bytes > 3 ? 2 : 1;
+}
+
template <> LIBC_INLINE size_t CharacterConverter::sizeAs<char32_t>() {
return 1;
}
@@ -229,7 +383,28 @@ template <> LIBC_INLINE size_t CharacterConverter::sizeAs<wchar_t>() {
return sizeAs<char32_t>();
}
-#endif // LIBC_TYPES_WCHAR_T_IS_UTF32
+#elif defined(LIBC_TYPES_WCHAR_T_IS_UTF16)
+
+LIBC_INLINE int CharacterConverter::push(wchar_t wchar) {
+ return push(static_cast<char16_t>(wchar));
+}
+
+LIBC_INLINE ErrorOr<wchar_t> CharacterConverter::pop_wchar() {
+ ErrorOr<char16_t> Result = pop_utf16();
+ if (!Result)
+ return Error(Result.error());
+ return static_cast<wchar_t>(*Result);
+}
+
+template <> LIBC_INLINE ErrorOr<wchar_t> CharacterConverter::pop() {
+ return pop_wchar();
+}
+
+template <> LIBC_INLINE size_t CharacterConverter::sizeAs<wchar_t>() {
+ return sizeAs<char16_t>();
+}
+
+#endif // LIBC_TYPES_WCHAR_T_IS_UTF32 || LIBC_TYPES_WCHAR_T_IS_UTF16
} // namespace internal
} // namespace LIBC_NAMESPACE_DECL
diff --git a/libc/test/src/__support/wchar/CMakeLists.txt b/libc/test/src/__support/wchar/CMakeLists.txt
index 259e896ee835d4..907950833dbcc0 100644
--- a/libc/test/src/__support/wchar/CMakeLists.txt
+++ b/libc/test/src/__support/wchar/CMakeLists.txt
@@ -41,6 +41,32 @@ add_libc_test(
libc.src.__support.wchar.mbstate
)
+add_libc_test(
+ utf8_to_16_test
+ SUITE
+ libc-support-tests
+ SRCS
+ utf8_to_16_test.cpp
+ DEPENDS
+ libc.src.__support.error_or
+ libc.src.__support.macros.properties.types
+ libc.src.__support.wchar.character_converter
+ libc.src.__support.wchar.mbstate
+)
+
+add_libc_test(
+ utf16_to_8_test
+ SUITE
+ libc-support-tests
+ SRCS
+ utf16_to_8_test.cpp
+ DEPENDS
+ libc.src.__support.error_or
+ libc.src.__support.macros.properties.types
+ libc.src.__support.wchar.character_converter
+ libc.src.__support.wchar.mbstate
+)
+
add_libc_test(
wcsnrtombs_test
SUITE
diff --git a/libc/test/src/__support/wchar/utf16_to_8_test.cpp b/libc/test/src/__support/wchar/utf16_to_8_test.cpp
new file mode 100644
index 00000000000000..737f17595ce107
--- /dev/null
+++ b/libc/test/src/__support/wchar/utf16_to_8_test.cpp
@@ -0,0 +1,235 @@
+//===-- Unittests for the CharacterConverter class (utf16 -> 8) -----------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#include "src/__support/common.h"
+#include "src/__support/macros/properties/types.h"
+#include "src/__support/wchar/character_converter.h"
+#include "src/__support/wchar/mbstate.h"
+
+#include "test/UnitTest/Test.h"
+
+#if defined(LIBC_TYPES_WCHAR_T_IS_UTF16)
+using TestCharTypesUTF16 = LIBC_NAMESPACE::testing::TypeList<char16_t, wchar_t>;
+#else
+using TestCharTypesUTF16 = LIBC_NAMESPACE::testing::TypeList<char16_t>;
+#endif
+
+TYPED_TEST(LlvmLibcCharacterConverterUTF16To8Test, OneByte,
+ TestCharTypesUTF16) {
+ using CharType16 = ParamType;
+
+ LIBC_NAMESPACE::internal::mbstate state;
+ LIBC_NAMESPACE::internal::CharacterConverter cr(&state);
+ cr.clear();
+
+ // utf8 1-byte encodings are identical to their utf16 representations
+ CharType16 utf16_A = 0x41; // 'A'
+ cr.push(utf16_A);
+ ASSERT_TRUE(cr.isFull());
+ auto popped = cr.pop_utf8();
+ ASSERT_TRUE(popped.has_value());
+ ASSERT_EQ(static_cast<char>(popped.value()), 'A');
+ ASSERT_TRUE(cr.isEmpty());
+
+ CharType16 utf16_B = 0x42; // 'B'
+ cr.push(utf16_B);
+ ASSERT_TRUE(cr.isFull());
+ popped = cr.pop_utf8();
+ ASSERT_TRUE(popped.has_value());
+ ASSERT_EQ(static_cast<char>(popped.value()), 'B');
+ ASSERT_TRUE(cr.isEmpty());
+
+ // should error if we try to pop another utf8 byte out
+ popped = cr.pop_utf8();
+ ASSERT_FALSE(popped.has_value());
+}
+
+TYPED_TEST(LlvmLibcCharacterConverterUTF16To8Test, TwoByte,
+ TestCharTypesUTF16) {
+ using CharType16 = ParamType;
+
+ LIBC_NAMESPACE::internal::mbstate state;
+ LIBC_NAMESPACE::internal::CharacterConverter cr(&state);
+ cr.clear();
+
+ // testing utf16: 0xff -> utf8: 0xc3 0xbf
+ CharType16 utf16 = 0xff;
+ cr.push(utf16);
+ ASSERT_TRUE(cr.isFull());
+ auto popped = cr.pop_utf8();
+ ASSERT_TRUE(popped.has_value());
+ ASSERT_EQ(static_cast<int>(popped.value()), 0xc3);
+ ASSERT_TRUE(cr.isPartiallyPopping());
+ popped = cr.pop_utf8();
+ ASSERT_TRUE(popped.has_value());
+ ASSERT_EQ(static_cast<int>(popped.value()), 0xbf);
+ ASSERT_TRUE(cr.isEmpty());
+
+ // testing utf16: 0x58e -> utf8: 0xd6 0x8e
+ utf16 = 0x58e;
+ cr.push(utf16);
+ ASSERT_TRUE(cr.isFull());
+ popped = cr.pop_utf8();
+ ASSERT_TRUE(popped.has_value());
+ ASSERT_EQ(static_cast<int>(popped.value()), 0xd6);
+ ASSERT_TRUE(cr.isPartiallyPopping());
+ popped = cr.pop_utf8();
+ ASSERT_TRUE(popped.has_value());
+ ASSERT_EQ(static_cast<int>(popped.value()), 0x8e);
+ ASSERT_TRUE(cr.isEmpty());
+
+ // should error if we try to pop another utf8 byte out
+ popped = cr.pop_utf8();
+ ASSERT_FALSE(popped.has_value());
+}
+
+TYPED_TEST(LlvmLibcCharacterConverterUTF16To8Test, ThreeByte,
+ TestCharTypesUTF16) {
+ using CharType16 = ParamType;
+
+ LIBC_NAMESPACE::internal::mbstate state;
+ LIBC_NAMESPACE::internal::CharacterConverter cr(&state);
+ cr.clear();
+
+ // testing utf16: 0xac15 -> utf8: 0xea 0xb0 0x95
+ CharType16 utf16 = 0xac15;
+ cr.push(utf16);
+ ASSERT_TRUE(cr.isFull());
+ auto popped = cr.pop_utf8();
+ ASSERT_TRUE(popped.has_value());
+ ASSERT_EQ(static_cast<int>(popped.value()), 0xea);
+ ASSERT_TRUE(cr.isPartiallyPopping());
+ popped = cr.pop_utf8();
+ ASSERT_TRUE(popped.has_value());
+ ASSERT_EQ(static_cast<int>(popped.value()), 0xb0);
+ ASSERT_TRUE(cr.isPartiallyPopping());
+ popped = cr.pop_utf8();
+ ASSERT_TRUE(popped.has_value());
+ ASSERT_EQ(static_cast<int>(popped.value()), 0x95);
+ ASSERT_TRUE(cr.isEmpty());
+
+ // testing utf16: 0x267b -> utf8: 0xe2 0x99 0xbb
+ utf16 = 0x267b;
+ cr.push(utf16);
+ ASSERT_TRUE(cr.isFull());
+ popped = cr.pop_utf8();
+ ASSERT_TRUE(popped.has_value());
+ ASSERT_EQ(static_cast<int>(popped.value()), 0xe2);
+ ASSERT_TRUE(cr.isPartiallyPopping());
+ popped = cr.pop_utf8();
+ ASSERT_TRUE(popped.has_value());
+ ASSERT_EQ(static_cast<int>(popped.value()), 0x99);
+ ASSERT_TRUE(cr.isPartiallyPopping());
+ popped = cr.pop_utf8();
+ ASSERT_TRUE(popped.has_value());
+ ASSERT_EQ(static_cast<int>(popped.value()), 0xbb);
+ ASSERT_TRUE(cr.isEmpty());
+
+ // should error if we try to pop another utf8 byte out
+ popped = cr.pop_utf8();
+ ASSERT_FALSE(popped.has_value());
+}
+
+TYPED_TEST(LlvmLibcCharacterConverterUTF16To8Test, FourByte,
+ TestCharTypesUTF16) {
+ using CharType16 = ParamType;
+
+ LIBC_NAMESPACE::internal::mbstate state;
+ LIBC_NAMESPACE::internal::CharacterConverter cr(&state);
+ cr.clear();
+
+ // testing utf16: 0xd83e 0xdd21 -> utf8: 0xf0 0x9f 0xa4 0xa1
+ CharType16 utf16_h = 0xd83e;
+ cr.push(utf16_h);
+ ASSERT_FALSE(cr.isPartiallyPopping());
+ CharType16 utf16_l = 0xdd21;
+ cr.push(utf16_l);
+ ASSERT_TRUE(cr.isFull());
+ auto popped = cr.pop_utf8();
+ ASSERT_TRUE(popped.has_value());
+ ASSERT_EQ(static_cast<int>(popped.value()), 0xf0);
+ ASSERT_TRUE(cr.isPartiallyPopping());
+ popped = cr.pop_utf8();
+ ASSERT_TRUE(popped.has_value());
+ ASSERT_EQ(static_cast<int>(popped.value()), 0x9f);
+ ASSERT_TRUE(cr.isPartiallyPopping());
+ popped = cr.pop_utf8();
+ ASSERT_TRUE(popped.has_value());
+ ASSERT_EQ(static_cast<int>(popped.value()), 0xa4);
+ ASSERT_TRUE(cr.isPartiallyPopping());
+ popped = cr.pop_utf8();
+ ASSERT_TRUE(popped.has_value());
+ ASSERT_EQ(static_cast<int>(popped.value()), 0xa1);
+ ASSERT_TRUE(cr.isEmpty());
+
+ // testing utf16: 0xd808 0xdd21 -> utf8: 0xf0 0x92 0x84 0xa1
+ utf16_h = 0xd808;
+ cr.push(utf16_h);
+ ASSERT_FALSE(cr.isPartiallyPopping());
+ utf16_l = 0xdd21;
+ cr.push(utf16_l);
+ ASSERT_TRUE(cr.isFull());
+ popped = cr.pop_utf8();
+ ASSERT_TRUE(popped.has_value());
+ ASSERT_EQ(static_cast<int>(popped.value()), 0xf0);
+ ASSERT_TRUE(cr.isPartiallyPopping());
+ popped = cr.pop_utf8();
+ ASSERT_TRUE(popped.has_value());
+ ASSERT_EQ(static_cast<int>(popped.value()), 0x92);
+ ASSERT_TRUE(cr.isPartiallyPopping());
+ popped = cr.pop_utf8();
+ ASSERT_TRUE(popped.has_value());
+ ASSERT_EQ(static_cast<int>(popped.value()), 0x84);
+ ASSERT_TRUE(cr.isPartiallyPopping());
+ popped = cr.pop_utf8();
+ ASSERT_TRUE(popped.has_value());
+ ASSERT_EQ(static_cast<int>(popped.value()), 0xa1);
+ ASSERT_TRUE(cr.isEmpty());
+
+ // should error if we try to pop another utf8 byte out
+ popped = cr.pop_utf8();
+ ASSERT_FALSE(popped.has_value());
+}
+
+TYPED_TEST(LlvmLibcCharacterConverterUTF16To8Test, CantPushMidConversion,
+ TestCharTypesUTF16) {
+ using CharType16 = ParamType;
+
+ LIBC_NAMESPACE::internal::mbstate state;
+ LIBC_NAMESPACE::internal::CharacterConverter cr(&state);
+ cr.clear();
+
+ // testing utf16: 0xd808 0xdd21 -> utf8: 0xf0 0x92 0x84 0xa1
+ CharType16 utf16_h = 0xd808;
+ ASSERT_EQ(cr.push(utf16_h), 0);
+ CharType16 utf16_l = 0xdd21;
+ ASSERT_EQ(cr.push(utf16_l), 0);
+ auto popped = cr.pop_utf8();
+ ASSERT_TRUE(popped.has_value());
+
+ // can't push a utf16 without finishing popping the utf8 bytes out
+ int err = cr.push(utf16_l);
+ ASSERT_EQ(err, -1);
+}
+
+TYPED_TEST(LlvmLibcCharacterConverterUTF16To8Test, CantPopMidPushing,
+ TestCharTypesUTF16) {
+ using CharType16 = ParamType;
+
+ LIBC_NAMESPACE::internal::mbstate state;
+ LIBC_NAMESPACE::internal::CharacterConverter cr(&state);
+ cr.clear();
+
+ // testing utf16: 0xd808 0xdd21 -> utf8: 0xf0 0x92 0x84 0xa1
+ CharType16 utf16_h = 0xd808;
+ ASSERT_EQ(cr.push(utf16_h), 0);
+
+ // can't pop a utf8 without finishing pushing the utf16 code units
+ ...
[truncated]
``````````
</details>
https://github.com/llvm/llvm-project/pull/230434
More information about the libc-commits
mailing list