[libc-commits] [libc] [libc] Add UTF-16 support in CharacterConverter (PR #230434)
via libc-commits
libc-commits at lists.llvm.org
Fri Oct 9 03:04:07 PDT 2026
https://github.com/yangbowen created https://github.com/llvm/llvm-project/pull/230434
This pull request adds support of UTF-16 in `CharacterConverter`. It does not use the functionality yet. Follow-ups will add uses to the functionality that this PR implements.
Aside from `push(char16_t)` and `pop_utf16()` that implements the conversion, another function `isPartiallyPopping()` is added, which distinguishes if the current state is "partially pushing" or "partially popping". Since UTF-16 can have 2 code units for 1 code point, this reporting function will be crucial to the follow-up uses.
For `isPartiallyPopping()` to work, UTF-8 overlong encodings must be rejected. They pose a security hazard anyway. This PR adds new checks that reject UTF-8 overlong encodings. This PR also adds new checks that reject `>U+10FFFF` inputs.
Tests are added to cover the above-mentioned functionalities. This includes testing that `isPartiallyPopping()` works in UTF-8 -> UTF-32 conversions, which the function is designed to work.
>From 5693e91825a6e2b4f5b596406307d70ead6665df Mon Sep 17 00:00:00 2001
From: yangbowen <ybw1234567 at hotmail.com>
Date: Thu, 8 Oct 2026 20:23:01 +0000
Subject: [PATCH] [libc] Add UTF-16 support in CharacterConverter
---
libc/hdr/types/CMakeLists.txt | 11 +
libc/hdr/types/char16_t.h | 27 ++
libc/src/__support/wchar/CMakeLists.txt | 2 +
.../src/__support/wchar/character_converter.h | 183 +++++++++++++-
libc/test/src/__support/wchar/CMakeLists.txt | 26 ++
.../src/__support/wchar/utf16_to_8_test.cpp | 235 ++++++++++++++++++
.../src/__support/wchar/utf32_to_8_test.cpp | 26 +-
.../src/__support/wchar/utf8_to_16_test.cpp | 183 ++++++++++++++
.../src/__support/wchar/utf8_to_32_test.cpp | 139 +++++++++++
9 files changed, 816 insertions(+), 16 deletions(-)
create mode 100644 libc/hdr/types/char16_t.h
create mode 100644 libc/test/src/__support/wchar/utf16_to_8_test.cpp
create mode 100644 libc/test/src/__support/wchar/utf8_to_16_test.cpp
diff --git a/libc/hdr/types/CMakeLists.txt b/libc/hdr/types/CMakeLists.txt
index 92a2985b74be72..6ca38970729cfd 100644
--- a/libc/hdr/types/CMakeLists.txt
+++ b/libc/hdr/types/CMakeLists.txt
@@ -9,6 +9,17 @@ add_proxy_header_library(
libc.include.uchar
)
+add_proxy_header_library(
+ char16_t
+ HDRS
+ char16_t.h
+ DEPENDS
+ libc.hdr.uchar_overlay
+ FULL_BUILD_DEPENDS
+ libc.include.llvm-libc-types.char16_t
+ libc.include.uchar
+)
+
add_proxy_header_library(
char32_t
HDRS
diff --git a/libc/hdr/types/char16_t.h b/libc/hdr/types/char16_t.h
new file mode 100644
index 00000000000000..a59d698243b090
--- /dev/null
+++ b/libc/hdr/types/char16_t.h
@@ -0,0 +1,27 @@
+//===-- Definition of char16_t.h ------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef LLVM_LIBC_HDR_TYPES_CHAR16_T_H
+#define LLVM_LIBC_HDR_TYPES_CHAR16_T_H
+
+#ifdef LIBC_FULL_BUILD
+
+#include "include/llvm-libc-types/char16_t.h"
+
+#else // overlay mode
+
+// MacOS doesn't provide uchar.h so we use the types provided by LLVM-libc.
+#ifdef __APPLE__
+#include "include/llvm-libc-types/char16_t.h"
+#else
+#include "hdr/uchar_overlay.h"
+#endif
+
+#endif // LIBC_FULL_BUILD
+
+#endif // LLVM_LIBC_HDR_TYPES_CHAR16_T_H
diff --git a/libc/src/__support/wchar/CMakeLists.txt b/libc/src/__support/wchar/CMakeLists.txt
index c4e7f0dbaa1786..91fd788fd07627 100644
--- a/libc/src/__support/wchar/CMakeLists.txt
+++ b/libc/src/__support/wchar/CMakeLists.txt
@@ -29,11 +29,13 @@ add_header_library(
DEPENDS
libc.hdr.errno_macros
libc.hdr.types.char8_t
+ libc.hdr.types.char16_t
libc.hdr.types.char32_t
libc.hdr.types.size_t
libc.src.__support.common
libc.src.__support.CPP.type_traits
libc.src.__support.error_or
+ libc.src.__support.libc_assert
libc.src.__support.macros.config
libc.src.__support.macros.properties.types
libc.src.__support.math_extras
diff --git a/libc/src/__support/wchar/character_converter.h b/libc/src/__support/wchar/character_converter.h
index f754a602d2cce0..f7ccf6f18b36cc 100644
--- a/libc/src/__support/wchar/character_converter.h
+++ b/libc/src/__support/wchar/character_converter.h
@@ -10,6 +10,7 @@
#define LLVM_LIBC_SRC___SUPPORT_CHARACTER_CONVERTER_H
#include "hdr/errno_macros.h"
+#include "hdr/types/char16_t.h"
#include "hdr/types/char32_t.h"
#include "hdr/types/char8_t.h"
#include "hdr/types/size_t.h"
@@ -18,6 +19,7 @@
#include "src/__support/CPP/type_traits.h"
#include "src/__support/common.h"
#include "src/__support/error_or.h"
+#include "src/__support/libc_assert.h"
#include "src/__support/macros/properties/types.h"
#include "src/__support/math_extras.h"
#include "src/__support/wchar/mbstate.h"
@@ -57,18 +59,65 @@ class CharacterConverter {
LIBC_INLINE bool isEmpty() { return state->bytes_stored == 0; }
bool isValidState();
+ /*
+ * When converting between UTF-16 and UTF-8, the converter might be in a state
+ * of either partially-pushing or partially-popping. For example consider the
+ * following sequence:
+ * * push((char8_t)0xF0);
+ * * push((char8_t)0xA4);
+ * * [A](partial=0x24,bytes_stored=2,total_bytes=4)
+ * * push((char8_t)0xAD);
+ * * push((char8_t)0xA2);
+ * * (partial=0x24B62,bytes_stored=4,total_bytes=4)
+ * * pop_utf16(); -> 0xD852
+ * * [B](partial=0x24B62,bytes_stored=2,total_bytes=4)
+ * * pop_utf16(); -> 0xDF62
+ * At point [A] and point [B], the stored bytes would both be 2. But at [A] we
+ * must continue reading input, while at [B] we must write the output low
+ * surrogate and clear the state.
+ * Thus we must have some means to distinguish between pushing and popping.
+ * Notice that: When pushing, we do left-shifts on state->partial before each
+ * push. When popping, we keep state->partial at the complete code point.
+ * Since we disallow overlong encodings, the complete UTF-8 code point is
+ * always greater than
+ * * MAX_VALUE_PER_UTF8_LEN[state->total_bytes - 2]
+ * but at most
+ * * MAX_VALUE_PER_UTF8_LEN[state->total_bytes - 1]
+ * If we are partially through pushing, state->partial is at most
+ * * char32 >> 6
+ * thus at most
+ * * MAX_VALUE_PER_UTF8_LEN[state->total_bytes - 1] >> 6
+ * Observe that, an item in MAX_VALUE_PER_UTF8_LEN right-shifted by 6 is
+ * always less than the preceding item. Based on these facts, we can determine
+ * if we are partially through pushing or popping, by comparing state->partial
+ * with MAX_VALUE_PER_UTF8_LEN[state->total_bytes - 2]
+ * Also when converting UTF-16 -> UTF-8, in a partial state,
+ * * bytes_stored=2
+ * * total_bytes=4
+ * so the threshold 0xFFFF is also correct.
+ */
+ LIBC_INLINE bool isPartiallyPopping() {
+ if (isEmpty() || isFull())
+ return false;
+ // When neither empty nor full, state->total_bytes is at least 2
+ char32_t threshold = MAX_VALUE_PER_UTF8_LEN[state->total_bytes - 2];
+ return state->partial > threshold;
+ }
+
template <typename CharType> size_t sizeAs();
int push(char8_t utf8_byte);
+ int push(char16_t utf16);
int push(char32_t utf32);
ErrorOr<char8_t> pop_utf8();
+ ErrorOr<char16_t> pop_utf16();
ErrorOr<char32_t> pop_utf32();
-#if defined(LIBC_TYPES_WCHAR_T_IS_UTF32)
+#if defined(LIBC_TYPES_WCHAR_T_IS_UTF32) || defined(LIBC_TYPES_WCHAR_T_IS_UTF16)
int push(wchar_t wchar);
ErrorOr<wchar_t> pop_wchar();
-#endif // LIBC_TYPES_WCHAR_T_IS_UTF32
+#endif // LIBC_TYPES_WCHAR_T_IS_UTF32 || LIBC_TYPES_WCHAR_T_IS_UTF16
template <typename CharType> ErrorOr<CharType> pop();
};
@@ -112,6 +161,11 @@ LIBC_INLINE int CharacterConverter::push(char8_t utf8_byte) {
state->bytes_stored++;
return 0;
}
+ // Pushing when partially popping is a misuse in the library
+ if (isPartiallyPopping()) {
+ clear();
+ return -1;
+ }
// Any subsequent push
// Adding 6 more bits so need to left shift
if (num_ones == 1 && !isFull()) {
@@ -119,6 +173,19 @@ LIBC_INLINE int CharacterConverter::push(char8_t utf8_byte) {
state->partial = state->partial << ENCODED_BITS_PER_UTF8;
state->partial |= byte;
state->bytes_stored++;
+ if (isFull()) {
+ // Detect and reject overlong encodings
+ // Needed for isPartiallyPopping to work
+ if (state->partial <= MAX_VALUE_PER_UTF8_LEN[state->total_bytes - 2]) {
+ clear();
+ return EILSEQ;
+ }
+ // Also reject out-of-range code points
+ if (state->partial > MAX_VALUE_PER_UTF8_LEN[state->total_bytes - 1]) {
+ clear();
+ return EILSEQ;
+ }
+ }
return 0;
}
@@ -127,6 +194,57 @@ LIBC_INLINE int CharacterConverter::push(char8_t utf8_byte) {
return EILSEQ;
}
+LIBC_INLINE int CharacterConverter::push(char16_t utf16) {
+ // Checking for high surrogate if first push
+ if (isEmpty()) {
+ bool is_surrogate = (utf16 & 0xF800) == 0xD800;
+ if (is_surrogate) {
+ bool is_high_surrogate = (utf16 & 0x400) == 0;
+ if (!is_high_surrogate) {
+ // unpaired surrogate -> reset the state
+ // bytes_stored and total_bytes will always be 0 here
+ state->partial = static_cast<char32_t>(0);
+ return EILSEQ;
+ }
+ // high surrogate
+ // code point between 0x10000 and 0x10FFFF
+ state->total_bytes = 4;
+ state->bytes_stored = 2;
+ // 10 bits each from the high surrogate and the low surrogate, and the
+ // full code point is from 0x10000, so we add 0x40 to partial, which after
+ // shifting becomes 0x10000
+ state->partial = static_cast<char32_t>(utf16 & 0x3FF) + 0x40;
+ } else {
+ state->partial = static_cast<char32_t>(utf16);
+ // determine number of utf-8 bytes needed to represent this utf16 value
+ uint8_t i;
+ for (i = 0; i < 2; i++) {
+ if (state->partial <= MAX_VALUE_PER_UTF8_LEN[i])
+ break;
+ }
+ state->total_bytes = i + 1;
+ state->bytes_stored = i + 1;
+ }
+ return 0;
+ }
+ // Pushing when partially popping is a misuse in the library
+ if (isFull() || isPartiallyPopping()) {
+ clear();
+ return -1;
+ }
+ bool is_low_surrogate = (utf16 & 0xFC00) == 0xDC00;
+ if (!is_low_surrogate) {
+ // unpaired surrogate -> reset the state
+ clear();
+ return EILSEQ;
+ }
+ char32_t low = utf16 & 0x3FF;
+ state->partial <<= 10;
+ state->partial |= low;
+ state->bytes_stored += 2;
+ return 0;
+}
+
LIBC_INLINE int CharacterConverter::push(char32_t utf32) {
// we can't be partially through a conversion when pushing a utf32 value
if (!isEmpty())
@@ -160,8 +278,36 @@ LIBC_INLINE ErrorOr<char32_t> CharacterConverter::pop_utf32() {
return utf32;
}
+LIBC_INLINE ErrorOr<char16_t> CharacterConverter::pop_utf16() {
+ if (!isFull() && !isPartiallyPopping())
+ return Error(-1);
+ char32_t output;
+ if (state->total_bytes <= 3) {
+ // no surrogates needed
+ output = state->partial;
+ // disallow using UTF-8 or UTF-32 to encode unpaired surrogates
+ bool is_surrogate = (output & 0xF800) == 0xD800;
+ if (is_surrogate) {
+ clear();
+ return Error(EILSEQ);
+ }
+ clear();
+ } else {
+ if (isFull()) {
+ // high surrogate first
+ output = ((state->partial - 0x10000) >> 10) | 0xD800;
+ state->bytes_stored = 2;
+ } else {
+ // low surrogate
+ output = ((state->partial - 0x10000) & 0x3FF) | 0xDC00;
+ clear();
+ }
+ }
+ return static_cast<char16_t>(output);
+}
+
LIBC_INLINE ErrorOr<char8_t> CharacterConverter::pop_utf8() {
- if (isEmpty())
+ if (!isFull() && !isPartiallyPopping())
return Error(-1);
constexpr char8_t FIRST_BYTE_HEADERS[] = {0, 0xC0, 0xE0, 0xF0};
@@ -196,6 +342,10 @@ template <> LIBC_INLINE ErrorOr<char8_t> CharacterConverter::pop() {
return pop_utf8();
}
+template <> LIBC_INLINE ErrorOr<char16_t> CharacterConverter::pop() {
+ return pop_utf16();
+}
+
template <> LIBC_INLINE ErrorOr<char32_t> CharacterConverter::pop() {
return pop_utf32();
}
@@ -204,6 +354,10 @@ template <> LIBC_INLINE size_t CharacterConverter::sizeAs<char8_t>() {
return state->total_bytes;
}
+template <> LIBC_INLINE size_t CharacterConverter::sizeAs<char16_t>() {
+ return state->total_bytes > 3 ? 2 : 1;
+}
+
template <> LIBC_INLINE size_t CharacterConverter::sizeAs<char32_t>() {
return 1;
}
@@ -229,7 +383,28 @@ template <> LIBC_INLINE size_t CharacterConverter::sizeAs<wchar_t>() {
return sizeAs<char32_t>();
}
-#endif // LIBC_TYPES_WCHAR_T_IS_UTF32
+#elif defined(LIBC_TYPES_WCHAR_T_IS_UTF16)
+
+LIBC_INLINE int CharacterConverter::push(wchar_t wchar) {
+ return push(static_cast<char16_t>(wchar));
+}
+
+LIBC_INLINE ErrorOr<wchar_t> CharacterConverter::pop_wchar() {
+ ErrorOr<char16_t> Result = pop_utf16();
+ if (!Result)
+ return Error(Result.error());
+ return static_cast<wchar_t>(*Result);
+}
+
+template <> LIBC_INLINE ErrorOr<wchar_t> CharacterConverter::pop() {
+ return pop_wchar();
+}
+
+template <> LIBC_INLINE size_t CharacterConverter::sizeAs<wchar_t>() {
+ return sizeAs<char16_t>();
+}
+
+#endif // LIBC_TYPES_WCHAR_T_IS_UTF32 || LIBC_TYPES_WCHAR_T_IS_UTF16
} // namespace internal
} // namespace LIBC_NAMESPACE_DECL
diff --git a/libc/test/src/__support/wchar/CMakeLists.txt b/libc/test/src/__support/wchar/CMakeLists.txt
index 259e896ee835d4..907950833dbcc0 100644
--- a/libc/test/src/__support/wchar/CMakeLists.txt
+++ b/libc/test/src/__support/wchar/CMakeLists.txt
@@ -41,6 +41,32 @@ add_libc_test(
libc.src.__support.wchar.mbstate
)
+add_libc_test(
+ utf8_to_16_test
+ SUITE
+ libc-support-tests
+ SRCS
+ utf8_to_16_test.cpp
+ DEPENDS
+ libc.src.__support.error_or
+ libc.src.__support.macros.properties.types
+ libc.src.__support.wchar.character_converter
+ libc.src.__support.wchar.mbstate
+)
+
+add_libc_test(
+ utf16_to_8_test
+ SUITE
+ libc-support-tests
+ SRCS
+ utf16_to_8_test.cpp
+ DEPENDS
+ libc.src.__support.error_or
+ libc.src.__support.macros.properties.types
+ libc.src.__support.wchar.character_converter
+ libc.src.__support.wchar.mbstate
+)
+
add_libc_test(
wcsnrtombs_test
SUITE
diff --git a/libc/test/src/__support/wchar/utf16_to_8_test.cpp b/libc/test/src/__support/wchar/utf16_to_8_test.cpp
new file mode 100644
index 00000000000000..737f17595ce107
--- /dev/null
+++ b/libc/test/src/__support/wchar/utf16_to_8_test.cpp
@@ -0,0 +1,235 @@
+//===-- Unittests for the CharacterConverter class (utf16 -> 8) -----------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#include "src/__support/common.h"
+#include "src/__support/macros/properties/types.h"
+#include "src/__support/wchar/character_converter.h"
+#include "src/__support/wchar/mbstate.h"
+
+#include "test/UnitTest/Test.h"
+
+#if defined(LIBC_TYPES_WCHAR_T_IS_UTF16)
+using TestCharTypesUTF16 = LIBC_NAMESPACE::testing::TypeList<char16_t, wchar_t>;
+#else
+using TestCharTypesUTF16 = LIBC_NAMESPACE::testing::TypeList<char16_t>;
+#endif
+
+TYPED_TEST(LlvmLibcCharacterConverterUTF16To8Test, OneByte,
+ TestCharTypesUTF16) {
+ using CharType16 = ParamType;
+
+ LIBC_NAMESPACE::internal::mbstate state;
+ LIBC_NAMESPACE::internal::CharacterConverter cr(&state);
+ cr.clear();
+
+ // utf8 1-byte encodings are identical to their utf16 representations
+ CharType16 utf16_A = 0x41; // 'A'
+ cr.push(utf16_A);
+ ASSERT_TRUE(cr.isFull());
+ auto popped = cr.pop_utf8();
+ ASSERT_TRUE(popped.has_value());
+ ASSERT_EQ(static_cast<char>(popped.value()), 'A');
+ ASSERT_TRUE(cr.isEmpty());
+
+ CharType16 utf16_B = 0x42; // 'B'
+ cr.push(utf16_B);
+ ASSERT_TRUE(cr.isFull());
+ popped = cr.pop_utf8();
+ ASSERT_TRUE(popped.has_value());
+ ASSERT_EQ(static_cast<char>(popped.value()), 'B');
+ ASSERT_TRUE(cr.isEmpty());
+
+ // should error if we try to pop another utf8 byte out
+ popped = cr.pop_utf8();
+ ASSERT_FALSE(popped.has_value());
+}
+
+TYPED_TEST(LlvmLibcCharacterConverterUTF16To8Test, TwoByte,
+ TestCharTypesUTF16) {
+ using CharType16 = ParamType;
+
+ LIBC_NAMESPACE::internal::mbstate state;
+ LIBC_NAMESPACE::internal::CharacterConverter cr(&state);
+ cr.clear();
+
+ // testing utf16: 0xff -> utf8: 0xc3 0xbf
+ CharType16 utf16 = 0xff;
+ cr.push(utf16);
+ ASSERT_TRUE(cr.isFull());
+ auto popped = cr.pop_utf8();
+ ASSERT_TRUE(popped.has_value());
+ ASSERT_EQ(static_cast<int>(popped.value()), 0xc3);
+ ASSERT_TRUE(cr.isPartiallyPopping());
+ popped = cr.pop_utf8();
+ ASSERT_TRUE(popped.has_value());
+ ASSERT_EQ(static_cast<int>(popped.value()), 0xbf);
+ ASSERT_TRUE(cr.isEmpty());
+
+ // testing utf16: 0x58e -> utf8: 0xd6 0x8e
+ utf16 = 0x58e;
+ cr.push(utf16);
+ ASSERT_TRUE(cr.isFull());
+ popped = cr.pop_utf8();
+ ASSERT_TRUE(popped.has_value());
+ ASSERT_EQ(static_cast<int>(popped.value()), 0xd6);
+ ASSERT_TRUE(cr.isPartiallyPopping());
+ popped = cr.pop_utf8();
+ ASSERT_TRUE(popped.has_value());
+ ASSERT_EQ(static_cast<int>(popped.value()), 0x8e);
+ ASSERT_TRUE(cr.isEmpty());
+
+ // should error if we try to pop another utf8 byte out
+ popped = cr.pop_utf8();
+ ASSERT_FALSE(popped.has_value());
+}
+
+TYPED_TEST(LlvmLibcCharacterConverterUTF16To8Test, ThreeByte,
+ TestCharTypesUTF16) {
+ using CharType16 = ParamType;
+
+ LIBC_NAMESPACE::internal::mbstate state;
+ LIBC_NAMESPACE::internal::CharacterConverter cr(&state);
+ cr.clear();
+
+ // testing utf16: 0xac15 -> utf8: 0xea 0xb0 0x95
+ CharType16 utf16 = 0xac15;
+ cr.push(utf16);
+ ASSERT_TRUE(cr.isFull());
+ auto popped = cr.pop_utf8();
+ ASSERT_TRUE(popped.has_value());
+ ASSERT_EQ(static_cast<int>(popped.value()), 0xea);
+ ASSERT_TRUE(cr.isPartiallyPopping());
+ popped = cr.pop_utf8();
+ ASSERT_TRUE(popped.has_value());
+ ASSERT_EQ(static_cast<int>(popped.value()), 0xb0);
+ ASSERT_TRUE(cr.isPartiallyPopping());
+ popped = cr.pop_utf8();
+ ASSERT_TRUE(popped.has_value());
+ ASSERT_EQ(static_cast<int>(popped.value()), 0x95);
+ ASSERT_TRUE(cr.isEmpty());
+
+ // testing utf16: 0x267b -> utf8: 0xe2 0x99 0xbb
+ utf16 = 0x267b;
+ cr.push(utf16);
+ ASSERT_TRUE(cr.isFull());
+ popped = cr.pop_utf8();
+ ASSERT_TRUE(popped.has_value());
+ ASSERT_EQ(static_cast<int>(popped.value()), 0xe2);
+ ASSERT_TRUE(cr.isPartiallyPopping());
+ popped = cr.pop_utf8();
+ ASSERT_TRUE(popped.has_value());
+ ASSERT_EQ(static_cast<int>(popped.value()), 0x99);
+ ASSERT_TRUE(cr.isPartiallyPopping());
+ popped = cr.pop_utf8();
+ ASSERT_TRUE(popped.has_value());
+ ASSERT_EQ(static_cast<int>(popped.value()), 0xbb);
+ ASSERT_TRUE(cr.isEmpty());
+
+ // should error if we try to pop another utf8 byte out
+ popped = cr.pop_utf8();
+ ASSERT_FALSE(popped.has_value());
+}
+
+TYPED_TEST(LlvmLibcCharacterConverterUTF16To8Test, FourByte,
+ TestCharTypesUTF16) {
+ using CharType16 = ParamType;
+
+ LIBC_NAMESPACE::internal::mbstate state;
+ LIBC_NAMESPACE::internal::CharacterConverter cr(&state);
+ cr.clear();
+
+ // testing utf16: 0xd83e 0xdd21 -> utf8: 0xf0 0x9f 0xa4 0xa1
+ CharType16 utf16_h = 0xd83e;
+ cr.push(utf16_h);
+ ASSERT_FALSE(cr.isPartiallyPopping());
+ CharType16 utf16_l = 0xdd21;
+ cr.push(utf16_l);
+ ASSERT_TRUE(cr.isFull());
+ auto popped = cr.pop_utf8();
+ ASSERT_TRUE(popped.has_value());
+ ASSERT_EQ(static_cast<int>(popped.value()), 0xf0);
+ ASSERT_TRUE(cr.isPartiallyPopping());
+ popped = cr.pop_utf8();
+ ASSERT_TRUE(popped.has_value());
+ ASSERT_EQ(static_cast<int>(popped.value()), 0x9f);
+ ASSERT_TRUE(cr.isPartiallyPopping());
+ popped = cr.pop_utf8();
+ ASSERT_TRUE(popped.has_value());
+ ASSERT_EQ(static_cast<int>(popped.value()), 0xa4);
+ ASSERT_TRUE(cr.isPartiallyPopping());
+ popped = cr.pop_utf8();
+ ASSERT_TRUE(popped.has_value());
+ ASSERT_EQ(static_cast<int>(popped.value()), 0xa1);
+ ASSERT_TRUE(cr.isEmpty());
+
+ // testing utf16: 0xd808 0xdd21 -> utf8: 0xf0 0x92 0x84 0xa1
+ utf16_h = 0xd808;
+ cr.push(utf16_h);
+ ASSERT_FALSE(cr.isPartiallyPopping());
+ utf16_l = 0xdd21;
+ cr.push(utf16_l);
+ ASSERT_TRUE(cr.isFull());
+ popped = cr.pop_utf8();
+ ASSERT_TRUE(popped.has_value());
+ ASSERT_EQ(static_cast<int>(popped.value()), 0xf0);
+ ASSERT_TRUE(cr.isPartiallyPopping());
+ popped = cr.pop_utf8();
+ ASSERT_TRUE(popped.has_value());
+ ASSERT_EQ(static_cast<int>(popped.value()), 0x92);
+ ASSERT_TRUE(cr.isPartiallyPopping());
+ popped = cr.pop_utf8();
+ ASSERT_TRUE(popped.has_value());
+ ASSERT_EQ(static_cast<int>(popped.value()), 0x84);
+ ASSERT_TRUE(cr.isPartiallyPopping());
+ popped = cr.pop_utf8();
+ ASSERT_TRUE(popped.has_value());
+ ASSERT_EQ(static_cast<int>(popped.value()), 0xa1);
+ ASSERT_TRUE(cr.isEmpty());
+
+ // should error if we try to pop another utf8 byte out
+ popped = cr.pop_utf8();
+ ASSERT_FALSE(popped.has_value());
+}
+
+TYPED_TEST(LlvmLibcCharacterConverterUTF16To8Test, CantPushMidConversion,
+ TestCharTypesUTF16) {
+ using CharType16 = ParamType;
+
+ LIBC_NAMESPACE::internal::mbstate state;
+ LIBC_NAMESPACE::internal::CharacterConverter cr(&state);
+ cr.clear();
+
+ // testing utf16: 0xd808 0xdd21 -> utf8: 0xf0 0x92 0x84 0xa1
+ CharType16 utf16_h = 0xd808;
+ ASSERT_EQ(cr.push(utf16_h), 0);
+ CharType16 utf16_l = 0xdd21;
+ ASSERT_EQ(cr.push(utf16_l), 0);
+ auto popped = cr.pop_utf8();
+ ASSERT_TRUE(popped.has_value());
+
+ // can't push a utf16 without finishing popping the utf8 bytes out
+ int err = cr.push(utf16_l);
+ ASSERT_EQ(err, -1);
+}
+
+TYPED_TEST(LlvmLibcCharacterConverterUTF16To8Test, CantPopMidPushing,
+ TestCharTypesUTF16) {
+ using CharType16 = ParamType;
+
+ LIBC_NAMESPACE::internal::mbstate state;
+ LIBC_NAMESPACE::internal::CharacterConverter cr(&state);
+ cr.clear();
+
+ // testing utf16: 0xd808 0xdd21 -> utf8: 0xf0 0x92 0x84 0xa1
+ CharType16 utf16_h = 0xd808;
+ ASSERT_EQ(cr.push(utf16_h), 0);
+
+ // can't pop a utf8 without finishing pushing the utf16 code units
+ auto popped = cr.pop_utf8();
+ ASSERT_FALSE(popped.has_value());
+}
diff --git a/libc/test/src/__support/wchar/utf32_to_8_test.cpp b/libc/test/src/__support/wchar/utf32_to_8_test.cpp
index 52e10e11911753..bc911050cff353 100644
--- a/libc/test/src/__support/wchar/utf32_to_8_test.cpp
+++ b/libc/test/src/__support/wchar/utf32_to_8_test.cpp
@@ -64,7 +64,7 @@ TYPED_TEST(LlvmLibcCharacterConverterUTF32To8Test, TwoByte,
auto popped = cr.pop_utf8();
ASSERT_TRUE(popped.has_value());
ASSERT_EQ(static_cast<int>(popped.value()), 0xc3);
- ASSERT_TRUE(!cr.isEmpty());
+ ASSERT_TRUE(cr.isPartiallyPopping());
popped = cr.pop_utf8();
ASSERT_TRUE(popped.has_value());
ASSERT_EQ(static_cast<int>(popped.value()), 0xbf);
@@ -77,7 +77,7 @@ TYPED_TEST(LlvmLibcCharacterConverterUTF32To8Test, TwoByte,
popped = cr.pop_utf8();
ASSERT_TRUE(popped.has_value());
ASSERT_EQ(static_cast<int>(popped.value()), 0xd6);
- ASSERT_TRUE(!cr.isEmpty());
+ ASSERT_TRUE(cr.isPartiallyPopping());
popped = cr.pop_utf8();
ASSERT_TRUE(popped.has_value());
ASSERT_EQ(static_cast<int>(popped.value()), 0x8e);
@@ -103,11 +103,11 @@ TYPED_TEST(LlvmLibcCharacterConverterUTF32To8Test, ThreeByte,
auto popped = cr.pop_utf8();
ASSERT_TRUE(popped.has_value());
ASSERT_EQ(static_cast<int>(popped.value()), 0xea);
- ASSERT_TRUE(!cr.isEmpty());
+ ASSERT_TRUE(cr.isPartiallyPopping());
popped = cr.pop_utf8();
ASSERT_TRUE(popped.has_value());
ASSERT_EQ(static_cast<int>(popped.value()), 0xb0);
- ASSERT_TRUE(!cr.isEmpty());
+ ASSERT_TRUE(cr.isPartiallyPopping());
popped = cr.pop_utf8();
ASSERT_TRUE(popped.has_value());
ASSERT_EQ(static_cast<int>(popped.value()), 0x95);
@@ -120,11 +120,11 @@ TYPED_TEST(LlvmLibcCharacterConverterUTF32To8Test, ThreeByte,
popped = cr.pop_utf8();
ASSERT_TRUE(popped.has_value());
ASSERT_EQ(static_cast<int>(popped.value()), 0xe2);
- ASSERT_TRUE(!cr.isEmpty());
+ ASSERT_TRUE(cr.isPartiallyPopping());
popped = cr.pop_utf8();
ASSERT_TRUE(popped.has_value());
ASSERT_EQ(static_cast<int>(popped.value()), 0x99);
- ASSERT_TRUE(!cr.isEmpty());
+ ASSERT_TRUE(cr.isPartiallyPopping());
popped = cr.pop_utf8();
ASSERT_TRUE(popped.has_value());
ASSERT_EQ(static_cast<int>(popped.value()), 0xbb);
@@ -150,15 +150,15 @@ TYPED_TEST(LlvmLibcCharacterConverterUTF32To8Test, FourByte,
auto popped = cr.pop_utf8();
ASSERT_TRUE(popped.has_value());
ASSERT_EQ(static_cast<int>(popped.value()), 0xf0);
- ASSERT_TRUE(!cr.isEmpty());
+ ASSERT_TRUE(cr.isPartiallyPopping());
popped = cr.pop_utf8();
ASSERT_TRUE(popped.has_value());
ASSERT_EQ(static_cast<int>(popped.value()), 0x9f);
- ASSERT_TRUE(!cr.isEmpty());
+ ASSERT_TRUE(cr.isPartiallyPopping());
popped = cr.pop_utf8();
ASSERT_TRUE(popped.has_value());
ASSERT_EQ(static_cast<int>(popped.value()), 0xa4);
- ASSERT_TRUE(!cr.isEmpty());
+ ASSERT_TRUE(cr.isPartiallyPopping());
popped = cr.pop_utf8();
ASSERT_TRUE(popped.has_value());
ASSERT_EQ(static_cast<int>(popped.value()), 0xa1);
@@ -171,15 +171,15 @@ TYPED_TEST(LlvmLibcCharacterConverterUTF32To8Test, FourByte,
popped = cr.pop_utf8();
ASSERT_TRUE(popped.has_value());
ASSERT_EQ(static_cast<int>(popped.value()), 0xf0);
- ASSERT_TRUE(!cr.isEmpty());
+ ASSERT_TRUE(cr.isPartiallyPopping());
popped = cr.pop_utf8();
ASSERT_TRUE(popped.has_value());
ASSERT_EQ(static_cast<int>(popped.value()), 0x92);
- ASSERT_TRUE(!cr.isEmpty());
+ ASSERT_TRUE(cr.isPartiallyPopping());
popped = cr.pop_utf8();
ASSERT_TRUE(popped.has_value());
ASSERT_EQ(static_cast<int>(popped.value()), 0x84);
- ASSERT_TRUE(!cr.isEmpty());
+ ASSERT_TRUE(cr.isPartiallyPopping());
popped = cr.pop_utf8();
ASSERT_TRUE(popped.has_value());
ASSERT_EQ(static_cast<int>(popped.value()), 0xa1);
@@ -213,6 +213,8 @@ TEST(LlvmLibcCharacterConverterUTF32To8Test, InvalidState) {
LIBC_NAMESPACE::internal::mbstate s1;
LIBC_NAMESPACE::internal::CharacterConverter c1(&s1);
ASSERT_TRUE(c1.isValidState());
+ ASSERT_TRUE(c1.isEmpty());
+ ASSERT_FALSE(c1.isPartiallyPopping());
LIBC_NAMESPACE::internal::mbstate s2{0, 2, 0};
LIBC_NAMESPACE::internal::CharacterConverter c2(&s2);
diff --git a/libc/test/src/__support/wchar/utf8_to_16_test.cpp b/libc/test/src/__support/wchar/utf8_to_16_test.cpp
new file mode 100644
index 00000000000000..ed38ddb7596b86
--- /dev/null
+++ b/libc/test/src/__support/wchar/utf8_to_16_test.cpp
@@ -0,0 +1,183 @@
+//===-- Unittests for character_converter utf8->utf16 ---------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#include "hdr/errno_macros.h"
+#include "src/__support/error_or.h"
+#include "src/__support/macros/properties/types.h"
+#include "src/__support/wchar/character_converter.h"
+#include "src/__support/wchar/mbstate.h"
+#include "test/UnitTest/Test.h"
+
+#if defined(LIBC_TYPES_WCHAR_T_IS_UTF16)
+using TestCharTypesUTF16 = LIBC_NAMESPACE::testing::TypeList<char16_t, wchar_t>;
+#else
+using TestCharTypesUTF16 = LIBC_NAMESPACE::testing::TypeList<char16_t>;
+#endif
+
+TYPED_TEST(LlvmLibcCharacterConverterUTF8To16Test, OneByte,
+ TestCharTypesUTF16) {
+ using CharType16 = ParamType;
+
+ LIBC_NAMESPACE::internal::mbstate state;
+ state.bytes_stored = 0;
+ state.total_bytes = 0;
+ char ch = 'A';
+
+ LIBC_NAMESPACE::internal::CharacterConverter char_conv(&state);
+ int err = char_conv.push(static_cast<char8_t>(ch));
+ auto wch = char_conv.pop<CharType16>();
+
+ ASSERT_EQ(err, 0);
+ ASSERT_TRUE(wch.has_value());
+ ASSERT_EQ(static_cast<int>(wch.value()), 65);
+}
+
+TYPED_TEST(LlvmLibcCharacterConverterUTF8To16Test, TwoBytes,
+ TestCharTypesUTF16) {
+ using CharType16 = ParamType;
+
+ LIBC_NAMESPACE::internal::mbstate state;
+ state.bytes_stored = 0;
+ state.total_bytes = 0;
+ const char ch[2] = {static_cast<char>(0xC2),
+ static_cast<char>(0x8E)}; // car symbol
+
+ LIBC_NAMESPACE::internal::CharacterConverter char_conv(&state);
+ char_conv.push(static_cast<char8_t>(ch[0]));
+ ASSERT_FALSE(char_conv.isPartiallyPopping());
+ char_conv.push(static_cast<char8_t>(ch[1]));
+ auto wch = char_conv.pop<CharType16>();
+
+ ASSERT_TRUE(wch.has_value());
+ ASSERT_EQ(static_cast<int>(wch.value()), 142);
+}
+
+TYPED_TEST(LlvmLibcCharacterConverterUTF8To16Test, ThreeBytes,
+ TestCharTypesUTF16) {
+ using CharType16 = ParamType;
+
+ LIBC_NAMESPACE::internal::mbstate state;
+ state.bytes_stored = 0;
+ state.total_bytes = 0;
+ const char ch[3] = {static_cast<char>(0xE2), static_cast<char>(0x88),
+ static_cast<char>(0x91)}; // ∑ sigma symbol
+
+ LIBC_NAMESPACE::internal::CharacterConverter char_conv(&state);
+ char_conv.push(static_cast<char8_t>(ch[0]));
+ ASSERT_FALSE(char_conv.isPartiallyPopping());
+ char_conv.push(static_cast<char8_t>(ch[1]));
+ ASSERT_FALSE(char_conv.isPartiallyPopping());
+ char_conv.push(static_cast<char8_t>(ch[2]));
+ auto wch = char_conv.pop<CharType16>();
+
+ ASSERT_TRUE(wch.has_value());
+ ASSERT_EQ(static_cast<int>(wch.value()), 8721);
+}
+
+TYPED_TEST(LlvmLibcCharacterConverterUTF8To16Test, FourBytes,
+ TestCharTypesUTF16) {
+ using CharType16 = ParamType;
+
+ LIBC_NAMESPACE::internal::mbstate state;
+ state.bytes_stored = 0;
+ state.total_bytes = 0;
+ const char ch[4] = {static_cast<char>(0xF0), static_cast<char>(0x9F),
+ static_cast<char>(0xA4),
+ static_cast<char>(0xA1)}; // 🤡 clown emoji
+
+ LIBC_NAMESPACE::internal::CharacterConverter char_conv(&state);
+ char_conv.push(static_cast<char8_t>(ch[0]));
+ ASSERT_FALSE(char_conv.isPartiallyPopping());
+ char_conv.push(static_cast<char8_t>(ch[1]));
+ ASSERT_FALSE(char_conv.isPartiallyPopping());
+ char_conv.push(static_cast<char8_t>(ch[2]));
+ ASSERT_FALSE(char_conv.isPartiallyPopping());
+ char_conv.push(static_cast<char8_t>(ch[3]));
+ auto wch_h = char_conv.pop<CharType16>();
+ ASSERT_TRUE(char_conv.isPartiallyPopping());
+ auto wch_l = char_conv.pop<CharType16>();
+
+ ASSERT_TRUE(wch_h.has_value());
+ ASSERT_EQ(static_cast<int>(wch_h.value()), 55358);
+ ASSERT_TRUE(wch_l.has_value());
+ ASSERT_EQ(static_cast<int>(wch_l.value()), 56609);
+}
+
+TYPED_TEST(LlvmLibcCharacterConverterUTF8To16Test, ValidTwoByteWithExtraRead,
+ TestCharTypesUTF16) {
+ using CharType16 = ParamType;
+
+ LIBC_NAMESPACE::internal::mbstate state;
+ state.bytes_stored = 0;
+ state.total_bytes = 0;
+ const char ch[3] = {static_cast<char>(0xC2), static_cast<char>(0x8E),
+ static_cast<char>(0x80)};
+
+ LIBC_NAMESPACE::internal::CharacterConverter char_conv(&state);
+ int err = char_conv.push(static_cast<char8_t>(ch[0]));
+ ASSERT_EQ(err, 0);
+ err = char_conv.push(static_cast<char8_t>(ch[1]));
+ ASSERT_EQ(err, 0);
+ // Should produce an error on 3rd byte
+ err = char_conv.push(static_cast<char8_t>(ch[2]));
+ ASSERT_EQ(err, EILSEQ);
+
+ // Should produce an error since mbstate was reset
+ auto wch = char_conv.pop<CharType16>();
+ ASSERT_FALSE(wch.has_value());
+}
+
+TYPED_TEST(LlvmLibcCharacterConverterUTF8To16Test, TwoValidTwoBytes,
+ TestCharTypesUTF16) {
+ using CharType16 = ParamType;
+
+ LIBC_NAMESPACE::internal::mbstate state;
+ state.bytes_stored = 0;
+ state.total_bytes = 0;
+ const char ch[4] = {static_cast<char>(0xC2), static_cast<char>(0x8E),
+ static_cast<char>(0xC7), static_cast<char>(0x8C)};
+
+ LIBC_NAMESPACE::internal::CharacterConverter char_conv(&state);
+ int err = char_conv.push(static_cast<char8_t>(ch[0]));
+ ASSERT_EQ(err, 0);
+ err = char_conv.push(static_cast<char8_t>(ch[1]));
+ ASSERT_EQ(err, 0);
+ auto wch = char_conv.pop<CharType16>();
+ ASSERT_TRUE(wch.has_value());
+ ASSERT_EQ(static_cast<int>(wch.value()), 142);
+
+ // Second two byte character
+ err = char_conv.push(static_cast<char8_t>(ch[2]));
+ ASSERT_EQ(err, 0);
+ err = char_conv.push(static_cast<char8_t>(ch[3]));
+ ASSERT_EQ(err, 0);
+ wch = char_conv.pop<CharType16>();
+ ASSERT_TRUE(wch.has_value());
+ ASSERT_EQ(static_cast<int>(wch.value()), 460);
+}
+
+TYPED_TEST(LlvmLibcCharacterConverterUTF8To16Test, InvalidPop,
+ TestCharTypesUTF16) {
+ using CharType16 = ParamType;
+
+ LIBC_NAMESPACE::internal::mbstate state;
+ state.bytes_stored = 0;
+ state.total_bytes = 0;
+ LIBC_NAMESPACE::internal::CharacterConverter char_conv(&state);
+ const char ch[2] = {static_cast<char>(0xC2), static_cast<char>(0x8E)};
+ int err = char_conv.push(static_cast<char8_t>(ch[0]));
+ ASSERT_EQ(err, 0);
+ auto wch = char_conv.pop<CharType16>();
+ ASSERT_FALSE(
+ wch.has_value()); // Should fail since we have not read enough bytes
+ err = char_conv.push(static_cast<char8_t>(ch[1]));
+ ASSERT_EQ(err, 0);
+ wch = char_conv.pop<CharType16>();
+ ASSERT_TRUE(wch.has_value());
+ ASSERT_EQ(static_cast<int>(wch.value()), 142);
+}
diff --git a/libc/test/src/__support/wchar/utf8_to_32_test.cpp b/libc/test/src/__support/wchar/utf8_to_32_test.cpp
index db74cddafcbd76..8e72371ca7c067 100644
--- a/libc/test/src/__support/wchar/utf8_to_32_test.cpp
+++ b/libc/test/src/__support/wchar/utf8_to_32_test.cpp
@@ -49,6 +49,7 @@ TYPED_TEST(LlvmLibcCharacterConverterUTF8To32Test, TwoBytes,
LIBC_NAMESPACE::internal::CharacterConverter char_conv(&state);
char_conv.push(static_cast<char8_t>(ch[0]));
+ ASSERT_FALSE(char_conv.isPartiallyPopping());
char_conv.push(static_cast<char8_t>(ch[1]));
auto wch = char_conv.pop<CharType32>();
@@ -68,7 +69,9 @@ TYPED_TEST(LlvmLibcCharacterConverterUTF8To32Test, ThreeBytes,
LIBC_NAMESPACE::internal::CharacterConverter char_conv(&state);
char_conv.push(static_cast<char8_t>(ch[0]));
+ ASSERT_FALSE(char_conv.isPartiallyPopping());
char_conv.push(static_cast<char8_t>(ch[1]));
+ ASSERT_FALSE(char_conv.isPartiallyPopping());
char_conv.push(static_cast<char8_t>(ch[2]));
auto wch = char_conv.pop<CharType32>();
@@ -89,8 +92,11 @@ TYPED_TEST(LlvmLibcCharacterConverterUTF8To32Test, FourBytes,
LIBC_NAMESPACE::internal::CharacterConverter char_conv(&state);
char_conv.push(static_cast<char8_t>(ch[0]));
+ ASSERT_FALSE(char_conv.isPartiallyPopping());
char_conv.push(static_cast<char8_t>(ch[1]));
+ ASSERT_FALSE(char_conv.isPartiallyPopping());
char_conv.push(static_cast<char8_t>(ch[2]));
+ ASSERT_FALSE(char_conv.isPartiallyPopping());
char_conv.push(static_cast<char8_t>(ch[3]));
auto wch = char_conv.pop<CharType32>();
@@ -223,3 +229,136 @@ TYPED_TEST(LlvmLibcCharacterConverterUTF8To32Test, InvalidPop,
ASSERT_TRUE(wch.has_value());
ASSERT_EQ(static_cast<int>(wch.value()), 142);
}
+
+TYPED_TEST(LlvmLibcCharacterConverterUTF8To32Test, RejectOverlongTwoBytes,
+ TestCharTypesUTF32) {
+ using CharType32 = ParamType;
+
+ LIBC_NAMESPACE::internal::mbstate state;
+ state.bytes_stored = 0;
+ state.total_bytes = 0;
+ const char ch[4] = {static_cast<char>(0xC1), static_cast<char>(0xBF),
+ static_cast<char>(0xC2), static_cast<char>(0x80)};
+
+ LIBC_NAMESPACE::internal::CharacterConverter char_conv(&state);
+ // Overlong encoding should be rejected U+007F
+ int err = char_conv.push(static_cast<char8_t>(ch[0]));
+ ASSERT_EQ(err, 0);
+ err = char_conv.push(static_cast<char8_t>(ch[1]));
+ ASSERT_EQ(err, EILSEQ);
+
+ // Not overlong, don't reject U+0080
+ err = char_conv.push(static_cast<char8_t>(ch[2]));
+ ASSERT_EQ(err, 0);
+ err = char_conv.push(static_cast<char8_t>(ch[3]));
+ ASSERT_EQ(err, 0);
+ auto wch = char_conv.pop<CharType32>();
+ ASSERT_TRUE(wch.has_value());
+ ASSERT_EQ(static_cast<int>(wch.value()), 128);
+}
+
+TYPED_TEST(LlvmLibcCharacterConverterUTF8To32Test, RejectOverlongThreeBytes,
+ TestCharTypesUTF32) {
+ using CharType32 = ParamType;
+
+ LIBC_NAMESPACE::internal::mbstate state;
+ state.bytes_stored = 0;
+ state.total_bytes = 0;
+ const char ch[6] = {static_cast<char>(0xE0), static_cast<char>(0x9F),
+ static_cast<char>(0xBF), static_cast<char>(0xE0),
+ static_cast<char>(0xA0), static_cast<char>(0x80)};
+
+ LIBC_NAMESPACE::internal::CharacterConverter char_conv(&state);
+ // Overlong encoding should be rejected U+07FF
+ int err = char_conv.push(static_cast<char8_t>(ch[0]));
+ ASSERT_EQ(err, 0);
+ err = char_conv.push(static_cast<char8_t>(ch[1]));
+ ASSERT_EQ(err, 0);
+ err = char_conv.push(static_cast<char8_t>(ch[2]));
+ ASSERT_EQ(err, EILSEQ);
+
+ // Not overlong, don't reject U+0800
+ err = char_conv.push(static_cast<char8_t>(ch[3]));
+ ASSERT_EQ(err, 0);
+ err = char_conv.push(static_cast<char8_t>(ch[4]));
+ ASSERT_EQ(err, 0);
+ err = char_conv.push(static_cast<char8_t>(ch[5]));
+ ASSERT_EQ(err, 0);
+ auto wch = char_conv.pop<CharType32>();
+ ASSERT_TRUE(wch.has_value());
+ ASSERT_EQ(static_cast<int>(wch.value()), 2048);
+}
+
+TYPED_TEST(LlvmLibcCharacterConverterUTF8To32Test, RejectOverlongFourBytes,
+ TestCharTypesUTF32) {
+ using CharType32 = ParamType;
+
+ LIBC_NAMESPACE::internal::mbstate state;
+ state.bytes_stored = 0;
+ state.total_bytes = 0;
+ const char ch[8] = {static_cast<char>(0xF0), static_cast<char>(0x8F),
+ static_cast<char>(0xBF), static_cast<char>(0xBF),
+ static_cast<char>(0xF0), static_cast<char>(0x90),
+ static_cast<char>(0x80), static_cast<char>(0x80)};
+
+ LIBC_NAMESPACE::internal::CharacterConverter char_conv(&state);
+ // Overlong encoding should be rejected U+FFFF
+ int err = char_conv.push(static_cast<char8_t>(ch[0]));
+ ASSERT_EQ(err, 0);
+ err = char_conv.push(static_cast<char8_t>(ch[1]));
+ ASSERT_EQ(err, 0);
+ err = char_conv.push(static_cast<char8_t>(ch[2]));
+ ASSERT_EQ(err, 0);
+ err = char_conv.push(static_cast<char8_t>(ch[3]));
+ ASSERT_EQ(err, EILSEQ);
+
+ // Not overlong, don't reject U+10000
+ err = char_conv.push(static_cast<char8_t>(ch[4]));
+ ASSERT_EQ(err, 0);
+ err = char_conv.push(static_cast<char8_t>(ch[5]));
+ ASSERT_EQ(err, 0);
+ err = char_conv.push(static_cast<char8_t>(ch[6]));
+ ASSERT_EQ(err, 0);
+ err = char_conv.push(static_cast<char8_t>(ch[7]));
+ ASSERT_EQ(err, 0);
+ auto wch = char_conv.pop<CharType32>();
+ ASSERT_TRUE(wch.has_value());
+ ASSERT_EQ(static_cast<int>(wch.value()), 65536);
+}
+
+TYPED_TEST(LlvmLibcCharacterConverterUTF8To32Test, RejectOutOfRange,
+ TestCharTypesUTF32) {
+ using CharType32 = ParamType;
+
+ LIBC_NAMESPACE::internal::mbstate state;
+ state.bytes_stored = 0;
+ state.total_bytes = 0;
+ const char ch[8] = {static_cast<char>(0xF4), static_cast<char>(0x90),
+ static_cast<char>(0x80), static_cast<char>(0x80),
+ static_cast<char>(0xF4), static_cast<char>(0x8F),
+ static_cast<char>(0xBF), static_cast<char>(0xBF)};
+
+ LIBC_NAMESPACE::internal::CharacterConverter char_conv(&state);
+ // Out of range code point should be rejected U+110000
+ int err = char_conv.push(static_cast<char8_t>(ch[0]));
+ ASSERT_EQ(err, 0);
+ err = char_conv.push(static_cast<char8_t>(ch[1]));
+ ASSERT_EQ(err, 0);
+ err = char_conv.push(static_cast<char8_t>(ch[2]));
+ ASSERT_EQ(err, 0);
+ err = char_conv.push(static_cast<char8_t>(ch[3]));
+ ASSERT_EQ(err, EILSEQ);
+
+ // In range, don't reject U+10FFFF
+ err = char_conv.push(static_cast<char8_t>(ch[4]));
+ ASSERT_EQ(err, 0);
+ err = char_conv.push(static_cast<char8_t>(ch[5]));
+ ASSERT_EQ(err, 0);
+ err = char_conv.push(static_cast<char8_t>(ch[6]));
+ ASSERT_EQ(err, 0);
+ err = char_conv.push(static_cast<char8_t>(ch[7]));
+ ASSERT_EQ(err, 0);
+ auto wch = char_conv.pop<CharType32>();
+ ASSERT_TRUE(wch.has_value());
+ ASSERT_EQ(static_cast<int>(wch.value()), 1114111);
+}
More information about the libc-commits
mailing list