[libc-commits] [libc] [libc] Add UTF-16 support in printf_core::wchar_writer (PR #230994)
via libc-commits
libc-commits at lists.llvm.org
Sun Oct 11 08:33:34 PDT 2026
https://github.com/yangbowen created https://github.com/llvm/llvm-project/pull/230994
This PR is based on #230917.
This pull request adds support for UTF-16 `wchar_t` in `printf_core::wchar_writer`, enabling handling of wide characters by `printf` and friends on UTF-16 `wchar_t` platforms. (disabled by config option `LIBC_CONF_PRINTF_DISABLE_WIDE`)
AI Tool Use: Chatbots are used for web searching. LLM-based agents are also used extensively for testing and checking bugs. All code is hand-written by a human. This PR description is also hand-written by a human.
>From 5693e91825a6e2b4f5b596406307d70ead6665df Mon Sep 17 00:00:00 2001
From: yangbowen <ybw1234567 at hotmail.com>
Date: Thu, 8 Oct 2026 20:23:01 +0000
Subject: [PATCH 1/3] [libc] Add UTF-16 support in CharacterConverter
---
libc/hdr/types/CMakeLists.txt | 11 +
libc/hdr/types/char16_t.h | 27 ++
libc/src/__support/wchar/CMakeLists.txt | 2 +
.../src/__support/wchar/character_converter.h | 183 +++++++++++++-
libc/test/src/__support/wchar/CMakeLists.txt | 26 ++
.../src/__support/wchar/utf16_to_8_test.cpp | 235 ++++++++++++++++++
.../src/__support/wchar/utf32_to_8_test.cpp | 26 +-
.../src/__support/wchar/utf8_to_16_test.cpp | 183 ++++++++++++++
.../src/__support/wchar/utf8_to_32_test.cpp | 139 +++++++++++
9 files changed, 816 insertions(+), 16 deletions(-)
create mode 100644 libc/hdr/types/char16_t.h
create mode 100644 libc/test/src/__support/wchar/utf16_to_8_test.cpp
create mode 100644 libc/test/src/__support/wchar/utf8_to_16_test.cpp
diff --git a/libc/hdr/types/CMakeLists.txt b/libc/hdr/types/CMakeLists.txt
index 92a2985b74be7..6ca38970729cf 100644
--- a/libc/hdr/types/CMakeLists.txt
+++ b/libc/hdr/types/CMakeLists.txt
@@ -9,6 +9,17 @@ add_proxy_header_library(
libc.include.uchar
)
+add_proxy_header_library(
+ char16_t
+ HDRS
+ char16_t.h
+ DEPENDS
+ libc.hdr.uchar_overlay
+ FULL_BUILD_DEPENDS
+ libc.include.llvm-libc-types.char16_t
+ libc.include.uchar
+)
+
add_proxy_header_library(
char32_t
HDRS
diff --git a/libc/hdr/types/char16_t.h b/libc/hdr/types/char16_t.h
new file mode 100644
index 0000000000000..a59d698243b09
--- /dev/null
+++ b/libc/hdr/types/char16_t.h
@@ -0,0 +1,27 @@
+//===-- Definition of char16_t.h ------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef LLVM_LIBC_HDR_TYPES_CHAR16_T_H
+#define LLVM_LIBC_HDR_TYPES_CHAR16_T_H
+
+#ifdef LIBC_FULL_BUILD
+
+#include "include/llvm-libc-types/char16_t.h"
+
+#else // overlay mode
+
+// MacOS doesn't provide uchar.h so we use the types provided by LLVM-libc.
+#ifdef __APPLE__
+#include "include/llvm-libc-types/char16_t.h"
+#else
+#include "hdr/uchar_overlay.h"
+#endif
+
+#endif // LIBC_FULL_BUILD
+
+#endif // LLVM_LIBC_HDR_TYPES_CHAR16_T_H
diff --git a/libc/src/__support/wchar/CMakeLists.txt b/libc/src/__support/wchar/CMakeLists.txt
index c4e7f0dbaa178..91fd788fd0762 100644
--- a/libc/src/__support/wchar/CMakeLists.txt
+++ b/libc/src/__support/wchar/CMakeLists.txt
@@ -29,11 +29,13 @@ add_header_library(
DEPENDS
libc.hdr.errno_macros
libc.hdr.types.char8_t
+ libc.hdr.types.char16_t
libc.hdr.types.char32_t
libc.hdr.types.size_t
libc.src.__support.common
libc.src.__support.CPP.type_traits
libc.src.__support.error_or
+ libc.src.__support.libc_assert
libc.src.__support.macros.config
libc.src.__support.macros.properties.types
libc.src.__support.math_extras
diff --git a/libc/src/__support/wchar/character_converter.h b/libc/src/__support/wchar/character_converter.h
index f754a602d2cce..f7ccf6f18b36c 100644
--- a/libc/src/__support/wchar/character_converter.h
+++ b/libc/src/__support/wchar/character_converter.h
@@ -10,6 +10,7 @@
#define LLVM_LIBC_SRC___SUPPORT_CHARACTER_CONVERTER_H
#include "hdr/errno_macros.h"
+#include "hdr/types/char16_t.h"
#include "hdr/types/char32_t.h"
#include "hdr/types/char8_t.h"
#include "hdr/types/size_t.h"
@@ -18,6 +19,7 @@
#include "src/__support/CPP/type_traits.h"
#include "src/__support/common.h"
#include "src/__support/error_or.h"
+#include "src/__support/libc_assert.h"
#include "src/__support/macros/properties/types.h"
#include "src/__support/math_extras.h"
#include "src/__support/wchar/mbstate.h"
@@ -57,18 +59,65 @@ class CharacterConverter {
LIBC_INLINE bool isEmpty() { return state->bytes_stored == 0; }
bool isValidState();
+ /*
+ * When converting between UTF-16 and UTF-8, the converter might be in a state
+ * of either partially-pushing or partially-popping. For example consider the
+ * following sequence:
+ * * push((char8_t)0xF0);
+ * * push((char8_t)0xA4);
+ * * [A](partial=0x24,bytes_stored=2,total_bytes=4)
+ * * push((char8_t)0xAD);
+ * * push((char8_t)0xA2);
+ * * (partial=0x24B62,bytes_stored=4,total_bytes=4)
+ * * pop_utf16(); -> 0xD852
+ * * [B](partial=0x24B62,bytes_stored=2,total_bytes=4)
+ * * pop_utf16(); -> 0xDF62
+ * At point [A] and point [B], the stored bytes would both be 2. But at [A] we
+ * must continue reading input, while at [B] we must write the output low
+ * surrogate and clear the state.
+ * Thus we must have some means to distinguish between pushing and popping.
+ * Notice that: When pushing, we do left-shifts on state->partial before each
+ * push. When popping, we keep state->partial at the complete code point.
+ * Since we disallow overlong encodings, the complete UTF-8 code point is
+ * always greater than
+ * * MAX_VALUE_PER_UTF8_LEN[state->total_bytes - 2]
+ * but at most
+ * * MAX_VALUE_PER_UTF8_LEN[state->total_bytes - 1]
+ * If we are partially through pushing, state->partial is at most
+ * * char32 >> 6
+ * thus at most
+ * * MAX_VALUE_PER_UTF8_LEN[state->total_bytes - 1] >> 6
+ * Observe that, an item in MAX_VALUE_PER_UTF8_LEN right-shifted by 6 is
+ * always less than the preceding item. Based on these facts, we can determine
+ * if we are partially through pushing or popping, by comparing state->partial
+ * with MAX_VALUE_PER_UTF8_LEN[state->total_bytes - 2]
+ * Also when converting UTF-16 -> UTF-8, in a partial state,
+ * * bytes_stored=2
+ * * total_bytes=4
+ * so the threshold 0xFFFF is also correct.
+ */
+ LIBC_INLINE bool isPartiallyPopping() {
+ if (isEmpty() || isFull())
+ return false;
+ // When neither empty nor full, state->total_bytes is at least 2
+ char32_t threshold = MAX_VALUE_PER_UTF8_LEN[state->total_bytes - 2];
+ return state->partial > threshold;
+ }
+
template <typename CharType> size_t sizeAs();
int push(char8_t utf8_byte);
+ int push(char16_t utf16);
int push(char32_t utf32);
ErrorOr<char8_t> pop_utf8();
+ ErrorOr<char16_t> pop_utf16();
ErrorOr<char32_t> pop_utf32();
-#if defined(LIBC_TYPES_WCHAR_T_IS_UTF32)
+#if defined(LIBC_TYPES_WCHAR_T_IS_UTF32) || defined(LIBC_TYPES_WCHAR_T_IS_UTF16)
int push(wchar_t wchar);
ErrorOr<wchar_t> pop_wchar();
-#endif // LIBC_TYPES_WCHAR_T_IS_UTF32
+#endif // LIBC_TYPES_WCHAR_T_IS_UTF32 || LIBC_TYPES_WCHAR_T_IS_UTF16
template <typename CharType> ErrorOr<CharType> pop();
};
@@ -112,6 +161,11 @@ LIBC_INLINE int CharacterConverter::push(char8_t utf8_byte) {
state->bytes_stored++;
return 0;
}
+ // Pushing when partially popping is a misuse in the library
+ if (isPartiallyPopping()) {
+ clear();
+ return -1;
+ }
// Any subsequent push
// Adding 6 more bits so need to left shift
if (num_ones == 1 && !isFull()) {
@@ -119,6 +173,19 @@ LIBC_INLINE int CharacterConverter::push(char8_t utf8_byte) {
state->partial = state->partial << ENCODED_BITS_PER_UTF8;
state->partial |= byte;
state->bytes_stored++;
+ if (isFull()) {
+ // Detect and reject overlong encodings
+ // Needed for isPartiallyPopping to work
+ if (state->partial <= MAX_VALUE_PER_UTF8_LEN[state->total_bytes - 2]) {
+ clear();
+ return EILSEQ;
+ }
+ // Also reject out-of-range code points
+ if (state->partial > MAX_VALUE_PER_UTF8_LEN[state->total_bytes - 1]) {
+ clear();
+ return EILSEQ;
+ }
+ }
return 0;
}
@@ -127,6 +194,57 @@ LIBC_INLINE int CharacterConverter::push(char8_t utf8_byte) {
return EILSEQ;
}
+LIBC_INLINE int CharacterConverter::push(char16_t utf16) {
+ // Checking for high surrogate if first push
+ if (isEmpty()) {
+ bool is_surrogate = (utf16 & 0xF800) == 0xD800;
+ if (is_surrogate) {
+ bool is_high_surrogate = (utf16 & 0x400) == 0;
+ if (!is_high_surrogate) {
+ // unpaired surrogate -> reset the state
+ // bytes_stored and total_bytes will always be 0 here
+ state->partial = static_cast<char32_t>(0);
+ return EILSEQ;
+ }
+ // high surrogate
+ // code point between 0x10000 and 0x10FFFF
+ state->total_bytes = 4;
+ state->bytes_stored = 2;
+ // 10 bits each from the high surrogate and the low surrogate, and the
+ // full code point is from 0x10000, so we add 0x40 to partial, which after
+ // shifting becomes 0x10000
+ state->partial = static_cast<char32_t>(utf16 & 0x3FF) + 0x40;
+ } else {
+ state->partial = static_cast<char32_t>(utf16);
+ // determine number of utf-8 bytes needed to represent this utf16 value
+ uint8_t i;
+ for (i = 0; i < 2; i++) {
+ if (state->partial <= MAX_VALUE_PER_UTF8_LEN[i])
+ break;
+ }
+ state->total_bytes = i + 1;
+ state->bytes_stored = i + 1;
+ }
+ return 0;
+ }
+ // Pushing when partially popping is a misuse in the library
+ if (isFull() || isPartiallyPopping()) {
+ clear();
+ return -1;
+ }
+ bool is_low_surrogate = (utf16 & 0xFC00) == 0xDC00;
+ if (!is_low_surrogate) {
+ // unpaired surrogate -> reset the state
+ clear();
+ return EILSEQ;
+ }
+ char32_t low = utf16 & 0x3FF;
+ state->partial <<= 10;
+ state->partial |= low;
+ state->bytes_stored += 2;
+ return 0;
+}
+
LIBC_INLINE int CharacterConverter::push(char32_t utf32) {
// we can't be partially through a conversion when pushing a utf32 value
if (!isEmpty())
@@ -160,8 +278,36 @@ LIBC_INLINE ErrorOr<char32_t> CharacterConverter::pop_utf32() {
return utf32;
}
+LIBC_INLINE ErrorOr<char16_t> CharacterConverter::pop_utf16() {
+ if (!isFull() && !isPartiallyPopping())
+ return Error(-1);
+ char32_t output;
+ if (state->total_bytes <= 3) {
+ // no surrogates needed
+ output = state->partial;
+ // disallow using UTF-8 or UTF-32 to encode unpaired surrogates
+ bool is_surrogate = (output & 0xF800) == 0xD800;
+ if (is_surrogate) {
+ clear();
+ return Error(EILSEQ);
+ }
+ clear();
+ } else {
+ if (isFull()) {
+ // high surrogate first
+ output = ((state->partial - 0x10000) >> 10) | 0xD800;
+ state->bytes_stored = 2;
+ } else {
+ // low surrogate
+ output = ((state->partial - 0x10000) & 0x3FF) | 0xDC00;
+ clear();
+ }
+ }
+ return static_cast<char16_t>(output);
+}
+
LIBC_INLINE ErrorOr<char8_t> CharacterConverter::pop_utf8() {
- if (isEmpty())
+ if (!isFull() && !isPartiallyPopping())
return Error(-1);
constexpr char8_t FIRST_BYTE_HEADERS[] = {0, 0xC0, 0xE0, 0xF0};
@@ -196,6 +342,10 @@ template <> LIBC_INLINE ErrorOr<char8_t> CharacterConverter::pop() {
return pop_utf8();
}
+template <> LIBC_INLINE ErrorOr<char16_t> CharacterConverter::pop() {
+ return pop_utf16();
+}
+
template <> LIBC_INLINE ErrorOr<char32_t> CharacterConverter::pop() {
return pop_utf32();
}
@@ -204,6 +354,10 @@ template <> LIBC_INLINE size_t CharacterConverter::sizeAs<char8_t>() {
return state->total_bytes;
}
+template <> LIBC_INLINE size_t CharacterConverter::sizeAs<char16_t>() {
+ return state->total_bytes > 3 ? 2 : 1;
+}
+
template <> LIBC_INLINE size_t CharacterConverter::sizeAs<char32_t>() {
return 1;
}
@@ -229,7 +383,28 @@ template <> LIBC_INLINE size_t CharacterConverter::sizeAs<wchar_t>() {
return sizeAs<char32_t>();
}
-#endif // LIBC_TYPES_WCHAR_T_IS_UTF32
+#elif defined(LIBC_TYPES_WCHAR_T_IS_UTF16)
+
+LIBC_INLINE int CharacterConverter::push(wchar_t wchar) {
+ return push(static_cast<char16_t>(wchar));
+}
+
+LIBC_INLINE ErrorOr<wchar_t> CharacterConverter::pop_wchar() {
+ ErrorOr<char16_t> Result = pop_utf16();
+ if (!Result)
+ return Error(Result.error());
+ return static_cast<wchar_t>(*Result);
+}
+
+template <> LIBC_INLINE ErrorOr<wchar_t> CharacterConverter::pop() {
+ return pop_wchar();
+}
+
+template <> LIBC_INLINE size_t CharacterConverter::sizeAs<wchar_t>() {
+ return sizeAs<char16_t>();
+}
+
+#endif // LIBC_TYPES_WCHAR_T_IS_UTF32 || LIBC_TYPES_WCHAR_T_IS_UTF16
} // namespace internal
} // namespace LIBC_NAMESPACE_DECL
diff --git a/libc/test/src/__support/wchar/CMakeLists.txt b/libc/test/src/__support/wchar/CMakeLists.txt
index 259e896ee835d..907950833dbcc 100644
--- a/libc/test/src/__support/wchar/CMakeLists.txt
+++ b/libc/test/src/__support/wchar/CMakeLists.txt
@@ -41,6 +41,32 @@ add_libc_test(
libc.src.__support.wchar.mbstate
)
+add_libc_test(
+ utf8_to_16_test
+ SUITE
+ libc-support-tests
+ SRCS
+ utf8_to_16_test.cpp
+ DEPENDS
+ libc.src.__support.error_or
+ libc.src.__support.macros.properties.types
+ libc.src.__support.wchar.character_converter
+ libc.src.__support.wchar.mbstate
+)
+
+add_libc_test(
+ utf16_to_8_test
+ SUITE
+ libc-support-tests
+ SRCS
+ utf16_to_8_test.cpp
+ DEPENDS
+ libc.src.__support.error_or
+ libc.src.__support.macros.properties.types
+ libc.src.__support.wchar.character_converter
+ libc.src.__support.wchar.mbstate
+)
+
add_libc_test(
wcsnrtombs_test
SUITE
diff --git a/libc/test/src/__support/wchar/utf16_to_8_test.cpp b/libc/test/src/__support/wchar/utf16_to_8_test.cpp
new file mode 100644
index 0000000000000..737f17595ce10
--- /dev/null
+++ b/libc/test/src/__support/wchar/utf16_to_8_test.cpp
@@ -0,0 +1,235 @@
+//===-- Unittests for the CharacterConverter class (utf16 -> 8) -----------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#include "src/__support/common.h"
+#include "src/__support/macros/properties/types.h"
+#include "src/__support/wchar/character_converter.h"
+#include "src/__support/wchar/mbstate.h"
+
+#include "test/UnitTest/Test.h"
+
+#if defined(LIBC_TYPES_WCHAR_T_IS_UTF16)
+using TestCharTypesUTF16 = LIBC_NAMESPACE::testing::TypeList<char16_t, wchar_t>;
+#else
+using TestCharTypesUTF16 = LIBC_NAMESPACE::testing::TypeList<char16_t>;
+#endif
+
+TYPED_TEST(LlvmLibcCharacterConverterUTF16To8Test, OneByte,
+ TestCharTypesUTF16) {
+ using CharType16 = ParamType;
+
+ LIBC_NAMESPACE::internal::mbstate state;
+ LIBC_NAMESPACE::internal::CharacterConverter cr(&state);
+ cr.clear();
+
+ // utf8 1-byte encodings are identical to their utf16 representations
+ CharType16 utf16_A = 0x41; // 'A'
+ cr.push(utf16_A);
+ ASSERT_TRUE(cr.isFull());
+ auto popped = cr.pop_utf8();
+ ASSERT_TRUE(popped.has_value());
+ ASSERT_EQ(static_cast<char>(popped.value()), 'A');
+ ASSERT_TRUE(cr.isEmpty());
+
+ CharType16 utf16_B = 0x42; // 'B'
+ cr.push(utf16_B);
+ ASSERT_TRUE(cr.isFull());
+ popped = cr.pop_utf8();
+ ASSERT_TRUE(popped.has_value());
+ ASSERT_EQ(static_cast<char>(popped.value()), 'B');
+ ASSERT_TRUE(cr.isEmpty());
+
+ // should error if we try to pop another utf8 byte out
+ popped = cr.pop_utf8();
+ ASSERT_FALSE(popped.has_value());
+}
+
+TYPED_TEST(LlvmLibcCharacterConverterUTF16To8Test, TwoByte,
+ TestCharTypesUTF16) {
+ using CharType16 = ParamType;
+
+ LIBC_NAMESPACE::internal::mbstate state;
+ LIBC_NAMESPACE::internal::CharacterConverter cr(&state);
+ cr.clear();
+
+ // testing utf16: 0xff -> utf8: 0xc3 0xbf
+ CharType16 utf16 = 0xff;
+ cr.push(utf16);
+ ASSERT_TRUE(cr.isFull());
+ auto popped = cr.pop_utf8();
+ ASSERT_TRUE(popped.has_value());
+ ASSERT_EQ(static_cast<int>(popped.value()), 0xc3);
+ ASSERT_TRUE(cr.isPartiallyPopping());
+ popped = cr.pop_utf8();
+ ASSERT_TRUE(popped.has_value());
+ ASSERT_EQ(static_cast<int>(popped.value()), 0xbf);
+ ASSERT_TRUE(cr.isEmpty());
+
+ // testing utf16: 0x58e -> utf8: 0xd6 0x8e
+ utf16 = 0x58e;
+ cr.push(utf16);
+ ASSERT_TRUE(cr.isFull());
+ popped = cr.pop_utf8();
+ ASSERT_TRUE(popped.has_value());
+ ASSERT_EQ(static_cast<int>(popped.value()), 0xd6);
+ ASSERT_TRUE(cr.isPartiallyPopping());
+ popped = cr.pop_utf8();
+ ASSERT_TRUE(popped.has_value());
+ ASSERT_EQ(static_cast<int>(popped.value()), 0x8e);
+ ASSERT_TRUE(cr.isEmpty());
+
+ // should error if we try to pop another utf8 byte out
+ popped = cr.pop_utf8();
+ ASSERT_FALSE(popped.has_value());
+}
+
+TYPED_TEST(LlvmLibcCharacterConverterUTF16To8Test, ThreeByte,
+ TestCharTypesUTF16) {
+ using CharType16 = ParamType;
+
+ LIBC_NAMESPACE::internal::mbstate state;
+ LIBC_NAMESPACE::internal::CharacterConverter cr(&state);
+ cr.clear();
+
+ // testing utf16: 0xac15 -> utf8: 0xea 0xb0 0x95
+ CharType16 utf16 = 0xac15;
+ cr.push(utf16);
+ ASSERT_TRUE(cr.isFull());
+ auto popped = cr.pop_utf8();
+ ASSERT_TRUE(popped.has_value());
+ ASSERT_EQ(static_cast<int>(popped.value()), 0xea);
+ ASSERT_TRUE(cr.isPartiallyPopping());
+ popped = cr.pop_utf8();
+ ASSERT_TRUE(popped.has_value());
+ ASSERT_EQ(static_cast<int>(popped.value()), 0xb0);
+ ASSERT_TRUE(cr.isPartiallyPopping());
+ popped = cr.pop_utf8();
+ ASSERT_TRUE(popped.has_value());
+ ASSERT_EQ(static_cast<int>(popped.value()), 0x95);
+ ASSERT_TRUE(cr.isEmpty());
+
+ // testing utf16: 0x267b -> utf8: 0xe2 0x99 0xbb
+ utf16 = 0x267b;
+ cr.push(utf16);
+ ASSERT_TRUE(cr.isFull());
+ popped = cr.pop_utf8();
+ ASSERT_TRUE(popped.has_value());
+ ASSERT_EQ(static_cast<int>(popped.value()), 0xe2);
+ ASSERT_TRUE(cr.isPartiallyPopping());
+ popped = cr.pop_utf8();
+ ASSERT_TRUE(popped.has_value());
+ ASSERT_EQ(static_cast<int>(popped.value()), 0x99);
+ ASSERT_TRUE(cr.isPartiallyPopping());
+ popped = cr.pop_utf8();
+ ASSERT_TRUE(popped.has_value());
+ ASSERT_EQ(static_cast<int>(popped.value()), 0xbb);
+ ASSERT_TRUE(cr.isEmpty());
+
+ // should error if we try to pop another utf8 byte out
+ popped = cr.pop_utf8();
+ ASSERT_FALSE(popped.has_value());
+}
+
+TYPED_TEST(LlvmLibcCharacterConverterUTF16To8Test, FourByte,
+ TestCharTypesUTF16) {
+ using CharType16 = ParamType;
+
+ LIBC_NAMESPACE::internal::mbstate state;
+ LIBC_NAMESPACE::internal::CharacterConverter cr(&state);
+ cr.clear();
+
+ // testing utf16: 0xd83e 0xdd21 -> utf8: 0xf0 0x9f 0xa4 0xa1
+ CharType16 utf16_h = 0xd83e;
+ cr.push(utf16_h);
+ ASSERT_FALSE(cr.isPartiallyPopping());
+ CharType16 utf16_l = 0xdd21;
+ cr.push(utf16_l);
+ ASSERT_TRUE(cr.isFull());
+ auto popped = cr.pop_utf8();
+ ASSERT_TRUE(popped.has_value());
+ ASSERT_EQ(static_cast<int>(popped.value()), 0xf0);
+ ASSERT_TRUE(cr.isPartiallyPopping());
+ popped = cr.pop_utf8();
+ ASSERT_TRUE(popped.has_value());
+ ASSERT_EQ(static_cast<int>(popped.value()), 0x9f);
+ ASSERT_TRUE(cr.isPartiallyPopping());
+ popped = cr.pop_utf8();
+ ASSERT_TRUE(popped.has_value());
+ ASSERT_EQ(static_cast<int>(popped.value()), 0xa4);
+ ASSERT_TRUE(cr.isPartiallyPopping());
+ popped = cr.pop_utf8();
+ ASSERT_TRUE(popped.has_value());
+ ASSERT_EQ(static_cast<int>(popped.value()), 0xa1);
+ ASSERT_TRUE(cr.isEmpty());
+
+ // testing utf16: 0xd808 0xdd21 -> utf8: 0xf0 0x92 0x84 0xa1
+ utf16_h = 0xd808;
+ cr.push(utf16_h);
+ ASSERT_FALSE(cr.isPartiallyPopping());
+ utf16_l = 0xdd21;
+ cr.push(utf16_l);
+ ASSERT_TRUE(cr.isFull());
+ popped = cr.pop_utf8();
+ ASSERT_TRUE(popped.has_value());
+ ASSERT_EQ(static_cast<int>(popped.value()), 0xf0);
+ ASSERT_TRUE(cr.isPartiallyPopping());
+ popped = cr.pop_utf8();
+ ASSERT_TRUE(popped.has_value());
+ ASSERT_EQ(static_cast<int>(popped.value()), 0x92);
+ ASSERT_TRUE(cr.isPartiallyPopping());
+ popped = cr.pop_utf8();
+ ASSERT_TRUE(popped.has_value());
+ ASSERT_EQ(static_cast<int>(popped.value()), 0x84);
+ ASSERT_TRUE(cr.isPartiallyPopping());
+ popped = cr.pop_utf8();
+ ASSERT_TRUE(popped.has_value());
+ ASSERT_EQ(static_cast<int>(popped.value()), 0xa1);
+ ASSERT_TRUE(cr.isEmpty());
+
+ // should error if we try to pop another utf8 byte out
+ popped = cr.pop_utf8();
+ ASSERT_FALSE(popped.has_value());
+}
+
+TYPED_TEST(LlvmLibcCharacterConverterUTF16To8Test, CantPushMidConversion,
+ TestCharTypesUTF16) {
+ using CharType16 = ParamType;
+
+ LIBC_NAMESPACE::internal::mbstate state;
+ LIBC_NAMESPACE::internal::CharacterConverter cr(&state);
+ cr.clear();
+
+ // testing utf16: 0xd808 0xdd21 -> utf8: 0xf0 0x92 0x84 0xa1
+ CharType16 utf16_h = 0xd808;
+ ASSERT_EQ(cr.push(utf16_h), 0);
+ CharType16 utf16_l = 0xdd21;
+ ASSERT_EQ(cr.push(utf16_l), 0);
+ auto popped = cr.pop_utf8();
+ ASSERT_TRUE(popped.has_value());
+
+ // can't push a utf16 without finishing popping the utf8 bytes out
+ int err = cr.push(utf16_l);
+ ASSERT_EQ(err, -1);
+}
+
+TYPED_TEST(LlvmLibcCharacterConverterUTF16To8Test, CantPopMidPushing,
+ TestCharTypesUTF16) {
+ using CharType16 = ParamType;
+
+ LIBC_NAMESPACE::internal::mbstate state;
+ LIBC_NAMESPACE::internal::CharacterConverter cr(&state);
+ cr.clear();
+
+ // testing utf16: 0xd808 0xdd21 -> utf8: 0xf0 0x92 0x84 0xa1
+ CharType16 utf16_h = 0xd808;
+ ASSERT_EQ(cr.push(utf16_h), 0);
+
+ // can't pop a utf8 without finishing pushing the utf16 code units
+ auto popped = cr.pop_utf8();
+ ASSERT_FALSE(popped.has_value());
+}
diff --git a/libc/test/src/__support/wchar/utf32_to_8_test.cpp b/libc/test/src/__support/wchar/utf32_to_8_test.cpp
index 52e10e1191175..bc911050cff35 100644
--- a/libc/test/src/__support/wchar/utf32_to_8_test.cpp
+++ b/libc/test/src/__support/wchar/utf32_to_8_test.cpp
@@ -64,7 +64,7 @@ TYPED_TEST(LlvmLibcCharacterConverterUTF32To8Test, TwoByte,
auto popped = cr.pop_utf8();
ASSERT_TRUE(popped.has_value());
ASSERT_EQ(static_cast<int>(popped.value()), 0xc3);
- ASSERT_TRUE(!cr.isEmpty());
+ ASSERT_TRUE(cr.isPartiallyPopping());
popped = cr.pop_utf8();
ASSERT_TRUE(popped.has_value());
ASSERT_EQ(static_cast<int>(popped.value()), 0xbf);
@@ -77,7 +77,7 @@ TYPED_TEST(LlvmLibcCharacterConverterUTF32To8Test, TwoByte,
popped = cr.pop_utf8();
ASSERT_TRUE(popped.has_value());
ASSERT_EQ(static_cast<int>(popped.value()), 0xd6);
- ASSERT_TRUE(!cr.isEmpty());
+ ASSERT_TRUE(cr.isPartiallyPopping());
popped = cr.pop_utf8();
ASSERT_TRUE(popped.has_value());
ASSERT_EQ(static_cast<int>(popped.value()), 0x8e);
@@ -103,11 +103,11 @@ TYPED_TEST(LlvmLibcCharacterConverterUTF32To8Test, ThreeByte,
auto popped = cr.pop_utf8();
ASSERT_TRUE(popped.has_value());
ASSERT_EQ(static_cast<int>(popped.value()), 0xea);
- ASSERT_TRUE(!cr.isEmpty());
+ ASSERT_TRUE(cr.isPartiallyPopping());
popped = cr.pop_utf8();
ASSERT_TRUE(popped.has_value());
ASSERT_EQ(static_cast<int>(popped.value()), 0xb0);
- ASSERT_TRUE(!cr.isEmpty());
+ ASSERT_TRUE(cr.isPartiallyPopping());
popped = cr.pop_utf8();
ASSERT_TRUE(popped.has_value());
ASSERT_EQ(static_cast<int>(popped.value()), 0x95);
@@ -120,11 +120,11 @@ TYPED_TEST(LlvmLibcCharacterConverterUTF32To8Test, ThreeByte,
popped = cr.pop_utf8();
ASSERT_TRUE(popped.has_value());
ASSERT_EQ(static_cast<int>(popped.value()), 0xe2);
- ASSERT_TRUE(!cr.isEmpty());
+ ASSERT_TRUE(cr.isPartiallyPopping());
popped = cr.pop_utf8();
ASSERT_TRUE(popped.has_value());
ASSERT_EQ(static_cast<int>(popped.value()), 0x99);
- ASSERT_TRUE(!cr.isEmpty());
+ ASSERT_TRUE(cr.isPartiallyPopping());
popped = cr.pop_utf8();
ASSERT_TRUE(popped.has_value());
ASSERT_EQ(static_cast<int>(popped.value()), 0xbb);
@@ -150,15 +150,15 @@ TYPED_TEST(LlvmLibcCharacterConverterUTF32To8Test, FourByte,
auto popped = cr.pop_utf8();
ASSERT_TRUE(popped.has_value());
ASSERT_EQ(static_cast<int>(popped.value()), 0xf0);
- ASSERT_TRUE(!cr.isEmpty());
+ ASSERT_TRUE(cr.isPartiallyPopping());
popped = cr.pop_utf8();
ASSERT_TRUE(popped.has_value());
ASSERT_EQ(static_cast<int>(popped.value()), 0x9f);
- ASSERT_TRUE(!cr.isEmpty());
+ ASSERT_TRUE(cr.isPartiallyPopping());
popped = cr.pop_utf8();
ASSERT_TRUE(popped.has_value());
ASSERT_EQ(static_cast<int>(popped.value()), 0xa4);
- ASSERT_TRUE(!cr.isEmpty());
+ ASSERT_TRUE(cr.isPartiallyPopping());
popped = cr.pop_utf8();
ASSERT_TRUE(popped.has_value());
ASSERT_EQ(static_cast<int>(popped.value()), 0xa1);
@@ -171,15 +171,15 @@ TYPED_TEST(LlvmLibcCharacterConverterUTF32To8Test, FourByte,
popped = cr.pop_utf8();
ASSERT_TRUE(popped.has_value());
ASSERT_EQ(static_cast<int>(popped.value()), 0xf0);
- ASSERT_TRUE(!cr.isEmpty());
+ ASSERT_TRUE(cr.isPartiallyPopping());
popped = cr.pop_utf8();
ASSERT_TRUE(popped.has_value());
ASSERT_EQ(static_cast<int>(popped.value()), 0x92);
- ASSERT_TRUE(!cr.isEmpty());
+ ASSERT_TRUE(cr.isPartiallyPopping());
popped = cr.pop_utf8();
ASSERT_TRUE(popped.has_value());
ASSERT_EQ(static_cast<int>(popped.value()), 0x84);
- ASSERT_TRUE(!cr.isEmpty());
+ ASSERT_TRUE(cr.isPartiallyPopping());
popped = cr.pop_utf8();
ASSERT_TRUE(popped.has_value());
ASSERT_EQ(static_cast<int>(popped.value()), 0xa1);
@@ -213,6 +213,8 @@ TEST(LlvmLibcCharacterConverterUTF32To8Test, InvalidState) {
LIBC_NAMESPACE::internal::mbstate s1;
LIBC_NAMESPACE::internal::CharacterConverter c1(&s1);
ASSERT_TRUE(c1.isValidState());
+ ASSERT_TRUE(c1.isEmpty());
+ ASSERT_FALSE(c1.isPartiallyPopping());
LIBC_NAMESPACE::internal::mbstate s2{0, 2, 0};
LIBC_NAMESPACE::internal::CharacterConverter c2(&s2);
diff --git a/libc/test/src/__support/wchar/utf8_to_16_test.cpp b/libc/test/src/__support/wchar/utf8_to_16_test.cpp
new file mode 100644
index 0000000000000..ed38ddb7596b8
--- /dev/null
+++ b/libc/test/src/__support/wchar/utf8_to_16_test.cpp
@@ -0,0 +1,183 @@
+//===-- Unittests for character_converter utf8->utf16 ---------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#include "hdr/errno_macros.h"
+#include "src/__support/error_or.h"
+#include "src/__support/macros/properties/types.h"
+#include "src/__support/wchar/character_converter.h"
+#include "src/__support/wchar/mbstate.h"
+#include "test/UnitTest/Test.h"
+
+#if defined(LIBC_TYPES_WCHAR_T_IS_UTF16)
+using TestCharTypesUTF16 = LIBC_NAMESPACE::testing::TypeList<char16_t, wchar_t>;
+#else
+using TestCharTypesUTF16 = LIBC_NAMESPACE::testing::TypeList<char16_t>;
+#endif
+
+TYPED_TEST(LlvmLibcCharacterConverterUTF8To16Test, OneByte,
+ TestCharTypesUTF16) {
+ using CharType16 = ParamType;
+
+ LIBC_NAMESPACE::internal::mbstate state;
+ state.bytes_stored = 0;
+ state.total_bytes = 0;
+ char ch = 'A';
+
+ LIBC_NAMESPACE::internal::CharacterConverter char_conv(&state);
+ int err = char_conv.push(static_cast<char8_t>(ch));
+ auto wch = char_conv.pop<CharType16>();
+
+ ASSERT_EQ(err, 0);
+ ASSERT_TRUE(wch.has_value());
+ ASSERT_EQ(static_cast<int>(wch.value()), 65);
+}
+
+TYPED_TEST(LlvmLibcCharacterConverterUTF8To16Test, TwoBytes,
+ TestCharTypesUTF16) {
+ using CharType16 = ParamType;
+
+ LIBC_NAMESPACE::internal::mbstate state;
+ state.bytes_stored = 0;
+ state.total_bytes = 0;
+ const char ch[2] = {static_cast<char>(0xC2),
+ static_cast<char>(0x8E)}; // car symbol
+
+ LIBC_NAMESPACE::internal::CharacterConverter char_conv(&state);
+ char_conv.push(static_cast<char8_t>(ch[0]));
+ ASSERT_FALSE(char_conv.isPartiallyPopping());
+ char_conv.push(static_cast<char8_t>(ch[1]));
+ auto wch = char_conv.pop<CharType16>();
+
+ ASSERT_TRUE(wch.has_value());
+ ASSERT_EQ(static_cast<int>(wch.value()), 142);
+}
+
+TYPED_TEST(LlvmLibcCharacterConverterUTF8To16Test, ThreeBytes,
+ TestCharTypesUTF16) {
+ using CharType16 = ParamType;
+
+ LIBC_NAMESPACE::internal::mbstate state;
+ state.bytes_stored = 0;
+ state.total_bytes = 0;
+ const char ch[3] = {static_cast<char>(0xE2), static_cast<char>(0x88),
+ static_cast<char>(0x91)}; // ∑ sigma symbol
+
+ LIBC_NAMESPACE::internal::CharacterConverter char_conv(&state);
+ char_conv.push(static_cast<char8_t>(ch[0]));
+ ASSERT_FALSE(char_conv.isPartiallyPopping());
+ char_conv.push(static_cast<char8_t>(ch[1]));
+ ASSERT_FALSE(char_conv.isPartiallyPopping());
+ char_conv.push(static_cast<char8_t>(ch[2]));
+ auto wch = char_conv.pop<CharType16>();
+
+ ASSERT_TRUE(wch.has_value());
+ ASSERT_EQ(static_cast<int>(wch.value()), 8721);
+}
+
+TYPED_TEST(LlvmLibcCharacterConverterUTF8To16Test, FourBytes,
+ TestCharTypesUTF16) {
+ using CharType16 = ParamType;
+
+ LIBC_NAMESPACE::internal::mbstate state;
+ state.bytes_stored = 0;
+ state.total_bytes = 0;
+ const char ch[4] = {static_cast<char>(0xF0), static_cast<char>(0x9F),
+ static_cast<char>(0xA4),
+ static_cast<char>(0xA1)}; // 🤡 clown emoji
+
+ LIBC_NAMESPACE::internal::CharacterConverter char_conv(&state);
+ char_conv.push(static_cast<char8_t>(ch[0]));
+ ASSERT_FALSE(char_conv.isPartiallyPopping());
+ char_conv.push(static_cast<char8_t>(ch[1]));
+ ASSERT_FALSE(char_conv.isPartiallyPopping());
+ char_conv.push(static_cast<char8_t>(ch[2]));
+ ASSERT_FALSE(char_conv.isPartiallyPopping());
+ char_conv.push(static_cast<char8_t>(ch[3]));
+ auto wch_h = char_conv.pop<CharType16>();
+ ASSERT_TRUE(char_conv.isPartiallyPopping());
+ auto wch_l = char_conv.pop<CharType16>();
+
+ ASSERT_TRUE(wch_h.has_value());
+ ASSERT_EQ(static_cast<int>(wch_h.value()), 55358);
+ ASSERT_TRUE(wch_l.has_value());
+ ASSERT_EQ(static_cast<int>(wch_l.value()), 56609);
+}
+
+TYPED_TEST(LlvmLibcCharacterConverterUTF8To16Test, ValidTwoByteWithExtraRead,
+ TestCharTypesUTF16) {
+ using CharType16 = ParamType;
+
+ LIBC_NAMESPACE::internal::mbstate state;
+ state.bytes_stored = 0;
+ state.total_bytes = 0;
+ const char ch[3] = {static_cast<char>(0xC2), static_cast<char>(0x8E),
+ static_cast<char>(0x80)};
+
+ LIBC_NAMESPACE::internal::CharacterConverter char_conv(&state);
+ int err = char_conv.push(static_cast<char8_t>(ch[0]));
+ ASSERT_EQ(err, 0);
+ err = char_conv.push(static_cast<char8_t>(ch[1]));
+ ASSERT_EQ(err, 0);
+ // Should produce an error on 3rd byte
+ err = char_conv.push(static_cast<char8_t>(ch[2]));
+ ASSERT_EQ(err, EILSEQ);
+
+ // Should produce an error since mbstate was reset
+ auto wch = char_conv.pop<CharType16>();
+ ASSERT_FALSE(wch.has_value());
+}
+
+TYPED_TEST(LlvmLibcCharacterConverterUTF8To16Test, TwoValidTwoBytes,
+ TestCharTypesUTF16) {
+ using CharType16 = ParamType;
+
+ LIBC_NAMESPACE::internal::mbstate state;
+ state.bytes_stored = 0;
+ state.total_bytes = 0;
+ const char ch[4] = {static_cast<char>(0xC2), static_cast<char>(0x8E),
+ static_cast<char>(0xC7), static_cast<char>(0x8C)};
+
+ LIBC_NAMESPACE::internal::CharacterConverter char_conv(&state);
+ int err = char_conv.push(static_cast<char8_t>(ch[0]));
+ ASSERT_EQ(err, 0);
+ err = char_conv.push(static_cast<char8_t>(ch[1]));
+ ASSERT_EQ(err, 0);
+ auto wch = char_conv.pop<CharType16>();
+ ASSERT_TRUE(wch.has_value());
+ ASSERT_EQ(static_cast<int>(wch.value()), 142);
+
+ // Second two byte character
+ err = char_conv.push(static_cast<char8_t>(ch[2]));
+ ASSERT_EQ(err, 0);
+ err = char_conv.push(static_cast<char8_t>(ch[3]));
+ ASSERT_EQ(err, 0);
+ wch = char_conv.pop<CharType16>();
+ ASSERT_TRUE(wch.has_value());
+ ASSERT_EQ(static_cast<int>(wch.value()), 460);
+}
+
+TYPED_TEST(LlvmLibcCharacterConverterUTF8To16Test, InvalidPop,
+ TestCharTypesUTF16) {
+ using CharType16 = ParamType;
+
+ LIBC_NAMESPACE::internal::mbstate state;
+ state.bytes_stored = 0;
+ state.total_bytes = 0;
+ LIBC_NAMESPACE::internal::CharacterConverter char_conv(&state);
+ const char ch[2] = {static_cast<char>(0xC2), static_cast<char>(0x8E)};
+ int err = char_conv.push(static_cast<char8_t>(ch[0]));
+ ASSERT_EQ(err, 0);
+ auto wch = char_conv.pop<CharType16>();
+ ASSERT_FALSE(
+ wch.has_value()); // Should fail since we have not read enough bytes
+ err = char_conv.push(static_cast<char8_t>(ch[1]));
+ ASSERT_EQ(err, 0);
+ wch = char_conv.pop<CharType16>();
+ ASSERT_TRUE(wch.has_value());
+ ASSERT_EQ(static_cast<int>(wch.value()), 142);
+}
diff --git a/libc/test/src/__support/wchar/utf8_to_32_test.cpp b/libc/test/src/__support/wchar/utf8_to_32_test.cpp
index db74cddafcbd7..8e72371ca7c06 100644
--- a/libc/test/src/__support/wchar/utf8_to_32_test.cpp
+++ b/libc/test/src/__support/wchar/utf8_to_32_test.cpp
@@ -49,6 +49,7 @@ TYPED_TEST(LlvmLibcCharacterConverterUTF8To32Test, TwoBytes,
LIBC_NAMESPACE::internal::CharacterConverter char_conv(&state);
char_conv.push(static_cast<char8_t>(ch[0]));
+ ASSERT_FALSE(char_conv.isPartiallyPopping());
char_conv.push(static_cast<char8_t>(ch[1]));
auto wch = char_conv.pop<CharType32>();
@@ -68,7 +69,9 @@ TYPED_TEST(LlvmLibcCharacterConverterUTF8To32Test, ThreeBytes,
LIBC_NAMESPACE::internal::CharacterConverter char_conv(&state);
char_conv.push(static_cast<char8_t>(ch[0]));
+ ASSERT_FALSE(char_conv.isPartiallyPopping());
char_conv.push(static_cast<char8_t>(ch[1]));
+ ASSERT_FALSE(char_conv.isPartiallyPopping());
char_conv.push(static_cast<char8_t>(ch[2]));
auto wch = char_conv.pop<CharType32>();
@@ -89,8 +92,11 @@ TYPED_TEST(LlvmLibcCharacterConverterUTF8To32Test, FourBytes,
LIBC_NAMESPACE::internal::CharacterConverter char_conv(&state);
char_conv.push(static_cast<char8_t>(ch[0]));
+ ASSERT_FALSE(char_conv.isPartiallyPopping());
char_conv.push(static_cast<char8_t>(ch[1]));
+ ASSERT_FALSE(char_conv.isPartiallyPopping());
char_conv.push(static_cast<char8_t>(ch[2]));
+ ASSERT_FALSE(char_conv.isPartiallyPopping());
char_conv.push(static_cast<char8_t>(ch[3]));
auto wch = char_conv.pop<CharType32>();
@@ -223,3 +229,136 @@ TYPED_TEST(LlvmLibcCharacterConverterUTF8To32Test, InvalidPop,
ASSERT_TRUE(wch.has_value());
ASSERT_EQ(static_cast<int>(wch.value()), 142);
}
+
+TYPED_TEST(LlvmLibcCharacterConverterUTF8To32Test, RejectOverlongTwoBytes,
+ TestCharTypesUTF32) {
+ using CharType32 = ParamType;
+
+ LIBC_NAMESPACE::internal::mbstate state;
+ state.bytes_stored = 0;
+ state.total_bytes = 0;
+ const char ch[4] = {static_cast<char>(0xC1), static_cast<char>(0xBF),
+ static_cast<char>(0xC2), static_cast<char>(0x80)};
+
+ LIBC_NAMESPACE::internal::CharacterConverter char_conv(&state);
+ // Overlong encoding should be rejected U+007F
+ int err = char_conv.push(static_cast<char8_t>(ch[0]));
+ ASSERT_EQ(err, 0);
+ err = char_conv.push(static_cast<char8_t>(ch[1]));
+ ASSERT_EQ(err, EILSEQ);
+
+ // Not overlong, don't reject U+0080
+ err = char_conv.push(static_cast<char8_t>(ch[2]));
+ ASSERT_EQ(err, 0);
+ err = char_conv.push(static_cast<char8_t>(ch[3]));
+ ASSERT_EQ(err, 0);
+ auto wch = char_conv.pop<CharType32>();
+ ASSERT_TRUE(wch.has_value());
+ ASSERT_EQ(static_cast<int>(wch.value()), 128);
+}
+
+TYPED_TEST(LlvmLibcCharacterConverterUTF8To32Test, RejectOverlongThreeBytes,
+ TestCharTypesUTF32) {
+ using CharType32 = ParamType;
+
+ LIBC_NAMESPACE::internal::mbstate state;
+ state.bytes_stored = 0;
+ state.total_bytes = 0;
+ const char ch[6] = {static_cast<char>(0xE0), static_cast<char>(0x9F),
+ static_cast<char>(0xBF), static_cast<char>(0xE0),
+ static_cast<char>(0xA0), static_cast<char>(0x80)};
+
+ LIBC_NAMESPACE::internal::CharacterConverter char_conv(&state);
+ // Overlong encoding should be rejected U+07FF
+ int err = char_conv.push(static_cast<char8_t>(ch[0]));
+ ASSERT_EQ(err, 0);
+ err = char_conv.push(static_cast<char8_t>(ch[1]));
+ ASSERT_EQ(err, 0);
+ err = char_conv.push(static_cast<char8_t>(ch[2]));
+ ASSERT_EQ(err, EILSEQ);
+
+ // Not overlong, don't reject U+0800
+ err = char_conv.push(static_cast<char8_t>(ch[3]));
+ ASSERT_EQ(err, 0);
+ err = char_conv.push(static_cast<char8_t>(ch[4]));
+ ASSERT_EQ(err, 0);
+ err = char_conv.push(static_cast<char8_t>(ch[5]));
+ ASSERT_EQ(err, 0);
+ auto wch = char_conv.pop<CharType32>();
+ ASSERT_TRUE(wch.has_value());
+ ASSERT_EQ(static_cast<int>(wch.value()), 2048);
+}
+
+TYPED_TEST(LlvmLibcCharacterConverterUTF8To32Test, RejectOverlongFourBytes,
+ TestCharTypesUTF32) {
+ using CharType32 = ParamType;
+
+ LIBC_NAMESPACE::internal::mbstate state;
+ state.bytes_stored = 0;
+ state.total_bytes = 0;
+ const char ch[8] = {static_cast<char>(0xF0), static_cast<char>(0x8F),
+ static_cast<char>(0xBF), static_cast<char>(0xBF),
+ static_cast<char>(0xF0), static_cast<char>(0x90),
+ static_cast<char>(0x80), static_cast<char>(0x80)};
+
+ LIBC_NAMESPACE::internal::CharacterConverter char_conv(&state);
+ // Overlong encoding should be rejected U+FFFF
+ int err = char_conv.push(static_cast<char8_t>(ch[0]));
+ ASSERT_EQ(err, 0);
+ err = char_conv.push(static_cast<char8_t>(ch[1]));
+ ASSERT_EQ(err, 0);
+ err = char_conv.push(static_cast<char8_t>(ch[2]));
+ ASSERT_EQ(err, 0);
+ err = char_conv.push(static_cast<char8_t>(ch[3]));
+ ASSERT_EQ(err, EILSEQ);
+
+ // Not overlong, don't reject U+10000
+ err = char_conv.push(static_cast<char8_t>(ch[4]));
+ ASSERT_EQ(err, 0);
+ err = char_conv.push(static_cast<char8_t>(ch[5]));
+ ASSERT_EQ(err, 0);
+ err = char_conv.push(static_cast<char8_t>(ch[6]));
+ ASSERT_EQ(err, 0);
+ err = char_conv.push(static_cast<char8_t>(ch[7]));
+ ASSERT_EQ(err, 0);
+ auto wch = char_conv.pop<CharType32>();
+ ASSERT_TRUE(wch.has_value());
+ ASSERT_EQ(static_cast<int>(wch.value()), 65536);
+}
+
+TYPED_TEST(LlvmLibcCharacterConverterUTF8To32Test, RejectOutOfRange,
+ TestCharTypesUTF32) {
+ using CharType32 = ParamType;
+
+ LIBC_NAMESPACE::internal::mbstate state;
+ state.bytes_stored = 0;
+ state.total_bytes = 0;
+ const char ch[8] = {static_cast<char>(0xF4), static_cast<char>(0x90),
+ static_cast<char>(0x80), static_cast<char>(0x80),
+ static_cast<char>(0xF4), static_cast<char>(0x8F),
+ static_cast<char>(0xBF), static_cast<char>(0xBF)};
+
+ LIBC_NAMESPACE::internal::CharacterConverter char_conv(&state);
+ // Out of range code point should be rejected U+110000
+ int err = char_conv.push(static_cast<char8_t>(ch[0]));
+ ASSERT_EQ(err, 0);
+ err = char_conv.push(static_cast<char8_t>(ch[1]));
+ ASSERT_EQ(err, 0);
+ err = char_conv.push(static_cast<char8_t>(ch[2]));
+ ASSERT_EQ(err, 0);
+ err = char_conv.push(static_cast<char8_t>(ch[3]));
+ ASSERT_EQ(err, EILSEQ);
+
+ // In range, don't reject U+10FFFF
+ err = char_conv.push(static_cast<char8_t>(ch[4]));
+ ASSERT_EQ(err, 0);
+ err = char_conv.push(static_cast<char8_t>(ch[5]));
+ ASSERT_EQ(err, 0);
+ err = char_conv.push(static_cast<char8_t>(ch[6]));
+ ASSERT_EQ(err, 0);
+ err = char_conv.push(static_cast<char8_t>(ch[7]));
+ ASSERT_EQ(err, 0);
+ auto wch = char_conv.pop<CharType32>();
+ ASSERT_TRUE(wch.has_value());
+ ASSERT_EQ(static_cast<int>(wch.value()), 1114111);
+}
>From aaccb6642f234062a57c9ea160e8e1b0e8517d67 Mon Sep 17 00:00:00 2001
From: yangbowen <ybw1234567 at hotmail.com>
Date: Thu, 8 Oct 2026 20:27:38 +0000
Subject: [PATCH 2/3] [libc] Add UTF-16 support in mbrtowc, wcrtomb, etc.
---
libc/src/__support/wchar/CMakeLists.txt | 24 +-
.../src/__support/wchar/character_converter.h | 113 ++++
libc/src/__support/wchar/mbrtowc.cpp | 52 --
libc/src/__support/wchar/mbrtowc.h | 12 +-
libc/src/__support/wchar/mbsnrtowcs.h | 5 +-
libc/src/__support/wchar/string_converter.h | 2 -
libc/src/__support/wchar/wcrtomb.h | 35 +-
libc/src/__support/wchar/wcsnrtombs.h | 28 +-
libc/src/stdlib/wctomb.cpp | 4 +-
libc/src/wchar/wcrtomb.cpp | 8 +-
libc/test/src/__support/wchar/CMakeLists.txt | 19 +-
.../wchar/string_converter_test_utf16.cpp | 546 ++++++++++++++++++
...st.cpp => string_converter_test_utf32.cpp} | 0
.../src/__support/wchar/wcsnrtombs_test.cpp | 225 +++++++-
libc/test/src/stdlib/mblen_test.cpp | 12 +
libc/test/src/stdlib/mbstowcs_test.cpp | 170 ++++++
libc/test/src/stdlib/mbtowc_test.cpp | 13 +
libc/test/src/stdlib/wcstombs_test.cpp | 75 ++-
libc/test/src/stdlib/wctomb_test.cpp | 27 +
libc/test/src/wchar/CMakeLists.txt | 3 +-
libc/test/src/wchar/mbrlen_test.cpp | 14 +
libc/test/src/wchar/mbrtowc_test.cpp | 29 +
libc/test/src/wchar/mbsinit_test.cpp | 16 +
libc/test/src/wchar/mbsnrtowcs_test.cpp | 207 +++++++
libc/test/src/wchar/mbsrtowcs_test.cpp | 181 ++++++
libc/test/src/wchar/wcrtomb_test.cpp | 36 ++
libc/test/src/wchar/wcsnrtombs_test.cpp | 184 +++++-
libc/test/src/wchar/wcsrtombs_test.cpp | 138 ++++-
28 files changed, 2045 insertions(+), 133 deletions(-)
delete mode 100644 libc/src/__support/wchar/mbrtowc.cpp
create mode 100644 libc/test/src/__support/wchar/string_converter_test_utf16.cpp
rename libc/test/src/__support/wchar/{string_converter_test.cpp => string_converter_test_utf32.cpp} (100%)
diff --git a/libc/src/__support/wchar/CMakeLists.txt b/libc/src/__support/wchar/CMakeLists.txt
index 91fd788fd0762..83462b9f0fc93 100644
--- a/libc/src/__support/wchar/CMakeLists.txt
+++ b/libc/src/__support/wchar/CMakeLists.txt
@@ -12,8 +12,6 @@ add_header_library(
HDRS
string_converter.h
DEPENDS
- libc.hdr.types.char8_t
- libc.hdr.types.char32_t
libc.hdr.types.size_t
libc.src.__support.common
libc.src.__support.CPP.type_traits
@@ -47,33 +45,29 @@ add_header_library(
HDRS
wcrtomb.h
DEPENDS
- libc.hdr.errno_macros
- libc.hdr.types.char32_t
libc.hdr.types.size_t
libc.hdr.types.wchar_t
libc.src.__support.common
libc.src.__support.error_or
- libc.src.__support.libc_assert
+ libc.src.__support.macros.config
libc.src.__support.macros.null_check
- .character_converter
.mbstate
+ .character_converter
)
-add_object_library(
+add_header_library(
mbrtowc
HDRS
mbrtowc.h
- SRCS
- mbrtowc.cpp
DEPENDS
- libc.hdr.errno_macros
- libc.hdr.types.wchar_t
libc.hdr.types.size_t
+ libc.hdr.types.wchar_t
libc.src.__support.common
libc.src.__support.error_or
libc.src.__support.macros.config
- .character_converter
+ libc.src.__support.macros.null_check
.mbstate
+ .character_converter
)
add_header_library(
@@ -82,8 +76,9 @@ add_header_library(
mbsnrtowcs.h
DEPENDS
libc.hdr.errno_macros
- libc.hdr.types.wchar_t
+ libc.hdr.types.char8_t
libc.hdr.types.size_t
+ libc.hdr.types.wchar_t
libc.src.__support.common
libc.src.__support.error_or
libc.src.__support.macros.config
@@ -100,14 +95,13 @@ add_header_library(
DEPENDS
libc.hdr.errno_macros
libc.hdr.types.char8_t
- libc.hdr.types.char32_t
libc.hdr.types.size_t
libc.hdr.types.wchar_t
libc.src.__support.common
libc.src.__support.error_or
libc.src.__support.macros.config
libc.src.__support.macros.null_check
- .string_converter
.character_converter
.mbstate
+ .string_converter
)
diff --git a/libc/src/__support/wchar/character_converter.h b/libc/src/__support/wchar/character_converter.h
index f7ccf6f18b36c..a34d3401a4b65 100644
--- a/libc/src/__support/wchar/character_converter.h
+++ b/libc/src/__support/wchar/character_converter.h
@@ -120,6 +120,21 @@ class CharacterConverter {
#endif // LIBC_TYPES_WCHAR_T_IS_UTF32 || LIBC_TYPES_WCHAR_T_IS_UTF16
template <typename CharType> ErrorOr<CharType> pop();
+
+ // mbrtowc, mbrtoc16, mbrtoc32
+ // mbrtoc16 may return a surrogate pair in 2 consecutive calls, and specify -3
+ // as a return value (for the second call). mbrtowc doesn't specify -3 as a
+ // return value. Set allowPartialPop to true for mbrtoc16.
+ template <typename T, bool allowPartialPop>
+ ErrorOr<size_t> mbrto_generic(T *__restrict dest_ptr,
+ const char *__restrict src_ptr,
+ size_t max_src_bytes);
+ // wcrtomb, c16rtomb, c32rtomb
+ // c16rtomb may consume a surrogate pair in 2 consecutive calls, and return 0
+ // on the first call. wcrtomb doesn't use that behavior. Set allowPartialPush
+ // to true for c16rtomb.
+ template <typename T, bool allowPartialPush>
+ ErrorOr<size_t> rtomb_generic(char *__restrict dest_ptr, T src);
};
LIBC_INLINE bool CharacterConverter::isValidState() {
@@ -406,6 +421,104 @@ template <> LIBC_INLINE size_t CharacterConverter::sizeAs<wchar_t>() {
#endif // LIBC_TYPES_WCHAR_T_IS_UTF32 || LIBC_TYPES_WCHAR_T_IS_UTF16
+template <typename T, bool allowPartialPop>
+LIBC_INLINE ErrorOr<size_t>
+CharacterConverter::mbrto_generic(T *__restrict dest_ptr,
+ const char *__restrict src_ptr,
+ size_t max_src_bytes) {
+ if (!isValidState())
+ return Error(EINVAL);
+ if (!allowPartialPop && isPartiallyPopping()) {
+ // don't allow a complete char32 spanning over 2 consecutive calls of char16
+ clear();
+ return Error(EILSEQ);
+ }
+
+ char empty_src = '\0';
+ if (src_ptr == nullptr) {
+ dest_ptr = nullptr;
+ src_ptr = &empty_src;
+ max_src_bytes = 1;
+ }
+
+ size_t i = 0;
+ // If we still have the lower part to write, don't read yet (UTF-16)
+ if (!isPartiallyPopping()) {
+ // Reading in bytes until we have a complete char32 or error
+ for (; i < max_src_bytes && !isFull(); ++i) {
+ int err = push(static_cast<char8_t>(src_ptr[i]));
+ // Encoding error
+ if (err != 0)
+ return Error(err);
+ }
+ }
+ if (!isFull() && !isPartiallyPopping()) {
+ // We haven't read in a full char32 yet.
+ // Incomplete but potentially valid
+ return (size_t)-2;
+ }
+ auto result = pop<T>();
+ if (!result.has_value()) {
+ return Error(result.error());
+ }
+ if (!allowPartialPop && !isEmpty()) {
+ // don't allow a complete char32 spanning over 2 consecutive calls of char16
+ clear();
+ return Error(EILSEQ);
+ }
+ if (dest_ptr != nullptr)
+ *dest_ptr = result.value();
+ if (i == 0) {
+ // lower part written, no input processed
+ return (size_t)-3;
+ }
+ // null terminator -> return 0
+ if (result.value() == T{})
+ return 0;
+ return i;
+}
+
+template <typename T, bool allowPartialPush>
+LIBC_INLINE ErrorOr<size_t>
+CharacterConverter::rtomb_generic(char *__restrict dest_ptr, T src) {
+ if (dest_ptr == nullptr)
+ src = {};
+
+ if (!isValidState())
+ return Error(EINVAL);
+ if (!allowPartialPush && !isEmpty()) {
+ // don't allow a complete char32 spanning over 2 consecutive calls of char16
+ clear();
+ return Error(EILSEQ);
+ }
+
+ int status = push(static_cast<T>(src));
+ if (status != 0)
+ return Error(status);
+
+ if (!isFull()) {
+ if (!allowPartialPush) {
+ // don't allow a complete char32 spanning over 2 consecutive calls of
+ // char16
+ clear();
+ return Error(EILSEQ);
+ }
+ return 0;
+ }
+
+ size_t count = 0;
+ for (; !isEmpty(); ++count) {
+ auto utf8 = pop_utf8(); // can never fail as long as the push succeeded
+ LIBC_ASSERT(utf8.has_value());
+
+ if (dest_ptr != nullptr) {
+ *dest_ptr = static_cast<char>(utf8.value());
+ dest_ptr++;
+ }
+ }
+ return count;
+}
+
} // namespace internal
} // namespace LIBC_NAMESPACE_DECL
diff --git a/libc/src/__support/wchar/mbrtowc.cpp b/libc/src/__support/wchar/mbrtowc.cpp
deleted file mode 100644
index 33a42a4a31110..0000000000000
--- a/libc/src/__support/wchar/mbrtowc.cpp
+++ /dev/null
@@ -1,52 +0,0 @@
-//===-- Implementation for mbrtowc function ---------------------*- C++ -*-===//
-//
-// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
-// See https://llvm.org/LICENSE.txt for license information.
-// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
-//
-//===----------------------------------------------------------------------===//
-
-#include "src/__support/wchar/mbrtowc.h"
-#include "hdr/errno_macros.h"
-#include "hdr/types/size_t.h"
-#include "hdr/types/wchar_t.h"
-#include "src/__support/common.h"
-#include "src/__support/error_or.h"
-#include "src/__support/macros/config.h"
-#include "src/__support/wchar/character_converter.h"
-#include "src/__support/wchar/mbstate.h"
-
-namespace LIBC_NAMESPACE_DECL {
-namespace internal {
-
-ErrorOr<size_t> mbrtowc(wchar_t *__restrict pwc, const char *__restrict src_ptr,
- size_t max_src_bytes, mbstate *__restrict ps) {
- CharacterConverter char_conv(ps);
- if (!char_conv.isValidState())
- return Error(EINVAL);
- if (src_ptr == nullptr)
- return 0;
- size_t i = 0;
- // Reading in bytes until we have a complete wc or error
- for (; i < max_src_bytes && !char_conv.isFull(); ++i) {
- int err = char_conv.push(static_cast<char8_t>(src_ptr[i]));
- // Encoding error
- if (err == EILSEQ)
- return Error(err);
- }
- auto wc = char_conv.pop_utf32();
- if (wc.has_value()) {
- if (pwc != nullptr)
- *pwc = wc.value();
- // null terminator -> return 0
- if (wc.value() == L'\0')
- return 0;
- return i;
- }
- // Incomplete but potentially valid
- return -2;
-}
-
-} // namespace internal
-
-} // namespace LIBC_NAMESPACE_DECL
diff --git a/libc/src/__support/wchar/mbrtowc.h b/libc/src/__support/wchar/mbrtowc.h
index d15c4844499a6..1c802fa9f8dfd 100644
--- a/libc/src/__support/wchar/mbrtowc.h
+++ b/libc/src/__support/wchar/mbrtowc.h
@@ -14,13 +14,21 @@
#include "src/__support/common.h"
#include "src/__support/error_or.h"
#include "src/__support/macros/config.h"
+#include "src/__support/macros/null_check.h"
+#include "src/__support/wchar/character_converter.h"
#include "src/__support/wchar/mbstate.h"
namespace LIBC_NAMESPACE_DECL {
namespace internal {
-ErrorOr<size_t> mbrtowc(wchar_t *__restrict pwc, const char *__restrict src_ptr,
- size_t max_src_bytes, mbstate *__restrict ps);
+LIBC_INLINE ErrorOr<size_t> mbrtowc(wchar_t *__restrict pwc,
+ const char *__restrict src_ptr,
+ size_t max_src_bytes,
+ mbstate *__restrict ps) {
+ LIBC_CRASH_ON_NULLPTR(ps);
+ CharacterConverter char_conv(ps);
+ return char_conv.mbrto_generic<wchar_t, false>(pwc, src_ptr, max_src_bytes);
+}
} // namespace internal
diff --git a/libc/src/__support/wchar/mbsnrtowcs.h b/libc/src/__support/wchar/mbsnrtowcs.h
index 50c3fa23a7d58..ebddc9802574d 100644
--- a/libc/src/__support/wchar/mbsnrtowcs.h
+++ b/libc/src/__support/wchar/mbsnrtowcs.h
@@ -10,6 +10,7 @@
#define LLVM_LIBC_SRC___SUPPORT_WCHAR_MBSNRTOWCS_H
#include "hdr/errno_macros.h"
+#include "hdr/types/char8_t.h"
#include "hdr/types/size_t.h"
#include "hdr/types/wchar_t.h"
#include "src/__support/common.h"
@@ -34,7 +35,7 @@ mbsnrtowcs(wchar_t *__restrict dst, const char **__restrict src,
StringConverter<char8_t> str_conv(reinterpret_cast<const char8_t *>(*src), ps,
max_dst_chars, max_src_bytes);
size_t dst_idx = 0;
- ErrorOr<char32_t> converted = str_conv.pop<char32_t>();
+ ErrorOr<wchar_t> converted = str_conv.pop<wchar_t>();
while (converted.has_value()) {
if (dst != nullptr)
dst[dst_idx] = converted.value();
@@ -45,7 +46,7 @@ mbsnrtowcs(wchar_t *__restrict dst, const char **__restrict src,
return dst_idx;
}
dst_idx++;
- converted = str_conv.pop<char32_t>();
+ converted = str_conv.pop<wchar_t>();
}
if (converted.error() == -1) { // if we hit conversion limit
diff --git a/libc/src/__support/wchar/string_converter.h b/libc/src/__support/wchar/string_converter.h
index 9e76570e11678..ebc8b4cc987e4 100644
--- a/libc/src/__support/wchar/string_converter.h
+++ b/libc/src/__support/wchar/string_converter.h
@@ -9,8 +9,6 @@
#ifndef LLVM_LIBC_SRC___SUPPORT_STRING_CONVERTER_H
#define LLVM_LIBC_SRC___SUPPORT_STRING_CONVERTER_H
-#include "hdr/types/char32_t.h"
-#include "hdr/types/char8_t.h"
#include "hdr/types/size_t.h"
#include "src/__support/CPP/type_traits.h"
#include "src/__support/common.h"
diff --git a/libc/src/__support/wchar/wcrtomb.h b/libc/src/__support/wchar/wcrtomb.h
index e7f2776b293a3..b8324be020674 100644
--- a/libc/src/__support/wchar/wcrtomb.h
+++ b/libc/src/__support/wchar/wcrtomb.h
@@ -9,46 +9,23 @@
#ifndef LLVM_LIBC_SRC__SUPPORT_WCHAR_WCRTOMB_H
#define LLVM_LIBC_SRC__SUPPORT_WCHAR_WCRTOMB_H
+#include "hdr/types/size_t.h"
+#include "hdr/types/wchar_t.h"
+#include "src/__support/common.h"
#include "src/__support/error_or.h"
+#include "src/__support/macros/config.h"
#include "src/__support/macros/null_check.h"
#include "src/__support/wchar/character_converter.h"
#include "src/__support/wchar/mbstate.h"
-#include "hdr/errno_macros.h"
-#include "hdr/types/char32_t.h"
-#include "hdr/types/size_t.h"
-#include "hdr/types/wchar_t.h"
-#include "src/__support/common.h"
-#include "src/__support/libc_assert.h"
-
namespace LIBC_NAMESPACE_DECL {
namespace internal {
LIBC_INLINE ErrorOr<size_t> wcrtomb(char *__restrict dest_ptr, wchar_t wc,
mbstate *__restrict ps) {
- LIBC_CRASH_ON_NULLPTR(dest_ptr);
LIBC_CRASH_ON_NULLPTR(ps);
- static_assert(sizeof(wchar_t) == 4);
-
- CharacterConverter cr(ps);
-
- if (!cr.isValidState())
- return Error(EINVAL);
-
- int status = cr.push(static_cast<char32_t>(wc));
- if (status != 0)
- return Error(status);
-
- size_t count = 0;
- while (!cr.isEmpty()) {
- auto utf8 = cr.pop_utf8(); // can never fail as long as the push succeeded
- LIBC_ASSERT(utf8.has_value());
-
- *dest_ptr = utf8.value();
- dest_ptr++;
- count++;
- }
- return count;
+ CharacterConverter char_conv(ps);
+ return char_conv.rtomb_generic<wchar_t, false>(dest_ptr, wc);
}
} // namespace internal
diff --git a/libc/src/__support/wchar/wcsnrtombs.h b/libc/src/__support/wchar/wcsnrtombs.h
index 5faf035145ff6..dd959757d9fd4 100644
--- a/libc/src/__support/wchar/wcsnrtombs.h
+++ b/libc/src/__support/wchar/wcsnrtombs.h
@@ -9,13 +9,15 @@
#ifndef LLVM_LIBC_SRC__SUPPORT_WCHAR_WCSNRTOMBS_H
#define LLVM_LIBC_SRC__SUPPORT_WCHAR_WCSNRTOMBS_H
-#include "hdr/types/char32_t.h"
+#include "hdr/errno_macros.h"
#include "hdr/types/char8_t.h"
#include "hdr/types/size_t.h"
#include "hdr/types/wchar_t.h"
#include "src/__support/common.h"
+#include "src/__support/error_or.h"
#include "src/__support/macros/config.h"
#include "src/__support/macros/null_check.h"
+#include "src/__support/wchar/character_converter.h"
#include "src/__support/wchar/mbstate.h"
#include "src/__support/wchar/string_converter.h"
@@ -35,9 +37,9 @@ wcsnrtombs(char *__restrict dest, const wchar_t **__restrict ptr_to_src,
if (dest == nullptr)
dest_len = SIZE_MAX;
- StringConverter<char32_t> str_conv(
- reinterpret_cast<const char32_t *>(*ptr_to_src), ps, dest_len,
- num_src_widechars);
+ StringConverter<wchar_t> str_conv(*ptr_to_src, ps, dest_len,
+ num_src_widechars);
+ size_t src_idx = 0;
size_t dst_idx = 0;
ErrorOr<char8_t> converted = str_conv.pop<char8_t>();
while (converted.has_value()) {
@@ -50,15 +52,27 @@ wcsnrtombs(char *__restrict dest, const wchar_t **__restrict ptr_to_src,
return dst_idx;
}
+ src_idx = str_conv.getSourceIndex();
dst_idx++;
converted = str_conv.pop<char8_t>();
}
+ // On error, we leave *ptr_to_src at just past the last successfully converted
+ // code point. See also:
+ // https://bugs.freebsd.org/bugzilla/show_bug.cgi?id=209907
if (dest != nullptr)
- *ptr_to_src += str_conv.getSourceIndex();
-
- if (converted.error() == -1) // if we hit conversion limit
+ *ptr_to_src += src_idx;
+ if (converted.error() == -1) {
+ if (!cr.isEmpty()) {
+ // (UTF-16) We disallow an unpaired high surrogate right at the end of the
+ // source string. We have no return value to indicate that situation, so
+ // we consider it an error in the source string.
+ cr.clear();
+ return Error(EILSEQ);
+ }
+ // if we hit conversion limit
return dst_idx;
+ }
return Error(converted.error());
}
diff --git a/libc/src/stdlib/wctomb.cpp b/libc/src/stdlib/wctomb.cpp
index 0ca1a84cd923e..20b17854cc5ed 100644
--- a/libc/src/stdlib/wctomb.cpp
+++ b/libc/src/stdlib/wctomb.cpp
@@ -24,8 +24,8 @@ LLVM_LIBC_FUNCTION(int, wctomb, (char *s, wchar_t wc)) {
auto result = internal::wcrtomb(s, wc, &internal_mbstate);
- if (!result.has_value()) { // invalid wide character
- libc_errno = EILSEQ;
+ if (!result.has_value()) {
+ libc_errno = result.error();
return -1;
}
diff --git a/libc/src/wchar/wcrtomb.cpp b/libc/src/wchar/wcrtomb.cpp
index 3e9df0599431e..b9548eb69b512 100644
--- a/libc/src/wchar/wcrtomb.cpp
+++ b/libc/src/wchar/wcrtomb.cpp
@@ -22,13 +22,7 @@ LLVM_LIBC_FUNCTION(size_t, wcrtomb,
(char *__restrict s, wchar_t wc, mbstate_t *__restrict ps)) {
static internal::mbstate internal_mbstate;
- // when s is nullptr, this is equivalent to wcrtomb(buf, L'\0', ps)
- char buf[sizeof(wchar_t) / sizeof(char)];
- if (s == nullptr) {
- s = buf;
- wc = L'\0';
- }
-
+ // the s==nullptr check is done by internal::wcrtomb
auto result = internal::wcrtomb(
s, wc,
ps == nullptr ? &internal_mbstate
diff --git a/libc/test/src/__support/wchar/CMakeLists.txt b/libc/test/src/__support/wchar/CMakeLists.txt
index 907950833dbcc..2c6f5be33d88a 100644
--- a/libc/test/src/__support/wchar/CMakeLists.txt
+++ b/libc/test/src/__support/wchar/CMakeLists.txt
@@ -1,11 +1,11 @@
add_custom_target(libc-support-wchar-tests)
add_libc_test(
- string_converter_test
+ string_converter_test_utf32
SUITE
libc-support-tests
SRCS
- string_converter_test.cpp
+ string_converter_test_utf32.cpp
DEPENDS
libc.src.__support.macros.properties.types
libc.src.__support.wchar.string_converter
@@ -15,6 +15,21 @@ add_libc_test(
libc.hdr.types.char32_t
)
+add_libc_test(
+ string_converter_test_utf16
+ SUITE
+ libc-support-tests
+ SRCS
+ string_converter_test_utf16.cpp
+ DEPENDS
+ libc.src.__support.macros.properties.types
+ libc.src.__support.wchar.string_converter
+ libc.src.__support.wchar.mbstate
+ libc.src.__support.error_or
+ libc.hdr.errno_macros
+ libc.hdr.types.char16_t
+)
+
add_libc_test(
utf8_to_32_test
SUITE
diff --git a/libc/test/src/__support/wchar/string_converter_test_utf16.cpp b/libc/test/src/__support/wchar/string_converter_test_utf16.cpp
new file mode 100644
index 0000000000000..c76cc7e67fd36
--- /dev/null
+++ b/libc/test/src/__support/wchar/string_converter_test_utf16.cpp
@@ -0,0 +1,546 @@
+//===-- Unittests for StringConverter class -------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#include "hdr/errno_macros.h"
+#include "hdr/types/char16_t.h"
+#include "hdr/types/char8_t.h"
+#include "src/__support/error_or.h"
+#include "src/__support/macros/properties/types.h"
+#include "src/__support/wchar/mbstate.h"
+#include "src/__support/wchar/string_converter.h"
+#include "test/UnitTest/Test.h"
+
+#if defined(LIBC_TYPES_WCHAR_T_IS_UTF16)
+using TestCharTypesUTF16 = LIBC_NAMESPACE::testing::TypeList<char16_t, wchar_t>;
+#else
+using TestCharTypesUTF16 = LIBC_NAMESPACE::testing::TypeList<char16_t>;
+#endif
+
+TYPED_TEST(LlvmLibcStringConverterTest, UTF8To16, TestCharTypesUTF16) {
+ using CharType16 = ParamType;
+
+ // first 4 bytes are clown emoji (🤡)
+ // next 3 bytes are sigma symbol (∑)
+ // next 2 bytes are y with diaeresis (ÿ)
+ // last byte is the letter A
+ const char *src = "\xF0\x9F\xA4\xA1\xE2\x88\x91\xC3\xBF\x41";
+ LIBC_NAMESPACE::internal::mbstate state;
+ LIBC_NAMESPACE::internal::StringConverter<char8_t> sc(
+ reinterpret_cast<const char8_t *>(src), &state, SIZE_MAX);
+
+ auto res = sc.pop<CharType16>();
+ ASSERT_TRUE(res.has_value());
+ ASSERT_EQ(static_cast<int>(res.value()), 0xd83e);
+ ASSERT_EQ(static_cast<int>(sc.getSourceIndex()), 4);
+
+ res = sc.pop<CharType16>();
+ ASSERT_TRUE(res.has_value());
+ ASSERT_EQ(static_cast<int>(res.value()), 0xdd21);
+ ASSERT_EQ(static_cast<int>(sc.getSourceIndex()), 4);
+
+ res = sc.pop<CharType16>();
+ ASSERT_TRUE(res.has_value());
+ ASSERT_EQ(static_cast<int>(res.value()), 0x2211);
+ ASSERT_EQ(static_cast<int>(sc.getSourceIndex()), 7);
+
+ res = sc.pop<CharType16>();
+ ASSERT_TRUE(res.has_value());
+ ASSERT_EQ(static_cast<int>(res.value()), 0xff);
+ ASSERT_EQ(static_cast<int>(sc.getSourceIndex()), 9);
+
+ res = sc.pop<CharType16>();
+ ASSERT_TRUE(res.has_value());
+ ASSERT_EQ(static_cast<int>(res.value()), 0x41);
+ ASSERT_EQ(static_cast<int>(sc.getSourceIndex()), 10);
+
+ res = sc.pop<CharType16>();
+ ASSERT_TRUE(res.has_value());
+ ASSERT_EQ(static_cast<int>(res.value()), 0);
+ ASSERT_EQ(static_cast<int>(sc.getSourceIndex()), 11);
+
+ res = sc.pop<CharType16>();
+ ASSERT_FALSE(res.has_value());
+ ASSERT_EQ(res.error(), -1);
+ ASSERT_EQ(static_cast<int>(sc.getSourceIndex()), 11);
+}
+
+TYPED_TEST(LlvmLibcStringConverterTest, UTF16To8, TestCharTypesUTF16) {
+ using CharType16 = ParamType;
+
+ // clown emoji, sigma symbol, y with diaeresis, letter A
+ const CharType16 src[] = {
+ static_cast<CharType16>(0xd83e), static_cast<CharType16>(0xdd21),
+ static_cast<CharType16>(0x2211), static_cast<CharType16>(0xff),
+ static_cast<CharType16>(0x41), static_cast<CharType16>(0x0)};
+ LIBC_NAMESPACE::internal::mbstate state;
+ LIBC_NAMESPACE::internal::StringConverter<CharType16> sc(src, &state,
+ SIZE_MAX);
+
+ auto res = sc.template pop<char8_t>();
+ ASSERT_TRUE(res.has_value());
+ ASSERT_EQ(static_cast<int>(res.value()), 0xF0);
+ ASSERT_EQ(static_cast<int>(sc.getSourceIndex()), 2);
+
+ res = sc.template pop<char8_t>();
+ ASSERT_TRUE(res.has_value());
+ ASSERT_EQ(static_cast<int>(res.value()), 0x9F);
+ ASSERT_EQ(static_cast<int>(sc.getSourceIndex()), 2);
+
+ res = sc.template pop<char8_t>();
+ ASSERT_TRUE(res.has_value());
+ ASSERT_EQ(static_cast<int>(res.value()), 0xA4);
+ ASSERT_EQ(static_cast<int>(sc.getSourceIndex()), 2);
+
+ res = sc.template pop<char8_t>();
+ ASSERT_TRUE(res.has_value());
+ ASSERT_EQ(static_cast<int>(res.value()), 0xA1);
+ ASSERT_EQ(static_cast<int>(sc.getSourceIndex()), 2);
+
+ // end of clown emoji, sigma symbol begins
+ res = sc.template pop<char8_t>();
+ ASSERT_TRUE(res.has_value());
+ ASSERT_EQ(static_cast<int>(res.value()), 0xE2);
+ ASSERT_EQ(static_cast<int>(sc.getSourceIndex()), 3);
+
+ res = sc.template pop<char8_t>();
+ ASSERT_TRUE(res.has_value());
+ ASSERT_EQ(static_cast<int>(res.value()), 0x88);
+ ASSERT_EQ(static_cast<int>(sc.getSourceIndex()), 3);
+
+ res = sc.template pop<char8_t>();
+ ASSERT_TRUE(res.has_value());
+ ASSERT_EQ(static_cast<int>(res.value()), 0x91);
+ ASSERT_EQ(static_cast<int>(sc.getSourceIndex()), 3);
+
+ // end of sigma symbol, y with diaeresis begins
+ res = sc.template pop<char8_t>();
+ ASSERT_TRUE(res.has_value());
+ ASSERT_EQ(static_cast<int>(res.value()), 0xC3);
+ ASSERT_EQ(static_cast<int>(sc.getSourceIndex()), 4);
+
+ res = sc.template pop<char8_t>();
+ ASSERT_TRUE(res.has_value());
+ ASSERT_EQ(static_cast<int>(res.value()), 0xBF);
+ ASSERT_EQ(static_cast<int>(sc.getSourceIndex()), 4);
+
+ // end of y with diaeresis, letter A begins
+ res = sc.template pop<char8_t>();
+ ASSERT_TRUE(res.has_value());
+ ASSERT_EQ(static_cast<int>(res.value()), 0x41);
+ ASSERT_EQ(static_cast<int>(sc.getSourceIndex()), 5);
+
+ // null byte
+ res = sc.template pop<char8_t>();
+ ASSERT_TRUE(res.has_value());
+ ASSERT_EQ(static_cast<int>(res.value()), 0);
+ ASSERT_EQ(static_cast<int>(sc.getSourceIndex()), 6);
+
+ res = sc.template pop<char8_t>();
+ ASSERT_FALSE(res.has_value());
+ ASSERT_EQ(res.error(), -1);
+ ASSERT_EQ(static_cast<int>(sc.getSourceIndex()), 6);
+}
+
+TYPED_TEST(LlvmLibcStringConverterTest, UTF16To8PartialRead,
+ TestCharTypesUTF16) {
+ using CharType16 = ParamType;
+
+ const CharType16 src[] = {
+ static_cast<CharType16>(0xd83e), static_cast<CharType16>(0xdd21),
+ static_cast<CharType16>(0x2211),
+ static_cast<CharType16>(0x0)}; // clown emoji, sigma symbol
+ LIBC_NAMESPACE::internal::mbstate state;
+ LIBC_NAMESPACE::internal::StringConverter<CharType16> sc(src, &state,
+ SIZE_MAX, 2);
+
+ auto res = sc.template pop<char8_t>();
+ ASSERT_TRUE(res.has_value());
+ ASSERT_EQ(static_cast<int>(res.value()), 0xF0);
+ ASSERT_EQ(static_cast<int>(sc.getSourceIndex()), 2);
+
+ res = sc.template pop<char8_t>();
+ ASSERT_TRUE(res.has_value());
+ ASSERT_EQ(static_cast<int>(res.value()), 0x9F);
+ ASSERT_EQ(static_cast<int>(sc.getSourceIndex()), 2);
+
+ res = sc.template pop<char8_t>();
+ ASSERT_TRUE(res.has_value());
+ ASSERT_EQ(static_cast<int>(res.value()), 0xA4);
+ ASSERT_EQ(static_cast<int>(sc.getSourceIndex()), 2);
+
+ res = sc.template pop<char8_t>();
+ ASSERT_TRUE(res.has_value());
+ ASSERT_EQ(static_cast<int>(res.value()), 0xA1);
+ ASSERT_EQ(static_cast<int>(sc.getSourceIndex()), 2);
+
+ // can only read 2 characters from source string, so error on next pop
+ res = sc.template pop<char8_t>();
+ ASSERT_FALSE(res.has_value());
+ ASSERT_EQ(res.error(), -1);
+}
+
+TYPED_TEST(LlvmLibcStringConverterTest, UTF8To16PartialRead,
+ TestCharTypesUTF16) {
+ using CharType16 = ParamType;
+
+ // first 4 bytes are clown emoji, then next 3 are sigma symbol
+ const char *src = "\xF0\x9F\xA4\xA1\xE2\x88\x91";
+ LIBC_NAMESPACE::internal::mbstate state;
+ LIBC_NAMESPACE::internal::StringConverter<char8_t> sc(
+ reinterpret_cast<const char8_t *>(src), &state, SIZE_MAX, 5);
+
+ auto res = sc.pop<CharType16>();
+ ASSERT_TRUE(res.has_value());
+ ASSERT_EQ(static_cast<int>(res.value()), 0xd83e);
+ ASSERT_EQ(static_cast<int>(sc.getSourceIndex()), 4);
+
+ res = sc.pop<CharType16>();
+ ASSERT_TRUE(res.has_value());
+ ASSERT_EQ(static_cast<int>(res.value()), 0xdd21);
+ ASSERT_EQ(static_cast<int>(sc.getSourceIndex()), 4);
+
+ res = sc.pop<CharType16>();
+ ASSERT_FALSE(res.has_value());
+ ASSERT_EQ(static_cast<int>(res.error()), -1);
+ ASSERT_EQ(static_cast<int>(sc.getSourceIndex()), 5);
+}
+
+TYPED_TEST(LlvmLibcStringConverterTest, UTF16To8ErrorHandling,
+ TestCharTypesUTF16) {
+ using CharType16 = ParamType;
+
+ const CharType16 src[] = {
+ static_cast<CharType16>(0xd83e), static_cast<CharType16>(0xdd21),
+ static_cast<CharType16>(0xdc00),
+ static_cast<CharType16>(0x0)}; // clown emoji, invalid utf16
+ LIBC_NAMESPACE::internal::mbstate state;
+ LIBC_NAMESPACE::internal::StringConverter<CharType16> sc(src, &state,
+ SIZE_MAX);
+
+ auto res = sc.template pop<char8_t>();
+ ASSERT_TRUE(res.has_value());
+ ASSERT_EQ(static_cast<int>(res.value()), 0xF0);
+ ASSERT_EQ(static_cast<int>(sc.getSourceIndex()), 2);
+
+ res = sc.template pop<char8_t>();
+ ASSERT_TRUE(res.has_value());
+ ASSERT_EQ(static_cast<int>(res.value()), 0x9F);
+ ASSERT_EQ(static_cast<int>(sc.getSourceIndex()), 2);
+
+ res = sc.template pop<char8_t>();
+ ASSERT_TRUE(res.has_value());
+ ASSERT_EQ(static_cast<int>(res.value()), 0xA4);
+ ASSERT_EQ(static_cast<int>(sc.getSourceIndex()), 2);
+
+ res = sc.template pop<char8_t>();
+ ASSERT_TRUE(res.has_value());
+ ASSERT_EQ(static_cast<int>(res.value()), 0xA1);
+ ASSERT_EQ(static_cast<int>(sc.getSourceIndex()), 2);
+
+ res = sc.template pop<char8_t>();
+ ASSERT_FALSE(res.has_value());
+ ASSERT_EQ(static_cast<int>(res.error()), EILSEQ);
+ ASSERT_EQ(static_cast<int>(sc.getSourceIndex()), 2);
+}
+
+TYPED_TEST(LlvmLibcStringConverterTest, UTF8To16ErrorHandling,
+ TestCharTypesUTF16) {
+ using CharType16 = ParamType;
+
+ // first 4 bytes are clown emoji (🤡)
+ // next 3 form an invalid character
+ const char *src = "\xF0\x9F\xA4\xA1\x90\x88\x30";
+ LIBC_NAMESPACE::internal::mbstate state;
+ LIBC_NAMESPACE::internal::StringConverter<char8_t> sc(
+ reinterpret_cast<const char8_t *>(src), &state, SIZE_MAX);
+
+ auto res = sc.pop<CharType16>();
+ ASSERT_TRUE(res.has_value());
+ ASSERT_EQ(static_cast<int>(res.value()), 0xd83e);
+ ASSERT_EQ(static_cast<int>(sc.getSourceIndex()), 4);
+
+ res = sc.pop<CharType16>();
+ ASSERT_TRUE(res.has_value());
+ ASSERT_EQ(static_cast<int>(res.value()), 0xdd21);
+ ASSERT_EQ(static_cast<int>(sc.getSourceIndex()), 4);
+
+ res = sc.pop<CharType16>();
+ ASSERT_FALSE(res.has_value());
+ ASSERT_EQ(static_cast<int>(res.error()), EILSEQ);
+ ASSERT_EQ(static_cast<int>(sc.getSourceIndex()), 4);
+}
+
+TYPED_TEST(LlvmLibcStringConverterTest, InvalidCharacterOutsideBounds,
+ TestCharTypesUTF16) {
+ using CharType16 = ParamType;
+
+ // if an invalid character exists in the source string but we don't have space
+ // to write it, we should return a "stop converting" error rather than an
+ // invalid character error
+
+ // first 4 bytes are clown emoji (🤡)
+ // next 3 form an invalid character
+ const char *src1 = "\xF0\x9F\xA4\xA1\x90\x88\x30";
+ LIBC_NAMESPACE::internal::mbstate ps1;
+ LIBC_NAMESPACE::internal::StringConverter<char8_t> sc1(
+ reinterpret_cast<const char8_t *>(src1), &ps1, 2);
+
+ auto res1 = sc1.pop<CharType16>();
+ ASSERT_TRUE(res1.has_value());
+ ASSERT_EQ(static_cast<int>(res1.value()), 0xd83e);
+ ASSERT_EQ(static_cast<int>(sc1.getSourceIndex()), 4);
+
+ res1 = sc1.pop<CharType16>();
+ ASSERT_TRUE(res1.has_value());
+ ASSERT_EQ(static_cast<int>(res1.value()), 0xdd21);
+ ASSERT_EQ(static_cast<int>(sc1.getSourceIndex()), 4);
+
+ res1 = sc1.pop<CharType16>();
+ ASSERT_FALSE(res1.has_value());
+ // no space to write error NOT invalid character error (EILSEQ)
+ ASSERT_EQ(static_cast<int>(res1.error()), -1);
+ ASSERT_EQ(static_cast<int>(sc1.getSourceIndex()), 4);
+
+ const CharType16 src2[] = {
+ static_cast<CharType16>(0xd83e), static_cast<CharType16>(0xdd21),
+ static_cast<CharType16>(0xdc00),
+ static_cast<CharType16>(0x0)}; // clown emoji, invalid utf16
+ LIBC_NAMESPACE::internal::mbstate ps2;
+ LIBC_NAMESPACE::internal::StringConverter<CharType16> sc2(src2, &ps2, 4);
+
+ auto res2 = sc2.template pop<char8_t>();
+ ASSERT_TRUE(res2.has_value());
+ ASSERT_EQ(static_cast<int>(res2.value()), 0xF0);
+ ASSERT_EQ(static_cast<int>(sc2.getSourceIndex()), 2);
+
+ res2 = sc2.template pop<char8_t>();
+ ASSERT_TRUE(res2.has_value());
+ ASSERT_EQ(static_cast<int>(res2.value()), 0x9F);
+ ASSERT_EQ(static_cast<int>(sc2.getSourceIndex()), 2);
+
+ res2 = sc2.template pop<char8_t>();
+ ASSERT_TRUE(res2.has_value());
+ ASSERT_EQ(static_cast<int>(res2.value()), 0xA4);
+ ASSERT_EQ(static_cast<int>(sc2.getSourceIndex()), 2);
+
+ res2 = sc2.template pop<char8_t>();
+ ASSERT_TRUE(res2.has_value());
+ ASSERT_EQ(static_cast<int>(res2.value()), 0xA1);
+ ASSERT_EQ(static_cast<int>(sc2.getSourceIndex()), 2);
+
+ res2 = sc2.template pop<char8_t>();
+ ASSERT_FALSE(res2.has_value());
+ // no space to write error NOT invalid character error (EILSEQ)
+ ASSERT_EQ(static_cast<int>(res2.error()), -1);
+ ASSERT_EQ(static_cast<int>(sc2.getSourceIndex()), 2);
+}
+
+TYPED_TEST(LlvmLibcStringConverterTest, MultipleStringConverters16To8,
+ TestCharTypesUTF16) {
+ using CharType16 = ParamType;
+
+ /*
+ We do NOT test partially popping a character and expecting the next
+ StringConverter to continue where we left off. This is not expected to work
+ and considered invalid.
+ */
+ const CharType16 src[] = {
+ static_cast<CharType16>(0xd83e), static_cast<CharType16>(0xdd21),
+ static_cast<CharType16>(0xff),
+ static_cast<CharType16>(0x0)}; // clown emoji, y with diaeresis (ÿ)
+ LIBC_NAMESPACE::internal::mbstate state;
+ LIBC_NAMESPACE::internal::StringConverter<CharType16> sc1(src, &state,
+ SIZE_MAX, 2);
+
+ auto res = sc1.template pop<char8_t>();
+ ASSERT_TRUE(res.has_value());
+ ASSERT_EQ(static_cast<int>(res.value()), 0xF0);
+ ASSERT_EQ(static_cast<int>(sc1.getSourceIndex()), 2);
+
+ res = sc1.template pop<char8_t>();
+ ASSERT_TRUE(res.has_value());
+ ASSERT_EQ(static_cast<int>(res.value()), 0x9F);
+ ASSERT_EQ(static_cast<int>(sc1.getSourceIndex()), 2);
+
+ res = sc1.template pop<char8_t>();
+ ASSERT_TRUE(res.has_value());
+ ASSERT_EQ(static_cast<int>(res.value()), 0xA4);
+ ASSERT_EQ(static_cast<int>(sc1.getSourceIndex()), 2);
+
+ res = sc1.template pop<char8_t>();
+ ASSERT_TRUE(res.has_value());
+ ASSERT_EQ(static_cast<int>(res.value()), 0xA1);
+ ASSERT_EQ(static_cast<int>(sc1.getSourceIndex()), 2);
+
+ // sc2 should pick up where sc1 left off and continue the conversion
+ LIBC_NAMESPACE::internal::StringConverter<CharType16> sc2(
+ src + sc1.getSourceIndex(), &state, SIZE_MAX, 1);
+
+ res = sc2.template pop<char8_t>();
+ ASSERT_TRUE(res.has_value());
+ ASSERT_EQ(static_cast<int>(res.value()), 0xC3);
+ ASSERT_EQ(static_cast<int>(sc2.getSourceIndex()), 1);
+
+ res = sc2.template pop<char8_t>();
+ ASSERT_TRUE(res.has_value());
+ ASSERT_EQ(static_cast<int>(res.value()), 0xBF);
+ ASSERT_EQ(static_cast<int>(sc2.getSourceIndex()), 1);
+}
+
+TYPED_TEST(LlvmLibcStringConverterTest, MultipleStringConverters8To16,
+ TestCharTypesUTF16) {
+ using CharType16 = ParamType;
+
+ const char *src = "\xF0\x9F\xA4\xA1"; // clown emoji
+ LIBC_NAMESPACE::internal::mbstate state;
+ LIBC_NAMESPACE::internal::StringConverter<char8_t> sc1(
+ reinterpret_cast<const char8_t *>(src), &state, SIZE_MAX, 2);
+
+ auto res = sc1.pop<CharType16>();
+ ASSERT_FALSE(res.has_value());
+ ASSERT_EQ(static_cast<int>(res.error()), -1);
+ ASSERT_EQ(static_cast<int>(sc1.getSourceIndex()), 2);
+
+ // sc2 should pick up where sc1 left off and continue the conversion
+ LIBC_NAMESPACE::internal::StringConverter<char8_t> sc2(
+ reinterpret_cast<const char8_t *>(src) + sc1.getSourceIndex(), &state,
+ SIZE_MAX, 3);
+
+ res = sc2.pop<CharType16>();
+ ASSERT_TRUE(res.has_value());
+ ASSERT_EQ(static_cast<int>(res.value()), 0xd83e);
+ ASSERT_EQ(static_cast<int>(sc2.getSourceIndex()), 2);
+
+ res = sc2.pop<CharType16>();
+ ASSERT_TRUE(res.has_value());
+ ASSERT_EQ(static_cast<int>(res.value()), 0xdd21);
+ ASSERT_EQ(static_cast<int>(sc2.getSourceIndex()), 2);
+
+ res = sc2.pop<CharType16>();
+ ASSERT_TRUE(res.has_value());
+ ASSERT_EQ(static_cast<int>(res.value()), 0);
+ ASSERT_EQ(static_cast<int>(sc2.getSourceIndex()), 3);
+}
+
+TYPED_TEST(LlvmLibcStringConverterTest, DestLimitUTF8To16, TestCharTypesUTF16) {
+ using CharType16 = ParamType;
+
+ const char *src = "\xF0\x9F\xA4\xA1\xF0\x9F\xA4\xA1"; // 2 clown emojis
+ LIBC_NAMESPACE::internal::mbstate state;
+ LIBC_NAMESPACE::internal::StringConverter<char8_t> sc(
+ reinterpret_cast<const char8_t *>(src), &state, 2);
+
+ auto res = sc.pop<CharType16>();
+ ASSERT_TRUE(res.has_value());
+ ASSERT_EQ(static_cast<int>(sc.getSourceIndex()), 4);
+
+ res = sc.pop<CharType16>();
+ ASSERT_TRUE(res.has_value());
+ ASSERT_EQ(static_cast<int>(sc.getSourceIndex()), 4);
+
+ res = sc.pop<CharType16>(); // no space to pop this into
+ ASSERT_FALSE(res.has_value());
+}
+
+TYPED_TEST(LlvmLibcStringConverterTest, DestLimitUTF16To8, TestCharTypesUTF16) {
+ using CharType16 = ParamType;
+
+ const CharType16 src[] = {static_cast<CharType16>(0xd83e),
+ static_cast<CharType16>(0xdd21),
+ static_cast<CharType16>(0xd83e),
+ static_cast<CharType16>(0xdd21)}; // 2 clown emojis
+ LIBC_NAMESPACE::internal::mbstate state;
+ LIBC_NAMESPACE::internal::StringConverter<CharType16> sc(src, &state, 5);
+
+ auto res = sc.template pop<char8_t>();
+ ASSERT_TRUE(res.has_value());
+ ASSERT_EQ(static_cast<int>(sc.getSourceIndex()), 2);
+
+ res = sc.template pop<char8_t>();
+ ASSERT_TRUE(res.has_value());
+ ASSERT_EQ(static_cast<int>(sc.getSourceIndex()), 2);
+
+ res = sc.template pop<char8_t>();
+ ASSERT_TRUE(res.has_value());
+ ASSERT_EQ(static_cast<int>(sc.getSourceIndex()), 2);
+
+ res = sc.template pop<char8_t>();
+ ASSERT_TRUE(res.has_value());
+ ASSERT_EQ(static_cast<int>(sc.getSourceIndex()), 2);
+
+ res = sc.template pop<char8_t>();
+ ASSERT_FALSE(res.has_value());
+ ASSERT_EQ(static_cast<int>(sc.getSourceIndex()), 2);
+}
+
+// The above tests concern situations that have equivalents in UTF-32
+// The below tests concern situations that are unique to UTF-16, mainly when
+// surrogate pairs are truncated by length limits
+
+TYPED_TEST(LlvmLibcStringConverterTest, DestLimitSurrogateTruncateUTF8To16,
+ TestCharTypesUTF16) {
+ using CharType16 = ParamType;
+
+ const char *src = "\xF0\x9F\xA4\xA1\xF0\x9F\xA4\xA1"; // 2 clown emojis
+ LIBC_NAMESPACE::internal::mbstate state;
+ LIBC_NAMESPACE::internal::StringConverter<char8_t> sc(
+ reinterpret_cast<const char8_t *>(src), &state, 3);
+
+ auto res = sc.pop<CharType16>();
+ ASSERT_TRUE(res.has_value());
+ ASSERT_EQ(static_cast<int>(sc.getSourceIndex()), 4);
+
+ res = sc.pop<CharType16>();
+ ASSERT_TRUE(res.has_value());
+ ASSERT_EQ(static_cast<int>(sc.getSourceIndex()), 4);
+
+ res = sc.pop<CharType16>(); // no space to pop the lower surrogate into
+ ASSERT_FALSE(res.has_value());
+ ASSERT_EQ(static_cast<int>(sc.getSourceIndex()), 4);
+}
+
+TYPED_TEST(LlvmLibcStringConverterTest, SrcLimitSurrogateTruncateUTF16To8,
+ TestCharTypesUTF16) {
+ using CharType16 = ParamType;
+
+ const CharType16 src[] = {
+ static_cast<CharType16>(0xd83e), static_cast<CharType16>(0xdd21),
+ static_cast<CharType16>(0x41), static_cast<CharType16>(0x0)};
+ LIBC_NAMESPACE::internal::mbstate state;
+ LIBC_NAMESPACE::internal::StringConverter<CharType16> sc1(src, &state,
+ SIZE_MAX, 1);
+
+ auto res = sc1.template pop<char8_t>();
+ ASSERT_FALSE(res.has_value());
+ ASSERT_EQ(static_cast<int>(res.error()), -1);
+ ASSERT_EQ(static_cast<int>(sc1.getSourceIndex()), 1);
+
+ // sc2 should pick up where sc1 left off and continue the conversion
+ LIBC_NAMESPACE::internal::StringConverter<CharType16> sc2(
+ src + sc1.getSourceIndex(), &state, SIZE_MAX, 3);
+
+ res = sc2.template pop<char8_t>();
+ ASSERT_TRUE(res.has_value());
+ ASSERT_EQ(static_cast<int>(res.value()), 0xF0);
+ ASSERT_EQ(static_cast<int>(sc2.getSourceIndex()), 1);
+
+ res = sc2.template pop<char8_t>();
+ ASSERT_TRUE(res.has_value());
+ ASSERT_EQ(static_cast<int>(res.value()), 0x9F);
+ ASSERT_EQ(static_cast<int>(sc2.getSourceIndex()), 1);
+
+ res = sc2.template pop<char8_t>();
+ ASSERT_TRUE(res.has_value());
+ ASSERT_EQ(static_cast<int>(res.value()), 0xA4);
+ ASSERT_EQ(static_cast<int>(sc2.getSourceIndex()), 1);
+
+ res = sc2.template pop<char8_t>();
+ ASSERT_TRUE(res.has_value());
+ ASSERT_EQ(static_cast<int>(res.value()), 0xA1);
+ ASSERT_EQ(static_cast<int>(sc2.getSourceIndex()), 1);
+}
diff --git a/libc/test/src/__support/wchar/string_converter_test.cpp b/libc/test/src/__support/wchar/string_converter_test_utf32.cpp
similarity index 100%
rename from libc/test/src/__support/wchar/string_converter_test.cpp
rename to libc/test/src/__support/wchar/string_converter_test_utf32.cpp
diff --git a/libc/test/src/__support/wchar/wcsnrtombs_test.cpp b/libc/test/src/__support/wchar/wcsnrtombs_test.cpp
index 2d431eddf4a6f..fd40c46d7c8f5 100644
--- a/libc/test/src/__support/wchar/wcsnrtombs_test.cpp
+++ b/libc/test/src/__support/wchar/wcsnrtombs_test.cpp
@@ -16,13 +16,7 @@
#include "src/__support/wchar/wcsnrtombs.h"
#include "test/UnitTest/Test.h"
-// TODO: add support for 16-bit widechars to remove this macro
-#ifdef LIBC_TARGET_OS_IS_WINDOWS
-TEST(LlvmLibcStringConverterTest, Windows) {
- // pass on windows for now
-}
-
-#else
+#if defined(LIBC_TYPES_WCHAR_T_IS_UTF32)
TEST(LlvmLibcWcsnrtombs, AllMultibyteLengths) {
LIBC_NAMESPACE::internal::mbstate state;
@@ -187,16 +181,209 @@ TEST(LlvmLibcWcsnrtombs, InvalidCharacter) {
const wchar_t *cur = src;
char mbs[11];
+ auto res = LIBC_NAMESPACE::internal::wcsnrtombs(mbs, &cur, 4, 7, &state1);
+ ASSERT_TRUE(res.has_value());
+ ASSERT_EQ(res.value(), static_cast<size_t>(7));
+
+ LIBC_NAMESPACE::internal::mbstate state2;
+ cur = src;
+ res = LIBC_NAMESPACE::internal::wcsnrtombs(mbs, &cur, 4, 11, &state2);
+ ASSERT_FALSE(res.has_value());
+ ASSERT_EQ(res.error(), EILSEQ);
+}
+
+#if defined(LIBC_ADD_NULL_CHECKS) && !defined(LIBC_HAS_SANITIZER)
+TEST(LlvmLibcWcsnrtombs, NullSrc) {
+ EXPECT_DEATH(
+ [] {
+ LIBC_NAMESPACE::internal::mbstate state;
+ char mbs[10];
+ LIBC_NAMESPACE::internal::wcsnrtombs(mbs, nullptr, 1, 1, &state);
+ },
+ WITH_SIGNAL(-1));
+}
+#endif // LIBC_HAS_ADDRESS_SANITIZER
+
+#elif defined(LIBC_TYPES_WCHAR_T_IS_UTF16)
+
+TEST(LlvmLibcWcsnrtombs, AllMultibyteLengths) {
+ LIBC_NAMESPACE::internal::mbstate state;
+
+ /// clown emoji, sigma symbol, y with diaeresis, letter A
+ const wchar_t src[] = {
+ static_cast<wchar_t>(0xd83e), static_cast<wchar_t>(0xdd21),
+ static_cast<wchar_t>(0x2211), static_cast<wchar_t>(0xff),
+ static_cast<wchar_t>(0x41), static_cast<wchar_t>(0x0)};
+ const wchar_t *cur = src;
+ char mbs[11];
+
+ auto res = LIBC_NAMESPACE::internal::wcsnrtombs(mbs, &cur, 6, 11, &state);
+ ASSERT_TRUE(res.has_value());
+ ASSERT_EQ(res.value(), static_cast<size_t>(10));
+ ASSERT_EQ(cur, nullptr);
+ ASSERT_EQ(mbs[0], '\xF0'); // clown begin
+ ASSERT_EQ(mbs[1], '\x9F');
+ ASSERT_EQ(mbs[2], '\xA4');
+ ASSERT_EQ(mbs[3], '\xA1');
+ ASSERT_EQ(mbs[4], '\xE2'); // sigma begin
+ ASSERT_EQ(mbs[5], '\x88');
+ ASSERT_EQ(mbs[6], '\x91');
+ ASSERT_EQ(mbs[7], '\xC3'); // y diaeresis begin
+ ASSERT_EQ(mbs[8], '\xBF');
+ ASSERT_EQ(mbs[9], '\x41'); // A begin
+ ASSERT_EQ(mbs[10], '\0'); // null terminator
+}
+
+TEST(LlvmLibcWcsnrtombs, DestLimit) {
+ LIBC_NAMESPACE::internal::mbstate state1;
+
+ /// clown emoji, sigma symbol, y with diaeresis, letter A
+ const wchar_t src[] = {
+ static_cast<wchar_t>(0xd83e), static_cast<wchar_t>(0xdd21),
+ static_cast<wchar_t>(0x2211), static_cast<wchar_t>(0xff),
+ static_cast<wchar_t>(0x41), static_cast<wchar_t>(0x0)};
+ const wchar_t *cur = src;
+
+ char mbs[11];
+ for (int i = 0; i < 11; ++i)
+ mbs[i] = '\x01'; // dummy initial values
+
+ auto res = LIBC_NAMESPACE::internal::wcsnrtombs(mbs, &cur, 6, 4, &state1);
+ ASSERT_TRUE(res.has_value());
+ ASSERT_EQ(res.value(), static_cast<size_t>(4));
+ ASSERT_EQ(cur, src + 2);
+ ASSERT_EQ(mbs[0], '\xF0');
+ ASSERT_EQ(mbs[1], '\x9F');
+ ASSERT_EQ(mbs[2], '\xA4');
+ ASSERT_EQ(mbs[3], '\xA1');
+ ASSERT_EQ(mbs[4], '\x01'); // didn't write more than 4 bytes
+
+ for (int i = 0; i < 11; ++i)
+ mbs[i] = '\x01'; // dummy initial values
+ LIBC_NAMESPACE::internal::mbstate state2;
+
+ // not enough bytes to convert the second character, so only converts one
+ cur = src;
+ res = LIBC_NAMESPACE::internal::wcsnrtombs(mbs, &cur, 6, 6, &state2);
+ ASSERT_TRUE(res.has_value());
+ ASSERT_EQ(res.value(), static_cast<size_t>(4));
+ ASSERT_EQ(cur, src + 2);
+ ASSERT_EQ(mbs[0], '\xF0');
+ ASSERT_EQ(mbs[1], '\x9F');
+ ASSERT_EQ(mbs[2], '\xA4');
+ ASSERT_EQ(mbs[3], '\xA1');
+ ASSERT_EQ(mbs[4], '\x01');
+}
+
+TEST(LlvmLibcWcsnrtombs, SrcLimit) {
+ LIBC_NAMESPACE::internal::mbstate state;
+
+ /// clown emoji, sigma symbol, y with diaeresis, letter A
+ const wchar_t src[] = {
+ static_cast<wchar_t>(0xd83e), static_cast<wchar_t>(0xdd21),
+ static_cast<wchar_t>(0x2211), static_cast<wchar_t>(0xff),
+ static_cast<wchar_t>(0x41), static_cast<wchar_t>(0x0)};
+ const wchar_t *cur = src;
+
+ char mbs[11];
+ for (int i = 0; i < 11; ++i)
+ mbs[i] = '\x01'; // dummy initial values
+
+ auto res = LIBC_NAMESPACE::internal::wcsnrtombs(mbs, &cur, 3, 11, &state);
+ ASSERT_TRUE(res.has_value());
+ ASSERT_EQ(res.value(), static_cast<size_t>(7));
+ ASSERT_EQ(cur, src + 3);
+ ASSERT_EQ(mbs[0], '\xF0'); // clown begin
+ ASSERT_EQ(mbs[1], '\x9F');
+ ASSERT_EQ(mbs[2], '\xA4');
+ ASSERT_EQ(mbs[3], '\xA1');
+ ASSERT_EQ(mbs[4], '\xE2'); // sigma begin
+ ASSERT_EQ(mbs[5], '\x88');
+ ASSERT_EQ(mbs[6], '\x91');
+ ASSERT_EQ(mbs[7], '\x01');
+
+ res = LIBC_NAMESPACE::internal::wcsnrtombs(mbs + res.value(), &cur, 100, 11,
+ &state);
+ ASSERT_TRUE(res.has_value());
+ ASSERT_EQ(res.value(), static_cast<size_t>(3));
+ ASSERT_EQ(cur, nullptr);
+ ASSERT_EQ(mbs[0], '\xF0'); // clown begin
+ ASSERT_EQ(mbs[1], '\x9F');
+ ASSERT_EQ(mbs[2], '\xA4');
+ ASSERT_EQ(mbs[3], '\xA1');
+ ASSERT_EQ(mbs[4], '\xE2'); // sigma begin
+ ASSERT_EQ(mbs[5], '\x88');
+ ASSERT_EQ(mbs[6], '\x91');
+ ASSERT_EQ(mbs[7], '\xC3'); // y diaeresis begin
+ ASSERT_EQ(mbs[8], '\xBF');
+ ASSERT_EQ(mbs[9], '\x41'); // A begin
+ ASSERT_EQ(mbs[10], '\0'); // null terminator
+}
+
+TEST(LlvmLibcWcsnrtombs, NullDest) {
+ LIBC_NAMESPACE::internal::mbstate state1;
+
+ const wchar_t src[] = {
+ static_cast<wchar_t>(0xd83e), static_cast<wchar_t>(0xdd21),
+ static_cast<wchar_t>(0x2211), static_cast<wchar_t>(0xff),
+ static_cast<wchar_t>(0x41), static_cast<wchar_t>(0x0)};
+ const wchar_t *cur = src;
+
+ // n parameter ignored when dest is null
+ auto res = LIBC_NAMESPACE::internal::wcsnrtombs(nullptr, &cur, 6, 1, &state1);
+ ASSERT_TRUE(res.has_value());
+ ASSERT_EQ(res.value(), static_cast<size_t>(10));
+ ASSERT_EQ(cur, src); // pointer not updated when dest = null
+
+ LIBC_NAMESPACE::internal::mbstate state2;
+ res = LIBC_NAMESPACE::internal::wcsnrtombs(nullptr, &cur, 6, 100, &state2);
+ ASSERT_TRUE(res.has_value());
+ ASSERT_EQ(res.value(), static_cast<size_t>(10));
+ ASSERT_EQ(cur, src);
+}
+
+TEST(LlvmLibcWcsnrtombs, InvalidState) {
+ // this is more thoroughly tested by CharacterConverter
+ LIBC_NAMESPACE::internal::mbstate state;
+ state.total_bytes = 100;
+
+ const wchar_t src[] = {
+ static_cast<wchar_t>(0xd83e), static_cast<wchar_t>(0xdd21),
+ static_cast<wchar_t>(0x2211), static_cast<wchar_t>(0xff),
+ static_cast<wchar_t>(0x41), static_cast<wchar_t>(0x0)};
+ const wchar_t *cur = src;
+
// n parameter ignored when dest is null
+ auto res = LIBC_NAMESPACE::internal::wcsnrtombs(nullptr, &cur, 6, 1, &state);
+ ASSERT_FALSE(res.has_value());
+ ASSERT_EQ(res.error(), EINVAL);
+}
+
+TEST(LlvmLibcWcsnrtombs, InvalidCharacter) {
+ LIBC_NAMESPACE::internal::mbstate state1;
+
+ const wchar_t src[] = {static_cast<wchar_t>(0xd83e),
+ static_cast<wchar_t>(0xdd21),
+ static_cast<wchar_t>(0x2211),
+ static_cast<wchar_t>(0xdc00), // invalid widechar
+ static_cast<wchar_t>(0x0)};
+ const wchar_t *cur = src;
+ char mbs[11];
+ for (int i = 0; i < 11; ++i)
+ mbs[i] = '\x01'; // dummy initial values
+
auto res = LIBC_NAMESPACE::internal::wcsnrtombs(mbs, &cur, 5, 7, &state1);
ASSERT_TRUE(res.has_value());
ASSERT_EQ(res.value(), static_cast<size_t>(7));
+ ASSERT_EQ(cur, src + 3);
LIBC_NAMESPACE::internal::mbstate state2;
cur = src;
res = LIBC_NAMESPACE::internal::wcsnrtombs(mbs, &cur, 5, 11, &state2);
ASSERT_FALSE(res.has_value());
ASSERT_EQ(res.error(), EILSEQ);
+ ASSERT_EQ(cur, src + 3);
+ ASSERT_EQ(mbs[7], '\x01');
}
#if defined(LIBC_ADD_NULL_CHECKS) && !defined(LIBC_HAS_SANITIZER)
@@ -210,4 +397,28 @@ TEST(LlvmLibcWcsnrtombs, NullSrc) {
WITH_SIGNAL(-1));
}
#endif // LIBC_HAS_ADDRESS_SANITIZER
+
+// The below tests concern situations that are unique to UTF-16, mainly when
+// surrogate pairs are truncated by length limits
+
+TEST(LlvmLibcWcsnrtombs, SrcLimitSurrogateTruncate) {
+ LIBC_NAMESPACE::internal::mbstate state;
+
+ const wchar_t src[] = {static_cast<wchar_t>(0xd83e),
+ static_cast<wchar_t>(0xdd21),
+ static_cast<wchar_t>(0x0)}; // clown emoji
+ const wchar_t *cur = src;
+
+ char mbs[5];
+ for (int i = 0; i < 5; ++i)
+ mbs[i] = '\x01'; // dummy initial values
+
+ auto res = LIBC_NAMESPACE::internal::wcsnrtombs(mbs, &cur, 1, 5, &state);
+ ASSERT_FALSE(res.has_value());
+ ASSERT_EQ(res.error(), EILSEQ);
+ ASSERT_EQ(cur, src);
+ ASSERT_EQ(mbs[0], '\x01');
+ ASSERT_TRUE(LIBC_NAMESPACE::internal::CharacterConverter(&state).isEmpty());
+}
+
#endif
diff --git a/libc/test/src/stdlib/mblen_test.cpp b/libc/test/src/stdlib/mblen_test.cpp
index 2ce5e7c939136..712fa141f2a29 100644
--- a/libc/test/src/stdlib/mblen_test.cpp
+++ b/libc/test/src/stdlib/mblen_test.cpp
@@ -61,6 +61,7 @@ TEST_F(LlvmLibcMBLenTest, ThreeByte) {
ASSERT_ERRNO_SUCCESS();
}
+#if defined(LIBC_TYPES_WCHAR_T_IS_UTF32)
TEST_F(LlvmLibcMBLenTest, FourByte) {
const char ch[4] = {static_cast<char>(0xF0), static_cast<char>(0x9F),
static_cast<char>(0xA4),
@@ -74,6 +75,17 @@ TEST_F(LlvmLibcMBLenTest, FourByte) {
ASSERT_EQ(n, -1);
ASSERT_ERRNO_SUCCESS();
}
+#elif defined(LIBC_TYPES_WCHAR_T_IS_UTF16)
+// UTF-16 mblen cannot process surrogate pair
+TEST_F(LlvmLibcMBLenTest, RejectFourByte) {
+ const char ch[4] = {static_cast<char>(0xF0), static_cast<char>(0x9F),
+ static_cast<char>(0xA4),
+ static_cast<char>(0xA1)}; // 🤡 clown emoji
+ int n = LIBC_NAMESPACE::mblen(ch, 4);
+ ASSERT_EQ(n, -1);
+ ASSERT_ERRNO_EQ(EILSEQ);
+}
+#endif
TEST_F(LlvmLibcMBLenTest, InvalidByte) {
const char ch[1] = {static_cast<char>(0x80)};
diff --git a/libc/test/src/stdlib/mbstowcs_test.cpp b/libc/test/src/stdlib/mbstowcs_test.cpp
index 125683a3eca97..4201a28268cf6 100644
--- a/libc/test/src/stdlib/mbstowcs_test.cpp
+++ b/libc/test/src/stdlib/mbstowcs_test.cpp
@@ -15,6 +15,8 @@
using LlvmLibcMBSToWCSTest = LIBC_NAMESPACE::testing::ErrnoCheckingTest;
+#if defined(LIBC_TYPES_WCHAR_T_IS_UTF32)
+
TEST_F(LlvmLibcMBSToWCSTest, OneByteOneChar) {
const char *ch = "A";
const char *original = ch;
@@ -169,3 +171,171 @@ TEST(LlvmLibcMBSToWCSTest, NullptrCrash) {
WITH_SIGNAL(-1));
}
#endif // LIBC_ADD_NULL_CHECKS
+
+#elif defined(LIBC_TYPES_WCHAR_T_IS_UTF16)
+
+TEST_F(LlvmLibcMBSToWCSTest, OneByteOneChar) {
+ const char *ch = "A";
+ const char *original = ch;
+ wchar_t dest[2];
+ size_t n = LIBC_NAMESPACE::mbstowcs(dest, ch, 1);
+ ASSERT_EQ(static_cast<char>(*dest), 'A');
+ ASSERT_EQ(static_cast<int>(n), 1);
+ // Making sure the pointer is not getting updated
+ ASSERT_EQ(ch, original);
+ ASSERT_ERRNO_SUCCESS();
+
+ n = LIBC_NAMESPACE::mbstowcs(dest + 1, ch + 1, 1);
+ ASSERT_EQ(static_cast<char>(dest[1]), '\0');
+ // Should not include null terminator
+ ASSERT_EQ(static_cast<int>(n), 0);
+ // Making sure the pointer is not getting updated
+ ASSERT_EQ(ch, original);
+ ASSERT_ERRNO_SUCCESS();
+}
+
+TEST_F(LlvmLibcMBSToWCSTest, FourByteOneChar) {
+ const char *src = "\xf0\x9f\x98\xb9"; // laughing cat emoji 😹
+ const char *original = src;
+ wchar_t dest[3];
+ size_t n = LIBC_NAMESPACE::mbstowcs(dest, src, 3);
+ ASSERT_ERRNO_SUCCESS();
+ ASSERT_EQ(static_cast<int>(dest[0]), 55357);
+ ASSERT_EQ(static_cast<int>(dest[1]), 56889);
+ ASSERT_TRUE(dest[2] == L'\0');
+ // Should not count null terminator in number
+ ASSERT_EQ(static_cast<int>(n), 2);
+ // Making sure the pointer is not getting updated
+ ASSERT_EQ(src, original);
+}
+
+TEST_F(LlvmLibcMBSToWCSTest, MultiByteTwoCharacters) {
+ // Two laughing cat emojis "😹😹"
+ const char *src = "\xf0\x9f\x98\xb9\xf0\x9f\x98\xb9";
+ const char *original = src;
+ wchar_t dest[5];
+ size_t n = LIBC_NAMESPACE::mbstowcs(dest, src, 5);
+ ASSERT_ERRNO_SUCCESS();
+ ASSERT_EQ(static_cast<int>(dest[0]), 55357);
+ ASSERT_EQ(static_cast<int>(dest[1]), 56889);
+ ASSERT_EQ(static_cast<int>(dest[2]), 55357);
+ ASSERT_EQ(static_cast<int>(dest[3]), 56889);
+ ASSERT_TRUE(dest[4] == L'\0');
+ // Should not count null terminator in number
+ ASSERT_EQ(static_cast<int>(n), 4);
+ // Making sure the pointer is not getting updated
+ ASSERT_EQ(src, original);
+}
+
+TEST_F(LlvmLibcMBSToWCSTest, MixedNumberOfBytes) {
+ // 'A', sigma symbol 'Σ', recycling symbol '♻', laughing cat emoji '😹'
+ const char *src = "A\xce\xa3\xe2\x99\xbb\xf0\x9f\x98\xb9";
+ const char *original = src;
+ wchar_t dest[6];
+ size_t n = LIBC_NAMESPACE::mbstowcs(dest, src, 6);
+ ASSERT_ERRNO_SUCCESS();
+ ASSERT_EQ(static_cast<char>(dest[0]), 'A');
+ ASSERT_EQ(static_cast<int>(dest[1]), 931);
+ ASSERT_EQ(static_cast<int>(dest[2]), 9851);
+ ASSERT_EQ(static_cast<int>(dest[3]), 55357);
+ ASSERT_EQ(static_cast<int>(dest[4]), 56889);
+ ASSERT_TRUE(dest[5] == L'\0');
+ // Should not count null terminator in number
+ ASSERT_EQ(static_cast<int>(n), 5);
+ // Making sure the pointer is not getting updated
+ ASSERT_EQ(src, original);
+}
+
+TEST_F(LlvmLibcMBSToWCSTest, ReadLessThanStringLength) {
+ // Four laughing cat emojis "😹😹😹😹"
+ const char *src =
+ "\xf0\x9f\x98\xb9\xf0\x9f\x98\xb9\xf0\x9f\x98\xb9\xf0\x9f\x98\xb9";
+ const char *original = src;
+ wchar_t dest[8] = {L'a', L'b', L'c', L'd', L'e', L'f', L'g', L'h'};
+ size_t n = LIBC_NAMESPACE::mbstowcs(dest, src, 6);
+ ASSERT_ERRNO_SUCCESS();
+ // Should have read 3 emojis
+ ASSERT_EQ(static_cast<int>(n), 6);
+ ASSERT_EQ(static_cast<int>(dest[0]), 55357);
+ ASSERT_EQ(static_cast<int>(dest[1]), 56889);
+ ASSERT_EQ(static_cast<int>(dest[2]), 55357);
+ ASSERT_EQ(static_cast<int>(dest[3]), 56889);
+ ASSERT_EQ(static_cast<int>(dest[4]), 55357);
+ ASSERT_EQ(static_cast<int>(dest[5]), 56889);
+ ASSERT_TRUE(dest[6] == L'g');
+ ASSERT_TRUE(dest[7] == L'h');
+ // Making sure the pointer is not getting updated
+ ASSERT_EQ(src, original);
+}
+
+TEST_F(LlvmLibcMBSToWCSTest, InvalidFirstByte) {
+ // 0x80 is invalid first byte of mb character
+ const char *src =
+ "\x80\x9f\x98\xb9\xf0\x9f\x98\xb9\xf0\x9f\x98\xb9\xf0\x9f\x98\xb9";
+ wchar_t dest[3];
+ size_t n = LIBC_NAMESPACE::mbstowcs(dest, src, 3);
+ // Should return error and set errno
+ ASSERT_EQ(static_cast<int>(n), -1);
+ ASSERT_ERRNO_EQ(EILSEQ);
+}
+
+TEST_F(LlvmLibcMBSToWCSTest, InvalidMiddleByte) {
+ // The 7th byte is invalid for a 4 byte character
+ const char *src =
+ "\xf0\x9f\x98\xb9\xf0\x9f\xf0\xb9\xf0\x9f\x98\xb9\xf0\x9f\x98\xb9";
+ const char *original = src;
+ wchar_t dest[3];
+ size_t n = LIBC_NAMESPACE::mbstowcs(dest, src, 5);
+ // Should return error and set errno
+ ASSERT_EQ(static_cast<int>(n), -1);
+ ASSERT_ERRNO_EQ(EILSEQ);
+ // Making sure the pointer is not getting updated
+ ASSERT_EQ(src, original);
+}
+
+TEST_F(LlvmLibcMBSToWCSTest, NullDestination) {
+ // Four laughing cat emojis "😹😹😹😹"
+ const char *src =
+ "\xf0\x9f\x98\xb9\xf0\x9f\x98\xb9\xf0\x9f\x98\xb9\xf0\x9f\x98\xb9";
+ const char *original = src;
+ size_t n = LIBC_NAMESPACE::mbstowcs(nullptr, src, 4);
+ ASSERT_ERRNO_SUCCESS();
+ // Null destination should ignore len and read till end of string
+ ASSERT_EQ(static_cast<int>(n), 8);
+ // Making sure the pointer is not getting updated
+ ASSERT_EQ(src, original);
+}
+
+TEST_F(LlvmLibcMBSToWCSTest, ErrnoChecks) {
+ // Two laughing cat emojis and invalid 3rd mb char (3rd byte of it)
+ const char *src =
+ "\xf0\x9f\x98\xb9\xf0\x9f\x98\xb9\xf0\x9f\xf0\xb9\xf0\x9f\x98\xb9";
+ const char *original = src;
+ wchar_t dest[7];
+ // First two bytes are valid --> should not set errno
+ size_t n = LIBC_NAMESPACE::mbstowcs(dest, src, 4);
+ ASSERT_ERRNO_SUCCESS();
+ ASSERT_EQ(static_cast<int>(n), 4);
+ ASSERT_EQ(static_cast<int>(dest[0]), 55357);
+ ASSERT_EQ(static_cast<int>(dest[1]), 56889);
+ ASSERT_EQ(static_cast<int>(dest[2]), 55357);
+ ASSERT_EQ(static_cast<int>(dest[3]), 56889);
+ // Making sure the pointer is not getting updated
+ ASSERT_EQ(src, original);
+ // Trying to read the 3rd byte should set errno
+ n = LIBC_NAMESPACE::mbstowcs(dest, src + 2, 2);
+ ASSERT_ERRNO_EQ(EILSEQ);
+ ASSERT_EQ(static_cast<int>(n), -1);
+ // Making sure the pointer is not getting updated
+ ASSERT_EQ(src, original);
+}
+
+#if defined(LIBC_ADD_NULL_CHECKS)
+TEST(LlvmLibcMBSToWCSTest, NullptrCrash) {
+ // Passing in a nullptr should crash the program.
+ EXPECT_DEATH([] { LIBC_NAMESPACE::mbstowcs(nullptr, nullptr, 1); },
+ WITH_SIGNAL(-1));
+}
+#endif // LIBC_ADD_NULL_CHECKS
+
+#endif
diff --git a/libc/test/src/stdlib/mbtowc_test.cpp b/libc/test/src/stdlib/mbtowc_test.cpp
index 7946e077b647e..60b9fb5e31de7 100644
--- a/libc/test/src/stdlib/mbtowc_test.cpp
+++ b/libc/test/src/stdlib/mbtowc_test.cpp
@@ -58,6 +58,7 @@ TEST_F(LlvmLibcMBToWCTest, ThreeByte) {
ASSERT_ERRNO_EQ(EILSEQ);
}
+#if defined(LIBC_TYPES_WCHAR_T_IS_UTF32)
TEST_F(LlvmLibcMBToWCTest, FourByte) {
const char ch[4] = {static_cast<char>(0xF0), static_cast<char>(0x9F),
static_cast<char>(0xA4),
@@ -72,6 +73,18 @@ TEST_F(LlvmLibcMBToWCTest, FourByte) {
ASSERT_EQ(n, -1);
ASSERT_ERRNO_EQ(EILSEQ);
}
+#elif defined(LIBC_TYPES_WCHAR_T_IS_UTF16)
+// UTF-16 mbtowc cannot process surrogate pair
+TEST_F(LlvmLibcMBToWCTest, RejectFourByte) {
+ const char ch[4] = {static_cast<char>(0xF0), static_cast<char>(0x9F),
+ static_cast<char>(0xA4),
+ static_cast<char>(0xA1)}; // 🤡 clown emoji
+ wchar_t dest[2];
+ int n = LIBC_NAMESPACE::mbtowc(dest, ch, 4);
+ ASSERT_EQ(n, -1);
+ ASSERT_ERRNO_EQ(EILSEQ);
+}
+#endif
TEST_F(LlvmLibcMBToWCTest, InvalidByte) {
const char ch[1] = {static_cast<char>(0x80)};
diff --git a/libc/test/src/stdlib/wcstombs_test.cpp b/libc/test/src/stdlib/wcstombs_test.cpp
index 792a4edb51b38..b5b9ba72f6fa9 100644
--- a/libc/test/src/stdlib/wcstombs_test.cpp
+++ b/libc/test/src/stdlib/wcstombs_test.cpp
@@ -15,6 +15,8 @@ using LlvmLibcWcstombs = LIBC_NAMESPACE::testing::ErrnoCheckingTest;
// these tests are fairly simple as this function just calls into the internal
// wcsnrtombs which is more thoroughly tested
+#if defined(LIBC_TYPES_WCHAR_T_IS_UTF32)
+
TEST_F(LlvmLibcWcstombs, AllMultibyteLengths) {
/// clown emoji, sigma symbol, y with diaeresis, letter A
const wchar_t src[] = {static_cast<wchar_t>(0x1f921),
@@ -76,9 +78,80 @@ TEST_F(LlvmLibcWcstombs, ErrnoTest) {
static_cast<wchar_t>(0x0)};
char mbs[11];
- // n parameter ignored when dest is null
ASSERT_EQ(LIBC_NAMESPACE::wcstombs(mbs, src, 7), static_cast<size_t>(7));
ASSERT_ERRNO_SUCCESS();
ASSERT_EQ(LIBC_NAMESPACE::wcstombs(mbs, src, 100), static_cast<size_t>(-1));
ASSERT_ERRNO_EQ(EILSEQ);
}
+
+#elif defined(LIBC_TYPES_WCHAR_T_IS_UTF16)
+
+TEST_F(LlvmLibcWcstombs, AllMultibyteLengths) {
+ /// clown emoji, sigma symbol, y with diaeresis, letter A
+ const wchar_t src[] = {
+ static_cast<wchar_t>(0xd83e), static_cast<wchar_t>(0xdd21),
+ static_cast<wchar_t>(0x2211), static_cast<wchar_t>(0xff),
+ static_cast<wchar_t>(0x41), static_cast<wchar_t>(0x0)};
+ char mbs[11];
+
+ ASSERT_EQ(LIBC_NAMESPACE::wcstombs(mbs, src, 11), static_cast<size_t>(10));
+ ASSERT_ERRNO_SUCCESS();
+ ASSERT_EQ(mbs[0], '\xF0'); // clown begin
+ ASSERT_EQ(mbs[1], '\x9F');
+ ASSERT_EQ(mbs[2], '\xA4');
+ ASSERT_EQ(mbs[3], '\xA1');
+ ASSERT_EQ(mbs[4], '\xE2'); // sigma begin
+ ASSERT_EQ(mbs[5], '\x88');
+ ASSERT_EQ(mbs[6], '\x91');
+ ASSERT_EQ(mbs[7], '\xC3'); // y diaeresis begin
+ ASSERT_EQ(mbs[8], '\xBF');
+ ASSERT_EQ(mbs[9], '\x41'); // A begin
+ ASSERT_EQ(mbs[10], '\0'); // null terminator
+}
+
+TEST_F(LlvmLibcWcstombs, DestLimit) {
+ /// clown emoji, sigma symbol, y with diaeresis, letter A
+ const wchar_t src[] = {
+ static_cast<wchar_t>(0xd83e), static_cast<wchar_t>(0xdd21),
+ static_cast<wchar_t>(0x2211), static_cast<wchar_t>(0xff),
+ static_cast<wchar_t>(0x41), static_cast<wchar_t>(0x0)};
+ char mbs[11];
+ for (int i = 0; i < 11; ++i)
+ mbs[i] = '\x01'; // dummy initial values
+
+ ASSERT_EQ(LIBC_NAMESPACE::wcstombs(mbs, src, 4), static_cast<size_t>(4));
+ ASSERT_ERRNO_SUCCESS();
+ ASSERT_EQ(mbs[0], '\xF0');
+ ASSERT_EQ(mbs[1], '\x9F');
+ ASSERT_EQ(mbs[2], '\xA4');
+ ASSERT_EQ(mbs[3], '\xA1');
+ ASSERT_EQ(mbs[4], '\x01'); // didn't write more than 4 bytes
+
+ for (int i = 0; i < 11; ++i)
+ mbs[i] = '\x01'; // dummy initial values
+
+ // not enough bytes to convert the second character, so only converts one
+ ASSERT_EQ(LIBC_NAMESPACE::wcstombs(mbs, src, 6), static_cast<size_t>(4));
+ ASSERT_ERRNO_SUCCESS();
+ ASSERT_EQ(mbs[0], '\xF0');
+ ASSERT_EQ(mbs[1], '\x9F');
+ ASSERT_EQ(mbs[2], '\xA4');
+ ASSERT_EQ(mbs[3], '\xA1');
+ ASSERT_EQ(mbs[4], '\x01');
+}
+
+TEST_F(LlvmLibcWcstombs, ErrnoTest) {
+ const wchar_t src[] = {static_cast<wchar_t>(0xd83e),
+ static_cast<wchar_t>(0xdd21),
+ static_cast<wchar_t>(0x2211),
+ static_cast<wchar_t>(0xdc00), // invalid widechar
+ static_cast<wchar_t>(0x0)};
+ char mbs[11];
+
+ ASSERT_EQ(LIBC_NAMESPACE::wcstombs(mbs, src, 7), static_cast<size_t>(7));
+ ASSERT_ERRNO_SUCCESS();
+ ASSERT_EQ(LIBC_NAMESPACE::wcstombs(mbs, src, 100), static_cast<size_t>(-1));
+ ASSERT_ERRNO_EQ(EILSEQ);
+}
+
+#endif
diff --git a/libc/test/src/stdlib/wctomb_test.cpp b/libc/test/src/stdlib/wctomb_test.cpp
index 56bebf87a2796..12ac1d5c1872e 100644
--- a/libc/test/src/stdlib/wctomb_test.cpp
+++ b/libc/test/src/stdlib/wctomb_test.cpp
@@ -43,6 +43,8 @@ TEST(LlvmLibcWCToMBTest, ThreeByte) {
ASSERT_EQ(mb[2], static_cast<char>(0x95));
}
+// UTF-16 wctomb cannot process this
+#if defined(LIBC_TYPES_WCHAR_T_IS_UTF32)
TEST(LlvmLibcWCToMBTest, FourByte) {
// testing utf32: 0x1f921 -> utf8: 0xf0 0x9f 0xa4 0xa1
wchar_t wc = 0x1f921;
@@ -54,6 +56,7 @@ TEST(LlvmLibcWCToMBTest, FourByte) {
ASSERT_EQ(mb[2], static_cast<char>(0xa4));
ASSERT_EQ(mb[3], static_cast<char>(0xa1));
}
+#endif
TEST(LlvmLibcWCToMBTest, NullString) {
wchar_t wc = L'A';
@@ -64,6 +67,8 @@ TEST(LlvmLibcWCToMBTest, NullString) {
ASSERT_EQ(cnt, 0);
}
+// UTF-16 wcrtomb cannot process this
+#if defined(LIBC_TYPES_WCHAR_T_IS_UTF32)
TEST(LlvmLibcWCToMBTest, InvalidWchar) {
wchar_t wc = 0x12ffff;
char mb[4];
@@ -71,3 +76,25 @@ TEST(LlvmLibcWCToMBTest, InvalidWchar) {
ASSERT_EQ(cnt, -1);
ASSERT_ERRNO_EQ(EILSEQ);
}
+#endif
+
+// Tests unique to UTF-16 wcrtomb
+#if defined(LIBC_TYPES_WCHAR_T_IS_UTF16)
+TEST_F(LlvmLibcWCToMBTest, InvalidWchar) {
+ wchar_t wc = 0xdc00;
+ char mb[4];
+ size_t cnt = LIBC_NAMESPACE::wctomb(mb, wc);
+ ASSERT_EQ(cnt, static_cast<size_t>(-1));
+ ASSERT_ERRNO_EQ(EILSEQ);
+}
+
+// We fail with EILSEQ on high surrogate, since we have no way to consume a
+// surrogate pair across 2 consecutive calls, unlike with c16rtomb
+TEST_F(LlvmLibcWCToMBTest, HighSurrogate) {
+ wchar_t wc = 0xd800;
+ char mb[4];
+ size_t cnt = LIBC_NAMESPACE::wctomb(mb, wc);
+ ASSERT_EQ(cnt, static_cast<size_t>(-1));
+ ASSERT_ERRNO_EQ(EILSEQ);
+}
+#endif
diff --git a/libc/test/src/wchar/CMakeLists.txt b/libc/test/src/wchar/CMakeLists.txt
index d511a1f0f4541..13269bbfc70d0 100644
--- a/libc/test/src/wchar/CMakeLists.txt
+++ b/libc/test/src/wchar/CMakeLists.txt
@@ -89,7 +89,7 @@ add_libc_test(
libc.hdr.errno_macros
libc.src.__support.wchar.mbstate
libc.src.string.memset
- libc.src.wchar.mbsrlen
+ libc.src.wchar.mbrlen
libc.hdr.types.mbstate_t
libc.hdr.types.wchar_t
libc.test.UnitTest.ErrnoCheckingTest
@@ -143,6 +143,7 @@ add_libc_test(
wcrtomb_test.cpp
DEPENDS
libc.hdr.errno_macros
+ libc.src.wchar.mbsinit
libc.src.wchar.wcrtomb
libc.src.string.memset
libc.hdr.types.wchar_t
diff --git a/libc/test/src/wchar/mbrlen_test.cpp b/libc/test/src/wchar/mbrlen_test.cpp
index 15892bd135d90..a31667e90a9fc 100644
--- a/libc/test/src/wchar/mbrlen_test.cpp
+++ b/libc/test/src/wchar/mbrlen_test.cpp
@@ -64,6 +64,7 @@ TEST_F(LlvmLibcMBRLenTest, ThreeByte) {
ASSERT_ERRNO_SUCCESS();
}
+#if defined(LIBC_TYPES_WCHAR_T_IS_UTF32)
TEST_F(LlvmLibcMBRLenTest, FourByte) {
const char ch[4] = {static_cast<char>(0xF0), static_cast<char>(0x9F),
static_cast<char>(0xA4),
@@ -89,6 +90,19 @@ TEST_F(LlvmLibcMBRLenTest, FourByte) {
ASSERT_EQ(static_cast<int>(n), 1);
ASSERT_ERRNO_SUCCESS();
}
+#elif defined(LIBC_TYPES_WCHAR_T_IS_UTF16)
+// UTF-16 mbrlen cannot process surrogate pair
+TEST_F(LlvmLibcMBRLenTest, RejectFourByte) {
+ const char ch[4] = {static_cast<char>(0xF0), static_cast<char>(0x9F),
+ static_cast<char>(0xA4),
+ static_cast<char>(0xA1)}; // 🤡 clown emoji
+ mbstate_t mb;
+ LIBC_NAMESPACE::memset(&mb, 0, sizeof(mbstate_t));
+ size_t n = LIBC_NAMESPACE::mbrlen(ch, 4, &mb);
+ ASSERT_EQ(static_cast<int>(n), -1);
+ ASSERT_ERRNO_EQ(EILSEQ);
+}
+#endif
TEST_F(LlvmLibcMBRLenTest, InvalidByte) {
const char ch[1] = {static_cast<char>(0x80)};
diff --git a/libc/test/src/wchar/mbrtowc_test.cpp b/libc/test/src/wchar/mbrtowc_test.cpp
index 5604d01782e87..48af78491df7d 100644
--- a/libc/test/src/wchar/mbrtowc_test.cpp
+++ b/libc/test/src/wchar/mbrtowc_test.cpp
@@ -76,6 +76,7 @@ TEST_F(LlvmLibcMBRToWCTest, ThreeByte) {
ASSERT_ERRNO_SUCCESS();
}
+#if defined(LIBC_TYPES_WCHAR_T_IS_UTF32)
TEST_F(LlvmLibcMBRToWCTest, FourByte) {
const char ch[4] = {static_cast<char>(0xF0), static_cast<char>(0x9F),
static_cast<char>(0xA4),
@@ -97,6 +98,20 @@ TEST_F(LlvmLibcMBRToWCTest, FourByte) {
ASSERT_EQ(static_cast<int>(*dest), 129313);
ASSERT_ERRNO_SUCCESS();
}
+#elif defined(LIBC_TYPES_WCHAR_T_IS_UTF16)
+// UTF-16 mbrtowc cannot process surrogate pair
+TEST_F(LlvmLibcMBRToWCTest, RejectFourByte) {
+ const char ch[4] = {static_cast<char>(0xF0), static_cast<char>(0x9F),
+ static_cast<char>(0xA4),
+ static_cast<char>(0xA1)}; // 🤡 clown emoji
+ wchar_t dest[2];
+ mbstate_t mb;
+ LIBC_NAMESPACE::memset(&mb, 0, sizeof(mbstate_t));
+ size_t n = LIBC_NAMESPACE::mbrtowc(dest, ch, 4, &mb);
+ ASSERT_EQ(static_cast<int>(n), -1);
+ ASSERT_ERRNO_EQ(EILSEQ);
+}
+#endif
TEST_F(LlvmLibcMBRToWCTest, InvalidByte) {
const char ch[1] = {static_cast<char>(0x80)};
@@ -190,6 +205,7 @@ TEST_F(LlvmLibcMBRToWCTest, NullString) {
ASSERT_ERRNO_SUCCESS();
}
+#if defined(LIBC_TYPES_WCHAR_T_IS_UTF32)
TEST_F(LlvmLibcMBRToWCTest, NullDest) {
const char ch[4] = {static_cast<char>(0xF0), static_cast<char>(0x9F),
static_cast<char>(0xA4),
@@ -201,6 +217,19 @@ TEST_F(LlvmLibcMBRToWCTest, NullDest) {
ASSERT_EQ(static_cast<int>(n), 4);
ASSERT_ERRNO_SUCCESS();
}
+#elif defined(LIBC_TYPES_WCHAR_T_IS_UTF16)
+// UTF-16 mbrtowc cannot process surrogate pair
+TEST_F(LlvmLibcMBRToWCTest, NullDest) {
+ const char ch[3] = {static_cast<char>(0xE2), static_cast<char>(0x88),
+ static_cast<char>(0x91)}; // ∑ sigma symbol
+ mbstate_t mb;
+ LIBC_NAMESPACE::memset(&mb, 0, sizeof(mbstate_t));
+ // reading nullptr should return correct size
+ size_t n = LIBC_NAMESPACE::mbrtowc(nullptr, ch, 10, &mb);
+ ASSERT_EQ(static_cast<int>(n), 3);
+ ASSERT_ERRNO_SUCCESS();
+}
+#endif
TEST_F(LlvmLibcMBRToWCTest, InvalidMBState) {
const char ch[4] = {static_cast<char>(0xC2), static_cast<char>(0x8E),
diff --git a/libc/test/src/wchar/mbsinit_test.cpp b/libc/test/src/wchar/mbsinit_test.cpp
index ecb48aa921608..93d3a3d11ceac 100644
--- a/libc/test/src/wchar/mbsinit_test.cpp
+++ b/libc/test/src/wchar/mbsinit_test.cpp
@@ -19,6 +19,7 @@ TEST(LlvmLibcMBSInitTest, EmptyState) {
ASSERT_NE(LIBC_NAMESPACE::mbsinit(nullptr), 0);
}
+#if defined(LIBC_TYPES_WCHAR_T_IS_UTF32)
TEST(LlvmLibcMBSInitTest, ConversionTest) {
const char *src = "\xf0\x9f\xa4\xa3"; // 4 byte emoji
wchar_t dest[2];
@@ -31,3 +32,18 @@ TEST(LlvmLibcMBSInitTest, ConversionTest) {
LIBC_NAMESPACE::mbrtowc(dest, src + 2, 2, &ps); // complete conversion
ASSERT_NE(LIBC_NAMESPACE::mbsinit(&ps), 0); // state should be reset now
}
+#elif defined(LIBC_TYPES_WCHAR_T_IS_UTF16)
+// UTF-16 mbrtowc cannot process surrogate pair
+TEST(LlvmLibcMBSInitTest, ConversionTest) {
+ const char *src = "\xe2\x88\x91"; // ∑ sigma symbol
+ wchar_t dest[2];
+ mbstate_t ps;
+ LIBC_NAMESPACE::memset(&ps, 0, sizeof(mbstate_t));
+
+ ASSERT_NE(LIBC_NAMESPACE::mbsinit(&ps), 0);
+ LIBC_NAMESPACE::mbrtowc(dest, src, 2, &ps); // partial conversion
+ ASSERT_EQ(LIBC_NAMESPACE::mbsinit(&ps), 0);
+ LIBC_NAMESPACE::mbrtowc(dest, src + 2, 1, &ps); // complete conversion
+ ASSERT_NE(LIBC_NAMESPACE::mbsinit(&ps), 0); // state should be reset now
+}
+#endif
diff --git a/libc/test/src/wchar/mbsnrtowcs_test.cpp b/libc/test/src/wchar/mbsnrtowcs_test.cpp
index 3cbe33c899a9a..49654cefbafdd 100644
--- a/libc/test/src/wchar/mbsnrtowcs_test.cpp
+++ b/libc/test/src/wchar/mbsnrtowcs_test.cpp
@@ -18,6 +18,8 @@
using LlvmLibcMBSNRToWCSTest = LIBC_NAMESPACE::testing::ErrnoCheckingTest;
+#if defined(LIBC_TYPES_WCHAR_T_IS_UTF32)
+
TEST_F(LlvmLibcMBSNRToWCSTest, OneByteOneChar) {
const char *ch = "A";
const char *original = ch;
@@ -210,3 +212,208 @@ TEST(LlvmLibcMBSNRToWCSTest, NullptrCrash) {
WITH_SIGNAL(-1));
}
#endif // LIBC_ADD_NULL_CHECKS
+
+#elif defined(LIBC_TYPES_WCHAR_T_IS_UTF16)
+
+TEST_F(LlvmLibcMBSNRToWCSTest, OneByteOneChar) {
+ const char *ch = "A";
+ const char *original = ch;
+ wchar_t dest[2];
+ mbstate_t mb;
+ LIBC_NAMESPACE::memset(&mb, 0, sizeof(mbstate_t));
+ size_t n = LIBC_NAMESPACE::mbsnrtowcs(dest, &ch, 1, 1, &mb);
+ ASSERT_EQ(static_cast<char>(*dest), 'A');
+ ASSERT_EQ(static_cast<int>(n), 1);
+ // Should point to null terminator now
+ ASSERT_EQ(ch, original + 1);
+ ASSERT_ERRNO_SUCCESS();
+
+ n = LIBC_NAMESPACE::mbsnrtowcs(dest + 1, &ch, 1, 1, &mb);
+ ASSERT_EQ(static_cast<char>(dest[1]), '\0');
+ // Should not include null terminator
+ ASSERT_EQ(static_cast<int>(n), 0);
+ // Should now be a nullptr
+ ASSERT_EQ(ch, nullptr);
+ ASSERT_ERRNO_SUCCESS();
+}
+
+TEST_F(LlvmLibcMBSNRToWCSTest, FourByteOneChar) {
+ const char *src = "\xf0\x9f\x98\xb9"; // laughing cat emoji 😹
+ const char *original = src;
+ wchar_t dest[3];
+ mbstate_t mb;
+ LIBC_NAMESPACE::memset(&mb, 0, sizeof(mbstate_t));
+ // Not enough bytes for the full character
+ size_t n = LIBC_NAMESPACE::mbsnrtowcs(dest, &src, 3, 3, &mb);
+ ASSERT_ERRNO_SUCCESS();
+ ASSERT_EQ(static_cast<int>(n), 0);
+ ASSERT_EQ(src, original + 3);
+ // Needs 2 more bytes (last byte of cat + null terminator)
+ n = LIBC_NAMESPACE::mbsnrtowcs(dest, &src, 2, 3, &mb);
+ ASSERT_ERRNO_SUCCESS();
+ // Does not include null terminator
+ ASSERT_EQ(static_cast<int>(n), 2);
+ ASSERT_EQ(src, nullptr);
+ ASSERT_EQ(static_cast<int>(dest[0]), 55357);
+ ASSERT_EQ(static_cast<int>(dest[1]), 56889);
+ ASSERT_TRUE(dest[2] == L'\0');
+}
+
+TEST_F(LlvmLibcMBSNRToWCSTest, MixedNumberOfBytes) {
+ // 'A', sigma symbol 'Σ', recycling symbol '♻', laughing cat emoji '😹'
+ const char *src = "A\xce\xa3\xe2\x99\xbb\xf0\x9f\x98\xb9";
+ const char *original = src;
+ wchar_t dest[6];
+ mbstate_t mb;
+ LIBC_NAMESPACE::memset(&mb, 0, sizeof(mbstate_t));
+
+ // Read 'A'
+ size_t n = LIBC_NAMESPACE::mbsnrtowcs(dest, &src, 1, 1, &mb);
+ ASSERT_ERRNO_SUCCESS();
+ ASSERT_EQ(static_cast<char>(dest[0]), 'A');
+ ASSERT_EQ(static_cast<int>(n), 1);
+ ASSERT_EQ(src, original + 1);
+
+ // Read sigma 'Σ'
+ n = LIBC_NAMESPACE::mbsnrtowcs(dest + 1, &src, 2, 1, &mb);
+ ASSERT_ERRNO_SUCCESS();
+ ASSERT_EQ(static_cast<int>(dest[1]), 931);
+ ASSERT_EQ(static_cast<int>(n), 1);
+ ASSERT_EQ(src, original + 3);
+
+ // Read recycling '♻'
+ n = LIBC_NAMESPACE::mbsnrtowcs(dest + 2, &src, 2, 5, &mb);
+ ASSERT_ERRNO_SUCCESS();
+ ASSERT_EQ(static_cast<int>(n), 0);
+ ASSERT_EQ(src, original + 5);
+ n = LIBC_NAMESPACE::mbsnrtowcs(dest + 2, &src, 1, 1, &mb);
+ ASSERT_ERRNO_SUCCESS();
+ ASSERT_EQ(static_cast<int>(n), 1);
+ ASSERT_EQ(src, original + 6);
+ ASSERT_EQ(static_cast<int>(dest[2]), 9851);
+
+ // Read laughing cat emoji '😹'
+ n = LIBC_NAMESPACE::mbsnrtowcs(dest + 3, &src, 4, 5, &mb);
+ ASSERT_ERRNO_SUCCESS();
+ ASSERT_EQ(static_cast<int>(n), 2);
+ ASSERT_EQ(src, original + 10);
+ ASSERT_EQ(static_cast<int>(dest[3]), 55357);
+ ASSERT_EQ(static_cast<int>(dest[4]), 56889);
+
+ n = LIBC_NAMESPACE::mbsnrtowcs(dest + 5, &src, 4, 4, nullptr);
+ ASSERT_TRUE(dest[5] == L'\0');
+ ASSERT_ERRNO_SUCCESS();
+ // Should not count null terminator in number
+ ASSERT_EQ(static_cast<int>(n), 0);
+ // Should now be a nullptr
+ ASSERT_EQ(src, nullptr);
+}
+
+TEST_F(LlvmLibcMBSNRToWCSTest, ReadLessThanStringLength) {
+ // Four laughing cat emojis "😹😹😹😹"
+ const char *src =
+ "\xf0\x9f\x98\xb9\xf0\x9f\x98\xb9\xf0\x9f\x98\xb9\xf0\x9f\x98\xb9";
+ const char *original = src;
+ wchar_t dest[8] = {L'a', L'b', L'c', L'd', L'e', L'f', L'g', L'h'};
+ size_t n = LIBC_NAMESPACE::mbsnrtowcs(dest, &src, 100, 6, nullptr);
+ ASSERT_ERRNO_SUCCESS();
+ // Should have read 3 emojis
+ ASSERT_EQ(static_cast<int>(n), 6);
+ ASSERT_EQ(static_cast<int>(dest[0]), 55357);
+ ASSERT_EQ(static_cast<int>(dest[1]), 56889);
+ ASSERT_EQ(static_cast<int>(dest[2]), 55357);
+ ASSERT_EQ(static_cast<int>(dest[3]), 56889);
+ ASSERT_EQ(static_cast<int>(dest[4]), 55357);
+ ASSERT_EQ(static_cast<int>(dest[5]), 56889);
+ ASSERT_TRUE(dest[6] == L'g');
+ ASSERT_TRUE(dest[7] == L'h');
+ // Read three laughing cat emojis, 12 bytes
+ ASSERT_EQ(src, original + 12);
+}
+
+TEST_F(LlvmLibcMBSNRToWCSTest, InvalidFirstByte) {
+ // 0x80 is invalid first byte of mb character
+ const char *src =
+ "\x80\x9f\x98\xb9\xf0\x9f\x98\xb9\xf0\x9f\x98\xb9\xf0\x9f\x98\xb9";
+ const char *original = src;
+ wchar_t dest[3];
+ size_t n = LIBC_NAMESPACE::mbsnrtowcs(dest, &src, 88, 88, nullptr);
+ // Should return error and set errno
+ ASSERT_EQ(static_cast<int>(n), -1);
+ ASSERT_ERRNO_EQ(EILSEQ);
+ // Should not update pointer
+ ASSERT_EQ(src, original);
+}
+
+TEST_F(LlvmLibcMBSNRToWCSTest, InvalidMiddleByte) {
+ // The 7th byte is invalid for a 4 byte character
+ const char *src =
+ "\xf0\x9f\x98\xb9\xf0\x9f\xf0\xb9\xf0\x9f\x98\xb9\xf0\x9f\x98\xb9";
+ const char *original = src;
+ wchar_t dest[3];
+ mbstate_t mb;
+ LIBC_NAMESPACE::memset(&mb, 0, sizeof(mbstate_t));
+ // Successfully read one character and first byte of the second character
+ size_t n = LIBC_NAMESPACE::mbsnrtowcs(dest, &src, 5, 88, &mb);
+ ASSERT_EQ(static_cast<int>(n), 2);
+ ASSERT_ERRNO_SUCCESS();
+ ASSERT_EQ(src, original + 5);
+ ASSERT_EQ(static_cast<int>(dest[0]), 55357);
+ ASSERT_EQ(static_cast<int>(dest[1]), 56889);
+
+ n = LIBC_NAMESPACE::mbsnrtowcs(dest + 1, &src, 5, 88, &mb);
+ // Should return error, set errno, and not update the pointer
+ ASSERT_EQ(static_cast<int>(n), -1);
+ ASSERT_ERRNO_EQ(EILSEQ);
+ ASSERT_EQ(src, original + 5);
+}
+
+TEST_F(LlvmLibcMBSNRToWCSTest, NullDestination) {
+ // Four laughing cat emojis "😹😹😹😹"
+ const char *src =
+ "\xf0\x9f\x98\xb9\xf0\x9f\x98\xb9\xf0\x9f\x98\xb9\xf0\x9f\x98\xb9";
+ const char *original = src;
+ size_t n = LIBC_NAMESPACE::mbsnrtowcs(nullptr, &src, 88, 88, nullptr);
+ ASSERT_ERRNO_SUCCESS();
+ // Null destination should ignore len and read till end of string
+ ASSERT_EQ(static_cast<int>(n), 8);
+ // It should also not change the src pointer
+ ASSERT_EQ(src, original);
+}
+
+TEST_F(LlvmLibcMBSNRToWCSTest, ErrnoChecks) {
+ // Two laughing cat emojis and invalid 3rd mb char (3rd byte of it)
+ const char *src =
+ "\xf0\x9f\x98\xb9\xf0\x9f\x98\xb9\xf0\x9f\xf0\xb9\xf0\x9f\x98\xb9";
+ const char *original = src;
+ wchar_t dest[5];
+ mbstate_t mb;
+ LIBC_NAMESPACE::memset(&mb, 0, sizeof(mbstate_t));
+ // First two bytes are valid --> should not set errno
+ size_t n = LIBC_NAMESPACE::mbsnrtowcs(dest, &src, 80, 4, &mb);
+ ASSERT_ERRNO_SUCCESS();
+ ASSERT_EQ(static_cast<int>(n), 4);
+ ASSERT_EQ(static_cast<int>(dest[0]), 55357);
+ ASSERT_EQ(static_cast<int>(dest[1]), 56889);
+ ASSERT_EQ(static_cast<int>(dest[0]), 55357);
+ ASSERT_EQ(static_cast<int>(dest[1]), 56889);
+ ASSERT_EQ(src, original + 8);
+
+ // Trying to read the 3rd byte should set errno
+ n = LIBC_NAMESPACE::mbsnrtowcs(dest + 2, &src, 4, 2, &mb);
+ ASSERT_ERRNO_EQ(EILSEQ);
+ ASSERT_EQ(static_cast<int>(n), -1);
+ // Should not move the pointer
+ ASSERT_EQ(src, original + 8);
+}
+
+#if defined(LIBC_ADD_NULL_CHECKS)
+TEST(LlvmLibcMBSNRToWCSTest, NullptrCrash) {
+ // Passing in a nullptr should crash the program.
+ EXPECT_DEATH(
+ [] { LIBC_NAMESPACE::mbsnrtowcs(nullptr, nullptr, 1, 1, nullptr); },
+ WITH_SIGNAL(-1));
+}
+#endif // LIBC_ADD_NULL_CHECKS
+
+#endif
diff --git a/libc/test/src/wchar/mbsrtowcs_test.cpp b/libc/test/src/wchar/mbsrtowcs_test.cpp
index 93d10d50f48c3..64300bad32f2a 100644
--- a/libc/test/src/wchar/mbsrtowcs_test.cpp
+++ b/libc/test/src/wchar/mbsrtowcs_test.cpp
@@ -18,6 +18,8 @@
using LlvmLibcMBSRToWCSTest = LIBC_NAMESPACE::testing::ErrnoCheckingTest;
+#if defined(LIBC_TYPES_WCHAR_T_IS_UTF32)
+
TEST_F(LlvmLibcMBSRToWCSTest, OneByteOneChar) {
const char *ch = "A";
const char *original = ch;
@@ -183,3 +185,182 @@ TEST(LlvmLibcMBSRToWCSTest, NullptrCrash) {
WITH_SIGNAL(-1));
}
#endif // LIBC_ADD_NULL_CHECKS
+
+#elif defined(LIBC_TYPES_WCHAR_T_IS_UTF16)
+
+TEST_F(LlvmLibcMBSRToWCSTest, OneByteOneChar) {
+ const char *ch = "A";
+ const char *original = ch;
+ wchar_t dest[2];
+ mbstate_t mb;
+ LIBC_NAMESPACE::memset(&mb, 0, sizeof(mbstate_t));
+ size_t n = LIBC_NAMESPACE::mbsrtowcs(dest, &ch, 1, &mb);
+ ASSERT_EQ(static_cast<char>(*dest), 'A');
+ ASSERT_EQ(static_cast<int>(n), 1);
+ // Should point to null terminator now
+ ASSERT_EQ(ch, original + 1);
+ ASSERT_ERRNO_SUCCESS();
+
+ n = LIBC_NAMESPACE::mbsrtowcs(dest + 1, &ch, 1, &mb);
+ ASSERT_EQ(static_cast<char>(dest[1]), '\0');
+ // Should not include null terminator
+ ASSERT_EQ(static_cast<int>(n), 0);
+ // Should now be a nullptr
+ ASSERT_EQ(ch, nullptr);
+ ASSERT_ERRNO_SUCCESS();
+}
+
+TEST_F(LlvmLibcMBSRToWCSTest, FourByteOneChar) {
+ const char *src = "\xf0\x9f\x98\xb9"; // laughing cat emoji 😹
+ wchar_t dest[3];
+ mbstate_t mb;
+ LIBC_NAMESPACE::memset(&mb, 0, sizeof(mbstate_t));
+ size_t n = LIBC_NAMESPACE::mbsrtowcs(dest, &src, 3, &mb);
+ ASSERT_ERRNO_SUCCESS();
+ ASSERT_EQ(static_cast<int>(dest[0]), 55357);
+ ASSERT_EQ(static_cast<int>(dest[1]), 56889);
+ ASSERT_TRUE(dest[2] == L'\0');
+ // Should not count null terminator in number
+ ASSERT_EQ(static_cast<int>(n), 2);
+ // Should now be a nullptr
+ ASSERT_EQ(src, nullptr);
+}
+
+TEST_F(LlvmLibcMBSRToWCSTest, MultiByteTwoCharacters) {
+ // Two laughing cat emojis "😹😹"
+ const char *src = "\xf0\x9f\x98\xb9\xf0\x9f\x98\xb9";
+ wchar_t dest[5];
+ mbstate_t mb;
+ LIBC_NAMESPACE::memset(&mb, 0, sizeof(mbstate_t));
+ size_t n = LIBC_NAMESPACE::mbsrtowcs(dest, &src, 5, &mb);
+ ASSERT_ERRNO_SUCCESS();
+ ASSERT_EQ(static_cast<int>(dest[0]), 55357);
+ ASSERT_EQ(static_cast<int>(dest[1]), 56889);
+ ASSERT_EQ(static_cast<int>(dest[2]), 55357);
+ ASSERT_EQ(static_cast<int>(dest[3]), 56889);
+ ASSERT_TRUE(dest[4] == L'\0');
+ // Should not count null terminator in number
+ ASSERT_EQ(static_cast<int>(n), 4);
+ // Should now be a nullptr
+ ASSERT_EQ(src, nullptr);
+}
+
+TEST_F(LlvmLibcMBSRToWCSTest, MixedNumberOfBytes) {
+ // 'A', sigma symbol 'Σ', recycling symbol '♻', laughing cat emoji '😹'
+ const char *src = "A\xce\xa3\xe2\x99\xbb\xf0\x9f\x98\xb9";
+ const char *original = src;
+ wchar_t dest[6];
+ size_t n = LIBC_NAMESPACE::mbsrtowcs(dest, &src, 5, nullptr);
+ ASSERT_ERRNO_SUCCESS();
+ ASSERT_EQ(static_cast<char>(dest[0]), 'A');
+ ASSERT_EQ(static_cast<int>(dest[1]), 931);
+ ASSERT_EQ(static_cast<int>(dest[2]), 9851);
+ ASSERT_EQ(static_cast<int>(dest[3]), 55357);
+ ASSERT_EQ(static_cast<int>(dest[4]), 56889);
+ // Should point to null terminator (byte at 10th index)
+ ASSERT_EQ(src, original + 10);
+ ASSERT_EQ(static_cast<int>(n), 5);
+ n = LIBC_NAMESPACE::mbsrtowcs(dest + 5, &src, 4, nullptr);
+ ASSERT_TRUE(dest[5] == L'\0');
+ ASSERT_ERRNO_SUCCESS();
+ // Should not count null terminator in number
+ ASSERT_EQ(static_cast<int>(n), 0);
+ // Should now be a nullptr
+ ASSERT_EQ(src, nullptr);
+}
+
+TEST_F(LlvmLibcMBSRToWCSTest, ReadLessThanStringLength) {
+ // Four laughing cat emojis "😹😹😹😹"
+ const char *src =
+ "\xf0\x9f\x98\xb9\xf0\x9f\x98\xb9\xf0\x9f\x98\xb9\xf0\x9f\x98\xb9";
+ const char *original = src;
+ wchar_t dest[8] = {L'a', L'b', L'c', L'd', L'e', L'f', L'g', L'h'};
+ size_t n = LIBC_NAMESPACE::mbsrtowcs(dest, &src, 6, nullptr);
+ ASSERT_ERRNO_SUCCESS();
+ // Should have read 3 emojis
+ ASSERT_EQ(static_cast<int>(n), 6);
+ ASSERT_EQ(static_cast<int>(dest[0]), 55357);
+ ASSERT_EQ(static_cast<int>(dest[1]), 56889);
+ ASSERT_EQ(static_cast<int>(dest[2]), 55357);
+ ASSERT_EQ(static_cast<int>(dest[3]), 56889);
+ ASSERT_EQ(static_cast<int>(dest[4]), 55357);
+ ASSERT_EQ(static_cast<int>(dest[5]), 56889);
+ ASSERT_TRUE(dest[6] == L'g');
+ ASSERT_TRUE(dest[7] == L'h');
+ // Read three laughing cat emojis, 12 bytes
+ ASSERT_EQ(src, original + 12);
+}
+
+TEST_F(LlvmLibcMBSRToWCSTest, InvalidFirstByte) {
+ // 0x80 is invalid first byte of mb character
+ const char *src =
+ "\x80\x9f\x98\xb9\xf0\x9f\x98\xb9\xf0\x9f\x98\xb9\xf0\x9f\x98\xb9";
+ const char *original = src;
+ wchar_t dest[3];
+ size_t n = LIBC_NAMESPACE::mbsrtowcs(dest, &src, 3, nullptr);
+ // Should return error and set errno
+ ASSERT_EQ(static_cast<int>(n), -1);
+ ASSERT_ERRNO_EQ(EILSEQ);
+ // Should not update pointer
+ ASSERT_EQ(src, original);
+}
+
+TEST_F(LlvmLibcMBSRToWCSTest, InvalidMiddleByte) {
+ // The 7th byte is invalid for a 4 byte character
+ const char *src =
+ "\xf0\x9f\x98\xb9\xf0\x9f\xf0\xb9\xf0\x9f\x98\xb9\xf0\x9f\x98\xb9";
+ const char *original = src;
+ wchar_t dest[3];
+ size_t n = LIBC_NAMESPACE::mbsrtowcs(dest, &src, 5, nullptr);
+ // Should return error, set errno, and not update the pointer
+ ASSERT_EQ(static_cast<int>(n), -1);
+ ASSERT_ERRNO_EQ(EILSEQ);
+ ASSERT_EQ(src, original);
+}
+
+TEST_F(LlvmLibcMBSRToWCSTest, NullDestination) {
+ // Four laughing cat emojis "😹😹😹😹"
+ const char *src =
+ "\xf0\x9f\x98\xb9\xf0\x9f\x98\xb9\xf0\x9f\x98\xb9\xf0\x9f\x98\xb9";
+ const char *original = src;
+ size_t n = LIBC_NAMESPACE::mbsrtowcs(nullptr, &src, 2, nullptr);
+ ASSERT_ERRNO_SUCCESS();
+ // Null destination should ignore len and read till end of string
+ ASSERT_EQ(static_cast<int>(n), 8);
+ // It should also not change the src pointer
+ ASSERT_EQ(src, original);
+}
+
+TEST_F(LlvmLibcMBSRToWCSTest, ErrnoChecks) {
+ // Two laughing cat emojis and invalid 3rd mb char (3rd byte of it)
+ const char *src =
+ "\xf0\x9f\x98\xb9\xf0\x9f\x98\xb9\xf0\x9f\xf0\xb9\xf0\x9f\x98\xb9";
+ const char *original = src;
+ wchar_t dest[7];
+ // First two bytes are valid --> should not set errno
+ size_t n = LIBC_NAMESPACE::mbsrtowcs(dest, &src, 4, nullptr);
+ ASSERT_ERRNO_SUCCESS();
+ ASSERT_EQ(static_cast<int>(n), 4);
+ ASSERT_EQ(static_cast<int>(dest[0]), 55357);
+ ASSERT_EQ(static_cast<int>(dest[1]), 56889);
+ ASSERT_EQ(static_cast<int>(dest[2]), 55357);
+ ASSERT_EQ(static_cast<int>(dest[3]), 56889);
+ ASSERT_EQ(src, original + 8);
+
+ // Trying to read the 3rd byte should set errno
+ n = LIBC_NAMESPACE::mbsrtowcs(dest, &src, 2, nullptr);
+ ASSERT_ERRNO_EQ(EILSEQ);
+ ASSERT_EQ(static_cast<int>(n), -1);
+ // Should not move the pointer
+ ASSERT_EQ(src, original + 8);
+}
+
+#if defined(LIBC_ADD_NULL_CHECKS)
+TEST(LlvmLibcMBSRToWCSTest, NullptrCrash) {
+ // Passing in a nullptr should crash the program.
+ EXPECT_DEATH([] { LIBC_NAMESPACE::mbsrtowcs(nullptr, nullptr, 1, nullptr); },
+ WITH_SIGNAL(-1));
+}
+#endif // LIBC_ADD_NULL_CHECKS
+
+#endif
diff --git a/libc/test/src/wchar/wcrtomb_test.cpp b/libc/test/src/wchar/wcrtomb_test.cpp
index 97aaf5ec7b37f..0af7ea2406175 100644
--- a/libc/test/src/wchar/wcrtomb_test.cpp
+++ b/libc/test/src/wchar/wcrtomb_test.cpp
@@ -11,6 +11,7 @@
#include "hdr/types/wchar_t.h"
#include "src/__support/wchar/mbstate.h"
#include "src/string/memset.h"
+#include "src/wchar/mbsinit.h"
#include "src/wchar/wcrtomb.h"
#include "test/UnitTest/ErrnoCheckingTest.h"
#include "test/UnitTest/Test.h"
@@ -55,6 +56,8 @@ TEST_F(LlvmLibcWCRToMBTest, ThreeByte) {
ASSERT_ERRNO_SUCCESS();
}
+// UTF-16 wcrtomb cannot process this
+#if defined(LIBC_TYPES_WCHAR_T_IS_UTF32)
TEST_F(LlvmLibcWCRToMBTest, FourByte) {
mbstate_t state;
LIBC_NAMESPACE::memset(&state, 0, sizeof(mbstate_t));
@@ -69,6 +72,7 @@ TEST_F(LlvmLibcWCRToMBTest, FourByte) {
ASSERT_EQ(mb[3], static_cast<char>(0xa1));
ASSERT_ERRNO_SUCCESS();
}
+#endif
TEST_F(LlvmLibcWCRToMBTest, NullString) {
mbstate_t state;
@@ -79,8 +83,10 @@ TEST_F(LlvmLibcWCRToMBTest, NullString) {
// should be equivalent to the call wcrtomb(buf, L'\0', state)
size_t cnt1 = LIBC_NAMESPACE::wcrtomb(nullptr, wc, &state);
ASSERT_ERRNO_SUCCESS();
+ ASSERT_TRUE(LIBC_NAMESPACE::mbsinit(&state) != 0);
size_t cnt2 = LIBC_NAMESPACE::wcrtomb(mb, L'\0', &state);
ASSERT_ERRNO_SUCCESS();
+ ASSERT_TRUE(LIBC_NAMESPACE::mbsinit(&state) != 0);
ASSERT_EQ(cnt1, cnt2);
}
@@ -93,6 +99,8 @@ TEST_F(LlvmLibcWCRToMBTest, NullState) {
ASSERT_EQ(cnt, static_cast<size_t>(1));
}
+// UTF-16 wcrtomb cannot process this
+#if defined(LIBC_TYPES_WCHAR_T_IS_UTF32)
TEST_F(LlvmLibcWCRToMBTest, InvalidWchar) {
mbstate_t state;
LIBC_NAMESPACE::memset(&state, 0, sizeof(mbstate_t));
@@ -102,6 +110,7 @@ TEST_F(LlvmLibcWCRToMBTest, InvalidWchar) {
ASSERT_EQ(cnt, static_cast<size_t>(-1));
ASSERT_ERRNO_EQ(EILSEQ);
}
+#endif
TEST_F(LlvmLibcWCRToMBTest, InvalidMBState) {
mbstate_t *state;
@@ -114,3 +123,30 @@ TEST_F(LlvmLibcWCRToMBTest, InvalidMBState) {
ASSERT_EQ(cnt, static_cast<size_t>(-1));
ASSERT_ERRNO_EQ(EINVAL);
}
+
+// Tests unique to UTF-16 wcrtomb
+#if defined(LIBC_TYPES_WCHAR_T_IS_UTF16)
+TEST_F(LlvmLibcWCRToMBTest, InvalidWchar) {
+ mbstate_t state;
+ LIBC_NAMESPACE::memset(&state, 0, sizeof(mbstate_t));
+ wchar_t wc = 0xdc00;
+ char mb[4];
+ size_t cnt = LIBC_NAMESPACE::wcrtomb(mb, wc, &state);
+ ASSERT_EQ(cnt, static_cast<size_t>(-1));
+ ASSERT_ERRNO_EQ(EILSEQ);
+ ASSERT_TRUE(LIBC_NAMESPACE::mbsinit(&state) != 0);
+}
+
+// We fail with EILSEQ on high surrogate, since we have no way to consume a
+// surrogate pair across 2 consecutive calls, unlike with c16rtomb
+TEST_F(LlvmLibcWCRToMBTest, HighSurrogate) {
+ mbstate_t state;
+ LIBC_NAMESPACE::memset(&state, 0, sizeof(mbstate_t));
+ wchar_t wc = 0xd800;
+ char mb[4];
+ size_t cnt = LIBC_NAMESPACE::wcrtomb(mb, wc, &state);
+ ASSERT_EQ(cnt, static_cast<size_t>(-1));
+ ASSERT_ERRNO_EQ(EILSEQ);
+ ASSERT_TRUE(LIBC_NAMESPACE::mbsinit(&state) != 0);
+}
+#endif
diff --git a/libc/test/src/wchar/wcsnrtombs_test.cpp b/libc/test/src/wchar/wcsnrtombs_test.cpp
index 04cf426d31cc7..7e51819c1111a 100644
--- a/libc/test/src/wchar/wcsnrtombs_test.cpp
+++ b/libc/test/src/wchar/wcsnrtombs_test.cpp
@@ -18,6 +18,8 @@ using LlvmLibcWcsnrtombs = LIBC_NAMESPACE::testing::ErrnoCheckingTest;
// these tests are fairly simple as this function just calls into the internal
// wcsnrtombs which is more thoroughly tested
+#if defined(LIBC_TYPES_WCHAR_T_IS_UTF32)
+
TEST_F(LlvmLibcWcsnrtombs, AllMultibyteLengths) {
mbstate_t state;
LIBC_NAMESPACE::memset(&state, 0, sizeof(mbstate_t));
@@ -142,13 +144,12 @@ TEST_F(LlvmLibcWcsnrtombs, ErrnoTest) {
const wchar_t *cur = src;
char mbs[11];
- // n parameter ignored when dest is null
- ASSERT_EQ(LIBC_NAMESPACE::wcsnrtombs(mbs, &cur, 5, 7, &state),
+ ASSERT_EQ(LIBC_NAMESPACE::wcsnrtombs(mbs, &cur, 4, 7, &state),
static_cast<size_t>(7));
ASSERT_ERRNO_SUCCESS();
LIBC_NAMESPACE::memset(&state, 0, sizeof(mbstate_t));
- ASSERT_EQ(LIBC_NAMESPACE::wcsnrtombs(mbs, &cur, 5, 100, &state),
+ ASSERT_EQ(LIBC_NAMESPACE::wcsnrtombs(mbs, &cur, 4, 100, &state),
static_cast<size_t>(-1));
ASSERT_ERRNO_EQ(EILSEQ);
}
@@ -190,3 +191,180 @@ TEST_F(LlvmLibcWcsnrtombs, NullState) {
ASSERT_EQ(mbs[3], '\xA1');
ASSERT_EQ(mbs[4], '\x01');
}
+
+#elif defined(LIBC_TYPES_WCHAR_T_IS_UTF16)
+
+TEST_F(LlvmLibcWcsnrtombs, AllMultibyteLengths) {
+ mbstate_t state;
+ LIBC_NAMESPACE::memset(&state, 0, sizeof(mbstate_t));
+
+ /// clown emoji, sigma symbol, y with diaeresis, letter A
+ const wchar_t src[] = {
+ static_cast<wchar_t>(0xd83e), static_cast<wchar_t>(0xdd21),
+ static_cast<wchar_t>(0x2211), static_cast<wchar_t>(0xff),
+ static_cast<wchar_t>(0x41), static_cast<wchar_t>(0x0)};
+ const wchar_t *cur = src;
+ char mbs[11];
+
+ ASSERT_EQ(LIBC_NAMESPACE::wcsnrtombs(mbs, &cur, 6, 11, &state),
+ static_cast<size_t>(10));
+ ASSERT_ERRNO_SUCCESS();
+ ASSERT_EQ(cur, nullptr);
+ ASSERT_EQ(mbs[0], '\xF0'); // clown begin
+ ASSERT_EQ(mbs[1], '\x9F');
+ ASSERT_EQ(mbs[2], '\xA4');
+ ASSERT_EQ(mbs[3], '\xA1');
+ ASSERT_EQ(mbs[4], '\xE2'); // sigma begin
+ ASSERT_EQ(mbs[5], '\x88');
+ ASSERT_EQ(mbs[6], '\x91');
+ ASSERT_EQ(mbs[7], '\xC3'); // y diaeresis begin
+ ASSERT_EQ(mbs[8], '\xBF');
+ ASSERT_EQ(mbs[9], '\x41'); // A begin
+ ASSERT_EQ(mbs[10], '\0'); // null terminator
+}
+
+TEST_F(LlvmLibcWcsnrtombs, DestLimit) {
+ mbstate_t state;
+ LIBC_NAMESPACE::memset(&state, 0, sizeof(mbstate_t));
+
+ /// clown emoji, sigma symbol, y with diaeresis, letter A
+ const wchar_t src[] = {
+ static_cast<wchar_t>(0xd83e), static_cast<wchar_t>(0xdd21),
+ static_cast<wchar_t>(0x2211), static_cast<wchar_t>(0xff),
+ static_cast<wchar_t>(0x41), static_cast<wchar_t>(0x0)};
+ const wchar_t *cur = src;
+
+ char mbs[11];
+ LIBC_NAMESPACE::memset(mbs, '\x01', 11); // dummy initial values
+
+ ASSERT_EQ(LIBC_NAMESPACE::wcsnrtombs(mbs, &cur, 6, 4, &state),
+ static_cast<size_t>(4));
+ ASSERT_ERRNO_SUCCESS();
+ ASSERT_EQ(cur, src + 2);
+ ASSERT_EQ(mbs[0], '\xF0');
+ ASSERT_EQ(mbs[1], '\x9F');
+ ASSERT_EQ(mbs[2], '\xA4');
+ ASSERT_EQ(mbs[3], '\xA1');
+ ASSERT_EQ(mbs[4], '\x01'); // didn't write more than 4 bytes
+
+ LIBC_NAMESPACE::memset(mbs, '\x01', 11); // dummy initial values
+ LIBC_NAMESPACE::memset(&state, 0, sizeof(mbstate_t));
+ cur = src;
+
+ // not enough bytes to convert the second character, so only converts one
+ ASSERT_EQ(LIBC_NAMESPACE::wcsnrtombs(mbs, &cur, 6, 6, &state),
+ static_cast<size_t>(4));
+ ASSERT_ERRNO_SUCCESS();
+ ASSERT_EQ(cur, src + 2);
+ ASSERT_EQ(mbs[0], '\xF0');
+ ASSERT_EQ(mbs[1], '\x9F');
+ ASSERT_EQ(mbs[2], '\xA4');
+ ASSERT_EQ(mbs[3], '\xA1');
+ ASSERT_EQ(mbs[4], '\x01');
+}
+
+TEST(LlvmLibcWcsnrtombs, SrcLimit) {
+ mbstate_t state;
+ LIBC_NAMESPACE::memset(&state, 0, sizeof(mbstate_t));
+
+ /// clown emoji, sigma symbol, y with diaeresis, letter A
+ const wchar_t src[] = {
+ static_cast<wchar_t>(0xd83e), static_cast<wchar_t>(0xdd21),
+ static_cast<wchar_t>(0x2211), static_cast<wchar_t>(0xff),
+ static_cast<wchar_t>(0x41), static_cast<wchar_t>(0x0)};
+ const wchar_t *cur = src;
+
+ char mbs[11];
+ LIBC_NAMESPACE::memset(mbs, '\x01', 11); // dummy initial values
+
+ auto res = LIBC_NAMESPACE::wcsnrtombs(mbs, &cur, 3, 11, &state);
+ ASSERT_ERRNO_SUCCESS();
+ ASSERT_EQ(res, static_cast<size_t>(7));
+ ASSERT_EQ(cur, src + 3);
+ ASSERT_EQ(mbs[0], '\xF0'); // clown begin
+ ASSERT_EQ(mbs[1], '\x9F');
+ ASSERT_EQ(mbs[2], '\xA4');
+ ASSERT_EQ(mbs[3], '\xA1');
+ ASSERT_EQ(mbs[4], '\xE2'); // sigma begin
+ ASSERT_EQ(mbs[5], '\x88');
+ ASSERT_EQ(mbs[6], '\x91');
+ ASSERT_EQ(mbs[7], '\x01');
+
+ res = LIBC_NAMESPACE::wcsnrtombs(mbs + res, &cur, 100, 11, &state);
+ ASSERT_ERRNO_SUCCESS();
+ ASSERT_EQ(res, static_cast<size_t>(3));
+ ASSERT_EQ(cur, nullptr);
+ ASSERT_EQ(mbs[0], '\xF0'); // clown begin
+ ASSERT_EQ(mbs[1], '\x9F');
+ ASSERT_EQ(mbs[2], '\xA4');
+ ASSERT_EQ(mbs[3], '\xA1');
+ ASSERT_EQ(mbs[4], '\xE2'); // sigma begin
+ ASSERT_EQ(mbs[5], '\x88');
+ ASSERT_EQ(mbs[6], '\x91');
+ ASSERT_EQ(mbs[7], '\xC3'); // y diaeresis begin
+ ASSERT_EQ(mbs[8], '\xBF');
+ ASSERT_EQ(mbs[9], '\x41'); // A begin
+ ASSERT_EQ(mbs[10], '\0'); // null terminator
+}
+
+TEST_F(LlvmLibcWcsnrtombs, ErrnoTest) {
+ mbstate_t state;
+ LIBC_NAMESPACE::memset(&state, 0, sizeof(mbstate_t));
+
+ const wchar_t src[] = {static_cast<wchar_t>(0xd83e),
+ static_cast<wchar_t>(0xdd21),
+ static_cast<wchar_t>(0x2211),
+ static_cast<wchar_t>(0xdc00), // invalid widechar
+ static_cast<wchar_t>(0x0)};
+ const wchar_t *cur = src;
+ char mbs[11];
+
+ ASSERT_EQ(LIBC_NAMESPACE::wcsnrtombs(mbs, &cur, 5, 7, &state),
+ static_cast<size_t>(7));
+ ASSERT_ERRNO_SUCCESS();
+
+ LIBC_NAMESPACE::memset(&state, 0, sizeof(mbstate_t));
+ ASSERT_EQ(LIBC_NAMESPACE::wcsnrtombs(mbs, &cur, 5, 100, &state),
+ static_cast<size_t>(-1));
+ ASSERT_ERRNO_EQ(EILSEQ);
+}
+
+TEST_F(LlvmLibcWcsnrtombs, NullState) {
+ // this test is the same as DestLimit except it uses a nullptr mbstate*
+
+ /// clown emoji, sigma symbol, y with diaeresis, letter A
+ const wchar_t src[] = {
+ static_cast<wchar_t>(0xd83e), static_cast<wchar_t>(0xdd21),
+ static_cast<wchar_t>(0x2211), static_cast<wchar_t>(0xff),
+ static_cast<wchar_t>(0x41), static_cast<wchar_t>(0x0)};
+ const wchar_t *cur = src;
+
+ char mbs[11];
+ LIBC_NAMESPACE::memset(mbs, '\x01', 11); // dummy initial values
+
+ ASSERT_EQ(LIBC_NAMESPACE::wcsnrtombs(mbs, &cur, 6, 4, nullptr),
+ static_cast<size_t>(4));
+ ASSERT_ERRNO_SUCCESS();
+ ASSERT_EQ(cur, src + 2);
+ ASSERT_EQ(mbs[0], '\xF0');
+ ASSERT_EQ(mbs[1], '\x9F');
+ ASSERT_EQ(mbs[2], '\xA4');
+ ASSERT_EQ(mbs[3], '\xA1');
+ ASSERT_EQ(mbs[4], '\x01'); // didn't write more than 4 bytes
+
+ LIBC_NAMESPACE::memset(mbs, '\x01', 11); // dummy initial values
+
+ // not enough bytes to convert the second character, so only converts one
+ cur = src;
+ ASSERT_EQ(LIBC_NAMESPACE::wcsnrtombs(mbs, &cur, 6, 6, nullptr),
+ static_cast<size_t>(4));
+ ASSERT_ERRNO_SUCCESS();
+ ASSERT_EQ(cur, src + 2);
+ ASSERT_EQ(mbs[0], '\xF0');
+ ASSERT_EQ(mbs[1], '\x9F');
+ ASSERT_EQ(mbs[2], '\xA4');
+ ASSERT_EQ(mbs[3], '\xA1');
+ ASSERT_EQ(mbs[4], '\x01');
+}
+
+#endif
diff --git a/libc/test/src/wchar/wcsrtombs_test.cpp b/libc/test/src/wchar/wcsrtombs_test.cpp
index 65c69e63aee0c..b90dd2754c391 100644
--- a/libc/test/src/wchar/wcsrtombs_test.cpp
+++ b/libc/test/src/wchar/wcsrtombs_test.cpp
@@ -18,6 +18,8 @@ using LlvmLibcWcsrtombs = LIBC_NAMESPACE::testing::ErrnoCheckingTest;
// these tests are fairly simple as this function just calls into the internal
// wcsnrtombs which is more thoroughly tested
+#if defined(LIBC_TYPES_WCHAR_T_IS_UTF32)
+
TEST_F(LlvmLibcWcsrtombs, AllMultibyteLengths) {
mbstate_t state;
LIBC_NAMESPACE::memset(&state, 0, sizeof(mbstate_t));
@@ -98,7 +100,6 @@ TEST_F(LlvmLibcWcsrtombs, ErrnoTest) {
const wchar_t *cur = src;
char mbs[11];
- // n parameter ignored when dest is null
ASSERT_EQ(LIBC_NAMESPACE::wcsrtombs(mbs, &cur, 7, &state),
static_cast<size_t>(7));
ASSERT_ERRNO_SUCCESS();
@@ -148,3 +149,138 @@ TEST_F(LlvmLibcWcsrtombs, NullState) {
ASSERT_EQ(mbs[3], '\xA1');
ASSERT_EQ(mbs[4], '\x01');
}
+
+#elif defined(LIBC_TYPES_WCHAR_T_IS_UTF16)
+
+TEST_F(LlvmLibcWcsrtombs, AllMultibyteLengths) {
+ mbstate_t state;
+ LIBC_NAMESPACE::memset(&state, 0, sizeof(mbstate_t));
+
+ /// clown emoji, sigma symbol, y with diaeresis, letter A
+ const wchar_t src[] = {
+ static_cast<wchar_t>(0xd83e), static_cast<wchar_t>(0xdd21),
+ static_cast<wchar_t>(0x2211), static_cast<wchar_t>(0xff),
+ static_cast<wchar_t>(0x41), static_cast<wchar_t>(0x0)};
+ const wchar_t *cur = src;
+ char mbs[11];
+
+ ASSERT_EQ(LIBC_NAMESPACE::wcsrtombs(mbs, &cur, 11, &state),
+ static_cast<size_t>(10));
+ ASSERT_ERRNO_SUCCESS();
+ ASSERT_EQ(cur, nullptr);
+ ASSERT_EQ(mbs[0], '\xF0'); // clown begin
+ ASSERT_EQ(mbs[1], '\x9F');
+ ASSERT_EQ(mbs[2], '\xA4');
+ ASSERT_EQ(mbs[3], '\xA1');
+ ASSERT_EQ(mbs[4], '\xE2'); // sigma begin
+ ASSERT_EQ(mbs[5], '\x88');
+ ASSERT_EQ(mbs[6], '\x91');
+ ASSERT_EQ(mbs[7], '\xC3'); // y diaeresis begin
+ ASSERT_EQ(mbs[8], '\xBF');
+ ASSERT_EQ(mbs[9], '\x41'); // A begin
+ ASSERT_EQ(mbs[10], '\0'); // null terminator
+}
+
+TEST_F(LlvmLibcWcsrtombs, DestLimit) {
+ mbstate_t state;
+ LIBC_NAMESPACE::memset(&state, 0, sizeof(mbstate_t));
+
+ /// clown emoji, sigma symbol, y with diaeresis, letter A
+ const wchar_t src[] = {
+ static_cast<wchar_t>(0xd83e), static_cast<wchar_t>(0xdd21),
+ static_cast<wchar_t>(0x2211), static_cast<wchar_t>(0xff),
+ static_cast<wchar_t>(0x41), static_cast<wchar_t>(0x0)};
+ const wchar_t *cur = src;
+
+ char mbs[11];
+ LIBC_NAMESPACE::memset(mbs, '\x01', 11); // dummy initial values
+
+ ASSERT_EQ(LIBC_NAMESPACE::wcsrtombs(mbs, &cur, 4, &state),
+ static_cast<size_t>(4));
+ ASSERT_ERRNO_SUCCESS();
+ ASSERT_EQ(cur, src + 2);
+ ASSERT_EQ(mbs[0], '\xF0');
+ ASSERT_EQ(mbs[1], '\x9F');
+ ASSERT_EQ(mbs[2], '\xA4');
+ ASSERT_EQ(mbs[3], '\xA1');
+ ASSERT_EQ(mbs[4], '\x01'); // didn't write more than 4 bytes
+
+ LIBC_NAMESPACE::memset(mbs, '\x01', 11); // dummy initial values
+ LIBC_NAMESPACE::memset(&state, 0, sizeof(mbstate_t));
+
+ // not enough bytes to convert the second character, so only converts one
+ cur = src;
+ ASSERT_EQ(LIBC_NAMESPACE::wcsrtombs(mbs, &cur, 6, &state),
+ static_cast<size_t>(4));
+ ASSERT_ERRNO_SUCCESS();
+ ASSERT_EQ(cur, src + 2);
+ ASSERT_EQ(mbs[0], '\xF0');
+ ASSERT_EQ(mbs[1], '\x9F');
+ ASSERT_EQ(mbs[2], '\xA4');
+ ASSERT_EQ(mbs[3], '\xA1');
+ ASSERT_EQ(mbs[4], '\x01');
+}
+
+TEST_F(LlvmLibcWcsrtombs, ErrnoTest) {
+ mbstate_t state;
+ LIBC_NAMESPACE::memset(&state, 0, sizeof(mbstate_t));
+
+ const wchar_t src[] = {static_cast<wchar_t>(0xd83e),
+ static_cast<wchar_t>(0xdd21),
+ static_cast<wchar_t>(0x2211),
+ static_cast<wchar_t>(0xdc00), // invalid widechar
+ static_cast<wchar_t>(0x0)};
+ const wchar_t *cur = src;
+ char mbs[11];
+
+ ASSERT_EQ(LIBC_NAMESPACE::wcsrtombs(mbs, &cur, 7, &state),
+ static_cast<size_t>(7));
+ ASSERT_ERRNO_SUCCESS();
+
+ LIBC_NAMESPACE::memset(&state, 0, sizeof(mbstate_t));
+ cur = src;
+
+ ASSERT_EQ(LIBC_NAMESPACE::wcsrtombs(mbs, &cur, 100, &state),
+ static_cast<size_t>(-1));
+ ASSERT_ERRNO_EQ(EILSEQ);
+}
+
+TEST_F(LlvmLibcWcsrtombs, NullState) {
+ // this test is the same as DestLimit except it uses a nullptr mbstate*
+
+ /// clown emoji, sigma symbol, y with diaeresis, letter A
+ const wchar_t src[] = {
+ static_cast<wchar_t>(0xd83e), static_cast<wchar_t>(0xdd21),
+ static_cast<wchar_t>(0x2211), static_cast<wchar_t>(0xff),
+ static_cast<wchar_t>(0x41), static_cast<wchar_t>(0x0)};
+ const wchar_t *cur = src;
+
+ char mbs[11];
+ LIBC_NAMESPACE::memset(mbs, '\x01', 11); // dummy initial values
+
+ ASSERT_EQ(LIBC_NAMESPACE::wcsrtombs(mbs, &cur, 4, nullptr),
+ static_cast<size_t>(4));
+ ASSERT_ERRNO_SUCCESS();
+ ASSERT_EQ(cur, src + 2);
+ ASSERT_EQ(mbs[0], '\xF0');
+ ASSERT_EQ(mbs[1], '\x9F');
+ ASSERT_EQ(mbs[2], '\xA4');
+ ASSERT_EQ(mbs[3], '\xA1');
+ ASSERT_EQ(mbs[4], '\x01'); // didn't write more than 4 bytes
+
+ LIBC_NAMESPACE::memset(mbs, '\x01', 11); // dummy initial values
+
+ // not enough bytes to convert the second character, so only converts one
+ cur = src;
+ ASSERT_EQ(LIBC_NAMESPACE::wcsrtombs(mbs, &cur, 6, nullptr),
+ static_cast<size_t>(4));
+ ASSERT_ERRNO_SUCCESS();
+ ASSERT_EQ(cur, src + 2);
+ ASSERT_EQ(mbs[0], '\xF0');
+ ASSERT_EQ(mbs[1], '\x9F');
+ ASSERT_EQ(mbs[2], '\xA4');
+ ASSERT_EQ(mbs[3], '\xA1');
+ ASSERT_EQ(mbs[4], '\x01');
+}
+
+#endif
>From 01418f4854839da6c127b49b3af519350b978b1b Mon Sep 17 00:00:00 2001
From: yangbowen <ybw1234567 at hotmail.com>
Date: Thu, 8 Oct 2026 20:31:04 +0000
Subject: [PATCH 3/3] [libc] Add UTF-16 support in printf_core::wchar_writer
---
libc/src/__support/printf_core/CMakeLists.txt | 1 -
.../printf_core/generic/error_mapper.h | 3 +++
.../__support/printf_core/string_converter.h | 25 +++++++++++--------
libc/test/src/stdio/sprintf_test.cpp | 15 +++++++++++
4 files changed, 32 insertions(+), 12 deletions(-)
diff --git a/libc/src/__support/printf_core/CMakeLists.txt b/libc/src/__support/printf_core/CMakeLists.txt
index 03b680010b00f..1a7cc00ad4e55 100644
--- a/libc/src/__support/printf_core/CMakeLists.txt
+++ b/libc/src/__support/printf_core/CMakeLists.txt
@@ -57,7 +57,6 @@ if(LIBC_CONF_PRINTF_DISABLE_WIDE)
else()
set(wchar_deps
libc.hdr.types.wchar_t
- libc.hdr.types.char32_t
libc.hdr.types.char8_t
libc.hdr.types.wint_t
libc.hdr.wchar_macros
diff --git a/libc/src/__support/printf_core/generic/error_mapper.h b/libc/src/__support/printf_core/generic/error_mapper.h
index a3f355b531b33..d63ba8db9a59b 100644
--- a/libc/src/__support/printf_core/generic/error_mapper.h
+++ b/libc/src/__support/printf_core/generic/error_mapper.h
@@ -35,6 +35,9 @@ LIBC_INLINE static int internal_error_to_errno(int internal_error) {
case FIXED_POINT_CONVERSION_ERROR:
case OVERFLOW_ERROR:
return ERANGE;
+ case ILLEGAL_WIDE_CHAR:
+ case MB_CONVERSION_ERROR:
+ return EILSEQ;
default:
LIBC_ASSERT(
false &&
diff --git a/libc/src/__support/printf_core/string_converter.h b/libc/src/__support/printf_core/string_converter.h
index 5352bd8306ba6..5900a6ac5f5c1 100644
--- a/libc/src/__support/printf_core/string_converter.h
+++ b/libc/src/__support/printf_core/string_converter.h
@@ -10,7 +10,6 @@
#define LLVM_LIBC_SRC___SUPPORT_PRINTF_CORE_STRING_CONVERTER_H
#ifndef LIBC_COPT_PRINTF_DISABLE_WIDE
-#include "hdr/types/char32_t.h"
#include "hdr/types/char8_t.h"
#include "src/__support/wchar/mbstate.h"
#include "src/__support/wchar/string_converter.h"
@@ -70,27 +69,29 @@ template <OverflowMode mode>
LIBC_INLINE int wchar_writer(Writer<mode> *writer,
const FormatSection &to_conv) {
size_t string_len = 0;
- const char32_t *wstr_ptr =
- reinterpret_cast<const char32_t *>(to_conv.conv_val_ptr);
+ const wchar_t *wstr_ptr =
+ reinterpret_cast<const wchar_t *>(to_conv.conv_val_ptr);
size_t precision =
to_conv.precision < 0 ? SIZE_MAX : static_cast<size_t>(to_conv.precision);
#ifndef LIBC_COPT_PRINTF_NO_NULLPTR_CHECKS
if (wstr_ptr == nullptr) {
- wstr_ptr = U"(null)";
+ wstr_ptr = L"(null)";
}
#endif // LIBC_COPT_PRINTF_NO_NULLPTR_CHECKS
internal::mbstate mbstate;
- internal::StringConverter<char32_t> length_counter(wstr_ptr, &mbstate,
- precision);
+ internal::StringConverter<wchar_t> length_counter(wstr_ptr, &mbstate,
+ precision);
- for (auto converted = length_counter.pop<char8_t>();
- converted.has_value() && converted.value() != '\0';
+ auto converted = length_counter.pop<char8_t>();
+ for (; converted.has_value() && converted.value() != '\0';
converted = length_counter.pop<char8_t>()) {
++string_len;
}
+ if (!converted.has_value() && converted.error() == EILSEQ)
+ return MB_CONVERSION_ERROR;
size_t padding_spaces = to_conv.min_width > static_cast<int>(string_len)
? to_conv.min_width - string_len
@@ -103,13 +104,15 @@ LIBC_INLINE int wchar_writer(Writer<mode> *writer,
}
mbstate = internal::mbstate();
- internal::StringConverter<char32_t> out_conv(wstr_ptr, &mbstate, precision);
+ internal::StringConverter<wchar_t> out_conv(wstr_ptr, &mbstate, precision);
- for (auto converted = out_conv.pop<char8_t>();
- converted.has_value() && converted.value() != '\0';
+ converted = out_conv.pop<char8_t>();
+ for (; converted.has_value() && converted.value() != '\0';
converted = out_conv.pop<char8_t>()) {
RET_IF_RESULT_NEGATIVE(writer->write(static_cast<char>(converted.value())));
}
+ if (!converted.has_value() && converted.error() == EILSEQ)
+ return MB_CONVERSION_ERROR;
// If the padding is on the right side, write the spaces last.
if (padding_spaces > 0 &&
diff --git a/libc/test/src/stdio/sprintf_test.cpp b/libc/test/src/stdio/sprintf_test.cpp
index f5b68b4d66858..0d1531cdc7f4d 100644
--- a/libc/test/src/stdio/sprintf_test.cpp
+++ b/libc/test/src/stdio/sprintf_test.cpp
@@ -3865,6 +3865,8 @@ TEST(LlvmLibcSprintfTest, WideCharConversion) {
EXPECT_EQ(written, 4);
ASSERT_STREQ_LEN(written, buff, " €");
+ // UTF-16 wide char cannot encode non-BMP character in a single wchar_t
+#if !defined(LIBC_TYPES_WCHAR_T_IS_UTF16)
// Grinning face emoji is a 4-byte UTF-8 character.
written = LIBC_NAMESPACE::sprintf(buff, "%lc", L'😀');
EXPECT_EQ(written, 4);
@@ -3889,15 +3891,22 @@ TEST(LlvmLibcSprintfTest, WideCharConversion) {
written = LIBC_NAMESPACE::sprintf(buff, "%3lc", L'😀');
EXPECT_EQ(written, 4);
ASSERT_STREQ_LEN(written, buff, "😀");
+#endif
// WEOF test.
EXPECT_EQ(LIBC_NAMESPACE::sprintf(buff, "%lc", WEOF), -1);
ASSERT_ERRNO_EQ(EILSEQ);
// Invalid wide character test
+#if defined(LIBC_TYPES_WCHAR_T_IS_UTF32)
EXPECT_EQ(LIBC_NAMESPACE::sprintf(buff, "%lc", static_cast<wint_t>(0x12ffff)),
-1);
ASSERT_ERRNO_EQ(EILSEQ);
+#elif defined(LIBC_TYPES_WCHAR_T_IS_UTF16)
+ EXPECT_EQ(LIBC_NAMESPACE::sprintf(buff, "%lc", static_cast<wint_t>(0xdc00)),
+ -1);
+ ASSERT_ERRNO_EQ(EILSEQ);
+#endif
}
TEST(LlvmLibcSprintfTest, WideCharStringConversion) {
@@ -4001,5 +4010,11 @@ TEST(LlvmLibcSprintfTest, WideCharStringConversion) {
written = LIBC_NAMESPACE::sprintf(buff, "%6.3ls", nullptr);
EXPECT_EQ(written, 6);
ASSERT_STREQ_LEN(written, buff, " (nu");
+
+#if defined(LIBC_TYPES_WCHAR_T_IS_UTF16)
+ // (UTF-16) Lone high surrogate at the end is rejected.
+ EXPECT_EQ(LIBC_NAMESPACE::sprintf(buff, "%ls", L"a\xD83D"), -1);
+ ASSERT_ERRNO_EQ(EILSEQ);
+#endif
}
#endif // !LIBC_COPT_PRINTF_DISABLE_WIDE && !LIBC_TARGET_ARCH_IS_GPU
More information about the libc-commits
mailing list