[libc-commits] [libc] [libc] Add support for wchar_t in StringConverter. (PR #211867)
Alex Strelnikov via libc-commits
libc-commits at lists.llvm.org
Wed Jul 29 08:51:24 PDT 2026
https://github.com/strel-12 updated https://github.com/llvm/llvm-project/pull/211867
>From a96d913492f011927d08af84bf897b3312c1c2c2 Mon Sep 17 00:00:00 2001
From: Alex Strelnikov <strel at google.com>
Date: Fri, 24 Jul 2026 16:28:15 +0000
Subject: [PATCH 1/2] Add support for wchar_t in StringConverter.
---
libc/src/__support/macros/properties/types.h | 13 +
libc/src/__support/wchar/CMakeLists.txt | 3 +-
.../src/__support/wchar/character_converter.h | 63 +++++
libc/test/UnitTest/CMakeLists.txt | 1 +
libc/test/UnitTest/LibcTest.cpp | 20 +-
libc/test/src/__support/wchar/CMakeLists.txt | 76 ++++-
.../__support/wchar/string_converter_test.cpp | 265 +++++++++++-------
.../src/__support/wchar/utf16_to_32_test.cpp | 34 +++
.../src/__support/wchar/utf16_to_8_test.cpp | 34 +++
.../src/__support/wchar/utf32_to_16_test.cpp | 36 +++
.../src/__support/wchar/utf32_to_8_test.cpp | 44 ++-
.../src/__support/wchar/utf8_to_16_test.cpp | 36 +++
.../src/__support/wchar/utf8_to_32_test.cpp | 60 ++--
13 files changed, 532 insertions(+), 153 deletions(-)
create mode 100644 libc/test/src/__support/wchar/utf16_to_32_test.cpp
create mode 100644 libc/test/src/__support/wchar/utf16_to_8_test.cpp
create mode 100644 libc/test/src/__support/wchar/utf32_to_16_test.cpp
create mode 100644 libc/test/src/__support/wchar/utf8_to_16_test.cpp
diff --git a/libc/src/__support/macros/properties/types.h b/libc/src/__support/macros/properties/types.h
index 505fa6d7957f8..8dc574cab6d13 100644
--- a/libc/src/__support/macros/properties/types.h
+++ b/libc/src/__support/macros/properties/types.h
@@ -20,6 +20,19 @@
#include "src/__support/macros/properties/cpu_features.h"
#include "src/__support/macros/properties/os.h"
+// Wide character encoding.
+#if defined(LIBC_COMPILER_IS_CLANG) || defined(LIBC_COMPILER_IS_GCC)
+#if defined(__SIZEOF_WCHAR_T__)
+#if __SIZEOF_WCHAR_T__ == 4
+#define LIBC_TYPES_WCHAR_T_IS_UTF32
+#elif __SIZEOF_WCHAR_T__ == 2
+#define LIBC_TYPES_WCHAR_T_IS_UTF16
+#endif
+#endif // __SIZEOF_WCHAR_T__
+#elif defined(LIBC_COMPILER_IS_MSVC)
+#define LIBC_TYPES_WCHAR_T_IS_UTF16
+#endif // LIBC_COMPILER
+
// 'long double' properties.
#if (LDBL_MANT_DIG == 53)
#define LIBC_TYPES_LONG_DOUBLE_IS_FLOAT64
diff --git a/libc/src/__support/wchar/CMakeLists.txt b/libc/src/__support/wchar/CMakeLists.txt
index ce42fc3bef202..c4e7f0dbaa178 100644
--- a/libc/src/__support/wchar/CMakeLists.txt
+++ b/libc/src/__support/wchar/CMakeLists.txt
@@ -19,7 +19,7 @@ add_header_library(
libc.src.__support.CPP.type_traits
libc.src.__support.error_or
.mbstate
- .character_converter
+ .character_converter
)
add_header_library(
@@ -35,6 +35,7 @@ add_header_library(
libc.src.__support.CPP.type_traits
libc.src.__support.error_or
libc.src.__support.macros.config
+ libc.src.__support.macros.properties.types
libc.src.__support.math_extras
.mbstate
)
diff --git a/libc/src/__support/wchar/character_converter.h b/libc/src/__support/wchar/character_converter.h
index 4da6970f08a4e..f0e3b4891ebc1 100644
--- a/libc/src/__support/wchar/character_converter.h
+++ b/libc/src/__support/wchar/character_converter.h
@@ -18,6 +18,7 @@
#include "src/__support/CPP/type_traits.h"
#include "src/__support/common.h"
#include "src/__support/error_or.h"
+#include "src/__support/macros/properties/types.h"
#include "src/__support/math_extras.h"
#include "src/__support/wchar/mbstate.h"
@@ -59,10 +60,14 @@ class CharacterConverter {
template <typename CharType> size_t sizeAs();
int push(char8_t utf8_byte);
+ int push(char16_t utf16);
int push(char32_t utf32);
+ int push(wchar_t wchar);
ErrorOr<char8_t> pop_utf8();
+ ErrorOr<char16_t> pop_utf16();
ErrorOr<char32_t> pop_utf32();
+ ErrorOr<wchar_t> pop_wchar();
template <typename CharType> ErrorOr<CharType> pop();
};
@@ -120,6 +125,12 @@ LIBC_INLINE int CharacterConverter::push(char8_t utf8_byte) {
return EILSEQ;
}
+LIBC_INLINE int CharacterConverter::push(char16_t utf16) {
+ // TODO: support UTF-16
+ (void)utf16;
+ return -1;
+}
+
LIBC_INLINE int CharacterConverter::push(char32_t utf32) {
// we can't be partially through a conversion when pushing a utf32 value
if (!isEmpty())
@@ -142,6 +153,16 @@ LIBC_INLINE int CharacterConverter::push(char32_t utf32) {
return EILSEQ;
}
+LIBC_INLINE int CharacterConverter::push(wchar_t wchar) {
+#if defined(LIBC_TYPES_WCHAR_T_IS_UTF32)
+ return push(static_cast<char32_t>(wchar));
+#elif defined(LIBC_TYPES_WCHAR_T_IS_UTF16)
+ return push(static_cast<char16_t>(wchar));
+#else
+ return -1;
+#endif
+}
+
LIBC_INLINE ErrorOr<char32_t> CharacterConverter::pop_utf32() {
// If pop is called too early, do not reset the state, use error to determine
// whether enough bytes have been pushed
@@ -153,6 +174,11 @@ LIBC_INLINE ErrorOr<char32_t> CharacterConverter::pop_utf32() {
return utf32;
}
+LIBC_INLINE ErrorOr<char16_t> CharacterConverter::pop_utf16() {
+ // TODO: support UTF-16
+ return Error(-1);
+}
+
LIBC_INLINE ErrorOr<char8_t> CharacterConverter::pop_utf8() {
if (isEmpty())
return Error(-1);
@@ -185,22 +211,59 @@ LIBC_INLINE ErrorOr<char8_t> CharacterConverter::pop_utf8() {
return static_cast<char8_t>(output);
}
+LIBC_INLINE ErrorOr<wchar_t> CharacterConverter::pop_wchar() {
+#if defined(LIBC_TYPES_WCHAR_T_IS_UTF32)
+ ErrorOr<char32_t> Result = pop_utf32();
+ return Result ? ErrorOr<wchar_t>(static_cast<wchar_t>(*Result))
+ : ErrorOr<wchar_t>(Error(Result.error()));
+#elif defined(LIBC_TYPES_WCHAR_T_IS_UTF16)
+ ErrorOr<char16_t> Result = pop_utf16();
+ return Result ? ErrorOr<wchar_t>(static_cast<wchar_t>(*Result))
+ : ErrorOr<wchar_t>(Error(Result.error()));
+#else
+ return Error(-1);
+#endif
+}
+
template <> LIBC_INLINE ErrorOr<char8_t> CharacterConverter::pop() {
return pop_utf8();
}
+template <> LIBC_INLINE ErrorOr<char16_t> CharacterConverter::pop() {
+ return pop_utf16();
+}
+
template <> LIBC_INLINE ErrorOr<char32_t> CharacterConverter::pop() {
return pop_utf32();
}
+template <> LIBC_INLINE ErrorOr<wchar_t> CharacterConverter::pop() {
+ return pop_wchar();
+}
+
template <> LIBC_INLINE size_t CharacterConverter::sizeAs<char8_t>() {
return state->total_bytes;
}
+template <> LIBC_INLINE size_t CharacterConverter::sizeAs<char16_t>() {
+ // TODO: support UTF-16
+ return 0;
+}
+
template <> LIBC_INLINE size_t CharacterConverter::sizeAs<char32_t>() {
return 1;
}
+template <> LIBC_INLINE size_t CharacterConverter::sizeAs<wchar_t>() {
+#if defined(LIBC_TYPES_WCHAR_T_IS_UTF32)
+ return sizeAs<char32_t>();
+#elif defined(LIBC_TYPES_WCHAR_T_IS_UTF16)
+ return sizeAs<char16_t>();
+#else
+ return 0;
+#endif
+}
+
} // namespace internal
} // namespace LIBC_NAMESPACE_DECL
diff --git a/libc/test/UnitTest/CMakeLists.txt b/libc/test/UnitTest/CMakeLists.txt
index 09e7655683fd5..300ff9ace0318 100644
--- a/libc/test/UnitTest/CMakeLists.txt
+++ b/libc/test/UnitTest/CMakeLists.txt
@@ -92,6 +92,7 @@ add_unittest_framework_library(
libc.src.__support.fixed_point.fx_rep
libc.src.__support.macros.properties.compiler
libc.src.__support.macros.properties.types
+ libc.src.__support.wchar.string_converter
libc.src.__support.uint128
${test_logger_osutil}
)
diff --git a/libc/test/UnitTest/LibcTest.cpp b/libc/test/UnitTest/LibcTest.cpp
index b03ceb6c7c77d..0148b41cd57f0 100644
--- a/libc/test/UnitTest/LibcTest.cpp
+++ b/libc/test/UnitTest/LibcTest.cpp
@@ -15,6 +15,7 @@
#include "src/__support/macros/config.h"
#include "src/__support/macros/properties/types.h" // LIBC_TYPES_HAS_INT128
#include "src/__support/uint128.h"
+#include "src/__support/wchar/string_converter.h"
#include "test/UnitTest/TestLogger.h"
#if __STDC_HOSTED__
@@ -71,18 +72,19 @@ cpp::string_view describeValue(const cpp::string &Value) { return Value; }
cpp::string_view describeValue(cpp::string_view Value) { return Value; }
cpp::string describeValue(cpp::wstring_view Value) {
- // TODO: Print `Value` as UTF-8 once `StringConverter` supports `wchar_t`.
- if (Value.empty())
- return "{}";
+ LIBC_NAMESPACE::internal::mbstate State;
+ LIBC_NAMESPACE::internal::StringConverter<wchar_t> StringConv(
+ Value.data(), &State, /* dstlen = */ SIZE_MAX, Value.size());
cpp::string S;
- S += '{';
- for (const wchar_t *Iter = Value.begin(); Iter + 1 != Value.end(); ++Iter) {
- S += cpp::to_string(*Iter);
- S += ',';
+ for (auto Conv = StringConv.pop<char8_t>(); Conv.has_value();
+ Conv = StringConv.pop<char8_t>()) {
+ S += static_cast<char>(*Conv);
}
- S += cpp::to_string(Value.back());
- S += '}';
+
+ if (S.empty() && !Value.empty())
+ S = cpp::string("<Failed Converstion To UTF-8>");
+
return S;
}
diff --git a/libc/test/src/__support/wchar/CMakeLists.txt b/libc/test/src/__support/wchar/CMakeLists.txt
index c112c83dbe9af..c7d0fc85c5b1d 100644
--- a/libc/test/src/__support/wchar/CMakeLists.txt
+++ b/libc/test/src/__support/wchar/CMakeLists.txt
@@ -1,38 +1,96 @@
add_custom_target(libc-support-wchar-tests)
add_libc_test(
- utf8_to_32_test
+ string_converter_test
SUITE
libc-support-tests
SRCS
- utf8_to_32_test.cpp
+ string_converter_test.cpp
DEPENDS
+ libc.src.__support.macros.properties.types
+ libc.src.__support.wchar.string_converter
+ libc.src.__support.wchar.mbstate
+ libc.src.__support.error_or
+ libc.hdr.errno_macros
+ libc.hdr.types.char32_t
+)
+
+add_libc_test(
+ utf8_to_16_test
+ SUITE
+ libc-support-tests
+ SRCS
+ utf8_to_16_test.cpp
+ DEPENDS
+ libc.src.__support.error_or
+ libc.src.__support.macros.properties.types
libc.src.__support.wchar.character_converter
+ libc.src.__support.wchar.mbstate
)
add_libc_test(
- utf32_to_8_test
+ utf8_to_32_test
SUITE
libc-support-tests
SRCS
- utf32_to_8_test.cpp
+ utf8_to_32_test.cpp
DEPENDS
+ libc.src.__support.error_or
+ libc.src.__support.macros.properties.types
libc.src.__support.wchar.character_converter
+ libc.src.__support.wchar.mbstate
)
+add_libc_test(
+ utf16_to_8_test
+ SUITE
+ libc-support-tests
+ SRCS
+ utf16_to_8_test.cpp
+ DEPENDS
+ libc.src.__support.error_or
+ libc.src.__support.macros.properties.types
+ libc.src.__support.wchar.character_converter
+ libc.src.__support.wchar.mbstate
+)
add_libc_test(
- string_converter_test
+ utf16_to_32_test
SUITE
libc-support-tests
SRCS
- string_converter_test.cpp
+ utf16_to_32_test.cpp
DEPENDS
- libc.src.__support.wchar.string_converter
+ libc.src.__support.error_or
+ libc.src.__support.macros.properties.types
+ libc.src.__support.wchar.character_converter
+ libc.src.__support.wchar.mbstate
+)
+
+add_libc_test(
+ utf32_to_8_test
+ SUITE
+ libc-support-tests
+ SRCS
+ utf32_to_8_test.cpp
+ DEPENDS
+ libc.src.__support.error_or
+ libc.src.__support.macros.properties.types
+ libc.src.__support.wchar.character_converter
libc.src.__support.wchar.mbstate
+)
+
+add_libc_test(
+ utf32_to_16_test
+ SUITE
+ libc-support-tests
+ SRCS
+ utf32_to_16_test.cpp
+ DEPENDS
libc.src.__support.error_or
- libc.hdr.errno_macros
- libc.hdr.types.char32_t
+ libc.src.__support.macros.properties.types
+ libc.src.__support.wchar.character_converter
+ libc.src.__support.wchar.mbstate
)
add_libc_test(
diff --git a/libc/test/src/__support/wchar/string_converter_test.cpp b/libc/test/src/__support/wchar/string_converter_test.cpp
index e45358ddc68c4..18293232d1138 100644
--- a/libc/test/src/__support/wchar/string_converter_test.cpp
+++ b/libc/test/src/__support/wchar/string_converter_test.cpp
@@ -10,21 +10,20 @@
#include "hdr/types/char32_t.h"
#include "hdr/types/char8_t.h"
#include "src/__support/error_or.h"
-#include "src/__support/macros/properties/os.h"
+#include "src/__support/macros/properties/types.h"
#include "src/__support/wchar/mbstate.h"
#include "src/__support/wchar/string_converter.h"
#include "test/UnitTest/Test.h"
-// TODO: add support for 16-bit widechars to StringConverter to remove this
-// macro
-#ifdef LIBC_TARGET_OS_IS_WINDOWS
-TEST(LlvmLibcStringConverterTest, Windows) {
- // pass on windows for now
-}
-
+#if defined(LIBC_TYPES_WCHAR_T_IS_UTF32)
+using TestCharTypesUTF32 = LIBC_NAMESPACE::testing::TypeList<char32_t, wchar_t>;
#else
+using TestCharTypesUTF32 = LIBC_NAMESPACE::testing::TypeList<char32_t>;
+#endif
+
+TYPED_TEST(LlvmLibcStringConverterTest, UTF8To32, TestCharTypesUTF32) {
+ using CharType32 = ParamType;
-TEST(LlvmLibcStringConverterTest, UTF8To32) {
// first 4 bytes are clown emoji (🤡)
// next 3 bytes are sigma symbol (∑)
// next 2 bytes are y with diaeresis (ÿ)
@@ -34,199 +33,213 @@ TEST(LlvmLibcStringConverterTest, UTF8To32) {
LIBC_NAMESPACE::internal::StringConverter<char8_t> sc(
reinterpret_cast<const char8_t *>(src), &state, SIZE_MAX);
- auto res = sc.pop<char32_t>();
+ auto res = sc.pop<CharType32>();
ASSERT_TRUE(res.has_value());
ASSERT_EQ(static_cast<int>(res.value()), 0x1f921);
ASSERT_EQ(static_cast<int>(sc.getSourceIndex()), 4);
- res = sc.pop<char32_t>();
+ res = sc.pop<CharType32>();
ASSERT_TRUE(res.has_value());
ASSERT_EQ(static_cast<int>(res.value()), 0x2211);
ASSERT_EQ(static_cast<int>(sc.getSourceIndex()), 7);
- res = sc.pop<char32_t>();
+ res = sc.pop<CharType32>();
ASSERT_TRUE(res.has_value());
ASSERT_EQ(static_cast<int>(res.value()), 0xff);
ASSERT_EQ(static_cast<int>(sc.getSourceIndex()), 9);
- res = sc.pop<char32_t>();
+ res = sc.pop<CharType32>();
ASSERT_TRUE(res.has_value());
ASSERT_EQ(static_cast<int>(res.value()), 0x41);
ASSERT_EQ(static_cast<int>(sc.getSourceIndex()), 10);
- res = sc.pop<char32_t>();
+ res = sc.pop<CharType32>();
ASSERT_TRUE(res.has_value());
ASSERT_EQ(static_cast<int>(res.value()), 0);
ASSERT_EQ(static_cast<int>(sc.getSourceIndex()), 11);
- res = sc.pop<char32_t>();
+ res = sc.pop<CharType32>();
ASSERT_FALSE(res.has_value());
ASSERT_EQ(res.error(), -1);
ASSERT_EQ(static_cast<int>(sc.getSourceIndex()), 11);
}
-TEST(LlvmLibcStringConverterTest, UTF32To8) {
+TYPED_TEST(LlvmLibcStringConverterTest, UTF32To8, TestCharTypesUTF32) {
+ using CharType32 = ParamType;
+
// clown emoji, sigma symbol, y with diaeresis, letter A
- const wchar_t src[] = {static_cast<wchar_t>(0x1f921),
- static_cast<wchar_t>(0x2211),
- static_cast<wchar_t>(0xff), static_cast<wchar_t>(0x41),
- static_cast<wchar_t>(0x0)};
+ const CharType32 src[] = {
+ static_cast<CharType32>(0x1f921), static_cast<CharType32>(0x2211),
+ static_cast<CharType32>(0xff), static_cast<CharType32>(0x41),
+ static_cast<CharType32>(0x0)};
LIBC_NAMESPACE::internal::mbstate state;
- LIBC_NAMESPACE::internal::StringConverter<char32_t> sc(
- reinterpret_cast<const char32_t *>(src), &state, SIZE_MAX);
+ LIBC_NAMESPACE::internal::StringConverter<CharType32> sc(src, &state,
+ SIZE_MAX);
- auto res = sc.pop<char8_t>();
+ auto res = sc.template pop<char8_t>();
ASSERT_TRUE(res.has_value());
ASSERT_EQ(static_cast<int>(res.value()), 0xF0);
ASSERT_EQ(static_cast<int>(sc.getSourceIndex()), 1);
- res = sc.pop<char8_t>();
+ res = sc.template pop<char8_t>();
ASSERT_TRUE(res.has_value());
ASSERT_EQ(static_cast<int>(res.value()), 0x9F);
ASSERT_EQ(static_cast<int>(sc.getSourceIndex()), 1);
- res = sc.pop<char8_t>();
+ res = sc.template pop<char8_t>();
ASSERT_TRUE(res.has_value());
ASSERT_EQ(static_cast<int>(res.value()), 0xA4);
ASSERT_EQ(static_cast<int>(sc.getSourceIndex()), 1);
- res = sc.pop<char8_t>();
+ res = sc.template pop<char8_t>();
ASSERT_TRUE(res.has_value());
ASSERT_EQ(static_cast<int>(res.value()), 0xA1);
ASSERT_EQ(static_cast<int>(sc.getSourceIndex()), 1);
// end of clown emoji, sigma symbol begins
- res = sc.pop<char8_t>();
+ res = sc.template pop<char8_t>();
ASSERT_TRUE(res.has_value());
ASSERT_EQ(static_cast<int>(res.value()), 0xE2);
ASSERT_EQ(static_cast<int>(sc.getSourceIndex()), 2);
- res = sc.pop<char8_t>();
+ res = sc.template pop<char8_t>();
ASSERT_TRUE(res.has_value());
ASSERT_EQ(static_cast<int>(res.value()), 0x88);
ASSERT_EQ(static_cast<int>(sc.getSourceIndex()), 2);
- res = sc.pop<char8_t>();
+ res = sc.template pop<char8_t>();
ASSERT_TRUE(res.has_value());
ASSERT_EQ(static_cast<int>(res.value()), 0x91);
ASSERT_EQ(static_cast<int>(sc.getSourceIndex()), 2);
// end of sigma symbol, y with diaeresis begins
- res = sc.pop<char8_t>();
+ res = sc.template pop<char8_t>();
ASSERT_TRUE(res.has_value());
ASSERT_EQ(static_cast<int>(res.value()), 0xC3);
ASSERT_EQ(static_cast<int>(sc.getSourceIndex()), 3);
- res = sc.pop<char8_t>();
+ res = sc.template pop<char8_t>();
ASSERT_TRUE(res.has_value());
ASSERT_EQ(static_cast<int>(res.value()), 0xBF);
ASSERT_EQ(static_cast<int>(sc.getSourceIndex()), 3);
// end of y with diaeresis, letter A begins
- res = sc.pop<char8_t>();
+ res = sc.template pop<char8_t>();
ASSERT_TRUE(res.has_value());
ASSERT_EQ(static_cast<int>(res.value()), 0x41);
ASSERT_EQ(static_cast<int>(sc.getSourceIndex()), 4);
// null byte
- res = sc.pop<char8_t>();
+ res = sc.template pop<char8_t>();
ASSERT_TRUE(res.has_value());
ASSERT_EQ(static_cast<int>(res.value()), 0);
ASSERT_EQ(static_cast<int>(sc.getSourceIndex()), 5);
- res = sc.pop<char8_t>();
+ res = sc.template pop<char8_t>();
ASSERT_FALSE(res.has_value());
ASSERT_EQ(res.error(), -1);
ASSERT_EQ(static_cast<int>(sc.getSourceIndex()), 5);
}
-TEST(LlvmLibcStringConverterTest, UTF32To8PartialRead) {
- const wchar_t src[] = {
- static_cast<wchar_t>(0x1f921), static_cast<wchar_t>(0x2211),
- static_cast<wchar_t>(0x0)}; // clown emoji, sigma symbol
+TYPED_TEST(LlvmLibcStringConverterTest, UTF32To8PartialRead,
+ TestCharTypesUTF32) {
+ using CharType32 = ParamType;
+
+ const CharType32 src[] = {
+ static_cast<CharType32>(0x1f921), static_cast<CharType32>(0x2211),
+ static_cast<CharType32>(0x0)}; // clown emoji, sigma symbol
LIBC_NAMESPACE::internal::mbstate state;
- LIBC_NAMESPACE::internal::StringConverter<char32_t> sc(
- reinterpret_cast<const char32_t *>(src), &state, SIZE_MAX, 1);
+ LIBC_NAMESPACE::internal::StringConverter<CharType32> sc(src, &state,
+ SIZE_MAX, 1);
- auto res = sc.pop<char8_t>();
+ auto res = sc.template pop<char8_t>();
ASSERT_TRUE(res.has_value());
ASSERT_EQ(static_cast<int>(res.value()), 0xF0);
ASSERT_EQ(static_cast<int>(sc.getSourceIndex()), 1);
- res = sc.pop<char8_t>();
+ res = sc.template pop<char8_t>();
ASSERT_TRUE(res.has_value());
ASSERT_EQ(static_cast<int>(res.value()), 0x9F);
ASSERT_EQ(static_cast<int>(sc.getSourceIndex()), 1);
- res = sc.pop<char8_t>();
+ res = sc.template pop<char8_t>();
ASSERT_TRUE(res.has_value());
ASSERT_EQ(static_cast<int>(res.value()), 0xA4);
ASSERT_EQ(static_cast<int>(sc.getSourceIndex()), 1);
- res = sc.pop<char8_t>();
+ res = sc.template pop<char8_t>();
ASSERT_TRUE(res.has_value());
ASSERT_EQ(static_cast<int>(res.value()), 0xA1);
ASSERT_EQ(static_cast<int>(sc.getSourceIndex()), 1);
// can only read 1 character from source string, so error on next pop
- res = sc.pop<char8_t>();
+ res = sc.template pop<char8_t>();
ASSERT_FALSE(res.has_value());
ASSERT_EQ(res.error(), -1);
}
-TEST(LlvmLibcStringConverterTest, UTF8To32PartialRead) {
+TYPED_TEST(LlvmLibcStringConverterTest, UTF8To32PartialRead,
+ TestCharTypesUTF32) {
+ using CharType32 = ParamType;
+
// first 4 bytes are clown emoji, then next 3 are sigma symbol
const char *src = "\xF0\x9F\xA4\xA1\xE2\x88\x91";
LIBC_NAMESPACE::internal::mbstate state;
LIBC_NAMESPACE::internal::StringConverter<char8_t> sc(
reinterpret_cast<const char8_t *>(src), &state, SIZE_MAX, 5);
- auto res = sc.pop<char32_t>();
+ auto res = sc.pop<CharType32>();
ASSERT_TRUE(res.has_value());
ASSERT_EQ(static_cast<int>(res.value()), 0x1f921);
ASSERT_EQ(static_cast<int>(sc.getSourceIndex()), 4);
- res = sc.pop<char32_t>();
+ res = sc.pop<CharType32>();
ASSERT_FALSE(res.has_value());
ASSERT_EQ(static_cast<int>(res.error()), -1);
ASSERT_EQ(static_cast<int>(sc.getSourceIndex()), 5);
}
-TEST(LlvmLibcStringConverterTest, UTF32To8ErrorHandling) {
- const wchar_t src[] = {
- static_cast<wchar_t>(0x1f921), static_cast<wchar_t>(0xffffff),
- static_cast<wchar_t>(0x0)}; // clown emoji, invalid utf32
+TYPED_TEST(LlvmLibcStringConverterTest, UTF32To8ErrorHandling,
+ TestCharTypesUTF32) {
+ using CharType32 = ParamType;
+
+ const CharType32 src[] = {
+ static_cast<CharType32>(0x1f921), static_cast<CharType32>(0xffffff),
+ static_cast<CharType32>(0x0)}; // clown emoji, invalid utf32
LIBC_NAMESPACE::internal::mbstate state;
- LIBC_NAMESPACE::internal::StringConverter<char32_t> sc(
- reinterpret_cast<const char32_t *>(src), &state, SIZE_MAX);
+ LIBC_NAMESPACE::internal::StringConverter<CharType32> sc(src, &state,
+ SIZE_MAX);
- auto res = sc.pop<char8_t>();
+ auto res = sc.template pop<char8_t>();
ASSERT_TRUE(res.has_value());
ASSERT_EQ(static_cast<int>(res.value()), 0xF0);
ASSERT_EQ(static_cast<int>(sc.getSourceIndex()), 1);
- res = sc.pop<char8_t>();
+ res = sc.template pop<char8_t>();
ASSERT_TRUE(res.has_value());
ASSERT_EQ(static_cast<int>(res.value()), 0x9F);
ASSERT_EQ(static_cast<int>(sc.getSourceIndex()), 1);
- res = sc.pop<char8_t>();
+ res = sc.template pop<char8_t>();
ASSERT_TRUE(res.has_value());
ASSERT_EQ(static_cast<int>(res.value()), 0xA4);
ASSERT_EQ(static_cast<int>(sc.getSourceIndex()), 1);
- res = sc.pop<char8_t>();
+ res = sc.template pop<char8_t>();
ASSERT_TRUE(res.has_value());
ASSERT_EQ(static_cast<int>(res.value()), 0xA1);
ASSERT_EQ(static_cast<int>(sc.getSourceIndex()), 1);
- res = sc.pop<char8_t>();
+ res = sc.template pop<char8_t>();
ASSERT_FALSE(res.has_value());
ASSERT_EQ(static_cast<int>(res.error()), EILSEQ);
ASSERT_EQ(static_cast<int>(sc.getSourceIndex()), 1);
}
-TEST(LlvmLibcStringConverterTest, UTF8To32ErrorHandling) {
+TYPED_TEST(LlvmLibcStringConverterTest, UTF8To32ErrorHandling,
+ TestCharTypesUTF32) {
+ using CharType32 = ParamType;
+
// first 4 bytes are clown emoji (🤡)
// next 3 form an invalid character
const char *src = "\xF0\x9F\xA4\xA1\x90\x88\x30";
@@ -234,18 +247,21 @@ TEST(LlvmLibcStringConverterTest, UTF8To32ErrorHandling) {
LIBC_NAMESPACE::internal::StringConverter<char8_t> sc(
reinterpret_cast<const char8_t *>(src), &state, SIZE_MAX);
- auto res = sc.pop<char32_t>();
+ auto res = sc.pop<CharType32>();
ASSERT_TRUE(res.has_value());
ASSERT_EQ(static_cast<int>(res.value()), 0x1f921);
ASSERT_EQ(static_cast<int>(sc.getSourceIndex()), 4);
- res = sc.pop<char32_t>();
+ res = sc.pop<CharType32>();
ASSERT_FALSE(res.has_value());
ASSERT_EQ(static_cast<int>(res.error()), EILSEQ);
ASSERT_EQ(static_cast<int>(sc.getSourceIndex()), 4);
}
-TEST(LlvmLibcStringConverterTest, InvalidCharacterOutsideBounds) {
+TYPED_TEST(LlvmLibcStringConverterTest, InvalidCharacterOutsideBounds,
+ TestCharTypesUTF32) {
+ using CharType32 = ParamType;
+
// if an invalid character exists in the source string but we don't have space
// to write it, we should return a "stop converting" error rather than an
// invalid character error
@@ -257,107 +273,111 @@ TEST(LlvmLibcStringConverterTest, InvalidCharacterOutsideBounds) {
LIBC_NAMESPACE::internal::StringConverter<char8_t> sc1(
reinterpret_cast<const char8_t *>(src1), &ps1, 1);
- auto res1 = sc1.pop<char32_t>();
+ auto res1 = sc1.pop<CharType32>();
ASSERT_TRUE(res1.has_value());
ASSERT_EQ(static_cast<int>(res1.value()), 0x1f921);
ASSERT_EQ(static_cast<int>(sc1.getSourceIndex()), 4);
- res1 = sc1.pop<char32_t>();
+ res1 = sc1.pop<CharType32>();
ASSERT_FALSE(res1.has_value());
// no space to write error NOT invalid character error (EILSEQ)
ASSERT_EQ(static_cast<int>(res1.error()), -1);
ASSERT_EQ(static_cast<int>(sc1.getSourceIndex()), 4);
- const wchar_t src2[] = {
- static_cast<wchar_t>(0x1f921), static_cast<wchar_t>(0xffffff),
- static_cast<wchar_t>(0x0)}; // clown emoji, invalid utf32
+ const CharType32 src2[] = {
+ static_cast<CharType32>(0x1f921), static_cast<CharType32>(0xffffff),
+ static_cast<CharType32>(0x0)}; // clown emoji, invalid utf32
LIBC_NAMESPACE::internal::mbstate ps2;
- LIBC_NAMESPACE::internal::StringConverter<char32_t> sc2(
- reinterpret_cast<const char32_t *>(src2), &ps2, 4);
+ LIBC_NAMESPACE::internal::StringConverter<CharType32> sc2(src2, &ps2, 4);
- auto res2 = sc2.pop<char8_t>();
+ auto res2 = sc2.template pop<char8_t>();
ASSERT_TRUE(res2.has_value());
ASSERT_EQ(static_cast<int>(res2.value()), 0xF0);
ASSERT_EQ(static_cast<int>(sc2.getSourceIndex()), 1);
- res2 = sc2.pop<char8_t>();
+ res2 = sc2.template pop<char8_t>();
ASSERT_TRUE(res2.has_value());
ASSERT_EQ(static_cast<int>(res2.value()), 0x9F);
ASSERT_EQ(static_cast<int>(sc2.getSourceIndex()), 1);
- res2 = sc2.pop<char8_t>();
+ res2 = sc2.template pop<char8_t>();
ASSERT_TRUE(res2.has_value());
ASSERT_EQ(static_cast<int>(res2.value()), 0xA4);
ASSERT_EQ(static_cast<int>(sc2.getSourceIndex()), 1);
- res2 = sc2.pop<char8_t>();
+ res2 = sc2.template pop<char8_t>();
ASSERT_TRUE(res2.has_value());
ASSERT_EQ(static_cast<int>(res2.value()), 0xA1);
ASSERT_EQ(static_cast<int>(sc2.getSourceIndex()), 1);
- res2 = sc2.pop<char8_t>();
+ res2 = sc2.template pop<char8_t>();
ASSERT_FALSE(res2.has_value());
// no space to write error NOT invalid character error (EILSEQ)
ASSERT_EQ(static_cast<int>(res2.error()), -1);
ASSERT_EQ(static_cast<int>(sc2.getSourceIndex()), 1);
}
-TEST(LlvmLibcStringConverterTest, MultipleStringConverters32To8) {
+TYPED_TEST(LlvmLibcStringConverterTest, MultipleStringConverters32To8,
+ TestCharTypesUTF32) {
+ using CharType32 = ParamType;
+
/*
We do NOT test partially popping a character and expecting the next
StringConverter to continue where we left off. This is not expected to work
and considered invalid.
*/
- const wchar_t src[] = {
- static_cast<wchar_t>(0x1f921), static_cast<wchar_t>(0xff),
- static_cast<wchar_t>(0x0)}; // clown emoji, y with diaeresis (ÿ)
+ const CharType32 src[] = {
+ static_cast<CharType32>(0x1f921), static_cast<CharType32>(0xff),
+ static_cast<CharType32>(0x0)}; // clown emoji, y with diaeresis (ÿ)
LIBC_NAMESPACE::internal::mbstate state;
- LIBC_NAMESPACE::internal::StringConverter<char32_t> sc1(
- reinterpret_cast<const char32_t *>(src), &state, SIZE_MAX, 1);
+ LIBC_NAMESPACE::internal::StringConverter<CharType32> sc1(src, &state,
+ SIZE_MAX, 1);
- auto res = sc1.pop<char8_t>();
+ auto res = sc1.template pop<char8_t>();
ASSERT_TRUE(res.has_value());
ASSERT_EQ(static_cast<int>(res.value()), 0xF0);
ASSERT_EQ(static_cast<int>(sc1.getSourceIndex()), 1);
- res = sc1.pop<char8_t>();
+ res = sc1.template pop<char8_t>();
ASSERT_TRUE(res.has_value());
ASSERT_EQ(static_cast<int>(res.value()), 0x9F);
ASSERT_EQ(static_cast<int>(sc1.getSourceIndex()), 1);
- res = sc1.pop<char8_t>();
+ res = sc1.template pop<char8_t>();
ASSERT_TRUE(res.has_value());
ASSERT_EQ(static_cast<int>(res.value()), 0xA4);
ASSERT_EQ(static_cast<int>(sc1.getSourceIndex()), 1);
- res = sc1.pop<char8_t>();
+ res = sc1.template pop<char8_t>();
ASSERT_TRUE(res.has_value());
ASSERT_EQ(static_cast<int>(res.value()), 0xA1);
ASSERT_EQ(static_cast<int>(sc1.getSourceIndex()), 1);
// sc2 should pick up where sc1 left off and continue the conversion
- LIBC_NAMESPACE::internal::StringConverter<char32_t> sc2(
- reinterpret_cast<const char32_t *>(src) + sc1.getSourceIndex(), &state,
- SIZE_MAX, 1);
+ LIBC_NAMESPACE::internal::StringConverter<CharType32> sc2(
+ src + sc1.getSourceIndex(), &state, SIZE_MAX, 1);
- res = sc2.pop<char8_t>();
+ res = sc2.template pop<char8_t>();
ASSERT_TRUE(res.has_value());
ASSERT_EQ(static_cast<int>(res.value()), 0xC3);
ASSERT_EQ(static_cast<int>(sc2.getSourceIndex()), 1);
- res = sc2.pop<char8_t>();
+ res = sc2.template pop<char8_t>();
ASSERT_TRUE(res.has_value());
ASSERT_EQ(static_cast<int>(res.value()), 0xBF);
ASSERT_EQ(static_cast<int>(sc2.getSourceIndex()), 1);
}
-TEST(LlvmLibcStringConverterTest, MultipleStringConverters8To32) {
+TYPED_TEST(LlvmLibcStringConverterTest, MultipleStringConverters8To32,
+ TestCharTypesUTF32) {
+ using CharType32 = ParamType;
+
const char *src = "\xF0\x9F\xA4\xA1"; // clown emoji
LIBC_NAMESPACE::internal::mbstate state;
LIBC_NAMESPACE::internal::StringConverter<char8_t> sc1(
reinterpret_cast<const char8_t *>(src), &state, SIZE_MAX, 2);
- auto res = sc1.pop<char32_t>();
+ auto res = sc1.pop<CharType32>();
ASSERT_FALSE(res.has_value());
ASSERT_EQ(static_cast<int>(res.error()), -1);
ASSERT_EQ(static_cast<int>(sc1.getSourceIndex()), 2);
@@ -367,57 +387,88 @@ TEST(LlvmLibcStringConverterTest, MultipleStringConverters8To32) {
reinterpret_cast<const char8_t *>(src) + sc1.getSourceIndex(), &state,
SIZE_MAX, 3);
- res = sc2.pop<char32_t>();
+ res = sc2.pop<CharType32>();
ASSERT_TRUE(res.has_value());
ASSERT_EQ(static_cast<int>(res.value()), 0x1f921);
ASSERT_EQ(static_cast<int>(sc2.getSourceIndex()), 2);
- res = sc2.pop<char32_t>();
+ res = sc2.pop<CharType32>();
ASSERT_TRUE(res.has_value());
ASSERT_EQ(static_cast<int>(res.value()), 0);
ASSERT_EQ(static_cast<int>(sc2.getSourceIndex()), 3);
}
-TEST(LlvmLibcStringConverterTest, DestLimitUTF8To32) {
+TYPED_TEST(LlvmLibcStringConverterTest, DestLimitUTF8To32, TestCharTypesUTF32) {
+ using CharType32 = ParamType;
+
const char *src = "\xF0\x9F\xA4\xA1\xF0\x9F\xA4\xA1"; // 2 clown emojis
LIBC_NAMESPACE::internal::mbstate state;
LIBC_NAMESPACE::internal::StringConverter<char8_t> sc(
reinterpret_cast<const char8_t *>(src), &state, 1);
- auto res = sc.pop<char32_t>();
+ auto res = sc.pop<CharType32>();
ASSERT_TRUE(res.has_value());
ASSERT_EQ(static_cast<int>(sc.getSourceIndex()), 4);
- res = sc.pop<char32_t>(); // no space to pop this into
+ res = sc.pop<CharType32>(); // no space to pop this into
ASSERT_FALSE(res.has_value());
}
-TEST(LlvmLibcStringConverterTest, DestLimitUTF32To8) {
- const wchar_t src[] = {static_cast<wchar_t>(0x1f921),
- static_cast<wchar_t>(0x1f921)}; // 2 clown emojis
+TYPED_TEST(LlvmLibcStringConverterTest, DestLimitUTF32To8, TestCharTypesUTF32) {
+ using CharType32 = ParamType;
+
+ const CharType32 src[] = {static_cast<CharType32>(0x1f921),
+ static_cast<CharType32>(0x1f921)}; // 2 clown emojis
LIBC_NAMESPACE::internal::mbstate state;
- LIBC_NAMESPACE::internal::StringConverter<char32_t> sc(
- reinterpret_cast<const char32_t *>(src), &state, 5);
+ LIBC_NAMESPACE::internal::StringConverter<CharType32> sc(src, &state, 5);
- auto res = sc.pop<char8_t>();
+ auto res = sc.template pop<char8_t>();
ASSERT_TRUE(res.has_value());
ASSERT_EQ(static_cast<int>(sc.getSourceIndex()), 1);
- res = sc.pop<char8_t>();
+ res = sc.template pop<char8_t>();
ASSERT_TRUE(res.has_value());
ASSERT_EQ(static_cast<int>(sc.getSourceIndex()), 1);
- res = sc.pop<char8_t>();
+ res = sc.template pop<char8_t>();
ASSERT_TRUE(res.has_value());
ASSERT_EQ(static_cast<int>(sc.getSourceIndex()), 1);
- res = sc.pop<char8_t>();
+ res = sc.template pop<char8_t>();
ASSERT_TRUE(res.has_value());
ASSERT_EQ(static_cast<int>(sc.getSourceIndex()), 1);
- res = sc.pop<char8_t>();
+ res = sc.template pop<char8_t>();
ASSERT_FALSE(res.has_value());
ASSERT_EQ(static_cast<int>(sc.getSourceIndex()), 1);
}
+#if defined(LIBC_TYPES_WCHAR_T_IS_UTF16)
+using TestCharTypesUTF16 = LIBC_NAMESPACE::testing::TypeList<char16_t, wchar_t>;
+#else
+using TestCharTypesUTF16 = LIBC_NAMESPACE::testing::TypeList<char16_t>;
#endif
+
+TYPED_TEST(LlvmLibcStringConverterTest, UTF16Fails, TestCharTypesUTF16) {
+ using CharType16 = ParamType;
+
+ const char32_t Src32[] = {static_cast<char32_t>('A'),
+ static_cast<char32_t>('B')};
+ LIBC_NAMESPACE::internal::mbstate State32;
+ LIBC_NAMESPACE::internal::StringConverter<char32_t> StringConv32(
+ Src32, &State32, /* dstlen = */ 2,
+ /* srclen = */ 2);
+
+ ASSERT_FALSE(StringConv32.pop<CharType16>().has_value());
+ ASSERT_EQ(StringConv32.getSourceIndex(), size_t{1});
+
+ const CharType16 Src16[] = {static_cast<CharType16>('A'),
+ static_cast<CharType16>('B')};
+ LIBC_NAMESPACE::internal::mbstate State16;
+ LIBC_NAMESPACE::internal::StringConverter<CharType16> StringConv16(
+ Src16, &State16, /* dstlen = */ 2,
+ /* srclen = */ 2);
+
+ ASSERT_FALSE(StringConv16.template pop<char32_t>().has_value());
+ ASSERT_EQ(StringConv16.getSourceIndex(), size_t{0});
+}
diff --git a/libc/test/src/__support/wchar/utf16_to_32_test.cpp b/libc/test/src/__support/wchar/utf16_to_32_test.cpp
new file mode 100644
index 0000000000000..8482a638f1872
--- /dev/null
+++ b/libc/test/src/__support/wchar/utf16_to_32_test.cpp
@@ -0,0 +1,34 @@
+//===----------------------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+///
+/// \file
+/// This file contains CharacterConverter unit tests for UTF-16 to UTF-32.
+///
+//===----------------------------------------------------------------------===//
+
+#include "src/__support/error_or.h"
+#include "src/__support/macros/properties/types.h"
+#include "src/__support/wchar/character_converter.h"
+#include "src/__support/wchar/mbstate.h"
+#include "test/UnitTest/Test.h"
+
+#if defined(LIBC_TYPES_WCHAR_T_IS_UTF16)
+using TestCharTypesUTF16 = LIBC_NAMESPACE::testing::TypeList<char16_t, wchar_t>;
+#else
+using TestCharTypesUTF16 = LIBC_NAMESPACE::testing::TypeList<char16_t>;
+#endif
+
+TYPED_TEST(LlvmLibcCharacterConverterUTF16To32Test, PushFails,
+ TestCharTypesUTF16) {
+ using CharType16 = ParamType;
+
+ LIBC_NAMESPACE::internal::mbstate State;
+ LIBC_NAMESPACE::internal::CharacterConverter CharConv(&State);
+ CharConv.clear();
+ ASSERT_EQ(CharConv.push(static_cast<CharType16>('A')), -1);
+}
diff --git a/libc/test/src/__support/wchar/utf16_to_8_test.cpp b/libc/test/src/__support/wchar/utf16_to_8_test.cpp
new file mode 100644
index 0000000000000..a80a0c4c8bcff
--- /dev/null
+++ b/libc/test/src/__support/wchar/utf16_to_8_test.cpp
@@ -0,0 +1,34 @@
+//===----------------------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+///
+/// \file
+/// This file contains CharacterConverter unit tests for UTF-16 to UTF-8.
+///
+//===----------------------------------------------------------------------===//
+
+#include "src/__support/error_or.h"
+#include "src/__support/macros/properties/types.h"
+#include "src/__support/wchar/character_converter.h"
+#include "src/__support/wchar/mbstate.h"
+#include "test/UnitTest/Test.h"
+
+#if defined(LIBC_TYPES_WCHAR_T_IS_UTF16)
+using TestCharTypesUTF16 = LIBC_NAMESPACE::testing::TypeList<char16_t, wchar_t>;
+#else
+using TestCharTypesUTF16 = LIBC_NAMESPACE::testing::TypeList<char16_t>;
+#endif
+
+TYPED_TEST(LlvmLibcCharacterConverterUTF16To8Test, PushFails,
+ TestCharTypesUTF16) {
+ using CharType16 = ParamType;
+
+ LIBC_NAMESPACE::internal::mbstate State;
+ LIBC_NAMESPACE::internal::CharacterConverter CharConv(&State);
+ CharConv.clear();
+ ASSERT_EQ(CharConv.push(static_cast<CharType16>('A')), -1);
+}
diff --git a/libc/test/src/__support/wchar/utf32_to_16_test.cpp b/libc/test/src/__support/wchar/utf32_to_16_test.cpp
new file mode 100644
index 0000000000000..c5287ef9a884b
--- /dev/null
+++ b/libc/test/src/__support/wchar/utf32_to_16_test.cpp
@@ -0,0 +1,36 @@
+//===----------------------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+///
+/// \file
+/// This file contains CharacterConverter unit tests for UTF-32 to UTF-16.
+///
+//===----------------------------------------------------------------------===//
+
+#include "src/__support/error_or.h"
+#include "src/__support/macros/properties/types.h"
+#include "src/__support/wchar/character_converter.h"
+#include "src/__support/wchar/mbstate.h"
+#include "test/UnitTest/Test.h"
+
+#if defined(LIBC_TYPES_WCHAR_T_IS_UTF16)
+using TestCharTypesUTF16 = LIBC_NAMESPACE::testing::TypeList<char16_t, wchar_t>;
+#else
+using TestCharTypesUTF16 = LIBC_NAMESPACE::testing::TypeList<char16_t>;
+#endif
+
+TYPED_TEST(LlvmLibcCharacterConverterUTF32To16Test, PopFails,
+ TestCharTypesUTF16) {
+ using CharType16 = ParamType;
+
+ LIBC_NAMESPACE::internal::mbstate State;
+ LIBC_NAMESPACE::internal::CharacterConverter CharConv(&State);
+ CharConv.clear();
+ ASSERT_EQ(CharConv.push(static_cast<char32_t>('A')), 0);
+
+ ASSERT_FALSE(CharConv.pop<CharType16>().has_value());
+}
diff --git a/libc/test/src/__support/wchar/utf32_to_8_test.cpp b/libc/test/src/__support/wchar/utf32_to_8_test.cpp
index 1ad523e148845..52e10e1191175 100644
--- a/libc/test/src/__support/wchar/utf32_to_8_test.cpp
+++ b/libc/test/src/__support/wchar/utf32_to_8_test.cpp
@@ -7,18 +7,28 @@
//===----------------------------------------------------------------------===//
#include "src/__support/common.h"
+#include "src/__support/macros/properties/types.h"
#include "src/__support/wchar/character_converter.h"
#include "src/__support/wchar/mbstate.h"
#include "test/UnitTest/Test.h"
-TEST(LlvmLibcCharacterConverterUTF32To8Test, OneByte) {
+#if defined(LIBC_TYPES_WCHAR_T_IS_UTF32)
+using TestCharTypesUTF32 = LIBC_NAMESPACE::testing::TypeList<char32_t, wchar_t>;
+#else
+using TestCharTypesUTF32 = LIBC_NAMESPACE::testing::TypeList<char32_t>;
+#endif
+
+TYPED_TEST(LlvmLibcCharacterConverterUTF32To8Test, OneByte,
+ TestCharTypesUTF32) {
+ using CharType32 = ParamType;
+
LIBC_NAMESPACE::internal::mbstate state;
LIBC_NAMESPACE::internal::CharacterConverter cr(&state);
cr.clear();
// utf8 1-byte encodings are identical to their utf32 representations
- char32_t utf32_A = 0x41; // 'A'
+ CharType32 utf32_A = 0x41; // 'A'
cr.push(utf32_A);
ASSERT_TRUE(cr.isFull());
auto popped = cr.pop_utf8();
@@ -26,7 +36,7 @@ TEST(LlvmLibcCharacterConverterUTF32To8Test, OneByte) {
ASSERT_EQ(static_cast<char>(popped.value()), 'A');
ASSERT_TRUE(cr.isEmpty());
- char32_t utf32_B = 0x42; // 'B'
+ CharType32 utf32_B = 0x42; // 'B'
cr.push(utf32_B);
ASSERT_TRUE(cr.isFull());
popped = cr.pop_utf8();
@@ -39,13 +49,16 @@ TEST(LlvmLibcCharacterConverterUTF32To8Test, OneByte) {
ASSERT_FALSE(popped.has_value());
}
-TEST(LlvmLibcCharacterConverterUTF32To8Test, TwoByte) {
+TYPED_TEST(LlvmLibcCharacterConverterUTF32To8Test, TwoByte,
+ TestCharTypesUTF32) {
+ using CharType32 = ParamType;
+
LIBC_NAMESPACE::internal::mbstate state;
LIBC_NAMESPACE::internal::CharacterConverter cr(&state);
cr.clear();
// testing utf32: 0xff -> utf8: 0xc3 0xbf
- char32_t utf32 = 0xff;
+ CharType32 utf32 = 0xff;
cr.push(utf32);
ASSERT_TRUE(cr.isFull());
auto popped = cr.pop_utf8();
@@ -75,13 +88,16 @@ TEST(LlvmLibcCharacterConverterUTF32To8Test, TwoByte) {
ASSERT_FALSE(popped.has_value());
}
-TEST(LlvmLibcCharacterConverterUTF32To8Test, ThreeByte) {
+TYPED_TEST(LlvmLibcCharacterConverterUTF32To8Test, ThreeByte,
+ TestCharTypesUTF32) {
+ using CharType32 = ParamType;
+
LIBC_NAMESPACE::internal::mbstate state;
LIBC_NAMESPACE::internal::CharacterConverter cr(&state);
cr.clear();
// testing utf32: 0xac15 -> utf8: 0xea 0xb0 0x95
- char32_t utf32 = 0xac15;
+ CharType32 utf32 = 0xac15;
cr.push(utf32);
ASSERT_TRUE(cr.isFull());
auto popped = cr.pop_utf8();
@@ -119,13 +135,16 @@ TEST(LlvmLibcCharacterConverterUTF32To8Test, ThreeByte) {
ASSERT_FALSE(popped.has_value());
}
-TEST(LlvmLibcCharacterConverterUTF32To8Test, FourByte) {
+TYPED_TEST(LlvmLibcCharacterConverterUTF32To8Test, FourByte,
+ TestCharTypesUTF32) {
+ using CharType32 = ParamType;
+
LIBC_NAMESPACE::internal::mbstate state;
LIBC_NAMESPACE::internal::CharacterConverter cr(&state);
cr.clear();
// testing utf32: 0x1f921 -> utf8: 0xf0 0x9f 0xa4 0xa1
- char32_t utf32 = 0x1f921;
+ CharType32 utf32 = 0x1f921;
cr.push(utf32);
ASSERT_TRUE(cr.isFull());
auto popped = cr.pop_utf8();
@@ -171,13 +190,16 @@ TEST(LlvmLibcCharacterConverterUTF32To8Test, FourByte) {
ASSERT_FALSE(popped.has_value());
}
-TEST(LlvmLibcCharacterConverterUTF32To8Test, CantPushMidConversion) {
+TYPED_TEST(LlvmLibcCharacterConverterUTF32To8Test, CantPushMidConversion,
+ TestCharTypesUTF32) {
+ using CharType32 = ParamType;
+
LIBC_NAMESPACE::internal::mbstate state;
LIBC_NAMESPACE::internal::CharacterConverter cr(&state);
cr.clear();
// testing utf32: 0x12121 -> utf8: 0xf0 0x92 0x84 0xa1
- char32_t utf32 = 0x12121;
+ CharType32 utf32 = 0x12121;
ASSERT_EQ(cr.push(utf32), 0);
auto popped = cr.pop_utf8();
ASSERT_TRUE(popped.has_value());
diff --git a/libc/test/src/__support/wchar/utf8_to_16_test.cpp b/libc/test/src/__support/wchar/utf8_to_16_test.cpp
new file mode 100644
index 0000000000000..ec3bc17165314
--- /dev/null
+++ b/libc/test/src/__support/wchar/utf8_to_16_test.cpp
@@ -0,0 +1,36 @@
+//===----------------------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+///
+/// \file
+/// This file contains CharacterConverter unit tests for UTF-8 to UTF-16.
+///
+//===----------------------------------------------------------------------===//
+
+#include "src/__support/error_or.h"
+#include "src/__support/macros/properties/types.h"
+#include "src/__support/wchar/character_converter.h"
+#include "src/__support/wchar/mbstate.h"
+#include "test/UnitTest/Test.h"
+
+#if defined(LIBC_TYPES_WCHAR_T_IS_UTF16)
+using TestCharTypesUTF16 = LIBC_NAMESPACE::testing::TypeList<char16_t, wchar_t>;
+#else
+using TestCharTypesUTF16 = LIBC_NAMESPACE::testing::TypeList<char16_t>;
+#endif
+
+TYPED_TEST(LlvmLibcCharacterConverterUTF8To16Test, PopFails,
+ TestCharTypesUTF16) {
+ using CharType16 = ParamType;
+
+ LIBC_NAMESPACE::internal::mbstate State;
+ LIBC_NAMESPACE::internal::CharacterConverter CharConv(&State);
+ CharConv.clear();
+ ASSERT_EQ(CharConv.push(static_cast<char8_t>('A')), 0);
+
+ ASSERT_FALSE(CharConv.pop<CharType16>().has_value());
+}
diff --git a/libc/test/src/__support/wchar/utf8_to_32_test.cpp b/libc/test/src/__support/wchar/utf8_to_32_test.cpp
index b419fb5d55414..db74cddafcbd7 100644
--- a/libc/test/src/__support/wchar/utf8_to_32_test.cpp
+++ b/libc/test/src/__support/wchar/utf8_to_32_test.cpp
@@ -8,11 +8,21 @@
#include "hdr/errno_macros.h"
#include "src/__support/error_or.h"
+#include "src/__support/macros/properties/types.h"
#include "src/__support/wchar/character_converter.h"
#include "src/__support/wchar/mbstate.h"
#include "test/UnitTest/Test.h"
-TEST(LlvmLibcCharacterConverterUTF8To32Test, OneByte) {
+#if defined(LIBC_TYPES_WCHAR_T_IS_UTF32)
+using TestCharTypesUTF32 = LIBC_NAMESPACE::testing::TypeList<char32_t, wchar_t>;
+#else
+using TestCharTypesUTF32 = LIBC_NAMESPACE::testing::TypeList<char32_t>;
+#endif
+
+TYPED_TEST(LlvmLibcCharacterConverterUTF8To32Test, OneByte,
+ TestCharTypesUTF32) {
+ using CharType32 = ParamType;
+
LIBC_NAMESPACE::internal::mbstate state;
state.bytes_stored = 0;
state.total_bytes = 0;
@@ -20,14 +30,17 @@ TEST(LlvmLibcCharacterConverterUTF8To32Test, OneByte) {
LIBC_NAMESPACE::internal::CharacterConverter char_conv(&state);
int err = char_conv.push(static_cast<char8_t>(ch));
- auto wch = char_conv.pop_utf32();
+ auto wch = char_conv.pop<CharType32>();
ASSERT_EQ(err, 0);
ASSERT_TRUE(wch.has_value());
ASSERT_EQ(static_cast<int>(wch.value()), 65);
}
-TEST(LlvmLibcCharacterConverterUTF8To32Test, TwoBytes) {
+TYPED_TEST(LlvmLibcCharacterConverterUTF8To32Test, TwoBytes,
+ TestCharTypesUTF32) {
+ using CharType32 = ParamType;
+
LIBC_NAMESPACE::internal::mbstate state;
state.bytes_stored = 0;
state.total_bytes = 0;
@@ -37,13 +50,16 @@ TEST(LlvmLibcCharacterConverterUTF8To32Test, TwoBytes) {
LIBC_NAMESPACE::internal::CharacterConverter char_conv(&state);
char_conv.push(static_cast<char8_t>(ch[0]));
char_conv.push(static_cast<char8_t>(ch[1]));
- auto wch = char_conv.pop_utf32();
+ auto wch = char_conv.pop<CharType32>();
ASSERT_TRUE(wch.has_value());
ASSERT_EQ(static_cast<int>(wch.value()), 142);
}
-TEST(LlvmLibcCharacterConverterUTF8To32Test, ThreeBytes) {
+TYPED_TEST(LlvmLibcCharacterConverterUTF8To32Test, ThreeBytes,
+ TestCharTypesUTF32) {
+ using CharType32 = ParamType;
+
LIBC_NAMESPACE::internal::mbstate state;
state.bytes_stored = 0;
state.total_bytes = 0;
@@ -54,13 +70,16 @@ TEST(LlvmLibcCharacterConverterUTF8To32Test, ThreeBytes) {
char_conv.push(static_cast<char8_t>(ch[0]));
char_conv.push(static_cast<char8_t>(ch[1]));
char_conv.push(static_cast<char8_t>(ch[2]));
- auto wch = char_conv.pop_utf32();
+ auto wch = char_conv.pop<CharType32>();
ASSERT_TRUE(wch.has_value());
ASSERT_EQ(static_cast<int>(wch.value()), 8721);
}
-TEST(LlvmLibcCharacterConverterUTF8To32Test, FourBytes) {
+TYPED_TEST(LlvmLibcCharacterConverterUTF8To32Test, FourBytes,
+ TestCharTypesUTF32) {
+ using CharType32 = ParamType;
+
LIBC_NAMESPACE::internal::mbstate state;
state.bytes_stored = 0;
state.total_bytes = 0;
@@ -73,7 +92,7 @@ TEST(LlvmLibcCharacterConverterUTF8To32Test, FourBytes) {
char_conv.push(static_cast<char8_t>(ch[1]));
char_conv.push(static_cast<char8_t>(ch[2]));
char_conv.push(static_cast<char8_t>(ch[3]));
- auto wch = char_conv.pop_utf32();
+ auto wch = char_conv.pop<CharType32>();
ASSERT_TRUE(wch.has_value());
ASSERT_EQ(static_cast<int>(wch.value()), 129313);
@@ -131,7 +150,10 @@ TEST(LlvmLibcCharacterConverterUTF8To32Test, InvalidLastByte) {
ASSERT_EQ(err, EILSEQ);
}
-TEST(LlvmLibcCharacterConverterUTF8To32Test, ValidTwoByteWithExtraRead) {
+TYPED_TEST(LlvmLibcCharacterConverterUTF8To32Test, ValidTwoByteWithExtraRead,
+ TestCharTypesUTF32) {
+ using CharType32 = ParamType;
+
LIBC_NAMESPACE::internal::mbstate state;
state.bytes_stored = 0;
state.total_bytes = 0;
@@ -148,11 +170,14 @@ TEST(LlvmLibcCharacterConverterUTF8To32Test, ValidTwoByteWithExtraRead) {
ASSERT_EQ(err, EILSEQ);
// Should produce an error since mbstate was reset
- auto wch = char_conv.pop_utf32();
+ auto wch = char_conv.pop<CharType32>();
ASSERT_FALSE(wch.has_value());
}
-TEST(LlvmLibcCharacterConverterUTF8To32Test, TwoValidTwoBytes) {
+TYPED_TEST(LlvmLibcCharacterConverterUTF8To32Test, TwoValidTwoBytes,
+ TestCharTypesUTF32) {
+ using CharType32 = ParamType;
+
LIBC_NAMESPACE::internal::mbstate state;
state.bytes_stored = 0;
state.total_bytes = 0;
@@ -164,7 +189,7 @@ TEST(LlvmLibcCharacterConverterUTF8To32Test, TwoValidTwoBytes) {
ASSERT_EQ(err, 0);
err = char_conv.push(static_cast<char8_t>(ch[1]));
ASSERT_EQ(err, 0);
- auto wch = char_conv.pop_utf32();
+ auto wch = char_conv.pop<CharType32>();
ASSERT_TRUE(wch.has_value());
ASSERT_EQ(static_cast<int>(wch.value()), 142);
@@ -173,12 +198,15 @@ TEST(LlvmLibcCharacterConverterUTF8To32Test, TwoValidTwoBytes) {
ASSERT_EQ(err, 0);
err = char_conv.push(static_cast<char8_t>(ch[3]));
ASSERT_EQ(err, 0);
- wch = char_conv.pop_utf32();
+ wch = char_conv.pop<CharType32>();
ASSERT_TRUE(wch.has_value());
ASSERT_EQ(static_cast<int>(wch.value()), 460);
}
-TEST(LlvmLibcCharacterConverterUTF8To32Test, InvalidPop) {
+TYPED_TEST(LlvmLibcCharacterConverterUTF8To32Test, InvalidPop,
+ TestCharTypesUTF32) {
+ using CharType32 = ParamType;
+
LIBC_NAMESPACE::internal::mbstate state;
state.bytes_stored = 0;
state.total_bytes = 0;
@@ -186,12 +214,12 @@ TEST(LlvmLibcCharacterConverterUTF8To32Test, InvalidPop) {
const char ch[2] = {static_cast<char>(0xC2), static_cast<char>(0x8E)};
int err = char_conv.push(static_cast<char8_t>(ch[0]));
ASSERT_EQ(err, 0);
- auto wch = char_conv.pop_utf32();
+ auto wch = char_conv.pop<CharType32>();
ASSERT_FALSE(
wch.has_value()); // Should fail since we have not read enough bytes
err = char_conv.push(static_cast<char8_t>(ch[1]));
ASSERT_EQ(err, 0);
- wch = char_conv.pop_utf32();
+ wch = char_conv.pop<CharType32>();
ASSERT_TRUE(wch.has_value());
ASSERT_EQ(static_cast<int>(wch.value()), 142);
}
>From c5663ffe02edcfb3207a58d21599ace0415bc999 Mon Sep 17 00:00:00 2001
From: Alex Strelnikov <strel at google.com>
Date: Wed, 29 Jul 2026 15:49:50 +0000
Subject: [PATCH 2/2] Use if instead of ternary op for readability. Fix typo.
---
libc/src/__support/wchar/character_converter.h | 10 ++++++----
libc/test/UnitTest/LibcTest.cpp | 2 +-
2 files changed, 7 insertions(+), 5 deletions(-)
diff --git a/libc/src/__support/wchar/character_converter.h b/libc/src/__support/wchar/character_converter.h
index f0e3b4891ebc1..e26a9ca249e0d 100644
--- a/libc/src/__support/wchar/character_converter.h
+++ b/libc/src/__support/wchar/character_converter.h
@@ -214,12 +214,14 @@ LIBC_INLINE ErrorOr<char8_t> CharacterConverter::pop_utf8() {
LIBC_INLINE ErrorOr<wchar_t> CharacterConverter::pop_wchar() {
#if defined(LIBC_TYPES_WCHAR_T_IS_UTF32)
ErrorOr<char32_t> Result = pop_utf32();
- return Result ? ErrorOr<wchar_t>(static_cast<wchar_t>(*Result))
- : ErrorOr<wchar_t>(Error(Result.error()));
+ if (!Result)
+ return Result.error();
+ return static_cast<wchar_t>(*Result);
#elif defined(LIBC_TYPES_WCHAR_T_IS_UTF16)
ErrorOr<char16_t> Result = pop_utf16();
- return Result ? ErrorOr<wchar_t>(static_cast<wchar_t>(*Result))
- : ErrorOr<wchar_t>(Error(Result.error()));
+ if (!Result)
+ return Result.error();
+ return static_cast<wchar_t>(*Result);
#else
return Error(-1);
#endif
diff --git a/libc/test/UnitTest/LibcTest.cpp b/libc/test/UnitTest/LibcTest.cpp
index 0148b41cd57f0..f5c4f69214083 100644
--- a/libc/test/UnitTest/LibcTest.cpp
+++ b/libc/test/UnitTest/LibcTest.cpp
@@ -83,7 +83,7 @@ cpp::string describeValue(cpp::wstring_view Value) {
}
if (S.empty() && !Value.empty())
- S = cpp::string("<Failed Converstion To UTF-8>");
+ S = cpp::string("<Failed Conversion To UTF-8>");
return S;
}
More information about the libc-commits
mailing list