[libc-commits] [libc] [libc] Implements UTF-16 wide char support (PR #229874)
via libc-commits
libc-commits at lists.llvm.org
Wed Oct 7 13:50:25 PDT 2026
https://github.com/yangbowen updated https://github.com/llvm/llvm-project/pull/229874
>From ce743b9ea86af0dcfad3c292ae3a084e41fe2221 Mon Sep 17 00:00:00 2001
From: yangbowen <ybw1234567 at hotmail.com>
Date: Tue, 6 Oct 2026 17:36:02 +0000
Subject: [PATCH] [libc] Implements UTF-16 wide char support
---
libc/config/windows/config.json | 10 +-
libc/hdr/types/CMakeLists.txt | 11 +
libc/hdr/types/char16_t.h | 27 ++
libc/include/llvm-libc-macros/stdlib-macros.h | 5 +-
libc/src/__support/CMakeLists.txt | 1 +
libc/src/__support/File/file.cpp | 28 +-
.../__support/printf_core/string_converter.h | 12 +-
libc/src/__support/uchar/CMakeLists.txt | 63 +++++
libc/src/__support/uchar/c16rtomb.h | 36 +++
libc/src/__support/uchar/c32rtomb.h | 36 +++
libc/src/__support/uchar/mbrtoc16.h | 37 +++
libc/src/__support/uchar/mbrtoc32.h | 37 +++
libc/src/__support/wchar/CMakeLists.txt | 26 +-
.../src/__support/wchar/character_converter.h | 253 +++++++++++++++++-
libc/src/__support/wchar/mbrtowc.cpp | 52 ----
libc/src/__support/wchar/mbrtowc.h | 12 +-
libc/src/__support/wchar/mbsnrtowcs.h | 5 +-
libc/src/__support/wchar/string_converter.h | 2 -
libc/src/__support/wchar/wcrtomb.h | 37 +--
libc/src/__support/wchar/wcsnrtombs.h | 9 +-
libc/src/wchar/CMakeLists.txt | 1 +
libc/src/wchar/wcrtomb.cpp | 3 +-
22 files changed, 566 insertions(+), 137 deletions(-)
create mode 100644 libc/hdr/types/char16_t.h
create mode 100644 libc/src/__support/uchar/CMakeLists.txt
create mode 100644 libc/src/__support/uchar/c16rtomb.h
create mode 100644 libc/src/__support/uchar/c32rtomb.h
create mode 100644 libc/src/__support/uchar/mbrtoc16.h
create mode 100644 libc/src/__support/uchar/mbrtoc32.h
delete mode 100644 libc/src/__support/wchar/mbrtowc.cpp
diff --git a/libc/config/windows/config.json b/libc/config/windows/config.json
index 6bb09db9d1aab4e..8e5ea141f66dd9b 100644
--- a/libc/config/windows/config.json
+++ b/libc/config/windows/config.json
@@ -1,8 +1,8 @@
{
- "printf": {
- "LIBC_CONF_PRINTF_DISABLE_WIDE": {
- "value": "true",
- "doc": "Disable handling wide characters for printf and friends."
- }
+ "printf": {
+ "LIBC_CONF_PRINTF_DISABLE_WIDE": {
+ "value": "false",
+ "doc": "Disable handling wide characters for printf and friends."
}
+ }
}
diff --git a/libc/hdr/types/CMakeLists.txt b/libc/hdr/types/CMakeLists.txt
index 92a2985b74be726..6ca38970729cfda 100644
--- a/libc/hdr/types/CMakeLists.txt
+++ b/libc/hdr/types/CMakeLists.txt
@@ -9,6 +9,17 @@ add_proxy_header_library(
libc.include.uchar
)
+add_proxy_header_library(
+ char16_t
+ HDRS
+ char16_t.h
+ DEPENDS
+ libc.hdr.uchar_overlay
+ FULL_BUILD_DEPENDS
+ libc.include.llvm-libc-types.char16_t
+ libc.include.uchar
+)
+
add_proxy_header_library(
char32_t
HDRS
diff --git a/libc/hdr/types/char16_t.h b/libc/hdr/types/char16_t.h
new file mode 100644
index 000000000000000..a59d698243b0907
--- /dev/null
+++ b/libc/hdr/types/char16_t.h
@@ -0,0 +1,27 @@
+//===-- Definition of char16_t.h ------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef LLVM_LIBC_HDR_TYPES_CHAR16_T_H
+#define LLVM_LIBC_HDR_TYPES_CHAR16_T_H
+
+#ifdef LIBC_FULL_BUILD
+
+#include "include/llvm-libc-types/char16_t.h"
+
+#else // overlay mode
+
+// MacOS doesn't provide uchar.h so we use the types provided by LLVM-libc.
+#ifdef __APPLE__
+#include "include/llvm-libc-types/char16_t.h"
+#else
+#include "hdr/uchar_overlay.h"
+#endif
+
+#endif // LIBC_FULL_BUILD
+
+#endif // LLVM_LIBC_HDR_TYPES_CHAR16_T_H
diff --git a/libc/include/llvm-libc-macros/stdlib-macros.h b/libc/include/llvm-libc-macros/stdlib-macros.h
index 2565c76be3c55c8..f326d4a35d16701 100644
--- a/libc/include/llvm-libc-macros/stdlib-macros.h
+++ b/libc/include/llvm-libc-macros/stdlib-macros.h
@@ -18,8 +18,9 @@
#define EXIT_FAILURE 1
#ifndef MB_CUR_MAX
-// We only support the "C" locale right now, so this is a constant byte.
-#define MB_CUR_MAX 1
+// We do UTF-32 <-> UTF-8 conversions, so MAX_UTF8_LENGTH
+// https://www.open-std.org/jtc1/sc22/wg14/issues/c90/issue0039.01.html
+#define MB_CUR_MAX 4
#endif // MB_CUR_MAX
#define RAND_MAX 2147483647
diff --git a/libc/src/__support/CMakeLists.txt b/libc/src/__support/CMakeLists.txt
index cc31ced36cf2ee7..4dd9fa67aefe300 100644
--- a/libc/src/__support/CMakeLists.txt
+++ b/libc/src/__support/CMakeLists.txt
@@ -518,6 +518,7 @@ add_subdirectory(fixed_point)
add_subdirectory(time)
+add_subdirectory(uchar)
add_subdirectory(wchar)
add_subdirectory(wctype)
diff --git a/libc/src/__support/File/file.cpp b/libc/src/__support/File/file.cpp
index c296b99e2ffe414..3ab499dfba9db5a 100644
--- a/libc/src/__support/File/file.cpp
+++ b/libc/src/__support/File/file.cpp
@@ -559,10 +559,12 @@ FileIOResult File::write_unlocked(const wchar_t *ws, size_t len) {
size_t written = 0;
for (size_t i = 0; i < len; ++i) {
internal::CharacterConverter cr(&mbstate);
- int push_err = cr.push(static_cast<char32_t>(ws[i]));
- if (push_err != 0) {
- err = true;
- return {written, push_err};
+ while (!cr.isFull()) {
+ int push_err = cr.push(ws[i]);
+ if (push_err != 0) {
+ err = true;
+ return {written, push_err};
+ }
}
// buffer the whole wchar to save on calls to write.
char buffer[4];
@@ -574,7 +576,7 @@ FileIOResult File::write_unlocked(const wchar_t *ws, size_t len) {
return {written, pop_res.error()};
}
char8_t byte = pop_res.value();
- buffer[char_size] = byte;
+ buffer[char_size] = static_cast<char>(byte);
++char_size;
}
auto write_res = write_unlocked_impl(buffer, char_size);
@@ -602,7 +604,7 @@ FileIOResult File::read_unlocked(wchar_t *ws, size_t len) {
}
size_t read_count = 0;
- for (size_t i = 0; i < len; ++i) {
+ for (size_t i = 0; i < len;) {
internal::CharacterConverter cr(&mbstate);
while (!cr.isFull()) {
uint8_t byte;
@@ -621,13 +623,15 @@ FileIOResult File::read_unlocked(wchar_t *ws, size_t len) {
return {read_count, push_err};
}
}
- auto pop_res = cr.pop<char32_t>();
- if (!pop_res.has_value()) {
- err = true;
- return {read_count, pop_res.error()};
+ while (!cr.isEmpty()) {
+ auto pop_res = cr.pop<wchar_t>();
+ if (!pop_res.has_value()) {
+ err = true;
+ return {read_count, pop_res.error()};
+ }
+ ws[i++] = pop_res.value();
+ ++read_count;
}
- ws[i] = static_cast<wchar_t>(pop_res.value());
- ++read_count;
}
return {read_count, 0};
}
diff --git a/libc/src/__support/printf_core/string_converter.h b/libc/src/__support/printf_core/string_converter.h
index 5352bd8306ba6cb..47c1aad3fea352b 100644
--- a/libc/src/__support/printf_core/string_converter.h
+++ b/libc/src/__support/printf_core/string_converter.h
@@ -70,21 +70,21 @@ template <OverflowMode mode>
LIBC_INLINE int wchar_writer(Writer<mode> *writer,
const FormatSection &to_conv) {
size_t string_len = 0;
- const char32_t *wstr_ptr =
- reinterpret_cast<const char32_t *>(to_conv.conv_val_ptr);
+ const wchar_t *wstr_ptr =
+ reinterpret_cast<const wchar_t *>(to_conv.conv_val_ptr);
size_t precision =
to_conv.precision < 0 ? SIZE_MAX : static_cast<size_t>(to_conv.precision);
#ifndef LIBC_COPT_PRINTF_NO_NULLPTR_CHECKS
if (wstr_ptr == nullptr) {
- wstr_ptr = U"(null)";
+ wstr_ptr = L"(null)";
}
#endif // LIBC_COPT_PRINTF_NO_NULLPTR_CHECKS
internal::mbstate mbstate;
- internal::StringConverter<char32_t> length_counter(wstr_ptr, &mbstate,
- precision);
+ internal::StringConverter<wchar_t> length_counter(wstr_ptr, &mbstate,
+ precision);
for (auto converted = length_counter.pop<char8_t>();
converted.has_value() && converted.value() != '\0';
@@ -103,7 +103,7 @@ LIBC_INLINE int wchar_writer(Writer<mode> *writer,
}
mbstate = internal::mbstate();
- internal::StringConverter<char32_t> out_conv(wstr_ptr, &mbstate, precision);
+ internal::StringConverter<wchar_t> out_conv(wstr_ptr, &mbstate, precision);
for (auto converted = out_conv.pop<char8_t>();
converted.has_value() && converted.value() != '\0';
diff --git a/libc/src/__support/uchar/CMakeLists.txt b/libc/src/__support/uchar/CMakeLists.txt
new file mode 100644
index 000000000000000..5364c117d5ebd9e
--- /dev/null
+++ b/libc/src/__support/uchar/CMakeLists.txt
@@ -0,0 +1,63 @@
+add_header_library(
+ c32rtomb
+ HDRS
+ c32rtomb.h
+ DEPENDS
+ libc.hdr.errno_macros
+ libc.hdr.types.char32_t
+ libc.hdr.types.size_t
+ libc.src.__support.common
+ libc.src.__support.error_or
+ libc.src.__support.macros.config
+ libc.src.__support.macros.null_check
+ libc.src.__support.wchar.character_converter
+ libc.src.__support.wchar.mbstate
+)
+
+add_header_library(
+ mbrtoc32
+ HDRS
+ mbrtoc32.h
+ DEPENDS
+ libc.hdr.errno_macros
+ libc.hdr.types.char32_t
+ libc.hdr.types.size_t
+ libc.src.__support.common
+ libc.src.__support.error_or
+ libc.src.__support.macros.config
+ libc.src.__support.macros.null_check
+ libc.src.__support.wchar.character_converter
+ libc.src.__support.wchar.mbstate
+)
+
+add_header_library(
+ c16rtomb
+ HDRS
+ c16rtomb.h
+ DEPENDS
+ libc.hdr.errno_macros
+ libc.hdr.types.char16_t
+ libc.hdr.types.size_t
+ libc.src.__support.common
+ libc.src.__support.error_or
+ libc.src.__support.macros.config
+ libc.src.__support.macros.null_check
+ libc.src.__support.wchar.character_converter
+ libc.src.__support.wchar.mbstate
+)
+
+add_header_library(
+ mbrtoc16
+ HDRS
+ mbrtoc16.h
+ DEPENDS
+ libc.hdr.errno_macros
+ libc.hdr.types.char16_t
+ libc.hdr.types.size_t
+ libc.src.__support.common
+ libc.src.__support.error_or
+ libc.src.__support.macros.config
+ libc.src.__support.macros.null_check
+ libc.src.__support.wchar.character_converter
+ libc.src.__support.wchar.mbstate
+)
diff --git a/libc/src/__support/uchar/c16rtomb.h b/libc/src/__support/uchar/c16rtomb.h
new file mode 100644
index 000000000000000..ff241e47bda12f1
--- /dev/null
+++ b/libc/src/__support/uchar/c16rtomb.h
@@ -0,0 +1,36 @@
+//===-- Implementation header for c16rtomb --------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef LLVM_LIBC_SRC__SUPPORT_UCHAR_C16RTOMB_H
+#define LLVM_LIBC_SRC__SUPPORT_UCHAR_C16RTOMB_H
+
+#include "hdr/errno_macros.h"
+#include "hdr/types/char16_t.h"
+#include "hdr/types/size_t.h"
+#include "src/__support/common.h"
+#include "src/__support/error_or.h"
+#include "src/__support/macros/config.h"
+#include "src/__support/macros/null_check.h"
+#include "src/__support/wchar/character_converter.h"
+#include "src/__support/wchar/mbstate.h"
+
+namespace LIBC_NAMESPACE_DECL {
+namespace internal {
+
+template <typename Char16Type = char16_t, typename Char8Type = char8_t>
+LIBC_INLINE ErrorOr<size_t> c16rtomb(Char8Type *__restrict dest_ptr,
+ Char16Type src, mbstate *__restrict ps) {
+ LIBC_CRASH_ON_NULLPTR(ps);
+ CharacterConverter char_conv(ps);
+ return char_conv.rtomb_generic<char16_t>(dest_ptr, src);
+}
+
+} // namespace internal
+} // namespace LIBC_NAMESPACE_DECL
+
+#endif // LLVM_LIBC_SRC__SUPPORT_UCHAR_C16RTOMB_H
diff --git a/libc/src/__support/uchar/c32rtomb.h b/libc/src/__support/uchar/c32rtomb.h
new file mode 100644
index 000000000000000..288c655cc7b34be
--- /dev/null
+++ b/libc/src/__support/uchar/c32rtomb.h
@@ -0,0 +1,36 @@
+//===-- Implementation header for c32rtomb --------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef LLVM_LIBC_SRC__SUPPORT_UCHAR_C32RTOMB_H
+#define LLVM_LIBC_SRC__SUPPORT_UCHAR_C32RTOMB_H
+
+#include "hdr/errno_macros.h"
+#include "hdr/types/char32_t.h"
+#include "hdr/types/size_t.h"
+#include "src/__support/common.h"
+#include "src/__support/error_or.h"
+#include "src/__support/macros/config.h"
+#include "src/__support/macros/null_check.h"
+#include "src/__support/wchar/character_converter.h"
+#include "src/__support/wchar/mbstate.h"
+
+namespace LIBC_NAMESPACE_DECL {
+namespace internal {
+
+template <typename Char32Type = char32_t, typename Char8Type = char8_t>
+LIBC_INLINE ErrorOr<size_t> c32rtomb(Char8Type *__restrict dest_ptr,
+ Char32Type src, mbstate *__restrict ps) {
+ LIBC_CRASH_ON_NULLPTR(ps);
+ CharacterConverter char_conv(ps);
+ return char_conv.rtomb_generic<char32_t>(dest_ptr, src);
+}
+
+} // namespace internal
+} // namespace LIBC_NAMESPACE_DECL
+
+#endif // LLVM_LIBC_SRC__SUPPORT_UCHAR_C32RTOMB_H
diff --git a/libc/src/__support/uchar/mbrtoc16.h b/libc/src/__support/uchar/mbrtoc16.h
new file mode 100644
index 000000000000000..c5b1e766141b79c
--- /dev/null
+++ b/libc/src/__support/uchar/mbrtoc16.h
@@ -0,0 +1,37 @@
+//===-- Implementation header for mbrtoc16 ----------------------*- C++ -*-===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef LLVM_LIBC_SRC__SUPPORT_UCHAR_MBRTOC16_H
+#define LLVM_LIBC_SRC__SUPPORT_UCHAR_MBRTOC16_H
+
+#include "hdr/errno_macros.h"
+#include "hdr/types/char16_t.h"
+#include "hdr/types/size_t.h"
+#include "src/__support/common.h"
+#include "src/__support/error_or.h"
+#include "src/__support/macros/config.h"
+#include "src/__support/macros/null_check.h"
+#include "src/__support/wchar/character_converter.h"
+#include "src/__support/wchar/mbstate.h"
+
+namespace LIBC_NAMESPACE_DECL {
+namespace internal {
+
+template <typename Char16Type = char16_t, typename Char8Type = char8_t>
+LIBC_INLINE ErrorOr<size_t>
+mbrtoc16(Char16Type *__restrict dest_ptr, const Char8Type *__restrict src_ptr,
+ size_t max_src_bytes, mbstate *__restrict ps) {
+ LIBC_CRASH_ON_NULLPTR(ps);
+ CharacterConverter char_conv(ps);
+ return char_conv.mbrto_generic<wchar_t>(dest_ptr, src_ptr, max_src_bytes);
+}
+
+} // namespace internal
+} // namespace LIBC_NAMESPACE_DECL
+
+#endif // LLVM_LIBC_SRC__SUPPORT_UCHAR_MBRTOC16_H
diff --git a/libc/src/__support/uchar/mbrtoc32.h b/libc/src/__support/uchar/mbrtoc32.h
new file mode 100644
index 000000000000000..76bfab07239c580
--- /dev/null
+++ b/libc/src/__support/uchar/mbrtoc32.h
@@ -0,0 +1,37 @@
+//===-- Implementation header for mbrtoc32 ----------------------*- C++ -*-===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef LLVM_LIBC_SRC__SUPPORT_UCHAR_MBRTOC32_H
+#define LLVM_LIBC_SRC__SUPPORT_UCHAR_MBRTOC32_H
+
+#include "hdr/errno_macros.h"
+#include "hdr/types/char32_t.h"
+#include "hdr/types/size_t.h"
+#include "src/__support/common.h"
+#include "src/__support/error_or.h"
+#include "src/__support/macros/config.h"
+#include "src/__support/macros/null_check.h"
+#include "src/__support/wchar/character_converter.h"
+#include "src/__support/wchar/mbstate.h"
+
+namespace LIBC_NAMESPACE_DECL {
+namespace internal {
+
+template <typename Char32Type = char32_t, typename Char8Type = char8_t>
+LIBC_INLINE ErrorOr<size_t>
+mbrtoc32(Char32Type *__restrict dest_ptr, const Char8Type *__restrict src_ptr,
+ size_t max_src_bytes, mbstate *__restrict ps) {
+ LIBC_CRASH_ON_NULLPTR(ps);
+ CharacterConverter char_conv(ps);
+ return char_conv.mbrto_generic<wchar_t>(dest_ptr, src_ptr, max_src_bytes);
+}
+
+} // namespace internal
+} // namespace LIBC_NAMESPACE_DECL
+
+#endif // LLVM_LIBC_SRC__SUPPORT_UCHAR_MBRTOC32_H
diff --git a/libc/src/__support/wchar/CMakeLists.txt b/libc/src/__support/wchar/CMakeLists.txt
index c4e7f0dbaa1786e..83462b9f0fc93c6 100644
--- a/libc/src/__support/wchar/CMakeLists.txt
+++ b/libc/src/__support/wchar/CMakeLists.txt
@@ -12,8 +12,6 @@ add_header_library(
HDRS
string_converter.h
DEPENDS
- libc.hdr.types.char8_t
- libc.hdr.types.char32_t
libc.hdr.types.size_t
libc.src.__support.common
libc.src.__support.CPP.type_traits
@@ -29,11 +27,13 @@ add_header_library(
DEPENDS
libc.hdr.errno_macros
libc.hdr.types.char8_t
+ libc.hdr.types.char16_t
libc.hdr.types.char32_t
libc.hdr.types.size_t
libc.src.__support.common
libc.src.__support.CPP.type_traits
libc.src.__support.error_or
+ libc.src.__support.libc_assert
libc.src.__support.macros.config
libc.src.__support.macros.properties.types
libc.src.__support.math_extras
@@ -45,33 +45,29 @@ add_header_library(
HDRS
wcrtomb.h
DEPENDS
- libc.hdr.errno_macros
- libc.hdr.types.char32_t
libc.hdr.types.size_t
libc.hdr.types.wchar_t
libc.src.__support.common
libc.src.__support.error_or
- libc.src.__support.libc_assert
+ libc.src.__support.macros.config
libc.src.__support.macros.null_check
- .character_converter
.mbstate
+ .character_converter
)
-add_object_library(
+add_header_library(
mbrtowc
HDRS
mbrtowc.h
- SRCS
- mbrtowc.cpp
DEPENDS
- libc.hdr.errno_macros
- libc.hdr.types.wchar_t
libc.hdr.types.size_t
+ libc.hdr.types.wchar_t
libc.src.__support.common
libc.src.__support.error_or
libc.src.__support.macros.config
- .character_converter
+ libc.src.__support.macros.null_check
.mbstate
+ .character_converter
)
add_header_library(
@@ -80,8 +76,9 @@ add_header_library(
mbsnrtowcs.h
DEPENDS
libc.hdr.errno_macros
- libc.hdr.types.wchar_t
+ libc.hdr.types.char8_t
libc.hdr.types.size_t
+ libc.hdr.types.wchar_t
libc.src.__support.common
libc.src.__support.error_or
libc.src.__support.macros.config
@@ -98,14 +95,13 @@ add_header_library(
DEPENDS
libc.hdr.errno_macros
libc.hdr.types.char8_t
- libc.hdr.types.char32_t
libc.hdr.types.size_t
libc.hdr.types.wchar_t
libc.src.__support.common
libc.src.__support.error_or
libc.src.__support.macros.config
libc.src.__support.macros.null_check
- .string_converter
.character_converter
.mbstate
+ .string_converter
)
diff --git a/libc/src/__support/wchar/character_converter.h b/libc/src/__support/wchar/character_converter.h
index f754a602d2cce0f..24d92a7576b6256 100644
--- a/libc/src/__support/wchar/character_converter.h
+++ b/libc/src/__support/wchar/character_converter.h
@@ -10,6 +10,7 @@
#define LLVM_LIBC_SRC___SUPPORT_CHARACTER_CONVERTER_H
#include "hdr/errno_macros.h"
+#include "hdr/types/char16_t.h"
#include "hdr/types/char32_t.h"
#include "hdr/types/char8_t.h"
#include "hdr/types/size_t.h"
@@ -18,6 +19,7 @@
#include "src/__support/CPP/type_traits.h"
#include "src/__support/common.h"
#include "src/__support/error_or.h"
+#include "src/__support/libc_assert.h"
#include "src/__support/macros/properties/types.h"
#include "src/__support/math_extras.h"
#include "src/__support/wchar/mbstate.h"
@@ -57,20 +59,76 @@ class CharacterConverter {
LIBC_INLINE bool isEmpty() { return state->bytes_stored == 0; }
bool isValidState();
+ /*
+ * When converting between UTF-16 and UTF-8, the converter might be in a state
+ * of either partially-pushing or partially-popping. For example consider the
+ * following sequence:
+ * * push((char8_t)0xF0);
+ * * push((char8_t)0xA4);
+ * * [A](partial=0x24,bytes_stored=2,total_bytes=4)
+ * * push((char8_t)0xAD);
+ * * push((char8_t)0xA2);
+ * * (partial=0x24B62,bytes_stored=4,total_bytes=4)
+ * * pop_utf16(); -> 0xD852
+ * * [B](partial=0x24B62,bytes_stored=2,total_bytes=4)
+ * * pop_utf16(); -> 0xDF62
+ * At point [A] and point [B], the stored bytes would both be 2. But at [A] we
+ * must continue reading input, while at [B] we must write the output low
+ * surrogate and clear the state.
+ * Thus we must have some means to distinguish between pushing and popping.
+ * Notice that: When pushing, we do left-shifts on state->partial before each
+ * push. When popping, we keep state->partial at the complete code point.
+ * Since we disallow overlong encodings, the complete UTF-8 code point is
+ * always greater than
+ * * MAX_VALUE_PER_UTF8_LEN[state->total_bytes - 2]
+ * but at most
+ * * MAX_VALUE_PER_UTF8_LEN[state->total_bytes - 1]
+ * If we are partially through pushing, state->partial is at most
+ * * char32 >> 6
+ * thus at most
+ * * MAX_VALUE_PER_UTF8_LEN[state->total_bytes - 1] >> 6
+ * Observe that, an item in MAX_VALUE_PER_UTF8_LEN right-shifted by 6 is
+ * always less than the preceding item. Based on these facts, we can determine
+ * if we are partially through pushing or popping, by comparing state->partial
+ * with MAX_VALUE_PER_UTF8_LEN[state->total_bytes - 2]
+ * Also when converting UTF-16 -> UTF-8, in a partial state,
+ * * bytes_stored=2
+ * * total_bytes=4
+ * so the threshold 0xFFFF is also correct.
+ */
+ LIBC_INLINE bool isPartiallyPopping() {
+ if (isEmpty() || isFull())
+ return false;
+ // When neither empty nor full, state->total_bytes is at least 2
+ char32_t threshold = MAX_VALUE_PER_UTF8_LEN[state->total_bytes - 2];
+ return state->partial > threshold;
+ }
+
template <typename CharType> size_t sizeAs();
int push(char8_t utf8_byte);
+ int push(char16_t utf16_byte);
int push(char32_t utf32);
ErrorOr<char8_t> pop_utf8();
+ ErrorOr<char16_t> pop_utf16();
ErrorOr<char32_t> pop_utf32();
-#if defined(LIBC_TYPES_WCHAR_T_IS_UTF32)
+#if defined(LIBC_TYPES_WCHAR_T_IS_UTF32) || defined(LIBC_TYPES_WCHAR_T_IS_UTF16)
int push(wchar_t wchar);
ErrorOr<wchar_t> pop_wchar();
-#endif // LIBC_TYPES_WCHAR_T_IS_UTF32
+#endif
template <typename CharType> ErrorOr<CharType> pop();
+
+ // mbrtowc, mbrtoc16, mbrtoc32
+ template <typename T>
+ ErrorOr<size_t> mbrto_generic(T *__restrict dest_ptr,
+ const char *__restrict src_ptr,
+ size_t max_src_bytes);
+ // wcrtomb, c16rtomb, c32rtomb
+ template <typename T>
+ ErrorOr<size_t> rtomb_generic(char *__restrict dest_ptr, T src);
};
LIBC_INLINE bool CharacterConverter::isValidState() {
@@ -97,7 +155,7 @@ LIBC_INLINE int CharacterConverter::push(char8_t utf8_byte) {
/* Since the format is 110xxxxx, 1110xxxx, and 11110xxx for 2, 3, and 4,
we will make the base mask with 7 ones and right shift it as necessary. */
constexpr size_t SIGNIFICANT_BITS = 7;
- char8_t base_mask =
+ constexpr char8_t base_mask =
static_cast<char8_t>(mask_trailing_ones<uint8_t, SIGNIFICANT_BITS>());
state->total_bytes = num_ones;
utf8_byte &= (base_mask >> num_ones);
@@ -119,6 +177,13 @@ LIBC_INLINE int CharacterConverter::push(char8_t utf8_byte) {
state->partial = state->partial << ENCODED_BITS_PER_UTF8;
state->partial |= byte;
state->bytes_stored++;
+ if (isFull()) {
+ // Detect and reject overlong encodings
+ if (state->partial <= MAX_VALUE_PER_UTF8_LEN[state->total_bytes - 2]) {
+ clear();
+ return EILSEQ;
+ }
+ }
return 0;
}
@@ -127,6 +192,53 @@ LIBC_INLINE int CharacterConverter::push(char8_t utf8_byte) {
return EILSEQ;
}
+LIBC_INLINE int CharacterConverter::push(char16_t utf16) {
+ // Checking for high surrogate if first push
+ if (isEmpty()) {
+ bool is_surrogate = (utf16 & 0xF800) == 0xD800;
+ if (is_surrogate) {
+ bool is_high_surrogate = (utf16 & 0x400) == 0;
+ if (!is_high_surrogate) {
+ // unpaired surrogate -> reset the state
+ // bytes_stored and total_bytes will always be 0 here
+ state->partial = static_cast<char32_t>(0);
+ return EILSEQ;
+ }
+ // high surrogate
+ // code point between 0x10000 and 0x10FFFF
+ state->total_bytes = 4;
+ state->bytes_stored = 2;
+ // 10 bits each from the high surrogate and the low surrogate, and the
+ // full code point is from 0x10000, so we add 0x40 to partial, which after
+ // shifting becomes 0x10000
+ state->partial = static_cast<char32_t>(utf16 & 0x3FF) + 0x40;
+ } else {
+ state->partial = static_cast<char32_t>(utf16);
+ // determine number of utf-8 bytes needed to represent this utf16 value
+ uint8_t i;
+ for (i = 0; i < 2; i++) {
+ if (state->partial <= MAX_VALUE_PER_UTF8_LEN[i])
+ break;
+ }
+ state->total_bytes = i + 1;
+ state->bytes_stored = i + 1;
+ }
+ return 0;
+ } else {
+ bool is_low_surrogate = (utf16 & 0xFC00) == 0xDC00;
+ if (!is_low_surrogate) {
+ // unpaired surrogate -> reset the state
+ clear();
+ return EILSEQ;
+ }
+ char32_t low = utf16 & 0x3FF;
+ state->partial <<= 10;
+ state->partial |= low;
+ state->bytes_stored += 2;
+ return 0;
+ }
+}
+
LIBC_INLINE int CharacterConverter::push(char32_t utf32) {
// we can't be partially through a conversion when pushing a utf32 value
if (!isEmpty())
@@ -160,6 +272,34 @@ LIBC_INLINE ErrorOr<char32_t> CharacterConverter::pop_utf32() {
return utf32;
}
+LIBC_INLINE ErrorOr<char16_t> CharacterConverter::pop_utf16() {
+ if (isEmpty())
+ return Error(-1);
+ char32_t output;
+ if (state->total_bytes <= 3) {
+ // no surrogates needed
+ output = state->partial;
+ // disallow using UTF-8 or UTF-32 to encode unpaired surrogates
+ bool is_surrogate = (output & 0xF800) == 0xD800;
+ if (is_surrogate) {
+ clear();
+ return Error(EILSEQ);
+ }
+ clear();
+ } else {
+ if (isFull()) {
+ // high surrogate first
+ output = ((state->partial - 0x10000) >> 10) | 0xD800;
+ state->bytes_stored = 2;
+ } else {
+ // low surrogate
+ output = ((state->partial - 0x10000) & 0x3FF) | 0xDC00;
+ clear();
+ }
+ }
+ return static_cast<char16_t>(output);
+}
+
LIBC_INLINE ErrorOr<char8_t> CharacterConverter::pop_utf8() {
if (isEmpty())
return Error(-1);
@@ -196,6 +336,10 @@ template <> LIBC_INLINE ErrorOr<char8_t> CharacterConverter::pop() {
return pop_utf8();
}
+template <> LIBC_INLINE ErrorOr<char16_t> CharacterConverter::pop() {
+ return pop_utf16();
+}
+
template <> LIBC_INLINE ErrorOr<char32_t> CharacterConverter::pop() {
return pop_utf32();
}
@@ -204,6 +348,10 @@ template <> LIBC_INLINE size_t CharacterConverter::sizeAs<char8_t>() {
return state->total_bytes;
}
+template <> LIBC_INLINE size_t CharacterConverter::sizeAs<char16_t>() {
+ return state->total_bytes > 3 ? 2 : 1;
+}
+
template <> LIBC_INLINE size_t CharacterConverter::sizeAs<char32_t>() {
return 1;
}
@@ -229,7 +377,104 @@ template <> LIBC_INLINE size_t CharacterConverter::sizeAs<wchar_t>() {
return sizeAs<char32_t>();
}
-#endif // LIBC_TYPES_WCHAR_T_IS_UTF32
+#elif defined(LIBC_TYPES_WCHAR_T_IS_UTF16)
+
+LIBC_INLINE int CharacterConverter::push(wchar_t wchar) {
+ return push(static_cast<char16_t>(wchar));
+}
+
+LIBC_INLINE ErrorOr<wchar_t> CharacterConverter::pop_wchar() {
+ ErrorOr<char16_t> Result = pop_utf16();
+ if (!Result)
+ return Error(Result.error());
+ return static_cast<wchar_t>(*Result);
+}
+
+template <> LIBC_INLINE ErrorOr<wchar_t> CharacterConverter::pop() {
+ return pop_wchar();
+}
+
+template <> LIBC_INLINE size_t CharacterConverter::sizeAs<wchar_t>() {
+ return sizeAs<char16_t>();
+}
+
+#endif
+
+template <typename T>
+LIBC_INLINE ErrorOr<size_t>
+CharacterConverter::mbrto_generic(T *__restrict dest_ptr,
+ const char *__restrict src_ptr,
+ size_t max_src_bytes) {
+ if (!isValidState())
+ return Error(EINVAL);
+
+ char empty_src = '\0';
+ if (src_ptr == nullptr) {
+ dest_ptr = nullptr;
+ src_ptr = &empty_src;
+ max_src_bytes = 1;
+ }
+
+ size_t i = 0;
+ // If we still have the lower part to write, don't read yet (UTF-16)
+ if (!isPartiallyPopping()) {
+ // Reading in bytes until we have a complete char32 or error
+ for (; i < max_src_bytes && !isFull(); ++i) {
+ int err = push(static_cast<char8_t>(src_ptr[i]));
+ // Encoding error
+ if (err != 0)
+ return Error(err);
+ }
+ }
+ auto result = pop<T>();
+ if (!result.has_value()) {
+ if (result.error() == -1) {
+ // Incomplete but potentially valid
+ return (size_t)-2;
+ } else {
+ return Error(result.error());
+ }
+ }
+ if (dest_ptr != nullptr)
+ *dest_ptr = result.value();
+ if (i == 0) {
+ // lower part written, no input processed
+ return (size_t)-3;
+ }
+ // null terminator -> return 0
+ if (result.value() == T{})
+ return 0;
+ return i;
+}
+
+template <typename T>
+LIBC_INLINE ErrorOr<size_t>
+CharacterConverter::rtomb_generic(char *__restrict dest_ptr, T src) {
+ if (dest_ptr == nullptr)
+ src = {};
+
+ if (!isValidState())
+ return Error(EINVAL);
+
+ int status = push(static_cast<char16_t>(src));
+ if (status != 0)
+ return Error(status);
+
+ if (!isFull())
+ return 0;
+
+ size_t count = 0;
+ for (; !isEmpty(); ++count) {
+ auto utf8 = pop_utf8(); // can never fail as long as the push succeeded
+ LIBC_ASSERT(utf8.has_value());
+
+ if (dest_ptr != nullptr) {
+ *dest_ptr = static_cast<char>(utf8.value());
+ dest_ptr++;
+ }
+ }
+ return count;
+}
} // namespace internal
} // namespace LIBC_NAMESPACE_DECL
diff --git a/libc/src/__support/wchar/mbrtowc.cpp b/libc/src/__support/wchar/mbrtowc.cpp
deleted file mode 100644
index 33a42a4a31110db..000000000000000
--- a/libc/src/__support/wchar/mbrtowc.cpp
+++ /dev/null
@@ -1,52 +0,0 @@
-//===-- Implementation for mbrtowc function ---------------------*- C++ -*-===//
-//
-// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
-// See https://llvm.org/LICENSE.txt for license information.
-// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
-//
-//===----------------------------------------------------------------------===//
-
-#include "src/__support/wchar/mbrtowc.h"
-#include "hdr/errno_macros.h"
-#include "hdr/types/size_t.h"
-#include "hdr/types/wchar_t.h"
-#include "src/__support/common.h"
-#include "src/__support/error_or.h"
-#include "src/__support/macros/config.h"
-#include "src/__support/wchar/character_converter.h"
-#include "src/__support/wchar/mbstate.h"
-
-namespace LIBC_NAMESPACE_DECL {
-namespace internal {
-
-ErrorOr<size_t> mbrtowc(wchar_t *__restrict pwc, const char *__restrict src_ptr,
- size_t max_src_bytes, mbstate *__restrict ps) {
- CharacterConverter char_conv(ps);
- if (!char_conv.isValidState())
- return Error(EINVAL);
- if (src_ptr == nullptr)
- return 0;
- size_t i = 0;
- // Reading in bytes until we have a complete wc or error
- for (; i < max_src_bytes && !char_conv.isFull(); ++i) {
- int err = char_conv.push(static_cast<char8_t>(src_ptr[i]));
- // Encoding error
- if (err == EILSEQ)
- return Error(err);
- }
- auto wc = char_conv.pop_utf32();
- if (wc.has_value()) {
- if (pwc != nullptr)
- *pwc = wc.value();
- // null terminator -> return 0
- if (wc.value() == L'\0')
- return 0;
- return i;
- }
- // Incomplete but potentially valid
- return -2;
-}
-
-} // namespace internal
-
-} // namespace LIBC_NAMESPACE_DECL
diff --git a/libc/src/__support/wchar/mbrtowc.h b/libc/src/__support/wchar/mbrtowc.h
index d15c4844499a618..6785bbc1363cfcc 100644
--- a/libc/src/__support/wchar/mbrtowc.h
+++ b/libc/src/__support/wchar/mbrtowc.h
@@ -14,13 +14,21 @@
#include "src/__support/common.h"
#include "src/__support/error_or.h"
#include "src/__support/macros/config.h"
+#include "src/__support/macros/null_check.h"
+#include "src/__support/wchar/character_converter.h"
#include "src/__support/wchar/mbstate.h"
namespace LIBC_NAMESPACE_DECL {
namespace internal {
-ErrorOr<size_t> mbrtowc(wchar_t *__restrict pwc, const char *__restrict src_ptr,
- size_t max_src_bytes, mbstate *__restrict ps);
+LIBC_INLINE ErrorOr<size_t> mbrtowc(wchar_t *__restrict pwc,
+ const char *__restrict src_ptr,
+ size_t max_src_bytes,
+ mbstate *__restrict ps) {
+ LIBC_CRASH_ON_NULLPTR(ps);
+ CharacterConverter char_conv(ps);
+ return char_conv.mbrto_generic<wchar_t>(pwc, src_ptr, max_src_bytes);
+}
} // namespace internal
diff --git a/libc/src/__support/wchar/mbsnrtowcs.h b/libc/src/__support/wchar/mbsnrtowcs.h
index 50c3fa23a7d58c2..ebddc9802574db0 100644
--- a/libc/src/__support/wchar/mbsnrtowcs.h
+++ b/libc/src/__support/wchar/mbsnrtowcs.h
@@ -10,6 +10,7 @@
#define LLVM_LIBC_SRC___SUPPORT_WCHAR_MBSNRTOWCS_H
#include "hdr/errno_macros.h"
+#include "hdr/types/char8_t.h"
#include "hdr/types/size_t.h"
#include "hdr/types/wchar_t.h"
#include "src/__support/common.h"
@@ -34,7 +35,7 @@ mbsnrtowcs(wchar_t *__restrict dst, const char **__restrict src,
StringConverter<char8_t> str_conv(reinterpret_cast<const char8_t *>(*src), ps,
max_dst_chars, max_src_bytes);
size_t dst_idx = 0;
- ErrorOr<char32_t> converted = str_conv.pop<char32_t>();
+ ErrorOr<wchar_t> converted = str_conv.pop<wchar_t>();
while (converted.has_value()) {
if (dst != nullptr)
dst[dst_idx] = converted.value();
@@ -45,7 +46,7 @@ mbsnrtowcs(wchar_t *__restrict dst, const char **__restrict src,
return dst_idx;
}
dst_idx++;
- converted = str_conv.pop<char32_t>();
+ converted = str_conv.pop<wchar_t>();
}
if (converted.error() == -1) { // if we hit conversion limit
diff --git a/libc/src/__support/wchar/string_converter.h b/libc/src/__support/wchar/string_converter.h
index 9e76570e1167893..ebc8b4cc987e49d 100644
--- a/libc/src/__support/wchar/string_converter.h
+++ b/libc/src/__support/wchar/string_converter.h
@@ -9,8 +9,6 @@
#ifndef LLVM_LIBC_SRC___SUPPORT_STRING_CONVERTER_H
#define LLVM_LIBC_SRC___SUPPORT_STRING_CONVERTER_H
-#include "hdr/types/char32_t.h"
-#include "hdr/types/char8_t.h"
#include "hdr/types/size_t.h"
#include "src/__support/CPP/type_traits.h"
#include "src/__support/common.h"
diff --git a/libc/src/__support/wchar/wcrtomb.h b/libc/src/__support/wchar/wcrtomb.h
index e7f2776b293a33a..26c9086e2d8adf4 100644
--- a/libc/src/__support/wchar/wcrtomb.h
+++ b/libc/src/__support/wchar/wcrtomb.h
@@ -9,46 +9,23 @@
#ifndef LLVM_LIBC_SRC__SUPPORT_WCHAR_WCRTOMB_H
#define LLVM_LIBC_SRC__SUPPORT_WCHAR_WCRTOMB_H
+#include "hdr/types/size_t.h"
+#include "hdr/types/wchar_t.h"
+#include "src/__support/common.h"
#include "src/__support/error_or.h"
+#include "src/__support/macros/config.h"
#include "src/__support/macros/null_check.h"
#include "src/__support/wchar/character_converter.h"
#include "src/__support/wchar/mbstate.h"
-#include "hdr/errno_macros.h"
-#include "hdr/types/char32_t.h"
-#include "hdr/types/size_t.h"
-#include "hdr/types/wchar_t.h"
-#include "src/__support/common.h"
-#include "src/__support/libc_assert.h"
-
namespace LIBC_NAMESPACE_DECL {
namespace internal {
LIBC_INLINE ErrorOr<size_t> wcrtomb(char *__restrict dest_ptr, wchar_t wc,
- mbstate *__restrict ps) {
- LIBC_CRASH_ON_NULLPTR(dest_ptr);
+ mbstate *__restrict ps) {
LIBC_CRASH_ON_NULLPTR(ps);
- static_assert(sizeof(wchar_t) == 4);
-
- CharacterConverter cr(ps);
-
- if (!cr.isValidState())
- return Error(EINVAL);
-
- int status = cr.push(static_cast<char32_t>(wc));
- if (status != 0)
- return Error(status);
-
- size_t count = 0;
- while (!cr.isEmpty()) {
- auto utf8 = cr.pop_utf8(); // can never fail as long as the push succeeded
- LIBC_ASSERT(utf8.has_value());
-
- *dest_ptr = utf8.value();
- dest_ptr++;
- count++;
- }
- return count;
+ CharacterConverter char_conv(ps);
+ return char_conv.rtomb_generic<wchar_t>(dest_ptr, wc);
}
} // namespace internal
diff --git a/libc/src/__support/wchar/wcsnrtombs.h b/libc/src/__support/wchar/wcsnrtombs.h
index 5faf035145ff699..97786815d69b641 100644
--- a/libc/src/__support/wchar/wcsnrtombs.h
+++ b/libc/src/__support/wchar/wcsnrtombs.h
@@ -9,13 +9,15 @@
#ifndef LLVM_LIBC_SRC__SUPPORT_WCHAR_WCSNRTOMBS_H
#define LLVM_LIBC_SRC__SUPPORT_WCHAR_WCSNRTOMBS_H
-#include "hdr/types/char32_t.h"
+#include "hdr/errno_macros.h"
#include "hdr/types/char8_t.h"
#include "hdr/types/size_t.h"
#include "hdr/types/wchar_t.h"
#include "src/__support/common.h"
+#include "src/__support/error_or.h"
#include "src/__support/macros/config.h"
#include "src/__support/macros/null_check.h"
+#include "src/__support/wchar/character_converter.h"
#include "src/__support/wchar/mbstate.h"
#include "src/__support/wchar/string_converter.h"
@@ -35,9 +37,8 @@ wcsnrtombs(char *__restrict dest, const wchar_t **__restrict ptr_to_src,
if (dest == nullptr)
dest_len = SIZE_MAX;
- StringConverter<char32_t> str_conv(
- reinterpret_cast<const char32_t *>(*ptr_to_src), ps, dest_len,
- num_src_widechars);
+ StringConverter<wchar_t> str_conv(*ptr_to_src, ps, dest_len,
+ num_src_widechars);
size_t dst_idx = 0;
ErrorOr<char8_t> converted = str_conv.pop<char8_t>();
while (converted.has_value()) {
diff --git a/libc/src/wchar/CMakeLists.txt b/libc/src/wchar/CMakeLists.txt
index 938b7ad2e4c737c..40145233b220c6b 100644
--- a/libc/src/wchar/CMakeLists.txt
+++ b/libc/src/wchar/CMakeLists.txt
@@ -149,6 +149,7 @@ add_entrypoint_object(
DEPENDS
libc.hdr.types.wchar_t
libc.hdr.types.mbstate_t
+ libc.include.llvm-libc-macros.limits_macros
libc.src.__support.libc_errno
libc.src.__support.wchar.wcrtomb
libc.src.__support.wchar.mbstate
diff --git a/libc/src/wchar/wcrtomb.cpp b/libc/src/wchar/wcrtomb.cpp
index 3e9df0599431ec5..b1d57f3d9d132c2 100644
--- a/libc/src/wchar/wcrtomb.cpp
+++ b/libc/src/wchar/wcrtomb.cpp
@@ -10,6 +10,7 @@
#include "hdr/types/mbstate_t.h"
#include "hdr/types/wchar_t.h"
+#include "include/llvm-libc-macros/limits-macros.h"
#include "src/__support/common.h"
#include "src/__support/libc_errno.h"
#include "src/__support/macros/config.h"
@@ -23,7 +24,7 @@ LLVM_LIBC_FUNCTION(size_t, wcrtomb,
static internal::mbstate internal_mbstate;
// when s is nullptr, this is equivalent to wcrtomb(buf, L'\0', ps)
- char buf[sizeof(wchar_t) / sizeof(char)];
+ char buf[MB_LEN_MAX];
if (s == nullptr) {
s = buf;
wc = L'\0';
More information about the libc-commits
mailing list