[libc-commits] [libc] POC: add snmalloc as an alternative allocator to libc (PR #122284)
Schrodinger ZHU Yifan via libc-commits
libc-commits at lists.llvm.org
Thu Jan 9 06:59:03 PST 2025
https://github.com/SchrodingerZhu created https://github.com/llvm/llvm-project/pull/122284
None
>From 6f3899a88028f3fdc193e9b90bd47f2e5214e840 Mon Sep 17 00:00:00 2001
From: Schrodinger ZHU Yifan <i at zhuyi.fan>
Date: Thu, 9 Jan 2025 16:24:32 +0800
Subject: [PATCH 1/2] [libc] implement sys/uio/writev
---
libc/config/linux/aarch64/entrypoints.txt | 3 +++
libc/config/linux/x86_64/entrypoints.txt | 3 +++
libc/hdr/types/CMakeLists.txt | 9 +++++++
libc/hdr/types/struct_iovec.h | 21 ++++++++++++++++
libc/include/CMakeLists.txt | 10 ++++++++
libc/include/sys/uio.h.def | 16 +++++++++++++
libc/include/sys/uio.yaml | 17 +++++++++++++
libc/src/sys/CMakeLists.txt | 1 +
libc/src/sys/uio/CMakeLists.txt | 10 ++++++++
libc/src/sys/uio/linux/CMakeLists.txt | 14 +++++++++++
libc/src/sys/uio/linux/writev.cpp | 27 +++++++++++++++++++++
libc/src/sys/uio/writev.h | 22 +++++++++++++++++
libc/test/src/sys/CMakeLists.txt | 1 +
libc/test/src/sys/uio/CMakeLists.txt | 15 ++++++++++++
libc/test/src/sys/uio/writev_test.cpp | 29 +++++++++++++++++++++++
15 files changed, 198 insertions(+)
create mode 100644 libc/hdr/types/struct_iovec.h
create mode 100644 libc/include/sys/uio.h.def
create mode 100644 libc/include/sys/uio.yaml
create mode 100644 libc/src/sys/uio/CMakeLists.txt
create mode 100644 libc/src/sys/uio/linux/CMakeLists.txt
create mode 100644 libc/src/sys/uio/linux/writev.cpp
create mode 100644 libc/src/sys/uio/writev.h
create mode 100644 libc/test/src/sys/uio/CMakeLists.txt
create mode 100644 libc/test/src/sys/uio/writev_test.cpp
diff --git a/libc/config/linux/aarch64/entrypoints.txt b/libc/config/linux/aarch64/entrypoints.txt
index 00f0c6a8bfb8e4..fc2b0e91c1286d 100644
--- a/libc/config/linux/aarch64/entrypoints.txt
+++ b/libc/config/linux/aarch64/entrypoints.txt
@@ -350,6 +350,9 @@ set(TARGET_LIBC_ENTRYPOINTS
# wchar.h entrypoints
libc.src.wchar.wctob
+
+ # sys/uio.h entrypoints
+ libc.src.sys.uio.writev
)
if(LLVM_LIBC_INCLUDE_SCUDO)
diff --git a/libc/config/linux/x86_64/entrypoints.txt b/libc/config/linux/x86_64/entrypoints.txt
index 7e549607716c02..e7b049c0a66388 100644
--- a/libc/config/linux/x86_64/entrypoints.txt
+++ b/libc/config/linux/x86_64/entrypoints.txt
@@ -350,6 +350,9 @@ set(TARGET_LIBC_ENTRYPOINTS
# wchar.h entrypoints
libc.src.wchar.wctob
libc.src.wchar.btowc
+
+ # sys/uio.h entrypoints
+ libc.src.sys.uio.writev
)
if(LLVM_LIBC_INCLUDE_SCUDO)
diff --git a/libc/hdr/types/CMakeLists.txt b/libc/hdr/types/CMakeLists.txt
index 1674de14201524..3dfa38a020fad0 100644
--- a/libc/hdr/types/CMakeLists.txt
+++ b/libc/hdr/types/CMakeLists.txt
@@ -333,3 +333,12 @@ add_proxy_header_library(
FULL_BUILD_DEPENDS
libc.include.llvm-libc-types.uid_t
)
+
+add_proxy_header_library(
+ struct_iovec
+ HDRS
+ struct_iovec.h
+ FULL_BUILD_DEPENDS
+ libc.include.llvm-libc-types.struct_iovec
+ libc.include.sys_uio
+)
diff --git a/libc/hdr/types/struct_iovec.h b/libc/hdr/types/struct_iovec.h
new file mode 100644
index 00000000000000..e34efdb8ddfe4b
--- /dev/null
+++ b/libc/hdr/types/struct_iovec.h
@@ -0,0 +1,21 @@
+//===-- Proxy for struct iovec -------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+#ifndef LLVM_LIBC_HDR_TYPES_STRUCT_FLOCK64_H
+#define LLVM_LIBC_HDR_TYPES_STRUCT_FLOCK64_H
+
+#ifdef LIBC_FULL_BUILD
+
+#include "include/llvm-libc-types/struct_iovec.h"
+
+#else
+
+#include <sys/uio.h>
+
+#endif // LIBC_FULL_BUILD
+
+#endif // LLVM_LIBC_HDR_TYPES_STRUCT_FLOCK64_H
diff --git a/libc/include/CMakeLists.txt b/libc/include/CMakeLists.txt
index 568bb05d923023..e5ceea360d3965 100644
--- a/libc/include/CMakeLists.txt
+++ b/libc/include/CMakeLists.txt
@@ -632,6 +632,16 @@ add_header_macro(
.llvm-libc-types.struct_utsname
)
+add_header_macro(
+ sys_uio
+ ../libc/include/sys/uio.yaml
+ sys/uio.h
+ DEPENDS
+ .llvm_libc_common_h
+ .llvm-libc-types.struct_iovec
+ .llvm-libc-types.ssize_t
+)
+
add_header_macro(
sys_wait
../libc/include/sys/wait.yaml
diff --git a/libc/include/sys/uio.h.def b/libc/include/sys/uio.h.def
new file mode 100644
index 00000000000000..c48aaf91d52f19
--- /dev/null
+++ b/libc/include/sys/uio.h.def
@@ -0,0 +1,16 @@
+//===-- POSIX header auxv.h -----------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef LLVM_LIBC_SYS_UIO_H
+#define LLVM_LIBC_SYS_UIO_H
+
+#include "__llvm-libc-common.h"
+
+%%public_api()
+
+#endif // LLVM_LIBC_SYS_UIO_H
diff --git a/libc/include/sys/uio.yaml b/libc/include/sys/uio.yaml
new file mode 100644
index 00000000000000..808d8ec790198e
--- /dev/null
+++ b/libc/include/sys/uio.yaml
@@ -0,0 +1,17 @@
+header: sys/uio.h
+header_template: uio.h.def
+macros: []
+types:
+ - type_name: struct_iovec
+ - type_name: ssize_t
+enums: []
+objects: []
+functions:
+ - name: writev
+ standards:
+ - POSIX
+ return_type: ssize_t
+ arguments:
+ - type: int
+ - type: const struct iovec *
+ - type: int
diff --git a/libc/src/sys/CMakeLists.txt b/libc/src/sys/CMakeLists.txt
index adc666b94202f7..bb177f11c6d628 100644
--- a/libc/src/sys/CMakeLists.txt
+++ b/libc/src/sys/CMakeLists.txt
@@ -11,3 +11,4 @@ add_subdirectory(statvfs)
add_subdirectory(utsname)
add_subdirectory(wait)
add_subdirectory(prctl)
+add_subdirectory(uio)
diff --git a/libc/src/sys/uio/CMakeLists.txt b/libc/src/sys/uio/CMakeLists.txt
new file mode 100644
index 00000000000000..6298f86cd937da
--- /dev/null
+++ b/libc/src/sys/uio/CMakeLists.txt
@@ -0,0 +1,10 @@
+if(EXISTS ${CMAKE_CURRENT_SOURCE_DIR}/${LIBC_TARGET_OS})
+ add_subdirectory(${CMAKE_CURRENT_SOURCE_DIR}/${LIBC_TARGET_OS})
+endif()
+
+add_entrypoint_object(
+ writev
+ ALIAS
+ DEPENDS
+ .${LIBC_TARGET_OS}.writev
+)
diff --git a/libc/src/sys/uio/linux/CMakeLists.txt b/libc/src/sys/uio/linux/CMakeLists.txt
new file mode 100644
index 00000000000000..85a7a3ae4d5c21
--- /dev/null
+++ b/libc/src/sys/uio/linux/CMakeLists.txt
@@ -0,0 +1,14 @@
+add_entrypoint_object(
+ writev
+ SRCS
+ writev.cpp
+ HDRS
+ ../writev.h
+ DEPENDS
+ libc.include.sys_syscall
+ libc.src.__support.OSUtil.osutil
+ libc.src.__support.common
+ libc.src.errno.errno
+ libc.hdr.types.ssize_t
+ libc.hdr.types.struct_iovec
+)
diff --git a/libc/src/sys/uio/linux/writev.cpp b/libc/src/sys/uio/linux/writev.cpp
new file mode 100644
index 00000000000000..a3bb8986d522ec
--- /dev/null
+++ b/libc/src/sys/uio/linux/writev.cpp
@@ -0,0 +1,27 @@
+//===-- Implementation file for writev ------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+#include "src/sys/uio/writev.h"
+#include "src/__support/OSUtil/syscall.h"
+#include "src/__support/common.h"
+#include "src/errno/libc_errno.h"
+#include <sys/syscall.h>
+
+namespace LIBC_NAMESPACE_DECL {
+
+LLVM_LIBC_FUNCTION(ssize_t, writev, (int fd, const iovec *iov, int iovcnt)) {
+ long ret = LIBC_NAMESPACE::syscall_impl<long>(SYS_writev, fd, iov, iovcnt);
+ // On failure, return -1 and set errno.
+ if (ret < 0) {
+ libc_errno = static_cast<int>(-ret);
+ return -1;
+ }
+ // On success, return number of bytes written.
+ return static_cast<ssize_t>(ret);
+}
+
+} // namespace LIBC_NAMESPACE_DECL
diff --git a/libc/src/sys/uio/writev.h b/libc/src/sys/uio/writev.h
new file mode 100644
index 00000000000000..787bc4b3044b0f
--- /dev/null
+++ b/libc/src/sys/uio/writev.h
@@ -0,0 +1,22 @@
+//===-- Implementation header for writev ----------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef LLVM_LIBC_SRC_SYS_UIO_WRITEV_H
+#define LLVM_LIBC_SRC_SYS_UIO_WRITEV_H
+
+#include "hdr/types/ssize_t.h"
+#include "hdr/types/struct_iovec.h"
+#include "src/__support/macros/config.h"
+
+namespace LIBC_NAMESPACE_DECL {
+
+ssize_t writev(int fd, const iovec *iov, int iovcnt);
+
+} // namespace LIBC_NAMESPACE_DECL
+
+#endif // LLVM_LIBC_SRC_SYS_UIO_WRITEV_H
diff --git a/libc/test/src/sys/CMakeLists.txt b/libc/test/src/sys/CMakeLists.txt
index dc0aa8bf7b75dc..9e9293aab628f4 100644
--- a/libc/test/src/sys/CMakeLists.txt
+++ b/libc/test/src/sys/CMakeLists.txt
@@ -11,3 +11,4 @@ add_subdirectory(wait)
add_subdirectory(prctl)
add_subdirectory(auxv)
add_subdirectory(epoll)
+add_subdirectory(uio)
diff --git a/libc/test/src/sys/uio/CMakeLists.txt b/libc/test/src/sys/uio/CMakeLists.txt
new file mode 100644
index 00000000000000..45f8d14c161792
--- /dev/null
+++ b/libc/test/src/sys/uio/CMakeLists.txt
@@ -0,0 +1,15 @@
+add_custom_target(libc_sys_uio_unittests)
+add_libc_unittest(
+ writev_test
+ SUITE
+ libc_sys_uio_unittests
+ SRCS
+ writev_test.cpp
+ DEPENDS
+ libc.src.errno.errno
+ libc.src.__support.common
+ libc.src.sys.uio.writev
+ libc.src.unistd.close
+ libc.src.fcntl.open
+ libc.test.UnitTest.ErrnoSetterMatcher
+)
diff --git a/libc/test/src/sys/uio/writev_test.cpp b/libc/test/src/sys/uio/writev_test.cpp
new file mode 100644
index 00000000000000..a9a314813182d2
--- /dev/null
+++ b/libc/test/src/sys/uio/writev_test.cpp
@@ -0,0 +1,29 @@
+//===-- Unittests for writev ----------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#include "src/fcntl/open.h"
+#include "src/sys/uio/writev.h"
+#include "src/unistd/close.h"
+#include "test/UnitTest/ErrnoSetterMatcher.h"
+#include "test/UnitTest/Test.h"
+
+using namespace LIBC_NAMESPACE::testing::ErrnoSetterMatcher;
+
+TEST(LlvmLibcSysUioWritevTest, SmokeTest) {
+ int fd = LIBC_NAMESPACE::open("/dev/null", O_WRONLY);
+ ASSERT_THAT(fd, returns(GT(0)).with_errno(EQ(0)));
+ const char *data = "Hello, World!\n";
+ struct iovec iov[2];
+ iov[0].iov_base = const_cast<char *>(data);
+ iov[0].iov_len = 7;
+ iov[1].iov_base = const_cast<char *>(data + 7);
+ iov[1].iov_len = 8;
+ ASSERT_THAT(LIBC_NAMESPACE::writev(fd, iov, 2),
+ returns(EQ(15)).with_errno(EQ(0)));
+ ASSERT_THAT(LIBC_NAMESPACE::close(fd), Succeeds());
+}
>From 29de0ba328913feeebeef1254b7d83f573619b99 Mon Sep 17 00:00:00 2001
From: Schrodinger ZHU Yifan <i at zhuyi.fan>
Date: Thu, 9 Jan 2025 22:57:57 +0800
Subject: [PATCH 2/2] POC: add snmalloc as an alternative allocator to libc
---
libc/include/sys/uio.h.def | 2 +-
libc/src/stdlib/CMakeLists.txt | 41 +++++++++++-
libc/src/stdlib/snmalloc/CMakeLists.txt | 75 ++++++++++++++++++++++
libc/src/stdlib/snmalloc/aligned_alloc.cpp | 0
libc/src/stdlib/snmalloc/calloc.cpp | 0
libc/src/stdlib/snmalloc/free.cpp | 0
libc/src/stdlib/snmalloc/malloc.cpp | 9 +++
libc/src/stdlib/snmalloc/mallopt.cpp | 0
libc/src/stdlib/snmalloc/realloc.cpp | 0
libc/src/stdlib/snmalloc/support.h | 14 ++++
10 files changed, 139 insertions(+), 2 deletions(-)
create mode 100644 libc/src/stdlib/snmalloc/CMakeLists.txt
create mode 100644 libc/src/stdlib/snmalloc/aligned_alloc.cpp
create mode 100644 libc/src/stdlib/snmalloc/calloc.cpp
create mode 100644 libc/src/stdlib/snmalloc/free.cpp
create mode 100644 libc/src/stdlib/snmalloc/malloc.cpp
create mode 100644 libc/src/stdlib/snmalloc/mallopt.cpp
create mode 100644 libc/src/stdlib/snmalloc/realloc.cpp
create mode 100644 libc/src/stdlib/snmalloc/support.h
diff --git a/libc/include/sys/uio.h.def b/libc/include/sys/uio.h.def
index c48aaf91d52f19..5f96c8f7e130d0 100644
--- a/libc/include/sys/uio.h.def
+++ b/libc/include/sys/uio.h.def
@@ -1,4 +1,4 @@
-//===-- POSIX header auxv.h -----------------------------------------------===//
+//===-- POSIX header sys/uio.h --------------------------------------------===//
//
// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
// See https://llvm.org/LICENSE.txt for license information.
diff --git a/libc/src/stdlib/CMakeLists.txt b/libc/src/stdlib/CMakeLists.txt
index 40ba9ead9a7ae6..7d0dfd1aa9da4b 100644
--- a/libc/src/stdlib/CMakeLists.txt
+++ b/libc/src/stdlib/CMakeLists.txt
@@ -324,7 +324,46 @@ add_entrypoint_object(
)
if(NOT LIBC_TARGET_OS_IS_BAREMETAL AND NOT LIBC_TARGET_OS_IS_GPU)
- if(LLVM_LIBC_INCLUDE_SCUDO)
+ if (NOT "${LLVM_LIBC_INCLUDE_SNMALLOC}" STREQUAL "")
+ message(STATUS "Including snmalloc as the allocator, source directory: ${LLVM_LIBC_INCLUDE_SNMALLOC}")
+ add_subdirectory(snmalloc)
+ add_entrypoint_object(
+ malloc
+ ALIAS
+ DEPENDS
+ .snmalloc.malloc
+ )
+ add_entrypoint_object(
+ calloc
+ ALIAS
+ DEPENDS
+ .snmalloc.calloc
+ )
+ add_entrypoint_object(
+ realloc
+ ALIAS
+ DEPENDS
+ .snmalloc.realloc
+ )
+ add_entrypoint_object(
+ aligned_alloc
+ ALIAS
+ DEPENDS
+ .snmalloc.aligned_alloc
+ )
+ add_entrypoint_object(
+ free
+ ALIAS
+ DEPENDS
+ .snmalloc.free
+ )
+ add_entrypoint_object(
+ mallopt
+ ALIAS
+ DEPENDS
+ .snmalloc.mallopt
+ )
+ elseif(LLVM_LIBC_INCLUDE_SCUDO)
set(SCUDO_DEPS "")
include(${LIBC_SOURCE_DIR}/../compiler-rt/cmake/Modules/AllSupportedArchDefs.cmake)
diff --git a/libc/src/stdlib/snmalloc/CMakeLists.txt b/libc/src/stdlib/snmalloc/CMakeLists.txt
new file mode 100644
index 00000000000000..3971ea42db1744
--- /dev/null
+++ b/libc/src/stdlib/snmalloc/CMakeLists.txt
@@ -0,0 +1,75 @@
+set(SNMALLOC_USE_SELF_VENDORED_STL ON CACHE BOOL "use freestanding snmalloc setup" FORCE)
+set(SNMALLOC_BUILD_TESTING OFF CACHE BOOL "disable snmalloc tests" FORCE)
+set(SNMALLOC_HEADER_ONLY_LIBRARY ON CACHE BOOL "use snmalloc as header only library" FORCE)
+
+# Disable installation
+macro (install)
+endmacro ()
+
+add_subdirectory(${LLVM_LIBC_INCLUDE_SNMALLOC} ${CMAKE_CURRENT_BINARY_DIR}/snmalloc EXCLUDE_FROM_ALL)
+
+target_compile_options(
+ snmalloc
+ INTERFACE
+ -ffreestanding
+ -nostdinc
+ -Wno-newline-eof
+ -Wno-extra-semi
+ -Wno-unused-command-line-argument
+ -Wno-ctad-maybe-unsupported
+ # TODO: define this
+ -DSTDERR_FILENO=2
+ -include ${CMAKE_CURRENT_SOURCE_DIR}/support.h
+ # include_directories does not propagate, use options instead
+ -isystem ${COMPILER_RESOURCE_DIR}/include
+ -isystem ${LIBC_INCLUDE_DIR}
+)
+add_dependencies(snmalloc libc-headers)
+
+add_entrypoint_object(
+ malloc
+ SRCS
+ malloc.cpp
+ DEPENDS
+ snmalloc
+)
+
+add_entrypoint_object(
+ calloc
+ SRCS
+ calloc.cpp
+ DEPENDS
+ snmalloc
+)
+
+add_entrypoint_object(
+ realloc
+ SRCS
+ realloc.cpp
+ DEPENDS
+ snmalloc
+)
+
+add_entrypoint_object(
+ aligned_alloc
+ SRCS
+ aligned_alloc.cpp
+ DEPENDS
+ snmalloc
+)
+
+add_entrypoint_object(
+ free
+ SRCS
+ free.cpp
+ DEPENDS
+ snmalloc
+)
+
+add_entrypoint_object(
+ mallopt
+ SRCS
+ mallopt.cpp
+ DEPENDS
+ snmalloc
+)
diff --git a/libc/src/stdlib/snmalloc/aligned_alloc.cpp b/libc/src/stdlib/snmalloc/aligned_alloc.cpp
new file mode 100644
index 00000000000000..e69de29bb2d1d6
diff --git a/libc/src/stdlib/snmalloc/calloc.cpp b/libc/src/stdlib/snmalloc/calloc.cpp
new file mode 100644
index 00000000000000..e69de29bb2d1d6
diff --git a/libc/src/stdlib/snmalloc/free.cpp b/libc/src/stdlib/snmalloc/free.cpp
new file mode 100644
index 00000000000000..e69de29bb2d1d6
diff --git a/libc/src/stdlib/snmalloc/malloc.cpp b/libc/src/stdlib/snmalloc/malloc.cpp
new file mode 100644
index 00000000000000..644f2eef0ed2bf
--- /dev/null
+++ b/libc/src/stdlib/snmalloc/malloc.cpp
@@ -0,0 +1,9 @@
+#include "src/stdlib/malloc.h"
+#include "snmalloc/snmalloc.h"
+#include "src/__support/common.h"
+
+namespace LIBC_NAMESPACE_DECL {
+LLVM_LIBC_FUNCTION(void *, malloc, (size_t size)) {
+ return snmalloc::libc::malloc(size);
+}
+} // namespace LIBC_NAMESPACE_DECL
diff --git a/libc/src/stdlib/snmalloc/mallopt.cpp b/libc/src/stdlib/snmalloc/mallopt.cpp
new file mode 100644
index 00000000000000..e69de29bb2d1d6
diff --git a/libc/src/stdlib/snmalloc/realloc.cpp b/libc/src/stdlib/snmalloc/realloc.cpp
new file mode 100644
index 00000000000000..e69de29bb2d1d6
diff --git a/libc/src/stdlib/snmalloc/support.h b/libc/src/stdlib/snmalloc/support.h
new file mode 100644
index 00000000000000..4909667c1a50d8
--- /dev/null
+++ b/libc/src/stdlib/snmalloc/support.h
@@ -0,0 +1,14 @@
+#include "src/sys/random/getrandom.h"
+
+// TODO: define this
+inline int getentropy(void *buf, size_t size) {
+ while (size > 0) {
+ ssize_t ret = LIBC_NAMESPACE::getrandom(buf, size, 0);
+ if (ret < 0) {
+ return -1;
+ }
+ buf = (char *)buf + ret;
+ size -= ret;
+ }
+ return 0;
+}
More information about the libc-commits
mailing list