[libc-commits] [libc] [libc][arc4random 3/4] implement global and local random state (PR #152617)
Schrodinger ZHU Yifan via libc-commits
libc-commits at lists.llvm.org
Wed Aug 27 07:03:39 PDT 2025
https://github.com/SchrodingerZhu updated https://github.com/llvm/llvm-project/pull/152617
>From 6c46017d9b0b5eb829adfd646f45efed2ba3367e Mon Sep 17 00:00:00 2001
From: Schrodinger ZHU Yifan <yifanzhu at rochester.edu>
Date: Wed, 30 Jul 2025 12:47:24 -0400
Subject: [PATCH 01/12] wip
---
libc/src/__support/aba_ptr.h | 70 ++++++++++++++++++++++++++++++
libc/src/__support/mpmc_stack.h | 75 +++++++++++++++++++++++++++++++++
2 files changed, 145 insertions(+)
create mode 100644 libc/src/__support/aba_ptr.h
create mode 100644 libc/src/__support/mpmc_stack.h
diff --git a/libc/src/__support/aba_ptr.h b/libc/src/__support/aba_ptr.h
new file mode 100644
index 0000000000000..c702aae017502
--- /dev/null
+++ b/libc/src/__support/aba_ptr.h
@@ -0,0 +1,70 @@
+//===-- Transactional Ptr for ABA prevention --------------------*- C++ -*-===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef LLVM_LIBC_SRC___SUPPORT_TAGGED_POINTER_H
+#define LLVM_LIBC_SRC___SUPPORT_TAGGED_POINTER_H
+
+#include "src/__support/common.h"
+#include "src/__support/threads/sleep.h"
+
+#ifdef __GCC_HAVE_SYNC_COMPARE_AND_SWAP_16
+#define LIBC_ABA_PTR_IS_ATOMIC true
+#else
+#define LIBC_ABA_PTR_IS_ATOMIC false
+#endif
+
+namespace LIBC_NAMESPACE_DECL {
+
+template <class T, bool IsAtomic> struct AbaPtrImpl {
+ union Impl {
+ struct alignas(2 * alignof(void *)) Atomic {
+ T *ptr;
+ __SIZE_TYPE__ tag;
+ } atomic;
+ struct Mutex {
+ T *ptr;
+ bool locked;
+ } mutex;
+ } impl;
+
+ LIBC_INLINE constexpr AbaPtrImpl(T *ptr)
+ : impl(IsAtomic ? Impl{.atomic{ptr, 0}} : Impl{.mutex{ptr, false}}) {}
+
+ /// User must guarantee that operation is redoable.
+ template <class Op> LIBC_INLINE void transaction(Op &&op) {
+ if constexpr (IsAtomic) {
+ for (;;) {
+ typename Impl::Atomic snapshot, next;
+ __atomic_load(&impl.atomic, &snapshot, __ATOMIC_RELAXED);
+ next.ptr = op(snapshot.ptr);
+ // Wrapping add for unsigned integers.
+ next.tag = snapshot.tag + 1;
+ if (__atomic_compare_exchange(&impl.atomic, &snapshot, &next, true,
+ __ATOMIC_ACQ_REL, __ATOMIC_RELAXED)) {
+ return;
+ }
+ }
+ } else {
+ // Acquire the lock.
+ while (__atomic_exchange_n(&impl.mutex.locked, true, __ATOMIC_ACQUIRE)) {
+ while (__atomic_load_n(&impl.mutex.locked, __ATOMIC_RELAXED)) {
+ LIBC_NAMESPACE::sleep_briefly();
+ }
+ }
+ impl.mutex.ptr = op(impl.mutex.ptr);
+ // Release the lock.
+ __atomic_store_n(&impl.mutex.locked, false, __ATOMIC_RELEASE);
+ }
+ }
+};
+
+template <class T> using AbaPtr = AbaPtrImpl<T, LIBC_ABA_PTR_IS_ATOMIC>;
+} // namespace LIBC_NAMESPACE_DECL
+
+#undef LIBC_ABA_PTR_IS_ATOMIC
+#endif
diff --git a/libc/src/__support/mpmc_stack.h b/libc/src/__support/mpmc_stack.h
new file mode 100644
index 0000000000000..4892c3926b4b0
--- /dev/null
+++ b/libc/src/__support/mpmc_stack.h
@@ -0,0 +1,75 @@
+//===-- Simple Lock-free MPMC Stack -----------------------------*- C++ -*-===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef LLVM_LIBC_SRC___SUPPORT_MPMC_STACK_H
+#define LLVM_LIBC_SRC___SUPPORT_MPMC_STACK_H
+
+#include "src/__support/CPP/atomic.h"
+#include "src/__support/CPP/new.h"
+#include "src/__support/CPP/optional.h"
+#include "src/__support/aba_ptr.h"
+
+namespace LIBC_NAMESPACE_DECL {
+template <class T> class MPMCStack {
+ struct Node {
+ cpp::Atomic<size_t> visitor;
+ AbaPtr<Node> next;
+ T value;
+
+ LIBC_INLINE Node(T val) : visitor(0), next(nullptr), value(val) {}
+ };
+ AbaPtr<Node> head;
+
+public:
+ static_assert(cpp::is_copy_constructible<T>::value,
+ "T must be copy constructible");
+ LIBC_INLINE MPMCStack() : head(nullptr) {}
+ LIBC_INLINE bool push(T value) {
+ AllocChecker ac;
+ Node *new_node = new Node(value, ac);
+ if (!ac) {
+ return false;
+ }
+ head.transaction([new_node](Node *old_head) {
+ new_node->next = old_head;
+ return new_node;
+ });
+ return true;
+ }
+ LIBC_INLINE cpp::optional<T> pop() {
+ cpp::optional<T> res;
+ Node *node;
+ head.transaction([&](Node *current_head) {
+ if (!current_head) {
+ res = cpp::nullopt;
+ return nullptr;
+ }
+ node = current_head;
+ node->visitor.fetch_add(1);
+ res = node->value;
+ auto next = node->next;
+ node->visitor.fetch_sub(1);
+ return next;
+ });
+ // On a successful transaction, a node is popped by us. So we must delete
+ // it. When we are at here, no one else can acquire
+ // new reference to the node, but we still need to wait until other threads
+ // inside the transaction who may potentially be holding a reference to the
+ // node.
+ if (res) {
+ // Spin until the node is no longer in use.
+ while (node->visitor.load() != 0)
+ LIBC_NAMESPACE::sleep_briefly();
+ delete node;
+ }
+ return res;
+ }
+};
+} // namespace LIBC_NAMESPACE_DECL
+
+#endif
>From 80b6c2445ec60853a7f081c42db53ca3e39c635a Mon Sep 17 00:00:00 2001
From: Schrodinger ZHU Yifan <yifanzhu at rochester.edu>
Date: Wed, 30 Jul 2025 12:52:35 -0400
Subject: [PATCH 02/12] remove some extra braces
---
libc/src/__support/aba_ptr.h | 10 ++++------
libc/src/__support/mpmc_stack.h | 3 +--
2 files changed, 5 insertions(+), 8 deletions(-)
diff --git a/libc/src/__support/aba_ptr.h b/libc/src/__support/aba_ptr.h
index c702aae017502..f7ed601dfbe70 100644
--- a/libc/src/__support/aba_ptr.h
+++ b/libc/src/__support/aba_ptr.h
@@ -45,17 +45,15 @@ template <class T, bool IsAtomic> struct AbaPtrImpl {
// Wrapping add for unsigned integers.
next.tag = snapshot.tag + 1;
if (__atomic_compare_exchange(&impl.atomic, &snapshot, &next, true,
- __ATOMIC_ACQ_REL, __ATOMIC_RELAXED)) {
+ __ATOMIC_ACQ_REL, __ATOMIC_RELAXED))
return;
- }
}
} else {
// Acquire the lock.
- while (__atomic_exchange_n(&impl.mutex.locked, true, __ATOMIC_ACQUIRE)) {
- while (__atomic_load_n(&impl.mutex.locked, __ATOMIC_RELAXED)) {
+ while (__atomic_exchange_n(&impl.mutex.locked, true, __ATOMIC_ACQUIRE))
+ while (__atomic_load_n(&impl.mutex.locked, __ATOMIC_RELAXED))
LIBC_NAMESPACE::sleep_briefly();
- }
- }
+
impl.mutex.ptr = op(impl.mutex.ptr);
// Release the lock.
__atomic_store_n(&impl.mutex.locked, false, __ATOMIC_RELEASE);
diff --git a/libc/src/__support/mpmc_stack.h b/libc/src/__support/mpmc_stack.h
index 4892c3926b4b0..819433f24380c 100644
--- a/libc/src/__support/mpmc_stack.h
+++ b/libc/src/__support/mpmc_stack.h
@@ -32,9 +32,8 @@ template <class T> class MPMCStack {
LIBC_INLINE bool push(T value) {
AllocChecker ac;
Node *new_node = new Node(value, ac);
- if (!ac) {
+ if (!ac)
return false;
- }
head.transaction([new_node](Node *old_head) {
new_node->next = old_head;
return new_node;
>From 79fdf8084e7bbb56d117d4a532453520d1d61fd4 Mon Sep 17 00:00:00 2001
From: Schrodinger ZHU Yifan <i at zhuyi.fan>
Date: Wed, 30 Jul 2025 16:21:43 -0400
Subject: [PATCH 03/12] add tests
---
libc/src/__support/CMakeLists.txt | 24 ++++++++
libc/src/__support/aba_ptr.h | 17 +++++-
libc/src/__support/mpmc_stack.h | 15 ++---
.../integration/src/__support/CMakeLists.txt | 15 +++++
.../src/__support/mpmc_stack_test.cpp | 61 +++++++++++++++++++
5 files changed, 124 insertions(+), 8 deletions(-)
create mode 100644 libc/test/integration/src/__support/mpmc_stack_test.cpp
diff --git a/libc/src/__support/CMakeLists.txt b/libc/src/__support/CMakeLists.txt
index 2196d9e23bba7..c9d89cf6fc286 100644
--- a/libc/src/__support/CMakeLists.txt
+++ b/libc/src/__support/CMakeLists.txt
@@ -398,6 +398,30 @@ add_header_library(
libc.src.__support.macros.attributes
)
+add_header_library(
+ aba_ptr
+ HDRS
+ aba_ptr.h
+ DEPENDS
+ libc.hdr.types.size_t
+ libc.src.__support.common
+ libc.src.__support.threads.sleep
+)
+
+add_header_library(
+ mpmc_stack
+ HDRS
+ mpmc_stack.h
+ DEPENDS
+ libc.src.__support.aba_ptr
+ libc.src.__support.common
+ libc.src.__support.CPP.atomic
+ libc.src.__support.CPP.new
+ libc.src.__support.CPP.optional
+ libc.src.__support.CPP.type_traits
+)
+
+
add_subdirectory(FPUtil)
add_subdirectory(OSUtil)
add_subdirectory(StringUtil)
diff --git a/libc/src/__support/aba_ptr.h b/libc/src/__support/aba_ptr.h
index f7ed601dfbe70..632cc466c295b 100644
--- a/libc/src/__support/aba_ptr.h
+++ b/libc/src/__support/aba_ptr.h
@@ -9,6 +9,7 @@
#ifndef LLVM_LIBC_SRC___SUPPORT_TAGGED_POINTER_H
#define LLVM_LIBC_SRC___SUPPORT_TAGGED_POINTER_H
+#include "hdr/types/size_t.h"
#include "src/__support/common.h"
#include "src/__support/threads/sleep.h"
@@ -24,7 +25,7 @@ template <class T, bool IsAtomic> struct AbaPtrImpl {
union Impl {
struct alignas(2 * alignof(void *)) Atomic {
T *ptr;
- __SIZE_TYPE__ tag;
+ size_t tag;
} atomic;
struct Mutex {
T *ptr;
@@ -59,6 +60,20 @@ template <class T, bool IsAtomic> struct AbaPtrImpl {
__atomic_store_n(&impl.mutex.locked, false, __ATOMIC_RELEASE);
}
}
+
+ LIBC_INLINE T *get() const {
+ if constexpr (IsAtomic) {
+ // Weak micro-architectures typically reguards simultaneous partial word
+ // loading and full word loading as a race condition. While there are
+ // implementations that uses racy read anyway, we still load the whole
+ // word to avoid any complications.
+ typename Impl::Atomic snapshot;
+ __atomic_load(&impl.atomic, &snapshot, __ATOMIC_RELAXED);
+ return snapshot.ptr;
+ } else {
+ return impl.mutex.ptr;
+ }
+ }
};
template <class T> using AbaPtr = AbaPtrImpl<T, LIBC_ABA_PTR_IS_ATOMIC>;
diff --git a/libc/src/__support/mpmc_stack.h b/libc/src/__support/mpmc_stack.h
index 819433f24380c..c6546bac9be9a 100644
--- a/libc/src/__support/mpmc_stack.h
+++ b/libc/src/__support/mpmc_stack.h
@@ -12,13 +12,14 @@
#include "src/__support/CPP/atomic.h"
#include "src/__support/CPP/new.h"
#include "src/__support/CPP/optional.h"
+#include "src/__support/CPP/type_traits.h"
#include "src/__support/aba_ptr.h"
namespace LIBC_NAMESPACE_DECL {
template <class T> class MPMCStack {
struct Node {
cpp::Atomic<size_t> visitor;
- AbaPtr<Node> next;
+ Node *next;
T value;
LIBC_INLINE Node(T val) : visitor(0), next(nullptr), value(val) {}
@@ -31,7 +32,7 @@ template <class T> class MPMCStack {
LIBC_INLINE MPMCStack() : head(nullptr) {}
LIBC_INLINE bool push(T value) {
AllocChecker ac;
- Node *new_node = new Node(value, ac);
+ Node *new_node = new (ac) Node(value);
if (!ac)
return false;
head.transaction([new_node](Node *old_head) {
@@ -41,17 +42,17 @@ template <class T> class MPMCStack {
return true;
}
LIBC_INLINE cpp::optional<T> pop() {
- cpp::optional<T> res;
- Node *node;
- head.transaction([&](Node *current_head) {
+ cpp::optional<T> res = cpp::nullopt;
+ Node *node = nullptr;
+ head.transaction([&](Node *current_head) -> Node * {
if (!current_head) {
res = cpp::nullopt;
return nullptr;
}
node = current_head;
node->visitor.fetch_add(1);
- res = node->value;
- auto next = node->next;
+ res = cpp::optional<T>{node->value};
+ Node *next = node->next;
node->visitor.fetch_sub(1);
return next;
});
diff --git a/libc/test/integration/src/__support/CMakeLists.txt b/libc/test/integration/src/__support/CMakeLists.txt
index b5b6557e8d689..93f54083f3c00 100644
--- a/libc/test/integration/src/__support/CMakeLists.txt
+++ b/libc/test/integration/src/__support/CMakeLists.txt
@@ -2,3 +2,18 @@ add_subdirectory(threads)
if(LIBC_TARGET_OS_IS_GPU)
add_subdirectory(GPU)
endif()
+
+add_libc_integration_test_suite(libc-support-integration-tests)
+
+add_integration_test(
+ mpmc_stack_test
+ SUITE
+ libc-support-integration-tests
+ SRCS
+ mpmc_stack_test.cpp
+ DEPENDS
+ libc.src.__support.mpmc_stack
+ libc.src.__support.threads.thread
+ libc.src.pthread.pthread_create
+ libc.src.pthread.pthread_join
+)
diff --git a/libc/test/integration/src/__support/mpmc_stack_test.cpp b/libc/test/integration/src/__support/mpmc_stack_test.cpp
new file mode 100644
index 0000000000000..3cc8237c9e179
--- /dev/null
+++ b/libc/test/integration/src/__support/mpmc_stack_test.cpp
@@ -0,0 +1,61 @@
+#include "src/__support/CPP/atomic.h"
+#include "src/__support/mpmc_stack.h"
+#include "src/pthread/pthread_create.h"
+#include "src/pthread/pthread_join.h"
+#include "test/IntegrationTest/test.h"
+
+using namespace LIBC_NAMESPACE;
+
+void smoke_test() {
+ MPMCStack<int> stack;
+ for (int i = 0; i <= 100; ++i)
+ if (!stack.push(i))
+ __builtin_trap();
+ for (int i = 100; i >= 0; --i)
+ if (*stack.pop() != i)
+ __builtin_trap();
+ if (stack.pop())
+ __builtin_trap(); // Should be empty now.
+}
+
+void multithread_test() {
+ constexpr static size_t NUM_THREADS = 5;
+ constexpr static size_t NUM_PUSHES = 100;
+ struct State {
+ MPMCStack<size_t> stack;
+ cpp::Atomic<size_t> counter = 0;
+ cpp::Atomic<bool> flags[NUM_PUSHES];
+ } state;
+ pthread_t threads[NUM_THREADS];
+ for (size_t i = 0; i < NUM_THREADS; ++i) {
+ LIBC_NAMESPACE::pthread_create(
+ &threads[i], nullptr,
+ [](void *arg) -> void * {
+ State *state = static_cast<State *>(arg);
+ for (;;) {
+ size_t current = state->counter.fetch_add(1);
+ if (current >= NUM_PUSHES)
+ break;
+ if (!state->stack.push(current))
+ __builtin_trap();
+ }
+ while (auto res = state->stack.pop())
+ state->flags[res.value()].store(true);
+ return nullptr;
+ },
+ &state);
+ }
+ for (pthread_t thread : threads)
+ LIBC_NAMESPACE::pthread_join(thread, nullptr);
+ while (cpp::optional<size_t> res = state.stack.pop())
+ state.flags[res.value()].store(true);
+ for (size_t i = 0; i < NUM_PUSHES; ++i)
+ if (!state.flags[i].load())
+ __builtin_trap();
+}
+
+TEST_MAIN() {
+ smoke_test();
+ multithread_test();
+ return 0;
+}
>From 7bb7c6d3b31a1eccbe30c435a15daf8e0ab37c13 Mon Sep 17 00:00:00 2001
From: Schrodinger ZHU Yifan <yifanzhu at rochester.edu>
Date: Fri, 8 Aug 2025 09:06:31 -0400
Subject: [PATCH 04/12] move changes
---
libc/src/__support/mpmc_stack.h | 36 +++++++++++-
.../src/__support/mpmc_stack_test.cpp | 58 +++++++++++++++++++
2 files changed, 92 insertions(+), 2 deletions(-)
diff --git a/libc/src/__support/mpmc_stack.h b/libc/src/__support/mpmc_stack.h
index c6546bac9be9a..df235c2c1dfac 100644
--- a/libc/src/__support/mpmc_stack.h
+++ b/libc/src/__support/mpmc_stack.h
@@ -12,7 +12,6 @@
#include "src/__support/CPP/atomic.h"
#include "src/__support/CPP/new.h"
#include "src/__support/CPP/optional.h"
-#include "src/__support/CPP/type_traits.h"
#include "src/__support/aba_ptr.h"
namespace LIBC_NAMESPACE_DECL {
@@ -29,7 +28,7 @@ template <class T> class MPMCStack {
public:
static_assert(cpp::is_copy_constructible<T>::value,
"T must be copy constructible");
- LIBC_INLINE MPMCStack() : head(nullptr) {}
+ LIBC_INLINE constexpr MPMCStack() : head(nullptr) {}
LIBC_INLINE bool push(T value) {
AllocChecker ac;
Node *new_node = new (ac) Node(value);
@@ -41,6 +40,39 @@ template <class T> class MPMCStack {
});
return true;
}
+ LIBC_INLINE bool push_all(T values[], size_t count) {
+ struct Guard {
+ size_t count;
+ Node **allocated;
+ LIBC_INLINE Guard(Node *allocated[]) : count(0), allocated(allocated) {}
+ LIBC_INLINE ~Guard() {
+ for (size_t i = 0; i < count; ++i)
+ delete allocated[i];
+ }
+ LIBC_INLINE void add(Node *node) { allocated[count++] = node; }
+ LIBC_INLINE void clear() { count = 0; }
+ };
+ // Variable sized array is a GNU extension.
+ __extension__ Node *allocated[count];
+ {
+ Guard guard(allocated);
+ for (size_t i = 0; i < count; ++i) {
+ AllocChecker ac;
+ Node *new_node = new (ac) Node(values[i]);
+ if (!ac)
+ return false;
+ guard.add(new_node);
+ if (i != 0)
+ new_node->next = allocated[i - 1];
+ }
+ guard.clear();
+ }
+ head.transaction([&allocated, count](Node *old_head) {
+ allocated[0]->next = old_head;
+ return allocated[count - 1];
+ });
+ return true;
+ }
LIBC_INLINE cpp::optional<T> pop() {
cpp::optional<T> res = cpp::nullopt;
Node *node = nullptr;
diff --git a/libc/test/integration/src/__support/mpmc_stack_test.cpp b/libc/test/integration/src/__support/mpmc_stack_test.cpp
index 3cc8237c9e179..9166a816a74fe 100644
--- a/libc/test/integration/src/__support/mpmc_stack_test.cpp
+++ b/libc/test/integration/src/__support/mpmc_stack_test.cpp
@@ -54,8 +54,66 @@ void multithread_test() {
__builtin_trap();
}
+void multithread_push_all_test() {
+ constexpr static size_t NUM_THREADS = 4;
+ constexpr static size_t BATCH_SIZE = 10;
+ constexpr static size_t NUM_BATCHES = 20;
+ struct State {
+ MPMCStack<size_t> stack;
+ cpp::Atomic<size_t> counter = 0;
+ cpp::Atomic<bool> flags[NUM_THREADS * BATCH_SIZE * NUM_BATCHES];
+ } state;
+ pthread_t threads[NUM_THREADS];
+
+ for (size_t i = 0; i < NUM_THREADS; ++i) {
+ LIBC_NAMESPACE::pthread_create(
+ &threads[i], nullptr,
+ [](void *arg) -> void * {
+ State *state = static_cast<State *>(arg);
+ size_t values[BATCH_SIZE];
+
+ for (size_t batch = 0; batch < NUM_BATCHES; ++batch) {
+ // Prepare batch of values
+ for (size_t j = 0; j < BATCH_SIZE; ++j) {
+ size_t current = state->counter.fetch_add(1);
+ values[j] = current;
+ }
+
+ // Push all values in batch
+ if (!state->stack.push_all(values, BATCH_SIZE))
+ __builtin_trap();
+ }
+
+ // Pop and mark all values
+ while (auto res = state->stack.pop()) {
+ size_t value = res.value();
+ if (value < NUM_THREADS * BATCH_SIZE * NUM_BATCHES)
+ state->flags[value].store(true);
+ }
+ return nullptr;
+ },
+ &state);
+ }
+
+ for (pthread_t thread : threads)
+ LIBC_NAMESPACE::pthread_join(thread, nullptr);
+
+ // Pop any remaining values
+ while (cpp::optional<size_t> res = state.stack.pop()) {
+ size_t value = res.value();
+ if (value < NUM_THREADS * BATCH_SIZE * NUM_BATCHES)
+ state.flags[value].store(true);
+ }
+
+ // Verify all values were processed
+ for (size_t i = 0; i < NUM_THREADS * BATCH_SIZE * NUM_BATCHES; ++i)
+ if (!state.flags[i].load())
+ __builtin_trap();
+}
+
TEST_MAIN() {
smoke_test();
multithread_test();
+ multithread_push_all_test();
return 0;
}
>From 436947e3ca6000918e183cead706ee75d7f8bcc8 Mon Sep 17 00:00:00 2001
From: Schrodinger ZHU Yifan <yifanzhu at rochester.edu>
Date: Fri, 8 Aug 2025 09:22:07 -0400
Subject: [PATCH 05/12] Update libc/src/__support/aba_ptr.h
Co-authored-by: Copilot <175728472+Copilot at users.noreply.github.com>
---
libc/src/__support/aba_ptr.h | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/libc/src/__support/aba_ptr.h b/libc/src/__support/aba_ptr.h
index 632cc466c295b..3e55aea8875a3 100644
--- a/libc/src/__support/aba_ptr.h
+++ b/libc/src/__support/aba_ptr.h
@@ -63,7 +63,7 @@ template <class T, bool IsAtomic> struct AbaPtrImpl {
LIBC_INLINE T *get() const {
if constexpr (IsAtomic) {
- // Weak micro-architectures typically reguards simultaneous partial word
+ // Weak micro-architectures typically regards simultaneous partial word
// loading and full word loading as a race condition. While there are
// implementations that uses racy read anyway, we still load the whole
// word to avoid any complications.
>From 7d58c0859c6bebca523d33eec86b604b5e60e585 Mon Sep 17 00:00:00 2001
From: Schrodinger ZHU Yifan <i at zhuyi.fan>
Date: Thu, 7 Aug 2025 20:42:25 -0400
Subject: [PATCH 06/12] WIP
---
libc/src/stdlib/linux/vsdo_rng.h | 129 +++++++++++++++++++++++++++++++
1 file changed, 129 insertions(+)
create mode 100644 libc/src/stdlib/linux/vsdo_rng.h
diff --git a/libc/src/stdlib/linux/vsdo_rng.h b/libc/src/stdlib/linux/vsdo_rng.h
new file mode 100644
index 0000000000000..cd4b955b23518
--- /dev/null
+++ b/libc/src/stdlib/linux/vsdo_rng.h
@@ -0,0 +1,129 @@
+//===-- vDSO based RNG ----------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef LIBC_SRC_STDLIB_LINUX_VSDO_RNG_H
+#define LIBC_SRC_STDLIB_LINUX_VSDO_RNG_H
+
+#include "src/__support/CPP/bit.h"
+#include "src/__support/CPP/mutex.h"
+#include "src/__support/OSUtil/linux/vdso.h"
+#include "src/__support/OSUtil/syscall.h"
+#include "src/__support/blockstore.h"
+#include "src/__support/common.h"
+#include "src/__support/macros/config.h"
+#include "src/__support/mpmc_stack.h"
+#include "src/__support/threads/callonce.h"
+#include "src/__support/threads/linux/raw_mutex.h"
+#include "src/sys/auxv/getauxval.h"
+
+namespace LIBC_NAMESPACE_DECL {
+namespace vsdo_rng {
+class GlobalState {
+public:
+ struct VGetrandomOpaqueParams {
+ unsigned int size_of_opaque_states;
+ unsigned int mmap_prot;
+ unsigned int mmap_flags;
+ unsigned int reserved[13];
+ };
+
+private:
+ struct Config {
+ size_t page_size;
+ size_t pages_per_alloc;
+ size_t states_per_page;
+ vdso::VDSOSymType<vdso::VDSOSym::GetRandom> getrandom;
+ VGetrandomOpaqueParams params;
+ };
+
+ // A lock-free stack of free opaque states.
+ MPMCStack<void *> free_list{};
+ // A mutex protecting the allocation of new pages.
+ RawMutex allocation_mutex{};
+ // A block store of allocated pages.
+ BlockStore<void *, 16> allocations{};
+
+ // Shared global configuration.
+ static CallOnceFlag config_flag;
+ static Config config;
+
+ // We grow the states by the number of CPUs. This function uses
+ // SYS_sched_getaffinity to get the number of CPUs.
+ LIBC_INLINE static size_t cpu_count();
+
+ // Grow available states. This function can fail if the system is out of
+ // memory.
+ LIBC_INLINE bool grow();
+
+public:
+ LIBC_INLINE constexpr GlobalState() {}
+ LIBC_INLINE static Config &get_config();
+ LIBC_INLINE ~GlobalState() {}
+};
+
+class LocalState {};
+
+LIBC_INLINE_VAR GlobalState::Config GlobalState::config{};
+LIBC_INLINE_VAR CallOnceFlag GlobalState::config_flag = 0;
+
+LIBC_INLINE size_t GlobalState::cpu_count() {
+ char cpu_set[128] = {0};
+ int res = LIBC_NAMESPACE::syscall_impl<int>(SYS_sched_getaffinity, 0,
+ sizeof(cpu_set), cpu_set);
+ if (res <= 0)
+ return 1;
+
+ size_t count = 0;
+ for (size_t i = 0; i < sizeof(cpu_set) / sizeof(unsigned long); ++i) {
+ unsigned long *mask_ptr = reinterpret_cast<unsigned long *>(cpu_set);
+ count += LIBC_NAMESPACE::cpp::popcount(mask_ptr[i]);
+ }
+
+ return count > 0 ? count : 1;
+}
+
+LIBC_INLINE GlobalState::Config &GlobalState::get_config() {
+ callonce(&config_flag, []() {
+ config.getrandom =
+ LIBC_NAMESPACE::vdso::TypedSymbol<vdso::VDSOSym::GetRandom>{};
+ if (!config.getrandom)
+ return;
+
+ // Call with special flag to get the desired configuration.
+ int res = config.getrandom(
+ /*buf=*/nullptr, /*count=*/0, /*flags=*/0,
+ /*opaque_states=*/&config.params,
+ /*size_of_opaque_states=*/~0);
+ if (res != 0)
+ return;
+
+ config.page_size = LIBC_NAMESPACE::getauxval(AT_PAGESZ);
+ if (!config.page_size)
+ return;
+
+ size_t count = cpu_count();
+
+ config.states_per_page =
+ config.page_size / config.params.size_of_opaque_states;
+
+ config.pages_per_alloc =
+ count / config.states_per_page + (count % config.states_per_page != 0);
+ });
+ return config;
+}
+
+LIBC_INLINE bool GlobalState::grow() {
+ // reserve a slot for the new page.
+ if (!allocations.push_back(nullptr))
+ return false;
+}
+
+} // namespace vsdo_rng
+} // namespace LIBC_NAMESPACE_DECL
+
+#endif // LIBC_SRC_STDLIB_LINUX_VSDO_RNG_H
>From 08c8af52fadd25ac398b5d39ad5c6d0dd59bd207 Mon Sep 17 00:00:00 2001
From: Schrodinger ZHU Yifan <i at zhuyi.fan>
Date: Fri, 8 Aug 2025 00:52:36 -0400
Subject: [PATCH 07/12] finish the state pool
---
libc/src/stdlib/linux/CMakeLists.txt | 20 ++
libc/src/stdlib/linux/vsdo_rng.h | 178 ++++++++++++++++--
.../src/stdlib/linux/CMakeLists.txt | 14 ++
.../src/stdlib/linux/vsdo_rng_test.cpp | 61 ++++++
4 files changed, 259 insertions(+), 14 deletions(-)
create mode 100644 libc/test/integration/src/stdlib/linux/CMakeLists.txt
create mode 100644 libc/test/integration/src/stdlib/linux/vsdo_rng_test.cpp
diff --git a/libc/src/stdlib/linux/CMakeLists.txt b/libc/src/stdlib/linux/CMakeLists.txt
index 1d3c00a5e0ddb..c2ecde3d658ba 100644
--- a/libc/src/stdlib/linux/CMakeLists.txt
+++ b/libc/src/stdlib/linux/CMakeLists.txt
@@ -9,3 +9,23 @@ add_entrypoint_object(
libc.src.signal.raise
libc.src.stdlib._Exit
)
+
+add_header_library(
+ vsdo_rng
+ HDRS
+ vsdo_rng.h
+ DEPENDS
+ libc.src.__support.threads.thread # For __cxa_thread_atexit_impl
+ libc.src.__support.CPP.algorithm
+ libc.src.__support.CPP.bit
+ libc.src.__support.CPP.mutex
+ libc.src.__support.CPP.optional
+ libc.src.__support.OSUtil.linux.vdso
+ libc.src.__support.OSUtil.osutil
+ libc.src.__support.macros.config
+ libc.src.__support.mpmc_stack
+ libc.src.__support.threads.callonce
+ libc.src.__support.threads.linux.raw_mutex
+ libc.src.sys.auxv.getauxval
+ libc.include.sys_syscall
+)
diff --git a/libc/src/stdlib/linux/vsdo_rng.h b/libc/src/stdlib/linux/vsdo_rng.h
index cd4b955b23518..9e1f8fab413b9 100644
--- a/libc/src/stdlib/linux/vsdo_rng.h
+++ b/libc/src/stdlib/linux/vsdo_rng.h
@@ -13,7 +13,6 @@
#include "src/__support/CPP/mutex.h"
#include "src/__support/OSUtil/linux/vdso.h"
#include "src/__support/OSUtil/syscall.h"
-#include "src/__support/blockstore.h"
#include "src/__support/common.h"
#include "src/__support/macros/config.h"
#include "src/__support/mpmc_stack.h"
@@ -23,6 +22,11 @@
namespace LIBC_NAMESPACE_DECL {
namespace vsdo_rng {
+extern "C" {
+using Destructor = void(void *);
+[[gnu::weak]] extern void *__dso_handle;
+int __cxa_thread_atexit_impl(Destructor *, void *, void *);
+}
class GlobalState {
public:
struct VGetrandomOpaqueParams {
@@ -32,7 +36,6 @@ class GlobalState {
unsigned int reserved[13];
};
-private:
struct Config {
size_t page_size;
size_t pages_per_alloc;
@@ -41,12 +44,11 @@ class GlobalState {
VGetrandomOpaqueParams params;
};
+private:
// A lock-free stack of free opaque states.
MPMCStack<void *> free_list{};
// A mutex protecting the allocation of new pages.
RawMutex allocation_mutex{};
- // A block store of allocated pages.
- BlockStore<void *, 16> allocations{};
// Shared global configuration.
static CallOnceFlag config_flag;
@@ -58,15 +60,79 @@ class GlobalState {
// Grow available states. This function can fail if the system is out of
// memory.
- LIBC_INLINE bool grow();
+ // - This routine assumes that the global config is valid.
+ // - On success, this routine returns one opaque state for direct use.
+ LIBC_INLINE void *grow();
public:
LIBC_INLINE constexpr GlobalState() {}
- LIBC_INLINE static Config &get_config();
- LIBC_INLINE ~GlobalState() {}
+ LIBC_INLINE static const Config &get_config();
+ LIBC_INLINE static const Config &get_config_unchecked() { return config; }
+ LIBC_INLINE void *get();
+ LIBC_INLINE void recycle(void *state);
};
-class LocalState {};
+LIBC_INLINE_VAR GlobalState global_state{};
+
+class LocalState {
+ bool in_flight = false;
+ bool failed = false;
+ void *state = nullptr;
+
+public:
+ struct Guard {
+ LocalState *tls;
+ LIBC_INLINE Guard(LocalState *tls) : tls(tls) {
+ tls->in_flight = true;
+ cpp::atomic_thread_fence(cpp::MemoryOrder::SEQ_CST);
+ }
+ LIBC_INLINE Guard(Guard &&other) : tls(other.tls) { other.tls = nullptr; }
+ LIBC_INLINE ~Guard() {
+ cpp::atomic_thread_fence(cpp::MemoryOrder::SEQ_CST);
+ if (tls)
+ tls->in_flight = false;
+ }
+ LIBC_INLINE void fill(void *buf, size_t size) const;
+ };
+ LIBC_INLINE constexpr LocalState() {}
+ LIBC_INLINE cpp::optional<Guard> get() {
+ if (in_flight)
+ return cpp::nullopt;
+
+ Guard guard(this);
+
+ if (!failed && !state) {
+ int register_res = __cxa_thread_atexit_impl(
+ [](void *self) {
+ auto *tls = static_cast<LocalState *>(self);
+ // Reject all future attempts to get a state.
+ void *state = tls->state;
+ tls->in_flight = true;
+ tls->failed = true;
+ tls->state = nullptr;
+ cpp::atomic_thread_fence(cpp::MemoryOrder::SEQ_CST);
+ if (state)
+ LIBC_NAMESPACE::vsdo_rng::global_state.recycle(state);
+ },
+ this, __dso_handle);
+ if (register_res == 0)
+ state = LIBC_NAMESPACE::vsdo_rng::global_state.get();
+ if (!state)
+ failed = true;
+ }
+
+ if (!state)
+ return cpp::nullopt;
+
+ return cpp::move(guard);
+ }
+};
+
+LIBC_INLINE_VAR LIBC_THREAD_LOCAL LocalState local_state{};
+
+//===----------------------------------------------------------------------===//
+// Implementation
+//===----------------------------------------------------------------------===//
LIBC_INLINE_VAR GlobalState::Config GlobalState::config{};
LIBC_INLINE_VAR CallOnceFlag GlobalState::config_flag = 0;
@@ -87,7 +153,7 @@ LIBC_INLINE size_t GlobalState::cpu_count() {
return count > 0 ? count : 1;
}
-LIBC_INLINE GlobalState::Config &GlobalState::get_config() {
+LIBC_INLINE const GlobalState::Config &GlobalState::get_config() {
callonce(&config_flag, []() {
config.getrandom =
LIBC_NAMESPACE::vdso::TypedSymbol<vdso::VDSOSym::GetRandom>{};
@@ -106,7 +172,7 @@ LIBC_INLINE GlobalState::Config &GlobalState::get_config() {
if (!config.page_size)
return;
- size_t count = cpu_count();
+ size_t count = cpp::max(cpu_count(), size_t{4});
config.states_per_page =
config.page_size / config.params.size_of_opaque_states;
@@ -117,10 +183,94 @@ LIBC_INLINE GlobalState::Config &GlobalState::get_config() {
return config;
}
-LIBC_INLINE bool GlobalState::grow() {
- // reserve a slot for the new page.
- if (!allocations.push_back(nullptr))
- return false;
+LIBC_INLINE void *GlobalState::grow() {
+ cpp::lock_guard guard(allocation_mutex);
+
+ // It is possible that when we finally grab the lock, other threads have
+ // successfully finished the allocation already. Hence, we first try if we
+ // can pop anything from the free list.
+ if (cpp::optional<void *> state = free_list.pop())
+ return *state;
+
+ long mmap_res = LIBC_NAMESPACE::syscall_impl<long>(
+ SYS_mmap, /*addr=*/nullptr,
+ /*length=*/config.page_size * config.pages_per_alloc,
+ /*prot=*/config.params.mmap_prot,
+ /*flags=*/config.params.mmap_flags,
+ /*fd=*/-1, /*offset=*/0);
+ if (mmap_res == -1 /* MAP_FAILED */)
+ return nullptr;
+
+ char *pages = reinterpret_cast<char *>(mmap_res);
+
+ // Initialize the page.
+ size_t total_states = config.pages_per_alloc * config.states_per_page;
+ size_t free_states = total_states - 1; // reserve one for direct use.
+ __extension__ void *opaque_states[total_states];
+ size_t index = 0;
+ for (size_t p = 0; p < config.pages_per_alloc; ++p) {
+ char *page = &pages[p * config.page_size];
+ for (size_t s = 0; s < config.states_per_page; ++s) {
+ void *state = &page[s * config.params.size_of_opaque_states];
+ opaque_states[index++] = state;
+ }
+ }
+
+ constexpr size_t RETRY_COUNT = 64;
+ for (size_t i = 0; i < RETRY_COUNT; ++i) {
+ if (free_list.push_all(opaque_states, free_states))
+ break;
+ // Abort if we are still short in memory after all these retries.
+ if (i + 1 == RETRY_COUNT) {
+ LIBC_NAMESPACE::syscall_impl<long>(
+ SYS_munmap, pages, config.page_size * config.pages_per_alloc);
+ return nullptr;
+ }
+ }
+
+ return opaque_states[free_states];
+}
+
+LIBC_INLINE void *GlobalState::get() {
+ const Config &config = get_config();
+ // If page size is not set, the global config is invalid. Early return.
+ if (!config.page_size)
+ return nullptr;
+
+ if (cpp::optional<void *> state = free_list.pop())
+ return *state;
+
+ // At this stage, we know that the config is valid.
+ return grow();
+}
+
+LIBC_INLINE void GlobalState::recycle(void *state) {
+ LIBC_ASSERT(state != nullptr);
+ constexpr size_t RETRY_COUNT = 64;
+ for (size_t i = 0; i < RETRY_COUNT; ++i)
+ if (free_list.push(state))
+ return;
+ // Otherwise, we just let it leak. It won't be too bad not to reuse the state
+ // since the OS can free the page if memory is tight.
+}
+
+//===----------------------------------------------------------------------===//
+// LocalState
+//===----------------------------------------------------------------------===//
+
+LIBC_INLINE void LocalState::Guard::fill(void *buf, size_t size) const {
+ LIBC_ASSERT(tls->state != nullptr);
+ char *cursor = reinterpret_cast<char *>(buf);
+ size_t remaining = size;
+ const auto &config = GlobalState::get_config_unchecked();
+ while (remaining > 0) {
+ int res = config.getrandom(cursor, remaining, /* default random flag */ 0,
+ tls->state, config.params.size_of_opaque_states);
+ if (res < 0)
+ continue;
+ remaining -= static_cast<size_t>(res);
+ cursor += res;
+ }
}
} // namespace vsdo_rng
diff --git a/libc/test/integration/src/stdlib/linux/CMakeLists.txt b/libc/test/integration/src/stdlib/linux/CMakeLists.txt
new file mode 100644
index 0000000000000..5afe67946cced
--- /dev/null
+++ b/libc/test/integration/src/stdlib/linux/CMakeLists.txt
@@ -0,0 +1,14 @@
+add_custom_target(stdlib-linux-integration-tests)
+add_dependencies(libc-integration-tests stdlib-linux-integration-tests)
+
+add_integration_test(
+ vsdo_rng_test
+ SUITE
+ stdlib-linux-integration-tests
+ SRCS
+ vsdo_rng_test.cpp
+ DEPENDS
+ libc.src.pthread.pthread_create
+ libc.src.pthread.pthread_join
+ libc.src.stdlib.linux.vsdo_rng
+)
diff --git a/libc/test/integration/src/stdlib/linux/vsdo_rng_test.cpp b/libc/test/integration/src/stdlib/linux/vsdo_rng_test.cpp
new file mode 100644
index 0000000000000..673ee36a590b1
--- /dev/null
+++ b/libc/test/integration/src/stdlib/linux/vsdo_rng_test.cpp
@@ -0,0 +1,61 @@
+//===-- Test for vsdo_rng functionality ----------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#include "src/pthread/pthread_create.h"
+#include "src/pthread/pthread_join.h"
+#include "src/stdlib/linux/vsdo_rng.h"
+#include "test/IntegrationTest/test.h"
+
+using namespace LIBC_NAMESPACE;
+
+void basic_test() {
+ // Test basic functionality
+ vsdo_rng::LocalState &local_state = vsdo_rng::local_state;
+
+ // Try to get a guard
+ if (auto guard = local_state.get()) {
+ // Fill a small buffer with random data
+ long long buffer[32] = {0};
+ guard->fill(buffer, sizeof(buffer));
+
+ // Basic sanity check - buffer should not have zero
+ for (auto &i : buffer)
+ if (i == 0)
+ __builtin_trap();
+ }
+ // If we can't get a guard, that's okay - the vDSO might not be available
+ // or the system might not support getrandom
+}
+
+void multithread_test() {
+ constexpr static size_t OUTER_REPEAT = 8;
+ constexpr static size_t INNER_REPEAT = 32;
+ constexpr static size_t NUM_THREADS = 16;
+ pthread_t threads[NUM_THREADS];
+
+ // Repeat outer loop so that
+ for (size_t r = 0; r < OUTER_REPEAT; ++r) {
+ for (pthread_t &thread : threads)
+ LIBC_NAMESPACE::pthread_create(
+ &thread, nullptr,
+ [](void *) -> void * {
+ for (size_t j = 0; j < INNER_REPEAT; ++j)
+ basic_test();
+ return nullptr;
+ },
+ nullptr);
+ for (pthread_t thread : threads)
+ LIBC_NAMESPACE::pthread_join(thread, nullptr);
+ }
+}
+
+TEST_MAIN() {
+ basic_test();
+ multithread_test();
+ return 0;
+}
>From 3e205970b767ce692a0a3df2a5031e45a8376668 Mon Sep 17 00:00:00 2001
From: Schrodinger ZHU Yifan <i at zhuyi.fan>
Date: Fri, 8 Aug 2025 00:58:36 -0400
Subject: [PATCH 08/12] update test
---
.../integration/src/stdlib/linux/vsdo_rng_test.cpp | 13 ++++++++++---
1 file changed, 10 insertions(+), 3 deletions(-)
diff --git a/libc/test/integration/src/stdlib/linux/vsdo_rng_test.cpp b/libc/test/integration/src/stdlib/linux/vsdo_rng_test.cpp
index 673ee36a590b1..1476b0931e3f9 100644
--- a/libc/test/integration/src/stdlib/linux/vsdo_rng_test.cpp
+++ b/libc/test/integration/src/stdlib/linux/vsdo_rng_test.cpp
@@ -20,13 +20,20 @@ void basic_test() {
// Try to get a guard
if (auto guard = local_state.get()) {
// Fill a small buffer with random data
- long long buffer[32] = {0};
+ char buffer[256]{};
guard->fill(buffer, sizeof(buffer));
- // Basic sanity check - buffer should not have zero
+ // Basic sanity check - count zero bytes.
+ // With 256 bytes, getting more than ~10 zero bytes would be suspicious
+ size_t zero_count = 0;
for (auto &i : buffer)
if (i == 0)
- __builtin_trap();
+ zero_count++;
+
+ // With uniform distribution, expect ~1 zero byte per 256 bytes
+ // Having more than 16 zero bytes in 256 bytes is very unlikely
+ if (zero_count > 16)
+ __builtin_trap();
}
// If we can't get a guard, that's okay - the vDSO might not be available
// or the system might not support getrandom
>From da8041f7b5a0fc3531bb2fb56d172738c9fdc28d Mon Sep 17 00:00:00 2001
From: Schrodinger ZHU Yifan <yifanzhu at rochester.edu>
Date: Fri, 8 Aug 2025 09:15:24 -0400
Subject: [PATCH 09/12] add some comment
---
libc/src/stdlib/linux/CMakeLists.txt | 2 +-
libc/src/stdlib/linux/vsdo_rng.h | 1 +
2 files changed, 2 insertions(+), 1 deletion(-)
diff --git a/libc/src/stdlib/linux/CMakeLists.txt b/libc/src/stdlib/linux/CMakeLists.txt
index c2ecde3d658ba..8bca9e726105b 100644
--- a/libc/src/stdlib/linux/CMakeLists.txt
+++ b/libc/src/stdlib/linux/CMakeLists.txt
@@ -26,6 +26,6 @@ add_header_library(
libc.src.__support.mpmc_stack
libc.src.__support.threads.callonce
libc.src.__support.threads.linux.raw_mutex
- libc.src.sys.auxv.getauxval
+ libc.src.sys.auxv.getauxval # TODO: remove public entrypoint dependency
libc.include.sys_syscall
)
diff --git a/libc/src/stdlib/linux/vsdo_rng.h b/libc/src/stdlib/linux/vsdo_rng.h
index 9e1f8fab413b9..023a125652e07 100644
--- a/libc/src/stdlib/linux/vsdo_rng.h
+++ b/libc/src/stdlib/linux/vsdo_rng.h
@@ -18,6 +18,7 @@
#include "src/__support/mpmc_stack.h"
#include "src/__support/threads/callonce.h"
#include "src/__support/threads/linux/raw_mutex.h"
+// TODO: this is public entrypoint, we should remove it later on.
#include "src/sys/auxv/getauxval.h"
namespace LIBC_NAMESPACE_DECL {
>From 1664b10999b13e34e170947aa71a8ebe0f0af3ea Mon Sep 17 00:00:00 2001
From: Schrodinger ZHU Yifan <yifanzhu at rochester.edu>
Date: Fri, 8 Aug 2025 09:33:22 -0400
Subject: [PATCH 10/12] fix typo
---
libc/src/stdlib/linux/CMakeLists.txt | 4 ++--
libc/src/stdlib/linux/{vsdo_rng.h => vdso_rng.h} | 14 +++++++-------
.../integration/src/stdlib/linux/CMakeLists.txt | 6 +++---
.../linux/{vsdo_rng_test.cpp => vdso_rng_test.cpp} | 6 +++---
4 files changed, 15 insertions(+), 15 deletions(-)
rename libc/src/stdlib/linux/{vsdo_rng.h => vdso_rng.h} (96%)
rename libc/test/integration/src/stdlib/linux/{vsdo_rng_test.cpp => vdso_rng_test.cpp} (92%)
diff --git a/libc/src/stdlib/linux/CMakeLists.txt b/libc/src/stdlib/linux/CMakeLists.txt
index 8bca9e726105b..85ad2bdab33ba 100644
--- a/libc/src/stdlib/linux/CMakeLists.txt
+++ b/libc/src/stdlib/linux/CMakeLists.txt
@@ -11,9 +11,9 @@ add_entrypoint_object(
)
add_header_library(
- vsdo_rng
+ vdso_rng
HDRS
- vsdo_rng.h
+ vdso_rng.h
DEPENDS
libc.src.__support.threads.thread # For __cxa_thread_atexit_impl
libc.src.__support.CPP.algorithm
diff --git a/libc/src/stdlib/linux/vsdo_rng.h b/libc/src/stdlib/linux/vdso_rng.h
similarity index 96%
rename from libc/src/stdlib/linux/vsdo_rng.h
rename to libc/src/stdlib/linux/vdso_rng.h
index 023a125652e07..fdfa5b1872561 100644
--- a/libc/src/stdlib/linux/vsdo_rng.h
+++ b/libc/src/stdlib/linux/vdso_rng.h
@@ -6,8 +6,8 @@
//
//===----------------------------------------------------------------------===//
-#ifndef LIBC_SRC_STDLIB_LINUX_VSDO_RNG_H
-#define LIBC_SRC_STDLIB_LINUX_VSDO_RNG_H
+#ifndef LIBC_SRC_STDLIB_LINUX_VDSO_RNG_H
+#define LIBC_SRC_STDLIB_LINUX_VDSO_RNG_H
#include "src/__support/CPP/bit.h"
#include "src/__support/CPP/mutex.h"
@@ -22,7 +22,7 @@
#include "src/sys/auxv/getauxval.h"
namespace LIBC_NAMESPACE_DECL {
-namespace vsdo_rng {
+namespace vdso_rng {
extern "C" {
using Destructor = void(void *);
[[gnu::weak]] extern void *__dso_handle;
@@ -113,11 +113,11 @@ class LocalState {
tls->state = nullptr;
cpp::atomic_thread_fence(cpp::MemoryOrder::SEQ_CST);
if (state)
- LIBC_NAMESPACE::vsdo_rng::global_state.recycle(state);
+ LIBC_NAMESPACE::vdso_rng::global_state.recycle(state);
},
this, __dso_handle);
if (register_res == 0)
- state = LIBC_NAMESPACE::vsdo_rng::global_state.get();
+ state = LIBC_NAMESPACE::vdso_rng::global_state.get();
if (!state)
failed = true;
}
@@ -274,7 +274,7 @@ LIBC_INLINE void LocalState::Guard::fill(void *buf, size_t size) const {
}
}
-} // namespace vsdo_rng
+} // namespace vdso_rng
} // namespace LIBC_NAMESPACE_DECL
-#endif // LIBC_SRC_STDLIB_LINUX_VSDO_RNG_H
+#endif // LIBC_SRC_STDLIB_LINUX_VDSO_RNG_H
diff --git a/libc/test/integration/src/stdlib/linux/CMakeLists.txt b/libc/test/integration/src/stdlib/linux/CMakeLists.txt
index 5afe67946cced..c81ce09f67456 100644
--- a/libc/test/integration/src/stdlib/linux/CMakeLists.txt
+++ b/libc/test/integration/src/stdlib/linux/CMakeLists.txt
@@ -2,13 +2,13 @@ add_custom_target(stdlib-linux-integration-tests)
add_dependencies(libc-integration-tests stdlib-linux-integration-tests)
add_integration_test(
- vsdo_rng_test
+ vdso_rng_test
SUITE
stdlib-linux-integration-tests
SRCS
- vsdo_rng_test.cpp
+ vdso_rng_test.cpp
DEPENDS
libc.src.pthread.pthread_create
libc.src.pthread.pthread_join
- libc.src.stdlib.linux.vsdo_rng
+ libc.src.stdlib.linux.vdso_rng
)
diff --git a/libc/test/integration/src/stdlib/linux/vsdo_rng_test.cpp b/libc/test/integration/src/stdlib/linux/vdso_rng_test.cpp
similarity index 92%
rename from libc/test/integration/src/stdlib/linux/vsdo_rng_test.cpp
rename to libc/test/integration/src/stdlib/linux/vdso_rng_test.cpp
index 1476b0931e3f9..b16d0cd8e896b 100644
--- a/libc/test/integration/src/stdlib/linux/vsdo_rng_test.cpp
+++ b/libc/test/integration/src/stdlib/linux/vdso_rng_test.cpp
@@ -1,4 +1,4 @@
-//===-- Test for vsdo_rng functionality ----------------------------------===//
+//===-- Test for vdso_rng functionality ----------------------------------===//
//
// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
// See https://llvm.org/LICENSE.txt for license information.
@@ -8,14 +8,14 @@
#include "src/pthread/pthread_create.h"
#include "src/pthread/pthread_join.h"
-#include "src/stdlib/linux/vsdo_rng.h"
+#include "src/stdlib/linux/vdso_rng.h"
#include "test/IntegrationTest/test.h"
using namespace LIBC_NAMESPACE;
void basic_test() {
// Test basic functionality
- vsdo_rng::LocalState &local_state = vsdo_rng::local_state;
+ vdso_rng::LocalState &local_state = vdso_rng::local_state;
// Try to get a guard
if (auto guard = local_state.get()) {
>From 689a2db07b1af847aae2311085b76f8bdd57df5f Mon Sep 17 00:00:00 2001
From: Schrodinger ZHU Yifan <yifanzhu at rochester.edu>
Date: Tue, 26 Aug 2025 10:26:35 -0400
Subject: [PATCH 11/12] remove all lockless changes
---
libc/src/__support/CMakeLists.txt | 24 ----
libc/src/__support/aba_ptr.h | 83 ------------
libc/src/__support/mpmc_stack.h | 107 ----------------
libc/src/stdlib/linux/CMakeLists.txt | 1 -
.../integration/src/__support/CMakeLists.txt | 13 --
.../src/__support/mpmc_stack_test.cpp | 119 ------------------
6 files changed, 347 deletions(-)
delete mode 100644 libc/src/__support/aba_ptr.h
delete mode 100644 libc/src/__support/mpmc_stack.h
delete mode 100644 libc/test/integration/src/__support/mpmc_stack_test.cpp
diff --git a/libc/src/__support/CMakeLists.txt b/libc/src/__support/CMakeLists.txt
index c9d89cf6fc286..2196d9e23bba7 100644
--- a/libc/src/__support/CMakeLists.txt
+++ b/libc/src/__support/CMakeLists.txt
@@ -398,30 +398,6 @@ add_header_library(
libc.src.__support.macros.attributes
)
-add_header_library(
- aba_ptr
- HDRS
- aba_ptr.h
- DEPENDS
- libc.hdr.types.size_t
- libc.src.__support.common
- libc.src.__support.threads.sleep
-)
-
-add_header_library(
- mpmc_stack
- HDRS
- mpmc_stack.h
- DEPENDS
- libc.src.__support.aba_ptr
- libc.src.__support.common
- libc.src.__support.CPP.atomic
- libc.src.__support.CPP.new
- libc.src.__support.CPP.optional
- libc.src.__support.CPP.type_traits
-)
-
-
add_subdirectory(FPUtil)
add_subdirectory(OSUtil)
add_subdirectory(StringUtil)
diff --git a/libc/src/__support/aba_ptr.h b/libc/src/__support/aba_ptr.h
deleted file mode 100644
index 3e55aea8875a3..0000000000000
--- a/libc/src/__support/aba_ptr.h
+++ /dev/null
@@ -1,83 +0,0 @@
-//===-- Transactional Ptr for ABA prevention --------------------*- C++ -*-===//
-//
-// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
-// See https://llvm.org/LICENSE.txt for license information.
-// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
-//
-//===----------------------------------------------------------------------===//
-
-#ifndef LLVM_LIBC_SRC___SUPPORT_TAGGED_POINTER_H
-#define LLVM_LIBC_SRC___SUPPORT_TAGGED_POINTER_H
-
-#include "hdr/types/size_t.h"
-#include "src/__support/common.h"
-#include "src/__support/threads/sleep.h"
-
-#ifdef __GCC_HAVE_SYNC_COMPARE_AND_SWAP_16
-#define LIBC_ABA_PTR_IS_ATOMIC true
-#else
-#define LIBC_ABA_PTR_IS_ATOMIC false
-#endif
-
-namespace LIBC_NAMESPACE_DECL {
-
-template <class T, bool IsAtomic> struct AbaPtrImpl {
- union Impl {
- struct alignas(2 * alignof(void *)) Atomic {
- T *ptr;
- size_t tag;
- } atomic;
- struct Mutex {
- T *ptr;
- bool locked;
- } mutex;
- } impl;
-
- LIBC_INLINE constexpr AbaPtrImpl(T *ptr)
- : impl(IsAtomic ? Impl{.atomic{ptr, 0}} : Impl{.mutex{ptr, false}}) {}
-
- /// User must guarantee that operation is redoable.
- template <class Op> LIBC_INLINE void transaction(Op &&op) {
- if constexpr (IsAtomic) {
- for (;;) {
- typename Impl::Atomic snapshot, next;
- __atomic_load(&impl.atomic, &snapshot, __ATOMIC_RELAXED);
- next.ptr = op(snapshot.ptr);
- // Wrapping add for unsigned integers.
- next.tag = snapshot.tag + 1;
- if (__atomic_compare_exchange(&impl.atomic, &snapshot, &next, true,
- __ATOMIC_ACQ_REL, __ATOMIC_RELAXED))
- return;
- }
- } else {
- // Acquire the lock.
- while (__atomic_exchange_n(&impl.mutex.locked, true, __ATOMIC_ACQUIRE))
- while (__atomic_load_n(&impl.mutex.locked, __ATOMIC_RELAXED))
- LIBC_NAMESPACE::sleep_briefly();
-
- impl.mutex.ptr = op(impl.mutex.ptr);
- // Release the lock.
- __atomic_store_n(&impl.mutex.locked, false, __ATOMIC_RELEASE);
- }
- }
-
- LIBC_INLINE T *get() const {
- if constexpr (IsAtomic) {
- // Weak micro-architectures typically regards simultaneous partial word
- // loading and full word loading as a race condition. While there are
- // implementations that uses racy read anyway, we still load the whole
- // word to avoid any complications.
- typename Impl::Atomic snapshot;
- __atomic_load(&impl.atomic, &snapshot, __ATOMIC_RELAXED);
- return snapshot.ptr;
- } else {
- return impl.mutex.ptr;
- }
- }
-};
-
-template <class T> using AbaPtr = AbaPtrImpl<T, LIBC_ABA_PTR_IS_ATOMIC>;
-} // namespace LIBC_NAMESPACE_DECL
-
-#undef LIBC_ABA_PTR_IS_ATOMIC
-#endif
diff --git a/libc/src/__support/mpmc_stack.h b/libc/src/__support/mpmc_stack.h
deleted file mode 100644
index df235c2c1dfac..0000000000000
--- a/libc/src/__support/mpmc_stack.h
+++ /dev/null
@@ -1,107 +0,0 @@
-//===-- Simple Lock-free MPMC Stack -----------------------------*- C++ -*-===//
-//
-// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
-// See https://llvm.org/LICENSE.txt for license information.
-// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
-//
-//===----------------------------------------------------------------------===//
-
-#ifndef LLVM_LIBC_SRC___SUPPORT_MPMC_STACK_H
-#define LLVM_LIBC_SRC___SUPPORT_MPMC_STACK_H
-
-#include "src/__support/CPP/atomic.h"
-#include "src/__support/CPP/new.h"
-#include "src/__support/CPP/optional.h"
-#include "src/__support/aba_ptr.h"
-
-namespace LIBC_NAMESPACE_DECL {
-template <class T> class MPMCStack {
- struct Node {
- cpp::Atomic<size_t> visitor;
- Node *next;
- T value;
-
- LIBC_INLINE Node(T val) : visitor(0), next(nullptr), value(val) {}
- };
- AbaPtr<Node> head;
-
-public:
- static_assert(cpp::is_copy_constructible<T>::value,
- "T must be copy constructible");
- LIBC_INLINE constexpr MPMCStack() : head(nullptr) {}
- LIBC_INLINE bool push(T value) {
- AllocChecker ac;
- Node *new_node = new (ac) Node(value);
- if (!ac)
- return false;
- head.transaction([new_node](Node *old_head) {
- new_node->next = old_head;
- return new_node;
- });
- return true;
- }
- LIBC_INLINE bool push_all(T values[], size_t count) {
- struct Guard {
- size_t count;
- Node **allocated;
- LIBC_INLINE Guard(Node *allocated[]) : count(0), allocated(allocated) {}
- LIBC_INLINE ~Guard() {
- for (size_t i = 0; i < count; ++i)
- delete allocated[i];
- }
- LIBC_INLINE void add(Node *node) { allocated[count++] = node; }
- LIBC_INLINE void clear() { count = 0; }
- };
- // Variable sized array is a GNU extension.
- __extension__ Node *allocated[count];
- {
- Guard guard(allocated);
- for (size_t i = 0; i < count; ++i) {
- AllocChecker ac;
- Node *new_node = new (ac) Node(values[i]);
- if (!ac)
- return false;
- guard.add(new_node);
- if (i != 0)
- new_node->next = allocated[i - 1];
- }
- guard.clear();
- }
- head.transaction([&allocated, count](Node *old_head) {
- allocated[0]->next = old_head;
- return allocated[count - 1];
- });
- return true;
- }
- LIBC_INLINE cpp::optional<T> pop() {
- cpp::optional<T> res = cpp::nullopt;
- Node *node = nullptr;
- head.transaction([&](Node *current_head) -> Node * {
- if (!current_head) {
- res = cpp::nullopt;
- return nullptr;
- }
- node = current_head;
- node->visitor.fetch_add(1);
- res = cpp::optional<T>{node->value};
- Node *next = node->next;
- node->visitor.fetch_sub(1);
- return next;
- });
- // On a successful transaction, a node is popped by us. So we must delete
- // it. When we are at here, no one else can acquire
- // new reference to the node, but we still need to wait until other threads
- // inside the transaction who may potentially be holding a reference to the
- // node.
- if (res) {
- // Spin until the node is no longer in use.
- while (node->visitor.load() != 0)
- LIBC_NAMESPACE::sleep_briefly();
- delete node;
- }
- return res;
- }
-};
-} // namespace LIBC_NAMESPACE_DECL
-
-#endif
diff --git a/libc/src/stdlib/linux/CMakeLists.txt b/libc/src/stdlib/linux/CMakeLists.txt
index 85ad2bdab33ba..b7246db377339 100644
--- a/libc/src/stdlib/linux/CMakeLists.txt
+++ b/libc/src/stdlib/linux/CMakeLists.txt
@@ -23,7 +23,6 @@ add_header_library(
libc.src.__support.OSUtil.linux.vdso
libc.src.__support.OSUtil.osutil
libc.src.__support.macros.config
- libc.src.__support.mpmc_stack
libc.src.__support.threads.callonce
libc.src.__support.threads.linux.raw_mutex
libc.src.sys.auxv.getauxval # TODO: remove public entrypoint dependency
diff --git a/libc/test/integration/src/__support/CMakeLists.txt b/libc/test/integration/src/__support/CMakeLists.txt
index 93f54083f3c00..7caef9ce1d401 100644
--- a/libc/test/integration/src/__support/CMakeLists.txt
+++ b/libc/test/integration/src/__support/CMakeLists.txt
@@ -4,16 +4,3 @@ if(LIBC_TARGET_OS_IS_GPU)
endif()
add_libc_integration_test_suite(libc-support-integration-tests)
-
-add_integration_test(
- mpmc_stack_test
- SUITE
- libc-support-integration-tests
- SRCS
- mpmc_stack_test.cpp
- DEPENDS
- libc.src.__support.mpmc_stack
- libc.src.__support.threads.thread
- libc.src.pthread.pthread_create
- libc.src.pthread.pthread_join
-)
diff --git a/libc/test/integration/src/__support/mpmc_stack_test.cpp b/libc/test/integration/src/__support/mpmc_stack_test.cpp
deleted file mode 100644
index 9166a816a74fe..0000000000000
--- a/libc/test/integration/src/__support/mpmc_stack_test.cpp
+++ /dev/null
@@ -1,119 +0,0 @@
-#include "src/__support/CPP/atomic.h"
-#include "src/__support/mpmc_stack.h"
-#include "src/pthread/pthread_create.h"
-#include "src/pthread/pthread_join.h"
-#include "test/IntegrationTest/test.h"
-
-using namespace LIBC_NAMESPACE;
-
-void smoke_test() {
- MPMCStack<int> stack;
- for (int i = 0; i <= 100; ++i)
- if (!stack.push(i))
- __builtin_trap();
- for (int i = 100; i >= 0; --i)
- if (*stack.pop() != i)
- __builtin_trap();
- if (stack.pop())
- __builtin_trap(); // Should be empty now.
-}
-
-void multithread_test() {
- constexpr static size_t NUM_THREADS = 5;
- constexpr static size_t NUM_PUSHES = 100;
- struct State {
- MPMCStack<size_t> stack;
- cpp::Atomic<size_t> counter = 0;
- cpp::Atomic<bool> flags[NUM_PUSHES];
- } state;
- pthread_t threads[NUM_THREADS];
- for (size_t i = 0; i < NUM_THREADS; ++i) {
- LIBC_NAMESPACE::pthread_create(
- &threads[i], nullptr,
- [](void *arg) -> void * {
- State *state = static_cast<State *>(arg);
- for (;;) {
- size_t current = state->counter.fetch_add(1);
- if (current >= NUM_PUSHES)
- break;
- if (!state->stack.push(current))
- __builtin_trap();
- }
- while (auto res = state->stack.pop())
- state->flags[res.value()].store(true);
- return nullptr;
- },
- &state);
- }
- for (pthread_t thread : threads)
- LIBC_NAMESPACE::pthread_join(thread, nullptr);
- while (cpp::optional<size_t> res = state.stack.pop())
- state.flags[res.value()].store(true);
- for (size_t i = 0; i < NUM_PUSHES; ++i)
- if (!state.flags[i].load())
- __builtin_trap();
-}
-
-void multithread_push_all_test() {
- constexpr static size_t NUM_THREADS = 4;
- constexpr static size_t BATCH_SIZE = 10;
- constexpr static size_t NUM_BATCHES = 20;
- struct State {
- MPMCStack<size_t> stack;
- cpp::Atomic<size_t> counter = 0;
- cpp::Atomic<bool> flags[NUM_THREADS * BATCH_SIZE * NUM_BATCHES];
- } state;
- pthread_t threads[NUM_THREADS];
-
- for (size_t i = 0; i < NUM_THREADS; ++i) {
- LIBC_NAMESPACE::pthread_create(
- &threads[i], nullptr,
- [](void *arg) -> void * {
- State *state = static_cast<State *>(arg);
- size_t values[BATCH_SIZE];
-
- for (size_t batch = 0; batch < NUM_BATCHES; ++batch) {
- // Prepare batch of values
- for (size_t j = 0; j < BATCH_SIZE; ++j) {
- size_t current = state->counter.fetch_add(1);
- values[j] = current;
- }
-
- // Push all values in batch
- if (!state->stack.push_all(values, BATCH_SIZE))
- __builtin_trap();
- }
-
- // Pop and mark all values
- while (auto res = state->stack.pop()) {
- size_t value = res.value();
- if (value < NUM_THREADS * BATCH_SIZE * NUM_BATCHES)
- state->flags[value].store(true);
- }
- return nullptr;
- },
- &state);
- }
-
- for (pthread_t thread : threads)
- LIBC_NAMESPACE::pthread_join(thread, nullptr);
-
- // Pop any remaining values
- while (cpp::optional<size_t> res = state.stack.pop()) {
- size_t value = res.value();
- if (value < NUM_THREADS * BATCH_SIZE * NUM_BATCHES)
- state.flags[value].store(true);
- }
-
- // Verify all values were processed
- for (size_t i = 0; i < NUM_THREADS * BATCH_SIZE * NUM_BATCHES; ++i)
- if (!state.flags[i].load())
- __builtin_trap();
-}
-
-TEST_MAIN() {
- smoke_test();
- multithread_test();
- multithread_push_all_test();
- return 0;
-}
>From b631c379bcdc501032a2b25a5e90955010ddeba4 Mon Sep 17 00:00:00 2001
From: Schrodinger ZHU Yifan <yifanzhu at rochester.edu>
Date: Wed, 27 Aug 2025 10:02:31 -0400
Subject: [PATCH 12/12] migrate to mutex based
---
libc/src/stdlib/linux/CMakeLists.txt | 1 -
libc/src/stdlib/linux/vdso_rng.h | 372 ++++++++++++++++-----------
2 files changed, 228 insertions(+), 145 deletions(-)
diff --git a/libc/src/stdlib/linux/CMakeLists.txt b/libc/src/stdlib/linux/CMakeLists.txt
index b7246db377339..39c75d6c62ec7 100644
--- a/libc/src/stdlib/linux/CMakeLists.txt
+++ b/libc/src/stdlib/linux/CMakeLists.txt
@@ -23,7 +23,6 @@ add_header_library(
libc.src.__support.OSUtil.linux.vdso
libc.src.__support.OSUtil.osutil
libc.src.__support.macros.config
- libc.src.__support.threads.callonce
libc.src.__support.threads.linux.raw_mutex
libc.src.sys.auxv.getauxval # TODO: remove public entrypoint dependency
libc.include.sys_syscall
diff --git a/libc/src/stdlib/linux/vdso_rng.h b/libc/src/stdlib/linux/vdso_rng.h
index fdfa5b1872561..636a06563ded0 100644
--- a/libc/src/stdlib/linux/vdso_rng.h
+++ b/libc/src/stdlib/linux/vdso_rng.h
@@ -15,12 +15,12 @@
#include "src/__support/OSUtil/syscall.h"
#include "src/__support/common.h"
#include "src/__support/macros/config.h"
-#include "src/__support/mpmc_stack.h"
-#include "src/__support/threads/callonce.h"
#include "src/__support/threads/linux/raw_mutex.h"
// TODO: this is public entrypoint, we should remove it later on.
#include "src/sys/auxv/getauxval.h"
+#include <linux/mman.h> //MREMAP_MAYMOVE
+
namespace LIBC_NAMESPACE_DECL {
namespace vdso_rng {
extern "C" {
@@ -28,36 +28,137 @@ using Destructor = void(void *);
[[gnu::weak]] extern void *__dso_handle;
int __cxa_thread_atexit_impl(Destructor *, void *, void *);
}
+
+//===----------------------------------------------------------------------===//
+// MMap Vector
+//===----------------------------------------------------------------------===//
+//
+// We don't want to use malloc inside the RNG implementation which complicates
+// the clean up on exiting. We use a raw mmap vector to maintain available
+// states. This vector grows linearly. As the pool is cached globally, we don't
+// clean it up on exit.
+//
+//===----------------------------------------------------------------------===//
+
+class PtrStore {
+ void **data;
+ void **usage;
+ void **end;
+
+ LIBC_INLINE size_t remaining_capacity() const { return end - usage; }
+ LIBC_INLINE size_t capacity() const { return end - data; }
+ LIBC_INLINE size_t size() const { return usage - data; }
+ LIBC_INLINE bool reserve(size_t incoming, size_t page_size) {
+ if (remaining_capacity() >= incoming)
+ return incoming;
+ size_t old_size = size();
+ size_t size_in_bytes = (old_size + incoming) * sizeof(void *);
+ size_t roundup = ((size_in_bytes + page_size - 1) / page_size) * page_size;
+ long sysret;
+ if (data == nullptr)
+ sysret = LIBC_NAMESPACE::syscall_impl<long>(
+ SYS_mmap, /*addr=*/nullptr,
+ /*length=*/roundup,
+ /*prot=*/PROT_READ | PROT_WRITE,
+ /*flags=*/MAP_PRIVATE | MAP_ANONYMOUS, /*fd=*/-1, /*offset=*/0);
+ else
+ sysret = LIBC_NAMESPACE::syscall_impl<long>(
+ SYS_mremap, /*old_address=*/data,
+ /*old_size=*/capacity() * sizeof(void *),
+ /*new_size=*/roundup, MREMAP_MAYMOVE, /*new_address=*/nullptr);
+ if (sysret == -1 /* MAP_FAILED */)
+ return false;
+ data = reinterpret_cast<void **>(sysret);
+ usage = data + old_size;
+ end = data + (roundup / sizeof(void *));
+ return true;
+ }
+
+public:
+ LIBC_INLINE constexpr PtrStore()
+ : data(nullptr), usage(nullptr), end(nullptr) {}
+ LIBC_INLINE void push_no_grow(void *ptr) {
+ LIBC_ASSERT(remaining_capacity() > 0);
+ *usage++ = ptr;
+ }
+ LIBC_INLINE void **bump(size_t count, size_t page_size) {
+ if (!reserve(count, page_size))
+ return nullptr;
+ void **result = usage;
+ usage += count;
+ return result;
+ }
+ LIBC_INLINE void *pop() {
+ if (size() == 0)
+ return nullptr;
+ return *--usage;
+ }
+};
+
+//===----------------------------------------------------------------------===//
+// Configuration
+//===----------------------------------------------------------------------===//
+
+struct VGetrandomOpaqueParams {
+ unsigned int size_of_opaque_states;
+ unsigned int mmap_prot;
+ unsigned int mmap_flags;
+ unsigned int reserved[13];
+};
+
+struct Config {
+ size_t page_size;
+ size_t pages_per_alloc;
+ size_t states_per_page;
+ vdso::VDSOSymType<vdso::VDSOSym::GetRandom> getrandom;
+ VGetrandomOpaqueParams params;
+
+ LIBC_INLINE bool is_valid() const { return page_size != 0; }
+};
+
+LIBC_INLINE_VAR constexpr size_t MINIMAL_STATE_NUMBER = 4;
+
+//===----------------------------------------------------------------------===//
+// Estimate the need of states based on CPU count. The global pool will grow
+// linearly at the rate of cpu_count().
+//===----------------------------------------------------------------------===//
+LIBC_INLINE size_t cpu_count() {
+ char cpu_set[128] = {0};
+ int res = LIBC_NAMESPACE::syscall_impl<int>(SYS_sched_getaffinity, 0,
+ sizeof(cpu_set), cpu_set);
+ if (res <= 0)
+ return 1;
+
+ size_t count = 0;
+ for (auto byte : cpu_set)
+ count += cpp::popcount(static_cast<unsigned char>(byte));
+
+ return count > 0 ? count : 1;
+}
+
+//===----------------------------------------------------------------------===//
+// Global State Pool
+//===----------------------------------------------------------------------===//
class GlobalState {
public:
- struct VGetrandomOpaqueParams {
- unsigned int size_of_opaque_states;
- unsigned int mmap_prot;
- unsigned int mmap_flags;
- unsigned int reserved[13];
- };
+ struct Result {
+ void *state;
+ vdso::VDSOSymType<vdso::VDSOSym::GetRandom> vgetrandom;
+ size_t size_of_opaque_states;
- struct Config {
- size_t page_size;
- size_t pages_per_alloc;
- size_t states_per_page;
- vdso::VDSOSymType<vdso::VDSOSym::GetRandom> getrandom;
- VGetrandomOpaqueParams params;
+ LIBC_INLINE static Result invalid() { return {nullptr, nullptr, 0}; }
+ LIBC_INLINE bool is_valid() const { return state != nullptr; }
};
private:
- // A lock-free stack of free opaque states.
- MPMCStack<void *> free_list{};
- // A mutex protecting the allocation of new pages.
- RawMutex allocation_mutex{};
+ // A mutex protecting the global state.
+ RawMutex lock;
- // Shared global configuration.
- static CallOnceFlag config_flag;
- static Config config;
+ // Free list
+ PtrStore free_list;
- // We grow the states by the number of CPUs. This function uses
- // SYS_sched_getaffinity to get the number of CPUs.
- LIBC_INLINE static size_t cpu_count();
+ // configuration.
+ Config config;
// Grow available states. This function can fail if the system is out of
// memory.
@@ -65,20 +166,29 @@ class GlobalState {
// - On success, this routine returns one opaque state for direct use.
LIBC_INLINE void *grow();
+ LIBC_INLINE void ensure_config();
+
public:
- LIBC_INLINE constexpr GlobalState() {}
- LIBC_INLINE static const Config &get_config();
- LIBC_INLINE static const Config &get_config_unchecked() { return config; }
- LIBC_INLINE void *get();
+ LIBC_INLINE constexpr GlobalState() : lock(), free_list(), config() {}
+ // Try acquire a state for local usage.
+ LIBC_INLINE Result get();
+ // Return a state back to the global pool.
LIBC_INLINE void recycle(void *state);
};
LIBC_INLINE_VAR GlobalState global_state{};
class LocalState {
+ // Whether there is an in-flight getrandom call. If so, we simply fallback to
+ // syscall.
bool in_flight = false;
+ // Whether previous attempt to get a state has failed. If so, we won't try
+ // again.
bool failed = false;
void *state = nullptr;
+ // Cache the function pointer locally
+ vdso::VDSOSymType<vdso::VDSOSym::GetRandom> vgetrandom = nullptr;
+ size_t size_of_opaque_states = 0;
public:
struct Guard {
@@ -96,37 +206,7 @@ class LocalState {
LIBC_INLINE void fill(void *buf, size_t size) const;
};
LIBC_INLINE constexpr LocalState() {}
- LIBC_INLINE cpp::optional<Guard> get() {
- if (in_flight)
- return cpp::nullopt;
-
- Guard guard(this);
-
- if (!failed && !state) {
- int register_res = __cxa_thread_atexit_impl(
- [](void *self) {
- auto *tls = static_cast<LocalState *>(self);
- // Reject all future attempts to get a state.
- void *state = tls->state;
- tls->in_flight = true;
- tls->failed = true;
- tls->state = nullptr;
- cpp::atomic_thread_fence(cpp::MemoryOrder::SEQ_CST);
- if (state)
- LIBC_NAMESPACE::vdso_rng::global_state.recycle(state);
- },
- this, __dso_handle);
- if (register_res == 0)
- state = LIBC_NAMESPACE::vdso_rng::global_state.get();
- if (!state)
- failed = true;
- }
-
- if (!state)
- return cpp::nullopt;
-
- return cpp::move(guard);
- }
+ LIBC_INLINE cpp::optional<Guard> get();
};
LIBC_INLINE_VAR LIBC_THREAD_LOCAL LocalState local_state{};
@@ -135,63 +215,47 @@ LIBC_INLINE_VAR LIBC_THREAD_LOCAL LocalState local_state{};
// Implementation
//===----------------------------------------------------------------------===//
-LIBC_INLINE_VAR GlobalState::Config GlobalState::config{};
-LIBC_INLINE_VAR CallOnceFlag GlobalState::config_flag = 0;
+LIBC_INLINE void GlobalState::ensure_config() {
+ // Already initialized.
+ if (config.is_valid())
+ return;
-LIBC_INLINE size_t GlobalState::cpu_count() {
- char cpu_set[128] = {0};
- int res = LIBC_NAMESPACE::syscall_impl<int>(SYS_sched_getaffinity, 0,
- sizeof(cpu_set), cpu_set);
- if (res <= 0)
- return 1;
+ config.getrandom =
+ LIBC_NAMESPACE::vdso::TypedSymbol<vdso::VDSOSym::GetRandom>{};
+ if (!config.getrandom)
+ return;
- size_t count = 0;
- for (size_t i = 0; i < sizeof(cpu_set) / sizeof(unsigned long); ++i) {
- unsigned long *mask_ptr = reinterpret_cast<unsigned long *>(cpu_set);
- count += LIBC_NAMESPACE::cpp::popcount(mask_ptr[i]);
- }
+ // Call with special flag to get the desired configuration.
+ int res = config.getrandom(
+ /*buf=*/nullptr, /*count=*/0, /*flags=*/0,
+ /*opaque_states=*/&config.params,
+ /*size_of_opaque_states=*/~0);
+ if (res != 0)
+ return;
- return count > 0 ? count : 1;
-}
+ auto page_size = static_cast<size_t>(LIBC_NAMESPACE::getauxval(AT_PAGESZ));
+ if (!page_size)
+ return;
+
+ size_t count = cpp::max(cpu_count(), MINIMAL_STATE_NUMBER);
+
+ config.states_per_page = page_size / config.params.size_of_opaque_states;
-LIBC_INLINE const GlobalState::Config &GlobalState::get_config() {
- callonce(&config_flag, []() {
- config.getrandom =
- LIBC_NAMESPACE::vdso::TypedSymbol<vdso::VDSOSym::GetRandom>{};
- if (!config.getrandom)
- return;
-
- // Call with special flag to get the desired configuration.
- int res = config.getrandom(
- /*buf=*/nullptr, /*count=*/0, /*flags=*/0,
- /*opaque_states=*/&config.params,
- /*size_of_opaque_states=*/~0);
- if (res != 0)
- return;
-
- config.page_size = LIBC_NAMESPACE::getauxval(AT_PAGESZ);
- if (!config.page_size)
- return;
-
- size_t count = cpp::max(cpu_count(), size_t{4});
-
- config.states_per_page =
- config.page_size / config.params.size_of_opaque_states;
-
- config.pages_per_alloc =
- count / config.states_per_page + (count % config.states_per_page != 0);
- });
- return config;
+ config.pages_per_alloc =
+ count / config.states_per_page + (count % config.states_per_page != 0);
+
+ // Finally set the page size to mark the config as valid.
+ config.page_size = page_size;
}
LIBC_INLINE void *GlobalState::grow() {
- cpp::lock_guard guard(allocation_mutex);
+ LIBC_ASSERT(config.is_valid());
+ size_t total_states = config.pages_per_alloc * config.states_per_page;
+ void **buffer = free_list.bump(total_states, config.page_size);
- // It is possible that when we finally grab the lock, other threads have
- // successfully finished the allocation already. Hence, we first try if we
- // can pop anything from the free list.
- if (cpp::optional<void *> state = free_list.pop())
- return *state;
+ // Failed to allocate space for pointers. Exit early.
+ if (!buffer)
+ return nullptr;
long mmap_res = LIBC_NAMESPACE::syscall_impl<long>(
SYS_mmap, /*addr=*/nullptr,
@@ -199,60 +263,44 @@ LIBC_INLINE void *GlobalState::grow() {
/*prot=*/config.params.mmap_prot,
/*flags=*/config.params.mmap_flags,
/*fd=*/-1, /*offset=*/0);
+
+ // Failed to allocate memory. Exit early.
if (mmap_res == -1 /* MAP_FAILED */)
return nullptr;
- char *pages = reinterpret_cast<char *>(mmap_res);
-
// Initialize the page.
- size_t total_states = config.pages_per_alloc * config.states_per_page;
- size_t free_states = total_states - 1; // reserve one for direct use.
- __extension__ void *opaque_states[total_states];
- size_t index = 0;
+ // Notice that states shall not go across page boundaries.
+ char *pages = reinterpret_cast<char *>(mmap_res);
for (size_t p = 0; p < config.pages_per_alloc; ++p) {
char *page = &pages[p * config.page_size];
for (size_t s = 0; s < config.states_per_page; ++s) {
void *state = &page[s * config.params.size_of_opaque_states];
- opaque_states[index++] = state;
- }
- }
-
- constexpr size_t RETRY_COUNT = 64;
- for (size_t i = 0; i < RETRY_COUNT; ++i) {
- if (free_list.push_all(opaque_states, free_states))
- break;
- // Abort if we are still short in memory after all these retries.
- if (i + 1 == RETRY_COUNT) {
- LIBC_NAMESPACE::syscall_impl<long>(
- SYS_munmap, pages, config.page_size * config.pages_per_alloc);
- return nullptr;
+ *buffer++ = state;
}
}
- return opaque_states[free_states];
+ // Return the last one. It should always be valid at this stage.
+ return free_list.pop();
}
-LIBC_INLINE void *GlobalState::get() {
- const Config &config = get_config();
+LIBC_INLINE GlobalState::Result GlobalState::get() {
+ cpp::lock_guard guard(lock);
+ ensure_config();
// If page size is not set, the global config is invalid. Early return.
- if (!config.page_size)
- return nullptr;
+ if (!config.is_valid())
+ return Result::invalid();
- if (cpp::optional<void *> state = free_list.pop())
- return *state;
+ if (void *state = free_list.pop())
+ return {state, config.getrandom, config.params.size_of_opaque_states};
// At this stage, we know that the config is valid.
- return grow();
+ return {grow(), config.getrandom, config.params.size_of_opaque_states};
}
LIBC_INLINE void GlobalState::recycle(void *state) {
+ cpp::lock_guard guard(lock);
LIBC_ASSERT(state != nullptr);
- constexpr size_t RETRY_COUNT = 64;
- for (size_t i = 0; i < RETRY_COUNT; ++i)
- if (free_list.push(state))
- return;
- // Otherwise, we just let it leak. It won't be too bad not to reuse the state
- // since the OS can free the page if memory is tight.
+ free_list.push_no_grow(state);
}
//===----------------------------------------------------------------------===//
@@ -263,10 +311,9 @@ LIBC_INLINE void LocalState::Guard::fill(void *buf, size_t size) const {
LIBC_ASSERT(tls->state != nullptr);
char *cursor = reinterpret_cast<char *>(buf);
size_t remaining = size;
- const auto &config = GlobalState::get_config_unchecked();
while (remaining > 0) {
- int res = config.getrandom(cursor, remaining, /* default random flag */ 0,
- tls->state, config.params.size_of_opaque_states);
+ int res = tls->vgetrandom(cursor, remaining, /* default random flag */ 0,
+ tls->state, tls->size_of_opaque_states);
if (res < 0)
continue;
remaining -= static_cast<size_t>(res);
@@ -274,6 +321,43 @@ LIBC_INLINE void LocalState::Guard::fill(void *buf, size_t size) const {
}
}
+LIBC_INLINE cpp::optional<LocalState::Guard> LocalState::get() {
+ if (in_flight)
+ return cpp::nullopt;
+
+ Guard guard(this);
+
+ // If uninitialized, try to get a state.
+ if (!failed && !state) {
+ int register_res = __cxa_thread_atexit_impl(
+ [](void *self) {
+ auto *tls = static_cast<LocalState *>(self);
+ // Reject all future attempts to get a state.
+ void *state = tls->state;
+ tls->in_flight = true;
+ tls->failed = true;
+ tls->state = nullptr;
+ cpp::atomic_thread_fence(cpp::MemoryOrder::SEQ_CST);
+ if (state)
+ global_state.recycle(state);
+ },
+ this, __dso_handle);
+ if (register_res == 0) {
+ GlobalState::Result result = global_state.get();
+ state = result.state;
+ vgetrandom = result.vgetrandom;
+ size_of_opaque_states = result.size_of_opaque_states;
+ }
+ if (!state)
+ failed = true;
+ }
+
+ if (!state)
+ return cpp::nullopt;
+
+ return cpp::move(guard);
+}
+
} // namespace vdso_rng
} // namespace LIBC_NAMESPACE_DECL
More information about the libc-commits
mailing list