[libcxx-commits] [libcxx] [libc++] Implement P2545R4 Read-Copy Update (RCU) (PR #175451)
via libcxx-commits
libcxx-commits at lists.llvm.org
Sat Sep 19 13:13:17 PDT 2026
https://github.com/huixie90 updated https://github.com/llvm/llvm-project/pull/175451
>From 3f5bce34ad7a12095486a6b2dcaf2e95effe17e6 Mon Sep 17 00:00:00 2001
From: Hui Xie <hui.xie1990 at gmail.com>
Date: Fri, 5 Dec 2025 18:41:50 +0000
Subject: [PATCH 01/65] init
---
libcxx/docs/FeatureTestMacroTable.rst | 2 +-
libcxx/include/CMakeLists.txt | 3 +
libcxx/include/__configuration/experimental.h | 1 +
libcxx/include/__rcu/rcu_domain.h | 27 ++++++++
libcxx/include/__rcu/rcu_obj_base.h | 27 ++++++++
libcxx/include/module.modulemap.in | 7 ++
libcxx/include/rcu | 54 +++++++++++++++
libcxx/include/version | 3 +
libcxx/modules/std.cppm.in | 1 +
libcxx/modules/std/rcu.inc | 2 -
.../rcu.version.compile.pass.cpp | 68 +++++++++++++++++++
.../version.version.compile.pass.cpp | 4 +-
.../generate_feature_test_macro_components.py | 3 +-
13 files changed, 196 insertions(+), 6 deletions(-)
create mode 100644 libcxx/include/__rcu/rcu_domain.h
create mode 100644 libcxx/include/__rcu/rcu_obj_base.h
create mode 100644 libcxx/include/rcu
create mode 100644 libcxx/test/std/language.support/support.limits/support.limits.general/rcu.version.compile.pass.cpp
diff --git a/libcxx/docs/FeatureTestMacroTable.rst b/libcxx/docs/FeatureTestMacroTable.rst
index 8329c6114d212..91ac2b997ccf7 100644
--- a/libcxx/docs/FeatureTestMacroTable.rst
+++ b/libcxx/docs/FeatureTestMacroTable.rst
@@ -522,7 +522,7 @@ Status
---------------------------------------------------------- -----------------
``__cpp_lib_raw_memory_algorithms`` ``202411L``
---------------------------------------------------------- -----------------
- ``__cpp_lib_rcu`` *unimplemented*
+ ``__cpp_lib_rcu`` ``202306L``
---------------------------------------------------------- -----------------
``__cpp_lib_reference_wrapper`` ``202403L``
---------------------------------------------------------- -----------------
diff --git a/libcxx/include/CMakeLists.txt b/libcxx/include/CMakeLists.txt
index caf90b0cbbafa..b0282cc6198c0 100644
--- a/libcxx/include/CMakeLists.txt
+++ b/libcxx/include/CMakeLists.txt
@@ -803,6 +803,8 @@ set(files
__ranges/views.h
__ranges/zip_transform_view.h
__ranges/zip_view.h
+ __rcu/rcu_domain.h
+ __rcu/rcu_obj_base.h
__split_buffer
__std_mbstate_t.h
__stop_token/atomic_unique_lock.h
@@ -1107,6 +1109,7 @@ set(files
ranges
ratio
regex
+ rcu
scoped_allocator
semaphore
set
diff --git a/libcxx/include/__configuration/experimental.h b/libcxx/include/__configuration/experimental.h
index bb38d8297c63d..6af9ea49f64a9 100644
--- a/libcxx/include/__configuration/experimental.h
+++ b/libcxx/include/__configuration/experimental.h
@@ -34,5 +34,6 @@
#define _LIBCPP_HAS_EXPERIMENTAL_SYNCSTREAM _LIBCPP_HAS_EXPERIMENTAL_LIBRARY
#define _LIBCPP_HAS_EXPERIMENTAL_HARDENING_OBSERVE_SEMANTIC _LIBCPP_HAS_EXPERIMENTAL_LIBRARY
#define _LIBCPP_HAS_EXPERIMENTAL_OPTIONAL_ITERATOR _LIBCPP_HAS_EXPERIMENTAL_LIBRARY
+#define _LIBCPP_HAS_EXPERIMENTAL_RCU _LIBCPP_HAS_EXPERIMENTAL_LIBRARY
#endif // _LIBCPP___CONFIGURATION_EXPERIMENTAL_H
diff --git a/libcxx/include/__rcu/rcu_domain.h b/libcxx/include/__rcu/rcu_domain.h
new file mode 100644
index 0000000000000..9b41cb7758458
--- /dev/null
+++ b/libcxx/include/__rcu/rcu_domain.h
@@ -0,0 +1,27 @@
+// -*- C++ -*-
+//===----------------------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef _LIBCPP___RCU_RCU_DOMAIN_H
+#define _LIBCPP___RCU_RCU_DOMAIN_H
+
+#include <__config>
+
+#if !defined(_LIBCPP_HAS_NO_PRAGMA_SYSTEM_HEADER)
+# pragma GCC system_header
+#endif
+
+_LIBCPP_BEGIN_NAMESPACE_STD
+
+#if _LIBCPP_STD_VER >= 26 && _LIBCPP_HAS_THREADS && _LIBCPP_HAS_EXPERIMENTAL_RCU
+
+#endif // _LIBCPP_STD_VER >= 26 && _LIBCPP_HAS_THREADS && _LIBCPP_HAS_EXPERIMENTAL_RCU
+
+_LIBCPP_END_NAMESPACE_STD
+
+#endif // _LIBCPP___RCU_RCU_DOMAIN_H
diff --git a/libcxx/include/__rcu/rcu_obj_base.h b/libcxx/include/__rcu/rcu_obj_base.h
new file mode 100644
index 0000000000000..faff897a1f159
--- /dev/null
+++ b/libcxx/include/__rcu/rcu_obj_base.h
@@ -0,0 +1,27 @@
+// -*- C++ -*-
+//===----------------------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef _LIBCPP___RCU_RCU_OBJ_BASE_H
+#define _LIBCPP___RCU_RCU_OBJ_BASE_H
+
+#include <__config>
+
+#if !defined(_LIBCPP_HAS_NO_PRAGMA_SYSTEM_HEADER)
+# pragma GCC system_header
+#endif
+
+_LIBCPP_BEGIN_NAMESPACE_STD
+
+#if _LIBCPP_STD_VER >= 26 && _LIBCPP_HAS_THREADS && _LIBCPP_HAS_EXPERIMENTAL_RCU
+
+#endif // _LIBCPP_STD_VER >= 26 && _LIBCPP_HAS_THREADS && _LIBCPP_HAS_EXPERIMENTAL_RCU
+
+_LIBCPP_END_NAMESPACE_STD
+
+#endif // _LIBCPP___RCU_RCU_OBJ_BASE_H
diff --git a/libcxx/include/module.modulemap.in b/libcxx/include/module.modulemap.in
index 29af487336365..7d3c90a1aa2e8 100644
--- a/libcxx/include/module.modulemap.in
+++ b/libcxx/include/module.modulemap.in
@@ -2063,6 +2063,13 @@ module std {
export *
}
+ module rcu {
+ module rcu_domain { header "__rcu/rcu_domain.h" }
+ module rcu_obj_base { header "__rcu/rcu_obj_base.h" }
+ header "rcu"
+ export *
+ }
+
module regex {
header "regex"
export *
diff --git a/libcxx/include/rcu b/libcxx/include/rcu
new file mode 100644
index 0000000000000..833aae4db6909
--- /dev/null
+++ b/libcxx/include/rcu
@@ -0,0 +1,54 @@
+// -*- C++ -*-
+//===----------------------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef _LIBCPP_RCU
+#define _LIBCPP_RCU
+
+/*
+
+namespace std {
+ // [saferecl.rcu.base], class template rcu_obj_base
+ template<class T, class D = default_delete<T>> class rcu_obj_base;
+
+ // [saferecl.rcu.domain], class rcu_domain
+ class rcu_domain;
+
+ // [saferecl.rcu.domain.func], non-member functions
+ rcu_domain& rcu_default_domain() noexcept;
+ void rcu_synchronize(rcu_domain& dom = rcu_default_domain()) noexcept;
+ void rcu_barrier(rcu_domain& dom = rcu_default_domain()) noexcept;
+ template<class T, class D = default_delete<T>>
+ void rcu_retire(T* p, D d = D(), rcu_domain& dom = rcu_default_domain());
+}
+
+*/
+
+#if __cplusplus < 201103L && defined(_LIBCPP_USE_FROZEN_CXX03_HEADERS)
+# include <__cxx03/__config>
+#else
+# include <__config>
+
+# if _LIBCPP_HAS_THREADS && _LIBCPP_HAS_EXPERIMENTAL_RCU
+
+# if _LIBCPP_STD_VER >= 26
+# include <__rcu/rcu_domain.h>
+# include <__rcu/rcu_obj_base.h>
+# endif
+
+# include <version>
+
+# if !defined(_LIBCPP_HAS_NO_PRAGMA_SYSTEM_HEADER)
+# pragma GCC system_header
+# endif
+
+# endif // _LIBCPP_HAS_THREADS && _LIBCPP_HAS_EXPERIMENTAL_RCU
+
+#endif // __cplusplus < 201103L && defined(_LIBCPP_USE_FROZEN_CXX03_HEADERS)
+
+#endif // _LIBCPP_RCU
diff --git a/libcxx/include/version b/libcxx/include/version
index de14038550c1c..9f20a3d843d89 100644
--- a/libcxx/include/version
+++ b/libcxx/include/version
@@ -641,6 +641,9 @@ __cpp_lib_void_t 201411L <type_traits>
# undef __cpp_lib_raw_memory_algorithms
# define __cpp_lib_raw_memory_algorithms 202411L
// # define __cpp_lib_rcu 202306L
+# if _LIBCPP_HAS_THREADS && _LIBCPP_HAS_EXPERIMENTAL_RCU
+# define __cpp_lib_rcu 202306L
+# endif
# define __cpp_lib_reference_wrapper 202403L
# define __cpp_lib_saturation_arithmetic 202603L
// # define __cpp_lib_senders 202406L
diff --git a/libcxx/modules/std.cppm.in b/libcxx/modules/std.cppm.in
index b9a491653659a..a7d41b5ff9573 100644
--- a/libcxx/modules/std.cppm.in
+++ b/libcxx/modules/std.cppm.in
@@ -91,6 +91,7 @@ module;
#include <random>
#include <ranges>
#include <ratio>
+#include <rcu>
#include <regex>
#include <scoped_allocator>
#include <semaphore>
diff --git a/libcxx/modules/std/rcu.inc b/libcxx/modules/std/rcu.inc
index abf5afad30c13..0dbad726474e7 100644
--- a/libcxx/modules/std/rcu.inc
+++ b/libcxx/modules/std/rcu.inc
@@ -8,7 +8,6 @@
//===----------------------------------------------------------------------===//
export namespace std {
-#if 0
# if _LIBCPP_STD_VER >= 23
// 2.2.3, class template rcu_obj_base using std::rcu_obj_base;
// 2.2.4, class rcu_domain
@@ -18,5 +17,4 @@ export namespace std {
using std::rcu_retire;
using std::rcu_synchronize;
# endif // _LIBCPP_STD_VER >= 23
-#endif
} // namespace std
diff --git a/libcxx/test/std/language.support/support.limits/support.limits.general/rcu.version.compile.pass.cpp b/libcxx/test/std/language.support/support.limits/support.limits.general/rcu.version.compile.pass.cpp
new file mode 100644
index 0000000000000..62331b8564d48
--- /dev/null
+++ b/libcxx/test/std/language.support/support.limits/support.limits.general/rcu.version.compile.pass.cpp
@@ -0,0 +1,68 @@
+//===----------------------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+// WARNING: This test was generated by generate_feature_test_macro_components.py
+// and should not be edited manually.
+
+// <rcu>
+
+// Test the feature test macros defined by <rcu>
+
+// clang-format off
+
+#include <rcu>
+#include "test_macros.h"
+
+#if TEST_STD_VER < 14
+
+# ifdef __cpp_lib_rcu
+# error "__cpp_lib_rcu should not be defined before c++26"
+# endif
+
+#elif TEST_STD_VER == 14
+
+# ifdef __cpp_lib_rcu
+# error "__cpp_lib_rcu should not be defined before c++26"
+# endif
+
+#elif TEST_STD_VER == 17
+
+# ifdef __cpp_lib_rcu
+# error "__cpp_lib_rcu should not be defined before c++26"
+# endif
+
+#elif TEST_STD_VER == 20
+
+# ifdef __cpp_lib_rcu
+# error "__cpp_lib_rcu should not be defined before c++26"
+# endif
+
+#elif TEST_STD_VER == 23
+
+# ifdef __cpp_lib_rcu
+# error "__cpp_lib_rcu should not be defined before c++26"
+# endif
+
+#elif TEST_STD_VER > 23
+
+# if !defined(_LIBCPP_VERSION) || (_LIBCPP_HAS_THREADS && _LIBCPP_HAS_EXPERIMENTAL_RCU)
+# ifndef __cpp_lib_rcu
+# error "__cpp_lib_rcu should be defined in c++26"
+# endif
+# if __cpp_lib_rcu != 202306L
+# error "__cpp_lib_rcu should have the value 202306L in c++26"
+# endif
+# else
+# ifdef __cpp_lib_rcu
+# error "__cpp_lib_rcu should not be defined when the requirement '!defined(_LIBCPP_VERSION) || (_LIBCPP_HAS_THREADS && _LIBCPP_HAS_EXPERIMENTAL_RCU)' is not met!"
+# endif
+# endif
+
+#endif // TEST_STD_VER > 23
+
+// clang-format on
diff --git a/libcxx/test/std/language.support/support.limits/support.limits.general/version.version.compile.pass.cpp b/libcxx/test/std/language.support/support.limits/support.limits.general/version.version.compile.pass.cpp
index 9c89124f6b65d..f42d105d993b4 100644
--- a/libcxx/test/std/language.support/support.limits/support.limits.general/version.version.compile.pass.cpp
+++ b/libcxx/test/std/language.support/support.limits/support.limits.general/version.version.compile.pass.cpp
@@ -8134,7 +8134,7 @@
# error "__cpp_lib_raw_memory_algorithms should have the value 202411L in c++26"
# endif
-# if !defined(_LIBCPP_VERSION)
+# if !defined(_LIBCPP_VERSION) || (_LIBCPP_HAS_THREADS && _LIBCPP_HAS_EXPERIMENTAL_RCU)
# ifndef __cpp_lib_rcu
# error "__cpp_lib_rcu should be defined in c++26"
# endif
@@ -8143,7 +8143,7 @@
# endif
# else
# ifdef __cpp_lib_rcu
-# error "__cpp_lib_rcu should not be defined because it is unimplemented in libc++!"
+# error "__cpp_lib_rcu should not be defined when the requirement '!defined(_LIBCPP_VERSION) || (_LIBCPP_HAS_THREADS && _LIBCPP_HAS_EXPERIMENTAL_RCU)' is not met!"
# endif
# endif
diff --git a/libcxx/utils/generate_feature_test_macro_components.py b/libcxx/utils/generate_feature_test_macro_components.py
index f97488ddca59d..3e2b2b8c2f542 100755
--- a/libcxx/utils/generate_feature_test_macro_components.py
+++ b/libcxx/utils/generate_feature_test_macro_components.py
@@ -1249,7 +1249,8 @@ def add_version_header(tc):
"headers": [
"rcu" # TODO verify this entry since the paper was underspecified.
],
- "unimplemented": True,
+ "test_suite_guard": "!defined(_LIBCPP_VERSION) || (_LIBCPP_HAS_THREADS && _LIBCPP_HAS_EXPERIMENTAL_RCU)",
+ "libcxx_guard": "_LIBCPP_HAS_THREADS && _LIBCPP_HAS_EXPERIMENTAL_RCU",
},
{
"name": "__cpp_lib_reference_from_temporary",
>From b54ec7367f946578830c2c0ba01ba94cce0e6ec5 Mon Sep 17 00:00:00 2001
From: Hui Xie <hui.xie1990 at gmail.com>
Date: Wed, 7 Jan 2026 21:05:23 +0000
Subject: [PATCH 02/65] init impl
---
libcxx/include/CMakeLists.txt | 1 +
libcxx/include/__rcu/rcu_domain.h | 211 ++++++++++++++++++
libcxx/include/__rcu/rcu_list.h | 36 +++
libcxx/include/__rcu/rcu_obj_base.h | 20 ++
libcxx/include/__rcu/readme.md | 69 ++++++
libcxx/include/module.modulemap.in | 1 +
.../saferecl/saferecl.rcu/general.pass.cpp | 117 ++++++++++
7 files changed, 455 insertions(+)
create mode 100644 libcxx/include/__rcu/rcu_list.h
create mode 100644 libcxx/include/__rcu/readme.md
create mode 100644 libcxx/test/std/thread/saferecl/saferecl.rcu/general.pass.cpp
diff --git a/libcxx/include/CMakeLists.txt b/libcxx/include/CMakeLists.txt
index b0282cc6198c0..a3e67d0be70dd 100644
--- a/libcxx/include/CMakeLists.txt
+++ b/libcxx/include/CMakeLists.txt
@@ -804,6 +804,7 @@ set(files
__ranges/zip_transform_view.h
__ranges/zip_view.h
__rcu/rcu_domain.h
+ __rcu/rcu_list.h
__rcu/rcu_obj_base.h
__split_buffer
__std_mbstate_t.h
diff --git a/libcxx/include/__rcu/rcu_domain.h b/libcxx/include/__rcu/rcu_domain.h
index 9b41cb7758458..fbd19933e171d 100644
--- a/libcxx/include/__rcu/rcu_domain.h
+++ b/libcxx/include/__rcu/rcu_domain.h
@@ -12,6 +12,16 @@
#include <__config>
+#include <__atomic/atomic.h>
+#include <atomic>
+#include <cstdint>
+#include <map>
+#include <mutex>
+#include <vector>
+
+// todo debug
+#include <cstdio>
+
#if !defined(_LIBCPP_HAS_NO_PRAGMA_SYSTEM_HEADER)
# pragma GCC system_header
#endif
@@ -20,6 +30,207 @@ _LIBCPP_BEGIN_NAMESPACE_STD
#if _LIBCPP_STD_VER >= 26 && _LIBCPP_HAS_THREADS && _LIBCPP_HAS_EXPERIMENTAL_RCU
+template <class _ThreadLocal>
+class __thread_local_owner {
+ // todo put globals in experimental dylib
+ inline static thread_local map<const __thread_local_owner*, _ThreadLocal> __thread_local_instances;
+
+ // Keep track of all thread-local instances owned by this owner.
+ // Only emplaced the first time a thread is trying to access its thread-local instance.
+ vector<atomic_ref<_ThreadLocal>> __owned_instances_;
+ mutex __mtx_;
+
+ void __register(_ThreadLocal& __obj) {
+ lock_guard<std::mutex> __lg(__mtx_);
+ __owned_instances_.emplace_back(__obj);
+ }
+
+public:
+ __thread_local_owner() = default;
+ __thread_local_owner(__thread_local_owner&&) = delete;
+
+ atomic_ref<_ThreadLocal> __get_current_thread_instance() {
+ auto __it = __thread_local_instances.find(this);
+ if (__it == __thread_local_instances.end()) {
+ auto [new_it, _] = __thread_local_instances.try_emplace(this, _ThreadLocal());
+ auto& __obj = new_it->second;
+ __register(__obj);
+ return atomic_ref(__obj);
+ }
+ return atomic_ref(__it->second);
+ }
+
+ template <class _Func>
+ void __for_each_owned_instances(_Func&& __f) {
+ unique_lock<std::mutex> __lock(__mtx_);
+ for (auto __instance : __owned_instances_) {
+ __f(__instance);
+ }
+ }
+};
+
+// Adopted the 2-phase implementation in the section
+// "3) General-Purpose RCU" of the paper
+// http://www.rdrop.com/users/paulmck/RCU/urcu-supp-accepted.2011.08.30a.pdf
+
+struct __reader_states {
+ // bit 15 is the grace period phase 0 or 1
+ // bits 0-14 is the reader nest level
+ //
+ // a thread can have nested reader locks, such as
+ // domain.lock(); // nest level = 1
+ // domain.lock(); // nest level = 2
+ // ...
+ // domain.unlock(); // nest level = 1
+ // domain.unlock(); // nest level = 0
+
+ static constexpr uint16_t __grace_period_phase_mask = 0b1000'0000'0000'0000;
+ static constexpr uint16_t __reader_nest_level_mask = 0b0111'1111'1111'1111;
+
+ using __state_type = uint16_t;
+
+ __thread_local_owner<__state_type> __per_thread_states_;
+
+ static uint16_t __get_grace_period_phase(__state_type __state) { return __state & __grace_period_phase_mask; }
+
+ static uint16_t __get_reader_nest_level(__state_type __state) { return __state & __reader_nest_level_mask; }
+
+ static bool __is_quiescent_state(__state_type __state) { return __get_reader_nest_level(__state) == 0; }
+};
+
+class rcu_domain {
+ __reader_states __reader_states_;
+
+ // only the highest bit is used for the phase.
+ std::atomic<__reader_states::__state_type> __global_reader_phase_{};
+
+ // only one thread is allowed to update concurrently
+ std::mutex __reclamation_mutex_; // todo this is not noexcept
+
+ std::atomic<bool> __waiting_flag_ = false;
+
+ rcu_domain() = default;
+
+ friend struct __rcu_domain_access;
+
+ // todo put globals in dylib
+ static rcu_domain& __rcu_default_domain() noexcept {
+ static rcu_domain __default_domain;
+ return __default_domain;
+ }
+
+ void __synchronize() noexcept {
+ __cxx_atomic_thread_fence(memory_order_seq_cst);
+ std::unique_lock __lk(__reclamation_mutex_);
+ std::printf("rcu_domain::__synchronize() going through phase 1\n");
+ __update_phase_and_wait();
+ __barrier();
+ std::printf("rcu_domain::__synchronize() going through phase 2\n");
+ __update_phase_and_wait();
+ __lk.unlock();
+ __cxx_atomic_thread_fence(memory_order_seq_cst);
+ }
+
+ void __update_phase_and_wait() noexcept {
+ // Flip the global phase
+ auto __old_phase =
+ __global_reader_phase_.fetch_xor(__reader_states::__grace_period_phase_mask, std::memory_order_relaxed);
+ auto __new_phase = __old_phase ^ __reader_states::__grace_period_phase_mask;
+ std::printf("rcu_domain::__update_phase_and_wait() new phase: 0x%04x\n", __new_phase);
+
+ __barrier();
+ // Wait for all threads to quiesce in the old phase
+ while (__any_reader_in_ongoing_grace_period(__new_phase)) {
+ __waiting_flag_.store(true, std::memory_order_relaxed);
+ __waiting_flag_.wait(true, std::memory_order_relaxed);
+ }
+ __waiting_flag_.store(false, std::memory_order_relaxed);
+ }
+
+ bool __any_reader_in_ongoing_grace_period(__reader_states::__state_type __global_phase) noexcept {
+ bool __any_ongoing = false;
+ __reader_states_.__per_thread_states_.__for_each_owned_instances(
+ [this, __global_phase, &__any_ongoing](atomic_ref<__reader_states::__state_type> __state) {
+ if (__is_grace_period_ongoing(__state.load(memory_order_relaxed), __global_phase)) {
+ __any_ongoing = true;
+ }
+ });
+ return __any_ongoing;
+ }
+
+ bool __is_grace_period_ongoing(__reader_states::__state_type __thread_state,
+ __reader_states::__state_type __global_phase) const noexcept {
+ // https://lwn.net/Articles/323929/
+ // The phase is flipped at the beginning of a grace period.
+ // Any readers that started before the flip will have the old phase
+ // and we consider them as ongoing that we need to wait for before we can close the grace period.
+ return !__reader_states::__is_quiescent_state(__thread_state) &&
+ __reader_states::__get_grace_period_phase(__thread_state) != __global_phase;
+ }
+
+ void __barrier() noexcept { asm volatile("" : : : "memory"); }
+
+public:
+ rcu_domain(const rcu_domain&) = delete;
+ rcu_domain& operator=(const rcu_domain&) = delete;
+
+ void printAllReaderStatesInHex() {
+ __reader_states_.__per_thread_states_.__for_each_owned_instances([](auto __state_ref) {
+ std::printf("Reader state: 0x%04x\n", __state_ref.load());
+ });
+ }
+
+ void lock() {
+ auto __current_thread_state_ref = __reader_states_.__per_thread_states_.__get_current_thread_instance();
+
+ if ((__reader_states::__is_quiescent_state(__current_thread_state_ref.load(memory_order_relaxed)))) {
+ // Entering a read-side critical section from a quiescent state.
+ __current_thread_state_ref.store(
+ __reader_states::__state_type(__global_reader_phase_.load(std::memory_order_relaxed) | uint16_t(1)),
+ memory_order_relaxed);
+ __cxx_atomic_thread_fence(memory_order_seq_cst);
+ } else {
+ // Already in read-side critical section, just increment the nest level.
+ __current_thread_state_ref.fetch_add(1, memory_order_relaxed);
+ }
+ }
+
+ bool try_lock() {
+ lock();
+ return true;
+ }
+
+ void unlock() {
+ auto __current_thread_state_ref = __reader_states_.__per_thread_states_.__get_current_thread_instance();
+ __cxx_atomic_thread_fence(memory_order_seq_cst);
+ // Decrement the nest level.
+ auto __old_state = __current_thread_state_ref.fetch_sub(1, memory_order_relaxed);
+
+ if (__reader_states::__get_reader_nest_level(__old_state) == 1 && __waiting_flag_.load(memory_order_relaxed)) {
+ // Transitioning to quiescent state, wake up waiters.
+ __waiting_flag_.store(false, std::memory_order_relaxed);
+ __waiting_flag_.notify_all();
+ }
+ }
+};
+
+struct __rcu_domain_access {
+ static rcu_domain& __rcu_default_domain() noexcept { return rcu_domain::__rcu_default_domain(); }
+ static void __rcu_synchronize(rcu_domain& __dom) noexcept { __dom.__synchronize(); }
+};
+
+// todo put it in the experimental dylib
+inline rcu_domain& rcu_default_domain() noexcept { return __rcu_domain_access::__rcu_default_domain(); }
+
+// todo put it in the experimental dylib
+inline void rcu_synchronize(rcu_domain& __dom = rcu_default_domain()) noexcept {
+ __rcu_domain_access::__rcu_synchronize(__dom);
+}
+
+void rcu_barrier(rcu_domain& dom = rcu_default_domain()) noexcept;
+template <class T, class D = default_delete<T>>
+void rcu_retire(T* p, D d = D(), rcu_domain& dom = rcu_default_domain());
+
#endif // _LIBCPP_STD_VER >= 26 && _LIBCPP_HAS_THREADS && _LIBCPP_HAS_EXPERIMENTAL_RCU
_LIBCPP_END_NAMESPACE_STD
diff --git a/libcxx/include/__rcu/rcu_list.h b/libcxx/include/__rcu/rcu_list.h
new file mode 100644
index 0000000000000..65ce35f1cd7a1
--- /dev/null
+++ b/libcxx/include/__rcu/rcu_list.h
@@ -0,0 +1,36 @@
+// -*- C++ -*-
+//===----------------------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef _LIBCPP___RCU_RCU_OBJ_BASE_H
+#define _LIBCPP___RCU_RCU_OBJ_BASE_H
+
+#include <__config>
+#include <__functional/function.h>
+#include <__rcu/rcu_domain.h>
+
+#if !defined(_LIBCPP_HAS_NO_PRAGMA_SYSTEM_HEADER)
+# pragma GCC system_header
+#endif
+
+_LIBCPP_BEGIN_NAMESPACE_STD
+
+#if _LIBCPP_STD_VER >= 26 && _LIBCPP_HAS_THREADS && _LIBCPP_HAS_EXPERIMENTAL_RCU
+
+struct __rcu_node {
+ function<void()> __callback_{};
+ __rcu_node* __next_ = nullptr;
+};
+
+struct __intrusive_linked_list_view {};
+
+#endif // _LIBCPP_STD_VER >= 26 && _LIBCPP_HAS_THREADS && _LIBCPP_HAS_EXPERIMENTAL_RCU
+
+_LIBCPP_END_NAMESPACE_STD
+
+#endif // _LIBCPP___RCU_RCU_OBJ_BASE_H
diff --git a/libcxx/include/__rcu/rcu_obj_base.h b/libcxx/include/__rcu/rcu_obj_base.h
index faff897a1f159..7b58b0c8dfee7 100644
--- a/libcxx/include/__rcu/rcu_obj_base.h
+++ b/libcxx/include/__rcu/rcu_obj_base.h
@@ -11,6 +11,9 @@
#define _LIBCPP___RCU_RCU_OBJ_BASE_H
#include <__config>
+#include <__memory/unique_ptr.h> // for default_delete
+#include <__rcu/rcu_domain.h>
+#include <__rcu/rcu_list.h>
#if !defined(_LIBCPP_HAS_NO_PRAGMA_SYSTEM_HEADER)
# pragma GCC system_header
@@ -20,6 +23,23 @@ _LIBCPP_BEGIN_NAMESPACE_STD
#if _LIBCPP_STD_VER >= 26 && _LIBCPP_HAS_THREADS && _LIBCPP_HAS_EXPERIMENTAL_RCU
+template <class T, class D = default_delete<T>>
+class rcu_obj_base : private __rcu_node {
+public:
+ void retire(D d = D(), rcu_domain& dom = rcu_default_domain()) noexcept;
+
+protected:
+ rcu_obj_base() = default;
+ rcu_obj_base(const rcu_obj_base&) = default;
+ rcu_obj_base(rcu_obj_base&&) = default;
+ rcu_obj_base& operator=(const rcu_obj_base&) = default;
+ rcu_obj_base& operator=(rcu_obj_base&&) = default;
+ ~rcu_obj_base() = default;
+
+private:
+ D deleter; // exposition only
+};
+
#endif // _LIBCPP_STD_VER >= 26 && _LIBCPP_HAS_THREADS && _LIBCPP_HAS_EXPERIMENTAL_RCU
_LIBCPP_END_NAMESPACE_STD
diff --git a/libcxx/include/__rcu/readme.md b/libcxx/include/__rcu/readme.md
new file mode 100644
index 0000000000000..ae83baf44dc61
--- /dev/null
+++ b/libcxx/include/__rcu/readme.md
@@ -0,0 +1,69 @@
+# folly
+
+
+## domain
+
+- `lock` :
+ - increment the counter with the epoch number == `_version`
+
+- `unlock`:
+ - decrement the counter (without arguments)
+
+- `retire` :
+ - push `cb` to `_q`
+ - if long time since last sync, try try_lock sync mutex and `half_sync` without blocking (to get some finished nodes)
+ - put the finished `cb`s to `executor` (immideately execute)
+
+- `half_sync` :
+ - `current` is the current `_version` number, `next` is +1
+ - move all nodes from `_q` to `queues[0]` (cannot just have a single `queue` and swap because of later readers) (thread safe with concurrent more push to `_q`)
+ - if is blocking, wait for zero for the epoch `next & 1`
+ - else, if epoch `next & 1` has reader is true, return
+ - at this stage, `next & 1` epoch reader zero, if late reader comes, it increments the `current` reader count. (no concurrent half_sync as it is mutex protected)
+ - move all nodes from `queue[1]` to `finished`, and move all nodes from `queue[0]` to `queue[1]`
+ - store `next` to `_version`
+ - notify threads that turn_.waitForTurn(next)
+
+- `sync`
+ - `current` is the current `_version` number, `target` is +2
+ - while true
+ - if current `work_` is smaller than `target` and cas to `target` succeeded,
+ `half_sync` until `version_` >= `target`, and run all the finished `cb`s, `return`
+ - else
+ - if `version_` >= `target` , `return`
+ - else `turn_.waitForTurn(work)` (so if other's target >= our target, other's second epoch half sync will unblock us)
+
+
+
+## executor
+
+- immediately invoke
+- queue if the `f` schedule another `f2` in the `executor`
+
+
+## example
+
+version == 0
+
+ T1 T2 T3
+
+Reader1 lock (0, 1)
+read obj0
+ obj = obj1
+ retire(obj0)
+ sync, target = 2
+ half, next = 1
+ wait_zero(epoch == 1)
+ cb0 -> queue[1]
+ version = 1
+ notify(1)
+ Read2 lock (1, 1)
+ read obj1
+ half, next = 2
+unlock (0,0)
+ wait_zero(epoch==0)
+ cb0 -> finished
+ run cb0
+ version = 2
+ notify(2)
+ unlock
diff --git a/libcxx/include/module.modulemap.in b/libcxx/include/module.modulemap.in
index 7d3c90a1aa2e8..230994c833553 100644
--- a/libcxx/include/module.modulemap.in
+++ b/libcxx/include/module.modulemap.in
@@ -2065,6 +2065,7 @@ module std {
module rcu {
module rcu_domain { header "__rcu/rcu_domain.h" }
+ module rcu_list { header "__rcu/rcu_list.h" }
module rcu_obj_base { header "__rcu/rcu_obj_base.h" }
header "rcu"
export *
diff --git a/libcxx/test/std/thread/saferecl/saferecl.rcu/general.pass.cpp b/libcxx/test/std/thread/saferecl/saferecl.rcu/general.pass.cpp
new file mode 100644
index 0000000000000..88412976f7a62
--- /dev/null
+++ b/libcxx/test/std/thread/saferecl/saferecl.rcu/general.pass.cpp
@@ -0,0 +1,117 @@
+//===----------------------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+//
+// UNSUPPORTED: no-threads
+// UNSUPPORTED: c++03, c++11, c++14, c++17, c++20, c++23
+
+// <rcu>
+
+#include <rcu>
+#include <thread>
+#include <chrono>
+#include <iostream>
+#include <print>
+
+#include "__rcu/rcu_domain.h"
+#include "make_test_thread.h"
+#include "test_macros.h"
+
+void log(auto start, std::string_view msg) {
+ auto now = std::chrono::system_clock::now();
+ std::println(std::cout, "[{:%H:%M:%S}] {}", now - start, msg);
+}
+
+int loop_num = 1;
+
+int main(int, char**) {
+ auto start = std::chrono::system_clock::now();
+ auto t1 = support::make_test_thread([start]() {
+ std::rcu_domain& dom = std::rcu_default_domain();
+ for (int i = 0; i < loop_num; ++i) {
+ log(start, "t1: entering rcu read-side critical section " + std::to_string(i));
+ dom.lock();
+ log(start, "t1: sleeping");
+ std::this_thread::sleep_for(std::chrono::seconds(1));
+ log(start, "t1: leaving rcu read-side critical section " + std::to_string(i));
+ dom.unlock();
+ log(start, "t1: printing all reader states");
+ dom.printAllReaderStatesInHex();
+ }
+ });
+
+ auto t2 = support::make_test_thread([start]() {
+ std::this_thread::sleep_for(std::chrono::milliseconds(200));
+ std::rcu_domain& dom = std::rcu_default_domain();
+ for (int i = 0; i < loop_num; ++i) {
+ log(start, "t2: entering rcu read-side critical section " + std::to_string(i));
+ dom.lock();
+ log(start, "t2: sleeping");
+ std::this_thread::sleep_for(std::chrono::seconds(1));
+ log(start, "t2: leaving rcu read-side critical section " + std::to_string(i));
+ dom.unlock();
+ log(start, "t2: printing all reader states");
+ dom.printAllReaderStatesInHex();
+ }
+ });
+
+ auto t3 = support::make_test_thread([start]() {
+ std::this_thread::sleep_for(std::chrono::milliseconds(1100));
+ std::rcu_domain& dom = std::rcu_default_domain();
+ for (int i = 0; i < loop_num; ++i) {
+ log(start, "t3: entering rcu read-side critical section " + std::to_string(i));
+ dom.lock();
+ log(start, "t3: sleeping");
+ std::this_thread::sleep_for(std::chrono::seconds(1));
+ log(start, "t3: leaving rcu read-side critical section " + std::to_string(i));
+ dom.unlock();
+ log(start, "t3: printing all reader states");
+ dom.printAllReaderStatesInHex();
+ }
+ });
+ auto t4 = support::make_test_thread([start]() {
+ std::this_thread::sleep_for(std::chrono::milliseconds(3000));
+ std::rcu_domain& dom = std::rcu_default_domain();
+ for (int i = 0; i < loop_num; ++i) {
+ log(start, "t4: entering rcu read-side critical section " + std::to_string(i));
+ dom.lock();
+ log(start, "t4: sleeping");
+ std::this_thread::sleep_for(std::chrono::seconds(1));
+ log(start, "t4: leaving rcu read-side critical section " + std::to_string(i));
+ dom.unlock();
+ log(start, "t4: printing all reader states");
+ dom.printAllReaderStatesInHex();
+ }
+ });
+ /*
+ auto t3 = support::make_test_thread([start]() {
+ std::rcu_domain& dom = std::rcu_default_domain();
+ log(start, "t3: entering rcu read-side critical section");
+ dom.lock();
+ log(start, "t3: sleeping");
+ std::this_thread::sleep_for(std::chrono::seconds(3));
+ log(start, "t3: leaving rcu read-side critical section");
+ dom.unlock();
+ }); */
+
+ std::this_thread::sleep_for(std::chrono::milliseconds(10));
+ for (int i = 0; i < loop_num+5; ++i) {
+
+ log(start, "t0: calling rcu_synchronize" + std::to_string(i));
+ std::rcu_synchronize();
+ log(start, "t0: rcu_synchronize returned" + std::to_string(i));
+ std::this_thread::sleep_for(std::chrono::seconds(1));
+ }
+
+
+ t1.join();
+ t2.join();
+ t3.join();
+ t4.join();
+
+ return 1;
+}
>From a9e729d717c166ed2a17539548bd61c8330c9ed6 Mon Sep 17 00:00:00 2001
From: Hui Xie <hui.xie1990 at gmail.com>
Date: Sun, 11 Jan 2026 18:59:42 +0000
Subject: [PATCH 03/65] callbacks
---
libcxx/include/__rcu/rcu_domain.h | 76 +++++++--
libcxx/include/__rcu/rcu_list.h | 51 +++++-
libcxx/include/__rcu/rcu_obj_base.h | 8 +-
.../read_write_lock_compare.pass.cpp | 154 ++++++++++++++++++
.../saferecl/saferecl.rcu/retire.pass.cpp | 150 +++++++++++++++++
5 files changed, 417 insertions(+), 22 deletions(-)
create mode 100644 libcxx/test/std/thread/saferecl/saferecl.rcu/read_write_lock_compare.pass.cpp
create mode 100644 libcxx/test/std/thread/saferecl/saferecl.rcu/retire.pass.cpp
diff --git a/libcxx/include/__rcu/rcu_domain.h b/libcxx/include/__rcu/rcu_domain.h
index fbd19933e171d..8863849f09362 100644
--- a/libcxx/include/__rcu/rcu_domain.h
+++ b/libcxx/include/__rcu/rcu_domain.h
@@ -13,6 +13,9 @@
#include <__config>
#include <__atomic/atomic.h>
+#include <__rcu/rcu_list.h>
+
+// todo replace with internal headers
#include <atomic>
#include <cstdint>
#include <map>
@@ -45,6 +48,8 @@ class __thread_local_owner {
__owned_instances_.emplace_back(__obj);
}
+ // todo: deregister on thread exit?
+
public:
__thread_local_owner() = default;
__thread_local_owner(__thread_local_owner&&) = delete;
@@ -99,19 +104,32 @@ struct __reader_states {
};
class rcu_domain {
+ // todo optimize the layout
+
__reader_states __reader_states_;
// only the highest bit is used for the phase.
std::atomic<__reader_states::__state_type> __global_reader_phase_{};
// only one thread is allowed to update concurrently
- std::mutex __reclamation_mutex_; // todo this is not noexcept
+ std::mutex __grace_period_mutex_; // todo this is not noexcept
+
+ std::atomic<bool> __grace_period_waiting_flag_ = false;
- std::atomic<bool> __waiting_flag_ = false;
+ // todo: maybe use a lock-free queue
+ std::mutex __retire_queue_mutex_; // todo this is not noexcept
+ __rcu_singly_list_view __retired_callback_queue_;
+
+ // these two queues do not need extra synchronization
+ // as they are always processed under the grace period mutex
+ __rcu_singly_list_view __callbacks_phase_1_;
+ __rcu_singly_list_view __callbacks_phase_2_;
rcu_domain() = default;
friend struct __rcu_domain_access;
+ template <class, class>
+ friend class rcu_obj_base;
// todo put globals in dylib
static rcu_domain& __rcu_default_domain() noexcept {
@@ -119,32 +137,58 @@ class rcu_domain {
return __default_domain;
}
+ template <class Callback>
+ void __retire_callback(Callback&& __cb) noexcept {
+ auto* __node = new __rcu_node();
+ __node->__callback_ = std::forward<Callback>(__cb);
+ std::unique_lock __lk(__retire_queue_mutex_);
+ __retired_callback_queue_.__push_back(__node);
+ }
+
void __synchronize() noexcept {
__cxx_atomic_thread_fence(memory_order_seq_cst);
- std::unique_lock __lk(__reclamation_mutex_);
- std::printf("rcu_domain::__synchronize() going through phase 1\n");
- __update_phase_and_wait();
+ std::unique_lock __lk(__grace_period_mutex_);
+ //std::printf("rcu_domain::__synchronize() going through phase 1\n");
+ auto __ready_callbacks = __update_phase_and_wait();
+
+ // Invoke the ready callbacks outside of the grace period mutex
+ __lk.unlock();
+ __ready_callbacks.__for_each([](auto* __node) { __node->__callback_(); });
+ __lk.lock();
+
__barrier();
- std::printf("rcu_domain::__synchronize() going through phase 2\n");
- __update_phase_and_wait();
+ //std::printf("rcu_domain::__synchronize() going through phase 2\n");
+ __ready_callbacks = __update_phase_and_wait();
+
+ // Invoke the ready callbacks outside of the grace period mutex
__lk.unlock();
+ __ready_callbacks.__for_each([](auto* __node) { __node->__callback_(); });
__cxx_atomic_thread_fence(memory_order_seq_cst);
}
- void __update_phase_and_wait() noexcept {
+ __rcu_singly_list_view __update_phase_and_wait() noexcept {
+ std::unique_lock __retire_lk(__retire_queue_mutex_);
+ __callbacks_phase_1_.__splice_back(__retired_callback_queue_);
+ __retire_lk.unlock();
+
// Flip the global phase
auto __old_phase =
__global_reader_phase_.fetch_xor(__reader_states::__grace_period_phase_mask, std::memory_order_relaxed);
auto __new_phase = __old_phase ^ __reader_states::__grace_period_phase_mask;
- std::printf("rcu_domain::__update_phase_and_wait() new phase: 0x%04x\n", __new_phase);
+ //std::printf("rcu_domain::__update_phase_and_wait() new phase: 0x%04x\n", __new_phase);
__barrier();
// Wait for all threads to quiesce in the old phase
while (__any_reader_in_ongoing_grace_period(__new_phase)) {
- __waiting_flag_.store(true, std::memory_order_relaxed);
- __waiting_flag_.wait(true, std::memory_order_relaxed);
+ __grace_period_waiting_flag_.store(true, std::memory_order_relaxed);
+ __grace_period_waiting_flag_.wait(true, std::memory_order_relaxed);
}
- __waiting_flag_.store(false, std::memory_order_relaxed);
+ __grace_period_waiting_flag_.store(false, std::memory_order_relaxed);
+
+ __rcu_singly_list_view __ready_callbacks;
+ __ready_callbacks.__splice_back(__callbacks_phase_2_);
+ __callbacks_phase_2_.__splice_back(__callbacks_phase_1_);
+ return __ready_callbacks;
}
bool __any_reader_in_ongoing_grace_period(__reader_states::__state_type __global_phase) noexcept {
@@ -206,10 +250,11 @@ class rcu_domain {
// Decrement the nest level.
auto __old_state = __current_thread_state_ref.fetch_sub(1, memory_order_relaxed);
- if (__reader_states::__get_reader_nest_level(__old_state) == 1 && __waiting_flag_.load(memory_order_relaxed)) {
+ if (__reader_states::__get_reader_nest_level(__old_state) == 1 &&
+ __grace_period_waiting_flag_.load(memory_order_relaxed)) {
// Transitioning to quiescent state, wake up waiters.
- __waiting_flag_.store(false, std::memory_order_relaxed);
- __waiting_flag_.notify_all();
+ __grace_period_waiting_flag_.store(false, std::memory_order_relaxed);
+ __grace_period_waiting_flag_.notify_all();
}
}
};
@@ -228,6 +273,7 @@ inline void rcu_synchronize(rcu_domain& __dom = rcu_default_domain()) noexcept {
}
void rcu_barrier(rcu_domain& dom = rcu_default_domain()) noexcept;
+
template <class T, class D = default_delete<T>>
void rcu_retire(T* p, D d = D(), rcu_domain& dom = rcu_default_domain());
diff --git a/libcxx/include/__rcu/rcu_list.h b/libcxx/include/__rcu/rcu_list.h
index 65ce35f1cd7a1..2547d7223a7cc 100644
--- a/libcxx/include/__rcu/rcu_list.h
+++ b/libcxx/include/__rcu/rcu_list.h
@@ -7,8 +7,8 @@
//
//===----------------------------------------------------------------------===//
-#ifndef _LIBCPP___RCU_RCU_OBJ_BASE_H
-#define _LIBCPP___RCU_RCU_OBJ_BASE_H
+#ifndef _LIBCPP___RCU_RCU_LIST_H
+#define _LIBCPP___RCU_RCU_LIST_H
#include <__config>
#include <__functional/function.h>
@@ -27,10 +27,53 @@ struct __rcu_node {
__rcu_node* __next_ = nullptr;
};
-struct __intrusive_linked_list_view {};
+class __rcu_singly_list_view {
+private:
+ __rcu_node* __head_ = nullptr;
+ __rcu_node* __tail_ = nullptr;
+
+public:
+ void __splice_back(__rcu_singly_list_view& __other) noexcept {
+ if (__other.__head_ == nullptr) {
+ return;
+ }
+ if (__head_ == nullptr) {
+ __head_ = __other.__head_;
+ __tail_ = __other.__tail_;
+ } else {
+ __tail_->__next_ = __other.__head_;
+ __tail_ = __other.__tail_;
+ }
+ __other.__head_ = nullptr;
+ __other.__tail_ = nullptr;
+ }
+
+ void __push_back(__rcu_node* __node) noexcept {
+ // assert(__node->__next_ == nullptr);
+ if (__head_ == nullptr) {
+ __head_ = __node;
+ __tail_ = __node;
+ } else {
+ __tail_->__next_ = __node;
+ __tail_ = __node;
+ }
+ }
+
+ template <class _Func>
+ void __for_each(_Func&& __f) noexcept {
+ __rcu_node* __current = __head_;
+ while (__current != nullptr) {
+ // __f could delete __current, so we need to get the next pointer first
+ auto __next = __current->__next_;
+ __f(__current);
+ __current = __next;
+ }
+ }
+};
+
#endif // _LIBCPP_STD_VER >= 26 && _LIBCPP_HAS_THREADS && _LIBCPP_HAS_EXPERIMENTAL_RCU
_LIBCPP_END_NAMESPACE_STD
-#endif // _LIBCPP___RCU_RCU_OBJ_BASE_H
+#endif // _LIBCPP___RCU_RCU_LIST_H
diff --git a/libcxx/include/__rcu/rcu_obj_base.h b/libcxx/include/__rcu/rcu_obj_base.h
index 7b58b0c8dfee7..3d1de353613a5 100644
--- a/libcxx/include/__rcu/rcu_obj_base.h
+++ b/libcxx/include/__rcu/rcu_obj_base.h
@@ -26,7 +26,11 @@ _LIBCPP_BEGIN_NAMESPACE_STD
template <class T, class D = default_delete<T>>
class rcu_obj_base : private __rcu_node {
public:
- void retire(D d = D(), rcu_domain& dom = rcu_default_domain()) noexcept;
+ void retire(D d = D(), rcu_domain& dom = rcu_default_domain()) noexcept {
+ auto ptr = static_cast<T*>(this);
+ dom.__retire_callback(
+ [ptr, d = std::move(d)]() mutable { d(ptr); });
+ }
protected:
rcu_obj_base() = default;
@@ -36,8 +40,6 @@ class rcu_obj_base : private __rcu_node {
rcu_obj_base& operator=(rcu_obj_base&&) = default;
~rcu_obj_base() = default;
-private:
- D deleter; // exposition only
};
#endif // _LIBCPP_STD_VER >= 26 && _LIBCPP_HAS_THREADS && _LIBCPP_HAS_EXPERIMENTAL_RCU
diff --git a/libcxx/test/std/thread/saferecl/saferecl.rcu/read_write_lock_compare.pass.cpp b/libcxx/test/std/thread/saferecl/saferecl.rcu/read_write_lock_compare.pass.cpp
new file mode 100644
index 0000000000000..2fc79013994e1
--- /dev/null
+++ b/libcxx/test/std/thread/saferecl/saferecl.rcu/read_write_lock_compare.pass.cpp
@@ -0,0 +1,154 @@
+//===----------------------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+//
+// UNSUPPORTED: no-threads
+// UNSUPPORTED: c++03, c++11, c++14, c++17, c++20, c++23
+
+// <rcu>
+
+#include <atomic>
+#include <rcu>
+#include <shared_mutex>
+#include <stop_token>
+#include <thread>
+#include <chrono>
+#include <iostream>
+#include <print>
+#include <string>
+
+#include "__rcu/rcu_domain.h"
+#include "make_test_thread.h"
+#include "test_macros.h"
+
+constexpr int num_reader = 4;
+const std::chrono::seconds test_time(10);
+
+struct alignas(128) MyObject : public std::rcu_obj_base<MyObject> {
+ std::string data_;
+
+ inline static int instance_count = 0;
+ MyObject() : data_(std::to_string(instance_count++) + " instance very very very long string") {}
+
+ void doWork() {
+ auto spin_for = [](std::chrono::microseconds us) {
+ auto start = std::chrono::high_resolution_clock::now();
+ while (std::chrono::high_resolution_clock::now() - start < us)
+ ;
+ };
+ using namespace std::chrono_literals;
+ spin_for(10us);
+ }
+};
+
+
+void test_read_write_lock() {
+ MyObject* globalObjRWLock = new MyObject();
+ std::shared_mutex globalObjMutex;
+
+ std::vector<std::jthread> readers;
+ readers.reserve(num_reader);
+
+ auto reader_func = [&globalObjRWLock, &globalObjMutex](std::stop_token token) {
+ int read_count = 0;
+ while (!token.stop_requested()) {
+ std::shared_lock<std::shared_mutex> lock(globalObjMutex);
+ globalObjRWLock->doWork();
+ ++read_count;
+ }
+ std::println("Reader thread read {} times", read_count);
+ };
+
+ auto writer_func = [&globalObjRWLock, &globalObjMutex](std::stop_token token) {
+ int write_count = 0;
+ while (!token.stop_requested()) {
+ auto newObj = new MyObject();
+ std::unique_lock<std::shared_mutex> lock(globalObjMutex);
+ auto oldObj = globalObjRWLock;
+ globalObjRWLock = newObj;
+ lock.unlock();
+ delete oldObj;
+ ++write_count;
+ std::this_thread::sleep_for(std::chrono::microseconds(100));
+ }
+ std::println("Writer thread wrote {} times", write_count);
+ };
+
+ for (int i = 0; i < num_reader; ++i) {
+ readers.emplace_back(reader_func);
+ }
+ std::jthread writer(writer_func);
+
+ std::this_thread::sleep_for(test_time);
+
+ for (auto& reader : readers) {
+ reader.request_stop();
+ }
+ writer.request_stop();
+}
+
+void test_rcu() {
+ std::rcu_domain& dom = std::rcu_default_domain();
+ std::atomic<MyObject*> global_obj_rcu = new MyObject();
+
+ std::vector<std::jthread> readers;
+ readers.reserve(num_reader);
+
+ auto reader_func = [&dom, &global_obj_rcu](std::stop_token token) {
+ int read_count = 0;
+ while (!token.stop_requested()) {
+ dom.lock();
+ auto obj = global_obj_rcu.load(std::memory_order_relaxed);
+ obj->doWork();
+ dom.unlock();
+ ++read_count;
+ }
+ std::println("RCU Reader thread read {} times", read_count);
+ };
+
+ auto writer_func = [&global_obj_rcu](std::stop_token token) {
+ int write_count = 0;
+ while (!token.stop_requested()) {
+ auto newObj = new MyObject();
+ auto oldObj = global_obj_rcu.exchange(newObj, std::memory_order_relaxed);
+ oldObj->retire();
+ ++write_count;
+ std::this_thread::sleep_for(std::chrono::microseconds(100));
+ }
+ std::println("RCU Writer thread wrote {} times", write_count);
+ };
+
+ auto syncer_func = [&dom](std::stop_token token) {
+ while (!token.stop_requested()) {
+ std::rcu_synchronize(dom);
+ std::this_thread::sleep_for(std::chrono::milliseconds(50));
+ }
+ };
+
+ for (int i = 0; i < num_reader; ++i) {
+ readers.emplace_back(reader_func);
+ }
+ std::jthread writer(writer_func);
+ std::jthread syncer(syncer_func);
+
+ std::this_thread::sleep_for(test_time);
+
+ for (auto& reader : readers) {
+ reader.request_stop();
+ }
+ writer.request_stop();
+ syncer.request_stop();
+ std::rcu_synchronize(dom);
+}
+
+int main(int, char**) {
+ std::println("Testing read-write lock:");
+ test_read_write_lock();
+ std::println("Testing RCU:");
+ test_rcu();
+ return 1;
+}
diff --git a/libcxx/test/std/thread/saferecl/saferecl.rcu/retire.pass.cpp b/libcxx/test/std/thread/saferecl/saferecl.rcu/retire.pass.cpp
new file mode 100644
index 0000000000000..dde6cdd323084
--- /dev/null
+++ b/libcxx/test/std/thread/saferecl/saferecl.rcu/retire.pass.cpp
@@ -0,0 +1,150 @@
+//===----------------------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+//
+// UNSUPPORTED: no-threads
+// UNSUPPORTED: c++03, c++11, c++14, c++17, c++20, c++23
+
+// <rcu>
+
+#include <rcu>
+#include <thread>
+#include <chrono>
+#include <iostream>
+#include <print>
+#include <string>
+
+#include "__rcu/rcu_domain.h"
+#include "make_test_thread.h"
+#include "test_macros.h"
+
+void log(auto start, std::string_view msg) {
+ auto now = std::chrono::system_clock::now();
+ std::println(std::cout, "[{:%H:%M:%S}] {}", now - start, msg);
+}
+
+int loop_num = 10;
+
+struct MyObject : public std::rcu_obj_base<MyObject> {
+ std::string data_;
+
+ inline static int instance_count = 0;
+ MyObject() : data_(std::to_string(instance_count++) + " instance very very very long string") {}
+
+ ~MyObject() {
+ std::println(std::cout, "MyObject {} destructor called", data_);
+ }
+};
+
+std::atomic<MyObject*> global_obj = nullptr;
+
+int main(int, char**) {
+ auto start = std::chrono::system_clock::now();
+ auto t1 = support::make_test_thread([start]() {
+ std::rcu_domain& dom = std::rcu_default_domain();
+ for (int i = 0; i < loop_num; ++i) {
+ log(start, "t1: entering rcu read-side critical section " + std::to_string(i));
+ dom.lock();
+ auto obj = global_obj.load();
+ log(start, "t1: reading: " + (obj ? obj->data_ : "nullptr"));
+ std::this_thread::sleep_for(std::chrono::seconds(1));
+ log(start, "t1: leaving rcu read-side critical section " + std::to_string(i) + " with object " +
+ (obj ? obj->data_ : "nullptr"));
+ dom.unlock();
+ log(start, "t1: printing all reader states");
+ dom.printAllReaderStatesInHex();
+ }
+ });
+
+ auto t2 = support::make_test_thread([start]() {
+ std::this_thread::sleep_for(std::chrono::milliseconds(200));
+ std::rcu_domain& dom = std::rcu_default_domain();
+ for (int i = 0; i < loop_num; ++i) {
+ log(start, "t2: entering rcu read-side critical section " + std::to_string(i));
+ dom.lock();
+ auto obj = global_obj.load();
+ log(start, "t2: reading: " + (obj ? obj->data_ : "nullptr"));
+ std::this_thread::sleep_for(std::chrono::seconds(1));
+ log(start, "t2: leaving rcu read-side critical section " + std::to_string(i) + " with object " +
+ (obj ? obj->data_ : "nullptr"));
+ dom.unlock();
+ log(start, "t2: printing all reader states");
+ dom.printAllReaderStatesInHex();
+ }
+ });
+
+ auto t3 = support::make_test_thread([start]() {
+ std::this_thread::sleep_for(std::chrono::milliseconds(1100));
+ std::rcu_domain& dom = std::rcu_default_domain();
+ for (int i = 0; i < loop_num; ++i) {
+ log(start, "t3: entering rcu read-side critical section " + std::to_string(i));
+ dom.lock();
+ auto obj = global_obj.load();
+ log(start, "t3: reading: " + (obj ? obj->data_ : "nullptr"));
+ std::this_thread::sleep_for(std::chrono::seconds(1));
+ log(start, "t3: leaving rcu read-side critical section " + std::to_string(i) + " with object " +
+ (obj ? obj->data_ : "nullptr"));
+ dom.unlock();
+ log(start, "t3: printing all reader states");
+ dom.printAllReaderStatesInHex();
+ }
+ });
+ auto t4 = support::make_test_thread([start]() {
+ std::this_thread::sleep_for(std::chrono::milliseconds(3000));
+ std::rcu_domain& dom = std::rcu_default_domain();
+ for (int i = 0; i < loop_num; ++i) {
+ log(start, "t4: entering rcu read-side critical section " + std::to_string(i));
+ dom.lock();
+ auto obj = global_obj.load();
+ log(start, "t4: reading: " + (obj ? obj->data_ : "nullptr"));
+ std::this_thread::sleep_for(std::chrono::seconds(1));
+ log(start, "t4: leaving rcu read-side critical section " + std::to_string(i) + " with object " +
+ (obj ? obj->data_ : "nullptr"));
+ dom.unlock();
+ log(start, "t4: printing all reader states");
+ dom.printAllReaderStatesInHex();
+ }
+ });
+
+ auto t5 = support::make_test_thread([start]() {
+ std::this_thread::sleep_for(std::chrono::milliseconds(3000));
+ for (int i = 0; i < loop_num; ++i) {
+ auto new_obj = new MyObject();
+
+ log(start, "t5: updating global to : " + new_obj->data_);
+ auto old = global_obj.exchange(new_obj);
+ log(start, "t5: retiring old object " + (old ? old->data_ : "nullptr"));
+ old->retire();
+ std::this_thread::sleep_for(std::chrono::seconds(1));
+ }
+ });
+ /*
+ auto t3 = support::make_test_thread([start]() {
+ std::rcu_domain& dom = std::rcu_default_domain();
+ log(start, "t3: entering rcu read-side critical section");
+ dom.lock();
+ log(start, "t3: sleeping");
+ std::this_thread::sleep_for(std::chrono::seconds(3));
+ log(start, "t3: leaving rcu read-side critical section");
+ dom.unlock();
+ }); */
+
+ std::this_thread::sleep_for(std::chrono::milliseconds(10));
+ for (int i = 0; i < loop_num + 5; ++i) {
+ log(start, "t0: calling rcu_synchronize" + std::to_string(i));
+ std::rcu_synchronize();
+ log(start, "t0: rcu_synchronize returned" + std::to_string(i));
+ std::this_thread::sleep_for(std::chrono::seconds(1));
+ }
+
+ t1.join();
+ t2.join();
+ t3.join();
+ t4.join();
+
+ return 1;
+}
>From 1d44f5caa2ac6ff21034c13b10ce50466aef924a Mon Sep 17 00:00:00 2001
From: Hui Xie <hui.xie1990 at gmail.com>
Date: Sun, 25 Jan 2026 10:46:22 +0000
Subject: [PATCH 04/65] review
---
libcxx/include/__rcu/rcu_domain.h | 12 +++---
libcxx/include/__rcu/readme.md | 69 -------------------------------
2 files changed, 6 insertions(+), 75 deletions(-)
delete mode 100644 libcxx/include/__rcu/readme.md
diff --git a/libcxx/include/__rcu/rcu_domain.h b/libcxx/include/__rcu/rcu_domain.h
index 8863849f09362..3ac3597280c25 100644
--- a/libcxx/include/__rcu/rcu_domain.h
+++ b/libcxx/include/__rcu/rcu_domain.h
@@ -33,17 +33,17 @@ _LIBCPP_BEGIN_NAMESPACE_STD
#if _LIBCPP_STD_VER >= 26 && _LIBCPP_HAS_THREADS && _LIBCPP_HAS_EXPERIMENTAL_RCU
-template <class _ThreadLocal>
+template <class _Tp>
class __thread_local_owner {
// todo put globals in experimental dylib
- inline static thread_local map<const __thread_local_owner*, _ThreadLocal> __thread_local_instances;
+ inline static thread_local map<const __thread_local_owner*, _Tp> __thread_local_instances;
// Keep track of all thread-local instances owned by this owner.
// Only emplaced the first time a thread is trying to access its thread-local instance.
- vector<atomic_ref<_ThreadLocal>> __owned_instances_;
+ vector<atomic_ref<_Tp>> __owned_instances_;
mutex __mtx_;
- void __register(_ThreadLocal& __obj) {
+ void __register(_Tp& __obj) {
lock_guard<std::mutex> __lg(__mtx_);
__owned_instances_.emplace_back(__obj);
}
@@ -54,10 +54,10 @@ class __thread_local_owner {
__thread_local_owner() = default;
__thread_local_owner(__thread_local_owner&&) = delete;
- atomic_ref<_ThreadLocal> __get_current_thread_instance() {
+ atomic_ref<_Tp> __get_current_thread_instance() {
auto __it = __thread_local_instances.find(this);
if (__it == __thread_local_instances.end()) {
- auto [new_it, _] = __thread_local_instances.try_emplace(this, _ThreadLocal());
+ auto [new_it, _] = __thread_local_instances.try_emplace(this, _Tp());
auto& __obj = new_it->second;
__register(__obj);
return atomic_ref(__obj);
diff --git a/libcxx/include/__rcu/readme.md b/libcxx/include/__rcu/readme.md
deleted file mode 100644
index ae83baf44dc61..0000000000000
--- a/libcxx/include/__rcu/readme.md
+++ /dev/null
@@ -1,69 +0,0 @@
-# folly
-
-
-## domain
-
-- `lock` :
- - increment the counter with the epoch number == `_version`
-
-- `unlock`:
- - decrement the counter (without arguments)
-
-- `retire` :
- - push `cb` to `_q`
- - if long time since last sync, try try_lock sync mutex and `half_sync` without blocking (to get some finished nodes)
- - put the finished `cb`s to `executor` (immideately execute)
-
-- `half_sync` :
- - `current` is the current `_version` number, `next` is +1
- - move all nodes from `_q` to `queues[0]` (cannot just have a single `queue` and swap because of later readers) (thread safe with concurrent more push to `_q`)
- - if is blocking, wait for zero for the epoch `next & 1`
- - else, if epoch `next & 1` has reader is true, return
- - at this stage, `next & 1` epoch reader zero, if late reader comes, it increments the `current` reader count. (no concurrent half_sync as it is mutex protected)
- - move all nodes from `queue[1]` to `finished`, and move all nodes from `queue[0]` to `queue[1]`
- - store `next` to `_version`
- - notify threads that turn_.waitForTurn(next)
-
-- `sync`
- - `current` is the current `_version` number, `target` is +2
- - while true
- - if current `work_` is smaller than `target` and cas to `target` succeeded,
- `half_sync` until `version_` >= `target`, and run all the finished `cb`s, `return`
- - else
- - if `version_` >= `target` , `return`
- - else `turn_.waitForTurn(work)` (so if other's target >= our target, other's second epoch half sync will unblock us)
-
-
-
-## executor
-
-- immediately invoke
-- queue if the `f` schedule another `f2` in the `executor`
-
-
-## example
-
-version == 0
-
- T1 T2 T3
-
-Reader1 lock (0, 1)
-read obj0
- obj = obj1
- retire(obj0)
- sync, target = 2
- half, next = 1
- wait_zero(epoch == 1)
- cb0 -> queue[1]
- version = 1
- notify(1)
- Read2 lock (1, 1)
- read obj1
- half, next = 2
-unlock (0,0)
- wait_zero(epoch==0)
- cb0 -> finished
- run cb0
- version = 2
- notify(2)
- unlock
>From c225d3f15eb3e7cd26cb50f42c46ca63e9a75a6c Mon Sep 17 00:00:00 2001
From: Hui Xie <hui.xie1990 at gmail.com>
Date: Sun, 22 Feb 2026 20:33:51 +0000
Subject: [PATCH 05/65] design doc
---
libcxx/docs/DesignDocs/RCU.rst | 312 +++++++++++++++++++++++++++++++++
libcxx/docs/index.rst | 1 +
2 files changed, 313 insertions(+)
create mode 100644 libcxx/docs/DesignDocs/RCU.rst
diff --git a/libcxx/docs/DesignDocs/RCU.rst b/libcxx/docs/DesignDocs/RCU.rst
new file mode 100644
index 0000000000000..c87ae8cde5677
--- /dev/null
+++ b/libcxx/docs/DesignDocs/RCU.rst
@@ -0,0 +1,312 @@
+====================
+``<rcu>`` Design
+====================
+
+.. contents::
+ :local:
+ :depth: 2
+
+
+Introduction
+============
+
+This is the C++ paper `Read-Copy Update (RCU) <https://www.open-std.org/jtc1/sc22/wg21/docs/papers/2023/p2545r4.pdf>`__.
+There are three designs discussed in the paper `Supplementary Material for User-Level Implementations of
+Read-Copy Update <http://www.rdrop.com/users/paulmck/RCU/urcu-supp-accepted.2011.08.30a.pdf>`__
+
+- Quiescent-State-Based Reclamation RCU
+- General-Purpose RCU
+- Low-Overhead RCU Via Signal Handling
+
+libc++ adopted a variation that closely resembles the "General-Purpose RCU" design in the above paper.
+
+
+Background Information
+======================
+
+`rcu` is an alternative to read-write locks and it is suitable for read-mostly data structures.
+Here is an example usage ::
+
+ struct Data /* members */ ;
+ std::atomic<Data*> data_;
+
+ // ==============================================
+ template <typename Func>
+ Result reader_op(Func fn) {
+ std::scoped_lock l(std::rcu_default_domain());
+ Data* p = data_;
+ // fn should not block too long or call
+ // rcu_synchronize(), rcu_barrier(), or
+ // rcu_retire(), directly or indirectly
+ return fn(p);
+ }
+
+ // ==============================================
+ // May be called concurrently with reader_op
+ void update(Data* newdata) {
+ Data* olddata = data_.exchange(newdata);
+ std::rcu_retire(olddata); // reclaim *olddata when safe
+ }
+
+There are several key details that an implementation of `rcu` must address:
+
+- On the reader side, `rcu_domain::lock` and `rcu_domain::unlock` must not block the thread while the writer thread
+ is performing updates.
+
+- On the writer side, `retire` -ing an object in principle should not block at all. However, it is conforming to
+ evaluate the `deleter` s inside `run_retire` .
+
+- On the writer side, `rcu_synchronize` should block until at least all the existing readers exit their critical sections
+ via `rcu_domain::unlock` . Note that this is a key difference between `rcu` and read-write locks:
+ In case a late reader enters the critical section after the writer thread has called `rcu_synchronize` and waiting, the
+ writer thread does not need to wait for the late reader to exit the critical section.
+
+- On the writer side, `run_barrier` should block until all the `retired` objects that happen before the `run_barrier` call are reclaimed.
+
+- The threads that are using `rcu` must be known by the `rcu` implementation states.
+
+
+Adopted Design
+==============
+
+The core idea of `rcu` can be described in this simple image from lwn.net
+
+.. image:: https://static.lwn.net/images/ns/kernel/rcu/GracePeriodGood.png
+
+- Each row is a thread. The last row is the writer thread and the rows above are the reader threads.
+- Each "Reader" block represents a critical section, which starts with `rcu_domain::lock` and ends with `rcu_domain::unlock` .
+- When `run_retire` is called from the writer thread, it starts the "Removal" block.
+- When `run_synchronize` is called from the writer thread, it starts the "Grace Period" block. We need to wait until all the
+ "Reader" blocks that started before the "Grace Period" started, to exit via `rcu_domain::unlock`, then we can end the "Grace Period".
+ Note that the "Grace Period" ends after the 4th row's "Reader" block ends. Also note that the "Grace Period" does not need to wait
+ for the late "Reader" blocks.
+- After the "Grace Period" ends, we can start the "Reclaim" block to reclaim the retired objects.
+
+Libc++ adopted a design that closely resembles the "General-Purpose RCU" design in the above paper.
+
+Some key details of this design are:
+
+- There is a global state which has two phases and it flips between the two phases.
+- Each thread stores its state: whether there is a reader in the critical section, and which phase it was when it entered the critical section.
+- When `run_synchronize` is called, it
+
+ - flips the global state to the next phase
+ - waits until all the threads that are in the critical section and entered in the previous phase to exit the critical section.
+ - flips the global state back to the original phase
+ - waits until all the threads that are in the critical section and entered in the next phase to exit the critical section.
+
+When `run_synchronize` returns, we can be sure that all the readers that were in the critical section before `run_synchronize` are now out of the critical section.
+The paper explains why we need to wait two phases instead of just one phase in detail. The key point is that, if we only wait for the readers in the previous phase to exit,
+there might be a late reader that enters the critical section after we flip the global state and before we wait for the previous phase's readers to exit.
+
+Implementation Terminology
+---------------------------
+
+- reader
+- grace period
+- quiescent state
+
+Implementation Details
+----------------------
+
+Design Questions
+================
+
+What thread(s) should the deleters run on
+-----------------------------------------
+
+Paul E. McKenney suggested to run those deleters on a background thread:
+
+ Folly currently uses an "inline executor", though some would like a
+ separate thread in some cases. The userspace RCU library always uses
+ background threads. By default only one, but there are ways to configure
+ more.
+
+We have doubts on this. As a standard library, we usually don't spin threads for users. And Paul E. McKenney
+has pointed out that:
+
+ Each library will have its own rules.
+ One advantage of an inline executor is that the deleters apply
+ backpressure against threads that flood the system with either .retire()
+ or rcu_retire(). Some compensation for the additional deadlocks,
+ I guess. But you can get the same effect by properly restricting where
+ the background threads run.
+
+And Thomas Rodgers (libstdc++'s RCU implementer) also said:
+
+ We don't normally do this, no, and I don't think I'd ever directly pursue such an approach in isolation. If,
+ on the other hand I could derive a permission from the user of library to create threads, by explicit use
+ of a system thread pool executor, perhaps then it would be ok to also use that executor to run other background
+ threads for things like this. Having said that, I think that is something that SG1 should probably discuss. On the
+ one hand spinning up a 'system' thread pool is going be potentially expensive and if the library user has done so,
+ you probably would not release those resources until termination. So, in that sense, you've 'paid' for a background
+ thread. But on the other hand, users of the library might find it surprising that then gave the library permission
+ to also do work on those threads in ways they might not expect.
+
+In libc++'s design, we would like to follow Folly's approach to run these deleters inline when `run_synchronize` or
+`run_barrier` is called.
+
+When should we run the deleters
+-------------------------------
+
+If we were to use the background thread approach, the deleters can be evaluated at any time after `run_synchronize`
+has marked the objects are safe to reclaim. e.g. Drain the queue periodically. However, if we were to run the deleters
+inline, we have to decide when to run them.
+
+There are few places we can run the deleters:
+
+- Inside `run_synchronize` after the writer thread is unblocked. This approach has the advantage that the writer thread
+ can reclaim the retired objects as soon as possible. However, it has the disadvantage that the writer thread may be
+ blocked for a long time if there are many retired objects to reclaim.
+
+- Inside `run_barrier` after the writer thread is unblocked. Since `run_barrier` is designed to block until all the retired
+ objects that happen before the `run_barrier` call are reclaimed, it is natural to run the deleters here.
+
+- Inside `run_retire` after the deleter is put into the queue. `run_retire` is designed not to block. However, if there are
+ objects that are safe to reclaim due to readers have exited their critical sections, and at the same time, there is no other
+ writer threads that are currently draining the deleter queue, we can take the opportunity to run the deleters.
+
+Folly currently takes the inline approach and runs the deleters at all three places mentioned above.
+
+In libc++'s design, we would like to start with the inline approach and only run the deleters inside `run_synchronize` and
+`run_barrier` for simplicity. We can consider running the deleters inside `run_retire` in the future if we find it is necessary
+to reduce the latency of reclaiming the retired objects.
+
+Almost all APIs are `noexcept` . Is it designed to avoid memory allocation and avoid using `mutex` ?
+-----------------------------------------------------------------------------------------------------
+
+We should avoid using `mutex` as much as we can. However, both Folly and Paul's implementation use `mutex` in some places.
+
+Paul E. McKenney mentioned that:
+
+ On the non-noexcept nature of mutexes, the best I could do was to say
+ that my reference implementation acquired its pthread_mutexes from C
+
+There are few places that might need to use `mutex`:
+
+- The first time a thread registers its existence with the `rcu` implementation. Depending on the implementation, this
+ can happen inside the `rcu_*` calls.
+
+- The add/remove of the deleter queue, if the queue is not a lock-free data structure.
+
+- The destruction of a thread, which might need to do some clean up work for the `rcu` states.
+
+Memory Allocation can throw as well. Luckily `run_retire` is not marked as `noexcept` in the paper, as `run_retire` is
+likely to allocate memory to create a type erased object into the deleter queue.
+
+However, there are other places where it might need to allocate memory. For example, see Folly's comments
+`here <https://github.com/facebook/folly/blob/c83b50725f554c4912a73b676746242a22211eca/folly/synchronization/Rcu.h#L343>`__ ::
+
+ * Note that despite the function being marked noexcept, an allocation
+ * may take place in folly::ThreadLocal the first time a thread enters a read
+ * region. Regardless, for now, we're marking this as noexcept to match the
+ * N4895 standard proposal:
+ *
+ * https://www.open-std.org/jtc1/sc22/wg21/docs/papers/2021/n4895.pdf
+ */
+ FOLLY_ALWAYS_INLINE void lock() noexcept {
+ counters_.increment(version_.load(std::memory_order_acquire));
+ }
+ FOLLY_ALWAYS_INLINE void unlock() noexcept { counters_.decrement(); }
+
+In libc++'s design, we will start with as less `mutex` usage as possible. In places where it is unavoidable, we could either
+wrap the internal API with a while loop, or use the `atomic_unique_lock` that we introduced with `stop_token` implementations.
+For memory allocations, we will only need them the first time we create some rcu_domain internal states or the first time registers a thread.
+We will consider whether we accepts a `terminate` call if this fails to allocate.
+
+
+How does `rcu` know about a thread?
+------------------------------------
+
+All design options require the `rcu_domain` to know whether a thread is in a critical section and stores each thread's state.
+The question is how does the `rcu_domain` know about a thread?
+
+Paul E. McKenney suggested that:
+
+ In a library, I also suggest automating each thread's initial call to register_thread().
+
+There are few options to automate the thread registration:
+
+- In the `std::thread` and `std::jthread` constructors, we can register them to the `rcu_domain` 's state. However, this approach
+ has the disadvantage that it does not cover threads that are not created by `std::thread` or `std::jthread` , for example, users
+ can create a thread through `pthread_create` and use that thread to call the `rcu` APIs. Another disadvantage is that most users
+ who create threads might not use `rcu` at all, so it might be wasteful to register all the threads by default.
+
+- In the `rcu_domain::lock` , we can check whether the current thread is registered, if not, we can register it. This approach has
+ the advantage that it can cover all threads, and it does not have the overhead of registering threads that do not use `rcu` at all.
+ However, it has the disadvantage that it might have a performance impact on the first call to `rcu_domain::lock` for each thread.
+ As the performance of `rcu_domain::lock` is critical, we need to make sure the thread registration process is as efficient as possible.
+
+Folly uses the second approach to automate the thread registration. Another caveat of this approach in Folly is that, the first call to
+`rcu_domain::lock` from a thread requires allocating memory to create the thread's state in the `rcu_domain` and it also needs to lock a
+`mutex` to protect the thread registration process.
+
+In libc++'s design, we will also use the second approach to automate the thread registration.
+
+Is `rcu_domain` singleton?
+--------------------------
+
+The APIs take a reference to `rcu_domain` with default value::
+
+ void rcu_synchronize(rcu_domain& dom = rcu_default_domain()) noexcept;
+
+ void rcu_barrier(rcu_domain& dom = rcu_default_domain()) noexcept;
+
+ template<class T, class D = default_delete<T>>
+ void rcu_retire(T* p, D d = D(), rcu_domain& dom = rcu_default_domain());
+
+However, the class `rcu_domain` has no user visible constructors::
+
+ class rcu_domain {
+ public:
+ rcu_domain(const rcu_domain&) = delete;
+ rcu_domain& operator=(const rcu_domain&) = delete;
+ };
+
+Paul E. McKenney confirmed that:
+
+ Yes, right now, just the one instance. Right now, its only purpose is to support C++ RAII RCU readers.
+ Later, we might add the ability to create separate RCU instances, similar to SRCU in the Linux kernel.
+
+Thomas Rodgers (libstdc++'s RCU implementer) also said:
+
+ Currently, liburcu doesn't actually support discrete domains, so internal to the shared library there is
+ only a single static instance of a type that wraps the relevant operations from the liburcu sources.
+ The `uintptr_t`` member is initialized with the address of this instance, and its size. Future implementations
+ could do something different behind this ABI contract, including a non-liburcu based approach.
+
+In libc++'s design, we will follow the same singleton pattern to simplify the design and implementation. Otherwise,
+lots of thread local storages will require a per-instance state.
+
+What about the ABI?
+-------------------
+
+The `rcu` library most functions are non-template. So it is possible to put the entire implementation details inside the dylib.
+
+Existing Implementations
+========================
+
+The C++ paper `Read-Copy Update (RCU) <https://www.open-std.org/jtc1/sc22/wg21/docs/papers/2023/p2545r4.pdf>`__
+discussed two reference implementations:
+
+- Folly
+- liburcu
+
+Folly
+-----
+
+liburcu
+-------
+
+There are 4 flavors of `rcu` implementations in liburcu (see doc `here <https://github.com/urcu/userspace-rcu/blob/master/README.md#usage-of-all-urcu-libraries>`__):
+
+- `memb`
+- `qsbr`
+- `mb`
+- `bp`
+
+
+libstdc++
+---------
+
+
diff --git a/libcxx/docs/index.rst b/libcxx/docs/index.rst
index e4f337270231e..abfe759fe36f2 100644
--- a/libcxx/docs/index.rst
+++ b/libcxx/docs/index.rst
@@ -226,6 +226,7 @@ Design Documents
DesignDocs/HeaderRemovalPolicy
DesignDocs/NoexceptPolicy
DesignDocs/PSTLIntegration
+ DesignDocs/RCU
DesignDocs/ThreadingSupportAPI
DesignDocs/UniquePtrTrivialAbi
DesignDocs/UnspecifiedBehaviorRandomization
>From bf599428733ad304e578c780a093c3e3cd43b3c6 Mon Sep 17 00:00:00 2001
From: Hui Xie <hui.xie1990 at gmail.com>
Date: Sat, 28 Feb 2026 15:57:23 +0000
Subject: [PATCH 06/65] refactor to hide implementation into dylib
---
libcxx/CMakeLists.txt | 6 +
libcxx/docs/DesignDocs/RCU.rst | 11 +-
libcxx/include/CMakeLists.txt | 1 -
libcxx/include/__rcu/rcu_domain.h | 251 ++---------------
libcxx/include/__rcu/rcu_obj_base.h | 26 +-
libcxx/include/module.modulemap.in | 1 -
libcxx/src/CMakeLists.txt | 7 +
.../experimental/include/rcu}/rcu_list.h | 9 +-
libcxx/src/experimental/rcu.cpp | 257 ++++++++++++++++++
9 files changed, 317 insertions(+), 252 deletions(-)
rename libcxx/{include/__rcu => src/experimental/include/rcu}/rcu_list.h (90%)
create mode 100644 libcxx/src/experimental/rcu.cpp
diff --git a/libcxx/CMakeLists.txt b/libcxx/CMakeLists.txt
index d163f0f579c39..442f8af669cab 100644
--- a/libcxx/CMakeLists.txt
+++ b/libcxx/CMakeLists.txt
@@ -31,6 +31,12 @@ list(INSERT CMAKE_MODULE_PATH 0
set(CMAKE_FOLDER "libc++")
+# Unset the C++ Standard: other projects sometimes override this global property, which
+# is incompatible with libc++ (since we set our own standard via other means).
+# TODO: Figure out which other part of the build is setting this and resolve the issue
+# at the root.
+unset(CMAKE_CXX_STANDARD)
+
set(LIBCXX_SOURCE_DIR ${CMAKE_CURRENT_SOURCE_DIR})
set(LIBCXX_BINARY_DIR ${CMAKE_CURRENT_BINARY_DIR})
diff --git a/libcxx/docs/DesignDocs/RCU.rst b/libcxx/docs/DesignDocs/RCU.rst
index c87ae8cde5677..ee1f2fe6fb993 100644
--- a/libcxx/docs/DesignDocs/RCU.rst
+++ b/libcxx/docs/DesignDocs/RCU.rst
@@ -169,8 +169,10 @@ There are few places we can run the deleters:
Folly currently takes the inline approach and runs the deleters at all three places mentioned above.
In libc++'s design, we would like to start with the inline approach and only run the deleters inside `run_synchronize` and
-`run_barrier` for simplicity. We can consider running the deleters inside `run_retire` in the future if we find it is necessary
-to reduce the latency of reclaiming the retired objects.
+`run_barrier` for simplicity. Whether or not running the deleters inside `run_retire` is debatable. On the one hand, running
+them will make `run_retire` take more time to return. But not running them will make the retired objects stay in the queue
+for a longer time. In an extreme case, if the user never calls `run_synchronize` or `run_barrier` after calling `run_retire`,
+the retired objects will never be reclaimed (without a background thread to drain the queue).
Almost all APIs are `noexcept` . Is it designed to avoid memory allocation and avoid using `mutex` ?
-----------------------------------------------------------------------------------------------------
@@ -309,4 +311,9 @@ There are 4 flavors of `rcu` implementations in liburcu (see doc `here <https://
libstdc++
---------
+According to Thomas Rodgers (libstdc++'s RCU implementer), their plan is to embed the subset of `liburcu` that implements the `memb`` and `mb` RCU flavors, with `memb` being the preferred choice,
+based on the presence of `SYS_membarrier` . Because `liburcu` is license compatible with libstdc++ they can directly embed the relevant source from `liburcu`
+
+libc++ Design
+=============
diff --git a/libcxx/include/CMakeLists.txt b/libcxx/include/CMakeLists.txt
index a3e67d0be70dd..b0282cc6198c0 100644
--- a/libcxx/include/CMakeLists.txt
+++ b/libcxx/include/CMakeLists.txt
@@ -804,7 +804,6 @@ set(files
__ranges/zip_transform_view.h
__ranges/zip_view.h
__rcu/rcu_domain.h
- __rcu/rcu_list.h
__rcu/rcu_obj_base.h
__split_buffer
__std_mbstate_t.h
diff --git a/libcxx/include/__rcu/rcu_domain.h b/libcxx/include/__rcu/rcu_domain.h
index 3ac3597280c25..71aed729afa93 100644
--- a/libcxx/include/__rcu/rcu_domain.h
+++ b/libcxx/include/__rcu/rcu_domain.h
@@ -11,19 +11,8 @@
#define _LIBCPP___RCU_RCU_DOMAIN_H
#include <__config>
-
-#include <__atomic/atomic.h>
-#include <__rcu/rcu_list.h>
-
-// todo replace with internal headers
-#include <atomic>
-#include <cstdint>
-#include <map>
-#include <mutex>
-#include <vector>
-
-// todo debug
-#include <cstdio>
+#include <__functional/function.h>
+#include <__memory/unique_ptr.h>
#if !defined(_LIBCPP_HAS_NO_PRAGMA_SYSTEM_HEADER)
# pragma GCC system_header
@@ -33,246 +22,52 @@ _LIBCPP_BEGIN_NAMESPACE_STD
#if _LIBCPP_STD_VER >= 26 && _LIBCPP_HAS_THREADS && _LIBCPP_HAS_EXPERIMENTAL_RCU
-template <class _Tp>
-class __thread_local_owner {
- // todo put globals in experimental dylib
- inline static thread_local map<const __thread_local_owner*, _Tp> __thread_local_instances;
-
- // Keep track of all thread-local instances owned by this owner.
- // Only emplaced the first time a thread is trying to access its thread-local instance.
- vector<atomic_ref<_Tp>> __owned_instances_;
- mutex __mtx_;
-
- void __register(_Tp& __obj) {
- lock_guard<std::mutex> __lg(__mtx_);
- __owned_instances_.emplace_back(__obj);
- }
-
- // todo: deregister on thread exit?
+struct __rcu_node {
+ function<void()> __callback_{};
+ __rcu_node* __next_ = nullptr;
-public:
- __thread_local_owner() = default;
- __thread_local_owner(__thread_local_owner&&) = delete;
-
- atomic_ref<_Tp> __get_current_thread_instance() {
- auto __it = __thread_local_instances.find(this);
- if (__it == __thread_local_instances.end()) {
- auto [new_it, _] = __thread_local_instances.try_emplace(this, _Tp());
- auto& __obj = new_it->second;
- __register(__obj);
- return atomic_ref(__obj);
- }
- return atomic_ref(__it->second);
- }
-
- template <class _Func>
- void __for_each_owned_instances(_Func&& __f) {
- unique_lock<std::mutex> __lock(__mtx_);
- for (auto __instance : __owned_instances_) {
- __f(__instance);
- }
- }
+ _LIBCPP_HIDE_FROM_ABI __rcu_node() {}
+ _LIBCPP_HIDE_FROM_ABI ~__rcu_node() {}
};
-// Adopted the 2-phase implementation in the section
-// "3) General-Purpose RCU" of the paper
-// http://www.rdrop.com/users/paulmck/RCU/urcu-supp-accepted.2011.08.30a.pdf
-
-struct __reader_states {
- // bit 15 is the grace period phase 0 or 1
- // bits 0-14 is the reader nest level
- //
- // a thread can have nested reader locks, such as
- // domain.lock(); // nest level = 1
- // domain.lock(); // nest level = 2
- // ...
- // domain.unlock(); // nest level = 1
- // domain.unlock(); // nest level = 0
-
- static constexpr uint16_t __grace_period_phase_mask = 0b1000'0000'0000'0000;
- static constexpr uint16_t __reader_nest_level_mask = 0b0111'1111'1111'1111;
-
- using __state_type = uint16_t;
-
- __thread_local_owner<__state_type> __per_thread_states_;
-
- static uint16_t __get_grace_period_phase(__state_type __state) { return __state & __grace_period_phase_mask; }
-
- static uint16_t __get_reader_nest_level(__state_type __state) { return __state & __reader_nest_level_mask; }
-
- static bool __is_quiescent_state(__state_type __state) { return __get_reader_nest_level(__state) == 0; }
-};
-
-class rcu_domain {
- // todo optimize the layout
-
- __reader_states __reader_states_;
-
- // only the highest bit is used for the phase.
- std::atomic<__reader_states::__state_type> __global_reader_phase_{};
-
- // only one thread is allowed to update concurrently
- std::mutex __grace_period_mutex_; // todo this is not noexcept
-
- std::atomic<bool> __grace_period_waiting_flag_ = false;
+class _LIBCPP_EXPORTED_FROM_ABI rcu_domain {
+ class __impl;
+ unique_ptr<__impl> __pimpl_;
- // todo: maybe use a lock-free queue
- std::mutex __retire_queue_mutex_; // todo this is not noexcept
- __rcu_singly_list_view __retired_callback_queue_;
-
- // these two queues do not need extra synchronization
- // as they are always processed under the grace period mutex
- __rcu_singly_list_view __callbacks_phase_1_;
- __rcu_singly_list_view __callbacks_phase_2_;
-
- rcu_domain() = default;
-
- friend struct __rcu_domain_access;
template <class, class>
friend class rcu_obj_base;
- // todo put globals in dylib
- static rcu_domain& __rcu_default_domain() noexcept {
- static rcu_domain __default_domain;
- return __default_domain;
- }
-
- template <class Callback>
- void __retire_callback(Callback&& __cb) noexcept {
- auto* __node = new __rcu_node();
- __node->__callback_ = std::forward<Callback>(__cb);
- std::unique_lock __lk(__retire_queue_mutex_);
- __retired_callback_queue_.__push_back(__node);
- }
-
- void __synchronize() noexcept {
- __cxx_atomic_thread_fence(memory_order_seq_cst);
- std::unique_lock __lk(__grace_period_mutex_);
- //std::printf("rcu_domain::__synchronize() going through phase 1\n");
- auto __ready_callbacks = __update_phase_and_wait();
+ friend rcu_domain& rcu_default_domain() noexcept;
+ friend void rcu_synchronize(rcu_domain&) noexcept;
- // Invoke the ready callbacks outside of the grace period mutex
- __lk.unlock();
- __ready_callbacks.__for_each([](auto* __node) { __node->__callback_(); });
- __lk.lock();
+ static rcu_domain& __rcu_default_domain() noexcept;
- __barrier();
- //std::printf("rcu_domain::__synchronize() going through phase 2\n");
- __ready_callbacks = __update_phase_and_wait();
+ rcu_domain();
- // Invoke the ready callbacks outside of the grace period mutex
- __lk.unlock();
- __ready_callbacks.__for_each([](auto* __node) { __node->__callback_(); });
- __cxx_atomic_thread_fence(memory_order_seq_cst);
- }
-
- __rcu_singly_list_view __update_phase_and_wait() noexcept {
- std::unique_lock __retire_lk(__retire_queue_mutex_);
- __callbacks_phase_1_.__splice_back(__retired_callback_queue_);
- __retire_lk.unlock();
-
- // Flip the global phase
- auto __old_phase =
- __global_reader_phase_.fetch_xor(__reader_states::__grace_period_phase_mask, std::memory_order_relaxed);
- auto __new_phase = __old_phase ^ __reader_states::__grace_period_phase_mask;
- //std::printf("rcu_domain::__update_phase_and_wait() new phase: 0x%04x\n", __new_phase);
-
- __barrier();
- // Wait for all threads to quiesce in the old phase
- while (__any_reader_in_ongoing_grace_period(__new_phase)) {
- __grace_period_waiting_flag_.store(true, std::memory_order_relaxed);
- __grace_period_waiting_flag_.wait(true, std::memory_order_relaxed);
- }
- __grace_period_waiting_flag_.store(false, std::memory_order_relaxed);
-
- __rcu_singly_list_view __ready_callbacks;
- __ready_callbacks.__splice_back(__callbacks_phase_2_);
- __callbacks_phase_2_.__splice_back(__callbacks_phase_1_);
- return __ready_callbacks;
- }
-
- bool __any_reader_in_ongoing_grace_period(__reader_states::__state_type __global_phase) noexcept {
- bool __any_ongoing = false;
- __reader_states_.__per_thread_states_.__for_each_owned_instances(
- [this, __global_phase, &__any_ongoing](atomic_ref<__reader_states::__state_type> __state) {
- if (__is_grace_period_ongoing(__state.load(memory_order_relaxed), __global_phase)) {
- __any_ongoing = true;
- }
- });
- return __any_ongoing;
- }
-
- bool __is_grace_period_ongoing(__reader_states::__state_type __thread_state,
- __reader_states::__state_type __global_phase) const noexcept {
- // https://lwn.net/Articles/323929/
- // The phase is flipped at the beginning of a grace period.
- // Any readers that started before the flip will have the old phase
- // and we consider them as ongoing that we need to wait for before we can close the grace period.
- return !__reader_states::__is_quiescent_state(__thread_state) &&
- __reader_states::__get_grace_period_phase(__thread_state) != __global_phase;
- }
-
- void __barrier() noexcept { asm volatile("" : : : "memory"); }
+ void __retire(__rcu_node*) noexcept;
public:
rcu_domain(const rcu_domain&) = delete;
rcu_domain& operator=(const rcu_domain&) = delete;
+ ~rcu_domain();
- void printAllReaderStatesInHex() {
- __reader_states_.__per_thread_states_.__for_each_owned_instances([](auto __state_ref) {
- std::printf("Reader state: 0x%04x\n", __state_ref.load());
- });
- }
-
- void lock() {
- auto __current_thread_state_ref = __reader_states_.__per_thread_states_.__get_current_thread_instance();
+ void printAllReaderStatesInHex();
- if ((__reader_states::__is_quiescent_state(__current_thread_state_ref.load(memory_order_relaxed)))) {
- // Entering a read-side critical section from a quiescent state.
- __current_thread_state_ref.store(
- __reader_states::__state_type(__global_reader_phase_.load(std::memory_order_relaxed) | uint16_t(1)),
- memory_order_relaxed);
- __cxx_atomic_thread_fence(memory_order_seq_cst);
- } else {
- // Already in read-side critical section, just increment the nest level.
- __current_thread_state_ref.fetch_add(1, memory_order_relaxed);
- }
- }
+ void lock() noexcept;
- bool try_lock() {
+ bool try_lock() noexcept {
lock();
return true;
}
- void unlock() {
- auto __current_thread_state_ref = __reader_states_.__per_thread_states_.__get_current_thread_instance();
- __cxx_atomic_thread_fence(memory_order_seq_cst);
- // Decrement the nest level.
- auto __old_state = __current_thread_state_ref.fetch_sub(1, memory_order_relaxed);
-
- if (__reader_states::__get_reader_nest_level(__old_state) == 1 &&
- __grace_period_waiting_flag_.load(memory_order_relaxed)) {
- // Transitioning to quiescent state, wake up waiters.
- __grace_period_waiting_flag_.store(false, std::memory_order_relaxed);
- __grace_period_waiting_flag_.notify_all();
- }
- }
-};
-
-struct __rcu_domain_access {
- static rcu_domain& __rcu_default_domain() noexcept { return rcu_domain::__rcu_default_domain(); }
- static void __rcu_synchronize(rcu_domain& __dom) noexcept { __dom.__synchronize(); }
+ void unlock() noexcept;
};
-// todo put it in the experimental dylib
-inline rcu_domain& rcu_default_domain() noexcept { return __rcu_domain_access::__rcu_default_domain(); }
+_LIBCPP_EXPORTED_FROM_ABI rcu_domain& rcu_default_domain() noexcept;
-// todo put it in the experimental dylib
-inline void rcu_synchronize(rcu_domain& __dom = rcu_default_domain()) noexcept {
- __rcu_domain_access::__rcu_synchronize(__dom);
-}
+_LIBCPP_EXPORTED_FROM_ABI void rcu_synchronize(rcu_domain& __dom = rcu_default_domain()) noexcept;
-void rcu_barrier(rcu_domain& dom = rcu_default_domain()) noexcept;
+_LIBCPP_EXPORTED_FROM_ABI void rcu_barrier(rcu_domain& __dom = rcu_default_domain()) noexcept;
template <class T, class D = default_delete<T>>
void rcu_retire(T* p, D d = D(), rcu_domain& dom = rcu_default_domain());
diff --git a/libcxx/include/__rcu/rcu_obj_base.h b/libcxx/include/__rcu/rcu_obj_base.h
index 3d1de353613a5..4dbfe1fd07720 100644
--- a/libcxx/include/__rcu/rcu_obj_base.h
+++ b/libcxx/include/__rcu/rcu_obj_base.h
@@ -11,9 +11,10 @@
#define _LIBCPP___RCU_RCU_OBJ_BASE_H
#include <__config>
+#include <__functional/function.h>
#include <__memory/unique_ptr.h> // for default_delete
#include <__rcu/rcu_domain.h>
-#include <__rcu/rcu_list.h>
+#include <__utility/move.h>
#if !defined(_LIBCPP_HAS_NO_PRAGMA_SYSTEM_HEADER)
# pragma GCC system_header
@@ -23,23 +24,22 @@ _LIBCPP_BEGIN_NAMESPACE_STD
#if _LIBCPP_STD_VER >= 26 && _LIBCPP_HAS_THREADS && _LIBCPP_HAS_EXPERIMENTAL_RCU
-template <class T, class D = default_delete<T>>
+template <class _Tp, class _Dp = default_delete<_Tp>>
class rcu_obj_base : private __rcu_node {
public:
- void retire(D d = D(), rcu_domain& dom = rcu_default_domain()) noexcept {
- auto ptr = static_cast<T*>(this);
- dom.__retire_callback(
- [ptr, d = std::move(d)]() mutable { d(ptr); });
+ _LIBCPP_HIDE_FROM_ABI void retire(_Dp __deleter = _Dp(), rcu_domain& __dom = rcu_default_domain()) noexcept {
+ auto __ptr = static_cast<_Tp*>(this);
+ __callback_ = [__ptr, __deleter = std::move(__deleter)]() mutable { __deleter(__ptr); };
+ __dom.__retire(this);
}
protected:
- rcu_obj_base() = default;
- rcu_obj_base(const rcu_obj_base&) = default;
- rcu_obj_base(rcu_obj_base&&) = default;
- rcu_obj_base& operator=(const rcu_obj_base&) = default;
- rcu_obj_base& operator=(rcu_obj_base&&) = default;
- ~rcu_obj_base() = default;
-
+ _LIBCPP_HIDE_FROM_ABI rcu_obj_base() = default;
+ _LIBCPP_HIDE_FROM_ABI rcu_obj_base(const rcu_obj_base&) = default;
+ _LIBCPP_HIDE_FROM_ABI rcu_obj_base(rcu_obj_base&&) = default;
+ _LIBCPP_HIDE_FROM_ABI rcu_obj_base& operator=(const rcu_obj_base&) = default;
+ _LIBCPP_HIDE_FROM_ABI rcu_obj_base& operator=(rcu_obj_base&&) = default;
+ _LIBCPP_HIDE_FROM_ABI ~rcu_obj_base() = default;
};
#endif // _LIBCPP_STD_VER >= 26 && _LIBCPP_HAS_THREADS && _LIBCPP_HAS_EXPERIMENTAL_RCU
diff --git a/libcxx/include/module.modulemap.in b/libcxx/include/module.modulemap.in
index 230994c833553..7d3c90a1aa2e8 100644
--- a/libcxx/include/module.modulemap.in
+++ b/libcxx/include/module.modulemap.in
@@ -2065,7 +2065,6 @@ module std {
module rcu {
module rcu_domain { header "__rcu/rcu_domain.h" }
- module rcu_list { header "__rcu/rcu_list.h" }
module rcu_obj_base { header "__rcu/rcu_obj_base.h" }
header "rcu"
export *
diff --git a/libcxx/src/CMakeLists.txt b/libcxx/src/CMakeLists.txt
index e029f9f29d5ea..51b9c32cc4eef 100644
--- a/libcxx/src/CMakeLists.txt
+++ b/libcxx/src/CMakeLists.txt
@@ -325,6 +325,13 @@ if (LIBCXX_ENABLE_LOCALIZATION AND LIBCXX_ENABLE_FILESYSTEM AND LIBCXX_ENABLE_TI
)
endif()
+if (LIBCXX_ENABLE_THREADS)
+ list(APPEND LIBCXX_EXPERIMENTAL_SOURCES
+ experimental/include/rcu/rcu_list.h
+ experimental/rcu.cpp
+ )
+endif()
+
add_library(cxx_experimental STATIC ${LIBCXX_EXPERIMENTAL_SOURCES})
target_link_libraries(cxx_experimental PUBLIC cxx-headers)
if (LIBCXX_ENABLE_SHARED AND TARGET cxx_shared)
diff --git a/libcxx/include/__rcu/rcu_list.h b/libcxx/src/experimental/include/rcu/rcu_list.h
similarity index 90%
rename from libcxx/include/__rcu/rcu_list.h
rename to libcxx/src/experimental/include/rcu/rcu_list.h
index 2547d7223a7cc..199c3c67e4275 100644
--- a/libcxx/include/__rcu/rcu_list.h
+++ b/libcxx/src/experimental/include/rcu/rcu_list.h
@@ -22,18 +22,13 @@ _LIBCPP_BEGIN_NAMESPACE_STD
#if _LIBCPP_STD_VER >= 26 && _LIBCPP_HAS_THREADS && _LIBCPP_HAS_EXPERIMENTAL_RCU
-struct __rcu_node {
- function<void()> __callback_{};
- __rcu_node* __next_ = nullptr;
-};
-
-class __rcu_singly_list_view {
+class rcu_singly_list_view {
private:
__rcu_node* __head_ = nullptr;
__rcu_node* __tail_ = nullptr;
public:
- void __splice_back(__rcu_singly_list_view& __other) noexcept {
+ void __splice_back(rcu_singly_list_view& __other) noexcept {
if (__other.__head_ == nullptr) {
return;
}
diff --git a/libcxx/src/experimental/rcu.cpp b/libcxx/src/experimental/rcu.cpp
new file mode 100644
index 0000000000000..f3b3a0990a908
--- /dev/null
+++ b/libcxx/src/experimental/rcu.cpp
@@ -0,0 +1,257 @@
+//===----------------------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#include <atomic>
+#include <memory>
+#include <mutex>
+#include <optional>
+#include <rcu>
+#include <vector>
+
+#include "__rcu/rcu_domain.h"
+#include "include/rcu/rcu_list.h"
+
+_LIBCPP_BEGIN_NAMESPACE_STD
+
+namespace {
+
+template <class Tp>
+class thread_local_container {
+ struct thread_entry {
+ Tp instance_;
+
+ thread_entry() : instance_() { register_instance(instance_); }
+ thread_entry(const thread_entry&) = delete;
+ thread_entry(thread_entry&&) = delete;
+
+ ~thread_entry() { deregister_instance(instance_); }
+ };
+
+ inline static thread_local optional<thread_entry> thread_entry_{};
+
+ // Keep track of all thread-local instances
+ // Only emplaced the first time a thread is trying to access its thread-local instance.
+ inline static vector<Tp*> instances_;
+ inline static mutex mtx_;
+
+ static void register_instance(Tp& obj) {
+ lock_guard<std::mutex> lg(mtx_);
+ instances_.emplace_back(&obj);
+ }
+
+ static void deregister_instance(Tp& obj) {
+ lock_guard<std::mutex> lg(mtx_);
+ instances_.erase(remove_if(instances_.begin(), instances_.end(), [&obj](Tp* instance) { return instance == &obj; }),
+ instances_.end());
+ }
+
+public:
+ thread_local_container() = delete;
+ thread_local_container(thread_local_container&&) = delete;
+
+ static atomic_ref<Tp> get_current_thread_instance() {
+ if (!thread_entry_.has_value()) {
+ auto& entry = thread_entry_.emplace();
+ return atomic_ref(entry.instance_);
+ }
+ return atomic_ref(thread_entry_->instance_);
+ }
+
+ template <class Func>
+ static void for_each(Func&& f) {
+ unique_lock<std::mutex> lock(mtx_);
+ for (auto instance : instances_) {
+ f(atomic_ref(*instance));
+ }
+ }
+};
+
+// Adopted the 2-phase implementation in the section
+// "3) General-Purpose RCU" of the paper
+// http://www.rdrop.com/users/paulmck/RCU/urcu-supp-accepted.2011.08.30a.pdf
+
+struct reader_states {
+ // bit 15 is the grace period phase 0 or 1
+ // bits 0-14 is the reader nest level
+ //
+ // a thread can have nested reader locks, such as
+ // domain.lock(); // nest level = 1
+ // domain.lock(); // nest level = 2
+ // ...
+ // domain.unlock(); // nest level = 1
+ // domain.unlock(); // nest level = 0
+
+ static constexpr uint16_t grace_period_phase_mask = 0b1000'0000'0000'0000;
+ static constexpr uint16_t reader_nest_level_mask = 0b0111'1111'1111'1111;
+
+ using state_type = uint16_t;
+
+ static uint16_t get_grace_period_phase(state_type state) { return state & grace_period_phase_mask; }
+
+ static uint16_t get_reader_nest_level(state_type state) { return state & reader_nest_level_mask; }
+
+ static bool is_quiescent_state(state_type state) { return get_reader_nest_level(state) == 0; }
+
+ static state_type make_state(state_type grace_period_phase, state_type reader_nest_level) {
+ return (grace_period_phase & grace_period_phase_mask) | (reader_nest_level & reader_nest_level_mask);
+ }
+};
+} // namespace
+
+class rcu_domain::__impl {
+ using per_thread_states = thread_local_container<reader_states::state_type>;
+
+ // only the highest bit is used for the phase.
+ std::atomic<reader_states::state_type> global_reader_phase_{};
+
+ // only one writer thread is allowed to call synchronize concurrently
+ std::mutex grace_period_mutex_; // todo this is not noexcept
+
+ // flag used for waking up writer threads waiting for all reader threads' quiescent state
+ std::atomic<bool> grace_period_waiting_flag_ = false;
+
+ // todo: maybe use a lock-free queue
+ std::mutex retire_queue_mutex_; // todo this is not noexcept
+ rcu_singly_list_view __retired_callback_queue_;
+
+ // these two queues do not need extra synchronization
+ // as they are always processed under the grace period mutex
+ rcu_singly_list_view callbacks_phase_1_;
+ rcu_singly_list_view callbacks_phase_2_;
+
+ friend class rcu_domain;
+
+ rcu_singly_list_view update_phase_and_wait() noexcept {
+ std::unique_lock retire_lk(retire_queue_mutex_);
+ callbacks_phase_1_.__splice_back(__retired_callback_queue_);
+ retire_lk.unlock();
+
+ // Flip the global phase
+ auto old_phase = global_reader_phase_.fetch_xor(reader_states::grace_period_phase_mask, std::memory_order_relaxed);
+ auto __new_phase = old_phase ^ reader_states::grace_period_phase_mask;
+ // std::printf("rcu_domain::update_phase_and_wait() new phase: 0x%04x\n", __new_phase);
+
+ __barrier();
+ // Wait for all threads to quiesce in the old phase
+ while (any_reader_in_ongoing_grace_period(__new_phase)) {
+ grace_period_waiting_flag_.store(true, std::memory_order_relaxed);
+ grace_period_waiting_flag_.wait(true, std::memory_order_relaxed);
+ }
+ grace_period_waiting_flag_.store(false, std::memory_order_relaxed);
+
+ rcu_singly_list_view ready_callbacks;
+ ready_callbacks.__splice_back(callbacks_phase_2_);
+ callbacks_phase_2_.__splice_back(callbacks_phase_1_);
+ return ready_callbacks;
+ }
+
+ bool any_reader_in_ongoing_grace_period(reader_states::state_type __global_phase) noexcept {
+ bool any_ongoing = false;
+ per_thread_states::for_each([this, __global_phase, &any_ongoing](atomic_ref<reader_states::state_type> __state) {
+ if (is_grace_period_ongoing(__state.load(memory_order_relaxed), __global_phase)) {
+ any_ongoing = true;
+ }
+ });
+ return any_ongoing;
+ }
+
+ bool is_grace_period_ongoing(reader_states::state_type thread_state,
+ reader_states::state_type global_phase) const noexcept {
+ // https://lwn.net/Articles/323929/
+ // The phase is flipped at the beginning of a grace period.
+ // Any readers that started before the flip will have the old phase
+ // and we consider them as ongoing that we need to wait for before we can close the grace period.
+ return !reader_states::is_quiescent_state(thread_state) &&
+ reader_states::get_grace_period_phase(thread_state) != global_phase;
+ }
+
+ void __barrier() noexcept { asm volatile("" : : : "memory"); }
+
+ void printAllReaderStatesInHex() {
+ per_thread_states::for_each([](auto __state_ref) { std::printf("Reader state: 0x%04x\n", __state_ref.load()); });
+ }
+
+public:
+ void lock() noexcept {
+ auto current_thread_state_ref = per_thread_states::get_current_thread_instance();
+
+ if ((reader_states::is_quiescent_state(current_thread_state_ref.load(memory_order_relaxed)))) {
+ // Entering a read-side critical section from a quiescent state.
+ current_thread_state_ref.store(
+ reader_states::make_state(global_reader_phase_.load(std::memory_order_relaxed), 1), memory_order_relaxed);
+ __cxx_atomic_thread_fence(memory_order_seq_cst);
+ } else {
+ // Already in read-side critical section, just increment the nest level.
+ current_thread_state_ref.fetch_add(1, memory_order_relaxed);
+ }
+ }
+
+ void unlock() noexcept {
+ auto current_thread_state_ref = per_thread_states::get_current_thread_instance();
+ __cxx_atomic_thread_fence(memory_order_seq_cst);
+ // Decrement the nest level.
+ auto old_state = current_thread_state_ref.fetch_sub(1, memory_order_relaxed);
+
+ if (reader_states::get_reader_nest_level(old_state) == 1 && grace_period_waiting_flag_.load(memory_order_relaxed)) {
+ // Transitioning to quiescent state, wake up waiters.
+ grace_period_waiting_flag_.store(false, std::memory_order_relaxed);
+ grace_period_waiting_flag_.notify_all();
+ }
+ }
+
+ void retire(__rcu_node* node) noexcept {
+ lock_guard<std::mutex> lk(retire_queue_mutex_);
+ __retired_callback_queue_.__push_back(node);
+ }
+
+ void synchronize() noexcept {
+ __cxx_atomic_thread_fence(memory_order_seq_cst);
+ std::unique_lock lk(grace_period_mutex_);
+
+ // std::printf("rcu_domain::__synchronize() going through phase 1\n");
+ auto ready_callbacks = update_phase_and_wait();
+
+ // Invoke the ready callbacks outside of the grace period mutex
+ lk.unlock();
+ ready_callbacks.__for_each([](auto* node) { node->__callback_(); });
+ lk.lock();
+
+ __barrier();
+ // std::printf("rcu_domain::__synchronize() going through phase 2\n");
+ ready_callbacks = update_phase_and_wait();
+
+ // Invoke the ready callbacks outside of the grace period mutex
+ lk.unlock();
+ ready_callbacks.__for_each([](auto* node) { node->__callback_(); });
+ __cxx_atomic_thread_fence(memory_order_seq_cst);
+ }
+};
+
+rcu_domain& rcu_domain::__rcu_default_domain() noexcept {
+ static rcu_domain default_domain;
+ return default_domain;
+}
+
+rcu_domain::rcu_domain() : __pimpl_(std::make_unique<__impl>()) {}
+rcu_domain::~rcu_domain() = default;
+
+void rcu_domain::printAllReaderStatesInHex() { __pimpl_->printAllReaderStatesInHex(); }
+
+void rcu_domain::lock() noexcept { __pimpl_->lock(); }
+
+void rcu_domain::unlock() noexcept { __pimpl_->unlock(); }
+
+void rcu_domain::__retire(__rcu_node* node) noexcept { __pimpl_->retire(node); }
+
+rcu_domain& rcu_default_domain() noexcept { return rcu_domain::__rcu_default_domain(); }
+
+void rcu_synchronize(rcu_domain& dom) noexcept { dom.__pimpl_->synchronize(); }
+
+void rcu_barrier(rcu_domain& dom) noexcept { rcu_synchronize(dom); }
+
+_LIBCPP_END_NAMESPACE_STD
>From dc1efe094765d73a2e9b95f7635a3d06b0a7cd4e Mon Sep 17 00:00:00 2001
From: Hui Xie <hui.xie1990 at gmail.com>
Date: Sat, 28 Feb 2026 16:38:30 +0000
Subject: [PATCH 07/65] move impl to internal linkage
---
libcxx/include/__rcu/rcu_domain.h | 16 +++++++++++++---
libcxx/src/experimental/rcu.cpp | 17 ++++++++---------
.../saferecl/saferecl.rcu/general.pass.cpp | 4 +++-
3 files changed, 24 insertions(+), 13 deletions(-)
diff --git a/libcxx/include/__rcu/rcu_domain.h b/libcxx/include/__rcu/rcu_domain.h
index 71aed729afa93..a055655e36b8c 100644
--- a/libcxx/include/__rcu/rcu_domain.h
+++ b/libcxx/include/__rcu/rcu_domain.h
@@ -13,6 +13,7 @@
#include <__config>
#include <__functional/function.h>
#include <__memory/unique_ptr.h>
+#include <__utility/move.h>
#if !defined(_LIBCPP_HAS_NO_PRAGMA_SYSTEM_HEADER)
# pragma GCC system_header
@@ -42,6 +43,13 @@ class _LIBCPP_EXPORTED_FROM_ABI rcu_domain {
static rcu_domain& __rcu_default_domain() noexcept;
+ template <class _Tp, class _Dp = default_delete<_Tp>>
+ _LIBCPP_HIDE_FROM_ABI void __rcu_retire_hidden_friend(_Tp* __tp, _Dp __deleter, rcu_domain& __dom) {
+ auto* __node = new __rcu_node();
+ __node->__callback_ = [__tp, __deleter = std::move(__deleter)]() mutable { __deleter(__tp); };
+ __dom.__retire(__node);
+ }
+
rcu_domain();
void __retire(__rcu_node*) noexcept;
@@ -55,7 +63,7 @@ class _LIBCPP_EXPORTED_FROM_ABI rcu_domain {
void lock() noexcept;
- bool try_lock() noexcept {
+ _LIBCPP_HIDE_FROM_ABI bool try_lock() noexcept {
lock();
return true;
}
@@ -69,8 +77,10 @@ _LIBCPP_EXPORTED_FROM_ABI void rcu_synchronize(rcu_domain& __dom = rcu_default_d
_LIBCPP_EXPORTED_FROM_ABI void rcu_barrier(rcu_domain& __dom = rcu_default_domain()) noexcept;
-template <class T, class D = default_delete<T>>
-void rcu_retire(T* p, D d = D(), rcu_domain& dom = rcu_default_domain());
+template <class _Tp, class _Dp = default_delete<_Tp>>
+_LIBCPP_HIDE_FROM_ABI void rcu_retire(_Tp* __tp, _Dp __deleter = _Dp(), rcu_domain& __dom = rcu_default_domain()) {
+ __rcu_retire_hidden_friend(__tp, std::move(__deleter), __dom);
+}
#endif // _LIBCPP_STD_VER >= 26 && _LIBCPP_HAS_THREADS && _LIBCPP_HAS_EXPERIMENTAL_RCU
diff --git a/libcxx/src/experimental/rcu.cpp b/libcxx/src/experimental/rcu.cpp
index f3b3a0990a908..6bf69b74c9b00 100644
--- a/libcxx/src/experimental/rcu.cpp
+++ b/libcxx/src/experimental/rcu.cpp
@@ -13,7 +13,6 @@
#include <rcu>
#include <vector>
-#include "__rcu/rcu_domain.h"
#include "include/rcu/rcu_list.h"
_LIBCPP_BEGIN_NAMESPACE_STD
@@ -101,9 +100,8 @@ struct reader_states {
return (grace_period_phase & grace_period_phase_mask) | (reader_nest_level & reader_nest_level_mask);
}
};
-} // namespace
-class rcu_domain::__impl {
+class rcu_domain_impl {
using per_thread_states = thread_local_container<reader_states::state_type>;
// only the highest bit is used for the phase.
@@ -172,10 +170,6 @@ class rcu_domain::__impl {
void __barrier() noexcept { asm volatile("" : : : "memory"); }
- void printAllReaderStatesInHex() {
- per_thread_states::for_each([](auto __state_ref) { std::printf("Reader state: 0x%04x\n", __state_ref.load()); });
- }
-
public:
void lock() noexcept {
auto current_thread_state_ref = per_thread_states::get_current_thread_instance();
@@ -213,7 +207,6 @@ class rcu_domain::__impl {
__cxx_atomic_thread_fence(memory_order_seq_cst);
std::unique_lock lk(grace_period_mutex_);
- // std::printf("rcu_domain::__synchronize() going through phase 1\n");
auto ready_callbacks = update_phase_and_wait();
// Invoke the ready callbacks outside of the grace period mutex
@@ -222,7 +215,6 @@ class rcu_domain::__impl {
lk.lock();
__barrier();
- // std::printf("rcu_domain::__synchronize() going through phase 2\n");
ready_callbacks = update_phase_and_wait();
// Invoke the ready callbacks outside of the grace period mutex
@@ -230,7 +222,14 @@ class rcu_domain::__impl {
ready_callbacks.__for_each([](auto* node) { node->__callback_(); });
__cxx_atomic_thread_fence(memory_order_seq_cst);
}
+
+ void printAllReaderStatesInHex() {
+ per_thread_states::for_each([](auto __state_ref) { std::printf("Reader state: 0x%04x\n", __state_ref.load()); });
+ }
};
+} // namespace
+
+class rcu_domain::__impl : public rcu_domain_impl {};
rcu_domain& rcu_domain::__rcu_default_domain() noexcept {
static rcu_domain default_domain;
diff --git a/libcxx/test/std/thread/saferecl/saferecl.rcu/general.pass.cpp b/libcxx/test/std/thread/saferecl/saferecl.rcu/general.pass.cpp
index 88412976f7a62..b94643db87c4b 100644
--- a/libcxx/test/std/thread/saferecl/saferecl.rcu/general.pass.cpp
+++ b/libcxx/test/std/thread/saferecl/saferecl.rcu/general.pass.cpp
@@ -26,7 +26,7 @@ void log(auto start, std::string_view msg) {
std::println(std::cout, "[{:%H:%M:%S}] {}", now - start, msg);
}
-int loop_num = 1;
+int loop_num = 10;
int main(int, char**) {
auto start = std::chrono::system_clock::now();
@@ -101,6 +101,8 @@ int main(int, char**) {
std::this_thread::sleep_for(std::chrono::milliseconds(10));
for (int i = 0; i < loop_num+5; ++i) {
+ log(start, "t0: printing all reader states before synchronize");
+ std::rcu_default_domain().printAllReaderStatesInHex();
log(start, "t0: calling rcu_synchronize" + std::to_string(i));
std::rcu_synchronize();
log(start, "t0: rcu_synchronize returned" + std::to_string(i));
>From d7513f36935ba5a772e22cd129f76a76ddd9ded0 Mon Sep 17 00:00:00 2001
From: Hui Xie <hui.xie1990 at gmail.com>
Date: Sat, 28 Feb 2026 18:58:12 +0000
Subject: [PATCH 08/65] more docs
---
libcxx/docs/DesignDocs/RCU.rst | 95 ++++++++++++++++++++++++++---
libcxx/include/__rcu/rcu_obj_base.h | 6 +-
2 files changed, 93 insertions(+), 8 deletions(-)
diff --git a/libcxx/docs/DesignDocs/RCU.rst b/libcxx/docs/DesignDocs/RCU.rst
index ee1f2fe6fb993..21ce05d25dff1 100644
--- a/libcxx/docs/DesignDocs/RCU.rst
+++ b/libcxx/docs/DesignDocs/RCU.rst
@@ -91,9 +91,9 @@ Some key details of this design are:
- When `run_synchronize` is called, it
- flips the global state to the next phase
- - waits until all the threads that are in the critical section and entered in the previous phase to exit the critical section.
+ - Going through a grace period: waits until all the threads that are in the critical section with the previous phase to exit the critical section.
- flips the global state back to the original phase
- - waits until all the threads that are in the critical section and entered in the next phase to exit the critical section.
+ - Going through another grace period: waits until all the threads that are in the critical section ith the next phase to exit the critical section.
When `run_synchronize` returns, we can be sure that all the readers that were in the critical section before `run_synchronize` are now out of the critical section.
The paper explains why we need to wait two phases instead of just one phase in detail. The key point is that, if we only wait for the readers in the previous phase to exit,
@@ -102,13 +102,96 @@ there might be a late reader that enters the critical section after we flip the
Implementation Terminology
---------------------------
-- reader
-- grace period
-- quiescent state
+- reader: A thread that calls `rcu_domain::lock` and `rcu_domain::unlock` to enter and exit the critical section.
+
+- quiescent state: When a thread is *not* in an RCU read-side critical section (between `lock` and `unlock`),
+ it is in a quiescent state.
+
+- grace period: Any time period during which every thread has been in at least one quiescent state is a grace period.
+ The grace period cannot end until all pre-existing readers have exited their critical sections. (see the image above)
Implementation Details
----------------------
+`thread_local_container`
+~~~~~~~~~~~~~~~~~~~~~~~~
+
+A helper class that manages thread local objects. It provides APIs to get the `thread_local` object for the current thread, and to iterate through all the `thread_local` objects from all threads.
+It only contains `static` member variables and functions.
+
+It uses `mutex` to protect the list of all object pointers. Since the list is only modified when a thread first calls `get_current_thread_instance` to register itself, and when a thread exits,
+the contention on the `mutex` should be low.
+
+TODO: We need to replace `mutex` as all non allocating APIs in `rcu` are designed to be `noexcept` and `mutex` can throw.
+
+`reader_states`
+~~~~~~~~~~~~~~~
+
+It defines the state of each reader thread. The state is essentially a pair of `(phase, lock_nested_level)` merged into a single `uint16_t` .
+
+- `phase` is a single bit as the global state only has two phases. This `phase` is the phase of the global state when the current thread enters the critical section.
+- `lock_nested_level` is the nested level of the `rcu_domain::lock` . As you can call `rcu_domain::lock` multiple times in the same thread, we need to keep track of the nested level.
+ When `lock_nested_level` is `0`, it means the current thread is in a quiescent state (not in a critical section).
+
+The `reader_states` class is also a `static` helper class which only provides static definitions and functions, instead of the wrapper of the `uint16_t` state. This is because we need
+the `atomic` integer API, e.g. `fetch_or`. An `atomic` of a wrapper struct won't have these APIs.
+
+`rcu_singly_list_view`
+~~~~~~~~~~~~~~~~~~~~~~
+
+A helper class that provides a view of the intrusive singly linked list of `__rcu_node` s. It also stores the back pointer of the list to make the push back operation more efficient.
+It provides APIs to push a node to the back of the list, splice from another list and to iterate through the list.
+
+For the derived class of `rcu_obj_base`, the `retire` is `noexcept`, which means that no memory allocation can happen when pushing it to the retired callback queue.
+
+`rcu_domain_impl`
+~~~~~~~~~~~~~~~~~
+
+This is the main class that implements the `rcu` logic. It contains
+
+- `std::atomic<reader_states::state_type> global_reader_phase_` : the global state that flips between two phases. The readers will record the phase when they enter the critical section, and the writer will flip the global state when it calls `run_synchronize` to start a new grace period.
+
+- `std::mutex grace_period_mutex_` : If we have multiple writer threads calling `run_synchronize` concurrenly, we need to make sure only one of them is performing the phase flipping and deleter queue draining.
+ TODO: `mutex` can throw, we need to consider how to replace it.
+
+- `std::atomic<bool> grace_period_waiting_flag_` : This flag is used to sleep/wake up the writer thread that is waiting for the grace period to end.
+
+- `std::mutex retire_queue_mutex_` and `rcu_singly_list_view __retired_callback_queue_` : This queue stores all the retired callbacks that are waiting for the grace period to end.
+ TODO: `mutex` can throw, we need to consider how to replace it.
+
+- `rcu_singly_list_view callbacks_phase_1_` and `rcu_singly_list_view callbacks_phase_2_` : These two queues are used to let the queued retired callbacks to go through two grace periods before invocation. No additional synchronization is needed for these two queues as they are only processed when the writer thread is holding the `grace_period_mutex_` .
+
+The domain has few operations:
+
+`lock`
+^^^^^^
+
+- If current was in quiescent state, we need to record the current global phase and set the nested level to 1.
+- If the current thread was already in the critical section, we just need to increment the nested level.
+
+`unlock`
+^^^^^^^^
+
+- We need to decrement the nested level.
+- If the nested level becomes 0, it means the thread is now in a quiescent state, we can notify the waiting writer thread if there is any.
+
+`retire`
+^^^^^^^^
+
+- We need to push the retired callback to the `__retired_callback_queue_`
+
+`synchronize`
+^^^^^^^^^^^^^
+
+- We need to go through two phases of grace period
+- For each phase of the grace period, we need to flip the phase, and wait until all the reading threads are either
+
+ - in a quiescent state, or
+ - in the critical section with the new phase.
+
+- After each phase ends, we can move the callbacks to the next phase's queue, and we can evaluate the callbacks that have gone through two phases.
+
+
Design Questions
================
@@ -315,5 +398,3 @@ According to Thomas Rodgers (libstdc++'s RCU implementer), their plan is to embe
based on the presence of `SYS_membarrier` . Because `liburcu` is license compatible with libstdc++ they can directly embed the relevant source from `liburcu`
-libc++ Design
-=============
diff --git a/libcxx/include/__rcu/rcu_obj_base.h b/libcxx/include/__rcu/rcu_obj_base.h
index 4dbfe1fd07720..a8d9383ac06e5 100644
--- a/libcxx/include/__rcu/rcu_obj_base.h
+++ b/libcxx/include/__rcu/rcu_obj_base.h
@@ -28,7 +28,11 @@ template <class _Tp, class _Dp = default_delete<_Tp>>
class rcu_obj_base : private __rcu_node {
public:
_LIBCPP_HIDE_FROM_ABI void retire(_Dp __deleter = _Dp(), rcu_domain& __dom = rcu_default_domain()) noexcept {
- auto __ptr = static_cast<_Tp*>(this);
+ auto __ptr = static_cast<_Tp*>(this);
+
+ // todo: std::function can throw on the assignment. Perhaps we can store
+ // the deleter in the class and either use virtual function or use
+ // function_ref here.
__callback_ = [__ptr, __deleter = std::move(__deleter)]() mutable { __deleter(__ptr); };
__dom.__retire(this);
}
>From 389d05c6bff28c704180054ad4baf0fe2201fc24 Mon Sep 17 00:00:00 2001
From: Hui Xie <hui.xie1990 at gmail.com>
Date: Sat, 28 Feb 2026 19:07:04 +0000
Subject: [PATCH 09/65] doc
---
libcxx/docs/DesignDocs/RCU.rst | 4 ++--
1 file changed, 2 insertions(+), 2 deletions(-)
diff --git a/libcxx/docs/DesignDocs/RCU.rst b/libcxx/docs/DesignDocs/RCU.rst
index 21ce05d25dff1..72d18332fbeb2 100644
--- a/libcxx/docs/DesignDocs/RCU.rst
+++ b/libcxx/docs/DesignDocs/RCU.rst
@@ -232,8 +232,8 @@ In libc++'s design, we would like to follow Folly's approach to run these delete
When should we run the deleters
-------------------------------
-If we were to use the background thread approach, the deleters can be evaluated at any time after `run_synchronize`
-has marked the objects are safe to reclaim. e.g. Drain the queue periodically. However, if we were to run the deleters
+If we were to use the background thread approach, the deleters can be evaluated at any time after the readers have exited
+their critical sections and objects are safe to reclaim. e.g. Drain the queue periodically. However, if we were to run the deleters
inline, we have to decide when to run them.
There are few places we can run the deleters:
>From 6e02db4205207d567f7da51eb60a25fcb3491ed7 Mon Sep 17 00:00:00 2001
From: Hui <hui.xie1990 at gmail.com>
Date: Sat, 8 Aug 2026 09:34:29 +0100
Subject: [PATCH 10/65] Update libcxx/docs/DesignDocs/RCU.rst
Co-authored-by: Louis Dionne <ldionne.2 at gmail.com>
---
libcxx/docs/DesignDocs/RCU.rst | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/libcxx/docs/DesignDocs/RCU.rst b/libcxx/docs/DesignDocs/RCU.rst
index 72d18332fbeb2..7c4539fa2209f 100644
--- a/libcxx/docs/DesignDocs/RCU.rst
+++ b/libcxx/docs/DesignDocs/RCU.rst
@@ -27,7 +27,7 @@ Background Information
`rcu` is an alternative to read-write locks and it is suitable for read-mostly data structures.
Here is an example usage ::
- struct Data /* members */ ;
+ struct Data { /* members */ };
std::atomic<Data*> data_;
// ==============================================
>From 279527da06f003d3da7e4eb2020a2456f81e1333 Mon Sep 17 00:00:00 2001
From: Hui <hui.xie1990 at gmail.com>
Date: Sat, 8 Aug 2026 09:35:05 +0100
Subject: [PATCH 11/65] Update libcxx/docs/DesignDocs/RCU.rst
Co-authored-by: Louis Dionne <ldionne.2 at gmail.com>
---
libcxx/docs/DesignDocs/RCU.rst | 3 +--
1 file changed, 1 insertion(+), 2 deletions(-)
diff --git a/libcxx/docs/DesignDocs/RCU.rst b/libcxx/docs/DesignDocs/RCU.rst
index 7c4539fa2209f..3bde73c46ae2d 100644
--- a/libcxx/docs/DesignDocs/RCU.rst
+++ b/libcxx/docs/DesignDocs/RCU.rst
@@ -53,8 +53,7 @@ There are several key details that an implementation of `rcu` must address:
- On the reader side, `rcu_domain::lock` and `rcu_domain::unlock` must not block the thread while the writer thread
is performing updates.
-- On the writer side, `retire` -ing an object in principle should not block at all. However, it is conforming to
- evaluate the `deleter` s inside `run_retire` .
+- On the writer side, `retire`-ing an object in principle should not block at all. However, it is technically conforming to block (e.g. evaluate the `deleter`s directly inside `run_retire`).
- On the writer side, `rcu_synchronize` should block until at least all the existing readers exit their critical sections
via `rcu_domain::unlock` . Note that this is a key difference between `rcu` and read-write locks:
>From eac19757295632b9c4e24ba87cc899e9144475bd Mon Sep 17 00:00:00 2001
From: Hui <hui.xie1990 at gmail.com>
Date: Sat, 8 Aug 2026 09:35:38 +0100
Subject: [PATCH 12/65] Update libcxx/docs/DesignDocs/RCU.rst
Co-authored-by: Louis Dionne <ldionne.2 at gmail.com>
---
libcxx/docs/DesignDocs/RCU.rst | 4 ++--
1 file changed, 2 insertions(+), 2 deletions(-)
diff --git a/libcxx/docs/DesignDocs/RCU.rst b/libcxx/docs/DesignDocs/RCU.rst
index 3bde73c46ae2d..49fc0eafcaf9a 100644
--- a/libcxx/docs/DesignDocs/RCU.rst
+++ b/libcxx/docs/DesignDocs/RCU.rst
@@ -57,8 +57,8 @@ There are several key details that an implementation of `rcu` must address:
- On the writer side, `rcu_synchronize` should block until at least all the existing readers exit their critical sections
via `rcu_domain::unlock` . Note that this is a key difference between `rcu` and read-write locks:
- In case a late reader enters the critical section after the writer thread has called `rcu_synchronize` and waiting, the
- writer thread does not need to wait for the late reader to exit the critical section.
+ In case a late reader enters the critical section after the writer thread has called `rcu_synchronize` and started waiting, the
+ writer thread does not need to wait for the late reader to exit the critical section. In contrast, a writer thread trying to acquire a read-write lock would have to wait until the late reader releases the lock before it can acquire it.
- On the writer side, `run_barrier` should block until all the `retired` objects that happen before the `run_barrier` call are reclaimed.
>From 390b5d302ef57fd4883d74a627dbead858330f2a Mon Sep 17 00:00:00 2001
From: Hui <hui.xie1990 at gmail.com>
Date: Sat, 8 Aug 2026 09:43:13 +0100
Subject: [PATCH 13/65] Update libcxx/docs/DesignDocs/RCU.rst
Co-authored-by: Louis Dionne <ldionne.2 at gmail.com>
---
libcxx/docs/DesignDocs/RCU.rst | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/libcxx/docs/DesignDocs/RCU.rst b/libcxx/docs/DesignDocs/RCU.rst
index 49fc0eafcaf9a..92828a99734fb 100644
--- a/libcxx/docs/DesignDocs/RCU.rst
+++ b/libcxx/docs/DesignDocs/RCU.rst
@@ -60,7 +60,7 @@ There are several key details that an implementation of `rcu` must address:
In case a late reader enters the critical section after the writer thread has called `rcu_synchronize` and started waiting, the
writer thread does not need to wait for the late reader to exit the critical section. In contrast, a writer thread trying to acquire a read-write lock would have to wait until the late reader releases the lock before it can acquire it.
-- On the writer side, `run_barrier` should block until all the `retired` objects that happen before the `run_barrier` call are reclaimed.
+- On the writer side, `rcu_barrier` should block until all the objects `retired` before the `rcu_barrier` call have been reclaimed.
- The threads that are using `rcu` must be known by the `rcu` implementation states.
>From 276f3bea62d60e316b49c0520a21f07eaeff03bb Mon Sep 17 00:00:00 2001
From: Hui <hui.xie1990 at gmail.com>
Date: Sat, 8 Aug 2026 09:43:55 +0100
Subject: [PATCH 14/65] Update libcxx/docs/DesignDocs/RCU.rst
Co-authored-by: Louis Dionne <ldionne.2 at gmail.com>
---
libcxx/docs/DesignDocs/RCU.rst | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/libcxx/docs/DesignDocs/RCU.rst b/libcxx/docs/DesignDocs/RCU.rst
index 92828a99734fb..80f1b061b30b1 100644
--- a/libcxx/docs/DesignDocs/RCU.rst
+++ b/libcxx/docs/DesignDocs/RCU.rst
@@ -48,7 +48,7 @@ Here is an example usage ::
std::rcu_retire(olddata); // reclaim *olddata when safe
}
-There are several key details that an implementation of `rcu` must address:
+There are several key properties that an implementation of `rcu` must satisfy:
- On the reader side, `rcu_domain::lock` and `rcu_domain::unlock` must not block the thread while the writer thread
is performing updates.
>From 349f56a73b1aa89b3cac3f6fc133368a2c4f2459 Mon Sep 17 00:00:00 2001
From: Hui <hui.xie1990 at gmail.com>
Date: Sat, 8 Aug 2026 09:44:15 +0100
Subject: [PATCH 15/65] Update libcxx/docs/DesignDocs/RCU.rst
Co-authored-by: Louis Dionne <ldionne.2 at gmail.com>
---
libcxx/docs/DesignDocs/RCU.rst | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/libcxx/docs/DesignDocs/RCU.rst b/libcxx/docs/DesignDocs/RCU.rst
index 80f1b061b30b1..10625106480bc 100644
--- a/libcxx/docs/DesignDocs/RCU.rst
+++ b/libcxx/docs/DesignDocs/RCU.rst
@@ -68,7 +68,7 @@ There are several key properties that an implementation of `rcu` must satisfy:
Adopted Design
==============
-The core idea of `rcu` can be described in this simple image from lwn.net
+The core idea of `rcu` can be described by this image from lwn.net
.. image:: https://static.lwn.net/images/ns/kernel/rcu/GracePeriodGood.png
>From d6252af78f51fe678d876fbc458f925d828cde22 Mon Sep 17 00:00:00 2001
From: Hui <hui.xie1990 at gmail.com>
Date: Sat, 8 Aug 2026 09:45:30 +0100
Subject: [PATCH 16/65] Update libcxx/docs/DesignDocs/RCU.rst
Co-authored-by: Louis Dionne <ldionne.2 at gmail.com>
---
libcxx/docs/DesignDocs/RCU.rst | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/libcxx/docs/DesignDocs/RCU.rst b/libcxx/docs/DesignDocs/RCU.rst
index 10625106480bc..6aadfee319327 100644
--- a/libcxx/docs/DesignDocs/RCU.rst
+++ b/libcxx/docs/DesignDocs/RCU.rst
@@ -74,7 +74,7 @@ The core idea of `rcu` can be described by this image from lwn.net
- Each row is a thread. The last row is the writer thread and the rows above are the reader threads.
- Each "Reader" block represents a critical section, which starts with `rcu_domain::lock` and ends with `rcu_domain::unlock` .
-- When `run_retire` is called from the writer thread, it starts the "Removal" block.
+- When `rcu_retire` is called from the writer thread, it starts the "Removal" block.
- When `run_synchronize` is called from the writer thread, it starts the "Grace Period" block. We need to wait until all the
"Reader" blocks that started before the "Grace Period" started, to exit via `rcu_domain::unlock`, then we can end the "Grace Period".
Note that the "Grace Period" ends after the 4th row's "Reader" block ends. Also note that the "Grace Period" does not need to wait
>From eb2b08bb7928313bb0c8ee31803a4baa6d5e5864 Mon Sep 17 00:00:00 2001
From: Hui <hui.xie1990 at gmail.com>
Date: Sat, 8 Aug 2026 09:45:50 +0100
Subject: [PATCH 17/65] Update libcxx/docs/DesignDocs/RCU.rst
Co-authored-by: Louis Dionne <ldionne.2 at gmail.com>
---
libcxx/docs/DesignDocs/RCU.rst | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/libcxx/docs/DesignDocs/RCU.rst b/libcxx/docs/DesignDocs/RCU.rst
index 6aadfee319327..4dc4822457f1c 100644
--- a/libcxx/docs/DesignDocs/RCU.rst
+++ b/libcxx/docs/DesignDocs/RCU.rst
@@ -92,7 +92,7 @@ Some key details of this design are:
- flips the global state to the next phase
- Going through a grace period: waits until all the threads that are in the critical section with the previous phase to exit the critical section.
- flips the global state back to the original phase
- - Going through another grace period: waits until all the threads that are in the critical section ith the next phase to exit the critical section.
+ - Going through another grace period: waits until all the threads that are in the critical section with the next phase to exit the critical section.
When `run_synchronize` returns, we can be sure that all the readers that were in the critical section before `run_synchronize` are now out of the critical section.
The paper explains why we need to wait two phases instead of just one phase in detail. The key point is that, if we only wait for the readers in the previous phase to exit,
>From 78c61c7be8bf9fde8cb91e956bd58e3346ec7ed0 Mon Sep 17 00:00:00 2001
From: Hui Xie <hui.xie1990 at gmail.com>
Date: Sat, 8 Aug 2026 10:11:06 +0100
Subject: [PATCH 18/65] synopsis
---
libcxx/docs/DesignDocs/RCU.rst | 24 +++++++++++++++++++
libcxx/include/__rcu/rcu_domain.h | 2 +-
libcxx/src/experimental/rcu.cpp | 4 ++--
.../saferecl/saferecl.rcu/general.pass.cpp | 10 ++++----
.../saferecl/saferecl.rcu/retire.pass.cpp | 8 +++----
5 files changed, 36 insertions(+), 12 deletions(-)
diff --git a/libcxx/docs/DesignDocs/RCU.rst b/libcxx/docs/DesignDocs/RCU.rst
index 4dc4822457f1c..d5cb8d6242764 100644
--- a/libcxx/docs/DesignDocs/RCU.rst
+++ b/libcxx/docs/DesignDocs/RCU.rst
@@ -20,6 +20,30 @@ Read-Copy Update <http://www.rdrop.com/users/paulmck/RCU/urcu-supp-accepted.2011
libc++ adopted a variation that closely resembles the "General-Purpose RCU" design in the above paper.
+Synopsis
+========
+::
+
+ class rcu_domain {
+ public:
+ void lock() noexcept;
+ bool try_lock() noexcept;
+ void unlock() noexcept;
+ };
+
+ rcu_domain& rcu_default_domain() noexcept;
+
+ void rcu_synchronize(rcu_domain& dom = rcu_default_domain()) noexcept;
+ void rcu_barrier(rcu_domain& dom = rcu_default_domain()) noexcept;
+
+ template<class T, class D = default_delete<T>>
+ void rcu_retire(T* p, D d = D(), rcu_domain& dom = rcu_default_domain());
+
+ template<class T, class D = default_delete<T>>
+ class rcu_obj_base {
+ public:
+ void retire(D d = D(), rcu_domain& dom = rcu_default_domain()) noexcept;
+ };
Background Information
======================
diff --git a/libcxx/include/__rcu/rcu_domain.h b/libcxx/include/__rcu/rcu_domain.h
index a055655e36b8c..7336a1d0a9220 100644
--- a/libcxx/include/__rcu/rcu_domain.h
+++ b/libcxx/include/__rcu/rcu_domain.h
@@ -59,7 +59,7 @@ class _LIBCPP_EXPORTED_FROM_ABI rcu_domain {
rcu_domain& operator=(const rcu_domain&) = delete;
~rcu_domain();
- void printAllReaderStatesInHex();
+ void debugPrintAllReaderStatesInHex();
void lock() noexcept;
diff --git a/libcxx/src/experimental/rcu.cpp b/libcxx/src/experimental/rcu.cpp
index 6bf69b74c9b00..3153b68dcd040 100644
--- a/libcxx/src/experimental/rcu.cpp
+++ b/libcxx/src/experimental/rcu.cpp
@@ -223,7 +223,7 @@ class rcu_domain_impl {
__cxx_atomic_thread_fence(memory_order_seq_cst);
}
- void printAllReaderStatesInHex() {
+ void debugPrintAllReaderStatesInHex() {
per_thread_states::for_each([](auto __state_ref) { std::printf("Reader state: 0x%04x\n", __state_ref.load()); });
}
};
@@ -239,7 +239,7 @@ rcu_domain& rcu_domain::__rcu_default_domain() noexcept {
rcu_domain::rcu_domain() : __pimpl_(std::make_unique<__impl>()) {}
rcu_domain::~rcu_domain() = default;
-void rcu_domain::printAllReaderStatesInHex() { __pimpl_->printAllReaderStatesInHex(); }
+void rcu_domain::debugPrintAllReaderStatesInHex() { __pimpl_->debugPrintAllReaderStatesInHex(); }
void rcu_domain::lock() noexcept { __pimpl_->lock(); }
diff --git a/libcxx/test/std/thread/saferecl/saferecl.rcu/general.pass.cpp b/libcxx/test/std/thread/saferecl/saferecl.rcu/general.pass.cpp
index b94643db87c4b..e2168d09210cc 100644
--- a/libcxx/test/std/thread/saferecl/saferecl.rcu/general.pass.cpp
+++ b/libcxx/test/std/thread/saferecl/saferecl.rcu/general.pass.cpp
@@ -40,7 +40,7 @@ int main(int, char**) {
log(start, "t1: leaving rcu read-side critical section " + std::to_string(i));
dom.unlock();
log(start, "t1: printing all reader states");
- dom.printAllReaderStatesInHex();
+ dom.debugPrintAllReaderStatesInHex();
}
});
@@ -55,7 +55,7 @@ int main(int, char**) {
log(start, "t2: leaving rcu read-side critical section " + std::to_string(i));
dom.unlock();
log(start, "t2: printing all reader states");
- dom.printAllReaderStatesInHex();
+ dom.debugPrintAllReaderStatesInHex();
}
});
@@ -70,7 +70,7 @@ int main(int, char**) {
log(start, "t3: leaving rcu read-side critical section " + std::to_string(i));
dom.unlock();
log(start, "t3: printing all reader states");
- dom.printAllReaderStatesInHex();
+ dom.debugPrintAllReaderStatesInHex();
}
});
auto t4 = support::make_test_thread([start]() {
@@ -84,7 +84,7 @@ int main(int, char**) {
log(start, "t4: leaving rcu read-side critical section " + std::to_string(i));
dom.unlock();
log(start, "t4: printing all reader states");
- dom.printAllReaderStatesInHex();
+ dom.debugPrintAllReaderStatesInHex();
}
});
/*
@@ -102,7 +102,7 @@ int main(int, char**) {
for (int i = 0; i < loop_num+5; ++i) {
log(start, "t0: printing all reader states before synchronize");
- std::rcu_default_domain().printAllReaderStatesInHex();
+ std::rcu_default_domain().debugPrintAllReaderStatesInHex();
log(start, "t0: calling rcu_synchronize" + std::to_string(i));
std::rcu_synchronize();
log(start, "t0: rcu_synchronize returned" + std::to_string(i));
diff --git a/libcxx/test/std/thread/saferecl/saferecl.rcu/retire.pass.cpp b/libcxx/test/std/thread/saferecl/saferecl.rcu/retire.pass.cpp
index dde6cdd323084..9f7ef08240091 100644
--- a/libcxx/test/std/thread/saferecl/saferecl.rcu/retire.pass.cpp
+++ b/libcxx/test/std/thread/saferecl/saferecl.rcu/retire.pass.cpp
@@ -56,7 +56,7 @@ int main(int, char**) {
(obj ? obj->data_ : "nullptr"));
dom.unlock();
log(start, "t1: printing all reader states");
- dom.printAllReaderStatesInHex();
+ dom.debugPrintAllReaderStatesInHex();
}
});
@@ -73,7 +73,7 @@ int main(int, char**) {
(obj ? obj->data_ : "nullptr"));
dom.unlock();
log(start, "t2: printing all reader states");
- dom.printAllReaderStatesInHex();
+ dom.debugPrintAllReaderStatesInHex();
}
});
@@ -90,7 +90,7 @@ int main(int, char**) {
(obj ? obj->data_ : "nullptr"));
dom.unlock();
log(start, "t3: printing all reader states");
- dom.printAllReaderStatesInHex();
+ dom.debugPrintAllReaderStatesInHex();
}
});
auto t4 = support::make_test_thread([start]() {
@@ -106,7 +106,7 @@ int main(int, char**) {
(obj ? obj->data_ : "nullptr"));
dom.unlock();
log(start, "t4: printing all reader states");
- dom.printAllReaderStatesInHex();
+ dom.debugPrintAllReaderStatesInHex();
}
});
>From e5d270847bd4e3136bea88fc50b5abc2f5e4d037 Mon Sep 17 00:00:00 2001
From: Hui Xie <hui.xie1990 at gmail.com>
Date: Sat, 8 Aug 2026 10:26:43 +0100
Subject: [PATCH 19/65] explicit export
---
libcxx/include/__rcu/rcu_domain.h | 2 ++
libcxx/src/experimental/rcu.cpp | 2 ++
2 files changed, 4 insertions(+)
diff --git a/libcxx/include/__rcu/rcu_domain.h b/libcxx/include/__rcu/rcu_domain.h
index 7336a1d0a9220..4ddd9e6190f51 100644
--- a/libcxx/include/__rcu/rcu_domain.h
+++ b/libcxx/include/__rcu/rcu_domain.h
@@ -20,6 +20,7 @@
#endif
_LIBCPP_BEGIN_NAMESPACE_STD
+_LIBCPP_BEGIN_EXPLICIT_ABI_ANNOTATIONS
#if _LIBCPP_STD_VER >= 26 && _LIBCPP_HAS_THREADS && _LIBCPP_HAS_EXPERIMENTAL_RCU
@@ -84,6 +85,7 @@ _LIBCPP_HIDE_FROM_ABI void rcu_retire(_Tp* __tp, _Dp __deleter = _Dp(), rcu_doma
#endif // _LIBCPP_STD_VER >= 26 && _LIBCPP_HAS_THREADS && _LIBCPP_HAS_EXPERIMENTAL_RCU
+_LIBCPP_END_EXPLICIT_ABI_ANNOTATIONS
_LIBCPP_END_NAMESPACE_STD
#endif // _LIBCPP___RCU_RCU_DOMAIN_H
diff --git a/libcxx/src/experimental/rcu.cpp b/libcxx/src/experimental/rcu.cpp
index 3153b68dcd040..dde4693dc1138 100644
--- a/libcxx/src/experimental/rcu.cpp
+++ b/libcxx/src/experimental/rcu.cpp
@@ -16,6 +16,7 @@
#include "include/rcu/rcu_list.h"
_LIBCPP_BEGIN_NAMESPACE_STD
+_LIBCPP_BEGIN_EXPLICIT_ABI_ANNOTATIONS
namespace {
@@ -253,4 +254,5 @@ void rcu_synchronize(rcu_domain& dom) noexcept { dom.__pimpl_->synchronize(); }
void rcu_barrier(rcu_domain& dom) noexcept { rcu_synchronize(dom); }
+_LIBCPP_END_EXPLICIT_ABI_ANNOTATIONS
_LIBCPP_END_NAMESPACE_STD
>From 85ade31dbc97c08098bfb8a2ba3984e874d9b398 Mon Sep 17 00:00:00 2001
From: Hui Xie <hui.xie1990 at gmail.com>
Date: Mon, 10 Aug 2026 16:49:37 +0100
Subject: [PATCH 20/65] typo
---
libcxx/docs/DesignDocs/RCU.rst | 34 +++++++++++++++++-----------------
1 file changed, 17 insertions(+), 17 deletions(-)
diff --git a/libcxx/docs/DesignDocs/RCU.rst b/libcxx/docs/DesignDocs/RCU.rst
index d5cb8d6242764..29a1070c7c914 100644
--- a/libcxx/docs/DesignDocs/RCU.rst
+++ b/libcxx/docs/DesignDocs/RCU.rst
@@ -77,7 +77,7 @@ There are several key properties that an implementation of `rcu` must satisfy:
- On the reader side, `rcu_domain::lock` and `rcu_domain::unlock` must not block the thread while the writer thread
is performing updates.
-- On the writer side, `retire`-ing an object in principle should not block at all. However, it is technically conforming to block (e.g. evaluate the `deleter`s directly inside `run_retire`).
+- On the writer side, `retire`-ing an object in principle should not block at all. However, it is technically conforming to block (e.g. evaluate the `deleter`s directly inside `rcu_retire`).
- On the writer side, `rcu_synchronize` should block until at least all the existing readers exit their critical sections
via `rcu_domain::unlock` . Note that this is a key difference between `rcu` and read-write locks:
@@ -99,7 +99,7 @@ The core idea of `rcu` can be described by this image from lwn.net
- Each row is a thread. The last row is the writer thread and the rows above are the reader threads.
- Each "Reader" block represents a critical section, which starts with `rcu_domain::lock` and ends with `rcu_domain::unlock` .
- When `rcu_retire` is called from the writer thread, it starts the "Removal" block.
-- When `run_synchronize` is called from the writer thread, it starts the "Grace Period" block. We need to wait until all the
+- When `rcu_synchronize` is called from the writer thread, it starts the "Grace Period" block. We need to wait until all the
"Reader" blocks that started before the "Grace Period" started, to exit via `rcu_domain::unlock`, then we can end the "Grace Period".
Note that the "Grace Period" ends after the 4th row's "Reader" block ends. Also note that the "Grace Period" does not need to wait
for the late "Reader" blocks.
@@ -111,14 +111,14 @@ Some key details of this design are:
- There is a global state which has two phases and it flips between the two phases.
- Each thread stores its state: whether there is a reader in the critical section, and which phase it was when it entered the critical section.
-- When `run_synchronize` is called, it
+- When `rcu_synchronize` is called, it
- flips the global state to the next phase
- Going through a grace period: waits until all the threads that are in the critical section with the previous phase to exit the critical section.
- flips the global state back to the original phase
- Going through another grace period: waits until all the threads that are in the critical section with the next phase to exit the critical section.
-When `run_synchronize` returns, we can be sure that all the readers that were in the critical section before `run_synchronize` are now out of the critical section.
+When `rcu_synchronize` returns, we can be sure that all the readers that were in the critical section before `rcu_synchronize` are now out of the critical section.
The paper explains why we need to wait two phases instead of just one phase in detail. The key point is that, if we only wait for the readers in the previous phase to exit,
there might be a late reader that enters the critical section after we flip the global state and before we wait for the previous phase's readers to exit.
@@ -172,9 +172,9 @@ For the derived class of `rcu_obj_base`, the `retire` is `noexcept`, which means
This is the main class that implements the `rcu` logic. It contains
-- `std::atomic<reader_states::state_type> global_reader_phase_` : the global state that flips between two phases. The readers will record the phase when they enter the critical section, and the writer will flip the global state when it calls `run_synchronize` to start a new grace period.
+- `std::atomic<reader_states::state_type> global_reader_phase_` : the global state that flips between two phases. The readers will record the phase when they enter the critical section, and the writer will flip the global state when it calls `rcu_synchronize` to start a new grace period.
-- `std::mutex grace_period_mutex_` : If we have multiple writer threads calling `run_synchronize` concurrenly, we need to make sure only one of them is performing the phase flipping and deleter queue draining.
+- `std::mutex grace_period_mutex_` : If we have multiple writer threads calling `rcu_synchronize` concurrenly, we need to make sure only one of them is performing the phase flipping and deleter queue draining.
TODO: `mutex` can throw, we need to consider how to replace it.
- `std::atomic<bool> grace_period_waiting_flag_` : This flag is used to sleep/wake up the writer thread that is waiting for the grace period to end.
@@ -249,8 +249,8 @@ And Thomas Rodgers (libstdc++'s RCU implementer) also said:
thread. But on the other hand, users of the library might find it surprising that then gave the library permission
to also do work on those threads in ways they might not expect.
-In libc++'s design, we would like to follow Folly's approach to run these deleters inline when `run_synchronize` or
-`run_barrier` is called.
+In libc++'s design, we would like to follow Folly's approach to run these deleters inline when `rcu_synchronize` or
+`rcu_barrier` is called.
When should we run the deleters
-------------------------------
@@ -261,23 +261,23 @@ inline, we have to decide when to run them.
There are few places we can run the deleters:
-- Inside `run_synchronize` after the writer thread is unblocked. This approach has the advantage that the writer thread
+- Inside `rcu_synchronize` after the writer thread is unblocked. This approach has the advantage that the writer thread
can reclaim the retired objects as soon as possible. However, it has the disadvantage that the writer thread may be
blocked for a long time if there are many retired objects to reclaim.
-- Inside `run_barrier` after the writer thread is unblocked. Since `run_barrier` is designed to block until all the retired
- objects that happen before the `run_barrier` call are reclaimed, it is natural to run the deleters here.
+- Inside `rcu_barrier` after the writer thread is unblocked. Since `rcu_barrier` is designed to block until all the retired
+ objects that happen before the `rcu_barrier` call are reclaimed, it is natural to run the deleters here.
-- Inside `run_retire` after the deleter is put into the queue. `run_retire` is designed not to block. However, if there are
+- Inside `rcu_retire` after the deleter is put into the queue. `rcu_retire` is designed not to block. However, if there are
objects that are safe to reclaim due to readers have exited their critical sections, and at the same time, there is no other
writer threads that are currently draining the deleter queue, we can take the opportunity to run the deleters.
Folly currently takes the inline approach and runs the deleters at all three places mentioned above.
-In libc++'s design, we would like to start with the inline approach and only run the deleters inside `run_synchronize` and
-`run_barrier` for simplicity. Whether or not running the deleters inside `run_retire` is debatable. On the one hand, running
-them will make `run_retire` take more time to return. But not running them will make the retired objects stay in the queue
-for a longer time. In an extreme case, if the user never calls `run_synchronize` or `run_barrier` after calling `run_retire`,
+In libc++'s design, we would like to start with the inline approach and only run the deleters inside `rcu_synchronize` and
+`rcu_barrier` for simplicity. Whether or not running the deleters inside `rcu_retire` is debatable. On the one hand, running
+them will make `rcu_retire` take more time to return. But not running them will make the retired objects stay in the queue
+for a longer time. In an extreme case, if the user never calls `rcu_synchronize` or `rcu_barrier` after calling `rcu_retire`,
the retired objects will never be reclaimed (without a background thread to drain the queue).
Almost all APIs are `noexcept` . Is it designed to avoid memory allocation and avoid using `mutex` ?
@@ -299,7 +299,7 @@ There are few places that might need to use `mutex`:
- The destruction of a thread, which might need to do some clean up work for the `rcu` states.
-Memory Allocation can throw as well. Luckily `run_retire` is not marked as `noexcept` in the paper, as `run_retire` is
+Memory Allocation can throw as well. Luckily `rcu_retire` is not marked as `noexcept` in the paper, as `rcu_retire` is
likely to allocate memory to create a type erased object into the deleter queue.
However, there are other places where it might need to allocate memory. For example, see Folly's comments
>From 05d9907b5ca66f233418290a1123aba30510760d Mon Sep 17 00:00:00 2001
From: Hui <hui.xie1990 at gmail.com>
Date: Sat, 15 Aug 2026 10:20:43 +0100
Subject: [PATCH 21/65] Update libcxx/docs/DesignDocs/RCU.rst
Co-authored-by: Louis Dionne <ldionne.2 at gmail.com>
---
libcxx/docs/DesignDocs/RCU.rst | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/libcxx/docs/DesignDocs/RCU.rst b/libcxx/docs/DesignDocs/RCU.rst
index 29a1070c7c914..c4df92c27b7df 100644
--- a/libcxx/docs/DesignDocs/RCU.rst
+++ b/libcxx/docs/DesignDocs/RCU.rst
@@ -190,7 +190,7 @@ The domain has few operations:
^^^^^^
- If current was in quiescent state, we need to record the current global phase and set the nested level to 1.
-- If the current thread was already in the critical section, we just need to increment the nested level.
+- If the current thread was already in a critical section, just increment the nested level.
`unlock`
^^^^^^^^
>From 06d60017098f450565f3f749e8ff3fdf9c8e78a8 Mon Sep 17 00:00:00 2001
From: Hui <hui.xie1990 at gmail.com>
Date: Sat, 15 Aug 2026 10:20:51 +0100
Subject: [PATCH 22/65] Update libcxx/docs/DesignDocs/RCU.rst
Co-authored-by: Louis Dionne <ldionne.2 at gmail.com>
---
libcxx/docs/DesignDocs/RCU.rst | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/libcxx/docs/DesignDocs/RCU.rst b/libcxx/docs/DesignDocs/RCU.rst
index c4df92c27b7df..69104a05e7511 100644
--- a/libcxx/docs/DesignDocs/RCU.rst
+++ b/libcxx/docs/DesignDocs/RCU.rst
@@ -274,7 +274,7 @@ There are few places we can run the deleters:
Folly currently takes the inline approach and runs the deleters at all three places mentioned above.
-In libc++'s design, we would like to start with the inline approach and only run the deleters inside `rcu_synchronize` and
+In libc++'s design, we use the inline approach and only run the deleters inside `rcu_synchronize` and
`rcu_barrier` for simplicity. Whether or not running the deleters inside `rcu_retire` is debatable. On the one hand, running
them will make `rcu_retire` take more time to return. But not running them will make the retired objects stay in the queue
for a longer time. In an extreme case, if the user never calls `rcu_synchronize` or `rcu_barrier` after calling `rcu_retire`,
>From 79722ddee132d95b682ddd7eb099765e9e7e697b Mon Sep 17 00:00:00 2001
From: Hui <hui.xie1990 at gmail.com>
Date: Sat, 15 Aug 2026 10:21:00 +0100
Subject: [PATCH 23/65] Update libcxx/docs/DesignDocs/RCU.rst
Co-authored-by: Louis Dionne <ldionne.2 at gmail.com>
---
libcxx/docs/DesignDocs/RCU.rst | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/libcxx/docs/DesignDocs/RCU.rst b/libcxx/docs/DesignDocs/RCU.rst
index 69104a05e7511..bd3a633589139 100644
--- a/libcxx/docs/DesignDocs/RCU.rst
+++ b/libcxx/docs/DesignDocs/RCU.rst
@@ -189,7 +189,7 @@ The domain has few operations:
`lock`
^^^^^^
-- If current was in quiescent state, we need to record the current global phase and set the nested level to 1.
+- If the current thread was in quiescent state, record the current global phase and set the nested level to 1.
- If the current thread was already in a critical section, just increment the nested level.
`unlock`
>From edee8b974b3b0c43b8d039ee5ad38f5456f867af Mon Sep 17 00:00:00 2001
From: Hui <hui.xie1990 at gmail.com>
Date: Sat, 15 Aug 2026 10:21:10 +0100
Subject: [PATCH 24/65] Update libcxx/docs/DesignDocs/RCU.rst
Co-authored-by: Louis Dionne <ldionne.2 at gmail.com>
---
libcxx/docs/DesignDocs/RCU.rst | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/libcxx/docs/DesignDocs/RCU.rst b/libcxx/docs/DesignDocs/RCU.rst
index bd3a633589139..102834c24f0fa 100644
--- a/libcxx/docs/DesignDocs/RCU.rst
+++ b/libcxx/docs/DesignDocs/RCU.rst
@@ -182,7 +182,7 @@ This is the main class that implements the `rcu` logic. It contains
- `std::mutex retire_queue_mutex_` and `rcu_singly_list_view __retired_callback_queue_` : This queue stores all the retired callbacks that are waiting for the grace period to end.
TODO: `mutex` can throw, we need to consider how to replace it.
-- `rcu_singly_list_view callbacks_phase_1_` and `rcu_singly_list_view callbacks_phase_2_` : These two queues are used to let the queued retired callbacks to go through two grace periods before invocation. No additional synchronization is needed for these two queues as they are only processed when the writer thread is holding the `grace_period_mutex_` .
+- `rcu_singly_list_view callbacks_phase_1_` and `rcu_singly_list_view callbacks_phase_2_` : These two queues are used to let the retired callbacks go through two grace periods before being invoked. No additional synchronization is needed for these two queues as they are only processed when the collector (?) thread is holding the `grace_period_mutex_` .
The domain has few operations:
>From 36577f77d816fdf03ff174d84be78cb95e4b6c01 Mon Sep 17 00:00:00 2001
From: Hui <hui.xie1990 at gmail.com>
Date: Sat, 15 Aug 2026 10:22:31 +0100
Subject: [PATCH 25/65] Update libcxx/docs/DesignDocs/RCU.rst
Co-authored-by: Louis Dionne <ldionne.2 at gmail.com>
---
libcxx/docs/DesignDocs/RCU.rst | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/libcxx/docs/DesignDocs/RCU.rst b/libcxx/docs/DesignDocs/RCU.rst
index 102834c24f0fa..f168652fb65bf 100644
--- a/libcxx/docs/DesignDocs/RCU.rst
+++ b/libcxx/docs/DesignDocs/RCU.rst
@@ -195,7 +195,7 @@ The domain has few operations:
`unlock`
^^^^^^^^
-- We need to decrement the nested level.
+- Decrement the nested level.
- If the nested level becomes 0, it means the thread is now in a quiescent state, we can notify the waiting writer thread if there is any.
`retire`
>From 317a0b2523d46ebf9965c66f3f4bf525c079bcf7 Mon Sep 17 00:00:00 2001
From: Hui <hui.xie1990 at gmail.com>
Date: Sat, 15 Aug 2026 10:22:44 +0100
Subject: [PATCH 26/65] Update libcxx/docs/DesignDocs/RCU.rst
Co-authored-by: Louis Dionne <ldionne.2 at gmail.com>
---
libcxx/docs/DesignDocs/RCU.rst | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/libcxx/docs/DesignDocs/RCU.rst b/libcxx/docs/DesignDocs/RCU.rst
index f168652fb65bf..634a25bdabc84 100644
--- a/libcxx/docs/DesignDocs/RCU.rst
+++ b/libcxx/docs/DesignDocs/RCU.rst
@@ -196,7 +196,7 @@ The domain has few operations:
^^^^^^^^
- Decrement the nested level.
-- If the nested level becomes 0, it means the thread is now in a quiescent state, we can notify the waiting writer thread if there is any.
+- If the nested level becomes 0, the thread is now in a quiescent state, we can notify the waiting collector thread if there is any.
`retire`
^^^^^^^^
>From ce2944f2a6e9bbc6795bce22ec42c7ee63aa79d2 Mon Sep 17 00:00:00 2001
From: Hui <hui.xie1990 at gmail.com>
Date: Sat, 15 Aug 2026 10:23:22 +0100
Subject: [PATCH 27/65] Update libcxx/docs/DesignDocs/RCU.rst
Co-authored-by: Louis Dionne <ldionne.2 at gmail.com>
---
libcxx/docs/DesignDocs/RCU.rst | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/libcxx/docs/DesignDocs/RCU.rst b/libcxx/docs/DesignDocs/RCU.rst
index 634a25bdabc84..a7f87a91e901d 100644
--- a/libcxx/docs/DesignDocs/RCU.rst
+++ b/libcxx/docs/DesignDocs/RCU.rst
@@ -212,7 +212,7 @@ The domain has few operations:
- in a quiescent state, or
- in the critical section with the new phase.
-- After each phase ends, we can move the callbacks to the next phase's queue, and we can evaluate the callbacks that have gone through two phases.
+- After each phase ends, we can move the callbacks to the next stage's queue, and we can evaluate the callbacks that have gone through two stages.
Design Questions
>From c55d37210138a1dbb6c58a7c01970921cfeeb3a5 Mon Sep 17 00:00:00 2001
From: Hui <hui.xie1990 at gmail.com>
Date: Sat, 15 Aug 2026 10:23:44 +0100
Subject: [PATCH 28/65] Update libcxx/docs/DesignDocs/RCU.rst
Co-authored-by: Louis Dionne <ldionne.2 at gmail.com>
---
libcxx/docs/DesignDocs/RCU.rst | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/libcxx/docs/DesignDocs/RCU.rst b/libcxx/docs/DesignDocs/RCU.rst
index a7f87a91e901d..a34197ff7144c 100644
--- a/libcxx/docs/DesignDocs/RCU.rst
+++ b/libcxx/docs/DesignDocs/RCU.rst
@@ -238,7 +238,7 @@ has pointed out that:
I guess. But you can get the same effect by properly restricting where
the background threads run.
-And Thomas Rodgers (libstdc++'s RCU implementer) also said:
+And Thomas Rodgers (libstdc++ contributor) also said:
We don't normally do this, no, and I don't think I'd ever directly pursue such an approach in isolation. If,
on the other hand I could derive a permission from the user of library to create threads, by explicit use
>From e3cc75ea1b003f53a53ba07ebb8f4b8abe17b8f1 Mon Sep 17 00:00:00 2001
From: Hui <hui.xie1990 at gmail.com>
Date: Sat, 15 Aug 2026 10:25:00 +0100
Subject: [PATCH 29/65] Update libcxx/docs/DesignDocs/RCU.rst
Co-authored-by: Louis Dionne <ldionne.2 at gmail.com>
---
libcxx/docs/DesignDocs/RCU.rst | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/libcxx/docs/DesignDocs/RCU.rst b/libcxx/docs/DesignDocs/RCU.rst
index a34197ff7144c..233c60fbfbd9c 100644
--- a/libcxx/docs/DesignDocs/RCU.rst
+++ b/libcxx/docs/DesignDocs/RCU.rst
@@ -256,7 +256,7 @@ When should we run the deleters
-------------------------------
If we were to use the background thread approach, the deleters can be evaluated at any time after the readers have exited
-their critical sections and objects are safe to reclaim. e.g. Drain the queue periodically. However, if we were to run the deleters
+their critical sections and objects are safe to reclaim. e.g. drain the queue periodically. However, if we were to run the deleters
inline, we have to decide when to run them.
There are few places we can run the deleters:
>From 5808ca9a1022e0b99805d3ca64a32555f08348b9 Mon Sep 17 00:00:00 2001
From: Hui Xie <hui.xie1990 at gmail.com>
Date: Wed, 19 Aug 2026 11:06:51 +0100
Subject: [PATCH 30/65] delete node
---
libcxx/include/__rcu/rcu_domain.h | 5 ++++-
1 file changed, 4 insertions(+), 1 deletion(-)
diff --git a/libcxx/include/__rcu/rcu_domain.h b/libcxx/include/__rcu/rcu_domain.h
index 4ddd9e6190f51..5c8a4667abca8 100644
--- a/libcxx/include/__rcu/rcu_domain.h
+++ b/libcxx/include/__rcu/rcu_domain.h
@@ -47,7 +47,10 @@ class _LIBCPP_EXPORTED_FROM_ABI rcu_domain {
template <class _Tp, class _Dp = default_delete<_Tp>>
_LIBCPP_HIDE_FROM_ABI void __rcu_retire_hidden_friend(_Tp* __tp, _Dp __deleter, rcu_domain& __dom) {
auto* __node = new __rcu_node();
- __node->__callback_ = [__tp, __deleter = std::move(__deleter)]() mutable { __deleter(__tp); };
+ __node->__callback_ = [__tp, __node, __deleter = std::move(__deleter)]() mutable {
+ __deleter(__tp);
+ delete __node;
+ };
__dom.__retire(__node);
}
>From e053cc94a5ee867097521f7837f1b9348db45860 Mon Sep 17 00:00:00 2001
From: Hui Xie <hui.xie1990 at gmail.com>
Date: Wed, 19 Aug 2026 13:50:46 +0100
Subject: [PATCH 31/65] documentation writer->collector
---
libcxx/docs/DesignDocs/RCU.rst | 28 ++++++++++++++--------------
1 file changed, 14 insertions(+), 14 deletions(-)
diff --git a/libcxx/docs/DesignDocs/RCU.rst b/libcxx/docs/DesignDocs/RCU.rst
index 233c60fbfbd9c..1c29aa5712246 100644
--- a/libcxx/docs/DesignDocs/RCU.rst
+++ b/libcxx/docs/DesignDocs/RCU.rst
@@ -75,16 +75,16 @@ Here is an example usage ::
There are several key properties that an implementation of `rcu` must satisfy:
- On the reader side, `rcu_domain::lock` and `rcu_domain::unlock` must not block the thread while the writer thread
- is performing updates.
+ is performing updates, or the collector thread is reclaiming the objects.
- On the writer side, `retire`-ing an object in principle should not block at all. However, it is technically conforming to block (e.g. evaluate the `deleter`s directly inside `rcu_retire`).
-- On the writer side, `rcu_synchronize` should block until at least all the existing readers exit their critical sections
+- On the collector side, `rcu_synchronize` should block until at least all the existing readers exit their critical sections
via `rcu_domain::unlock` . Note that this is a key difference between `rcu` and read-write locks:
- In case a late reader enters the critical section after the writer thread has called `rcu_synchronize` and started waiting, the
- writer thread does not need to wait for the late reader to exit the critical section. In contrast, a writer thread trying to acquire a read-write lock would have to wait until the late reader releases the lock before it can acquire it.
+ In case a late reader enters the critical section after the collector thread has called `rcu_synchronize` and started waiting, the
+ collector thread does not need to wait for the late reader to exit the critical section. In contrast, a collector thread trying to acquire a read-write lock would have to wait until the late reader releases the lock before it can acquire it.
-- On the writer side, `rcu_barrier` should block until all the objects `retired` before the `rcu_barrier` call have been reclaimed.
+- On the collector side, `rcu_barrier` should block until all the objects `retired` before the `rcu_barrier` call have been reclaimed.
- The threads that are using `rcu` must be known by the `rcu` implementation states.
@@ -96,10 +96,10 @@ The core idea of `rcu` can be described by this image from lwn.net
.. image:: https://static.lwn.net/images/ns/kernel/rcu/GracePeriodGood.png
-- Each row is a thread. The last row is the writer thread and the rows above are the reader threads.
+- Each row is a thread. The last row is the collector thread and the rows above are the reader threads.
- Each "Reader" block represents a critical section, which starts with `rcu_domain::lock` and ends with `rcu_domain::unlock` .
- When `rcu_retire` is called from the writer thread, it starts the "Removal" block.
-- When `rcu_synchronize` is called from the writer thread, it starts the "Grace Period" block. We need to wait until all the
+- When `rcu_synchronize` is called from the collector thread, it starts the "Grace Period" block. We need to wait until all the
"Reader" blocks that started before the "Grace Period" started, to exit via `rcu_domain::unlock`, then we can end the "Grace Period".
Note that the "Grace Period" ends after the 4th row's "Reader" block ends. Also note that the "Grace Period" does not need to wait
for the late "Reader" blocks.
@@ -172,12 +172,12 @@ For the derived class of `rcu_obj_base`, the `retire` is `noexcept`, which means
This is the main class that implements the `rcu` logic. It contains
-- `std::atomic<reader_states::state_type> global_reader_phase_` : the global state that flips between two phases. The readers will record the phase when they enter the critical section, and the writer will flip the global state when it calls `rcu_synchronize` to start a new grace period.
+- `std::atomic<reader_states::state_type> global_reader_phase_` : the global state that flips between two phases. The readers will record the phase when they enter the critical section, and the collector will flip the global state when it calls `rcu_synchronize` to start a new grace period.
-- `std::mutex grace_period_mutex_` : If we have multiple writer threads calling `rcu_synchronize` concurrenly, we need to make sure only one of them is performing the phase flipping and deleter queue draining.
+- `std::mutex grace_period_mutex_` : If we have multiple collector threads calling `rcu_synchronize` concurrenly, we need to make sure only one of them is performing the phase flipping and deleter queue draining.
TODO: `mutex` can throw, we need to consider how to replace it.
-- `std::atomic<bool> grace_period_waiting_flag_` : This flag is used to sleep/wake up the writer thread that is waiting for the grace period to end.
+- `std::atomic<bool> grace_period_waiting_flag_` : This flag is used to sleep/wake up the collector thread that is waiting for the grace period to end.
- `std::mutex retire_queue_mutex_` and `rcu_singly_list_view __retired_callback_queue_` : This queue stores all the retired callbacks that are waiting for the grace period to end.
TODO: `mutex` can throw, we need to consider how to replace it.
@@ -261,16 +261,16 @@ inline, we have to decide when to run them.
There are few places we can run the deleters:
-- Inside `rcu_synchronize` after the writer thread is unblocked. This approach has the advantage that the writer thread
- can reclaim the retired objects as soon as possible. However, it has the disadvantage that the writer thread may be
+- Inside `rcu_synchronize` after the collector thread is unblocked. This approach has the advantage that the collector thread
+ can reclaim the retired objects as soon as possible. However, it has the disadvantage that the collector thread may be
blocked for a long time if there are many retired objects to reclaim.
-- Inside `rcu_barrier` after the writer thread is unblocked. Since `rcu_barrier` is designed to block until all the retired
+- Inside `rcu_barrier` after the collector thread is unblocked. Since `rcu_barrier` is designed to block until all the retired
objects that happen before the `rcu_barrier` call are reclaimed, it is natural to run the deleters here.
- Inside `rcu_retire` after the deleter is put into the queue. `rcu_retire` is designed not to block. However, if there are
objects that are safe to reclaim due to readers have exited their critical sections, and at the same time, there is no other
- writer threads that are currently draining the deleter queue, we can take the opportunity to run the deleters.
+ collector/writer threads that are currently draining the deleter queue, we can take the opportunity to run the deleters.
Folly currently takes the inline approach and runs the deleters at all three places mentioned above.
>From f4d1bd18365e300c7c4f563b7b7fcf6c0fe4d0f0 Mon Sep 17 00:00:00 2001
From: Hui Xie <hui.xie1990 at gmail.com>
Date: Wed, 19 Aug 2026 15:26:34 +0100
Subject: [PATCH 32/65] queue lock free
---
libcxx/docs/DesignDocs/RCU.rst | 5 +-
.../src/experimental/include/rcu/rcu_list.h | 65 ++++++--
.../include/rcu/thread_local_container.h | 85 ++++++++++
libcxx/src/experimental/rcu.cpp | 61 +------
...d_write_lock_compare_multi_writer.pass.cpp | 154 ++++++++++++++++++
5 files changed, 301 insertions(+), 69 deletions(-)
create mode 100644 libcxx/src/experimental/include/rcu/thread_local_container.h
create mode 100644 libcxx/test/std/thread/saferecl/saferecl.rcu/read_write_lock_compare_multi_writer.pass.cpp
diff --git a/libcxx/docs/DesignDocs/RCU.rst b/libcxx/docs/DesignDocs/RCU.rst
index 1c29aa5712246..68aadd0f4c696 100644
--- a/libcxx/docs/DesignDocs/RCU.rst
+++ b/libcxx/docs/DesignDocs/RCU.rst
@@ -180,7 +180,10 @@ This is the main class that implements the `rcu` logic. It contains
- `std::atomic<bool> grace_period_waiting_flag_` : This flag is used to sleep/wake up the collector thread that is waiting for the grace period to end.
- `std::mutex retire_queue_mutex_` and `rcu_singly_list_view __retired_callback_queue_` : This queue stores all the retired callbacks that are waiting for the grace period to end.
- TODO: `mutex` can throw, we need to consider how to replace it.
+
+ - TODO: `mutex` can throw, we need to consider how to replace it.
+
+ - Design 2: The access pattern of this list in this implementation is very special. The writer threads are only going to push to this queue. And the collector threads are only going to splice the entire queue. It is easy to implement it with a single `atomic<pair<Head*, Tail*>>`. It will be efficient if the platform supports 16 bytes CAS. To reduce contention between multiple writer threads retiring objects, we let each thread have a thread local queue
- `rcu_singly_list_view callbacks_phase_1_` and `rcu_singly_list_view callbacks_phase_2_` : These two queues are used to let the retired callbacks go through two grace periods before being invoked. No additional synchronization is needed for these two queues as they are only processed when the collector (?) thread is holding the `grace_period_mutex_` .
diff --git a/libcxx/src/experimental/include/rcu/rcu_list.h b/libcxx/src/experimental/include/rcu/rcu_list.h
index 199c3c67e4275..60d1380765ac0 100644
--- a/libcxx/src/experimental/include/rcu/rcu_list.h
+++ b/libcxx/src/experimental/include/rcu/rcu_list.h
@@ -13,6 +13,9 @@
#include <__config>
#include <__functional/function.h>
#include <__rcu/rcu_domain.h>
+#include <atomic>
+
+#include "thread_local_container.h"
#if !defined(_LIBCPP_HAS_NO_PRAGMA_SYSTEM_HEADER)
# pragma GCC system_header
@@ -22,6 +25,8 @@ _LIBCPP_BEGIN_NAMESPACE_STD
#if _LIBCPP_STD_VER >= 26 && _LIBCPP_HAS_THREADS && _LIBCPP_HAS_EXPERIMENTAL_RCU
+class rcu_thread_local_list_view;
+
class rcu_singly_list_view {
private:
__rcu_node* __head_ = nullptr;
@@ -43,16 +48,8 @@ class rcu_singly_list_view {
__other.__tail_ = nullptr;
}
- void __push_back(__rcu_node* __node) noexcept {
- // assert(__node->__next_ == nullptr);
- if (__head_ == nullptr) {
- __head_ = __node;
- __tail_ = __node;
- } else {
- __tail_->__next_ = __node;
- __tail_ = __node;
- }
- }
+ void __splice_back(rcu_thread_local_list_view& __other) noexcept;
+
template <class _Func>
void __for_each(_Func&& __f) noexcept {
@@ -66,6 +63,54 @@ class rcu_singly_list_view {
}
};
+class rcu_thread_local_list_view {
+ struct alignas(2*sizeof(void*)) thread_entry {
+ __rcu_node* __head_ = nullptr;
+ __rcu_node* __tail_ = nullptr;
+ };
+
+ using per_thread_entries = thread_local_container<thread_entry>;
+
+ friend class rcu_singly_list_view;
+
+public:
+ void __push_front(__rcu_node* __node) noexcept {
+ atomic_ref<thread_entry> entry_ref = per_thread_entries::get_current_thread_instance();
+ auto expected_entry = entry_ref.load(std::memory_order_relaxed);
+ auto original_next = __node->__next_;
+ while (true) {
+ auto new_entry = [&] {
+ if (expected_entry.__head_ == nullptr) {
+ return thread_entry{__node, __node};
+ } else {
+ __node->__next_ = expected_entry.__head_;
+ return thread_entry{__node, expected_entry.__tail_};
+ }
+ }();
+ if (entry_ref.compare_exchange_weak(
+ expected_entry, new_entry, std::memory_order_acq_rel, std::memory_order_relaxed)) {
+ break;
+ } else {
+ __node->__next_ = original_next;
+ }
+ }
+ }
+};
+
+void rcu_singly_list_view::__splice_back(rcu_thread_local_list_view& __other) noexcept {
+ using thread_entry = rcu_thread_local_list_view::thread_entry;
+ const auto splice_single_entry = [this](atomic_ref<thread_entry> entry_ref) noexcept {
+ if (entry_ref.load(std::memory_order_relaxed).__head_ == nullptr) {
+ return;
+ }
+ auto entry = entry_ref.exchange(thread_entry{nullptr, nullptr}, std::memory_order_acq_rel);
+ rcu_singly_list_view tmp;
+ tmp.__head_ = entry.__head_;
+ tmp.__tail_ = entry.__tail_;
+ this->__splice_back(tmp);
+ };
+ rcu_thread_local_list_view::per_thread_entries::for_each(splice_single_entry);
+}
#endif // _LIBCPP_STD_VER >= 26 && _LIBCPP_HAS_THREADS && _LIBCPP_HAS_EXPERIMENTAL_RCU
diff --git a/libcxx/src/experimental/include/rcu/thread_local_container.h b/libcxx/src/experimental/include/rcu/thread_local_container.h
new file mode 100644
index 0000000000000..5b11b6dcab504
--- /dev/null
+++ b/libcxx/src/experimental/include/rcu/thread_local_container.h
@@ -0,0 +1,85 @@
+// -*- C++ -*-
+//===----------------------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef _LIBCPP___RCU_THREAD_LOCAL_CONTAINER_H
+#define _LIBCPP___RCU_THREAD_LOCAL_CONTAINER_H
+
+#include <__config>
+#include <__functional/function.h>
+#include <__rcu/rcu_domain.h>
+
+#include <atomic>
+#include <optional>
+#include <mutex>
+#include <vector>
+
+#if !defined(_LIBCPP_HAS_NO_PRAGMA_SYSTEM_HEADER)
+# pragma GCC system_header
+#endif
+
+_LIBCPP_BEGIN_NAMESPACE_STD
+
+#if _LIBCPP_STD_VER >= 26 && _LIBCPP_HAS_THREADS && _LIBCPP_HAS_EXPERIMENTAL_RCU
+
+template <class Tp>
+class thread_local_container {
+ struct thread_entry {
+ Tp instance_;
+
+ thread_entry() : instance_() { register_instance(instance_); }
+ thread_entry(const thread_entry&) = delete;
+ thread_entry(thread_entry&&) = delete;
+
+ ~thread_entry() { deregister_instance(instance_); }
+ };
+
+ inline static thread_local optional<thread_entry> thread_entry_{};
+
+ // Keep track of all thread-local instances
+ // Only emplaced the first time a thread is trying to access its thread-local instance.
+ inline static vector<Tp*> instances_;
+ inline static mutex mtx_;
+
+ static void register_instance(Tp& obj) {
+ lock_guard<std::mutex> lg(mtx_);
+ instances_.emplace_back(&obj);
+ }
+
+ static void deregister_instance(Tp& obj) {
+ lock_guard<std::mutex> lg(mtx_);
+ instances_.erase(remove_if(instances_.begin(), instances_.end(), [&obj](Tp* instance) { return instance == &obj; }),
+ instances_.end());
+ }
+
+public:
+ thread_local_container() = delete;
+ thread_local_container(thread_local_container&&) = delete;
+
+ static atomic_ref<Tp> get_current_thread_instance() {
+ if (!thread_entry_.has_value()) {
+ auto& entry = thread_entry_.emplace();
+ return atomic_ref(entry.instance_);
+ }
+ return atomic_ref(thread_entry_->instance_);
+ }
+
+ template <class Func>
+ static void for_each(Func&& f) {
+ unique_lock<std::mutex> lock(mtx_);
+ for (auto instance : instances_) {
+ f(atomic_ref(*instance));
+ }
+ }
+};
+
+#endif // _LIBCPP_STD_VER >= 26 && _LIBCPP_HAS_THREADS && _LIBCPP_HAS_EXPERIMENTAL_RCU
+
+_LIBCPP_END_NAMESPACE_STD
+
+#endif // _LIBCPP___RCU_THREAD_LOCAL_CONTAINER_H
diff --git a/libcxx/src/experimental/rcu.cpp b/libcxx/src/experimental/rcu.cpp
index dde4693dc1138..a585d38a8ef5e 100644
--- a/libcxx/src/experimental/rcu.cpp
+++ b/libcxx/src/experimental/rcu.cpp
@@ -9,67 +9,16 @@
#include <atomic>
#include <memory>
#include <mutex>
-#include <optional>
#include <rcu>
-#include <vector>
#include "include/rcu/rcu_list.h"
+#include "include/rcu/thread_local_container.h"
_LIBCPP_BEGIN_NAMESPACE_STD
_LIBCPP_BEGIN_EXPLICIT_ABI_ANNOTATIONS
namespace {
-template <class Tp>
-class thread_local_container {
- struct thread_entry {
- Tp instance_;
-
- thread_entry() : instance_() { register_instance(instance_); }
- thread_entry(const thread_entry&) = delete;
- thread_entry(thread_entry&&) = delete;
-
- ~thread_entry() { deregister_instance(instance_); }
- };
-
- inline static thread_local optional<thread_entry> thread_entry_{};
-
- // Keep track of all thread-local instances
- // Only emplaced the first time a thread is trying to access its thread-local instance.
- inline static vector<Tp*> instances_;
- inline static mutex mtx_;
-
- static void register_instance(Tp& obj) {
- lock_guard<std::mutex> lg(mtx_);
- instances_.emplace_back(&obj);
- }
-
- static void deregister_instance(Tp& obj) {
- lock_guard<std::mutex> lg(mtx_);
- instances_.erase(remove_if(instances_.begin(), instances_.end(), [&obj](Tp* instance) { return instance == &obj; }),
- instances_.end());
- }
-
-public:
- thread_local_container() = delete;
- thread_local_container(thread_local_container&&) = delete;
-
- static atomic_ref<Tp> get_current_thread_instance() {
- if (!thread_entry_.has_value()) {
- auto& entry = thread_entry_.emplace();
- return atomic_ref(entry.instance_);
- }
- return atomic_ref(thread_entry_->instance_);
- }
-
- template <class Func>
- static void for_each(Func&& f) {
- unique_lock<std::mutex> lock(mtx_);
- for (auto instance : instances_) {
- f(atomic_ref(*instance));
- }
- }
-};
// Adopted the 2-phase implementation in the section
// "3) General-Purpose RCU" of the paper
@@ -115,8 +64,7 @@ class rcu_domain_impl {
std::atomic<bool> grace_period_waiting_flag_ = false;
// todo: maybe use a lock-free queue
- std::mutex retire_queue_mutex_; // todo this is not noexcept
- rcu_singly_list_view __retired_callback_queue_;
+ rcu_thread_local_list_view __retired_callback_queue_;
// these two queues do not need extra synchronization
// as they are always processed under the grace period mutex
@@ -126,9 +74,7 @@ class rcu_domain_impl {
friend class rcu_domain;
rcu_singly_list_view update_phase_and_wait() noexcept {
- std::unique_lock retire_lk(retire_queue_mutex_);
callbacks_phase_1_.__splice_back(__retired_callback_queue_);
- retire_lk.unlock();
// Flip the global phase
auto old_phase = global_reader_phase_.fetch_xor(reader_states::grace_period_phase_mask, std::memory_order_relaxed);
@@ -200,8 +146,7 @@ class rcu_domain_impl {
}
void retire(__rcu_node* node) noexcept {
- lock_guard<std::mutex> lk(retire_queue_mutex_);
- __retired_callback_queue_.__push_back(node);
+ __retired_callback_queue_.__push_front(node);
}
void synchronize() noexcept {
diff --git a/libcxx/test/std/thread/saferecl/saferecl.rcu/read_write_lock_compare_multi_writer.pass.cpp b/libcxx/test/std/thread/saferecl/saferecl.rcu/read_write_lock_compare_multi_writer.pass.cpp
new file mode 100644
index 0000000000000..2fc79013994e1
--- /dev/null
+++ b/libcxx/test/std/thread/saferecl/saferecl.rcu/read_write_lock_compare_multi_writer.pass.cpp
@@ -0,0 +1,154 @@
+//===----------------------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+//
+// UNSUPPORTED: no-threads
+// UNSUPPORTED: c++03, c++11, c++14, c++17, c++20, c++23
+
+// <rcu>
+
+#include <atomic>
+#include <rcu>
+#include <shared_mutex>
+#include <stop_token>
+#include <thread>
+#include <chrono>
+#include <iostream>
+#include <print>
+#include <string>
+
+#include "__rcu/rcu_domain.h"
+#include "make_test_thread.h"
+#include "test_macros.h"
+
+constexpr int num_reader = 4;
+const std::chrono::seconds test_time(10);
+
+struct alignas(128) MyObject : public std::rcu_obj_base<MyObject> {
+ std::string data_;
+
+ inline static int instance_count = 0;
+ MyObject() : data_(std::to_string(instance_count++) + " instance very very very long string") {}
+
+ void doWork() {
+ auto spin_for = [](std::chrono::microseconds us) {
+ auto start = std::chrono::high_resolution_clock::now();
+ while (std::chrono::high_resolution_clock::now() - start < us)
+ ;
+ };
+ using namespace std::chrono_literals;
+ spin_for(10us);
+ }
+};
+
+
+void test_read_write_lock() {
+ MyObject* globalObjRWLock = new MyObject();
+ std::shared_mutex globalObjMutex;
+
+ std::vector<std::jthread> readers;
+ readers.reserve(num_reader);
+
+ auto reader_func = [&globalObjRWLock, &globalObjMutex](std::stop_token token) {
+ int read_count = 0;
+ while (!token.stop_requested()) {
+ std::shared_lock<std::shared_mutex> lock(globalObjMutex);
+ globalObjRWLock->doWork();
+ ++read_count;
+ }
+ std::println("Reader thread read {} times", read_count);
+ };
+
+ auto writer_func = [&globalObjRWLock, &globalObjMutex](std::stop_token token) {
+ int write_count = 0;
+ while (!token.stop_requested()) {
+ auto newObj = new MyObject();
+ std::unique_lock<std::shared_mutex> lock(globalObjMutex);
+ auto oldObj = globalObjRWLock;
+ globalObjRWLock = newObj;
+ lock.unlock();
+ delete oldObj;
+ ++write_count;
+ std::this_thread::sleep_for(std::chrono::microseconds(100));
+ }
+ std::println("Writer thread wrote {} times", write_count);
+ };
+
+ for (int i = 0; i < num_reader; ++i) {
+ readers.emplace_back(reader_func);
+ }
+ std::jthread writer(writer_func);
+
+ std::this_thread::sleep_for(test_time);
+
+ for (auto& reader : readers) {
+ reader.request_stop();
+ }
+ writer.request_stop();
+}
+
+void test_rcu() {
+ std::rcu_domain& dom = std::rcu_default_domain();
+ std::atomic<MyObject*> global_obj_rcu = new MyObject();
+
+ std::vector<std::jthread> readers;
+ readers.reserve(num_reader);
+
+ auto reader_func = [&dom, &global_obj_rcu](std::stop_token token) {
+ int read_count = 0;
+ while (!token.stop_requested()) {
+ dom.lock();
+ auto obj = global_obj_rcu.load(std::memory_order_relaxed);
+ obj->doWork();
+ dom.unlock();
+ ++read_count;
+ }
+ std::println("RCU Reader thread read {} times", read_count);
+ };
+
+ auto writer_func = [&global_obj_rcu](std::stop_token token) {
+ int write_count = 0;
+ while (!token.stop_requested()) {
+ auto newObj = new MyObject();
+ auto oldObj = global_obj_rcu.exchange(newObj, std::memory_order_relaxed);
+ oldObj->retire();
+ ++write_count;
+ std::this_thread::sleep_for(std::chrono::microseconds(100));
+ }
+ std::println("RCU Writer thread wrote {} times", write_count);
+ };
+
+ auto syncer_func = [&dom](std::stop_token token) {
+ while (!token.stop_requested()) {
+ std::rcu_synchronize(dom);
+ std::this_thread::sleep_for(std::chrono::milliseconds(50));
+ }
+ };
+
+ for (int i = 0; i < num_reader; ++i) {
+ readers.emplace_back(reader_func);
+ }
+ std::jthread writer(writer_func);
+ std::jthread syncer(syncer_func);
+
+ std::this_thread::sleep_for(test_time);
+
+ for (auto& reader : readers) {
+ reader.request_stop();
+ }
+ writer.request_stop();
+ syncer.request_stop();
+ std::rcu_synchronize(dom);
+}
+
+int main(int, char**) {
+ std::println("Testing read-write lock:");
+ test_read_write_lock();
+ std::println("Testing RCU:");
+ test_rcu();
+ return 1;
+}
>From f2b33f34cb048c1e8dc5aef0f0b04a9fe5987a7a Mon Sep 17 00:00:00 2001
From: Hui Xie <hui.xie1990 at gmail.com>
Date: Sat, 5 Sep 2026 16:52:05 +0100
Subject: [PATCH 33/65] review
---
libcxx/docs/DesignDocs/RCU.rst | 42 +++++++++++++++----
libcxx/src/experimental/rcu.cpp | 15 ++++---
...d_write_lock_compare_multi_writer.pass.cpp | 25 +++++++++--
3 files changed, 61 insertions(+), 21 deletions(-)
diff --git a/libcxx/docs/DesignDocs/RCU.rst b/libcxx/docs/DesignDocs/RCU.rst
index 68aadd0f4c696..83e93a4f6579b 100644
--- a/libcxx/docs/DesignDocs/RCU.rst
+++ b/libcxx/docs/DesignDocs/RCU.rst
@@ -162,8 +162,14 @@ the `atomic` integer API, e.g. `fetch_or`. An `atomic` of a wrapper struct won't
`rcu_singly_list_view`
~~~~~~~~~~~~~~~~~~~~~~
-A helper class that provides a view of the intrusive singly linked list of `__rcu_node` s. It also stores the back pointer of the list to make the push back operation more efficient.
-It provides APIs to push a node to the back of the list, splice from another list and to iterate through the list.
+A helper class that provides a view of the intrusive singly linked list of `__rcu_node` s. It also stores the back pointer of the list to make the splice back operation more efficient.
+It provides APIs to splice from another `rcu_singly_list_view` or `rcu_thread_local_list_view`, and to iterate through the list.
+
+`rcu_thread_local_list_view`
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+A thread safe helper class that provides a view of the intrusive singly linked list of `__rcu_node` s. It provides APIs to push a node into the list, or to be spliced to another `rcu_singly_list_view`.
+It uses `thread_local` storage to avoid contentions from multiple threads pushing nodes into the list.
For the derived class of `rcu_obj_base`, the `retire` is `noexcept`, which means that no memory allocation can happen when pushing it to the retired callback queue.
@@ -179,13 +185,9 @@ This is the main class that implements the `rcu` logic. It contains
- `std::atomic<bool> grace_period_waiting_flag_` : This flag is used to sleep/wake up the collector thread that is waiting for the grace period to end.
-- `std::mutex retire_queue_mutex_` and `rcu_singly_list_view __retired_callback_queue_` : This queue stores all the retired callbacks that are waiting for the grace period to end.
-
- - TODO: `mutex` can throw, we need to consider how to replace it.
+- `rcu_thread_local_list_view retired_queue_stage0_;` : All the retired objects are directly pushed to this queue first.
- - Design 2: The access pattern of this list in this implementation is very special. The writer threads are only going to push to this queue. And the collector threads are only going to splice the entire queue. It is easy to implement it with a single `atomic<pair<Head*, Tail*>>`. It will be efficient if the platform supports 16 bytes CAS. To reduce contention between multiple writer threads retiring objects, we let each thread have a thread local queue
-
-- `rcu_singly_list_view callbacks_phase_1_` and `rcu_singly_list_view callbacks_phase_2_` : These two queues are used to let the retired callbacks go through two grace periods before being invoked. No additional synchronization is needed for these two queues as they are only processed when the collector (?) thread is holding the `grace_period_mutex_` .
+- `rcu_singly_list_view retired_queue_stage1_` and `rcu_singly_list_view retired_queue_stage2_` : These two queues are used to let the retired callbacks go through two grace periods before being invoked. No additional synchronization is needed for these two queues as they are only processed when the collector thread is holding the `grace_period_mutex_` .
The domain has few operations:
@@ -204,7 +206,7 @@ The domain has few operations:
`retire`
^^^^^^^^
-- We need to push the retired callback to the `__retired_callback_queue_`
+- We need to push the retired callback to the `retired_queue_stage0_`
`synchronize`
^^^^^^^^^^^^^
@@ -217,6 +219,25 @@ The domain has few operations:
- After each phase ends, we can move the callbacks to the next stage's queue, and we can evaluate the callbacks that have gone through two stages.
+- Detailed Algorithms of going through one grace period:
+
+ 1. lock `grace_period_mutex_`
+ 2. Promote stage 0 to stage 1
+ 3. Flip the global phase
+ 4. Wait for the current grace period to end
+ 5. Promote stage 2 to the local ready list
+ 6. Promote stage 1 to stage 2
+ 7. unlock `grace_period_mutex_`
+ 8. Drain the local ready list
+
+- Post condition of each cycle of this algorithm:
+
+ 1. Nodes that were previously inside stage 2 queue will be drained
+ 2. Nodes that were previously inside stage 0 will end up in stage 2
+ 3. stage 1 queue was empty before the cycle and empty after the cycle (maybe we should make it a local variable instead?)
+
+- Note: in this design, both `rcu_retire` and `rcu_synchronize` calls this `synchronize` operation.
+
Design Questions
================
@@ -255,6 +276,9 @@ And Thomas Rodgers (libstdc++ contributor) also said:
In libc++'s design, we would like to follow Folly's approach to run these deleters inline when `rcu_synchronize` or
`rcu_barrier` is called.
+If we create a libc++ owned collector thread that performs stage migration and potentially stage 2 collection, there is a challenge that creating that thread is something each platform would have to teach us how to do
+(e.g. embedded threads can't actually create a thread with the `std::thread` constructor). There's also no precedent for libc++ to create a thread on the user's behalf.
+
When should we run the deleters
-------------------------------
diff --git a/libcxx/src/experimental/rcu.cpp b/libcxx/src/experimental/rcu.cpp
index a585d38a8ef5e..2f4552b2e9cee 100644
--- a/libcxx/src/experimental/rcu.cpp
+++ b/libcxx/src/experimental/rcu.cpp
@@ -63,18 +63,17 @@ class rcu_domain_impl {
// flag used for waking up writer threads waiting for all reader threads' quiescent state
std::atomic<bool> grace_period_waiting_flag_ = false;
- // todo: maybe use a lock-free queue
- rcu_thread_local_list_view __retired_callback_queue_;
+ rcu_thread_local_list_view retired_queue_stage0_;
// these two queues do not need extra synchronization
// as they are always processed under the grace period mutex
- rcu_singly_list_view callbacks_phase_1_;
- rcu_singly_list_view callbacks_phase_2_;
+ rcu_singly_list_view retired_queue_stage1_;
+ rcu_singly_list_view retired_queue_stage2_;
friend class rcu_domain;
rcu_singly_list_view update_phase_and_wait() noexcept {
- callbacks_phase_1_.__splice_back(__retired_callback_queue_);
+ retired_queue_stage1_.__splice_back(retired_queue_stage0_);
// Flip the global phase
auto old_phase = global_reader_phase_.fetch_xor(reader_states::grace_period_phase_mask, std::memory_order_relaxed);
@@ -90,8 +89,8 @@ class rcu_domain_impl {
grace_period_waiting_flag_.store(false, std::memory_order_relaxed);
rcu_singly_list_view ready_callbacks;
- ready_callbacks.__splice_back(callbacks_phase_2_);
- callbacks_phase_2_.__splice_back(callbacks_phase_1_);
+ ready_callbacks.__splice_back(retired_queue_stage2_);
+ retired_queue_stage2_.__splice_back(retired_queue_stage1_);
return ready_callbacks;
}
@@ -146,7 +145,7 @@ class rcu_domain_impl {
}
void retire(__rcu_node* node) noexcept {
- __retired_callback_queue_.__push_front(node);
+ retired_queue_stage0_.__push_front(node);
}
void synchronize() noexcept {
diff --git a/libcxx/test/std/thread/saferecl/saferecl.rcu/read_write_lock_compare_multi_writer.pass.cpp b/libcxx/test/std/thread/saferecl/saferecl.rcu/read_write_lock_compare_multi_writer.pass.cpp
index 2fc79013994e1..d345997d89444 100644
--- a/libcxx/test/std/thread/saferecl/saferecl.rcu/read_write_lock_compare_multi_writer.pass.cpp
+++ b/libcxx/test/std/thread/saferecl/saferecl.rcu/read_write_lock_compare_multi_writer.pass.cpp
@@ -26,6 +26,7 @@
#include "test_macros.h"
constexpr int num_reader = 4;
+constexpr int num_writer = 4;
const std::chrono::seconds test_time(10);
struct alignas(128) MyObject : public std::rcu_obj_base<MyObject> {
@@ -53,6 +54,9 @@ void test_read_write_lock() {
std::vector<std::jthread> readers;
readers.reserve(num_reader);
+ std::vector<std::jthread> writers;
+ writers.reserve(num_writer);
+
auto reader_func = [&globalObjRWLock, &globalObjMutex](std::stop_token token) {
int read_count = 0;
while (!token.stop_requested()) {
@@ -81,14 +85,19 @@ void test_read_write_lock() {
for (int i = 0; i < num_reader; ++i) {
readers.emplace_back(reader_func);
}
- std::jthread writer(writer_func);
+ for (int i = 0; i < num_writer; ++i) {
+ writers.emplace_back(writer_func);
+ }
std::this_thread::sleep_for(test_time);
for (auto& reader : readers) {
reader.request_stop();
}
- writer.request_stop();
+
+ for (auto& writer : writers) {
+ writer.request_stop();
+ }
}
void test_rcu() {
@@ -98,6 +107,9 @@ void test_rcu() {
std::vector<std::jthread> readers;
readers.reserve(num_reader);
+ std::vector<std::jthread> writers;
+ writers.reserve(num_writer);
+
auto reader_func = [&dom, &global_obj_rcu](std::stop_token token) {
int read_count = 0;
while (!token.stop_requested()) {
@@ -132,7 +144,10 @@ void test_rcu() {
for (int i = 0; i < num_reader; ++i) {
readers.emplace_back(reader_func);
}
- std::jthread writer(writer_func);
+
+ for (int i = 0; i < num_writer; ++i) {
+ writers.emplace_back(writer_func);
+ }
std::jthread syncer(syncer_func);
std::this_thread::sleep_for(test_time);
@@ -140,7 +155,9 @@ void test_rcu() {
for (auto& reader : readers) {
reader.request_stop();
}
- writer.request_stop();
+ for (auto& writer : writers) {
+ writer.request_stop();
+ }
syncer.request_stop();
std::rcu_synchronize(dom);
}
>From 37c3891b2a0dabfb9690170ff92674b2a671d8ce Mon Sep 17 00:00:00 2001
From: Hui Xie <hui.xie1990 at gmail.com>
Date: Sun, 6 Sep 2026 09:55:52 +0100
Subject: [PATCH 34/65] remove std::function, replace it with function_ref
---
libcxx/include/__rcu/rcu_domain.h | 27 +++++++++++++------
libcxx/include/__rcu/rcu_obj_base.h | 16 ++++++-----
libcxx/src/experimental/rcu.cpp | 4 +++
.../read_write_lock_compare.pass.cpp | 26 ++++++++++++------
4 files changed, 50 insertions(+), 23 deletions(-)
diff --git a/libcxx/include/__rcu/rcu_domain.h b/libcxx/include/__rcu/rcu_domain.h
index 5c8a4667abca8..7788412daf9f2 100644
--- a/libcxx/include/__rcu/rcu_domain.h
+++ b/libcxx/include/__rcu/rcu_domain.h
@@ -11,7 +11,7 @@
#define _LIBCPP___RCU_RCU_DOMAIN_H
#include <__config>
-#include <__functional/function.h>
+#include <__functional/function_ref.h>
#include <__memory/unique_ptr.h>
#include <__utility/move.h>
@@ -25,13 +25,28 @@ _LIBCPP_BEGIN_EXPLICIT_ABI_ANNOTATIONS
#if _LIBCPP_STD_VER >= 26 && _LIBCPP_HAS_THREADS && _LIBCPP_HAS_EXPERIMENTAL_RCU
struct __rcu_node {
- function<void()> __callback_{};
- __rcu_node* __next_ = nullptr;
+ function_ref<void()> __callback_ = std::cw<[] {}>;
+ __rcu_node* __next_ = nullptr;
_LIBCPP_HIDE_FROM_ABI __rcu_node() {}
_LIBCPP_HIDE_FROM_ABI ~__rcu_node() {}
};
+template <class _Tp, class _Deleter>
+struct __rcu_node_with_deleter : __rcu_node {
+ _Tp* __obj_;
+ _LIBCPP_NO_UNIQUE_ADDRESS _Deleter __deleter_ = _Deleter();
+
+ _LIBCPP_HIDE_FROM_ABI __rcu_node_with_deleter(_Tp* __obj, _Deleter __deleter) : __obj_(__obj), __deleter_(__deleter) {
+ __callback_ = function_ref<void()>(std::cw<&__rcu_node_with_deleter::__destroy>, this);
+ }
+
+ _LIBCPP_HIDE_FROM_ABI void __destroy() const {
+ __deleter_(__obj_);
+ delete this;
+ }
+};
+
class _LIBCPP_EXPORTED_FROM_ABI rcu_domain {
class __impl;
unique_ptr<__impl> __pimpl_;
@@ -46,11 +61,7 @@ class _LIBCPP_EXPORTED_FROM_ABI rcu_domain {
template <class _Tp, class _Dp = default_delete<_Tp>>
_LIBCPP_HIDE_FROM_ABI void __rcu_retire_hidden_friend(_Tp* __tp, _Dp __deleter, rcu_domain& __dom) {
- auto* __node = new __rcu_node();
- __node->__callback_ = [__tp, __node, __deleter = std::move(__deleter)]() mutable {
- __deleter(__tp);
- delete __node;
- };
+ auto* __node = new __rcu_node_with_deleter<_Tp, _Dp>(__tp, std::move(__deleter));
__dom.__retire(__node);
}
diff --git a/libcxx/include/__rcu/rcu_obj_base.h b/libcxx/include/__rcu/rcu_obj_base.h
index a8d9383ac06e5..ccf653b22e6f6 100644
--- a/libcxx/include/__rcu/rcu_obj_base.h
+++ b/libcxx/include/__rcu/rcu_obj_base.h
@@ -10,8 +10,9 @@
#ifndef _LIBCPP___RCU_RCU_OBJ_BASE_H
#define _LIBCPP___RCU_RCU_OBJ_BASE_H
+#include "__configuration/attributes.h"
#include <__config>
-#include <__functional/function.h>
+#include <__functional/function_ref.h>
#include <__memory/unique_ptr.h> // for default_delete
#include <__rcu/rcu_domain.h>
#include <__utility/move.h>
@@ -28,12 +29,8 @@ template <class _Tp, class _Dp = default_delete<_Tp>>
class rcu_obj_base : private __rcu_node {
public:
_LIBCPP_HIDE_FROM_ABI void retire(_Dp __deleter = _Dp(), rcu_domain& __dom = rcu_default_domain()) noexcept {
- auto __ptr = static_cast<_Tp*>(this);
-
- // todo: std::function can throw on the assignment. Perhaps we can store
- // the deleter in the class and either use virtual function or use
- // function_ref here.
- __callback_ = [__ptr, __deleter = std::move(__deleter)]() mutable { __deleter(__ptr); };
+ __deleter_ = std::move(__deleter);
+ __callback_ = function_ref<void()>(std::cw<&rcu_obj_base::__destroy>, this);
__dom.__retire(this);
}
@@ -44,6 +41,11 @@ class rcu_obj_base : private __rcu_node {
_LIBCPP_HIDE_FROM_ABI rcu_obj_base& operator=(const rcu_obj_base&) = default;
_LIBCPP_HIDE_FROM_ABI rcu_obj_base& operator=(rcu_obj_base&&) = default;
_LIBCPP_HIDE_FROM_ABI ~rcu_obj_base() = default;
+
+private:
+ _LIBCPP_HIDE_FROM_ABI void __destroy() { __deleter_(static_cast<_Tp*>(this)); }
+
+ _LIBCPP_NO_UNIQUE_ADDRESS _Dp __deleter_ = _Dp();
};
#endif // _LIBCPP_STD_VER >= 26 && _LIBCPP_HAS_THREADS && _LIBCPP_HAS_EXPERIMENTAL_RCU
diff --git a/libcxx/src/experimental/rcu.cpp b/libcxx/src/experimental/rcu.cpp
index 2f4552b2e9cee..4532780cced1b 100644
--- a/libcxx/src/experimental/rcu.cpp
+++ b/libcxx/src/experimental/rcu.cpp
@@ -14,6 +14,10 @@
#include "include/rcu/rcu_list.h"
#include "include/rcu/thread_local_container.h"
+// todo: remove debug print
+#include <cstdio>
+//
+
_LIBCPP_BEGIN_NAMESPACE_STD
_LIBCPP_BEGIN_EXPLICIT_ABI_ANNOTATIONS
diff --git a/libcxx/test/std/thread/saferecl/saferecl.rcu/read_write_lock_compare.pass.cpp b/libcxx/test/std/thread/saferecl/saferecl.rcu/read_write_lock_compare.pass.cpp
index 2fc79013994e1..8d637b7c05bfd 100644
--- a/libcxx/test/std/thread/saferecl/saferecl.rcu/read_write_lock_compare.pass.cpp
+++ b/libcxx/test/std/thread/saferecl/saferecl.rcu/read_write_lock_compare.pass.cpp
@@ -12,6 +12,7 @@
// <rcu>
#include <atomic>
+#include <cstddef>
#include <rcu>
#include <shared_mutex>
#include <stop_token>
@@ -28,11 +29,14 @@
constexpr int num_reader = 4;
const std::chrono::seconds test_time(10);
+
struct alignas(128) MyObject : public std::rcu_obj_base<MyObject> {
std::string data_;
+ std::atomic<size_t>& destruction_count_;
inline static int instance_count = 0;
- MyObject() : data_(std::to_string(instance_count++) + " instance very very very long string") {}
+ MyObject(std::atomic<size_t>& count) : data_(std::to_string(instance_count++) + " instance very very very long string"), destruction_count_(count) {}
+ ~MyObject() {destruction_count_.fetch_add(1, std::memory_order_relaxed);}
void doWork() {
auto spin_for = [](std::chrono::microseconds us) {
@@ -47,12 +51,14 @@ struct alignas(128) MyObject : public std::rcu_obj_base<MyObject> {
void test_read_write_lock() {
- MyObject* globalObjRWLock = new MyObject();
+ std::atomic<size_t> destruction_count = 0;
+ MyObject* globalObjRWLock = new MyObject(destruction_count);
std::shared_mutex globalObjMutex;
std::vector<std::jthread> readers;
readers.reserve(num_reader);
+
auto reader_func = [&globalObjRWLock, &globalObjMutex](std::stop_token token) {
int read_count = 0;
while (!token.stop_requested()) {
@@ -63,10 +69,10 @@ void test_read_write_lock() {
std::println("Reader thread read {} times", read_count);
};
- auto writer_func = [&globalObjRWLock, &globalObjMutex](std::stop_token token) {
+ auto writer_func = [&globalObjRWLock, &globalObjMutex, &destruction_count](std::stop_token token) {
int write_count = 0;
while (!token.stop_requested()) {
- auto newObj = new MyObject();
+ auto newObj = new MyObject(destruction_count);
std::unique_lock<std::shared_mutex> lock(globalObjMutex);
auto oldObj = globalObjRWLock;
globalObjRWLock = newObj;
@@ -89,15 +95,18 @@ void test_read_write_lock() {
reader.request_stop();
}
writer.request_stop();
+ std::println("Writer thread destruction {} times", destruction_count.load());
}
void test_rcu() {
+ std::atomic<size_t> destruction_count = 0;
std::rcu_domain& dom = std::rcu_default_domain();
- std::atomic<MyObject*> global_obj_rcu = new MyObject();
+ std::atomic<MyObject*> global_obj_rcu = new MyObject(destruction_count);
std::vector<std::jthread> readers;
readers.reserve(num_reader);
+
auto reader_func = [&dom, &global_obj_rcu](std::stop_token token) {
int read_count = 0;
while (!token.stop_requested()) {
@@ -110,10 +119,10 @@ void test_rcu() {
std::println("RCU Reader thread read {} times", read_count);
};
- auto writer_func = [&global_obj_rcu](std::stop_token token) {
+ auto writer_func = [&global_obj_rcu, &destruction_count](std::stop_token token) {
int write_count = 0;
while (!token.stop_requested()) {
- auto newObj = new MyObject();
+ auto newObj = new MyObject(destruction_count);
auto oldObj = global_obj_rcu.exchange(newObj, std::memory_order_relaxed);
oldObj->retire();
++write_count;
@@ -124,7 +133,7 @@ void test_rcu() {
auto syncer_func = [&dom](std::stop_token token) {
while (!token.stop_requested()) {
- std::rcu_synchronize(dom);
+ std::rcu_synchronize(dom);
std::this_thread::sleep_for(std::chrono::milliseconds(50));
}
};
@@ -143,6 +152,7 @@ void test_rcu() {
writer.request_stop();
syncer.request_stop();
std::rcu_synchronize(dom);
+ std::println("RCU Writer thread destruction {} times", destruction_count.load());
}
int main(int, char**) {
>From a7a3c3aa12a247a64f25c690ff69df72a09077d4 Mon Sep 17 00:00:00 2001
From: Hui Xie <hui.xie1990 at gmail.com>
Date: Sun, 6 Sep 2026 12:59:16 +0100
Subject: [PATCH 35/65] sync no drain
---
libcxx/docs/DesignDocs/RCU.rst | 47 ++++++++++---------
libcxx/include/__rcu/rcu_domain.h | 1 +
libcxx/src/experimental/rcu.cpp | 42 ++++++++++-------
.../saferecl/saferecl.rcu/general.pass.cpp | 1 -
.../read_write_lock_compare.pass.cpp | 7 ++-
...d_write_lock_compare_multi_writer.pass.cpp | 3 +-
6 files changed, 57 insertions(+), 44 deletions(-)
diff --git a/libcxx/docs/DesignDocs/RCU.rst b/libcxx/docs/DesignDocs/RCU.rst
index 83e93a4f6579b..f24b5c7659235 100644
--- a/libcxx/docs/DesignDocs/RCU.rst
+++ b/libcxx/docs/DesignDocs/RCU.rst
@@ -99,7 +99,7 @@ The core idea of `rcu` can be described by this image from lwn.net
- Each row is a thread. The last row is the collector thread and the rows above are the reader threads.
- Each "Reader" block represents a critical section, which starts with `rcu_domain::lock` and ends with `rcu_domain::unlock` .
- When `rcu_retire` is called from the writer thread, it starts the "Removal" block.
-- When `rcu_synchronize` is called from the collector thread, it starts the "Grace Period" block. We need to wait until all the
+- When `rcu_synchronize`/`rcu_barrier` is called from the collector thread, it starts the "Grace Period" block. We need to wait until all the
"Reader" blocks that started before the "Grace Period" started, to exit via `rcu_domain::unlock`, then we can end the "Grace Period".
Note that the "Grace Period" ends after the 4th row's "Reader" block ends. Also note that the "Grace Period" does not need to wait
for the late "Reader" blocks.
@@ -111,14 +111,14 @@ Some key details of this design are:
- There is a global state which has two phases and it flips between the two phases.
- Each thread stores its state: whether there is a reader in the critical section, and which phase it was when it entered the critical section.
-- When `rcu_synchronize` is called, it
+- When `rcu_synchronize`/`rcu_barrier` is called, it
- flips the global state to the next phase
- Going through a grace period: waits until all the threads that are in the critical section with the previous phase to exit the critical section.
- flips the global state back to the original phase
- Going through another grace period: waits until all the threads that are in the critical section with the next phase to exit the critical section.
-When `rcu_synchronize` returns, we can be sure that all the readers that were in the critical section before `rcu_synchronize` are now out of the critical section.
+When `rcu_synchronize`/`rcu_barrier` returns, we can be sure that all the readers that were in the critical section before `rcu_synchronize`/`rcu_barrier` are now out of the critical section.
The paper explains why we need to wait two phases instead of just one phase in detail. The key point is that, if we only wait for the readers in the previous phase to exit,
there might be a late reader that enters the critical section after we flip the global state and before we wait for the previous phase's readers to exit.
@@ -187,7 +187,8 @@ This is the main class that implements the `rcu` logic. It contains
- `rcu_thread_local_list_view retired_queue_stage0_;` : All the retired objects are directly pushed to this queue first.
-- `rcu_singly_list_view retired_queue_stage1_` and `rcu_singly_list_view retired_queue_stage2_` : These two queues are used to let the retired callbacks go through two grace periods before being invoked. No additional synchronization is needed for these two queues as they are only processed when the collector thread is holding the `grace_period_mutex_` .
+- `rcu_singly_list_view retired_queue_stage1_` and `rcu_singly_list_view retired_queue_stage2_` : These two queues are used to let the retired callbacks go through two grace periods before being invoked. No additional synchronization is needed for these two queues as they are only processed when the collector thread is holding the `grace_period_mutex_`.
+ Note that `_retired_queue_stage2_` queue holds the callbacks that are ready to be called.
The domain has few operations:
@@ -222,21 +223,23 @@ The domain has few operations:
- Detailed Algorithms of going through one grace period:
1. lock `grace_period_mutex_`
- 2. Promote stage 0 to stage 1
+ 2. Move stage 0 to a local working queue
3. Flip the global phase
4. Wait for the current grace period to end
- 5. Promote stage 2 to the local ready list
- 6. Promote stage 1 to stage 2
+ 5. Promote stage 1 to stage 2
+ 6. Promote local work queue (from stage 0) to stage 1
7. unlock `grace_period_mutex_`
- 8. Drain the local ready list
- Post condition of each cycle of this algorithm:
- 1. Nodes that were previously inside stage 2 queue will be drained
- 2. Nodes that were previously inside stage 0 will end up in stage 2
- 3. stage 1 queue was empty before the cycle and empty after the cycle (maybe we should make it a local variable instead?)
+ - Nodes that were previously inside stage 0 will end up in stage 1
+ - Nodes that were previously inside stage 1 will end up in stage 2
-- Note: in this design, both `rcu_retire` and `rcu_synchronize` calls this `synchronize` operation.
+- Note: in this design, This `synchronize` operation has a `invoke_callback` parameter. `rcu_synchronize` calls it with `false`,
+ while `rcu_barrier` calls it with `true`, which means that the stage 2 queue will be drained and invoked at the end of each
+ grace period
+
+- Note: stage 0 queue is not guarded by `grace_period_mutex_`
Design Questions
@@ -273,7 +276,7 @@ And Thomas Rodgers (libstdc++ contributor) also said:
thread. But on the other hand, users of the library might find it surprising that then gave the library permission
to also do work on those threads in ways they might not expect.
-In libc++'s design, we would like to follow Folly's approach to run these deleters inline when `rcu_synchronize` or
+In libc++'s design, we would like to follow Folly's approach to run these deleters inline when
`rcu_barrier` is called.
If we create a libc++ owned collector thread that performs stage migration and potentially stage 2 collection, there is a challenge that creating that thread is something each platform would have to teach us how to do
@@ -288,23 +291,25 @@ inline, we have to decide when to run them.
There are few places we can run the deleters:
-- Inside `rcu_synchronize` after the collector thread is unblocked. This approach has the advantage that the collector thread
+- Inside `rcu_barrier` after the collector thread is unblocked. This approach has the advantage that the collector thread
can reclaim the retired objects as soon as possible. However, it has the disadvantage that the collector thread may be
- blocked for a long time if there are many retired objects to reclaim.
-
-- Inside `rcu_barrier` after the collector thread is unblocked. Since `rcu_barrier` is designed to block until all the retired
+ blocked for a long time if there are many retired objects to reclaim. Since `rcu_barrier` is designed to block until all the retired
objects that happen before the `rcu_barrier` call are reclaimed, it is natural to run the deleters here.
+- Inside `rcu_synchronize` after the collector thread is unblocked. However, the standard spec does not explicitly suggests that this function
+ maybe invoke evaluations.
+
- Inside `rcu_retire` after the deleter is put into the queue. `rcu_retire` is designed not to block. However, if there are
objects that are safe to reclaim due to readers have exited their critical sections, and at the same time, there is no other
collector/writer threads that are currently draining the deleter queue, we can take the opportunity to run the deleters.
Folly currently takes the inline approach and runs the deleters at all three places mentioned above.
-In libc++'s design, we use the inline approach and only run the deleters inside `rcu_synchronize` and
-`rcu_barrier` for simplicity. Whether or not running the deleters inside `rcu_retire` is debatable. On the one hand, running
-them will make `rcu_retire` take more time to return. But not running them will make the retired objects stay in the queue
-for a longer time. In an extreme case, if the user never calls `rcu_synchronize` or `rcu_barrier` after calling `rcu_retire`,
+In libc++'s design, we use the inline approach and only run the deleters inside
+`rcu_barrier` for simplicity. The standard spec does not suggest we should invoke deleters in `rcu_synchronize`.
+Whether or not running the deleters inside `rcu_retire` is debatable. On the one hand, running
+them will make `rcu_retire` take more time to return. But not running them will make the retired objects stay in the queue
+for a longer time. In an extreme case, if the user never calls `rcu_barrier` after calling `rcu_retire`,
the retired objects will never be reclaimed (without a background thread to drain the queue).
Almost all APIs are `noexcept` . Is it designed to avoid memory allocation and avoid using `mutex` ?
diff --git a/libcxx/include/__rcu/rcu_domain.h b/libcxx/include/__rcu/rcu_domain.h
index 7788412daf9f2..b7580ad53e221 100644
--- a/libcxx/include/__rcu/rcu_domain.h
+++ b/libcxx/include/__rcu/rcu_domain.h
@@ -56,6 +56,7 @@ class _LIBCPP_EXPORTED_FROM_ABI rcu_domain {
friend rcu_domain& rcu_default_domain() noexcept;
friend void rcu_synchronize(rcu_domain&) noexcept;
+ friend void rcu_barrier(rcu_domain&) noexcept;
static rcu_domain& __rcu_default_domain() noexcept;
diff --git a/libcxx/src/experimental/rcu.cpp b/libcxx/src/experimental/rcu.cpp
index 4532780cced1b..1810f086cc394 100644
--- a/libcxx/src/experimental/rcu.cpp
+++ b/libcxx/src/experimental/rcu.cpp
@@ -76,8 +76,9 @@ class rcu_domain_impl {
friend class rcu_domain;
- rcu_singly_list_view update_phase_and_wait() noexcept {
- retired_queue_stage1_.__splice_back(retired_queue_stage0_);
+ void update_phase_and_wait() noexcept {
+ rcu_singly_list_view working_queue;
+ working_queue.__splice_back(retired_queue_stage0_);
// Flip the global phase
auto old_phase = global_reader_phase_.fetch_xor(reader_states::grace_period_phase_mask, std::memory_order_relaxed);
@@ -92,10 +93,8 @@ class rcu_domain_impl {
}
grace_period_waiting_flag_.store(false, std::memory_order_relaxed);
- rcu_singly_list_view ready_callbacks;
- ready_callbacks.__splice_back(retired_queue_stage2_);
retired_queue_stage2_.__splice_back(retired_queue_stage1_);
- return ready_callbacks;
+ retired_queue_stage1_.__splice_back(working_queue);
}
bool any_reader_in_ongoing_grace_period(reader_states::state_type __global_phase) noexcept {
@@ -152,23 +151,34 @@ class rcu_domain_impl {
retired_queue_stage0_.__push_front(node);
}
- void synchronize() noexcept {
+ void synchronize(bool invoke_callback) noexcept {
__cxx_atomic_thread_fence(memory_order_seq_cst);
std::unique_lock lk(grace_period_mutex_);
- auto ready_callbacks = update_phase_and_wait();
+ update_phase_and_wait();
- // Invoke the ready callbacks outside of the grace period mutex
- lk.unlock();
- ready_callbacks.__for_each([](auto* node) { node->__callback_(); });
- lk.lock();
+ if (invoke_callback) {
+ rcu_singly_list_view ready_callbacks;
+ ready_callbacks.__splice_back(retired_queue_stage2_);
+
+ // Invoke the ready callbacks outside of the grace period mutex
+ lk.unlock();
+ ready_callbacks.__for_each([](auto* node) { node->__callback_(); });
+ lk.lock();
+ }
__barrier();
- ready_callbacks = update_phase_and_wait();
+ update_phase_and_wait();
- // Invoke the ready callbacks outside of the grace period mutex
lk.unlock();
- ready_callbacks.__for_each([](auto* node) { node->__callback_(); });
+
+ if (invoke_callback) {
+ rcu_singly_list_view ready_callbacks;
+ ready_callbacks.__splice_back(retired_queue_stage2_);
+
+ // Invoke the ready callbacks outside of the grace period mutex
+ ready_callbacks.__for_each([](auto* node) { node->__callback_(); });
+ }
__cxx_atomic_thread_fence(memory_order_seq_cst);
}
@@ -198,9 +208,9 @@ void rcu_domain::__retire(__rcu_node* node) noexcept { __pimpl_->retire(node); }
rcu_domain& rcu_default_domain() noexcept { return rcu_domain::__rcu_default_domain(); }
-void rcu_synchronize(rcu_domain& dom) noexcept { dom.__pimpl_->synchronize(); }
+void rcu_synchronize(rcu_domain& dom) noexcept { dom.__pimpl_->synchronize(false); }
-void rcu_barrier(rcu_domain& dom) noexcept { rcu_synchronize(dom); }
+void rcu_barrier(rcu_domain& dom) noexcept { dom.__pimpl_->synchronize(true); }
_LIBCPP_END_EXPLICIT_ABI_ANNOTATIONS
_LIBCPP_END_NAMESPACE_STD
diff --git a/libcxx/test/std/thread/saferecl/saferecl.rcu/general.pass.cpp b/libcxx/test/std/thread/saferecl/saferecl.rcu/general.pass.cpp
index e2168d09210cc..81e74bf45861f 100644
--- a/libcxx/test/std/thread/saferecl/saferecl.rcu/general.pass.cpp
+++ b/libcxx/test/std/thread/saferecl/saferecl.rcu/general.pass.cpp
@@ -17,7 +17,6 @@
#include <iostream>
#include <print>
-#include "__rcu/rcu_domain.h"
#include "make_test_thread.h"
#include "test_macros.h"
diff --git a/libcxx/test/std/thread/saferecl/saferecl.rcu/read_write_lock_compare.pass.cpp b/libcxx/test/std/thread/saferecl/saferecl.rcu/read_write_lock_compare.pass.cpp
index 8d637b7c05bfd..5aa7c82d1602c 100644
--- a/libcxx/test/std/thread/saferecl/saferecl.rcu/read_write_lock_compare.pass.cpp
+++ b/libcxx/test/std/thread/saferecl/saferecl.rcu/read_write_lock_compare.pass.cpp
@@ -22,7 +22,6 @@
#include <print>
#include <string>
-#include "__rcu/rcu_domain.h"
#include "make_test_thread.h"
#include "test_macros.h"
@@ -133,7 +132,7 @@ void test_rcu() {
auto syncer_func = [&dom](std::stop_token token) {
while (!token.stop_requested()) {
- std::rcu_synchronize(dom);
+ std::rcu_barrier(dom);
std::this_thread::sleep_for(std::chrono::milliseconds(50));
}
};
@@ -151,8 +150,8 @@ void test_rcu() {
}
writer.request_stop();
syncer.request_stop();
- std::rcu_synchronize(dom);
- std::println("RCU Writer thread destruction {} times", destruction_count.load());
+ std::rcu_barrier(dom);
+ std::println("RCU collector thread destruction {} times", destruction_count.load());
}
int main(int, char**) {
diff --git a/libcxx/test/std/thread/saferecl/saferecl.rcu/read_write_lock_compare_multi_writer.pass.cpp b/libcxx/test/std/thread/saferecl/saferecl.rcu/read_write_lock_compare_multi_writer.pass.cpp
index d345997d89444..aa1562d295c6d 100644
--- a/libcxx/test/std/thread/saferecl/saferecl.rcu/read_write_lock_compare_multi_writer.pass.cpp
+++ b/libcxx/test/std/thread/saferecl/saferecl.rcu/read_write_lock_compare_multi_writer.pass.cpp
@@ -21,7 +21,6 @@
#include <print>
#include <string>
-#include "__rcu/rcu_domain.h"
#include "make_test_thread.h"
#include "test_macros.h"
@@ -136,7 +135,7 @@ void test_rcu() {
auto syncer_func = [&dom](std::stop_token token) {
while (!token.stop_requested()) {
- std::rcu_synchronize(dom);
+ std::rcu_barrier(dom);
std::this_thread::sleep_for(std::chrono::milliseconds(50));
}
};
>From ff3150ca9641e6697b5a7b14edd027f7e720e8f3 Mon Sep 17 00:00:00 2001
From: Hui Xie <hui.xie1990 at gmail.com>
Date: Sun, 6 Sep 2026 13:25:08 +0100
Subject: [PATCH 36/65] code tidy up
---
libcxx/include/__rcu/rcu_domain.h | 3 -
libcxx/include/__rcu/rcu_obj_base.h | 6 +-
.../src/experimental/include/rcu/rcu_list.h | 69 +++++++++----------
.../include/rcu/thread_local_container.h | 7 +-
libcxx/src/experimental/rcu.cpp | 39 +++++------
5 files changed, 58 insertions(+), 66 deletions(-)
diff --git a/libcxx/include/__rcu/rcu_domain.h b/libcxx/include/__rcu/rcu_domain.h
index b7580ad53e221..66e51375fef3f 100644
--- a/libcxx/include/__rcu/rcu_domain.h
+++ b/libcxx/include/__rcu/rcu_domain.h
@@ -27,9 +27,6 @@ _LIBCPP_BEGIN_EXPLICIT_ABI_ANNOTATIONS
struct __rcu_node {
function_ref<void()> __callback_ = std::cw<[] {}>;
__rcu_node* __next_ = nullptr;
-
- _LIBCPP_HIDE_FROM_ABI __rcu_node() {}
- _LIBCPP_HIDE_FROM_ABI ~__rcu_node() {}
};
template <class _Tp, class _Deleter>
diff --git a/libcxx/include/__rcu/rcu_obj_base.h b/libcxx/include/__rcu/rcu_obj_base.h
index ccf653b22e6f6..a889d0144a8bd 100644
--- a/libcxx/include/__rcu/rcu_obj_base.h
+++ b/libcxx/include/__rcu/rcu_obj_base.h
@@ -10,7 +10,6 @@
#ifndef _LIBCPP___RCU_RCU_OBJ_BASE_H
#define _LIBCPP___RCU_RCU_OBJ_BASE_H
-#include "__configuration/attributes.h"
#include <__config>
#include <__functional/function_ref.h>
#include <__memory/unique_ptr.h> // for default_delete
@@ -29,13 +28,12 @@ template <class _Tp, class _Dp = default_delete<_Tp>>
class rcu_obj_base : private __rcu_node {
public:
_LIBCPP_HIDE_FROM_ABI void retire(_Dp __deleter = _Dp(), rcu_domain& __dom = rcu_default_domain()) noexcept {
- __deleter_ = std::move(__deleter);
- __callback_ = function_ref<void()>(std::cw<&rcu_obj_base::__destroy>, this);
+ __deleter_ = std::move(__deleter);
__dom.__retire(this);
}
protected:
- _LIBCPP_HIDE_FROM_ABI rcu_obj_base() = default;
+ _LIBCPP_HIDE_FROM_ABI rcu_obj_base() { __callback_ = function_ref<void()>(std::cw<&rcu_obj_base::__destroy>, this); }
_LIBCPP_HIDE_FROM_ABI rcu_obj_base(const rcu_obj_base&) = default;
_LIBCPP_HIDE_FROM_ABI rcu_obj_base(rcu_obj_base&&) = default;
_LIBCPP_HIDE_FROM_ABI rcu_obj_base& operator=(const rcu_obj_base&) = default;
diff --git a/libcxx/src/experimental/include/rcu/rcu_list.h b/libcxx/src/experimental/include/rcu/rcu_list.h
index 60d1380765ac0..3eb3dc5750795 100644
--- a/libcxx/src/experimental/include/rcu/rcu_list.h
+++ b/libcxx/src/experimental/include/rcu/rcu_list.h
@@ -29,44 +29,43 @@ class rcu_thread_local_list_view;
class rcu_singly_list_view {
private:
- __rcu_node* __head_ = nullptr;
- __rcu_node* __tail_ = nullptr;
+ __rcu_node* head_ = nullptr;
+ __rcu_node* tail_ = nullptr;
public:
- void __splice_back(rcu_singly_list_view& __other) noexcept {
- if (__other.__head_ == nullptr) {
+ void splice_back(rcu_singly_list_view& other) noexcept {
+ if (other.head_ == nullptr) {
return;
}
- if (__head_ == nullptr) {
- __head_ = __other.__head_;
- __tail_ = __other.__tail_;
+ if (head_ == nullptr) {
+ head_ = other.head_;
+ tail_ = other.tail_;
} else {
- __tail_->__next_ = __other.__head_;
- __tail_ = __other.__tail_;
+ tail_->__next_ = other.head_;
+ tail_ = other.tail_;
}
- __other.__head_ = nullptr;
- __other.__tail_ = nullptr;
+ other.head_ = nullptr;
+ other.tail_ = nullptr;
}
- void __splice_back(rcu_thread_local_list_view& __other) noexcept;
+ void splice_back(rcu_thread_local_list_view& __other) noexcept;
-
- template <class _Func>
- void __for_each(_Func&& __f) noexcept {
- __rcu_node* __current = __head_;
- while (__current != nullptr) {
+ template <class Func>
+ void for_each(Func&& f) noexcept {
+ __rcu_node* current = head_;
+ while (current != nullptr) {
// __f could delete __current, so we need to get the next pointer first
- auto __next = __current->__next_;
- __f(__current);
- __current = __next;
+ auto __next = current->__next_;
+ f(current);
+ current = __next;
}
}
};
class rcu_thread_local_list_view {
- struct alignas(2*sizeof(void*)) thread_entry {
- __rcu_node* __head_ = nullptr;
- __rcu_node* __tail_ = nullptr;
+ struct alignas(2 * sizeof(void*)) thread_entry {
+ __rcu_node* head_ = nullptr;
+ __rcu_node* tail_ = nullptr;
};
using per_thread_entries = thread_local_container<thread_entry>;
@@ -74,40 +73,40 @@ class rcu_thread_local_list_view {
friend class rcu_singly_list_view;
public:
- void __push_front(__rcu_node* __node) noexcept {
+ void push_front(__rcu_node* node) noexcept {
atomic_ref<thread_entry> entry_ref = per_thread_entries::get_current_thread_instance();
auto expected_entry = entry_ref.load(std::memory_order_relaxed);
- auto original_next = __node->__next_;
+ auto original_next = node->__next_;
while (true) {
auto new_entry = [&] {
- if (expected_entry.__head_ == nullptr) {
- return thread_entry{__node, __node};
+ if (expected_entry.head_ == nullptr) {
+ return thread_entry{node, node};
} else {
- __node->__next_ = expected_entry.__head_;
- return thread_entry{__node, expected_entry.__tail_};
+ node->__next_ = expected_entry.head_;
+ return thread_entry{node, expected_entry.tail_};
}
}();
if (entry_ref.compare_exchange_weak(
expected_entry, new_entry, std::memory_order_acq_rel, std::memory_order_relaxed)) {
break;
} else {
- __node->__next_ = original_next;
+ node->__next_ = original_next;
}
}
}
};
-void rcu_singly_list_view::__splice_back(rcu_thread_local_list_view& __other) noexcept {
+void rcu_singly_list_view::splice_back(rcu_thread_local_list_view& __other) noexcept {
using thread_entry = rcu_thread_local_list_view::thread_entry;
const auto splice_single_entry = [this](atomic_ref<thread_entry> entry_ref) noexcept {
- if (entry_ref.load(std::memory_order_relaxed).__head_ == nullptr) {
+ if (entry_ref.load(std::memory_order_relaxed).head_ == nullptr) {
return;
}
auto entry = entry_ref.exchange(thread_entry{nullptr, nullptr}, std::memory_order_acq_rel);
rcu_singly_list_view tmp;
- tmp.__head_ = entry.__head_;
- tmp.__tail_ = entry.__tail_;
- this->__splice_back(tmp);
+ tmp.head_ = entry.head_;
+ tmp.tail_ = entry.tail_;
+ this->splice_back(tmp);
};
rcu_thread_local_list_view::per_thread_entries::for_each(splice_single_entry);
}
diff --git a/libcxx/src/experimental/include/rcu/thread_local_container.h b/libcxx/src/experimental/include/rcu/thread_local_container.h
index 5b11b6dcab504..973b8279db78a 100644
--- a/libcxx/src/experimental/include/rcu/thread_local_container.h
+++ b/libcxx/src/experimental/include/rcu/thread_local_container.h
@@ -15,8 +15,8 @@
#include <__rcu/rcu_domain.h>
#include <atomic>
-#include <optional>
#include <mutex>
+#include <optional>
#include <vector>
#if !defined(_LIBCPP_HAS_NO_PRAGMA_SYSTEM_HEADER)
@@ -53,8 +53,9 @@ class thread_local_container {
static void deregister_instance(Tp& obj) {
lock_guard<std::mutex> lg(mtx_);
- instances_.erase(remove_if(instances_.begin(), instances_.end(), [&obj](Tp* instance) { return instance == &obj; }),
- instances_.end());
+ instances_.erase(
+ std::remove_if(instances_.begin(), instances_.end(), [&obj](Tp* instance) { return instance == &obj; }),
+ instances_.end());
}
public:
diff --git a/libcxx/src/experimental/rcu.cpp b/libcxx/src/experimental/rcu.cpp
index 1810f086cc394..0841f512da904 100644
--- a/libcxx/src/experimental/rcu.cpp
+++ b/libcxx/src/experimental/rcu.cpp
@@ -23,7 +23,6 @@ _LIBCPP_BEGIN_EXPLICIT_ABI_ANNOTATIONS
namespace {
-
// Adopted the 2-phase implementation in the section
// "3) General-Purpose RCU" of the paper
// http://www.rdrop.com/users/paulmck/RCU/urcu-supp-accepted.2011.08.30a.pdf
@@ -78,29 +77,29 @@ class rcu_domain_impl {
void update_phase_and_wait() noexcept {
rcu_singly_list_view working_queue;
- working_queue.__splice_back(retired_queue_stage0_);
+ working_queue.splice_back(retired_queue_stage0_);
// Flip the global phase
auto old_phase = global_reader_phase_.fetch_xor(reader_states::grace_period_phase_mask, std::memory_order_relaxed);
- auto __new_phase = old_phase ^ reader_states::grace_period_phase_mask;
- // std::printf("rcu_domain::update_phase_and_wait() new phase: 0x%04x\n", __new_phase);
+ auto new_phase = old_phase ^ reader_states::grace_period_phase_mask;
+ // std::printf("rcu_domain::update_phase_and_wait() new phase: 0x%04x\n", new_phase);
- __barrier();
+ barrier();
// Wait for all threads to quiesce in the old phase
- while (any_reader_in_ongoing_grace_period(__new_phase)) {
+ while (any_reader_in_ongoing_grace_period(new_phase)) {
grace_period_waiting_flag_.store(true, std::memory_order_relaxed);
grace_period_waiting_flag_.wait(true, std::memory_order_relaxed);
}
grace_period_waiting_flag_.store(false, std::memory_order_relaxed);
- retired_queue_stage2_.__splice_back(retired_queue_stage1_);
- retired_queue_stage1_.__splice_back(working_queue);
+ retired_queue_stage2_.splice_back(retired_queue_stage1_);
+ retired_queue_stage1_.splice_back(working_queue);
}
- bool any_reader_in_ongoing_grace_period(reader_states::state_type __global_phase) noexcept {
+ bool any_reader_in_ongoing_grace_period(reader_states::state_type global_phase) noexcept {
bool any_ongoing = false;
- per_thread_states::for_each([this, __global_phase, &any_ongoing](atomic_ref<reader_states::state_type> __state) {
- if (is_grace_period_ongoing(__state.load(memory_order_relaxed), __global_phase)) {
+ per_thread_states::for_each([this, global_phase, &any_ongoing](atomic_ref<reader_states::state_type> state) {
+ if (is_grace_period_ongoing(state.load(memory_order_relaxed), global_phase)) {
any_ongoing = true;
}
});
@@ -117,7 +116,7 @@ class rcu_domain_impl {
reader_states::get_grace_period_phase(thread_state) != global_phase;
}
- void __barrier() noexcept { asm volatile("" : : : "memory"); }
+ void barrier() noexcept { asm volatile("" : : : "memory"); }
public:
void lock() noexcept {
@@ -147,9 +146,7 @@ class rcu_domain_impl {
}
}
- void retire(__rcu_node* node) noexcept {
- retired_queue_stage0_.__push_front(node);
- }
+ void retire(__rcu_node* node) noexcept { retired_queue_stage0_.push_front(node); }
void synchronize(bool invoke_callback) noexcept {
__cxx_atomic_thread_fence(memory_order_seq_cst);
@@ -159,31 +156,31 @@ class rcu_domain_impl {
if (invoke_callback) {
rcu_singly_list_view ready_callbacks;
- ready_callbacks.__splice_back(retired_queue_stage2_);
+ ready_callbacks.splice_back(retired_queue_stage2_);
// Invoke the ready callbacks outside of the grace period mutex
lk.unlock();
- ready_callbacks.__for_each([](auto* node) { node->__callback_(); });
+ ready_callbacks.for_each([](auto* node) { node->__callback_(); });
lk.lock();
}
- __barrier();
+ barrier();
update_phase_and_wait();
lk.unlock();
if (invoke_callback) {
rcu_singly_list_view ready_callbacks;
- ready_callbacks.__splice_back(retired_queue_stage2_);
+ ready_callbacks.splice_back(retired_queue_stage2_);
// Invoke the ready callbacks outside of the grace period mutex
- ready_callbacks.__for_each([](auto* node) { node->__callback_(); });
+ ready_callbacks.for_each([](auto* node) { node->__callback_(); });
}
__cxx_atomic_thread_fence(memory_order_seq_cst);
}
void debugPrintAllReaderStatesInHex() {
- per_thread_states::for_each([](auto __state_ref) { std::printf("Reader state: 0x%04x\n", __state_ref.load()); });
+ per_thread_states::for_each([](auto state_ref) { std::printf("Reader state: 0x%04x\n", state_ref.load()); });
}
};
} // namespace
>From 97c448091c71a4d77a9e4d54b1af6b2cec63592f Mon Sep 17 00:00:00 2001
From: Hui Xie <hui.xie1990 at gmail.com>
Date: Tue, 8 Sep 2026 15:29:35 +0100
Subject: [PATCH 37/65] rcu_obj_base test
---
libcxx/include/__rcu/rcu_obj_base.h | 12 ++-
.../saferecl.rcu.base/mandates.verify.cpp | 47 ++++++++++
.../saferecl/saferecl.rcu/general.pass.cpp | 2 +-
.../read_write_lock_compare.pass.cpp | 2 +-
...d_write_lock_compare_multi_writer.pass.cpp | 2 +-
.../saferecl/saferecl.rcu/retire.pass.cpp | 2 +-
.../saferecl.rcu.base/ctor.pass.cpp | 61 +++++++++++++
.../incomplete.compile.pass.cpp | 21 +++++
.../saferecl.rcu.base/retire.pass.cpp | 90 +++++++++++++++++++
9 files changed, 233 insertions(+), 6 deletions(-)
create mode 100644 libcxx/test/libcxx/thread/saferecl/saferecl.rcu/saferecl.rcu.base/mandates.verify.cpp
create mode 100644 libcxx/test/std/thread/saferecl/saferecl.rcu/saferecl.rcu.base/ctor.pass.cpp
create mode 100644 libcxx/test/std/thread/saferecl/saferecl.rcu/saferecl.rcu.base/incomplete.compile.pass.cpp
create mode 100644 libcxx/test/std/thread/saferecl/saferecl.rcu/saferecl.rcu.base/retire.pass.cpp
diff --git a/libcxx/include/__rcu/rcu_obj_base.h b/libcxx/include/__rcu/rcu_obj_base.h
index a889d0144a8bd..d75b90ade4eb7 100644
--- a/libcxx/include/__rcu/rcu_obj_base.h
+++ b/libcxx/include/__rcu/rcu_obj_base.h
@@ -14,6 +14,8 @@
#include <__functional/function_ref.h>
#include <__memory/unique_ptr.h> // for default_delete
#include <__rcu/rcu_domain.h>
+#include <__type_traits/is_assignable.h>
+#include <__type_traits/is_constructible.h>
#include <__utility/move.h>
#if !defined(_LIBCPP_HAS_NO_PRAGMA_SYSTEM_HEADER)
@@ -26,14 +28,20 @@ _LIBCPP_BEGIN_NAMESPACE_STD
template <class _Tp, class _Dp = default_delete<_Tp>>
class rcu_obj_base : private __rcu_node {
+ static_assert(std::is_default_constructible_v<_Dp>);
+ static_assert(std::is_move_assignable_v<_Dp>);
+ static_assert(requires(_Tp* __ptr, _Dp __d) { __d(__ptr); }, "Deleter must be callable with an object pointer.");
+
public:
_LIBCPP_HIDE_FROM_ABI void retire(_Dp __deleter = _Dp(), rcu_domain& __dom = rcu_default_domain()) noexcept {
- __deleter_ = std::move(__deleter);
+ static_assert(std::is_base_of_v<rcu_obj_base, _Tp>, "T must be an rcu-protectable type.");
+ __deleter_ = std::move(__deleter);
+ __callback_ = function_ref<void()>(std::cw<&rcu_obj_base::__destroy>, this);
__dom.__retire(this);
}
protected:
- _LIBCPP_HIDE_FROM_ABI rcu_obj_base() { __callback_ = function_ref<void()>(std::cw<&rcu_obj_base::__destroy>, this); }
+ _LIBCPP_HIDE_FROM_ABI rcu_obj_base() = default;
_LIBCPP_HIDE_FROM_ABI rcu_obj_base(const rcu_obj_base&) = default;
_LIBCPP_HIDE_FROM_ABI rcu_obj_base(rcu_obj_base&&) = default;
_LIBCPP_HIDE_FROM_ABI rcu_obj_base& operator=(const rcu_obj_base&) = default;
diff --git a/libcxx/test/libcxx/thread/saferecl/saferecl.rcu/saferecl.rcu.base/mandates.verify.cpp b/libcxx/test/libcxx/thread/saferecl/saferecl.rcu/saferecl.rcu.base/mandates.verify.cpp
new file mode 100644
index 0000000000000..75f76494bce8b
--- /dev/null
+++ b/libcxx/test/libcxx/thread/saferecl/saferecl.rcu/saferecl.rcu.base/mandates.verify.cpp
@@ -0,0 +1,47 @@
+//===----------------------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+//
+// UNSUPPORTED: no-threads
+// REQUIRES: std-at-least-c++26
+
+// template<class T, class D = default_delete<T>>
+// class rcu_obj_base
+// T may be an incomplete type. It shall be complete before any member of the resulting specialization of rcu_obj_base is referenced.
+
+#include <rcu>
+#include <type_traits>
+
+// D shall be a function object type ([function.objects]) for which, given a value d of type D and a value ptr of type T*, the expression d(ptr) is valid.
+struct A {};
+struct B : std::rcu_obj_base<B, A> {};
+// expected-error-re@*:* {{static assertion failed {{.*}}Deleter must be callable with an object pointer.}}
+
+// D shall meet the requirements for Cpp17DefaultConstructible and Cpp17MoveAssignable.
+struct NonDefault {
+ NonDefault(int) {}
+ void operator()(auto...) const {}
+};
+struct C : std::rcu_obj_base<C, NonDefault> {};
+// expected-error-re@*:* {{static assertion failed {{.*}}std::is_default_constructible_v<NonDefault>}}
+
+struct NonMoveAssignable {
+ NonMoveAssignable() = default;
+ NonMoveAssignable(NonMoveAssignable&&) = default;
+ NonMoveAssignable& operator==(NonMoveAssignable&&) = delete;
+};
+struct D : std::rcu_obj_base<D, NonMoveAssignable> {};
+// expected-error-re@*:* {{static assertion failed {{.*}}std::is_move_assignable_v<NonMoveAssignable>}}
+
+// void retire(D d = D(), rcu_domain& dom = rcu_default_domain()) noexcept;
+// Mandates: T is an rcu-protectable type.
+struct E : std::rcu_obj_base<A> {};
+void test(E* e) {
+ e->retire();
+ // expected-error-re@*:* {{static assertion failed {{.*}}T must be an rcu-protectable type.}}
+ // expected-error-re@*:* {{static_cast {{.*}}}}
+}
diff --git a/libcxx/test/std/thread/saferecl/saferecl.rcu/general.pass.cpp b/libcxx/test/std/thread/saferecl/saferecl.rcu/general.pass.cpp
index 81e74bf45861f..aac4247498c09 100644
--- a/libcxx/test/std/thread/saferecl/saferecl.rcu/general.pass.cpp
+++ b/libcxx/test/std/thread/saferecl/saferecl.rcu/general.pass.cpp
@@ -7,7 +7,7 @@
//===----------------------------------------------------------------------===//
//
// UNSUPPORTED: no-threads
-// UNSUPPORTED: c++03, c++11, c++14, c++17, c++20, c++23
+// REQUIRES: std-at-least-c++26
// <rcu>
diff --git a/libcxx/test/std/thread/saferecl/saferecl.rcu/read_write_lock_compare.pass.cpp b/libcxx/test/std/thread/saferecl/saferecl.rcu/read_write_lock_compare.pass.cpp
index 5aa7c82d1602c..f96a89ddd165a 100644
--- a/libcxx/test/std/thread/saferecl/saferecl.rcu/read_write_lock_compare.pass.cpp
+++ b/libcxx/test/std/thread/saferecl/saferecl.rcu/read_write_lock_compare.pass.cpp
@@ -7,7 +7,7 @@
//===----------------------------------------------------------------------===//
//
// UNSUPPORTED: no-threads
-// UNSUPPORTED: c++03, c++11, c++14, c++17, c++20, c++23
+// REQUIRES: std-at-least-c++26
// <rcu>
diff --git a/libcxx/test/std/thread/saferecl/saferecl.rcu/read_write_lock_compare_multi_writer.pass.cpp b/libcxx/test/std/thread/saferecl/saferecl.rcu/read_write_lock_compare_multi_writer.pass.cpp
index aa1562d295c6d..98a2d5d42b8ad 100644
--- a/libcxx/test/std/thread/saferecl/saferecl.rcu/read_write_lock_compare_multi_writer.pass.cpp
+++ b/libcxx/test/std/thread/saferecl/saferecl.rcu/read_write_lock_compare_multi_writer.pass.cpp
@@ -7,7 +7,7 @@
//===----------------------------------------------------------------------===//
//
// UNSUPPORTED: no-threads
-// UNSUPPORTED: c++03, c++11, c++14, c++17, c++20, c++23
+// REQUIRES: std-at-least-c++26
// <rcu>
diff --git a/libcxx/test/std/thread/saferecl/saferecl.rcu/retire.pass.cpp b/libcxx/test/std/thread/saferecl/saferecl.rcu/retire.pass.cpp
index 9f7ef08240091..03cfeb572c2dd 100644
--- a/libcxx/test/std/thread/saferecl/saferecl.rcu/retire.pass.cpp
+++ b/libcxx/test/std/thread/saferecl/saferecl.rcu/retire.pass.cpp
@@ -7,7 +7,7 @@
//===----------------------------------------------------------------------===//
//
// UNSUPPORTED: no-threads
-// UNSUPPORTED: c++03, c++11, c++14, c++17, c++20, c++23
+// REQUIRES: std-at-least-c++26
// <rcu>
diff --git a/libcxx/test/std/thread/saferecl/saferecl.rcu/saferecl.rcu.base/ctor.pass.cpp b/libcxx/test/std/thread/saferecl/saferecl.rcu/saferecl.rcu.base/ctor.pass.cpp
new file mode 100644
index 0000000000000..7a5765b7fc6a0
--- /dev/null
+++ b/libcxx/test/std/thread/saferecl/saferecl.rcu/saferecl.rcu.base/ctor.pass.cpp
@@ -0,0 +1,61 @@
+//===----------------------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+//
+// UNSUPPORTED: no-threads
+// REQUIRES: std-at-least-c++26
+
+// protected:
+// rcu_obj_base() = default;
+// rcu_obj_base(const rcu_obj_base&) = default;
+// rcu_obj_base(rcu_obj_base&&) = default;
+// rcu_obj_base& operator=(const rcu_obj_base&) = default;
+// rcu_obj_base& operator=(rcu_obj_base&&) = default;
+// ~rcu_obj_base() = default;
+
+#include <cassert>
+#include <rcu>
+#include <thread>
+#include <type_traits>
+
+#include "make_test_thread.h"
+#include "test_macros.h"
+
+class A : public std::rcu_obj_base<A> {};
+
+static_assert(!std::is_default_constructible_v<std::rcu_obj_base<A>>);
+static_assert(!std::is_copy_constructible_v<std::rcu_obj_base<A>>);
+static_assert(!std::is_move_constructible_v<std::rcu_obj_base<A>>);
+static_assert(!std::is_copy_assignable_v<std::rcu_obj_base<A>>);
+static_assert(!std::is_move_assignable_v<std::rcu_obj_base<A>>);
+static_assert(!std::is_destructible_v<std::rcu_obj_base<A>>);
+
+static_assert(std::is_default_constructible_v<A>);
+static_assert(std::is_copy_constructible_v<A>);
+static_assert(std::is_move_constructible_v<A>);
+static_assert(std::is_copy_assignable_v<A>);
+static_assert(std::is_move_assignable_v<A>);
+static_assert(std::is_destructible_v<A>);
+
+// If D is trivially copyable, all specializations of rcu_obj_base<T, D> are trivially copyable.
+struct TrivialDeleter {
+ void operator()(auto...) const {}
+};
+struct B : std::rcu_obj_base<B, TrivialDeleter> {};
+static_assert(std::is_trivially_copyable_v<std::rcu_obj_base<A, TrivialDeleter>>);
+
+int main(int, char**) {
+ {
+ A a1;
+ A a2(a1);
+ A a3(std::move(a1));
+ a1 = a2;
+ a1 = std::move(a3);
+ }
+
+ return 0;
+}
diff --git a/libcxx/test/std/thread/saferecl/saferecl.rcu/saferecl.rcu.base/incomplete.compile.pass.cpp b/libcxx/test/std/thread/saferecl/saferecl.rcu/saferecl.rcu.base/incomplete.compile.pass.cpp
new file mode 100644
index 0000000000000..2a3674c593c7b
--- /dev/null
+++ b/libcxx/test/std/thread/saferecl/saferecl.rcu/saferecl.rcu.base/incomplete.compile.pass.cpp
@@ -0,0 +1,21 @@
+//===----------------------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+//
+// UNSUPPORTED: no-threads
+// REQUIRES: std-at-least-c++26
+
+// template<class T, class D = default_delete<T>>
+// class rcu_obj_base
+// T may be an incomplete type. It shall be complete before any member of the resulting specialization of rcu_obj_base is referenced.
+
+#include <rcu>
+#include <type_traits>
+
+class A;
+
+void test(std::rcu_obj_base<A>*) {}
diff --git a/libcxx/test/std/thread/saferecl/saferecl.rcu/saferecl.rcu.base/retire.pass.cpp b/libcxx/test/std/thread/saferecl/saferecl.rcu/saferecl.rcu.base/retire.pass.cpp
new file mode 100644
index 0000000000000..de15abb4a1026
--- /dev/null
+++ b/libcxx/test/std/thread/saferecl/saferecl.rcu/saferecl.rcu.base/retire.pass.cpp
@@ -0,0 +1,90 @@
+//===----------------------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+//
+// UNSUPPORTED: no-threads
+// REQUIRES: std-at-least-c++26
+
+// void retire(D d = D(), rcu_domain& dom = rcu_default_domain()) noexcept;
+
+#include <cassert>
+#include <rcu>
+#include <thread>
+
+#include "make_test_thread.h"
+#include "test_macros.h"
+
+class TestClass1 : public std::rcu_obj_base<TestClass1> {
+ int& destruction_count_;
+
+public:
+ TestClass1(int& c) : destruction_count_(c) {}
+ ~TestClass1() { ++destruction_count_; }
+};
+
+struct Deleter {
+ int i = 0;
+ template <class T>
+ void operator()(T* ptr) const {
+ ptr->i_ = i;
+ delete ptr;
+ }
+};
+
+struct CustomDeleter : std::rcu_obj_base<CustomDeleter, Deleter> {
+ int& i_;
+ int& destruction_count_;
+ CustomDeleter(int& i, int& destruction_count) : i_(i), destruction_count_(destruction_count) {}
+ ~CustomDeleter() { ++destruction_count_; }
+};
+
+// Test noexcept
+template <class T, class... Args>
+concept NothrowRetireable = requires(T t, Args... args) {
+ { t.retire(args...) } noexcept;
+};
+
+static_assert(NothrowRetireable<TestClass1>);
+static_assert(NothrowRetireable<CustomDeleter>);
+static_assert(NothrowRetireable<CustomDeleter, Deleter>);
+static_assert(NothrowRetireable<CustomDeleter, Deleter, std::rcu_domain>);
+
+int main(int, char**) {
+ {
+ // retire();
+
+ int destruction_count = 0;
+ auto* ptr = new TestClass1(destruction_count);
+ ptr->retire();
+ std::rcu_barrier();
+ assert(destruction_count == 1);
+ }
+
+ {
+ // retire(deleter);
+ int i = 0;
+ int destruction_count = 0;
+ auto* ptr = new CustomDeleter(i, destruction_count);
+ ptr->retire(Deleter{42});
+ std::rcu_barrier();
+ assert(destruction_count == 1);
+ assert(i == 42);
+ }
+
+ {
+ // retire(deleter, domain);
+ int i = 0;
+ int destruction_count = 0;
+ auto* ptr = new CustomDeleter(i, destruction_count);
+ ptr->retire(Deleter{42}, std::rcu_default_domain());
+ std::rcu_barrier();
+ assert(destruction_count == 1);
+ assert(i == 42);
+ }
+
+ return 0;
+}
>From 4af40aefd0a127d8b2cd27fc66ba607c74dfae56 Mon Sep 17 00:00:00 2001
From: Hui Xie <hui.xie1990 at gmail.com>
Date: Tue, 8 Sep 2026 20:21:21 +0100
Subject: [PATCH 38/65] domain tests to be continued
---
libcxx/include/__rcu/rcu_domain.h | 7 +-
libcxx/src/experimental/rcu.cpp | 15 +-
.../saferecl.rcu.base/mandates.verify.cpp | 6 +-
.../saferecl.rcu.domain/mandates.verify.cpp | 25 +++
.../saferecl.rcu.base/retire.pass.cpp | 3 +
.../saferecl.rcu.domain/ctor.compile.pass.cpp | 22 +++
.../saferecl.rcu.domain/lock_unlock.pass.cpp | 143 ++++++++++++++++++
.../rcu_default_domain.pass.cpp | 38 +++++
.../saferecl.rcu.domain/rcu_retire.pass.cpp | 82 ++++++++++
9 files changed, 326 insertions(+), 15 deletions(-)
create mode 100644 libcxx/test/libcxx/thread/saferecl/saferecl.rcu/saferecl.rcu.domain/mandates.verify.cpp
create mode 100644 libcxx/test/std/thread/saferecl/saferecl.rcu/saferecl.rcu.domain/ctor.compile.pass.cpp
create mode 100644 libcxx/test/std/thread/saferecl/saferecl.rcu/saferecl.rcu.domain/lock_unlock.pass.cpp
create mode 100644 libcxx/test/std/thread/saferecl/saferecl.rcu/saferecl.rcu.domain/rcu_default_domain.pass.cpp
create mode 100644 libcxx/test/std/thread/saferecl/saferecl.rcu/saferecl.rcu.domain/rcu_retire.pass.cpp
diff --git a/libcxx/include/__rcu/rcu_domain.h b/libcxx/include/__rcu/rcu_domain.h
index 66e51375fef3f..fb542967129de 100644
--- a/libcxx/include/__rcu/rcu_domain.h
+++ b/libcxx/include/__rcu/rcu_domain.h
@@ -13,6 +13,7 @@
#include <__config>
#include <__functional/function_ref.h>
#include <__memory/unique_ptr.h>
+#include <__type_traits/is_constructible.h>
#include <__utility/move.h>
#if !defined(_LIBCPP_HAS_NO_PRAGMA_SYSTEM_HEADER)
@@ -57,8 +58,8 @@ class _LIBCPP_EXPORTED_FROM_ABI rcu_domain {
static rcu_domain& __rcu_default_domain() noexcept;
- template <class _Tp, class _Dp = default_delete<_Tp>>
- _LIBCPP_HIDE_FROM_ABI void __rcu_retire_hidden_friend(_Tp* __tp, _Dp __deleter, rcu_domain& __dom) {
+ template <class _Tp, class _Dp>
+ friend _LIBCPP_HIDE_FROM_ABI void __rcu_retire_hidden_friend(_Tp* __tp, _Dp __deleter, rcu_domain& __dom) {
auto* __node = new __rcu_node_with_deleter<_Tp, _Dp>(__tp, std::move(__deleter));
__dom.__retire(__node);
}
@@ -92,6 +93,8 @@ _LIBCPP_EXPORTED_FROM_ABI void rcu_barrier(rcu_domain& __dom = rcu_default_domai
template <class _Tp, class _Dp = default_delete<_Tp>>
_LIBCPP_HIDE_FROM_ABI void rcu_retire(_Tp* __tp, _Dp __deleter = _Dp(), rcu_domain& __dom = rcu_default_domain()) {
+ static_assert(std::is_move_constructible_v<_Dp>);
+ static_assert(requires(_Dp __dp, _Tp* __ptr) { __dp(__ptr); }, "Deleter must be callable with a pointer");
__rcu_retire_hidden_friend(__tp, std::move(__deleter), __dom);
}
diff --git a/libcxx/src/experimental/rcu.cpp b/libcxx/src/experimental/rcu.cpp
index 0841f512da904..6f6026329c504 100644
--- a/libcxx/src/experimental/rcu.cpp
+++ b/libcxx/src/experimental/rcu.cpp
@@ -84,7 +84,8 @@ class rcu_domain_impl {
auto new_phase = old_phase ^ reader_states::grace_period_phase_mask;
// std::printf("rcu_domain::update_phase_and_wait() new phase: 0x%04x\n", new_phase);
- barrier();
+ std::atomic_signal_fence(std::memory_order_seq_cst);
+
// Wait for all threads to quiesce in the old phase
while (any_reader_in_ongoing_grace_period(new_phase)) {
grace_period_waiting_flag_.store(true, std::memory_order_relaxed);
@@ -116,8 +117,6 @@ class rcu_domain_impl {
reader_states::get_grace_period_phase(thread_state) != global_phase;
}
- void barrier() noexcept { asm volatile("" : : : "memory"); }
-
public:
void lock() noexcept {
auto current_thread_state_ref = per_thread_states::get_current_thread_instance();
@@ -126,7 +125,7 @@ class rcu_domain_impl {
// Entering a read-side critical section from a quiescent state.
current_thread_state_ref.store(
reader_states::make_state(global_reader_phase_.load(std::memory_order_relaxed), 1), memory_order_relaxed);
- __cxx_atomic_thread_fence(memory_order_seq_cst);
+ std::atomic_thread_fence(memory_order_seq_cst);
} else {
// Already in read-side critical section, just increment the nest level.
current_thread_state_ref.fetch_add(1, memory_order_relaxed);
@@ -135,7 +134,7 @@ class rcu_domain_impl {
void unlock() noexcept {
auto current_thread_state_ref = per_thread_states::get_current_thread_instance();
- __cxx_atomic_thread_fence(memory_order_seq_cst);
+ std::atomic_thread_fence(memory_order_seq_cst);
// Decrement the nest level.
auto old_state = current_thread_state_ref.fetch_sub(1, memory_order_relaxed);
@@ -149,7 +148,7 @@ class rcu_domain_impl {
void retire(__rcu_node* node) noexcept { retired_queue_stage0_.push_front(node); }
void synchronize(bool invoke_callback) noexcept {
- __cxx_atomic_thread_fence(memory_order_seq_cst);
+ std::atomic_thread_fence(memory_order_seq_cst);
std::unique_lock lk(grace_period_mutex_);
update_phase_and_wait();
@@ -164,7 +163,7 @@ class rcu_domain_impl {
lk.lock();
}
- barrier();
+ std::atomic_signal_fence(memory_order_seq_cst);
update_phase_and_wait();
lk.unlock();
@@ -176,7 +175,7 @@ class rcu_domain_impl {
// Invoke the ready callbacks outside of the grace period mutex
ready_callbacks.for_each([](auto* node) { node->__callback_(); });
}
- __cxx_atomic_thread_fence(memory_order_seq_cst);
+ std::atomic_thread_fence(memory_order_seq_cst);
}
void debugPrintAllReaderStatesInHex() {
diff --git a/libcxx/test/libcxx/thread/saferecl/saferecl.rcu/saferecl.rcu.base/mandates.verify.cpp b/libcxx/test/libcxx/thread/saferecl/saferecl.rcu/saferecl.rcu.base/mandates.verify.cpp
index 75f76494bce8b..3796ae4cba1e8 100644
--- a/libcxx/test/libcxx/thread/saferecl/saferecl.rcu/saferecl.rcu.base/mandates.verify.cpp
+++ b/libcxx/test/libcxx/thread/saferecl/saferecl.rcu/saferecl.rcu.base/mandates.verify.cpp
@@ -9,10 +9,6 @@
// UNSUPPORTED: no-threads
// REQUIRES: std-at-least-c++26
-// template<class T, class D = default_delete<T>>
-// class rcu_obj_base
-// T may be an incomplete type. It shall be complete before any member of the resulting specialization of rcu_obj_base is referenced.
-
#include <rcu>
#include <type_traits>
@@ -32,7 +28,7 @@ struct C : std::rcu_obj_base<C, NonDefault> {};
struct NonMoveAssignable {
NonMoveAssignable() = default;
NonMoveAssignable(NonMoveAssignable&&) = default;
- NonMoveAssignable& operator==(NonMoveAssignable&&) = delete;
+ NonMoveAssignable& operator=(NonMoveAssignable&&) = delete;
};
struct D : std::rcu_obj_base<D, NonMoveAssignable> {};
// expected-error-re@*:* {{static assertion failed {{.*}}std::is_move_assignable_v<NonMoveAssignable>}}
diff --git a/libcxx/test/libcxx/thread/saferecl/saferecl.rcu/saferecl.rcu.domain/mandates.verify.cpp b/libcxx/test/libcxx/thread/saferecl/saferecl.rcu/saferecl.rcu.domain/mandates.verify.cpp
new file mode 100644
index 0000000000000..42deda0d727e1
--- /dev/null
+++ b/libcxx/test/libcxx/thread/saferecl/saferecl.rcu/saferecl.rcu.domain/mandates.verify.cpp
@@ -0,0 +1,25 @@
+//===----------------------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+//
+// UNSUPPORTED: no-threads
+// REQUIRES: std-at-least-c++26
+
+#include <rcu>
+#include <type_traits>
+
+// template<class T, class D = default_delete<T>>
+// void rcu_retire(T* p, D d = D(), rcu_domain& dom = rcu_default_domain());
+// Mandates: is_move_constructible_v<D> is true and the expression d(p) is well-formed.
+
+struct NotDeleter {};
+
+void test(int* ptr) {
+ std::rcu_retire(ptr, NotDeleter{});
+ // expected-error-re@*:* {{static assertion failed {{.*}}Deleter must be callable with a pointer}}
+ // expected-error-re@*:* {{{{.*}}}}
+}
diff --git a/libcxx/test/std/thread/saferecl/saferecl.rcu/saferecl.rcu.base/retire.pass.cpp b/libcxx/test/std/thread/saferecl/saferecl.rcu/saferecl.rcu.base/retire.pass.cpp
index de15abb4a1026..1dd17cda6cadd 100644
--- a/libcxx/test/std/thread/saferecl/saferecl.rcu/saferecl.rcu.base/retire.pass.cpp
+++ b/libcxx/test/std/thread/saferecl/saferecl.rcu/saferecl.rcu.base/retire.pass.cpp
@@ -85,6 +85,9 @@ int main(int, char**) {
assert(destruction_count == 1);
assert(i == 42);
}
+ {
+ // todo: multi threaded test
+ }
return 0;
}
diff --git a/libcxx/test/std/thread/saferecl/saferecl.rcu/saferecl.rcu.domain/ctor.compile.pass.cpp b/libcxx/test/std/thread/saferecl/saferecl.rcu/saferecl.rcu.domain/ctor.compile.pass.cpp
new file mode 100644
index 0000000000000..671804546d412
--- /dev/null
+++ b/libcxx/test/std/thread/saferecl/saferecl.rcu/saferecl.rcu.domain/ctor.compile.pass.cpp
@@ -0,0 +1,22 @@
+//===----------------------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+//
+// UNSUPPORTED: no-threads
+// REQUIRES: std-at-least-c++26
+
+// rcu_domain(const rcu_domain&) = delete;
+// rcu_domain& operator=(const rcu_domain&) = delete;
+
+#include <rcu>
+#include <type_traits>
+
+static_assert(!std::is_default_constructible_v<std::rcu_domain>);
+static_assert(!std::is_copy_constructible_v<std::rcu_domain>);
+static_assert(!std::is_move_constructible_v<std::rcu_domain>);
+static_assert(!std::is_copy_assignable_v<std::rcu_domain>);
+static_assert(!std::is_move_assignable_v<std::rcu_domain>);
diff --git a/libcxx/test/std/thread/saferecl/saferecl.rcu/saferecl.rcu.domain/lock_unlock.pass.cpp b/libcxx/test/std/thread/saferecl/saferecl.rcu/saferecl.rcu.domain/lock_unlock.pass.cpp
new file mode 100644
index 0000000000000..7a0fc7b1ec676
--- /dev/null
+++ b/libcxx/test/std/thread/saferecl/saferecl.rcu/saferecl.rcu.domain/lock_unlock.pass.cpp
@@ -0,0 +1,143 @@
+//===----------------------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+//
+// UNSUPPORTED: no-threads
+// REQUIRES: std-at-least-c++26
+
+// void lock() noexcept;
+// bool try_lock() noexcept;
+// void unlock() noexcept;
+
+#include <atomic>
+#include <cassert>
+#include <chrono>
+#include <cstddef>
+#include <latch>
+#include <rcu>
+#include <thread>
+#include <utility>
+
+#include "make_test_thread.h"
+#include "test_macros.h"
+
+static_assert(noexcept(std::declval<std::rcu_domain>().lock()));
+static_assert(noexcept(std::declval<std::rcu_domain>().unlock()));
+static_assert(noexcept(std::declval<std::rcu_domain>().try_lock()));
+
+int main(int, char**) {
+ {
+ // lock opens rcu protected region
+ // unlock closes it
+ std::atomic_bool flag = false;
+ auto& dom = std::rcu_default_domain();
+ dom.lock();
+
+ auto t = support::make_test_jthread([&] {
+ std::rcu_synchronize();
+ flag = true;
+ });
+
+ std::this_thread::sleep_for(std::chrono::milliseconds(10));
+ // flag should not be set because rcu_synchronize should block after lock
+ assert(flag.load() == false);
+ dom.unlock(); // should unblock rcu_synchronize
+ }
+
+ {
+ // lock/unlock can be nested
+
+ std::atomic_bool flag = false;
+ auto& dom = std::rcu_default_domain();
+ dom.lock();
+ dom.lock();
+ dom.lock();
+
+ auto t = support::make_test_jthread([&] {
+ std::rcu_synchronize();
+ flag = true;
+ });
+
+ std::this_thread::sleep_for(std::chrono::milliseconds(10));
+ // flag should not be set because rcu_synchronize should block after lock
+ assert(flag.load() == false);
+
+ dom.unlock();
+
+ std::this_thread::sleep_for(std::chrono::milliseconds(10));
+ // flag should not be set because rcu_synchronize should block after lock
+ assert(flag.load() == false);
+
+ dom.unlock();
+
+ std::this_thread::sleep_for(std::chrono::milliseconds(10));
+ // flag should not be set because rcu_synchronize should block after lock
+ assert(flag.load() == false);
+
+ dom.unlock();
+ }
+
+ {
+ // multi-threaded lock/unlock
+ constexpr std::size_t num_threads = 4;
+ std::atomic_bool flag = false;
+ std::array<std::atomic_bool, 4> unlock_signal{};
+ std::latch lock_latch(num_threads);
+
+ std::vector<std::jthread> threads;
+ threads.reserve(num_threads);
+
+ for (size_t i = 0; i < num_threads; ++i) {
+ threads.push_back(support::make_test_jthread([&, i] {
+ auto& dom = std::rcu_default_domain();
+ dom.lock();
+ lock_latch.arrive_and_wait();
+ unlock_signal[i].wait(false);
+ dom.unlock();
+ }));
+ }
+
+ auto sync_thread = support::make_test_jthread([&] {
+ lock_latch.wait();
+ std::rcu_synchronize();
+ flag = true;
+ });
+
+ lock_latch.wait();
+ std::this_thread::sleep_for(std::chrono::milliseconds(10));
+ assert(flag.load() == false);
+
+ for (std::size_t i = 0; i < num_threads - 1; ++i) {
+ unlock_signal[i].store(true);
+ unlock_signal[i].notify_all();
+ std::this_thread::sleep_for(std::chrono::milliseconds(10));
+ assert(flag.load() == false);
+ }
+
+ unlock_signal[3].store(true);
+ unlock_signal[3].notify_all();
+ }
+ {
+ // try_lock
+ std::atomic_bool flag = false;
+ auto& dom = std::rcu_default_domain();
+ std::same_as<bool> decltype(auto) r = dom.try_lock();
+ assert(r);
+
+ auto t = support::make_test_jthread([&] {
+ std::rcu_synchronize();
+ flag = true;
+ });
+
+ std::this_thread::sleep_for(std::chrono::milliseconds(10));
+ // flag should not be set because rcu_synchronize should block after lock
+ assert(flag.load() == false);
+ dom.unlock(); // should unblock rcu_synchronize
+ }
+
+ return 0;
+}
diff --git a/libcxx/test/std/thread/saferecl/saferecl.rcu/saferecl.rcu.domain/rcu_default_domain.pass.cpp b/libcxx/test/std/thread/saferecl/saferecl.rcu/saferecl.rcu.domain/rcu_default_domain.pass.cpp
new file mode 100644
index 0000000000000..12370fc1b4e75
--- /dev/null
+++ b/libcxx/test/std/thread/saferecl/saferecl.rcu/saferecl.rcu.domain/rcu_default_domain.pass.cpp
@@ -0,0 +1,38 @@
+//===----------------------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+//
+// UNSUPPORTED: no-threads
+// REQUIRES: std-at-least-c++26
+
+// rcu_domain& rcu_default_domain() noexcept;
+
+#include <atomic>
+#include <cassert>
+#include <chrono>
+#include <cstddef>
+#include <latch>
+#include <rcu>
+#include <thread>
+#include <utility>
+
+#include "make_test_thread.h"
+#include "test_macros.h"
+
+static_assert(noexcept(std::rcu_default_domain()));
+
+int main(int, char**) {
+ {
+ std::same_as<std::rcu_domain&> decltype(auto) dom1 = std::rcu_default_domain();
+ std::same_as<std::rcu_domain&> decltype(auto) dom2 = std::rcu_default_domain();
+ assert(&dom1 == &dom2);
+ dom1.lock();
+ dom1.unlock();
+ }
+
+ return 0;
+}
diff --git a/libcxx/test/std/thread/saferecl/saferecl.rcu/saferecl.rcu.domain/rcu_retire.pass.cpp b/libcxx/test/std/thread/saferecl/saferecl.rcu/saferecl.rcu.domain/rcu_retire.pass.cpp
new file mode 100644
index 0000000000000..0395a879021ae
--- /dev/null
+++ b/libcxx/test/std/thread/saferecl/saferecl.rcu/saferecl.rcu.domain/rcu_retire.pass.cpp
@@ -0,0 +1,82 @@
+//===----------------------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+//
+// UNSUPPORTED: no-threads
+// REQUIRES: std-at-least-c++26
+
+// template<class T, class D = default_delete<T>>
+// void rcu_retire(T* p, D d = D(), rcu_domain& dom = rcu_default_domain());
+
+#include <cassert>
+#include <rcu>
+#include <thread>
+
+#include "make_test_thread.h"
+#include "test_macros.h"
+
+class TestClass1 {
+ int& destruction_count_;
+
+public:
+ TestClass1(int& c) : destruction_count_(c) {}
+ ~TestClass1() { ++destruction_count_; }
+};
+
+struct Deleter {
+ int i = 0;
+ template <class T>
+ void operator()(T* ptr) const {
+ ptr->i_ = i;
+ delete ptr;
+ }
+};
+
+struct CustomDeleter {
+ int& i_;
+ int& destruction_count_;
+ CustomDeleter(int& i, int& destruction_count) : i_(i), destruction_count_(destruction_count) {}
+ ~CustomDeleter() { ++destruction_count_; }
+};
+
+int main(int, char**) {
+ {
+ // retire();
+
+ int destruction_count = 0;
+ auto* ptr = new TestClass1(destruction_count);
+ std::rcu_retire(ptr);
+ std::rcu_barrier();
+ assert(destruction_count == 1);
+ }
+ {
+ // retire(deleter);
+ int i = 0;
+ int destruction_count = 0;
+ auto* ptr = new CustomDeleter(i, destruction_count);
+ std::rcu_retire(ptr, Deleter{42});
+ std::rcu_barrier();
+ assert(destruction_count == 1);
+ assert(i == 42);
+ }
+
+ {
+ // retire(deleter, domain);
+ int i = 0;
+ int destruction_count = 0;
+ auto* ptr = new CustomDeleter(i, destruction_count);
+ std::rcu_retire(ptr, Deleter{42}, std::rcu_default_domain());
+ std::rcu_barrier();
+ assert(destruction_count == 1);
+ assert(i == 42);
+ }
+ {
+ // todo multi threaded test
+ }
+
+ return 0;
+}
>From edc5c19e7440e774fe4ec991a05b341a34ebeed0 Mon Sep 17 00:00:00 2001
From: Hui Xie <hui.xie1990 at gmail.com>
Date: Thu, 10 Sep 2026 17:21:35 +0100
Subject: [PATCH 39/65] retire stress test + fix race on draining queue
---
libcxx/src/experimental/rcu.cpp | 7 -
.../read_write_lock_compare.pass.cpp | 4 +-
...d_write_lock_compare_multi_writer.pass.cpp | 4 +-
.../saferecl.rcu.base/retire.pass.cpp | 134 ++++++++++++++++-
.../saferecl.rcu.base/retire_stress.pass.cpp | 92 ++++++++++++
.../saferecl.rcu.domain/rcu_retire.pass.cpp | 140 +++++++++++++++++-
.../rcu_retire_stress.pass.cpp | 93 ++++++++++++
7 files changed, 458 insertions(+), 16 deletions(-)
create mode 100644 libcxx/test/std/thread/saferecl/saferecl.rcu/saferecl.rcu.base/retire_stress.pass.cpp
create mode 100644 libcxx/test/std/thread/saferecl/saferecl.rcu/saferecl.rcu.domain/rcu_retire_stress.pass.cpp
diff --git a/libcxx/src/experimental/rcu.cpp b/libcxx/src/experimental/rcu.cpp
index 6f6026329c504..e7c29f6fe57ea 100644
--- a/libcxx/src/experimental/rcu.cpp
+++ b/libcxx/src/experimental/rcu.cpp
@@ -156,23 +156,16 @@ class rcu_domain_impl {
if (invoke_callback) {
rcu_singly_list_view ready_callbacks;
ready_callbacks.splice_back(retired_queue_stage2_);
-
- // Invoke the ready callbacks outside of the grace period mutex
- lk.unlock();
ready_callbacks.for_each([](auto* node) { node->__callback_(); });
- lk.lock();
}
std::atomic_signal_fence(memory_order_seq_cst);
update_phase_and_wait();
- lk.unlock();
if (invoke_callback) {
rcu_singly_list_view ready_callbacks;
ready_callbacks.splice_back(retired_queue_stage2_);
-
- // Invoke the ready callbacks outside of the grace period mutex
ready_callbacks.for_each([](auto* node) { node->__callback_(); });
}
std::atomic_thread_fence(memory_order_seq_cst);
diff --git a/libcxx/test/std/thread/saferecl/saferecl.rcu/read_write_lock_compare.pass.cpp b/libcxx/test/std/thread/saferecl/saferecl.rcu/read_write_lock_compare.pass.cpp
index f96a89ddd165a..fa7e134ee8043 100644
--- a/libcxx/test/std/thread/saferecl/saferecl.rcu/read_write_lock_compare.pass.cpp
+++ b/libcxx/test/std/thread/saferecl/saferecl.rcu/read_write_lock_compare.pass.cpp
@@ -110,7 +110,7 @@ void test_rcu() {
int read_count = 0;
while (!token.stop_requested()) {
dom.lock();
- auto obj = global_obj_rcu.load(std::memory_order_relaxed);
+ auto obj = global_obj_rcu.load();
obj->doWork();
dom.unlock();
++read_count;
@@ -122,7 +122,7 @@ void test_rcu() {
int write_count = 0;
while (!token.stop_requested()) {
auto newObj = new MyObject(destruction_count);
- auto oldObj = global_obj_rcu.exchange(newObj, std::memory_order_relaxed);
+ auto oldObj = global_obj_rcu.exchange(newObj);
oldObj->retire();
++write_count;
std::this_thread::sleep_for(std::chrono::microseconds(100));
diff --git a/libcxx/test/std/thread/saferecl/saferecl.rcu/read_write_lock_compare_multi_writer.pass.cpp b/libcxx/test/std/thread/saferecl/saferecl.rcu/read_write_lock_compare_multi_writer.pass.cpp
index 98a2d5d42b8ad..ac9799439c8d8 100644
--- a/libcxx/test/std/thread/saferecl/saferecl.rcu/read_write_lock_compare_multi_writer.pass.cpp
+++ b/libcxx/test/std/thread/saferecl/saferecl.rcu/read_write_lock_compare_multi_writer.pass.cpp
@@ -113,7 +113,7 @@ void test_rcu() {
int read_count = 0;
while (!token.stop_requested()) {
dom.lock();
- auto obj = global_obj_rcu.load(std::memory_order_relaxed);
+ auto obj = global_obj_rcu.load();
obj->doWork();
dom.unlock();
++read_count;
@@ -125,7 +125,7 @@ void test_rcu() {
int write_count = 0;
while (!token.stop_requested()) {
auto newObj = new MyObject();
- auto oldObj = global_obj_rcu.exchange(newObj, std::memory_order_relaxed);
+ auto oldObj = global_obj_rcu.exchange(newObj);
oldObj->retire();
++write_count;
std::this_thread::sleep_for(std::chrono::microseconds(100));
diff --git a/libcxx/test/std/thread/saferecl/saferecl.rcu/saferecl.rcu.base/retire.pass.cpp b/libcxx/test/std/thread/saferecl/saferecl.rcu/saferecl.rcu.base/retire.pass.cpp
index 1dd17cda6cadd..5b27073b48c8d 100644
--- a/libcxx/test/std/thread/saferecl/saferecl.rcu/saferecl.rcu.base/retire.pass.cpp
+++ b/libcxx/test/std/thread/saferecl/saferecl.rcu/saferecl.rcu.base/retire.pass.cpp
@@ -11,9 +11,12 @@
// void retire(D d = D(), rcu_domain& dom = rcu_default_domain()) noexcept;
+#include <atomic>
#include <cassert>
+#include <latch>
#include <rcu>
#include <thread>
+#include <vector>
#include "make_test_thread.h"
#include "test_macros.h"
@@ -86,7 +89,136 @@ int main(int, char**) {
assert(i == 42);
}
{
- // todo: multi threaded test
+ // different thread
+ std::latch unlock_latch(1);
+
+ auto& dom = std::rcu_default_domain();
+ dom.lock();
+
+ auto t = support::make_test_jthread([&] {
+ int destruction_count = 0;
+ auto* ptr = new TestClass1(destruction_count);
+ ptr->retire();
+ std::this_thread::sleep_for(std::chrono::milliseconds(10));
+
+ // should not be destroyed yet as the main thread is still holding the lock
+ assert(destruction_count == 0);
+
+ unlock_latch.count_down();
+
+ std::rcu_barrier();
+ assert(destruction_count == 1);
+ });
+
+ unlock_latch.wait();
+
+ std::this_thread::sleep_for(std::chrono::milliseconds(10));
+ dom.unlock();
+ }
+ {
+ // multiple reader threads
+ constexpr int num_readers = 4;
+ std::latch unlock_latch(1);
+ std::atomic<int> reader_locked = 0;
+
+ std::vector<std::jthread> reader_threads;
+ for (int i = 0; i < num_readers; ++i) {
+ reader_threads.emplace_back([&] {
+ auto& dom = std::rcu_default_domain();
+ dom.lock();
+ ++reader_locked;
+ unlock_latch.wait();
+
+ std::this_thread::sleep_for(std::chrono::milliseconds(10));
+ dom.unlock();
+ });
+ }
+
+ auto writer_thread = support::make_test_jthread([&] {
+ int destruction_count = 0;
+ auto* ptr = new TestClass1(destruction_count);
+ ptr->retire();
+ while (reader_locked.load() < 1) {
+ std::this_thread::sleep_for(std::chrono::milliseconds(10));
+ }
+
+ // should not be destroyed yet as the main thread is still holding the lock
+ assert(destruction_count == 0);
+
+ unlock_latch.count_down();
+
+ std::rcu_barrier();
+ assert(destruction_count == 1);
+ });
+ }
+ {
+ // multiple writer threads
+ constexpr int num_writers = 4;
+ std::latch unlock_latch(num_writers);
+
+ auto& dom = std::rcu_default_domain();
+ dom.lock();
+
+ std::vector<std::jthread> writer_threads;
+ for (int i = 0; i < num_writers; ++i) {
+ writer_threads.emplace_back([&] {
+ int destruction_count = 0;
+ auto* ptr = new TestClass1(destruction_count);
+ ptr->retire();
+ std::this_thread::sleep_for(std::chrono::milliseconds(10));
+
+ // should not be destroyed yet as the main thread is still holding the lock
+ assert(destruction_count == 0);
+
+ unlock_latch.count_down();
+
+ std::rcu_barrier();
+ assert(destruction_count == 1);
+ });
+ }
+
+ unlock_latch.wait();
+
+ std::this_thread::sleep_for(std::chrono::milliseconds(10));
+ dom.unlock();
+ }
+ {
+ // multiple reader and writer threa constexpr int num_readers = 4;
+ constexpr int num_readers = 4;
+ constexpr int num_writers = 3;
+ std::latch unlock_latch(num_writers);
+ std::atomic<int> reader_locked = 0;
+
+ std::vector<std::jthread> reader_threads;
+ for (int i = 0; i < num_readers; ++i) {
+ reader_threads.emplace_back([&] {
+ auto& dom = std::rcu_default_domain();
+ dom.lock();
+ ++reader_locked;
+ unlock_latch.wait();
+
+ std::this_thread::sleep_for(std::chrono::milliseconds(10));
+ dom.unlock();
+ });
+ };
+
+ std::vector<std::jthread> writer_threads;
+ for (int i = 0; i < num_writers; ++i) {
+ writer_threads.emplace_back([&] {
+ int destruction_count = 0;
+ auto* ptr = new TestClass1(destruction_count);
+ ptr->retire();
+ std::this_thread::sleep_for(std::chrono::milliseconds(10));
+
+ // should not be destroyed yet as the main thread is still holding the lock
+ assert(destruction_count == 0);
+
+ unlock_latch.count_down();
+
+ std::rcu_barrier();
+ assert(destruction_count == 1);
+ });
+ }
}
return 0;
diff --git a/libcxx/test/std/thread/saferecl/saferecl.rcu/saferecl.rcu.base/retire_stress.pass.cpp b/libcxx/test/std/thread/saferecl/saferecl.rcu/saferecl.rcu.base/retire_stress.pass.cpp
new file mode 100644
index 0000000000000..bce1f113cf1d8
--- /dev/null
+++ b/libcxx/test/std/thread/saferecl/saferecl.rcu/saferecl.rcu.base/retire_stress.pass.cpp
@@ -0,0 +1,92 @@
+//===----------------------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+//
+// UNSUPPORTED: no-threads
+// REQUIRES: std-at-least-c++26
+
+// void retire(D d = D(), rcu_domain& dom = rcu_default_domain()) noexcept;
+
+#include <atomic>
+#include <cassert>
+#include <rcu>
+#include <thread>
+#include <vector>
+
+#include "make_test_thread.h"
+#include "test_macros.h"
+
+struct TestClass : std::rcu_obj_base<TestClass> {
+ size_t index_;
+
+ TestClass(size_t i) : index_(i) {}
+ TestClass(const TestClass&) = delete;
+ TestClass(TestClass&&) = delete;
+ TestClass& operator=(const TestClass&) = delete;
+ TestClass& operator=(TestClass&&) = delete;
+ ~TestClass();
+};
+
+struct ObjectStore {
+ static constexpr size_t max_objects = 10000;
+ static inline std::atomic<size_t> next_index = 0;
+ static inline std::vector<std::atomic_bool> object_destroyed = std::vector<std::atomic_bool>(max_objects);
+
+ static TestClass* create() {
+ size_t index = next_index.fetch_add(1, std::memory_order_relaxed);
+ return index < max_objects ? new TestClass(index) : nullptr;
+ }
+
+ static bool is_destroyed(size_t index) {
+ return ObjectStore::object_destroyed[index].load(std::memory_order_relaxed);
+ }
+};
+
+TestClass::~TestClass() {
+ assert(!ObjectStore::is_destroyed(index_));
+ ObjectStore::object_destroyed[index_].store(true, std::memory_order_relaxed);
+}
+
+int main(int, char**) {
+ {
+ // stress test
+ constexpr int num_readers = 4;
+ constexpr int num_writers = 3;
+
+ std::atomic<TestClass*> global_ptr = ObjectStore::create();
+
+ std::vector<std::jthread> reader_threads;
+ for (int i = 0; i < num_readers; ++i) {
+ reader_threads.emplace_back([&](std::stop_token stop_token) {
+ while (!stop_token.stop_requested()) {
+ auto& dom = std::rcu_default_domain();
+ dom.lock();
+ auto ptr = global_ptr.load();
+ assert(!ObjectStore::is_destroyed(ptr->index_));
+ dom.unlock();
+ }
+ });
+ }
+
+ std::vector<std::jthread> writer_threads;
+ for (int i = 0; i < num_writers; ++i) {
+ writer_threads.emplace_back([&]() {
+ TestClass* ptr = ObjectStore::create();
+ for (; ptr != nullptr; ptr = ObjectStore::create()) {
+ auto old = global_ptr.exchange(ptr);
+ auto oldIndex = old->index_;
+ old->retire();
+ std::rcu_barrier();
+
+ assert(ObjectStore::is_destroyed(oldIndex));
+ }
+ });
+ }
+ }
+
+ return 0;
+}
diff --git a/libcxx/test/std/thread/saferecl/saferecl.rcu/saferecl.rcu.domain/rcu_retire.pass.cpp b/libcxx/test/std/thread/saferecl/saferecl.rcu/saferecl.rcu.domain/rcu_retire.pass.cpp
index 0395a879021ae..47490cbb9cc01 100644
--- a/libcxx/test/std/thread/saferecl/saferecl.rcu/saferecl.rcu.domain/rcu_retire.pass.cpp
+++ b/libcxx/test/std/thread/saferecl/saferecl.rcu/saferecl.rcu.domain/rcu_retire.pass.cpp
@@ -12,9 +12,12 @@
// template<class T, class D = default_delete<T>>
// void rcu_retire(T* p, D d = D(), rcu_domain& dom = rcu_default_domain());
+#include <atomic>
#include <cassert>
+#include <latch>
#include <rcu>
#include <thread>
+#include <vector>
#include "make_test_thread.h"
#include "test_macros.h"
@@ -45,7 +48,7 @@ struct CustomDeleter {
int main(int, char**) {
{
- // retire();
+ // rcu_retire();
int destruction_count = 0;
auto* ptr = new TestClass1(destruction_count);
@@ -54,7 +57,7 @@ int main(int, char**) {
assert(destruction_count == 1);
}
{
- // retire(deleter);
+ // rcu_retire(deleter);
int i = 0;
int destruction_count = 0;
auto* ptr = new CustomDeleter(i, destruction_count);
@@ -65,7 +68,7 @@ int main(int, char**) {
}
{
- // retire(deleter, domain);
+ // rcu_retire(deleter, domain);
int i = 0;
int destruction_count = 0;
auto* ptr = new CustomDeleter(i, destruction_count);
@@ -75,7 +78,136 @@ int main(int, char**) {
assert(i == 42);
}
{
- // todo multi threaded test
+ // different thread
+ std::latch unlock_latch(1);
+
+ auto& dom = std::rcu_default_domain();
+ dom.lock();
+
+ auto t = support::make_test_jthread([&] {
+ int destruction_count = 0;
+ auto* ptr = new TestClass1(destruction_count);
+ std::rcu_retire(ptr);
+ std::this_thread::sleep_for(std::chrono::milliseconds(10));
+
+ // should not be destroyed yet as the main thread is still holding the lock
+ assert(destruction_count == 0);
+
+ unlock_latch.count_down();
+
+ std::rcu_barrier();
+ assert(destruction_count == 1);
+ });
+
+ unlock_latch.wait();
+
+ std::this_thread::sleep_for(std::chrono::milliseconds(10));
+ dom.unlock();
+ }
+ {
+ // multiple reader threads
+ constexpr int num_readers = 4;
+ std::latch unlock_latch(1);
+ std::atomic<int> reader_locked = 0;
+
+ std::vector<std::jthread> reader_threads;
+ for (int i = 0; i < num_readers; ++i) {
+ reader_threads.emplace_back([&] {
+ auto& dom = std::rcu_default_domain();
+ dom.lock();
+ ++reader_locked;
+ unlock_latch.wait();
+
+ std::this_thread::sleep_for(std::chrono::milliseconds(10));
+ dom.unlock();
+ });
+ }
+
+ auto writer_thread = support::make_test_jthread([&] {
+ int destruction_count = 0;
+ auto* ptr = new TestClass1(destruction_count);
+ std::rcu_retire(ptr);
+ while (reader_locked.load() < 1) {
+ std::this_thread::sleep_for(std::chrono::milliseconds(10));
+ }
+
+ // should not be destroyed yet as the main thread is still holding the lock
+ assert(destruction_count == 0);
+
+ unlock_latch.count_down();
+
+ std::rcu_barrier();
+ assert(destruction_count == 1);
+ });
+ }
+ {
+ // multiple writer threads
+ constexpr int num_writers = 4;
+ std::latch unlock_latch(num_writers);
+
+ auto& dom = std::rcu_default_domain();
+ dom.lock();
+
+ std::vector<std::jthread> writer_threads;
+ for (int i = 0; i < num_writers; ++i) {
+ writer_threads.emplace_back([&] {
+ int destruction_count = 0;
+ auto* ptr = new TestClass1(destruction_count);
+ std::rcu_retire(ptr);
+ std::this_thread::sleep_for(std::chrono::milliseconds(10));
+
+ // should not be destroyed yet as the main thread is still holding the lock
+ assert(destruction_count == 0);
+
+ unlock_latch.count_down();
+
+ std::rcu_barrier();
+ assert(destruction_count == 1);
+ });
+ }
+
+ unlock_latch.wait();
+
+ std::this_thread::sleep_for(std::chrono::milliseconds(10));
+ dom.unlock();
+ }
+ {
+ // multiple reader and writer threa constexpr int num_readers = 4;
+ constexpr int num_readers = 4;
+ constexpr int num_writers = 3;
+ std::latch unlock_latch(num_writers);
+ std::atomic<int> reader_locked = 0;
+
+ std::vector<std::jthread> reader_threads;
+ for (int i = 0; i < num_readers; ++i) {
+ reader_threads.emplace_back([&] {
+ auto& dom = std::rcu_default_domain();
+ dom.lock();
+ ++reader_locked;
+ unlock_latch.wait();
+
+ std::this_thread::sleep_for(std::chrono::milliseconds(10));
+ dom.unlock();
+ });
+ };
+
+ std::vector<std::jthread> writer_threads;
+ for (int i = 0; i < num_writers; ++i) {
+ writer_threads.emplace_back([&] {
+ int destruction_count = 0;
+ auto* ptr = new TestClass1(destruction_count);
+ std::rcu_retire(ptr);
+ std::this_thread::sleep_for(std::chrono::milliseconds(10));
+
+ // should not be destroyed yet as the main thread is still holding the lock
+ assert(destruction_count == 0);
+
+ unlock_latch.count_down();
+
+ std::rcu_barrier();
+ assert(destruction_count == 1);
+ });
+ }
}
return 0;
diff --git a/libcxx/test/std/thread/saferecl/saferecl.rcu/saferecl.rcu.domain/rcu_retire_stress.pass.cpp b/libcxx/test/std/thread/saferecl/saferecl.rcu/saferecl.rcu.domain/rcu_retire_stress.pass.cpp
new file mode 100644
index 0000000000000..c4053bd0bf3be
--- /dev/null
+++ b/libcxx/test/std/thread/saferecl/saferecl.rcu/saferecl.rcu.domain/rcu_retire_stress.pass.cpp
@@ -0,0 +1,93 @@
+//===----------------------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+//
+// UNSUPPORTED: no-threads
+// REQUIRES: std-at-least-c++26
+
+// template<class T, class D = default_delete<T>>
+// void rcu_retire(T* p, D d = D(), rcu_domain& dom = rcu_default_domain());
+
+#include <atomic>
+#include <cassert>
+#include <rcu>
+#include <thread>
+#include <vector>
+
+#include "make_test_thread.h"
+#include "test_macros.h"
+
+struct TestClass {
+ size_t index_;
+
+ TestClass(size_t i) : index_(i) {}
+ TestClass(const TestClass&) = delete;
+ TestClass(TestClass&&) = delete;
+ TestClass& operator=(const TestClass&) = delete;
+ TestClass& operator=(TestClass&&) = delete;
+ ~TestClass();
+};
+
+struct ObjectStore {
+ static constexpr size_t max_objects = 10000;
+ static inline std::atomic<size_t> next_index = 0;
+ static inline std::vector<std::atomic_bool> object_destroyed = std::vector<std::atomic_bool>(max_objects);
+
+ static TestClass* create() {
+ size_t index = next_index.fetch_add(1, std::memory_order_relaxed);
+ return index < max_objects ? new TestClass(index) : nullptr;
+ }
+
+ static bool is_destroyed(size_t index) {
+ return ObjectStore::object_destroyed[index].load(std::memory_order_relaxed);
+ }
+};
+
+TestClass::~TestClass() {
+ assert(!ObjectStore::is_destroyed(index_));
+ ObjectStore::object_destroyed[index_].store(true, std::memory_order_relaxed);
+}
+
+int main(int, char**) {
+ {
+ // stress test
+ constexpr int num_readers = 4;
+ constexpr int num_writers = 3;
+
+ std::atomic<TestClass*> global_ptr = ObjectStore::create();
+
+ std::vector<std::jthread> reader_threads;
+ for (int i = 0; i < num_readers; ++i) {
+ reader_threads.emplace_back([&](std::stop_token stop_token) {
+ while (!stop_token.stop_requested()) {
+ auto& dom = std::rcu_default_domain();
+ dom.lock();
+ auto ptr = global_ptr.load();
+ assert(!ObjectStore::is_destroyed(ptr->index_));
+ dom.unlock();
+ }
+ });
+ }
+
+ std::vector<std::jthread> writer_threads;
+ for (int i = 0; i < num_writers; ++i) {
+ writer_threads.emplace_back([&]() {
+ TestClass* ptr = ObjectStore::create();
+ for (; ptr != nullptr; ptr = ObjectStore::create()) {
+ auto old = global_ptr.exchange(ptr);
+ auto oldIndex = old->index_;
+ std::rcu_retire(old);
+ std::rcu_barrier();
+
+ assert(ObjectStore::is_destroyed(oldIndex));
+ }
+ });
+ }
+ }
+
+ return 0;
+}
>From 8bbc265aed55b441fd095858c81ac9a794e663a7 Mon Sep 17 00:00:00 2001
From: Hui Xie <hui.xie1990 at gmail.com>
Date: Thu, 10 Sep 2026 17:32:03 +0100
Subject: [PATCH 40/65] use raw funciton pointer
---
libcxx/include/__rcu/rcu_domain.h | 15 ++++++++-------
libcxx/include/__rcu/rcu_obj_base.h | 8 +++++---
libcxx/src/experimental/rcu.cpp | 4 ++--
3 files changed, 15 insertions(+), 12 deletions(-)
diff --git a/libcxx/include/__rcu/rcu_domain.h b/libcxx/include/__rcu/rcu_domain.h
index fb542967129de..ee966775792b1 100644
--- a/libcxx/include/__rcu/rcu_domain.h
+++ b/libcxx/include/__rcu/rcu_domain.h
@@ -11,7 +11,6 @@
#define _LIBCPP___RCU_RCU_DOMAIN_H
#include <__config>
-#include <__functional/function_ref.h>
#include <__memory/unique_ptr.h>
#include <__type_traits/is_constructible.h>
#include <__utility/move.h>
@@ -26,8 +25,9 @@ _LIBCPP_BEGIN_EXPLICIT_ABI_ANNOTATIONS
#if _LIBCPP_STD_VER >= 26 && _LIBCPP_HAS_THREADS && _LIBCPP_HAS_EXPERIMENTAL_RCU
struct __rcu_node {
- function_ref<void()> __callback_ = std::cw<[] {}>;
- __rcu_node* __next_ = nullptr;
+ using __cb_type = void(__rcu_node*);
+ __cb_type* __callback_ = [](__rcu_node*) {};
+ __rcu_node* __next_ = nullptr;
};
template <class _Tp, class _Deleter>
@@ -36,12 +36,13 @@ struct __rcu_node_with_deleter : __rcu_node {
_LIBCPP_NO_UNIQUE_ADDRESS _Deleter __deleter_ = _Deleter();
_LIBCPP_HIDE_FROM_ABI __rcu_node_with_deleter(_Tp* __obj, _Deleter __deleter) : __obj_(__obj), __deleter_(__deleter) {
- __callback_ = function_ref<void()>(std::cw<&__rcu_node_with_deleter::__destroy>, this);
+ __callback_ = &__rcu_node_with_deleter::__destroy;
}
- _LIBCPP_HIDE_FROM_ABI void __destroy() const {
- __deleter_(__obj_);
- delete this;
+ _LIBCPP_HIDE_FROM_ABI static void __destroy(__rcu_node* __node) {
+ auto __self = static_cast<__rcu_node_with_deleter*>(__node);
+ __self->__deleter_(__self->__obj_);
+ delete __self;
}
};
diff --git a/libcxx/include/__rcu/rcu_obj_base.h b/libcxx/include/__rcu/rcu_obj_base.h
index d75b90ade4eb7..056f72628a9a5 100644
--- a/libcxx/include/__rcu/rcu_obj_base.h
+++ b/libcxx/include/__rcu/rcu_obj_base.h
@@ -11,7 +11,6 @@
#define _LIBCPP___RCU_RCU_OBJ_BASE_H
#include <__config>
-#include <__functional/function_ref.h>
#include <__memory/unique_ptr.h> // for default_delete
#include <__rcu/rcu_domain.h>
#include <__type_traits/is_assignable.h>
@@ -36,7 +35,7 @@ class rcu_obj_base : private __rcu_node {
_LIBCPP_HIDE_FROM_ABI void retire(_Dp __deleter = _Dp(), rcu_domain& __dom = rcu_default_domain()) noexcept {
static_assert(std::is_base_of_v<rcu_obj_base, _Tp>, "T must be an rcu-protectable type.");
__deleter_ = std::move(__deleter);
- __callback_ = function_ref<void()>(std::cw<&rcu_obj_base::__destroy>, this);
+ __callback_ = &rcu_obj_base::__destroy;
__dom.__retire(this);
}
@@ -49,7 +48,10 @@ class rcu_obj_base : private __rcu_node {
_LIBCPP_HIDE_FROM_ABI ~rcu_obj_base() = default;
private:
- _LIBCPP_HIDE_FROM_ABI void __destroy() { __deleter_(static_cast<_Tp*>(this)); }
+ _LIBCPP_HIDE_FROM_ABI static void __destroy(__rcu_node* __node) {
+ auto __self = static_cast<_Tp*>(__node);
+ __self->__deleter_(__self);
+ }
_LIBCPP_NO_UNIQUE_ADDRESS _Dp __deleter_ = _Dp();
};
diff --git a/libcxx/src/experimental/rcu.cpp b/libcxx/src/experimental/rcu.cpp
index e7c29f6fe57ea..d8a78db7cdd75 100644
--- a/libcxx/src/experimental/rcu.cpp
+++ b/libcxx/src/experimental/rcu.cpp
@@ -156,7 +156,7 @@ class rcu_domain_impl {
if (invoke_callback) {
rcu_singly_list_view ready_callbacks;
ready_callbacks.splice_back(retired_queue_stage2_);
- ready_callbacks.for_each([](auto* node) { node->__callback_(); });
+ ready_callbacks.for_each([](auto* node) { node->__callback_(node); });
}
std::atomic_signal_fence(memory_order_seq_cst);
@@ -166,7 +166,7 @@ class rcu_domain_impl {
if (invoke_callback) {
rcu_singly_list_view ready_callbacks;
ready_callbacks.splice_back(retired_queue_stage2_);
- ready_callbacks.for_each([](auto* node) { node->__callback_(); });
+ ready_callbacks.for_each([](auto* node) { node->__callback_(node); });
}
std::atomic_thread_fence(memory_order_seq_cst);
}
>From 85fd361f36bc65fe7572254dc829452e32c66b5e Mon Sep 17 00:00:00 2001
From: Hui <hui.xie1990 at gmail.com>
Date: Sat, 12 Sep 2026 10:19:00 +0100
Subject: [PATCH 41/65] Update libcxx/docs/DesignDocs/RCU.rst
Co-authored-by: Louis Dionne <ldionne.2 at gmail.com>
---
libcxx/docs/DesignDocs/RCU.rst | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/libcxx/docs/DesignDocs/RCU.rst b/libcxx/docs/DesignDocs/RCU.rst
index f24b5c7659235..c4453c41d0f11 100644
--- a/libcxx/docs/DesignDocs/RCU.rst
+++ b/libcxx/docs/DesignDocs/RCU.rst
@@ -80,7 +80,7 @@ There are several key properties that an implementation of `rcu` must satisfy:
- On the writer side, `retire`-ing an object in principle should not block at all. However, it is technically conforming to block (e.g. evaluate the `deleter`s directly inside `rcu_retire`).
- On the collector side, `rcu_synchronize` should block until at least all the existing readers exit their critical sections
- via `rcu_domain::unlock` . Note that this is a key difference between `rcu` and read-write locks:
+ via `rcu_domain::unlock`. Note that this is a key difference between `rcu` and read-write locks:
In case a late reader enters the critical section after the collector thread has called `rcu_synchronize` and started waiting, the
collector thread does not need to wait for the late reader to exit the critical section. In contrast, a collector thread trying to acquire a read-write lock would have to wait until the late reader releases the lock before it can acquire it.
>From 80af9a1180562e6eaaa435e5d09d4b53050b6ef7 Mon Sep 17 00:00:00 2001
From: Hui <hui.xie1990 at gmail.com>
Date: Sat, 12 Sep 2026 10:19:23 +0100
Subject: [PATCH 42/65] Update libcxx/docs/DesignDocs/RCU.rst
Co-authored-by: Louis Dionne <ldionne.2 at gmail.com>
---
libcxx/docs/DesignDocs/RCU.rst | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/libcxx/docs/DesignDocs/RCU.rst b/libcxx/docs/DesignDocs/RCU.rst
index c4453c41d0f11..13491f94b4dc5 100644
--- a/libcxx/docs/DesignDocs/RCU.rst
+++ b/libcxx/docs/DesignDocs/RCU.rst
@@ -86,7 +86,7 @@ There are several key properties that an implementation of `rcu` must satisfy:
- On the collector side, `rcu_barrier` should block until all the objects `retired` before the `rcu_barrier` call have been reclaimed.
-- The threads that are using `rcu` must be known by the `rcu` implementation states.
+- The threads that are participating in `rcu` must be known by the `rcu` implementation.
Adopted Design
>From c558bf8baa7b999b338fe27b91c9d477423debc7 Mon Sep 17 00:00:00 2001
From: Hui <hui.xie1990 at gmail.com>
Date: Sat, 12 Sep 2026 10:19:34 +0100
Subject: [PATCH 43/65] Update libcxx/docs/DesignDocs/RCU.rst
Co-authored-by: Louis Dionne <ldionne.2 at gmail.com>
---
libcxx/docs/DesignDocs/RCU.rst | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/libcxx/docs/DesignDocs/RCU.rst b/libcxx/docs/DesignDocs/RCU.rst
index 13491f94b4dc5..8b70128bf66f1 100644
--- a/libcxx/docs/DesignDocs/RCU.rst
+++ b/libcxx/docs/DesignDocs/RCU.rst
@@ -147,7 +147,7 @@ the contention on the `mutex` should be low.
TODO: We need to replace `mutex` as all non allocating APIs in `rcu` are designed to be `noexcept` and `mutex` can throw.
-`reader_states`
+`struct reader_states`
~~~~~~~~~~~~~~~
It defines the state of each reader thread. The state is essentially a pair of `(phase, lock_nested_level)` merged into a single `uint16_t` .
>From bdbf75b78b9c956fc265bea0ef3921fcdc30307f Mon Sep 17 00:00:00 2001
From: Hui <hui.xie1990 at gmail.com>
Date: Sat, 12 Sep 2026 10:19:47 +0100
Subject: [PATCH 44/65] Update libcxx/docs/DesignDocs/RCU.rst
Co-authored-by: Louis Dionne <ldionne.2 at gmail.com>
---
libcxx/docs/DesignDocs/RCU.rst | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/libcxx/docs/DesignDocs/RCU.rst b/libcxx/docs/DesignDocs/RCU.rst
index 8b70128bf66f1..24c55a09ac10e 100644
--- a/libcxx/docs/DesignDocs/RCU.rst
+++ b/libcxx/docs/DesignDocs/RCU.rst
@@ -109,7 +109,7 @@ Libc++ adopted a design that closely resembles the "General-Purpose RCU" design
Some key details of this design are:
-- There is a global state which has two phases and it flips between the two phases.
+- There is a global state which has two phases and flips between the two.
- Each thread stores its state: whether there is a reader in the critical section, and which phase it was when it entered the critical section.
- When `rcu_synchronize`/`rcu_barrier` is called, it
>From 71ae98ca3f92d1a899477b09d9311e24abbb97b3 Mon Sep 17 00:00:00 2001
From: Hui <hui.xie1990 at gmail.com>
Date: Sat, 12 Sep 2026 10:20:15 +0100
Subject: [PATCH 45/65] Update libcxx/docs/DesignDocs/RCU.rst
Co-authored-by: Louis Dionne <ldionne.2 at gmail.com>
---
libcxx/docs/DesignDocs/RCU.rst | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/libcxx/docs/DesignDocs/RCU.rst b/libcxx/docs/DesignDocs/RCU.rst
index 24c55a09ac10e..1efef8be818c0 100644
--- a/libcxx/docs/DesignDocs/RCU.rst
+++ b/libcxx/docs/DesignDocs/RCU.rst
@@ -110,7 +110,7 @@ Libc++ adopted a design that closely resembles the "General-Purpose RCU" design
Some key details of this design are:
- There is a global state which has two phases and flips between the two.
-- Each thread stores its state: whether there is a reader in the critical section, and which phase it was when it entered the critical section.
+- Each thread stores its state: whether there is a reader in the critical section, and which phase was active when it entered the critical section.
- When `rcu_synchronize`/`rcu_barrier` is called, it
- flips the global state to the next phase
>From acd5796998f94ce12a89219bd45dd5ec2b1672fe Mon Sep 17 00:00:00 2001
From: Hui <hui.xie1990 at gmail.com>
Date: Sat, 12 Sep 2026 10:20:30 +0100
Subject: [PATCH 46/65] Update libcxx/docs/DesignDocs/RCU.rst
Co-authored-by: Louis Dionne <ldionne.2 at gmail.com>
---
libcxx/docs/DesignDocs/RCU.rst | 10 +++++-----
1 file changed, 5 insertions(+), 5 deletions(-)
diff --git a/libcxx/docs/DesignDocs/RCU.rst b/libcxx/docs/DesignDocs/RCU.rst
index 1efef8be818c0..59e726eabb4c1 100644
--- a/libcxx/docs/DesignDocs/RCU.rst
+++ b/libcxx/docs/DesignDocs/RCU.rst
@@ -111,12 +111,12 @@ Some key details of this design are:
- There is a global state which has two phases and flips between the two.
- Each thread stores its state: whether there is a reader in the critical section, and which phase was active when it entered the critical section.
-- When `rcu_synchronize`/`rcu_barrier` is called, it
+- Assume the two phases are `a` and `b`, and the current phase is `a`. When `rcu_synchronize`/`rcu_barrier` is called, it
- - flips the global state to the next phase
- - Going through a grace period: waits until all the threads that are in the critical section with the previous phase to exit the critical section.
- - flips the global state back to the original phase
- - Going through another grace period: waits until all the threads that are in the critical section with the next phase to exit the critical section.
+ - flips the global state to `b`
+ - goes through a grace period: waits until all the threads that are in the critical section with phase `a` have exited their critical section
+ - flips the global state back to phase `a`
+ - goes through another grace period: waits until all the threads that are in the critical section with phase `b` have exited their critical section
When `rcu_synchronize`/`rcu_barrier` returns, we can be sure that all the readers that were in the critical section before `rcu_synchronize`/`rcu_barrier` are now out of the critical section.
The paper explains why we need to wait two phases instead of just one phase in detail. The key point is that, if we only wait for the readers in the previous phase to exit,
>From 642a3c3d30e7c1152a39fb2ef092f5a1520cfdef Mon Sep 17 00:00:00 2001
From: Hui <hui.xie1990 at gmail.com>
Date: Sat, 12 Sep 2026 10:20:45 +0100
Subject: [PATCH 47/65] Update libcxx/docs/DesignDocs/RCU.rst
Co-authored-by: Louis Dionne <ldionne.2 at gmail.com>
---
libcxx/docs/DesignDocs/RCU.rst | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/libcxx/docs/DesignDocs/RCU.rst b/libcxx/docs/DesignDocs/RCU.rst
index 59e726eabb4c1..2744104f94ef9 100644
--- a/libcxx/docs/DesignDocs/RCU.rst
+++ b/libcxx/docs/DesignDocs/RCU.rst
@@ -118,7 +118,7 @@ Some key details of this design are:
- flips the global state back to phase `a`
- goes through another grace period: waits until all the threads that are in the critical section with phase `b` have exited their critical section
-When `rcu_synchronize`/`rcu_barrier` returns, we can be sure that all the readers that were in the critical section before `rcu_synchronize`/`rcu_barrier` are now out of the critical section.
+When `rcu_synchronize`/`rcu_barrier` returns, we can be sure that all the readers that were in the critical section before the call are now out of the critical section.
The paper explains why we need to wait two phases instead of just one phase in detail. The key point is that, if we only wait for the readers in the previous phase to exit,
there might be a late reader that enters the critical section after we flip the global state and before we wait for the previous phase's readers to exit.
>From fb416b3658246751824978d9e58519b4f9f0a3e7 Mon Sep 17 00:00:00 2001
From: Hui <hui.xie1990 at gmail.com>
Date: Sat, 12 Sep 2026 10:21:01 +0100
Subject: [PATCH 48/65] Update libcxx/docs/DesignDocs/RCU.rst
Co-authored-by: Louis Dionne <ldionne.2 at gmail.com>
---
libcxx/docs/DesignDocs/RCU.rst | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/libcxx/docs/DesignDocs/RCU.rst b/libcxx/docs/DesignDocs/RCU.rst
index 2744104f94ef9..4104498659a2e 100644
--- a/libcxx/docs/DesignDocs/RCU.rst
+++ b/libcxx/docs/DesignDocs/RCU.rst
@@ -136,7 +136,7 @@ Implementation Terminology
Implementation Details
----------------------
-`thread_local_container`
+`class thread_local_container`
~~~~~~~~~~~~~~~~~~~~~~~~
A helper class that manages thread local objects. It provides APIs to get the `thread_local` object for the current thread, and to iterate through all the `thread_local` objects from all threads.
>From 44f3aa08bd1e4be6f04ba5f8c0eadade271f8b92 Mon Sep 17 00:00:00 2001
From: Hui Xie <hui.xie1990 at gmail.com>
Date: Sat, 12 Sep 2026 18:20:02 +0100
Subject: [PATCH 49/65] update test
---
.../saferecl.rcu/read_write_lock_compare.pass.cpp | 8 ++++----
.../read_write_lock_compare_multi_writer.pass.cpp | 10 +++++-----
2 files changed, 9 insertions(+), 9 deletions(-)
diff --git a/libcxx/test/std/thread/saferecl/saferecl.rcu/read_write_lock_compare.pass.cpp b/libcxx/test/std/thread/saferecl/saferecl.rcu/read_write_lock_compare.pass.cpp
index fa7e134ee8043..9c357762a1870 100644
--- a/libcxx/test/std/thread/saferecl/saferecl.rcu/read_write_lock_compare.pass.cpp
+++ b/libcxx/test/std/thread/saferecl/saferecl.rcu/read_write_lock_compare.pass.cpp
@@ -38,13 +38,13 @@ struct alignas(128) MyObject : public std::rcu_obj_base<MyObject> {
~MyObject() {destruction_count_.fetch_add(1, std::memory_order_relaxed);}
void doWork() {
- auto spin_for = [](std::chrono::microseconds us) {
+ [[maybe_unused]]auto spin_for = [](std::chrono::microseconds us) {
auto start = std::chrono::high_resolution_clock::now();
while (std::chrono::high_resolution_clock::now() - start < us)
;
};
using namespace std::chrono_literals;
- spin_for(10us);
+ //spin_for(10us);
}
};
@@ -78,7 +78,7 @@ void test_read_write_lock() {
lock.unlock();
delete oldObj;
++write_count;
- std::this_thread::sleep_for(std::chrono::microseconds(100));
+ // std::this_thread::sleep_for(std::chrono::microseconds(100));
}
std::println("Writer thread wrote {} times", write_count);
};
@@ -125,7 +125,7 @@ void test_rcu() {
auto oldObj = global_obj_rcu.exchange(newObj);
oldObj->retire();
++write_count;
- std::this_thread::sleep_for(std::chrono::microseconds(100));
+ // std::this_thread::sleep_for(std::chrono::microseconds(100));
}
std::println("RCU Writer thread wrote {} times", write_count);
};
diff --git a/libcxx/test/std/thread/saferecl/saferecl.rcu/read_write_lock_compare_multi_writer.pass.cpp b/libcxx/test/std/thread/saferecl/saferecl.rcu/read_write_lock_compare_multi_writer.pass.cpp
index ac9799439c8d8..79aa775e8dddf 100644
--- a/libcxx/test/std/thread/saferecl/saferecl.rcu/read_write_lock_compare_multi_writer.pass.cpp
+++ b/libcxx/test/std/thread/saferecl/saferecl.rcu/read_write_lock_compare_multi_writer.pass.cpp
@@ -35,13 +35,13 @@ struct alignas(128) MyObject : public std::rcu_obj_base<MyObject> {
MyObject() : data_(std::to_string(instance_count++) + " instance very very very long string") {}
void doWork() {
- auto spin_for = [](std::chrono::microseconds us) {
+ [[maybe_unused]] auto spin_for = [](std::chrono::microseconds us) {
auto start = std::chrono::high_resolution_clock::now();
while (std::chrono::high_resolution_clock::now() - start < us)
;
};
using namespace std::chrono_literals;
- spin_for(10us);
+ // spin_for(10us);
}
};
@@ -76,7 +76,7 @@ void test_read_write_lock() {
lock.unlock();
delete oldObj;
++write_count;
- std::this_thread::sleep_for(std::chrono::microseconds(100));
+ // std::this_thread::sleep_for(std::chrono::microseconds(100));
}
std::println("Writer thread wrote {} times", write_count);
};
@@ -128,7 +128,7 @@ void test_rcu() {
auto oldObj = global_obj_rcu.exchange(newObj);
oldObj->retire();
++write_count;
- std::this_thread::sleep_for(std::chrono::microseconds(100));
+ // std::this_thread::sleep_for(std::chrono::microseconds(100));
}
std::println("RCU Writer thread wrote {} times", write_count);
};
@@ -158,7 +158,7 @@ void test_rcu() {
writer.request_stop();
}
syncer.request_stop();
- std::rcu_synchronize(dom);
+ std::rcu_barrier(dom);
}
int main(int, char**) {
>From ffc2ca61476f4beedc9af03532e7f8f4064d6d21 Mon Sep 17 00:00:00 2001
From: Hui Xie <hui.xie1990 at gmail.com>
Date: Sat, 12 Sep 2026 18:37:58 +0100
Subject: [PATCH 50/65] atomic queue
---
.../src/experimental/include/rcu/rcu_list.h | 52 +++++++++----------
libcxx/src/experimental/rcu.cpp | 2 +-
2 files changed, 26 insertions(+), 28 deletions(-)
diff --git a/libcxx/src/experimental/include/rcu/rcu_list.h b/libcxx/src/experimental/include/rcu/rcu_list.h
index 3eb3dc5750795..3d8b4f83d4ecb 100644
--- a/libcxx/src/experimental/include/rcu/rcu_list.h
+++ b/libcxx/src/experimental/include/rcu/rcu_list.h
@@ -14,6 +14,7 @@
#include <__functional/function.h>
#include <__rcu/rcu_domain.h>
#include <atomic>
+#include <type_traits>
#include "thread_local_container.h"
@@ -25,7 +26,7 @@ _LIBCPP_BEGIN_NAMESPACE_STD
#if _LIBCPP_STD_VER >= 26 && _LIBCPP_HAS_THREADS && _LIBCPP_HAS_EXPERIMENTAL_RCU
-class rcu_thread_local_list_view;
+class rcu_atomic_list_view;
class rcu_singly_list_view {
private:
@@ -48,7 +49,7 @@ class rcu_singly_list_view {
other.tail_ = nullptr;
}
- void splice_back(rcu_thread_local_list_view& __other) noexcept;
+ void splice_back(rcu_atomic_list_view& __other) noexcept;
template <class Func>
void for_each(Func&& f) noexcept {
@@ -62,31 +63,32 @@ class rcu_singly_list_view {
}
};
-class rcu_thread_local_list_view {
- struct alignas(2 * sizeof(void*)) thread_entry {
- __rcu_node* head_ = nullptr;
- __rcu_node* tail_ = nullptr;
- };
+struct alignas(2 * sizeof(void*)) rcu_atomic_list_view_entry {
+ __rcu_node* head_ = nullptr;
+ __rcu_node* tail_ = nullptr;
+};
+
+class rcu_atomic_list_view {
+ using entry = rcu_atomic_list_view_entry;
- using per_thread_entries = thread_local_container<thread_entry>;
+ std::atomic<entry> entry_{};
friend class rcu_singly_list_view;
public:
void push_front(__rcu_node* node) noexcept {
- atomic_ref<thread_entry> entry_ref = per_thread_entries::get_current_thread_instance();
- auto expected_entry = entry_ref.load(std::memory_order_relaxed);
- auto original_next = node->__next_;
+ auto expected_entry = entry_.load(std::memory_order_relaxed);
+ auto original_next = node->__next_;
while (true) {
auto new_entry = [&] {
if (expected_entry.head_ == nullptr) {
- return thread_entry{node, node};
+ return entry{node, node};
} else {
node->__next_ = expected_entry.head_;
- return thread_entry{node, expected_entry.tail_};
+ return entry{node, expected_entry.tail_};
}
}();
- if (entry_ref.compare_exchange_weak(
+ if (entry_.compare_exchange_weak(
expected_entry, new_entry, std::memory_order_acq_rel, std::memory_order_relaxed)) {
break;
} else {
@@ -96,19 +98,15 @@ class rcu_thread_local_list_view {
}
};
-void rcu_singly_list_view::splice_back(rcu_thread_local_list_view& __other) noexcept {
- using thread_entry = rcu_thread_local_list_view::thread_entry;
- const auto splice_single_entry = [this](atomic_ref<thread_entry> entry_ref) noexcept {
- if (entry_ref.load(std::memory_order_relaxed).head_ == nullptr) {
- return;
- }
- auto entry = entry_ref.exchange(thread_entry{nullptr, nullptr}, std::memory_order_acq_rel);
- rcu_singly_list_view tmp;
- tmp.head_ = entry.head_;
- tmp.tail_ = entry.tail_;
- this->splice_back(tmp);
- };
- rcu_thread_local_list_view::per_thread_entries::for_each(splice_single_entry);
+void rcu_singly_list_view::splice_back(rcu_atomic_list_view& __other) noexcept {
+ if (__other.entry_.load(std::memory_order_relaxed).head_ == nullptr) {
+ return;
+ }
+ auto entry = __other.entry_.exchange(rcu_atomic_list_view::entry{nullptr, nullptr}, std::memory_order_acq_rel);
+ rcu_singly_list_view tmp;
+ tmp.head_ = entry.head_;
+ tmp.tail_ = entry.tail_;
+ this->splice_back(tmp);
}
#endif // _LIBCPP_STD_VER >= 26 && _LIBCPP_HAS_THREADS && _LIBCPP_HAS_EXPERIMENTAL_RCU
diff --git a/libcxx/src/experimental/rcu.cpp b/libcxx/src/experimental/rcu.cpp
index d8a78db7cdd75..87cdf3c0e863a 100644
--- a/libcxx/src/experimental/rcu.cpp
+++ b/libcxx/src/experimental/rcu.cpp
@@ -66,7 +66,7 @@ class rcu_domain_impl {
// flag used for waking up writer threads waiting for all reader threads' quiescent state
std::atomic<bool> grace_period_waiting_flag_ = false;
- rcu_thread_local_list_view retired_queue_stage0_;
+ rcu_atomic_list_view retired_queue_stage0_;
// these two queues do not need extra synchronization
// as they are always processed under the grace period mutex
>From f9563e82820ff0cc119b4a299b0c67b0976def24 Mon Sep 17 00:00:00 2001
From: Hui Xie <hui.xie1990 at gmail.com>
Date: Sat, 12 Sep 2026 18:52:16 +0100
Subject: [PATCH 51/65] Revert "atomic queue"
This reverts commit 1ccdc65167e4aa78362461c5b91cdccf048e1fbc.
---
.../src/experimental/include/rcu/rcu_list.h | 52 ++++++++++---------
libcxx/src/experimental/rcu.cpp | 2 +-
2 files changed, 28 insertions(+), 26 deletions(-)
diff --git a/libcxx/src/experimental/include/rcu/rcu_list.h b/libcxx/src/experimental/include/rcu/rcu_list.h
index 3d8b4f83d4ecb..3eb3dc5750795 100644
--- a/libcxx/src/experimental/include/rcu/rcu_list.h
+++ b/libcxx/src/experimental/include/rcu/rcu_list.h
@@ -14,7 +14,6 @@
#include <__functional/function.h>
#include <__rcu/rcu_domain.h>
#include <atomic>
-#include <type_traits>
#include "thread_local_container.h"
@@ -26,7 +25,7 @@ _LIBCPP_BEGIN_NAMESPACE_STD
#if _LIBCPP_STD_VER >= 26 && _LIBCPP_HAS_THREADS && _LIBCPP_HAS_EXPERIMENTAL_RCU
-class rcu_atomic_list_view;
+class rcu_thread_local_list_view;
class rcu_singly_list_view {
private:
@@ -49,7 +48,7 @@ class rcu_singly_list_view {
other.tail_ = nullptr;
}
- void splice_back(rcu_atomic_list_view& __other) noexcept;
+ void splice_back(rcu_thread_local_list_view& __other) noexcept;
template <class Func>
void for_each(Func&& f) noexcept {
@@ -63,32 +62,31 @@ class rcu_singly_list_view {
}
};
-struct alignas(2 * sizeof(void*)) rcu_atomic_list_view_entry {
- __rcu_node* head_ = nullptr;
- __rcu_node* tail_ = nullptr;
-};
-
-class rcu_atomic_list_view {
- using entry = rcu_atomic_list_view_entry;
+class rcu_thread_local_list_view {
+ struct alignas(2 * sizeof(void*)) thread_entry {
+ __rcu_node* head_ = nullptr;
+ __rcu_node* tail_ = nullptr;
+ };
- std::atomic<entry> entry_{};
+ using per_thread_entries = thread_local_container<thread_entry>;
friend class rcu_singly_list_view;
public:
void push_front(__rcu_node* node) noexcept {
- auto expected_entry = entry_.load(std::memory_order_relaxed);
- auto original_next = node->__next_;
+ atomic_ref<thread_entry> entry_ref = per_thread_entries::get_current_thread_instance();
+ auto expected_entry = entry_ref.load(std::memory_order_relaxed);
+ auto original_next = node->__next_;
while (true) {
auto new_entry = [&] {
if (expected_entry.head_ == nullptr) {
- return entry{node, node};
+ return thread_entry{node, node};
} else {
node->__next_ = expected_entry.head_;
- return entry{node, expected_entry.tail_};
+ return thread_entry{node, expected_entry.tail_};
}
}();
- if (entry_.compare_exchange_weak(
+ if (entry_ref.compare_exchange_weak(
expected_entry, new_entry, std::memory_order_acq_rel, std::memory_order_relaxed)) {
break;
} else {
@@ -98,15 +96,19 @@ class rcu_atomic_list_view {
}
};
-void rcu_singly_list_view::splice_back(rcu_atomic_list_view& __other) noexcept {
- if (__other.entry_.load(std::memory_order_relaxed).head_ == nullptr) {
- return;
- }
- auto entry = __other.entry_.exchange(rcu_atomic_list_view::entry{nullptr, nullptr}, std::memory_order_acq_rel);
- rcu_singly_list_view tmp;
- tmp.head_ = entry.head_;
- tmp.tail_ = entry.tail_;
- this->splice_back(tmp);
+void rcu_singly_list_view::splice_back(rcu_thread_local_list_view& __other) noexcept {
+ using thread_entry = rcu_thread_local_list_view::thread_entry;
+ const auto splice_single_entry = [this](atomic_ref<thread_entry> entry_ref) noexcept {
+ if (entry_ref.load(std::memory_order_relaxed).head_ == nullptr) {
+ return;
+ }
+ auto entry = entry_ref.exchange(thread_entry{nullptr, nullptr}, std::memory_order_acq_rel);
+ rcu_singly_list_view tmp;
+ tmp.head_ = entry.head_;
+ tmp.tail_ = entry.tail_;
+ this->splice_back(tmp);
+ };
+ rcu_thread_local_list_view::per_thread_entries::for_each(splice_single_entry);
}
#endif // _LIBCPP_STD_VER >= 26 && _LIBCPP_HAS_THREADS && _LIBCPP_HAS_EXPERIMENTAL_RCU
diff --git a/libcxx/src/experimental/rcu.cpp b/libcxx/src/experimental/rcu.cpp
index 87cdf3c0e863a..d8a78db7cdd75 100644
--- a/libcxx/src/experimental/rcu.cpp
+++ b/libcxx/src/experimental/rcu.cpp
@@ -66,7 +66,7 @@ class rcu_domain_impl {
// flag used for waking up writer threads waiting for all reader threads' quiescent state
std::atomic<bool> grace_period_waiting_flag_ = false;
- rcu_atomic_list_view retired_queue_stage0_;
+ rcu_thread_local_list_view retired_queue_stage0_;
// these two queues do not need extra synchronization
// as they are always processed under the grace period mutex
>From 129dc249e5978b0397ee91233ce7a5a4ceeb2867 Mon Sep 17 00:00:00 2001
From: Hui Xie <hui.xie1990 at gmail.com>
Date: Sat, 12 Sep 2026 19:28:32 +0100
Subject: [PATCH 52/65] refactor the thread local queue
---
.../src/experimental/include/rcu/rcu_list.h | 61 +++++++------------
libcxx/src/experimental/rcu.cpp | 12 ++--
2 files changed, 31 insertions(+), 42 deletions(-)
diff --git a/libcxx/src/experimental/include/rcu/rcu_list.h b/libcxx/src/experimental/include/rcu/rcu_list.h
index 3eb3dc5750795..21065e95e9790 100644
--- a/libcxx/src/experimental/include/rcu/rcu_list.h
+++ b/libcxx/src/experimental/include/rcu/rcu_list.h
@@ -25,14 +25,15 @@ _LIBCPP_BEGIN_NAMESPACE_STD
#if _LIBCPP_STD_VER >= 26 && _LIBCPP_HAS_THREADS && _LIBCPP_HAS_EXPERIMENTAL_RCU
-class rcu_thread_local_list_view;
-
class rcu_singly_list_view {
private:
__rcu_node* head_ = nullptr;
__rcu_node* tail_ = nullptr;
public:
+ rcu_singly_list_view() = default;
+ rcu_singly_list_view(__rcu_node* head, __rcu_node* tail) : head_(head), tail_(tail) {}
+
void splice_back(rcu_singly_list_view& other) noexcept {
if (other.head_ == nullptr) {
return;
@@ -48,8 +49,6 @@ class rcu_singly_list_view {
other.tail_ = nullptr;
}
- void splice_back(rcu_thread_local_list_view& __other) noexcept;
-
template <class Func>
void for_each(Func&& f) noexcept {
__rcu_node* current = head_;
@@ -62,54 +61,40 @@ class rcu_singly_list_view {
}
};
-class rcu_thread_local_list_view {
- struct alignas(2 * sizeof(void*)) thread_entry {
- __rcu_node* head_ = nullptr;
- __rcu_node* tail_ = nullptr;
- };
-
- using per_thread_entries = thread_local_container<thread_entry>;
-
- friend class rcu_singly_list_view;
+struct alignas(2 * sizeof(void*)) rcu_atomic_list_view {
+ __rcu_node* head_ = nullptr;
+ __rcu_node* tail_ = nullptr;
-public:
- void push_front(__rcu_node* node) noexcept {
- atomic_ref<thread_entry> entry_ref = per_thread_entries::get_current_thread_instance();
- auto expected_entry = entry_ref.load(std::memory_order_relaxed);
- auto original_next = node->__next_;
+ static void push_front(atomic_ref<rcu_atomic_list_view> view_ref, __rcu_node* node) noexcept {
+ auto expected_view = view_ref.load(std::memory_order_relaxed);
+ auto original_next = node->__next_;
while (true) {
- auto new_entry = [&] {
- if (expected_entry.head_ == nullptr) {
- return thread_entry{node, node};
+ auto new_view = [&] {
+ if (expected_view.head_ == nullptr) {
+ return rcu_atomic_list_view{node, node};
} else {
- node->__next_ = expected_entry.head_;
- return thread_entry{node, expected_entry.tail_};
+ node->__next_ = expected_view.head_;
+ return rcu_atomic_list_view{node, expected_view.tail_};
}
}();
- if (entry_ref.compare_exchange_weak(
- expected_entry, new_entry, std::memory_order_acq_rel, std::memory_order_relaxed)) {
+ if (view_ref.compare_exchange_weak(
+ expected_view, new_view, std::memory_order_acq_rel, std::memory_order_relaxed)) {
break;
} else {
node->__next_ = original_next;
}
}
}
-};
-void rcu_singly_list_view::splice_back(rcu_thread_local_list_view& __other) noexcept {
- using thread_entry = rcu_thread_local_list_view::thread_entry;
- const auto splice_single_entry = [this](atomic_ref<thread_entry> entry_ref) noexcept {
- if (entry_ref.load(std::memory_order_relaxed).head_ == nullptr) {
+ static void splice_back(atomic_ref<rcu_atomic_list_view> from_ref, rcu_singly_list_view& to) noexcept {
+ if (from_ref.load(std::memory_order_relaxed).head_ == nullptr) {
return;
}
- auto entry = entry_ref.exchange(thread_entry{nullptr, nullptr}, std::memory_order_acq_rel);
- rcu_singly_list_view tmp;
- tmp.head_ = entry.head_;
- tmp.tail_ = entry.tail_;
- this->splice_back(tmp);
- };
- rcu_thread_local_list_view::per_thread_entries::for_each(splice_single_entry);
-}
+ auto from = from_ref.exchange(rcu_atomic_list_view{nullptr, nullptr}, std::memory_order_acq_rel);
+ rcu_singly_list_view tmp(from.head_, from.tail_);
+ to.splice_back(tmp);
+ }
+};
#endif // _LIBCPP_STD_VER >= 26 && _LIBCPP_HAS_THREADS && _LIBCPP_HAS_EXPERIMENTAL_RCU
diff --git a/libcxx/src/experimental/rcu.cpp b/libcxx/src/experimental/rcu.cpp
index d8a78db7cdd75..a0f8784bd6ca5 100644
--- a/libcxx/src/experimental/rcu.cpp
+++ b/libcxx/src/experimental/rcu.cpp
@@ -66,7 +66,7 @@ class rcu_domain_impl {
// flag used for waking up writer threads waiting for all reader threads' quiescent state
std::atomic<bool> grace_period_waiting_flag_ = false;
- rcu_thread_local_list_view retired_queue_stage0_;
+ using per_thread_retired_queue_stage0 = thread_local_container<rcu_atomic_list_view>;
// these two queues do not need extra synchronization
// as they are always processed under the grace period mutex
@@ -77,7 +77,9 @@ class rcu_domain_impl {
void update_phase_and_wait() noexcept {
rcu_singly_list_view working_queue;
- working_queue.splice_back(retired_queue_stage0_);
+ per_thread_retired_queue_stage0::for_each([&working_queue](atomic_ref<rcu_atomic_list_view> thread_local_stage0) {
+ rcu_atomic_list_view::splice_back(thread_local_stage0, working_queue);
+ });
// Flip the global phase
auto old_phase = global_reader_phase_.fetch_xor(reader_states::grace_period_phase_mask, std::memory_order_relaxed);
@@ -145,7 +147,10 @@ class rcu_domain_impl {
}
}
- void retire(__rcu_node* node) noexcept { retired_queue_stage0_.push_front(node); }
+ void retire(__rcu_node* node) noexcept {
+ auto thread_local_stage0_ref = per_thread_retired_queue_stage0::get_current_thread_instance();
+ rcu_atomic_list_view::push_front(thread_local_stage0_ref, node);
+ }
void synchronize(bool invoke_callback) noexcept {
std::atomic_thread_fence(memory_order_seq_cst);
@@ -162,7 +167,6 @@ class rcu_domain_impl {
std::atomic_signal_fence(memory_order_seq_cst);
update_phase_and_wait();
-
if (invoke_callback) {
rcu_singly_list_view ready_callbacks;
ready_callbacks.splice_back(retired_queue_stage2_);
>From a2f80b530c5fdcf4d91600512e0c0bf646ddd287 Mon Sep 17 00:00:00 2001
From: Hui Xie <hui.xie1990 at gmail.com>
Date: Sun, 13 Sep 2026 07:59:25 +0100
Subject: [PATCH 53/65] Revert "refactor the thread local queue"
This reverts commit 02610850dd7131dc04427f6f3214a5c07552ed07.
---
.../src/experimental/include/rcu/rcu_list.h | 61 ++++++++++++-------
libcxx/src/experimental/rcu.cpp | 12 ++--
2 files changed, 42 insertions(+), 31 deletions(-)
diff --git a/libcxx/src/experimental/include/rcu/rcu_list.h b/libcxx/src/experimental/include/rcu/rcu_list.h
index 21065e95e9790..3eb3dc5750795 100644
--- a/libcxx/src/experimental/include/rcu/rcu_list.h
+++ b/libcxx/src/experimental/include/rcu/rcu_list.h
@@ -25,15 +25,14 @@ _LIBCPP_BEGIN_NAMESPACE_STD
#if _LIBCPP_STD_VER >= 26 && _LIBCPP_HAS_THREADS && _LIBCPP_HAS_EXPERIMENTAL_RCU
+class rcu_thread_local_list_view;
+
class rcu_singly_list_view {
private:
__rcu_node* head_ = nullptr;
__rcu_node* tail_ = nullptr;
public:
- rcu_singly_list_view() = default;
- rcu_singly_list_view(__rcu_node* head, __rcu_node* tail) : head_(head), tail_(tail) {}
-
void splice_back(rcu_singly_list_view& other) noexcept {
if (other.head_ == nullptr) {
return;
@@ -49,6 +48,8 @@ class rcu_singly_list_view {
other.tail_ = nullptr;
}
+ void splice_back(rcu_thread_local_list_view& __other) noexcept;
+
template <class Func>
void for_each(Func&& f) noexcept {
__rcu_node* current = head_;
@@ -61,40 +62,54 @@ class rcu_singly_list_view {
}
};
-struct alignas(2 * sizeof(void*)) rcu_atomic_list_view {
- __rcu_node* head_ = nullptr;
- __rcu_node* tail_ = nullptr;
+class rcu_thread_local_list_view {
+ struct alignas(2 * sizeof(void*)) thread_entry {
+ __rcu_node* head_ = nullptr;
+ __rcu_node* tail_ = nullptr;
+ };
+
+ using per_thread_entries = thread_local_container<thread_entry>;
- static void push_front(atomic_ref<rcu_atomic_list_view> view_ref, __rcu_node* node) noexcept {
- auto expected_view = view_ref.load(std::memory_order_relaxed);
- auto original_next = node->__next_;
+ friend class rcu_singly_list_view;
+
+public:
+ void push_front(__rcu_node* node) noexcept {
+ atomic_ref<thread_entry> entry_ref = per_thread_entries::get_current_thread_instance();
+ auto expected_entry = entry_ref.load(std::memory_order_relaxed);
+ auto original_next = node->__next_;
while (true) {
- auto new_view = [&] {
- if (expected_view.head_ == nullptr) {
- return rcu_atomic_list_view{node, node};
+ auto new_entry = [&] {
+ if (expected_entry.head_ == nullptr) {
+ return thread_entry{node, node};
} else {
- node->__next_ = expected_view.head_;
- return rcu_atomic_list_view{node, expected_view.tail_};
+ node->__next_ = expected_entry.head_;
+ return thread_entry{node, expected_entry.tail_};
}
}();
- if (view_ref.compare_exchange_weak(
- expected_view, new_view, std::memory_order_acq_rel, std::memory_order_relaxed)) {
+ if (entry_ref.compare_exchange_weak(
+ expected_entry, new_entry, std::memory_order_acq_rel, std::memory_order_relaxed)) {
break;
} else {
node->__next_ = original_next;
}
}
}
+};
- static void splice_back(atomic_ref<rcu_atomic_list_view> from_ref, rcu_singly_list_view& to) noexcept {
- if (from_ref.load(std::memory_order_relaxed).head_ == nullptr) {
+void rcu_singly_list_view::splice_back(rcu_thread_local_list_view& __other) noexcept {
+ using thread_entry = rcu_thread_local_list_view::thread_entry;
+ const auto splice_single_entry = [this](atomic_ref<thread_entry> entry_ref) noexcept {
+ if (entry_ref.load(std::memory_order_relaxed).head_ == nullptr) {
return;
}
- auto from = from_ref.exchange(rcu_atomic_list_view{nullptr, nullptr}, std::memory_order_acq_rel);
- rcu_singly_list_view tmp(from.head_, from.tail_);
- to.splice_back(tmp);
- }
-};
+ auto entry = entry_ref.exchange(thread_entry{nullptr, nullptr}, std::memory_order_acq_rel);
+ rcu_singly_list_view tmp;
+ tmp.head_ = entry.head_;
+ tmp.tail_ = entry.tail_;
+ this->splice_back(tmp);
+ };
+ rcu_thread_local_list_view::per_thread_entries::for_each(splice_single_entry);
+}
#endif // _LIBCPP_STD_VER >= 26 && _LIBCPP_HAS_THREADS && _LIBCPP_HAS_EXPERIMENTAL_RCU
diff --git a/libcxx/src/experimental/rcu.cpp b/libcxx/src/experimental/rcu.cpp
index a0f8784bd6ca5..d8a78db7cdd75 100644
--- a/libcxx/src/experimental/rcu.cpp
+++ b/libcxx/src/experimental/rcu.cpp
@@ -66,7 +66,7 @@ class rcu_domain_impl {
// flag used for waking up writer threads waiting for all reader threads' quiescent state
std::atomic<bool> grace_period_waiting_flag_ = false;
- using per_thread_retired_queue_stage0 = thread_local_container<rcu_atomic_list_view>;
+ rcu_thread_local_list_view retired_queue_stage0_;
// these two queues do not need extra synchronization
// as they are always processed under the grace period mutex
@@ -77,9 +77,7 @@ class rcu_domain_impl {
void update_phase_and_wait() noexcept {
rcu_singly_list_view working_queue;
- per_thread_retired_queue_stage0::for_each([&working_queue](atomic_ref<rcu_atomic_list_view> thread_local_stage0) {
- rcu_atomic_list_view::splice_back(thread_local_stage0, working_queue);
- });
+ working_queue.splice_back(retired_queue_stage0_);
// Flip the global phase
auto old_phase = global_reader_phase_.fetch_xor(reader_states::grace_period_phase_mask, std::memory_order_relaxed);
@@ -147,10 +145,7 @@ class rcu_domain_impl {
}
}
- void retire(__rcu_node* node) noexcept {
- auto thread_local_stage0_ref = per_thread_retired_queue_stage0::get_current_thread_instance();
- rcu_atomic_list_view::push_front(thread_local_stage0_ref, node);
- }
+ void retire(__rcu_node* node) noexcept { retired_queue_stage0_.push_front(node); }
void synchronize(bool invoke_callback) noexcept {
std::atomic_thread_fence(memory_order_seq_cst);
@@ -167,6 +162,7 @@ class rcu_domain_impl {
std::atomic_signal_fence(memory_order_seq_cst);
update_phase_and_wait();
+
if (invoke_callback) {
rcu_singly_list_view ready_callbacks;
ready_callbacks.splice_back(retired_queue_stage2_);
>From 521c2f4f67a1be71c7a1d484883c65c91d3a9210 Mon Sep 17 00:00:00 2001
From: Hui Xie <hui.xie1990 at gmail.com>
Date: Sun, 13 Sep 2026 07:59:52 +0100
Subject: [PATCH 54/65] "atomic queue, refactor thread local to remove
atomic_ref"
This reverts commit 2e84a72927cc29a09869ed85b6ce927c48afe16f.
---
.../src/experimental/include/rcu/rcu_list.h | 52 ++++++------
.../include/rcu/thread_local_container.h | 13 +--
libcxx/src/experimental/rcu.cpp | 82 +++++++++++--------
3 files changed, 83 insertions(+), 64 deletions(-)
diff --git a/libcxx/src/experimental/include/rcu/rcu_list.h b/libcxx/src/experimental/include/rcu/rcu_list.h
index 3eb3dc5750795..3d8b4f83d4ecb 100644
--- a/libcxx/src/experimental/include/rcu/rcu_list.h
+++ b/libcxx/src/experimental/include/rcu/rcu_list.h
@@ -14,6 +14,7 @@
#include <__functional/function.h>
#include <__rcu/rcu_domain.h>
#include <atomic>
+#include <type_traits>
#include "thread_local_container.h"
@@ -25,7 +26,7 @@ _LIBCPP_BEGIN_NAMESPACE_STD
#if _LIBCPP_STD_VER >= 26 && _LIBCPP_HAS_THREADS && _LIBCPP_HAS_EXPERIMENTAL_RCU
-class rcu_thread_local_list_view;
+class rcu_atomic_list_view;
class rcu_singly_list_view {
private:
@@ -48,7 +49,7 @@ class rcu_singly_list_view {
other.tail_ = nullptr;
}
- void splice_back(rcu_thread_local_list_view& __other) noexcept;
+ void splice_back(rcu_atomic_list_view& __other) noexcept;
template <class Func>
void for_each(Func&& f) noexcept {
@@ -62,31 +63,32 @@ class rcu_singly_list_view {
}
};
-class rcu_thread_local_list_view {
- struct alignas(2 * sizeof(void*)) thread_entry {
- __rcu_node* head_ = nullptr;
- __rcu_node* tail_ = nullptr;
- };
+struct alignas(2 * sizeof(void*)) rcu_atomic_list_view_entry {
+ __rcu_node* head_ = nullptr;
+ __rcu_node* tail_ = nullptr;
+};
+
+class rcu_atomic_list_view {
+ using entry = rcu_atomic_list_view_entry;
- using per_thread_entries = thread_local_container<thread_entry>;
+ std::atomic<entry> entry_{};
friend class rcu_singly_list_view;
public:
void push_front(__rcu_node* node) noexcept {
- atomic_ref<thread_entry> entry_ref = per_thread_entries::get_current_thread_instance();
- auto expected_entry = entry_ref.load(std::memory_order_relaxed);
- auto original_next = node->__next_;
+ auto expected_entry = entry_.load(std::memory_order_relaxed);
+ auto original_next = node->__next_;
while (true) {
auto new_entry = [&] {
if (expected_entry.head_ == nullptr) {
- return thread_entry{node, node};
+ return entry{node, node};
} else {
node->__next_ = expected_entry.head_;
- return thread_entry{node, expected_entry.tail_};
+ return entry{node, expected_entry.tail_};
}
}();
- if (entry_ref.compare_exchange_weak(
+ if (entry_.compare_exchange_weak(
expected_entry, new_entry, std::memory_order_acq_rel, std::memory_order_relaxed)) {
break;
} else {
@@ -96,19 +98,15 @@ class rcu_thread_local_list_view {
}
};
-void rcu_singly_list_view::splice_back(rcu_thread_local_list_view& __other) noexcept {
- using thread_entry = rcu_thread_local_list_view::thread_entry;
- const auto splice_single_entry = [this](atomic_ref<thread_entry> entry_ref) noexcept {
- if (entry_ref.load(std::memory_order_relaxed).head_ == nullptr) {
- return;
- }
- auto entry = entry_ref.exchange(thread_entry{nullptr, nullptr}, std::memory_order_acq_rel);
- rcu_singly_list_view tmp;
- tmp.head_ = entry.head_;
- tmp.tail_ = entry.tail_;
- this->splice_back(tmp);
- };
- rcu_thread_local_list_view::per_thread_entries::for_each(splice_single_entry);
+void rcu_singly_list_view::splice_back(rcu_atomic_list_view& __other) noexcept {
+ if (__other.entry_.load(std::memory_order_relaxed).head_ == nullptr) {
+ return;
+ }
+ auto entry = __other.entry_.exchange(rcu_atomic_list_view::entry{nullptr, nullptr}, std::memory_order_acq_rel);
+ rcu_singly_list_view tmp;
+ tmp.head_ = entry.head_;
+ tmp.tail_ = entry.tail_;
+ this->splice_back(tmp);
}
#endif // _LIBCPP_STD_VER >= 26 && _LIBCPP_HAS_THREADS && _LIBCPP_HAS_EXPERIMENTAL_RCU
diff --git a/libcxx/src/experimental/include/rcu/thread_local_container.h b/libcxx/src/experimental/include/rcu/thread_local_container.h
index 973b8279db78a..04a0eb7ce2267 100644
--- a/libcxx/src/experimental/include/rcu/thread_local_container.h
+++ b/libcxx/src/experimental/include/rcu/thread_local_container.h
@@ -14,7 +14,6 @@
#include <__functional/function.h>
#include <__rcu/rcu_domain.h>
-#include <atomic>
#include <mutex>
#include <optional>
#include <vector>
@@ -27,6 +26,10 @@ _LIBCPP_BEGIN_NAMESPACE_STD
#if _LIBCPP_STD_VER >= 26 && _LIBCPP_HAS_THREADS && _LIBCPP_HAS_EXPERIMENTAL_RCU
+// Tp must be thread-safe itself between
+// - the operation that is done by the object from get_current_thread_instance calls
+// - and the operation that for_each
+// since there is no mutex guarding between them
template <class Tp>
class thread_local_container {
struct thread_entry {
@@ -62,19 +65,19 @@ class thread_local_container {
thread_local_container() = delete;
thread_local_container(thread_local_container&&) = delete;
- static atomic_ref<Tp> get_current_thread_instance() {
+ static Tp& get_current_thread_instance() {
if (!thread_entry_.has_value()) {
auto& entry = thread_entry_.emplace();
- return atomic_ref(entry.instance_);
+ return entry.instance_;
}
- return atomic_ref(thread_entry_->instance_);
+ return thread_entry_->instance_;
}
template <class Func>
static void for_each(Func&& f) {
unique_lock<std::mutex> lock(mtx_);
for (auto instance : instances_) {
- f(atomic_ref(*instance));
+ f(*instance);
}
}
};
diff --git a/libcxx/src/experimental/rcu.cpp b/libcxx/src/experimental/rcu.cpp
index d8a78db7cdd75..213f28b8c17b3 100644
--- a/libcxx/src/experimental/rcu.cpp
+++ b/libcxx/src/experimental/rcu.cpp
@@ -7,6 +7,7 @@
//===----------------------------------------------------------------------===//
#include <atomic>
+#include <cstdint>
#include <memory>
#include <mutex>
#include <rcu>
@@ -43,19 +44,41 @@ struct reader_states {
using state_type = uint16_t;
- static uint16_t get_grace_period_phase(state_type state) { return state & grace_period_phase_mask; }
+ std::atomic<state_type> state_{};
- static uint16_t get_reader_nest_level(state_type state) { return state & reader_nest_level_mask; }
+ bool is_quiescent_state() const noexcept {
+ return get_reader_nest_level(state_.load(std::memory_order_relaxed)) == 0;
+ }
- static bool is_quiescent_state(state_type state) { return get_reader_nest_level(state) == 0; }
+ bool is_grace_period_ongoing_at_phase(state_type global_phase) const noexcept {
+ auto state = state_.load(std::memory_order_relaxed);
+ return !is_quiescent_state(state) && get_grace_period_phase(state) != global_phase;
+ }
- static state_type make_state(state_type grace_period_phase, state_type reader_nest_level) {
- return (grace_period_phase & grace_period_phase_mask) | (reader_nest_level & reader_nest_level_mask);
+ void set_state(state_type grace_period_phase, state_type reader_nest_level) {
+ auto state = (grace_period_phase & grace_period_phase_mask) | (reader_nest_level & reader_nest_level_mask);
+ state_.store(state, memory_order_relaxed);
}
+
+ void increment_nest_level() noexcept { state_.fetch_add(1, memory_order_relaxed); }
+
+ // return previous nested level
+ uint16_t decrement_nest_level() noexcept {
+ auto old_state = state_.fetch_sub(1, memory_order_relaxed);
+ return get_reader_nest_level(old_state);
+ }
+
+ state_type debug_get_state() const noexcept { return state_.load(); }
+
+private:
+ static uint16_t get_grace_period_phase(state_type state) noexcept { return state & grace_period_phase_mask; }
+ static bool is_quiescent_state(state_type state) noexcept { return get_reader_nest_level(state) == 0; }
+
+ static uint16_t get_reader_nest_level(state_type state) noexcept { return state & reader_nest_level_mask; }
};
class rcu_domain_impl {
- using per_thread_states = thread_local_container<reader_states::state_type>;
+ using per_thread_states = thread_local_container<reader_states>;
// only the highest bit is used for the phase.
std::atomic<reader_states::state_type> global_reader_phase_{};
@@ -66,7 +89,7 @@ class rcu_domain_impl {
// flag used for waking up writer threads waiting for all reader threads' quiescent state
std::atomic<bool> grace_period_waiting_flag_ = false;
- rcu_thread_local_list_view retired_queue_stage0_;
+ using per_thread_retired_queue_stage0 = thread_local_container<rcu_atomic_list_view>;
// these two queues do not need extra synchronization
// as they are always processed under the grace period mutex
@@ -77,7 +100,9 @@ class rcu_domain_impl {
void update_phase_and_wait() noexcept {
rcu_singly_list_view working_queue;
- working_queue.splice_back(retired_queue_stage0_);
+ per_thread_retired_queue_stage0::for_each([&working_queue](rcu_atomic_list_view& stage0_list) {
+ working_queue.splice_back(stage0_list);
+ });
// Flip the global phase
auto old_phase = global_reader_phase_.fetch_xor(reader_states::grace_period_phase_mask, std::memory_order_relaxed);
@@ -99,53 +124,45 @@ class rcu_domain_impl {
bool any_reader_in_ongoing_grace_period(reader_states::state_type global_phase) noexcept {
bool any_ongoing = false;
- per_thread_states::for_each([this, global_phase, &any_ongoing](atomic_ref<reader_states::state_type> state) {
- if (is_grace_period_ongoing(state.load(memory_order_relaxed), global_phase)) {
+ per_thread_states::for_each([global_phase, &any_ongoing](reader_states& state) {
+ if (state.is_grace_period_ongoing_at_phase(global_phase)) {
any_ongoing = true;
}
});
return any_ongoing;
}
- bool is_grace_period_ongoing(reader_states::state_type thread_state,
- reader_states::state_type global_phase) const noexcept {
- // https://lwn.net/Articles/323929/
- // The phase is flipped at the beginning of a grace period.
- // Any readers that started before the flip will have the old phase
- // and we consider them as ongoing that we need to wait for before we can close the grace period.
- return !reader_states::is_quiescent_state(thread_state) &&
- reader_states::get_grace_period_phase(thread_state) != global_phase;
- }
-
public:
void lock() noexcept {
- auto current_thread_state_ref = per_thread_states::get_current_thread_instance();
+ reader_states& current_thread_state = per_thread_states::get_current_thread_instance();
- if ((reader_states::is_quiescent_state(current_thread_state_ref.load(memory_order_relaxed)))) {
- // Entering a read-side critical section from a quiescent state.
- current_thread_state_ref.store(
- reader_states::make_state(global_reader_phase_.load(std::memory_order_relaxed), 1), memory_order_relaxed);
+ if (current_thread_state.is_quiescent_state()) {
+ // Enter critical section by setting the nested level from 0 -> 1
+ current_thread_state.set_state(global_reader_phase_.load(memory_order_relaxed), 1);
std::atomic_thread_fence(memory_order_seq_cst);
} else {
// Already in read-side critical section, just increment the nest level.
- current_thread_state_ref.fetch_add(1, memory_order_relaxed);
+ current_thread_state.increment_nest_level();
}
}
void unlock() noexcept {
- auto current_thread_state_ref = per_thread_states::get_current_thread_instance();
+ reader_states& current_thread_state = per_thread_states::get_current_thread_instance();
std::atomic_thread_fence(memory_order_seq_cst);
// Decrement the nest level.
- auto old_state = current_thread_state_ref.fetch_sub(1, memory_order_relaxed);
+ auto old_nested_level = current_thread_state.decrement_nest_level();
- if (reader_states::get_reader_nest_level(old_state) == 1 && grace_period_waiting_flag_.load(memory_order_relaxed)) {
+ if (old_nested_level == 1 && grace_period_waiting_flag_.load(memory_order_relaxed)) {
// Transitioning to quiescent state, wake up waiters.
grace_period_waiting_flag_.store(false, std::memory_order_relaxed);
grace_period_waiting_flag_.notify_all();
}
}
- void retire(__rcu_node* node) noexcept { retired_queue_stage0_.push_front(node); }
+ void retire(__rcu_node* node) noexcept {
+ rcu_atomic_list_view& stage0_queue = per_thread_retired_queue_stage0::get_current_thread_instance();
+ stage0_queue.push_front(node);
+ }
void synchronize(bool invoke_callback) noexcept {
std::atomic_thread_fence(memory_order_seq_cst);
@@ -162,7 +179,6 @@ class rcu_domain_impl {
std::atomic_signal_fence(memory_order_seq_cst);
update_phase_and_wait();
-
if (invoke_callback) {
rcu_singly_list_view ready_callbacks;
ready_callbacks.splice_back(retired_queue_stage2_);
@@ -172,7 +188,9 @@ class rcu_domain_impl {
}
void debugPrintAllReaderStatesInHex() {
- per_thread_states::for_each([](auto state_ref) { std::printf("Reader state: 0x%04x\n", state_ref.load()); });
+ per_thread_states::for_each([](reader_states& state) {
+ std::printf("Reader state: 0x%04x\n", state.debug_get_state());
+ });
}
};
} // namespace
>From c0bc5eb5539d1e4a443d690ebf4cdba5cd7a570f Mon Sep 17 00:00:00 2001
From: Hui Xie <hui.xie1990 at gmail.com>
Date: Sun, 13 Sep 2026 11:07:45 +0100
Subject: [PATCH 55/65] review
---
libcxx/docs/DesignDocs/RCU.rst | 47 +++++++++++--------
libcxx/include/__rcu/rcu_domain.h | 2 +-
libcxx/src/experimental/rcu.cpp | 4 +-
.../saferecl/saferecl.rcu/general.pass.cpp | 10 ++--
.../saferecl/saferecl.rcu/retire.pass.cpp | 8 ++--
5 files changed, 40 insertions(+), 31 deletions(-)
diff --git a/libcxx/docs/DesignDocs/RCU.rst b/libcxx/docs/DesignDocs/RCU.rst
index 4104498659a2e..e4d2eb7c36e9f 100644
--- a/libcxx/docs/DesignDocs/RCU.rst
+++ b/libcxx/docs/DesignDocs/RCU.rst
@@ -33,7 +33,10 @@ Synopsis
rcu_domain& rcu_default_domain() noexcept;
+ // blocks until all previous readers exit the critical section
void rcu_synchronize(rcu_domain& dom = rcu_default_domain()) noexcept;
+
+ // blocks until all deleters of previously retired objects run
void rcu_barrier(rcu_domain& dom = rcu_default_domain()) noexcept;
template<class T, class D = default_delete<T>>
@@ -54,6 +57,7 @@ Here is an example usage ::
struct Data { /* members */ };
std::atomic<Data*> data_;
+ // reader thread
// ==============================================
template <typename Func>
Result reader_op(Func fn) {
@@ -65,6 +69,7 @@ Here is an example usage ::
return fn(p);
}
+ // writer thread
// ==============================================
// May be called concurrently with reader_op
void update(Data* newdata) {
@@ -72,12 +77,20 @@ Here is an example usage ::
std::rcu_retire(olddata); // reclaim *olddata when safe
}
+ // collector thread
+ // ==============================================
+ void collect_garbage(std::stop_token st) {
+ while (!st.stop_requested()) {
+ std::rcu_barrier();
+ std::this_thread::sleep(1s);
+ }
+ }
+
There are several key properties that an implementation of `rcu` must satisfy:
-- On the reader side, `rcu_domain::lock` and `rcu_domain::unlock` must not block the thread while the writer thread
- is performing updates, or the collector thread is reclaiming the objects.
+- On the reader side, `rcu_domain::lock` and `rcu_domain::unlock` must not block the thread while the writer thread is performing updates, other readers are `lock` ing the critical section, or the collector thread is reclaiming the objects.
-- On the writer side, `retire`-ing an object in principle should not block at all. However, it is technically conforming to block (e.g. evaluate the `deleter`s directly inside `rcu_retire`).
+- On the writer side, our interpretation is that `retire`-ing an object should not block for quality of implementation reasons. However, it is technically conforming to block (e.g. evaluate the `deleter`s directly inside `rcu_retire`).
- On the collector side, `rcu_synchronize` should block until at least all the existing readers exit their critical sections
via `rcu_domain::unlock`. Note that this is a key difference between `rcu` and read-write locks:
@@ -137,7 +150,7 @@ Implementation Details
----------------------
`class thread_local_container`
-~~~~~~~~~~~~~~~~~~~~~~~~
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
A helper class that manages thread local objects. It provides APIs to get the `thread_local` object for the current thread, and to iterate through all the `thread_local` objects from all threads.
It only contains `static` member variables and functions.
@@ -148,7 +161,7 @@ the contention on the `mutex` should be low.
TODO: We need to replace `mutex` as all non allocating APIs in `rcu` are designed to be `noexcept` and `mutex` can throw.
`struct reader_states`
-~~~~~~~~~~~~~~~
+~~~~~~~~~~~~~~~~~~~~~~
It defines the state of each reader thread. The state is essentially a pair of `(phase, lock_nested_level)` merged into a single `uint16_t` .
@@ -156,28 +169,24 @@ It defines the state of each reader thread. The state is essentially a pair of `
- `lock_nested_level` is the nested level of the `rcu_domain::lock` . As you can call `rcu_domain::lock` multiple times in the same thread, we need to keep track of the nested level.
When `lock_nested_level` is `0`, it means the current thread is in a quiescent state (not in a critical section).
-The `reader_states` class is also a `static` helper class which only provides static definitions and functions, instead of the wrapper of the `uint16_t` state. This is because we need
-the `atomic` integer API, e.g. `fetch_or`. An `atomic` of a wrapper struct won't have these APIs.
-
-`rcu_singly_list_view`
-~~~~~~~~~~~~~~~~~~~~~~
+`class rcu_singly_list_view`
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~
A helper class that provides a view of the intrusive singly linked list of `__rcu_node` s. It also stores the back pointer of the list to make the splice back operation more efficient.
-It provides APIs to splice from another `rcu_singly_list_view` or `rcu_thread_local_list_view`, and to iterate through the list.
+It provides APIs to splice from another `rcu_singly_list_view` or `rcu_atomic_list_view`, and to iterate through the list.
-`rcu_thread_local_list_view`
-~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+`class rcu_atomic_list_view`
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
A thread safe helper class that provides a view of the intrusive singly linked list of `__rcu_node` s. It provides APIs to push a node into the list, or to be spliced to another `rcu_singly_list_view`.
-It uses `thread_local` storage to avoid contentions from multiple threads pushing nodes into the list.
-For the derived class of `rcu_obj_base`, the `retire` is `noexcept`, which means that no memory allocation can happen when pushing it to the retired callback queue.
-
-`rcu_domain_impl`
-~~~~~~~~~~~~~~~~~
+`class rcu_domain_impl`
+~~~~~~~~~~~~~~~~~~~~~~~~
This is the main class that implements the `rcu` logic. It contains
+- `using per_thread_states = thread_local_container<reader_states>`. It contains each thread's `reader_states`. This is not a member but static thread_local objects.
+
- `std::atomic<reader_states::state_type> global_reader_phase_` : the global state that flips between two phases. The readers will record the phase when they enter the critical section, and the collector will flip the global state when it calls `rcu_synchronize` to start a new grace period.
- `std::mutex grace_period_mutex_` : If we have multiple collector threads calling `rcu_synchronize` concurrenly, we need to make sure only one of them is performing the phase flipping and deleter queue draining.
@@ -185,7 +194,7 @@ This is the main class that implements the `rcu` logic. It contains
- `std::atomic<bool> grace_period_waiting_flag_` : This flag is used to sleep/wake up the collector thread that is waiting for the grace period to end.
-- `rcu_thread_local_list_view retired_queue_stage0_;` : All the retired objects are directly pushed to this queue first.
+- `using per_thread_retired_queue_stage0 = thread_local_container<rcu_atomic_list_view>` : All the retired objects are directly pushed to this queue first. This is not a member but static thread_local objects.
- `rcu_singly_list_view retired_queue_stage1_` and `rcu_singly_list_view retired_queue_stage2_` : These two queues are used to let the retired callbacks go through two grace periods before being invoked. No additional synchronization is needed for these two queues as they are only processed when the collector thread is holding the `grace_period_mutex_`.
Note that `_retired_queue_stage2_` queue holds the callbacks that are ready to be called.
diff --git a/libcxx/include/__rcu/rcu_domain.h b/libcxx/include/__rcu/rcu_domain.h
index ee966775792b1..7e0e94461a15a 100644
--- a/libcxx/include/__rcu/rcu_domain.h
+++ b/libcxx/include/__rcu/rcu_domain.h
@@ -74,7 +74,7 @@ class _LIBCPP_EXPORTED_FROM_ABI rcu_domain {
rcu_domain& operator=(const rcu_domain&) = delete;
~rcu_domain();
- void debugPrintAllReaderStatesInHex();
+ void __debug_print_all_reader_states_in_hex();
void lock() noexcept;
diff --git a/libcxx/src/experimental/rcu.cpp b/libcxx/src/experimental/rcu.cpp
index 213f28b8c17b3..350ccf2d6dea1 100644
--- a/libcxx/src/experimental/rcu.cpp
+++ b/libcxx/src/experimental/rcu.cpp
@@ -187,7 +187,7 @@ class rcu_domain_impl {
std::atomic_thread_fence(memory_order_seq_cst);
}
- void debugPrintAllReaderStatesInHex() {
+ void __debug_print_all_reader_states_in_hex() {
per_thread_states::for_each([](reader_states& state) {
std::printf("Reader state: 0x%04x\n", state.debug_get_state());
});
@@ -205,7 +205,7 @@ rcu_domain& rcu_domain::__rcu_default_domain() noexcept {
rcu_domain::rcu_domain() : __pimpl_(std::make_unique<__impl>()) {}
rcu_domain::~rcu_domain() = default;
-void rcu_domain::debugPrintAllReaderStatesInHex() { __pimpl_->debugPrintAllReaderStatesInHex(); }
+void rcu_domain::__debug_print_all_reader_states_in_hex() { __pimpl_->__debug_print_all_reader_states_in_hex(); }
void rcu_domain::lock() noexcept { __pimpl_->lock(); }
diff --git a/libcxx/test/std/thread/saferecl/saferecl.rcu/general.pass.cpp b/libcxx/test/std/thread/saferecl/saferecl.rcu/general.pass.cpp
index aac4247498c09..1f0275ddc1cbd 100644
--- a/libcxx/test/std/thread/saferecl/saferecl.rcu/general.pass.cpp
+++ b/libcxx/test/std/thread/saferecl/saferecl.rcu/general.pass.cpp
@@ -39,7 +39,7 @@ int main(int, char**) {
log(start, "t1: leaving rcu read-side critical section " + std::to_string(i));
dom.unlock();
log(start, "t1: printing all reader states");
- dom.debugPrintAllReaderStatesInHex();
+ dom.__debug_print_all_reader_states_in_hex();
}
});
@@ -54,7 +54,7 @@ int main(int, char**) {
log(start, "t2: leaving rcu read-side critical section " + std::to_string(i));
dom.unlock();
log(start, "t2: printing all reader states");
- dom.debugPrintAllReaderStatesInHex();
+ dom.__debug_print_all_reader_states_in_hex();
}
});
@@ -69,7 +69,7 @@ int main(int, char**) {
log(start, "t3: leaving rcu read-side critical section " + std::to_string(i));
dom.unlock();
log(start, "t3: printing all reader states");
- dom.debugPrintAllReaderStatesInHex();
+ dom.__debug_print_all_reader_states_in_hex();
}
});
auto t4 = support::make_test_thread([start]() {
@@ -83,7 +83,7 @@ int main(int, char**) {
log(start, "t4: leaving rcu read-side critical section " + std::to_string(i));
dom.unlock();
log(start, "t4: printing all reader states");
- dom.debugPrintAllReaderStatesInHex();
+ dom.__debug_print_all_reader_states_in_hex();
}
});
/*
@@ -101,7 +101,7 @@ int main(int, char**) {
for (int i = 0; i < loop_num+5; ++i) {
log(start, "t0: printing all reader states before synchronize");
- std::rcu_default_domain().debugPrintAllReaderStatesInHex();
+ std::rcu_default_domain().__debug_print_all_reader_states_in_hex();
log(start, "t0: calling rcu_synchronize" + std::to_string(i));
std::rcu_synchronize();
log(start, "t0: rcu_synchronize returned" + std::to_string(i));
diff --git a/libcxx/test/std/thread/saferecl/saferecl.rcu/retire.pass.cpp b/libcxx/test/std/thread/saferecl/saferecl.rcu/retire.pass.cpp
index 03cfeb572c2dd..1573cafc03a60 100644
--- a/libcxx/test/std/thread/saferecl/saferecl.rcu/retire.pass.cpp
+++ b/libcxx/test/std/thread/saferecl/saferecl.rcu/retire.pass.cpp
@@ -56,7 +56,7 @@ int main(int, char**) {
(obj ? obj->data_ : "nullptr"));
dom.unlock();
log(start, "t1: printing all reader states");
- dom.debugPrintAllReaderStatesInHex();
+ dom.__debug_print_all_reader_states_in_hex();
}
});
@@ -73,7 +73,7 @@ int main(int, char**) {
(obj ? obj->data_ : "nullptr"));
dom.unlock();
log(start, "t2: printing all reader states");
- dom.debugPrintAllReaderStatesInHex();
+ dom.__debug_print_all_reader_states_in_hex();
}
});
@@ -90,7 +90,7 @@ int main(int, char**) {
(obj ? obj->data_ : "nullptr"));
dom.unlock();
log(start, "t3: printing all reader states");
- dom.debugPrintAllReaderStatesInHex();
+ dom.__debug_print_all_reader_states_in_hex();
}
});
auto t4 = support::make_test_thread([start]() {
@@ -106,7 +106,7 @@ int main(int, char**) {
(obj ? obj->data_ : "nullptr"));
dom.unlock();
log(start, "t4: printing all reader states");
- dom.debugPrintAllReaderStatesInHex();
+ dom.__debug_print_all_reader_states_in_hex();
}
});
>From 62372853337528302d6c5190dcaeb6665ef008ba Mon Sep 17 00:00:00 2001
From: Hui Xie <hui.xie1990 at gmail.com>
Date: Sun, 13 Sep 2026 11:16:32 +0100
Subject: [PATCH 56/65] remove double negative
---
libcxx/src/experimental/rcu.cpp | 6 +++---
1 file changed, 3 insertions(+), 3 deletions(-)
diff --git a/libcxx/src/experimental/rcu.cpp b/libcxx/src/experimental/rcu.cpp
index 350ccf2d6dea1..88c61eb5355bb 100644
--- a/libcxx/src/experimental/rcu.cpp
+++ b/libcxx/src/experimental/rcu.cpp
@@ -52,7 +52,7 @@ struct reader_states {
bool is_grace_period_ongoing_at_phase(state_type global_phase) const noexcept {
auto state = state_.load(std::memory_order_relaxed);
- return !is_quiescent_state(state) && get_grace_period_phase(state) != global_phase;
+ return !is_quiescent_state(state) && get_grace_period_phase(state) == global_phase;
}
void set_state(state_type grace_period_phase, state_type reader_nest_level) {
@@ -106,13 +106,13 @@ class rcu_domain_impl {
// Flip the global phase
auto old_phase = global_reader_phase_.fetch_xor(reader_states::grace_period_phase_mask, std::memory_order_relaxed);
- auto new_phase = old_phase ^ reader_states::grace_period_phase_mask;
+ // auto new_phase = old_phase ^ reader_states::grace_period_phase_mask;
// std::printf("rcu_domain::update_phase_and_wait() new phase: 0x%04x\n", new_phase);
std::atomic_signal_fence(std::memory_order_seq_cst);
// Wait for all threads to quiesce in the old phase
- while (any_reader_in_ongoing_grace_period(new_phase)) {
+ while (any_reader_in_ongoing_grace_period(old_phase)) {
grace_period_waiting_flag_.store(true, std::memory_order_relaxed);
grace_period_waiting_flag_.wait(true, std::memory_order_relaxed);
}
>From c35e6e7b095e3e88e2124b7e15245891b3075155 Mon Sep 17 00:00:00 2001
From: Hui Xie <hui.xie1990 at gmail.com>
Date: Sun, 13 Sep 2026 20:24:26 +0100
Subject: [PATCH 57/65] fix writer threads finish with pending thread local
stage 0
---
.../src/experimental/include/rcu/rcu_list.h | 38 +++-
.../include/rcu/thread_local_container.h | 18 +-
libcxx/src/experimental/rcu.cpp | 12 +-
.../saferecl.rcu.base/retire.pass.cpp | 2 +-
.../saferecl.rcu.domain/rcu_barrier.pass.cpp | 179 ++++++++++++++++++
.../rcu_barrier_stress.pass.cpp | 111 +++++++++++
.../saferecl.rcu.domain/rcu_retire.pass.cpp | 2 +-
7 files changed, 349 insertions(+), 13 deletions(-)
create mode 100644 libcxx/test/std/thread/saferecl/saferecl.rcu/saferecl.rcu.domain/rcu_barrier.pass.cpp
create mode 100644 libcxx/test/std/thread/saferecl/saferecl.rcu/saferecl.rcu.domain/rcu_barrier_stress.pass.cpp
diff --git a/libcxx/src/experimental/include/rcu/rcu_list.h b/libcxx/src/experimental/include/rcu/rcu_list.h
index 3d8b4f83d4ecb..3c1a882d51e5a 100644
--- a/libcxx/src/experimental/include/rcu/rcu_list.h
+++ b/libcxx/src/experimental/include/rcu/rcu_list.h
@@ -49,7 +49,7 @@ class rcu_singly_list_view {
other.tail_ = nullptr;
}
- void splice_back(rcu_atomic_list_view& __other) noexcept;
+ void splice_back(rcu_atomic_list_view& other) noexcept;
template <class Func>
void for_each(Func&& f) noexcept {
@@ -96,13 +96,43 @@ class rcu_atomic_list_view {
}
}
}
+
+ void splice_front(rcu_atomic_list_view& other) noexcept {
+ if (other.entry_.load(std::memory_order_relaxed).head_ == nullptr) {
+ return;
+ }
+
+ entry to_be_inserted = other.entry_.exchange(entry{nullptr, nullptr}, std::memory_order_relaxed);
+
+ auto original_next = to_be_inserted.tail_->__next_;
+
+ auto expected_entry = entry_.load(std::memory_order_relaxed);
+
+ while (true) {
+ auto new_entry = [&] {
+ if (expected_entry.head_ == nullptr) {
+ return to_be_inserted;
+ } else {
+ to_be_inserted.tail_->__next_ = expected_entry.head_;
+ return entry{to_be_inserted.head_, expected_entry.tail_};
+ }
+ }();
+
+ if (entry_.compare_exchange_weak(
+ expected_entry, new_entry, std::memory_order_acq_rel, std::memory_order_relaxed)) {
+ break;
+ } else {
+ to_be_inserted.tail_->__next_ = original_next;
+ }
+ }
+ }
};
-void rcu_singly_list_view::splice_back(rcu_atomic_list_view& __other) noexcept {
- if (__other.entry_.load(std::memory_order_relaxed).head_ == nullptr) {
+void rcu_singly_list_view::splice_back(rcu_atomic_list_view& other) noexcept {
+ if (other.entry_.load(std::memory_order_relaxed).head_ == nullptr) {
return;
}
- auto entry = __other.entry_.exchange(rcu_atomic_list_view::entry{nullptr, nullptr}, std::memory_order_acq_rel);
+ auto entry = other.entry_.exchange(rcu_atomic_list_view::entry{nullptr, nullptr}, std::memory_order_acq_rel);
rcu_singly_list_view tmp;
tmp.head_ = entry.head_;
tmp.tail_ = entry.tail_;
diff --git a/libcxx/src/experimental/include/rcu/thread_local_container.h b/libcxx/src/experimental/include/rcu/thread_local_container.h
index 04a0eb7ce2267..2b779623b508b 100644
--- a/libcxx/src/experimental/include/rcu/thread_local_container.h
+++ b/libcxx/src/experimental/include/rcu/thread_local_container.h
@@ -11,7 +11,7 @@
#define _LIBCPP___RCU_THREAD_LOCAL_CONTAINER_H
#include <__config>
-#include <__functional/function.h>
+#include <__functional/function_ref.h>
#include <__rcu/rcu_domain.h>
#include <mutex>
@@ -26,20 +26,26 @@ _LIBCPP_BEGIN_NAMESPACE_STD
#if _LIBCPP_STD_VER >= 26 && _LIBCPP_HAS_THREADS && _LIBCPP_HAS_EXPERIMENTAL_RCU
-// Tp must be thread-safe itself between
+// Tp must be thread-safe itself between
// - the operation that is done by the object from get_current_thread_instance calls
// - and the operation that for_each
// since there is no mutex guarding between them
template <class Tp>
class thread_local_container {
+ static void empty_callback(Tp&) noexcept {}
+
struct thread_entry {
Tp instance_;
+ function_ref<void(Tp&) noexcept> pre_dtor_callback_;
- thread_entry() : instance_() { register_instance(instance_); }
+ thread_entry( function_ref<void(Tp&) noexcept> cb) : instance_(), pre_dtor_callback_(cb) { register_instance(instance_); }
thread_entry(const thread_entry&) = delete;
thread_entry(thread_entry&&) = delete;
- ~thread_entry() { deregister_instance(instance_); }
+ ~thread_entry() {
+ deregister_instance(instance_);
+ pre_dtor_callback_(instance_);
+ }
};
inline static thread_local optional<thread_entry> thread_entry_{};
@@ -65,9 +71,9 @@ class thread_local_container {
thread_local_container() = delete;
thread_local_container(thread_local_container&&) = delete;
- static Tp& get_current_thread_instance() {
+ static Tp& get_current_thread_instance(function_ref<void(Tp&) noexcept> cb = cw<&empty_callback>) {
if (!thread_entry_.has_value()) {
- auto& entry = thread_entry_.emplace();
+ auto& entry = thread_entry_.emplace(cb);
return entry.instance_;
}
return thread_entry_->instance_;
diff --git a/libcxx/src/experimental/rcu.cpp b/libcxx/src/experimental/rcu.cpp
index 88c61eb5355bb..0a2c8745a8309 100644
--- a/libcxx/src/experimental/rcu.cpp
+++ b/libcxx/src/experimental/rcu.cpp
@@ -89,6 +89,10 @@ class rcu_domain_impl {
// flag used for waking up writer threads waiting for all reader threads' quiescent state
std::atomic<bool> grace_period_waiting_flag_ = false;
+ // stage 0 queue is thread local. In case a thread dies with non-empty list in stage 0,
+ // those nodes will be move into the orphaned_stage0_ on destruction
+ rcu_atomic_list_view retired_queue_orphaned_stage0_;
+
using per_thread_retired_queue_stage0 = thread_local_container<rcu_atomic_list_view>;
// these two queues do not need extra synchronization
@@ -103,6 +107,7 @@ class rcu_domain_impl {
per_thread_retired_queue_stage0::for_each([&working_queue](rcu_atomic_list_view& stage0_list) {
working_queue.splice_back(stage0_list);
});
+ working_queue.splice_back(retired_queue_orphaned_stage0_);
// Flip the global phase
auto old_phase = global_reader_phase_.fetch_xor(reader_states::grace_period_phase_mask, std::memory_order_relaxed);
@@ -132,6 +137,10 @@ class rcu_domain_impl {
return any_ongoing;
}
+ void move_to_orphan_list_on_destruction(rcu_atomic_list_view& stage0_to_be_destroyed) noexcept {
+ retired_queue_orphaned_stage0_.splice_front(stage0_to_be_destroyed);
+ }
+
public:
void lock() noexcept {
reader_states& current_thread_state = per_thread_states::get_current_thread_instance();
@@ -160,7 +169,8 @@ class rcu_domain_impl {
}
void retire(__rcu_node* node) noexcept {
- rcu_atomic_list_view& stage0_queue = per_thread_retired_queue_stage0::get_current_thread_instance();
+ rcu_atomic_list_view& stage0_queue = per_thread_retired_queue_stage0::get_current_thread_instance(
+ function_ref(std::cw<&rcu_domain_impl::move_to_orphan_list_on_destruction>, this));
stage0_queue.push_front(node);
}
diff --git a/libcxx/test/std/thread/saferecl/saferecl.rcu/saferecl.rcu.base/retire.pass.cpp b/libcxx/test/std/thread/saferecl/saferecl.rcu/saferecl.rcu.base/retire.pass.cpp
index 5b27073b48c8d..cb9917e12ee24 100644
--- a/libcxx/test/std/thread/saferecl/saferecl.rcu/saferecl.rcu.base/retire.pass.cpp
+++ b/libcxx/test/std/thread/saferecl/saferecl.rcu/saferecl.rcu.base/retire.pass.cpp
@@ -183,7 +183,7 @@ int main(int, char**) {
dom.unlock();
}
{
- // multiple reader and writer threa constexpr int num_readers = 4;
+ // multiple reader and writer threads
constexpr int num_readers = 4;
constexpr int num_writers = 3;
std::latch unlock_latch(num_writers);
diff --git a/libcxx/test/std/thread/saferecl/saferecl.rcu/saferecl.rcu.domain/rcu_barrier.pass.cpp b/libcxx/test/std/thread/saferecl/saferecl.rcu/saferecl.rcu.domain/rcu_barrier.pass.cpp
new file mode 100644
index 0000000000000..b5ff929eb1553
--- /dev/null
+++ b/libcxx/test/std/thread/saferecl/saferecl.rcu/saferecl.rcu.domain/rcu_barrier.pass.cpp
@@ -0,0 +1,179 @@
+//===----------------------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+//
+// UNSUPPORTED: no-threads
+// REQUIRES: std-at-least-c++26
+
+// void rcu_barrier(rcu_domain& dom = rcu_default_domain()) noexcept;
+
+#include <atomic>
+#include <cassert>
+#include <chrono>
+#include <rcu>
+#include <thread>
+#include <vector>
+
+#include "make_test_thread.h"
+#include "test_macros.h"
+
+static_assert(noexcept(std::rcu_barrier()));
+static_assert(noexcept(std::rcu_barrier(std::rcu_default_domain())));
+
+class TestClass1 {
+ int& destruction_count_;
+
+public:
+ TestClass1(int& c) : destruction_count_(c) {}
+ ~TestClass1() { ++destruction_count_; }
+};
+
+struct TestClass2 {
+ std::atomic<int>& dtor_count_;
+
+ TestClass2(std::atomic<int>& ctor_count, std::atomic<int>& dtor_count) : dtor_count_(dtor_count) {
+ ctor_count.fetch_add(1, std::memory_order_relaxed);
+ }
+
+ ~TestClass2() { dtor_count_.fetch_add(1, std::memory_order_relaxed); }
+};
+
+int main(int, char**) {
+ {
+ // rcu_barrier();
+
+ int destruction_count = 0;
+ auto* ptr = new TestClass1(destruction_count);
+ std::rcu_retire(ptr);
+ std::rcu_barrier();
+ assert(destruction_count == 1);
+ }
+ {
+ // rcu_retire(domain);
+
+ int destruction_count = 0;
+ auto* ptr = new TestClass1(destruction_count);
+ std::rcu_retire(ptr);
+ std::rcu_barrier(std::rcu_default_domain());
+ assert(destruction_count == 1);
+ }
+ {
+ // different thread
+
+ int destruction_count = 0;
+ auto* ptr = new TestClass1(destruction_count);
+ std::rcu_retire(ptr);
+
+ auto t = support::make_test_jthread([&] {
+ std::rcu_barrier();
+ assert(destruction_count == 1);
+ });
+ }
+ {
+ // multiple collector threads
+ int destruction_count = 0;
+ auto* ptr = new TestClass1(destruction_count);
+ std::rcu_retire(ptr);
+
+ constexpr int num_collectors = 4;
+ std::vector<std::jthread> collector_threads;
+ for (int i = 0; i < num_collectors; ++i) {
+ collector_threads.push_back(support::make_test_jthread([&] {
+ std::rcu_barrier();
+ assert(destruction_count == 1);
+ }));
+ }
+ }
+
+ {
+ // multiple writer and collector threads
+ constexpr int num_writers = 3;
+ constexpr int num_collectors = 3;
+
+ std::atomic<int> ctor_count = 0;
+ std::atomic<int> dtor_count = 0;
+
+ std::vector<std::jthread> writer_threads;
+ for (int i = 0; i < num_writers; ++i) {
+ writer_threads.emplace_back([&](std::stop_token st) {
+ while (!st.stop_requested()) {
+ auto* ptr = new TestClass2(ctor_count, dtor_count);
+ std::rcu_retire(ptr);
+ std::this_thread::sleep_for(std::chrono::microseconds(1));
+ }
+ });
+ }
+
+ std::vector<std::jthread> collector_threads;
+ for (int i = 0; i < num_collectors; ++i) {
+ collector_threads.emplace_back([&](std::stop_token st) {
+ while (!st.stop_requested()) {
+ std::rcu_barrier();
+ std::this_thread::sleep_for(std::chrono::microseconds(5));
+ }
+ });
+ }
+
+ std::this_thread::sleep_for(std::chrono::milliseconds(10));
+
+ writer_threads.clear();
+ collector_threads.clear();
+ assert(ctor_count.load() == dtor_count.load());
+ }
+
+ {
+ // multiple reader, writer and collector threads
+ constexpr int num_readers = 2;
+ constexpr int num_writers = 2;
+ constexpr int num_collectors = 2;
+
+ std::atomic<int> ctor_count = 0;
+ std::atomic<int> dtor_count = 0;
+
+ std::vector<std::jthread> reader_threads;
+ for (int i = 0; i < num_readers; ++i) {
+ reader_threads.emplace_back([&](std::stop_token st) {
+ while (!st.stop_requested()) {
+ auto& dom = std::rcu_default_domain();
+ dom.lock();
+ std::this_thread::sleep_for(std::chrono::microseconds(5));
+ dom.unlock();
+ }
+ });
+ }
+
+ std::vector<std::jthread> writer_threads;
+ for (int i = 0; i < num_writers; ++i) {
+ writer_threads.emplace_back([&](std::stop_token st) {
+ while (!st.stop_requested()) {
+ auto* ptr = new TestClass2(ctor_count, dtor_count);
+ std::rcu_retire(ptr);
+ std::this_thread::sleep_for(std::chrono::microseconds(5));
+ }
+ });
+ }
+
+ std::vector<std::jthread> collector_threads;
+ for (int i = 0; i < num_collectors; ++i) {
+ collector_threads.emplace_back([&](std::stop_token st) {
+ while (!st.stop_requested()) {
+ std::rcu_barrier();
+ std::this_thread::sleep_for(std::chrono::microseconds(5));
+ }
+ });
+ }
+
+ std::this_thread::sleep_for(std::chrono::milliseconds(10));
+
+ reader_threads.clear();
+ writer_threads.clear();
+ collector_threads.clear();
+ assert(ctor_count.load() == dtor_count.load());
+ }
+
+ return 0;
+}
diff --git a/libcxx/test/std/thread/saferecl/saferecl.rcu/saferecl.rcu.domain/rcu_barrier_stress.pass.cpp b/libcxx/test/std/thread/saferecl/saferecl.rcu/saferecl.rcu.domain/rcu_barrier_stress.pass.cpp
new file mode 100644
index 0000000000000..8f324b59ae64d
--- /dev/null
+++ b/libcxx/test/std/thread/saferecl/saferecl.rcu/saferecl.rcu.domain/rcu_barrier_stress.pass.cpp
@@ -0,0 +1,111 @@
+//===----------------------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+//
+// UNSUPPORTED: no-threads
+// REQUIRES: std-at-least-c++26
+
+// void rcu_barrier(rcu_domain& dom = rcu_default_domain()) noexcept;
+
+#include <atomic>
+#include <cassert>
+#include <rcu>
+#include <stop_token>
+#include <thread>
+#include <vector>
+
+#include "make_test_thread.h"
+#include "test_macros.h"
+
+struct TestClass {
+ size_t index_;
+
+ TestClass(size_t i) : index_(i) {}
+ TestClass(const TestClass&) = delete;
+ TestClass(TestClass&&) = delete;
+ TestClass& operator=(const TestClass&) = delete;
+ TestClass& operator=(TestClass&&) = delete;
+ ~TestClass();
+};
+
+struct ObjectStore {
+ static constexpr size_t max_objects = 100000;
+ static inline std::atomic<size_t> next_index = 0;
+ static inline std::vector<std::atomic_bool> object_destroyed = std::vector<std::atomic_bool>(max_objects);
+
+ static TestClass* create() {
+ size_t index = next_index.fetch_add(1, std::memory_order_relaxed);
+ return index < max_objects ? new TestClass(index) : nullptr;
+ }
+
+ static bool is_destroyed(size_t index) {
+ return ObjectStore::object_destroyed[index].load(std::memory_order_relaxed);
+ }
+};
+
+TestClass::~TestClass() {
+ assert(!ObjectStore::is_destroyed(index_));
+ ObjectStore::object_destroyed[index_].store(true, std::memory_order_relaxed);
+}
+
+int main(int, char**) {
+ {
+ // stress test
+ constexpr int num_readers = 3;
+ constexpr int num_writers = 3;
+ constexpr int num_collectors = 2;
+
+ std::atomic<TestClass*> global_ptr = ObjectStore::create();
+
+ std::vector<std::jthread> reader_threads;
+ for (int i = 0; i < num_readers; ++i) {
+ reader_threads.emplace_back([&](std::stop_token stop_token) {
+ while (!stop_token.stop_requested()) {
+ auto& dom = std::rcu_default_domain();
+ dom.lock();
+ auto ptr = global_ptr.load();
+ assert(!ObjectStore::is_destroyed(ptr->index_));
+ dom.unlock();
+ }
+ });
+ }
+
+ std::vector<std::jthread> writer_threads;
+ for (int i = 0; i < num_writers; ++i) {
+ writer_threads.emplace_back([&]() {
+ TestClass* ptr = ObjectStore::create();
+ for (; ptr != nullptr; ptr = ObjectStore::create()) {
+ auto old = global_ptr.exchange(ptr);
+ std::rcu_retire(old);
+ }
+ });
+ }
+
+ std::vector<std::jthread> collector_threads;
+ for (int i = 0; i < num_collectors; ++i) {
+ collector_threads.emplace_back([&](std::stop_token st) {
+ while (!st.stop_requested()) {
+ std::rcu_barrier();
+ }
+ std::rcu_barrier();
+ });
+ }
+
+ reader_threads.clear();
+ writer_threads.clear();
+ collector_threads.clear();
+
+ auto g = global_ptr.exchange(nullptr);
+ delete g;
+
+ for (size_t i = 0; i < ObjectStore::max_objects; ++i) {
+ assert(ObjectStore::is_destroyed(i));
+ }
+ }
+
+ return 0;
+}
diff --git a/libcxx/test/std/thread/saferecl/saferecl.rcu/saferecl.rcu.domain/rcu_retire.pass.cpp b/libcxx/test/std/thread/saferecl/saferecl.rcu/saferecl.rcu.domain/rcu_retire.pass.cpp
index 47490cbb9cc01..5a9d38b2f9673 100644
--- a/libcxx/test/std/thread/saferecl/saferecl.rcu/saferecl.rcu.domain/rcu_retire.pass.cpp
+++ b/libcxx/test/std/thread/saferecl/saferecl.rcu/saferecl.rcu.domain/rcu_retire.pass.cpp
@@ -172,7 +172,7 @@ int main(int, char**) {
dom.unlock();
}
{
- // multiple reader and writer threa constexpr int num_readers = 4;
+ // multiple reader and writer threads
constexpr int num_readers = 4;
constexpr int num_writers = 3;
std::latch unlock_latch(num_writers);
>From 4f0ea1bfcecb52c58ead772713ea0708a5fad6be Mon Sep 17 00:00:00 2001
From: Hui Xie <hui.xie1990 at gmail.com>
Date: Sun, 13 Sep 2026 21:23:35 +0100
Subject: [PATCH 58/65] fix aba in orphon list
---
libcxx/src/experimental/rcu.cpp | 8 ++++++--
1 file changed, 6 insertions(+), 2 deletions(-)
diff --git a/libcxx/src/experimental/rcu.cpp b/libcxx/src/experimental/rcu.cpp
index 0a2c8745a8309..a72f61a431a55 100644
--- a/libcxx/src/experimental/rcu.cpp
+++ b/libcxx/src/experimental/rcu.cpp
@@ -91,7 +91,8 @@ class rcu_domain_impl {
// stage 0 queue is thread local. In case a thread dies with non-empty list in stage 0,
// those nodes will be move into the orphaned_stage0_ on destruction
- rcu_atomic_list_view retired_queue_orphaned_stage0_;
+ rcu_singly_list_view retired_queue_orphaned_stage0_;
+ std::mutex retired_queue_orphaned_stage0_mutex_;
using per_thread_retired_queue_stage0 = thread_local_container<rcu_atomic_list_view>;
@@ -107,7 +108,9 @@ class rcu_domain_impl {
per_thread_retired_queue_stage0::for_each([&working_queue](rcu_atomic_list_view& stage0_list) {
working_queue.splice_back(stage0_list);
});
+ std::unique_lock lk(retired_queue_orphaned_stage0_mutex_);
working_queue.splice_back(retired_queue_orphaned_stage0_);
+ lk.unlock();
// Flip the global phase
auto old_phase = global_reader_phase_.fetch_xor(reader_states::grace_period_phase_mask, std::memory_order_relaxed);
@@ -138,7 +141,8 @@ class rcu_domain_impl {
}
void move_to_orphan_list_on_destruction(rcu_atomic_list_view& stage0_to_be_destroyed) noexcept {
- retired_queue_orphaned_stage0_.splice_front(stage0_to_be_destroyed);
+ std::lock_guard g(retired_queue_orphaned_stage0_mutex_);
+ retired_queue_orphaned_stage0_.splice_back(stage0_to_be_destroyed);
}
public:
>From 3c2dce9424cd3ef7b9d11399126581826fb8bb94 Mon Sep 17 00:00:00 2001
From: Hui Xie <hui.xie1990 at gmail.com>
Date: Mon, 14 Sep 2026 13:01:45 +0100
Subject: [PATCH 59/65] sync tests
---
.../include/rcu/thread_local_container.h | 2 +-
.../rcu_synchronize.pass.cpp | 99 +++++++++++++++++++
.../rcu_synchronize_stress.pass.cpp | 71 +++++++++++++
3 files changed, 171 insertions(+), 1 deletion(-)
create mode 100644 libcxx/test/std/thread/saferecl/saferecl.rcu/saferecl.rcu.domain/rcu_synchronize.pass.cpp
create mode 100644 libcxx/test/std/thread/saferecl/saferecl.rcu/saferecl.rcu.domain/rcu_synchronize_stress.pass.cpp
diff --git a/libcxx/src/experimental/include/rcu/thread_local_container.h b/libcxx/src/experimental/include/rcu/thread_local_container.h
index 2b779623b508b..a9a6aa2e47f0e 100644
--- a/libcxx/src/experimental/include/rcu/thread_local_container.h
+++ b/libcxx/src/experimental/include/rcu/thread_local_container.h
@@ -43,8 +43,8 @@ class thread_local_container {
thread_entry(thread_entry&&) = delete;
~thread_entry() {
- deregister_instance(instance_);
pre_dtor_callback_(instance_);
+ deregister_instance(instance_);
}
};
diff --git a/libcxx/test/std/thread/saferecl/saferecl.rcu/saferecl.rcu.domain/rcu_synchronize.pass.cpp b/libcxx/test/std/thread/saferecl/saferecl.rcu/saferecl.rcu.domain/rcu_synchronize.pass.cpp
new file mode 100644
index 0000000000000..2fd602fffb0bc
--- /dev/null
+++ b/libcxx/test/std/thread/saferecl/saferecl.rcu/saferecl.rcu.domain/rcu_synchronize.pass.cpp
@@ -0,0 +1,99 @@
+//===----------------------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+//
+// UNSUPPORTED: no-threads
+// REQUIRES: std-at-least-c++26
+
+// void rcu_synchronize(rcu_domain& dom = rcu_default_domain()) noexcept;
+
+#include <atomic>
+#include <cassert>
+#include <chrono>
+#include <rcu>
+#include <thread>
+#include <vector>
+
+#include "make_test_thread.h"
+#include "test_macros.h"
+
+static_assert(noexcept(std::rcu_synchronize()));
+static_assert(noexcept(std::rcu_synchronize(std::rcu_default_domain())));
+
+int main(int, char**) {
+ {
+ // rcu_synchronize();
+
+ std::atomic_bool sync_returned = false;
+ std::atomic_bool unlock_called = false;
+ auto& dom = std::rcu_default_domain();
+ dom.lock();
+
+ auto t = support::make_test_jthread([&] {
+ std::rcu_synchronize();
+ assert(unlock_called.load(std::memory_order_relaxed));
+ sync_returned = true;
+ });
+
+ std::this_thread::sleep_for(std::chrono::milliseconds(10));
+ // flag should not be set because rcu_synchronize should block after lock
+ assert(!sync_returned.load(std::memory_order_relaxed));
+
+ unlock_called.store(true, std::memory_order_relaxed);
+ dom.unlock(); // should unblock rcu_synchronize
+ }
+
+ {
+ // rcu_synchronize(dom);
+
+ std::atomic_bool sync_returned = false;
+ std::atomic_bool unlock_called = false;
+ auto& dom = std::rcu_default_domain();
+ dom.lock();
+
+ auto t = support::make_test_jthread([&] {
+ std::rcu_synchronize(dom);
+ assert(unlock_called.load(std::memory_order_relaxed));
+ sync_returned = true;
+ });
+
+ std::this_thread::sleep_for(std::chrono::milliseconds(10));
+ // flag should not be set because rcu_synchronize should block after lock
+ assert(!sync_returned.load(std::memory_order_relaxed));
+
+ unlock_called.store(true, std::memory_order_relaxed);
+ dom.unlock(); // should unblock rcu_synchronize
+ }
+ {
+ // multithreaded synchronize
+ constexpr size_t num_sync_threads = 4;
+ std::array<std::atomic_bool, num_sync_threads> sync_returned{};
+ std::atomic_bool unlock_called = false;
+ auto& dom = std::rcu_default_domain();
+ dom.lock();
+
+ std::vector<std::jthread> sync_threads;
+ for (size_t i = 0; i < num_sync_threads; ++i) {
+ sync_threads.push_back(support::make_test_jthread([&, i] {
+ std::rcu_synchronize();
+ assert(unlock_called.load(std::memory_order_relaxed));
+ sync_returned[i].store(true, std::memory_order_relaxed);
+ }));
+ }
+
+ std::this_thread::sleep_for(std::chrono::milliseconds(10));
+ // flag should not be set because rcu_synchronize should block after lock
+ for (auto&& sync_ret : sync_returned) {
+ assert(!sync_ret.load(std::memory_order_relaxed));
+ }
+
+ unlock_called.store(true, std::memory_order_relaxed);
+ dom.unlock(); // should unblock rcu_synchronize
+ }
+
+ return 0;
+}
diff --git a/libcxx/test/std/thread/saferecl/saferecl.rcu/saferecl.rcu.domain/rcu_synchronize_stress.pass.cpp b/libcxx/test/std/thread/saferecl/saferecl.rcu/saferecl.rcu.domain/rcu_synchronize_stress.pass.cpp
new file mode 100644
index 0000000000000..da29ff7cb1a4e
--- /dev/null
+++ b/libcxx/test/std/thread/saferecl/saferecl.rcu/saferecl.rcu.domain/rcu_synchronize_stress.pass.cpp
@@ -0,0 +1,71 @@
+//===----------------------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+//
+// UNSUPPORTED: no-threads
+// REQUIRES: std-at-least-c++26
+
+// void rcu_synchronize(rcu_domain& dom = rcu_default_domain()) noexcept;
+
+#include <atomic>
+#include <cassert>
+#include <chrono>
+#include <rcu>
+#include <thread>
+#include <vector>
+#include "make_test_thread.h"
+#include "test_macros.h"
+
+static_assert(noexcept(std::rcu_synchronize()));
+static_assert(noexcept(std::rcu_synchronize(std::rcu_default_domain())));
+
+int main(int, char**) {
+ {
+ constexpr size_t num_reader_threads = 4;
+ constexpr size_t num_sync_threads = 4;
+ constexpr int max_reader_iteration = 10000;
+ std::array<std::atomic<int>, num_reader_threads> lock_indicies{};
+ std::array<std::atomic<int>, num_reader_threads> unlock_indicies{};
+ auto& dom = std::rcu_default_domain();
+
+ std::vector<std::jthread> reader_threads;
+ for (size_t i = 0; i < num_reader_threads; ++i) {
+ reader_threads.push_back(support::make_test_jthread([&, i] {
+ for (int count = 0; count < max_reader_iteration; ++count) {
+ dom.lock();
+ lock_indicies[i].store(count, std::memory_order_relaxed);
+ std::this_thread::sleep_for(std::chrono::microseconds(10));
+
+ unlock_indicies[i].store(count, std::memory_order_relaxed);
+ dom.unlock();
+ }
+ }));
+ }
+
+ std::vector<std::jthread> sync_threads;
+ for (size_t i = 0; i < num_sync_threads; ++i) {
+ sync_threads.push_back(support::make_test_jthread([&](std::stop_token st) {
+ while (!st.stop_requested()) {
+ std::array<int, num_reader_threads> pre_sync_reader_count{};
+ for (size_t j = 0; j < num_reader_threads; ++j) {
+ pre_sync_reader_count[i] = lock_indicies[i].load(std::memory_order_relaxed);
+ }
+
+ std::rcu_synchronize();
+ for (size_t j = 0; j < num_reader_threads; ++j) {
+ assert(unlock_indicies[j].load(std::memory_order_relaxed) >= pre_sync_reader_count[j]);
+ }
+ }
+ }));
+ }
+
+ reader_threads.clear();
+ sync_threads.clear();
+ }
+
+ return 0;
+}
>From 4c2b9d91223c0698a0ba77f11b5a0ab17e11137d Mon Sep 17 00:00:00 2001
From: Hui Xie <hui.xie1990 at gmail.com>
Date: Sat, 19 Sep 2026 18:17:13 +0100
Subject: [PATCH 60/65] test
---
.../saferecl/saferecl.rcu/general.pass.cpp | 2 +-
.../read_write_lock_compare.pass.cpp | 2 +-
.../read_write_lock_compare_fair.pass.cpp | 179 ++++++++++++++++++
...d_write_lock_compare_multi_writer.pass.cpp | 2 +-
4 files changed, 182 insertions(+), 3 deletions(-)
create mode 100644 libcxx/test/std/thread/saferecl/saferecl.rcu/read_write_lock_compare_fair.pass.cpp
diff --git a/libcxx/test/std/thread/saferecl/saferecl.rcu/general.pass.cpp b/libcxx/test/std/thread/saferecl/saferecl.rcu/general.pass.cpp
index 1f0275ddc1cbd..6e971af9ae1e4 100644
--- a/libcxx/test/std/thread/saferecl/saferecl.rcu/general.pass.cpp
+++ b/libcxx/test/std/thread/saferecl/saferecl.rcu/general.pass.cpp
@@ -114,5 +114,5 @@ int main(int, char**) {
t3.join();
t4.join();
- return 1;
+ return 0;
}
diff --git a/libcxx/test/std/thread/saferecl/saferecl.rcu/read_write_lock_compare.pass.cpp b/libcxx/test/std/thread/saferecl/saferecl.rcu/read_write_lock_compare.pass.cpp
index 9c357762a1870..1ac44129c67ba 100644
--- a/libcxx/test/std/thread/saferecl/saferecl.rcu/read_write_lock_compare.pass.cpp
+++ b/libcxx/test/std/thread/saferecl/saferecl.rcu/read_write_lock_compare.pass.cpp
@@ -159,5 +159,5 @@ int main(int, char**) {
test_read_write_lock();
std::println("Testing RCU:");
test_rcu();
- return 1;
+ return 0;
}
diff --git a/libcxx/test/std/thread/saferecl/saferecl.rcu/read_write_lock_compare_fair.pass.cpp b/libcxx/test/std/thread/saferecl/saferecl.rcu/read_write_lock_compare_fair.pass.cpp
new file mode 100644
index 0000000000000..6d82c18112e7a
--- /dev/null
+++ b/libcxx/test/std/thread/saferecl/saferecl.rcu/read_write_lock_compare_fair.pass.cpp
@@ -0,0 +1,179 @@
+//===----------------------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+//
+// UNSUPPORTED: no-threads
+// REQUIRES: std-at-least-c++26
+
+// <rcu>
+
+#include <atomic>
+#include <cstddef>
+#include <rcu>
+#include <shared_mutex>
+#include <stop_token>
+#include <thread>
+#include <chrono>
+#include <iostream>
+#include <print>
+#include <string>
+
+#include "make_test_thread.h"
+#include "test_macros.h"
+
+constexpr int num_reader = 4;
+const std::chrono::seconds test_time(10);
+
+struct alignas(128) MyObject : public std::rcu_obj_base<MyObject> {
+ std::string data_;
+ std::atomic<size_t>* destruction_count_;
+
+ inline static int instance_count = 0;
+ MyObject(std::atomic<size_t>& count)
+ : data_(std::to_string(instance_count++) + " instance very very very long string"), destruction_count_(&count) {}
+ ~MyObject() { destruction_count_->fetch_add(1, std::memory_order_relaxed); }
+
+ void doWork() {
+ [[maybe_unused]] auto spin_for = [](std::chrono::microseconds us) {
+ auto start = std::chrono::high_resolution_clock::now();
+ while (std::chrono::high_resolution_clock::now() - start < us)
+ ;
+ };
+ using namespace std::chrono_literals;
+ //spin_for(10us);
+ }
+};
+
+struct alignas(128) MyObjectRWL {
+ std::string data_;
+ std::atomic<size_t>* destruction_count_;
+
+ inline static int instance_count = 0;
+ MyObjectRWL(std::atomic<size_t>& count)
+ : data_(std::to_string(instance_count++) + " instance very very very long string"), destruction_count_(&count) {}
+ ~MyObjectRWL() { destruction_count_->fetch_add(1, std::memory_order_relaxed); }
+MyObjectRWL(const MyObjectRWL&) = default;
+MyObjectRWL& operator=(const MyObjectRWL&) = default;
+
+ void doWork() {
+ [[maybe_unused]] auto spin_for = [](std::chrono::microseconds us) {
+ auto start = std::chrono::high_resolution_clock::now();
+ while (std::chrono::high_resolution_clock::now() - start < us)
+ ;
+ };
+ using namespace std::chrono_literals;
+ //spin_for(10us);
+ }
+};
+
+void test_read_write_lock() {
+ std::atomic<size_t> destruction_count = 0;
+ MyObjectRWL globalObjRWLock(destruction_count);
+ std::shared_mutex globalObjMutex;
+
+ std::vector<std::jthread> readers;
+ readers.reserve(num_reader);
+
+ auto reader_func = [&globalObjRWLock, &globalObjMutex](std::stop_token token) {
+ int read_count = 0;
+ while (!token.stop_requested()) {
+ std::shared_lock<std::shared_mutex> lock(globalObjMutex);
+ globalObjRWLock.doWork();
+ ++read_count;
+ }
+ std::println("Reader thread read {} times", read_count);
+ };
+
+ auto writer_func = [&globalObjRWLock, &globalObjMutex, &destruction_count](std::stop_token token) {
+ int write_count = 0;
+ while (!token.stop_requested()) {
+ std::unique_lock<std::shared_mutex> lock(globalObjMutex);
+ globalObjRWLock = MyObjectRWL(destruction_count);
+ lock.unlock();
+ ++write_count;
+ // std::this_thread::sleep_for(std::chrono::microseconds(100));
+ }
+ std::println("Writer thread wrote {} times", write_count);
+ };
+
+ for (int i = 0; i < num_reader; ++i) {
+ readers.emplace_back(reader_func);
+ }
+ std::jthread writer(writer_func);
+
+ std::this_thread::sleep_for(test_time);
+
+ for (auto& reader : readers) {
+ reader.request_stop();
+ }
+ writer.request_stop();
+ std::println("Writer thread destruction {} times", destruction_count.load());
+}
+
+void test_rcu() {
+ std::atomic<size_t> destruction_count = 0;
+ std::rcu_domain& dom = std::rcu_default_domain();
+ std::atomic<MyObject*> global_obj_rcu = new MyObject(destruction_count);
+
+ std::vector<std::jthread> readers;
+ readers.reserve(num_reader);
+
+ auto reader_func = [&dom, &global_obj_rcu](std::stop_token token) {
+ int read_count = 0;
+ while (!token.stop_requested()) {
+ dom.lock();
+ auto obj = global_obj_rcu.load();
+ obj->doWork();
+ dom.unlock();
+ ++read_count;
+ }
+ std::println("RCU Reader thread read {} times", read_count);
+ };
+
+ auto writer_func = [&global_obj_rcu, &destruction_count](std::stop_token token) {
+ int write_count = 0;
+ while (!token.stop_requested()) {
+ auto newObj = new MyObject(destruction_count);
+ auto oldObj = global_obj_rcu.exchange(newObj);
+ oldObj->retire();
+ ++write_count;
+ // std::this_thread::sleep_for(std::chrono::microseconds(100));
+ }
+ std::println("RCU Writer thread wrote {} times", write_count);
+ };
+
+ auto syncer_func = [&dom](std::stop_token token) {
+ while (!token.stop_requested()) {
+ std::rcu_barrier(dom);
+ std::this_thread::sleep_for(std::chrono::milliseconds(50));
+ }
+ };
+
+ for (int i = 0; i < num_reader; ++i) {
+ readers.emplace_back(reader_func);
+ }
+ std::jthread writer(writer_func);
+ std::jthread syncer(syncer_func);
+
+ std::this_thread::sleep_for(test_time);
+
+ for (auto& reader : readers) {
+ reader.request_stop();
+ }
+ writer.request_stop();
+ syncer.request_stop();
+ std::rcu_barrier(dom);
+ std::println("RCU collector thread destruction {} times", destruction_count.load());
+}
+
+int main(int, char**) {
+ std::println("Testing read-write lock:");
+ test_read_write_lock();
+ std::println("Testing RCU:");
+ test_rcu();
+ return 0;
+}
diff --git a/libcxx/test/std/thread/saferecl/saferecl.rcu/read_write_lock_compare_multi_writer.pass.cpp b/libcxx/test/std/thread/saferecl/saferecl.rcu/read_write_lock_compare_multi_writer.pass.cpp
index 79aa775e8dddf..5ab26cc0928d9 100644
--- a/libcxx/test/std/thread/saferecl/saferecl.rcu/read_write_lock_compare_multi_writer.pass.cpp
+++ b/libcxx/test/std/thread/saferecl/saferecl.rcu/read_write_lock_compare_multi_writer.pass.cpp
@@ -166,5 +166,5 @@ int main(int, char**) {
test_read_write_lock();
std::println("Testing RCU:");
test_rcu();
- return 1;
+ return 0;
}
>From 058ee1b1b0b4958633ebdfeefa5ca29bcccebc06 Mon Sep 17 00:00:00 2001
From: Hui Xie <hui.xie1990 at gmail.com>
Date: Sat, 19 Sep 2026 18:37:48 +0100
Subject: [PATCH 61/65] format and ci
---
libcxx/include/version | 1 -
libcxx/modules/std/rcu.inc | 4 +--
.../src/experimental/include/rcu/rcu_list.h | 30 -------------------
.../include/rcu/thread_local_container.h | 4 ++-
.../saferecl.rcu.base/mandates.verify.cpp | 4 +--
.../rcu.version.compile.pass.cpp | 19 ++++++++++--
.../version.version.compile.pass.cpp | 4 +--
.../saferecl/saferecl.rcu/general.pass.cpp | 22 +++++++-------
.../read_write_lock_compare.pass.cpp | 21 ++++++-------
.../read_write_lock_compare_fair.pass.cpp | 4 +--
...d_write_lock_compare_multi_writer.pass.cpp | 11 ++++---
.../saferecl/saferecl.rcu/retire.pass.cpp | 26 ++++++++--------
12 files changed, 66 insertions(+), 84 deletions(-)
diff --git a/libcxx/include/version b/libcxx/include/version
index 9f20a3d843d89..a49f275f85d4b 100644
--- a/libcxx/include/version
+++ b/libcxx/include/version
@@ -640,7 +640,6 @@ __cpp_lib_void_t 201411L <type_traits>
# define __cpp_lib_ratio 202306L
# undef __cpp_lib_raw_memory_algorithms
# define __cpp_lib_raw_memory_algorithms 202411L
-// # define __cpp_lib_rcu 202306L
# if _LIBCPP_HAS_THREADS && _LIBCPP_HAS_EXPERIMENTAL_RCU
# define __cpp_lib_rcu 202306L
# endif
diff --git a/libcxx/modules/std/rcu.inc b/libcxx/modules/std/rcu.inc
index 0dbad726474e7..90e936e4c4fb1 100644
--- a/libcxx/modules/std/rcu.inc
+++ b/libcxx/modules/std/rcu.inc
@@ -8,7 +8,7 @@
//===----------------------------------------------------------------------===//
export namespace std {
-# if _LIBCPP_STD_VER >= 23
+#if _LIBCPP_STD_VER >= 26
// 2.2.3, class template rcu_obj_base using std::rcu_obj_base;
// 2.2.4, class rcu_domain
using std::rcu_domain;
@@ -16,5 +16,5 @@ export namespace std {
using std::rcu_barrier;
using std::rcu_retire;
using std::rcu_synchronize;
-# endif // _LIBCPP_STD_VER >= 23
+#endif // _LIBCPP_STD_VER >= 26
} // namespace std
diff --git a/libcxx/src/experimental/include/rcu/rcu_list.h b/libcxx/src/experimental/include/rcu/rcu_list.h
index 3c1a882d51e5a..1483a1ea34fb5 100644
--- a/libcxx/src/experimental/include/rcu/rcu_list.h
+++ b/libcxx/src/experimental/include/rcu/rcu_list.h
@@ -96,36 +96,6 @@ class rcu_atomic_list_view {
}
}
}
-
- void splice_front(rcu_atomic_list_view& other) noexcept {
- if (other.entry_.load(std::memory_order_relaxed).head_ == nullptr) {
- return;
- }
-
- entry to_be_inserted = other.entry_.exchange(entry{nullptr, nullptr}, std::memory_order_relaxed);
-
- auto original_next = to_be_inserted.tail_->__next_;
-
- auto expected_entry = entry_.load(std::memory_order_relaxed);
-
- while (true) {
- auto new_entry = [&] {
- if (expected_entry.head_ == nullptr) {
- return to_be_inserted;
- } else {
- to_be_inserted.tail_->__next_ = expected_entry.head_;
- return entry{to_be_inserted.head_, expected_entry.tail_};
- }
- }();
-
- if (entry_.compare_exchange_weak(
- expected_entry, new_entry, std::memory_order_acq_rel, std::memory_order_relaxed)) {
- break;
- } else {
- to_be_inserted.tail_->__next_ = original_next;
- }
- }
- }
};
void rcu_singly_list_view::splice_back(rcu_atomic_list_view& other) noexcept {
diff --git a/libcxx/src/experimental/include/rcu/thread_local_container.h b/libcxx/src/experimental/include/rcu/thread_local_container.h
index a9a6aa2e47f0e..2373980c75ca4 100644
--- a/libcxx/src/experimental/include/rcu/thread_local_container.h
+++ b/libcxx/src/experimental/include/rcu/thread_local_container.h
@@ -38,7 +38,9 @@ class thread_local_container {
Tp instance_;
function_ref<void(Tp&) noexcept> pre_dtor_callback_;
- thread_entry( function_ref<void(Tp&) noexcept> cb) : instance_(), pre_dtor_callback_(cb) { register_instance(instance_); }
+ thread_entry(function_ref<void(Tp&) noexcept> cb) : instance_(), pre_dtor_callback_(cb) {
+ register_instance(instance_);
+ }
thread_entry(const thread_entry&) = delete;
thread_entry(thread_entry&&) = delete;
diff --git a/libcxx/test/libcxx/thread/saferecl/saferecl.rcu/saferecl.rcu.base/mandates.verify.cpp b/libcxx/test/libcxx/thread/saferecl/saferecl.rcu/saferecl.rcu.base/mandates.verify.cpp
index 3796ae4cba1e8..2c26dc91ce0d2 100644
--- a/libcxx/test/libcxx/thread/saferecl/saferecl.rcu/saferecl.rcu.base/mandates.verify.cpp
+++ b/libcxx/test/libcxx/thread/saferecl/saferecl.rcu/saferecl.rcu.base/mandates.verify.cpp
@@ -26,8 +26,8 @@ struct C : std::rcu_obj_base<C, NonDefault> {};
// expected-error-re@*:* {{static assertion failed {{.*}}std::is_default_constructible_v<NonDefault>}}
struct NonMoveAssignable {
- NonMoveAssignable() = default;
- NonMoveAssignable(NonMoveAssignable&&) = default;
+ NonMoveAssignable() = default;
+ NonMoveAssignable(NonMoveAssignable&&) = default;
NonMoveAssignable& operator=(NonMoveAssignable&&) = delete;
};
struct D : std::rcu_obj_base<D, NonMoveAssignable> {};
diff --git a/libcxx/test/std/language.support/support.limits/support.limits.general/rcu.version.compile.pass.cpp b/libcxx/test/std/language.support/support.limits/support.limits.general/rcu.version.compile.pass.cpp
index 62331b8564d48..e5737ee267402 100644
--- a/libcxx/test/std/language.support/support.limits/support.limits.general/rcu.version.compile.pass.cpp
+++ b/libcxx/test/std/language.support/support.limits/support.limits.general/rcu.version.compile.pass.cpp
@@ -48,7 +48,7 @@
# error "__cpp_lib_rcu should not be defined before c++26"
# endif
-#elif TEST_STD_VER > 23
+#elif TEST_STD_VER == 26
# if !defined(_LIBCPP_VERSION) || (_LIBCPP_HAS_THREADS && _LIBCPP_HAS_EXPERIMENTAL_RCU)
# ifndef __cpp_lib_rcu
@@ -63,6 +63,21 @@
# endif
# endif
-#endif // TEST_STD_VER > 23
+#elif TEST_STD_VER > 26
+
+# if !defined(_LIBCPP_VERSION) || (_LIBCPP_HAS_THREADS && _LIBCPP_HAS_EXPERIMENTAL_RCU)
+# ifndef __cpp_lib_rcu
+# error "__cpp_lib_rcu should be defined in c++29"
+# endif
+# if __cpp_lib_rcu != 202306L
+# error "__cpp_lib_rcu should have the value 202306L in c++29"
+# endif
+# else
+# ifdef __cpp_lib_rcu
+# error "__cpp_lib_rcu should not be defined when the requirement '!defined(_LIBCPP_VERSION) || (_LIBCPP_HAS_THREADS && _LIBCPP_HAS_EXPERIMENTAL_RCU)' is not met!"
+# endif
+# endif
+
+#endif // TEST_STD_VER > 26
// clang-format on
diff --git a/libcxx/test/std/language.support/support.limits/support.limits.general/version.version.compile.pass.cpp b/libcxx/test/std/language.support/support.limits/support.limits.general/version.version.compile.pass.cpp
index f42d105d993b4..d9b7291a5857f 100644
--- a/libcxx/test/std/language.support/support.limits/support.limits.general/version.version.compile.pass.cpp
+++ b/libcxx/test/std/language.support/support.limits/support.limits.general/version.version.compile.pass.cpp
@@ -10247,7 +10247,7 @@
# error "__cpp_lib_raw_memory_algorithms should have the value 202411L in c++29"
# endif
-# if !defined(_LIBCPP_VERSION)
+# if !defined(_LIBCPP_VERSION) || (_LIBCPP_HAS_THREADS && _LIBCPP_HAS_EXPERIMENTAL_RCU)
# ifndef __cpp_lib_rcu
# error "__cpp_lib_rcu should be defined in c++29"
# endif
@@ -10256,7 +10256,7 @@
# endif
# else
# ifdef __cpp_lib_rcu
-# error "__cpp_lib_rcu should not be defined because it is unimplemented in libc++!"
+# error "__cpp_lib_rcu should not be defined when the requirement '!defined(_LIBCPP_VERSION) || (_LIBCPP_HAS_THREADS && _LIBCPP_HAS_EXPERIMENTAL_RCU)' is not met!"
# endif
# endif
diff --git a/libcxx/test/std/thread/saferecl/saferecl.rcu/general.pass.cpp b/libcxx/test/std/thread/saferecl/saferecl.rcu/general.pass.cpp
index 6e971af9ae1e4..5755bd244d386 100644
--- a/libcxx/test/std/thread/saferecl/saferecl.rcu/general.pass.cpp
+++ b/libcxx/test/std/thread/saferecl/saferecl.rcu/general.pass.cpp
@@ -38,7 +38,7 @@ int main(int, char**) {
std::this_thread::sleep_for(std::chrono::seconds(1));
log(start, "t1: leaving rcu read-side critical section " + std::to_string(i));
dom.unlock();
- log(start, "t1: printing all reader states");
+ log(start, "t1: printing all reader states");
dom.__debug_print_all_reader_states_in_hex();
}
});
@@ -53,7 +53,7 @@ int main(int, char**) {
std::this_thread::sleep_for(std::chrono::seconds(1));
log(start, "t2: leaving rcu read-side critical section " + std::to_string(i));
dom.unlock();
- log(start, "t2: printing all reader states");
+ log(start, "t2: printing all reader states");
dom.__debug_print_all_reader_states_in_hex();
}
});
@@ -68,7 +68,7 @@ int main(int, char**) {
std::this_thread::sleep_for(std::chrono::seconds(1));
log(start, "t3: leaving rcu read-side critical section " + std::to_string(i));
dom.unlock();
- log(start, "t3: printing all reader states");
+ log(start, "t3: printing all reader states");
dom.__debug_print_all_reader_states_in_hex();
}
});
@@ -82,7 +82,7 @@ int main(int, char**) {
std::this_thread::sleep_for(std::chrono::seconds(1));
log(start, "t4: leaving rcu read-side critical section " + std::to_string(i));
dom.unlock();
- log(start, "t4: printing all reader states");
+ log(start, "t4: printing all reader states");
dom.__debug_print_all_reader_states_in_hex();
}
});
@@ -98,17 +98,15 @@ int main(int, char**) {
}); */
std::this_thread::sleep_for(std::chrono::milliseconds(10));
- for (int i = 0; i < loop_num+5; ++i) {
-
- log(start, "t0: printing all reader states before synchronize");
- std::rcu_default_domain().__debug_print_all_reader_states_in_hex();
- log(start, "t0: calling rcu_synchronize" + std::to_string(i));
- std::rcu_synchronize();
- log(start, "t0: rcu_synchronize returned" + std::to_string(i));
+ for (int i = 0; i < loop_num + 5; ++i) {
+ log(start, "t0: printing all reader states before synchronize");
+ std::rcu_default_domain().__debug_print_all_reader_states_in_hex();
+ log(start, "t0: calling rcu_synchronize" + std::to_string(i));
+ std::rcu_synchronize();
+ log(start, "t0: rcu_synchronize returned" + std::to_string(i));
std::this_thread::sleep_for(std::chrono::seconds(1));
}
-
t1.join();
t2.join();
t3.join();
diff --git a/libcxx/test/std/thread/saferecl/saferecl.rcu/read_write_lock_compare.pass.cpp b/libcxx/test/std/thread/saferecl/saferecl.rcu/read_write_lock_compare.pass.cpp
index 1ac44129c67ba..35cadffc2fbe4 100644
--- a/libcxx/test/std/thread/saferecl/saferecl.rcu/read_write_lock_compare.pass.cpp
+++ b/libcxx/test/std/thread/saferecl/saferecl.rcu/read_write_lock_compare.pass.cpp
@@ -28,17 +28,17 @@
constexpr int num_reader = 4;
const std::chrono::seconds test_time(10);
-
struct alignas(128) MyObject : public std::rcu_obj_base<MyObject> {
std::string data_;
std::atomic<size_t>& destruction_count_;
inline static int instance_count = 0;
- MyObject(std::atomic<size_t>& count) : data_(std::to_string(instance_count++) + " instance very very very long string"), destruction_count_(count) {}
- ~MyObject() {destruction_count_.fetch_add(1, std::memory_order_relaxed);}
+ MyObject(std::atomic<size_t>& count)
+ : data_(std::to_string(instance_count++) + " instance very very very long string"), destruction_count_(count) {}
+ ~MyObject() { destruction_count_.fetch_add(1, std::memory_order_relaxed); }
void doWork() {
- [[maybe_unused]]auto spin_for = [](std::chrono::microseconds us) {
+ [[maybe_unused]] auto spin_for = [](std::chrono::microseconds us) {
auto start = std::chrono::high_resolution_clock::now();
while (std::chrono::high_resolution_clock::now() - start < us)
;
@@ -48,16 +48,14 @@ struct alignas(128) MyObject : public std::rcu_obj_base<MyObject> {
}
};
-
void test_read_write_lock() {
std::atomic<size_t> destruction_count = 0;
- MyObject* globalObjRWLock = new MyObject(destruction_count);
+ MyObject* globalObjRWLock = new MyObject(destruction_count);
std::shared_mutex globalObjMutex;
std::vector<std::jthread> readers;
readers.reserve(num_reader);
-
auto reader_func = [&globalObjRWLock, &globalObjMutex](std::stop_token token) {
int read_count = 0;
while (!token.stop_requested()) {
@@ -78,7 +76,7 @@ void test_read_write_lock() {
lock.unlock();
delete oldObj;
++write_count;
- // std::this_thread::sleep_for(std::chrono::microseconds(100));
+ // std::this_thread::sleep_for(std::chrono::microseconds(100));
}
std::println("Writer thread wrote {} times", write_count);
};
@@ -99,13 +97,12 @@ void test_read_write_lock() {
void test_rcu() {
std::atomic<size_t> destruction_count = 0;
- std::rcu_domain& dom = std::rcu_default_domain();
+ std::rcu_domain& dom = std::rcu_default_domain();
std::atomic<MyObject*> global_obj_rcu = new MyObject(destruction_count);
std::vector<std::jthread> readers;
readers.reserve(num_reader);
-
auto reader_func = [&dom, &global_obj_rcu](std::stop_token token) {
int read_count = 0;
while (!token.stop_requested()) {
@@ -125,14 +122,14 @@ void test_rcu() {
auto oldObj = global_obj_rcu.exchange(newObj);
oldObj->retire();
++write_count;
- // std::this_thread::sleep_for(std::chrono::microseconds(100));
+ // std::this_thread::sleep_for(std::chrono::microseconds(100));
}
std::println("RCU Writer thread wrote {} times", write_count);
};
auto syncer_func = [&dom](std::stop_token token) {
while (!token.stop_requested()) {
- std::rcu_barrier(dom);
+ std::rcu_barrier(dom);
std::this_thread::sleep_for(std::chrono::milliseconds(50));
}
};
diff --git a/libcxx/test/std/thread/saferecl/saferecl.rcu/read_write_lock_compare_fair.pass.cpp b/libcxx/test/std/thread/saferecl/saferecl.rcu/read_write_lock_compare_fair.pass.cpp
index 6d82c18112e7a..349fed456453a 100644
--- a/libcxx/test/std/thread/saferecl/saferecl.rcu/read_write_lock_compare_fair.pass.cpp
+++ b/libcxx/test/std/thread/saferecl/saferecl.rcu/read_write_lock_compare_fair.pass.cpp
@@ -56,8 +56,8 @@ struct alignas(128) MyObjectRWL {
MyObjectRWL(std::atomic<size_t>& count)
: data_(std::to_string(instance_count++) + " instance very very very long string"), destruction_count_(&count) {}
~MyObjectRWL() { destruction_count_->fetch_add(1, std::memory_order_relaxed); }
-MyObjectRWL(const MyObjectRWL&) = default;
-MyObjectRWL& operator=(const MyObjectRWL&) = default;
+ MyObjectRWL(const MyObjectRWL&) = default;
+ MyObjectRWL& operator=(const MyObjectRWL&) = default;
void doWork() {
[[maybe_unused]] auto spin_for = [](std::chrono::microseconds us) {
diff --git a/libcxx/test/std/thread/saferecl/saferecl.rcu/read_write_lock_compare_multi_writer.pass.cpp b/libcxx/test/std/thread/saferecl/saferecl.rcu/read_write_lock_compare_multi_writer.pass.cpp
index 5ab26cc0928d9..e230379a61e01 100644
--- a/libcxx/test/std/thread/saferecl/saferecl.rcu/read_write_lock_compare_multi_writer.pass.cpp
+++ b/libcxx/test/std/thread/saferecl/saferecl.rcu/read_write_lock_compare_multi_writer.pass.cpp
@@ -35,17 +35,16 @@ struct alignas(128) MyObject : public std::rcu_obj_base<MyObject> {
MyObject() : data_(std::to_string(instance_count++) + " instance very very very long string") {}
void doWork() {
- [[maybe_unused]] auto spin_for = [](std::chrono::microseconds us) {
+ [[maybe_unused]] auto spin_for = [](std::chrono::microseconds us) {
auto start = std::chrono::high_resolution_clock::now();
while (std::chrono::high_resolution_clock::now() - start < us)
;
};
using namespace std::chrono_literals;
- // spin_for(10us);
+ // spin_for(10us);
}
};
-
void test_read_write_lock() {
MyObject* globalObjRWLock = new MyObject();
std::shared_mutex globalObjMutex;
@@ -76,7 +75,7 @@ void test_read_write_lock() {
lock.unlock();
delete oldObj;
++write_count;
- // std::this_thread::sleep_for(std::chrono::microseconds(100));
+ // std::this_thread::sleep_for(std::chrono::microseconds(100));
}
std::println("Writer thread wrote {} times", write_count);
};
@@ -100,7 +99,7 @@ void test_read_write_lock() {
}
void test_rcu() {
- std::rcu_domain& dom = std::rcu_default_domain();
+ std::rcu_domain& dom = std::rcu_default_domain();
std::atomic<MyObject*> global_obj_rcu = new MyObject();
std::vector<std::jthread> readers;
@@ -128,7 +127,7 @@ void test_rcu() {
auto oldObj = global_obj_rcu.exchange(newObj);
oldObj->retire();
++write_count;
- // std::this_thread::sleep_for(std::chrono::microseconds(100));
+ // std::this_thread::sleep_for(std::chrono::microseconds(100));
}
std::println("RCU Writer thread wrote {} times", write_count);
};
diff --git a/libcxx/test/std/thread/saferecl/saferecl.rcu/retire.pass.cpp b/libcxx/test/std/thread/saferecl/saferecl.rcu/retire.pass.cpp
index 1573cafc03a60..922808c5afa5b 100644
--- a/libcxx/test/std/thread/saferecl/saferecl.rcu/retire.pass.cpp
+++ b/libcxx/test/std/thread/saferecl/saferecl.rcu/retire.pass.cpp
@@ -35,9 +35,7 @@ struct MyObject : public std::rcu_obj_base<MyObject> {
inline static int instance_count = 0;
MyObject() : data_(std::to_string(instance_count++) + " instance very very very long string") {}
- ~MyObject() {
- std::println(std::cout, "MyObject {} destructor called", data_);
- }
+ ~MyObject() { std::println(std::cout, "MyObject {} destructor called", data_); }
};
std::atomic<MyObject*> global_obj = nullptr;
@@ -52,8 +50,9 @@ int main(int, char**) {
auto obj = global_obj.load();
log(start, "t1: reading: " + (obj ? obj->data_ : "nullptr"));
std::this_thread::sleep_for(std::chrono::seconds(1));
- log(start, "t1: leaving rcu read-side critical section " + std::to_string(i) + " with object " +
- (obj ? obj->data_ : "nullptr"));
+ log(start,
+ "t1: leaving rcu read-side critical section " + std::to_string(i) + " with object " +
+ (obj ? obj->data_ : "nullptr"));
dom.unlock();
log(start, "t1: printing all reader states");
dom.__debug_print_all_reader_states_in_hex();
@@ -69,8 +68,9 @@ int main(int, char**) {
auto obj = global_obj.load();
log(start, "t2: reading: " + (obj ? obj->data_ : "nullptr"));
std::this_thread::sleep_for(std::chrono::seconds(1));
- log(start, "t2: leaving rcu read-side critical section " + std::to_string(i) + " with object " +
- (obj ? obj->data_ : "nullptr"));
+ log(start,
+ "t2: leaving rcu read-side critical section " + std::to_string(i) + " with object " +
+ (obj ? obj->data_ : "nullptr"));
dom.unlock();
log(start, "t2: printing all reader states");
dom.__debug_print_all_reader_states_in_hex();
@@ -86,8 +86,9 @@ int main(int, char**) {
auto obj = global_obj.load();
log(start, "t3: reading: " + (obj ? obj->data_ : "nullptr"));
std::this_thread::sleep_for(std::chrono::seconds(1));
- log(start, "t3: leaving rcu read-side critical section " + std::to_string(i) + " with object " +
- (obj ? obj->data_ : "nullptr"));
+ log(start,
+ "t3: leaving rcu read-side critical section " + std::to_string(i) + " with object " +
+ (obj ? obj->data_ : "nullptr"));
dom.unlock();
log(start, "t3: printing all reader states");
dom.__debug_print_all_reader_states_in_hex();
@@ -102,8 +103,9 @@ int main(int, char**) {
auto obj = global_obj.load();
log(start, "t4: reading: " + (obj ? obj->data_ : "nullptr"));
std::this_thread::sleep_for(std::chrono::seconds(1));
- log(start, "t4: leaving rcu read-side critical section " + std::to_string(i) + " with object " +
- (obj ? obj->data_ : "nullptr"));
+ log(start,
+ "t4: leaving rcu read-side critical section " + std::to_string(i) + " with object " +
+ (obj ? obj->data_ : "nullptr"));
dom.unlock();
log(start, "t4: printing all reader states");
dom.__debug_print_all_reader_states_in_hex();
@@ -146,5 +148,5 @@ int main(int, char**) {
t3.join();
t4.join();
- return 1;
+ return 0;
}
>From 9794258e3d05d3a86dc0175a172c14646ab74ad4 Mon Sep 17 00:00:00 2001
From: Hui Xie <hui.xie1990 at gmail.com>
Date: Sat, 19 Sep 2026 18:54:53 +0100
Subject: [PATCH 62/65] silence 16 bytes atomci warning
---
libcxx/src/experimental/rcu.cpp | 7 ++++++-
1 file changed, 6 insertions(+), 1 deletion(-)
diff --git a/libcxx/src/experimental/rcu.cpp b/libcxx/src/experimental/rcu.cpp
index a72f61a431a55..e344794dfdf0d 100644
--- a/libcxx/src/experimental/rcu.cpp
+++ b/libcxx/src/experimental/rcu.cpp
@@ -6,9 +6,14 @@
//
//===----------------------------------------------------------------------===//
+#include <__config>
+
+_LIBCPP_DIAGNOSTIC_PUSH
+_LIBCPP_CLANG_DIAGNOSTIC_IGNORED("-Watomic-alignment")
#include <atomic>
+_LIBCPP_DIAGNOSTIC_POP
+
#include <cstdint>
-#include <memory>
#include <mutex>
#include <rcu>
>From 3c1b6949024fb61425ffe25e9d5589b0c2869f4c Mon Sep 17 00:00:00 2001
From: Hui Xie <hui.xie1990 at gmail.com>
Date: Sat, 19 Sep 2026 20:53:57 +0100
Subject: [PATCH 63/65] CI
---
libcxx/include/__rcu/rcu_domain.h | 45 +++++++++++--------
libcxx/include/__rcu/rcu_obj_base.h | 1 +
libcxx/modules/std/rcu.inc | 2 +-
.../test/libcxx/transitive_includes/cxx03.csv | 1 +
.../test/libcxx/transitive_includes/cxx11.csv | 1 +
.../test/libcxx/transitive_includes/cxx14.csv | 1 +
.../test/libcxx/transitive_includes/cxx17.csv | 1 +
.../test/libcxx/transitive_includes/cxx20.csv | 1 +
.../test/libcxx/transitive_includes/cxx23.csv | 1 +
.../test/libcxx/transitive_includes/cxx26.csv | 4 ++
.../test/libcxx/transitive_includes/cxx29.csv | 4 ++
11 files changed, 43 insertions(+), 19 deletions(-)
diff --git a/libcxx/include/__rcu/rcu_domain.h b/libcxx/include/__rcu/rcu_domain.h
index 7e0e94461a15a..d58edbd7148e3 100644
--- a/libcxx/include/__rcu/rcu_domain.h
+++ b/libcxx/include/__rcu/rcu_domain.h
@@ -19,15 +19,29 @@
# pragma GCC system_header
#endif
+_LIBCPP_PUSH_MACROS
+#include <__undef_macros>
+
_LIBCPP_BEGIN_NAMESPACE_STD
_LIBCPP_BEGIN_EXPLICIT_ABI_ANNOTATIONS
#if _LIBCPP_STD_VER >= 26 && _LIBCPP_HAS_THREADS && _LIBCPP_HAS_EXPERIMENTAL_RCU
+class rcu_domain;
+
+_LIBCPP_EXPORTED_FROM_ABI rcu_domain& rcu_default_domain() noexcept;
+
+_LIBCPP_EXPORTED_FROM_ABI void rcu_synchronize(rcu_domain& __dom = rcu_default_domain()) noexcept;
+
+_LIBCPP_EXPORTED_FROM_ABI void rcu_barrier(rcu_domain& __dom = rcu_default_domain()) noexcept;
+
+template <class _Tp, class _Dp = default_delete<_Tp>>
+_LIBCPP_HIDE_FROM_ABI void rcu_retire(_Tp*, _Dp = _Dp(), rcu_domain& __dom = rcu_default_domain());
+
struct __rcu_node {
- using __cb_type = void(__rcu_node*);
- __cb_type* __callback_ = [](__rcu_node*) {};
- __rcu_node* __next_ = nullptr;
+ using __cb_type _LIBCPP_NODEBUG = void(__rcu_node*);
+ __cb_type* __callback_ = [](__rcu_node*) {};
+ __rcu_node* __next_ = nullptr;
};
template <class _Tp, class _Deleter>
@@ -57,13 +71,10 @@ class _LIBCPP_EXPORTED_FROM_ABI rcu_domain {
friend void rcu_synchronize(rcu_domain&) noexcept;
friend void rcu_barrier(rcu_domain&) noexcept;
- static rcu_domain& __rcu_default_domain() noexcept;
+ template <class _Tp, class _Dp >
+ friend void rcu_retire(_Tp*, _Dp, rcu_domain&);
- template <class _Tp, class _Dp>
- friend _LIBCPP_HIDE_FROM_ABI void __rcu_retire_hidden_friend(_Tp* __tp, _Dp __deleter, rcu_domain& __dom) {
- auto* __node = new __rcu_node_with_deleter<_Tp, _Dp>(__tp, std::move(__deleter));
- __dom.__retire(__node);
- }
+ static rcu_domain& __rcu_default_domain() noexcept;
rcu_domain();
@@ -86,17 +97,13 @@ class _LIBCPP_EXPORTED_FROM_ABI rcu_domain {
void unlock() noexcept;
};
-_LIBCPP_EXPORTED_FROM_ABI rcu_domain& rcu_default_domain() noexcept;
-
-_LIBCPP_EXPORTED_FROM_ABI void rcu_synchronize(rcu_domain& __dom = rcu_default_domain()) noexcept;
-
-_LIBCPP_EXPORTED_FROM_ABI void rcu_barrier(rcu_domain& __dom = rcu_default_domain()) noexcept;
-
-template <class _Tp, class _Dp = default_delete<_Tp>>
-_LIBCPP_HIDE_FROM_ABI void rcu_retire(_Tp* __tp, _Dp __deleter = _Dp(), rcu_domain& __dom = rcu_default_domain()) {
+template <class _Tp, class _Dp >
+_LIBCPP_HIDE_FROM_ABI void rcu_retire(_Tp* __tp, _Dp __deleter, rcu_domain& __dom) {
static_assert(std::is_move_constructible_v<_Dp>);
static_assert(requires(_Dp __dp, _Tp* __ptr) { __dp(__ptr); }, "Deleter must be callable with a pointer");
- __rcu_retire_hidden_friend(__tp, std::move(__deleter), __dom);
+
+ auto* __node = new __rcu_node_with_deleter<_Tp, _Dp>(__tp, std::move(__deleter));
+ __dom.__retire(__node);
}
#endif // _LIBCPP_STD_VER >= 26 && _LIBCPP_HAS_THREADS && _LIBCPP_HAS_EXPERIMENTAL_RCU
@@ -104,4 +111,6 @@ _LIBCPP_HIDE_FROM_ABI void rcu_retire(_Tp* __tp, _Dp __deleter = _Dp(), rcu_doma
_LIBCPP_END_EXPLICIT_ABI_ANNOTATIONS
_LIBCPP_END_NAMESPACE_STD
+_LIBCPP_POP_MACROS
+
#endif // _LIBCPP___RCU_RCU_DOMAIN_H
diff --git a/libcxx/include/__rcu/rcu_obj_base.h b/libcxx/include/__rcu/rcu_obj_base.h
index 056f72628a9a5..4a154a2a81edd 100644
--- a/libcxx/include/__rcu/rcu_obj_base.h
+++ b/libcxx/include/__rcu/rcu_obj_base.h
@@ -14,6 +14,7 @@
#include <__memory/unique_ptr.h> // for default_delete
#include <__rcu/rcu_domain.h>
#include <__type_traits/is_assignable.h>
+#include <__type_traits/is_base_of.h>
#include <__type_traits/is_constructible.h>
#include <__utility/move.h>
diff --git a/libcxx/modules/std/rcu.inc b/libcxx/modules/std/rcu.inc
index 90e936e4c4fb1..40d3cfbe18ac7 100644
--- a/libcxx/modules/std/rcu.inc
+++ b/libcxx/modules/std/rcu.inc
@@ -12,7 +12,7 @@ export namespace std {
// 2.2.3, class template rcu_obj_base using std::rcu_obj_base;
// 2.2.4, class rcu_domain
using std::rcu_domain;
- using std::rcu_default_domain();
+ using std::rcu_default_domain;
using std::rcu_barrier;
using std::rcu_retire;
using std::rcu_synchronize;
diff --git a/libcxx/test/libcxx/transitive_includes/cxx03.csv b/libcxx/test/libcxx/transitive_includes/cxx03.csv
index ad2f90062c078..91518d3c4d512 100644
--- a/libcxx/test/libcxx/transitive_includes/cxx03.csv
+++ b/libcxx/test/libcxx/transitive_includes/cxx03.csv
@@ -598,6 +598,7 @@ ranges version
ratio climits
ratio cstdint
ratio version
+rcu version
regex cctype
regex climits
regex clocale
diff --git a/libcxx/test/libcxx/transitive_includes/cxx11.csv b/libcxx/test/libcxx/transitive_includes/cxx11.csv
index 672df0a488c9e..a6814ca897696 100644
--- a/libcxx/test/libcxx/transitive_includes/cxx11.csv
+++ b/libcxx/test/libcxx/transitive_includes/cxx11.csv
@@ -600,6 +600,7 @@ ranges version
ratio climits
ratio cstdint
ratio version
+rcu version
regex cctype
regex climits
regex clocale
diff --git a/libcxx/test/libcxx/transitive_includes/cxx14.csv b/libcxx/test/libcxx/transitive_includes/cxx14.csv
index f496a00270386..f258c219c4141 100644
--- a/libcxx/test/libcxx/transitive_includes/cxx14.csv
+++ b/libcxx/test/libcxx/transitive_includes/cxx14.csv
@@ -604,6 +604,7 @@ ranges version
ratio climits
ratio cstdint
ratio version
+rcu version
regex cctype
regex climits
regex clocale
diff --git a/libcxx/test/libcxx/transitive_includes/cxx17.csv b/libcxx/test/libcxx/transitive_includes/cxx17.csv
index e0a92259202c4..0093dd1a28b88 100644
--- a/libcxx/test/libcxx/transitive_includes/cxx17.csv
+++ b/libcxx/test/libcxx/transitive_includes/cxx17.csv
@@ -676,6 +676,7 @@ ranges version
ratio climits
ratio cstdint
ratio version
+rcu version
regex cctype
regex climits
regex clocale
diff --git a/libcxx/test/libcxx/transitive_includes/cxx20.csv b/libcxx/test/libcxx/transitive_includes/cxx20.csv
index 3c5e9439f100c..3663a36618be0 100644
--- a/libcxx/test/libcxx/transitive_includes/cxx20.csv
+++ b/libcxx/test/libcxx/transitive_includes/cxx20.csv
@@ -757,6 +757,7 @@ ranges version
ratio climits
ratio cstdint
ratio version
+rcu version
regex cctype
regex climits
regex clocale
diff --git a/libcxx/test/libcxx/transitive_includes/cxx23.csv b/libcxx/test/libcxx/transitive_includes/cxx23.csv
index 8c03f1c3c2251..1d4f2333c9355 100644
--- a/libcxx/test/libcxx/transitive_includes/cxx23.csv
+++ b/libcxx/test/libcxx/transitive_includes/cxx23.csv
@@ -800,6 +800,7 @@ ranges version
ratio climits
ratio cstdint
ratio version
+rcu version
regex cctype
regex climits
regex clocale
diff --git a/libcxx/test/libcxx/transitive_includes/cxx26.csv b/libcxx/test/libcxx/transitive_includes/cxx26.csv
index 800d65aed4704..5912efac4282f 100644
--- a/libcxx/test/libcxx/transitive_includes/cxx26.csv
+++ b/libcxx/test/libcxx/transitive_includes/cxx26.csv
@@ -782,6 +782,10 @@ ranges version
ratio climits
ratio cstdint
ratio version
+rcu cstdint
+rcu cstring
+rcu limits
+rcu version
regex cctype
regex climits
regex clocale
diff --git a/libcxx/test/libcxx/transitive_includes/cxx29.csv b/libcxx/test/libcxx/transitive_includes/cxx29.csv
index 800d65aed4704..5912efac4282f 100644
--- a/libcxx/test/libcxx/transitive_includes/cxx29.csv
+++ b/libcxx/test/libcxx/transitive_includes/cxx29.csv
@@ -782,6 +782,10 @@ ranges version
ratio climits
ratio cstdint
ratio version
+rcu cstdint
+rcu cstring
+rcu limits
+rcu version
regex cctype
regex climits
regex clocale
>From 378ff473878d07fce14e2bcb5b6d552ff33d786e Mon Sep 17 00:00:00 2001
From: Hui Xie <hui.xie1990 at gmail.com>
Date: Sat, 19 Sep 2026 20:58:10 +0100
Subject: [PATCH 64/65] CI
---
libcxx/include/CMakeLists.txt | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/libcxx/include/CMakeLists.txt b/libcxx/include/CMakeLists.txt
index b0282cc6198c0..a63841660f16d 100644
--- a/libcxx/include/CMakeLists.txt
+++ b/libcxx/include/CMakeLists.txt
@@ -1108,8 +1108,8 @@ set(files
random
ranges
ratio
- regex
rcu
+ regex
scoped_allocator
semaphore
set
>From bce3e57207c324e422adf32e27af53d2d206bf85 Mon Sep 17 00:00:00 2001
From: Hui Xie <hui.xie1990 at gmail.com>
Date: Sat, 19 Sep 2026 21:12:32 +0100
Subject: [PATCH 65/65] ci
---
libcxx/modules/std/rcu.inc | 4 ++--
1 file changed, 2 insertions(+), 2 deletions(-)
diff --git a/libcxx/modules/std/rcu.inc b/libcxx/modules/std/rcu.inc
index 40d3cfbe18ac7..75adb1c25704f 100644
--- a/libcxx/modules/std/rcu.inc
+++ b/libcxx/modules/std/rcu.inc
@@ -11,9 +11,9 @@ export namespace std {
#if _LIBCPP_STD_VER >= 26
// 2.2.3, class template rcu_obj_base using std::rcu_obj_base;
// 2.2.4, class rcu_domain
- using std::rcu_domain;
- using std::rcu_default_domain;
using std::rcu_barrier;
+ using std::rcu_default_domain;
+ using std::rcu_domain;
using std::rcu_retire;
using std::rcu_synchronize;
#endif // _LIBCPP_STD_VER >= 26
More information about the libcxx-commits
mailing list