[libcxx-commits] [libcxx] [libc++] P2530R3: Implementation of std::hazard_pointer (PR #218218)
Nikita Grivin via libcxx-commits
libcxx-commits at lists.llvm.org
Thu Sep 17 02:51:50 PDT 2026
================
@@ -0,0 +1,916 @@
+//===----------------------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#include <__hazard_pointer/domain.h>
+
+#include <__assert>
+#include <__atomic/atomic.h>
+#include <__atomic/fence.h>
+#include <__atomic/memory_order.h>
+#include <__bit/bit_ceil.h>
+#include <__chrono/duration.h>
+#include <__chrono/steady_clock.h>
+#include <__mutex/once_flag.h>
+#include <__thread/support.h>
+#include <cstddef>
+#include <cstdint>
+#include <new>
+
+#if _LIBCPP_HAZARD_POINTER_ASYMMETRIC_FENCE
+# if defined(__linux__)
+# include <cerrno>
+# include <sys/mman.h>
+# include <sys/syscall.h>
+# include <unistd.h>
+# elif defined(_WIN32)
+# define WIN32_LEAN_AND_MEAN
+# define VC_EXTRA_LEAN
+# include <windows.h>
+# endif
+#endif
+
+// The engine behind <hazard_pointer> implements the single default domain the Standard specifies.
+// Nothing in this file is ABI: only the three functions at the bottom, and the two structs in
+// <__hazard_pointer/domain.h> they take/return, are.
+//
+// Terminology: a *record* is a hazard pointer (its slot holds the protected node address); a *node* is
+// the header embedded in every hazard-protectable object; the *available list* holds free records; each
+// thread keeps a small cache of free records; retired nodes wait in eight sharded lists until a
+// reclamation pass -- run inline by whichever thread's retire() crosses the threshold -- deletes the
+// unprotected ones.
+
+#if defined(__SANITIZE_THREAD__)
+# define _LIBCPP_HAZARD_POINTER_TSAN 1
+#elif defined(__has_feature)
+# if __has_feature(thread_sanitizer)
+# define _LIBCPP_HAZARD_POINTER_TSAN 1
+# endif
+#endif
+#ifndef _LIBCPP_HAZARD_POINTER_TSAN
+# define _LIBCPP_HAZARD_POINTER_TSAN 0
+#endif
+
+#if defined(__ELF__) && __has_feature(cxx_thread_local)
+# define _LIBCPP_HAZARD_POINTER_THREAD_LOCAL 1
+#else
+# define _LIBCPP_HAZARD_POINTER_THREAD_LOCAL 0
+#endif
+
+_LIBCPP_BEGIN_NAMESPACE_STD
+
+namespace {
+
+// Configuration
+
+constexpr int __retired_count_threshold = 1000; // a pass runs once this many objects are retired ...
+constexpr int __threshold_multiplier = 2; // ... or twice the record count, whichever is larger ...
+constexpr uintptr_t __sync_time_period_s = 2; // ... or two seconds after the previous pass
+constexpr int __shard_count = 8;
+constexpr int __shard_mask = __shard_count - 1;
+constexpr int __ignored_low_bits = 8;
+constexpr uintptr_t __lock_bit = 1;
+constexpr uint8_t __thread_cache_capacity = 9;
+// Records are laid out one per cache line (see __hazard_pointer_record). Where over-aligned allocation is
+// not available, std::__libcpp_allocate can only promise fundamental alignment, so fall back to that.
+#if !_LIBCPP_HAS_ALIGNED_ALLOCATION
+constexpr size_t __cache_line_size = alignof(max_align_t);
+#elif defined(__GCC_DESTRUCTIVE_SIZE)
+constexpr size_t __cache_line_size = __GCC_DESTRUCTIVE_SIZE;
+#else
+constexpr size_t __cache_line_size = 64;
+#endif
+
+// 64-bit mixer shared by the shard function and the protected set.
+inline size_t __mix_pointer(uintptr_t __x) noexcept {
+ if constexpr (sizeof(uintptr_t) == 8) {
+ __x ^= __x >> 32;
+ __x *= 0x9E3779B97F4A7C15ULL;
+ __x ^= __x >> 29;
+ } else {
+ __x ^= __x >> 16;
+ __x *= 0x9E3779B1U;
+ __x ^= __x >> 15;
+ }
+ return static_cast<size_t>(__x);
+}
+
+// Record
+
+// One cache line per record: records of different threads must not share a line (every protect() writes
+// to it); targets without over-aligned allocation fall back to fundamental alignment. Records are created
+// by the domain's grow array and never destroyed.
+struct alignas(__cache_line_size) __hazard_pointer_record : __hazard_pointer_slot {
+ __hazard_pointer_record* __next_available_ = nullptr; // available-list link; meaningful only while free
+ __hazard_pointer_record() noexcept : __hazard_pointer_slot{nullptr} {}
+};
+
+// Grow-only array
+
+// Concurrent grow-only array with wait-free indexed access and reference stability. operator[](i) is
+// valid for every i: if i >= __size() the array grows to bit_ceil(i + 1), default-constructing every
+// element up to the new capacity. Growth is a race decided by one CAS; losers destroy what they built.
+// Each growth round is one allocation holding {header}{_Tp* list[capacity]}{_Tp slab[capacity - base]};
+// older rounds stay alive on a chain, so element addresses never change. __size() is *not* monotonic -- a
+// delayed store from a smaller round can lower it after a larger one published -- but it is always a lower
+// bound on what has been constructed, which is all operator[]'s fast path needs. Never destroyed (the domain
+// is immortal). _Tp must be nothrow default constructible.
+template <class _Tp>
+class __grow_array {
+ struct __round {
+ __round* __next_ = nullptr; // the previous (smaller) round; owns elements [0, __next_->__size_)
+ size_t __size_ = 0; // > __next_->__size_
+ _Tp** __list() noexcept { return reinterpret_cast<_Tp**>(reinterpret_cast<byte*>(this) + sizeof(__round)); }
+ };
+
+ atomic<size_t> __size_{0};
+ atomic<__round*> __array_{nullptr};
+
+ static constexpr size_t __alignment() noexcept {
+ return alignof(_Tp) > alignof(max_align_t) ? alignof(_Tp) : alignof(max_align_t);
+ }
+ static constexpr size_t __ceil_to(size_t __value, size_t __alignment) noexcept {
+ return (__value + __alignment - 1) / __alignment * __alignment;
+ }
+ static constexpr size_t __bytes(size_t __capacity, size_t __base) noexcept {
+ return __ceil_to(sizeof(__round) + __capacity * sizeof(_Tp*), __alignment()) +
+ __ceil_to((__capacity - __base) * sizeof(_Tp), __alignment());
+ }
+ static _Tp* __slab(__round* __curr) noexcept {
+ uintptr_t __list_end = reinterpret_cast<uintptr_t>(__curr->__list() + __curr->__size_);
+ return reinterpret_cast<_Tp*>(__ceil_to(static_cast<size_t>(__list_end), __alignment()));
+ }
+
+ // Builds a round of __capacity elements on top of __next; returns nullptr (and updates __next) when
+ // another thread published a round meanwhile. May throw bad_alloc.
+ __round* __new_round(size_t __capacity, __round*& __next) {
+ size_t __base = __next ? __next->__size_ : 0;
+ void* __raw = std::__libcpp_allocate<byte>(__element_count(__bytes(__capacity, __base)), __alignment());
+ __round* __curr = ::new (__raw) __round{};
+ __curr->__next_ = __next;
+ __curr->__size_ = __capacity;
+ _Tp* __slab_ptr = __slab(__curr);
+ _Tp** __list = __curr->__list();
+ for (size_t __i = 0; __i < __base; ++__i)
+ __list[__i] = __next->__list()[__i];
+ for (size_t __i = __base; __i < __capacity; ++__i)
+ __list[__i] = nullptr;
+ for (size_t __i = __base; __i < __capacity; ++__i) {
+ __round* __seen = __array_.load(memory_order_acquire);
+ if (__seen != __next) { // race lost early: stop building
+ __next = __seen;
+ __del_round(__curr);
+ return nullptr;
+ }
+ __list[__i] = ::new (static_cast<void*>(&__slab_ptr[__i - __base])) _Tp();
+ }
+ return __curr;
+ }
+
+ void __del_round(__round* __curr) noexcept {
+ size_t __built = __curr->__size_;
+ __round* __next = __curr->__next_;
+ size_t __base = __next ? __next->__size_ : 0;
+ size_t __size = __bytes(__curr->__size_, __base);
+ _Tp** __list = __curr->__list();
+ while (__built > __base && __list[__built - 1] == nullptr) // never-constructed tail
+ --__built;
+ for (size_t __i = 0; __i < __built - __base; ++__i)
+ __list[__built - 1 - __i]->~_Tp();
+ std::__libcpp_deallocate<byte>(reinterpret_cast<byte*>(__curr), __element_count(__size), __alignment());
+ }
+
+ __round* __at_slow(size_t __index) {
+ // Optimistic concurrency: no mutex, so element constructors may themselves use the array.
+ __round* __p = __array_.load(memory_order_acquire);
+ size_t __capacity = std::__bit_ceil(__index + 1);
+ while (true) {
+ if (__p && __index < __p->__size_)
+ return __p;
+ __round* __q = __new_round(__capacity, __p); // __p is updated when the race is lost early
+ if (__q == nullptr)
+ continue;
+ if (__array_.compare_exchange_strong(__p, __q, memory_order_acq_rel, memory_order_acquire)) {
+ __size_.store(__capacity, memory_order_release);
+ return __q;
+ }
+ __del_round(__q);
+ }
+ }
+
+public:
+ constexpr __grow_array() noexcept = default;
+ __grow_array(const __grow_array&) = delete;
+ __grow_array& operator=(const __grow_array&) = delete;
+
+ size_t __size() const noexcept { return __size_.load(memory_order_acquire); }
+
+ // Reference to element __index, growing the array if needed. May throw bad_alloc.
+ _Tp& operator[](size_t __index) {
+ bool __fast = __index < __size_.load(memory_order_acquire);
+ __round* __p = __fast ? __array_.load(memory_order_acquire) : __at_slow(__index);
+ return *__p->__list()[__index];
+ }
+
+ // Pointers to the first min(__count, size) elements constructed so far, in index order; __count is
+ // clamped. Returns nullptr when the array is still empty.
+ _Tp* const* __pointers(size_t& __count) const noexcept {
+ __round* __p = __array_.load(memory_order_acquire);
+ if (__p == nullptr) {
+ __count = 0;
+ return nullptr;
+ }
+ if (__count > __p->__size_)
+ __count = __p->__size_;
+ return __p->__list();
+ }
+};
+
+// Per-thread cache of free records
+
+// A per-thread stack of up to 9 free records. It sits in front of the domain's available list:
+// __hazard_pointer_acquire() pops from it and __hazard_pointer_release() pushes to it, so the steady state
+// costs no atomics on the domain's shared state. Reached through __tls_cache (one TLS load) where
+// _LIBCPP_HAZARD_POINTER_THREAD_LOCAL, else through the pthread key (call_once fast path + __libcpp_tls_get);
+// see __hazard_pointer_domain::__thread_cache. The key is always what registers the thread-exit hook.
+struct __hazard_pointer_thread_cache {
+ __hazard_pointer_record* __entries_[__thread_cache_capacity] = {};
+ uint8_t __count_ = 0;
+
+ __hazard_pointer_record* __try_get() noexcept { return __count_ > 0 ? __entries_[--__count_] : nullptr; }
+
+ bool __try_put(__hazard_pointer_record* __record) noexcept {
+ if (__count_ < __thread_cache_capacity) {
+ __entries_[__count_++] = __record;
+ return true;
+ }
+ return false;
+ }
+};
+
+// Sentinel stored by __at_thread_exit once a thread's cache has been torn down (in __tls_cache, or in the
+// TLS key on the pthread-only path). While it is there, __thread_cache() reports "no cache", so a
+// hazard_pointer destroyed by a destructor that runs after ours releases its record straight to the domain
+// instead of touching the freed cache.
+//
+// With __tls_cache the sentinel stays for the rest of the thread's life. On the pthread-only path it is
+// armed when the cache is torn down; POSIX clears a TSD value before each destructor round, so a later
+// round sees null again, not the sentinel. We deliberately do not re-arm it there: re-arming would keep
+// this thread in destructor rounds after sanitizer runtimes have already finalized it. Consequently, on
+// that path a make_hazard_pointer() called from a foreign TSD destructor in a later round may build a fresh
+// cache that is never evicted -- a bounded leak of that cache object plus up to __thread_cache_capacity
+// records. Records are immortal, so the cache object is the actual leak.
+__hazard_pointer_thread_cache __dead_thread_cache;
+
+#if _LIBCPP_HAZARD_POINTER_THREAD_LOCAL
+[[__gnu__::__tls_model__("initial-exec")]] constinit thread_local __hazard_pointer_thread_cache* __tls_cache = nullptr;
+#endif
+
+// Intrusive lists of retired nodes
+
+using __node = __hazard_pointer_obj_node;
+
+// A (head, tail, count) list of nodes; single-threaded.
+struct __node_list {
+ __node* __head_ = nullptr;
+ __node* __tail_ = nullptr;
+ int __count_ = 0;
+
+ bool __empty() const noexcept { return __head_ == nullptr; }
+
+ void __push(__node* __n) noexcept {
+ __n->__next_ = nullptr;
+ if (__tail_ != nullptr)
+ __tail_->__next_ = __n;
+ else
+ __head_ = __n;
+ __tail_ = __n;
+ ++__count_;
+ }
+
+ void __clear() noexcept {
+ __head_ = nullptr;
+ __tail_ = nullptr;
+ __count_ = 0;
+ }
+};
+
+// A head-only list shared between threads: lock-free push of a whole __node_list, wait-free pop-all.
+struct __shared_node_list {
+ atomic<__node*> __head_{nullptr};
+
+ void __push(__node_list& __list) noexcept { // prepends the whole list, kept in order
+ if (__list.__empty())
+ return;
+ __node* __old_head = __head_.load(memory_order_acquire);
+ do {
+ __list.__tail_->__next_ = __old_head;
+ } while (!__head_.compare_exchange_weak(__old_head, __list.__head_, memory_order_acq_rel, memory_order_acquire));
+ __list.__clear();
+ }
+
+ __node* __pop_all() noexcept { return __head_.exchange(nullptr, memory_order_acq_rel); }
+
+ bool __empty() const noexcept { return __head_.load(memory_order_acquire) == nullptr; }
+};
+
+// The set of protected addresses built by every reclamation pass
+
+// Open addressing, power-of-two table, __mix_pointer as the hash. Storage comes from nothrow new;
+// __insert() returns false when the table cannot grow, in which case the pass falls back to matching
+// each retired node against the records directly (see __hazard_pointer_domain::__match_reclaim).
+class __protected_set {
+ uintptr_t* __slots_ = nullptr; // 0 = empty; only non-null addresses are inserted
+ size_t __capacity_ = 0;
+ size_t __mask_ = 0;
+ size_t __size_ = 0;
+
+ void __insert_no_grow(uintptr_t __key) noexcept {
+ size_t __i = __mix_pointer(__key) & __mask_;
+ while (__slots_[__i] != 0) {
+ if (__slots_[__i] == __key)
+ return;
+ __i = (__i + 1) & __mask_;
+ }
+ __slots_[__i] = __key;
+ ++__size_;
+ }
+
+ bool __grow() noexcept {
+ size_t __new_capacity = __capacity_ == 0 ? 16 : __capacity_ * 2;
+ uintptr_t* __new_slots = ::new (nothrow) uintptr_t[__new_capacity](); // zero-initialised
+ if (__new_slots == nullptr)
+ return false;
+ uintptr_t* __old = __slots_;
+ size_t __old_capacity = __capacity_;
+ __slots_ = __new_slots;
+ __capacity_ = __new_capacity;
+ __mask_ = __new_capacity - 1;
+ __size_ = 0;
+ for (size_t __i = 0; __i < __old_capacity; ++__i)
+ if (__old[__i] != 0)
+ __insert_no_grow(__old[__i]);
+ delete[] __old;
+ return true;
+ }
+
+public:
+ __protected_set() = default;
+ __protected_set(const __protected_set&) = delete;
+ __protected_set& operator=(const __protected_set&) = delete;
+ ~__protected_set() { delete[] __slots_; }
+
+ bool __insert(const void* __p) noexcept {
+ if ((__size_ + 1) * 2 > __capacity_ && !__grow())
+ return false;
+ __insert_no_grow(reinterpret_cast<uintptr_t>(__p));
+ return true;
+ }
+
+ bool __contains(const void* __p) const noexcept {
+ if (__size_ == 0)
+ return false;
+ uintptr_t __key = reinterpret_cast<uintptr_t>(__p);
+ size_t __i = __mix_pointer(__key) & __mask_;
+ while (__slots_[__i] != 0) {
+ if (__slots_[__i] == __key)
+ return true;
+ __i = (__i + 1) & __mask_;
+ }
+ return false;
+ }
+};
+
+// Fences of the reader/reclaimer protocol (see <__hazard_pointer/domain.h>)
+
+// The fence every retire() issues before publishing its node, and the one every reclamation pass issues
+// before scanning the slots. With asymmetric fences the retire side (like the reader side, inline in
+// the headers) is a compiler barrier and the reclaimer side is a barrier on every CPU currently running
+// a thread of this process; otherwise both are seq_cst fences.
+#if _LIBCPP_HAZARD_POINTER_ASYMMETRIC_FENCE
----------------
MindSpectre wrote:
Required explicit approval
https://github.com/llvm/llvm-project/pull/218218
More information about the libcxx-commits
mailing list