[libc-commits] [libc] [libc] add internal rwlock implementation (PR #91142)
via libc-commits
libc-commits at lists.llvm.org
Sun May 5 13:24:12 PDT 2024
llvmbot wrote:
<!--LLVM PR SUMMARY COMMENT-->
@llvm/pr-subscribers-libc
Author: Schrodinger ZHU Yifan (SchrodingerZhu)
<details>
<summary>Changes</summary>
---
Patch is 27.96 KiB, truncated to 20.00 KiB below, full version: https://github.com/llvm/llvm-project/pull/91142.diff
8 Files Affected:
- (modified) libc/src/__support/CPP/atomic.h (+20)
- (modified) libc/src/__support/threads/linux/CMakeLists.txt (+24-4)
- (modified) libc/src/__support/threads/linux/futex_word.h (+49-1)
- (added) libc/src/__support/threads/linux/rwlock.h (+412)
- (modified) libc/test/src/__support/CMakeLists.txt (+1)
- (added) libc/test/src/__support/threads/CMakeLists.txt (+3)
- (added) libc/test/src/__support/threads/linux/CMakeLists.txt (+13)
- (added) libc/test/src/__support/threads/linux/rwlock_test.cpp (+172)
``````````diff
diff --git a/libc/src/__support/CPP/atomic.h b/libc/src/__support/CPP/atomic.h
index 5e428940565b99..1392197321a80e 100644
--- a/libc/src/__support/CPP/atomic.h
+++ b/libc/src/__support/CPP/atomic.h
@@ -101,6 +101,26 @@ template <typename T> struct Atomic {
int(mem_ord), int(mem_ord));
}
+ // Atomic compare exchange
+ LIBC_INLINE bool compare_exchange_strong(
+ T &expected, T desired, MemoryOrder success_order,
+ MemoryOrder failure_order,
+ [[maybe_unused]] MemoryScope mem_scope = cpp::MemoryScope::DEVICE) {
+ return __atomic_compare_exchange_n(&val, &expected, desired, false,
+ int(success_order), int(failure_order));
+ }
+
+ LIBC_INLINE bool compare_exchange_weak(
+ T &expected, T new_val, MemoryOrder success_order,
+ MemoryOrder failure_order,
+ [[maybe_unused]] MemoryScope mem_scope = cpp::MemoryScope::DEVICE) {
+ return __atomic_compare_exchange_n(
+ &val, &expected, new_val,
+ /* is_weak */ true,
+ /* success_memorder */ static_cast<int>(success_order),
+ /* failure_memorder */ static_cast<int>(failure_order));
+ }
+
T exchange(T desired, MemoryOrder mem_ord = MemoryOrder::SEQ_CST,
[[maybe_unused]] MemoryScope mem_scope = MemoryScope::DEVICE) {
#if __has_builtin(__scoped_atomic_exchange_n)
diff --git a/libc/src/__support/threads/linux/CMakeLists.txt b/libc/src/__support/threads/linux/CMakeLists.txt
index 87a7a66ac6ea57..36edf99092792e 100644
--- a/libc/src/__support/threads/linux/CMakeLists.txt
+++ b/libc/src/__support/threads/linux/CMakeLists.txt
@@ -1,13 +1,21 @@
+if(NOT TARGET libc.src.__support.OSUtil.osutil)
+ return()
+endif()
+
add_header_library(
futex_word_type
HDRS
futex_word.h
+ DEPENDS
+ libc.src.__support.CPP.optional
+ libc.src.__support.CPP.atomic
+ libc.src.__support.CPP.limits
+ libc.src.__support.OSUtil.osutil
+ libc.include.sys_syscall
+ libc.src.time.clock_gettime
+ libc.src.errno.errno
)
-if(NOT TARGET libc.src.__support.OSUtil.osutil)
- return()
-endif()
-
add_header_library(
mutex
HDRS
@@ -20,6 +28,18 @@ add_header_library(
libc.src.__support.threads.mutex_common
)
+add_header_library(
+ rwlock
+ HDRS
+ rwlock.h
+ DEPENDS
+ .futex_word_type
+ libc.include.sys_syscall
+ libc.src.__support.CPP.atomic
+ libc.src.__support.CPP.optional
+ libc.src.__support.libc_assert
+)
+
add_object_library(
thread
SRCS
diff --git a/libc/src/__support/threads/linux/futex_word.h b/libc/src/__support/threads/linux/futex_word.h
index 67159b81b56132..aba650e2906bbe 100644
--- a/libc/src/__support/threads/linux/futex_word.h
+++ b/libc/src/__support/threads/linux/futex_word.h
@@ -9,9 +9,15 @@
#ifndef LLVM_LIBC_SRC___SUPPORT_THREADS_LINUX_FUTEX_WORD_H
#define LLVM_LIBC_SRC___SUPPORT_THREADS_LINUX_FUTEX_WORD_H
+#include "include/llvm-libc-types/struct_timespec.h"
+#include "src/__support/CPP/atomic.h"
+#include "src/__support/CPP/limits.h"
+#include "src/__support/CPP/optional.h"
+#include "src/__support/OSUtil/syscall.h"
+#include "src/errno/libc_errno.h"
+#include <linux/futex.h>
#include <stdint.h>
#include <sys/syscall.h>
-
namespace LIBC_NAMESPACE {
// Futexes are 32 bits in size on all platforms, including 64-bit platforms.
@@ -25,6 +31,48 @@ constexpr auto FUTEX_SYSCALL_ID = SYS_futex_time64;
#error "futex and futex_time64 syscalls not available."
#endif
+// Returns false on timeout, and true in all other cases.
+LIBC_INLINE bool futex_wait(cpp::Atomic<FutexWordType> &futex,
+ FutexWordType expected,
+ cpp::optional<::timespec> abs_timeout,
+ bool is_shared = false) {
+ for (;;) {
+ if (futex.load(cpp::MemoryOrder::RELAXED) != expected)
+ return true;
+ // Use FUTEX_WAIT_BITSET rather than FUTEX_WAIT to be able to give an
+ // absolute time rather than a relative time.
+ long ret =
+ syscall_impl<long>(FUTEX_SYSCALL_ID, &futex,
+ is_shared ? FUTEX_WAIT_BITSET
+ : (FUTEX_WAIT_BITSET | FUTEX_PRIVATE_FLAG),
+ expected, abs_timeout ? &*abs_timeout : nullptr,
+ nullptr, FUTEX_BITSET_MATCH_ANY);
+ switch (ret) {
+ case -EINTR:
+ continue;
+ case -ETIMEDOUT:
+ return false;
+ default:
+ return true;
+ }
+ }
+}
+
+LIBC_INLINE bool futex_wake_one(cpp::Atomic<FutexWordType> &futex,
+ bool is_shared = false) {
+ long ret = syscall_impl<long>(FUTEX_SYSCALL_ID, &futex,
+ is_shared ? FUTEX_WAKE : FUTEX_WAKE_PRIVATE, 1,
+ 0, 0, 0);
+ return ret > 0;
+}
+
+LIBC_INLINE void futex_wake_all(cpp::Atomic<FutexWordType> &futex,
+ bool is_shared = false) {
+ syscall_impl<long>(FUTEX_SYSCALL_ID, &futex,
+ is_shared ? FUTEX_WAKE : FUTEX_WAKE_PRIVATE,
+ cpp::numeric_limits<int>::max(), 0, 0, 0);
+}
+
} // namespace LIBC_NAMESPACE
#endif // LLVM_LIBC_SRC___SUPPORT_THREADS_LINUX_FUTEX_WORD_H
diff --git a/libc/src/__support/threads/linux/rwlock.h b/libc/src/__support/threads/linux/rwlock.h
new file mode 100644
index 00000000000000..5a1c1f3cb518b2
--- /dev/null
+++ b/libc/src/__support/threads/linux/rwlock.h
@@ -0,0 +1,412 @@
+//===--- Implementation of a Linux rwlock class ------------------*- C++-*-===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef LLVM_LIBC_SRC___SUPPORT_THREADS_LINUX_MUTEX_H
+#define LLVM_LIBC_SRC___SUPPORT_THREADS_LINUX_MUTEX_H
+
+#include "src/__support/CPP/atomic.h"
+#include "src/__support/CPP/optional.h"
+#include "src/__support/libc_assert.h"
+#include "src/__support/macros/attributes.h"
+#include "src/__support/threads/linux/futex_word.h"
+
+#include "src/__support/threads/sleep.h"
+#include "src/errno/libc_errno.h"
+#include "src/time/clock_gettime.h"
+
+namespace LIBC_NAMESPACE {
+// A rwlock implementation using futexes.
+// Code is largely based on the Rust standard library implementation.
+// https://github.com/rust-lang/rust/blob/22a5267c83a3e17f2b763279eb24bb632c45dc6b/library/std/src/sys/sync/rwlock/futex.rs
+struct RwLock {
+ // The state consists of a 30-bit reader counter, a 'readers waiting' flag,
+ // and a 'writers waiting' flag. Bits 0..30:
+ // 0: Unlocked
+ // 1..=0x3FFF_FFFE: Locked by N readers
+ // 0x3FFF_FFFF: Write locked
+ // Bit 30: Readers are waiting on this futex.
+ // Bit 31: Writers are waiting on the writer_notify futex.
+ cpp::Atomic<FutexWordType> state;
+ // The 'condition variable' to notify writers through.
+ // Incremented on every signal.
+ cpp::Atomic<FutexWordType> writer_notify;
+ // If the rwlock is shared between processes.
+ bool is_shared;
+
+ LIBC_INLINE_VAR static constexpr FutexWordType READ_LOCKED = 1u;
+ LIBC_INLINE_VAR static constexpr FutexWordType MASK = (1u << 30) - 1u;
+ LIBC_INLINE_VAR static constexpr FutexWordType WRITE_LOCKED = MASK;
+ LIBC_INLINE_VAR static constexpr FutexWordType MAX_READERS = MASK - 1u;
+ LIBC_INLINE_VAR static constexpr FutexWordType READERS_WAITING = 1u << 30u;
+ LIBC_INLINE_VAR static constexpr FutexWordType WRITERS_WAITING = 1u << 31u;
+ LIBC_INLINE_VAR static constexpr int_fast32_t SPIN_LIMIT = 100u;
+ LIBC_INLINE static constexpr bool is_unlocked(FutexWordType state) {
+ return (state & MASK) == 0u;
+ }
+ LIBC_INLINE static constexpr bool is_write_locked(FutexWordType state) {
+ return (state & MASK) == WRITE_LOCKED;
+ }
+ LIBC_INLINE static constexpr bool has_readers_waiting(FutexWordType state) {
+ return (state & READERS_WAITING) != 0u;
+ }
+ LIBC_INLINE static constexpr bool has_writers_waiting(FutexWordType state) {
+ return (state & WRITERS_WAITING) != 0u;
+ }
+ // This also returns false if the counter could overflow if we tried to read
+ // lock it.
+ //
+ // We don't allow read-locking if there's readers waiting, even if the lock is
+ // unlocked and there's no writers waiting. The only situation when this
+ // happens is after unlocking, at which point the unlocking thread might be
+ // waking up writers, which have priority over readers. The unlocking thread
+ // will clear the readers waiting bit and wake up readers, if necessary.
+ LIBC_INLINE static constexpr bool is_read_lockable(FutexWordType state) {
+ return (state & MASK) < MAX_READERS && !has_readers_waiting(state) &&
+ !has_writers_waiting(state);
+ }
+ LIBC_INLINE static constexpr bool
+ has_reached_max_readers(FutexWordType state) {
+ return (state & MASK) == MAX_READERS;
+ }
+
+ // Convert a relative timeout to an absolute timespec.
+ LIBC_INLINE static void abs_timeout(cpp::optional<timespec> &timeout) {
+ if (!timeout)
+ return;
+
+ int errno_backup = libc_errno;
+ timespec now;
+ // if we failed to get time, we move timeout to infinity by setting it to
+ // nullopt.
+ if (LIBC_NAMESPACE::clock_gettime(CLOCK_MONOTONIC, &now) != 0) {
+ timeout = cpp::nullopt;
+ libc_errno = errno_backup;
+ return;
+ }
+ if (__builtin_add_overflow(now.tv_sec, timeout->tv_sec, &timeout->tv_sec)) {
+ timeout = cpp::nullopt;
+ return;
+ }
+ if (__builtin_add_overflow(now.tv_nsec, timeout->tv_nsec,
+ &timeout->tv_nsec)) {
+ timeout = cpp::nullopt;
+ return;
+ }
+ if (now.tv_nsec >= 1'000'000'000) {
+ if (__builtin_add_overflow(now.tv_sec, 1, &timeout->tv_sec)) {
+ timeout = cpp::nullopt;
+ return;
+ }
+ timeout->tv_nsec -= 1'000'000'000;
+ }
+ }
+
+ template <typename F>
+ LIBC_INLINE bool fetch_update(cpp::Atomic<FutexWordType> &__restrict word,
+ FutexWordType &__restrict prev,
+ cpp::MemoryOrder set_order,
+ cpp::MemoryOrder fetch_order, F &&f) {
+ prev = word.load(fetch_order);
+ // It is okay to have spurious failures here as we are in a loop.
+ while (cpp::optional<FutexWordType> new_val = f(prev))
+ if (word.compare_exchange_weak(prev, *new_val, set_order, fetch_order))
+ return true;
+
+ return false;
+ }
+
+ template <typename F>
+ LIBC_INLINE FutexWordType spin_until(int_fast32_t spin, F &&f) {
+ for (;;) {
+ FutexWordType state = this->state.load(cpp::MemoryOrder::RELAXED);
+ if (f(state) || spin == 0)
+ return state;
+ // pause the pipeline to avoid extra memory loads due to speculation
+ sleep_briefly();
+ --spin;
+ }
+ }
+
+ LIBC_INLINE static bool
+ is_timeout(const cpp::optional<timespec> &abs_timeout) {
+ if (!abs_timeout)
+ return false;
+ timespec now;
+ LIBC_NAMESPACE::clock_gettime(CLOCK_MONOTONIC, &now);
+ if (now.tv_sec > abs_timeout->tv_sec ||
+ (now.tv_sec == abs_timeout->tv_sec &&
+ now.tv_nsec >= abs_timeout->tv_nsec))
+ return true;
+ return false;
+ }
+
+ LIBC_INLINE FutexWordType spin_read(int_fast32_t spin) {
+ // Stop spinning when it's unlocked or read locked, or when there's waiting
+ // threads.
+ return spin_until(spin, [&](FutexWordType state) -> bool {
+ return !is_write_locked(state) || has_readers_waiting(state) ||
+ has_writers_waiting(state) || is_unlocked(state);
+ });
+ }
+
+ LIBC_INLINE FutexWordType spin_write(int_fast32_t spin) {
+ // Stop spinning when it's unlocked or when there's waiting writers, to keep
+ // things somewhat fair.
+ return spin_until(spin, [&](FutexWordType state) -> bool {
+ return is_unlocked(state) || has_writers_waiting(state);
+ });
+ }
+
+ [[gnu::cold]]
+ LIBC_INLINE bool
+ read_contented(cpp::optional<timespec> abs_timeout = cpp::nullopt) {
+ // Notice that the timeout is not checked during the fast spin loop.
+ FutexWordType prev = this->spin_read(SPIN_LIMIT);
+ int lockable_loop = 0;
+ for (;;) {
+ // We have a chance to lock it, go ahead and have a try.
+ if (is_read_lockable(prev)) {
+ if (this->state.compare_exchange_weak(prev, prev + READ_LOCKED,
+ cpp::MemoryOrder::ACQUIRE,
+ cpp::MemoryOrder::RELAXED))
+ return true;
+ // Timeout is checked the first time we enter the loop since there is
+ // always spin before this.
+ if (lockable_loop == 0 && is_timeout(abs_timeout)) {
+ libc_errno = ETIMEDOUT;
+ return false;
+ }
+ // In the rare case, we somehow stuck in the lockable loop for a long
+ // time. We should give a chance to allow timeout checking. We do this
+ // everytime we meet the SPIN_LIMIT.
+ lockable_loop = (lockable_loop + 1) % SPIN_LIMIT;
+ // we continue to spin if we failed to lock it.
+ // notice that prev is updated by the compare_exchange_weak.
+ continue;
+ }
+
+ if (has_reached_max_readers(prev)) {
+ // The read lock could not be acquired because the maximum
+ // number of read locks for rwlock has been exceeded.
+ libc_errno = EAGAIN;
+ return false;
+ }
+
+ // Make sure the readers waiting bit is set before we go to sleep.
+ // Strong CAS is required as this is a one-time operation.
+ if (!has_readers_waiting(prev) &&
+ !this->state.compare_exchange_strong(prev, prev | READERS_WAITING,
+ cpp::MemoryOrder::RELAXED))
+ continue;
+
+ if (!futex_wait(this->state, prev | READERS_WAITING, abs_timeout,
+ is_shared)) {
+ // timeout happened.
+ libc_errno = ETIMEDOUT;
+ return false;
+ }
+
+ prev = this->spin_read(SPIN_LIMIT);
+ lockable_loop = 0;
+ }
+ }
+
+ /// Wake up a single writer.
+ LIBC_INLINE bool wake_writer() {
+ writer_notify.fetch_add(1, cpp::MemoryOrder::RELEASE);
+ return futex_wake_one(writer_notify);
+ }
+
+ /// Wake up waiting threads after unlocking.
+ ///
+ /// If both are waiting, this will wake up only one writer, but will fall
+ /// back to waking up readers if there was no writer to wake up.
+ [[gnu::cold]]
+ LIBC_INLINE void wake_writer_or_readers(FutexWordType prev) {
+ LIBC_ASSERT(is_unlocked(prev));
+ // The readers waiting bit might be turned on at any point now,
+ // since readers will block when there's anything waiting.
+ // Writers will just lock the lock though, regardless of the waiting bits,
+ // so we don't have to worry about the writer waiting bit.
+ //
+ // If the lock gets locked in the meantime, we don't have to do
+ // anything, because then the thread that locked the lock will take
+ // care of waking up waiters when it unlocks.
+
+ // If only writers are waiting, wake one of them up.
+ if (prev == WRITERS_WAITING)
+ if (this->state.compare_exchange_strong(
+ prev, 0, cpp::MemoryOrder::RELAXED, cpp::MemoryOrder::RELAXED))
+ return (void)wake_writer();
+ // notice that prev is updated.
+
+ // If both writers and readers are waiting, leave the readers waiting
+ // and only wake up one writer.
+ if (prev == (WRITERS_WAITING + READERS_WAITING)) {
+ // set READERS_WAITING and check if someone else locked the lock.
+ if (!this->state.compare_exchange_strong(prev, READERS_WAITING,
+ cpp::MemoryOrder::RELAXED,
+ cpp::MemoryOrder::RELAXED))
+ return; // someone else locked the lock.
+
+ if (wake_writer())
+ return;
+
+ // No writers were actually blocked on futex_wait, so we continue
+ // to wake up readers instead, since we can't be sure if we notified a
+ // writer.
+ prev = READERS_WAITING;
+ }
+
+ if (prev == READERS_WAITING &&
+ this->state.compare_exchange_strong(prev, 0, cpp::MemoryOrder::RELAXED,
+ cpp::MemoryOrder::RELAXED))
+ futex_wake_all(this->state);
+ }
+
+ [[gnu::cold]]
+ LIBC_INLINE bool write_contented(cpp::optional<timespec> abs_timeout) {
+ FutexWordType prev = this->spin_write(SPIN_LIMIT);
+ FutexWordType other_writers_waiting = 0;
+ int lockable_loop = 0;
+ for (;;) {
+ // We have a chance to lock it, go ahead and have a try.
+ if (is_unlocked(prev)) {
+ if (this->state.compare_exchange_weak(
+ prev, prev | WRITE_LOCKED | other_writers_waiting,
+ cpp::MemoryOrder::ACQUIRE, cpp::MemoryOrder::RELAXED))
+ return true;
+ // Timeout is checked the first time we enter the loop since there is
+ // always spin before this.
+ if (lockable_loop == 0 && is_timeout(abs_timeout)) {
+ libc_errno = ETIMEDOUT;
+ return false;
+ }
+ // In the rare case, we somehow stuck in the lockable loop for a long
+ // time. We should give a chance to allow timeout checking. We do this
+ // everytime we meet the SPIN_LIMIT.
+ lockable_loop = (lockable_loop + 1) % SPIN_LIMIT;
+ // we continue to spin if we failed to lock it.
+ // notice that prev is updated by the compare_exchange_weak.
+ continue;
+ }
+
+ // Set the waiting bit indicating that we're waiting on it.
+ if (!has_writers_waiting(prev) &&
+ !this->state.compare_exchange_strong(prev, prev | WRITERS_WAITING,
+ cpp::MemoryOrder::RELAXED))
+ continue;
+
+ // Other writers might be waiting now too, so we should make sure
+ // we keep that bit on once we manage lock it.
+ other_writers_waiting = WRITERS_WAITING;
+
+ // Examine the notification counter before we check if `state` has
+ // changed, to make sure we don't miss any notifications.
+ FutexWordType seq = this->writer_notify.load(cpp::MemoryOrder::ACQUIRE);
+
+ // Don't go to sleep if the lock has become available,
+ // or if the writers waiting bit is no longer set.
+ prev = this->state.load(cpp::MemoryOrder::RELAXED);
+ if (is_unlocked(prev) || !has_writers_waiting(prev))
+ continue;
+
+ // Wait for the state to change.
+ if (!futex_wait(this->writer_notify, seq, abs_timeout, is_shared)) {
+ // timeout happened.
+ libc_errno = ETIMEDOUT;
+ return false;
+ }
+
+ // Spin again after waking up.
+ prev = this->spin_write(SPIN_LIMIT);
+ lockable_loop = 0;
+ }
+ }
+
+public:
+ LIBC_INLINE explicit constexpr RwLock(bool shared = false)
+ : state(FutexWordType(0)), writer_notify(FutexWordType(0)),
+ is_shared(shared) {}
+
+ LIBC_INLINE bool try_read() {
+ FutexWordType prev;
+ // set order is the success order for the inner compare_exchange_weak.
+ // it effectively make the store ordering to be relaxed.
+ return fetch_update(this->state, prev, cpp::MemoryOrder::ACQUIRE,
+ cpp::MemoryOrder::RELAXED,
+ [](FutexWordType x) -> cpp::optional<FutexWordType> {
+ if (is_read_lockable(x))
+ return {x + READ_LOCKED};
+ return cpp::nullopt;
+ });
+ }
+
+ LIBC_INLINE bool read(cpp::optional<timespec> timeout = cpp::nullopt) {
+ // record absoulte time if timeout is provided.
+ abs_timeout(timeout);
+ FutexWordType prev = this->state.load(cpp::MemoryOrder::RELAXED);
+ // It is okay to have spurious failures here as we will fallback to
+ // contented routine.
+ if (!is_read_lockable(prev) ||
+ !this->state.compare_exchange_weak(prev, prev + READ_LOCKED,
+ cpp::MemoryOrder::ACQUIRE,
+ cpp::MemoryOrder::RELAXED))
+ return read_contented(timeout);
+ return true;
+ }
+
+ LIBC_INLINE void read_unlock() {
+ FutexWordType pre...
[truncated]
``````````
</details>
https://github.com/llvm/llvm-project/pull/91142
More information about the libc-commits
mailing list