[libc-commits] [libc] [libc] add internal rwlock implementation (PR #91142)
Schrodinger ZHU Yifan via libc-commits
libc-commits at lists.llvm.org
Sun May 5 13:29:45 PDT 2024
https://github.com/SchrodingerZhu updated https://github.com/llvm/llvm-project/pull/91142
>From bed03df045b3f33d8b49a71e12fbc0e0845fa071 Mon Sep 17 00:00:00 2001
From: Schrodinger ZHU Yifan <yifanzhu at rochester.edu>
Date: Sun, 5 May 2024 16:22:22 -0400
Subject: [PATCH 1/4] [libc] add internal rwlock implementation
---
libc/src/__support/CPP/atomic.h | 20 +
.../__support/threads/linux/CMakeLists.txt | 28 +-
libc/src/__support/threads/linux/futex_word.h | 50 ++-
libc/src/__support/threads/linux/rwlock.h | 412 ++++++++++++++++++
libc/test/src/__support/CMakeLists.txt | 1 +
.../test/src/__support/threads/CMakeLists.txt | 3 +
.../__support/threads/linux/CMakeLists.txt | 13 +
.../__support/threads/linux/rwlock_test.cpp | 172 ++++++++
8 files changed, 694 insertions(+), 5 deletions(-)
create mode 100644 libc/src/__support/threads/linux/rwlock.h
create mode 100644 libc/test/src/__support/threads/CMakeLists.txt
create mode 100644 libc/test/src/__support/threads/linux/CMakeLists.txt
create mode 100644 libc/test/src/__support/threads/linux/rwlock_test.cpp
diff --git a/libc/src/__support/CPP/atomic.h b/libc/src/__support/CPP/atomic.h
index 5e428940565b99..1392197321a80e 100644
--- a/libc/src/__support/CPP/atomic.h
+++ b/libc/src/__support/CPP/atomic.h
@@ -101,6 +101,26 @@ template <typename T> struct Atomic {
int(mem_ord), int(mem_ord));
}
+ // Atomic compare exchange
+ LIBC_INLINE bool compare_exchange_strong(
+ T &expected, T desired, MemoryOrder success_order,
+ MemoryOrder failure_order,
+ [[maybe_unused]] MemoryScope mem_scope = cpp::MemoryScope::DEVICE) {
+ return __atomic_compare_exchange_n(&val, &expected, desired, false,
+ int(success_order), int(failure_order));
+ }
+
+ LIBC_INLINE bool compare_exchange_weak(
+ T &expected, T new_val, MemoryOrder success_order,
+ MemoryOrder failure_order,
+ [[maybe_unused]] MemoryScope mem_scope = cpp::MemoryScope::DEVICE) {
+ return __atomic_compare_exchange_n(
+ &val, &expected, new_val,
+ /* is_weak */ true,
+ /* success_memorder */ static_cast<int>(success_order),
+ /* failure_memorder */ static_cast<int>(failure_order));
+ }
+
T exchange(T desired, MemoryOrder mem_ord = MemoryOrder::SEQ_CST,
[[maybe_unused]] MemoryScope mem_scope = MemoryScope::DEVICE) {
#if __has_builtin(__scoped_atomic_exchange_n)
diff --git a/libc/src/__support/threads/linux/CMakeLists.txt b/libc/src/__support/threads/linux/CMakeLists.txt
index 87a7a66ac6ea57..36edf99092792e 100644
--- a/libc/src/__support/threads/linux/CMakeLists.txt
+++ b/libc/src/__support/threads/linux/CMakeLists.txt
@@ -1,13 +1,21 @@
+if(NOT TARGET libc.src.__support.OSUtil.osutil)
+ return()
+endif()
+
add_header_library(
futex_word_type
HDRS
futex_word.h
+ DEPENDS
+ libc.src.__support.CPP.optional
+ libc.src.__support.CPP.atomic
+ libc.src.__support.CPP.limits
+ libc.src.__support.OSUtil.osutil
+ libc.include.sys_syscall
+ libc.src.time.clock_gettime
+ libc.src.errno.errno
)
-if(NOT TARGET libc.src.__support.OSUtil.osutil)
- return()
-endif()
-
add_header_library(
mutex
HDRS
@@ -20,6 +28,18 @@ add_header_library(
libc.src.__support.threads.mutex_common
)
+add_header_library(
+ rwlock
+ HDRS
+ rwlock.h
+ DEPENDS
+ .futex_word_type
+ libc.include.sys_syscall
+ libc.src.__support.CPP.atomic
+ libc.src.__support.CPP.optional
+ libc.src.__support.libc_assert
+)
+
add_object_library(
thread
SRCS
diff --git a/libc/src/__support/threads/linux/futex_word.h b/libc/src/__support/threads/linux/futex_word.h
index 67159b81b56132..aba650e2906bbe 100644
--- a/libc/src/__support/threads/linux/futex_word.h
+++ b/libc/src/__support/threads/linux/futex_word.h
@@ -9,9 +9,15 @@
#ifndef LLVM_LIBC_SRC___SUPPORT_THREADS_LINUX_FUTEX_WORD_H
#define LLVM_LIBC_SRC___SUPPORT_THREADS_LINUX_FUTEX_WORD_H
+#include "include/llvm-libc-types/struct_timespec.h"
+#include "src/__support/CPP/atomic.h"
+#include "src/__support/CPP/limits.h"
+#include "src/__support/CPP/optional.h"
+#include "src/__support/OSUtil/syscall.h"
+#include "src/errno/libc_errno.h"
+#include <linux/futex.h>
#include <stdint.h>
#include <sys/syscall.h>
-
namespace LIBC_NAMESPACE {
// Futexes are 32 bits in size on all platforms, including 64-bit platforms.
@@ -25,6 +31,48 @@ constexpr auto FUTEX_SYSCALL_ID = SYS_futex_time64;
#error "futex and futex_time64 syscalls not available."
#endif
+// Returns false on timeout, and true in all other cases.
+LIBC_INLINE bool futex_wait(cpp::Atomic<FutexWordType> &futex,
+ FutexWordType expected,
+ cpp::optional<::timespec> abs_timeout,
+ bool is_shared = false) {
+ for (;;) {
+ if (futex.load(cpp::MemoryOrder::RELAXED) != expected)
+ return true;
+ // Use FUTEX_WAIT_BITSET rather than FUTEX_WAIT to be able to give an
+ // absolute time rather than a relative time.
+ long ret =
+ syscall_impl<long>(FUTEX_SYSCALL_ID, &futex,
+ is_shared ? FUTEX_WAIT_BITSET
+ : (FUTEX_WAIT_BITSET | FUTEX_PRIVATE_FLAG),
+ expected, abs_timeout ? &*abs_timeout : nullptr,
+ nullptr, FUTEX_BITSET_MATCH_ANY);
+ switch (ret) {
+ case -EINTR:
+ continue;
+ case -ETIMEDOUT:
+ return false;
+ default:
+ return true;
+ }
+ }
+}
+
+LIBC_INLINE bool futex_wake_one(cpp::Atomic<FutexWordType> &futex,
+ bool is_shared = false) {
+ long ret = syscall_impl<long>(FUTEX_SYSCALL_ID, &futex,
+ is_shared ? FUTEX_WAKE : FUTEX_WAKE_PRIVATE, 1,
+ 0, 0, 0);
+ return ret > 0;
+}
+
+LIBC_INLINE void futex_wake_all(cpp::Atomic<FutexWordType> &futex,
+ bool is_shared = false) {
+ syscall_impl<long>(FUTEX_SYSCALL_ID, &futex,
+ is_shared ? FUTEX_WAKE : FUTEX_WAKE_PRIVATE,
+ cpp::numeric_limits<int>::max(), 0, 0, 0);
+}
+
} // namespace LIBC_NAMESPACE
#endif // LLVM_LIBC_SRC___SUPPORT_THREADS_LINUX_FUTEX_WORD_H
diff --git a/libc/src/__support/threads/linux/rwlock.h b/libc/src/__support/threads/linux/rwlock.h
new file mode 100644
index 00000000000000..5a1c1f3cb518b2
--- /dev/null
+++ b/libc/src/__support/threads/linux/rwlock.h
@@ -0,0 +1,412 @@
+//===--- Implementation of a Linux rwlock class ------------------*- C++-*-===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef LLVM_LIBC_SRC___SUPPORT_THREADS_LINUX_MUTEX_H
+#define LLVM_LIBC_SRC___SUPPORT_THREADS_LINUX_MUTEX_H
+
+#include "src/__support/CPP/atomic.h"
+#include "src/__support/CPP/optional.h"
+#include "src/__support/libc_assert.h"
+#include "src/__support/macros/attributes.h"
+#include "src/__support/threads/linux/futex_word.h"
+
+#include "src/__support/threads/sleep.h"
+#include "src/errno/libc_errno.h"
+#include "src/time/clock_gettime.h"
+
+namespace LIBC_NAMESPACE {
+// A rwlock implementation using futexes.
+// Code is largely based on the Rust standard library implementation.
+// https://github.com/rust-lang/rust/blob/22a5267c83a3e17f2b763279eb24bb632c45dc6b/library/std/src/sys/sync/rwlock/futex.rs
+struct RwLock {
+ // The state consists of a 30-bit reader counter, a 'readers waiting' flag,
+ // and a 'writers waiting' flag. Bits 0..30:
+ // 0: Unlocked
+ // 1..=0x3FFF_FFFE: Locked by N readers
+ // 0x3FFF_FFFF: Write locked
+ // Bit 30: Readers are waiting on this futex.
+ // Bit 31: Writers are waiting on the writer_notify futex.
+ cpp::Atomic<FutexWordType> state;
+ // The 'condition variable' to notify writers through.
+ // Incremented on every signal.
+ cpp::Atomic<FutexWordType> writer_notify;
+ // If the rwlock is shared between processes.
+ bool is_shared;
+
+ LIBC_INLINE_VAR static constexpr FutexWordType READ_LOCKED = 1u;
+ LIBC_INLINE_VAR static constexpr FutexWordType MASK = (1u << 30) - 1u;
+ LIBC_INLINE_VAR static constexpr FutexWordType WRITE_LOCKED = MASK;
+ LIBC_INLINE_VAR static constexpr FutexWordType MAX_READERS = MASK - 1u;
+ LIBC_INLINE_VAR static constexpr FutexWordType READERS_WAITING = 1u << 30u;
+ LIBC_INLINE_VAR static constexpr FutexWordType WRITERS_WAITING = 1u << 31u;
+ LIBC_INLINE_VAR static constexpr int_fast32_t SPIN_LIMIT = 100u;
+ LIBC_INLINE static constexpr bool is_unlocked(FutexWordType state) {
+ return (state & MASK) == 0u;
+ }
+ LIBC_INLINE static constexpr bool is_write_locked(FutexWordType state) {
+ return (state & MASK) == WRITE_LOCKED;
+ }
+ LIBC_INLINE static constexpr bool has_readers_waiting(FutexWordType state) {
+ return (state & READERS_WAITING) != 0u;
+ }
+ LIBC_INLINE static constexpr bool has_writers_waiting(FutexWordType state) {
+ return (state & WRITERS_WAITING) != 0u;
+ }
+ // This also returns false if the counter could overflow if we tried to read
+ // lock it.
+ //
+ // We don't allow read-locking if there's readers waiting, even if the lock is
+ // unlocked and there's no writers waiting. The only situation when this
+ // happens is after unlocking, at which point the unlocking thread might be
+ // waking up writers, which have priority over readers. The unlocking thread
+ // will clear the readers waiting bit and wake up readers, if necessary.
+ LIBC_INLINE static constexpr bool is_read_lockable(FutexWordType state) {
+ return (state & MASK) < MAX_READERS && !has_readers_waiting(state) &&
+ !has_writers_waiting(state);
+ }
+ LIBC_INLINE static constexpr bool
+ has_reached_max_readers(FutexWordType state) {
+ return (state & MASK) == MAX_READERS;
+ }
+
+ // Convert a relative timeout to an absolute timespec.
+ LIBC_INLINE static void abs_timeout(cpp::optional<timespec> &timeout) {
+ if (!timeout)
+ return;
+
+ int errno_backup = libc_errno;
+ timespec now;
+ // if we failed to get time, we move timeout to infinity by setting it to
+ // nullopt.
+ if (LIBC_NAMESPACE::clock_gettime(CLOCK_MONOTONIC, &now) != 0) {
+ timeout = cpp::nullopt;
+ libc_errno = errno_backup;
+ return;
+ }
+ if (__builtin_add_overflow(now.tv_sec, timeout->tv_sec, &timeout->tv_sec)) {
+ timeout = cpp::nullopt;
+ return;
+ }
+ if (__builtin_add_overflow(now.tv_nsec, timeout->tv_nsec,
+ &timeout->tv_nsec)) {
+ timeout = cpp::nullopt;
+ return;
+ }
+ if (now.tv_nsec >= 1'000'000'000) {
+ if (__builtin_add_overflow(now.tv_sec, 1, &timeout->tv_sec)) {
+ timeout = cpp::nullopt;
+ return;
+ }
+ timeout->tv_nsec -= 1'000'000'000;
+ }
+ }
+
+ template <typename F>
+ LIBC_INLINE bool fetch_update(cpp::Atomic<FutexWordType> &__restrict word,
+ FutexWordType &__restrict prev,
+ cpp::MemoryOrder set_order,
+ cpp::MemoryOrder fetch_order, F &&f) {
+ prev = word.load(fetch_order);
+ // It is okay to have spurious failures here as we are in a loop.
+ while (cpp::optional<FutexWordType> new_val = f(prev))
+ if (word.compare_exchange_weak(prev, *new_val, set_order, fetch_order))
+ return true;
+
+ return false;
+ }
+
+ template <typename F>
+ LIBC_INLINE FutexWordType spin_until(int_fast32_t spin, F &&f) {
+ for (;;) {
+ FutexWordType state = this->state.load(cpp::MemoryOrder::RELAXED);
+ if (f(state) || spin == 0)
+ return state;
+ // pause the pipeline to avoid extra memory loads due to speculation
+ sleep_briefly();
+ --spin;
+ }
+ }
+
+ LIBC_INLINE static bool
+ is_timeout(const cpp::optional<timespec> &abs_timeout) {
+ if (!abs_timeout)
+ return false;
+ timespec now;
+ LIBC_NAMESPACE::clock_gettime(CLOCK_MONOTONIC, &now);
+ if (now.tv_sec > abs_timeout->tv_sec ||
+ (now.tv_sec == abs_timeout->tv_sec &&
+ now.tv_nsec >= abs_timeout->tv_nsec))
+ return true;
+ return false;
+ }
+
+ LIBC_INLINE FutexWordType spin_read(int_fast32_t spin) {
+ // Stop spinning when it's unlocked or read locked, or when there's waiting
+ // threads.
+ return spin_until(spin, [&](FutexWordType state) -> bool {
+ return !is_write_locked(state) || has_readers_waiting(state) ||
+ has_writers_waiting(state) || is_unlocked(state);
+ });
+ }
+
+ LIBC_INLINE FutexWordType spin_write(int_fast32_t spin) {
+ // Stop spinning when it's unlocked or when there's waiting writers, to keep
+ // things somewhat fair.
+ return spin_until(spin, [&](FutexWordType state) -> bool {
+ return is_unlocked(state) || has_writers_waiting(state);
+ });
+ }
+
+ [[gnu::cold]]
+ LIBC_INLINE bool
+ read_contented(cpp::optional<timespec> abs_timeout = cpp::nullopt) {
+ // Notice that the timeout is not checked during the fast spin loop.
+ FutexWordType prev = this->spin_read(SPIN_LIMIT);
+ int lockable_loop = 0;
+ for (;;) {
+ // We have a chance to lock it, go ahead and have a try.
+ if (is_read_lockable(prev)) {
+ if (this->state.compare_exchange_weak(prev, prev + READ_LOCKED,
+ cpp::MemoryOrder::ACQUIRE,
+ cpp::MemoryOrder::RELAXED))
+ return true;
+ // Timeout is checked the first time we enter the loop since there is
+ // always spin before this.
+ if (lockable_loop == 0 && is_timeout(abs_timeout)) {
+ libc_errno = ETIMEDOUT;
+ return false;
+ }
+ // In the rare case, we somehow stuck in the lockable loop for a long
+ // time. We should give a chance to allow timeout checking. We do this
+ // everytime we meet the SPIN_LIMIT.
+ lockable_loop = (lockable_loop + 1) % SPIN_LIMIT;
+ // we continue to spin if we failed to lock it.
+ // notice that prev is updated by the compare_exchange_weak.
+ continue;
+ }
+
+ if (has_reached_max_readers(prev)) {
+ // The read lock could not be acquired because the maximum
+ // number of read locks for rwlock has been exceeded.
+ libc_errno = EAGAIN;
+ return false;
+ }
+
+ // Make sure the readers waiting bit is set before we go to sleep.
+ // Strong CAS is required as this is a one-time operation.
+ if (!has_readers_waiting(prev) &&
+ !this->state.compare_exchange_strong(prev, prev | READERS_WAITING,
+ cpp::MemoryOrder::RELAXED))
+ continue;
+
+ if (!futex_wait(this->state, prev | READERS_WAITING, abs_timeout,
+ is_shared)) {
+ // timeout happened.
+ libc_errno = ETIMEDOUT;
+ return false;
+ }
+
+ prev = this->spin_read(SPIN_LIMIT);
+ lockable_loop = 0;
+ }
+ }
+
+ /// Wake up a single writer.
+ LIBC_INLINE bool wake_writer() {
+ writer_notify.fetch_add(1, cpp::MemoryOrder::RELEASE);
+ return futex_wake_one(writer_notify);
+ }
+
+ /// Wake up waiting threads after unlocking.
+ ///
+ /// If both are waiting, this will wake up only one writer, but will fall
+ /// back to waking up readers if there was no writer to wake up.
+ [[gnu::cold]]
+ LIBC_INLINE void wake_writer_or_readers(FutexWordType prev) {
+ LIBC_ASSERT(is_unlocked(prev));
+ // The readers waiting bit might be turned on at any point now,
+ // since readers will block when there's anything waiting.
+ // Writers will just lock the lock though, regardless of the waiting bits,
+ // so we don't have to worry about the writer waiting bit.
+ //
+ // If the lock gets locked in the meantime, we don't have to do
+ // anything, because then the thread that locked the lock will take
+ // care of waking up waiters when it unlocks.
+
+ // If only writers are waiting, wake one of them up.
+ if (prev == WRITERS_WAITING)
+ if (this->state.compare_exchange_strong(
+ prev, 0, cpp::MemoryOrder::RELAXED, cpp::MemoryOrder::RELAXED))
+ return (void)wake_writer();
+ // notice that prev is updated.
+
+ // If both writers and readers are waiting, leave the readers waiting
+ // and only wake up one writer.
+ if (prev == (WRITERS_WAITING + READERS_WAITING)) {
+ // set READERS_WAITING and check if someone else locked the lock.
+ if (!this->state.compare_exchange_strong(prev, READERS_WAITING,
+ cpp::MemoryOrder::RELAXED,
+ cpp::MemoryOrder::RELAXED))
+ return; // someone else locked the lock.
+
+ if (wake_writer())
+ return;
+
+ // No writers were actually blocked on futex_wait, so we continue
+ // to wake up readers instead, since we can't be sure if we notified a
+ // writer.
+ prev = READERS_WAITING;
+ }
+
+ if (prev == READERS_WAITING &&
+ this->state.compare_exchange_strong(prev, 0, cpp::MemoryOrder::RELAXED,
+ cpp::MemoryOrder::RELAXED))
+ futex_wake_all(this->state);
+ }
+
+ [[gnu::cold]]
+ LIBC_INLINE bool write_contented(cpp::optional<timespec> abs_timeout) {
+ FutexWordType prev = this->spin_write(SPIN_LIMIT);
+ FutexWordType other_writers_waiting = 0;
+ int lockable_loop = 0;
+ for (;;) {
+ // We have a chance to lock it, go ahead and have a try.
+ if (is_unlocked(prev)) {
+ if (this->state.compare_exchange_weak(
+ prev, prev | WRITE_LOCKED | other_writers_waiting,
+ cpp::MemoryOrder::ACQUIRE, cpp::MemoryOrder::RELAXED))
+ return true;
+ // Timeout is checked the first time we enter the loop since there is
+ // always spin before this.
+ if (lockable_loop == 0 && is_timeout(abs_timeout)) {
+ libc_errno = ETIMEDOUT;
+ return false;
+ }
+ // In the rare case, we somehow stuck in the lockable loop for a long
+ // time. We should give a chance to allow timeout checking. We do this
+ // everytime we meet the SPIN_LIMIT.
+ lockable_loop = (lockable_loop + 1) % SPIN_LIMIT;
+ // we continue to spin if we failed to lock it.
+ // notice that prev is updated by the compare_exchange_weak.
+ continue;
+ }
+
+ // Set the waiting bit indicating that we're waiting on it.
+ if (!has_writers_waiting(prev) &&
+ !this->state.compare_exchange_strong(prev, prev | WRITERS_WAITING,
+ cpp::MemoryOrder::RELAXED))
+ continue;
+
+ // Other writers might be waiting now too, so we should make sure
+ // we keep that bit on once we manage lock it.
+ other_writers_waiting = WRITERS_WAITING;
+
+ // Examine the notification counter before we check if `state` has
+ // changed, to make sure we don't miss any notifications.
+ FutexWordType seq = this->writer_notify.load(cpp::MemoryOrder::ACQUIRE);
+
+ // Don't go to sleep if the lock has become available,
+ // or if the writers waiting bit is no longer set.
+ prev = this->state.load(cpp::MemoryOrder::RELAXED);
+ if (is_unlocked(prev) || !has_writers_waiting(prev))
+ continue;
+
+ // Wait for the state to change.
+ if (!futex_wait(this->writer_notify, seq, abs_timeout, is_shared)) {
+ // timeout happened.
+ libc_errno = ETIMEDOUT;
+ return false;
+ }
+
+ // Spin again after waking up.
+ prev = this->spin_write(SPIN_LIMIT);
+ lockable_loop = 0;
+ }
+ }
+
+public:
+ LIBC_INLINE explicit constexpr RwLock(bool shared = false)
+ : state(FutexWordType(0)), writer_notify(FutexWordType(0)),
+ is_shared(shared) {}
+
+ LIBC_INLINE bool try_read() {
+ FutexWordType prev;
+ // set order is the success order for the inner compare_exchange_weak.
+ // it effectively make the store ordering to be relaxed.
+ return fetch_update(this->state, prev, cpp::MemoryOrder::ACQUIRE,
+ cpp::MemoryOrder::RELAXED,
+ [](FutexWordType x) -> cpp::optional<FutexWordType> {
+ if (is_read_lockable(x))
+ return {x + READ_LOCKED};
+ return cpp::nullopt;
+ });
+ }
+
+ LIBC_INLINE bool read(cpp::optional<timespec> timeout = cpp::nullopt) {
+ // record absoulte time if timeout is provided.
+ abs_timeout(timeout);
+ FutexWordType prev = this->state.load(cpp::MemoryOrder::RELAXED);
+ // It is okay to have spurious failures here as we will fallback to
+ // contented routine.
+ if (!is_read_lockable(prev) ||
+ !this->state.compare_exchange_weak(prev, prev + READ_LOCKED,
+ cpp::MemoryOrder::ACQUIRE,
+ cpp::MemoryOrder::RELAXED))
+ return read_contented(timeout);
+ return true;
+ }
+
+ LIBC_INLINE void read_unlock() {
+ FutexWordType prev =
+ this->state.fetch_sub(READ_LOCKED, cpp::MemoryOrder::RELEASE) -
+ READ_LOCKED;
+ // It's impossible for a reader to be waiting on a read-locked RwLock,
+ // except if there is also a writer waiting.
+ LIBC_ASSERT(!has_readers_waiting(state) || has_writers_waiting(state));
+
+ // Wake up a writer if we were the last reader and there's a writer waiting.
+ if (is_unlocked(prev) && has_writers_waiting(prev))
+ wake_writer_or_readers(prev);
+ }
+ LIBC_INLINE bool try_write() {
+ FutexWordType prev;
+ return fetch_update(this->state, prev, cpp::MemoryOrder::ACQUIRE,
+ cpp::MemoryOrder::RELAXED,
+ [](FutexWordType s) -> cpp::optional<FutexWordType> {
+ if (is_unlocked(s))
+ return s + WRITE_LOCKED;
+ return cpp::nullopt;
+ });
+ }
+ LIBC_INLINE bool write(cpp::optional<timespec> timeout = cpp::nullopt) {
+ // record absoulte time if timeout is provided.
+ abs_timeout(timeout);
+ FutexWordType prev = 0;
+ // It is okay to have spurious failures here as we will fallback to
+ // contented routine.
+ if (!this->state.compare_exchange_weak(prev, WRITE_LOCKED,
+ cpp::MemoryOrder::ACQUIRE,
+ cpp::MemoryOrder::RELAXED))
+ return write_contented(timeout);
+ return true;
+ }
+ LIBC_INLINE void write_unlock() {
+ FutexWordType prev =
+ this->state.fetch_sub(WRITE_LOCKED, cpp::MemoryOrder::RELEASE) -
+ WRITE_LOCKED;
+
+ LIBC_ASSERT(is_unlocked(prev));
+
+ if (has_writers_waiting(prev) || has_readers_waiting(prev))
+ wake_writer_or_readers(prev);
+ }
+};
+
+} // namespace LIBC_NAMESPACE
+
+#endif // LLVM_LIBC_SRC___SUPPORT_THREADS_LINUX_MUTEX_H
diff --git a/libc/test/src/__support/CMakeLists.txt b/libc/test/src/__support/CMakeLists.txt
index 5d1230f5f3a70f..7c084b5d284643 100644
--- a/libc/test/src/__support/CMakeLists.txt
+++ b/libc/test/src/__support/CMakeLists.txt
@@ -206,3 +206,4 @@ add_subdirectory(OSUtil)
add_subdirectory(FPUtil)
add_subdirectory(fixed_point)
add_subdirectory(HashTable)
+add_subdirectory(threads)
diff --git a/libc/test/src/__support/threads/CMakeLists.txt b/libc/test/src/__support/threads/CMakeLists.txt
new file mode 100644
index 00000000000000..b4bbe81c92ff2e
--- /dev/null
+++ b/libc/test/src/__support/threads/CMakeLists.txt
@@ -0,0 +1,3 @@
+if(EXISTS ${CMAKE_CURRENT_SOURCE_DIR}/${LIBC_TARGET_OS})
+ add_subdirectory(${LIBC_TARGET_OS})
+endif()
diff --git a/libc/test/src/__support/threads/linux/CMakeLists.txt b/libc/test/src/__support/threads/linux/CMakeLists.txt
new file mode 100644
index 00000000000000..ba332dcb0015fe
--- /dev/null
+++ b/libc/test/src/__support/threads/linux/CMakeLists.txt
@@ -0,0 +1,13 @@
+add_custom_target(libc___support_threads_linux_rwlock_unittests)
+add_libc_unittest(
+ rwlock_test
+ SUITE
+ libc___support_threads_linux_rwlock_unittests
+ SRCS
+ rwlock_test.cpp
+ DEPENDS
+ libc.src.__support.threads.linux.rwlock
+ libc.test.UnitTest.ErrnoSetterMatcher
+ libc.src.sys.mman.mmap
+ libc.src.sys.mman.munmap
+)
diff --git a/libc/test/src/__support/threads/linux/rwlock_test.cpp b/libc/test/src/__support/threads/linux/rwlock_test.cpp
new file mode 100644
index 00000000000000..c259cc7f9511f2
--- /dev/null
+++ b/libc/test/src/__support/threads/linux/rwlock_test.cpp
@@ -0,0 +1,172 @@
+#include "src/__support/CPP/atomic.h"
+#include "src/__support/libc_assert.h"
+#include "src/__support/threads/linux/rwlock.h"
+#include "src/errno/libc_errno.h"
+#include "src/sys/mman/mmap.h"
+#include "src/sys/mman/munmap.h"
+#include "test/UnitTest/Test.h"
+#include <linux/mman.h>
+#include <llvm-libc-types/pid_t.h>
+#include <pthread.h>
+extern "C" pid_t fork(void);
+extern "C" [[noreturn]] void exit(int);
+namespace LIBC_NAMESPACE {
+TEST(LlvmLibcRwLock, Smoke) {
+ RwLock l{};
+ l.read();
+ l.read_unlock();
+ l.write();
+ l.write_unlock();
+ l.read();
+ l.read();
+ l.read_unlock();
+ l.read_unlock();
+ l.write();
+ l.write_unlock();
+}
+
+TEST(LlvmLibcRwLock, TrivialReadLock) {
+ RwLock l{};
+ pthread_t t[100];
+ for (int i = 0; i < 100; i++) {
+ pthread_create(
+ &t[i], nullptr,
+ [](void *arg) -> void * {
+ RwLock *l = static_cast<RwLock *>(arg);
+ if (!l->try_read())
+ l->read();
+ l->read_unlock();
+ return nullptr;
+ },
+ &l);
+ }
+ for (int i = 0; i < 100; i++) {
+ pthread_join(t[i], nullptr);
+ }
+}
+
+TEST(LlvmLibcRwLock, TrivialWriteLock) {
+ struct Data {
+ RwLock lock{};
+ cpp::Atomic<int> i = 0;
+ } data;
+
+ pthread_t t[100];
+ for (int i = 0; i < 100; i++) {
+ pthread_create(
+ &t[i], nullptr,
+ [](void *arg) -> void * {
+ Data *data = static_cast<Data *>(arg);
+ data->lock.write();
+ int x = data->i.load(cpp::MemoryOrder::RELAXED);
+ x++;
+ data->i.store(x, cpp::MemoryOrder::RELAXED);
+ data->lock.write_unlock();
+ return nullptr;
+ },
+ &data);
+ }
+ for (int i = 0; i < 100; i++) {
+ pthread_join(t[i], nullptr);
+ }
+ ASSERT_EQ(data.i.load(cpp::MemoryOrder::RELAXED), 100);
+}
+
+TEST(LlvmLibcRwLock, DeadWriteTimeoutRead) {
+ RwLock l{};
+ l.write();
+ pthread_t t[100];
+ for (int i = 0; i < 100; i++) {
+ pthread_create(
+ &t[i], nullptr,
+ [](void *arg) -> void * {
+ RwLock *l = static_cast<RwLock *>(arg);
+ if (l->try_read())
+ __builtin_trap();
+ if (l->try_write())
+ __builtin_trap();
+ ::timespec ts{};
+ ts.tv_sec = 1;
+ if (l->read(ts))
+ __builtin_trap();
+ if (libc_errno != ETIMEDOUT)
+ __builtin_trap();
+ return nullptr;
+ },
+ &l);
+ }
+ for (int i = 0; i < 100; i++) {
+ pthread_join(t[i], nullptr);
+ }
+}
+
+TEST(LlvmLibcRwLock, DeadReadTimeoutWrite) {
+ RwLock l{};
+ l.read();
+ pthread_t t[100];
+ for (int i = 0; i < 100; i++) {
+ pthread_create(
+ &t[i], nullptr,
+ [](void *arg) -> void * {
+ RwLock *l = static_cast<RwLock *>(arg);
+ // l->read_unlock();
+ ::timespec ts{};
+ ts.tv_sec = 1;
+ if (l->try_write())
+ __builtin_trap();
+ if (l->write(ts))
+ __builtin_trap();
+ if (libc_errno != ETIMEDOUT)
+ __builtin_trap();
+ return nullptr;
+ },
+ &l);
+ }
+ for (int i = 0; i < 100; i++) {
+ pthread_join(t[i], nullptr);
+ }
+}
+
+TEST(LlvmLibcRwLock, Fork) {
+ static constexpr struct Data {
+ RwLock lock{true};
+ cpp::Atomic<int> i = 0;
+ cpp::Atomic<int> pending = 2;
+ } DEFAULT;
+
+ void *mmap_addr = mmap(nullptr, sizeof(Data), PROT_READ | PROT_WRITE,
+ MAP_ANONYMOUS | MAP_SHARED, -1, 0);
+ ASSERT_NE(mmap_addr, MAP_FAILED);
+ Data *data = static_cast<Data *>(mmap_addr);
+ // placement new is not available as we cannot include <new>
+ __builtin_memcpy(data, &DEFAULT, sizeof(Data));
+ int pid = ::fork();
+
+ pthread_t t[100];
+ for (int i = 0; i < 100; i++) {
+ pthread_create(
+ &t[i], nullptr,
+ [](void *arg) -> void * {
+ Data *data = static_cast<Data *>(arg);
+ data->lock.write();
+ int x = data->i.load(cpp::MemoryOrder::RELAXED);
+ x++;
+ data->i.store(x, cpp::MemoryOrder::RELAXED);
+ data->lock.write_unlock();
+ return nullptr;
+ },
+ data);
+ }
+ for (int i = 0; i < 100; i++) {
+ pthread_join(t[i], nullptr);
+ }
+ data->pending.fetch_sub(1);
+ while (data->pending)
+ /* not UB as operations are atomic */;
+ // early exit to avoid pollute the test output
+ if (pid == 0)
+ ::exit(0);
+ ASSERT_EQ(data->i.load(cpp::MemoryOrder::RELAXED), 200);
+ ASSERT_EQ(munmap(mmap_addr, sizeof(Data)), 0);
+}
+} // namespace LIBC_NAMESPACE
\ No newline at end of file
>From 06b04cee62d1338b6613d4477a9322fff8fbbb3e Mon Sep 17 00:00:00 2001
From: Schrodinger ZHU Yifan <yifanzhu at rochester.edu>
Date: Sun, 5 May 2024 16:24:25 -0400
Subject: [PATCH 2/4] [libc] correct format
---
libc/src/__support/threads/linux/rwlock.h | 6 +++---
libc/test/src/__support/threads/linux/rwlock_test.cpp | 2 +-
2 files changed, 4 insertions(+), 4 deletions(-)
diff --git a/libc/src/__support/threads/linux/rwlock.h b/libc/src/__support/threads/linux/rwlock.h
index 5a1c1f3cb518b2..4098be7d133014 100644
--- a/libc/src/__support/threads/linux/rwlock.h
+++ b/libc/src/__support/threads/linux/rwlock.h
@@ -6,8 +6,8 @@
//
//===----------------------------------------------------------------------===//
-#ifndef LLVM_LIBC_SRC___SUPPORT_THREADS_LINUX_MUTEX_H
-#define LLVM_LIBC_SRC___SUPPORT_THREADS_LINUX_MUTEX_H
+#ifndef LLVM_LIBC_SRC___SUPPORT_THREADS_LINUX_RWLOCK_H
+#define LLVM_LIBC_SRC___SUPPORT_THREADS_LINUX_RWLOCK_H
#include "src/__support/CPP/atomic.h"
#include "src/__support/CPP/optional.h"
@@ -409,4 +409,4 @@ struct RwLock {
} // namespace LIBC_NAMESPACE
-#endif // LLVM_LIBC_SRC___SUPPORT_THREADS_LINUX_MUTEX_H
+#endif // LLVM_LIBC_SRC___SUPPORT_THREADS_LINUX_RWLOCK_H
diff --git a/libc/test/src/__support/threads/linux/rwlock_test.cpp b/libc/test/src/__support/threads/linux/rwlock_test.cpp
index c259cc7f9511f2..464bb9218db91b 100644
--- a/libc/test/src/__support/threads/linux/rwlock_test.cpp
+++ b/libc/test/src/__support/threads/linux/rwlock_test.cpp
@@ -169,4 +169,4 @@ TEST(LlvmLibcRwLock, Fork) {
ASSERT_EQ(data->i.load(cpp::MemoryOrder::RELAXED), 200);
ASSERT_EQ(munmap(mmap_addr, sizeof(Data)), 0);
}
-} // namespace LIBC_NAMESPACE
\ No newline at end of file
+} // namespace LIBC_NAMESPACE
>From 462f03d507ebe5a0361fcad88be138d1ea8002c8 Mon Sep 17 00:00:00 2001
From: Schrodinger ZHU Yifan <yifanzhu at rochester.edu>
Date: Sun, 5 May 2024 16:25:38 -0400
Subject: [PATCH 3/4] [libc] adjust check ordering
---
libc/test/src/__support/threads/linux/rwlock_test.cpp | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/libc/test/src/__support/threads/linux/rwlock_test.cpp b/libc/test/src/__support/threads/linux/rwlock_test.cpp
index 464bb9218db91b..7984d92daa68c9 100644
--- a/libc/test/src/__support/threads/linux/rwlock_test.cpp
+++ b/libc/test/src/__support/threads/linux/rwlock_test.cpp
@@ -163,10 +163,10 @@ TEST(LlvmLibcRwLock, Fork) {
data->pending.fetch_sub(1);
while (data->pending)
/* not UB as operations are atomic */;
+ ASSERT_EQ(data->i.load(cpp::MemoryOrder::RELAXED), 200);
// early exit to avoid pollute the test output
if (pid == 0)
::exit(0);
- ASSERT_EQ(data->i.load(cpp::MemoryOrder::RELAXED), 200);
ASSERT_EQ(munmap(mmap_addr, sizeof(Data)), 0);
}
} // namespace LIBC_NAMESPACE
>From 3a6790cc0927d68ebe150f4617dc5675e5355d12 Mon Sep 17 00:00:00 2001
From: Schrodinger ZHU Yifan <yifanzhu at rochester.edu>
Date: Sun, 5 May 2024 16:29:34 -0400
Subject: [PATCH 4/4] fix deadlock
---
libc/src/__support/threads/linux/rwlock.h | 4 ++--
1 file changed, 2 insertions(+), 2 deletions(-)
diff --git a/libc/src/__support/threads/linux/rwlock.h b/libc/src/__support/threads/linux/rwlock.h
index 4098be7d133014..cadff0c362fa08 100644
--- a/libc/src/__support/threads/linux/rwlock.h
+++ b/libc/src/__support/threads/linux/rwlock.h
@@ -219,7 +219,7 @@ struct RwLock {
/// Wake up a single writer.
LIBC_INLINE bool wake_writer() {
writer_notify.fetch_add(1, cpp::MemoryOrder::RELEASE);
- return futex_wake_one(writer_notify);
+ return futex_wake_one(writer_notify, is_shared);
}
/// Wake up waiting threads after unlocking.
@@ -266,7 +266,7 @@ struct RwLock {
if (prev == READERS_WAITING &&
this->state.compare_exchange_strong(prev, 0, cpp::MemoryOrder::RELAXED,
cpp::MemoryOrder::RELAXED))
- futex_wake_all(this->state);
+ futex_wake_all(this->state, is_shared);
}
[[gnu::cold]]
More information about the libc-commits
mailing list