[libc-commits] [libc] [libc] add internal rwlock implementation (PR #91142)

[via libc-commits]

================
@@ -0,0 +1,412 @@
+//===--- Implementation of a Linux rwlock class ------------------*- C++-*-===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef LLVM_LIBC_SRC___SUPPORT_THREADS_LINUX_RWLOCK_H
+#define LLVM_LIBC_SRC___SUPPORT_THREADS_LINUX_RWLOCK_H
+
+#include "src/__support/CPP/atomic.h"
+#include "src/__support/CPP/optional.h"
+#include "src/__support/libc_assert.h"
+#include "src/__support/macros/attributes.h"
+#include "src/__support/threads/linux/futex_word.h"
+
+#include "src/__support/threads/sleep.h"
+#include "src/errno/libc_errno.h"
+#include "src/time/clock_gettime.h"
+
+namespace LIBC_NAMESPACE {
+// A rwlock implementation using futexes.
+// Code is largely based on the Rust standard library implementation.
+// https://github.com/rust-lang/rust/blob/22a5267c83a3e17f2b763279eb24bb632c45dc6b/library/std/src/sys/sync/rwlock/futex.rs
+class RwLock {
+  // The state consists of a 30-bit reader counter, a 'readers waiting' flag,
+  // and a 'writers waiting' flag. Bits 0..30:
+  //   0: Unlocked
+  //   1..=0x3FFF_FFFE: Locked by N readers
+  //   0x3FFF_FFFF: Write locked
+  // Bit 30: Readers are waiting on this futex.
+  // Bit 31: Writers are waiting on the writer_notify futex.
+  cpp::Atomic<FutexWordType> state;
+  // The 'condition variable' to notify writers through.
+  // Incremented on every signal.
+  cpp::Atomic<FutexWordType> writer_notify;
+  // If the rwlock is shared between processes.
+  bool is_shared;
+
+  LIBC_INLINE_VAR static constexpr FutexWordType READ_LOCKED = 1u;
+  LIBC_INLINE_VAR static constexpr FutexWordType MASK = (1u << 30) - 1u;
+  LIBC_INLINE_VAR static constexpr FutexWordType WRITE_LOCKED = MASK;
+  LIBC_INLINE_VAR static constexpr FutexWordType MAX_READERS = MASK - 1u;
+  LIBC_INLINE_VAR static constexpr FutexWordType READERS_WAITING = 1u << 30u;
+  LIBC_INLINE_VAR static constexpr FutexWordType WRITERS_WAITING = 1u << 31u;
+  LIBC_INLINE_VAR static constexpr int_fast32_t SPIN_LIMIT = 100u;
+  LIBC_INLINE static constexpr bool is_unlocked(FutexWordType state) {
+    return (state & MASK) == 0u;
+  }
+  LIBC_INLINE static constexpr bool is_write_locked(FutexWordType state) {
+    return (state & MASK) == WRITE_LOCKED;
+  }
+  LIBC_INLINE static constexpr bool has_readers_waiting(FutexWordType state) {
+    return (state & READERS_WAITING) != 0u;
+  }
+  LIBC_INLINE static constexpr bool has_writers_waiting(FutexWordType state) {
+    return (state & WRITERS_WAITING) != 0u;
+  }
+  // This also returns false if the counter could overflow if we tried to read
+  // lock it.
+  //
+  // We don't allow read-locking if there's readers waiting, even if the lock is
+  // unlocked and there's no writers waiting. The only situation when this
+  // happens is after unlocking, at which point the unlocking thread might be
+  // waking up writers, which have priority over readers. The unlocking thread
+  // will clear the readers waiting bit and wake up readers, if necessary.
+  LIBC_INLINE static constexpr bool is_read_lockable(FutexWordType state) {
+    return (state & MASK) < MAX_READERS && !has_readers_waiting(state) &&
+           !has_writers_waiting(state);
+  }
+  LIBC_INLINE static constexpr bool
+  has_reached_max_readers(FutexWordType state) {
+    return (state & MASK) == MAX_READERS;
+  }
+
+  // Convert a relative timeout to an absolute timespec.
+  LIBC_INLINE static void abs_timeout(cpp::optional<timespec> &timeout) {
+    if (!timeout)
+      return;
+
+    int errno_backup = libc_errno;
+    timespec now;
+    // if we failed to get time, we move timeout to infinity by setting it to
+    // nullopt.
+    if (LIBC_NAMESPACE::clock_gettime(CLOCK_MONOTONIC, &now) != 0) {
+      timeout = cpp::nullopt;
+      libc_errno = errno_backup;
+      return;
+    }
+    if (__builtin_add_overflow(now.tv_sec, timeout->tv_sec, &timeout->tv_sec)) {
+      timeout = cpp::nullopt;
+      return;
+    }
+    if (__builtin_add_overflow(now.tv_nsec, timeout->tv_nsec,
+                               &timeout->tv_nsec)) {
+      timeout = cpp::nullopt;
+      return;
+    }
+    if (now.tv_nsec >= 1'000'000'000) {
+      if (__builtin_add_overflow(now.tv_sec, 1, &timeout->tv_sec)) {
+        timeout = cpp::nullopt;
+        return;
+      }
+      timeout->tv_nsec -= 1'000'000'000;
+    }
+  }
+
+  template <typename F>
+  LIBC_INLINE bool fetch_update(cpp::Atomic<FutexWordType> &__restrict word,
+                                FutexWordType &__restrict prev,
+                                cpp::MemoryOrder set_order,
+                                cpp::MemoryOrder fetch_order, F &&f) {
+    prev = word.load(fetch_order);
+    // It is okay to have spurious failures here as we are in a loop.
+    while (cpp::optional<FutexWordType> new_val = f(prev))
+      if (word.compare_exchange_weak(prev, *new_val, set_order, fetch_order))
+        return true;
+
+    return false;
+  }
+
+  template <typename F>
+  LIBC_INLINE FutexWordType spin_until(int_fast32_t spin, F &&f) {
+    for (;;) {
+      FutexWordType state = this->state.load(cpp::MemoryOrder::RELAXED);
+      if (f(state) || spin == 0)
+        return state;
+      // pause the pipeline to avoid extra memory loads due to speculation
+      sleep_briefly();
+      --spin;
+    }
+  }
+
+  LIBC_INLINE static bool
+  is_timeout(const cpp::optional<timespec> &abs_timeout) {
+    if (!abs_timeout)
+      return false;
+    timespec now;
+    LIBC_NAMESPACE::clock_gettime(CLOCK_MONOTONIC, &now);
+    if (now.tv_sec > abs_timeout->tv_sec ||
+        (now.tv_sec == abs_timeout->tv_sec &&
+         now.tv_nsec >= abs_timeout->tv_nsec))
+      return true;
+    return false;
+  }
+
+  LIBC_INLINE FutexWordType spin_read(int_fast32_t spin) {
+    // Stop spinning when it's unlocked or read locked, or when there's waiting
+    // threads.
+    return spin_until(spin, [&](FutexWordType state) -> bool {
+      return !is_write_locked(state) || has_readers_waiting(state) ||
+             has_writers_waiting(state) || is_unlocked(state);
----------------
QuarticCat wrote:

The `is_unlocked(state)` here seems redundant as `!is_write_locked(state)` implies either unlocked or read locked.

https://github.com/llvm/llvm-project/pull/91142