[libc-commits] [libc] [libc] rework mutex (PR #92168)
Schrodinger ZHU Yifan via libc-commits
libc-commits at lists.llvm.org
Tue May 28 22:13:04 PDT 2024
https://github.com/SchrodingerZhu updated https://github.com/llvm/llvm-project/pull/92168
>From 3c4dc296a7512376fdaa8a874021f40324a5ff49 Mon Sep 17 00:00:00 2001
From: Schrodinger ZHU Yifan <yifanzhu at rochester.edu>
Date: Tue, 14 May 2024 15:52:46 -0400
Subject: [PATCH 01/15] [libc] rework mutex
---
libc/config/config.json | 10 ++
libc/docs/configure.rst | 3 +
libc/hdr/types/CMakeLists.txt | 9 ++
libc/hdr/types/pid_t.h | 22 +++
libc/src/__support/File/dir.h | 2 +-
libc/src/__support/File/file.h | 2 +-
libc/src/__support/threads/fork_callbacks.cpp | 3 +-
libc/src/__support/threads/gpu/mutex.h | 2 +-
.../__support/threads/linux/CMakeLists.txt | 23 +++
libc/src/__support/threads/linux/mutex.h | 133 +++++++-----------
libc/src/__support/threads/linux/raw_mutex.h | 116 +++++++++++++++
libc/src/__support/threads/thread.cpp | 4 +-
libc/src/pthread/pthread_mutex_init.cpp | 5 +-
libc/src/pthread/pthread_mutexattr.h | 5 +
libc/src/stdlib/atexit.cpp | 2 +-
libc/src/threads/linux/CMakeLists.txt | 1 +
libc/src/threads/mtx_init.cpp | 3 +-
.../__support/threads/thread_detach_test.cpp | 2 +-
18 files changed, 253 insertions(+), 94 deletions(-)
create mode 100644 libc/hdr/types/pid_t.h
create mode 100644 libc/src/__support/threads/linux/raw_mutex.h
diff --git a/libc/config/config.json b/libc/config/config.json
index d6ef891b9f260..ee51ed03cdace 100644
--- a/libc/config/config.json
+++ b/libc/config/config.json
@@ -40,5 +40,15 @@
"value": true,
"doc": "Enable -fstack-protector-strong to defend against stack smashing attack."
}
+ },
+ "pthread": {
+ "LIBC_CONF_TIMEOUT_ENSURE_MONOTONICITY": {
+ "value": true,
+ "doc": "Automatically adjust timeout to CLOCK_MONOTONIC."
+ },
+ "LIBC_CONF_RAW_MUTEX_DEFAULT_SPIN_COUNT": {
+ "value": 100,
+ "doc": "Default number of spins before blocking if a mutex is contented."
+ }
}
}
diff --git a/libc/docs/configure.rst b/libc/docs/configure.rst
index 8f8c44caa1153..34ceadb071991 100644
--- a/libc/docs/configure.rst
+++ b/libc/docs/configure.rst
@@ -34,6 +34,9 @@ to learn about the defaults for your platform and target.
- ``LIBC_CONF_PRINTF_DISABLE_INDEX_MODE``: Disable index mode in the printf format string.
- ``LIBC_CONF_PRINTF_DISABLE_WRITE_INT``: Disable handling of %n in printf format string.
- ``LIBC_CONF_PRINTF_FLOAT_TO_STR_USE_MEGA_LONG_DOUBLE_TABLE``: Use large table for better printf long double performance.
+* **"pthread" options**
+ - ``LIBC_CONF_RAW_MUTEX_DEFAULT_SPIN_COUNT``: Default number of spins before blocking if a mutex is contented.
+ - ``LIBC_CONF_TIMEOUT_ENSURE_MONOTONICITY``: Automatically adjust timeout to CLOCK_MONOTONIC.
* **"string" options**
- ``LIBC_CONF_MEMSET_X86_USE_SOFTWARE_PREFETCHING``: Inserts prefetch for write instructions (PREFETCHW) for memset on x86 to recover performance when hardware prefetcher is disabled.
- ``LIBC_CONF_STRING_UNSAFE_WIDE_READ``: Read more than a byte at a time to perform byte-string operations like strlen.
diff --git a/libc/hdr/types/CMakeLists.txt b/libc/hdr/types/CMakeLists.txt
index 3a1bb2f3c340f..8e87642f66088 100644
--- a/libc/hdr/types/CMakeLists.txt
+++ b/libc/hdr/types/CMakeLists.txt
@@ -108,3 +108,12 @@ add_proxy_header_library(
libc.include.llvm-libc-types.struct_timeval
libc.include.sys_time
)
+
+add_proxy_header_library(
+ pid_t
+ HDRS
+ pid_t.h
+ FULL_BUILD_DEPENDS
+ libc.include.llvm-libc-types.pid_t
+ libc.include.sys_types
+)
diff --git a/libc/hdr/types/pid_t.h b/libc/hdr/types/pid_t.h
new file mode 100644
index 0000000000000..34306fc4118cf
--- /dev/null
+++ b/libc/hdr/types/pid_t.h
@@ -0,0 +1,22 @@
+//===-- Proxy for pid_t ---------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef LLVM_LIBC_HDR_TYPES_PID_T_H
+#define LLVM_LIBC_HDR_TYPES_PID_T_H
+
+#ifdef LIBC_FULL_BUILD
+
+#include "include/llvm-libc-types/pid_t.h"
+
+#else // Overlay mode
+
+#include <sys/types.h>
+
+#endif // LLVM_LIBC_FULL_BUILD
+
+#endif // LLVM_LIBC_HDR_TYPES_PID_T_H
diff --git a/libc/src/__support/File/dir.h b/libc/src/__support/File/dir.h
index a2f50f88e0e8b..ab445b6c0bec5 100644
--- a/libc/src/__support/File/dir.h
+++ b/libc/src/__support/File/dir.h
@@ -54,7 +54,7 @@ class Dir {
Dir(const Dir &) = delete;
explicit Dir(int fdesc)
- : fd(fdesc), readptr(0), fillsize(0), mutex(false, false, false) {}
+ : fd(fdesc), readptr(0), fillsize(0), mutex(false, false, false, false) {}
~Dir() = default;
Dir &operator=(const Dir &) = delete;
diff --git a/libc/src/__support/File/file.h b/libc/src/__support/File/file.h
index eafd3ab7d9107..6b41056cf40bd 100644
--- a/libc/src/__support/File/file.h
+++ b/libc/src/__support/File/file.h
@@ -154,7 +154,7 @@ class File {
uint8_t *buffer, size_t buffer_size, int buffer_mode,
bool owned, ModeFlags modeflags)
: platform_write(wf), platform_read(rf), platform_seek(sf),
- platform_close(cf), mutex(false, false, false), ungetc_buf(0),
+ platform_close(cf), mutex(false, false, false, false), ungetc_buf(0),
buf(buffer), bufsize(buffer_size), bufmode(buffer_mode), own_buf(owned),
mode(modeflags), pos(0), prev_op(FileOp::NONE), read_limit(0),
eof(false), err(false) {
diff --git a/libc/src/__support/threads/fork_callbacks.cpp b/libc/src/__support/threads/fork_callbacks.cpp
index 6efaf62f135ae..9ad3f38892f8f 100644
--- a/libc/src/__support/threads/fork_callbacks.cpp
+++ b/libc/src/__support/threads/fork_callbacks.cpp
@@ -33,7 +33,8 @@ class AtForkCallbackManager {
size_t next_index;
public:
- constexpr AtForkCallbackManager() : mtx(false, false, false), next_index(0) {}
+ constexpr AtForkCallbackManager()
+ : mtx(false, false, false, false), next_index(0) {}
bool register_triple(const ForkCallbackTriple &triple) {
cpp::lock_guard lock(mtx);
diff --git a/libc/src/__support/threads/gpu/mutex.h b/libc/src/__support/threads/gpu/mutex.h
index 71d0ef04cbfe5..9554083679335 100644
--- a/libc/src/__support/threads/gpu/mutex.h
+++ b/libc/src/__support/threads/gpu/mutex.h
@@ -19,7 +19,7 @@ namespace LIBC_NAMESPACE {
/// define the Mutex interface and require that only a single thread executes
/// code requiring a mutex lock.
struct Mutex {
- LIBC_INLINE constexpr Mutex(bool, bool, bool) {}
+ LIBC_INLINE constexpr Mutex(bool, bool, bool, bool) {}
LIBC_INLINE MutexError lock() { return MutexError::NONE; }
LIBC_INLINE MutexError unlock() { return MutexError::NONE; }
diff --git a/libc/src/__support/threads/linux/CMakeLists.txt b/libc/src/__support/threads/linux/CMakeLists.txt
index f6913ef083428..b57592473af39 100644
--- a/libc/src/__support/threads/linux/CMakeLists.txt
+++ b/libc/src/__support/threads/linux/CMakeLists.txt
@@ -22,12 +22,35 @@ add_header_library(
libc.src.__support.time.linux.abs_timeout
)
+set(raw_mutex_additional_flags)
+if (LIBC_CONF_TIMEOUT_ENSURE_MONOTONICITY)
+ set(raw_mutex_additional_flags -DLIBC_COPT_TIMEOUT_ENSURE_MONOTONICITY)
+endif()
+
+add_header_library(
+ raw_mutex
+ HDRS
+ mutex.h
+ DEPENDS
+ .futex_utils
+ libc.src.__support.threads.sleep
+ libc.src.__support.time.linux.abs_timeout
+ libc.src.__support.time.linux.monotonicity
+ libc.src.__support.CPP.optional
+ libc.hdr.types.pid_t
+ COMPILE_OPTIONS
+ -DLIBC_COPT_RAW_MUTEX_DEFAULT_SPIN_COUNT=${LIBC_CONF_RAW_MUTEX_DEFAULT_SPIN_COUNT}
+ ${raw_mutex_additional_flags}
+
+)
+
add_header_library(
mutex
HDRS
mutex.h
DEPENDS
.futex_utils
+ .raw_mutex
libc.src.__support.threads.mutex_common
)
diff --git a/libc/src/__support/threads/linux/mutex.h b/libc/src/__support/threads/linux/mutex.h
index 6702de4651686..915919cba37e6 100644
--- a/libc/src/__support/threads/linux/mutex.h
+++ b/libc/src/__support/threads/linux/mutex.h
@@ -9,114 +9,81 @@
#ifndef LLVM_LIBC_SRC___SUPPORT_THREADS_LINUX_MUTEX_H
#define LLVM_LIBC_SRC___SUPPORT_THREADS_LINUX_MUTEX_H
+#include "hdr/types/pid_t.h"
+#include "src/__support/CPP/optional.h"
#include "src/__support/threads/linux/futex_utils.h"
+#include "src/__support/threads/linux/raw_mutex.h"
#include "src/__support/threads/mutex_common.h"
namespace LIBC_NAMESPACE {
-struct Mutex {
- unsigned char timed;
+
+// TODO: support shared/recursive/robust mutexes.
+class Mutex final : private internal::RawMutex {
+ // reserved timed, may be useful when combined with other flags.
+ [[maybe_unused]] unsigned char timed;
unsigned char recursive;
unsigned char robust;
+ unsigned char pshared;
- void *owner;
+ // TLS address may not work across forked processes. Use thead id instead.
+ pid_t owner;
unsigned long long lock_count;
- Futex futex_word;
-
enum class LockState : FutexWordType {
- Free,
- Locked,
- Waiting,
+ Free = UNLOCKED,
+ Locked = LOCKED,
+ Waiting = CONTENTED,
};
public:
- constexpr Mutex(bool istimed, bool isrecursive, bool isrobust)
- : timed(istimed), recursive(isrecursive), robust(isrobust),
- owner(nullptr), lock_count(0),
- futex_word(FutexWordType(LockState::Free)) {}
-
- static MutexError init(Mutex *mutex, bool istimed, bool isrecur,
- bool isrobust) {
- mutex->timed = istimed;
+ LIBC_INLINE constexpr Mutex(bool is_timed, bool is_recursive, bool is_robust,
+ bool is_pshared)
+ : internal::RawMutex(), timed(is_timed), recursive(is_recursive),
+ robust(is_robust), pshared(is_pshared), owner(0), lock_count(0) {}
+
+ LIBC_INLINE static MutexError init(Mutex *mutex, bool is_timed, bool isrecur,
+ bool isrobust, bool is_pshared) {
+ internal::RawMutex::init(mutex);
+ mutex->timed = is_timed;
mutex->recursive = isrecur;
mutex->robust = isrobust;
- mutex->owner = nullptr;
+ mutex->pshared = is_pshared;
+ mutex->owner = 0;
mutex->lock_count = 0;
- mutex->futex_word.set(FutexWordType(LockState::Free));
return MutexError::NONE;
}
- static MutexError destroy(Mutex *) { return MutexError::NONE; }
-
- MutexError reset();
-
- MutexError lock() {
- bool was_waiting = false;
- while (true) {
- FutexWordType mutex_status = FutexWordType(LockState::Free);
- FutexWordType locked_status = FutexWordType(LockState::Locked);
+ LIBC_INLINE static MutexError destroy(Mutex *) { return MutexError::NONE; }
- if (futex_word.compare_exchange_strong(
- mutex_status, FutexWordType(LockState::Locked))) {
- if (was_waiting)
- futex_word = FutexWordType(LockState::Waiting);
- return MutexError::NONE;
- }
-
- switch (LockState(mutex_status)) {
- case LockState::Waiting:
- // If other threads are waiting already, then join them. Note that the
- // futex syscall will block if the futex data is still
- // `LockState::Waiting` (the 4th argument to the syscall function
- // below.)
- futex_word.wait(FutexWordType(LockState::Waiting));
- was_waiting = true;
- // Once woken up/unblocked, try everything all over.
- continue;
- case LockState::Locked:
- // Mutex has been locked by another thread so set the status to
- // LockState::Waiting.
- if (futex_word.compare_exchange_strong(
- locked_status, FutexWordType(LockState::Waiting))) {
- // If we are able to set the futex data to `LockState::Waiting`, then
- // we will wait for the futex to be woken up. Note again that the
- // following syscall will block only if the futex data is still
- // `LockState::Waiting`.
- futex_word.wait(FutexWordType(LockState::Waiting));
- was_waiting = true;
- }
- continue;
- case LockState::Free:
- // If it was LockState::Free, we shouldn't be here at all.
- return MutexError::BAD_LOCK_STATE;
- }
- }
+ LIBC_INLINE MutexError lock() {
+ this->internal::RawMutex::lock(
+ /* timeout */ cpp::nullopt,
+ /* is_pshared */ this->pshared,
+ /* spin_count */ LIBC_COPT_RAW_MUTEX_DEFAULT_SPIN_COUNT);
+ // TODO: record owner and lock count.
+ return MutexError::NONE;
}
- MutexError unlock() {
- while (true) {
- FutexWordType mutex_status = FutexWordType(LockState::Waiting);
- if (futex_word.compare_exchange_strong(mutex_status,
- FutexWordType(LockState::Free))) {
- // If any thread is waiting to be woken up, then do it.
- futex_word.notify_one();
- return MutexError::NONE;
- }
+ LIBC_INLINE MutexError timed_lock(internal::AbsTimeout abs_time) {
+ if (this->internal::RawMutex::lock(
+ abs_time, /* is_shared */ this->pshared,
+ /* spin_count */ LIBC_COPT_RAW_MUTEX_DEFAULT_SPIN_COUNT))
+ return MutexError::NONE;
+ return MutexError::TIMEOUT;
+ }
- if (mutex_status == FutexWordType(LockState::Locked)) {
- // If nobody was waiting at this point, just free it.
- if (futex_word.compare_exchange_strong(mutex_status,
- FutexWordType(LockState::Free)))
- return MutexError::NONE;
- } else {
- // This can happen, for example if some thread tries to unlock an
- // already free mutex.
- return MutexError::UNLOCK_WITHOUT_LOCK;
- }
- }
+ LIBC_INLINE MutexError unlock() {
+ if (this->internal::RawMutex::unlock(/* is_shared */ this->pshared))
+ return MutexError::NONE;
+ return MutexError::UNLOCK_WITHOUT_LOCK;
}
- MutexError trylock();
+ LIBC_INLINE MutexError try_lock() {
+ if (this->internal::RawMutex::try_lock())
+ return MutexError::NONE;
+ return MutexError::BUSY;
+ }
+ friend struct CndVar;
};
} // namespace LIBC_NAMESPACE
diff --git a/libc/src/__support/threads/linux/raw_mutex.h b/libc/src/__support/threads/linux/raw_mutex.h
new file mode 100644
index 0000000000000..93da725d66d61
--- /dev/null
+++ b/libc/src/__support/threads/linux/raw_mutex.h
@@ -0,0 +1,116 @@
+//===--- Implementation of a Linux RawMutex class ---------------*- C++ -*-===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+#ifndef LLVM_LIBC_SRC___SUPPORT_THREADS_LINUX_RAW_MUTEX_H
+#define LLVM_LIBC_SRC___SUPPORT_THREADS_LINUX_RAW_MUTEX_H
+
+#include "futex_word.h"
+#include "src/__support/CPP/optional.h"
+#include "src/__support/common.h"
+#include "src/__support/macros/attributes.h"
+#include "src/__support/threads/linux/futex_utils.h"
+#include "src/__support/threads/sleep.h"
+#include "src/__support/time/linux/abs_timeout.h"
+#ifdef LIBC_COPT_TIMEOUT_ENSURE_MONOTONICITY
+#include "src/__support/time/linux/monotonicity.h"
+#endif
+
+#ifndef LIBC_COPT_RAW_MUTEX_DEFAULT_SPIN_COUNT
+#define LIBC_COPT_RAW_MUTEX_DEFAULT_SPIN_COUNT 100
+#endif
+
+namespace LIBC_NAMESPACE {
+namespace internal {
+// Lock is a simple timable lock for internal usage.
+// This is separated from Mutex because this one does not need to consider
+// rubustness and reentrancy. Also, this one has spin optimization for shorter
+// critical sections.
+class RawMutex {
+protected:
+ Futex futex;
+ LIBC_INLINE_VAR static constexpr FutexWordType UNLOCKED = 0b00;
+ LIBC_INLINE_VAR static constexpr FutexWordType LOCKED = 0b01;
+ LIBC_INLINE_VAR static constexpr FutexWordType CONTENTED = 0b10;
+
+ LIBC_INLINE FutexWordType spin(uint_fast32_t spin_count) {
+ FutexWordType result;
+ for (;;) {
+ result = futex.load(cpp::MemoryOrder::RELAXED);
+ // spin until one of the following conditions is met:
+ // - the mutex is unlocked
+ // - the mutex is contented
+ // - the spin count reaches 0
+ if (result != LOCKED || spin_count == 0)
+ return result;
+ // Pause the pipeline to avoid extraneous memory operations due to
+ // speculation.
+ sleep_briefly();
+ spin_count--;
+ };
+ }
+
+ // Return true if the lock is acquired. Return false if timeout happens before
+ // the lock is acquired.
+ [[gnu::cold]] LIBC_INLINE bool
+ lock_contented(cpp::optional<Futex::Timeout> timeout, bool is_pshared,
+ uint_fast32_t spin_count) {
+ FutexWordType state = spin(spin_count);
+ // Before go into contention state, try to grab the lock.
+ if (state == UNLOCKED &&
+ futex.compare_exchange_strong(state, LOCKED, cpp::MemoryOrder::ACQUIRE,
+ cpp::MemoryOrder::RELAXED))
+ return true;
+#ifdef LIBC_COPT_TIMEOUT_ENSURE_MONOTONICITY
+ if (timeout)
+ ensure_monotonicity(*timeout);
+#endif
+ for (;;) {
+ // Try to grab the lock if it is unlocked. Mark contented if it is locked.
+ if (state != CONTENTED &&
+ futex.exchange(CONTENTED, cpp::MemoryOrder::ACQUIRE) == UNLOCKED)
+ return true;
+ // Contention persists. Park the thread and wait for further notification.
+ if (-ETIMEDOUT == futex.wait(CONTENTED, timeout, is_pshared))
+ return false;
+ // Continue to spin after waking up.
+ state = spin(spin_count);
+ }
+ }
+
+ [[gnu::cold]] LIBC_INLINE void wake(bool is_pshared) {
+ futex.notify_one(is_pshared);
+ }
+
+public:
+ LIBC_INLINE static void init(RawMutex *mutex) { mutex->futex = UNLOCKED; }
+ LIBC_INLINE constexpr RawMutex() : futex(UNLOCKED) {}
+ LIBC_INLINE bool try_lock() {
+ FutexWordType expected = UNLOCKED;
+ // Use strong version since this is a one-time operation.
+ return futex.compare_exchange_strong(
+ expected, LOCKED, cpp::MemoryOrder::ACQUIRE, cpp::MemoryOrder::RELAXED);
+ }
+ LIBC_INLINE bool
+ lock(cpp::optional<Futex::Timeout> timeout = cpp::nullopt,
+ bool is_shared = false,
+ uint_fast32_t spin_count = LIBC_COPT_RAW_MUTEX_DEFAULT_SPIN_COUNT) {
+ // Timeout will not be checked if immediate lock is possible.
+ return try_lock() || lock_contented(timeout, is_shared, spin_count);
+ }
+ LIBC_INLINE bool unlock(bool is_pshared = false) {
+ FutexWordType prev = futex.exchange(UNLOCKED, cpp::MemoryOrder::RELEASE);
+ // if there is someone waiting, wake them up
+ if (prev == CONTENTED)
+ wake(is_pshared);
+ // Detect invalid unlock operation.
+ return prev == LOCKED;
+ }
+};
+} // namespace internal
+} // namespace LIBC_NAMESPACE
+
+#endif // LLVM_LIBC_SRC___SUPPORT_THREADS_LINUX_RAW_MUTEX_H
diff --git a/libc/src/__support/threads/thread.cpp b/libc/src/__support/threads/thread.cpp
index 7b02f8246e24b..2bea01bf332b3 100644
--- a/libc/src/__support/threads/thread.cpp
+++ b/libc/src/__support/threads/thread.cpp
@@ -54,7 +54,7 @@ class TSSKeyMgr {
cpp::array<TSSKeyUnit, TSS_KEY_COUNT> units;
public:
- constexpr TSSKeyMgr() : mtx(false, false, false) {}
+ constexpr TSSKeyMgr() : mtx(false, false, false, false) {}
cpp::optional<unsigned int> new_key(TSSDtor *dtor) {
cpp::lock_guard lock(mtx);
@@ -111,7 +111,7 @@ class ThreadAtExitCallbackMgr {
FixedVector<AtExitUnit, 1024> callback_list;
public:
- constexpr ThreadAtExitCallbackMgr() : mtx(false, false, false) {}
+ constexpr ThreadAtExitCallbackMgr() : mtx(false, false, false, false) {}
int add_callback(AtExitCallback *callback, void *obj) {
cpp::lock_guard lock(mtx);
diff --git a/libc/src/pthread/pthread_mutex_init.cpp b/libc/src/pthread/pthread_mutex_init.cpp
index f64d1fe68ad20..35615c449c044 100644
--- a/libc/src/pthread/pthread_mutex_init.cpp
+++ b/libc/src/pthread/pthread_mutex_init.cpp
@@ -26,9 +26,10 @@ LLVM_LIBC_FUNCTION(int, pthread_mutex_init,
const pthread_mutexattr_t *__restrict attr)) {
auto mutexattr = attr == nullptr ? DEFAULT_MUTEXATTR : *attr;
auto err =
- Mutex::init(reinterpret_cast<Mutex *>(m), false,
+ Mutex::init(reinterpret_cast<Mutex *>(m), /* timeout is supported */ true,
get_mutexattr_type(mutexattr) & PTHREAD_MUTEX_RECURSIVE,
- get_mutexattr_robust(mutexattr) & PTHREAD_MUTEX_ROBUST);
+ get_mutexattr_robust(mutexattr) & PTHREAD_MUTEX_ROBUST,
+ get_mutexattr_pshared(mutexattr) & PTHREAD_PROCESS_SHARED);
return err == MutexError::NONE ? 0 : EAGAIN;
}
diff --git a/libc/src/pthread/pthread_mutexattr.h b/libc/src/pthread/pthread_mutexattr.h
index 8b435854416a7..292ceebe049f9 100644
--- a/libc/src/pthread/pthread_mutexattr.h
+++ b/libc/src/pthread/pthread_mutexattr.h
@@ -43,6 +43,11 @@ LIBC_INLINE int get_mutexattr_robust(pthread_mutexattr_t attr) {
unsigned(PThreadMutexAttrPos::ROBUST_SHIFT);
}
+LIBC_INLINE int get_mutexattr_pshared(pthread_mutexattr_t attr) {
+ return (attr & unsigned(PThreadMutexAttrPos::PSHARED_MASK)) >>
+ unsigned(PThreadMutexAttrPos::PSHARED_SHIFT);
+}
+
} // namespace LIBC_NAMESPACE
#endif // LLVM_LIBC_SRC_PTHREAD_PTHREAD_MUTEXATTR_H
diff --git a/libc/src/stdlib/atexit.cpp b/libc/src/stdlib/atexit.cpp
index 4f0497444773d..fd24811000123 100644
--- a/libc/src/stdlib/atexit.cpp
+++ b/libc/src/stdlib/atexit.cpp
@@ -17,7 +17,7 @@ namespace LIBC_NAMESPACE {
namespace {
-Mutex handler_list_mtx(false, false, false);
+Mutex handler_list_mtx(false, false, false, false);
using AtExitCallback = void(void *);
using StdCAtExitCallback = void(void);
diff --git a/libc/src/threads/linux/CMakeLists.txt b/libc/src/threads/linux/CMakeLists.txt
index a5a02e47aab38..6c8e0845faf4c 100644
--- a/libc/src/threads/linux/CMakeLists.txt
+++ b/libc/src/threads/linux/CMakeLists.txt
@@ -9,6 +9,7 @@ add_header_library(
libc.src.__support.CPP.mutex
libc.src.__support.OSUtil.osutil
libc.src.__support.threads.mutex
+ libc.src.__support.threads.linux.raw_mutex
libc.src.__support.threads.linux.futex_utils
)
diff --git a/libc/src/threads/mtx_init.cpp b/libc/src/threads/mtx_init.cpp
index 74d08d33b116b..7cd848d12ee6d 100644
--- a/libc/src/threads/mtx_init.cpp
+++ b/libc/src/threads/mtx_init.cpp
@@ -20,7 +20,8 @@ static_assert(sizeof(Mutex) <= sizeof(mtx_t),
LLVM_LIBC_FUNCTION(int, mtx_init, (mtx_t * m, int type)) {
auto err = Mutex::init(reinterpret_cast<Mutex *>(m), type & mtx_timed,
- type & mtx_recursive, 0);
+ type & mtx_recursive, /* is_robust */ false,
+ /* is_pshared */ false);
return err == MutexError::NONE ? thrd_success : thrd_error;
}
diff --git a/libc/test/integration/src/__support/threads/thread_detach_test.cpp b/libc/test/integration/src/__support/threads/thread_detach_test.cpp
index 697d991d07547..b11e704551aab 100644
--- a/libc/test/integration/src/__support/threads/thread_detach_test.cpp
+++ b/libc/test/integration/src/__support/threads/thread_detach_test.cpp
@@ -10,7 +10,7 @@
#include "src/__support/threads/thread.h"
#include "test/IntegrationTest/test.h"
-LIBC_NAMESPACE::Mutex mutex(false, false, false);
+LIBC_NAMESPACE::Mutex mutex(false, false, false, false);
int func(void *) {
mutex.lock();
>From 70922e20d23316e53461c7c1267999655cdfd9d1 Mon Sep 17 00:00:00 2001
From: Schrodinger ZHU Yifan <yifanzhu at rochester.edu>
Date: Tue, 14 May 2024 16:30:31 -0400
Subject: [PATCH 02/15] [libc] fix missing dep
---
libc/include/llvm-libc-types/CMakeLists.txt | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/libc/include/llvm-libc-types/CMakeLists.txt b/libc/include/llvm-libc-types/CMakeLists.txt
index 018b6c58316c3..9e9a2fabd5fa4 100644
--- a/libc/include/llvm-libc-types/CMakeLists.txt
+++ b/libc/include/llvm-libc-types/CMakeLists.txt
@@ -7,7 +7,7 @@ add_header(__call_once_func_t HDR __call_once_func_t.h)
add_header(__exec_argv_t HDR __exec_argv_t.h)
add_header(__exec_envp_t HDR __exec_envp_t.h)
add_header(__futex_word HDR __futex_word.h)
-add_header(__mutex_type HDR __mutex_type.h DEPENDS .__futex_word)
+add_header(__mutex_type HDR __mutex_type.h DEPENDS .__futex_word .pid_t)
add_header(__pthread_once_func_t HDR __pthread_once_func_t.h)
add_header(__pthread_start_t HDR __pthread_start_t.h)
add_header(__pthread_tss_dtor_t HDR __pthread_tss_dtor_t.h)
>From 718529c907ba934a1691b7dcf0e3cec206e09d3c Mon Sep 17 00:00:00 2001
From: Schrodinger ZHU Yifan <yifanzhu at rochester.edu>
Date: Tue, 14 May 2024 16:34:17 -0400
Subject: [PATCH 03/15] [libc] adjust fields
---
libc/include/llvm-libc-types/CMakeLists.txt | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/libc/include/llvm-libc-types/CMakeLists.txt b/libc/include/llvm-libc-types/CMakeLists.txt
index 9e9a2fabd5fa4..8ac7be391fb2e 100644
--- a/libc/include/llvm-libc-types/CMakeLists.txt
+++ b/libc/include/llvm-libc-types/CMakeLists.txt
@@ -7,6 +7,7 @@ add_header(__call_once_func_t HDR __call_once_func_t.h)
add_header(__exec_argv_t HDR __exec_argv_t.h)
add_header(__exec_envp_t HDR __exec_envp_t.h)
add_header(__futex_word HDR __futex_word.h)
+add_header(pid_t HDR pid_t.h)
add_header(__mutex_type HDR __mutex_type.h DEPENDS .__futex_word .pid_t)
add_header(__pthread_once_func_t HDR __pthread_once_func_t.h)
add_header(__pthread_start_t HDR __pthread_start_t.h)
@@ -45,7 +46,6 @@ add_header(mtx_t HDR mtx_t.h DEPENDS .__futex_word .__mutex_type)
add_header(nlink_t HDR nlink_t.h)
add_header(off_t HDR off_t.h)
add_header(once_flag HDR once_flag.h DEPENDS .__futex_word)
-add_header(pid_t HDR pid_t.h)
add_header(posix_spawn_file_actions_t HDR posix_spawn_file_actions_t.h)
add_header(posix_spawnattr_t HDR posix_spawnattr_t.h)
add_header(pthread_attr_t HDR pthread_attr_t.h DEPENDS .size_t)
>From 299c563361e4928786f4788f5712ee51d22e2495 Mon Sep 17 00:00:00 2001
From: Schrodinger ZHU Yifan <yifanzhu at rochester.edu>
Date: Tue, 14 May 2024 16:34:53 -0400
Subject: [PATCH 04/15] [libc] fix typo
---
libc/src/__support/threads/linux/mutex.h | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/libc/src/__support/threads/linux/mutex.h b/libc/src/__support/threads/linux/mutex.h
index 915919cba37e6..4a49d7c68b2e5 100644
--- a/libc/src/__support/threads/linux/mutex.h
+++ b/libc/src/__support/threads/linux/mutex.h
@@ -25,7 +25,7 @@ class Mutex final : private internal::RawMutex {
unsigned char robust;
unsigned char pshared;
- // TLS address may not work across forked processes. Use thead id instead.
+ // TLS address may not work across forked processes. Use thread id instead.
pid_t owner;
unsigned long long lock_count;
>From 0148b55150e311cbd84dc23ba31aab75944c617d Mon Sep 17 00:00:00 2001
From: Schrodinger ZHU Yifan <yifanzhu at rochester.edu>
Date: Tue, 14 May 2024 16:37:01 -0400
Subject: [PATCH 05/15] [libc] fix unlock detection
---
libc/src/__support/threads/linux/raw_mutex.h | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/libc/src/__support/threads/linux/raw_mutex.h b/libc/src/__support/threads/linux/raw_mutex.h
index 93da725d66d61..f2209f50cc458 100644
--- a/libc/src/__support/threads/linux/raw_mutex.h
+++ b/libc/src/__support/threads/linux/raw_mutex.h
@@ -107,7 +107,7 @@ class RawMutex {
if (prev == CONTENTED)
wake(is_pshared);
// Detect invalid unlock operation.
- return prev == LOCKED;
+ return prev != UNLOCKED;
}
};
} // namespace internal
>From be017f0be3a06cfc40a76df23079e8c3ef699bb2 Mon Sep 17 00:00:00 2001
From: Schrodinger ZHU Yifan <yifanzhu at rochester.edu>
Date: Tue, 14 May 2024 16:37:44 -0400
Subject: [PATCH 06/15] [libc] fix typo
---
libc/src/__support/threads/linux/raw_mutex.h | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/libc/src/__support/threads/linux/raw_mutex.h b/libc/src/__support/threads/linux/raw_mutex.h
index f2209f50cc458..778d036851af4 100644
--- a/libc/src/__support/threads/linux/raw_mutex.h
+++ b/libc/src/__support/threads/linux/raw_mutex.h
@@ -27,7 +27,7 @@ namespace LIBC_NAMESPACE {
namespace internal {
// Lock is a simple timable lock for internal usage.
// This is separated from Mutex because this one does not need to consider
-// rubustness and reentrancy. Also, this one has spin optimization for shorter
+// robustness and reentrancy. Also, this one has spin optimization for shorter
// critical sections.
class RawMutex {
protected:
>From 64ce89f8821d3e758ec5af842b80b82bfa102a40 Mon Sep 17 00:00:00 2001
From: Schrodinger ZHU Yifan <yifanzhu at rochester.edu>
Date: Tue, 14 May 2024 18:41:58 -0400
Subject: [PATCH 07/15] [libc] more cleanups
---
libc/src/__support/threads/linux/mutex.h | 9 +--------
1 file changed, 1 insertion(+), 8 deletions(-)
diff --git a/libc/src/__support/threads/linux/mutex.h b/libc/src/__support/threads/linux/mutex.h
index 4a49d7c68b2e5..ea43892820bc2 100644
--- a/libc/src/__support/threads/linux/mutex.h
+++ b/libc/src/__support/threads/linux/mutex.h
@@ -20,7 +20,7 @@ namespace LIBC_NAMESPACE {
// TODO: support shared/recursive/robust mutexes.
class Mutex final : private internal::RawMutex {
// reserved timed, may be useful when combined with other flags.
- [[maybe_unused]] unsigned char timed;
+ unsigned char timed;
unsigned char recursive;
unsigned char robust;
unsigned char pshared;
@@ -29,12 +29,6 @@ class Mutex final : private internal::RawMutex {
pid_t owner;
unsigned long long lock_count;
- enum class LockState : FutexWordType {
- Free = UNLOCKED,
- Locked = LOCKED,
- Waiting = CONTENTED,
- };
-
public:
LIBC_INLINE constexpr Mutex(bool is_timed, bool is_recursive, bool is_robust,
bool is_pshared)
@@ -83,7 +77,6 @@ class Mutex final : private internal::RawMutex {
return MutexError::NONE;
return MutexError::BUSY;
}
- friend struct CndVar;
};
} // namespace LIBC_NAMESPACE
>From 8dd098e8849f092659cdb85756c00c731aa18eac Mon Sep 17 00:00:00 2001
From: Schrodinger ZHU Yifan <yifanzhu at rochester.edu>
Date: Tue, 14 May 2024 20:04:06 -0400
Subject: [PATCH 08/15] [libc] reword `contented` to `in contenion`
---
libc/config/config.json | 2 +-
libc/docs/configure.rst | 2 +-
libc/src/__support/threads/linux/raw_mutex.h | 21 ++++++++++----------
3 files changed, 13 insertions(+), 12 deletions(-)
diff --git a/libc/config/config.json b/libc/config/config.json
index ee51ed03cdace..ec6cd2f9769b7 100644
--- a/libc/config/config.json
+++ b/libc/config/config.json
@@ -48,7 +48,7 @@
},
"LIBC_CONF_RAW_MUTEX_DEFAULT_SPIN_COUNT": {
"value": 100,
- "doc": "Default number of spins before blocking if a mutex is contented."
+ "doc": "Default number of spins before blocking if a mutex is in contention."
}
}
}
diff --git a/libc/docs/configure.rst b/libc/docs/configure.rst
index 34ceadb071991..64657f9eabadf 100644
--- a/libc/docs/configure.rst
+++ b/libc/docs/configure.rst
@@ -35,7 +35,7 @@ to learn about the defaults for your platform and target.
- ``LIBC_CONF_PRINTF_DISABLE_WRITE_INT``: Disable handling of %n in printf format string.
- ``LIBC_CONF_PRINTF_FLOAT_TO_STR_USE_MEGA_LONG_DOUBLE_TABLE``: Use large table for better printf long double performance.
* **"pthread" options**
- - ``LIBC_CONF_RAW_MUTEX_DEFAULT_SPIN_COUNT``: Default number of spins before blocking if a mutex is contented.
+ - ``LIBC_CONF_RAW_MUTEX_DEFAULT_SPIN_COUNT``: Default number of spins before blocking if a mutex is in contention.
- ``LIBC_CONF_TIMEOUT_ENSURE_MONOTONICITY``: Automatically adjust timeout to CLOCK_MONOTONIC.
* **"string" options**
- ``LIBC_CONF_MEMSET_X86_USE_SOFTWARE_PREFETCHING``: Inserts prefetch for write instructions (PREFETCHW) for memset on x86 to recover performance when hardware prefetcher is disabled.
diff --git a/libc/src/__support/threads/linux/raw_mutex.h b/libc/src/__support/threads/linux/raw_mutex.h
index 778d036851af4..cc598e8db1d75 100644
--- a/libc/src/__support/threads/linux/raw_mutex.h
+++ b/libc/src/__support/threads/linux/raw_mutex.h
@@ -34,7 +34,7 @@ class RawMutex {
Futex futex;
LIBC_INLINE_VAR static constexpr FutexWordType UNLOCKED = 0b00;
LIBC_INLINE_VAR static constexpr FutexWordType LOCKED = 0b01;
- LIBC_INLINE_VAR static constexpr FutexWordType CONTENTED = 0b10;
+ LIBC_INLINE_VAR static constexpr FutexWordType IN_CONTENTION = 0b10;
LIBC_INLINE FutexWordType spin(uint_fast32_t spin_count) {
FutexWordType result;
@@ -42,7 +42,7 @@ class RawMutex {
result = futex.load(cpp::MemoryOrder::RELAXED);
// spin until one of the following conditions is met:
// - the mutex is unlocked
- // - the mutex is contented
+ // - the mutex is in contention
// - the spin count reaches 0
if (result != LOCKED || spin_count == 0)
return result;
@@ -56,8 +56,8 @@ class RawMutex {
// Return true if the lock is acquired. Return false if timeout happens before
// the lock is acquired.
[[gnu::cold]] LIBC_INLINE bool
- lock_contented(cpp::optional<Futex::Timeout> timeout, bool is_pshared,
- uint_fast32_t spin_count) {
+ lock_slow(cpp::optional<Futex::Timeout> timeout, bool is_pshared,
+ uint_fast32_t spin_count) {
FutexWordType state = spin(spin_count);
// Before go into contention state, try to grab the lock.
if (state == UNLOCKED &&
@@ -69,12 +69,13 @@ class RawMutex {
ensure_monotonicity(*timeout);
#endif
for (;;) {
- // Try to grab the lock if it is unlocked. Mark contented if it is locked.
- if (state != CONTENTED &&
- futex.exchange(CONTENTED, cpp::MemoryOrder::ACQUIRE) == UNLOCKED)
+ // Try to grab the lock if it is unlocked. Mark the contention flag if it
+ // is locked.
+ if (state != IN_CONTENTION &&
+ futex.exchange(IN_CONTENTION, cpp::MemoryOrder::ACQUIRE) == UNLOCKED)
return true;
// Contention persists. Park the thread and wait for further notification.
- if (-ETIMEDOUT == futex.wait(CONTENTED, timeout, is_pshared))
+ if (-ETIMEDOUT == futex.wait(IN_CONTENTION, timeout, is_pshared))
return false;
// Continue to spin after waking up.
state = spin(spin_count);
@@ -99,12 +100,12 @@ class RawMutex {
bool is_shared = false,
uint_fast32_t spin_count = LIBC_COPT_RAW_MUTEX_DEFAULT_SPIN_COUNT) {
// Timeout will not be checked if immediate lock is possible.
- return try_lock() || lock_contented(timeout, is_shared, spin_count);
+ return try_lock() || lock_slow(timeout, is_shared, spin_count);
}
LIBC_INLINE bool unlock(bool is_pshared = false) {
FutexWordType prev = futex.exchange(UNLOCKED, cpp::MemoryOrder::RELEASE);
// if there is someone waiting, wake them up
- if (prev == CONTENTED)
+ if (prev == IN_CONTENTION)
wake(is_pshared);
// Detect invalid unlock operation.
return prev != UNLOCKED;
>From 2521dfb48aaca5851e417e64fa0009a1a5fe91f4 Mon Sep 17 00:00:00 2001
From: Schrodinger ZHU Yifan <yifanzhu at rochester.edu>
Date: Thu, 16 May 2024 14:29:14 -0400
Subject: [PATCH 09/15] add raw_mutex tests
---
libc/test/src/__support/CMakeLists.txt | 1 +
.../test/src/__support/threads/CMakeLists.txt | 4 ++
.../__support/threads/linux/CMakeLists.txt | 12 ++++
.../threads/linux/raw_mutex_test.cpp | 69 +++++++++++++++++++
4 files changed, 86 insertions(+)
create mode 100644 libc/test/src/__support/threads/CMakeLists.txt
create mode 100644 libc/test/src/__support/threads/linux/CMakeLists.txt
create mode 100644 libc/test/src/__support/threads/linux/raw_mutex_test.cpp
diff --git a/libc/test/src/__support/CMakeLists.txt b/libc/test/src/__support/CMakeLists.txt
index 8bdc56ee59ccc..663aa2bb82cae 100644
--- a/libc/test/src/__support/CMakeLists.txt
+++ b/libc/test/src/__support/CMakeLists.txt
@@ -207,3 +207,4 @@ add_subdirectory(FPUtil)
add_subdirectory(fixed_point)
add_subdirectory(HashTable)
add_subdirectory(time)
+add_subdirectory(threads)
diff --git a/libc/test/src/__support/threads/CMakeLists.txt b/libc/test/src/__support/threads/CMakeLists.txt
new file mode 100644
index 0000000000000..70d68abba6e30
--- /dev/null
+++ b/libc/test/src/__support/threads/CMakeLists.txt
@@ -0,0 +1,4 @@
+add_custom_target(libc-support-threads-tests)
+if(EXISTS ${CMAKE_CURRENT_SOURCE_DIR}/${LIBC_TARGET_OS})
+ add_subdirectory(${LIBC_TARGET_OS})
+endif()
diff --git a/libc/test/src/__support/threads/linux/CMakeLists.txt b/libc/test/src/__support/threads/linux/CMakeLists.txt
new file mode 100644
index 0000000000000..ce9becb923c3a
--- /dev/null
+++ b/libc/test/src/__support/threads/linux/CMakeLists.txt
@@ -0,0 +1,12 @@
+add_libc_test(
+ raw_mutex_test
+ SUITE
+ libc-support-threads-tests
+ SRCS
+ raw_mutex_test.cpp
+ DEPENDS
+ libc.src.__support.threads.linux.raw_mutex
+ libc.src.sys.mman.mmap
+ libc.src.sys.mman.munmap
+ libc.src.stdlib.exit
+)
diff --git a/libc/test/src/__support/threads/linux/raw_mutex_test.cpp b/libc/test/src/__support/threads/linux/raw_mutex_test.cpp
new file mode 100644
index 0000000000000..baf8965986c2c
--- /dev/null
+++ b/libc/test/src/__support/threads/linux/raw_mutex_test.cpp
@@ -0,0 +1,69 @@
+#include "include/llvm-libc-macros/linux/time-macros.h"
+#include "src/__support/CPP/atomic.h"
+#include "src/__support/OSUtil/syscall.h"
+#include "src/__support/threads/linux/raw_mutex.h"
+#include "src/__support/threads/sleep.h"
+#include "src/__support/time/linux/clock_gettime.h"
+#include "src/stdlib/exit.h"
+#include "src/sys/mman/mmap.h"
+#include "src/sys/mman/munmap.h"
+#include "test/UnitTest/Test.h"
+#include <sys/syscall.h>
+
+TEST(LlvmLibcSupportThreadsRawMutexTest, SmokeTest) {
+ LIBC_NAMESPACE::internal::RawMutex mutex;
+ ASSERT_TRUE(mutex.lock());
+ ASSERT_TRUE(mutex.unlock());
+ ASSERT_TRUE(mutex.try_lock());
+ ASSERT_FALSE(mutex.try_lock());
+ ASSERT_TRUE(mutex.unlock());
+ ASSERT_FALSE(mutex.unlock());
+}
+
+TEST(LlvmLibcSupportThreadsRawMutexTest, Timeout) {
+ LIBC_NAMESPACE::internal::RawMutex mutex;
+ ASSERT_TRUE(mutex.lock());
+ timespec ts;
+ LIBC_NAMESPACE::internal::clock_gettime(CLOCK_MONOTONIC, &ts);
+ ts.tv_sec += 1;
+ // Timeout will be respected when deadlock happens.
+ auto timeout = LIBC_NAMESPACE::internal::AbsTimeout::from_timespec(ts, false);
+ ASSERT_TRUE(timeout.has_value());
+ ASSERT_FALSE(mutex.lock(*timeout));
+ ASSERT_TRUE(mutex.unlock());
+ ASSERT_TRUE(mutex.lock(*timeout));
+ ASSERT_TRUE(mutex.unlock());
+ // If a lock can be acquired directly, expired timeout will not count.
+ ASSERT_TRUE(mutex.lock(*timeout));
+ ASSERT_TRUE(mutex.unlock());
+}
+
+TEST(LlvmLibcSupportThreadsRawMutexTest, PSharedLock) {
+ struct SharedData {
+ LIBC_NAMESPACE::internal::RawMutex mutex;
+ LIBC_NAMESPACE::cpp::Atomic<size_t> finished;
+ int data;
+ };
+ void *addr =
+ LIBC_NAMESPACE::mmap(nullptr, sizeof(SharedData), PROT_READ | PROT_WRITE,
+ MAP_ANONYMOUS | MAP_SHARED, -1, 0);
+ ASSERT_NE(addr, MAP_FAILED);
+ auto *shared = reinterpret_cast<SharedData *>(addr);
+ shared->data = 0;
+ LIBC_NAMESPACE::internal::RawMutex::init(&shared->mutex);
+ // Avoid pull in our own implementation of pthread_t.
+ long pid = LIBC_NAMESPACE::syscall_impl<long>(SYS_fork);
+ for (int i = 0; i < 10000; ++i) {
+ shared->mutex.lock(LIBC_NAMESPACE::cpp::nullopt, true);
+ shared->data++;
+ shared->mutex.unlock(true);
+ }
+ // Mark the thread as finished.
+ shared->finished.fetch_add(1);
+ // early exit to avoid output pollution
+ if (pid == 0)
+ LIBC_NAMESPACE::exit(0);
+ while (shared->finished.load() != 2)
+ LIBC_NAMESPACE::sleep_briefly();
+ ASSERT_EQ(shared->data, 20000);
+}
>From 7fe1a4538ca6e306c89c0389a9c4c2b749d40dff Mon Sep 17 00:00:00 2001
From: Schrodinger ZHU Yifan <yifanzhu at rochester.edu>
Date: Thu, 16 May 2024 14:39:12 -0400
Subject: [PATCH 10/15] unmap the page after test
---
libc/test/src/__support/threads/linux/raw_mutex_test.cpp | 1 +
1 file changed, 1 insertion(+)
diff --git a/libc/test/src/__support/threads/linux/raw_mutex_test.cpp b/libc/test/src/__support/threads/linux/raw_mutex_test.cpp
index baf8965986c2c..94ecf2e4e06ff 100644
--- a/libc/test/src/__support/threads/linux/raw_mutex_test.cpp
+++ b/libc/test/src/__support/threads/linux/raw_mutex_test.cpp
@@ -66,4 +66,5 @@ TEST(LlvmLibcSupportThreadsRawMutexTest, PSharedLock) {
while (shared->finished.load() != 2)
LIBC_NAMESPACE::sleep_briefly();
ASSERT_EQ(shared->data, 20000);
+ LIBC_NAMESPACE::munmap(addr, sizeof(SharedData));
}
>From 749f23e767d8f4623e4e9a51096a204586991718 Mon Sep 17 00:00:00 2001
From: Schrodinger ZHU Yifan <yifanzhu at rochester.edu>
Date: Thu, 16 May 2024 19:04:37 -0400
Subject: [PATCH 11/15] address CR
---
libc/src/__support/threads/linux/mutex.h | 14 ++++++--------
libc/src/__support/threads/linux/raw_mutex.h | 6 ++++--
.../src/__support/threads/linux/raw_mutex_test.cpp | 8 ++++++++
3 files changed, 18 insertions(+), 10 deletions(-)
diff --git a/libc/src/__support/threads/linux/mutex.h b/libc/src/__support/threads/linux/mutex.h
index ea43892820bc2..292c61fb2d789 100644
--- a/libc/src/__support/threads/linux/mutex.h
+++ b/libc/src/__support/threads/linux/mutex.h
@@ -49,29 +49,27 @@ class Mutex final : private internal::RawMutex {
LIBC_INLINE static MutexError destroy(Mutex *) { return MutexError::NONE; }
+ // TODO: record owner and lock count.
LIBC_INLINE MutexError lock() {
this->internal::RawMutex::lock(
- /* timeout */ cpp::nullopt,
- /* is_pshared */ this->pshared,
- /* spin_count */ LIBC_COPT_RAW_MUTEX_DEFAULT_SPIN_COUNT);
- // TODO: record owner and lock count.
+ /* timeout */ cpp::nullopt, this->pshared);
return MutexError::NONE;
}
+ // TODO: record owner and lock count.
LIBC_INLINE MutexError timed_lock(internal::AbsTimeout abs_time) {
- if (this->internal::RawMutex::lock(
- abs_time, /* is_shared */ this->pshared,
- /* spin_count */ LIBC_COPT_RAW_MUTEX_DEFAULT_SPIN_COUNT))
+ if (this->internal::RawMutex::lock(abs_time, this->pshared))
return MutexError::NONE;
return MutexError::TIMEOUT;
}
LIBC_INLINE MutexError unlock() {
- if (this->internal::RawMutex::unlock(/* is_shared */ this->pshared))
+ if (this->internal::RawMutex::unlock(this->pshared))
return MutexError::NONE;
return MutexError::UNLOCK_WITHOUT_LOCK;
}
+ // TODO: record owner and lock count.
LIBC_INLINE MutexError try_lock() {
if (this->internal::RawMutex::try_lock())
return MutexError::NONE;
diff --git a/libc/src/__support/threads/linux/raw_mutex.h b/libc/src/__support/threads/linux/raw_mutex.h
index cc598e8db1d75..704aa6350298b 100644
--- a/libc/src/__support/threads/linux/raw_mutex.h
+++ b/libc/src/__support/threads/linux/raw_mutex.h
@@ -75,7 +75,7 @@ class RawMutex {
futex.exchange(IN_CONTENTION, cpp::MemoryOrder::ACQUIRE) == UNLOCKED)
return true;
// Contention persists. Park the thread and wait for further notification.
- if (-ETIMEDOUT == futex.wait(IN_CONTENTION, timeout, is_pshared))
+ if (ETIMEDOUT == -futex.wait(IN_CONTENTION, timeout, is_pshared))
return false;
// Continue to spin after waking up.
state = spin(spin_count);
@@ -100,7 +100,9 @@ class RawMutex {
bool is_shared = false,
uint_fast32_t spin_count = LIBC_COPT_RAW_MUTEX_DEFAULT_SPIN_COUNT) {
// Timeout will not be checked if immediate lock is possible.
- return try_lock() || lock_slow(timeout, is_shared, spin_count);
+ if (try_lock())
+ return true;
+ return lock_slow(timeout, is_shared, spin_count);
}
LIBC_INLINE bool unlock(bool is_pshared = false) {
FutexWordType prev = futex.exchange(UNLOCKED, cpp::MemoryOrder::RELEASE);
diff --git a/libc/test/src/__support/threads/linux/raw_mutex_test.cpp b/libc/test/src/__support/threads/linux/raw_mutex_test.cpp
index 94ecf2e4e06ff..a6991f8366c8e 100644
--- a/libc/test/src/__support/threads/linux/raw_mutex_test.cpp
+++ b/libc/test/src/__support/threads/linux/raw_mutex_test.cpp
@@ -1,3 +1,11 @@
+//===-- Unittests for Linux's RawMutex ------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
#include "include/llvm-libc-macros/linux/time-macros.h"
#include "src/__support/CPP/atomic.h"
#include "src/__support/OSUtil/syscall.h"
>From ab5ba24232fe6d0e2d4cdf466a1fbe5285b21b23 Mon Sep 17 00:00:00 2001
From: Schrodinger ZHU Yifan <yifanzhu at rochester.edu>
Date: Thu, 23 May 2024 23:42:54 -0400
Subject: [PATCH 12/15] fix condvar after merge
---
libc/src/__support/threads/CndVar.h | 7 ++++---
libc/src/__support/threads/linux/CMakeLists.txt | 1 +
libc/src/__support/threads/linux/CndVar.cpp | 5 +++--
libc/src/__support/threads/linux/raw_mutex.h | 2 ++
4 files changed, 10 insertions(+), 5 deletions(-)
diff --git a/libc/src/__support/threads/CndVar.h b/libc/src/__support/threads/CndVar.h
index baa2a686c57d4..3ba22212ea744 100644
--- a/libc/src/__support/threads/CndVar.h
+++ b/libc/src/__support/threads/CndVar.h
@@ -10,6 +10,7 @@
#define LLVM_LIBC___SUPPORT_SRC_THREADS_LINUX_CNDVAR_H
#include "src/__support/threads/linux/futex_utils.h" // Futex
+#include "src/__support/threads/linux/raw_mutex.h" // RawMutex
#include "src/__support/threads/mutex.h" // Mutex
#include <stdint.h> // uint32_t
@@ -29,12 +30,12 @@ struct CndVar {
CndWaiter *waitq_front;
CndWaiter *waitq_back;
- Mutex qmtx;
+ internal::RawMutex qmtx;
static int init(CndVar *cv) {
cv->waitq_front = cv->waitq_back = nullptr;
- auto err = Mutex::init(&cv->qmtx, false, false, false);
- return err == MutexError::NONE ? 0 : -1;
+ internal::RawMutex::init(&cv->qmtx);
+ return 0;
}
static void destroy(CndVar *cv) {
diff --git a/libc/src/__support/threads/linux/CMakeLists.txt b/libc/src/__support/threads/linux/CMakeLists.txt
index b57592473af39..ef0421edc2516 100644
--- a/libc/src/__support/threads/linux/CMakeLists.txt
+++ b/libc/src/__support/threads/linux/CMakeLists.txt
@@ -98,5 +98,6 @@ add_object_library(
libc.src.__support.OSUtil.osutil
libc.src.__support.threads.linux.futex_word_type
libc.src.__support.threads.mutex
+ libc.src.__support.threads.linux.raw_mutex
libc.src.__support.CPP.mutex
)
diff --git a/libc/src/__support/threads/linux/CndVar.cpp b/libc/src/__support/threads/linux/CndVar.cpp
index b3a0fdbda4e9e..6a8ef09e45ae1 100644
--- a/libc/src/__support/threads/linux/CndVar.cpp
+++ b/libc/src/__support/threads/linux/CndVar.cpp
@@ -10,6 +10,7 @@
#include "src/__support/CPP/mutex.h"
#include "src/__support/OSUtil/syscall.h" // syscall_impl
#include "src/__support/threads/linux/futex_word.h" // FutexWordType
+#include "src/__support/threads/linux/raw_mutex.h" // RawMutex
#include "src/__support/threads/mutex.h" // Mutex
#include <sys/syscall.h> // For syscall numbers.
@@ -74,11 +75,11 @@ void CndVar::notify_one() {
if (waitq_front == nullptr)
waitq_back = nullptr;
- qmtx.futex_word = FutexWordType(Mutex::LockState::Free);
+ qmtx.futex = internal::RawMutex::UNLOCKED;
// this is a special WAKE_OP, so we use syscall directly
LIBC_NAMESPACE::syscall_impl<long>(
- FUTEX_SYSCALL_ID, &qmtx.futex_word.val, FUTEX_WAKE_OP, 1, 1,
+ FUTEX_SYSCALL_ID, &qmtx.futex.val, FUTEX_WAKE_OP, 1, 1,
&first->futex_word.val,
FUTEX_OP(FUTEX_OP_SET, WS_Signalled, FUTEX_OP_CMP_EQ, WS_Waiting));
}
diff --git a/libc/src/__support/threads/linux/raw_mutex.h b/libc/src/__support/threads/linux/raw_mutex.h
index 704aa6350298b..075ef946678d0 100644
--- a/libc/src/__support/threads/linux/raw_mutex.h
+++ b/libc/src/__support/threads/linux/raw_mutex.h
@@ -24,6 +24,7 @@
#endif
namespace LIBC_NAMESPACE {
+struct CndVar;
namespace internal {
// Lock is a simple timable lock for internal usage.
// This is separated from Mutex because this one does not need to consider
@@ -112,6 +113,7 @@ class RawMutex {
// Detect invalid unlock operation.
return prev != UNLOCKED;
}
+ friend struct ::LIBC_NAMESPACE::CndVar;
};
} // namespace internal
} // namespace LIBC_NAMESPACE
>From 0580610c9db71492a88980d70ef995aba0691855 Mon Sep 17 00:00:00 2001
From: Schrodinger ZHU Yifan <yifanzhu at rochester.edu>
Date: Fri, 24 May 2024 00:04:50 -0400
Subject: [PATCH 13/15] fix header
---
libc/include/llvm-libc-types/CMakeLists.txt | 2 +-
libc/include/llvm-libc-types/cnd_t.h | 4 ++--
2 files changed, 3 insertions(+), 3 deletions(-)
diff --git a/libc/include/llvm-libc-types/CMakeLists.txt b/libc/include/llvm-libc-types/CMakeLists.txt
index 8ac7be391fb2e..ee2c910b85b00 100644
--- a/libc/include/llvm-libc-types/CMakeLists.txt
+++ b/libc/include/llvm-libc-types/CMakeLists.txt
@@ -21,7 +21,7 @@ add_header(blksize_t HDR blksize_t.h)
add_header(cc_t HDR cc_t.h)
add_header(clock_t HDR clock_t.h)
add_header(clockid_t HDR clockid_t.h)
-add_header(cnd_t HDR cnd_t.h)
+add_header(cnd_t HDR cnd_t.h DEPENDS .__futex_word)
add_header(cookie_io_functions_t HDR cookie_io_functions_t.h DEPENDS .off64_t .ssize_t)
add_header(cpu_set_t HDR cpu_set_t.h)
add_header(double_t HDR double_t.h)
diff --git a/libc/include/llvm-libc-types/cnd_t.h b/libc/include/llvm-libc-types/cnd_t.h
index 1159ac4357921..266dfbb588926 100644
--- a/libc/include/llvm-libc-types/cnd_t.h
+++ b/libc/include/llvm-libc-types/cnd_t.h
@@ -9,12 +9,12 @@
#ifndef LLVM_LIBC_TYPES_CND_T_H
#define LLVM_LIBC_TYPES_CND_T_H
-#include "mtx_t.h"
+#include "llvm-libc-types/__futex_word.h"
typedef struct {
void *__qfront;
void *__qback;
- mtx_t __qmtx;
+ __futex_word __qmtx;
} cnd_t;
#endif // LLVM_LIBC_TYPES_CND_T_H
>From eab3caa961d1d9315b4b4fe33051b77b53458e44 Mon Sep 17 00:00:00 2001
From: Yifan Zhu <yifzhu at nvidia.com>
Date: Fri, 24 May 2024 16:50:29 -0700
Subject: [PATCH 14/15] address code reviews
---
libc/config/config.json | 6 +--
libc/docs/configure.rst | 4 +-
libc/src/__support/File/dir.h | 16 ++++---
libc/src/__support/File/file.h | 9 ++--
libc/src/__support/threads/CndVar.h | 11 ++---
libc/src/__support/threads/fork_callbacks.cpp | 4 +-
.../__support/threads/linux/CMakeLists.txt | 4 +-
libc/src/__support/threads/linux/CndVar.cpp | 2 +-
libc/src/__support/threads/linux/mutex.h | 27 +++++++-----
libc/src/__support/threads/linux/raw_mutex.h | 44 +++++++++++--------
libc/src/__support/threads/thread.cpp | 8 +++-
libc/src/pthread/pthread_mutex_init.cpp | 2 +-
libc/src/stdlib/atexit.cpp | 3 +-
.../__support/threads/thread_detach_test.cpp | 3 +-
.../threads/linux/raw_mutex_test.cpp | 15 ++++---
15 files changed, 95 insertions(+), 63 deletions(-)
diff --git a/libc/config/config.json b/libc/config/config.json
index ec6cd2f9769b7..974354a4d1ab5 100644
--- a/libc/config/config.json
+++ b/libc/config/config.json
@@ -44,11 +44,11 @@
"pthread": {
"LIBC_CONF_TIMEOUT_ENSURE_MONOTONICITY": {
"value": true,
- "doc": "Automatically adjust timeout to CLOCK_MONOTONIC."
+ "doc": "Automatically adjust timeout to CLOCK_MONOTONIC (default to true). POSIX API may require CLOCK_REALTIME, which can be unstable and leading to unexpected behavior. This option will convert the real-time timestamp to monotonic timestamp relative to the time of call."
},
"LIBC_CONF_RAW_MUTEX_DEFAULT_SPIN_COUNT": {
"value": 100,
- "doc": "Default number of spins before blocking if a mutex is in contention."
+ "doc": "Default number of spins before blocking if a mutex is in contention (default to 100)."
}
}
-}
+}
\ No newline at end of file
diff --git a/libc/docs/configure.rst b/libc/docs/configure.rst
index 64657f9eabadf..77ade07714fdf 100644
--- a/libc/docs/configure.rst
+++ b/libc/docs/configure.rst
@@ -35,8 +35,8 @@ to learn about the defaults for your platform and target.
- ``LIBC_CONF_PRINTF_DISABLE_WRITE_INT``: Disable handling of %n in printf format string.
- ``LIBC_CONF_PRINTF_FLOAT_TO_STR_USE_MEGA_LONG_DOUBLE_TABLE``: Use large table for better printf long double performance.
* **"pthread" options**
- - ``LIBC_CONF_RAW_MUTEX_DEFAULT_SPIN_COUNT``: Default number of spins before blocking if a mutex is in contention.
- - ``LIBC_CONF_TIMEOUT_ENSURE_MONOTONICITY``: Automatically adjust timeout to CLOCK_MONOTONIC.
+ - ``LIBC_CONF_RAW_MUTEX_DEFAULT_SPIN_COUNT``: Default number of spins before blocking if a mutex is in contention (default to 100).
+ - ``LIBC_CONF_TIMEOUT_ENSURE_MONOTONICITY``: Automatically adjust timeout to CLOCK_MONOTONIC (default to true). POSIX API may require CLOCK_REALTIME, which can be unstable and leading to unexpected behavior. This option will convert the real-time timestamp to monotonic timestamp relative to the time of call.
* **"string" options**
- ``LIBC_CONF_MEMSET_X86_USE_SOFTWARE_PREFETCHING``: Inserts prefetch for write instructions (PREFETCHW) for memset on x86 to recover performance when hardware prefetcher is disabled.
- ``LIBC_CONF_STRING_UNSAFE_WIDE_READ``: Read more than a byte at a time to perform byte-string operations like strlen.
diff --git a/libc/src/__support/File/dir.h b/libc/src/__support/File/dir.h
index ab445b6c0bec5..9f8c8dbb0c36e 100644
--- a/libc/src/__support/File/dir.h
+++ b/libc/src/__support/File/dir.h
@@ -50,14 +50,16 @@ class Dir {
// A directory is to be opened by the static method open and closed
// by the close method. So, all constructors and destructor are declared
// as private. Inappropriate constructors are declared as deleted.
- Dir() = delete;
- Dir(const Dir &) = delete;
+ LIBC_INLINE Dir() = delete;
+ LIBC_INLINE Dir(const Dir &) = delete;
- explicit Dir(int fdesc)
- : fd(fdesc), readptr(0), fillsize(0), mutex(false, false, false, false) {}
- ~Dir() = default;
+ LIBC_INLINE explicit Dir(int fdesc)
+ : fd(fdesc), readptr(0), fillsize(0),
+ mutex(/*timed=*/false, /*recursive=*/false, /*robust=*/false,
+ /*pshared=*/false) {}
+ LIBC_INLINE ~Dir() = default;
- Dir &operator=(const Dir &) = delete;
+ LIBC_INLINE Dir &operator=(const Dir &) = delete;
public:
static ErrorOr<Dir *> open(const char *path);
@@ -69,7 +71,7 @@ class Dir {
// cleaned up.
int close();
- int getfd() { return fd; }
+ LIBC_INLINE int getfd() { return fd; }
};
} // namespace LIBC_NAMESPACE
diff --git a/libc/src/__support/File/file.h b/libc/src/__support/File/file.h
index 6b41056cf40bd..06154871485ce 100644
--- a/libc/src/__support/File/file.h
+++ b/libc/src/__support/File/file.h
@@ -154,10 +154,11 @@ class File {
uint8_t *buffer, size_t buffer_size, int buffer_mode,
bool owned, ModeFlags modeflags)
: platform_write(wf), platform_read(rf), platform_seek(sf),
- platform_close(cf), mutex(false, false, false, false), ungetc_buf(0),
- buf(buffer), bufsize(buffer_size), bufmode(buffer_mode), own_buf(owned),
- mode(modeflags), pos(0), prev_op(FileOp::NONE), read_limit(0),
- eof(false), err(false) {
+ platform_close(cf), mutex(/*timed=*/false, /*recursive=*/false,
+ /*robust=*/false, /*pshared=*/false),
+ ungetc_buf(0), buf(buffer), bufsize(buffer_size), bufmode(buffer_mode),
+ own_buf(owned), mode(modeflags), pos(0), prev_op(FileOp::NONE),
+ read_limit(0), eof(false), err(false) {
adjust_buf();
}
diff --git a/libc/src/__support/threads/CndVar.h b/libc/src/__support/threads/CndVar.h
index 3ba22212ea744..55f6e02b354cb 100644
--- a/libc/src/__support/threads/CndVar.h
+++ b/libc/src/__support/threads/CndVar.h
@@ -17,7 +17,7 @@
namespace LIBC_NAMESPACE {
-struct CndVar {
+class CndVar {
enum CndWaiterStatus : uint32_t {
WS_Waiting = 0xE,
WS_Signalled = 0x5,
@@ -30,15 +30,16 @@ struct CndVar {
CndWaiter *waitq_front;
CndWaiter *waitq_back;
- internal::RawMutex qmtx;
+ RawMutex qmtx;
- static int init(CndVar *cv) {
+public:
+ LIBC_INLINE static int init(CndVar *cv) {
cv->waitq_front = cv->waitq_back = nullptr;
- internal::RawMutex::init(&cv->qmtx);
+ RawMutex::init(&cv->qmtx);
return 0;
}
- static void destroy(CndVar *cv) {
+ LIBC_INLINE static void destroy(CndVar *cv) {
cv->waitq_front = cv->waitq_back = nullptr;
}
diff --git a/libc/src/__support/threads/fork_callbacks.cpp b/libc/src/__support/threads/fork_callbacks.cpp
index 9ad3f38892f8f..385a2bcb58168 100644
--- a/libc/src/__support/threads/fork_callbacks.cpp
+++ b/libc/src/__support/threads/fork_callbacks.cpp
@@ -34,7 +34,9 @@ class AtForkCallbackManager {
public:
constexpr AtForkCallbackManager()
- : mtx(false, false, false, false), next_index(0) {}
+ : mtx(/*timed=*/false, /*recursive=*/false, /*robust=*/false,
+ /*pshared=*/false),
+ next_index(0) {}
bool register_triple(const ForkCallbackTriple &triple) {
cpp::lock_guard lock(mtx);
diff --git a/libc/src/__support/threads/linux/CMakeLists.txt b/libc/src/__support/threads/linux/CMakeLists.txt
index ef0421edc2516..9bf88ccc84557 100644
--- a/libc/src/__support/threads/linux/CMakeLists.txt
+++ b/libc/src/__support/threads/linux/CMakeLists.txt
@@ -24,7 +24,9 @@ add_header_library(
set(raw_mutex_additional_flags)
if (LIBC_CONF_TIMEOUT_ENSURE_MONOTONICITY)
- set(raw_mutex_additional_flags -DLIBC_COPT_TIMEOUT_ENSURE_MONOTONICITY)
+ set(raw_mutex_additional_flags -DLIBC_COPT_TIMEOUT_ENSURE_MONOTONICITY=1)
+else()
+ set(raw_mutex_additional_flags -DLIBC_COPT_TIMEOUT_ENSURE_MONOTONICITY=0)
endif()
add_header_library(
diff --git a/libc/src/__support/threads/linux/CndVar.cpp b/libc/src/__support/threads/linux/CndVar.cpp
index 6a8ef09e45ae1..5a720271c08ff 100644
--- a/libc/src/__support/threads/linux/CndVar.cpp
+++ b/libc/src/__support/threads/linux/CndVar.cpp
@@ -75,7 +75,7 @@ void CndVar::notify_one() {
if (waitq_front == nullptr)
waitq_back = nullptr;
- qmtx.futex = internal::RawMutex::UNLOCKED;
+ qmtx.futex = RawMutex::UNLOCKED;
// this is a special WAKE_OP, so we use syscall directly
LIBC_NAMESPACE::syscall_impl<long>(
diff --git a/libc/src/__support/threads/linux/mutex.h b/libc/src/__support/threads/linux/mutex.h
index 292c61fb2d789..79506d7e6911a 100644
--- a/libc/src/__support/threads/linux/mutex.h
+++ b/libc/src/__support/threads/linux/mutex.h
@@ -11,6 +11,7 @@
#include "hdr/types/pid_t.h"
#include "src/__support/CPP/optional.h"
+#include "src/__support/libc_assert.h"
#include "src/__support/threads/linux/futex_utils.h"
#include "src/__support/threads/linux/raw_mutex.h"
#include "src/__support/threads/mutex_common.h"
@@ -18,7 +19,7 @@
namespace LIBC_NAMESPACE {
// TODO: support shared/recursive/robust mutexes.
-class Mutex final : private internal::RawMutex {
+class Mutex final : private RawMutex {
// reserved timed, may be useful when combined with other flags.
unsigned char timed;
unsigned char recursive;
@@ -32,12 +33,12 @@ class Mutex final : private internal::RawMutex {
public:
LIBC_INLINE constexpr Mutex(bool is_timed, bool is_recursive, bool is_robust,
bool is_pshared)
- : internal::RawMutex(), timed(is_timed), recursive(is_recursive),
- robust(is_robust), pshared(is_pshared), owner(0), lock_count(0) {}
+ : RawMutex(), timed(is_timed), recursive(is_recursive), robust(is_robust),
+ pshared(is_pshared), owner(0), lock_count(0) {}
LIBC_INLINE static MutexError init(Mutex *mutex, bool is_timed, bool isrecur,
bool isrobust, bool is_pshared) {
- internal::RawMutex::init(mutex);
+ RawMutex::init(mutex);
mutex->timed = is_timed;
mutex->recursive = isrecur;
mutex->robust = isrobust;
@@ -47,31 +48,37 @@ class Mutex final : private internal::RawMutex {
return MutexError::NONE;
}
- LIBC_INLINE static MutexError destroy(Mutex *) { return MutexError::NONE; }
+ LIBC_INLINE static MutexError destroy(Mutex *lock) {
+ LIBC_ASSERT(lock->owner == 0 && lock->lock_count == 0 &&
+ "Mutex destroyed while being locked.");
+ RawMutex::destroy(lock);
+ return MutexError::NONE;
+ }
// TODO: record owner and lock count.
LIBC_INLINE MutexError lock() {
- this->internal::RawMutex::lock(
- /* timeout */ cpp::nullopt, this->pshared);
+ // Since timeout is not specified, we do not need to check the return value.
+ this->RawMutex::lock(
+ /* timeout=*/cpp::nullopt, this->pshared);
return MutexError::NONE;
}
// TODO: record owner and lock count.
LIBC_INLINE MutexError timed_lock(internal::AbsTimeout abs_time) {
- if (this->internal::RawMutex::lock(abs_time, this->pshared))
+ if (this->RawMutex::lock(abs_time, this->pshared))
return MutexError::NONE;
return MutexError::TIMEOUT;
}
LIBC_INLINE MutexError unlock() {
- if (this->internal::RawMutex::unlock(this->pshared))
+ if (this->RawMutex::unlock(this->pshared))
return MutexError::NONE;
return MutexError::UNLOCK_WITHOUT_LOCK;
}
// TODO: record owner and lock count.
LIBC_INLINE MutexError try_lock() {
- if (this->internal::RawMutex::try_lock())
+ if (this->RawMutex::try_lock())
return MutexError::NONE;
return MutexError::BUSY;
}
diff --git a/libc/src/__support/threads/linux/raw_mutex.h b/libc/src/__support/threads/linux/raw_mutex.h
index 075ef946678d0..a3059d1b4fecf 100644
--- a/libc/src/__support/threads/linux/raw_mutex.h
+++ b/libc/src/__support/threads/linux/raw_mutex.h
@@ -8,24 +8,31 @@
#ifndef LLVM_LIBC_SRC___SUPPORT_THREADS_LINUX_RAW_MUTEX_H
#define LLVM_LIBC_SRC___SUPPORT_THREADS_LINUX_RAW_MUTEX_H
-#include "futex_word.h"
#include "src/__support/CPP/optional.h"
#include "src/__support/common.h"
+#include "src/__support/libc_assert.h"
#include "src/__support/macros/attributes.h"
#include "src/__support/threads/linux/futex_utils.h"
+#include "src/__support/threads/linux/futex_word.h"
#include "src/__support/threads/sleep.h"
#include "src/__support/time/linux/abs_timeout.h"
-#ifdef LIBC_COPT_TIMEOUT_ENSURE_MONOTONICITY
+
+#ifndef LIBC_COPT_TIMEOUT_ENSURE_MONOTONICITY
+#define LIBC_COPT_TIMEOUT_ENSURE_MONOTONICITY 1
+#warning "LIBC_COPT_TIMEOUT_ENSURE_MONOTONICITY is not defined, defaulting to 1"
+#endif
+
+#if LIBC_COPT_TIMEOUT_ENSURE_MONOTONICITY
#include "src/__support/time/linux/monotonicity.h"
#endif
#ifndef LIBC_COPT_RAW_MUTEX_DEFAULT_SPIN_COUNT
#define LIBC_COPT_RAW_MUTEX_DEFAULT_SPIN_COUNT 100
+#warning \
+ "LIBC_COPT_RAW_MUTEX_DEFAULT_SPIN_COUNT is not defined, defaulting to 100"
#endif
namespace LIBC_NAMESPACE {
-struct CndVar;
-namespace internal {
// Lock is a simple timable lock for internal usage.
// This is separated from Mutex because this one does not need to consider
// robustness and reentrancy. Also, this one has spin optimization for shorter
@@ -37,7 +44,8 @@ class RawMutex {
LIBC_INLINE_VAR static constexpr FutexWordType LOCKED = 0b01;
LIBC_INLINE_VAR static constexpr FutexWordType IN_CONTENTION = 0b10;
- LIBC_INLINE FutexWordType spin(uint_fast32_t spin_count) {
+private:
+ LIBC_INLINE FutexWordType spin(int spin_count) {
FutexWordType result;
for (;;) {
result = futex.load(cpp::MemoryOrder::RELAXED);
@@ -56,16 +64,16 @@ class RawMutex {
// Return true if the lock is acquired. Return false if timeout happens before
// the lock is acquired.
- [[gnu::cold]] LIBC_INLINE bool
- lock_slow(cpp::optional<Futex::Timeout> timeout, bool is_pshared,
- uint_fast32_t spin_count) {
+ LIBC_INLINE bool lock_slow(cpp::optional<Futex::Timeout> timeout,
+ bool is_pshared, int spin_count) {
FutexWordType state = spin(spin_count);
// Before go into contention state, try to grab the lock.
if (state == UNLOCKED &&
futex.compare_exchange_strong(state, LOCKED, cpp::MemoryOrder::ACQUIRE,
cpp::MemoryOrder::RELAXED))
return true;
-#ifdef LIBC_COPT_TIMEOUT_ENSURE_MONOTONICITY
+#if LIBC_COPT_TIMEOUT_ENSURE_MONOTONICITY
+ /* ADL should kick in */
if (timeout)
ensure_monotonicity(*timeout);
#endif
@@ -83,14 +91,12 @@ class RawMutex {
}
}
- [[gnu::cold]] LIBC_INLINE void wake(bool is_pshared) {
- futex.notify_one(is_pshared);
- }
+ LIBC_INLINE void wake(bool is_pshared) { futex.notify_one(is_pshared); }
public:
LIBC_INLINE static void init(RawMutex *mutex) { mutex->futex = UNLOCKED; }
LIBC_INLINE constexpr RawMutex() : futex(UNLOCKED) {}
- LIBC_INLINE bool try_lock() {
+ [[nodiscard]] LIBC_INLINE bool try_lock() {
FutexWordType expected = UNLOCKED;
// Use strong version since this is a one-time operation.
return futex.compare_exchange_strong(
@@ -99,23 +105,25 @@ class RawMutex {
LIBC_INLINE bool
lock(cpp::optional<Futex::Timeout> timeout = cpp::nullopt,
bool is_shared = false,
- uint_fast32_t spin_count = LIBC_COPT_RAW_MUTEX_DEFAULT_SPIN_COUNT) {
+ int spin_count = LIBC_COPT_RAW_MUTEX_DEFAULT_SPIN_COUNT) {
// Timeout will not be checked if immediate lock is possible.
- if (try_lock())
+ if (LIBC_LIKELY(try_lock()))
return true;
return lock_slow(timeout, is_shared, spin_count);
}
LIBC_INLINE bool unlock(bool is_pshared = false) {
FutexWordType prev = futex.exchange(UNLOCKED, cpp::MemoryOrder::RELEASE);
// if there is someone waiting, wake them up
- if (prev == IN_CONTENTION)
+ if (LIBC_UNLIKELY(prev == IN_CONTENTION))
wake(is_pshared);
// Detect invalid unlock operation.
return prev != UNLOCKED;
}
- friend struct ::LIBC_NAMESPACE::CndVar;
+ LIBC_INLINE void static destroy([[maybe_unused]] RawMutex *lock) {
+ LIBC_ASSERT(lock->futex == UNLOCKED && "Mutex destroyed while used.");
+ }
+ friend class CndVar;
};
-} // namespace internal
} // namespace LIBC_NAMESPACE
#endif // LLVM_LIBC_SRC___SUPPORT_THREADS_LINUX_RAW_MUTEX_H
diff --git a/libc/src/__support/threads/thread.cpp b/libc/src/__support/threads/thread.cpp
index 2bea01bf332b3..d7bedb8ca67d0 100644
--- a/libc/src/__support/threads/thread.cpp
+++ b/libc/src/__support/threads/thread.cpp
@@ -54,7 +54,9 @@ class TSSKeyMgr {
cpp::array<TSSKeyUnit, TSS_KEY_COUNT> units;
public:
- constexpr TSSKeyMgr() : mtx(false, false, false, false) {}
+ constexpr TSSKeyMgr()
+ : mtx(/*timed=*/false, /*recursive=*/false, /*robust=*/false,
+ /*pshared=*/false) {}
cpp::optional<unsigned int> new_key(TSSDtor *dtor) {
cpp::lock_guard lock(mtx);
@@ -111,7 +113,9 @@ class ThreadAtExitCallbackMgr {
FixedVector<AtExitUnit, 1024> callback_list;
public:
- constexpr ThreadAtExitCallbackMgr() : mtx(false, false, false, false) {}
+ constexpr ThreadAtExitCallbackMgr()
+ : mtx(/*timed=*/false, /*recursive=*/false, /*robust=*/false,
+ /*pshared=*/false) {}
int add_callback(AtExitCallback *callback, void *obj) {
cpp::lock_guard lock(mtx);
diff --git a/libc/src/pthread/pthread_mutex_init.cpp b/libc/src/pthread/pthread_mutex_init.cpp
index 35615c449c044..3914669b1c73e 100644
--- a/libc/src/pthread/pthread_mutex_init.cpp
+++ b/libc/src/pthread/pthread_mutex_init.cpp
@@ -26,7 +26,7 @@ LLVM_LIBC_FUNCTION(int, pthread_mutex_init,
const pthread_mutexattr_t *__restrict attr)) {
auto mutexattr = attr == nullptr ? DEFAULT_MUTEXATTR : *attr;
auto err =
- Mutex::init(reinterpret_cast<Mutex *>(m), /* timeout is supported */ true,
+ Mutex::init(reinterpret_cast<Mutex *>(m), /*is_timed=*/true,
get_mutexattr_type(mutexattr) & PTHREAD_MUTEX_RECURSIVE,
get_mutexattr_robust(mutexattr) & PTHREAD_MUTEX_ROBUST,
get_mutexattr_pshared(mutexattr) & PTHREAD_PROCESS_SHARED);
diff --git a/libc/src/stdlib/atexit.cpp b/libc/src/stdlib/atexit.cpp
index fd24811000123..9e37c4cf256c9 100644
--- a/libc/src/stdlib/atexit.cpp
+++ b/libc/src/stdlib/atexit.cpp
@@ -17,7 +17,8 @@ namespace LIBC_NAMESPACE {
namespace {
-Mutex handler_list_mtx(false, false, false, false);
+Mutex handler_list_mtx(/*timed=*/false, /*recursive=*/false, /*robust=*/false,
+ /*pshared=*/false);
using AtExitCallback = void(void *);
using StdCAtExitCallback = void(void);
diff --git a/libc/test/integration/src/__support/threads/thread_detach_test.cpp b/libc/test/integration/src/__support/threads/thread_detach_test.cpp
index b11e704551aab..995f8b0050b58 100644
--- a/libc/test/integration/src/__support/threads/thread_detach_test.cpp
+++ b/libc/test/integration/src/__support/threads/thread_detach_test.cpp
@@ -10,7 +10,8 @@
#include "src/__support/threads/thread.h"
#include "test/IntegrationTest/test.h"
-LIBC_NAMESPACE::Mutex mutex(false, false, false, false);
+LIBC_NAMESPACE::Mutex mutex(/*timed=*/false, /*recursive=*/false,
+ /*robust=*/false, /*pshared=*/false);
int func(void *) {
mutex.lock();
diff --git a/libc/test/src/__support/threads/linux/raw_mutex_test.cpp b/libc/test/src/__support/threads/linux/raw_mutex_test.cpp
index a6991f8366c8e..814f6f14c87b9 100644
--- a/libc/test/src/__support/threads/linux/raw_mutex_test.cpp
+++ b/libc/test/src/__support/threads/linux/raw_mutex_test.cpp
@@ -19,7 +19,7 @@
#include <sys/syscall.h>
TEST(LlvmLibcSupportThreadsRawMutexTest, SmokeTest) {
- LIBC_NAMESPACE::internal::RawMutex mutex;
+ LIBC_NAMESPACE::RawMutex mutex;
ASSERT_TRUE(mutex.lock());
ASSERT_TRUE(mutex.unlock());
ASSERT_TRUE(mutex.try_lock());
@@ -29,7 +29,7 @@ TEST(LlvmLibcSupportThreadsRawMutexTest, SmokeTest) {
}
TEST(LlvmLibcSupportThreadsRawMutexTest, Timeout) {
- LIBC_NAMESPACE::internal::RawMutex mutex;
+ LIBC_NAMESPACE::RawMutex mutex;
ASSERT_TRUE(mutex.lock());
timespec ts;
LIBC_NAMESPACE::internal::clock_gettime(CLOCK_MONOTONIC, &ts);
@@ -37,18 +37,21 @@ TEST(LlvmLibcSupportThreadsRawMutexTest, Timeout) {
// Timeout will be respected when deadlock happens.
auto timeout = LIBC_NAMESPACE::internal::AbsTimeout::from_timespec(ts, false);
ASSERT_TRUE(timeout.has_value());
+ // The following will timeout
ASSERT_FALSE(mutex.lock(*timeout));
ASSERT_TRUE(mutex.unlock());
- ASSERT_TRUE(mutex.lock(*timeout));
+ // Test that the mutex works after the timeout.
+ ASSERT_TRUE(mutex.lock());
ASSERT_TRUE(mutex.unlock());
// If a lock can be acquired directly, expired timeout will not count.
+ // Notice that the timeout is already reached during preivous deadlock.
ASSERT_TRUE(mutex.lock(*timeout));
ASSERT_TRUE(mutex.unlock());
}
TEST(LlvmLibcSupportThreadsRawMutexTest, PSharedLock) {
struct SharedData {
- LIBC_NAMESPACE::internal::RawMutex mutex;
+ LIBC_NAMESPACE::RawMutex mutex;
LIBC_NAMESPACE::cpp::Atomic<size_t> finished;
int data;
};
@@ -58,7 +61,7 @@ TEST(LlvmLibcSupportThreadsRawMutexTest, PSharedLock) {
ASSERT_NE(addr, MAP_FAILED);
auto *shared = reinterpret_cast<SharedData *>(addr);
shared->data = 0;
- LIBC_NAMESPACE::internal::RawMutex::init(&shared->mutex);
+ LIBC_NAMESPACE::RawMutex::init(&shared->mutex);
// Avoid pull in our own implementation of pthread_t.
long pid = LIBC_NAMESPACE::syscall_impl<long>(SYS_fork);
for (int i = 0; i < 10000; ++i) {
@@ -68,7 +71,7 @@ TEST(LlvmLibcSupportThreadsRawMutexTest, PSharedLock) {
}
// Mark the thread as finished.
shared->finished.fetch_add(1);
- // early exit to avoid output pollution
+ // let the child exit early to avoid output pollution
if (pid == 0)
LIBC_NAMESPACE::exit(0);
while (shared->finished.load() != 2)
>From b9abf31d2e027bfa09632f058fc62563b7f8ec3b Mon Sep 17 00:00:00 2001
From: Yifan Zhu <yifzhu at nvidia.com>
Date: Mon, 27 May 2024 09:26:58 -0700
Subject: [PATCH 15/15] fix formatter stripped trailing lines
---
libc/config/config.json | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/libc/config/config.json b/libc/config/config.json
index 974354a4d1ab5..d3d1ff1e28716 100644
--- a/libc/config/config.json
+++ b/libc/config/config.json
@@ -51,4 +51,4 @@
"doc": "Default number of spins before blocking if a mutex is in contention (default to 100)."
}
}
-}
\ No newline at end of file
+}
More information about the libc-commits
mailing list