[libc-commits] [libc] [llvm] [libc][CndVar] reimplmement conditional variable with FIFO ordering (PR #192748)

[Michael Jones via libc-commits]

================
@@ -6,49 +6,406 @@
 //
 //===----------------------------------------------------------------------===//
 
-#ifndef LLVM_LIBC___SUPPORT_SRC_THREADS_LINUX_CNDVAR_H
-#define LLVM_LIBC___SUPPORT_SRC_THREADS_LINUX_CNDVAR_H
+#ifndef LLVM_LIBC_SRC___SUPPORT_THREADS_CNDVAR_H
+#define LLVM_LIBC_SRC___SUPPORT_THREADS_CNDVAR_H
 
 #include "hdr/stdint_proxy.h" // uint32_t
+#include "src/__support/CPP/mutex.h"
+#include "src/__support/CPP/new.h"
 #include "src/__support/macros/config.h"
-#include "src/__support/threads/futex_utils.h"       // Futex
-#include "src/__support/threads/mutex.h"             // Mutex
-#include "src/__support/threads/raw_mutex.h"         // RawMutex
+#include "src/__support/threads/futex_utils.h" // Futex
+#include "src/__support/threads/mutex.h"       // Mutex
+#include "src/__support/threads/raw_mutex.h"   // RawMutex
+
+#ifndef LIBC_COPT_TIMEOUT_ENSURE_MONOTONICITY
+#define LIBC_COPT_TIMEOUT_ENSURE_MONOTONICITY 1
+#endif
 
 namespace LIBC_NAMESPACE_DECL {
 
-class CndVar {
-  enum CndWaiterStatus : uint32_t {
-    WS_Waiting = 0xE,
-    WS_Signalled = 0x5,
+enum class CndVarResult {
+  // The waiter successfully received a signal.
+  Success,
+  // Error occurs during mutex acquisition.
+  MutexError,
+  // Timeout occurs.
+  Timeout,
+};
+
+class PrivateCndVar {
+  // A single-waiter multiple-notifier barrier used to keep
+  // track of cancellation threads. We use this barrier to
+  // ensure in-queue threads that have posted their cancellation
+  // request have finished dequeue themselves.
+  class CancellationBarrier {
+    LIBC_INLINE_VAR static constexpr size_t SPIN_LIMIT = 100;
+    LIBC_INLINE_VAR static constexpr size_t CANCEL_STEP = 2;
+    LIBC_INLINE_VAR static constexpr size_t SLEEPING_BIT = 1;
+
+    // LSB indicates whether the waiter is in sleeping state.
+    Futex futex;
+
+  public:
+    LIBC_INLINE CancellationBarrier() : futex(0) {}
+    // Add one more notification request.
+    LIBC_INLINE void add_one() {
+      futex.fetch_add(CANCEL_STEP, cpp::MemoryOrder::RELAXED);
+    }
+    // Send notification to one waiter.
+    LIBC_INLINE void notify() {
+      FutexWordType res = futex.fetch_sub(CANCEL_STEP);
+      // Only need to goto syscall if waiter is sleep and we are the last one
+      if (res <= (CANCEL_STEP | SLEEPING_BIT) && (res & SLEEPING_BIT) != 0)
+        futex.notify_one();
+    }
+    LIBC_INLINE void wait() {
+      size_t spin = 0;
+      while (auto remaining = futex.load(cpp::MemoryOrder::RELAXED)) {
+        // Set LSB to 1 to indicate that the waiter is entering sleeping
+        // state.
+        FutexWordType new_val = remaining | SLEEPING_BIT;
+        if (spin > SPIN_LIMIT &&
+            futex.compare_exchange_strong(remaining, new_val)) {
+          futex.wait(new_val, /*timeout=*/cpp::nullopt, /*is_pshared=*/false);
+          futex.fetch_sub(1);
+          spin = 0;
+        }
+        sleep_briefly();
+        spin++;
+      }
+    }
+  };
+
+  enum WaiterState : uint8_t {
+    // Initial state after entering the wait queue.
+    Waiting = 0,
+    // A signal has been received.
+    Signalled = 1,
+    // A cancellation has been requested.
+    Cancelled = 2,
+    // The thread has been requeued to the mutex.
+    Requeued = 3,
+  };
+
+  struct WaiterHeader {
+    WaiterHeader *prev;
+    WaiterHeader *next;
+
+    // We use cyclic dummy node to avoid handing corner cases.
+    LIBC_INLINE void ensure_queue_initialization() {
+      if (LIBC_UNLIKELY(prev == nullptr))
+        prev = next = this;
+    }
+
+    // Assume `this` the dummy node of queue. Push back `waiter` to the queue.
+    LIBC_INLINE void push_back(WaiterHeader *waiter) {
+      ensure_queue_initialization();
+      waiter->next = this;
+      waiter->prev = prev;
+      waiter->next->prev = waiter;
+      waiter->prev->next = waiter;
+    }
+
+    // Remove `waiter` from the queue.
+    LIBC_INLINE static void remove(WaiterHeader *waiter) {
+      waiter->next->prev = waiter->prev;
+      waiter->prev->next = waiter->next;
+      waiter->prev = waiter->next = waiter;
+    }
+
+    // Assume `this` the dummy node of queue. Pop the first waiter from the
+    // queue.
+    LIBC_INLINE WaiterHeader *pop_front() {
+      ensure_queue_initialization();
+      if (next == this)
+        return nullptr;
+      WaiterHeader *first = next;
+      remove(first);
+      return first;
+    }
   };
 
-  struct CndWaiter {
-    Futex futex_word = WS_Waiting;
-    CndWaiter *next = nullptr;
+  // This node will be on the per-thread stack.
+  struct CndWaiter : WaiterHeader {
+    cpp::Atomic<CancellationBarrier *> cancellation_barrier;
+    RawMutex barrier;
+    cpp::Atomic<uint8_t> state;
+
+    LIBC_INLINE CndWaiter()
+        : WaiterHeader{}, cancellation_barrier(nullptr), barrier{},
+          state{Waiting} {
+      // this lock should always success as no contention is possible
+      (void)barrier.try_lock();
----------------
michaelrj-google wrote:

I'd recommend adding an assert here to check this.

https://github.com/llvm/llvm-project/pull/192748