[libc-commits] [libc] 6d61d83 - [libc] inline fast path of callonce (#96226)

[via libc-commits]

Author: Schrodinger ZHU Yifan
Date: 2024-06-27T08:48:13-07:00
New Revision: 6d61d83736ca23c336472e7f2bda59b7bb307bba

URL: https://github.com/llvm/llvm-project/commit/6d61d83736ca23c336472e7f2bda59b7bb307bba
DIFF: https://github.com/llvm/llvm-project/commit/6d61d83736ca23c336472e7f2bda59b7bb307bba.diff

LOG: [libc] inline fast path of callonce (#96226)

Split from #91572

---------

Co-authored-by: Nick Desaulniers (paternity leave) <nickdesaulniers at users.noreply.github.com>

Added: 
    libc/src/__support/threads/linux/callonce.h

Modified: 
    libc/src/__support/threads/callonce.h
    libc/src/__support/threads/linux/CMakeLists.txt
    libc/src/__support/threads/linux/callonce.cpp

Removed: 
    


################################################################################
diff  --git a/libc/src/__support/threads/callonce.h b/libc/src/__support/threads/callonce.h
index b3d6813f7dda9..67f4c99657107 100644
--- a/libc/src/__support/threads/callonce.h
+++ b/libc/src/__support/threads/callonce.h
@@ -9,13 +9,32 @@
 #ifndef LLVM_LIBC_SRC___SUPPORT_THREADS_CALLONCE_H
 #define LLVM_LIBC_SRC___SUPPORT_THREADS_CALLONCE_H
 
+#include "src/__support/macros/optimization.h" // LIBC_LIKELY
+
+// Plaform specific routines, provides:
+// - OnceFlag definition
+// - callonce_impl::callonce_fastpath for fast path check
+// - callonce_impl::callonce_slowpath for slow path execution
+#ifdef __linux__
+#include "src/__support/threads/linux/callonce.h"
+#else
+#error "callonce is not supported on this platform"
+#endif
+
 namespace LIBC_NAMESPACE {
 
-struct CallOnceFlag;
+// Common definitions
 using CallOnceCallback = void(void);
+namespace callonce_impl {
+int callonce_slowpath(CallOnceFlag *flag, CallOnceCallback *callback);
+} // namespace callonce_impl
 
-int callonce(CallOnceFlag *flag, CallOnceCallback *callback);
+LIBC_INLINE int callonce(CallOnceFlag *flag, CallOnceCallback *callback) {
+  if (LIBC_LIKELY(callonce_impl::callonce_fastpath(flag)))
+    return 0;
 
+  return callonce_impl::callonce_slowpath(flag, callback);
+}
 } // namespace LIBC_NAMESPACE
 
 #endif // LLVM_LIBC_SRC___SUPPORT_THREADS_CALLONCE_H

diff  --git a/libc/src/__support/threads/linux/CMakeLists.txt b/libc/src/__support/threads/linux/CMakeLists.txt
index 95e509b7a825d..8b7971584e77e 100644
--- a/libc/src/__support/threads/linux/CMakeLists.txt
+++ b/libc/src/__support/threads/linux/CMakeLists.txt
@@ -100,8 +100,10 @@ add_object_library(
     callonce.cpp
   HDRS
     ../callonce.h
+    callonce.h
   DEPENDS
     .futex_utils
+    libc.src.__support.macros.optimization
 )
 
 add_object_library(

diff  --git a/libc/src/__support/threads/linux/callonce.cpp b/libc/src/__support/threads/linux/callonce.cpp
index b48a514a44875..24d376f447fba 100644
--- a/libc/src/__support/threads/linux/callonce.cpp
+++ b/libc/src/__support/threads/linux/callonce.cpp
@@ -7,27 +7,16 @@
 //===----------------------------------------------------------------------===//
 
 #include "src/__support/threads/callonce.h"
-#include "src/__support/macros/optimization.h"
+#include "src/__support/threads/linux/callonce.h"
 #include "src/__support/threads/linux/futex_utils.h"
 
 namespace LIBC_NAMESPACE {
-
-static constexpr FutexWordType NOT_CALLED = 0x0;
-static constexpr FutexWordType START = 0x11;
-static constexpr FutexWordType WAITING = 0x22;
-static constexpr FutexWordType FINISH = 0x33;
-
-int callonce(CallOnceFlag *flag, CallOnceCallback *func) {
+namespace callonce_impl {
+int callonce_slowpath(CallOnceFlag *flag, CallOnceCallback *func) {
   auto *futex_word = reinterpret_cast<Futex *>(flag);
 
   FutexWordType not_called = NOT_CALLED;
 
-  // Avoid cmpxchg operation if the function has already been called.
-  // The destination operand of cmpxchg may receive a write cycle without
-  // regard to the result of the comparison
-  if (LIBC_LIKELY(futex_word->load(cpp::MemoryOrder::RELAXED) == FINISH))
-    return 0;
-
   // The call_once call can return only after the called function |func|
   // returns. So, we use futexes to synchronize calls with the same flag value.
   if (futex_word->compare_exchange_strong(not_called, START)) {
@@ -46,5 +35,5 @@ int callonce(CallOnceFlag *flag, CallOnceCallback *func) {
 
   return 0;
 }
-
+} // namespace callonce_impl
 } // namespace LIBC_NAMESPACE

diff  --git a/libc/src/__support/threads/linux/callonce.h b/libc/src/__support/threads/linux/callonce.h
new file mode 100644
index 0000000000000..315cc6149e9ec
--- /dev/null
+++ b/libc/src/__support/threads/linux/callonce.h
@@ -0,0 +1,31 @@
+//===-- Linux callonce fastpath -------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+#ifndef LLVM_LIBC_SRC___SUPPORT_THREADS_LINUX_CALLONCE_H
+#define LLVM_LIBC_SRC___SUPPORT_THREADS_LINUX_CALLONCE_H
+
+#include "src/__support/threads/linux/futex_utils.h"
+
+namespace LIBC_NAMESPACE {
+using CallOnceFlag = Futex;
+
+namespace callonce_impl {
+static constexpr FutexWordType NOT_CALLED = 0x0;
+static constexpr FutexWordType START = 0x11;
+static constexpr FutexWordType WAITING = 0x22;
+static constexpr FutexWordType FINISH = 0x33;
+
+// Avoid cmpxchg operation if the function has already been called.
+// The destination operand of cmpxchg may receive a write cycle without
+// regard to the result of the comparison.
+LIBC_INLINE bool callonce_fastpath(CallOnceFlag *flag) {
+  return flag->load(cpp::MemoryOrder::RELAXED) == FINISH;
+}
+} // namespace callonce_impl
+
+} // namespace LIBC_NAMESPACE
+#endif // LLVM_LIBC_SRC___SUPPORT_THREADS_LINUX_CALLONCE_H