[libc-commits] [libc] [libc] inline fast path of callonce (PR #96226)
Schrodinger ZHU Yifan via libc-commits
libc-commits at lists.llvm.org
Wed Jun 26 17:59:03 PDT 2024
https://github.com/SchrodingerZhu updated https://github.com/llvm/llvm-project/pull/96226
>From 3fef09f475f32cf06fcb0e04480b6138e2a043f1 Mon Sep 17 00:00:00 2001
From: Schrodinger ZHU Yifan <i at zhuyi.fan>
Date: Thu, 20 Jun 2024 11:47:41 -0700
Subject: [PATCH 1/4] [libc] inline fast path of callonce
---
libc/src/__support/threads/callonce.h | 29 +++++++++++++++++--
libc/src/__support/threads/linux/callonce.cpp | 18 ++----------
2 files changed, 30 insertions(+), 17 deletions(-)
diff --git a/libc/src/__support/threads/callonce.h b/libc/src/__support/threads/callonce.h
index b3d6813f7dda9..55199b6e30906 100644
--- a/libc/src/__support/threads/callonce.h
+++ b/libc/src/__support/threads/callonce.h
@@ -9,13 +9,38 @@
#ifndef LLVM_LIBC_SRC___SUPPORT_THREADS_CALLONCE_H
#define LLVM_LIBC_SRC___SUPPORT_THREADS_CALLONCE_H
+#include "src/__support/macros/optimization.h"
+
+#ifdef __linux__
+#include "src/__support/threads/linux/futex_utils.h"
+#else
+#error "callonce is not supported on this platform"
+#endif
+
namespace LIBC_NAMESPACE {
-struct CallOnceFlag;
+#ifdef __linux__
+using CallOnceFlag = Futex;
+#endif
using CallOnceCallback = void(void);
+namespace callonce_impl {
+static constexpr FutexWordType NOT_CALLED = 0x0;
+static constexpr FutexWordType START = 0x11;
+static constexpr FutexWordType WAITING = 0x22;
+static constexpr FutexWordType FINISH = 0x33;
+int callonce_slowpath(CallOnceFlag *flag, CallOnceCallback *callback);
+} // namespace callonce_impl
-int callonce(CallOnceFlag *flag, CallOnceCallback *callback);
+LIBC_INLINE int callonce(CallOnceFlag *flag, CallOnceCallback *callback) {
+ using namespace callonce_impl;
+ // Avoid cmpxchg operation if the function has already been called.
+ // The destination operand of cmpxchg may receive a write cycle without
+ // regard to the result of the comparison
+ if (LIBC_LIKELY(flag->load(cpp::MemoryOrder::RELAXED) == FINISH))
+ return 0;
+ return callonce_slowpath(flag, callback);
+}
} // namespace LIBC_NAMESPACE
#endif // LLVM_LIBC_SRC___SUPPORT_THREADS_CALLONCE_H
diff --git a/libc/src/__support/threads/linux/callonce.cpp b/libc/src/__support/threads/linux/callonce.cpp
index b48a514a44875..2bea2da2ae21d 100644
--- a/libc/src/__support/threads/linux/callonce.cpp
+++ b/libc/src/__support/threads/linux/callonce.cpp
@@ -7,27 +7,15 @@
//===----------------------------------------------------------------------===//
#include "src/__support/threads/callonce.h"
-#include "src/__support/macros/optimization.h"
#include "src/__support/threads/linux/futex_utils.h"
namespace LIBC_NAMESPACE {
-
-static constexpr FutexWordType NOT_CALLED = 0x0;
-static constexpr FutexWordType START = 0x11;
-static constexpr FutexWordType WAITING = 0x22;
-static constexpr FutexWordType FINISH = 0x33;
-
-int callonce(CallOnceFlag *flag, CallOnceCallback *func) {
+namespace callonce_impl {
+int callonce_slowpath(CallOnceFlag *flag, CallOnceCallback *func) {
auto *futex_word = reinterpret_cast<Futex *>(flag);
FutexWordType not_called = NOT_CALLED;
- // Avoid cmpxchg operation if the function has already been called.
- // The destination operand of cmpxchg may receive a write cycle without
- // regard to the result of the comparison
- if (LIBC_LIKELY(futex_word->load(cpp::MemoryOrder::RELAXED) == FINISH))
- return 0;
-
// The call_once call can return only after the called function |func|
// returns. So, we use futexes to synchronize calls with the same flag value.
if (futex_word->compare_exchange_strong(not_called, START)) {
@@ -46,5 +34,5 @@ int callonce(CallOnceFlag *flag, CallOnceCallback *func) {
return 0;
}
-
+} // namespace callonce_impl
} // namespace LIBC_NAMESPACE
>From c6bb43daf5cb4a2032e6bc21a39305372a6cbeff Mon Sep 17 00:00:00 2001
From: Schrodinger ZHU Yifan <i at zhuyi.fan>
Date: Tue, 25 Jun 2024 21:54:40 -0700
Subject: [PATCH 2/4] address CRs
---
libc/src/__support/threads/callonce.h | 24 ++++++--------
.../__support/threads/linux/CMakeLists.txt | 2 ++
libc/src/__support/threads/linux/callonce.h | 31 +++++++++++++++++++
3 files changed, 42 insertions(+), 15 deletions(-)
create mode 100644 libc/src/__support/threads/linux/callonce.h
diff --git a/libc/src/__support/threads/callonce.h b/libc/src/__support/threads/callonce.h
index 55199b6e30906..1ba8982211ad7 100644
--- a/libc/src/__support/threads/callonce.h
+++ b/libc/src/__support/threads/callonce.h
@@ -9,37 +9,31 @@
#ifndef LLVM_LIBC_SRC___SUPPORT_THREADS_CALLONCE_H
#define LLVM_LIBC_SRC___SUPPORT_THREADS_CALLONCE_H
-#include "src/__support/macros/optimization.h"
+#include "src/__support/macros/optimization.h" // For LIBC_LIKELY
+// Plaform specific routines, provides:
+// - OnceFlag definition
+// - callonce_impl::callonce_fastpath for fast path check
+// - callonce_impl::callonce_slowpath for slow path execution
#ifdef __linux__
-#include "src/__support/threads/linux/futex_utils.h"
+#include "src/__support/threads/linux/callonce.h"
#else
#error "callonce is not supported on this platform"
#endif
namespace LIBC_NAMESPACE {
-#ifdef __linux__
-using CallOnceFlag = Futex;
-#endif
+// Common definitions
using CallOnceCallback = void(void);
namespace callonce_impl {
-static constexpr FutexWordType NOT_CALLED = 0x0;
-static constexpr FutexWordType START = 0x11;
-static constexpr FutexWordType WAITING = 0x22;
-static constexpr FutexWordType FINISH = 0x33;
int callonce_slowpath(CallOnceFlag *flag, CallOnceCallback *callback);
} // namespace callonce_impl
LIBC_INLINE int callonce(CallOnceFlag *flag, CallOnceCallback *callback) {
- using namespace callonce_impl;
- // Avoid cmpxchg operation if the function has already been called.
- // The destination operand of cmpxchg may receive a write cycle without
- // regard to the result of the comparison
- if (LIBC_LIKELY(flag->load(cpp::MemoryOrder::RELAXED) == FINISH))
+ if (LIBC_LIKELY(callonce_impl::callonce_fastpath(flag)))
return 0;
- return callonce_slowpath(flag, callback);
+ return callonce_impl::callonce_slowpath(flag, callback);
}
} // namespace LIBC_NAMESPACE
diff --git a/libc/src/__support/threads/linux/CMakeLists.txt b/libc/src/__support/threads/linux/CMakeLists.txt
index 95e509b7a825d..8b7971584e77e 100644
--- a/libc/src/__support/threads/linux/CMakeLists.txt
+++ b/libc/src/__support/threads/linux/CMakeLists.txt
@@ -100,8 +100,10 @@ add_object_library(
callonce.cpp
HDRS
../callonce.h
+ callonce.h
DEPENDS
.futex_utils
+ libc.src.__support.macros.optimization
)
add_object_library(
diff --git a/libc/src/__support/threads/linux/callonce.h b/libc/src/__support/threads/linux/callonce.h
new file mode 100644
index 0000000000000..315cc6149e9ec
--- /dev/null
+++ b/libc/src/__support/threads/linux/callonce.h
@@ -0,0 +1,31 @@
+//===-- Linux callonce fastpath -------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+#ifndef LLVM_LIBC_SRC___SUPPORT_THREADS_LINUX_CALLONCE_H
+#define LLVM_LIBC_SRC___SUPPORT_THREADS_LINUX_CALLONCE_H
+
+#include "src/__support/threads/linux/futex_utils.h"
+
+namespace LIBC_NAMESPACE {
+using CallOnceFlag = Futex;
+
+namespace callonce_impl {
+static constexpr FutexWordType NOT_CALLED = 0x0;
+static constexpr FutexWordType START = 0x11;
+static constexpr FutexWordType WAITING = 0x22;
+static constexpr FutexWordType FINISH = 0x33;
+
+// Avoid cmpxchg operation if the function has already been called.
+// The destination operand of cmpxchg may receive a write cycle without
+// regard to the result of the comparison.
+LIBC_INLINE bool callonce_fastpath(CallOnceFlag *flag) {
+ return flag->load(cpp::MemoryOrder::RELAXED) == FINISH;
+}
+} // namespace callonce_impl
+
+} // namespace LIBC_NAMESPACE
+#endif // LLVM_LIBC_SRC___SUPPORT_THREADS_LINUX_CALLONCE_H
>From c2bff314b0c6e7b11142a3510ff9f175ed411c17 Mon Sep 17 00:00:00 2001
From: Schrodinger ZHU Yifan <yifanzhu at rochester.edu>
Date: Wed, 26 Jun 2024 09:58:00 -0700
Subject: [PATCH 3/4] Update callonce.h
Co-authored-by: Nick Desaulniers (paternity leave) <nickdesaulniers at users.noreply.github.com>
---
libc/src/__support/threads/callonce.h | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/libc/src/__support/threads/callonce.h b/libc/src/__support/threads/callonce.h
index 1ba8982211ad7..67f4c99657107 100644
--- a/libc/src/__support/threads/callonce.h
+++ b/libc/src/__support/threads/callonce.h
@@ -9,7 +9,7 @@
#ifndef LLVM_LIBC_SRC___SUPPORT_THREADS_CALLONCE_H
#define LLVM_LIBC_SRC___SUPPORT_THREADS_CALLONCE_H
-#include "src/__support/macros/optimization.h" // For LIBC_LIKELY
+#include "src/__support/macros/optimization.h" // LIBC_LIKELY
// Plaform specific routines, provides:
// - OnceFlag definition
>From ead664d31a7b06bd7388ccd3c237552c6023a411 Mon Sep 17 00:00:00 2001
From: Schrodinger ZHU Yifan <i at zhuyi.fan>
Date: Wed, 26 Jun 2024 17:58:48 -0700
Subject: [PATCH 4/4] address CRs
---
libc/src/__support/threads/linux/callonce.cpp | 1 +
1 file changed, 1 insertion(+)
diff --git a/libc/src/__support/threads/linux/callonce.cpp b/libc/src/__support/threads/linux/callonce.cpp
index 2bea2da2ae21d..24d376f447fba 100644
--- a/libc/src/__support/threads/linux/callonce.cpp
+++ b/libc/src/__support/threads/linux/callonce.cpp
@@ -7,6 +7,7 @@
//===----------------------------------------------------------------------===//
#include "src/__support/threads/callonce.h"
+#include "src/__support/threads/linux/callonce.h"
#include "src/__support/threads/linux/futex_utils.h"
namespace LIBC_NAMESPACE {
More information about the libc-commits
mailing list