[compiler-rt] aa85c6f - [compiler-rt] Fix atomic support functions on 32-bit architectures
Alex Richardson via llvm-commits
llvm-commits at lists.llvm.org
Mon Sep 21 02:21:49 PDT 2020
Author: Alex Richardson
Date: 2020-09-21T10:21:11+01:00
New Revision: aa85c6f2a528792e2ff778a36fb6f35a01e8191c
URL: https://github.com/llvm/llvm-project/commit/aa85c6f2a528792e2ff778a36fb6f35a01e8191c
DIFF: https://github.com/llvm/llvm-project/commit/aa85c6f2a528792e2ff778a36fb6f35a01e8191c.diff
LOG: [compiler-rt] Fix atomic support functions on 32-bit architectures
The code currently uses __c11_atomic_is_lock_free() to detect whether an
atomic operation is natively supported. However, this can result in a
runtime function call to determine whether the given operation is lock-free
and clang generating a call to e.g. __atomic_load_8 since the branch is
not a constant zero. Since we are implementing those runtime functions, we
must avoid those calls. This patch replaces __c11_atomic_is_lock_free()
with __atomic_always_lock_free() which always results in a compile-time
constant value. This problem was found while compiling atomic.c for MIPS32
since the -Watomic-alignment warning was being triggered and objdump showed
an undefined reference to _atomic_is_lock_free.
In addition to fixing 32-bit platforms this also enables the 16-byte case
that was disabled in r153779 (185f2edd70a34d28b305df0cd8ce519ecbca2cfd).
Reviewed By: efriedma
Differential Revision: https://reviews.llvm.org/D86510
Added:
Modified:
compiler-rt/lib/builtins/atomic.c
Removed:
################################################################################
diff --git a/compiler-rt/lib/builtins/atomic.c b/compiler-rt/lib/builtins/atomic.c
index 8634a72e77d1..b20369d0c48b 100644
--- a/compiler-rt/lib/builtins/atomic.c
+++ b/compiler-rt/lib/builtins/atomic.c
@@ -120,45 +120,39 @@ static __inline Lock *lock_for_pointer(void *ptr) {
return locks + (hash & SPINLOCK_MASK);
}
-/// Macros for determining whether a size is lock free. Clang can not yet
-/// codegen __atomic_is_lock_free(16), so for now we assume 16-byte values are
-/// not lock free.
-#define IS_LOCK_FREE_1 __c11_atomic_is_lock_free(1)
-#define IS_LOCK_FREE_2 __c11_atomic_is_lock_free(2)
-#define IS_LOCK_FREE_4 __c11_atomic_is_lock_free(4)
-#define IS_LOCK_FREE_8 __c11_atomic_is_lock_free(8)
-#define IS_LOCK_FREE_16 0
+/// Macros for determining whether a size is lock free.
+#define ATOMIC_ALWAYS_LOCK_FREE_OR_ALIGNED_LOCK_FREE(size, p) \
+ (__atomic_always_lock_free(size, p) || \
+ (__atomic_always_lock_free(size, 0) && ((uintptr_t)p % size) == 0))
+#define IS_LOCK_FREE_1(p) ATOMIC_ALWAYS_LOCK_FREE_OR_ALIGNED_LOCK_FREE(1, p)
+#define IS_LOCK_FREE_2(p) ATOMIC_ALWAYS_LOCK_FREE_OR_ALIGNED_LOCK_FREE(2, p)
+#define IS_LOCK_FREE_4(p) ATOMIC_ALWAYS_LOCK_FREE_OR_ALIGNED_LOCK_FREE(4, p)
+#define IS_LOCK_FREE_8(p) ATOMIC_ALWAYS_LOCK_FREE_OR_ALIGNED_LOCK_FREE(8, p)
+#define IS_LOCK_FREE_16(p) ATOMIC_ALWAYS_LOCK_FREE_OR_ALIGNED_LOCK_FREE(16, p)
/// Macro that calls the compiler-generated lock-free versions of functions
/// when they exist.
-#define LOCK_FREE_CASES() \
+#define TRY_LOCK_FREE_CASE(n, type, ptr) \
+ case n: \
+ if (IS_LOCK_FREE_##n(ptr)) { \
+ LOCK_FREE_ACTION(type); \
+ } \
+ break;
+#ifdef __SIZEOF_INT128__
+#define TRY_LOCK_FREE_CASE_16(p) TRY_LOCK_FREE_CASE(16, __uint128_t, p)
+#else
+#define TRY_LOCK_FREE_CASE_16(p) /* __uint128_t not available */
+#endif
+
+#define LOCK_FREE_CASES(ptr) \
do { \
switch (size) { \
- case 1: \
- if (IS_LOCK_FREE_1) { \
- LOCK_FREE_ACTION(uint8_t); \
- } \
- break; \
- case 2: \
- if (IS_LOCK_FREE_2) { \
- LOCK_FREE_ACTION(uint16_t); \
- } \
- break; \
- case 4: \
- if (IS_LOCK_FREE_4) { \
- LOCK_FREE_ACTION(uint32_t); \
- } \
- break; \
- case 8: \
- if (IS_LOCK_FREE_8) { \
- LOCK_FREE_ACTION(uint64_t); \
- } \
- break; \
- case 16: \
- if (IS_LOCK_FREE_16) { \
- /* FIXME: __uint128_t isn't available on 32 bit platforms. \
- LOCK_FREE_ACTION(__uint128_t);*/ \
- } \
+ TRY_LOCK_FREE_CASE(1, uint8_t, ptr) \
+ TRY_LOCK_FREE_CASE(2, uint16_t, ptr) \
+ TRY_LOCK_FREE_CASE(4, uint32_t, ptr) \
+ TRY_LOCK_FREE_CASE(8, uint64_t, ptr) \
+ TRY_LOCK_FREE_CASE_16(ptr) /* __uint128_t may not be supported */ \
+ default: \
break; \
} \
} while (0)
@@ -169,7 +163,7 @@ void __atomic_load_c(int size, void *src, void *dest, int model) {
#define LOCK_FREE_ACTION(type) \
*((type *)dest) = __c11_atomic_load((_Atomic(type) *)src, model); \
return;
- LOCK_FREE_CASES();
+ LOCK_FREE_CASES(src);
#undef LOCK_FREE_ACTION
Lock *l = lock_for_pointer(src);
lock(l);
@@ -183,7 +177,7 @@ void __atomic_store_c(int size, void *dest, void *src, int model) {
#define LOCK_FREE_ACTION(type) \
__c11_atomic_store((_Atomic(type) *)dest, *(type *)src, model); \
return;
- LOCK_FREE_CASES();
+ LOCK_FREE_CASES(dest);
#undef LOCK_FREE_ACTION
Lock *l = lock_for_pointer(dest);
lock(l);
@@ -202,7 +196,7 @@ int __atomic_compare_exchange_c(int size, void *ptr, void *expected,
return __c11_atomic_compare_exchange_strong( \
(_Atomic(type) *)ptr, (type *)expected, *(type *)desired, success, \
failure)
- LOCK_FREE_CASES();
+ LOCK_FREE_CASES(ptr);
#undef LOCK_FREE_ACTION
Lock *l = lock_for_pointer(ptr);
lock(l);
@@ -223,7 +217,7 @@ void __atomic_exchange_c(int size, void *ptr, void *val, void *old, int model) {
*(type *)old = \
__c11_atomic_exchange((_Atomic(type) *)ptr, *(type *)val, model); \
return;
- LOCK_FREE_CASES();
+ LOCK_FREE_CASES(ptr);
#undef LOCK_FREE_ACTION
Lock *l = lock_for_pointer(ptr);
lock(l);
@@ -253,7 +247,7 @@ void __atomic_exchange_c(int size, void *ptr, void *val, void *old, int model) {
#define OPTIMISED_CASE(n, lockfree, type) \
type __atomic_load_##n(type *src, int model) { \
- if (lockfree) \
+ if (lockfree(src)) \
return __c11_atomic_load((_Atomic(type) *)src, model); \
Lock *l = lock_for_pointer(src); \
lock(l); \
@@ -266,7 +260,7 @@ OPTIMISED_CASES
#define OPTIMISED_CASE(n, lockfree, type) \
void __atomic_store_##n(type *dest, type val, int model) { \
- if (lockfree) { \
+ if (lockfree(dest)) { \
__c11_atomic_store((_Atomic(type) *)dest, val, model); \
return; \
} \
@@ -281,7 +275,7 @@ OPTIMISED_CASES
#define OPTIMISED_CASE(n, lockfree, type) \
type __atomic_exchange_##n(type *dest, type val, int model) { \
- if (lockfree) \
+ if (lockfree(dest)) \
return __c11_atomic_exchange((_Atomic(type) *)dest, val, model); \
Lock *l = lock_for_pointer(dest); \
lock(l); \
@@ -296,7 +290,7 @@ OPTIMISED_CASES
#define OPTIMISED_CASE(n, lockfree, type) \
bool __atomic_compare_exchange_##n(type *ptr, type *expected, type desired, \
int success, int failure) { \
- if (lockfree) \
+ if (lockfree(ptr)) \
return __c11_atomic_compare_exchange_strong( \
(_Atomic(type) *)ptr, expected, desired, success, failure); \
Lock *l = lock_for_pointer(ptr); \
@@ -318,7 +312,7 @@ OPTIMISED_CASES
////////////////////////////////////////////////////////////////////////////////
#define ATOMIC_RMW(n, lockfree, type, opname, op) \
type __atomic_fetch_##opname##_##n(type *ptr, type val, int model) { \
- if (lockfree) \
+ if (lockfree(ptr)) \
return __c11_atomic_fetch_##opname((_Atomic(type) *)ptr, val, model); \
Lock *l = lock_for_pointer(ptr); \
lock(l); \
More information about the llvm-commits
mailing list