[compiler-rt] r302300 - [scudo] Add Android support
Kostya Kortchinsky via llvm-commits
llvm-commits at lists.llvm.org
Fri May 5 14:38:23 PDT 2017
Author: cryptoad
Date: Fri May 5 16:38:22 2017
New Revision: 302300
URL: http://llvm.org/viewvc/llvm-project?rev=302300&view=rev
Log:
[scudo] Add Android support
Summary:
This change adds Android support to the allocator (but doesn't yet enable it in
the cmake config), and should be the last fragment of the rewritten change
D31947.
Android has more memory constraints than other platforms, so the idea of a
unique context per thread would not have worked. The alternative chosen is to
allocate a set of contexts based on the number of cores on the machine, and
share those contexts within the threads. Contexts can be dynamically reassigned
to threads to prevent contention, based on a scheme suggested by @dvyuokv in
the initial review.
Additionally, given that Android doesn't support ELF TLS (only emutls for now),
we use the TSan TLS slot to make things faster: Scudo is mutually exclusive
with other sanitizers so this shouldn't cause any problem.
An additional change made here, is replacing `thread_local` by `THREADLOCAL`
and using the initial-exec thread model in the non-Android version to prevent
extraneous weak definition and checks on the relevant variables.
Reviewers: kcc, dvyukov, alekseyshl
Reviewed By: alekseyshl
Subscribers: srhines, mgorny, llvm-commits
Differential Revision: https://reviews.llvm.org/D32649
Added:
compiler-rt/trunk/lib/scudo/scudo_tls_android.cpp
compiler-rt/trunk/lib/scudo/scudo_tls_android.inc
compiler-rt/trunk/lib/scudo/scudo_tls_context_android.inc
compiler-rt/trunk/lib/scudo/scudo_tls_context_linux.inc
compiler-rt/trunk/lib/scudo/scudo_tls_linux.inc
- copied, changed from r302299, compiler-rt/trunk/lib/scudo/scudo_tls_linux.h
Removed:
compiler-rt/trunk/lib/scudo/scudo_tls_linux.h
Modified:
compiler-rt/trunk/lib/scudo/CMakeLists.txt
compiler-rt/trunk/lib/scudo/scudo_allocator.cpp
compiler-rt/trunk/lib/scudo/scudo_allocator.h
compiler-rt/trunk/lib/scudo/scudo_tls.h
compiler-rt/trunk/lib/scudo/scudo_tls_linux.cpp
Modified: compiler-rt/trunk/lib/scudo/CMakeLists.txt
URL: http://llvm.org/viewvc/llvm-project/compiler-rt/trunk/lib/scudo/CMakeLists.txt?rev=302300&r1=302299&r2=302300&view=diff
==============================================================================
--- compiler-rt/trunk/lib/scudo/CMakeLists.txt (original)
+++ compiler-rt/trunk/lib/scudo/CMakeLists.txt Fri May 5 16:38:22 2017
@@ -14,6 +14,7 @@ set(SCUDO_SOURCES
scudo_interceptors.cpp
scudo_new_delete.cpp
scudo_termination.cpp
+ scudo_tls_android.cpp
scudo_tls_linux.cpp
scudo_utils.cpp)
Modified: compiler-rt/trunk/lib/scudo/scudo_allocator.cpp
URL: http://llvm.org/viewvc/llvm-project/compiler-rt/trunk/lib/scudo/scudo_allocator.cpp?rev=302300&r1=302299&r2=302300&view=diff
==============================================================================
--- compiler-rt/trunk/lib/scudo/scudo_allocator.cpp (original)
+++ compiler-rt/trunk/lib/scudo/scudo_allocator.cpp Fri May 5 16:38:22 2017
@@ -368,11 +368,12 @@ struct ScudoAllocator {
void *Ptr;
uptr Salt;
uptr AllocationAlignment = FromPrimary ? MinAlignment : Alignment;
- ScudoThreadContext *ThreadContext = getThreadContext();
+ ScudoThreadContext *ThreadContext = getThreadContextAndLock();
if (LIKELY(ThreadContext)) {
Salt = getPrng(ThreadContext)->getNext();
Ptr = BackendAllocator.Allocate(getAllocatorCache(ThreadContext),
NeededSize, AllocationAlignment);
+ ThreadContext->unlock();
} else {
SpinMutexLock l(&FallbackMutex);
Salt = FallbackPrng.getNext();
@@ -434,9 +435,10 @@ struct ScudoAllocator {
if (BypassQuarantine) {
Chunk->eraseHeader();
void *Ptr = Chunk->getAllocBeg(Header);
- ScudoThreadContext *ThreadContext = getThreadContext();
+ ScudoThreadContext *ThreadContext = getThreadContextAndLock();
if (LIKELY(ThreadContext)) {
getBackendAllocator().Deallocate(getAllocatorCache(ThreadContext), Ptr);
+ ThreadContext->unlock();
} else {
SpinMutexLock Lock(&FallbackMutex);
getBackendAllocator().Deallocate(&FallbackAllocatorCache, Ptr);
@@ -445,12 +447,13 @@ struct ScudoAllocator {
UnpackedHeader NewHeader = *Header;
NewHeader.State = ChunkQuarantine;
Chunk->compareExchangeHeader(&NewHeader, Header);
- ScudoThreadContext *ThreadContext = getThreadContext();
+ ScudoThreadContext *ThreadContext = getThreadContextAndLock();
if (LIKELY(ThreadContext)) {
AllocatorQuarantine.Put(getQuarantineCache(ThreadContext),
QuarantineCallback(
getAllocatorCache(ThreadContext)),
Chunk, Size);
+ ThreadContext->unlock();
} else {
SpinMutexLock l(&FallbackMutex);
AllocatorQuarantine.Put(&FallbackQuarantineCache,
Modified: compiler-rt/trunk/lib/scudo/scudo_allocator.h
URL: http://llvm.org/viewvc/llvm-project/compiler-rt/trunk/lib/scudo/scudo_allocator.h?rev=302300&r1=302299&r2=302300&view=diff
==============================================================================
--- compiler-rt/trunk/lib/scudo/scudo_allocator.h (original)
+++ compiler-rt/trunk/lib/scudo/scudo_allocator.h Fri May 5 16:38:22 2017
@@ -72,7 +72,13 @@ const uptr AlignedChunkHeaderSize =
#if SANITIZER_CAN_USE_ALLOCATOR64
const uptr AllocatorSpace = ~0ULL;
-const uptr AllocatorSize = 0x40000000000ULL; // 4TB.
+# if defined(__aarch64__) && SANITIZER_ANDROID
+const uptr AllocatorSize = 0x4000000000ULL; // 256G.
+# elif defined(__aarch64__)
+const uptr AllocatorSize = 0x10000000000ULL; // 1T.
+# else
+const uptr AllocatorSize = 0x40000000000ULL; // 4T.
+# endif
typedef DefaultSizeClassMap SizeClassMap;
struct AP {
static const uptr kSpaceBeg = AllocatorSpace;
Modified: compiler-rt/trunk/lib/scudo/scudo_tls.h
URL: http://llvm.org/viewvc/llvm-project/compiler-rt/trunk/lib/scudo/scudo_tls.h?rev=302300&r1=302299&r2=302300&view=diff
==============================================================================
--- compiler-rt/trunk/lib/scudo/scudo_tls.h (original)
+++ compiler-rt/trunk/lib/scudo/scudo_tls.h Fri May 5 16:38:22 2017
@@ -19,10 +19,16 @@
#include "scudo_allocator.h"
#include "scudo_utils.h"
+#include "sanitizer_common/sanitizer_linux.h"
+#include "sanitizer_common/sanitizer_platform.h"
+
namespace __scudo {
-struct ALIGNED(64) ScudoThreadContext {
- public:
+// Platform specific base thread context definitions.
+#include "scudo_tls_context_android.inc"
+#include "scudo_tls_context_linux.inc"
+
+struct ALIGNED(64) ScudoThreadContext : public ScudoThreadContextPlatform {
AllocatorCache Cache;
Xorshift128Plus Prng;
uptr QuarantineCachePlaceHolder[4];
@@ -32,8 +38,9 @@ struct ALIGNED(64) ScudoThreadContext {
void initThread();
-// Fastpath functions are defined in the following platform specific headers.
-#include "scudo_tls_linux.h"
+// Platform specific dastpath functions definitions.
+#include "scudo_tls_android.inc"
+#include "scudo_tls_linux.inc"
} // namespace __scudo
Added: compiler-rt/trunk/lib/scudo/scudo_tls_android.cpp
URL: http://llvm.org/viewvc/llvm-project/compiler-rt/trunk/lib/scudo/scudo_tls_android.cpp?rev=302300&view=auto
==============================================================================
--- compiler-rt/trunk/lib/scudo/scudo_tls_android.cpp (added)
+++ compiler-rt/trunk/lib/scudo/scudo_tls_android.cpp Fri May 5 16:38:22 2017
@@ -0,0 +1,95 @@
+//===-- scudo_tls_android.cpp -----------------------------------*- C++ -*-===//
+//
+// The LLVM Compiler Infrastructure
+//
+// This file is distributed under the University of Illinois Open Source
+// License. See LICENSE.TXT for details.
+//
+//===----------------------------------------------------------------------===//
+///
+/// Scudo thread local structure implementation for Android.
+///
+//===----------------------------------------------------------------------===//
+
+#include "sanitizer_common/sanitizer_platform.h"
+
+#if SANITIZER_LINUX && SANITIZER_ANDROID
+
+#include "scudo_tls.h"
+
+#include <pthread.h>
+
+namespace __scudo {
+
+static pthread_once_t GlobalInitialized = PTHREAD_ONCE_INIT;
+static pthread_key_t PThreadKey;
+
+static atomic_uint32_t ThreadContextCurrentIndex;
+static ScudoThreadContext *ThreadContexts;
+static uptr NumberOfContexts;
+
+// sysconf(_SC_NPROCESSORS_{CONF,ONLN}) cannot be used as they allocate memory.
+static uptr getNumberOfCPUs() {
+ cpu_set_t CPUs;
+ CHECK_EQ(sched_getaffinity(0, sizeof(cpu_set_t), &CPUs), 0);
+ return CPU_COUNT(&CPUs);
+}
+
+static void initOnce() {
+ // Hack: TLS_SLOT_TSAN was introduced in N. To be able to use it on M for
+ // testing, we create an unused key. Since the key_data array follows the tls
+ // array, it basically gives us the extra entry we need.
+ // TODO(kostyak): remove and restrict to N and above.
+ CHECK_EQ(pthread_key_create(&PThreadKey, NULL), 0);
+ initScudo();
+ NumberOfContexts = getNumberOfCPUs();
+ ThreadContexts = reinterpret_cast<ScudoThreadContext *>(
+ MmapOrDie(sizeof(ScudoThreadContext) * NumberOfContexts, __func__));
+ for (int i = 0; i < NumberOfContexts; i++)
+ ThreadContexts[i].init();
+}
+
+void initThread() {
+ pthread_once(&GlobalInitialized, initOnce);
+ // Initial context assignment is done in a plain round-robin fashion.
+ u32 Index = atomic_fetch_add(&ThreadContextCurrentIndex, 1,
+ memory_order_relaxed);
+ ScudoThreadContext *ThreadContext =
+ &ThreadContexts[Index % NumberOfContexts];
+ *get_android_tls_ptr() = reinterpret_cast<uptr>(ThreadContext);
+}
+
+ScudoThreadContext *getThreadContextAndLockSlow() {
+ ScudoThreadContext *ThreadContext;
+ // Go through all the contexts and find the first unlocked one.
+ for (u32 i = 0; i < NumberOfContexts; i++) {
+ ThreadContext = &ThreadContexts[i];
+ if (ThreadContext->tryLock()) {
+ *get_android_tls_ptr() = reinterpret_cast<uptr>(ThreadContext);
+ return ThreadContext;
+ }
+ }
+ // No luck, find the one with the lowest precedence, and slow lock it.
+ u64 Precedence = UINT64_MAX;
+ for (u32 i = 0; i < NumberOfContexts; i++) {
+ u64 SlowLockPrecedence = ThreadContexts[i].getSlowLockPrecedence();
+ if (SlowLockPrecedence && SlowLockPrecedence < Precedence) {
+ ThreadContext = &ThreadContexts[i];
+ Precedence = SlowLockPrecedence;
+ }
+ }
+ if (LIKELY(Precedence != UINT64_MAX)) {
+ ThreadContext->lock();
+ *get_android_tls_ptr() = reinterpret_cast<uptr>(ThreadContext);
+ return ThreadContext;
+ }
+ // Last resort (can this happen?), stick with the current one.
+ ThreadContext =
+ reinterpret_cast<ScudoThreadContext *>(*get_android_tls_ptr());
+ ThreadContext->lock();
+ return ThreadContext;
+}
+
+} // namespace __scudo
+
+#endif // SANITIZER_LINUX && SANITIZER_ANDROID
Added: compiler-rt/trunk/lib/scudo/scudo_tls_android.inc
URL: http://llvm.org/viewvc/llvm-project/compiler-rt/trunk/lib/scudo/scudo_tls_android.inc?rev=302300&view=auto
==============================================================================
--- compiler-rt/trunk/lib/scudo/scudo_tls_android.inc (added)
+++ compiler-rt/trunk/lib/scudo/scudo_tls_android.inc Fri May 5 16:38:22 2017
@@ -0,0 +1,44 @@
+//===-- scudo_tls_android.inc -----------------------------------*- C++ -*-===//
+//
+// The LLVM Compiler Infrastructure
+//
+// This file is distributed under the University of Illinois Open Source
+// License. See LICENSE.TXT for details.
+//
+//===----------------------------------------------------------------------===//
+///
+/// Scudo thread local structure fastpath functions implementation for Android.
+///
+//===----------------------------------------------------------------------===//
+
+#ifndef SCUDO_TLS_ANDROID_H_
+#define SCUDO_TLS_ANDROID_H_
+
+#ifndef SCUDO_TLS_H_
+# error "This file must be included inside scudo_tls.h."
+#endif // SCUDO_TLS_H_
+
+#if SANITIZER_LINUX && SANITIZER_ANDROID
+
+ALWAYS_INLINE void initThreadMaybe() {
+ if (LIKELY(*get_android_tls_ptr()))
+ return;
+ initThread();
+}
+
+ScudoThreadContext *getThreadContextAndLockSlow();
+
+ALWAYS_INLINE ScudoThreadContext *getThreadContextAndLock() {
+ ScudoThreadContext *ThreadContext =
+ reinterpret_cast<ScudoThreadContext *>(*get_android_tls_ptr());
+ CHECK(ThreadContext);
+ // Try to lock the currently associated context.
+ if (ThreadContext->tryLock())
+ return ThreadContext;
+ // If it failed, go the slow path.
+ return getThreadContextAndLockSlow();
+}
+
+#endif // SANITIZER_LINUX && SANITIZER_ANDROID
+
+#endif // SCUDO_TLS_ANDROID_H_
Added: compiler-rt/trunk/lib/scudo/scudo_tls_context_android.inc
URL: http://llvm.org/viewvc/llvm-project/compiler-rt/trunk/lib/scudo/scudo_tls_context_android.inc?rev=302300&view=auto
==============================================================================
--- compiler-rt/trunk/lib/scudo/scudo_tls_context_android.inc (added)
+++ compiler-rt/trunk/lib/scudo/scudo_tls_context_android.inc Fri May 5 16:38:22 2017
@@ -0,0 +1,54 @@
+//===-- scudo_tls_context_android.inc ---------------------------*- C++ -*-===//
+//
+// The LLVM Compiler Infrastructure
+//
+// This file is distributed under the University of Illinois Open Source
+// License. See LICENSE.TXT for details.
+//
+//===----------------------------------------------------------------------===//
+///
+/// Android specific base thread context definition.
+///
+//===----------------------------------------------------------------------===//
+
+#ifndef SCUDO_TLS_CONTEXT_ANDROID_INC_
+#define SCUDO_TLS_CONTEXT_ANDROID_INC_
+
+#ifndef SCUDO_TLS_H_
+# error "This file must be included inside scudo_tls.h."
+#endif // SCUDO_TLS_H_
+
+#if SANITIZER_LINUX && SANITIZER_ANDROID
+
+struct ScudoThreadContextPlatform {
+ INLINE bool tryLock() {
+ if (Mutex.TryLock()) {
+ atomic_store_relaxed(&SlowLockPrecedence, 0);
+ return true;
+ }
+ if (atomic_load_relaxed(&SlowLockPrecedence) == 0)
+ atomic_store_relaxed(&SlowLockPrecedence, NanoTime());
+ return false;
+ }
+
+ INLINE void lock() {
+ Mutex.Lock();
+ atomic_store_relaxed(&SlowLockPrecedence, 0);
+ }
+
+ INLINE void unlock() {
+ Mutex.Unlock();
+ }
+
+ INLINE u64 getSlowLockPrecedence() {
+ return atomic_load_relaxed(&SlowLockPrecedence);
+ }
+
+ private:
+ StaticSpinMutex Mutex;
+ atomic_uint64_t SlowLockPrecedence;
+};
+
+#endif // SANITIZER_LINUX && SANITIZER_ANDROID
+
+#endif // SCUDO_TLS_CONTEXT_ANDROID_INC_
Added: compiler-rt/trunk/lib/scudo/scudo_tls_context_linux.inc
URL: http://llvm.org/viewvc/llvm-project/compiler-rt/trunk/lib/scudo/scudo_tls_context_linux.inc?rev=302300&view=auto
==============================================================================
--- compiler-rt/trunk/lib/scudo/scudo_tls_context_linux.inc (added)
+++ compiler-rt/trunk/lib/scudo/scudo_tls_context_linux.inc Fri May 5 16:38:22 2017
@@ -0,0 +1,29 @@
+//===-- scudo_tls_context_linux.inc -----------------------------*- C++ -*-===//
+//
+// The LLVM Compiler Infrastructure
+//
+// This file is distributed under the University of Illinois Open Source
+// License. See LICENSE.TXT for details.
+//
+//===----------------------------------------------------------------------===//
+///
+/// Linux specific base thread context definition.
+///
+//===----------------------------------------------------------------------===//
+
+#ifndef SCUDO_TLS_CONTEXT_LINUX_INC_
+#define SCUDO_TLS_CONTEXT_LINUX_INC_
+
+#ifndef SCUDO_TLS_H_
+# error "This file must be included inside scudo_tls.h."
+#endif // SCUDO_TLS_H_
+
+#if SANITIZER_LINUX && !SANITIZER_ANDROID
+
+struct ScudoThreadContextPlatform {
+ ALWAYS_INLINE void unlock() {}
+};
+
+#endif // SANITIZER_LINUX && !SANITIZER_ANDROID
+
+#endif // SCUDO_TLS_CONTEXT_LINUX_INC_
Modified: compiler-rt/trunk/lib/scudo/scudo_tls_linux.cpp
URL: http://llvm.org/viewvc/llvm-project/compiler-rt/trunk/lib/scudo/scudo_tls_linux.cpp?rev=302300&r1=302299&r2=302300&view=diff
==============================================================================
--- compiler-rt/trunk/lib/scudo/scudo_tls_linux.cpp (original)
+++ compiler-rt/trunk/lib/scudo/scudo_tls_linux.cpp Fri May 5 16:38:22 2017
@@ -14,7 +14,7 @@
#include "sanitizer_common/sanitizer_platform.h"
-#if SANITIZER_LINUX
+#if SANITIZER_LINUX && !SANITIZER_ANDROID
#include "scudo_tls.h"
@@ -26,8 +26,10 @@ namespace __scudo {
static pthread_once_t GlobalInitialized = PTHREAD_ONCE_INIT;
static pthread_key_t PThreadKey;
-thread_local ThreadState ScudoThreadState = ThreadNotInitialized;
-thread_local ScudoThreadContext ThreadLocalContext;
+__attribute__((tls_model("initial-exec")))
+THREADLOCAL ThreadState ScudoThreadState = ThreadNotInitialized;
+__attribute__((tls_model("initial-exec")))
+THREADLOCAL ScudoThreadContext ThreadLocalContext;
static void teardownThread(void *Ptr) {
uptr Iteration = reinterpret_cast<uptr>(Ptr);
@@ -59,4 +61,4 @@ void initThread() {
} // namespace __scudo
-#endif // SANITIZER_LINUX
+#endif // SANITIZER_LINUX && !SANITIZER_ANDROID
Removed: compiler-rt/trunk/lib/scudo/scudo_tls_linux.h
URL: http://llvm.org/viewvc/llvm-project/compiler-rt/trunk/lib/scudo/scudo_tls_linux.h?rev=302299&view=auto
==============================================================================
--- compiler-rt/trunk/lib/scudo/scudo_tls_linux.h (original)
+++ compiler-rt/trunk/lib/scudo/scudo_tls_linux.h (removed)
@@ -1,48 +0,0 @@
-//===-- scudo_tls_linux.h ---------------------------------------*- C++ -*-===//
-//
-// The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
-//
-//===----------------------------------------------------------------------===//
-///
-/// Scudo thread local structure fastpath functions implementation for platforms
-/// supporting thread_local.
-///
-//===----------------------------------------------------------------------===//
-
-#ifndef SCUDO_TLS_LINUX_H_
-#define SCUDO_TLS_LINUX_H_
-
-#ifndef SCUDO_TLS_H_
-# error "This file must be included inside scudo_tls.h."
-#endif // SCUDO_TLS_H_
-
-#include "sanitizer_common/sanitizer_platform.h"
-
-#if SANITIZER_LINUX
-
-enum ThreadState : u8 {
- ThreadNotInitialized = 0,
- ThreadInitialized,
- ThreadTornDown,
-};
-extern thread_local ThreadState ScudoThreadState;
-extern thread_local ScudoThreadContext ThreadLocalContext;
-
-ALWAYS_INLINE void initThreadMaybe() {
- if (LIKELY(ScudoThreadState != ThreadNotInitialized))
- return;
- initThread();
-}
-
-ALWAYS_INLINE ScudoThreadContext *getThreadContext() {
- if (UNLIKELY(ScudoThreadState == ThreadTornDown))
- return nullptr;
- return &ThreadLocalContext;
-}
-
-#endif // SANITIZER_LINUX
-
-#endif // SCUDO_TLS_LINUX_H_
Copied: compiler-rt/trunk/lib/scudo/scudo_tls_linux.inc (from r302299, compiler-rt/trunk/lib/scudo/scudo_tls_linux.h)
URL: http://llvm.org/viewvc/llvm-project/compiler-rt/trunk/lib/scudo/scudo_tls_linux.inc?p2=compiler-rt/trunk/lib/scudo/scudo_tls_linux.inc&p1=compiler-rt/trunk/lib/scudo/scudo_tls_linux.h&r1=302299&r2=302300&rev=302300&view=diff
==============================================================================
--- compiler-rt/trunk/lib/scudo/scudo_tls_linux.h (original)
+++ compiler-rt/trunk/lib/scudo/scudo_tls_linux.inc Fri May 5 16:38:22 2017
@@ -1,4 +1,4 @@
-//===-- scudo_tls_linux.h ---------------------------------------*- C++ -*-===//
+//===-- scudo_tls_linux.inc -------------------------------------*- C++ -*-===//
//
// The LLVM Compiler Infrastructure
//
@@ -19,17 +19,17 @@
# error "This file must be included inside scudo_tls.h."
#endif // SCUDO_TLS_H_
-#include "sanitizer_common/sanitizer_platform.h"
-
-#if SANITIZER_LINUX
+#if SANITIZER_LINUX && !SANITIZER_ANDROID
enum ThreadState : u8 {
ThreadNotInitialized = 0,
ThreadInitialized,
ThreadTornDown,
};
-extern thread_local ThreadState ScudoThreadState;
-extern thread_local ScudoThreadContext ThreadLocalContext;
+__attribute__((tls_model("initial-exec")))
+extern THREADLOCAL ThreadState ScudoThreadState;
+__attribute__((tls_model("initial-exec")))
+extern THREADLOCAL ScudoThreadContext ThreadLocalContext;
ALWAYS_INLINE void initThreadMaybe() {
if (LIKELY(ScudoThreadState != ThreadNotInitialized))
@@ -37,12 +37,12 @@ ALWAYS_INLINE void initThreadMaybe() {
initThread();
}
-ALWAYS_INLINE ScudoThreadContext *getThreadContext() {
+ALWAYS_INLINE ScudoThreadContext *getThreadContextAndLock() {
if (UNLIKELY(ScudoThreadState == ThreadTornDown))
return nullptr;
return &ThreadLocalContext;
}
-#endif // SANITIZER_LINUX
+#endif // SANITIZER_LINUX && !SANITIZER_ANDROID
#endif // SCUDO_TLS_LINUX_H_
More information about the llvm-commits
mailing list