[compiler-rt] r334463 - [builtins] Delay emutls deallocation for one round

[Ryan Prichard via llvm-commits]

Author: rprichard
Date: Mon Jun 11 18:32:26 2018
New Revision: 334463

URL: http://llvm.org/viewvc/llvm-project?rev=334463&view=rev
Log:
[builtins] Delay emutls deallocation for one round

Summary:
With Android/Bionic, delay deallocation to round 2 of 4. It must run after
C++ thread_local destructors have been called, but before the final 2
rounds, because emutls calls free, and jemalloc then needs another 2
rounds to free its thread-specific data.

Fixes https://github.com/android-ndk/ndk/issues/687

Reviewers: cmtice, srhines, jyknight, chh, echristo

Reviewed By: srhines, chh, echristo

Subscribers: echristo, delcypher, llvm-commits, #sanitizers

Differential Revision: https://reviews.llvm.org/D46978

Modified:
    compiler-rt/trunk/lib/builtins/emutls.c

Modified: compiler-rt/trunk/lib/builtins/emutls.c
URL: http://llvm.org/viewvc/llvm-project/compiler-rt/trunk/lib/builtins/emutls.c?rev=334463&r1=334462&r2=334463&view=diff
==============================================================================
--- compiler-rt/trunk/lib/builtins/emutls.c (original)
+++ compiler-rt/trunk/lib/builtins/emutls.c Mon Jun 11 18:32:26 2018
@@ -14,7 +14,22 @@
 #include "int_lib.h"
 #include "int_util.h"
 
+#ifdef __BIONIC__
+/* There are 4 pthread key cleanup rounds on Bionic. Delay emutls deallocation
+   to round 2. We need to delay deallocation because:
+    - Android versions older than M lack __cxa_thread_atexit_impl, so apps
+      use a pthread key destructor to call C++ destructors.
+    - Apps might use __thread/thread_local variables in pthread destructors.
+   We can't wait until the final two rounds, because jemalloc needs two rounds
+   after the final malloc/free call to free its thread-specific data (see
+   https://reviews.llvm.org/D46978#1107507). */
+#define EMUTLS_SKIP_DESTRUCTOR_ROUNDS 1
+#else
+#define EMUTLS_SKIP_DESTRUCTOR_ROUNDS 0
+#endif
+
 typedef struct emutls_address_array {
+    uintptr_t skip_destructor_rounds;
     uintptr_t size;  /* number of elements in the 'data' array */
     void* data[];
 } emutls_address_array;
@@ -65,9 +80,30 @@ static __inline void emutls_memalign_fre
 #endif
 }
 
+static __inline void emutls_setspecific(emutls_address_array *value) {
+    pthread_setspecific(emutls_pthread_key, (void*) value);
+}
+
+static __inline emutls_address_array* emutls_getspecific() {
+    return (emutls_address_array*) pthread_getspecific(emutls_pthread_key);
+}
+
 static void emutls_key_destructor(void* ptr) {
-    emutls_shutdown((emutls_address_array*)ptr);
-    free(ptr);
+    emutls_address_array *array = (emutls_address_array*)ptr;
+    if (array->skip_destructor_rounds > 0) {
+        /* emutls is deallocated using a pthread key destructor. These
+         * destructors are called in several rounds to accommodate destructor
+         * functions that (re)initialize key values with pthread_setspecific.
+         * Delay the emutls deallocation to accommodate other end-of-thread
+         * cleanup tasks like calling thread_local destructors (e.g. the
+         * __cxa_thread_atexit fallback in libc++abi).
+         */
+        array->skip_destructor_rounds--;
+        emutls_setspecific(array);
+    } else {
+        emutls_shutdown(array);
+        free(ptr);
+    }
 }
 
 static __inline void emutls_init(void) {
@@ -88,14 +124,6 @@ static __inline void emutls_unlock() {
     pthread_mutex_unlock(&emutls_mutex);
 }
 
-static __inline void emutls_setspecific(emutls_address_array *value) {
-    pthread_setspecific(emutls_pthread_key, (void*) value);
-}
-
-static __inline emutls_address_array* emutls_getspecific() {
-    return (emutls_address_array*) pthread_getspecific(emutls_pthread_key);
-}
-
 #else /* _WIN32 */
 
 #include <windows.h>
@@ -338,8 +366,10 @@ emutls_get_address_array(uintptr_t index
     if (array == NULL) {
         uintptr_t new_size = emutls_new_data_array_size(index);
         array = (emutls_address_array*) malloc(emutls_asize(new_size));
-        if (array)
+        if (array) {
             memset(array->data, 0, new_size * sizeof(void*));
+            array->skip_destructor_rounds = EMUTLS_SKIP_DESTRUCTOR_ROUNDS;
+        }
         emutls_check_array_set_size(array, new_size);
     } else if (index > array->size) {
         uintptr_t orig_size = array->size;