[compiler-rt] 8a300de - [sanitizer] Make DTLS_on_tls_get_addr signal safer
Vitaly Buka via llvm-commits
llvm-commits at lists.llvm.org
Tue Dec 1 16:16:15 PST 2020
Author: Vitaly Buka
Date: 2020-12-01T16:16:04-08:00
New Revision: 8a300deb3e4602eee7745db5281b5ac728e44971
URL: https://github.com/llvm/llvm-project/commit/8a300deb3e4602eee7745db5281b5ac728e44971
DIFF: https://github.com/llvm/llvm-project/commit/8a300deb3e4602eee7745db5281b5ac728e44971.diff
LOG: [sanitizer] Make DTLS_on_tls_get_addr signal safer
Avoid relocating DTV table and use linked list of mmap-ed pages.
Reviewed By: eugenis
Differential Revision: https://reviews.llvm.org/D92428
Added:
compiler-rt/test/sanitizer_common/TestCases/Linux/resize_tls_dynamic.cpp
Modified:
compiler-rt/lib/sanitizer_common/sanitizer_tls_get_addr.cpp
compiler-rt/lib/sanitizer_common/sanitizer_tls_get_addr.h
Removed:
################################################################################
diff --git a/compiler-rt/lib/sanitizer_common/sanitizer_tls_get_addr.cpp b/compiler-rt/lib/sanitizer_common/sanitizer_tls_get_addr.cpp
index 10748f96420a..1f664b6cf5b8 100644
--- a/compiler-rt/lib/sanitizer_common/sanitizer_tls_get_addr.cpp
+++ b/compiler-rt/lib/sanitizer_common/sanitizer_tls_get_addr.cpp
@@ -12,6 +12,7 @@
#include "sanitizer_tls_get_addr.h"
+#include "sanitizer_atomic.h"
#include "sanitizer_flags.h"
#include "sanitizer_platform_interceptors.h"
@@ -42,39 +43,54 @@ static atomic_uintptr_t number_of_live_dtls;
static const uptr kDestroyedThread = -1;
-static inline void DTLS_Deallocate(DTLS::DTV *dtv, uptr size) {
- if (!size) return;
- VReport(2, "__tls_get_addr: DTLS_Deallocate %p %zd\n", dtv, size);
- UnmapOrDie(dtv, size * sizeof(DTLS::DTV));
+static void DTLS_Deallocate(DTLS::DTVBlock *block) {
+ VReport(2, "__tls_get_addr: DTLS_Deallocate %p %zd\n", block);
+ UnmapOrDie(block, sizeof(DTLS::DTVBlock));
atomic_fetch_sub(&number_of_live_dtls, 1, memory_order_relaxed);
}
-static inline void DTLS_Resize(uptr new_size) {
- if (dtls.dtv_size >= new_size) return;
- new_size = RoundUpToPowerOfTwo(new_size);
- new_size = Max(new_size, 4096UL / sizeof(DTLS::DTV));
- DTLS::DTV *new_dtv =
- (DTLS::DTV *)MmapOrDie(new_size * sizeof(DTLS::DTV), "DTLS_Resize");
+static DTLS::DTVBlock *DTLS_NextBlock(atomic_uintptr_t *cur) {
+ uptr v = atomic_load(cur, memory_order_acquire);
+ if (v == kDestroyedThread)
+ return nullptr;
+ DTLS::DTVBlock *next = (DTLS::DTVBlock *)v;
+ if (next)
+ return next;
+ DTLS::DTVBlock *new_dtv =
+ (DTLS::DTVBlock *)MmapOrDie(sizeof(DTLS::DTVBlock), "DTLS_NextBlock");
+ uptr prev = 0;
+ if (!atomic_compare_exchange_strong(cur, &prev, (uptr)new_dtv,
+ memory_order_seq_cst)) {
+ UnmapOrDie(new_dtv, sizeof(DTLS::DTVBlock));
+ return (DTLS::DTVBlock *)prev;
+ }
uptr num_live_dtls =
atomic_fetch_add(&number_of_live_dtls, 1, memory_order_relaxed);
- VReport(2, "__tls_get_addr: DTLS_Resize %p %zd\n", &dtls, num_live_dtls);
- CHECK_LT(num_live_dtls, 1 << 20);
- uptr old_dtv_size = dtls.dtv_size;
- DTLS::DTV *old_dtv = dtls.dtv;
- if (old_dtv_size)
- internal_memcpy(new_dtv, dtls.dtv, dtls.dtv_size * sizeof(DTLS::DTV));
- dtls.dtv = new_dtv;
- dtls.dtv_size = new_size;
- if (old_dtv_size)
- DTLS_Deallocate(old_dtv, old_dtv_size);
+ VReport(2, "__tls_get_addr: DTLS_NextBlock %p %zd\n", &dtls, num_live_dtls);
+ return new_dtv;
+}
+
+static DTLS::DTV *DTLS_Find(uptr id) {
+ VReport(2, "__tls_get_addr: DTLS_Find %p %zd\n", &dtls, id);
+ static constexpr uptr kPerBlock = ARRAY_SIZE(DTLS::DTVBlock::dtvs);
+ DTLS::DTVBlock *cur = DTLS_NextBlock(&dtls.dtv_block);
+ if (!cur)
+ return nullptr;
+ for (; id >= kPerBlock; id -= kPerBlock) cur = DTLS_NextBlock(&cur->next);
+ return cur->dtvs + id;
}
void DTLS_Destroy() {
if (!common_flags()->intercept_tls_get_addr) return;
- VReport(2, "__tls_get_addr: DTLS_Destroy %p %zd\n", &dtls, dtls.dtv_size);
- uptr s = dtls.dtv_size;
- dtls.dtv_size = kDestroyedThread; // Do this before unmap for AS-safety.
- DTLS_Deallocate(dtls.dtv, s);
+ VReport(2, "__tls_get_addr: DTLS_Destroy %p\n", &dtls);
+ DTLS::DTVBlock *block = (DTLS::DTVBlock *)atomic_exchange(
+ &dtls.dtv_block, kDestroyedThread, memory_order_release);
+ while (block) {
+ DTLS::DTVBlock *next =
+ (DTLS::DTVBlock *)atomic_load(&block->next, memory_order_acquire);
+ DTLS_Deallocate(block);
+ block = next;
+ }
}
#if defined(__powerpc64__) || defined(__mips__)
@@ -96,9 +112,9 @@ DTLS::DTV *DTLS_on_tls_get_addr(void *arg_void, void *res,
if (!common_flags()->intercept_tls_get_addr) return 0;
TlsGetAddrParam *arg = reinterpret_cast<TlsGetAddrParam *>(arg_void);
uptr dso_id = arg->dso_id;
- if (dtls.dtv_size == kDestroyedThread) return 0;
- DTLS_Resize(dso_id + 1);
- if (dtls.dtv[dso_id].beg) return 0;
+ DTLS::DTV *dtv = DTLS_Find(dso_id);
+ if (!dtv || dtv->beg)
+ return 0;
uptr tls_size = 0;
uptr tls_beg = reinterpret_cast<uptr>(res) - arg->offset - kDtvOffset;
VReport(2, "__tls_get_addr: %p {%p,%p} => %p; tls_beg: %p; sp: %p "
@@ -126,9 +142,9 @@ DTLS::DTV *DTLS_on_tls_get_addr(void *arg_void, void *res,
// This may happen inside the DTOR of main thread, so just ignore it.
tls_size = 0;
}
- dtls.dtv[dso_id].beg = tls_beg;
- dtls.dtv[dso_id].size = tls_size;
- return dtls.dtv + dso_id;
+ dtv->beg = tls_beg;
+ dtv->size = tls_size;
+ return dtv;
}
void DTLS_on_libc_memalign(void *ptr, uptr size) {
@@ -141,7 +157,8 @@ void DTLS_on_libc_memalign(void *ptr, uptr size) {
DTLS *DTLS_Get() { return &dtls; }
bool DTLSInDestruction(DTLS *dtls) {
- return dtls->dtv_size == kDestroyedThread;
+ return atomic_load(&dtls->dtv_block, memory_order_relaxed) ==
+ kDestroyedThread;
}
#else
diff --git a/compiler-rt/lib/sanitizer_common/sanitizer_tls_get_addr.h b/compiler-rt/lib/sanitizer_common/sanitizer_tls_get_addr.h
index cfdef4a2cd4b..a599c0bbc75c 100644
--- a/compiler-rt/lib/sanitizer_common/sanitizer_tls_get_addr.h
+++ b/compiler-rt/lib/sanitizer_common/sanitizer_tls_get_addr.h
@@ -28,6 +28,7 @@
#ifndef SANITIZER_TLS_GET_ADDR_H
#define SANITIZER_TLS_GET_ADDR_H
+#include "sanitizer_atomic.h"
#include "sanitizer_common.h"
namespace __sanitizer {
@@ -38,9 +39,14 @@ struct DTLS {
struct DTV {
uptr beg, size;
};
+ struct DTVBlock {
+ atomic_uintptr_t next;
+ DTV dtvs[(4096UL - sizeof(next)) / sizeof(DTLS::DTV)];
+ };
+
+ static_assert(sizeof(DTVBlock) <= 4096UL, "Unexpected block size");
- uptr dtv_size;
- DTV *dtv; // dtv_size elements, allocated by MmapOrDie.
+ atomic_uintptr_t dtv_block;
// Auxiliary fields, don't access them outside sanitizer_tls_get_addr.cpp
uptr last_memalign_size;
@@ -49,7 +55,13 @@ struct DTLS {
template <typename Fn>
void ForEachDVT(DTLS *dtls, const Fn &fn) {
- for (uptr j = 0; j < dtls->dtv_size; ++j) fn(dtls->dtv[j], j);
+ DTLS::DTVBlock *block =
+ (DTLS::DTVBlock *)atomic_load(&dtls->dtv_block, memory_order_acquire);
+ while (block) {
+ int id = 0;
+ for (auto &d : block->dtvs) fn(d, id++);
+ block = (DTLS::DTVBlock *)atomic_load(&block->next, memory_order_acquire);
+ }
}
// Returns pointer and size of a linker-allocated TLS block.
diff --git a/compiler-rt/test/sanitizer_common/TestCases/Linux/resize_tls_dynamic.cpp b/compiler-rt/test/sanitizer_common/TestCases/Linux/resize_tls_dynamic.cpp
new file mode 100644
index 000000000000..8330384a305a
--- /dev/null
+++ b/compiler-rt/test/sanitizer_common/TestCases/Linux/resize_tls_dynamic.cpp
@@ -0,0 +1,55 @@
+// RUN: %clangxx %s -DBUILD_DSO -fPIC -shared -o %t.so
+// RUN: %clangxx --std=c++11 %s -o %t
+// RUN: %env_tool_opts=verbosity=2 %run %t 2>&1 | FileCheck %s
+
+// Does not call __tls_get_addr
+// UNSUPPORTED: i386-linux
+
+// Do not intercept __tls_get_addr
+// UNSUPPORTED: lsan, ubsan, android
+
+#include <string.h>
+
+#ifndef BUILD_DSO
+
+#include <assert.h>
+#include <dlfcn.h>
+#include <stdio.h>
+#include <stdlib.h>
+
+char buff[10000];
+
+int main(int argc, char *argv[]) {
+ sprintf(buff, "rm -f %s.so.*", argv[0]);
+ system(buff);
+
+ void *prev_handle = 0;
+ for (int i = 0; i < 300; ++i) {
+ sprintf(buff, "cp %s.so %s.so.%d", argv[0], argv[0], i);
+ system(buff);
+
+ sprintf(buff, "%s.so.%d", argv[0], i);
+ void *handle = dlopen(buff, RTLD_LAZY);
+ assert(handle != 0);
+ assert(handle != prev_handle);
+ prev_handle = handle;
+
+ typedef void (*FnType)(char c);
+ ((FnType)dlsym(handle, "StoreToTLS"))(i);
+ }
+ return 0;
+}
+
+#else // BUILD_DSO
+__thread char huge_thread_local_array[1 << 12];
+
+extern "C" void StoreToTLS(char c) {
+ memset(huge_thread_local_array, c, sizeof(huge_thread_local_array));
+}
+#endif // BUILD_DSO
+
+// CHECK: DTLS_Find [[DTLS:0x[a-f0-9]+]] {{[0-9]+}}
+// CHECK-NEXT: DTLS_NextBlock [[DTLS]] 0
+// CHECK: DTLS_Find [[DTLS:0x[a-f0-9]+]] 255
+// CHECK-NEXT: DTLS_NextBlock [[DTLS]] 1
+// CHECK-NOT: DTLS_NextBlock
\ No newline at end of file
More information about the llvm-commits
mailing list