[compiler-rt] cb0e14c - [sanitizer] Switch dlsym hack to internal_allocator
Vitaly Buka via llvm-commits
llvm-commits at lists.llvm.org
Fri Nov 12 16:11:21 PST 2021
Author: Vitaly Buka
Date: 2021-11-12T16:11:10-08:00
New Revision: cb0e14ce6dcdd614a7207f4ce6fcf81a164471ab
URL: https://github.com/llvm/llvm-project/commit/cb0e14ce6dcdd614a7207f4ce6fcf81a164471ab
DIFF: https://github.com/llvm/llvm-project/commit/cb0e14ce6dcdd614a7207f4ce6fcf81a164471ab.diff
LOG: [sanitizer] Switch dlsym hack to internal_allocator
Since glibc 2.34, dlsym does
1. malloc 1
2. malloc 2
3. free pointer from malloc 1
4. free pointer from malloc 2
These sequence was not handled by trivial dlsym hack.
This fixes https://bugs.llvm.org/show_bug.cgi?id=52278
Reviewed By: eugenis, morehouse
Differential Revision: https://reviews.llvm.org/D112588
Added:
compiler-rt/lib/sanitizer_common/sanitizer_allocator_dlsym.h
Modified:
compiler-rt/lib/asan/asan_malloc_linux.cpp
compiler-rt/lib/dfsan/dfsan_interceptors.cpp
compiler-rt/lib/hwasan/hwasan_allocation_functions.cpp
compiler-rt/lib/lsan/lsan_common.h
compiler-rt/lib/lsan/lsan_interceptors.cpp
compiler-rt/lib/memprof/memprof_malloc_linux.cpp
compiler-rt/lib/msan/msan_interceptors.cpp
compiler-rt/lib/sanitizer_common/CMakeLists.txt
compiler-rt/lib/sanitizer_common/sanitizer_malloc_mac.inc
Removed:
################################################################################
diff --git a/compiler-rt/lib/asan/asan_malloc_linux.cpp b/compiler-rt/lib/asan/asan_malloc_linux.cpp
index 362ca0275fd2..bab80b96f584 100644
--- a/compiler-rt/lib/asan/asan_malloc_linux.cpp
+++ b/compiler-rt/lib/asan/asan_malloc_linux.cpp
@@ -21,108 +21,66 @@
# include "asan_interceptors.h"
# include "asan_internal.h"
# include "asan_stack.h"
+# include "lsan/lsan_common.h"
# include "sanitizer_common/sanitizer_allocator_checks.h"
+# include "sanitizer_common/sanitizer_allocator_dlsym.h"
# include "sanitizer_common/sanitizer_errno.h"
# include "sanitizer_common/sanitizer_tls_get_addr.h"
// ---------------------- Replacement functions ---------------- {{{1
using namespace __asan;
-static uptr allocated_for_dlsym;
-static uptr last_dlsym_alloc_size_in_words;
-static const uptr kDlsymAllocPoolSize = 1024;
-static uptr alloc_memory_for_dlsym[kDlsymAllocPoolSize];
-
-static inline bool IsInDlsymAllocPool(const void *ptr) {
- uptr off = (uptr)ptr - (uptr)alloc_memory_for_dlsym;
- return off < allocated_for_dlsym * sizeof(alloc_memory_for_dlsym[0]);
-}
-
-static void *AllocateFromLocalPool(uptr size_in_bytes) {
- uptr size_in_words = RoundUpTo(size_in_bytes, kWordSize) / kWordSize;
- void *mem = (void*)&alloc_memory_for_dlsym[allocated_for_dlsym];
- last_dlsym_alloc_size_in_words = size_in_words;
- allocated_for_dlsym += size_in_words;
- CHECK_LT(allocated_for_dlsym, kDlsymAllocPoolSize);
- return mem;
-}
-
-static void DeallocateFromLocalPool(const void *ptr) {
- // Hack: since glibc 2.27 dlsym no longer uses stack-allocated memory to store
- // error messages and instead uses malloc followed by free. To avoid pool
- // exhaustion due to long object filenames, handle that special case here.
- uptr prev_offset = allocated_for_dlsym - last_dlsym_alloc_size_in_words;
- void *prev_mem = (void*)&alloc_memory_for_dlsym[prev_offset];
- if (prev_mem == ptr) {
- REAL(memset)(prev_mem, 0, last_dlsym_alloc_size_in_words * kWordSize);
- allocated_for_dlsym = prev_offset;
- last_dlsym_alloc_size_in_words = 0;
+struct DlsymAlloc : public DlSymAllocator<DlsymAlloc> {
+ static bool UseImpl() { return asan_init_is_running; }
+ static void OnAllocate(const void *ptr, uptr size) {
+# if CAN_SANITIZE_LEAKS
+ // Suppress leaks from dlerror(). Previously dlsym hack on global array was
+ // used by leak sanitizer as a root region.
+ __lsan_register_root_region(ptr, size);
+# endif
}
-}
-
-static inline bool MaybeInDlsym() {
- // Fuchsia doesn't use dlsym-based interceptors.
- return !SANITIZER_FUCHSIA && asan_init_is_running;
-}
-
-static inline bool UseLocalPool() { return MaybeInDlsym(); }
-
-static void *ReallocFromLocalPool(void *ptr, uptr size) {
- const uptr offset = (uptr)ptr - (uptr)alloc_memory_for_dlsym;
- const uptr copy_size = Min(size, kDlsymAllocPoolSize - offset);
- void *new_ptr;
- if (UNLIKELY(UseLocalPool())) {
- new_ptr = AllocateFromLocalPool(size);
- } else {
- ENSURE_ASAN_INITED();
- GET_STACK_TRACE_MALLOC;
- new_ptr = asan_malloc(size, &stack);
+ static void OnFree(const void *ptr, uptr size) {
+# if CAN_SANITIZE_LEAKS
+ __lsan_unregister_root_region(ptr, size);
+# endif
}
- internal_memcpy(new_ptr, ptr, copy_size);
- return new_ptr;
-}
+};
INTERCEPTOR(void, free, void *ptr) {
- if (UNLIKELY(IsInDlsymAllocPool(ptr))) {
- DeallocateFromLocalPool(ptr);
- return;
- }
+ if (DlsymAlloc::PointerIsMine(ptr))
+ return DlsymAlloc::Free(ptr);
GET_STACK_TRACE_FREE;
asan_free(ptr, &stack, FROM_MALLOC);
}
#if SANITIZER_INTERCEPT_CFREE
INTERCEPTOR(void, cfree, void *ptr) {
- if (UNLIKELY(IsInDlsymAllocPool(ptr)))
- return;
+ if (DlsymAlloc::PointerIsMine(ptr))
+ return DlsymAlloc::Free(ptr);
GET_STACK_TRACE_FREE;
asan_free(ptr, &stack, FROM_MALLOC);
}
#endif // SANITIZER_INTERCEPT_CFREE
INTERCEPTOR(void*, malloc, uptr size) {
- if (UNLIKELY(UseLocalPool()))
- // Hack: dlsym calls malloc before REAL(malloc) is retrieved from dlsym.
- return AllocateFromLocalPool(size);
+ if (DlsymAlloc::Use())
+ return DlsymAlloc::Allocate(size);
ENSURE_ASAN_INITED();
GET_STACK_TRACE_MALLOC;
return asan_malloc(size, &stack);
}
INTERCEPTOR(void*, calloc, uptr nmemb, uptr size) {
- if (UNLIKELY(UseLocalPool()))
- // Hack: dlsym calls calloc before REAL(calloc) is retrieved from dlsym.
- return AllocateFromLocalPool(nmemb * size);
+ if (DlsymAlloc::Use())
+ return DlsymAlloc::Callocate(nmemb, size);
ENSURE_ASAN_INITED();
GET_STACK_TRACE_MALLOC;
return asan_calloc(nmemb, size, &stack);
}
INTERCEPTOR(void*, realloc, void *ptr, uptr size) {
- if (UNLIKELY(IsInDlsymAllocPool(ptr)))
- return ReallocFromLocalPool(ptr, size);
- if (UNLIKELY(UseLocalPool()))
- return AllocateFromLocalPool(size);
+ if (DlsymAlloc::Use() || DlsymAlloc::PointerIsMine(ptr))
+ return DlsymAlloc::Realloc(ptr, size);
ENSURE_ASAN_INITED();
GET_STACK_TRACE_MALLOC;
return asan_realloc(ptr, size, &stack);
diff --git a/compiler-rt/lib/dfsan/dfsan_interceptors.cpp b/compiler-rt/lib/dfsan/dfsan_interceptors.cpp
index d6b114a7acb5..d8fb9ea86618 100644
--- a/compiler-rt/lib/dfsan/dfsan_interceptors.cpp
+++ b/compiler-rt/lib/dfsan/dfsan_interceptors.cpp
@@ -17,6 +17,7 @@
#include "dfsan/dfsan.h"
#include "dfsan/dfsan_thread.h"
#include "interception/interception.h"
+#include "sanitizer_common/sanitizer_allocator_dlsym.h"
#include "sanitizer_common/sanitizer_allocator_interface.h"
#include "sanitizer_common/sanitizer_common.h"
#include "sanitizer_common/sanitizer_errno.h"
@@ -26,11 +27,11 @@
using namespace __sanitizer;
-namespace {
+static bool interceptors_initialized;
-bool interceptors_initialized;
-
-} // namespace
+struct DlsymAlloc : public DlSymAllocator<DlsymAlloc> {
+ static bool UseImpl() { return !__dfsan::dfsan_inited; }
+};
INTERCEPTOR(void *, reallocarray, void *ptr, SIZE_T nmemb, SIZE_T size) {
return __dfsan::dfsan_reallocarray(ptr, nmemb, size);
@@ -47,63 +48,37 @@ INTERCEPTOR(void *, aligned_alloc, SIZE_T alignment, SIZE_T size) {
return __dfsan::dfsan_aligned_alloc(alignment, size);
}
-static uptr allocated_for_dlsym;
-static const uptr kDlsymAllocPoolSize = 1024;
-static uptr alloc_memory_for_dlsym[kDlsymAllocPoolSize];
-
-static bool IsInDlsymAllocPool(const void *ptr) {
- uptr off = (uptr)ptr - (uptr)alloc_memory_for_dlsym;
- return off < sizeof(alloc_memory_for_dlsym);
-}
-
-static void *AllocateFromLocalPool(uptr size_in_bytes) {
- uptr size_in_words = RoundUpTo(size_in_bytes, kWordSize) / kWordSize;
- void *mem = (void *)&alloc_memory_for_dlsym[allocated_for_dlsym];
- allocated_for_dlsym += size_in_words;
- CHECK_LT(allocated_for_dlsym, kDlsymAllocPoolSize);
- return mem;
-}
-
INTERCEPTOR(void *, calloc, SIZE_T nmemb, SIZE_T size) {
- if (UNLIKELY(!__dfsan::dfsan_inited))
- // Hack: dlsym calls calloc before REAL(calloc) is retrieved from dlsym.
- return AllocateFromLocalPool(nmemb * size);
+ if (DlsymAlloc::Use())
+ return DlsymAlloc::Callocate(nmemb, size);
return __dfsan::dfsan_calloc(nmemb, size);
}
INTERCEPTOR(void *, realloc, void *ptr, SIZE_T size) {
- if (UNLIKELY(IsInDlsymAllocPool(ptr))) {
- uptr offset = (uptr)ptr - (uptr)alloc_memory_for_dlsym;
- uptr copy_size = Min(size, kDlsymAllocPoolSize - offset);
- void *new_ptr;
- if (UNLIKELY(!__dfsan::dfsan_inited)) {
- new_ptr = AllocateFromLocalPool(copy_size);
- } else {
- copy_size = size;
- new_ptr = __dfsan::dfsan_malloc(copy_size);
- }
- internal_memcpy(new_ptr, ptr, copy_size);
- return new_ptr;
- }
+ if (DlsymAlloc::Use() || DlsymAlloc::PointerIsMine(ptr))
+ return DlsymAlloc::Realloc(ptr, size);
return __dfsan::dfsan_realloc(ptr, size);
}
INTERCEPTOR(void *, malloc, SIZE_T size) {
- if (UNLIKELY(!__dfsan::dfsan_inited))
- // Hack: dlsym calls malloc before REAL(malloc) is retrieved from dlsym.
- return AllocateFromLocalPool(size);
+ if (DlsymAlloc::Use())
+ return DlsymAlloc::Allocate(size);
return __dfsan::dfsan_malloc(size);
}
INTERCEPTOR(void, free, void *ptr) {
- if (!ptr || UNLIKELY(IsInDlsymAllocPool(ptr)))
+ if (!ptr)
return;
+ if (DlsymAlloc::PointerIsMine(ptr))
+ return DlsymAlloc::Free(ptr);
return __dfsan::dfsan_deallocate(ptr);
}
INTERCEPTOR(void, cfree, void *ptr) {
- if (!ptr || UNLIKELY(IsInDlsymAllocPool(ptr)))
+ if (!ptr)
return;
+ if (DlsymAlloc::PointerIsMine(ptr))
+ return DlsymAlloc::Free(ptr);
return __dfsan::dfsan_deallocate(ptr);
}
diff --git a/compiler-rt/lib/hwasan/hwasan_allocation_functions.cpp b/compiler-rt/lib/hwasan/hwasan_allocation_functions.cpp
index aecc892b7455..9cd82dbabd19 100644
--- a/compiler-rt/lib/hwasan/hwasan_allocation_functions.cpp
+++ b/compiler-rt/lib/hwasan/hwasan_allocation_functions.cpp
@@ -14,6 +14,7 @@
#include "hwasan.h"
#include "interception/interception.h"
+#include "sanitizer_common/sanitizer_allocator_dlsym.h"
#include "sanitizer_common/sanitizer_allocator_interface.h"
#include "sanitizer_common/sanitizer_tls_get_addr.h"
@@ -21,22 +22,9 @@
using namespace __hwasan;
-static uptr allocated_for_dlsym;
-static const uptr kDlsymAllocPoolSize = 1024;
-static uptr alloc_memory_for_dlsym[kDlsymAllocPoolSize];
-
-static bool IsInDlsymAllocPool(const void *ptr) {
- uptr off = (uptr)ptr - (uptr)alloc_memory_for_dlsym;
- return off < sizeof(alloc_memory_for_dlsym);
-}
-
-static void *AllocateFromLocalPool(uptr size_in_bytes) {
- uptr size_in_words = RoundUpTo(size_in_bytes, kWordSize) / kWordSize;
- void *mem = (void *)&alloc_memory_for_dlsym[allocated_for_dlsym];
- allocated_for_dlsym += size_in_words;
- CHECK_LT(allocated_for_dlsym, kDlsymAllocPoolSize);
- return mem;
-}
+struct DlsymAlloc : public DlSymAllocator<DlsymAlloc> {
+ static bool UseImpl() { return !hwasan_inited; }
+};
extern "C" {
@@ -83,16 +71,20 @@ void *__sanitizer_pvalloc(uptr size) {
SANITIZER_INTERFACE_ATTRIBUTE
void __sanitizer_free(void *ptr) {
- if (!ptr || UNLIKELY(IsInDlsymAllocPool(ptr)))
+ if (!ptr)
return;
+ if (DlsymAlloc::PointerIsMine(ptr))
+ return DlsymAlloc::Free(ptr);
GET_MALLOC_STACK_TRACE;
hwasan_free(ptr, &stack);
}
SANITIZER_INTERFACE_ATTRIBUTE
void __sanitizer_cfree(void *ptr) {
- if (!ptr || UNLIKELY(IsInDlsymAllocPool(ptr)))
+ if (!ptr)
return;
+ if (DlsymAlloc::PointerIsMine(ptr))
+ return DlsymAlloc::Free(ptr);
GET_MALLOC_STACK_TRACE;
hwasan_free(ptr, &stack);
}
@@ -119,29 +111,16 @@ void __sanitizer_malloc_stats(void) {
SANITIZER_INTERFACE_ATTRIBUTE
void *__sanitizer_calloc(uptr nmemb, uptr size) {
- if (UNLIKELY(!hwasan_inited))
- // Hack: dlsym calls calloc before REAL(calloc) is retrieved from dlsym.
- return AllocateFromLocalPool(nmemb * size);
+ if (DlsymAlloc::Use())
+ return DlsymAlloc::Callocate(nmemb, size);
GET_MALLOC_STACK_TRACE;
return hwasan_calloc(nmemb, size, &stack);
}
SANITIZER_INTERFACE_ATTRIBUTE
void *__sanitizer_realloc(void *ptr, uptr size) {
- if (UNLIKELY(IsInDlsymAllocPool(ptr))) {
- uptr offset = (uptr)ptr - (uptr)alloc_memory_for_dlsym;
- uptr copy_size = Min(size, kDlsymAllocPoolSize - offset);
- void *new_ptr;
- if (UNLIKELY(!hwasan_inited)) {
- new_ptr = AllocateFromLocalPool(copy_size);
- } else {
- copy_size = size;
- GET_MALLOC_STACK_TRACE;
- new_ptr = hwasan_malloc(copy_size, &stack);
- }
- internal_memcpy(new_ptr, ptr, copy_size);
- return new_ptr;
- }
+ if (DlsymAlloc::Use() || DlsymAlloc::PointerIsMine(ptr))
+ return DlsymAlloc::Realloc(ptr, size);
GET_MALLOC_STACK_TRACE;
return hwasan_realloc(ptr, size, &stack);
}
@@ -156,9 +135,8 @@ SANITIZER_INTERFACE_ATTRIBUTE
void *__sanitizer_malloc(uptr size) {
if (UNLIKELY(!hwasan_init_is_running))
ENSURE_HWASAN_INITED();
- if (UNLIKELY(!hwasan_inited))
- // Hack: dlsym calls malloc before REAL(malloc) is retrieved from dlsym.
- return AllocateFromLocalPool(size);
+ if (DlsymAlloc::Use())
+ return DlsymAlloc::Allocate(size);
GET_MALLOC_STACK_TRACE;
return hwasan_malloc(size, &stack);
}
diff --git a/compiler-rt/lib/lsan/lsan_common.h b/compiler-rt/lib/lsan/lsan_common.h
index 1f828fac67ea..f9b55e4e8006 100644
--- a/compiler-rt/lib/lsan/lsan_common.h
+++ b/compiler-rt/lib/lsan/lsan_common.h
@@ -280,6 +280,13 @@ int __lsan_is_turned_off();
SANITIZER_INTERFACE_ATTRIBUTE SANITIZER_WEAK_ATTRIBUTE
const char *__lsan_default_suppressions();
+
+SANITIZER_INTERFACE_ATTRIBUTE
+void __lsan_register_root_region(const void *p, __lsan::uptr size);
+
+SANITIZER_INTERFACE_ATTRIBUTE
+void __lsan_unregister_root_region(const void *p, __lsan::uptr size);
+
} // extern "C"
#endif // LSAN_COMMON_H
diff --git a/compiler-rt/lib/lsan/lsan_interceptors.cpp b/compiler-rt/lib/lsan/lsan_interceptors.cpp
index 90a90a56c54c..22999d567f62 100644
--- a/compiler-rt/lib/lsan/lsan_interceptors.cpp
+++ b/compiler-rt/lib/lsan/lsan_interceptors.cpp
@@ -13,6 +13,7 @@
#include "interception/interception.h"
#include "sanitizer_common/sanitizer_allocator.h"
+#include "sanitizer_common/sanitizer_allocator_dlsym.h"
#include "sanitizer_common/sanitizer_allocator_report.h"
#include "sanitizer_common/sanitizer_atomic.h"
#include "sanitizer_common/sanitizer_common.h"
@@ -43,6 +44,22 @@ int pthread_key_create(unsigned *key, void (*destructor)(void* v));
int pthread_setspecific(unsigned key, const void *v);
}
+struct DlsymAlloc : DlSymAllocator<DlsymAlloc> {
+ static bool UseImpl() { return lsan_init_is_running; }
+ static void OnAllocate(const void *ptr, uptr size) {
+#if CAN_SANITIZE_LEAKS
+ // Suppress leaks from dlerror(). Previously dlsym hack on global array was
+ // used by leak sanitizer as a root region.
+ __lsan_register_root_region(ptr, size);
+#endif
+ }
+ static void OnFree(const void *ptr, uptr size) {
+#if CAN_SANITIZE_LEAKS
+ __lsan_unregister_root_region(ptr, size);
+#endif
+ }
+};
+
///// Malloc/free interceptors. /////
namespace std {
@@ -52,41 +69,34 @@ namespace std {
#if !SANITIZER_MAC
INTERCEPTOR(void*, malloc, uptr size) {
+ if (DlsymAlloc::Use())
+ return DlsymAlloc::Allocate(size);
ENSURE_LSAN_INITED;
GET_STACK_TRACE_MALLOC;
return lsan_malloc(size, stack);
}
INTERCEPTOR(void, free, void *p) {
+ if (DlsymAlloc::PointerIsMine(p))
+ return DlsymAlloc::Free(p);
ENSURE_LSAN_INITED;
lsan_free(p);
}
INTERCEPTOR(void*, calloc, uptr nmemb, uptr size) {
- // This hack is not required for Fuchsia because there are no dlsym calls
- // involved in setting up interceptors.
-#if !SANITIZER_FUCHSIA
- if (lsan_init_is_running) {
- // Hack: dlsym calls calloc before REAL(calloc) is retrieved from dlsym.
- const uptr kCallocPoolSize = 1024;
- static uptr calloc_memory_for_dlsym[kCallocPoolSize];
- static uptr allocated;
- uptr size_in_words = ((nmemb * size) + kWordSize - 1) / kWordSize;
- void *mem = (void*)&calloc_memory_for_dlsym[allocated];
- allocated += size_in_words;
- CHECK(allocated < kCallocPoolSize);
- return mem;
- }
-#endif // !SANITIZER_FUCHSIA
+ if (DlsymAlloc::Use())
+ return DlsymAlloc::Callocate(nmemb, size);
ENSURE_LSAN_INITED;
GET_STACK_TRACE_MALLOC;
return lsan_calloc(nmemb, size, stack);
}
-INTERCEPTOR(void*, realloc, void *q, uptr size) {
+INTERCEPTOR(void *, realloc, void *ptr, uptr size) {
+ if (DlsymAlloc::Use() || DlsymAlloc::PointerIsMine(ptr))
+ return DlsymAlloc::Realloc(ptr, size);
ENSURE_LSAN_INITED;
GET_STACK_TRACE_MALLOC;
- return lsan_realloc(q, size, stack);
+ return lsan_realloc(ptr, size, stack);
}
INTERCEPTOR(void*, reallocarray, void *q, uptr nmemb, uptr size) {
diff --git a/compiler-rt/lib/memprof/memprof_malloc_linux.cpp b/compiler-rt/lib/memprof/memprof_malloc_linux.cpp
index 155ce0abde5e..ef753fcaa4ad 100644
--- a/compiler-rt/lib/memprof/memprof_malloc_linux.cpp
+++ b/compiler-rt/lib/memprof/memprof_malloc_linux.cpp
@@ -23,104 +23,52 @@
#include "memprof_internal.h"
#include "memprof_stack.h"
#include "sanitizer_common/sanitizer_allocator_checks.h"
+#include "sanitizer_common/sanitizer_allocator_dlsym.h"
#include "sanitizer_common/sanitizer_errno.h"
#include "sanitizer_common/sanitizer_tls_get_addr.h"
// ---------------------- Replacement functions ---------------- {{{1
using namespace __memprof;
-static uptr allocated_for_dlsym;
-static uptr last_dlsym_alloc_size_in_words;
-static const uptr kDlsymAllocPoolSize = 1024;
-static uptr alloc_memory_for_dlsym[kDlsymAllocPoolSize];
-
-static inline bool IsInDlsymAllocPool(const void *ptr) {
- uptr off = (uptr)ptr - (uptr)alloc_memory_for_dlsym;
- return off < allocated_for_dlsym * sizeof(alloc_memory_for_dlsym[0]);
-}
-
-static void *AllocateFromLocalPool(uptr size_in_bytes) {
- uptr size_in_words = RoundUpTo(size_in_bytes, kWordSize) / kWordSize;
- void *mem = (void *)&alloc_memory_for_dlsym[allocated_for_dlsym];
- last_dlsym_alloc_size_in_words = size_in_words;
- allocated_for_dlsym += size_in_words;
- CHECK_LT(allocated_for_dlsym, kDlsymAllocPoolSize);
- return mem;
-}
-
-static void DeallocateFromLocalPool(const void *ptr) {
- // Hack: since glibc 2.27 dlsym no longer uses stack-allocated memory to store
- // error messages and instead uses malloc followed by free. To avoid pool
- // exhaustion due to long object filenames, handle that special case here.
- uptr prev_offset = allocated_for_dlsym - last_dlsym_alloc_size_in_words;
- void *prev_mem = (void *)&alloc_memory_for_dlsym[prev_offset];
- if (prev_mem == ptr) {
- REAL(memset)(prev_mem, 0, last_dlsym_alloc_size_in_words * kWordSize);
- allocated_for_dlsym = prev_offset;
- last_dlsym_alloc_size_in_words = 0;
- }
-}
-
-static inline bool MaybeInDlsym() { return memprof_init_is_running; }
-
-static inline bool UseLocalPool() { return MaybeInDlsym(); }
-
-static void *ReallocFromLocalPool(void *ptr, uptr size) {
- const uptr offset = (uptr)ptr - (uptr)alloc_memory_for_dlsym;
- const uptr copy_size = Min(size, kDlsymAllocPoolSize - offset);
- void *new_ptr;
- if (UNLIKELY(UseLocalPool())) {
- new_ptr = AllocateFromLocalPool(size);
- } else {
- ENSURE_MEMPROF_INITED();
- GET_STACK_TRACE_MALLOC;
- new_ptr = memprof_malloc(size, &stack);
- }
- internal_memcpy(new_ptr, ptr, copy_size);
- return new_ptr;
-}
+struct DlsymAlloc : public DlSymAllocator<DlsymAlloc> {
+ static bool UseImpl() { return memprof_init_is_running; }
+};
INTERCEPTOR(void, free, void *ptr) {
- if (UNLIKELY(IsInDlsymAllocPool(ptr))) {
- DeallocateFromLocalPool(ptr);
- return;
- }
+ if (DlsymAlloc::PointerIsMine(ptr))
+ return DlsymAlloc::Free(ptr);
GET_STACK_TRACE_FREE;
memprof_free(ptr, &stack, FROM_MALLOC);
}
#if SANITIZER_INTERCEPT_CFREE
INTERCEPTOR(void, cfree, void *ptr) {
- if (UNLIKELY(IsInDlsymAllocPool(ptr)))
- return;
+ if (DlsymAlloc::PointerIsMine(ptr))
+ return DlsymAlloc::Free(ptr);
GET_STACK_TRACE_FREE;
memprof_free(ptr, &stack, FROM_MALLOC);
}
#endif // SANITIZER_INTERCEPT_CFREE
INTERCEPTOR(void *, malloc, uptr size) {
- if (UNLIKELY(UseLocalPool()))
- // Hack: dlsym calls malloc before REAL(malloc) is retrieved from dlsym.
- return AllocateFromLocalPool(size);
+ if (DlsymAlloc::Use())
+ return DlsymAlloc::Allocate(size);
ENSURE_MEMPROF_INITED();
GET_STACK_TRACE_MALLOC;
return memprof_malloc(size, &stack);
}
INTERCEPTOR(void *, calloc, uptr nmemb, uptr size) {
- if (UNLIKELY(UseLocalPool()))
- // Hack: dlsym calls calloc before REAL(calloc) is retrieved from dlsym.
- return AllocateFromLocalPool(nmemb * size);
+ if (DlsymAlloc::Use())
+ return DlsymAlloc::Callocate(nmemb, size);
ENSURE_MEMPROF_INITED();
GET_STACK_TRACE_MALLOC;
return memprof_calloc(nmemb, size, &stack);
}
INTERCEPTOR(void *, realloc, void *ptr, uptr size) {
- if (UNLIKELY(IsInDlsymAllocPool(ptr)))
- return ReallocFromLocalPool(ptr, size);
- if (UNLIKELY(UseLocalPool()))
- return AllocateFromLocalPool(size);
+ if (DlsymAlloc::Use() || DlsymAlloc::PointerIsMine(ptr))
+ return DlsymAlloc::Realloc(ptr, size);
ENSURE_MEMPROF_INITED();
GET_STACK_TRACE_MALLOC;
return memprof_realloc(ptr, size, &stack);
diff --git a/compiler-rt/lib/msan/msan_interceptors.cpp b/compiler-rt/lib/msan/msan_interceptors.cpp
index f55f363a8730..eaa3b3ae9404 100644
--- a/compiler-rt/lib/msan/msan_interceptors.cpp
+++ b/compiler-rt/lib/msan/msan_interceptors.cpp
@@ -22,8 +22,8 @@
#include "msan_report.h"
#include "msan_thread.h"
#include "sanitizer_common/sanitizer_allocator.h"
+#include "sanitizer_common/sanitizer_allocator_dlsym.h"
#include "sanitizer_common/sanitizer_allocator_interface.h"
-#include "sanitizer_common/sanitizer_allocator_internal.h"
#include "sanitizer_common/sanitizer_atomic.h"
#include "sanitizer_common/sanitizer_common.h"
#include "sanitizer_common/sanitizer_errno.h"
@@ -75,22 +75,9 @@ bool IsInInterceptorScope() {
return in_interceptor_scope;
}
-static uptr allocated_for_dlsym;
-static const uptr kDlsymAllocPoolSize = 1024;
-static uptr alloc_memory_for_dlsym[kDlsymAllocPoolSize];
-
-static bool IsInDlsymAllocPool(const void *ptr) {
- uptr off = (uptr)ptr - (uptr)alloc_memory_for_dlsym;
- return off < sizeof(alloc_memory_for_dlsym);
-}
-
-static void *AllocateFromLocalPool(uptr size_in_bytes) {
- uptr size_in_words = RoundUpTo(size_in_bytes, kWordSize) / kWordSize;
- void *mem = (void *)&alloc_memory_for_dlsym[allocated_for_dlsym];
- allocated_for_dlsym += size_in_words;
- CHECK_LT(allocated_for_dlsym, kDlsymAllocPoolSize);
- return mem;
-}
+struct DlsymAlloc : public DlSymAllocator<DlsymAlloc> {
+ static bool UseImpl() { return !msan_inited; }
+};
#define ENSURE_MSAN_INITED() do { \
CHECK(!msan_init_is_running); \
@@ -221,14 +208,20 @@ INTERCEPTOR(void *, pvalloc, SIZE_T size) {
#endif
INTERCEPTOR(void, free, void *ptr) {
- if (!ptr || UNLIKELY(IsInDlsymAllocPool(ptr))) return;
+ if (UNLIKELY(!ptr))
+ return;
+ if (DlsymAlloc::PointerIsMine(ptr))
+ return DlsymAlloc::Free(ptr);
GET_MALLOC_STACK_TRACE;
MsanDeallocate(&stack, ptr);
}
#if !SANITIZER_FREEBSD && !SANITIZER_NETBSD
INTERCEPTOR(void, cfree, void *ptr) {
- if (!ptr || UNLIKELY(IsInDlsymAllocPool(ptr))) return;
+ if (UNLIKELY(!ptr))
+ return;
+ if (DlsymAlloc::PointerIsMine(ptr))
+ return DlsymAlloc::Free(ptr);
GET_MALLOC_STACK_TRACE;
MsanDeallocate(&stack, ptr);
}
@@ -879,27 +872,14 @@ INTERCEPTOR(int, epoll_pwait, int epfd, void *events, int maxevents,
INTERCEPTOR(void *, calloc, SIZE_T nmemb, SIZE_T size) {
GET_MALLOC_STACK_TRACE;
- if (UNLIKELY(!msan_inited))
- // Hack: dlsym calls calloc before REAL(calloc) is retrieved from dlsym.
- return AllocateFromLocalPool(nmemb * size);
+ if (DlsymAlloc::Use())
+ return DlsymAlloc::Callocate(nmemb, size);
return msan_calloc(nmemb, size, &stack);
}
INTERCEPTOR(void *, realloc, void *ptr, SIZE_T size) {
- if (UNLIKELY(IsInDlsymAllocPool(ptr))) {
- uptr offset = (uptr)ptr - (uptr)alloc_memory_for_dlsym;
- uptr copy_size = Min(size, kDlsymAllocPoolSize - offset);
- void *new_ptr;
- if (UNLIKELY(!msan_inited)) {
- new_ptr = AllocateFromLocalPool(copy_size);
- } else {
- copy_size = size;
- GET_MALLOC_STACK_TRACE;
- new_ptr = msan_malloc(copy_size, &stack);
- }
- internal_memcpy(new_ptr, ptr, copy_size);
- return new_ptr;
- }
+ if (DlsymAlloc::Use() || DlsymAlloc::PointerIsMine(ptr))
+ return DlsymAlloc::Realloc(ptr, size);
GET_MALLOC_STACK_TRACE;
return msan_realloc(ptr, size, &stack);
}
@@ -910,9 +890,8 @@ INTERCEPTOR(void *, reallocarray, void *ptr, SIZE_T nmemb, SIZE_T size) {
}
INTERCEPTOR(void *, malloc, SIZE_T size) {
- if (UNLIKELY(!msan_inited))
- // Hack: dlsym calls malloc before REAL(malloc) is retrieved from dlsym.
- return AllocateFromLocalPool(size);
+ if (DlsymAlloc::Use())
+ return DlsymAlloc::Allocate(size);
GET_MALLOC_STACK_TRACE;
return msan_malloc(size, &stack);
}
diff --git a/compiler-rt/lib/sanitizer_common/CMakeLists.txt b/compiler-rt/lib/sanitizer_common/CMakeLists.txt
index bd830c2f3668..9320a834ef7a 100644
--- a/compiler-rt/lib/sanitizer_common/CMakeLists.txt
+++ b/compiler-rt/lib/sanitizer_common/CMakeLists.txt
@@ -100,6 +100,7 @@ set(SANITIZER_IMPL_HEADERS
sanitizer_allocator.h
sanitizer_allocator_checks.h
sanitizer_allocator_combined.h
+ sanitizer_allocator_dlsym.h
sanitizer_allocator_interface.h
sanitizer_allocator_internal.h
sanitizer_allocator_local_cache.h
diff --git a/compiler-rt/lib/sanitizer_common/sanitizer_allocator_dlsym.h b/compiler-rt/lib/sanitizer_common/sanitizer_allocator_dlsym.h
new file mode 100644
index 000000000000..92b1373ef84d
--- /dev/null
+++ b/compiler-rt/lib/sanitizer_common/sanitizer_allocator_dlsym.h
@@ -0,0 +1,79 @@
+//===-- sanitizer_allocator_dlsym.h -----------------------------*- C++ -*-===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+//
+// Hack: Sanitizer initializer calls dlsym which may need to allocate and call
+// back into uninitialized sanitizer.
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef SANITIZER_ALLOCATOR_DLSYM_H
+#define SANITIZER_ALLOCATOR_DLSYM_H
+
+#include "sanitizer_allocator_internal.h"
+
+namespace __sanitizer {
+
+template <typename Details>
+struct DlSymAllocator {
+ static bool Use() {
+ // Fuchsia doesn't use dlsym-based interceptors.
+ return !SANITIZER_FUCHSIA && UNLIKELY(Details::UseImpl());
+ }
+
+ static bool PointerIsMine(const void *ptr) {
+ // Fuchsia doesn't use dlsym-based interceptors.
+ return !SANITIZER_FUCHSIA &&
+ UNLIKELY(internal_allocator()->FromPrimary(ptr));
+ }
+
+ static void *Allocate(uptr size_in_bytes) {
+ void *ptr = InternalAlloc(size_in_bytes, nullptr, kWordSize);
+ CHECK(internal_allocator()->FromPrimary(ptr));
+ Details::OnAllocate(ptr,
+ internal_allocator()->GetActuallyAllocatedSize(ptr));
+ return ptr;
+ }
+
+ static void *Callocate(SIZE_T nmemb, SIZE_T size) {
+ void *ptr = InternalCalloc(nmemb, size);
+ CHECK(internal_allocator()->FromPrimary(ptr));
+ Details::OnAllocate(ptr,
+ internal_allocator()->GetActuallyAllocatedSize(ptr));
+ return ptr;
+ }
+
+ static void Free(void *ptr) {
+ uptr size = internal_allocator()->GetActuallyAllocatedSize(ptr);
+ Details::OnFree(ptr, size);
+ InternalFree(ptr);
+ }
+
+ static void *Realloc(void *ptr, uptr new_size) {
+ if (!ptr)
+ return Allocate(new_size);
+ CHECK(internal_allocator()->FromPrimary(ptr));
+ if (!new_size) {
+ Free(ptr);
+ return nullptr;
+ }
+ uptr size = internal_allocator()->GetActuallyAllocatedSize(ptr);
+ uptr memcpy_size = Min(new_size, size);
+ void *new_ptr = Allocate(new_size);
+ if (new_ptr)
+ internal_memcpy(new_ptr, ptr, memcpy_size);
+ Free(ptr);
+ return new_ptr;
+ }
+
+ static void OnAllocate(const void *ptr, uptr size) {}
+ static void OnFree(const void *ptr, uptr size) {}
+};
+
+} // namespace __sanitizer
+
+#endif // SANITIZER_ALLOCATOR_DLSYM_H
diff --git a/compiler-rt/lib/sanitizer_common/sanitizer_malloc_mac.inc b/compiler-rt/lib/sanitizer_common/sanitizer_malloc_mac.inc
index e3b664f68b61..764e2cef5e74 100644
--- a/compiler-rt/lib/sanitizer_common/sanitizer_malloc_mac.inc
+++ b/compiler-rt/lib/sanitizer_common/sanitizer_malloc_mac.inc
@@ -23,6 +23,7 @@
#include <sys/mman.h>
#include "interception/interception.h"
+#include "sanitizer_common/sanitizer_allocator_dlsym.h"
#include "sanitizer_common/sanitizer_mac.h"
// Similar code is used in Google Perftools,
@@ -192,20 +193,15 @@ void *__sanitizer_mz_malloc(malloc_zone_t *zone, uptr size) {
return p;
}
+struct DlsymAlloc : public DlSymAllocator<DlsymAlloc> {
+ static bool UseImpl() { return !COMMON_MALLOC_SANITIZER_INITIALIZED; }
+};
+
extern "C"
SANITIZER_INTERFACE_ATTRIBUTE
void *__sanitizer_mz_calloc(malloc_zone_t *zone, size_t nmemb, size_t size) {
- if (UNLIKELY(!COMMON_MALLOC_SANITIZER_INITIALIZED)) {
- // Hack: dlsym calls calloc before REAL(calloc) is retrieved from dlsym.
- const size_t kCallocPoolSize = 1024;
- static uptr calloc_memory_for_dlsym[kCallocPoolSize];
- static size_t allocated;
- size_t size_in_words = ((nmemb * size) + kWordSize - 1) / kWordSize;
- void *mem = (void*)&calloc_memory_for_dlsym[allocated];
- allocated += size_in_words;
- CHECK(allocated < kCallocPoolSize);
- return mem;
- }
+ if (DlsymAlloc::Use())
+ return DlsymAlloc::Callocate(nmemb, size);
COMMON_MALLOC_CALLOC(nmemb, size);
return p;
}
@@ -223,6 +219,8 @@ extern "C"
SANITIZER_INTERFACE_ATTRIBUTE
void __sanitizer_mz_free(malloc_zone_t *zone, void *ptr) {
if (!ptr) return;
+ if (DlsymAlloc::PointerIsMine(ptr))
+ return DlsymAlloc::Free(ptr);
COMMON_MALLOC_FREE(ptr);
}
More information about the llvm-commits
mailing list