[llvm-branch-commits] [compiler-rt] [CopyProf] Add standalone allocator. (PR #235453)
Snehasish Kumar via llvm-branch-commits
llvm-branch-commits at lists.llvm.org
Wed Nov 4 21:25:54 PST 2026
================
@@ -0,0 +1,312 @@
+//===-- copyprof_allocator.cpp -------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+///
+/// This file implements the standalone memory allocator for CopyProf, managing
+/// internal dynamic allocations required during profiling.
+///
+//===----------------------------------------------------------------------===//
+
+#include "copyprof_allocator.h"
+
+#include <pthread.h>
+
+#include "copyprof_interceptors.h"
+#include "copyprof_interface_internal.h"
+#include "copyprof_shadow.h"
+#include "sanitizer_common/sanitizer_allocator.h"
+#include "sanitizer_common/sanitizer_allocator_checks.h"
+#include "sanitizer_common/sanitizer_allocator_interface.h"
+#include "sanitizer_common/sanitizer_allocator_report.h"
+#include "sanitizer_common/sanitizer_common.h"
+#include "sanitizer_common/sanitizer_errno.h"
+#include "sanitizer_common/sanitizer_flags.h"
+#include "sanitizer_common/sanitizer_internal_defs.h"
+#include "sanitizer_common/sanitizer_platform_limits_posix.h"
+#include "sanitizer_common/sanitizer_stacktrace.h"
+
+namespace __copyprof {
+namespace {
+
+using namespace __sanitizer;
+
+// Callbacks invoked by the sanitizer allocator whenever it maps or unmaps
+// memory. They are no-ops, as CopyProf does not need to track the allocator's
+// mappings.
+struct CopyProfMapUnmapCallback {
+ // Called when the primary allocator maps memory for a region from which it
+ // serves small and medium sized allocations.
+ void OnMap(uptr p, uptr size) const {}
+ // Called when the secondary allocator maps memory for an allocation that is
+ // too large for the primary allocator. Each such allocation is mmap()ed
+ // separately.
+ void OnMapSecondary(uptr p, uptr size, uptr user_begin,
+ uptr user_size) const {}
+ // Called when either allocator unmaps memory.
+ void OnUnmap(uptr p, uptr size) const {}
+};
+
+struct CopyProfMetadata {
+ // The requested allocation size in bytes.
+ uptr requested_size;
+};
+
+struct AllocatorParameters {
+ static constexpr uptr kSpaceBeg = ~(uptr)0;
+ static constexpr uptr kSpaceSize = 0x40000000000ULL;
+ static constexpr uptr kMetadataSize = sizeof(CopyProfMetadata);
+ static constexpr uptr kFlags = 0;
+ using SizeClassMap = DefaultSizeClassMap;
+ using MapUnmapCallback = CopyProfMapUnmapCallback;
+ using AddressSpaceView = LocalAddressSpaceView;
+};
+
+using PrimaryAllocator = SizeClassAllocator64<AllocatorParameters>;
+using Allocator = CombinedAllocator<PrimaryAllocator>;
+using AllocatorCache = Allocator::AllocatorCache;
+
+// Per-thread allocator cache. It is mmap()ed, and thus zero-initialized as
+// `AllocatorCache` requires, and committed back to `allocator` when its thread
+// exits. As CopyProf does not intercept thread creation, the cache is created
+// by the thread's first (de)allocation, which also arms the TSD destructor that
+// commits it back.
+pthread_key_t thread_cache_key;
+__attribute__((
+ tls_model("initial-exec"))) THREADLOCAL AllocatorCache* thread_cache;
+// Set once the exiting thread's cache has been committed back. Later TSD
+// destructors may still (de)allocate on this thread so they use the fallback
+// cache, as a re-created cache could leak.
+__attribute__((
+ tls_model("initial-exec"))) THREADLOCAL bool thread_cache_committed_back;
+
+uptr ThreadCacheSize() {
+ return RoundUpTo(sizeof(AllocatorCache), GetPageSizeCached());
+}
+
+// Returns the calling thread's allocator cache, or `nullptr` if it has already
+// been committed back.
+AllocatorCache* GetThreadCache() {
+ if (LIKELY(thread_cache))
+ return thread_cache;
+ if (UNLIKELY(thread_cache_committed_back))
+ return nullptr;
+ thread_cache =
+ static_cast<AllocatorCache*>(MmapOrDie(ThreadCacheSize(), __func__));
+ CHECK_EQ(0, pthread_setspecific(thread_cache_key, thread_cache));
+ return thread_cache;
+}
+
+// Custom allocator that keeps track of allocation sizes.
+struct CopyProfAllocator {
+ // Heap allocations must be at least 8-byte aligned so independent
+ // allocations never share a shadow byte (avoiding data races).
+ static_assert(kMinAlignment == 1 << kShadowScale);
+
+ Allocator allocator;
+ // Fallback allocator cache and mutex used in the slow path when no per-thread
+ // allocator cache is available.
+ StaticSpinMutex fallback_mutex;
+ AllocatorCache fallback_allocator_cache;
+
+ void Init() {
+ SetAllocatorMayReturnNull(common_flags()->allocator_may_return_null);
+ allocator.Init(common_flags()->allocator_release_to_os_interval_ms);
+ }
+
+ CopyProfMetadata* GetMetadata(const void* ptr) {
+ return reinterpret_cast<CopyProfMetadata*>(allocator.GetMetaData(ptr));
+ }
+
+ // `align` must be a non-zero power of two; callers are responsible for
+ // validating it.
+ void* Allocate(uptr size, uptr align = kMinAlignment) {
+ if (UNLIKELY(!copyprof_is_initialized))
+ __copyprof_init_once();
+ align = (align < kMinAlignment) ? kMinAlignment : align;
+ CHECK(IsPowerOfTwo(align));
+ // C++ standard requires zero-sized allocations to return a non-null
+ // pointer.
+ if (size == 0)
+ size = 1;
+ void* allocated;
+ if (AllocatorCache* cache = GetThreadCache()) {
+ allocated = allocator.Allocate(cache, size, align);
+ } else {
+ SpinMutexLock lock(&fallback_mutex);
+ allocated = allocator.Allocate(&fallback_allocator_cache, size, align);
+ }
+
+ if (UNLIKELY(!allocated)) {
+ SetAllocatorOutOfMemory();
+ if (AllocatorMayReturnNull())
+ return nullptr;
+ UNINITIALIZED BufferedStackTrace stack;
+ stack.Unwind(StackTrace::GetCurrentPc(), GET_CURRENT_FRAME(),
+ /*context=*/nullptr, common_flags()->fast_unwind_on_fatal);
+ ReportOutOfMemory(size, &stack);
+ }
+ GetMetadata(allocated)->requested_size = size;
+ return allocated;
+ }
+
+ void Deallocate(void* ptr) {
+ if (UNLIKELY(!ptr))
+ return;
+ if (AllocatorCache* cache = GetThreadCache()) {
+ allocator.Deallocate(cache, ptr);
+ } else {
+ SpinMutexLock lock(&fallback_mutex);
+ allocator.Deallocate(&fallback_allocator_cache, ptr);
+ }
+ }
+
+ void* Calloc(uptr num, uptr size) {
+ if (UNLIKELY(CheckForCallocOverflow(size, num))) {
+ if (AllocatorMayReturnNull())
+ return nullptr;
+ UNINITIALIZED BufferedStackTrace stack;
+ stack.Unwind(StackTrace::GetCurrentPc(), GET_CURRENT_FRAME(),
+ /*context=*/nullptr, common_flags()->fast_unwind_on_fatal);
+ ReportCallocOverflow(num, size, &stack);
+ }
+ void* ptr = Allocate(num * size);
+ // Memory from the secondary allocator is freshly mmap()ed, and thus already
+ // zeroed.
+ if (ptr && allocator.FromPrimary(ptr))
+ REAL(memset)(ptr, 0, num * size);
+ return ptr;
+ }
+
+ // Implemented via `Allocate` and `Deallocate` rather than
+ // `CombinedAllocator::Reallocate`, s.t. metadata is updated as for any other
+ // (de)allocation.
+ void* Reallocate(void* old_ptr, uptr new_size) {
+ CHECK(old_ptr && new_size);
+ void* new_ptr = Allocate(new_size);
+ if (new_ptr) {
+ CHECK_NE(REAL(memcpy), nullptr);
+ uptr memcpy_size = Min(new_size, GetMetadata(old_ptr)->requested_size);
+ REAL(memcpy)(new_ptr, old_ptr, memcpy_size);
+ Deallocate(old_ptr);
+ }
+ return new_ptr;
+ }
+};
+
+CopyProfAllocator instance;
+
+// TSD destructor that commits the exiting thread's cache back to `allocator`.
+void CommitBackThreadCache(void* cache) {
+ thread_cache = nullptr;
+ thread_cache_committed_back = true;
+ instance.allocator.SwallowCache(static_cast<AllocatorCache*>(cache));
+ UnmapOrDie(cache, ThreadCacheSize());
+}
+
+} // namespace
+
+void InitializeCopyProfAllocator() {
+ instance.Init();
+ CHECK_EQ(0, pthread_key_create(&thread_cache_key, CommitBackThreadCache));
+}
+
+void* CopyProfAllocate(uptr size, uptr align) {
+ if (UNLIKELY(align == 0 || !IsPowerOfTwo(align))) {
+ errno = errno_EINVAL;
+ if (AllocatorMayReturnNull())
+ return nullptr;
+ UNINITIALIZED BufferedStackTrace stack;
+ stack.Unwind(StackTrace::GetCurrentPc(), GET_CURRENT_FRAME(),
+ /*context=*/nullptr, common_flags()->fast_unwind_on_fatal);
+ ReportInvalidAllocationAlignment(align, &stack);
+ }
+ return SetErrnoOnNull(instance.Allocate(size, align));
+}
+
+void CopyProfDeallocate(void* ptr, uptr size, uptr align) {
----------------
snehasish wrote:
Maybe drop the names of unused params?
https://github.com/llvm/llvm-project/pull/235453
More information about the llvm-branch-commits
mailing list