[compiler-rt] [asan] Plumb may_return_null through Allocator::Allocate/asan_memalign (PR #196413)
Justin T. Gibbs via llvm-commits
llvm-commits at lists.llvm.org
Fri May 8 14:51:17 PDT 2026
https://github.com/scsiguy updated https://github.com/llvm/llvm-project/pull/196413
>From bb9b5ce5a30951b7b60188b1d67ad22fef781f25 Mon Sep 17 00:00:00 2001
From: "Justin T. Gibbs" <gibbs at scsiguy.com>
Date: Thu, 7 May 2026 12:55:48 -0700
Subject: [PATCH] [asan] Plumb may_return_null through Allocator::Allocate and
asan_memalign
Allocator::Allocate() and asan_memalign() previously consulted the global
AllocatorMayReturnNull() flag internally to decide whether OOM should
yield nullptr or abort via Report*+Die(). Lift that decision to the
caller as an explicit bool parameter so future operator new wrappers can
override it.
All existing call sites pass AllocatorMayReturnNull() (the test harness
in asan_noinst_test.cpp passes false explicitly, matching the default
flag value it ran under) so observable behavior is unchanged.
NFC.
---
compiler-rt/lib/asan/asan_allocator.cpp | 76 ++++++++++++-------
compiler-rt/lib/asan/asan_allocator.h | 3 +-
compiler-rt/lib/asan/asan_malloc_linux.cpp | 4 +-
compiler-rt/lib/asan/asan_malloc_mac.cpp | 10 ++-
.../lib/asan/tests/asan_noinst_test.cpp | 3 +-
5 files changed, 62 insertions(+), 34 deletions(-)
diff --git a/compiler-rt/lib/asan/asan_allocator.cpp b/compiler-rt/lib/asan/asan_allocator.cpp
index 46ba7e16da9b2..4faa7c355fabe 100644
--- a/compiler-rt/lib/asan/asan_allocator.cpp
+++ b/compiler-rt/lib/asan/asan_allocator.cpp
@@ -536,12 +536,16 @@ struct Allocator {
}
// -------------------- Allocation/Deallocation routines ---------------
- void *Allocate(uptr size, uptr alignment, BufferedStackTrace *stack,
- AllocType alloc_type, bool can_fill) {
+ // may_return_null tells Allocate() whether OOM should produce a nullptr
+ // (true) or a fatal Report*+Die() (false). Existing callers pass
+ // AllocatorMayReturnNull() so the global flag continues to control
+ // behavior.
+ void* Allocate(uptr size, uptr alignment, BufferedStackTrace* stack,
+ AllocType alloc_type, bool can_fill, bool may_return_null) {
if (UNLIKELY(!AsanInited()))
AsanInitFromRtl();
if (UNLIKELY(IsRssLimitExceeded())) {
- if (AllocatorMayReturnNull())
+ if (may_return_null)
return nullptr;
ReportRssLimitExceeded(stack);
}
@@ -578,7 +582,7 @@ struct Allocator {
CHECK(IsAligned(needed_size, min_alignment));
if (size > kMaxAllowedMallocSize || needed_size > kMaxAllowedMallocSize ||
size > max_user_defined_malloc_size) {
- if (AllocatorMayReturnNull()) {
+ if (may_return_null) {
Report("WARNING: AddressSanitizer failed to allocate 0x%zx bytes\n",
size);
return nullptr;
@@ -600,7 +604,7 @@ struct Allocator {
}
if (UNLIKELY(!allocated)) {
SetAllocatorOutOfMemory();
- if (AllocatorMayReturnNull())
+ if (may_return_null)
return nullptr;
ReportOutOfMemory(size, stack);
}
@@ -798,7 +802,8 @@ struct Allocator {
thread_stats.reallocs++;
thread_stats.realloced += new_size;
- void *new_ptr = Allocate(new_size, 8, stack, FROM_MALLOC, true);
+ void* new_ptr = Allocate(new_size, 8, stack, FROM_MALLOC, /*can_fill=*/true,
+ AllocatorMayReturnNull());
if (new_ptr) {
u8 chunk_state = atomic_load(&m->chunk_state, memory_order_acquire);
if (chunk_state != CHUNK_ALLOCATED)
@@ -820,7 +825,8 @@ struct Allocator {
return nullptr;
ReportCallocOverflow(nmemb, size, stack);
}
- void* ptr = Allocate(nmemb * size, align, stack, FROM_MALLOC, false);
+ void* ptr = Allocate(nmemb * size, align, stack, FROM_MALLOC,
+ /*can_fill=*/false, AllocatorMayReturnNull());
// If the memory comes from the secondary allocator no need to clear it
// as it comes directly from mmap.
if (ptr && allocator.FromPrimary(ptr))
@@ -1067,7 +1073,9 @@ void asan_free_aligned_sized(void* ptr, uptr alignment, uptr size,
}
void *asan_malloc(uptr size, BufferedStackTrace *stack) {
- return SetErrnoOnNull(instance.Allocate(size, 8, stack, FROM_MALLOC, true));
+ return SetErrnoOnNull(instance.Allocate(size, 8, stack, FROM_MALLOC,
+ /*can_fill=*/true,
+ AllocatorMayReturnNull()));
}
void *asan_calloc(uptr nmemb, uptr size, BufferedStackTrace *stack) {
@@ -1076,7 +1084,9 @@ void *asan_calloc(uptr nmemb, uptr size, BufferedStackTrace *stack) {
#if SANITIZER_AIX
void* asan_vec_malloc(uptr size, BufferedStackTrace* stack) {
- return SetErrnoOnNull(instance.Allocate(size, 16, stack, FROM_MALLOC, true));
+ return SetErrnoOnNull(instance.Allocate(size, 16, stack, FROM_MALLOC,
+ /*can_fill=*/true,
+ AllocatorMayReturnNull()));
}
void* asan_vec_calloc(uptr nmemb, uptr size, BufferedStackTrace* stack) {
@@ -1097,7 +1107,9 @@ void *asan_reallocarray(void *p, uptr nmemb, uptr size,
void *asan_realloc(void *p, uptr size, BufferedStackTrace *stack) {
if (!p)
- return SetErrnoOnNull(instance.Allocate(size, 8, stack, FROM_MALLOC, true));
+ return SetErrnoOnNull(instance.Allocate(size, 8, stack, FROM_MALLOC,
+ /*can_fill=*/true,
+ AllocatorMayReturnNull()));
if (size == 0) {
if (flags()->allocator_frees_and_returns_null_on_realloc_zero) {
instance.Deallocate(p, 0, 0, stack, FROM_MALLOC);
@@ -1110,8 +1122,9 @@ void *asan_realloc(void *p, uptr size, BufferedStackTrace *stack) {
}
void *asan_valloc(uptr size, BufferedStackTrace *stack) {
- return SetErrnoOnNull(
- instance.Allocate(size, GetPageSizeCached(), stack, FROM_MALLOC, true));
+ return SetErrnoOnNull(instance.Allocate(size, GetPageSizeCached(), stack,
+ FROM_MALLOC, /*can_fill=*/true,
+ AllocatorMayReturnNull()));
}
void *asan_pvalloc(uptr size, BufferedStackTrace *stack) {
@@ -1124,19 +1137,26 @@ void *asan_pvalloc(uptr size, BufferedStackTrace *stack) {
}
// pvalloc(0) should allocate one page.
size = size ? RoundUpTo(size, PageSize) : PageSize;
- return SetErrnoOnNull(
- instance.Allocate(size, PageSize, stack, FROM_MALLOC, true));
-}
-
-void *asan_memalign(uptr alignment, uptr size, BufferedStackTrace *stack) {
+ return SetErrnoOnNull(instance.Allocate(size, PageSize, stack, FROM_MALLOC,
+ /*can_fill=*/true,
+ AllocatorMayReturnNull()));
+}
+
+// may_return_null controls behavior on every failure path (alignment,
+// oversize, OOM): true forces nullptr; false routes through the fatal
+// ReportInvalidAllocationAlignment / ReportAllocationSizeTooBig /
+// ReportOutOfMemory + Die() path. Existing callers pass
+// AllocatorMayReturnNull() to honor the global flag.
+void* asan_memalign(uptr alignment, uptr size, BufferedStackTrace* stack,
+ bool may_return_null) {
if (UNLIKELY(!IsPowerOfTwo(alignment))) {
errno = errno_EINVAL;
- if (AllocatorMayReturnNull())
+ if (may_return_null)
return nullptr;
ReportInvalidAllocationAlignment(alignment, stack);
}
- return SetErrnoOnNull(
- instance.Allocate(size, alignment, stack, FROM_MALLOC, true));
+ return SetErrnoOnNull(instance.Allocate(size, alignment, stack, FROM_MALLOC,
+ /*can_fill=*/true, may_return_null));
}
void *asan_aligned_alloc(uptr alignment, uptr size, BufferedStackTrace *stack) {
@@ -1146,8 +1166,9 @@ void *asan_aligned_alloc(uptr alignment, uptr size, BufferedStackTrace *stack) {
return nullptr;
ReportInvalidAlignedAllocAlignment(size, alignment, stack);
}
- return SetErrnoOnNull(
- instance.Allocate(size, alignment, stack, FROM_MALLOC, true));
+ return SetErrnoOnNull(instance.Allocate(size, alignment, stack, FROM_MALLOC,
+ /*can_fill=*/true,
+ AllocatorMayReturnNull()));
}
int asan_posix_memalign(void **memptr, uptr alignment, uptr size,
@@ -1157,7 +1178,8 @@ int asan_posix_memalign(void **memptr, uptr alignment, uptr size,
return errno_EINVAL;
ReportInvalidPosixMemalignAlignment(alignment, stack);
}
- void *ptr = instance.Allocate(size, alignment, stack, FROM_MALLOC, true);
+ void* ptr = instance.Allocate(size, alignment, stack, FROM_MALLOC,
+ /*can_fill=*/true, AllocatorMayReturnNull());
if (UNLIKELY(!ptr))
// OOM error is already taken care of by Allocate.
return errno_ENOMEM;
@@ -1191,7 +1213,8 @@ namespace {
void *asan_new(uptr size, BufferedStackTrace *stack, bool array) {
return SetErrnoOnNull(
- instance.Allocate(size, 0, stack, array ? FROM_NEW_BR : FROM_NEW, true));
+ instance.Allocate(size, 0, stack, array ? FROM_NEW_BR : FROM_NEW,
+ /*can_fill=*/true, AllocatorMayReturnNull()));
}
void *asan_new_aligned(uptr size, uptr alignment, BufferedStackTrace *stack,
@@ -1202,8 +1225,9 @@ void *asan_new_aligned(uptr size, uptr alignment, BufferedStackTrace *stack,
return nullptr;
ReportInvalidAllocationAlignment(alignment, stack);
}
- return SetErrnoOnNull(instance.Allocate(
- size, alignment, stack, array ? FROM_NEW_BR : FROM_NEW, true));
+ return SetErrnoOnNull(
+ instance.Allocate(size, alignment, stack, array ? FROM_NEW_BR : FROM_NEW,
+ /*can_fill=*/true, AllocatorMayReturnNull()));
}
void asan_delete(void *ptr, BufferedStackTrace *stack, bool array) {
diff --git a/compiler-rt/lib/asan/asan_allocator.h b/compiler-rt/lib/asan/asan_allocator.h
index a02d1434a273d..9a16310ef86e4 100644
--- a/compiler-rt/lib/asan/asan_allocator.h
+++ b/compiler-rt/lib/asan/asan_allocator.h
@@ -275,7 +275,8 @@ struct AsanThreadLocalMallocStorage {
AsanThreadLocalMallocStorage() {}
};
-void *asan_memalign(uptr alignment, uptr size, BufferedStackTrace *stack);
+void* asan_memalign(uptr alignment, uptr size, BufferedStackTrace* stack,
+ bool may_return_null);
void asan_free(void *ptr, BufferedStackTrace *stack);
void asan_free_sized(void* ptr, uptr size, BufferedStackTrace* stack);
void asan_free_aligned_sized(void* ptr, uptr alignment, uptr size,
diff --git a/compiler-rt/lib/asan/asan_malloc_linux.cpp b/compiler-rt/lib/asan/asan_malloc_linux.cpp
index 20a231d345710..750c84ab1279a 100644
--- a/compiler-rt/lib/asan/asan_malloc_linux.cpp
+++ b/compiler-rt/lib/asan/asan_malloc_linux.cpp
@@ -138,12 +138,12 @@ INTERCEPTOR(void*, reallocarray, void *ptr, uptr nmemb, uptr size) {
#if SANITIZER_INTERCEPT_MEMALIGN
INTERCEPTOR(void*, memalign, uptr boundary, uptr size) {
GET_STACK_TRACE_MALLOC;
- return asan_memalign(boundary, size, &stack);
+ return asan_memalign(boundary, size, &stack, AllocatorMayReturnNull());
}
INTERCEPTOR(void*, __libc_memalign, uptr boundary, uptr size) {
GET_STACK_TRACE_MALLOC;
- return asan_memalign(boundary, size, &stack);
+ return asan_memalign(boundary, size, &stack, AllocatorMayReturnNull());
}
#endif // SANITIZER_INTERCEPT_MEMALIGN
diff --git a/compiler-rt/lib/asan/asan_malloc_mac.cpp b/compiler-rt/lib/asan/asan_malloc_mac.cpp
index a442bdbbaa4d3..c4f58ac85a3db 100644
--- a/compiler-rt/lib/asan/asan_malloc_mac.cpp
+++ b/compiler-rt/lib/asan/asan_malloc_mac.cpp
@@ -31,7 +31,7 @@ using namespace __asan;
# define COMMON_MALLOC_FORCE_UNLOCK() asan_mz_force_unlock()
# define COMMON_MALLOC_MEMALIGN(alignment, size) \
GET_STACK_TRACE_MALLOC; \
- void *p = asan_memalign(alignment, size, &stack)
+ void* p = asan_memalign(alignment, size, &stack, AllocatorMayReturnNull())
# define COMMON_MALLOC_MALLOC(size) \
GET_STACK_TRACE_MALLOC; \
void *p = asan_malloc(size, &stack)
@@ -44,9 +44,11 @@ using namespace __asan;
# define COMMON_MALLOC_POSIX_MEMALIGN(memptr, alignment, size) \
GET_STACK_TRACE_MALLOC; \
int res = asan_posix_memalign(memptr, alignment, size, &stack);
-# define COMMON_MALLOC_VALLOC(size) \
- GET_STACK_TRACE_MALLOC; \
- void *p = asan_memalign(GetPageSizeCached(), size, &stack);
+# define COMMON_MALLOC_VALLOC(size) \
+ GET_STACK_TRACE_MALLOC; \
+ void* p = asan_memalign(GetPageSizeCached(), size, &stack, \
+ AllocatorMayReturnNull());
+
# define COMMON_MALLOC_FREE(ptr) \
GET_STACK_TRACE_FREE; \
asan_free(ptr, &stack);
diff --git a/compiler-rt/lib/asan/tests/asan_noinst_test.cpp b/compiler-rt/lib/asan/tests/asan_noinst_test.cpp
index cda3764b24f11..0a88c1a8e10ab 100644
--- a/compiler-rt/lib/asan/tests/asan_noinst_test.cpp
+++ b/compiler-rt/lib/asan/tests/asan_noinst_test.cpp
@@ -80,7 +80,8 @@ static void *MallocStress(void *NumOfItrPtr) {
case 2: size += 4096; break;
}
size_t alignment = 1 << (my_rand_r(&seed) % 10 + 1);
- char *ptr = (char *)__asan::asan_memalign(alignment, size, &stack2);
+ char* ptr = (char*)__asan::asan_memalign(alignment, size, &stack2,
+ /*may_return_null=*/false);
EXPECT_EQ(size, __asan::asan_malloc_usable_size(ptr, 0, 0));
vec.push_back(ptr);
ptr[0] = 0;
More information about the llvm-commits
mailing list