[compiler-rt] [asan] Plumb may_return_null through Allocator::Allocate/asan_memalign (PR #196413)

Justin T. Gibbs via llvm-commits llvm-commits at lists.llvm.org
Fri May 8 14:51:17 PDT 2026


https://github.com/scsiguy updated https://github.com/llvm/llvm-project/pull/196413

>From bb9b5ce5a30951b7b60188b1d67ad22fef781f25 Mon Sep 17 00:00:00 2001
From: "Justin T. Gibbs" <gibbs at scsiguy.com>
Date: Thu, 7 May 2026 12:55:48 -0700
Subject: [PATCH] [asan] Plumb may_return_null through Allocator::Allocate and
 asan_memalign

Allocator::Allocate() and asan_memalign() previously consulted the global
AllocatorMayReturnNull() flag internally to decide whether OOM should
yield nullptr or abort via Report*+Die(). Lift that decision to the
caller as an explicit bool parameter so future operator new wrappers can
override it.

All existing call sites pass AllocatorMayReturnNull() (the test harness
in asan_noinst_test.cpp passes false explicitly, matching the default
flag value it ran under) so observable behavior is unchanged.

NFC.
---
 compiler-rt/lib/asan/asan_allocator.cpp       | 76 ++++++++++++-------
 compiler-rt/lib/asan/asan_allocator.h         |  3 +-
 compiler-rt/lib/asan/asan_malloc_linux.cpp    |  4 +-
 compiler-rt/lib/asan/asan_malloc_mac.cpp      | 10 ++-
 .../lib/asan/tests/asan_noinst_test.cpp       |  3 +-
 5 files changed, 62 insertions(+), 34 deletions(-)

diff --git a/compiler-rt/lib/asan/asan_allocator.cpp b/compiler-rt/lib/asan/asan_allocator.cpp
index 46ba7e16da9b2..4faa7c355fabe 100644
--- a/compiler-rt/lib/asan/asan_allocator.cpp
+++ b/compiler-rt/lib/asan/asan_allocator.cpp
@@ -536,12 +536,16 @@ struct Allocator {
   }
 
   // -------------------- Allocation/Deallocation routines ---------------
-  void *Allocate(uptr size, uptr alignment, BufferedStackTrace *stack,
-                 AllocType alloc_type, bool can_fill) {
+  // may_return_null tells Allocate() whether OOM should produce a nullptr
+  // (true) or a fatal Report*+Die() (false). Existing callers pass
+  // AllocatorMayReturnNull() so the global flag continues to control
+  // behavior.
+  void* Allocate(uptr size, uptr alignment, BufferedStackTrace* stack,
+                 AllocType alloc_type, bool can_fill, bool may_return_null) {
     if (UNLIKELY(!AsanInited()))
       AsanInitFromRtl();
     if (UNLIKELY(IsRssLimitExceeded())) {
-      if (AllocatorMayReturnNull())
+      if (may_return_null)
         return nullptr;
       ReportRssLimitExceeded(stack);
     }
@@ -578,7 +582,7 @@ struct Allocator {
     CHECK(IsAligned(needed_size, min_alignment));
     if (size > kMaxAllowedMallocSize || needed_size > kMaxAllowedMallocSize ||
         size > max_user_defined_malloc_size) {
-      if (AllocatorMayReturnNull()) {
+      if (may_return_null) {
         Report("WARNING: AddressSanitizer failed to allocate 0x%zx bytes\n",
                size);
         return nullptr;
@@ -600,7 +604,7 @@ struct Allocator {
     }
     if (UNLIKELY(!allocated)) {
       SetAllocatorOutOfMemory();
-      if (AllocatorMayReturnNull())
+      if (may_return_null)
         return nullptr;
       ReportOutOfMemory(size, stack);
     }
@@ -798,7 +802,8 @@ struct Allocator {
     thread_stats.reallocs++;
     thread_stats.realloced += new_size;
 
-    void *new_ptr = Allocate(new_size, 8, stack, FROM_MALLOC, true);
+    void* new_ptr = Allocate(new_size, 8, stack, FROM_MALLOC, /*can_fill=*/true,
+                             AllocatorMayReturnNull());
     if (new_ptr) {
       u8 chunk_state = atomic_load(&m->chunk_state, memory_order_acquire);
       if (chunk_state != CHUNK_ALLOCATED)
@@ -820,7 +825,8 @@ struct Allocator {
         return nullptr;
       ReportCallocOverflow(nmemb, size, stack);
     }
-    void* ptr = Allocate(nmemb * size, align, stack, FROM_MALLOC, false);
+    void* ptr = Allocate(nmemb * size, align, stack, FROM_MALLOC,
+                         /*can_fill=*/false, AllocatorMayReturnNull());
     // If the memory comes from the secondary allocator no need to clear it
     // as it comes directly from mmap.
     if (ptr && allocator.FromPrimary(ptr))
@@ -1067,7 +1073,9 @@ void asan_free_aligned_sized(void* ptr, uptr alignment, uptr size,
 }
 
 void *asan_malloc(uptr size, BufferedStackTrace *stack) {
-  return SetErrnoOnNull(instance.Allocate(size, 8, stack, FROM_MALLOC, true));
+  return SetErrnoOnNull(instance.Allocate(size, 8, stack, FROM_MALLOC,
+                                          /*can_fill=*/true,
+                                          AllocatorMayReturnNull()));
 }
 
 void *asan_calloc(uptr nmemb, uptr size, BufferedStackTrace *stack) {
@@ -1076,7 +1084,9 @@ void *asan_calloc(uptr nmemb, uptr size, BufferedStackTrace *stack) {
 
 #if SANITIZER_AIX
 void* asan_vec_malloc(uptr size, BufferedStackTrace* stack) {
-  return SetErrnoOnNull(instance.Allocate(size, 16, stack, FROM_MALLOC, true));
+  return SetErrnoOnNull(instance.Allocate(size, 16, stack, FROM_MALLOC,
+                                          /*can_fill=*/true,
+                                          AllocatorMayReturnNull()));
 }
 
 void* asan_vec_calloc(uptr nmemb, uptr size, BufferedStackTrace* stack) {
@@ -1097,7 +1107,9 @@ void *asan_reallocarray(void *p, uptr nmemb, uptr size,
 
 void *asan_realloc(void *p, uptr size, BufferedStackTrace *stack) {
   if (!p)
-    return SetErrnoOnNull(instance.Allocate(size, 8, stack, FROM_MALLOC, true));
+    return SetErrnoOnNull(instance.Allocate(size, 8, stack, FROM_MALLOC,
+                                            /*can_fill=*/true,
+                                            AllocatorMayReturnNull()));
   if (size == 0) {
     if (flags()->allocator_frees_and_returns_null_on_realloc_zero) {
       instance.Deallocate(p, 0, 0, stack, FROM_MALLOC);
@@ -1110,8 +1122,9 @@ void *asan_realloc(void *p, uptr size, BufferedStackTrace *stack) {
 }
 
 void *asan_valloc(uptr size, BufferedStackTrace *stack) {
-  return SetErrnoOnNull(
-      instance.Allocate(size, GetPageSizeCached(), stack, FROM_MALLOC, true));
+  return SetErrnoOnNull(instance.Allocate(size, GetPageSizeCached(), stack,
+                                          FROM_MALLOC, /*can_fill=*/true,
+                                          AllocatorMayReturnNull()));
 }
 
 void *asan_pvalloc(uptr size, BufferedStackTrace *stack) {
@@ -1124,19 +1137,26 @@ void *asan_pvalloc(uptr size, BufferedStackTrace *stack) {
   }
   // pvalloc(0) should allocate one page.
   size = size ? RoundUpTo(size, PageSize) : PageSize;
-  return SetErrnoOnNull(
-      instance.Allocate(size, PageSize, stack, FROM_MALLOC, true));
-}
-
-void *asan_memalign(uptr alignment, uptr size, BufferedStackTrace *stack) {
+  return SetErrnoOnNull(instance.Allocate(size, PageSize, stack, FROM_MALLOC,
+                                          /*can_fill=*/true,
+                                          AllocatorMayReturnNull()));
+}
+
+// may_return_null controls behavior on every failure path (alignment,
+// oversize, OOM): true forces nullptr; false routes through the fatal
+// ReportInvalidAllocationAlignment / ReportAllocationSizeTooBig /
+// ReportOutOfMemory + Die() path. Existing callers pass
+// AllocatorMayReturnNull() to honor the global flag.
+void* asan_memalign(uptr alignment, uptr size, BufferedStackTrace* stack,
+                    bool may_return_null) {
   if (UNLIKELY(!IsPowerOfTwo(alignment))) {
     errno = errno_EINVAL;
-    if (AllocatorMayReturnNull())
+    if (may_return_null)
       return nullptr;
     ReportInvalidAllocationAlignment(alignment, stack);
   }
-  return SetErrnoOnNull(
-      instance.Allocate(size, alignment, stack, FROM_MALLOC, true));
+  return SetErrnoOnNull(instance.Allocate(size, alignment, stack, FROM_MALLOC,
+                                          /*can_fill=*/true, may_return_null));
 }
 
 void *asan_aligned_alloc(uptr alignment, uptr size, BufferedStackTrace *stack) {
@@ -1146,8 +1166,9 @@ void *asan_aligned_alloc(uptr alignment, uptr size, BufferedStackTrace *stack) {
       return nullptr;
     ReportInvalidAlignedAllocAlignment(size, alignment, stack);
   }
-  return SetErrnoOnNull(
-      instance.Allocate(size, alignment, stack, FROM_MALLOC, true));
+  return SetErrnoOnNull(instance.Allocate(size, alignment, stack, FROM_MALLOC,
+                                          /*can_fill=*/true,
+                                          AllocatorMayReturnNull()));
 }
 
 int asan_posix_memalign(void **memptr, uptr alignment, uptr size,
@@ -1157,7 +1178,8 @@ int asan_posix_memalign(void **memptr, uptr alignment, uptr size,
       return errno_EINVAL;
     ReportInvalidPosixMemalignAlignment(alignment, stack);
   }
-  void *ptr = instance.Allocate(size, alignment, stack, FROM_MALLOC, true);
+  void* ptr = instance.Allocate(size, alignment, stack, FROM_MALLOC,
+                                /*can_fill=*/true, AllocatorMayReturnNull());
   if (UNLIKELY(!ptr))
     // OOM error is already taken care of by Allocate.
     return errno_ENOMEM;
@@ -1191,7 +1213,8 @@ namespace {
 
 void *asan_new(uptr size, BufferedStackTrace *stack, bool array) {
   return SetErrnoOnNull(
-      instance.Allocate(size, 0, stack, array ? FROM_NEW_BR : FROM_NEW, true));
+      instance.Allocate(size, 0, stack, array ? FROM_NEW_BR : FROM_NEW,
+                        /*can_fill=*/true, AllocatorMayReturnNull()));
 }
 
 void *asan_new_aligned(uptr size, uptr alignment, BufferedStackTrace *stack,
@@ -1202,8 +1225,9 @@ void *asan_new_aligned(uptr size, uptr alignment, BufferedStackTrace *stack,
       return nullptr;
     ReportInvalidAllocationAlignment(alignment, stack);
   }
-  return SetErrnoOnNull(instance.Allocate(
-      size, alignment, stack, array ? FROM_NEW_BR : FROM_NEW, true));
+  return SetErrnoOnNull(
+      instance.Allocate(size, alignment, stack, array ? FROM_NEW_BR : FROM_NEW,
+                        /*can_fill=*/true, AllocatorMayReturnNull()));
 }
 
 void asan_delete(void *ptr, BufferedStackTrace *stack, bool array) {
diff --git a/compiler-rt/lib/asan/asan_allocator.h b/compiler-rt/lib/asan/asan_allocator.h
index a02d1434a273d..9a16310ef86e4 100644
--- a/compiler-rt/lib/asan/asan_allocator.h
+++ b/compiler-rt/lib/asan/asan_allocator.h
@@ -275,7 +275,8 @@ struct AsanThreadLocalMallocStorage {
   AsanThreadLocalMallocStorage() {}
 };
 
-void *asan_memalign(uptr alignment, uptr size, BufferedStackTrace *stack);
+void* asan_memalign(uptr alignment, uptr size, BufferedStackTrace* stack,
+                    bool may_return_null);
 void asan_free(void *ptr, BufferedStackTrace *stack);
 void asan_free_sized(void* ptr, uptr size, BufferedStackTrace* stack);
 void asan_free_aligned_sized(void* ptr, uptr alignment, uptr size,
diff --git a/compiler-rt/lib/asan/asan_malloc_linux.cpp b/compiler-rt/lib/asan/asan_malloc_linux.cpp
index 20a231d345710..750c84ab1279a 100644
--- a/compiler-rt/lib/asan/asan_malloc_linux.cpp
+++ b/compiler-rt/lib/asan/asan_malloc_linux.cpp
@@ -138,12 +138,12 @@ INTERCEPTOR(void*, reallocarray, void *ptr, uptr nmemb, uptr size) {
 #if SANITIZER_INTERCEPT_MEMALIGN
 INTERCEPTOR(void*, memalign, uptr boundary, uptr size) {
   GET_STACK_TRACE_MALLOC;
-  return asan_memalign(boundary, size, &stack);
+  return asan_memalign(boundary, size, &stack, AllocatorMayReturnNull());
 }
 
 INTERCEPTOR(void*, __libc_memalign, uptr boundary, uptr size) {
   GET_STACK_TRACE_MALLOC;
-  return asan_memalign(boundary, size, &stack);
+  return asan_memalign(boundary, size, &stack, AllocatorMayReturnNull());
 }
 #endif // SANITIZER_INTERCEPT_MEMALIGN
 
diff --git a/compiler-rt/lib/asan/asan_malloc_mac.cpp b/compiler-rt/lib/asan/asan_malloc_mac.cpp
index a442bdbbaa4d3..c4f58ac85a3db 100644
--- a/compiler-rt/lib/asan/asan_malloc_mac.cpp
+++ b/compiler-rt/lib/asan/asan_malloc_mac.cpp
@@ -31,7 +31,7 @@ using namespace __asan;
 #  define COMMON_MALLOC_FORCE_UNLOCK() asan_mz_force_unlock()
 #  define COMMON_MALLOC_MEMALIGN(alignment, size) \
     GET_STACK_TRACE_MALLOC;                       \
-    void *p = asan_memalign(alignment, size, &stack)
+    void* p = asan_memalign(alignment, size, &stack, AllocatorMayReturnNull())
 #  define COMMON_MALLOC_MALLOC(size) \
     GET_STACK_TRACE_MALLOC;          \
     void *p = asan_malloc(size, &stack)
@@ -44,9 +44,11 @@ using namespace __asan;
 #  define COMMON_MALLOC_POSIX_MEMALIGN(memptr, alignment, size) \
     GET_STACK_TRACE_MALLOC;                                     \
     int res = asan_posix_memalign(memptr, alignment, size, &stack);
-#  define COMMON_MALLOC_VALLOC(size) \
-    GET_STACK_TRACE_MALLOC;          \
-    void *p = asan_memalign(GetPageSizeCached(), size, &stack);
+#  define COMMON_MALLOC_VALLOC(size)                           \
+    GET_STACK_TRACE_MALLOC;                                    \
+    void* p = asan_memalign(GetPageSizeCached(), size, &stack, \
+                            AllocatorMayReturnNull());
+
 #  define COMMON_MALLOC_FREE(ptr) \
     GET_STACK_TRACE_FREE;         \
     asan_free(ptr, &stack);
diff --git a/compiler-rt/lib/asan/tests/asan_noinst_test.cpp b/compiler-rt/lib/asan/tests/asan_noinst_test.cpp
index cda3764b24f11..0a88c1a8e10ab 100644
--- a/compiler-rt/lib/asan/tests/asan_noinst_test.cpp
+++ b/compiler-rt/lib/asan/tests/asan_noinst_test.cpp
@@ -80,7 +80,8 @@ static void *MallocStress(void *NumOfItrPtr) {
         case 2: size += 4096; break;
       }
       size_t alignment = 1 << (my_rand_r(&seed) % 10 + 1);
-      char *ptr = (char *)__asan::asan_memalign(alignment, size, &stack2);
+      char* ptr = (char*)__asan::asan_memalign(alignment, size, &stack2,
+                                               /*may_return_null=*/false);
       EXPECT_EQ(size, __asan::asan_malloc_usable_size(ptr, 0, 0));
       vec.push_back(ptr);
       ptr[0] = 0;



More information about the llvm-commits mailing list