[compiler-rt] [asan] Throw std::bad_alloc from operator new on allocation failure (PR #196388)
Justin T. Gibbs via llvm-commits
llvm-commits at lists.llvm.org
Fri May 8 14:45:54 PDT 2026
https://github.com/scsiguy updated https://github.com/llvm/llvm-project/pull/196388
>From f83e2a28032373d208ab5f87f9d7f9b8278caef0 Mon Sep 17 00:00:00 2001
From: "Justin T. Gibbs" <gibbs at scsiguy.com>
Date: Thu, 7 May 2026 12:55:48 -0700
Subject: [PATCH 1/2] [asan] Plumb may_return_null through Allocator::Allocate
and asan_memalign
Allocator::Allocate() and asan_memalign() previously consulted the global
AllocatorMayReturnNull() flag internally to decide whether OOM should
yield nullptr or abort via Report*+Die(). Lift that decision to the
caller as an explicit bool parameter so future operator new wrappers can
override it.
All existing call sites pass AllocatorMayReturnNull() (the test harness
in asan_noinst_test.cpp passes false explicitly, matching the default
flag value it ran under) so observable behavior is unchanged.
NFC.
---
compiler-rt/lib/asan/asan_allocator.cpp | 76 ++++++++++++-------
compiler-rt/lib/asan/asan_allocator.h | 3 +-
compiler-rt/lib/asan/asan_malloc_linux.cpp | 4 +-
compiler-rt/lib/asan/asan_malloc_mac.cpp | 10 ++-
.../lib/asan/tests/asan_noinst_test.cpp | 3 +-
5 files changed, 62 insertions(+), 34 deletions(-)
diff --git a/compiler-rt/lib/asan/asan_allocator.cpp b/compiler-rt/lib/asan/asan_allocator.cpp
index 46ba7e16da9b2..4faa7c355fabe 100644
--- a/compiler-rt/lib/asan/asan_allocator.cpp
+++ b/compiler-rt/lib/asan/asan_allocator.cpp
@@ -536,12 +536,16 @@ struct Allocator {
}
// -------------------- Allocation/Deallocation routines ---------------
- void *Allocate(uptr size, uptr alignment, BufferedStackTrace *stack,
- AllocType alloc_type, bool can_fill) {
+ // may_return_null tells Allocate() whether OOM should produce a nullptr
+ // (true) or a fatal Report*+Die() (false). Existing callers pass
+ // AllocatorMayReturnNull() so the global flag continues to control
+ // behavior.
+ void* Allocate(uptr size, uptr alignment, BufferedStackTrace* stack,
+ AllocType alloc_type, bool can_fill, bool may_return_null) {
if (UNLIKELY(!AsanInited()))
AsanInitFromRtl();
if (UNLIKELY(IsRssLimitExceeded())) {
- if (AllocatorMayReturnNull())
+ if (may_return_null)
return nullptr;
ReportRssLimitExceeded(stack);
}
@@ -578,7 +582,7 @@ struct Allocator {
CHECK(IsAligned(needed_size, min_alignment));
if (size > kMaxAllowedMallocSize || needed_size > kMaxAllowedMallocSize ||
size > max_user_defined_malloc_size) {
- if (AllocatorMayReturnNull()) {
+ if (may_return_null) {
Report("WARNING: AddressSanitizer failed to allocate 0x%zx bytes\n",
size);
return nullptr;
@@ -600,7 +604,7 @@ struct Allocator {
}
if (UNLIKELY(!allocated)) {
SetAllocatorOutOfMemory();
- if (AllocatorMayReturnNull())
+ if (may_return_null)
return nullptr;
ReportOutOfMemory(size, stack);
}
@@ -798,7 +802,8 @@ struct Allocator {
thread_stats.reallocs++;
thread_stats.realloced += new_size;
- void *new_ptr = Allocate(new_size, 8, stack, FROM_MALLOC, true);
+ void* new_ptr = Allocate(new_size, 8, stack, FROM_MALLOC, /*can_fill=*/true,
+ AllocatorMayReturnNull());
if (new_ptr) {
u8 chunk_state = atomic_load(&m->chunk_state, memory_order_acquire);
if (chunk_state != CHUNK_ALLOCATED)
@@ -820,7 +825,8 @@ struct Allocator {
return nullptr;
ReportCallocOverflow(nmemb, size, stack);
}
- void* ptr = Allocate(nmemb * size, align, stack, FROM_MALLOC, false);
+ void* ptr = Allocate(nmemb * size, align, stack, FROM_MALLOC,
+ /*can_fill=*/false, AllocatorMayReturnNull());
// If the memory comes from the secondary allocator no need to clear it
// as it comes directly from mmap.
if (ptr && allocator.FromPrimary(ptr))
@@ -1067,7 +1073,9 @@ void asan_free_aligned_sized(void* ptr, uptr alignment, uptr size,
}
void *asan_malloc(uptr size, BufferedStackTrace *stack) {
- return SetErrnoOnNull(instance.Allocate(size, 8, stack, FROM_MALLOC, true));
+ return SetErrnoOnNull(instance.Allocate(size, 8, stack, FROM_MALLOC,
+ /*can_fill=*/true,
+ AllocatorMayReturnNull()));
}
void *asan_calloc(uptr nmemb, uptr size, BufferedStackTrace *stack) {
@@ -1076,7 +1084,9 @@ void *asan_calloc(uptr nmemb, uptr size, BufferedStackTrace *stack) {
#if SANITIZER_AIX
void* asan_vec_malloc(uptr size, BufferedStackTrace* stack) {
- return SetErrnoOnNull(instance.Allocate(size, 16, stack, FROM_MALLOC, true));
+ return SetErrnoOnNull(instance.Allocate(size, 16, stack, FROM_MALLOC,
+ /*can_fill=*/true,
+ AllocatorMayReturnNull()));
}
void* asan_vec_calloc(uptr nmemb, uptr size, BufferedStackTrace* stack) {
@@ -1097,7 +1107,9 @@ void *asan_reallocarray(void *p, uptr nmemb, uptr size,
void *asan_realloc(void *p, uptr size, BufferedStackTrace *stack) {
if (!p)
- return SetErrnoOnNull(instance.Allocate(size, 8, stack, FROM_MALLOC, true));
+ return SetErrnoOnNull(instance.Allocate(size, 8, stack, FROM_MALLOC,
+ /*can_fill=*/true,
+ AllocatorMayReturnNull()));
if (size == 0) {
if (flags()->allocator_frees_and_returns_null_on_realloc_zero) {
instance.Deallocate(p, 0, 0, stack, FROM_MALLOC);
@@ -1110,8 +1122,9 @@ void *asan_realloc(void *p, uptr size, BufferedStackTrace *stack) {
}
void *asan_valloc(uptr size, BufferedStackTrace *stack) {
- return SetErrnoOnNull(
- instance.Allocate(size, GetPageSizeCached(), stack, FROM_MALLOC, true));
+ return SetErrnoOnNull(instance.Allocate(size, GetPageSizeCached(), stack,
+ FROM_MALLOC, /*can_fill=*/true,
+ AllocatorMayReturnNull()));
}
void *asan_pvalloc(uptr size, BufferedStackTrace *stack) {
@@ -1124,19 +1137,26 @@ void *asan_pvalloc(uptr size, BufferedStackTrace *stack) {
}
// pvalloc(0) should allocate one page.
size = size ? RoundUpTo(size, PageSize) : PageSize;
- return SetErrnoOnNull(
- instance.Allocate(size, PageSize, stack, FROM_MALLOC, true));
-}
-
-void *asan_memalign(uptr alignment, uptr size, BufferedStackTrace *stack) {
+ return SetErrnoOnNull(instance.Allocate(size, PageSize, stack, FROM_MALLOC,
+ /*can_fill=*/true,
+ AllocatorMayReturnNull()));
+}
+
+// may_return_null controls behavior on every failure path (alignment,
+// oversize, OOM): true forces nullptr; false routes through the fatal
+// ReportInvalidAllocationAlignment / ReportAllocationSizeTooBig /
+// ReportOutOfMemory + Die() path. Existing callers pass
+// AllocatorMayReturnNull() to honor the global flag.
+void* asan_memalign(uptr alignment, uptr size, BufferedStackTrace* stack,
+ bool may_return_null) {
if (UNLIKELY(!IsPowerOfTwo(alignment))) {
errno = errno_EINVAL;
- if (AllocatorMayReturnNull())
+ if (may_return_null)
return nullptr;
ReportInvalidAllocationAlignment(alignment, stack);
}
- return SetErrnoOnNull(
- instance.Allocate(size, alignment, stack, FROM_MALLOC, true));
+ return SetErrnoOnNull(instance.Allocate(size, alignment, stack, FROM_MALLOC,
+ /*can_fill=*/true, may_return_null));
}
void *asan_aligned_alloc(uptr alignment, uptr size, BufferedStackTrace *stack) {
@@ -1146,8 +1166,9 @@ void *asan_aligned_alloc(uptr alignment, uptr size, BufferedStackTrace *stack) {
return nullptr;
ReportInvalidAlignedAllocAlignment(size, alignment, stack);
}
- return SetErrnoOnNull(
- instance.Allocate(size, alignment, stack, FROM_MALLOC, true));
+ return SetErrnoOnNull(instance.Allocate(size, alignment, stack, FROM_MALLOC,
+ /*can_fill=*/true,
+ AllocatorMayReturnNull()));
}
int asan_posix_memalign(void **memptr, uptr alignment, uptr size,
@@ -1157,7 +1178,8 @@ int asan_posix_memalign(void **memptr, uptr alignment, uptr size,
return errno_EINVAL;
ReportInvalidPosixMemalignAlignment(alignment, stack);
}
- void *ptr = instance.Allocate(size, alignment, stack, FROM_MALLOC, true);
+ void* ptr = instance.Allocate(size, alignment, stack, FROM_MALLOC,
+ /*can_fill=*/true, AllocatorMayReturnNull());
if (UNLIKELY(!ptr))
// OOM error is already taken care of by Allocate.
return errno_ENOMEM;
@@ -1191,7 +1213,8 @@ namespace {
void *asan_new(uptr size, BufferedStackTrace *stack, bool array) {
return SetErrnoOnNull(
- instance.Allocate(size, 0, stack, array ? FROM_NEW_BR : FROM_NEW, true));
+ instance.Allocate(size, 0, stack, array ? FROM_NEW_BR : FROM_NEW,
+ /*can_fill=*/true, AllocatorMayReturnNull()));
}
void *asan_new_aligned(uptr size, uptr alignment, BufferedStackTrace *stack,
@@ -1202,8 +1225,9 @@ void *asan_new_aligned(uptr size, uptr alignment, BufferedStackTrace *stack,
return nullptr;
ReportInvalidAllocationAlignment(alignment, stack);
}
- return SetErrnoOnNull(instance.Allocate(
- size, alignment, stack, array ? FROM_NEW_BR : FROM_NEW, true));
+ return SetErrnoOnNull(
+ instance.Allocate(size, alignment, stack, array ? FROM_NEW_BR : FROM_NEW,
+ /*can_fill=*/true, AllocatorMayReturnNull()));
}
void asan_delete(void *ptr, BufferedStackTrace *stack, bool array) {
diff --git a/compiler-rt/lib/asan/asan_allocator.h b/compiler-rt/lib/asan/asan_allocator.h
index a02d1434a273d..9a16310ef86e4 100644
--- a/compiler-rt/lib/asan/asan_allocator.h
+++ b/compiler-rt/lib/asan/asan_allocator.h
@@ -275,7 +275,8 @@ struct AsanThreadLocalMallocStorage {
AsanThreadLocalMallocStorage() {}
};
-void *asan_memalign(uptr alignment, uptr size, BufferedStackTrace *stack);
+void* asan_memalign(uptr alignment, uptr size, BufferedStackTrace* stack,
+ bool may_return_null);
void asan_free(void *ptr, BufferedStackTrace *stack);
void asan_free_sized(void* ptr, uptr size, BufferedStackTrace* stack);
void asan_free_aligned_sized(void* ptr, uptr alignment, uptr size,
diff --git a/compiler-rt/lib/asan/asan_malloc_linux.cpp b/compiler-rt/lib/asan/asan_malloc_linux.cpp
index 20a231d345710..750c84ab1279a 100644
--- a/compiler-rt/lib/asan/asan_malloc_linux.cpp
+++ b/compiler-rt/lib/asan/asan_malloc_linux.cpp
@@ -138,12 +138,12 @@ INTERCEPTOR(void*, reallocarray, void *ptr, uptr nmemb, uptr size) {
#if SANITIZER_INTERCEPT_MEMALIGN
INTERCEPTOR(void*, memalign, uptr boundary, uptr size) {
GET_STACK_TRACE_MALLOC;
- return asan_memalign(boundary, size, &stack);
+ return asan_memalign(boundary, size, &stack, AllocatorMayReturnNull());
}
INTERCEPTOR(void*, __libc_memalign, uptr boundary, uptr size) {
GET_STACK_TRACE_MALLOC;
- return asan_memalign(boundary, size, &stack);
+ return asan_memalign(boundary, size, &stack, AllocatorMayReturnNull());
}
#endif // SANITIZER_INTERCEPT_MEMALIGN
diff --git a/compiler-rt/lib/asan/asan_malloc_mac.cpp b/compiler-rt/lib/asan/asan_malloc_mac.cpp
index a442bdbbaa4d3..c4f58ac85a3db 100644
--- a/compiler-rt/lib/asan/asan_malloc_mac.cpp
+++ b/compiler-rt/lib/asan/asan_malloc_mac.cpp
@@ -31,7 +31,7 @@ using namespace __asan;
# define COMMON_MALLOC_FORCE_UNLOCK() asan_mz_force_unlock()
# define COMMON_MALLOC_MEMALIGN(alignment, size) \
GET_STACK_TRACE_MALLOC; \
- void *p = asan_memalign(alignment, size, &stack)
+ void* p = asan_memalign(alignment, size, &stack, AllocatorMayReturnNull())
# define COMMON_MALLOC_MALLOC(size) \
GET_STACK_TRACE_MALLOC; \
void *p = asan_malloc(size, &stack)
@@ -44,9 +44,11 @@ using namespace __asan;
# define COMMON_MALLOC_POSIX_MEMALIGN(memptr, alignment, size) \
GET_STACK_TRACE_MALLOC; \
int res = asan_posix_memalign(memptr, alignment, size, &stack);
-# define COMMON_MALLOC_VALLOC(size) \
- GET_STACK_TRACE_MALLOC; \
- void *p = asan_memalign(GetPageSizeCached(), size, &stack);
+# define COMMON_MALLOC_VALLOC(size) \
+ GET_STACK_TRACE_MALLOC; \
+ void* p = asan_memalign(GetPageSizeCached(), size, &stack, \
+ AllocatorMayReturnNull());
+
# define COMMON_MALLOC_FREE(ptr) \
GET_STACK_TRACE_FREE; \
asan_free(ptr, &stack);
diff --git a/compiler-rt/lib/asan/tests/asan_noinst_test.cpp b/compiler-rt/lib/asan/tests/asan_noinst_test.cpp
index cda3764b24f11..0a88c1a8e10ab 100644
--- a/compiler-rt/lib/asan/tests/asan_noinst_test.cpp
+++ b/compiler-rt/lib/asan/tests/asan_noinst_test.cpp
@@ -80,7 +80,8 @@ static void *MallocStress(void *NumOfItrPtr) {
case 2: size += 4096; break;
}
size_t alignment = 1 << (my_rand_r(&seed) % 10 + 1);
- char *ptr = (char *)__asan::asan_memalign(alignment, size, &stack2);
+ char* ptr = (char*)__asan::asan_memalign(alignment, size, &stack2,
+ /*may_return_null=*/false);
EXPECT_EQ(size, __asan::asan_malloc_usable_size(ptr, 0, 0));
vec.push_back(ptr);
ptr[0] = 0;
>From 447ffa37e1c285e7975d860b8b4597164c638e10 Mon Sep 17 00:00:00 2001
From: "Justin T. Gibbs" <gibbs at scsiguy.com>
Date: Thu, 7 May 2026 12:57:59 -0700
Subject: [PATCH 2/2] [asan] Throw std::bad_alloc from operator new on
allocation failure
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit
The changed allocator behavior is 'opt-in' via allocator_may_return_null=1.
The throwing forms of operator new in asan_new_delete.cpp now honor [new.delete.single]/3: on allocation failure they run std::get_new_handler() in a loop until either the allocation succeeds or the handler is null, at which point they either throw std::bad_alloc — when allocator_may_return_null=1 — or fall back to the historical fatal ReportOutOfMemory diagnostic when the flag is left at its default false. The nothrow forms honor /4 by behaving as if they call the throwing form within a try/catch — the same handler chain runs, and a thrown std::bad_alloc converts to a nullptr return (or, when the flag is at its default false, falls back to ReportOutOfMemory + Die() to preserve historical behavior).
Both forms now invoke the new_handler chain unconditionally regardless of allocator_may_return_null, on every supported platform including Windows. This closes the long-standing TODO in asan_new_delete.cpp ("throw std::bad_alloc instead of dying on OOM") and aligns ASan with libstdc++ / libc++ / tcmalloc. See https://github.com/google/sanitizers/issues/295.
Implementation:
* asan_allocator: the throwing / nothrow operator new wrappers (asan_new / asan_new_aligned and the array variants) now force may_return_null=true on Allocate() so the lambdas in OPERATOR_NEW_BODY* always observe nullptr on failure — never abort. asan_new_aligned's alignment-validation path now always returns nullptr, letting the new_handler chain decide rather than aborting unconditionally on bad alignment.
* asan_new_delete: include <new> on non-Windows targets; replace the unconditional fake std::nothrow_t / std::align_val_t with a Windows-only fake-std namespace that also forward-declares std::get_new_handler (provided by the C++ runtime the asan DLL already links against). Add three small templates that capture the entire new-handling protocol: RunNewHandlerChain runs std::get_new_handler() per [new.delete.single]/3+/4 until success or chain exhaustion; NewImplNothrowExhausted is the chain-exhausted decision used by the nothrow form (nullptr if AllocatorMayReturnNull(), else ReportOutOfMemory + Die()); NewImplThrowing and NewImplNothrow are thin wrappers that compose the two. Platform variation lives in two #if blocks inside the templates rather than in a per-platform pair of OPERATOR_NEW_BODY* macros. The eight OPERATOR_NEW_BODY* macros are a single shared set across platforms.
* Throwing form chain-exhausted contract: the throwing form must never return nullptr to its caller (per [basic.stc.dynamic.allocation]/3 it returns a non-null pointer or propagates an exception). NewImplThrowing therefore does not route through NewImplNothrowExhausted — it inlines its own chain-exhausted handling: throw std::bad_alloc on Linux/Apple/etc. with the flag set, otherwise abort via ReportOutOfMemory + Die(). On Windows, where exceptions are unavailable, it always aborts on chain exhaustion regardless of allocator_may_return_null.
* The nothrow operator new / delete overloads are now declared NOEXCEPT to match the standard library's exception specifications, which become visible once <new> is included.
* Windows: the asan runtime is built without exceptions and cannot include <new>, so the throwing form can't throw std::bad_alloc. It still runs the std::get_new_handler() chain on every allocation failure (handler-invocation half of /3) and falls back to the historical ReportOutOfMemory + Die() path on chain exhaustion — independently of allocator_may_return_null, since returning nullptr would violate the throwing-new contract. The nothrow form is fully /4-conformant on Windows: handler chain runs, then nullptr (allocator_may_return_null=1) or ReportOutOfMemory + Die() (default flag). Handler-thrown exceptions on Windows remain undefined behavior per the no-exceptions build, as they have always been on Windows asan.
* sanitizer_common/sanitizer_flags.inc: rewrite the allocator_may_return_null docstring to enumerate the four regimes (malloc / throwing-new / nothrow-new × flag false/true) and clarify that both new forms always run the handler chain regardless of the flag.
* CMake: define ASAN_CXX_CFLAGS and ASAN_DYNAMIC_CXX_CFLAGS as the base ASan flags minus -fno-exceptions and -nostdinc++ plus -fexceptions. Apply to RTAsan_cxx (static C++ slice) and RTAsan_dynamic (dynamic build, which is C++-only by construction). RTTI stays disabled; libstdc++ / libc++abi provide the std::bad_alloc typeinfo.
* Diagnostic change for the operator-new path: because asan_new now forces may_return_null=true on Allocate(), the in-place "allocation-size-too-big" diagnostic that previously fired for oversize requests inside Allocate is replaced by a "WARNING: AddressSanitizer failed to allocate" line (from Allocate's may_return_null=true branch) followed by the chain-exhausted "out-of-memory" SUMMARY (or std::bad_alloc throw on flag=1). Other failure paths (malloc / calloc / realloc / aligned_alloc / posix_memalign) continue to emit "allocation-size-too-big" as before.
Behavior matrix (handler chain runs in every cell):
| allocator_may_return_null=0 (default) | allocator_may_return_null=1
--------------------------- | ------------------------------------- | -----------------------------------
throwing operator new | runs handler chain, then | runs handler chain, then throws
(Linux/Apple/etc.) | ReportOutOfMemory + Die() if | std::bad_alloc if chain exhausted
| chain exhausted (vanilla diagnostic) | (NEW)
--------------------------- | ------------------------------------- | -----------------------------------
throwing operator new | runs handler chain, then ReportOutOfMemory + Die() if chain exhausted —
(Windows, both flag values)| identical for both flag values, since the throwing form must never
| return nullptr and Windows has no exceptions to throw.
--------------------------- | ------------------------------------- | -----------------------------------
nothrow operator new | runs handler chain, then | runs handler chain, then returns
(all platforms) | ReportOutOfMemory + Die() if | nullptr if chain exhausted
| chain exhausted (vanilla diagnostic) | (vanilla under /4)
--------------------------- | ------------------------------------- | -----------------------------------
malloc / aligned_alloc / | abort + ReportOutOfMemory (vanilla) | nullptr (vanilla)
posix_memalign / realloc | |
The handler chain runs unconditionally — both forms are standards-conformant w.r.t. handler invocation regardless of allocator_may_return_null. Vanilla pre-patch fatal-abort behavior is preserved by default for users who haven't installed a handler and haven't opted into the throwing semantics, on every platform.
Test Plan:
New tests under compiler-rt/test/asan/TestCases/ covering the eight OPERATOR_NEW_BODY* macro paths plus both flag values:
Opt-in (allocator_may_return_null=1):
* throw_bad_alloc_oversize.cpp — array throws bad_alloc
* throw_bad_alloc_aligned.cpp — aligned array throws bad_alloc
* throw_bad_alloc_single.cpp — single-object throws bad_alloc
* throw_bad_alloc_aligned_single.cpp — aligned single-object throws bad_alloc
* nothrow_new_returns_null.cpp — array nothrow returns null
* nothrow_new_single_returns_null.cpp — single-object nothrow returns null
* nothrow_new_aligned_single_returns_null.cpp — aligned single-object nothrow returns null
* nothrow_new_aligned_array_returns_null.cpp — aligned array nothrow returns null
* new_handler_invocation.cpp — handler runs, can break loop
* new_handler_throws_other.cpp — non-bad_alloc handler exception propagates
Default flag (allocator_may_return_null=0) — abort path:
* throw_bad_alloc_default_aborts.cpp — throwing new aborts with vanilla diagnostic
* nothrow_new_default_aborts.cpp — nothrow new aborts with vanilla diagnostic
The four nothrow tests run on Windows too (the Windows nothrow path is now standards-conformant); the throwing tests stay UNSUPPORTED on Windows since they require std::bad_alloc to be catchable.
Regression: existing malloc-size-too-big.cpp continues to assert that malloc OOM under allocator_may_return_null=0 aborts.
Updated existing sanitizer_common tests for the new-operator behavior:
* compiler-rt/test/sanitizer_common/TestCases/allocator_returns_null.cpp
* compiler-rt/test/sanitizer_common/TestCases/max_allocation_size.cpp
* compiler-rt/test/sanitizer_common/TestCases/Linux/allocator_returns_null_std.cpp
For the flag=0 cells (CHECK-nCRASH, CHECK-nnCRASH, the std test), the expected SUMMARY pattern is loosened to (allocation-size-too-big|out-of-memory) — other sanitizers still emit "allocation-size-too-big" (their Allocate hits the in-place size-too-big report); asan post-patch emits "out-of-memory" because asan_new now forces may_return_null=true on Allocate so the chain-exhausted abort path runs instead.
For the flag=1 + throwing-new cell, the test now wraps the operator new call in a try/catch around std::bad_alloc and converges asan to the same observable as the nothrow path (errno=ENOMEM, x=nullptr -> CHECK-NULL). This validates that the bad_alloc throw is catchable from user code (the whole point of the new contract), rather than just observing that the process aborts. Other sanitizers' operator new still calls ReportOutOfMemory + Die() before any throw could occur, so their catch never runs — the cell-1 RUN line splits via lit %if asan to route them to the unchanged CHECK-nCRASH-OOM (process aborts with "out-of-memory" SUMMARY).
Verified locally: all 3 sanitizer_common tests pass under asan-x86_64-Linux, and all 12 new asan-specific TestCases pass under both X86_64LinuxConfig and X86_64LinuxDynamicConfig.
Manual end-to-end verification against a real-world deserialization regression where attacker-controlled input drives an uncapped std::string::resize through operator new: default ASAN_OPTIONS aborts with the historical ReportOutOfMemory + Die() diagnostic stack; with ASAN_OPTIONS=allocator_may_return_null=1 the same input causes std::bad_alloc to be thrown and caught by the caller's existing exception handler, allowing graceful recovery instead of process termination.
Depends on the preceding NFC change "[asan] Plumb may_return_null through Allocator::Allocate and asan_memalign".
---
compiler-rt/lib/asan/CMakeLists.txt | 19 +-
compiler-rt/lib/asan/asan_allocator.cpp | 29 +--
compiler-rt/lib/asan/asan_new_delete.cpp | 187 +++++++++++++-----
.../lib/sanitizer_common/sanitizer_flags.inc | 13 +-
.../asan/TestCases/new_handler_invocation.cpp | 45 +++++
.../TestCases/new_handler_throws_other.cpp | 36 ++++
...nothrow_new_aligned_array_returns_null.cpp | 27 +++
...othrow_new_aligned_single_returns_null.cpp | 26 +++
.../TestCases/nothrow_new_default_aborts.cpp | 33 ++++
.../TestCases/nothrow_new_returns_null.cpp | 26 +++
.../nothrow_new_single_returns_null.cpp | 25 +++
.../TestCases/throw_bad_alloc_aligned.cpp | 36 ++++
.../throw_bad_alloc_aligned_single.cpp | 34 ++++
.../throw_bad_alloc_default_aborts.cpp | 31 +++
.../TestCases/throw_bad_alloc_oversize.cpp | 34 ++++
.../asan/TestCases/throw_bad_alloc_single.cpp | 35 ++++
.../Linux/allocator_returns_null_std.cpp | 6 +-
.../TestCases/allocator_returns_null.cpp | 32 ++-
.../TestCases/max_allocation_size.cpp | 35 +++-
19 files changed, 627 insertions(+), 82 deletions(-)
create mode 100644 compiler-rt/test/asan/TestCases/new_handler_invocation.cpp
create mode 100644 compiler-rt/test/asan/TestCases/new_handler_throws_other.cpp
create mode 100644 compiler-rt/test/asan/TestCases/nothrow_new_aligned_array_returns_null.cpp
create mode 100644 compiler-rt/test/asan/TestCases/nothrow_new_aligned_single_returns_null.cpp
create mode 100644 compiler-rt/test/asan/TestCases/nothrow_new_default_aborts.cpp
create mode 100644 compiler-rt/test/asan/TestCases/nothrow_new_returns_null.cpp
create mode 100644 compiler-rt/test/asan/TestCases/nothrow_new_single_returns_null.cpp
create mode 100644 compiler-rt/test/asan/TestCases/throw_bad_alloc_aligned.cpp
create mode 100644 compiler-rt/test/asan/TestCases/throw_bad_alloc_aligned_single.cpp
create mode 100644 compiler-rt/test/asan/TestCases/throw_bad_alloc_default_aborts.cpp
create mode 100644 compiler-rt/test/asan/TestCases/throw_bad_alloc_oversize.cpp
create mode 100644 compiler-rt/test/asan/TestCases/throw_bad_alloc_single.cpp
diff --git a/compiler-rt/lib/asan/CMakeLists.txt b/compiler-rt/lib/asan/CMakeLists.txt
index 6085f18426dff..2838771da5330 100644
--- a/compiler-rt/lib/asan/CMakeLists.txt
+++ b/compiler-rt/lib/asan/CMakeLists.txt
@@ -143,6 +143,17 @@ set(ASAN_DYNAMIC_CFLAGS ${ASAN_CFLAGS})
append_list_if(COMPILER_RT_HAS_FTLS_MODEL_INITIAL_EXEC
-ftls-model=initial-exec ASAN_DYNAMIC_CFLAGS)
+# asan_new_delete.cpp throws std::bad_alloc, so its TU needs -fexceptions
+# and access to <new> (drop -nostdinc++). RTTI stays disabled — libstdc++ /
+# libc++abi supply the bad_alloc typeinfo.
+set(ASAN_CXX_CFLAGS ${ASAN_CFLAGS})
+list(REMOVE_ITEM ASAN_CXX_CFLAGS -fno-exceptions -nostdinc++)
+append_list_if(COMPILER_RT_HAS_FEXCEPTIONS_FLAG -fexceptions ASAN_CXX_CFLAGS)
+set(ASAN_DYNAMIC_CXX_CFLAGS ${ASAN_DYNAMIC_CFLAGS})
+list(REMOVE_ITEM ASAN_DYNAMIC_CXX_CFLAGS -fno-exceptions -nostdinc++)
+append_list_if(COMPILER_RT_HAS_FEXCEPTIONS_FLAG -fexceptions
+ ASAN_DYNAMIC_CXX_CFLAGS)
+
# LLVM turns /OPT:ICF back on when LLVM_ENABLE_PDBs is set
# we _REALLY_ need to turn it back off for ASAN, because the way
# asan emulates weak functions from DLLs requires NOICF
@@ -167,7 +178,10 @@ add_compiler_rt_object_libraries(RTAsan_dynamic
ARCHS ${ASAN_SUPPORTED_ARCH}
SOURCES ${ASAN_SOURCES} ${ASAN_CXX_SOURCES}
ADDITIONAL_HEADERS ${ASAN_HEADERS}
- CFLAGS ${ASAN_DYNAMIC_CFLAGS}
+ # Build the whole dynamic ASan with -fexceptions rather than splitting it
+ # into C and C++ slices — only asan_new_delete.cpp uses exceptions but the
+ # extra flag is harmless on the C sources and avoids the build-system split.
+ CFLAGS ${ASAN_DYNAMIC_CXX_CFLAGS}
DEFS ${ASAN_DYNAMIC_DEFINITIONS})
if(NOT APPLE)
@@ -181,7 +195,8 @@ if(NOT APPLE)
ARCHS ${ASAN_SUPPORTED_ARCH}
SOURCES ${ASAN_CXX_SOURCES}
ADDITIONAL_HEADERS ${ASAN_HEADERS}
- CFLAGS ${ASAN_CFLAGS}
+ # asan_new_delete.cpp needs -fexceptions to throw bad_alloc on OOM.
+ CFLAGS ${ASAN_CXX_CFLAGS}
DEFS ${ASAN_COMMON_DEFINITIONS})
add_compiler_rt_object_libraries(RTAsan_static
ARCHS ${ASAN_SUPPORTED_ARCH}
diff --git a/compiler-rt/lib/asan/asan_allocator.cpp b/compiler-rt/lib/asan/asan_allocator.cpp
index 4faa7c355fabe..7c0d834e90a27 100644
--- a/compiler-rt/lib/asan/asan_allocator.cpp
+++ b/compiler-rt/lib/asan/asan_allocator.cpp
@@ -536,10 +536,11 @@ struct Allocator {
}
// -------------------- Allocation/Deallocation routines ---------------
- // may_return_null tells Allocate() whether OOM should produce a nullptr
- // (true) or a fatal Report*+Die() (false). Existing callers pass
- // AllocatorMayReturnNull() so the global flag continues to control
- // behavior.
+ // may_return_null: true returns nullptr on failure, false aborts via
+ // Report*+Die(). C-allocator entry points pass AllocatorMayReturnNull() to
+ // honor the global flag; operator new wrappers pass true unconditionally so
+ // std::get_new_handler() runs before the runtime gives up — see
+ // asan_new_delete.cpp.
void* Allocate(uptr size, uptr alignment, BufferedStackTrace* stack,
AllocType alloc_type, bool can_fill, bool may_return_null) {
if (UNLIKELY(!AsanInited()))
@@ -1142,11 +1143,10 @@ void *asan_pvalloc(uptr size, BufferedStackTrace *stack) {
AllocatorMayReturnNull()));
}
-// may_return_null controls behavior on every failure path (alignment,
-// oversize, OOM): true forces nullptr; false routes through the fatal
-// ReportInvalidAllocationAlignment / ReportAllocationSizeTooBig /
-// ReportOutOfMemory + Die() path. Existing callers pass
-// AllocatorMayReturnNull() to honor the global flag.
+// may_return_null applies to every failure path (alignment / oversize / OOM)
+// with the same semantics as Allocate() above. Same caller policy too: the
+// C-allocator entry points (memalign / __libc_memalign / Mac-zone callers)
+// pass AllocatorMayReturnNull(); operator new wrappers pass true.
void* asan_memalign(uptr alignment, uptr size, BufferedStackTrace* stack,
bool may_return_null) {
if (UNLIKELY(!IsPowerOfTwo(alignment))) {
@@ -1211,23 +1211,24 @@ uptr asan_malloc_usable_size(const void *ptr, uptr pc, uptr bp) {
namespace {
+// Force may_return_null=true so operator new wrappers in asan_new_delete.cpp
+// can run the std::get_new_handler() chain before deciding what to do (per
+// [new.delete.single]/3+/4).
void *asan_new(uptr size, BufferedStackTrace *stack, bool array) {
return SetErrnoOnNull(
instance.Allocate(size, 0, stack, array ? FROM_NEW_BR : FROM_NEW,
- /*can_fill=*/true, AllocatorMayReturnNull()));
+ /*can_fill=*/true, /*may_return_null=*/true));
}
void *asan_new_aligned(uptr size, uptr alignment, BufferedStackTrace *stack,
bool array) {
if (UNLIKELY(alignment == 0 || !IsPowerOfTwo(alignment))) {
errno = errno_EINVAL;
- if (AllocatorMayReturnNull())
- return nullptr;
- ReportInvalidAllocationAlignment(alignment, stack);
+ return nullptr;
}
return SetErrnoOnNull(
instance.Allocate(size, alignment, stack, array ? FROM_NEW_BR : FROM_NEW,
- /*can_fill=*/true, AllocatorMayReturnNull()));
+ /*can_fill=*/true, /*may_return_null=*/true));
}
void asan_delete(void *ptr, BufferedStackTrace *stack, bool array) {
diff --git a/compiler-rt/lib/asan/asan_new_delete.cpp b/compiler-rt/lib/asan/asan_new_delete.cpp
index d7ed5b570728b..85a11f7a4fab1 100644
--- a/compiler-rt/lib/asan/asan_new_delete.cpp
+++ b/compiler-rt/lib/asan/asan_new_delete.cpp
@@ -19,6 +19,15 @@
#include "asan_stack.h"
#include "interception/interception.h"
+// <new> is unavailable on Windows because the asan runtime is built there
+// without exceptions; the fake std-namespace block below covers that case,
+// and per-platform branching lives in the templates further down. See
+// https://github.com/google/sanitizers/issues/295 for the history of
+// enabling exceptions in the Windows asan runtime.
+#if !SANITIZER_WINDOWS
+# include <new>
+#endif
+
// C++ operators can't have dllexport attributes on Windows. We export them
// anyway by passing extra -export flags to the linker, which is exactly that
// dllexport would normally do. We need to export them in order to make the
@@ -51,51 +60,131 @@ using namespace __asan;
// This code has issues on OSX.
// See https://github.com/google/sanitizers/issues/131.
-// Fake std::nothrow_t and std::align_val_t to avoid including <new>.
+#if SANITIZER_WINDOWS
+// Forward-declare just enough of std for the operator overrides and for
+// RunNewHandlerChain below. std::get_new_handler is supplied by the C++
+// runtime the asan DLL already links against (vcruntime / msvcprt / mingw
+// libstdc++) — it doesn't need to be in <new>.
namespace std {
struct nothrow_t {};
-enum class align_val_t: size_t {};
+enum class align_val_t : size_t {};
+using new_handler = void (*)();
+new_handler get_new_handler() noexcept;
} // namespace std
+#endif // SANITIZER_WINDOWS
+
+// All eight operator new variants route through three templates so the
+// per-platform difference (Linux/Apple throw, Windows abort) is centralized
+// here rather than duplicated in eight OPERATOR_NEW_BODY* macros.
+//
+// Contract: the Alloc callable must always return nullptr on failure, never
+// abort. asan_new / asan_new_array / asan_new_aligned / asan_new_array_aligned
+// in asan_allocator.cpp force may_return_null=true to honor this.
+
+// Runs std::get_new_handler() per [new.delete.single]/3+/4 until the
+// allocation succeeds or the handler is null. A handler that throws
+// propagates out of this function — callers wrap in try/catch as needed.
+template <typename Alloc>
+static void* RunNewHandlerChain(Alloc alloc) {
+ for (;;) {
+ void* res = alloc();
+ if (LIKELY(res != nullptr))
+ return res;
+ std::new_handler handler = std::get_new_handler();
+ if (!handler)
+ return nullptr;
+ handler();
+ }
+}
+
+// Chain-exhausted decision for the nothrow form: nullptr if
+// allocator_may_return_null is set, else ReportOutOfMemory + Die(). The
+// throwing form intentionally does NOT route through here — it must never
+// return nullptr to its caller (per [basic.stc.dynamic.allocation]/3) so on
+// Windows, where it can't throw bad_alloc, it must abort regardless of the
+// flag.
+static void* NewImplNothrowExhausted(uptr size, BufferedStackTrace* stack) {
+ if (AllocatorMayReturnNull())
+ return nullptr;
+ ReportOutOfMemory(size, stack);
+ __builtin_unreachable();
+}
-// TODO(alekseyshl): throw std::bad_alloc instead of dying on OOM.
-// For local pool allocation, align to SHADOW_GRANULARITY to match asan
-// allocator behavior.
-#define OPERATOR_NEW_BODY \
- GET_STACK_TRACE_MALLOC; \
- void *res = asan_new(size, &stack); \
- if (UNLIKELY(!res)) \
- ReportOutOfMemory(size, &stack); \
- return res
+// Throwing operator new: chain, then on exhaustion throw std::bad_alloc
+// (non-Windows + allocator_may_return_null=1) or abort via
+// ReportOutOfMemory + Die() (default flag, or Windows for any flag value).
+template <typename Alloc>
+static void* NewImplThrowing(uptr size, BufferedStackTrace* stack,
+ Alloc alloc) {
+ void* res = RunNewHandlerChain(alloc);
+ if (LIKELY(res != nullptr))
+ return res;
+#if !SANITIZER_WINDOWS
+ if (AllocatorMayReturnNull())
+ throw std::bad_alloc();
+#endif
+ ReportOutOfMemory(size, stack);
+ __builtin_unreachable();
+}
+
+// Nothrow operator new: per [new.delete.single]/4 behaves as-if the throwing
+// form is called within a try/catch. On Windows there's no try/catch; a
+// user handler that throws there yields UB (asan is built without
+// exceptions on Windows).
+template <typename Alloc>
+static void* NewImplNothrow(uptr size, BufferedStackTrace* stack,
+ Alloc alloc) noexcept {
+#if !SANITIZER_WINDOWS
+ try {
+ void* res = RunNewHandlerChain(alloc);
+ if (LIKELY(res != nullptr))
+ return res;
+ return NewImplNothrowExhausted(size, stack);
+ } catch (...) {
+ return nullptr;
+ }
+#else
+ void* res = RunNewHandlerChain(alloc);
+ if (LIKELY(res != nullptr))
+ return res;
+ return NewImplNothrowExhausted(size, stack);
+#endif
+}
+
+#define OPERATOR_NEW_BODY \
+ GET_STACK_TRACE_MALLOC; \
+ return NewImplThrowing(size, &stack, [&]() { return asan_new(size, &stack); })
#define OPERATOR_NEW_BODY_NOTHROW \
GET_STACK_TRACE_MALLOC; \
- return asan_new(size, &stack)
-#define OPERATOR_NEW_BODY_ARRAY \
- GET_STACK_TRACE_MALLOC; \
- void *res = asan_new_array(size, &stack); \
- if (UNLIKELY(!res)) \
- ReportOutOfMemory(size, &stack); \
- return res
+ return NewImplNothrow(size, &stack, [&]() { return asan_new(size, &stack); })
+#define OPERATOR_NEW_BODY_ARRAY \
+ GET_STACK_TRACE_MALLOC; \
+ return NewImplThrowing(size, &stack, \
+ [&]() { return asan_new_array(size, &stack); })
#define OPERATOR_NEW_BODY_ARRAY_NOTHROW \
GET_STACK_TRACE_MALLOC; \
- return asan_new_array(size, &stack)
-#define OPERATOR_NEW_BODY_ALIGN \
- GET_STACK_TRACE_MALLOC; \
- void *res = asan_new_aligned(size, static_cast<uptr>(align), &stack); \
- if (UNLIKELY(!res)) \
- ReportOutOfMemory(size, &stack); \
- return res
-#define OPERATOR_NEW_BODY_ALIGN_NOTHROW \
- GET_STACK_TRACE_MALLOC; \
- return asan_new_aligned(size, static_cast<uptr>(align), &stack)
-#define OPERATOR_NEW_BODY_ALIGN_ARRAY \
- GET_STACK_TRACE_MALLOC; \
- void *res = asan_new_array_aligned(size, static_cast<uptr>(align), &stack); \
- if (UNLIKELY(!res)) \
- ReportOutOfMemory(size, &stack); \
- return res
-#define OPERATOR_NEW_BODY_ALIGN_ARRAY_NOTHROW \
- GET_STACK_TRACE_MALLOC; \
- return asan_new_array_aligned(size, static_cast<uptr>(align), &stack)
+ return NewImplNothrow(size, &stack, \
+ [&]() { return asan_new_array(size, &stack); })
+#define OPERATOR_NEW_BODY_ALIGN \
+ GET_STACK_TRACE_MALLOC; \
+ return NewImplThrowing(size, &stack, [&]() { \
+ return asan_new_aligned(size, static_cast<uptr>(align), &stack); \
+ })
+#define OPERATOR_NEW_BODY_ALIGN_NOTHROW \
+ GET_STACK_TRACE_MALLOC; \
+ return NewImplNothrow(size, &stack, [&]() { \
+ return asan_new_aligned(size, static_cast<uptr>(align), &stack); \
+ })
+#define OPERATOR_NEW_BODY_ALIGN_ARRAY \
+ GET_STACK_TRACE_MALLOC; \
+ return NewImplThrowing(size, &stack, [&]() { \
+ return asan_new_array_aligned(size, static_cast<uptr>(align), &stack); \
+ })
+#define OPERATOR_NEW_BODY_ALIGN_ARRAY_NOTHROW \
+ GET_STACK_TRACE_MALLOC; \
+ return NewImplNothrow(size, &stack, [&]() { \
+ return asan_new_array_aligned(size, static_cast<uptr>(align), &stack); \
+ })
// On OS X it's not enough to just provide our own 'operator new' and
// 'operator delete' implementations, because they're going to be in the
@@ -110,11 +199,11 @@ void *operator new(size_t size) { OPERATOR_NEW_BODY; }
CXX_OPERATOR_ATTRIBUTE
void *operator new[](size_t size) { OPERATOR_NEW_BODY_ARRAY; }
CXX_OPERATOR_ATTRIBUTE
-void *operator new(size_t size, std::nothrow_t const &) {
+void* operator new(size_t size, std::nothrow_t const&) NOEXCEPT {
OPERATOR_NEW_BODY_NOTHROW;
}
CXX_OPERATOR_ATTRIBUTE
-void *operator new[](size_t size, std::nothrow_t const &) {
+void* operator new[](size_t size, std::nothrow_t const&) NOEXCEPT {
OPERATOR_NEW_BODY_ARRAY_NOTHROW;
}
CXX_OPERATOR_ATTRIBUTE
@@ -126,13 +215,13 @@ void *operator new[](size_t size, std::align_val_t align) {
OPERATOR_NEW_BODY_ALIGN_ARRAY;
}
CXX_OPERATOR_ATTRIBUTE
-void *operator new(size_t size, std::align_val_t align,
- std::nothrow_t const &) {
+void* operator new(size_t size, std::align_val_t align,
+ std::nothrow_t const&) NOEXCEPT {
OPERATOR_NEW_BODY_ALIGN_NOTHROW;
}
CXX_OPERATOR_ATTRIBUTE
-void *operator new[](size_t size, std::align_val_t align,
- std::nothrow_t const &) {
+void* operator new[](size_t size, std::align_val_t align,
+ std::nothrow_t const&) NOEXCEPT {
OPERATOR_NEW_BODY_ALIGN_ARRAY_NOTHROW;
}
@@ -178,11 +267,11 @@ void operator delete(void *ptr) NOEXCEPT { OPERATOR_DELETE_BODY; }
CXX_OPERATOR_ATTRIBUTE
void operator delete[](void *ptr) NOEXCEPT { OPERATOR_DELETE_BODY_ARRAY; }
CXX_OPERATOR_ATTRIBUTE
-void operator delete(void *ptr, std::nothrow_t const &) {
+void operator delete(void* ptr, std::nothrow_t const&) NOEXCEPT {
OPERATOR_DELETE_BODY;
}
CXX_OPERATOR_ATTRIBUTE
-void operator delete[](void *ptr, std::nothrow_t const &) {
+void operator delete[](void* ptr, std::nothrow_t const&) NOEXCEPT {
OPERATOR_DELETE_BODY_ARRAY;
}
CXX_OPERATOR_ATTRIBUTE
@@ -202,13 +291,13 @@ void operator delete[](void *ptr, std::align_val_t align) NOEXCEPT {
OPERATOR_DELETE_BODY_ALIGN_ARRAY;
}
CXX_OPERATOR_ATTRIBUTE
-void operator delete(void *ptr, std::align_val_t align,
- std::nothrow_t const &) {
+void operator delete(void* ptr, std::align_val_t align,
+ std::nothrow_t const&) NOEXCEPT {
OPERATOR_DELETE_BODY_ALIGN;
}
CXX_OPERATOR_ATTRIBUTE
-void operator delete[](void *ptr, std::align_val_t align,
- std::nothrow_t const &) {
+void operator delete[](void* ptr, std::align_val_t align,
+ std::nothrow_t const&) NOEXCEPT {
OPERATOR_DELETE_BODY_ALIGN_ARRAY;
}
CXX_OPERATOR_ATTRIBUTE
diff --git a/compiler-rt/lib/sanitizer_common/sanitizer_flags.inc b/compiler-rt/lib/sanitizer_common/sanitizer_flags.inc
index 5f449907f6011..1c6f774927086 100644
--- a/compiler-rt/lib/sanitizer_common/sanitizer_flags.inc
+++ b/compiler-rt/lib/sanitizer_common/sanitizer_flags.inc
@@ -83,8 +83,17 @@ COMMON_FLAG(
"detect_leaks=false, or if __lsan_do_leak_check() is called before the "
"handler has a chance to run.")
COMMON_FLAG(bool, allocator_may_return_null, false,
- "If false, the allocator will crash instead of returning 0 on "
- "out-of-memory.")
+ "Controls allocator behavior on out-of-memory. C-allocator entry "
+ "points (malloc / calloc / realloc / aligned_alloc / "
+ "posix_memalign): default=false reports and aborts, true returns "
+ "nullptr. AddressSanitizer's operator new always runs the "
+ "std::get_new_handler() chain first per [new.delete.single]/3+/4 "
+ "regardless of this flag; on chain exhaustion default=false "
+ "reports and aborts for both forms, true throws std::bad_alloc "
+ "(throwing form) or returns nullptr (nothrow form). On Windows "
+ "the asan runtime is built without exceptions, so the throwing "
+ "form aborts even with true; the nothrow form still returns "
+ "nullptr.")
COMMON_FLAG(bool, print_summary, true,
"If false, disable printing error summaries in addition to error "
"reports.")
diff --git a/compiler-rt/test/asan/TestCases/new_handler_invocation.cpp b/compiler-rt/test/asan/TestCases/new_handler_invocation.cpp
new file mode 100644
index 0000000000000..cd526eb764ccd
--- /dev/null
+++ b/compiler-rt/test/asan/TestCases/new_handler_invocation.cpp
@@ -0,0 +1,45 @@
+// Throwing operator new must invoke std::new_handler before throwing
+// std::bad_alloc, per [new.delete.single]/3 (and /4 for the nothrow form).
+// Opt-in via allocator_may_return_null=1: the handler chain runs regardless
+// of the flag, but the chain-exhausted action only becomes "throw bad_alloc"
+// when the flag is set; otherwise it's ReportOutOfMemory + Die().
+
+// RUN: %clangxx_asan -O0 %s -o %t
+// RUN: %env_asan_opts=allocator_may_return_null=1 %run %t 2>&1 | FileCheck %s
+
+// UNSUPPORTED: target={{.*windows.*}}
+// REQUIRES: stable-runtime
+
+#include <cstdio>
+#include <new>
+
+static const size_t kHugeSize =
+#if __LP64__ || defined(_WIN64)
+ (1ULL << 40) + 1;
+#else
+ (3UL << 30) + 1;
+#endif
+
+static int handler_calls = 0;
+
+static void my_handler() {
+ ++handler_calls;
+ fprintf(stderr, "handler call %d\n", handler_calls);
+ // Break the loop. A real handler would free memory and return; this
+ // allocation is unrecoverable so we throw to terminate.
+ throw std::bad_alloc();
+}
+
+int main() {
+ std::set_new_handler(my_handler);
+ try {
+ char *p = new char[kHugeSize];
+ fprintf(stderr, "FAIL: allocation unexpectedly returned %p\n", p);
+ } catch (const std::bad_alloc &) {
+ fprintf(stderr, "caught bad_alloc after %d handler call(s)\n",
+ handler_calls);
+ }
+ // CHECK: handler call 1
+ // CHECK: caught bad_alloc after 1 handler call(s)
+ return 0;
+}
diff --git a/compiler-rt/test/asan/TestCases/new_handler_throws_other.cpp b/compiler-rt/test/asan/TestCases/new_handler_throws_other.cpp
new file mode 100644
index 0000000000000..8d64959228dd7
--- /dev/null
+++ b/compiler-rt/test/asan/TestCases/new_handler_throws_other.cpp
@@ -0,0 +1,36 @@
+// If the registered std::new_handler throws an exception other than
+// std::bad_alloc, that exception must propagate out of operator new
+// unmodified. Opt-in via allocator_may_return_null=1.
+
+// RUN: %clangxx_asan -O0 %s -o %t
+// RUN: %env_asan_opts=allocator_may_return_null=1 %run %t 2>&1 | FileCheck %s
+
+// UNSUPPORTED: target={{.*windows.*}}
+// REQUIRES: stable-runtime
+
+#include <cstdio>
+#include <new>
+#include <stdexcept>
+
+static const size_t kHugeSize =
+#if __LP64__ || defined(_WIN64)
+ (1ULL << 40) + 1;
+#else
+ (3UL << 30) + 1;
+#endif
+
+static void my_handler() { throw std::runtime_error("oom-policy"); }
+
+int main() {
+ std::set_new_handler(my_handler);
+ try {
+ char *p = new char[kHugeSize];
+ fprintf(stderr, "FAIL: allocation unexpectedly returned %p\n", p);
+ } catch (const std::bad_alloc &) {
+ fprintf(stderr, "FAIL: caught bad_alloc instead of runtime_error\n");
+ } catch (const std::runtime_error &e) {
+ fprintf(stderr, "caught runtime_error: %s\n", e.what());
+ }
+ // CHECK: caught runtime_error: oom-policy
+ return 0;
+}
diff --git a/compiler-rt/test/asan/TestCases/nothrow_new_aligned_array_returns_null.cpp b/compiler-rt/test/asan/TestCases/nothrow_new_aligned_array_returns_null.cpp
new file mode 100644
index 0000000000000..1430c24f5cf57
--- /dev/null
+++ b/compiler-rt/test/asan/TestCases/nothrow_new_aligned_array_returns_null.cpp
@@ -0,0 +1,27 @@
+// Aligned array nothrow operator new must return nullptr on allocation
+// failure (OPERATOR_NEW_BODY_ALIGN_ARRAY_NOTHROW). Opt-in via
+// allocator_may_return_null=1.
+
+// RUN: %clangxx_asan -O0 -std=c++17 %s -o %t
+// RUN: %env_asan_opts=allocator_may_return_null=1 %run %t 2>&1 | FileCheck %s
+
+// REQUIRES: stable-runtime
+
+#include <cstdio>
+#include <new>
+
+struct alignas(64) HugeAligned {
+#if __LP64__ || defined(_WIN64)
+ char data[(1ULL << 40) + 1];
+#else
+ char data[(3UL << 30) + 1];
+#endif
+};
+
+int main() {
+ HugeAligned *p = new (std::nothrow) HugeAligned[1];
+ fprintf(stderr, "nothrow aligned array returned %s\n",
+ p ? "non-null" : "null");
+ // CHECK: nothrow aligned array returned null
+ return 0;
+}
diff --git a/compiler-rt/test/asan/TestCases/nothrow_new_aligned_single_returns_null.cpp b/compiler-rt/test/asan/TestCases/nothrow_new_aligned_single_returns_null.cpp
new file mode 100644
index 0000000000000..c0096f54bc255
--- /dev/null
+++ b/compiler-rt/test/asan/TestCases/nothrow_new_aligned_single_returns_null.cpp
@@ -0,0 +1,26 @@
+// Aligned single-object nothrow operator new must return nullptr on
+// allocation failure (OPERATOR_NEW_BODY_ALIGN_NOTHROW). Opt-in via
+// allocator_may_return_null=1.
+
+// RUN: %clangxx_asan -O0 -std=c++17 %s -o %t
+// RUN: %env_asan_opts=allocator_may_return_null=1 %run %t 2>&1 | FileCheck %s
+
+// REQUIRES: stable-runtime
+
+#include <cstdio>
+#include <new>
+
+struct alignas(64) HugeAligned {
+#if __LP64__ || defined(_WIN64)
+ char data[(1ULL << 40) + 1];
+#else
+ char data[(3UL << 30) + 1];
+#endif
+};
+
+int main() {
+ HugeAligned *p = new (std::nothrow) HugeAligned;
+ fprintf(stderr, "nothrow aligned returned %s\n", p ? "non-null" : "null");
+ // CHECK: nothrow aligned returned null
+ return 0;
+}
diff --git a/compiler-rt/test/asan/TestCases/nothrow_new_default_aborts.cpp b/compiler-rt/test/asan/TestCases/nothrow_new_default_aborts.cpp
new file mode 100644
index 0000000000000..f35b8c7bea92e
--- /dev/null
+++ b/compiler-rt/test/asan/TestCases/nothrow_new_default_aborts.cpp
@@ -0,0 +1,33 @@
+// With allocator_may_return_null=false (default), nothrow operator new
+// aborts on OOM. The handler chain runs (per [new.delete.single]/4); on
+// chain exhaustion the runtime emits the asan diagnostic and Die()s rather
+// than returning nullptr.
+
+// RUN: %clangxx_asan -O0 %s -o %t
+// RUN: not %run %t 2>&1 | FileCheck %s
+
+// REQUIRES: stable-runtime
+
+#include <cstdio>
+#include <new>
+
+static const size_t kHugeSize =
+#if __LP64__ || defined(_WIN64)
+ (1ULL << 40) + 1;
+#else
+ (3UL << 30) + 1;
+#endif
+
+int main() {
+ // No new_handler installed -> chain exhausts immediately -> default flag
+ // selects the abort path.
+ char *p = new (std::nothrow) char[kHugeSize];
+ fprintf(stderr, "FAIL: allocation unexpectedly returned %p\n", p);
+ return 0;
+}
+
+// Linux's secondary mmap fails first (out of memory) and Windows's
+// kMaxAllowedMallocSize check trips first (requested allocation size); both
+// prove the default-flag abort path was taken.
+// CHECK: AddressSanitizer: {{out of memory|requested allocation size}}
+// CHECK: ABORTING
diff --git a/compiler-rt/test/asan/TestCases/nothrow_new_returns_null.cpp b/compiler-rt/test/asan/TestCases/nothrow_new_returns_null.cpp
new file mode 100644
index 0000000000000..19e73ca3c58d7
--- /dev/null
+++ b/compiler-rt/test/asan/TestCases/nothrow_new_returns_null.cpp
@@ -0,0 +1,26 @@
+// Per [new.delete.single]/4, nothrow operator new must return nullptr on
+// allocation failure after running the new_handler chain. Opt-in via
+// allocator_may_return_null=1; the default-flag abort case is covered by
+// nothrow_new_default_aborts.cpp.
+
+// RUN: %clangxx_asan -O0 %s -o %t
+// RUN: %env_asan_opts=allocator_may_return_null=1 %run %t 2>&1 | FileCheck %s
+
+// REQUIRES: stable-runtime
+
+#include <cstdio>
+#include <new>
+
+static const size_t kHugeSize =
+#if __LP64__ || defined(_WIN64)
+ (1ULL << 40) + 1;
+#else
+ (3UL << 30) + 1;
+#endif
+
+int main() {
+ char *p = new (std::nothrow) char[kHugeSize];
+ fprintf(stderr, "nothrow returned %s\n", p ? "non-null" : "null");
+ // CHECK: nothrow returned null
+ return 0;
+}
diff --git a/compiler-rt/test/asan/TestCases/nothrow_new_single_returns_null.cpp b/compiler-rt/test/asan/TestCases/nothrow_new_single_returns_null.cpp
new file mode 100644
index 0000000000000..23c264c618437
--- /dev/null
+++ b/compiler-rt/test/asan/TestCases/nothrow_new_single_returns_null.cpp
@@ -0,0 +1,25 @@
+// Single-object nothrow operator new must return nullptr on allocation
+// failure (OPERATOR_NEW_BODY_NOTHROW). Opt-in via allocator_may_return_null=1.
+
+// RUN: %clangxx_asan -O0 %s -o %t
+// RUN: %env_asan_opts=allocator_may_return_null=1 %run %t 2>&1 | FileCheck %s
+
+// REQUIRES: stable-runtime
+
+#include <cstdio>
+#include <new>
+
+struct alignas(1) Huge {
+#if __LP64__ || defined(_WIN64)
+ char data[(1ULL << 40) + 1];
+#else
+ char data[(3UL << 30) + 1];
+#endif
+};
+
+int main() {
+ Huge *p = new (std::nothrow) Huge;
+ fprintf(stderr, "nothrow returned %s\n", p ? "non-null" : "null");
+ // CHECK: nothrow returned null
+ return 0;
+}
diff --git a/compiler-rt/test/asan/TestCases/throw_bad_alloc_aligned.cpp b/compiler-rt/test/asan/TestCases/throw_bad_alloc_aligned.cpp
new file mode 100644
index 0000000000000..7384a2fa91f2b
--- /dev/null
+++ b/compiler-rt/test/asan/TestCases/throw_bad_alloc_aligned.cpp
@@ -0,0 +1,36 @@
+// Throwing aligned operator new must throw std::bad_alloc on allocation
+// failure, just like the unaligned form. Opt-in via allocator_may_return_null=1.
+
+// RUN: %clangxx_asan -O0 -std=c++17 %s -o %t
+// RUN: %env_asan_opts=allocator_may_return_null=1 %run %t 2>&1 | FileCheck %s
+
+// UNSUPPORTED: target={{.*windows.*}}
+// REQUIRES: stable-runtime
+
+#include <cstdio>
+#include <new>
+
+static const size_t kHugeSize =
+#if __LP64__ || defined(_WIN64)
+ (1ULL << 40) + 1;
+#else
+ (3UL << 30) + 1;
+#endif
+
+struct alignas(64) Aligned {
+ char data[1];
+};
+
+int main() {
+ bool caught = false;
+ try {
+ Aligned *p = new Aligned[kHugeSize];
+ fprintf(stderr, "FAIL: allocation unexpectedly returned %p\n", p);
+ } catch (const std::bad_alloc &) {
+ caught = true;
+ }
+ if (caught)
+ fprintf(stderr, "caught bad_alloc\n");
+ // CHECK: caught bad_alloc
+ return 0;
+}
diff --git a/compiler-rt/test/asan/TestCases/throw_bad_alloc_aligned_single.cpp b/compiler-rt/test/asan/TestCases/throw_bad_alloc_aligned_single.cpp
new file mode 100644
index 0000000000000..e9966cb459080
--- /dev/null
+++ b/compiler-rt/test/asan/TestCases/throw_bad_alloc_aligned_single.cpp
@@ -0,0 +1,34 @@
+// Aligned single-object throwing operator new must throw std::bad_alloc on
+// allocation failure (OPERATOR_NEW_BODY_ALIGN). Opt-in via
+// allocator_may_return_null=1.
+
+// RUN: %clangxx_asan -O0 -std=c++17 %s -o %t
+// RUN: %env_asan_opts=allocator_may_return_null=1 %run %t 2>&1 | FileCheck %s
+
+// UNSUPPORTED: target={{.*windows.*}}
+// REQUIRES: stable-runtime
+
+#include <cstdio>
+#include <new>
+
+struct alignas(64) HugeAligned {
+#if __LP64__ || defined(_WIN64)
+ char data[(1ULL << 40) + 1];
+#else
+ char data[(3UL << 30) + 1];
+#endif
+};
+
+int main() {
+ bool caught = false;
+ try {
+ HugeAligned *p = new HugeAligned;
+ fprintf(stderr, "FAIL: allocation unexpectedly returned %p\n", p);
+ } catch (const std::bad_alloc &) {
+ caught = true;
+ }
+ if (caught)
+ fprintf(stderr, "caught bad_alloc\n");
+ // CHECK: caught bad_alloc
+ return 0;
+}
diff --git a/compiler-rt/test/asan/TestCases/throw_bad_alloc_default_aborts.cpp b/compiler-rt/test/asan/TestCases/throw_bad_alloc_default_aborts.cpp
new file mode 100644
index 0000000000000..408747f083922
--- /dev/null
+++ b/compiler-rt/test/asan/TestCases/throw_bad_alloc_default_aborts.cpp
@@ -0,0 +1,31 @@
+// With allocator_may_return_null=false (default), throwing operator new
+// aborts on OOM. The handler chain runs (per [new.delete.single]/3); on
+// chain exhaustion the runtime emits the asan ERROR + SUMMARY block and
+// Die()s.
+
+// RUN: %clangxx_asan -O0 %s -o %t
+// RUN: not %run %t 2>&1 | FileCheck %s
+
+// UNSUPPORTED: target={{.*windows.*}}
+// REQUIRES: stable-runtime
+
+#include <cstdio>
+#include <new>
+
+static const size_t kHugeSize =
+#if __LP64__ || defined(_WIN64)
+ (1ULL << 40) + 1;
+#else
+ (3UL << 30) + 1;
+#endif
+
+int main() {
+ // No new_handler installed -> chain exhausts immediately -> default flag
+ // selects the abort path.
+ char *p = new char[kHugeSize];
+ fprintf(stderr, "FAIL: allocation unexpectedly returned %p\n", p);
+ return 0;
+}
+
+// CHECK: AddressSanitizer: out of memory
+// CHECK: ABORTING
diff --git a/compiler-rt/test/asan/TestCases/throw_bad_alloc_oversize.cpp b/compiler-rt/test/asan/TestCases/throw_bad_alloc_oversize.cpp
new file mode 100644
index 0000000000000..6fd4983139279
--- /dev/null
+++ b/compiler-rt/test/asan/TestCases/throw_bad_alloc_oversize.cpp
@@ -0,0 +1,34 @@
+// Throwing operator new must throw std::bad_alloc on allocation failure
+// (here triggered by an oversize request) rather than aborting. Opt-in via
+// allocator_may_return_null=1.
+
+// RUN: %clangxx_asan -O0 %s -o %t
+// RUN: %env_asan_opts=allocator_may_return_null=1 %run %t 2>&1 | FileCheck %s
+
+// Windows asan can't throw bad_alloc; see asan_new_delete.cpp.
+// UNSUPPORTED: target={{.*windows.*}}
+// REQUIRES: stable-runtime
+
+#include <cstdio>
+#include <new>
+
+static const size_t kHugeSize =
+#if __LP64__ || defined(_WIN64)
+ (1ULL << 40) + 1;
+#else
+ (3UL << 30) + 1;
+#endif
+
+int main() {
+ bool caught = false;
+ try {
+ char *p = new char[kHugeSize];
+ fprintf(stderr, "FAIL: allocation unexpectedly returned %p\n", p);
+ } catch (const std::bad_alloc &) {
+ caught = true;
+ }
+ if (caught)
+ fprintf(stderr, "caught bad_alloc\n");
+ // CHECK: caught bad_alloc
+ return 0;
+}
diff --git a/compiler-rt/test/asan/TestCases/throw_bad_alloc_single.cpp b/compiler-rt/test/asan/TestCases/throw_bad_alloc_single.cpp
new file mode 100644
index 0000000000000..da019cb0a9f15
--- /dev/null
+++ b/compiler-rt/test/asan/TestCases/throw_bad_alloc_single.cpp
@@ -0,0 +1,35 @@
+// Single-object throwing operator new must throw std::bad_alloc on
+// allocation failure (OPERATOR_NEW_BODY). Opt-in via
+// allocator_may_return_null=1.
+
+// RUN: %clangxx_asan -O0 %s -o %t
+// RUN: %env_asan_opts=allocator_may_return_null=1 %run %t 2>&1 | FileCheck %s
+
+// UNSUPPORTED: target={{.*windows.*}}
+// REQUIRES: stable-runtime
+
+#include <cstdio>
+#include <new>
+
+// Single object whose size alone exceeds the allocator's limit.
+struct alignas(1) Huge {
+#if __LP64__ || defined(_WIN64)
+ char data[(1ULL << 40) + 1];
+#else
+ char data[(3UL << 30) + 1];
+#endif
+};
+
+int main() {
+ bool caught = false;
+ try {
+ Huge *p = new Huge;
+ fprintf(stderr, "FAIL: allocation unexpectedly returned %p\n", p);
+ } catch (const std::bad_alloc &) {
+ caught = true;
+ }
+ if (caught)
+ fprintf(stderr, "caught bad_alloc\n");
+ // CHECK: caught bad_alloc
+ return 0;
+}
diff --git a/compiler-rt/test/sanitizer_common/TestCases/Linux/allocator_returns_null_std.cpp b/compiler-rt/test/sanitizer_common/TestCases/Linux/allocator_returns_null_std.cpp
index 812cf049f2a9b..3d315cff02fd8 100644
--- a/compiler-rt/test/sanitizer_common/TestCases/Linux/allocator_returns_null_std.cpp
+++ b/compiler-rt/test/sanitizer_common/TestCases/Linux/allocator_returns_null_std.cpp
@@ -27,4 +27,8 @@ int main(int argc, char **argv) {
}
// CHECK: #{{[0-9]+.*}}allocator_returns_null_std.cpp
-// CHECK: {{SUMMARY: .*Sanitizer: allocation-size-too-big.*allocator_returns_null_std.cpp.*}} in main
+// std::vector::resize uses throwing operator new[]. asan forces
+// may_return_null=true on Allocate so std::get_new_handler() runs first; the
+// chain-exhausted abort path emits "out-of-memory" rather than the in-place
+// "allocation-size-too-big" emitted by other sanitizers.
+// CHECK: {{SUMMARY: .*Sanitizer: (allocation-size-too-big|out-of-memory).*allocator_returns_null_std.cpp.*}} in main
diff --git a/compiler-rt/test/sanitizer_common/TestCases/allocator_returns_null.cpp b/compiler-rt/test/sanitizer_common/TestCases/allocator_returns_null.cpp
index ca6f637b9a3f5..de7dcf0712197 100644
--- a/compiler-rt/test/sanitizer_common/TestCases/allocator_returns_null.cpp
+++ b/compiler-rt/test/sanitizer_common/TestCases/allocator_returns_null.cpp
@@ -28,14 +28,20 @@
// RUN: | FileCheck %s --check-prefix=CHECK-NULL
// RUN: %env_tool_opts=allocator_may_return_null=0 not %run %t new 2>&1 \
// RUN: | FileCheck %s --check-prefix=CHECK-nCRASH
-// RUN: %env_tool_opts=allocator_may_return_null=1 not %run %t new 2>&1 \
-// RUN: | FileCheck %s --check-prefix=CHECK-nCRASH-OOM
+// flag=1 + throwing new: asan throws bad_alloc, the test catches it and
+// converges to CHECK-NULL; other sanitizers abort inside operator new.
+// RUN: %if asan %{ %env_tool_opts=allocator_may_return_null=1 %run %t new 2>&1 | FileCheck %s --check-prefix=CHECK-NULL %}
+// RUN: %if !asan %{ %env_tool_opts=allocator_may_return_null=1 not %run %t new 2>&1 | FileCheck %s --check-prefix=CHECK-nCRASH-OOM %}
// RUN: %env_tool_opts=allocator_may_return_null=0 not %run %t new-nothrow 2>&1 \
// RUN: | FileCheck %s --check-prefix=CHECK-nnCRASH
// RUN: %env_tool_opts=allocator_may_return_null=1 %run %t new-nothrow 2>&1 \
// RUN: | FileCheck %s --check-prefix=CHECK-NULL
// TODO(alekseyshl): win32 is disabled due to failing errno tests, fix it there.
+// Windows asan would also fail the flag=1 + new cell above: its runtime is
+// built without exceptions and never throws bad_alloc, so the throwing form
+// always falls back to ReportOutOfMemory + Die(). Re-enabling this test on
+// Windows requires tightening the %if asan dispatch to exclude Windows.
// UNSUPPORTED: ubsan, target={{.*windows-msvc.*}}
#include <assert.h>
@@ -79,7 +85,14 @@ int main(int argc, char **argv) {
assert(*t == 42);
free(t);
} else if (!strcmp(action, "new")) {
- x = operator new(kMaxAllowedMallocSizePlusOne);
+ try {
+ x = operator new(kMaxAllowedMallocSizePlusOne);
+ assert(0 && "throwing operator new returned without throwing -- "
+ "violates [basic.stc.dynamic.allocation]/3");
+ } catch (const std::bad_alloc &) {
+ x = nullptr;
+ errno = ENOMEM;
+ }
} else if (!strcmp(action, "new-nothrow")) {
x = operator new(kMaxAllowedMallocSizePlusOne, std::nothrow);
} else {
@@ -110,13 +123,18 @@ int main(int argc, char **argv) {
// CHECK-mrCRASH: {{SUMMARY: .*Sanitizer: allocation-size-too-big.*allocator_returns_null.cpp.*}} in main
// CHECK-nCRASH: new:
// CHECK-nCRASH: #{{[0-9]+.*}}allocator_returns_null.cpp
-// CHECK-nCRASH: {{SUMMARY: .*Sanitizer: allocation-size-too-big.*allocator_returns_null.cpp.*}} in main
+// asan's throwing/nothrow operator new forces may_return_null=true on
+// Allocate so std::get_new_handler() runs first; the chain-exhausted abort
+// path emits "out-of-memory" rather than the in-place
+// "allocation-size-too-big" emitted by other sanitizers. Same alternation
+// applies to CHECK-nnCRASH.
+// CHECK-nCRASH: {{SUMMARY: .*Sanitizer: (allocation-size-too-big|out-of-memory).*allocator_returns_null.cpp.*}} in main
// CHECK-nCRASH-OOM: new:
-// CHECK-nCRASH-O#{{[0-9]+.*}}allocator_returns_null.cpp
+// CHECK-nCRASH-OOM: #{{[0-9]+.*}}allocator_returns_null.cpp
// CHECK-nCRASH-OOM: {{SUMMARY: .*Sanitizer: out-of-memory.*allocator_returns_null.cpp.*}} in main
// CHECK-nnCRASH: new-nothrow:
// CHECK-nnCRASH: #{{[0-9]+.*}}allocator_returns_null.cpp
-// CHECK-nnCRASH: {{SUMMARY: .*Sanitizer: allocation-size-too-big.*allocator_returns_null.cpp.*}} in main
+// CHECK-nnCRASH: {{SUMMARY: .*Sanitizer: (allocation-size-too-big|out-of-memory).*allocator_returns_null.cpp.*}} in main
-// CHECK-NULL: {{malloc|calloc|calloc-overflow|realloc|realloc-after-malloc|new-nothrow}}
+// CHECK-NULL: {{malloc|calloc|calloc-overflow|realloc|realloc-after-malloc|new-nothrow|new}}
// CHECK-NULL: errno: 12, x: 0
diff --git a/compiler-rt/test/sanitizer_common/TestCases/max_allocation_size.cpp b/compiler-rt/test/sanitizer_common/TestCases/max_allocation_size.cpp
index 2fde16fbed3d2..94e71b9e8835c 100644
--- a/compiler-rt/test/sanitizer_common/TestCases/max_allocation_size.cpp
+++ b/compiler-rt/test/sanitizer_common/TestCases/max_allocation_size.cpp
@@ -28,8 +28,10 @@
// RUN: | FileCheck %s --check-prefix=CHECK-NULL
// RUN: %env_tool_opts=max_allocation_size_mb=2:allocator_may_return_null=0 \
// RUN: not %run %t new 2>&1 | FileCheck %s --check-prefix=CHECK-nCRASH
-// RUN: %env_tool_opts=max_allocation_size_mb=2:allocator_may_return_null=1 \
-// RUN: not %run %t new 2>&1 | FileCheck %s --check-prefix=CHECK-nCRASH-OOM
+// flag=1 + throwing new: asan throws bad_alloc, allocate() catches it and
+// converges to CHECK-NULL; other sanitizers abort inside operator new.
+// RUN: %if asan %{ %env_tool_opts=max_allocation_size_mb=2:allocator_may_return_null=1 %run %t new 2>&1 | FileCheck %s --check-prefix=CHECK-NULL %}
+// RUN: %if !asan %{ %env_tool_opts=max_allocation_size_mb=2:allocator_may_return_null=1 not %run %t new 2>&1 | FileCheck %s --check-prefix=CHECK-nCRASH-OOM %}
// RUN: %env_tool_opts=max_allocation_size_mb=2:allocator_may_return_null=0 \
// RUN: not %run %t new-nothrow 2>&1 \
// RUN: | FileCheck %s --check-prefix=CHECK-nnCRASH
@@ -41,6 +43,10 @@
// RUN: %run %t strndup 2>&1 | FileCheck %s --check-prefix=CHECK-NULL
// win32 is disabled due to failing errno tests.
+// Windows asan would also fail the flag=1 + new cell above: its runtime is
+// built without exceptions and never throws bad_alloc, so the throwing form
+// always falls back to ReportOutOfMemory + Die(). Re-enabling this test on
+// Windows requires tightening the %if asan dispatch to exclude Windows.
// UNSUPPORTED: ubsan, target={{.*windows-msvc.*}}
// Symbolizer needs to allocated memory when reporting.
@@ -70,8 +76,18 @@ static void *allocate(const char *Action, size_t Size) {
free(P);
return nullptr;
}
- if (!strcmp(Action, "new"))
- return ::operator new(Size);
+ if (!strcmp(Action, "new")) {
+ try {
+ void *p = ::operator new(Size);
+ assert(p != nullptr &&
+ "throwing operator new returned nullptr without throwing -- "
+ "violates [basic.stc.dynamic.allocation]/3");
+ return p;
+ } catch (const std::bad_alloc &) {
+ errno = ENOMEM;
+ return nullptr;
+ }
+ }
if (!strcmp(Action, "new-nothrow"))
return ::operator new(Size, std::nothrow);
if (!strcmp(Action, "strndup")) {
@@ -136,18 +152,23 @@ int main(int Argc, char **Argv) {
// CHECK-mrCRASH: {{SUMMARY: .*Sanitizer: allocation-size-too-big.* in allocate}}
// CHECK-nCRASH: new:
// CHECK-nCRASH: #{{[0-9]+.*}}max_allocation_size.cpp
-// CHECK-nCRASH: {{SUMMARY: .*Sanitizer: allocation-size-too-big.* in allocate}}
+// asan's throwing/nothrow operator new forces may_return_null=true on
+// Allocate so std::get_new_handler() runs first; the chain-exhausted abort
+// path emits "out-of-memory" rather than the in-place
+// "allocation-size-too-big" emitted by other sanitizers. Same alternation
+// applies to CHECK-nnCRASH.
+// CHECK-nCRASH: {{SUMMARY: .*Sanitizer: (allocation-size-too-big|out-of-memory).* in allocate}}
// CHECK-nCRASH-OOM: new:
// CHECK-nCRASH-OOM: #{{[0-9]+.*}}max_allocation_size.cpp
// CHECK-nCRASH-OOM: {{SUMMARY: .*Sanitizer: out-of-memory.* in allocate}}
// CHECK-nnCRASH: new-nothrow:
// CHECK-nnCRASH: #{{[0-9]+.*}}max_allocation_size.cpp
-// CHECK-nnCRASH: {{SUMMARY: .*Sanitizer: allocation-size-too-big.* in allocate}}
+// CHECK-nnCRASH: {{SUMMARY: .*Sanitizer: (allocation-size-too-big|out-of-memory).* in allocate}}
// CHECK-sCRASH: strndup:
// CHECK-sCRASH: #{{[0-9]+.*}}max_allocation_size.cpp
// CHECK-sCRASH: {{SUMMARY: .*Sanitizer: allocation-size-too-big.*}}
-// CHECK-NULL: {{malloc|calloc|calloc-overflow|realloc|realloc-after-malloc|new-nothrow|strndup}}
+// CHECK-NULL: {{malloc|calloc|calloc-overflow|realloc|realloc-after-malloc|new-nothrow|new|strndup}}
// CHECK-NULL: errno: 12, P: 0
//
// CHECK-NOTNULL-NOT: P: 0
More information about the llvm-commits
mailing list