[compiler-rt] [scudo] Handle failed mmap for allocation ring buffer (PR #71817)

[Florian Mayer via llvm-commits]

https://github.com/fmayer created https://github.com/llvm/llvm-project/pull/71817

None

>From 02f73587c36a808922bf365b460940ccf4ea398c Mon Sep 17 00:00:00 2001
From: Florian Mayer <fmayer at google.com>
Date: Thu, 9 Nov 2023 06:09:14 -0800
Subject: [PATCH] [scudo] Handle failed mmap for allocation ring buffer

---
 compiler-rt/lib/scudo/standalone/combined.h | 11 +++++++++--
 1 file changed, 9 insertions(+), 2 deletions(-)

diff --git a/compiler-rt/lib/scudo/standalone/combined.h b/compiler-rt/lib/scudo/standalone/combined.h
index b1700e5ecef7f5b..088e27eeb4c7439 100644
--- a/compiler-rt/lib/scudo/standalone/combined.h
+++ b/compiler-rt/lib/scudo/standalone/combined.h
@@ -1281,7 +1281,8 @@ class Allocator {
 
   void storeSecondaryAllocationStackMaybe(const Options &Options, void *Ptr,
                                           uptr Size) {
-    if (!UNLIKELY(Options.get(OptionBit::TrackAllocationStacks)))
+    if (!UNLIKELY(Options.get(OptionBit::TrackAllocationStacks)) ||
+        RawRingBuffer == nullptr)
       return;
 
     u32 Trace = collectStackTrace();
@@ -1296,7 +1297,8 @@ class Allocator {
 
   void storeDeallocationStackMaybe(const Options &Options, void *Ptr,
                                    u8 PrevTag, uptr Size) {
-    if (!UNLIKELY(Options.get(OptionBit::TrackAllocationStacks)))
+    if (!UNLIKELY(Options.get(OptionBit::TrackAllocationStacks)) ||
+        RawRingBuffer == nullptr)
       return;
 
     auto *Ptr32 = reinterpret_cast<u32 *>(Ptr);
@@ -1501,6 +1503,11 @@ class Allocator {
                 getPageSizeCached()),
         "scudo:ring_buffer");
     RawRingBuffer = reinterpret_cast<char *>(MemMap.getBase());
+    if (RawRingBuffer == nullptr) {
+      Printf("Failed to allocate allocation ring buffer of size %d",
+             getFlags()->allocation_ring_buffer_size);
+      return;
+    }
     auto *RingBuffer = reinterpret_cast<AllocationRingBuffer *>(RawRingBuffer);
     RingBuffer->MemMap = MemMap;
     RingBuffer->Size = AllocationRingBufferSize;