[compiler-rt] resize stack depot for allocation ring buffer (PR #74515)
via llvm-commits
llvm-commits at lists.llvm.org
Tue Dec 5 11:06:54 PST 2023
llvmbot wrote:
<!--LLVM PR SUMMARY COMMENT-->
@llvm/pr-subscribers-compiler-rt-sanitizer
Author: Florian Mayer (fmayer)
<details>
<summary>Changes</summary>
---
Full diff: https://github.com/llvm/llvm-project/pull/74515.diff
5 Files Affected:
- (modified) compiler-rt/lib/scudo/standalone/combined.h (+61-21)
- (modified) compiler-rt/lib/scudo/standalone/fuzz/get_error_info_fuzzer.cpp (+10-9)
- (modified) compiler-rt/lib/scudo/standalone/platform.h (-10)
- (modified) compiler-rt/lib/scudo/standalone/stack_depot.h (+50-30)
- (modified) compiler-rt/lib/scudo/standalone/wrappers_c_bionic.cpp (+1-1)
``````````diff
diff --git a/compiler-rt/lib/scudo/standalone/combined.h b/compiler-rt/lib/scudo/standalone/combined.h
index 72e5caa026e4d..c6027ae25f318 100644
--- a/compiler-rt/lib/scudo/standalone/combined.h
+++ b/compiler-rt/lib/scudo/standalone/combined.h
@@ -14,6 +14,7 @@
#include "flags.h"
#include "flags_parser.h"
#include "local_cache.h"
+#include "mem_map.h"
#include "memtag.h"
#include "options.h"
#include "quarantine.h"
@@ -289,7 +290,9 @@ class Allocator {
uptr Size =
android_unsafe_frame_pointer_chase(Stack, MaxTraceSize + DiscardFrames);
Size = Min<uptr>(Size, MaxTraceSize + DiscardFrames);
- return Depot.insert(Stack + Min<uptr>(DiscardFrames, Size), Stack + Size);
+ return reinterpret_cast<StackDepot *>(RawStackDepot)
+ ->insert(RawStackDepot, Stack + Min<uptr>(DiscardFrames, Size),
+ Stack + Size);
#else
return 0;
#endif
@@ -916,8 +919,14 @@ class Allocator {
Primary.Options.clear(OptionBit::AddLargeAllocationSlack);
}
- const char *getStackDepotAddress() const {
- return reinterpret_cast<const char *>(&Depot);
+ const char *getStackDepotAddress() {
+ initThreadMaybe();
+ return RawStackDepot;
+ }
+
+ uptr getStackDepotSize() {
+ initThreadMaybe();
+ return StackDepotSize;
}
const char *getRegionInfoArrayAddress() const {
@@ -954,13 +963,14 @@ class Allocator {
static const uptr MaxTraceSize = 64;
- static void collectTraceMaybe(const StackDepot *Depot,
+ static void collectTraceMaybe(const char *RawStackDepot,
uintptr_t (&Trace)[MaxTraceSize], u32 Hash) {
+ auto *Depot = reinterpret_cast<const StackDepot *>(RawStackDepot);
uptr RingPos, Size;
- if (!Depot->find(Hash, &RingPos, &Size))
+ if (!Depot->find(RawStackDepot, Hash, &RingPos, &Size))
return;
for (unsigned I = 0; I != Size && I != MaxTraceSize; ++I)
- Trace[I] = static_cast<uintptr_t>((*Depot)[RingPos + I]);
+ Trace[I] = static_cast<uintptr_t>(Depot->at(RawStackDepot, RingPos + I));
}
static void getErrorInfo(struct scudo_error_info *ErrorInfo,
@@ -973,25 +983,24 @@ class Allocator {
MemoryAddr + MemorySize < MemoryAddr)
return;
- auto *Depot = reinterpret_cast<const StackDepot *>(DepotPtr);
size_t NextErrorReport = 0;
// Check for OOB in the current block and the two surrounding blocks. Beyond
// that, UAF is more likely.
if (extractTag(FaultAddr) != 0)
- getInlineErrorInfo(ErrorInfo, NextErrorReport, FaultAddr, Depot,
+ getInlineErrorInfo(ErrorInfo, NextErrorReport, FaultAddr, DepotPtr,
RegionInfoPtr, Memory, MemoryTags, MemoryAddr,
MemorySize, 0, 2);
// Check the ring buffer. For primary allocations this will only find UAF;
// for secondary allocations we can find either UAF or OOB.
- getRingBufferErrorInfo(ErrorInfo, NextErrorReport, FaultAddr, Depot,
+ getRingBufferErrorInfo(ErrorInfo, NextErrorReport, FaultAddr, DepotPtr,
RingBufferPtr);
// Check for OOB in the 28 blocks surrounding the 3 we checked earlier.
// Beyond that we are likely to hit false positives.
if (extractTag(FaultAddr) != 0)
- getInlineErrorInfo(ErrorInfo, NextErrorReport, FaultAddr, Depot,
+ getInlineErrorInfo(ErrorInfo, NextErrorReport, FaultAddr, DepotPtr,
RegionInfoPtr, Memory, MemoryTags, MemoryAddr,
MemorySize, 2, 16);
}
@@ -1039,7 +1048,8 @@ class Allocator {
uptr GuardedAllocSlotSize = 0;
#endif // GWP_ASAN_HOOKS
- StackDepot Depot;
+ char *RawStackDepot = nullptr;
+ uptr StackDepotSize = 0;
struct AllocationRingBuffer {
struct Entry {
@@ -1255,7 +1265,8 @@ class Allocator {
}
void storePrimaryAllocationStackMaybe(const Options &Options, void *Ptr) {
- if (!UNLIKELY(Options.get(OptionBit::TrackAllocationStacks)))
+ if (!UNLIKELY(Options.get(OptionBit::TrackAllocationStacks)) ||
+ !RawRingBuffer)
return;
auto *Ptr32 = reinterpret_cast<u32 *>(Ptr);
Ptr32[MemTagAllocationTraceIndex] = collectStackTrace();
@@ -1288,7 +1299,8 @@ class Allocator {
void storeSecondaryAllocationStackMaybe(const Options &Options, void *Ptr,
uptr Size) {
- if (!UNLIKELY(Options.get(OptionBit::TrackAllocationStacks)))
+ if (!UNLIKELY(Options.get(OptionBit::TrackAllocationStacks)) ||
+ !RawRingBuffer)
return;
u32 Trace = collectStackTrace();
@@ -1303,7 +1315,8 @@ class Allocator {
void storeDeallocationStackMaybe(const Options &Options, void *Ptr,
u8 PrevTag, uptr Size) {
- if (!UNLIKELY(Options.get(OptionBit::TrackAllocationStacks)))
+ if (!UNLIKELY(Options.get(OptionBit::TrackAllocationStacks)) ||
+ !RawRingBuffer)
return;
auto *Ptr32 = reinterpret_cast<u32 *>(Ptr);
@@ -1324,7 +1337,7 @@ class Allocator {
static void getInlineErrorInfo(struct scudo_error_info *ErrorInfo,
size_t &NextErrorReport, uintptr_t FaultAddr,
- const StackDepot *Depot,
+ const char *RawStackDepot,
const char *RegionInfoPtr, const char *Memory,
const char *MemoryTags, uintptr_t MemoryAddr,
size_t MemorySize, size_t MinDistance,
@@ -1389,8 +1402,10 @@ class Allocator {
UntaggedFaultAddr < ChunkAddr ? BUFFER_UNDERFLOW : BUFFER_OVERFLOW;
R->allocation_address = ChunkAddr;
R->allocation_size = Header.SizeOrUnusedBytes;
- collectTraceMaybe(Depot, R->allocation_trace,
- Data[MemTagAllocationTraceIndex]);
+ if (RawStackDepot) {
+ collectTraceMaybe(RawStackDepot, R->allocation_trace,
+ Data[MemTagAllocationTraceIndex]);
+ }
R->allocation_tid = Data[MemTagAllocationTidIndex];
return NextErrorReport == NumErrorReports;
};
@@ -1407,11 +1422,11 @@ class Allocator {
static void getRingBufferErrorInfo(struct scudo_error_info *ErrorInfo,
size_t &NextErrorReport,
uintptr_t FaultAddr,
- const StackDepot *Depot,
+ const char *RawStackDepot,
const char *RingBufferPtr) {
auto *RingBuffer =
reinterpret_cast<const AllocationRingBuffer *>(RingBufferPtr);
- if (!RingBuffer || RingBuffer->Size == 0)
+ if (!RingBuffer || RingBuffer->Size == 0 || !RawStackDepot)
return;
uptr Pos = atomic_load_relaxed(&RingBuffer->Pos);
@@ -1470,9 +1485,10 @@ class Allocator {
R->allocation_address = UntaggedEntryPtr;
R->allocation_size = EntrySize;
- collectTraceMaybe(Depot, R->allocation_trace, AllocationTrace);
+ collectTraceMaybe(RawStackDepot, R->allocation_trace, AllocationTrace);
R->allocation_tid = AllocationTid;
- collectTraceMaybe(Depot, R->deallocation_trace, DeallocationTrace);
+ collectTraceMaybe(RawStackDepot, R->deallocation_trace,
+ DeallocationTrace);
R->deallocation_tid = DeallocationTid;
}
}
@@ -1501,6 +1517,27 @@ class Allocator {
return;
u32 AllocationRingBufferSize =
static_cast<u32>(getFlags()->allocation_ring_buffer_size);
+ // We store alloc and free stacks for each entry.
+ constexpr auto kStacksPerRingBufferEntry = 2;
+ u32 TabSize = static_cast<u32>(roundUpPowerOfTwo(kStacksPerRingBufferEntry *
+ AllocationRingBufferSize));
+ constexpr auto kFramesPerStack = 8;
+ static_assert(isPowerOfTwo(kFramesPerStack));
+ u32 RingSize = static_cast<u32>(TabSize * kFramesPerStack);
+ DCHECK(isPowerOfTwo(RingSize));
+ static_assert(sizeof(StackDepot) % alignof(atomic_u64) == 0);
+
+ StackDepotSize =
+ roundUp(sizeof(StackDepot) + sizeof(atomic_u64) * RingSize +
+ sizeof(atomic_u32) * TabSize,
+ getPageSizeCached());
+ MemMapT DepotMap;
+ DepotMap.map(
+ /*Addr=*/0U, StackDepotSize, "scudo:stack_depot");
+ RawStackDepot = reinterpret_cast<char *>(DepotMap.getBase());
+ auto *Depot = reinterpret_cast<StackDepot *>(DepotMap.getBase());
+ Depot->init(DepotMap, RingSize, TabSize);
+
MemMapT MemMap;
MemMap.map(
/*Addr=*/0U,
@@ -1524,6 +1561,9 @@ class Allocator {
MemMap.unmap(MemMap.getBase(), MemMap.getCapacity());
}
RawRingBuffer = nullptr;
+ if (RawStackDepot) {
+ reinterpret_cast<StackDepot *>(RawStackDepot)->free();
+ }
}
static constexpr size_t ringBufferSizeInBytes(u32 AllocationRingBufferSize) {
diff --git a/compiler-rt/lib/scudo/standalone/fuzz/get_error_info_fuzzer.cpp b/compiler-rt/lib/scudo/standalone/fuzz/get_error_info_fuzzer.cpp
index 74456450a4761..dd3da623ca5e9 100644
--- a/compiler-rt/lib/scudo/standalone/fuzz/get_error_info_fuzzer.cpp
+++ b/compiler-rt/lib/scudo/standalone/fuzz/get_error_info_fuzzer.cpp
@@ -29,13 +29,13 @@ extern "C" int LLVMFuzzerTestOneInput(uint8_t *Data, size_t Size) {
size_t MemorySize = (MemoryAndTags.length() / 17) * 16;
const char *MemoryTags = Memory + MemorySize;
- std::string StackDepotBytes =
+ scudo::StackDepot Depot;
+ std::string DepotRingBytes =
FDP.ConsumeRandomLengthString(FDP.remaining_bytes());
- std::vector<char> StackDepot(sizeof(scudo::StackDepot), 0);
- for (size_t i = 0; i < StackDepotBytes.length() && i < StackDepot.size();
- ++i) {
- StackDepot[i] = StackDepotBytes[i];
- }
+ std::string DepotTabBytes =
+ FDP.ConsumeRandomLengthString(FDP.remaining_bytes());
+ Depot.SetRingForFuzzing(DepotRingBytes.c_str(), DepotRingBytes.size());
+ Depot.SetTabForFuzzing(DepotTabBytes.c_str(), DepotTabBytes.size());
std::string RegionInfoBytes =
FDP.ConsumeRandomLengthString(FDP.remaining_bytes());
@@ -52,8 +52,9 @@ extern "C" int LLVMFuzzerTestOneInput(uint8_t *Data, size_t Size) {
return 0;
scudo_error_info ErrorInfo;
- AllocatorT::getErrorInfo(&ErrorInfo, FaultAddr, StackDepot.data(),
- RegionInfo.data(), RingBufferBytes.data(), Memory,
- MemoryTags, MemoryAddr, MemorySize);
+ AllocatorT::getErrorInfo(&ErrorInfo, FaultAddr,
+ reinterpret_cast<char *>(&Depot), RegionInfo.data(),
+ RingBufferBytes.data(), Memory, MemoryTags,
+ MemoryAddr, MemorySize);
return 0;
}
diff --git a/compiler-rt/lib/scudo/standalone/platform.h b/compiler-rt/lib/scudo/standalone/platform.h
index b71a86be7669f..5af1275e32d2b 100644
--- a/compiler-rt/lib/scudo/standalone/platform.h
+++ b/compiler-rt/lib/scudo/standalone/platform.h
@@ -63,16 +63,6 @@
#define SCUDO_CAN_USE_MTE (SCUDO_LINUX || SCUDO_TRUSTY)
#endif
-// Use smaller table sizes for fuzzing in order to reduce input size.
-// Trusty just has less available memory.
-#ifndef SCUDO_SMALL_STACK_DEPOT
-#if defined(SCUDO_FUZZ) || SCUDO_TRUSTY
-#define SCUDO_SMALL_STACK_DEPOT 1
-#else
-#define SCUDO_SMALL_STACK_DEPOT 0
-#endif
-#endif
-
#ifndef SCUDO_ENABLE_HOOKS
#define SCUDO_ENABLE_HOOKS 0
#endif
diff --git a/compiler-rt/lib/scudo/standalone/stack_depot.h b/compiler-rt/lib/scudo/standalone/stack_depot.h
index 12c35eb2a4f33..b812658c95463 100644
--- a/compiler-rt/lib/scudo/standalone/stack_depot.h
+++ b/compiler-rt/lib/scudo/standalone/stack_depot.h
@@ -10,6 +10,8 @@
#define SCUDO_STACK_DEPOT_H_
#include "atomic_helpers.h"
+#include "common.h"
+#include "mem_map.h"
#include "mutex.h"
namespace scudo {
@@ -38,7 +40,7 @@ class MurMur2HashBuilder {
}
};
-class StackDepot {
+class alignas(16) StackDepot {
HybridMutex RingEndMu;
u32 RingEnd = 0;
@@ -62,47 +64,62 @@ class StackDepot {
// This is achieved by re-checking the hash of the stack trace before
// returning the trace.
-#if SCUDO_SMALL_STACK_DEPOT
- static const uptr TabBits = 4;
-#else
- static const uptr TabBits = 16;
-#endif
- static const uptr TabSize = 1 << TabBits;
- static const uptr TabMask = TabSize - 1;
- atomic_u32 Tab[TabSize] = {};
-
-#if SCUDO_SMALL_STACK_DEPOT
- static const uptr RingBits = 4;
-#else
- static const uptr RingBits = 19;
-#endif
- static const uptr RingSize = 1 << RingBits;
- static const uptr RingMask = RingSize - 1;
- atomic_u64 Ring[RingSize] = {};
+ MemMapT DepotMap;
+ uptr RingSize = 0;
+ uptr RingMask = 0;
+ uptr TabMask = 0;
+
+ atomic_u64 *Ring(char *RawStackDepot) {
+ return reinterpret_cast<atomic_u64 *>(RawStackDepot + sizeof(StackDepot));
+ }
+
+ atomic_u32 *Tab(char *RawStackDepot) {
+ return reinterpret_cast<atomic_u32 *>(RawStackDepot + sizeof(StackDepot) +
+ sizeof(atomic_u64) * RingSize);
+ }
+
+ const atomic_u64 *Ring(const char *RawStackDepot) const {
+ return reinterpret_cast<const atomic_u64 *>(RawStackDepot +
+ sizeof(StackDepot));
+ }
+
+ const atomic_u32 *Tab(const char *RawStackDepot) const {
+ return reinterpret_cast<const atomic_u32 *>(
+ RawStackDepot + sizeof(StackDepot) + sizeof(atomic_u64) * RingSize);
+ }
public:
+ void init(MemMapT Map, uptr RingSz, uptr TabSz) {
+ DCHECK(isPowerOfTwo(RingSz));
+ DCHECK(isPowerOfTwo(TabSz));
+ DepotMap = Map;
+ RingSize = RingSz;
+ RingMask = RingSz - 1;
+ TabMask = TabSz - 1;
+ }
+
// Insert hash of the stack trace [Begin, End) into the stack depot, and
// return the hash.
- u32 insert(uptr *Begin, uptr *End) {
+ u32 insert(char *RawStackDepot, uptr *Begin, uptr *End) {
MurMur2HashBuilder B;
for (uptr *I = Begin; I != End; ++I)
B.add(u32(*I) >> 2);
u32 Hash = B.get();
u32 Pos = Hash & TabMask;
- u32 RingPos = atomic_load_relaxed(&Tab[Pos]);
- u64 Entry = atomic_load_relaxed(&Ring[RingPos]);
+ u32 RingPos = atomic_load_relaxed(&Tab(RawStackDepot)[Pos]);
+ u64 Entry = atomic_load_relaxed(&Ring(RawStackDepot)[RingPos]);
u64 Id = (u64(End - Begin) << 33) | (u64(Hash) << 1) | 1;
if (Entry == Id)
return Hash;
ScopedLock Lock(RingEndMu);
RingPos = RingEnd;
- atomic_store_relaxed(&Tab[Pos], RingPos);
- atomic_store_relaxed(&Ring[RingPos], Id);
+ atomic_store_relaxed(&Tab(RawStackDepot)[Pos], RingPos);
+ atomic_store_relaxed(&Ring(RawStackDepot)[RingPos], Id);
for (uptr *I = Begin; I != End; ++I) {
RingPos = (RingPos + 1) & RingMask;
- atomic_store_relaxed(&Ring[RingPos], *I);
+ atomic_store_relaxed(&Ring(RawStackDepot)[RingPos], *I);
}
RingEnd = (RingPos + 1) & RingMask;
return Hash;
@@ -111,12 +128,13 @@ class StackDepot {
// Look up a stack trace by hash. Returns true if successful. The trace may be
// accessed via operator[] passing indexes between *RingPosPtr and
// *RingPosPtr + *SizePtr.
- bool find(u32 Hash, uptr *RingPosPtr, uptr *SizePtr) const {
+ bool find(const char *RawStackDepot, u32 Hash, uptr *RingPosPtr,
+ uptr *SizePtr) const {
u32 Pos = Hash & TabMask;
- u32 RingPos = atomic_load_relaxed(&Tab[Pos]);
+ u32 RingPos = atomic_load_relaxed(&Tab(RawStackDepot)[Pos]);
if (RingPos >= RingSize)
return false;
- u64 Entry = atomic_load_relaxed(&Ring[RingPos]);
+ u64 Entry = atomic_load_relaxed(&Ring(RawStackDepot)[RingPos]);
u64 HashWithTagBit = (u64(Hash) << 1) | 1;
if ((Entry & 0x1ffffffff) != HashWithTagBit)
return false;
@@ -128,13 +146,15 @@ class StackDepot {
MurMur2HashBuilder B;
for (uptr I = 0; I != Size; ++I) {
RingPos = (RingPos + 1) & RingMask;
- B.add(u32(atomic_load_relaxed(&Ring[RingPos])) >> 2);
+ B.add(u32(atomic_load_relaxed(&Ring(RawStackDepot)[RingPos])) >> 2);
}
return B.get() == Hash;
}
- u64 operator[](uptr RingPos) const {
- return atomic_load_relaxed(&Ring[RingPos & RingMask]);
+ void free() { DepotMap.unmap(DepotMap.getBase(), DepotMap.getCapacity()); }
+
+ u64 at(const char *RawStackDepot, uptr RingPos) const {
+ return atomic_load_relaxed(&Ring(RawStackDepot)[RingPos & RingMask]);
}
};
diff --git a/compiler-rt/lib/scudo/standalone/wrappers_c_bionic.cpp b/compiler-rt/lib/scudo/standalone/wrappers_c_bionic.cpp
index 4fed44779b902..ee78b76efd310 100644
--- a/compiler-rt/lib/scudo/standalone/wrappers_c_bionic.cpp
+++ b/compiler-rt/lib/scudo/standalone/wrappers_c_bionic.cpp
@@ -54,7 +54,7 @@ INTERFACE const char *__scudo_get_stack_depot_addr() {
}
INTERFACE size_t __scudo_get_stack_depot_size() {
- return sizeof(scudo::StackDepot);
+ return Allocator.getStackDepotSize();
}
INTERFACE const char *__scudo_get_region_info_addr() {
``````````
</details>
https://github.com/llvm/llvm-project/pull/74515
More information about the llvm-commits
mailing list