[compiler-rt] 405ceaa - [scudo] Manage pages with MemMap in SizeClassAllocator64
Chia-hung Duan via llvm-commits
llvm-commits at lists.llvm.org
Wed Apr 5 14:35:04 PDT 2023
Author: Chia-hung Duan
Date: 2023-04-05T21:28:54Z
New Revision: 405ceaa09de04332384ab4e12a2ae5a79eb7d10d
URL: https://github.com/llvm/llvm-project/commit/405ceaa09de04332384ab4e12a2ae5a79eb7d10d
DIFF: https://github.com/llvm/llvm-project/commit/405ceaa09de04332384ab4e12a2ae5a79eb7d10d.diff
LOG: [scudo] Manage pages with MemMap in SizeClassAllocator64
Introduce a new data structure to manage the allocated pages from the
system. This is meant to deprecate certain memory system call wrappers
in Scudo, e.g., map()/unmap(). Besides, we would like to make
MapPlatformData to be appeared in platform specific data structure only.
Given that there are several allocators in Scudo and each of them has
different way of page management. The deprecation will be done in
several CLs. In this commit, we start from SizeClassAllocator64.
Reviewed By: cferris
Differential Revision: https://reviews.llvm.org/D146009
Added:
compiler-rt/lib/scudo/standalone/mem_map.cpp
compiler-rt/lib/scudo/standalone/mem_map.h
compiler-rt/lib/scudo/standalone/mem_map_base.h
Modified:
compiler-rt/lib/scudo/standalone/CMakeLists.txt
compiler-rt/lib/scudo/standalone/primary64.h
compiler-rt/lib/scudo/standalone/release.h
Removed:
################################################################################
diff --git a/compiler-rt/lib/scudo/standalone/CMakeLists.txt b/compiler-rt/lib/scudo/standalone/CMakeLists.txt
index 6fcd4deddf716..b2f4ff81c9aad 100644
--- a/compiler-rt/lib/scudo/standalone/CMakeLists.txt
+++ b/compiler-rt/lib/scudo/standalone/CMakeLists.txt
@@ -71,6 +71,8 @@ set(SCUDO_HEADERS
list.h
local_cache.h
memtag.h
+ mem_map.h
+ mem_map_base.h
mutex.h
options.h
platform.h
@@ -104,6 +106,7 @@ set(SCUDO_SOURCES
flags.cpp
fuchsia.cpp
linux.cpp
+ mem_map.cpp
release.cpp
report.cpp
rss_limit_checker.cpp
diff --git a/compiler-rt/lib/scudo/standalone/mem_map.cpp b/compiler-rt/lib/scudo/standalone/mem_map.cpp
new file mode 100644
index 0000000000000..36cfc1f834226
--- /dev/null
+++ b/compiler-rt/lib/scudo/standalone/mem_map.cpp
@@ -0,0 +1,73 @@
+//===-- mem_map.cpp ---------------------------------------------*- C++ -*-===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#include "mem_map.h"
+
+#include "common.h"
+
+namespace scudo {
+
+bool MemMapDefault::mapImpl(uptr Addr, uptr Size, const char *Name,
+ uptr Flags) {
+ void *MappedAddr =
+ ::scudo::map(reinterpret_cast<void *>(Addr), Size, Name, Flags, &Data);
+ if (MappedAddr == nullptr)
+ return false;
+ Base = reinterpret_cast<uptr>(MappedAddr);
+ Capacity = Size;
+ return true;
+}
+
+void MemMapDefault::unmapImpl(uptr Addr, uptr Size) {
+ if (Size == Capacity) {
+ Base = Capacity = 0;
+ } else {
+ if (Base == Addr)
+ Base = Addr + Size;
+ Capacity -= Size;
+ }
+
+ ::scudo::unmap(reinterpret_cast<void *>(Addr), Size, UNMAP_ALL, &Data);
+}
+
+bool MemMapDefault::remapImpl(uptr Addr, uptr Size, const char *Name,
+ uptr Flags) {
+ void *RemappedAddr =
+ ::scudo::map(reinterpret_cast<void *>(Addr), Size, Name, Flags, &Data);
+ return reinterpret_cast<uptr>(RemappedAddr) == Addr;
+}
+
+void MemMapDefault::releaseAndZeroPagesToOSImpl(uptr From, uptr Size) {
+ return ::scudo::releasePagesToOS(Base, From - Base, Size, &Data);
+}
+
+void ReservedMemoryDefault::releaseImpl() {
+ ::scudo::unmap(reinterpret_cast<void *>(Base), Capacity, UNMAP_ALL, &Data);
+}
+
+bool ReservedMemoryDefault::createImpl(uptr Addr, uptr Size, const char *Name,
+ uptr Flags) {
+ void *Reserved = ::scudo::map(reinterpret_cast<void *>(Addr), Size, Name,
+ Flags | MAP_NOACCESS, &Data);
+ if (Reserved == nullptr)
+ return false;
+
+ Base = reinterpret_cast<uptr>(Reserved);
+ Capacity = Size;
+
+ return true;
+}
+
+ReservedMemoryDefault::MemMapT ReservedMemoryDefault::dispatchImpl(uptr Addr,
+ uptr Size) {
+ ReservedMemoryDefault::MemMapT NewMap(Addr, Size);
+ NewMap.setMapPlatformData(Data);
+ return NewMap;
+}
+
+} // namespace scudo
diff --git a/compiler-rt/lib/scudo/standalone/mem_map.h b/compiler-rt/lib/scudo/standalone/mem_map.h
new file mode 100644
index 0000000000000..58fbe9d01b7ed
--- /dev/null
+++ b/compiler-rt/lib/scudo/standalone/mem_map.h
@@ -0,0 +1,87 @@
+//===-- mem_map.h -----------------------------------------------*- C++ -*-===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef SCUDO_MEM_MAP_H_
+#define SCUDO_MEM_MAP_H_
+
+#include "mem_map_base.h"
+
+#include "common.h"
+#include "internal_defs.h"
+
+// TODO: This is only used for `MapPlatformData`. Remove these includes when we
+// have all three platform specific `MemMap` and `ReservedMemory`
+// implementations.
+#include "fuchsia.h"
+#include "linux.h"
+#include "trusty.h"
+
+namespace scudo {
+
+// This will be deprecated when every allocator has been supported by each
+// platform's `MemMap` implementation.
+class MemMapDefault final : public MemMapBase<MemMapDefault> {
+public:
+ constexpr MemMapDefault() = default;
+ MemMapDefault(uptr Base, uptr Capacity) : Base(Base), Capacity(Capacity) {}
+
+ // Impls for base functions.
+ bool mapImpl(uptr Addr, uptr Size, const char *Name, uptr Flags);
+ void unmapImpl(uptr Addr, uptr Size);
+ bool remapImpl(uptr Addr, uptr Size, const char *Name, uptr Flags);
+ void releasePagesToOSImpl(uptr From, uptr Size) {
+ return releaseAndZeroPagesToOSImpl(From, Size);
+ }
+ void releaseAndZeroPagesToOSImpl(uptr From, uptr Size);
+ uptr getBaseImpl() { return Base; }
+ uptr getCapacityImpl() { return Capacity; }
+
+ void setMapPlatformData(MapPlatformData &NewData) { Data = NewData; }
+
+private:
+ uptr Base = 0;
+ uptr Capacity = 0;
+ MapPlatformData Data = {};
+};
+
+// This will be deprecated when every allocator has been supported by each
+// platform's `MemMap` implementation.
+class ReservedMemoryDefault final
+ : public ReservedMemory<ReservedMemoryDefault, MemMapDefault> {
+public:
+ constexpr ReservedMemoryDefault() = default;
+
+ bool createImpl(uptr Addr, uptr Size, const char *Name, uptr Flags);
+ void releaseImpl();
+ MemMapT dispatchImpl(uptr Addr, uptr Size);
+ uptr getBaseImpl() { return Base; }
+ uptr getCapacityImpl() { return Capacity; }
+
+private:
+ uptr Base = 0;
+ uptr Capacity = 0;
+ MapPlatformData Data = {};
+};
+
+#if SCUDO_LINUX
+using ReservedMemoryT = ReservedMemoryDefault;
+using MemMapT = ReservedMemoryT::MemMapT;
+#elif SCUDO_FUCHSIA
+using ReservedMemoryT = ReservedMemoryDefault;
+using MemMapT = ReservedMemoryT::MemMapT;
+#elif SCUDO_TRUSTY
+using ReservedMemoryT = ReservedMemoryDefault;
+using MemMapT = ReservedMemoryT::MemMapT;
+#else
+#error \
+ "Unsupported platform, please implement the ReservedMemory for your platform!"
+#endif
+
+} // namespace scudo
+
+#endif // SCUDO_MEM_MAP_H_
diff --git a/compiler-rt/lib/scudo/standalone/mem_map_base.h b/compiler-rt/lib/scudo/standalone/mem_map_base.h
new file mode 100644
index 0000000000000..f2b6a27736130
--- /dev/null
+++ b/compiler-rt/lib/scudo/standalone/mem_map_base.h
@@ -0,0 +1,121 @@
+//===-- mem_map_base.h ------------------------------------------*- C++ -*-===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef SCUDO_MEM_MAP_BASE_H_
+#define SCUDO_MEM_MAP_BASE_H_
+
+#include "common.h"
+
+namespace scudo {
+
+// In Scudo, every memory operation will be fulfilled through a
+// platform-specific `MemMap` instance. The essential APIs are listed in the
+// `MemMapBase` below. This is implemented in CRTP, so for each implementation,
+// it has to implement all of the 'Impl' named functions.
+template <class Derived> class MemMapBase {
+public:
+ constexpr MemMapBase() = default;
+
+ // This is used to map a new set of contiguous pages. Note that the `Addr` is
+ // only a suggestion to the system.
+ bool map(uptr Addr, uptr Size, const char *Name, uptr Flags = 0) {
+ DCHECK(!isAllocated());
+ return invokeImpl(&Derived::mapImpl, Addr, Size, Name, Flags);
+ }
+
+ // This is used to unmap partial/full pages from the beginning or the end.
+ // I.e., the result pages are expected to be still contiguous.
+ void unmap(uptr Addr, uptr Size) {
+ DCHECK(isAllocated());
+ DCHECK((Addr == getBase()) || (Addr + Size == getBase() + getCapacity()));
+ invokeImpl(&Derived::unmapImpl, Addr, Size);
+ }
+
+ // This is used to remap a mapped range (either from map() or dispatched from
+ // ReservedMemory). For example, we have reserved several pages and then we
+ // want to remap them with
diff erent accessibility.
+ bool remap(uptr Addr, uptr Size, const char *Name, uptr Flags = 0) {
+ DCHECK(isAllocated());
+ DCHECK((Addr >= getBase()) || (Addr + Size <= getBase() + getCapacity()));
+ return invokeImpl(&Derived::remapImpl, Addr, Size, Name, Flags);
+ }
+
+ // Suggest releasing a set of contiguous physical pages back to the OS. Note
+ // that only physical pages are supposed to be released. Any release of
+ // virtual pages may lead to undefined behavior.
+ void releasePagesToOS(uptr From, uptr Size) {
+ DCHECK(isAllocated());
+ DCHECK((From >= getBase()) || (From + Size <= getBase() + getCapacity()));
+ invokeImpl(&Derived::releasePagesToOSImpl, From, Size);
+ }
+ // This is similar to the above one except that any subsequent access to the
+ // released pages will return with zero-filled pages.
+ void releaseAndZeroPagesToOS(uptr From, uptr Size) {
+ DCHECK(isAllocated());
+ DCHECK((From >= getBase()) || (From + Size <= getBase() + getCapacity()));
+ invokeImpl(&Derived::releaseAndZeroPagesToOSImpl, From, Size);
+ }
+
+ uptr getBase() { return invokeImpl(&Derived::getBaseImpl); }
+ uptr getCapacity() { return invokeImpl(&Derived::getCapacityImpl); }
+
+ bool isAllocated() { return getBase() != 0U; }
+
+protected:
+ template <typename R, typename... Args>
+ R invokeImpl(R (Derived::*MemFn)(Args...), Args... args) {
+ return (static_cast<Derived *>(this)->*MemFn)(args...);
+ }
+};
+
+// `ReservedMemory` is a special memory handle which can be viewed as a page
+// allocator. `ReservedMemory` will reserve a contiguous pages and the later
+// page request can be fulfilled at the designated address. This is used when
+// we want to ensure the virtual address of the MemMap will be in a known range.
+// This is implemented in CRTP, so for each
+// implementation, it has to implement all of the 'Impl' named functions.
+template <class Derived, typename MemMapTy> class ReservedMemory {
+public:
+ using MemMapT = MemMapTy;
+ constexpr ReservedMemory() = default;
+
+ // Reserve a chunk of memory at a suggested address.
+ bool create(uptr Addr, uptr Size, const char *Name, uptr Flags = 0) {
+ DCHECK(!isCreated());
+ return invokeImpl(&Derived::createImpl, Addr, Size, Name, Flags);
+ }
+
+ // Release the entire reserved memory.
+ void release() {
+ DCHECK(isCreated());
+ invokeImpl(&Derived::releaseImpl);
+ }
+
+ // Dispatch a sub-range of reserved memory. Note that any fragmentation of
+ // the reserved pages is managed by each implementation.
+ MemMapT dispatch(uptr Addr, uptr Size) {
+ DCHECK(isCreated());
+ DCHECK((Addr >= getBase()) || (Addr + Size <= getBase() + getCapacity()));
+ return invokeImpl(&Derived::dispatchImpl, Addr, Size);
+ }
+
+ uptr getBase() { return invokeImpl(&Derived::getBaseImpl); }
+ uptr getCapacity() { return invokeImpl(&Derived::getCapacityImpl); }
+
+ bool isCreated() { return getBase() != 0U; }
+
+protected:
+ template <typename R, typename... Args>
+ R invokeImpl(R (Derived::*MemFn)(Args...), Args... args) {
+ return (static_cast<Derived *>(this)->*MemFn)(args...);
+ }
+};
+
+} // namespace scudo
+
+#endif // SCUDO_MEM_MAP_BASE_H_
diff --git a/compiler-rt/lib/scudo/standalone/primary64.h b/compiler-rt/lib/scudo/standalone/primary64.h
index 00b7758b4c43c..dad6d263a083d 100644
--- a/compiler-rt/lib/scudo/standalone/primary64.h
+++ b/compiler-rt/lib/scudo/standalone/primary64.h
@@ -13,6 +13,7 @@
#include "common.h"
#include "list.h"
#include "local_cache.h"
+#include "mem_map.h"
#include "memtag.h"
#include "options.h"
#include "release.h"
@@ -103,10 +104,11 @@ template <typename Config> class SizeClassAllocator64 {
SmallerBlockReleasePageDelta =
PagesInGroup * (1 + MinSizeClass / 16U) / 100;
- DCHECK_EQ(PrimaryBase, 0U);
// Reserve the space required for the Primary.
- PrimaryBase = reinterpret_cast<uptr>(map(
- nullptr, PrimarySize, "scudo:primary_reserve", MAP_NOACCESS, &Data));
+ CHECK(ReservedMemory.create(/*Addr=*/0U, PrimarySize,
+ "scudo:primary_reserve"));
+ PrimaryBase = ReservedMemory.getBase();
+ DCHECK_NE(PrimaryBase, 0U);
u32 Seed;
const u64 Time = getMonotonicTimeFast();
@@ -141,8 +143,7 @@ template <typename Config> class SizeClassAllocator64 {
*Region = {};
}
if (PrimaryBase)
- unmap(reinterpret_cast<void *>(PrimaryBase), PrimarySize, UNMAP_ALL,
- &Data);
+ ReservedMemory.release();
PrimaryBase = 0U;
}
@@ -449,7 +450,7 @@ template <typename Config> class SizeClassAllocator64 {
uptr AllocatedUser GUARDED_BY(Mutex) = 0;
// The minimum size of pushed blocks to trigger page release.
uptr TryReleaseThreshold GUARDED_BY(Mutex) = 0;
- MapPlatformData Data GUARDED_BY(Mutex) = {};
+ MemMapT MemMap = {};
ReleaseToOsInfo ReleaseInfo GUARDED_BY(Mutex) = {};
bool Exhausted GUARDED_BY(Mutex) = false;
};
@@ -459,11 +460,13 @@ template <typename Config> class SizeClassAllocator64 {
};
static_assert(sizeof(RegionInfo) % SCUDO_CACHE_LINE_SIZE == 0, "");
+ // TODO: `PrimaryBase` can be obtained from ReservedMemory. This needs to be
+ // deprecated.
uptr PrimaryBase = 0;
+ ReservedMemoryT ReservedMemory = {};
// The minimum size of pushed blocks that we will try to release the pages in
// that size class.
uptr SmallerBlockReleasePageDelta = 0;
- MapPlatformData Data = {};
atomic_s32 ReleaseToOsIntervalMs = {};
alignas(SCUDO_CACHE_LINE_SIZE) RegionInfo RegionInfoArray[NumClasses];
@@ -742,14 +745,18 @@ template <typename Config> class SizeClassAllocator64 {
Region->Exhausted = true;
return false;
}
- if (MappedUser == 0)
- Region->Data = Data;
- if (UNLIKELY(!map(
- reinterpret_cast<void *>(RegionBeg + MappedUser), MapSize,
- "scudo:primary",
+ // TODO: Consider allocating MemMap in init().
+ if (!Region->MemMap.isAllocated()) {
+ Region->MemMap = ReservedMemory.dispatch(
+ getRegionBaseByClassId(ClassId), RegionSize);
+ }
+ DCHECK(Region->MemMap.isAllocated());
+
+ if (UNLIKELY(!Region->MemMap.remap(
+ RegionBeg + MappedUser, MapSize, "scudo:primary",
MAP_ALLOWNOMEM | MAP_RESIZABLE |
- (useMemoryTagging<Config>(Options.load()) ? MAP_MEMTAG : 0),
- &Region->Data))) {
+ (useMemoryTagging<Config>(Options.load()) ? MAP_MEMTAG
+ : 0)))) {
return false;
}
Region->MappedUser += MapSize;
@@ -1082,7 +1089,8 @@ template <typename Config> class SizeClassAllocator64 {
const uptr ReleaseRangeSize = ReleaseEnd - ReleaseBase;
const uptr ReleaseOffset = ReleaseBase - Region->RegionBeg;
- ReleaseRecorder Recorder(Region->RegionBeg, ReleaseOffset, &Region->Data);
+ RegionReleaseRecorder<MemMapT> Recorder(&Region->MemMap, Region->RegionBeg,
+ ReleaseOffset);
PageReleaseContext Context(BlockSize, /*NumberOfRegions=*/1U,
ReleaseRangeSize, ReleaseOffset);
diff --git a/compiler-rt/lib/scudo/standalone/release.h b/compiler-rt/lib/scudo/standalone/release.h
index bb66a5d2c409c..b7a0131863368 100644
--- a/compiler-rt/lib/scudo/standalone/release.h
+++ b/compiler-rt/lib/scudo/standalone/release.h
@@ -11,11 +11,42 @@
#include "common.h"
#include "list.h"
+#include "mem_map.h"
#include "mutex.h"
#include "thread_annotations.h"
namespace scudo {
+template <typename MemMapT> class RegionReleaseRecorder {
+public:
+ RegionReleaseRecorder(MemMapT *RegionMemMap, uptr Base, uptr Offset = 0)
+ : RegionMemMap(RegionMemMap), Base(Base), Offset(Offset) {}
+
+ uptr getReleasedRangesCount() const { return ReleasedRangesCount; }
+
+ uptr getReleasedBytes() const { return ReleasedBytes; }
+
+ uptr getBase() const { return Base; }
+
+ // Releases [From, To) range of pages back to OS. Note that `From` and `To`
+ // are offseted from `Base` + Offset.
+ void releasePageRangeToOS(uptr From, uptr To) {
+ const uptr Size = To - From;
+ RegionMemMap->releasePagesToOS(getBase() + Offset + From, Size);
+ ReleasedRangesCount++;
+ ReleasedBytes += Size;
+ }
+
+private:
+ uptr ReleasedRangesCount = 0;
+ uptr ReleasedBytes = 0;
+ MemMapT *RegionMemMap = nullptr;
+ uptr Base = 0;
+ // The release offset from Base. This is used when we know a given range after
+ // Base will not be released.
+ uptr Offset = 0;
+};
+
class ReleaseRecorder {
public:
ReleaseRecorder(uptr Base, uptr Offset = 0, MapPlatformData *Data = nullptr)
More information about the llvm-commits
mailing list