[compiler-rt] 8aaefa9 - [scudo] Add a method to force release everything.
Christopher Ferris via llvm-commits
llvm-commits at lists.llvm.org
Wed Mar 15 17:29:47 PDT 2023
Author: Christopher Ferris
Date: 2023-03-15T17:29:19-07:00
New Revision: 8aaefa92436cf7f66e107795d8513db1285952ac
URL: https://github.com/llvm/llvm-project/commit/8aaefa92436cf7f66e107795d8513db1285952ac
DIFF: https://github.com/llvm/llvm-project/commit/8aaefa92436cf7f66e107795d8513db1285952ac.diff
LOG: [scudo] Add a method to force release everything.
The force flag to releaseToOSMaybe does not release everything
since it is an expensive operation. Modify the release flag to
have three states: normal, force, forceall. Force behaves the same
as setting Force to true from before this change. Forceall will
release everything regardless of how much time it takes, or
how much there is to release.
In addition, add a new mallopt that will call the release function
with the forceall flag set.
Reviewed By: Chia-hungDuan
Differential Revision: https://reviews.llvm.org/D146106
Added:
Modified:
compiler-rt/lib/scudo/standalone/combined.h
compiler-rt/lib/scudo/standalone/common.h
compiler-rt/lib/scudo/standalone/include/scudo/interface.h
compiler-rt/lib/scudo/standalone/primary32.h
compiler-rt/lib/scudo/standalone/primary64.h
compiler-rt/lib/scudo/standalone/tests/combined_test.cpp
compiler-rt/lib/scudo/standalone/tests/primary_test.cpp
compiler-rt/lib/scudo/standalone/wrappers_c.inc
Removed:
################################################################################
diff --git a/compiler-rt/lib/scudo/standalone/combined.h b/compiler-rt/lib/scudo/standalone/combined.h
index 52dbf6b145264..250eba0f4ddad 100644
--- a/compiler-rt/lib/scudo/standalone/combined.h
+++ b/compiler-rt/lib/scudo/standalone/combined.h
@@ -745,9 +745,9 @@ class Allocator {
Str.output();
}
- void releaseToOS() {
+ void releaseToOS(ReleaseToOS ReleaseType) {
initThreadMaybe();
- Primary.releaseToOS();
+ Primary.releaseToOS(ReleaseType);
Secondary.releaseToOS();
}
diff --git a/compiler-rt/lib/scudo/standalone/common.h b/compiler-rt/lib/scudo/standalone/common.h
index e1ca264f46a69..82e6cf4aee618 100644
--- a/compiler-rt/lib/scudo/standalone/common.h
+++ b/compiler-rt/lib/scudo/standalone/common.h
@@ -215,6 +215,13 @@ enum class Option : u8 {
MaxTSDsCount, // Number of usable TSDs for the shared registry.
};
+enum class ReleaseToOS : u8 {
+ Normal, // Follow the normal rules for releasing pages to the OS
+ Force, // Force release pages to the OS, but avoid cases that take too long.
+ ForceAll, // Force release every page possible regardless of how long it will
+ // take.
+};
+
constexpr unsigned char PatternFillByte = 0xAB;
enum FillContentsMode {
diff --git a/compiler-rt/lib/scudo/standalone/include/scudo/interface.h b/compiler-rt/lib/scudo/standalone/include/scudo/interface.h
index 23bcfba3982ac..3c083ed7f9da2 100644
--- a/compiler-rt/lib/scudo/standalone/include/scudo/interface.h
+++ b/compiler-rt/lib/scudo/standalone/include/scudo/interface.h
@@ -118,6 +118,10 @@ size_t __scudo_get_ring_buffer_size(void);
#define M_PURGE -101
#endif
+#ifndef M_PURGE_ALL
+#define M_PURGE_ALL -104
+#endif
+
// Tune the allocator's choice of memory tags to make it more likely that
// a certain class of memory errors will be detected. The value argument should
// be one of the M_MEMTAG_TUNING_* constants below.
diff --git a/compiler-rt/lib/scudo/standalone/primary32.h b/compiler-rt/lib/scudo/standalone/primary32.h
index 5d3bc658f6e89..3be689fb0e608 100644
--- a/compiler-rt/lib/scudo/standalone/primary32.h
+++ b/compiler-rt/lib/scudo/standalone/primary32.h
@@ -281,14 +281,14 @@ template <typename Config> class SizeClassAllocator32 {
return true;
}
- uptr releaseToOS() {
+ uptr releaseToOS(ReleaseToOS ReleaseType) {
uptr TotalReleasedBytes = 0;
for (uptr I = 0; I < NumClasses; I++) {
if (I == SizeClassMap::BatchClassId)
continue;
SizeClassInfo *Sci = getSizeClassInfo(I);
ScopedLock L(Sci->Mutex);
- TotalReleasedBytes += releaseToOSMaybe(Sci, I, /*Force=*/true);
+ TotalReleasedBytes += releaseToOSMaybe(Sci, I, ReleaseType);
}
return TotalReleasedBytes;
}
@@ -727,7 +727,8 @@ template <typename Config> class SizeClassAllocator32 {
}
NOINLINE uptr releaseToOSMaybe(SizeClassInfo *Sci, uptr ClassId,
- bool Force = false) REQUIRES(Sci->Mutex) {
+ ReleaseToOS ReleaseType = ReleaseToOS::Normal)
+ REQUIRES(Sci->Mutex) {
const uptr BlockSize = getSizeByClassId(ClassId);
const uptr PageSize = getPageSizeCached();
@@ -743,16 +744,18 @@ template <typename Config> class SizeClassAllocator32 {
if (BytesPushed < PageSize)
return 0; // Nothing new to release.
- const bool CheckDensity = BlockSize < PageSize / 16U;
+ const bool CheckDensity =
+ BlockSize < PageSize / 16U && ReleaseType != ReleaseToOS::ForceAll;
// Releasing smaller blocks is expensive, so we want to make sure that a
// significant amount of bytes are free, and that there has been a good
// amount of batches pushed to the freelist before attempting to release.
if (CheckDensity) {
- if (!Force && BytesPushed < Sci->AllocatedUser / 16U)
+ if (ReleaseType == ReleaseToOS::Normal &&
+ BytesPushed < Sci->AllocatedUser / 16U)
return 0;
}
- if (!Force) {
+ if (ReleaseType == ReleaseToOS::Normal) {
const s32 IntervalMs = atomic_load_relaxed(&ReleaseToOsIntervalMs);
if (IntervalMs < 0)
return 0;
diff --git a/compiler-rt/lib/scudo/standalone/primary64.h b/compiler-rt/lib/scudo/standalone/primary64.h
index 48ec2cbf0671a..1cb6d02f6cd6a 100644
--- a/compiler-rt/lib/scudo/standalone/primary64.h
+++ b/compiler-rt/lib/scudo/standalone/primary64.h
@@ -321,14 +321,14 @@ template <typename Config> class SizeClassAllocator64 {
return true;
}
- uptr releaseToOS() {
+ uptr releaseToOS(ReleaseToOS ReleaseType) {
uptr TotalReleasedBytes = 0;
for (uptr I = 0; I < NumClasses; I++) {
if (I == SizeClassMap::BatchClassId)
continue;
RegionInfo *Region = getRegionInfo(I);
ScopedLock L(Region->Mutex);
- TotalReleasedBytes += releaseToOSMaybe(Region, I, /*Force=*/true);
+ TotalReleasedBytes += releaseToOSMaybe(Region, I, ReleaseType);
}
return TotalReleasedBytes;
}
@@ -805,7 +805,8 @@ template <typename Config> class SizeClassAllocator64 {
}
NOINLINE uptr releaseToOSMaybe(RegionInfo *Region, uptr ClassId,
- bool Force = false) REQUIRES(Region->Mutex) {
+ ReleaseToOS ReleaseType = ReleaseToOS::Normal)
+ REQUIRES(Region->Mutex) {
const uptr BlockSize = getSizeByClassId(ClassId);
const uptr PageSize = getPageSizeCached();
@@ -821,16 +822,18 @@ template <typename Config> class SizeClassAllocator64 {
if (BytesPushed < PageSize)
return 0; // Nothing new to release.
- const bool CheckDensity = BlockSize < PageSize / 16U;
+ const bool CheckDensity =
+ BlockSize < PageSize / 16U && ReleaseType != ReleaseToOS::ForceAll;
// Releasing smaller blocks is expensive, so we want to make sure that a
// significant amount of bytes are free, and that there has been a good
// amount of batches pushed to the freelist before attempting to release.
if (CheckDensity) {
- if (!Force && BytesPushed < Region->AllocatedUser / 16U)
+ if (ReleaseType == ReleaseToOS::Normal &&
+ BytesPushed < Region->AllocatedUser / 16U)
return 0;
}
- if (!Force) {
+ if (ReleaseType == ReleaseToOS::Normal) {
const s32 IntervalMs = atomic_load_relaxed(&ReleaseToOsIntervalMs);
if (IntervalMs < 0)
return 0;
diff --git a/compiler-rt/lib/scudo/standalone/tests/combined_test.cpp b/compiler-rt/lib/scudo/standalone/tests/combined_test.cpp
index 6f4fa748ed93a..550fdd3dd7606 100644
--- a/compiler-rt/lib/scudo/standalone/tests/combined_test.cpp
+++ b/compiler-rt/lib/scudo/standalone/tests/combined_test.cpp
@@ -92,7 +92,7 @@ template <class TypeParam> struct ScudoCombinedTest : public Test {
Allocator = std::make_unique<AllocatorT>();
}
~ScudoCombinedTest() {
- Allocator->releaseToOS();
+ Allocator->releaseToOS(scudo::ReleaseToOS::Force);
UseQuarantine = true;
}
@@ -412,7 +412,7 @@ SCUDO_TYPED_TEST(ScudoCombinedDeathTest, DisableMemoryTagging) {
reinterpret_cast<char *>(P)[2048] = 0xaa;
Allocator->deallocate(P, Origin);
- Allocator->releaseToOS();
+ Allocator->releaseToOS(scudo::ReleaseToOS::Force);
}
}
@@ -488,7 +488,7 @@ SCUDO_TYPED_TEST(ScudoCombinedTest, ThreadedCombined) {
}
for (auto &T : Threads)
T.join();
- Allocator->releaseToOS();
+ Allocator->releaseToOS(scudo::ReleaseToOS::Force);
}
// Test that multiple instantiations of the allocator have not messed up the
@@ -601,7 +601,7 @@ TEST(ScudoCombinedTest, FullRegion) {
// operation without issue.
SCUDO_TYPED_TEST(ScudoCombinedTest, ReleaseToOS) {
auto *Allocator = this->Allocator.get();
- Allocator->releaseToOS();
+ Allocator->releaseToOS(scudo::ReleaseToOS::Force);
}
SCUDO_TYPED_TEST(ScudoCombinedTest, OddEven) {
@@ -740,7 +740,7 @@ TEST(ScudoCombinedTest, BasicTrustyConfig) {
auto *TSD = Allocator->getTSDRegistry()->getTSDAndLock(&UnlockRequired);
TSD->getCache().drain();
- Allocator->releaseToOS();
+ Allocator->releaseToOS(scudo::ReleaseToOS::Force);
}
#endif
diff --git a/compiler-rt/lib/scudo/standalone/tests/primary_test.cpp b/compiler-rt/lib/scudo/standalone/tests/primary_test.cpp
index c7ebcc3f82fbd..51a7038ac7870 100644
--- a/compiler-rt/lib/scudo/standalone/tests/primary_test.cpp
+++ b/compiler-rt/lib/scudo/standalone/tests/primary_test.cpp
@@ -161,7 +161,7 @@ SCUDO_TYPED_TEST(ScudoPrimaryTest, BasicPrimary) {
Cache.deallocate(ClassId, Pointers[J]);
}
Cache.destroy(nullptr);
- Allocator->releaseToOS();
+ Allocator->releaseToOS(scudo::ReleaseToOS::Force);
scudo::ScopedString Str;
Allocator->getStats(&Str);
Str.output();
@@ -215,7 +215,7 @@ TEST(ScudoPrimaryTest, Primary64OOM) {
Cache.deallocate(Primary::SizeClassMap::BatchClassId, B);
}
Cache.destroy(nullptr);
- Allocator.releaseToOS();
+ Allocator.releaseToOS(scudo::ReleaseToOS::Force);
scudo::ScopedString Str;
Allocator.getStats(&Str);
Str.output();
@@ -253,7 +253,7 @@ SCUDO_TYPED_TEST(ScudoPrimaryTest, PrimaryIterate) {
V.pop_back();
}
Cache.destroy(nullptr);
- Allocator->releaseToOS();
+ Allocator->releaseToOS(scudo::ReleaseToOS::Force);
scudo::ScopedString Str;
Allocator->getStats(&Str);
Str.output();
@@ -300,7 +300,7 @@ SCUDO_TYPED_TEST(ScudoPrimaryTest, PrimaryThreaded) {
}
for (auto &T : Threads)
T.join();
- Allocator->releaseToOS();
+ Allocator->releaseToOS(scudo::ReleaseToOS::Force);
scudo::ScopedString Str;
Allocator->getStats(&Str);
Str.output();
@@ -322,7 +322,7 @@ SCUDO_TYPED_TEST(ScudoPrimaryTest, ReleaseToOS) {
EXPECT_NE(P, nullptr);
Cache.deallocate(ClassId, P);
Cache.destroy(nullptr);
- EXPECT_GT(Allocator->releaseToOS(), 0U);
+ EXPECT_GT(Allocator->releaseToOS(scudo::ReleaseToOS::Force), 0U);
}
SCUDO_TYPED_TEST(ScudoPrimaryTest, MemoryGroup) {
diff --git a/compiler-rt/lib/scudo/standalone/wrappers_c.inc b/compiler-rt/lib/scudo/standalone/wrappers_c.inc
index 37e336ee09d62..106a2875a1fef 100644
--- a/compiler-rt/lib/scudo/standalone/wrappers_c.inc
+++ b/compiler-rt/lib/scudo/standalone/wrappers_c.inc
@@ -188,7 +188,10 @@ INTERFACE WEAK int SCUDO_PREFIX(mallopt)(int param, int value) {
static_cast<scudo::sptr>(value));
return 1;
} else if (param == M_PURGE) {
- SCUDO_ALLOCATOR.releaseToOS();
+ SCUDO_ALLOCATOR.releaseToOS(scudo::ReleaseToOS::Force);
+ return 1;
+ } else if (param == M_PURGE_ALL) {
+ SCUDO_ALLOCATOR.releaseToOS(scudo::ReleaseToOS::ForceAll);
return 1;
} else {
scudo::Option option;
More information about the llvm-commits
mailing list