[compiler-rt] [scudo] Add a method to use a hard-coded page size (PR #106646)
via llvm-commits
llvm-commits at lists.llvm.org
Thu Aug 29 17:16:26 PDT 2024
llvmbot wrote:
<!--LLVM PR SUMMARY COMMENT-->
@llvm/pr-subscribers-compiler-rt-sanitizer
Author: Christopher Ferris (cferris1000)
<details>
<summary>Changes</summary>
Currently, only Android supports using a hard-code page size. Make this a bit more generic so any platform that wants to can use this.
In addition, add a getPageSizeLogCached() function since this value is used in release.h and can avoid keeping this value around in objects.
Finally, change some of the release.h page size multiplies to shifts using the new page size log value.
---
Full diff: https://github.com/llvm/llvm-project/pull/106646.diff
5 Files Affected:
- (modified) compiler-rt/lib/scudo/standalone/combined.h (+5)
- (modified) compiler-rt/lib/scudo/standalone/common.cpp (+7-1)
- (modified) compiler-rt/lib/scudo/standalone/common.h (+29-4)
- (modified) compiler-rt/lib/scudo/standalone/platform.h (+5)
- (modified) compiler-rt/lib/scudo/standalone/release.h (+13-13)
``````````diff
diff --git a/compiler-rt/lib/scudo/standalone/combined.h b/compiler-rt/lib/scudo/standalone/combined.h
index 9c26282e6f860a..c37779784a6217 100644
--- a/compiler-rt/lib/scudo/standalone/combined.h
+++ b/compiler-rt/lib/scudo/standalone/combined.h
@@ -140,6 +140,11 @@ class Allocator {
typedef typename QuarantineT::CacheT QuarantineCacheT;
void init() {
+#if !defined(SCUDO_PAGE_SIZE)
+ // Make sure that the page size is initialized if it's not a constant.
+ CHECK_NE(getPageSizeCached(), 0U);
+#endif
+
performSanityChecks();
// Check if hardware CRC32 is supported in the binary and by the platform,
diff --git a/compiler-rt/lib/scudo/standalone/common.cpp b/compiler-rt/lib/scudo/standalone/common.cpp
index 06e930638f6f97..fb95240a721255 100644
--- a/compiler-rt/lib/scudo/standalone/common.cpp
+++ b/compiler-rt/lib/scudo/standalone/common.cpp
@@ -12,13 +12,19 @@
namespace scudo {
-uptr PageSizeCached;
+#if !defined(SCUDO_PAGE_SIZE)
+uptr PageSizeCached = 0;
+uptr PageSizeLogCached = 0;
uptr getPageSize();
+// This must be called in the init path or there could be a race if multiple
+// threads try to set the cached values.
uptr getPageSizeSlow() {
PageSizeCached = getPageSize();
CHECK_NE(PageSizeCached, 0);
+ PageSizeLogCached = static_cast<uptr>(__builtin_ctzl(PageSizeCached));
return PageSizeCached;
}
+#endif
} // namespace scudo
diff --git a/compiler-rt/lib/scudo/standalone/common.h b/compiler-rt/lib/scudo/standalone/common.h
index 151fbd317e74d3..ee9029bab3b069 100644
--- a/compiler-rt/lib/scudo/standalone/common.h
+++ b/compiler-rt/lib/scudo/standalone/common.h
@@ -133,18 +133,43 @@ inline void computePercentage(uptr Numerator, uptr Denominator, uptr *Integral,
// Platform specific functions.
+#if defined(SCUDO_PAGE_SIZE)
+
+inline constexpr uptr getPageSizeCached() {
+ return SCUDO_PAGE_SIZE;
+}
+
+inline constexpr uptr getPageSizeSlow() {
+ return getPageSizeCached();
+}
+
+inline constexpr uptr getPageSizeLogCached() {
+ return static_cast<uptr>(__builtin_ctzl(SCUDO_PAGE_SIZE));
+}
+
+#else
+
extern uptr PageSizeCached;
+extern uptr PageSizeLogCached;
+
uptr getPageSizeSlow();
+
inline uptr getPageSizeCached() {
-#if SCUDO_ANDROID && defined(PAGE_SIZE)
- // Most Android builds have a build-time constant page size.
- return PAGE_SIZE;
-#endif
if (LIKELY(PageSizeCached))
return PageSizeCached;
return getPageSizeSlow();
}
+inline uptr getPageSizeLogCached() {
+ if (LIKELY(PageSizeLogCached)) {
+ return PageSizeLogCached;
+ }
+ getPageSizeSlow();
+ return PageSizeLogCached;
+}
+
+#endif
+
// Returns 0 if the number of CPUs could not be determined.
u32 getNumberOfCPUs();
diff --git a/compiler-rt/lib/scudo/standalone/platform.h b/compiler-rt/lib/scudo/standalone/platform.h
index 5af1275e32d2b6..3f017faaee78f3 100644
--- a/compiler-rt/lib/scudo/standalone/platform.h
+++ b/compiler-rt/lib/scudo/standalone/platform.h
@@ -21,6 +21,11 @@
// See https://android.googlesource.com/platform/bionic/+/master/docs/defines.md
#if defined(__BIONIC__)
#define SCUDO_ANDROID 1
+// Transitive includes of unistd.h will get PAGE_SIZE if it is defined.
+#include <unistd.h>
+#if defined(PAGE_SIZE)
+#define SCUDO_PAGE_SIZE PAGE_SIZE
+#endif
#else
#define SCUDO_ANDROID 0
#endif
diff --git a/compiler-rt/lib/scudo/standalone/release.h b/compiler-rt/lib/scudo/standalone/release.h
index 69f926e3f8680b..f8177a85ee395b 100644
--- a/compiler-rt/lib/scudo/standalone/release.h
+++ b/compiler-rt/lib/scudo/standalone/release.h
@@ -88,7 +88,7 @@ class FragmentationRecorder {
void releasePageRangeToOS(uptr From, uptr To) {
DCHECK_EQ((To - From) % getPageSizeCached(), 0U);
- ReleasedPagesCount += (To - From) / getPageSizeCached();
+ ReleasedPagesCount += (To - From) >> getPageSizeLogCached();
}
private:
@@ -348,7 +348,7 @@ class RegionPageMap {
template <class ReleaseRecorderT> class FreePagesRangeTracker {
public:
explicit FreePagesRangeTracker(ReleaseRecorderT &Recorder)
- : Recorder(Recorder), PageSizeLog(getLog2(getPageSizeCached())) {}
+ : Recorder(Recorder) {}
void processNextPage(bool Released) {
if (Released) {
@@ -372,6 +372,7 @@ template <class ReleaseRecorderT> class FreePagesRangeTracker {
private:
void closeOpenedRange() {
if (InRange) {
+ const uptr PageSizeLog = getPageSizeLogCached();
Recorder.releasePageRangeToOS((CurrentRangeStatePage << PageSizeLog),
(CurrentPage << PageSizeLog));
InRange = false;
@@ -379,7 +380,6 @@ template <class ReleaseRecorderT> class FreePagesRangeTracker {
}
ReleaseRecorderT &Recorder;
- const uptr PageSizeLog;
bool InRange = false;
uptr CurrentPage = 0;
uptr CurrentRangeStatePage = 0;
@@ -389,7 +389,7 @@ struct PageReleaseContext {
PageReleaseContext(uptr BlockSize, uptr NumberOfRegions, uptr ReleaseSize,
uptr ReleaseOffset = 0)
: BlockSize(BlockSize), NumberOfRegions(NumberOfRegions) {
- PageSize = getPageSizeCached();
+ const uptr PageSize = getPageSizeCached();
if (BlockSize <= PageSize) {
if (PageSize % BlockSize == 0) {
// Same number of chunks per page, no cross overs.
@@ -408,7 +408,7 @@ struct PageReleaseContext {
SameBlockCountPerPage = false;
}
} else {
- if (BlockSize % PageSize == 0) {
+ if ((BlockSize & (PageSize - 1)) == 0) {
// One chunk covers multiple pages, no cross overs.
FullPagesBlockCountMax = 1;
SameBlockCountPerPage = true;
@@ -427,8 +427,8 @@ struct PageReleaseContext {
if (NumberOfRegions != 1)
DCHECK_EQ(ReleaseOffset, 0U);
- PagesCount = roundUp(ReleaseSize, PageSize) / PageSize;
- PageSizeLog = getLog2(PageSize);
+ const uptr PageSizeLog = getPageSizeLogCached();
+ PagesCount = roundUp(ReleaseSize, PageSize) >> PageSizeLog;
ReleasePageOffset = ReleaseOffset >> PageSizeLog;
}
@@ -451,6 +451,7 @@ struct PageReleaseContext {
// RegionSize, it's not necessary to be aligned with page size.
bool markRangeAsAllCounted(uptr From, uptr To, uptr Base,
const uptr RegionIndex, const uptr RegionSize) {
+ const uptr PageSize = getPageSizeCached();
DCHECK_LT(From, To);
DCHECK_LE(To, Base + RegionSize);
DCHECK_EQ(From % PageSize, 0U);
@@ -544,6 +545,7 @@ struct PageReleaseContext {
if (!ensurePageMapAllocated())
return false;
+ const uptr PageSize = getPageSizeCached();
if (MayContainLastBlockInRegion) {
const uptr LastBlockInRegion =
((RegionSize / BlockSize) - 1U) * BlockSize;
@@ -605,17 +607,15 @@ struct PageReleaseContext {
return true;
}
- uptr getPageIndex(uptr P) { return (P >> PageSizeLog) - ReleasePageOffset; }
- uptr getReleaseOffset() { return ReleasePageOffset << PageSizeLog; }
+ uptr getPageIndex(uptr P) { return (P >> getPageSizeLogCached()) - ReleasePageOffset; }
+ uptr getReleaseOffset() { return ReleasePageOffset << getPageSizeLogCached(); }
uptr BlockSize;
uptr NumberOfRegions;
// For partial region marking, some pages in front are not needed to be
// counted.
uptr ReleasePageOffset;
- uptr PageSize;
uptr PagesCount;
- uptr PageSizeLog;
uptr FullPagesBlockCountMax;
bool SameBlockCountPerPage;
RegionPageMap PageMap;
@@ -628,7 +628,7 @@ template <class ReleaseRecorderT, typename SkipRegionT>
NOINLINE void
releaseFreeMemoryToOS(PageReleaseContext &Context,
ReleaseRecorderT &Recorder, SkipRegionT SkipRegion) {
- const uptr PageSize = Context.PageSize;
+ const uptr PageSize = getPageSizeCached();
const uptr BlockSize = Context.BlockSize;
const uptr PagesCount = Context.PagesCount;
const uptr NumberOfRegions = Context.NumberOfRegions;
@@ -671,7 +671,7 @@ releaseFreeMemoryToOS(PageReleaseContext &Context,
uptr PrevPageBoundary = 0;
uptr CurrentBoundary = 0;
if (ReleasePageOffset > 0) {
- PrevPageBoundary = ReleasePageOffset * PageSize;
+ PrevPageBoundary = ReleasePageOffset << getPageSizeLogCached();
CurrentBoundary = roundUpSlow(PrevPageBoundary, BlockSize);
}
for (uptr J = 0; J < PagesCount; J++) {
``````````
</details>
https://github.com/llvm/llvm-project/pull/106646
More information about the llvm-commits
mailing list