[compiler-rt] 4634a48 - [scudo] Add a method to use a hard-coded page size (#106646)
via llvm-commits
llvm-commits at lists.llvm.org
Thu Sep 5 13:43:37 PDT 2024
Author: Christopher Ferris
Date: 2024-09-05T13:43:34-07:00
New Revision: 4634a480e0e5aa3116b397369fe3877a8dfe4dc0
URL: https://github.com/llvm/llvm-project/commit/4634a480e0e5aa3116b397369fe3877a8dfe4dc0
DIFF: https://github.com/llvm/llvm-project/commit/4634a480e0e5aa3116b397369fe3877a8dfe4dc0.diff
LOG: [scudo] Add a method to use a hard-coded page size (#106646)
Currently, only Android supports using a hard-code page size. Make this
a bit more generic so any platform that wants to can use this.
In addition, add a getPageSizeLogCached() function since this value is
used in release.h and can avoid keeping this value around in objects.
Finally, change some of the release.h page size multiplies to shifts
using the new page size log value.
Added:
Modified:
compiler-rt/lib/scudo/standalone/combined.h
compiler-rt/lib/scudo/standalone/common.cpp
compiler-rt/lib/scudo/standalone/common.h
compiler-rt/lib/scudo/standalone/linux.cpp
compiler-rt/lib/scudo/standalone/platform.h
compiler-rt/lib/scudo/standalone/release.h
Removed:
################################################################################
diff --git a/compiler-rt/lib/scudo/standalone/combined.h b/compiler-rt/lib/scudo/standalone/combined.h
index 9c26282e6f860a..a5f1bc388e8824 100644
--- a/compiler-rt/lib/scudo/standalone/combined.h
+++ b/compiler-rt/lib/scudo/standalone/combined.h
@@ -140,6 +140,9 @@ class Allocator {
typedef typename QuarantineT::CacheT QuarantineCacheT;
void init() {
+ // Make sure that the page size is initialized if it's not a constant.
+ CHECK_NE(getPageSizeCached(), 0U);
+
performSanityChecks();
// Check if hardware CRC32 is supported in the binary and by the platform,
diff --git a/compiler-rt/lib/scudo/standalone/common.cpp b/compiler-rt/lib/scudo/standalone/common.cpp
index 06e930638f6f97..80134c39e757de 100644
--- a/compiler-rt/lib/scudo/standalone/common.cpp
+++ b/compiler-rt/lib/scudo/standalone/common.cpp
@@ -12,13 +12,21 @@
namespace scudo {
-uptr PageSizeCached;
+#if !defined(SCUDO_PAGE_SIZE)
+uptr PageSizeCached = 0;
+uptr PageSizeLogCached = 0;
+
+// Must be defined in platform specific code.
uptr getPageSize();
+// This must be called in the init path or there could be a race if multiple
+// threads try to set the cached values.
uptr getPageSizeSlow() {
PageSizeCached = getPageSize();
CHECK_NE(PageSizeCached, 0);
+ PageSizeLogCached = getLog2(PageSizeCached);
return PageSizeCached;
}
+#endif
} // namespace scudo
diff --git a/compiler-rt/lib/scudo/standalone/common.h b/compiler-rt/lib/scudo/standalone/common.h
index 151fbd317e74d3..e5dfda2e9072a0 100644
--- a/compiler-rt/lib/scudo/standalone/common.h
+++ b/compiler-rt/lib/scudo/standalone/common.h
@@ -133,18 +133,40 @@ inline void computePercentage(uptr Numerator, uptr Denominator, uptr *Integral,
// Platform specific functions.
+#if defined(SCUDO_PAGE_SIZE)
+
+inline constexpr uptr getPageSizeCached() { return SCUDO_PAGE_SIZE; }
+
+inline constexpr uptr getPageSizeSlow() { return getPageSizeCached(); }
+
+inline constexpr uptr getPageSizeLogCached() {
+ return static_cast<uptr>(__builtin_ctzl(SCUDO_PAGE_SIZE));
+}
+
+#else
+
extern uptr PageSizeCached;
+extern uptr PageSizeLogCached;
+
uptr getPageSizeSlow();
+
inline uptr getPageSizeCached() {
-#if SCUDO_ANDROID && defined(PAGE_SIZE)
- // Most Android builds have a build-time constant page size.
- return PAGE_SIZE;
-#endif
if (LIKELY(PageSizeCached))
return PageSizeCached;
return getPageSizeSlow();
}
+inline uptr getPageSizeLogCached() {
+ if (LIKELY(PageSizeLogCached))
+ return PageSizeLogCached;
+ // PageSizeLogCached and PageSizeCached are both set in getPageSizeSlow()
+ getPageSizeSlow();
+ DCHECK_NE(PageSizeLogCached, 0);
+ return PageSizeLogCached;
+}
+
+#endif
+
// Returns 0 if the number of CPUs could not be determined.
u32 getNumberOfCPUs();
diff --git a/compiler-rt/lib/scudo/standalone/linux.cpp b/compiler-rt/lib/scudo/standalone/linux.cpp
index 27469510810986..6cc8e0c786e06a 100644
--- a/compiler-rt/lib/scudo/standalone/linux.cpp
+++ b/compiler-rt/lib/scudo/standalone/linux.cpp
@@ -40,7 +40,10 @@
namespace scudo {
+#if !defined(SCUDO_PAGE_SIZE)
+// This function is only used when page size is not hard-coded.
uptr getPageSize() { return static_cast<uptr>(sysconf(_SC_PAGESIZE)); }
+#endif
void NORETURN die() { abort(); }
diff --git a/compiler-rt/lib/scudo/standalone/platform.h b/compiler-rt/lib/scudo/standalone/platform.h
index 5af1275e32d2b6..3f017faaee78f3 100644
--- a/compiler-rt/lib/scudo/standalone/platform.h
+++ b/compiler-rt/lib/scudo/standalone/platform.h
@@ -21,6 +21,11 @@
// See https://android.googlesource.com/platform/bionic/+/master/docs/defines.md
#if defined(__BIONIC__)
#define SCUDO_ANDROID 1
+// Transitive includes of unistd.h will get PAGE_SIZE if it is defined.
+#include <unistd.h>
+#if defined(PAGE_SIZE)
+#define SCUDO_PAGE_SIZE PAGE_SIZE
+#endif
#else
#define SCUDO_ANDROID 0
#endif
diff --git a/compiler-rt/lib/scudo/standalone/release.h b/compiler-rt/lib/scudo/standalone/release.h
index 69f926e3f8680b..51abdd82aa5389 100644
--- a/compiler-rt/lib/scudo/standalone/release.h
+++ b/compiler-rt/lib/scudo/standalone/release.h
@@ -88,7 +88,7 @@ class FragmentationRecorder {
void releasePageRangeToOS(uptr From, uptr To) {
DCHECK_EQ((To - From) % getPageSizeCached(), 0U);
- ReleasedPagesCount += (To - From) / getPageSizeCached();
+ ReleasedPagesCount += (To - From) >> getPageSizeLogCached();
}
private:
@@ -348,7 +348,7 @@ class RegionPageMap {
template <class ReleaseRecorderT> class FreePagesRangeTracker {
public:
explicit FreePagesRangeTracker(ReleaseRecorderT &Recorder)
- : Recorder(Recorder), PageSizeLog(getLog2(getPageSizeCached())) {}
+ : Recorder(Recorder) {}
void processNextPage(bool Released) {
if (Released) {
@@ -372,6 +372,7 @@ template <class ReleaseRecorderT> class FreePagesRangeTracker {
private:
void closeOpenedRange() {
if (InRange) {
+ const uptr PageSizeLog = getPageSizeLogCached();
Recorder.releasePageRangeToOS((CurrentRangeStatePage << PageSizeLog),
(CurrentPage << PageSizeLog));
InRange = false;
@@ -379,7 +380,6 @@ template <class ReleaseRecorderT> class FreePagesRangeTracker {
}
ReleaseRecorderT &Recorder;
- const uptr PageSizeLog;
bool InRange = false;
uptr CurrentPage = 0;
uptr CurrentRangeStatePage = 0;
@@ -389,7 +389,7 @@ struct PageReleaseContext {
PageReleaseContext(uptr BlockSize, uptr NumberOfRegions, uptr ReleaseSize,
uptr ReleaseOffset = 0)
: BlockSize(BlockSize), NumberOfRegions(NumberOfRegions) {
- PageSize = getPageSizeCached();
+ const uptr PageSize = getPageSizeCached();
if (BlockSize <= PageSize) {
if (PageSize % BlockSize == 0) {
// Same number of chunks per page, no cross overs.
@@ -408,7 +408,7 @@ struct PageReleaseContext {
SameBlockCountPerPage = false;
}
} else {
- if (BlockSize % PageSize == 0) {
+ if ((BlockSize & (PageSize - 1)) == 0) {
// One chunk covers multiple pages, no cross overs.
FullPagesBlockCountMax = 1;
SameBlockCountPerPage = true;
@@ -427,8 +427,8 @@ struct PageReleaseContext {
if (NumberOfRegions != 1)
DCHECK_EQ(ReleaseOffset, 0U);
- PagesCount = roundUp(ReleaseSize, PageSize) / PageSize;
- PageSizeLog = getLog2(PageSize);
+ const uptr PageSizeLog = getPageSizeLogCached();
+ PagesCount = roundUp(ReleaseSize, PageSize) >> PageSizeLog;
ReleasePageOffset = ReleaseOffset >> PageSizeLog;
}
@@ -451,6 +451,7 @@ struct PageReleaseContext {
// RegionSize, it's not necessary to be aligned with page size.
bool markRangeAsAllCounted(uptr From, uptr To, uptr Base,
const uptr RegionIndex, const uptr RegionSize) {
+ const uptr PageSize = getPageSizeCached();
DCHECK_LT(From, To);
DCHECK_LE(To, Base + RegionSize);
DCHECK_EQ(From % PageSize, 0U);
@@ -544,6 +545,7 @@ struct PageReleaseContext {
if (!ensurePageMapAllocated())
return false;
+ const uptr PageSize = getPageSizeCached();
if (MayContainLastBlockInRegion) {
const uptr LastBlockInRegion =
((RegionSize / BlockSize) - 1U) * BlockSize;
@@ -605,17 +607,19 @@ struct PageReleaseContext {
return true;
}
- uptr getPageIndex(uptr P) { return (P >> PageSizeLog) - ReleasePageOffset; }
- uptr getReleaseOffset() { return ReleasePageOffset << PageSizeLog; }
+ uptr getPageIndex(uptr P) {
+ return (P >> getPageSizeLogCached()) - ReleasePageOffset;
+ }
+ uptr getReleaseOffset() {
+ return ReleasePageOffset << getPageSizeLogCached();
+ }
uptr BlockSize;
uptr NumberOfRegions;
// For partial region marking, some pages in front are not needed to be
// counted.
uptr ReleasePageOffset;
- uptr PageSize;
uptr PagesCount;
- uptr PageSizeLog;
uptr FullPagesBlockCountMax;
bool SameBlockCountPerPage;
RegionPageMap PageMap;
@@ -628,7 +632,7 @@ template <class ReleaseRecorderT, typename SkipRegionT>
NOINLINE void
releaseFreeMemoryToOS(PageReleaseContext &Context,
ReleaseRecorderT &Recorder, SkipRegionT SkipRegion) {
- const uptr PageSize = Context.PageSize;
+ const uptr PageSize = getPageSizeCached();
const uptr BlockSize = Context.BlockSize;
const uptr PagesCount = Context.PagesCount;
const uptr NumberOfRegions = Context.NumberOfRegions;
@@ -671,7 +675,7 @@ releaseFreeMemoryToOS(PageReleaseContext &Context,
uptr PrevPageBoundary = 0;
uptr CurrentBoundary = 0;
if (ReleasePageOffset > 0) {
- PrevPageBoundary = ReleasePageOffset * PageSize;
+ PrevPageBoundary = ReleasePageOffset << getPageSizeLogCached();
CurrentBoundary = roundUpSlow(PrevPageBoundary, BlockSize);
}
for (uptr J = 0; J < PagesCount; J++) {
More information about the llvm-commits
mailing list