[compiler-rt] [scudo] Add fragmentation info for each memory group (PR #107475)

[via llvm-commits]

https://github.com/ChiaHungDuan updated https://github.com/llvm/llvm-project/pull/107475

>From 7cc79ddf2df1cab02a5e26cd30b35a8385e305ca Mon Sep 17 00:00:00 2001
From: Chia-hung Duan <chiahungduan at google.com>
Date: Thu, 5 Sep 2024 21:51:38 +0000
Subject: [PATCH 1/2] [scudo] Add fragmentation info for each memory group

This information helps with tuning the heuristic of selecting memory
groups to release the unused pages.
---
 compiler-rt/lib/scudo/standalone/primary32.h  |  6 ++
 compiler-rt/lib/scudo/standalone/primary64.h  | 57 +++++++++++++++++++
 compiler-rt/lib/scudo/standalone/release.h    | 16 ++++++
 .../scudo/standalone/tests/primary_test.cpp   |  1 +
 4 files changed, 80 insertions(+)

diff --git a/compiler-rt/lib/scudo/standalone/primary32.h b/compiler-rt/lib/scudo/standalone/primary32.h
index 87264471be2c1a..9ef5e7102e3220 100644
--- a/compiler-rt/lib/scudo/standalone/primary32.h
+++ b/compiler-rt/lib/scudo/standalone/primary32.h
@@ -332,6 +332,12 @@ template <typename Config> class SizeClassAllocator32 {
     }
   }
 
+  void getMemoryGroupFragmentationInfo(ScopedString *Str) {
+    // Each region is also a memory group because region size is the same as
+    // group size.
+    getFragmentationInfo(Str);
+  }
+
   bool setOption(Option O, sptr Value) {
     if (O == Option::ReleaseInterval) {
       const s32 Interval = Max(
diff --git a/compiler-rt/lib/scudo/standalone/primary64.h b/compiler-rt/lib/scudo/standalone/primary64.h
index ffcc22fea0c6ea..1bcfc54235b1ad 100644
--- a/compiler-rt/lib/scudo/standalone/primary64.h
+++ b/compiler-rt/lib/scudo/standalone/primary64.h
@@ -395,6 +395,18 @@ template <typename Config> class SizeClassAllocator64 {
     }
   }
 
+  void getMemoryGroupFragmentationInfo(ScopedString *Str) {
+    Str->append(
+        "Fragmentation Stats: SizeClassAllocator64: page size = %zu bytes\n",
+        getPageSizeCached());
+
+    for (uptr I = 1; I < NumClasses; I++) {
+      RegionInfo *Region = getRegionInfo(I);
+      ScopedLock L(Region->MMLock);
+      getMemoryGroupFragmentationInfoInRegion(Region, I, Str);
+    }
+  }
+
   bool setOption(Option O, sptr Value) {
     if (O == Option::ReleaseInterval) {
       const s32 Interval = Max(
@@ -1193,6 +1205,51 @@ template <typename Config> class SizeClassAllocator64 {
                 AllocatedPagesCount, InUseBytes >> 10, Integral, Fractional);
   }
 
+  void getMemoryGroupFragmentationInfoInRegion(RegionInfo *Region, uptr ClassId,
+                                               ScopedString *Str)
+      REQUIRES(Region->MMLock) {
+    const uptr BlockSize = getSizeByClassId(ClassId);
+    const uptr AllocatedUserEnd =
+        Region->MemMapInfo.AllocatedUser + Region->RegionBeg;
+
+    SinglyLinkedList<BatchGroupT> GroupsToRelease;
+    {
+      ScopedLock L(Region->FLLock);
+      GroupsToRelease = Region->FreeListInfo.BlockList;
+      Region->FreeListInfo.BlockList.clear();
+    }
+
+    constexpr uptr GroupSize = (1UL << GroupSizeLog);
+    constexpr uptr MaxNumGroups = RegionSize / GroupSize;
+
+    MemoryGroupFragmentationRecorder<GroupSize, MaxNumGroups> Recorder;
+    if (!GroupsToRelease.empty()) {
+      PageReleaseContext Context =
+          markFreeBlocks(Region, BlockSize, AllocatedUserEnd,
+                         getCompactPtrBaseByClassId(ClassId), GroupsToRelease);
+      auto SkipRegion = [](UNUSED uptr RegionIndex) { return false; };
+      releaseFreeMemoryToOS(Context, Recorder, SkipRegion);
+
+      mergeGroupsToReleaseBack(Region, GroupsToRelease);
+    }
+
+    Str->append("MemoryGroupFragmentationInfo in Region %zu (%zu)\n", ClassId,
+                BlockSize);
+
+    const uptr MaxNumGroupsInUse =
+        roundUp(Region->MemMapInfo.AllocatedUser, GroupSize) / GroupSize;
+    for (uptr I = 0; I < MaxNumGroupsInUse; ++I) {
+
+      uptr Integral;
+      uptr Fractional;
+      computePercentage(Recorder.NumPagesInOneGroup -
+                            Recorder.getNumFreePages(I),
+                        Recorder.NumPagesInOneGroup, &Integral, &Fractional);
+      Str->append("MemoryGroup #%zu (0x%zx): util: %3zu.%02zu%%\n", I,
+                  Region->RegionBeg + I * GroupSize, Integral, Fractional);
+    }
+  }
+
   NOINLINE uptr releaseToOSMaybe(RegionInfo *Region, uptr ClassId,
                                  ReleaseToOS ReleaseType = ReleaseToOS::Normal)
       REQUIRES(Region->MMLock) EXCLUDES(Region->FLLock) {
diff --git a/compiler-rt/lib/scudo/standalone/release.h b/compiler-rt/lib/scudo/standalone/release.h
index 51abdd82aa5389..6353dafde31609 100644
--- a/compiler-rt/lib/scudo/standalone/release.h
+++ b/compiler-rt/lib/scudo/standalone/release.h
@@ -95,6 +95,22 @@ class FragmentationRecorder {
   uptr ReleasedPagesCount = 0;
 };
 
+template <uptr GroupSize, uptr NumGroups>
+class MemoryGroupFragmentationRecorder {
+public:
+  const uptr NumPagesInOneGroup = GroupSize / getPageSizeCached();
+
+  void releasePageRangeToOS(uptr From, uptr To) {
+    for (uptr I = From / getPageSizeCached(); I < To / getPageSizeCached(); ++I)
+      ++FreePagesCount[I / NumPagesInOneGroup];
+  }
+
+  uptr getNumFreePages(uptr GroupId) { return FreePagesCount[GroupId]; }
+
+private:
+  uptr FreePagesCount[NumGroups] = {};
+};
+
 // A buffer pool which holds a fixed number of static buffers of `uptr` elements
 // for fast buffer allocation. If the request size is greater than
 // `StaticBufferNumElements` or if all the static buffers are in use, it'll
diff --git a/compiler-rt/lib/scudo/standalone/tests/primary_test.cpp b/compiler-rt/lib/scudo/standalone/tests/primary_test.cpp
index 1cf3bb51db0e71..0636fe73c0b591 100644
--- a/compiler-rt/lib/scudo/standalone/tests/primary_test.cpp
+++ b/compiler-rt/lib/scudo/standalone/tests/primary_test.cpp
@@ -386,6 +386,7 @@ SCUDO_TYPED_TEST(ScudoPrimaryTest, PrimaryThreaded) {
   scudo::ScopedString Str;
   Allocator->getStats(&Str);
   Allocator->getFragmentationInfo(&Str);
+  Allocator->getMemoryGroupFragmentationInfo(&Str);
   Str.output();
 }
 

>From ffb5194dcbeedbcf7a214b6f66115edef810066b Mon Sep 17 00:00:00 2001
From: Chia-hung Duan <chiahungduan at google.com>
Date: Mon, 9 Sep 2024 18:41:04 +0000
Subject: [PATCH 2/2] Address review comment

---
 compiler-rt/lib/scudo/standalone/primary64.h | 1 -
 1 file changed, 1 deletion(-)

diff --git a/compiler-rt/lib/scudo/standalone/primary64.h b/compiler-rt/lib/scudo/standalone/primary64.h
index 1bcfc54235b1ad..045070d0e34de9 100644
--- a/compiler-rt/lib/scudo/standalone/primary64.h
+++ b/compiler-rt/lib/scudo/standalone/primary64.h
@@ -1239,7 +1239,6 @@ template <typename Config> class SizeClassAllocator64 {
     const uptr MaxNumGroupsInUse =
         roundUp(Region->MemMapInfo.AllocatedUser, GroupSize) / GroupSize;
     for (uptr I = 0; I < MaxNumGroupsInUse; ++I) {
-
       uptr Integral;
       uptr Fractional;
       computePercentage(Recorder.NumPagesInOneGroup -