[llvm] Tweak BumpPtrAllocator to benefit the hot path (PR #90571)

[Owen Anderson via llvm-commits]

https://github.com/resistor created https://github.com/llvm/llvm-project/pull/90571

This takes the form of three consecutive but related changes:
- Mark the fast path of BumpPtrAllocator as likely-taken.
- Move the slow path of BumpPtrAllocator to a separate function.
- Mark the slow path of BumpPtrAllocator as noinline.

Overall, this saves geomean 0.4% userspace instructions on CTMark -O3,
and 0.98% on CTMark -O0 -g.

http://llvm-compile-time-tracker.com/compare.php?from=e1622e189e8c0ef457bfac528f90a7a930d9aad2&to=9eb53a4ed3af4a55e769ae1dd22d034b63d046e3&stat=instructions%3Au


>From 60c04319287d80c17a3eea43023cc90ae87c4b96 Mon Sep 17 00:00:00 2001
From: Owen Anderson <resistor at mac.com>
Date: Mon, 29 Apr 2024 14:16:25 -0600
Subject: [PATCH 1/3] Mark the fast path of BumpPtrAllocator as likely.

---
 llvm/include/llvm/Support/Allocator.h | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/llvm/include/llvm/Support/Allocator.h b/llvm/include/llvm/Support/Allocator.h
index c1e5c6d2853bd5..cb721c45893f79 100644
--- a/llvm/include/llvm/Support/Allocator.h
+++ b/llvm/include/llvm/Support/Allocator.h
@@ -159,9 +159,9 @@ class BumpPtrAllocatorImpl
 #endif
 
     // Check if we have enough space.
-    if (Adjustment + SizeToAllocate <= size_t(End - CurPtr)
-        // We can't return nullptr even for a zero-sized allocation!
-        && CurPtr != nullptr) {
+    if (LLVM_LIKELY(Adjustment + SizeToAllocate <= size_t(End - CurPtr)
+                    // We can't return nullptr even for a zero-sized allocation!
+                    && CurPtr != nullptr)) {
       char *AlignedPtr = CurPtr + Adjustment;
       CurPtr = AlignedPtr + SizeToAllocate;
       // Update the allocation point of this memory block in MemorySanitizer.

>From 5471dd94f88f3e79ccc6b43935d6e62c7540b917 Mon Sep 17 00:00:00 2001
From: Owen Anderson <resistor at mac.com>
Date: Sun, 28 Apr 2024 10:15:59 -0600
Subject: [PATCH 2/3] Move the slow path of BumpPtrAllocator to a separate
 function.

---
 llvm/include/llvm/Support/Allocator.h | 5 +++++
 1 file changed, 5 insertions(+)

diff --git a/llvm/include/llvm/Support/Allocator.h b/llvm/include/llvm/Support/Allocator.h
index cb721c45893f79..26e275fc682dfb 100644
--- a/llvm/include/llvm/Support/Allocator.h
+++ b/llvm/include/llvm/Support/Allocator.h
@@ -173,6 +173,11 @@ class BumpPtrAllocatorImpl
       return AlignedPtr;
     }
 
+    return AllocateSlow(SizeToAllocate, Alignment);
+  }
+
+  LLVM_ATTRIBUTE_RETURNS_NONNULL void *AllocateSlow(size_t SizeToAllocate,
+                                                    Align Alignment) {
     // If Size is really big, allocate a separate slab for it.
     size_t PaddedSize = SizeToAllocate + Alignment.value() - 1;
     if (PaddedSize > SizeThreshold) {

>From 9eb53a4ed3af4a55e769ae1dd22d034b63d046e3 Mon Sep 17 00:00:00 2001
From: Owen Anderson <resistor at mac.com>
Date: Mon, 29 Apr 2024 14:18:27 -0600
Subject: [PATCH 3/3] Mark the slow path of BumpPtrAllocator as noinline.

---
 llvm/include/llvm/Support/Allocator.h | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/llvm/include/llvm/Support/Allocator.h b/llvm/include/llvm/Support/Allocator.h
index 26e275fc682dfb..873d453650187e 100644
--- a/llvm/include/llvm/Support/Allocator.h
+++ b/llvm/include/llvm/Support/Allocator.h
@@ -176,8 +176,8 @@ class BumpPtrAllocatorImpl
     return AllocateSlow(SizeToAllocate, Alignment);
   }
 
-  LLVM_ATTRIBUTE_RETURNS_NONNULL void *AllocateSlow(size_t SizeToAllocate,
-                                                    Align Alignment) {
+  LLVM_ATTRIBUTE_RETURNS_NONNULL LLVM_ATTRIBUTE_NOINLINE void *
+  AllocateSlow(size_t SizeToAllocate, Align Alignment) {
     // If Size is really big, allocate a separate slab for it.
     size_t PaddedSize = SizeToAllocate + Alignment.value() - 1;
     if (PaddedSize > SizeThreshold) {