[llvm] Tweak BumpPtrAllocator to benefit the hot path (PR #90571)

[Owen Anderson via llvm-commits]

https://github.com/resistor updated https://github.com/llvm/llvm-project/pull/90571

>From ab3ff6612a018e0bfed9eb7666890592ce6776df Mon Sep 17 00:00:00 2001
From: Owen Anderson <resistor at mac.com>
Date: Mon, 29 Apr 2024 14:16:25 -0600
Subject: [PATCH] Tweak BumpPtrAllocator to benefit the hot path.

This takes the form of three consecutive but related changes:
- Mark the fast path of BumpPtrAllocator as likely-taken.
- Move the slow path of BumpPtrAllocator to a separate function.
- Mark the slow path of BumpPtrAllocator as noinline.

Overall, this saves geomean 0.4% userspace instructions on CTMark -O3,
and 0.98% on CTMark -O0 -g.
---
 llvm/include/llvm/Support/Allocator.h | 11 ++++++++---
 1 file changed, 8 insertions(+), 3 deletions(-)

diff --git a/llvm/include/llvm/Support/Allocator.h b/llvm/include/llvm/Support/Allocator.h
index c1e5c6d2853bd5..873d453650187e 100644
--- a/llvm/include/llvm/Support/Allocator.h
+++ b/llvm/include/llvm/Support/Allocator.h
@@ -159,9 +159,9 @@ class BumpPtrAllocatorImpl
 #endif
 
     // Check if we have enough space.
-    if (Adjustment + SizeToAllocate <= size_t(End - CurPtr)
-        // We can't return nullptr even for a zero-sized allocation!
-        && CurPtr != nullptr) {
+    if (LLVM_LIKELY(Adjustment + SizeToAllocate <= size_t(End - CurPtr)
+                    // We can't return nullptr even for a zero-sized allocation!
+                    && CurPtr != nullptr)) {
       char *AlignedPtr = CurPtr + Adjustment;
       CurPtr = AlignedPtr + SizeToAllocate;
       // Update the allocation point of this memory block in MemorySanitizer.
@@ -173,6 +173,11 @@ class BumpPtrAllocatorImpl
       return AlignedPtr;
     }
 
+    return AllocateSlow(SizeToAllocate, Alignment);
+  }
+
+  LLVM_ATTRIBUTE_RETURNS_NONNULL LLVM_ATTRIBUTE_NOINLINE void *
+  AllocateSlow(size_t SizeToAllocate, Align Alignment) {
     // If Size is really big, allocate a separate slab for it.
     size_t PaddedSize = SizeToAllocate + Alignment.value() - 1;
     if (PaddedSize > SizeThreshold) {