[compiler-rt] r301184 - [sanitizer] Cache SizeClassForTransferBatch in the 32-bit local cache

[Kostya Kortchinsky via llvm-commits]

Author: cryptoad
Date: Mon Apr 24 09:53:38 2017
New Revision: 301184

URL: http://llvm.org/viewvc/llvm-project?rev=301184&view=rev
Log:
[sanitizer] Cache SizeClassForTransferBatch in the 32-bit local cache

Summary:
`SizeClassForTransferBatch` is expensive and is called for every `CreateBatch`
and `DestroyBatch`. Caching it means `kNumClasses` calls in `InitCache`
instead. This should be a performance gain if more than `kNumClasses / 2`
batches are created and destroyed during the lifetime of the local cache.

I have chosen to fully remove the function and putting the code in `InitCache`,
which is a debatable choice.

In single threaded benchmarks leveraging primary backed allocations, this turns
out to be a sizeable gain in performances (greater than 5%). In multithreaded
benchmarks leveraging everything, it is less significant but still an
improvement (about 1%).

Reviewers: kcc, dvyukov, alekseyshl

Reviewed By: dvyukov

Subscribers: kubamracek, llvm-commits

Differential Revision: https://reviews.llvm.org/D32365

Modified:
    compiler-rt/trunk/lib/sanitizer_common/sanitizer_allocator_local_cache.h

Modified: compiler-rt/trunk/lib/sanitizer_common/sanitizer_allocator_local_cache.h
URL: http://llvm.org/viewvc/llvm-project/compiler-rt/trunk/lib/sanitizer_common/sanitizer_allocator_local_cache.h?rev=301184&r1=301183&r2=301184&view=diff
==============================================================================
--- compiler-rt/trunk/lib/sanitizer_common/sanitizer_allocator_local_cache.h (original)
+++ compiler-rt/trunk/lib/sanitizer_common/sanitizer_allocator_local_cache.h Mon Apr 24 09:53:38 2017
@@ -180,6 +180,7 @@ struct SizeClassAllocator32LocalCache {
     uptr count;
     uptr max_count;
     uptr class_size;
+    uptr class_id_for_transfer_batch;
     void *batch[2 * TransferBatch::kMaxNumCached];
   };
   PerClass per_class_[kNumClasses];
@@ -188,32 +189,31 @@ struct SizeClassAllocator32LocalCache {
   void InitCache() {
     if (per_class_[1].max_count)
       return;
+    // TransferBatch class is declared in SizeClassAllocator.
+    uptr class_id_for_transfer_batch =
+        SizeClassMap::ClassID(sizeof(TransferBatch));
     for (uptr i = 0; i < kNumClasses; i++) {
       PerClass *c = &per_class_[i];
-      c->max_count = 2 * TransferBatch::MaxCached(i);
+      uptr max_cached = TransferBatch::MaxCached(i);
+      c->max_count = 2 * max_cached;
       c->class_size = Allocator::ClassIdToSize(i);
+      // We transfer chunks between central and thread-local free lists in
+      // batches. For small size classes we allocate batches separately. For
+      // large size classes we may use one of the chunks to store the batch.
+      // sizeof(TransferBatch) must be a power of 2 for more efficient
+      // allocation.
+      c->class_id_for_transfer_batch = (c->class_size <
+          TransferBatch::AllocationSizeRequiredForNElements(max_cached)) ?
+              class_id_for_transfer_batch : 0;
     }
   }
 
-  // TransferBatch class is declared in SizeClassAllocator.
-  // We transfer chunks between central and thread-local free lists in batches.
-  // For small size classes we allocate batches separately.
-  // For large size classes we may use one of the chunks to store the batch.
-  // sizeof(TransferBatch) must be a power of 2 for more efficient allocation.
-  static uptr SizeClassForTransferBatch(uptr class_id) {
-    if (Allocator::ClassIdToSize(class_id) <
-        TransferBatch::AllocationSizeRequiredForNElements(
-            TransferBatch::MaxCached(class_id)))
-      return SizeClassMap::ClassID(sizeof(TransferBatch));
-    return 0;
-  }
-
   // Returns a TransferBatch suitable for class_id.
   // For small size classes allocates the batch from the allocator.
   // For large size classes simply returns b.
   TransferBatch *CreateBatch(uptr class_id, SizeClassAllocator *allocator,
                              TransferBatch *b) {
-    if (uptr batch_class_id = SizeClassForTransferBatch(class_id))
+    if (uptr batch_class_id = per_class_[class_id].class_id_for_transfer_batch)
       return (TransferBatch*)Allocate(allocator, batch_class_id);
     return b;
   }
@@ -223,7 +223,7 @@ struct SizeClassAllocator32LocalCache {
   // Does notthing for large size classes.
   void DestroyBatch(uptr class_id, SizeClassAllocator *allocator,
                     TransferBatch *b) {
-    if (uptr batch_class_id = SizeClassForTransferBatch(class_id))
+    if (uptr batch_class_id = per_class_[class_id].class_id_for_transfer_batch)
       Deallocate(allocator, batch_class_id, b);
   }