[compiler-rt] r300216 - Cache size per class size in SizeClassAllocatorXLocalCache.

[Alex Shlyapnikov via llvm-commits]

Author: alekseyshl
Date: Thu Apr 13 11:49:16 2017
New Revision: 300216

URL: http://llvm.org/viewvc/llvm-project?rev=300216&view=rev
Log:
Cache size per class size in SizeClassAllocatorXLocalCache.

Summary:
Allocator::ClassIdToSize() is not free and calling it in every
Allocate/Deallocate has noticeable impact on perf.
Reapplying D31991 with the appropriate fixes.

Reviewers: cryptoad

Subscribers: kubamracek, llvm-commits

Differential Revision: https://reviews.llvm.org/D32024

Modified:
    compiler-rt/trunk/lib/sanitizer_common/sanitizer_allocator_local_cache.h

Modified: compiler-rt/trunk/lib/sanitizer_common/sanitizer_allocator_local_cache.h
URL: http://llvm.org/viewvc/llvm-project/compiler-rt/trunk/lib/sanitizer_common/sanitizer_allocator_local_cache.h?rev=300216&r1=300215&r2=300216&view=diff
==============================================================================
--- compiler-rt/trunk/lib/sanitizer_common/sanitizer_allocator_local_cache.h (original)
+++ compiler-rt/trunk/lib/sanitizer_common/sanitizer_allocator_local_cache.h Thu Apr 13 11:49:16 2017
@@ -45,10 +45,10 @@ struct SizeClassAllocator64LocalCache {
   void *Allocate(SizeClassAllocator *allocator, uptr class_id) {
     CHECK_NE(class_id, 0UL);
     CHECK_LT(class_id, kNumClasses);
-    stats_.Add(AllocatorStatAllocated, Allocator::ClassIdToSize(class_id));
     PerClass *c = &per_class_[class_id];
     if (UNLIKELY(c->count == 0))
       Refill(c, allocator, class_id);
+    stats_.Add(AllocatorStatAllocated, c->class_size);
     CHECK_GT(c->count, 0);
     CompactPtrT chunk = c->chunks[--c->count];
     void *res = reinterpret_cast<void *>(allocator->CompactPtrToPointer(
@@ -62,8 +62,8 @@ struct SizeClassAllocator64LocalCache {
     // If the first allocator call on a new thread is a deallocation, then
     // max_count will be zero, leading to check failure.
     InitCache();
-    stats_.Sub(AllocatorStatAllocated, Allocator::ClassIdToSize(class_id));
     PerClass *c = &per_class_[class_id];
+    stats_.Sub(AllocatorStatAllocated, c->class_size);
     CHECK_NE(c->max_count, 0UL);
     if (UNLIKELY(c->count == c->max_count))
       Drain(c, allocator, class_id, c->max_count / 2);
@@ -85,6 +85,7 @@ struct SizeClassAllocator64LocalCache {
   struct PerClass {
     u32 count;
     u32 max_count;
+    uptr class_size;
     CompactPtrT chunks[2 * SizeClassMap::kMaxNumCachedHint];
   };
   PerClass per_class_[kNumClasses];
@@ -96,6 +97,7 @@ struct SizeClassAllocator64LocalCache {
     for (uptr i = 0; i < kNumClasses; i++) {
       PerClass *c = &per_class_[i];
       c->max_count = 2 * SizeClassMap::MaxCachedHint(i);
+      c->class_size = Allocator::ClassIdToSize(i);
     }
   }
 
@@ -141,10 +143,10 @@ struct SizeClassAllocator32LocalCache {
   void *Allocate(SizeClassAllocator *allocator, uptr class_id) {
     CHECK_NE(class_id, 0UL);
     CHECK_LT(class_id, kNumClasses);
-    stats_.Add(AllocatorStatAllocated, Allocator::ClassIdToSize(class_id));
     PerClass *c = &per_class_[class_id];
     if (UNLIKELY(c->count == 0))
       Refill(allocator, class_id);
+    stats_.Add(AllocatorStatAllocated, c->class_size);
     void *res = c->batch[--c->count];
     PREFETCH(c->batch[c->count - 1]);
     return res;
@@ -156,8 +158,8 @@ struct SizeClassAllocator32LocalCache {
     // If the first allocator call on a new thread is a deallocation, then
     // max_count will be zero, leading to check failure.
     InitCache();
-    stats_.Sub(AllocatorStatAllocated, Allocator::ClassIdToSize(class_id));
     PerClass *c = &per_class_[class_id];
+    stats_.Sub(AllocatorStatAllocated, c->class_size);
     CHECK_NE(c->max_count, 0UL);
     if (UNLIKELY(c->count == c->max_count))
       Drain(allocator, class_id);
@@ -177,6 +179,7 @@ struct SizeClassAllocator32LocalCache {
   struct PerClass {
     uptr count;
     uptr max_count;
+    uptr class_size;
     void *batch[2 * TransferBatch::kMaxNumCached];
   };
   PerClass per_class_[kNumClasses];
@@ -188,6 +191,7 @@ struct SizeClassAllocator32LocalCache {
     for (uptr i = 0; i < kNumClasses; i++) {
       PerClass *c = &per_class_[i];
       c->max_count = 2 * TransferBatch::MaxCached(i);
+      c->class_size = Allocator::ClassIdToSize(i);
     }
   }