From 4ba77b58fd117d808570d5bd57471bba154e2778 Mon Sep 17 00:00:00 2001
From: Kostya Kortchinsky <kostyak@google.com>
Date: Mon, 24 Apr 2017 14:53:38 +0000
Subject: [sanitizer] Cache SizeClassForTransferBatch in the 32-bit local cache

Summary:
`SizeClassForTransferBatch` is expensive and is called for every `CreateBatch`
and `DestroyBatch`. Caching it means `kNumClasses` calls in `InitCache`
instead. This should be a performance gain if more than `kNumClasses / 2`
batches are created and destroyed during the lifetime of the local cache.

I have chosen to fully remove the function and putting the code in `InitCache`,
which is a debatable choice.

In single threaded benchmarks leveraging primary backed allocations, this turns
out to be a sizeable gain in performances (greater than 5%). In multithreaded
benchmarks leveraging everything, it is less significant but still an
improvement (about 1%).

Reviewers: kcc, dvyukov, alekseyshl

Reviewed By: dvyukov

Subscribers: kubamracek, llvm-commits

Differential Revision: https://reviews.llvm.org/D32365

git-svn-id: https://llvm.org/svn/llvm-project/compiler-rt/trunk@301184 91177308-0d34-0410-b5e6-96231b3b80d8
---
 .../sanitizer_allocator_local_cache.h              | 32 +++++++++++-----------
 1 file changed, 16 insertions(+), 16 deletions(-)

(limited to 'lib')

diff --git a/lib/sanitizer_common/sanitizer_allocator_local_cache.h b/lib/sanitizer_common/sanitizer_allocator_local_cache.h
index d6c66604e..b3729bf55 100644
--- a/lib/sanitizer_common/sanitizer_allocator_local_cache.h
+++ b/lib/sanitizer_common/sanitizer_allocator_local_cache.h
@@ -180,6 +180,7 @@ struct SizeClassAllocator32LocalCache {
     uptr count;
     uptr max_count;
     uptr class_size;
+    uptr class_id_for_transfer_batch;
     void *batch[2 * TransferBatch::kMaxNumCached];
   };
   PerClass per_class_[kNumClasses];
@@ -188,32 +189,31 @@ struct SizeClassAllocator32LocalCache {
   void InitCache() {
     if (per_class_[1].max_count)
       return;
+    // TransferBatch class is declared in SizeClassAllocator.
+    uptr class_id_for_transfer_batch =
+        SizeClassMap::ClassID(sizeof(TransferBatch));
     for (uptr i = 0; i < kNumClasses; i++) {
       PerClass *c = &per_class_[i];
-      c->max_count = 2 * TransferBatch::MaxCached(i);
+      uptr max_cached = TransferBatch::MaxCached(i);
+      c->max_count = 2 * max_cached;
       c->class_size = Allocator::ClassIdToSize(i);
+      // We transfer chunks between central and thread-local free lists in
+      // batches. For small size classes we allocate batches separately. For
+      // large size classes we may use one of the chunks to store the batch.
+      // sizeof(TransferBatch) must be a power of 2 for more efficient
+      // allocation.
+      c->class_id_for_transfer_batch = (c->class_size <
+          TransferBatch::AllocationSizeRequiredForNElements(max_cached)) ?
+              class_id_for_transfer_batch : 0;
     }
   }
 
-  // TransferBatch class is declared in SizeClassAllocator.
-  // We transfer chunks between central and thread-local free lists in batches.
-  // For small size classes we allocate batches separately.
-  // For large size classes we may use one of the chunks to store the batch.
-  // sizeof(TransferBatch) must be a power of 2 for more efficient allocation.
-  static uptr SizeClassForTransferBatch(uptr class_id) {
-    if (Allocator::ClassIdToSize(class_id) <
-        TransferBatch::AllocationSizeRequiredForNElements(
-            TransferBatch::MaxCached(class_id)))
-      return SizeClassMap::ClassID(sizeof(TransferBatch));
-    return 0;
-  }
-
   // Returns a TransferBatch suitable for class_id.
   // For small size classes allocates the batch from the allocator.
   // For large size classes simply returns b.
   TransferBatch *CreateBatch(uptr class_id, SizeClassAllocator *allocator,
                              TransferBatch *b) {
-    if (uptr batch_class_id = SizeClassForTransferBatch(class_id))
+    if (uptr batch_class_id = per_class_[class_id].class_id_for_transfer_batch)
       return (TransferBatch*)Allocate(allocator, batch_class_id);
     return b;
   }
@@ -223,7 +223,7 @@ struct SizeClassAllocator32LocalCache {
   // Does notthing for large size classes.
   void DestroyBatch(uptr class_id, SizeClassAllocator *allocator,
                     TransferBatch *b) {
-    if (uptr batch_class_id = SizeClassForTransferBatch(class_id))
+    if (uptr batch_class_id = per_class_[class_id].class_id_for_transfer_batch)
       Deallocate(allocator, batch_class_id, b);
   }
 
-- 
cgit v1.2.3