[compiler-rt] r190852 - [asan] further speedup use-after-return: simplify deallocation of fake frames. ~ 20% speedup.

[Kostya Serebryany]

Author: kcc
Date: Tue Sep 17 02:42:54 2013
New Revision: 190852

URL: http://llvm.org/viewvc/llvm-project?rev=190852&view=rev
Log:
[asan] further speedup use-after-return: simplify deallocation of fake frames. ~ 20% speedup.

Modified:
    compiler-rt/trunk/lib/asan/asan_fake_stack.cc
    compiler-rt/trunk/lib/asan/asan_fake_stack.h
    compiler-rt/trunk/lib/asan/tests/asan_fake_stack_test.cc

Modified: compiler-rt/trunk/lib/asan/asan_fake_stack.cc
URL: http://llvm.org/viewvc/llvm-project/compiler-rt/trunk/lib/asan/asan_fake_stack.cc?rev=190852&r1=190851&r2=190852&view=diff
==============================================================================
--- compiler-rt/trunk/lib/asan/asan_fake_stack.cc (original)
+++ compiler-rt/trunk/lib/asan/asan_fake_stack.cc Tue Sep 17 02:42:54 2013
@@ -63,25 +63,13 @@ FakeFrame *FakeStack::Allocate(uptr stac
     FakeFrame *res = reinterpret_cast<FakeFrame *>(
         GetFrame(stack_size_log, class_id, pos));
     res->real_stack = real_stack;
+    *SavedFlagPtr(reinterpret_cast<uptr>(res), class_id) = &flags[pos];
     return res;
   }
   CHECK(0 && "Failed to allocate a fake stack frame");
   return 0;
 }
 
-ALWAYS_INLINE USED
-void FakeStack::Deallocate(FakeFrame *ff, uptr stack_size_log, uptr class_id,
-                           uptr real_stack) {
-  u8 *base = GetFrame(stack_size_log, class_id, 0);
-  u8 *cur = reinterpret_cast<u8 *>(ff);
-  CHECK_LE(base, cur);
-  CHECK_LT(cur, base + (1UL << stack_size_log));
-  uptr pos = (cur - base) >> (kMinStackFrameSizeLog + class_id);
-  u8 *flags = GetFlags(stack_size_log, class_id);
-  CHECK_EQ(flags[pos], 1);
-  flags[pos] = 0;
-}
-
 uptr FakeStack::AddrIsInFakeStack(uptr ptr) {
   uptr stack_size_log = this->stack_size_log();
   uptr beg = reinterpret_cast<uptr>(GetFrame(stack_size_log, 0, 0));
@@ -161,9 +149,7 @@ ALWAYS_INLINE uptr OnMalloc(uptr class_i
 ALWAYS_INLINE void OnFree(uptr ptr, uptr class_id, uptr size, uptr real_stack) {
   if (ptr == real_stack)
     return;
-  FakeStack *fs = GetFakeStackFast();  // Must not be 0.
-  FakeFrame *ff = reinterpret_cast<FakeFrame *>(ptr);
-  fs->Deallocate(ff, fs->stack_size_log(), class_id, real_stack);
+  FakeStack::Deallocate(ptr, class_id);
   SetShadow(ptr, size, class_id, kMagic8);
 }
 

Modified: compiler-rt/trunk/lib/asan/asan_fake_stack.h
URL: http://llvm.org/viewvc/llvm-project/compiler-rt/trunk/lib/asan/asan_fake_stack.h?rev=190852&r1=190851&r2=190852&view=diff
==============================================================================
--- compiler-rt/trunk/lib/asan/asan_fake_stack.h (original)
+++ compiler-rt/trunk/lib/asan/asan_fake_stack.h Tue Sep 17 02:42:54 2013
@@ -133,9 +133,10 @@ class FakeStack {
   // Allocate the fake frame.
   FakeFrame *Allocate(uptr stack_size_log, uptr class_id, uptr real_stack);
 
-  // Deallocate the fake frame.
-  void Deallocate(FakeFrame *ff, uptr stack_size_log, uptr class_id,
-                  uptr real_stack);
+  // Deallocate the fake frame: read the saved flag address and write 0 there.
+  static void Deallocate(uptr x, uptr class_id) {
+    **SavedFlagPtr(x, class_id) = 0;
+  }
 
   // Poison the entire FakeStack's shadow with the magic value.
   void PoisonAll(u8 magic);
@@ -148,6 +149,13 @@ class FakeStack {
     return 1UL << (class_id + kMinStackFrameSizeLog);
   }
 
+  // The fake frame is guaranteed to have a right redzone.
+  // We use the last word of that redzone to store the address of the flag
+  // that corresponds to the current frame to make faster deallocation.
+  static u8 **SavedFlagPtr(uptr x, uptr class_id) {
+    return reinterpret_cast<u8 **>(x + BytesInSizeClass(class_id) - sizeof(x));
+  }
+
   uptr stack_size_log() const { return stack_size_log_; }
 
   void HandleNoReturn();

Modified: compiler-rt/trunk/lib/asan/tests/asan_fake_stack_test.cc
URL: http://llvm.org/viewvc/llvm-project/compiler-rt/trunk/lib/asan/tests/asan_fake_stack_test.cc?rev=190852&r1=190851&r2=190852&view=diff
==============================================================================
--- compiler-rt/trunk/lib/asan/tests/asan_fake_stack_test.cc (original)
+++ compiler-rt/trunk/lib/asan/tests/asan_fake_stack_test.cc Tue Sep 17 02:42:54 2013
@@ -128,7 +128,7 @@ TEST(FakeStack, Allocate) {
     }
     for (std::map<FakeFrame *, uptr>::iterator it = s.begin(); it != s.end();
          ++it) {
-      fs->Deallocate(it->first, stack_size_log, it->second, 0);
+      fs->Deallocate(reinterpret_cast<uptr>(it->first), it->second);
     }
   }
   fs->Destroy();
@@ -141,7 +141,7 @@ static void RecursiveFunction(FakeStack
     RecursiveFunction(fs, depth - 1);
     RecursiveFunction(fs, depth - 1);
   }
-  fs->Deallocate(ff, fs->stack_size_log(), class_id, 0);
+  fs->Deallocate(reinterpret_cast<uptr>(ff), class_id);
 }
 
 TEST(FakeStack, RecursiveStressTest) {