[compiler-rt] r195549 - [sanitizer] use 16-byte aligned bzero in performance critical place (mostly for lsan)

[Kostya Serebryany]

Author: kcc
Date: Sat Nov 23 06:49:19 2013
New Revision: 195549

URL: http://llvm.org/viewvc/llvm-project?rev=195549&view=rev
Log:
[sanitizer] use 16-byte aligned bzero in performance critical place (mostly for lsan)

Modified:
    compiler-rt/trunk/lib/sanitizer_common/sanitizer_allocator.h
    compiler-rt/trunk/lib/sanitizer_common/sanitizer_libc.cc
    compiler-rt/trunk/lib/sanitizer_common/sanitizer_libc.h

Modified: compiler-rt/trunk/lib/sanitizer_common/sanitizer_allocator.h
URL: http://llvm.org/viewvc/llvm-project/compiler-rt/trunk/lib/sanitizer_common/sanitizer_allocator.h?rev=195549&r1=195548&r2=195549&view=diff
==============================================================================
--- compiler-rt/trunk/lib/sanitizer_common/sanitizer_allocator.h (original)
+++ compiler-rt/trunk/lib/sanitizer_common/sanitizer_allocator.h Sat Nov 23 06:49:19 2013
@@ -1191,7 +1191,7 @@ class CombinedAllocator {
     if (alignment > 8)
       CHECK_EQ(reinterpret_cast<uptr>(res) & (alignment - 1), 0);
     if (cleared && res)
-      internal_memset(res, 0, size);
+      internal_bzero_aligned16(res, RoundUpTo(size, 16));
     return res;
   }
 

Modified: compiler-rt/trunk/lib/sanitizer_common/sanitizer_libc.cc
URL: http://llvm.org/viewvc/llvm-project/compiler-rt/trunk/lib/sanitizer_common/sanitizer_libc.cc?rev=195549&r1=195548&r2=195549&view=diff
==============================================================================
--- compiler-rt/trunk/lib/sanitizer_common/sanitizer_libc.cc (original)
+++ compiler-rt/trunk/lib/sanitizer_common/sanitizer_libc.cc Sat Nov 23 06:49:19 2013
@@ -16,6 +16,11 @@
 
 namespace __sanitizer {
 
+// Make the compiler think that something is going on there.
+static inline void break_optimization(void *arg) {
+  __asm__ __volatile__("" : : "r" (arg) : "memory");
+}
+
 s64 internal_atoll(const char *nptr) {
   return internal_simple_strtoll(nptr, (char**)0, 10);
 }
@@ -62,6 +67,16 @@ void *internal_memmove(void *dest, const
   return dest;
 }
 
+// Semi-fast bzero for 16-aligned data. Still far from peak performance.
+void internal_bzero_aligned16(void *s, uptr n) {
+  struct S16 { u64 a, b; } ALIGNED(16);
+  CHECK_EQ((reinterpret_cast<uptr>(s) | n) & 15, 0);
+  for (S16 *p = reinterpret_cast<S16*>(s), *end = p + n / 16; p < end; p++) {
+    p->a = p->b = 0;
+    break_optimization(0);  // Make sure this does not become memset.
+  }
+}
+
 void *internal_memset(void* s, int c, uptr n) {
   // The next line prevents Clang from making a call to memset() instead of the
   // loop below.

Modified: compiler-rt/trunk/lib/sanitizer_common/sanitizer_libc.h
URL: http://llvm.org/viewvc/llvm-project/compiler-rt/trunk/lib/sanitizer_common/sanitizer_libc.h?rev=195549&r1=195548&r2=195549&view=diff
==============================================================================
--- compiler-rt/trunk/lib/sanitizer_common/sanitizer_libc.h (original)
+++ compiler-rt/trunk/lib/sanitizer_common/sanitizer_libc.h Sat Nov 23 06:49:19 2013
@@ -29,6 +29,8 @@ void *internal_memchr(const void *s, int
 int internal_memcmp(const void* s1, const void* s2, uptr n);
 void *internal_memcpy(void *dest, const void *src, uptr n);
 void *internal_memmove(void *dest, const void *src, uptr n);
+// Set [s, s + n) to 0. Both s and n should be 16-aligned.
+void internal_bzero_aligned16(void *s, uptr n);
 // Should not be used in performance-critical places.
 void *internal_memset(void *s, int c, uptr n);
 char* internal_strchr(const char *s, int c);