[compiler-rt] 03e345b - [asan] Avoid few branches on memcpy hot path

[Vitaly Buka via llvm-commits]

Author: Vitaly Buka
Date: 2022-08-08T19:20:06-07:00
New Revision: 03e345b5198d22ac7a4fd9798010fd1da16a25ca

URL: https://github.com/llvm/llvm-project/commit/03e345b5198d22ac7a4fd9798010fd1da16a25ca
DIFF: https://github.com/llvm/llvm-project/commit/03e345b5198d22ac7a4fd9798010fd1da16a25ca.diff

LOG: [asan] Avoid few branches on memcpy hot path

Added: 
    

Modified: 
    compiler-rt/lib/asan/asan_interceptors_memintrinsics.h
    compiler-rt/lib/asan/asan_internal.h
    compiler-rt/lib/asan/asan_rtl.cpp

Removed: 
    


################################################################################
diff  --git a/compiler-rt/lib/asan/asan_interceptors_memintrinsics.h b/compiler-rt/lib/asan/asan_interceptors_memintrinsics.h
index 178827ab0351..391e72b826d3 100644
--- a/compiler-rt/lib/asan/asan_interceptors_memintrinsics.h
+++ b/compiler-rt/lib/asan/asan_interceptors_memintrinsics.h
@@ -80,18 +80,14 @@ struct AsanInterceptorContext {
 // See http://llvm.org/bugs/show_bug.cgi?id=11763.
 #define ASAN_MEMCPY_IMPL(ctx, to, from, size)                 \
   do {                                                        \
-    if (UNLIKELY(!asan_inited))                               \
-      return internal_memcpy(to, from, size);                 \
-    if (asan_init_is_running) {                               \
-      return REAL(memcpy)(to, from, size);                    \
-    }                                                         \
-    ENSURE_ASAN_INITED();                                     \
-    if (LIKELY(flags()->replace_intrin)) {                    \
-      if (LIKELY(to != from)) {                             \
+    if (LIKELY(replace_intrin_cached)) {                      \
+      if (LIKELY(to != from)) {                               \
         CHECK_RANGES_OVERLAP("memcpy", to, size, from, size); \
       }                                                       \
       ASAN_READ_RANGE(ctx, from, size);                       \
       ASAN_WRITE_RANGE(ctx, to, size);                        \
+    } else if (UNLIKELY(!asan_inited)) {                      \
+      return internal_memcpy(to, from, size);                 \
     }                                                         \
     return REAL(memcpy)(to, from, size);                      \
   } while (0)
@@ -99,24 +95,17 @@ struct AsanInterceptorContext {
 // memset is called inside Printf.
 #define ASAN_MEMSET_IMPL(ctx, block, c, size) \
   do {                                        \
-    if (UNLIKELY(!asan_inited))               \
-      return internal_memset(block, c, size); \
-    if (asan_init_is_running) {               \
-      return REAL(memset)(block, c, size);    \
-    }                                         \
-    ENSURE_ASAN_INITED();                     \
-    if (LIKELY(flags()->replace_intrin)) {    \
+    if (LIKELY(replace_intrin_cached)) {      \
       ASAN_WRITE_RANGE(ctx, block, size);     \
+    } else if (UNLIKELY(!asan_inited)) {      \
+      return internal_memset(block, c, size); \
     }                                         \
     return REAL(memset)(block, c, size);      \
   } while (0)
 
 #define ASAN_MEMMOVE_IMPL(ctx, to, from, size) \
   do {                                         \
-    if (UNLIKELY(!asan_inited))                \
-      return internal_memmove(to, from, size); \
-    ENSURE_ASAN_INITED();                      \
-    if (LIKELY(flags()->replace_intrin)) {     \
+    if (LIKELY(replace_intrin_cached)) {       \
       ASAN_READ_RANGE(ctx, from, size);        \
       ASAN_WRITE_RANGE(ctx, to, size);         \
     }                                          \

diff  --git a/compiler-rt/lib/asan/asan_internal.h b/compiler-rt/lib/asan/asan_internal.h
index 9c46f225116e..959fdec26042 100644
--- a/compiler-rt/lib/asan/asan_internal.h
+++ b/compiler-rt/lib/asan/asan_internal.h
@@ -133,6 +133,7 @@ void InstallAtExitCheckLeaks();
 extern int asan_inited;
 // Used to avoid infinite recursion in __asan_init().
 extern bool asan_init_is_running;
+extern bool replace_intrin_cached;
 extern void (*death_callback)(void);
 // These magic values are written to shadow for better error
 // reporting.

diff  --git a/compiler-rt/lib/asan/asan_rtl.cpp b/compiler-rt/lib/asan/asan_rtl.cpp
index 2bbf0ac5240a..88f66de5669a 100644
--- a/compiler-rt/lib/asan/asan_rtl.cpp
+++ b/compiler-rt/lib/asan/asan_rtl.cpp
@@ -73,6 +73,7 @@ static void CheckUnwind() {
 // -------------------------- Globals --------------------- {{{1
 int asan_inited;
 bool asan_init_is_running;
+bool replace_intrin_cached;
 
 #if !ASAN_FIXED_MAPPING
 uptr kHighMemEnd, kMidMemBeg, kMidMemEnd;
@@ -451,6 +452,7 @@ static void AsanInitInternal() {
 
   // On Linux AsanThread::ThreadStart() calls malloc() that's why asan_inited
   // should be set to 1 prior to initializing the threads.
+  replace_intrin_cached = flags()->replace_intrin;
   asan_inited = 1;
   asan_init_is_running = false;