[compiler-rt] r178239 - [ASan] Speed-up initialization-order checking: create and use fast versions of PoisonShadow functions, store copies of __asan_global descriptors in a vector instead of list of pointers. This gives 3x speedup on both benchmarks and real binaries with lots of globals.
Alexey Samsonov
samsonov at google.com
Thu Mar 28 08:42:43 PDT 2013
Author: samsonov
Date: Thu Mar 28 10:42:43 2013
New Revision: 178239
URL: http://llvm.org/viewvc/llvm-project?rev=178239&view=rev
Log:
[ASan] Speed-up initialization-order checking: create and use fast versions of PoisonShadow functions, store copies of __asan_global descriptors in a vector instead of list of pointers. This gives 3x speedup on both benchmarks and real binaries with lots of globals.
Added:
compiler-rt/trunk/lib/asan/asan_poisoning.h
Modified:
compiler-rt/trunk/lib/asan/asan_allocator2.cc
compiler-rt/trunk/lib/asan/asan_fake_stack.cc
compiler-rt/trunk/lib/asan/asan_globals.cc
compiler-rt/trunk/lib/asan/asan_interceptors.cc
compiler-rt/trunk/lib/asan/asan_internal.h
compiler-rt/trunk/lib/asan/asan_poisoning.cc
compiler-rt/trunk/lib/asan/asan_rtl.cc
compiler-rt/trunk/lib/asan/asan_thread.cc
Modified: compiler-rt/trunk/lib/asan/asan_allocator2.cc
URL: http://llvm.org/viewvc/llvm-project/compiler-rt/trunk/lib/asan/asan_allocator2.cc?rev=178239&r1=178238&r2=178239&view=diff
==============================================================================
--- compiler-rt/trunk/lib/asan/asan_allocator2.cc (original)
+++ compiler-rt/trunk/lib/asan/asan_allocator2.cc Thu Mar 28 10:42:43 2013
@@ -19,6 +19,7 @@
#if ASAN_ALLOCATOR_VERSION == 2
#include "asan_mapping.h"
+#include "asan_poisoning.h"
#include "asan_report.h"
#include "asan_thread.h"
#include "sanitizer_common/sanitizer_allocator.h"
Modified: compiler-rt/trunk/lib/asan/asan_fake_stack.cc
URL: http://llvm.org/viewvc/llvm-project/compiler-rt/trunk/lib/asan/asan_fake_stack.cc?rev=178239&r1=178238&r2=178239&view=diff
==============================================================================
--- compiler-rt/trunk/lib/asan/asan_fake_stack.cc (original)
+++ compiler-rt/trunk/lib/asan/asan_fake_stack.cc Thu Mar 28 10:42:43 2013
@@ -12,6 +12,7 @@
// FakeStack is used to detect use-after-return bugs.
//===----------------------------------------------------------------------===//
#include "asan_allocator.h"
+#include "asan_poisoning.h"
#include "asan_thread.h"
namespace __asan {
Modified: compiler-rt/trunk/lib/asan/asan_globals.cc
URL: http://llvm.org/viewvc/llvm-project/compiler-rt/trunk/lib/asan/asan_globals.cc?rev=178239&r1=178238&r2=178239&view=diff
==============================================================================
--- compiler-rt/trunk/lib/asan/asan_globals.cc (original)
+++ compiler-rt/trunk/lib/asan/asan_globals.cc Thu Mar 28 10:42:43 2013
@@ -14,11 +14,14 @@
#include "asan_interceptors.h"
#include "asan_internal.h"
#include "asan_mapping.h"
+#include "asan_poisoning.h"
#include "asan_report.h"
#include "asan_stack.h"
#include "asan_stats.h"
#include "asan_thread.h"
+#include "sanitizer_common/sanitizer_common.h"
#include "sanitizer_common/sanitizer_mutex.h"
+#include "sanitizer_common/sanitizer_placement_new.h"
namespace __asan {
@@ -32,15 +35,22 @@ struct ListOfGlobals {
static BlockingMutex mu_for_globals(LINKER_INITIALIZED);
static LowLevelAllocator allocator_for_globals;
static ListOfGlobals *list_of_all_globals;
-static ListOfGlobals *list_of_dynamic_init_globals;
-static void PoisonRedZones(const Global &g) {
+static const int kDynamicInitGlobalsInitialCapacity = 512;
+typedef InternalVector<Global> VectorOfGlobals;
+// Lazy-initialized and never deleted.
+static VectorOfGlobals *dynamic_init_globals;
+
+ALWAYS_INLINE INLINE void PoisonShadowForGlobal(const Global *g, u8 value) {
+ FastPoisonShadow(g->beg, g->size_with_redzone, value);
+}
+
+ALWAYS_INLINE INLINE void PoisonRedZones(const Global &g) {
uptr aligned_size = RoundUpTo(g.size, SHADOW_GRANULARITY);
- PoisonShadow(g.beg + aligned_size, g.size_with_redzone - aligned_size,
- kAsanGlobalRedzoneMagic);
+ FastPoisonShadow(g.beg + aligned_size, g.size_with_redzone - aligned_size,
+ kAsanGlobalRedzoneMagic);
if (g.size != aligned_size) {
- // partial right redzone
- PoisonShadowPartialRightRedzone(
+ FastPoisonShadowPartialRightRedzone(
g.beg + RoundDownTo(g.size, SHADOW_GRANULARITY),
g.size % SHADOW_GRANULARITY,
SHADOW_GRANULARITY,
@@ -78,17 +88,20 @@ static void RegisterGlobal(const Global
CHECK(AddrIsInMem(g->beg));
CHECK(AddrIsAlignedByGranularity(g->beg));
CHECK(AddrIsAlignedByGranularity(g->size_with_redzone));
- PoisonRedZones(*g);
+ if (flags()->poison_heap)
+ PoisonRedZones(*g);
ListOfGlobals *l =
(ListOfGlobals*)allocator_for_globals.Allocate(sizeof(ListOfGlobals));
l->g = g;
l->next = list_of_all_globals;
list_of_all_globals = l;
if (g->has_dynamic_init) {
- l = (ListOfGlobals*)allocator_for_globals.Allocate(sizeof(ListOfGlobals));
- l->g = g;
- l->next = list_of_dynamic_init_globals;
- list_of_dynamic_init_globals = l;
+ if (dynamic_init_globals == 0) {
+ void *mem = allocator_for_globals.Allocate(sizeof(VectorOfGlobals));
+ dynamic_init_globals = new(mem)
+ VectorOfGlobals(kDynamicInitGlobalsInitialCapacity);
+ }
+ dynamic_init_globals->push_back(*g);
}
}
@@ -98,36 +111,13 @@ static void UnregisterGlobal(const Globa
CHECK(AddrIsInMem(g->beg));
CHECK(AddrIsAlignedByGranularity(g->beg));
CHECK(AddrIsAlignedByGranularity(g->size_with_redzone));
- PoisonShadow(g->beg, g->size_with_redzone, 0);
+ if (flags()->poison_heap)
+ PoisonShadowForGlobal(g, 0);
// We unpoison the shadow memory for the global but we do not remove it from
// the list because that would require O(n^2) time with the current list
// implementation. It might not be worth doing anyway.
}
-// Poison all shadow memory for a single global.
-static void PoisonGlobalAndRedzones(const Global *g) {
- CHECK(asan_inited);
- CHECK(flags()->check_initialization_order);
- CHECK(AddrIsInMem(g->beg));
- CHECK(AddrIsAlignedByGranularity(g->beg));
- CHECK(AddrIsAlignedByGranularity(g->size_with_redzone));
- if (flags()->report_globals >= 3)
- Printf("DynInitPoison : %s\n", g->name);
- PoisonShadow(g->beg, g->size_with_redzone, kAsanInitializationOrderMagic);
-}
-
-static void UnpoisonGlobal(const Global *g) {
- CHECK(asan_inited);
- CHECK(flags()->check_initialization_order);
- CHECK(AddrIsInMem(g->beg));
- CHECK(AddrIsAlignedByGranularity(g->beg));
- CHECK(AddrIsAlignedByGranularity(g->size_with_redzone));
- if (flags()->report_globals >= 3)
- Printf("DynInitUnpoison: %s\n", g->name);
- PoisonShadow(g->beg, g->size_with_redzone, 0);
- PoisonRedZones(*g);
-}
-
} // namespace __asan
// ---------------------- Interface ---------------- {{{1
@@ -157,13 +147,19 @@ void __asan_unregister_globals(__asan_gl
// poisons all global variables not defined in this TU, so that a dynamic
// initializer can only touch global variables in the same TU.
void __asan_before_dynamic_init(const char *module_name) {
- if (!flags()->check_initialization_order) return;
- CHECK(list_of_dynamic_init_globals);
+ if (!flags()->check_initialization_order ||
+ !flags()->poison_heap)
+ return;
+ CHECK(dynamic_init_globals);
CHECK(module_name);
+ CHECK(asan_inited);
BlockingMutexLock lock(&mu_for_globals);
- for (ListOfGlobals *l = list_of_dynamic_init_globals; l; l = l->next) {
- if (l->g->module_name != module_name)
- PoisonGlobalAndRedzones(l->g);
+ if (flags()->report_globals >= 3)
+ Printf("DynInitPoison module: %s\n", module_name);
+ for (uptr i = 0, n = dynamic_init_globals->size(); i < n; ++i) {
+ const Global *g = &(*dynamic_init_globals)[i];
+ if (g->module_name != module_name)
+ PoisonShadowForGlobal(g, kAsanInitializationOrderMagic);
}
}
@@ -171,8 +167,17 @@ void __asan_before_dynamic_init(const ch
// all dynamically initialized globals except for those defined in the current
// TU are poisoned. It simply unpoisons all dynamically initialized globals.
void __asan_after_dynamic_init() {
- if (!flags()->check_initialization_order) return;
+ if (!flags()->check_initialization_order ||
+ !flags()->poison_heap)
+ return;
+ CHECK(asan_inited);
BlockingMutexLock lock(&mu_for_globals);
- for (ListOfGlobals *l = list_of_dynamic_init_globals; l; l = l->next)
- UnpoisonGlobal(l->g);
+ // FIXME: Optionally report that we're unpoisoning globals from a module.
+ for (uptr i = 0, n = dynamic_init_globals->size(); i < n; ++i) {
+ const Global *g = &(*dynamic_init_globals)[i];
+ // Unpoison the whole global.
+ PoisonShadowForGlobal(g, 0);
+ // Poison redzones back.
+ PoisonRedZones(*g);
+ }
}
Modified: compiler-rt/trunk/lib/asan/asan_interceptors.cc
URL: http://llvm.org/viewvc/llvm-project/compiler-rt/trunk/lib/asan/asan_interceptors.cc?rev=178239&r1=178238&r2=178239&view=diff
==============================================================================
--- compiler-rt/trunk/lib/asan/asan_interceptors.cc (original)
+++ compiler-rt/trunk/lib/asan/asan_interceptors.cc Thu Mar 28 10:42:43 2013
@@ -17,6 +17,7 @@
#include "asan_intercepted_functions.h"
#include "asan_internal.h"
#include "asan_mapping.h"
+#include "asan_poisoning.h"
#include "asan_report.h"
#include "asan_stack.h"
#include "asan_stats.h"
Modified: compiler-rt/trunk/lib/asan/asan_internal.h
URL: http://llvm.org/viewvc/llvm-project/compiler-rt/trunk/lib/asan/asan_internal.h?rev=178239&r1=178238&r2=178239&view=diff
==============================================================================
--- compiler-rt/trunk/lib/asan/asan_internal.h (original)
+++ compiler-rt/trunk/lib/asan/asan_internal.h Thu Mar 28 10:42:43 2013
@@ -100,16 +100,6 @@ void AsanTSDSet(void *tsd);
void AppendToErrorMessageBuffer(const char *buffer);
-// asan_poisoning.cc
-// Poisons the shadow memory for "size" bytes starting from "addr".
-void PoisonShadow(uptr addr, uptr size, u8 value);
-// Poisons the shadow memory for "redzone_size" bytes starting from
-// "addr + size".
-void PoisonShadowPartialRightRedzone(uptr addr,
- uptr size,
- uptr redzone_size,
- u8 value);
-
// Platfrom-specific options.
#if SANITIZER_MAC
bool PlatformHasDifferentMemcpyAndMemmove();
Modified: compiler-rt/trunk/lib/asan/asan_poisoning.cc
URL: http://llvm.org/viewvc/llvm-project/compiler-rt/trunk/lib/asan/asan_poisoning.cc?rev=178239&r1=178238&r2=178239&view=diff
==============================================================================
--- compiler-rt/trunk/lib/asan/asan_poisoning.cc (original)
+++ compiler-rt/trunk/lib/asan/asan_poisoning.cc Thu Mar 28 10:42:43 2013
@@ -12,9 +12,7 @@
// Shadow memory poisoning by ASan RTL and by user application.
//===----------------------------------------------------------------------===//
-#include "asan_interceptors.h"
-#include "asan_internal.h"
-#include "asan_mapping.h"
+#include "asan_poisoning.h"
#include "sanitizer_common/sanitizer_libc.h"
namespace __asan {
@@ -22,11 +20,11 @@ namespace __asan {
void PoisonShadow(uptr addr, uptr size, u8 value) {
if (!flags()->poison_heap) return;
CHECK(AddrIsAlignedByGranularity(addr));
+ CHECK(AddrIsInMem(addr));
CHECK(AddrIsAlignedByGranularity(addr + size));
- uptr shadow_beg = MemToShadow(addr);
- uptr shadow_end = MemToShadow(addr + size - SHADOW_GRANULARITY) + 1;
+ CHECK(AddrIsInMem(addr + size - SHADOW_GRANULARITY));
CHECK(REAL(memset) != 0);
- REAL(memset)((void*)shadow_beg, value, shadow_end - shadow_beg);
+ FastPoisonShadow(addr, size, value);
}
void PoisonShadowPartialRightRedzone(uptr addr,
@@ -35,20 +33,10 @@ void PoisonShadowPartialRightRedzone(upt
u8 value) {
if (!flags()->poison_heap) return;
CHECK(AddrIsAlignedByGranularity(addr));
- u8 *shadow = (u8*)MemToShadow(addr);
- for (uptr i = 0; i < redzone_size;
- i += SHADOW_GRANULARITY, shadow++) {
- if (i + SHADOW_GRANULARITY <= size) {
- *shadow = 0; // fully addressable
- } else if (i >= size) {
- *shadow = (SHADOW_GRANULARITY == 128) ? 0xff : value; // unaddressable
- } else {
- *shadow = size - i; // first size-i bytes are addressable
- }
- }
+ CHECK(AddrIsInMem(addr));
+ FastPoisonShadowPartialRightRedzone(addr, size, redzone_size, value);
}
-
struct ShadowSegmentEndpoint {
u8 *chunk;
s8 offset; // in [0, SHADOW_GRANULARITY)
Added: compiler-rt/trunk/lib/asan/asan_poisoning.h
URL: http://llvm.org/viewvc/llvm-project/compiler-rt/trunk/lib/asan/asan_poisoning.h?rev=178239&view=auto
==============================================================================
--- compiler-rt/trunk/lib/asan/asan_poisoning.h (added)
+++ compiler-rt/trunk/lib/asan/asan_poisoning.h Thu Mar 28 10:42:43 2013
@@ -0,0 +1,58 @@
+//===-- asan_poisoning.h ----------------------------------------*- C++ -*-===//
+//
+// The LLVM Compiler Infrastructure
+//
+// This file is distributed under the University of Illinois Open Source
+// License. See LICENSE.TXT for details.
+//
+//===----------------------------------------------------------------------===//
+//
+// This file is a part of AddressSanitizer, an address sanity checker.
+//
+// Shadow memory poisoning by ASan RTL and by user application.
+//===----------------------------------------------------------------------===//
+
+#include "asan_interceptors.h"
+#include "asan_internal.h"
+#include "asan_mapping.h"
+
+namespace __asan {
+
+// Poisons the shadow memory for "size" bytes starting from "addr".
+void PoisonShadow(uptr addr, uptr size, u8 value);
+
+// Poisons the shadow memory for "redzone_size" bytes starting from
+// "addr + size".
+void PoisonShadowPartialRightRedzone(uptr addr,
+ uptr size,
+ uptr redzone_size,
+ u8 value);
+
+// Fast versions of PoisonShadow and PoisonShadowPartialRightRedzone that
+// assume that memory addresses are properly aligned. Use in
+// performance-critical code with care.
+ALWAYS_INLINE INLINE void FastPoisonShadow(uptr aligned_beg, uptr aligned_size,
+ u8 value) {
+ DCHECK(flags()->poison_heap);
+ uptr shadow_beg = MEM_TO_SHADOW(aligned_beg);
+ uptr shadow_end = MEM_TO_SHADOW(
+ aligned_beg + aligned_size - SHADOW_GRANULARITY) + 1;
+ REAL(memset)((void*)shadow_beg, value, shadow_end - shadow_beg);
+}
+
+ALWAYS_INLINE INLINE void FastPoisonShadowPartialRightRedzone(
+ uptr aligned_addr, uptr size, uptr redzone_size, u8 value) {
+ DCHECK(flags()->poison_heap);
+ u8 *shadow = (u8*)MEM_TO_SHADOW(aligned_addr);
+ for (uptr i = 0; i < redzone_size; i += SHADOW_GRANULARITY, shadow++) {
+ if (i + SHADOW_GRANULARITY <= size) {
+ *shadow = 0; // fully addressable
+ } else if (i >= size) {
+ *shadow = (SHADOW_GRANULARITY == 128) ? 0xff : value; // unaddressable
+ } else {
+ *shadow = size - i; // first size-i bytes are addressable
+ }
+ }
+}
+
+} // namespace __asan
Modified: compiler-rt/trunk/lib/asan/asan_rtl.cc
URL: http://llvm.org/viewvc/llvm-project/compiler-rt/trunk/lib/asan/asan_rtl.cc?rev=178239&r1=178238&r2=178239&view=diff
==============================================================================
--- compiler-rt/trunk/lib/asan/asan_rtl.cc (original)
+++ compiler-rt/trunk/lib/asan/asan_rtl.cc Thu Mar 28 10:42:43 2013
@@ -15,6 +15,7 @@
#include "asan_interceptors.h"
#include "asan_internal.h"
#include "asan_mapping.h"
+#include "asan_poisoning.h"
#include "asan_report.h"
#include "asan_stack.h"
#include "asan_stats.h"
Modified: compiler-rt/trunk/lib/asan/asan_thread.cc
URL: http://llvm.org/viewvc/llvm-project/compiler-rt/trunk/lib/asan/asan_thread.cc?rev=178239&r1=178238&r2=178239&view=diff
==============================================================================
--- compiler-rt/trunk/lib/asan/asan_thread.cc (original)
+++ compiler-rt/trunk/lib/asan/asan_thread.cc Thu Mar 28 10:42:43 2013
@@ -13,6 +13,7 @@
//===----------------------------------------------------------------------===//
#include "asan_allocator.h"
#include "asan_interceptors.h"
+#include "asan_poisoning.h"
#include "asan_stack.h"
#include "asan_thread.h"
#include "asan_mapping.h"
More information about the llvm-commits
mailing list