[compiler-rt] r207630 - [asan] make AsanCoverage lock-free (and AS-safe)

[Kostya Serebryany]

Author: kcc
Date: Wed Apr 30 05:40:48 2014
New Revision: 207630

URL: http://llvm.org/viewvc/llvm-project?rev=207630&view=rev
Log:
[asan] make AsanCoverage lock-free (and AS-safe)

Modified:
    compiler-rt/trunk/include/sanitizer/common_interface_defs.h
    compiler-rt/trunk/lib/asan/asan_rtl.cc
    compiler-rt/trunk/lib/sanitizer_common/sanitizer_coverage.cc
    compiler-rt/trunk/lib/sanitizer_common/sanitizer_internal_defs.h

Modified: compiler-rt/trunk/include/sanitizer/common_interface_defs.h
URL: http://llvm.org/viewvc/llvm-project/compiler-rt/trunk/include/sanitizer/common_interface_defs.h?rev=207630&r1=207629&r2=207630&view=diff
==============================================================================
--- compiler-rt/trunk/include/sanitizer/common_interface_defs.h (original)
+++ compiler-rt/trunk/include/sanitizer/common_interface_defs.h Wed Apr 30 05:40:48 2014
@@ -47,6 +47,8 @@ extern "C" {
   void __sanitizer_unaligned_store32(void *p, uint32_t x);
   void __sanitizer_unaligned_store64(void *p, uint64_t x);
 
+  // Initialize coverage.
+  void __sanitizer_cov_init();
   // Record and dump coverage info.
   void __sanitizer_cov_dump();
 

Modified: compiler-rt/trunk/lib/asan/asan_rtl.cc
URL: http://llvm.org/viewvc/llvm-project/compiler-rt/trunk/lib/asan/asan_rtl.cc?rev=207630&r1=207629&r2=207630&view=diff
==============================================================================
--- compiler-rt/trunk/lib/asan/asan_rtl.cc (original)
+++ compiler-rt/trunk/lib/asan/asan_rtl.cc Wed Apr 30 05:40:48 2014
@@ -664,8 +664,10 @@ static void AsanInitInternal() {
   if (flags()->atexit)
     Atexit(asan_atexit);
 
-  if (flags()->coverage)
+  if (flags()->coverage) {
+    __sanitizer_cov_init();
     Atexit(__sanitizer_cov_dump);
+  }
 
   // interceptors
   InitTlsSize();

Modified: compiler-rt/trunk/lib/sanitizer_common/sanitizer_coverage.cc
URL: http://llvm.org/viewvc/llvm-project/compiler-rt/trunk/lib/sanitizer_common/sanitizer_coverage.cc?rev=207630&r1=207629&r2=207630&view=diff
==============================================================================
--- compiler-rt/trunk/lib/sanitizer_common/sanitizer_coverage.cc (original)
+++ compiler-rt/trunk/lib/sanitizer_common/sanitizer_coverage.cc Wed Apr 30 05:40:48 2014
@@ -40,23 +40,31 @@
 #include "sanitizer_stacktrace.h"
 #include "sanitizer_flags.h"
 
-struct CovData {
-  BlockingMutex mu;
-  InternalMmapVector<uptr> v;
-  atomic_uint32_t guard;
-};
-
-static uptr cov_data_placeholder[(sizeof(CovData) / sizeof(uptr)) + 1];
-COMPILER_CHECK(sizeof(cov_data_placeholder) >= sizeof(CovData));
-static CovData *cov_data = reinterpret_cast<CovData*>(cov_data_placeholder);
+atomic_uint32_t dump_once_guard;  // Ensure that CovDump runs only once.
+
+// pc_array is the array containing the covered PCs.
+// To make the pc_array thread- and AS- safe it has to be large enough.
+// 128M counters "ought to be enough for anybody" (4M on 32-bit).
+// pc_array is allocated with MmapNoReserveOrDie and so it uses only as
+// much RAM as it really needs.
+static const uptr kPcArraySize = FIRST_32_SECOND_64(1 << 22, 1 << 27);
+static uptr *pc_array;
+static atomic_uintptr_t pc_array_index;
 
 namespace __sanitizer {
 
 // Simply add the pc into the vector under lock. If the function is called more
 // than once for a given PC it will be inserted multiple times, which is fine.
 static void CovAdd(uptr pc) {
-  BlockingMutexLock lock(&cov_data->mu);
-  cov_data->v.push_back(pc);
+  if (!pc_array) return;
+  uptr idx = atomic_fetch_add(&pc_array_index, 1, memory_order_relaxed);
+  CHECK_LT(idx, kPcArraySize);
+  pc_array[idx] = pc;
+}
+
+void CovInit() {
+  pc_array = reinterpret_cast<uptr *>(
+      MmapNoReserveOrDie(sizeof(uptr) * kPcArraySize, "CovInit"));
 }
 
 static inline bool CompareLess(const uptr &a, const uptr &b) {
@@ -66,13 +74,13 @@ static inline bool CompareLess(const upt
 // Dump the coverage on disk.
 void CovDump() {
 #if !SANITIZER_WINDOWS
-  if (atomic_fetch_add(&cov_data->guard, 1, memory_order_relaxed) != 0) return;
-  BlockingMutexLock lock(&cov_data->mu);
-  InternalMmapVector<uptr> &v = cov_data->v;
-  InternalSort(&v, v.size(), CompareLess);
-  InternalMmapVector<u32> offsets(v.size());
-  const uptr *vb = v.data();
-  const uptr *ve = vb + v.size();
+  if (atomic_fetch_add(&dump_once_guard, 1, memory_order_relaxed))
+    return;
+  uptr size = atomic_load(&pc_array_index, memory_order_relaxed);
+  InternalSort(&pc_array, size, CompareLess);
+  InternalMmapVector<u32> offsets(size);
+  const uptr *vb = pc_array;
+  const uptr *ve = vb + size;
   MemoryMappingLayout proc_maps(/*cache_enabled*/false);
   uptr mb, me, off, prot;
   InternalScopedBuffer<char> module(4096);
@@ -116,4 +124,5 @@ SANITIZER_INTERFACE_ATTRIBUTE void __san
   CovAdd(StackTrace::GetPreviousInstructionPc(GET_CALLER_PC()));
 }
 SANITIZER_INTERFACE_ATTRIBUTE void __sanitizer_cov_dump() { CovDump(); }
+SANITIZER_INTERFACE_ATTRIBUTE void __sanitizer_cov_init() { CovInit(); }
 }  // extern "C"

Modified: compiler-rt/trunk/lib/sanitizer_common/sanitizer_internal_defs.h
URL: http://llvm.org/viewvc/llvm-project/compiler-rt/trunk/lib/sanitizer_common/sanitizer_internal_defs.h?rev=207630&r1=207629&r2=207630&view=diff
==============================================================================
--- compiler-rt/trunk/lib/sanitizer_common/sanitizer_internal_defs.h (original)
+++ compiler-rt/trunk/lib/sanitizer_common/sanitizer_internal_defs.h Wed Apr 30 05:40:48 2014
@@ -104,6 +104,7 @@ extern "C" {
   void __sanitizer_report_error_summary(const char *error_summary);
 
   SANITIZER_INTERFACE_ATTRIBUTE void __sanitizer_cov_dump();
+  SANITIZER_INTERFACE_ATTRIBUTE void __sanitizer_cov_init();
   SANITIZER_INTERFACE_ATTRIBUTE void __sanitizer_cov();
   SANITIZER_INTERFACE_ATTRIBUTE
   void __sanitizer_annotate_contiguous_container(const void *beg,