<div dir="ltr"><div class="gmail_default" style><br></div><div class="gmail_extra"><br><br><div class="gmail_quote">On Fri, Jan 11, 2013 at 12:07 PM, Dmitry Vyukov <span dir="ltr"><<a href="mailto:dvyukov@google.com" target="_blank">dvyukov@google.com</a>></span> wrote:<br>
<blockquote class="gmail_quote" style="margin:0 0 0 .8ex;border-left:1px #ccc solid;padding-left:1ex">Author: dvyukov<br>
Date: Fri Jan 11 02:07:43 2013<br>
New Revision: 172183<br>
<br>
URL: <a href="http://llvm.org/viewvc/llvm-project?rev=172183&view=rev" target="_blank">http://llvm.org/viewvc/llvm-project?rev=172183&view=rev</a><br>
Log:<br>
asan: Refactor asan memory quarantine.<br>
<br>
Added:<br>
compiler-rt/trunk/lib/sanitizer_common/sanitizer_quarantine.h<br>
Modified:<br>
compiler-rt/trunk/lib/asan/asan_allocator.h<br>
compiler-rt/trunk/lib/asan/asan_allocator2.cc<br>
compiler-rt/trunk/lib/sanitizer_common/CMakeLists.txt<br>
<br>
Modified: compiler-rt/trunk/lib/asan/asan_allocator.h<br>
URL: <a href="http://llvm.org/viewvc/llvm-project/compiler-rt/trunk/lib/asan/asan_allocator.h?rev=172183&r1=172182&r2=172183&view=diff" target="_blank">http://llvm.org/viewvc/llvm-project/compiler-rt/trunk/lib/asan/asan_allocator.h?rev=172183&r1=172182&r2=172183&view=diff</a><br>
==============================================================================<br>
--- compiler-rt/trunk/lib/asan/asan_allocator.h (original)<br>
+++ compiler-rt/trunk/lib/asan/asan_allocator.h Fri Jan 11 02:07:43 2013<br>
@@ -23,7 +23,7 @@<br>
// to a new one (version 2). The change is quite intrusive so both allocators<br>
// will co-exist in the source base for a while. The actual allocator is chosen<br>
// at build time by redefining this macrozz.<br>
-#define ASAN_ALLOCATOR_VERSION 1<br>
+#define ASAN_ALLOCATOR_VERSION 2<br>
<br>
namespace __asan {<br>
<br>
@@ -98,16 +98,20 @@<br>
<br>
struct AsanThreadLocalMallocStorage {<br>
explicit AsanThreadLocalMallocStorage(LinkerInitialized x)<br>
- : quarantine_(x) { }<br>
+#if ASAN_ALLOCATOR_VERSION == 1<br>
+ : quarantine_(x)<br>
+#endif<br>
+ { }<br>
AsanThreadLocalMallocStorage() {<br>
CHECK(REAL(memset));<br>
REAL(memset)(this, 0, sizeof(AsanThreadLocalMallocStorage));<br>
}<br>
<br>
- AsanChunkFifoList quarantine_;<br>
#if ASAN_ALLOCATOR_VERSION == 1<br>
+ AsanChunkFifoList quarantine_;<br>
AsanChunk *free_lists_[kNumberOfSizeClasses];<br>
#else<br>
+ uptr quarantine_cache[16];<br>
uptr allocator2_cache[1024]; // Opaque.<br>
#endif<br>
void CommitBack();<br>
<br>
Modified: compiler-rt/trunk/lib/asan/asan_allocator2.cc<br>
URL: <a href="http://llvm.org/viewvc/llvm-project/compiler-rt/trunk/lib/asan/asan_allocator2.cc?rev=172183&r1=172182&r2=172183&view=diff" target="_blank">http://llvm.org/viewvc/llvm-project/compiler-rt/trunk/lib/asan/asan_allocator2.cc?rev=172183&r1=172182&r2=172183&view=diff</a><br>
==============================================================================<br>
--- compiler-rt/trunk/lib/asan/asan_allocator2.cc (original)<br>
+++ compiler-rt/trunk/lib/asan/asan_allocator2.cc Fri Jan 11 02:07:43 2013<br>
@@ -27,6 +27,7 @@<br>
#include "sanitizer_common/sanitizer_internal_defs.h"<br>
#include "sanitizer_common/sanitizer_list.h"<br>
#include "sanitizer_common/sanitizer_stackdepot.h"<br>
+#include "sanitizer_common/sanitizer_quarantine.h"<br>
<br>
namespace __asan {<br>
<br>
@@ -92,15 +93,6 @@<br>
<br>
static const uptr kReturnOnZeroMalloc = 2048; // Zero page is protected.<br>
<br>
-static int inited = 0;<br>
-<br>
-static void Init() {<br>
- if (inited) return;<br>
- __asan_init();<br>
- inited = true; // this must happen before any threads are created.<br>
- allocator.Init();<br>
-}<br>
-<br>
// Every chunk of memory allocated by this allocator can be in one of 3 states:<br>
// CHUNK_AVAILABLE: the chunk is in the free list and ready to be allocated.<br>
// CHUNK_ALLOCATED: the chunk is allocated and not yet freed.<br>
@@ -246,31 +238,26 @@<br>
chunk_->FreeStackSize());<br>
}<br>
<br>
-class Quarantine: public AsanChunkFifoList {<br>
- public:<br>
- void SwallowThreadLocalQuarantine(AsanThreadLocalMallocStorage *ms) {<br>
- AsanChunkFifoList *q = &ms->quarantine_;<br>
- if (!q->size()) return;<br>
- AsanChunkFifoList tmp;<br>
- uptr max_size = flags()->quarantine_size;<br>
- {<br>
- SpinMutexLock l(&mutex_);<br>
- PushList(q);<br>
- while (size() > max_size)<br>
- tmp.Push(Pop());<br>
- }<br>
- while (!tmp.empty())<br>
- Deallocate(tmp.Pop(), ms);<br>
- }<br>
-<br>
- void BypassThreadLocalQuarantine(AsanChunk *m) {<br>
- SpinMutexLock l(&mutex_);<br>
- Push(m);<br>
+struct QuarantineCallback;<br>
+typedef Quarantine<QuarantineCallback, AsanChunk> AsanQuarantine;<br>
+typedef AsanQuarantine::Cache QuarantineCache;<br>
+static AsanQuarantine quarantine(LINKER_INITIALIZED);<br>
+static QuarantineCache fallback_quarantine_cache(LINKER_INITIALIZED);<br>
+static AllocatorCache fallback_allocator_cache;<br>
+static SpinMutex fallback_mutex;<br>
+<br>
+QuarantineCache *GetQuarantineCache(AsanThreadLocalMallocStorage *ms) {<br>
+ DCHECK(ms);<br></blockquote><div><br></div><div style>You've left DCHECKs</div><div style> </div><blockquote class="gmail_quote" style="margin:0 0 0 .8ex;border-left:1px #ccc solid;padding-left:1ex">
+ DCHECK_LE(sizeof(QuarantineCache), sizeof(ms->quarantine_cache));<br>
+ return reinterpret_cast<QuarantineCache *>(ms->quarantine_cache);<br>
+}<br>
+<br>
+struct QuarantineCallback {<br>
+ explicit QuarantineCallback(AllocatorCache *cache)<br>
+ : cache_(cache) {<br>
}<br>
<br>
- private:<br>
- static void Deallocate(AsanChunk *m, AsanThreadLocalMallocStorage *ms) {<br>
- CHECK(m);<br>
+ void Recycle(AsanChunk *m) {<br>
CHECK(m->chunk_state == CHUNK_QUARANTINE);<br>
m->chunk_state = CHUNK_AVAILABLE;<br>
CHECK_NE(m->alloc_tid, kInvalidTid);<br>
@@ -290,34 +277,27 @@<br>
thread_stats.real_frees++;<br>
thread_stats.really_freed += m->UsedSize();<br>
<br>
- allocator.Deallocate(GetAllocatorCache(ms), p);<br>
+ allocator.Deallocate(cache_, p);<br>
}<br>
- SpinMutex mutex_;<br>
-};<br>
<br>
-static Quarantine quarantine;<br>
+ void *Allocate(uptr size) {<br>
+ return allocator.Allocate(cache_, size, 0, false);<br>
+ }<br>
<br>
-void AsanChunkFifoList::PushList(AsanChunkFifoList *q) {<br>
- CHECK(q->size() > 0);<br>
- size_ += q->size();<br>
- append_back(q);<br>
- q->clear();<br>
-}<br>
-<br>
-void AsanChunkFifoList::Push(AsanChunk *n) {<br>
- push_back(n);<br>
- size_ += n->UsedSize();<br>
-}<br>
-<br>
-// Interesting performance observation: this function takes up to 15% of overal<br>
-// allocator time. That's because *first_ has been evicted from cache long time<br>
-// ago. Not sure if we can or want to do anything with this.<br>
-AsanChunk *AsanChunkFifoList::Pop() {<br>
- CHECK(first_);<br>
- AsanChunk *res = front();<br>
- size_ -= res->UsedSize();<br>
- pop_front();<br>
- return res;<br>
+ void Deallocate(void *p) {<br>
+ allocator.Deallocate(cache_, p);<br>
+ }<br>
+<br>
+ AllocatorCache *cache_;<br>
+};<br>
+<br>
+static void Init() {<br>
+ static int inited = 0;<br>
+ if (inited) return;<br>
+ __asan_init();<br>
+ inited = true; // this must happen before any threads are created.<br>
+ allocator.Init();<br>
+ quarantine.Init((uptr)flags()->quarantine_size, kMaxThreadLocalQuarantine);<br>
}<br>
<br>
static void *Allocate(uptr size, uptr alignment, StackTrace *stack,<br>
@@ -468,13 +448,13 @@<br>
<br>
// Push into quarantine.<br>
if (t) {<br>
- AsanChunkFifoList &q = t->malloc_storage().quarantine_;<br>
- q.Push(m);<br>
-<br>
- if (q.size() > kMaxThreadLocalQuarantine)<br>
- quarantine.SwallowThreadLocalQuarantine(&t->malloc_storage());<br>
+ AsanThreadLocalMallocStorage *ms = &t->malloc_storage();<br>
+ AllocatorCache *ac = GetAllocatorCache(ms);<br>
+ quarantine.Put(GetQuarantineCache(ms), QuarantineCallback(ac), m);<br>
} else {<br>
- quarantine.BypassThreadLocalQuarantine(m);<br>
+ SpinMutexLock l(&fallback_mutex);<br>
+ AllocatorCache *ac = &fallback_allocator_cache;<br>
+ quarantine.Put(&fallback_quarantine_cache, QuarantineCallback(ac), m);<br>
}<br>
<br>
ASAN_FREE_HOOK(ptr);<br>
@@ -586,7 +566,8 @@<br>
}<br>
<br>
void AsanThreadLocalMallocStorage::CommitBack() {<br>
- quarantine.SwallowThreadLocalQuarantine(this);<br>
+ AllocatorCache *ac = GetAllocatorCache(this);<br>
+ quarantine.Drain(GetQuarantineCache(this), QuarantineCallback(ac));<br>
allocator.SwallowCache(GetAllocatorCache(this));<br>
}<br>
<br>
<br>
Modified: compiler-rt/trunk/lib/sanitizer_common/CMakeLists.txt<br>
URL: <a href="http://llvm.org/viewvc/llvm-project/compiler-rt/trunk/lib/sanitizer_common/CMakeLists.txt?rev=172183&r1=172182&r2=172183&view=diff" target="_blank">http://llvm.org/viewvc/llvm-project/compiler-rt/trunk/lib/sanitizer_common/CMakeLists.txt?rev=172183&r1=172182&r2=172183&view=diff</a><br>
==============================================================================<br>
--- compiler-rt/trunk/lib/sanitizer_common/CMakeLists.txt (original)<br>
+++ compiler-rt/trunk/lib/sanitizer_common/CMakeLists.txt Fri Jan 11 02:07:43 2013<br>
@@ -38,6 +38,7 @@<br>
sanitizer_placement_new.h<br>
sanitizer_platform_interceptors.h<br>
sanitizer_procmaps.h<br>
+ sanitizer_quarantine.h<br>
sanitizer_report_decorator.h<br>
sanitizer_stackdepot.h<br>
sanitizer_stacktrace.h<br>
<br>
Added: compiler-rt/trunk/lib/sanitizer_common/sanitizer_quarantine.h<br>
URL: <a href="http://llvm.org/viewvc/llvm-project/compiler-rt/trunk/lib/sanitizer_common/sanitizer_quarantine.h?rev=172183&view=auto" target="_blank">http://llvm.org/viewvc/llvm-project/compiler-rt/trunk/lib/sanitizer_common/sanitizer_quarantine.h?rev=172183&view=auto</a><br>
==============================================================================<br>
--- compiler-rt/trunk/lib/sanitizer_common/sanitizer_quarantine.h (added)<br>
+++ compiler-rt/trunk/lib/sanitizer_common/sanitizer_quarantine.h Fri Jan 11 02:07:43 2013<br>
@@ -0,0 +1,107 @@<br>
+//===-- sanitizer_quarantine.h ----------------------------------*- C++ -*-===//<br>
+//<br>
+// The LLVM Compiler Infrastructure<br>
+//<br>
+// This file is distributed under the University of Illinois Open Source<br>
+// License. See LICENSE.TXT for details.<br>
+//<br>
+//===----------------------------------------------------------------------===//<br>
+//<br>
+// Memory quarantine for AddressSanitizer and potentially other tools.<br>
+// Quarantine caches some specified amount of memory in per-thread caches,<br>
+// then evicts to global FIFO queue. When the queue reaches specified threshold,<br>
+// oldest memory is recycled.<br>
+//<br>
+//===----------------------------------------------------------------------===//<br>
+<br>
+#ifndef SANITIZER_QUARANTINE_H<br>
+#define SANITIZER_QUARANTINE_H<br>
+<br>
+#include "sanitizer_internal_defs.h"<br>
+#include "sanitizer_mutex.h"<br>
+<br>
+namespace __sanitizer {<br>
+<br>
+template<typename Node> class QuarantineCache;<br>
+<br>
+// The callback interface is:<br>
+// Callback cb;<br>
+// Node *ptr;<br>
+// cb.Recycle(ptr);<br>
+template<typename Callback, typename Node><br>
+class Quarantine {<br>
+ public:<br>
+ typedef QuarantineCache<Node> Cache;<br>
+<br>
+ explicit Quarantine(LinkerInitialized)<br>
+ : cache_(LINKER_INITIALIZED) {<br>
+ }<br>
+<br>
+ void Init(uptr size, uptr cache_size) {<br>
+ max_size_ = size;<br>
+ max_cache_size_ = cache_size;<br>
+ }<br>
+<br>
+ void Put(Cache *c, Callback cb, Node *ptr) {<br>
+ c->Enqueue(ptr);<br>
+ if (c->Size() > max_cache_size_)<br>
+ Drain(c, cb);<br>
+ }<br>
+<br>
+ void Drain(Cache *c, Callback cb) {<br>
+ SpinMutexLock l(&mutex_);<br>
+ while (Node *ptr = c->Dequeue())<br>
+ cache_.Enqueue(ptr);<br>
+ while (cache_.Size() > max_size_) {<br>
+ Node *ptr = cache_.Dequeue();<br>
+ cb.Recycle(ptr);<br>
+ }<br>
+ }<br>
+<br>
+ private:<br>
+ SpinMutex mutex_;<br>
+ uptr max_size_;<br>
+ uptr max_cache_size_;<br>
+ Cache cache_;<br>
+};<br>
+<br>
+// Per-thread cache of memory blocks (essentially FIFO queue).<br>
+template<typename Node><br>
+class QuarantineCache {<br>
+ public:<br>
+ explicit QuarantineCache(LinkerInitialized) {<br>
+ }<br>
+<br>
+ uptr Size() const {<br>
+ return size_;<br>
+ }<br>
+<br>
+ void Enqueue(Node *ptr) {<br>
+ size_ += ptr->UsedSize();<br>
+ ptr->next = 0;<br>
+ if (tail_)<br>
+ tail_->next = ptr;<br>
+ else<br>
+ head_ = ptr;<br>
+ tail_ = ptr;<br>
+ }<br>
+<br>
+ Node *Dequeue() {<br>
+ Node *ptr = head_;<br>
+ if (ptr == 0)<br>
+ return 0;<br>
+ head_ = ptr->next;<br>
+ if (head_ == 0)<br>
+ tail_ = 0;<br>
+ size_ -= ptr->UsedSize();<br>
+ return ptr;<br>
+ }<br>
+<br>
+ private:<br>
+ Node *head_;<br>
+ Node *tail_;<br>
+ uptr size_;<br>
+};<br>
+}<br>
+<br>
+#endif // #ifndef SANITIZER_QUARANTINE_H<br>
<br>
<br>
_______________________________________________<br>
llvm-commits mailing list<br>
<a href="mailto:llvm-commits@cs.uiuc.edu">llvm-commits@cs.uiuc.edu</a><br>
<a href="http://lists.cs.uiuc.edu/mailman/listinfo/llvm-commits" target="_blank">http://lists.cs.uiuc.edu/mailman/listinfo/llvm-commits</a><br>
</blockquote></div><br></div></div>