[llvm-commits] [compiler-rt] r172183 - in /compiler-rt/trunk/lib: asan/asan_allocator.h asan/asan_allocator2.cc sanitizer_common/CMakeLists.txt sanitizer_common/sanitizer_quarantine.h
Dmitry Vyukov
dvyukov at google.com
Fri Jan 11 00:07:44 PST 2013
Author: dvyukov
Date: Fri Jan 11 02:07:43 2013
New Revision: 172183
URL: http://llvm.org/viewvc/llvm-project?rev=172183&view=rev
Log:
asan: Refactor asan memory quarantine.
Added:
compiler-rt/trunk/lib/sanitizer_common/sanitizer_quarantine.h
Modified:
compiler-rt/trunk/lib/asan/asan_allocator.h
compiler-rt/trunk/lib/asan/asan_allocator2.cc
compiler-rt/trunk/lib/sanitizer_common/CMakeLists.txt
Modified: compiler-rt/trunk/lib/asan/asan_allocator.h
URL: http://llvm.org/viewvc/llvm-project/compiler-rt/trunk/lib/asan/asan_allocator.h?rev=172183&r1=172182&r2=172183&view=diff
==============================================================================
--- compiler-rt/trunk/lib/asan/asan_allocator.h (original)
+++ compiler-rt/trunk/lib/asan/asan_allocator.h Fri Jan 11 02:07:43 2013
@@ -23,7 +23,7 @@
// to a new one (version 2). The change is quite intrusive so both allocators
// will co-exist in the source base for a while. The actual allocator is chosen
// at build time by redefining this macrozz.
-#define ASAN_ALLOCATOR_VERSION 1
+#define ASAN_ALLOCATOR_VERSION 2
namespace __asan {
@@ -98,16 +98,20 @@
struct AsanThreadLocalMallocStorage {
explicit AsanThreadLocalMallocStorage(LinkerInitialized x)
- : quarantine_(x) { }
+#if ASAN_ALLOCATOR_VERSION == 1
+ : quarantine_(x)
+#endif
+ { }
AsanThreadLocalMallocStorage() {
CHECK(REAL(memset));
REAL(memset)(this, 0, sizeof(AsanThreadLocalMallocStorage));
}
- AsanChunkFifoList quarantine_;
#if ASAN_ALLOCATOR_VERSION == 1
+ AsanChunkFifoList quarantine_;
AsanChunk *free_lists_[kNumberOfSizeClasses];
#else
+ uptr quarantine_cache[16];
uptr allocator2_cache[1024]; // Opaque.
#endif
void CommitBack();
Modified: compiler-rt/trunk/lib/asan/asan_allocator2.cc
URL: http://llvm.org/viewvc/llvm-project/compiler-rt/trunk/lib/asan/asan_allocator2.cc?rev=172183&r1=172182&r2=172183&view=diff
==============================================================================
--- compiler-rt/trunk/lib/asan/asan_allocator2.cc (original)
+++ compiler-rt/trunk/lib/asan/asan_allocator2.cc Fri Jan 11 02:07:43 2013
@@ -27,6 +27,7 @@
#include "sanitizer_common/sanitizer_internal_defs.h"
#include "sanitizer_common/sanitizer_list.h"
#include "sanitizer_common/sanitizer_stackdepot.h"
+#include "sanitizer_common/sanitizer_quarantine.h"
namespace __asan {
@@ -92,15 +93,6 @@
static const uptr kReturnOnZeroMalloc = 2048; // Zero page is protected.
-static int inited = 0;
-
-static void Init() {
- if (inited) return;
- __asan_init();
- inited = true; // this must happen before any threads are created.
- allocator.Init();
-}
-
// Every chunk of memory allocated by this allocator can be in one of 3 states:
// CHUNK_AVAILABLE: the chunk is in the free list and ready to be allocated.
// CHUNK_ALLOCATED: the chunk is allocated and not yet freed.
@@ -246,31 +238,26 @@
chunk_->FreeStackSize());
}
-class Quarantine: public AsanChunkFifoList {
- public:
- void SwallowThreadLocalQuarantine(AsanThreadLocalMallocStorage *ms) {
- AsanChunkFifoList *q = &ms->quarantine_;
- if (!q->size()) return;
- AsanChunkFifoList tmp;
- uptr max_size = flags()->quarantine_size;
- {
- SpinMutexLock l(&mutex_);
- PushList(q);
- while (size() > max_size)
- tmp.Push(Pop());
- }
- while (!tmp.empty())
- Deallocate(tmp.Pop(), ms);
- }
-
- void BypassThreadLocalQuarantine(AsanChunk *m) {
- SpinMutexLock l(&mutex_);
- Push(m);
+struct QuarantineCallback;
+typedef Quarantine<QuarantineCallback, AsanChunk> AsanQuarantine;
+typedef AsanQuarantine::Cache QuarantineCache;
+static AsanQuarantine quarantine(LINKER_INITIALIZED);
+static QuarantineCache fallback_quarantine_cache(LINKER_INITIALIZED);
+static AllocatorCache fallback_allocator_cache;
+static SpinMutex fallback_mutex;
+
+QuarantineCache *GetQuarantineCache(AsanThreadLocalMallocStorage *ms) {
+ DCHECK(ms);
+ DCHECK_LE(sizeof(QuarantineCache), sizeof(ms->quarantine_cache));
+ return reinterpret_cast<QuarantineCache *>(ms->quarantine_cache);
+}
+
+struct QuarantineCallback {
+ explicit QuarantineCallback(AllocatorCache *cache)
+ : cache_(cache) {
}
- private:
- static void Deallocate(AsanChunk *m, AsanThreadLocalMallocStorage *ms) {
- CHECK(m);
+ void Recycle(AsanChunk *m) {
CHECK(m->chunk_state == CHUNK_QUARANTINE);
m->chunk_state = CHUNK_AVAILABLE;
CHECK_NE(m->alloc_tid, kInvalidTid);
@@ -290,34 +277,27 @@
thread_stats.real_frees++;
thread_stats.really_freed += m->UsedSize();
- allocator.Deallocate(GetAllocatorCache(ms), p);
+ allocator.Deallocate(cache_, p);
}
- SpinMutex mutex_;
-};
-static Quarantine quarantine;
+ void *Allocate(uptr size) {
+ return allocator.Allocate(cache_, size, 0, false);
+ }
-void AsanChunkFifoList::PushList(AsanChunkFifoList *q) {
- CHECK(q->size() > 0);
- size_ += q->size();
- append_back(q);
- q->clear();
-}
-
-void AsanChunkFifoList::Push(AsanChunk *n) {
- push_back(n);
- size_ += n->UsedSize();
-}
-
-// Interesting performance observation: this function takes up to 15% of overal
-// allocator time. That's because *first_ has been evicted from cache long time
-// ago. Not sure if we can or want to do anything with this.
-AsanChunk *AsanChunkFifoList::Pop() {
- CHECK(first_);
- AsanChunk *res = front();
- size_ -= res->UsedSize();
- pop_front();
- return res;
+ void Deallocate(void *p) {
+ allocator.Deallocate(cache_, p);
+ }
+
+ AllocatorCache *cache_;
+};
+
+static void Init() {
+ static int inited = 0;
+ if (inited) return;
+ __asan_init();
+ inited = true; // this must happen before any threads are created.
+ allocator.Init();
+ quarantine.Init((uptr)flags()->quarantine_size, kMaxThreadLocalQuarantine);
}
static void *Allocate(uptr size, uptr alignment, StackTrace *stack,
@@ -468,13 +448,13 @@
// Push into quarantine.
if (t) {
- AsanChunkFifoList &q = t->malloc_storage().quarantine_;
- q.Push(m);
-
- if (q.size() > kMaxThreadLocalQuarantine)
- quarantine.SwallowThreadLocalQuarantine(&t->malloc_storage());
+ AsanThreadLocalMallocStorage *ms = &t->malloc_storage();
+ AllocatorCache *ac = GetAllocatorCache(ms);
+ quarantine.Put(GetQuarantineCache(ms), QuarantineCallback(ac), m);
} else {
- quarantine.BypassThreadLocalQuarantine(m);
+ SpinMutexLock l(&fallback_mutex);
+ AllocatorCache *ac = &fallback_allocator_cache;
+ quarantine.Put(&fallback_quarantine_cache, QuarantineCallback(ac), m);
}
ASAN_FREE_HOOK(ptr);
@@ -586,7 +566,8 @@
}
void AsanThreadLocalMallocStorage::CommitBack() {
- quarantine.SwallowThreadLocalQuarantine(this);
+ AllocatorCache *ac = GetAllocatorCache(this);
+ quarantine.Drain(GetQuarantineCache(this), QuarantineCallback(ac));
allocator.SwallowCache(GetAllocatorCache(this));
}
Modified: compiler-rt/trunk/lib/sanitizer_common/CMakeLists.txt
URL: http://llvm.org/viewvc/llvm-project/compiler-rt/trunk/lib/sanitizer_common/CMakeLists.txt?rev=172183&r1=172182&r2=172183&view=diff
==============================================================================
--- compiler-rt/trunk/lib/sanitizer_common/CMakeLists.txt (original)
+++ compiler-rt/trunk/lib/sanitizer_common/CMakeLists.txt Fri Jan 11 02:07:43 2013
@@ -38,6 +38,7 @@
sanitizer_placement_new.h
sanitizer_platform_interceptors.h
sanitizer_procmaps.h
+ sanitizer_quarantine.h
sanitizer_report_decorator.h
sanitizer_stackdepot.h
sanitizer_stacktrace.h
Added: compiler-rt/trunk/lib/sanitizer_common/sanitizer_quarantine.h
URL: http://llvm.org/viewvc/llvm-project/compiler-rt/trunk/lib/sanitizer_common/sanitizer_quarantine.h?rev=172183&view=auto
==============================================================================
--- compiler-rt/trunk/lib/sanitizer_common/sanitizer_quarantine.h (added)
+++ compiler-rt/trunk/lib/sanitizer_common/sanitizer_quarantine.h Fri Jan 11 02:07:43 2013
@@ -0,0 +1,107 @@
+//===-- sanitizer_quarantine.h ----------------------------------*- C++ -*-===//
+//
+// The LLVM Compiler Infrastructure
+//
+// This file is distributed under the University of Illinois Open Source
+// License. See LICENSE.TXT for details.
+//
+//===----------------------------------------------------------------------===//
+//
+// Memory quarantine for AddressSanitizer and potentially other tools.
+// Quarantine caches some specified amount of memory in per-thread caches,
+// then evicts to global FIFO queue. When the queue reaches specified threshold,
+// oldest memory is recycled.
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef SANITIZER_QUARANTINE_H
+#define SANITIZER_QUARANTINE_H
+
+#include "sanitizer_internal_defs.h"
+#include "sanitizer_mutex.h"
+
+namespace __sanitizer {
+
+template<typename Node> class QuarantineCache;
+
+// The callback interface is:
+// Callback cb;
+// Node *ptr;
+// cb.Recycle(ptr);
+template<typename Callback, typename Node>
+class Quarantine {
+ public:
+ typedef QuarantineCache<Node> Cache;
+
+ explicit Quarantine(LinkerInitialized)
+ : cache_(LINKER_INITIALIZED) {
+ }
+
+ void Init(uptr size, uptr cache_size) {
+ max_size_ = size;
+ max_cache_size_ = cache_size;
+ }
+
+ void Put(Cache *c, Callback cb, Node *ptr) {
+ c->Enqueue(ptr);
+ if (c->Size() > max_cache_size_)
+ Drain(c, cb);
+ }
+
+ void Drain(Cache *c, Callback cb) {
+ SpinMutexLock l(&mutex_);
+ while (Node *ptr = c->Dequeue())
+ cache_.Enqueue(ptr);
+ while (cache_.Size() > max_size_) {
+ Node *ptr = cache_.Dequeue();
+ cb.Recycle(ptr);
+ }
+ }
+
+ private:
+ SpinMutex mutex_;
+ uptr max_size_;
+ uptr max_cache_size_;
+ Cache cache_;
+};
+
+// Per-thread cache of memory blocks (essentially FIFO queue).
+template<typename Node>
+class QuarantineCache {
+ public:
+ explicit QuarantineCache(LinkerInitialized) {
+ }
+
+ uptr Size() const {
+ return size_;
+ }
+
+ void Enqueue(Node *ptr) {
+ size_ += ptr->UsedSize();
+ ptr->next = 0;
+ if (tail_)
+ tail_->next = ptr;
+ else
+ head_ = ptr;
+ tail_ = ptr;
+ }
+
+ Node *Dequeue() {
+ Node *ptr = head_;
+ if (ptr == 0)
+ return 0;
+ head_ = ptr->next;
+ if (head_ == 0)
+ tail_ = 0;
+ size_ -= ptr->UsedSize();
+ return ptr;
+ }
+
+ private:
+ Node *head_;
+ Node *tail_;
+ uptr size_;
+};
+}
+
+#endif // #ifndef SANITIZER_QUARANTINE_H
More information about the llvm-commits
mailing list