[llvm-commits] [compiler-rt] r172183 - in /compiler-rt/trunk/lib: asan/asan_allocator.h asan/asan_allocator2.cc sanitizer_common/CMakeLists.txt sanitizer_common/sanitizer_quarantine.h
Kostya Serebryany
kcc at google.com
Fri Jan 11 00:18:33 PST 2013
On Fri, Jan 11, 2013 at 12:07 PM, Dmitry Vyukov <dvyukov at google.com> wrote:
> Author: dvyukov
> Date: Fri Jan 11 02:07:43 2013
> New Revision: 172183
>
> URL: http://llvm.org/viewvc/llvm-project?rev=172183&view=rev
> Log:
> asan: Refactor asan memory quarantine.
>
> Added:
> compiler-rt/trunk/lib/sanitizer_common/sanitizer_quarantine.h
> Modified:
> compiler-rt/trunk/lib/asan/asan_allocator.h
> compiler-rt/trunk/lib/asan/asan_allocator2.cc
> compiler-rt/trunk/lib/sanitizer_common/CMakeLists.txt
>
> Modified: compiler-rt/trunk/lib/asan/asan_allocator.h
> URL:
> http://llvm.org/viewvc/llvm-project/compiler-rt/trunk/lib/asan/asan_allocator.h?rev=172183&r1=172182&r2=172183&view=diff
>
> ==============================================================================
> --- compiler-rt/trunk/lib/asan/asan_allocator.h (original)
> +++ compiler-rt/trunk/lib/asan/asan_allocator.h Fri Jan 11 02:07:43 2013
> @@ -23,7 +23,7 @@
> // to a new one (version 2). The change is quite intrusive so both
> allocators
> // will co-exist in the source base for a while. The actual allocator is
> chosen
> // at build time by redefining this macrozz.
> -#define ASAN_ALLOCATOR_VERSION 1
> +#define ASAN_ALLOCATOR_VERSION 2
>
> namespace __asan {
>
> @@ -98,16 +98,20 @@
>
> struct AsanThreadLocalMallocStorage {
> explicit AsanThreadLocalMallocStorage(LinkerInitialized x)
> - : quarantine_(x) { }
> +#if ASAN_ALLOCATOR_VERSION == 1
> + : quarantine_(x)
> +#endif
> + { }
> AsanThreadLocalMallocStorage() {
> CHECK(REAL(memset));
> REAL(memset)(this, 0, sizeof(AsanThreadLocalMallocStorage));
> }
>
> - AsanChunkFifoList quarantine_;
> #if ASAN_ALLOCATOR_VERSION == 1
> + AsanChunkFifoList quarantine_;
> AsanChunk *free_lists_[kNumberOfSizeClasses];
> #else
> + uptr quarantine_cache[16];
> uptr allocator2_cache[1024]; // Opaque.
> #endif
> void CommitBack();
>
> Modified: compiler-rt/trunk/lib/asan/asan_allocator2.cc
> URL:
> http://llvm.org/viewvc/llvm-project/compiler-rt/trunk/lib/asan/asan_allocator2.cc?rev=172183&r1=172182&r2=172183&view=diff
>
> ==============================================================================
> --- compiler-rt/trunk/lib/asan/asan_allocator2.cc (original)
> +++ compiler-rt/trunk/lib/asan/asan_allocator2.cc Fri Jan 11 02:07:43 2013
> @@ -27,6 +27,7 @@
> #include "sanitizer_common/sanitizer_internal_defs.h"
> #include "sanitizer_common/sanitizer_list.h"
> #include "sanitizer_common/sanitizer_stackdepot.h"
> +#include "sanitizer_common/sanitizer_quarantine.h"
>
> namespace __asan {
>
> @@ -92,15 +93,6 @@
>
> static const uptr kReturnOnZeroMalloc = 2048; // Zero page is protected.
>
> -static int inited = 0;
> -
> -static void Init() {
> - if (inited) return;
> - __asan_init();
> - inited = true; // this must happen before any threads are created.
> - allocator.Init();
> -}
> -
> // Every chunk of memory allocated by this allocator can be in one of 3
> states:
> // CHUNK_AVAILABLE: the chunk is in the free list and ready to be
> allocated.
> // CHUNK_ALLOCATED: the chunk is allocated and not yet freed.
> @@ -246,31 +238,26 @@
> chunk_->FreeStackSize());
> }
>
> -class Quarantine: public AsanChunkFifoList {
> - public:
> - void SwallowThreadLocalQuarantine(AsanThreadLocalMallocStorage *ms) {
> - AsanChunkFifoList *q = &ms->quarantine_;
> - if (!q->size()) return;
> - AsanChunkFifoList tmp;
> - uptr max_size = flags()->quarantine_size;
> - {
> - SpinMutexLock l(&mutex_);
> - PushList(q);
> - while (size() > max_size)
> - tmp.Push(Pop());
> - }
> - while (!tmp.empty())
> - Deallocate(tmp.Pop(), ms);
> - }
> -
> - void BypassThreadLocalQuarantine(AsanChunk *m) {
> - SpinMutexLock l(&mutex_);
> - Push(m);
> +struct QuarantineCallback;
> +typedef Quarantine<QuarantineCallback, AsanChunk> AsanQuarantine;
> +typedef AsanQuarantine::Cache QuarantineCache;
> +static AsanQuarantine quarantine(LINKER_INITIALIZED);
> +static QuarantineCache fallback_quarantine_cache(LINKER_INITIALIZED);
> +static AllocatorCache fallback_allocator_cache;
> +static SpinMutex fallback_mutex;
> +
> +QuarantineCache *GetQuarantineCache(AsanThreadLocalMallocStorage *ms) {
> + DCHECK(ms);
>
You've left DCHECKs
> + DCHECK_LE(sizeof(QuarantineCache), sizeof(ms->quarantine_cache));
> + return reinterpret_cast<QuarantineCache *>(ms->quarantine_cache);
> +}
> +
> +struct QuarantineCallback {
> + explicit QuarantineCallback(AllocatorCache *cache)
> + : cache_(cache) {
> }
>
> - private:
> - static void Deallocate(AsanChunk *m, AsanThreadLocalMallocStorage *ms) {
> - CHECK(m);
> + void Recycle(AsanChunk *m) {
> CHECK(m->chunk_state == CHUNK_QUARANTINE);
> m->chunk_state = CHUNK_AVAILABLE;
> CHECK_NE(m->alloc_tid, kInvalidTid);
> @@ -290,34 +277,27 @@
> thread_stats.real_frees++;
> thread_stats.really_freed += m->UsedSize();
>
> - allocator.Deallocate(GetAllocatorCache(ms), p);
> + allocator.Deallocate(cache_, p);
> }
> - SpinMutex mutex_;
> -};
>
> -static Quarantine quarantine;
> + void *Allocate(uptr size) {
> + return allocator.Allocate(cache_, size, 0, false);
> + }
>
> -void AsanChunkFifoList::PushList(AsanChunkFifoList *q) {
> - CHECK(q->size() > 0);
> - size_ += q->size();
> - append_back(q);
> - q->clear();
> -}
> -
> -void AsanChunkFifoList::Push(AsanChunk *n) {
> - push_back(n);
> - size_ += n->UsedSize();
> -}
> -
> -// Interesting performance observation: this function takes up to 15% of
> overal
> -// allocator time. That's because *first_ has been evicted from cache
> long time
> -// ago. Not sure if we can or want to do anything with this.
> -AsanChunk *AsanChunkFifoList::Pop() {
> - CHECK(first_);
> - AsanChunk *res = front();
> - size_ -= res->UsedSize();
> - pop_front();
> - return res;
> + void Deallocate(void *p) {
> + allocator.Deallocate(cache_, p);
> + }
> +
> + AllocatorCache *cache_;
> +};
> +
> +static void Init() {
> + static int inited = 0;
> + if (inited) return;
> + __asan_init();
> + inited = true; // this must happen before any threads are created.
> + allocator.Init();
> + quarantine.Init((uptr)flags()->quarantine_size,
> kMaxThreadLocalQuarantine);
> }
>
> static void *Allocate(uptr size, uptr alignment, StackTrace *stack,
> @@ -468,13 +448,13 @@
>
> // Push into quarantine.
> if (t) {
> - AsanChunkFifoList &q = t->malloc_storage().quarantine_;
> - q.Push(m);
> -
> - if (q.size() > kMaxThreadLocalQuarantine)
> - quarantine.SwallowThreadLocalQuarantine(&t->malloc_storage());
> + AsanThreadLocalMallocStorage *ms = &t->malloc_storage();
> + AllocatorCache *ac = GetAllocatorCache(ms);
> + quarantine.Put(GetQuarantineCache(ms), QuarantineCallback(ac), m);
> } else {
> - quarantine.BypassThreadLocalQuarantine(m);
> + SpinMutexLock l(&fallback_mutex);
> + AllocatorCache *ac = &fallback_allocator_cache;
> + quarantine.Put(&fallback_quarantine_cache, QuarantineCallback(ac), m);
> }
>
> ASAN_FREE_HOOK(ptr);
> @@ -586,7 +566,8 @@
> }
>
> void AsanThreadLocalMallocStorage::CommitBack() {
> - quarantine.SwallowThreadLocalQuarantine(this);
> + AllocatorCache *ac = GetAllocatorCache(this);
> + quarantine.Drain(GetQuarantineCache(this), QuarantineCallback(ac));
> allocator.SwallowCache(GetAllocatorCache(this));
> }
>
>
> Modified: compiler-rt/trunk/lib/sanitizer_common/CMakeLists.txt
> URL:
> http://llvm.org/viewvc/llvm-project/compiler-rt/trunk/lib/sanitizer_common/CMakeLists.txt?rev=172183&r1=172182&r2=172183&view=diff
>
> ==============================================================================
> --- compiler-rt/trunk/lib/sanitizer_common/CMakeLists.txt (original)
> +++ compiler-rt/trunk/lib/sanitizer_common/CMakeLists.txt Fri Jan 11
> 02:07:43 2013
> @@ -38,6 +38,7 @@
> sanitizer_placement_new.h
> sanitizer_platform_interceptors.h
> sanitizer_procmaps.h
> + sanitizer_quarantine.h
> sanitizer_report_decorator.h
> sanitizer_stackdepot.h
> sanitizer_stacktrace.h
>
> Added: compiler-rt/trunk/lib/sanitizer_common/sanitizer_quarantine.h
> URL:
> http://llvm.org/viewvc/llvm-project/compiler-rt/trunk/lib/sanitizer_common/sanitizer_quarantine.h?rev=172183&view=auto
>
> ==============================================================================
> --- compiler-rt/trunk/lib/sanitizer_common/sanitizer_quarantine.h (added)
> +++ compiler-rt/trunk/lib/sanitizer_common/sanitizer_quarantine.h Fri Jan
> 11 02:07:43 2013
> @@ -0,0 +1,107 @@
> +//===-- sanitizer_quarantine.h ----------------------------------*- C++
> -*-===//
> +//
> +// The LLVM Compiler Infrastructure
> +//
> +// This file is distributed under the University of Illinois Open Source
> +// License. See LICENSE.TXT for details.
> +//
>
> +//===----------------------------------------------------------------------===//
> +//
> +// Memory quarantine for AddressSanitizer and potentially other tools.
> +// Quarantine caches some specified amount of memory in per-thread caches,
> +// then evicts to global FIFO queue. When the queue reaches specified
> threshold,
> +// oldest memory is recycled.
> +//
>
> +//===----------------------------------------------------------------------===//
> +
> +#ifndef SANITIZER_QUARANTINE_H
> +#define SANITIZER_QUARANTINE_H
> +
> +#include "sanitizer_internal_defs.h"
> +#include "sanitizer_mutex.h"
> +
> +namespace __sanitizer {
> +
> +template<typename Node> class QuarantineCache;
> +
> +// The callback interface is:
> +// Callback cb;
> +// Node *ptr;
> +// cb.Recycle(ptr);
> +template<typename Callback, typename Node>
> +class Quarantine {
> + public:
> + typedef QuarantineCache<Node> Cache;
> +
> + explicit Quarantine(LinkerInitialized)
> + : cache_(LINKER_INITIALIZED) {
> + }
> +
> + void Init(uptr size, uptr cache_size) {
> + max_size_ = size;
> + max_cache_size_ = cache_size;
> + }
> +
> + void Put(Cache *c, Callback cb, Node *ptr) {
> + c->Enqueue(ptr);
> + if (c->Size() > max_cache_size_)
> + Drain(c, cb);
> + }
> +
> + void Drain(Cache *c, Callback cb) {
> + SpinMutexLock l(&mutex_);
> + while (Node *ptr = c->Dequeue())
> + cache_.Enqueue(ptr);
> + while (cache_.Size() > max_size_) {
> + Node *ptr = cache_.Dequeue();
> + cb.Recycle(ptr);
> + }
> + }
> +
> + private:
> + SpinMutex mutex_;
> + uptr max_size_;
> + uptr max_cache_size_;
> + Cache cache_;
> +};
> +
> +// Per-thread cache of memory blocks (essentially FIFO queue).
> +template<typename Node>
> +class QuarantineCache {
> + public:
> + explicit QuarantineCache(LinkerInitialized) {
> + }
> +
> + uptr Size() const {
> + return size_;
> + }
> +
> + void Enqueue(Node *ptr) {
> + size_ += ptr->UsedSize();
> + ptr->next = 0;
> + if (tail_)
> + tail_->next = ptr;
> + else
> + head_ = ptr;
> + tail_ = ptr;
> + }
> +
> + Node *Dequeue() {
> + Node *ptr = head_;
> + if (ptr == 0)
> + return 0;
> + head_ = ptr->next;
> + if (head_ == 0)
> + tail_ = 0;
> + size_ -= ptr->UsedSize();
> + return ptr;
> + }
> +
> + private:
> + Node *head_;
> + Node *tail_;
> + uptr size_;
> +};
> +}
> +
> +#endif // #ifndef SANITIZER_QUARANTINE_H
>
>
> _______________________________________________
> llvm-commits mailing list
> llvm-commits at cs.uiuc.edu
> http://lists.cs.uiuc.edu/mailman/listinfo/llvm-commits
>
-------------- next part --------------
An HTML attachment was scrubbed...
URL: <http://lists.llvm.org/pipermail/llvm-commits/attachments/20130111/a12d9e61/attachment.html>
More information about the llvm-commits
mailing list