[compiler-rt] 51aa61e - [hwasan] Fix lock contention on thread creation.

[Evgenii Stepanov via llvm-commits]

Author: Evgenii Stepanov
Date: 2021-04-14T17:13:15-07:00
New Revision: 51aa61e74bdb394d89217350ddcb0fd04a21e32f

URL: https://github.com/llvm/llvm-project/commit/51aa61e74bdb394d89217350ddcb0fd04a21e32f
DIFF: https://github.com/llvm/llvm-project/commit/51aa61e74bdb394d89217350ddcb0fd04a21e32f.diff

LOG: [hwasan] Fix lock contention on thread creation.

Do not hold the free/live thread list lock longer than necessary.
This change speeds up the following benchmark 10x.

constexpr int kTopThreads = 50;
constexpr int kChildThreads = 20;
constexpr int kChildIterations = 8;

void Thread() {
  for (int i = 0; i < kChildIterations; ++i) {
    std::vector<std::thread> threads;
    for (int i = 0; i < kChildThreads; ++i)
      threads.emplace_back([](){});
    for (auto& t : threads)
      t.join();
  }
}

int main() {
  std::vector<std::thread> threads;
  for (int i = 0; i < kTopThreads; ++i)
    threads.emplace_back(Thread);
  for (auto& t : threads)
    t.join();
}

Differential Revision: https://reviews.llvm.org/D100348

Added: 
    

Modified: 
    compiler-rt/lib/hwasan/hwasan_thread_list.h

Removed: 
    


################################################################################
diff  --git a/compiler-rt/lib/hwasan/hwasan_thread_list.h b/compiler-rt/lib/hwasan/hwasan_thread_list.h
index e596bde36662..3b89bad59cc3 100644
--- a/compiler-rt/lib/hwasan/hwasan_thread_list.h
+++ b/compiler-rt/lib/hwasan/hwasan_thread_list.h
@@ -86,17 +86,22 @@ class HwasanThreadList {
   }
 
   Thread *CreateCurrentThread() {
-    Thread *t;
+    Thread *t = nullptr;
     {
-      SpinMutexLock l(&list_mutex_);
+      SpinMutexLock l(&free_list_mutex_);
       if (!free_list_.empty()) {
         t = free_list_.back();
         free_list_.pop_back();
-        uptr start = (uptr)t - ring_buffer_size_;
-        internal_memset((void *)start, 0, ring_buffer_size_ + sizeof(Thread));
-      } else {
-        t = AllocThread();
       }
+    }
+    if (t) {
+      uptr start = (uptr)t - ring_buffer_size_;
+      internal_memset((void *)start, 0, ring_buffer_size_ + sizeof(Thread));
+    } else {
+      t = AllocThread();
+    }
+    {
+      SpinMutexLock l(&live_list_mutex_);
       live_list_.push_back(t);
     }
     t->Init((uptr)t - ring_buffer_size_, ring_buffer_size_);
@@ -110,6 +115,7 @@ class HwasanThreadList {
   }
 
   void RemoveThreadFromLiveList(Thread *t) {
+    SpinMutexLock l(&live_list_mutex_);
     for (Thread *&t2 : live_list_)
       if (t2 == t) {
         // To remove t2, copy the last element of the list in t2's position, and
@@ -124,10 +130,10 @@ class HwasanThreadList {
   void ReleaseThread(Thread *t) {
     RemoveThreadStats(t);
     t->Destroy();
-    SpinMutexLock l(&list_mutex_);
+    DontNeedThread(t);
     RemoveThreadFromLiveList(t);
+    SpinMutexLock l(&free_list_mutex_);
     free_list_.push_back(t);
-    DontNeedThread(t);
   }
 
   Thread *GetThreadByBufferAddress(uptr p) {
@@ -144,7 +150,7 @@ class HwasanThreadList {
 
   template <class CB>
   void VisitAllLiveThreads(CB cb) {
-    SpinMutexLock l(&list_mutex_);
+    SpinMutexLock l(&live_list_mutex_);
     for (Thread *t : live_list_) cb(t);
   }
 
@@ -180,9 +186,10 @@ class HwasanThreadList {
   uptr ring_buffer_size_;
   uptr thread_alloc_size_;
 
+  SpinMutex free_list_mutex_;
   InternalMmapVector<Thread *> free_list_;
+  SpinMutex live_list_mutex_;
   InternalMmapVector<Thread *> live_list_;
-  SpinMutex list_mutex_;
 
   ThreadStats stats_;
   SpinMutex stats_mutex_;