[compiler-rt] [rtsan][compiler-rt] Prevent UB hang in rtsan lock unit tests (PR #104733)

[Chris Apple via llvm-commits]

https://github.com/cjappl updated https://github.com/llvm/llvm-project/pull/104733

>From 847319a698abd51b2bce0ae67a63821bcc672fa1 Mon Sep 17 00:00:00 2001
From: Chris Apple <cja-private at pm.me>
Date: Sun, 18 Aug 2024 07:19:36 -0700
Subject: [PATCH 1/2] Prevent hang in rwlock tests

---
 .../rtsan/tests/rtsan_test_interceptors.cpp   | 130 ++++++++++++++----
 1 file changed, 107 insertions(+), 23 deletions(-)

diff --git a/compiler-rt/lib/rtsan/tests/rtsan_test_interceptors.cpp b/compiler-rt/lib/rtsan/tests/rtsan_test_interceptors.cpp
index f5b016089087df..966e5514062881 100644
--- a/compiler-rt/lib/rtsan/tests/rtsan_test_interceptors.cpp
+++ b/compiler-rt/lib/rtsan/tests/rtsan_test_interceptors.cpp
@@ -328,26 +328,64 @@ TEST(TestRtsanInterceptors, PthreadCreateDiesWhenRealtime) {
   ExpectNonRealtimeSurvival(Func);
 }
 
-TEST(TestRtsanInterceptors, PthreadMutexLockDiesWhenRealtime) {
-  auto Func = []() {
-    pthread_mutex_t mutex{};
+class PthreadMutexLockTest : public ::testing::Test {
+protected:
+  void SetUp() override {
+    pthread_mutex_init(&mutex, nullptr);
+    is_locked = false;
+  }
+
+  void TearDown() override {
+    if (is_locked) {
+      Unlock();
+    }
+    pthread_mutex_destroy(&mutex);
+  }
+
+  void Lock() {
+    ASSERT_TRUE(!is_locked);
     pthread_mutex_lock(&mutex);
-  };
+    is_locked = true;
+  }
+
+  void Unlock() {
+    ASSERT_TRUE(is_locked);
+    pthread_mutex_unlock(&mutex);
+    is_locked = false;
+  }
+
+private:
+  pthread_mutex_t mutex;
+  bool is_locked;
+};
+
+TEST_F(PthreadMutexLockTest, PthreadMutexLockDiesWhenRealtime) {
+  auto Func = [this]() { Lock(); };
 
   ExpectRealtimeDeath(Func, "pthread_mutex_lock");
+}
+
+TEST_F(PthreadMutexLockTest, PthreadMutexLockSurvivesWhenNotRealtime) {
+  auto Func = [this]() { Lock(); };
+
   ExpectNonRealtimeSurvival(Func);
 }
 
-TEST(TestRtsanInterceptors, PthreadMutexUnlockDiesWhenRealtime) {
-  auto Func = []() {
-    pthread_mutex_t mutex{};
-    pthread_mutex_unlock(&mutex);
-  };
+TEST_F(PthreadMutexLockTest, PthreadMutexUnlockDiesWhenRealtime) {
+  Lock();
+  auto Func = [this]() { Unlock(); };
 
   ExpectRealtimeDeath(Func, "pthread_mutex_unlock");
   ExpectNonRealtimeSurvival(Func);
 }
 
+TEST_F(PthreadMutexLockTest, PthreadMutexUnlockSurvivesWhenNotRealtime) {
+  Lock();
+  auto Func = [this]() { Unlock(); };
+
+  ExpectNonRealtimeSurvival(Func);
+}
+
 TEST(TestRtsanInterceptors, PthreadMutexJoinDiesWhenRealtime) {
   auto Func = []() {
     pthread_t thread{};
@@ -431,30 +469,76 @@ TEST(TestRtsanInterceptors, PthreadCondWaitDiesWhenRealtime) {
   pthread_mutex_destroy(&mutex);
 }
 
-TEST(TestRtsanInterceptors, PthreadRwlockRdlockDiesWhenRealtime) {
-  auto Func = []() {
-    pthread_rwlock_t rw_lock;
+class PthreadRwlockTest : public ::testing::Test {
+protected:
+  void SetUp() override {
+    pthread_rwlock_init(&rw_lock, nullptr);
+    is_locked = false;
+  }
+
+  void TearDown() override {
+    if (is_locked) {
+      Unlock();
+    }
+    pthread_rwlock_destroy(&rw_lock);
+  }
+
+  void RdLock() {
+    ASSERT_TRUE(!is_locked);
     pthread_rwlock_rdlock(&rw_lock);
-  };
+    is_locked = true;
+  }
+
+  void WrLock() {
+    ASSERT_TRUE(!is_locked);
+    pthread_rwlock_wrlock(&rw_lock);
+    is_locked = true;
+  }
+
+  void Unlock() {
+    ASSERT_TRUE(is_locked);
+    pthread_rwlock_unlock(&rw_lock);
+    is_locked = false;
+  }
+
+private:
+  pthread_rwlock_t rw_lock;
+  bool is_locked;
+};
+
+TEST_F(PthreadRwlockTest, PthreadRwlockRdlockDiesWhenRealtime) {
+  auto Func = [this]() { RdLock(); };
   ExpectRealtimeDeath(Func, "pthread_rwlock_rdlock");
+}
+
+TEST_F(PthreadRwlockTest, PthreadRwlockRdlockSurvivesWhenNonRealtime) {
+  auto Func = [this]() { RdLock(); };
   ExpectNonRealtimeSurvival(Func);
 }
 
-TEST(TestRtsanInterceptors, PthreadRwlockUnlockDiesWhenRealtime) {
-  auto Func = []() {
-    pthread_rwlock_t rw_lock;
-    pthread_rwlock_unlock(&rw_lock);
-  };
+TEST_F(PthreadRwlockTest, PthreadRwlockUnlockDiesWhenRealtime) {
+  RdLock();
+
+  auto Func = [this]() { Unlock(); };
   ExpectRealtimeDeath(Func, "pthread_rwlock_unlock");
+}
+
+TEST_F(PthreadRwlockTest, PthreadRwlockUnlockSurvivesWhenNonRealtime) {
+  RdLock();
+
+  auto Func = [this]() { Unlock(); };
   ExpectNonRealtimeSurvival(Func);
 }
 
-TEST(TestRtsanInterceptors, PthreadRwlockWrlockDiesWhenRealtime) {
-  auto Func = []() {
-    pthread_rwlock_t rw_lock;
-    pthread_rwlock_wrlock(&rw_lock);
-  };
+TEST_F(PthreadRwlockTest, PthreadRwlockWrlockDiesWhenRealtime) {
+  auto Func = [this]() { WrLock(); };
+
   ExpectRealtimeDeath(Func, "pthread_rwlock_wrlock");
+}
+
+TEST_F(PthreadRwlockTest, PthreadRwlockWrlockSurvivesWhenNonRealtime) {
+  auto Func = [this]() { WrLock(); };
+
   ExpectNonRealtimeSurvival(Func);
 }
 

>From 0911cba2bf8223f5ba998d883490958485c73635 Mon Sep 17 00:00:00 2001
From: Chris Apple <cja-private at pm.me>
Date: Tue, 20 Aug 2024 07:15:04 -0700
Subject: [PATCH 2/2] [PR] Maskray - remove unnecessary braces

---
 compiler-rt/lib/rtsan/tests/rtsan_test_interceptors.cpp | 8 ++++----
 1 file changed, 4 insertions(+), 4 deletions(-)

diff --git a/compiler-rt/lib/rtsan/tests/rtsan_test_interceptors.cpp b/compiler-rt/lib/rtsan/tests/rtsan_test_interceptors.cpp
index 966e5514062881..8861104068c8e9 100644
--- a/compiler-rt/lib/rtsan/tests/rtsan_test_interceptors.cpp
+++ b/compiler-rt/lib/rtsan/tests/rtsan_test_interceptors.cpp
@@ -336,9 +336,9 @@ class PthreadMutexLockTest : public ::testing::Test {
   }
 
   void TearDown() override {
-    if (is_locked) {
+    if (is_locked)
       Unlock();
-    }
+
     pthread_mutex_destroy(&mutex);
   }
 
@@ -477,9 +477,9 @@ class PthreadRwlockTest : public ::testing::Test {
   }
 
   void TearDown() override {
-    if (is_locked) {
+    if (is_locked)
       Unlock();
-    }
+
     pthread_rwlock_destroy(&rw_lock);
   }