[compiler-rt] [compiler-rt][nsan] Add two steps in init process (PR #98807)

[Dmitriy Chestnykh via llvm-commits]

https://github.com/chestnykh updated https://github.com/llvm/llvm-project/pull/98807

>From 05494145465cc1307dcd19d029b8cd94ffd4c5b6 Mon Sep 17 00:00:00 2001
From: Dmitry Chestnykh <dm.chestnykh at gmail.com>
Date: Sun, 14 Jul 2024 13:05:02 +0300
Subject: [PATCH 1/2] [compiler-rt][nsan] Add two steps in init process

- Install deadly signal handlers to print an informative
report if something like SIGSEGV was happened
- Disable coredump creation
If NSAN_OPTIONS contains `abort_on_error=1` the process
hangs because the kernel tries to create a very huge core

Fix #98806
---
 compiler-rt/lib/nsan/nsan.cpp | 18 +++++++++++++++++-
 1 file changed, 17 insertions(+), 1 deletion(-)

diff --git a/compiler-rt/lib/nsan/nsan.cpp b/compiler-rt/lib/nsan/nsan.cpp
index 7a5f013579dfb..efcbf6a1bedb4 100644
--- a/compiler-rt/lib/nsan/nsan.cpp
+++ b/compiler-rt/lib/nsan/nsan.cpp
@@ -535,7 +535,10 @@ int32_t checkFT(const FT value, ShadowFT Shadow, CheckTypeT CheckType,
   }
 
   if (flags().halt_on_error) {
-    Printf("Exiting\n");
+    if (common_flags()->abort_on_error)
+      Printf("ABORTING\n");
+    else
+      Printf("Exiting\n");
     Die();
   }
   return flags().resume_after_warning ? kResumeFromValue : kContinueWithShadow;
@@ -776,6 +779,16 @@ extern "C" SANITIZER_INTERFACE_ATTRIBUTE void __nsan_dump_shadow_args() {
   printf("args tag: %lx\n", __nsan_shadow_args_tag);
 }
 
+static void OnStackUnwind(const SignalContext &sig, const void *,
+                          BufferedStackTrace *stack) {
+  stack->Unwind(StackTrace::GetNextInstructionPc(sig.pc), sig.bp, sig.context,
+                common_flags()->fast_unwind_on_fatal);
+}
+
+static void NsanOnDeadlySignal(int signo, void *siginfo, void *context) {
+  HandleDeadlySignal(siginfo, context, GetTid(), &OnStackUnwind, nullptr);
+}
+
 bool __nsan::nsan_initialized;
 bool __nsan::nsan_init_is_running;
 
@@ -789,6 +802,9 @@ extern "C" SANITIZER_INTERFACE_ATTRIBUTE void __nsan_init() {
   InitializeSuppressions();
   InitializePlatformEarly();
 
+  DisableCoreDumperIfNecessary();
+  InstallDeadlySignalHandlers(NsanOnDeadlySignal);
+
   if (!MmapFixedNoReserve(TypesAddr(), UnusedAddr() - TypesAddr()))
     Die();
 

>From 78d5eb44e48a47f9863709dfe624164e6733929c Mon Sep 17 00:00:00 2001
From: Dmitry Chestnykh <dm.chestnykh at gmail.com>
Date: Mon, 15 Jul 2024 07:13:15 +0300
Subject: [PATCH 2/2] [compiler-rt][nsan] Don't install signal handler

---
 compiler-rt/lib/nsan/nsan.cpp | 11 -----------
 1 file changed, 11 deletions(-)

diff --git a/compiler-rt/lib/nsan/nsan.cpp b/compiler-rt/lib/nsan/nsan.cpp
index efcbf6a1bedb4..0b7bbc9ed5991 100644
--- a/compiler-rt/lib/nsan/nsan.cpp
+++ b/compiler-rt/lib/nsan/nsan.cpp
@@ -779,16 +779,6 @@ extern "C" SANITIZER_INTERFACE_ATTRIBUTE void __nsan_dump_shadow_args() {
   printf("args tag: %lx\n", __nsan_shadow_args_tag);
 }
 
-static void OnStackUnwind(const SignalContext &sig, const void *,
-                          BufferedStackTrace *stack) {
-  stack->Unwind(StackTrace::GetNextInstructionPc(sig.pc), sig.bp, sig.context,
-                common_flags()->fast_unwind_on_fatal);
-}
-
-static void NsanOnDeadlySignal(int signo, void *siginfo, void *context) {
-  HandleDeadlySignal(siginfo, context, GetTid(), &OnStackUnwind, nullptr);
-}
-
 bool __nsan::nsan_initialized;
 bool __nsan::nsan_init_is_running;
 
@@ -803,7 +793,6 @@ extern "C" SANITIZER_INTERFACE_ATTRIBUTE void __nsan_init() {
   InitializePlatformEarly();
 
   DisableCoreDumperIfNecessary();
-  InstallDeadlySignalHandlers(NsanOnDeadlySignal);
 
   if (!MmapFixedNoReserve(TypesAddr(), UnusedAddr() - TypesAddr()))
     Die();