[compiler-rt] [compiler-rt] Avoid generating coredumps when piped to a tool (PR #83701)

[Alexander Richardson via llvm-commits]

https://github.com/arichardson updated https://github.com/llvm/llvm-project/pull/83701

>From 8aabfa9dc6cbd38ad53e06f992e557ef9540a447 Mon Sep 17 00:00:00 2001
From: Alex Richardson <alexrichardson at google.com>
Date: Sat, 2 Mar 2024 19:05:55 -0800
Subject: [PATCH 1/2] =?UTF-8?q?[=F0=9D=98=80=F0=9D=97=BD=F0=9D=97=BF]=20in?=
 =?UTF-8?q?itial=20version?=
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

Created using spr 1.3.4
---
 .../sanitizer_posix_libcdep.cpp               | 21 ++++++++++++++++++-
 1 file changed, 20 insertions(+), 1 deletion(-)

diff --git a/compiler-rt/lib/sanitizer_common/sanitizer_posix_libcdep.cpp b/compiler-rt/lib/sanitizer_common/sanitizer_posix_libcdep.cpp
index e88e654eec5a1c..7472a256870c16 100644
--- a/compiler-rt/lib/sanitizer_common/sanitizer_posix_libcdep.cpp
+++ b/compiler-rt/lib/sanitizer_common/sanitizer_posix_libcdep.cpp
@@ -103,9 +103,28 @@ static void setlim(int res, rlim_t lim) {
 }
 
 void DisableCoreDumperIfNecessary() {
+  rlimit rlim;
+  CHECK_EQ(0, getrlimit(RLIMIT_CORE, &rlim));
   if (common_flags()->disable_coredump) {
-    setlim(RLIMIT_CORE, 0);
+#  ifdef __linux__
+    // On Linux, if the kernel.core_pattern sysctl starts with a '|' (i.e. it
+    // is being piped to a coredump handler such as systemd-coredumpd), the
+    // kernel ignores RLIMIT_CORE (since we aren't creating a file in the file
+    // system) except for the magic value of 1, that disables coredumps when
+    // piping. 1 byte is also too small for any kind of valid core dump, so it
+    // also disables coredumps if kernel.core_pattern creates files directly.
+    rlim.rlim_cur = Min<rlim_t>(1, rlim.rlim_max);
+#  else
+    rlim.rlim_cur = 0;
+#  endif
+  } else {
+    // Set the limit to 1GB to avoid blocking the system for multiple minutes
+    // while dumping all (potentially huge) mappings.
+    // We are only setting the soft limit, so if it's really wanted, users can
+    // override it to generate a massive core dump.
+    rlim.rlim_cur = Min<rlim_t>(1024 * 1024 * 1024, rlim.rlim_max);
   }
+  CHECK_EQ(0, setrlimit(RLIMIT_CORE, &rlim));
 }
 
 bool StackSizeIsUnlimited() {

>From 347dab6d1b39beb2d79a32165be15c8bc057dea5 Mon Sep 17 00:00:00 2001
From: Alex Richardson <alexrichardson at google.com>
Date: Sat, 2 Mar 2024 19:40:18 -0800
Subject: [PATCH 2/2] update test

Created using spr 1.3.4
---
 compiler-rt/test/sanitizer_common/TestCases/corelimit.cpp | 7 ++++++-
 1 file changed, 6 insertions(+), 1 deletion(-)

diff --git a/compiler-rt/test/sanitizer_common/TestCases/corelimit.cpp b/compiler-rt/test/sanitizer_common/TestCases/corelimit.cpp
index 2378a4cfdced12..fed2e1d89cbffa 100644
--- a/compiler-rt/test/sanitizer_common/TestCases/corelimit.cpp
+++ b/compiler-rt/test/sanitizer_common/TestCases/corelimit.cpp
@@ -10,7 +10,12 @@ int main() {
   getrlimit(RLIMIT_CORE, &lim_core);
   void *p;
   if (sizeof(p) == 8) {
-    assert(0 == lim_core.rlim_cur);
+#ifdef __linux__
+    // See comments in DisableCoreDumperIfNecessary().
+    assert(lim_core.rlim_cur == 1);
+#else
+    assert(lim_core.rlim_cur == 0);
+#endif
   }
   return 0;
 }