[compiler-rt] [scudo] Change tests that use setrlimit to cause mmap to fail. (PR #87004)

[Christopher Ferris via llvm-commits]

https://github.com/cferris1000 created https://github.com/llvm/llvm-project/pull/87004

It appears that qemu does not actually cause mmap to fail when calling setrlimit to limit the address space size. In the two tests that use setrlimit, detect if mmap still works and skip the tests in that case.

Since all Android targets should support setrlimit, compile out the mmap check code for them.

>From 52c6145803384fd3546568b41e7e6f4038510eb8 Mon Sep 17 00:00:00 2001
From: Christopher Ferris <cferris at google.com>
Date: Thu, 28 Mar 2024 13:12:51 -0700
Subject: [PATCH] [scudo] Change tests that use setrlimit to cause mmap to
 fail.

It appears that qemu does not actually cause mmap to fail when
calling setrlimit to limit the address space size. In the two
tests that use setrlimit, detect if mmap still works and skip
the tests in that case.

Since all Android targets should support setrlimit, compile out
the mmap check code for them.
---
 .../lib/scudo/standalone/tests/strings_test.cpp     | 13 +++++++++++++
 .../lib/scudo/standalone/tests/vector_test.cpp      | 11 +++++++++++
 2 files changed, 24 insertions(+)

diff --git a/compiler-rt/lib/scudo/standalone/tests/strings_test.cpp b/compiler-rt/lib/scudo/standalone/tests/strings_test.cpp
index 17a596d712d0ca..c28ee7287aba14 100644
--- a/compiler-rt/lib/scudo/standalone/tests/strings_test.cpp
+++ b/compiler-rt/lib/scudo/standalone/tests/strings_test.cpp
@@ -129,6 +129,8 @@ TEST(ScudoStringsTest, Padding) {
 }
 
 #if defined(__linux__)
+
+#include <sys/mman.h>
 #include <sys/resource.h>
 
 TEST(ScudoStringsTest, CapacityIncreaseFails) {
@@ -136,9 +138,20 @@ TEST(ScudoStringsTest, CapacityIncreaseFails) {
 
   rlimit Limit = {};
   EXPECT_EQ(0, getrlimit(RLIMIT_AS, &Limit));
+
   rlimit EmptyLimit = {.rlim_cur = 0, .rlim_max = Limit.rlim_max};
   EXPECT_EQ(0, setrlimit(RLIMIT_AS, &EmptyLimit));
 
+#if !SCUDO_ANDROID
+  // qemu does not honor the setrlimit, so verify before proceeding.
+  void *ptr = mmap(nullptr, 100, PROT_READ | PROT_WRITE, MAP_ANONYMOUS | MAP_PRIVATE, -1, 0);
+  if (ptr != MAP_FAILED) {
+    munmap(ptr, 100);
+    setrlimit(RLIMIT_AS, &Limit);
+    GTEST_SKIP() << "Limiting address space does not prevent mmap.";
+  }
+#endif
+
   // Test requires that the default length is at least 6 characters.
   scudo::uptr MaxSize = Str.capacity();
   EXPECT_LE(6u, MaxSize);
diff --git a/compiler-rt/lib/scudo/standalone/tests/vector_test.cpp b/compiler-rt/lib/scudo/standalone/tests/vector_test.cpp
index add62c5a42a3e4..e65c5d6cc2759e 100644
--- a/compiler-rt/lib/scudo/standalone/tests/vector_test.cpp
+++ b/compiler-rt/lib/scudo/standalone/tests/vector_test.cpp
@@ -44,6 +44,7 @@ TEST(ScudoVectorTest, ResizeReduction) {
 
 #if defined(__linux__)
 
+#include <sys/mman.h>
 #include <sys/resource.h>
 
 // Verify that if the reallocate fails, nothing new is added.
@@ -58,6 +59,16 @@ TEST(ScudoVectorTest, ReallocateFails) {
   rlimit EmptyLimit = {.rlim_cur = 0, .rlim_max = Limit.rlim_max};
   EXPECT_EQ(0, setrlimit(RLIMIT_AS, &EmptyLimit));
 
+#if !SCUDO_ANDROID
+  // qemu does not honor the setrlimit, so verify before proceeding.
+  void *ptr = mmap(nullptr, 100, PROT_READ | PROT_WRITE, MAP_ANONYMOUS | MAP_PRIVATE, -1, 0);
+  if (ptr != MAP_FAILED) {
+    munmap(ptr, 100);
+    setrlimit(RLIMIT_AS, &Limit);
+    GTEST_SKIP() << "Limiting address space does not prevent mmap.";
+  }
+#endif
+
   V.resize(capacity);
   // Set the last element so we can check it later.
   V.back() = '\0';