[llvm-commits] [compiler-rt] r165214 - in /compiler-rt/trunk/lib: asan/lit_tests/Linux/rlimit_mmap_test.cc sanitizer_common/sanitizer_posix.cc

[Kostya Serebryany]

Author: kcc
Date: Thu Oct  4 02:21:09 2012
New Revision: 165214

URL: http://llvm.org/viewvc/llvm-project?rev=165214&view=rev
Log:
[asan] properly report mmap failure

Added:
    compiler-rt/trunk/lib/asan/lit_tests/Linux/rlimit_mmap_test.cc
Modified:
    compiler-rt/trunk/lib/sanitizer_common/sanitizer_posix.cc

Added: compiler-rt/trunk/lib/asan/lit_tests/Linux/rlimit_mmap_test.cc
URL: http://llvm.org/viewvc/llvm-project/compiler-rt/trunk/lib/asan/lit_tests/Linux/rlimit_mmap_test.cc?rev=165214&view=auto
==============================================================================
--- compiler-rt/trunk/lib/asan/lit_tests/Linux/rlimit_mmap_test.cc (added)
+++ compiler-rt/trunk/lib/asan/lit_tests/Linux/rlimit_mmap_test.cc Thu Oct  4 02:21:09 2012
@@ -0,0 +1,16 @@
+// Check that we properly report mmap failure.
+// RUN: %clangxx_asan %s -o %t && %t 2>&1 | FileCheck %s
+#include <stdlib.h>
+#include <assert.h>
+#include <sys/time.h>
+#include <sys/resource.h>
+
+static volatile void *x;
+
+int main(int argc, char **argv) {
+  struct rlimit mmap_resource_limit = { 0, 0 };
+  assert(0 == setrlimit(RLIMIT_AS, &mmap_resource_limit));
+  x = malloc(10000000);
+// CHECK: AddressSanitizer is unable to mmap
+  return 0;
+}

Modified: compiler-rt/trunk/lib/sanitizer_common/sanitizer_posix.cc
URL: http://llvm.org/viewvc/llvm-project/compiler-rt/trunk/lib/sanitizer_common/sanitizer_posix.cc?rev=165214&r1=165213&r2=165214&view=diff
==============================================================================
--- compiler-rt/trunk/lib/sanitizer_common/sanitizer_posix.cc (original)
+++ compiler-rt/trunk/lib/sanitizer_common/sanitizer_posix.cc Thu Oct  4 02:21:09 2012
@@ -47,6 +47,14 @@
                             PROT_READ | PROT_WRITE,
                             MAP_PRIVATE | MAP_ANON, -1, 0);
   if (res == (void*)-1) {
+    static int recursion_count;
+    if (recursion_count) {
+      // The Report() and CHECK calls below may call mmap recursively and fail.
+      // If we went into recursion, just die.
+      RawWrite("AddressSanitizer is unable to mmap\n");
+      Die();
+    }
+    recursion_count++;
     Report("ERROR: Failed to allocate 0x%zx (%zd) bytes of %s: %s\n",
            size, size, mem_type, strerror(errno));
     DumpProcessMap();