[compiler-rt] r177741 - tsan: return 0 on malloc() failure instead of crashing

[Dmitry Vyukov]

Author: dvyukov
Date: Fri Mar 22 12:06:22 2013
New Revision: 177741

URL: http://llvm.org/viewvc/llvm-project?rev=177741&view=rev
Log:
tsan: return 0 on malloc() failure instead of crashing


Added:
    compiler-rt/trunk/lib/tsan/lit_tests/malloc_overflow.cc
Modified:
    compiler-rt/trunk/lib/tsan/rtl/tsan_interceptors.cc
    compiler-rt/trunk/lib/tsan/rtl/tsan_mman.cc

Added: compiler-rt/trunk/lib/tsan/lit_tests/malloc_overflow.cc
URL: http://llvm.org/viewvc/llvm-project/compiler-rt/trunk/lib/tsan/lit_tests/malloc_overflow.cc?rev=177741&view=auto
==============================================================================
--- compiler-rt/trunk/lib/tsan/lit_tests/malloc_overflow.cc (added)
+++ compiler-rt/trunk/lib/tsan/lit_tests/malloc_overflow.cc Fri Mar 22 12:06:22 2013
@@ -0,0 +1,22 @@
+// RUN: %clangxx_tsan -O1 %s -o %t && %t 2>&1 | FileCheck %s
+#include <stdio.h>
+#include <stdlib.h>
+
+int main() {
+  void *p = malloc((size_t)-1);
+  if (p != 0)
+    printf("FAIL malloc(-1) = %p\n", p);
+  p = malloc((size_t)-1 / 2);
+  if (p != 0)
+    printf("FAIL malloc(-1/2) = %p\n", p);
+  p = calloc((size_t)-1, (size_t)-1);
+  if (p != 0)
+    printf("FAIL calloc(-1, -1) = %p\n", p);
+  p = calloc((size_t)-1 / 2, (size_t)-1 / 2);
+  if (p != 0)
+    printf("FAIL calloc(-1/2, -1/2) = %p\n", p);
+  printf("OK\n");
+}
+
+// CHECK-NOT: FAIL
+// CHECK-NOT: failed to allocate

Modified: compiler-rt/trunk/lib/tsan/rtl/tsan_interceptors.cc
URL: http://llvm.org/viewvc/llvm-project/compiler-rt/trunk/lib/tsan/rtl/tsan_interceptors.cc?rev=177741&r1=177740&r2=177741&view=diff
==============================================================================
--- compiler-rt/trunk/lib/tsan/rtl/tsan_interceptors.cc (original)
+++ compiler-rt/trunk/lib/tsan/rtl/tsan_interceptors.cc Fri Mar 22 12:06:22 2013
@@ -356,7 +356,8 @@ TSAN_INTERCEPTOR(void*, calloc, uptr siz
   {
     SCOPED_INTERCEPTOR_RAW(calloc, size, n);
     p = user_alloc(thr, pc, n * size);
-    if (p) internal_memset(p, 0, n * size);
+    if (p)
+      internal_memset(p, 0, n * size);
   }
   invoke_malloc_hook(p, n * size);
   return p;

Modified: compiler-rt/trunk/lib/tsan/rtl/tsan_mman.cc
URL: http://llvm.org/viewvc/llvm-project/compiler-rt/trunk/lib/tsan/rtl/tsan_mman.cc?rev=177741&r1=177740&r2=177741&view=diff
==============================================================================
--- compiler-rt/trunk/lib/tsan/rtl/tsan_mman.cc (original)
+++ compiler-rt/trunk/lib/tsan/rtl/tsan_mman.cc Fri Mar 22 12:06:22 2013
@@ -101,6 +101,8 @@ static void SignalUnsafeCall(ThreadState
 
 void *user_alloc(ThreadState *thr, uptr pc, uptr sz, uptr align) {
   CHECK_GT(thr->in_rtl, 0);
+  if ((sz >= (1ull << 40)) || (align >= (1ull << 40)))
+    return 0;
   void *p = allocator()->Allocate(&thr->alloc_cache, sz, align);
   if (p == 0)
     return 0;