[llvm-commits] [compiler-rt] r173443 - in /compiler-rt/trunk/lib: asan/tests/asan_noinst_test.cc sanitizer_common/sanitizer_allocator.cc

[Kostya Serebryany]

Author: kcc
Date: Fri Jan 25 06:22:21 2013
New Revision: 173443

URL: http://llvm.org/viewvc/llvm-project?rev=173443&view=rev
Log:
[sanitizer] improve the calloc overflow check (spotted by samsonov@)

Modified:
    compiler-rt/trunk/lib/asan/tests/asan_noinst_test.cc
    compiler-rt/trunk/lib/sanitizer_common/sanitizer_allocator.cc

Modified: compiler-rt/trunk/lib/asan/tests/asan_noinst_test.cc
URL: http://llvm.org/viewvc/llvm-project/compiler-rt/trunk/lib/asan/tests/asan_noinst_test.cc?rev=173443&r1=173442&r2=173443&view=diff
==============================================================================
--- compiler-rt/trunk/lib/asan/tests/asan_noinst_test.cc (original)
+++ compiler-rt/trunk/lib/asan/tests/asan_noinst_test.cc Fri Jan 25 06:22:21 2013
@@ -840,3 +840,12 @@
   void *p = calloc(kArraySize, kArraySize2);  // Should return 0.
   EXPECT_EQ(0L, Ident(p));
 }
+
+TEST(AddressSanitizerInterface, CallocOverflow2) {
+#if SANITIZER_WORDSIZE == 32
+  size_t kArraySize = 112;
+  volatile size_t kArraySize2 = 43878406;
+  void *p = calloc(kArraySize, kArraySize2);  // Should return 0.
+  EXPECT_EQ(0L, Ident(p));
+#endif
+}

Modified: compiler-rt/trunk/lib/sanitizer_common/sanitizer_allocator.cc
URL: http://llvm.org/viewvc/llvm-project/compiler-rt/trunk/lib/sanitizer_common/sanitizer_allocator.cc?rev=173443&r1=173442&r2=173443&view=diff
==============================================================================
--- compiler-rt/trunk/lib/sanitizer_common/sanitizer_allocator.cc (original)
+++ compiler-rt/trunk/lib/sanitizer_common/sanitizer_allocator.cc Fri Jan 25 06:22:21 2013
@@ -76,8 +76,9 @@
 }
 
 bool CallocShouldReturnNullDueToOverflow(uptr size, uptr n) {
-  uptr mul = size * n;
-  return mul < size || mul < n;
+  if (!size) return false;
+  uptr max = (uptr)-1L;
+  return (max / size) < n;
 }
 
 }  // namespace __sanitizer