[compiler-rt] r201664 - [asan] Improve stack overflow detection.

[Evgeniy Stepanov]

Author: eugenis
Date: Wed Feb 19 05:02:46 2014
New Revision: 201664

URL: http://llvm.org/viewvc/llvm-project?rev=201664&view=rev
Log:
[asan] Improve stack overflow detection.

There are more cases when those manifest as an access below SP.

Modified:
    compiler-rt/trunk/lib/asan/asan_report.cc
    compiler-rt/trunk/test/asan/TestCases/stack-overflow.cc

Modified: compiler-rt/trunk/lib/asan/asan_report.cc
URL: http://llvm.org/viewvc/llvm-project/compiler-rt/trunk/lib/asan/asan_report.cc?rev=201664&r1=201663&r2=201664&view=diff
==============================================================================
--- compiler-rt/trunk/lib/asan/asan_report.cc (original)
+++ compiler-rt/trunk/lib/asan/asan_report.cc Wed Feb 19 05:02:46 2014
@@ -570,14 +570,12 @@ class ScopedInErrorReport {
 
 static bool IsStackOverflow(uptr addr, uptr sp) {
   uptr stack_frame_bottom = sp;
-#ifdef __x86_64__
-  stack_frame_bottom -= 128; // x86_64 stack redzone
-#else
-  // call stores return value 1 word below SP.
-  stack_frame_bottom -= sizeof(uptr);
-#endif
-  // Access below sp (+ redzone on x86_64) is probably something else (like
-  // stack of another thread).
+  // x86_64 stack redzone: leaf functions can access up to 128 bytes below SP.
+  // ARM has push-multiple instruction that stores up to 64(?) bytes below SP.
+  stack_frame_bottom -= 128;
+
+  // Access below SP (minus redzone) is probably something else (like stack of
+  // another thread).
   if (addr < stack_frame_bottom)
     return false;
 

Modified: compiler-rt/trunk/test/asan/TestCases/stack-overflow.cc
URL: http://llvm.org/viewvc/llvm-project/compiler-rt/trunk/test/asan/TestCases/stack-overflow.cc?rev=201664&r1=201663&r2=201664&view=diff
==============================================================================
--- compiler-rt/trunk/test/asan/TestCases/stack-overflow.cc (original)
+++ compiler-rt/trunk/test/asan/TestCases/stack-overflow.cc Wed Feb 19 05:02:46 2014
@@ -1,6 +1,17 @@
+// Test ASan detection of stack-overflow condition.
+
+// RUN: %clangxx_asan -O0 %s -DSMALL_FRAME -o %t && ASAN_OPTIONS=use_sigaltstack=1 not %t 2>&1 | FileCheck %s
+// RUN: %clangxx_asan -O3 %s -DSMALL_FRAME -o %t && ASAN_OPTIONS=use_sigaltstack=1 not %t 2>&1 | FileCheck %s
+// RUN: %clangxx_asan -O0 %s -DSAVE_ALL_THE_REGISTERS -o %t && ASAN_OPTIONS=use_sigaltstack=1 not %t 2>&1 | FileCheck %s
+// RUN: %clangxx_asan -O3 %s -DSAVE_ALL_THE_REGISTERS -o %t && ASAN_OPTIONS=use_sigaltstack=1 not %t 2>&1 | FileCheck %s
+// RUN: %clangxx_asan -O0 %s -o %t && ASAN_OPTIONS=use_sigaltstack=1 not %t 2>&1 | FileCheck %s
+// RUN: %clangxx_asan -O3 %s -o %t && ASAN_OPTIONS=use_sigaltstack=1 not %t 2>&1 | FileCheck %s
+
+// RUN: %clangxx_asan -O0 %s -DTHREAD -DSMALL_FRAME -o %t && ASAN_OPTIONS=use_sigaltstack=1 not %t 2>&1 | FileCheck %s
+// RUN: %clangxx_asan -O3 %s -DTHREAD -DSMALL_FRAME -o %t && ASAN_OPTIONS=use_sigaltstack=1 not %t 2>&1 | FileCheck %s
+// RUN: %clangxx_asan -O0 %s -DTHREAD -DSAVE_ALL_THE_REGISTERS -o %t && ASAN_OPTIONS=use_sigaltstack=1 not %t 2>&1 | FileCheck %s
+// RUN: %clangxx_asan -O3 %s -DTHREAD -DSAVE_ALL_THE_REGISTERS -o %t && ASAN_OPTIONS=use_sigaltstack=1 not %t 2>&1 | FileCheck %s
 // RUN: %clangxx_asan -O0 %s -DTHREAD -o %t && ASAN_OPTIONS=use_sigaltstack=1 not %t 2>&1 | FileCheck %s
-// RUN: %clangxx_asan -O1 %s -DTHREAD -o %t && ASAN_OPTIONS=use_sigaltstack=1 not %t 2>&1 | FileCheck %s
-// RUN: %clangxx_asan -O2 %s -DTHREAD -o %t && ASAN_OPTIONS=use_sigaltstack=1 not %t 2>&1 | FileCheck %s
 // RUN: %clangxx_asan -O3 %s -DTHREAD -o %t && ASAN_OPTIONS=use_sigaltstack=1 not %t 2>&1 | FileCheck %s
 
 #include <assert.h>
@@ -9,24 +20,63 @@
 
 const int BS = 1024;
 volatile char x;
+volatile int y = 1;
+volatile int z0, z1, z2, z3, z4, z5, z6, z7, z8, z9, z10, z11, z12, z13;
+
+void recursive_func(char *p) {
+#if defined(SMALL_FRAME)
+  char *buf = 0;
+#elif defined(SAVE_ALL_THE_REGISTERS)
+  char *buf = 0;
+  int t0, t1, t2, t3, t4, t5, t6, t7, t8, t9, t10, t11, t12, t13;
+  t0 = z0;
+  t1 = z1;
+  t2 = z2;
+  t3 = z3;
+  t4 = z4;
+  t5 = z5;
+  t6 = z6;
+  t7 = z7;
+  t8 = z8;
+  t9 = z9;
+  t10 = z10;
+  t11 = z11;
+  t12 = z12;
+  t13 = z13;
 
-void large_frame_func(char *p, int level) {
+  z0 = t0;
+  z1 = t1;
+  z2 = t2;
+  z3 = t3;
+  z4 = t4;
+  z5 = t5;
+  z6 = t6;
+  z7 = t7;
+  z8 = t8;
+  z9 = t9;
+  z10 = t10;
+  z11 = t11;
+  z12 = t12;
+  z13 = t13;
+#else
   char buf[BS];
   if (p)
     assert(p - buf >= BS);
   buf[rand() % BS] = 1;
   buf[rand() % BS] = 2;
   x = buf[rand() % BS];
-  volatile int y = 1;
+#endif
   if (y)
-    large_frame_func(buf, level + 1);
+    recursive_func(buf);
+  x = 1; // prevent tail call optimization
   // CHECK: {{stack-overflow on address 0x.* \(pc 0x.* sp 0x.* bp 0x.* T.*\)}}
-  // Frame 0 may be anywhere (in rand(), for example).
-  // CHECK: {{    #. 0x.* in large_frame_func.*stack-overflow.cc:}}[[@LINE-3]]
+  // If stack overflow happens during function prologue, stack trace may be
+  // corrupted. Unwind tables are not always 100% exact there.
+  // For this reason, we don't do any further checks.
 }
 
 void *ThreadFn(void* unused) {
-  large_frame_func(0, 0);
+  recursive_func(0);
   return 0;
 }
 
@@ -36,7 +86,7 @@ int main(int argc, char **argv) {
   pthread_create(&t, 0, ThreadFn, 0);
   pthread_join(t, 0);
 #else
-  large_frame_func(0, 0);
+  recursive_func(0);
 #endif
   return 0;
 }