[llvm] r288281 - [libFuzzer] extend -rss_limit_mb to crash instantly on a single malloc that exceeds the limit

[Kostya Serebryany via llvm-commits]

Author: kcc
Date: Wed Nov 30 16:39:35 2016
New Revision: 288281

URL: http://llvm.org/viewvc/llvm-project?rev=288281&view=rev
Log:
[libFuzzer] extend -rss_limit_mb to crash instantly on a single malloc that exceeds the limit

Added:
    llvm/trunk/lib/Fuzzer/test/OutOfMemorySingleLargeMallocTest.cpp
Modified:
    llvm/trunk/lib/Fuzzer/FuzzerInternal.h
    llvm/trunk/lib/Fuzzer/FuzzerLoop.cpp
    llvm/trunk/lib/Fuzzer/test/CMakeLists.txt
    llvm/trunk/lib/Fuzzer/test/fuzzer-oom-with-profile.test
    llvm/trunk/lib/Fuzzer/test/fuzzer-oom.test

Modified: llvm/trunk/lib/Fuzzer/FuzzerInternal.h
URL: http://llvm.org/viewvc/llvm-project/llvm/trunk/lib/Fuzzer/FuzzerInternal.h?rev=288281&r1=288280&r2=288281&view=diff
==============================================================================
--- llvm/trunk/lib/Fuzzer/FuzzerInternal.h (original)
+++ llvm/trunk/lib/Fuzzer/FuzzerInternal.h Wed Nov 30 16:39:35 2016
@@ -105,6 +105,8 @@ public:
   void TryDetectingAMemoryLeak(const uint8_t *Data, size_t Size,
                                bool DuringInitialCorpusExecution);
 
+  void HandleMalloc(size_t Size);
+
 private:
   void AlarmCallback();
   void CrashCallback();

Modified: llvm/trunk/lib/Fuzzer/FuzzerLoop.cpp
URL: http://llvm.org/viewvc/llvm-project/llvm/trunk/lib/Fuzzer/FuzzerLoop.cpp?rev=288281&r1=288280&r2=288281&view=diff
==============================================================================
--- llvm/trunk/lib/Fuzzer/FuzzerLoop.cpp (original)
+++ llvm/trunk/lib/Fuzzer/FuzzerLoop.cpp Wed Nov 30 16:39:35 2016
@@ -140,6 +140,7 @@ static MallocFreeTracer AllocTracer;
 
 void MallocHook(const volatile void *ptr, size_t size) {
   size_t N = AllocTracer.Mallocs++;
+  F->HandleMalloc(size);
   if (int TraceLevel = AllocTracer.TraceLevel) {
     Printf("MALLOC[%zd] %p %zd\n", N, ptr, size);
     if (TraceLevel >= 2 && EF)
@@ -155,6 +156,21 @@ void FreeHook(const volatile void *ptr)
   }
 }
 
+// Crash on a single malloc that exceeds the rss limit.
+void Fuzzer::HandleMalloc(size_t Size) {
+  if ((Size >> 20) < (size_t)Options.RssLimitMb)
+    return;
+  Printf("==%d== ERROR: libFuzzer: out-of-memory (malloc(%zd))\n", GetPid(),
+         Size);
+  Printf("   To change the out-of-memory limit use -rss_limit_mb=<N>\n\n");
+  if (EF->__sanitizer_print_stack_trace)
+    EF->__sanitizer_print_stack_trace();
+  DumpCurrentUnit("oom-");
+  Printf("SUMMARY: libFuzzer: out-of-memory\n");
+  PrintFinalStats();
+  _Exit(Options.ErrorExitCode); // Stop right now.
+}
+
 Fuzzer::Fuzzer(UserCallback CB, InputCorpus &Corpus, MutationDispatcher &MD,
                FuzzingOptions Options)
     : CB(CB), Corpus(Corpus), MD(MD), Options(Options) {

Modified: llvm/trunk/lib/Fuzzer/test/CMakeLists.txt
URL: http://llvm.org/viewvc/llvm-project/llvm/trunk/lib/Fuzzer/test/CMakeLists.txt?rev=288281&r1=288280&r2=288281&view=diff
==============================================================================
--- llvm/trunk/lib/Fuzzer/test/CMakeLists.txt (original)
+++ llvm/trunk/lib/Fuzzer/test/CMakeLists.txt Wed Nov 30 16:39:35 2016
@@ -85,6 +85,7 @@ set(Tests
   NthRunCrashTest
   OneHugeAllocTest
   OutOfMemoryTest
+  OutOfMemorySingleLargeMallocTest
   RepeatedMemcmp
   RepeatedBytesTest
   SimpleCmpTest

Added: llvm/trunk/lib/Fuzzer/test/OutOfMemorySingleLargeMallocTest.cpp
URL: http://llvm.org/viewvc/llvm-project/llvm/trunk/lib/Fuzzer/test/OutOfMemorySingleLargeMallocTest.cpp?rev=288281&view=auto
==============================================================================
--- llvm/trunk/lib/Fuzzer/test/OutOfMemorySingleLargeMallocTest.cpp (added)
+++ llvm/trunk/lib/Fuzzer/test/OutOfMemorySingleLargeMallocTest.cpp Wed Nov 30 16:39:35 2016
@@ -0,0 +1,28 @@
+// This file is distributed under the University of Illinois Open Source
+// License. See LICENSE.TXT for details.
+
+// Tests OOM handling.
+#include <assert.h>
+#include <cstdint>
+#include <cstdlib>
+#include <cstddef>
+#include <cstring>
+#include <iostream>
+#include <unistd.h>
+
+static volatile char *SinkPtr;
+
+extern "C" int LLVMFuzzerTestOneInput(const uint8_t *Data, size_t Size) {
+  if (Size > 0 && Data[0] == 'H') {
+    if (Size > 1 && Data[1] == 'i') {
+      if (Size > 2 && Data[2] == '!') {
+          size_t kSize = 0xff000000U;
+          char *p = new char[kSize];
+          SinkPtr = p;
+          delete [] p;
+      }
+    }
+  }
+  return 0;
+}
+

Modified: llvm/trunk/lib/Fuzzer/test/fuzzer-oom-with-profile.test
URL: http://llvm.org/viewvc/llvm-project/llvm/trunk/lib/Fuzzer/test/fuzzer-oom-with-profile.test?rev=288281&r1=288280&r2=288281&view=diff
==============================================================================
--- llvm/trunk/lib/Fuzzer/test/fuzzer-oom-with-profile.test (original)
+++ llvm/trunk/lib/Fuzzer/test/fuzzer-oom-with-profile.test Wed Nov 30 16:39:35 2016
@@ -1,6 +1,6 @@
 REQUIRES: linux
-RUN: not LLVMFuzzer-OutOfMemoryTest -rss_limit_mb=10 2>&1 | FileCheck %s
-CHECK: ERROR: libFuzzer: out-of-memory (used: {{.*}}; limit: 10Mb)
+RUN: not LLVMFuzzer-OutOfMemoryTest -rss_limit_mb=300 2>&1 | FileCheck %s
+CHECK: ERROR: libFuzzer: out-of-memory (used: {{.*}}; limit: 300Mb)
 CHECK: Live Heap Allocations
 CHECK: Test unit written to ./oom-
 SUMMARY: libFuzzer: out-of-memory

Modified: llvm/trunk/lib/Fuzzer/test/fuzzer-oom.test
URL: http://llvm.org/viewvc/llvm-project/llvm/trunk/lib/Fuzzer/test/fuzzer-oom.test?rev=288281&r1=288280&r2=288281&view=diff
==============================================================================
--- llvm/trunk/lib/Fuzzer/test/fuzzer-oom.test (original)
+++ llvm/trunk/lib/Fuzzer/test/fuzzer-oom.test Wed Nov 30 16:39:35 2016
@@ -1,4 +1,8 @@
-RUN: not LLVMFuzzer-OutOfMemoryTest -rss_limit_mb=10 2>&1 | FileCheck %s
-CHECK: ERROR: libFuzzer: out-of-memory (used: {{.*}}; limit: 10Mb)
+RUN: not LLVMFuzzer-OutOfMemoryTest -rss_limit_mb=300 2>&1 | FileCheck %s
+CHECK: ERROR: libFuzzer: out-of-memory (used: {{.*}}; limit: 300Mb)
 CHECK: Test unit written to ./oom-
 SUMMARY: libFuzzer: out-of-memory
+
+RUN: not LLVMFuzzer-OutOfMemorySingleLargeMallocTest 2>&1 | FileCheck %s --check-prefix=SINGLE_LARGE_MALLOC
+SINGLE_LARGE_MALLOC: libFuzzer: out-of-memory (malloc(42{{.*}}))
+SINGLE_LARGE_MALLOC: in LLVMFuzzerTestOneInput