[llvm] r268807 - [libFuzzer] add exeprimental -rss_limit_mb flag to fight against OOMs

[Kostya Serebryany via llvm-commits]

Author: kcc
Date: Fri May  6 16:58:35 2016
New Revision: 268807

URL: http://llvm.org/viewvc/llvm-project?rev=268807&view=rev
Log:
[libFuzzer] add exeprimental -rss_limit_mb flag to fight against OOMs

Modified:
    llvm/trunk/lib/Fuzzer/FuzzerDriver.cpp
    llvm/trunk/lib/Fuzzer/FuzzerFlags.def
    llvm/trunk/lib/Fuzzer/FuzzerInternal.h
    llvm/trunk/lib/Fuzzer/FuzzerLoop.cpp

Modified: llvm/trunk/lib/Fuzzer/FuzzerDriver.cpp
URL: http://llvm.org/viewvc/llvm-project/llvm/trunk/lib/Fuzzer/FuzzerDriver.cpp?rev=268807&r1=268806&r2=268807&view=diff
==============================================================================
--- llvm/trunk/lib/Fuzzer/FuzzerDriver.cpp (original)
+++ llvm/trunk/lib/Fuzzer/FuzzerDriver.cpp Fri May  6 16:58:35 2016
@@ -234,6 +234,21 @@ static int RunInMultipleProcesses(const
   return HasErrors ? 1 : 0;
 }
 
+static void RssThread(Fuzzer *F, size_t RssLimitMb) {
+  while (true) {
+    std::this_thread::sleep_for(std::chrono::seconds(1));
+    size_t Peak = GetPeakRSSMb();
+    if (Peak > RssLimitMb)
+      F->RssLimitCallback(Peak, RssLimitMb);
+  }
+}
+
+static void StartRssThread(Fuzzer *F, size_t RssLimitMb) {
+  if (!RssLimitMb) return;
+  std::thread T(RssThread, F, RssLimitMb);
+  T.detach();
+}
+
 int RunOneTest(Fuzzer *F, const char *InputFilePath) {
   Unit U = FileToVector(InputFilePath);
   Unit PreciseSizedU(U);
@@ -331,6 +346,8 @@ static int FuzzerDriver(const std::vecto
     if (U.size() <= Word::GetMaxSize())
       MD.AddWordToManualDictionary(Word(U.data(), U.size()));
 
+  StartRssThread(&F, Flags.rss_limit_mb);
+
   // Timer
   if (Flags.timeout > 0)
     SetTimer(Flags.timeout / 2 + 1);

Modified: llvm/trunk/lib/Fuzzer/FuzzerFlags.def
URL: http://llvm.org/viewvc/llvm-project/llvm/trunk/lib/Fuzzer/FuzzerFlags.def?rev=268807&r1=268806&r2=268807&view=diff
==============================================================================
--- llvm/trunk/lib/Fuzzer/FuzzerFlags.def (original)
+++ llvm/trunk/lib/Fuzzer/FuzzerFlags.def Fri May  6 16:58:35 2016
@@ -81,6 +81,8 @@ FUZZER_FLAG_INT(close_fd_mask, 0, "If 1,
     "Be careful, this will also close e.g. asan's stderr/stdout.")
 FUZZER_FLAG_INT(detect_leaks, 1, "If 1, and if LeakSanitizer is enabled "
     "try to detect memory leaks during fuzzing (i.e. not only at shut down).")
+FUZZER_FLAG_INT(rss_limit_mb, 0, "If non-zero, the fuzzer will exit upon"
+    "reaching this limit of RSS memory usage.")
 
 FUZZER_DEPRECATED_FLAG(exit_on_first)
 FUZZER_DEPRECATED_FLAG(save_minimized_corpus)

Modified: llvm/trunk/lib/Fuzzer/FuzzerInternal.h
URL: http://llvm.org/viewvc/llvm-project/llvm/trunk/lib/Fuzzer/FuzzerInternal.h?rev=268807&r1=268806&r2=268807&view=diff
==============================================================================
--- llvm/trunk/lib/Fuzzer/FuzzerInternal.h (original)
+++ llvm/trunk/lib/Fuzzer/FuzzerInternal.h Fri May  6 16:58:35 2016
@@ -353,6 +353,7 @@ public:
   MutationDispatcher &GetMD() { return MD; }
   void PrintFinalStats();
   void SetMaxLen(size_t MaxLen);
+  void RssLimitCallback(size_t RssPeakMb, size_t RssLimitMb);
 
 private:
   void AlarmCallback();

Modified: llvm/trunk/lib/Fuzzer/FuzzerLoop.cpp
URL: http://llvm.org/viewvc/llvm-project/llvm/trunk/lib/Fuzzer/FuzzerLoop.cpp?rev=268807&r1=268806&r2=268807&view=diff
==============================================================================
--- llvm/trunk/lib/Fuzzer/FuzzerLoop.cpp (original)
+++ llvm/trunk/lib/Fuzzer/FuzzerLoop.cpp Fri May  6 16:58:35 2016
@@ -176,6 +176,17 @@ void Fuzzer::AlarmCallback() {
   }
 }
 
+void Fuzzer::RssLimitCallback(size_t RssPeakMb, size_t RssLimitMb) {
+  Printf("==%d== ERROR: libFuzzer: out-of-memory (used: %zdMb; limit: %zdMb)\n",
+         GetPid(), RssPeakMb, RssLimitMb);
+  Printf("*****************************************************************\n");
+  Printf("** Experimental! TODO: dump the stack trace and the reproducer **\n");
+  Printf("*****************************************************************\n");
+  Printf("SUMMARY: libFuzzer: out-of-memory\n");
+  PrintFinalStats();
+  _Exit(Options.ErrorExitCode); // Stop right now.
+}
+
 void Fuzzer::PrintStats(const char *Where, const char *End) {
   size_t ExecPerSec = execPerSec();
   if (Options.OutputCSV) {