[compiler-rt] r311972 - [libFuzzer] refactoring: move reading the seed corpus closer to where it's consumed; NFC

[Kostya Serebryany via llvm-commits]

Author: kcc
Date: Mon Aug 28 19:05:01 2017
New Revision: 311972

URL: http://llvm.org/viewvc/llvm-project?rev=311972&view=rev
Log:
[libFuzzer] refactoring: move reading the seed corpus closer to where it's consumed; NFC

Modified:
    compiler-rt/trunk/lib/fuzzer/FuzzerDriver.cpp
    compiler-rt/trunk/lib/fuzzer/FuzzerInternal.h
    compiler-rt/trunk/lib/fuzzer/FuzzerLoop.cpp

Modified: compiler-rt/trunk/lib/fuzzer/FuzzerDriver.cpp
URL: http://llvm.org/viewvc/llvm-project/compiler-rt/trunk/lib/fuzzer/FuzzerDriver.cpp?rev=311972&r1=311971&r2=311972&view=diff
==============================================================================
--- compiler-rt/trunk/lib/fuzzer/FuzzerDriver.cpp (original)
+++ compiler-rt/trunk/lib/fuzzer/FuzzerDriver.cpp Mon Aug 28 19:05:01 2017
@@ -558,8 +558,6 @@ int FuzzerDriver(int *argc, char ***argv
   if (Flags.workers > 0 && Flags.jobs > 0)
     return RunInMultipleProcesses(Args, Flags.workers, Flags.jobs);
 
-  const size_t kMaxSaneLen = 1 << 20;
-  const size_t kMinDefaultLen = 4096;
   FuzzingOptions Options;
   Options.Verbosity = Flags.verbosity;
   Options.MaxLen = Flags.max_len;
@@ -702,8 +700,10 @@ int FuzzerDriver(int *argc, char ***argv
   }
 
   if (Flags.merge) {
+    const size_t kDefaultMaxMergeLen = 1 << 20;
     if (Options.MaxLen == 0)
-      F->SetMaxInputLen(kMaxSaneLen);
+      F->SetMaxInputLen(kDefaultMaxMergeLen);
+
     if (Flags.merge_control_file)
       F->CrashResistantMergeInternalStep(Flags.merge_control_file);
     else
@@ -713,16 +713,16 @@ int FuzzerDriver(int *argc, char ***argv
     exit(0);
   }
 
-  size_t TemporaryMaxLen = Options.MaxLen ? Options.MaxLen : kMaxSaneLen;
-
-  UnitVector InitialCorpus;
-  for (auto &Inp : *Inputs) {
-    Printf("Loading corpus dir: %s\n", Inp.c_str());
-    ReadDirToVectorOfUnits(Inp.c_str(), &InitialCorpus, nullptr,
-                           TemporaryMaxLen, /*ExitOnError=*/false);
-  }
 
   if (Flags.analyze_dict) {
+    size_t MaxLen = INT_MAX;  // Large max length.
+    UnitVector InitialCorpus;
+    for (auto &Inp : *Inputs) {
+      Printf("Loading corpus dir: %s\n", Inp.c_str());
+      ReadDirToVectorOfUnits(Inp.c_str(), &InitialCorpus, nullptr,
+                             MaxLen, /*ExitOnError=*/false);
+    }
+
     if (Dictionary.empty() || Inputs->empty()) {
       Printf("ERROR: can't analyze dict without dict and corpus provided\n");
       return 1;
@@ -735,21 +735,7 @@ int FuzzerDriver(int *argc, char ***argv
     exit(0);
   }
 
-  if (Options.MaxLen == 0) {
-    size_t MaxLen = 0;
-    for (auto &U : InitialCorpus)
-      MaxLen = std::max(U.size(), MaxLen);
-    F->SetMaxInputLen(std::min(std::max(kMinDefaultLen, MaxLen), kMaxSaneLen));
-  }
-
-  if (InitialCorpus.empty()) {
-    InitialCorpus.push_back(Unit({'\n'}));  // Valid ASCII input.
-    if (Options.Verbosity)
-      Printf("INFO: A corpus is not provided, starting from an empty corpus\n");
-  }
-  F->ShuffleAndMinimize(&InitialCorpus);
-  InitialCorpus.clear();  // Don't need this memory any more.
-  F->Loop();
+  F->Loop(*Inputs);
 
   if (Flags.verbosity)
     Printf("Done %zd runs in %zd second(s)\n", F->getTotalNumberOfRuns(),

Modified: compiler-rt/trunk/lib/fuzzer/FuzzerInternal.h
URL: http://llvm.org/viewvc/llvm-project/compiler-rt/trunk/lib/fuzzer/FuzzerInternal.h?rev=311972&r1=311971&r2=311972&view=diff
==============================================================================
--- compiler-rt/trunk/lib/fuzzer/FuzzerInternal.h (original)
+++ compiler-rt/trunk/lib/fuzzer/FuzzerInternal.h Mon Aug 28 19:05:01 2017
@@ -35,7 +35,8 @@ public:
   Fuzzer(UserCallback CB, InputCorpus &Corpus, MutationDispatcher &MD,
          FuzzingOptions Options);
   ~Fuzzer();
-  void Loop();
+  void Loop(const Vector<std::string> &CorpusDirs);
+  void ReadAndExecuteSeedCorpora(const Vector<std::string> &CorpusDirs);
   void MinimizeCrashLoop(const Unit &U);
   void ShuffleAndMinimize(UnitVector *V);
   void RereadOutputCorpus(size_t MaxSize);

Modified: compiler-rt/trunk/lib/fuzzer/FuzzerLoop.cpp
URL: http://llvm.org/viewvc/llvm-project/compiler-rt/trunk/lib/fuzzer/FuzzerLoop.cpp?rev=311972&r1=311971&r2=311972&view=diff
==============================================================================
--- compiler-rt/trunk/lib/fuzzer/FuzzerLoop.cpp (original)
+++ compiler-rt/trunk/lib/fuzzer/FuzzerLoop.cpp Mon Aug 28 19:05:01 2017
@@ -380,7 +380,8 @@ void Fuzzer::ShuffleCorpus(UnitVector *V
 }
 
 void Fuzzer::ShuffleAndMinimize(UnitVector *InitialCorpus) {
-  Printf("#0\tREAD units: %zd\n", InitialCorpus->size());
+  Printf("#0\tREAD units: %zd; rss: %zdMb\n", InitialCorpus->size(),
+         GetPeakRSSMb());
   if (Options.ShuffleAtStartUp)
     ShuffleCorpus(InitialCorpus);
 
@@ -624,7 +625,33 @@ void Fuzzer::MutateAndTestOne() {
   }
 }
 
-void Fuzzer::Loop() {
+void Fuzzer::ReadAndExecuteSeedCorpora(const Vector<std::string> &CorpusDirs) {
+  const size_t kMaxSaneLen = 1 << 20;
+  const size_t kMinDefaultLen = 4096;
+  size_t TemporaryMaxLen = Options.MaxLen ? Options.MaxLen : kMaxSaneLen;
+  UnitVector InitialCorpus;
+  for (auto &Inp : CorpusDirs) {
+    Printf("Loading corpus dir: %s\n", Inp.c_str());
+    ReadDirToVectorOfUnits(Inp.c_str(), &InitialCorpus, nullptr,
+                           TemporaryMaxLen, /*ExitOnError=*/false);
+  }
+  if (Options.MaxLen == 0) {
+    size_t MaxLen = 0;
+    for (auto &U : InitialCorpus)
+      MaxLen = std::max(U.size(), MaxLen);
+    SetMaxInputLen(std::min(std::max(kMinDefaultLen, MaxLen), kMaxSaneLen));
+  }
+
+  if (InitialCorpus.empty()) {
+    InitialCorpus.push_back(Unit({'\n'}));  // Valid ASCII input.
+    if (Options.Verbosity)
+      Printf("INFO: A corpus is not provided, starting from an empty corpus\n");
+  }
+  ShuffleAndMinimize(&InitialCorpus);
+}
+
+void Fuzzer::Loop(const Vector<std::string> &CorpusDirs) {
+  ReadAndExecuteSeedCorpora(CorpusDirs);
   TPC.SetPrintNewPCs(Options.PrintNewCovPcs);
   TPC.SetPrintNewFuncs(Options.PrintNewCovFuncs);
   system_clock::time_point LastCorpusReload = system_clock::now();