[compiler-rt] r353891 - [libFuzzer] move the implementation of the fork mode into a separate file
Kostya Serebryany via llvm-commits
llvm-commits at lists.llvm.org
Tue Feb 12 14:48:55 PST 2019
Author: kcc
Date: Tue Feb 12 14:48:55 2019
New Revision: 353891
URL: http://llvm.org/viewvc/llvm-project?rev=353891&view=rev
Log:
[libFuzzer] move the implementation of the fork mode into a separate file
Added:
compiler-rt/trunk/lib/fuzzer/FuzzerFork.cpp
compiler-rt/trunk/lib/fuzzer/FuzzerFork.h
Modified:
compiler-rt/trunk/lib/fuzzer/CMakeLists.txt
compiler-rt/trunk/lib/fuzzer/FuzzerDriver.cpp
compiler-rt/trunk/lib/fuzzer/FuzzerIO.cpp
compiler-rt/trunk/lib/fuzzer/FuzzerIO.h
Modified: compiler-rt/trunk/lib/fuzzer/CMakeLists.txt
URL: http://llvm.org/viewvc/llvm-project/compiler-rt/trunk/lib/fuzzer/CMakeLists.txt?rev=353891&r1=353890&r2=353891&view=diff
==============================================================================
--- compiler-rt/trunk/lib/fuzzer/CMakeLists.txt (original)
+++ compiler-rt/trunk/lib/fuzzer/CMakeLists.txt Tue Feb 12 14:48:55 2019
@@ -6,6 +6,7 @@ set(LIBFUZZER_SOURCES
FuzzerExtFunctionsWeak.cpp
FuzzerExtFunctionsWindows.cpp
FuzzerExtraCounters.cpp
+ FuzzerFork.cpp
FuzzerIO.cpp
FuzzerIOPosix.cpp
FuzzerIOWindows.cpp
@@ -32,6 +33,7 @@ set(LIBFUZZER_HEADERS
FuzzerExtFunctions.def
FuzzerExtFunctions.h
FuzzerFlags.def
+ FuzzerFork.h
FuzzerIO.h
FuzzerInterface.h
FuzzerInternal.h
Modified: compiler-rt/trunk/lib/fuzzer/FuzzerDriver.cpp
URL: http://llvm.org/viewvc/llvm-project/compiler-rt/trunk/lib/fuzzer/FuzzerDriver.cpp?rev=353891&r1=353890&r2=353891&view=diff
==============================================================================
--- compiler-rt/trunk/lib/fuzzer/FuzzerDriver.cpp (original)
+++ compiler-rt/trunk/lib/fuzzer/FuzzerDriver.cpp Tue Feb 12 14:48:55 2019
@@ -10,13 +10,14 @@
#include "FuzzerCommand.h"
#include "FuzzerCorpus.h"
+#include "FuzzerFork.h"
#include "FuzzerIO.h"
#include "FuzzerInterface.h"
#include "FuzzerInternal.h"
+#include "FuzzerMerge.h"
#include "FuzzerMutate.h"
#include "FuzzerRandom.h"
#include "FuzzerTracePC.h"
-#include "FuzzerMerge.h"
#include <algorithm>
#include <atomic>
#include <chrono>
@@ -306,11 +307,6 @@ static std::string GetDedupTokenFromFile
return S.substr(Beg, End - Beg);
}
-static std::string TempPath(const char *Extension) {
- return DirPlusFile(TmpDir(),
- "libFuzzerTemp." + std::to_string(GetPid()) + Extension);
-}
-
int CleanseCrashInput(const Vector<std::string> &Args,
const FuzzingOptions &Options) {
if (Inputs->size() != 1 || !Flags.exact_artifact_path) {
@@ -471,91 +467,6 @@ int MinimizeCrashInputInternalStep(Fuzze
return 0;
}
-// This is just a skeleton of an experimental -fork=1 feature.
-void FuzzWithFork(Fuzzer *F, const FuzzingOptions &Options,
- const Vector<std::string> &Args,
- const Vector<std::string> &Corpora) {
- Printf("INFO: -fork=1: doing fuzzing in a separate process in order to "
- "be more resistant to crashes, timeouts, and OOMs\n");
- auto Rand = F->GetMD().GetRand();
-
- Vector<SizedFile> Corpus;
- for (auto &Dir : Corpora)
- GetSizedFilesFromDir(Dir, &Corpus);
- std::sort(Corpus.begin(), Corpus.end());
- auto CFPath = TempPath(".fork");
- auto LogPath = TempPath(".log");
-
- Vector<std::string> Files;
- Set<uint32_t> Features;
- if (!Corpus.empty()) {
- CrashResistantMerge(Args, {}, Corpus, &Files, {}, &Features, CFPath, false);
- RemoveFile(CFPath);
- }
- auto TempDir = TempPath("Dir");
- MkDir(TempDir);
- Printf("INFO: -fork=1: %zd seeds, starting to fuzz; scratch: %s\n",
- Files.size(), TempDir.c_str());
-
- Command BaseCmd(Args);
- BaseCmd.removeFlag("fork");
- for (auto &C : Corpora) // Remove all corpora from the args.
- BaseCmd.removeArgument(C);
- if (!BaseCmd.hasFlag("max_total_time"))
- BaseCmd.addFlag("max_total_time", "60");
- BaseCmd.addArgument(TempDir);
- int ExitCode = 0;
- for (size_t i = 0; i < 1000000; i++) {
- // TODO: take new files from disk e.g. those generated by another process.
- Command Cmd(BaseCmd);
- if (Files.size() >= 2)
- Cmd.addFlag("seed_inputs",
- Files[Rand.SkewTowardsLast(Files.size())] + "," +
- Files[Rand.SkewTowardsLast(Files.size())]);
- Cmd.setOutputFile(LogPath);
- Cmd.combineOutAndErr();
- RmFilesInDir(TempDir);
- ExitCode = ExecuteCommand(Cmd);
- // Printf("done [%d] %s\n", ExitCode, Cmd.toString().c_str());
- if (ExitCode == Options.InterruptExitCode)
- break;
- Vector<SizedFile> TempFiles;
- Vector<std::string>FilesToAdd;
- Set<uint32_t> NewFeatures;
- GetSizedFilesFromDir(TempDir, &TempFiles);
- CrashResistantMerge(Args, {}, TempFiles, &FilesToAdd, Features,
- &NewFeatures, CFPath, false);
- RemoveFile(CFPath);
- for (auto &Path : FilesToAdd) {
- auto NewPath = F->WriteToOutputCorpus(FileToVector(Path, Options.MaxLen));
- if (!NewPath.empty())
- Files.push_back(NewPath);
- }
- Features.insert(NewFeatures.begin(), NewFeatures.end());
- Printf("INFO: temp_files: %zd files_added: %zd newft: %zd ft: %zd\n",
- TempFiles.size(), FilesToAdd.size(), NewFeatures.size(),
- Features.size());
- // Continue if our crash is one of the ignorred ones.
- if (Options.IgnoreTimeouts && ExitCode == Options.TimeoutExitCode)
- continue;
- if (Options.IgnoreOOMs && ExitCode == Options.OOMExitCode)
- continue;
- // And exit if we don't ignore this crash.
- if (ExitCode != 0) {
- Printf("INFO: log from the inner process:\n%s",
- FileToString(LogPath).c_str());
- break;
- }
- }
-
- RmFilesInDir(TempDir);
- RmDir(TempDir);
-
- // Use the exit code from the last child process.
- Printf("Fork: exiting: %d\n", ExitCode);
- exit(ExitCode);
-}
-
void Merge(Fuzzer *F, FuzzingOptions &Options, const Vector<std::string> &Args,
const Vector<std::string> &Corpora, const char *CFPathOrNull) {
if (Corpora.size() < 2) {
@@ -814,7 +725,7 @@ int FuzzerDriver(int *argc, char ***argv
}
if (Flags.fork)
- FuzzWithFork(F, Options, Args, *Inputs);
+ FuzzWithFork(F->GetMD().GetRand(), Options, Args, *Inputs);
if (Flags.merge)
Merge(F, Options, Args, *Inputs, Flags.merge_control_file);
Added: compiler-rt/trunk/lib/fuzzer/FuzzerFork.cpp
URL: http://llvm.org/viewvc/llvm-project/compiler-rt/trunk/lib/fuzzer/FuzzerFork.cpp?rev=353891&view=auto
==============================================================================
--- compiler-rt/trunk/lib/fuzzer/FuzzerFork.cpp (added)
+++ compiler-rt/trunk/lib/fuzzer/FuzzerFork.cpp Tue Feb 12 14:48:55 2019
@@ -0,0 +1,127 @@
+//===- FuzzerFork.cpp - run fuzzing in separate subprocesses --------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+// Spawn an orchestrate separate fuzzing processes.
+//===----------------------------------------------------------------------===//
+
+#include "FuzzerCommand.h"
+#include "FuzzerFork.h"
+#include "FuzzerIO.h"
+#include "FuzzerMerge.h"
+#include "FuzzerSHA1.h"
+#include "FuzzerUtil.h"
+
+namespace fuzzer {
+
+// This is just a skeleton of an experimental -fork=1 feature.
+void FuzzWithFork(Random &Rand, const FuzzingOptions &Options,
+ const Vector<std::string> &Args,
+ const Vector<std::string> &CorpusDirs) {
+ Printf("INFO: -fork=1: doing fuzzing in a separate process in order to "
+ "be more resistant to crashes, timeouts, and OOMs\n");
+
+ Vector<SizedFile> SeedFiles;
+ for (auto &Dir : CorpusDirs)
+ GetSizedFilesFromDir(Dir, &SeedFiles);
+ std::sort(SeedFiles.begin(), SeedFiles.end());
+ auto CFPath = TempPath(".fork");
+ auto LogPath = TempPath(".log");
+ auto TempDir = TempPath(".scratch_dir");
+ std::string MainCorpusDir;
+ if (CorpusDirs.empty())
+ MkDir(MainCorpusDir = TempPath(".corpus_dir"));
+ else
+ MainCorpusDir = CorpusDirs[0];
+ MkDir(TempDir);
+
+ Vector<std::string> Files;
+ Set<uint32_t> Features;
+ if (!SeedFiles.empty()) {
+ CrashResistantMerge(Args, {}, SeedFiles, &Files, {}, &Features, CFPath,
+ false);
+ RemoveFile(CFPath);
+ }
+ Printf("INFO: -fork=1: %zd seeds, starting to fuzz; scratch: %s\n",
+ Files.size(), TempDir.c_str());
+
+ Command BaseCmd(Args);
+ BaseCmd.removeFlag("fork");
+ for (auto &C : CorpusDirs) // Remove all corpora from the args.
+ BaseCmd.removeArgument(C);
+ BaseCmd.addArgument(TempDir);
+ BaseCmd.addFlag("len_control", "0"); // len_control is bad for short runs.
+ BaseCmd.addFlag("reload", "0"); // working in an isolated dir, no reload.
+ int ExitCode = 0;
+ int max_total_time = 1;
+ for (size_t i = 0; i < 1000000; i++) {
+ // TODO: take new files from disk e.g. those generated by another process.
+ Command Cmd(BaseCmd);
+ if (size_t CorpusSubsetSize = std::min(Files.size(), (size_t)10)) {
+ std::string Seeds;
+ for (size_t i = 0; i < CorpusSubsetSize; i++) {
+ if (i) Seeds += ",";
+ Seeds += Files[Rand.SkewTowardsLast(Files.size())];
+ }
+ Cmd.addFlag("seed_inputs", Seeds);
+ }
+ if (Options.MaxTotalTimeSec > max_total_time)
+ max_total_time++;
+ if (!Cmd.hasFlag("max_total_time"))
+ Cmd.addFlag("max_total_time", std::to_string(max_total_time));
+ Cmd.setOutputFile(LogPath);
+ Cmd.combineOutAndErr();
+ RmFilesInDir(TempDir);
+ ExitCode = ExecuteCommand(Cmd);
+ // Printf("done [%d] %s\n", ExitCode, Cmd.toString().c_str());
+ if (ExitCode == Options.InterruptExitCode)
+ break;
+ Vector<SizedFile> TempFiles;
+ Vector<std::string>FilesToAdd;
+ Set<uint32_t> NewFeatures;
+ GetSizedFilesFromDir(TempDir, &TempFiles);
+ if (!TempFiles.empty())
+ CrashResistantMerge(Args, {}, TempFiles, &FilesToAdd, Features,
+ &NewFeatures, CFPath, false);
+ RemoveFile(CFPath);
+ for (auto &Path : FilesToAdd) {
+ auto U = FileToVector(Path);
+ auto NewPath = DirPlusFile(MainCorpusDir, Hash(U));
+ WriteToFile(U, NewPath);
+ Files.push_back(NewPath);
+ }
+ Features.insert(NewFeatures.begin(), NewFeatures.end());
+ Printf("INFO: temp_files: %zd files_added: %zd newft: %zd ft: %zd\n",
+ TempFiles.size(), FilesToAdd.size(), NewFeatures.size(),
+ Features.size());
+ // Continue if our crash is one of the ignorred ones.
+ if (Options.IgnoreTimeouts && ExitCode == Options.TimeoutExitCode)
+ continue;
+ if (Options.IgnoreOOMs && ExitCode == Options.OOMExitCode)
+ continue;
+ // And exit if we don't ignore this crash.
+ if (ExitCode != 0) {
+ Printf("INFO: log from the inner process:\n%s",
+ FileToString(LogPath).c_str());
+ break;
+ }
+ }
+
+ RmFilesInDir(TempDir);
+ RmDir(TempDir);
+
+ if (CorpusDirs.empty()) {
+ RmFilesInDir(MainCorpusDir);
+ RmDir(MainCorpusDir);
+ }
+
+ // Use the exit code from the last child process.
+ Printf("Fork: exiting: %d\n", ExitCode);
+ exit(ExitCode);
+}
+
+} // namespace fuzzer
+
Added: compiler-rt/trunk/lib/fuzzer/FuzzerFork.h
URL: http://llvm.org/viewvc/llvm-project/compiler-rt/trunk/lib/fuzzer/FuzzerFork.h?rev=353891&view=auto
==============================================================================
--- compiler-rt/trunk/lib/fuzzer/FuzzerFork.h (added)
+++ compiler-rt/trunk/lib/fuzzer/FuzzerFork.h Tue Feb 12 14:48:55 2019
@@ -0,0 +1,24 @@
+//===- FuzzerFork.h - run fuzzing in sub-processes --------------*- C++ -* ===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef LLVM_FUZZER_FORK_H
+#define LLVM_FUZZER_FORK_H
+
+#include "FuzzerDefs.h"
+#include "FuzzerOptions.h"
+#include "FuzzerRandom.h"
+
+#include <string>
+
+namespace fuzzer {
+void FuzzWithFork(Random &Rand, const FuzzingOptions &Options,
+ const Vector<std::string> &Args,
+ const Vector<std::string> &CorpusDirs);
+} // namespace fuzzer
+
+#endif // LLVM_FUZZER_FORK_H
Modified: compiler-rt/trunk/lib/fuzzer/FuzzerIO.cpp
URL: http://llvm.org/viewvc/llvm-project/compiler-rt/trunk/lib/fuzzer/FuzzerIO.cpp?rev=353891&r1=353890&r2=353891&view=diff
==============================================================================
--- compiler-rt/trunk/lib/fuzzer/FuzzerIO.cpp (original)
+++ compiler-rt/trunk/lib/fuzzer/FuzzerIO.cpp Tue Feb 12 14:48:55 2019
@@ -8,9 +8,10 @@
// IO functions.
//===----------------------------------------------------------------------===//
-#include "FuzzerIO.h"
#include "FuzzerDefs.h"
#include "FuzzerExtFunctions.h"
+#include "FuzzerIO.h"
+#include "FuzzerUtil.h"
#include <algorithm>
#include <cstdarg>
#include <fstream>
@@ -141,4 +142,9 @@ void RmFilesInDir(const std::string &Pat
RemoveFile(F);
}
+std::string TempPath(const char *Extension) {
+ return DirPlusFile(TmpDir(),
+ "libFuzzerTemp." + std::to_string(GetPid()) + Extension);
+}
+
} // namespace fuzzer
Modified: compiler-rt/trunk/lib/fuzzer/FuzzerIO.h
URL: http://llvm.org/viewvc/llvm-project/compiler-rt/trunk/lib/fuzzer/FuzzerIO.h?rev=353891&r1=353890&r2=353891&view=diff
==============================================================================
--- compiler-rt/trunk/lib/fuzzer/FuzzerIO.h (original)
+++ compiler-rt/trunk/lib/fuzzer/FuzzerIO.h Tue Feb 12 14:48:55 2019
@@ -39,6 +39,8 @@ std::string DirName(const std::string &F
// Returns path to a TmpDir.
std::string TmpDir();
+std::string TempPath(const char *Extension);
+
bool IsInterestingCoverageFile(const std::string &FileName);
void DupAndCloseStderr();
More information about the llvm-commits
mailing list