[compiler-rt] r353910 - [libFuzzer] a bit of refactoring of the fork mode
Kostya Serebryany via llvm-commits
llvm-commits at lists.llvm.org
Tue Feb 12 20:04:45 PST 2019
Author: kcc
Date: Tue Feb 12 20:04:45 2019
New Revision: 353910
URL: http://llvm.org/viewvc/llvm-project?rev=353910&view=rev
Log:
[libFuzzer] a bit of refactoring of the fork mode
Modified:
compiler-rt/trunk/lib/fuzzer/FuzzerFork.cpp
compiler-rt/trunk/lib/fuzzer/FuzzerIO.cpp
compiler-rt/trunk/lib/fuzzer/FuzzerIO.h
compiler-rt/trunk/lib/fuzzer/FuzzerIOPosix.cpp
compiler-rt/trunk/lib/fuzzer/FuzzerIOWindows.cpp
compiler-rt/trunk/lib/fuzzer/FuzzerMerge.cpp
compiler-rt/trunk/test/fuzzer/merge-control-file.test
Modified: compiler-rt/trunk/lib/fuzzer/FuzzerFork.cpp
URL: http://llvm.org/viewvc/llvm-project/compiler-rt/trunk/lib/fuzzer/FuzzerFork.cpp?rev=353910&r1=353909&r2=353910&view=diff
==============================================================================
--- compiler-rt/trunk/lib/fuzzer/FuzzerFork.cpp (original)
+++ compiler-rt/trunk/lib/fuzzer/FuzzerFork.cpp Tue Feb 12 20:04:45 2019
@@ -5,7 +5,7 @@
// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
//
//===----------------------------------------------------------------------===//
-// Spawn an orchestrate separate fuzzing processes.
+// Spawn and orchestrate separate fuzzing processes.
//===----------------------------------------------------------------------===//
#include "FuzzerCommand.h"
@@ -17,6 +17,65 @@
namespace fuzzer {
+struct FuzzJob {
+ // Inputs.
+ Command Cmd;
+ Vector<std::string> Files;
+ std::string CorpusDir;
+ std::string LogPath;
+ std::string CFPath;
+ int MaxTotalTimeSec;
+
+ // Fuzzing Outputs.
+ int ExitCode;
+};
+
+struct GlobalEnv {
+ const Vector<std::string> *Args;
+ std::string MainCorpusDir;
+ Set<uint32_t> Features;
+ Vector<std::string> Files;
+};
+
+void RunOneFuzzingJob(FuzzJob *Job) {
+ Command &Cmd = Job->Cmd;
+ if (!Job->Files.empty()) {
+ std::string Seeds;
+ for (const auto &File : Job->Files)
+ Seeds += (Seeds.empty() ? "" : ",") + File;
+ Cmd.addFlag("seed_inputs", Seeds);
+ }
+ Cmd.addFlag("max_total_time", std::to_string(Job->MaxTotalTimeSec));
+ Cmd.setOutputFile(Job->LogPath);
+ Cmd.combineOutAndErr();
+ Cmd.addArgument(Job->CorpusDir);
+ RmDirRecursive(Job->CorpusDir);
+ MkDir(Job->CorpusDir);
+ Job->ExitCode = ExecuteCommand(Cmd);
+}
+
+void RunOneMergeJob(GlobalEnv *Env, FuzzJob *Job) {
+ Vector<SizedFile> TempFiles;
+ GetSizedFilesFromDir(Job->CorpusDir, &TempFiles);
+
+ Vector<std::string>FilesToAdd;
+ Set<uint32_t> NewFeatures;
+ CrashResistantMerge(*Env->Args, {}, TempFiles, &FilesToAdd, Env->Features,
+ &NewFeatures, Job->CFPath, false);
+ RemoveFile(Job->CFPath);
+ for (auto &Path : FilesToAdd) {
+ auto U = FileToVector(Path);
+ auto NewPath = DirPlusFile(Env->MainCorpusDir, Hash(U));
+ WriteToFile(U, NewPath);
+ Env->Files.push_back(NewPath);
+ }
+ RmDirRecursive(Job->CorpusDir);
+ Env->Features.insert(NewFeatures.begin(), NewFeatures.end());
+ Printf("INFO: temp_files: %zd files_added: %zd newft: %zd ft: %zd\n",
+ TempFiles.size(), FilesToAdd.size(), NewFeatures.size(),
+ Env->Features.size());
+}
+
// This is just a skeleton of an experimental -fork=1 feature.
void FuzzWithFork(Random &Rand, const FuzzingOptions &Options,
const Vector<std::string> &Args,
@@ -24,99 +83,78 @@ void FuzzWithFork(Random &Rand, const Fu
Printf("INFO: -fork=1: doing fuzzing in a separate process in order to "
"be more resistant to crashes, timeouts, and OOMs\n");
+ GlobalEnv Env;
+ Env.Args = &Args;
+
Vector<SizedFile> SeedFiles;
for (auto &Dir : CorpusDirs)
GetSizedFilesFromDir(Dir, &SeedFiles);
std::sort(SeedFiles.begin(), SeedFiles.end());
- auto CFPath = TempPath(".fork");
- auto LogPath = TempPath(".log");
- auto TempDir = TempPath(".scratch_dir");
- std::string MainCorpusDir;
+ auto TempDir = TempPath(".dir");
+ RmDirRecursive(TempDir); // just in case there is a leftover from an old run.
+ MkDir(TempDir);
+
+ auto CFPath = DirPlusFile(TempDir, "merge.txt");
+ auto LogPath = DirPlusFile(TempDir, "sub.log");
+
if (CorpusDirs.empty())
- MkDir(MainCorpusDir = TempPath(".corpus_dir"));
+ MkDir(Env.MainCorpusDir = DirPlusFile(TempDir, "C"));
else
- MainCorpusDir = CorpusDirs[0];
- MkDir(TempDir);
+ Env.MainCorpusDir = CorpusDirs[0];
- Vector<std::string> Files;
- Set<uint32_t> Features;
- if (!SeedFiles.empty()) {
- CrashResistantMerge(Args, {}, SeedFiles, &Files, {}, &Features, CFPath,
- false);
- RemoveFile(CFPath);
- }
+ auto TempCorpusDir = DirPlusFile(TempDir, "C0");
+
+ CrashResistantMerge(*Env.Args, {}, SeedFiles, &Env.Files, {}, &Env.Features,
+ CFPath, false);
+ RemoveFile(CFPath);
Printf("INFO: -fork=1: %zd seeds, starting to fuzz; scratch: %s\n",
- Files.size(), TempDir.c_str());
+ Env.Files.size(), TempDir.c_str());
- Command BaseCmd(Args);
+ Command BaseCmd(*Env.Args);
BaseCmd.removeFlag("fork");
for (auto &C : CorpusDirs) // Remove all corpora from the args.
BaseCmd.removeArgument(C);
- BaseCmd.addArgument(TempDir);
- BaseCmd.addFlag("len_control", "0"); // len_control is bad for short runs.
BaseCmd.addFlag("reload", "0"); // working in an isolated dir, no reload.
int ExitCode = 0;
- int max_total_time = 1;
- for (size_t i = 0; i < 1000000; i++) {
+
+
+ for (size_t i = 1; i < 1000000; i++) {
// TODO: take new files from disk e.g. those generated by another process.
- Command Cmd(BaseCmd);
- if (size_t CorpusSubsetSize = std::min(Files.size(), (size_t)10)) {
- std::string Seeds;
- for (size_t i = 0; i < CorpusSubsetSize; i++) {
- if (i) Seeds += ",";
- Seeds += Files[Rand.SkewTowardsLast(Files.size())];
- }
- Cmd.addFlag("seed_inputs", Seeds);
- }
- if (Options.MaxTotalTimeSec > max_total_time)
- max_total_time++;
- if (!Cmd.hasFlag("max_total_time"))
- Cmd.addFlag("max_total_time", std::to_string(max_total_time));
- Cmd.setOutputFile(LogPath);
- Cmd.combineOutAndErr();
- RmFilesInDir(TempDir);
- ExitCode = ExecuteCommand(Cmd);
- // Printf("done [%d] %s\n", ExitCode, Cmd.toString().c_str());
- if (ExitCode == Options.InterruptExitCode)
+
+ FuzzJob Job;
+ Job.Cmd = BaseCmd;
+ if (size_t CorpusSubsetSize = std::min(Env.Files.size(), (size_t)100))
+ for (size_t i = 0; i < CorpusSubsetSize; i++)
+ Job.Files.push_back(Env.Files[Rand.SkewTowardsLast(Env.Files.size())]);
+ Job.CorpusDir = TempCorpusDir;
+ Job.LogPath = LogPath;
+ Job.CFPath = CFPath;
+ // Start from very short runs and gradually increase them.
+ Job.MaxTotalTimeSec = std::min(300, (int)i);
+ RunOneFuzzingJob(&Job);
+
+ if (Options.Verbosity >= 2)
+ Printf("done [%d] %s\n", Job.ExitCode, Job.Cmd.toString().c_str());
+ if (Job.ExitCode == Options.InterruptExitCode)
break;
- Vector<SizedFile> TempFiles;
- Vector<std::string>FilesToAdd;
- Set<uint32_t> NewFeatures;
- GetSizedFilesFromDir(TempDir, &TempFiles);
- if (!TempFiles.empty())
- CrashResistantMerge(Args, {}, TempFiles, &FilesToAdd, Features,
- &NewFeatures, CFPath, false);
- RemoveFile(CFPath);
- for (auto &Path : FilesToAdd) {
- auto U = FileToVector(Path);
- auto NewPath = DirPlusFile(MainCorpusDir, Hash(U));
- WriteToFile(U, NewPath);
- Files.push_back(NewPath);
- }
- Features.insert(NewFeatures.begin(), NewFeatures.end());
- Printf("INFO: temp_files: %zd files_added: %zd newft: %zd ft: %zd\n",
- TempFiles.size(), FilesToAdd.size(), NewFeatures.size(),
- Features.size());
+
+ RunOneMergeJob(&Env, &Job);
+
// Continue if our crash is one of the ignorred ones.
- if (Options.IgnoreTimeouts && ExitCode == Options.TimeoutExitCode)
+ if (Options.IgnoreTimeouts && Job.ExitCode == Options.TimeoutExitCode)
continue;
- if (Options.IgnoreOOMs && ExitCode == Options.OOMExitCode)
+ if (Options.IgnoreOOMs && Job.ExitCode == Options.OOMExitCode)
continue;
// And exit if we don't ignore this crash.
- if (ExitCode != 0) {
+ if (Job.ExitCode != 0) {
Printf("INFO: log from the inner process:\n%s",
FileToString(LogPath).c_str());
+ ExitCode = Job.ExitCode;
break;
}
}
- RmFilesInDir(TempDir);
- RmDir(TempDir);
-
- if (CorpusDirs.empty()) {
- RmFilesInDir(MainCorpusDir);
- RmDir(MainCorpusDir);
- }
+ RmDirRecursive(TempDir);
// Use the exit code from the last child process.
Printf("Fork: exiting: %d\n", ExitCode);
Modified: compiler-rt/trunk/lib/fuzzer/FuzzerIO.cpp
URL: http://llvm.org/viewvc/llvm-project/compiler-rt/trunk/lib/fuzzer/FuzzerIO.cpp?rev=353910&r1=353909&r2=353910&view=diff
==============================================================================
--- compiler-rt/trunk/lib/fuzzer/FuzzerIO.cpp (original)
+++ compiler-rt/trunk/lib/fuzzer/FuzzerIO.cpp Tue Feb 12 20:04:45 2019
@@ -135,11 +135,11 @@ void VPrintf(bool Verbose, const char *F
fflush(OutputFile);
}
-void RmFilesInDir(const std::string &Path) {
- Vector<std::string> Files;
- ListFilesInDirRecursive(Path, 0, &Files, /*TopDir*/true);
- for (auto &F : Files)
- RemoveFile(F);
+void RmDirRecursive(const std::string &Dir) {
+ IterateDirRecurisve(
+ Dir, [](const std::string &Path) {},
+ [](const std::string &Path) { RmDir(Path); },
+ [](const std::string &Path) { RemoveFile(Path); });
}
std::string TempPath(const char *Extension) {
Modified: compiler-rt/trunk/lib/fuzzer/FuzzerIO.h
URL: http://llvm.org/viewvc/llvm-project/compiler-rt/trunk/lib/fuzzer/FuzzerIO.h?rev=353910&r1=353909&r2=353910&view=diff
==============================================================================
--- compiler-rt/trunk/lib/fuzzer/FuzzerIO.h (original)
+++ compiler-rt/trunk/lib/fuzzer/FuzzerIO.h Tue Feb 12 20:04:45 2019
@@ -60,6 +60,16 @@ size_t FileSize(const std::string &Path)
void ListFilesInDirRecursive(const std::string &Dir, long *Epoch,
Vector<std::string> *V, bool TopDir);
+void RmDirRecursive(const std::string &Dir);
+
+// Iterate files and dirs inside Dir, recursively.
+// Call DirPreCallback/DirPostCallback on dirs before/after
+// calling FileCallback on files.
+void IterateDirRecurisve(const std::string &Dir,
+ void (*DirPreCallback)(const std::string &Dir),
+ void (*DirPostCallback)(const std::string &Dir),
+ void (*FileCallback)(const std::string &Dir));
+
struct SizedFile {
std::string File;
size_t Size;
@@ -86,7 +96,6 @@ intptr_t GetHandleFromFd(int fd);
void MkDir(const std::string &Path);
void RmDir(const std::string &Path);
-void RmFilesInDir(const std::string &Path);
} // namespace fuzzer
Modified: compiler-rt/trunk/lib/fuzzer/FuzzerIOPosix.cpp
URL: http://llvm.org/viewvc/llvm-project/compiler-rt/trunk/lib/fuzzer/FuzzerIOPosix.cpp?rev=353910&r1=353909&r2=353910&view=diff
==============================================================================
--- compiler-rt/trunk/lib/fuzzer/FuzzerIOPosix.cpp (original)
+++ compiler-rt/trunk/lib/fuzzer/FuzzerIOPosix.cpp Tue Feb 12 20:04:45 2019
@@ -78,6 +78,28 @@ void ListFilesInDirRecursive(const std::
*Epoch = E;
}
+
+void IterateDirRecurisve(const std::string &Dir,
+ void (*DirPreCallback)(const std::string &Dir),
+ void (*DirPostCallback)(const std::string &Dir),
+ void (*FileCallback)(const std::string &Dir)) {
+ DirPreCallback(Dir);
+ DIR *D = opendir(Dir.c_str());
+ if (!D) return;
+ while (auto E = readdir(D)) {
+ std::string Path = DirPlusFile(Dir, E->d_name);
+ if (E->d_type == DT_REG || E->d_type == DT_LNK ||
+ (E->d_type == DT_UNKNOWN && IsFile(Path)))
+ FileCallback(Path);
+ else if ((E->d_type == DT_DIR ||
+ (E->d_type == DT_UNKNOWN && IsDirectory(Path))) &&
+ *E->d_name != '.')
+ IterateDirRecurisve(Path, DirPreCallback, DirPostCallback, FileCallback);
+ }
+ closedir(D);
+ DirPostCallback(Dir);
+}
+
char GetSeparator() {
return '/';
}
Modified: compiler-rt/trunk/lib/fuzzer/FuzzerIOWindows.cpp
URL: http://llvm.org/viewvc/llvm-project/compiler-rt/trunk/lib/fuzzer/FuzzerIOWindows.cpp?rev=353910&r1=353909&r2=353910&view=diff
==============================================================================
--- compiler-rt/trunk/lib/fuzzer/FuzzerIOWindows.cpp (original)
+++ compiler-rt/trunk/lib/fuzzer/FuzzerIOWindows.cpp Tue Feb 12 20:04:45 2019
@@ -140,6 +140,15 @@ void ListFilesInDirRecursive(const std::
*Epoch = E;
}
+
+void IterateDirRecurisve(const std::string &Dir,
+ void (*DirPreCallback)(const std::string &Dir),
+ void (*DirPostCallback)(const std::string &Dir),
+ void (*FileCallback)(const std::string &Dir)) {
+ // Unimplemented.
+ // TODO: implement, and then implement ListFilesInDirRecursive via this one.
+}
+
char GetSeparator() {
return '\\';
}
Modified: compiler-rt/trunk/lib/fuzzer/FuzzerMerge.cpp
URL: http://llvm.org/viewvc/llvm-project/compiler-rt/trunk/lib/fuzzer/FuzzerMerge.cpp?rev=353910&r1=353909&r2=353910&view=diff
==============================================================================
--- compiler-rt/trunk/lib/fuzzer/FuzzerMerge.cpp (original)
+++ compiler-rt/trunk/lib/fuzzer/FuzzerMerge.cpp Tue Feb 12 20:04:45 2019
@@ -261,6 +261,7 @@ void CrashResistantMerge(const Vector<st
const Set<uint32_t> &InitialFeatures,
Set<uint32_t> *NewFeatures, const std::string &CFPath,
bool V /*Verbose*/) {
+ if (NewCorpus.empty() && OldCorpus.empty()) return; // Nothing to merge.
size_t NumAttempts = 0;
if (FileSize(CFPath)) {
VPrintf(V, "MERGE-OUTER: non-empty control file provided: '%s'\n",
Modified: compiler-rt/trunk/test/fuzzer/merge-control-file.test
URL: http://llvm.org/viewvc/llvm-project/compiler-rt/trunk/test/fuzzer/merge-control-file.test?rev=353910&r1=353909&r2=353910&view=diff
==============================================================================
--- compiler-rt/trunk/test/fuzzer/merge-control-file.test (original)
+++ compiler-rt/trunk/test/fuzzer/merge-control-file.test Tue Feb 12 20:04:45 2019
@@ -12,10 +12,11 @@ RUN: echo ..Z... > %t/T0/3
# Test what happens if the control file is junk.
+RUN: rm -f %t/T1/*; cp %t/T0/* %t/T1
RUN: echo JUNK > %t/MCF
-RUN: not %run %t/T.exe -merge=1 %t/T1 %t/T2 -merge_control_file=%t/MCF 2>&1 | FileCheck %s --check-prefix=JUNK
+RUN: %run %t/T.exe -merge=1 %t/T1 %t/T2 -merge_control_file=%t/MCF 2>&1 | FileCheck %s --check-prefix=JUNK
RUN: echo 3 > %t/MCF; echo 0 >> %t/MCF; echo %t/T1/1 >> %t/MCF
-RUN: not %run %t/T.exe -merge=1 %t/T1 %t/T2 -merge_control_file=%t/MCF 2>&1 | FileCheck %s --check-prefix=JUNK
+RUN: %run %t/T.exe -merge=1 %t/T1 %t/T2 -merge_control_file=%t/MCF 2>&1 | FileCheck %s --check-prefix=JUNK
JUNK: MERGE-OUTER: non-empty control file provided: {{.*}}MCF
JUNK: MERGE-OUTER: bad control file, will overwrite it
More information about the llvm-commits
mailing list