[compiler-rt] r353775 - [libFuzzer] extend the -fork=1 functionality. Still not fully usable, but good enough for the first unit test
Kostya Serebryany via llvm-commits
llvm-commits at lists.llvm.org
Mon Feb 11 16:12:33 PST 2019
Author: kcc
Date: Mon Feb 11 16:12:33 2019
New Revision: 353775
URL: http://llvm.org/viewvc/llvm-project?rev=353775&view=rev
Log:
[libFuzzer] extend the -fork=1 functionality. Still not fully usable, but good enough for the first unit test
Modified:
compiler-rt/trunk/lib/fuzzer/FuzzerDriver.cpp
compiler-rt/trunk/lib/fuzzer/FuzzerFlags.def
compiler-rt/trunk/lib/fuzzer/FuzzerIO.cpp
compiler-rt/trunk/lib/fuzzer/FuzzerIO.h
compiler-rt/trunk/lib/fuzzer/FuzzerIOPosix.cpp
compiler-rt/trunk/lib/fuzzer/FuzzerIOWindows.cpp
compiler-rt/trunk/lib/fuzzer/FuzzerInternal.h
compiler-rt/trunk/lib/fuzzer/FuzzerLoop.cpp
compiler-rt/trunk/lib/fuzzer/FuzzerMerge.cpp
compiler-rt/trunk/lib/fuzzer/FuzzerRandom.h
compiler-rt/trunk/lib/fuzzer/FuzzerUtilLinux.cpp
compiler-rt/trunk/test/fuzzer/simple.test
Modified: compiler-rt/trunk/lib/fuzzer/FuzzerDriver.cpp
URL: http://llvm.org/viewvc/llvm-project/compiler-rt/trunk/lib/fuzzer/FuzzerDriver.cpp?rev=353775&r1=353774&r2=353775&view=diff
==============================================================================
--- compiler-rt/trunk/lib/fuzzer/FuzzerDriver.cpp (original)
+++ compiler-rt/trunk/lib/fuzzer/FuzzerDriver.cpp Mon Feb 11 16:12:33 2019
@@ -472,46 +472,76 @@ int MinimizeCrashInputInternalStep(Fuzze
}
// This is just a skeleton of an experimental -fork=1 feature.
-void FuzzWithFork(const FuzzingOptions &Options,
+void FuzzWithFork(Fuzzer *F, const FuzzingOptions &Options,
const Vector<std::string> &Args,
const Vector<std::string> &Corpora) {
Printf("INFO: -fork=1: doing fuzzing in a separate process in order to "
"be more resistant to crashes, timeouts, and OOMs\n");
+ auto Rand = F->GetMD().GetRand();
Vector<SizedFile> Corpus;
for (auto &Dir : Corpora)
GetSizedFilesFromDir(Dir, &Corpus);
std::sort(Corpus.begin(), Corpus.end());
+ auto CFPath = TempPath(".fork");
Vector<std::string> Files;
Set<uint32_t> Features;
if (!Corpus.empty()) {
- auto CFPath = TempPath(".fork");
CrashResistantMerge(Args, {}, Corpus, &Files, {}, &Features, CFPath);
RemoveFile(CFPath);
}
- Printf("INFO: -fork=1: %zd seeds, starting to fuzz\n", Files.size());
+ auto TempDir = TempPath("Dir");
+ MkDir(TempDir);
+ Printf("INFO: -fork=1: %zd seeds, starting to fuzz; scratch: %s\n",
+ Files.size(), TempDir.c_str());
- Command Cmd(Args);
- Cmd.removeFlag("fork");
+ Command BaseCmd(Args);
+ BaseCmd.removeFlag("fork");
for (auto &C : Corpora) // Remove all corpora from the args.
- Cmd.removeArgument(C);
- if (Files.size() >= 2)
- Cmd.addFlag("seed_inputs",
- Files.back() + "," + Files[Files.size() - 2]);
- Cmd.addFlag("runs", "1000000");
- Cmd.addFlag("max_total_time", "30");
- for (size_t i = 0; i < 1000; i++) {
+ BaseCmd.removeArgument(C);
+ BaseCmd.addFlag("runs", "1000000");
+ BaseCmd.addFlag("max_total_time", "30");
+ BaseCmd.addArgument(TempDir);
+ int ExitCode = 0;
+ for (size_t i = 0; i < 1000000; i++) {
+ // TODO: take new files from disk e.g. those generated by another process.
+ Command Cmd(BaseCmd);
+ if (Files.size() >= 2)
+ Cmd.addFlag("seed_inputs",
+ Files[Rand.SkewTowardsLast(Files.size())] + "," +
+ Files[Rand.SkewTowardsLast(Files.size())]);
Printf("RUN %s\n", Cmd.toString().c_str());
- int ExitCode = ExecuteCommand(Cmd);
+ RmFilesInDir(TempDir);
+ ExitCode = ExecuteCommand(Cmd);
+ Printf("Exit code: %d\n", ExitCode);
if (ExitCode == Options.InterruptExitCode)
- exit(0);
- if (ExitCode == Options.TimeoutExitCode || ExitCode == Options.OOMExitCode)
- continue;
+ break;
+ Vector<SizedFile> TempFiles;
+ Vector<std::string>FilesToAdd;
+ Set<uint32_t> NewFeatures;
+ GetSizedFilesFromDir(TempDir, &TempFiles);
+ CrashResistantMerge(Args, {}, TempFiles, &FilesToAdd, Features,
+ &NewFeatures, CFPath);
+ RemoveFile(CFPath);
+ for (auto &Path : FilesToAdd) {
+ auto NewPath = F->WriteToOutputCorpus(FileToVector(Path, Options.MaxLen));
+ if (!NewPath.empty())
+ Files.push_back(NewPath);
+ }
+ Features.insert(NewFeatures.begin(), NewFeatures.end());
+ Printf("INFO: temp_files: %zd files_added: %zd newft: %zd ft: %zd\n",
+ TempFiles.size(), FilesToAdd.size(), NewFeatures.size(),
+ Features.size());
if (ExitCode != 0) break;
}
- exit(0);
+ RmFilesInDir(TempDir);
+ RmDir(TempDir);
+
+ // Use the exit code from the last child process.
+ Printf("Fork: exiting: %d\n", ExitCode);
+ exit(ExitCode);
}
void Merge(Fuzzer *F, FuzzingOptions &Options, const Vector<std::string> &Args,
@@ -770,7 +800,7 @@ int FuzzerDriver(int *argc, char ***argv
}
if (Flags.fork)
- FuzzWithFork(Options, Args, *Inputs);
+ FuzzWithFork(F, Options, Args, *Inputs);
if (Flags.merge)
Merge(F, Options, Args, *Inputs, Flags.merge_control_file);
Modified: compiler-rt/trunk/lib/fuzzer/FuzzerFlags.def
URL: http://llvm.org/viewvc/llvm-project/compiler-rt/trunk/lib/fuzzer/FuzzerFlags.def?rev=353775&r1=353774&r2=353775&view=diff
==============================================================================
--- compiler-rt/trunk/lib/fuzzer/FuzzerFlags.def (original)
+++ compiler-rt/trunk/lib/fuzzer/FuzzerFlags.def Mon Feb 11 16:12:33 2019
@@ -36,7 +36,7 @@ FUZZER_FLAG_INT(
"If one unit runs more than this number of seconds the process will abort.")
FUZZER_FLAG_INT(error_exitcode, 77, "When libFuzzer itself reports a bug "
"this exit code will be used.")
-FUZZER_FLAG_INT(timeout_exitcode, 77, "When libFuzzer reports a timeout "
+FUZZER_FLAG_INT(timeout_exitcode, 70, "When libFuzzer reports a timeout "
"this exit code will be used.")
FUZZER_FLAG_INT(max_total_time, 0, "If positive, indicates the maximal total "
"time in seconds to run the fuzzer.")
Modified: compiler-rt/trunk/lib/fuzzer/FuzzerIO.cpp
URL: http://llvm.org/viewvc/llvm-project/compiler-rt/trunk/lib/fuzzer/FuzzerIO.cpp?rev=353775&r1=353774&r2=353775&view=diff
==============================================================================
--- compiler-rt/trunk/lib/fuzzer/FuzzerIO.cpp (original)
+++ compiler-rt/trunk/lib/fuzzer/FuzzerIO.cpp Mon Feb 11 16:12:33 2019
@@ -125,4 +125,11 @@ void Printf(const char *Fmt, ...) {
fflush(OutputFile);
}
+void RmFilesInDir(const std::string &Path) {
+ Vector<std::string> Files;
+ ListFilesInDirRecursive(Path, 0, &Files, /*TopDir*/true);
+ for (auto &F : Files)
+ RemoveFile(F);
+}
+
} // namespace fuzzer
Modified: compiler-rt/trunk/lib/fuzzer/FuzzerIO.h
URL: http://llvm.org/viewvc/llvm-project/compiler-rt/trunk/lib/fuzzer/FuzzerIO.h?rev=353775&r1=353774&r2=353775&view=diff
==============================================================================
--- compiler-rt/trunk/lib/fuzzer/FuzzerIO.h (original)
+++ compiler-rt/trunk/lib/fuzzer/FuzzerIO.h Mon Feb 11 16:12:33 2019
@@ -81,6 +81,10 @@ void DiscardOutput(int Fd);
intptr_t GetHandleFromFd(int fd);
+void MkDir(const std::string &Path);
+void RmDir(const std::string &Path);
+void RmFilesInDir(const std::string &Path);
+
} // namespace fuzzer
#endif // LLVM_FUZZER_IO_H
Modified: compiler-rt/trunk/lib/fuzzer/FuzzerIOPosix.cpp
URL: http://llvm.org/viewvc/llvm-project/compiler-rt/trunk/lib/fuzzer/FuzzerIOPosix.cpp?rev=353775&r1=353774&r2=353775&view=diff
==============================================================================
--- compiler-rt/trunk/lib/fuzzer/FuzzerIOPosix.cpp (original)
+++ compiler-rt/trunk/lib/fuzzer/FuzzerIOPosix.cpp Mon Feb 11 16:12:33 2019
@@ -136,11 +136,18 @@ bool IsInterestingCoverageFile(const std
return true;
}
-
void RawPrint(const char *Str) {
write(2, Str, strlen(Str));
}
+void MkDir(const std::string &Path) {
+ mkdir(Path.c_str(), 0700);
+}
+
+void RmDir(const std::string &Path) {
+ rmdir(Path.c_str());
+}
+
} // namespace fuzzer
#endif // LIBFUZZER_POSIX
Modified: compiler-rt/trunk/lib/fuzzer/FuzzerIOWindows.cpp
URL: http://llvm.org/viewvc/llvm-project/compiler-rt/trunk/lib/fuzzer/FuzzerIOWindows.cpp?rev=353775&r1=353774&r2=353775&view=diff
==============================================================================
--- compiler-rt/trunk/lib/fuzzer/FuzzerIOWindows.cpp (original)
+++ compiler-rt/trunk/lib/fuzzer/FuzzerIOWindows.cpp Mon Feb 11 16:12:33 2019
@@ -336,6 +336,14 @@ void RawPrint(const char *Str) {
_write(2, Str, strlen(Str));
}
+void MkDir(const std::string &Path) {
+ Printf("MkDir: unimplemented\n");
+}
+
+void RmDir(const std::string &Path) {
+ Printf("RmDir: unimplemented\n");
+}
+
} // namespace fuzzer
#endif // LIBFUZZER_WINDOWS
Modified: compiler-rt/trunk/lib/fuzzer/FuzzerInternal.h
URL: http://llvm.org/viewvc/llvm-project/compiler-rt/trunk/lib/fuzzer/FuzzerInternal.h?rev=353775&r1=353774&r2=353775&view=diff
==============================================================================
--- compiler-rt/trunk/lib/fuzzer/FuzzerInternal.h (original)
+++ compiler-rt/trunk/lib/fuzzer/FuzzerInternal.h Mon Feb 11 16:12:33 2019
@@ -87,7 +87,7 @@ public:
void HandleMalloc(size_t Size);
static void MaybeExitGracefully();
- void WriteToOutputCorpus(const Unit &U);
+ std::string WriteToOutputCorpus(const Unit &U);
private:
void AlarmCallback();
Modified: compiler-rt/trunk/lib/fuzzer/FuzzerLoop.cpp
URL: http://llvm.org/viewvc/llvm-project/compiler-rt/trunk/lib/fuzzer/FuzzerLoop.cpp?rev=353775&r1=353774&r2=353775&view=diff
==============================================================================
--- compiler-rt/trunk/lib/fuzzer/FuzzerLoop.cpp (original)
+++ compiler-rt/trunk/lib/fuzzer/FuzzerLoop.cpp Mon Feb 11 16:12:33 2019
@@ -537,15 +537,16 @@ void Fuzzer::ExecuteCallback(const uint8
delete[] DataCopy;
}
-void Fuzzer::WriteToOutputCorpus(const Unit &U) {
+std::string Fuzzer::WriteToOutputCorpus(const Unit &U) {
if (Options.OnlyASCII)
assert(IsASCII(U));
if (Options.OutputCorpus.empty())
- return;
+ return "";
std::string Path = DirPlusFile(Options.OutputCorpus, Hash(U));
WriteToFile(U, Path);
if (Options.Verbosity >= 2)
Printf("Written %zd bytes to %s\n", U.size(), Path.c_str());
+ return Path;
}
void Fuzzer::WriteUnitToFileWithPrefix(const Unit &U, const char *Prefix) {
Modified: compiler-rt/trunk/lib/fuzzer/FuzzerMerge.cpp
URL: http://llvm.org/viewvc/llvm-project/compiler-rt/trunk/lib/fuzzer/FuzzerMerge.cpp?rev=353775&r1=353774&r2=353775&view=diff
==============================================================================
--- compiler-rt/trunk/lib/fuzzer/FuzzerMerge.cpp (original)
+++ compiler-rt/trunk/lib/fuzzer/FuzzerMerge.cpp Mon Feb 11 16:12:33 2019
@@ -120,28 +120,28 @@ size_t Merger::ApproximateMemoryConsumpt
return Res;
}
-// Decides which files need to be merged (add thost to NewFiles).
+// Decides which files need to be merged (add those to NewFiles).
// Returns the number of new features added.
size_t Merger::Merge(const Set<uint32_t> &InitialFeatures,
- Set<uint32_t> *AllFeatures,
+ Set<uint32_t> *NewFeatures,
Vector<std::string> *NewFiles) {
NewFiles->clear();
assert(NumFilesInFirstCorpus <= Files.size());
- *AllFeatures = InitialFeatures;
+ Set<uint32_t> AllFeatures = InitialFeatures;
// What features are in the initial corpus?
for (size_t i = 0; i < NumFilesInFirstCorpus; i++) {
auto &Cur = Files[i].Features;
- AllFeatures->insert(Cur.begin(), Cur.end());
+ AllFeatures.insert(Cur.begin(), Cur.end());
}
- size_t InitialNumFeatures = AllFeatures->size();
+ size_t InitialNumFeatures = AllFeatures.size();
// Remove all features that we already know from all other inputs.
for (size_t i = NumFilesInFirstCorpus; i < Files.size(); i++) {
auto &Cur = Files[i].Features;
Vector<uint32_t> Tmp;
- std::set_difference(Cur.begin(), Cur.end(), AllFeatures->begin(),
- AllFeatures->end(), std::inserter(Tmp, Tmp.begin()));
+ std::set_difference(Cur.begin(), Cur.end(), AllFeatures.begin(),
+ AllFeatures.end(), std::inserter(Tmp, Tmp.begin()));
Cur.swap(Tmp);
}
@@ -161,12 +161,17 @@ size_t Merger::Merge(const Set<uint32_t>
auto &Cur = Files[i].Features;
// Printf("%s -> sz %zd ft %zd\n", Files[i].Name.c_str(),
// Files[i].Size, Cur.size());
- size_t OldSize = AllFeatures->size();
- AllFeatures->insert(Cur.begin(), Cur.end());
- if (AllFeatures->size() > OldSize)
+ bool FoundNewFeatures = false;
+ for (auto Fe: Cur) {
+ if (AllFeatures.insert(Fe).second) {
+ FoundNewFeatures = true;
+ NewFeatures->insert(Fe);
+ }
+ }
+ if (FoundNewFeatures)
NewFiles->push_back(Files[i].Name);
}
- return AllFeatures->size() - InitialNumFeatures;
+ return AllFeatures.size() - InitialNumFeatures;
}
Set<uint32_t> Merger::AllFeatures() const {
Modified: compiler-rt/trunk/lib/fuzzer/FuzzerRandom.h
URL: http://llvm.org/viewvc/llvm-project/compiler-rt/trunk/lib/fuzzer/FuzzerRandom.h?rev=353775&r1=353774&r2=353775&view=diff
==============================================================================
--- compiler-rt/trunk/lib/fuzzer/FuzzerRandom.h (original)
+++ compiler-rt/trunk/lib/fuzzer/FuzzerRandom.h Mon Feb 11 16:12:33 2019
@@ -20,6 +20,11 @@ class Random : public std::mt19937 {
result_type operator()() { return this->std::mt19937::operator()(); }
size_t Rand() { return this->operator()(); }
size_t RandBool() { return Rand() % 2; }
+ size_t SkewTowardsLast(size_t n) {
+ size_t T = this->operator()(n * n);
+ size_t Res = sqrt(T);
+ return Res;
+ }
size_t operator()(size_t n) { return n ? Rand() % n : 0; }
intptr_t operator()(intptr_t From, intptr_t To) {
assert(From < To);
Modified: compiler-rt/trunk/lib/fuzzer/FuzzerUtilLinux.cpp
URL: http://llvm.org/viewvc/llvm-project/compiler-rt/trunk/lib/fuzzer/FuzzerUtilLinux.cpp?rev=353775&r1=353774&r2=353775&view=diff
==============================================================================
--- compiler-rt/trunk/lib/fuzzer/FuzzerUtilLinux.cpp (original)
+++ compiler-rt/trunk/lib/fuzzer/FuzzerUtilLinux.cpp Mon Feb 11 16:12:33 2019
@@ -13,12 +13,18 @@
#include "FuzzerCommand.h"
#include <stdlib.h>
+#include <sys/types.h>
+#include <sys/wait.h>
+
namespace fuzzer {
int ExecuteCommand(const Command &Cmd) {
std::string CmdLine = Cmd.toString();
- return system(CmdLine.c_str());
+ int exit_code = system(CmdLine.c_str());
+ if (WIFEXITED(exit_code))
+ return WEXITSTATUS(exit_code);
+ return exit_code;
}
} // namespace fuzzer
Modified: compiler-rt/trunk/test/fuzzer/simple.test
URL: http://llvm.org/viewvc/llvm-project/compiler-rt/trunk/test/fuzzer/simple.test?rev=353775&r1=353774&r2=353775&view=diff
==============================================================================
--- compiler-rt/trunk/test/fuzzer/simple.test (original)
+++ compiler-rt/trunk/test/fuzzer/simple.test Mon Feb 11 16:12:33 2019
@@ -1,7 +1,8 @@
CHECK: BINGO
RUN: %cpp_compiler %S/SimpleTest.cpp -o %t-SimpleTest
-RUN: not %run %t-SimpleTest 2>&1 | FileCheck %s
+RUN: not %run %t-SimpleTest 2>&1 | FileCheck %s
+RUN: not %run %t-SimpleTest -fork=1 2>&1 | FileCheck %s
# only_ascii mode. Will perform some minimal self-validation.
RUN: not %run %t-SimpleTest -only_ascii=1 2>&1
More information about the llvm-commits
mailing list