[compiler-rt] r363470 - [libFuzzer] implement a better queue for the fork mode. Add an internal flag -stop_file to allow graceful shutdown of fuzzing. Enhance the logging in the fork mode
Kostya Serebryany via llvm-commits
llvm-commits at lists.llvm.org
Fri Jun 14 15:56:50 PDT 2019
Author: kcc
Date: Fri Jun 14 15:56:50 2019
New Revision: 363470
URL: http://llvm.org/viewvc/llvm-project?rev=363470&view=rev
Log:
[libFuzzer] implement a better queue for the fork mode. Add an internal flag -stop_file to allow graceful shutdown of fuzzing. Enhance the logging in the fork mode
Modified:
compiler-rt/trunk/lib/fuzzer/FuzzerDriver.cpp
compiler-rt/trunk/lib/fuzzer/FuzzerFlags.def
compiler-rt/trunk/lib/fuzzer/FuzzerFork.cpp
compiler-rt/trunk/lib/fuzzer/FuzzerLoop.cpp
compiler-rt/trunk/lib/fuzzer/FuzzerOptions.h
compiler-rt/trunk/test/fuzzer/only-some-bytes-fork.test
Modified: compiler-rt/trunk/lib/fuzzer/FuzzerDriver.cpp
URL: http://llvm.org/viewvc/llvm-project/compiler-rt/trunk/lib/fuzzer/FuzzerDriver.cpp?rev=363470&r1=363469&r2=363470&view=diff
==============================================================================
--- compiler-rt/trunk/lib/fuzzer/FuzzerDriver.cpp (original)
+++ compiler-rt/trunk/lib/fuzzer/FuzzerDriver.cpp Fri Jun 14 15:56:50 2019
@@ -709,6 +709,8 @@ int FuzzerDriver(int *argc, char ***argv
if (Flags.collect_data_flow)
Options.CollectDataFlow = Flags.collect_data_flow;
Options.LazyCounters = Flags.lazy_counters;
+ if (Flags.stop_file)
+ Options.StopFile = Flags.stop_file;
unsigned Seed = Flags.seed;
// Initialize Seed.
Modified: compiler-rt/trunk/lib/fuzzer/FuzzerFlags.def
URL: http://llvm.org/viewvc/llvm-project/compiler-rt/trunk/lib/fuzzer/FuzzerFlags.def?rev=363470&r1=363469&r2=363470&view=diff
==============================================================================
--- compiler-rt/trunk/lib/fuzzer/FuzzerFlags.def (original)
+++ compiler-rt/trunk/lib/fuzzer/FuzzerFlags.def Fri Jun 14 15:56:50 2019
@@ -50,6 +50,7 @@ FUZZER_FLAG_INT(ignore_crashes, 0, "Igno
FUZZER_FLAG_INT(merge, 0, "If 1, the 2-nd, 3-rd, etc corpora will be "
"merged into the 1-st corpus. Only interesting units will be taken. "
"This flag can be used to minimize a corpus.")
+FUZZER_FLAG_STRING(stop_file, "Stop fuzzing ASAP if this file exists")
FUZZER_FLAG_STRING(merge_inner, "internal flag")
FUZZER_FLAG_STRING(merge_control_file,
"Specify a control file used for the merge process. "
Modified: compiler-rt/trunk/lib/fuzzer/FuzzerFork.cpp
URL: http://llvm.org/viewvc/llvm-project/compiler-rt/trunk/lib/fuzzer/FuzzerFork.cpp?rev=363470&r1=363469&r2=363470&view=diff
==============================================================================
--- compiler-rt/trunk/lib/fuzzer/FuzzerFork.cpp (original)
+++ compiler-rt/trunk/lib/fuzzer/FuzzerFork.cpp Fri Jun 14 15:56:50 2019
@@ -19,6 +19,7 @@
#include <atomic>
#include <chrono>
+#include <condition_variable>
#include <fstream>
#include <memory>
#include <mutex>
@@ -68,6 +69,9 @@ struct FuzzJob {
std::string LogPath;
std::string SeedListPath;
std::string CFPath;
+ size_t JobId;
+
+ int DftTimeInSeconds = 0;
// Fuzzing Outputs.
int ExitCode;
@@ -102,6 +106,8 @@ struct GlobalEnv {
size_t NumRuns = 0;
+ std::string StopFile() { return DirPlusFile(TempDir, "STOP"); }
+
size_t secondsSinceProcessStartUp() const {
return std::chrono::duration_cast<std::chrono::seconds>(
std::chrono::system_clock::now() - ProcessStartTime)
@@ -119,6 +125,7 @@ struct GlobalEnv {
Cmd.addFlag("print_final_stats", "1");
Cmd.addFlag("print_funcs", "0"); // no need to spend time symbolizing.
Cmd.addFlag("max_total_time", std::to_string(std::min((size_t)300, JobId)));
+ Cmd.addFlag("stop_file", StopFile());
if (!DataFlowBinary.empty()) {
Cmd.addFlag("data_flow_trace", DFTDir);
if (!Cmd.hasFlag("focus_function"))
@@ -128,11 +135,14 @@ struct GlobalEnv {
std::string Seeds;
if (size_t CorpusSubsetSize =
std::min(Files.size(), (size_t)sqrt(Files.size() + 2))) {
+ auto Time1 = std::chrono::system_clock::now();
for (size_t i = 0; i < CorpusSubsetSize; i++) {
auto &SF = Files[Rand->SkewTowardsLast(Files.size())];
Seeds += (Seeds.empty() ? "" : ",") + SF;
CollectDFT(SF);
}
+ auto Time2 = std::chrono::system_clock::now();
+ Job->DftTimeInSeconds = duration_cast<seconds>(Time2 - Time1).count();
}
if (!Seeds.empty()) {
Job->SeedListPath =
@@ -144,6 +154,7 @@ struct GlobalEnv {
Job->CorpusDir = DirPlusFile(TempDir, "C" + std::to_string(JobId));
Job->FeaturesDir = DirPlusFile(TempDir, "F" + std::to_string(JobId));
Job->CFPath = DirPlusFile(TempDir, std::to_string(JobId) + ".merge");
+ Job->JobId = JobId;
Cmd.addArgument(Job->CorpusDir);
@@ -189,6 +200,13 @@ struct GlobalEnv {
}
}
}
+ // if (!FilesToAdd.empty() || Job->ExitCode != 0)
+ Printf("#%zd: cov: %zd ft: %zd corp: %zd exec/s %zd "
+ "oom/timeout/crash: %zd/%zd/%zd time: %zds job: %zd dft_time: %d\n",
+ NumRuns, Cov.size(), Features.size(), Files.size(),
+ Stats.average_exec_per_sec, NumOOMs, NumTimeouts, NumCrashes,
+ secondsSinceProcessStartUp(), Job->JobId, Job->DftTimeInSeconds);
+
if (MergeCandidates.empty()) return;
Vector<std::string> FilesToAdd;
@@ -209,12 +227,6 @@ struct GlobalEnv {
PrintPC(" NEW_FUNC: %p %F %L\n", "",
TPC.GetNextInstructionPc(TE->PC));
- if (!FilesToAdd.empty() || Job->ExitCode != 0)
- Printf("#%zd: cov: %zd ft: %zd corp: %zd exec/s %zd "
- "oom/timeout/crash: %zd/%zd/%zd time: %zds\n", NumRuns,
- Cov.size(), Features.size(), Files.size(),
- Stats.average_exec_per_sec,
- NumOOMs, NumTimeouts, NumCrashes, secondsSinceProcessStartUp());
}
@@ -239,28 +251,29 @@ struct GlobalEnv {
struct JobQueue {
std::queue<FuzzJob *> Qu;
std::mutex Mu;
+ std::condition_variable Cv;
void Push(FuzzJob *Job) {
- std::lock_guard<std::mutex> Lock(Mu);
- Qu.push(Job);
+ {
+ std::lock_guard<std::mutex> Lock(Mu);
+ Qu.push(Job);
+ }
+ Cv.notify_one();
}
FuzzJob *Pop() {
- std::lock_guard<std::mutex> Lock(Mu);
- if (Qu.empty()) return nullptr;
+ std::unique_lock<std::mutex> Lk(Mu);
+ // std::lock_guard<std::mutex> Lock(Mu);
+ Cv.wait(Lk, [&]{return !Qu.empty();});
+ assert(!Qu.empty());
auto Job = Qu.front();
Qu.pop();
return Job;
}
};
-void WorkerThread(std::atomic<bool> *Stop, JobQueue *FuzzQ, JobQueue *MergeQ) {
- while (!Stop->load()) {
- auto Job = FuzzQ->Pop();
+void WorkerThread(JobQueue *FuzzQ, JobQueue *MergeQ) {
+ while (auto Job = FuzzQ->Pop()) {
// Printf("WorkerThread: job %p\n", Job);
- if (!Job) {
- SleepSeconds(1);
- continue;
- }
Job->ExitCode = ExecuteCommand(Job->Cmd);
MergeQ->Push(Job);
}
@@ -307,27 +320,29 @@ void FuzzWithFork(Random &Rand, const Fu
int ExitCode = 0;
JobQueue FuzzQ, MergeQ;
- std::atomic<bool> Stop(false);
+
+ auto StopJobs = [&]() {
+ for (int i = 0; i < NumJobs; i++)
+ FuzzQ.Push(nullptr);
+ MergeQ.Push(nullptr);
+ WriteToFile(Unit({1}), Env.StopFile());
+ };
size_t JobId = 1;
Vector<std::thread> Threads;
for (int t = 0; t < NumJobs; t++) {
- Threads.push_back(std::thread(WorkerThread, &Stop, &FuzzQ, &MergeQ));
+ Threads.push_back(std::thread(WorkerThread, &FuzzQ, &MergeQ));
FuzzQ.Push(Env.CreateNewJob(JobId++));
}
while (true) {
std::unique_ptr<FuzzJob> Job(MergeQ.Pop());
- if (!Job) {
- if (Stop)
- break;
- SleepSeconds(1);
- continue;
- }
+ if (!Job)
+ break;
ExitCode = Job->ExitCode;
if (ExitCode == Options.InterruptExitCode) {
Printf("==%lu== libFuzzer: a child was interrupted; exiting\n", GetPid());
- Stop = true;
+ StopJobs();
break;
}
Fuzzer::MaybeExitGracefully();
@@ -352,7 +367,8 @@ void FuzzWithFork(Random &Rand, const Fu
// And exit if we don't ignore this crash.
Printf("INFO: log from the inner process:\n%s",
FileToString(Job->LogPath).c_str());
- Stop = true;
+ StopJobs();
+ break;
}
}
@@ -360,22 +376,22 @@ void FuzzWithFork(Random &Rand, const Fu
// This is not precise, since other threads are still running
// and we will wait while joining them.
// We also don't stop instantly: other jobs need to finish.
- if (Options.MaxTotalTimeSec > 0 && !Stop &&
+ if (Options.MaxTotalTimeSec > 0 &&
Env.secondsSinceProcessStartUp() >= (size_t)Options.MaxTotalTimeSec) {
Printf("INFO: fuzzed for %zd seconds, wrapping up soon\n",
Env.secondsSinceProcessStartUp());
- Stop = true;
+ StopJobs();
+ break;
}
- if (!Stop && Env.NumRuns >= Options.MaxNumberOfRuns) {
+ if (Env.NumRuns >= Options.MaxNumberOfRuns) {
Printf("INFO: fuzzed for %zd iterations, wrapping up soon\n",
Env.NumRuns);
- Stop = true;
+ StopJobs();
+ break;
}
- if (!Stop)
- FuzzQ.Push(Env.CreateNewJob(JobId++));
+ FuzzQ.Push(Env.CreateNewJob(JobId++));
}
- Stop = true;
for (auto &T : Threads)
T.join();
Modified: compiler-rt/trunk/lib/fuzzer/FuzzerLoop.cpp
URL: http://llvm.org/viewvc/llvm-project/compiler-rt/trunk/lib/fuzzer/FuzzerLoop.cpp?rev=363470&r1=363469&r2=363470&view=diff
==============================================================================
--- compiler-rt/trunk/lib/fuzzer/FuzzerLoop.cpp (original)
+++ compiler-rt/trunk/lib/fuzzer/FuzzerLoop.cpp Fri Jun 14 15:56:50 2019
@@ -801,6 +801,9 @@ void Fuzzer::Loop(Vector<SizedFile> &Cor
while (true) {
auto Now = system_clock::now();
+ if (!Options.StopFile.empty() &&
+ !FileToVector(Options.StopFile, 1, false).empty())
+ break;
if (duration_cast<seconds>(Now - LastCorpusReload).count() >=
Options.ReloadIntervalSec) {
RereadOutputCorpus(MaxInputLen);
Modified: compiler-rt/trunk/lib/fuzzer/FuzzerOptions.h
URL: http://llvm.org/viewvc/llvm-project/compiler-rt/trunk/lib/fuzzer/FuzzerOptions.h?rev=363470&r1=363469&r2=363470&view=diff
==============================================================================
--- compiler-rt/trunk/lib/fuzzer/FuzzerOptions.h (original)
+++ compiler-rt/trunk/lib/fuzzer/FuzzerOptions.h Fri Jun 14 15:56:50 2019
@@ -53,6 +53,7 @@ struct FuzzingOptions {
std::string DataFlowTrace;
std::string CollectDataFlow;
std::string FeaturesDir;
+ std::string StopFile;
bool SaveArtifacts = true;
bool PrintNEW = true; // Print a status line when new units are found;
bool PrintNewCovPcs = false;
Modified: compiler-rt/trunk/test/fuzzer/only-some-bytes-fork.test
URL: http://llvm.org/viewvc/llvm-project/compiler-rt/trunk/test/fuzzer/only-some-bytes-fork.test?rev=363470&r1=363469&r2=363470&view=diff
==============================================================================
--- compiler-rt/trunk/test/fuzzer/only-some-bytes-fork.test (original)
+++ compiler-rt/trunk/test/fuzzer/only-some-bytes-fork.test Fri Jun 14 15:56:50 2019
@@ -8,5 +8,5 @@ RUN: %cpp_compiler %S/OnlySomeBytesTest.
# Test that the fork mode can collect and use the DFT
RUN: rm -rf %t && mkdir %t
-RUN: not %t-Fuzz -collect_data_flow=%t-DFT -use_value_profile=1 -runs=100000000 -fork=1 2> %t/log
+RUN: not %t-Fuzz -collect_data_flow=%t-DFT -use_value_profile=1 -runs=100000000 -fork=20 2> %t/log
RUN: grep BINGO %t/log
More information about the llvm-commits
mailing list