[compiler-rt] r317767 - [libFuzzer] allow merge to resume after being preempted
Kostya Serebryany via llvm-commits
llvm-commits at lists.llvm.org
Wed Nov 8 21:49:28 PST 2017
Author: kcc
Date: Wed Nov 8 21:49:28 2017
New Revision: 317767
URL: http://llvm.org/viewvc/llvm-project?rev=317767&view=rev
Log:
[libFuzzer] allow merge to resume after being preempted
Added:
compiler-rt/trunk/test/fuzzer/merge-control-file.test
Modified:
compiler-rt/trunk/lib/fuzzer/FuzzerFlags.def
compiler-rt/trunk/lib/fuzzer/FuzzerMerge.cpp
Modified: compiler-rt/trunk/lib/fuzzer/FuzzerFlags.def
URL: http://llvm.org/viewvc/llvm-project/compiler-rt/trunk/lib/fuzzer/FuzzerFlags.def?rev=317767&r1=317766&r2=317767&view=diff
==============================================================================
--- compiler-rt/trunk/lib/fuzzer/FuzzerFlags.def (original)
+++ compiler-rt/trunk/lib/fuzzer/FuzzerFlags.def Wed Nov 8 21:49:28 2017
@@ -41,6 +41,8 @@ FUZZER_FLAG_INT(merge, 0, "If 1, the 2-n
FUZZER_FLAG_STRING(merge_inner, "internal flag")
FUZZER_FLAG_STRING(merge_control_file,
"Specify a control file used for the merge proccess. "
+ "If a merge process gets killed it tries to leave this file "
+ "in a state suitable for resuming the merge. "
"By default a temporary file will be used.")
FUZZER_FLAG_STRING(save_coverage_summary, "Experimental:"
" save coverage summary to a given file."
Modified: compiler-rt/trunk/lib/fuzzer/FuzzerMerge.cpp
URL: http://llvm.org/viewvc/llvm-project/compiler-rt/trunk/lib/fuzzer/FuzzerMerge.cpp?rev=317767&r1=317766&r2=317767&view=diff
==============================================================================
--- compiler-rt/trunk/lib/fuzzer/FuzzerMerge.cpp (original)
+++ compiler-rt/trunk/lib/fuzzer/FuzzerMerge.cpp Wed Nov 8 21:49:28 2017
@@ -256,6 +256,22 @@ void Fuzzer::CrashResistantMergeInternal
}
}
+static void WriteNewControlFile(const std::string &CFPath,
+ const Vector<SizedFile> &AllFiles,
+ size_t NumFilesInFirstCorpus) {
+ RemoveFile(CFPath);
+ std::ofstream ControlFile(CFPath);
+ ControlFile << AllFiles.size() << "\n";
+ ControlFile << NumFilesInFirstCorpus << "\n";
+ for (auto &SF: AllFiles)
+ ControlFile << SF.File << "\n";
+ if (!ControlFile) {
+ Printf("MERGE-OUTER: failed to write to the control file: %s\n",
+ CFPath.c_str());
+ exit(1);
+ }
+}
+
// Outer process. Does not call the target code and thus sohuld not fail.
void Fuzzer::CrashResistantMerge(const Vector<std::string> &Args,
const Vector<std::string> &Corpora,
@@ -266,46 +282,64 @@ void Fuzzer::CrashResistantMerge(const V
Printf("Merge requires two or more corpus dirs\n");
return;
}
- Vector<SizedFile> AllFiles;
- GetSizedFilesFromDir(Corpora[0], &AllFiles);
- size_t NumFilesInFirstCorpus = AllFiles.size();
- std::sort(AllFiles.begin(), AllFiles.end());
- for (size_t i = 1; i < Corpora.size(); i++)
- GetSizedFilesFromDir(Corpora[i], &AllFiles);
- std::sort(AllFiles.begin() + NumFilesInFirstCorpus, AllFiles.end());
- Printf("MERGE-OUTER: %zd files, %zd in the initial corpus\n",
- AllFiles.size(), NumFilesInFirstCorpus);
auto CFPath =
MergeControlFilePathOrNull
? MergeControlFilePathOrNull
: DirPlusFile(TmpDir(),
"libFuzzerTemp." + std::to_string(GetPid()) + ".txt");
- // Write the control file.
- RemoveFile(CFPath);
- std::ofstream ControlFile(CFPath);
- ControlFile << AllFiles.size() << "\n";
- ControlFile << NumFilesInFirstCorpus << "\n";
- for (auto &SF: AllFiles)
- ControlFile << SF.File << "\n";
- if (!ControlFile) {
- Printf("MERGE-OUTER: failed to write to the control file: %s\n",
- CFPath.c_str());
- exit(1);
+
+ size_t NumAttempts = 0;
+ if (MergeControlFilePathOrNull && FileSize(MergeControlFilePathOrNull)) {
+ Printf("MERGE-OUTER: non-empty control file provided: '%s'\n",
+ MergeControlFilePathOrNull);
+ Merger M;
+ std::ifstream IF(MergeControlFilePathOrNull);
+ if (M.Parse(IF, /*ParseCoverage=*/false)) {
+ Printf("MERGE-OUTER: control file ok, %zd files total,"
+ " first not processed file %zd\n",
+ M.Files.size(), M.FirstNotProcessedFile);
+ if (!M.LastFailure.empty())
+ Printf("MERGE-OUTER: '%s' will be skipped as unlucky "
+ "(merge has stumbled on it the last time)\n",
+ M.LastFailure.c_str());
+ if (M.FirstNotProcessedFile >= M.Files.size()) {
+ Printf("MERGE-OUTER: nothing to do, merge has been completed before\n");
+ exit(0);
+ }
+
+ NumAttempts = M.Files.size() - M.FirstNotProcessedFile;
+ } else {
+ Printf("MERGE-OUTER: bad control file, will overwrite it\n");
+ }
+ }
+
+ if (!NumAttempts) {
+ // The supplied control file is empty or bad, create a fresh one.
+ Vector<SizedFile> AllFiles;
+ GetSizedFilesFromDir(Corpora[0], &AllFiles);
+ size_t NumFilesInFirstCorpus = AllFiles.size();
+ std::sort(AllFiles.begin(), AllFiles.end());
+ for (size_t i = 1; i < Corpora.size(); i++)
+ GetSizedFilesFromDir(Corpora[i], &AllFiles);
+ std::sort(AllFiles.begin() + NumFilesInFirstCorpus, AllFiles.end());
+ Printf("MERGE-OUTER: %zd files, %zd in the initial corpus\n",
+ AllFiles.size(), NumFilesInFirstCorpus);
+ WriteNewControlFile(CFPath, AllFiles, NumFilesInFirstCorpus);
+ NumAttempts = AllFiles.size();
}
- ControlFile.close();
- // Execute the inner process untill it passes.
+ // Execute the inner process until it passes.
// Every inner process should execute at least one input.
auto BaseCmd = SplitBefore("-ignore_remaining_args=1",
CloneArgsWithoutX(Args, "merge"));
bool Success = false;
- for (size_t i = 1; i <= AllFiles.size(); i++) {
- Printf("MERGE-OUTER: attempt %zd\n", i);
+ for (size_t Attempt = 1; Attempt <= NumAttempts; Attempt++) {
+ Printf("MERGE-OUTER: attempt %zd\n", Attempt);
auto ExitCode =
ExecuteCommand(BaseCmd.first + " -merge_control_file=" + CFPath +
" -merge_inner=1 " + BaseCmd.second);
if (!ExitCode) {
- Printf("MERGE-OUTER: succesfull in %zd attempt(s)\n", i);
+ Printf("MERGE-OUTER: succesfull in %zd attempt(s)\n", Attempt);
Success = true;
break;
}
Added: compiler-rt/trunk/test/fuzzer/merge-control-file.test
URL: http://llvm.org/viewvc/llvm-project/compiler-rt/trunk/test/fuzzer/merge-control-file.test?rev=317767&view=auto
==============================================================================
--- compiler-rt/trunk/test/fuzzer/merge-control-file.test (added)
+++ compiler-rt/trunk/test/fuzzer/merge-control-file.test Wed Nov 8 21:49:28 2017
@@ -0,0 +1,47 @@
+RUN: mkdir -p %t
+RUN: %cpp_compiler %S/FullCoverageSetTest.cpp -o %t/T
+
+RUN: rm -rf %t/T0 %t/T1 %t/T2
+RUN: mkdir -p %t/T0 %t/T1 %t/T2
+RUN: echo F..... > %t/T0/1
+RUN: echo .U.... > %t/T0/2
+RUN: echo ..Z... > %t/T0/3
+
+# Test what happens if the control file is junk.
+
+RUN: echo JUNK > %t/MCF
+RUN: not %t/T -merge=1 %t/T1 %t/T2 -merge_control_file=%t/MCF 2>&1 | FileCheck %s --check-prefix=JUNK
+RUN: echo 3 > %t/MCF; echo 0 >> %t/MCF; echo %t/T1/1 >> %t/MCF
+RUN: not %t/T -merge=1 %t/T1 %t/T2 -merge_control_file=%t/MCF 2>&1 | FileCheck %s --check-prefix=JUNK
+JUNK: MERGE-OUTER: non-empty control file provided: {{.*}}MCF
+JUNK: MERGE-OUTER: bad control file, will overwrite it
+
+
+# Check valid control files
+
+RUN: rm -f %t/T1/*; cp %t/T0/* %t/T1
+RUN: echo 3 > %t/MCF; echo 0 >> %t/MCF; echo %t/T1/1 >> %t/MCF; echo %t/T1/2 >> %t/MCF; echo %t/T1/3 >> %t/MCF
+RUN: %t/T -merge=1 %t/T1 %t/T2 -merge_control_file=%t/MCF 2>&1 | FileCheck %s --check-prefix=OK_0
+OK_0: MERGE-OUTER: control file ok, 3 files total, first not processed file 0
+OK_0: MERGE-OUTER: 3 new files with {{.*}} new features added
+
+RUN: rm -f %t/T1/*; cp %t/T0/* %t/T1
+RUN: echo 3 > %t/MCF; echo 0 >> %t/MCF; echo %t/T1/1 >> %t/MCF; echo %t/T1/2 >> %t/MCF; echo %t/T1/3 >> %t/MCF
+RUN: echo STARTED 0 1 >> %t/MCF
+RUN: echo DONE 0 11 >> %t/MCF
+RUN: echo STARTED 1 2 >> %t/MCF
+RUN: echo DONE 1 12 >> %t/MCF
+RUN: %t/T -merge=1 %t/T1 %t/T2 -merge_control_file=%t/MCF 2>&1 | FileCheck %s --check-prefix=OK_2
+OK_2: MERGE-OUTER: control file ok, 3 files total, first not processed file 2
+OK_2: MERGE-OUTER: 3 new files with {{.*}} new features added
+
+RUN: rm -f %t/T1/*; cp %t/T0/* %t/T1
+RUN: echo 3 > %t/MCF; echo 0 >> %t/MCF; echo %t/T1/1 >> %t/MCF; echo %t/T1/2 >> %t/MCF; echo %t/T1/3 >> %t/MCF
+RUN: echo STARTED 0 1 >> %t/MCF
+RUN: echo DONE 0 11 >> %t/MCF
+RUN: echo STARTED 1 2 >> %t/MCF
+RUN: echo DONE 1 12 >> %t/MCF
+RUN: echo STARTED 2 2 >> %t/MCF
+RUN: echo DONE 2 13 >> %t/MCF
+RUN: %t/T -merge=1 %t/T1 %t/T2 -merge_control_file=%t/MCF 2>&1 | FileCheck %s --check-prefix=OK_3
+OK_3: MERGE-OUTER: nothing to do, merge has been completed before
More information about the llvm-commits
mailing list