[compiler-rt] r358317 - [libFuzzer] add -features_dir= flag to dump unique input features on disk
Kostya Serebryany via llvm-commits
llvm-commits at lists.llvm.org
Fri Apr 12 17:20:32 PDT 2019
Author: kcc
Date: Fri Apr 12 17:20:31 2019
New Revision: 358317
URL: http://llvm.org/viewvc/llvm-project?rev=358317&view=rev
Log:
[libFuzzer] add -features_dir= flag to dump unique input features on disk
Added:
compiler-rt/trunk/test/fuzzer/features_dir.test
Modified:
compiler-rt/trunk/lib/fuzzer/FuzzerCorpus.h
compiler-rt/trunk/lib/fuzzer/FuzzerDriver.cpp
compiler-rt/trunk/lib/fuzzer/FuzzerFlags.def
compiler-rt/trunk/lib/fuzzer/FuzzerIO.cpp
compiler-rt/trunk/lib/fuzzer/FuzzerIO.h
compiler-rt/trunk/lib/fuzzer/FuzzerIOPosix.cpp
compiler-rt/trunk/lib/fuzzer/FuzzerIOWindows.cpp
compiler-rt/trunk/lib/fuzzer/FuzzerLoop.cpp
compiler-rt/trunk/lib/fuzzer/FuzzerOptions.h
Modified: compiler-rt/trunk/lib/fuzzer/FuzzerCorpus.h
URL: http://llvm.org/viewvc/llvm-project/compiler-rt/trunk/lib/fuzzer/FuzzerCorpus.h?rev=358317&r1=358316&r2=358317&view=diff
==============================================================================
--- compiler-rt/trunk/lib/fuzzer/FuzzerCorpus.h (original)
+++ compiler-rt/trunk/lib/fuzzer/FuzzerCorpus.h Fri Apr 12 17:20:31 2019
@@ -85,9 +85,10 @@ class InputCorpus {
bool empty() const { return Inputs.empty(); }
const Unit &operator[] (size_t Idx) const { return Inputs[Idx]->U; }
- void AddToCorpus(const Unit &U, size_t NumFeatures, bool MayDeleteFile,
- bool HasFocusFunction, const Vector<uint32_t> &FeatureSet,
- const DataFlowTrace &DFT, const InputInfo *BaseII) {
+ InputInfo *AddToCorpus(const Unit &U, size_t NumFeatures, bool MayDeleteFile,
+ bool HasFocusFunction,
+ const Vector<uint32_t> &FeatureSet,
+ const DataFlowTrace &DFT, const InputInfo *BaseII) {
assert(!U.empty());
if (FeatureDebug)
Printf("ADD_TO_CORPUS %zd NF %zd\n", Inputs.size(), NumFeatures);
@@ -113,6 +114,7 @@ class InputCorpus {
UpdateCorpusDistribution();
PrintCorpus();
// ValidateFeatureSet();
+ return &II;
}
// Debug-only
Modified: compiler-rt/trunk/lib/fuzzer/FuzzerDriver.cpp
URL: http://llvm.org/viewvc/llvm-project/compiler-rt/trunk/lib/fuzzer/FuzzerDriver.cpp?rev=358317&r1=358316&r2=358317&view=diff
==============================================================================
--- compiler-rt/trunk/lib/fuzzer/FuzzerDriver.cpp (original)
+++ compiler-rt/trunk/lib/fuzzer/FuzzerDriver.cpp Fri Apr 12 17:20:31 2019
@@ -659,6 +659,8 @@ int FuzzerDriver(int *argc, char ***argv
Options.FocusFunction = Flags.focus_function;
if (Flags.data_flow_trace)
Options.DataFlowTrace = Flags.data_flow_trace;
+ if (Flags.features_dir)
+ Options.FeaturesDir = Flags.features_dir;
Options.LazyCounters = Flags.lazy_counters;
unsigned Seed = Flags.seed;
Modified: compiler-rt/trunk/lib/fuzzer/FuzzerFlags.def
URL: http://llvm.org/viewvc/llvm-project/compiler-rt/trunk/lib/fuzzer/FuzzerFlags.def?rev=358317&r1=358316&r2=358317&view=diff
==============================================================================
--- compiler-rt/trunk/lib/fuzzer/FuzzerFlags.def (original)
+++ compiler-rt/trunk/lib/fuzzer/FuzzerFlags.def Fri Apr 12 17:20:31 2019
@@ -67,6 +67,10 @@ FUZZER_FLAG_INT(cleanse_crash, 0, "If 1,
" Use with -exact_artifact_path to specify the output."
)
FUZZER_FLAG_INT(minimize_crash_internal_step, 0, "internal flag")
+FUZZER_FLAG_STRING(features_dir, "internal flag. Used to dump feature sets on disk."
+ "Every time a new input is added to the corpus, a corresponding file in the features_dir"
+ " is created containing the unique features of that input."
+ " Features are stored in binary format.")
FUZZER_FLAG_INT(use_counters, 1, "Use coverage counters")
FUZZER_FLAG_INT(use_memmem, 1,
"Use hints from intercepting memmem, strstr, etc")
Modified: compiler-rt/trunk/lib/fuzzer/FuzzerIO.cpp
URL: http://llvm.org/viewvc/llvm-project/compiler-rt/trunk/lib/fuzzer/FuzzerIO.cpp?rev=358317&r1=358316&r2=358317&view=diff
==============================================================================
--- compiler-rt/trunk/lib/fuzzer/FuzzerIO.cpp (original)
+++ compiler-rt/trunk/lib/fuzzer/FuzzerIO.cpp Fri Apr 12 17:20:31 2019
@@ -61,10 +61,14 @@ void CopyFileToErr(const std::string &Pa
}
void WriteToFile(const Unit &U, const std::string &Path) {
+ WriteToFile(U.data(), U.size(), Path);
+}
+
+void WriteToFile(const uint8_t *Data, size_t Size, const std::string &Path) {
// Use raw C interface because this function may be called from a sig handler.
FILE *Out = fopen(Path.c_str(), "wb");
if (!Out) return;
- fwrite(U.data(), sizeof(U[0]), U.size(), Out);
+ fwrite(Data, sizeof(Data[0]), Size, Out);
fclose(Out);
}
Modified: compiler-rt/trunk/lib/fuzzer/FuzzerIO.h
URL: http://llvm.org/viewvc/llvm-project/compiler-rt/trunk/lib/fuzzer/FuzzerIO.h?rev=358317&r1=358316&r2=358317&view=diff
==============================================================================
--- compiler-rt/trunk/lib/fuzzer/FuzzerIO.h (original)
+++ compiler-rt/trunk/lib/fuzzer/FuzzerIO.h Fri Apr 12 17:20:31 2019
@@ -24,6 +24,7 @@ std::string FileToString(const std::stri
void CopyFileToErr(const std::string &Path);
+void WriteToFile(const uint8_t *Data, size_t Size, const std::string &Path);
void WriteToFile(const Unit &U, const std::string &Path);
void ReadDirToVectorOfUnits(const char *Path, Vector<Unit> *V,
@@ -89,6 +90,7 @@ int CloseFile(int Fd);
int DuplicateFile(int Fd);
void RemoveFile(const std::string &Path);
+void RenameFile(const std::string &OldPath, const std::string &NewPath);
void DiscardOutput(int Fd);
Modified: compiler-rt/trunk/lib/fuzzer/FuzzerIOPosix.cpp
URL: http://llvm.org/viewvc/llvm-project/compiler-rt/trunk/lib/fuzzer/FuzzerIOPosix.cpp?rev=358317&r1=358316&r2=358317&view=diff
==============================================================================
--- compiler-rt/trunk/lib/fuzzer/FuzzerIOPosix.cpp (original)
+++ compiler-rt/trunk/lib/fuzzer/FuzzerIOPosix.cpp Fri Apr 12 17:20:31 2019
@@ -120,6 +120,10 @@ void RemoveFile(const std::string &Path)
unlink(Path.c_str());
}
+void RenameFile(const std::string &OldPath, const std::string &NewPath) {
+ rename(OldPath.c_str(), NewPath.c_str());
+}
+
void DiscardOutput(int Fd) {
FILE* Temp = fopen("/dev/null", "w");
if (!Temp)
Modified: compiler-rt/trunk/lib/fuzzer/FuzzerIOWindows.cpp
URL: http://llvm.org/viewvc/llvm-project/compiler-rt/trunk/lib/fuzzer/FuzzerIOWindows.cpp?rev=358317&r1=358316&r2=358317&view=diff
==============================================================================
--- compiler-rt/trunk/lib/fuzzer/FuzzerIOWindows.cpp (original)
+++ compiler-rt/trunk/lib/fuzzer/FuzzerIOWindows.cpp Fri Apr 12 17:20:31 2019
@@ -219,6 +219,10 @@ void RemoveFile(const std::string &Path)
_unlink(Path.c_str());
}
+void RenameFile(const std::string &OldPath, const std::string &NewPath) {
+ rename(OldPath.c_str(), NewPath.c_str());
+}
+
void DiscardOutput(int Fd) {
FILE* Temp = fopen("nul", "w");
if (!Temp)
Modified: compiler-rt/trunk/lib/fuzzer/FuzzerLoop.cpp
URL: http://llvm.org/viewvc/llvm-project/compiler-rt/trunk/lib/fuzzer/FuzzerLoop.cpp?rev=358317&r1=358316&r2=358317&view=diff
==============================================================================
--- compiler-rt/trunk/lib/fuzzer/FuzzerLoop.cpp (original)
+++ compiler-rt/trunk/lib/fuzzer/FuzzerLoop.cpp Fri Apr 12 17:20:31 2019
@@ -445,6 +445,24 @@ void Fuzzer::PrintPulseAndReportSlowInpu
}
}
+static void WriteFeatureSetToFile(const std::string &FeaturesDir,
+ const uint8_t Sha1[],
+ const Vector<uint32_t> &FeatureSet) {
+ if (FeaturesDir.empty() || FeatureSet.empty()) return;
+ WriteToFile(reinterpret_cast<const uint8_t *>(FeatureSet.data()),
+ FeatureSet.size() * sizeof(FeatureSet[0]),
+ DirPlusFile(FeaturesDir, Sha1ToString(Sha1)));
+ Printf("Features: %s\n", Sha1ToString(Sha1).c_str());
+}
+
+static void RenameFeatureSetFile(const std::string &FeaturesDir,
+ const std::string &OldFile,
+ const std::string &NewFile) {
+ if (FeaturesDir.empty()) return;
+ RenameFile(DirPlusFile(FeaturesDir, OldFile),
+ DirPlusFile(FeaturesDir, NewFile));
+}
+
bool Fuzzer::RunOne(const uint8_t *Data, size_t Size, bool MayDeleteFile,
InputInfo *II, bool *FoundUniqFeatures) {
if (!Size)
@@ -469,15 +487,21 @@ bool Fuzzer::RunOne(const uint8_t *Data,
size_t NumNewFeatures = Corpus.NumFeatureUpdates() - NumUpdatesBefore;
if (NumNewFeatures) {
TPC.UpdateObservedPCs();
- Corpus.AddToCorpus({Data, Data + Size}, NumNewFeatures, MayDeleteFile,
- TPC.ObservedFocusFunction(), UniqFeatureSetTmp, DFT, II);
+ auto NewII = Corpus.AddToCorpus({Data, Data + Size}, NumNewFeatures,
+ MayDeleteFile, TPC.ObservedFocusFunction(),
+ UniqFeatureSetTmp, DFT, II);
+ WriteFeatureSetToFile(Options.FeaturesDir, NewII->Sha1,
+ NewII->UniqFeatureSet);
return true;
}
if (II && FoundUniqFeaturesOfII &&
II->DataFlowTraceForFocusFunction.empty() &&
FoundUniqFeaturesOfII == II->UniqFeatureSet.size() &&
II->U.size() > Size) {
+ auto OldFeaturesFile = Sha1ToString(II->Sha1);
Corpus.Replace(II, {Data, Data + Size});
+ RenameFeatureSetFile(Options.FeaturesDir, OldFeaturesFile,
+ Sha1ToString(II->Sha1));
return true;
}
return false;
Modified: compiler-rt/trunk/lib/fuzzer/FuzzerOptions.h
URL: http://llvm.org/viewvc/llvm-project/compiler-rt/trunk/lib/fuzzer/FuzzerOptions.h?rev=358317&r1=358316&r2=358317&view=diff
==============================================================================
--- compiler-rt/trunk/lib/fuzzer/FuzzerOptions.h (original)
+++ compiler-rt/trunk/lib/fuzzer/FuzzerOptions.h Fri Apr 12 17:20:31 2019
@@ -51,6 +51,7 @@ struct FuzzingOptions {
std::string ExitOnItem;
std::string FocusFunction;
std::string DataFlowTrace;
+ std::string FeaturesDir;
bool SaveArtifacts = true;
bool PrintNEW = true; // Print a status line when new units are found;
bool PrintNewCovPcs = false;
Added: compiler-rt/trunk/test/fuzzer/features_dir.test
URL: http://llvm.org/viewvc/llvm-project/compiler-rt/trunk/test/fuzzer/features_dir.test?rev=358317&view=auto
==============================================================================
--- compiler-rt/trunk/test/fuzzer/features_dir.test (added)
+++ compiler-rt/trunk/test/fuzzer/features_dir.test Fri Apr 12 17:20:31 2019
@@ -0,0 +1,7 @@
+# Tests -features_dir=F
+# REQUIRES: linux
+RUN: %cpp_compiler %S/SimpleTest.cpp -o %t-SimpleTest
+RUN: rm -rf %t-C %t-F
+RUN: mkdir %t-C %t-F
+RUN: not %run %t-SimpleTest %t-C -features_dir=%t-F
+RUN: for c in %t-C/*; do f=%t-F/$(basename $c); echo looking for $f; [ -a $f ]; done
More information about the llvm-commits
mailing list