[llvm] r283682 - [libFuzzer] when shrinking the corpus, delete evicted files previously created by the current process
Kostya Serebryany via llvm-commits
llvm-commits at lists.llvm.org
Sat Oct 8 16:24:46 PDT 2016
Author: kcc
Date: Sat Oct 8 18:24:45 2016
New Revision: 283682
URL: http://llvm.org/viewvc/llvm-project?rev=283682&view=rev
Log:
[libFuzzer] when shrinking the corpus, delete evicted files previously created by the current process
Modified:
llvm/trunk/lib/Fuzzer/FuzzerCorpus.h
llvm/trunk/lib/Fuzzer/FuzzerDefs.h
llvm/trunk/lib/Fuzzer/FuzzerDriver.cpp
llvm/trunk/lib/Fuzzer/FuzzerFlags.def
llvm/trunk/lib/Fuzzer/FuzzerIO.cpp
llvm/trunk/lib/Fuzzer/FuzzerLoop.cpp
llvm/trunk/lib/Fuzzer/build.sh
llvm/trunk/lib/Fuzzer/test/FuzzerUnittest.cpp
Modified: llvm/trunk/lib/Fuzzer/FuzzerCorpus.h
URL: http://llvm.org/viewvc/llvm-project/llvm/trunk/lib/Fuzzer/FuzzerCorpus.h?rev=283682&r1=283681&r2=283682&view=diff
==============================================================================
--- llvm/trunk/lib/Fuzzer/FuzzerCorpus.h (original)
+++ llvm/trunk/lib/Fuzzer/FuzzerCorpus.h Sat Oct 8 18:24:45 2016
@@ -30,12 +30,13 @@ struct InputInfo {
// Stats.
size_t NumExecutedMutations = 0;
size_t NumSuccessfullMutations = 0;
+ bool MayDeleteFile = false;
};
class InputCorpus {
public:
static const size_t kFeatureSetSize = 1 << 16;
- InputCorpus() {
+ InputCorpus(const std::string &OutputCorpus) : OutputCorpus(OutputCorpus) {
memset(InputSizesPerFeature, 0, sizeof(InputSizesPerFeature));
memset(SmallestElementPerFeature, 0, sizeof(SmallestElementPerFeature));
}
@@ -58,7 +59,7 @@ class InputCorpus {
}
bool empty() const { return Inputs.empty(); }
const Unit &operator[] (size_t Idx) const { return Inputs[Idx]->U; }
- void AddToCorpus(const Unit &U, size_t NumFeatures) {
+ void AddToCorpus(const Unit &U, size_t NumFeatures, bool MayDeleteFile = false) {
assert(!U.empty());
uint8_t Hash[kSHA1NumBytes];
if (FeatureDebug)
@@ -69,6 +70,7 @@ class InputCorpus {
InputInfo &II = *Inputs.back();
II.U = U;
II.NumFeatures = NumFeatures;
+ II.MayDeleteFile = MayDeleteFile;
memcpy(II.Sha1, Hash, kSHA1NumBytes);
UpdateCorpusDistribution();
ValidateFeatureSet();
@@ -112,20 +114,27 @@ class InputCorpus {
Printf("\n");
}
+ void DeleteInput(size_t Idx) {
+ InputInfo &II = *Inputs[Idx];
+ if (!OutputCorpus.empty() && II.MayDeleteFile)
+ DeleteFile(DirPlusFile(OutputCorpus, Sha1ToString(II.Sha1)));
+ Unit().swap(II.U);
+ if (FeatureDebug)
+ Printf("EVICTED %zd\n", Idx);
+ }
+
bool AddFeature(size_t Idx, uint32_t NewSize, bool Shrink) {
assert(NewSize);
Idx = Idx % kFeatureSetSize;
uint32_t OldSize = GetFeature(Idx);
if (OldSize == 0 || (Shrink && OldSize > NewSize)) {
if (OldSize > 0) {
- InputInfo &II = *Inputs[SmallestElementPerFeature[Idx]];
+ size_t OldIdx = SmallestElementPerFeature[Idx];
+ InputInfo &II = *Inputs[OldIdx];
assert(II.NumFeatures > 0);
II.NumFeatures--;
- if (II.NumFeatures == 0) {
- Unit().swap(II.U);
- if (FeatureDebug)
- Printf("EVICTED %zd\n", SmallestElementPerFeature[Idx]);
- }
+ if (II.NumFeatures == 0)
+ DeleteInput(OldIdx);
}
if (FeatureDebug)
Printf("ADD FEATURE %zd sz %d\n", Idx, NewSize);
@@ -191,6 +200,8 @@ private:
bool CountingFeatures = false;
uint32_t InputSizesPerFeature[kFeatureSetSize];
uint32_t SmallestElementPerFeature[kFeatureSetSize];
+
+ std::string OutputCorpus;
};
} // namespace fuzzer
Modified: llvm/trunk/lib/Fuzzer/FuzzerDefs.h
URL: http://llvm.org/viewvc/llvm-project/llvm/trunk/lib/Fuzzer/FuzzerDefs.h?rev=283682&r1=283681&r2=283682&view=diff
==============================================================================
--- llvm/trunk/lib/Fuzzer/FuzzerDefs.h (original)
+++ llvm/trunk/lib/Fuzzer/FuzzerDefs.h Sat Oct 8 18:24:45 2016
@@ -60,11 +60,13 @@ int FuzzerDriver(int *argc, char ***argv
bool IsFile(const std::string &Path);
long GetEpoch(const std::string &Path);
std::string FileToString(const std::string &Path);
-Unit FileToVector(const std::string &Path, size_t MaxSize = 0);
+Unit FileToVector(const std::string &Path, size_t MaxSize = 0,
+ bool ExitOnError = true);
void ReadDirToVectorOfUnits(const char *Path, std::vector<Unit> *V,
- long *Epoch, size_t MaxSize);
+ long *Epoch, size_t MaxSize, bool ExitOnError);
void WriteToFile(const Unit &U, const std::string &Path);
void CopyFileToErr(const std::string &Path);
+void DeleteFile(const std::string &Path);
// Returns "Dir/FileName" or equivalent for the current OS.
std::string DirPlusFile(const std::string &DirPath,
const std::string &FileName);
Modified: llvm/trunk/lib/Fuzzer/FuzzerDriver.cpp
URL: http://llvm.org/viewvc/llvm-project/llvm/trunk/lib/Fuzzer/FuzzerDriver.cpp?rev=283682&r1=283681&r2=283682&view=diff
==============================================================================
--- llvm/trunk/lib/Fuzzer/FuzzerDriver.cpp (original)
+++ llvm/trunk/lib/Fuzzer/FuzzerDriver.cpp Sat Oct 8 18:24:45 2016
@@ -443,7 +443,7 @@ int FuzzerDriver(int *argc, char ***argv
Random Rand(Seed);
MutationDispatcher MD(Rand, Options);
- InputCorpus Corpus;
+ InputCorpus Corpus(Options.OutputCorpus);
Fuzzer F(Callback, Corpus, MD, Options);
for (auto &U: Dictionary)
@@ -500,7 +500,8 @@ int FuzzerDriver(int *argc, char ***argv
UnitVector InitialCorpus;
for (auto &Inp : *Inputs) {
Printf("Loading corpus dir: %s\n", Inp.c_str());
- ReadDirToVectorOfUnits(Inp.c_str(), &InitialCorpus, nullptr, TemporaryMaxLen);
+ ReadDirToVectorOfUnits(Inp.c_str(), &InitialCorpus, nullptr,
+ TemporaryMaxLen, /*ExitOnError=*/false);
}
if (Options.MaxLen == 0) {
Modified: llvm/trunk/lib/Fuzzer/FuzzerFlags.def
URL: http://llvm.org/viewvc/llvm-project/llvm/trunk/lib/Fuzzer/FuzzerFlags.def?rev=283682&r1=283681&r2=283682&view=diff
==============================================================================
--- llvm/trunk/lib/Fuzzer/FuzzerFlags.def (original)
+++ llvm/trunk/lib/Fuzzer/FuzzerFlags.def Sat Oct 8 18:24:45 2016
@@ -56,7 +56,7 @@ FUZZER_FLAG_INT(jobs, 0, "Number of jobs
FUZZER_FLAG_INT(workers, 0,
"Number of simultaneous worker processes to run the jobs."
" If zero, \"min(jobs,NumberOfCpuCores()/2)\" is used.")
-FUZZER_FLAG_INT(reload, 10,
+FUZZER_FLAG_INT(reload, 1,
"Reload the main corpus every <N> seconds to get new units"
" discovered by other processes. If 0, disabled")
FUZZER_FLAG_INT(report_slow_units, 10,
Modified: llvm/trunk/lib/Fuzzer/FuzzerIO.cpp
URL: http://llvm.org/viewvc/llvm-project/llvm/trunk/lib/Fuzzer/FuzzerIO.cpp?rev=283682&r1=283681&r2=283682&view=diff
==============================================================================
--- llvm/trunk/lib/Fuzzer/FuzzerIO.cpp (original)
+++ llvm/trunk/lib/Fuzzer/FuzzerIO.cpp Sat Oct 8 18:24:45 2016
@@ -60,9 +60,9 @@ static void ListFilesInDirRecursive(cons
*Epoch = E;
}
-Unit FileToVector(const std::string &Path, size_t MaxSize) {
+Unit FileToVector(const std::string &Path, size_t MaxSize, bool ExitOnError) {
std::ifstream T(Path);
- if (!T) {
+ if (ExitOnError && !T) {
Printf("No such directory: %s; exiting\n", Path.c_str());
exit(1);
}
@@ -78,6 +78,10 @@ Unit FileToVector(const std::string &Pat
return Res;
}
+void DeleteFile(const std::string &Path) {
+ unlink(Path.c_str());
+}
+
std::string FileToString(const std::string &Path) {
std::ifstream T(Path);
return std::string((std::istreambuf_iterator<char>(T)),
@@ -97,7 +101,7 @@ void WriteToFile(const Unit &U, const st
}
void ReadDirToVectorOfUnits(const char *Path, std::vector<Unit> *V,
- long *Epoch, size_t MaxSize) {
+ long *Epoch, size_t MaxSize, bool ExitOnError) {
long E = Epoch ? *Epoch : 0;
std::vector<std::string> Files;
ListFilesInDirRecursive(Path, Epoch, &Files, /*TopDir*/true);
@@ -108,7 +112,9 @@ void ReadDirToVectorOfUnits(const char *
NumLoaded++;
if ((NumLoaded & (NumLoaded - 1)) == 0 && NumLoaded >= 1024)
Printf("Loaded %zd/%zd files from %s\n", NumLoaded, Files.size(), Path);
- V->push_back(FileToVector(X, MaxSize));
+ auto S = FileToVector(X, MaxSize, ExitOnError);
+ if (!S.empty())
+ V->push_back(S);
}
}
Modified: llvm/trunk/lib/Fuzzer/FuzzerLoop.cpp
URL: http://llvm.org/viewvc/llvm-project/llvm/trunk/lib/Fuzzer/FuzzerLoop.cpp?rev=283682&r1=283681&r2=283682&view=diff
==============================================================================
--- llvm/trunk/lib/Fuzzer/FuzzerLoop.cpp (original)
+++ llvm/trunk/lib/Fuzzer/FuzzerLoop.cpp Sat Oct 8 18:24:45 2016
@@ -386,7 +386,8 @@ void Fuzzer::RereadOutputCorpus(size_t M
if (Options.OutputCorpus.empty() || !Options.ReloadIntervalSec) return;
std::vector<Unit> AdditionalCorpus;
ReadDirToVectorOfUnits(Options.OutputCorpus.c_str(), &AdditionalCorpus,
- &EpochOfLastReadOfOutputCorpus, MaxSize);
+ &EpochOfLastReadOfOutputCorpus, MaxSize,
+ /*ExitOnError*/ false);
if (Options.Verbosity >= 2)
Printf("Reload: read %zd new units.\n", AdditionalCorpus.size());
bool Reloaded = false;
@@ -605,9 +606,9 @@ void Fuzzer::Merge(const std::vector<std
assert(MaxInputLen > 0);
UnitVector Initial, Extra;
- ReadDirToVectorOfUnits(Corpora[0].c_str(), &Initial, nullptr, MaxInputLen);
+ ReadDirToVectorOfUnits(Corpora[0].c_str(), &Initial, nullptr, MaxInputLen, true);
for (auto &C : ExtraCorpora)
- ReadDirToVectorOfUnits(C.c_str(), &Extra, nullptr, MaxInputLen);
+ ReadDirToVectorOfUnits(C.c_str(), &Extra, nullptr, MaxInputLen, true);
if (!Initial.empty()) {
Printf("=== Minimizing the initial corpus of %zd units\n", Initial.size());
@@ -685,8 +686,8 @@ void Fuzzer::MutateAndTestOne() {
StartTraceRecording();
II.NumExecutedMutations++;
if (size_t NumFeatures = RunOne(CurrentUnitData, Size)) {
- Corpus.AddToCorpus({CurrentUnitData, CurrentUnitData + Size},
- NumFeatures);
+ Corpus.AddToCorpus({CurrentUnitData, CurrentUnitData + Size}, NumFeatures,
+ /*MayDeleteFile=*/true);
ReportNewCoverage(&II, {CurrentUnitData, CurrentUnitData + Size});
CheckExitOnItem();
}
@@ -711,7 +712,7 @@ void Fuzzer::Loop() {
if (duration_cast<seconds>(Now - LastCorpusReload).count() >=
Options.ReloadIntervalSec) {
RereadOutputCorpus(MaxInputLen);
- LastCorpusReload = Now;
+ LastCorpusReload = system_clock::now();
}
if (TotalNumberOfRuns >= Options.MaxNumberOfRuns)
break;
Modified: llvm/trunk/lib/Fuzzer/build.sh
URL: http://llvm.org/viewvc/llvm-project/llvm/trunk/lib/Fuzzer/build.sh?rev=283682&r1=283681&r2=283682&view=diff
==============================================================================
--- llvm/trunk/lib/Fuzzer/build.sh (original)
+++ llvm/trunk/lib/Fuzzer/build.sh Sat Oct 8 18:24:45 2016
@@ -1,7 +1,7 @@
#!/bin/bash
LIBFUZZER_SRC_DIR=$(dirname $0)
for f in $LIBFUZZER_SRC_DIR/*.cpp; do
- clang -g -O2 -std=c++11 $f -c &
+ clang -g -O2 -fno-omit-frame-pointer -std=c++11 $f -c &
done
wait
rm -f libFuzzer.a
Modified: llvm/trunk/lib/Fuzzer/test/FuzzerUnittest.cpp
URL: http://llvm.org/viewvc/llvm-project/llvm/trunk/lib/Fuzzer/test/FuzzerUnittest.cpp?rev=283682&r1=283681&r2=283682&view=diff
==============================================================================
--- llvm/trunk/lib/Fuzzer/test/FuzzerUnittest.cpp (original)
+++ llvm/trunk/lib/Fuzzer/test/FuzzerUnittest.cpp Sat Oct 8 18:24:45 2016
@@ -579,7 +579,7 @@ TEST(FuzzerUtil, Base64) {
TEST(Corpus, Distribution) {
Random Rand(0);
- InputCorpus C;
+ InputCorpus C("");
size_t N = 10;
size_t TriesPerUnit = 1<<20;
for (size_t i = 0; i < N; i++)
More information about the llvm-commits
mailing list