[llvm] r236906 - [lib/Fuzzer] use -fsanitize-coverage=trace-cmp when building LLVM with LLVM_USE_SANITIZE_COVERAGE; in lib/Fuzzer try to reload the corpus to pick up new units from other processes
Kostya Serebryany
kcc at google.com
Fri May 8 14:30:55 PDT 2015
Author: kcc
Date: Fri May 8 16:30:55 2015
New Revision: 236906
URL: http://llvm.org/viewvc/llvm-project?rev=236906&view=rev
Log:
[lib/Fuzzer] use -fsanitize-coverage=trace-cmp when building LLVM with LLVM_USE_SANITIZE_COVERAGE; in lib/Fuzzer try to reload the corpus to pick up new units from other processes
Modified:
llvm/trunk/cmake/modules/HandleLLVMOptions.cmake
llvm/trunk/lib/Fuzzer/FuzzerDriver.cpp
llvm/trunk/lib/Fuzzer/FuzzerFlags.def
llvm/trunk/lib/Fuzzer/FuzzerIO.cpp
llvm/trunk/lib/Fuzzer/FuzzerInternal.h
llvm/trunk/lib/Fuzzer/FuzzerLoop.cpp
llvm/trunk/lib/Fuzzer/test/dfsan/CMakeLists.txt
Modified: llvm/trunk/cmake/modules/HandleLLVMOptions.cmake
URL: http://llvm.org/viewvc/llvm-project/llvm/trunk/cmake/modules/HandleLLVMOptions.cmake?rev=236906&r1=236905&r2=236906&view=diff
==============================================================================
--- llvm/trunk/cmake/modules/HandleLLVMOptions.cmake (original)
+++ llvm/trunk/cmake/modules/HandleLLVMOptions.cmake Fri May 8 16:30:55 2015
@@ -476,7 +476,7 @@ if(LLVM_USE_SANITIZER)
message(WARNING "LLVM_USE_SANITIZER is not supported on this platform.")
endif()
if (LLVM_USE_SANITIZE_COVERAGE)
- append("-fsanitize-coverage=edge,indirect-calls,8bit-counters" CMAKE_C_FLAGS CMAKE_CXX_FLAGS)
+ append("-fsanitize-coverage=edge,indirect-calls,8bit-counters,trace-cmp" CMAKE_C_FLAGS CMAKE_CXX_FLAGS)
endif()
endif()
Modified: llvm/trunk/lib/Fuzzer/FuzzerDriver.cpp
URL: http://llvm.org/viewvc/llvm-project/llvm/trunk/lib/Fuzzer/FuzzerDriver.cpp?rev=236906&r1=236905&r2=236906&view=diff
==============================================================================
--- llvm/trunk/lib/Fuzzer/FuzzerDriver.cpp (original)
+++ llvm/trunk/lib/Fuzzer/FuzzerDriver.cpp Fri May 8 16:30:55 2015
@@ -207,6 +207,7 @@ int FuzzerDriver(int argc, char **argv,
Options.PreferSmallDuringInitialShuffle =
Flags.prefer_small_during_initial_shuffle;
Options.Tokens = ReadTokensFile(Flags.tokens);
+ Options.Reload = Flags.reload;
if (Flags.runs >= 0)
Options.MaxNumberOfRuns = Flags.runs;
if (!inputs.empty())
@@ -235,8 +236,10 @@ int FuzzerDriver(int argc, char **argv,
if (Flags.apply_tokens)
return ApplyTokens(F, Flags.apply_tokens);
+ F.RereadOutputCorpus();
for (auto &inp : inputs)
- F.ReadDir(inp);
+ if (inp != Options.OutputCorpus)
+ F.ReadDir(inp, nullptr);
if (F.CorpusSize() == 0)
F.AddToCorpus(Unit()); // Can't fuzz empty corpus, so add an empty input.
Modified: llvm/trunk/lib/Fuzzer/FuzzerFlags.def
URL: http://llvm.org/viewvc/llvm-project/llvm/trunk/lib/Fuzzer/FuzzerFlags.def?rev=236906&r1=236905&r2=236906&view=diff
==============================================================================
--- llvm/trunk/lib/Fuzzer/FuzzerFlags.def (original)
+++ llvm/trunk/lib/Fuzzer/FuzzerFlags.def Fri May 8 16:30:55 2015
@@ -46,7 +46,9 @@ FUZZER_FLAG_INT(workers, 0,
"Number of simultaneous worker processes to run the jobs.")
FUZZER_FLAG_INT(dfsan, 1, "Use DFSan for taint-guided mutations. No-op unless "
"the DFSan instrumentation was compiled in.")
-
+FUZZER_FLAG_INT(reload, 1,
+ "Reload the main corpus periodically to get new units"
+ "discovered by other processes.")
FUZZER_FLAG_STRING(tokens, "Use the file with tokens (one token per line) to"
" fuzz a token based input language.")
FUZZER_FLAG_STRING(apply_tokens, "Read the given input file, substitute bytes "
Modified: llvm/trunk/lib/Fuzzer/FuzzerIO.cpp
URL: http://llvm.org/viewvc/llvm-project/llvm/trunk/lib/Fuzzer/FuzzerIO.cpp?rev=236906&r1=236905&r2=236906&view=diff
==============================================================================
--- llvm/trunk/lib/Fuzzer/FuzzerIO.cpp (original)
+++ llvm/trunk/lib/Fuzzer/FuzzerIO.cpp Fri May 8 16:30:55 2015
@@ -13,10 +13,26 @@
#include <iterator>
#include <fstream>
#include <dirent.h>
+#include <sys/types.h>
+#include <sys/stat.h>
+#include <unistd.h>
+
namespace fuzzer {
-static std::vector<std::string> ListFilesInDir(const std::string &Dir) {
+static long GetEpoch(const std::string &Path) {
+ struct stat St;
+ if (stat(Path.c_str(), &St)) return 0;
+ return St.st_mtime;
+}
+
+static std::vector<std::string> ListFilesInDir(const std::string &Dir,
+ long *Epoch) {
std::vector<std::string> V;
+ if (Epoch) {
+ auto E = GetEpoch(Dir.c_str());
+ if (*Epoch >= E) return V;
+ *Epoch = E;
+ }
DIR *D = opendir(Dir.c_str());
if (!D) return V;
while (auto E = readdir(D)) {
@@ -50,9 +66,14 @@ void WriteToFile(const Unit &U, const st
OF.write((const char*)U.data(), U.size());
}
-void ReadDirToVectorOfUnits(const char *Path, std::vector<Unit> *V) {
- for (auto &X : ListFilesInDir(Path))
- V->push_back(FileToVector(DirPlusFile(Path, X)));
+void ReadDirToVectorOfUnits(const char *Path, std::vector<Unit> *V,
+ long *Epoch) {
+ long E = Epoch ? *Epoch : 0;
+ for (auto &X : ListFilesInDir(Path, Epoch)) {
+ auto FilePath = DirPlusFile(Path, X);
+ if (Epoch && GetEpoch(FilePath) < E) continue;
+ V->push_back(FileToVector(FilePath));
+ }
}
std::string DirPlusFile(const std::string &DirPath,
Modified: llvm/trunk/lib/Fuzzer/FuzzerInternal.h
URL: http://llvm.org/viewvc/llvm-project/llvm/trunk/lib/Fuzzer/FuzzerInternal.h?rev=236906&r1=236905&r2=236906&view=diff
==============================================================================
--- llvm/trunk/lib/Fuzzer/FuzzerInternal.h (original)
+++ llvm/trunk/lib/Fuzzer/FuzzerInternal.h Fri May 8 16:30:55 2015
@@ -16,6 +16,7 @@
#include <string>
#include <vector>
#include <unordered_set>
+#include <set>
#include "FuzzerInterface.h"
@@ -25,7 +26,8 @@ using namespace std::chrono;
std::string FileToString(const std::string &Path);
Unit FileToVector(const std::string &Path);
-void ReadDirToVectorOfUnits(const char *Path, std::vector<Unit> *V);
+void ReadDirToVectorOfUnits(const char *Path, std::vector<Unit> *V,
+ long *Epoch);
void WriteToFile(const Unit &U, const std::string &Path);
void CopyFileToErr(const std::string &Path);
// Returns "Dir/FileName" or equivalent for the current OS.
@@ -54,6 +56,7 @@ class Fuzzer {
bool UseFullCoverageSet = false;
bool UseCoveragePairs = false;
bool UseDFSan = false;
+ bool Reload = true;
int PreferSmallDuringInitialShuffle = -1;
size_t MaxNumberOfRuns = ULONG_MAX;
std::string OutputCorpus;
@@ -65,9 +68,10 @@ class Fuzzer {
void ShuffleAndMinimize();
void InitializeDFSan();
size_t CorpusSize() const { return Corpus.size(); }
- void ReadDir(const std::string &Path) {
- ReadDirToVectorOfUnits(Path.c_str(), &Corpus);
+ void ReadDir(const std::string &Path, long *Epoch) {
+ ReadDirToVectorOfUnits(Path.c_str(), &Corpus, Epoch);
}
+ void RereadOutputCorpus();
// Save the current corpus to OutputCorpus.
void SaveCorpus();
@@ -116,6 +120,7 @@ class Fuzzer {
size_t TotalNumberOfRuns = 0;
std::vector<Unit> Corpus;
+ std::set<Unit> UnitsAddedAfterInitialLoad;
std::unordered_set<uintptr_t> FullCoverageSets;
std::unordered_set<uint64_t> CoveragePairs;
@@ -132,6 +137,7 @@ class Fuzzer {
system_clock::time_point ProcessStartTime = system_clock::now();
system_clock::time_point UnitStartTime;
long TimeOfLongestUnitInSeconds = 0;
+ long EpochOfLastReadOfOutputCorpus = 0;
};
}; // namespace fuzzer
Modified: llvm/trunk/lib/Fuzzer/FuzzerLoop.cpp
URL: http://llvm.org/viewvc/llvm-project/llvm/trunk/lib/Fuzzer/FuzzerLoop.cpp?rev=236906&r1=236905&r2=236906&view=diff
==============================================================================
--- llvm/trunk/lib/Fuzzer/FuzzerLoop.cpp (original)
+++ llvm/trunk/lib/Fuzzer/FuzzerLoop.cpp Fri May 8 16:30:55 2015
@@ -86,6 +86,30 @@ void Fuzzer::PrintStats(const char *Wher
<< End;
}
+void Fuzzer::RereadOutputCorpus() {
+ if (Options.OutputCorpus.empty()) return;
+ std::vector<Unit> AdditionalCorpus;
+ ReadDirToVectorOfUnits(Options.OutputCorpus.c_str(), &AdditionalCorpus,
+ &EpochOfLastReadOfOutputCorpus);
+ if (Corpus.empty()) {
+ Corpus = AdditionalCorpus;
+ return;
+ }
+ if (!Options.Reload) return;
+ for (auto &X : AdditionalCorpus) {
+ if (X.size() > (size_t)Options.MaxLen)
+ X.resize(Options.MaxLen);
+ if (UnitsAddedAfterInitialLoad.insert(X).second) {
+ Corpus.push_back(X);
+ CurrentUnit.clear();
+ CurrentUnit.insert(CurrentUnit.begin(), X.begin(), X.end());
+ size_t NewCoverage = RunOne(CurrentUnit);
+ if (NewCoverage && Options.Verbosity >= 1)
+ PrintStats("RELOAD", NewCoverage);
+ }
+ }
+}
+
void Fuzzer::ShuffleAndMinimize() {
size_t MaxCov = 0;
bool PreferSmall =
@@ -268,6 +292,7 @@ void Fuzzer::SaveCorpus() {
void Fuzzer::ReportNewCoverage(size_t NewCoverage, const Unit &U) {
if (!NewCoverage) return;
Corpus.push_back(U);
+ UnitsAddedAfterInitialLoad.insert(U);
PrintStats("NEW ", NewCoverage, "");
if (Options.Verbosity) {
std::cerr << " L: " << U.size();
@@ -299,6 +324,7 @@ void Fuzzer::MutateAndTestOne(Unit *U) {
void Fuzzer::Loop(size_t NumIterations) {
for (size_t i = 1; i <= NumIterations; i++) {
for (size_t J1 = 0; J1 < Corpus.size(); J1++) {
+ RereadOutputCorpus();
if (TotalNumberOfRuns >= Options.MaxNumberOfRuns)
return;
// First, simply mutate the unit w/o doing crosses.
Modified: llvm/trunk/lib/Fuzzer/test/dfsan/CMakeLists.txt
URL: http://llvm.org/viewvc/llvm-project/llvm/trunk/lib/Fuzzer/test/dfsan/CMakeLists.txt?rev=236906&r1=236905&r2=236906&view=diff
==============================================================================
--- llvm/trunk/lib/Fuzzer/test/dfsan/CMakeLists.txt (original)
+++ llvm/trunk/lib/Fuzzer/test/dfsan/CMakeLists.txt Fri May 8 16:30:55 2015
@@ -1,7 +1,7 @@
# These tests depend on both coverage and dfsan instrumentation.
set(CMAKE_CXX_FLAGS_RELEASE
- "${LIBFUZZER_FLAGS_BASE} -O0 -fno-sanitize=all -fsanitize=dataflow -mllvm -sanitizer-coverage-experimental-trace-compares=1")
+ "${LIBFUZZER_FLAGS_BASE} -O0 -fno-sanitize=all -fsanitize=dataflow")
foreach(Test ${DFSanTests})
add_executable(LLVMFuzzer-${Test}
More information about the llvm-commits
mailing list