[compiler-rt] r353584 - [libFuzzer] more refactoring; change some of the exit codes (timeout, OOM, interrupt) so that the parent process can distinguish those
Kostya Serebryany via llvm-commits
llvm-commits at lists.llvm.org
Fri Feb 8 16:16:21 PST 2019
Author: kcc
Date: Fri Feb 8 16:16:21 2019
New Revision: 353584
URL: http://llvm.org/viewvc/llvm-project?rev=353584&view=rev
Log:
[libFuzzer] more refactoring; change some of the exit codes (timeout, OOM, interrupt) so that the parent process can distinguish those
Modified:
compiler-rt/trunk/lib/fuzzer/FuzzerDriver.cpp
compiler-rt/trunk/lib/fuzzer/FuzzerLoop.cpp
compiler-rt/trunk/lib/fuzzer/FuzzerMerge.cpp
compiler-rt/trunk/lib/fuzzer/FuzzerMerge.h
compiler-rt/trunk/lib/fuzzer/FuzzerOptions.h
compiler-rt/trunk/lib/fuzzer/tests/FuzzerUnittest.cpp
Modified: compiler-rt/trunk/lib/fuzzer/FuzzerDriver.cpp
URL: http://llvm.org/viewvc/llvm-project/compiler-rt/trunk/lib/fuzzer/FuzzerDriver.cpp?rev=353584&r1=353583&r2=353584&view=diff
==============================================================================
--- compiler-rt/trunk/lib/fuzzer/FuzzerDriver.cpp (original)
+++ compiler-rt/trunk/lib/fuzzer/FuzzerDriver.cpp Fri Feb 8 16:16:21 2019
@@ -475,22 +475,27 @@ int MinimizeCrashInputInternalStep(Fuzze
void FuzzWithFork(const FuzzingOptions &Options,
const Vector<std::string> &Args,
const Vector<std::string> &Corpora) {
- auto CFPath = TempPath(".fork");
Printf("INFO: -fork=1: doing fuzzing in a separate process in order to "
"be more resistant to crashes, timeouts, and OOMs\n");
-
Vector<SizedFile> Corpus;
for (auto &Dir : Corpora)
GetSizedFilesFromDir(Dir, &Corpus);
std::sort(Corpus.begin(), Corpus.end());
- auto Files = CrashResistantMerge(Args, {}, Corpus, CFPath);
- Printf("INFO: -fork=1: seed corpus analyzed, %zd seeds chosen, starting to "
- "fuzz in separate processes\n", Files.size());
+ Vector<std::string> Files;
+ Set<uint32_t> Features;
+ if (!Corpus.empty()) {
+ auto CFPath = TempPath(".fork");
+ CrashResistantMerge(Args, {}, Corpus, &Files, {}, &Features, CFPath);
+ RemoveFile(CFPath);
+ }
+ Printf("INFO: -fork=1: %zd seeds, starting to fuzz\n", Files.size());
Command Cmd(Args);
Cmd.removeFlag("fork");
+ for (auto &C : Corpora) // Remove all corpora from the args.
+ Cmd.removeArgument(C);
if (Files.size() >= 2)
Cmd.addFlag("seed_inputs",
Files.back() + "," + Files[Files.size() - 2]);
@@ -499,11 +504,13 @@ void FuzzWithFork(const FuzzingOptions &
for (size_t i = 0; i < 1000; i++) {
Printf("RUN %s\n", Cmd.toString().c_str());
int ExitCode = ExecuteCommand(Cmd);
- // TODO: sniff the crash, ignore OOMs and timeouts.
+ if (ExitCode == Options.InterruptExitCode)
+ exit(0);
+ if (ExitCode == Options.TimeoutExitCode || ExitCode == Options.OOMExitCode)
+ continue;
if (ExitCode != 0) break;
}
- RemoveFile(CFPath);
exit(0);
}
@@ -522,8 +529,11 @@ void Merge(Fuzzer *F, FuzzingOptions &Op
std::sort(NewCorpus.begin(), NewCorpus.end());
std::string CFPath = CFPathOrNull ? CFPathOrNull : TempPath(".txt");
- auto Files = CrashResistantMerge(Args, OldCorpus, NewCorpus, CFPath);
- for (auto &Path : Files)
+ Vector<std::string> NewFiles;
+ Set<uint32_t> NewFeatures;
+ CrashResistantMerge(Args, OldCorpus, NewCorpus, &NewFiles, {}, &NewFeatures,
+ CFPath);
+ for (auto &Path : NewFiles)
F->WriteToOutputCorpus(FileToVector(Path, Options.MaxLen));
// We are done, delete the control file if it was a temporary one.
if (!Flags.merge_control_file)
Modified: compiler-rt/trunk/lib/fuzzer/FuzzerLoop.cpp
URL: http://llvm.org/viewvc/llvm-project/compiler-rt/trunk/lib/fuzzer/FuzzerLoop.cpp?rev=353584&r1=353583&r2=353584&view=diff
==============================================================================
--- compiler-rt/trunk/lib/fuzzer/FuzzerLoop.cpp (original)
+++ compiler-rt/trunk/lib/fuzzer/FuzzerLoop.cpp Fri Feb 8 16:16:21 2019
@@ -131,7 +131,7 @@ void Fuzzer::HandleMalloc(size_t Size) {
DumpCurrentUnit("oom-");
Printf("SUMMARY: libFuzzer: out-of-memory\n");
PrintFinalStats();
- _Exit(Options.ErrorExitCode); // Stop right now.
+ _Exit(Options.OOMExitCode); // Stop right now.
}
Fuzzer::Fuzzer(UserCallback CB, InputCorpus &Corpus, MutationDispatcher &MD,
@@ -265,7 +265,8 @@ void Fuzzer::MaybeExitGracefully() {
void Fuzzer::InterruptCallback() {
Printf("==%lu== libFuzzer: run interrupted; exiting\n", GetPid());
PrintFinalStats();
- _Exit(0); // Stop right now, don't perform any at-exit actions.
+ // Stop right now, don't perform any at-exit actions.
+ _Exit(Options.InterruptExitCode);
}
NO_SANITIZE_MEMORY
@@ -314,7 +315,7 @@ void Fuzzer::RssLimitCallback() {
DumpCurrentUnit("oom-");
Printf("SUMMARY: libFuzzer: out-of-memory\n");
PrintFinalStats();
- _Exit(Options.ErrorExitCode); // Stop right now.
+ _Exit(Options.OOMExitCode); // Stop right now.
}
void Fuzzer::PrintStats(const char *Where, const char *End, size_t Units) {
Modified: compiler-rt/trunk/lib/fuzzer/FuzzerMerge.cpp
URL: http://llvm.org/viewvc/llvm-project/compiler-rt/trunk/lib/fuzzer/FuzzerMerge.cpp?rev=353584&r1=353583&r2=353584&view=diff
==============================================================================
--- compiler-rt/trunk/lib/fuzzer/FuzzerMerge.cpp (original)
+++ compiler-rt/trunk/lib/fuzzer/FuzzerMerge.cpp Fri Feb 8 16:16:21 2019
@@ -122,25 +122,26 @@ size_t Merger::ApproximateMemoryConsumpt
// Decides which files need to be merged (add thost to NewFiles).
// Returns the number of new features added.
-size_t Merger::Merge(const Set<uint32_t> &InitialFeatures,
+size_t Merger::Merge(const Set<uint32_t> &InitialFeatures,
+ Set<uint32_t> *AllFeatures,
Vector<std::string> *NewFiles) {
NewFiles->clear();
assert(NumFilesInFirstCorpus <= Files.size());
- Set<uint32_t> AllFeatures(InitialFeatures);
+ *AllFeatures = InitialFeatures;
// What features are in the initial corpus?
for (size_t i = 0; i < NumFilesInFirstCorpus; i++) {
auto &Cur = Files[i].Features;
- AllFeatures.insert(Cur.begin(), Cur.end());
+ AllFeatures->insert(Cur.begin(), Cur.end());
}
- size_t InitialNumFeatures = AllFeatures.size();
+ size_t InitialNumFeatures = AllFeatures->size();
// Remove all features that we already know from all other inputs.
for (size_t i = NumFilesInFirstCorpus; i < Files.size(); i++) {
auto &Cur = Files[i].Features;
Vector<uint32_t> Tmp;
- std::set_difference(Cur.begin(), Cur.end(), AllFeatures.begin(),
- AllFeatures.end(), std::inserter(Tmp, Tmp.begin()));
+ std::set_difference(Cur.begin(), Cur.end(), AllFeatures->begin(),
+ AllFeatures->end(), std::inserter(Tmp, Tmp.begin()));
Cur.swap(Tmp);
}
@@ -160,12 +161,12 @@ size_t Merger::Merge(const Set<uint32_t>
auto &Cur = Files[i].Features;
// Printf("%s -> sz %zd ft %zd\n", Files[i].Name.c_str(),
// Files[i].Size, Cur.size());
- size_t OldSize = AllFeatures.size();
- AllFeatures.insert(Cur.begin(), Cur.end());
- if (AllFeatures.size() > OldSize)
+ size_t OldSize = AllFeatures->size();
+ AllFeatures->insert(Cur.begin(), Cur.end());
+ if (AllFeatures->size() > OldSize)
NewFiles->push_back(Files[i].Name);
}
- return AllFeatures.size() - InitialNumFeatures;
+ return AllFeatures->size() - InitialNumFeatures;
}
Set<uint32_t> Merger::AllFeatures() const {
@@ -248,10 +249,12 @@ static void WriteNewControlFile(const st
}
// Outer process. Does not call the target code and thus should not fail.
-Vector<std::string>
-CrashResistantMerge(const Vector<std::string> &Args,
+void CrashResistantMerge(const Vector<std::string> &Args,
const Vector<SizedFile> &OldCorpus,
const Vector<SizedFile> &NewCorpus,
+ Vector<std::string> *NewFiles,
+ const Set<uint32_t> &InitialFeatures,
+ Set<uint32_t> *NewFeatures,
const std::string &CFPath) {
size_t NumAttempts = 0;
if (FileSize(CFPath)) {
@@ -319,12 +322,9 @@ CrashResistantMerge(const Vector<std::st
IF.close();
Printf("MERGE-OUTER: consumed %zdMb (%zdMb rss) to parse the control file\n",
M.ApproximateMemoryConsumption() >> 20, GetPeakRSSMb());
- Set<uint32_t> InitialFeatures;
- Vector<std::string> NewFiles;
- size_t NumNewFeatures = M.Merge(InitialFeatures, &NewFiles);
+ size_t NumNewFeatures = M.Merge(InitialFeatures, NewFeatures, NewFiles);
Printf("MERGE-OUTER: %zd new files with %zd new features added\n",
- NewFiles.size(), NumNewFeatures);
- return NewFiles;
+ NewFiles->size(), NumNewFeatures);
}
} // namespace fuzzer
Modified: compiler-rt/trunk/lib/fuzzer/FuzzerMerge.h
URL: http://llvm.org/viewvc/llvm-project/compiler-rt/trunk/lib/fuzzer/FuzzerMerge.h?rev=353584&r1=353583&r2=353584&view=diff
==============================================================================
--- compiler-rt/trunk/lib/fuzzer/FuzzerMerge.h (original)
+++ compiler-rt/trunk/lib/fuzzer/FuzzerMerge.h Fri Feb 8 16:16:21 2019
@@ -64,16 +64,19 @@ struct Merger {
bool Parse(const std::string &Str, bool ParseCoverage);
void ParseOrExit(std::istream &IS, bool ParseCoverage);
size_t Merge(const Set<uint32_t> &InitialFeatures,
+ Set<uint32_t> *NewFeatures,
Vector<std::string> *NewFiles);
size_t ApproximateMemoryConsumption() const;
Set<uint32_t> AllFeatures() const;
};
-Vector<std::string>
-CrashResistantMerge(const Vector<std::string> &Args,
- const Vector<SizedFile> &OldCorpus,
- const Vector<SizedFile> &NewCorpus,
- const std::string &CFPath);
+void CrashResistantMerge(const Vector<std::string> &Args,
+ const Vector<SizedFile> &OldCorpus,
+ const Vector<SizedFile> &NewCorpus,
+ Vector<std::string> *NewFiles,
+ const Set<uint32_t> &InitialFeatures,
+ Set<uint32_t> *NewFeatures,
+ const std::string &CFPath);
} // namespace fuzzer
Modified: compiler-rt/trunk/lib/fuzzer/FuzzerOptions.h
URL: http://llvm.org/viewvc/llvm-project/compiler-rt/trunk/lib/fuzzer/FuzzerOptions.h?rev=353584&r1=353583&r2=353584&view=diff
==============================================================================
--- compiler-rt/trunk/lib/fuzzer/FuzzerOptions.h (original)
+++ compiler-rt/trunk/lib/fuzzer/FuzzerOptions.h Fri Feb 8 16:16:21 2019
@@ -19,7 +19,9 @@ struct FuzzingOptions {
size_t MaxLen = 0;
size_t LenControl = 1000;
int UnitTimeoutSec = 300;
- int TimeoutExitCode = 77;
+ int TimeoutExitCode = 70;
+ int OOMExitCode = 71;
+ int InterruptExitCode = 72;
int ErrorExitCode = 77;
int MaxTotalTimeSec = 0;
int RssLimitMb = 0;
Modified: compiler-rt/trunk/lib/fuzzer/tests/FuzzerUnittest.cpp
URL: http://llvm.org/viewvc/llvm-project/compiler-rt/trunk/lib/fuzzer/tests/FuzzerUnittest.cpp?rev=353584&r1=353583&r2=353584&view=diff
==============================================================================
--- compiler-rt/trunk/lib/fuzzer/tests/FuzzerUnittest.cpp (original)
+++ compiler-rt/trunk/lib/fuzzer/tests/FuzzerUnittest.cpp Fri Feb 8 16:16:21 2019
@@ -644,8 +644,9 @@ static void Merge(const std::string &Inp
size_t NumNewFeatures) {
Merger M;
Vector<std::string> NewFiles;
+ Set<uint32_t> NewFeatures;
EXPECT_TRUE(M.Parse(Input, true));
- EXPECT_EQ(NumNewFeatures, M.Merge({}, &NewFiles));
+ EXPECT_EQ(NumNewFeatures, M.Merge({}, &NewFeatures, &NewFiles));
EQ(NewFiles, Result);
}
@@ -689,6 +690,7 @@ TEST(Merge, Good) {
Vector<std::string> NewFiles;
+ Set<uint32_t> NewFeatures;
EXPECT_TRUE(M.Parse("3\n2\nAA\nBB\nC\n"
"STARTED 0 1000\nDONE 0 1 2 3\n"
@@ -702,7 +704,7 @@ TEST(Merge, Good) {
EQ(M.Files[0].Features, {1, 2, 3});
EQ(M.Files[1].Features, {4, 5, 6});
EQ(M.Files[2].Features, {1, 3, 6});
- EXPECT_EQ(0U, M.Merge({}, &NewFiles));
+ EXPECT_EQ(0U, M.Merge({}, &NewFeatures, &NewFiles));
EQ(NewFiles, {});
EXPECT_TRUE(M.Parse("3\n1\nA\nB\nC\n"
@@ -713,7 +715,7 @@ TEST(Merge, Good) {
EQ(M.Files[0].Features, {1, 2, 3});
EQ(M.Files[1].Features, {4, 5, 6});
EQ(M.Files[2].Features, {1, 3, 6});
- EXPECT_EQ(3U, M.Merge({}, &NewFiles));
+ EXPECT_EQ(3U, M.Merge({}, &NewFeatures, &NewFiles));
EQ(NewFiles, {"B"});
// Same as the above, but with InitialFeatures.
@@ -727,7 +729,7 @@ TEST(Merge, Good) {
InitialFeatures.insert(1);
InitialFeatures.insert(2);
InitialFeatures.insert(3);
- EXPECT_EQ(3U, M.Merge(InitialFeatures, &NewFiles));
+ EXPECT_EQ(3U, M.Merge(InitialFeatures, &NewFeatures, &NewFiles));
EQ(NewFiles, {"B"});
}
More information about the llvm-commits
mailing list