[compiler-rt] r336597 - [libFuzzer] Mutation tracking and logging implemented
Matt Morehouse via llvm-commits
llvm-commits at lists.llvm.org
Mon Jul 9 13:17:52 PDT 2018
Author: morehouse
Date: Mon Jul 9 13:17:52 2018
New Revision: 336597
URL: http://llvm.org/viewvc/llvm-project?rev=336597&view=rev
Log:
[libFuzzer] Mutation tracking and logging implemented
Code now exists to track number of mutations that are used in fuzzing in
total and ones that produce new coverage. The stats are currently being
dumped to the command line.
Patch By: Kode Williams
Differntial Revision: https://reviews.llvm.org/D48054
Added:
compiler-rt/trunk/test/fuzzer/fuzzer-mutationstats.test
Modified:
compiler-rt/trunk/lib/fuzzer/FuzzerDriver.cpp
compiler-rt/trunk/lib/fuzzer/FuzzerFlags.def
compiler-rt/trunk/lib/fuzzer/FuzzerLoop.cpp
compiler-rt/trunk/lib/fuzzer/FuzzerMutate.cpp
compiler-rt/trunk/lib/fuzzer/FuzzerMutate.h
compiler-rt/trunk/lib/fuzzer/FuzzerOptions.h
Modified: compiler-rt/trunk/lib/fuzzer/FuzzerDriver.cpp
URL: http://llvm.org/viewvc/llvm-project/compiler-rt/trunk/lib/fuzzer/FuzzerDriver.cpp?rev=336597&r1=336596&r2=336597&view=diff
==============================================================================
--- compiler-rt/trunk/lib/fuzzer/FuzzerDriver.cpp (original)
+++ compiler-rt/trunk/lib/fuzzer/FuzzerDriver.cpp Mon Jul 9 13:17:52 2018
@@ -613,6 +613,7 @@ int FuzzerDriver(int *argc, char ***argv
Options.PrintNewCovPcs = Flags.print_pcs;
Options.PrintNewCovFuncs = Flags.print_funcs;
Options.PrintFinalStats = Flags.print_final_stats;
+ Options.PrintMutationStats = Flags.print_mutation_stats;
Options.PrintCorpusStats = Flags.print_corpus_stats;
Options.PrintCoverage = Flags.print_coverage;
Options.DumpCoverage = Flags.dump_coverage;
Modified: compiler-rt/trunk/lib/fuzzer/FuzzerFlags.def
URL: http://llvm.org/viewvc/llvm-project/compiler-rt/trunk/lib/fuzzer/FuzzerFlags.def?rev=336597&r1=336596&r2=336597&view=diff
==============================================================================
--- compiler-rt/trunk/lib/fuzzer/FuzzerFlags.def (original)
+++ compiler-rt/trunk/lib/fuzzer/FuzzerFlags.def Mon Jul 9 13:17:52 2018
@@ -153,3 +153,4 @@ FUZZER_DEPRECATED_FLAG(use_equivalence_s
FUZZER_FLAG_INT(analyze_dict, 0, "Experimental")
FUZZER_DEPRECATED_FLAG(use_clang_coverage)
FUZZER_FLAG_STRING(data_flow_trace, "Experimental: use the data flow trace")
+FUZZER_FLAG_INT(print_mutation_stats, 0, "Experimental")
Modified: compiler-rt/trunk/lib/fuzzer/FuzzerLoop.cpp
URL: http://llvm.org/viewvc/llvm-project/compiler-rt/trunk/lib/fuzzer/FuzzerLoop.cpp?rev=336597&r1=336596&r2=336597&view=diff
==============================================================================
--- compiler-rt/trunk/lib/fuzzer/FuzzerLoop.cpp (original)
+++ compiler-rt/trunk/lib/fuzzer/FuzzerLoop.cpp Mon Jul 9 13:17:52 2018
@@ -355,6 +355,8 @@ void Fuzzer::PrintFinalStats() {
TPC.DumpCoverage();
if (Options.PrintCorpusStats)
Corpus.PrintStats();
+ if (Options.PrintMutationStats)
+ MD.PrintMutationStats();
if (!Options.PrintFinalStats)
return;
size_t ExecPerSec = execPerSec();
Modified: compiler-rt/trunk/lib/fuzzer/FuzzerMutate.cpp
URL: http://llvm.org/viewvc/llvm-project/compiler-rt/trunk/lib/fuzzer/FuzzerMutate.cpp?rev=336597&r1=336596&r2=336597&view=diff
==============================================================================
--- compiler-rt/trunk/lib/fuzzer/FuzzerMutate.cpp (original)
+++ compiler-rt/trunk/lib/fuzzer/FuzzerMutate.cpp Mon Jul 9 13:17:52 2018
@@ -58,6 +58,10 @@ MutationDispatcher::MutationDispatcher(R
if (EF->LLVMFuzzerCustomCrossOver)
Mutators.push_back(
{&MutationDispatcher::Mutate_CustomCrossOver, "CustomCrossOver"});
+
+ // Initialize mutation statistic counters.
+ TotalMutations.resize(Mutators.size(), 0);
+ UsefulMutations.resize(Mutators.size(), 0);
}
static char RandCh(Random &Rand) {
@@ -261,9 +265,9 @@ size_t MutationDispatcher::Mutate_AddWor
DE = MakeDictionaryEntryFromCMP(X.A, X.B, Data, Size);
} break;
case 3: if (Options.UseMemmem) {
- auto X = TPC.MMT.Get(Rand.Rand());
- DE = DictionaryEntry(X);
- } break;
+ auto X = TPC.MMT.Get(Rand.Rand());
+ DE = DictionaryEntry(X);
+ } break;
default:
assert(0);
}
@@ -431,18 +435,18 @@ size_t MutationDispatcher::Mutate_CrossO
auto &U = MutateInPlaceHere;
size_t NewSize = 0;
switch(Rand(3)) {
- case 0:
- NewSize = CrossOver(Data, Size, O.data(), O.size(), U.data(), U.size());
- break;
- case 1:
- NewSize = InsertPartOf(O.data(), O.size(), U.data(), U.size(), MaxSize);
- if (!NewSize)
- NewSize = CopyPartOf(O.data(), O.size(), U.data(), U.size());
- break;
- case 2:
+ case 0:
+ NewSize = CrossOver(Data, Size, O.data(), O.size(), U.data(), U.size());
+ break;
+ case 1:
+ NewSize = InsertPartOf(O.data(), O.size(), U.data(), U.size(), MaxSize);
+ if (!NewSize)
NewSize = CopyPartOf(O.data(), O.size(), U.data(), U.size());
- break;
- default: assert(0);
+ break;
+ case 2:
+ NewSize = CopyPartOf(O.data(), O.size(), U.data(), U.size());
+ break;
+ default: assert(0);
}
assert(NewSize > 0 && "CrossOver returned empty unit");
assert(NewSize <= MaxSize && "CrossOver returned overisized unit");
@@ -451,7 +455,7 @@ size_t MutationDispatcher::Mutate_CrossO
}
void MutationDispatcher::StartMutationSequence() {
- CurrentMutatorSequence.clear();
+ CurrentMutatorIdxSequence.clear();
CurrentDictionaryEntrySequence.clear();
}
@@ -465,6 +469,7 @@ void MutationDispatcher::RecordSuccessfu
if (!PersistentAutoDictionary.ContainsWord(DE->GetW()))
PersistentAutoDictionary.push_back({DE->GetW(), 1});
}
+ RecordUsefulMutations();
}
void MutationDispatcher::PrintRecommendedDictionary() {
@@ -484,9 +489,9 @@ void MutationDispatcher::PrintRecommende
}
void MutationDispatcher::PrintMutationSequence() {
- Printf("MS: %zd ", CurrentMutatorSequence.size());
- for (auto M : CurrentMutatorSequence)
- Printf("%s-", M.Name);
+ Printf("MS: %zd ", CurrentMutatorIdxSequence.size());
+ for (auto M : CurrentMutatorIdxSequence)
+ Printf("%s-", Mutators[M].Name);
if (!CurrentDictionaryEntrySequence.empty()) {
Printf(" DE: ");
for (auto DE : CurrentDictionaryEntrySequence) {
@@ -514,12 +519,14 @@ size_t MutationDispatcher::MutateImpl(ui
// in which case they will return 0.
// Try several times before returning un-mutated data.
for (int Iter = 0; Iter < 100; Iter++) {
- auto M = Mutators[Rand(Mutators.size())];
+ size_t MutatorIdx = Rand(Mutators.size());
+ auto M = Mutators[MutatorIdx];
size_t NewSize = (this->*(M.Fn))(Data, Size, MaxSize);
if (NewSize && NewSize <= MaxSize) {
if (Options.OnlyASCII)
ToASCII(Data, NewSize);
- CurrentMutatorSequence.push_back(M);
+ CurrentMutatorIdxSequence.push_back(MutatorIdx);
+ TotalMutations[MutatorIdx]++;
return NewSize;
}
}
@@ -532,4 +539,23 @@ void MutationDispatcher::AddWordToManual
{W, std::numeric_limits<size_t>::max()});
}
+void MutationDispatcher::RecordUsefulMutations() {
+ for (const size_t M : CurrentMutatorIdxSequence)
+ UsefulMutations[M]++;
+}
+
+void MutationDispatcher::PrintMutationStats() {
+ Printf("\nstat::mutation_usefulness: ");
+ for (size_t i = 0; i < Mutators.size(); i++) {
+ double UsefulPercentage =
+ TotalMutations[i]
+ ? (100.0 * UsefulMutations[i]) / TotalMutations[i]
+ : 0;
+ Printf("%.3f", UsefulPercentage);
+ if (i < Mutators.size() - 1)
+ Printf(",");
+ }
+ Printf("\n");
+}
+
} // namespace fuzzer
Modified: compiler-rt/trunk/lib/fuzzer/FuzzerMutate.h
URL: http://llvm.org/viewvc/llvm-project/compiler-rt/trunk/lib/fuzzer/FuzzerMutate.h?rev=336597&r1=336596&r2=336597&view=diff
==============================================================================
--- compiler-rt/trunk/lib/fuzzer/FuzzerMutate.h (original)
+++ compiler-rt/trunk/lib/fuzzer/FuzzerMutate.h Mon Jul 9 13:17:52 2018
@@ -86,8 +86,11 @@ public:
Random &GetRand() { return Rand; }
-private:
+ void PrintMutationStats();
+
+ void RecordUsefulMutations();
+private:
struct Mutator {
size_t (MutationDispatcher::*Fn)(uint8_t *Data, size_t Size, size_t Max);
const char *Name;
@@ -128,8 +131,8 @@ private:
// entries that led to successful discoveries in the past mutations.
Dictionary PersistentAutoDictionary;
- Vector<Mutator> CurrentMutatorSequence;
Vector<DictionaryEntry *> CurrentDictionaryEntrySequence;
+ Vector<size_t> CurrentMutatorIdxSequence;
static const size_t kCmpDictionaryEntriesDequeSize = 16;
DictionaryEntry CmpDictionaryEntriesDeque[kCmpDictionaryEntriesDequeSize];
@@ -143,6 +146,11 @@ private:
Vector<Mutator> Mutators;
Vector<Mutator> DefaultMutators;
+
+ // A total count of each mutation used in the fuzzing process.
+ Vector<uint64_t> TotalMutations;
+ // The number of each mutation that resulted in new coverage.
+ Vector<uint64_t> UsefulMutations;
};
} // namespace fuzzer
Modified: compiler-rt/trunk/lib/fuzzer/FuzzerOptions.h
URL: http://llvm.org/viewvc/llvm-project/compiler-rt/trunk/lib/fuzzer/FuzzerOptions.h?rev=336597&r1=336596&r2=336597&view=diff
==============================================================================
--- compiler-rt/trunk/lib/fuzzer/FuzzerOptions.h (original)
+++ compiler-rt/trunk/lib/fuzzer/FuzzerOptions.h Mon Jul 9 13:17:52 2018
@@ -52,6 +52,7 @@ struct FuzzingOptions {
bool PrintNewCovPcs = false;
int PrintNewCovFuncs = 0;
bool PrintFinalStats = false;
+ bool PrintMutationStats = false;
bool PrintCorpusStats = false;
bool PrintCoverage = false;
bool DumpCoverage = false;
Added: compiler-rt/trunk/test/fuzzer/fuzzer-mutationstats.test
URL: http://llvm.org/viewvc/llvm-project/compiler-rt/trunk/test/fuzzer/fuzzer-mutationstats.test?rev=336597&view=auto
==============================================================================
--- compiler-rt/trunk/test/fuzzer/fuzzer-mutationstats.test (added)
+++ compiler-rt/trunk/test/fuzzer/fuzzer-mutationstats.test Mon Jul 9 13:17:52 2018
@@ -0,0 +1,5 @@
+RUN: %cpp_compiler %S/SimpleTest.cpp -o %t-MutationStatsTest
+RUN: not %run %t-MutationStatsTest -print_mutation_stats=1 2>&1 | FileCheck %s
+
+# Ensures there are some non-zero values in the usefulness percentages printed.
+CHECK: stat::mutation_usefulness: {{[0-9]+\.[0-9]+}}
More information about the llvm-commits
mailing list