[compiler-rt] r337324 - [libFuzzer] Mutation tracking and logging implemented.
Max Moroz via llvm-commits
llvm-commits at lists.llvm.org
Tue Jul 17 13:37:40 PDT 2018
Author: dor1s
Date: Tue Jul 17 13:37:40 2018
New Revision: 337324
URL: http://llvm.org/viewvc/llvm-project?rev=337324&view=rev
Log:
[libFuzzer] Mutation tracking and logging implemented.
Summary:
Code now exists to track number of mutations that are used in fuzzing in total
and ones that produce new coverage. The stats are currently being dumped to the
command line.
Patch by Kodé Williams (@kodewilliams).
Reviewers: metzman, Dor1s, morehouse, kcc
Reviewed By: Dor1s, morehouse, kcc
Subscribers: delcypher, kubamracek, kcc, morehouse, llvm-commits, #sanitizers, mgorny
Differential Revision: https://reviews.llvm.org/D48054
Added:
compiler-rt/trunk/test/fuzzer/fuzzer-mutationstats.test
Modified:
compiler-rt/trunk/lib/fuzzer/FuzzerDriver.cpp
compiler-rt/trunk/lib/fuzzer/FuzzerFlags.def
compiler-rt/trunk/lib/fuzzer/FuzzerLoop.cpp
compiler-rt/trunk/lib/fuzzer/FuzzerMutate.cpp
compiler-rt/trunk/lib/fuzzer/FuzzerMutate.h
compiler-rt/trunk/lib/fuzzer/FuzzerOptions.h
Modified: compiler-rt/trunk/lib/fuzzer/FuzzerDriver.cpp
URL: http://llvm.org/viewvc/llvm-project/compiler-rt/trunk/lib/fuzzer/FuzzerDriver.cpp?rev=337324&r1=337323&r2=337324&view=diff
==============================================================================
--- compiler-rt/trunk/lib/fuzzer/FuzzerDriver.cpp (original)
+++ compiler-rt/trunk/lib/fuzzer/FuzzerDriver.cpp Tue Jul 17 13:37:40 2018
@@ -615,6 +615,7 @@ int FuzzerDriver(int *argc, char ***argv
Options.PrintNewCovPcs = Flags.print_pcs;
Options.PrintNewCovFuncs = Flags.print_funcs;
Options.PrintFinalStats = Flags.print_final_stats;
+ Options.PrintMutationStats = Flags.print_mutation_stats;
Options.PrintCorpusStats = Flags.print_corpus_stats;
Options.PrintCoverage = Flags.print_coverage;
Options.PrintUnstableStats = Flags.print_unstable_stats;
Modified: compiler-rt/trunk/lib/fuzzer/FuzzerFlags.def
URL: http://llvm.org/viewvc/llvm-project/compiler-rt/trunk/lib/fuzzer/FuzzerFlags.def?rev=337324&r1=337323&r2=337324&view=diff
==============================================================================
--- compiler-rt/trunk/lib/fuzzer/FuzzerFlags.def (original)
+++ compiler-rt/trunk/lib/fuzzer/FuzzerFlags.def Tue Jul 17 13:37:40 2018
@@ -155,3 +155,4 @@ FUZZER_DEPRECATED_FLAG(use_equivalence_s
FUZZER_FLAG_INT(analyze_dict, 0, "Experimental")
FUZZER_DEPRECATED_FLAG(use_clang_coverage)
FUZZER_FLAG_STRING(data_flow_trace, "Experimental: use the data flow trace")
+FUZZER_FLAG_INT(print_mutation_stats, 0, "Experimental")
Modified: compiler-rt/trunk/lib/fuzzer/FuzzerLoop.cpp
URL: http://llvm.org/viewvc/llvm-project/compiler-rt/trunk/lib/fuzzer/FuzzerLoop.cpp?rev=337324&r1=337323&r2=337324&view=diff
==============================================================================
--- compiler-rt/trunk/lib/fuzzer/FuzzerLoop.cpp (original)
+++ compiler-rt/trunk/lib/fuzzer/FuzzerLoop.cpp Tue Jul 17 13:37:40 2018
@@ -360,6 +360,7 @@ void Fuzzer::PrintFinalStats() {
TPC.DumpCoverage();
if (Options.PrintCorpusStats)
Corpus.PrintStats();
+ if (Options.PrintMutationStats) MD.PrintMutationStats();
if (!Options.PrintFinalStats)
return;
size_t ExecPerSec = execPerSec();
Modified: compiler-rt/trunk/lib/fuzzer/FuzzerMutate.cpp
URL: http://llvm.org/viewvc/llvm-project/compiler-rt/trunk/lib/fuzzer/FuzzerMutate.cpp?rev=337324&r1=337323&r2=337324&view=diff
==============================================================================
--- compiler-rt/trunk/lib/fuzzer/FuzzerMutate.cpp (original)
+++ compiler-rt/trunk/lib/fuzzer/FuzzerMutate.cpp Tue Jul 17 13:37:40 2018
@@ -30,34 +30,36 @@ MutationDispatcher::MutationDispatcher(R
DefaultMutators.insert(
DefaultMutators.begin(),
{
- {&MutationDispatcher::Mutate_EraseBytes, "EraseBytes"},
- {&MutationDispatcher::Mutate_InsertByte, "InsertByte"},
+ {&MutationDispatcher::Mutate_EraseBytes, "EraseBytes", 0, 0},
+ {&MutationDispatcher::Mutate_InsertByte, "InsertByte", 0, 0},
{&MutationDispatcher::Mutate_InsertRepeatedBytes,
- "InsertRepeatedBytes"},
- {&MutationDispatcher::Mutate_ChangeByte, "ChangeByte"},
- {&MutationDispatcher::Mutate_ChangeBit, "ChangeBit"},
- {&MutationDispatcher::Mutate_ShuffleBytes, "ShuffleBytes"},
- {&MutationDispatcher::Mutate_ChangeASCIIInteger, "ChangeASCIIInt"},
- {&MutationDispatcher::Mutate_ChangeBinaryInteger, "ChangeBinInt"},
- {&MutationDispatcher::Mutate_CopyPart, "CopyPart"},
- {&MutationDispatcher::Mutate_CrossOver, "CrossOver"},
+ "InsertRepeatedBytes", 0, 0},
+ {&MutationDispatcher::Mutate_ChangeByte, "ChangeByte", 0, 0},
+ {&MutationDispatcher::Mutate_ChangeBit, "ChangeBit", 0, 0},
+ {&MutationDispatcher::Mutate_ShuffleBytes, "ShuffleBytes", 0, 0},
+ {&MutationDispatcher::Mutate_ChangeASCIIInteger, "ChangeASCIIInt", 0,
+ 0},
+ {&MutationDispatcher::Mutate_ChangeBinaryInteger, "ChangeBinInt", 0,
+ 0},
+ {&MutationDispatcher::Mutate_CopyPart, "CopyPart", 0, 0},
+ {&MutationDispatcher::Mutate_CrossOver, "CrossOver", 0, 0},
{&MutationDispatcher::Mutate_AddWordFromManualDictionary,
- "ManualDict"},
+ "ManualDict", 0, 0},
{&MutationDispatcher::Mutate_AddWordFromPersistentAutoDictionary,
- "PersAutoDict"},
+ "PersAutoDict", 0, 0},
});
if(Options.UseCmp)
DefaultMutators.push_back(
- {&MutationDispatcher::Mutate_AddWordFromTORC, "CMP"});
+ {&MutationDispatcher::Mutate_AddWordFromTORC, "CMP", 0, 0});
if (EF->LLVMFuzzerCustomMutator)
- Mutators.push_back({&MutationDispatcher::Mutate_Custom, "Custom"});
+ Mutators.push_back({&MutationDispatcher::Mutate_Custom, "Custom", 0, 0});
else
Mutators = DefaultMutators;
if (EF->LLVMFuzzerCustomCrossOver)
Mutators.push_back(
- {&MutationDispatcher::Mutate_CustomCrossOver, "CustomCrossOver"});
+ {&MutationDispatcher::Mutate_CustomCrossOver, "CustomCrossOver", 0, 0});
}
static char RandCh(Random &Rand) {
@@ -464,6 +466,7 @@ void MutationDispatcher::RecordSuccessfu
if (!PersistentAutoDictionary.ContainsWord(DE->GetW()))
PersistentAutoDictionary.push_back({DE->GetW(), 1});
}
+ RecordUsefulMutations();
}
void MutationDispatcher::PrintRecommendedDictionary() {
@@ -484,8 +487,7 @@ void MutationDispatcher::PrintRecommende
void MutationDispatcher::PrintMutationSequence() {
Printf("MS: %zd ", CurrentMutatorSequence.size());
- for (auto M : CurrentMutatorSequence)
- Printf("%s-", M.Name);
+ for (auto M : CurrentMutatorSequence) Printf("%s-", M->Name);
if (!CurrentDictionaryEntrySequence.empty()) {
Printf(" DE: ");
for (auto DE : CurrentDictionaryEntrySequence) {
@@ -513,12 +515,13 @@ size_t MutationDispatcher::MutateImpl(ui
// in which case they will return 0.
// Try several times before returning un-mutated data.
for (int Iter = 0; Iter < 100; Iter++) {
- auto M = Mutators[Rand(Mutators.size())];
- size_t NewSize = (this->*(M.Fn))(Data, Size, MaxSize);
+ auto M = &Mutators[Rand(Mutators.size())];
+ size_t NewSize = (this->*(M->Fn))(Data, Size, MaxSize);
if (NewSize && NewSize <= MaxSize) {
if (Options.OnlyASCII)
ToASCII(Data, NewSize);
CurrentMutatorSequence.push_back(M);
+ M->TotalCount++;
return NewSize;
}
}
@@ -531,4 +534,21 @@ void MutationDispatcher::AddWordToManual
{W, std::numeric_limits<size_t>::max()});
}
+void MutationDispatcher::RecordUsefulMutations() {
+ for (auto M : CurrentMutatorSequence) M->UsefulCount++;
+}
+
+void MutationDispatcher::PrintMutationStats() {
+ Printf("\nstat::mutation_usefulness: ");
+ for (size_t i = 0; i < Mutators.size(); i++) {
+ double UsefulPercentage =
+ Mutators[i].TotalCount
+ ? (100.0 * Mutators[i].UsefulCount) / Mutators[i].TotalCount
+ : 0;
+ Printf("%.3f", UsefulPercentage);
+ if (i < Mutators.size() - 1) Printf(",");
+ }
+ Printf("\n");
+}
+
} // namespace fuzzer
Modified: compiler-rt/trunk/lib/fuzzer/FuzzerMutate.h
URL: http://llvm.org/viewvc/llvm-project/compiler-rt/trunk/lib/fuzzer/FuzzerMutate.h?rev=337324&r1=337323&r2=337324&view=diff
==============================================================================
--- compiler-rt/trunk/lib/fuzzer/FuzzerMutate.h (original)
+++ compiler-rt/trunk/lib/fuzzer/FuzzerMutate.h Tue Jul 17 13:37:40 2018
@@ -86,11 +86,16 @@ public:
Random &GetRand() { return Rand; }
-private:
+ void PrintMutationStats();
+ void RecordUsefulMutations();
+
+ private:
struct Mutator {
size_t (MutationDispatcher::*Fn)(uint8_t *Data, size_t Size, size_t Max);
const char *Name;
+ uint64_t UsefulCount;
+ uint64_t TotalCount;
};
size_t AddWordFromDictionary(Dictionary &D, uint8_t *Data, size_t Size,
@@ -128,8 +133,8 @@ private:
// entries that led to successful discoveries in the past mutations.
Dictionary PersistentAutoDictionary;
- Vector<Mutator> CurrentMutatorSequence;
Vector<DictionaryEntry *> CurrentDictionaryEntrySequence;
+ Vector<Mutator *> CurrentMutatorSequence;
static const size_t kCmpDictionaryEntriesDequeSize = 16;
DictionaryEntry CmpDictionaryEntriesDeque[kCmpDictionaryEntriesDequeSize];
Modified: compiler-rt/trunk/lib/fuzzer/FuzzerOptions.h
URL: http://llvm.org/viewvc/llvm-project/compiler-rt/trunk/lib/fuzzer/FuzzerOptions.h?rev=337324&r1=337323&r2=337324&view=diff
==============================================================================
--- compiler-rt/trunk/lib/fuzzer/FuzzerOptions.h (original)
+++ compiler-rt/trunk/lib/fuzzer/FuzzerOptions.h Tue Jul 17 13:37:40 2018
@@ -52,6 +52,7 @@ struct FuzzingOptions {
bool PrintNewCovPcs = false;
int PrintNewCovFuncs = 0;
bool PrintFinalStats = false;
+ bool PrintMutationStats = false;
bool PrintCorpusStats = false;
bool PrintCoverage = false;
bool PrintUnstableStats = false;
Added: compiler-rt/trunk/test/fuzzer/fuzzer-mutationstats.test
URL: http://llvm.org/viewvc/llvm-project/compiler-rt/trunk/test/fuzzer/fuzzer-mutationstats.test?rev=337324&view=auto
==============================================================================
--- compiler-rt/trunk/test/fuzzer/fuzzer-mutationstats.test (added)
+++ compiler-rt/trunk/test/fuzzer/fuzzer-mutationstats.test Tue Jul 17 13:37:40 2018
@@ -0,0 +1,5 @@
+RUN: %cpp_compiler %S/SimpleTest.cpp -o %t-MutationStatsTest
+RUN: not %run %t-MutationStatsTest -print_mutation_stats=1 2>&1 | FileCheck %s
+
+# Ensures there are some non-zero values in the usefulness percentages printed.
+CHECK: stat::mutation_usefulness: {{[0-9]+\.[0-9]+}}
More information about the llvm-commits
mailing list