[compiler-rt] r340976 - [libFuzzer] Remove mutation stats and weighted mutation selection.
Max Moroz via llvm-commits
llvm-commits at lists.llvm.org
Wed Aug 29 14:53:15 PDT 2018
Author: dor1s
Date: Wed Aug 29 14:53:15 2018
New Revision: 340976
URL: http://llvm.org/viewvc/llvm-project?rev=340976&view=rev
Log:
[libFuzzer] Remove mutation stats and weighted mutation selection.
Summary:
This was an experimental feature. After evaluating it with:
1) https://github.com/google/fuzzer-test-suite/tree/master/engine-comparison
2) enabling on real world fuzz targets running at ClusterFuzz and OSS-Fuzz
The following conclusions were made:
1) With fuzz targets that have reached a code coverage plateau, the feature does
not improve libFuzzer's ability to discover new coverage and may actually
negatively impact it.
2) With fuzz targets that have not yet reached a code coverage plateau, the
feature might speed up new units discovery in some cases, but it is quite
rare and hard to confirm with a high level on confidence.
Revert of https://reviews.llvm.org/D48054 and https://reviews.llvm.org/D49621.
Reviewers: metzman, morehouse
Reviewed By: metzman, morehouse
Subscribers: delcypher, #sanitizers, llvm-commits, kcc
Differential Revision: https://reviews.llvm.org/D51455
Removed:
compiler-rt/trunk/test/fuzzer/fuzzer-mutationstats.test
Modified:
compiler-rt/trunk/lib/fuzzer/FuzzerDriver.cpp
compiler-rt/trunk/lib/fuzzer/FuzzerFlags.def
compiler-rt/trunk/lib/fuzzer/FuzzerLoop.cpp
compiler-rt/trunk/lib/fuzzer/FuzzerMutate.cpp
compiler-rt/trunk/lib/fuzzer/FuzzerMutate.h
compiler-rt/trunk/lib/fuzzer/FuzzerOptions.h
Modified: compiler-rt/trunk/lib/fuzzer/FuzzerDriver.cpp
URL: http://llvm.org/viewvc/llvm-project/compiler-rt/trunk/lib/fuzzer/FuzzerDriver.cpp?rev=340976&r1=340975&r2=340976&view=diff
==============================================================================
--- compiler-rt/trunk/lib/fuzzer/FuzzerDriver.cpp (original)
+++ compiler-rt/trunk/lib/fuzzer/FuzzerDriver.cpp Wed Aug 29 14:53:15 2018
@@ -615,8 +615,6 @@ int FuzzerDriver(int *argc, char ***argv
Options.PrintNewCovPcs = Flags.print_pcs;
Options.PrintNewCovFuncs = Flags.print_funcs;
Options.PrintFinalStats = Flags.print_final_stats;
- Options.PrintMutationStats = Flags.print_mutation_stats;
- Options.UseWeightedMutations = Flags.use_weighted_mutations;
Options.PrintCorpusStats = Flags.print_corpus_stats;
Options.PrintCoverage = Flags.print_coverage;
Options.PrintUnstableStats = Flags.print_unstable_stats;
Modified: compiler-rt/trunk/lib/fuzzer/FuzzerFlags.def
URL: http://llvm.org/viewvc/llvm-project/compiler-rt/trunk/lib/fuzzer/FuzzerFlags.def?rev=340976&r1=340975&r2=340976&view=diff
==============================================================================
--- compiler-rt/trunk/lib/fuzzer/FuzzerFlags.def (original)
+++ compiler-rt/trunk/lib/fuzzer/FuzzerFlags.def Wed Aug 29 14:53:15 2018
@@ -162,6 +162,3 @@ FUZZER_DEPRECATED_FLAG(use_equivalence_s
FUZZER_FLAG_INT(analyze_dict, 0, "Experimental")
FUZZER_DEPRECATED_FLAG(use_clang_coverage)
FUZZER_FLAG_STRING(data_flow_trace, "Experimental: use the data flow trace")
-FUZZER_FLAG_INT(print_mutation_stats, 0, "Experimental")
-FUZZER_FLAG_INT(use_weighted_mutations, 0, "Experimental: If 1, fuzzing will "
- "favor mutations that perform better during runtime.")
Modified: compiler-rt/trunk/lib/fuzzer/FuzzerLoop.cpp
URL: http://llvm.org/viewvc/llvm-project/compiler-rt/trunk/lib/fuzzer/FuzzerLoop.cpp?rev=340976&r1=340975&r2=340976&view=diff
==============================================================================
--- compiler-rt/trunk/lib/fuzzer/FuzzerLoop.cpp (original)
+++ compiler-rt/trunk/lib/fuzzer/FuzzerLoop.cpp Wed Aug 29 14:53:15 2018
@@ -38,7 +38,6 @@
namespace fuzzer {
static const size_t kMaxUnitSizeToPrint = 256;
-static const size_t kUpdateMutationWeightRuns = 10000;
thread_local bool Fuzzer::IsMyThread;
@@ -361,7 +360,6 @@ void Fuzzer::PrintFinalStats() {
TPC.DumpCoverage();
if (Options.PrintCorpusStats)
Corpus.PrintStats();
- if (Options.PrintMutationStats) MD.PrintMutationStats();
if (!Options.PrintFinalStats)
return;
size_t ExecPerSec = execPerSec();
@@ -550,9 +548,6 @@ static bool LooseMemeq(const uint8_t *A,
void Fuzzer::ExecuteCallback(const uint8_t *Data, size_t Size) {
TPC.RecordInitialStack();
- if (Options.UseWeightedMutations &&
- TotalNumberOfRuns % kUpdateMutationWeightRuns == 0)
- MD.UpdateDistribution();
TotalNumberOfRuns++;
assert(InFuzzingThread());
if (SMR.IsClient())
Modified: compiler-rt/trunk/lib/fuzzer/FuzzerMutate.cpp
URL: http://llvm.org/viewvc/llvm-project/compiler-rt/trunk/lib/fuzzer/FuzzerMutate.cpp?rev=340976&r1=340975&r2=340976&view=diff
==============================================================================
--- compiler-rt/trunk/lib/fuzzer/FuzzerMutate.cpp (original)
+++ compiler-rt/trunk/lib/fuzzer/FuzzerMutate.cpp Wed Aug 29 14:53:15 2018
@@ -30,41 +30,34 @@ MutationDispatcher::MutationDispatcher(R
DefaultMutators.insert(
DefaultMutators.begin(),
{
- // Initialize useful and total mutation counts as 1 in order to
- // have mutation stats (i.e. weights) with equal non-zero values.
- {&MutationDispatcher::Mutate_EraseBytes, "EraseBytes", 1, 1},
- {&MutationDispatcher::Mutate_InsertByte, "InsertByte", 1, 1},
+ {&MutationDispatcher::Mutate_EraseBytes, "EraseBytes"},
+ {&MutationDispatcher::Mutate_InsertByte, "InsertByte"},
{&MutationDispatcher::Mutate_InsertRepeatedBytes,
- "InsertRepeatedBytes", 1, 1},
- {&MutationDispatcher::Mutate_ChangeByte, "ChangeByte", 1, 1},
- {&MutationDispatcher::Mutate_ChangeBit, "ChangeBit", 1, 1},
- {&MutationDispatcher::Mutate_ShuffleBytes, "ShuffleBytes", 1, 1},
- {&MutationDispatcher::Mutate_ChangeASCIIInteger, "ChangeASCIIInt", 1,
- 1},
- {&MutationDispatcher::Mutate_ChangeBinaryInteger, "ChangeBinInt", 1,
- 1},
- {&MutationDispatcher::Mutate_CopyPart, "CopyPart", 1, 1},
- {&MutationDispatcher::Mutate_CrossOver, "CrossOver", 1, 1},
+ "InsertRepeatedBytes"},
+ {&MutationDispatcher::Mutate_ChangeByte, "ChangeByte"},
+ {&MutationDispatcher::Mutate_ChangeBit, "ChangeBit"},
+ {&MutationDispatcher::Mutate_ShuffleBytes, "ShuffleBytes"},
+ {&MutationDispatcher::Mutate_ChangeASCIIInteger, "ChangeASCIIInt"},
+ {&MutationDispatcher::Mutate_ChangeBinaryInteger, "ChangeBinInt"},
+ {&MutationDispatcher::Mutate_CopyPart, "CopyPart"},
+ {&MutationDispatcher::Mutate_CrossOver, "CrossOver"},
{&MutationDispatcher::Mutate_AddWordFromManualDictionary,
- "ManualDict", 1, 1},
+ "ManualDict"},
{&MutationDispatcher::Mutate_AddWordFromPersistentAutoDictionary,
- "PersAutoDict", 1, 1},
+ "PersAutoDict"},
});
if(Options.UseCmp)
DefaultMutators.push_back(
- {&MutationDispatcher::Mutate_AddWordFromTORC, "CMP", 1, 1});
+ {&MutationDispatcher::Mutate_AddWordFromTORC, "CMP"});
if (EF->LLVMFuzzerCustomMutator)
- Mutators.push_back({&MutationDispatcher::Mutate_Custom, "Custom", 1, 1});
+ Mutators.push_back({&MutationDispatcher::Mutate_Custom, "Custom"});
else
Mutators = DefaultMutators;
if (EF->LLVMFuzzerCustomCrossOver)
Mutators.push_back(
- {&MutationDispatcher::Mutate_CustomCrossOver, "CustomCrossOver", 1, 1});
-
- // For weighted mutation selection, init with uniform weights distribution.
- Stats.resize(Mutators.size());
+ {&MutationDispatcher::Mutate_CustomCrossOver, "CustomCrossOver"});
}
static char RandCh(Random &Rand) {
@@ -471,7 +464,6 @@ void MutationDispatcher::RecordSuccessfu
if (!PersistentAutoDictionary.ContainsWord(DE->GetW()))
PersistentAutoDictionary.push_back({DE->GetW(), 1});
}
- RecordUsefulMutations();
}
void MutationDispatcher::PrintRecommendedDictionary() {
@@ -492,7 +484,8 @@ void MutationDispatcher::PrintRecommende
void MutationDispatcher::PrintMutationSequence() {
Printf("MS: %zd ", CurrentMutatorSequence.size());
- for (auto M : CurrentMutatorSequence) Printf("%s-", M->Name);
+ for (auto M : CurrentMutatorSequence)
+ Printf("%s-", M.Name);
if (!CurrentDictionaryEntrySequence.empty()) {
Printf(" DE: ");
for (auto DE : CurrentDictionaryEntrySequence) {
@@ -519,20 +512,13 @@ size_t MutationDispatcher::MutateImpl(ui
// Some mutations may fail (e.g. can't insert more bytes if Size == MaxSize),
// in which case they will return 0.
// Try several times before returning un-mutated data.
- Mutator *M = nullptr;
for (int Iter = 0; Iter < 100; Iter++) {
- // Even when using weighted mutations, fallback to the default selection in
- // 20% of cases.
- if (Options.UseWeightedMutations && Rand(5))
- M = &Mutators[WeightedIndex()];
- else
- M = &Mutators[Rand(Mutators.size())];
- size_t NewSize = (this->*(M->Fn))(Data, Size, MaxSize);
+ auto M = Mutators[Rand(Mutators.size())];
+ size_t NewSize = (this->*(M.Fn))(Data, Size, MaxSize);
if (NewSize && NewSize <= MaxSize) {
if (Options.OnlyASCII)
ToASCII(Data, NewSize);
CurrentMutatorSequence.push_back(M);
- M->TotalCount++;
return NewSize;
}
}
@@ -573,34 +559,4 @@ void MutationDispatcher::AddWordToManual
{W, std::numeric_limits<size_t>::max()});
}
-void MutationDispatcher::RecordUsefulMutations() {
- for (auto M : CurrentMutatorSequence) M->UsefulCount++;
-}
-
-void MutationDispatcher::PrintMutationStats() {
- Printf("\nstat::mutation_usefulness: ");
- UpdateMutationStats();
- for (size_t i = 0; i < Stats.size(); i++) {
- Printf("%.3f", 100 * Stats[i]);
- if (i < Stats.size() - 1)
- Printf(",");
- else
- Printf("\n");
- }
-}
-
-void MutationDispatcher::UpdateMutationStats() {
- // Calculate usefulness statistic for each mutation
- for (size_t i = 0; i < Stats.size(); i++)
- Stats[i] =
- static_cast<double>(Mutators[i].UsefulCount) / Mutators[i].TotalCount;
-}
-
-void MutationDispatcher::UpdateDistribution() {
- UpdateMutationStats();
- Distribution = std::discrete_distribution<size_t>(Stats.begin(), Stats.end());
-}
-
-size_t MutationDispatcher::WeightedIndex() { return Distribution(GetRand()); }
-
} // namespace fuzzer
Modified: compiler-rt/trunk/lib/fuzzer/FuzzerMutate.h
URL: http://llvm.org/viewvc/llvm-project/compiler-rt/trunk/lib/fuzzer/FuzzerMutate.h?rev=340976&r1=340975&r2=340976&view=diff
==============================================================================
--- compiler-rt/trunk/lib/fuzzer/FuzzerMutate.h (original)
+++ compiler-rt/trunk/lib/fuzzer/FuzzerMutate.h Wed Aug 29 14:53:15 2018
@@ -93,29 +93,10 @@ public:
Random &GetRand() { return Rand; }
- /// Records tally of mutations resulting in new coverage, for usefulness
- /// metric.
- void RecordUsefulMutations();
-
- /// Outputs usefulness stats on command line if option is enabled.
- void PrintMutationStats();
-
- /// Recalculates mutation stats based on latest run data.
- void UpdateMutationStats();
-
- /// Sets weights based on mutation performance during fuzzer run.
- void UpdateDistribution();
-
- /// Returns the index of a mutation based on how useful it has been.
- /// Favors mutations with higher usefulness ratios but can return any index.
- size_t WeightedIndex();
-
private:
struct Mutator {
size_t (MutationDispatcher::*Fn)(uint8_t *Data, size_t Size, size_t Max);
const char *Name;
- uint64_t UsefulCount;
- uint64_t TotalCount;
};
size_t AddWordFromDictionary(Dictionary &D, uint8_t *Data, size_t Size,
@@ -154,7 +135,6 @@ public:
Dictionary PersistentAutoDictionary;
Vector<DictionaryEntry *> CurrentDictionaryEntrySequence;
- Vector<Mutator *> CurrentMutatorSequence;
static const size_t kCmpDictionaryEntriesDequeSize = 16;
DictionaryEntry CmpDictionaryEntriesDeque[kCmpDictionaryEntriesDequeSize];
@@ -169,10 +149,7 @@ public:
Vector<Mutator> Mutators;
Vector<Mutator> DefaultMutators;
-
- // Used to weight mutations based on usefulness.
- Vector<double> Stats;
- std::discrete_distribution<size_t> Distribution;
+ Vector<Mutator> CurrentMutatorSequence;
};
} // namespace fuzzer
Modified: compiler-rt/trunk/lib/fuzzer/FuzzerOptions.h
URL: http://llvm.org/viewvc/llvm-project/compiler-rt/trunk/lib/fuzzer/FuzzerOptions.h?rev=340976&r1=340975&r2=340976&view=diff
==============================================================================
--- compiler-rt/trunk/lib/fuzzer/FuzzerOptions.h (original)
+++ compiler-rt/trunk/lib/fuzzer/FuzzerOptions.h Wed Aug 29 14:53:15 2018
@@ -52,8 +52,6 @@ struct FuzzingOptions {
bool PrintNewCovPcs = false;
int PrintNewCovFuncs = 0;
bool PrintFinalStats = false;
- bool PrintMutationStats = false;
- bool UseWeightedMutations = false;
bool PrintCorpusStats = false;
bool PrintCoverage = false;
bool PrintUnstableStats = false;
Removed: compiler-rt/trunk/test/fuzzer/fuzzer-mutationstats.test
URL: http://llvm.org/viewvc/llvm-project/compiler-rt/trunk/test/fuzzer/fuzzer-mutationstats.test?rev=340975&view=auto
==============================================================================
--- compiler-rt/trunk/test/fuzzer/fuzzer-mutationstats.test (original)
+++ compiler-rt/trunk/test/fuzzer/fuzzer-mutationstats.test (removed)
@@ -1,10 +0,0 @@
-RUN: %cpp_compiler %S/SimpleTest.cpp -o %t-MutationStatsTest
-RUN: not %run %t-MutationStatsTest -print_mutation_stats=1 2>&1 | FileCheck %s --check-prefix=STAT
-
-# Ensures there are some non-zero values in the usefulness percentages printed.
-STAT: stat::mutation_usefulness: {{[0-9]+\.[0-9]+}}
-
-# Weighted mutations only trigger after first 10,000 runs, hence flag.
-RUN: not %run %t-MutationStatsTest -use_weighted_mutations=1 -seed=1 -runs=100000 2>&1 | FileCheck %s --check-prefix=WEIGHTED
-
-WEIGHTED: BINGO
More information about the llvm-commits
mailing list