[llvm] r285049 - [libFuzzer] simplify the code for use_cmp, also use the position hint when available, add a test
Kostya Serebryany via llvm-commits
llvm-commits at lists.llvm.org
Mon Oct 24 19:04:43 PDT 2016
Author: kcc
Date: Mon Oct 24 21:04:43 2016
New Revision: 285049
URL: http://llvm.org/viewvc/llvm-project?rev=285049&view=rev
Log:
[libFuzzer] simplify the code for use_cmp, also use the position hint when available, add a test
Added:
llvm/trunk/lib/Fuzzer/test/swap-cmp.test
Modified:
llvm/trunk/lib/Fuzzer/FuzzerDictionary.h
llvm/trunk/lib/Fuzzer/FuzzerLoop.cpp
llvm/trunk/lib/Fuzzer/FuzzerMutate.cpp
llvm/trunk/lib/Fuzzer/FuzzerMutate.h
llvm/trunk/lib/Fuzzer/FuzzerTracePC.cpp
llvm/trunk/lib/Fuzzer/FuzzerTracePC.h
llvm/trunk/lib/Fuzzer/test/SimpleCmpTest.cpp
llvm/trunk/lib/Fuzzer/test/SwapCmpTest.cpp
Modified: llvm/trunk/lib/Fuzzer/FuzzerDictionary.h
URL: http://llvm.org/viewvc/llvm-project/llvm/trunk/lib/Fuzzer/FuzzerDictionary.h?rev=285049&r1=285048&r2=285049&view=diff
==============================================================================
--- llvm/trunk/lib/Fuzzer/FuzzerDictionary.h (original)
+++ llvm/trunk/lib/Fuzzer/FuzzerDictionary.h Mon Oct 24 21:04:43 2016
@@ -68,6 +68,13 @@ class DictionaryEntry {
size_t GetUseCount() const { return UseCount; }
size_t GetSuccessCount() const {return SuccessCount; }
+ void Print(const char *PrintAfter = "\n") {
+ PrintASCII(W.data(), W.size());
+ if (HasPositionHint())
+ Printf("@%zd", GetPositionHint());
+ Printf("%s", PrintAfter);
+ }
+
private:
Word W;
size_t PositionHint = std::numeric_limits<size_t>::max();
Modified: llvm/trunk/lib/Fuzzer/FuzzerLoop.cpp
URL: http://llvm.org/viewvc/llvm-project/llvm/trunk/lib/Fuzzer/FuzzerLoop.cpp?rev=285049&r1=285048&r2=285049&view=diff
==============================================================================
--- llvm/trunk/lib/Fuzzer/FuzzerLoop.cpp (original)
+++ llvm/trunk/lib/Fuzzer/FuzzerLoop.cpp Mon Oct 24 21:04:43 2016
@@ -478,9 +478,6 @@ size_t Fuzzer::RunOne(const uint8_t *Dat
Res = 1;
}
- if (Res && Options.UseCmp)
- TPC.ProcessTORC(MD.GetTraceCmpDictionary(), CurrentUnitData, Size);
-
auto TimeOfUnit =
duration_cast<seconds>(UnitStopTime - UnitStartTime).count();
if (!(TotalNumberOfRuns & (TotalNumberOfRuns - 1)) &&
@@ -514,8 +511,6 @@ void Fuzzer::ExecuteCallback(const uint8
UnitStartTime = system_clock::now();
ResetCounters(); // Reset coverage right before the callback.
TPC.ResetMaps();
- if (Options.UseCmp)
- TPC.ResetTORC();
if (Options.UseCounters)
TPC.ResetGuards();
int Res = CB(DataCopy, Size);
Modified: llvm/trunk/lib/Fuzzer/FuzzerMutate.cpp
URL: http://llvm.org/viewvc/llvm-project/llvm/trunk/lib/Fuzzer/FuzzerMutate.cpp?rev=285049&r1=285048&r2=285049&view=diff
==============================================================================
--- llvm/trunk/lib/Fuzzer/FuzzerMutate.cpp (original)
+++ llvm/trunk/lib/Fuzzer/FuzzerMutate.cpp Mon Oct 24 21:04:43 2016
@@ -51,8 +51,7 @@ MutationDispatcher::MutationDispatcher(R
});
if(Options.UseCmp)
DefaultMutators.push_back(
- {&MutationDispatcher::Mutate_AddWordFromTraceCmpDictionary,
- "TraceCmpDict"});
+ {&MutationDispatcher::Mutate_AddWordFromTORC, "CMP"});
if (EF->LLVMFuzzerCustomMutator)
Mutators.push_back({&MutationDispatcher::Mutate_Custom, "Custom"});
@@ -175,24 +174,13 @@ size_t MutationDispatcher::Mutate_AddWor
return AddWordFromDictionary(TempAutoDictionary, Data, Size, MaxSize);
}
-size_t MutationDispatcher::Mutate_AddWordFromTraceCmpDictionary(
- uint8_t *Data, size_t Size, size_t MaxSize) {
- return AddWordFromDictionary(TraceCmpDictionary, Data, Size, MaxSize);
-}
-
-size_t MutationDispatcher::Mutate_AddWordFromPersistentAutoDictionary(
- uint8_t *Data, size_t Size, size_t MaxSize) {
- return AddWordFromDictionary(PersistentAutoDictionary, Data, Size, MaxSize);
-}
-
-size_t MutationDispatcher::AddWordFromDictionary(Dictionary &D, uint8_t *Data,
- size_t Size, size_t MaxSize) {
- if (Size > MaxSize) return 0;
- if (D.empty()) return 0;
- DictionaryEntry &DE = D[Rand(D.size())];
+size_t MutationDispatcher::ApplyDictionaryEntry(uint8_t *Data, size_t Size,
+ size_t MaxSize,
+ DictionaryEntry &DE) {
const Word &W = DE.GetW();
bool UsePositionHint = DE.HasPositionHint() &&
- DE.GetPositionHint() + W.size() < Size && Rand.RandBool();
+ DE.GetPositionHint() + W.size() < Size &&
+ Rand.RandBool();
if (Rand.RandBool()) { // Insert W.
if (Size + W.size() > MaxSize) return 0;
size_t Idx = UsePositionHint ? DE.GetPositionHint() : Rand(Size + 1);
@@ -204,6 +192,92 @@ size_t MutationDispatcher::AddWordFromDi
size_t Idx = UsePositionHint ? DE.GetPositionHint() : Rand(Size - W.size());
memcpy(Data + Idx, W.data(), W.size());
}
+ return Size;
+}
+
+// Somewhere in the past we have observed a comparison instructions
+// with arguments Arg1 Arg2. This function tries to guess a dictionary
+// entry that will satisfy that comparison.
+// It first tries to find one of the arguments (possibly swapped) in the
+// input and if it succeeds it creates a DE with a position hint.
+// Otherwise it creates a DE with one of the arguments w/o a position hint.
+template <class T>
+DictionaryEntry MutationDispatcher::MakeDictionaryEntryFromCMP(
+ T Arg1, T Arg2, const uint8_t *Data, size_t Size) {
+ ScopedDoingMyOwnMemmem scoped_doing_my_own_memmem;
+ if (Rand.RandBool()) Arg1 = Bswap(Arg1);
+ if (Rand.RandBool()) Arg2 = Bswap(Arg2);
+ bool HandleFirst = Rand.RandBool();
+ T ExistingBytes, DesiredBytes;
+ Word W;
+ const uint8_t *End = Data + Size;
+ for (int Arg = 0; Arg < 2; Arg++) {
+ ExistingBytes = HandleFirst ? Arg1 : Arg2;
+ DesiredBytes = HandleFirst ? Arg2 : Arg1;
+ HandleFirst = !HandleFirst;
+ W.Set(reinterpret_cast<uint8_t*>(&DesiredBytes), sizeof(T));
+ const size_t kMaxNumPositions = 8;
+ size_t Positions[kMaxNumPositions];
+ size_t NumPositions = 0;
+ for (const uint8_t *Cur = Data;
+ Cur < End && NumPositions < kMaxNumPositions; Cur++) {
+ Cur = (uint8_t *)memmem(Cur, End - Cur, &ExistingBytes, sizeof(T));
+ if (!Cur) break;
+ Positions[NumPositions++] = Cur - Data;
+ }
+ if (!NumPositions) break;
+ return DictionaryEntry(W, Positions[Rand(NumPositions)]);
+ }
+ DictionaryEntry DE(W);
+ return DE;
+}
+
+size_t MutationDispatcher::Mutate_AddWordFromTORC(
+ uint8_t *Data, size_t Size, size_t MaxSize) {
+ Word W;
+ DictionaryEntry DE;
+ bool Debug = false;
+ if (Rand.RandBool()) {
+ auto X = TPC.TORC8.Get(Rand.Rand());
+ DE = MakeDictionaryEntryFromCMP(X.A, X.B, Data, Size);
+ if (X.A > 10000 &&X.B > 10000) Debug = false;
+ if (Debug) {
+ Printf("ZZZ %zx %zx\n", X.A, X.B);
+ DE.Print();
+ }
+ } else {
+ auto X = TPC.TORC4.Get(Rand.Rand());
+ if ((X.A >> 16) == 0 && (X.B >> 16) == 0 && Rand.RandBool())
+ DE = MakeDictionaryEntryFromCMP((uint16_t)X.A, (uint16_t)X.B, Data,
+ Size);
+ else
+ DE = MakeDictionaryEntryFromCMP(X.A, X.B, Data, Size);
+ }
+ Size = ApplyDictionaryEntry(Data, Size, MaxSize, DE);
+ if (!Size) return 0;
+ if (Debug) {
+ Printf("DONE\n");
+ }
+ DictionaryEntry &DERef =
+ CmpDictionaryEntriesDeque[CmpDictionaryEntriesDequeIdx++ %
+ kCmpDictionaryEntriesDequeSize];
+ DERef = DE;
+ CurrentDictionaryEntrySequence.push_back(&DERef);
+ return Size;
+}
+
+size_t MutationDispatcher::Mutate_AddWordFromPersistentAutoDictionary(
+ uint8_t *Data, size_t Size, size_t MaxSize) {
+ return AddWordFromDictionary(PersistentAutoDictionary, Data, Size, MaxSize);
+}
+
+size_t MutationDispatcher::AddWordFromDictionary(Dictionary &D, uint8_t *Data,
+ size_t Size, size_t MaxSize) {
+ if (Size > MaxSize) return 0;
+ if (D.empty()) return 0;
+ DictionaryEntry &DE = D[Rand(D.size())];
+ Size = ApplyDictionaryEntry(Data, Size, MaxSize, DE);
+ if (!Size) return 0;
DE.IncUseCount();
CurrentDictionaryEntrySequence.push_back(&DE);
return Size;
Modified: llvm/trunk/lib/Fuzzer/FuzzerMutate.h
URL: http://llvm.org/viewvc/llvm-project/llvm/trunk/lib/Fuzzer/FuzzerMutate.h?rev=285049&r1=285048&r2=285049&view=diff
==============================================================================
--- llvm/trunk/lib/Fuzzer/FuzzerMutate.h (original)
+++ llvm/trunk/lib/Fuzzer/FuzzerMutate.h Mon Oct 24 21:04:43 2016
@@ -55,9 +55,8 @@ public:
size_t Mutate_AddWordFromTemporaryAutoDictionary(uint8_t *Data, size_t Size,
size_t MaxSize);
- /// Mutates data by adding a word from the trace-cmp dictionary.
- size_t Mutate_AddWordFromTraceCmpDictionary(uint8_t *Data, size_t Size,
- size_t MaxSize);
+ /// Mutates data by adding a word from the TORC.
+ size_t Mutate_AddWordFromTORC(uint8_t *Data, size_t Size, size_t MaxSize);
/// Mutates data by adding a word from the persistent automatic dictionary.
size_t Mutate_AddWordFromPersistentAutoDictionary(uint8_t *Data, size_t Size,
@@ -92,8 +91,6 @@ public:
Random &GetRand() { return Rand; }
- Dictionary *GetTraceCmpDictionary() { return &TraceCmpDictionary; }
-
private:
struct Mutator {
@@ -110,6 +107,12 @@ private:
size_t ToSize, size_t MaxToSize);
size_t CopyPartOf(const uint8_t *From, size_t FromSize, uint8_t *To,
size_t ToSize);
+ size_t ApplyDictionaryEntry(uint8_t *Data, size_t Size, size_t MaxSize,
+ DictionaryEntry &DE);
+
+ template <class T>
+ DictionaryEntry MakeDictionaryEntryFromCMP(T Arg1, T Arg2,
+ const uint8_t *Data, size_t Size);
Random &Rand;
const FuzzingOptions &Options;
@@ -123,11 +126,13 @@ private:
// entries that led to successfull discoveries in the past mutations.
Dictionary PersistentAutoDictionary;
- // Dictionary from tracing CMP instructions.
- Dictionary TraceCmpDictionary;
-
std::vector<Mutator> CurrentMutatorSequence;
std::vector<DictionaryEntry *> CurrentDictionaryEntrySequence;
+
+ static const size_t kCmpDictionaryEntriesDequeSize = 16;
+ DictionaryEntry CmpDictionaryEntriesDeque[kCmpDictionaryEntriesDequeSize];
+ size_t CmpDictionaryEntriesDequeIdx = 0;
+
const InputCorpus *Corpus = nullptr;
std::vector<uint8_t> MutateInPlaceHere;
Modified: llvm/trunk/lib/Fuzzer/FuzzerTracePC.cpp
URL: http://llvm.org/viewvc/llvm-project/llvm/trunk/lib/Fuzzer/FuzzerTracePC.cpp?rev=285049&r1=285048&r2=285049&view=diff
==============================================================================
--- llvm/trunk/lib/Fuzzer/FuzzerTracePC.cpp (original)
+++ llvm/trunk/lib/Fuzzer/FuzzerTracePC.cpp Mon Oct 24 21:04:43 2016
@@ -272,55 +272,6 @@ void TracePC::HandleCmp(void *PC, T Arg1
HandleValueProfile(Idx);
}
-void TracePC::ProcessTORC(Dictionary *Dict, const uint8_t *Data, size_t Size) {
- TORCToDict(TORC8, Dict, Data, Size);
- TORCToDict(TORC4, Dict, Data, Size);
-}
-
-template <class T>
-void TracePC::TORCToDict(const TableOfRecentCompares<T, kTORCSize> &TORC,
- Dictionary *Dict, const uint8_t *Data, size_t Size) {
- ScopedDoingMyOwnMemmem scoped_doing_my_own_memmem;
- for (size_t i = 0; i < TORC.kSize; i++) {
- T A[2] = {TORC.Table[i][0], TORC.Table[i][1]};
- if (!A[0] && !A[1]) continue;
- for (int j = 0; j < 2; j++)
- TORCToDict(Dict, A[j], A[!j], Data, Size);
- }
-}
-
-template <class T>
-void TracePC::TORCToDict(Dictionary *Dict, T FindInData, T Substitute,
- const uint8_t *Data, size_t Size) {
- if (FindInData == Substitute) return;
- if (sizeof(T) == 4) {
- uint16_t HigherBytes = Substitute >> sizeof(T) * 4;
- if (HigherBytes == 0 || HigherBytes == 0xffff)
- TORCToDict(Dict, static_cast<uint16_t>(FindInData),
- static_cast<uint16_t>(Substitute), Data, Size);
- }
- const size_t DataSize = sizeof(T);
- const uint8_t *End = Data + Size;
- int Attempts = 3;
- for (int DoSwap = 0; DoSwap <= 1; DoSwap++) {
- for (const uint8_t *Cur = Data; Cur < End && Attempts--; Cur++) {
- Cur = (uint8_t *)memmem(Cur, End - Cur, &FindInData, DataSize);
- if (!Cur)
- break;
- size_t Pos = Cur - Data;
- Word W(reinterpret_cast<uint8_t *>(&Substitute), sizeof(Substitute));
- DictionaryEntry DE(W, Pos);
- // TODO: evict all entries from Dic if it's full.
- Dict->push_back(DE);
- // Printf("Dict[%zd] TORC%zd %llx => %llx pos %zd\n", Dict->size(),
- // sizeof(T),
- // (uint64_t)FindInData, (uint64_t)Substitute, Pos);
- }
- FindInData = Bswap(FindInData);
- Substitute = Bswap(Substitute);
- }
-}
-
} // namespace fuzzer
extern "C" {
Modified: llvm/trunk/lib/Fuzzer/FuzzerTracePC.h
URL: http://llvm.org/viewvc/llvm-project/llvm/trunk/lib/Fuzzer/FuzzerTracePC.h?rev=285049&r1=285048&r2=285049&view=diff
==============================================================================
--- llvm/trunk/lib/Fuzzer/FuzzerTracePC.h (original)
+++ llvm/trunk/lib/Fuzzer/FuzzerTracePC.h Mon Oct 24 21:04:43 2016
@@ -27,13 +27,18 @@ namespace fuzzer {
template<class T, size_t kSizeT>
struct TableOfRecentCompares {
static const size_t kSize = kSizeT;
+ struct Pair {
+ T A, B;
+ };
void Insert(size_t Idx, T Arg1, T Arg2) {
Idx = Idx % kSize;
- Table[Idx][0] = Arg1;
- Table[Idx][1] = Arg2;
+ Table[Idx].A = Arg1;
+ Table[Idx].B = Arg2;
}
- void Clear() { memset(Table, 0, sizeof(Table)); }
- T Table[kSize][2];
+
+ Pair Get(size_t I) { return Table[I % kSize]; }
+
+ Pair Table[kSize];
};
class TracePC {
@@ -67,11 +72,6 @@ class TracePC {
memset(Counters, 0, sizeof(Counters));
}
- void ResetTORC() {
- TORC4.Clear();
- TORC8.Clear();
- }
-
void UpdateFeatureSet(size_t CurrentElementIdx, size_t CurrentElementSize);
void PrintFeatureSet();
@@ -88,7 +88,9 @@ class TracePC {
bool UsingTracePcGuard() const {return NumModules; }
- void ProcessTORC(Dictionary *Dict, const uint8_t *Data, size_t Size);
+ static const size_t kTORCSize = 1 << 5;
+ TableOfRecentCompares<uint32_t, kTORCSize> TORC4;
+ TableOfRecentCompares<uint64_t, kTORCSize> TORC8;
private:
bool UseCounters = false;
@@ -113,9 +115,6 @@ private:
static const size_t kNumCounters = 1 << 14;
alignas(8) uint8_t Counters[kNumCounters];
- static const size_t kTORCSize = 1 << 12;
- TableOfRecentCompares<uint32_t, kTORCSize> TORC4;
- TableOfRecentCompares<uint64_t, kTORCSize> TORC8;
void TORCInsert(size_t Idx, uint8_t Arg1, uint8_t Arg2) {
// Do nothing, too small to be interesting.
}
@@ -129,13 +128,6 @@ private:
TORC8.Insert(Idx, Arg1, Arg2);
}
- template <class T>
- void TORCToDict(const TableOfRecentCompares<T, kTORCSize> &TORC,
- Dictionary *Dict, const uint8_t *Data, size_t Size);
- template <class T>
- void TORCToDict(Dictionary *Dict, T FindInData, T Substitute,
- const uint8_t *Data, size_t Size);
-
static const size_t kNumPCs = 1 << 24;
uintptr_t PCs[kNumPCs];
Modified: llvm/trunk/lib/Fuzzer/test/SimpleCmpTest.cpp
URL: http://llvm.org/viewvc/llvm-project/llvm/trunk/lib/Fuzzer/test/SimpleCmpTest.cpp?rev=285049&r1=285048&r2=285049&view=diff
==============================================================================
--- llvm/trunk/lib/Fuzzer/test/SimpleCmpTest.cpp (original)
+++ llvm/trunk/lib/Fuzzer/test/SimpleCmpTest.cpp Mon Oct 24 21:04:43 2016
@@ -25,7 +25,8 @@ extern "C" int LLVMFuzzerTestOneInput(co
z < -10000 &&
z >= -10005 &&
z != -10003 &&
- a == 4242) {
+ a == 4242 &&
+ true) {
fprintf(stderr, "BINGO; Found the target: size %zd (%zd, %zd, %d, %d), exiting.\n",
Size, x, y, z, a);
exit(1);
Modified: llvm/trunk/lib/Fuzzer/test/SwapCmpTest.cpp
URL: http://llvm.org/viewvc/llvm-project/llvm/trunk/lib/Fuzzer/test/SwapCmpTest.cpp?rev=285049&r1=285048&r2=285049&view=diff
==============================================================================
--- llvm/trunk/lib/Fuzzer/test/SwapCmpTest.cpp (original)
+++ llvm/trunk/lib/Fuzzer/test/SwapCmpTest.cpp Mon Oct 24 21:04:43 2016
@@ -21,8 +21,9 @@ extern "C" int LLVMFuzzerTestOneInput(co
z = __builtin_bswap16(z);
if (x == 0x46555A5A5A5A5546ULL &&
+ z == 0x4F4B &&
y == 0x66757A7A &&
- z == 0x4F4B
+ true
) {
if (Data[Size - 3] == 'z') {
fprintf(stderr, "BINGO; Found the target\n");
Added: llvm/trunk/lib/Fuzzer/test/swap-cmp.test
URL: http://llvm.org/viewvc/llvm-project/llvm/trunk/lib/Fuzzer/test/swap-cmp.test?rev=285049&view=auto
==============================================================================
--- llvm/trunk/lib/Fuzzer/test/swap-cmp.test (added)
+++ llvm/trunk/lib/Fuzzer/test/swap-cmp.test Mon Oct 24 21:04:43 2016
@@ -0,0 +1,2 @@
+CHECK: BINGO
+RUN: not LLVMFuzzer-SwapCmpTest -seed=1 -use_cmp=1 -runs=10000000 2>&1 | FileCheck %s
More information about the llvm-commits
mailing list