<div dir="ltr"><br><div class="gmail_extra"><br><div class="gmail_quote">On Fri, Jan 15, 2016 at 7:53 PM, Kostya Serebryany via llvm-commits <span dir="ltr"><<a href="mailto:llvm-commits@lists.llvm.org" target="_blank">llvm-commits@lists.llvm.org</a>></span> wrote:<br><blockquote class="gmail_quote" style="margin:0 0 0 .8ex;border-left:1px #ccc solid;padding-left:1ex">Author: kcc<br>
Date: Fri Jan 15 21:53:32 2016<br>
New Revision: 257985<br>
<br>
URL: <a href="http://llvm.org/viewvc/llvm-project?rev=257985&view=rev" rel="noreferrer" target="_blank">http://llvm.org/viewvc/llvm-project?rev=257985&view=rev</a><br>
Log:<br>
[libFuzzer] replace vector with a simpler data structure in the Dictionaries to avoid memory allocations on hot path<br>
<br>
Modified:<br>
llvm/trunk/lib/Fuzzer/FuzzerDriver.cpp<br>
llvm/trunk/lib/Fuzzer/FuzzerInternal.h<br>
llvm/trunk/lib/Fuzzer/FuzzerMutate.cpp<br>
llvm/trunk/lib/Fuzzer/FuzzerTraceState.cpp<br>
llvm/trunk/lib/Fuzzer/FuzzerUtil.cpp<br>
llvm/trunk/lib/Fuzzer/test/FuzzerUnittest.cpp<br>
<br>
Modified: llvm/trunk/lib/Fuzzer/FuzzerDriver.cpp<br>
URL: <a href="http://llvm.org/viewvc/llvm-project/llvm/trunk/lib/Fuzzer/FuzzerDriver.cpp?rev=257985&r1=257984&r2=257985&view=diff" rel="noreferrer" target="_blank">http://llvm.org/viewvc/llvm-project/llvm/trunk/lib/Fuzzer/FuzzerDriver.cpp?rev=257985&r1=257984&r2=257985&view=diff</a><br>
==============================================================================<br>
--- llvm/trunk/lib/Fuzzer/FuzzerDriver.cpp (original)<br>
+++ llvm/trunk/lib/Fuzzer/FuzzerDriver.cpp Fri Jan 15 21:53:32 2016<br>
@@ -301,7 +301,8 @@ int FuzzerDriver(const std::vector<std::<br>
Fuzzer F(USF, Options);<br>
<br>
for (auto &U: Dictionary)<br>
- USF.GetMD().AddWordToManualDictionary(U);<br>
+ if (U.size() <= Word::GetMaxSize())<br>
+ USF.GetMD().AddWordToManualDictionary(Word(U.data(), U.size()));<br>
<br>
// Timer<br>
if (Flags.timeout > 0)<br>
<br>
Modified: llvm/trunk/lib/Fuzzer/FuzzerInternal.h<br>
URL: <a href="http://llvm.org/viewvc/llvm-project/llvm/trunk/lib/Fuzzer/FuzzerInternal.h?rev=257985&r1=257984&r2=257985&view=diff" rel="noreferrer" target="_blank">http://llvm.org/viewvc/llvm-project/llvm/trunk/lib/Fuzzer/FuzzerInternal.h?rev=257985&r1=257984&r2=257985&view=diff</a><br>
==============================================================================<br>
--- llvm/trunk/lib/Fuzzer/FuzzerInternal.h (original)<br>
+++ llvm/trunk/lib/Fuzzer/FuzzerInternal.h Fri Jan 15 21:53:32 2016<br>
@@ -18,6 +18,7 @@<br>
#include <cstddef><br>
#include <cstdlib><br>
#include <string><br>
+#include <string.h><br>
#include <vector><br>
#include <unordered_set><br>
<br>
@@ -27,6 +28,40 @@ namespace fuzzer {<br>
using namespace std::chrono;<br>
typedef std::vector<uint8_t> Unit;<br>
<br>
+// A simple POD sized array of bytes.<br>
+template<size_t kMaxSize><br>
+class FixedWord {<br>
+ public:<br>
+<br>
+ FixedWord() : Size(0) {}<br>
+ FixedWord(const uint8_t *B, uint8_t S) { Set(B, S); }<br>
+<br>
+ void Set(const uint8_t *B, uint8_t S) {<br>
+ assert(S <= kMaxSize);<br>
+ memcpy(Data, B, S);<br>
+ Size = S;<br>
+ }<br>
+<br>
+ bool operator == (const FixedWord<kMaxSize> &w) const {<br>
+ return Size == w.Size && 0 == memcmp(Data, w.Data, Size);<br>
+ }<br>
+<br>
+ bool operator < (const FixedWord<kMaxSize> &w) const {<br>
+ if (Size != w.Size) return Size < w.Size;<br>
+ return memcmp(Data, w.Data, Size) < 0;<br>
+ }<br>
+<br>
+ static size_t GetMaxSize() { return kMaxSize; }<br>
+ const uint8_t *data() const { return Data; }<br>
+ uint8_t size() const { return Size; }<br>
+<br>
+ private:<br>
+ uint8_t Size;<br>
+ uint8_t Data[kMaxSize];<br>
+};<br>
+<br>
+typedef FixedWord<27> Word; // 28 bytes.<br>
+<br>
std::string FileToString(const std::string &Path);<br>
Unit FileToVector(const std::string &Path);<br>
void ReadDirToVectorOfUnits(const char *Path, std::vector<Unit> *V,<br>
@@ -43,6 +78,7 @@ void PrintHexArray(const uint8_t *Data,<br>
const char *PrintAfter = "");<br>
void PrintASCII(const uint8_t *Data, size_t Size, const char *PrintAfter = "");<br>
void PrintASCII(const Unit &U, const char *PrintAfter = "");<br>
+void PrintASCII(const Word &W, const char *PrintAfter = "");<br>
std::string Hash(const Unit &U);<br>
void SetTimer(int Seconds);<br>
std::string Base64(const Unit &U);<br>
@@ -118,9 +154,9 @@ class MutationDispatcher {<br>
size_t CrossOver(const uint8_t *Data1, size_t Size1, const uint8_t *Data2,<br>
size_t Size2, uint8_t *Out, size_t MaxOutSize);<br>
<br>
- void AddWordToManualDictionary(const Unit &Word);<br>
+ void AddWordToManualDictionary(const Word &W);<br>
<br>
- void AddWordToAutoDictionary(const Unit &Word, size_t PositionHint);<br>
+ void AddWordToAutoDictionary(const Word &W, size_t PositionHint);<br>
void ClearAutoDictionary();<br>
void PrintRecommendedDictionary();<br>
<br>
<br>
Modified: llvm/trunk/lib/Fuzzer/FuzzerMutate.cpp<br>
URL: <a href="http://llvm.org/viewvc/llvm-project/llvm/trunk/lib/Fuzzer/FuzzerMutate.cpp?rev=257985&r1=257984&r2=257985&view=diff" rel="noreferrer" target="_blank">http://llvm.org/viewvc/llvm-project/llvm/trunk/lib/Fuzzer/FuzzerMutate.cpp?rev=257985&r1=257984&r2=257985&view=diff</a><br>
==============================================================================<br>
--- llvm/trunk/lib/Fuzzer/FuzzerMutate.cpp (original)<br>
+++ llvm/trunk/lib/Fuzzer/FuzzerMutate.cpp Fri Jan 15 21:53:32 2016<br>
@@ -23,16 +23,15 @@ struct Mutator {<br>
};<br>
<br>
struct DictionaryEntry {<br>
- Unit Word;<br>
+ Word W;<br>
size_t PositionHint;<br>
};<br>
<br>
struct Dictionary : public std::vector<DictionaryEntry>{<br>
- bool ContainsWord(const Unit &W) const {<br>
+ bool ContainsWord(const Word &W) const {<br>
return end() !=<br>
- std::find_if(begin(), end(), [&](const DictionaryEntry &DE) {<br>
- return DE.Word == W;<br>
- });<br>
+ std::find_if(begin(), end(),<br>
+ [&](const DictionaryEntry &DE) { return DE.W == W; });<br></blockquote><div><br></div><div>This ^ looks like std::any_of, perhaps? (maybe even llvm::any_of, for the ultimate convenience)</div><div> </div><blockquote class="gmail_quote" style="margin:0 0 0 .8ex;border-left:1px #ccc solid;padding-left:1ex">
}<br>
};<br>
<br>
@@ -161,20 +160,20 @@ size_t MutationDispatcher::Impl::AddWord<br>
size_t MaxSize) {<br>
if (D.empty()) return 0;<br>
const DictionaryEntry &DE = D[Rand(D.size())];<br>
- const Unit &Word = DE.Word;<br>
+ const Word &W = DE.W;<br>
size_t PositionHint = DE.PositionHint;<br>
bool UsePositionHint = PositionHint != std::numeric_limits<size_t>::max() &&<br>
- PositionHint + Word.size() < Size && Rand.RandBool();<br>
- if (Rand.RandBool()) { // Insert Word.<br>
- if (Size + Word.size() > MaxSize) return 0;<br>
+ PositionHint + W.size() < Size && Rand.RandBool();<br>
+ if (Rand.RandBool()) { // Insert W.<br>
+ if (Size + W.size() > MaxSize) return 0;<br>
size_t Idx = UsePositionHint ? PositionHint : Rand(Size + 1);<br>
- memmove(Data + Idx + Word.size(), Data + Idx, Size - Idx);<br>
- memcpy(Data + Idx, Word.data(), Word.size());<br>
- Size += Word.size();<br>
- } else { // Overwrite some bytes with Word.<br>
- if (Word.size() > Size) return 0;<br>
- size_t Idx = UsePositionHint ? PositionHint : Rand(Size - Word.size());<br>
- memcpy(Data + Idx, Word.data(), Word.size());<br>
+ memmove(Data + Idx + W.size(), Data + Idx, Size - Idx);<br>
+ memcpy(Data + Idx, W.data(), W.size());<br>
+ Size += W.size();<br>
+ } else { // Overwrite some bytes with W.<br>
+ if (W.size() > Size) return 0;<br>
+ size_t Idx = UsePositionHint ? PositionHint : Rand(Size - W.size());<br>
+ memcpy(Data + Idx, W.data(), W.size());<br>
}<br>
CurrentDictionaryEntrySequence.push_back(DE);<br>
return Size;<br>
@@ -238,16 +237,16 @@ void MutationDispatcher::StartMutationSe<br>
void MutationDispatcher::RecordSuccessfulMutationSequence() {<br>
for (auto &DE : MDImpl->CurrentDictionaryEntrySequence)<br>
// Linear search is fine here as this happens seldom.<br>
- if (!MDImpl->PersistentAutoDictionary.ContainsWord(DE.Word))<br>
+ if (!MDImpl->PersistentAutoDictionary.ContainsWord(DE.W))<br>
MDImpl->PersistentAutoDictionary.push_back(<br>
- {DE.Word, std::numeric_limits<size_t>::max()});<br>
+ {DE.W, std::numeric_limits<size_t>::max()});<br>
}<br>
<br>
void MutationDispatcher::PrintRecommendedDictionary() {<br>
- std::vector<Unit> V;<br>
+ std::vector<Word> V;<br>
for (auto &DE : MDImpl->PersistentAutoDictionary)<br>
- if (!MDImpl->ManualDictionary.ContainsWord(DE.Word))<br>
- V.push_back(DE.Word);<br>
+ if (!MDImpl->ManualDictionary.ContainsWord(DE.W))<br>
+ V.push_back(DE.W);<br>
if (V.empty()) return;<br>
Printf("###### Recommended dictionary. ######\n");<br>
for (auto &U: V) {<br>
@@ -265,7 +264,7 @@ void MutationDispatcher::PrintMutationSe<br>
Printf(" DE: ");<br>
for (auto &DE : MDImpl->CurrentDictionaryEntrySequence) {<br>
Printf("\"");<br>
- PrintASCII(DE.Word, "\"-");<br>
+ PrintASCII(DE.W, "\"-");<br>
}<br>
}<br>
}<br>
@@ -299,16 +298,16 @@ void MutationDispatcher::SetCorpus(const<br>
MDImpl->SetCorpus(Corpus);<br>
}<br>
<br>
-void MutationDispatcher::AddWordToManualDictionary(const Unit &Word) {<br>
+void MutationDispatcher::AddWordToManualDictionary(const Word &W) {<br>
MDImpl->ManualDictionary.push_back(<br>
- {Word, std::numeric_limits<size_t>::max()});<br>
+ {W, std::numeric_limits<size_t>::max()});<br>
}<br>
<br>
-void MutationDispatcher::AddWordToAutoDictionary(const Unit &Word,<br>
+void MutationDispatcher::AddWordToAutoDictionary(const Word &W,<br>
size_t PositionHint) {<br>
static const size_t kMaxAutoDictSize = 1 << 14;<br>
if (MDImpl->TempAutoDictionary.size() >= kMaxAutoDictSize) return;<br>
- MDImpl->TempAutoDictionary.push_back({Word, PositionHint});<br>
+ MDImpl->TempAutoDictionary.push_back({W, PositionHint});<br>
}<br>
<br>
void MutationDispatcher::ClearAutoDictionary() {<br>
<br>
Modified: llvm/trunk/lib/Fuzzer/FuzzerTraceState.cpp<br>
URL: <a href="http://llvm.org/viewvc/llvm-project/llvm/trunk/lib/Fuzzer/FuzzerTraceState.cpp?rev=257985&r1=257984&r2=257985&view=diff" rel="noreferrer" target="_blank">http://llvm.org/viewvc/llvm-project/llvm/trunk/lib/Fuzzer/FuzzerTraceState.cpp?rev=257985&r1=257984&r2=257985&view=diff</a><br>
==============================================================================<br>
--- llvm/trunk/lib/Fuzzer/FuzzerTraceState.cpp (original)<br>
+++ llvm/trunk/lib/Fuzzer/FuzzerTraceState.cpp Fri Jan 15 21:53:32 2016<br>
@@ -164,14 +164,10 @@ struct LabelRange {<br>
<br>
// For now, very simple: put Size bytes of Data at position Pos.<br>
struct TraceBasedMutation {<br>
- static const size_t kMaxSize = 28;<br>
- uint32_t Pos : 24;<br>
- uint32_t Size : 8;<br>
- uint8_t Data[kMaxSize];<br>
+ uint32_t Pos;<br>
+ Word W;<br>
};<br>
<br>
-const size_t TraceBasedMutation::kMaxSize;<br>
-<br>
// Declared as static globals for faster checks inside the hooks.<br>
static bool RecordingTraces = false;<br>
static bool RecordingMemcmp = false;<br>
@@ -223,12 +219,11 @@ class TraceState {<br>
RecordingMemcmp = false;<br>
for (size_t i = 0; i < NumMutations; i++) {<br>
auto &M = Mutations[i];<br>
- Unit U(M.Data, M.Data + M.Size);<br>
if (Options.Verbosity >= 2) {<br>
- AutoDictUnitCounts[U]++;<br>
+ AutoDictUnitCounts[M.W]++;<br>
AutoDictAdds++;<br>
if ((AutoDictAdds & (AutoDictAdds - 1)) == 0) {<br>
- typedef std::pair<size_t, Unit> CU;<br>
+ typedef std::pair<size_t, Word> CU;<br>
std::vector<CU> CountedUnits;<br>
for (auto &I : AutoDictUnitCounts)<br>
CountedUnits.push_back(std::make_pair(I.second, I.first));<br>
@@ -242,17 +237,15 @@ class TraceState {<br>
}<br>
}<br>
}<br>
- USF.GetMD().AddWordToAutoDictionary(U, M.Pos);<br>
+ USF.GetMD().AddWordToAutoDictionary(M.W, M.Pos);<br>
}<br>
}<br>
<br>
void AddMutation(uint32_t Pos, uint32_t Size, const uint8_t *Data) {<br>
if (NumMutations >= kMaxMutations) return;<br>
- assert(Size <= TraceBasedMutation::kMaxSize);<br>
auto &M = Mutations[NumMutations++];<br>
M.Pos = Pos;<br>
- M.Size = Size;<br>
- memcpy(M.Data, Data, Size);<br>
+ M.W.Set(Data, Size);<br>
}<br>
<br>
void AddMutation(uint32_t Pos, uint32_t Size, uint64_t Data) {<br>
@@ -274,7 +267,7 @@ class TraceState {<br>
const Fuzzer::FuzzingOptions &Options;<br>
uint8_t **CurrentUnitData;<br>
size_t *CurrentUnitSize;<br>
- std::map<Unit, size_t> AutoDictUnitCounts;<br>
+ std::map<Word, size_t> AutoDictUnitCounts;<br>
size_t AutoDictAdds = 0;<br>
static thread_local bool IsMyThread;<br>
};<br>
@@ -423,7 +416,7 @@ void TraceState::TraceCmpCallback(uintpt<br>
void TraceState::TraceMemcmpCallback(size_t CmpSize, const uint8_t *Data1,<br>
const uint8_t *Data2) {<br>
if (!RecordingMemcmp || !IsMyThread) return;<br>
- CmpSize = std::min(CmpSize, TraceBasedMutation::kMaxSize);<br>
+ CmpSize = std::min(CmpSize, Word::GetMaxSize());<br>
int Added2 = TryToAddDesiredData(Data1, Data2, CmpSize);<br>
int Added1 = TryToAddDesiredData(Data2, Data1, CmpSize);<br>
if ((Added1 || Added2) && Options.Verbosity >= 3) {<br>
<br>
Modified: llvm/trunk/lib/Fuzzer/FuzzerUtil.cpp<br>
URL: <a href="http://llvm.org/viewvc/llvm-project/llvm/trunk/lib/Fuzzer/FuzzerUtil.cpp?rev=257985&r1=257984&r2=257985&view=diff" rel="noreferrer" target="_blank">http://llvm.org/viewvc/llvm-project/llvm/trunk/lib/Fuzzer/FuzzerUtil.cpp?rev=257985&r1=257984&r2=257985&view=diff</a><br>
==============================================================================<br>
--- llvm/trunk/lib/Fuzzer/FuzzerUtil.cpp (original)<br>
+++ llvm/trunk/lib/Fuzzer/FuzzerUtil.cpp Fri Jan 15 21:53:32 2016<br>
@@ -49,10 +49,12 @@ void PrintASCII(const uint8_t *Data, siz<br>
Printf("%s", PrintAfter);<br>
}<br>
<br>
+void PrintASCII(const Word &W, const char *PrintAfter) {<br>
+ PrintASCII(W.data(), W.size(), PrintAfter);<br>
+}<br>
+<br>
void PrintASCII(const Unit &U, const char *PrintAfter) {<br>
- for (auto X : U)<br>
- PrintASCIIByte(X);<br>
- Printf("%s", PrintAfter);<br>
+ PrintASCII(U.data(), U.size(), PrintAfter);<br>
}<br>
<br>
std::string Hash(const Unit &U) {<br>
<br>
Modified: llvm/trunk/lib/Fuzzer/test/FuzzerUnittest.cpp<br>
URL: <a href="http://llvm.org/viewvc/llvm-project/llvm/trunk/lib/Fuzzer/test/FuzzerUnittest.cpp?rev=257985&r1=257984&r2=257985&view=diff" rel="noreferrer" target="_blank">http://llvm.org/viewvc/llvm-project/llvm/trunk/lib/Fuzzer/test/FuzzerUnittest.cpp?rev=257985&r1=257984&r2=257985&view=diff</a><br>
==============================================================================<br>
--- llvm/trunk/lib/Fuzzer/test/FuzzerUnittest.cpp (original)<br>
+++ llvm/trunk/lib/Fuzzer/test/FuzzerUnittest.cpp Fri Jan 15 21:53:32 2016<br>
@@ -247,8 +247,8 @@ void TestAddWordFromDictionary(Mutator M<br>
MutationDispatcher MD(Rand);<br>
uint8_t Word1[4] = {0xAA, 0xBB, 0xCC, 0xDD};<br>
uint8_t Word2[3] = {0xFF, 0xEE, 0xEF};<br>
- MD.AddWordToManualDictionary(Unit(Word1, Word1 + sizeof(Word1)));<br>
- MD.AddWordToManualDictionary(Unit(Word2, Word2 + sizeof(Word2)));<br>
+ MD.AddWordToManualDictionary(Word(Word1, sizeof(Word1)));<br>
+ MD.AddWordToManualDictionary(Word(Word2, sizeof(Word2)));<br>
int FoundMask = 0;<br>
uint8_t CH0[7] = {0x00, 0x11, 0x22, 0xAA, 0xBB, 0xCC, 0xDD};<br>
uint8_t CH1[7] = {0x00, 0x11, 0xAA, 0xBB, 0xCC, 0xDD, 0x22};<br>
@@ -285,16 +285,16 @@ TEST(FuzzerMutate, AddWordFromDictionary<br>
void TestAddWordFromDictionaryWithHint(Mutator M, int NumIter) {<br>
FuzzerRandomLibc Rand(0);<br>
MutationDispatcher MD(Rand);<br>
- uint8_t Word[] = {0xAA, 0xBB, 0xCC, 0xDD, 0xFF, 0xEE, 0xEF};<br>
+ uint8_t W[] = {0xAA, 0xBB, 0xCC, 0xDD, 0xFF, 0xEE, 0xEF};<br>
size_t PosHint = 7777;<br>
- MD.AddWordToAutoDictionary(Unit(Word, Word + sizeof(Word)), PosHint);<br>
+ MD.AddWordToAutoDictionary(Word(W, sizeof(W)), PosHint);<br>
int FoundMask = 0;<br>
for (int i = 0; i < NumIter; i++) {<br>
uint8_t T[10000];<br>
memset(T, 0, sizeof(T));<br>
size_t NewSize = (MD.*M)(T, 9000, 10000);<br>
- if (NewSize >= PosHint + sizeof(Word) &&<br>
- !memcmp(Word, T + PosHint, sizeof(Word)))<br>
+ if (NewSize >= PosHint + sizeof(W) &&<br>
+ !memcmp(W, T + PosHint, sizeof(W)))<br>
FoundMask = 1;<br>
}<br>
EXPECT_EQ(FoundMask, 1);<br>
<br>
<br>
_______________________________________________<br>
llvm-commits mailing list<br>
<a href="mailto:llvm-commits@lists.llvm.org">llvm-commits@lists.llvm.org</a><br>
<a href="http://lists.llvm.org/cgi-bin/mailman/listinfo/llvm-commits" rel="noreferrer" target="_blank">http://lists.llvm.org/cgi-bin/mailman/listinfo/llvm-commits</a><br>
</blockquote></div><br></div></div>