<div dir="ltr">Thanks!</div><div class="gmail_extra"><br><div class="gmail_quote">On Fri, Oct 23, 2015 at 2:10 PM, Kostya Serebryany <span dir="ltr"><<a href="mailto:kcc@google.com" target="_blank">kcc@google.com</a>></span> wrote:<br><blockquote class="gmail_quote" style="margin:0 0 0 .8ex;border-left:1px #ccc solid;padding-left:1ex"><div dir="ltr"><br><div class="gmail_extra"><br><div class="gmail_quote"><span class="">On Fri, Oct 23, 2015 at 1:59 PM, Sean Silva <span dir="ltr"><<a href="mailto:chisophugis@gmail.com" target="_blank">chisophugis@gmail.com</a>></span> wrote:<br><blockquote class="gmail_quote" style="margin:0px 0px 0px 0.8ex;border-left-width:1px;border-left-color:rgb(204,204,204);border-left-style:solid;padding-left:1ex"><div dir="ltr">Sorry I haven't been keeping up to date. The replacement for this feature is to use a user-supplied mutator? <a href="http://llvm.org/docs/LibFuzzer.html#user-supplied-mutators" target="_blank">http://llvm.org/docs/LibFuzzer.html#user-supplied-mutators</a></div></blockquote><div><br></div></span><div>No, AFL-style dictionaries: </div><div><a href="http://llvm.org/docs/LibFuzzer.html#dictionaries" target="_blank">http://llvm.org/docs/LibFuzzer.html#dictionaries</a></div><div><div class="h5"><div><br></div><div><br></div><div> </div><blockquote class="gmail_quote" style="margin:0px 0px 0px 0.8ex;border-left-width:1px;border-left-color:rgb(204,204,204);border-left-style:solid;padding-left:1ex"><div><div><div class="gmail_extra"><br><div class="gmail_quote">On Thu, Oct 22, 2015 at 2:48 PM, Kostya Serebryany via llvm-commits <span dir="ltr"><<a href="mailto:llvm-commits@lists.llvm.org" target="_blank">llvm-commits@lists.llvm.org</a>></span> wrote:<br><blockquote class="gmail_quote" style="margin:0px 0px 0px 0.8ex;border-left-width:1px;border-left-color:rgb(204,204,204);border-left-style:solid;padding-left:1ex">Author: kcc<br>
Date: Thu Oct 22 16:48:09 2015<br>
New Revision: 251069<br>
<br>
URL: <a href="http://llvm.org/viewvc/llvm-project?rev=251069&view=rev" rel="noreferrer" target="_blank">http://llvm.org/viewvc/llvm-project?rev=251069&view=rev</a><br>
Log:<br>
[libFuzzer] remove the deprecated 'tokens' feature<br>
<br>
Removed:<br>
llvm/trunk/lib/Fuzzer/test/CxxTokensTest.cpp<br>
Modified:<br>
llvm/trunk/lib/Fuzzer/FuzzerDriver.cpp<br>
llvm/trunk/lib/Fuzzer/FuzzerFlags.def<br>
llvm/trunk/lib/Fuzzer/FuzzerInternal.h<br>
llvm/trunk/lib/Fuzzer/FuzzerLoop.cpp<br>
llvm/trunk/lib/Fuzzer/test/CMakeLists.txt<br>
llvm/trunk/lib/Fuzzer/test/fuzzer.test<br>
<br>
Modified: llvm/trunk/lib/Fuzzer/FuzzerDriver.cpp<br>
URL: <a href="http://llvm.org/viewvc/llvm-project/llvm/trunk/lib/Fuzzer/FuzzerDriver.cpp?rev=251069&r1=251068&r2=251069&view=diff" rel="noreferrer" target="_blank">http://llvm.org/viewvc/llvm-project/llvm/trunk/lib/Fuzzer/FuzzerDriver.cpp?rev=251069&r1=251068&r2=251069&view=diff</a><br>
==============================================================================<br>
--- llvm/trunk/lib/Fuzzer/FuzzerDriver.cpp (original)<br>
+++ llvm/trunk/lib/Fuzzer/FuzzerDriver.cpp Thu Oct 22 16:48:09 2015<br>
@@ -182,26 +182,6 @@ static int RunInMultipleProcesses(const<br>
return HasErrors ? 1 : 0;<br>
}<br>
<br>
-std::vector<std::string> ReadTokensFile(const char *TokensFilePath) {<br>
- if (!TokensFilePath) return {};<br>
- std::string TokensFileContents = FileToString(TokensFilePath);<br>
- std::istringstream ISS(TokensFileContents);<br>
- std::vector<std::string> Res = {std::istream_iterator<std::string>{ISS},<br>
- std::istream_iterator<std::string>{}};<br>
- Res.push_back(" ");<br>
- Res.push_back("\t");<br>
- Res.push_back("\n");<br>
- return Res;<br>
-}<br>
-<br>
-int ApplyTokens(const Fuzzer &F, const char *InputFilePath) {<br>
- Unit U = FileToVector(InputFilePath);<br>
- auto T = F.SubstituteTokens(U);<br>
- T.push_back(0);<br>
- Printf("%s", T.data());<br>
- return 0;<br>
-}<br>
-<br>
int RunOneTest(Fuzzer *F, const char *InputFilePath) {<br>
Unit U = FileToVector(InputFilePath);<br>
F->ExecuteCallback(U);<br>
@@ -258,7 +238,6 @@ int FuzzerDriver(const std::vector<std::<br>
Options.ShuffleAtStartUp = Flags.shuffle;<br>
Options.PreferSmallDuringInitialShuffle =<br>
Flags.prefer_small_during_initial_shuffle;<br>
- Options.Tokens = ReadTokensFile(Flags.deprecated_tokens);<br>
Options.Reload = Flags.reload;<br>
Options.OnlyASCII = Flags.only_ascii;<br>
Options.TBMDepth = Flags.tbm_depth;<br>
@@ -282,9 +261,6 @@ int FuzzerDriver(const std::vector<std::<br>
<br>
Fuzzer F(USF, Options);<br>
<br>
- if (Flags.apply_tokens)<br>
- return ApplyTokens(F, Flags.apply_tokens);<br>
-<br>
// Timer<br>
if (Flags.timeout > 0)<br>
SetTimer(Flags.timeout / 2 + 1);<br>
@@ -300,13 +276,6 @@ int FuzzerDriver(const std::vector<std::<br>
Printf("Seed: %u\n", Seed);<br>
USF.GetRand().ResetSeed(Seed);<br>
<br>
- if (Flags.verbosity >= 2) {<br>
- Printf("Tokens: {");<br>
- for (auto &T : Options.Tokens)<br>
- Printf("%s,", T.c_str());<br>
- Printf("}\n");<br>
- }<br>
-<br>
F.RereadOutputCorpus();<br>
for (auto &inp : *Inputs)<br>
if (inp != Options.OutputCorpus)<br>
<br>
Modified: llvm/trunk/lib/Fuzzer/FuzzerFlags.def<br>
URL: <a href="http://llvm.org/viewvc/llvm-project/llvm/trunk/lib/Fuzzer/FuzzerFlags.def?rev=251069&r1=251068&r2=251069&view=diff" rel="noreferrer" target="_blank">http://llvm.org/viewvc/llvm-project/llvm/trunk/lib/Fuzzer/FuzzerFlags.def?rev=251069&r1=251068&r2=251069&view=diff</a><br>
==============================================================================<br>
--- llvm/trunk/lib/Fuzzer/FuzzerFlags.def (original)<br>
+++ llvm/trunk/lib/Fuzzer/FuzzerFlags.def Thu Oct 22 16:48:09 2015<br>
@@ -47,11 +47,6 @@ FUZZER_FLAG_INT(workers, 0,<br>
FUZZER_FLAG_INT(reload, 1,<br>
"Reload the main corpus periodically to get new units"<br>
" discovered by other processes.")<br>
-FUZZER_FLAG_STRING(deprecated_tokens,<br>
- "Use the file with tokens (one token per line) to"<br>
- " fuzz a token based input language.")<br>
-FUZZER_FLAG_STRING(apply_tokens, "Read the given input file, substitute bytes "<br>
- " with tokens and write the result to stdout.")<br>
FUZZER_FLAG_STRING(sync_command, "Execute an external command "<br>
"\"<sync_command> <test_corpus>\" "<br>
"to synchronize the test corpus.")<br>
<br>
Modified: llvm/trunk/lib/Fuzzer/FuzzerInternal.h<br>
URL: <a href="http://llvm.org/viewvc/llvm-project/llvm/trunk/lib/Fuzzer/FuzzerInternal.h?rev=251069&r1=251068&r2=251069&view=diff" rel="noreferrer" target="_blank">http://llvm.org/viewvc/llvm-project/llvm/trunk/lib/Fuzzer/FuzzerInternal.h?rev=251069&r1=251068&r2=251069&view=diff</a><br>
==============================================================================<br>
--- llvm/trunk/lib/Fuzzer/FuzzerInternal.h (original)<br>
+++ llvm/trunk/lib/Fuzzer/FuzzerInternal.h Thu Oct 22 16:48:09 2015<br>
@@ -93,7 +93,6 @@ class Fuzzer {<br>
std::string OutputCorpus;<br>
std::string SyncCommand;<br>
std::string ArtifactPrefix = "./";<br>
- std::vector<std::string> Tokens;<br>
std::vector<Unit> Dictionary;<br>
bool SaveArtifacts = true;<br>
};<br>
@@ -119,7 +118,6 @@ class Fuzzer {<br>
<br>
static void StaticAlarmCallback();<br>
<br>
- Unit SubstituteTokens(const Unit &U) const;<br>
void ExecuteCallback(const Unit &U);<br>
<br>
private:<br>
@@ -133,7 +131,7 @@ class Fuzzer {<br>
void WriteToOutputCorpus(const Unit &U);<br>
void WriteUnitToFileWithPrefix(const Unit &U, const char *Prefix);<br>
void PrintStats(const char *Where, size_t Cov, const char *End = "\n");<br>
- void PrintUnitInASCIIOrTokens(const Unit &U, const char *PrintAfter = "");<br>
+ void PrintUnitInASCII(const Unit &U, const char *PrintAfter = "");<br>
<br>
void SyncCorpus();<br>
<br>
<br>
Modified: llvm/trunk/lib/Fuzzer/FuzzerLoop.cpp<br>
URL: <a href="http://llvm.org/viewvc/llvm-project/llvm/trunk/lib/Fuzzer/FuzzerLoop.cpp?rev=251069&r1=251068&r2=251069&view=diff" rel="noreferrer" target="_blank">http://llvm.org/viewvc/llvm-project/llvm/trunk/lib/Fuzzer/FuzzerLoop.cpp?rev=251069&r1=251068&r2=251069&view=diff</a><br>
==============================================================================<br>
--- llvm/trunk/lib/Fuzzer/FuzzerLoop.cpp (original)<br>
+++ llvm/trunk/lib/Fuzzer/FuzzerLoop.cpp Thu Oct 22 16:48:09 2015<br>
@@ -35,14 +35,8 @@ void Fuzzer::SetDeathCallback() {<br>
__sanitizer_set_death_callback(StaticDeathCallback);<br>
}<br>
<br>
-void Fuzzer::PrintUnitInASCIIOrTokens(const Unit &U, const char *PrintAfter) {<br>
- if (Options.Tokens.empty()) {<br>
- PrintASCII(U, PrintAfter);<br>
- } else {<br>
- auto T = SubstituteTokens(U);<br>
- T.push_back(0);<br>
- Printf("%s%s", T.data(), PrintAfter);<br>
- }<br>
+void Fuzzer::PrintUnitInASCII(const Unit &U, const char *PrintAfter) {<br>
+ PrintASCII(U, PrintAfter);<br>
}<br>
<br>
void Fuzzer::StaticDeathCallback() {<br>
@@ -54,7 +48,7 @@ void Fuzzer::DeathCallback() {<br>
Printf("DEATH:\n");<br>
if (CurrentUnit.size() <= kMaxUnitSizeToPrint) {<br>
Print(CurrentUnit, "\n");<br>
- PrintUnitInASCIIOrTokens(CurrentUnit, "\n");<br>
+ PrintUnitInASCII(CurrentUnit, "\n");<br>
}<br>
WriteUnitToFileWithPrefix(CurrentUnit, "crash-");<br>
}<br>
@@ -77,7 +71,7 @@ void Fuzzer::AlarmCallback() {<br>
Options.UnitTimeoutSec);<br>
if (CurrentUnit.size() <= kMaxUnitSizeToPrint) {<br>
Print(CurrentUnit, "\n");<br>
- PrintUnitInASCIIOrTokens(CurrentUnit, "\n");<br>
+ PrintUnitInASCII(CurrentUnit, "\n");<br>
}<br>
WriteUnitToFileWithPrefix(CurrentUnit, "timeout-");<br>
Printf("==%d== ERROR: libFuzzer: timeout after %d seconds\n", GetPid(),<br>
@@ -191,28 +185,9 @@ void Fuzzer::RunOneAndUpdateCorpus(Unit<br>
ReportNewCoverage(RunOne(U), U);<br>
}<br>
<br>
-Unit Fuzzer::SubstituteTokens(const Unit &U) const {<br>
- Unit Res;<br>
- for (auto Idx : U) {<br>
- if (Idx < Options.Tokens.size()) {<br>
- std::string Token = Options.Tokens[Idx];<br>
- Res.insert(Res.end(), Token.begin(), Token.end());<br>
- } else {<br>
- Res.push_back(' ');<br>
- }<br>
- }<br>
- // FIXME: Apply DFSan labels.<br>
- return Res;<br>
-}<br>
-<br>
void Fuzzer::ExecuteCallback(const Unit &U) {<br>
- int Res = 0;<br>
- if (Options.Tokens.empty()) {<br>
- Res = USF.TargetFunction(U.data(), U.size());<br>
- } else {<br>
- auto T = SubstituteTokens(U);<br>
- Res = USF.TargetFunction(T.data(), T.size());<br>
- }<br>
+ int Res = USF.TargetFunction(U.data(), U.size());<br>
+ (void)Res;<br>
assert(Res == 0);<br>
}<br>
<br>
@@ -278,7 +253,7 @@ void Fuzzer::ReportNewCoverage(size_t Ne<br>
Printf(" L: %zd", U.size());<br>
if (U.size() < 30) {<br>
Printf(" ");<br>
- PrintUnitInASCIIOrTokens(U, "\t");<br>
+ PrintUnitInASCII(U, "\t");<br>
Print(U);<br>
}<br>
Printf("\n");<br>
<br>
Modified: llvm/trunk/lib/Fuzzer/test/CMakeLists.txt<br>
URL: <a href="http://llvm.org/viewvc/llvm-project/llvm/trunk/lib/Fuzzer/test/CMakeLists.txt?rev=251069&r1=251068&r2=251069&view=diff" rel="noreferrer" target="_blank">http://llvm.org/viewvc/llvm-project/llvm/trunk/lib/Fuzzer/test/CMakeLists.txt?rev=251069&r1=251068&r2=251069&view=diff</a><br>
==============================================================================<br>
--- llvm/trunk/lib/Fuzzer/test/CMakeLists.txt (original)<br>
+++ llvm/trunk/lib/Fuzzer/test/CMakeLists.txt Thu Oct 22 16:48:09 2015<br>
@@ -14,7 +14,6 @@ set(DFSanTests<br>
<br>
set(Tests<br>
CounterTest<br>
- CxxTokensTest<br>
FourIndependentBranchesTest<br>
FullCoverageSetTest<br>
InfiniteTest<br>
<br>
Removed: llvm/trunk/lib/Fuzzer/test/CxxTokensTest.cpp<br>
URL: <a href="http://llvm.org/viewvc/llvm-project/llvm/trunk/lib/Fuzzer/test/CxxTokensTest.cpp?rev=251068&view=auto" rel="noreferrer" target="_blank">http://llvm.org/viewvc/llvm-project/llvm/trunk/lib/Fuzzer/test/CxxTokensTest.cpp?rev=251068&view=auto</a><br>
==============================================================================<br>
--- llvm/trunk/lib/Fuzzer/test/CxxTokensTest.cpp (original)<br>
+++ llvm/trunk/lib/Fuzzer/test/CxxTokensTest.cpp (removed)<br>
@@ -1,25 +0,0 @@<br>
-// Simple test for a fuzzer. The fuzzer must find a sequence of C++ tokens.<br>
-#include <cstdint><br>
-#include <cstdlib><br>
-#include <cstddef><br>
-#include <cstring><br>
-#include <iostream><br>
-<br>
-static void Found() {<br>
- std::cout << "BINGO; Found the target, exiting\n";<br>
- exit(1);<br>
-}<br>
-<br>
-extern "C" int LLVMFuzzerTestOneInput(const uint8_t *Data, size_t Size) {<br>
- // looking for "thread_local unsigned A;"<br>
- if (Size < 24) return 0;<br>
- if (0 == memcmp(&Data[0], "thread_local", 12))<br>
- if (Data[12] == ' ')<br>
- if (0 == memcmp(&Data[13], "unsigned", 8))<br>
- if (Data[21] == ' ')<br>
- if (Data[22] == 'A')<br>
- if (Data[23] == ';')<br>
- Found();<br>
- return 0;<br>
-}<br>
-<br>
<br>
Modified: llvm/trunk/lib/Fuzzer/test/fuzzer.test<br>
URL: <a href="http://llvm.org/viewvc/llvm-project/llvm/trunk/lib/Fuzzer/test/fuzzer.test?rev=251069&r1=251068&r2=251069&view=diff" rel="noreferrer" target="_blank">http://llvm.org/viewvc/llvm-project/llvm/trunk/lib/Fuzzer/test/fuzzer.test?rev=251069&r1=251068&r2=251069&view=diff</a><br>
==============================================================================<br>
--- llvm/trunk/lib/Fuzzer/test/fuzzer.test (original)<br>
+++ llvm/trunk/lib/Fuzzer/test/fuzzer.test Thu Oct 22 16:48:09 2015<br>
@@ -37,8 +37,6 @@ RUN: not LLVMFuzzer-CounterTest -use_cou<br>
<br>
RUN: not LLVMFuzzer-SimpleCmpTest -use_traces=1 -seed=1 -runs=1000000 -timeout=5 2>&1 | FileCheck %s<br>
<br>
-RUN: not LLVMFuzzer-CxxTokensTest -seed=1 -timeout=15 -deprecated_tokens=%S/../cxx_fuzzer_tokens.txt 2>&1 | FileCheck %s<br>
-<br>
RUN: not LLVMFuzzer-UserSuppliedFuzzerTest -seed=1 -timeout=15 2>&1 | FileCheck %s<br>
<br>
RUN: not LLVMFuzzer-MemcmpTest -use_traces=1 -seed=1 -runs=100000 2>&1 | FileCheck %s<br>
<br>
<br>
_______________________________________________<br>
llvm-commits mailing list<br>
<a href="mailto:llvm-commits@lists.llvm.org" target="_blank">llvm-commits@lists.llvm.org</a><br>
<a href="http://lists.llvm.org/cgi-bin/mailman/listinfo/llvm-commits" rel="noreferrer" target="_blank">http://lists.llvm.org/cgi-bin/mailman/listinfo/llvm-commits</a><br>
</blockquote></div><br></div>
</div></div></blockquote></div></div></div><br></div></div>
</blockquote></div><br></div>