[compiler-rt] r360712 - [libFuzzer] reimplement DFT's collect_data_flow inside libFuzzer so that we don't need external python scripts
Kostya Serebryany via llvm-commits
llvm-commits at lists.llvm.org
Tue May 14 14:47:35 PDT 2019
Author: kcc
Date: Tue May 14 14:47:35 2019
New Revision: 360712
URL: http://llvm.org/viewvc/llvm-project?rev=360712&view=rev
Log:
[libFuzzer] reimplement DFT's collect_data_flow inside libFuzzer so that we don't need external python scripts
Modified:
compiler-rt/trunk/lib/fuzzer/FuzzerDataFlowTrace.cpp
compiler-rt/trunk/lib/fuzzer/FuzzerDataFlowTrace.h
compiler-rt/trunk/test/fuzzer/dataflow.test
compiler-rt/trunk/test/fuzzer/only-some-bytes.test
Modified: compiler-rt/trunk/lib/fuzzer/FuzzerDataFlowTrace.cpp
URL: http://llvm.org/viewvc/llvm-project/compiler-rt/trunk/lib/fuzzer/FuzzerDataFlowTrace.cpp?rev=360712&r1=360711&r2=360712&view=diff
==============================================================================
--- compiler-rt/trunk/lib/fuzzer/FuzzerDataFlowTrace.cpp (original)
+++ compiler-rt/trunk/lib/fuzzer/FuzzerDataFlowTrace.cpp Tue May 14 14:47:35 2019
@@ -9,14 +9,21 @@
//===----------------------------------------------------------------------===//
#include "FuzzerDataFlowTrace.h"
+
+#include "FuzzerCommand.h"
#include "FuzzerIO.h"
#include "FuzzerRandom.h"
+#include "FuzzerSHA1.h"
+#include "FuzzerUtil.h"
#include <cstdlib>
#include <fstream>
#include <numeric>
+#include <queue>
#include <sstream>
#include <string>
+#include <unordered_map>
+#include <unordered_set>
#include <vector>
namespace fuzzer {
@@ -78,6 +85,7 @@ Vector<double> BlockCoverage::FunctionWe
for (auto It : Functions) {
auto FunctionID = It.first;
auto Counters = It.second;
+ assert(FunctionID < NumFunctions);
auto &Weight = Res[FunctionID];
Weight = 1000.; // this function is covered.
Weight /= SmallestNonZeroCounter(Counters);
@@ -97,10 +105,57 @@ void DataFlowTrace::ReadCoverage(const s
}
}
-void DataFlowTrace::Init(const std::string &DirPath,
+static void DFTStringAppendToVector(Vector<uint8_t> *DFT,
+ const std::string_view DFTString) {
+ assert(DFT->size() == DFTString.size());
+ for (size_t I = 0, Len = DFT->size(); I < Len; I++)
+ (*DFT)[I] = DFTString[I] == '1';
+}
+
+// converts a string of '0' and '1' into a Vector<uint8_t>
+static Vector<uint8_t> DFTStringToVector(const std::string_view DFTString) {
+ Vector<uint8_t> DFT(DFTString.size());
+ DFTStringAppendToVector(&DFT, DFTString);
+ return DFT;
+}
+
+static std::ostream &operator<<(std::ostream &OS, const Vector<uint8_t> &DFT) {
+ for (auto B : DFT)
+ OS << (B ? "1" : "0");
+ return OS;
+}
+
+static bool ParseError(const char *Err, const std::string &Line) {
+ Printf("DataFlowTrace: parse error: %s: Line: %s\n", Err, Line.c_str());
+ return false;
+};
+
+static bool ParseDFTLine(const std::string &Line, size_t *FunctionNum,
+ std::string_view *DFTString) {
+ if (!Line.empty() && Line[0] != 'F')
+ return false; // Ignore coverage.
+ size_t SpacePos = Line.find(' ');
+ if (SpacePos == std::string::npos)
+ return ParseError("no space in the trace line", Line);
+ if (Line.empty() || Line[0] != 'F')
+ return ParseError("the trace line doesn't start with 'F'", Line);
+ *FunctionNum = std::atol(Line.c_str() + 1);
+ const char *Beg = Line.c_str() + SpacePos + 1;
+ const char *End = Line.c_str() + Line.size();
+ assert(Beg < End);
+ size_t Len = End - Beg;
+ for (size_t I = 0; I < Len; I++) {
+ if (Beg[I] != '0' && Beg[I] != '1')
+ return ParseError("the trace should contain only 0 or 1", Line);
+ }
+ *DFTString = Beg;
+ return true;
+}
+
+bool DataFlowTrace::Init(const std::string &DirPath,
std::string *FocusFunction,
Random &Rand) {
- if (DirPath.empty()) return;
+ if (DirPath.empty()) return false;
Printf("INFO: DataFlowTrace: reading from '%s'\n", DirPath.c_str());
Vector<SizedFile> Files;
GetSizedFilesFromDir(DirPath, &Files);
@@ -144,7 +199,7 @@ void DataFlowTrace::Init(const std::stri
}
if (!NumFunctions || FocusFuncIdx == SIZE_MAX || Files.size() <= 1)
- return;
+ return false;
// Read traces.
size_t NumTraceFiles = 0;
@@ -152,41 +207,23 @@ void DataFlowTrace::Init(const std::stri
for (auto &SF : Files) {
auto Name = Basename(SF.File);
if (Name == kFunctionsTxt) continue;
- auto ParseError = [&](const char *Err) {
- Printf("DataFlowTrace: parse error: %s\n File: %s\n Line: %s\n", Err,
- Name.c_str(), L.c_str());
- };
NumTraceFiles++;
// Printf("=== %s\n", Name.c_str());
std::ifstream IF(SF.File);
while (std::getline(IF, L, '\n')) {
- if (!L.empty() && L[0] == 'C')
- continue; // Ignore coverage.
- size_t SpacePos = L.find(' ');
- if (SpacePos == std::string::npos)
- return ParseError("no space in the trace line");
- if (L.empty() || L[0] != 'F')
- return ParseError("the trace line doesn't start with 'F'");
- size_t N = std::atol(L.c_str() + 1);
- if (N >= NumFunctions)
- return ParseError("N is greater than the number of functions");
- if (N == FocusFuncIdx) {
+ size_t FunctionNum = 0;
+ std::string_view DFTString;
+ if (ParseDFTLine(L, &FunctionNum, &DFTString) &&
+ FunctionNum == FocusFuncIdx) {
NumTracesWithFocusFunction++;
- const char *Beg = L.c_str() + SpacePos + 1;
- const char *End = L.c_str() + L.size();
- assert(Beg < End);
- size_t Len = End - Beg;
- Vector<uint8_t> V(Len);
- for (size_t I = 0; I < Len; I++) {
- if (Beg[I] != '0' && Beg[I] != '1')
- ParseError("the trace should contain only 0 or 1");
- V[I] = Beg[I] == '1';
- }
- Traces[Name] = V;
+
+ if (FunctionNum >= NumFunctions)
+ return ParseError("N is greater than the number of functions", L);
+ Traces[Name] = DFTStringToVector(DFTString);
// Print just a few small traces.
- if (NumTracesWithFocusFunction <= 3 && Len <= 16)
- Printf("%s => |%s|\n", Name.c_str(), L.c_str() + SpacePos + 1);
- break; // No need to parse the following lines.
+ if (NumTracesWithFocusFunction <= 3 && DFTString.size() <= 16)
+ Printf("%s => |%s|\n", Name.c_str(), std::string(DFTString).c_str());
+ break; // No need to parse the following lines.
}
}
}
@@ -194,13 +231,87 @@ void DataFlowTrace::Init(const std::stri
Printf("INFO: DataFlowTrace: %zd trace files, %zd functions, "
"%zd traces with focus function\n",
NumTraceFiles, NumFunctions, NumTracesWithFocusFunction);
+ return true;
}
int CollectDataFlow(const std::string &DFTBinary, const std::string &DirPath,
const Vector<SizedFile> &CorporaFiles) {
- Printf("INFO: collecting data flow for %zd files\n", CorporaFiles.size());
+ Printf("INFO: collecting data flow: bin: %s dir: %s files: %zd\n",
+ DFTBinary.c_str(), DirPath.c_str(), CorporaFiles.size());
+ MkDir(DirPath);
+ auto Temp = TempPath(".dft");
+ for (auto &F : CorporaFiles) {
+ // For every input F we need to collect the data flow and the coverage.
+ // Data flow collection may fail if we request too many DFSan tags at once.
+ // So, we start from requesting all tags in range [0,Size) and if that fails
+ // we then request tags in [0,Size/2) and [Size/2, Size), and so on.
+ // Function number => DFT.
+ std::unordered_map<size_t, Vector<uint8_t>> DFTMap;
+ std::unordered_set<std::string> Cov;
+ std::queue<std::pair<size_t, size_t>> Q;
+ Q.push({0, F.Size});
+ while (!Q.empty()) {
+ auto R = Q.front();
+ Printf("\n\n\n********* Trying: [%zd, %zd)\n", R.first, R.second);
+ Q.pop();
+ Command Cmd;
+ Cmd.addArgument(DFTBinary);
+ Cmd.addArgument(std::to_string(R.first));
+ Cmd.addArgument(std::to_string(R.second));
+ Cmd.addArgument(F.File);
+ Cmd.addArgument(Temp);
+ Printf("CMD: %s\n", Cmd.toString().c_str());
+ if (ExecuteCommand(Cmd)) {
+ // DFSan has failed, collect tags for two subsets.
+ if (R.second - R.first >= 2) {
+ size_t Mid = (R.second + R.first) / 2;
+ Q.push({R.first, Mid});
+ Q.push({Mid, R.second});
+ }
+ } else {
+ Printf("********* Success: [%zd, %zd)\n", R.first, R.second);
+ std::ifstream IF(Temp);
+ std::string L;
+ while (std::getline(IF, L, '\n')) {
+ // Data flow collection has succeeded.
+ // Merge the results with the other runs.
+ if (L.empty()) continue;
+ if (L[0] == 'C') {
+ // Take coverage lines as is, they will be the same in all attempts.
+ Cov.insert(L);
+ } else if (L[0] == 'F') {
+ size_t FunctionNum = 0;
+ std::string_view DFTString;
+ if (ParseDFTLine(L, &FunctionNum, &DFTString)) {
+ auto &DFT = DFTMap[FunctionNum];
+ if (DFT.empty()) {
+ // Haven't seen this function before, take DFT as is.
+ DFT = DFTStringToVector(DFTString);
+ } else if (DFT.size() == DFTString.size()) {
+ // Have seen this function already, merge DFTs.
+ DFTStringAppendToVector(&DFT, DFTString);
+ }
+ }
+ }
+ }
+ }
+ }
+ auto OutPath = DirPlusFile(DirPath, Hash(FileToVector(F.File)));
+ // Dump combined DFT to disk.
+ Printf("Producing DFT for %s\n", OutPath.c_str());
+ std::ofstream OF(OutPath);
+ for (auto &DFT: DFTMap)
+ OF << "F" << DFT.first << " " << DFT.second << std::endl;
+ for (auto &C : Cov)
+ OF << C << std::endl;
+ }
+ RemoveFile(Temp);
+ // Write functions.txt.
+ Command Cmd;
+ Cmd.addArgument(DFTBinary);
+ Cmd.setOutputFile(DirPlusFile(DirPath, "functions.txt"));
+ ExecuteCommand(Cmd);
return 0;
}
} // namespace fuzzer
-
Modified: compiler-rt/trunk/lib/fuzzer/FuzzerDataFlowTrace.h
URL: http://llvm.org/viewvc/llvm-project/compiler-rt/trunk/lib/fuzzer/FuzzerDataFlowTrace.h?rev=360712&r1=360711&r2=360712&view=diff
==============================================================================
--- compiler-rt/trunk/lib/fuzzer/FuzzerDataFlowTrace.h (original)
+++ compiler-rt/trunk/lib/fuzzer/FuzzerDataFlowTrace.h Tue May 14 14:47:35 2019
@@ -111,7 +111,7 @@ class BlockCoverage {
class DataFlowTrace {
public:
void ReadCoverage(const std::string &DirPath);
- void Init(const std::string &DirPath, std::string *FocusFunction,
+ bool Init(const std::string &DirPath, std::string *FocusFunction,
Random &Rand);
void Clear() { Traces.clear(); }
const Vector<uint8_t> *Get(const std::string &InputSha1) const {
Modified: compiler-rt/trunk/test/fuzzer/dataflow.test
URL: http://llvm.org/viewvc/llvm-project/compiler-rt/trunk/test/fuzzer/dataflow.test?rev=360712&r1=360711&r2=360712&view=diff
==============================================================================
--- compiler-rt/trunk/test/fuzzer/dataflow.test (original)
+++ compiler-rt/trunk/test/fuzzer/dataflow.test Tue May 14 14:47:35 2019
@@ -74,6 +74,11 @@ RUN:%libfuzzer_src/scripts/merge_data_fl
# Test collect_data_flow
RUN: %libfuzzer_src/scripts/collect_data_flow.py %t-ThreeFunctionsTestDF %t/IN/FUZZMU | sort | FileCheck %s --check-prefix=IN_FUZZMU
+# Test libFuzzer's built in DFT collection.
+RUN: rm -rf %t-DFT
+RUN: %t-ThreeFunctionsTest -collect_data_flow=%t-ThreeFunctionsTestDF -data_flow_trace=%t-DFT %t/IN/FUZZMU
+RUN: cat %t-DFT/* | sort | FileCheck %s --check-prefix=IN_FUZZMU
+
IN_FUZZMU: F0 1111001
IN_FUZZMU: F1 0000100
IN_FUZZMU: F2 0000011
@@ -88,10 +93,19 @@ RUN: %t-ExplodeDFSanLabelsTestDF 4 6 %t
# Or we can use collect_data_flow
RUN: %libfuzzer_src/scripts/collect_data_flow.py %t-ExplodeDFSanLabelsTestDF %t/IN/1234567890123456
+# Test libFuzzer's builtin collect_data_flow.
+RUN: %t-ThreeFunctionsTest -collect_data_flow=%t-ThreeFunctionsTestDF -data_flow_trace=%t-DFT %t/IN/1234567890123456
+
# Test that we can run collect_data_flow on the entire corpus dir
RUN: rm -rf %t/OUT
RUN: %libfuzzer_src/scripts/collect_data_flow.py %t-ThreeFunctionsTestDF %t/IN %t/OUT
RUN: %t-ThreeFunctionsTest -data_flow_trace=%t/OUT -runs=0 -focus_function=Func2 2>&1 | FileCheck %s --check-prefix=USE_DATA_FLOW_TRACE
+
+RUN: rm -rf %t/OUT
+RUN: %t-ThreeFunctionsTest -collect_data_flow=%t-ThreeFunctionsTestDF -data_flow_trace=%t/OUT %t/IN
+RUN: %t-ThreeFunctionsTest -data_flow_trace=%t/OUT -runs=0 -focus_function=Func2 2>&1 | FileCheck %s --check-prefix=USE_DATA_FLOW_TRACE
+
+
USE_DATA_FLOW_TRACE: INFO: DataFlowTrace: reading from {{.*}}/OUT
USE_DATA_FLOW_TRACE-DAG: ca8eefe2fd5d6b32028f355fafa3e739a6bf5edc => |000001|
USE_DATA_FLOW_TRACE-DAG: d28cb407e8e1a702c72d25473f0553d3ec172262 => |0000011|
@@ -99,12 +113,14 @@ USE_DATA_FLOW_TRACE: INFO: DataFlowTrace
USE_DATA_FLOW_TRACE: INFO: Focus function is set to 'Func2'
# Test that we can run collect_data_flow on a long input (>2**16 bytes)
-RUN: rm -rf %t/OUT
RUN: printf "%0.sA" {1..150001} > %t/IN/very_long_input
+RUN: rm -rf %t/OUT
RUN: %libfuzzer_src/scripts/collect_data_flow.py %t-ThreeFunctionsTestDF %t/IN/very_long_input %t/OUT | FileCheck %s --check-prefix=COLLECT_TRACE_FOR_LONG_INPUT
+RUN: rm -rf %t/OUT
+RUN: %t-ThreeFunctionsTest -collect_data_flow=%t-ThreeFunctionsTestDF -data_flow_trace=%t/OUT %t/IN/very_long_input 2>&1 | FileCheck %s --check-prefix=COLLECT_TRACE_FOR_LONG_INPUT
RUN: rm %t/IN/very_long_input
-COLLECT_TRACE_FOR_LONG_INPUT: ******* Trying:{{[ ]+}}[0, 150001]
-COLLECT_TRACE_FOR_LONG_INPUT: ******* Trying:{{[ ]+}}[75000, 150001]
-COLLECT_TRACE_FOR_LONG_INPUT: ******* Trying:{{[ ]+}}[112500, 150001]
-COLLECT_TRACE_FOR_LONG_INPUT: ******* Success:{{[ ]+}}[{{[0123456789]+}}, 150001]
-COLLECT_TRACE_FOR_LONG_INPUT: ******* Success:{{[ ]+}}[0, {{[0123456789]+}}]
+COLLECT_TRACE_FOR_LONG_INPUT: ******* Trying:{{[ ]+}}[0, 150001
+COLLECT_TRACE_FOR_LONG_INPUT-DAG: ******* Trying:{{[ ]+}}[75000, 150001
+COLLECT_TRACE_FOR_LONG_INPUT-DAG: ******* Trying:{{[ ]+}}[112500, 150001
+COLLECT_TRACE_FOR_LONG_INPUT-DAG: ******* Success:{{[ ]+}}[{{[0123456789]+}}, 150001
+COLLECT_TRACE_FOR_LONG_INPUT-DAG: ******* Success:{{[ ]+}}[0, {{[0123456789]+}}
Modified: compiler-rt/trunk/test/fuzzer/only-some-bytes.test
URL: http://llvm.org/viewvc/llvm-project/compiler-rt/trunk/test/fuzzer/only-some-bytes.test?rev=360712&r1=360711&r2=360712&view=diff
==============================================================================
--- compiler-rt/trunk/test/fuzzer/only-some-bytes.test (original)
+++ compiler-rt/trunk/test/fuzzer/only-some-bytes.test Tue May 14 14:47:35 2019
@@ -23,7 +23,7 @@ RUN: %t-Fuzz -focus_function=f0 -runs=0
ONE_FOCUSED_INPUT: INFO: 1/3 inputs touch the focus function
RUN: rm -rf %t/IN_DFT
-RUN: %libfuzzer_src/scripts/collect_data_flow.py %t-DFT %t/IN %t/IN_DFT > /dev/null 2>&1
+RUN: %t-Fuzz -collect_data_flow=%t-DFT %t/IN -data_flow_trace=%t/IN_DFT > /dev/null 2>&1
# Repeat twice to make sure that the inputs with DFT are not removed from the corpus.
RUN: %t-Fuzz -focus_function=f0 -data_flow_trace=%t/IN_DFT -runs=100 %t/IN 2>&1 | FileCheck %s --check-prefix=HAVE_DFT
@@ -32,7 +32,7 @@ HAVE_DFT: INFO: 1/{{.*}} inputs have the
# Collect DFT, then use it.
RUN: rm -rf %t/C %t/C1 && mkdir %t/C %t/C1 && cp %t/IN/* %t/C
-RUN: rm -rf %t/C_DFT && %libfuzzer_src/scripts/collect_data_flow.py %t-DFT %t/C %t/C_DFT > /dev/null 2>&1
+RUN: rm -rf %t/C_DFT && %t-Fuzz -collect_data_flow=%t-DFT %t/C -data_flow_trace=%t/C_DFT > /dev/null 2>&1
RUN: not %t-Fuzz -focus_function=f0 -data_flow_trace=%t/C_DFT -seed=1 -runs=1000000 -use_value_profile=1 %t/C1 %t/C 2> %t/log
RUN: grep BINGO %t/log
More information about the llvm-commits
mailing list