[compiler-rt] e2d0b44 - [DFSan] Add efficient fast16labels instrumentation mode.
Matt Morehouse via llvm-commits
llvm-commits at lists.llvm.org
Wed Jul 29 12:00:05 PDT 2020
Author: Matt Morehouse
Date: 2020-07-29T18:58:47Z
New Revision: e2d0b44a7cd261218c9e527d23eb5d13425afe8b
URL: https://github.com/llvm/llvm-project/commit/e2d0b44a7cd261218c9e527d23eb5d13425afe8b
DIFF: https://github.com/llvm/llvm-project/commit/e2d0b44a7cd261218c9e527d23eb5d13425afe8b.diff
LOG: [DFSan] Add efficient fast16labels instrumentation mode.
Adds the -fast-16-labels flag, which enables efficient instrumentation
for DFSan when the user needs <=16 labels. The instrumentation
eliminates most branches and most calls to __dfsan_union or
__dfsan_union_load.
Reviewed By: vitalybuka
Differential Revision: https://reviews.llvm.org/D84371
Added:
llvm/test/Instrumentation/DataFlowSanitizer/fast16labels.ll
Modified:
clang/docs/DataFlowSanitizer.rst
compiler-rt/lib/dfsan/dfsan.cpp
compiler-rt/lib/dfsan/dfsan_flags.inc
compiler-rt/lib/dfsan/done_abilist.txt
compiler-rt/lib/fuzzer/FuzzerDataFlowTrace.cpp
compiler-rt/lib/fuzzer/dataflow/DataFlow.cpp
compiler-rt/test/dfsan/fast16labels.c
compiler-rt/test/fuzzer/dataflow.test
compiler-rt/test/fuzzer/only-some-bytes-fork.test
compiler-rt/test/fuzzer/only-some-bytes.test
llvm/lib/Transforms/Instrumentation/DataFlowSanitizer.cpp
Removed:
################################################################################
diff --git a/clang/docs/DataFlowSanitizer.rst b/clang/docs/DataFlowSanitizer.rst
index 44956037490a..cd8c5869a017 100644
--- a/clang/docs/DataFlowSanitizer.rst
+++ b/clang/docs/DataFlowSanitizer.rst
@@ -174,6 +174,58 @@ the correct labels are propagated.
return 0;
}
+fast16labels mode
+=================
+
+If you need 16 or fewer labels, you can use fast16labels instrumentation for
+less CPU and code size overhead. To use fast16labels instrumentation, you'll
+need to specify `-fsanitize=dataflow -mllvm -dfsan-fast-16-labels` in your
+compile and link commands and use a modified API for creating and managing
+labels.
+
+In fast16labels mode, base labels are simply 16-bit unsigned integers that are
+powers of 2 (i.e. 1, 2, 4, 8, ..., 32768), and union labels are created by ORing
+base labels. In this mode DFSan does not manage any label metadata, so the
+functions `dfsan_create_label`, `dfsan_union`, `dfsan_get_label_info`,
+`dfsan_has_label`, `dfsan_has_label_with_desc`, `dfsan_get_label_count`, and
+`dfsan_dump_labels` are unsupported. Instead of using them, the user should
+maintain any necessary metadata about base labels themselves.
+
+For example:
+
+.. code-block:: c++
+
+ #include <sanitizer/dfsan_interface.h>
+ #include <assert.h>
+
+ int main(void) {
+ int i = 100;
+ int j = 200;
+ int k = 300;
+ dfsan_label i_label = 1;
+ dfsan_label j_label = 2;
+ dfsan_label k_label = 4;
+ dfsan_set_label(i_label, &i, sizeof(i));
+ dfsan_set_label(j_label, &j, sizeof(j));
+ dfsan_set_label(k_label, &k, sizeof(k));
+
+ dfsan_label ij_label = dfsan_get_label(i + j);
+
+ assert(ij_label & i_label); // ij_label has i_label
+ assert(ij_label & j_label); // ij_label has j_label
+ assert(!(ij_label & k_label)); // ij_label doesn't have k_label
+ assert(ij_label == 3) // Verifies all of the above
+
+ dfsan_label ijk_label = dfsan_get_label(i + j + k);
+
+ assert(ijk_label & i_label); // ijk_label has i_label
+ assert(ijk_label & j_label); // ijk_label has j_label
+ assert(ijk_label & k_label); // ijk_label has k_label
+ assert(ijk_label == 7); // Verifies all of the above
+
+ return 0;
+ }
+
Current status
==============
diff --git a/compiler-rt/lib/dfsan/dfsan.cpp b/compiler-rt/lib/dfsan/dfsan.cpp
index 105989c93ab1..678f6c1183e0 100644
--- a/compiler-rt/lib/dfsan/dfsan.cpp
+++ b/compiler-rt/lib/dfsan/dfsan.cpp
@@ -18,15 +18,16 @@
// prefixed __dfsan_.
//===----------------------------------------------------------------------===//
+#include "dfsan/dfsan.h"
+
#include "sanitizer_common/sanitizer_atomic.h"
#include "sanitizer_common/sanitizer_common.h"
#include "sanitizer_common/sanitizer_file.h"
-#include "sanitizer_common/sanitizer_flags.h"
#include "sanitizer_common/sanitizer_flag_parser.h"
+#include "sanitizer_common/sanitizer_flags.h"
+#include "sanitizer_common/sanitizer_internal_defs.h"
#include "sanitizer_common/sanitizer_libc.h"
-#include "dfsan/dfsan.h"
-
using namespace __dfsan;
typedef atomic_uint16_t atomic_dfsan_label;
@@ -158,18 +159,10 @@ static void dfsan_check_label(dfsan_label label) {
}
}
-static void ReportUnsupportedFast16(const char *func) {
- Report("FATAL: DataFlowSanitizer: %s is unsupported in fast16labels mode\n",
- func);
- Die();
-}
-
// Resolves the union of two unequal labels. Nonequality is a precondition for
// this function (the instrumentation pass inlines the equality test).
extern "C" SANITIZER_INTERFACE_ATTRIBUTE
dfsan_label __dfsan_union(dfsan_label l1, dfsan_label l2) {
- if (flags().fast16labels)
- return l1 | l2;
DCHECK_NE(l1, l2);
if (l1 == 0)
@@ -224,6 +217,14 @@ dfsan_label __dfsan_union_load(const dfsan_label *ls, uptr n) {
return label;
}
+extern "C" SANITIZER_INTERFACE_ATTRIBUTE
+dfsan_label __dfsan_union_load_fast16labels(const dfsan_label *ls, uptr n) {
+ dfsan_label label = ls[0];
+ for (uptr i = 1; i != n; ++i)
+ label |= ls[i];
+ return label;
+}
+
extern "C" SANITIZER_INTERFACE_ATTRIBUTE
void __dfsan_unimplemented(char *fname) {
if (flags().warn_unimplemented)
@@ -259,8 +260,6 @@ dfsan_union(dfsan_label l1, dfsan_label l2) {
extern "C" SANITIZER_INTERFACE_ATTRIBUTE
dfsan_label dfsan_create_label(const char *desc, void *userdata) {
- if (flags().fast16labels)
- ReportUnsupportedFast16("dfsan_create_label");
dfsan_label label =
atomic_fetch_add(&__dfsan_last_label, 1, memory_order_relaxed) + 1;
dfsan_check_label(label);
@@ -319,15 +318,11 @@ dfsan_read_label(const void *addr, uptr size) {
extern "C" SANITIZER_INTERFACE_ATTRIBUTE
const struct dfsan_label_info *dfsan_get_label_info(dfsan_label label) {
- if (flags().fast16labels)
- ReportUnsupportedFast16("dfsan_get_label_info");
return &__dfsan_label_info[label];
}
extern "C" SANITIZER_INTERFACE_ATTRIBUTE int
dfsan_has_label(dfsan_label label, dfsan_label elem) {
- if (flags().fast16labels)
- return label & elem;
if (label == elem)
return true;
const dfsan_label_info *info = dfsan_get_label_info(label);
@@ -340,8 +335,6 @@ dfsan_has_label(dfsan_label label, dfsan_label elem) {
extern "C" SANITIZER_INTERFACE_ATTRIBUTE dfsan_label
dfsan_has_label_with_desc(dfsan_label label, const char *desc) {
- if (flags().fast16labels)
- ReportUnsupportedFast16("dfsan_has_label_with_desc");
const dfsan_label_info *info = dfsan_get_label_info(label);
if (info->l1 != 0) {
return dfsan_has_label_with_desc(info->l1, desc) ||
@@ -361,9 +354,6 @@ dfsan_get_label_count(void) {
extern "C" SANITIZER_INTERFACE_ATTRIBUTE void
dfsan_dump_labels(int fd) {
- if (flags().fast16labels)
- return;
-
dfsan_label last_label =
atomic_load(&__dfsan_last_label, memory_order_relaxed);
for (uptr l = 1; l <= last_label; ++l) {
diff --git a/compiler-rt/lib/dfsan/dfsan_flags.inc b/compiler-rt/lib/dfsan/dfsan_flags.inc
index 29db73b98278..cdd0035c9b2d 100644
--- a/compiler-rt/lib/dfsan/dfsan_flags.inc
+++ b/compiler-rt/lib/dfsan/dfsan_flags.inc
@@ -29,7 +29,3 @@ DFSAN_FLAG(
DFSAN_FLAG(const char *, dump_labels_at_exit, "", "The path of the file where "
"to dump the labels when the "
"program terminates.")
-DFSAN_FLAG(bool, fast16labels, false,
- "Enables experimental mode where DFSan supports only 16 power-of-2 labels "
- "(1, 2, 4, 8, ... 32768) and the label union is computed as a bit-wise OR."
-)
diff --git a/compiler-rt/lib/dfsan/done_abilist.txt b/compiler-rt/lib/dfsan/done_abilist.txt
index 52f3ff5ef239..540f0a4c6bfc 100644
--- a/compiler-rt/lib/dfsan/done_abilist.txt
+++ b/compiler-rt/lib/dfsan/done_abilist.txt
@@ -28,6 +28,8 @@ fun:dfsan_set_write_callback=uninstrumented
fun:dfsan_set_write_callback=custom
fun:dfsan_flush=uninstrumented
fun:dfsan_flush=discard
+fun:dfsan_use_fast16labels=uninstrumented
+fun:dfsan_use_fast16labels=discard
###############################################################################
# glibc
diff --git a/compiler-rt/lib/fuzzer/FuzzerDataFlowTrace.cpp b/compiler-rt/lib/fuzzer/FuzzerDataFlowTrace.cpp
index 48df8e668604..0e9cdf7e66b1 100644
--- a/compiler-rt/lib/fuzzer/FuzzerDataFlowTrace.cpp
+++ b/compiler-rt/lib/fuzzer/FuzzerDataFlowTrace.cpp
@@ -253,7 +253,7 @@ int CollectDataFlow(const std::string &DFTBinary, const std::string &DirPath,
return 1;
}
- static char DFSanEnv[] = "DFSAN_OPTIONS=fast16labels=1:warn_unimplemented=0";
+ static char DFSanEnv[] = "DFSAN_OPTIONS=warn_unimplemented=0";
putenv(DFSanEnv);
MkDir(DirPath);
for (auto &F : CorporaFiles) {
diff --git a/compiler-rt/lib/fuzzer/dataflow/DataFlow.cpp b/compiler-rt/lib/fuzzer/dataflow/DataFlow.cpp
index 8bf4e25b8b15..78b3f9abd5cf 100644
--- a/compiler-rt/lib/fuzzer/dataflow/DataFlow.cpp
+++ b/compiler-rt/lib/fuzzer/dataflow/DataFlow.cpp
@@ -17,9 +17,11 @@
// and also provides basic-block coverage for every input.
//
// Build:
-// 1. Compile this file (DataFlow.cpp) with -fsanitize=dataflow and -O2.
+// 1. Compile this file (DataFlow.cpp) with -fsanitize=dataflow -mllvm
+// -dfsan-fast-16-labels and -O2.
// 2. Compile DataFlowCallbacks.cpp with -O2 -fPIC.
// 3. Build the fuzz target with -g -fsanitize=dataflow
+// -mllvm -dfsan-fast-16-labels
// -fsanitize-coverage=trace-pc-guard,pc-table,bb,trace-cmp
// 4. Link those together with -fsanitize=dataflow
//
@@ -36,7 +38,7 @@
// Run:
// # Collect data flow and coverage for INPUT_FILE
// # write to OUTPUT_FILE (default: stdout)
-// export DFSAN_OPTIONS=fast16labels=1:warn_unimplemented=0
+// export DFSAN_OPTIONS=warn_unimplemented=0
// ./a.out INPUT_FILE [OUTPUT_FILE]
//
// # Print all instrumented functions. llvm-symbolizer must be present in PATH
diff --git a/compiler-rt/test/dfsan/fast16labels.c b/compiler-rt/test/dfsan/fast16labels.c
index 269d1e2efc0a..72aaf38bfed4 100644
--- a/compiler-rt/test/dfsan/fast16labels.c
+++ b/compiler-rt/test/dfsan/fast16labels.c
@@ -1,14 +1,8 @@
-// RUN: %clang_dfsan %s -o %t
-// RUN: DFSAN_OPTIONS=fast16labels=1 %run %t
-// RUN: DFSAN_OPTIONS=fast16labels=1 not %run %t dfsan_create_label 2>&1 \
-// RUN: | FileCheck %s --check-prefix=CREATE-LABEL
-// RUN: DFSAN_OPTIONS=fast16labels=1 not %run %t dfsan_get_label_info 2>&1 \
-// RUN: | FileCheck %s --check-prefix=GET-LABEL-INFO
-// RUN: DFSAN_OPTIONS=fast16labels=1 not %run %t dfsan_has_label_with_desc \
-// RUN: 2>&1 | FileCheck %s --check-prefix=HAS-LABEL-WITH-DESC
-//
-// Tests DFSAN_OPTIONS=fast16labels=1
+// RUN: %clang_dfsan %s -mllvm -dfsan-fast-16-labels -o %t
+// RUN: %run %t
//
+// Tests fast16labels mode.
+
#include <sanitizer/dfsan_interface.h>
#include <assert.h>
@@ -20,19 +14,6 @@ int foo(int a, int b) {
}
int main(int argc, char *argv[]) {
- // Death tests for unsupported API usage.
- const char *command = (argc < 2) ? "" : argv[1];
- // CREATE-LABEL: FATAL: DataFlowSanitizer: dfsan_create_label is unsupported
- if (strcmp(command, "dfsan_create_label") == 0)
- dfsan_create_label("", NULL);
- // GET-LABEL-INFO: FATAL: DataFlowSanitizer: dfsan_get_label_info is unsupported
- if (strcmp(command, "dfsan_get_label_info") == 0)
- dfsan_get_label_info(1);
- // HAS-LABEL-WITH-DESC: FATAL: DataFlowSanitizer: dfsan_has_label_with_desc is unsupported
- if (strcmp(command, "dfsan_has_label_with_desc") == 0)
- dfsan_has_label_with_desc(1, "");
-
- // Supported usage.
int a = 10;
int b = 20;
dfsan_set_label(8, &a, sizeof(a));
@@ -43,7 +24,4 @@ int main(int argc, char *argv[]) {
dfsan_label l = dfsan_get_label(c);
printf("C: 0x%x\n", l);
assert(l == 520); // OR of the other two labels.
- assert(dfsan_has_label(l, 8));
- assert(dfsan_has_label(l, 512));
- assert(!dfsan_has_label(l, 1));
}
diff --git a/compiler-rt/test/fuzzer/dataflow.test b/compiler-rt/test/fuzzer/dataflow.test
index 93cd580b9d52..391f160d0044 100644
--- a/compiler-rt/test/fuzzer/dataflow.test
+++ b/compiler-rt/test/fuzzer/dataflow.test
@@ -2,10 +2,10 @@
REQUIRES: linux, x86_64
# Build the tracer and the test.
-RUN: %no_fuzzer_cpp_compiler -c -fno-sanitize=all -fsanitize=dataflow %S/../../lib/fuzzer/dataflow/DataFlow.cpp -o %t-DataFlow.o
+RUN: %no_fuzzer_cpp_compiler -c -fno-sanitize=all -fsanitize=dataflow -mllvm -dfsan-fast-16-labels %S/../../lib/fuzzer/dataflow/DataFlow.cpp -o %t-DataFlow.o
RUN: %no_fuzzer_cpp_compiler -c -fno-sanitize=all -fPIC %S/../../lib/fuzzer/dataflow/DataFlowCallbacks.cpp -o %t-DataFlowCallbacks.o
-RUN: %no_fuzzer_cpp_compiler -fno-sanitize=all -fsanitize=dataflow -fsanitize-coverage=trace-pc-guard,pc-table,bb,trace-cmp %S/ThreeFunctionsTest.cpp %t-DataFlow*.o -o %t-ThreeFunctionsTestDF
-RUN: %no_fuzzer_cpp_compiler -fno-sanitize=all -fsanitize=dataflow -fsanitize-coverage=trace-pc-guard,pc-table,bb,trace-cmp %S/Labels20Test.cpp %t-DataFlow*.o -o %t-Labels20TestDF
+RUN: %no_fuzzer_cpp_compiler -fno-sanitize=all -fsanitize=dataflow -mllvm -dfsan-fast-16-labels -fsanitize-coverage=trace-pc-guard,pc-table,bb,trace-cmp %S/ThreeFunctionsTest.cpp %t-DataFlow*.o -o %t-ThreeFunctionsTestDF
+RUN: %no_fuzzer_cpp_compiler -fno-sanitize=all -fsanitize=dataflow -mllvm -dfsan-fast-16-labels -fsanitize-coverage=trace-pc-guard,pc-table,bb,trace-cmp %S/Labels20Test.cpp %t-DataFlow*.o -o %t-Labels20TestDF
RUN: %cpp_compiler %S/ThreeFunctionsTest.cpp -o %t-ThreeFunctionsTest
# Dump the function list.
@@ -30,7 +30,7 @@ RUN: echo -n FUZxxxxxxxxxxxxxxxxx > %t/IN20/FUZxxxxxxxxxxxxxxxxx
RUN: echo -n FUxxxxxxxxxxxxxxxxxx > %t/IN20/FUxxxxxxxxxxxxxxxxxx
-RUN: export DFSAN_OPTIONS=fast16labels=1:warn_unimplemented=0
+RUN: export DFSAN_OPTIONS=warn_unimplemented=0
# This test assumes that the functions in ThreeFunctionsTestDF are instrumented
# in a specific order:
diff --git a/compiler-rt/test/fuzzer/only-some-bytes-fork.test b/compiler-rt/test/fuzzer/only-some-bytes-fork.test
index ef03c24fd2c6..a4142dbbb3b9 100644
--- a/compiler-rt/test/fuzzer/only-some-bytes-fork.test
+++ b/compiler-rt/test/fuzzer/only-some-bytes-fork.test
@@ -2,9 +2,9 @@
REQUIRES: linux, x86_64
# Build the tracer and the test.
-RUN: %no_fuzzer_cpp_compiler -c -fno-sanitize=all -fsanitize=dataflow %S/../../lib/fuzzer/dataflow/DataFlow.cpp -o %t-DataFlow.o
+RUN: %no_fuzzer_cpp_compiler -c -fno-sanitize=all -fsanitize=dataflow -mllvm -dfsan-fast-16-labels %S/../../lib/fuzzer/dataflow/DataFlow.cpp -o %t-DataFlow.o
RUN: %no_fuzzer_cpp_compiler -c -fno-sanitize=all -fPIC %S/../../lib/fuzzer/dataflow/DataFlowCallbacks.cpp -o %t-DataFlowCallbacks.o
-RUN: %no_fuzzer_cpp_compiler -fno-sanitize=all -fsanitize=dataflow -fsanitize-coverage=trace-pc-guard,pc-table,bb,trace-cmp %S/OnlySomeBytesTest.cpp %t-DataFlow*.o -o %t-DFT
+RUN: %no_fuzzer_cpp_compiler -fno-sanitize=all -fsanitize=dataflow -mllvm -dfsan-fast-16-labels -fsanitize-coverage=trace-pc-guard,pc-table,bb,trace-cmp %S/OnlySomeBytesTest.cpp %t-DataFlow*.o -o %t-DFT
RUN: %cpp_compiler %S/OnlySomeBytesTest.cpp -o %t-Fuzz
# Test that the fork mode can collect and use the DFT
diff --git a/compiler-rt/test/fuzzer/only-some-bytes.test b/compiler-rt/test/fuzzer/only-some-bytes.test
index 62703f398f76..9a31ebfc93c1 100644
--- a/compiler-rt/test/fuzzer/only-some-bytes.test
+++ b/compiler-rt/test/fuzzer/only-some-bytes.test
@@ -2,9 +2,9 @@
REQUIRES: linux, x86_64
# Build the tracer and the test.
-RUN: %no_fuzzer_cpp_compiler -c -fno-sanitize=all -fsanitize=dataflow %S/../../lib/fuzzer/dataflow/DataFlow.cpp -o %t-DataFlow.o
+RUN: %no_fuzzer_cpp_compiler -c -fno-sanitize=all -fsanitize=dataflow -mllvm -dfsan-fast-16-labels %S/../../lib/fuzzer/dataflow/DataFlow.cpp -o %t-DataFlow.o
RUN: %no_fuzzer_cpp_compiler -c -fno-sanitize=all -fPIC %S/../../lib/fuzzer/dataflow/DataFlowCallbacks.cpp -o %t-DataFlowCallbacks.o
-RUN: %no_fuzzer_cpp_compiler -fno-sanitize=all -fsanitize=dataflow -fsanitize-coverage=trace-pc-guard,pc-table,bb,trace-cmp %S/OnlySomeBytesTest.cpp %t-DataFlow*.o -o %t-DFT
+RUN: %no_fuzzer_cpp_compiler -fno-sanitize=all -fsanitize=dataflow -mllvm -dfsan-fast-16-labels -fsanitize-coverage=trace-pc-guard,pc-table,bb,trace-cmp %S/OnlySomeBytesTest.cpp %t-DataFlow*.o -o %t-DFT
RUN: %cpp_compiler %S/OnlySomeBytesTest.cpp -o %t-Fuzz
# Prepare the inputs.
diff --git a/llvm/lib/Transforms/Instrumentation/DataFlowSanitizer.cpp b/llvm/lib/Transforms/Instrumentation/DataFlowSanitizer.cpp
index 317b0904876a..feb1abb18777 100644
--- a/llvm/lib/Transforms/Instrumentation/DataFlowSanitizer.cpp
+++ b/llvm/lib/Transforms/Instrumentation/DataFlowSanitizer.cpp
@@ -178,6 +178,14 @@ static cl::opt<bool> ClEventCallbacks(
cl::desc("Insert calls to __dfsan_*_callback functions on data events."),
cl::Hidden, cl::init(false));
+// Use a distinct bit for each base label, enabling faster unions with less
+// instrumentation. Limits the max number of base labels to 16.
+static cl::opt<bool> ClFast16Labels(
+ "dfsan-fast-16-labels",
+ cl::desc("Use more efficient instrumentation, limiting the number of "
+ "labels to 16."),
+ cl::Hidden, cl::init(false));
+
static StringRef GetGlobalTypeString(const GlobalValue &G) {
// Types of GlobalVariables are always pointer types.
Type *GType = G.getValueType();
@@ -362,6 +370,7 @@ class DataFlowSanitizer {
FunctionCallee DFSanUnionFn;
FunctionCallee DFSanCheckedUnionFn;
FunctionCallee DFSanUnionLoadFn;
+ FunctionCallee DFSanUnionLoadFast16LabelsFn;
FunctionCallee DFSanUnimplementedFn;
FunctionCallee DFSanSetLabelFn;
FunctionCallee DFSanNonzeroLabelFn;
@@ -748,6 +757,17 @@ void DataFlowSanitizer::initializeRuntimeFunctions(Module &M) {
DFSanUnionLoadFn =
Mod->getOrInsertFunction("__dfsan_union_load", DFSanUnionLoadFnTy, AL);
}
+ {
+ AttributeList AL;
+ AL = AL.addAttribute(M.getContext(), AttributeList::FunctionIndex,
+ Attribute::NoUnwind);
+ AL = AL.addAttribute(M.getContext(), AttributeList::FunctionIndex,
+ Attribute::ReadOnly);
+ AL = AL.addAttribute(M.getContext(), AttributeList::ReturnIndex,
+ Attribute::ZExt);
+ DFSanUnionLoadFast16LabelsFn = Mod->getOrInsertFunction(
+ "__dfsan_union_load_fast16labels", DFSanUnionLoadFnTy, AL);
+ }
DFSanUnimplementedFn =
Mod->getOrInsertFunction("__dfsan_unimplemented", DFSanUnimplementedFnTy);
{
@@ -810,6 +830,7 @@ bool DataFlowSanitizer::runImpl(Module &M) {
&i != DFSanUnionFn.getCallee()->stripPointerCasts() &&
&i != DFSanCheckedUnionFn.getCallee()->stripPointerCasts() &&
&i != DFSanUnionLoadFn.getCallee()->stripPointerCasts() &&
+ &i != DFSanUnionLoadFast16LabelsFn.getCallee()->stripPointerCasts() &&
&i != DFSanUnimplementedFn.getCallee()->stripPointerCasts() &&
&i != DFSanSetLabelFn.getCallee()->stripPointerCasts() &&
&i != DFSanNonzeroLabelFn.getCallee()->stripPointerCasts() &&
@@ -1144,7 +1165,10 @@ Value *DFSanFunction::combineShadows(Value *V1, Value *V2, Instruction *Pos) {
return CCS.Shadow;
IRBuilder<> IRB(Pos);
- if (AvoidNewBlocks) {
+ if (ClFast16Labels) {
+ CCS.Block = Pos->getParent();
+ CCS.Shadow = IRB.CreateOr(V1, V2);
+ } else if (AvoidNewBlocks) {
CallInst *Call = IRB.CreateCall(DFS.DFSanCheckedUnionFn, {V1, V2});
Call->addAttribute(AttributeList::ReturnIndex, Attribute::ZExt);
Call->addParamAttr(0, Attribute::ZExt);
@@ -1254,6 +1278,30 @@ Value *DFSanFunction::loadShadow(Value *Addr, uint64_t Size, uint64_t Align,
IRB.CreateAlignedLoad(DFS.ShadowTy, ShadowAddr1, ShadowAlign), Pos);
}
}
+
+ if (ClFast16Labels && Size % (64 / DFS.ShadowWidthBits) == 0) {
+ // First OR all the WideShadows, then OR individual shadows within the
+ // combined WideShadow. This is fewer instructions than ORing shadows
+ // individually.
+ IRBuilder<> IRB(Pos);
+ Value *WideAddr =
+ IRB.CreateBitCast(ShadowAddr, Type::getInt64PtrTy(*DFS.Ctx));
+ Value *CombinedWideShadow =
+ IRB.CreateAlignedLoad(IRB.getInt64Ty(), WideAddr, ShadowAlign);
+ for (uint64_t Ofs = 64 / DFS.ShadowWidthBits; Ofs != Size;
+ Ofs += 64 / DFS.ShadowWidthBits) {
+ WideAddr = IRB.CreateGEP(Type::getInt64Ty(*DFS.Ctx), WideAddr,
+ ConstantInt::get(DFS.IntptrTy, 1));
+ Value *NextWideShadow =
+ IRB.CreateAlignedLoad(IRB.getInt64Ty(), WideAddr, ShadowAlign);
+ CombinedWideShadow = IRB.CreateOr(CombinedWideShadow, NextWideShadow);
+ }
+ for (unsigned Width = 32; Width >= DFS.ShadowWidthBits; Width >>= 1) {
+ Value *ShrShadow = IRB.CreateLShr(CombinedWideShadow, Width);
+ CombinedWideShadow = IRB.CreateOr(CombinedWideShadow, ShrShadow);
+ }
+ return IRB.CreateTrunc(CombinedWideShadow, DFS.ShadowTy);
+ }
if (!AvoidNewBlocks && Size % (64 / DFS.ShadowWidthBits) == 0) {
// Fast path for the common case where each byte has identical shadow: load
// shadow 64 bits at a time, fall out to a __dfsan_union_load call if any
@@ -1320,8 +1368,10 @@ Value *DFSanFunction::loadShadow(Value *Addr, uint64_t Size, uint64_t Align,
}
IRBuilder<> IRB(Pos);
+ FunctionCallee &UnionLoadFn =
+ ClFast16Labels ? DFS.DFSanUnionLoadFast16LabelsFn : DFS.DFSanUnionLoadFn;
CallInst *FallbackCall = IRB.CreateCall(
- DFS.DFSanUnionLoadFn, {ShadowAddr, ConstantInt::get(DFS.IntptrTy, Size)});
+ UnionLoadFn, {ShadowAddr, ConstantInt::get(DFS.IntptrTy, Size)});
FallbackCall->addAttribute(AttributeList::ReturnIndex, Attribute::ZExt);
return FallbackCall;
}
diff --git a/llvm/test/Instrumentation/DataFlowSanitizer/fast16labels.ll b/llvm/test/Instrumentation/DataFlowSanitizer/fast16labels.ll
new file mode 100644
index 000000000000..85f99f5b1828
--- /dev/null
+++ b/llvm/test/Instrumentation/DataFlowSanitizer/fast16labels.ll
@@ -0,0 +1,100 @@
+; Test that -dfsan-fast-16-labels mode uses inline ORs rather than calling
+; __dfsan_union or __dfsan_union_load.
+; RUN: opt < %s -dfsan -dfsan-fast-16-labels -S | FileCheck %s --implicit-check-not="call{{.*}}__dfsan_union"
+target datalayout = "e-p:64:64:64-i1:8:8-i8:8:8-i16:16:16-i32:32:32-i64:64:64-f32:32:32-f64:64:64-v64:64:64-v128:128:128-a0:0:64-s0:64:64-f80:128:128-n8:16:32:64-S128"
+target triple = "x86_64-unknown-linux-gnu"
+
+define i8 @add(i8 %a, i8 %b) {
+ ; CHECK-LABEL: define i8 @"dfs$add"
+ ; CHECK-DAG: %[[ALABEL:.*]] = load{{.*}}__dfsan_arg_tls, i64 0, i64 0
+ ; CHECK-DAG: %[[BLABEL:.*]] = load{{.*}}__dfsan_arg_tls, i64 0, i64 1
+ ; CHECK: %[[ADDLABEL:.*]] = or i16 %[[ALABEL]], %[[BLABEL]]
+ ; CHECK: add i8
+ ; CHECK: store i16 %[[ADDLABEL]], i16* @__dfsan_retval_tls
+ ; CHECK: ret i8
+ %c = add i8 %a, %b
+ ret i8 %c
+}
+
+define i8 @load8(i8* %p) {
+ ; CHECK-LABEL: define i8 @"dfs$load8"
+ ; CHECK: load i16, i16*
+ ; CHECK: ptrtoint i8* {{.*}} to i64
+ ; CHECK: and i64
+ ; CHECK: mul i64
+ ; CHECK: inttoptr i64
+ ; CHECK: load i16, i16*
+ ; CHECK: or i16
+ ; CHECK: load i8, i8*
+ ; CHECK: store i16 {{.*}} @__dfsan_retval_tls
+ ; CHECK: ret i8
+
+ %a = load i8, i8* %p
+ ret i8 %a
+}
+
+define i16 @load16(i16* %p) {
+ ; CHECK-LABEL: define i16 @"dfs$load16"
+ ; CHECK: ptrtoint i16*
+ ; CHECK: and i64
+ ; CHECK: mul i64
+ ; CHECK: inttoptr i64 {{.*}} i16*
+ ; CHECK: getelementptr i16
+ ; CHECK: load i16, i16*
+ ; CHECK: load i16, i16*
+ ; CHECK: or i16
+ ; CHECK: or i16
+ ; CHECK: load i16, i16*
+ ; CHECK: store {{.*}} @__dfsan_retval_tls
+ ; CHECK: ret i16
+
+ %a = load i16, i16* %p
+ ret i16 %a
+}
+
+define i32 @load32(i32* %p) {
+ ; CHECK-LABEL: define i32 @"dfs$load32"
+ ; CHECK: ptrtoint i32*
+ ; CHECK: and i64
+ ; CHECK: mul i64
+ ; CHECK: inttoptr i64 {{.*}} i16*
+ ; CHECK: bitcast i16* {{.*}} i64*
+ ; CHECK: load i64, i64*
+ ; CHECK: lshr i64 {{.*}}, 32
+ ; CHECK: or i64
+ ; CHECK: lshr i64 {{.*}}, 16
+ ; CHECK: or i64
+ ; CHECK: trunc i64 {{.*}} i16
+ ; CHECK: or i16
+ ; CHECK: load i32, i32*
+ ; CHECK: store i16 {{.*}} @__dfsan_retval_tls
+ ; CHECK: ret i32
+
+ %a = load i32, i32* %p
+ ret i32 %a
+}
+
+define i64 @load64(i64* %p) {
+ ; CHECK-LABEL: define i64 @"dfs$load64"
+ ; CHECK: ptrtoint i64*
+ ; CHECK: and i64
+ ; CHECK: mul i64
+ ; CHECK: inttoptr i64 {{.*}} i16*
+ ; CHECK: bitcast i16* {{.*}} i64*
+ ; CHECK: load i64, i64*
+ ; CHECK: getelementptr i64, i64* {{.*}}, i64 1
+ ; CHECK: load i64, i64*
+ ; CHECK: or i64
+ ; CHECK: lshr i64 {{.*}}, 32
+ ; CHECK: or i64
+ ; CHECK: lshr i64 {{.*}}, 16
+ ; CHECK: or i64
+ ; CHECK: trunc i64 {{.*}} i16
+ ; CHECK: or i16
+ ; CHECK: load i64, i64*
+ ; CHECK: store i16 {{.*}} @__dfsan_retval_tls
+ ; CHECK: ret i64
+
+ %a = load i64, i64* %p
+ ret i64 %a
+}
More information about the llvm-commits
mailing list