[compiler-rt] [CompilerRT] Add numerical sanitizer (PR #94322)
Alexander Shaposhnikov via llvm-commits
llvm-commits at lists.llvm.org
Wed Jun 19 12:41:52 PDT 2024
https://github.com/alexander-shaposhnikov updated https://github.com/llvm/llvm-project/pull/94322
>From 42d00684275a9117bcddd3a0291549a6d9bcdd12 Mon Sep 17 00:00:00 2001
From: Alexander Shaposhnikov <ashaposhnikov at google.com>
Date: Mon, 17 Jun 2024 21:56:59 +0000
Subject: [PATCH] [CompilerRT] Add numerical sanitizer
---
.../cmake/Modules/AllSupportedArchDefs.cmake | 1 +
compiler-rt/cmake/config-ix.cmake | 13 +-
.../include/sanitizer/nsan_interface.h | 75 ++
compiler-rt/lib/nsan/CMakeLists.txt | 56 ++
compiler-rt/lib/nsan/nsan.cc | 821 ++++++++++++++++++
compiler-rt/lib/nsan/nsan.h | 225 +++++
compiler-rt/lib/nsan/nsan.syms.extra | 2 +
compiler-rt/lib/nsan/nsan_flags.cc | 78 ++
compiler-rt/lib/nsan/nsan_flags.h | 35 +
compiler-rt/lib/nsan/nsan_flags.inc | 49 ++
compiler-rt/lib/nsan/nsan_interceptors.cc | 364 ++++++++
compiler-rt/lib/nsan/nsan_platform.h | 135 +++
compiler-rt/lib/nsan/nsan_stats.cc | 158 ++++
compiler-rt/lib/nsan/nsan_stats.h | 92 ++
compiler-rt/lib/nsan/nsan_suppressions.cc | 77 ++
compiler-rt/lib/nsan/nsan_suppressions.h | 28 +
compiler-rt/lib/nsan/tests/CMakeLists.txt | 53 ++
compiler-rt/lib/nsan/tests/NSanUnitTest.cpp | 67 ++
.../lib/nsan/tests/nsan_unit_test_main.cpp | 18 +
compiler-rt/test/nsan/CMakeLists.txt | 19 +
compiler-rt/test/nsan/Unit/lit.site.cfg.py.in | 10 +
compiler-rt/test/nsan/lit.cfg.py | 1 +
compiler-rt/test/nsan/lit.site.cfg.py.in | 14 +
23 files changed, 2390 insertions(+), 1 deletion(-)
create mode 100644 compiler-rt/include/sanitizer/nsan_interface.h
create mode 100644 compiler-rt/lib/nsan/CMakeLists.txt
create mode 100644 compiler-rt/lib/nsan/nsan.cc
create mode 100644 compiler-rt/lib/nsan/nsan.h
create mode 100644 compiler-rt/lib/nsan/nsan.syms.extra
create mode 100644 compiler-rt/lib/nsan/nsan_flags.cc
create mode 100644 compiler-rt/lib/nsan/nsan_flags.h
create mode 100644 compiler-rt/lib/nsan/nsan_flags.inc
create mode 100644 compiler-rt/lib/nsan/nsan_interceptors.cc
create mode 100644 compiler-rt/lib/nsan/nsan_platform.h
create mode 100644 compiler-rt/lib/nsan/nsan_stats.cc
create mode 100644 compiler-rt/lib/nsan/nsan_stats.h
create mode 100644 compiler-rt/lib/nsan/nsan_suppressions.cc
create mode 100644 compiler-rt/lib/nsan/nsan_suppressions.h
create mode 100644 compiler-rt/lib/nsan/tests/CMakeLists.txt
create mode 100644 compiler-rt/lib/nsan/tests/NSanUnitTest.cpp
create mode 100644 compiler-rt/lib/nsan/tests/nsan_unit_test_main.cpp
create mode 100644 compiler-rt/test/nsan/CMakeLists.txt
create mode 100644 compiler-rt/test/nsan/Unit/lit.site.cfg.py.in
create mode 100644 compiler-rt/test/nsan/lit.cfg.py
create mode 100644 compiler-rt/test/nsan/lit.site.cfg.py.in
diff --git a/compiler-rt/cmake/Modules/AllSupportedArchDefs.cmake b/compiler-rt/cmake/Modules/AllSupportedArchDefs.cmake
index 2fe06273a814c..ac4a71202384d 100644
--- a/compiler-rt/cmake/Modules/AllSupportedArchDefs.cmake
+++ b/compiler-rt/cmake/Modules/AllSupportedArchDefs.cmake
@@ -61,6 +61,7 @@ else()
endif()
set(ALL_MSAN_SUPPORTED_ARCH ${X86_64} ${MIPS64} ${ARM64} ${PPC64} ${S390X}
${LOONGARCH64})
+set(ALL_NSAN_SUPPORTED_ARCH ${X86_64})
set(ALL_HWASAN_SUPPORTED_ARCH ${X86_64} ${ARM64} ${RISCV64})
set(ALL_MEMPROF_SUPPORTED_ARCH ${X86_64})
set(ALL_PROFILE_SUPPORTED_ARCH ${X86} ${X86_64} ${ARM32} ${ARM64} ${PPC32} ${PPC64}
diff --git a/compiler-rt/cmake/config-ix.cmake b/compiler-rt/cmake/config-ix.cmake
index bddaa37579fd7..75e4d3677703a 100644
--- a/compiler-rt/cmake/config-ix.cmake
+++ b/compiler-rt/cmake/config-ix.cmake
@@ -609,6 +609,9 @@ if(APPLE)
list_intersect(MSAN_SUPPORTED_ARCH
ALL_MSAN_SUPPORTED_ARCH
SANITIZER_COMMON_SUPPORTED_ARCH)
+ list_intersect(NSAN_SUPPORTED_ARCH
+ ALL_NSAN_SUPPORTED_ARCH
+ SANITIZER_COMMON_SUPPORTED_ARCH)
list_intersect(HWASAN_SUPPORTED_ARCH
ALL_HWASAN_SUPPORTED_ARCH
SANITIZER_COMMON_SUPPORTED_ARCH)
@@ -678,6 +681,7 @@ else()
filter_available_targets(SHADOWCALLSTACK_SUPPORTED_ARCH
${ALL_SHADOWCALLSTACK_SUPPORTED_ARCH})
filter_available_targets(GWP_ASAN_SUPPORTED_ARCH ${ALL_GWP_ASAN_SUPPORTED_ARCH})
+ filter_available_targets(NSAN_SUPPORTED_ARCH ${ALL_NSAN_SUPPORTED_ARCH})
filter_available_targets(ORC_SUPPORTED_ARCH ${ALL_ORC_SUPPORTED_ARCH})
endif()
@@ -712,7 +716,7 @@ if(COMPILER_RT_SUPPORTED_ARCH)
endif()
message(STATUS "Compiler-RT supported architectures: ${COMPILER_RT_SUPPORTED_ARCH}")
-set(ALL_SANITIZERS asan;dfsan;msan;hwasan;tsan;safestack;cfi;scudo_standalone;ubsan_minimal;gwp_asan;asan_abi)
+set(ALL_SANITIZERS asan;dfsan;msan;hwasan;tsan;safestack;cfi;scudo_standalone;ubsan_minimal;gwp_asan;nsan;asan_abi)
set(COMPILER_RT_SANITIZERS_TO_BUILD all CACHE STRING
"sanitizers to build if supported on the target (all;${ALL_SANITIZERS})")
list_replace(COMPILER_RT_SANITIZERS_TO_BUILD all "${ALL_SANITIZERS}")
@@ -897,4 +901,11 @@ if (GWP_ASAN_SUPPORTED_ARCH AND
else()
set(COMPILER_RT_HAS_GWP_ASAN FALSE)
endif()
+
+if (COMPILER_RT_HAS_SANITIZER_COMMON AND NSAN_SUPPORTED_ARCH AND
+ OS_NAME MATCHES "Linux")
+ set(COMPILER_RT_HAS_NSAN TRUE)
+else()
+ set(COMPILER_RT_HAS_NSAN FALSE)
+endif()
pythonize_bool(COMPILER_RT_HAS_GWP_ASAN)
diff --git a/compiler-rt/include/sanitizer/nsan_interface.h b/compiler-rt/include/sanitizer/nsan_interface.h
new file mode 100644
index 0000000000000..057ca0473bb3c
--- /dev/null
+++ b/compiler-rt/include/sanitizer/nsan_interface.h
@@ -0,0 +1,75 @@
+//===-- sanitizer/nsan_interface.h ------------------------------*- C++ -*-===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+//
+// Public interface for nsan.
+//
+//===----------------------------------------------------------------------===//
+#ifndef SANITIZER_NSAN_INTERFACE_H
+#define SANITIZER_NSAN_INTERFACE_H
+
+#include <sanitizer/common_interface_defs.h>
+
+#ifdef __cplusplus
+extern "C" {
+#endif
+
+/// User-provided default option settings.
+///
+/// You can provide your own implementation of this function to return a string
+/// containing NSan runtime options (for example,
+/// <c>verbosity=1:halt_on_error=0</c>).
+///
+/// \returns Default options string.
+const char *__nsan_default_options(void);
+
+// Dumps nsan shadow data for a block of `size_bytes` bytes of application
+// memory at location `addr`.
+//
+// Each line contains application address, shadow types, then values.
+// Unknown types are shown as `__`, while known values are shown as
+// `f`, `d`, `l` for float, double, and long double respectively. Position is
+// shown as a single hex digit. The shadow value itself appears on the line that
+// contains the first byte of the value.
+// FIXME: Show both shadow and application value.
+//
+// Example: `__nsan_dump_shadow_mem(addr, 32, 8, 0)` might print:
+//
+// 0x0add7359: __ f0 f1 f2 f3 __ __ __ (42.000)
+// 0x0add7361: __ d1 d2 d3 d4 d5 d6 d7
+// 0x0add7369: d8 f0 f1 f2 f3 __ __ f2 (-1.000) (12.5)
+// 0x0add7371: f3 __ __ __ __ __ __ __
+//
+// This means that there is:
+// - a shadow double for the float at address 0x0add7360, with value 42;
+// - a shadow float128 for the double at address 0x0add7362, with value -1;
+// - a shadow double for the float at address 0x0add736a, with value 12.5;
+// There was also a shadow double for the float at address 0x0add736e, but bytes
+// f0 and f1 were overwritten by one or several stores, so that the shadow value
+// is no longer valid.
+// The argument `reserved` can be any value. Its true value is provided by the
+// instrumentation.
+void __nsan_dump_shadow_mem(const char *addr, size_t size_bytes,
+ size_t bytes_per_line, size_t reserved);
+
+// Explicitly dumps a value.
+// FIXME: vector versions ?
+void __nsan_dump_float(float value);
+void __nsan_dump_double(double value);
+void __nsan_dump_longdouble(long double value);
+
+// Explicitly checks a value.
+// FIXME: vector versions ?
+void __nsan_check_float(float value);
+void __nsan_check_double(double value);
+void __nsan_check_longdouble(long double value);
+
+#ifdef __cplusplus
+} // extern "C"
+#endif
+
+#endif // SANITIZER_NSAN_INTERFACE_H
diff --git a/compiler-rt/lib/nsan/CMakeLists.txt b/compiler-rt/lib/nsan/CMakeLists.txt
new file mode 100644
index 0000000000000..ae94c96d60727
--- /dev/null
+++ b/compiler-rt/lib/nsan/CMakeLists.txt
@@ -0,0 +1,56 @@
+add_compiler_rt_component(nsan)
+
+include_directories(..)
+
+set(NSAN_SOURCES
+ nsan.cc
+ nsan_flags.cc
+ nsan_interceptors.cc
+ nsan_stats.cc
+ nsan_suppressions.cc
+)
+
+set(NSAN_HEADERS
+ nsan.h
+ nsan_flags.h
+ nsan_flags.inc
+ nsan_platform.h
+ nsan_stats.h
+ nsan_suppressions.h
+)
+
+append_list_if(COMPILER_RT_HAS_FPIC_FLAG -fPIC NSAN_CFLAGS)
+
+set(NSAN_DYNAMIC_LINK_FLAGS ${SANITIZER_COMMON_LINK_FLAGS})
+
+set(NSAN_CFLAGS ${SANITIZER_COMMON_CFLAGS})
+
+if (COMPILER_RT_HAS_NSAN)
+ foreach(arch ${NSAN_SUPPORTED_ARCH})
+ add_compiler_rt_runtime(
+ clang_rt.nsan
+ STATIC
+ ARCHS ${arch}
+ SOURCES ${NSAN_SOURCES}
+ $<TARGET_OBJECTS:RTInterception.${arch}>
+ $<TARGET_OBJECTS:RTSanitizerCommon.${arch}>
+ $<TARGET_OBJECTS:RTSanitizerCommonLibc.${arch}>
+ $<TARGET_OBJECTS:RTSanitizerCommonCoverage.${arch}>
+ $<TARGET_OBJECTS:RTSanitizerCommonSymbolizer.${arch}>
+ $<TARGET_OBJECTS:RTUbsan.${arch}>
+ ADDITIONAL_HEADERS ${NSAN_HEADERS}
+ CFLAGS ${NSAN_CFLAGS}
+ PARENT_TARGET nsan
+ )
+ endforeach()
+
+ add_compiler_rt_object_libraries(RTNsan
+ ARCHS ${NSAN_SUPPORTED_ARCH}
+ SOURCES ${NSAN_SOURCES}
+ ADDITIONAL_HEADERS ${NSAN_HEADERS}
+ CFLAGS ${NSAN_CFLAGS})
+endif()
+
+if(COMPILER_RT_INCLUDE_TESTS)
+ add_subdirectory(tests)
+endif()
diff --git a/compiler-rt/lib/nsan/nsan.cc b/compiler-rt/lib/nsan/nsan.cc
new file mode 100644
index 0000000000000..d0d29ddfba0e3
--- /dev/null
+++ b/compiler-rt/lib/nsan/nsan.cc
@@ -0,0 +1,821 @@
+//===-- nsan.cc -----------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+//
+// NumericalStabilitySanitizer runtime.
+//
+// This implements:
+// - The public nsan interface (include/sanitizer/nsan_interface.h).
+// - The private nsan interface (./nsan.h).
+// - The internal instrumentation interface. These are function emitted by the
+// instrumentation pass:
+// * __nsan_get_shadow_ptr_for_{float,double,longdouble}_load
+// These return the shadow memory pointer for loading the shadow value,
+// after checking that the types are consistent. If the types are not
+// consistent, returns nullptr.
+// * __nsan_get_shadow_ptr_for_{float,double,longdouble}_store
+// Sets the shadow types appropriately and returns the shadow memory
+// pointer for storing the shadow value.
+// * __nsan_internal_check_{float,double,long double}_{f,d,l} checks the
+// accuracy of a value against its shadow and emits a warning depending
+// on the runtime configuration. The middle part indicates the type of
+// the application value, the suffix (f,d,l) indicates the type of the
+// shadow, and depends on the instrumentation configuration.
+// * __nsan_fcmp_fail_* emits a warning for an fcmp instruction whose
+// corresponding shadow fcmp result differs.
+//
+//===----------------------------------------------------------------------===//
+
+#include <assert.h>
+#include <math.h>
+#include <stdint.h>
+#include <stdio.h>
+#include <stdlib.h>
+
+#include "sanitizer_common/sanitizer_atomic.h"
+#include "sanitizer_common/sanitizer_common.h"
+#include "sanitizer_common/sanitizer_libc.h"
+#include "sanitizer_common/sanitizer_report_decorator.h"
+#include "sanitizer_common/sanitizer_stacktrace.h"
+#include "sanitizer_common/sanitizer_symbolizer.h"
+
+#include "nsan/nsan.h"
+#include "nsan/nsan_flags.h"
+#include "nsan/nsan_stats.h"
+#include "nsan/nsan_suppressions.h"
+
+using namespace __sanitizer;
+using namespace __nsan;
+
+static constexpr const int kMaxVectorWidth = 8;
+
+// When copying application memory, we also copy its shadow and shadow type.
+// FIXME: We could provide fixed-size versions that would nicely
+// vectorize for known sizes.
+extern "C" SANITIZER_INTERFACE_ATTRIBUTE void
+__nsan_copy_values(const u8 *daddr, const u8 *saddr, uptr size) {
+ internal_memmove((void *)getShadowTypeAddrFor(daddr),
+ getShadowTypeAddrFor(saddr), size);
+ internal_memmove((void *)getShadowAddrFor(daddr), getShadowAddrFor(saddr),
+ size * kShadowScale);
+}
+
+// FIXME: We could provide fixed-size versions that would nicely
+// vectorize for known sizes.
+extern "C" SANITIZER_INTERFACE_ATTRIBUTE void
+__nsan_set_value_unknown(const u8 *addr, uptr size) {
+ internal_memset((void *)getShadowTypeAddrFor(addr), 0, size);
+}
+
+namespace __nsan {
+
+const char *FTInfo<float>::kCppTypeName = "float";
+const char *FTInfo<double>::kCppTypeName = "double";
+const char *FTInfo<long double>::kCppTypeName = "long double";
+const char *FTInfo<__float128>::kCppTypeName = "__float128";
+
+const char FTInfo<float>::kTypePattern[sizeof(float)];
+const char FTInfo<double>::kTypePattern[sizeof(double)];
+const char FTInfo<long double>::kTypePattern[sizeof(long double)];
+
+// Helper for __nsan_dump_shadow_mem: Reads the value at address `Ptr`,
+// identified by its type id.
+template <typename ShadowFT> __float128 readShadowInternal(const u8 *Ptr) {
+ ShadowFT Shadow;
+ __builtin_memcpy(&Shadow, Ptr, sizeof(Shadow));
+ return Shadow;
+}
+
+__float128 readShadow(const u8 *Ptr, const char ShadowTypeId) {
+ switch (ShadowTypeId) {
+ case 'd':
+ return readShadowInternal<double>(Ptr);
+ case 'l':
+ return readShadowInternal<long double>(Ptr);
+ case 'q':
+ return readShadowInternal<__float128>(Ptr);
+ default:
+ return 0.0;
+ }
+}
+
+class Decorator : public __sanitizer::SanitizerCommonDecorator {
+public:
+ Decorator() : SanitizerCommonDecorator() {}
+ const char *Warning() { return Red(); }
+ const char *Name() { return Green(); }
+ const char *End() { return Default(); }
+};
+
+namespace {
+
+// Workaround for the fact that Printf() does not support floats.
+struct PrintBuffer {
+ char Buffer[64];
+};
+template <typename FT> struct FTPrinter {};
+
+template <> struct FTPrinter<double> {
+ static PrintBuffer dec(double Value) {
+ PrintBuffer Result;
+ snprintf(Result.Buffer, sizeof(Result.Buffer) - 1, "%.20f", Value);
+ return Result;
+ }
+ static PrintBuffer hex(double Value) {
+ PrintBuffer Result;
+ snprintf(Result.Buffer, sizeof(Result.Buffer) - 1, "%.20a", Value);
+ return Result;
+ }
+};
+
+template <> struct FTPrinter<float> : FTPrinter<double> {};
+
+template <> struct FTPrinter<long double> {
+ static PrintBuffer dec(long double Value) {
+ PrintBuffer Result;
+ snprintf(Result.Buffer, sizeof(Result.Buffer) - 1, "%.20Lf", Value);
+ return Result;
+ }
+ static PrintBuffer hex(long double Value) {
+ PrintBuffer Result;
+ snprintf(Result.Buffer, sizeof(Result.Buffer) - 1, "%.20La", Value);
+ return Result;
+ }
+};
+
+// FIXME: print with full precision.
+template <> struct FTPrinter<__float128> : FTPrinter<long double> {};
+
+// This is a template so that there are no implicit conversions.
+template <typename FT> inline FT ftAbs(FT V);
+
+template <> inline long double ftAbs(long double V) { return fabsl(V); }
+template <> inline double ftAbs(double V) { return fabs(V); }
+
+// We don't care about nans.
+// std::abs(__float128) code is suboptimal and generates a function call to
+// __getf2().
+template <typename FT> inline FT ftAbs(FT V) { return V >= FT{0} ? V : -V; }
+
+template <typename FT1, typename FT2, bool Enable> struct LargestFTImpl {
+ using type = FT2;
+};
+
+template <typename FT1, typename FT2> struct LargestFTImpl<FT1, FT2, true> {
+ using type = FT1;
+};
+
+template <typename FT1, typename FT2>
+using LargestFT =
+ typename LargestFTImpl<FT1, FT2, (sizeof(FT1) > sizeof(FT2))>::type;
+
+template <typename T> T max(T a, T b) { return a < b ? b : a; }
+
+} // end anonymous namespace
+
+} // end namespace __nsan
+
+void __sanitizer::BufferedStackTrace::UnwindImpl(uptr pc, uptr bp,
+ void *context,
+ bool request_fast,
+ u32 max_depth) {
+ using namespace __nsan;
+ return Unwind(max_depth, pc, bp, context, 0, 0, false);
+}
+
+extern "C" SANITIZER_INTERFACE_ATTRIBUTE void __nsan_print_accumulated_stats() {
+ if (nsan_stats)
+ nsan_stats->print();
+}
+
+static void nsanAtexit() {
+ Printf("Numerical Sanitizer exit stats:\n");
+ __nsan_print_accumulated_stats();
+ nsan_stats = nullptr;
+}
+
+// The next three functions return a pointer for storing a shadow value for `n`
+// values, after setting the shadow types. We return the pointer instead of
+// storing ourselves because it avoids having to rely on the calling convention
+// around long double being the same for nsan and the target application.
+// We have to have 3 versions because we need to know which type we are storing
+// since we are setting the type shadow memory.
+template <typename FT> static u8 *getShadowPtrForStore(u8 *StoreAddr, uptr N) {
+ unsigned char *ShadowType = getShadowTypeAddrFor(StoreAddr);
+ for (uptr I = 0; I < N; ++I) {
+ __builtin_memcpy(ShadowType + I * sizeof(FT), FTInfo<FT>::kTypePattern,
+ sizeof(FTInfo<FT>::kTypePattern));
+ }
+ return getShadowAddrFor(StoreAddr);
+}
+
+extern "C" SANITIZER_INTERFACE_ATTRIBUTE u8 *
+__nsan_get_shadow_ptr_for_float_store(u8 *store_addr, uptr n) {
+ return getShadowPtrForStore<float>(store_addr, n);
+}
+
+extern "C" SANITIZER_INTERFACE_ATTRIBUTE u8 *
+__nsan_get_shadow_ptr_for_double_store(u8 *store_addr, uptr n) {
+ return getShadowPtrForStore<double>(store_addr, n);
+}
+
+extern "C" SANITIZER_INTERFACE_ATTRIBUTE u8 *
+__nsan_get_shadow_ptr_for_longdouble_store(u8 *store_addr, uptr n) {
+ return getShadowPtrForStore<long double>(store_addr, n);
+}
+
+template <typename FT> static bool isValidShadowType(const u8 *ShadowType) {
+ return __builtin_memcmp(ShadowType, FTInfo<FT>::kTypePattern, sizeof(FT)) ==
+ 0;
+}
+
+template <int kSize, typename T> static bool isZero(const T *Ptr) {
+ constexpr const char kZeros[kSize] = {}; // Zero initialized.
+ return __builtin_memcmp(Ptr, kZeros, kSize) == 0;
+}
+
+template <typename FT> static bool isUnknownShadowType(const u8 *ShadowType) {
+ return isZero<sizeof(FTInfo<FT>::kTypePattern)>(ShadowType);
+}
+
+// The three folowing functions check that the address stores a complete
+// shadow value of the given type and return a pointer for loading.
+// They return nullptr if the type of the value is unknown or incomplete.
+template <typename FT>
+static const u8 *getShadowPtrForLoad(const u8 *LoadAddr, uptr N) {
+ const u8 *const ShadowType = getShadowTypeAddrFor(LoadAddr);
+ for (uptr I = 0; I < N; ++I) {
+ if (!isValidShadowType<FT>(ShadowType + I * sizeof(FT))) {
+ // If loadtracking stats are enabled, log loads with invalid types
+ // (tampered with through type punning).
+ if (flags().enable_loadtracking_stats) {
+ if (isUnknownShadowType<FT>(ShadowType + I * sizeof(FT))) {
+ // Warn only if the value is non-zero. Zero is special because
+ // applications typically initialize large buffers to zero in an
+ // untyped way.
+ if (!isZero<sizeof(FT)>(LoadAddr)) {
+ GET_CALLER_PC_BP;
+ nsan_stats->addUnknownLoadTrackingEvent(pc, bp);
+ }
+ } else {
+ GET_CALLER_PC_BP;
+ nsan_stats->addInvalidLoadTrackingEvent(pc, bp);
+ }
+ }
+ return nullptr;
+ }
+ }
+ return getShadowAddrFor(LoadAddr);
+}
+
+extern "C" SANITIZER_INTERFACE_ATTRIBUTE const u8 *
+__nsan_get_shadow_ptr_for_float_load(const u8 *load_addr, uptr n) {
+ return getShadowPtrForLoad<float>(load_addr, n);
+}
+
+extern "C" SANITIZER_INTERFACE_ATTRIBUTE const u8 *
+__nsan_get_shadow_ptr_for_double_load(const u8 *load_addr, uptr n) {
+ return getShadowPtrForLoad<double>(load_addr, n);
+}
+
+extern "C" SANITIZER_INTERFACE_ATTRIBUTE const u8 *
+__nsan_get_shadow_ptr_for_longdouble_load(const u8 *load_addr, uptr n) {
+ return getShadowPtrForLoad<long double>(load_addr, n);
+}
+
+// Returns the raw shadow pointer. The returned pointer should be considered
+// opaque.
+extern "C" SANITIZER_INTERFACE_ATTRIBUTE u8 *
+__nsan_internal_get_raw_shadow_ptr(const u8 *addr) {
+ return getShadowAddrFor(const_cast<u8 *>(addr));
+}
+
+// Returns the raw shadow type pointer. The returned pointer should be
+// considered opaque.
+extern "C" SANITIZER_INTERFACE_ATTRIBUTE u8 *
+__nsan_internal_get_raw_shadow_type_ptr(const u8 *addr) {
+ return reinterpret_cast<u8 *>(getShadowTypeAddrFor(const_cast<u8 *>(addr)));
+}
+
+static ValueType getValueType(u8 c) { return static_cast<ValueType>(c & 0x3); }
+
+static int getValuePos(u8 c) { return c >> kValueSizeSizeBits; }
+
+// Checks the consistency of the value types at the given type pointer.
+// If the value is inconsistent, returns ValueType::kUnknown. Else, return the
+// consistent type.
+template <typename FT> static bool checkValueConsistency(const u8 *ShadowType) {
+ const int Pos = getValuePos(*ShadowType);
+ // Check that all bytes from the start of the value are ordered.
+ for (uptr I = 0; I < sizeof(FT); ++I) {
+ const u8 T = *(ShadowType - Pos + I);
+ if (!(getValueType(T) == FTInfo<FT>::kValueType && getValuePos(T) == I)) {
+ return false;
+ }
+ }
+ return true;
+}
+
+// The instrumentation automatically appends `shadow_value_type_ids`, see
+// maybeAddSuffixForNsanInterface.
+extern "C" SANITIZER_INTERFACE_ATTRIBUTE void
+__nsan_dump_shadow_mem(const u8 *addr, size_t size_bytes, size_t bytes_per_line,
+ size_t shadow_value_type_ids) {
+ const u8 *const ShadowType = getShadowTypeAddrFor(addr);
+ const u8 *const Shadow = getShadowAddrFor(addr);
+
+ constexpr int kMaxNumDecodedValues = 16;
+ __float128 DecodedValues[kMaxNumDecodedValues];
+ int NumDecodedValues = 0;
+ if (bytes_per_line > 4 * kMaxNumDecodedValues) {
+ bytes_per_line = 4 * kMaxNumDecodedValues;
+ }
+
+ // We keep track of the current type and position as we go.
+ ValueType LastValueTy = kUnknownValueType;
+ int LastPos = -1;
+ size_t Offset = 0;
+ for (size_t R = 0; R < (size_bytes + bytes_per_line - 1) / bytes_per_line;
+ ++R) {
+ printf("%p: ", (void *)(addr + R * bytes_per_line));
+ for (size_t C = 0; C < bytes_per_line && Offset < size_bytes; ++C) {
+ const ValueType ValueTy = getValueType(ShadowType[Offset]);
+ const int pos = getValuePos(ShadowType[Offset]);
+ if (ValueTy == LastValueTy && pos == LastPos + 1) {
+ ++LastPos;
+ } else {
+ LastValueTy = ValueTy;
+ LastPos = pos == 0 ? 0 : -1;
+ }
+
+ switch (ValueTy) {
+ case kUnknownValueType:
+ printf("__ ");
+ break;
+ case kFloatValueType:
+ printf("f%x ", pos);
+ if (LastPos == sizeof(float) - 1) {
+ DecodedValues[NumDecodedValues] =
+ readShadow(Shadow + kShadowScale * (Offset + 1 - sizeof(float)),
+ static_cast<char>(shadow_value_type_ids & 0xff));
+ ++NumDecodedValues;
+ }
+ break;
+ case kDoubleValueType:
+ printf("d%x ", pos);
+ if (LastPos == sizeof(double) - 1) {
+ DecodedValues[NumDecodedValues] = readShadow(
+ Shadow + kShadowScale * (Offset + 1 - sizeof(double)),
+ static_cast<char>((shadow_value_type_ids >> 8) & 0xff));
+ ++NumDecodedValues;
+ }
+ break;
+ case kFp80ValueType:
+ printf("l%x ", pos);
+ if (LastPos == sizeof(long double) - 1) {
+ DecodedValues[NumDecodedValues] = readShadow(
+ Shadow + kShadowScale * (Offset + 1 - sizeof(long double)),
+ static_cast<char>((shadow_value_type_ids >> 16) & 0xff));
+ ++NumDecodedValues;
+ }
+ break;
+ }
+ ++Offset;
+ }
+ for (int I = 0; I < NumDecodedValues; ++I) {
+ printf(" (%s)", FTPrinter<__float128>::dec(DecodedValues[I]).Buffer);
+ }
+ NumDecodedValues = 0;
+ printf("\n");
+ }
+}
+
+SANITIZER_INTERFACE_ATTRIBUTE
+ALIGNED(16)
+THREADLOCAL
+uptr __nsan_shadow_ret_tag = 0;
+
+SANITIZER_INTERFACE_ATTRIBUTE
+ALIGNED(16)
+THREADLOCAL
+char __nsan_shadow_ret_ptr[kMaxVectorWidth * sizeof(__float128)];
+
+SANITIZER_INTERFACE_ATTRIBUTE
+ALIGNED(16)
+THREADLOCAL
+uptr __nsan_shadow_args_tag = 0;
+
+// Maximum number of args. This should be enough for anyone (tm). An alternate
+// scheme is to have the generated code create an alloca and make
+// __nsan_shadow_args_ptr point ot the alloca.
+constexpr const int kMaxNumArgs = 128;
+SANITIZER_INTERFACE_ATTRIBUTE
+ALIGNED(16)
+THREADLOCAL
+char __nsan_shadow_args_ptr[kMaxVectorWidth * kMaxNumArgs * sizeof(__float128)];
+
+enum ContinuationType { // Keep in sync with instrumentation pass.
+ kContinueWithShadow = 0,
+ kResumeFromValue = 1,
+};
+
+// Checks the consistency between application and shadow value. Returns true
+// when the instrumented code should resume computations from the original value
+// rather than the shadow value. This prevents one error to propagate to all
+// subsequent operations. This behaviour is tunable with flags.
+template <typename FT, typename ShadowFT>
+int32_t checkFT(const FT Value, ShadowFT Shadow, CheckTypeT CheckType,
+ uptr CheckArg) {
+ // We do all comparisons in the InternalFT domain, which is the largest FT
+ // type.
+ using InternalFT = LargestFT<FT, ShadowFT>;
+ const InternalFT CheckValue = Value;
+ const InternalFT CheckShadow = Shadow;
+
+ // See this article for an interesting discussion of how to compare floats:
+ // https://randomascii.wordpress.com/2012/02/25/comparing-floating-point-numbers-2012-edition/
+ static constexpr const FT Eps = FTInfo<FT>::kEpsilon;
+
+ const InternalFT AbsErr = ftAbs(CheckValue - CheckShadow);
+
+ if (flags().enable_check_stats) {
+ GET_CALLER_PC_BP;
+ // We are re-computing `Largest` here because this is a cold branch, and we
+ // want to avoid having to move the computation of `Largest` before the
+ // absolute value check when this branch is not taken.
+ const InternalFT Largest = max(ftAbs(CheckValue), ftAbs(CheckShadow));
+ nsan_stats->addCheck(CheckType, pc, bp, AbsErr / Largest);
+ }
+
+ // Note: writing the comparison that way ensures that when `AbsErr` is Nan
+ // (value and shadow are inf or -inf), we pass the test.
+ if (!(AbsErr >= flags().cached_absolute_error_threshold))
+ return kContinueWithShadow;
+
+ const InternalFT Largest = max(ftAbs(CheckValue), ftAbs(CheckShadow));
+ if (AbsErr * (1ull << flags().log2_max_relative_error) <= Largest)
+ return kContinueWithShadow; // No problem here.
+
+ if (!flags().disable_warnings) {
+ GET_CALLER_PC_BP;
+ BufferedStackTrace stack;
+ stack.Unwind(pc, bp, nullptr, false);
+ if (GetSuppressionForStack(&stack, CheckKind::Consistency)) {
+ // FIXME: optionally print.
+ return flags().resume_after_suppression ? kResumeFromValue
+ : kContinueWithShadow;
+ }
+
+ Decorator D;
+ Printf("%s", D.Warning());
+ // Printf does not support float formatting.
+ char RelErrBuf[64] = "inf";
+ if (Largest > Eps) {
+ snprintf(RelErrBuf, sizeof(RelErrBuf) - 1, "%.20Lf%% (2^%.0Lf epsilons)",
+ static_cast<long double>(100.0 * AbsErr / Largest),
+ log2l(static_cast<long double>(AbsErr / Largest / Eps)));
+ }
+ char UlpErrBuf[128] = "";
+ const double ShadowUlpDiff = getULPDiff(CheckValue, CheckShadow);
+ if (ShadowUlpDiff != kMaxULPDiff) {
+ // This is the ULP diff in the internal domain. The user actually cares
+ // about that in the original domain.
+ const double UlpDiff =
+ ShadowUlpDiff / (u64{1} << (FTInfo<InternalFT>::kMantissaBits -
+ FTInfo<FT>::kMantissaBits));
+ snprintf(UlpErrBuf, sizeof(UlpErrBuf) - 1,
+ "(%.0f ULPs == %.1f digits == %.1f bits)", UlpDiff,
+ log10(UlpDiff), log2(UlpDiff));
+ }
+ Printf("WARNING: NumericalStabilitySanitizer: inconsistent shadow results");
+ switch (CheckType) {
+ case CheckTypeT::kUnknown:
+ case CheckTypeT::kFcmp:
+ case CheckTypeT::kMaxCheckType:
+ break;
+ case CheckTypeT::kRet:
+ Printf(" while checking return value");
+ break;
+ case CheckTypeT::kArg:
+ Printf(" while checking call argument #%d", static_cast<int>(CheckArg));
+ break;
+ case CheckTypeT::kLoad:
+ Printf(
+ " while checking load from address 0x%lx. This is due to incorrect "
+ "shadow memory tracking, typically due to uninstrumented code "
+ "writing to memory.",
+ CheckArg);
+ break;
+ case CheckTypeT::kStore:
+ Printf(" while checking store to address 0x%lx", CheckArg);
+ break;
+ case CheckTypeT::kInsert:
+ Printf(" while checking vector insert");
+ break;
+ case CheckTypeT::kUser:
+ Printf(" in user-initiated check");
+ break;
+ }
+ using ValuePrinter = FTPrinter<FT>;
+ using ShadowPrinter = FTPrinter<ShadowFT>;
+ Printf("\n"
+ "%-12s precision (native): dec: %s hex: %s\n"
+ "%-12s precision (shadow): dec: %s hex: %s\n"
+ "shadow truncated to %-12s: dec: %s hex: %s\n"
+ "Relative error: %s\n"
+ "Absolute error: %s\n"
+ "%s\n",
+ FTInfo<FT>::kCppTypeName, ValuePrinter::dec(Value).Buffer,
+ ValuePrinter::hex(Value).Buffer, FTInfo<ShadowFT>::kCppTypeName,
+ ShadowPrinter::dec(Shadow).Buffer, ShadowPrinter::hex(Shadow).Buffer,
+ FTInfo<FT>::kCppTypeName, ValuePrinter::dec(Shadow).Buffer,
+ ValuePrinter::hex(Shadow).Buffer, RelErrBuf,
+ ValuePrinter::hex(AbsErr).Buffer, UlpErrBuf);
+ stack.Print();
+ }
+
+ if (flags().enable_warning_stats) {
+ GET_CALLER_PC_BP;
+ nsan_stats->addWarning(CheckType, pc, bp, AbsErr / Largest);
+ }
+
+ if (flags().halt_on_error) {
+ Printf("Exiting\n");
+ Die();
+ }
+ return flags().resume_after_warning ? kResumeFromValue : kContinueWithShadow;
+}
+
+extern "C" SANITIZER_INTERFACE_ATTRIBUTE int32_t __nsan_internal_check_float_d(
+ float value, double shadow, int32_t check_type, uptr check_arg) {
+ return checkFT(value, shadow, static_cast<CheckTypeT>(check_type), check_arg);
+}
+
+extern "C" SANITIZER_INTERFACE_ATTRIBUTE int32_t __nsan_internal_check_double_l(
+ double value, long double shadow, int32_t check_type, uptr check_arg) {
+ return checkFT(value, shadow, static_cast<CheckTypeT>(check_type), check_arg);
+}
+
+extern "C" SANITIZER_INTERFACE_ATTRIBUTE int32_t __nsan_internal_check_double_q(
+ double value, __float128 shadow, int32_t check_type, uptr check_arg) {
+ return checkFT(value, shadow, static_cast<CheckTypeT>(check_type), check_arg);
+}
+
+extern "C" SANITIZER_INTERFACE_ATTRIBUTE int32_t
+__nsan_internal_check_longdouble_q(long double value, __float128 shadow,
+ int32_t check_type, uptr check_arg) {
+ return checkFT(value, shadow, static_cast<CheckTypeT>(check_type), check_arg);
+}
+
+static const char *getTruthValueName(bool v) { return v ? "true" : "false"; }
+
+// This uses the same values as CmpInst::Predicate.
+static const char *getPredicateName(int v) {
+ switch (v) {
+ case 0:
+ return "(false)";
+ case 1:
+ return "==";
+ case 2:
+ return ">";
+ case 3:
+ return ">=";
+ case 4:
+ return "<";
+ case 5:
+ return "<=";
+ case 6:
+ return "!=";
+ case 7:
+ return "(ordered)";
+ case 8:
+ return "(unordered)";
+ case 9:
+ return "==";
+ case 10:
+ return ">";
+ case 11:
+ return ">=";
+ case 12:
+ return "<";
+ case 13:
+ return "<=";
+ case 14:
+ return "!=";
+ case 15:
+ return "(true)";
+ }
+ return "??";
+}
+
+template <typename FT, typename ShadowFT>
+void fCmpFailFT(const FT Lhs, const FT Rhs, ShadowFT LhsShadow,
+ ShadowFT RhsShadow, int Predicate, bool Result,
+ bool ShadowResult) {
+ if (Result == ShadowResult) {
+ // When a vector comparison fails, we fail each element of the comparison
+ // to simplify instrumented code. Skip elements where the shadow comparison
+ // gave the same result as the original one.
+ return;
+ }
+
+ GET_CALLER_PC_BP;
+ BufferedStackTrace Stack;
+ Stack.Unwind(pc, bp, nullptr, false);
+
+ if (GetSuppressionForStack(&Stack, CheckKind::Fcmp)) {
+ // FIXME: optionally print.
+ return;
+ }
+
+ if (flags().enable_warning_stats) {
+ nsan_stats->addWarning(CheckTypeT::kFcmp, pc, bp, 0.0);
+ }
+
+ if (flags().disable_warnings) {
+ return;
+ }
+
+ // FIXME: ideally we would print the shadow value as FP128. Right now because
+ // we truncate to long double we can sometimes see stuff like:
+ // shadow <value> == <value> (false)
+ using ValuePrinter = FTPrinter<FT>;
+ using ShadowPrinter = FTPrinter<ShadowFT>;
+ Decorator D;
+ const char *const PredicateName = getPredicateName(Predicate);
+ Printf("%s", D.Warning());
+ Printf("WARNING: NumericalStabilitySanitizer: floating-point comparison "
+ "results depend on precision\n"
+ "%-12s precision dec (native): %s %s %s (%s)\n"
+ "%-12s precision dec (shadow): %s %s %s (%s)\n"
+ "%-12s precision hex (native): %s %s %s (%s)\n"
+ "%-12s precision hex (shadow): %s %s %s (%s)\n"
+ "%s",
+ // Native, decimal.
+ FTInfo<FT>::kCppTypeName, ValuePrinter::dec(Lhs).Buffer, PredicateName,
+ ValuePrinter::dec(Rhs).Buffer, getTruthValueName(Result),
+ // Shadow, decimal
+ FTInfo<ShadowFT>::kCppTypeName, ShadowPrinter::dec(LhsShadow).Buffer,
+ PredicateName, ShadowPrinter::dec(RhsShadow).Buffer,
+ getTruthValueName(ShadowResult),
+ // Native, hex.
+ FTInfo<FT>::kCppTypeName, ValuePrinter::hex(Lhs).Buffer, PredicateName,
+ ValuePrinter::hex(Rhs).Buffer, getTruthValueName(Result),
+ // Shadow, hex
+ FTInfo<ShadowFT>::kCppTypeName, ShadowPrinter::hex(LhsShadow).Buffer,
+ PredicateName, ShadowPrinter::hex(RhsShadow).Buffer,
+ getTruthValueName(ShadowResult), D.End());
+ Printf("%s", D.Default());
+ Stack.Print();
+ if (flags().halt_on_error) {
+ Printf("Exiting\n");
+ Die();
+ }
+}
+
+extern "C" SANITIZER_INTERFACE_ATTRIBUTE void
+__nsan_fcmp_fail_float_d(float lhs, float rhs, double lhs_shadow,
+ double rhs_shadow, int predicate, bool result,
+ bool shadow_result) {
+ fCmpFailFT(lhs, rhs, lhs_shadow, rhs_shadow, predicate, result,
+ shadow_result);
+}
+
+extern "C" SANITIZER_INTERFACE_ATTRIBUTE void
+__nsan_fcmp_fail_double_q(double lhs, double rhs, __float128 lhs_shadow,
+ __float128 rhs_shadow, int predicate, bool result,
+ bool shadow_result) {
+ fCmpFailFT(lhs, rhs, lhs_shadow, rhs_shadow, predicate, result,
+ shadow_result);
+}
+
+extern "C" SANITIZER_INTERFACE_ATTRIBUTE void
+__nsan_fcmp_fail_double_l(double lhs, double rhs, long double lhs_shadow,
+ long double rhs_shadow, int predicate, bool result,
+ bool shadow_result) {
+ fCmpFailFT(lhs, rhs, lhs_shadow, rhs_shadow, predicate, result,
+ shadow_result);
+}
+
+extern "C" SANITIZER_INTERFACE_ATTRIBUTE void
+__nsan_fcmp_fail_longdouble_q(long double lhs, long double rhs,
+ __float128 lhs_shadow, __float128 rhs_shadow,
+ int predicate, bool result, bool shadow_result) {
+ fCmpFailFT(lhs, rhs, lhs_shadow, rhs_shadow, predicate, result,
+ shadow_result);
+}
+
+template <typename FT> void checkFTFromShadowStack(const FT Value) {
+ // Get the shadow 2FT value from the shadow stack. Note that
+ // __nsan_check_{float,double,long double} is a function like any other, so
+ // the instrumentation will have placed the shadow value on the shadow stack.
+ using ShadowFT = typename FTInfo<FT>::shadow_type;
+ ShadowFT Shadow;
+ __builtin_memcpy(&Shadow, __nsan_shadow_args_ptr, sizeof(ShadowFT));
+ checkFT(Value, Shadow, CheckTypeT::kUser, 0);
+}
+
+// FIXME: Add suffixes and let the instrumentation pass automatically add
+// suffixes.
+extern "C" SANITIZER_INTERFACE_ATTRIBUTE void __nsan_check_float(float Value) {
+ assert(__nsan_shadow_args_tag == (uptr)&__nsan_check_float &&
+ "__nsan_check_float called from non-instrumented function");
+ checkFTFromShadowStack(Value);
+}
+
+extern "C" SANITIZER_INTERFACE_ATTRIBUTE void
+__nsan_check_double(double Value) {
+ assert(__nsan_shadow_args_tag == (uptr)&__nsan_check_double &&
+ "__nsan_check_double called from non-instrumented function");
+ checkFTFromShadowStack(Value);
+}
+
+extern "C" SANITIZER_INTERFACE_ATTRIBUTE void
+__nsan_check_longdouble(long double Value) {
+ assert(__nsan_shadow_args_tag == (uptr)&__nsan_check_longdouble &&
+ "__nsan_check_longdouble called from non-instrumented function");
+ checkFTFromShadowStack(Value);
+}
+
+template <typename FT> static void dumpFTFromShadowStack(const FT Value) {
+ // Get the shadow 2FT value from the shadow stack. Note that
+ // __nsan_dump_{float,double,long double} is a function like any other, so
+ // the instrumentation will have placed the shadow value on the shadow stack.
+ using ShadowFT = typename FTInfo<FT>::shadow_type;
+ ShadowFT Shadow;
+ __builtin_memcpy(&Shadow, __nsan_shadow_args_ptr, sizeof(ShadowFT));
+ using ValuePrinter = FTPrinter<FT>;
+ using ShadowPrinter = FTPrinter<typename FTInfo<FT>::shadow_type>;
+ printf("value dec:%s hex:%s\n"
+ "shadow dec:%s hex:%s\n",
+ ValuePrinter::dec(Value).Buffer, ValuePrinter::hex(Value).Buffer,
+ ShadowPrinter::dec(Shadow).Buffer, ShadowPrinter::hex(Shadow).Buffer);
+}
+
+extern "C" SANITIZER_INTERFACE_ATTRIBUTE void __nsan_dump_float(float Value) {
+ assert(__nsan_shadow_args_tag == (uptr)&__nsan_dump_float &&
+ "__nsan_dump_float called from non-instrumented function");
+ dumpFTFromShadowStack(Value);
+}
+
+extern "C" SANITIZER_INTERFACE_ATTRIBUTE void __nsan_dump_double(double Value) {
+ assert(__nsan_shadow_args_tag == (uptr)&__nsan_dump_double &&
+ "__nsan_dump_double called from non-instrumented function");
+ dumpFTFromShadowStack(Value);
+}
+
+extern "C" SANITIZER_INTERFACE_ATTRIBUTE void
+__nsan_dump_longdouble(long double Value) {
+ assert(__nsan_shadow_args_tag == (uptr)&__nsan_dump_longdouble &&
+ "__nsan_dump_longdouble called from non-instrumented function");
+ dumpFTFromShadowStack(Value);
+}
+
+extern "C" SANITIZER_INTERFACE_ATTRIBUTE void __nsan_dump_shadow_ret() {
+ printf("ret tag: %lx\n", __nsan_shadow_ret_tag);
+ double V;
+ __builtin_memcpy(&V, __nsan_shadow_ret_ptr, sizeof(double));
+ printf("double Value: %f\n", V);
+ // FIXME: float128 value.
+}
+
+extern "C" SANITIZER_INTERFACE_ATTRIBUTE void __nsan_dump_shadow_args() {
+ printf("args tag: %lx\n", __nsan_shadow_args_tag);
+}
+
+namespace __nsan {
+bool NsanInitialized = false;
+bool NsanInitIsRunning;
+} // end namespace __nsan
+
+extern "C" SANITIZER_INTERFACE_ATTRIBUTE void __nsan_init() {
+ CHECK(!NsanInitIsRunning);
+ if (NsanInitialized)
+ return;
+ NsanInitIsRunning = true;
+
+ InitializeFlags();
+ InitializeSuppressions();
+ InitializePlatformEarly();
+
+ if (!MmapFixedNoReserve(TypesAddr(), UnusedAddr() - TypesAddr()))
+ Die();
+
+ initializeInterceptors();
+
+ initializeStats();
+ if (flags().print_stats_on_exit)
+ Atexit(nsanAtexit);
+
+ NsanInitIsRunning = false;
+ NsanInitialized = true;
+}
+
+#if SANITIZER_CAN_USE_PREINIT_ARRAY
+__attribute__((section(".preinit_array"),
+ used)) static void (*nsan_init_ptr)() = __nsan_init;
+#endif
diff --git a/compiler-rt/lib/nsan/nsan.h b/compiler-rt/lib/nsan/nsan.h
new file mode 100644
index 0000000000000..6ab64d3aecb38
--- /dev/null
+++ b/compiler-rt/lib/nsan/nsan.h
@@ -0,0 +1,225 @@
+//===-- nsan.h -------------------------------------------------*- C++ -*-===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+//
+// This file is a part of NumericalStabilitySanitizer.
+//
+// Private NSan header.
+//===----------------------------------------------------------------------===//
+
+#ifndef NSAN_H
+#define NSAN_H
+
+#include "sanitizer_common/sanitizer_internal_defs.h"
+
+using __sanitizer::sptr;
+using __sanitizer::u16;
+using __sanitizer::u8;
+using __sanitizer::uptr;
+
+#include "nsan_platform.h"
+
+#include <assert.h>
+#include <float.h>
+#include <limits.h>
+#include <math.h>
+#include <stdio.h>
+
+// Private nsan interface. Used e.g. by interceptors.
+extern "C" {
+
+// This marks the shadow type of the given block of application memory as
+// unknown.
+// printf-free (see comment in nsan_interceptors.cc).
+void __nsan_set_value_unknown(const u8 *addr, uptr size);
+
+// Copies annotations in the shadow memory for a block of application memory to
+// a new address. This function is used together with memory-copying functions
+// in application memory, e.g. the instrumentation inserts
+// `__nsan_copy_values(dest, src, size)` after builtin calls to
+// `memcpy(dest, src, size)`. Intercepted memcpy calls also call this function.
+// printf-free (see comment in nsan_interceptors.cc).
+void __nsan_copy_values(const u8 *daddr, const u8 *saddr, uptr size);
+
+SANITIZER_INTERFACE_ATTRIBUTE SANITIZER_WEAK_ATTRIBUTE const char *
+__nsan_default_options();
+}
+
+namespace __nsan {
+
+extern bool NsanInitialized;
+extern bool NsanInitIsRunning;
+
+void initializeInterceptors();
+
+// See notes in nsan_platform.
+// printf-free (see comment in nsan_interceptors.cc).
+inline u8 *getShadowAddrFor(u8 *Ptr) {
+ uptr AppOffset = ((uptr)Ptr) & ShadowMask();
+ return (u8 *)(AppOffset * kShadowScale + ShadowAddr());
+}
+
+// printf-free (see comment in nsan_interceptors.cc).
+inline const u8 *getShadowAddrFor(const u8 *Ptr) {
+ return getShadowAddrFor(const_cast<u8 *>(Ptr));
+}
+
+// printf-free (see comment in nsan_interceptors.cc).
+inline u8 *getShadowTypeAddrFor(u8 *Ptr) {
+ uptr AppOffset = ((uptr)Ptr) & ShadowMask();
+ return (u8 *)(AppOffset + TypesAddr());
+}
+
+// printf-free (see comment in nsan_interceptors.cc).
+inline const u8 *getShadowTypeAddrFor(const u8 *Ptr) {
+ return getShadowTypeAddrFor(const_cast<u8 *>(Ptr));
+}
+
+// Information about value types and their shadow counterparts.
+template <typename FT> struct FTInfo {};
+template <> struct FTInfo<float> {
+ using orig_type = float;
+ using orig_bits_type = __sanitizer::u32;
+ using mantissa_bits_type = __sanitizer::u32;
+ using shadow_type = double;
+ static const char *kCppTypeName;
+ static constexpr unsigned kMantissaBits = 23;
+ static constexpr int kExponentBits = 8;
+ static constexpr int kExponentBias = 127;
+ static constexpr int kValueType = kFloatValueType;
+ static constexpr char kTypePattern[sizeof(float)] = {
+ static_cast<unsigned char>(kValueType | (0 << kValueSizeSizeBits)),
+ static_cast<unsigned char>(kValueType | (1 << kValueSizeSizeBits)),
+ static_cast<unsigned char>(kValueType | (2 << kValueSizeSizeBits)),
+ static_cast<unsigned char>(kValueType | (3 << kValueSizeSizeBits)),
+ };
+ static constexpr const float kEpsilon = FLT_EPSILON;
+};
+template <> struct FTInfo<double> {
+ using orig_type = double;
+ using orig_bits_type = __sanitizer::u64;
+ using mantissa_bits_type = __sanitizer::u64;
+ using shadow_type = __float128;
+ static const char *kCppTypeName;
+ static constexpr unsigned kMantissaBits = 52;
+ static constexpr int kExponentBits = 11;
+ static constexpr int kExponentBias = 1023;
+ static constexpr int kValueType = kDoubleValueType;
+ static constexpr char kTypePattern[sizeof(double)] = {
+ static_cast<unsigned char>(kValueType | (0 << kValueSizeSizeBits)),
+ static_cast<unsigned char>(kValueType | (1 << kValueSizeSizeBits)),
+ static_cast<unsigned char>(kValueType | (2 << kValueSizeSizeBits)),
+ static_cast<unsigned char>(kValueType | (3 << kValueSizeSizeBits)),
+ static_cast<unsigned char>(kValueType | (4 << kValueSizeSizeBits)),
+ static_cast<unsigned char>(kValueType | (5 << kValueSizeSizeBits)),
+ static_cast<unsigned char>(kValueType | (6 << kValueSizeSizeBits)),
+ static_cast<unsigned char>(kValueType | (7 << kValueSizeSizeBits)),
+ };
+ static constexpr const float kEpsilon = DBL_EPSILON;
+};
+template <> struct FTInfo<long double> {
+ using orig_type = long double;
+ using mantissa_bits_type = __sanitizer::u64;
+ using shadow_type = __float128;
+ static const char *kCppTypeName;
+ static constexpr unsigned kMantissaBits = 63;
+ static constexpr int kExponentBits = 15;
+ static constexpr int kExponentBias = (1 << (kExponentBits - 1)) - 1;
+ static constexpr int kValueType = kFp80ValueType;
+ static constexpr char kTypePattern[sizeof(long double)] = {
+ static_cast<unsigned char>(kValueType | (0 << kValueSizeSizeBits)),
+ static_cast<unsigned char>(kValueType | (1 << kValueSizeSizeBits)),
+ static_cast<unsigned char>(kValueType | (2 << kValueSizeSizeBits)),
+ static_cast<unsigned char>(kValueType | (3 << kValueSizeSizeBits)),
+ static_cast<unsigned char>(kValueType | (4 << kValueSizeSizeBits)),
+ static_cast<unsigned char>(kValueType | (5 << kValueSizeSizeBits)),
+ static_cast<unsigned char>(kValueType | (6 << kValueSizeSizeBits)),
+ static_cast<unsigned char>(kValueType | (7 << kValueSizeSizeBits)),
+ static_cast<unsigned char>(kValueType | (8 << kValueSizeSizeBits)),
+ static_cast<unsigned char>(kValueType | (9 << kValueSizeSizeBits)),
+ static_cast<unsigned char>(kValueType | (10 << kValueSizeSizeBits)),
+ static_cast<unsigned char>(kValueType | (11 << kValueSizeSizeBits)),
+ static_cast<unsigned char>(kValueType | (12 << kValueSizeSizeBits)),
+ static_cast<unsigned char>(kValueType | (13 << kValueSizeSizeBits)),
+ static_cast<unsigned char>(kValueType | (14 << kValueSizeSizeBits)),
+ static_cast<unsigned char>(kValueType | (15 << kValueSizeSizeBits)),
+ };
+ static constexpr const float kEpsilon = LDBL_EPSILON;
+};
+
+template <> struct FTInfo<__float128> {
+ using orig_type = __float128;
+ using orig_bits_type = __uint128_t;
+ using mantissa_bits_type = __uint128_t;
+ static const char *kCppTypeName;
+ static constexpr unsigned kMantissaBits = 112;
+ static constexpr int kExponentBits = 15;
+ static constexpr int kExponentBias = (1 << (kExponentBits - 1)) - 1;
+};
+
+constexpr double kMaxULPDiff = INFINITY;
+
+// Helper for getULPDiff that works on bit representations.
+template <typename BT> double getULPDiffBits(BT V1Bits, BT V2Bits) {
+ // If the integer representations of two same-sign floats are subtracted then
+ // the absolute value of the result is equal to one plus the number of
+ // representable floats between them.
+ return V1Bits >= V2Bits ? V1Bits - V2Bits : V2Bits - V1Bits;
+}
+
+// Returns the the number of floating point values between V1 and V2, capped to
+// u64max. Return 0 for (-0.0,0.0).
+template <typename FT> double getULPDiff(FT V1, FT V2) {
+ if (V1 == V2) {
+ return 0; // Typically, -0.0 and 0.0
+ }
+ using BT = typename FTInfo<FT>::orig_bits_type;
+ static_assert(sizeof(FT) == sizeof(BT), "not implemented");
+ static_assert(sizeof(BT) <= 64, "not implemented");
+ BT V1Bits;
+ __builtin_memcpy(&V1Bits, &V1, sizeof(BT));
+ BT V2Bits;
+ __builtin_memcpy(&V2Bits, &V2, sizeof(BT));
+ // Check whether the signs differ. IEEE-754 float types always store the sign
+ // in the most significant bit. NaNs and infinities are handled by the calling
+ // code.
+ constexpr BT kSignMask = BT{1} << (CHAR_BIT * sizeof(BT) - 1);
+ if ((V1Bits ^ V2Bits) & kSignMask) {
+ // Signs differ. We can get the ULPs as `getULPDiff(negative_number, -0.0)
+ // + getULPDiff(0.0, positive_number)`.
+ if (V1Bits & kSignMask) {
+ return getULPDiffBits<BT>(V1Bits, kSignMask) +
+ getULPDiffBits<BT>(0, V2Bits);
+ } else {
+ return getULPDiffBits<BT>(V2Bits, kSignMask) +
+ getULPDiffBits<BT>(0, V1Bits);
+ }
+ }
+ return getULPDiffBits(V1Bits, V2Bits);
+}
+
+// FIXME: This needs mor work: Because there is no 80-bit integer type, we have
+// to go through __uint128_t. Therefore the assumptions about the sign bit do
+// not hold.
+template <> inline double getULPDiff(long double V1, long double V2) {
+ using BT = __uint128_t;
+ BT V1Bits = 0;
+ __builtin_memcpy(&V1Bits, &V1, sizeof(long double));
+ BT V2Bits = 0;
+ __builtin_memcpy(&V2Bits, &V2, sizeof(long double));
+ if ((V1Bits ^ V2Bits) & (BT{1} << (CHAR_BIT * sizeof(BT) - 1)))
+ return (V1 == V2) ? __sanitizer::u64{0} : kMaxULPDiff; // Signs differ.
+ // If the integer representations of two same-sign floats are subtracted then
+ // the absolute value of the result is equal to one plus the number of
+ // representable floats between them.
+ BT Diff = V1Bits >= V2Bits ? V1Bits - V2Bits : V2Bits - V1Bits;
+ return Diff >= kMaxULPDiff ? kMaxULPDiff : Diff;
+}
+
+} // end namespace __nsan
+
+#endif // NSAN_H
diff --git a/compiler-rt/lib/nsan/nsan.syms.extra b/compiler-rt/lib/nsan/nsan.syms.extra
new file mode 100644
index 0000000000000..f3be6d39736b7
--- /dev/null
+++ b/compiler-rt/lib/nsan/nsan.syms.extra
@@ -0,0 +1,2 @@
+nsan_*
+__nsan_*
diff --git a/compiler-rt/lib/nsan/nsan_flags.cc b/compiler-rt/lib/nsan/nsan_flags.cc
new file mode 100644
index 0000000000000..09a9dad7ce041
--- /dev/null
+++ b/compiler-rt/lib/nsan/nsan_flags.cc
@@ -0,0 +1,78 @@
+//===-- nsan_flags.cc -----------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+//
+// This file is a part of NumericalStabilitySanitizer.
+//
+//===----------------------------------------------------------------------===//
+
+#include "nsan_flags.h"
+
+#include "sanitizer_common/sanitizer_flag_parser.h"
+#include "sanitizer_common/sanitizer_flags.h"
+
+namespace __nsan {
+
+SANITIZER_INTERFACE_WEAK_DEF(const char *, __nsan_default_options, void) {
+ return "";
+}
+
+using namespace __sanitizer;
+
+Flags flags_data;
+
+void Flags::SetDefaults() {
+#define NSAN_FLAG(Type, Name, DefaultValue, Description) Name = DefaultValue;
+#include "nsan_flags.inc"
+#undef NSAN_FLAG
+}
+
+void Flags::PopulateCache() {
+ cached_absolute_error_threshold =
+ 1.0 / (1ull << log2_absolute_error_threshold);
+}
+
+static void RegisterNSanFlags(FlagParser *parser, Flags *f) {
+#define NSAN_FLAG(Type, Name, DefaultValue, Description) \
+ RegisterFlag(parser, #Name, Description, &f->Name);
+#include "nsan_flags.inc"
+#undef NSAN_FLAG
+}
+
+static const char *MaybeCallNsanDefaultOptions() {
+ return (&__nsan_default_options) ? __nsan_default_options() : "";
+}
+
+void InitializeFlags() {
+ SetCommonFlagsDefaults();
+ {
+ CommonFlags cf;
+ cf.CopyFrom(*common_flags());
+ cf.external_symbolizer_path = GetEnv("NSAN_SYMBOLIZER_PATH");
+ OverrideCommonFlags(cf);
+ }
+
+ flags().SetDefaults();
+
+ FlagParser parser;
+ RegisterCommonFlags(&parser);
+ RegisterNSanFlags(&parser, &flags());
+
+ const char *nsan_default_options = MaybeCallNsanDefaultOptions();
+ parser.ParseString(nsan_default_options);
+
+ parser.ParseString(GetEnv("NSAN_OPTIONS"));
+ InitializeCommonFlags();
+ if (Verbosity())
+ ReportUnrecognizedFlags();
+ if (common_flags()->help)
+ parser.PrintFlagDescriptions();
+
+ flags().PopulateCache();
+}
+
+} // namespace __nsan
diff --git a/compiler-rt/lib/nsan/nsan_flags.h b/compiler-rt/lib/nsan/nsan_flags.h
new file mode 100644
index 0000000000000..ea5390b6e2ab8
--- /dev/null
+++ b/compiler-rt/lib/nsan/nsan_flags.h
@@ -0,0 +1,35 @@
+//===-- nsan_flags.h --------------------------------------------*- C++ -*-===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+//
+// This file is a part of NumericalStabilitySanitizer.
+//===----------------------------------------------------------------------===//
+
+#ifndef NSAN_FLAGS_H
+#define NSAN_FLAGS_H
+
+namespace __nsan {
+
+struct Flags {
+#define NSAN_FLAG(Type, Name, DefaultValue, Description) Type Name;
+#include "nsan_flags.inc"
+#undef NSAN_FLAG
+
+ double cached_absolute_error_threshold = 0.0;
+
+ void SetDefaults();
+ void PopulateCache();
+};
+
+extern Flags flags_data;
+inline Flags &flags() { return flags_data; }
+
+void InitializeFlags();
+
+} // namespace __nsan
+
+#endif
diff --git a/compiler-rt/lib/nsan/nsan_flags.inc b/compiler-rt/lib/nsan/nsan_flags.inc
new file mode 100644
index 0000000000000..63c15475f6754
--- /dev/null
+++ b/compiler-rt/lib/nsan/nsan_flags.inc
@@ -0,0 +1,49 @@
+//===-- nsan_flags.inc ------------------------------------------*- C++ -*-===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+//
+// NSan runtime flags.
+//
+//===----------------------------------------------------------------------===//
+#ifndef NSAN_FLAG
+#error "Define NSAN_FLAG prior to including this file!"
+#endif
+
+// NSAN_FLAG(Type, Name, DefaultValue, Description)
+// See COMMON_FLAG in sanitizer_flags.inc for more details.
+
+NSAN_FLAG(bool, halt_on_error, true, "If true, halt after the first error.")
+NSAN_FLAG(bool, resume_after_warning, true,
+ "If true, we resume resume the computation from the original "
+ "application floating-point value after a warning. If false, "
+ "computations continue with the shadow value.")
+NSAN_FLAG(const char *, suppressions, "", "Suppressions file name.")
+NSAN_FLAG(bool, resume_after_suppression, true,
+ "If true, a suppression will also resume the computation from the FT"
+ " domain. If false, output is suppressed but the shadow value is"
+ " retained.")
+// FIXME: should this be specified in units of epsilon instead?
+NSAN_FLAG(int, log2_max_relative_error, 19,
+ "Log2 maximum admissible relative error, e.g. 19 means max relative "
+ "error of 1/2^19 ~= 0.000002.")
+NSAN_FLAG(int, log2_absolute_error_threshold, 32,
+ "Log2 maximum admissible absolute error. Any numbers closer than "
+ "1/2^n are considered to be the same.")
+NSAN_FLAG(bool, disable_warnings, false,
+ "If true, disable warning printing. This is useful to only compute "
+ "stats.")
+NSAN_FLAG(bool, enable_check_stats, false,
+ "If true, compute check stats, i.e. for each line, the number of "
+ "times a check was performed on this line.")
+NSAN_FLAG(bool, enable_warning_stats, false,
+ "If true, compute warning stats, i.e. for each line, the number of "
+ "times a warning was emitted for this line.")
+NSAN_FLAG(bool, enable_loadtracking_stats, false,
+ "If true, compute load tracking stats, i.e. for each load from "
+ "memory, the number of times nsan resumed from the original value "
+ "due to invalid or unknown types.")
+NSAN_FLAG(bool, print_stats_on_exit, false, "If true, print stats on exit.")
diff --git a/compiler-rt/lib/nsan/nsan_interceptors.cc b/compiler-rt/lib/nsan/nsan_interceptors.cc
new file mode 100644
index 0000000000000..394d788888268
--- /dev/null
+++ b/compiler-rt/lib/nsan/nsan_interceptors.cc
@@ -0,0 +1,364 @@
+//===-- nsan_interceptors.cc ----------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+//
+// Interceptors for standard library functions.
+//
+// A note about `printf`: Make sure none of the interceptor code calls any
+// part of the nsan framework that can call `printf`, since this could create
+// a loop (`printf` itself uses the libc). printf-free functions are documented
+// as such in nsan.h.
+//
+//===----------------------------------------------------------------------===//
+
+#include "interception/interception.h"
+#include "nsan/nsan.h"
+#include "sanitizer_common/sanitizer_common.h"
+
+#include <wchar.h>
+
+#if SANITIZER_LINUX
+extern "C" int mallopt(int param, int value);
+#endif
+
+using namespace __sanitizer;
+using __nsan::NsanInitialized;
+using __nsan::NsanInitIsRunning;
+
+static constexpr uptr kEarlyAllocBufSize = 16384;
+static uptr AllocatedBytes;
+static char EarlyAllocBuf[kEarlyAllocBufSize];
+
+static bool isInEarlyAllocBuf(const void *Ptr) {
+ return ((uptr)Ptr >= (uptr)EarlyAllocBuf &&
+ ((uptr)Ptr - (uptr)EarlyAllocBuf) < sizeof(EarlyAllocBuf));
+}
+
+static u8 *toU8Ptr(wchar_t *ptr) { return reinterpret_cast<u8 *>(ptr); }
+
+static const u8 *toU8Ptr(const wchar_t *ptr) {
+ return reinterpret_cast<const u8 *>(ptr);
+}
+
+template <typename T> T min(T a, T b) { return a < b ? a : b; }
+
+// Handle allocation requests early (before all interceptors are setup). dlsym,
+// for example, calls calloc.
+static void *handleEarlyAlloc(uptr Size) {
+ void *Mem = (void *)&EarlyAllocBuf[AllocatedBytes];
+ AllocatedBytes += Size;
+ CHECK_LT(AllocatedBytes, kEarlyAllocBufSize);
+ return Mem;
+}
+
+INTERCEPTOR(void *, memset, void *Dst, int V, uptr Size) {
+ // NOTE: This guard is needed because nsan's initialization code might call
+ // memset.
+ if (!NsanInitialized && REAL(memset) == nullptr)
+ return internal_memset(Dst, V, Size);
+
+ void *Res = REAL(memset)(Dst, V, Size);
+ __nsan_set_value_unknown(static_cast<u8 *>(Dst), Size);
+ return Res;
+}
+
+INTERCEPTOR(wchar_t *, wmemset, wchar_t *Dst, wchar_t V, uptr Size) {
+ wchar_t *Res = REAL(wmemset)(Dst, V, Size);
+ __nsan_set_value_unknown(toU8Ptr(Dst), sizeof(wchar_t) * Size);
+ return Res;
+}
+
+INTERCEPTOR(void *, memmove, void *Dst, const void *Src, uptr Size) {
+ // NOTE: This guard is needed because nsan's initialization code might call
+ // memmove.
+ if (!NsanInitialized && REAL(memmove) == nullptr)
+ return internal_memmove(Dst, Src, Size);
+
+ void *Res = REAL(memmove)(Dst, Src, Size);
+ __nsan_copy_values(static_cast<u8 *>(Dst), static_cast<const u8 *>(Src),
+ Size);
+ return Res;
+}
+
+INTERCEPTOR(wchar_t *, wmemmove, wchar_t *Dst, const wchar_t *Src, uptr Size) {
+ wchar_t *Res = REAL(wmemmove)(Dst, Src, Size);
+ __nsan_copy_values(toU8Ptr(Dst), toU8Ptr(Src), sizeof(wchar_t) * Size);
+ return Res;
+}
+
+INTERCEPTOR(void *, memcpy, void *Dst, const void *Src, uptr Size) {
+ // NOTE: This guard is needed because nsan's initialization code might call
+ // memcpy.
+ if (!NsanInitialized && REAL(memcpy) == nullptr) {
+ // memmove is used here because on some platforms this will also
+ // intercept the memmove implementation.
+ return internal_memmove(Dst, Src, Size);
+ }
+
+ void *Res = REAL(memcpy)(Dst, Src, Size);
+ __nsan_copy_values(static_cast<u8 *>(Dst), static_cast<const u8 *>(Src),
+ Size);
+ return Res;
+}
+
+INTERCEPTOR(wchar_t *, wmemcpy, wchar_t *Dst, const wchar_t *Src, uptr Size) {
+ wchar_t *Res = REAL(wmemcpy)(Dst, Src, Size);
+ __nsan_copy_values(toU8Ptr(Dst), toU8Ptr(Src), sizeof(wchar_t) * Size);
+ return Res;
+}
+
+INTERCEPTOR(void *, malloc, uptr Size) {
+ // NOTE: This guard is needed because nsan's initialization code might call
+ // malloc.
+ if (NsanInitIsRunning && REAL(malloc) == nullptr)
+ return handleEarlyAlloc(Size);
+
+ void *Res = REAL(malloc)(Size);
+ if (Res)
+ __nsan_set_value_unknown(static_cast<u8 *>(Res), Size);
+ return Res;
+}
+
+INTERCEPTOR(void *, realloc, void *Ptr, uptr Size) {
+ void *Res = REAL(realloc)(Ptr, Size);
+ // FIXME: We might want to copy the types from the original allocation
+ // (although that would require that we know its size).
+ if (Res)
+ __nsan_set_value_unknown(static_cast<u8 *>(Res), Size);
+ return Res;
+}
+
+INTERCEPTOR(void *, calloc, uptr Nmemb, uptr Size) {
+ // NOTE: This guard is needed because nsan's initialization code might call
+ // calloc.
+ if (NsanInitIsRunning && REAL(calloc) == nullptr) {
+ // Note: EarlyAllocBuf is initialized with zeros.
+ return handleEarlyAlloc(Nmemb * Size);
+ }
+
+ void *Res = REAL(calloc)(Nmemb, Size);
+ if (Res)
+ __nsan_set_value_unknown(static_cast<u8 *>(Res), Nmemb * Size);
+ return Res;
+}
+
+INTERCEPTOR(void, free, void *P) {
+ // There are only a few early allocation requests, so we simply skip the free.
+ if (isInEarlyAllocBuf(P))
+ return;
+ REAL(free)(P);
+}
+
+INTERCEPTOR(void *, valloc, uptr Size) {
+ void *const Res = REAL(valloc)(Size);
+ if (Res)
+ __nsan_set_value_unknown(static_cast<u8 *>(Res), Size);
+ return Res;
+}
+
+INTERCEPTOR(void *, memalign, uptr Alignment, uptr Size) {
+ void *const Res = REAL(memalign)(Alignment, Size);
+ if (Res)
+ __nsan_set_value_unknown(static_cast<u8 *>(Res), Size);
+ return Res;
+}
+
+INTERCEPTOR(void *, __libc_memalign, uptr Alignment, uptr Size) {
+ void *const Res = REAL(__libc_memalign)(Alignment, Size);
+ if (Res)
+ __nsan_set_value_unknown(static_cast<u8 *>(Res), Size);
+ return Res;
+}
+
+INTERCEPTOR(void *, pvalloc, uptr Size) {
+ void *const Res = REAL(pvalloc)(Size);
+ if (Res)
+ __nsan_set_value_unknown(static_cast<u8 *>(Res), Size);
+ return Res;
+}
+
+INTERCEPTOR(void *, aligned_alloc, uptr Alignment, uptr Size) {
+ void *const Res = REAL(aligned_alloc)(Alignment, Size);
+ if (Res)
+ __nsan_set_value_unknown(static_cast<u8 *>(Res), Size);
+ return Res;
+}
+
+INTERCEPTOR(int, posix_memalign, void **Memptr, uptr Alignment, uptr Size) {
+ int Res = REAL(posix_memalign)(Memptr, Alignment, Size);
+ if (Res == 0 && *Memptr)
+ __nsan_set_value_unknown(static_cast<u8 *>(*Memptr), Size);
+ return Res;
+}
+
+INTERCEPTOR(char *, strfry, char *S) {
+ const auto Len = internal_strlen(S);
+ char *Res = REAL(strfry)(S);
+ if (Res)
+ __nsan_set_value_unknown(reinterpret_cast<u8 *>(S), Len);
+ return Res;
+}
+
+INTERCEPTOR(char *, strsep, char **Stringp, const char *Delim) {
+ char *OrigStringp = REAL(strsep)(Stringp, Delim);
+ if (Stringp != nullptr) {
+ // The previous character has been overwritten with a '\0' char.
+ __nsan_set_value_unknown(reinterpret_cast<u8 *>(*Stringp) - 1, 1);
+ }
+ return OrigStringp;
+}
+
+INTERCEPTOR(char *, strtok, char *Str, const char *Delim) {
+ // This is overly conservative, but the probability that modern code is using
+ // strtok on double data is essentially zero anyway.
+ if (Str)
+ __nsan_set_value_unknown(reinterpret_cast<u8 *>(Str), internal_strlen(Str));
+ return REAL(strtok)(Str, Delim);
+}
+
+static void nsanCopyZeroTerminated(char *Dst, const char *Src, uptr N) {
+ __nsan_copy_values(reinterpret_cast<u8 *>(Dst),
+ reinterpret_cast<const u8 *>(Src), N); // Data.
+ __nsan_set_value_unknown(reinterpret_cast<u8 *>(Dst) + N, 1); // Terminator.
+}
+
+static void nsanWCopyZeroTerminated(wchar_t *Dst, const wchar_t *Src, uptr N) {
+ __nsan_copy_values(toU8Ptr(Dst), toU8Ptr(Src), sizeof(wchar_t) * N);
+ __nsan_set_value_unknown(toU8Ptr(Dst + N), sizeof(wchar_t));
+}
+
+INTERCEPTOR(char *, strdup, const char *S) {
+ char *Res = REAL(strdup)(S);
+ if (Res) {
+ nsanCopyZeroTerminated(Res, S, internal_strlen(S));
+ }
+ return Res;
+}
+
+INTERCEPTOR(wchar_t *, wcsdup, const wchar_t *S) {
+ wchar_t *Res = REAL(wcsdup)(S);
+ if (Res) {
+ nsanWCopyZeroTerminated(Res, S, wcslen(S));
+ }
+ return Res;
+}
+
+INTERCEPTOR(char *, strndup, const char *S, uptr Size) {
+ char *Res = REAL(strndup)(S, Size);
+ if (Res) {
+ nsanCopyZeroTerminated(Res, S, min(internal_strlen(S), Size));
+ }
+ return Res;
+}
+
+INTERCEPTOR(char *, strcpy, char *Dst, const char *Src) {
+ char *Res = REAL(strcpy)(Dst, Src);
+ nsanCopyZeroTerminated(Dst, Src, internal_strlen(Src));
+ return Res;
+}
+
+INTERCEPTOR(wchar_t *, wcscpy, wchar_t *Dst, const wchar_t *Src) {
+ wchar_t *Res = REAL(wcscpy)(Dst, Src);
+ nsanWCopyZeroTerminated(Dst, Src, wcslen(Src));
+ return Res;
+}
+
+INTERCEPTOR(char *, strncpy, char *Dst, const char *Src, uptr Size) {
+ char *Res = REAL(strncpy)(Dst, Src, Size);
+ nsanCopyZeroTerminated(Dst, Src, min(Size, internal_strlen(Src)));
+ return Res;
+}
+
+INTERCEPTOR(char *, strcat, char *Dst, const char *Src) {
+ const auto DstLenBeforeCat = internal_strlen(Dst);
+ char *Res = REAL(strcat)(Dst, Src);
+ nsanCopyZeroTerminated(Dst + DstLenBeforeCat, Src, internal_strlen(Src));
+ return Res;
+}
+
+INTERCEPTOR(wchar_t *, wcscat, wchar_t *Dst, const wchar_t *Src) {
+ const auto DstLenBeforeCat = wcslen(Dst);
+ wchar_t *Res = REAL(wcscat)(Dst, Src);
+ nsanWCopyZeroTerminated(Dst + DstLenBeforeCat, Src, wcslen(Src));
+ return Res;
+}
+
+INTERCEPTOR(char *, strncat, char *Dst, const char *Src, uptr Size) {
+ const auto DstLen = internal_strlen(Dst);
+ char *Res = REAL(strncat)(Dst, Src, Size);
+ nsanCopyZeroTerminated(Dst + DstLen, Src, min(Size, internal_strlen(Src)));
+ return Res;
+}
+
+INTERCEPTOR(char *, stpcpy, char *Dst, const char *Src) {
+ char *Res = REAL(stpcpy)(Dst, Src);
+ nsanCopyZeroTerminated(Dst, Src, internal_strlen(Src));
+ return Res;
+}
+
+INTERCEPTOR(wchar_t *, wcpcpy, wchar_t *Dst, const wchar_t *Src) {
+ wchar_t *Res = REAL(wcpcpy)(Dst, Src);
+ nsanWCopyZeroTerminated(Dst, Src, wcslen(Src));
+ return Res;
+}
+
+INTERCEPTOR(uptr, strxfrm, char *Dst, const char *Src, uptr Size) {
+ // This is overly conservative, but this function should very rarely be used.
+ __nsan_set_value_unknown(reinterpret_cast<u8 *>(Dst), internal_strlen(Dst));
+ const uptr Res = REAL(strxfrm)(Dst, Src, Size);
+ return Res;
+}
+
+namespace __nsan {
+void initializeInterceptors() {
+ static bool Initialized = false;
+ CHECK(!Initialized);
+
+ // Instruct libc malloc to consume less memory.
+#if SANITIZER_LINUX
+ mallopt(1, 0); // M_MXFAST
+ mallopt(-3, 32 * 1024); // M_MMAP_THRESHOLD
+#endif
+
+ INTERCEPT_FUNCTION(malloc);
+ INTERCEPT_FUNCTION(calloc);
+ INTERCEPT_FUNCTION(free);
+ INTERCEPT_FUNCTION(realloc);
+ INTERCEPT_FUNCTION(valloc);
+ INTERCEPT_FUNCTION(memalign);
+ INTERCEPT_FUNCTION(__libc_memalign);
+ INTERCEPT_FUNCTION(pvalloc);
+ INTERCEPT_FUNCTION(aligned_alloc);
+ INTERCEPT_FUNCTION(posix_memalign);
+
+ INTERCEPT_FUNCTION(memset);
+ INTERCEPT_FUNCTION(wmemset);
+ INTERCEPT_FUNCTION(memmove);
+ INTERCEPT_FUNCTION(wmemmove);
+ INTERCEPT_FUNCTION(memcpy);
+ INTERCEPT_FUNCTION(wmemcpy);
+
+ INTERCEPT_FUNCTION(strdup);
+ INTERCEPT_FUNCTION(wcsdup);
+ INTERCEPT_FUNCTION(strndup);
+ INTERCEPT_FUNCTION(stpcpy);
+ INTERCEPT_FUNCTION(wcpcpy);
+ INTERCEPT_FUNCTION(strcpy);
+ INTERCEPT_FUNCTION(wcscpy);
+ INTERCEPT_FUNCTION(strncpy);
+ INTERCEPT_FUNCTION(strcat);
+ INTERCEPT_FUNCTION(wcscat);
+ INTERCEPT_FUNCTION(strncat);
+ INTERCEPT_FUNCTION(strxfrm);
+
+ INTERCEPT_FUNCTION(strfry);
+ INTERCEPT_FUNCTION(strsep);
+ INTERCEPT_FUNCTION(strtok);
+
+ Initialized = 1;
+}
+} // end namespace __nsan
diff --git a/compiler-rt/lib/nsan/nsan_platform.h b/compiler-rt/lib/nsan/nsan_platform.h
new file mode 100644
index 0000000000000..c9d4cacd8c889
--- /dev/null
+++ b/compiler-rt/lib/nsan/nsan_platform.h
@@ -0,0 +1,135 @@
+//===------------------------ nsan_platform.h -------------------*- C++ -*-===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+//
+// Platform specific information for NSan.
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef NSAN_PLATFORM_H
+#define NSAN_PLATFORM_H
+
+namespace __nsan {
+
+// NSan uses two regions of memory to store information:
+// - 'shadow memory' stores the shadow copies of numerical values stored in
+// application memory.
+// - 'shadow types' is used to determine which value type each byte of memory
+// belongs to. This makes sure that we always know whether a shadow value is
+// valid. Shadow values may be tampered with using access through other
+// pointer types (type punning). Each byte stores:
+// - bit 1-0: whether the corresponding value is of unknown (00),
+// float (01), double (10), or long double (11) type.
+// - bit 5-2: the index of this byte in the value, or 0000 if type is
+// unknown.
+// This allows handling unaligned loat load/stores by checking that a load
+// with a given alignment corresponds to the alignment of the store.
+// Any store of a non-floating point type invalidates the corresponding
+// bytes, so that subsequent overlapping loads (aligned or not) know that
+// the corresponding shadow value is no longer valid.
+
+// On Linux/x86_64, memory is laid out as follows:
+//
+// +--------------------+ 0x800000000000 (top of memory)
+// | application memory |
+// +--------------------+ 0x700000008000 (kAppAddr)
+// | |
+// | unused |
+// | |
+// +--------------------+ 0x400000000000 (kUnusedAddr)
+// | shadow memory |
+// +--------------------+ 0x200000000000 (kShadowAddr)
+// | shadow types |
+// +--------------------+ 0x100000000000 (kTypesAddr)
+// | reserved by kernel |
+// +--------------------+ 0x000000000000
+//
+//
+// To derive a shadow memory address from an application memory address,
+// bits 44-46 are cleared to bring the address into the range
+// [0x000000000000,0x100000000000). We scale to account for the fact that a
+// shadow value takes twice as much space as the original value.
+// Then we add kShadowAddr to put the shadow relative offset into the shadow
+// memory. See getShadowAddrFor().
+// The process is similar for the shadow types.
+
+// The ratio of app to shadow memory.
+enum { kShadowScale = 2 };
+
+// The original value type of a byte in app memory. Uses LLVM terminology:
+// https://llvm.org/docs/LangRef.html#floating-point-types
+// FIXME: support half and bfloat.
+enum ValueType {
+ kUnknownValueType = 0,
+ kFloatValueType = 1, // LLVM float, shadow type double.
+ kDoubleValueType = 2, // LLVM double, shadow type fp128.
+ kFp80ValueType = 3, // LLVM x86_fp80, shadow type fp128.
+};
+
+// The size of ValueType encoding, in bits.
+enum {
+ kValueSizeSizeBits = 2,
+};
+
+#if defined(__x86_64__)
+struct Mapping {
+ // FIXME: kAppAddr == 0x700000000000 ?
+ static const uptr kAppAddr = 0x700000008000;
+ static const uptr kUnusedAddr = 0x400000000000;
+ static const uptr kShadowAddr = 0x200000000000;
+ static const uptr kTypesAddr = 0x100000000000;
+ static const uptr kShadowMask = ~0x700000000000;
+};
+#else
+#error "NSan not supported for this platform!"
+#endif
+
+enum MappingType {
+ MAPPING_APP_ADDR,
+ MAPPING_UNUSED_ADDR,
+ MAPPING_SHADOW_ADDR,
+ MAPPING_TYPES_ADDR,
+ MAPPING_SHADOW_MASK
+};
+
+template <typename Mapping, int Type> uptr MappingImpl() {
+ switch (Type) {
+ case MAPPING_APP_ADDR:
+ return Mapping::kAppAddr;
+ case MAPPING_UNUSED_ADDR:
+ return Mapping::kUnusedAddr;
+ case MAPPING_SHADOW_ADDR:
+ return Mapping::kShadowAddr;
+ case MAPPING_TYPES_ADDR:
+ return Mapping::kTypesAddr;
+ case MAPPING_SHADOW_MASK:
+ return Mapping::kShadowMask;
+ }
+}
+
+template <int Type> uptr MappingArchImpl() {
+ return MappingImpl<Mapping, Type>();
+}
+
+ALWAYS_INLINE
+uptr AppAddr() { return MappingArchImpl<MAPPING_APP_ADDR>(); }
+
+ALWAYS_INLINE
+uptr UnusedAddr() { return MappingArchImpl<MAPPING_UNUSED_ADDR>(); }
+
+ALWAYS_INLINE
+uptr ShadowAddr() { return MappingArchImpl<MAPPING_SHADOW_ADDR>(); }
+
+ALWAYS_INLINE
+uptr TypesAddr() { return MappingArchImpl<MAPPING_TYPES_ADDR>(); }
+
+ALWAYS_INLINE
+uptr ShadowMask() { return MappingArchImpl<MAPPING_SHADOW_MASK>(); }
+
+} // end namespace __nsan
+
+#endif
diff --git a/compiler-rt/lib/nsan/nsan_stats.cc b/compiler-rt/lib/nsan/nsan_stats.cc
new file mode 100644
index 0000000000000..4e7a0c5810192
--- /dev/null
+++ b/compiler-rt/lib/nsan/nsan_stats.cc
@@ -0,0 +1,158 @@
+//===-- nsan_stats.cc -----------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+//
+// This file is a part of NumericalStabilitySanitizer.
+//
+// NumericalStabilitySanitizer statistics.
+//===----------------------------------------------------------------------===//
+
+#include "nsan/nsan_stats.h"
+
+#include "sanitizer_common/sanitizer_common.h"
+#include "sanitizer_common/sanitizer_placement_new.h"
+#include "sanitizer_common/sanitizer_stackdepot.h"
+#include "sanitizer_common/sanitizer_stacktrace.h"
+#include "sanitizer_common/sanitizer_symbolizer.h"
+
+#include <assert.h>
+#include <stdio.h>
+
+namespace __nsan {
+
+using namespace __sanitizer;
+
+Stats::Stats() {
+ CheckAndWarnings.Initialize(0);
+ TrackedLoads.Initialize(0);
+}
+
+Stats::~Stats() { Printf("deleting nsan stats\n"); }
+
+static uptr key(CheckTypeT CheckType, u32 StackId) {
+ return static_cast<uptr>(CheckType) +
+ StackId * static_cast<uptr>(CheckTypeT::kMaxCheckType);
+}
+
+template <typename MapT, typename VectorT, typename Fn>
+void UpdateEntry(CheckTypeT CheckTy, uptr PC, uptr BP, MapT *Map,
+ VectorT *Vector, Mutex *Mutex, Fn F) {
+ BufferedStackTrace Stack;
+ Stack.Unwind(PC, BP, nullptr, false);
+ u32 StackId = StackDepotPut(Stack);
+ typename MapT::Handle Handle(Map, key(CheckTy, StackId));
+ Lock L(Mutex);
+ if (Handle.created()) {
+ typename VectorT::value_type Entry;
+ Entry.StackId = StackId;
+ Entry.CheckTy = CheckTy;
+ F(Entry);
+ Vector->push_back(Entry);
+ } else {
+ auto &Entry = (*Vector)[*Handle];
+ F(Entry);
+ }
+}
+
+void Stats::addCheck(CheckTypeT CheckTy, uptr PC, uptr BP, double RelErr) {
+ UpdateEntry(CheckTy, PC, BP, &CheckAndWarningsMap, &CheckAndWarnings,
+ &CheckAndWarningsMutex, [RelErr](CheckAndWarningsValue &Entry) {
+ ++Entry.NumChecks;
+ if (RelErr > Entry.MaxRelativeError) {
+ Entry.MaxRelativeError = RelErr;
+ }
+ });
+}
+
+void Stats::addWarning(CheckTypeT CheckTy, uptr PC, uptr BP, double RelErr) {
+ UpdateEntry(CheckTy, PC, BP, &CheckAndWarningsMap, &CheckAndWarnings,
+ &CheckAndWarningsMutex, [RelErr](CheckAndWarningsValue &Entry) {
+ ++Entry.NumWarnings;
+ if (RelErr > Entry.MaxRelativeError) {
+ Entry.MaxRelativeError = RelErr;
+ }
+ });
+}
+
+void Stats::addInvalidLoadTrackingEvent(uptr PC, uptr BP) {
+ UpdateEntry(CheckTypeT::kLoad, PC, BP, &LoadTrackingMap, &TrackedLoads,
+ &TrackedLoadsMutex,
+ [](LoadTrackingValue &Entry) { ++Entry.NumInvalid; });
+}
+
+void Stats::addUnknownLoadTrackingEvent(uptr PC, uptr BP) {
+ UpdateEntry(CheckTypeT::kLoad, PC, BP, &LoadTrackingMap, &TrackedLoads,
+ &TrackedLoadsMutex,
+ [](LoadTrackingValue &Entry) { ++Entry.NumUnknown; });
+}
+
+static const char *CheckTypeDisplay(CheckTypeT CheckType) {
+ switch (CheckType) {
+ case CheckTypeT::kUnknown:
+ return "unknown";
+ case CheckTypeT::kRet:
+ return "return";
+ case CheckTypeT::kArg:
+ return "argument";
+ case CheckTypeT::kLoad:
+ return "load";
+ case CheckTypeT::kStore:
+ return "store";
+ case CheckTypeT::kInsert:
+ return "vector insert";
+ case CheckTypeT::kUser:
+ return "user-initiated";
+ case CheckTypeT::kFcmp:
+ return "fcmp";
+ case CheckTypeT::kMaxCheckType:
+ return "[max]";
+ }
+ assert(false && "unknown CheckType case");
+ return "";
+}
+
+void Stats::print() const {
+ {
+ Lock L(&CheckAndWarningsMutex);
+ for (const auto &Entry : CheckAndWarnings) {
+ Printf("warned %llu times out of %llu %s checks ", Entry.NumWarnings,
+ Entry.NumChecks, CheckTypeDisplay(Entry.CheckTy));
+ if (Entry.NumWarnings > 0) {
+ char RelErrBuf[64];
+ snprintf(RelErrBuf, sizeof(RelErrBuf) - 1, "%f",
+ Entry.MaxRelativeError * 100.0);
+ Printf("(max relative error: %s%%) ", RelErrBuf);
+ }
+ Printf("at:\n");
+ StackDepotGet(Entry.StackId).Print();
+ }
+ }
+
+ {
+ Lock L(&TrackedLoadsMutex);
+ u64 TotalInvalidLoadTracking = 0;
+ u64 TotalUnknownLoadTracking = 0;
+ for (const auto &Entry : TrackedLoads) {
+ TotalInvalidLoadTracking += Entry.NumInvalid;
+ TotalUnknownLoadTracking += Entry.NumUnknown;
+ Printf("invalid/unknown type for %llu/%llu loads at:\n", Entry.NumInvalid,
+ Entry.NumUnknown);
+ StackDepotGet(Entry.StackId).Print();
+ }
+ Printf(
+ "There were %llu/%llu floating-point loads where the shadow type was "
+ "invalid/unknown.\n",
+ TotalInvalidLoadTracking, TotalUnknownLoadTracking);
+ }
+}
+
+ALIGNED(64) static char StatsPlaceholder[sizeof(Stats)];
+Stats *nsan_stats = nullptr;
+
+void initializeStats() { nsan_stats = new (StatsPlaceholder) Stats(); }
+
+} // namespace __nsan
diff --git a/compiler-rt/lib/nsan/nsan_stats.h b/compiler-rt/lib/nsan/nsan_stats.h
new file mode 100644
index 0000000000000..7e8c7bb7bf12d
--- /dev/null
+++ b/compiler-rt/lib/nsan/nsan_stats.h
@@ -0,0 +1,92 @@
+//===-- nsan_stats.h --------------------------------------------*- C++- *-===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+//
+// This file is a part of NumericalStabilitySanitizer.
+//
+// NSan statistics. This class counts the number of checks per code location,
+// and is used to output statistics (typically when using
+// `disable_warnings=1,enable_check_stats=1,enable_warning_stats=1`).
+//===----------------------------------------------------------------------===//
+
+#ifndef NSAN_STATS_H
+#define NSAN_STATS_H
+
+#include "sanitizer_common/sanitizer_addrhashmap.h"
+#include "sanitizer_common/sanitizer_internal_defs.h"
+#include "sanitizer_common/sanitizer_mutex.h"
+
+namespace __nsan {
+
+enum class CheckTypeT {
+ kUnknown = 0,
+ kRet,
+ kArg,
+ kLoad,
+ kStore,
+ kInsert,
+ kUser, // User initiated.
+ kFcmp,
+ kMaxCheckType,
+};
+
+class Stats {
+public:
+ Stats();
+ ~Stats();
+
+ // Signal that we checked the instruction at the given address.
+ void addCheck(CheckTypeT CheckType, __sanitizer::uptr PC,
+ __sanitizer::uptr BP, double RelErr);
+ // Signal that we warned for the instruction at the given address.
+ void addWarning(CheckTypeT CheckType, __sanitizer::uptr PC,
+ __sanitizer::uptr BP, double RelErr);
+
+ // Signal that we detected a floating-point load where the shadow type was
+ // invalid.
+ void addInvalidLoadTrackingEvent(__sanitizer::uptr PC, __sanitizer::uptr BP);
+ // Signal that we detected a floating-point load where the shadow type was
+ // unknown but the value was nonzero.
+ void addUnknownLoadTrackingEvent(__sanitizer::uptr PC, __sanitizer::uptr BP);
+
+ void print() const;
+
+private:
+ using IndexMap = __sanitizer::AddrHashMap<__sanitizer::uptr, 11>;
+
+ struct CheckAndWarningsValue {
+ CheckTypeT CheckTy;
+ __sanitizer::u32 StackId = 0;
+ __sanitizer::u64 NumChecks = 0;
+ __sanitizer::u64 NumWarnings = 0;
+ // This is a bitcasted double. Doubles have the nice idea to be ordered as
+ // ints.
+ double MaxRelativeError = 0;
+ };
+ // Maps key(CheckType, StackId) to indices in CheckAndWarnings.
+ IndexMap CheckAndWarningsMap;
+ __sanitizer::InternalMmapVectorNoCtor<CheckAndWarningsValue> CheckAndWarnings;
+ mutable __sanitizer::Mutex CheckAndWarningsMutex;
+
+ struct LoadTrackingValue {
+ CheckTypeT CheckTy;
+ __sanitizer::u32 StackId = 0;
+ __sanitizer::u64 NumInvalid = 0;
+ __sanitizer::u64 NumUnknown = 0;
+ };
+ // Maps key(CheckTypeT::kLoad, StackId) to indices in TrackedLoads.
+ IndexMap LoadTrackingMap;
+ __sanitizer::InternalMmapVectorNoCtor<LoadTrackingValue> TrackedLoads;
+ mutable __sanitizer::Mutex TrackedLoadsMutex;
+};
+
+extern Stats *nsan_stats;
+void initializeStats();
+
+} // namespace __nsan
+
+#endif // NSAN_STATS_H
diff --git a/compiler-rt/lib/nsan/nsan_suppressions.cc b/compiler-rt/lib/nsan/nsan_suppressions.cc
new file mode 100644
index 0000000000000..9529a8e1be7d8
--- /dev/null
+++ b/compiler-rt/lib/nsan/nsan_suppressions.cc
@@ -0,0 +1,77 @@
+//===-- nsan_suppressions.cc ----------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#include "nsan_suppressions.h"
+
+#include "sanitizer_common/sanitizer_placement_new.h"
+#include "sanitizer_common/sanitizer_stacktrace.h"
+#include "sanitizer_common/sanitizer_symbolizer.h"
+
+#include "nsan_flags.h"
+
+// Can be overriden in frontend.
+SANITIZER_WEAK_DEFAULT_IMPL
+const char *__nsan_default_suppressions() { return 0; }
+
+namespace __nsan {
+
+const char *const kSuppressionFcmp = "fcmp";
+const char *const kSuppressionConsistency = "consistency";
+
+using namespace __sanitizer;
+
+ALIGNED(64) static char SuppressionPlaceholder[sizeof(SuppressionContext)];
+static SuppressionContext *SuppressionCtx = nullptr;
+
+// The order should match the enum CheckKind.
+static const char *kSuppressionTypes[] = {kSuppressionFcmp,
+ kSuppressionConsistency};
+
+void InitializeSuppressions() {
+ CHECK_EQ(nullptr, SuppressionCtx);
+ SuppressionCtx = new (SuppressionPlaceholder)
+ SuppressionContext(kSuppressionTypes, ARRAY_SIZE(kSuppressionTypes));
+ SuppressionCtx->ParseFromFile(flags().suppressions);
+ SuppressionCtx->Parse(__nsan_default_suppressions());
+}
+
+static Suppression *GetSuppressionForAddr(uptr Addr, const char *SupprType) {
+ Suppression *S = nullptr;
+
+ // Suppress by module name.
+ SuppressionContext *Suppressions = SuppressionCtx;
+ if (const char *ModuleName =
+ Symbolizer::GetOrInit()->GetModuleNameForPc(Addr)) {
+ if (Suppressions->Match(ModuleName, SupprType, &S))
+ return S;
+ }
+
+ // Suppress by file or function name.
+ SymbolizedStack *Frames = Symbolizer::GetOrInit()->SymbolizePC(Addr);
+ for (SymbolizedStack *Cur = Frames; Cur; Cur = Cur->next) {
+ if (Suppressions->Match(Cur->info.function, SupprType, &S) ||
+ Suppressions->Match(Cur->info.file, SupprType, &S)) {
+ break;
+ }
+ }
+ Frames->ClearAll();
+ return S;
+}
+
+Suppression *GetSuppressionForStack(const StackTrace *Stack, CheckKind K) {
+ for (uptr I = 0, E = Stack->size; I < E; I++) {
+ Suppression *S = GetSuppressionForAddr(
+ StackTrace::GetPreviousInstructionPc(Stack->trace[I]),
+ kSuppressionTypes[static_cast<int>(K)]);
+ if (S)
+ return S;
+ }
+ return nullptr;
+}
+
+} // end namespace __nsan
diff --git a/compiler-rt/lib/nsan/nsan_suppressions.h b/compiler-rt/lib/nsan/nsan_suppressions.h
new file mode 100644
index 0000000000000..0ac81fcf154da
--- /dev/null
+++ b/compiler-rt/lib/nsan/nsan_suppressions.h
@@ -0,0 +1,28 @@
+//===-- nsan_suppressions.h -------------------------------------*- C++ -*-===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+//
+// Defines nsan suppression rules.
+//===----------------------------------------------------------------------===//
+
+#ifndef NSAN_SUPPRESSIONS_H
+#define NSAN_SUPPRESSIONS_H
+
+#include "sanitizer_common/sanitizer_suppressions.h"
+
+namespace __nsan {
+
+enum class CheckKind { Fcmp, Consistency };
+
+void InitializeSuppressions();
+
+__sanitizer::Suppression *
+GetSuppressionForStack(const __sanitizer::StackTrace *Stack, CheckKind K);
+
+} // namespace __nsan
+
+#endif
diff --git a/compiler-rt/lib/nsan/tests/CMakeLists.txt b/compiler-rt/lib/nsan/tests/CMakeLists.txt
new file mode 100644
index 0000000000000..e6674e12b548b
--- /dev/null
+++ b/compiler-rt/lib/nsan/tests/CMakeLists.txt
@@ -0,0 +1,53 @@
+include(CompilerRTCompile)
+
+set(NSAN_UNITTEST_CFLAGS
+ ${COMPILER_RT_UNITTEST_CFLAGS}
+ ${COMPILER_RT_GTEST_CFLAGS}
+ -I${COMPILER_RT_SOURCE_DIR}/lib/
+ -DSANITIZER_COMMON_REDEFINE_BUILTINS_IN_STD
+ -O2
+ -g
+ -fno-omit-frame-pointer)
+
+file(GLOB NSAN_HEADERS ../*.h)
+set(NSAN_UNITTESTS
+ NSanUnitTest.cpp
+ nsan_unit_test_main.cpp)
+
+add_custom_target(NsanUnitTests)
+
+# set(NSAN_UNITTEST_LINK_FLAGS ${COMPILER_RT_UNITTEST_LINK_FLAGS} -ldl)
+# list(APPEND NSAN_UNITTEST_LINK_FLAGS --driver-mode=g++)
+
+if(COMPILER_RT_DEFAULT_TARGET_ARCH IN_LIST NSAN_SUPPORTED_ARCH)
+ # NSan unit tests are only run on the host machine.
+ set(arch ${COMPILER_RT_DEFAULT_TARGET_ARCH})
+
+ set(NSAN_TEST_RUNTIME RTNsanTest.${arch})
+
+ set(NSAN_TEST_RUNTIME_OBJECTS
+ $<TARGET_OBJECTS:RTNsan.${arch}>
+ $<TARGET_OBJECTS:RTInterception.${arch}>
+ $<TARGET_OBJECTS:RTSanitizerCommon.${arch}>
+ $<TARGET_OBJECTS:RTSanitizerCommonLibc.${arch}>
+ $<TARGET_OBJECTS:RTSanitizerCommonSymbolizer.${arch}>)
+
+ add_library(${NSAN_TEST_RUNTIME} STATIC
+ ${NSAN_TEST_RUNTIME_OBJECTS})
+
+ set_target_properties(${NSAN_TEST_RUNTIME} PROPERTIES
+ ARCHIVE_OUTPUT_DIRECTORY ${CMAKE_CURRENT_BINARY_DIR}
+ FOLDER "Compiler-RT Runtime tests")
+
+ set(NsanTestObjects)
+ generate_compiler_rt_tests(NsanTestObjects
+ NsanUnitTests "Nsan-${arch}-Test" ${arch}
+ SOURCES ${NSAN_UNITTESTS} ${COMPILER_RT_GTEST_SOURCE}
+ RUNTIME ${NSAN_TEST_RUNTIME}
+ DEPS ${NSAN_UNIT_TEST_HEADERS}
+ CFLAGS ${NSAN_UNITTEST_CFLAGS}
+ LINK_FLAGS ${NSAN_UNITTEST_LINK_FLAGS})
+ set_target_properties(NsanUnitTests PROPERTIES
+ RUNTIME_OUTPUT_DIRECTORY ${CMAKE_CURRENT_BINARY_DIR})
+endif()
+
diff --git a/compiler-rt/lib/nsan/tests/NSanUnitTest.cpp b/compiler-rt/lib/nsan/tests/NSanUnitTest.cpp
new file mode 100644
index 0000000000000..3c6b505aaf7a1
--- /dev/null
+++ b/compiler-rt/lib/nsan/tests/NSanUnitTest.cpp
@@ -0,0 +1,67 @@
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+
+// Do not attempt to use LLVM ostream etc from gtest.
+// #define GTEST_NO_LLVM_SUPPORT 1
+
+#include "../nsan.h"
+#include "gtest/gtest.h"
+
+#include <cmath>
+
+namespace __nsan {
+
+template <typename FT, auto next> void TestFT() {
+ // Basic local tests anchored at 0.0.
+ ASSERT_EQ(getULPDiff<FT>(0.0, 0.0), 0);
+ ASSERT_EQ(getULPDiff<FT>(-0.0, 0.0), 0);
+ ASSERT_EQ(getULPDiff<FT>(next(-0.0, -1.0), 0.0), 1);
+ ASSERT_EQ(getULPDiff<FT>(next(0.0, 1.0), -0.0), 1);
+ ASSERT_EQ(getULPDiff<FT>(next(-0.0, -1.0), next(0.0, 1.0)), 2);
+ // Basic local tests anchored at 2.0.
+ ASSERT_EQ(getULPDiff<FT>(next(2.0, 1.0), 2.0), 1);
+ ASSERT_EQ(getULPDiff<FT>(next(2.0, 3.0), 2.0), 1);
+ ASSERT_EQ(getULPDiff<FT>(next(2.0, 1.0), next(2.0, 3.0)), 2);
+
+ ASSERT_NE(getULPDiff<FT>(-0.01, 0.01), kMaxULPDiff);
+
+ // Basic local tests anchored at a random number.
+ const FT X = 4863.5123;
+ const FT To = 2 * X;
+ FT Y = X;
+ ASSERT_EQ(getULPDiff<FT>(X, Y), 0);
+ ASSERT_EQ(getULPDiff<FT>(-X, -Y), 0);
+ Y = next(Y, To);
+ ASSERT_EQ(getULPDiff<FT>(X, Y), 1);
+ ASSERT_EQ(getULPDiff<FT>(-X, -Y), 1);
+ Y = next(Y, To);
+ ASSERT_EQ(getULPDiff<FT>(X, Y), 2);
+ ASSERT_EQ(getULPDiff<FT>(-X, -Y), 2);
+ Y = next(Y, To);
+ ASSERT_EQ(getULPDiff<FT>(X, Y), 3);
+ ASSERT_EQ(getULPDiff<FT>(-X, -Y), 3);
+
+ // Values with larger differences.
+ static constexpr const __sanitizer::u64 MantissaSize =
+ __sanitizer::u64{1} << FTInfo<FT>::kMantissaBits;
+ ASSERT_EQ(getULPDiff<FT>(1.0, next(2.0, 1.0)), MantissaSize - 1);
+ ASSERT_EQ(getULPDiff<FT>(1.0, 2.0), MantissaSize);
+ ASSERT_EQ(getULPDiff<FT>(1.0, next(2.0, 3.0)), MantissaSize + 1);
+ ASSERT_EQ(getULPDiff<FT>(1.0, 3.0), (3 * MantissaSize) / 2);
+}
+
+TEST(NSanTest, Float) { TestFT<float, nextafterf>(); }
+
+TEST(NSanTest, Double) {
+ TestFT<double, static_cast<double (*)(double, double)>(nextafter)>();
+}
+
+TEST(NSanTest, Float128) {
+ // Very basic tests. FIXME: improve when we have nextafter<__float128>.
+ ASSERT_EQ(getULPDiff<__float128>(0.0, 0.0), 0);
+ ASSERT_EQ(getULPDiff<__float128>(-0.0, 0.0), 0);
+ ASSERT_NE(getULPDiff<__float128>(-0.01, 0.01), kMaxULPDiff);
+}
+
+} // end namespace __nsan
diff --git a/compiler-rt/lib/nsan/tests/nsan_unit_test_main.cpp b/compiler-rt/lib/nsan/tests/nsan_unit_test_main.cpp
new file mode 100644
index 0000000000000..86ad5bbfef17e
--- /dev/null
+++ b/compiler-rt/lib/nsan/tests/nsan_unit_test_main.cpp
@@ -0,0 +1,18 @@
+//===-- nsan_unit_test_main.cpp -------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+//
+// This file is a part of NSan.
+//
+//===----------------------------------------------------------------------===//
+#include "gtest/gtest.h"
+
+int main(int argc, char **argv) {
+ testing::GTEST_FLAG(death_test_style) = "threadsafe";
+ testing::InitGoogleTest(&argc, argv);
+ return RUN_ALL_TESTS();
+}
diff --git a/compiler-rt/test/nsan/CMakeLists.txt b/compiler-rt/test/nsan/CMakeLists.txt
new file mode 100644
index 0000000000000..fb73587574fba
--- /dev/null
+++ b/compiler-rt/test/nsan/CMakeLists.txt
@@ -0,0 +1,19 @@
+set(NSAN_LIT_SOURCE_DIR ${CMAKE_CURRENT_SOURCE_DIR})
+
+set(NSAN_TESTSUITES)
+set(NSAN_TEST_DEPS ${SANITIZER_COMMON_LIT_TEST_DEPS} nsan)
+
+if(COMPILER_RT_INCLUDE_TESTS AND
+ COMPILER_RT_LIBCXX_PATH AND
+ COMPILER_RT_LIBCXXABI_PATH)
+ configure_lit_site_cfg(
+ ${CMAKE_CURRENT_SOURCE_DIR}/Unit/lit.site.cfg.py.in
+ ${CMAKE_CURRENT_BINARY_DIR}/Unit/lit.site.cfg.py)
+ list(APPEND NSAN_TEST_DEPS NsanUnitTests)
+ list(APPEND NSAN_TESTSUITES ${CMAKE_CURRENT_BINARY_DIR}/Unit)
+endif()
+
+add_lit_testsuite(check-nsan "Running the numerical stability sanitizer tests"
+ ${NSAN_TESTSUITES}
+ DEPENDS ${NSAN_TEST_DEPS}
+ )
diff --git a/compiler-rt/test/nsan/Unit/lit.site.cfg.py.in b/compiler-rt/test/nsan/Unit/lit.site.cfg.py.in
new file mode 100644
index 0000000000000..f8243f8ccb41b
--- /dev/null
+++ b/compiler-rt/test/nsan/Unit/lit.site.cfg.py.in
@@ -0,0 +1,10 @@
+ at LIT_SITE_CFG_IN_HEADER@
+
+# Load common config for all compiler-rt unit tests.
+lit_config.load_config(config, "@COMPILER_RT_BINARY_DIR@/unittests/lit.common.unit.configured")
+
+# Setup config name.
+config.name = 'NumericalStabilitySanitizer-Unit'
+
+config.test_exec_root = "@COMPILER_RT_BINARY_DIR@/lib/nsan/tests"
+config.test_source_root = config.test_exec_root
diff --git a/compiler-rt/test/nsan/lit.cfg.py b/compiler-rt/test/nsan/lit.cfg.py
new file mode 100644
index 0000000000000..8b137891791fe
--- /dev/null
+++ b/compiler-rt/test/nsan/lit.cfg.py
@@ -0,0 +1 @@
+
diff --git a/compiler-rt/test/nsan/lit.site.cfg.py.in b/compiler-rt/test/nsan/lit.site.cfg.py.in
new file mode 100644
index 0000000000000..69fd057e75748
--- /dev/null
+++ b/compiler-rt/test/nsan/lit.site.cfg.py.in
@@ -0,0 +1,14 @@
+ at LIT_SITE_CFG_IN_HEADER@
+
+# Tool-specific config options.
+config.name_suffix = "- at CONFIG_NAME@"
+config.target_cflags = "@NSAN_TEST_TARGET_CFLAGS@"
+config.target_arch = "@NSAN_TEST_TARGET_ARCH@"
+config.use_lld = @NSAN_TEST_USE_LLD@
+config.use_thinlto = @NSAN_TEST_USE_THINLTO@
+
+# Load common config for all compiler-rt lit tests.
+lit_config.load_config(config, "@COMPILER_RT_BINARY_DIR@/test/lit.common.configured")
+
+# Load tool-specific config that would do the real work.
+lit_config.load_config(config, "@NSAN_LIT_SOURCE_DIR@/lit.cfg.py")
More information about the llvm-commits
mailing list