[llvm] [LLVM][Instrumentation] Add numerical sanitizer (PR #85916)
Vitaly Buka via llvm-commits
llvm-commits at lists.llvm.org
Tue Jun 4 16:21:30 PDT 2024
================
@@ -0,0 +1,2233 @@
+//===-- NumericalStabilitySanitizer.cpp -----------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+//
+// This file is a part of NumericalStabilitySanitizer.
+//
+//===----------------------------------------------------------------------===//
+
+#include "llvm/Transforms/Instrumentation/NumericalStabilitySanitizer.h"
+
+#include <cstdint>
+#include <unordered_map>
+
+#include "llvm/ADT/SmallPtrSet.h"
+#include "llvm/ADT/SmallString.h"
+#include "llvm/ADT/SmallVector.h"
+#include "llvm/ADT/Statistic.h"
+#include "llvm/ADT/StringExtras.h"
+#include "llvm/Analysis/CaptureTracking.h"
+#include "llvm/Analysis/TargetLibraryInfo.h"
+#include "llvm/Analysis/ValueTracking.h"
+#include "llvm/IR/DataLayout.h"
+#include "llvm/IR/Function.h"
+#include "llvm/IR/IRBuilder.h"
+#include "llvm/IR/IntrinsicInst.h"
+#include "llvm/IR/Intrinsics.h"
+#include "llvm/IR/LLVMContext.h"
+#include "llvm/IR/Metadata.h"
+#include "llvm/IR/Module.h"
+#include "llvm/IR/Type.h"
+#include "llvm/InitializePasses.h"
+#include "llvm/ProfileData/InstrProf.h"
+#include "llvm/Support/CommandLine.h"
+#include "llvm/Support/Debug.h"
+#include "llvm/Support/MathExtras.h"
+#include "llvm/Support/Regex.h"
+#include "llvm/Support/raw_ostream.h"
+#include "llvm/Transforms/Instrumentation.h"
+#include "llvm/Transforms/Utils/BasicBlockUtils.h"
+#include "llvm/Transforms/Utils/EscapeEnumerator.h"
+#include "llvm/Transforms/Utils/Local.h"
+#include "llvm/Transforms/Utils/ModuleUtils.h"
+
+using namespace llvm;
+
+#define DEBUG_TYPE "nsan"
+
+STATISTIC(NumInstrumentedFTLoads,
+ "Number of instrumented floating-point loads");
+
+STATISTIC(NumInstrumentedFTCalls,
+ "Number of instrumented floating-point calls");
+STATISTIC(NumInstrumentedFTRets,
+ "Number of instrumented floating-point returns");
+STATISTIC(NumInstrumentedFTStores,
+ "Number of instrumented floating-point stores");
+STATISTIC(NumInstrumentedNonFTStores,
+ "Number of instrumented non floating-point stores");
+STATISTIC(
+ NumInstrumentedNonFTMemcpyStores,
+ "Number of instrumented non floating-point stores with memcpy semantics");
+STATISTIC(NumInstrumentedFCmp, "Number of instrumented fcmps");
+
+// Using smaller shadow types types can help improve speed. For example, `dlq`
+// is 3x slower to 5x faster in opt mode and 2-6x faster in dbg mode compared to
+// `dqq`.
+static cl::opt<std::string> ClShadowMapping(
+ "nsan-shadow-type-mapping", cl::init("dqq"),
+ cl::desc("One shadow type id for each of `float`, `double`, `long double`. "
+ "`d`,`l`,`q`,`e` mean double, x86_fp80, fp128 (quad) and "
+ "ppc_fp128 (extended double) respectively. The default is to "
+ "shadow `float` as `double`, and `double` and `x86_fp80` as "
+ "`fp128`"),
+ cl::Hidden);
+
+static cl::opt<bool>
+ ClInstrumentFCmp("nsan-instrument-fcmp", cl::init(true),
+ cl::desc("Instrument floating-point comparisons"),
+ cl::Hidden);
+
+static cl::opt<std::string> ClCheckFunctionsFilter(
+ "check-functions-filter",
+ cl::desc("Only emit checks for arguments of functions "
+ "whose names match the given regular expression"),
+ cl::value_desc("regex"));
+
+static cl::opt<bool> ClTruncateFCmpEq(
+ "nsan-truncate-fcmp-eq", cl::init(true),
+ cl::desc(
+ "This flag controls the behaviour of fcmp equality comparisons:"
+ "For equality comparisons such as `x == 0.0f`, we can perform the "
+ "shadow check in the shadow (`x_shadow == 0.0) == (x == 0.0f)`) or app "
+ " domain (`(trunc(x_shadow) == 0.0f) == (x == 0.0f)`). This helps "
+ "catch the case when `x_shadow` is accurate enough (and therefore "
+ "close enough to zero) so that `trunc(x_shadow)` is zero even though "
+ "both `x` and `x_shadow` are not. "),
+ cl::Hidden);
+
+// When there is external, uninstrumented code writing to memory, the shadow
+// memory can get out of sync with the application memory. Enabling this flag
+// emits consistency checks for loads to catch this situation.
+// When everything is instrumented, this is not strictly necessary because any
+// load should have a corresponding store, but can help debug cases when the
+// framework did a bad job at tracking shadow memory modifications by failing on
+// load rather than store.
+// FIXME: provide a way to resume computations from the FT value when the load
+// is inconsistent. This ensures that further computations are not polluted.
+static cl::opt<bool> ClCheckLoads("nsan-check-loads", cl::init(false),
+ cl::desc("Check floating-point load"),
+ cl::Hidden);
+
+static cl::opt<bool> ClCheckStores("nsan-check-stores", cl::init(true),
+ cl::desc("Check floating-point stores"),
+ cl::Hidden);
+
+static cl::opt<bool> ClCheckRet("nsan-check-ret", cl::init(true),
+ cl::desc("Check floating-point return values"),
+ cl::Hidden);
+
+// LLVM may store constant floats as bitcasted ints.
+// It's not really necessary to shadow such stores,
+// if the shadow value is unknown the framework will re-extend it on load
+// anyway. Moreover, because of size collisions (e.g. bf16 vs f16) it is
+// impossible to determine the floating-point type based on the size.
+// However, for debugging purposes it can be useful to model such stores.
+static cl::opt<bool> ClPropagateNonFTConstStoresAsFT(
+ "nsan-propagate-non-ft-const-stores-as-ft", cl::init(false),
+ cl::desc(
+ "Propagate non floating-point const stores as floating point values."
+ "For debugging purposes only"),
+ cl::Hidden);
+
+static constexpr StringLiteral kNsanModuleCtorName("nsan.module_ctor");
+static constexpr StringLiteral kNsanInitName("__nsan_init");
+
+// The following values must be kept in sync with the runtime.
+static constexpr const int kShadowScale = 2;
+static constexpr const int kMaxVectorWidth = 8;
+static constexpr const int kMaxNumArgs = 128;
+static constexpr const int kMaxShadowTypeSizeBytes = 16; // fp128
+
+namespace {
+
+// Defines the characteristics (type id, type, and floating-point semantics)
+// attached for all possible shadow types.
+class ShadowTypeConfig {
+public:
+ static std::unique_ptr<ShadowTypeConfig> fromNsanTypeId(char TypeId);
+
+ // The LLVM Type corresponding to the shadow type.
+ virtual Type *getType(LLVMContext &Context) const = 0;
+
+ // The nsan type id of the shadow type (`d`, `l`, `q`, ...).
+ virtual char getNsanTypeId() const = 0;
+
+ virtual ~ShadowTypeConfig() = default;
+};
+
+template <char NsanTypeId>
+class ShadowTypeConfigImpl : public ShadowTypeConfig {
+public:
+ char getNsanTypeId() const override { return NsanTypeId; }
+ static constexpr const char kNsanTypeId = NsanTypeId;
+};
+
+// `double` (`d`) shadow type.
+class F64ShadowConfig : public ShadowTypeConfigImpl<'d'> {
+ Type *getType(LLVMContext &Context) const override {
+ return Type::getDoubleTy(Context);
+ }
+};
+
+// `x86_fp80` (`l`) shadow type: X86 long double.
+class F80ShadowConfig : public ShadowTypeConfigImpl<'l'> {
+ Type *getType(LLVMContext &Context) const override {
+ return Type::getX86_FP80Ty(Context);
+ }
+};
+
+// `fp128` (`q`) shadow type.
+class F128ShadowConfig : public ShadowTypeConfigImpl<'q'> {
+ Type *getType(LLVMContext &Context) const override {
+ return Type::getFP128Ty(Context);
+ }
+};
+
+// `ppc_fp128` (`e`) shadow type: IBM extended double with 106 bits of mantissa.
+class PPC128ShadowConfig : public ShadowTypeConfigImpl<'e'> {
+ Type *getType(LLVMContext &Context) const override {
+ return Type::getPPC_FP128Ty(Context);
+ }
+};
+
+// Creates a ShadowTypeConfig given its type id.
+std::unique_ptr<ShadowTypeConfig>
+ShadowTypeConfig::fromNsanTypeId(const char TypeId) {
+ switch (TypeId) {
+ case F64ShadowConfig::kNsanTypeId:
+ return std::make_unique<F64ShadowConfig>();
+ case F80ShadowConfig::kNsanTypeId:
+ return std::make_unique<F80ShadowConfig>();
+ case F128ShadowConfig::kNsanTypeId:
+ return std::make_unique<F128ShadowConfig>();
+ case PPC128ShadowConfig::kNsanTypeId:
+ return std::make_unique<PPC128ShadowConfig>();
+ }
+ report_fatal_error("nsan: invalid shadow type id '" + Twine(TypeId) + "'");
+}
+
+// An enum corresponding to shadow value types. Used as indices in arrays, so
+// not an `enum class`.
+enum FTValueType { kFloat, kDouble, kLongDouble, kNumValueTypes };
+
+// If `FT` corresponds to a primitive FTValueType, return it.
+static std::optional<FTValueType> ftValueTypeFromType(Type *FT) {
+ if (FT->isFloatTy())
+ return kFloat;
+ if (FT->isDoubleTy())
+ return kDouble;
+ if (FT->isX86_FP80Ty())
+ return kLongDouble;
+ return {};
+}
+
+// Returns the LLVM type for an FTValueType.
+static Type *typeFromFTValueType(FTValueType VT, LLVMContext &Context) {
+ switch (VT) {
+ case kFloat:
+ return Type::getFloatTy(Context);
+ case kDouble:
+ return Type::getDoubleTy(Context);
+ case kLongDouble:
+ return Type::getX86_FP80Ty(Context);
+ case kNumValueTypes:
+ return nullptr;
+ }
+}
+
+// Returns the type name for an FTValueType.
+static const char *typeNameFromFTValueType(FTValueType VT) {
+ switch (VT) {
+ case kFloat:
+ return "float";
+ case kDouble:
+ return "double";
+ case kLongDouble:
+ return "longdouble";
+ case kNumValueTypes:
+ return nullptr;
+ }
+}
+
+// A specific mapping configuration of application type to shadow type for nsan
+// (see -nsan-shadow-mapping flag).
+class MappingConfig {
+public:
+ bool initialize(LLVMContext *C) {
+ if (ClShadowMapping.size() != 3) {
+ errs() << "Invalid nsan mapping: " << ClShadowMapping << "\n";
+ }
+ Context = C;
+ unsigned ShadowTypeSizeBits[kNumValueTypes];
+ for (int VT = 0; VT < kNumValueTypes; ++VT) {
+ auto Config = ShadowTypeConfig::fromNsanTypeId(ClShadowMapping[VT]);
+ if (Config == nullptr)
+ return false;
+ const unsigned AppTypeSize =
+ typeFromFTValueType(static_cast<FTValueType>(VT), *C)
+ ->getScalarSizeInBits();
+ const unsigned ShadowTypeSize =
+ Config->getType(*C)->getScalarSizeInBits();
+ // Check that the shadow type size is at most kShadowScale times the
+ // application type size, so that shadow memory compoutations are valid.
+ if (ShadowTypeSize > kShadowScale * AppTypeSize)
+ report_fatal_error("Invalid nsan mapping f" + Twine(AppTypeSize) +
+ "->f" + Twine(ShadowTypeSize) +
+ ": The shadow type size should be at most " +
+ Twine(kShadowScale) +
+ " times the application type size");
+ ShadowTypeSizeBits[VT] = ShadowTypeSize;
+ Configs[VT] = std::move(Config);
+ }
+
+ // Check that the mapping is monotonous. This is required because if one
+ // does an fpextend of `float->long double` in application code, nsan is
+ // going to do an fpextend of `shadow(float) -> shadow(long double)` in
+ // shadow code. This will fail in `qql` mode, since nsan would be
+ // fpextending `f128->long`, which is invalid.
+ // FIXME: Relax this.
+ if (ShadowTypeSizeBits[kFloat] > ShadowTypeSizeBits[kDouble] ||
+ ShadowTypeSizeBits[kDouble] > ShadowTypeSizeBits[kLongDouble])
+ report_fatal_error("Invalid nsan mapping: { float->f" +
+ Twine(ShadowTypeSizeBits[kFloat]) + "; double->f" +
+ Twine(ShadowTypeSizeBits[kDouble]) +
+ "; long double->f" +
+ Twine(ShadowTypeSizeBits[kLongDouble]) + " }");
+ return true;
+ }
+
+ const ShadowTypeConfig &byValueType(FTValueType VT) const {
+ assert(VT < FTValueType::kNumValueTypes && "invalid value type");
+ return *Configs[VT];
+ }
+
+ // Returns the extended shadow type for a given application type.
+ Type *getExtendedFPType(Type *FT) const {
+ if (const auto VT = ftValueTypeFromType(FT))
+ return Configs[*VT]->getType(*Context);
+ if (FT->isVectorTy()) {
+ auto *VecTy = cast<VectorType>(FT);
+ // FIXME: add support for scalable vector types.
+ if (VecTy->isScalableTy())
+ return nullptr;
+ Type *ExtendedScalar = getExtendedFPType(VecTy->getElementType());
+ return ExtendedScalar
+ ? VectorType::get(ExtendedScalar, VecTy->getElementCount())
+ : nullptr;
+ }
+ return nullptr;
+ }
+
+private:
+ LLVMContext *Context = nullptr;
+ std::unique_ptr<ShadowTypeConfig> Configs[FTValueType::kNumValueTypes];
+};
+
+// The memory extents of a type specifies how many elements of a given
+// FTValueType needs to be stored when storing this type.
+struct MemoryExtents {
+ FTValueType ValueType;
+ uint64_t NumElts;
+};
+static MemoryExtents getMemoryExtentsOrDie(Type *FT) {
+ if (const auto VT = ftValueTypeFromType(FT))
+ return {*VT, 1};
+ if (FT->isVectorTy()) {
+ auto *VecTy = cast<VectorType>(FT);
+ const auto ScalarExtents = getMemoryExtentsOrDie(VecTy->getElementType());
+ return {ScalarExtents.ValueType,
+ ScalarExtents.NumElts * VecTy->getElementCount().getFixedValue()};
+ }
+ llvm_unreachable("invalid value type");
+}
+
+// The location of a check. Passed as parameters to runtime checking functions.
+class CheckLoc {
+public:
+ // Creates a location that references an application memory location.
+ static CheckLoc makeStore(Value *Address) {
+ CheckLoc Result(kStore);
+ Result.Address = Address;
+ return Result;
+ }
+ static CheckLoc makeLoad(Value *Address) {
+ CheckLoc Result(kLoad);
+ Result.Address = Address;
+ return Result;
+ }
+
+ // Creates a location that references an argument, given by id.
+ static CheckLoc makeArg(int ArgId) {
+ CheckLoc Result(kArg);
+ Result.ArgId = ArgId;
+ return Result;
+ }
+
+ // Creates a location that references the return value of a function.
+ static CheckLoc makeRet() { return CheckLoc(kRet); }
+
+ // Creates a location that references a vector insert.
+ static CheckLoc makeInsert() { return CheckLoc(kInsert); }
+
+ // Returns the CheckType of location this refers to, as an integer-typed LLVM
+ // IR value.
+ Value *getType(LLVMContext &C) const {
+ return ConstantInt::get(Type::getInt32Ty(C), static_cast<int>(CheckTy));
+ }
+
+ // Returns a CheckType-specific value representing details of the location
+ // (e.g. application address for loads or stores), as an `IntptrTy`-typed LLVM
+ // IR value.
+ Value *getValue(Type *IntptrTy, IRBuilder<> &Builder) const {
+ switch (CheckTy) {
+ case kUnknown:
+ llvm_unreachable("unknown type");
+ case kRet:
+ case kInsert:
+ return ConstantInt::get(IntptrTy, 0);
+ case kArg:
+ return ConstantInt::get(IntptrTy, ArgId);
+ case kLoad:
+ case kStore:
+ return Builder.CreatePtrToInt(Address, IntptrTy);
+ }
+ }
+
+private:
+ // Must be kept in sync with the runtime.
+ enum CheckType {
+ kUnknown = 0,
+ kRet,
+ kArg,
+ kLoad,
+ kStore,
+ kInsert,
+ };
+ explicit CheckLoc(CheckType CheckTy) : CheckTy(CheckTy) {}
+
+ const CheckType CheckTy;
+ Value *Address = nullptr;
+ int ArgId = -1;
+};
+
+// A map of LLVM IR values to shadow LLVM IR values.
+class ValueToShadowMap {
+public:
+ explicit ValueToShadowMap(MappingConfig *Config) : Config(Config) {}
+
+ // Sets the shadow value for a value. Asserts that the value does not already
+ // have a value.
+ void setShadow(Value *V, Value *Shadow) {
+ assert(V);
+ assert(Shadow);
+ [[maybe_unused]] const bool Inserted = Map.emplace(V, Shadow).second;
+ LLVM_DEBUG({
+ if (!Inserted) {
+ if (const auto *const I = dyn_cast<Instruction>(V))
+ errs() << I->getFunction()->getName() << ": ";
+ errs() << "duplicate shadow (" << V << "): ";
+ V->dump();
+ }
+ });
+ assert(Inserted && "duplicate shadow");
+ }
+
+ // Returns true if the value already has a shadow (including if the value is a
+ // constant). If true, calling getShadow() is valid.
+ bool hasShadow(Value *V) const {
+ return isa<Constant>(V) || (Map.find(V) != Map.end());
+ }
+
+ // Returns the shadow value for a given value. Asserts that the value has
+ // a shadow value. Lazily creates shadows for constant values.
+ Value *getShadow(Value *V) const {
+ if (Constant *C = dyn_cast<Constant>(V))
+ return getShadowConstant(C);
+ const auto ShadowValIt = Map.find(V);
+ assert(ShadowValIt != Map.end() && "shadow val does not exist");
+ assert(ShadowValIt->second && "shadow val is null");
+ return ShadowValIt->second;
+ }
+
+ bool empty() const { return Map.empty(); }
+
+private:
+ // Extends a constant application value to its shadow counterpart.
+ APFloat extendConstantFP(APFloat CV, const fltSemantics &To) const {
+ bool LosesInfo = false;
+ CV.convert(To, APFloatBase::rmTowardZero, &LosesInfo);
+ return CV;
+ }
+
+ // Returns the shadow constant for the given application constant.
+ Constant *getShadowConstant(Constant *C) const {
+ if (UndefValue *U = dyn_cast<UndefValue>(C)) {
+ return UndefValue::get(Config->getExtendedFPType(U->getType()));
+ }
+ if (ConstantFP *CFP = dyn_cast<ConstantFP>(C)) {
+ // Floating-point constants.
+ Type *Ty = Config->getExtendedFPType(CFP->getType());
+ return ConstantFP::get(
+ Ty, extendConstantFP(CFP->getValueAPF(), Ty->getFltSemantics()));
+ }
+ // Vector, array, or aggregate constants.
+ if (C->getType()->isVectorTy()) {
+ SmallVector<Constant *, 8> Elements;
+ for (int I = 0, E = cast<VectorType>(C->getType())
+ ->getElementCount()
+ .getFixedValue();
+ I < E; ++I)
+ Elements.push_back(getShadowConstant(C->getAggregateElement(I)));
+ return ConstantVector::get(Elements);
+ }
+ llvm_unreachable("unimplemented");
+ }
+
+ MappingConfig *const Config;
+ std::unordered_map<Value *, Value *> Map;
+};
+
+/// Instantiating NumericalStabilitySanitizer inserts the nsan runtime library
+/// API function declarations into the module if they don't exist already.
+/// Instantiating ensures the __nsan_init function is in the list of global
+/// constructors for the module.
+class NumericalStabilitySanitizer {
+public:
+ bool sanitizeFunction(Function &F, const TargetLibraryInfo &TLI);
+
+private:
+ void initialize(Module &M);
+ bool instrumentMemIntrinsic(MemIntrinsic *MI);
+ void maybeAddSuffixForNsanInterface(CallBase *CI);
+ bool addrPointsToConstantData(Value *Addr);
+ void maybeCreateShadowValue(Instruction &Root, const TargetLibraryInfo &TLI,
+ ValueToShadowMap &Map);
+ Value *createShadowValueWithOperandsAvailable(Instruction &Inst,
+ const TargetLibraryInfo &TLI,
+ const ValueToShadowMap &Map);
+ PHINode *maybeCreateShadowPhi(PHINode &Phi, const TargetLibraryInfo &TLI);
+ void createShadowArguments(Function &F, const TargetLibraryInfo &TLI,
+ ValueToShadowMap &Map);
+
+ void populateShadowStack(CallBase &CI, const TargetLibraryInfo &TLI,
+ const ValueToShadowMap &Map);
+
+ void propagateShadowValues(Instruction &Inst, const TargetLibraryInfo &TLI,
+ const ValueToShadowMap &Map);
+ Value *emitCheck(Value *V, Value *ShadowV, IRBuilder<> &Builder,
+ CheckLoc Loc);
+ Value *emitCheckInternal(Value *V, Value *ShadowV, IRBuilder<> &Builder,
+ CheckLoc Loc);
+ void emitFCmpCheck(FCmpInst &FCmp, const ValueToShadowMap &Map);
+
+ // Value creation handlers.
+ Value *handleLoad(LoadInst &Load, Type *VT, Type *ExtendedVT);
+ Value *handleTrunc(FPTruncInst &Trunc, Type *VT, Type *ExtendedVT,
+ const ValueToShadowMap &Map);
+ Value *handleExt(FPExtInst &Ext, Type *VT, Type *ExtendedVT,
+ const ValueToShadowMap &Map);
+ Value *handleCallBase(CallBase &Call, Type *VT, Type *ExtendedVT,
+ const TargetLibraryInfo &TLI,
+ const ValueToShadowMap &Map, IRBuilder<> &Builder);
+ Value *maybeHandleKnownCallBase(CallBase &Call, Type *VT, Type *ExtendedVT,
+ const TargetLibraryInfo &TLI,
+ const ValueToShadowMap &Map,
+ IRBuilder<> &Builder);
+
+ // Value propagation handlers.
+ void propagateFTStore(StoreInst &Store, Type *VT, Type *ExtendedVT,
+ const ValueToShadowMap &Map);
+ void propagateNonFTStore(StoreInst &Store, Type *VT,
+ const ValueToShadowMap &Map);
+
+ MappingConfig Config;
+ LLVMContext *Context = nullptr;
+ IntegerType *IntptrTy = nullptr;
+ FunctionCallee NsanGetShadowPtrForStore[FTValueType::kNumValueTypes];
+ FunctionCallee NsanGetShadowPtrForLoad[FTValueType::kNumValueTypes];
+ FunctionCallee NsanCheckValue[FTValueType::kNumValueTypes];
+ FunctionCallee NsanFCmpFail[FTValueType::kNumValueTypes];
+ FunctionCallee NsanCopyValues;
+ FunctionCallee NsanSetValueUnknown;
+ FunctionCallee NsanGetRawShadowTypePtr;
+ FunctionCallee NsanGetRawShadowPtr;
+ GlobalValue *NsanShadowRetTag;
+
+ Type *NsanShadowRetType;
+ GlobalValue *NsanShadowRetPtr;
+
+ GlobalValue *NsanShadowArgsTag;
+
+ Type *NsanShadowArgsType;
+ GlobalValue *NsanShadowArgsPtr;
+
+ std::optional<Regex> CheckFunctionsFilter;
+};
+
+void insertModuleCtor(Module &M) {
+ getOrCreateSanitizerCtorAndInitFunctions(
+ M, kNsanModuleCtorName, kNsanInitName, /*InitArgTypes=*/{},
+ /*InitArgs=*/{},
+ // This callback is invoked when the functions are created the first
+ // time. Hook them into the global ctors list in that case:
+ [&](Function *Ctor, FunctionCallee) { appendToGlobalCtors(M, Ctor, 0); });
+}
+
+} // end anonymous namespace
+
+PreservedAnalyses
+NumericalStabilitySanitizerPass::run(Function &F,
+ FunctionAnalysisManager &FAM) {
+ NumericalStabilitySanitizer Nsan;
+ if (Nsan.sanitizeFunction(F, FAM.getResult<TargetLibraryAnalysis>(F)))
+ return PreservedAnalyses::none();
+ return PreservedAnalyses::all();
+}
+
+PreservedAnalyses
+NumericalStabilitySanitizerPass::run(Module &M, ModuleAnalysisManager &MAM) {
+ insertModuleCtor(M);
+ return PreservedAnalyses::none();
+}
+
+static GlobalValue *createThreadLocalGV(const char *Name, Module &M, Type *Ty) {
+ return dyn_cast<GlobalValue>(M.getOrInsertGlobal(Name, Ty, [&M, Ty, Name] {
+ return new GlobalVariable(M, Ty, false, GlobalVariable::ExternalLinkage,
+ nullptr, Name, nullptr,
+ GlobalVariable::InitialExecTLSModel);
+ }));
+}
+
+void NumericalStabilitySanitizer::initialize(Module &M) {
+ const DataLayout &DL = M.getDataLayout();
+ Context = &M.getContext();
+ IntptrTy = DL.getIntPtrType(*Context);
+ Type *PtrTy = PointerType::getUnqual(*Context);
+ Type *Int32Ty = Type::getInt32Ty(*Context);
+ Type *Int1Ty = Type::getInt1Ty(*Context);
+ Type *VoidTy = Type::getVoidTy(*Context);
+
+ AttributeList Attr;
+ Attr = Attr.addFnAttribute(*Context, Attribute::NoUnwind);
+ // Initialize the runtime values (functions and global variables).
+ for (int I = 0; I < kNumValueTypes; ++I) {
+ const FTValueType VT = static_cast<FTValueType>(I);
+ const char *const VTName = typeNameFromFTValueType(VT);
+ Type *const VTTy = typeFromFTValueType(VT, *Context);
+
+ // Load/store.
+ const std::string GetterPrefix =
+ std::string("__nsan_get_shadow_ptr_for_") + VTName;
+ NsanGetShadowPtrForStore[VT] = M.getOrInsertFunction(
+ GetterPrefix + "_store", Attr, PtrTy, PtrTy, IntptrTy);
+ NsanGetShadowPtrForLoad[VT] = M.getOrInsertFunction(
+ GetterPrefix + "_load", Attr, PtrTy, PtrTy, IntptrTy);
+
+ // Check.
+ const auto &ShadowConfig = Config.byValueType(VT);
+ Type *ShadowTy = ShadowConfig.getType(*Context);
+ NsanCheckValue[VT] =
+ M.getOrInsertFunction(std::string("__nsan_internal_check_") + VTName +
+ "_" + ShadowConfig.getNsanTypeId(),
+ Attr, Int32Ty, VTTy, ShadowTy, Int32Ty, IntptrTy);
+ NsanFCmpFail[VT] = M.getOrInsertFunction(
+ std::string("__nsan_fcmp_fail_") + VTName + "_" +
+ ShadowConfig.getNsanTypeId(),
+ Attr, VoidTy, VTTy, VTTy, ShadowTy, ShadowTy, Int32Ty, Int1Ty, Int1Ty);
+ }
+
+ NsanCopyValues = M.getOrInsertFunction("__nsan_copy_values", Attr, VoidTy,
+ PtrTy, PtrTy, IntptrTy);
+ NsanSetValueUnknown = M.getOrInsertFunction("__nsan_set_value_unknown", Attr,
+ VoidTy, PtrTy, IntptrTy);
+
+ // FIXME: Add attributes nofree, nosync, readnone, readonly,
+ NsanGetRawShadowTypePtr = M.getOrInsertFunction(
+ "__nsan_internal_get_raw_shadow_type_ptr", Attr, PtrTy, PtrTy);
+ NsanGetRawShadowPtr = M.getOrInsertFunction(
+ "__nsan_internal_get_raw_shadow_ptr", Attr, PtrTy, PtrTy);
+
+ NsanShadowRetTag = createThreadLocalGV("__nsan_shadow_ret_tag", M, IntptrTy);
+
+ NsanShadowRetType = ArrayType::get(Type::getInt8Ty(*Context),
+ kMaxVectorWidth * kMaxShadowTypeSizeBytes);
+ NsanShadowRetPtr =
+ createThreadLocalGV("__nsan_shadow_ret_ptr", M, NsanShadowRetType);
+
+ NsanShadowArgsTag =
+ createThreadLocalGV("__nsan_shadow_args_tag", M, IntptrTy);
+
+ NsanShadowArgsType =
+ ArrayType::get(Type::getInt8Ty(*Context),
+ kMaxVectorWidth * kMaxNumArgs * kMaxShadowTypeSizeBytes);
+
+ NsanShadowArgsPtr =
+ createThreadLocalGV("__nsan_shadow_args_ptr", M, NsanShadowArgsType);
+
+ if (!ClCheckFunctionsFilter.empty()) {
+ Regex R = Regex(ClCheckFunctionsFilter);
+ std::string RegexError;
+ assert(R.isValid(RegexError));
+ CheckFunctionsFilter = std::move(R);
+ }
+}
+
+// Returns true if the given LLVM Value points to constant data (typically, a
+// global variable reference).
+bool NumericalStabilitySanitizer::addrPointsToConstantData(Value *Addr) {
+ // If this is a GEP, just analyze its pointer operand.
+ if (GetElementPtrInst *GEP = dyn_cast<GetElementPtrInst>(Addr))
+ Addr = GEP->getPointerOperand();
+
+ if (GlobalVariable *GV = dyn_cast<GlobalVariable>(Addr)) {
+ return GV->isConstant();
+ }
+ return false;
+}
+
+// This instruments the function entry to create shadow arguments.
+// Pseudocode:
+// if (this_fn_ptr == __nsan_shadow_args_tag) {
+// s(arg0) = LOAD<sizeof(arg0)>(__nsan_shadow_args);
+// s(arg1) = LOAD<sizeof(arg1)>(__nsan_shadow_args + sizeof(arg0));
+// ...
+// __nsan_shadow_args_tag = 0;
+// } else {
+// s(arg0) = fext(arg0);
+// s(arg1) = fext(arg1);
+// ...
+// }
+void NumericalStabilitySanitizer::createShadowArguments(
+ Function &F, const TargetLibraryInfo &TLI, ValueToShadowMap &Map) {
+ assert(!F.getIntrinsicID() && "found a definition of an intrinsic");
+
+ // Do not bother if there are no FP args.
+ if (all_of(F.args(), [this](const Argument &Arg) {
+ return Config.getExtendedFPType(Arg.getType()) == nullptr;
+ }))
+ return;
+
+ const DataLayout &DL = F.getParent()->getDataLayout();
+ IRBuilder<> Builder(F.getEntryBlock().getFirstNonPHI());
+ // The function has shadow args if the shadow args tag matches the function
+ // address.
+ Value *HasShadowArgs = Builder.CreateICmpEQ(
+ Builder.CreateLoad(IntptrTy, NsanShadowArgsTag, /*isVolatile=*/false),
+ Builder.CreatePtrToInt(&F, IntptrTy));
+
+ unsigned ShadowArgsOffsetBytes = 0;
+ for (Argument &Arg : F.args()) {
+ Type *VT = Arg.getType();
+ Type *ExtendedVT = Config.getExtendedFPType(VT);
+ if (ExtendedVT == nullptr)
+ continue; // Not an FT value.
+ Value *L = Builder.CreateAlignedLoad(
+ ExtendedVT,
+ Builder.CreateConstGEP2_64(NsanShadowArgsType, NsanShadowArgsPtr, 0,
+ ShadowArgsOffsetBytes),
+ Align(1), /*isVolatile=*/false);
+ Value *Shadow = Builder.CreateSelect(
+ HasShadowArgs, L,
+ Builder.CreateCast(Instruction::FPExt, &Arg, ExtendedVT));
+ Map.setShadow(&Arg, Shadow);
+ TypeSize SlotSize = DL.getTypeStoreSize(ExtendedVT);
+ assert(!SlotSize.isScalable() && "unsupported");
+ ShadowArgsOffsetBytes += SlotSize;
+ }
+ Builder.CreateStore(ConstantInt::get(IntptrTy, 0), NsanShadowArgsTag);
+}
+
+// Returns true if the instrumentation should emit code to check arguments
+// before a function call.
+static bool shouldCheckArgs(CallBase &CI, const TargetLibraryInfo &TLI,
+ const std::optional<Regex> &CheckFunctionsFilter) {
+
+ Function *Fn = CI.getCalledFunction();
+
+ if (CheckFunctionsFilter) {
+ // Skip checking args of indirect calls.
+ if (Fn == nullptr)
+ return false;
+ if (CheckFunctionsFilter->match(Fn->getName()))
+ return true;
+ return false;
+ }
+
+ if (Fn == nullptr)
+ return true; // Always check args of indirect calls.
+
+ // Never check nsan functions, the user called them for a reason.
+ if (Fn->getName().starts_with("__nsan_"))
+ return false;
+
+ const auto ID = Fn->getIntrinsicID();
+ LibFunc LFunc = LibFunc::NumLibFuncs;
+ // Always check args of unknown functions.
+ if (ID == Intrinsic::ID() && !TLI.getLibFunc(*Fn, LFunc))
+ return true;
+
+ // Do not check args of an `fabs` call that is used for a comparison.
+ // This is typically used for `fabs(a-b) < tolerance`, where what matters is
+ // the result of the comparison, which is already caught be the fcmp checks.
+ if (ID == Intrinsic::fabs || LFunc == LibFunc_fabsf ||
+ LFunc == LibFunc_fabs || LFunc == LibFunc_fabsl)
+ for (const auto &U : CI.users())
+ if (isa<CmpInst>(U))
+ return false;
+
+ return true; // Default is check.
+}
+
+// Populates the shadow call stack (which contains shadow values for every
+// floating-point parameter to the function).
+void NumericalStabilitySanitizer::populateShadowStack(
+ CallBase &CI, const TargetLibraryInfo &TLI, const ValueToShadowMap &Map) {
+ // Do not create a shadow stack for inline asm.
+ if (CI.isInlineAsm())
+ return;
+
+ // Do not bother if there are no FP args.
+ if (all_of(CI.operands(), [this](const Value *Arg) {
+ return Config.getExtendedFPType(Arg->getType()) == nullptr;
+ }))
+ return;
+
+ IRBuilder<> Builder(&CI);
+ SmallVector<Value *, 8> ArgShadows;
+ const bool ShouldCheckArgs = shouldCheckArgs(CI, TLI, CheckFunctionsFilter);
+ int ArgId = -1;
+ for (Value *Arg : CI.operands()) {
+ ++ArgId;
+ if (Config.getExtendedFPType(Arg->getType()) == nullptr)
+ continue; // Not an FT value.
+ Value *ArgShadow = Map.getShadow(Arg);
+ ArgShadows.push_back(ShouldCheckArgs ? emitCheck(Arg, ArgShadow, Builder,
+ CheckLoc::makeArg(ArgId))
+ : ArgShadow);
+ }
+
+ // Do not create shadow stacks for intrinsics/known lib funcs.
+ if (Function *Fn = CI.getCalledFunction()) {
+ LibFunc LFunc;
+ if (Fn->isIntrinsic() || TLI.getLibFunc(*Fn, LFunc))
+ return;
+ }
+
+ const DataLayout &DL = CI.getModule()->getDataLayout();
+ // Set the shadow stack tag.
+ Builder.CreateStore(CI.getCalledOperand(), NsanShadowArgsTag);
+ TypeSize ShadowArgsOffsetBytes = TypeSize::getFixed(0);
+
+ unsigned ShadowArgId = 0;
+ for (const Value *Arg : CI.operands()) {
+ Type *const VT = Arg->getType();
+ Type *const ExtendedVT = Config.getExtendedFPType(VT);
+ if (ExtendedVT == nullptr)
+ continue; // Not an FT value.
+ Builder.CreateAlignedStore(
+ ArgShadows[ShadowArgId++],
+ Builder.CreateConstGEP2_64(NsanShadowArgsType, NsanShadowArgsPtr, 0,
+ ShadowArgsOffsetBytes),
+ Align(1), /*isVolatile=*/false);
+ TypeSize SlotSize = DL.getTypeStoreSize(ExtendedVT);
+ assert(!SlotSize.isScalable() && "unsupported");
+ ShadowArgsOffsetBytes += SlotSize;
+ }
+}
+
+// Internal part of emitCheck(). Returns a value that indicates whether
+// computation should continue with the shadow or resume by re-fextending the
+// value.
+enum ContinuationType { // Keep in sync with runtime.
----------------
vitalybuka wrote:
same for the rest
https://github.com/llvm/llvm-project/pull/85916
More information about the llvm-commits
mailing list