[llvm] Add numerical sanitizer (PR #85916)
Alexander Shaposhnikov via llvm-commits
llvm-commits at lists.llvm.org
Tue May 21 18:21:21 PDT 2024
================
@@ -0,0 +1,2263 @@
+//===-- NumericalStabilitySanitizer.cpp -----------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+//
+// This file is a part of NumericalStabilitySanitizer.
+//
+//===----------------------------------------------------------------------===//
+
+#include "llvm/Transforms/Instrumentation/NumericalStabilitySanitizer.h"
+
+#include <cstdint>
+#include <unordered_map>
+
+#include "llvm/ADT/SmallPtrSet.h"
+#include "llvm/ADT/SmallString.h"
+#include "llvm/ADT/SmallVector.h"
+#include "llvm/ADT/Statistic.h"
+#include "llvm/ADT/StringExtras.h"
+#include "llvm/Analysis/CaptureTracking.h"
+#include "llvm/Analysis/TargetLibraryInfo.h"
+#include "llvm/Analysis/ValueTracking.h"
+#include "llvm/IR/DataLayout.h"
+#include "llvm/IR/Function.h"
+#include "llvm/IR/IRBuilder.h"
+#include "llvm/IR/IntrinsicInst.h"
+#include "llvm/IR/Intrinsics.h"
+#include "llvm/IR/LLVMContext.h"
+#include "llvm/IR/Metadata.h"
+#include "llvm/IR/Module.h"
+#include "llvm/IR/Type.h"
+#include "llvm/InitializePasses.h"
+#include "llvm/ProfileData/InstrProf.h"
+#include "llvm/Support/CommandLine.h"
+#include "llvm/Support/Debug.h"
+#include "llvm/Support/MathExtras.h"
+#include "llvm/Support/Regex.h"
+#include "llvm/Support/raw_ostream.h"
+#include "llvm/Transforms/Instrumentation.h"
+#include "llvm/Transforms/Utils/BasicBlockUtils.h"
+#include "llvm/Transforms/Utils/EscapeEnumerator.h"
+#include "llvm/Transforms/Utils/Local.h"
+#include "llvm/Transforms/Utils/ModuleUtils.h"
+
+using namespace llvm;
+
+#define DEBUG_TYPE "nsan"
+
+STATISTIC(NumInstrumentedFTLoads,
+ "Number of instrumented floating-point loads");
+
+STATISTIC(NumInstrumentedFTCalls,
+ "Number of instrumented floating-point calls");
+STATISTIC(NumInstrumentedFTRets,
+ "Number of instrumented floating-point returns");
+STATISTIC(NumInstrumentedFTStores,
+ "Number of instrumented floating-point stores");
+STATISTIC(NumInstrumentedNonFTStores,
+ "Number of instrumented non floating-point stores");
+STATISTIC(
+ NumInstrumentedNonFTMemcpyStores,
+ "Number of instrumented non floating-point stores with memcpy semantics");
+STATISTIC(NumInstrumentedFCmp, "Number of instrumented fcmps");
+
+// Using smaller shadow types types can help improve speed. For example, `dlq`
+// is 3x slower to 5x faster in opt mode and 2-6x faster in dbg mode compared to
+// `dqq`.
+static cl::opt<std::string> ClShadowMapping(
+ "nsan-shadow-type-mapping", cl::init("dqq"),
+ cl::desc("One shadow type id for each of `float`, `double`, `long double`. "
+ "`d`,`l`,`q`,`e` mean double, x86_fp80, fp128 (quad) and "
+ "ppc_fp128 (extended double) respectively. The default is to "
+ "shadow `float` as `double`, and `double` and `x86_fp80` as "
+ "`fp128`"),
+ cl::Hidden);
+
+static cl::opt<bool>
+ ClInstrumentFCmp("nsan-instrument-fcmp", cl::init(true),
+ cl::desc("Instrument floating-point comparisons"),
+ cl::Hidden);
+
+static cl::opt<std::string> ClCheckFunctionsFilter(
+ "check-functions-filter",
+ cl::desc("Only emit checks for arguments of functions "
+ "whose names match the given regular expression"),
+ cl::value_desc("regex"));
+
+static cl::opt<bool> ClTruncateFCmpEq(
+ "nsan-truncate-fcmp-eq", cl::init(true),
+ cl::desc(
+ "This flag controls the behaviour of fcmp equality comparisons:"
+ "For equality comparisons such as `x == 0.0f`, we can perform the "
+ "shadow check in the shadow (`x_shadow == 0.0) == (x == 0.0f)`) or app "
+ " domain (`(trunc(x_shadow) == 0.0f) == (x == 0.0f)`). This helps "
+ "catch the case when `x_shadow` is accurate enough (and therefore "
+ "close enough to zero) so that `trunc(x_shadow)` is zero even though "
+ "both `x` and `x_shadow` are not. "),
+ cl::Hidden);
+
+// When there is external, uninstrumented code writing to memory, the shadow
+// memory can get out of sync with the application memory. Enabling this flag
+// emits consistency checks for loads to catch this situation.
+// When everything is instrumented, this is not strictly necessary because any
+// load should have a corresponding store, but can help debug cases when the
+// framework did a bad job at tracking shadow memory modifications by failing on
+// load rather than store.
+// FIXME: provide a way to resume computations from the FT value when the load
+// is inconsistent. This ensures that further computations are not polluted.
+static cl::opt<bool> ClCheckLoads("nsan-check-loads", cl::init(false),
+ cl::desc("Check floating-point load"),
+ cl::Hidden);
+
+static cl::opt<bool> ClCheckStores("nsan-check-stores", cl::init(true),
+ cl::desc("Check floating-point stores"),
+ cl::Hidden);
+
+static cl::opt<bool> ClCheckRet("nsan-check-ret", cl::init(true),
+ cl::desc("Check floating-point return values"),
+ cl::Hidden);
+
+static const char *const kNsanModuleCtorName = "nsan.module_ctor";
+static const char *const kNsanInitName = "__nsan_init";
+
+// The following values must be kept in sync with the runtime.
+static constexpr const int kShadowScale = 2;
+static constexpr const int kMaxVectorWidth = 8;
+static constexpr const int kMaxNumArgs = 128;
+static constexpr const int kMaxShadowTypeSizeBytes = 16; // fp128
+
+namespace {
+
+// Defines the characteristics (type id, type, and floating-point semantics)
+// attached for all possible shadow types.
+class ShadowTypeConfig {
+public:
+ static std::unique_ptr<ShadowTypeConfig> fromNsanTypeId(char TypeId);
+ // The floating-point semantics of the shadow type.
+ virtual const fltSemantics &semantics() const = 0;
+
+ // The LLVM Type corresponding to the shadow type.
+ virtual Type *getType(LLVMContext &Context) const = 0;
+
+ // The nsan type id of the shadow type (`d`, `l`, `q`, ...).
+ virtual char getNsanTypeId() const = 0;
+
+ virtual ~ShadowTypeConfig() {}
+};
+
+template <char NsanTypeId>
+class ShadowTypeConfigImpl : public ShadowTypeConfig {
+public:
+ char getNsanTypeId() const override { return NsanTypeId; }
+ static constexpr const char kNsanTypeId = NsanTypeId;
+};
+
+// `double` (`d`) shadow type.
+class F64ShadowConfig : public ShadowTypeConfigImpl<'d'> {
+ const fltSemantics &semantics() const override {
+ return APFloat::IEEEdouble();
+ }
+ Type *getType(LLVMContext &Context) const override {
+ return Type::getDoubleTy(Context);
+ }
+};
+
+// `x86_fp80` (`l`) shadow type: X86 long double.
+class F80ShadowConfig : public ShadowTypeConfigImpl<'l'> {
+ const fltSemantics &semantics() const override {
+ return APFloat::x87DoubleExtended();
+ }
+ Type *getType(LLVMContext &Context) const override {
+ return Type::getX86_FP80Ty(Context);
+ }
+};
+
+// `fp128` (`q`) shadow type.
+class F128ShadowConfig : public ShadowTypeConfigImpl<'q'> {
+ const fltSemantics &semantics() const override { return APFloat::IEEEquad(); }
+ Type *getType(LLVMContext &Context) const override {
+ return Type::getFP128Ty(Context);
+ }
+};
+
+// `ppc_fp128` (`e`) shadow type: IBM extended double with 106 bits of mantissa.
+class PPC128ShadowConfig : public ShadowTypeConfigImpl<'e'> {
+ const fltSemantics &semantics() const override {
+ return APFloat::PPCDoubleDouble();
+ }
+ Type *getType(LLVMContext &Context) const override {
+ return Type::getPPC_FP128Ty(Context);
+ }
+};
+
+// Creates a ShadowTypeConfig given its type id.
+std::unique_ptr<ShadowTypeConfig>
+ShadowTypeConfig::fromNsanTypeId(const char TypeId) {
+ switch (TypeId) {
+ case F64ShadowConfig::kNsanTypeId:
+ return std::make_unique<F64ShadowConfig>();
+ case F80ShadowConfig::kNsanTypeId:
+ return std::make_unique<F80ShadowConfig>();
+ case F128ShadowConfig::kNsanTypeId:
+ return std::make_unique<F128ShadowConfig>();
+ case PPC128ShadowConfig::kNsanTypeId:
+ return std::make_unique<PPC128ShadowConfig>();
+ }
+ errs() << "nsan: invalid shadow type id'" << TypeId << "'\n";
+ return nullptr;
+}
+
+// An enum corresponding to shadow value types. Used as indices in arrays, so
+// not an `enum class`.
+enum FTValueType { kFloat, kDouble, kLongDouble, kNumValueTypes };
+
+static FTValueType semanticsToFTValueType(const fltSemantics &Sem) {
+ if (&Sem == &APFloat::IEEEsingle()) {
+ return kFloat;
+ } else if (&Sem == &APFloat::IEEEdouble()) {
+ return kDouble;
+ } else if (&Sem == &APFloat::x87DoubleExtended()) {
+ return kLongDouble;
+ }
+ llvm_unreachable("semantics are not one of the handled types");
+}
+
+// If `FT` corresponds to a primitive FTValueType, return it.
+static std::optional<FTValueType> ftValueTypeFromType(Type *FT) {
+ if (FT->isFloatTy())
+ return kFloat;
+ if (FT->isDoubleTy())
+ return kDouble;
+ if (FT->isX86_FP80Ty())
+ return kLongDouble;
+ return {};
+}
+
+// Returns the LLVM type for an FTValueType.
+static Type *typeFromFTValueType(FTValueType VT, LLVMContext &Context) {
+ switch (VT) {
+ case kFloat:
+ return Type::getFloatTy(Context);
+ case kDouble:
+ return Type::getDoubleTy(Context);
+ case kLongDouble:
+ return Type::getX86_FP80Ty(Context);
+ case kNumValueTypes:
+ return nullptr;
+ }
+}
+
+// Returns the type name for an FTValueType.
+static const char *typeNameFromFTValueType(FTValueType VT) {
+ switch (VT) {
+ case kFloat:
+ return "float";
+ case kDouble:
+ return "double";
+ case kLongDouble:
+ return "longdouble";
+ case kNumValueTypes:
+ return nullptr;
+ }
+}
+
+// A specific mapping configuration of application type to shadow type for nsan
+// (see -nsan-shadow-mapping flag).
+class MappingConfig {
+public:
+ bool initialize(LLVMContext *C) {
+ if (ClShadowMapping.size() != 3) {
+ errs() << "Invalid nsan mapping: " << ClShadowMapping << "\n";
+ }
+ Context = C;
+ unsigned ShadowTypeSizeBits[kNumValueTypes];
+ for (int VT = 0; VT < kNumValueTypes; ++VT) {
+ auto Config = ShadowTypeConfig::fromNsanTypeId(ClShadowMapping[VT]);
+ if (Config == nullptr)
+ return false;
+ const unsigned AppTypeSize =
+ typeFromFTValueType(static_cast<FTValueType>(VT), *C)
+ ->getScalarSizeInBits();
+ const unsigned ShadowTypeSize =
+ Config->getType(*C)->getScalarSizeInBits();
+ // Check that the shadow type size is at most kShadowScale times the
+ // application type size, so that shadow memory compoutations are valid.
+ if (ShadowTypeSize > kShadowScale * AppTypeSize) {
+ errs() << "Invalid nsan mapping f" << AppTypeSize << "->f"
+ << ShadowTypeSize << ": The shadow type size should be at most "
+ << kShadowScale << " times the application type size\n";
+ return false;
+ }
+ ShadowTypeSizeBits[VT] = ShadowTypeSize;
+ Configs[VT] = std::move(Config);
+ }
+
+ // Check that the mapping is monotonous. This is required because if one
+ // does an fpextend of `float->long double` in application code, nsan is
+ // going to do an fpextend of `shadow(float) -> shadow(long double)` in
+ // shadow code. This will fail in `qql` mode, since nsan would be
+ // fpextending `f128->long`, which is invalid.
+ // FIXME: Relax this.
+ if (ShadowTypeSizeBits[kFloat] > ShadowTypeSizeBits[kDouble] ||
+ ShadowTypeSizeBits[kDouble] > ShadowTypeSizeBits[kLongDouble]) {
+ errs() << "Invalid nsan mapping: { float->f" << ShadowTypeSizeBits[kFloat]
+ << "; double->f" << ShadowTypeSizeBits[kDouble]
+ << "; long double->f" << ShadowTypeSizeBits[kLongDouble] << " }\n";
+ return false;
+ }
+ return true;
+ }
+
+ const ShadowTypeConfig &byValueType(FTValueType VT) const {
+ assert(VT < FTValueType::kNumValueTypes && "invalid value type");
+ return *Configs[VT];
+ }
+
+ const ShadowTypeConfig &bySemantics(const fltSemantics &Sem) const {
+ return byValueType(semanticsToFTValueType(Sem));
+ }
+
+ // Returns the extended shadow type for a given application type.
+ Type *getExtendedFPType(Type *FT) const {
+ if (const auto VT = ftValueTypeFromType(FT))
+ return Configs[*VT]->getType(*Context);
+ if (FT->isVectorTy()) {
+ auto *VecTy = cast<VectorType>(FT);
+ Type *ExtendedScalar = getExtendedFPType(VecTy->getElementType());
+ return ExtendedScalar
+ ? VectorType::get(ExtendedScalar, VecTy->getElementCount())
+ : nullptr;
+ }
+ return nullptr;
+ }
+
+private:
+ LLVMContext *Context = nullptr;
+ std::unique_ptr<ShadowTypeConfig> Configs[FTValueType::kNumValueTypes];
+};
+
+// The memory extents of a type specifies how many elements of a given
+// FTValueType needs to be stored when storing this type.
+struct MemoryExtents {
+ FTValueType ValueType;
+ uint64_t NumElts;
+};
+static MemoryExtents getMemoryExtentsOrDie(Type *FT) {
+ if (const auto VT = ftValueTypeFromType(FT))
+ return {*VT, 1};
+ if (FT->isVectorTy()) {
+ auto *VecTy = cast<VectorType>(FT);
+ const auto ScalarExtents = getMemoryExtentsOrDie(VecTy->getElementType());
+ return {ScalarExtents.ValueType,
+ ScalarExtents.NumElts * VecTy->getElementCount().getFixedValue()};
+ }
+ llvm_unreachable("invalid value type");
+}
+
+// The location of a check. Passed as parameters to runtime checking functions.
+class CheckLoc {
+public:
+ // Creates a location that references an application memory location.
+ static CheckLoc makeStore(Value *Address) {
+ CheckLoc Result(kStore);
+ Result.Address = Address;
+ return Result;
+ }
+ static CheckLoc makeLoad(Value *Address) {
+ CheckLoc Result(kLoad);
+ Result.Address = Address;
+ return Result;
+ }
+
+ // Creates a location that references an argument, given by id.
+ static CheckLoc makeArg(int ArgId) {
+ CheckLoc Result(kArg);
+ Result.ArgId = ArgId;
+ return Result;
+ }
+
+ // Creates a location that references the return value of a function.
+ static CheckLoc makeRet() { return CheckLoc(kRet); }
+
+ // Creates a location that references a vector insert.
+ static CheckLoc makeInsert() { return CheckLoc(kInsert); }
+
+ // Returns the CheckType of location this refers to, as an integer-typed LLVM
+ // IR value.
+ Value *getType(LLVMContext &C) const {
+ return ConstantInt::get(Type::getInt32Ty(C), static_cast<int>(CheckTy));
+ }
+
+ // Returns a CheckType-specific value representing details of the location
+ // (e.g. application address for loads or stores), as an `IntptrTy`-typed LLVM
+ // IR value.
+ Value *getValue(Type *IntptrTy, IRBuilder<> &Builder) const {
+ switch (CheckTy) {
+ case kUnknown:
+ llvm_unreachable("unknown type");
+ case kRet:
+ case kInsert:
+ return ConstantInt::get(IntptrTy, 0);
+ case kArg:
+ return ConstantInt::get(IntptrTy, ArgId);
+ case kLoad:
+ case kStore:
+ return Builder.CreatePtrToInt(Address, IntptrTy);
+ }
+ }
+
+private:
+ // Must be kept in sync with the runtime.
+ enum CheckType {
+ kUnknown = 0,
+ kRet,
+ kArg,
+ kLoad,
+ kStore,
+ kInsert,
+ };
+ explicit CheckLoc(CheckType CheckTy) : CheckTy(CheckTy) {}
+
+ const CheckType CheckTy;
+ Value *Address = nullptr;
+ int ArgId = -1;
+};
+
+// A map of LLVM IR values to shadow LLVM IR values.
+class ValueToShadowMap {
+public:
+ explicit ValueToShadowMap(MappingConfig *Config) : Config(Config) {}
+
+ // Sets the shadow value for a value. Asserts that the value does not already
+ // have a value.
+ void setShadow(Value *V, Value *Shadow) {
+ assert(V);
+ assert(Shadow);
+ const bool Inserted = Map.emplace(V, Shadow).second;
+#ifdef LLVM_ENABLE_DUMP
+ if (!Inserted) {
+ if (const auto *const I = dyn_cast<Instruction>(V))
+ I->getParent()->getParent()->dump();
+ errs() << "duplicate shadow (" << V << "): ";
+ V->dump();
+ }
+#endif
+ assert(Inserted && "duplicate shadow");
+ (void)Inserted;
+ }
+
+ // Returns true if the value already has a shadow (including if the value is a
+ // constant). If true, calling getShadow() is valid.
+ bool hasShadow(Value *V) const {
+ return isa<Constant>(V) || (Map.find(V) != Map.end());
+ }
+
+ // Returns the shadow value for a given value. Asserts that the value has
+ // a shadow value. Lazily creates shadows for constant values.
+ Value *getShadow(Value *V) const {
+ assert(V);
+ if (Constant *C = dyn_cast<Constant>(V))
+ return getShadowConstant(C);
+ const auto ShadowValIt = Map.find(V);
+ assert(ShadowValIt != Map.end() && "shadow val does not exist");
+ assert(ShadowValIt->second && "shadow val is null");
+ return ShadowValIt->second;
+ }
+
+ bool empty() const { return Map.empty(); }
+
+private:
+ // Extends a constant application value to its shadow counterpart.
+ APFloat extendConstantFP(APFloat CV) const {
+ bool LosesInfo = false;
+ CV.convert(Config->bySemantics(CV.getSemantics()).semantics(),
+ APFloatBase::rmTowardZero, &LosesInfo);
+ return CV;
+ }
+
+ // Returns the shadow constant for the given application constant.
+ Constant *getShadowConstant(Constant *C) const {
+ if (UndefValue *U = dyn_cast<UndefValue>(C)) {
+ return UndefValue::get(Config->getExtendedFPType(U->getType()));
+ }
+ if (ConstantFP *CFP = dyn_cast<ConstantFP>(C)) {
+ // Floating-point constants.
+ return ConstantFP::get(Config->getExtendedFPType(CFP->getType()),
+ extendConstantFP(CFP->getValueAPF()));
+ }
+ // Vector, array, or aggregate constants.
+ if (C->getType()->isVectorTy()) {
+ SmallVector<Constant *, 8> Elements;
+ for (int I = 0, E = cast<VectorType>(C->getType())
+ ->getElementCount()
+ .getFixedValue();
+ I < E; ++I)
+ Elements.push_back(getShadowConstant(C->getAggregateElement(I)));
+ return ConstantVector::get(Elements);
+ }
+ llvm_unreachable("unimplemented");
+ }
+
+ MappingConfig *const Config;
+ std::unordered_map<Value *, Value *> Map;
+};
+
+/// Instantiating NumericalStabilitySanitizer inserts the nsan runtime library
+/// API function declarations into the module if they don't exist already.
+/// Instantiating ensures the __nsan_init function is in the list of global
+/// constructors for the module.
+class NumericalStabilitySanitizer {
+public:
+ bool sanitizeFunction(Function &F, const TargetLibraryInfo &TLI);
+
+private:
+ void initialize(Module &M);
+ bool instrumentMemIntrinsic(MemIntrinsic *MI);
+ void maybeAddSuffixForNsanInterface(CallBase *CI);
+ bool addrPointsToConstantData(Value *Addr);
+ void maybeCreateShadowValue(Instruction &Root, const TargetLibraryInfo &TLI,
+ ValueToShadowMap &Map);
+ Value *createShadowValueWithOperandsAvailable(Instruction &Inst,
+ const TargetLibraryInfo &TLI,
+ const ValueToShadowMap &Map);
+ PHINode *maybeCreateShadowPhi(PHINode &Phi, const TargetLibraryInfo &TLI);
+ void createShadowArguments(Function &F, const TargetLibraryInfo &TLI,
+ ValueToShadowMap &Map);
+
+ void populateShadowStack(CallBase &CI, const TargetLibraryInfo &TLI,
+ const ValueToShadowMap &Map);
+
+ void propagateShadowValues(Instruction &Inst, const TargetLibraryInfo &TLI,
+ const ValueToShadowMap &Map);
+ Value *emitCheck(Value *V, Value *ShadowV, IRBuilder<> &Builder,
+ CheckLoc Loc);
+ Value *emitCheckInternal(Value *V, Value *ShadowV, IRBuilder<> &Builder,
+ CheckLoc Loc);
+ void emitFCmpCheck(FCmpInst &FCmp, const ValueToShadowMap &Map);
+ Value *getCalleeAddress(CallBase &Call, IRBuilder<> &Builder) const;
+
+ // Value creation handlers.
+ Value *handleLoad(LoadInst &Load, Type *VT, Type *ExtendedVT);
+ Value *handleTrunc(FPTruncInst &Trunc, Type *VT, Type *ExtendedVT,
+ const ValueToShadowMap &Map);
+ Value *handleExt(FPExtInst &Ext, Type *VT, Type *ExtendedVT,
+ const ValueToShadowMap &Map);
+ Value *handleCallBase(CallBase &Call, Type *VT, Type *ExtendedVT,
+ const TargetLibraryInfo &TLI,
+ const ValueToShadowMap &Map, IRBuilder<> &Builder);
+ Value *maybeHandleKnownCallBase(CallBase &Call, Type *VT, Type *ExtendedVT,
+ const TargetLibraryInfo &TLI,
+ const ValueToShadowMap &Map,
+ IRBuilder<> &Builder);
+
+ // Value propagation handlers.
+ void propagateFTStore(StoreInst &Store, Type *VT, Type *ExtendedVT,
+ const ValueToShadowMap &Map);
+ void propagateNonFTStore(StoreInst &Store, Type *VT,
+ const ValueToShadowMap &Map);
+
+ MappingConfig Config;
+ LLVMContext *Context = nullptr;
+ IntegerType *IntptrTy = nullptr;
+ FunctionCallee NsanGetShadowPtrForStore[FTValueType::kNumValueTypes];
+ FunctionCallee NsanGetShadowPtrForLoad[FTValueType::kNumValueTypes];
+ FunctionCallee NsanCheckValue[FTValueType::kNumValueTypes];
+ FunctionCallee NsanFCmpFail[FTValueType::kNumValueTypes];
+ FunctionCallee NsanCopyValues;
+ FunctionCallee NsanSetValueUnknown;
+ FunctionCallee NsanGetRawShadowTypePtr;
+ FunctionCallee NsanGetRawShadowPtr;
+ GlobalValue *NsanShadowRetTag;
+
+ Type *NsanShadowRetType;
+ GlobalValue *NsanShadowRetPtr;
+
+ GlobalValue *NsanShadowArgsTag;
+
+ Type *NsanShadowArgsType;
+ GlobalValue *NsanShadowArgsPtr;
+
+ std::optional<Regex> CheckFunctionsFilter;
+};
+
+void insertModuleCtor(Module &M) {
+ getOrCreateSanitizerCtorAndInitFunctions(
+ M, kNsanModuleCtorName, kNsanInitName, /*InitArgTypes=*/{},
+ /*InitArgs=*/{},
+ // This callback is invoked when the functions are created the first
+ // time. Hook them into the global ctors list in that case:
+ [&](Function *Ctor, FunctionCallee) { appendToGlobalCtors(M, Ctor, 0); });
+}
+
+} // end anonymous namespace
+
+PreservedAnalyses
+NumericalStabilitySanitizerPass::run(Function &F,
+ FunctionAnalysisManager &FAM) {
+ NumericalStabilitySanitizer Nsan;
+ if (Nsan.sanitizeFunction(F, FAM.getResult<TargetLibraryAnalysis>(F)))
+ return PreservedAnalyses::none();
+ return PreservedAnalyses::all();
+}
+
+PreservedAnalyses
+NumericalStabilitySanitizerPass::run(Module &M, ModuleAnalysisManager &MAM) {
+ insertModuleCtor(M);
+ return PreservedAnalyses::none();
+}
+
+static GlobalValue *createThreadLocalGV(const char *Name, Module &M, Type *Ty) {
+ return dyn_cast<GlobalValue>(M.getOrInsertGlobal(Name, Ty, [&M, Ty, Name] {
+ return new GlobalVariable(M, Ty, false, GlobalVariable::ExternalLinkage,
+ nullptr, Name, nullptr,
+ GlobalVariable::InitialExecTLSModel);
+ }));
+}
+
+void NumericalStabilitySanitizer::initialize(Module &M) {
+ const DataLayout &DL = M.getDataLayout();
+ Context = &M.getContext();
+ IntptrTy = DL.getIntPtrType(*Context);
+ Type *PtrTy = PointerType::getUnqual(*Context);
+ Type *Int32Ty = Type::getInt32Ty(*Context);
+ Type *Int1Ty = Type::getInt1Ty(*Context);
+ Type *VoidTy = Type::getVoidTy(*Context);
+
+ AttributeList Attr;
+ Attr = Attr.addFnAttribute(*Context, Attribute::NoUnwind);
+ // Initialize the runtime values (functions and global variables).
+ for (int I = 0; I < kNumValueTypes; ++I) {
+ const FTValueType VT = static_cast<FTValueType>(I);
+ const char *const VTName = typeNameFromFTValueType(VT);
+ Type *const VTTy = typeFromFTValueType(VT, *Context);
+
+ // Load/store.
+ const std::string GetterPrefix =
+ std::string("__nsan_get_shadow_ptr_for_") + VTName;
+ NsanGetShadowPtrForStore[VT] = M.getOrInsertFunction(
+ GetterPrefix + "_store", Attr, PtrTy, PtrTy, IntptrTy);
+ NsanGetShadowPtrForLoad[VT] = M.getOrInsertFunction(
+ GetterPrefix + "_load", Attr, PtrTy, PtrTy, IntptrTy);
+
+ // Check.
+ const auto &ShadowConfig = Config.byValueType(VT);
+ Type *ShadowTy = ShadowConfig.getType(*Context);
+ NsanCheckValue[VT] =
+ M.getOrInsertFunction(std::string("__nsan_internal_check_") + VTName +
+ "_" + ShadowConfig.getNsanTypeId(),
+ Attr, Int32Ty, VTTy, ShadowTy, Int32Ty, IntptrTy);
+ NsanFCmpFail[VT] = M.getOrInsertFunction(
+ std::string("__nsan_fcmp_fail_") + VTName + "_" +
+ ShadowConfig.getNsanTypeId(),
+ Attr, VoidTy, VTTy, VTTy, ShadowTy, ShadowTy, Int32Ty, Int1Ty, Int1Ty);
+ }
+
+ NsanCopyValues = M.getOrInsertFunction("__nsan_copy_values", Attr, VoidTy,
+ PtrTy, PtrTy, IntptrTy);
+ NsanSetValueUnknown = M.getOrInsertFunction("__nsan_set_value_unknown", Attr,
+ VoidTy, PtrTy, IntptrTy);
+
+ // FIXME: Add attributes nofree, nosync, readnone, readonly,
+ NsanGetRawShadowTypePtr = M.getOrInsertFunction(
+ "__nsan_internal_get_raw_shadow_type_ptr", Attr, PtrTy, PtrTy);
+ NsanGetRawShadowPtr = M.getOrInsertFunction(
+ "__nsan_internal_get_raw_shadow_ptr", Attr, PtrTy, PtrTy);
+
+ NsanShadowRetTag = createThreadLocalGV("__nsan_shadow_ret_tag", M, IntptrTy);
+
+ NsanShadowRetType = ArrayType::get(Type::getInt8Ty(*Context),
+ kMaxVectorWidth * kMaxShadowTypeSizeBytes);
+ NsanShadowRetPtr =
+ createThreadLocalGV("__nsan_shadow_ret_ptr", M, NsanShadowRetType);
+
+ NsanShadowArgsTag =
+ createThreadLocalGV("__nsan_shadow_args_tag", M, IntptrTy);
+
+ NsanShadowArgsType =
+ ArrayType::get(Type::getInt8Ty(*Context),
+ kMaxVectorWidth * kMaxNumArgs * kMaxShadowTypeSizeBytes);
+
+ NsanShadowArgsPtr =
+ createThreadLocalGV("__nsan_shadow_args_ptr", M, NsanShadowArgsType);
+
+ if (!ClCheckFunctionsFilter.empty()) {
+ Regex R = Regex(ClCheckFunctionsFilter);
+ std::string RegexError;
+ assert(R.isValid(RegexError));
+ CheckFunctionsFilter = std::move(R);
+ }
+}
+
+// Returns true if the given LLVM Value points to constant data (typically, a
+// global variable reference).
+bool NumericalStabilitySanitizer::addrPointsToConstantData(Value *Addr) {
+ // If this is a GEP, just analyze its pointer operand.
+ if (GetElementPtrInst *GEP = dyn_cast<GetElementPtrInst>(Addr))
+ Addr = GEP->getPointerOperand();
+
+ if (GlobalVariable *GV = dyn_cast<GlobalVariable>(Addr)) {
+ return GV->isConstant();
+ }
+ return false;
+}
+
+// This instruments the function entry to create shadow arguments.
+// Pseudocode:
+// if (this_fn_ptr == __nsan_shadow_args_tag) {
+// s(arg0) = LOAD<sizeof(arg0)>(__nsan_shadow_args);
+// s(arg1) = LOAD<sizeof(arg1)>(__nsan_shadow_args + sizeof(arg0));
+// ...
+// __nsan_shadow_args_tag = 0;
+// } else {
+// s(arg0) = fext(arg0);
+// s(arg1) = fext(arg1);
+// ...
+// }
+void NumericalStabilitySanitizer::createShadowArguments(
+ Function &F, const TargetLibraryInfo &TLI, ValueToShadowMap &Map) {
+ assert(!F.getIntrinsicID() && "found a definition of an intrinsic");
+
+ // Do not bother if there are no FP args.
+ if (all_of(F.args(), [this](const Argument &Arg) {
+ return Config.getExtendedFPType(Arg.getType()) == nullptr;
+ }))
+ return;
+
+ const DataLayout &DL = F.getParent()->getDataLayout();
+ IRBuilder<> Builder(F.getEntryBlock().getFirstNonPHI());
+ // The function has shadow args if the shadow args tag matches the function
+ // address.
+ Value *HasShadowArgs = Builder.CreateICmpEQ(
+ Builder.CreateLoad(IntptrTy, NsanShadowArgsTag, /*isVolatile=*/false),
+ Builder.CreatePtrToInt(&F, IntptrTy));
+
+ unsigned ShadowArgsOffsetBytes = 0;
+ for (Argument &Arg : F.args()) {
+ Type *const VT = Arg.getType();
+ Type *const ExtendedVT = Config.getExtendedFPType(VT);
+ if (ExtendedVT == nullptr)
+ continue; // Not an FT value.
+ Value *L = Builder.CreateAlignedLoad(
+ ExtendedVT,
+ Builder.CreateConstGEP2_64(NsanShadowArgsType, NsanShadowArgsPtr, 0,
+ ShadowArgsOffsetBytes),
+ Align(1), /*isVolatile=*/false);
+ Value *Shadow = Builder.CreateSelect(
+ HasShadowArgs, L,
+ Builder.CreateCast(Instruction::FPExt, &Arg, ExtendedVT));
+ Map.setShadow(&Arg, Shadow);
+ TypeSize SlotSize = DL.getTypeStoreSize(ExtendedVT);
+ assert(!SlotSize.isScalable() && "unsupported");
+ ShadowArgsOffsetBytes += SlotSize.getFixedValue();
+ }
+ Builder.CreateStore(ConstantInt::get(IntptrTy, 0), NsanShadowArgsTag);
+}
+
+// Returns true if the instrumentation should emit code to check arguments
+// before a function call.
+static bool shouldCheckArgs(CallBase &CI, const TargetLibraryInfo &TLI,
+ const std::optional<Regex> &CheckFunctionsFilter) {
+
+ Function *Fn = CI.getCalledFunction();
+
+ if (CheckFunctionsFilter) {
+ // Skip checking args of indirect calls.
+ if (Fn == nullptr)
+ return false;
+ if (CheckFunctionsFilter->match(Fn->getName()))
+ return true;
+ return false;
+ }
+
+ if (Fn == nullptr)
+ return true; // Always check args of indirect calls.
+
+ // Never check nsan functions, the user called them for a reason.
+ if (Fn->getName().starts_with("__nsan_"))
+ return false;
+
+ const auto ID = Fn->getIntrinsicID();
+ LibFunc LFunc = LibFunc::NumLibFuncs;
+ // Always check args of unknown functions.
+ if (ID == Intrinsic::ID() && !TLI.getLibFunc(*Fn, LFunc))
+ return true;
+
+ // Do not check args of an `fabs` call that is used for a comparison.
+ // This is typically used for `fabs(a-b) < tolerance`, where what matters is
+ // the result of the comparison, which is already caught be the fcmp checks.
+ if (ID == Intrinsic::fabs || LFunc == LibFunc_fabsf ||
+ LFunc == LibFunc_fabs || LFunc == LibFunc_fabsl)
+ for (const auto &U : CI.users())
+ if (isa<CmpInst>(U))
+ return false;
+
+ return true; // Default is check.
+}
+
+// Populates the shadow call stack (which contains shadow values for every
+// floating-point parameter to the function).
+void NumericalStabilitySanitizer::populateShadowStack(
+ CallBase &CI, const TargetLibraryInfo &TLI, const ValueToShadowMap &Map) {
+ // Do not create a shadow stack for inline asm.
+ if (CI.isInlineAsm())
+ return;
+
+ // Do not bother if there are no FP args.
+ if (all_of(CI.operands(), [this](const Value *Arg) {
+ return Config.getExtendedFPType(Arg->getType()) == nullptr;
+ }))
+ return;
+
+ IRBuilder<> Builder(&CI);
+ SmallVector<Value *, 8> ArgShadows;
+ const bool ShouldCheckArgs = shouldCheckArgs(CI, TLI, CheckFunctionsFilter);
+ int ArgId = -1;
+ for (Value *Arg : CI.operands()) {
+ ++ArgId;
+ if (Config.getExtendedFPType(Arg->getType()) == nullptr)
+ continue; // Not an FT value.
+ Value *ArgShadow = Map.getShadow(Arg);
+ ArgShadows.push_back(ShouldCheckArgs ? emitCheck(Arg, ArgShadow, Builder,
+ CheckLoc::makeArg(ArgId))
+ : ArgShadow);
+ }
+
+ // Do not create shadow stacks for intrinsics/known lib funcs.
+ if (Function *Fn = CI.getCalledFunction()) {
+ LibFunc LFunc;
+ if (Fn->isIntrinsic() || TLI.getLibFunc(*Fn, LFunc))
+ return;
+ }
+
+ const DataLayout &DL =
+ CI.getParent()->getParent()->getParent()->getDataLayout();
+ // Set the shadow stack tag.
+ Builder.CreateStore(getCalleeAddress(CI, Builder), NsanShadowArgsTag);
+ unsigned ShadowArgsOffsetBytes = 0;
+
+ unsigned ShadowArgId = 0;
+ for (const Value *Arg : CI.operands()) {
+ Type *const VT = Arg->getType();
+ Type *const ExtendedVT = Config.getExtendedFPType(VT);
+ if (ExtendedVT == nullptr)
+ continue; // Not an FT value.
+ Builder.CreateAlignedStore(
+ ArgShadows[ShadowArgId++],
+ Builder.CreateConstGEP2_64(NsanShadowArgsType, NsanShadowArgsPtr, 0,
+ ShadowArgsOffsetBytes),
+ Align(1), /*isVolatile=*/false);
+ TypeSize SlotSize = DL.getTypeStoreSize(ExtendedVT);
+ assert(!SlotSize.isScalable() && "unsupported");
+ ShadowArgsOffsetBytes += SlotSize.getFixedValue();
+ }
+}
+
+// Internal part of emitCheck(). Returns a value that indicates whether
+// computation should continue with the shadow or resume by re-fextending the
+// value.
+enum ContinuationType { // Keep in sync with runtime.
+ kContinueWithShadow = 0,
+ kResumeFromValue = 1,
+};
+Value *NumericalStabilitySanitizer::emitCheckInternal(Value *V, Value *ShadowV,
+ IRBuilder<> &Builder,
+ CheckLoc Loc) {
+ // Do not emit checks for constant values, this is redundant.
+ if (isa<Constant>(V))
+ return ConstantInt::get(Builder.getInt32Ty(), kContinueWithShadow);
+
+ Type *const Ty = V->getType();
+ if (const auto VT = ftValueTypeFromType(Ty))
+ return Builder.CreateCall(
+ NsanCheckValue[*VT],
+ {V, ShadowV, Loc.getType(*Context), Loc.getValue(IntptrTy, Builder)});
+
+ if (Ty->isVectorTy()) {
+ auto *VecTy = cast<VectorType>(Ty);
+ Value *CheckResult = nullptr;
+ for (int I = 0, E = VecTy->getElementCount().getFixedValue(); I < E; ++I) {
+ // We resume if any element resumes. Another option would be to create a
+ // vector shuffle with the array of ContinueWithShadow, but that is too
+ // complex.
+ Value *ExtractedV = Builder.CreateExtractElement(V, I);
+ Value *ExtractedShadowV = Builder.CreateExtractElement(ShadowV, I);
+ Value *ComponentCheckResult =
+ emitCheckInternal(ExtractedV, ExtractedShadowV, Builder, Loc);
+ CheckResult = CheckResult
+ ? Builder.CreateOr(CheckResult, ComponentCheckResult)
+ : ComponentCheckResult;
+ }
+ return CheckResult;
+ }
+ if (Ty->isArrayTy()) {
+ Value *CheckResult = nullptr;
+ for (int I = 0, E = Ty->getArrayNumElements(); I < E; ++I) {
+ Value *ExtractedV = Builder.CreateExtractElement(V, I);
+ Value *ExtractedShadowV = Builder.CreateExtractElement(ShadowV, I);
+ Value *ComponentCheckResult =
+ emitCheckInternal(ExtractedV, ExtractedShadowV, Builder, Loc);
+ CheckResult = CheckResult
+ ? Builder.CreateOr(CheckResult, ComponentCheckResult)
+ : ComponentCheckResult;
+ }
+ return CheckResult;
+ }
+ if (Ty->isStructTy()) {
+ Value *CheckResult = nullptr;
+ for (int I = 0, E = Ty->getStructNumElements(); I < E; ++I) {
+ if (Config.getExtendedFPType(Ty->getStructElementType(I)) == nullptr)
+ continue; // Only check FT values.
+ Value *ExtractedV = Builder.CreateExtractElement(V, I);
+ Value *ExtractedShadowV = Builder.CreateExtractElement(ShadowV, I);
+ Value *ComponentCheckResult =
+ emitCheckInternal(ExtractedV, ExtractedShadowV, Builder, Loc);
+ CheckResult = CheckResult
+ ? Builder.CreateOr(CheckResult, ComponentCheckResult)
+ : ComponentCheckResult;
+ }
+ assert(CheckResult && "struct with no FT element");
+ return CheckResult;
+ }
+
+ llvm_unreachable("not implemented");
+}
+
+// Inserts a runtime check of V against its shadow value ShadowV.
+// We check values whenever they escape: on return, call, stores, and
+// insertvalue.
+// Returns the shadow value that should be used to continue the computations,
+// depending on the answer from the runtime.
+// FIXME: Should we check on select ? phi ?
+Value *NumericalStabilitySanitizer::emitCheck(Value *V, Value *ShadowV,
+ IRBuilder<> &Builder,
+ CheckLoc Loc) {
+ // Do not emit checks for constant values, this is redundant.
+ if (isa<Constant>(V))
+ return ShadowV;
+
+ if (Instruction *Inst = dyn_cast<Instruction>(V)) {
+ Function *F = Inst->getFunction();
+ if (CheckFunctionsFilter &&
+ !(F && CheckFunctionsFilter->match(F->getName()))) {
+ return ShadowV;
+ }
+ }
+
+ Value *CheckResult = emitCheckInternal(V, ShadowV, Builder, Loc);
+ Value *ICmpEQ = Builder.CreateICmpEQ(
+ CheckResult, ConstantInt::get(Builder.getInt32Ty(), kResumeFromValue));
+ return Builder.CreateSelect(
+ ICmpEQ,
+ Builder.CreateCast(Instruction::FPExt, V,
+ Config.getExtendedFPType(V->getType())),
+ ShadowV);
+}
+
+static Instruction *getNextInstructionOrDie(Instruction &Inst) {
+ assert(Inst.getNextNode() && "instruction is a terminator");
+ return Inst.getNextNode();
+}
+
+// Inserts a check that fcmp on shadow values are consistent with that on base
+// values.
+void NumericalStabilitySanitizer::emitFCmpCheck(FCmpInst &FCmp,
+ const ValueToShadowMap &Map) {
+ if (!ClInstrumentFCmp)
+ return;
+
+ Function *F = FCmp.getFunction();
+ if (CheckFunctionsFilter &&
+ !(F && CheckFunctionsFilter->match(F->getName()))) {
+ return;
+ }
+
+ Value *LHS = FCmp.getOperand(0);
+ if (Config.getExtendedFPType(LHS->getType()) == nullptr)
+ return;
+ Value *RHS = FCmp.getOperand(1);
+
+ // Split the basic block. On mismatch, we'll jump to the new basic block with
+ // a call to the runtime for error reporting.
+ BasicBlock *FCmpBB = FCmp.getParent();
+ BasicBlock *NextBB = FCmpBB->splitBasicBlock(getNextInstructionOrDie(FCmp));
+ // Remove the newly created terminator unconditional branch.
+ FCmpBB->back().eraseFromParent();
+ BasicBlock *FailBB =
+ BasicBlock::Create(*Context, "", FCmpBB->getParent(), NextBB);
+
+ // Create the shadow fcmp and comparison between the fcmps.
+ IRBuilder<> FCmpBuilder(FCmpBB);
+ FCmpBuilder.SetCurrentDebugLocation(FCmp.getDebugLoc());
+ Value *ShadowLHS = Map.getShadow(LHS);
+ Value *ShadowRHS = Map.getShadow(RHS);
+ // See comment on ClTruncateFCmpEq.
+ if (FCmp.isEquality() && ClTruncateFCmpEq) {
+ Type *Ty = ShadowLHS->getType();
+ ShadowLHS = FCmpBuilder.CreateCast(
+ Instruction::FPExt,
+ FCmpBuilder.CreateCast(Instruction::FPTrunc, ShadowLHS, LHS->getType()),
+ Ty);
+ ShadowRHS = FCmpBuilder.CreateCast(
+ Instruction::FPExt,
+ FCmpBuilder.CreateCast(Instruction::FPTrunc, ShadowRHS, RHS->getType()),
+ Ty);
+ }
+ Value *ShadowFCmp =
+ FCmpBuilder.CreateFCmp(FCmp.getPredicate(), ShadowLHS, ShadowRHS);
+ Value *OriginalAndShadowFcmpMatch =
+ FCmpBuilder.CreateICmpEQ(&FCmp, ShadowFCmp);
+
+ if (OriginalAndShadowFcmpMatch->getType()->isVectorTy()) {
+ // If we have a vector type, `OriginalAndShadowFcmpMatch` is a vector of i1,
+ // where an element is true if the corresponding elements in original and
+ // shadow are the same. We want all elements to be 1.
+ OriginalAndShadowFcmpMatch =
+ FCmpBuilder.CreateAndReduce(OriginalAndShadowFcmpMatch);
+ }
+
+ FCmpBuilder.CreateCondBr(OriginalAndShadowFcmpMatch, NextBB, FailBB);
+
+ // Fill in FailBB.
+ IRBuilder<> FailBuilder(FailBB);
+ FailBuilder.SetCurrentDebugLocation(FCmp.getDebugLoc());
+
+ const auto EmitFailCall = [this, &FCmp, &FCmpBuilder,
+ &FailBuilder](Value *L, Value *R, Value *ShadowL,
+ Value *ShadowR, Value *Result,
+ Value *ShadowResult) {
+ Type *FT = L->getType();
+ FunctionCallee *Callee = nullptr;
+ if (FT->isFloatTy()) {
+ Callee = &(NsanFCmpFail[kFloat]);
+ } else if (FT->isDoubleTy()) {
+ Callee = &(NsanFCmpFail[kDouble]);
+ } else if (FT->isX86_FP80Ty()) {
+ // FIXME: make NsanFCmpFailLongDouble work.
+ Callee = &(NsanFCmpFail[kDouble]);
+ L = FailBuilder.CreateCast(Instruction::FPTrunc, L,
+ Type::getDoubleTy(*Context));
+ R = FailBuilder.CreateCast(Instruction::FPTrunc, L,
+ Type::getDoubleTy(*Context));
+ } else {
+ llvm_unreachable("not implemented");
+ }
+ FailBuilder.CreateCall(*Callee, {L, R, ShadowL, ShadowR,
+ ConstantInt::get(FCmpBuilder.getInt32Ty(),
+ FCmp.getPredicate()),
+ Result, ShadowResult});
+ };
+ if (LHS->getType()->isVectorTy()) {
+ for (int I = 0, E = cast<VectorType>(LHS->getType())
+ ->getElementCount()
+ .getFixedValue();
+ I < E; ++I) {
+ Value *L = FailBuilder.CreateExtractElement(LHS, I);
+ Value *R = FailBuilder.CreateExtractElement(RHS, I);
+ Value *ShadowL = FailBuilder.CreateExtractElement(ShadowLHS, I);
+ Value *ShadowR = FailBuilder.CreateExtractElement(ShadowRHS, I);
+ Value *Result = FailBuilder.CreateExtractElement(&FCmp, I);
+ Value *ShadowResult = FailBuilder.CreateExtractElement(ShadowFCmp, I);
+ EmitFailCall(L, R, ShadowL, ShadowR, Result, ShadowResult);
+ }
+ } else {
+ EmitFailCall(LHS, RHS, ShadowLHS, ShadowRHS, &FCmp, ShadowFCmp);
+ }
+ FailBuilder.CreateBr(NextBB);
+
+ ++NumInstrumentedFCmp;
+}
+
+// Creates a shadow phi value for any phi that defines a value of FT type.
+PHINode *NumericalStabilitySanitizer::maybeCreateShadowPhi(
+ PHINode &Phi, const TargetLibraryInfo &TLI) {
+ Type *const VT = Phi.getType();
+ Type *const ExtendedVT = Config.getExtendedFPType(VT);
+ if (ExtendedVT == nullptr)
+ return nullptr; // Not an FT value.
+ // The phi operands are shadow values and are not available when the phi is
+ // created. They will be populated in a final phase, once all shadow values
+ // have been created.
+ PHINode *Shadow = PHINode::Create(ExtendedVT, Phi.getNumIncomingValues());
+ Shadow->insertAfter(&Phi);
+ return Shadow;
+}
+
+Value *NumericalStabilitySanitizer::handleLoad(LoadInst &Load, Type *VT,
+ Type *ExtendedVT) {
+ IRBuilder<> Builder(getNextInstructionOrDie(Load));
+ Builder.SetCurrentDebugLocation(Load.getDebugLoc());
+ if (addrPointsToConstantData(Load.getPointerOperand())) {
+ // No need to look into the shadow memory, the value is a constant. Just
+ // convert from FT to 2FT.
+ return Builder.CreateFPExt(&Load, ExtendedVT);
+ }
+
+ // if (%shadowptr == &)
+ // %shadow = fpext %v
+ // else
+ // %shadow = load (ptrcast %shadow_ptr))
+ // Considered options here:
+ // - Have `NsanGetShadowPtrForLoad` return a fixed address
+ // &__nsan_unknown_value_shadow_address that is valid to load from, and
+ // use a select. This has the advantage that the generated IR is simpler.
+ // - Have `NsanGetShadowPtrForLoad` return nullptr. Because `select` does
+ // not short-circuit, dereferencing the returned pointer is no longer an
+ // option, have to split and create a separate basic block. This has the
+ // advantage of being easier to debug because it crashes if we ever mess
+ // up.
+
+ const auto Extents = getMemoryExtentsOrDie(VT);
+ Value *ShadowPtr = Builder.CreateCall(
+ NsanGetShadowPtrForLoad[Extents.ValueType],
+ {Load.getPointerOperand(), ConstantInt::get(IntptrTy, Extents.NumElts)});
+ ++NumInstrumentedFTLoads;
+
+ // Split the basic block.
+ BasicBlock *LoadBB = Load.getParent();
+ BasicBlock *NextBB = LoadBB->splitBasicBlock(Builder.GetInsertPoint());
+ // Create the two options for creating the shadow value.
+ BasicBlock *ShadowLoadBB =
+ BasicBlock::Create(*Context, "", LoadBB->getParent(), NextBB);
+ BasicBlock *FExtBB =
+ BasicBlock::Create(*Context, "", LoadBB->getParent(), NextBB);
+
+ // Replace the newly created terminator unconditional branch by a conditional
+ // branch to one of the options.
+ {
+ LoadBB->back().eraseFromParent();
+ IRBuilder<> LoadBBBuilder(LoadBB); // The old builder has been invalidated.
+ LoadBBBuilder.SetCurrentDebugLocation(Load.getDebugLoc());
+ LoadBBBuilder.CreateCondBr(LoadBBBuilder.CreateIsNull(ShadowPtr), FExtBB,
+ ShadowLoadBB);
+ }
+
+ // Fill in ShadowLoadBB.
+ IRBuilder<> ShadowLoadBBBuilder(ShadowLoadBB);
+ ShadowLoadBBBuilder.SetCurrentDebugLocation(Load.getDebugLoc());
+ Value *ShadowLoad = ShadowLoadBBBuilder.CreateAlignedLoad(
+ ExtendedVT, ShadowPtr, Align(1), Load.isVolatile());
+ if (ClCheckLoads) {
+ ShadowLoad = emitCheck(&Load, ShadowLoad, ShadowLoadBBBuilder,
+ CheckLoc::makeLoad(Load.getPointerOperand()));
+ }
+ ShadowLoadBBBuilder.CreateBr(NextBB);
+
+ // Fill in FExtBB.
+ IRBuilder<> FExtBBBuilder(FExtBB);
+ FExtBBBuilder.SetCurrentDebugLocation(Load.getDebugLoc());
+ Value *const FExt =
+ FExtBBBuilder.CreateCast(Instruction::FPExt, &Load, ExtendedVT);
+ FExtBBBuilder.CreateBr(NextBB);
+
+ // The shadow value come from any of the options.
+ IRBuilder<> NextBBBuilder(&*NextBB->begin());
+ NextBBBuilder.SetCurrentDebugLocation(Load.getDebugLoc());
+ PHINode *ShadowPhi = NextBBBuilder.CreatePHI(ExtendedVT, 2);
+ ShadowPhi->addIncoming(ShadowLoad, ShadowLoadBB);
+ ShadowPhi->addIncoming(FExt, FExtBB);
+ return ShadowPhi;
+}
+
+Value *NumericalStabilitySanitizer::handleTrunc(FPTruncInst &Trunc, Type *VT,
+ Type *ExtendedVT,
+ const ValueToShadowMap &Map) {
+ Value *const OrigSource = Trunc.getOperand(0);
+ Type *const OrigSourceTy = OrigSource->getType();
+ Type *const ExtendedSourceTy = Config.getExtendedFPType(OrigSourceTy);
+
+ // When truncating:
+ // - (A) If the source has a shadow, we truncate from the shadow, else we
+ // truncate from the original source.
+ // - (B) If the shadow of the source is larger than the shadow of the dest,
+ // we still need a truncate. Else, the shadow of the source is the same
+ // type as the shadow of the dest (because mappings are non-decreasing), so
+ // we don't need to emit a truncate.
+ // Examples,
+ // with a mapping of {f32->f64;f64->f80;f80->f128}
+ // fptrunc double %1 to float -> fptrunc x86_fp80 s(%1) to double
+ // fptrunc x86_fp80 %1 to float -> fptrunc fp128 s(%1) to double
+ // fptrunc fp128 %1 to float -> fptrunc fp128 %1 to double
+ // fptrunc x86_fp80 %1 to double -> x86_fp80 s(%1)
+ // fptrunc fp128 %1 to double -> fptrunc fp128 %1 to x86_fp80
+ // fptrunc fp128 %1 to x86_fp80 -> fp128 %1
+ // with a mapping of {f32->f64;f64->f128;f80->f128}
+ // fptrunc double %1 to float -> fptrunc fp128 s(%1) to double
+ // fptrunc x86_fp80 %1 to float -> fptrunc fp128 s(%1) to double
+ // fptrunc fp128 %1 to float -> fptrunc fp128 %1 to double
+ // fptrunc x86_fp80 %1 to double -> fp128 %1
+ // fptrunc fp128 %1 to double -> fp128 %1
+ // fptrunc fp128 %1 to x86_fp80 -> fp128 %1
+ // with a mapping of {f32->f32;f64->f32;f80->f64}
+ // fptrunc double %1 to float -> float s(%1)
+ // fptrunc x86_fp80 %1 to float -> fptrunc double s(%1) to float
+ // fptrunc fp128 %1 to float -> fptrunc fp128 %1 to float
+ // fptrunc x86_fp80 %1 to double -> fptrunc double s(%1) to float
+ // fptrunc fp128 %1 to double -> fptrunc fp128 %1 to float
+ // fptrunc fp128 %1 to x86_fp80 -> fptrunc fp128 %1 to double
+
+ // See (A) above.
+ Value *const Source =
+ ExtendedSourceTy ? Map.getShadow(OrigSource) : OrigSource;
+ Type *const SourceTy = ExtendedSourceTy ? ExtendedSourceTy : OrigSourceTy;
+ // See (B) above.
+ if (SourceTy == ExtendedVT)
+ return Source;
+
+ Instruction *const Shadow =
+ CastInst::Create(Instruction::FPTrunc, Source, ExtendedVT);
+ Shadow->insertAfter(&Trunc);
+ return Shadow;
+}
+
+Value *NumericalStabilitySanitizer::handleExt(FPExtInst &Ext, Type *VT,
+ Type *ExtendedVT,
+ const ValueToShadowMap &Map) {
+ Value *const OrigSource = Ext.getOperand(0);
+ Type *const OrigSourceTy = OrigSource->getType();
+ Type *const ExtendedSourceTy = Config.getExtendedFPType(OrigSourceTy);
+ // When extending:
+ // - (A) If the source has a shadow, we extend from the shadow, else we
+ // extend from the original source.
+ // - (B) If the shadow of the dest is larger than the shadow of the source,
+ // we still need an extend. Else, the shadow of the source is the same
+ // type as the shadow of the dest (because mappings are non-decreasing), so
+ // we don't need to emit an extend.
+ // Examples,
+ // with a mapping of {f32->f64;f64->f80;f80->f128}
+ // fpext half %1 to float -> fpext half %1 to double
+ // fpext half %1 to double -> fpext half %1 to x86_fp80
+ // fpext half %1 to x86_fp80 -> fpext half %1 to fp128
+ // fpext float %1 to double -> double s(%1)
+ // fpext float %1 to x86_fp80 -> fpext double s(%1) to fp128
+ // fpext double %1 to x86_fp80 -> fpext x86_fp80 s(%1) to fp128
+ // with a mapping of {f32->f64;f64->f128;f80->f128}
+ // fpext half %1 to float -> fpext half %1 to double
+ // fpext half %1 to double -> fpext half %1 to fp128
+ // fpext half %1 to x86_fp80 -> fpext half %1 to fp128
+ // fpext float %1 to double -> fpext double s(%1) to fp128
+ // fpext float %1 to x86_fp80 -> fpext double s(%1) to fp128
+ // fpext double %1 to x86_fp80 -> fp128 s(%1)
+ // with a mapping of {f32->f32;f64->f32;f80->f64}
+ // fpext half %1 to float -> fpext half %1 to float
+ // fpext half %1 to double -> fpext half %1 to float
+ // fpext half %1 to x86_fp80 -> fpext half %1 to double
+ // fpext float %1 to double -> s(%1)
+ // fpext float %1 to x86_fp80 -> fpext float s(%1) to double
+ // fpext double %1 to x86_fp80 -> fpext float s(%1) to double
+
+ // See (A) above.
+ Value *Source = ExtendedSourceTy ? Map.getShadow(OrigSource) : OrigSource;
+ Type *SourceTy = ExtendedSourceTy ? ExtendedSourceTy : OrigSourceTy;
+ // See (B) above.
+ if (SourceTy == ExtendedVT)
+ return Source;
+
+ Instruction *Shadow =
+ CastInst::Create(Instruction::FPExt, Source, ExtendedVT);
+ Shadow->insertAfter(&Ext);
+ return Shadow;
+}
+
+// Returns a value with the address of the callee.
+Value *
+NumericalStabilitySanitizer::getCalleeAddress(CallBase &Call,
+ IRBuilder<> &Builder) const {
+ if (Function *Fn = Call.getCalledFunction()) {
+ // We're calling a statically known function.
+ return Builder.CreatePtrToInt(Fn, IntptrTy);
+ } else {
+ // We're calling a function through a function pointer.
+ return Builder.CreatePtrToInt(Call.getCalledOperand(), IntptrTy);
+ }
+}
+
+namespace {
+
+// FIXME: This should be tablegen-ed.
+
+struct KnownIntrinsic {
+ struct WidenedIntrinsic {
+ const char *NarrowName;
+ Intrinsic::ID ID; // wide id.
+ using FnTypeFactory = FunctionType *(*)(LLVMContext &);
+ FnTypeFactory MakeFnTy;
+ };
+
+ static const char *get(LibFunc LFunc);
+
+ // Given an intrinsic with an `FT` argument, try to find a wider intrinsic
+ // that applies the same operation on the shadow argument.
+ // Options are:
+ // - pass in the ID and full function type,
+ // - pass in the name, which includes the function type through mangling.
+ static const WidenedIntrinsic *widen(StringRef Name);
+
+private:
+ struct LFEntry {
+ LibFunc LFunc;
+ const char *IntrinsicName;
+ };
+ static const LFEntry kLibfuncIntrinsics[];
+
+ static const WidenedIntrinsic kWidenedIntrinsics[];
+};
+
+FunctionType *Make_Double_Double(LLVMContext &C) {
+ return FunctionType::get(Type::getDoubleTy(C), {Type::getDoubleTy(C)}, false);
+}
+
+FunctionType *Make_X86FP80_X86FP80(LLVMContext &C) {
+ return FunctionType::get(Type::getX86_FP80Ty(C), {Type::getX86_FP80Ty(C)},
+ false);
+}
+
+FunctionType *Make_Double_DoubleI32(LLVMContext &C) {
+ return FunctionType::get(Type::getDoubleTy(C),
+ {Type::getDoubleTy(C), Type::getInt32Ty(C)}, false);
+}
+
+FunctionType *Make_X86FP80_X86FP80I32(LLVMContext &C) {
+ return FunctionType::get(Type::getX86_FP80Ty(C),
+ {Type::getX86_FP80Ty(C), Type::getInt32Ty(C)},
+ false);
+}
+
+FunctionType *Make_Double_DoubleDouble(LLVMContext &C) {
+ return FunctionType::get(Type::getDoubleTy(C),
+ {Type::getDoubleTy(C), Type::getDoubleTy(C)}, false);
+}
+
+FunctionType *Make_X86FP80_X86FP80X86FP80(LLVMContext &C) {
+ return FunctionType::get(Type::getX86_FP80Ty(C),
+ {Type::getX86_FP80Ty(C), Type::getX86_FP80Ty(C)},
+ false);
+}
+
+FunctionType *Make_Double_DoubleDoubleDouble(LLVMContext &C) {
+ return FunctionType::get(
+ Type::getDoubleTy(C),
+ {Type::getDoubleTy(C), Type::getDoubleTy(C), Type::getDoubleTy(C)},
+ false);
+}
+
+FunctionType *Make_X86FP80_X86FP80X86FP80X86FP80(LLVMContext &C) {
+ return FunctionType::get(
+ Type::getX86_FP80Ty(C),
+ {Type::getX86_FP80Ty(C), Type::getX86_FP80Ty(C), Type::getX86_FP80Ty(C)},
+ false);
+}
+
+const KnownIntrinsic::WidenedIntrinsic KnownIntrinsic::kWidenedIntrinsics[] = {
+ // FIXME: Right now we ignore vector intrinsics.
+ // This is hard because we have to model the semantics of the intrinsics,
+ // e.g. llvm.x86.sse2.min.sd means extract first element, min, insert back.
+ // Intrinsics that take any non-vector FT types:
+ // NOTE: Right now because of https://bugs.llvm.org/show_bug.cgi?id=45399
+ // for f128 we need to use Make_X86FP80_X86FP80 (go to a lower precision and
+ // come back).
+ {"llvm.sqrt.f32", Intrinsic::sqrt, Make_Double_Double},
----------------
alexander-shaposhnikov wrote:
Yeah, I agree, this solution is not great. In its current state, it primarily functions as a way to create a static table (type, intrinsic, factory). We probably should replace this mechanism entirely, either by generating the table with TableGen or by eliminating the use of mangled names and passing type information differently.
I've tried the latter, but it's very tedious. I agree with the point, but honestly would prefer to fix it later.
https://github.com/llvm/llvm-project/pull/85916
More information about the llvm-commits
mailing list