[llvm-branch-commits] [llvm] [LLVM] Add DeviceAddressSanitizer instrumentation pass (PR #242443)
Yaxun Liu via llvm-branch-commits
llvm-branch-commits at lists.llvm.org
Fri Nov 13 19:26:13 PST 2026
================
@@ -0,0 +1,879 @@
+//===- DeviceAddressSanitizer.cpp - device address sanitizer --------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+//
+// Bounds checking for device code. Allocations live in [2^45, 2^46). That
+// window is 32 regions of 2^40 bytes, one per size class, in virtual memory:
+//
+// bits [45] = 1 in the window
+// bits [44:40] = C size class
+// bits [39:0] = offset in region C
+//
+// Class C has chunks of 2^(6 + C) bytes. Chunk index is offset >> (6 + C).
+// Metadata is an array of i64 at the high end of the region, one word per
+// reachable chunk, indexed backwards from the last byte:
+//
+// [ chunk 0 | chunk 1 | ... | unused | ... | md N-1 | ... | md 1 | md 0 ]
+// |<------ 2^40 bytes ----------------------------------------------- >|
+//
+// We assume that the metadata is unchanged inside of the module by asserting it
+// does not alias with anything else. This allows the load to be hoisted, but
+// degrades use-after-free identification. An operation that changes metadata
+// state, like memory allocation, should not be instrumented by this pass.
+//
+// %v = load i32, ptr addrspace(1) %p
+//
+// becomes:
+//
+// ; bound: decode base(%p)
+// %addr = ptrtoint ptr addrspace(1) %p to i64
+// %valid = icmp eq i64 (lshr %addr, 45), 1 ; bit 45: in the window?
+// %class = and i64 (lshr %addr, 40), 31 ; bits [44:40]: size class
+// %shift = add i64 %class, 6 ; chunk size 2^(6+C)
+// %inregion = and i64 %addr, (1<<40)-1 ; offset in this class
+// %idx = lshr i64 %inregion, %shift ; chunk index ("block")
+// %slot = or i64 (shl %idx, 3), 7 ; last byte of md word idx
+// %meta = sub i64 (%addr or (1<<40)-1), %slot ; region top minus slot
+// %ep = inttoptr (select %valid, %meta, @__dasan_no_entry)
+// %entry = load i64, ptr addrspace(1) %ep, align 8, !alias.scope !0
+// %size = and i64 %entry, (1<<37)-1 ; live allocation size
+// %off = ashr i64 (and %entry, ~((1<<38)-1)), 32 ; chunk-base offset
+// %chunkoff = sub i64 %inregion, (shl %idx, %shift)
+// %start = sub i64 %addr, (sub %chunkoff, %off) ; object base
+//
+// ; check: this access against that bound
+// %oob = icmp uge i64 (%addr - %start + 3), %size ; last byte of i32
+// %bad = and i1 %valid, %oob ; ignore non-window ptrs
+// br i1 %bad, label %fail, label %go, !prof !unlikely
+// fail:
+// call void @__dasan_report_load(ptr %p, i64 4, i64 %start, i64 %size)
+// unreachable
+// go:
+// %v = load i32, ptr addrspace(1) %p, !noalias !0
+//
+//===----------------------------------------------------------------------===//
+
+#include "llvm/Transforms/Instrumentation/DeviceAddressSanitizer.h"
+#include "llvm/ADT/DenseMap.h"
+#include "llvm/ADT/STLExtras.h"
+#include "llvm/ADT/SmallVector.h"
+#include "llvm/ADT/Statistic.h"
+#include "llvm/Analysis/InstSimplifyFolder.h"
+#include "llvm/Analysis/MemoryBuiltins.h"
+#include "llvm/Analysis/ValueTracking.h"
+#include "llvm/IR/Constants.h"
+#include "llvm/IR/DataLayout.h"
+#include "llvm/IR/DerivedTypes.h"
+#include "llvm/IR/IRBuilder.h"
+#include "llvm/IR/InstIterator.h"
+#include "llvm/IR/Instructions.h"
+#include "llvm/IR/IntrinsicInst.h"
+#include "llvm/IR/IntrinsicsAMDGPU.h"
+#include "llvm/IR/MDBuilder.h"
+#include "llvm/IR/Module.h"
+#include "llvm/Support/AMDGPUAddrSpace.h"
+#include "llvm/Support/CommandLine.h"
+#include "llvm/Support/MathExtras.h"
+#include "llvm/Transforms/Instrumentation/AddressSanitizerCommon.h"
+#include "llvm/Transforms/Utils/BasicBlockUtils.h"
+#include "llvm/Transforms/Utils/Instrumentation.h"
+#include "llvm/Transforms/Utils/ModuleUtils.h"
+
+using namespace llvm;
+
+#define DEBUG_TYPE "dasan"
+
+STATISTIC(NumInstrumentedReads, "Number of instrumented reads");
+STATISTIC(NumInstrumentedWrites, "Number of instrumented writes");
+STATISTIC(NumDecodes, "Number of base pointer decodes emitted");
+
+// Constants describing the layout of the tracked memory region.
+static constexpr uint64_t kSpaceSizeLog = 45;
+static constexpr uint64_t kNumClassesLog = 5;
+static constexpr uint64_t kClassMask = (1ULL << kNumClassesLog) - 1;
+static constexpr uint64_t kRegionSizeLog = kSpaceSizeLog - kNumClassesLog;
+static constexpr uint64_t kRegionMask = (1ULL << kRegionSizeLog) - 1;
+static constexpr uint64_t kMinSizeLog = 6;
+static constexpr uint64_t kMinAlignment = 1ULL << kMinSizeLog;
+static constexpr uint64_t kMetadataSizeLog = 3;
+static constexpr uint64_t kMetadataSize = 1ULL << kMetadataSizeLog;
+static constexpr uint64_t kSizeBits = 37;
+static constexpr uint64_t kSizeMask = (1ULL << kSizeBits) - 1;
+static constexpr uint64_t kOffsetShift = kSizeBits + 1;
+static constexpr uint64_t kOffsetFieldMask = ~((1ULL << kOffsetShift) - 1);
+static constexpr uint64_t kOffsetScaleLog = kMinSizeLog;
+static constexpr unsigned kWordBits = 64;
+static_assert(kOffsetShift < kWordBits, "an entry is one word");
+
+static constexpr uint64_t kMetadataBytes =
+ (1ULL << (kRegionSizeLog - kMinSizeLog)) << kMetadataSizeLog;
+
+static constexpr uint64_t countSizeClasses() {
+ uint64_t N = 0;
+ while (N <= kClassMask &&
+ (1ULL << (kMinSizeLog + N)) <= (kRegionMask + 1) - kMetadataBytes)
+ ++N;
+ return N;
+}
+static_assert(countSizeClasses() << kRegionSizeLog == 1ULL << kSpaceSizeLog,
+ "the layout must keep every size class");
+static_assert(kMinSizeLog + countSizeClasses() - 1 <= kSizeBits,
+ "the largest chunk outgrows its entry");
+
+static constexpr char kReportLoadName[] = "__dasan_report_load";
+static constexpr char kReportStoreName[] = "__dasan_report_store";
+static constexpr char kReportSharedLoadName[] = "__dasan_report_shared_load";
+static constexpr char kReportSharedStoreName[] = "__dasan_report_shared_store";
+static constexpr char kRecoverSuffix[] = "_noabort";
+static constexpr char kModuleMarkerName[] = "__dasan_instrumented";
+static constexpr char kGlobalGuardName[] = "__dasan_guard";
+static constexpr char kNoEntryName[] = "__dasan_no_entry";
+static constexpr char kGlobalsSectionName[] = ".rodata.dasan.globals";
+static constexpr char kGlobalsDataSectionName[] = ".data.dasan.globals";
+static constexpr char kGlobalsRelRoSectionName[] = ".data.rel.ro.dasan.globals";
+static constexpr char kGlobalsBssSectionName[] = ".sbss.dasan.globals";
+
+static cl::opt<bool> ClInstrumentReads("dasan-instrument-reads", cl::Hidden,
+ cl::init(true));
+static cl::opt<bool> ClInstrumentWrites("dasan-instrument-writes", cl::Hidden,
+ cl::init(true));
+static cl::opt<bool> ClInstrumentAtomics("dasan-instrument-atomics", cl::Hidden,
+ cl::init(true));
+static cl::opt<bool> ClRecover("dasan-recover", cl::Hidden, cl::init(false));
+static cl::opt<bool> ClTrapOnError("dasan-trap-on-error", cl::Hidden,
+ cl::init(false));
+static cl::opt<cl::boolOrDefault>
+ ClInstrumentGlobals("dasan-instrument-globals", cl::Hidden,
+ cl::desc("Wrap globals into the dasan window"));
+static cl::opt<bool> ClInstrumentShared("dasan-instrument-shared", cl::Hidden,
+ cl::init(true));
+
+namespace {
+
+// A single memory access.
+struct Access {
+ Instruction *I;
+ BasicBlock *Block;
+ Value *Ptr;
+ Value *Len;
+ uint64_t FixedLen;
+ MaybeAlign Alignment;
+ bool IsWrite;
+};
+
+// Access information from its associated metadata.
+struct Bound {
+ Value *Start;
+ Value *Size;
+ Value *Valid;
+};
+
+class NoSanitizeInserter final : public IRBuilderDefaultInserter {
+public:
+ void InsertHelper(Instruction *I, const Twine &Name,
+ BasicBlock::iterator IP) const {
+ IRBuilderDefaultInserter::InsertHelper(I, Name, IP);
+ I->setMetadata(LLVMContext::MD_nosanitize,
+ MDNode::get(I->getContext(), {}));
+ }
+};
+
+using BuilderTy = IRBuilder<InstSimplifyFolder, NoSanitizeInserter>;
+
+class DeviceAddressSanitizer {
+public:
+ DeviceAddressSanitizer(Module &M, const DeviceAddressSanitizerOptions &Opts)
+ : M(M), C(M.getContext()), DL(M.getDataLayout()),
+ IntptrTy(DL.getIntPtrType(C)), Int32Ty(Type::getInt32Ty(C)),
+ Recover(ClRecover.getNumOccurrences() ? ClRecover : Opts.Recover),
+ CheckShared(ClInstrumentShared && M.getTargetTriple().isAMDGPU()),
+ GlobalsAS(DL.getDefaultGlobalsAddressSpace()) {}
+
+ bool run();
+
+private:
+ void initializeCallbacks();
+ bool placeGlobals();
+ void emitModuleMarker();
+ bool instrumentFunction(Function &F);
+ void collect(Instruction &I, SmallVectorImpl<Access> &Accesses) const;
+ bool instrumentPointer(Value *Ptr) const;
+ bool shouldPlaceGlobal(const GlobalVariable &G) const;
+ bool isGlobalAS(unsigned AS) const;
+ bool isLocalAS(unsigned AS) const;
+ bool isPrivateAS(unsigned AS) const;
+ bool relocateGlobals() const;
+
+ void initEntryScope();
+ void markNoAliasEntry(Instruction *I) const;
+ Bound heapBound(BuilderTy &IRB, Value *Base, bool KnownMine = false);
+ Bound sharedBound(BuilderTy &IRB, Value *Ptr, Value *Limit);
+ void check(BuilderTy &IRB, const Access &A, const Bound &B, Instruction *At,
+ bool Shared, BasicBlock *Trap);
+ Value *sharedIntPtr(BuilderTy &IRB, Value *P);
+ Value *groupSegmentLimit(BuilderTy &IRB, Function &F);
+ void splitTo(BuilderTy &IRB, Instruction *At, Value *Bad, BasicBlock *Fail);
+ BasicBlock *createTrap(Function &F);
+ void emitReport(BuilderTy &IRB, Value *Addr, Value *Len, Value *Start,
+ Value *Size, bool Shared, bool IsWrite);
+ GlobalVariable *sizedSharedGV(const Value *P) const;
+ GlobalVariable *anySharedGV(const Value *P) const;
+ uint64_t sharedAllocSize(const GlobalVariable &G) const;
+ bool aborting() const { return !Recover || ClTrapOnError; }
+
+ Constant *intptr(uint64_t V) const { return ConstantInt::get(IntptrTy, V); }
+ Constant *int32(uint32_t V) const { return ConstantInt::get(Int32Ty, V); }
+ void setPoint(BuilderTy &IRB, Instruction *At) const {
+ IRB.SetInsertPoint(At);
+ IRB.SetCurrentDebugLocation(Loc);
+ }
+
+ Module &M;
+ LLVMContext &C;
+ const DataLayout &DL;
+ IntegerType *IntptrTy;
+ Type *Int32Ty;
+ bool Recover;
+ bool CheckShared;
+ unsigned GlobalsAS;
+ FunctionCallee Report[2];
+ FunctionCallee ReportShared[2];
+ Constant *NoEntry = nullptr;
+ MDNode *EntryScope = nullptr;
+ DebugLoc Loc;
+};
+
+} // namespace
+
+bool DeviceAddressSanitizer::isGlobalAS(unsigned AS) const {
+ if (!M.getTargetTriple().isAMDGPU())
+ return AS == 0 || AS == GlobalsAS;
+ return AS == AMDGPUAS::FLAT_ADDRESS || AS == AMDGPUAS::GLOBAL_ADDRESS ||
+ AS == AMDGPUAS::CONSTANT_ADDRESS;
+}
+
+bool DeviceAddressSanitizer::isLocalAS(unsigned AS) const {
+ return M.getTargetTriple().isAMDGPU() && AS == AMDGPUAS::LOCAL_ADDRESS;
+}
+
+bool DeviceAddressSanitizer::isPrivateAS(unsigned AS) const {
+ return M.getTargetTriple().isAMDGPU() && AS == AMDGPUAS::PRIVATE_ADDRESS;
+}
+
+bool DeviceAddressSanitizer::relocateGlobals() const {
+ switch (ClInstrumentGlobals) {
+ case cl::boolOrDefault::BOU_TRUE:
+ return true;
+ case cl::boolOrDefault::BOU_FALSE:
+ return false;
+ case cl::boolOrDefault::BOU_UNSET:
+ return M.getTargetTriple().isAMDGPU();
+ }
+ llvm_unreachable("invalid -dasan-instrument-globals");
+}
+
+// Special accesses that should not be instrumented like the dispatch packet.
+static bool isSpecialMemory(const Value *Obj) {
+ const auto *II = dyn_cast<IntrinsicInst>(Obj);
+ if (!II)
+ return false;
+ switch (II->getIntrinsicID()) {
+ case Intrinsic::amdgcn_dispatch_ptr:
+ case Intrinsic::amdgcn_implicitarg_ptr:
+ case Intrinsic::amdgcn_kernarg_segment_ptr:
+ case Intrinsic::amdgcn_queue_ptr:
+ case Intrinsic::amdgcn_implicit_buffer_ptr:
+ return true;
+ default:
+ return false;
+ }
+}
+
+bool DeviceAddressSanitizer::instrumentPointer(Value *Ptr) const {
+ unsigned AS = Ptr->getType()->getPointerAddressSpace();
+ if (isLocalAS(AS))
+ return CheckShared;
+ if (isPrivateAS(AS))
+ return false;
+ if (!isGlobalAS(AS))
+ return false;
+ const Value *Obj = getUnderlyingObject(Ptr);
+ return !isa<AllocaInst>(Obj) && !isSpecialMemory(Obj);
+}
+
+void DeviceAddressSanitizer::collect(Instruction &I,
+ SmallVectorImpl<Access> &Accesses) const {
+ auto Add = [&](Value *Ptr, Value *Len, uint64_t Fixed, MaybeAlign A,
+ bool IsWrite) {
+ if (!(IsWrite ? ClInstrumentWrites : ClInstrumentReads))
+ return;
+ if (!instrumentPointer(Ptr))
+ return;
+ Accesses.push_back({&I, I.getParent(), Ptr, Len,
+ Fixed <= UINT32_MAX ? Fixed : 0, A, IsWrite});
+ };
+ auto AddTyped = [&](Value *Ptr, Type *Ty, MaybeAlign A, bool IsWrite) {
+ TypeSize Size = DL.getTypeStoreSize(Ty);
+ if (Size.isScalable() || Size.isZero())
+ return;
+ Add(Ptr, intptr(Size.getFixedValue()), Size.getFixedValue(), A, IsWrite);
+ };
+
+ // Collect every interesting instruction that touches memory.
+ if (auto *LI = dyn_cast<LoadInst>(&I))
+ AddTyped(LI->getPointerOperand(), LI->getType(), LI->getAlign(), false);
+ else if (auto *SI = dyn_cast<StoreInst>(&I))
+ AddTyped(SI->getPointerOperand(), SI->getValueOperand()->getType(),
+ SI->getAlign(), true);
+ else if (auto *RMW = dyn_cast<AtomicRMWInst>(&I)) {
+ if (ClInstrumentAtomics)
+ AddTyped(RMW->getPointerOperand(), RMW->getValOperand()->getType(),
+ std::nullopt, true);
+ } else if (auto *XCHG = dyn_cast<AtomicCmpXchgInst>(&I)) {
+ if (ClInstrumentAtomics)
+ AddTyped(XCHG->getPointerOperand(), XCHG->getCompareOperand()->getType(),
+ std::nullopt, true);
+ } else if (auto *MI = dyn_cast<AnyMemIntrinsic>(&I)) {
----------------
yxsamliu wrote:
Please check masked loads and stores too. O3 can produce these, but DASan currently skips them. Only active lanes should be checked. Please add tests.
https://github.com/llvm/llvm-project/pull/242443
More information about the llvm-branch-commits
mailing list