[llvm-branch-commits] [llvm] [LLVM] Add DeviceAddressSanitizer instrumentation pass (PR #242443)

Joseph Huber via llvm-branch-commits llvm-branch-commits at lists.llvm.org
Sat Nov 14 11:19:28 PST 2026


================
@@ -0,0 +1,879 @@
+//===- DeviceAddressSanitizer.cpp - device address sanitizer --------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+//
+// Bounds checking for device code. Allocations live in [2^45, 2^46). That
+// window is 32 regions of 2^40 bytes, one per size class, in virtual memory:
+//
+//   bits [45]     = 1                      in the window
+//   bits [44:40]  = C                      size class
+//   bits [39:0]   = offset in region C
+//
+// Class C has chunks of 2^(6 + C) bytes. Chunk index is offset >> (6 + C).
+// Metadata is an array of i64 at the high end of the region, one word per
+// reachable chunk, indexed backwards from the last byte:
+//
+//   [ chunk 0 | chunk 1 | ... | unused | ... | md N-1 | ... | md 1 | md 0 ]
+//   |<------ 2^40 bytes ----------------------------------------------- >|
+//
+// We assume that the metadata is unchanged inside of the module by asserting it
+// does not alias with anything else. This allows the load to be hoisted, but
+// degrades use-after-free identification. An operation that changes metadata
+// state, like memory allocation, should not be instrumented by this pass.
+//
+//   %v = load i32, ptr addrspace(1) %p
+//
+// becomes:
+//
+//   ; bound: decode base(%p)
+//   %addr     = ptrtoint ptr addrspace(1) %p to i64
+//   %valid    = icmp eq i64 (lshr %addr, 45), 1      ; bit 45: in the window?
+//   %class    = and i64 (lshr %addr, 40), 31         ; bits [44:40]: size class
+//   %shift    = add i64 %class, 6                    ; chunk size 2^(6+C)
+//   %inregion = and i64 %addr, (1<<40)-1             ; offset in this class
+//   %idx      = lshr i64 %inregion, %shift           ; chunk index ("block")
+//   %slot     = or i64 (shl %idx, 3), 7              ; last byte of md word idx
+//   %meta     = sub i64 (%addr or (1<<40)-1), %slot  ; region top minus slot
+//   %ep       = inttoptr (select %valid, %meta, @__dasan_no_entry)
+//   %entry    = load i64, ptr addrspace(1) %ep, align 8, !alias.scope !0
+//   %size     = and i64 %entry, (1<<37)-1            ; live allocation size
+//   %off      = ashr i64 (and %entry, ~((1<<38)-1)), 32 ; chunk-base offset
+//   %chunkoff = sub i64 %inregion, (shl %idx, %shift)
+//   %start    = sub i64 %addr, (sub %chunkoff, %off) ; object base
+//
+//   ; check: this access against that bound
+//   %oob = icmp uge i64 (%addr - %start + 3), %size   ; last byte of i32
+//   %bad = and i1 %valid, %oob                        ; ignore non-window ptrs
+//   br i1 %bad, label %fail, label %go, !prof !unlikely
+// fail:
+//   call void @__dasan_report_load(ptr %p, i64 4, i64 %start, i64 %size)
+//   unreachable
+// go:
+//   %v = load i32, ptr addrspace(1) %p, !noalias !0
+//
+//===----------------------------------------------------------------------===//
+
+#include "llvm/Transforms/Instrumentation/DeviceAddressSanitizer.h"
+#include "llvm/ADT/DenseMap.h"
+#include "llvm/ADT/STLExtras.h"
+#include "llvm/ADT/SmallVector.h"
+#include "llvm/ADT/Statistic.h"
+#include "llvm/Analysis/InstSimplifyFolder.h"
+#include "llvm/Analysis/MemoryBuiltins.h"
+#include "llvm/Analysis/ValueTracking.h"
+#include "llvm/IR/Constants.h"
+#include "llvm/IR/DataLayout.h"
+#include "llvm/IR/DerivedTypes.h"
+#include "llvm/IR/IRBuilder.h"
+#include "llvm/IR/InstIterator.h"
+#include "llvm/IR/Instructions.h"
+#include "llvm/IR/IntrinsicInst.h"
+#include "llvm/IR/IntrinsicsAMDGPU.h"
+#include "llvm/IR/MDBuilder.h"
+#include "llvm/IR/Module.h"
+#include "llvm/Support/AMDGPUAddrSpace.h"
+#include "llvm/Support/CommandLine.h"
+#include "llvm/Support/MathExtras.h"
+#include "llvm/Transforms/Instrumentation/AddressSanitizerCommon.h"
+#include "llvm/Transforms/Utils/BasicBlockUtils.h"
+#include "llvm/Transforms/Utils/Instrumentation.h"
+#include "llvm/Transforms/Utils/ModuleUtils.h"
+
+using namespace llvm;
+
+#define DEBUG_TYPE "dasan"
+
+STATISTIC(NumInstrumentedReads, "Number of instrumented reads");
+STATISTIC(NumInstrumentedWrites, "Number of instrumented writes");
+STATISTIC(NumDecodes, "Number of base pointer decodes emitted");
+
+// Constants describing the layout of the tracked memory region.
+static constexpr uint64_t kSpaceSizeLog = 45;
+static constexpr uint64_t kNumClassesLog = 5;
+static constexpr uint64_t kClassMask = (1ULL << kNumClassesLog) - 1;
+static constexpr uint64_t kRegionSizeLog = kSpaceSizeLog - kNumClassesLog;
+static constexpr uint64_t kRegionMask = (1ULL << kRegionSizeLog) - 1;
+static constexpr uint64_t kMinSizeLog = 6;
+static constexpr uint64_t kMinAlignment = 1ULL << kMinSizeLog;
+static constexpr uint64_t kMetadataSizeLog = 3;
+static constexpr uint64_t kMetadataSize = 1ULL << kMetadataSizeLog;
+static constexpr uint64_t kSizeBits = 37;
+static constexpr uint64_t kSizeMask = (1ULL << kSizeBits) - 1;
+static constexpr uint64_t kOffsetShift = kSizeBits + 1;
+static constexpr uint64_t kOffsetFieldMask = ~((1ULL << kOffsetShift) - 1);
+static constexpr uint64_t kOffsetScaleLog = kMinSizeLog;
+static constexpr unsigned kWordBits = 64;
+static_assert(kOffsetShift < kWordBits, "an entry is one word");
+
+static constexpr uint64_t kMetadataBytes =
+    (1ULL << (kRegionSizeLog - kMinSizeLog)) << kMetadataSizeLog;
+
+static constexpr uint64_t countSizeClasses() {
+  uint64_t N = 0;
+  while (N <= kClassMask &&
+         (1ULL << (kMinSizeLog + N)) <= (kRegionMask + 1) - kMetadataBytes)
+    ++N;
+  return N;
+}
+static_assert(countSizeClasses() << kRegionSizeLog == 1ULL << kSpaceSizeLog,
+              "the layout must keep every size class");
+static_assert(kMinSizeLog + countSizeClasses() - 1 <= kSizeBits,
+              "the largest chunk outgrows its entry");
+
+static constexpr char kReportLoadName[] = "__dasan_report_load";
+static constexpr char kReportStoreName[] = "__dasan_report_store";
+static constexpr char kReportSharedLoadName[] = "__dasan_report_shared_load";
+static constexpr char kReportSharedStoreName[] = "__dasan_report_shared_store";
+static constexpr char kRecoverSuffix[] = "_noabort";
+static constexpr char kModuleMarkerName[] = "__dasan_instrumented";
+static constexpr char kGlobalGuardName[] = "__dasan_guard";
+static constexpr char kNoEntryName[] = "__dasan_no_entry";
+static constexpr char kGlobalsSectionName[] = ".rodata.dasan.globals";
+static constexpr char kGlobalsDataSectionName[] = ".data.dasan.globals";
+static constexpr char kGlobalsRelRoSectionName[] = ".data.rel.ro.dasan.globals";
+static constexpr char kGlobalsBssSectionName[] = ".sbss.dasan.globals";
+
+static cl::opt<bool> ClInstrumentReads("dasan-instrument-reads", cl::Hidden,
+                                       cl::init(true));
+static cl::opt<bool> ClInstrumentWrites("dasan-instrument-writes", cl::Hidden,
+                                        cl::init(true));
+static cl::opt<bool> ClInstrumentAtomics("dasan-instrument-atomics", cl::Hidden,
+                                         cl::init(true));
+static cl::opt<bool> ClRecover("dasan-recover", cl::Hidden, cl::init(false));
+static cl::opt<bool> ClTrapOnError("dasan-trap-on-error", cl::Hidden,
+                                   cl::init(false));
+static cl::opt<cl::boolOrDefault>
+    ClInstrumentGlobals("dasan-instrument-globals", cl::Hidden,
+                        cl::desc("Wrap globals into the dasan window"));
+static cl::opt<bool> ClInstrumentShared("dasan-instrument-shared", cl::Hidden,
+                                        cl::init(true));
+
+namespace {
+
+// A single memory access.
+struct Access {
+  Instruction *I;
+  BasicBlock *Block;
+  Value *Ptr;
+  Value *Len;
+  uint64_t FixedLen;
+  MaybeAlign Alignment;
+  bool IsWrite;
+};
+
+// Access information from its associated metadata.
+struct Bound {
+  Value *Start;
+  Value *Size;
+  Value *Valid;
+};
+
+class NoSanitizeInserter final : public IRBuilderDefaultInserter {
+public:
+  void InsertHelper(Instruction *I, const Twine &Name,
+                    BasicBlock::iterator IP) const {
+    IRBuilderDefaultInserter::InsertHelper(I, Name, IP);
+    I->setMetadata(LLVMContext::MD_nosanitize,
+                   MDNode::get(I->getContext(), {}));
+  }
+};
+
+using BuilderTy = IRBuilder<InstSimplifyFolder, NoSanitizeInserter>;
+
+class DeviceAddressSanitizer {
+public:
+  DeviceAddressSanitizer(Module &M, const DeviceAddressSanitizerOptions &Opts)
+      : M(M), C(M.getContext()), DL(M.getDataLayout()),
+        IntptrTy(DL.getIntPtrType(C)), Int32Ty(Type::getInt32Ty(C)),
+        Recover(ClRecover.getNumOccurrences() ? ClRecover : Opts.Recover),
+        CheckShared(ClInstrumentShared && M.getTargetTriple().isAMDGPU()),
+        GlobalsAS(DL.getDefaultGlobalsAddressSpace()) {}
+
+  bool run();
+
+private:
+  void initializeCallbacks();
+  bool placeGlobals();
+  void emitModuleMarker();
+  bool instrumentFunction(Function &F);
+  void collect(Instruction &I, SmallVectorImpl<Access> &Accesses) const;
+  bool instrumentPointer(Value *Ptr) const;
+  bool shouldPlaceGlobal(const GlobalVariable &G) const;
+  bool isGlobalAS(unsigned AS) const;
+  bool isLocalAS(unsigned AS) const;
+  bool isPrivateAS(unsigned AS) const;
+  bool relocateGlobals() const;
+
+  void initEntryScope();
+  void markNoAliasEntry(Instruction *I) const;
+  Bound heapBound(BuilderTy &IRB, Value *Base, bool KnownMine = false);
+  Bound sharedBound(BuilderTy &IRB, Value *Ptr, Value *Limit);
+  void check(BuilderTy &IRB, const Access &A, const Bound &B, Instruction *At,
+             bool Shared, BasicBlock *Trap);
+  Value *sharedIntPtr(BuilderTy &IRB, Value *P);
+  Value *groupSegmentLimit(BuilderTy &IRB, Function &F);
+  void splitTo(BuilderTy &IRB, Instruction *At, Value *Bad, BasicBlock *Fail);
+  BasicBlock *createTrap(Function &F);
+  void emitReport(BuilderTy &IRB, Value *Addr, Value *Len, Value *Start,
+                  Value *Size, bool Shared, bool IsWrite);
+  GlobalVariable *sizedSharedGV(const Value *P) const;
+  GlobalVariable *anySharedGV(const Value *P) const;
+  uint64_t sharedAllocSize(const GlobalVariable &G) const;
+  bool aborting() const { return !Recover || ClTrapOnError; }
+
+  Constant *intptr(uint64_t V) const { return ConstantInt::get(IntptrTy, V); }
+  Constant *int32(uint32_t V) const { return ConstantInt::get(Int32Ty, V); }
+  void setPoint(BuilderTy &IRB, Instruction *At) const {
+    IRB.SetInsertPoint(At);
+    IRB.SetCurrentDebugLocation(Loc);
+  }
+
+  Module &M;
+  LLVMContext &C;
+  const DataLayout &DL;
+  IntegerType *IntptrTy;
+  Type *Int32Ty;
+  bool Recover;
+  bool CheckShared;
+  unsigned GlobalsAS;
+  FunctionCallee Report[2];
+  FunctionCallee ReportShared[2];
+  Constant *NoEntry = nullptr;
+  MDNode *EntryScope = nullptr;
+  DebugLoc Loc;
+};
+
+} // namespace
+
+bool DeviceAddressSanitizer::isGlobalAS(unsigned AS) const {
+  if (!M.getTargetTriple().isAMDGPU())
+    return AS == 0 || AS == GlobalsAS;
+  return AS == AMDGPUAS::FLAT_ADDRESS || AS == AMDGPUAS::GLOBAL_ADDRESS ||
+         AS == AMDGPUAS::CONSTANT_ADDRESS;
+}
+
+bool DeviceAddressSanitizer::isLocalAS(unsigned AS) const {
+  return M.getTargetTriple().isAMDGPU() && AS == AMDGPUAS::LOCAL_ADDRESS;
+}
+
+bool DeviceAddressSanitizer::isPrivateAS(unsigned AS) const {
+  return M.getTargetTriple().isAMDGPU() && AS == AMDGPUAS::PRIVATE_ADDRESS;
+}
+
+bool DeviceAddressSanitizer::relocateGlobals() const {
+  switch (ClInstrumentGlobals) {
+  case cl::boolOrDefault::BOU_TRUE:
+    return true;
+  case cl::boolOrDefault::BOU_FALSE:
+    return false;
+  case cl::boolOrDefault::BOU_UNSET:
+    return M.getTargetTriple().isAMDGPU();
+  }
+  llvm_unreachable("invalid -dasan-instrument-globals");
+}
+
+// Special accesses that should not be instrumented like the dispatch packet.
+static bool isSpecialMemory(const Value *Obj) {
+  const auto *II = dyn_cast<IntrinsicInst>(Obj);
+  if (!II)
+    return false;
+  switch (II->getIntrinsicID()) {
+  case Intrinsic::amdgcn_dispatch_ptr:
+  case Intrinsic::amdgcn_implicitarg_ptr:
+  case Intrinsic::amdgcn_kernarg_segment_ptr:
+  case Intrinsic::amdgcn_queue_ptr:
+  case Intrinsic::amdgcn_implicit_buffer_ptr:
+    return true;
+  default:
+    return false;
+  }
+}
+
+bool DeviceAddressSanitizer::instrumentPointer(Value *Ptr) const {
+  unsigned AS = Ptr->getType()->getPointerAddressSpace();
+  if (isLocalAS(AS))
+    return CheckShared;
+  if (isPrivateAS(AS))
+    return false;
+  if (!isGlobalAS(AS))
+    return false;
+  const Value *Obj = getUnderlyingObject(Ptr);
+  return !isa<AllocaInst>(Obj) && !isSpecialMemory(Obj);
+}
+
+void DeviceAddressSanitizer::collect(Instruction &I,
+                                     SmallVectorImpl<Access> &Accesses) const {
+  auto Add = [&](Value *Ptr, Value *Len, uint64_t Fixed, MaybeAlign A,
+                 bool IsWrite) {
+    if (!(IsWrite ? ClInstrumentWrites : ClInstrumentReads))
+      return;
+    if (!instrumentPointer(Ptr))
+      return;
+    Accesses.push_back({&I, I.getParent(), Ptr, Len,
+                        Fixed <= UINT32_MAX ? Fixed : 0, A, IsWrite});
+  };
+  auto AddTyped = [&](Value *Ptr, Type *Ty, MaybeAlign A, bool IsWrite) {
+    TypeSize Size = DL.getTypeStoreSize(Ty);
+    if (Size.isScalable() || Size.isZero())
+      return;
+    Add(Ptr, intptr(Size.getFixedValue()), Size.getFixedValue(), A, IsWrite);
+  };
+
+  // Collect every interesting instruction that touches memory.
+  if (auto *LI = dyn_cast<LoadInst>(&I))
+    AddTyped(LI->getPointerOperand(), LI->getType(), LI->getAlign(), false);
+  else if (auto *SI = dyn_cast<StoreInst>(&I))
+    AddTyped(SI->getPointerOperand(), SI->getValueOperand()->getType(),
+             SI->getAlign(), true);
+  else if (auto *RMW = dyn_cast<AtomicRMWInst>(&I)) {
+    if (ClInstrumentAtomics)
+      AddTyped(RMW->getPointerOperand(), RMW->getValOperand()->getType(),
+               std::nullopt, true);
+  } else if (auto *XCHG = dyn_cast<AtomicCmpXchgInst>(&I)) {
+    if (ClInstrumentAtomics)
+      AddTyped(XCHG->getPointerOperand(), XCHG->getCompareOperand()->getType(),
+               std::nullopt, true);
+  } else if (auto *MI = dyn_cast<AnyMemIntrinsic>(&I)) {
----------------
jhuber6 wrote:

There are a lot of intrinsics that will get checking in the future. I am trying to keep the patch as small as possible and plan to add those in a future patch. Left a TODO for now.

https://github.com/llvm/llvm-project/pull/242443


More information about the llvm-branch-commits mailing list