[llvm-branch-commits] [llvm] [LLVM] Add a ConcurrencySanitizer LLVM pass (PR #225780)
Matt Arsenault via llvm-branch-commits
llvm-branch-commits at lists.llvm.org
Fri Oct 2 02:02:47 PDT 2026
================
@@ -0,0 +1,613 @@
+//===- ConcurrencySanitizer.cpp - watchpoint race detector ----------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+//
+// This file implements instrumentation for ConcurrencySanitizer, a sampling
+// data-race detector. Memory accesses are preceded by runtime probes while the
+// original operations remain unchanged.
+//
+//===----------------------------------------------------------------------===//
+
+#include "llvm/Transforms/Instrumentation/ConcurrencySanitizer.h"
+#include "llvm/ADT/BitmaskEnum.h"
+#include "llvm/ADT/DenseSet.h"
+#include "llvm/ADT/SmallVector.h"
+#include "llvm/ADT/Statistic.h"
+#include "llvm/ADT/StringExtras.h"
+#include "llvm/ADT/bit.h"
+#include "llvm/Analysis/CaptureTracking.h"
+#include "llvm/Analysis/TargetLibraryInfo.h"
+#include "llvm/Analysis/ValueTracking.h"
+#include "llvm/IR/DataLayout.h"
+#include "llvm/IR/IRBuilder.h"
+#include "llvm/IR/Instructions.h"
+#include "llvm/IR/IntrinsicInst.h"
+#include "llvm/IR/Intrinsics.h"
+#include "llvm/IR/Module.h"
+#include "llvm/ProfileData/InstrProf.h"
+#include "llvm/Support/AMDGPUAddrSpace.h"
+#include "llvm/Support/CommandLine.h"
+#include "llvm/Support/NVPTXAddrSpace.h"
+#include "llvm/TargetParser/Triple.h"
+#include "llvm/Transforms/Utils/EscapeEnumerator.h"
+#include "llvm/Transforms/Utils/Instrumentation.h"
+#include "llvm/Transforms/Utils/Local.h"
+#include "llvm/Transforms/Utils/ModuleUtils.h"
+
+using namespace llvm;
+
+#define DEBUG_TYPE "csan"
+
+static cl::opt<bool> ClInstrumentMemoryAccesses(
+ "csan-instrument-memory-accesses", cl::init(true),
+ cl::desc("Instrument memory accesses"), cl::Hidden);
+static cl::opt<bool>
+ ClInstrumentFuncEntryExit("csan-instrument-func-entry-exit", cl::init(true),
+ cl::desc("Instrument function entry and exit"),
+ cl::Hidden);
+static cl::opt<bool> ClHandleCxxExceptions(
+ "csan-handle-cxx-exceptions", cl::init(true),
+ cl::desc("Handle C++ exceptions (insert cleanup blocks for unwinding)"),
+ cl::Hidden);
+static cl::opt<bool> ClInstrumentAtomics("csan-instrument-atomics",
+ cl::init(true),
+ cl::desc("Instrument atomics"),
+ cl::Hidden);
+static cl::opt<bool> ClInstrumentMemIntrinsics(
+ "csan-instrument-memintrinsics", cl::init(true),
+ cl::desc("Instrument memintrinsics (memset/memcpy/memmove)"), cl::Hidden);
+static cl::opt<bool> ClDistinguishVolatile(
+ "csan-distinguish-volatile", cl::init(false),
+ cl::desc("Treat aligned volatile accesses up to 8 bytes as atomic"),
+ cl::Hidden);
+STATISTIC(NumInstrumentedReads, "Number of instrumented reads");
+STATISTIC(NumInstrumentedWrites, "Number of instrumented writes");
+STATISTIC(NumOmittedReadsBeforeWrite,
+ "Number of reads ignored due to following writes");
+
+static constexpr char kCsanModuleCtorName[] = "csan.module_ctor";
+static constexpr char kCsanInitName[] = "__csan_init";
+
+namespace {
+
+/// Must match CSAN_ACCESS_* in compiler-rt/lib/csan/csan_defs.h.
+enum class AccessFlags : unsigned {
+ None = 0,
+ Atomic = 1u << 0,
+ Compound = 1u << 1,
+ LLVM_MARK_AS_BITMASK_ENUM(Compound),
+};
+
+static bool isAtomicMemoryAccess(const Instruction *I) {
+ auto SSID = getAtomicSyncScopeID(I);
+ if (!SSID)
+ return false;
+ if (isa<LoadInst>(I) || isa<StoreInst>(I))
+ return *SSID != SyncScope::SingleThread;
+ return true;
+}
+
+static Value *getCallbackAddress(IRBuilderBase &IRB, Value *Addr) {
+ return IRB.CreateAddrSpaceCast(Addr, IRB.getPtrTy());
+}
+
+static ConstantInt *createOrdering(IRBuilderBase &IRB, AtomicOrdering Ord) {
+ uint32_t Value = 0;
+ switch (Ord) {
+ case AtomicOrdering::NotAtomic:
+ llvm_unreachable("unexpected atomic ordering");
+ case AtomicOrdering::Unordered:
+ case AtomicOrdering::Monotonic:
+ Value = 0;
+ break;
+ case AtomicOrdering::Acquire:
+ Value = 2;
+ break;
+ case AtomicOrdering::Release:
+ Value = 3;
+ break;
+ case AtomicOrdering::AcquireRelease:
+ Value = 4;
+ break;
+ case AtomicOrdering::SequentiallyConsistent:
+ Value = 5;
+ break;
+ }
+ return IRB.getInt32(Value);
+}
+
+static int getAccessSizeIndex(Type *Ty, const DataLayout &DL) {
+ assert(Ty->isSized());
+ if (Ty->isScalableTy())
+ return -1;
+ uint32_t TypeSize = DL.getTypeStoreSizeInBits(Ty);
+ if (TypeSize != 8 && TypeSize != 16 && TypeSize != 32 && TypeSize != 64 &&
+ TypeSize != 128)
+ return -1;
+ unsigned Idx = llvm::countr_zero(TypeSize / 8);
+ return static_cast<int>(Idx);
+}
+
+static bool addressSpaceMayRace(const Triple &T, unsigned AS) {
+ if (T.isAMDGPU())
+ // GDS and buffer fat pointers cannot form a generic watchpoint key.
+ return AS == AMDGPUAS::FLAT_ADDRESS || AS == AMDGPUAS::GLOBAL_ADDRESS ||
+ AS == AMDGPUAS::LOCAL_ADDRESS;
+ if (T.isNVPTX())
+ return AS == NVPTXAS::ADDRESS_SPACE_GENERIC ||
+ AS == NVPTXAS::ADDRESS_SPACE_GLOBAL ||
+ AS == NVPTXAS::ADDRESS_SPACE_SHARED ||
+ AS == NVPTXAS::ADDRESS_SPACE_SHARED_CLUSTER;
+ if (T.isSPIRV())
+ // FIXME: No exposed address spaces for SPIR-V.
+ return false;
+ return AS == 0;
+}
+
+struct ConcurrencySanitizer {
+ bool sanitizeFunction(Function &F, const TargetLibraryInfo &TLI);
+
+private:
+ struct MemoryAccessLists {
+ SmallVector<Instruction *, 8> LoadsAndStores;
+ SmallVector<Instruction *, 8> AtomicAccesses;
+ SmallVector<IntrinsicInst *, 8> MemIntrinCalls;
+ bool HasCalls = false;
+ };
+
+ void initialize(Module &M, const TargetLibraryInfo &TLI);
+ void collectMemoryAccesses(Function &F, MemoryAccessLists &Out);
+ bool instrumentLoadOrStore(Instruction *I, const DataLayout &DL);
+ bool instrumentAtomic(Instruction *I, const DataLayout &DL);
+ bool instrumentMemIntrinsic(IntrinsicInst *II, const DataLayout &DL);
+ bool insertAccessProbe(Instruction *I, Value *Addr, Type *AccessTy,
+ const DataLayout &DL, bool IsWrite, bool IsCompound,
+ bool IsAtomic);
+ bool shouldInstrumentAddress(Value *Addr) const;
+ bool shouldInstrumentAccess(Instruction *I) const;
+ void insertFuncEntryExit(Function &F);
+ void insertRuntimeIgnores(Function &F);
+ Constant *getFlags(AccessFlags Flags) const {
+ return ConstantInt::get(FlagsTy, to_underlying(Flags));
+ }
+
+ Module *Mod = nullptr;
+ Type *IntptrTy = nullptr;
+ IntegerType *FlagsTy = nullptr;
+
+ // Accesses sizes are powers of two: 1, 2, 4, 8, 16.
+ static const size_t kNumAccessSizes = 5;
+ // void __csan_readN(ptr, i32);
+ FunctionCallee CsanRead[kNumAccessSizes];
+ // void __csan_writeN(ptr, i32);
+ FunctionCallee CsanWrite[kNumAccessSizes];
+ // void __csan_unaligned_readN(ptr, i32);
+ FunctionCallee CsanUnalignedRead[kNumAccessSizes];
+ // void __csan_unaligned_writeN(ptr, i32);
+ FunctionCallee CsanUnalignedWrite[kNumAccessSizes];
+ // void __csan_read_writeN(ptr, i32);
+ FunctionCallee CsanCompoundRW[kNumAccessSizes];
+ // void __csan_unaligned_read_writeN(ptr, i32);
+ FunctionCallee CsanUnalignedCompoundRW[kNumAccessSizes];
+ // void __csan_func_entry(ptr);
+ FunctionCallee CsanFuncEntry;
+ // void __csan_func_exit();
+ FunctionCallee CsanFuncExit;
+ // void __csan_ignore_thread_begin();
+ FunctionCallee CsanIgnoreBegin;
+ // void __csan_ignore_thread_end();
+ FunctionCallee CsanIgnoreEnd;
+ // void __csan_read_range(ptr, intptr_t, i32);
+ FunctionCallee CsanReadRange;
+ // void __csan_write_range(ptr, intptr_t, i32);
+ FunctionCallee CsanWriteRange;
+ // void __csan_atomic_thread_fence(i32);
+ FunctionCallee CsanAtomicThreadFence;
+ // void __csan_atomic_signal_fence(i32);
+ FunctionCallee CsanAtomicSignalFence;
+};
+
+void insertModuleCtor(Module &M) {
+ getOrCreateSanitizerCtorAndInitFunctions(
+ M, kCsanModuleCtorName, kCsanInitName, /*InitArgTypes=*/{},
+ /*InitArgs=*/{},
+ [&](Function *Ctor, FunctionCallee) { appendToGlobalCtors(M, Ctor, 0); });
+}
+
+} // namespace
+
+PreservedAnalyses ConcurrencySanitizerPass::run(Function &F,
+ FunctionAnalysisManager &FAM) {
+ ConcurrencySanitizer CSan;
+ if (CSan.sanitizeFunction(F, FAM.getResult<TargetLibraryAnalysis>(F)))
+ return PreservedAnalyses::none();
+ return PreservedAnalyses::all();
+}
+
+PreservedAnalyses ModuleConcurrencySanitizerPass::run(Module &M,
+ ModuleAnalysisManager &) {
+ if (checkIfAlreadyInstrumented(M, "nosanitize_concurrency"))
+ return PreservedAnalyses::all();
+ insertModuleCtor(M);
+ return PreservedAnalyses::none();
+}
+
+void ConcurrencySanitizer::initialize(Module &M, const TargetLibraryInfo &TLI) {
+ LLVMContext &Ctx = M.getContext();
+ Mod = &M;
+ IntptrTy = M.getDataLayout().getIntPtrType(Ctx);
+ FlagsTy = Type::getInt32Ty(Ctx);
+
+ AttributeList Attr = AttributeList().addFnAttribute(Ctx, Attribute::NoUnwind);
+ IRBuilder<> IRB(Ctx);
+ Type *VoidTy = IRB.getVoidTy();
+ Type *PtrTy = IRB.getPtrTy();
+ CsanFuncEntry =
+ M.getOrInsertFunction("__csan_func_entry", Attr, VoidTy, PtrTy);
+ CsanFuncExit = M.getOrInsertFunction("__csan_func_exit", Attr, VoidTy);
+ CsanIgnoreBegin =
+ M.getOrInsertFunction("__csan_ignore_thread_begin", Attr, VoidTy);
+ CsanIgnoreEnd =
+ M.getOrInsertFunction("__csan_ignore_thread_end", Attr, VoidTy);
+ for (unsigned I = 0; I < kNumAccessSizes; ++I) {
+ std::string ByteSize = utostr(1U << I);
+ auto AccessFn = [&](const Twine &Name) {
+ return M.getOrInsertFunction(("__csan_" + Name).str(), Attr, VoidTy,
+ PtrTy, FlagsTy);
+ };
----------------
arsenm wrote:
Can you avoid programmatically building the function names? Eventually these should get RuntimeLibcallsInfo entries
https://github.com/llvm/llvm-project/pull/225780
More information about the llvm-branch-commits
mailing list