[llvm] [SLP] Support memory runtime alias checks (PR #203631)
Alexey Bataev via llvm-commits
llvm-commits at lists.llvm.org
Thu Jun 18 09:57:57 PDT 2026
================
@@ -24388,6 +24552,437 @@ Value *BoUpSLP::vectorizeTree(TreeEntry *E) {
return nullptr;
}
+bool BoUpSLP::isRuntimeCheckableAliasPair(Instruction *Inst1,
+ Instruction *Inst2) {
+ // Only simple (non-volatile, non-atomic) accesses may be reordered once
+ // aliasing is ruled out at runtime; volatile/atomic ordering must be kept.
+ if (!isSimple(Inst1) || !isSimple(Inst2))
+ return false;
+ Value *Ptr1 = getLoadStorePointerOperand(Inst1);
+ Value *Ptr2 = getLoadStorePointerOperand(Inst2);
+ // Only simple load/store accesses have a single pointer operand whose
+ // accessed range can be bounded and compared at runtime.
+ if (!Ptr1 || !Ptr2)
+ return false;
+ if (Ptr1->getType()->getPointerAddressSpace() !=
+ Ptr2->getType()->getPointerAddressSpace())
+ return false;
+ const Value *Base1 = getUnderlyingObject(Ptr1);
+ const Value *Base2 = getUnderlyingObject(Ptr2);
+ // A range check is only meaningful between two distinct, identifiable
+ // objects.
+ if (Base1 == Base2 || isa<UndefValue>(Base1) || isa<UndefValue>(Base2))
+ return false;
+ return true;
+}
+
+bool BoUpSLP::isCoveredByExistingVersionCheck(BasicBlock *BB,
+ Instruction *Inst1,
+ Instruction *Inst2) const {
+ const auto It = VersionedBlockCheckedPairs.find(BB);
+ if (It == VersionedBlockCheckedPairs.end())
+ return false;
+ Value *Ptr1 = getLoadStorePointerOperand(Inst1);
+ Value *Ptr2 = getLoadStorePointerOperand(Inst2);
+ if (!Ptr1 || !Ptr2)
+ return false;
+ const Value *Base1 = getUnderlyingObject(Ptr1);
+ const Value *Base2 = getUnderlyingObject(Ptr2);
+ if (Base1 == Base2)
+ return false;
+ if (Base2 < Base1)
+ std::swap(Base1, Base2);
+ return It->second.contains({Base1, Base2});
+}
+
+/// Returns true if \p BB's body already contains vector instructions, e.g.
+/// from an earlier SLP vectorization in the same pass.
+static bool blockBodyHasVectorInstructions(BasicBlock *BB) {
+ for (Instruction &I : *BB) {
+ if (isa<PHINode>(&I) || I.isTerminator())
+ continue;
+ // A vector-producing instruction (vector load, binop, shuffle, etc.) has a
+ // vector result type.
+ if (getValueType(&I)->isVectorTy())
+ return true;
+ }
+ return false;
+}
+
+void BoUpSLP::captureRuntimeCheckBodySnapshot() {
+ RTOrigBodyOrder.clear();
+ if (!TryRuntimeAliasChecks || !RTChecks.BB)
+ return;
+ BasicBlock *BB = RTChecks.BB;
+ for (Instruction &I : *BB)
+ if (!isa<PHINode>(&I) && !I.isTerminator())
+ RTOrigBodyOrder.push_back(&I);
+}
+
+bool BoUpSLP::recordRuntimeAliasCheck(BasicBlock *BB, Instruction *Inst1,
+ Instruction *Inst2) {
+ Value *Ptr1 = getLoadStorePointerOperand(Inst1);
+ Value *Ptr2 = getLoadStorePointerOperand(Inst2);
+ if (!Ptr1 || !Ptr2)
+ return false;
+ const Value *Base1 = getUnderlyingObject(Ptr1);
+ const Value *Base2 = getUnderlyingObject(Ptr2);
+ if (Base1 == Base2)
+ return false;
+ // Only a single block can be versioned per attempt.
+ if (RTChecks.BB && RTChecks.BB != BB)
+ return false;
+ // Normalize the pair order so duplicate checks collapse.
+ if (Base2 < Base1)
+ std::swap(Base1, Base2);
+ auto Pair = std::make_pair(Base1, Base2);
+ // After the checks have been validated and bounded, do not introduce new
+ // pairs.
+ if (RTChecksFinalized)
+ return RTChecks.BasePairs.contains(Pair);
+ RTChecks.BB = BB;
+ RTChecks.BasePairs.insert(Pair);
+ return true;
+}
+
+bool BoUpSLP::canVersionForRuntimeChecks() {
+ if (RTChecks.BasePairs.empty() || !RTChecks.BB)
+ return false;
+ if (RTChecks.BasePairs.size() > SLPMaxRuntimeAliasChecks)
+ return false;
+ BasicBlock *BB = RTChecks.BB;
+ // Never version a scalar fallback block: it is the safe, original-order copy
+ // taken when aliasing is detected and must stay scalar.
+ if (ScalarFallbackBlocks.contains(BB))
+ return false;
+ // Versioning duplicates the block body; only straight-line code outside any
+ // loop is handled for now, to avoid LoopInfo and region updates.
+ if (!LI || LI->getLoopFor(BB))
+ return false;
+ if (!BB->getTerminator())
+ return false;
+ if (!DT)
+ return false;
+ // The scalar fallback must be a faithful copy of the original scalar body.
+ // Reject blocks that were already partially vectorized earlier in this pass.
+ if (blockBodyHasVectorInstructions(BB))
+ return false;
+
+ // Versioning duplicates the original block body into a scalar fallback.
+ // Calls that cannot be duplicated or whose semantics depend on the call being
+ // immediately followed by a return cannot be cloned into the diamond.
+ if (BB->getTerminatingMustTailCall())
+ return false;
+ if (any_of(*BB, [](Instruction &I) {
+ auto *CB = dyn_cast<CallBase>(&I);
+ return CB && (CB->cannotDuplicate() || CB->isConvergent());
+ }))
+ return false;
+
+ // The scalar fallback is a clone of the body. Operands defined inside the
+ // body are remapped to their clones, but operands defined outside the body
+ // (header PHIs, dominating definitions) are reused as-is by the clone. If
+ // such an outside operand is itself vectorized by this tree, vectorizeTree()
+ // will delete its scalar, leaving the clone with a dangling, out-of-tree use.
+ // Versioning cannot model that, so bail out.
+ for (Instruction &I : *BB) {
+ if (isa<PHINode>(&I) || I.isTerminator())
+ continue;
+ for (Value *Op : I.operands()) {
+ auto *OpI = dyn_cast<Instruction>(Op);
+ if (!OpI)
+ continue;
+ bool DefinedInBody =
+ OpI->getParent() == BB && !isa<PHINode>(OpI) && !OpI->isTerminator();
+ if (!DefinedInBody && isVectorized(OpI))
+ return false;
+ }
+ }
+
+ SmallPtrSet<const Value *, 8> Bases;
+ for (const auto &P : RTChecks.BasePairs) {
+ Bases.insert(P.first);
+ Bases.insert(P.second);
+ }
+ // Every base object must be available in the (PHI-only) header where the
+ // guard branch is emitted.
+ if (any_of(Bases, [&](const Value *Base) {
+ const auto *I = dyn_cast<Instruction>(Base);
+ return I && (I->getParent() == BB || !DT->dominates(I, BB));
+ }))
+ return false;
+
+ // Compute the accessed address range [Low, High) for every involved base.
+ RTChecks.Bounds.clear();
+ SmallMapVector<Value *, std::pair<const SCEV *, const SCEV *>, 4> OffBounds;
+ unsigned NumMemInsts = 0;
+ for (Instruction &I : *BB) {
+ if (!I.mayReadOrWriteMemory())
+ continue;
+ // The dependency scan truncates beyond MaxMemDepDistance. Only version
+ // blocks small enough that the scan is never truncated, so every
+ // conflicting base pair is guaranteed to be recorded.
+ if (++NumMemInsts > MaxMemDepDistance)
+ return false;
+ Value *Ptr = getLoadStorePointerOperand(&I);
+ // A non-simple memory access (e.g. a call) keeps its dependencies, so its
+ // relative order with kept-dependency partners is preserved inside the
+ // duplicated body and it does not need a bound.
+ if (!Ptr)
+ continue;
+ Value *Base = const_cast<Value *>(getUnderlyingObject(Ptr));
+ if (!Bases.contains(Base))
+ continue;
+ TypeSize TS = DL->getTypeStoreSize(getLoadStoreType(&I));
+ if (TS.isScalable())
+ return false;
+ Type *IntTy = DL->getIntPtrType(Base->getType());
+ const SCEV *PtrSC = SE->getPtrToIntExpr(SE->getSCEV(Ptr), IntTy);
+ const SCEV *BaseSC = SE->getPtrToIntExpr(SE->getSCEV(Base), IntTy);
+ if (isa<SCEVCouldNotCompute>(PtrSC) || isa<SCEVCouldNotCompute>(BaseSC))
+ return false;
+ // Byte offset of this access from its base. Require a constant so the bound
+ // stays base-plus-constant, and non-negative so the unsigned [Low, High)
+ // range below actually covers the access.
+ const auto *Off = dyn_cast<SCEVConstant>(SE->getMinusSCEV(PtrSC, BaseSC));
+ if (!Off || Off->getAPInt().isNegative())
+ return false;
+ const SCEV *EndOff =
+ SE->getAddExpr(Off, SE->getConstant(IntTy, TS.getFixedValue()));
+ auto [It, Inserted] = OffBounds.try_emplace(Base, Off, EndOff);
+ if (!Inserted) {
+ // Constant operands, so these fold to a constant min/max offset rather
+ // than emitting a runtime umin/umax.
+ It->second.first = SE->getUMinExpr(It->second.first, Off);
+ It->second.second = SE->getUMaxExpr(It->second.second, EndOff);
+ }
+ }
+ // Materialize the absolute [Low, High) = base + [minOff, maxEndOff). The
+ // offsets are constants, so each bound is a base-plus-constant expression
+ // that expands to a single add off the base address (no runtime umin/umax).
+ for (const auto &[Base, Off] : OffBounds) {
+ Type *IntTy = DL->getIntPtrType(Base->getType());
+ const SCEV *BaseSC = SE->getPtrToIntExpr(SE->getSCEV(Base), IntTy);
+ RTChecks.Bounds.try_emplace(Base, SE->getAddExpr(BaseSC, Off.first),
+ SE->getAddExpr(BaseSC, Off.second));
+ }
+ // Every involved base must contribute at least one bounded access, and its
+ // bounds must be expandable at the guard (so the checks only reference values
+ // available in the header).
+ SCEVExpander Exp(*SE, "slp.rtcheck");
+ // The guard is emitted after the header PHIs, so validate expandability at
+ // the first non-PHI: only values available in the header (PHIs, arguments,
+ // values defined before the block) dominate that point.
+ Instruction *GuardPt = &*BB->getFirstNonPHIIt();
+ if (any_of(Bases, [&](const Value *Base) {
+ auto *It = RTChecks.Bounds.find(Base);
+ return It == RTChecks.Bounds.end() ||
+ !Exp.isSafeToExpandAt(It->second.first, GuardPt) ||
+ !Exp.isSafeToExpandAt(It->second.second, GuardPt);
+ }))
+ return false;
+
+ // No SSA value defined in the body may escape the versioned region: that
+ // would require a merge PHI in the continuation block, which is not yet
+ // supported. A use by the terminator counts as an escape because the
+ // terminator is moved into the continuation block.
+ for (Instruction &I : *BB) {
+ if (isa<PHINode>(&I) || I.isTerminator())
+ continue;
+ if (any_of(I.users(), [&](User *U) {
+ auto *UI = dyn_cast<Instruction>(U);
+ return !UI || UI->getParent() != BB || UI->isTerminator() ||
+ isa<PHINode>(UI);
+ }))
+ return false;
+ }
+
+ // The runtime checks and the guard branch run on both the vector and the
+ // scalar fallback path. When aliasing is detected at runtime the scalar code
+ // executes unchanged plus the check overhead, so bound the check cost
+ // relative to the guarded scalar region to avoid pessimizing that path too
+ // much. The scalar region cost is the cost of the (current, still scalar)
+ // block body; no IR is emitted here.
+ constexpr TTI::TargetCostKind CostKind = TTI::TCK_RecipThroughput;
+ InstructionCost ScalarCost = 0;
+ for (Instruction &I : *BB) {
+ if (isa<PHINode>(&I) || I.isTerminator())
+ continue;
+ ScalarCost += TTI->getInstructionCost(&I, CostKind);
+ }
+ InstructionCost CheckCost = getRuntimeChecksCost();
+ if (!ScalarCost.isValid() || !CheckCost.isValid())
+ return false;
+ if (CheckCost * 100 >
+ ScalarCost * SLPRuntimeAliasChecksMaxScalarCostPercent.getValue())
+ return false;
+
+ // Freeze the check set so the final scheduleBlock() keeps the same dropped
+ // dependencies the bounds were just computed for.
+ RTChecksFinalized = true;
+ return true;
+}
+
+InstructionCost BoUpSLP::getRuntimeChecksCost() const {
+ if (RTChecks.BasePairs.empty())
+ return 0;
+ LLVMContext &Ctx = F->getContext();
+ Type *IntTy = DL->getIntPtrType(Ctx);
+ Type *I1Ty = Type::getInt1Ty(Ctx);
+ constexpr TTI::TargetCostKind Kind = TTI::TCK_RecipThroughput;
+ unsigned NumBases = RTChecks.Bounds.size();
+ unsigned NumPairs = RTChecks.BasePairs.size();
+ InstructionCost Cost = 0;
+ // Per base: a [Low, High) address pair. canVersionForRuntimeChecks() folds
+ // each bound to a base-plus-constant offset, so it expands to two integer
+ // adds (one per bound) with no runtime umin/umax reduction.
+ Cost += TTI->getArithmeticInstrCost(Instruction::Add, IntTy, Kind) *
+ (2 * NumBases);
+ // Two integer compares and one logical and per checked pair.
+ InstructionCost CmpCost = TTI->getCmpSelInstrCost(
+ Instruction::ICmp, IntTy, I1Ty, CmpInst::ICMP_ULT, Kind);
+ InstructionCost AndCost =
+ TTI->getArithmeticInstrCost(Instruction::And, I1Ty, Kind);
+ Cost += (CmpCost * 2 + AndCost) * NumPairs;
+ // Or-reduction of the per-pair conflicts.
+ if (NumPairs > 1)
+ Cost += TTI->getArithmeticInstrCost(Instruction::Or, I1Ty, Kind) *
+ (NumPairs - 1);
+ // The guard branch.
+ Cost += TTI->getCFInstrCost(Instruction::CondBr, Kind);
+ return Cost;
+}
+
+Value *BoUpSLP::emitRuntimeAliasCheck(IRBuilderBase &Builder,
+ SCEVExpander &Exp) {
+ // Materialize the [Low, High) address bounds for every involved base from
+ // their SCEVs at the builder's insertion point.
+ SmallDenseMap<const Value *, std::pair<Value *, Value *>, 4> Range;
+ for (const auto &[Base, Bound] : RTChecks.Bounds) {
+ Type *IntTy = DL->getIntPtrType(Base->getType());
+ Value *Low =
+ Exp.expandCodeFor(Bound.first, IntTy, Builder.GetInsertPoint());
+ Value *High =
+ Exp.expandCodeFor(Bound.second, IntTy, Builder.GetInsertPoint());
+ Range[Base] = {Low, High};
+ }
+ Value *Cond = nullptr;
+ for (const auto &P : RTChecks.BasePairs) {
+ std::pair<Value *, Value *> A = Range.lookup(P.first);
+ std::pair<Value *, Value *> B = Range.lookup(P.second);
+ assert(A.first && A.second && B.first && B.second &&
+ "Missing bounds for a checked base object");
+ // The objects overlap iff (A.Low < B.High) && (B.Low < A.High).
+ Value *C0 = Builder.CreateICmpULT(A.first, B.second, "rt.bound0");
+ Value *C1 = Builder.CreateICmpULT(B.first, A.second, "rt.bound1");
+ Value *Conflict = Builder.CreateAnd(C0, C1, "rt.conflict");
+ Cond =
+ Cond ? Builder.CreateOr(Cond, Conflict, "rt.conflict.all") : Conflict;
+ }
+ return Builder.CreateFreeze(Cond, "rt.guard");
+}
+
+void BoUpSLP::versionBlocksForRuntimeChecks() {
+ // Only version when this is the optimistic attempt that collected the checks.
+ if (!TryRuntimeAliasChecks || RTChecks.BasePairs.empty() || !RTChecks.BB ||
+ RTOrigBodyOrder.empty())
+ return;
+ BasicBlock *BB = RTChecks.BB;
+ Function *Fn = BB->getParent();
+ LLVMContext &Ctx = BB->getContext();
+ Instruction *Term = BB->getTerminator();
+ assert(Term && "Versioned block must have a terminator");
+
+ // Build the diamond:
+ // BB (header: PHIs + guard) -> {VecBB, ScalarBB} -> Tail (terminator).
+ BasicBlock *VecBB = BasicBlock::Create(Ctx, BB->getName() + ".rtvec", Fn);
+ BasicBlock *ScalarBB =
+ BasicBlock::Create(Ctx, BB->getName() + ".rtscalar", Fn);
+ BasicBlock *Tail = BasicBlock::Create(Ctx, BB->getName() + ".rtcont", Fn);
+
+ // Clone the body into the scalar fallback block, in the original program
+ // order (RTOrigBodyOrder).
+ SmallDenseMap<Value *, Value *, 16> VMap;
+ for (Instruction *I : RTOrigBodyOrder) {
+ assert(I->getParent() == BB &&
+ "RTOrigBodyOrder entry no longer in the versioned block");
+ Instruction *C = I->clone();
+ if (I->hasName())
+ C->setName(I->getName() + ".scalar");
+ C->insertInto(ScalarBB, ScalarBB->end());
+ VMap[I] = C;
+ }
+ // Remap the clones to use the cloned definitions; references to values
+ // defined outside the body (header or function arguments) are unchanged.
+ for (Instruction &I : *ScalarBB)
+ for (Use &U : I.operands())
+ if (Value *V = VMap.lookup(U.get()))
+ U.set(V);
+
+ // Move the body (everything but PHIs and the terminator) into the vector
+ // block in the scheduled order.
+ for (Instruction &I : make_early_inc_range(*BB)) {
+ if (isa<PHINode>(&I) || I.isTerminator())
+ continue;
+ I.removeFromParent();
+ I.insertInto(VecBB, VecBB->end());
+ }
+
+ // Emit the runtime alias check before the still-present terminator, so the
+ // SCEV bounds are expanded at a valid insertion point.
+ IRBuilder<> ChkBuilder(Term);
+ ChkBuilder.SetCurrentDebugLocation(Term->getDebugLoc());
+ SCEVExpander Exp(*SE, "slp.rtcheck");
+ Value *Cond = emitRuntimeAliasCheck(ChkBuilder, Exp);
+
+ // Move the original terminator into the continuation block and replace it
+ // with the guard branch.
+ Term->removeFromParent();
+ Term->insertInto(Tail, Tail->end());
+ UncondBrInst::Create(Tail, VecBB);
+ UncondBrInst::Create(Tail, ScalarBB);
+ ChkBuilder.SetInsertPoint(BB);
+ ChkBuilder.CreateCondBr(Cond, ScalarBB, VecBB);
+
+ // The continuation block is the new predecessor of the original successors.
+ for (BasicBlock *Succ : successors(Term))
+ Succ->replacePhiUsesWith(BB, Tail);
+
+ // Keep the dominator tree valid for the remainder of the run.
+ if (DT) {
+ DomTreeUpdater DTU(*DT, DomTreeUpdater::UpdateStrategy::Eager);
+ SmallVector<DominatorTree::UpdateType> Updates = {
+ {DominatorTree::Insert, BB, VecBB},
+ {DominatorTree::Insert, BB, ScalarBB},
+ {DominatorTree::Insert, VecBB, Tail},
+ {DominatorTree::Insert, ScalarBB, Tail}};
+ for (BasicBlock *Succ : successors(Term)) {
+ Updates.push_back({DominatorTree::Insert, Tail, Succ});
+ Updates.push_back({DominatorTree::Delete, BB, Succ});
----------------
alexey-bataev wrote:
After recomputation BB still dominates Succ transitively, which is the intended result - we only changed the edge set, not a dominance claim.
https://github.com/llvm/llvm-project/pull/203631
More information about the llvm-commits
mailing list