[llvm] [SLP] Support memory runtime alias checks (PR #203631)
via llvm-commits
llvm-commits at lists.llvm.org
Fri Jun 12 13:51:37 PDT 2026
llvmorg-github-actions[bot] wrote:
<!--LLVM PR SUMMARY COMMENT-->
@llvm/pr-subscribers-llvm-transforms
Author: Alexey Bataev (alexey-bataev)
<details>
<summary>Changes</summary>
Vectorize straight-line code blocked by runtime-checkable may-alias
dependencies by versioning the block. Drop the deps, and if the tree is
profitable, emit base-object address-range overlap checks branching to a
vector fast path or an original-order scalar fallback.
Fixes #<!-- -->201534
---
Patch is 51.43 KiB, truncated to 20.00 KiB below, full version: https://github.com/llvm/llvm-project/pull/203631.diff
4 Files Affected:
- (modified) llvm/include/llvm/Transforms/Vectorize/SLPVectorizer.h (+13)
- (modified) llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp (+696-18)
- (modified) llvm/test/Transforms/SLPVectorizer/AArch64/loadi8.ll (+41-10)
- (modified) llvm/test/Transforms/SLPVectorizer/X86/runtime-alias-checks.ll (+90)
``````````diff
diff --git a/llvm/include/llvm/Transforms/Vectorize/SLPVectorizer.h b/llvm/include/llvm/Transforms/Vectorize/SLPVectorizer.h
index af010994107a5..bb8db7272cb63 100644
--- a/llvm/include/llvm/Transforms/Vectorize/SLPVectorizer.h
+++ b/llvm/include/llvm/Transforms/Vectorize/SLPVectorizer.h
@@ -171,11 +171,24 @@ struct SLPVectorizerPass : public OptionalPassInfoMixin<SLPVectorizerPass> {
unsigned Idx, unsigned MinVF,
unsigned &Size);
+ /// Single vectorization attempt for a store chain. \p vectorizeStoreChain
+ /// wraps this to retry once with runtime alias checks enabled when the
+ /// normal attempt is blocked only by runtime-checkable may-alias
+ /// dependencies.
+ std::optional<bool> vectorizeStoreChainImpl(ArrayRef<Value *> Chain,
+ slpvectorizer::BoUpSLP &R,
+ unsigned Idx, unsigned MinVF,
+ unsigned &Size);
+
bool vectorizeStores(
ArrayRef<StoreInst *> Stores, slpvectorizer::BoUpSLP &R,
DenseSet<std::tuple<Value *, Value *, Value *, Value *, unsigned>>
&Visited);
+ /// Set by runImpl() when runtime alias check versioning changed the CFG, so
+ /// run() can drop CFG-analysis preservation only when necessary.
+ bool CFGChanged = false;
+
/// The store instructions in a basic block organized by base pointer.
StoreListMap Stores;
diff --git a/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp b/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp
index 3d5b0ebb99f4a..f34ef76557751 100644
--- a/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp
+++ b/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp
@@ -37,6 +37,7 @@
#include "llvm/Analysis/CodeMetrics.h"
#include "llvm/Analysis/ConstantFolding.h"
#include "llvm/Analysis/DemandedBits.h"
+#include "llvm/Analysis/DomTreeUpdater.h"
#include "llvm/Analysis/GlobalsModRef.h"
#include "llvm/Analysis/IVDescriptors.h"
#include "llvm/Analysis/Loads.h"
@@ -51,6 +52,7 @@
#include "llvm/Analysis/VectorUtils.h"
#include "llvm/IR/Attributes.h"
#include "llvm/IR/BasicBlock.h"
+#include "llvm/IR/CFG.h"
#include "llvm/IR/Constant.h"
#include "llvm/IR/Constants.h"
#include "llvm/IR/DataLayout.h"
@@ -278,6 +280,28 @@ static cl::opt<bool> PerLaneGatherScale(
cl::desc("Use per-lane execution scale for gather/buildvector tree "
"entries to model LICM-hoistable buildvector sequences."));
+/// Enable versioning of a basic block with runtime alias checks.
+static cl::opt<bool> SLPEnableRuntimeAliasChecks(
+ "slp-vectorize-with-runtime-alias-checks", cl::init(true), cl::Hidden,
+ cl::desc("Allow SLP to version a block with runtime alias checks to "
+ "vectorize trees blocked by may-alias memory dependencies."));
+
+/// Maximum number of runtime alias checks (one per pair of base objects) that
+/// may guard a single versioned region.
+static cl::opt<unsigned> SLPMaxRuntimeAliasChecks(
+ "slp-max-runtime-alias-checks", cl::init(8), cl::Hidden,
+ cl::desc("The maximum number of runtime alias checks generated to guard a "
+ "single SLP-vectorized region."));
+
+/// The runtime checks and the guard branch execute on both the vector and the
+/// scalar fallback path, so they add overhead to the scalar code.
+static cl::opt<unsigned> SLPRuntimeAliasChecksMaxScalarCostPercent(
+ "slp-runtime-alias-checks-max-scalar-cost-percent", cl::init(25),
+ cl::Hidden,
+ cl::desc("Maximum SLP runtime alias check cost, as a percentage of the "
+ "guarded scalar region cost, before versioning is rejected to "
+ "avoid pessimizing the scalar fallback path."));
+
// Limit the number of alias checks. The limit is chosen so that
// it has no negative effect on the llvm benchmarks.
static const unsigned AliasedCheckLimit = 10;
@@ -2204,7 +2228,64 @@ class slpvectorizer::BoUpSLP {
/// Construct a vectorizable tree that starts at \p Roots.
void buildTree(ArrayRef<Value *> Roots);
- /// Return the scalars of the root node.
+ /// Returns true if the last buildTree() observed a may-alias memory
+ /// dependency between two distinct, range-checkable base objects, i.e. a
+ /// dependency that could be turned into a runtime alias check.
+ bool hasRuntimeCheckableBlockers() const {
+ return HasRuntimeCheckableBlockers;
+ }
+
+ /// Records whether a may-alias dependency between distinct, range-checkable
+ /// base objects has been observed, so the caller can decide to retry with
+ /// runtime alias checks enabled.
+ void setHasRuntimeCheckableBlockers(bool V) {
+ HasRuntimeCheckableBlockers = V;
+ }
+
+ /// Returns true if the current vectorization attempt may drop
+ /// runtime-checkable may-alias dependencies and guard the region with
+ /// runtime alias checks.
+ bool isTryingRuntimeAliasChecks() const { return TryRuntimeAliasChecks; }
+
+ /// Enables or disables dropping runtime-checkable may-alias dependencies in
+ /// favor of runtime alias checks for the current vectorization attempt.
+ void setTryRuntimeAliasChecks(bool V) { TryRuntimeAliasChecks = V; }
+
+ /// Resets the runtime alias check data.
+ void resetRuntimeAliasCheckState() {
+ HasRuntimeCheckableBlockers = false;
+ RTChecksFinalized = false;
+ RTChecks.clear();
+ RTOrigBodyOrder.clear();
+ }
+
+ /// Snapshots RTChecks.BB's body (non-PHI, non-terminator) into
+ /// RTOrigBodyOrder in program order, for the scalar fallback.
+ void captureRuntimeCheckBodySnapshot();
+
+ /// Returns true if the runtime alias checks can be safely emitted to guard
+ /// the vectorized region.
+ bool canVersionForRuntimeChecks();
+
+ /// Returns true if \p BB is a scalar fallback block created by runtime alias
+ /// check versioning.
+ bool isScalarFallbackBlock(BasicBlock *BB) const {
+ return ScalarFallbackBlocks.contains(BB);
+ }
+
+ /// Returns the modeled cost of the runtime alias checks collected during the
+ /// last (optimistic) buildTree().
+ InstructionCost getRuntimeChecksCost() const;
+
+ /// Returns true if the last (optimistic) buildTree() collected any runtime
+ /// alias checks that must guard the vectorized region.
+ bool hasRuntimeAliasChecks() const { return !RTChecks.BasePairs.empty(); }
+
+ /// Returns true if vectorization changed the CFG (i.e. a block was versioned
+ /// with runtime alias checks). When true, CFG analyses must not be preserved.
+ bool isCFGChanged() const { return CFGChanged; }
+
+ /// Returns the scalars of the root node.
ArrayRef<Value *> getRootNodeScalars() const {
assert(!VectorizableTree.empty() && "No graph to get the first node from");
return VectorizableTree.front()->Scalars;
@@ -2305,6 +2386,9 @@ class slpvectorizer::BoUpSLP {
ExternalUses.clear();
ExternalUsesAsOriginalScalar.clear();
ExternalUsesWithNonUsers.clear();
+ RTChecks.clear();
+ HasRuntimeCheckableBlockers = false;
+ RTChecksFinalized = false;
for (auto &Iter : BlocksSchedules) {
BlockScheduling *BS = Iter.second.get();
BS->clear();
@@ -5035,6 +5119,81 @@ class slpvectorizer::BoUpSLP {
return Aliased;
}
+ /// Returns true if the may-alias dependency between simple load/store
+ /// instructions \p Inst1 and \p Inst2 could be disambiguated by a runtime
+ /// alias check.
+ bool isRuntimeCheckableAliasPair(Instruction *Inst1, Instruction *Inst2);
+
+ /// Records the (distinct base object) pair behind the may-alias dependency
+ /// of \p Inst1 and \p Inst2 as a runtime alias check guarding the region in
+ /// block \p BB. Returns true if the pair was recorded.
+ bool recordRuntimeAliasCheck(BasicBlock *BB, Instruction *Inst1,
+ Instruction *Inst2);
+
+ /// Emits the collected runtime alias checks and versions the affected block,
+ /// duplicating its body into a scalar fallback guarded by the checks.
+ void versionBlocksForRuntimeChecks();
+
+ /// Builds the i1 value that is true when any pair of checked base objects
+ /// overlaps at runtime. The base address bounds are materialized from their
+ /// SCEVs with \p Exp.
+ Value *emitRuntimeAliasCheck(IRBuilderBase &Builder, SCEVExpander &Exp);
+
+ /// Data to model and emit the runtime alias checks.
+ struct RuntimeAliasCheckInfo {
+ /// The block whose body is guarded by the checks. Exactly one block is
+ /// supported per attempt.
+ BasicBlock *BB = nullptr;
+ /// Pairs of base objects that must be proven disjoint.
+ SmallSetVector<std::pair<const Value *, const Value *>, 4> BasePairs;
+ /// Accessed address range [Low, High) for each involved base object.
+ SmallMapVector<const Value *, std::pair<const SCEV *, const SCEV *>, 4>
+ Bounds;
+
+ void clear() {
+ BB = nullptr;
+ BasePairs.clear();
+ Bounds.clear();
+ }
+ };
+
+ /// When true, scheduling drops may-alias memory dependencies between
+ /// distinct, range-checkable base objects and records them as runtime alias
+ /// checks instead.
+ bool TryRuntimeAliasChecks = false;
+
+ /// Runtime alias checks collected during the last optimistic buildTree().
+ RuntimeAliasCheckInfo RTChecks;
+
+ /// Base-object pairs already proven disjoint by the block's runtime alias
+ /// check.
+ SmallDenseMap<BasicBlock *,
+ SmallDenseSet<std::pair<const Value *, const Value *>, 4>, 2>
+ VersionedBlockCheckedPairs;
+
+ /// Scalar fallback blocks.
+ SmallPtrSet<BasicBlock *, 4> ScalarFallbackBlocks;
+
+ /// Returns true if a may-alias dependency between the simple load/store
+ /// instructions \p Inst1 and \p Inst2 in block \p BB is already covered by a
+ /// runtime alias check emitted for \p BB by a previous versioning.
+ bool isCoveredByExistingVersionCheck(BasicBlock *BB, Instruction *Inst1,
+ Instruction *Inst2) const;
+
+ /// True, if a may-alias dependency between distinct, range-checkable base
+ /// objects is observed (whether or not it was dropped).
+ bool HasRuntimeCheckableBlockers = false;
+
+ /// Runtime checks are validated and bounded the collected checks.
+ bool RTChecksFinalized = false;
+
+ /// Set when a block was versioned with runtime alias checks, which changes
+ /// the CFG. Used to drop CFG-analysis preservation for the run.
+ bool CFGChanged = false;
+
+ /// Guarded block body (non-PHI, non-terminator) in original source order.
+ SmallVector<Instruction *> RTOrigBodyOrder;
+
using AliasCacheKey = std::pair<Instruction *, Instruction *>;
/// Cache for alias results.
@@ -5618,6 +5777,7 @@ class slpvectorizer::BoUpSLP {
ScheduleCopyableDataMapByUsers.clear();
ReadyInsts.clear();
RecalcCopyableOperandDeps.clear();
+ IgnoredMemDeps.clear();
ScheduleStart = nullptr;
ScheduleEnd = nullptr;
FirstLoadStoreInRegion = nullptr;
@@ -6488,6 +6648,10 @@ class slpvectorizer::BoUpSLP {
/// recomputing the same operand more than once.
SmallSetVector<ScheduleData *, 8> RecalcCopyableOperandDeps;
+ /// Ordered pairs (Src, Dst) of memory instructions whose may-alias
+ /// dependency has been dropped in favor of a runtime alias check.
+ SmallDenseSet<std::pair<Instruction *, Instruction *>, 8> IgnoredMemDeps;
+
/// The ID of the scheduling region. For a new vectorization iteration this
/// is incremented which "removes" all ScheduleData from the region.
/// Make sure that the initial SchedulingRegionID is greater than the
@@ -24388,6 +24552,417 @@ Value *BoUpSLP::vectorizeTree(TreeEntry *E) {
return nullptr;
}
+bool BoUpSLP::isRuntimeCheckableAliasPair(Instruction *Inst1,
+ Instruction *Inst2) {
+ // Only simple (non-volatile, non-atomic) accesses may be reordered once
+ // aliasing is ruled out at runtime; volatile/atomic ordering must be kept.
+ if (!isSimple(Inst1) || !isSimple(Inst2))
+ return false;
+ Value *Ptr1 = getLoadStorePointerOperand(Inst1);
+ Value *Ptr2 = getLoadStorePointerOperand(Inst2);
+ // Only simple load/store accesses have a single pointer operand whose
+ // accessed range can be bounded and compared at runtime.
+ if (!Ptr1 || !Ptr2)
+ return false;
+ if (Ptr1->getType()->getPointerAddressSpace() !=
+ Ptr2->getType()->getPointerAddressSpace())
+ return false;
+ const Value *Base1 = getUnderlyingObject(Ptr1);
+ const Value *Base2 = getUnderlyingObject(Ptr2);
+ // A range check is only meaningful between two distinct, identifiable
+ // objects.
+ if (Base1 == Base2 || isa<UndefValue>(Base1) || isa<UndefValue>(Base2))
+ return false;
+ return true;
+}
+
+bool BoUpSLP::isCoveredByExistingVersionCheck(BasicBlock *BB,
+ Instruction *Inst1,
+ Instruction *Inst2) const {
+ const auto It = VersionedBlockCheckedPairs.find(BB);
+ if (It == VersionedBlockCheckedPairs.end())
+ return false;
+ Value *Ptr1 = getLoadStorePointerOperand(Inst1);
+ Value *Ptr2 = getLoadStorePointerOperand(Inst2);
+ if (!Ptr1 || !Ptr2)
+ return false;
+ const Value *Base1 = getUnderlyingObject(Ptr1);
+ const Value *Base2 = getUnderlyingObject(Ptr2);
+ if (Base1 == Base2)
+ return false;
+ if (Base2 < Base1)
+ std::swap(Base1, Base2);
+ return It->second.contains({Base1, Base2});
+}
+
+/// Returns true if \p BB's body already contains vector instructions, e.g.
+/// from an earlier SLP vectorization in the same pass.
+static bool blockBodyHasVectorInstructions(BasicBlock *BB) {
+ for (Instruction &I : *BB) {
+ if (isa<PHINode>(&I) || I.isTerminator())
+ continue;
+ // A vector-producing instruction (vector load, binop, shuffle, etc.) has a
+ // vector result type.
+ if (getValueType(&I)->isVectorTy())
+ return true;
+ }
+ return false;
+}
+
+void BoUpSLP::captureRuntimeCheckBodySnapshot() {
+ RTOrigBodyOrder.clear();
+ if (!TryRuntimeAliasChecks || !RTChecks.BB)
+ return;
+ BasicBlock *BB = RTChecks.BB;
+ for (Instruction &I : *BB)
+ if (!isa<PHINode>(&I) && !I.isTerminator())
+ RTOrigBodyOrder.push_back(&I);
+}
+
+bool BoUpSLP::recordRuntimeAliasCheck(BasicBlock *BB, Instruction *Inst1,
+ Instruction *Inst2) {
+ Value *Ptr1 = getLoadStorePointerOperand(Inst1);
+ Value *Ptr2 = getLoadStorePointerOperand(Inst2);
+ if (!Ptr1 || !Ptr2)
+ return false;
+ const Value *Base1 = getUnderlyingObject(Ptr1);
+ const Value *Base2 = getUnderlyingObject(Ptr2);
+ if (Base1 == Base2)
+ return false;
+ // Only a single block can be versioned per attempt.
+ if (RTChecks.BB && RTChecks.BB != BB)
+ return false;
+ // Normalize the pair order so duplicate checks collapse.
+ if (Base2 < Base1)
+ std::swap(Base1, Base2);
+ auto Pair = std::make_pair(Base1, Base2);
+ // After the checks have been validated and bounded, do not introduce new
+ // pairs.
+ if (RTChecksFinalized)
+ return RTChecks.BasePairs.contains(Pair);
+ RTChecks.BB = BB;
+ RTChecks.BasePairs.insert(Pair);
+ return true;
+}
+
+bool BoUpSLP::canVersionForRuntimeChecks() {
+ if (RTChecks.BasePairs.empty() || !RTChecks.BB)
+ return false;
+ if (RTChecks.BasePairs.size() > SLPMaxRuntimeAliasChecks)
+ return false;
+ BasicBlock *BB = RTChecks.BB;
+ // Never version a scalar fallback block: it is the safe, original-order copy
+ // taken when aliasing is detected and must stay scalar.
+ if (ScalarFallbackBlocks.contains(BB))
+ return false;
+ // Versioning duplicates the block body; only straight-line code outside any
+ // loop is handled for now, to avoid LoopInfo and region updates.
+ if (!LI || LI->getLoopFor(BB))
+ return false;
+ if (!BB->getTerminator())
+ return false;
+ if (!DT)
+ return false;
+ // The scalar fallback must be a faithful copy of the original scalar body.
+ // Reject blocks that were already partially vectorized earlier in this pass.
+ if (blockBodyHasVectorInstructions(BB))
+ return false;
+
+ // Versioning duplicates the original block body into a scalar fallback.
+ // Calls that cannot be duplicated or whose semantics depend on the call being
+ // immediately followed by a return cannot be cloned into the diamond.
+ if (BB->getTerminatingMustTailCall())
+ return false;
+ if (any_of(*BB, [](Instruction &I) {
+ auto *CB = dyn_cast<CallBase>(&I);
+ return CB && (CB->cannotDuplicate() || CB->isConvergent());
+ }))
+ return false;
+
+ SmallPtrSet<const Value *, 8> Bases;
+ for (const auto &P : RTChecks.BasePairs) {
+ Bases.insert(P.first);
+ Bases.insert(P.second);
+ }
+ // Every base object must be available in the (PHI-only) header where the
+ // guard branch is emitted.
+ if (any_of(Bases, [&](const Value *Base) {
+ const auto *I = dyn_cast<Instruction>(Base);
+ return I && (I->getParent() == BB || !DT->dominates(I, BB));
+ }))
+ return false;
+
+ // Compute the accessed address range [Low, High) for every involved base.
+ RTChecks.Bounds.clear();
+ SmallMapVector<Value *, std::pair<const SCEV *, const SCEV *>, 4> OffBounds;
+ unsigned NumMemInsts = 0;
+ for (Instruction &I : *BB) {
+ if (!I.mayReadOrWriteMemory())
+ continue;
+ // The dependency scan truncates beyond MaxMemDepDistance. Only version
+ // blocks small enough that the scan is never truncated, so every
+ // conflicting base pair is guaranteed to be recorded.
+ if (++NumMemInsts > MaxMemDepDistance)
+ return false;
+ Value *Ptr = getLoadStorePointerOperand(&I);
+ // A non-simple memory access (e.g. a call) keeps its dependencies, so its
+ // relative order with kept-dependency partners is preserved inside the
+ // duplicated body and it does not need a bound.
+ if (!Ptr)
+ continue;
+ Value *Base = const_cast<Value *>(getUnderlyingObject(Ptr));
+ if (!Bases.contains(Base))
+ continue;
+ TypeSize TS = DL->getTypeStoreSize(getLoadStoreType(&I));
+ if (TS.isScalable())
+ return false;
+ Type *IntTy = DL->getIntPtrType(Base->getType());
+ const SCEV *PtrSC = SE->getPtrToIntExpr(SE->getSCEV(Ptr), IntTy);
+ const SCEV *BaseSC = SE->getPtrToIntExpr(SE->getSCEV(Base), IntTy);
+ if (isa<SCEVCouldNotCompute>(PtrSC) || isa<SCEVCouldNotCompute>(BaseSC))
+ return false;
+ // Byte offset of this access from its base. Require a constant so the bound
+ // stays base-plus-constant, and non-negative so the unsigned [Low, High)
+ // range below actually covers the access.
+ const auto *Off = dyn_cast<SCEVConstant>(SE->getMinusSCEV(PtrSC, BaseSC));
+ if (!Off || Off->getAPInt().isNegative())
+ return false;
+ const SCEV *EndOff =
+ SE->getAddExpr(Off, SE->getConstant(IntTy, TS.getFixedValue()));
+ auto [It, Inserted] = OffBounds.try_emplace(Base, Off, EndOff);
+ if (!Inserted) {
+ // Constant operands, so these fold to a constant min/max offset rather
+ // than emitting a runtime umin/umax.
+ It->second.first = SE->getUMinExpr(It->second.first, Off);
+ It->second.second = SE->getUMaxExpr(It->second.second, EndOff);
+ }
+ }
+ // Materialize the absolute [Low, High) = base + [minOff, maxEndOff). The
+ // offsets are constants, so each bound is a base-plus-constant expression
+ // that expands to a single add off the base address (no runtime umin/umax).
+ for (const auto &[Base, Off] : OffBounds) {
+ Type *IntTy = DL->getIntPtrType(Base->getType());
+ const SCEV *BaseSC = SE->getPtrToIntExpr(SE->getSCEV(Base), IntTy);
+ RTChecks.Bounds.try_emplace(Base, SE->getAddExpr(BaseSC, Off.first),
+ SE->getAddExpr(BaseSC, Off.second));
+ }
+ // Every involved base must contribute at least one bounded access, and its
+ // bounds must be expandable at the guard (so the checks only reference values
+ // available in the header).
+ SCEVExpander Exp(*SE, "slp.rtcheck");
+ // The guard is emitted after the header PHIs, so validate expandability at
+ // the first non-PHI: only values available in the header (PHIs, arguments,
+ // values defined before the block) dominate that point.
+ Instruction *GuardPt = &*BB->getFirstNonPHIIt();
+ if (any_of(Bases, [&](const Value *Base) {
+ auto *It = RTChecks.Bounds.find(Base);
+ return ...
[truncated]
``````````
</details>
https://github.com/llvm/llvm-project/pull/203631
More information about the llvm-commits
mailing list