[llvm] Revert "[SLP] Support memory runtime alias checks" (PR #209173)
via llvm-commits
llvm-commits at lists.llvm.org
Mon Jul 13 06:12:21 PDT 2026
llvmorg-github-actions[bot] wrote:
<!--LLVM PR SUMMARY COMMENT-->
@llvm/pr-subscribers-llvm-transforms
Author: Alexey Bataev (alexey-bataev)
<details>
<summary>Changes</summary>
This reverts commit d4c6c8623e9bf9aa75ad049f55423223228116b3 to address
postcommit review https://github.com/llvm/llvm-project/pull/203631?email_source=notifications&email_token=ABI45DXYNFIHECEWI5IGYJT5ETMNLA5CNFSNUABKM5UWIORPF5TWS5BNNB2WEL2QOVWGYUTFOF2WK43UKJSXM2LFO4XTINRYGQ4TAMRWGAZ2M4TFMFZW63VMON2GC5DFL5RWQYLOM5S2KZLWMVXHJLDGN5XXIZLSL5RWY2LDNM#pullrequestreview-4684902603
---
Patch is 73.56 KiB, truncated to 20.00 KiB below, full version: https://github.com/llvm/llvm-project/pull/209173.diff
4 Files Affected:
- (modified) llvm/include/llvm/Transforms/Vectorize/SLPVectorizer.h (-13)
- (modified) llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp (+17-825)
- (modified) llvm/test/Transforms/SLPVectorizer/AArch64/loadi8.ll (+10-41)
- (modified) llvm/test/Transforms/SLPVectorizer/X86/runtime-alias-checks.ll (-315)
``````````diff
diff --git a/llvm/include/llvm/Transforms/Vectorize/SLPVectorizer.h b/llvm/include/llvm/Transforms/Vectorize/SLPVectorizer.h
index 538d62626b37b..b3f5718ddfda4 100644
--- a/llvm/include/llvm/Transforms/Vectorize/SLPVectorizer.h
+++ b/llvm/include/llvm/Transforms/Vectorize/SLPVectorizer.h
@@ -172,25 +172,12 @@ struct SLPVectorizerPass : public OptionalPassInfoMixin<SLPVectorizerPass> {
unsigned Idx, unsigned MinVF,
unsigned &Size);
- /// Single vectorization attempt for a store chain. \p vectorizeStoreChain
- /// wraps this to retry once with runtime alias checks enabled when the
- /// normal attempt is blocked only by runtime-checkable may-alias
- /// dependencies.
- std::optional<bool> vectorizeStoreChainImpl(ArrayRef<Value *> Chain,
- slpvectorizer::BoUpSLP &R,
- unsigned Idx, unsigned MinVF,
- unsigned &Size);
-
bool vectorizeStores(
ArrayRef<StoreInst *> Stores, slpvectorizer::BoUpSLP &R,
DenseSet<std::tuple<Value *, Value *, Value *, Value *, unsigned>>
&Visited,
bool AllowMaskedStores = true);
- /// Set by runImpl() when runtime alias check versioning changed the CFG, so
- /// run() can drop CFG-analysis preservation only when necessary.
- bool CFGChanged = false;
-
/// The store instructions in a basic block organized by base pointer.
StoreListMap Stores;
diff --git a/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp b/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp
index a70aa1d35e02f..7e22ba3bd149c 100644
--- a/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp
+++ b/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp
@@ -38,7 +38,6 @@
#include "llvm/Analysis/CodeMetrics.h"
#include "llvm/Analysis/ConstantFolding.h"
#include "llvm/Analysis/DemandedBits.h"
-#include "llvm/Analysis/DomTreeUpdater.h"
#include "llvm/Analysis/GlobalsModRef.h"
#include "llvm/Analysis/IVDescriptors.h"
#include "llvm/Analysis/Loads.h"
@@ -53,7 +52,6 @@
#include "llvm/Analysis/VectorUtils.h"
#include "llvm/IR/Attributes.h"
#include "llvm/IR/BasicBlock.h"
-#include "llvm/IR/CFG.h"
#include "llvm/IR/Constant.h"
#include "llvm/IR/Constants.h"
#include "llvm/IR/DataLayout.h"
@@ -291,28 +289,6 @@ static cl::opt<bool> PerLaneGatherScale(
cl::desc("Use per-lane execution scale for gather/buildvector tree "
"entries to model LICM-hoistable buildvector sequences."));
-/// Enable versioning of a basic block with runtime alias checks.
-static cl::opt<bool> SLPEnableRuntimeAliasChecks(
- "slp-vectorize-with-runtime-alias-checks", cl::init(true), cl::Hidden,
- cl::desc("Allow SLP to version a block with runtime alias checks to "
- "vectorize trees blocked by may-alias memory dependencies."));
-
-/// Maximum number of runtime alias checks (one per pair of base objects) that
-/// may guard a single versioned region.
-static cl::opt<unsigned> SLPMaxRuntimeAliasChecks(
- "slp-max-runtime-alias-checks", cl::init(8), cl::Hidden,
- cl::desc("The maximum number of runtime alias checks generated to guard a "
- "single SLP-vectorized region."));
-
-/// The runtime checks and the guard branch execute on both the vector and the
-/// scalar fallback path, so they add overhead to the scalar code.
-static cl::opt<unsigned> SLPRuntimeAliasChecksMaxScalarCostPercent(
- "slp-runtime-alias-checks-max-scalar-cost-percent", cl::init(25),
- cl::Hidden,
- cl::desc("Maximum SLP runtime alias check cost, as a percentage of the "
- "guarded scalar region cost, before versioning is rejected to "
- "avoid pessimizing the scalar fallback path."));
-
// Limit the number of alias checks. The limit is chosen so that
// it has no negative effect on the llvm benchmarks.
static const unsigned AliasedCheckLimit = 10;
@@ -2103,93 +2079,7 @@ class slpvectorizer::BoUpSLP {
/// Construct a vectorizable tree that starts at \p Roots.
void buildTree(ArrayRef<Value *> Roots);
- /// Returns true if the last buildTree() observed a may-alias memory
- /// dependency between two distinct, range-checkable base objects, i.e. a
- /// dependency that could be turned into a runtime alias check.
- bool hasRuntimeCheckableBlockers() const {
- return HasRuntimeCheckableBlockers;
- }
-
- /// Records whether a may-alias dependency between distinct, range-checkable
- /// base objects has been observed, so the caller can decide to retry with
- /// runtime alias checks enabled.
- void setHasRuntimeCheckableBlockers(bool V) {
- HasRuntimeCheckableBlockers = V;
- }
-
- /// Returns true if the last buildTree() kept a may-alias memory dependency
- /// that is not runtime-checkable (call or a non-simple mem access). Such a
- /// dependency cannot be dropped, so a runtime-checks retry cannot unblock the
- /// region and would be pure overhead.
- bool hasNonCheckableMemBlocker() const { return HasNonCheckableMemBlocker; }
-
- /// Records that a non-runtime-checkable may-alias dependency was kept.
- void setHasNonCheckableMemBlocker(bool V) { HasNonCheckableMemBlocker = V; }
-
- /// Returns true if the current vectorization attempt may drop
- /// runtime-checkable may-alias dependencies and guard the region with
- /// runtime alias checks.
- bool isTryingRuntimeAliasChecks() const { return TryRuntimeAliasChecks; }
-
- /// Enables or disables dropping runtime-checkable may-alias dependencies in
- /// favor of runtime alias checks for the current vectorization attempt.
- void setTryRuntimeAliasChecks(bool V) { TryRuntimeAliasChecks = V; }
-
- /// Resets the runtime alias check data.
- void resetRuntimeAliasCheckState() {
- HasRuntimeCheckableBlockers = false;
- HasNonCheckableMemBlocker = false;
- RTChecksFinalized = false;
- RTChecks.clear();
- RTOrigBodyOrder.clear();
- }
-
- /// Snapshots RTChecks.BB's body (non-PHI, non-terminator) into
- /// RTOrigBodyOrder in program order, for the scalar fallback.
- void captureRuntimeCheckBodySnapshot();
-
- /// Returns true if \p BB satisfies the block-level preconditions for runtime
- /// alias check versioning (straight-line, outside any loop, duplicable, not a
- /// scalar fallback, function not optimized for size). These checks do not
- /// depend on the collected checks, so they can gate the (expensive)
- /// optimistic retry before any tree is rebuilt.
- bool canVersionBlockForRuntimeChecks(BasicBlock *BB) const;
-
- /// Returns true if the runtime alias checks can be safely emitted to guard
- /// the vectorized region.
- bool canVersionForRuntimeChecks();
-
- /// Returns true if \p BB is a scalar fallback block created by runtime alias
- /// check versioning.
- bool isScalarFallbackBlock(BasicBlock *BB) const {
- return ScalarFallbackBlocks.contains(BB);
- }
-
- /// Returns true if an optimistic runtime-checks versioning attempt already
- /// failed for \p BB, so further retries in the same block can be skipped.
- bool runtimeChecksFailedForBlock(BasicBlock *BB) const {
- return FailedRuntimeChecksBlocks.contains(BB);
- }
-
- /// Records that an optimistic runtime-checks versioning attempt failed for
- /// \p BB.
- void markRuntimeChecksFailedForBlock(BasicBlock *BB) {
- FailedRuntimeChecksBlocks.insert(BB);
- }
-
- /// Returns the modeled cost of the runtime alias checks collected during the
- /// last (optimistic) buildTree().
- InstructionCost getRuntimeChecksCost() const;
-
- /// Returns true if the last (optimistic) buildTree() collected any runtime
- /// alias checks that must guard the vectorized region.
- bool hasRuntimeAliasChecks() const { return !RTChecks.BasePairs.empty(); }
-
- /// Returns true if vectorization changed the CFG (i.e. a block was versioned
- /// with runtime alias checks). When true, CFG analyses must not be preserved.
- bool isCFGChanged() const { return CFGChanged; }
-
- /// Returns the scalars of the root node.
+ /// Return the scalars of the root node.
ArrayRef<Value *> getRootNodeScalars() const {
assert(!VectorizableTree.empty() && "No graph to get the first node from");
return VectorizableTree.front()->Scalars;
@@ -2296,10 +2186,6 @@ class slpvectorizer::BoUpSLP {
ExternalUses.clear();
ExternalUsesAsOriginalScalar.clear();
ExternalUsesWithNonUsers.clear();
- RTChecks.clear();
- HasRuntimeCheckableBlockers = false;
- HasNonCheckableMemBlocker = false;
- RTChecksFinalized = false;
for (auto &Iter : BlocksSchedules) {
BlockScheduling *BS = Iter.second.get();
BS->clear();
@@ -5048,89 +4934,6 @@ class slpvectorizer::BoUpSLP {
return Aliased;
}
- /// Returns true if the may-alias dependency between simple load/store
- /// instructions \p Inst1 and \p Inst2 could be disambiguated by a runtime
- /// alias check.
- bool isRuntimeCheckableAliasPair(Instruction *Inst1, Instruction *Inst2);
-
- /// Records the (distinct base object) pair behind the may-alias dependency
- /// of \p Inst1 and \p Inst2 as a runtime alias check guarding the region in
- /// block \p BB. Returns true if the pair was recorded.
- bool recordRuntimeAliasCheck(BasicBlock *BB, Instruction *Inst1,
- Instruction *Inst2);
-
- /// Emits the collected runtime alias checks and versions the affected block,
- /// duplicating its body into a scalar fallback guarded by the checks.
- void versionBlocksForRuntimeChecks();
-
- /// Builds the i1 value that is true when any pair of checked base objects
- /// overlaps at runtime. The base address bounds are materialized from their
- /// SCEVs with \p Exp.
- Value *emitRuntimeAliasCheck(IRBuilderBase &Builder, SCEVExpander &Exp);
-
- /// Data to model and emit the runtime alias checks.
- struct RuntimeAliasCheckInfo {
- /// The block whose body is guarded by the checks. Exactly one block is
- /// supported per attempt.
- BasicBlock *BB = nullptr;
- /// Pairs of base objects that must be proven disjoint.
- SmallSetVector<std::pair<const Value *, const Value *>, 4> BasePairs;
- /// Accessed address range [Low, High) for each involved base object.
- SmallMapVector<const Value *, std::pair<const SCEV *, const SCEV *>, 4>
- Bounds;
-
- void clear() {
- BB = nullptr;
- BasePairs.clear();
- Bounds.clear();
- }
- };
-
- /// When true, scheduling drops may-alias memory dependencies between
- /// distinct, range-checkable base objects and records them as runtime alias
- /// checks instead.
- bool TryRuntimeAliasChecks = false;
-
- /// Runtime alias checks collected during the last optimistic buildTree().
- RuntimeAliasCheckInfo RTChecks;
-
- /// Base-object pairs already proven disjoint by the block's runtime alias
- /// check.
- SmallDenseMap<BasicBlock *,
- SmallDenseSet<std::pair<const Value *, const Value *>, 4>, 2>
- VersionedBlockCheckedPairs;
-
- /// Scalar fallback blocks.
- SmallPtrSet<BasicBlock *, 4> ScalarFallbackBlocks;
-
- /// Blocks for which a runtime-checks versioning attempt was made
- /// and did not produce a profitable versioning.
- SmallPtrSet<BasicBlock *, 8> FailedRuntimeChecksBlocks;
-
- /// Returns true if a may-alias dependency between the simple load/store
- /// instructions \p Inst1 and \p Inst2 in block \p BB is already covered by a
- /// runtime alias check emitted for \p BB by a previous versioning.
- bool isCoveredByExistingVersionCheck(BasicBlock *BB, Instruction *Inst1,
- Instruction *Inst2) const;
-
- /// True, if a may-alias dependency between distinct, range-checkable base
- /// objects is observed (whether or not it was dropped).
- bool HasRuntimeCheckableBlockers = false;
-
- /// True, if a kept may-alias dependency is not runtime-checkable (call or a
- /// non-simple memaccess).
- bool HasNonCheckableMemBlocker = false;
-
- /// Runtime checks are validated and bounded the collected checks.
- bool RTChecksFinalized = false;
-
- /// Set when a block was versioned with runtime alias checks, which changes
- /// the CFG. Used to drop CFG-analysis preservation for the run.
- bool CFGChanged = false;
-
- /// Guarded block body (non-PHI, non-terminator) in original source order.
- SmallVector<Instruction *> RTOrigBodyOrder;
-
using AliasCacheKey = std::pair<Instruction *, Instruction *>;
/// Cache for alias results.
@@ -5714,7 +5517,6 @@ class slpvectorizer::BoUpSLP {
ScheduleCopyableDataMapByUsers.clear();
ReadyInsts.clear();
RecalcCopyableOperandDeps.clear();
- IgnoredMemDeps.clear();
ScheduleStart = nullptr;
ScheduleEnd = nullptr;
FirstLoadStoreInRegion = nullptr;
@@ -6589,10 +6391,6 @@ class slpvectorizer::BoUpSLP {
/// recomputing the same operand more than once.
SmallSetVector<ScheduleData *, 8> RecalcCopyableOperandDeps;
- /// Ordered pairs (Src, Dst) of memory instructions whose may-alias
- /// dependency has been dropped in favor of a runtime alias check.
- SmallDenseSet<std::pair<Instruction *, Instruction *>, 8> IgnoredMemDeps;
-
/// The ID of the scheduling region. For a new vectorization iteration this
/// is incremented which "removes" all ScheduleData from the region.
/// Make sure that the initial SchedulingRegionID is greater than the
@@ -24809,457 +24607,6 @@ Value *BoUpSLP::vectorizeTree(TreeEntry *E) {
return nullptr;
}
-bool BoUpSLP::isRuntimeCheckableAliasPair(Instruction *Inst1,
- Instruction *Inst2) {
- // Only simple (non-volatile, non-atomic) accesses may be reordered once
- // aliasing is ruled out at runtime; volatile/atomic ordering must be kept.
- if (!isSimple(Inst1) || !isSimple(Inst2))
- return false;
- Value *Ptr1 = getLoadStorePointerOperand(Inst1);
- Value *Ptr2 = getLoadStorePointerOperand(Inst2);
- // Only simple load/store accesses have a single pointer operand whose
- // accessed range can be bounded and compared at runtime.
- if (!Ptr1 || !Ptr2)
- return false;
- if (Ptr1->getType()->getPointerAddressSpace() !=
- Ptr2->getType()->getPointerAddressSpace())
- return false;
- const Value *Base1 = getUnderlyingObject(Ptr1);
- const Value *Base2 = getUnderlyingObject(Ptr2);
- // A runtime range check is only meaningful between two distinct, identifiable
- // objects: two accesses to the same base differ by a compile-time offset, so
- // they are resolved statically and a base-range check would fold to a
- // constant predicate rather than a useful runtime guard.
- if (Base1 == Base2 || isa<UndefValue>(Base1) || isa<UndefValue>(Base2))
- return false;
- return true;
-}
-
-bool BoUpSLP::isCoveredByExistingVersionCheck(BasicBlock *BB,
- Instruction *Inst1,
- Instruction *Inst2) const {
- const auto It = VersionedBlockCheckedPairs.find(BB);
- if (It == VersionedBlockCheckedPairs.end())
- return false;
- Value *Ptr1 = getLoadStorePointerOperand(Inst1);
- Value *Ptr2 = getLoadStorePointerOperand(Inst2);
- if (!Ptr1 || !Ptr2)
- return false;
- const Value *Base1 = getUnderlyingObject(Ptr1);
- const Value *Base2 = getUnderlyingObject(Ptr2);
- if (Base1 == Base2)
- return false;
- if (Base2 < Base1)
- std::swap(Base1, Base2);
- return It->second.contains({Base1, Base2});
-}
-
-/// Returns true if \p BB's body already contains vector instructions, e.g.
-/// from an earlier SLP vectorization in the same pass.
-static bool blockBodyHasVectorInstructions(BasicBlock *BB) {
- for (Instruction &I : *BB) {
- if (isa<PHINode>(&I) || I.isTerminator())
- continue;
- // A vector-producing instruction (vector load, binop, shuffle, etc.) has a
- // vector result type.
- if (getValueType(&I)->isVectorTy())
- return true;
- }
- return false;
-}
-
-void BoUpSLP::captureRuntimeCheckBodySnapshot() {
- RTOrigBodyOrder.clear();
- if (!TryRuntimeAliasChecks || !RTChecks.BB)
- return;
- BasicBlock *BB = RTChecks.BB;
- for (Instruction &I : *BB)
- if (!isa<PHINode>(&I) && !I.isTerminator())
- RTOrigBodyOrder.push_back(&I);
-}
-
-bool BoUpSLP::recordRuntimeAliasCheck(BasicBlock *BB, Instruction *Inst1,
- Instruction *Inst2) {
- Value *Ptr1 = getLoadStorePointerOperand(Inst1);
- Value *Ptr2 = getLoadStorePointerOperand(Inst2);
- if (!Ptr1 || !Ptr2)
- return false;
- const Value *Base1 = getUnderlyingObject(Ptr1);
- const Value *Base2 = getUnderlyingObject(Ptr2);
- if (Base1 == Base2)
- return false;
- // Only a single block can be versioned per attempt.
- if (RTChecks.BB && RTChecks.BB != BB)
- return false;
- // Normalize the pair order so duplicate checks collapse.
- if (Base2 < Base1)
- std::swap(Base1, Base2);
- auto Pair = std::make_pair(Base1, Base2);
- // After the checks have been validated and bounded, do not introduce new
- // pairs.
- if (RTChecksFinalized)
- return RTChecks.BasePairs.contains(Pair);
- RTChecks.BB = BB;
- RTChecks.BasePairs.insert(Pair);
- return true;
-}
-
-bool BoUpSLP::canVersionBlockForRuntimeChecks(BasicBlock *BB) const {
- assert(BB && "Expected a block to version for runtime checks.");
- if (!BB->getTerminator())
- return false;
- // Versioning duplicates the block body, increasing code size on the guarded
- // path; do not version when the function is optimized for size (-Os/-Oz).
- if (F->hasOptSize())
- return false;
- if (!DT || !LI)
- return false;
- if (!DT->isReachableFromEntry(BB))
- return false;
- // Versioning duplicates the block body; only straight-line code outside any
- // loop is handled for now, to avoid LoopInfo and region updates.
- if (LI->getLoopFor(BB))
- return false;
- // Never version a scalar fallback block: it is the safe, original-order copy
- // taken when aliasing is detected and must stay scalar.
- if (ScalarFallbackBlocks.contains(BB))
- return false;
- // The scalar fallback must be a faithful copy of the original scalar body.
- // Reject blocks that were already partially vectorized earlier in this pass.
- if (blockBodyHasVectorInstructions(BB))
- return false;
- // Versioning duplicates the original block body into a scalar fallback.
- // Calls that cannot be duplicated or whose semantics depend on the call being
- // immediately followed by a return cannot be cloned into the diamond.
- if (BB->getTerminatingMustTailCall())
- return false;
- if (any_of(*BB, [](Instruction &I) {
- auto *CB = dyn_cast<CallBase>(&I);
- return CB && (CB->cannotDuplicate() || CB->isConvergent());
- }))
- return false;
- return true;
-}
-
-bool BoUpSLP::canVersionForRuntimeChecks() {
- if (RTChecks.BasePairs.empty() || !RTChecks.BB)
- return false;
- if (RTChecks.BasePairs.size() > SLPMaxRuntimeAliasChecks)
- return false;
- BasicBlock *BB = RTChecks.BB;
- // Block-level preconditions (loop/optsize/duplicability/...) are the same
- // ones used to gate the optimistic retry, so reuse them here.
- if (!canVersionBlockForRuntimeChecks(BB))
- return false;
-
- // The scalar fallback is a clone of the body. Operands defined inside the
- // body are remapped to their clones, but operands defined outside the body
- // (header PHIs, dominating definitions) are reused as-is by the clone. If
- // such an outside operand is itself vectorized by this tree, vectorizeTree()
- // will delete its scalar, leaving the clone with a dangling, out-of-tree use.
- // Versioning cannot model that, so bail out.
- for (Instruction &I : *BB) {
- if (isa<PHINode>(&I) || I.isTerminator())
- continue;
- for (Value *Op : I.operands()) {
- auto *OpI = dyn_cast<Instruction>(Op);
- if (!OpI)
- continue;
- bool DefinedInBody =
- OpI->getParent() == BB && !isa<PHINode>(OpI) && !OpI->isTerminator();
- if (!DefinedInBody && isVectorized(OpI))
- return false;
- }
- }
-
- SmallPtrSet<const Value *, 8> Bases;
- for (const auto &P : RTChecks.BasePairs) {
- Bases.insert(P.first);
- Bases.insert(P.second);
- }
- // Every base object must be available in the (PHI-only) header where the
- // guard branch is emitted.
- if (any_...
[truncated]
``````````
</details>
https://github.com/llvm/llvm-project/pull/209173
More information about the llvm-commits
mailing list