[llvm] [SLP] Support memory runtime alias checks (PR #203631)
Alexey Bataev via llvm-commits
llvm-commits at lists.llvm.org
Sun Jun 14 05:10:31 PDT 2026
https://github.com/alexey-bataev updated https://github.com/llvm/llvm-project/pull/203631
>From 3c846b25afe9c679ebc2d2f276a93ef0d609aea3 Mon Sep 17 00:00:00 2001
From: Alexey Bataev <a.bataev at outlook.com>
Date: Fri, 12 Jun 2026 13:50:40 -0700
Subject: [PATCH] =?UTF-8?q?[=F0=9D=98=80=F0=9D=97=BD=F0=9D=97=BF]=20initia?=
=?UTF-8?q?l=20version?=
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit
Created using spr 1.3.7
---
.../llvm/Transforms/Vectorize/SLPVectorizer.h | 13 +
.../Transforms/Vectorize/SLPVectorizer.cpp | 714 +++++++++++++++++-
.../SLPVectorizer/AArch64/loadi8.ll | 51 +-
.../SLPVectorizer/X86/runtime-alias-checks.ll | 90 +++
4 files changed, 840 insertions(+), 28 deletions(-)
diff --git a/llvm/include/llvm/Transforms/Vectorize/SLPVectorizer.h b/llvm/include/llvm/Transforms/Vectorize/SLPVectorizer.h
index af010994107a5..bb8db7272cb63 100644
--- a/llvm/include/llvm/Transforms/Vectorize/SLPVectorizer.h
+++ b/llvm/include/llvm/Transforms/Vectorize/SLPVectorizer.h
@@ -171,11 +171,24 @@ struct SLPVectorizerPass : public OptionalPassInfoMixin<SLPVectorizerPass> {
unsigned Idx, unsigned MinVF,
unsigned &Size);
+ /// Single vectorization attempt for a store chain. \p vectorizeStoreChain
+ /// wraps this to retry once with runtime alias checks enabled when the
+ /// normal attempt is blocked only by runtime-checkable may-alias
+ /// dependencies.
+ std::optional<bool> vectorizeStoreChainImpl(ArrayRef<Value *> Chain,
+ slpvectorizer::BoUpSLP &R,
+ unsigned Idx, unsigned MinVF,
+ unsigned &Size);
+
bool vectorizeStores(
ArrayRef<StoreInst *> Stores, slpvectorizer::BoUpSLP &R,
DenseSet<std::tuple<Value *, Value *, Value *, Value *, unsigned>>
&Visited);
+ /// Set by runImpl() when runtime alias check versioning changed the CFG, so
+ /// run() can drop CFG-analysis preservation only when necessary.
+ bool CFGChanged = false;
+
/// The store instructions in a basic block organized by base pointer.
StoreListMap Stores;
diff --git a/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp b/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp
index 3d5b0ebb99f4a..f34ef76557751 100644
--- a/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp
+++ b/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp
@@ -37,6 +37,7 @@
#include "llvm/Analysis/CodeMetrics.h"
#include "llvm/Analysis/ConstantFolding.h"
#include "llvm/Analysis/DemandedBits.h"
+#include "llvm/Analysis/DomTreeUpdater.h"
#include "llvm/Analysis/GlobalsModRef.h"
#include "llvm/Analysis/IVDescriptors.h"
#include "llvm/Analysis/Loads.h"
@@ -51,6 +52,7 @@
#include "llvm/Analysis/VectorUtils.h"
#include "llvm/IR/Attributes.h"
#include "llvm/IR/BasicBlock.h"
+#include "llvm/IR/CFG.h"
#include "llvm/IR/Constant.h"
#include "llvm/IR/Constants.h"
#include "llvm/IR/DataLayout.h"
@@ -278,6 +280,28 @@ static cl::opt<bool> PerLaneGatherScale(
cl::desc("Use per-lane execution scale for gather/buildvector tree "
"entries to model LICM-hoistable buildvector sequences."));
+/// Enable versioning of a basic block with runtime alias checks.
+static cl::opt<bool> SLPEnableRuntimeAliasChecks(
+ "slp-vectorize-with-runtime-alias-checks", cl::init(true), cl::Hidden,
+ cl::desc("Allow SLP to version a block with runtime alias checks to "
+ "vectorize trees blocked by may-alias memory dependencies."));
+
+/// Maximum number of runtime alias checks (one per pair of base objects) that
+/// may guard a single versioned region.
+static cl::opt<unsigned> SLPMaxRuntimeAliasChecks(
+ "slp-max-runtime-alias-checks", cl::init(8), cl::Hidden,
+ cl::desc("The maximum number of runtime alias checks generated to guard a "
+ "single SLP-vectorized region."));
+
+/// The runtime checks and the guard branch execute on both the vector and the
+/// scalar fallback path, so they add overhead to the scalar code.
+static cl::opt<unsigned> SLPRuntimeAliasChecksMaxScalarCostPercent(
+ "slp-runtime-alias-checks-max-scalar-cost-percent", cl::init(25),
+ cl::Hidden,
+ cl::desc("Maximum SLP runtime alias check cost, as a percentage of the "
+ "guarded scalar region cost, before versioning is rejected to "
+ "avoid pessimizing the scalar fallback path."));
+
// Limit the number of alias checks. The limit is chosen so that
// it has no negative effect on the llvm benchmarks.
static const unsigned AliasedCheckLimit = 10;
@@ -2204,7 +2228,64 @@ class slpvectorizer::BoUpSLP {
/// Construct a vectorizable tree that starts at \p Roots.
void buildTree(ArrayRef<Value *> Roots);
- /// Return the scalars of the root node.
+ /// Returns true if the last buildTree() observed a may-alias memory
+ /// dependency between two distinct, range-checkable base objects, i.e. a
+ /// dependency that could be turned into a runtime alias check.
+ bool hasRuntimeCheckableBlockers() const {
+ return HasRuntimeCheckableBlockers;
+ }
+
+ /// Records whether a may-alias dependency between distinct, range-checkable
+ /// base objects has been observed, so the caller can decide to retry with
+ /// runtime alias checks enabled.
+ void setHasRuntimeCheckableBlockers(bool V) {
+ HasRuntimeCheckableBlockers = V;
+ }
+
+ /// Returns true if the current vectorization attempt may drop
+ /// runtime-checkable may-alias dependencies and guard the region with
+ /// runtime alias checks.
+ bool isTryingRuntimeAliasChecks() const { return TryRuntimeAliasChecks; }
+
+ /// Enables or disables dropping runtime-checkable may-alias dependencies in
+ /// favor of runtime alias checks for the current vectorization attempt.
+ void setTryRuntimeAliasChecks(bool V) { TryRuntimeAliasChecks = V; }
+
+ /// Resets the runtime alias check data.
+ void resetRuntimeAliasCheckState() {
+ HasRuntimeCheckableBlockers = false;
+ RTChecksFinalized = false;
+ RTChecks.clear();
+ RTOrigBodyOrder.clear();
+ }
+
+ /// Snapshots RTChecks.BB's body (non-PHI, non-terminator) into
+ /// RTOrigBodyOrder in program order, for the scalar fallback.
+ void captureRuntimeCheckBodySnapshot();
+
+ /// Returns true if the runtime alias checks can be safely emitted to guard
+ /// the vectorized region.
+ bool canVersionForRuntimeChecks();
+
+ /// Returns true if \p BB is a scalar fallback block created by runtime alias
+ /// check versioning.
+ bool isScalarFallbackBlock(BasicBlock *BB) const {
+ return ScalarFallbackBlocks.contains(BB);
+ }
+
+ /// Returns the modeled cost of the runtime alias checks collected during the
+ /// last (optimistic) buildTree().
+ InstructionCost getRuntimeChecksCost() const;
+
+ /// Returns true if the last (optimistic) buildTree() collected any runtime
+ /// alias checks that must guard the vectorized region.
+ bool hasRuntimeAliasChecks() const { return !RTChecks.BasePairs.empty(); }
+
+ /// Returns true if vectorization changed the CFG (i.e. a block was versioned
+ /// with runtime alias checks). When true, CFG analyses must not be preserved.
+ bool isCFGChanged() const { return CFGChanged; }
+
+ /// Returns the scalars of the root node.
ArrayRef<Value *> getRootNodeScalars() const {
assert(!VectorizableTree.empty() && "No graph to get the first node from");
return VectorizableTree.front()->Scalars;
@@ -2305,6 +2386,9 @@ class slpvectorizer::BoUpSLP {
ExternalUses.clear();
ExternalUsesAsOriginalScalar.clear();
ExternalUsesWithNonUsers.clear();
+ RTChecks.clear();
+ HasRuntimeCheckableBlockers = false;
+ RTChecksFinalized = false;
for (auto &Iter : BlocksSchedules) {
BlockScheduling *BS = Iter.second.get();
BS->clear();
@@ -5035,6 +5119,81 @@ class slpvectorizer::BoUpSLP {
return Aliased;
}
+ /// Returns true if the may-alias dependency between simple load/store
+ /// instructions \p Inst1 and \p Inst2 could be disambiguated by a runtime
+ /// alias check.
+ bool isRuntimeCheckableAliasPair(Instruction *Inst1, Instruction *Inst2);
+
+ /// Records the (distinct base object) pair behind the may-alias dependency
+ /// of \p Inst1 and \p Inst2 as a runtime alias check guarding the region in
+ /// block \p BB. Returns true if the pair was recorded.
+ bool recordRuntimeAliasCheck(BasicBlock *BB, Instruction *Inst1,
+ Instruction *Inst2);
+
+ /// Emits the collected runtime alias checks and versions the affected block,
+ /// duplicating its body into a scalar fallback guarded by the checks.
+ void versionBlocksForRuntimeChecks();
+
+ /// Builds the i1 value that is true when any pair of checked base objects
+ /// overlaps at runtime. The base address bounds are materialized from their
+ /// SCEVs with \p Exp.
+ Value *emitRuntimeAliasCheck(IRBuilderBase &Builder, SCEVExpander &Exp);
+
+ /// Data to model and emit the runtime alias checks.
+ struct RuntimeAliasCheckInfo {
+ /// The block whose body is guarded by the checks. Exactly one block is
+ /// supported per attempt.
+ BasicBlock *BB = nullptr;
+ /// Pairs of base objects that must be proven disjoint.
+ SmallSetVector<std::pair<const Value *, const Value *>, 4> BasePairs;
+ /// Accessed address range [Low, High) for each involved base object.
+ SmallMapVector<const Value *, std::pair<const SCEV *, const SCEV *>, 4>
+ Bounds;
+
+ void clear() {
+ BB = nullptr;
+ BasePairs.clear();
+ Bounds.clear();
+ }
+ };
+
+ /// When true, scheduling drops may-alias memory dependencies between
+ /// distinct, range-checkable base objects and records them as runtime alias
+ /// checks instead.
+ bool TryRuntimeAliasChecks = false;
+
+ /// Runtime alias checks collected during the last optimistic buildTree().
+ RuntimeAliasCheckInfo RTChecks;
+
+ /// Base-object pairs already proven disjoint by the block's runtime alias
+ /// check.
+ SmallDenseMap<BasicBlock *,
+ SmallDenseSet<std::pair<const Value *, const Value *>, 4>, 2>
+ VersionedBlockCheckedPairs;
+
+ /// Scalar fallback blocks.
+ SmallPtrSet<BasicBlock *, 4> ScalarFallbackBlocks;
+
+ /// Returns true if a may-alias dependency between the simple load/store
+ /// instructions \p Inst1 and \p Inst2 in block \p BB is already covered by a
+ /// runtime alias check emitted for \p BB by a previous versioning.
+ bool isCoveredByExistingVersionCheck(BasicBlock *BB, Instruction *Inst1,
+ Instruction *Inst2) const;
+
+ /// True, if a may-alias dependency between distinct, range-checkable base
+ /// objects is observed (whether or not it was dropped).
+ bool HasRuntimeCheckableBlockers = false;
+
+ /// Runtime checks are validated and bounded the collected checks.
+ bool RTChecksFinalized = false;
+
+ /// Set when a block was versioned with runtime alias checks, which changes
+ /// the CFG. Used to drop CFG-analysis preservation for the run.
+ bool CFGChanged = false;
+
+ /// Guarded block body (non-PHI, non-terminator) in original source order.
+ SmallVector<Instruction *> RTOrigBodyOrder;
+
using AliasCacheKey = std::pair<Instruction *, Instruction *>;
/// Cache for alias results.
@@ -5618,6 +5777,7 @@ class slpvectorizer::BoUpSLP {
ScheduleCopyableDataMapByUsers.clear();
ReadyInsts.clear();
RecalcCopyableOperandDeps.clear();
+ IgnoredMemDeps.clear();
ScheduleStart = nullptr;
ScheduleEnd = nullptr;
FirstLoadStoreInRegion = nullptr;
@@ -6488,6 +6648,10 @@ class slpvectorizer::BoUpSLP {
/// recomputing the same operand more than once.
SmallSetVector<ScheduleData *, 8> RecalcCopyableOperandDeps;
+ /// Ordered pairs (Src, Dst) of memory instructions whose may-alias
+ /// dependency has been dropped in favor of a runtime alias check.
+ SmallDenseSet<std::pair<Instruction *, Instruction *>, 8> IgnoredMemDeps;
+
/// The ID of the scheduling region. For a new vectorization iteration this
/// is incremented which "removes" all ScheduleData from the region.
/// Make sure that the initial SchedulingRegionID is greater than the
@@ -24388,6 +24552,417 @@ Value *BoUpSLP::vectorizeTree(TreeEntry *E) {
return nullptr;
}
+bool BoUpSLP::isRuntimeCheckableAliasPair(Instruction *Inst1,
+ Instruction *Inst2) {
+ // Only simple (non-volatile, non-atomic) accesses may be reordered once
+ // aliasing is ruled out at runtime; volatile/atomic ordering must be kept.
+ if (!isSimple(Inst1) || !isSimple(Inst2))
+ return false;
+ Value *Ptr1 = getLoadStorePointerOperand(Inst1);
+ Value *Ptr2 = getLoadStorePointerOperand(Inst2);
+ // Only simple load/store accesses have a single pointer operand whose
+ // accessed range can be bounded and compared at runtime.
+ if (!Ptr1 || !Ptr2)
+ return false;
+ if (Ptr1->getType()->getPointerAddressSpace() !=
+ Ptr2->getType()->getPointerAddressSpace())
+ return false;
+ const Value *Base1 = getUnderlyingObject(Ptr1);
+ const Value *Base2 = getUnderlyingObject(Ptr2);
+ // A range check is only meaningful between two distinct, identifiable
+ // objects.
+ if (Base1 == Base2 || isa<UndefValue>(Base1) || isa<UndefValue>(Base2))
+ return false;
+ return true;
+}
+
+bool BoUpSLP::isCoveredByExistingVersionCheck(BasicBlock *BB,
+ Instruction *Inst1,
+ Instruction *Inst2) const {
+ const auto It = VersionedBlockCheckedPairs.find(BB);
+ if (It == VersionedBlockCheckedPairs.end())
+ return false;
+ Value *Ptr1 = getLoadStorePointerOperand(Inst1);
+ Value *Ptr2 = getLoadStorePointerOperand(Inst2);
+ if (!Ptr1 || !Ptr2)
+ return false;
+ const Value *Base1 = getUnderlyingObject(Ptr1);
+ const Value *Base2 = getUnderlyingObject(Ptr2);
+ if (Base1 == Base2)
+ return false;
+ if (Base2 < Base1)
+ std::swap(Base1, Base2);
+ return It->second.contains({Base1, Base2});
+}
+
+/// Returns true if \p BB's body already contains vector instructions, e.g.
+/// from an earlier SLP vectorization in the same pass.
+static bool blockBodyHasVectorInstructions(BasicBlock *BB) {
+ for (Instruction &I : *BB) {
+ if (isa<PHINode>(&I) || I.isTerminator())
+ continue;
+ // A vector-producing instruction (vector load, binop, shuffle, etc.) has a
+ // vector result type.
+ if (getValueType(&I)->isVectorTy())
+ return true;
+ }
+ return false;
+}
+
+void BoUpSLP::captureRuntimeCheckBodySnapshot() {
+ RTOrigBodyOrder.clear();
+ if (!TryRuntimeAliasChecks || !RTChecks.BB)
+ return;
+ BasicBlock *BB = RTChecks.BB;
+ for (Instruction &I : *BB)
+ if (!isa<PHINode>(&I) && !I.isTerminator())
+ RTOrigBodyOrder.push_back(&I);
+}
+
+bool BoUpSLP::recordRuntimeAliasCheck(BasicBlock *BB, Instruction *Inst1,
+ Instruction *Inst2) {
+ Value *Ptr1 = getLoadStorePointerOperand(Inst1);
+ Value *Ptr2 = getLoadStorePointerOperand(Inst2);
+ if (!Ptr1 || !Ptr2)
+ return false;
+ const Value *Base1 = getUnderlyingObject(Ptr1);
+ const Value *Base2 = getUnderlyingObject(Ptr2);
+ if (Base1 == Base2)
+ return false;
+ // Only a single block can be versioned per attempt.
+ if (RTChecks.BB && RTChecks.BB != BB)
+ return false;
+ // Normalize the pair order so duplicate checks collapse.
+ if (Base2 < Base1)
+ std::swap(Base1, Base2);
+ auto Pair = std::make_pair(Base1, Base2);
+ // After the checks have been validated and bounded, do not introduce new
+ // pairs.
+ if (RTChecksFinalized)
+ return RTChecks.BasePairs.contains(Pair);
+ RTChecks.BB = BB;
+ RTChecks.BasePairs.insert(Pair);
+ return true;
+}
+
+bool BoUpSLP::canVersionForRuntimeChecks() {
+ if (RTChecks.BasePairs.empty() || !RTChecks.BB)
+ return false;
+ if (RTChecks.BasePairs.size() > SLPMaxRuntimeAliasChecks)
+ return false;
+ BasicBlock *BB = RTChecks.BB;
+ // Never version a scalar fallback block: it is the safe, original-order copy
+ // taken when aliasing is detected and must stay scalar.
+ if (ScalarFallbackBlocks.contains(BB))
+ return false;
+ // Versioning duplicates the block body; only straight-line code outside any
+ // loop is handled for now, to avoid LoopInfo and region updates.
+ if (!LI || LI->getLoopFor(BB))
+ return false;
+ if (!BB->getTerminator())
+ return false;
+ if (!DT)
+ return false;
+ // The scalar fallback must be a faithful copy of the original scalar body.
+ // Reject blocks that were already partially vectorized earlier in this pass.
+ if (blockBodyHasVectorInstructions(BB))
+ return false;
+
+ // Versioning duplicates the original block body into a scalar fallback.
+ // Calls that cannot be duplicated or whose semantics depend on the call being
+ // immediately followed by a return cannot be cloned into the diamond.
+ if (BB->getTerminatingMustTailCall())
+ return false;
+ if (any_of(*BB, [](Instruction &I) {
+ auto *CB = dyn_cast<CallBase>(&I);
+ return CB && (CB->cannotDuplicate() || CB->isConvergent());
+ }))
+ return false;
+
+ SmallPtrSet<const Value *, 8> Bases;
+ for (const auto &P : RTChecks.BasePairs) {
+ Bases.insert(P.first);
+ Bases.insert(P.second);
+ }
+ // Every base object must be available in the (PHI-only) header where the
+ // guard branch is emitted.
+ if (any_of(Bases, [&](const Value *Base) {
+ const auto *I = dyn_cast<Instruction>(Base);
+ return I && (I->getParent() == BB || !DT->dominates(I, BB));
+ }))
+ return false;
+
+ // Compute the accessed address range [Low, High) for every involved base.
+ RTChecks.Bounds.clear();
+ SmallMapVector<Value *, std::pair<const SCEV *, const SCEV *>, 4> OffBounds;
+ unsigned NumMemInsts = 0;
+ for (Instruction &I : *BB) {
+ if (!I.mayReadOrWriteMemory())
+ continue;
+ // The dependency scan truncates beyond MaxMemDepDistance. Only version
+ // blocks small enough that the scan is never truncated, so every
+ // conflicting base pair is guaranteed to be recorded.
+ if (++NumMemInsts > MaxMemDepDistance)
+ return false;
+ Value *Ptr = getLoadStorePointerOperand(&I);
+ // A non-simple memory access (e.g. a call) keeps its dependencies, so its
+ // relative order with kept-dependency partners is preserved inside the
+ // duplicated body and it does not need a bound.
+ if (!Ptr)
+ continue;
+ Value *Base = const_cast<Value *>(getUnderlyingObject(Ptr));
+ if (!Bases.contains(Base))
+ continue;
+ TypeSize TS = DL->getTypeStoreSize(getLoadStoreType(&I));
+ if (TS.isScalable())
+ return false;
+ Type *IntTy = DL->getIntPtrType(Base->getType());
+ const SCEV *PtrSC = SE->getPtrToIntExpr(SE->getSCEV(Ptr), IntTy);
+ const SCEV *BaseSC = SE->getPtrToIntExpr(SE->getSCEV(Base), IntTy);
+ if (isa<SCEVCouldNotCompute>(PtrSC) || isa<SCEVCouldNotCompute>(BaseSC))
+ return false;
+ // Byte offset of this access from its base. Require a constant so the bound
+ // stays base-plus-constant, and non-negative so the unsigned [Low, High)
+ // range below actually covers the access.
+ const auto *Off = dyn_cast<SCEVConstant>(SE->getMinusSCEV(PtrSC, BaseSC));
+ if (!Off || Off->getAPInt().isNegative())
+ return false;
+ const SCEV *EndOff =
+ SE->getAddExpr(Off, SE->getConstant(IntTy, TS.getFixedValue()));
+ auto [It, Inserted] = OffBounds.try_emplace(Base, Off, EndOff);
+ if (!Inserted) {
+ // Constant operands, so these fold to a constant min/max offset rather
+ // than emitting a runtime umin/umax.
+ It->second.first = SE->getUMinExpr(It->second.first, Off);
+ It->second.second = SE->getUMaxExpr(It->second.second, EndOff);
+ }
+ }
+ // Materialize the absolute [Low, High) = base + [minOff, maxEndOff). The
+ // offsets are constants, so each bound is a base-plus-constant expression
+ // that expands to a single add off the base address (no runtime umin/umax).
+ for (const auto &[Base, Off] : OffBounds) {
+ Type *IntTy = DL->getIntPtrType(Base->getType());
+ const SCEV *BaseSC = SE->getPtrToIntExpr(SE->getSCEV(Base), IntTy);
+ RTChecks.Bounds.try_emplace(Base, SE->getAddExpr(BaseSC, Off.first),
+ SE->getAddExpr(BaseSC, Off.second));
+ }
+ // Every involved base must contribute at least one bounded access, and its
+ // bounds must be expandable at the guard (so the checks only reference values
+ // available in the header).
+ SCEVExpander Exp(*SE, "slp.rtcheck");
+ // The guard is emitted after the header PHIs, so validate expandability at
+ // the first non-PHI: only values available in the header (PHIs, arguments,
+ // values defined before the block) dominate that point.
+ Instruction *GuardPt = &*BB->getFirstNonPHIIt();
+ if (any_of(Bases, [&](const Value *Base) {
+ auto *It = RTChecks.Bounds.find(Base);
+ return It == RTChecks.Bounds.end() ||
+ !Exp.isSafeToExpandAt(It->second.first, GuardPt) ||
+ !Exp.isSafeToExpandAt(It->second.second, GuardPt);
+ }))
+ return false;
+
+ // No SSA value defined in the body may escape the versioned region: that
+ // would require a merge PHI in the continuation block, which is not yet
+ // supported. A use by the terminator counts as an escape because the
+ // terminator is moved into the continuation block.
+ for (Instruction &I : *BB) {
+ if (isa<PHINode>(&I) || I.isTerminator())
+ continue;
+ if (any_of(I.users(), [&](User *U) {
+ auto *UI = dyn_cast<Instruction>(U);
+ return !UI || UI->getParent() != BB || UI->isTerminator() ||
+ isa<PHINode>(UI);
+ }))
+ return false;
+ }
+
+ // The runtime checks and the guard branch run on both the vector and the
+ // scalar fallback path. When aliasing is detected at runtime the scalar code
+ // executes unchanged plus the check overhead, so bound the check cost
+ // relative to the guarded scalar region to avoid pessimizing that path too
+ // much. The scalar region cost is the cost of the (current, still scalar)
+ // block body; no IR is emitted here.
+ constexpr TTI::TargetCostKind CostKind = TTI::TCK_RecipThroughput;
+ InstructionCost ScalarCost = 0;
+ for (Instruction &I : *BB) {
+ if (isa<PHINode>(&I) || I.isTerminator())
+ continue;
+ ScalarCost += TTI->getInstructionCost(&I, CostKind);
+ }
+ InstructionCost CheckCost = getRuntimeChecksCost();
+ if (!ScalarCost.isValid() || !CheckCost.isValid())
+ return false;
+ if (CheckCost * 100 >
+ ScalarCost * SLPRuntimeAliasChecksMaxScalarCostPercent.getValue())
+ return false;
+
+ // Freeze the check set so the final scheduleBlock() keeps the same dropped
+ // dependencies the bounds were just computed for.
+ RTChecksFinalized = true;
+ return true;
+}
+
+InstructionCost BoUpSLP::getRuntimeChecksCost() const {
+ if (RTChecks.BasePairs.empty())
+ return 0;
+ LLVMContext &Ctx = F->getContext();
+ Type *IntTy = DL->getIntPtrType(Ctx);
+ Type *I1Ty = Type::getInt1Ty(Ctx);
+ constexpr TTI::TargetCostKind Kind = TTI::TCK_RecipThroughput;
+ unsigned NumBases = RTChecks.Bounds.size();
+ unsigned NumPairs = RTChecks.BasePairs.size();
+ InstructionCost Cost = 0;
+ // Per base: a [Low, High) address pair. canVersionForRuntimeChecks() folds
+ // each bound to a base-plus-constant offset, so it expands to two integer
+ // adds (one per bound) with no runtime umin/umax reduction.
+ Cost += TTI->getArithmeticInstrCost(Instruction::Add, IntTy, Kind) *
+ (2 * NumBases);
+ // Two integer compares and one logical and per checked pair.
+ InstructionCost CmpCost = TTI->getCmpSelInstrCost(
+ Instruction::ICmp, IntTy, I1Ty, CmpInst::ICMP_ULT, Kind);
+ InstructionCost AndCost =
+ TTI->getArithmeticInstrCost(Instruction::And, I1Ty, Kind);
+ Cost += (CmpCost * 2 + AndCost) * NumPairs;
+ // Or-reduction of the per-pair conflicts.
+ if (NumPairs > 1)
+ Cost += TTI->getArithmeticInstrCost(Instruction::Or, I1Ty, Kind) *
+ (NumPairs - 1);
+ // The guard branch.
+ Cost += TTI->getCFInstrCost(Instruction::CondBr, Kind);
+ return Cost;
+}
+
+Value *BoUpSLP::emitRuntimeAliasCheck(IRBuilderBase &Builder,
+ SCEVExpander &Exp) {
+ // Materialize the [Low, High) address bounds for every involved base from
+ // their SCEVs at the builder's insertion point.
+ SmallDenseMap<const Value *, std::pair<Value *, Value *>, 4> Range;
+ for (const auto &[Base, Bound] : RTChecks.Bounds) {
+ Type *IntTy = DL->getIntPtrType(Base->getType());
+ Value *Low =
+ Exp.expandCodeFor(Bound.first, IntTy, Builder.GetInsertPoint());
+ Value *High =
+ Exp.expandCodeFor(Bound.second, IntTy, Builder.GetInsertPoint());
+ Range[Base] = {Low, High};
+ }
+ Value *Cond = nullptr;
+ for (const auto &P : RTChecks.BasePairs) {
+ std::pair<Value *, Value *> A = Range.lookup(P.first);
+ std::pair<Value *, Value *> B = Range.lookup(P.second);
+ assert(A.first && A.second && B.first && B.second &&
+ "Missing bounds for a checked base object");
+ // The objects overlap iff (A.Low < B.High) && (B.Low < A.High).
+ Value *C0 = Builder.CreateICmpULT(A.first, B.second, "rt.bound0");
+ Value *C1 = Builder.CreateICmpULT(B.first, A.second, "rt.bound1");
+ Value *Conflict = Builder.CreateAnd(C0, C1, "rt.conflict");
+ Cond =
+ Cond ? Builder.CreateOr(Cond, Conflict, "rt.conflict.all") : Conflict;
+ }
+ return Builder.CreateFreeze(Cond, "rt.guard");
+}
+
+void BoUpSLP::versionBlocksForRuntimeChecks() {
+ // Only version when this is the optimistic attempt that collected the checks.
+ if (!TryRuntimeAliasChecks || RTChecks.BasePairs.empty() || !RTChecks.BB ||
+ RTOrigBodyOrder.empty())
+ return;
+ BasicBlock *BB = RTChecks.BB;
+ Function *Fn = BB->getParent();
+ LLVMContext &Ctx = BB->getContext();
+ Instruction *Term = BB->getTerminator();
+ assert(Term && "Versioned block must have a terminator");
+
+ // Build the diamond:
+ // BB (header: PHIs + guard) -> {VecBB, ScalarBB} -> Tail (terminator).
+ BasicBlock *VecBB = BasicBlock::Create(Ctx, BB->getName() + ".rtvec", Fn);
+ BasicBlock *ScalarBB =
+ BasicBlock::Create(Ctx, BB->getName() + ".rtscalar", Fn);
+ BasicBlock *Tail = BasicBlock::Create(Ctx, BB->getName() + ".rtcont", Fn);
+
+ // Clone the body into the scalar fallback block, in the original program
+ // order (RTOrigBodyOrder).
+ SmallDenseMap<Value *, Value *, 16> VMap;
+ for (Instruction *I : RTOrigBodyOrder) {
+ assert(I->getParent() == BB &&
+ "RTOrigBodyOrder entry no longer in the versioned block");
+ Instruction *C = I->clone();
+ if (I->hasName())
+ C->setName(I->getName() + ".scalar");
+ C->insertInto(ScalarBB, ScalarBB->end());
+ VMap[I] = C;
+ }
+ // Remap the clones to use the cloned definitions; references to values
+ // defined outside the body (header or function arguments) are unchanged.
+ for (Instruction &I : *ScalarBB)
+ for (Use &U : I.operands())
+ if (Value *V = VMap.lookup(U.get()))
+ U.set(V);
+
+ // Move the body (everything but PHIs and the terminator) into the vector
+ // block in the scheduled order.
+ for (Instruction &I : make_early_inc_range(*BB)) {
+ if (isa<PHINode>(&I) || I.isTerminator())
+ continue;
+ I.removeFromParent();
+ I.insertInto(VecBB, VecBB->end());
+ }
+
+ // Emit the runtime alias check before the still-present terminator, so the
+ // SCEV bounds are expanded at a valid insertion point.
+ IRBuilder<> ChkBuilder(Term);
+ ChkBuilder.SetCurrentDebugLocation(Term->getDebugLoc());
+ SCEVExpander Exp(*SE, "slp.rtcheck");
+ Value *Cond = emitRuntimeAliasCheck(ChkBuilder, Exp);
+
+ // Move the original terminator into the continuation block and replace it
+ // with the guard branch.
+ Term->removeFromParent();
+ Term->insertInto(Tail, Tail->end());
+ UncondBrInst::Create(Tail, VecBB);
+ UncondBrInst::Create(Tail, ScalarBB);
+ ChkBuilder.SetInsertPoint(BB);
+ ChkBuilder.CreateCondBr(Cond, ScalarBB, VecBB);
+
+ // The continuation block is the new predecessor of the original successors.
+ for (BasicBlock *Succ : successors(Term))
+ Succ->replacePhiUsesWith(BB, Tail);
+
+ // Keep the dominator tree valid for the remainder of the run.
+ if (DT) {
+ DomTreeUpdater DTU(*DT, DomTreeUpdater::UpdateStrategy::Eager);
+ SmallVector<DominatorTree::UpdateType> Updates = {
+ {DominatorTree::Insert, BB, VecBB},
+ {DominatorTree::Insert, BB, ScalarBB},
+ {DominatorTree::Insert, VecBB, Tail},
+ {DominatorTree::Insert, ScalarBB, Tail}};
+ for (BasicBlock *Succ : successors(Term)) {
+ Updates.push_back({DominatorTree::Insert, Tail, Succ});
+ Updates.push_back({DominatorTree::Delete, BB, Succ});
+ }
+ DTU.applyUpdates(Updates);
+ // Refresh DFS numbers: getLastInstructionInBundle() may use them to order
+ // instructions across blocks for nodes emitted after versioning.
+ DT->updateDFSNumbers();
+ }
+
+ // Record the checked base pairs for the fast-path block so that subsequent
+ // vectorization there can reuse this guard instead of versioning again.
+ SmallDenseSet<std::pair<const Value *, const Value *>, 4> &Covered =
+ VersionedBlockCheckedPairs[VecBB];
+ Covered.insert_range(RTChecks.BasePairs);
+ // The scalar fallback must never be vectorized or versioned again.
+ ScalarFallbackBlocks.insert(ScalarBB);
+
+ // The checks have been emitted; clear so any later vectorizeTree() in the
+ // same attempt does not version again.
+ RTChecks.clear();
+ RTChecksFinalized = false;
+ RTOrigBodyOrder.clear();
+ // The CFG was modified; CFG analyses can no longer be preserved.
+ CFGChanged = true;
+}
+
Value *BoUpSLP::vectorizeTree() {
ExtraValueToDebugLocsMap ExternallyUsedValues;
return vectorizeTree(ExternallyUsedValues);
@@ -24421,6 +24996,13 @@ Value *BoUpSLP::vectorizeTree(
(void)getLastInstructionInBundle(TE.get());
}
+ // If this tree was scheduled by dropping may-alias memory dependencies in
+ // favor of runtime alias checks, materialize the checks and version the
+ // affected block before emitting the vector code. Scheduling has already
+ // run, so moving the body into the fast-path block is safe, and the
+ // dominator tree is updated for the rest of the run.
+ versionBlocksForRuntimeChecks();
+
if (ReductionRoot)
Builder.SetInsertPoint(ReductionRoot->getParent(),
ReductionRoot->getIterator());
@@ -26312,23 +26894,69 @@ void BoUpSLP::BlockScheduling::calculateDependencies(
// the whole loop (even if the loop is fast, it's quadratic).
// It's important for the loop break condition (see below) to
// check this limit even between two read-only instructions.
- if (DistToSrc >= MaxMemDepDistance ||
- ((SrcMayWrite || DepDest->getInst()->mayWriteToMemory()) &&
- (IsNonSimpleSrc || NumAliased >= AliasedCheckLimit ||
- SLP->isAliased(SrcLoc, SrcInst, DepDest->getInst())))) {
-
- // We increment the counter only if the locations are aliased
- // (instead of counting all alias checks). This gives a better
- // balance between reduced runtime and accurate dependencies.
- NumAliased++;
+ Instruction *DepInst = DepDest->getInst();
+ bool MayConflict = SrcMayWrite || DepInst->mayWriteToMemory();
+ bool AddDep = false;
+ if (DistToSrc >= MaxMemDepDistance) {
+ AddDep = true;
+ } else if (MayConflict &&
+ (IsNonSimpleSrc || NumAliased >= AliasedCheckLimit)) {
+ AddDep = true;
+ } else if (MayConflict && SLP->isAliased(SrcLoc, SrcInst, DepInst)) {
+ AddDep = true;
+ }
+
+ if (AddDep) {
+ // Try to turn a may-alias dependency between distinct, range-checkable
+ // base objects into a runtime alias check. This covers both concrete
+ // aliases and the conservative dependencies added once the
+ // per-instruction alias-check limit is reached, so that a high alias
+ // count cannot block a region that is otherwise versionable. The
+ // dropped dependency is recorded so the final scheduleBlock()
+ // recomputation keeps it dropped consistently.
+ bool Dropped = false;
+ if (MayConflict && SLPEnableRuntimeAliasChecks) {
+ auto Key = std::make_pair(SrcInst, DepInst);
+ // A dependency dropped during buildTree() must stay dropped on every
+ // later recomputation (e.g. the final scheduleBlock()), independent
+ // of the current mode, so the schedule remains consistent.
+ if (IgnoredMemDeps.contains(Key)) {
+ Dropped = true;
+ } else if (SLP->isCoveredByExistingVersionCheck(BB, SrcInst,
+ DepInst)) {
+ // This block is the fast path of an earlier versioning whose
+ // runtime check already proved these bases disjoint, so the
+ // dependency can be dropped without emitting a new check.
+ Dropped = true;
+ } else if (SLP->isTryingRuntimeAliasChecks()) {
+ if (SLP->isRuntimeCheckableAliasPair(SrcInst, DepInst) &&
+ SLP->recordRuntimeAliasCheck(BB, SrcInst, DepInst)) {
+ IgnoredMemDeps.insert(Key);
+ Dropped = true;
+ }
+ } else if (!SLP->hasRuntimeCheckableBlockers() &&
+ SLP->isRuntimeCheckableAliasPair(SrcInst, DepInst)) {
+ // Record only that a runtime check could unblock this region; the
+ // retry will collect the actual checks.
+ SLP->setHasRuntimeCheckableBlockers(true);
+ }
+ }
- DepDest->addMemoryDependency(BundleMember);
- BundleMember->incDependencies();
- if (!DepDest->isScheduled())
- BundleMember->incrementUnscheduledDeps(1);
- if (!DepDest->hasValidDependencies() ||
- (InsertInReadyList && DepDest->isReady()))
- WorkList.push_back(DepDest);
+ if (!Dropped) {
+ // Count only kept dependencies toward the alias-check limit. Counting
+ // dropped ones could prematurely trigger the conservative
+ // (limit-based) dependencies above and add spurious dependencies
+ // (e.g. between non-overlapping stores to the same base) that would
+ // block scheduling of an otherwise versionable region.
+ NumAliased++;
+ DepDest->addMemoryDependency(BundleMember);
+ BundleMember->incDependencies();
+ if (!DepDest->isScheduled())
+ BundleMember->incrementUnscheduledDeps(1);
+ if (!DepDest->hasValidDependencies() ||
+ (InsertInReadyList && DepDest->isReady()))
+ WorkList.push_back(DepDest);
+ }
}
// Example, explaining the loop break condition: Let's assume our
@@ -27549,7 +28177,13 @@ PreservedAnalyses SLPVectorizerPass::run(Function &F, FunctionAnalysisManager &A
return PreservedAnalyses::all();
PreservedAnalyses PA;
- PA.preserveSet<CFGAnalyses>();
+ // Runtime alias check versioning changes the CFG. In that case only the
+ // dominator tree is kept up to date (via DomTreeUpdater); other CFG analyses
+ // must be recomputed. Otherwise SLP leaves the CFG intact.
+ if (CFGChanged)
+ PA.preserve<DominatorTreeAnalysis>();
+ else
+ PA.preserveSet<CFGAnalyses>();
return PA;
}
@@ -27604,6 +28238,11 @@ bool SLPVectorizerPass::runImpl(Function &F, ScalarEvolution *SE_,
if (BB->isEHPad() || isa_and_nonnull<UnreachableInst>(BB->getTerminator()))
continue;
+ // Skip scalar fallback blocks created by runtime alias check versioning:
+ // they duplicate the original scalar code and must not be vectorized again.
+ if (R.isScalarFallbackBlock(BB))
+ continue;
+
// Start new block - clear the list of reduction roots.
R.clearReductionData();
collectSeedInstructions(BB);
@@ -27632,6 +28271,9 @@ bool SLPVectorizerPass::runImpl(Function &F, ScalarEvolution *SE_,
R.optimizeGatherSequence();
LLVM_DEBUG(dbgs() << "SLP: vectorized \"" << F.getName() << "\"\n");
}
+ // Propagate whether runtime alias check versioning changed the CFG, so the
+ // caller can drop CFG-analysis preservation only when necessary.
+ CFGChanged = R.isCFGChanged();
return Changed;
}
@@ -27639,7 +28281,31 @@ std::optional<bool>
SLPVectorizerPass::vectorizeStoreChain(ArrayRef<Value *> Chain, BoUpSLP &R,
unsigned Idx, unsigned MinVF,
unsigned &Size) {
+ std::optional<bool> Res = vectorizeStoreChainImpl(Chain, R, Idx, MinVF, Size);
+ // Retry once with runtime alias checks, but only when the normal attempt
+ // could not even schedule the bundle, i.e. it was blocked by
+ // a memory dependency, and that dependency is runtime-checkable.
+ if (Res.has_value() || !SLPEnableRuntimeAliasChecks ||
+ R.isTryingRuntimeAliasChecks() || !R.hasRuntimeCheckableBlockers())
+ return Res;
+ R.setTryRuntimeAliasChecks(true);
+ unsigned RTSize = Size;
+ std::optional<bool> RTRes =
+ vectorizeStoreChainImpl(Chain, R, Idx, MinVF, RTSize);
+ R.setTryRuntimeAliasChecks(false);
+ if (RTRes && *RTRes) {
+ Size = RTSize;
+ return RTRes;
+ }
+ return Res;
+}
+
+std::optional<bool>
+SLPVectorizerPass::vectorizeStoreChainImpl(ArrayRef<Value *> Chain, BoUpSLP &R,
+ unsigned Idx, unsigned MinVF,
+ unsigned &Size) {
Size = 0;
+ R.resetRuntimeAliasCheckState();
LLVM_DEBUG(dbgs() << "SLP: Analyzing a store chain of length " << Chain.size()
<< "\n");
const unsigned Sz = R.getVectorElementSize(Chain[0]);
@@ -27699,6 +28365,8 @@ SLPVectorizerPass::vectorizeStoreChain(ArrayRef<Value *> Chain, BoUpSLP &R,
R.reorderTopToBottom();
R.reorderBottomToTop();
}
+ if (R.isTryingRuntimeAliasChecks())
+ R.captureRuntimeCheckBodySnapshot();
R.transformNodes();
R.computeMinimumValueSizes();
@@ -27710,6 +28378,16 @@ SLPVectorizerPass::vectorizeStoreChain(ArrayRef<Value *> Chain, BoUpSLP &R,
Size = 2; // cut off masked gather small trees
InstructionCost Cost = R.getTreeCost(TreeCost);
+ // When the tree was scheduled by dropping may-alias dependencies, account
+ // for the runtime alias checks in the cost and make sure the region can
+ // actually be versioned. If it cannot, abandon this attempt: the dropped
+ // dependencies are unused because the tree is not vectorized.
+ if (R.isTryingRuntimeAliasChecks() && R.hasRuntimeAliasChecks()) {
+ if (!R.canVersionForRuntimeChecks())
+ return false;
+ Cost += R.getRuntimeChecksCost();
+ }
+
LLVM_DEBUG(dbgs() << "SLP: Found cost = " << Cost << " for VF=" << VF << "\n");
if (Cost < -SLPCostThreshold) {
LLVM_DEBUG(dbgs() << "SLP: Decided to vectorize cost = " << Cost << "\n");
diff --git a/llvm/test/Transforms/SLPVectorizer/AArch64/loadi8.ll b/llvm/test/Transforms/SLPVectorizer/AArch64/loadi8.ll
index a5371cd051a5b..58da711cf5434 100644
--- a/llvm/test/Transforms/SLPVectorizer/AArch64/loadi8.ll
+++ b/llvm/test/Transforms/SLPVectorizer/AArch64/loadi8.ll
@@ -88,24 +88,53 @@ entry:
define void @f_alias(ptr nocapture %dst, ptr nocapture readonly %src, ptr nocapture readonly %w) {
; CHECK-LABEL: @f_alias(
; CHECK-NEXT: entry:
+; CHECK-NEXT: [[DST2:%.*]] = ptrtoint ptr [[DST:%.*]] to i64
+; CHECK-NEXT: [[SRC1:%.*]] = ptrtoint ptr [[SRC:%.*]] to i64
+; CHECK-NEXT: [[TMP19:%.*]] = add i64 [[SRC1]], 4
+; CHECK-NEXT: [[TMP20:%.*]] = add i64 [[DST2]], 4
+; CHECK-NEXT: [[RT_BOUND0:%.*]] = icmp ult i64 [[DST2]], [[TMP19]]
+; CHECK-NEXT: [[RT_BOUND1:%.*]] = icmp ult i64 [[SRC1]], [[TMP20]]
+; CHECK-NEXT: [[RT_CONFLICT:%.*]] = and i1 [[RT_BOUND0]], [[RT_BOUND1]]
+; CHECK-NEXT: [[RT_GUARD:%.*]] = freeze i1 [[RT_CONFLICT]]
+; CHECK-NEXT: br i1 [[RT_GUARD]], label [[ENTRY_RTSCALAR:%.*]], label [[ENTRY_RTVEC:%.*]]
+; CHECK: entry.rtvec:
; CHECK-NEXT: [[TMP0:%.*]] = load i32, ptr [[W:%.*]], align 16
; CHECK-NEXT: [[OFFSET:%.*]] = getelementptr inbounds [[STRUCT_WEIGHT_T:%.*]], ptr [[W]], i64 0, i32 1
; CHECK-NEXT: [[TMP1:%.*]] = load i32, ptr [[OFFSET]], align 4
-; CHECK-NEXT: [[TMP2:%.*]] = load i8, ptr [[SRC:%.*]], align 1
+; CHECK-NEXT: [[TMP21:%.*]] = load <4 x i8>, ptr [[SRC]], align 1
+; CHECK-NEXT: [[TMP22:%.*]] = zext <4 x i8> [[TMP21]] to <4 x i32>
+; CHECK-NEXT: [[TMP23:%.*]] = insertelement <4 x i32> poison, i32 [[TMP0]], i32 0
+; CHECK-NEXT: [[TMP24:%.*]] = shufflevector <4 x i32> [[TMP23]], <4 x i32> poison, <4 x i32> zeroinitializer
+; CHECK-NEXT: [[TMP25:%.*]] = mul nsw <4 x i32> [[TMP24]], [[TMP22]]
+; CHECK-NEXT: [[TMP26:%.*]] = insertelement <4 x i32> poison, i32 [[TMP1]], i32 0
+; CHECK-NEXT: [[TMP10:%.*]] = shufflevector <4 x i32> [[TMP26]], <4 x i32> poison, <4 x i32> zeroinitializer
+; CHECK-NEXT: [[TMP11:%.*]] = add nsw <4 x i32> [[TMP25]], [[TMP10]]
+; CHECK-NEXT: [[TMP12:%.*]] = icmp ult <4 x i32> [[TMP11]], splat (i32 256)
+; CHECK-NEXT: [[TMP13:%.*]] = icmp sgt <4 x i32> [[TMP11]], zeroinitializer
+; CHECK-NEXT: [[TMP14:%.*]] = sext <4 x i1> [[TMP13]] to <4 x i32>
+; CHECK-NEXT: [[TMP15:%.*]] = select <4 x i1> [[TMP12]], <4 x i32> [[TMP11]], <4 x i32> [[TMP14]]
+; CHECK-NEXT: [[TMP16:%.*]] = trunc <4 x i32> [[TMP15]] to <4 x i8>
+; CHECK-NEXT: store <4 x i8> [[TMP16]], ptr [[DST]], align 1
+; CHECK-NEXT: br label [[ENTRY_RTCONT:%.*]]
+; CHECK: entry.rtscalar:
+; CHECK-NEXT: [[TMP17:%.*]] = load i32, ptr [[W]], align 16
+; CHECK-NEXT: [[OFFSET_SCALAR:%.*]] = getelementptr inbounds [[STRUCT_WEIGHT_T]], ptr [[W]], i64 0, i32 1
+; CHECK-NEXT: [[TMP18:%.*]] = load i32, ptr [[OFFSET_SCALAR]], align 4
+; CHECK-NEXT: [[TMP2:%.*]] = load i8, ptr [[SRC]], align 1
; CHECK-NEXT: [[CONV:%.*]] = zext i8 [[TMP2]] to i32
-; CHECK-NEXT: [[MUL:%.*]] = mul nsw i32 [[TMP0]], [[CONV]]
-; CHECK-NEXT: [[ADD:%.*]] = add nsw i32 [[MUL]], [[TMP1]]
+; CHECK-NEXT: [[MUL_SCALAR:%.*]] = mul nsw i32 [[TMP17]], [[CONV]]
+; CHECK-NEXT: [[ADD:%.*]] = add nsw i32 [[MUL_SCALAR]], [[TMP18]]
; CHECK-NEXT: [[TOBOOL_NOT_I:%.*]] = icmp ult i32 [[ADD]], 256
; CHECK-NEXT: [[TMP3:%.*]] = icmp sgt i32 [[ADD]], 0
; CHECK-NEXT: [[SHR_I:%.*]] = sext i1 [[TMP3]] to i32
; CHECK-NEXT: [[COND_I:%.*]] = select i1 [[TOBOOL_NOT_I]], i32 [[ADD]], i32 [[SHR_I]]
; CHECK-NEXT: [[CONV_I:%.*]] = trunc i32 [[COND_I]] to i8
-; CHECK-NEXT: store i8 [[CONV_I]], ptr [[DST:%.*]], align 1
+; CHECK-NEXT: store i8 [[CONV_I]], ptr [[DST]], align 1
; CHECK-NEXT: [[ARRAYIDX_1:%.*]] = getelementptr inbounds i8, ptr [[SRC]], i64 1
; CHECK-NEXT: [[TMP4:%.*]] = load i8, ptr [[ARRAYIDX_1]], align 1
; CHECK-NEXT: [[CONV_1:%.*]] = zext i8 [[TMP4]] to i32
-; CHECK-NEXT: [[MUL_1:%.*]] = mul nsw i32 [[TMP0]], [[CONV_1]]
-; CHECK-NEXT: [[ADD_1:%.*]] = add nsw i32 [[MUL_1]], [[TMP1]]
+; CHECK-NEXT: [[MUL_1_SCALAR:%.*]] = mul nsw i32 [[TMP17]], [[CONV_1]]
+; CHECK-NEXT: [[ADD_1:%.*]] = add nsw i32 [[MUL_1_SCALAR]], [[TMP18]]
; CHECK-NEXT: [[TOBOOL_NOT_I_1:%.*]] = icmp ult i32 [[ADD_1]], 256
; CHECK-NEXT: [[TMP5:%.*]] = icmp sgt i32 [[ADD_1]], 0
; CHECK-NEXT: [[SHR_I_1:%.*]] = sext i1 [[TMP5]] to i32
@@ -116,8 +145,8 @@ define void @f_alias(ptr nocapture %dst, ptr nocapture readonly %src, ptr nocapt
; CHECK-NEXT: [[ARRAYIDX_2:%.*]] = getelementptr inbounds i8, ptr [[SRC]], i64 2
; CHECK-NEXT: [[TMP6:%.*]] = load i8, ptr [[ARRAYIDX_2]], align 1
; CHECK-NEXT: [[CONV_2:%.*]] = zext i8 [[TMP6]] to i32
-; CHECK-NEXT: [[MUL_2:%.*]] = mul nsw i32 [[TMP0]], [[CONV_2]]
-; CHECK-NEXT: [[ADD_2:%.*]] = add nsw i32 [[MUL_2]], [[TMP1]]
+; CHECK-NEXT: [[MUL_2_SCALAR:%.*]] = mul nsw i32 [[TMP17]], [[CONV_2]]
+; CHECK-NEXT: [[ADD_2:%.*]] = add nsw i32 [[MUL_2_SCALAR]], [[TMP18]]
; CHECK-NEXT: [[TOBOOL_NOT_I_2:%.*]] = icmp ult i32 [[ADD_2]], 256
; CHECK-NEXT: [[TMP7:%.*]] = icmp sgt i32 [[ADD_2]], 0
; CHECK-NEXT: [[SHR_I_2:%.*]] = sext i1 [[TMP7]] to i32
@@ -128,8 +157,8 @@ define void @f_alias(ptr nocapture %dst, ptr nocapture readonly %src, ptr nocapt
; CHECK-NEXT: [[ARRAYIDX_3:%.*]] = getelementptr inbounds i8, ptr [[SRC]], i64 3
; CHECK-NEXT: [[TMP8:%.*]] = load i8, ptr [[ARRAYIDX_3]], align 1
; CHECK-NEXT: [[CONV_3:%.*]] = zext i8 [[TMP8]] to i32
-; CHECK-NEXT: [[MUL_3:%.*]] = mul nsw i32 [[TMP0]], [[CONV_3]]
-; CHECK-NEXT: [[ADD_3:%.*]] = add nsw i32 [[MUL_3]], [[TMP1]]
+; CHECK-NEXT: [[MUL_3_SCALAR:%.*]] = mul nsw i32 [[TMP17]], [[CONV_3]]
+; CHECK-NEXT: [[ADD_3:%.*]] = add nsw i32 [[MUL_3_SCALAR]], [[TMP18]]
; CHECK-NEXT: [[TOBOOL_NOT_I_3:%.*]] = icmp ult i32 [[ADD_3]], 256
; CHECK-NEXT: [[TMP9:%.*]] = icmp sgt i32 [[ADD_3]], 0
; CHECK-NEXT: [[SHR_I_3:%.*]] = sext i1 [[TMP9]] to i32
@@ -137,6 +166,8 @@ define void @f_alias(ptr nocapture %dst, ptr nocapture readonly %src, ptr nocapt
; CHECK-NEXT: [[CONV_I_3:%.*]] = trunc i32 [[COND_I_3]] to i8
; CHECK-NEXT: [[ARRAYIDX2_3:%.*]] = getelementptr inbounds i8, ptr [[DST]], i64 3
; CHECK-NEXT: store i8 [[CONV_I_3]], ptr [[ARRAYIDX2_3]], align 1
+; CHECK-NEXT: br label [[ENTRY_RTCONT]]
+; CHECK: entry.rtcont:
; CHECK-NEXT: ret void
;
entry:
diff --git a/llvm/test/Transforms/SLPVectorizer/X86/runtime-alias-checks.ll b/llvm/test/Transforms/SLPVectorizer/X86/runtime-alias-checks.ll
index 08f6bd5c46571..b6fa05abb5795 100644
--- a/llvm/test/Transforms/SLPVectorizer/X86/runtime-alias-checks.ll
+++ b/llvm/test/Transforms/SLPVectorizer/X86/runtime-alias-checks.ll
@@ -1,5 +1,6 @@
; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --version 6
; RUN: opt -passes=slp-vectorizer -mtriple=x86_64-unknown-linux-gnu -mcpu=skylake-avx512 -S < %s | FileCheck %s
+; RUN: opt -passes=slp-vectorizer -mtriple=x86_64-unknown-linux-gnu -mcpu=skylake-avx512 -slp-vectorize-with-runtime-alias-checks=false -S < %s | FileCheck %s --check-prefix=NOCHK
declare void @nodup() noduplicate
declare void @musttail_callee(ptr, ptr, ptr)
@@ -8,6 +9,35 @@ define void @test(ptr %dst, ptr %x, ptr %y) {
; CHECK-LABEL: define void @test(
; CHECK-SAME: ptr [[DST:%.*]], ptr [[X:%.*]], ptr [[Y:%.*]]) #[[ATTR1:[0-9]+]] {
; CHECK-NEXT: [[ENTRY:.*:]]
+; CHECK-NEXT: [[DST10:%.*]] = ptrtoint ptr [[DST]] to i64
+; CHECK-NEXT: [[Y9:%.*]] = ptrtoint ptr [[Y]] to i64
+; CHECK-NEXT: [[X8:%.*]] = ptrtoint ptr [[X]] to i64
+; CHECK-NEXT: [[TMP0:%.*]] = add i64 [[X8]], 64
+; CHECK-NEXT: [[TMP1:%.*]] = add i64 [[Y9]], 64
+; CHECK-NEXT: [[TMP2:%.*]] = add i64 [[DST10]], 64
+; CHECK-NEXT: [[RT_BOUND0:%.*]] = icmp ult i64 [[DST10]], [[TMP0]]
+; CHECK-NEXT: [[RT_BOUND1:%.*]] = icmp ult i64 [[X8]], [[TMP2]]
+; CHECK-NEXT: [[RT_CONFLICT:%.*]] = and i1 [[RT_BOUND0]], [[RT_BOUND1]]
+; CHECK-NEXT: [[RT_BOUND011:%.*]] = icmp ult i64 [[DST10]], [[TMP1]]
+; CHECK-NEXT: [[RT_BOUND112:%.*]] = icmp ult i64 [[Y9]], [[TMP2]]
+; CHECK-NEXT: [[RT_CONFLICT13:%.*]] = and i1 [[RT_BOUND011]], [[RT_BOUND112]]
+; CHECK-NEXT: [[RT_CONFLICT_ALL:%.*]] = or i1 [[RT_CONFLICT]], [[RT_CONFLICT13]]
+; CHECK-NEXT: [[RT_GUARD:%.*]] = freeze i1 [[RT_CONFLICT_ALL]]
+; CHECK-NEXT: br i1 [[RT_GUARD]], label %[[ENTRY_RTSCALAR:.*]], label %[[ENTRY_RTVEC:.*]]
+; CHECK: [[ENTRY_RTVEC]]:
+; CHECK-NEXT: [[TMP3:%.*]] = load <4 x double>, ptr [[X]], align 8
+; CHECK-NEXT: [[TMP4:%.*]] = load <4 x double>, ptr [[Y]], align 8
+; CHECK-NEXT: [[TMP5:%.*]] = fdiv <4 x double> [[TMP3]], [[TMP4]]
+; CHECK-NEXT: store <4 x double> [[TMP5]], ptr [[DST]], align 8
+; CHECK-NEXT: [[X4P1:%.*]] = getelementptr inbounds double, ptr [[X]], i64 4
+; CHECK-NEXT: [[Y4P1:%.*]] = getelementptr inbounds double, ptr [[Y]], i64 4
+; CHECK-NEXT: [[DST8:%.*]] = getelementptr inbounds double, ptr [[DST]], i64 4
+; CHECK-NEXT: [[TMP6:%.*]] = load <4 x double>, ptr [[X4P1]], align 8
+; CHECK-NEXT: [[TMP7:%.*]] = load <4 x double>, ptr [[Y4P1]], align 8
+; CHECK-NEXT: [[TMP8:%.*]] = fdiv <4 x double> [[TMP6]], [[TMP7]]
+; CHECK-NEXT: store <4 x double> [[TMP8]], ptr [[DST8]], align 8
+; CHECK-NEXT: br label %[[ENTRY_RTCONT:.*]]
+; CHECK: [[ENTRY_RTSCALAR]]:
; CHECK-NEXT: [[X0:%.*]] = load double, ptr [[X]], align 8
; CHECK-NEXT: [[Y0:%.*]] = load double, ptr [[Y]], align 8
; CHECK-NEXT: [[D0:%.*]] = fdiv double [[X0]], [[Y0]]
@@ -61,8 +91,68 @@ define void @test(ptr %dst, ptr %x, ptr %y) {
; CHECK-NEXT: [[D7:%.*]] = fdiv double [[X7]], [[Y7]]
; CHECK-NEXT: [[DST7:%.*]] = getelementptr inbounds double, ptr [[DST]], i64 7
; CHECK-NEXT: store double [[D7]], ptr [[DST7]], align 8
+; CHECK-NEXT: br label %[[ENTRY_RTCONT]]
+; CHECK: [[ENTRY_RTCONT]]:
; CHECK-NEXT: ret void
;
+; NOCHK-LABEL: define void @test(
+; NOCHK-SAME: ptr [[DST:%.*]], ptr [[X:%.*]], ptr [[Y:%.*]]) #[[ATTR0:[0-9]+]] {
+; NOCHK-NEXT: [[ENTRY:.*:]]
+; NOCHK-NEXT: [[X0:%.*]] = load double, ptr [[X]], align 8
+; NOCHK-NEXT: [[Y0:%.*]] = load double, ptr [[Y]], align 8
+; NOCHK-NEXT: [[D0:%.*]] = fdiv double [[X0]], [[Y0]]
+; NOCHK-NEXT: store double [[D0]], ptr [[DST]], align 8
+; NOCHK-NEXT: [[X1P:%.*]] = getelementptr inbounds double, ptr [[X]], i64 1
+; NOCHK-NEXT: [[X1:%.*]] = load double, ptr [[X1P]], align 8
+; NOCHK-NEXT: [[Y1P:%.*]] = getelementptr inbounds double, ptr [[Y]], i64 1
+; NOCHK-NEXT: [[Y1:%.*]] = load double, ptr [[Y1P]], align 8
+; NOCHK-NEXT: [[D1:%.*]] = fdiv double [[X1]], [[Y1]]
+; NOCHK-NEXT: [[DST1:%.*]] = getelementptr inbounds double, ptr [[DST]], i64 1
+; NOCHK-NEXT: store double [[D1]], ptr [[DST1]], align 8
+; NOCHK-NEXT: [[X2P:%.*]] = getelementptr inbounds double, ptr [[X]], i64 2
+; NOCHK-NEXT: [[X2:%.*]] = load double, ptr [[X2P]], align 8
+; NOCHK-NEXT: [[Y2P:%.*]] = getelementptr inbounds double, ptr [[Y]], i64 2
+; NOCHK-NEXT: [[Y2:%.*]] = load double, ptr [[Y2P]], align 8
+; NOCHK-NEXT: [[D2:%.*]] = fdiv double [[X2]], [[Y2]]
+; NOCHK-NEXT: [[DST2:%.*]] = getelementptr inbounds double, ptr [[DST]], i64 2
+; NOCHK-NEXT: store double [[D2]], ptr [[DST2]], align 8
+; NOCHK-NEXT: [[X3P:%.*]] = getelementptr inbounds double, ptr [[X]], i64 3
+; NOCHK-NEXT: [[X3:%.*]] = load double, ptr [[X3P]], align 8
+; NOCHK-NEXT: [[Y3P:%.*]] = getelementptr inbounds double, ptr [[Y]], i64 3
+; NOCHK-NEXT: [[Y3:%.*]] = load double, ptr [[Y3P]], align 8
+; NOCHK-NEXT: [[D3:%.*]] = fdiv double [[X3]], [[Y3]]
+; NOCHK-NEXT: [[DST3:%.*]] = getelementptr inbounds double, ptr [[DST]], i64 3
+; NOCHK-NEXT: store double [[D3]], ptr [[DST3]], align 8
+; NOCHK-NEXT: [[X4P:%.*]] = getelementptr inbounds double, ptr [[X]], i64 4
+; NOCHK-NEXT: [[X4:%.*]] = load double, ptr [[X4P]], align 8
+; NOCHK-NEXT: [[Y4P:%.*]] = getelementptr inbounds double, ptr [[Y]], i64 4
+; NOCHK-NEXT: [[Y4:%.*]] = load double, ptr [[Y4P]], align 8
+; NOCHK-NEXT: [[D4:%.*]] = fdiv double [[X4]], [[Y4]]
+; NOCHK-NEXT: [[DST4:%.*]] = getelementptr inbounds double, ptr [[DST]], i64 4
+; NOCHK-NEXT: store double [[D4]], ptr [[DST4]], align 8
+; NOCHK-NEXT: [[X5P:%.*]] = getelementptr inbounds double, ptr [[X]], i64 5
+; NOCHK-NEXT: [[X5:%.*]] = load double, ptr [[X5P]], align 8
+; NOCHK-NEXT: [[Y5P:%.*]] = getelementptr inbounds double, ptr [[Y]], i64 5
+; NOCHK-NEXT: [[Y5:%.*]] = load double, ptr [[Y5P]], align 8
+; NOCHK-NEXT: [[D5:%.*]] = fdiv double [[X5]], [[Y5]]
+; NOCHK-NEXT: [[DST5:%.*]] = getelementptr inbounds double, ptr [[DST]], i64 5
+; NOCHK-NEXT: store double [[D5]], ptr [[DST5]], align 8
+; NOCHK-NEXT: [[X6P:%.*]] = getelementptr inbounds double, ptr [[X]], i64 6
+; NOCHK-NEXT: [[X6:%.*]] = load double, ptr [[X6P]], align 8
+; NOCHK-NEXT: [[Y6P:%.*]] = getelementptr inbounds double, ptr [[Y]], i64 6
+; NOCHK-NEXT: [[Y6:%.*]] = load double, ptr [[Y6P]], align 8
+; NOCHK-NEXT: [[D6:%.*]] = fdiv double [[X6]], [[Y6]]
+; NOCHK-NEXT: [[DST6:%.*]] = getelementptr inbounds double, ptr [[DST]], i64 6
+; NOCHK-NEXT: store double [[D6]], ptr [[DST6]], align 8
+; NOCHK-NEXT: [[X7P:%.*]] = getelementptr inbounds double, ptr [[X]], i64 7
+; NOCHK-NEXT: [[X7:%.*]] = load double, ptr [[X7P]], align 8
+; NOCHK-NEXT: [[Y7P:%.*]] = getelementptr inbounds double, ptr [[Y]], i64 7
+; NOCHK-NEXT: [[Y7:%.*]] = load double, ptr [[Y7P]], align 8
+; NOCHK-NEXT: [[D7:%.*]] = fdiv double [[X7]], [[Y7]]
+; NOCHK-NEXT: [[DST7:%.*]] = getelementptr inbounds double, ptr [[DST]], i64 7
+; NOCHK-NEXT: store double [[D7]], ptr [[DST7]], align 8
+; NOCHK-NEXT: ret void
+;
entry:
%x0 = load double, ptr %x, align 8
%y0 = load double, ptr %y, align 8
More information about the llvm-commits
mailing list