[llvm] Revert "[SLP] Support memory runtime alias checks" (PR #209173)
Alexey Bataev via llvm-commits
llvm-commits at lists.llvm.org
Mon Jul 13 06:08:14 PDT 2026
https://github.com/alexey-bataev created https://github.com/llvm/llvm-project/pull/209173
This reverts commit d4c6c8623e9bf9aa75ad049f55423223228116b3 to address
postcommit review https://github.com/llvm/llvm-project/pull/203631?email_source=notifications&email_token=ABI45DXYNFIHECEWI5IGYJT5ETMNLA5CNFSNUABKM5UWIORPF5TWS5BNNB2WEL2QOVWGYUTFOF2WK43UKJSXM2LFO4XTINRYGQ4TAMRWGAZ2M4TFMFZW63VMON2GC5DFL5RWQYLOM5S2KZLWMVXHJLDGN5XXIZLSL5RWY2LDNM#pullrequestreview-4684902603
>From 3f783010a23ede96c82efaddec0b1e3e934ab75c Mon Sep 17 00:00:00 2001
From: Alexey Bataev <a.bataev at outlook.com>
Date: Mon, 13 Jul 2026 06:08:01 -0700
Subject: [PATCH] =?UTF-8?q?[=F0=9D=98=80=F0=9D=97=BD=F0=9D=97=BF]=20initia?=
=?UTF-8?q?l=20version?=
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit
Created using spr 1.3.7
---
.../llvm/Transforms/Vectorize/SLPVectorizer.h | 13 -
.../Transforms/Vectorize/SLPVectorizer.cpp | 842 +-----------------
.../SLPVectorizer/AArch64/loadi8.ll | 51 +-
.../SLPVectorizer/X86/runtime-alias-checks.ll | 315 -------
4 files changed, 27 insertions(+), 1194 deletions(-)
diff --git a/llvm/include/llvm/Transforms/Vectorize/SLPVectorizer.h b/llvm/include/llvm/Transforms/Vectorize/SLPVectorizer.h
index 538d62626b37b..b3f5718ddfda4 100644
--- a/llvm/include/llvm/Transforms/Vectorize/SLPVectorizer.h
+++ b/llvm/include/llvm/Transforms/Vectorize/SLPVectorizer.h
@@ -172,25 +172,12 @@ struct SLPVectorizerPass : public OptionalPassInfoMixin<SLPVectorizerPass> {
unsigned Idx, unsigned MinVF,
unsigned &Size);
- /// Single vectorization attempt for a store chain. \p vectorizeStoreChain
- /// wraps this to retry once with runtime alias checks enabled when the
- /// normal attempt is blocked only by runtime-checkable may-alias
- /// dependencies.
- std::optional<bool> vectorizeStoreChainImpl(ArrayRef<Value *> Chain,
- slpvectorizer::BoUpSLP &R,
- unsigned Idx, unsigned MinVF,
- unsigned &Size);
-
bool vectorizeStores(
ArrayRef<StoreInst *> Stores, slpvectorizer::BoUpSLP &R,
DenseSet<std::tuple<Value *, Value *, Value *, Value *, unsigned>>
&Visited,
bool AllowMaskedStores = true);
- /// Set by runImpl() when runtime alias check versioning changed the CFG, so
- /// run() can drop CFG-analysis preservation only when necessary.
- bool CFGChanged = false;
-
/// The store instructions in a basic block organized by base pointer.
StoreListMap Stores;
diff --git a/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp b/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp
index a70aa1d35e02f..7e22ba3bd149c 100644
--- a/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp
+++ b/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp
@@ -38,7 +38,6 @@
#include "llvm/Analysis/CodeMetrics.h"
#include "llvm/Analysis/ConstantFolding.h"
#include "llvm/Analysis/DemandedBits.h"
-#include "llvm/Analysis/DomTreeUpdater.h"
#include "llvm/Analysis/GlobalsModRef.h"
#include "llvm/Analysis/IVDescriptors.h"
#include "llvm/Analysis/Loads.h"
@@ -53,7 +52,6 @@
#include "llvm/Analysis/VectorUtils.h"
#include "llvm/IR/Attributes.h"
#include "llvm/IR/BasicBlock.h"
-#include "llvm/IR/CFG.h"
#include "llvm/IR/Constant.h"
#include "llvm/IR/Constants.h"
#include "llvm/IR/DataLayout.h"
@@ -291,28 +289,6 @@ static cl::opt<bool> PerLaneGatherScale(
cl::desc("Use per-lane execution scale for gather/buildvector tree "
"entries to model LICM-hoistable buildvector sequences."));
-/// Enable versioning of a basic block with runtime alias checks.
-static cl::opt<bool> SLPEnableRuntimeAliasChecks(
- "slp-vectorize-with-runtime-alias-checks", cl::init(true), cl::Hidden,
- cl::desc("Allow SLP to version a block with runtime alias checks to "
- "vectorize trees blocked by may-alias memory dependencies."));
-
-/// Maximum number of runtime alias checks (one per pair of base objects) that
-/// may guard a single versioned region.
-static cl::opt<unsigned> SLPMaxRuntimeAliasChecks(
- "slp-max-runtime-alias-checks", cl::init(8), cl::Hidden,
- cl::desc("The maximum number of runtime alias checks generated to guard a "
- "single SLP-vectorized region."));
-
-/// The runtime checks and the guard branch execute on both the vector and the
-/// scalar fallback path, so they add overhead to the scalar code.
-static cl::opt<unsigned> SLPRuntimeAliasChecksMaxScalarCostPercent(
- "slp-runtime-alias-checks-max-scalar-cost-percent", cl::init(25),
- cl::Hidden,
- cl::desc("Maximum SLP runtime alias check cost, as a percentage of the "
- "guarded scalar region cost, before versioning is rejected to "
- "avoid pessimizing the scalar fallback path."));
-
// Limit the number of alias checks. The limit is chosen so that
// it has no negative effect on the llvm benchmarks.
static const unsigned AliasedCheckLimit = 10;
@@ -2103,93 +2079,7 @@ class slpvectorizer::BoUpSLP {
/// Construct a vectorizable tree that starts at \p Roots.
void buildTree(ArrayRef<Value *> Roots);
- /// Returns true if the last buildTree() observed a may-alias memory
- /// dependency between two distinct, range-checkable base objects, i.e. a
- /// dependency that could be turned into a runtime alias check.
- bool hasRuntimeCheckableBlockers() const {
- return HasRuntimeCheckableBlockers;
- }
-
- /// Records whether a may-alias dependency between distinct, range-checkable
- /// base objects has been observed, so the caller can decide to retry with
- /// runtime alias checks enabled.
- void setHasRuntimeCheckableBlockers(bool V) {
- HasRuntimeCheckableBlockers = V;
- }
-
- /// Returns true if the last buildTree() kept a may-alias memory dependency
- /// that is not runtime-checkable (call or a non-simple mem access). Such a
- /// dependency cannot be dropped, so a runtime-checks retry cannot unblock the
- /// region and would be pure overhead.
- bool hasNonCheckableMemBlocker() const { return HasNonCheckableMemBlocker; }
-
- /// Records that a non-runtime-checkable may-alias dependency was kept.
- void setHasNonCheckableMemBlocker(bool V) { HasNonCheckableMemBlocker = V; }
-
- /// Returns true if the current vectorization attempt may drop
- /// runtime-checkable may-alias dependencies and guard the region with
- /// runtime alias checks.
- bool isTryingRuntimeAliasChecks() const { return TryRuntimeAliasChecks; }
-
- /// Enables or disables dropping runtime-checkable may-alias dependencies in
- /// favor of runtime alias checks for the current vectorization attempt.
- void setTryRuntimeAliasChecks(bool V) { TryRuntimeAliasChecks = V; }
-
- /// Resets the runtime alias check data.
- void resetRuntimeAliasCheckState() {
- HasRuntimeCheckableBlockers = false;
- HasNonCheckableMemBlocker = false;
- RTChecksFinalized = false;
- RTChecks.clear();
- RTOrigBodyOrder.clear();
- }
-
- /// Snapshots RTChecks.BB's body (non-PHI, non-terminator) into
- /// RTOrigBodyOrder in program order, for the scalar fallback.
- void captureRuntimeCheckBodySnapshot();
-
- /// Returns true if \p BB satisfies the block-level preconditions for runtime
- /// alias check versioning (straight-line, outside any loop, duplicable, not a
- /// scalar fallback, function not optimized for size). These checks do not
- /// depend on the collected checks, so they can gate the (expensive)
- /// optimistic retry before any tree is rebuilt.
- bool canVersionBlockForRuntimeChecks(BasicBlock *BB) const;
-
- /// Returns true if the runtime alias checks can be safely emitted to guard
- /// the vectorized region.
- bool canVersionForRuntimeChecks();
-
- /// Returns true if \p BB is a scalar fallback block created by runtime alias
- /// check versioning.
- bool isScalarFallbackBlock(BasicBlock *BB) const {
- return ScalarFallbackBlocks.contains(BB);
- }
-
- /// Returns true if an optimistic runtime-checks versioning attempt already
- /// failed for \p BB, so further retries in the same block can be skipped.
- bool runtimeChecksFailedForBlock(BasicBlock *BB) const {
- return FailedRuntimeChecksBlocks.contains(BB);
- }
-
- /// Records that an optimistic runtime-checks versioning attempt failed for
- /// \p BB.
- void markRuntimeChecksFailedForBlock(BasicBlock *BB) {
- FailedRuntimeChecksBlocks.insert(BB);
- }
-
- /// Returns the modeled cost of the runtime alias checks collected during the
- /// last (optimistic) buildTree().
- InstructionCost getRuntimeChecksCost() const;
-
- /// Returns true if the last (optimistic) buildTree() collected any runtime
- /// alias checks that must guard the vectorized region.
- bool hasRuntimeAliasChecks() const { return !RTChecks.BasePairs.empty(); }
-
- /// Returns true if vectorization changed the CFG (i.e. a block was versioned
- /// with runtime alias checks). When true, CFG analyses must not be preserved.
- bool isCFGChanged() const { return CFGChanged; }
-
- /// Returns the scalars of the root node.
+ /// Return the scalars of the root node.
ArrayRef<Value *> getRootNodeScalars() const {
assert(!VectorizableTree.empty() && "No graph to get the first node from");
return VectorizableTree.front()->Scalars;
@@ -2296,10 +2186,6 @@ class slpvectorizer::BoUpSLP {
ExternalUses.clear();
ExternalUsesAsOriginalScalar.clear();
ExternalUsesWithNonUsers.clear();
- RTChecks.clear();
- HasRuntimeCheckableBlockers = false;
- HasNonCheckableMemBlocker = false;
- RTChecksFinalized = false;
for (auto &Iter : BlocksSchedules) {
BlockScheduling *BS = Iter.second.get();
BS->clear();
@@ -5048,89 +4934,6 @@ class slpvectorizer::BoUpSLP {
return Aliased;
}
- /// Returns true if the may-alias dependency between simple load/store
- /// instructions \p Inst1 and \p Inst2 could be disambiguated by a runtime
- /// alias check.
- bool isRuntimeCheckableAliasPair(Instruction *Inst1, Instruction *Inst2);
-
- /// Records the (distinct base object) pair behind the may-alias dependency
- /// of \p Inst1 and \p Inst2 as a runtime alias check guarding the region in
- /// block \p BB. Returns true if the pair was recorded.
- bool recordRuntimeAliasCheck(BasicBlock *BB, Instruction *Inst1,
- Instruction *Inst2);
-
- /// Emits the collected runtime alias checks and versions the affected block,
- /// duplicating its body into a scalar fallback guarded by the checks.
- void versionBlocksForRuntimeChecks();
-
- /// Builds the i1 value that is true when any pair of checked base objects
- /// overlaps at runtime. The base address bounds are materialized from their
- /// SCEVs with \p Exp.
- Value *emitRuntimeAliasCheck(IRBuilderBase &Builder, SCEVExpander &Exp);
-
- /// Data to model and emit the runtime alias checks.
- struct RuntimeAliasCheckInfo {
- /// The block whose body is guarded by the checks. Exactly one block is
- /// supported per attempt.
- BasicBlock *BB = nullptr;
- /// Pairs of base objects that must be proven disjoint.
- SmallSetVector<std::pair<const Value *, const Value *>, 4> BasePairs;
- /// Accessed address range [Low, High) for each involved base object.
- SmallMapVector<const Value *, std::pair<const SCEV *, const SCEV *>, 4>
- Bounds;
-
- void clear() {
- BB = nullptr;
- BasePairs.clear();
- Bounds.clear();
- }
- };
-
- /// When true, scheduling drops may-alias memory dependencies between
- /// distinct, range-checkable base objects and records them as runtime alias
- /// checks instead.
- bool TryRuntimeAliasChecks = false;
-
- /// Runtime alias checks collected during the last optimistic buildTree().
- RuntimeAliasCheckInfo RTChecks;
-
- /// Base-object pairs already proven disjoint by the block's runtime alias
- /// check.
- SmallDenseMap<BasicBlock *,
- SmallDenseSet<std::pair<const Value *, const Value *>, 4>, 2>
- VersionedBlockCheckedPairs;
-
- /// Scalar fallback blocks.
- SmallPtrSet<BasicBlock *, 4> ScalarFallbackBlocks;
-
- /// Blocks for which a runtime-checks versioning attempt was made
- /// and did not produce a profitable versioning.
- SmallPtrSet<BasicBlock *, 8> FailedRuntimeChecksBlocks;
-
- /// Returns true if a may-alias dependency between the simple load/store
- /// instructions \p Inst1 and \p Inst2 in block \p BB is already covered by a
- /// runtime alias check emitted for \p BB by a previous versioning.
- bool isCoveredByExistingVersionCheck(BasicBlock *BB, Instruction *Inst1,
- Instruction *Inst2) const;
-
- /// True, if a may-alias dependency between distinct, range-checkable base
- /// objects is observed (whether or not it was dropped).
- bool HasRuntimeCheckableBlockers = false;
-
- /// True, if a kept may-alias dependency is not runtime-checkable (call or a
- /// non-simple memaccess).
- bool HasNonCheckableMemBlocker = false;
-
- /// Runtime checks are validated and bounded the collected checks.
- bool RTChecksFinalized = false;
-
- /// Set when a block was versioned with runtime alias checks, which changes
- /// the CFG. Used to drop CFG-analysis preservation for the run.
- bool CFGChanged = false;
-
- /// Guarded block body (non-PHI, non-terminator) in original source order.
- SmallVector<Instruction *> RTOrigBodyOrder;
-
using AliasCacheKey = std::pair<Instruction *, Instruction *>;
/// Cache for alias results.
@@ -5714,7 +5517,6 @@ class slpvectorizer::BoUpSLP {
ScheduleCopyableDataMapByUsers.clear();
ReadyInsts.clear();
RecalcCopyableOperandDeps.clear();
- IgnoredMemDeps.clear();
ScheduleStart = nullptr;
ScheduleEnd = nullptr;
FirstLoadStoreInRegion = nullptr;
@@ -6589,10 +6391,6 @@ class slpvectorizer::BoUpSLP {
/// recomputing the same operand more than once.
SmallSetVector<ScheduleData *, 8> RecalcCopyableOperandDeps;
- /// Ordered pairs (Src, Dst) of memory instructions whose may-alias
- /// dependency has been dropped in favor of a runtime alias check.
- SmallDenseSet<std::pair<Instruction *, Instruction *>, 8> IgnoredMemDeps;
-
/// The ID of the scheduling region. For a new vectorization iteration this
/// is incremented which "removes" all ScheduleData from the region.
/// Make sure that the initial SchedulingRegionID is greater than the
@@ -24809,457 +24607,6 @@ Value *BoUpSLP::vectorizeTree(TreeEntry *E) {
return nullptr;
}
-bool BoUpSLP::isRuntimeCheckableAliasPair(Instruction *Inst1,
- Instruction *Inst2) {
- // Only simple (non-volatile, non-atomic) accesses may be reordered once
- // aliasing is ruled out at runtime; volatile/atomic ordering must be kept.
- if (!isSimple(Inst1) || !isSimple(Inst2))
- return false;
- Value *Ptr1 = getLoadStorePointerOperand(Inst1);
- Value *Ptr2 = getLoadStorePointerOperand(Inst2);
- // Only simple load/store accesses have a single pointer operand whose
- // accessed range can be bounded and compared at runtime.
- if (!Ptr1 || !Ptr2)
- return false;
- if (Ptr1->getType()->getPointerAddressSpace() !=
- Ptr2->getType()->getPointerAddressSpace())
- return false;
- const Value *Base1 = getUnderlyingObject(Ptr1);
- const Value *Base2 = getUnderlyingObject(Ptr2);
- // A runtime range check is only meaningful between two distinct, identifiable
- // objects: two accesses to the same base differ by a compile-time offset, so
- // they are resolved statically and a base-range check would fold to a
- // constant predicate rather than a useful runtime guard.
- if (Base1 == Base2 || isa<UndefValue>(Base1) || isa<UndefValue>(Base2))
- return false;
- return true;
-}
-
-bool BoUpSLP::isCoveredByExistingVersionCheck(BasicBlock *BB,
- Instruction *Inst1,
- Instruction *Inst2) const {
- const auto It = VersionedBlockCheckedPairs.find(BB);
- if (It == VersionedBlockCheckedPairs.end())
- return false;
- Value *Ptr1 = getLoadStorePointerOperand(Inst1);
- Value *Ptr2 = getLoadStorePointerOperand(Inst2);
- if (!Ptr1 || !Ptr2)
- return false;
- const Value *Base1 = getUnderlyingObject(Ptr1);
- const Value *Base2 = getUnderlyingObject(Ptr2);
- if (Base1 == Base2)
- return false;
- if (Base2 < Base1)
- std::swap(Base1, Base2);
- return It->second.contains({Base1, Base2});
-}
-
-/// Returns true if \p BB's body already contains vector instructions, e.g.
-/// from an earlier SLP vectorization in the same pass.
-static bool blockBodyHasVectorInstructions(BasicBlock *BB) {
- for (Instruction &I : *BB) {
- if (isa<PHINode>(&I) || I.isTerminator())
- continue;
- // A vector-producing instruction (vector load, binop, shuffle, etc.) has a
- // vector result type.
- if (getValueType(&I)->isVectorTy())
- return true;
- }
- return false;
-}
-
-void BoUpSLP::captureRuntimeCheckBodySnapshot() {
- RTOrigBodyOrder.clear();
- if (!TryRuntimeAliasChecks || !RTChecks.BB)
- return;
- BasicBlock *BB = RTChecks.BB;
- for (Instruction &I : *BB)
- if (!isa<PHINode>(&I) && !I.isTerminator())
- RTOrigBodyOrder.push_back(&I);
-}
-
-bool BoUpSLP::recordRuntimeAliasCheck(BasicBlock *BB, Instruction *Inst1,
- Instruction *Inst2) {
- Value *Ptr1 = getLoadStorePointerOperand(Inst1);
- Value *Ptr2 = getLoadStorePointerOperand(Inst2);
- if (!Ptr1 || !Ptr2)
- return false;
- const Value *Base1 = getUnderlyingObject(Ptr1);
- const Value *Base2 = getUnderlyingObject(Ptr2);
- if (Base1 == Base2)
- return false;
- // Only a single block can be versioned per attempt.
- if (RTChecks.BB && RTChecks.BB != BB)
- return false;
- // Normalize the pair order so duplicate checks collapse.
- if (Base2 < Base1)
- std::swap(Base1, Base2);
- auto Pair = std::make_pair(Base1, Base2);
- // After the checks have been validated and bounded, do not introduce new
- // pairs.
- if (RTChecksFinalized)
- return RTChecks.BasePairs.contains(Pair);
- RTChecks.BB = BB;
- RTChecks.BasePairs.insert(Pair);
- return true;
-}
-
-bool BoUpSLP::canVersionBlockForRuntimeChecks(BasicBlock *BB) const {
- assert(BB && "Expected a block to version for runtime checks.");
- if (!BB->getTerminator())
- return false;
- // Versioning duplicates the block body, increasing code size on the guarded
- // path; do not version when the function is optimized for size (-Os/-Oz).
- if (F->hasOptSize())
- return false;
- if (!DT || !LI)
- return false;
- if (!DT->isReachableFromEntry(BB))
- return false;
- // Versioning duplicates the block body; only straight-line code outside any
- // loop is handled for now, to avoid LoopInfo and region updates.
- if (LI->getLoopFor(BB))
- return false;
- // Never version a scalar fallback block: it is the safe, original-order copy
- // taken when aliasing is detected and must stay scalar.
- if (ScalarFallbackBlocks.contains(BB))
- return false;
- // The scalar fallback must be a faithful copy of the original scalar body.
- // Reject blocks that were already partially vectorized earlier in this pass.
- if (blockBodyHasVectorInstructions(BB))
- return false;
- // Versioning duplicates the original block body into a scalar fallback.
- // Calls that cannot be duplicated or whose semantics depend on the call being
- // immediately followed by a return cannot be cloned into the diamond.
- if (BB->getTerminatingMustTailCall())
- return false;
- if (any_of(*BB, [](Instruction &I) {
- auto *CB = dyn_cast<CallBase>(&I);
- return CB && (CB->cannotDuplicate() || CB->isConvergent());
- }))
- return false;
- return true;
-}
-
-bool BoUpSLP::canVersionForRuntimeChecks() {
- if (RTChecks.BasePairs.empty() || !RTChecks.BB)
- return false;
- if (RTChecks.BasePairs.size() > SLPMaxRuntimeAliasChecks)
- return false;
- BasicBlock *BB = RTChecks.BB;
- // Block-level preconditions (loop/optsize/duplicability/...) are the same
- // ones used to gate the optimistic retry, so reuse them here.
- if (!canVersionBlockForRuntimeChecks(BB))
- return false;
-
- // The scalar fallback is a clone of the body. Operands defined inside the
- // body are remapped to their clones, but operands defined outside the body
- // (header PHIs, dominating definitions) are reused as-is by the clone. If
- // such an outside operand is itself vectorized by this tree, vectorizeTree()
- // will delete its scalar, leaving the clone with a dangling, out-of-tree use.
- // Versioning cannot model that, so bail out.
- for (Instruction &I : *BB) {
- if (isa<PHINode>(&I) || I.isTerminator())
- continue;
- for (Value *Op : I.operands()) {
- auto *OpI = dyn_cast<Instruction>(Op);
- if (!OpI)
- continue;
- bool DefinedInBody =
- OpI->getParent() == BB && !isa<PHINode>(OpI) && !OpI->isTerminator();
- if (!DefinedInBody && isVectorized(OpI))
- return false;
- }
- }
-
- SmallPtrSet<const Value *, 8> Bases;
- for (const auto &P : RTChecks.BasePairs) {
- Bases.insert(P.first);
- Bases.insert(P.second);
- }
- // Every base object must be available in the (PHI-only) header where the
- // guard branch is emitted.
- if (any_of(Bases, [&](const Value *Base) {
- const auto *I = dyn_cast<Instruction>(Base);
- return I && (I->getParent() == BB || !DT->dominates(I, BB));
- }))
- return false;
-
- // Compute the accessed address range [Low, High) for every involved base.
- RTChecks.Bounds.clear();
- SmallMapVector<Value *, std::pair<const SCEV *, const SCEV *>, 4> OffBounds;
- unsigned NumMemInsts = 0;
- for (Instruction &I : *BB) {
- if (!I.mayReadOrWriteMemory())
- continue;
- // The dependency scan truncates beyond MaxMemDepDistance. Only version
- // blocks small enough that the scan is never truncated, so every
- // conflicting base pair is guaranteed to be recorded.
- if (++NumMemInsts > MaxMemDepDistance)
- return false;
- Value *Ptr = getLoadStorePointerOperand(&I);
- // A non-simple memory access (e.g. a call) keeps its dependencies, so its
- // relative order with kept-dependency partners is preserved inside the
- // duplicated body and it does not need a bound.
- if (!Ptr)
- continue;
- Value *Base = const_cast<Value *>(getUnderlyingObject(Ptr));
- if (!Bases.contains(Base))
- continue;
- TypeSize TS = DL->getTypeStoreSize(getLoadStoreType(&I));
- if (TS.isScalable())
- return false;
- Type *IntTy = DL->getIntPtrType(Base->getType());
- const SCEV *PtrSC = SE->getPtrToIntExpr(SE->getSCEV(Ptr), IntTy);
- const SCEV *BaseSC = SE->getPtrToIntExpr(SE->getSCEV(Base), IntTy);
- if (isa<SCEVCouldNotCompute>(PtrSC) || isa<SCEVCouldNotCompute>(BaseSC))
- return false;
- // Byte offset of this access from its base. Require a constant so the bound
- // stays base-plus-constant, and non-negative so the unsigned [Low, High)
- // range below actually covers the access.
- // TODO: Support variable offsets (e.g. base + %idx) which would need
- // runtime umin/umax bounds and a proof that the offset range cannot wrap,
- // plus a richer cost model.
- const auto *Off = dyn_cast<SCEVConstant>(SE->getMinusSCEV(PtrSC, BaseSC));
- if (!Off || Off->getAPInt().isNegative())
- return false;
- const SCEV *EndOff =
- SE->getAddExpr(Off, SE->getConstant(IntTy, TS.getFixedValue()));
- auto [It, Inserted] = OffBounds.try_emplace(Base, Off, EndOff);
- if (!Inserted) {
- // Constant operands, so these fold to a constant min/max offset rather
- // than emitting a runtime umin/umax.
- It->second.first = SE->getUMinExpr(It->second.first, Off);
- It->second.second = SE->getUMaxExpr(It->second.second, EndOff);
- }
- }
- // Materialize the absolute [Low, High) = base + [minOff, maxEndOff). The
- // offsets are constants, so each bound is a base-plus-constant expression
- // that expands to a single add off the base address (no runtime umin/umax).
- for (const auto &[Base, Off] : OffBounds) {
- Type *IntTy = DL->getIntPtrType(Base->getType());
- const SCEV *BaseSC = SE->getPtrToIntExpr(SE->getSCEV(Base), IntTy);
- RTChecks.Bounds.try_emplace(Base, SE->getAddExpr(BaseSC, Off.first),
- SE->getAddExpr(BaseSC, Off.second));
- }
- // Every involved base must contribute at least one bounded access, and its
- // bounds must be expandable at the guard (so the checks only reference values
- // available in the header).
- SCEVExpander Exp(*SE, "slp.rtcheck");
- // The guard is emitted after the header PHIs, so validate expandability at
- // the first non-PHI: only values available in the header (PHIs, arguments,
- // values defined before the block) dominate that point.
- Instruction *GuardPt = &*BB->getFirstNonPHIIt();
- if (any_of(Bases, [&](const Value *Base) {
- auto *It = RTChecks.Bounds.find(Base);
- return It == RTChecks.Bounds.end() ||
- !Exp.isSafeToExpandAt(It->second.first, GuardPt) ||
- !Exp.isSafeToExpandAt(It->second.second, GuardPt);
- }))
- return false;
-
- // No SSA value defined in the body may escape the versioned region: that
- // would require a merge PHI in the continuation block, which is not yet
- // supported. A use by the terminator counts as an escape because the
- // terminator is moved into the continuation block.
- for (Instruction &I : *BB) {
- if (isa<PHINode>(&I) || I.isTerminator())
- continue;
- if (any_of(I.users(), [&](User *U) {
- auto *UI = dyn_cast<Instruction>(U);
- return !UI || UI->getParent() != BB || UI->isTerminator();
- }))
- return false;
- }
-
- // The runtime checks and the guard branch run on both the vector and the
- // scalar fallback path. When aliasing is detected at runtime the scalar code
- // executes unchanged plus the check overhead, so bound the check cost
- // relative to the guarded scalar region to avoid pessimizing that path too
- // much. The scalar region cost is the cost of the (current, still scalar)
- // block body; no IR is emitted here.
- constexpr TTI::TargetCostKind CostKind = TTI::TCK_RecipThroughput;
- InstructionCost ScalarCost = 0;
- for (Instruction &I : *BB) {
- if (isa<PHINode>(&I) || I.isTerminator())
- continue;
- ScalarCost += TTI->getInstructionCost(&I, CostKind);
- }
- InstructionCost CheckCost = getRuntimeChecksCost();
- if (!ScalarCost.isValid() || !CheckCost.isValid())
- return false;
- if (CheckCost * 100 >
- ScalarCost * SLPRuntimeAliasChecksMaxScalarCostPercent.getValue())
- return false;
-
- // Freeze the check set so the final scheduleBlock() keeps the same dropped
- // dependencies the bounds were just computed for.
- RTChecksFinalized = true;
- return true;
-}
-
-InstructionCost BoUpSLP::getRuntimeChecksCost() const {
- if (RTChecks.BasePairs.empty())
- return 0;
- LLVMContext &Ctx = F->getContext();
- Type *IntTy = DL->getIntPtrType(Ctx);
- Type *I1Ty = Type::getInt1Ty(Ctx);
- constexpr TTI::TargetCostKind Kind = TTI::TCK_RecipThroughput;
- unsigned NumBases = RTChecks.Bounds.size();
- unsigned NumPairs = RTChecks.BasePairs.size();
- InstructionCost Cost = 0;
- // Per base: a [Low, High) address pair. canVersionForRuntimeChecks() folds
- // each bound to a base-plus-constant offset, so it expands to two integer
- // adds (one per bound) with no runtime umin/umax reduction.
- Cost += TTI->getArithmeticInstrCost(Instruction::Add, IntTy, Kind) *
- (2 * NumBases);
- // Two integer compares and one logical and per checked pair.
- InstructionCost CmpCost = TTI->getCmpSelInstrCost(
- Instruction::ICmp, IntTy, I1Ty, CmpInst::ICMP_ULT, Kind);
- InstructionCost AndCost =
- TTI->getArithmeticInstrCost(Instruction::And, I1Ty, Kind);
- Cost += (CmpCost * 2 + AndCost) * NumPairs;
- // Or-reduction of the per-pair conflicts.
- if (NumPairs > 1)
- Cost += TTI->getArithmeticInstrCost(Instruction::Or, I1Ty, Kind) *
- (NumPairs - 1);
- // The guard branch.
- Cost += TTI->getCFInstrCost(Instruction::CondBr, Kind);
- return Cost;
-}
-
-Value *BoUpSLP::emitRuntimeAliasCheck(IRBuilderBase &Builder,
- SCEVExpander &Exp) {
- // Materialize the [Low, High) address bounds for every involved base from
- // their SCEVs at the builder's insertion point.
- SmallDenseMap<const Value *, std::pair<Value *, Value *>, 4> Range;
- for (const auto &[Base, Bound] : RTChecks.Bounds) {
- Type *IntTy = DL->getIntPtrType(Base->getType());
- Value *Low =
- Exp.expandCodeFor(Bound.first, IntTy, Builder.GetInsertPoint());
- Value *High =
- Exp.expandCodeFor(Bound.second, IntTy, Builder.GetInsertPoint());
- Range[Base] = {Low, High};
- }
- Value *Cond = nullptr;
- for (const auto &P : RTChecks.BasePairs) {
- std::pair<Value *, Value *> A = Range.lookup(P.first);
- std::pair<Value *, Value *> B = Range.lookup(P.second);
- assert(A.first && A.second && B.first && B.second &&
- "Missing bounds for a checked base object");
- // The objects overlap iff (A.Low < B.High) && (B.Low < A.High).
- Value *C0 = Builder.CreateICmpULT(A.first, B.second, "rt.bound0");
- Value *C1 = Builder.CreateICmpULT(B.first, A.second, "rt.bound1");
- Value *Conflict = Builder.CreateAnd(C0, C1, "rt.conflict");
- Cond =
- Cond ? Builder.CreateOr(Cond, Conflict, "rt.conflict.all") : Conflict;
- }
- return Builder.CreateFreeze(Cond, "rt.guard");
-}
-
-void BoUpSLP::versionBlocksForRuntimeChecks() {
- // Only version when this is the optimistic attempt that collected the checks.
- if (!TryRuntimeAliasChecks || RTChecks.BasePairs.empty() || !RTChecks.BB ||
- RTOrigBodyOrder.empty())
- return;
- BasicBlock *BB = RTChecks.BB;
- Function *Fn = BB->getParent();
- LLVMContext &Ctx = BB->getContext();
- Instruction *Term = BB->getTerminator();
- assert(Term && "Versioned block must have a terminator");
-
- // Build the diamond:
- // BB (header: PHIs + guard) -> {VecBB, ScalarBB} -> Tail (terminator).
- BasicBlock *VecBB = BasicBlock::Create(Ctx, BB->getName() + ".rtvec", Fn);
- BasicBlock *ScalarBB =
- BasicBlock::Create(Ctx, BB->getName() + ".rtscalar", Fn);
- BasicBlock *Tail = BasicBlock::Create(Ctx, BB->getName() + ".rtcont", Fn);
-
- // Clone the body into the scalar fallback block, in the original program
- // order (RTOrigBodyOrder).
- SmallDenseMap<Value *, Value *, 16> VMap;
- for (Instruction *I : RTOrigBodyOrder) {
- assert(I->getParent() == BB &&
- "RTOrigBodyOrder entry no longer in the versioned block");
- Instruction *C = I->clone();
- if (I->hasName())
- C->setName(I->getName() + ".scalar");
- C->insertInto(ScalarBB, ScalarBB->end());
- VMap[I] = C;
- }
- // Remap the clones to use the cloned definitions; references to values
- // defined outside the body (header or function arguments) are unchanged.
- for (Instruction &I : *ScalarBB)
- for (Use &U : I.operands())
- if (Value *V = VMap.lookup(U.get()))
- U.set(V);
-
- // Move the body (everything but PHIs and the terminator) into the vector
- // block in the scheduled order.
- for (Instruction &I : make_early_inc_range(*BB)) {
- if (isa<PHINode>(&I) || I.isTerminator())
- continue;
- I.removeFromParent();
- I.insertInto(VecBB, VecBB->end());
- }
-
- // Emit the runtime alias check before the still-present terminator, so the
- // SCEV bounds are expanded at a valid insertion point.
- IRBuilder<> ChkBuilder(Term);
- ChkBuilder.SetCurrentDebugLocation(Term->getDebugLoc());
- SCEVExpander Exp(*SE, "slp.rtcheck");
- Value *Cond = emitRuntimeAliasCheck(ChkBuilder, Exp);
-
- // Move the original terminator into the continuation block and replace it
- // with the guard branch.
- Term->removeFromParent();
- Term->insertInto(Tail, Tail->end());
- UncondBrInst::Create(Tail, VecBB);
- UncondBrInst::Create(Tail, ScalarBB);
- ChkBuilder.SetInsertPoint(BB);
- ChkBuilder.CreateCondBr(Cond, ScalarBB, VecBB);
-
- // The continuation block is the new predecessor of the original successors.
- for (BasicBlock *Succ : successors(Term))
- Succ->replacePhiUsesWith(BB, Tail);
-
- // Keep the dominator tree valid for the remainder of the run.
- if (DT) {
- DomTreeUpdater DTU(*DT, DomTreeUpdater::UpdateStrategy::Eager);
- SmallVector<DominatorTree::UpdateType> Updates = {
- {DominatorTree::Insert, BB, VecBB},
- {DominatorTree::Insert, BB, ScalarBB},
- {DominatorTree::Insert, VecBB, Tail},
- {DominatorTree::Insert, ScalarBB, Tail}};
- for (BasicBlock *Succ : successors(Term)) {
- Updates.push_back({DominatorTree::Insert, Tail, Succ});
- Updates.push_back({DominatorTree::Delete, BB, Succ});
- }
- DTU.applyUpdates(Updates);
- // Refresh DFS numbers: getLastInstructionInBundle() may use them to order
- // instructions across blocks for nodes emitted after versioning.
- DT->updateDFSNumbers();
- }
-
- // Record the checked base pairs for the fast-path block so that subsequent
- // vectorization there can reuse this guard instead of versioning again.
- SmallDenseSet<std::pair<const Value *, const Value *>, 4> &Covered =
- VersionedBlockCheckedPairs[VecBB];
- Covered.insert_range(RTChecks.BasePairs);
- // The scalar fallback must never be vectorized or versioned again.
- // TODO: check that scalar fallback does not miss vectorization opportunity
- // without runtime checks.
- ScalarFallbackBlocks.insert(ScalarBB);
-
- // The checks have been emitted; clear so any later vectorizeTree() in the
- // same attempt does not version again.
- RTChecks.clear();
- RTChecksFinalized = false;
- RTOrigBodyOrder.clear();
- // The CFG was modified; CFG analyses can no longer be preserved.
- CFGChanged = true;
-}
-
Value *BoUpSLP::vectorizeTree() {
ExtraValueToDebugLocsMap ExternallyUsedValues;
return vectorizeTree(ExternallyUsedValues);
@@ -25293,13 +24640,6 @@ Value *BoUpSLP::vectorizeTree(
(void)getLastInstructionInBundle(TE.get());
}
- // If this tree was scheduled by dropping may-alias memory dependencies in
- // favor of runtime alias checks, materialize the checks and version the
- // affected block before emitting the vector code. Scheduling has already
- // run, so moving the body into the fast-path block is safe, and the
- // dominator tree is updated for the rest of the run.
- versionBlocksForRuntimeChecks();
-
if (ReductionRoot)
Builder.SetInsertPoint(ReductionRoot->getParent(),
ReductionRoot->getIterator());
@@ -27202,69 +26542,23 @@ void BoUpSLP::BlockScheduling::calculateDependencies(
// the whole loop (even if the loop is fast, it's quadratic).
// It's important for the loop break condition (see below) to
// check this limit even between two read-only instructions.
- Instruction *DepInst = DepDest->getInst();
- bool MayConflict = SrcMayWrite || DepInst->mayWriteToMemory();
if (DistToSrc >= MaxMemDepDistance ||
- (MayConflict && (IsNonSimpleSrc || NumAliased >= AliasedCheckLimit ||
- SLP->isAliased(SrcLoc, SrcInst, DepInst)))) {
- // Try to turn a may-alias dependency between distinct, range-checkable
- // base objects into a runtime alias check. This covers both concrete
- // aliases and the conservative dependencies added once the
- // per-instruction alias-check limit is reached, so that a high alias
- // count cannot block a region that is otherwise versionable. The
- // dropped dependency is recorded so the final scheduleBlock()
- // recomputation keeps it dropped consistently.
- bool Dropped = false;
- if (MayConflict && SLPEnableRuntimeAliasChecks) {
- auto Key = std::make_pair(SrcInst, DepInst);
- // A dependency dropped during buildTree() must stay dropped on every
- // later recomputation (e.g. the final scheduleBlock()), independent
- // of the current mode, so the schedule remains consistent.
- if (IgnoredMemDeps.contains(Key)) {
- Dropped = true;
- } else if (SLP->isCoveredByExistingVersionCheck(BB, SrcInst,
- DepInst)) {
- // This block is the fast path of an earlier versioning whose
- // runtime check already proved these bases disjoint, so the
- // dependency can be dropped without emitting a new check.
- Dropped = true;
- } else if (SLP->isTryingRuntimeAliasChecks()) {
- if (SLP->isRuntimeCheckableAliasPair(SrcInst, DepInst) &&
- SLP->recordRuntimeAliasCheck(BB, SrcInst, DepInst)) {
- IgnoredMemDeps.insert(Key);
- Dropped = true;
- }
- } else if (!SLP->hasNonCheckableMemBlocker()) {
- // A dependency on a non-simple access (call, a volatile/atomic, or
- // anything without a single load/store pointer) cannot be
- // range-checked, so it is a hard blocker a retry cannot drop. A
- // simple runtime-checkable may-alias pair could instead be dropped
- // by a runtime check. Other simple pairs (e.g. same base) are
- // resolved statically and are not blockers.
- if (IsNonSimpleSrc || !getLoadStorePointerOperand(DepInst) ||
- !isSimple(DepInst))
- SLP->setHasNonCheckableMemBlocker(true);
- else if (!SLP->hasRuntimeCheckableBlockers() &&
- SLP->isRuntimeCheckableAliasPair(SrcInst, DepInst))
- SLP->setHasRuntimeCheckableBlockers(true);
- }
- }
+ ((SrcMayWrite || DepDest->getInst()->mayWriteToMemory()) &&
+ (IsNonSimpleSrc || NumAliased >= AliasedCheckLimit ||
+ SLP->isAliased(SrcLoc, SrcInst, DepDest->getInst())))) {
- if (!Dropped) {
- // Count only kept dependencies toward the alias-check limit. Counting
- // dropped ones could prematurely trigger the conservative
- // (limit-based) dependencies above and add spurious dependencies
- // (e.g. between non-overlapping stores to the same base) that would
- // block scheduling of an otherwise versionable region.
- NumAliased++;
- DepDest->addMemoryDependency(BundleMember);
- BundleMember->incDependencies();
- if (!DepDest->isScheduled())
- BundleMember->incrementUnscheduledDeps(1);
- if (!DepDest->hasValidDependencies() ||
- (InsertInReadyList && DepDest->isReady()))
- WorkList.push_back(DepDest);
- }
+ // We increment the counter only if the locations are aliased
+ // (instead of counting all alias checks). This gives a better
+ // balance between reduced runtime and accurate dependencies.
+ NumAliased++;
+
+ DepDest->addMemoryDependency(BundleMember);
+ BundleMember->incDependencies();
+ if (!DepDest->isScheduled())
+ BundleMember->incrementUnscheduledDeps(1);
+ if (!DepDest->hasValidDependencies() ||
+ (InsertInReadyList && DepDest->isReady()))
+ WorkList.push_back(DepDest);
}
// Example, explaining the loop break condition: Let's assume our
@@ -28487,13 +27781,7 @@ PreservedAnalyses SLPVectorizerPass::run(Function &F, FunctionAnalysisManager &A
return PreservedAnalyses::all();
PreservedAnalyses PA;
- // Runtime alias check versioning changes the CFG. In that case only the
- // dominator tree is kept up to date (via DomTreeUpdater); other CFG analyses
- // must be recomputed. Otherwise SLP leaves the CFG intact.
- if (CFGChanged)
- PA.preserve<DominatorTreeAnalysis>();
- else
- PA.preserveSet<CFGAnalyses>();
+ PA.preserveSet<CFGAnalyses>();
return PA;
}
@@ -28548,11 +27836,6 @@ bool SLPVectorizerPass::runImpl(Function &F, ScalarEvolution *SE_,
if (BB->isEHPad() || isa_and_nonnull<UnreachableInst>(BB->getTerminator()))
continue;
- // Skip scalar fallback blocks created by runtime alias check versioning:
- // they duplicate the original scalar code and must not be vectorized again.
- if (R.isScalarFallbackBlock(BB))
- continue;
-
// Start new block - clear the list of reduction roots.
R.clearReductionData();
collectSeedInstructions(BB);
@@ -28581,9 +27864,6 @@ bool SLPVectorizerPass::runImpl(Function &F, ScalarEvolution *SE_,
R.optimizeGatherSequence();
LLVM_DEBUG(dbgs() << "SLP: vectorized \"" << F.getName() << "\"\n");
}
- // Propagate whether runtime alias check versioning changed the CFG, so the
- // caller can drop CFG-analysis preservation only when necessary.
- CFGChanged = R.isCFGChanged();
return Changed;
}
@@ -28591,67 +27871,7 @@ std::optional<bool>
SLPVectorizerPass::vectorizeStoreChain(ArrayRef<Value *> Chain, BoUpSLP &R,
unsigned Idx, unsigned MinVF,
unsigned &Size) {
- std::optional<bool> Res = vectorizeStoreChainImpl(Chain, R, Idx, MinVF, Size);
- assert(!R.isTryingRuntimeAliasChecks() &&
- "Unexpected nested runtime alias check attempt");
- // Retry once with runtime alias checks, but only when the normal attempt
- // could not even schedule the bundle, i.e. it was blocked by
- // a memory dependency, and that dependency is runtime-checkable. If the
- // region also kept a non-checkable blocker, dropping the checkable deps
- // cannot unblock it.
- if (Res.has_value() || !SLPEnableRuntimeAliasChecks ||
- !R.hasRuntimeCheckableBlockers() || R.hasNonCheckableMemBlocker())
- return Res;
- // getTreeCost() unconditionally rejects a VF=2 tree whose vector
- // instruction count exceeds its scalar instruction count (the
- // SLPInstCountCheck heuristic), because the cost model is known to
- // underestimate shuffle/insert/extract overhead at that width. That check
- // returns an invalid cost that no threshold, including the runtime-checks
- // one, can outweigh. Retrying such a chain would pay the full tree rebuild
- // cost for the one width the cost model is least trusted for, on top of
- // the extra guard overhead, so skip it instead of wasting the attempt.
- if (Chain.size() == 2)
- return Res;
- // Only retry when scheduling blocked the chain (a store or its value bundle
- // could not be scheduled), which dropping may-alias deps can fix. A chain
- // gathered for structural reasons (e.g. non-consecutive addresses) would
- // gather again, so the second buildTree() would be pure overhead with no
- // vectorization benefit.
- Value *FirstStore = Chain.front();
- if (!R.isNotScheduled(FirstStore) &&
- !R.isNotScheduled(cast<StoreInst>(FirstStore)->getValueOperand()))
- return Res;
- BasicBlock *BB = cast<Instruction>(FirstStore)->getParent();
- // If an earlier optimistic attempt already failed for this block, do not
- // retry for other chains or smaller VFs in the same block.
- if (R.runtimeChecksFailedForBlock(BB))
- return Res;
- // The retry rebuilds the whole tree; also skip it when the chain's block can
- // never be versioned (e.g. it is inside a loop).
- if (!R.canVersionBlockForRuntimeChecks(BB)) {
- R.markRuntimeChecksFailedForBlock(BB);
- return Res;
- }
- R.setTryRuntimeAliasChecks(true);
- unsigned RTSize = Size;
- std::optional<bool> RTRes =
- vectorizeStoreChainImpl(Chain, R, Idx, MinVF, RTSize);
- R.setTryRuntimeAliasChecks(false);
- if (RTRes && *RTRes) {
- Size = RTSize;
- return RTRes;
- }
- // The versioning attempt failed for this block, skip the (expensive) retry.
- R.markRuntimeChecksFailedForBlock(BB);
- return Res;
-}
-
-std::optional<bool>
-SLPVectorizerPass::vectorizeStoreChainImpl(ArrayRef<Value *> Chain, BoUpSLP &R,
- unsigned Idx, unsigned MinVF,
- unsigned &Size) {
Size = 0;
- R.resetRuntimeAliasCheckState();
LLVM_DEBUG(dbgs() << "SLP: Analyzing a store chain of length " << Chain.size()
<< "\n");
const unsigned Sz = R.getVectorElementSize(Chain[0]);
@@ -28711,8 +27931,6 @@ SLPVectorizerPass::vectorizeStoreChainImpl(ArrayRef<Value *> Chain, BoUpSLP &R,
R.reorderTopToBottom();
R.reorderBottomToTop();
}
- if (R.isTryingRuntimeAliasChecks())
- R.captureRuntimeCheckBodySnapshot();
R.transformNodes();
R.computeMinimumValueSizes();
@@ -28724,32 +27942,6 @@ SLPVectorizerPass::vectorizeStoreChainImpl(ArrayRef<Value *> Chain, BoUpSLP &R,
Size = 2; // cut off masked gather small trees
InstructionCost Cost = R.getTreeCost(TreeCost);
- // When the tree was scheduled by dropping may-alias dependencies, account
- // for the runtime alias checks in the cost and make sure the region can
- // actually be versioned. If it cannot, abandon this attempt: the dropped
- // dependencies are unused because the tree is not vectorized.
- // The full check cost is charged against the vector-path benefit here; the
- // scalar-path overhead is bounded separately in canVersionForRuntimeChecks()
- // (a fraction of the scalar body cost).
- if (R.isTryingRuntimeAliasChecks() && R.hasRuntimeAliasChecks()) {
- // getRuntimeChecksCost() only ever sums non-negative per-instruction
- // throughput costs (address adds, compares, and/or, the guard branch), so
- // adding it can only push Cost up, never down, and an already-invalid
- // Cost (e.g. rejected by one of getTreeCost()'s own heuristics) stays
- // invalid either way. So if Cost already fails the threshold below, no
- // checks cost can rescue it: skip the (expensive) SCEV/dominance/escape
- // analysis in canVersionForRuntimeChecks() instead of running it on a
- // tree that is already rejected.
- if (Cost >= -SLPCostThreshold)
- return false;
- if (!R.canVersionForRuntimeChecks())
- return false;
- // TODO: Add probability-weighted model (benefit scaled by the expected
- // non-alias odds) would be more precise but needs
- // branch-probability/profile data.
- Cost += R.getRuntimeChecksCost();
- }
-
LLVM_DEBUG(dbgs() << "SLP: Found cost = " << Cost << " for VF=" << VF << "\n");
if (Cost < -SLPCostThreshold) {
LLVM_DEBUG(dbgs() << "SLP: Decided to vectorize cost = " << Cost << "\n");
diff --git a/llvm/test/Transforms/SLPVectorizer/AArch64/loadi8.ll b/llvm/test/Transforms/SLPVectorizer/AArch64/loadi8.ll
index 9812e5392f979..91a560f7782ac 100644
--- a/llvm/test/Transforms/SLPVectorizer/AArch64/loadi8.ll
+++ b/llvm/test/Transforms/SLPVectorizer/AArch64/loadi8.ll
@@ -88,53 +88,24 @@ entry:
define void @f_alias(ptr nocapture %dst, ptr nocapture readonly %src, ptr nocapture readonly %w) {
; CHECK-LABEL: @f_alias(
; CHECK-NEXT: entry:
-; CHECK-NEXT: [[DST2:%.*]] = ptrtoint ptr [[DST:%.*]] to i64
-; CHECK-NEXT: [[SRC1:%.*]] = ptrtoint ptr [[SRC:%.*]] to i64
-; CHECK-NEXT: [[TMP19:%.*]] = add i64 [[SRC1]], 4
-; CHECK-NEXT: [[TMP20:%.*]] = add i64 [[DST2]], 4
-; CHECK-NEXT: [[RT_BOUND0:%.*]] = icmp ult i64 [[DST2]], [[TMP19]]
-; CHECK-NEXT: [[RT_BOUND1:%.*]] = icmp ult i64 [[SRC1]], [[TMP20]]
-; CHECK-NEXT: [[RT_CONFLICT:%.*]] = and i1 [[RT_BOUND0]], [[RT_BOUND1]]
-; CHECK-NEXT: [[RT_GUARD:%.*]] = freeze i1 [[RT_CONFLICT]]
-; CHECK-NEXT: br i1 [[RT_GUARD]], label [[ENTRY_RTSCALAR:%.*]], label [[ENTRY_RTVEC:%.*]]
-; CHECK: entry.rtvec:
; CHECK-NEXT: [[TMP0:%.*]] = load i32, ptr [[W:%.*]], align 16
; CHECK-NEXT: [[OFFSET:%.*]] = getelementptr inbounds [[STRUCT_WEIGHT_T:%.*]], ptr [[W]], i64 0, i32 1
; CHECK-NEXT: [[TMP1:%.*]] = load i32, ptr [[OFFSET]], align 4
-; CHECK-NEXT: [[TMP21:%.*]] = load <4 x i8>, ptr [[SRC]], align 1
-; CHECK-NEXT: [[TMP22:%.*]] = zext <4 x i8> [[TMP21]] to <4 x i32>
-; CHECK-NEXT: [[TMP23:%.*]] = insertelement <4 x i32> poison, i32 [[TMP0]], i64 0
-; CHECK-NEXT: [[TMP24:%.*]] = shufflevector <4 x i32> [[TMP23]], <4 x i32> poison, <4 x i32> zeroinitializer
-; CHECK-NEXT: [[TMP25:%.*]] = mul nsw <4 x i32> [[TMP24]], [[TMP22]]
-; CHECK-NEXT: [[TMP26:%.*]] = insertelement <4 x i32> poison, i32 [[TMP1]], i64 0
-; CHECK-NEXT: [[TMP10:%.*]] = shufflevector <4 x i32> [[TMP26]], <4 x i32> poison, <4 x i32> zeroinitializer
-; CHECK-NEXT: [[TMP11:%.*]] = add nsw <4 x i32> [[TMP25]], [[TMP10]]
-; CHECK-NEXT: [[TMP12:%.*]] = icmp ult <4 x i32> [[TMP11]], splat (i32 256)
-; CHECK-NEXT: [[TMP13:%.*]] = icmp sgt <4 x i32> [[TMP11]], zeroinitializer
-; CHECK-NEXT: [[TMP14:%.*]] = sext <4 x i1> [[TMP13]] to <4 x i32>
-; CHECK-NEXT: [[TMP15:%.*]] = select <4 x i1> [[TMP12]], <4 x i32> [[TMP11]], <4 x i32> [[TMP14]]
-; CHECK-NEXT: [[TMP16:%.*]] = trunc <4 x i32> [[TMP15]] to <4 x i8>
-; CHECK-NEXT: store <4 x i8> [[TMP16]], ptr [[DST]], align 1
-; CHECK-NEXT: br label [[ENTRY_RTCONT:%.*]]
-; CHECK: entry.rtscalar:
-; CHECK-NEXT: [[TMP17:%.*]] = load i32, ptr [[W]], align 16
-; CHECK-NEXT: [[OFFSET_SCALAR:%.*]] = getelementptr inbounds [[STRUCT_WEIGHT_T]], ptr [[W]], i64 0, i32 1
-; CHECK-NEXT: [[TMP18:%.*]] = load i32, ptr [[OFFSET_SCALAR]], align 4
-; CHECK-NEXT: [[TMP2:%.*]] = load i8, ptr [[SRC]], align 1
+; CHECK-NEXT: [[TMP2:%.*]] = load i8, ptr [[SRC:%.*]], align 1
; CHECK-NEXT: [[CONV:%.*]] = zext i8 [[TMP2]] to i32
-; CHECK-NEXT: [[MUL_SCALAR:%.*]] = mul nsw i32 [[TMP17]], [[CONV]]
-; CHECK-NEXT: [[ADD:%.*]] = add nsw i32 [[MUL_SCALAR]], [[TMP18]]
+; CHECK-NEXT: [[MUL:%.*]] = mul nsw i32 [[TMP0]], [[CONV]]
+; CHECK-NEXT: [[ADD:%.*]] = add nsw i32 [[MUL]], [[TMP1]]
; CHECK-NEXT: [[TOBOOL_NOT_I:%.*]] = icmp ult i32 [[ADD]], 256
; CHECK-NEXT: [[TMP3:%.*]] = icmp sgt i32 [[ADD]], 0
; CHECK-NEXT: [[SHR_I:%.*]] = sext i1 [[TMP3]] to i32
; CHECK-NEXT: [[COND_I:%.*]] = select i1 [[TOBOOL_NOT_I]], i32 [[ADD]], i32 [[SHR_I]]
; CHECK-NEXT: [[CONV_I:%.*]] = trunc i32 [[COND_I]] to i8
-; CHECK-NEXT: store i8 [[CONV_I]], ptr [[DST]], align 1
+; CHECK-NEXT: store i8 [[CONV_I]], ptr [[DST:%.*]], align 1
; CHECK-NEXT: [[ARRAYIDX_1:%.*]] = getelementptr inbounds i8, ptr [[SRC]], i64 1
; CHECK-NEXT: [[TMP4:%.*]] = load i8, ptr [[ARRAYIDX_1]], align 1
; CHECK-NEXT: [[CONV_1:%.*]] = zext i8 [[TMP4]] to i32
-; CHECK-NEXT: [[MUL_1_SCALAR:%.*]] = mul nsw i32 [[TMP17]], [[CONV_1]]
-; CHECK-NEXT: [[ADD_1:%.*]] = add nsw i32 [[MUL_1_SCALAR]], [[TMP18]]
+; CHECK-NEXT: [[MUL_1:%.*]] = mul nsw i32 [[TMP0]], [[CONV_1]]
+; CHECK-NEXT: [[ADD_1:%.*]] = add nsw i32 [[MUL_1]], [[TMP1]]
; CHECK-NEXT: [[TOBOOL_NOT_I_1:%.*]] = icmp ult i32 [[ADD_1]], 256
; CHECK-NEXT: [[TMP5:%.*]] = icmp sgt i32 [[ADD_1]], 0
; CHECK-NEXT: [[SHR_I_1:%.*]] = sext i1 [[TMP5]] to i32
@@ -145,8 +116,8 @@ define void @f_alias(ptr nocapture %dst, ptr nocapture readonly %src, ptr nocapt
; CHECK-NEXT: [[ARRAYIDX_2:%.*]] = getelementptr inbounds i8, ptr [[SRC]], i64 2
; CHECK-NEXT: [[TMP6:%.*]] = load i8, ptr [[ARRAYIDX_2]], align 1
; CHECK-NEXT: [[CONV_2:%.*]] = zext i8 [[TMP6]] to i32
-; CHECK-NEXT: [[MUL_2_SCALAR:%.*]] = mul nsw i32 [[TMP17]], [[CONV_2]]
-; CHECK-NEXT: [[ADD_2:%.*]] = add nsw i32 [[MUL_2_SCALAR]], [[TMP18]]
+; CHECK-NEXT: [[MUL_2:%.*]] = mul nsw i32 [[TMP0]], [[CONV_2]]
+; CHECK-NEXT: [[ADD_2:%.*]] = add nsw i32 [[MUL_2]], [[TMP1]]
; CHECK-NEXT: [[TOBOOL_NOT_I_2:%.*]] = icmp ult i32 [[ADD_2]], 256
; CHECK-NEXT: [[TMP7:%.*]] = icmp sgt i32 [[ADD_2]], 0
; CHECK-NEXT: [[SHR_I_2:%.*]] = sext i1 [[TMP7]] to i32
@@ -157,8 +128,8 @@ define void @f_alias(ptr nocapture %dst, ptr nocapture readonly %src, ptr nocapt
; CHECK-NEXT: [[ARRAYIDX_3:%.*]] = getelementptr inbounds i8, ptr [[SRC]], i64 3
; CHECK-NEXT: [[TMP8:%.*]] = load i8, ptr [[ARRAYIDX_3]], align 1
; CHECK-NEXT: [[CONV_3:%.*]] = zext i8 [[TMP8]] to i32
-; CHECK-NEXT: [[MUL_3_SCALAR:%.*]] = mul nsw i32 [[TMP17]], [[CONV_3]]
-; CHECK-NEXT: [[ADD_3:%.*]] = add nsw i32 [[MUL_3_SCALAR]], [[TMP18]]
+; CHECK-NEXT: [[MUL_3:%.*]] = mul nsw i32 [[TMP0]], [[CONV_3]]
+; CHECK-NEXT: [[ADD_3:%.*]] = add nsw i32 [[MUL_3]], [[TMP1]]
; CHECK-NEXT: [[TOBOOL_NOT_I_3:%.*]] = icmp ult i32 [[ADD_3]], 256
; CHECK-NEXT: [[TMP9:%.*]] = icmp sgt i32 [[ADD_3]], 0
; CHECK-NEXT: [[SHR_I_3:%.*]] = sext i1 [[TMP9]] to i32
@@ -166,8 +137,6 @@ define void @f_alias(ptr nocapture %dst, ptr nocapture readonly %src, ptr nocapt
; CHECK-NEXT: [[CONV_I_3:%.*]] = trunc i32 [[COND_I_3]] to i8
; CHECK-NEXT: [[ARRAYIDX2_3:%.*]] = getelementptr inbounds i8, ptr [[DST]], i64 3
; CHECK-NEXT: store i8 [[CONV_I_3]], ptr [[ARRAYIDX2_3]], align 1
-; CHECK-NEXT: br label [[ENTRY_RTCONT]]
-; CHECK: entry.rtcont:
; CHECK-NEXT: ret void
;
entry:
diff --git a/llvm/test/Transforms/SLPVectorizer/X86/runtime-alias-checks.ll b/llvm/test/Transforms/SLPVectorizer/X86/runtime-alias-checks.ll
index d643a44dcf91a..bd3a867f93077 100644
--- a/llvm/test/Transforms/SLPVectorizer/X86/runtime-alias-checks.ll
+++ b/llvm/test/Transforms/SLPVectorizer/X86/runtime-alias-checks.ll
@@ -1,6 +1,5 @@
; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --version 6
; RUN: opt -passes=slp-vectorizer -mtriple=x86_64-unknown-linux-gnu -mcpu=skylake-avx512 -S < %s | FileCheck %s
-; RUN: opt -passes=slp-vectorizer -mtriple=x86_64-unknown-linux-gnu -mcpu=skylake-avx512 -slp-vectorize-with-runtime-alias-checks=false -S < %s | FileCheck %s --check-prefix=NOCHK
declare void @nodup() noduplicate
declare void @musttail_callee(ptr, ptr, ptr)
@@ -9,35 +8,6 @@ define void @test(ptr %dst, ptr %x, ptr %y) {
; CHECK-LABEL: define void @test(
; CHECK-SAME: ptr [[DST:%.*]], ptr [[X:%.*]], ptr [[Y:%.*]]) #[[ATTR1:[0-9]+]] {
; CHECK-NEXT: [[ENTRY:.*:]]
-; CHECK-NEXT: [[DST10:%.*]] = ptrtoint ptr [[DST]] to i64
-; CHECK-NEXT: [[Y9:%.*]] = ptrtoint ptr [[Y]] to i64
-; CHECK-NEXT: [[X8:%.*]] = ptrtoint ptr [[X]] to i64
-; CHECK-NEXT: [[TMP0:%.*]] = add i64 [[X8]], 64
-; CHECK-NEXT: [[TMP1:%.*]] = add i64 [[Y9]], 64
-; CHECK-NEXT: [[TMP2:%.*]] = add i64 [[DST10]], 64
-; CHECK-NEXT: [[RT_BOUND0:%.*]] = icmp ult i64 [[DST10]], [[TMP0]]
-; CHECK-NEXT: [[RT_BOUND1:%.*]] = icmp ult i64 [[X8]], [[TMP2]]
-; CHECK-NEXT: [[RT_CONFLICT:%.*]] = and i1 [[RT_BOUND0]], [[RT_BOUND1]]
-; CHECK-NEXT: [[RT_BOUND011:%.*]] = icmp ult i64 [[DST10]], [[TMP1]]
-; CHECK-NEXT: [[RT_BOUND112:%.*]] = icmp ult i64 [[Y9]], [[TMP2]]
-; CHECK-NEXT: [[RT_CONFLICT13:%.*]] = and i1 [[RT_BOUND011]], [[RT_BOUND112]]
-; CHECK-NEXT: [[RT_CONFLICT_ALL:%.*]] = or i1 [[RT_CONFLICT]], [[RT_CONFLICT13]]
-; CHECK-NEXT: [[RT_GUARD:%.*]] = freeze i1 [[RT_CONFLICT_ALL]]
-; CHECK-NEXT: br i1 [[RT_GUARD]], label %[[ENTRY_RTSCALAR:.*]], label %[[ENTRY_RTVEC:.*]]
-; CHECK: [[ENTRY_RTVEC]]:
-; CHECK-NEXT: [[TMP3:%.*]] = load <4 x double>, ptr [[X]], align 8
-; CHECK-NEXT: [[TMP4:%.*]] = load <4 x double>, ptr [[Y]], align 8
-; CHECK-NEXT: [[TMP5:%.*]] = fdiv <4 x double> [[TMP3]], [[TMP4]]
-; CHECK-NEXT: store <4 x double> [[TMP5]], ptr [[DST]], align 8
-; CHECK-NEXT: [[X4P1:%.*]] = getelementptr inbounds double, ptr [[X]], i64 4
-; CHECK-NEXT: [[Y4P1:%.*]] = getelementptr inbounds double, ptr [[Y]], i64 4
-; CHECK-NEXT: [[DST8:%.*]] = getelementptr inbounds double, ptr [[DST]], i64 4
-; CHECK-NEXT: [[TMP6:%.*]] = load <4 x double>, ptr [[X4P1]], align 8
-; CHECK-NEXT: [[TMP7:%.*]] = load <4 x double>, ptr [[Y4P1]], align 8
-; CHECK-NEXT: [[TMP8:%.*]] = fdiv <4 x double> [[TMP6]], [[TMP7]]
-; CHECK-NEXT: store <4 x double> [[TMP8]], ptr [[DST8]], align 8
-; CHECK-NEXT: br label %[[ENTRY_RTCONT:.*]]
-; CHECK: [[ENTRY_RTSCALAR]]:
; CHECK-NEXT: [[X0:%.*]] = load double, ptr [[X]], align 8
; CHECK-NEXT: [[Y0:%.*]] = load double, ptr [[Y]], align 8
; CHECK-NEXT: [[D0:%.*]] = fdiv double [[X0]], [[Y0]]
@@ -91,68 +61,8 @@ define void @test(ptr %dst, ptr %x, ptr %y) {
; CHECK-NEXT: [[D7:%.*]] = fdiv double [[X7]], [[Y7]]
; CHECK-NEXT: [[DST7:%.*]] = getelementptr inbounds double, ptr [[DST]], i64 7
; CHECK-NEXT: store double [[D7]], ptr [[DST7]], align 8
-; CHECK-NEXT: br label %[[ENTRY_RTCONT]]
-; CHECK: [[ENTRY_RTCONT]]:
; CHECK-NEXT: ret void
;
-; NOCHK-LABEL: define void @test(
-; NOCHK-SAME: ptr [[DST:%.*]], ptr [[X:%.*]], ptr [[Y:%.*]]) #[[ATTR1:[0-9]+]] {
-; NOCHK-NEXT: [[ENTRY:.*:]]
-; NOCHK-NEXT: [[X0:%.*]] = load double, ptr [[X]], align 8
-; NOCHK-NEXT: [[Y0:%.*]] = load double, ptr [[Y]], align 8
-; NOCHK-NEXT: [[D0:%.*]] = fdiv double [[X0]], [[Y0]]
-; NOCHK-NEXT: store double [[D0]], ptr [[DST]], align 8
-; NOCHK-NEXT: [[X1P:%.*]] = getelementptr inbounds double, ptr [[X]], i64 1
-; NOCHK-NEXT: [[X1:%.*]] = load double, ptr [[X1P]], align 8
-; NOCHK-NEXT: [[Y1P:%.*]] = getelementptr inbounds double, ptr [[Y]], i64 1
-; NOCHK-NEXT: [[Y1:%.*]] = load double, ptr [[Y1P]], align 8
-; NOCHK-NEXT: [[D1:%.*]] = fdiv double [[X1]], [[Y1]]
-; NOCHK-NEXT: [[DST1:%.*]] = getelementptr inbounds double, ptr [[DST]], i64 1
-; NOCHK-NEXT: store double [[D1]], ptr [[DST1]], align 8
-; NOCHK-NEXT: [[X2P:%.*]] = getelementptr inbounds double, ptr [[X]], i64 2
-; NOCHK-NEXT: [[X2:%.*]] = load double, ptr [[X2P]], align 8
-; NOCHK-NEXT: [[Y2P:%.*]] = getelementptr inbounds double, ptr [[Y]], i64 2
-; NOCHK-NEXT: [[Y2:%.*]] = load double, ptr [[Y2P]], align 8
-; NOCHK-NEXT: [[D2:%.*]] = fdiv double [[X2]], [[Y2]]
-; NOCHK-NEXT: [[DST2:%.*]] = getelementptr inbounds double, ptr [[DST]], i64 2
-; NOCHK-NEXT: store double [[D2]], ptr [[DST2]], align 8
-; NOCHK-NEXT: [[X3P:%.*]] = getelementptr inbounds double, ptr [[X]], i64 3
-; NOCHK-NEXT: [[X3:%.*]] = load double, ptr [[X3P]], align 8
-; NOCHK-NEXT: [[Y3P:%.*]] = getelementptr inbounds double, ptr [[Y]], i64 3
-; NOCHK-NEXT: [[Y3:%.*]] = load double, ptr [[Y3P]], align 8
-; NOCHK-NEXT: [[D3:%.*]] = fdiv double [[X3]], [[Y3]]
-; NOCHK-NEXT: [[DST3:%.*]] = getelementptr inbounds double, ptr [[DST]], i64 3
-; NOCHK-NEXT: store double [[D3]], ptr [[DST3]], align 8
-; NOCHK-NEXT: [[X4P:%.*]] = getelementptr inbounds double, ptr [[X]], i64 4
-; NOCHK-NEXT: [[X4:%.*]] = load double, ptr [[X4P]], align 8
-; NOCHK-NEXT: [[Y4P:%.*]] = getelementptr inbounds double, ptr [[Y]], i64 4
-; NOCHK-NEXT: [[Y4:%.*]] = load double, ptr [[Y4P]], align 8
-; NOCHK-NEXT: [[D4:%.*]] = fdiv double [[X4]], [[Y4]]
-; NOCHK-NEXT: [[DST4:%.*]] = getelementptr inbounds double, ptr [[DST]], i64 4
-; NOCHK-NEXT: store double [[D4]], ptr [[DST4]], align 8
-; NOCHK-NEXT: [[X5P:%.*]] = getelementptr inbounds double, ptr [[X]], i64 5
-; NOCHK-NEXT: [[X5:%.*]] = load double, ptr [[X5P]], align 8
-; NOCHK-NEXT: [[Y5P:%.*]] = getelementptr inbounds double, ptr [[Y]], i64 5
-; NOCHK-NEXT: [[Y5:%.*]] = load double, ptr [[Y5P]], align 8
-; NOCHK-NEXT: [[D5:%.*]] = fdiv double [[X5]], [[Y5]]
-; NOCHK-NEXT: [[DST5:%.*]] = getelementptr inbounds double, ptr [[DST]], i64 5
-; NOCHK-NEXT: store double [[D5]], ptr [[DST5]], align 8
-; NOCHK-NEXT: [[X6P:%.*]] = getelementptr inbounds double, ptr [[X]], i64 6
-; NOCHK-NEXT: [[X6:%.*]] = load double, ptr [[X6P]], align 8
-; NOCHK-NEXT: [[Y6P:%.*]] = getelementptr inbounds double, ptr [[Y]], i64 6
-; NOCHK-NEXT: [[Y6:%.*]] = load double, ptr [[Y6P]], align 8
-; NOCHK-NEXT: [[D6:%.*]] = fdiv double [[X6]], [[Y6]]
-; NOCHK-NEXT: [[DST6:%.*]] = getelementptr inbounds double, ptr [[DST]], i64 6
-; NOCHK-NEXT: store double [[D6]], ptr [[DST6]], align 8
-; NOCHK-NEXT: [[X7P:%.*]] = getelementptr inbounds double, ptr [[X]], i64 7
-; NOCHK-NEXT: [[X7:%.*]] = load double, ptr [[X7P]], align 8
-; NOCHK-NEXT: [[Y7P:%.*]] = getelementptr inbounds double, ptr [[Y]], i64 7
-; NOCHK-NEXT: [[Y7:%.*]] = load double, ptr [[Y7P]], align 8
-; NOCHK-NEXT: [[D7:%.*]] = fdiv double [[X7]], [[Y7]]
-; NOCHK-NEXT: [[DST7:%.*]] = getelementptr inbounds double, ptr [[DST]], i64 7
-; NOCHK-NEXT: store double [[D7]], ptr [[DST7]], align 8
-; NOCHK-NEXT: ret void
-;
entry:
%x0 = load double, ptr %x, align 8
%y0 = load double, ptr %y, align 8
@@ -270,65 +180,6 @@ define void @reject_noduplicate_call(ptr %dst, ptr %x, ptr %y) {
; CHECK-NEXT: call void @nodup()
; CHECK-NEXT: ret void
;
-; NOCHK-LABEL: define void @reject_noduplicate_call(
-; NOCHK-SAME: ptr [[DST:%.*]], ptr [[X:%.*]], ptr [[Y:%.*]]) #[[ATTR1]] {
-; NOCHK-NEXT: [[ENTRY:.*:]]
-; NOCHK-NEXT: [[X0:%.*]] = load double, ptr [[X]], align 8
-; NOCHK-NEXT: [[Y0:%.*]] = load double, ptr [[Y]], align 8
-; NOCHK-NEXT: [[D0:%.*]] = fdiv double [[X0]], [[Y0]]
-; NOCHK-NEXT: store double [[D0]], ptr [[DST]], align 8
-; NOCHK-NEXT: [[X1P:%.*]] = getelementptr inbounds double, ptr [[X]], i64 1
-; NOCHK-NEXT: [[X1:%.*]] = load double, ptr [[X1P]], align 8
-; NOCHK-NEXT: [[Y1P:%.*]] = getelementptr inbounds double, ptr [[Y]], i64 1
-; NOCHK-NEXT: [[Y1:%.*]] = load double, ptr [[Y1P]], align 8
-; NOCHK-NEXT: [[D1:%.*]] = fdiv double [[X1]], [[Y1]]
-; NOCHK-NEXT: [[DST1:%.*]] = getelementptr inbounds double, ptr [[DST]], i64 1
-; NOCHK-NEXT: store double [[D1]], ptr [[DST1]], align 8
-; NOCHK-NEXT: [[X2P:%.*]] = getelementptr inbounds double, ptr [[X]], i64 2
-; NOCHK-NEXT: [[X2:%.*]] = load double, ptr [[X2P]], align 8
-; NOCHK-NEXT: [[Y2P:%.*]] = getelementptr inbounds double, ptr [[Y]], i64 2
-; NOCHK-NEXT: [[Y2:%.*]] = load double, ptr [[Y2P]], align 8
-; NOCHK-NEXT: [[D2:%.*]] = fdiv double [[X2]], [[Y2]]
-; NOCHK-NEXT: [[DST2:%.*]] = getelementptr inbounds double, ptr [[DST]], i64 2
-; NOCHK-NEXT: store double [[D2]], ptr [[DST2]], align 8
-; NOCHK-NEXT: [[X3P:%.*]] = getelementptr inbounds double, ptr [[X]], i64 3
-; NOCHK-NEXT: [[X3:%.*]] = load double, ptr [[X3P]], align 8
-; NOCHK-NEXT: [[Y3P:%.*]] = getelementptr inbounds double, ptr [[Y]], i64 3
-; NOCHK-NEXT: [[Y3:%.*]] = load double, ptr [[Y3P]], align 8
-; NOCHK-NEXT: [[D3:%.*]] = fdiv double [[X3]], [[Y3]]
-; NOCHK-NEXT: [[DST3:%.*]] = getelementptr inbounds double, ptr [[DST]], i64 3
-; NOCHK-NEXT: store double [[D3]], ptr [[DST3]], align 8
-; NOCHK-NEXT: [[X4P:%.*]] = getelementptr inbounds double, ptr [[X]], i64 4
-; NOCHK-NEXT: [[X4:%.*]] = load double, ptr [[X4P]], align 8
-; NOCHK-NEXT: [[Y4P:%.*]] = getelementptr inbounds double, ptr [[Y]], i64 4
-; NOCHK-NEXT: [[Y4:%.*]] = load double, ptr [[Y4P]], align 8
-; NOCHK-NEXT: [[D4:%.*]] = fdiv double [[X4]], [[Y4]]
-; NOCHK-NEXT: [[DST4:%.*]] = getelementptr inbounds double, ptr [[DST]], i64 4
-; NOCHK-NEXT: store double [[D4]], ptr [[DST4]], align 8
-; NOCHK-NEXT: [[X5P:%.*]] = getelementptr inbounds double, ptr [[X]], i64 5
-; NOCHK-NEXT: [[X5:%.*]] = load double, ptr [[X5P]], align 8
-; NOCHK-NEXT: [[Y5P:%.*]] = getelementptr inbounds double, ptr [[Y]], i64 5
-; NOCHK-NEXT: [[Y5:%.*]] = load double, ptr [[Y5P]], align 8
-; NOCHK-NEXT: [[D5:%.*]] = fdiv double [[X5]], [[Y5]]
-; NOCHK-NEXT: [[DST5:%.*]] = getelementptr inbounds double, ptr [[DST]], i64 5
-; NOCHK-NEXT: store double [[D5]], ptr [[DST5]], align 8
-; NOCHK-NEXT: [[X6P:%.*]] = getelementptr inbounds double, ptr [[X]], i64 6
-; NOCHK-NEXT: [[X6:%.*]] = load double, ptr [[X6P]], align 8
-; NOCHK-NEXT: [[Y6P:%.*]] = getelementptr inbounds double, ptr [[Y]], i64 6
-; NOCHK-NEXT: [[Y6:%.*]] = load double, ptr [[Y6P]], align 8
-; NOCHK-NEXT: [[D6:%.*]] = fdiv double [[X6]], [[Y6]]
-; NOCHK-NEXT: [[DST6:%.*]] = getelementptr inbounds double, ptr [[DST]], i64 6
-; NOCHK-NEXT: store double [[D6]], ptr [[DST6]], align 8
-; NOCHK-NEXT: [[X7P:%.*]] = getelementptr inbounds double, ptr [[X]], i64 7
-; NOCHK-NEXT: [[X7:%.*]] = load double, ptr [[X7P]], align 8
-; NOCHK-NEXT: [[Y7P:%.*]] = getelementptr inbounds double, ptr [[Y]], i64 7
-; NOCHK-NEXT: [[Y7:%.*]] = load double, ptr [[Y7P]], align 8
-; NOCHK-NEXT: [[D7:%.*]] = fdiv double [[X7]], [[Y7]]
-; NOCHK-NEXT: [[DST7:%.*]] = getelementptr inbounds double, ptr [[DST]], i64 7
-; NOCHK-NEXT: store double [[D7]], ptr [[DST7]], align 8
-; NOCHK-NEXT: call void @nodup()
-; NOCHK-NEXT: ret void
-;
entry:
%x0 = load double, ptr %x, align 8
%y0 = load double, ptr %y, align 8
@@ -447,65 +298,6 @@ define void @reject_musttail(ptr %dst, ptr %x, ptr %y) {
; CHECK-NEXT: musttail call void @musttail_callee(ptr [[DST]], ptr [[X]], ptr [[Y]])
; CHECK-NEXT: ret void
;
-; NOCHK-LABEL: define void @reject_musttail(
-; NOCHK-SAME: ptr [[DST:%.*]], ptr [[X:%.*]], ptr [[Y:%.*]]) #[[ATTR1]] {
-; NOCHK-NEXT: [[ENTRY:.*:]]
-; NOCHK-NEXT: [[X0:%.*]] = load double, ptr [[X]], align 8
-; NOCHK-NEXT: [[Y0:%.*]] = load double, ptr [[Y]], align 8
-; NOCHK-NEXT: [[D0:%.*]] = fdiv double [[X0]], [[Y0]]
-; NOCHK-NEXT: store double [[D0]], ptr [[DST]], align 8
-; NOCHK-NEXT: [[X1P:%.*]] = getelementptr inbounds double, ptr [[X]], i64 1
-; NOCHK-NEXT: [[X1:%.*]] = load double, ptr [[X1P]], align 8
-; NOCHK-NEXT: [[Y1P:%.*]] = getelementptr inbounds double, ptr [[Y]], i64 1
-; NOCHK-NEXT: [[Y1:%.*]] = load double, ptr [[Y1P]], align 8
-; NOCHK-NEXT: [[D1:%.*]] = fdiv double [[X1]], [[Y1]]
-; NOCHK-NEXT: [[DST1:%.*]] = getelementptr inbounds double, ptr [[DST]], i64 1
-; NOCHK-NEXT: store double [[D1]], ptr [[DST1]], align 8
-; NOCHK-NEXT: [[X2P:%.*]] = getelementptr inbounds double, ptr [[X]], i64 2
-; NOCHK-NEXT: [[X2:%.*]] = load double, ptr [[X2P]], align 8
-; NOCHK-NEXT: [[Y2P:%.*]] = getelementptr inbounds double, ptr [[Y]], i64 2
-; NOCHK-NEXT: [[Y2:%.*]] = load double, ptr [[Y2P]], align 8
-; NOCHK-NEXT: [[D2:%.*]] = fdiv double [[X2]], [[Y2]]
-; NOCHK-NEXT: [[DST2:%.*]] = getelementptr inbounds double, ptr [[DST]], i64 2
-; NOCHK-NEXT: store double [[D2]], ptr [[DST2]], align 8
-; NOCHK-NEXT: [[X3P:%.*]] = getelementptr inbounds double, ptr [[X]], i64 3
-; NOCHK-NEXT: [[X3:%.*]] = load double, ptr [[X3P]], align 8
-; NOCHK-NEXT: [[Y3P:%.*]] = getelementptr inbounds double, ptr [[Y]], i64 3
-; NOCHK-NEXT: [[Y3:%.*]] = load double, ptr [[Y3P]], align 8
-; NOCHK-NEXT: [[D3:%.*]] = fdiv double [[X3]], [[Y3]]
-; NOCHK-NEXT: [[DST3:%.*]] = getelementptr inbounds double, ptr [[DST]], i64 3
-; NOCHK-NEXT: store double [[D3]], ptr [[DST3]], align 8
-; NOCHK-NEXT: [[X4P:%.*]] = getelementptr inbounds double, ptr [[X]], i64 4
-; NOCHK-NEXT: [[X4:%.*]] = load double, ptr [[X4P]], align 8
-; NOCHK-NEXT: [[Y4P:%.*]] = getelementptr inbounds double, ptr [[Y]], i64 4
-; NOCHK-NEXT: [[Y4:%.*]] = load double, ptr [[Y4P]], align 8
-; NOCHK-NEXT: [[D4:%.*]] = fdiv double [[X4]], [[Y4]]
-; NOCHK-NEXT: [[DST4:%.*]] = getelementptr inbounds double, ptr [[DST]], i64 4
-; NOCHK-NEXT: store double [[D4]], ptr [[DST4]], align 8
-; NOCHK-NEXT: [[X5P:%.*]] = getelementptr inbounds double, ptr [[X]], i64 5
-; NOCHK-NEXT: [[X5:%.*]] = load double, ptr [[X5P]], align 8
-; NOCHK-NEXT: [[Y5P:%.*]] = getelementptr inbounds double, ptr [[Y]], i64 5
-; NOCHK-NEXT: [[Y5:%.*]] = load double, ptr [[Y5P]], align 8
-; NOCHK-NEXT: [[D5:%.*]] = fdiv double [[X5]], [[Y5]]
-; NOCHK-NEXT: [[DST5:%.*]] = getelementptr inbounds double, ptr [[DST]], i64 5
-; NOCHK-NEXT: store double [[D5]], ptr [[DST5]], align 8
-; NOCHK-NEXT: [[X6P:%.*]] = getelementptr inbounds double, ptr [[X]], i64 6
-; NOCHK-NEXT: [[X6:%.*]] = load double, ptr [[X6P]], align 8
-; NOCHK-NEXT: [[Y6P:%.*]] = getelementptr inbounds double, ptr [[Y]], i64 6
-; NOCHK-NEXT: [[Y6:%.*]] = load double, ptr [[Y6P]], align 8
-; NOCHK-NEXT: [[D6:%.*]] = fdiv double [[X6]], [[Y6]]
-; NOCHK-NEXT: [[DST6:%.*]] = getelementptr inbounds double, ptr [[DST]], i64 6
-; NOCHK-NEXT: store double [[D6]], ptr [[DST6]], align 8
-; NOCHK-NEXT: [[X7P:%.*]] = getelementptr inbounds double, ptr [[X]], i64 7
-; NOCHK-NEXT: [[X7:%.*]] = load double, ptr [[X7P]], align 8
-; NOCHK-NEXT: [[Y7P:%.*]] = getelementptr inbounds double, ptr [[Y]], i64 7
-; NOCHK-NEXT: [[Y7:%.*]] = load double, ptr [[Y7P]], align 8
-; NOCHK-NEXT: [[D7:%.*]] = fdiv double [[X7]], [[Y7]]
-; NOCHK-NEXT: [[DST7:%.*]] = getelementptr inbounds double, ptr [[DST]], i64 7
-; NOCHK-NEXT: store double [[D7]], ptr [[DST7]], align 8
-; NOCHK-NEXT: musttail call void @musttail_callee(ptr [[DST]], ptr [[X]], ptr [[Y]])
-; NOCHK-NEXT: ret void
-;
entry:
%x0 = load double, ptr %x, align 8
%y0 = load double, ptr %y, align 8
@@ -614,55 +406,6 @@ define void @reject_outside_operand_vectorized(ptr %q, ptr %p) {
; CHECK-NEXT: store i32 [[Q4O]], ptr [[Q]], align 4
; CHECK-NEXT: ret void
;
-; NOCHK-LABEL: define void @reject_outside_operand_vectorized(
-; NOCHK-SAME: ptr [[Q:%.*]], ptr [[P:%.*]]) #[[ATTR1]] {
-; NOCHK-NEXT: [[ENTRY:.*]]:
-; NOCHK-NEXT: [[BUF:%.*]] = alloca [5 x i32], align 16
-; NOCHK-NEXT: br i1 true, label %[[PRE:.*]], label %[[BODY:.*]]
-; NOCHK: [[PRE]]:
-; NOCHK-NEXT: store i32 0, ptr [[BUF]], align 4
-; NOCHK-NEXT: [[T1:%.*]] = getelementptr i8, ptr [[BUF]], i64 4
-; NOCHK-NEXT: [[PRE1:%.*]] = load i32, ptr [[T1]], align 4
-; NOCHK-NEXT: [[T2:%.*]] = getelementptr i8, ptr [[BUF]], i64 8
-; NOCHK-NEXT: [[PRE2:%.*]] = load i32, ptr [[T2]], align 8
-; NOCHK-NEXT: br label %[[BODY]]
-; NOCHK: [[BODY]]:
-; NOCHK-NEXT: [[PH1:%.*]] = phi i32 [ [[PRE2]], %[[PRE]] ], [ 0, %[[ENTRY]] ]
-; NOCHK-NEXT: [[PH2:%.*]] = phi i32 [ [[PRE1]], %[[PRE]] ], [ 0, %[[ENTRY]] ]
-; NOCHK-NEXT: store i32 0, ptr null, align 4
-; NOCHK-NEXT: [[Q0:%.*]] = load i32, ptr [[Q]], align 4
-; NOCHK-NEXT: [[Q0O:%.*]] = or i32 [[Q0]], 1
-; NOCHK-NEXT: store i32 [[Q0O]], ptr [[Q]], align 4
-; NOCHK-NEXT: [[P4:%.*]] = getelementptr i8, ptr [[P]], i64 4
-; NOCHK-NEXT: [[L4:%.*]] = load i32, ptr [[P4]], align 4
-; NOCHK-NEXT: [[O4:%.*]] = or i32 [[PH2]], [[L4]]
-; NOCHK-NEXT: store i32 [[O4]], ptr [[P4]], align 4
-; NOCHK-NEXT: [[Q1:%.*]] = load i32, ptr [[Q]], align 4
-; NOCHK-NEXT: [[Q1O:%.*]] = or i32 [[Q1]], 1
-; NOCHK-NEXT: store i32 [[Q1O]], ptr [[Q]], align 4
-; NOCHK-NEXT: [[P8:%.*]] = getelementptr i8, ptr [[P]], i64 8
-; NOCHK-NEXT: [[L8:%.*]] = load i32, ptr [[P8]], align 4
-; NOCHK-NEXT: [[O8:%.*]] = or i32 [[PH1]], [[L8]]
-; NOCHK-NEXT: store i32 [[O8]], ptr [[P8]], align 4
-; NOCHK-NEXT: [[Q2:%.*]] = load i32, ptr [[Q]], align 4
-; NOCHK-NEXT: [[Q2O:%.*]] = or i32 [[Q2]], 1
-; NOCHK-NEXT: store i32 [[Q2O]], ptr [[Q]], align 4
-; NOCHK-NEXT: [[P12:%.*]] = getelementptr i8, ptr [[P]], i64 12
-; NOCHK-NEXT: [[L12:%.*]] = load i32, ptr [[P12]], align 4
-; NOCHK-NEXT: [[O12:%.*]] = or i32 [[L12]], 1
-; NOCHK-NEXT: store i32 [[O12]], ptr [[P12]], align 4
-; NOCHK-NEXT: [[Q3:%.*]] = load i32, ptr [[Q]], align 4
-; NOCHK-NEXT: [[Q3O:%.*]] = or i32 [[Q3]], 1
-; NOCHK-NEXT: store i32 [[Q3O]], ptr [[Q]], align 4
-; NOCHK-NEXT: [[P16:%.*]] = getelementptr i8, ptr [[P]], i64 16
-; NOCHK-NEXT: [[L16:%.*]] = load i32, ptr [[P16]], align 4
-; NOCHK-NEXT: [[O16:%.*]] = or i32 [[L16]], 1
-; NOCHK-NEXT: store i32 [[O16]], ptr [[P]], align 4
-; NOCHK-NEXT: [[Q4:%.*]] = load i32, ptr [[Q]], align 4
-; NOCHK-NEXT: [[Q4O:%.*]] = or i32 [[Q4]], 1
-; NOCHK-NEXT: store i32 [[Q4O]], ptr [[Q]], align 4
-; NOCHK-NEXT: ret void
-;
entry:
%buf = alloca [5 x i32], align 16
br i1 true, label %pre, label %body
@@ -772,64 +515,6 @@ define void @reject_optsize(ptr %dst, ptr %x, ptr %y) optsize {
; CHECK-NEXT: store double [[D7]], ptr [[DST7]], align 8
; CHECK-NEXT: ret void
;
-; NOCHK-LABEL: define void @reject_optsize(
-; NOCHK-SAME: ptr [[DST:%.*]], ptr [[X:%.*]], ptr [[Y:%.*]]) #[[ATTR2:[0-9]+]] {
-; NOCHK-NEXT: [[ENTRY:.*:]]
-; NOCHK-NEXT: [[X0:%.*]] = load double, ptr [[X]], align 8
-; NOCHK-NEXT: [[Y0:%.*]] = load double, ptr [[Y]], align 8
-; NOCHK-NEXT: [[D0:%.*]] = fdiv double [[X0]], [[Y0]]
-; NOCHK-NEXT: store double [[D0]], ptr [[DST]], align 8
-; NOCHK-NEXT: [[X1P:%.*]] = getelementptr inbounds double, ptr [[X]], i64 1
-; NOCHK-NEXT: [[X1:%.*]] = load double, ptr [[X1P]], align 8
-; NOCHK-NEXT: [[Y1P:%.*]] = getelementptr inbounds double, ptr [[Y]], i64 1
-; NOCHK-NEXT: [[Y1:%.*]] = load double, ptr [[Y1P]], align 8
-; NOCHK-NEXT: [[D1:%.*]] = fdiv double [[X1]], [[Y1]]
-; NOCHK-NEXT: [[DST1:%.*]] = getelementptr inbounds double, ptr [[DST]], i64 1
-; NOCHK-NEXT: store double [[D1]], ptr [[DST1]], align 8
-; NOCHK-NEXT: [[X2P:%.*]] = getelementptr inbounds double, ptr [[X]], i64 2
-; NOCHK-NEXT: [[X2:%.*]] = load double, ptr [[X2P]], align 8
-; NOCHK-NEXT: [[Y2P:%.*]] = getelementptr inbounds double, ptr [[Y]], i64 2
-; NOCHK-NEXT: [[Y2:%.*]] = load double, ptr [[Y2P]], align 8
-; NOCHK-NEXT: [[D2:%.*]] = fdiv double [[X2]], [[Y2]]
-; NOCHK-NEXT: [[DST2:%.*]] = getelementptr inbounds double, ptr [[DST]], i64 2
-; NOCHK-NEXT: store double [[D2]], ptr [[DST2]], align 8
-; NOCHK-NEXT: [[X3P:%.*]] = getelementptr inbounds double, ptr [[X]], i64 3
-; NOCHK-NEXT: [[X3:%.*]] = load double, ptr [[X3P]], align 8
-; NOCHK-NEXT: [[Y3P:%.*]] = getelementptr inbounds double, ptr [[Y]], i64 3
-; NOCHK-NEXT: [[Y3:%.*]] = load double, ptr [[Y3P]], align 8
-; NOCHK-NEXT: [[D3:%.*]] = fdiv double [[X3]], [[Y3]]
-; NOCHK-NEXT: [[DST3:%.*]] = getelementptr inbounds double, ptr [[DST]], i64 3
-; NOCHK-NEXT: store double [[D3]], ptr [[DST3]], align 8
-; NOCHK-NEXT: [[X4P:%.*]] = getelementptr inbounds double, ptr [[X]], i64 4
-; NOCHK-NEXT: [[X4:%.*]] = load double, ptr [[X4P]], align 8
-; NOCHK-NEXT: [[Y4P:%.*]] = getelementptr inbounds double, ptr [[Y]], i64 4
-; NOCHK-NEXT: [[Y4:%.*]] = load double, ptr [[Y4P]], align 8
-; NOCHK-NEXT: [[D4:%.*]] = fdiv double [[X4]], [[Y4]]
-; NOCHK-NEXT: [[DST4:%.*]] = getelementptr inbounds double, ptr [[DST]], i64 4
-; NOCHK-NEXT: store double [[D4]], ptr [[DST4]], align 8
-; NOCHK-NEXT: [[X5P:%.*]] = getelementptr inbounds double, ptr [[X]], i64 5
-; NOCHK-NEXT: [[X5:%.*]] = load double, ptr [[X5P]], align 8
-; NOCHK-NEXT: [[Y5P:%.*]] = getelementptr inbounds double, ptr [[Y]], i64 5
-; NOCHK-NEXT: [[Y5:%.*]] = load double, ptr [[Y5P]], align 8
-; NOCHK-NEXT: [[D5:%.*]] = fdiv double [[X5]], [[Y5]]
-; NOCHK-NEXT: [[DST5:%.*]] = getelementptr inbounds double, ptr [[DST]], i64 5
-; NOCHK-NEXT: store double [[D5]], ptr [[DST5]], align 8
-; NOCHK-NEXT: [[X6P:%.*]] = getelementptr inbounds double, ptr [[X]], i64 6
-; NOCHK-NEXT: [[X6:%.*]] = load double, ptr [[X6P]], align 8
-; NOCHK-NEXT: [[Y6P:%.*]] = getelementptr inbounds double, ptr [[Y]], i64 6
-; NOCHK-NEXT: [[Y6:%.*]] = load double, ptr [[Y6P]], align 8
-; NOCHK-NEXT: [[D6:%.*]] = fdiv double [[X6]], [[Y6]]
-; NOCHK-NEXT: [[DST6:%.*]] = getelementptr inbounds double, ptr [[DST]], i64 6
-; NOCHK-NEXT: store double [[D6]], ptr [[DST6]], align 8
-; NOCHK-NEXT: [[X7P:%.*]] = getelementptr inbounds double, ptr [[X]], i64 7
-; NOCHK-NEXT: [[X7:%.*]] = load double, ptr [[X7P]], align 8
-; NOCHK-NEXT: [[Y7P:%.*]] = getelementptr inbounds double, ptr [[Y]], i64 7
-; NOCHK-NEXT: [[Y7:%.*]] = load double, ptr [[Y7P]], align 8
-; NOCHK-NEXT: [[D7:%.*]] = fdiv double [[X7]], [[Y7]]
-; NOCHK-NEXT: [[DST7:%.*]] = getelementptr inbounds double, ptr [[DST]], i64 7
-; NOCHK-NEXT: store double [[D7]], ptr [[DST7]], align 8
-; NOCHK-NEXT: ret void
-;
entry:
%x0 = load double, ptr %x, align 8
%y0 = load double, ptr %y, align 8
More information about the llvm-commits
mailing list