[llvm] [WIP][SLP] Allow external uses to exist as both an extract and a rematerialization during the pass (PR #211680)
Ryan Buchner via llvm-commits
llvm-commits at lists.llvm.org
Fri Aug 7 18:00:01 PDT 2026
https://github.com/bababuck updated https://github.com/llvm/llvm-project/pull/211680
>From 2094b9aca9ef4d23d14cd675a98d46943a3c07fe Mon Sep 17 00:00:00 2001
From: Ryan Buchner <buchner.ryan at gmail.com>
Date: Tue, 21 Jul 2026 10:35:38 -0700
Subject: [PATCH 01/22] [SLP] Allow external uses to exist as both an extract
and a rematerialization during the pass
Will allow better pattern matching and more aggresive rematerialization.
---
.../Transforms/Vectorize/SLPVectorizer.cpp | 498 ++++++++++++--
.../SLPVectorizer/AArch64/getelementptr2.ll | 4 +-
.../AArch64/multiple_reduction.ll | 637 +-----------------
.../RISCV/buildvector-all-external-scalars.ll | 46 +-
.../SLPVectorizer/RISCV/external.ll | 98 +++
.../RISCV/mixed-extracts-types.ll | 8 +-
.../RISCV/runtime-strided-stores.ll | 252 ++++---
.../strided-loads-with-external-use-ptr.ll | 8 +-
.../Transforms/SLPVectorizer/X86/PR35628_1.ll | 6 +-
.../test/Transforms/SLPVectorizer/X86/call.ll | 17 +-
.../X86/entries-shuffled-diff-sizes.ll | 10 +-
.../X86/extractelement-multi-register-use.ll | 10 +-
.../X86/extractelemets-extended-by-poison.ll | 4 +-
.../X86/gathered-shuffle-resized.ll | 14 +-
.../SLPVectorizer/X86/horizontal-list.ll | 17 +-
.../SLPVectorizer/X86/horizontal-minmax.ll | 25 +-
.../X86/split-node-last-inst-vectorized.ll | 2 +-
.../X86/split-vector-operand-with-reuses.ll | 24 +-
18 files changed, 810 insertions(+), 870 deletions(-)
create mode 100644 llvm/test/Transforms/SLPVectorizer/RISCV/external.ll
diff --git a/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp b/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp
index d3e491247f93f..99eb20ae9e3e8 100644
--- a/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp
+++ b/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp
@@ -883,16 +883,23 @@ static SmallBitVector isUndefVector(const Value *V,
/// %2 = mul <4 x i8> %1, %1
/// ret <4 x i8> %2
/// Mask will return the Shuffle Mask equivalent to the extracted elements.
+/// CouldBeExtract maps deferred scalar forms back to their corresponding
+/// extractelement candidates.
/// TODO: Can we split off and reuse the shuffle mask detection from
/// ShuffleVectorInst/getShuffleCost?
-static std::optional<TargetTransformInfo::ShuffleKind>
-isFixedVectorShuffle(ArrayRef<Value *> VL, SmallVectorImpl<int> &Mask,
- AssumptionCache *AC) {
- const auto *It = find_if(VL, IsaPred<ExtractElementInst>);
+static std::optional<TargetTransformInfo::ShuffleKind> isFixedVectorShuffle(
+ ArrayRef<Value *> VL, SmallVectorImpl<int> &Mask, AssumptionCache *AC,
+ const DenseMap<Value *, ExtractElementInst *> &CouldBeExtract) {
+ auto GetExtract = [&CouldBeExtract](Value *V) -> ExtractElementInst * {
+ if (auto *EI = dyn_cast<ExtractElementInst>(V))
+ return EI;
+ return CouldBeExtract.lookup(V);
+ };
+ const auto *It = find_if(VL, [&](Value *V) { return GetExtract(V); });
if (It == VL.end())
return std::nullopt;
- unsigned Size = accumulate(VL, 0u, [](unsigned S, Value *V) {
- auto *EI = dyn_cast<ExtractElementInst>(V);
+ unsigned Size = accumulate(VL, 0u, [&](unsigned S, Value *V) {
+ auto *EI = GetExtract(V);
if (!EI)
return S;
auto *VTy = dyn_cast<FixedVectorType>(EI->getVectorOperandType());
@@ -904,7 +911,7 @@ isFixedVectorShuffle(ArrayRef<Value *> VL, SmallVectorImpl<int> &Mask,
Value *Vec1 = nullptr;
Value *Vec2 = nullptr;
bool HasNonUndefVec = any_of(VL, [&](Value *V) {
- auto *EE = dyn_cast<ExtractElementInst>(V);
+ auto *EE = GetExtract(V);
if (!EE)
return false;
Value *Vec = EE->getVectorOperand();
@@ -919,7 +926,10 @@ isFixedVectorShuffle(ArrayRef<Value *> VL, SmallVectorImpl<int> &Mask,
// Undef can be represented as an undef element in a vector.
if (isa<UndefValue>(VL[I]))
continue;
- auto *EI = cast<ExtractElementInst>(VL[I]);
+ auto *EI = GetExtract(VL[I]);
+ if (!EI)
+ return std::nullopt;
+
if (isa<ScalableVectorType>(EI->getVectorOperandType()))
return std::nullopt;
auto *Vec = EI->getVectorOperand();
@@ -1021,6 +1031,22 @@ static bool isValidForAlternation(unsigned Opcode) {
return !Instruction::isIntDivRem(Opcode);
}
+/// \returns true if \p Scalar can stay rematerialized during vectorization and
+/// be switched to an extractelement later.
+/// For now, only support load and load-cast pairs since those are simpler to
+/// handle and are commonly profitable to rematerialize
+static bool isDeferredExtractable(Value *Scalar) {
+ if (isa<VectorType>(Scalar->getType()))
+ return false;
+ if (isa<LoadInst>(Scalar))
+ return true;
+ if (auto *CI = dyn_cast<CastInst>(Scalar)) {
+ auto *LI = dyn_cast<LoadInst>(CI->getOperand(0));
+ return LI && LI->hasOneUse();
+ }
+ return false;
+}
+
namespace {
/// Helper class that determines VL can use the same opcode.
@@ -2304,6 +2330,9 @@ class slpvectorizer::BoUpSLP {
CompressEntryToData.clear();
ExternalUses.clear();
ExternalUsesAsOriginalScalar.clear();
+ ExternalUsesAsExtract.clear();
+ ExternalUsesAsRematCostTmp.clear();
+ ExternalUsesAsExtractCostTmp.clear();
ExternalUsesWithNonUsers.clear();
for (auto &Iter : BlocksSchedules) {
BlockScheduling *BS = Iter.second.get();
@@ -3782,6 +3811,26 @@ class slpvectorizer::BoUpSLP {
/// is delayed until BoUpSLP is destructed.
void eraseInstruction(Instruction *I) {
DeletedInstructions.insert(I);
+ if (auto It = DeferredScalarsToExtract.find(I);
+ It != DeferredScalarsToExtract.end()) {
+ SmallPtrSet<Instruction *, 2> ProcessedExtracts;
+ for (DeferredExtractType &DET : It->getSecond()) {
+ auto *E = cast<Instruction>(DET.NewInst);
+ if (!ProcessedExtracts.insert(E).second)
+ continue;
+ bool LiveUsers = false;
+ for (Use &U : E->uses())
+ if (!isDeleted(cast<Instruction>(U.getUser()))) {
+ LiveUsers = true;
+ break;
+ }
+ if (!LiveUsers) {
+ LLVM_DEBUG(dbgs() << "SLP: \tErasing scalar:" << *E << ".\n");
+ eraseInstruction(E);
+ }
+ }
+ }
+ DeferredScalarsToExtract.erase(I);
}
/// Remove instructions from the parent function and clear the operands of \p
@@ -3931,6 +3980,17 @@ class slpvectorizer::BoUpSLP {
SmallVectorImpl<Value *> &Op2,
OrdersType &ReorderIndices) const;
+ // Create ExtractElement instructions that we deferred creating earlier
+ // to allow for better vectorization of chains using those values
+ void emitDeferredExtracts();
+
+ const DenseMap<Value *, ExtractElementInst *> &getCouldBeExtract() const {
+ return CouldBeExtract;
+ }
+ const DenseMap<Value *, Instruction *> &getCouldBeRemat() const {
+ return CouldBeRemat;
+ }
+
~BoUpSLP();
private:
@@ -4055,6 +4115,20 @@ class slpvectorizer::BoUpSLP {
/// Vectorize a single entry in the tree.
Value *vectorizeTree(TreeEntry *E);
+ struct DeferredExtractType {
+ Value *Scalar;
+ Value *NewInst;
+ llvm::User *User;
+ DeferredExtractType(Value *Scalar, Value *NewInst, llvm::User *User)
+ : Scalar(Scalar), NewInst(NewInst), User(User) {}
+ };
+
+ /// Track external uses that are more profitable as extracts
+ /// We rematerialize the value initially, but on cleanup
+ /// we need to swap back in the extract instruction
+ DenseMap<Value *, SmallVector<DeferredExtractType, 2>>
+ DeferredScalarsToExtract;
+
/// Vectorize a single entry in the tree, the \p Idx-th operand of the entry
/// \p E.
Value *vectorizeOperand(TreeEntry *E, unsigned NodeIdx);
@@ -4733,7 +4807,33 @@ class slpvectorizer::BoUpSLP {
ReuseShuffleIndices.end());
if (ReorderIndices.empty()) {
Last->Scalars.assign(VL.begin(), VL.end());
- if (S)
+
+ /// In a gather node, any rematerialized values ought to be treated as
+ /// extractelements instead
+ /// Update the parent nodes operands to match
+ bool ReplacedByExtractCandidate = false;
+ if (EntryState == TreeEntry::NeedToGather) {
+ for (auto &VPtr : Last->Scalars)
+ if (auto *NewVPtr = CouldBeExtract.lookup(VPtr)) {
+ VPtr = NewVPtr;
+ ReplacedByExtractCandidate = true;
+ }
+ if (ReplacedByExtractCandidate) {
+ LoadEntriesToVectorize.remove(Last->Idx);
+ ScalarsVectorizationLegality Legality =
+ getScalarsVectorizationLegality(Last->Scalars, 0, UserTreeIdx);
+ InstructionsState S = Legality.getInstructionsState();
+ if (S)
+ Last->setOperations(S);
+ if (UserTreeIdx.UserTE)
+ for (auto &VPtr :
+ UserTreeIdx.UserTE->getOperand(UserTreeIdx.EdgeIdx))
+ if (auto *NewVPtr = CouldBeExtract.lookup(VPtr))
+ VPtr = NewVPtr;
+ }
+ }
+
+ if (S && !ReplacedByExtractCandidate)
Last->setOperations(S);
} else {
// Reorder scalars and build final mask.
@@ -4753,7 +4853,7 @@ class slpvectorizer::BoUpSLP {
assert(S && "Split nodes must have operations.");
Last->setOperations(S);
SmallPtrSet<Value *, 4> Processed;
- for (Value *V : VL) {
+ for (Value *V : Last->Scalars) {
auto *I = dyn_cast<Instruction>(V);
if (!I)
continue;
@@ -4771,11 +4871,12 @@ class slpvectorizer::BoUpSLP {
if (isa<PHINode>(S.getMainOp()) ||
isVectorLikeInstWithConstOps(S.getMainOp()) ||
(!S.areInstructionsWithCopyableElements() &&
- doesNotNeedToSchedule(VL)) ||
- all_of(VL, [&](Value *V) { return S.isNonSchedulable(V); }))
+ doesNotNeedToSchedule(Last->Scalars)) ||
+ all_of(Last->Scalars,
+ [&](Value *V) { return S.isNonSchedulable(V); }))
Last->setDoesNotNeedToSchedule();
SmallPtrSet<Value *, 4> Processed;
- for (Value *V : VL) {
+ for (Value *V : Last->Scalars) {
if (isa<PoisonValue>(V))
continue;
if (S.isCopyableElement(V)) {
@@ -4799,7 +4900,7 @@ class slpvectorizer::BoUpSLP {
#if !defined(NDEBUG) || defined(EXPENSIVE_CHECKS)
auto *BundleMember = Bundle.getBundle().begin();
SmallPtrSet<Value *, 4> Processed;
- for (Value *V : VL) {
+ for (Value *V : Last->Scalars) {
if (S.isNonSchedulable(V) || !Processed.insert(V).second)
continue;
++BundleMember;
@@ -4812,7 +4913,7 @@ class slpvectorizer::BoUpSLP {
} else {
// Build a map for gathered scalars to the nodes where they are used.
bool AllConstsOrCasts = true;
- for (Value *V : VL) {
+ for (Value *V : Last->Scalars) {
if (S && S.areInstructionsWithCopyableElements() &&
S.isCopyableElement(V))
Last->addCopyableElement(V);
@@ -4827,7 +4928,7 @@ class slpvectorizer::BoUpSLP {
if (AllConstsOrCasts)
CastMaxMinBWSizes =
std::make_pair(std::numeric_limits<unsigned>::max(), 1);
- MustGather.insert_range(VL);
+ MustGather.insert_range(Last->Scalars);
}
if (UserTreeIdx.UserTE)
@@ -5069,11 +5170,29 @@ class slpvectorizer::BoUpSLP {
/// after vectorization.
UserList ExternalUses;
- /// A list of GEPs which can be reaplced by scalar GEPs instead of
- /// extractelement instructions.
+ /// A list of scalars that can be used as scalars first instead of immediate
+ /// extractelement materialization.
SmallPtrSet<Value *, 4> ExternalUsesAsOriginalScalar;
- /// A list of scalar to be extracted without specific user necause of too many
+ /// Map a scalar rematerialized form to the matching extractelement.
+ DenseMap<Value *, ExtractElementInst *> CouldBeExtract;
+ /// Reverse mapping used by scheduling: extractelement back to remat scalar.
+ DenseMap<Value *, Instruction *> CouldBeRemat;
+
+ /// Cases where extraction is estimated as more profitable but want to delay
+ /// extraction to allow for better vectorization in the interim. These values
+ /// are converted to extracts in a late cleanup step after primary SLP
+ /// rewriting.
+ SmallPtrSet<Value *, 4> ExternalUsesAsExtract;
+ /// Per-tree extract profitability cost for ExternalUsesAsExtract scalars.
+ DenseMap<const Value *, InstructionCost> ExternalUsesAsExtractCost;
+ DenseMap<const Value *, InstructionCost> ExternalUsesAsRematCost;
+ /// Cache the costs for the current tree since we may not keep around these
+ /// extracts if the tree is rejected
+ DenseMap<const Value *, InstructionCost> ExternalUsesAsExtractCostTmp;
+ DenseMap<const Value *, InstructionCost> ExternalUsesAsRematCostTmp;
+
+ /// A list of scalar to be extracted without specific user because of too many
/// uses.
SmallPtrSet<Value *, 4> ExternalUsesWithNonUsers;
@@ -5606,8 +5725,8 @@ class slpvectorizer::BoUpSLP {
/// extractelements/insertelements only or nodes with instructions, with
/// uses/operands outside of the block.
struct BlockScheduling {
- BlockScheduling(BasicBlock *BB)
- : BB(BB), ChunkSize(BB->size()), ChunkPos(ChunkSize) {}
+ BlockScheduling(BasicBlock *BB, BoUpSLP &R)
+ : BB(BB), ChunkSize(BB->size()), ChunkPos(ChunkSize), R(R) {}
void clear() {
ScheduledBundles.clear();
@@ -6005,7 +6124,7 @@ class slpvectorizer::BoUpSLP {
// Copyable data is used only once (uses itself).
TotalOpCount = OperandsUses[In] = 1;
} else {
- for (const Use &U : In->operands()) {
+ auto HandleOneOp = [&](const Use &U) {
if (auto *I = dyn_cast<Instruction>(U.get())) {
auto Res = OperandsUses.try_emplace(I, 0);
unsigned ExtraDeps = 1;
@@ -6021,7 +6140,15 @@ class slpvectorizer::BoUpSLP {
Res.first->getSecond() += ExtraDeps;
TotalOpCount += ExtraDeps;
}
- }
+ };
+ for (const Use &U : In->operands())
+ HandleOneOp(U);
+ // Track the operands from the extractelement copy
+ // as well to make sure the dependency on the vector
+ // is tracked
+ if (auto *EI = R.getCouldBeExtract().lookup(In))
+ for (const Use &U : EI->operands())
+ HandleOneOp(U);
}
// Decrement the unscheduled counter and insert to ready list if
// ready.
@@ -6121,6 +6248,10 @@ class slpvectorizer::BoUpSLP {
Bundle->getTreeEntry()->getOperand(OpIdx)[Lane])) {
LLVM_DEBUG(dbgs() << "SLP: check for readiness (def): "
<< *I << "\n");
+ // The scheduling node works on the rematerialize version
+ // of the extract
+ if (auto *RI = R.getCouldBeRemat().lookup(I))
+ I = RI;
DecrUnschedForInst(
I, Bundle->getTreeEntry(), OpIdx, Checked,
Bundle->getTreeEntry()->isExpandedOperand(In, OpIdx));
@@ -6396,6 +6527,9 @@ class slpvectorizer::BoUpSLP {
/// of ScheduleDataChunks.
int ChunkPos;
+ /// Use to access information about deferred extracts
+ BoUpSLP &R;
+
/// Attaches ScheduleData to Instruction.
/// Note that the mapping survives during all vectorization iterations, i.e.
/// ScheduleData structures are recycled.
@@ -12879,6 +13013,38 @@ void BoUpSLP::buildTreeRec(ArrayRef<Value *> VLRef, unsigned Depth,
SmallVector<int> ReuseShuffleIndices;
SmallVector<Value *> VL(VLRef);
+ // Can this bundle be represented as extracts from the same vector
+ auto IsExtractLikeBundle = [&](ArrayRef<Value *> Scalars) -> bool {
+ if (!all_of(Scalars, [&](const Value *Scalar) -> bool {
+ return CouldBeExtract.contains(Scalar);
+ }))
+ return false;
+
+ auto *BaseEE = CouldBeExtract.lookup(Scalars[0]);
+ Value *BaseVec = BaseEE->getVectorOperand();
+ for (const Value *S : Scalars) {
+ auto *EE = CouldBeExtract.lookup(S);
+ // Only handle simple case where all elements come from the same vector
+ // If needs to be from multiple vectors, better off leaving in current
+ // form. Will handle replacing individual rematerializations with extracts
+ // during gather node creation.
+ Value *Vec = EE->getVectorOperand();
+ if (Vec != BaseVec)
+ return false;
+ std::optional<unsigned> Lane = getExtractIndex(EE);
+ if (!Lane)
+ return false;
+ }
+ return true;
+ };
+ auto ReplaceWithExtractCandidates = [&](SmallVectorImpl<Value *> &Scalars) {
+ if (!IsExtractLikeBundle(Scalars))
+ return;
+ for (Value *&VPtr : Scalars) {
+ auto *Extract = CouldBeExtract.lookup(VPtr);
+ VPtr = Extract;
+ }
+ };
// Tries to build split node.
auto TrySplitNode = [&](const InstructionsState &LocalState) {
@@ -13079,7 +13245,7 @@ void BoUpSLP::buildTreeRec(ArrayRef<Value *> VLRef, unsigned Depth,
BasicBlock *BB = VL0->getParent();
auto &BSRef = BlocksSchedules[BB];
if (!BSRef)
- BSRef = std::make_unique<BlockScheduling>(BB);
+ BSRef = std::make_unique<BlockScheduling>(BB, *this);
BlockScheduling &BS = *BSRef;
@@ -13104,6 +13270,9 @@ void BoUpSLP::buildTreeRec(ArrayRef<Value *> VLRef, unsigned Depth,
}
InstructionsCompatibilityAnalysis Analysis(*DT, *DL, *TTI, *TLI);
SmallVector<ValueList> Operands = Analysis.buildOperands(S, VL);
+ for (auto &Ops : Operands)
+ ReplaceWithExtractCandidates(Ops);
+
ScheduleBundle Empty;
ScheduleBundle &Bundle = BundlePtr.value() ? *BundlePtr.value() : Empty;
LLVM_DEBUG(dbgs() << "SLP: We are able to schedule this bundle.\n");
@@ -13401,6 +13570,11 @@ void BoUpSLP::buildTreeRec(ArrayRef<Value *> VLRef, unsigned Depth,
Operands[0] = Ops.getVL(0);
Operands[1] = Ops.getVL(1);
}
+ // Try to replace again after shuffling operands
+ // TODO: Make VLOperands aware of deferred extracts
+ for (auto &Ops : Operands)
+ ReplaceWithExtractCandidates(Ops);
+
TE->setOperands(Operands);
for (unsigned I : seq<unsigned>(VL0->getNumOperands()))
buildTreeRec(TE->getOperand(I), Depth + 1, {TE, I});
@@ -13818,11 +13992,17 @@ unsigned BoUpSLP::getNumVectorInsts() const {
// ExtractElement gathers from the same source vector become a single
// shufflevector. Collect source vectors globally across all gather
// entries and count once at the end.
- if (all_of(TE.Scalars,
- IsaPred<ExtractElementInst, UndefValue, Constant>)) {
- for (Value *V : TE.Scalars)
+ if (all_of(TE.Scalars, [&](Value *V) {
+ return isa<ExtractElementInst, UndefValue, Constant>(V) ||
+ CouldBeExtract.contains(V);
+ })) {
+ for (Value *V : TE.Scalars) {
if (auto *EE = dyn_cast<ExtractElementInst>(V))
GatherExtractSourceVecs.insert(EE->getVectorOperand());
+ else if (CouldBeExtract.contains(V))
+ if (auto *EE = CouldBeExtract.lookup(V))
+ GatherExtractSourceVecs.insert(EE->getVectorOperand());
+ }
} else {
for (Value *V : TE.Scalars) {
if (!isConstant(V))
@@ -16141,7 +16321,8 @@ class BoUpSLP::ShuffleCostEstimator : public BaseShuffleAnalysis {
CheckedExtracts(CheckedExtracts) {}
Value *adjustExtracts(const TreeEntry *E, MutableArrayRef<int> Mask,
ArrayRef<std::optional<TTI::ShuffleKind>> ShuffleKinds,
- unsigned NumParts, bool &UseVecBaseAsInput) {
+ unsigned NumParts, bool &UseVecBaseAsInput,
+ const DenseMap<Value *, Instruction *> &CouldBeRemat) {
UseVecBaseAsInput = false;
if (Mask.empty())
return nullptr;
@@ -16214,8 +16395,21 @@ class BoUpSLP::ShuffleCostEstimator : public BaseShuffleAnalysis {
continue;
unsigned Idx = *EEIdx;
// Take credit for instruction that will become dead.
- if (EE->hasOneUse() || !PrevNodeFound) {
- Instruction *Ext = EE->user_back();
+ // If looking at an extract instruction that was also
+ // rematerialized, the uses are stored by the rematerialized
+ // instruction.
+ bool OneUse;
+ if (auto *RI = CouldBeRemat.lookup(EE))
+ OneUse = RI->hasOneUse();
+ else
+ OneUse = EE->hasOneUse();
+ if (OneUse || !PrevNodeFound) {
+ Instruction *Ext;
+ if (auto *RI = CouldBeRemat.lookup(EE)) {
+ Ext = RI->user_back();
+ } else {
+ Ext = EE->user_back();
+ }
if (isa<SExtInst, ZExtInst>(Ext) &&
all_of(Ext->users(), IsaPred<GetElementPtrInst>)) {
// Use getExtractWithExtendCost() to calculate the cost of
@@ -16330,11 +16524,13 @@ class BoUpSLP::ShuffleCostEstimator : public BaseShuffleAnalysis {
[&](auto P) {
if (P.value() == PoisonMaskElem)
return Mask[P.index()] == PoisonMaskElem;
- auto *EI = cast<ExtractElementInst>(
+ auto *EI = dyn_cast<ExtractElementInst>(
cast<const TreeEntry *>(InVectors.front())
->getOrdered(P.index()));
- return EI->getVectorOperand() == V1 ||
- EI->getVectorOperand() == V2;
+ if (EI)
+ return EI->getVectorOperand() == V1 ||
+ EI->getVectorOperand() == V2;
+ return false;
}) &&
"Expected extractelement vectors.");
}
@@ -16362,8 +16558,9 @@ class BoUpSLP::ShuffleCostEstimator : public BaseShuffleAnalysis {
isa<UndefValue>(Scalar);
if (isa<Constant>(V1))
return true;
- auto *EI = cast<ExtractElementInst>(Scalar);
- return EI->getVectorOperand() == V1;
+ if (auto *EI = dyn_cast<ExtractElementInst>(Scalar))
+ return EI->getVectorOperand() == V1;
+ return false;
}) &&
"Expected only tree entry for extractelement vectors.");
return;
@@ -17032,6 +17229,18 @@ BoUpSLP::getEntryCost(const TreeEntry *E, ArrayRef<Value *> VectorizedVals,
for (unsigned I = 0; I < Sz; ++I) {
if (UsedScalars.test(I))
continue;
+ if (auto *Inst = dyn_cast<Instruction>(UniqueValues[I])) {
+ if (auto It = ExternalUsesAsExtractCost.find(Inst);
+ It != ExternalUsesAsExtractCost.end()) {
+ ScalarCost += It->second;
+ continue;
+ }
+ if (auto It = ExternalUsesAsRematCost.find(Inst);
+ It != ExternalUsesAsRematCost.end()) {
+ ScalarCost += It->second;
+ continue;
+ }
+ }
ScalarCost += ScalarEltCost(I);
}
}
@@ -18154,7 +18363,7 @@ bool BoUpSLP::isFullyVectorizableTinyTree(bool ForReduction) const {
(((TE->hasState() &&
TE->getOpcode() == Instruction::ExtractElement) ||
all_of(TE->Scalars, IsaPred<ExtractElementInst, UndefValue>)) &&
- isFixedVectorShuffle(TE->Scalars, Mask, AC)) ||
+ isFixedVectorShuffle(TE->Scalars, Mask, AC, CouldBeExtract)) ||
(TE->hasState() && TE->getOpcode() == Instruction::Load &&
!TE->isAltShuffle()) ||
any_of(TE->Scalars, IsaPred<LoadInst>));
@@ -20008,9 +20217,14 @@ InstructionCost BoUpSLP::getTreeCost(InstructionCost TreeCost,
LLVM_DEBUG(dbgs() << " ExtractElement cost for " << *ScalarTy << " from "
<< *VecTy << ": " << ExtraCost << "\n");
}
- // Leave the scalar instructions as is if they are cheaper than extracts.
+ // Keep the scalar instruction first when it can be safely used as scalar.
+ // Track cases where extraction is more profitable and convert those to
+ // extracts in a late cleanup step.
if (Entry->Idx != 0 || Entry->getOpcode() == Instruction::GetElementPtr ||
- Entry->getOpcode() == Instruction::Load) {
+ Entry->getOpcode() == Instruction::Load ||
+ (Entry->getOpcode() == Instruction::ZExt &&
+ getOperandEntry(Entry, 0)->getOperations().valid() &&
+ getOperandEntry(Entry, 0)->getOpcode() == Instruction::Load)) {
// Checks if the user of the external scalar is phi in loop body.
auto IsPhiInLoop = [&](const ExternalUser &U) {
if (auto *Phi = dyn_cast_if_present<PHINode>(U.User)) {
@@ -20088,7 +20302,8 @@ InstructionCost BoUpSLP::getTreeCost(InstructionCost TreeCost,
}) <= 2;
if (IsProfitablePHIUser) {
KeepScalar = true;
- } else if (KeepScalar && ScalarCost != TTI::TCC_Free &&
+ } else if (KeepScalar && !isDeferredExtractable(EU.Scalar) &&
+ ScalarCost != TTI::TCC_Free &&
ExtraCost - ScalarCost <= TTI::TCC_Basic &&
(!GatheredLoadsEntriesFirst.has_value() ||
Entry->Idx < *GatheredLoadsEntriesFirst)) {
@@ -20109,6 +20324,13 @@ InstructionCost BoUpSLP::getTreeCost(InstructionCost TreeCost,
}
if (KeepScalar) {
ExternalUsesAsOriginalScalar.insert(EU.Scalar);
+ if (isDeferredExtractable(EU.Scalar)) {
+ auto [ItCost, Inserted] =
+ ExternalUsesAsRematCostTmp.try_emplace(EU.Scalar, ScalarCost);
+ if (!Inserted)
+ ItCost->second = std::min(ItCost->second, ScalarCost);
+ }
+
for (Value *V : Inst->operands()) {
// Struct operands cannot be rebuilt by the !User extraction
// path (it has no insertvalue chain), so leave their existing
@@ -20140,16 +20362,22 @@ InstructionCost BoUpSLP::getTreeCost(InstructionCost TreeCost,
}
}
}
+ } else if (isDeferredExtractable(EU.Scalar)) {
+ ExternalUsesAsExtract.insert(EU.Scalar);
+ auto [ItCost, Inserted] =
+ ExternalUsesAsExtractCostTmp.try_emplace(Inst, ExtraCost);
+ if (!Inserted)
+ ItCost->second = std::min(ItCost->second, ExtraCost);
}
}
}
- // Scale the extract cost by the execution frequency of the block where
- // codegen will place the extractelement. That block is the nearest common
- // dominator of all effective use sites (precomputed in ScalarToExtractBlock
- // above), which is order-independent. For scalars kept as originals the
- // existing ScaleCost path (user-block based) remains correct, since the
- // scalar instruction executes at its definition site's frequency.
+ // Scale the extract/scalar cost by the execution frequency of the block
+ // where codegen will place the extractelement. That block is the nearest
+ // common dominator of all effective use sites (precomputed in
+ // ScalarToExtractBlock above), which is order-independent. For scalars kept
+ // as originals use the existing ScaleCost path (user-block based), since
+ // scalar materialization executes at its definition site's frequency.
if (!ExternalUsesAsOriginalScalar.contains(EU.Scalar)) {
if (ExtraCost.isValid() && ExtraCost != 0) {
if (!EU.User) {
@@ -20545,7 +20773,7 @@ BoUpSLP::tryToGatherSingleRegisterExtractElements(
// Check that gather of extractelements can be represented as just a
// shuffle of a single/two vectors the scalars are extracted from.
std::optional<TTI::ShuffleKind> Res =
- isFixedVectorShuffle(GatheredExtracts, Mask, AC);
+ isFixedVectorShuffle(GatheredExtracts, Mask, AC, CouldBeExtract);
if (!Res || all_of(Mask, equal_to(PoisonMaskElem))) {
// TODO: try to check other subsets if possible.
// Restore the original VL if attempt was not successful.
@@ -20560,11 +20788,6 @@ BoUpSLP::tryToGatherSingleRegisterExtractElements(
std::swap(VL[I], GatheredExtracts[I]);
continue;
}
- auto *EI = dyn_cast<ExtractElementInst>(VL[I]);
- if (!EI || !isa<FixedVectorType>(EI->getVectorOperandType()) ||
- !isa<ConstantInt, UndefValue>(EI->getIndexOperand()) ||
- is_contained(UndefVectorExtracts, I))
- continue;
}
return Res;
}
@@ -22096,7 +22319,8 @@ class BoUpSLP::ShuffleInstructionBuilder final : public BaseShuffleAnalysis {
/// Adjusts extractelements after reusing them.
Value *adjustExtracts(const TreeEntry *E, MutableArrayRef<int> Mask,
ArrayRef<std::optional<TTI::ShuffleKind>> ShuffleKinds,
- unsigned NumParts, bool &UseVecBaseAsInput) {
+ unsigned NumParts, bool &UseVecBaseAsInput,
+ const DenseMap<Value *, Instruction *> &CouldBeRemat) {
UseVecBaseAsInput = false;
SmallPtrSet<Value *, 4> UniqueBases;
Value *VecBase = nullptr;
@@ -22606,6 +22830,11 @@ ResTy BoUpSLP::processBuildVector(const TreeEntry *E, Type *ScalarTy,
auto *VecTy = getWidenedType(ScalarTy, GatheredScalars.size());
unsigned NumParts =
::getNumberOfParts(*TTI, VecTy, ScalarTy, GatheredScalars.size());
+ auto GetGatheredExtract = [&](unsigned Idx) -> ExtractElementInst * {
+ if (auto *EI = dyn_cast<ExtractElementInst>(StoredGS[Idx]))
+ return EI;
+ return CouldBeExtract.lookup(StoredGS[Idx]);
+ };
if (!all_of(GatheredScalars, IsaPred<UndefValue>)) {
// Check for gathered extracts.
bool Resized = false;
@@ -22616,8 +22845,9 @@ ResTy BoUpSLP::processBuildVector(const TreeEntry *E, Type *ScalarTy,
for (auto [Idx, I] : enumerate(ExtractMask)) {
if (I == PoisonMaskElem)
continue;
- if (ArrayRef<TreeEntry *> TEs = getTreeEntries(
- cast<ExtractElementInst>(StoredGS[Idx])->getVectorOperand());
+ auto *EI = GetGatheredExtract(Idx);
+ assert(EI && "Expected only extracts to have been gathered");
+ if (ArrayRef<TreeEntry *> TEs = getTreeEntries(EI->getVectorOperand());
!TEs.empty())
ExtractEntries.append(TEs.begin(), TEs.end());
}
@@ -22630,7 +22860,8 @@ ResTy BoUpSLP::processBuildVector(const TreeEntry *E, Type *ScalarTy,
return *Delayed;
}
if (Value *VecBase = ShuffleBuilder.adjustExtracts(
- E, ExtractMask, ExtractShuffles, NumParts, UseVecBaseAsInput)) {
+ E, ExtractMask, ExtractShuffles, NumParts, UseVecBaseAsInput,
+ CouldBeRemat)) {
ExtractVecBase = VecBase;
if (auto *VecBaseTy = dyn_cast<FixedVectorType>(VecBase->getType()))
if (VF == VecBaseTy->getNumElements() &&
@@ -22838,7 +23069,8 @@ ResTy BoUpSLP::processBuildVector(const TreeEntry *E, Type *ScalarTy,
continue;
if (isa<UndefValue>(StoredGS[I]))
continue;
- auto *EI = cast<ExtractElementInst>(StoredGS[I]);
+ auto *EI = GetGatheredExtract(I);
+ assert(EI && "Expected only extracts to have been gathered");
Value *VecOp = EI->getVectorOperand();
if (ArrayRef<TreeEntry *> TEs = getTreeEntries(VecOp);
!TEs.empty() && TEs.front()->VectorizedValue)
@@ -24615,6 +24847,7 @@ Value *BoUpSLP::vectorizeTree(
assert(Vec && "Can't find vectorizable value");
Value *Lane = Builder.getInt32(ExternalUse.Lane);
+ bool ExtractAnyways = false;
auto ExtractAndExtendIfNeeded = [&](Value *Vec) {
if (isa<InsertValueInst>(Scalar))
return Vec;
@@ -24622,7 +24855,9 @@ Value *BoUpSLP::vectorizeTree(
Value *Ex = nullptr;
Value *ExV = nullptr;
auto *Inst = dyn_cast<Instruction>(Scalar);
- bool ReplaceInst = Inst && ExternalUsesAsOriginalScalar.contains(Inst);
+ bool ReplaceInst = Inst &&
+ ExternalUsesAsOriginalScalar.contains(Inst) &&
+ !ExtractAnyways;
// For struct-typed scalars, the User must be an ExtractValueInst that
// describes which struct field is being extracted. Copy its indices
// into an owning SmallVector so the cache key survives erasure of the
@@ -24635,7 +24870,7 @@ Value *BoUpSLP::vectorizeTree(
}
auto Key = std::make_pair(Scalar, Indices);
auto It = ScalarToEEs.find(Key);
- if (It != ScalarToEEs.end()) {
+ if (It != ScalarToEEs.end() && !ExtractAnyways) {
// No need to emit many extracts, just move the only one in the
// current block.
auto EEIt = It->second.find(ReplaceInst ? Inst->getParent()
@@ -24774,6 +25009,54 @@ Value *BoUpSLP::vectorizeTree(
VectorToInsertElement.try_emplace(Vec, IE);
return Vec;
};
+ auto GetUnderlyingInsts =
+ [](Value *Extract,
+ Value *Remat) -> std::pair<ExtractElementInst *, Instruction *> {
+ auto *ExtractCast = dyn_cast<CastInst>(Extract);
+ if (ExtractCast) {
+ auto *RematCast = dyn_cast<CastInst>(Remat);
+ assert(RematCast && "Expected Remat to be extended if extract is");
+ Extract = ExtractCast->getOperand(0);
+ Remat = RematCast->getOperand(0);
+ }
+ return {dyn_cast<ExtractElementInst>(Extract),
+ dyn_cast<Instruction>(Remat)};
+ };
+ auto TrackDeferredExtract = [&](Instruction *Inst, Value *Replacement) {
+ if (!Inst)
+ return;
+ if (ExternalUsesAsOriginalScalar.contains(Inst)) {
+ if (CouldBeExtract.contains(Replacement))
+ return;
+ ExtractAnyways = true;
+ Value *ReplacedExtract = ExtractAndExtendIfNeeded(Vec);
+ auto P = GetUnderlyingInsts(ReplacedExtract, Replacement);
+ auto *EI = P.first;
+ auto *RI = P.second;
+ assert(EI && RI && "Expected to find underlying instructions");
+ if (ExternalUsesAsRematCostTmp.contains(Inst)) {
+ InstructionCost RematCost = ExternalUsesAsRematCostTmp.lookup(Inst);
+ ExternalUsesAsRematCost.try_emplace(EI, RematCost);
+ }
+ CouldBeExtract.try_emplace(RI, EI);
+ CouldBeRemat.try_emplace(EI, RI);
+ ExtractAnyways = false;
+ return;
+ }
+ if (!ExternalUsesAsExtract.contains(Inst) ||
+ CouldBeExtract.contains(Inst))
+ return;
+ auto P = GetUnderlyingInsts(Replacement, Inst);
+ auto *EI = P.first;
+ auto *RI = P.second;
+ assert(EI && RI && "Expected to find underlying instructions");
+ if (ExternalUsesAsExtractCostTmp.contains(Inst)) {
+ InstructionCost ExtractCost = ExternalUsesAsExtractCostTmp.lookup(Inst);
+ ExternalUsesAsExtractCost.try_emplace(RI, ExtractCost);
+ }
+ CouldBeExtract.try_emplace(RI, EI);
+ CouldBeRemat.try_emplace(EI, RI);
+ };
// If User == nullptr, the Scalar remains as scalar in vectorized
// instructions or is used as extra arg. Generate ExtractElement instruction
// and update the record for this scalar in ExternallyUsedValues.
@@ -24825,6 +25108,7 @@ Value *BoUpSLP::vectorizeTree(
} else {
Builder.SetInsertPoint(&F->getEntryBlock(), F->getEntryBlock().begin());
}
+ bool IsDeferredScalar = isDeferredExtractable(Scalar);
Value *NewInst = ExtractAndExtendIfNeeded(Vec);
// Required to update internally referenced instructions.
if (Scalar != NewInst) {
@@ -24833,6 +25117,8 @@ Value *BoUpSLP::vectorizeTree(
"Extractelements should not be replaced.");
Scalar->replaceAllUsesWith(NewInst);
}
+ if (IsDeferredScalar)
+ TrackDeferredExtract(dyn_cast<Instruction>(Scalar), NewInst);
continue;
}
@@ -24922,15 +25208,20 @@ Value *BoUpSLP::vectorizeTree(
}
} else {
Builder.SetInsertPoint(cast<Instruction>(User));
+ bool IsDeferredScalar = isDeferredExtractable(Scalar);
Value *NewInst = ExtractAndExtendIfNeeded(Vec);
if (isa<StructType>(Scalar->getType()) &&
isa_and_nonnull<ExtractValueInst>(User) &&
!isa<StructType>(NewInst->getType())) {
User->replaceAllUsesWith(NewInst);
eraseInstruction(cast<Instruction>(User));
+ } else if (ExternalUsesAsExtract.contains(Scalar)) {
+ DeferredScalarsToExtract[Scalar].emplace_back(Scalar, NewInst, User);
} else {
User->replaceUsesOfWith(Scalar, NewInst);
}
+ if (IsDeferredScalar)
+ TrackDeferredExtract(dyn_cast<Instruction>(Scalar), NewInst);
}
} else {
Builder.SetInsertPoint(&F->getEntryBlock(), F->getEntryBlock().begin());
@@ -25093,6 +25384,9 @@ Value *BoUpSLP::vectorizeTree(
continue;
if (!isa<Instruction>(Scalar) || Entry->isCopyableElement(Scalar))
continue;
+ if (DeferredScalarsToExtract.contains(Scalar) ||
+ CouldBeRemat.contains(Scalar))
+ continue;
#ifndef NDEBUG
Type *Ty = Scalar->getType();
if (!Ty->isVoidTy()) {
@@ -25245,6 +25539,50 @@ Value *BoUpSLP::vectorizeTree(
return Vec;
}
+void BoUpSLP::emitDeferredExtracts() {
+ SmallVector<DeferredExtractType> DeferredExtracts;
+ for (const auto &Entry : DeferredScalarsToExtract)
+ append_range(DeferredExtracts, Entry.second);
+ for (const auto &DET : DeferredExtracts) {
+ auto *UI = cast<Instruction>(DET.User);
+ if (isDeleted(UI))
+ continue;
+ DET.User->replaceUsesOfWith(DET.Scalar, DET.NewInst);
+ LLVM_DEBUG(dbgs() << "SLP: Delayed replacement:" << *UI << ".\n");
+ }
+ DeferredScalarsToExtract.clear();
+ for (const auto &DET : DeferredExtracts) {
+ LLVM_DEBUG(dbgs() << "SLP: \tErasing scalar:" << *DET.Scalar << ".\n");
+ auto *I = cast<Instruction>(DET.Scalar);
+ if (isDeleted(I))
+ continue;
+ assert((I->use_empty() || all_of(I->uses(),
+ [&](Use &U) {
+ return isDeleted(
+ cast<Instruction>(U.getUser()));
+ })) &&
+ "trying to erase instruction with users.");
+ eraseInstruction(I);
+ }
+ for (const auto &P : CouldBeExtract) {
+ if (auto *Ext = P.second) {
+ if (!isDeleted(Ext)) {
+ unsigned NumUses = Ext->getNumUses();
+ if (!NumUses) {
+ eraseInstruction(Ext);
+ } else if (NumUses == 1) {
+ Value *User = Ext->uses().begin()->getUser();
+ if (auto *CI = dyn_cast<CastInst>(User);
+ CI && CI->getNumUses() == 0) {
+ eraseInstruction(CI);
+ eraseInstruction(Ext);
+ }
+ }
+ }
+ }
+ }
+}
+
void BoUpSLP::optimizeGatherSequence() {
LLVM_DEBUG(dbgs() << "SLP: Optimizing " << GatherShuffleExtractSeq.size()
<< " gather sequences instructions.\n");
@@ -26044,12 +26382,35 @@ void BoUpSLP::BlockScheduling::initScheduleData(Instruction *FromI,
continue;
ScheduleData *SD = ScheduleDataMap.lookup(I);
if (!SD) {
- SD = allocateScheduleDataChunks();
+ // Both an extract and its rematerialization ought to be scheduled
+ // together
+ if (auto *EI = R.getCouldBeExtract().lookup(I)) {
+ SD = ScheduleDataMap.lookup(EI);
+ if (!SD) {
+ SD = allocateScheduleDataChunks();
+ ScheduleDataMap[EI] = SD;
+ }
+ } else if (auto *RI = R.getCouldBeRemat().lookup(I)) {
+ SD = ScheduleDataMap.lookup(RI);
+ if (!SD) {
+ SD = allocateScheduleDataChunks();
+ ScheduleDataMap[RI] = SD;
+ }
+ } else {
+ SD = allocateScheduleDataChunks();
+ }
ScheduleDataMap[I] = SD;
}
- assert(!isInSchedulingRegion(*SD) &&
+ bool IsSharedNode =
+ R.getCouldBeExtract().contains(I) || R.getCouldBeRemat().contains(I);
+ assert((!isInSchedulingRegion(*SD) || IsSharedNode) &&
"new ScheduleData already in scheduling region");
- SD->init(SchedulingRegionID, I);
+ if (!isInSchedulingRegion(*SD)) {
+ if (auto *RI = R.getCouldBeRemat().lookup(I))
+ SD->init(SchedulingRegionID, RI);
+ else
+ SD->init(SchedulingRegionID, I);
+ }
auto CanIgnoreLoad = [](const Instruction *I) {
const auto *LI = dyn_cast<LoadInst>(I);
@@ -26592,9 +26953,15 @@ void BoUpSLP::scheduleBlock(const BoUpSLP &R, BlockScheduling *BS) {
} else {
auto *SD = cast<ScheduleData>(Picked);
Instruction *PickedInst = SD->getInst();
- if (PickedInst->getNextNode() != LastScheduledInst)
+ bool ShouldMove = PickedInst->getNextNode() != LastScheduledInst;
+ if (ShouldMove)
PickedInst->moveAfter(LastScheduledInst->getPrevNode());
LastScheduledInst = PickedInst;
+ if (auto *EI = CouldBeExtract.lookup(PickedInst)) {
+ if (ShouldMove)
+ EI->moveAfter(LastScheduledInst->getPrevNode());
+ LastScheduledInst = EI;
+ }
}
auto Invalid = InstructionsState::invalid();
BS->schedule(R, Invalid, EdgeInfo(), Picked, ReadyInsts);
@@ -27628,6 +27995,8 @@ bool SLPVectorizerPass::runImpl(Function &F, ScalarEvolution *SE_,
}
}
+ R.emitDeferredExtracts();
+
if (Changed) {
R.optimizeGatherSequence();
LLVM_DEBUG(dbgs() << "SLP: vectorized \"" << F.getName() << "\"\n");
@@ -29667,7 +30036,8 @@ class HorizontalReduction {
TrackedToOrig.push_back(RV);
}
SmallVector<int> Mask;
- if (isFixedVectorShuffle(CommonCandidates, Mask, AC)) {
+ if (isFixedVectorShuffle(CommonCandidates, Mask, AC,
+ V.getCouldBeExtract())) {
++I;
Candidates.swap(CommonCandidates);
ShuffledExtracts = true;
@@ -31656,7 +32026,7 @@ bool SLPVectorizerPass::vectorizeInsertElementInst(InsertElementInst *IEI,
SmallVector<int> Mask;
if (!findBuildAggregate(IEI, TTI, BuildVectorOpds, BuildVectorInsts, R) ||
(all_of(BuildVectorOpds, IsaPred<ExtractElementInst, UndefValue>) &&
- isFixedVectorShuffle(BuildVectorOpds, Mask, AC)))
+ isFixedVectorShuffle(BuildVectorOpds, Mask, AC, R.getCouldBeExtract())))
return false;
if (MaxVFOnly && BuildVectorInsts.size() == 2) {
diff --git a/llvm/test/Transforms/SLPVectorizer/AArch64/getelementptr2.ll b/llvm/test/Transforms/SLPVectorizer/AArch64/getelementptr2.ll
index 4847149c87a18..5cc4fe846109b 100644
--- a/llvm/test/Transforms/SLPVectorizer/AArch64/getelementptr2.ll
+++ b/llvm/test/Transforms/SLPVectorizer/AArch64/getelementptr2.ll
@@ -1,8 +1,8 @@
; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
;test_i16_extend NOTE: Assertions have been autogenerated by utils/update_test_checks.py
-; RUN: opt -S -mtriple=aarch64--linux-gnu -passes=slp-vectorizer -slp-threshold=-5 -pass-remarks-output=%t < %s | FileCheck %s
+; RUN: opt -S -mtriple=aarch64--linux-gnu -passes=slp-vectorizer -slp-threshold=-4 -pass-remarks-output=%t < %s | FileCheck %s
; RUN: cat %t | FileCheck -check-prefix=YAML %s
-; RUN: opt -S -mtriple=aarch64--linux-gnu -passes=slp-vectorizer -slp-threshold=-5 -pass-remarks-output=%t < %s | FileCheck %s
+; RUN: opt -S -mtriple=aarch64--linux-gnu -passes=slp-vectorizer -slp-threshold=-4 -pass-remarks-output=%t < %s | FileCheck %s
; RUN: cat %t | FileCheck -check-prefix=YAML %s
diff --git a/llvm/test/Transforms/SLPVectorizer/AArch64/multiple_reduction.ll b/llvm/test/Transforms/SLPVectorizer/AArch64/multiple_reduction.ll
index fe3db7d462e8e..c44c3bbf32553 100644
--- a/llvm/test/Transforms/SLPVectorizer/AArch64/multiple_reduction.ll
+++ b/llvm/test/Transforms/SLPVectorizer/AArch64/multiple_reduction.ll
@@ -14,387 +14,40 @@ define i64 @straight(ptr nocapture noundef readonly %p, i32 noundef %st) {
; CHECK-LABEL: @straight(
; CHECK-NEXT: entry:
; CHECK-NEXT: [[IDX_EXT:%.*]] = sext i32 [[ST:%.*]] to i64
-; CHECK-NEXT: [[TMP0:%.*]] = load i16, ptr [[P:%.*]], align 2
-; CHECK-NEXT: [[CONV:%.*]] = zext i16 [[TMP0]] to i32
-; CHECK-NEXT: [[MUL:%.*]] = mul nuw nsw i32 [[CONV]], [[CONV]]
-; CHECK-NEXT: [[ARRAYIDX_1:%.*]] = getelementptr inbounds i16, ptr [[P]], i64 1
-; CHECK-NEXT: [[TMP1:%.*]] = load i16, ptr [[ARRAYIDX_1]], align 2
-; CHECK-NEXT: [[CONV_1:%.*]] = zext i16 [[TMP1]] to i32
-; CHECK-NEXT: [[ADD_1:%.*]] = add nuw nsw i32 [[CONV]], [[CONV_1]]
-; CHECK-NEXT: [[MUL_1:%.*]] = mul nuw nsw i32 [[CONV_1]], [[CONV_1]]
-; CHECK-NEXT: [[ADD11_1:%.*]] = add nuw i32 [[MUL_1]], [[MUL]]
-; CHECK-NEXT: [[ARRAYIDX_2:%.*]] = getelementptr inbounds i16, ptr [[P]], i64 2
-; CHECK-NEXT: [[TMP2:%.*]] = load i16, ptr [[ARRAYIDX_2]], align 2
-; CHECK-NEXT: [[CONV_2:%.*]] = zext i16 [[TMP2]] to i32
-; CHECK-NEXT: [[ADD_2:%.*]] = add nuw nsw i32 [[ADD_1]], [[CONV_2]]
-; CHECK-NEXT: [[MUL_2:%.*]] = mul nuw nsw i32 [[CONV_2]], [[CONV_2]]
-; CHECK-NEXT: [[ADD11_2:%.*]] = add i32 [[MUL_2]], [[ADD11_1]]
-; CHECK-NEXT: [[ARRAYIDX_3:%.*]] = getelementptr inbounds i16, ptr [[P]], i64 3
-; CHECK-NEXT: [[TMP3:%.*]] = load i16, ptr [[ARRAYIDX_3]], align 2
-; CHECK-NEXT: [[CONV_3:%.*]] = zext i16 [[TMP3]] to i32
-; CHECK-NEXT: [[ADD_3:%.*]] = add nuw nsw i32 [[ADD_2]], [[CONV_3]]
-; CHECK-NEXT: [[MUL_3:%.*]] = mul nuw nsw i32 [[CONV_3]], [[CONV_3]]
-; CHECK-NEXT: [[ADD11_3:%.*]] = add i32 [[MUL_3]], [[ADD11_2]]
-; CHECK-NEXT: [[ARRAYIDX_4:%.*]] = getelementptr inbounds i16, ptr [[P]], i64 4
-; CHECK-NEXT: [[TMP4:%.*]] = load i16, ptr [[ARRAYIDX_4]], align 2
-; CHECK-NEXT: [[CONV_4:%.*]] = zext i16 [[TMP4]] to i32
-; CHECK-NEXT: [[ADD_4:%.*]] = add nuw nsw i32 [[ADD_3]], [[CONV_4]]
-; CHECK-NEXT: [[MUL_4:%.*]] = mul nuw nsw i32 [[CONV_4]], [[CONV_4]]
-; CHECK-NEXT: [[ADD11_4:%.*]] = add i32 [[MUL_4]], [[ADD11_3]]
-; CHECK-NEXT: [[ARRAYIDX_5:%.*]] = getelementptr inbounds i16, ptr [[P]], i64 5
-; CHECK-NEXT: [[TMP5:%.*]] = load i16, ptr [[ARRAYIDX_5]], align 2
-; CHECK-NEXT: [[CONV_5:%.*]] = zext i16 [[TMP5]] to i32
-; CHECK-NEXT: [[ADD_5:%.*]] = add nuw nsw i32 [[ADD_4]], [[CONV_5]]
-; CHECK-NEXT: [[MUL_5:%.*]] = mul nuw nsw i32 [[CONV_5]], [[CONV_5]]
-; CHECK-NEXT: [[ADD11_5:%.*]] = add i32 [[MUL_5]], [[ADD11_4]]
-; CHECK-NEXT: [[ARRAYIDX_6:%.*]] = getelementptr inbounds i16, ptr [[P]], i64 6
-; CHECK-NEXT: [[TMP6:%.*]] = load i16, ptr [[ARRAYIDX_6]], align 2
-; CHECK-NEXT: [[CONV_6:%.*]] = zext i16 [[TMP6]] to i32
-; CHECK-NEXT: [[ADD_6:%.*]] = add nuw nsw i32 [[ADD_5]], [[CONV_6]]
-; CHECK-NEXT: [[MUL_6:%.*]] = mul nuw nsw i32 [[CONV_6]], [[CONV_6]]
-; CHECK-NEXT: [[ADD11_6:%.*]] = add i32 [[MUL_6]], [[ADD11_5]]
-; CHECK-NEXT: [[ARRAYIDX_7:%.*]] = getelementptr inbounds i16, ptr [[P]], i64 7
-; CHECK-NEXT: [[TMP7:%.*]] = load i16, ptr [[ARRAYIDX_7]], align 2
-; CHECK-NEXT: [[CONV_7:%.*]] = zext i16 [[TMP7]] to i32
-; CHECK-NEXT: [[ADD_7:%.*]] = add nuw nsw i32 [[ADD_6]], [[CONV_7]]
-; CHECK-NEXT: [[MUL_7:%.*]] = mul nuw nsw i32 [[CONV_7]], [[CONV_7]]
-; CHECK-NEXT: [[ADD11_7:%.*]] = add i32 [[MUL_7]], [[ADD11_6]]
-; CHECK-NEXT: [[ADD_PTR:%.*]] = getelementptr inbounds i16, ptr [[P]], i64 [[IDX_EXT]]
-; CHECK-NEXT: [[TMP8:%.*]] = load i16, ptr [[ADD_PTR]], align 2
-; CHECK-NEXT: [[CONV_140:%.*]] = zext i16 [[TMP8]] to i32
-; CHECK-NEXT: [[ADD_141:%.*]] = add nuw nsw i32 [[ADD_7]], [[CONV_140]]
-; CHECK-NEXT: [[MUL_142:%.*]] = mul nuw nsw i32 [[CONV_140]], [[CONV_140]]
-; CHECK-NEXT: [[ADD11_143:%.*]] = add i32 [[MUL_142]], [[ADD11_7]]
-; CHECK-NEXT: [[ARRAYIDX_1_1:%.*]] = getelementptr inbounds i16, ptr [[ADD_PTR]], i64 1
-; CHECK-NEXT: [[TMP9:%.*]] = load i16, ptr [[ARRAYIDX_1_1]], align 2
-; CHECK-NEXT: [[CONV_1_1:%.*]] = zext i16 [[TMP9]] to i32
-; CHECK-NEXT: [[ADD_1_1:%.*]] = add nuw nsw i32 [[ADD_141]], [[CONV_1_1]]
-; CHECK-NEXT: [[MUL_1_1:%.*]] = mul nuw nsw i32 [[CONV_1_1]], [[CONV_1_1]]
-; CHECK-NEXT: [[ADD11_1_1:%.*]] = add i32 [[MUL_1_1]], [[ADD11_143]]
-; CHECK-NEXT: [[ARRAYIDX_2_1:%.*]] = getelementptr inbounds i16, ptr [[ADD_PTR]], i64 2
-; CHECK-NEXT: [[TMP10:%.*]] = load i16, ptr [[ARRAYIDX_2_1]], align 2
-; CHECK-NEXT: [[CONV_2_1:%.*]] = zext i16 [[TMP10]] to i32
-; CHECK-NEXT: [[ADD_2_1:%.*]] = add nuw nsw i32 [[ADD_1_1]], [[CONV_2_1]]
-; CHECK-NEXT: [[MUL_2_1:%.*]] = mul nuw nsw i32 [[CONV_2_1]], [[CONV_2_1]]
-; CHECK-NEXT: [[ADD11_2_1:%.*]] = add i32 [[MUL_2_1]], [[ADD11_1_1]]
-; CHECK-NEXT: [[ARRAYIDX_3_1:%.*]] = getelementptr inbounds i16, ptr [[ADD_PTR]], i64 3
-; CHECK-NEXT: [[TMP11:%.*]] = load i16, ptr [[ARRAYIDX_3_1]], align 2
-; CHECK-NEXT: [[CONV_3_1:%.*]] = zext i16 [[TMP11]] to i32
-; CHECK-NEXT: [[ADD_3_1:%.*]] = add nuw nsw i32 [[ADD_2_1]], [[CONV_3_1]]
-; CHECK-NEXT: [[MUL_3_1:%.*]] = mul nuw nsw i32 [[CONV_3_1]], [[CONV_3_1]]
-; CHECK-NEXT: [[ADD11_3_1:%.*]] = add i32 [[MUL_3_1]], [[ADD11_2_1]]
-; CHECK-NEXT: [[ARRAYIDX_4_1:%.*]] = getelementptr inbounds i16, ptr [[ADD_PTR]], i64 4
-; CHECK-NEXT: [[TMP12:%.*]] = load i16, ptr [[ARRAYIDX_4_1]], align 2
-; CHECK-NEXT: [[CONV_4_1:%.*]] = zext i16 [[TMP12]] to i32
-; CHECK-NEXT: [[ADD_4_1:%.*]] = add nuw nsw i32 [[ADD_3_1]], [[CONV_4_1]]
-; CHECK-NEXT: [[MUL_4_1:%.*]] = mul nuw nsw i32 [[CONV_4_1]], [[CONV_4_1]]
-; CHECK-NEXT: [[ADD11_4_1:%.*]] = add i32 [[MUL_4_1]], [[ADD11_3_1]]
-; CHECK-NEXT: [[ARRAYIDX_5_1:%.*]] = getelementptr inbounds i16, ptr [[ADD_PTR]], i64 5
-; CHECK-NEXT: [[TMP13:%.*]] = load i16, ptr [[ARRAYIDX_5_1]], align 2
-; CHECK-NEXT: [[CONV_5_1:%.*]] = zext i16 [[TMP13]] to i32
-; CHECK-NEXT: [[ADD_5_1:%.*]] = add nuw nsw i32 [[ADD_4_1]], [[CONV_5_1]]
-; CHECK-NEXT: [[MUL_5_1:%.*]] = mul nuw nsw i32 [[CONV_5_1]], [[CONV_5_1]]
-; CHECK-NEXT: [[ADD11_5_1:%.*]] = add i32 [[MUL_5_1]], [[ADD11_4_1]]
-; CHECK-NEXT: [[ARRAYIDX_6_1:%.*]] = getelementptr inbounds i16, ptr [[ADD_PTR]], i64 6
-; CHECK-NEXT: [[TMP14:%.*]] = load i16, ptr [[ARRAYIDX_6_1]], align 2
-; CHECK-NEXT: [[CONV_6_1:%.*]] = zext i16 [[TMP14]] to i32
-; CHECK-NEXT: [[ADD_6_1:%.*]] = add nuw nsw i32 [[ADD_5_1]], [[CONV_6_1]]
-; CHECK-NEXT: [[MUL_6_1:%.*]] = mul nuw nsw i32 [[CONV_6_1]], [[CONV_6_1]]
-; CHECK-NEXT: [[ADD11_6_1:%.*]] = add i32 [[MUL_6_1]], [[ADD11_5_1]]
-; CHECK-NEXT: [[ARRAYIDX_7_1:%.*]] = getelementptr inbounds i16, ptr [[ADD_PTR]], i64 7
-; CHECK-NEXT: [[TMP15:%.*]] = load i16, ptr [[ARRAYIDX_7_1]], align 2
-; CHECK-NEXT: [[CONV_7_1:%.*]] = zext i16 [[TMP15]] to i32
-; CHECK-NEXT: [[ADD_7_1:%.*]] = add nuw nsw i32 [[ADD_6_1]], [[CONV_7_1]]
-; CHECK-NEXT: [[MUL_7_1:%.*]] = mul nuw nsw i32 [[CONV_7_1]], [[CONV_7_1]]
-; CHECK-NEXT: [[ADD11_7_1:%.*]] = add i32 [[MUL_7_1]], [[ADD11_6_1]]
+; CHECK-NEXT: [[ADD_PTR:%.*]] = getelementptr inbounds i16, ptr [[P:%.*]], i64 [[IDX_EXT]]
; CHECK-NEXT: [[ADD_PTR_1:%.*]] = getelementptr inbounds i16, ptr [[ADD_PTR]], i64 [[IDX_EXT]]
-; CHECK-NEXT: [[TMP16:%.*]] = load i16, ptr [[ADD_PTR_1]], align 2
-; CHECK-NEXT: [[CONV_244:%.*]] = zext i16 [[TMP16]] to i32
-; CHECK-NEXT: [[ADD_245:%.*]] = add nuw nsw i32 [[ADD_7_1]], [[CONV_244]]
-; CHECK-NEXT: [[MUL_246:%.*]] = mul nuw nsw i32 [[CONV_244]], [[CONV_244]]
-; CHECK-NEXT: [[ADD11_247:%.*]] = add i32 [[MUL_246]], [[ADD11_7_1]]
-; CHECK-NEXT: [[ARRAYIDX_1_2:%.*]] = getelementptr inbounds i16, ptr [[ADD_PTR_1]], i64 1
-; CHECK-NEXT: [[TMP17:%.*]] = load i16, ptr [[ARRAYIDX_1_2]], align 2
-; CHECK-NEXT: [[CONV_1_2:%.*]] = zext i16 [[TMP17]] to i32
-; CHECK-NEXT: [[ADD_1_2:%.*]] = add nuw nsw i32 [[ADD_245]], [[CONV_1_2]]
-; CHECK-NEXT: [[MUL_1_2:%.*]] = mul nuw nsw i32 [[CONV_1_2]], [[CONV_1_2]]
-; CHECK-NEXT: [[ADD11_1_2:%.*]] = add i32 [[MUL_1_2]], [[ADD11_247]]
-; CHECK-NEXT: [[ARRAYIDX_2_2:%.*]] = getelementptr inbounds i16, ptr [[ADD_PTR_1]], i64 2
-; CHECK-NEXT: [[TMP18:%.*]] = load i16, ptr [[ARRAYIDX_2_2]], align 2
-; CHECK-NEXT: [[CONV_2_2:%.*]] = zext i16 [[TMP18]] to i32
-; CHECK-NEXT: [[ADD_2_2:%.*]] = add nuw nsw i32 [[ADD_1_2]], [[CONV_2_2]]
-; CHECK-NEXT: [[MUL_2_2:%.*]] = mul nuw nsw i32 [[CONV_2_2]], [[CONV_2_2]]
-; CHECK-NEXT: [[ADD11_2_2:%.*]] = add i32 [[MUL_2_2]], [[ADD11_1_2]]
-; CHECK-NEXT: [[ARRAYIDX_3_2:%.*]] = getelementptr inbounds i16, ptr [[ADD_PTR_1]], i64 3
-; CHECK-NEXT: [[TMP19:%.*]] = load i16, ptr [[ARRAYIDX_3_2]], align 2
-; CHECK-NEXT: [[CONV_3_2:%.*]] = zext i16 [[TMP19]] to i32
-; CHECK-NEXT: [[ADD_3_2:%.*]] = add nuw nsw i32 [[ADD_2_2]], [[CONV_3_2]]
-; CHECK-NEXT: [[MUL_3_2:%.*]] = mul nuw nsw i32 [[CONV_3_2]], [[CONV_3_2]]
-; CHECK-NEXT: [[ADD11_3_2:%.*]] = add i32 [[MUL_3_2]], [[ADD11_2_2]]
-; CHECK-NEXT: [[ARRAYIDX_4_2:%.*]] = getelementptr inbounds i16, ptr [[ADD_PTR_1]], i64 4
-; CHECK-NEXT: [[TMP20:%.*]] = load i16, ptr [[ARRAYIDX_4_2]], align 2
-; CHECK-NEXT: [[CONV_4_2:%.*]] = zext i16 [[TMP20]] to i32
-; CHECK-NEXT: [[ADD_4_2:%.*]] = add nuw nsw i32 [[ADD_3_2]], [[CONV_4_2]]
-; CHECK-NEXT: [[MUL_4_2:%.*]] = mul nuw nsw i32 [[CONV_4_2]], [[CONV_4_2]]
-; CHECK-NEXT: [[ADD11_4_2:%.*]] = add i32 [[MUL_4_2]], [[ADD11_3_2]]
-; CHECK-NEXT: [[ARRAYIDX_5_2:%.*]] = getelementptr inbounds i16, ptr [[ADD_PTR_1]], i64 5
-; CHECK-NEXT: [[TMP21:%.*]] = load i16, ptr [[ARRAYIDX_5_2]], align 2
-; CHECK-NEXT: [[CONV_5_2:%.*]] = zext i16 [[TMP21]] to i32
-; CHECK-NEXT: [[ADD_5_2:%.*]] = add nuw nsw i32 [[ADD_4_2]], [[CONV_5_2]]
-; CHECK-NEXT: [[MUL_5_2:%.*]] = mul nuw nsw i32 [[CONV_5_2]], [[CONV_5_2]]
-; CHECK-NEXT: [[ADD11_5_2:%.*]] = add i32 [[MUL_5_2]], [[ADD11_4_2]]
-; CHECK-NEXT: [[ARRAYIDX_6_2:%.*]] = getelementptr inbounds i16, ptr [[ADD_PTR_1]], i64 6
-; CHECK-NEXT: [[TMP22:%.*]] = load i16, ptr [[ARRAYIDX_6_2]], align 2
-; CHECK-NEXT: [[CONV_6_2:%.*]] = zext i16 [[TMP22]] to i32
-; CHECK-NEXT: [[ADD_6_2:%.*]] = add nuw nsw i32 [[ADD_5_2]], [[CONV_6_2]]
-; CHECK-NEXT: [[MUL_6_2:%.*]] = mul nuw nsw i32 [[CONV_6_2]], [[CONV_6_2]]
-; CHECK-NEXT: [[ADD11_6_2:%.*]] = add i32 [[MUL_6_2]], [[ADD11_5_2]]
-; CHECK-NEXT: [[ARRAYIDX_7_2:%.*]] = getelementptr inbounds i16, ptr [[ADD_PTR_1]], i64 7
-; CHECK-NEXT: [[TMP23:%.*]] = load i16, ptr [[ARRAYIDX_7_2]], align 2
-; CHECK-NEXT: [[CONV_7_2:%.*]] = zext i16 [[TMP23]] to i32
-; CHECK-NEXT: [[ADD_7_2:%.*]] = add nuw nsw i32 [[ADD_6_2]], [[CONV_7_2]]
-; CHECK-NEXT: [[MUL_7_2:%.*]] = mul nuw nsw i32 [[CONV_7_2]], [[CONV_7_2]]
-; CHECK-NEXT: [[ADD11_7_2:%.*]] = add i32 [[MUL_7_2]], [[ADD11_6_2]]
; CHECK-NEXT: [[ADD_PTR_2:%.*]] = getelementptr inbounds i16, ptr [[ADD_PTR_1]], i64 [[IDX_EXT]]
-; CHECK-NEXT: [[TMP24:%.*]] = load i16, ptr [[ADD_PTR_2]], align 2
-; CHECK-NEXT: [[CONV_348:%.*]] = zext i16 [[TMP24]] to i32
-; CHECK-NEXT: [[ADD_349:%.*]] = add nuw nsw i32 [[ADD_7_2]], [[CONV_348]]
-; CHECK-NEXT: [[MUL_350:%.*]] = mul nuw nsw i32 [[CONV_348]], [[CONV_348]]
-; CHECK-NEXT: [[ADD11_351:%.*]] = add i32 [[MUL_350]], [[ADD11_7_2]]
-; CHECK-NEXT: [[ARRAYIDX_1_3:%.*]] = getelementptr inbounds i16, ptr [[ADD_PTR_2]], i64 1
-; CHECK-NEXT: [[TMP25:%.*]] = load i16, ptr [[ARRAYIDX_1_3]], align 2
-; CHECK-NEXT: [[CONV_1_3:%.*]] = zext i16 [[TMP25]] to i32
-; CHECK-NEXT: [[ADD_1_3:%.*]] = add nuw nsw i32 [[ADD_349]], [[CONV_1_3]]
-; CHECK-NEXT: [[MUL_1_3:%.*]] = mul nuw nsw i32 [[CONV_1_3]], [[CONV_1_3]]
-; CHECK-NEXT: [[ADD11_1_3:%.*]] = add i32 [[MUL_1_3]], [[ADD11_351]]
-; CHECK-NEXT: [[ARRAYIDX_2_3:%.*]] = getelementptr inbounds i16, ptr [[ADD_PTR_2]], i64 2
-; CHECK-NEXT: [[TMP26:%.*]] = load i16, ptr [[ARRAYIDX_2_3]], align 2
-; CHECK-NEXT: [[CONV_2_3:%.*]] = zext i16 [[TMP26]] to i32
-; CHECK-NEXT: [[ADD_2_3:%.*]] = add nuw nsw i32 [[ADD_1_3]], [[CONV_2_3]]
-; CHECK-NEXT: [[MUL_2_3:%.*]] = mul nuw nsw i32 [[CONV_2_3]], [[CONV_2_3]]
-; CHECK-NEXT: [[ADD11_2_3:%.*]] = add i32 [[MUL_2_3]], [[ADD11_1_3]]
-; CHECK-NEXT: [[ARRAYIDX_3_3:%.*]] = getelementptr inbounds i16, ptr [[ADD_PTR_2]], i64 3
-; CHECK-NEXT: [[TMP27:%.*]] = load i16, ptr [[ARRAYIDX_3_3]], align 2
-; CHECK-NEXT: [[CONV_3_3:%.*]] = zext i16 [[TMP27]] to i32
-; CHECK-NEXT: [[ADD_3_3:%.*]] = add nuw nsw i32 [[ADD_2_3]], [[CONV_3_3]]
-; CHECK-NEXT: [[MUL_3_3:%.*]] = mul nuw nsw i32 [[CONV_3_3]], [[CONV_3_3]]
-; CHECK-NEXT: [[ADD11_3_3:%.*]] = add i32 [[MUL_3_3]], [[ADD11_2_3]]
-; CHECK-NEXT: [[ARRAYIDX_4_3:%.*]] = getelementptr inbounds i16, ptr [[ADD_PTR_2]], i64 4
-; CHECK-NEXT: [[TMP28:%.*]] = load i16, ptr [[ARRAYIDX_4_3]], align 2
-; CHECK-NEXT: [[CONV_4_3:%.*]] = zext i16 [[TMP28]] to i32
-; CHECK-NEXT: [[ADD_4_3:%.*]] = add nuw nsw i32 [[ADD_3_3]], [[CONV_4_3]]
-; CHECK-NEXT: [[MUL_4_3:%.*]] = mul nuw nsw i32 [[CONV_4_3]], [[CONV_4_3]]
-; CHECK-NEXT: [[ADD11_4_3:%.*]] = add i32 [[MUL_4_3]], [[ADD11_3_3]]
-; CHECK-NEXT: [[ARRAYIDX_5_3:%.*]] = getelementptr inbounds i16, ptr [[ADD_PTR_2]], i64 5
-; CHECK-NEXT: [[TMP29:%.*]] = load i16, ptr [[ARRAYIDX_5_3]], align 2
-; CHECK-NEXT: [[CONV_5_3:%.*]] = zext i16 [[TMP29]] to i32
-; CHECK-NEXT: [[ADD_5_3:%.*]] = add nuw nsw i32 [[ADD_4_3]], [[CONV_5_3]]
-; CHECK-NEXT: [[MUL_5_3:%.*]] = mul nuw nsw i32 [[CONV_5_3]], [[CONV_5_3]]
-; CHECK-NEXT: [[ADD11_5_3:%.*]] = add i32 [[MUL_5_3]], [[ADD11_4_3]]
-; CHECK-NEXT: [[ARRAYIDX_6_3:%.*]] = getelementptr inbounds i16, ptr [[ADD_PTR_2]], i64 6
-; CHECK-NEXT: [[TMP30:%.*]] = load i16, ptr [[ARRAYIDX_6_3]], align 2
-; CHECK-NEXT: [[CONV_6_3:%.*]] = zext i16 [[TMP30]] to i32
-; CHECK-NEXT: [[ADD_6_3:%.*]] = add nuw nsw i32 [[ADD_5_3]], [[CONV_6_3]]
-; CHECK-NEXT: [[MUL_6_3:%.*]] = mul nuw nsw i32 [[CONV_6_3]], [[CONV_6_3]]
-; CHECK-NEXT: [[ADD11_6_3:%.*]] = add i32 [[MUL_6_3]], [[ADD11_5_3]]
-; CHECK-NEXT: [[ARRAYIDX_7_3:%.*]] = getelementptr inbounds i16, ptr [[ADD_PTR_2]], i64 7
-; CHECK-NEXT: [[TMP31:%.*]] = load i16, ptr [[ARRAYIDX_7_3]], align 2
-; CHECK-NEXT: [[CONV_7_3:%.*]] = zext i16 [[TMP31]] to i32
-; CHECK-NEXT: [[ADD_7_3:%.*]] = add nuw nsw i32 [[ADD_6_3]], [[CONV_7_3]]
-; CHECK-NEXT: [[MUL_7_3:%.*]] = mul nuw nsw i32 [[CONV_7_3]], [[CONV_7_3]]
-; CHECK-NEXT: [[ADD11_7_3:%.*]] = add i32 [[MUL_7_3]], [[ADD11_6_3]]
; CHECK-NEXT: [[ADD_PTR_3:%.*]] = getelementptr inbounds i16, ptr [[ADD_PTR_2]], i64 [[IDX_EXT]]
-; CHECK-NEXT: [[TMP32:%.*]] = load i16, ptr [[ADD_PTR_3]], align 2
-; CHECK-NEXT: [[CONV_452:%.*]] = zext i16 [[TMP32]] to i32
-; CHECK-NEXT: [[ADD_453:%.*]] = add nuw nsw i32 [[ADD_7_3]], [[CONV_452]]
-; CHECK-NEXT: [[MUL_454:%.*]] = mul nuw nsw i32 [[CONV_452]], [[CONV_452]]
-; CHECK-NEXT: [[ADD11_455:%.*]] = add i32 [[MUL_454]], [[ADD11_7_3]]
-; CHECK-NEXT: [[ARRAYIDX_1_4:%.*]] = getelementptr inbounds i16, ptr [[ADD_PTR_3]], i64 1
-; CHECK-NEXT: [[TMP33:%.*]] = load i16, ptr [[ARRAYIDX_1_4]], align 2
-; CHECK-NEXT: [[CONV_1_4:%.*]] = zext i16 [[TMP33]] to i32
-; CHECK-NEXT: [[ADD_1_4:%.*]] = add nuw nsw i32 [[ADD_453]], [[CONV_1_4]]
-; CHECK-NEXT: [[MUL_1_4:%.*]] = mul nuw nsw i32 [[CONV_1_4]], [[CONV_1_4]]
-; CHECK-NEXT: [[ADD11_1_4:%.*]] = add i32 [[MUL_1_4]], [[ADD11_455]]
-; CHECK-NEXT: [[ARRAYIDX_2_4:%.*]] = getelementptr inbounds i16, ptr [[ADD_PTR_3]], i64 2
-; CHECK-NEXT: [[TMP34:%.*]] = load i16, ptr [[ARRAYIDX_2_4]], align 2
-; CHECK-NEXT: [[CONV_2_4:%.*]] = zext i16 [[TMP34]] to i32
-; CHECK-NEXT: [[ADD_2_4:%.*]] = add nuw nsw i32 [[ADD_1_4]], [[CONV_2_4]]
-; CHECK-NEXT: [[MUL_2_4:%.*]] = mul nuw nsw i32 [[CONV_2_4]], [[CONV_2_4]]
-; CHECK-NEXT: [[ADD11_2_4:%.*]] = add i32 [[MUL_2_4]], [[ADD11_1_4]]
-; CHECK-NEXT: [[ARRAYIDX_3_4:%.*]] = getelementptr inbounds i16, ptr [[ADD_PTR_3]], i64 3
-; CHECK-NEXT: [[TMP35:%.*]] = load i16, ptr [[ARRAYIDX_3_4]], align 2
-; CHECK-NEXT: [[CONV_3_4:%.*]] = zext i16 [[TMP35]] to i32
-; CHECK-NEXT: [[ADD_3_4:%.*]] = add nuw nsw i32 [[ADD_2_4]], [[CONV_3_4]]
-; CHECK-NEXT: [[MUL_3_4:%.*]] = mul nuw nsw i32 [[CONV_3_4]], [[CONV_3_4]]
-; CHECK-NEXT: [[ADD11_3_4:%.*]] = add i32 [[MUL_3_4]], [[ADD11_2_4]]
-; CHECK-NEXT: [[ARRAYIDX_4_4:%.*]] = getelementptr inbounds i16, ptr [[ADD_PTR_3]], i64 4
-; CHECK-NEXT: [[TMP36:%.*]] = load i16, ptr [[ARRAYIDX_4_4]], align 2
-; CHECK-NEXT: [[CONV_4_4:%.*]] = zext i16 [[TMP36]] to i32
-; CHECK-NEXT: [[ADD_4_4:%.*]] = add nuw nsw i32 [[ADD_3_4]], [[CONV_4_4]]
-; CHECK-NEXT: [[MUL_4_4:%.*]] = mul nuw nsw i32 [[CONV_4_4]], [[CONV_4_4]]
-; CHECK-NEXT: [[ADD11_4_4:%.*]] = add i32 [[MUL_4_4]], [[ADD11_3_4]]
-; CHECK-NEXT: [[ARRAYIDX_5_4:%.*]] = getelementptr inbounds i16, ptr [[ADD_PTR_3]], i64 5
-; CHECK-NEXT: [[TMP37:%.*]] = load i16, ptr [[ARRAYIDX_5_4]], align 2
-; CHECK-NEXT: [[CONV_5_4:%.*]] = zext i16 [[TMP37]] to i32
-; CHECK-NEXT: [[ADD_5_4:%.*]] = add nuw nsw i32 [[ADD_4_4]], [[CONV_5_4]]
-; CHECK-NEXT: [[MUL_5_4:%.*]] = mul nuw nsw i32 [[CONV_5_4]], [[CONV_5_4]]
-; CHECK-NEXT: [[ADD11_5_4:%.*]] = add i32 [[MUL_5_4]], [[ADD11_4_4]]
-; CHECK-NEXT: [[ARRAYIDX_6_4:%.*]] = getelementptr inbounds i16, ptr [[ADD_PTR_3]], i64 6
-; CHECK-NEXT: [[TMP38:%.*]] = load i16, ptr [[ARRAYIDX_6_4]], align 2
-; CHECK-NEXT: [[CONV_6_4:%.*]] = zext i16 [[TMP38]] to i32
-; CHECK-NEXT: [[ADD_6_4:%.*]] = add nuw nsw i32 [[ADD_5_4]], [[CONV_6_4]]
-; CHECK-NEXT: [[MUL_6_4:%.*]] = mul nuw nsw i32 [[CONV_6_4]], [[CONV_6_4]]
-; CHECK-NEXT: [[ADD11_6_4:%.*]] = add i32 [[MUL_6_4]], [[ADD11_5_4]]
-; CHECK-NEXT: [[ARRAYIDX_7_4:%.*]] = getelementptr inbounds i16, ptr [[ADD_PTR_3]], i64 7
-; CHECK-NEXT: [[TMP39:%.*]] = load i16, ptr [[ARRAYIDX_7_4]], align 2
-; CHECK-NEXT: [[CONV_7_4:%.*]] = zext i16 [[TMP39]] to i32
-; CHECK-NEXT: [[ADD_7_4:%.*]] = add nuw nsw i32 [[ADD_6_4]], [[CONV_7_4]]
-; CHECK-NEXT: [[MUL_7_4:%.*]] = mul nuw nsw i32 [[CONV_7_4]], [[CONV_7_4]]
-; CHECK-NEXT: [[ADD11_7_4:%.*]] = add i32 [[MUL_7_4]], [[ADD11_6_4]]
; CHECK-NEXT: [[ADD_PTR_4:%.*]] = getelementptr inbounds i16, ptr [[ADD_PTR_3]], i64 [[IDX_EXT]]
-; CHECK-NEXT: [[TMP40:%.*]] = load i16, ptr [[ADD_PTR_4]], align 2
-; CHECK-NEXT: [[CONV_556:%.*]] = zext i16 [[TMP40]] to i32
-; CHECK-NEXT: [[ADD_557:%.*]] = add nuw nsw i32 [[ADD_7_4]], [[CONV_556]]
-; CHECK-NEXT: [[MUL_558:%.*]] = mul nuw nsw i32 [[CONV_556]], [[CONV_556]]
-; CHECK-NEXT: [[ADD11_559:%.*]] = add i32 [[MUL_558]], [[ADD11_7_4]]
-; CHECK-NEXT: [[ARRAYIDX_1_5:%.*]] = getelementptr inbounds i16, ptr [[ADD_PTR_4]], i64 1
-; CHECK-NEXT: [[TMP41:%.*]] = load i16, ptr [[ARRAYIDX_1_5]], align 2
-; CHECK-NEXT: [[CONV_1_5:%.*]] = zext i16 [[TMP41]] to i32
-; CHECK-NEXT: [[ADD_1_5:%.*]] = add nuw nsw i32 [[ADD_557]], [[CONV_1_5]]
-; CHECK-NEXT: [[MUL_1_5:%.*]] = mul nuw nsw i32 [[CONV_1_5]], [[CONV_1_5]]
-; CHECK-NEXT: [[ADD11_1_5:%.*]] = add i32 [[MUL_1_5]], [[ADD11_559]]
-; CHECK-NEXT: [[ARRAYIDX_2_5:%.*]] = getelementptr inbounds i16, ptr [[ADD_PTR_4]], i64 2
-; CHECK-NEXT: [[TMP42:%.*]] = load i16, ptr [[ARRAYIDX_2_5]], align 2
-; CHECK-NEXT: [[CONV_2_5:%.*]] = zext i16 [[TMP42]] to i32
-; CHECK-NEXT: [[ADD_2_5:%.*]] = add nuw nsw i32 [[ADD_1_5]], [[CONV_2_5]]
-; CHECK-NEXT: [[MUL_2_5:%.*]] = mul nuw nsw i32 [[CONV_2_5]], [[CONV_2_5]]
-; CHECK-NEXT: [[ADD11_2_5:%.*]] = add i32 [[MUL_2_5]], [[ADD11_1_5]]
-; CHECK-NEXT: [[ARRAYIDX_3_5:%.*]] = getelementptr inbounds i16, ptr [[ADD_PTR_4]], i64 3
-; CHECK-NEXT: [[TMP43:%.*]] = load i16, ptr [[ARRAYIDX_3_5]], align 2
-; CHECK-NEXT: [[CONV_3_5:%.*]] = zext i16 [[TMP43]] to i32
-; CHECK-NEXT: [[ADD_3_5:%.*]] = add nuw nsw i32 [[ADD_2_5]], [[CONV_3_5]]
-; CHECK-NEXT: [[MUL_3_5:%.*]] = mul nuw nsw i32 [[CONV_3_5]], [[CONV_3_5]]
-; CHECK-NEXT: [[ADD11_3_5:%.*]] = add i32 [[MUL_3_5]], [[ADD11_2_5]]
-; CHECK-NEXT: [[ARRAYIDX_4_5:%.*]] = getelementptr inbounds i16, ptr [[ADD_PTR_4]], i64 4
-; CHECK-NEXT: [[TMP44:%.*]] = load i16, ptr [[ARRAYIDX_4_5]], align 2
-; CHECK-NEXT: [[CONV_4_5:%.*]] = zext i16 [[TMP44]] to i32
-; CHECK-NEXT: [[ADD_4_5:%.*]] = add nuw nsw i32 [[ADD_3_5]], [[CONV_4_5]]
-; CHECK-NEXT: [[MUL_4_5:%.*]] = mul nuw nsw i32 [[CONV_4_5]], [[CONV_4_5]]
-; CHECK-NEXT: [[ADD11_4_5:%.*]] = add i32 [[MUL_4_5]], [[ADD11_3_5]]
-; CHECK-NEXT: [[ARRAYIDX_5_5:%.*]] = getelementptr inbounds i16, ptr [[ADD_PTR_4]], i64 5
-; CHECK-NEXT: [[TMP45:%.*]] = load i16, ptr [[ARRAYIDX_5_5]], align 2
-; CHECK-NEXT: [[CONV_5_5:%.*]] = zext i16 [[TMP45]] to i32
-; CHECK-NEXT: [[ADD_5_5:%.*]] = add nuw nsw i32 [[ADD_4_5]], [[CONV_5_5]]
-; CHECK-NEXT: [[MUL_5_5:%.*]] = mul nuw nsw i32 [[CONV_5_5]], [[CONV_5_5]]
-; CHECK-NEXT: [[ADD11_5_5:%.*]] = add i32 [[MUL_5_5]], [[ADD11_4_5]]
-; CHECK-NEXT: [[ARRAYIDX_6_5:%.*]] = getelementptr inbounds i16, ptr [[ADD_PTR_4]], i64 6
-; CHECK-NEXT: [[TMP46:%.*]] = load i16, ptr [[ARRAYIDX_6_5]], align 2
-; CHECK-NEXT: [[CONV_6_5:%.*]] = zext i16 [[TMP46]] to i32
-; CHECK-NEXT: [[ADD_6_5:%.*]] = add nuw nsw i32 [[ADD_5_5]], [[CONV_6_5]]
-; CHECK-NEXT: [[MUL_6_5:%.*]] = mul nuw nsw i32 [[CONV_6_5]], [[CONV_6_5]]
-; CHECK-NEXT: [[ADD11_6_5:%.*]] = add i32 [[MUL_6_5]], [[ADD11_5_5]]
-; CHECK-NEXT: [[ARRAYIDX_7_5:%.*]] = getelementptr inbounds i16, ptr [[ADD_PTR_4]], i64 7
-; CHECK-NEXT: [[TMP47:%.*]] = load i16, ptr [[ARRAYIDX_7_5]], align 2
-; CHECK-NEXT: [[CONV_7_5:%.*]] = zext i16 [[TMP47]] to i32
-; CHECK-NEXT: [[ADD_7_5:%.*]] = add nuw nsw i32 [[ADD_6_5]], [[CONV_7_5]]
-; CHECK-NEXT: [[MUL_7_5:%.*]] = mul nuw nsw i32 [[CONV_7_5]], [[CONV_7_5]]
-; CHECK-NEXT: [[ADD11_7_5:%.*]] = add i32 [[MUL_7_5]], [[ADD11_6_5]]
; CHECK-NEXT: [[ADD_PTR_5:%.*]] = getelementptr inbounds i16, ptr [[ADD_PTR_4]], i64 [[IDX_EXT]]
-; CHECK-NEXT: [[TMP48:%.*]] = load i16, ptr [[ADD_PTR_5]], align 2
-; CHECK-NEXT: [[CONV_660:%.*]] = zext i16 [[TMP48]] to i32
-; CHECK-NEXT: [[ADD_661:%.*]] = add nuw nsw i32 [[ADD_7_5]], [[CONV_660]]
-; CHECK-NEXT: [[MUL_662:%.*]] = mul nuw nsw i32 [[CONV_660]], [[CONV_660]]
-; CHECK-NEXT: [[ADD11_663:%.*]] = add i32 [[MUL_662]], [[ADD11_7_5]]
-; CHECK-NEXT: [[ARRAYIDX_1_6:%.*]] = getelementptr inbounds i16, ptr [[ADD_PTR_5]], i64 1
-; CHECK-NEXT: [[TMP49:%.*]] = load i16, ptr [[ARRAYIDX_1_6]], align 2
-; CHECK-NEXT: [[CONV_1_6:%.*]] = zext i16 [[TMP49]] to i32
-; CHECK-NEXT: [[ADD_1_6:%.*]] = add nuw nsw i32 [[ADD_661]], [[CONV_1_6]]
-; CHECK-NEXT: [[MUL_1_6:%.*]] = mul nuw nsw i32 [[CONV_1_6]], [[CONV_1_6]]
-; CHECK-NEXT: [[ADD11_1_6:%.*]] = add i32 [[MUL_1_6]], [[ADD11_663]]
-; CHECK-NEXT: [[ARRAYIDX_2_6:%.*]] = getelementptr inbounds i16, ptr [[ADD_PTR_5]], i64 2
-; CHECK-NEXT: [[TMP50:%.*]] = load i16, ptr [[ARRAYIDX_2_6]], align 2
-; CHECK-NEXT: [[CONV_2_6:%.*]] = zext i16 [[TMP50]] to i32
-; CHECK-NEXT: [[ADD_2_6:%.*]] = add nuw nsw i32 [[ADD_1_6]], [[CONV_2_6]]
-; CHECK-NEXT: [[MUL_2_6:%.*]] = mul nuw nsw i32 [[CONV_2_6]], [[CONV_2_6]]
-; CHECK-NEXT: [[ADD11_2_6:%.*]] = add i32 [[MUL_2_6]], [[ADD11_1_6]]
-; CHECK-NEXT: [[ARRAYIDX_3_6:%.*]] = getelementptr inbounds i16, ptr [[ADD_PTR_5]], i64 3
-; CHECK-NEXT: [[TMP51:%.*]] = load i16, ptr [[ARRAYIDX_3_6]], align 2
-; CHECK-NEXT: [[CONV_3_6:%.*]] = zext i16 [[TMP51]] to i32
-; CHECK-NEXT: [[ADD_3_6:%.*]] = add nuw nsw i32 [[ADD_2_6]], [[CONV_3_6]]
-; CHECK-NEXT: [[MUL_3_6:%.*]] = mul nuw nsw i32 [[CONV_3_6]], [[CONV_3_6]]
-; CHECK-NEXT: [[ADD11_3_6:%.*]] = add i32 [[MUL_3_6]], [[ADD11_2_6]]
-; CHECK-NEXT: [[ARRAYIDX_4_6:%.*]] = getelementptr inbounds i16, ptr [[ADD_PTR_5]], i64 4
-; CHECK-NEXT: [[TMP52:%.*]] = load i16, ptr [[ARRAYIDX_4_6]], align 2
-; CHECK-NEXT: [[CONV_4_6:%.*]] = zext i16 [[TMP52]] to i32
-; CHECK-NEXT: [[ADD_4_6:%.*]] = add nuw nsw i32 [[ADD_3_6]], [[CONV_4_6]]
-; CHECK-NEXT: [[MUL_4_6:%.*]] = mul nuw nsw i32 [[CONV_4_6]], [[CONV_4_6]]
-; CHECK-NEXT: [[ADD11_4_6:%.*]] = add i32 [[MUL_4_6]], [[ADD11_3_6]]
-; CHECK-NEXT: [[ARRAYIDX_5_6:%.*]] = getelementptr inbounds i16, ptr [[ADD_PTR_5]], i64 5
-; CHECK-NEXT: [[TMP53:%.*]] = load i16, ptr [[ARRAYIDX_5_6]], align 2
-; CHECK-NEXT: [[CONV_5_6:%.*]] = zext i16 [[TMP53]] to i32
-; CHECK-NEXT: [[ADD_5_6:%.*]] = add nuw nsw i32 [[ADD_4_6]], [[CONV_5_6]]
-; CHECK-NEXT: [[MUL_5_6:%.*]] = mul nuw nsw i32 [[CONV_5_6]], [[CONV_5_6]]
-; CHECK-NEXT: [[ADD11_5_6:%.*]] = add i32 [[MUL_5_6]], [[ADD11_4_6]]
-; CHECK-NEXT: [[ARRAYIDX_6_6:%.*]] = getelementptr inbounds i16, ptr [[ADD_PTR_5]], i64 6
-; CHECK-NEXT: [[TMP54:%.*]] = load i16, ptr [[ARRAYIDX_6_6]], align 2
-; CHECK-NEXT: [[CONV_6_6:%.*]] = zext i16 [[TMP54]] to i32
-; CHECK-NEXT: [[ADD_6_6:%.*]] = add nuw nsw i32 [[ADD_5_6]], [[CONV_6_6]]
-; CHECK-NEXT: [[MUL_6_6:%.*]] = mul nuw nsw i32 [[CONV_6_6]], [[CONV_6_6]]
-; CHECK-NEXT: [[ADD11_6_6:%.*]] = add i32 [[MUL_6_6]], [[ADD11_5_6]]
-; CHECK-NEXT: [[ARRAYIDX_7_6:%.*]] = getelementptr inbounds i16, ptr [[ADD_PTR_5]], i64 7
-; CHECK-NEXT: [[TMP55:%.*]] = load i16, ptr [[ARRAYIDX_7_6]], align 2
-; CHECK-NEXT: [[CONV_7_6:%.*]] = zext i16 [[TMP55]] to i32
-; CHECK-NEXT: [[ADD_7_6:%.*]] = add nuw nsw i32 [[ADD_6_6]], [[CONV_7_6]]
-; CHECK-NEXT: [[MUL_7_6:%.*]] = mul nuw nsw i32 [[CONV_7_6]], [[CONV_7_6]]
-; CHECK-NEXT: [[ADD11_7_6:%.*]] = add i32 [[MUL_7_6]], [[ADD11_6_6]]
; CHECK-NEXT: [[ADD_PTR_6:%.*]] = getelementptr inbounds i16, ptr [[ADD_PTR_5]], i64 [[IDX_EXT]]
-; CHECK-NEXT: [[TMP56:%.*]] = load i16, ptr [[ADD_PTR_6]], align 2
-; CHECK-NEXT: [[CONV_764:%.*]] = zext i16 [[TMP56]] to i32
-; CHECK-NEXT: [[ADD_765:%.*]] = add nuw nsw i32 [[ADD_7_6]], [[CONV_764]]
-; CHECK-NEXT: [[MUL_766:%.*]] = mul nuw nsw i32 [[CONV_764]], [[CONV_764]]
-; CHECK-NEXT: [[ADD11_767:%.*]] = add i32 [[MUL_766]], [[ADD11_7_6]]
-; CHECK-NEXT: [[ARRAYIDX_1_7:%.*]] = getelementptr inbounds i16, ptr [[ADD_PTR_6]], i64 1
-; CHECK-NEXT: [[TMP57:%.*]] = load i16, ptr [[ARRAYIDX_1_7]], align 2
-; CHECK-NEXT: [[CONV_1_7:%.*]] = zext i16 [[TMP57]] to i32
-; CHECK-NEXT: [[ADD_1_7:%.*]] = add nuw nsw i32 [[ADD_765]], [[CONV_1_7]]
-; CHECK-NEXT: [[MUL_1_7:%.*]] = mul nuw nsw i32 [[CONV_1_7]], [[CONV_1_7]]
-; CHECK-NEXT: [[ADD11_1_7:%.*]] = add i32 [[MUL_1_7]], [[ADD11_767]]
-; CHECK-NEXT: [[ARRAYIDX_2_7:%.*]] = getelementptr inbounds i16, ptr [[ADD_PTR_6]], i64 2
-; CHECK-NEXT: [[TMP58:%.*]] = load i16, ptr [[ARRAYIDX_2_7]], align 2
-; CHECK-NEXT: [[CONV_2_7:%.*]] = zext i16 [[TMP58]] to i32
-; CHECK-NEXT: [[ADD_2_7:%.*]] = add nuw nsw i32 [[ADD_1_7]], [[CONV_2_7]]
-; CHECK-NEXT: [[MUL_2_7:%.*]] = mul nuw nsw i32 [[CONV_2_7]], [[CONV_2_7]]
-; CHECK-NEXT: [[ADD11_2_7:%.*]] = add i32 [[MUL_2_7]], [[ADD11_1_7]]
-; CHECK-NEXT: [[ARRAYIDX_3_7:%.*]] = getelementptr inbounds i16, ptr [[ADD_PTR_6]], i64 3
-; CHECK-NEXT: [[TMP59:%.*]] = load i16, ptr [[ARRAYIDX_3_7]], align 2
-; CHECK-NEXT: [[CONV_3_7:%.*]] = zext i16 [[TMP59]] to i32
-; CHECK-NEXT: [[ADD_3_7:%.*]] = add nuw nsw i32 [[ADD_2_7]], [[CONV_3_7]]
-; CHECK-NEXT: [[MUL_3_7:%.*]] = mul nuw nsw i32 [[CONV_3_7]], [[CONV_3_7]]
-; CHECK-NEXT: [[ADD11_3_7:%.*]] = add i32 [[MUL_3_7]], [[ADD11_2_7]]
-; CHECK-NEXT: [[ARRAYIDX_4_7:%.*]] = getelementptr inbounds i16, ptr [[ADD_PTR_6]], i64 4
-; CHECK-NEXT: [[TMP60:%.*]] = load i16, ptr [[ARRAYIDX_4_7]], align 2
-; CHECK-NEXT: [[CONV_4_7:%.*]] = zext i16 [[TMP60]] to i32
-; CHECK-NEXT: [[ADD_4_7:%.*]] = add nuw nsw i32 [[ADD_3_7]], [[CONV_4_7]]
-; CHECK-NEXT: [[MUL_4_7:%.*]] = mul nuw nsw i32 [[CONV_4_7]], [[CONV_4_7]]
-; CHECK-NEXT: [[ADD11_4_7:%.*]] = add i32 [[MUL_4_7]], [[ADD11_3_7]]
-; CHECK-NEXT: [[ARRAYIDX_5_7:%.*]] = getelementptr inbounds i16, ptr [[ADD_PTR_6]], i64 5
-; CHECK-NEXT: [[TMP61:%.*]] = load i16, ptr [[ARRAYIDX_5_7]], align 2
-; CHECK-NEXT: [[CONV_5_7:%.*]] = zext i16 [[TMP61]] to i32
-; CHECK-NEXT: [[ADD_5_7:%.*]] = add nuw nsw i32 [[ADD_4_7]], [[CONV_5_7]]
-; CHECK-NEXT: [[MUL_5_7:%.*]] = mul nuw nsw i32 [[CONV_5_7]], [[CONV_5_7]]
-; CHECK-NEXT: [[ADD11_5_7:%.*]] = add i32 [[MUL_5_7]], [[ADD11_4_7]]
-; CHECK-NEXT: [[ARRAYIDX_6_7:%.*]] = getelementptr inbounds i16, ptr [[ADD_PTR_6]], i64 6
-; CHECK-NEXT: [[TMP62:%.*]] = load i16, ptr [[ARRAYIDX_6_7]], align 2
-; CHECK-NEXT: [[CONV_6_7:%.*]] = zext i16 [[TMP62]] to i32
-; CHECK-NEXT: [[ADD_6_7:%.*]] = add nuw nsw i32 [[ADD_5_7]], [[CONV_6_7]]
-; CHECK-NEXT: [[MUL_6_7:%.*]] = mul nuw nsw i32 [[CONV_6_7]], [[CONV_6_7]]
-; CHECK-NEXT: [[ADD11_6_7:%.*]] = add i32 [[MUL_6_7]], [[ADD11_5_7]]
-; CHECK-NEXT: [[ARRAYIDX_7_7:%.*]] = getelementptr inbounds i16, ptr [[ADD_PTR_6]], i64 7
-; CHECK-NEXT: [[TMP63:%.*]] = load i16, ptr [[ARRAYIDX_7_7]], align 2
-; CHECK-NEXT: [[CONV_7_7:%.*]] = zext i16 [[TMP63]] to i32
-; CHECK-NEXT: [[ADD_7_7:%.*]] = add nuw nsw i32 [[ADD_6_7]], [[CONV_7_7]]
-; CHECK-NEXT: [[MUL_7_7:%.*]] = mul nuw nsw i32 [[CONV_7_7]], [[CONV_7_7]]
-; CHECK-NEXT: [[ADD11_7_7:%.*]] = add i32 [[MUL_7_7]], [[ADD11_6_7]]
+; CHECK-NEXT: [[TMP1:%.*]] = load <8 x i16>, ptr [[P]], align 2
+; CHECK-NEXT: [[TMP10:%.*]] = load <8 x i16>, ptr [[ADD_PTR]], align 2
+; CHECK-NEXT: [[TMP19:%.*]] = load <8 x i16>, ptr [[ADD_PTR_1]], align 2
+; CHECK-NEXT: [[TMP28:%.*]] = load <8 x i16>, ptr [[ADD_PTR_2]], align 2
+; CHECK-NEXT: [[TMP37:%.*]] = load <8 x i16>, ptr [[ADD_PTR_3]], align 2
+; CHECK-NEXT: [[TMP46:%.*]] = load <8 x i16>, ptr [[ADD_PTR_4]], align 2
+; CHECK-NEXT: [[TMP55:%.*]] = load <8 x i16>, ptr [[ADD_PTR_5]], align 2
+; CHECK-NEXT: [[TMP64:%.*]] = load <8 x i16>, ptr [[ADD_PTR_6]], align 2
+; CHECK-NEXT: [[TMP72:%.*]] = shufflevector <8 x i16> [[TMP64]], <8 x i16> poison, <64 x i32> <i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>
+; CHECK-NEXT: [[TMP73:%.*]] = shufflevector <8 x i16> [[TMP55]], <8 x i16> poison, <64 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
+; CHECK-NEXT: [[TMP74:%.*]] = shufflevector <8 x i16> [[TMP64]], <8 x i16> [[TMP55]], <64 x i32> <i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15, i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>
+; CHECK-NEXT: [[TMP75:%.*]] = shufflevector <8 x i16> [[TMP46]], <8 x i16> poison, <64 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
+; CHECK-NEXT: [[TMP76:%.*]] = shufflevector <64 x i16> [[TMP74]], <64 x i16> [[TMP75]], <64 x i32> <i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 64, i32 65, i32 66, i32 67, i32 68, i32 69, i32 70, i32 71, i32 48, i32 49, i32 50, i32 51, i32 52, i32 53, i32 54, i32 55, i32 56, i32 57, i32 58, i32 59, i32 60, i32 61, i32 62, i32 63>
+; CHECK-NEXT: [[TMP77:%.*]] = shufflevector <8 x i16> [[TMP37]], <8 x i16> poison, <64 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
+; CHECK-NEXT: [[TMP78:%.*]] = shufflevector <64 x i16> [[TMP76]], <64 x i16> [[TMP77]], <64 x i32> <i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 64, i32 65, i32 66, i32 67, i32 68, i32 69, i32 70, i32 71, i32 40, i32 41, i32 42, i32 43, i32 44, i32 45, i32 46, i32 47, i32 48, i32 49, i32 50, i32 51, i32 52, i32 53, i32 54, i32 55, i32 56, i32 57, i32 58, i32 59, i32 60, i32 61, i32 62, i32 63>
+; CHECK-NEXT: [[TMP79:%.*]] = shufflevector <8 x i16> [[TMP28]], <8 x i16> poison, <64 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
+; CHECK-NEXT: [[TMP80:%.*]] = shufflevector <64 x i16> [[TMP78]], <64 x i16> [[TMP79]], <64 x i32> <i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 64, i32 65, i32 66, i32 67, i32 68, i32 69, i32 70, i32 71, i32 32, i32 33, i32 34, i32 35, i32 36, i32 37, i32 38, i32 39, i32 40, i32 41, i32 42, i32 43, i32 44, i32 45, i32 46, i32 47, i32 48, i32 49, i32 50, i32 51, i32 52, i32 53, i32 54, i32 55, i32 56, i32 57, i32 58, i32 59, i32 60, i32 61, i32 62, i32 63>
+; CHECK-NEXT: [[TMP81:%.*]] = shufflevector <8 x i16> [[TMP19]], <8 x i16> poison, <64 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
+; CHECK-NEXT: [[TMP82:%.*]] = shufflevector <64 x i16> [[TMP80]], <64 x i16> [[TMP81]], <64 x i32> <i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 64, i32 65, i32 66, i32 67, i32 68, i32 69, i32 70, i32 71, i32 24, i32 25, i32 26, i32 27, i32 28, i32 29, i32 30, i32 31, i32 32, i32 33, i32 34, i32 35, i32 36, i32 37, i32 38, i32 39, i32 40, i32 41, i32 42, i32 43, i32 44, i32 45, i32 46, i32 47, i32 48, i32 49, i32 50, i32 51, i32 52, i32 53, i32 54, i32 55, i32 56, i32 57, i32 58, i32 59, i32 60, i32 61, i32 62, i32 63>
+; CHECK-NEXT: [[TMP83:%.*]] = shufflevector <8 x i16> [[TMP1]], <8 x i16> poison, <64 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
+; CHECK-NEXT: [[TMP84:%.*]] = shufflevector <64 x i16> [[TMP82]], <64 x i16> [[TMP83]], <64 x i32> <i32 64, i32 65, i32 66, i32 67, i32 68, i32 69, i32 70, i32 71, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 16, i32 17, i32 18, i32 19, i32 20, i32 21, i32 22, i32 23, i32 24, i32 25, i32 26, i32 27, i32 28, i32 29, i32 30, i32 31, i32 32, i32 33, i32 34, i32 35, i32 36, i32 37, i32 38, i32 39, i32 40, i32 41, i32 42, i32 43, i32 44, i32 45, i32 46, i32 47, i32 48, i32 49, i32 50, i32 51, i32 52, i32 53, i32 54, i32 55, i32 56, i32 57, i32 58, i32 59, i32 60, i32 61, i32 62, i32 63>
+; CHECK-NEXT: [[TMP85:%.*]] = shufflevector <8 x i16> [[TMP10]], <8 x i16> poison, <64 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
+; CHECK-NEXT: [[TMP86:%.*]] = shufflevector <64 x i16> [[TMP84]], <64 x i16> [[TMP85]], <64 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 64, i32 65, i32 66, i32 67, i32 68, i32 69, i32 70, i32 71, i32 16, i32 17, i32 18, i32 19, i32 20, i32 21, i32 22, i32 23, i32 24, i32 25, i32 26, i32 27, i32 28, i32 29, i32 30, i32 31, i32 32, i32 33, i32 34, i32 35, i32 36, i32 37, i32 38, i32 39, i32 40, i32 41, i32 42, i32 43, i32 44, i32 45, i32 46, i32 47, i32 48, i32 49, i32 50, i32 51, i32 52, i32 53, i32 54, i32 55, i32 56, i32 57, i32 58, i32 59, i32 60, i32 61, i32 62, i32 63>
+; CHECK-NEXT: [[TMP87:%.*]] = zext <64 x i16> [[TMP86]] to <64 x i32>
+; CHECK-NEXT: [[ADD_7_7:%.*]] = call i32 @llvm.vector.reduce.add.v64i32(<64 x i32> [[TMP87]])
+; CHECK-NEXT: [[TMP88:%.*]] = mul nuw nsw <64 x i32> [[TMP87]], [[TMP87]]
+; CHECK-NEXT: [[ADD11_7_7:%.*]] = call i32 @llvm.vector.reduce.add.v64i32(<64 x i32> [[TMP88]])
; CHECK-NEXT: [[CONV15:%.*]] = zext i32 [[ADD_7_7]] to i64
; CHECK-NEXT: [[CONV16:%.*]] = zext i32 [[ADD11_7_7]] to i64
; CHECK-NEXT: [[SHL:%.*]] = shl nuw i64 [[CONV16]], 32
@@ -790,237 +443,3 @@ entry:
%add17 = or i64 %shl, %conv15
ret i64 %add17
}
-
-define i64 @looped(ptr nocapture noundef readonly %p, i32 noundef %st) {
-; CHECK-LABEL: @looped(
-; CHECK-NEXT: entry:
-; CHECK-NEXT: [[IDX_EXT:%.*]] = sext i32 [[ST:%.*]] to i64
-; CHECK-NEXT: br label [[FOR_COND1_PREHEADER:%.*]]
-; CHECK: for.cond1.preheader:
-; CHECK-NEXT: [[Y_038:%.*]] = phi i32 [ 0, [[ENTRY:%.*]] ], [ [[INC13:%.*]], [[FOR_COND1_PREHEADER]] ]
-; CHECK-NEXT: [[SQ_037:%.*]] = phi i32 [ 0, [[ENTRY]] ], [ [[OP_RDX:%.*]], [[FOR_COND1_PREHEADER]] ]
-; CHECK-NEXT: [[SM_036:%.*]] = phi i32 [ 0, [[ENTRY]] ], [ [[OP_RDX1:%.*]], [[FOR_COND1_PREHEADER]] ]
-; CHECK-NEXT: [[P_ADDR_035:%.*]] = phi ptr [ [[P:%.*]], [[ENTRY]] ], [ [[ADD_PTR:%.*]], [[FOR_COND1_PREHEADER]] ]
-; CHECK-NEXT: [[TMP0:%.*]] = load i16, ptr [[P_ADDR_035]], align 2
-; CHECK-NEXT: [[CONV:%.*]] = zext i16 [[TMP0]] to i32
-; CHECK-NEXT: [[ADD:%.*]] = add i32 [[SM_036]], [[CONV]]
-; CHECK-NEXT: [[MUL:%.*]] = mul nuw nsw i32 [[CONV]], [[CONV]]
-; CHECK-NEXT: [[ADD11:%.*]] = add i32 [[MUL]], [[SQ_037]]
-; CHECK-NEXT: [[ARRAYIDX_1:%.*]] = getelementptr inbounds i16, ptr [[P_ADDR_035]], i64 1
-; CHECK-NEXT: [[TMP1:%.*]] = load i16, ptr [[ARRAYIDX_1]], align 2
-; CHECK-NEXT: [[CONV_1:%.*]] = zext i16 [[TMP1]] to i32
-; CHECK-NEXT: [[ADD_1:%.*]] = add i32 [[ADD]], [[CONV_1]]
-; CHECK-NEXT: [[MUL_1:%.*]] = mul nuw nsw i32 [[CONV_1]], [[CONV_1]]
-; CHECK-NEXT: [[ADD11_1:%.*]] = add i32 [[MUL_1]], [[ADD11]]
-; CHECK-NEXT: [[ARRAYIDX_2:%.*]] = getelementptr inbounds i16, ptr [[P_ADDR_035]], i64 2
-; CHECK-NEXT: [[TMP2:%.*]] = load i16, ptr [[ARRAYIDX_2]], align 2
-; CHECK-NEXT: [[CONV_2:%.*]] = zext i16 [[TMP2]] to i32
-; CHECK-NEXT: [[ADD_2:%.*]] = add i32 [[ADD_1]], [[CONV_2]]
-; CHECK-NEXT: [[MUL_2:%.*]] = mul nuw nsw i32 [[CONV_2]], [[CONV_2]]
-; CHECK-NEXT: [[ADD11_2:%.*]] = add i32 [[MUL_2]], [[ADD11_1]]
-; CHECK-NEXT: [[ARRAYIDX_3:%.*]] = getelementptr inbounds i16, ptr [[P_ADDR_035]], i64 3
-; CHECK-NEXT: [[TMP3:%.*]] = load i16, ptr [[ARRAYIDX_3]], align 2
-; CHECK-NEXT: [[CONV_3:%.*]] = zext i16 [[TMP3]] to i32
-; CHECK-NEXT: [[ADD_3:%.*]] = add i32 [[ADD_2]], [[CONV_3]]
-; CHECK-NEXT: [[MUL_3:%.*]] = mul nuw nsw i32 [[CONV_3]], [[CONV_3]]
-; CHECK-NEXT: [[ADD11_3:%.*]] = add i32 [[MUL_3]], [[ADD11_2]]
-; CHECK-NEXT: [[ARRAYIDX_4:%.*]] = getelementptr inbounds i16, ptr [[P_ADDR_035]], i64 4
-; CHECK-NEXT: [[TMP4:%.*]] = load i16, ptr [[ARRAYIDX_4]], align 2
-; CHECK-NEXT: [[CONV_4:%.*]] = zext i16 [[TMP4]] to i32
-; CHECK-NEXT: [[ADD_4:%.*]] = add i32 [[ADD_3]], [[CONV_4]]
-; CHECK-NEXT: [[MUL_4:%.*]] = mul nuw nsw i32 [[CONV_4]], [[CONV_4]]
-; CHECK-NEXT: [[ADD11_4:%.*]] = add i32 [[MUL_4]], [[ADD11_3]]
-; CHECK-NEXT: [[ARRAYIDX_5:%.*]] = getelementptr inbounds i16, ptr [[P_ADDR_035]], i64 5
-; CHECK-NEXT: [[TMP5:%.*]] = load i16, ptr [[ARRAYIDX_5]], align 2
-; CHECK-NEXT: [[CONV_5:%.*]] = zext i16 [[TMP5]] to i32
-; CHECK-NEXT: [[ADD_5:%.*]] = add i32 [[ADD_4]], [[CONV_5]]
-; CHECK-NEXT: [[MUL_5:%.*]] = mul nuw nsw i32 [[CONV_5]], [[CONV_5]]
-; CHECK-NEXT: [[ADD11_5:%.*]] = add i32 [[MUL_5]], [[ADD11_4]]
-; CHECK-NEXT: [[ARRAYIDX_6:%.*]] = getelementptr inbounds i16, ptr [[P_ADDR_035]], i64 6
-; CHECK-NEXT: [[TMP6:%.*]] = load i16, ptr [[ARRAYIDX_6]], align 2
-; CHECK-NEXT: [[CONV_6:%.*]] = zext i16 [[TMP6]] to i32
-; CHECK-NEXT: [[ADD_6:%.*]] = add i32 [[ADD_5]], [[CONV_6]]
-; CHECK-NEXT: [[MUL_6:%.*]] = mul nuw nsw i32 [[CONV_6]], [[CONV_6]]
-; CHECK-NEXT: [[ADD11_6:%.*]] = add i32 [[MUL_6]], [[ADD11_5]]
-; CHECK-NEXT: [[ARRAYIDX_7:%.*]] = getelementptr inbounds i16, ptr [[P_ADDR_035]], i64 7
-; CHECK-NEXT: [[TMP7:%.*]] = load i16, ptr [[ARRAYIDX_7]], align 2
-; CHECK-NEXT: [[CONV_7:%.*]] = zext i16 [[TMP7]] to i32
-; CHECK-NEXT: [[ADD_7:%.*]] = add i32 [[ADD_6]], [[CONV_7]]
-; CHECK-NEXT: [[MUL_7:%.*]] = mul nuw nsw i32 [[CONV_7]], [[CONV_7]]
-; CHECK-NEXT: [[ADD11_7:%.*]] = add i32 [[MUL_7]], [[ADD11_6]]
-; CHECK-NEXT: [[ARRAYIDX_8:%.*]] = getelementptr inbounds i16, ptr [[P_ADDR_035]], i64 8
-; CHECK-NEXT: [[TMP8:%.*]] = load i16, ptr [[ARRAYIDX_8]], align 2
-; CHECK-NEXT: [[CONV_8:%.*]] = zext i16 [[TMP8]] to i32
-; CHECK-NEXT: [[ADD_8:%.*]] = add i32 [[ADD_7]], [[CONV_8]]
-; CHECK-NEXT: [[MUL_8:%.*]] = mul nuw nsw i32 [[CONV_8]], [[CONV_8]]
-; CHECK-NEXT: [[ADD11_8:%.*]] = add i32 [[MUL_8]], [[ADD11_7]]
-; CHECK-NEXT: [[ARRAYIDX_9:%.*]] = getelementptr inbounds i16, ptr [[P_ADDR_035]], i64 9
-; CHECK-NEXT: [[TMP9:%.*]] = load i16, ptr [[ARRAYIDX_9]], align 2
-; CHECK-NEXT: [[CONV_9:%.*]] = zext i16 [[TMP9]] to i32
-; CHECK-NEXT: [[ADD_9:%.*]] = add i32 [[ADD_8]], [[CONV_9]]
-; CHECK-NEXT: [[MUL_9:%.*]] = mul nuw nsw i32 [[CONV_9]], [[CONV_9]]
-; CHECK-NEXT: [[ADD11_9:%.*]] = add i32 [[MUL_9]], [[ADD11_8]]
-; CHECK-NEXT: [[ARRAYIDX_10:%.*]] = getelementptr inbounds i16, ptr [[P_ADDR_035]], i64 10
-; CHECK-NEXT: [[TMP10:%.*]] = load i16, ptr [[ARRAYIDX_10]], align 2
-; CHECK-NEXT: [[CONV_10:%.*]] = zext i16 [[TMP10]] to i32
-; CHECK-NEXT: [[ADD_10:%.*]] = add i32 [[ADD_9]], [[CONV_10]]
-; CHECK-NEXT: [[MUL_10:%.*]] = mul nuw nsw i32 [[CONV_10]], [[CONV_10]]
-; CHECK-NEXT: [[ADD11_10:%.*]] = add i32 [[MUL_10]], [[ADD11_9]]
-; CHECK-NEXT: [[ARRAYIDX_11:%.*]] = getelementptr inbounds i16, ptr [[P_ADDR_035]], i64 11
-; CHECK-NEXT: [[TMP11:%.*]] = load i16, ptr [[ARRAYIDX_11]], align 2
-; CHECK-NEXT: [[CONV_11:%.*]] = zext i16 [[TMP11]] to i32
-; CHECK-NEXT: [[ADD_11:%.*]] = add i32 [[ADD_10]], [[CONV_11]]
-; CHECK-NEXT: [[MUL_11:%.*]] = mul nuw nsw i32 [[CONV_11]], [[CONV_11]]
-; CHECK-NEXT: [[ADD11_11:%.*]] = add i32 [[MUL_11]], [[ADD11_10]]
-; CHECK-NEXT: [[ARRAYIDX_12:%.*]] = getelementptr inbounds i16, ptr [[P_ADDR_035]], i64 12
-; CHECK-NEXT: [[TMP12:%.*]] = load i16, ptr [[ARRAYIDX_12]], align 2
-; CHECK-NEXT: [[CONV_12:%.*]] = zext i16 [[TMP12]] to i32
-; CHECK-NEXT: [[ADD_12:%.*]] = add i32 [[ADD_11]], [[CONV_12]]
-; CHECK-NEXT: [[MUL_12:%.*]] = mul nuw nsw i32 [[CONV_12]], [[CONV_12]]
-; CHECK-NEXT: [[ADD11_12:%.*]] = add i32 [[MUL_12]], [[ADD11_11]]
-; CHECK-NEXT: [[ARRAYIDX_13:%.*]] = getelementptr inbounds i16, ptr [[P_ADDR_035]], i64 13
-; CHECK-NEXT: [[TMP13:%.*]] = load i16, ptr [[ARRAYIDX_13]], align 2
-; CHECK-NEXT: [[CONV_13:%.*]] = zext i16 [[TMP13]] to i32
-; CHECK-NEXT: [[ADD_13:%.*]] = add i32 [[ADD_12]], [[CONV_13]]
-; CHECK-NEXT: [[MUL_13:%.*]] = mul nuw nsw i32 [[CONV_13]], [[CONV_13]]
-; CHECK-NEXT: [[ADD11_13:%.*]] = add i32 [[MUL_13]], [[ADD11_12]]
-; CHECK-NEXT: [[ARRAYIDX_14:%.*]] = getelementptr inbounds i16, ptr [[P_ADDR_035]], i64 14
-; CHECK-NEXT: [[TMP14:%.*]] = load i16, ptr [[ARRAYIDX_14]], align 2
-; CHECK-NEXT: [[CONV_14:%.*]] = zext i16 [[TMP14]] to i32
-; CHECK-NEXT: [[ADD_14:%.*]] = add i32 [[ADD_13]], [[CONV_14]]
-; CHECK-NEXT: [[MUL_14:%.*]] = mul nuw nsw i32 [[CONV_14]], [[CONV_14]]
-; CHECK-NEXT: [[ADD11_14:%.*]] = add i32 [[MUL_14]], [[ADD11_13]]
-; CHECK-NEXT: [[ARRAYIDX_15:%.*]] = getelementptr inbounds i16, ptr [[P_ADDR_035]], i64 15
-; CHECK-NEXT: [[TMP15:%.*]] = load i16, ptr [[ARRAYIDX_15]], align 2
-; CHECK-NEXT: [[CONV_15:%.*]] = zext i16 [[TMP15]] to i32
-; CHECK-NEXT: [[OP_RDX1]] = add i32 [[ADD_14]], [[CONV_15]]
-; CHECK-NEXT: [[MUL_15:%.*]] = mul nuw nsw i32 [[CONV_15]], [[CONV_15]]
-; CHECK-NEXT: [[OP_RDX]] = add i32 [[MUL_15]], [[ADD11_14]]
-; CHECK-NEXT: [[ADD_PTR]] = getelementptr inbounds i16, ptr [[P_ADDR_035]], i64 [[IDX_EXT]]
-; CHECK-NEXT: [[INC13]] = add nuw nsw i32 [[Y_038]], 1
-; CHECK-NEXT: [[EXITCOND_NOT:%.*]] = icmp eq i32 [[INC13]], 16
-; CHECK-NEXT: br i1 [[EXITCOND_NOT]], label [[FOR_COND_CLEANUP:%.*]], label [[FOR_COND1_PREHEADER]]
-; CHECK: for.cond.cleanup:
-; CHECK-NEXT: [[CONV15:%.*]] = zext i32 [[OP_RDX1]] to i64
-; CHECK-NEXT: [[CONV16:%.*]] = zext i32 [[OP_RDX]] to i64
-; CHECK-NEXT: [[SHL:%.*]] = shl nuw i64 [[CONV16]], 32
-; CHECK-NEXT: [[ADD17:%.*]] = or i64 [[SHL]], [[CONV15]]
-; CHECK-NEXT: ret i64 [[ADD17]]
-;
-entry:
- %idx.ext = sext i32 %st to i64
- br label %for.cond1.preheader
-
-for.cond1.preheader: ; preds = %entry, %for.cond1.preheader
- %y.038 = phi i32 [ 0, %entry ], [ %inc13, %for.cond1.preheader ]
- %sq.037 = phi i32 [ 0, %entry ], [ %add11.15, %for.cond1.preheader ]
- %sm.036 = phi i32 [ 0, %entry ], [ %add.15, %for.cond1.preheader ]
- %p.addr.035 = phi ptr [ %p, %entry ], [ %add.ptr, %for.cond1.preheader ]
- %0 = load i16, ptr %p.addr.035, align 2
- %conv = zext i16 %0 to i32
- %add = add i32 %sm.036, %conv
- %mul = mul nuw nsw i32 %conv, %conv
- %add11 = add i32 %mul, %sq.037
- %arrayidx.1 = getelementptr inbounds i16, ptr %p.addr.035, i64 1
- %1 = load i16, ptr %arrayidx.1, align 2
- %conv.1 = zext i16 %1 to i32
- %add.1 = add i32 %add, %conv.1
- %mul.1 = mul nuw nsw i32 %conv.1, %conv.1
- %add11.1 = add i32 %mul.1, %add11
- %arrayidx.2 = getelementptr inbounds i16, ptr %p.addr.035, i64 2
- %2 = load i16, ptr %arrayidx.2, align 2
- %conv.2 = zext i16 %2 to i32
- %add.2 = add i32 %add.1, %conv.2
- %mul.2 = mul nuw nsw i32 %conv.2, %conv.2
- %add11.2 = add i32 %mul.2, %add11.1
- %arrayidx.3 = getelementptr inbounds i16, ptr %p.addr.035, i64 3
- %3 = load i16, ptr %arrayidx.3, align 2
- %conv.3 = zext i16 %3 to i32
- %add.3 = add i32 %add.2, %conv.3
- %mul.3 = mul nuw nsw i32 %conv.3, %conv.3
- %add11.3 = add i32 %mul.3, %add11.2
- %arrayidx.4 = getelementptr inbounds i16, ptr %p.addr.035, i64 4
- %4 = load i16, ptr %arrayidx.4, align 2
- %conv.4 = zext i16 %4 to i32
- %add.4 = add i32 %add.3, %conv.4
- %mul.4 = mul nuw nsw i32 %conv.4, %conv.4
- %add11.4 = add i32 %mul.4, %add11.3
- %arrayidx.5 = getelementptr inbounds i16, ptr %p.addr.035, i64 5
- %5 = load i16, ptr %arrayidx.5, align 2
- %conv.5 = zext i16 %5 to i32
- %add.5 = add i32 %add.4, %conv.5
- %mul.5 = mul nuw nsw i32 %conv.5, %conv.5
- %add11.5 = add i32 %mul.5, %add11.4
- %arrayidx.6 = getelementptr inbounds i16, ptr %p.addr.035, i64 6
- %6 = load i16, ptr %arrayidx.6, align 2
- %conv.6 = zext i16 %6 to i32
- %add.6 = add i32 %add.5, %conv.6
- %mul.6 = mul nuw nsw i32 %conv.6, %conv.6
- %add11.6 = add i32 %mul.6, %add11.5
- %arrayidx.7 = getelementptr inbounds i16, ptr %p.addr.035, i64 7
- %7 = load i16, ptr %arrayidx.7, align 2
- %conv.7 = zext i16 %7 to i32
- %add.7 = add i32 %add.6, %conv.7
- %mul.7 = mul nuw nsw i32 %conv.7, %conv.7
- %add11.7 = add i32 %mul.7, %add11.6
- %arrayidx.8 = getelementptr inbounds i16, ptr %p.addr.035, i64 8
- %8 = load i16, ptr %arrayidx.8, align 2
- %conv.8 = zext i16 %8 to i32
- %add.8 = add i32 %add.7, %conv.8
- %mul.8 = mul nuw nsw i32 %conv.8, %conv.8
- %add11.8 = add i32 %mul.8, %add11.7
- %arrayidx.9 = getelementptr inbounds i16, ptr %p.addr.035, i64 9
- %9 = load i16, ptr %arrayidx.9, align 2
- %conv.9 = zext i16 %9 to i32
- %add.9 = add i32 %add.8, %conv.9
- %mul.9 = mul nuw nsw i32 %conv.9, %conv.9
- %add11.9 = add i32 %mul.9, %add11.8
- %arrayidx.10 = getelementptr inbounds i16, ptr %p.addr.035, i64 10
- %10 = load i16, ptr %arrayidx.10, align 2
- %conv.10 = zext i16 %10 to i32
- %add.10 = add i32 %add.9, %conv.10
- %mul.10 = mul nuw nsw i32 %conv.10, %conv.10
- %add11.10 = add i32 %mul.10, %add11.9
- %arrayidx.11 = getelementptr inbounds i16, ptr %p.addr.035, i64 11
- %11 = load i16, ptr %arrayidx.11, align 2
- %conv.11 = zext i16 %11 to i32
- %add.11 = add i32 %add.10, %conv.11
- %mul.11 = mul nuw nsw i32 %conv.11, %conv.11
- %add11.11 = add i32 %mul.11, %add11.10
- %arrayidx.12 = getelementptr inbounds i16, ptr %p.addr.035, i64 12
- %12 = load i16, ptr %arrayidx.12, align 2
- %conv.12 = zext i16 %12 to i32
- %add.12 = add i32 %add.11, %conv.12
- %mul.12 = mul nuw nsw i32 %conv.12, %conv.12
- %add11.12 = add i32 %mul.12, %add11.11
- %arrayidx.13 = getelementptr inbounds i16, ptr %p.addr.035, i64 13
- %13 = load i16, ptr %arrayidx.13, align 2
- %conv.13 = zext i16 %13 to i32
- %add.13 = add i32 %add.12, %conv.13
- %mul.13 = mul nuw nsw i32 %conv.13, %conv.13
- %add11.13 = add i32 %mul.13, %add11.12
- %arrayidx.14 = getelementptr inbounds i16, ptr %p.addr.035, i64 14
- %14 = load i16, ptr %arrayidx.14, align 2
- %conv.14 = zext i16 %14 to i32
- %add.14 = add i32 %add.13, %conv.14
- %mul.14 = mul nuw nsw i32 %conv.14, %conv.14
- %add11.14 = add i32 %mul.14, %add11.13
- %arrayidx.15 = getelementptr inbounds i16, ptr %p.addr.035, i64 15
- %15 = load i16, ptr %arrayidx.15, align 2
- %conv.15 = zext i16 %15 to i32
- %add.15 = add i32 %add.14, %conv.15
- %mul.15 = mul nuw nsw i32 %conv.15, %conv.15
- %add11.15 = add i32 %mul.15, %add11.14
- %add.ptr = getelementptr inbounds i16, ptr %p.addr.035, i64 %idx.ext
- %inc13 = add nuw nsw i32 %y.038, 1
- %exitcond.not = icmp eq i32 %inc13, 16
- br i1 %exitcond.not, label %for.cond.cleanup, label %for.cond1.preheader
-
-for.cond.cleanup: ; preds = %for.cond1.preheader
- %conv15 = zext i32 %add.15 to i64
- %conv16 = zext i32 %add11.15 to i64
- %shl = shl nuw i64 %conv16, 32
- %add17 = or i64 %shl, %conv15
- ret i64 %add17
-}
-
diff --git a/llvm/test/Transforms/SLPVectorizer/RISCV/buildvector-all-external-scalars.ll b/llvm/test/Transforms/SLPVectorizer/RISCV/buildvector-all-external-scalars.ll
index 97ac1f612caf5..1d49e83341f4e 100644
--- a/llvm/test/Transforms/SLPVectorizer/RISCV/buildvector-all-external-scalars.ll
+++ b/llvm/test/Transforms/SLPVectorizer/RISCV/buildvector-all-external-scalars.ll
@@ -234,20 +234,14 @@ define void @test(ptr %__last.addr.011.i.i, ptr %call3) {
; EXP-NEXT: [[TMP8:%.*]] = load float, ptr [[__LAST_ADDR_0_I_I]], align 4
; EXP-NEXT: store float [[TMP8]], ptr [[INCDEC_PTR2_I_I]], align 4
; EXP-NEXT: store float [[TMP7]], ptr [[__LAST_ADDR_0_I_I]], align 4
-; EXP-NEXT: [[INCDEC_PTR2_I_I_1:%.*]] = getelementptr inbounds nuw i8, ptr [[INCDEC_PTR2_I_I]], i32 4
+; EXP-NEXT: [[INCDEC_PTR2_I_I_2:%.*]] = getelementptr inbounds nuw i8, ptr [[INCDEC_PTR2_I_I]], i32 4
; EXP-NEXT: [[__LAST_ADDR_0_I_I_1:%.*]] = getelementptr inbounds i8, ptr [[__LAST_ADDR_0_I_I]], i32 -4
-; EXP-NEXT: [[TMP9:%.*]] = load float, ptr [[INCDEC_PTR2_I_I_1]], align 4
-; EXP-NEXT: [[TMP10:%.*]] = load float, ptr [[__LAST_ADDR_0_I_I_1]], align 4
-; EXP-NEXT: store float [[TMP10]], ptr [[INCDEC_PTR2_I_I_1]], align 4
-; EXP-NEXT: store float [[TMP9]], ptr [[__LAST_ADDR_0_I_I_1]], align 4
-; EXP-NEXT: [[INCDEC_PTR2_I_I_2:%.*]] = getelementptr inbounds nuw i8, ptr [[INCDEC_PTR2_I_I_1]], i32 4
-; EXP-NEXT: [[__LAST_ADDR_0_I_I_2:%.*]] = getelementptr inbounds i8, ptr [[__LAST_ADDR_0_I_I_1]], i32 -4
; EXP-NEXT: [[TMP11:%.*]] = load float, ptr [[INCDEC_PTR2_I_I_2]], align 4
-; EXP-NEXT: [[TMP12:%.*]] = load float, ptr [[__LAST_ADDR_0_I_I_2]], align 4
+; EXP-NEXT: [[TMP12:%.*]] = load float, ptr [[__LAST_ADDR_0_I_I_1]], align 4
; EXP-NEXT: store float [[TMP12]], ptr [[INCDEC_PTR2_I_I_2]], align 4
-; EXP-NEXT: store float [[TMP11]], ptr [[__LAST_ADDR_0_I_I_2]], align 4
+; EXP-NEXT: store float [[TMP11]], ptr [[__LAST_ADDR_0_I_I_1]], align 4
; EXP-NEXT: [[INCDEC_PTR2_I_I_3:%.*]] = getelementptr inbounds nuw i8, ptr [[INCDEC_PTR2_I_I_2]], i32 4
-; EXP-NEXT: [[__LAST_ADDR_0_I_I_3:%.*]] = getelementptr inbounds i8, ptr [[__LAST_ADDR_0_I_I_2]], i32 -4
+; EXP-NEXT: [[__LAST_ADDR_0_I_I_3:%.*]] = getelementptr inbounds i8, ptr [[__LAST_ADDR_0_I_I_1]], i32 -4
; EXP-NEXT: [[TMP13:%.*]] = load float, ptr [[INCDEC_PTR2_I_I_3]], align 4
; EXP-NEXT: [[TMP14:%.*]] = load float, ptr [[__LAST_ADDR_0_I_I_3]], align 4
; EXP-NEXT: store float [[TMP14]], ptr [[INCDEC_PTR2_I_I_3]], align 4
@@ -348,29 +342,35 @@ define void @test(ptr %__last.addr.011.i.i, ptr %call3) {
; EXP-NEXT: [[TMP46:%.*]] = load float, ptr [[__LAST_ADDR_0_I_I_19]], align 4
; EXP-NEXT: store float [[TMP46]], ptr [[INCDEC_PTR2_I_I_19]], align 4
; EXP-NEXT: store float [[TMP45]], ptr [[__LAST_ADDR_0_I_I_19]], align 4
-; EXP-NEXT: [[__LAST_ADDR_0_I_I_20:%.*]] = getelementptr inbounds i8, ptr [[__LAST_ADDR_0_I_I_19]], i32 -4
-; EXP-NEXT: [[TMP47:%.*]] = insertelement <2 x ptr> poison, ptr [[INCDEC_PTR2_I_I_19]], i32 0
-; EXP-NEXT: [[TMP48:%.*]] = insertelement <2 x ptr> [[TMP47]], ptr [[__LAST_ADDR_0_I_I_19]], i32 1
-; EXP-NEXT: [[TMP49:%.*]] = getelementptr inbounds i8, <2 x ptr> [[TMP48]], <2 x i32> <i32 4, i32 -4>
-; EXP-NEXT: [[INCDEC_PTR2_I_I_20:%.*]] = getelementptr inbounds nuw i8, ptr [[INCDEC_PTR2_I_I_19]], i32 4
-; EXP-NEXT: [[TMP50:%.*]] = load float, ptr [[INCDEC_PTR2_I_I_20]], align 4
+; EXP-NEXT: [[__LAST_ADDR_0_I_I_20:%.*]] = getelementptr inbounds nuw i8, ptr [[INCDEC_PTR2_I_I_19]], i32 4
+; EXP-NEXT: [[INCDEC_PTR2_I_I_20:%.*]] = getelementptr inbounds i8, ptr [[__LAST_ADDR_0_I_I_19]], i32 -4
; EXP-NEXT: [[TMP51:%.*]] = load float, ptr [[__LAST_ADDR_0_I_I_20]], align 4
-; EXP-NEXT: store float [[TMP51]], ptr [[INCDEC_PTR2_I_I_20]], align 4
+; EXP-NEXT: [[TMP50:%.*]] = load float, ptr [[INCDEC_PTR2_I_I_20]], align 4
; EXP-NEXT: store float [[TMP50]], ptr [[__LAST_ADDR_0_I_I_20]], align 4
+; EXP-NEXT: store float [[TMP51]], ptr [[INCDEC_PTR2_I_I_20]], align 4
+; EXP-NEXT: [[__LAST_ADDR_0_I_I_21:%.*]] = getelementptr inbounds i8, ptr [[INCDEC_PTR2_I_I_20]], i32 -4
; EXP-NEXT: [[TMP52:%.*]] = insertelement <2 x ptr> poison, ptr [[__LAST_ADDR_0_I_I_20]], i32 0
; EXP-NEXT: [[TMP53:%.*]] = insertelement <2 x ptr> [[TMP52]], ptr [[INCDEC_PTR2_I_I_20]], i32 1
-; EXP-NEXT: [[TMP54:%.*]] = getelementptr inbounds i8, <2 x ptr> [[TMP53]], <2 x i32> <i32 -4, i32 4>
-; EXP-NEXT: [[__LAST_ADDR_0_I_I_21:%.*]] = getelementptr inbounds i8, ptr [[__LAST_ADDR_0_I_I_20]], i32 -4
-; EXP-NEXT: [[TMP55:%.*]] = getelementptr inbounds i8, <2 x ptr> [[TMP49]], <2 x i32> <i32 4, i32 -4>
-; EXP-NEXT: [[INCDEC_PTR2_I_I_21:%.*]] = getelementptr inbounds nuw i8, ptr [[INCDEC_PTR2_I_I_20]], i32 4
+; EXP-NEXT: [[TMP49:%.*]] = getelementptr inbounds i8, <2 x ptr> [[TMP53]], <2 x i32> <i32 4, i32 -4>
+; EXP-NEXT: [[INCDEC_PTR2_I_I_21:%.*]] = getelementptr inbounds nuw i8, ptr [[__LAST_ADDR_0_I_I_20]], i32 4
; EXP-NEXT: [[TMP56:%.*]] = load float, ptr [[INCDEC_PTR2_I_I_21]], align 4
; EXP-NEXT: [[TMP57:%.*]] = load float, ptr [[__LAST_ADDR_0_I_I_21]], align 4
; EXP-NEXT: store float [[TMP57]], ptr [[INCDEC_PTR2_I_I_21]], align 4
; EXP-NEXT: store float [[TMP56]], ptr [[__LAST_ADDR_0_I_I_21]], align 4
+; EXP-NEXT: [[TMP96:%.*]] = insertelement <2 x ptr> poison, ptr [[__LAST_ADDR_0_I_I_21]], i32 0
+; EXP-NEXT: [[TMP97:%.*]] = insertelement <2 x ptr> [[TMP96]], ptr [[INCDEC_PTR2_I_I_21]], i32 1
+; EXP-NEXT: [[TMP54:%.*]] = getelementptr inbounds i8, <2 x ptr> [[TMP97]], <2 x i32> <i32 -4, i32 4>
+; EXP-NEXT: [[__LAST_ADDR_0_I_I_32:%.*]] = getelementptr inbounds i8, ptr [[__LAST_ADDR_0_I_I_21]], i32 -4
+; EXP-NEXT: [[TMP55:%.*]] = getelementptr inbounds i8, <2 x ptr> [[TMP49]], <2 x i32> <i32 4, i32 -4>
+; EXP-NEXT: [[INCDEC_PTR2_I_I_32:%.*]] = getelementptr inbounds nuw i8, ptr [[INCDEC_PTR2_I_I_21]], i32 4
+; EXP-NEXT: [[TMP98:%.*]] = load float, ptr [[INCDEC_PTR2_I_I_32]], align 4
+; EXP-NEXT: [[TMP99:%.*]] = load float, ptr [[__LAST_ADDR_0_I_I_32]], align 4
+; EXP-NEXT: store float [[TMP99]], ptr [[INCDEC_PTR2_I_I_32]], align 4
+; EXP-NEXT: store float [[TMP98]], ptr [[__LAST_ADDR_0_I_I_32]], align 4
; EXP-NEXT: [[TMP58:%.*]] = getelementptr inbounds i8, <2 x ptr> [[TMP54]], <2 x i32> <i32 -4, i32 4>
-; EXP-NEXT: [[__LAST_ADDR_0_I_I_22:%.*]] = getelementptr inbounds i8, ptr [[__LAST_ADDR_0_I_I_21]], i32 -4
+; EXP-NEXT: [[__LAST_ADDR_0_I_I_22:%.*]] = getelementptr inbounds i8, ptr [[__LAST_ADDR_0_I_I_32]], i32 -4
; EXP-NEXT: [[TMP59:%.*]] = getelementptr inbounds i8, <2 x ptr> [[TMP55]], <2 x i32> <i32 4, i32 -4>
-; EXP-NEXT: [[INCDEC_PTR2_I_I_22:%.*]] = getelementptr inbounds nuw i8, ptr [[INCDEC_PTR2_I_I_21]], i32 4
+; EXP-NEXT: [[INCDEC_PTR2_I_I_22:%.*]] = getelementptr inbounds nuw i8, ptr [[INCDEC_PTR2_I_I_32]], i32 4
; EXP-NEXT: [[TMP60:%.*]] = load float, ptr [[INCDEC_PTR2_I_I_22]], align 4
; EXP-NEXT: [[TMP61:%.*]] = load float, ptr [[__LAST_ADDR_0_I_I_22]], align 4
; EXP-NEXT: store float [[TMP61]], ptr [[INCDEC_PTR2_I_I_22]], align 4
diff --git a/llvm/test/Transforms/SLPVectorizer/RISCV/external.ll b/llvm/test/Transforms/SLPVectorizer/RISCV/external.ll
new file mode 100644
index 0000000000000..efe2fdd25e2a0
--- /dev/null
+++ b/llvm/test/Transforms/SLPVectorizer/RISCV/external.ll
@@ -0,0 +1,98 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --version 6
+; RUN: opt < %s -passes=slp-vectorizer -mtriple=riscv64 -mattr=+v -S | FileCheck %s --check-prefixes=DEFAULT
+
+define void @simple_copy(ptr %dest, ptr %p, ptr %dest2, ptr %dest3) {
+;, float %l0, float %l1, float %l2, float %l3, float %l4, float %l5, float %l6, float %l7) {
+; DEFAULT-LABEL: define void @simple_copy(
+; DEFAULT-SAME: ptr [[DEST:%.*]], ptr [[P:%.*]], ptr [[DEST2:%.*]], ptr [[DEST3:%.*]]) #[[ATTR0:[0-9]+]] {
+; DEFAULT-NEXT: [[ENTRY:.*:]]
+; DEFAULT-NEXT: [[INC7:%.*]] = getelementptr inbounds float, ptr [[P]], i64 7
+; DEFAULT-NEXT: [[TMP3:%.*]] = load float, ptr [[INC7]], align 2
+; DEFAULT-NEXT: [[TMP0:%.*]] = load <8 x float>, ptr [[P]], align 4
+; DEFAULT-NEXT: store <8 x float> [[TMP0]], ptr [[DEST]], align 4
+; DEFAULT-NEXT: [[D4:%.*]] = getelementptr inbounds float, ptr [[DEST2]], i64 4
+; DEFAULT-NEXT: [[D5:%.*]] = getelementptr inbounds float, ptr [[DEST2]], i64 5
+; DEFAULT-NEXT: [[D6:%.*]] = getelementptr inbounds float, ptr [[DEST2]], i64 6
+; DEFAULT-NEXT: [[D7:%.*]] = getelementptr inbounds float, ptr [[DEST2]], i64 7
+; DEFAULT-NEXT: [[TMP1:%.*]] = shufflevector <8 x float> [[TMP0]], <8 x float> poison, <4 x i32> <i32 0, i32 1, i32 2, i32 3>
+; DEFAULT-NEXT: store <4 x float> [[TMP1]], ptr [[DEST2]], align 4
+; DEFAULT-NEXT: store float [[TMP3]], ptr [[D7]], align 2
+; DEFAULT-NEXT: ret void
+;
+entry:
+ %inc1 = getelementptr inbounds float, ptr %p, i64 1
+ %inc2 = getelementptr inbounds float, ptr %p, i64 2
+ %inc3 = getelementptr inbounds float, ptr %p, i64 3
+ %inc4 = getelementptr inbounds float, ptr %p, i64 4
+ %inc5 = getelementptr inbounds float, ptr %p, i64 5
+ %inc6 = getelementptr inbounds float, ptr %p, i64 6
+ %inc7 = getelementptr inbounds float, ptr %p, i64 7
+ %l0 = load float, ptr %p, align 4
+ %l1 = load float, ptr %inc1, align 2
+ %l2 = load float, ptr %inc2, align 2
+ %l3 = load float, ptr %inc3, align 2
+ %l4 = load float, ptr %inc4, align 2
+ %l5 = load float, ptr %inc5, align 2
+ %l6 = load float, ptr %inc6, align 2
+ %l7 = load float, ptr %inc7, align 2
+
+; %e0 = fadd float %l0, %l0
+; %e1 = fadd float %l1, %l1
+; %e2 = fadd float %l2, %l2
+; %e3 = fadd float %l3, %l3
+; %e4 = fadd float %l4, %l4
+; %e5 = fadd float %l5, %l5
+; %e6 = fadd float %l6, %l6
+; %e7 = fadd float %l7, %l7
+
+; %red0 = fadd float %e0, %e1
+; %red1 = fadd float %red0, %e2
+; %red2 = fadd float %red1, %e3
+; %red3 = fadd float %red2, %e4
+; %red4 = fadd float %red3, %e5
+; %red5 = fadd float %red4, %e6
+; %red6 = fadd float %red5, %e7
+ %i1 = getelementptr inbounds float, ptr %dest, i64 1
+ %i2 = getelementptr inbounds float, ptr %dest, i64 2
+ %i3 = getelementptr inbounds float, ptr %dest, i64 3
+ %i4 = getelementptr inbounds float, ptr %dest, i64 4
+ %i5 = getelementptr inbounds float, ptr %dest, i64 5
+ %i6 = getelementptr inbounds float, ptr %dest, i64 6
+ %i7 = getelementptr inbounds float, ptr %dest, i64 7
+ store float %l0, ptr %dest, align 4
+ store float %l1, ptr %i1, align 2
+ store float %l2, ptr %i2, align 2
+ store float %l3, ptr %i3, align 2
+ store float %l4, ptr %i4, align 2
+ store float %l5, ptr %i5, align 2
+ store float %l6, ptr %i6, align 2
+ store float %l7, ptr %i7, align 2
+
+ %d1 = getelementptr inbounds float, ptr %dest2, i64 1
+ %d2 = getelementptr inbounds float, ptr %dest2, i64 2
+ %d3 = getelementptr inbounds float, ptr %dest2, i64 3
+ %d4 = getelementptr inbounds float, ptr %dest2, i64 4
+ %d5 = getelementptr inbounds float, ptr %dest2, i64 5
+ %d6 = getelementptr inbounds float, ptr %dest2, i64 6
+ %d7 = getelementptr inbounds float, ptr %dest2, i64 7
+ store float %l0, ptr %dest2, align 4
+ store float %l1, ptr %d1, align 2
+ store float %l2, ptr %d2, align 2
+ store float %l3, ptr %d3, align 2
+; store float %l4, ptr %d4, align 2
+; store float %l5, ptr %d5, align 2
+; store float %l6, ptr %d6, align 2
+ store float %l7, ptr %d7, align 2
+
+; %r0 = fadd float %e0, %e1
+; %r1 = fadd float %r0, %e2
+; %r2 = fadd float %r1, %e3
+; %r3 = fadd float %r2, %e4
+; %r4 = fadd float %r3, %e5
+; %r5 = fadd float %r4, %e6
+; %r6 = fadd float %r5, %e7
+; store float %r2, ptr %dest2, align 4
+
+; store float %e7, ptr %dest3, align 4
+ ret void
+}
diff --git a/llvm/test/Transforms/SLPVectorizer/RISCV/mixed-extracts-types.ll b/llvm/test/Transforms/SLPVectorizer/RISCV/mixed-extracts-types.ll
index 56555efb063ed..125fe69820d5c 100644
--- a/llvm/test/Transforms/SLPVectorizer/RISCV/mixed-extracts-types.ll
+++ b/llvm/test/Transforms/SLPVectorizer/RISCV/mixed-extracts-types.ll
@@ -6,15 +6,15 @@ define i32 @test() {
; CHECK-SAME: ) #[[ATTR0:[0-9]+]] {
; CHECK-NEXT: entry:
; CHECK-NEXT: [[VECTOR_RECUR_EXTRACT:%.*]] = extractelement <vscale x 4 x i8> zeroinitializer, i64 0
+; CHECK-NEXT: [[CONV5:%.*]] = sext i8 [[VECTOR_RECUR_EXTRACT]] to i32
+; CHECK-NEXT: store i32 [[CONV5]], ptr getelementptr ([0 x i32], ptr null, i64 0, i64 -14), align 4
; CHECK-NEXT: [[TMP0:%.*]] = load <2 x i8>, ptr getelementptr ([9 x i8], ptr null, i64 -2, i64 5), align 1
; CHECK-NEXT: [[TMP3:%.*]] = load i8, ptr getelementptr ([9 x i8], ptr null, i64 -2, i64 5), align 1
; CHECK-NEXT: [[TMP1:%.*]] = icmp ne <2 x i8> [[TMP0]], zeroinitializer
; CHECK-NEXT: [[TMP2:%.*]] = zext <2 x i1> [[TMP1]] to <2 x i16>
; CHECK-NEXT: store <2 x i16> [[TMP2]], ptr getelementptr ([0 x i16], ptr null, i64 0, i64 -14), align 2
-; CHECK-NEXT: [[TMP4:%.*]] = insertelement <2 x i8> poison, i8 [[VECTOR_RECUR_EXTRACT]], i32 0
-; CHECK-NEXT: [[TMP5:%.*]] = insertelement <2 x i8> [[TMP4]], i8 [[TMP3]], i32 1
-; CHECK-NEXT: [[TMP6:%.*]] = sext <2 x i8> [[TMP5]] to <2 x i32>
-; CHECK-NEXT: store <2 x i32> [[TMP6]], ptr getelementptr ([0 x i32], ptr null, i64 0, i64 -14), align 4
+; CHECK-NEXT: [[CONV5_1:%.*]] = sext i8 [[TMP3]] to i32
+; CHECK-NEXT: store i32 [[CONV5_1]], ptr getelementptr ([0 x i32], ptr null, i64 0, i64 -13), align 4
; CHECK-NEXT: ret i32 0
;
entry:
diff --git a/llvm/test/Transforms/SLPVectorizer/RISCV/runtime-strided-stores.ll b/llvm/test/Transforms/SLPVectorizer/RISCV/runtime-strided-stores.ll
index 0d5b1271bc86e..a6b4a48566aaa 100644
--- a/llvm/test/Transforms/SLPVectorizer/RISCV/runtime-strided-stores.ll
+++ b/llvm/test/Transforms/SLPVectorizer/RISCV/runtime-strided-stores.ll
@@ -6,7 +6,21 @@ define void @runtime_stride(ptr %pl, ptr %ps, i64 %stride) {
; CHECK-LABEL: define void @runtime_stride(
; CHECK-SAME: ptr [[PL:%.*]], ptr [[PS:%.*]], i64 [[STRIDE:%.*]]) #[[ATTR0:[0-9]+]] {
; CHECK-NEXT: [[GEP_L0:%.*]] = getelementptr i8, ptr [[PL]], i64 0
-; CHECK-NEXT: [[TMP1:%.*]] = load <8 x i8>, ptr [[GEP_L0]], align 1
+; CHECK-NEXT: [[GEP_L1:%.*]] = getelementptr i8, ptr [[PL]], i64 1
+; CHECK-NEXT: [[GEP_L2:%.*]] = getelementptr i8, ptr [[PL]], i64 2
+; CHECK-NEXT: [[GEP_L3:%.*]] = getelementptr i8, ptr [[PL]], i64 3
+; CHECK-NEXT: [[GEP_L4:%.*]] = getelementptr i8, ptr [[PL]], i64 4
+; CHECK-NEXT: [[GEP_L5:%.*]] = getelementptr i8, ptr [[PL]], i64 5
+; CHECK-NEXT: [[GEP_L6:%.*]] = getelementptr i8, ptr [[PL]], i64 6
+; CHECK-NEXT: [[GEP_L7:%.*]] = getelementptr i8, ptr [[PL]], i64 7
+; CHECK-NEXT: [[TMP9:%.*]] = load i8, ptr [[GEP_L7]], align 1
+; CHECK-NEXT: [[TMP8:%.*]] = load i8, ptr [[GEP_L6]], align 1
+; CHECK-NEXT: [[TMP7:%.*]] = load i8, ptr [[GEP_L5]], align 1
+; CHECK-NEXT: [[TMP6:%.*]] = load i8, ptr [[GEP_L4]], align 1
+; CHECK-NEXT: [[TMP5:%.*]] = load i8, ptr [[GEP_L3]], align 1
+; CHECK-NEXT: [[TMP4:%.*]] = load i8, ptr [[GEP_L2]], align 1
+; CHECK-NEXT: [[TMP3:%.*]] = load i8, ptr [[GEP_L1]], align 1
+; CHECK-NEXT: [[TMP2:%.*]] = load i8, ptr [[GEP_L0]], align 1
; CHECK-NEXT: [[GEP_S0:%.*]] = getelementptr i8, ptr [[PS]], i64 0
; CHECK-NEXT: [[GEP_S1:%.*]] = getelementptr i8, ptr [[GEP_S0]], i64 [[STRIDE]]
; CHECK-NEXT: [[GEP_S2:%.*]] = getelementptr i8, ptr [[GEP_S1]], i64 [[STRIDE]]
@@ -15,21 +29,13 @@ define void @runtime_stride(ptr %pl, ptr %ps, i64 %stride) {
; CHECK-NEXT: [[GEP_S5:%.*]] = getelementptr i8, ptr [[GEP_S4]], i64 [[STRIDE]]
; CHECK-NEXT: [[GEP_S6:%.*]] = getelementptr i8, ptr [[GEP_S5]], i64 [[STRIDE]]
; CHECK-NEXT: [[GEP_S7:%.*]] = getelementptr i8, ptr [[GEP_S6]], i64 [[STRIDE]]
-; CHECK-NEXT: [[TMP2:%.*]] = extractelement <8 x i8> [[TMP1]], i32 0
; CHECK-NEXT: store i8 [[TMP2]], ptr [[GEP_S0]], align 1
-; CHECK-NEXT: [[TMP3:%.*]] = extractelement <8 x i8> [[TMP1]], i32 1
; CHECK-NEXT: store i8 [[TMP3]], ptr [[GEP_S1]], align 1
-; CHECK-NEXT: [[TMP4:%.*]] = extractelement <8 x i8> [[TMP1]], i32 2
; CHECK-NEXT: store i8 [[TMP4]], ptr [[GEP_S2]], align 1
-; CHECK-NEXT: [[TMP5:%.*]] = extractelement <8 x i8> [[TMP1]], i32 3
; CHECK-NEXT: store i8 [[TMP5]], ptr [[GEP_S3]], align 1
-; CHECK-NEXT: [[TMP6:%.*]] = extractelement <8 x i8> [[TMP1]], i32 4
; CHECK-NEXT: store i8 [[TMP6]], ptr [[GEP_S4]], align 1
-; CHECK-NEXT: [[TMP7:%.*]] = extractelement <8 x i8> [[TMP1]], i32 5
; CHECK-NEXT: store i8 [[TMP7]], ptr [[GEP_S5]], align 1
-; CHECK-NEXT: [[TMP8:%.*]] = extractelement <8 x i8> [[TMP1]], i32 6
; CHECK-NEXT: store i8 [[TMP8]], ptr [[GEP_S6]], align 1
-; CHECK-NEXT: [[TMP9:%.*]] = extractelement <8 x i8> [[TMP1]], i32 7
; CHECK-NEXT: store i8 [[TMP9]], ptr [[GEP_S7]], align 1
; CHECK-NEXT: ret void
;
@@ -76,7 +82,21 @@ define void @runtime_stride_complex_scev(ptr %pl, ptr %ps, i64 %stride, i64 %str
; CHECK-LABEL: define void @runtime_stride_complex_scev(
; CHECK-SAME: ptr [[PL:%.*]], ptr [[PS:%.*]], i64 [[STRIDE:%.*]], i64 [[STRIDE1:%.*]]) #[[ATTR0]] {
; CHECK-NEXT: [[GEP_L0:%.*]] = getelementptr i8, ptr [[PL]], i64 0
-; CHECK-NEXT: [[TMP1:%.*]] = load <8 x i8>, ptr [[GEP_L0]], align 1
+; CHECK-NEXT: [[GEP_L1:%.*]] = getelementptr i8, ptr [[PL]], i64 1
+; CHECK-NEXT: [[GEP_L2:%.*]] = getelementptr i8, ptr [[PL]], i64 2
+; CHECK-NEXT: [[GEP_L3:%.*]] = getelementptr i8, ptr [[PL]], i64 3
+; CHECK-NEXT: [[GEP_L4:%.*]] = getelementptr i8, ptr [[PL]], i64 4
+; CHECK-NEXT: [[GEP_L5:%.*]] = getelementptr i8, ptr [[PL]], i64 5
+; CHECK-NEXT: [[GEP_L6:%.*]] = getelementptr i8, ptr [[PL]], i64 6
+; CHECK-NEXT: [[GEP_L7:%.*]] = getelementptr i8, ptr [[PL]], i64 7
+; CHECK-NEXT: [[TMP9:%.*]] = load i8, ptr [[GEP_L7]], align 1
+; CHECK-NEXT: [[TMP8:%.*]] = load i8, ptr [[GEP_L6]], align 1
+; CHECK-NEXT: [[TMP7:%.*]] = load i8, ptr [[GEP_L5]], align 1
+; CHECK-NEXT: [[TMP6:%.*]] = load i8, ptr [[GEP_L4]], align 1
+; CHECK-NEXT: [[TMP5:%.*]] = load i8, ptr [[GEP_L3]], align 1
+; CHECK-NEXT: [[TMP4:%.*]] = load i8, ptr [[GEP_L2]], align 1
+; CHECK-NEXT: [[TMP3:%.*]] = load i8, ptr [[GEP_L1]], align 1
+; CHECK-NEXT: [[TMP2:%.*]] = load i8, ptr [[GEP_L0]], align 1
; CHECK-NEXT: [[GEP_S0:%.*]] = getelementptr i8, ptr [[PS]], i64 0
; CHECK-NEXT: [[GEP_S1:%.*]] = getelementptr i8, ptr [[GEP_S0]], i64 [[STRIDE]]
; CHECK-NEXT: [[GEP_S2:%.*]] = getelementptr i8, ptr [[GEP_S1]], i64 [[STRIDE]]
@@ -93,21 +113,13 @@ define void @runtime_stride_complex_scev(ptr %pl, ptr %ps, i64 %stride, i64 %str
; CHECK-NEXT: [[GEP_SS5:%.*]] = getelementptr i8, ptr [[GEP_S5]], i64 [[STRIDE1]]
; CHECK-NEXT: [[GEP_SS6:%.*]] = getelementptr i8, ptr [[GEP_S6]], i64 [[STRIDE1]]
; CHECK-NEXT: [[GEP_SS7:%.*]] = getelementptr i8, ptr [[GEP_S7]], i64 [[STRIDE1]]
-; CHECK-NEXT: [[TMP2:%.*]] = extractelement <8 x i8> [[TMP1]], i32 0
; CHECK-NEXT: store i8 [[TMP2]], ptr [[GEP_SS0]], align 1
-; CHECK-NEXT: [[TMP3:%.*]] = extractelement <8 x i8> [[TMP1]], i32 1
; CHECK-NEXT: store i8 [[TMP3]], ptr [[GEP_SS1]], align 1
-; CHECK-NEXT: [[TMP4:%.*]] = extractelement <8 x i8> [[TMP1]], i32 2
; CHECK-NEXT: store i8 [[TMP4]], ptr [[GEP_SS2]], align 1
-; CHECK-NEXT: [[TMP5:%.*]] = extractelement <8 x i8> [[TMP1]], i32 3
; CHECK-NEXT: store i8 [[TMP5]], ptr [[GEP_SS3]], align 1
-; CHECK-NEXT: [[TMP6:%.*]] = extractelement <8 x i8> [[TMP1]], i32 4
; CHECK-NEXT: store i8 [[TMP6]], ptr [[GEP_SS4]], align 1
-; CHECK-NEXT: [[TMP7:%.*]] = extractelement <8 x i8> [[TMP1]], i32 5
; CHECK-NEXT: store i8 [[TMP7]], ptr [[GEP_SS5]], align 1
-; CHECK-NEXT: [[TMP8:%.*]] = extractelement <8 x i8> [[TMP1]], i32 6
; CHECK-NEXT: store i8 [[TMP8]], ptr [[GEP_SS6]], align 1
-; CHECK-NEXT: [[TMP9:%.*]] = extractelement <8 x i8> [[TMP1]], i32 7
; CHECK-NEXT: store i8 [[TMP9]], ptr [[GEP_SS7]], align 1
; CHECK-NEXT: ret void
;
@@ -164,7 +176,21 @@ define void @two_runtime_strides(ptr %pl, ptr %ps, i64 %stride) {
; CHECK-LABEL: define void @two_runtime_strides(
; CHECK-SAME: ptr [[PL:%.*]], ptr [[PS:%.*]], i64 [[STRIDE:%.*]]) #[[ATTR0]] {
; CHECK-NEXT: [[GEP_L0:%.*]] = getelementptr i8, ptr [[PL]], i64 0
-; CHECK-NEXT: [[TMP1:%.*]] = load <8 x i8>, ptr [[GEP_L0]], align 1
+; CHECK-NEXT: [[GEP_L1:%.*]] = getelementptr i8, ptr [[PL]], i64 1
+; CHECK-NEXT: [[GEP_L2:%.*]] = getelementptr i8, ptr [[PL]], i64 2
+; CHECK-NEXT: [[GEP_L3:%.*]] = getelementptr i8, ptr [[PL]], i64 3
+; CHECK-NEXT: [[GEP_L4:%.*]] = getelementptr i8, ptr [[PL]], i64 4
+; CHECK-NEXT: [[GEP_L5:%.*]] = getelementptr i8, ptr [[PL]], i64 5
+; CHECK-NEXT: [[GEP_L6:%.*]] = getelementptr i8, ptr [[PL]], i64 6
+; CHECK-NEXT: [[GEP_L7:%.*]] = getelementptr i8, ptr [[PL]], i64 7
+; CHECK-NEXT: [[TMP9:%.*]] = load i8, ptr [[GEP_L7]], align 1
+; CHECK-NEXT: [[TMP8:%.*]] = load i8, ptr [[GEP_L6]], align 1
+; CHECK-NEXT: [[TMP7:%.*]] = load i8, ptr [[GEP_L5]], align 1
+; CHECK-NEXT: [[TMP6:%.*]] = load i8, ptr [[GEP_L4]], align 1
+; CHECK-NEXT: [[TMP5:%.*]] = load i8, ptr [[GEP_L3]], align 1
+; CHECK-NEXT: [[TMP4:%.*]] = load i8, ptr [[GEP_L2]], align 1
+; CHECK-NEXT: [[TMP3:%.*]] = load i8, ptr [[GEP_L1]], align 1
+; CHECK-NEXT: [[TMP2:%.*]] = load i8, ptr [[GEP_L0]], align 1
; CHECK-NEXT: [[PS1:%.*]] = mul i64 [[STRIDE]], 16
; CHECK-NEXT: [[GEP_S0:%.*]] = getelementptr i8, ptr [[PS]], i64 0
; CHECK-NEXT: [[GEP_S1:%.*]] = getelementptr i8, ptr [[GEP_S0]], i64 [[STRIDE]]
@@ -174,21 +200,13 @@ define void @two_runtime_strides(ptr %pl, ptr %ps, i64 %stride) {
; CHECK-NEXT: [[GEP_S5:%.*]] = getelementptr i8, ptr [[GEP_S4]], i64 [[STRIDE]]
; CHECK-NEXT: [[GEP_S6:%.*]] = getelementptr i8, ptr [[GEP_S5]], i64 [[STRIDE]]
; CHECK-NEXT: [[GEP_S7:%.*]] = getelementptr i8, ptr [[GEP_S6]], i64 [[STRIDE]]
-; CHECK-NEXT: [[TMP2:%.*]] = extractelement <8 x i8> [[TMP1]], i32 0
; CHECK-NEXT: store i8 [[TMP2]], ptr [[GEP_S0]], align 1
-; CHECK-NEXT: [[TMP3:%.*]] = extractelement <8 x i8> [[TMP1]], i32 1
; CHECK-NEXT: store i8 [[TMP3]], ptr [[GEP_S1]], align 1
-; CHECK-NEXT: [[TMP4:%.*]] = extractelement <8 x i8> [[TMP1]], i32 2
; CHECK-NEXT: store i8 [[TMP4]], ptr [[GEP_S2]], align 1
-; CHECK-NEXT: [[TMP5:%.*]] = extractelement <8 x i8> [[TMP1]], i32 3
; CHECK-NEXT: store i8 [[TMP5]], ptr [[GEP_S3]], align 1
-; CHECK-NEXT: [[TMP6:%.*]] = extractelement <8 x i8> [[TMP1]], i32 4
; CHECK-NEXT: store i8 [[TMP6]], ptr [[GEP_S4]], align 1
-; CHECK-NEXT: [[TMP7:%.*]] = extractelement <8 x i8> [[TMP1]], i32 5
; CHECK-NEXT: store i8 [[TMP7]], ptr [[GEP_S5]], align 1
-; CHECK-NEXT: [[TMP8:%.*]] = extractelement <8 x i8> [[TMP1]], i32 6
; CHECK-NEXT: store i8 [[TMP8]], ptr [[GEP_S6]], align 1
-; CHECK-NEXT: [[TMP9:%.*]] = extractelement <8 x i8> [[TMP1]], i32 7
; CHECK-NEXT: store i8 [[TMP9]], ptr [[GEP_S7]], align 1
; CHECK-NEXT: ret void
;
@@ -236,7 +254,21 @@ define void @runtime_strides_constant_gap(ptr %pl, ptr %ps, i64 %stride) {
; CHECK-LABEL: define void @runtime_strides_constant_gap(
; CHECK-SAME: ptr [[PL:%.*]], ptr [[PS:%.*]], i64 [[STRIDE:%.*]]) #[[ATTR0]] {
; CHECK-NEXT: [[GEP_L0:%.*]] = getelementptr i8, ptr [[PL]], i64 0
-; CHECK-NEXT: [[TMP1:%.*]] = load <8 x i8>, ptr [[GEP_L0]], align 1
+; CHECK-NEXT: [[GEP_L1:%.*]] = getelementptr i8, ptr [[PL]], i64 1
+; CHECK-NEXT: [[GEP_L2:%.*]] = getelementptr i8, ptr [[PL]], i64 2
+; CHECK-NEXT: [[GEP_L3:%.*]] = getelementptr i8, ptr [[PL]], i64 3
+; CHECK-NEXT: [[GEP_L4:%.*]] = getelementptr i8, ptr [[PL]], i64 4
+; CHECK-NEXT: [[GEP_L5:%.*]] = getelementptr i8, ptr [[PL]], i64 5
+; CHECK-NEXT: [[GEP_L6:%.*]] = getelementptr i8, ptr [[PL]], i64 6
+; CHECK-NEXT: [[GEP_L7:%.*]] = getelementptr i8, ptr [[PL]], i64 7
+; CHECK-NEXT: [[TMP9:%.*]] = load i8, ptr [[GEP_L7]], align 1
+; CHECK-NEXT: [[TMP8:%.*]] = load i8, ptr [[GEP_L6]], align 1
+; CHECK-NEXT: [[TMP7:%.*]] = load i8, ptr [[GEP_L5]], align 1
+; CHECK-NEXT: [[TMP6:%.*]] = load i8, ptr [[GEP_L4]], align 1
+; CHECK-NEXT: [[TMP5:%.*]] = load i8, ptr [[GEP_L3]], align 1
+; CHECK-NEXT: [[TMP4:%.*]] = load i8, ptr [[GEP_L2]], align 1
+; CHECK-NEXT: [[TMP3:%.*]] = load i8, ptr [[GEP_L1]], align 1
+; CHECK-NEXT: [[TMP2:%.*]] = load i8, ptr [[GEP_L0]], align 1
; CHECK-NEXT: [[GEP_S0:%.*]] = getelementptr i8, ptr [[PS]], i64 0
; CHECK-NEXT: [[GEP_S1:%.*]] = getelementptr i8, ptr [[GEP_S0]], i64 [[STRIDE]]
; CHECK-NEXT: [[GEP_S2:%.*]] = getelementptr i8, ptr [[GEP_S1]], i64 [[STRIDE]]
@@ -245,21 +277,13 @@ define void @runtime_strides_constant_gap(ptr %pl, ptr %ps, i64 %stride) {
; CHECK-NEXT: [[GEP_S5:%.*]] = getelementptr i8, ptr [[GEP_S4]], i64 [[STRIDE]]
; CHECK-NEXT: [[GEP_S6:%.*]] = getelementptr i8, ptr [[GEP_S5]], i64 [[STRIDE]]
; CHECK-NEXT: [[GEP_S7:%.*]] = getelementptr i8, ptr [[GEP_S6]], i64 [[STRIDE]]
-; CHECK-NEXT: [[TMP2:%.*]] = extractelement <8 x i8> [[TMP1]], i32 0
; CHECK-NEXT: store i8 [[TMP2]], ptr [[GEP_S0]], align 1
-; CHECK-NEXT: [[TMP3:%.*]] = extractelement <8 x i8> [[TMP1]], i32 1
; CHECK-NEXT: store i8 [[TMP3]], ptr [[GEP_S1]], align 1
-; CHECK-NEXT: [[TMP4:%.*]] = extractelement <8 x i8> [[TMP1]], i32 2
; CHECK-NEXT: store i8 [[TMP4]], ptr [[GEP_S2]], align 1
-; CHECK-NEXT: [[TMP5:%.*]] = extractelement <8 x i8> [[TMP1]], i32 3
; CHECK-NEXT: store i8 [[TMP5]], ptr [[GEP_S3]], align 1
-; CHECK-NEXT: [[TMP6:%.*]] = extractelement <8 x i8> [[TMP1]], i32 4
; CHECK-NEXT: store i8 [[TMP6]], ptr [[GEP_S4]], align 1
-; CHECK-NEXT: [[TMP7:%.*]] = extractelement <8 x i8> [[TMP1]], i32 5
; CHECK-NEXT: store i8 [[TMP7]], ptr [[GEP_S5]], align 1
-; CHECK-NEXT: [[TMP8:%.*]] = extractelement <8 x i8> [[TMP1]], i32 6
; CHECK-NEXT: store i8 [[TMP8]], ptr [[GEP_S6]], align 1
-; CHECK-NEXT: [[TMP9:%.*]] = extractelement <8 x i8> [[TMP1]], i32 7
; CHECK-NEXT: store i8 [[TMP9]], ptr [[GEP_S7]], align 1
; CHECK-NEXT: ret void
;
@@ -306,7 +330,21 @@ define void @overlapping_strides(ptr %pl, ptr %ps, i64 %stride) {
; CHECK-LABEL: define void @overlapping_strides(
; CHECK-SAME: ptr [[PL:%.*]], ptr [[PS:%.*]], i64 [[STRIDE:%.*]]) #[[ATTR0]] {
; CHECK-NEXT: [[GEP_L0:%.*]] = getelementptr i8, ptr [[PL]], i64 0
-; CHECK-NEXT: [[TMP1:%.*]] = load <8 x i8>, ptr [[GEP_L0]], align 1
+; CHECK-NEXT: [[GEP_L1:%.*]] = getelementptr i8, ptr [[PL]], i64 1
+; CHECK-NEXT: [[GEP_L2:%.*]] = getelementptr i8, ptr [[PL]], i64 2
+; CHECK-NEXT: [[GEP_L3:%.*]] = getelementptr i8, ptr [[PL]], i64 3
+; CHECK-NEXT: [[GEP_L4:%.*]] = getelementptr i8, ptr [[PL]], i64 4
+; CHECK-NEXT: [[GEP_L5:%.*]] = getelementptr i8, ptr [[PL]], i64 5
+; CHECK-NEXT: [[GEP_L6:%.*]] = getelementptr i8, ptr [[PL]], i64 6
+; CHECK-NEXT: [[GEP_L7:%.*]] = getelementptr i8, ptr [[PL]], i64 7
+; CHECK-NEXT: [[TMP9:%.*]] = load i8, ptr [[GEP_L7]], align 1
+; CHECK-NEXT: [[TMP8:%.*]] = load i8, ptr [[GEP_L6]], align 1
+; CHECK-NEXT: [[TMP7:%.*]] = load i8, ptr [[GEP_L5]], align 1
+; CHECK-NEXT: [[TMP6:%.*]] = load i8, ptr [[GEP_L4]], align 1
+; CHECK-NEXT: [[TMP5:%.*]] = load i8, ptr [[GEP_L3]], align 1
+; CHECK-NEXT: [[TMP4:%.*]] = load i8, ptr [[GEP_L2]], align 1
+; CHECK-NEXT: [[TMP3:%.*]] = load i8, ptr [[GEP_L1]], align 1
+; CHECK-NEXT: [[TMP2:%.*]] = load i8, ptr [[GEP_L0]], align 1
; CHECK-NEXT: [[STRIDE2:%.*]] = mul i64 [[STRIDE]], 1
; CHECK-NEXT: [[STRIDE1:%.*]] = mul i64 [[STRIDE]], 2
; CHECK-NEXT: [[GEP_S0:%.*]] = getelementptr i8, ptr [[PS]], i64 0
@@ -317,21 +355,13 @@ define void @overlapping_strides(ptr %pl, ptr %ps, i64 %stride) {
; CHECK-NEXT: [[GEP_S5:%.*]] = getelementptr i8, ptr [[GEP_S4]], i64 [[STRIDE2]]
; CHECK-NEXT: [[GEP_S6:%.*]] = getelementptr i8, ptr [[GEP_S5]], i64 [[STRIDE2]]
; CHECK-NEXT: [[GEP_S7:%.*]] = getelementptr i8, ptr [[GEP_S6]], i64 [[STRIDE2]]
-; CHECK-NEXT: [[TMP2:%.*]] = extractelement <8 x i8> [[TMP1]], i32 0
; CHECK-NEXT: store i8 [[TMP2]], ptr [[GEP_S0]], align 1
-; CHECK-NEXT: [[TMP3:%.*]] = extractelement <8 x i8> [[TMP1]], i32 1
; CHECK-NEXT: store i8 [[TMP3]], ptr [[GEP_S1]], align 1
-; CHECK-NEXT: [[TMP4:%.*]] = extractelement <8 x i8> [[TMP1]], i32 2
; CHECK-NEXT: store i8 [[TMP4]], ptr [[GEP_S2]], align 1
-; CHECK-NEXT: [[TMP5:%.*]] = extractelement <8 x i8> [[TMP1]], i32 3
; CHECK-NEXT: store i8 [[TMP5]], ptr [[GEP_S3]], align 1
-; CHECK-NEXT: [[TMP6:%.*]] = extractelement <8 x i8> [[TMP1]], i32 4
; CHECK-NEXT: store i8 [[TMP6]], ptr [[GEP_S4]], align 1
-; CHECK-NEXT: [[TMP7:%.*]] = extractelement <8 x i8> [[TMP1]], i32 5
; CHECK-NEXT: store i8 [[TMP7]], ptr [[GEP_S5]], align 1
-; CHECK-NEXT: [[TMP8:%.*]] = extractelement <8 x i8> [[TMP1]], i32 6
; CHECK-NEXT: store i8 [[TMP8]], ptr [[GEP_S6]], align 1
-; CHECK-NEXT: [[TMP9:%.*]] = extractelement <8 x i8> [[TMP1]], i32 7
; CHECK-NEXT: store i8 [[TMP9]], ptr [[GEP_S7]], align 1
; CHECK-NEXT: ret void
;
@@ -380,9 +410,14 @@ define void @runtime_stride_unit_stride(ptr %pl, ptr %ps, i64 %stride) {
; CHECK-LABEL: define void @runtime_stride_unit_stride(
; CHECK-SAME: ptr [[PL:%.*]], ptr [[PS:%.*]], i64 [[STRIDE:%.*]]) #[[ATTR0]] {
; CHECK-NEXT: [[GEP_L0:%.*]] = getelementptr i8, ptr [[PL]], i64 0
+; CHECK-NEXT: [[GEP_L1:%.*]] = getelementptr i8, ptr [[PL]], i64 1
+; CHECK-NEXT: [[GEP_L2:%.*]] = getelementptr i8, ptr [[PL]], i64 2
; CHECK-NEXT: [[GEP_L3:%.*]] = getelementptr i8, ptr [[PL]], i64 3
-; CHECK-NEXT: [[TMP3:%.*]] = call <8 x i8> @llvm.masked.load.v8i8.p0(ptr align 1 [[GEP_L0]], <8 x i1> <i1 true, i1 true, i1 true, i1 false, i1 false, i1 false, i1 false, i1 true>, <8 x i8> poison)
-; CHECK-NEXT: [[TMP2:%.*]] = shufflevector <8 x i8> [[TMP3]], <8 x i8> poison, <4 x i32> <i32 0, i32 1, i32 2, i32 7>
+; CHECK-NEXT: [[GEP_L7:%.*]] = getelementptr i8, ptr [[PL]], i64 7
+; CHECK-NEXT: [[LOAD7:%.*]] = load i8, ptr [[GEP_L7]], align 1
+; CHECK-NEXT: [[LOAD2:%.*]] = load i8, ptr [[GEP_L2]], align 1
+; CHECK-NEXT: [[TMP5:%.*]] = load i8, ptr [[GEP_L1]], align 1
+; CHECK-NEXT: [[TMP4:%.*]] = load i8, ptr [[GEP_L0]], align 1
; CHECK-NEXT: [[GEP_S0:%.*]] = getelementptr i8, ptr [[PS]], i64 0
; CHECK-NEXT: [[GEP_S1:%.*]] = getelementptr i8, ptr [[GEP_S0]], i64 [[STRIDE]]
; CHECK-NEXT: [[GEP_S2:%.*]] = getelementptr i8, ptr [[GEP_S1]], i64 [[STRIDE]]
@@ -392,14 +427,10 @@ define void @runtime_stride_unit_stride(ptr %pl, ptr %ps, i64 %stride) {
; CHECK-NEXT: [[GEP_S6:%.*]] = getelementptr i8, ptr [[GEP_S5]], i64 1
; CHECK-NEXT: [[GEP_S7:%.*]] = getelementptr i8, ptr [[GEP_S6]], i64 1
; CHECK-NEXT: [[TMP1:%.*]] = load <4 x i8>, ptr [[GEP_L3]], align 1
-; CHECK-NEXT: [[TMP4:%.*]] = extractelement <4 x i8> [[TMP2]], i32 0
; CHECK-NEXT: store i8 [[TMP4]], ptr [[GEP_S0]], align 1
-; CHECK-NEXT: [[TMP5:%.*]] = extractelement <4 x i8> [[TMP2]], i32 1
; CHECK-NEXT: store i8 [[TMP5]], ptr [[GEP_S1]], align 1
-; CHECK-NEXT: [[LOAD2:%.*]] = extractelement <4 x i8> [[TMP2]], i32 2
; CHECK-NEXT: store i8 [[LOAD2]], ptr [[GEP_S2]], align 1
; CHECK-NEXT: store <4 x i8> [[TMP1]], ptr [[GEP_S3]], align 1
-; CHECK-NEXT: [[LOAD7:%.*]] = extractelement <4 x i8> [[TMP2]], i32 3
; CHECK-NEXT: store i8 [[LOAD7]], ptr [[GEP_S7]], align 1
; CHECK-NEXT: ret void
;
@@ -447,7 +478,13 @@ define void @unit_stride_runtime_stride(ptr %pl, ptr %ps, i64 %stride) {
; CHECK-SAME: ptr [[PL:%.*]], ptr [[PS:%.*]], i64 [[STRIDE:%.*]]) #[[ATTR0]] {
; CHECK-NEXT: [[GEP_L0:%.*]] = getelementptr i8, ptr [[PL]], i64 0
; CHECK-NEXT: [[GEP_L4:%.*]] = getelementptr i8, ptr [[PL]], i64 4
-; CHECK-NEXT: [[TMP2:%.*]] = load <4 x i8>, ptr [[GEP_L4]], align 1
+; CHECK-NEXT: [[GEP_L5:%.*]] = getelementptr i8, ptr [[PL]], i64 5
+; CHECK-NEXT: [[GEP_L6:%.*]] = getelementptr i8, ptr [[PL]], i64 6
+; CHECK-NEXT: [[GEP_L7:%.*]] = getelementptr i8, ptr [[PL]], i64 7
+; CHECK-NEXT: [[TMP6:%.*]] = load i8, ptr [[GEP_L7]], align 1
+; CHECK-NEXT: [[TMP5:%.*]] = load i8, ptr [[GEP_L6]], align 1
+; CHECK-NEXT: [[TMP4:%.*]] = load i8, ptr [[GEP_L5]], align 1
+; CHECK-NEXT: [[TMP3:%.*]] = load i8, ptr [[GEP_L4]], align 1
; CHECK-NEXT: [[GEP_S0:%.*]] = getelementptr i8, ptr [[PS]], i64 0
; CHECK-NEXT: [[GEP_S1:%.*]] = getelementptr i8, ptr [[GEP_S0]], i64 1
; CHECK-NEXT: [[GEP_S2:%.*]] = getelementptr i8, ptr [[GEP_S1]], i64 1
@@ -458,13 +495,9 @@ define void @unit_stride_runtime_stride(ptr %pl, ptr %ps, i64 %stride) {
; CHECK-NEXT: [[GEP_S7:%.*]] = getelementptr i8, ptr [[GEP_S6]], i64 [[STRIDE]]
; CHECK-NEXT: [[TMP1:%.*]] = load <4 x i8>, ptr [[GEP_L0]], align 1
; CHECK-NEXT: store <4 x i8> [[TMP1]], ptr [[GEP_S0]], align 1
-; CHECK-NEXT: [[TMP3:%.*]] = extractelement <4 x i8> [[TMP2]], i32 0
; CHECK-NEXT: store i8 [[TMP3]], ptr [[GEP_S4]], align 1
-; CHECK-NEXT: [[TMP4:%.*]] = extractelement <4 x i8> [[TMP2]], i32 1
; CHECK-NEXT: store i8 [[TMP4]], ptr [[GEP_S5]], align 1
-; CHECK-NEXT: [[TMP5:%.*]] = extractelement <4 x i8> [[TMP2]], i32 2
; CHECK-NEXT: store i8 [[TMP5]], ptr [[GEP_S6]], align 1
-; CHECK-NEXT: [[TMP6:%.*]] = extractelement <4 x i8> [[TMP2]], i32 3
; CHECK-NEXT: store i8 [[TMP6]], ptr [[GEP_S7]], align 1
; CHECK-NEXT: ret void
;
@@ -511,21 +544,23 @@ define void @runtime_stride_constant_stride(ptr %pl, ptr %ps, i64 %stride) {
; CHECK-LABEL: define void @runtime_stride_constant_stride(
; CHECK-SAME: ptr [[PL:%.*]], ptr [[PS:%.*]], i64 [[STRIDE:%.*]]) #[[ATTR0]] {
; CHECK-NEXT: [[GEP_L0:%.*]] = getelementptr i8, ptr [[PL]], i64 0
+; CHECK-NEXT: [[GEP_L1:%.*]] = getelementptr i8, ptr [[PL]], i64 1
+; CHECK-NEXT: [[GEP_L2:%.*]] = getelementptr i8, ptr [[PL]], i64 2
+; CHECK-NEXT: [[GEP_L3:%.*]] = getelementptr i8, ptr [[PL]], i64 3
; CHECK-NEXT: [[GEP_L4:%.*]] = getelementptr i8, ptr [[PL]], i64 4
-; CHECK-NEXT: [[TMP1:%.*]] = load <4 x i8>, ptr [[GEP_L0]], align 1
+; CHECK-NEXT: [[TMP6:%.*]] = load i8, ptr [[GEP_L3]], align 1
+; CHECK-NEXT: [[TMP5:%.*]] = load i8, ptr [[GEP_L2]], align 1
+; CHECK-NEXT: [[TMP4:%.*]] = load i8, ptr [[GEP_L1]], align 1
+; CHECK-NEXT: [[TMP7:%.*]] = load i8, ptr [[GEP_L0]], align 1
; CHECK-NEXT: [[GEP_S0:%.*]] = getelementptr i8, ptr [[PS]], i64 0
; CHECK-NEXT: [[GEP_S1:%.*]] = getelementptr i8, ptr [[GEP_S0]], i64 [[STRIDE]]
; CHECK-NEXT: [[GEP_S2:%.*]] = getelementptr i8, ptr [[GEP_S1]], i64 [[STRIDE]]
; CHECK-NEXT: [[GEP_S3:%.*]] = getelementptr i8, ptr [[GEP_S2]], i64 [[STRIDE]]
; CHECK-NEXT: [[GEP_S4:%.*]] = getelementptr i8, ptr [[GEP_S3]], i64 1
; CHECK-NEXT: [[TMP3:%.*]] = load <4 x i8>, ptr [[GEP_L4]], align 1
-; CHECK-NEXT: [[TMP7:%.*]] = extractelement <4 x i8> [[TMP1]], i32 0
; CHECK-NEXT: store i8 [[TMP7]], ptr [[GEP_S0]], align 1
-; CHECK-NEXT: [[TMP4:%.*]] = extractelement <4 x i8> [[TMP1]], i32 1
; CHECK-NEXT: store i8 [[TMP4]], ptr [[GEP_S1]], align 1
-; CHECK-NEXT: [[TMP5:%.*]] = extractelement <4 x i8> [[TMP1]], i32 2
; CHECK-NEXT: store i8 [[TMP5]], ptr [[GEP_S2]], align 1
-; CHECK-NEXT: [[TMP6:%.*]] = extractelement <4 x i8> [[TMP1]], i32 3
; CHECK-NEXT: store i8 [[TMP6]], ptr [[GEP_S3]], align 1
; CHECK-NEXT: call void @llvm.experimental.vp.strided.store.v4i8.p0.i64(<4 x i8> [[TMP3]], ptr align 1 [[GEP_S4]], i64 2, <4 x i1> splat (i1 true), i32 4)
; CHECK-NEXT: ret void
@@ -574,7 +609,13 @@ define void @constant_stride_runtime_stride(ptr %pl, ptr %ps, i64 %stride) {
; CHECK-SAME: ptr [[PL:%.*]], ptr [[PS:%.*]], i64 [[STRIDE:%.*]]) #[[ATTR0]] {
; CHECK-NEXT: [[GEP_L0:%.*]] = getelementptr i8, ptr [[PL]], i64 0
; CHECK-NEXT: [[GEP_L4:%.*]] = getelementptr i8, ptr [[PL]], i64 4
-; CHECK-NEXT: [[TMP2:%.*]] = load <4 x i8>, ptr [[GEP_L4]], align 1
+; CHECK-NEXT: [[GEP_L5:%.*]] = getelementptr i8, ptr [[PL]], i64 5
+; CHECK-NEXT: [[GEP_L6:%.*]] = getelementptr i8, ptr [[PL]], i64 6
+; CHECK-NEXT: [[GEP_L7:%.*]] = getelementptr i8, ptr [[PL]], i64 7
+; CHECK-NEXT: [[TMP6:%.*]] = load i8, ptr [[GEP_L7]], align 1
+; CHECK-NEXT: [[TMP5:%.*]] = load i8, ptr [[GEP_L6]], align 1
+; CHECK-NEXT: [[TMP4:%.*]] = load i8, ptr [[GEP_L5]], align 1
+; CHECK-NEXT: [[TMP3:%.*]] = load i8, ptr [[GEP_L4]], align 1
; CHECK-NEXT: [[GEP_S0:%.*]] = getelementptr i8, ptr [[PS]], i64 0
; CHECK-NEXT: [[GEP_S1:%.*]] = getelementptr i8, ptr [[GEP_S0]], i64 2
; CHECK-NEXT: [[GEP_S2:%.*]] = getelementptr i8, ptr [[GEP_S1]], i64 2
@@ -585,13 +626,9 @@ define void @constant_stride_runtime_stride(ptr %pl, ptr %ps, i64 %stride) {
; CHECK-NEXT: [[GEP_S7:%.*]] = getelementptr i8, ptr [[GEP_S6]], i64 [[STRIDE]]
; CHECK-NEXT: [[TMP1:%.*]] = load <4 x i8>, ptr [[GEP_L0]], align 1
; CHECK-NEXT: call void @llvm.experimental.vp.strided.store.v4i8.p0.i64(<4 x i8> [[TMP1]], ptr align 1 [[GEP_S0]], i64 2, <4 x i1> splat (i1 true), i32 4)
-; CHECK-NEXT: [[TMP3:%.*]] = extractelement <4 x i8> [[TMP2]], i32 0
; CHECK-NEXT: store i8 [[TMP3]], ptr [[GEP_S4]], align 1
-; CHECK-NEXT: [[TMP4:%.*]] = extractelement <4 x i8> [[TMP2]], i32 1
; CHECK-NEXT: store i8 [[TMP4]], ptr [[GEP_S5]], align 1
-; CHECK-NEXT: [[TMP5:%.*]] = extractelement <4 x i8> [[TMP2]], i32 2
; CHECK-NEXT: store i8 [[TMP5]], ptr [[GEP_S6]], align 1
-; CHECK-NEXT: [[TMP6:%.*]] = extractelement <4 x i8> [[TMP2]], i32 3
; CHECK-NEXT: store i8 [[TMP6]], ptr [[GEP_S7]], align 1
; CHECK-NEXT: ret void
;
@@ -638,18 +675,18 @@ define void @overlap(ptr %pl, ptr %ps, i64 %stride) {
; CHECK-LABEL: define void @overlap(
; CHECK-SAME: ptr [[PL:%.*]], ptr [[PS:%.*]], i64 [[STRIDE:%.*]]) #[[ATTR0]] {
; CHECK-NEXT: [[GEP_L0:%.*]] = getelementptr i8, ptr [[PL]], i64 0
+; CHECK-NEXT: [[GEP_L1:%.*]] = getelementptr i8, ptr [[PL]], i64 1
; CHECK-NEXT: [[GEP_L2:%.*]] = getelementptr i8, ptr [[PL]], i64 2
; CHECK-NEXT: [[GEP_L3:%.*]] = getelementptr i8, ptr [[PL]], i64 3
-; CHECK-NEXT: [[TMP2:%.*]] = load <2 x i8>, ptr [[GEP_L0]], align 1
+; CHECK-NEXT: [[TMP4:%.*]] = load i8, ptr [[GEP_L1]], align 1
+; CHECK-NEXT: [[TMP3:%.*]] = load i8, ptr [[GEP_L0]], align 1
; CHECK-NEXT: [[LOAD2:%.*]] = load i8, ptr [[GEP_L2]], align 1
; CHECK-NEXT: [[GEP_S0:%.*]] = getelementptr i8, ptr [[PS]], i64 0
; CHECK-NEXT: [[GEP_S1:%.*]] = getelementptr i8, ptr [[GEP_S0]], i64 [[STRIDE]]
; CHECK-NEXT: [[GEP_S2:%.*]] = getelementptr i8, ptr [[GEP_S1]], i64 [[STRIDE]]
; CHECK-NEXT: [[GEP_S3:%.*]] = getelementptr i8, ptr [[GEP_S2]], i64 [[STRIDE]]
; CHECK-NEXT: [[TMP1:%.*]] = load <4 x i8>, ptr [[GEP_L3]], align 1
-; CHECK-NEXT: [[TMP3:%.*]] = extractelement <2 x i8> [[TMP2]], i32 0
; CHECK-NEXT: store i8 [[TMP3]], ptr [[GEP_S0]], align 1
-; CHECK-NEXT: [[TMP4:%.*]] = extractelement <2 x i8> [[TMP2]], i32 1
; CHECK-NEXT: store i8 [[TMP4]], ptr [[GEP_S1]], align 1
; CHECK-NEXT: store i8 [[LOAD2]], ptr [[GEP_S2]], align 1
; CHECK-NEXT: store <4 x i8> [[TMP1]], ptr [[GEP_S3]], align 1
@@ -694,18 +731,18 @@ define void @overlap_constant_stride(ptr %pl, ptr %ps, i64 %stride) {
; CHECK-LABEL: define void @overlap_constant_stride(
; CHECK-SAME: ptr [[PL:%.*]], ptr [[PS:%.*]], i64 [[STRIDE:%.*]]) #[[ATTR0]] {
; CHECK-NEXT: [[GEP_L0:%.*]] = getelementptr i8, ptr [[PL]], i64 0
+; CHECK-NEXT: [[GEP_L1:%.*]] = getelementptr i8, ptr [[PL]], i64 1
; CHECK-NEXT: [[GEP_L2:%.*]] = getelementptr i8, ptr [[PL]], i64 2
; CHECK-NEXT: [[GEP_L3:%.*]] = getelementptr i8, ptr [[PL]], i64 3
-; CHECK-NEXT: [[TMP2:%.*]] = load <2 x i8>, ptr [[GEP_L0]], align 1
+; CHECK-NEXT: [[TMP4:%.*]] = load i8, ptr [[GEP_L1]], align 1
+; CHECK-NEXT: [[TMP3:%.*]] = load i8, ptr [[GEP_L0]], align 1
; CHECK-NEXT: [[LOAD2:%.*]] = load i8, ptr [[GEP_L2]], align 1
; CHECK-NEXT: [[GEP_S0:%.*]] = getelementptr i8, ptr [[PS]], i64 0
; CHECK-NEXT: [[GEP_S1:%.*]] = getelementptr i8, ptr [[GEP_S0]], i64 [[STRIDE]]
; CHECK-NEXT: [[GEP_S2:%.*]] = getelementptr i8, ptr [[GEP_S1]], i64 [[STRIDE]]
; CHECK-NEXT: [[GEP_S3:%.*]] = getelementptr i8, ptr [[GEP_S2]], i64 [[STRIDE]]
; CHECK-NEXT: [[TMP1:%.*]] = load <4 x i8>, ptr [[GEP_L3]], align 1
-; CHECK-NEXT: [[TMP3:%.*]] = extractelement <2 x i8> [[TMP2]], i32 0
; CHECK-NEXT: store i8 [[TMP3]], ptr [[GEP_S0]], align 1
-; CHECK-NEXT: [[TMP4:%.*]] = extractelement <2 x i8> [[TMP2]], i32 1
; CHECK-NEXT: store i8 [[TMP4]], ptr [[GEP_S1]], align 1
; CHECK-NEXT: store i8 [[LOAD2]], ptr [[GEP_S2]], align 1
; CHECK-NEXT: call void @llvm.experimental.vp.strided.store.v4i8.p0.i64(<4 x i8> [[TMP1]], ptr align 1 [[GEP_S3]], i64 2, <4 x i1> splat (i1 true), i32 4)
@@ -750,7 +787,21 @@ define void @runtime_stride_reorder(ptr %pl, ptr %ps, i64 %stride) {
; CHECK-LABEL: define void @runtime_stride_reorder(
; CHECK-SAME: ptr [[PL:%.*]], ptr [[PS:%.*]], i64 [[STRIDE:%.*]]) #[[ATTR0]] {
; CHECK-NEXT: [[GEP_L0:%.*]] = getelementptr i8, ptr [[PL]], i64 0
-; CHECK-NEXT: [[TMP1:%.*]] = load <8 x i8>, ptr [[GEP_L0]], align 1
+; CHECK-NEXT: [[GEP_L1:%.*]] = getelementptr i8, ptr [[PL]], i64 2
+; CHECK-NEXT: [[GEP_L2:%.*]] = getelementptr i8, ptr [[PL]], i64 1
+; CHECK-NEXT: [[GEP_L3:%.*]] = getelementptr i8, ptr [[PL]], i64 4
+; CHECK-NEXT: [[GEP_L4:%.*]] = getelementptr i8, ptr [[PL]], i64 7
+; CHECK-NEXT: [[GEP_L5:%.*]] = getelementptr i8, ptr [[PL]], i64 5
+; CHECK-NEXT: [[GEP_L6:%.*]] = getelementptr i8, ptr [[PL]], i64 6
+; CHECK-NEXT: [[GEP_L7:%.*]] = getelementptr i8, ptr [[PL]], i64 3
+; CHECK-NEXT: [[TMP9:%.*]] = load i8, ptr [[GEP_L7]], align 1
+; CHECK-NEXT: [[TMP8:%.*]] = load i8, ptr [[GEP_L6]], align 1
+; CHECK-NEXT: [[TMP7:%.*]] = load i8, ptr [[GEP_L5]], align 1
+; CHECK-NEXT: [[TMP6:%.*]] = load i8, ptr [[GEP_L4]], align 1
+; CHECK-NEXT: [[TMP5:%.*]] = load i8, ptr [[GEP_L3]], align 1
+; CHECK-NEXT: [[TMP4:%.*]] = load i8, ptr [[GEP_L2]], align 1
+; CHECK-NEXT: [[TMP3:%.*]] = load i8, ptr [[GEP_L1]], align 1
+; CHECK-NEXT: [[TMP2:%.*]] = load i8, ptr [[GEP_L0]], align 1
; CHECK-NEXT: [[GEP_S0:%.*]] = getelementptr i8, ptr [[PS]], i64 0
; CHECK-NEXT: [[GEP_S1:%.*]] = getelementptr i8, ptr [[GEP_S0]], i64 [[STRIDE]]
; CHECK-NEXT: [[GEP_S2:%.*]] = getelementptr i8, ptr [[GEP_S1]], i64 [[STRIDE]]
@@ -759,21 +810,13 @@ define void @runtime_stride_reorder(ptr %pl, ptr %ps, i64 %stride) {
; CHECK-NEXT: [[GEP_S5:%.*]] = getelementptr i8, ptr [[GEP_S4]], i64 [[STRIDE]]
; CHECK-NEXT: [[GEP_S6:%.*]] = getelementptr i8, ptr [[GEP_S5]], i64 [[STRIDE]]
; CHECK-NEXT: [[GEP_S7:%.*]] = getelementptr i8, ptr [[GEP_S6]], i64 [[STRIDE]]
-; CHECK-NEXT: [[TMP2:%.*]] = extractelement <8 x i8> [[TMP1]], i32 0
; CHECK-NEXT: store i8 [[TMP2]], ptr [[GEP_S0]], align 1
-; CHECK-NEXT: [[TMP3:%.*]] = extractelement <8 x i8> [[TMP1]], i32 2
; CHECK-NEXT: store i8 [[TMP3]], ptr [[GEP_S1]], align 1
-; CHECK-NEXT: [[TMP4:%.*]] = extractelement <8 x i8> [[TMP1]], i32 1
; CHECK-NEXT: store i8 [[TMP4]], ptr [[GEP_S2]], align 1
-; CHECK-NEXT: [[TMP5:%.*]] = extractelement <8 x i8> [[TMP1]], i32 4
; CHECK-NEXT: store i8 [[TMP5]], ptr [[GEP_S3]], align 1
-; CHECK-NEXT: [[TMP6:%.*]] = extractelement <8 x i8> [[TMP1]], i32 7
; CHECK-NEXT: store i8 [[TMP6]], ptr [[GEP_S4]], align 1
-; CHECK-NEXT: [[TMP7:%.*]] = extractelement <8 x i8> [[TMP1]], i32 5
; CHECK-NEXT: store i8 [[TMP7]], ptr [[GEP_S5]], align 1
-; CHECK-NEXT: [[TMP8:%.*]] = extractelement <8 x i8> [[TMP1]], i32 6
; CHECK-NEXT: store i8 [[TMP8]], ptr [[GEP_S6]], align 1
-; CHECK-NEXT: [[TMP9:%.*]] = extractelement <8 x i8> [[TMP1]], i32 3
; CHECK-NEXT: store i8 [[TMP9]], ptr [[GEP_S7]], align 1
; CHECK-NEXT: ret void
;
@@ -820,7 +863,21 @@ define void @runtime_stride_unschedulable(ptr %pl, ptr %ps, i64 %stride) {
; CHECK-LABEL: define void @runtime_stride_unschedulable(
; CHECK-SAME: ptr [[PL:%.*]], ptr [[PS:%.*]], i64 [[STRIDE:%.*]]) #[[ATTR0]] {
; CHECK-NEXT: [[GEP_L0:%.*]] = getelementptr i8, ptr [[PL]], i64 0
-; CHECK-NEXT: [[TMP1:%.*]] = load <8 x i8>, ptr [[GEP_L0]], align 1
+; CHECK-NEXT: [[GEP_L1:%.*]] = getelementptr i8, ptr [[PL]], i64 1
+; CHECK-NEXT: [[GEP_L2:%.*]] = getelementptr i8, ptr [[PL]], i64 2
+; CHECK-NEXT: [[GEP_L3:%.*]] = getelementptr i8, ptr [[PL]], i64 3
+; CHECK-NEXT: [[GEP_L4:%.*]] = getelementptr i8, ptr [[PL]], i64 4
+; CHECK-NEXT: [[GEP_L5:%.*]] = getelementptr i8, ptr [[PL]], i64 5
+; CHECK-NEXT: [[GEP_L6:%.*]] = getelementptr i8, ptr [[PL]], i64 6
+; CHECK-NEXT: [[GEP_L7:%.*]] = getelementptr i8, ptr [[PL]], i64 7
+; CHECK-NEXT: [[TMP8:%.*]] = load i8, ptr [[GEP_L7]], align 1
+; CHECK-NEXT: [[TMP7:%.*]] = load i8, ptr [[GEP_L6]], align 1
+; CHECK-NEXT: [[TMP11:%.*]] = load i8, ptr [[GEP_L5]], align 1
+; CHECK-NEXT: [[TMP6:%.*]] = load i8, ptr [[GEP_L4]], align 1
+; CHECK-NEXT: [[TMP10:%.*]] = load i8, ptr [[GEP_L3]], align 1
+; CHECK-NEXT: [[TMP9:%.*]] = load i8, ptr [[GEP_L2]], align 1
+; CHECK-NEXT: [[TMP5:%.*]] = load i8, ptr [[GEP_L1]], align 1
+; CHECK-NEXT: [[TMP4:%.*]] = load i8, ptr [[GEP_L0]], align 1
; CHECK-NEXT: [[GEP_S0:%.*]] = getelementptr i8, ptr [[PS]], i64 0
; CHECK-NEXT: [[GEP_S1:%.*]] = getelementptr i8, ptr [[GEP_S0]], i64 [[STRIDE]]
; CHECK-NEXT: [[GEP_S2:%.*]] = getelementptr i8, ptr [[GEP_S1]], i64 [[STRIDE]]
@@ -829,21 +886,13 @@ define void @runtime_stride_unschedulable(ptr %pl, ptr %ps, i64 %stride) {
; CHECK-NEXT: [[GEP_S5:%.*]] = getelementptr i8, ptr [[GEP_S4]], i64 [[STRIDE]]
; CHECK-NEXT: [[GEP_S6:%.*]] = getelementptr i8, ptr [[GEP_S5]], i64 [[STRIDE]]
; CHECK-NEXT: [[GEP_S7:%.*]] = getelementptr i8, ptr [[GEP_S6]], i64 [[STRIDE]]
-; CHECK-NEXT: [[TMP4:%.*]] = extractelement <8 x i8> [[TMP1]], i32 0
; CHECK-NEXT: store i8 [[TMP4]], ptr [[GEP_S0]], align 1
-; CHECK-NEXT: [[TMP5:%.*]] = extractelement <8 x i8> [[TMP1]], i32 1
; CHECK-NEXT: store i8 [[TMP5]], ptr [[GEP_S6]], align 1
-; CHECK-NEXT: [[TMP9:%.*]] = extractelement <8 x i8> [[TMP1]], i32 2
; CHECK-NEXT: store i8 [[TMP9]], ptr [[GEP_S2]], align 1
-; CHECK-NEXT: [[TMP10:%.*]] = extractelement <8 x i8> [[TMP1]], i32 3
; CHECK-NEXT: store i8 [[TMP10]], ptr [[GEP_S3]], align 1
-; CHECK-NEXT: [[TMP6:%.*]] = extractelement <8 x i8> [[TMP1]], i32 4
; CHECK-NEXT: store i8 [[TMP6]], ptr [[GEP_S4]], align 1
-; CHECK-NEXT: [[TMP11:%.*]] = extractelement <8 x i8> [[TMP1]], i32 5
; CHECK-NEXT: store i8 [[TMP11]], ptr [[GEP_S5]], align 1
-; CHECK-NEXT: [[TMP7:%.*]] = extractelement <8 x i8> [[TMP1]], i32 6
; CHECK-NEXT: store i8 [[TMP7]], ptr [[GEP_S1]], align 1
-; CHECK-NEXT: [[TMP8:%.*]] = extractelement <8 x i8> [[TMP1]], i32 7
; CHECK-NEXT: store i8 [[TMP8]], ptr [[GEP_S7]], align 1
; CHECK-NEXT: ret void
;
@@ -919,13 +968,22 @@ define void @runtime_stride_diff_types(ptr %pl, ptr %ps, i64 %stride) {
; CHECK-SAME: ptr [[PL:%.*]], ptr [[PS:%.*]], i64 [[STRIDE:%.*]]) #[[ATTR0]] {
; CHECK-NEXT: [[GEP_L0:%.*]] = getelementptr i8, ptr [[PL]], i64 0
; CHECK-NEXT: [[GEP_L1:%.*]] = getelementptr i8, ptr [[PL]], i64 1
+; CHECK-NEXT: [[GEP_L2:%.*]] = getelementptr i8, ptr [[PL]], i64 2
+; CHECK-NEXT: [[GEP_L3:%.*]] = getelementptr i8, ptr [[PL]], i64 3
+; CHECK-NEXT: [[GEP_L4:%.*]] = getelementptr i8, ptr [[PL]], i64 4
+; CHECK-NEXT: [[GEP_L5:%.*]] = getelementptr i8, ptr [[PL]], i64 5
; CHECK-NEXT: [[GEP_L6:%.*]] = getelementptr i8, ptr [[PL]], i64 6
; CHECK-NEXT: [[GEP_L7:%.*]] = getelementptr i8, ptr [[PL]], i64 7
-; CHECK-NEXT: [[TMP1:%.*]] = call <5 x i8> @llvm.masked.load.v5i8.p0(ptr align 1 [[GEP_L0]], <5 x i1> <i1 true, i1 false, i1 true, i1 true, i1 true>, <5 x i8> poison)
-; CHECK-NEXT: [[TMP2:%.*]] = shufflevector <5 x i8> [[TMP1]], <5 x i8> poison, <4 x i32> <i32 0, i32 2, i32 3, i32 4>
+; CHECK-NEXT: [[TMP9:%.*]] = load i8, ptr [[GEP_L4]], align 1
+; CHECK-NEXT: [[TMP8:%.*]] = load i8, ptr [[GEP_L3]], align 1
+; CHECK-NEXT: [[TMP7:%.*]] = load i8, ptr [[GEP_L2]], align 1
+; CHECK-NEXT: [[TMP4:%.*]] = load i8, ptr [[GEP_L0]], align 1
; CHECK-NEXT: [[LOAD6:%.*]] = load i8, ptr [[GEP_L6]], align 1
; CHECK-NEXT: [[LOAD7:%.*]] = load i8, ptr [[GEP_L7]], align 1
-; CHECK-NEXT: [[TMP3:%.*]] = call <2 x i8> @llvm.experimental.vp.strided.load.v2i8.p0.i64(ptr align 1 [[GEP_L1]], i64 4, <2 x i1> splat (i1 true), i32 2)
+; CHECK-NEXT: [[LOAD5:%.*]] = load i8, ptr [[GEP_L5]], align 1
+; CHECK-NEXT: [[LOAD1:%.*]] = load i8, ptr [[GEP_L1]], align 1
+; CHECK-NEXT: [[TMP11:%.*]] = zext i8 [[LOAD5]] to i16
+; CHECK-NEXT: [[TMP6:%.*]] = zext i8 [[LOAD1]] to i16
; CHECK-NEXT: [[GEP_S0:%.*]] = getelementptr i8, ptr [[PS]], i64 0
; CHECK-NEXT: [[GEP_S1:%.*]] = getelementptr i8, ptr [[GEP_S0]], i64 [[STRIDE]]
; CHECK-NEXT: [[GEP_S2:%.*]] = getelementptr i8, ptr [[GEP_S1]], i64 [[STRIDE]]
@@ -934,19 +992,11 @@ define void @runtime_stride_diff_types(ptr %pl, ptr %ps, i64 %stride) {
; CHECK-NEXT: [[GEP_S5:%.*]] = getelementptr i8, ptr [[GEP_S4]], i64 [[STRIDE]]
; CHECK-NEXT: [[GEP_S6:%.*]] = getelementptr i8, ptr [[GEP_S5]], i64 [[STRIDE]]
; CHECK-NEXT: [[GEP_S7:%.*]] = getelementptr i8, ptr [[GEP_S6]], i64 [[STRIDE]]
-; CHECK-NEXT: [[TMP4:%.*]] = extractelement <4 x i8> [[TMP2]], i32 0
; CHECK-NEXT: store i8 [[TMP4]], ptr [[GEP_S0]], align 1
-; CHECK-NEXT: [[TMP5:%.*]] = extractelement <2 x i8> [[TMP3]], i32 0
-; CHECK-NEXT: [[TMP6:%.*]] = zext i8 [[TMP5]] to i16
; CHECK-NEXT: store i16 [[TMP6]], ptr [[GEP_S1]], align 2
-; CHECK-NEXT: [[TMP7:%.*]] = extractelement <4 x i8> [[TMP2]], i32 1
; CHECK-NEXT: store i8 [[TMP7]], ptr [[GEP_S2]], align 1
-; CHECK-NEXT: [[TMP8:%.*]] = extractelement <4 x i8> [[TMP2]], i32 2
; CHECK-NEXT: store i8 [[TMP8]], ptr [[GEP_S3]], align 1
-; CHECK-NEXT: [[TMP9:%.*]] = extractelement <4 x i8> [[TMP2]], i32 3
; CHECK-NEXT: store i8 [[TMP9]], ptr [[GEP_S4]], align 1
-; CHECK-NEXT: [[TMP10:%.*]] = extractelement <2 x i8> [[TMP3]], i32 1
-; CHECK-NEXT: [[TMP11:%.*]] = zext i8 [[TMP10]] to i16
; CHECK-NEXT: store i16 [[TMP11]], ptr [[GEP_S5]], align 2
; CHECK-NEXT: store i8 [[LOAD6]], ptr [[GEP_S6]], align 1
; CHECK-NEXT: store i8 [[LOAD7]], ptr [[GEP_S7]], align 1
diff --git a/llvm/test/Transforms/SLPVectorizer/RISCV/strided-loads-with-external-use-ptr.ll b/llvm/test/Transforms/SLPVectorizer/RISCV/strided-loads-with-external-use-ptr.ll
index 8fddb2e21e248..a1800363d90b8 100644
--- a/llvm/test/Transforms/SLPVectorizer/RISCV/strided-loads-with-external-use-ptr.ll
+++ b/llvm/test/Transforms/SLPVectorizer/RISCV/strided-loads-with-external-use-ptr.ll
@@ -15,11 +15,9 @@ define i16 @test() {
; CHECK-NEXT: [[PEDGE_061_I:%.*]] = phi ptr [ [[INCDEC_PTR_I:%.*]], [[WHILE_BODY_I]] ], [ null, [[ENTRY]] ]
; CHECK-NEXT: [[INCDEC_PTR_I]] = getelementptr [[S]], ptr [[PEDGE_061_I]], i64 -1
; CHECK-NEXT: [[PPREV_0_I]] = getelementptr [[S]], ptr [[PPREV_062_I]], i64 -1
-; CHECK-NEXT: [[TMP1:%.*]] = call <3 x i16> @llvm.masked.load.v3i16.p0(ptr align 2 [[PPREV_0_I]], <3 x i1> <i1 true, i1 false, i1 true>, <3 x i16> poison)
-; CHECK-NEXT: [[TMP2:%.*]] = shufflevector <3 x i16> [[TMP1]], <3 x i16> poison, <2 x i32> <i32 0, i32 2>
-; CHECK-NEXT: [[TMP3:%.*]] = extractelement <2 x i16> [[TMP2]], i32 0
-; CHECK-NEXT: [[TMP4:%.*]] = extractelement <2 x i16> [[TMP2]], i32 1
-; CHECK-NEXT: [[CMP_I178:%.*]] = icmp ult i16 [[TMP4]], [[TMP3]]
+; CHECK-NEXT: [[TMP1:%.*]] = load i16, ptr [[PPREV_0_I]], align 2
+; CHECK-NEXT: [[TMP2:%.*]] = load i16, ptr [[INCDEC_PTR_I]], align 2
+; CHECK-NEXT: [[CMP_I178:%.*]] = icmp ult i16 [[TMP2]], [[TMP1]]
; CHECK-NEXT: br label [[WHILE_BODY_I]]
;
entry:
diff --git a/llvm/test/Transforms/SLPVectorizer/X86/PR35628_1.ll b/llvm/test/Transforms/SLPVectorizer/X86/PR35628_1.ll
index e9aa434dec03d..22022a84a217f 100644
--- a/llvm/test/Transforms/SLPVectorizer/X86/PR35628_1.ll
+++ b/llvm/test/Transforms/SLPVectorizer/X86/PR35628_1.ll
@@ -10,9 +10,11 @@ define void @mainTest(ptr %ptr) #0 {
; CHECK: loop:
; CHECK-NEXT: [[DUMMY_PHI:%.*]] = phi i32 [ 1, [[ENTRY:%.*]] ], [ [[OP_RDX3:%.*]], [[LOOP]] ]
; CHECK-NEXT: [[TMP0:%.*]] = getelementptr inbounds i32, ptr [[PTR]], i64 1
+; CHECK-NEXT: [[TMP8:%.*]] = getelementptr inbounds i32, ptr [[PTR]], i64 2
+; CHECK-NEXT: [[TMP9:%.*]] = getelementptr inbounds i32, ptr [[PTR]], i64 3
+; CHECK-NEXT: [[TMP2:%.*]] = load i32, ptr [[TMP9]], align 4
; CHECK-NEXT: [[TMP1:%.*]] = load <4 x i32>, ptr [[PTR]], align 4
-; CHECK-NEXT: [[TMP2:%.*]] = extractelement <4 x i32> [[TMP1]], i32 3
-; CHECK-NEXT: [[TMP3:%.*]] = extractelement <4 x i32> [[TMP1]], i32 2
+; CHECK-NEXT: [[TMP3:%.*]] = load i32, ptr [[TMP8]], align 4
; CHECK-NEXT: [[TMP4:%.*]] = load i32, ptr [[TMP0]], align 4
; CHECK-NEXT: [[TMP5:%.*]] = mul <4 x i32> [[TMP1]], [[TMP1]]
; CHECK-NEXT: [[TMP6:%.*]] = sext i32 [[TMP3]] to i64
diff --git a/llvm/test/Transforms/SLPVectorizer/X86/call.ll b/llvm/test/Transforms/SLPVectorizer/X86/call.ll
index 9e19aa9e93095..1f559a195e76b 100644
--- a/llvm/test/Transforms/SLPVectorizer/X86/call.ll
+++ b/llvm/test/Transforms/SLPVectorizer/X86/call.ll
@@ -131,11 +131,12 @@ define void @sqrt_libm_no_errno(ptr %a, ptr %b) {
define void @sqrt_libm_errno(ptr %a, ptr %b) {
; CHECK-LABEL: @sqrt_libm_errno(
-; CHECK-NEXT: [[TMP1:%.*]] = load <2 x double>, ptr [[A:%.*]], align 8
+; CHECK-NEXT: [[IDX1:%.*]] = getelementptr inbounds double, ptr [[A:%.*]], i64 1
+; CHECK-NEXT: [[A1:%.*]] = load double, ptr [[IDX1]], align 8
+; CHECK-NEXT: [[TMP1:%.*]] = load <2 x double>, ptr [[A]], align 8
; CHECK-NEXT: [[TMP2:%.*]] = extractelement <2 x double> [[TMP1]], i32 0
; CHECK-NEXT: [[SQRT1:%.*]] = tail call nnan double @sqrt(double [[TMP2]]) #[[ATTR4:[0-9]+]]
-; CHECK-NEXT: [[TMP3:%.*]] = extractelement <2 x double> [[TMP1]], i32 1
-; CHECK-NEXT: [[SQRT2:%.*]] = tail call nnan double @sqrt(double [[TMP3]]) #[[ATTR4]]
+; CHECK-NEXT: [[SQRT2:%.*]] = tail call nnan double @sqrt(double [[A1]]) #[[ATTR4]]
; CHECK-NEXT: store double [[SQRT1]], ptr [[B:%.*]], align 8
; CHECK-NEXT: [[IDX2:%.*]] = getelementptr inbounds double, ptr [[B]], i64 1
; CHECK-NEXT: store double [[SQRT2]], ptr [[IDX2]], align 8
@@ -155,11 +156,11 @@ define void @sqrt_libm_errno(ptr %a, ptr %b) {
; Negative test case
define void @round_custom(ptr %a, ptr %b) {
; CHECK-LABEL: @round_custom(
-; CHECK-NEXT: [[TMP1:%.*]] = load <2 x i64>, ptr [[A:%.*]], align 8
-; CHECK-NEXT: [[TMP2:%.*]] = extractelement <2 x i64> [[TMP1]], i32 0
-; CHECK-NEXT: [[ROUND1:%.*]] = tail call i64 @round(i64 [[TMP2]]) #[[ATTR5:[0-9]+]]
-; CHECK-NEXT: [[TMP3:%.*]] = extractelement <2 x i64> [[TMP1]], i32 1
-; CHECK-NEXT: [[ROUND2:%.*]] = tail call i64 @round(i64 [[TMP3]]) #[[ATTR5]]
+; CHECK-NEXT: [[IDX1:%.*]] = getelementptr inbounds i64, ptr [[A:%.*]], i64 1
+; CHECK-NEXT: [[A1:%.*]] = load i64, ptr [[IDX1]], align 8
+; CHECK-NEXT: [[A0:%.*]] = load i64, ptr [[A]], align 8
+; CHECK-NEXT: [[ROUND1:%.*]] = tail call i64 @round(i64 [[A0]]) #[[ATTR5:[0-9]+]]
+; CHECK-NEXT: [[ROUND2:%.*]] = tail call i64 @round(i64 [[A1]]) #[[ATTR5]]
; CHECK-NEXT: store i64 [[ROUND1]], ptr [[B:%.*]], align 8
; CHECK-NEXT: [[IDX2:%.*]] = getelementptr inbounds i64, ptr [[B]], i64 1
; CHECK-NEXT: store i64 [[ROUND2]], ptr [[IDX2]], align 8
diff --git a/llvm/test/Transforms/SLPVectorizer/X86/entries-shuffled-diff-sizes.ll b/llvm/test/Transforms/SLPVectorizer/X86/entries-shuffled-diff-sizes.ll
index b99a1c2d83394..c5d3f514c5dc3 100644
--- a/llvm/test/Transforms/SLPVectorizer/X86/entries-shuffled-diff-sizes.ll
+++ b/llvm/test/Transforms/SLPVectorizer/X86/entries-shuffled-diff-sizes.ll
@@ -18,12 +18,12 @@ define void @test() {
; CHECK-NEXT: [[TMP7:%.*]] = load <16 x float>, ptr getelementptr ([16000 x i8], ptr @GLOB, i64 0, i64 1272), align 16
; CHECK-NEXT: [[TMP11:%.*]] = load <2 x float>, ptr getelementptr ([16000 x i8], ptr @GLOB, i64 0, i64 1620), align 4
; CHECK-NEXT: [[TMP9:%.*]] = shufflevector <2 x float> [[TMP11]], <2 x float> poison, <16 x i32> <i32 0, i32 1, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
-; CHECK-NEXT: [[TMP10:%.*]] = shufflevector <16 x float> [[TMP9]], <16 x float> [[TMP7]], <16 x i32> <i32 poison, i32 0, i32 20, i32 1, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
-; CHECK-NEXT: [[TMP19:%.*]] = shufflevector <2 x float> [[TMP3]], <2 x float> poison, <16 x i32> <i32 0, i32 1, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
-; CHECK-NEXT: [[TMP14:%.*]] = shufflevector <16 x float> [[TMP19]], <16 x float> [[TMP10]], <16 x i32> <i32 1, i32 1, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 19, i32 19, i32 19, i32 19, i32 18>
-; CHECK-NEXT: [[TMP17:%.*]] = shufflevector <16 x float> [[TMP7]], <16 x float> [[TMP14]], <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15, i32 18>
+; CHECK-NEXT: [[TMP10:%.*]] = shufflevector <16 x float> [[TMP9]], <16 x float> [[TMP7]], <16 x i32> <i32 poison, i32 0, i32 1, i32 30, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
+; CHECK-NEXT: [[TMP12:%.*]] = shufflevector <2 x float> [[TMP3]], <2 x float> poison, <16 x i32> <i32 0, i32 1, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
+; CHECK-NEXT: [[TMP14:%.*]] = shufflevector <16 x float> [[TMP12]], <16 x float> [[TMP10]], <16 x i32> <i32 1, i32 1, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 18, i32 18, i32 18, i32 19>
+; CHECK-NEXT: [[TMP17:%.*]] = shufflevector <16 x float> [[TMP7]], <16 x float> [[TMP14]], <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 15, i32 28>
; CHECK-NEXT: [[TMP18:%.*]] = fmul reassoc ninf nsz arcp contract afn <16 x float> [[TMP14]], [[TMP17]]
-; CHECK-NEXT: [[TMP15:%.*]] = shufflevector <16 x float> [[TMP18]], <16 x float> poison, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 15, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14>
+; CHECK-NEXT: [[TMP15:%.*]] = shufflevector <16 x float> [[TMP18]], <16 x float> poison, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 15, i32 14>
; CHECK-NEXT: store <16 x float> [[TMP15]], ptr getelementptr ([16000 x i8], ptr @GLOB, i64 0, i64 2992), align 16
; CHECK-NEXT: ret void
;
diff --git a/llvm/test/Transforms/SLPVectorizer/X86/extractelement-multi-register-use.ll b/llvm/test/Transforms/SLPVectorizer/X86/extractelement-multi-register-use.ll
index c945799b38122..c45f6b54f39c5 100644
--- a/llvm/test/Transforms/SLPVectorizer/X86/extractelement-multi-register-use.ll
+++ b/llvm/test/Transforms/SLPVectorizer/X86/extractelement-multi-register-use.ll
@@ -10,10 +10,10 @@ define void @test(double %i) {
; CHECK-NEXT: [[TMP3:%.*]] = fsub <2 x double> zeroinitializer, [[TMP2]]
; CHECK-NEXT: [[I96:%.*]] = fsub double poison, 0.000000e+00
; CHECK-NEXT: [[I75:%.*]] = fsub double 0.000000e+00, [[I]]
-; CHECK-NEXT: [[TMP4:%.*]] = insertelement <2 x double> <double poison, double 0.000000e+00>, double [[I]], i32 0
-; CHECK-NEXT: [[TMP8:%.*]] = shufflevector <2 x double> [[TMP4]], <2 x double> <double 0.000000e+00, double poison>, <2 x i32> <i32 2, i32 0>
-; CHECK-NEXT: [[TMP15:%.*]] = fsub <2 x double> [[TMP4]], [[TMP8]]
-; CHECK-NEXT: [[TMP5:%.*]] = shufflevector <2 x double> [[TMP15]], <2 x double> poison, <8 x i32> <i32 0, i32 1, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
+; CHECK-NEXT: [[TMP8:%.*]] = insertelement <2 x double> <double poison, double 0.000000e+00>, double [[I]], i32 0
+; CHECK-NEXT: [[TMP9:%.*]] = shufflevector <2 x double> [[TMP8]], <2 x double> <double 0.000000e+00, double poison>, <2 x i32> <i32 2, i32 0>
+; CHECK-NEXT: [[TMP4:%.*]] = fsub <2 x double> [[TMP8]], [[TMP9]]
+; CHECK-NEXT: [[TMP5:%.*]] = shufflevector <2 x double> [[TMP4]], <2 x double> poison, <8 x i32> <i32 0, i32 1, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
; CHECK-NEXT: [[TMP6:%.*]] = shufflevector <8 x double> <double 0.000000e+00, double 0.000000e+00, double poison, double poison, double 0.000000e+00, double poison, double poison, double poison>, <8 x double> [[TMP5]], <8 x i32> <i32 0, i32 1, i32 8, i32 9, i32 4, i32 poison, i32 poison, i32 poison>
; CHECK-NEXT: [[TMP7:%.*]] = insertelement <8 x double> [[TMP6]], double [[I75]], i32 5
; CHECK-NEXT: [[TMP10:%.*]] = shufflevector <8 x double> [[TMP7]], <8 x double> poison, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 2, i32 3>
@@ -23,7 +23,7 @@ define void @test(double %i) {
; CHECK-NEXT: [[TMP14:%.*]] = fcmp ult <8 x double> [[TMP13]], zeroinitializer
; CHECK-NEXT: br label [[BB116:%.*]]
; CHECK: bb116:
-; CHECK-NEXT: [[TMP16:%.*]] = extractelement <2 x double> [[TMP15]], i32 0
+; CHECK-NEXT: [[TMP16:%.*]] = extractelement <2 x double> [[TMP4]], i32 0
; CHECK-NEXT: [[I117:%.*]] = fmul double 0.000000e+00, [[TMP16]]
; CHECK-NEXT: [[I119:%.*]] = fmul double 0.000000e+00, [[I96]]
; CHECK-NEXT: [[I120:%.*]] = fadd double [[I117]], [[I119]]
diff --git a/llvm/test/Transforms/SLPVectorizer/X86/extractelemets-extended-by-poison.ll b/llvm/test/Transforms/SLPVectorizer/X86/extractelemets-extended-by-poison.ll
index 132865da252c9..ccca930e44ba7 100644
--- a/llvm/test/Transforms/SLPVectorizer/X86/extractelemets-extended-by-poison.ll
+++ b/llvm/test/Transforms/SLPVectorizer/X86/extractelemets-extended-by-poison.ll
@@ -5,7 +5,8 @@ define i32 @test() {
; CHECK-LABEL: define i32 @test() {
; CHECK-NEXT: [[ENTRY:.*:]]
; CHECK-NEXT: [[TMP0:%.*]] = load <4 x i64>, ptr null, align 16
-; CHECK-NEXT: [[TMP12:%.*]] = extractelement <4 x i64> [[TMP0]], i32 1
+; CHECK-NEXT: [[TMP12:%.*]] = load i64, ptr getelementptr inbounds nuw (i8, ptr null, i64 8), align 8
+; CHECK-NEXT: [[TMP9:%.*]] = load i64, ptr null, align 16
; CHECK-NEXT: [[TMP13:%.*]] = or i64 [[TMP12]], 0
; CHECK-NEXT: [[TMP3:%.*]] = shufflevector <4 x i64> [[TMP0]], <4 x i64> poison, <6 x i32> <i32 0, i32 1, i32 2, i32 3, i32 poison, i32 1>
; CHECK-NEXT: [[TMP4:%.*]] = shufflevector <6 x i64> [[TMP3]], <6 x i64> <i64 poison, i64 poison, i64 poison, i64 poison, i64 0, i64 poison>, <6 x i32> <i32 poison, i32 poison, i32 poison, i32 poison, i32 10, i32 5>
@@ -17,7 +18,6 @@ define i32 @test() {
; CHECK-NEXT: [[TMP14:%.*]] = shufflevector <6 x i32> [[TMP11]], <6 x i32> poison, <8 x i32> <i32 0, i32 1, i32 1, i32 2, i32 2, i32 5, i32 4, i32 3>
; CHECK-NEXT: [[TMP15:%.*]] = add <8 x i32> [[TMP14]], zeroinitializer
; CHECK-NEXT: [[TMP8:%.*]] = add <16 x i32> [[TMP7]], zeroinitializer
-; CHECK-NEXT: [[TMP9:%.*]] = extractelement <4 x i64> [[TMP0]], i32 0
; CHECK-NEXT: [[INC_3_3_I_1:%.*]] = or i64 [[TMP9]], 0
; CHECK-NEXT: [[TMP16:%.*]] = shufflevector <16 x i32> [[TMP8]], <16 x i32> poison, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>
; CHECK-NEXT: [[RDX_OP:%.*]] = or <8 x i32> [[TMP16]], [[TMP15]]
diff --git a/llvm/test/Transforms/SLPVectorizer/X86/gathered-shuffle-resized.ll b/llvm/test/Transforms/SLPVectorizer/X86/gathered-shuffle-resized.ll
index 6df57f8c6a0bb..9acb41e1c12c4 100644
--- a/llvm/test/Transforms/SLPVectorizer/X86/gathered-shuffle-resized.ll
+++ b/llvm/test/Transforms/SLPVectorizer/X86/gathered-shuffle-resized.ll
@@ -11,16 +11,16 @@ define ptr @test(ptr %0, ptr %args_gep) {
; CHECK-NEXT: [[TMP2:%.*]] = shufflevector <2 x i32> [[TMP1]], <2 x i32> poison, <16 x i32> <i32 0, i32 0, i32 0, i32 0, i32 0, i32 0, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 0, i32 0, i32 0, i32 0>
; CHECK-NEXT: [[TMP3:%.*]] = icmp eq <16 x i32> [[TMP2]], zeroinitializer
; CHECK-NEXT: [[TMP4:%.*]] = zext <16 x i1> [[TMP3]] to <16 x i8>
-; CHECK-NEXT: store <16 x i8> [[TMP4]], ptr [[ARG26]], align 32
; CHECK-NEXT: [[TMP5:%.*]] = getelementptr i8, ptr [[ARG26]], i64 17
; CHECK-NEXT: [[TMP6:%.*]] = getelementptr i8, ptr [[ARG1]], i64 8
; CHECK-NEXT: [[TMP7:%.*]] = getelementptr i8, ptr [[ARG1]], i64 12
-; CHECK-NEXT: [[TMP8:%.*]] = load i32, ptr [[TMP7]], align 4, !noalias [[META0:![0-9]+]]
-; CHECK-NEXT: [[TMP9:%.*]] = load <2 x i32>, ptr [[TMP6]], align 8, !noalias [[META0]]
-; CHECK-NEXT: [[TMP10:%.*]] = shufflevector <2 x i32> [[TMP1]], <2 x i32> poison, <16 x i32> <i32 0, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
-; CHECK-NEXT: [[TMP11:%.*]] = shufflevector <2 x i32> [[TMP9]], <2 x i32> poison, <16 x i32> <i32 0, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 1, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
-; CHECK-NEXT: [[TMP12:%.*]] = shufflevector <16 x i32> [[TMP10]], <16 x i32> [[TMP11]], <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 16, i32 8, i32 9, i32 10, i32 11, i32 12, i32 22, i32 14, i32 15>
-; CHECK-NEXT: [[TMP13:%.*]] = shufflevector <16 x i32> [[TMP12]], <16 x i32> poison, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 7, i32 7, i32 7, i32 7, i32 7, i32 13, i32 13, i32 13>
+; CHECK-NEXT: store <16 x i8> [[TMP4]], ptr [[ARG26]], align 32
+; CHECK-NEXT: [[TMP9:%.*]] = load <2 x i32>, ptr [[TMP6]], align 8, !noalias [[META0:![0-9]+]]
+; CHECK-NEXT: [[TMP8:%.*]] = load i32, ptr [[TMP7]], align 4, !noalias [[META0]]
+; CHECK-NEXT: [[TMP10:%.*]] = shufflevector <2 x i32> [[TMP1]], <2 x i32> poison, <4 x i32> <i32 0, i32 1, i32 poison, i32 poison>
+; CHECK-NEXT: [[TMP11:%.*]] = shufflevector <2 x i32> [[TMP9]], <2 x i32> poison, <16 x i32> <i32 poison, i32 poison, i32 0, i32 1, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
+; CHECK-NEXT: [[TMP12:%.*]] = shufflevector <4 x i32> [[TMP10]], <4 x i32> poison, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
+; CHECK-NEXT: [[TMP13:%.*]] = shufflevector <2 x i32> [[TMP1]], <2 x i32> [[TMP9]], <16 x i32> <i32 0, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 3, i32 3, i32 3>
; CHECK-NEXT: [[TMP14:%.*]] = icmp eq <16 x i32> [[TMP13]], zeroinitializer
; CHECK-NEXT: [[TMP15:%.*]] = zext <16 x i1> [[TMP14]] to <16 x i8>
; CHECK-NEXT: store <16 x i8> [[TMP15]], ptr [[TMP5]], align 1
diff --git a/llvm/test/Transforms/SLPVectorizer/X86/horizontal-list.ll b/llvm/test/Transforms/SLPVectorizer/X86/horizontal-list.ll
index 4e434a61e1f1c..1ca2518ddf1df 100644
--- a/llvm/test/Transforms/SLPVectorizer/X86/horizontal-list.ll
+++ b/llvm/test/Transforms/SLPVectorizer/X86/horizontal-list.ll
@@ -914,14 +914,19 @@ define float @extra_args_no_fast(ptr %x, float %a, float %b) {
; THRESHOLD-LABEL: @extra_args_no_fast(
; THRESHOLD-NEXT: [[ADDC:%.*]] = fadd fast float [[B:%.*]], 3.000000e+00
; THRESHOLD-NEXT: [[ADD:%.*]] = fadd fast float [[A:%.*]], [[ADDC]]
-; THRESHOLD-NEXT: [[TMP1:%.*]] = load <4 x float>, ptr [[X:%.*]], align 4
-; THRESHOLD-NEXT: [[T0:%.*]] = extractelement <4 x float> [[TMP1]], i32 0
-; THRESHOLD-NEXT: [[ADD1:%.*]] = fadd fast float [[T0]], [[ADD]]
-; THRESHOLD-NEXT: [[T1:%.*]] = extractelement <4 x float> [[TMP1]], i32 1
+; THRESHOLD-NEXT: [[ARRAYIDX3:%.*]] = getelementptr inbounds float, ptr [[X:%.*]], i64 1
+; THRESHOLD-NEXT: [[ARRAYIDX3_1:%.*]] = getelementptr inbounds float, ptr [[X]], i64 2
+; THRESHOLD-NEXT: [[ARRAYIDX3_2:%.*]] = getelementptr inbounds float, ptr [[X]], i64 3
+; THRESHOLD-NEXT: [[TMP1:%.*]] = load <4 x float>, ptr [[X]], align 4
+; THRESHOLD-NEXT: [[T3:%.*]] = load float, ptr [[ARRAYIDX3_2]], align 4
+; THRESHOLD-NEXT: [[TMP2:%.*]] = load <4 x float>, ptr [[X]], align 4
+; THRESHOLD-NEXT: [[T2:%.*]] = load float, ptr [[ARRAYIDX3_1]], align 4
+; THRESHOLD-NEXT: [[T1:%.*]] = load float, ptr [[ARRAYIDX3]], align 4
+; THRESHOLD-NEXT: [[TMP3:%.*]] = extractelement <4 x float> [[TMP1]], i32 0
+; THRESHOLD-NEXT: [[TMP4:%.*]] = extractelement <4 x float> [[TMP2]], i32 0
+; THRESHOLD-NEXT: [[ADD1:%.*]] = fadd fast float [[TMP3]], [[ADD]]
; THRESHOLD-NEXT: [[ADD4:%.*]] = fadd fast float [[T1]], [[ADD1]]
-; THRESHOLD-NEXT: [[T2:%.*]] = extractelement <4 x float> [[TMP1]], i32 2
; THRESHOLD-NEXT: [[ADD4_1:%.*]] = fadd float [[T2]], [[ADD4]]
-; THRESHOLD-NEXT: [[T3:%.*]] = extractelement <4 x float> [[TMP1]], i32 3
; THRESHOLD-NEXT: [[ADD4_2:%.*]] = fadd fast float [[T3]], [[ADD4_1]]
; THRESHOLD-NEXT: [[ADD5:%.*]] = fadd fast float [[ADD4_2]], [[A]]
; THRESHOLD-NEXT: ret float [[ADD5]]
diff --git a/llvm/test/Transforms/SLPVectorizer/X86/horizontal-minmax.ll b/llvm/test/Transforms/SLPVectorizer/X86/horizontal-minmax.ll
index b2b9363565277..03c34b4cab092 100644
--- a/llvm/test/Transforms/SLPVectorizer/X86/horizontal-minmax.ll
+++ b/llvm/test/Transforms/SLPVectorizer/X86/horizontal-minmax.ll
@@ -268,9 +268,9 @@ define float @maxf8(float) {
; DEFAULT-NEXT: ret float [[TMP23]]
;
; THRESH-LABEL: @maxf8(
+; THRESH-NEXT: [[TMP4:%.*]] = load float, ptr getelementptr inbounds ([32 x float], ptr @arr1, i64 0, i64 1), align 4
; THRESH-NEXT: [[TMP2:%.*]] = load <2 x float>, ptr @arr1, align 16
; THRESH-NEXT: [[TMP3:%.*]] = extractelement <2 x float> [[TMP2]], i32 0
-; THRESH-NEXT: [[TMP4:%.*]] = extractelement <2 x float> [[TMP2]], i32 1
; THRESH-NEXT: [[TMP5:%.*]] = fcmp fast ogt float [[TMP3]], [[TMP4]]
; THRESH-NEXT: [[TMP6:%.*]] = select i1 [[TMP5]], float [[TMP3]], float [[TMP4]]
; THRESH-NEXT: [[TMP7:%.*]] = load float, ptr getelementptr inbounds ([32 x float], ptr @arr1, i64 0, i64 2), align 8
@@ -371,9 +371,9 @@ define float @maxf16(float) {
; DEFAULT-NEXT: ret float [[TMP47]]
;
; THRESH-LABEL: @maxf16(
+; THRESH-NEXT: [[TMP4:%.*]] = load float, ptr getelementptr inbounds ([32 x float], ptr @arr1, i64 0, i64 1), align 4
; THRESH-NEXT: [[TMP2:%.*]] = load <2 x float>, ptr @arr1, align 16
; THRESH-NEXT: [[TMP3:%.*]] = extractelement <2 x float> [[TMP2]], i32 0
-; THRESH-NEXT: [[TMP4:%.*]] = extractelement <2 x float> [[TMP2]], i32 1
; THRESH-NEXT: [[TMP5:%.*]] = fcmp fast ogt float [[TMP3]], [[TMP4]]
; THRESH-NEXT: [[TMP6:%.*]] = select i1 [[TMP5]], float [[TMP3]], float [[TMP4]]
; THRESH-NEXT: [[TMP7:%.*]] = load float, ptr getelementptr inbounds ([32 x float], ptr @arr1, i64 0, i64 2), align 8
@@ -570,9 +570,9 @@ define float @maxf32(float) {
; DEFAULT-NEXT: ret float [[TMP95]]
;
; THRESH-LABEL: @maxf32(
+; THRESH-NEXT: [[TMP4:%.*]] = load float, ptr getelementptr inbounds ([32 x float], ptr @arr1, i64 0, i64 1), align 4
; THRESH-NEXT: [[TMP2:%.*]] = load <2 x float>, ptr @arr1, align 16
; THRESH-NEXT: [[TMP3:%.*]] = extractelement <2 x float> [[TMP2]], i32 0
-; THRESH-NEXT: [[TMP4:%.*]] = extractelement <2 x float> [[TMP2]], i32 1
; THRESH-NEXT: [[TMP5:%.*]] = fcmp fast ogt float [[TMP3]], [[TMP4]]
; THRESH-NEXT: [[TMP6:%.*]] = select i1 [[TMP5]], float [[TMP3]], float [[TMP4]]
; THRESH-NEXT: [[TMP7:%.*]] = load float, ptr getelementptr inbounds ([32 x float], ptr @arr1, i64 0, i64 2), align 8
@@ -785,9 +785,8 @@ define i32 @maxi8_mutiple_uses(i32) {
; DEFAULT-NEXT: ret i32 [[OP_RDX5]]
;
; THRESH-LABEL: @maxi8_mutiple_uses(
-; THRESH-NEXT: [[TMP2:%.*]] = load <2 x i32>, ptr @arr, align 16
-; THRESH-NEXT: [[TMP3:%.*]] = extractelement <2 x i32> [[TMP2]], i32 0
-; THRESH-NEXT: [[TMP4:%.*]] = extractelement <2 x i32> [[TMP2]], i32 1
+; THRESH-NEXT: [[TMP4:%.*]] = load i32, ptr getelementptr inbounds ([32 x i32], ptr @arr, i64 0, i64 1), align 4
+; THRESH-NEXT: [[TMP3:%.*]] = load i32, ptr @arr, align 16
; THRESH-NEXT: [[TMP5:%.*]] = icmp sgt i32 [[TMP3]], [[TMP4]]
; THRESH-NEXT: [[TMP6:%.*]] = select i1 [[TMP5]], i32 [[TMP3]], i32 [[TMP4]]
; THRESH-NEXT: [[TMP7:%.*]] = load <4 x i32>, ptr getelementptr inbounds ([32 x i32], ptr @arr, i64 0, i64 2), align 8
@@ -855,15 +854,14 @@ define i32 @maxi8_mutiple_uses2(i32) {
; DEFAULT-NEXT: ret i32 [[TMP17]]
;
; THRESH-LABEL: @maxi8_mutiple_uses2(
-; THRESH-NEXT: [[TMP2:%.*]] = load <4 x i32>, ptr @arr, align 16
-; THRESH-NEXT: [[TMP3:%.*]] = extractelement <4 x i32> [[TMP2]], i32 0
-; THRESH-NEXT: [[TMP4:%.*]] = extractelement <4 x i32> [[TMP2]], i32 1
+; THRESH-NEXT: [[TMP10:%.*]] = load i32, ptr getelementptr inbounds ([32 x i32], ptr @arr, i64 0, i64 3), align 4
+; THRESH-NEXT: [[TMP7:%.*]] = load i32, ptr getelementptr inbounds ([32 x i32], ptr @arr, i64 0, i64 2), align 8
+; THRESH-NEXT: [[TMP4:%.*]] = load i32, ptr getelementptr inbounds ([32 x i32], ptr @arr, i64 0, i64 1), align 4
+; THRESH-NEXT: [[TMP3:%.*]] = load i32, ptr @arr, align 16
; THRESH-NEXT: [[TMP5:%.*]] = icmp sgt i32 [[TMP3]], [[TMP4]]
; THRESH-NEXT: [[TMP6:%.*]] = select i1 [[TMP5]], i32 [[TMP3]], i32 [[TMP4]]
-; THRESH-NEXT: [[TMP7:%.*]] = extractelement <4 x i32> [[TMP2]], i32 2
; THRESH-NEXT: [[TMP8:%.*]] = icmp sgt i32 [[TMP6]], [[TMP7]]
; THRESH-NEXT: [[TMP9:%.*]] = select i1 [[TMP8]], i32 [[TMP6]], i32 [[TMP7]]
-; THRESH-NEXT: [[TMP10:%.*]] = extractelement <4 x i32> [[TMP2]], i32 3
; THRESH-NEXT: [[TMP11:%.*]] = icmp sgt i32 [[TMP9]], [[TMP10]]
; THRESH-NEXT: [[TMP12:%.*]] = select i1 [[TMP11]], i32 [[TMP9]], i32 [[TMP10]]
; THRESH-NEXT: [[TMP13:%.*]] = load i32, ptr getelementptr inbounds ([32 x i32], ptr @arr, i64 0, i64 4), align 16
@@ -969,9 +967,8 @@ define ptr @maxp8(i32) {
; DEFAULT-NEXT: ret ptr [[TMP23]]
;
; THRESH-LABEL: @maxp8(
-; THRESH-NEXT: [[TMP2:%.*]] = load <2 x ptr>, ptr @arrp, align 16
-; THRESH-NEXT: [[TMP3:%.*]] = extractelement <2 x ptr> [[TMP2]], i32 0
-; THRESH-NEXT: [[TMP4:%.*]] = extractelement <2 x ptr> [[TMP2]], i32 1
+; THRESH-NEXT: [[TMP4:%.*]] = load ptr, ptr getelementptr inbounds ([32 x ptr], ptr @arrp, i64 0, i64 1), align 4
+; THRESH-NEXT: [[TMP3:%.*]] = load ptr, ptr @arrp, align 16
; THRESH-NEXT: [[TMP5:%.*]] = icmp ugt ptr [[TMP3]], [[TMP4]]
; THRESH-NEXT: [[TMP6:%.*]] = select i1 [[TMP5]], ptr [[TMP3]], ptr [[TMP4]]
; THRESH-NEXT: [[TMP7:%.*]] = load ptr, ptr getelementptr inbounds ([32 x ptr], ptr @arrp, i64 0, i64 2), align 8
diff --git a/llvm/test/Transforms/SLPVectorizer/X86/split-node-last-inst-vectorized.ll b/llvm/test/Transforms/SLPVectorizer/X86/split-node-last-inst-vectorized.ll
index 47a25a089c69e..96e9dd06026fb 100644
--- a/llvm/test/Transforms/SLPVectorizer/X86/split-node-last-inst-vectorized.ll
+++ b/llvm/test/Transforms/SLPVectorizer/X86/split-node-last-inst-vectorized.ll
@@ -14,7 +14,7 @@ define void @test(ptr %0, <8 x i8> %1) {
; CHECK-NEXT: [[TMP12:%.*]] = insertelement <8 x i8> poison, i8 [[TMP6]], i32 0
; CHECK-NEXT: [[TMP9:%.*]] = insertelement <8 x i8> [[TMP12]], i8 [[TMP3]], i32 1
; CHECK-NEXT: [[TMP10:%.*]] = shufflevector <8 x i8> [[TMP9]], <8 x i8> poison, <8 x i32> <i32 0, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1>
-; CHECK-NEXT: [[TMP13:%.*]] = or <8 x i8> [[TMP10]], [[TMP7]]
+; CHECK-NEXT: [[TMP13:%.*]] = or <8 x i8> [[TMP7]], [[TMP10]]
; CHECK-NEXT: store <8 x i8> [[TMP13]], ptr [[TMP11]], align 4
; CHECK-NEXT: [[TMP14:%.*]] = shufflevector <8 x i8> [[TMP1]], <8 x i8> poison, <8 x i32> <i32 0, i32 0, i32 2, i32 3, i32 4, i32 5, i32 0, i32 7>
; CHECK-NEXT: [[TMP15:%.*]] = or <8 x i8> [[TMP7]], [[TMP14]]
diff --git a/llvm/test/Transforms/SLPVectorizer/X86/split-vector-operand-with-reuses.ll b/llvm/test/Transforms/SLPVectorizer/X86/split-vector-operand-with-reuses.ll
index 972a58cecc822..cfe4c1b492a6b 100644
--- a/llvm/test/Transforms/SLPVectorizer/X86/split-vector-operand-with-reuses.ll
+++ b/llvm/test/Transforms/SLPVectorizer/X86/split-vector-operand-with-reuses.ll
@@ -12,27 +12,27 @@ define void @test(ptr %p) {
; CHECK-NEXT: [[TMP2:%.*]] = shufflevector <4 x i32> [[TMP1]], <4 x i32> poison, <4 x i32> <i32 2, i32 3, i32 0, i32 1>
; CHECK-NEXT: [[TMP19:%.*]] = shufflevector <4 x i32> [[TMP2]], <4 x i32> poison, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
; CHECK-NEXT: [[TMP3:%.*]] = shufflevector <16 x i32> <i32 0, i32 0, i32 0, i32 0, i32 poison, i32 poison, i32 poison, i32 poison, i32 0, i32 0, i32 0, i32 0, i32 0, i32 0, i32 0, i32 0>, <16 x i32> [[TMP19]], <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 16, i32 17, i32 18, i32 19, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15>
-; CHECK-NEXT: [[TMP4:%.*]] = shufflevector <4 x i32> [[TMP2]], <4 x i32> poison, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 0, i32 1, i32 2, i32 3>
-; CHECK-NEXT: [[TMP5:%.*]] = load <4 x i32>, ptr [[ARRAYIDX7_US_I_841]], align 4
-; CHECK-NEXT: [[TMP6:%.*]] = shufflevector <8 x i32> [[TMP4]], <8 x i32> poison, <12 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 poison, i32 poison, i32 poison, i32 poison>
-; CHECK-NEXT: [[TMP20:%.*]] = shufflevector <4 x i32> [[TMP5]], <4 x i32> poison, <12 x i32> <i32 0, i32 1, i32 2, i32 3, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
-; CHECK-NEXT: [[TMP7:%.*]] = shufflevector <12 x i32> [[TMP6]], <12 x i32> [[TMP20]], <12 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 12, i32 13, i32 14, i32 15>
-; CHECK-NEXT: [[TMP8:%.*]] = shufflevector <4 x i32> [[TMP5]], <4 x i32> poison, <16 x i32> <i32 poison, i32 poison, i32 2, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
-; CHECK-NEXT: [[TMP9:%.*]] = shufflevector <4 x i32> [[TMP5]], <4 x i32> poison, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
+; CHECK-NEXT: [[TMP5:%.*]] = or <4 x i32> [[TMP2]], zeroinitializer
+; CHECK-NEXT: [[TMP7:%.*]] = srem <4 x i32> [[TMP5]], <i32 0, i32 0, i32 1, i32 1>
+; CHECK-NEXT: [[TMP8:%.*]] = load <8 x i32>, ptr [[ARRAYIDX7_US_I_841]], align 4
+; CHECK-NEXT: [[TMP15:%.*]] = shufflevector <8 x i32> [[TMP8]], <8 x i32> poison, <16 x i32> <i32 poison, i32 poison, i32 2, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
+; CHECK-NEXT: [[TMP9:%.*]] = shufflevector <8 x i32> [[TMP8]], <8 x i32> poison, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
; CHECK-NEXT: [[TMP10:%.*]] = shufflevector <16 x i32> <i32 0, i32 0, i32 0, i32 0, i32 poison, i32 poison, i32 poison, i32 poison, i32 0, i32 0, i32 0, i32 0, i32 0, i32 0, i32 0, i32 0>, <16 x i32> [[TMP9]], <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 18, i32 poison, i32 poison, i32 poison, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15>
; CHECK-NEXT: [[TMP11:%.*]] = insertelement <16 x i32> [[TMP10]], i32 [[TMP0]], i32 6
; CHECK-NEXT: [[TMP12:%.*]] = shufflevector <16 x i32> [[TMP11]], <16 x i32> poison, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 4, i32 6, i32 6, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15>
; CHECK-NEXT: [[TMP13:%.*]] = add <16 x i32> [[TMP3]], [[TMP12]]
; CHECK-NEXT: [[TMP14:%.*]] = srem <16 x i32> [[TMP13]], <i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 0, i32 0, i32 1, i32 1, i32 1, i32 1>
-; CHECK-NEXT: [[TMP15:%.*]] = or <12 x i32> [[TMP7]], zeroinitializer
-; CHECK-NEXT: [[TMP16:%.*]] = srem <12 x i32> [[TMP15]], <i32 0, i32 0, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1>
+; CHECK-NEXT: [[TMP16:%.*]] = or <8 x i32> [[TMP8]], zeroinitializer
+; CHECK-NEXT: [[TMP20:%.*]] = srem <8 x i32> [[TMP16]], splat (i32 1)
; CHECK-NEXT: br label %[[FOR_COND1_PREHEADER_US_I:.*]]
; CHECK: [[FOR_COND1_PREHEADER_US_I]]:
-; CHECK-NEXT: [[A_PROMOTED253537_US_I:%.*]] = phi i32 [ [[OP_RDX8:%.*]], %[[FOR_COND1_PREHEADER_US_I]] ], [ 0, %[[ENTRY]] ]
+; CHECK-NEXT: [[A_PROMOTED253537_US_I:%.*]] = phi i32 [ [[OP_RDX6:%.*]], %[[FOR_COND1_PREHEADER_US_I]] ], [ 0, %[[ENTRY]] ]
; CHECK-NEXT: [[TMP17:%.*]] = call i32 @llvm.vector.reduce.add.v16i32(<16 x i32> [[TMP14]])
-; CHECK-NEXT: [[TMP18:%.*]] = call i32 @llvm.vector.reduce.add.v12i32(<12 x i32> [[TMP16]])
+; CHECK-NEXT: [[TMP21:%.*]] = call i32 @llvm.vector.reduce.add.v4i32(<4 x i32> [[TMP7]])
+; CHECK-NEXT: [[TMP18:%.*]] = call i32 @llvm.vector.reduce.add.v8i32(<8 x i32> [[TMP20]])
; CHECK-NEXT: [[OP_RDX:%.*]] = add i32 [[TMP18]], [[TMP17]]
-; CHECK-NEXT: [[OP_RDX8]] = add i32 [[OP_RDX]], 0
+; CHECK-NEXT: [[OP_RDX5:%.*]] = add i32 [[TMP21]], 0
+; CHECK-NEXT: [[OP_RDX6]] = add i32 [[OP_RDX]], [[OP_RDX5]]
; CHECK-NEXT: br label %[[FOR_COND1_PREHEADER_US_I]]
;
entry:
>From 29e3400aa868df32b06db91f6a791b3b8ad7c98a Mon Sep 17 00:00:00 2001
From: Ryan Buchner <buchner.ryan at gmail.com>
Date: Tue, 21 Jul 2026 10:35:38 -0700
Subject: [PATCH 02/22] Fix accidentally deleted test
---
.../AArch64/multiple_reduction.ll | 201 +++++++++++++++---
1 file changed, 174 insertions(+), 27 deletions(-)
diff --git a/llvm/test/Transforms/SLPVectorizer/AArch64/multiple_reduction.ll b/llvm/test/Transforms/SLPVectorizer/AArch64/multiple_reduction.ll
index c44c3bbf32553..89b0bf3f2179d 100644
--- a/llvm/test/Transforms/SLPVectorizer/AArch64/multiple_reduction.ll
+++ b/llvm/test/Transforms/SLPVectorizer/AArch64/multiple_reduction.ll
@@ -21,33 +21,33 @@ define i64 @straight(ptr nocapture noundef readonly %p, i32 noundef %st) {
; CHECK-NEXT: [[ADD_PTR_4:%.*]] = getelementptr inbounds i16, ptr [[ADD_PTR_3]], i64 [[IDX_EXT]]
; CHECK-NEXT: [[ADD_PTR_5:%.*]] = getelementptr inbounds i16, ptr [[ADD_PTR_4]], i64 [[IDX_EXT]]
; CHECK-NEXT: [[ADD_PTR_6:%.*]] = getelementptr inbounds i16, ptr [[ADD_PTR_5]], i64 [[IDX_EXT]]
-; CHECK-NEXT: [[TMP1:%.*]] = load <8 x i16>, ptr [[P]], align 2
-; CHECK-NEXT: [[TMP10:%.*]] = load <8 x i16>, ptr [[ADD_PTR]], align 2
-; CHECK-NEXT: [[TMP19:%.*]] = load <8 x i16>, ptr [[ADD_PTR_1]], align 2
-; CHECK-NEXT: [[TMP28:%.*]] = load <8 x i16>, ptr [[ADD_PTR_2]], align 2
-; CHECK-NEXT: [[TMP37:%.*]] = load <8 x i16>, ptr [[ADD_PTR_3]], align 2
-; CHECK-NEXT: [[TMP46:%.*]] = load <8 x i16>, ptr [[ADD_PTR_4]], align 2
-; CHECK-NEXT: [[TMP55:%.*]] = load <8 x i16>, ptr [[ADD_PTR_5]], align 2
-; CHECK-NEXT: [[TMP64:%.*]] = load <8 x i16>, ptr [[ADD_PTR_6]], align 2
-; CHECK-NEXT: [[TMP72:%.*]] = shufflevector <8 x i16> [[TMP64]], <8 x i16> poison, <64 x i32> <i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>
-; CHECK-NEXT: [[TMP73:%.*]] = shufflevector <8 x i16> [[TMP55]], <8 x i16> poison, <64 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
-; CHECK-NEXT: [[TMP74:%.*]] = shufflevector <8 x i16> [[TMP64]], <8 x i16> [[TMP55]], <64 x i32> <i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15, i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>
-; CHECK-NEXT: [[TMP75:%.*]] = shufflevector <8 x i16> [[TMP46]], <8 x i16> poison, <64 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
-; CHECK-NEXT: [[TMP76:%.*]] = shufflevector <64 x i16> [[TMP74]], <64 x i16> [[TMP75]], <64 x i32> <i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 64, i32 65, i32 66, i32 67, i32 68, i32 69, i32 70, i32 71, i32 48, i32 49, i32 50, i32 51, i32 52, i32 53, i32 54, i32 55, i32 56, i32 57, i32 58, i32 59, i32 60, i32 61, i32 62, i32 63>
-; CHECK-NEXT: [[TMP77:%.*]] = shufflevector <8 x i16> [[TMP37]], <8 x i16> poison, <64 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
-; CHECK-NEXT: [[TMP78:%.*]] = shufflevector <64 x i16> [[TMP76]], <64 x i16> [[TMP77]], <64 x i32> <i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 64, i32 65, i32 66, i32 67, i32 68, i32 69, i32 70, i32 71, i32 40, i32 41, i32 42, i32 43, i32 44, i32 45, i32 46, i32 47, i32 48, i32 49, i32 50, i32 51, i32 52, i32 53, i32 54, i32 55, i32 56, i32 57, i32 58, i32 59, i32 60, i32 61, i32 62, i32 63>
-; CHECK-NEXT: [[TMP79:%.*]] = shufflevector <8 x i16> [[TMP28]], <8 x i16> poison, <64 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
-; CHECK-NEXT: [[TMP80:%.*]] = shufflevector <64 x i16> [[TMP78]], <64 x i16> [[TMP79]], <64 x i32> <i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 64, i32 65, i32 66, i32 67, i32 68, i32 69, i32 70, i32 71, i32 32, i32 33, i32 34, i32 35, i32 36, i32 37, i32 38, i32 39, i32 40, i32 41, i32 42, i32 43, i32 44, i32 45, i32 46, i32 47, i32 48, i32 49, i32 50, i32 51, i32 52, i32 53, i32 54, i32 55, i32 56, i32 57, i32 58, i32 59, i32 60, i32 61, i32 62, i32 63>
-; CHECK-NEXT: [[TMP81:%.*]] = shufflevector <8 x i16> [[TMP19]], <8 x i16> poison, <64 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
-; CHECK-NEXT: [[TMP82:%.*]] = shufflevector <64 x i16> [[TMP80]], <64 x i16> [[TMP81]], <64 x i32> <i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 64, i32 65, i32 66, i32 67, i32 68, i32 69, i32 70, i32 71, i32 24, i32 25, i32 26, i32 27, i32 28, i32 29, i32 30, i32 31, i32 32, i32 33, i32 34, i32 35, i32 36, i32 37, i32 38, i32 39, i32 40, i32 41, i32 42, i32 43, i32 44, i32 45, i32 46, i32 47, i32 48, i32 49, i32 50, i32 51, i32 52, i32 53, i32 54, i32 55, i32 56, i32 57, i32 58, i32 59, i32 60, i32 61, i32 62, i32 63>
-; CHECK-NEXT: [[TMP83:%.*]] = shufflevector <8 x i16> [[TMP1]], <8 x i16> poison, <64 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
-; CHECK-NEXT: [[TMP84:%.*]] = shufflevector <64 x i16> [[TMP82]], <64 x i16> [[TMP83]], <64 x i32> <i32 64, i32 65, i32 66, i32 67, i32 68, i32 69, i32 70, i32 71, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 16, i32 17, i32 18, i32 19, i32 20, i32 21, i32 22, i32 23, i32 24, i32 25, i32 26, i32 27, i32 28, i32 29, i32 30, i32 31, i32 32, i32 33, i32 34, i32 35, i32 36, i32 37, i32 38, i32 39, i32 40, i32 41, i32 42, i32 43, i32 44, i32 45, i32 46, i32 47, i32 48, i32 49, i32 50, i32 51, i32 52, i32 53, i32 54, i32 55, i32 56, i32 57, i32 58, i32 59, i32 60, i32 61, i32 62, i32 63>
-; CHECK-NEXT: [[TMP85:%.*]] = shufflevector <8 x i16> [[TMP10]], <8 x i16> poison, <64 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
-; CHECK-NEXT: [[TMP86:%.*]] = shufflevector <64 x i16> [[TMP84]], <64 x i16> [[TMP85]], <64 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 64, i32 65, i32 66, i32 67, i32 68, i32 69, i32 70, i32 71, i32 16, i32 17, i32 18, i32 19, i32 20, i32 21, i32 22, i32 23, i32 24, i32 25, i32 26, i32 27, i32 28, i32 29, i32 30, i32 31, i32 32, i32 33, i32 34, i32 35, i32 36, i32 37, i32 38, i32 39, i32 40, i32 41, i32 42, i32 43, i32 44, i32 45, i32 46, i32 47, i32 48, i32 49, i32 50, i32 51, i32 52, i32 53, i32 54, i32 55, i32 56, i32 57, i32 58, i32 59, i32 60, i32 61, i32 62, i32 63>
-; CHECK-NEXT: [[TMP87:%.*]] = zext <64 x i16> [[TMP86]] to <64 x i32>
-; CHECK-NEXT: [[ADD_7_7:%.*]] = call i32 @llvm.vector.reduce.add.v64i32(<64 x i32> [[TMP87]])
-; CHECK-NEXT: [[TMP88:%.*]] = mul nuw nsw <64 x i32> [[TMP87]], [[TMP87]]
-; CHECK-NEXT: [[ADD11_7_7:%.*]] = call i32 @llvm.vector.reduce.add.v64i32(<64 x i32> [[TMP88]])
+; CHECK-NEXT: [[TMP0:%.*]] = load <8 x i16>, ptr [[P]], align 2
+; CHECK-NEXT: [[TMP1:%.*]] = load <8 x i16>, ptr [[ADD_PTR]], align 2
+; CHECK-NEXT: [[TMP2:%.*]] = load <8 x i16>, ptr [[ADD_PTR_1]], align 2
+; CHECK-NEXT: [[TMP3:%.*]] = load <8 x i16>, ptr [[ADD_PTR_2]], align 2
+; CHECK-NEXT: [[TMP4:%.*]] = load <8 x i16>, ptr [[ADD_PTR_3]], align 2
+; CHECK-NEXT: [[TMP5:%.*]] = load <8 x i16>, ptr [[ADD_PTR_4]], align 2
+; CHECK-NEXT: [[TMP6:%.*]] = load <8 x i16>, ptr [[ADD_PTR_5]], align 2
+; CHECK-NEXT: [[TMP7:%.*]] = load <8 x i16>, ptr [[ADD_PTR_6]], align 2
+; CHECK-NEXT: [[TMP8:%.*]] = shufflevector <8 x i16> [[TMP7]], <8 x i16> poison, <64 x i32> <i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>
+; CHECK-NEXT: [[TMP9:%.*]] = shufflevector <8 x i16> [[TMP6]], <8 x i16> poison, <64 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
+; CHECK-NEXT: [[TMP10:%.*]] = shufflevector <8 x i16> [[TMP7]], <8 x i16> [[TMP6]], <64 x i32> <i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15, i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>
+; CHECK-NEXT: [[TMP11:%.*]] = shufflevector <8 x i16> [[TMP5]], <8 x i16> poison, <64 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
+; CHECK-NEXT: [[TMP12:%.*]] = shufflevector <64 x i16> [[TMP10]], <64 x i16> [[TMP11]], <64 x i32> <i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 64, i32 65, i32 66, i32 67, i32 68, i32 69, i32 70, i32 71, i32 48, i32 49, i32 50, i32 51, i32 52, i32 53, i32 54, i32 55, i32 56, i32 57, i32 58, i32 59, i32 60, i32 61, i32 62, i32 63>
+; CHECK-NEXT: [[TMP13:%.*]] = shufflevector <8 x i16> [[TMP4]], <8 x i16> poison, <64 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
+; CHECK-NEXT: [[TMP14:%.*]] = shufflevector <64 x i16> [[TMP12]], <64 x i16> [[TMP13]], <64 x i32> <i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 64, i32 65, i32 66, i32 67, i32 68, i32 69, i32 70, i32 71, i32 40, i32 41, i32 42, i32 43, i32 44, i32 45, i32 46, i32 47, i32 48, i32 49, i32 50, i32 51, i32 52, i32 53, i32 54, i32 55, i32 56, i32 57, i32 58, i32 59, i32 60, i32 61, i32 62, i32 63>
+; CHECK-NEXT: [[TMP15:%.*]] = shufflevector <8 x i16> [[TMP3]], <8 x i16> poison, <64 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
+; CHECK-NEXT: [[TMP16:%.*]] = shufflevector <64 x i16> [[TMP14]], <64 x i16> [[TMP15]], <64 x i32> <i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 64, i32 65, i32 66, i32 67, i32 68, i32 69, i32 70, i32 71, i32 32, i32 33, i32 34, i32 35, i32 36, i32 37, i32 38, i32 39, i32 40, i32 41, i32 42, i32 43, i32 44, i32 45, i32 46, i32 47, i32 48, i32 49, i32 50, i32 51, i32 52, i32 53, i32 54, i32 55, i32 56, i32 57, i32 58, i32 59, i32 60, i32 61, i32 62, i32 63>
+; CHECK-NEXT: [[TMP17:%.*]] = shufflevector <8 x i16> [[TMP2]], <8 x i16> poison, <64 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
+; CHECK-NEXT: [[TMP18:%.*]] = shufflevector <64 x i16> [[TMP16]], <64 x i16> [[TMP17]], <64 x i32> <i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 64, i32 65, i32 66, i32 67, i32 68, i32 69, i32 70, i32 71, i32 24, i32 25, i32 26, i32 27, i32 28, i32 29, i32 30, i32 31, i32 32, i32 33, i32 34, i32 35, i32 36, i32 37, i32 38, i32 39, i32 40, i32 41, i32 42, i32 43, i32 44, i32 45, i32 46, i32 47, i32 48, i32 49, i32 50, i32 51, i32 52, i32 53, i32 54, i32 55, i32 56, i32 57, i32 58, i32 59, i32 60, i32 61, i32 62, i32 63>
+; CHECK-NEXT: [[TMP19:%.*]] = shufflevector <8 x i16> [[TMP0]], <8 x i16> poison, <64 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
+; CHECK-NEXT: [[TMP20:%.*]] = shufflevector <64 x i16> [[TMP18]], <64 x i16> [[TMP19]], <64 x i32> <i32 64, i32 65, i32 66, i32 67, i32 68, i32 69, i32 70, i32 71, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 16, i32 17, i32 18, i32 19, i32 20, i32 21, i32 22, i32 23, i32 24, i32 25, i32 26, i32 27, i32 28, i32 29, i32 30, i32 31, i32 32, i32 33, i32 34, i32 35, i32 36, i32 37, i32 38, i32 39, i32 40, i32 41, i32 42, i32 43, i32 44, i32 45, i32 46, i32 47, i32 48, i32 49, i32 50, i32 51, i32 52, i32 53, i32 54, i32 55, i32 56, i32 57, i32 58, i32 59, i32 60, i32 61, i32 62, i32 63>
+; CHECK-NEXT: [[TMP21:%.*]] = shufflevector <8 x i16> [[TMP1]], <8 x i16> poison, <64 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
+; CHECK-NEXT: [[TMP22:%.*]] = shufflevector <64 x i16> [[TMP20]], <64 x i16> [[TMP21]], <64 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 64, i32 65, i32 66, i32 67, i32 68, i32 69, i32 70, i32 71, i32 16, i32 17, i32 18, i32 19, i32 20, i32 21, i32 22, i32 23, i32 24, i32 25, i32 26, i32 27, i32 28, i32 29, i32 30, i32 31, i32 32, i32 33, i32 34, i32 35, i32 36, i32 37, i32 38, i32 39, i32 40, i32 41, i32 42, i32 43, i32 44, i32 45, i32 46, i32 47, i32 48, i32 49, i32 50, i32 51, i32 52, i32 53, i32 54, i32 55, i32 56, i32 57, i32 58, i32 59, i32 60, i32 61, i32 62, i32 63>
+; CHECK-NEXT: [[TMP23:%.*]] = zext <64 x i16> [[TMP22]] to <64 x i32>
+; CHECK-NEXT: [[ADD_7_7:%.*]] = call i32 @llvm.vector.reduce.add.v64i32(<64 x i32> [[TMP23]])
+; CHECK-NEXT: [[TMP25:%.*]] = mul nuw nsw <64 x i32> [[TMP23]], [[TMP23]]
+; CHECK-NEXT: [[ADD11_7_7:%.*]] = call i32 @llvm.vector.reduce.add.v64i32(<64 x i32> [[TMP25]])
; CHECK-NEXT: [[CONV15:%.*]] = zext i32 [[ADD_7_7]] to i64
; CHECK-NEXT: [[CONV16:%.*]] = zext i32 [[ADD11_7_7]] to i64
; CHECK-NEXT: [[SHL:%.*]] = shl nuw i64 [[CONV16]], 32
@@ -443,3 +443,150 @@ entry:
%add17 = or i64 %shl, %conv15
ret i64 %add17
}
+
+define i64 @looped(ptr nocapture noundef readonly %p, i32 noundef %st) {
+; CHECK-LABEL: @looped(
+; CHECK-NEXT: entry:
+; CHECK-NEXT: [[IDX_EXT:%.*]] = sext i32 [[ST:%.*]] to i64
+; CHECK-NEXT: br label [[FOR_COND1_PREHEADER:%.*]]
+; CHECK: for.cond1.preheader:
+; CHECK-NEXT: [[Y_038:%.*]] = phi i32 [ 0, [[ENTRY:%.*]] ], [ [[INC13:%.*]], [[FOR_COND1_PREHEADER]] ]
+; CHECK-NEXT: [[SQ_037:%.*]] = phi i32 [ 0, [[ENTRY]] ], [ [[OP_RDX:%.*]], [[FOR_COND1_PREHEADER]] ]
+; CHECK-NEXT: [[SM_036:%.*]] = phi i32 [ 0, [[ENTRY]] ], [ [[OP_RDX1:%.*]], [[FOR_COND1_PREHEADER]] ]
+; CHECK-NEXT: [[P_ADDR_035:%.*]] = phi ptr [ [[P:%.*]], [[ENTRY]] ], [ [[ADD_PTR:%.*]], [[FOR_COND1_PREHEADER]] ]
+; CHECK-NEXT: [[TMP0:%.*]] = load <16 x i16>, ptr [[P_ADDR_035]], align 2
+; CHECK-NEXT: [[TMP1:%.*]] = zext <16 x i16> [[TMP0]] to <16 x i32>
+; CHECK-NEXT: [[TMP2:%.*]] = zext <16 x i16> [[TMP0]] to <16 x i32>
+; CHECK-NEXT: [[TMP3:%.*]] = mul nuw nsw <16 x i32> [[TMP1]], [[TMP1]]
+; CHECK-NEXT: [[TMP4:%.*]] = call i32 @llvm.vector.reduce.add.v16i32(<16 x i32> [[TMP2]])
+; CHECK-NEXT: [[OP_RDX1]] = add i32 [[TMP4]], [[SM_036]]
+; CHECK-NEXT: [[TMP5:%.*]] = call i32 @llvm.vector.reduce.add.v16i32(<16 x i32> [[TMP3]])
+; CHECK-NEXT: [[OP_RDX]] = add i32 [[TMP5]], [[SQ_037]]
+; CHECK-NEXT: [[ADD_PTR]] = getelementptr inbounds i16, ptr [[P_ADDR_035]], i64 [[IDX_EXT]]
+; CHECK-NEXT: [[INC13]] = add nuw nsw i32 [[Y_038]], 1
+; CHECK-NEXT: [[EXITCOND_NOT:%.*]] = icmp eq i32 [[INC13]], 16
+; CHECK-NEXT: br i1 [[EXITCOND_NOT]], label [[FOR_COND_CLEANUP:%.*]], label [[FOR_COND1_PREHEADER]]
+; CHECK: for.cond.cleanup:
+; CHECK-NEXT: [[CONV15:%.*]] = zext i32 [[OP_RDX1]] to i64
+; CHECK-NEXT: [[CONV16:%.*]] = zext i32 [[OP_RDX]] to i64
+; CHECK-NEXT: [[SHL:%.*]] = shl nuw i64 [[CONV16]], 32
+; CHECK-NEXT: [[ADD17:%.*]] = or i64 [[SHL]], [[CONV15]]
+; CHECK-NEXT: ret i64 [[ADD17]]
+;
+entry:
+ %idx.ext = sext i32 %st to i64
+ br label %for.cond1.preheader
+
+for.cond1.preheader: ; preds = %entry, %for.cond1.preheader
+ %y.038 = phi i32 [ 0, %entry ], [ %inc13, %for.cond1.preheader ]
+ %sq.037 = phi i32 [ 0, %entry ], [ %add11.15, %for.cond1.preheader ]
+ %sm.036 = phi i32 [ 0, %entry ], [ %add.15, %for.cond1.preheader ]
+ %p.addr.035 = phi ptr [ %p, %entry ], [ %add.ptr, %for.cond1.preheader ]
+ %0 = load i16, ptr %p.addr.035, align 2
+ %conv = zext i16 %0 to i32
+ %add = add i32 %sm.036, %conv
+ %mul = mul nuw nsw i32 %conv, %conv
+ %add11 = add i32 %mul, %sq.037
+ %arrayidx.1 = getelementptr inbounds i16, ptr %p.addr.035, i64 1
+ %1 = load i16, ptr %arrayidx.1, align 2
+ %conv.1 = zext i16 %1 to i32
+ %add.1 = add i32 %add, %conv.1
+ %mul.1 = mul nuw nsw i32 %conv.1, %conv.1
+ %add11.1 = add i32 %mul.1, %add11
+ %arrayidx.2 = getelementptr inbounds i16, ptr %p.addr.035, i64 2
+ %2 = load i16, ptr %arrayidx.2, align 2
+ %conv.2 = zext i16 %2 to i32
+ %add.2 = add i32 %add.1, %conv.2
+ %mul.2 = mul nuw nsw i32 %conv.2, %conv.2
+ %add11.2 = add i32 %mul.2, %add11.1
+ %arrayidx.3 = getelementptr inbounds i16, ptr %p.addr.035, i64 3
+ %3 = load i16, ptr %arrayidx.3, align 2
+ %conv.3 = zext i16 %3 to i32
+ %add.3 = add i32 %add.2, %conv.3
+ %mul.3 = mul nuw nsw i32 %conv.3, %conv.3
+ %add11.3 = add i32 %mul.3, %add11.2
+ %arrayidx.4 = getelementptr inbounds i16, ptr %p.addr.035, i64 4
+ %4 = load i16, ptr %arrayidx.4, align 2
+ %conv.4 = zext i16 %4 to i32
+ %add.4 = add i32 %add.3, %conv.4
+ %mul.4 = mul nuw nsw i32 %conv.4, %conv.4
+ %add11.4 = add i32 %mul.4, %add11.3
+ %arrayidx.5 = getelementptr inbounds i16, ptr %p.addr.035, i64 5
+ %5 = load i16, ptr %arrayidx.5, align 2
+ %conv.5 = zext i16 %5 to i32
+ %add.5 = add i32 %add.4, %conv.5
+ %mul.5 = mul nuw nsw i32 %conv.5, %conv.5
+ %add11.5 = add i32 %mul.5, %add11.4
+ %arrayidx.6 = getelementptr inbounds i16, ptr %p.addr.035, i64 6
+ %6 = load i16, ptr %arrayidx.6, align 2
+ %conv.6 = zext i16 %6 to i32
+ %add.6 = add i32 %add.5, %conv.6
+ %mul.6 = mul nuw nsw i32 %conv.6, %conv.6
+ %add11.6 = add i32 %mul.6, %add11.5
+ %arrayidx.7 = getelementptr inbounds i16, ptr %p.addr.035, i64 7
+ %7 = load i16, ptr %arrayidx.7, align 2
+ %conv.7 = zext i16 %7 to i32
+ %add.7 = add i32 %add.6, %conv.7
+ %mul.7 = mul nuw nsw i32 %conv.7, %conv.7
+ %add11.7 = add i32 %mul.7, %add11.6
+ %arrayidx.8 = getelementptr inbounds i16, ptr %p.addr.035, i64 8
+ %8 = load i16, ptr %arrayidx.8, align 2
+ %conv.8 = zext i16 %8 to i32
+ %add.8 = add i32 %add.7, %conv.8
+ %mul.8 = mul nuw nsw i32 %conv.8, %conv.8
+ %add11.8 = add i32 %mul.8, %add11.7
+ %arrayidx.9 = getelementptr inbounds i16, ptr %p.addr.035, i64 9
+ %9 = load i16, ptr %arrayidx.9, align 2
+ %conv.9 = zext i16 %9 to i32
+ %add.9 = add i32 %add.8, %conv.9
+ %mul.9 = mul nuw nsw i32 %conv.9, %conv.9
+ %add11.9 = add i32 %mul.9, %add11.8
+ %arrayidx.10 = getelementptr inbounds i16, ptr %p.addr.035, i64 10
+ %10 = load i16, ptr %arrayidx.10, align 2
+ %conv.10 = zext i16 %10 to i32
+ %add.10 = add i32 %add.9, %conv.10
+ %mul.10 = mul nuw nsw i32 %conv.10, %conv.10
+ %add11.10 = add i32 %mul.10, %add11.9
+ %arrayidx.11 = getelementptr inbounds i16, ptr %p.addr.035, i64 11
+ %11 = load i16, ptr %arrayidx.11, align 2
+ %conv.11 = zext i16 %11 to i32
+ %add.11 = add i32 %add.10, %conv.11
+ %mul.11 = mul nuw nsw i32 %conv.11, %conv.11
+ %add11.11 = add i32 %mul.11, %add11.10
+ %arrayidx.12 = getelementptr inbounds i16, ptr %p.addr.035, i64 12
+ %12 = load i16, ptr %arrayidx.12, align 2
+ %conv.12 = zext i16 %12 to i32
+ %add.12 = add i32 %add.11, %conv.12
+ %mul.12 = mul nuw nsw i32 %conv.12, %conv.12
+ %add11.12 = add i32 %mul.12, %add11.11
+ %arrayidx.13 = getelementptr inbounds i16, ptr %p.addr.035, i64 13
+ %13 = load i16, ptr %arrayidx.13, align 2
+ %conv.13 = zext i16 %13 to i32
+ %add.13 = add i32 %add.12, %conv.13
+ %mul.13 = mul nuw nsw i32 %conv.13, %conv.13
+ %add11.13 = add i32 %mul.13, %add11.12
+ %arrayidx.14 = getelementptr inbounds i16, ptr %p.addr.035, i64 14
+ %14 = load i16, ptr %arrayidx.14, align 2
+ %conv.14 = zext i16 %14 to i32
+ %add.14 = add i32 %add.13, %conv.14
+ %mul.14 = mul nuw nsw i32 %conv.14, %conv.14
+ %add11.14 = add i32 %mul.14, %add11.13
+ %arrayidx.15 = getelementptr inbounds i16, ptr %p.addr.035, i64 15
+ %15 = load i16, ptr %arrayidx.15, align 2
+ %conv.15 = zext i16 %15 to i32
+ %add.15 = add i32 %add.14, %conv.15
+ %mul.15 = mul nuw nsw i32 %conv.15, %conv.15
+ %add11.15 = add i32 %mul.15, %add11.14
+ %add.ptr = getelementptr inbounds i16, ptr %p.addr.035, i64 %idx.ext
+ %inc13 = add nuw nsw i32 %y.038, 1
+ %exitcond.not = icmp eq i32 %inc13, 16
+ br i1 %exitcond.not, label %for.cond.cleanup, label %for.cond1.preheader
+
+for.cond.cleanup: ; preds = %for.cond1.preheader
+ %conv15 = zext i32 %add.15 to i64
+ %conv16 = zext i32 %add11.15 to i64
+ %shl = shl nuw i64 %conv16, 32
+ %add17 = or i64 %shl, %conv15
+ ret i64 %add17
+}
+
>From 38a8a965f54c7c0fcce020538d8756a472ce10f1 Mon Sep 17 00:00:00 2001
From: Ryan Buchner <buchner.ryan at gmail.com>
Date: Thu, 23 Jul 2026 15:53:15 -0700
Subject: [PATCH 03/22] Fixup noop change to test
---
.../X86/extractelement-multi-register-use.ll | 10 +++++-----
1 file changed, 5 insertions(+), 5 deletions(-)
diff --git a/llvm/test/Transforms/SLPVectorizer/X86/extractelement-multi-register-use.ll b/llvm/test/Transforms/SLPVectorizer/X86/extractelement-multi-register-use.ll
index c45f6b54f39c5..c945799b38122 100644
--- a/llvm/test/Transforms/SLPVectorizer/X86/extractelement-multi-register-use.ll
+++ b/llvm/test/Transforms/SLPVectorizer/X86/extractelement-multi-register-use.ll
@@ -10,10 +10,10 @@ define void @test(double %i) {
; CHECK-NEXT: [[TMP3:%.*]] = fsub <2 x double> zeroinitializer, [[TMP2]]
; CHECK-NEXT: [[I96:%.*]] = fsub double poison, 0.000000e+00
; CHECK-NEXT: [[I75:%.*]] = fsub double 0.000000e+00, [[I]]
-; CHECK-NEXT: [[TMP8:%.*]] = insertelement <2 x double> <double poison, double 0.000000e+00>, double [[I]], i32 0
-; CHECK-NEXT: [[TMP9:%.*]] = shufflevector <2 x double> [[TMP8]], <2 x double> <double 0.000000e+00, double poison>, <2 x i32> <i32 2, i32 0>
-; CHECK-NEXT: [[TMP4:%.*]] = fsub <2 x double> [[TMP8]], [[TMP9]]
-; CHECK-NEXT: [[TMP5:%.*]] = shufflevector <2 x double> [[TMP4]], <2 x double> poison, <8 x i32> <i32 0, i32 1, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
+; CHECK-NEXT: [[TMP4:%.*]] = insertelement <2 x double> <double poison, double 0.000000e+00>, double [[I]], i32 0
+; CHECK-NEXT: [[TMP8:%.*]] = shufflevector <2 x double> [[TMP4]], <2 x double> <double 0.000000e+00, double poison>, <2 x i32> <i32 2, i32 0>
+; CHECK-NEXT: [[TMP15:%.*]] = fsub <2 x double> [[TMP4]], [[TMP8]]
+; CHECK-NEXT: [[TMP5:%.*]] = shufflevector <2 x double> [[TMP15]], <2 x double> poison, <8 x i32> <i32 0, i32 1, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
; CHECK-NEXT: [[TMP6:%.*]] = shufflevector <8 x double> <double 0.000000e+00, double 0.000000e+00, double poison, double poison, double 0.000000e+00, double poison, double poison, double poison>, <8 x double> [[TMP5]], <8 x i32> <i32 0, i32 1, i32 8, i32 9, i32 4, i32 poison, i32 poison, i32 poison>
; CHECK-NEXT: [[TMP7:%.*]] = insertelement <8 x double> [[TMP6]], double [[I75]], i32 5
; CHECK-NEXT: [[TMP10:%.*]] = shufflevector <8 x double> [[TMP7]], <8 x double> poison, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 2, i32 3>
@@ -23,7 +23,7 @@ define void @test(double %i) {
; CHECK-NEXT: [[TMP14:%.*]] = fcmp ult <8 x double> [[TMP13]], zeroinitializer
; CHECK-NEXT: br label [[BB116:%.*]]
; CHECK: bb116:
-; CHECK-NEXT: [[TMP16:%.*]] = extractelement <2 x double> [[TMP4]], i32 0
+; CHECK-NEXT: [[TMP16:%.*]] = extractelement <2 x double> [[TMP15]], i32 0
; CHECK-NEXT: [[I117:%.*]] = fmul double 0.000000e+00, [[TMP16]]
; CHECK-NEXT: [[I119:%.*]] = fmul double 0.000000e+00, [[I96]]
; CHECK-NEXT: [[I120:%.*]] = fadd double [[I117]], [[I119]]
>From 911a6627bb74b90a05d61c9b29bcb02885ed2bf3 Mon Sep 17 00:00:00 2001
From: Ryan Buchner <buchner.ryan at gmail.com>
Date: Sat, 25 Jul 2026 21:44:00 -0700
Subject: [PATCH 04/22] [SLP] Adjust cost as separate RematAdjustment cost
---
.../Transforms/Vectorize/SLPVectorizer.cpp | 46 ++++++++++++-------
1 file changed, 29 insertions(+), 17 deletions(-)
diff --git a/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp b/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp
index 99eb20ae9e3e8..48c1b8c45e5ae 100644
--- a/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp
+++ b/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp
@@ -17229,18 +17229,6 @@ BoUpSLP::getEntryCost(const TreeEntry *E, ArrayRef<Value *> VectorizedVals,
for (unsigned I = 0; I < Sz; ++I) {
if (UsedScalars.test(I))
continue;
- if (auto *Inst = dyn_cast<Instruction>(UniqueValues[I])) {
- if (auto It = ExternalUsesAsExtractCost.find(Inst);
- It != ExternalUsesAsExtractCost.end()) {
- ScalarCost += It->second;
- continue;
- }
- if (auto It = ExternalUsesAsRematCost.find(Inst);
- It != ExternalUsesAsRematCost.end()) {
- ScalarCost += It->second;
- continue;
- }
- }
ScalarCost += ScalarEltCost(I);
}
}
@@ -17283,7 +17271,29 @@ BoUpSLP::getEntryCost(const TreeEntry *E, ArrayRef<Value *> VectorizedVals,
VecCost += SpillsReloads;
LLVM_DEBUG(dumpTreeCosts(E, CommonCost, VecCost - CommonCost,
ScalarCost, "Calculated costs for Tree"));
- return VecCost - ScalarCost;
+
+ InstructionCost RematAdjustment = 0;
+ for (auto *UV : UniqueValues) {
+ if (auto *Inst = dyn_cast<Instruction>(UV)) {
+ if (auto It = ExternalUsesAsExtractCost.find(Inst);
+ It != ExternalUsesAsExtractCost.end()) {
+ RematAdjustment += It->second;
+ continue;
+ }
+ if (auto It = ExternalUsesAsRematCost.find(Inst);
+ It != ExternalUsesAsRematCost.end()) {
+ RematAdjustment += It->second;
+ continue;
+ }
+ }
+ }
+ if (RematAdjustment != 0)
+ LLVM_DEBUG({
+ dbgs() << "SLP: Adjusting cost by " << RematAdjustment
+ << " to account for rematerialization\n.";
+ });
+
+ return VecCost - ScalarCost + RematAdjustment;
};
// Calculate cost difference from vectorizing set of GEPs.
// Negative value means vectorizing is profitable.
@@ -20325,10 +20335,11 @@ InstructionCost BoUpSLP::getTreeCost(InstructionCost TreeCost,
if (KeepScalar) {
ExternalUsesAsOriginalScalar.insert(EU.Scalar);
if (isDeferredExtractable(EU.Scalar)) {
+ InstructionCost RematDelta = ExtraCost - ScalarCost;
auto [ItCost, Inserted] =
- ExternalUsesAsRematCostTmp.try_emplace(EU.Scalar, ScalarCost);
+ ExternalUsesAsRematCostTmp.try_emplace(EU.Scalar, RematDelta);
if (!Inserted)
- ItCost->second = std::min(ItCost->second, ScalarCost);
+ ItCost->second = std::min(ItCost->second, RematDelta);
}
for (Value *V : Inst->operands()) {
@@ -20364,10 +20375,11 @@ InstructionCost BoUpSLP::getTreeCost(InstructionCost TreeCost,
}
} else if (isDeferredExtractable(EU.Scalar)) {
ExternalUsesAsExtract.insert(EU.Scalar);
+ InstructionCost ExtractDelta = ScalarCost - ExtraCost;
auto [ItCost, Inserted] =
- ExternalUsesAsExtractCostTmp.try_emplace(Inst, ExtraCost);
+ ExternalUsesAsExtractCostTmp.try_emplace(Inst, ExtractDelta);
if (!Inserted)
- ItCost->second = std::min(ItCost->second, ExtraCost);
+ ItCost->second = std::min(ItCost->second, ExtractDelta);
}
}
}
>From 45dde65fe12bc01b58e90b4a47357103b8f1998e Mon Sep 17 00:00:00 2001
From: Ryan Buchner <buchner.ryan at gmail.com>
Date: Sun, 26 Jul 2026 21:04:44 -0700
Subject: [PATCH 05/22] [SLP] Check on UsedScalar when calculating
RematAdjustment
---
llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp | 6 ++++--
1 file changed, 4 insertions(+), 2 deletions(-)
diff --git a/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp b/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp
index 48c1b8c45e5ae..52bf17f2ca0ce 100644
--- a/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp
+++ b/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp
@@ -17273,8 +17273,10 @@ BoUpSLP::getEntryCost(const TreeEntry *E, ArrayRef<Value *> VectorizedVals,
ScalarCost, "Calculated costs for Tree"));
InstructionCost RematAdjustment = 0;
- for (auto *UV : UniqueValues) {
- if (auto *Inst = dyn_cast<Instruction>(UV)) {
+ for (unsigned I : seq<unsigned>(0, Sz)) {
+ if (UsedScalars.test(I))
+ continue;
+ if (auto *Inst = dyn_cast<Instruction>(UniqueValues[I])) {
if (auto It = ExternalUsesAsExtractCost.find(Inst);
It != ExternalUsesAsExtractCost.end()) {
RematAdjustment += It->second;
>From bb89c953c5391173791535c2ab081de3482e7c55 Mon Sep 17 00:00:00 2001
From: Ryan Buchner <buchner.ryan at gmail.com>
Date: Tue, 28 Jul 2026 16:38:33 -0700
Subject: [PATCH 06/22] [SLP] Bug fixes
---
.../Transforms/Vectorize/SLPVectorizer.cpp | 32 ++++++++++++++-----
1 file changed, 24 insertions(+), 8 deletions(-)
diff --git a/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp b/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp
index 52bf17f2ca0ce..bc89a06d683cc 100644
--- a/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp
+++ b/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp
@@ -13045,6 +13045,10 @@ void BoUpSLP::buildTreeRec(ArrayRef<Value *> VLRef, unsigned Depth,
VPtr = Extract;
}
};
+ // Normally we replace at the time of operand creation but if first
+ // Entry in tree these VL were never operands
+ if (Depth == 0)
+ ReplaceWithExtractCandidates(VL);
// Tries to build split node.
auto TrySplitNode = [&](const InstructionsState &LocalState) {
@@ -25036,7 +25040,8 @@ Value *BoUpSLP::vectorizeTree(
return {dyn_cast<ExtractElementInst>(Extract),
dyn_cast<Instruction>(Remat)};
};
- auto TrackDeferredExtract = [&](Instruction *Inst, Value *Replacement) {
+ auto TrackDeferredExtract = [&](Instruction *Inst, Value *Replacement,
+ llvm::User *User) {
if (!Inst)
return;
if (ExternalUsesAsOriginalScalar.contains(Inst)) {
@@ -25070,6 +25075,10 @@ Value *BoUpSLP::vectorizeTree(
}
CouldBeExtract.try_emplace(RI, EI);
CouldBeRemat.try_emplace(EI, RI);
+ if (User)
+ User->replaceUsesOfWith(EI, RI);
+ else
+ EI->replaceAllUsesWith(RI);
};
// If User == nullptr, the Scalar remains as scalar in vectorized
// instructions or is used as extra arg. Generate ExtractElement instruction
@@ -25131,8 +25140,11 @@ Value *BoUpSLP::vectorizeTree(
"Extractelements should not be replaced.");
Scalar->replaceAllUsesWith(NewInst);
}
+ if (ExternalUsesAsExtract.contains(Scalar))
+ DeferredScalarsToExtract[Scalar].emplace_back(Scalar, NewInst, User);
+
if (IsDeferredScalar)
- TrackDeferredExtract(dyn_cast<Instruction>(Scalar), NewInst);
+ TrackDeferredExtract(dyn_cast<Instruction>(Scalar), NewInst, User);
continue;
}
@@ -25235,7 +25247,7 @@ Value *BoUpSLP::vectorizeTree(
User->replaceUsesOfWith(Scalar, NewInst);
}
if (IsDeferredScalar)
- TrackDeferredExtract(dyn_cast<Instruction>(Scalar), NewInst);
+ TrackDeferredExtract(dyn_cast<Instruction>(Scalar), NewInst, User);
}
} else {
Builder.SetInsertPoint(&F->getEntryBlock(), F->getEntryBlock().begin());
@@ -25558,11 +25570,15 @@ void BoUpSLP::emitDeferredExtracts() {
for (const auto &Entry : DeferredScalarsToExtract)
append_range(DeferredExtracts, Entry.second);
for (const auto &DET : DeferredExtracts) {
- auto *UI = cast<Instruction>(DET.User);
- if (isDeleted(UI))
- continue;
- DET.User->replaceUsesOfWith(DET.Scalar, DET.NewInst);
- LLVM_DEBUG(dbgs() << "SLP: Delayed replacement:" << *UI << ".\n");
+ auto *UI = cast_or_null<Instruction>(DET.User);
+ if (!DET.User) {
+ DET.Scalar->replaceAllUsesWith(DET.NewInst);
+ } else {
+ if (isDeleted(UI))
+ continue;
+ DET.User->replaceUsesOfWith(DET.Scalar, DET.NewInst);
+ LLVM_DEBUG(dbgs() << "SLP: Delayed replacement:" << *UI << ".\n");
+ }
}
DeferredScalarsToExtract.clear();
for (const auto &DET : DeferredExtracts) {
>From afed8f174eca08b01a00c59eadaaf51eb6643b4e Mon Sep 17 00:00:00 2001
From: Ryan Buchner <buchner.ryan at gmail.com>
Date: Mon, 27 Jul 2026 09:12:47 -0700
Subject: [PATCH 07/22] [SLP] Don't modify costing behavior even if deferrable
and only handle basic load cases
---
.../Transforms/Vectorize/SLPVectorizer.cpp | 48 +-
.../SLPVectorizer/AArch64/getelementptr2.ll | 4 +-
.../AArch64/multiple_reduction.ll | 506 ++++++++++++++++--
.../SLPVectorizer/RISCV/external.ll | 3 +-
.../RISCV/runtime-strided-stores.ll | 252 ++++-----
.../strided-loads-with-external-use-ptr.ll | 8 +-
.../Transforms/SLPVectorizer/X86/PR35628_1.ll | 6 +-
.../test/Transforms/SLPVectorizer/X86/call.ll | 17 +-
.../X86/extractelemets-extended-by-poison.ll | 4 +-
.../SLPVectorizer/X86/horizontal-list.ll | 17 +-
.../SLPVectorizer/X86/horizontal-minmax.ll | 25 +-
11 files changed, 622 insertions(+), 268 deletions(-)
diff --git a/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp b/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp
index bc89a06d683cc..c82ebccda3ef1 100644
--- a/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp
+++ b/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp
@@ -1038,13 +1038,7 @@ static bool isValidForAlternation(unsigned Opcode) {
static bool isDeferredExtractable(Value *Scalar) {
if (isa<VectorType>(Scalar->getType()))
return false;
- if (isa<LoadInst>(Scalar))
- return true;
- if (auto *CI = dyn_cast<CastInst>(Scalar)) {
- auto *LI = dyn_cast<LoadInst>(CI->getOperand(0));
- return LI && LI->hasOneUse();
- }
- return false;
+ return isa<LoadInst>(Scalar);
}
namespace {
@@ -20237,10 +20231,7 @@ InstructionCost BoUpSLP::getTreeCost(InstructionCost TreeCost,
// Track cases where extraction is more profitable and convert those to
// extracts in a late cleanup step.
if (Entry->Idx != 0 || Entry->getOpcode() == Instruction::GetElementPtr ||
- Entry->getOpcode() == Instruction::Load ||
- (Entry->getOpcode() == Instruction::ZExt &&
- getOperandEntry(Entry, 0)->getOperations().valid() &&
- getOperandEntry(Entry, 0)->getOpcode() == Instruction::Load)) {
+ Entry->getOpcode() == Instruction::Load) {
// Checks if the user of the external scalar is phi in loop body.
auto IsPhiInLoop = [&](const ExternalUser &U) {
if (auto *Phi = dyn_cast_if_present<PHINode>(U.User)) {
@@ -20318,8 +20309,7 @@ InstructionCost BoUpSLP::getTreeCost(InstructionCost TreeCost,
}) <= 2;
if (IsProfitablePHIUser) {
KeepScalar = true;
- } else if (KeepScalar && !isDeferredExtractable(EU.Scalar) &&
- ScalarCost != TTI::TCC_Free &&
+ } else if (KeepScalar && ScalarCost != TTI::TCC_Free &&
ExtraCost - ScalarCost <= TTI::TCC_Basic &&
(!GatheredLoadsEntriesFirst.has_value() ||
Entry->Idx < *GatheredLoadsEntriesFirst)) {
@@ -25027,32 +25017,18 @@ Value *BoUpSLP::vectorizeTree(
VectorToInsertElement.try_emplace(Vec, IE);
return Vec;
};
- auto GetUnderlyingInsts =
- [](Value *Extract,
- Value *Remat) -> std::pair<ExtractElementInst *, Instruction *> {
- auto *ExtractCast = dyn_cast<CastInst>(Extract);
- if (ExtractCast) {
- auto *RematCast = dyn_cast<CastInst>(Remat);
- assert(RematCast && "Expected Remat to be extended if extract is");
- Extract = ExtractCast->getOperand(0);
- Remat = RematCast->getOperand(0);
- }
- return {dyn_cast<ExtractElementInst>(Extract),
- dyn_cast<Instruction>(Remat)};
- };
auto TrackDeferredExtract = [&](Instruction *Inst, Value *Replacement,
llvm::User *User) {
if (!Inst)
return;
if (ExternalUsesAsOriginalScalar.contains(Inst)) {
- if (CouldBeExtract.contains(Replacement))
- return;
ExtractAnyways = true;
Value *ReplacedExtract = ExtractAndExtendIfNeeded(Vec);
- auto P = GetUnderlyingInsts(ReplacedExtract, Replacement);
- auto *EI = P.first;
- auto *RI = P.second;
+ auto *EI = dyn_cast<ExtractElementInst>(ReplacedExtract);
+ auto *RI = dyn_cast<Instruction>(Replacement);
assert(EI && RI && "Expected to find underlying instructions");
+ if (CouldBeExtract.contains(RI))
+ return;
if (ExternalUsesAsRematCostTmp.contains(Inst)) {
InstructionCost RematCost = ExternalUsesAsRematCostTmp.lookup(Inst);
ExternalUsesAsRematCost.try_emplace(EI, RematCost);
@@ -25062,13 +25038,11 @@ Value *BoUpSLP::vectorizeTree(
ExtractAnyways = false;
return;
}
- if (!ExternalUsesAsExtract.contains(Inst) ||
- CouldBeExtract.contains(Inst))
- return;
- auto P = GetUnderlyingInsts(Replacement, Inst);
- auto *EI = P.first;
- auto *RI = P.second;
+ auto *EI = dyn_cast<ExtractElementInst>(Replacement);
+ auto *RI = Inst;
assert(EI && RI && "Expected to find underlying instructions");
+ if (!ExternalUsesAsExtract.contains(RI) || CouldBeExtract.contains(RI))
+ return;
if (ExternalUsesAsExtractCostTmp.contains(Inst)) {
InstructionCost ExtractCost = ExternalUsesAsExtractCostTmp.lookup(Inst);
ExternalUsesAsExtractCost.try_emplace(RI, ExtractCost);
diff --git a/llvm/test/Transforms/SLPVectorizer/AArch64/getelementptr2.ll b/llvm/test/Transforms/SLPVectorizer/AArch64/getelementptr2.ll
index 5cc4fe846109b..4847149c87a18 100644
--- a/llvm/test/Transforms/SLPVectorizer/AArch64/getelementptr2.ll
+++ b/llvm/test/Transforms/SLPVectorizer/AArch64/getelementptr2.ll
@@ -1,8 +1,8 @@
; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
;test_i16_extend NOTE: Assertions have been autogenerated by utils/update_test_checks.py
-; RUN: opt -S -mtriple=aarch64--linux-gnu -passes=slp-vectorizer -slp-threshold=-4 -pass-remarks-output=%t < %s | FileCheck %s
+; RUN: opt -S -mtriple=aarch64--linux-gnu -passes=slp-vectorizer -slp-threshold=-5 -pass-remarks-output=%t < %s | FileCheck %s
; RUN: cat %t | FileCheck -check-prefix=YAML %s
-; RUN: opt -S -mtriple=aarch64--linux-gnu -passes=slp-vectorizer -slp-threshold=-4 -pass-remarks-output=%t < %s | FileCheck %s
+; RUN: opt -S -mtriple=aarch64--linux-gnu -passes=slp-vectorizer -slp-threshold=-5 -pass-remarks-output=%t < %s | FileCheck %s
; RUN: cat %t | FileCheck -check-prefix=YAML %s
diff --git a/llvm/test/Transforms/SLPVectorizer/AArch64/multiple_reduction.ll b/llvm/test/Transforms/SLPVectorizer/AArch64/multiple_reduction.ll
index 89b0bf3f2179d..fe3db7d462e8e 100644
--- a/llvm/test/Transforms/SLPVectorizer/AArch64/multiple_reduction.ll
+++ b/llvm/test/Transforms/SLPVectorizer/AArch64/multiple_reduction.ll
@@ -14,40 +14,387 @@ define i64 @straight(ptr nocapture noundef readonly %p, i32 noundef %st) {
; CHECK-LABEL: @straight(
; CHECK-NEXT: entry:
; CHECK-NEXT: [[IDX_EXT:%.*]] = sext i32 [[ST:%.*]] to i64
-; CHECK-NEXT: [[ADD_PTR:%.*]] = getelementptr inbounds i16, ptr [[P:%.*]], i64 [[IDX_EXT]]
+; CHECK-NEXT: [[TMP0:%.*]] = load i16, ptr [[P:%.*]], align 2
+; CHECK-NEXT: [[CONV:%.*]] = zext i16 [[TMP0]] to i32
+; CHECK-NEXT: [[MUL:%.*]] = mul nuw nsw i32 [[CONV]], [[CONV]]
+; CHECK-NEXT: [[ARRAYIDX_1:%.*]] = getelementptr inbounds i16, ptr [[P]], i64 1
+; CHECK-NEXT: [[TMP1:%.*]] = load i16, ptr [[ARRAYIDX_1]], align 2
+; CHECK-NEXT: [[CONV_1:%.*]] = zext i16 [[TMP1]] to i32
+; CHECK-NEXT: [[ADD_1:%.*]] = add nuw nsw i32 [[CONV]], [[CONV_1]]
+; CHECK-NEXT: [[MUL_1:%.*]] = mul nuw nsw i32 [[CONV_1]], [[CONV_1]]
+; CHECK-NEXT: [[ADD11_1:%.*]] = add nuw i32 [[MUL_1]], [[MUL]]
+; CHECK-NEXT: [[ARRAYIDX_2:%.*]] = getelementptr inbounds i16, ptr [[P]], i64 2
+; CHECK-NEXT: [[TMP2:%.*]] = load i16, ptr [[ARRAYIDX_2]], align 2
+; CHECK-NEXT: [[CONV_2:%.*]] = zext i16 [[TMP2]] to i32
+; CHECK-NEXT: [[ADD_2:%.*]] = add nuw nsw i32 [[ADD_1]], [[CONV_2]]
+; CHECK-NEXT: [[MUL_2:%.*]] = mul nuw nsw i32 [[CONV_2]], [[CONV_2]]
+; CHECK-NEXT: [[ADD11_2:%.*]] = add i32 [[MUL_2]], [[ADD11_1]]
+; CHECK-NEXT: [[ARRAYIDX_3:%.*]] = getelementptr inbounds i16, ptr [[P]], i64 3
+; CHECK-NEXT: [[TMP3:%.*]] = load i16, ptr [[ARRAYIDX_3]], align 2
+; CHECK-NEXT: [[CONV_3:%.*]] = zext i16 [[TMP3]] to i32
+; CHECK-NEXT: [[ADD_3:%.*]] = add nuw nsw i32 [[ADD_2]], [[CONV_3]]
+; CHECK-NEXT: [[MUL_3:%.*]] = mul nuw nsw i32 [[CONV_3]], [[CONV_3]]
+; CHECK-NEXT: [[ADD11_3:%.*]] = add i32 [[MUL_3]], [[ADD11_2]]
+; CHECK-NEXT: [[ARRAYIDX_4:%.*]] = getelementptr inbounds i16, ptr [[P]], i64 4
+; CHECK-NEXT: [[TMP4:%.*]] = load i16, ptr [[ARRAYIDX_4]], align 2
+; CHECK-NEXT: [[CONV_4:%.*]] = zext i16 [[TMP4]] to i32
+; CHECK-NEXT: [[ADD_4:%.*]] = add nuw nsw i32 [[ADD_3]], [[CONV_4]]
+; CHECK-NEXT: [[MUL_4:%.*]] = mul nuw nsw i32 [[CONV_4]], [[CONV_4]]
+; CHECK-NEXT: [[ADD11_4:%.*]] = add i32 [[MUL_4]], [[ADD11_3]]
+; CHECK-NEXT: [[ARRAYIDX_5:%.*]] = getelementptr inbounds i16, ptr [[P]], i64 5
+; CHECK-NEXT: [[TMP5:%.*]] = load i16, ptr [[ARRAYIDX_5]], align 2
+; CHECK-NEXT: [[CONV_5:%.*]] = zext i16 [[TMP5]] to i32
+; CHECK-NEXT: [[ADD_5:%.*]] = add nuw nsw i32 [[ADD_4]], [[CONV_5]]
+; CHECK-NEXT: [[MUL_5:%.*]] = mul nuw nsw i32 [[CONV_5]], [[CONV_5]]
+; CHECK-NEXT: [[ADD11_5:%.*]] = add i32 [[MUL_5]], [[ADD11_4]]
+; CHECK-NEXT: [[ARRAYIDX_6:%.*]] = getelementptr inbounds i16, ptr [[P]], i64 6
+; CHECK-NEXT: [[TMP6:%.*]] = load i16, ptr [[ARRAYIDX_6]], align 2
+; CHECK-NEXT: [[CONV_6:%.*]] = zext i16 [[TMP6]] to i32
+; CHECK-NEXT: [[ADD_6:%.*]] = add nuw nsw i32 [[ADD_5]], [[CONV_6]]
+; CHECK-NEXT: [[MUL_6:%.*]] = mul nuw nsw i32 [[CONV_6]], [[CONV_6]]
+; CHECK-NEXT: [[ADD11_6:%.*]] = add i32 [[MUL_6]], [[ADD11_5]]
+; CHECK-NEXT: [[ARRAYIDX_7:%.*]] = getelementptr inbounds i16, ptr [[P]], i64 7
+; CHECK-NEXT: [[TMP7:%.*]] = load i16, ptr [[ARRAYIDX_7]], align 2
+; CHECK-NEXT: [[CONV_7:%.*]] = zext i16 [[TMP7]] to i32
+; CHECK-NEXT: [[ADD_7:%.*]] = add nuw nsw i32 [[ADD_6]], [[CONV_7]]
+; CHECK-NEXT: [[MUL_7:%.*]] = mul nuw nsw i32 [[CONV_7]], [[CONV_7]]
+; CHECK-NEXT: [[ADD11_7:%.*]] = add i32 [[MUL_7]], [[ADD11_6]]
+; CHECK-NEXT: [[ADD_PTR:%.*]] = getelementptr inbounds i16, ptr [[P]], i64 [[IDX_EXT]]
+; CHECK-NEXT: [[TMP8:%.*]] = load i16, ptr [[ADD_PTR]], align 2
+; CHECK-NEXT: [[CONV_140:%.*]] = zext i16 [[TMP8]] to i32
+; CHECK-NEXT: [[ADD_141:%.*]] = add nuw nsw i32 [[ADD_7]], [[CONV_140]]
+; CHECK-NEXT: [[MUL_142:%.*]] = mul nuw nsw i32 [[CONV_140]], [[CONV_140]]
+; CHECK-NEXT: [[ADD11_143:%.*]] = add i32 [[MUL_142]], [[ADD11_7]]
+; CHECK-NEXT: [[ARRAYIDX_1_1:%.*]] = getelementptr inbounds i16, ptr [[ADD_PTR]], i64 1
+; CHECK-NEXT: [[TMP9:%.*]] = load i16, ptr [[ARRAYIDX_1_1]], align 2
+; CHECK-NEXT: [[CONV_1_1:%.*]] = zext i16 [[TMP9]] to i32
+; CHECK-NEXT: [[ADD_1_1:%.*]] = add nuw nsw i32 [[ADD_141]], [[CONV_1_1]]
+; CHECK-NEXT: [[MUL_1_1:%.*]] = mul nuw nsw i32 [[CONV_1_1]], [[CONV_1_1]]
+; CHECK-NEXT: [[ADD11_1_1:%.*]] = add i32 [[MUL_1_1]], [[ADD11_143]]
+; CHECK-NEXT: [[ARRAYIDX_2_1:%.*]] = getelementptr inbounds i16, ptr [[ADD_PTR]], i64 2
+; CHECK-NEXT: [[TMP10:%.*]] = load i16, ptr [[ARRAYIDX_2_1]], align 2
+; CHECK-NEXT: [[CONV_2_1:%.*]] = zext i16 [[TMP10]] to i32
+; CHECK-NEXT: [[ADD_2_1:%.*]] = add nuw nsw i32 [[ADD_1_1]], [[CONV_2_1]]
+; CHECK-NEXT: [[MUL_2_1:%.*]] = mul nuw nsw i32 [[CONV_2_1]], [[CONV_2_1]]
+; CHECK-NEXT: [[ADD11_2_1:%.*]] = add i32 [[MUL_2_1]], [[ADD11_1_1]]
+; CHECK-NEXT: [[ARRAYIDX_3_1:%.*]] = getelementptr inbounds i16, ptr [[ADD_PTR]], i64 3
+; CHECK-NEXT: [[TMP11:%.*]] = load i16, ptr [[ARRAYIDX_3_1]], align 2
+; CHECK-NEXT: [[CONV_3_1:%.*]] = zext i16 [[TMP11]] to i32
+; CHECK-NEXT: [[ADD_3_1:%.*]] = add nuw nsw i32 [[ADD_2_1]], [[CONV_3_1]]
+; CHECK-NEXT: [[MUL_3_1:%.*]] = mul nuw nsw i32 [[CONV_3_1]], [[CONV_3_1]]
+; CHECK-NEXT: [[ADD11_3_1:%.*]] = add i32 [[MUL_3_1]], [[ADD11_2_1]]
+; CHECK-NEXT: [[ARRAYIDX_4_1:%.*]] = getelementptr inbounds i16, ptr [[ADD_PTR]], i64 4
+; CHECK-NEXT: [[TMP12:%.*]] = load i16, ptr [[ARRAYIDX_4_1]], align 2
+; CHECK-NEXT: [[CONV_4_1:%.*]] = zext i16 [[TMP12]] to i32
+; CHECK-NEXT: [[ADD_4_1:%.*]] = add nuw nsw i32 [[ADD_3_1]], [[CONV_4_1]]
+; CHECK-NEXT: [[MUL_4_1:%.*]] = mul nuw nsw i32 [[CONV_4_1]], [[CONV_4_1]]
+; CHECK-NEXT: [[ADD11_4_1:%.*]] = add i32 [[MUL_4_1]], [[ADD11_3_1]]
+; CHECK-NEXT: [[ARRAYIDX_5_1:%.*]] = getelementptr inbounds i16, ptr [[ADD_PTR]], i64 5
+; CHECK-NEXT: [[TMP13:%.*]] = load i16, ptr [[ARRAYIDX_5_1]], align 2
+; CHECK-NEXT: [[CONV_5_1:%.*]] = zext i16 [[TMP13]] to i32
+; CHECK-NEXT: [[ADD_5_1:%.*]] = add nuw nsw i32 [[ADD_4_1]], [[CONV_5_1]]
+; CHECK-NEXT: [[MUL_5_1:%.*]] = mul nuw nsw i32 [[CONV_5_1]], [[CONV_5_1]]
+; CHECK-NEXT: [[ADD11_5_1:%.*]] = add i32 [[MUL_5_1]], [[ADD11_4_1]]
+; CHECK-NEXT: [[ARRAYIDX_6_1:%.*]] = getelementptr inbounds i16, ptr [[ADD_PTR]], i64 6
+; CHECK-NEXT: [[TMP14:%.*]] = load i16, ptr [[ARRAYIDX_6_1]], align 2
+; CHECK-NEXT: [[CONV_6_1:%.*]] = zext i16 [[TMP14]] to i32
+; CHECK-NEXT: [[ADD_6_1:%.*]] = add nuw nsw i32 [[ADD_5_1]], [[CONV_6_1]]
+; CHECK-NEXT: [[MUL_6_1:%.*]] = mul nuw nsw i32 [[CONV_6_1]], [[CONV_6_1]]
+; CHECK-NEXT: [[ADD11_6_1:%.*]] = add i32 [[MUL_6_1]], [[ADD11_5_1]]
+; CHECK-NEXT: [[ARRAYIDX_7_1:%.*]] = getelementptr inbounds i16, ptr [[ADD_PTR]], i64 7
+; CHECK-NEXT: [[TMP15:%.*]] = load i16, ptr [[ARRAYIDX_7_1]], align 2
+; CHECK-NEXT: [[CONV_7_1:%.*]] = zext i16 [[TMP15]] to i32
+; CHECK-NEXT: [[ADD_7_1:%.*]] = add nuw nsw i32 [[ADD_6_1]], [[CONV_7_1]]
+; CHECK-NEXT: [[MUL_7_1:%.*]] = mul nuw nsw i32 [[CONV_7_1]], [[CONV_7_1]]
+; CHECK-NEXT: [[ADD11_7_1:%.*]] = add i32 [[MUL_7_1]], [[ADD11_6_1]]
; CHECK-NEXT: [[ADD_PTR_1:%.*]] = getelementptr inbounds i16, ptr [[ADD_PTR]], i64 [[IDX_EXT]]
+; CHECK-NEXT: [[TMP16:%.*]] = load i16, ptr [[ADD_PTR_1]], align 2
+; CHECK-NEXT: [[CONV_244:%.*]] = zext i16 [[TMP16]] to i32
+; CHECK-NEXT: [[ADD_245:%.*]] = add nuw nsw i32 [[ADD_7_1]], [[CONV_244]]
+; CHECK-NEXT: [[MUL_246:%.*]] = mul nuw nsw i32 [[CONV_244]], [[CONV_244]]
+; CHECK-NEXT: [[ADD11_247:%.*]] = add i32 [[MUL_246]], [[ADD11_7_1]]
+; CHECK-NEXT: [[ARRAYIDX_1_2:%.*]] = getelementptr inbounds i16, ptr [[ADD_PTR_1]], i64 1
+; CHECK-NEXT: [[TMP17:%.*]] = load i16, ptr [[ARRAYIDX_1_2]], align 2
+; CHECK-NEXT: [[CONV_1_2:%.*]] = zext i16 [[TMP17]] to i32
+; CHECK-NEXT: [[ADD_1_2:%.*]] = add nuw nsw i32 [[ADD_245]], [[CONV_1_2]]
+; CHECK-NEXT: [[MUL_1_2:%.*]] = mul nuw nsw i32 [[CONV_1_2]], [[CONV_1_2]]
+; CHECK-NEXT: [[ADD11_1_2:%.*]] = add i32 [[MUL_1_2]], [[ADD11_247]]
+; CHECK-NEXT: [[ARRAYIDX_2_2:%.*]] = getelementptr inbounds i16, ptr [[ADD_PTR_1]], i64 2
+; CHECK-NEXT: [[TMP18:%.*]] = load i16, ptr [[ARRAYIDX_2_2]], align 2
+; CHECK-NEXT: [[CONV_2_2:%.*]] = zext i16 [[TMP18]] to i32
+; CHECK-NEXT: [[ADD_2_2:%.*]] = add nuw nsw i32 [[ADD_1_2]], [[CONV_2_2]]
+; CHECK-NEXT: [[MUL_2_2:%.*]] = mul nuw nsw i32 [[CONV_2_2]], [[CONV_2_2]]
+; CHECK-NEXT: [[ADD11_2_2:%.*]] = add i32 [[MUL_2_2]], [[ADD11_1_2]]
+; CHECK-NEXT: [[ARRAYIDX_3_2:%.*]] = getelementptr inbounds i16, ptr [[ADD_PTR_1]], i64 3
+; CHECK-NEXT: [[TMP19:%.*]] = load i16, ptr [[ARRAYIDX_3_2]], align 2
+; CHECK-NEXT: [[CONV_3_2:%.*]] = zext i16 [[TMP19]] to i32
+; CHECK-NEXT: [[ADD_3_2:%.*]] = add nuw nsw i32 [[ADD_2_2]], [[CONV_3_2]]
+; CHECK-NEXT: [[MUL_3_2:%.*]] = mul nuw nsw i32 [[CONV_3_2]], [[CONV_3_2]]
+; CHECK-NEXT: [[ADD11_3_2:%.*]] = add i32 [[MUL_3_2]], [[ADD11_2_2]]
+; CHECK-NEXT: [[ARRAYIDX_4_2:%.*]] = getelementptr inbounds i16, ptr [[ADD_PTR_1]], i64 4
+; CHECK-NEXT: [[TMP20:%.*]] = load i16, ptr [[ARRAYIDX_4_2]], align 2
+; CHECK-NEXT: [[CONV_4_2:%.*]] = zext i16 [[TMP20]] to i32
+; CHECK-NEXT: [[ADD_4_2:%.*]] = add nuw nsw i32 [[ADD_3_2]], [[CONV_4_2]]
+; CHECK-NEXT: [[MUL_4_2:%.*]] = mul nuw nsw i32 [[CONV_4_2]], [[CONV_4_2]]
+; CHECK-NEXT: [[ADD11_4_2:%.*]] = add i32 [[MUL_4_2]], [[ADD11_3_2]]
+; CHECK-NEXT: [[ARRAYIDX_5_2:%.*]] = getelementptr inbounds i16, ptr [[ADD_PTR_1]], i64 5
+; CHECK-NEXT: [[TMP21:%.*]] = load i16, ptr [[ARRAYIDX_5_2]], align 2
+; CHECK-NEXT: [[CONV_5_2:%.*]] = zext i16 [[TMP21]] to i32
+; CHECK-NEXT: [[ADD_5_2:%.*]] = add nuw nsw i32 [[ADD_4_2]], [[CONV_5_2]]
+; CHECK-NEXT: [[MUL_5_2:%.*]] = mul nuw nsw i32 [[CONV_5_2]], [[CONV_5_2]]
+; CHECK-NEXT: [[ADD11_5_2:%.*]] = add i32 [[MUL_5_2]], [[ADD11_4_2]]
+; CHECK-NEXT: [[ARRAYIDX_6_2:%.*]] = getelementptr inbounds i16, ptr [[ADD_PTR_1]], i64 6
+; CHECK-NEXT: [[TMP22:%.*]] = load i16, ptr [[ARRAYIDX_6_2]], align 2
+; CHECK-NEXT: [[CONV_6_2:%.*]] = zext i16 [[TMP22]] to i32
+; CHECK-NEXT: [[ADD_6_2:%.*]] = add nuw nsw i32 [[ADD_5_2]], [[CONV_6_2]]
+; CHECK-NEXT: [[MUL_6_2:%.*]] = mul nuw nsw i32 [[CONV_6_2]], [[CONV_6_2]]
+; CHECK-NEXT: [[ADD11_6_2:%.*]] = add i32 [[MUL_6_2]], [[ADD11_5_2]]
+; CHECK-NEXT: [[ARRAYIDX_7_2:%.*]] = getelementptr inbounds i16, ptr [[ADD_PTR_1]], i64 7
+; CHECK-NEXT: [[TMP23:%.*]] = load i16, ptr [[ARRAYIDX_7_2]], align 2
+; CHECK-NEXT: [[CONV_7_2:%.*]] = zext i16 [[TMP23]] to i32
+; CHECK-NEXT: [[ADD_7_2:%.*]] = add nuw nsw i32 [[ADD_6_2]], [[CONV_7_2]]
+; CHECK-NEXT: [[MUL_7_2:%.*]] = mul nuw nsw i32 [[CONV_7_2]], [[CONV_7_2]]
+; CHECK-NEXT: [[ADD11_7_2:%.*]] = add i32 [[MUL_7_2]], [[ADD11_6_2]]
; CHECK-NEXT: [[ADD_PTR_2:%.*]] = getelementptr inbounds i16, ptr [[ADD_PTR_1]], i64 [[IDX_EXT]]
+; CHECK-NEXT: [[TMP24:%.*]] = load i16, ptr [[ADD_PTR_2]], align 2
+; CHECK-NEXT: [[CONV_348:%.*]] = zext i16 [[TMP24]] to i32
+; CHECK-NEXT: [[ADD_349:%.*]] = add nuw nsw i32 [[ADD_7_2]], [[CONV_348]]
+; CHECK-NEXT: [[MUL_350:%.*]] = mul nuw nsw i32 [[CONV_348]], [[CONV_348]]
+; CHECK-NEXT: [[ADD11_351:%.*]] = add i32 [[MUL_350]], [[ADD11_7_2]]
+; CHECK-NEXT: [[ARRAYIDX_1_3:%.*]] = getelementptr inbounds i16, ptr [[ADD_PTR_2]], i64 1
+; CHECK-NEXT: [[TMP25:%.*]] = load i16, ptr [[ARRAYIDX_1_3]], align 2
+; CHECK-NEXT: [[CONV_1_3:%.*]] = zext i16 [[TMP25]] to i32
+; CHECK-NEXT: [[ADD_1_3:%.*]] = add nuw nsw i32 [[ADD_349]], [[CONV_1_3]]
+; CHECK-NEXT: [[MUL_1_3:%.*]] = mul nuw nsw i32 [[CONV_1_3]], [[CONV_1_3]]
+; CHECK-NEXT: [[ADD11_1_3:%.*]] = add i32 [[MUL_1_3]], [[ADD11_351]]
+; CHECK-NEXT: [[ARRAYIDX_2_3:%.*]] = getelementptr inbounds i16, ptr [[ADD_PTR_2]], i64 2
+; CHECK-NEXT: [[TMP26:%.*]] = load i16, ptr [[ARRAYIDX_2_3]], align 2
+; CHECK-NEXT: [[CONV_2_3:%.*]] = zext i16 [[TMP26]] to i32
+; CHECK-NEXT: [[ADD_2_3:%.*]] = add nuw nsw i32 [[ADD_1_3]], [[CONV_2_3]]
+; CHECK-NEXT: [[MUL_2_3:%.*]] = mul nuw nsw i32 [[CONV_2_3]], [[CONV_2_3]]
+; CHECK-NEXT: [[ADD11_2_3:%.*]] = add i32 [[MUL_2_3]], [[ADD11_1_3]]
+; CHECK-NEXT: [[ARRAYIDX_3_3:%.*]] = getelementptr inbounds i16, ptr [[ADD_PTR_2]], i64 3
+; CHECK-NEXT: [[TMP27:%.*]] = load i16, ptr [[ARRAYIDX_3_3]], align 2
+; CHECK-NEXT: [[CONV_3_3:%.*]] = zext i16 [[TMP27]] to i32
+; CHECK-NEXT: [[ADD_3_3:%.*]] = add nuw nsw i32 [[ADD_2_3]], [[CONV_3_3]]
+; CHECK-NEXT: [[MUL_3_3:%.*]] = mul nuw nsw i32 [[CONV_3_3]], [[CONV_3_3]]
+; CHECK-NEXT: [[ADD11_3_3:%.*]] = add i32 [[MUL_3_3]], [[ADD11_2_3]]
+; CHECK-NEXT: [[ARRAYIDX_4_3:%.*]] = getelementptr inbounds i16, ptr [[ADD_PTR_2]], i64 4
+; CHECK-NEXT: [[TMP28:%.*]] = load i16, ptr [[ARRAYIDX_4_3]], align 2
+; CHECK-NEXT: [[CONV_4_3:%.*]] = zext i16 [[TMP28]] to i32
+; CHECK-NEXT: [[ADD_4_3:%.*]] = add nuw nsw i32 [[ADD_3_3]], [[CONV_4_3]]
+; CHECK-NEXT: [[MUL_4_3:%.*]] = mul nuw nsw i32 [[CONV_4_3]], [[CONV_4_3]]
+; CHECK-NEXT: [[ADD11_4_3:%.*]] = add i32 [[MUL_4_3]], [[ADD11_3_3]]
+; CHECK-NEXT: [[ARRAYIDX_5_3:%.*]] = getelementptr inbounds i16, ptr [[ADD_PTR_2]], i64 5
+; CHECK-NEXT: [[TMP29:%.*]] = load i16, ptr [[ARRAYIDX_5_3]], align 2
+; CHECK-NEXT: [[CONV_5_3:%.*]] = zext i16 [[TMP29]] to i32
+; CHECK-NEXT: [[ADD_5_3:%.*]] = add nuw nsw i32 [[ADD_4_3]], [[CONV_5_3]]
+; CHECK-NEXT: [[MUL_5_3:%.*]] = mul nuw nsw i32 [[CONV_5_3]], [[CONV_5_3]]
+; CHECK-NEXT: [[ADD11_5_3:%.*]] = add i32 [[MUL_5_3]], [[ADD11_4_3]]
+; CHECK-NEXT: [[ARRAYIDX_6_3:%.*]] = getelementptr inbounds i16, ptr [[ADD_PTR_2]], i64 6
+; CHECK-NEXT: [[TMP30:%.*]] = load i16, ptr [[ARRAYIDX_6_3]], align 2
+; CHECK-NEXT: [[CONV_6_3:%.*]] = zext i16 [[TMP30]] to i32
+; CHECK-NEXT: [[ADD_6_3:%.*]] = add nuw nsw i32 [[ADD_5_3]], [[CONV_6_3]]
+; CHECK-NEXT: [[MUL_6_3:%.*]] = mul nuw nsw i32 [[CONV_6_3]], [[CONV_6_3]]
+; CHECK-NEXT: [[ADD11_6_3:%.*]] = add i32 [[MUL_6_3]], [[ADD11_5_3]]
+; CHECK-NEXT: [[ARRAYIDX_7_3:%.*]] = getelementptr inbounds i16, ptr [[ADD_PTR_2]], i64 7
+; CHECK-NEXT: [[TMP31:%.*]] = load i16, ptr [[ARRAYIDX_7_3]], align 2
+; CHECK-NEXT: [[CONV_7_3:%.*]] = zext i16 [[TMP31]] to i32
+; CHECK-NEXT: [[ADD_7_3:%.*]] = add nuw nsw i32 [[ADD_6_3]], [[CONV_7_3]]
+; CHECK-NEXT: [[MUL_7_3:%.*]] = mul nuw nsw i32 [[CONV_7_3]], [[CONV_7_3]]
+; CHECK-NEXT: [[ADD11_7_3:%.*]] = add i32 [[MUL_7_3]], [[ADD11_6_3]]
; CHECK-NEXT: [[ADD_PTR_3:%.*]] = getelementptr inbounds i16, ptr [[ADD_PTR_2]], i64 [[IDX_EXT]]
+; CHECK-NEXT: [[TMP32:%.*]] = load i16, ptr [[ADD_PTR_3]], align 2
+; CHECK-NEXT: [[CONV_452:%.*]] = zext i16 [[TMP32]] to i32
+; CHECK-NEXT: [[ADD_453:%.*]] = add nuw nsw i32 [[ADD_7_3]], [[CONV_452]]
+; CHECK-NEXT: [[MUL_454:%.*]] = mul nuw nsw i32 [[CONV_452]], [[CONV_452]]
+; CHECK-NEXT: [[ADD11_455:%.*]] = add i32 [[MUL_454]], [[ADD11_7_3]]
+; CHECK-NEXT: [[ARRAYIDX_1_4:%.*]] = getelementptr inbounds i16, ptr [[ADD_PTR_3]], i64 1
+; CHECK-NEXT: [[TMP33:%.*]] = load i16, ptr [[ARRAYIDX_1_4]], align 2
+; CHECK-NEXT: [[CONV_1_4:%.*]] = zext i16 [[TMP33]] to i32
+; CHECK-NEXT: [[ADD_1_4:%.*]] = add nuw nsw i32 [[ADD_453]], [[CONV_1_4]]
+; CHECK-NEXT: [[MUL_1_4:%.*]] = mul nuw nsw i32 [[CONV_1_4]], [[CONV_1_4]]
+; CHECK-NEXT: [[ADD11_1_4:%.*]] = add i32 [[MUL_1_4]], [[ADD11_455]]
+; CHECK-NEXT: [[ARRAYIDX_2_4:%.*]] = getelementptr inbounds i16, ptr [[ADD_PTR_3]], i64 2
+; CHECK-NEXT: [[TMP34:%.*]] = load i16, ptr [[ARRAYIDX_2_4]], align 2
+; CHECK-NEXT: [[CONV_2_4:%.*]] = zext i16 [[TMP34]] to i32
+; CHECK-NEXT: [[ADD_2_4:%.*]] = add nuw nsw i32 [[ADD_1_4]], [[CONV_2_4]]
+; CHECK-NEXT: [[MUL_2_4:%.*]] = mul nuw nsw i32 [[CONV_2_4]], [[CONV_2_4]]
+; CHECK-NEXT: [[ADD11_2_4:%.*]] = add i32 [[MUL_2_4]], [[ADD11_1_4]]
+; CHECK-NEXT: [[ARRAYIDX_3_4:%.*]] = getelementptr inbounds i16, ptr [[ADD_PTR_3]], i64 3
+; CHECK-NEXT: [[TMP35:%.*]] = load i16, ptr [[ARRAYIDX_3_4]], align 2
+; CHECK-NEXT: [[CONV_3_4:%.*]] = zext i16 [[TMP35]] to i32
+; CHECK-NEXT: [[ADD_3_4:%.*]] = add nuw nsw i32 [[ADD_2_4]], [[CONV_3_4]]
+; CHECK-NEXT: [[MUL_3_4:%.*]] = mul nuw nsw i32 [[CONV_3_4]], [[CONV_3_4]]
+; CHECK-NEXT: [[ADD11_3_4:%.*]] = add i32 [[MUL_3_4]], [[ADD11_2_4]]
+; CHECK-NEXT: [[ARRAYIDX_4_4:%.*]] = getelementptr inbounds i16, ptr [[ADD_PTR_3]], i64 4
+; CHECK-NEXT: [[TMP36:%.*]] = load i16, ptr [[ARRAYIDX_4_4]], align 2
+; CHECK-NEXT: [[CONV_4_4:%.*]] = zext i16 [[TMP36]] to i32
+; CHECK-NEXT: [[ADD_4_4:%.*]] = add nuw nsw i32 [[ADD_3_4]], [[CONV_4_4]]
+; CHECK-NEXT: [[MUL_4_4:%.*]] = mul nuw nsw i32 [[CONV_4_4]], [[CONV_4_4]]
+; CHECK-NEXT: [[ADD11_4_4:%.*]] = add i32 [[MUL_4_4]], [[ADD11_3_4]]
+; CHECK-NEXT: [[ARRAYIDX_5_4:%.*]] = getelementptr inbounds i16, ptr [[ADD_PTR_3]], i64 5
+; CHECK-NEXT: [[TMP37:%.*]] = load i16, ptr [[ARRAYIDX_5_4]], align 2
+; CHECK-NEXT: [[CONV_5_4:%.*]] = zext i16 [[TMP37]] to i32
+; CHECK-NEXT: [[ADD_5_4:%.*]] = add nuw nsw i32 [[ADD_4_4]], [[CONV_5_4]]
+; CHECK-NEXT: [[MUL_5_4:%.*]] = mul nuw nsw i32 [[CONV_5_4]], [[CONV_5_4]]
+; CHECK-NEXT: [[ADD11_5_4:%.*]] = add i32 [[MUL_5_4]], [[ADD11_4_4]]
+; CHECK-NEXT: [[ARRAYIDX_6_4:%.*]] = getelementptr inbounds i16, ptr [[ADD_PTR_3]], i64 6
+; CHECK-NEXT: [[TMP38:%.*]] = load i16, ptr [[ARRAYIDX_6_4]], align 2
+; CHECK-NEXT: [[CONV_6_4:%.*]] = zext i16 [[TMP38]] to i32
+; CHECK-NEXT: [[ADD_6_4:%.*]] = add nuw nsw i32 [[ADD_5_4]], [[CONV_6_4]]
+; CHECK-NEXT: [[MUL_6_4:%.*]] = mul nuw nsw i32 [[CONV_6_4]], [[CONV_6_4]]
+; CHECK-NEXT: [[ADD11_6_4:%.*]] = add i32 [[MUL_6_4]], [[ADD11_5_4]]
+; CHECK-NEXT: [[ARRAYIDX_7_4:%.*]] = getelementptr inbounds i16, ptr [[ADD_PTR_3]], i64 7
+; CHECK-NEXT: [[TMP39:%.*]] = load i16, ptr [[ARRAYIDX_7_4]], align 2
+; CHECK-NEXT: [[CONV_7_4:%.*]] = zext i16 [[TMP39]] to i32
+; CHECK-NEXT: [[ADD_7_4:%.*]] = add nuw nsw i32 [[ADD_6_4]], [[CONV_7_4]]
+; CHECK-NEXT: [[MUL_7_4:%.*]] = mul nuw nsw i32 [[CONV_7_4]], [[CONV_7_4]]
+; CHECK-NEXT: [[ADD11_7_4:%.*]] = add i32 [[MUL_7_4]], [[ADD11_6_4]]
; CHECK-NEXT: [[ADD_PTR_4:%.*]] = getelementptr inbounds i16, ptr [[ADD_PTR_3]], i64 [[IDX_EXT]]
+; CHECK-NEXT: [[TMP40:%.*]] = load i16, ptr [[ADD_PTR_4]], align 2
+; CHECK-NEXT: [[CONV_556:%.*]] = zext i16 [[TMP40]] to i32
+; CHECK-NEXT: [[ADD_557:%.*]] = add nuw nsw i32 [[ADD_7_4]], [[CONV_556]]
+; CHECK-NEXT: [[MUL_558:%.*]] = mul nuw nsw i32 [[CONV_556]], [[CONV_556]]
+; CHECK-NEXT: [[ADD11_559:%.*]] = add i32 [[MUL_558]], [[ADD11_7_4]]
+; CHECK-NEXT: [[ARRAYIDX_1_5:%.*]] = getelementptr inbounds i16, ptr [[ADD_PTR_4]], i64 1
+; CHECK-NEXT: [[TMP41:%.*]] = load i16, ptr [[ARRAYIDX_1_5]], align 2
+; CHECK-NEXT: [[CONV_1_5:%.*]] = zext i16 [[TMP41]] to i32
+; CHECK-NEXT: [[ADD_1_5:%.*]] = add nuw nsw i32 [[ADD_557]], [[CONV_1_5]]
+; CHECK-NEXT: [[MUL_1_5:%.*]] = mul nuw nsw i32 [[CONV_1_5]], [[CONV_1_5]]
+; CHECK-NEXT: [[ADD11_1_5:%.*]] = add i32 [[MUL_1_5]], [[ADD11_559]]
+; CHECK-NEXT: [[ARRAYIDX_2_5:%.*]] = getelementptr inbounds i16, ptr [[ADD_PTR_4]], i64 2
+; CHECK-NEXT: [[TMP42:%.*]] = load i16, ptr [[ARRAYIDX_2_5]], align 2
+; CHECK-NEXT: [[CONV_2_5:%.*]] = zext i16 [[TMP42]] to i32
+; CHECK-NEXT: [[ADD_2_5:%.*]] = add nuw nsw i32 [[ADD_1_5]], [[CONV_2_5]]
+; CHECK-NEXT: [[MUL_2_5:%.*]] = mul nuw nsw i32 [[CONV_2_5]], [[CONV_2_5]]
+; CHECK-NEXT: [[ADD11_2_5:%.*]] = add i32 [[MUL_2_5]], [[ADD11_1_5]]
+; CHECK-NEXT: [[ARRAYIDX_3_5:%.*]] = getelementptr inbounds i16, ptr [[ADD_PTR_4]], i64 3
+; CHECK-NEXT: [[TMP43:%.*]] = load i16, ptr [[ARRAYIDX_3_5]], align 2
+; CHECK-NEXT: [[CONV_3_5:%.*]] = zext i16 [[TMP43]] to i32
+; CHECK-NEXT: [[ADD_3_5:%.*]] = add nuw nsw i32 [[ADD_2_5]], [[CONV_3_5]]
+; CHECK-NEXT: [[MUL_3_5:%.*]] = mul nuw nsw i32 [[CONV_3_5]], [[CONV_3_5]]
+; CHECK-NEXT: [[ADD11_3_5:%.*]] = add i32 [[MUL_3_5]], [[ADD11_2_5]]
+; CHECK-NEXT: [[ARRAYIDX_4_5:%.*]] = getelementptr inbounds i16, ptr [[ADD_PTR_4]], i64 4
+; CHECK-NEXT: [[TMP44:%.*]] = load i16, ptr [[ARRAYIDX_4_5]], align 2
+; CHECK-NEXT: [[CONV_4_5:%.*]] = zext i16 [[TMP44]] to i32
+; CHECK-NEXT: [[ADD_4_5:%.*]] = add nuw nsw i32 [[ADD_3_5]], [[CONV_4_5]]
+; CHECK-NEXT: [[MUL_4_5:%.*]] = mul nuw nsw i32 [[CONV_4_5]], [[CONV_4_5]]
+; CHECK-NEXT: [[ADD11_4_5:%.*]] = add i32 [[MUL_4_5]], [[ADD11_3_5]]
+; CHECK-NEXT: [[ARRAYIDX_5_5:%.*]] = getelementptr inbounds i16, ptr [[ADD_PTR_4]], i64 5
+; CHECK-NEXT: [[TMP45:%.*]] = load i16, ptr [[ARRAYIDX_5_5]], align 2
+; CHECK-NEXT: [[CONV_5_5:%.*]] = zext i16 [[TMP45]] to i32
+; CHECK-NEXT: [[ADD_5_5:%.*]] = add nuw nsw i32 [[ADD_4_5]], [[CONV_5_5]]
+; CHECK-NEXT: [[MUL_5_5:%.*]] = mul nuw nsw i32 [[CONV_5_5]], [[CONV_5_5]]
+; CHECK-NEXT: [[ADD11_5_5:%.*]] = add i32 [[MUL_5_5]], [[ADD11_4_5]]
+; CHECK-NEXT: [[ARRAYIDX_6_5:%.*]] = getelementptr inbounds i16, ptr [[ADD_PTR_4]], i64 6
+; CHECK-NEXT: [[TMP46:%.*]] = load i16, ptr [[ARRAYIDX_6_5]], align 2
+; CHECK-NEXT: [[CONV_6_5:%.*]] = zext i16 [[TMP46]] to i32
+; CHECK-NEXT: [[ADD_6_5:%.*]] = add nuw nsw i32 [[ADD_5_5]], [[CONV_6_5]]
+; CHECK-NEXT: [[MUL_6_5:%.*]] = mul nuw nsw i32 [[CONV_6_5]], [[CONV_6_5]]
+; CHECK-NEXT: [[ADD11_6_5:%.*]] = add i32 [[MUL_6_5]], [[ADD11_5_5]]
+; CHECK-NEXT: [[ARRAYIDX_7_5:%.*]] = getelementptr inbounds i16, ptr [[ADD_PTR_4]], i64 7
+; CHECK-NEXT: [[TMP47:%.*]] = load i16, ptr [[ARRAYIDX_7_5]], align 2
+; CHECK-NEXT: [[CONV_7_5:%.*]] = zext i16 [[TMP47]] to i32
+; CHECK-NEXT: [[ADD_7_5:%.*]] = add nuw nsw i32 [[ADD_6_5]], [[CONV_7_5]]
+; CHECK-NEXT: [[MUL_7_5:%.*]] = mul nuw nsw i32 [[CONV_7_5]], [[CONV_7_5]]
+; CHECK-NEXT: [[ADD11_7_5:%.*]] = add i32 [[MUL_7_5]], [[ADD11_6_5]]
; CHECK-NEXT: [[ADD_PTR_5:%.*]] = getelementptr inbounds i16, ptr [[ADD_PTR_4]], i64 [[IDX_EXT]]
+; CHECK-NEXT: [[TMP48:%.*]] = load i16, ptr [[ADD_PTR_5]], align 2
+; CHECK-NEXT: [[CONV_660:%.*]] = zext i16 [[TMP48]] to i32
+; CHECK-NEXT: [[ADD_661:%.*]] = add nuw nsw i32 [[ADD_7_5]], [[CONV_660]]
+; CHECK-NEXT: [[MUL_662:%.*]] = mul nuw nsw i32 [[CONV_660]], [[CONV_660]]
+; CHECK-NEXT: [[ADD11_663:%.*]] = add i32 [[MUL_662]], [[ADD11_7_5]]
+; CHECK-NEXT: [[ARRAYIDX_1_6:%.*]] = getelementptr inbounds i16, ptr [[ADD_PTR_5]], i64 1
+; CHECK-NEXT: [[TMP49:%.*]] = load i16, ptr [[ARRAYIDX_1_6]], align 2
+; CHECK-NEXT: [[CONV_1_6:%.*]] = zext i16 [[TMP49]] to i32
+; CHECK-NEXT: [[ADD_1_6:%.*]] = add nuw nsw i32 [[ADD_661]], [[CONV_1_6]]
+; CHECK-NEXT: [[MUL_1_6:%.*]] = mul nuw nsw i32 [[CONV_1_6]], [[CONV_1_6]]
+; CHECK-NEXT: [[ADD11_1_6:%.*]] = add i32 [[MUL_1_6]], [[ADD11_663]]
+; CHECK-NEXT: [[ARRAYIDX_2_6:%.*]] = getelementptr inbounds i16, ptr [[ADD_PTR_5]], i64 2
+; CHECK-NEXT: [[TMP50:%.*]] = load i16, ptr [[ARRAYIDX_2_6]], align 2
+; CHECK-NEXT: [[CONV_2_6:%.*]] = zext i16 [[TMP50]] to i32
+; CHECK-NEXT: [[ADD_2_6:%.*]] = add nuw nsw i32 [[ADD_1_6]], [[CONV_2_6]]
+; CHECK-NEXT: [[MUL_2_6:%.*]] = mul nuw nsw i32 [[CONV_2_6]], [[CONV_2_6]]
+; CHECK-NEXT: [[ADD11_2_6:%.*]] = add i32 [[MUL_2_6]], [[ADD11_1_6]]
+; CHECK-NEXT: [[ARRAYIDX_3_6:%.*]] = getelementptr inbounds i16, ptr [[ADD_PTR_5]], i64 3
+; CHECK-NEXT: [[TMP51:%.*]] = load i16, ptr [[ARRAYIDX_3_6]], align 2
+; CHECK-NEXT: [[CONV_3_6:%.*]] = zext i16 [[TMP51]] to i32
+; CHECK-NEXT: [[ADD_3_6:%.*]] = add nuw nsw i32 [[ADD_2_6]], [[CONV_3_6]]
+; CHECK-NEXT: [[MUL_3_6:%.*]] = mul nuw nsw i32 [[CONV_3_6]], [[CONV_3_6]]
+; CHECK-NEXT: [[ADD11_3_6:%.*]] = add i32 [[MUL_3_6]], [[ADD11_2_6]]
+; CHECK-NEXT: [[ARRAYIDX_4_6:%.*]] = getelementptr inbounds i16, ptr [[ADD_PTR_5]], i64 4
+; CHECK-NEXT: [[TMP52:%.*]] = load i16, ptr [[ARRAYIDX_4_6]], align 2
+; CHECK-NEXT: [[CONV_4_6:%.*]] = zext i16 [[TMP52]] to i32
+; CHECK-NEXT: [[ADD_4_6:%.*]] = add nuw nsw i32 [[ADD_3_6]], [[CONV_4_6]]
+; CHECK-NEXT: [[MUL_4_6:%.*]] = mul nuw nsw i32 [[CONV_4_6]], [[CONV_4_6]]
+; CHECK-NEXT: [[ADD11_4_6:%.*]] = add i32 [[MUL_4_6]], [[ADD11_3_6]]
+; CHECK-NEXT: [[ARRAYIDX_5_6:%.*]] = getelementptr inbounds i16, ptr [[ADD_PTR_5]], i64 5
+; CHECK-NEXT: [[TMP53:%.*]] = load i16, ptr [[ARRAYIDX_5_6]], align 2
+; CHECK-NEXT: [[CONV_5_6:%.*]] = zext i16 [[TMP53]] to i32
+; CHECK-NEXT: [[ADD_5_6:%.*]] = add nuw nsw i32 [[ADD_4_6]], [[CONV_5_6]]
+; CHECK-NEXT: [[MUL_5_6:%.*]] = mul nuw nsw i32 [[CONV_5_6]], [[CONV_5_6]]
+; CHECK-NEXT: [[ADD11_5_6:%.*]] = add i32 [[MUL_5_6]], [[ADD11_4_6]]
+; CHECK-NEXT: [[ARRAYIDX_6_6:%.*]] = getelementptr inbounds i16, ptr [[ADD_PTR_5]], i64 6
+; CHECK-NEXT: [[TMP54:%.*]] = load i16, ptr [[ARRAYIDX_6_6]], align 2
+; CHECK-NEXT: [[CONV_6_6:%.*]] = zext i16 [[TMP54]] to i32
+; CHECK-NEXT: [[ADD_6_6:%.*]] = add nuw nsw i32 [[ADD_5_6]], [[CONV_6_6]]
+; CHECK-NEXT: [[MUL_6_6:%.*]] = mul nuw nsw i32 [[CONV_6_6]], [[CONV_6_6]]
+; CHECK-NEXT: [[ADD11_6_6:%.*]] = add i32 [[MUL_6_6]], [[ADD11_5_6]]
+; CHECK-NEXT: [[ARRAYIDX_7_6:%.*]] = getelementptr inbounds i16, ptr [[ADD_PTR_5]], i64 7
+; CHECK-NEXT: [[TMP55:%.*]] = load i16, ptr [[ARRAYIDX_7_6]], align 2
+; CHECK-NEXT: [[CONV_7_6:%.*]] = zext i16 [[TMP55]] to i32
+; CHECK-NEXT: [[ADD_7_6:%.*]] = add nuw nsw i32 [[ADD_6_6]], [[CONV_7_6]]
+; CHECK-NEXT: [[MUL_7_6:%.*]] = mul nuw nsw i32 [[CONV_7_6]], [[CONV_7_6]]
+; CHECK-NEXT: [[ADD11_7_6:%.*]] = add i32 [[MUL_7_6]], [[ADD11_6_6]]
; CHECK-NEXT: [[ADD_PTR_6:%.*]] = getelementptr inbounds i16, ptr [[ADD_PTR_5]], i64 [[IDX_EXT]]
-; CHECK-NEXT: [[TMP0:%.*]] = load <8 x i16>, ptr [[P]], align 2
-; CHECK-NEXT: [[TMP1:%.*]] = load <8 x i16>, ptr [[ADD_PTR]], align 2
-; CHECK-NEXT: [[TMP2:%.*]] = load <8 x i16>, ptr [[ADD_PTR_1]], align 2
-; CHECK-NEXT: [[TMP3:%.*]] = load <8 x i16>, ptr [[ADD_PTR_2]], align 2
-; CHECK-NEXT: [[TMP4:%.*]] = load <8 x i16>, ptr [[ADD_PTR_3]], align 2
-; CHECK-NEXT: [[TMP5:%.*]] = load <8 x i16>, ptr [[ADD_PTR_4]], align 2
-; CHECK-NEXT: [[TMP6:%.*]] = load <8 x i16>, ptr [[ADD_PTR_5]], align 2
-; CHECK-NEXT: [[TMP7:%.*]] = load <8 x i16>, ptr [[ADD_PTR_6]], align 2
-; CHECK-NEXT: [[TMP8:%.*]] = shufflevector <8 x i16> [[TMP7]], <8 x i16> poison, <64 x i32> <i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>
-; CHECK-NEXT: [[TMP9:%.*]] = shufflevector <8 x i16> [[TMP6]], <8 x i16> poison, <64 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
-; CHECK-NEXT: [[TMP10:%.*]] = shufflevector <8 x i16> [[TMP7]], <8 x i16> [[TMP6]], <64 x i32> <i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15, i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>
-; CHECK-NEXT: [[TMP11:%.*]] = shufflevector <8 x i16> [[TMP5]], <8 x i16> poison, <64 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
-; CHECK-NEXT: [[TMP12:%.*]] = shufflevector <64 x i16> [[TMP10]], <64 x i16> [[TMP11]], <64 x i32> <i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 64, i32 65, i32 66, i32 67, i32 68, i32 69, i32 70, i32 71, i32 48, i32 49, i32 50, i32 51, i32 52, i32 53, i32 54, i32 55, i32 56, i32 57, i32 58, i32 59, i32 60, i32 61, i32 62, i32 63>
-; CHECK-NEXT: [[TMP13:%.*]] = shufflevector <8 x i16> [[TMP4]], <8 x i16> poison, <64 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
-; CHECK-NEXT: [[TMP14:%.*]] = shufflevector <64 x i16> [[TMP12]], <64 x i16> [[TMP13]], <64 x i32> <i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 64, i32 65, i32 66, i32 67, i32 68, i32 69, i32 70, i32 71, i32 40, i32 41, i32 42, i32 43, i32 44, i32 45, i32 46, i32 47, i32 48, i32 49, i32 50, i32 51, i32 52, i32 53, i32 54, i32 55, i32 56, i32 57, i32 58, i32 59, i32 60, i32 61, i32 62, i32 63>
-; CHECK-NEXT: [[TMP15:%.*]] = shufflevector <8 x i16> [[TMP3]], <8 x i16> poison, <64 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
-; CHECK-NEXT: [[TMP16:%.*]] = shufflevector <64 x i16> [[TMP14]], <64 x i16> [[TMP15]], <64 x i32> <i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 64, i32 65, i32 66, i32 67, i32 68, i32 69, i32 70, i32 71, i32 32, i32 33, i32 34, i32 35, i32 36, i32 37, i32 38, i32 39, i32 40, i32 41, i32 42, i32 43, i32 44, i32 45, i32 46, i32 47, i32 48, i32 49, i32 50, i32 51, i32 52, i32 53, i32 54, i32 55, i32 56, i32 57, i32 58, i32 59, i32 60, i32 61, i32 62, i32 63>
-; CHECK-NEXT: [[TMP17:%.*]] = shufflevector <8 x i16> [[TMP2]], <8 x i16> poison, <64 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
-; CHECK-NEXT: [[TMP18:%.*]] = shufflevector <64 x i16> [[TMP16]], <64 x i16> [[TMP17]], <64 x i32> <i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 64, i32 65, i32 66, i32 67, i32 68, i32 69, i32 70, i32 71, i32 24, i32 25, i32 26, i32 27, i32 28, i32 29, i32 30, i32 31, i32 32, i32 33, i32 34, i32 35, i32 36, i32 37, i32 38, i32 39, i32 40, i32 41, i32 42, i32 43, i32 44, i32 45, i32 46, i32 47, i32 48, i32 49, i32 50, i32 51, i32 52, i32 53, i32 54, i32 55, i32 56, i32 57, i32 58, i32 59, i32 60, i32 61, i32 62, i32 63>
-; CHECK-NEXT: [[TMP19:%.*]] = shufflevector <8 x i16> [[TMP0]], <8 x i16> poison, <64 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
-; CHECK-NEXT: [[TMP20:%.*]] = shufflevector <64 x i16> [[TMP18]], <64 x i16> [[TMP19]], <64 x i32> <i32 64, i32 65, i32 66, i32 67, i32 68, i32 69, i32 70, i32 71, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 16, i32 17, i32 18, i32 19, i32 20, i32 21, i32 22, i32 23, i32 24, i32 25, i32 26, i32 27, i32 28, i32 29, i32 30, i32 31, i32 32, i32 33, i32 34, i32 35, i32 36, i32 37, i32 38, i32 39, i32 40, i32 41, i32 42, i32 43, i32 44, i32 45, i32 46, i32 47, i32 48, i32 49, i32 50, i32 51, i32 52, i32 53, i32 54, i32 55, i32 56, i32 57, i32 58, i32 59, i32 60, i32 61, i32 62, i32 63>
-; CHECK-NEXT: [[TMP21:%.*]] = shufflevector <8 x i16> [[TMP1]], <8 x i16> poison, <64 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
-; CHECK-NEXT: [[TMP22:%.*]] = shufflevector <64 x i16> [[TMP20]], <64 x i16> [[TMP21]], <64 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 64, i32 65, i32 66, i32 67, i32 68, i32 69, i32 70, i32 71, i32 16, i32 17, i32 18, i32 19, i32 20, i32 21, i32 22, i32 23, i32 24, i32 25, i32 26, i32 27, i32 28, i32 29, i32 30, i32 31, i32 32, i32 33, i32 34, i32 35, i32 36, i32 37, i32 38, i32 39, i32 40, i32 41, i32 42, i32 43, i32 44, i32 45, i32 46, i32 47, i32 48, i32 49, i32 50, i32 51, i32 52, i32 53, i32 54, i32 55, i32 56, i32 57, i32 58, i32 59, i32 60, i32 61, i32 62, i32 63>
-; CHECK-NEXT: [[TMP23:%.*]] = zext <64 x i16> [[TMP22]] to <64 x i32>
-; CHECK-NEXT: [[ADD_7_7:%.*]] = call i32 @llvm.vector.reduce.add.v64i32(<64 x i32> [[TMP23]])
-; CHECK-NEXT: [[TMP25:%.*]] = mul nuw nsw <64 x i32> [[TMP23]], [[TMP23]]
-; CHECK-NEXT: [[ADD11_7_7:%.*]] = call i32 @llvm.vector.reduce.add.v64i32(<64 x i32> [[TMP25]])
+; CHECK-NEXT: [[TMP56:%.*]] = load i16, ptr [[ADD_PTR_6]], align 2
+; CHECK-NEXT: [[CONV_764:%.*]] = zext i16 [[TMP56]] to i32
+; CHECK-NEXT: [[ADD_765:%.*]] = add nuw nsw i32 [[ADD_7_6]], [[CONV_764]]
+; CHECK-NEXT: [[MUL_766:%.*]] = mul nuw nsw i32 [[CONV_764]], [[CONV_764]]
+; CHECK-NEXT: [[ADD11_767:%.*]] = add i32 [[MUL_766]], [[ADD11_7_6]]
+; CHECK-NEXT: [[ARRAYIDX_1_7:%.*]] = getelementptr inbounds i16, ptr [[ADD_PTR_6]], i64 1
+; CHECK-NEXT: [[TMP57:%.*]] = load i16, ptr [[ARRAYIDX_1_7]], align 2
+; CHECK-NEXT: [[CONV_1_7:%.*]] = zext i16 [[TMP57]] to i32
+; CHECK-NEXT: [[ADD_1_7:%.*]] = add nuw nsw i32 [[ADD_765]], [[CONV_1_7]]
+; CHECK-NEXT: [[MUL_1_7:%.*]] = mul nuw nsw i32 [[CONV_1_7]], [[CONV_1_7]]
+; CHECK-NEXT: [[ADD11_1_7:%.*]] = add i32 [[MUL_1_7]], [[ADD11_767]]
+; CHECK-NEXT: [[ARRAYIDX_2_7:%.*]] = getelementptr inbounds i16, ptr [[ADD_PTR_6]], i64 2
+; CHECK-NEXT: [[TMP58:%.*]] = load i16, ptr [[ARRAYIDX_2_7]], align 2
+; CHECK-NEXT: [[CONV_2_7:%.*]] = zext i16 [[TMP58]] to i32
+; CHECK-NEXT: [[ADD_2_7:%.*]] = add nuw nsw i32 [[ADD_1_7]], [[CONV_2_7]]
+; CHECK-NEXT: [[MUL_2_7:%.*]] = mul nuw nsw i32 [[CONV_2_7]], [[CONV_2_7]]
+; CHECK-NEXT: [[ADD11_2_7:%.*]] = add i32 [[MUL_2_7]], [[ADD11_1_7]]
+; CHECK-NEXT: [[ARRAYIDX_3_7:%.*]] = getelementptr inbounds i16, ptr [[ADD_PTR_6]], i64 3
+; CHECK-NEXT: [[TMP59:%.*]] = load i16, ptr [[ARRAYIDX_3_7]], align 2
+; CHECK-NEXT: [[CONV_3_7:%.*]] = zext i16 [[TMP59]] to i32
+; CHECK-NEXT: [[ADD_3_7:%.*]] = add nuw nsw i32 [[ADD_2_7]], [[CONV_3_7]]
+; CHECK-NEXT: [[MUL_3_7:%.*]] = mul nuw nsw i32 [[CONV_3_7]], [[CONV_3_7]]
+; CHECK-NEXT: [[ADD11_3_7:%.*]] = add i32 [[MUL_3_7]], [[ADD11_2_7]]
+; CHECK-NEXT: [[ARRAYIDX_4_7:%.*]] = getelementptr inbounds i16, ptr [[ADD_PTR_6]], i64 4
+; CHECK-NEXT: [[TMP60:%.*]] = load i16, ptr [[ARRAYIDX_4_7]], align 2
+; CHECK-NEXT: [[CONV_4_7:%.*]] = zext i16 [[TMP60]] to i32
+; CHECK-NEXT: [[ADD_4_7:%.*]] = add nuw nsw i32 [[ADD_3_7]], [[CONV_4_7]]
+; CHECK-NEXT: [[MUL_4_7:%.*]] = mul nuw nsw i32 [[CONV_4_7]], [[CONV_4_7]]
+; CHECK-NEXT: [[ADD11_4_7:%.*]] = add i32 [[MUL_4_7]], [[ADD11_3_7]]
+; CHECK-NEXT: [[ARRAYIDX_5_7:%.*]] = getelementptr inbounds i16, ptr [[ADD_PTR_6]], i64 5
+; CHECK-NEXT: [[TMP61:%.*]] = load i16, ptr [[ARRAYIDX_5_7]], align 2
+; CHECK-NEXT: [[CONV_5_7:%.*]] = zext i16 [[TMP61]] to i32
+; CHECK-NEXT: [[ADD_5_7:%.*]] = add nuw nsw i32 [[ADD_4_7]], [[CONV_5_7]]
+; CHECK-NEXT: [[MUL_5_7:%.*]] = mul nuw nsw i32 [[CONV_5_7]], [[CONV_5_7]]
+; CHECK-NEXT: [[ADD11_5_7:%.*]] = add i32 [[MUL_5_7]], [[ADD11_4_7]]
+; CHECK-NEXT: [[ARRAYIDX_6_7:%.*]] = getelementptr inbounds i16, ptr [[ADD_PTR_6]], i64 6
+; CHECK-NEXT: [[TMP62:%.*]] = load i16, ptr [[ARRAYIDX_6_7]], align 2
+; CHECK-NEXT: [[CONV_6_7:%.*]] = zext i16 [[TMP62]] to i32
+; CHECK-NEXT: [[ADD_6_7:%.*]] = add nuw nsw i32 [[ADD_5_7]], [[CONV_6_7]]
+; CHECK-NEXT: [[MUL_6_7:%.*]] = mul nuw nsw i32 [[CONV_6_7]], [[CONV_6_7]]
+; CHECK-NEXT: [[ADD11_6_7:%.*]] = add i32 [[MUL_6_7]], [[ADD11_5_7]]
+; CHECK-NEXT: [[ARRAYIDX_7_7:%.*]] = getelementptr inbounds i16, ptr [[ADD_PTR_6]], i64 7
+; CHECK-NEXT: [[TMP63:%.*]] = load i16, ptr [[ARRAYIDX_7_7]], align 2
+; CHECK-NEXT: [[CONV_7_7:%.*]] = zext i16 [[TMP63]] to i32
+; CHECK-NEXT: [[ADD_7_7:%.*]] = add nuw nsw i32 [[ADD_6_7]], [[CONV_7_7]]
+; CHECK-NEXT: [[MUL_7_7:%.*]] = mul nuw nsw i32 [[CONV_7_7]], [[CONV_7_7]]
+; CHECK-NEXT: [[ADD11_7_7:%.*]] = add i32 [[MUL_7_7]], [[ADD11_6_7]]
; CHECK-NEXT: [[CONV15:%.*]] = zext i32 [[ADD_7_7]] to i64
; CHECK-NEXT: [[CONV16:%.*]] = zext i32 [[ADD11_7_7]] to i64
; CHECK-NEXT: [[SHL:%.*]] = shl nuw i64 [[CONV16]], 32
@@ -454,14 +801,101 @@ define i64 @looped(ptr nocapture noundef readonly %p, i32 noundef %st) {
; CHECK-NEXT: [[SQ_037:%.*]] = phi i32 [ 0, [[ENTRY]] ], [ [[OP_RDX:%.*]], [[FOR_COND1_PREHEADER]] ]
; CHECK-NEXT: [[SM_036:%.*]] = phi i32 [ 0, [[ENTRY]] ], [ [[OP_RDX1:%.*]], [[FOR_COND1_PREHEADER]] ]
; CHECK-NEXT: [[P_ADDR_035:%.*]] = phi ptr [ [[P:%.*]], [[ENTRY]] ], [ [[ADD_PTR:%.*]], [[FOR_COND1_PREHEADER]] ]
-; CHECK-NEXT: [[TMP0:%.*]] = load <16 x i16>, ptr [[P_ADDR_035]], align 2
-; CHECK-NEXT: [[TMP1:%.*]] = zext <16 x i16> [[TMP0]] to <16 x i32>
-; CHECK-NEXT: [[TMP2:%.*]] = zext <16 x i16> [[TMP0]] to <16 x i32>
-; CHECK-NEXT: [[TMP3:%.*]] = mul nuw nsw <16 x i32> [[TMP1]], [[TMP1]]
-; CHECK-NEXT: [[TMP4:%.*]] = call i32 @llvm.vector.reduce.add.v16i32(<16 x i32> [[TMP2]])
-; CHECK-NEXT: [[OP_RDX1]] = add i32 [[TMP4]], [[SM_036]]
-; CHECK-NEXT: [[TMP5:%.*]] = call i32 @llvm.vector.reduce.add.v16i32(<16 x i32> [[TMP3]])
-; CHECK-NEXT: [[OP_RDX]] = add i32 [[TMP5]], [[SQ_037]]
+; CHECK-NEXT: [[TMP0:%.*]] = load i16, ptr [[P_ADDR_035]], align 2
+; CHECK-NEXT: [[CONV:%.*]] = zext i16 [[TMP0]] to i32
+; CHECK-NEXT: [[ADD:%.*]] = add i32 [[SM_036]], [[CONV]]
+; CHECK-NEXT: [[MUL:%.*]] = mul nuw nsw i32 [[CONV]], [[CONV]]
+; CHECK-NEXT: [[ADD11:%.*]] = add i32 [[MUL]], [[SQ_037]]
+; CHECK-NEXT: [[ARRAYIDX_1:%.*]] = getelementptr inbounds i16, ptr [[P_ADDR_035]], i64 1
+; CHECK-NEXT: [[TMP1:%.*]] = load i16, ptr [[ARRAYIDX_1]], align 2
+; CHECK-NEXT: [[CONV_1:%.*]] = zext i16 [[TMP1]] to i32
+; CHECK-NEXT: [[ADD_1:%.*]] = add i32 [[ADD]], [[CONV_1]]
+; CHECK-NEXT: [[MUL_1:%.*]] = mul nuw nsw i32 [[CONV_1]], [[CONV_1]]
+; CHECK-NEXT: [[ADD11_1:%.*]] = add i32 [[MUL_1]], [[ADD11]]
+; CHECK-NEXT: [[ARRAYIDX_2:%.*]] = getelementptr inbounds i16, ptr [[P_ADDR_035]], i64 2
+; CHECK-NEXT: [[TMP2:%.*]] = load i16, ptr [[ARRAYIDX_2]], align 2
+; CHECK-NEXT: [[CONV_2:%.*]] = zext i16 [[TMP2]] to i32
+; CHECK-NEXT: [[ADD_2:%.*]] = add i32 [[ADD_1]], [[CONV_2]]
+; CHECK-NEXT: [[MUL_2:%.*]] = mul nuw nsw i32 [[CONV_2]], [[CONV_2]]
+; CHECK-NEXT: [[ADD11_2:%.*]] = add i32 [[MUL_2]], [[ADD11_1]]
+; CHECK-NEXT: [[ARRAYIDX_3:%.*]] = getelementptr inbounds i16, ptr [[P_ADDR_035]], i64 3
+; CHECK-NEXT: [[TMP3:%.*]] = load i16, ptr [[ARRAYIDX_3]], align 2
+; CHECK-NEXT: [[CONV_3:%.*]] = zext i16 [[TMP3]] to i32
+; CHECK-NEXT: [[ADD_3:%.*]] = add i32 [[ADD_2]], [[CONV_3]]
+; CHECK-NEXT: [[MUL_3:%.*]] = mul nuw nsw i32 [[CONV_3]], [[CONV_3]]
+; CHECK-NEXT: [[ADD11_3:%.*]] = add i32 [[MUL_3]], [[ADD11_2]]
+; CHECK-NEXT: [[ARRAYIDX_4:%.*]] = getelementptr inbounds i16, ptr [[P_ADDR_035]], i64 4
+; CHECK-NEXT: [[TMP4:%.*]] = load i16, ptr [[ARRAYIDX_4]], align 2
+; CHECK-NEXT: [[CONV_4:%.*]] = zext i16 [[TMP4]] to i32
+; CHECK-NEXT: [[ADD_4:%.*]] = add i32 [[ADD_3]], [[CONV_4]]
+; CHECK-NEXT: [[MUL_4:%.*]] = mul nuw nsw i32 [[CONV_4]], [[CONV_4]]
+; CHECK-NEXT: [[ADD11_4:%.*]] = add i32 [[MUL_4]], [[ADD11_3]]
+; CHECK-NEXT: [[ARRAYIDX_5:%.*]] = getelementptr inbounds i16, ptr [[P_ADDR_035]], i64 5
+; CHECK-NEXT: [[TMP5:%.*]] = load i16, ptr [[ARRAYIDX_5]], align 2
+; CHECK-NEXT: [[CONV_5:%.*]] = zext i16 [[TMP5]] to i32
+; CHECK-NEXT: [[ADD_5:%.*]] = add i32 [[ADD_4]], [[CONV_5]]
+; CHECK-NEXT: [[MUL_5:%.*]] = mul nuw nsw i32 [[CONV_5]], [[CONV_5]]
+; CHECK-NEXT: [[ADD11_5:%.*]] = add i32 [[MUL_5]], [[ADD11_4]]
+; CHECK-NEXT: [[ARRAYIDX_6:%.*]] = getelementptr inbounds i16, ptr [[P_ADDR_035]], i64 6
+; CHECK-NEXT: [[TMP6:%.*]] = load i16, ptr [[ARRAYIDX_6]], align 2
+; CHECK-NEXT: [[CONV_6:%.*]] = zext i16 [[TMP6]] to i32
+; CHECK-NEXT: [[ADD_6:%.*]] = add i32 [[ADD_5]], [[CONV_6]]
+; CHECK-NEXT: [[MUL_6:%.*]] = mul nuw nsw i32 [[CONV_6]], [[CONV_6]]
+; CHECK-NEXT: [[ADD11_6:%.*]] = add i32 [[MUL_6]], [[ADD11_5]]
+; CHECK-NEXT: [[ARRAYIDX_7:%.*]] = getelementptr inbounds i16, ptr [[P_ADDR_035]], i64 7
+; CHECK-NEXT: [[TMP7:%.*]] = load i16, ptr [[ARRAYIDX_7]], align 2
+; CHECK-NEXT: [[CONV_7:%.*]] = zext i16 [[TMP7]] to i32
+; CHECK-NEXT: [[ADD_7:%.*]] = add i32 [[ADD_6]], [[CONV_7]]
+; CHECK-NEXT: [[MUL_7:%.*]] = mul nuw nsw i32 [[CONV_7]], [[CONV_7]]
+; CHECK-NEXT: [[ADD11_7:%.*]] = add i32 [[MUL_7]], [[ADD11_6]]
+; CHECK-NEXT: [[ARRAYIDX_8:%.*]] = getelementptr inbounds i16, ptr [[P_ADDR_035]], i64 8
+; CHECK-NEXT: [[TMP8:%.*]] = load i16, ptr [[ARRAYIDX_8]], align 2
+; CHECK-NEXT: [[CONV_8:%.*]] = zext i16 [[TMP8]] to i32
+; CHECK-NEXT: [[ADD_8:%.*]] = add i32 [[ADD_7]], [[CONV_8]]
+; CHECK-NEXT: [[MUL_8:%.*]] = mul nuw nsw i32 [[CONV_8]], [[CONV_8]]
+; CHECK-NEXT: [[ADD11_8:%.*]] = add i32 [[MUL_8]], [[ADD11_7]]
+; CHECK-NEXT: [[ARRAYIDX_9:%.*]] = getelementptr inbounds i16, ptr [[P_ADDR_035]], i64 9
+; CHECK-NEXT: [[TMP9:%.*]] = load i16, ptr [[ARRAYIDX_9]], align 2
+; CHECK-NEXT: [[CONV_9:%.*]] = zext i16 [[TMP9]] to i32
+; CHECK-NEXT: [[ADD_9:%.*]] = add i32 [[ADD_8]], [[CONV_9]]
+; CHECK-NEXT: [[MUL_9:%.*]] = mul nuw nsw i32 [[CONV_9]], [[CONV_9]]
+; CHECK-NEXT: [[ADD11_9:%.*]] = add i32 [[MUL_9]], [[ADD11_8]]
+; CHECK-NEXT: [[ARRAYIDX_10:%.*]] = getelementptr inbounds i16, ptr [[P_ADDR_035]], i64 10
+; CHECK-NEXT: [[TMP10:%.*]] = load i16, ptr [[ARRAYIDX_10]], align 2
+; CHECK-NEXT: [[CONV_10:%.*]] = zext i16 [[TMP10]] to i32
+; CHECK-NEXT: [[ADD_10:%.*]] = add i32 [[ADD_9]], [[CONV_10]]
+; CHECK-NEXT: [[MUL_10:%.*]] = mul nuw nsw i32 [[CONV_10]], [[CONV_10]]
+; CHECK-NEXT: [[ADD11_10:%.*]] = add i32 [[MUL_10]], [[ADD11_9]]
+; CHECK-NEXT: [[ARRAYIDX_11:%.*]] = getelementptr inbounds i16, ptr [[P_ADDR_035]], i64 11
+; CHECK-NEXT: [[TMP11:%.*]] = load i16, ptr [[ARRAYIDX_11]], align 2
+; CHECK-NEXT: [[CONV_11:%.*]] = zext i16 [[TMP11]] to i32
+; CHECK-NEXT: [[ADD_11:%.*]] = add i32 [[ADD_10]], [[CONV_11]]
+; CHECK-NEXT: [[MUL_11:%.*]] = mul nuw nsw i32 [[CONV_11]], [[CONV_11]]
+; CHECK-NEXT: [[ADD11_11:%.*]] = add i32 [[MUL_11]], [[ADD11_10]]
+; CHECK-NEXT: [[ARRAYIDX_12:%.*]] = getelementptr inbounds i16, ptr [[P_ADDR_035]], i64 12
+; CHECK-NEXT: [[TMP12:%.*]] = load i16, ptr [[ARRAYIDX_12]], align 2
+; CHECK-NEXT: [[CONV_12:%.*]] = zext i16 [[TMP12]] to i32
+; CHECK-NEXT: [[ADD_12:%.*]] = add i32 [[ADD_11]], [[CONV_12]]
+; CHECK-NEXT: [[MUL_12:%.*]] = mul nuw nsw i32 [[CONV_12]], [[CONV_12]]
+; CHECK-NEXT: [[ADD11_12:%.*]] = add i32 [[MUL_12]], [[ADD11_11]]
+; CHECK-NEXT: [[ARRAYIDX_13:%.*]] = getelementptr inbounds i16, ptr [[P_ADDR_035]], i64 13
+; CHECK-NEXT: [[TMP13:%.*]] = load i16, ptr [[ARRAYIDX_13]], align 2
+; CHECK-NEXT: [[CONV_13:%.*]] = zext i16 [[TMP13]] to i32
+; CHECK-NEXT: [[ADD_13:%.*]] = add i32 [[ADD_12]], [[CONV_13]]
+; CHECK-NEXT: [[MUL_13:%.*]] = mul nuw nsw i32 [[CONV_13]], [[CONV_13]]
+; CHECK-NEXT: [[ADD11_13:%.*]] = add i32 [[MUL_13]], [[ADD11_12]]
+; CHECK-NEXT: [[ARRAYIDX_14:%.*]] = getelementptr inbounds i16, ptr [[P_ADDR_035]], i64 14
+; CHECK-NEXT: [[TMP14:%.*]] = load i16, ptr [[ARRAYIDX_14]], align 2
+; CHECK-NEXT: [[CONV_14:%.*]] = zext i16 [[TMP14]] to i32
+; CHECK-NEXT: [[ADD_14:%.*]] = add i32 [[ADD_13]], [[CONV_14]]
+; CHECK-NEXT: [[MUL_14:%.*]] = mul nuw nsw i32 [[CONV_14]], [[CONV_14]]
+; CHECK-NEXT: [[ADD11_14:%.*]] = add i32 [[MUL_14]], [[ADD11_13]]
+; CHECK-NEXT: [[ARRAYIDX_15:%.*]] = getelementptr inbounds i16, ptr [[P_ADDR_035]], i64 15
+; CHECK-NEXT: [[TMP15:%.*]] = load i16, ptr [[ARRAYIDX_15]], align 2
+; CHECK-NEXT: [[CONV_15:%.*]] = zext i16 [[TMP15]] to i32
+; CHECK-NEXT: [[OP_RDX1]] = add i32 [[ADD_14]], [[CONV_15]]
+; CHECK-NEXT: [[MUL_15:%.*]] = mul nuw nsw i32 [[CONV_15]], [[CONV_15]]
+; CHECK-NEXT: [[OP_RDX]] = add i32 [[MUL_15]], [[ADD11_14]]
; CHECK-NEXT: [[ADD_PTR]] = getelementptr inbounds i16, ptr [[P_ADDR_035]], i64 [[IDX_EXT]]
; CHECK-NEXT: [[INC13]] = add nuw nsw i32 [[Y_038]], 1
; CHECK-NEXT: [[EXITCOND_NOT:%.*]] = icmp eq i32 [[INC13]], 16
diff --git a/llvm/test/Transforms/SLPVectorizer/RISCV/external.ll b/llvm/test/Transforms/SLPVectorizer/RISCV/external.ll
index efe2fdd25e2a0..5074d21246d27 100644
--- a/llvm/test/Transforms/SLPVectorizer/RISCV/external.ll
+++ b/llvm/test/Transforms/SLPVectorizer/RISCV/external.ll
@@ -6,8 +6,6 @@ define void @simple_copy(ptr %dest, ptr %p, ptr %dest2, ptr %dest3) {
; DEFAULT-LABEL: define void @simple_copy(
; DEFAULT-SAME: ptr [[DEST:%.*]], ptr [[P:%.*]], ptr [[DEST2:%.*]], ptr [[DEST3:%.*]]) #[[ATTR0:[0-9]+]] {
; DEFAULT-NEXT: [[ENTRY:.*:]]
-; DEFAULT-NEXT: [[INC7:%.*]] = getelementptr inbounds float, ptr [[P]], i64 7
-; DEFAULT-NEXT: [[TMP3:%.*]] = load float, ptr [[INC7]], align 2
; DEFAULT-NEXT: [[TMP0:%.*]] = load <8 x float>, ptr [[P]], align 4
; DEFAULT-NEXT: store <8 x float> [[TMP0]], ptr [[DEST]], align 4
; DEFAULT-NEXT: [[D4:%.*]] = getelementptr inbounds float, ptr [[DEST2]], i64 4
@@ -16,6 +14,7 @@ define void @simple_copy(ptr %dest, ptr %p, ptr %dest2, ptr %dest3) {
; DEFAULT-NEXT: [[D7:%.*]] = getelementptr inbounds float, ptr [[DEST2]], i64 7
; DEFAULT-NEXT: [[TMP1:%.*]] = shufflevector <8 x float> [[TMP0]], <8 x float> poison, <4 x i32> <i32 0, i32 1, i32 2, i32 3>
; DEFAULT-NEXT: store <4 x float> [[TMP1]], ptr [[DEST2]], align 4
+; DEFAULT-NEXT: [[TMP3:%.*]] = extractelement <8 x float> [[TMP0]], i32 7
; DEFAULT-NEXT: store float [[TMP3]], ptr [[D7]], align 2
; DEFAULT-NEXT: ret void
;
diff --git a/llvm/test/Transforms/SLPVectorizer/RISCV/runtime-strided-stores.ll b/llvm/test/Transforms/SLPVectorizer/RISCV/runtime-strided-stores.ll
index a6b4a48566aaa..0d5b1271bc86e 100644
--- a/llvm/test/Transforms/SLPVectorizer/RISCV/runtime-strided-stores.ll
+++ b/llvm/test/Transforms/SLPVectorizer/RISCV/runtime-strided-stores.ll
@@ -6,21 +6,7 @@ define void @runtime_stride(ptr %pl, ptr %ps, i64 %stride) {
; CHECK-LABEL: define void @runtime_stride(
; CHECK-SAME: ptr [[PL:%.*]], ptr [[PS:%.*]], i64 [[STRIDE:%.*]]) #[[ATTR0:[0-9]+]] {
; CHECK-NEXT: [[GEP_L0:%.*]] = getelementptr i8, ptr [[PL]], i64 0
-; CHECK-NEXT: [[GEP_L1:%.*]] = getelementptr i8, ptr [[PL]], i64 1
-; CHECK-NEXT: [[GEP_L2:%.*]] = getelementptr i8, ptr [[PL]], i64 2
-; CHECK-NEXT: [[GEP_L3:%.*]] = getelementptr i8, ptr [[PL]], i64 3
-; CHECK-NEXT: [[GEP_L4:%.*]] = getelementptr i8, ptr [[PL]], i64 4
-; CHECK-NEXT: [[GEP_L5:%.*]] = getelementptr i8, ptr [[PL]], i64 5
-; CHECK-NEXT: [[GEP_L6:%.*]] = getelementptr i8, ptr [[PL]], i64 6
-; CHECK-NEXT: [[GEP_L7:%.*]] = getelementptr i8, ptr [[PL]], i64 7
-; CHECK-NEXT: [[TMP9:%.*]] = load i8, ptr [[GEP_L7]], align 1
-; CHECK-NEXT: [[TMP8:%.*]] = load i8, ptr [[GEP_L6]], align 1
-; CHECK-NEXT: [[TMP7:%.*]] = load i8, ptr [[GEP_L5]], align 1
-; CHECK-NEXT: [[TMP6:%.*]] = load i8, ptr [[GEP_L4]], align 1
-; CHECK-NEXT: [[TMP5:%.*]] = load i8, ptr [[GEP_L3]], align 1
-; CHECK-NEXT: [[TMP4:%.*]] = load i8, ptr [[GEP_L2]], align 1
-; CHECK-NEXT: [[TMP3:%.*]] = load i8, ptr [[GEP_L1]], align 1
-; CHECK-NEXT: [[TMP2:%.*]] = load i8, ptr [[GEP_L0]], align 1
+; CHECK-NEXT: [[TMP1:%.*]] = load <8 x i8>, ptr [[GEP_L0]], align 1
; CHECK-NEXT: [[GEP_S0:%.*]] = getelementptr i8, ptr [[PS]], i64 0
; CHECK-NEXT: [[GEP_S1:%.*]] = getelementptr i8, ptr [[GEP_S0]], i64 [[STRIDE]]
; CHECK-NEXT: [[GEP_S2:%.*]] = getelementptr i8, ptr [[GEP_S1]], i64 [[STRIDE]]
@@ -29,13 +15,21 @@ define void @runtime_stride(ptr %pl, ptr %ps, i64 %stride) {
; CHECK-NEXT: [[GEP_S5:%.*]] = getelementptr i8, ptr [[GEP_S4]], i64 [[STRIDE]]
; CHECK-NEXT: [[GEP_S6:%.*]] = getelementptr i8, ptr [[GEP_S5]], i64 [[STRIDE]]
; CHECK-NEXT: [[GEP_S7:%.*]] = getelementptr i8, ptr [[GEP_S6]], i64 [[STRIDE]]
+; CHECK-NEXT: [[TMP2:%.*]] = extractelement <8 x i8> [[TMP1]], i32 0
; CHECK-NEXT: store i8 [[TMP2]], ptr [[GEP_S0]], align 1
+; CHECK-NEXT: [[TMP3:%.*]] = extractelement <8 x i8> [[TMP1]], i32 1
; CHECK-NEXT: store i8 [[TMP3]], ptr [[GEP_S1]], align 1
+; CHECK-NEXT: [[TMP4:%.*]] = extractelement <8 x i8> [[TMP1]], i32 2
; CHECK-NEXT: store i8 [[TMP4]], ptr [[GEP_S2]], align 1
+; CHECK-NEXT: [[TMP5:%.*]] = extractelement <8 x i8> [[TMP1]], i32 3
; CHECK-NEXT: store i8 [[TMP5]], ptr [[GEP_S3]], align 1
+; CHECK-NEXT: [[TMP6:%.*]] = extractelement <8 x i8> [[TMP1]], i32 4
; CHECK-NEXT: store i8 [[TMP6]], ptr [[GEP_S4]], align 1
+; CHECK-NEXT: [[TMP7:%.*]] = extractelement <8 x i8> [[TMP1]], i32 5
; CHECK-NEXT: store i8 [[TMP7]], ptr [[GEP_S5]], align 1
+; CHECK-NEXT: [[TMP8:%.*]] = extractelement <8 x i8> [[TMP1]], i32 6
; CHECK-NEXT: store i8 [[TMP8]], ptr [[GEP_S6]], align 1
+; CHECK-NEXT: [[TMP9:%.*]] = extractelement <8 x i8> [[TMP1]], i32 7
; CHECK-NEXT: store i8 [[TMP9]], ptr [[GEP_S7]], align 1
; CHECK-NEXT: ret void
;
@@ -82,21 +76,7 @@ define void @runtime_stride_complex_scev(ptr %pl, ptr %ps, i64 %stride, i64 %str
; CHECK-LABEL: define void @runtime_stride_complex_scev(
; CHECK-SAME: ptr [[PL:%.*]], ptr [[PS:%.*]], i64 [[STRIDE:%.*]], i64 [[STRIDE1:%.*]]) #[[ATTR0]] {
; CHECK-NEXT: [[GEP_L0:%.*]] = getelementptr i8, ptr [[PL]], i64 0
-; CHECK-NEXT: [[GEP_L1:%.*]] = getelementptr i8, ptr [[PL]], i64 1
-; CHECK-NEXT: [[GEP_L2:%.*]] = getelementptr i8, ptr [[PL]], i64 2
-; CHECK-NEXT: [[GEP_L3:%.*]] = getelementptr i8, ptr [[PL]], i64 3
-; CHECK-NEXT: [[GEP_L4:%.*]] = getelementptr i8, ptr [[PL]], i64 4
-; CHECK-NEXT: [[GEP_L5:%.*]] = getelementptr i8, ptr [[PL]], i64 5
-; CHECK-NEXT: [[GEP_L6:%.*]] = getelementptr i8, ptr [[PL]], i64 6
-; CHECK-NEXT: [[GEP_L7:%.*]] = getelementptr i8, ptr [[PL]], i64 7
-; CHECK-NEXT: [[TMP9:%.*]] = load i8, ptr [[GEP_L7]], align 1
-; CHECK-NEXT: [[TMP8:%.*]] = load i8, ptr [[GEP_L6]], align 1
-; CHECK-NEXT: [[TMP7:%.*]] = load i8, ptr [[GEP_L5]], align 1
-; CHECK-NEXT: [[TMP6:%.*]] = load i8, ptr [[GEP_L4]], align 1
-; CHECK-NEXT: [[TMP5:%.*]] = load i8, ptr [[GEP_L3]], align 1
-; CHECK-NEXT: [[TMP4:%.*]] = load i8, ptr [[GEP_L2]], align 1
-; CHECK-NEXT: [[TMP3:%.*]] = load i8, ptr [[GEP_L1]], align 1
-; CHECK-NEXT: [[TMP2:%.*]] = load i8, ptr [[GEP_L0]], align 1
+; CHECK-NEXT: [[TMP1:%.*]] = load <8 x i8>, ptr [[GEP_L0]], align 1
; CHECK-NEXT: [[GEP_S0:%.*]] = getelementptr i8, ptr [[PS]], i64 0
; CHECK-NEXT: [[GEP_S1:%.*]] = getelementptr i8, ptr [[GEP_S0]], i64 [[STRIDE]]
; CHECK-NEXT: [[GEP_S2:%.*]] = getelementptr i8, ptr [[GEP_S1]], i64 [[STRIDE]]
@@ -113,13 +93,21 @@ define void @runtime_stride_complex_scev(ptr %pl, ptr %ps, i64 %stride, i64 %str
; CHECK-NEXT: [[GEP_SS5:%.*]] = getelementptr i8, ptr [[GEP_S5]], i64 [[STRIDE1]]
; CHECK-NEXT: [[GEP_SS6:%.*]] = getelementptr i8, ptr [[GEP_S6]], i64 [[STRIDE1]]
; CHECK-NEXT: [[GEP_SS7:%.*]] = getelementptr i8, ptr [[GEP_S7]], i64 [[STRIDE1]]
+; CHECK-NEXT: [[TMP2:%.*]] = extractelement <8 x i8> [[TMP1]], i32 0
; CHECK-NEXT: store i8 [[TMP2]], ptr [[GEP_SS0]], align 1
+; CHECK-NEXT: [[TMP3:%.*]] = extractelement <8 x i8> [[TMP1]], i32 1
; CHECK-NEXT: store i8 [[TMP3]], ptr [[GEP_SS1]], align 1
+; CHECK-NEXT: [[TMP4:%.*]] = extractelement <8 x i8> [[TMP1]], i32 2
; CHECK-NEXT: store i8 [[TMP4]], ptr [[GEP_SS2]], align 1
+; CHECK-NEXT: [[TMP5:%.*]] = extractelement <8 x i8> [[TMP1]], i32 3
; CHECK-NEXT: store i8 [[TMP5]], ptr [[GEP_SS3]], align 1
+; CHECK-NEXT: [[TMP6:%.*]] = extractelement <8 x i8> [[TMP1]], i32 4
; CHECK-NEXT: store i8 [[TMP6]], ptr [[GEP_SS4]], align 1
+; CHECK-NEXT: [[TMP7:%.*]] = extractelement <8 x i8> [[TMP1]], i32 5
; CHECK-NEXT: store i8 [[TMP7]], ptr [[GEP_SS5]], align 1
+; CHECK-NEXT: [[TMP8:%.*]] = extractelement <8 x i8> [[TMP1]], i32 6
; CHECK-NEXT: store i8 [[TMP8]], ptr [[GEP_SS6]], align 1
+; CHECK-NEXT: [[TMP9:%.*]] = extractelement <8 x i8> [[TMP1]], i32 7
; CHECK-NEXT: store i8 [[TMP9]], ptr [[GEP_SS7]], align 1
; CHECK-NEXT: ret void
;
@@ -176,21 +164,7 @@ define void @two_runtime_strides(ptr %pl, ptr %ps, i64 %stride) {
; CHECK-LABEL: define void @two_runtime_strides(
; CHECK-SAME: ptr [[PL:%.*]], ptr [[PS:%.*]], i64 [[STRIDE:%.*]]) #[[ATTR0]] {
; CHECK-NEXT: [[GEP_L0:%.*]] = getelementptr i8, ptr [[PL]], i64 0
-; CHECK-NEXT: [[GEP_L1:%.*]] = getelementptr i8, ptr [[PL]], i64 1
-; CHECK-NEXT: [[GEP_L2:%.*]] = getelementptr i8, ptr [[PL]], i64 2
-; CHECK-NEXT: [[GEP_L3:%.*]] = getelementptr i8, ptr [[PL]], i64 3
-; CHECK-NEXT: [[GEP_L4:%.*]] = getelementptr i8, ptr [[PL]], i64 4
-; CHECK-NEXT: [[GEP_L5:%.*]] = getelementptr i8, ptr [[PL]], i64 5
-; CHECK-NEXT: [[GEP_L6:%.*]] = getelementptr i8, ptr [[PL]], i64 6
-; CHECK-NEXT: [[GEP_L7:%.*]] = getelementptr i8, ptr [[PL]], i64 7
-; CHECK-NEXT: [[TMP9:%.*]] = load i8, ptr [[GEP_L7]], align 1
-; CHECK-NEXT: [[TMP8:%.*]] = load i8, ptr [[GEP_L6]], align 1
-; CHECK-NEXT: [[TMP7:%.*]] = load i8, ptr [[GEP_L5]], align 1
-; CHECK-NEXT: [[TMP6:%.*]] = load i8, ptr [[GEP_L4]], align 1
-; CHECK-NEXT: [[TMP5:%.*]] = load i8, ptr [[GEP_L3]], align 1
-; CHECK-NEXT: [[TMP4:%.*]] = load i8, ptr [[GEP_L2]], align 1
-; CHECK-NEXT: [[TMP3:%.*]] = load i8, ptr [[GEP_L1]], align 1
-; CHECK-NEXT: [[TMP2:%.*]] = load i8, ptr [[GEP_L0]], align 1
+; CHECK-NEXT: [[TMP1:%.*]] = load <8 x i8>, ptr [[GEP_L0]], align 1
; CHECK-NEXT: [[PS1:%.*]] = mul i64 [[STRIDE]], 16
; CHECK-NEXT: [[GEP_S0:%.*]] = getelementptr i8, ptr [[PS]], i64 0
; CHECK-NEXT: [[GEP_S1:%.*]] = getelementptr i8, ptr [[GEP_S0]], i64 [[STRIDE]]
@@ -200,13 +174,21 @@ define void @two_runtime_strides(ptr %pl, ptr %ps, i64 %stride) {
; CHECK-NEXT: [[GEP_S5:%.*]] = getelementptr i8, ptr [[GEP_S4]], i64 [[STRIDE]]
; CHECK-NEXT: [[GEP_S6:%.*]] = getelementptr i8, ptr [[GEP_S5]], i64 [[STRIDE]]
; CHECK-NEXT: [[GEP_S7:%.*]] = getelementptr i8, ptr [[GEP_S6]], i64 [[STRIDE]]
+; CHECK-NEXT: [[TMP2:%.*]] = extractelement <8 x i8> [[TMP1]], i32 0
; CHECK-NEXT: store i8 [[TMP2]], ptr [[GEP_S0]], align 1
+; CHECK-NEXT: [[TMP3:%.*]] = extractelement <8 x i8> [[TMP1]], i32 1
; CHECK-NEXT: store i8 [[TMP3]], ptr [[GEP_S1]], align 1
+; CHECK-NEXT: [[TMP4:%.*]] = extractelement <8 x i8> [[TMP1]], i32 2
; CHECK-NEXT: store i8 [[TMP4]], ptr [[GEP_S2]], align 1
+; CHECK-NEXT: [[TMP5:%.*]] = extractelement <8 x i8> [[TMP1]], i32 3
; CHECK-NEXT: store i8 [[TMP5]], ptr [[GEP_S3]], align 1
+; CHECK-NEXT: [[TMP6:%.*]] = extractelement <8 x i8> [[TMP1]], i32 4
; CHECK-NEXT: store i8 [[TMP6]], ptr [[GEP_S4]], align 1
+; CHECK-NEXT: [[TMP7:%.*]] = extractelement <8 x i8> [[TMP1]], i32 5
; CHECK-NEXT: store i8 [[TMP7]], ptr [[GEP_S5]], align 1
+; CHECK-NEXT: [[TMP8:%.*]] = extractelement <8 x i8> [[TMP1]], i32 6
; CHECK-NEXT: store i8 [[TMP8]], ptr [[GEP_S6]], align 1
+; CHECK-NEXT: [[TMP9:%.*]] = extractelement <8 x i8> [[TMP1]], i32 7
; CHECK-NEXT: store i8 [[TMP9]], ptr [[GEP_S7]], align 1
; CHECK-NEXT: ret void
;
@@ -254,21 +236,7 @@ define void @runtime_strides_constant_gap(ptr %pl, ptr %ps, i64 %stride) {
; CHECK-LABEL: define void @runtime_strides_constant_gap(
; CHECK-SAME: ptr [[PL:%.*]], ptr [[PS:%.*]], i64 [[STRIDE:%.*]]) #[[ATTR0]] {
; CHECK-NEXT: [[GEP_L0:%.*]] = getelementptr i8, ptr [[PL]], i64 0
-; CHECK-NEXT: [[GEP_L1:%.*]] = getelementptr i8, ptr [[PL]], i64 1
-; CHECK-NEXT: [[GEP_L2:%.*]] = getelementptr i8, ptr [[PL]], i64 2
-; CHECK-NEXT: [[GEP_L3:%.*]] = getelementptr i8, ptr [[PL]], i64 3
-; CHECK-NEXT: [[GEP_L4:%.*]] = getelementptr i8, ptr [[PL]], i64 4
-; CHECK-NEXT: [[GEP_L5:%.*]] = getelementptr i8, ptr [[PL]], i64 5
-; CHECK-NEXT: [[GEP_L6:%.*]] = getelementptr i8, ptr [[PL]], i64 6
-; CHECK-NEXT: [[GEP_L7:%.*]] = getelementptr i8, ptr [[PL]], i64 7
-; CHECK-NEXT: [[TMP9:%.*]] = load i8, ptr [[GEP_L7]], align 1
-; CHECK-NEXT: [[TMP8:%.*]] = load i8, ptr [[GEP_L6]], align 1
-; CHECK-NEXT: [[TMP7:%.*]] = load i8, ptr [[GEP_L5]], align 1
-; CHECK-NEXT: [[TMP6:%.*]] = load i8, ptr [[GEP_L4]], align 1
-; CHECK-NEXT: [[TMP5:%.*]] = load i8, ptr [[GEP_L3]], align 1
-; CHECK-NEXT: [[TMP4:%.*]] = load i8, ptr [[GEP_L2]], align 1
-; CHECK-NEXT: [[TMP3:%.*]] = load i8, ptr [[GEP_L1]], align 1
-; CHECK-NEXT: [[TMP2:%.*]] = load i8, ptr [[GEP_L0]], align 1
+; CHECK-NEXT: [[TMP1:%.*]] = load <8 x i8>, ptr [[GEP_L0]], align 1
; CHECK-NEXT: [[GEP_S0:%.*]] = getelementptr i8, ptr [[PS]], i64 0
; CHECK-NEXT: [[GEP_S1:%.*]] = getelementptr i8, ptr [[GEP_S0]], i64 [[STRIDE]]
; CHECK-NEXT: [[GEP_S2:%.*]] = getelementptr i8, ptr [[GEP_S1]], i64 [[STRIDE]]
@@ -277,13 +245,21 @@ define void @runtime_strides_constant_gap(ptr %pl, ptr %ps, i64 %stride) {
; CHECK-NEXT: [[GEP_S5:%.*]] = getelementptr i8, ptr [[GEP_S4]], i64 [[STRIDE]]
; CHECK-NEXT: [[GEP_S6:%.*]] = getelementptr i8, ptr [[GEP_S5]], i64 [[STRIDE]]
; CHECK-NEXT: [[GEP_S7:%.*]] = getelementptr i8, ptr [[GEP_S6]], i64 [[STRIDE]]
+; CHECK-NEXT: [[TMP2:%.*]] = extractelement <8 x i8> [[TMP1]], i32 0
; CHECK-NEXT: store i8 [[TMP2]], ptr [[GEP_S0]], align 1
+; CHECK-NEXT: [[TMP3:%.*]] = extractelement <8 x i8> [[TMP1]], i32 1
; CHECK-NEXT: store i8 [[TMP3]], ptr [[GEP_S1]], align 1
+; CHECK-NEXT: [[TMP4:%.*]] = extractelement <8 x i8> [[TMP1]], i32 2
; CHECK-NEXT: store i8 [[TMP4]], ptr [[GEP_S2]], align 1
+; CHECK-NEXT: [[TMP5:%.*]] = extractelement <8 x i8> [[TMP1]], i32 3
; CHECK-NEXT: store i8 [[TMP5]], ptr [[GEP_S3]], align 1
+; CHECK-NEXT: [[TMP6:%.*]] = extractelement <8 x i8> [[TMP1]], i32 4
; CHECK-NEXT: store i8 [[TMP6]], ptr [[GEP_S4]], align 1
+; CHECK-NEXT: [[TMP7:%.*]] = extractelement <8 x i8> [[TMP1]], i32 5
; CHECK-NEXT: store i8 [[TMP7]], ptr [[GEP_S5]], align 1
+; CHECK-NEXT: [[TMP8:%.*]] = extractelement <8 x i8> [[TMP1]], i32 6
; CHECK-NEXT: store i8 [[TMP8]], ptr [[GEP_S6]], align 1
+; CHECK-NEXT: [[TMP9:%.*]] = extractelement <8 x i8> [[TMP1]], i32 7
; CHECK-NEXT: store i8 [[TMP9]], ptr [[GEP_S7]], align 1
; CHECK-NEXT: ret void
;
@@ -330,21 +306,7 @@ define void @overlapping_strides(ptr %pl, ptr %ps, i64 %stride) {
; CHECK-LABEL: define void @overlapping_strides(
; CHECK-SAME: ptr [[PL:%.*]], ptr [[PS:%.*]], i64 [[STRIDE:%.*]]) #[[ATTR0]] {
; CHECK-NEXT: [[GEP_L0:%.*]] = getelementptr i8, ptr [[PL]], i64 0
-; CHECK-NEXT: [[GEP_L1:%.*]] = getelementptr i8, ptr [[PL]], i64 1
-; CHECK-NEXT: [[GEP_L2:%.*]] = getelementptr i8, ptr [[PL]], i64 2
-; CHECK-NEXT: [[GEP_L3:%.*]] = getelementptr i8, ptr [[PL]], i64 3
-; CHECK-NEXT: [[GEP_L4:%.*]] = getelementptr i8, ptr [[PL]], i64 4
-; CHECK-NEXT: [[GEP_L5:%.*]] = getelementptr i8, ptr [[PL]], i64 5
-; CHECK-NEXT: [[GEP_L6:%.*]] = getelementptr i8, ptr [[PL]], i64 6
-; CHECK-NEXT: [[GEP_L7:%.*]] = getelementptr i8, ptr [[PL]], i64 7
-; CHECK-NEXT: [[TMP9:%.*]] = load i8, ptr [[GEP_L7]], align 1
-; CHECK-NEXT: [[TMP8:%.*]] = load i8, ptr [[GEP_L6]], align 1
-; CHECK-NEXT: [[TMP7:%.*]] = load i8, ptr [[GEP_L5]], align 1
-; CHECK-NEXT: [[TMP6:%.*]] = load i8, ptr [[GEP_L4]], align 1
-; CHECK-NEXT: [[TMP5:%.*]] = load i8, ptr [[GEP_L3]], align 1
-; CHECK-NEXT: [[TMP4:%.*]] = load i8, ptr [[GEP_L2]], align 1
-; CHECK-NEXT: [[TMP3:%.*]] = load i8, ptr [[GEP_L1]], align 1
-; CHECK-NEXT: [[TMP2:%.*]] = load i8, ptr [[GEP_L0]], align 1
+; CHECK-NEXT: [[TMP1:%.*]] = load <8 x i8>, ptr [[GEP_L0]], align 1
; CHECK-NEXT: [[STRIDE2:%.*]] = mul i64 [[STRIDE]], 1
; CHECK-NEXT: [[STRIDE1:%.*]] = mul i64 [[STRIDE]], 2
; CHECK-NEXT: [[GEP_S0:%.*]] = getelementptr i8, ptr [[PS]], i64 0
@@ -355,13 +317,21 @@ define void @overlapping_strides(ptr %pl, ptr %ps, i64 %stride) {
; CHECK-NEXT: [[GEP_S5:%.*]] = getelementptr i8, ptr [[GEP_S4]], i64 [[STRIDE2]]
; CHECK-NEXT: [[GEP_S6:%.*]] = getelementptr i8, ptr [[GEP_S5]], i64 [[STRIDE2]]
; CHECK-NEXT: [[GEP_S7:%.*]] = getelementptr i8, ptr [[GEP_S6]], i64 [[STRIDE2]]
+; CHECK-NEXT: [[TMP2:%.*]] = extractelement <8 x i8> [[TMP1]], i32 0
; CHECK-NEXT: store i8 [[TMP2]], ptr [[GEP_S0]], align 1
+; CHECK-NEXT: [[TMP3:%.*]] = extractelement <8 x i8> [[TMP1]], i32 1
; CHECK-NEXT: store i8 [[TMP3]], ptr [[GEP_S1]], align 1
+; CHECK-NEXT: [[TMP4:%.*]] = extractelement <8 x i8> [[TMP1]], i32 2
; CHECK-NEXT: store i8 [[TMP4]], ptr [[GEP_S2]], align 1
+; CHECK-NEXT: [[TMP5:%.*]] = extractelement <8 x i8> [[TMP1]], i32 3
; CHECK-NEXT: store i8 [[TMP5]], ptr [[GEP_S3]], align 1
+; CHECK-NEXT: [[TMP6:%.*]] = extractelement <8 x i8> [[TMP1]], i32 4
; CHECK-NEXT: store i8 [[TMP6]], ptr [[GEP_S4]], align 1
+; CHECK-NEXT: [[TMP7:%.*]] = extractelement <8 x i8> [[TMP1]], i32 5
; CHECK-NEXT: store i8 [[TMP7]], ptr [[GEP_S5]], align 1
+; CHECK-NEXT: [[TMP8:%.*]] = extractelement <8 x i8> [[TMP1]], i32 6
; CHECK-NEXT: store i8 [[TMP8]], ptr [[GEP_S6]], align 1
+; CHECK-NEXT: [[TMP9:%.*]] = extractelement <8 x i8> [[TMP1]], i32 7
; CHECK-NEXT: store i8 [[TMP9]], ptr [[GEP_S7]], align 1
; CHECK-NEXT: ret void
;
@@ -410,14 +380,9 @@ define void @runtime_stride_unit_stride(ptr %pl, ptr %ps, i64 %stride) {
; CHECK-LABEL: define void @runtime_stride_unit_stride(
; CHECK-SAME: ptr [[PL:%.*]], ptr [[PS:%.*]], i64 [[STRIDE:%.*]]) #[[ATTR0]] {
; CHECK-NEXT: [[GEP_L0:%.*]] = getelementptr i8, ptr [[PL]], i64 0
-; CHECK-NEXT: [[GEP_L1:%.*]] = getelementptr i8, ptr [[PL]], i64 1
-; CHECK-NEXT: [[GEP_L2:%.*]] = getelementptr i8, ptr [[PL]], i64 2
; CHECK-NEXT: [[GEP_L3:%.*]] = getelementptr i8, ptr [[PL]], i64 3
-; CHECK-NEXT: [[GEP_L7:%.*]] = getelementptr i8, ptr [[PL]], i64 7
-; CHECK-NEXT: [[LOAD7:%.*]] = load i8, ptr [[GEP_L7]], align 1
-; CHECK-NEXT: [[LOAD2:%.*]] = load i8, ptr [[GEP_L2]], align 1
-; CHECK-NEXT: [[TMP5:%.*]] = load i8, ptr [[GEP_L1]], align 1
-; CHECK-NEXT: [[TMP4:%.*]] = load i8, ptr [[GEP_L0]], align 1
+; CHECK-NEXT: [[TMP3:%.*]] = call <8 x i8> @llvm.masked.load.v8i8.p0(ptr align 1 [[GEP_L0]], <8 x i1> <i1 true, i1 true, i1 true, i1 false, i1 false, i1 false, i1 false, i1 true>, <8 x i8> poison)
+; CHECK-NEXT: [[TMP2:%.*]] = shufflevector <8 x i8> [[TMP3]], <8 x i8> poison, <4 x i32> <i32 0, i32 1, i32 2, i32 7>
; CHECK-NEXT: [[GEP_S0:%.*]] = getelementptr i8, ptr [[PS]], i64 0
; CHECK-NEXT: [[GEP_S1:%.*]] = getelementptr i8, ptr [[GEP_S0]], i64 [[STRIDE]]
; CHECK-NEXT: [[GEP_S2:%.*]] = getelementptr i8, ptr [[GEP_S1]], i64 [[STRIDE]]
@@ -427,10 +392,14 @@ define void @runtime_stride_unit_stride(ptr %pl, ptr %ps, i64 %stride) {
; CHECK-NEXT: [[GEP_S6:%.*]] = getelementptr i8, ptr [[GEP_S5]], i64 1
; CHECK-NEXT: [[GEP_S7:%.*]] = getelementptr i8, ptr [[GEP_S6]], i64 1
; CHECK-NEXT: [[TMP1:%.*]] = load <4 x i8>, ptr [[GEP_L3]], align 1
+; CHECK-NEXT: [[TMP4:%.*]] = extractelement <4 x i8> [[TMP2]], i32 0
; CHECK-NEXT: store i8 [[TMP4]], ptr [[GEP_S0]], align 1
+; CHECK-NEXT: [[TMP5:%.*]] = extractelement <4 x i8> [[TMP2]], i32 1
; CHECK-NEXT: store i8 [[TMP5]], ptr [[GEP_S1]], align 1
+; CHECK-NEXT: [[LOAD2:%.*]] = extractelement <4 x i8> [[TMP2]], i32 2
; CHECK-NEXT: store i8 [[LOAD2]], ptr [[GEP_S2]], align 1
; CHECK-NEXT: store <4 x i8> [[TMP1]], ptr [[GEP_S3]], align 1
+; CHECK-NEXT: [[LOAD7:%.*]] = extractelement <4 x i8> [[TMP2]], i32 3
; CHECK-NEXT: store i8 [[LOAD7]], ptr [[GEP_S7]], align 1
; CHECK-NEXT: ret void
;
@@ -478,13 +447,7 @@ define void @unit_stride_runtime_stride(ptr %pl, ptr %ps, i64 %stride) {
; CHECK-SAME: ptr [[PL:%.*]], ptr [[PS:%.*]], i64 [[STRIDE:%.*]]) #[[ATTR0]] {
; CHECK-NEXT: [[GEP_L0:%.*]] = getelementptr i8, ptr [[PL]], i64 0
; CHECK-NEXT: [[GEP_L4:%.*]] = getelementptr i8, ptr [[PL]], i64 4
-; CHECK-NEXT: [[GEP_L5:%.*]] = getelementptr i8, ptr [[PL]], i64 5
-; CHECK-NEXT: [[GEP_L6:%.*]] = getelementptr i8, ptr [[PL]], i64 6
-; CHECK-NEXT: [[GEP_L7:%.*]] = getelementptr i8, ptr [[PL]], i64 7
-; CHECK-NEXT: [[TMP6:%.*]] = load i8, ptr [[GEP_L7]], align 1
-; CHECK-NEXT: [[TMP5:%.*]] = load i8, ptr [[GEP_L6]], align 1
-; CHECK-NEXT: [[TMP4:%.*]] = load i8, ptr [[GEP_L5]], align 1
-; CHECK-NEXT: [[TMP3:%.*]] = load i8, ptr [[GEP_L4]], align 1
+; CHECK-NEXT: [[TMP2:%.*]] = load <4 x i8>, ptr [[GEP_L4]], align 1
; CHECK-NEXT: [[GEP_S0:%.*]] = getelementptr i8, ptr [[PS]], i64 0
; CHECK-NEXT: [[GEP_S1:%.*]] = getelementptr i8, ptr [[GEP_S0]], i64 1
; CHECK-NEXT: [[GEP_S2:%.*]] = getelementptr i8, ptr [[GEP_S1]], i64 1
@@ -495,9 +458,13 @@ define void @unit_stride_runtime_stride(ptr %pl, ptr %ps, i64 %stride) {
; CHECK-NEXT: [[GEP_S7:%.*]] = getelementptr i8, ptr [[GEP_S6]], i64 [[STRIDE]]
; CHECK-NEXT: [[TMP1:%.*]] = load <4 x i8>, ptr [[GEP_L0]], align 1
; CHECK-NEXT: store <4 x i8> [[TMP1]], ptr [[GEP_S0]], align 1
+; CHECK-NEXT: [[TMP3:%.*]] = extractelement <4 x i8> [[TMP2]], i32 0
; CHECK-NEXT: store i8 [[TMP3]], ptr [[GEP_S4]], align 1
+; CHECK-NEXT: [[TMP4:%.*]] = extractelement <4 x i8> [[TMP2]], i32 1
; CHECK-NEXT: store i8 [[TMP4]], ptr [[GEP_S5]], align 1
+; CHECK-NEXT: [[TMP5:%.*]] = extractelement <4 x i8> [[TMP2]], i32 2
; CHECK-NEXT: store i8 [[TMP5]], ptr [[GEP_S6]], align 1
+; CHECK-NEXT: [[TMP6:%.*]] = extractelement <4 x i8> [[TMP2]], i32 3
; CHECK-NEXT: store i8 [[TMP6]], ptr [[GEP_S7]], align 1
; CHECK-NEXT: ret void
;
@@ -544,23 +511,21 @@ define void @runtime_stride_constant_stride(ptr %pl, ptr %ps, i64 %stride) {
; CHECK-LABEL: define void @runtime_stride_constant_stride(
; CHECK-SAME: ptr [[PL:%.*]], ptr [[PS:%.*]], i64 [[STRIDE:%.*]]) #[[ATTR0]] {
; CHECK-NEXT: [[GEP_L0:%.*]] = getelementptr i8, ptr [[PL]], i64 0
-; CHECK-NEXT: [[GEP_L1:%.*]] = getelementptr i8, ptr [[PL]], i64 1
-; CHECK-NEXT: [[GEP_L2:%.*]] = getelementptr i8, ptr [[PL]], i64 2
-; CHECK-NEXT: [[GEP_L3:%.*]] = getelementptr i8, ptr [[PL]], i64 3
; CHECK-NEXT: [[GEP_L4:%.*]] = getelementptr i8, ptr [[PL]], i64 4
-; CHECK-NEXT: [[TMP6:%.*]] = load i8, ptr [[GEP_L3]], align 1
-; CHECK-NEXT: [[TMP5:%.*]] = load i8, ptr [[GEP_L2]], align 1
-; CHECK-NEXT: [[TMP4:%.*]] = load i8, ptr [[GEP_L1]], align 1
-; CHECK-NEXT: [[TMP7:%.*]] = load i8, ptr [[GEP_L0]], align 1
+; CHECK-NEXT: [[TMP1:%.*]] = load <4 x i8>, ptr [[GEP_L0]], align 1
; CHECK-NEXT: [[GEP_S0:%.*]] = getelementptr i8, ptr [[PS]], i64 0
; CHECK-NEXT: [[GEP_S1:%.*]] = getelementptr i8, ptr [[GEP_S0]], i64 [[STRIDE]]
; CHECK-NEXT: [[GEP_S2:%.*]] = getelementptr i8, ptr [[GEP_S1]], i64 [[STRIDE]]
; CHECK-NEXT: [[GEP_S3:%.*]] = getelementptr i8, ptr [[GEP_S2]], i64 [[STRIDE]]
; CHECK-NEXT: [[GEP_S4:%.*]] = getelementptr i8, ptr [[GEP_S3]], i64 1
; CHECK-NEXT: [[TMP3:%.*]] = load <4 x i8>, ptr [[GEP_L4]], align 1
+; CHECK-NEXT: [[TMP7:%.*]] = extractelement <4 x i8> [[TMP1]], i32 0
; CHECK-NEXT: store i8 [[TMP7]], ptr [[GEP_S0]], align 1
+; CHECK-NEXT: [[TMP4:%.*]] = extractelement <4 x i8> [[TMP1]], i32 1
; CHECK-NEXT: store i8 [[TMP4]], ptr [[GEP_S1]], align 1
+; CHECK-NEXT: [[TMP5:%.*]] = extractelement <4 x i8> [[TMP1]], i32 2
; CHECK-NEXT: store i8 [[TMP5]], ptr [[GEP_S2]], align 1
+; CHECK-NEXT: [[TMP6:%.*]] = extractelement <4 x i8> [[TMP1]], i32 3
; CHECK-NEXT: store i8 [[TMP6]], ptr [[GEP_S3]], align 1
; CHECK-NEXT: call void @llvm.experimental.vp.strided.store.v4i8.p0.i64(<4 x i8> [[TMP3]], ptr align 1 [[GEP_S4]], i64 2, <4 x i1> splat (i1 true), i32 4)
; CHECK-NEXT: ret void
@@ -609,13 +574,7 @@ define void @constant_stride_runtime_stride(ptr %pl, ptr %ps, i64 %stride) {
; CHECK-SAME: ptr [[PL:%.*]], ptr [[PS:%.*]], i64 [[STRIDE:%.*]]) #[[ATTR0]] {
; CHECK-NEXT: [[GEP_L0:%.*]] = getelementptr i8, ptr [[PL]], i64 0
; CHECK-NEXT: [[GEP_L4:%.*]] = getelementptr i8, ptr [[PL]], i64 4
-; CHECK-NEXT: [[GEP_L5:%.*]] = getelementptr i8, ptr [[PL]], i64 5
-; CHECK-NEXT: [[GEP_L6:%.*]] = getelementptr i8, ptr [[PL]], i64 6
-; CHECK-NEXT: [[GEP_L7:%.*]] = getelementptr i8, ptr [[PL]], i64 7
-; CHECK-NEXT: [[TMP6:%.*]] = load i8, ptr [[GEP_L7]], align 1
-; CHECK-NEXT: [[TMP5:%.*]] = load i8, ptr [[GEP_L6]], align 1
-; CHECK-NEXT: [[TMP4:%.*]] = load i8, ptr [[GEP_L5]], align 1
-; CHECK-NEXT: [[TMP3:%.*]] = load i8, ptr [[GEP_L4]], align 1
+; CHECK-NEXT: [[TMP2:%.*]] = load <4 x i8>, ptr [[GEP_L4]], align 1
; CHECK-NEXT: [[GEP_S0:%.*]] = getelementptr i8, ptr [[PS]], i64 0
; CHECK-NEXT: [[GEP_S1:%.*]] = getelementptr i8, ptr [[GEP_S0]], i64 2
; CHECK-NEXT: [[GEP_S2:%.*]] = getelementptr i8, ptr [[GEP_S1]], i64 2
@@ -626,9 +585,13 @@ define void @constant_stride_runtime_stride(ptr %pl, ptr %ps, i64 %stride) {
; CHECK-NEXT: [[GEP_S7:%.*]] = getelementptr i8, ptr [[GEP_S6]], i64 [[STRIDE]]
; CHECK-NEXT: [[TMP1:%.*]] = load <4 x i8>, ptr [[GEP_L0]], align 1
; CHECK-NEXT: call void @llvm.experimental.vp.strided.store.v4i8.p0.i64(<4 x i8> [[TMP1]], ptr align 1 [[GEP_S0]], i64 2, <4 x i1> splat (i1 true), i32 4)
+; CHECK-NEXT: [[TMP3:%.*]] = extractelement <4 x i8> [[TMP2]], i32 0
; CHECK-NEXT: store i8 [[TMP3]], ptr [[GEP_S4]], align 1
+; CHECK-NEXT: [[TMP4:%.*]] = extractelement <4 x i8> [[TMP2]], i32 1
; CHECK-NEXT: store i8 [[TMP4]], ptr [[GEP_S5]], align 1
+; CHECK-NEXT: [[TMP5:%.*]] = extractelement <4 x i8> [[TMP2]], i32 2
; CHECK-NEXT: store i8 [[TMP5]], ptr [[GEP_S6]], align 1
+; CHECK-NEXT: [[TMP6:%.*]] = extractelement <4 x i8> [[TMP2]], i32 3
; CHECK-NEXT: store i8 [[TMP6]], ptr [[GEP_S7]], align 1
; CHECK-NEXT: ret void
;
@@ -675,18 +638,18 @@ define void @overlap(ptr %pl, ptr %ps, i64 %stride) {
; CHECK-LABEL: define void @overlap(
; CHECK-SAME: ptr [[PL:%.*]], ptr [[PS:%.*]], i64 [[STRIDE:%.*]]) #[[ATTR0]] {
; CHECK-NEXT: [[GEP_L0:%.*]] = getelementptr i8, ptr [[PL]], i64 0
-; CHECK-NEXT: [[GEP_L1:%.*]] = getelementptr i8, ptr [[PL]], i64 1
; CHECK-NEXT: [[GEP_L2:%.*]] = getelementptr i8, ptr [[PL]], i64 2
; CHECK-NEXT: [[GEP_L3:%.*]] = getelementptr i8, ptr [[PL]], i64 3
-; CHECK-NEXT: [[TMP4:%.*]] = load i8, ptr [[GEP_L1]], align 1
-; CHECK-NEXT: [[TMP3:%.*]] = load i8, ptr [[GEP_L0]], align 1
+; CHECK-NEXT: [[TMP2:%.*]] = load <2 x i8>, ptr [[GEP_L0]], align 1
; CHECK-NEXT: [[LOAD2:%.*]] = load i8, ptr [[GEP_L2]], align 1
; CHECK-NEXT: [[GEP_S0:%.*]] = getelementptr i8, ptr [[PS]], i64 0
; CHECK-NEXT: [[GEP_S1:%.*]] = getelementptr i8, ptr [[GEP_S0]], i64 [[STRIDE]]
; CHECK-NEXT: [[GEP_S2:%.*]] = getelementptr i8, ptr [[GEP_S1]], i64 [[STRIDE]]
; CHECK-NEXT: [[GEP_S3:%.*]] = getelementptr i8, ptr [[GEP_S2]], i64 [[STRIDE]]
; CHECK-NEXT: [[TMP1:%.*]] = load <4 x i8>, ptr [[GEP_L3]], align 1
+; CHECK-NEXT: [[TMP3:%.*]] = extractelement <2 x i8> [[TMP2]], i32 0
; CHECK-NEXT: store i8 [[TMP3]], ptr [[GEP_S0]], align 1
+; CHECK-NEXT: [[TMP4:%.*]] = extractelement <2 x i8> [[TMP2]], i32 1
; CHECK-NEXT: store i8 [[TMP4]], ptr [[GEP_S1]], align 1
; CHECK-NEXT: store i8 [[LOAD2]], ptr [[GEP_S2]], align 1
; CHECK-NEXT: store <4 x i8> [[TMP1]], ptr [[GEP_S3]], align 1
@@ -731,18 +694,18 @@ define void @overlap_constant_stride(ptr %pl, ptr %ps, i64 %stride) {
; CHECK-LABEL: define void @overlap_constant_stride(
; CHECK-SAME: ptr [[PL:%.*]], ptr [[PS:%.*]], i64 [[STRIDE:%.*]]) #[[ATTR0]] {
; CHECK-NEXT: [[GEP_L0:%.*]] = getelementptr i8, ptr [[PL]], i64 0
-; CHECK-NEXT: [[GEP_L1:%.*]] = getelementptr i8, ptr [[PL]], i64 1
; CHECK-NEXT: [[GEP_L2:%.*]] = getelementptr i8, ptr [[PL]], i64 2
; CHECK-NEXT: [[GEP_L3:%.*]] = getelementptr i8, ptr [[PL]], i64 3
-; CHECK-NEXT: [[TMP4:%.*]] = load i8, ptr [[GEP_L1]], align 1
-; CHECK-NEXT: [[TMP3:%.*]] = load i8, ptr [[GEP_L0]], align 1
+; CHECK-NEXT: [[TMP2:%.*]] = load <2 x i8>, ptr [[GEP_L0]], align 1
; CHECK-NEXT: [[LOAD2:%.*]] = load i8, ptr [[GEP_L2]], align 1
; CHECK-NEXT: [[GEP_S0:%.*]] = getelementptr i8, ptr [[PS]], i64 0
; CHECK-NEXT: [[GEP_S1:%.*]] = getelementptr i8, ptr [[GEP_S0]], i64 [[STRIDE]]
; CHECK-NEXT: [[GEP_S2:%.*]] = getelementptr i8, ptr [[GEP_S1]], i64 [[STRIDE]]
; CHECK-NEXT: [[GEP_S3:%.*]] = getelementptr i8, ptr [[GEP_S2]], i64 [[STRIDE]]
; CHECK-NEXT: [[TMP1:%.*]] = load <4 x i8>, ptr [[GEP_L3]], align 1
+; CHECK-NEXT: [[TMP3:%.*]] = extractelement <2 x i8> [[TMP2]], i32 0
; CHECK-NEXT: store i8 [[TMP3]], ptr [[GEP_S0]], align 1
+; CHECK-NEXT: [[TMP4:%.*]] = extractelement <2 x i8> [[TMP2]], i32 1
; CHECK-NEXT: store i8 [[TMP4]], ptr [[GEP_S1]], align 1
; CHECK-NEXT: store i8 [[LOAD2]], ptr [[GEP_S2]], align 1
; CHECK-NEXT: call void @llvm.experimental.vp.strided.store.v4i8.p0.i64(<4 x i8> [[TMP1]], ptr align 1 [[GEP_S3]], i64 2, <4 x i1> splat (i1 true), i32 4)
@@ -787,21 +750,7 @@ define void @runtime_stride_reorder(ptr %pl, ptr %ps, i64 %stride) {
; CHECK-LABEL: define void @runtime_stride_reorder(
; CHECK-SAME: ptr [[PL:%.*]], ptr [[PS:%.*]], i64 [[STRIDE:%.*]]) #[[ATTR0]] {
; CHECK-NEXT: [[GEP_L0:%.*]] = getelementptr i8, ptr [[PL]], i64 0
-; CHECK-NEXT: [[GEP_L1:%.*]] = getelementptr i8, ptr [[PL]], i64 2
-; CHECK-NEXT: [[GEP_L2:%.*]] = getelementptr i8, ptr [[PL]], i64 1
-; CHECK-NEXT: [[GEP_L3:%.*]] = getelementptr i8, ptr [[PL]], i64 4
-; CHECK-NEXT: [[GEP_L4:%.*]] = getelementptr i8, ptr [[PL]], i64 7
-; CHECK-NEXT: [[GEP_L5:%.*]] = getelementptr i8, ptr [[PL]], i64 5
-; CHECK-NEXT: [[GEP_L6:%.*]] = getelementptr i8, ptr [[PL]], i64 6
-; CHECK-NEXT: [[GEP_L7:%.*]] = getelementptr i8, ptr [[PL]], i64 3
-; CHECK-NEXT: [[TMP9:%.*]] = load i8, ptr [[GEP_L7]], align 1
-; CHECK-NEXT: [[TMP8:%.*]] = load i8, ptr [[GEP_L6]], align 1
-; CHECK-NEXT: [[TMP7:%.*]] = load i8, ptr [[GEP_L5]], align 1
-; CHECK-NEXT: [[TMP6:%.*]] = load i8, ptr [[GEP_L4]], align 1
-; CHECK-NEXT: [[TMP5:%.*]] = load i8, ptr [[GEP_L3]], align 1
-; CHECK-NEXT: [[TMP4:%.*]] = load i8, ptr [[GEP_L2]], align 1
-; CHECK-NEXT: [[TMP3:%.*]] = load i8, ptr [[GEP_L1]], align 1
-; CHECK-NEXT: [[TMP2:%.*]] = load i8, ptr [[GEP_L0]], align 1
+; CHECK-NEXT: [[TMP1:%.*]] = load <8 x i8>, ptr [[GEP_L0]], align 1
; CHECK-NEXT: [[GEP_S0:%.*]] = getelementptr i8, ptr [[PS]], i64 0
; CHECK-NEXT: [[GEP_S1:%.*]] = getelementptr i8, ptr [[GEP_S0]], i64 [[STRIDE]]
; CHECK-NEXT: [[GEP_S2:%.*]] = getelementptr i8, ptr [[GEP_S1]], i64 [[STRIDE]]
@@ -810,13 +759,21 @@ define void @runtime_stride_reorder(ptr %pl, ptr %ps, i64 %stride) {
; CHECK-NEXT: [[GEP_S5:%.*]] = getelementptr i8, ptr [[GEP_S4]], i64 [[STRIDE]]
; CHECK-NEXT: [[GEP_S6:%.*]] = getelementptr i8, ptr [[GEP_S5]], i64 [[STRIDE]]
; CHECK-NEXT: [[GEP_S7:%.*]] = getelementptr i8, ptr [[GEP_S6]], i64 [[STRIDE]]
+; CHECK-NEXT: [[TMP2:%.*]] = extractelement <8 x i8> [[TMP1]], i32 0
; CHECK-NEXT: store i8 [[TMP2]], ptr [[GEP_S0]], align 1
+; CHECK-NEXT: [[TMP3:%.*]] = extractelement <8 x i8> [[TMP1]], i32 2
; CHECK-NEXT: store i8 [[TMP3]], ptr [[GEP_S1]], align 1
+; CHECK-NEXT: [[TMP4:%.*]] = extractelement <8 x i8> [[TMP1]], i32 1
; CHECK-NEXT: store i8 [[TMP4]], ptr [[GEP_S2]], align 1
+; CHECK-NEXT: [[TMP5:%.*]] = extractelement <8 x i8> [[TMP1]], i32 4
; CHECK-NEXT: store i8 [[TMP5]], ptr [[GEP_S3]], align 1
+; CHECK-NEXT: [[TMP6:%.*]] = extractelement <8 x i8> [[TMP1]], i32 7
; CHECK-NEXT: store i8 [[TMP6]], ptr [[GEP_S4]], align 1
+; CHECK-NEXT: [[TMP7:%.*]] = extractelement <8 x i8> [[TMP1]], i32 5
; CHECK-NEXT: store i8 [[TMP7]], ptr [[GEP_S5]], align 1
+; CHECK-NEXT: [[TMP8:%.*]] = extractelement <8 x i8> [[TMP1]], i32 6
; CHECK-NEXT: store i8 [[TMP8]], ptr [[GEP_S6]], align 1
+; CHECK-NEXT: [[TMP9:%.*]] = extractelement <8 x i8> [[TMP1]], i32 3
; CHECK-NEXT: store i8 [[TMP9]], ptr [[GEP_S7]], align 1
; CHECK-NEXT: ret void
;
@@ -863,21 +820,7 @@ define void @runtime_stride_unschedulable(ptr %pl, ptr %ps, i64 %stride) {
; CHECK-LABEL: define void @runtime_stride_unschedulable(
; CHECK-SAME: ptr [[PL:%.*]], ptr [[PS:%.*]], i64 [[STRIDE:%.*]]) #[[ATTR0]] {
; CHECK-NEXT: [[GEP_L0:%.*]] = getelementptr i8, ptr [[PL]], i64 0
-; CHECK-NEXT: [[GEP_L1:%.*]] = getelementptr i8, ptr [[PL]], i64 1
-; CHECK-NEXT: [[GEP_L2:%.*]] = getelementptr i8, ptr [[PL]], i64 2
-; CHECK-NEXT: [[GEP_L3:%.*]] = getelementptr i8, ptr [[PL]], i64 3
-; CHECK-NEXT: [[GEP_L4:%.*]] = getelementptr i8, ptr [[PL]], i64 4
-; CHECK-NEXT: [[GEP_L5:%.*]] = getelementptr i8, ptr [[PL]], i64 5
-; CHECK-NEXT: [[GEP_L6:%.*]] = getelementptr i8, ptr [[PL]], i64 6
-; CHECK-NEXT: [[GEP_L7:%.*]] = getelementptr i8, ptr [[PL]], i64 7
-; CHECK-NEXT: [[TMP8:%.*]] = load i8, ptr [[GEP_L7]], align 1
-; CHECK-NEXT: [[TMP7:%.*]] = load i8, ptr [[GEP_L6]], align 1
-; CHECK-NEXT: [[TMP11:%.*]] = load i8, ptr [[GEP_L5]], align 1
-; CHECK-NEXT: [[TMP6:%.*]] = load i8, ptr [[GEP_L4]], align 1
-; CHECK-NEXT: [[TMP10:%.*]] = load i8, ptr [[GEP_L3]], align 1
-; CHECK-NEXT: [[TMP9:%.*]] = load i8, ptr [[GEP_L2]], align 1
-; CHECK-NEXT: [[TMP5:%.*]] = load i8, ptr [[GEP_L1]], align 1
-; CHECK-NEXT: [[TMP4:%.*]] = load i8, ptr [[GEP_L0]], align 1
+; CHECK-NEXT: [[TMP1:%.*]] = load <8 x i8>, ptr [[GEP_L0]], align 1
; CHECK-NEXT: [[GEP_S0:%.*]] = getelementptr i8, ptr [[PS]], i64 0
; CHECK-NEXT: [[GEP_S1:%.*]] = getelementptr i8, ptr [[GEP_S0]], i64 [[STRIDE]]
; CHECK-NEXT: [[GEP_S2:%.*]] = getelementptr i8, ptr [[GEP_S1]], i64 [[STRIDE]]
@@ -886,13 +829,21 @@ define void @runtime_stride_unschedulable(ptr %pl, ptr %ps, i64 %stride) {
; CHECK-NEXT: [[GEP_S5:%.*]] = getelementptr i8, ptr [[GEP_S4]], i64 [[STRIDE]]
; CHECK-NEXT: [[GEP_S6:%.*]] = getelementptr i8, ptr [[GEP_S5]], i64 [[STRIDE]]
; CHECK-NEXT: [[GEP_S7:%.*]] = getelementptr i8, ptr [[GEP_S6]], i64 [[STRIDE]]
+; CHECK-NEXT: [[TMP4:%.*]] = extractelement <8 x i8> [[TMP1]], i32 0
; CHECK-NEXT: store i8 [[TMP4]], ptr [[GEP_S0]], align 1
+; CHECK-NEXT: [[TMP5:%.*]] = extractelement <8 x i8> [[TMP1]], i32 1
; CHECK-NEXT: store i8 [[TMP5]], ptr [[GEP_S6]], align 1
+; CHECK-NEXT: [[TMP9:%.*]] = extractelement <8 x i8> [[TMP1]], i32 2
; CHECK-NEXT: store i8 [[TMP9]], ptr [[GEP_S2]], align 1
+; CHECK-NEXT: [[TMP10:%.*]] = extractelement <8 x i8> [[TMP1]], i32 3
; CHECK-NEXT: store i8 [[TMP10]], ptr [[GEP_S3]], align 1
+; CHECK-NEXT: [[TMP6:%.*]] = extractelement <8 x i8> [[TMP1]], i32 4
; CHECK-NEXT: store i8 [[TMP6]], ptr [[GEP_S4]], align 1
+; CHECK-NEXT: [[TMP11:%.*]] = extractelement <8 x i8> [[TMP1]], i32 5
; CHECK-NEXT: store i8 [[TMP11]], ptr [[GEP_S5]], align 1
+; CHECK-NEXT: [[TMP7:%.*]] = extractelement <8 x i8> [[TMP1]], i32 6
; CHECK-NEXT: store i8 [[TMP7]], ptr [[GEP_S1]], align 1
+; CHECK-NEXT: [[TMP8:%.*]] = extractelement <8 x i8> [[TMP1]], i32 7
; CHECK-NEXT: store i8 [[TMP8]], ptr [[GEP_S7]], align 1
; CHECK-NEXT: ret void
;
@@ -968,22 +919,13 @@ define void @runtime_stride_diff_types(ptr %pl, ptr %ps, i64 %stride) {
; CHECK-SAME: ptr [[PL:%.*]], ptr [[PS:%.*]], i64 [[STRIDE:%.*]]) #[[ATTR0]] {
; CHECK-NEXT: [[GEP_L0:%.*]] = getelementptr i8, ptr [[PL]], i64 0
; CHECK-NEXT: [[GEP_L1:%.*]] = getelementptr i8, ptr [[PL]], i64 1
-; CHECK-NEXT: [[GEP_L2:%.*]] = getelementptr i8, ptr [[PL]], i64 2
-; CHECK-NEXT: [[GEP_L3:%.*]] = getelementptr i8, ptr [[PL]], i64 3
-; CHECK-NEXT: [[GEP_L4:%.*]] = getelementptr i8, ptr [[PL]], i64 4
-; CHECK-NEXT: [[GEP_L5:%.*]] = getelementptr i8, ptr [[PL]], i64 5
; CHECK-NEXT: [[GEP_L6:%.*]] = getelementptr i8, ptr [[PL]], i64 6
; CHECK-NEXT: [[GEP_L7:%.*]] = getelementptr i8, ptr [[PL]], i64 7
-; CHECK-NEXT: [[TMP9:%.*]] = load i8, ptr [[GEP_L4]], align 1
-; CHECK-NEXT: [[TMP8:%.*]] = load i8, ptr [[GEP_L3]], align 1
-; CHECK-NEXT: [[TMP7:%.*]] = load i8, ptr [[GEP_L2]], align 1
-; CHECK-NEXT: [[TMP4:%.*]] = load i8, ptr [[GEP_L0]], align 1
+; CHECK-NEXT: [[TMP1:%.*]] = call <5 x i8> @llvm.masked.load.v5i8.p0(ptr align 1 [[GEP_L0]], <5 x i1> <i1 true, i1 false, i1 true, i1 true, i1 true>, <5 x i8> poison)
+; CHECK-NEXT: [[TMP2:%.*]] = shufflevector <5 x i8> [[TMP1]], <5 x i8> poison, <4 x i32> <i32 0, i32 2, i32 3, i32 4>
; CHECK-NEXT: [[LOAD6:%.*]] = load i8, ptr [[GEP_L6]], align 1
; CHECK-NEXT: [[LOAD7:%.*]] = load i8, ptr [[GEP_L7]], align 1
-; CHECK-NEXT: [[LOAD5:%.*]] = load i8, ptr [[GEP_L5]], align 1
-; CHECK-NEXT: [[LOAD1:%.*]] = load i8, ptr [[GEP_L1]], align 1
-; CHECK-NEXT: [[TMP11:%.*]] = zext i8 [[LOAD5]] to i16
-; CHECK-NEXT: [[TMP6:%.*]] = zext i8 [[LOAD1]] to i16
+; CHECK-NEXT: [[TMP3:%.*]] = call <2 x i8> @llvm.experimental.vp.strided.load.v2i8.p0.i64(ptr align 1 [[GEP_L1]], i64 4, <2 x i1> splat (i1 true), i32 2)
; CHECK-NEXT: [[GEP_S0:%.*]] = getelementptr i8, ptr [[PS]], i64 0
; CHECK-NEXT: [[GEP_S1:%.*]] = getelementptr i8, ptr [[GEP_S0]], i64 [[STRIDE]]
; CHECK-NEXT: [[GEP_S2:%.*]] = getelementptr i8, ptr [[GEP_S1]], i64 [[STRIDE]]
@@ -992,11 +934,19 @@ define void @runtime_stride_diff_types(ptr %pl, ptr %ps, i64 %stride) {
; CHECK-NEXT: [[GEP_S5:%.*]] = getelementptr i8, ptr [[GEP_S4]], i64 [[STRIDE]]
; CHECK-NEXT: [[GEP_S6:%.*]] = getelementptr i8, ptr [[GEP_S5]], i64 [[STRIDE]]
; CHECK-NEXT: [[GEP_S7:%.*]] = getelementptr i8, ptr [[GEP_S6]], i64 [[STRIDE]]
+; CHECK-NEXT: [[TMP4:%.*]] = extractelement <4 x i8> [[TMP2]], i32 0
; CHECK-NEXT: store i8 [[TMP4]], ptr [[GEP_S0]], align 1
+; CHECK-NEXT: [[TMP5:%.*]] = extractelement <2 x i8> [[TMP3]], i32 0
+; CHECK-NEXT: [[TMP6:%.*]] = zext i8 [[TMP5]] to i16
; CHECK-NEXT: store i16 [[TMP6]], ptr [[GEP_S1]], align 2
+; CHECK-NEXT: [[TMP7:%.*]] = extractelement <4 x i8> [[TMP2]], i32 1
; CHECK-NEXT: store i8 [[TMP7]], ptr [[GEP_S2]], align 1
+; CHECK-NEXT: [[TMP8:%.*]] = extractelement <4 x i8> [[TMP2]], i32 2
; CHECK-NEXT: store i8 [[TMP8]], ptr [[GEP_S3]], align 1
+; CHECK-NEXT: [[TMP9:%.*]] = extractelement <4 x i8> [[TMP2]], i32 3
; CHECK-NEXT: store i8 [[TMP9]], ptr [[GEP_S4]], align 1
+; CHECK-NEXT: [[TMP10:%.*]] = extractelement <2 x i8> [[TMP3]], i32 1
+; CHECK-NEXT: [[TMP11:%.*]] = zext i8 [[TMP10]] to i16
; CHECK-NEXT: store i16 [[TMP11]], ptr [[GEP_S5]], align 2
; CHECK-NEXT: store i8 [[LOAD6]], ptr [[GEP_S6]], align 1
; CHECK-NEXT: store i8 [[LOAD7]], ptr [[GEP_S7]], align 1
diff --git a/llvm/test/Transforms/SLPVectorizer/RISCV/strided-loads-with-external-use-ptr.ll b/llvm/test/Transforms/SLPVectorizer/RISCV/strided-loads-with-external-use-ptr.ll
index a1800363d90b8..8fddb2e21e248 100644
--- a/llvm/test/Transforms/SLPVectorizer/RISCV/strided-loads-with-external-use-ptr.ll
+++ b/llvm/test/Transforms/SLPVectorizer/RISCV/strided-loads-with-external-use-ptr.ll
@@ -15,9 +15,11 @@ define i16 @test() {
; CHECK-NEXT: [[PEDGE_061_I:%.*]] = phi ptr [ [[INCDEC_PTR_I:%.*]], [[WHILE_BODY_I]] ], [ null, [[ENTRY]] ]
; CHECK-NEXT: [[INCDEC_PTR_I]] = getelementptr [[S]], ptr [[PEDGE_061_I]], i64 -1
; CHECK-NEXT: [[PPREV_0_I]] = getelementptr [[S]], ptr [[PPREV_062_I]], i64 -1
-; CHECK-NEXT: [[TMP1:%.*]] = load i16, ptr [[PPREV_0_I]], align 2
-; CHECK-NEXT: [[TMP2:%.*]] = load i16, ptr [[INCDEC_PTR_I]], align 2
-; CHECK-NEXT: [[CMP_I178:%.*]] = icmp ult i16 [[TMP2]], [[TMP1]]
+; CHECK-NEXT: [[TMP1:%.*]] = call <3 x i16> @llvm.masked.load.v3i16.p0(ptr align 2 [[PPREV_0_I]], <3 x i1> <i1 true, i1 false, i1 true>, <3 x i16> poison)
+; CHECK-NEXT: [[TMP2:%.*]] = shufflevector <3 x i16> [[TMP1]], <3 x i16> poison, <2 x i32> <i32 0, i32 2>
+; CHECK-NEXT: [[TMP3:%.*]] = extractelement <2 x i16> [[TMP2]], i32 0
+; CHECK-NEXT: [[TMP4:%.*]] = extractelement <2 x i16> [[TMP2]], i32 1
+; CHECK-NEXT: [[CMP_I178:%.*]] = icmp ult i16 [[TMP4]], [[TMP3]]
; CHECK-NEXT: br label [[WHILE_BODY_I]]
;
entry:
diff --git a/llvm/test/Transforms/SLPVectorizer/X86/PR35628_1.ll b/llvm/test/Transforms/SLPVectorizer/X86/PR35628_1.ll
index 22022a84a217f..e9aa434dec03d 100644
--- a/llvm/test/Transforms/SLPVectorizer/X86/PR35628_1.ll
+++ b/llvm/test/Transforms/SLPVectorizer/X86/PR35628_1.ll
@@ -10,11 +10,9 @@ define void @mainTest(ptr %ptr) #0 {
; CHECK: loop:
; CHECK-NEXT: [[DUMMY_PHI:%.*]] = phi i32 [ 1, [[ENTRY:%.*]] ], [ [[OP_RDX3:%.*]], [[LOOP]] ]
; CHECK-NEXT: [[TMP0:%.*]] = getelementptr inbounds i32, ptr [[PTR]], i64 1
-; CHECK-NEXT: [[TMP8:%.*]] = getelementptr inbounds i32, ptr [[PTR]], i64 2
-; CHECK-NEXT: [[TMP9:%.*]] = getelementptr inbounds i32, ptr [[PTR]], i64 3
-; CHECK-NEXT: [[TMP2:%.*]] = load i32, ptr [[TMP9]], align 4
; CHECK-NEXT: [[TMP1:%.*]] = load <4 x i32>, ptr [[PTR]], align 4
-; CHECK-NEXT: [[TMP3:%.*]] = load i32, ptr [[TMP8]], align 4
+; CHECK-NEXT: [[TMP2:%.*]] = extractelement <4 x i32> [[TMP1]], i32 3
+; CHECK-NEXT: [[TMP3:%.*]] = extractelement <4 x i32> [[TMP1]], i32 2
; CHECK-NEXT: [[TMP4:%.*]] = load i32, ptr [[TMP0]], align 4
; CHECK-NEXT: [[TMP5:%.*]] = mul <4 x i32> [[TMP1]], [[TMP1]]
; CHECK-NEXT: [[TMP6:%.*]] = sext i32 [[TMP3]] to i64
diff --git a/llvm/test/Transforms/SLPVectorizer/X86/call.ll b/llvm/test/Transforms/SLPVectorizer/X86/call.ll
index 1f559a195e76b..9e19aa9e93095 100644
--- a/llvm/test/Transforms/SLPVectorizer/X86/call.ll
+++ b/llvm/test/Transforms/SLPVectorizer/X86/call.ll
@@ -131,12 +131,11 @@ define void @sqrt_libm_no_errno(ptr %a, ptr %b) {
define void @sqrt_libm_errno(ptr %a, ptr %b) {
; CHECK-LABEL: @sqrt_libm_errno(
-; CHECK-NEXT: [[IDX1:%.*]] = getelementptr inbounds double, ptr [[A:%.*]], i64 1
-; CHECK-NEXT: [[A1:%.*]] = load double, ptr [[IDX1]], align 8
-; CHECK-NEXT: [[TMP1:%.*]] = load <2 x double>, ptr [[A]], align 8
+; CHECK-NEXT: [[TMP1:%.*]] = load <2 x double>, ptr [[A:%.*]], align 8
; CHECK-NEXT: [[TMP2:%.*]] = extractelement <2 x double> [[TMP1]], i32 0
; CHECK-NEXT: [[SQRT1:%.*]] = tail call nnan double @sqrt(double [[TMP2]]) #[[ATTR4:[0-9]+]]
-; CHECK-NEXT: [[SQRT2:%.*]] = tail call nnan double @sqrt(double [[A1]]) #[[ATTR4]]
+; CHECK-NEXT: [[TMP3:%.*]] = extractelement <2 x double> [[TMP1]], i32 1
+; CHECK-NEXT: [[SQRT2:%.*]] = tail call nnan double @sqrt(double [[TMP3]]) #[[ATTR4]]
; CHECK-NEXT: store double [[SQRT1]], ptr [[B:%.*]], align 8
; CHECK-NEXT: [[IDX2:%.*]] = getelementptr inbounds double, ptr [[B]], i64 1
; CHECK-NEXT: store double [[SQRT2]], ptr [[IDX2]], align 8
@@ -156,11 +155,11 @@ define void @sqrt_libm_errno(ptr %a, ptr %b) {
; Negative test case
define void @round_custom(ptr %a, ptr %b) {
; CHECK-LABEL: @round_custom(
-; CHECK-NEXT: [[IDX1:%.*]] = getelementptr inbounds i64, ptr [[A:%.*]], i64 1
-; CHECK-NEXT: [[A1:%.*]] = load i64, ptr [[IDX1]], align 8
-; CHECK-NEXT: [[A0:%.*]] = load i64, ptr [[A]], align 8
-; CHECK-NEXT: [[ROUND1:%.*]] = tail call i64 @round(i64 [[A0]]) #[[ATTR5:[0-9]+]]
-; CHECK-NEXT: [[ROUND2:%.*]] = tail call i64 @round(i64 [[A1]]) #[[ATTR5]]
+; CHECK-NEXT: [[TMP1:%.*]] = load <2 x i64>, ptr [[A:%.*]], align 8
+; CHECK-NEXT: [[TMP2:%.*]] = extractelement <2 x i64> [[TMP1]], i32 0
+; CHECK-NEXT: [[ROUND1:%.*]] = tail call i64 @round(i64 [[TMP2]]) #[[ATTR5:[0-9]+]]
+; CHECK-NEXT: [[TMP3:%.*]] = extractelement <2 x i64> [[TMP1]], i32 1
+; CHECK-NEXT: [[ROUND2:%.*]] = tail call i64 @round(i64 [[TMP3]]) #[[ATTR5]]
; CHECK-NEXT: store i64 [[ROUND1]], ptr [[B:%.*]], align 8
; CHECK-NEXT: [[IDX2:%.*]] = getelementptr inbounds i64, ptr [[B]], i64 1
; CHECK-NEXT: store i64 [[ROUND2]], ptr [[IDX2]], align 8
diff --git a/llvm/test/Transforms/SLPVectorizer/X86/extractelemets-extended-by-poison.ll b/llvm/test/Transforms/SLPVectorizer/X86/extractelemets-extended-by-poison.ll
index ccca930e44ba7..132865da252c9 100644
--- a/llvm/test/Transforms/SLPVectorizer/X86/extractelemets-extended-by-poison.ll
+++ b/llvm/test/Transforms/SLPVectorizer/X86/extractelemets-extended-by-poison.ll
@@ -5,8 +5,7 @@ define i32 @test() {
; CHECK-LABEL: define i32 @test() {
; CHECK-NEXT: [[ENTRY:.*:]]
; CHECK-NEXT: [[TMP0:%.*]] = load <4 x i64>, ptr null, align 16
-; CHECK-NEXT: [[TMP12:%.*]] = load i64, ptr getelementptr inbounds nuw (i8, ptr null, i64 8), align 8
-; CHECK-NEXT: [[TMP9:%.*]] = load i64, ptr null, align 16
+; CHECK-NEXT: [[TMP12:%.*]] = extractelement <4 x i64> [[TMP0]], i32 1
; CHECK-NEXT: [[TMP13:%.*]] = or i64 [[TMP12]], 0
; CHECK-NEXT: [[TMP3:%.*]] = shufflevector <4 x i64> [[TMP0]], <4 x i64> poison, <6 x i32> <i32 0, i32 1, i32 2, i32 3, i32 poison, i32 1>
; CHECK-NEXT: [[TMP4:%.*]] = shufflevector <6 x i64> [[TMP3]], <6 x i64> <i64 poison, i64 poison, i64 poison, i64 poison, i64 0, i64 poison>, <6 x i32> <i32 poison, i32 poison, i32 poison, i32 poison, i32 10, i32 5>
@@ -18,6 +17,7 @@ define i32 @test() {
; CHECK-NEXT: [[TMP14:%.*]] = shufflevector <6 x i32> [[TMP11]], <6 x i32> poison, <8 x i32> <i32 0, i32 1, i32 1, i32 2, i32 2, i32 5, i32 4, i32 3>
; CHECK-NEXT: [[TMP15:%.*]] = add <8 x i32> [[TMP14]], zeroinitializer
; CHECK-NEXT: [[TMP8:%.*]] = add <16 x i32> [[TMP7]], zeroinitializer
+; CHECK-NEXT: [[TMP9:%.*]] = extractelement <4 x i64> [[TMP0]], i32 0
; CHECK-NEXT: [[INC_3_3_I_1:%.*]] = or i64 [[TMP9]], 0
; CHECK-NEXT: [[TMP16:%.*]] = shufflevector <16 x i32> [[TMP8]], <16 x i32> poison, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>
; CHECK-NEXT: [[RDX_OP:%.*]] = or <8 x i32> [[TMP16]], [[TMP15]]
diff --git a/llvm/test/Transforms/SLPVectorizer/X86/horizontal-list.ll b/llvm/test/Transforms/SLPVectorizer/X86/horizontal-list.ll
index 1ca2518ddf1df..4e434a61e1f1c 100644
--- a/llvm/test/Transforms/SLPVectorizer/X86/horizontal-list.ll
+++ b/llvm/test/Transforms/SLPVectorizer/X86/horizontal-list.ll
@@ -914,19 +914,14 @@ define float @extra_args_no_fast(ptr %x, float %a, float %b) {
; THRESHOLD-LABEL: @extra_args_no_fast(
; THRESHOLD-NEXT: [[ADDC:%.*]] = fadd fast float [[B:%.*]], 3.000000e+00
; THRESHOLD-NEXT: [[ADD:%.*]] = fadd fast float [[A:%.*]], [[ADDC]]
-; THRESHOLD-NEXT: [[ARRAYIDX3:%.*]] = getelementptr inbounds float, ptr [[X:%.*]], i64 1
-; THRESHOLD-NEXT: [[ARRAYIDX3_1:%.*]] = getelementptr inbounds float, ptr [[X]], i64 2
-; THRESHOLD-NEXT: [[ARRAYIDX3_2:%.*]] = getelementptr inbounds float, ptr [[X]], i64 3
-; THRESHOLD-NEXT: [[TMP1:%.*]] = load <4 x float>, ptr [[X]], align 4
-; THRESHOLD-NEXT: [[T3:%.*]] = load float, ptr [[ARRAYIDX3_2]], align 4
-; THRESHOLD-NEXT: [[TMP2:%.*]] = load <4 x float>, ptr [[X]], align 4
-; THRESHOLD-NEXT: [[T2:%.*]] = load float, ptr [[ARRAYIDX3_1]], align 4
-; THRESHOLD-NEXT: [[T1:%.*]] = load float, ptr [[ARRAYIDX3]], align 4
-; THRESHOLD-NEXT: [[TMP3:%.*]] = extractelement <4 x float> [[TMP1]], i32 0
-; THRESHOLD-NEXT: [[TMP4:%.*]] = extractelement <4 x float> [[TMP2]], i32 0
-; THRESHOLD-NEXT: [[ADD1:%.*]] = fadd fast float [[TMP3]], [[ADD]]
+; THRESHOLD-NEXT: [[TMP1:%.*]] = load <4 x float>, ptr [[X:%.*]], align 4
+; THRESHOLD-NEXT: [[T0:%.*]] = extractelement <4 x float> [[TMP1]], i32 0
+; THRESHOLD-NEXT: [[ADD1:%.*]] = fadd fast float [[T0]], [[ADD]]
+; THRESHOLD-NEXT: [[T1:%.*]] = extractelement <4 x float> [[TMP1]], i32 1
; THRESHOLD-NEXT: [[ADD4:%.*]] = fadd fast float [[T1]], [[ADD1]]
+; THRESHOLD-NEXT: [[T2:%.*]] = extractelement <4 x float> [[TMP1]], i32 2
; THRESHOLD-NEXT: [[ADD4_1:%.*]] = fadd float [[T2]], [[ADD4]]
+; THRESHOLD-NEXT: [[T3:%.*]] = extractelement <4 x float> [[TMP1]], i32 3
; THRESHOLD-NEXT: [[ADD4_2:%.*]] = fadd fast float [[T3]], [[ADD4_1]]
; THRESHOLD-NEXT: [[ADD5:%.*]] = fadd fast float [[ADD4_2]], [[A]]
; THRESHOLD-NEXT: ret float [[ADD5]]
diff --git a/llvm/test/Transforms/SLPVectorizer/X86/horizontal-minmax.ll b/llvm/test/Transforms/SLPVectorizer/X86/horizontal-minmax.ll
index 03c34b4cab092..b2b9363565277 100644
--- a/llvm/test/Transforms/SLPVectorizer/X86/horizontal-minmax.ll
+++ b/llvm/test/Transforms/SLPVectorizer/X86/horizontal-minmax.ll
@@ -268,9 +268,9 @@ define float @maxf8(float) {
; DEFAULT-NEXT: ret float [[TMP23]]
;
; THRESH-LABEL: @maxf8(
-; THRESH-NEXT: [[TMP4:%.*]] = load float, ptr getelementptr inbounds ([32 x float], ptr @arr1, i64 0, i64 1), align 4
; THRESH-NEXT: [[TMP2:%.*]] = load <2 x float>, ptr @arr1, align 16
; THRESH-NEXT: [[TMP3:%.*]] = extractelement <2 x float> [[TMP2]], i32 0
+; THRESH-NEXT: [[TMP4:%.*]] = extractelement <2 x float> [[TMP2]], i32 1
; THRESH-NEXT: [[TMP5:%.*]] = fcmp fast ogt float [[TMP3]], [[TMP4]]
; THRESH-NEXT: [[TMP6:%.*]] = select i1 [[TMP5]], float [[TMP3]], float [[TMP4]]
; THRESH-NEXT: [[TMP7:%.*]] = load float, ptr getelementptr inbounds ([32 x float], ptr @arr1, i64 0, i64 2), align 8
@@ -371,9 +371,9 @@ define float @maxf16(float) {
; DEFAULT-NEXT: ret float [[TMP47]]
;
; THRESH-LABEL: @maxf16(
-; THRESH-NEXT: [[TMP4:%.*]] = load float, ptr getelementptr inbounds ([32 x float], ptr @arr1, i64 0, i64 1), align 4
; THRESH-NEXT: [[TMP2:%.*]] = load <2 x float>, ptr @arr1, align 16
; THRESH-NEXT: [[TMP3:%.*]] = extractelement <2 x float> [[TMP2]], i32 0
+; THRESH-NEXT: [[TMP4:%.*]] = extractelement <2 x float> [[TMP2]], i32 1
; THRESH-NEXT: [[TMP5:%.*]] = fcmp fast ogt float [[TMP3]], [[TMP4]]
; THRESH-NEXT: [[TMP6:%.*]] = select i1 [[TMP5]], float [[TMP3]], float [[TMP4]]
; THRESH-NEXT: [[TMP7:%.*]] = load float, ptr getelementptr inbounds ([32 x float], ptr @arr1, i64 0, i64 2), align 8
@@ -570,9 +570,9 @@ define float @maxf32(float) {
; DEFAULT-NEXT: ret float [[TMP95]]
;
; THRESH-LABEL: @maxf32(
-; THRESH-NEXT: [[TMP4:%.*]] = load float, ptr getelementptr inbounds ([32 x float], ptr @arr1, i64 0, i64 1), align 4
; THRESH-NEXT: [[TMP2:%.*]] = load <2 x float>, ptr @arr1, align 16
; THRESH-NEXT: [[TMP3:%.*]] = extractelement <2 x float> [[TMP2]], i32 0
+; THRESH-NEXT: [[TMP4:%.*]] = extractelement <2 x float> [[TMP2]], i32 1
; THRESH-NEXT: [[TMP5:%.*]] = fcmp fast ogt float [[TMP3]], [[TMP4]]
; THRESH-NEXT: [[TMP6:%.*]] = select i1 [[TMP5]], float [[TMP3]], float [[TMP4]]
; THRESH-NEXT: [[TMP7:%.*]] = load float, ptr getelementptr inbounds ([32 x float], ptr @arr1, i64 0, i64 2), align 8
@@ -785,8 +785,9 @@ define i32 @maxi8_mutiple_uses(i32) {
; DEFAULT-NEXT: ret i32 [[OP_RDX5]]
;
; THRESH-LABEL: @maxi8_mutiple_uses(
-; THRESH-NEXT: [[TMP4:%.*]] = load i32, ptr getelementptr inbounds ([32 x i32], ptr @arr, i64 0, i64 1), align 4
-; THRESH-NEXT: [[TMP3:%.*]] = load i32, ptr @arr, align 16
+; THRESH-NEXT: [[TMP2:%.*]] = load <2 x i32>, ptr @arr, align 16
+; THRESH-NEXT: [[TMP3:%.*]] = extractelement <2 x i32> [[TMP2]], i32 0
+; THRESH-NEXT: [[TMP4:%.*]] = extractelement <2 x i32> [[TMP2]], i32 1
; THRESH-NEXT: [[TMP5:%.*]] = icmp sgt i32 [[TMP3]], [[TMP4]]
; THRESH-NEXT: [[TMP6:%.*]] = select i1 [[TMP5]], i32 [[TMP3]], i32 [[TMP4]]
; THRESH-NEXT: [[TMP7:%.*]] = load <4 x i32>, ptr getelementptr inbounds ([32 x i32], ptr @arr, i64 0, i64 2), align 8
@@ -854,14 +855,15 @@ define i32 @maxi8_mutiple_uses2(i32) {
; DEFAULT-NEXT: ret i32 [[TMP17]]
;
; THRESH-LABEL: @maxi8_mutiple_uses2(
-; THRESH-NEXT: [[TMP10:%.*]] = load i32, ptr getelementptr inbounds ([32 x i32], ptr @arr, i64 0, i64 3), align 4
-; THRESH-NEXT: [[TMP7:%.*]] = load i32, ptr getelementptr inbounds ([32 x i32], ptr @arr, i64 0, i64 2), align 8
-; THRESH-NEXT: [[TMP4:%.*]] = load i32, ptr getelementptr inbounds ([32 x i32], ptr @arr, i64 0, i64 1), align 4
-; THRESH-NEXT: [[TMP3:%.*]] = load i32, ptr @arr, align 16
+; THRESH-NEXT: [[TMP2:%.*]] = load <4 x i32>, ptr @arr, align 16
+; THRESH-NEXT: [[TMP3:%.*]] = extractelement <4 x i32> [[TMP2]], i32 0
+; THRESH-NEXT: [[TMP4:%.*]] = extractelement <4 x i32> [[TMP2]], i32 1
; THRESH-NEXT: [[TMP5:%.*]] = icmp sgt i32 [[TMP3]], [[TMP4]]
; THRESH-NEXT: [[TMP6:%.*]] = select i1 [[TMP5]], i32 [[TMP3]], i32 [[TMP4]]
+; THRESH-NEXT: [[TMP7:%.*]] = extractelement <4 x i32> [[TMP2]], i32 2
; THRESH-NEXT: [[TMP8:%.*]] = icmp sgt i32 [[TMP6]], [[TMP7]]
; THRESH-NEXT: [[TMP9:%.*]] = select i1 [[TMP8]], i32 [[TMP6]], i32 [[TMP7]]
+; THRESH-NEXT: [[TMP10:%.*]] = extractelement <4 x i32> [[TMP2]], i32 3
; THRESH-NEXT: [[TMP11:%.*]] = icmp sgt i32 [[TMP9]], [[TMP10]]
; THRESH-NEXT: [[TMP12:%.*]] = select i1 [[TMP11]], i32 [[TMP9]], i32 [[TMP10]]
; THRESH-NEXT: [[TMP13:%.*]] = load i32, ptr getelementptr inbounds ([32 x i32], ptr @arr, i64 0, i64 4), align 16
@@ -967,8 +969,9 @@ define ptr @maxp8(i32) {
; DEFAULT-NEXT: ret ptr [[TMP23]]
;
; THRESH-LABEL: @maxp8(
-; THRESH-NEXT: [[TMP4:%.*]] = load ptr, ptr getelementptr inbounds ([32 x ptr], ptr @arrp, i64 0, i64 1), align 4
-; THRESH-NEXT: [[TMP3:%.*]] = load ptr, ptr @arrp, align 16
+; THRESH-NEXT: [[TMP2:%.*]] = load <2 x ptr>, ptr @arrp, align 16
+; THRESH-NEXT: [[TMP3:%.*]] = extractelement <2 x ptr> [[TMP2]], i32 0
+; THRESH-NEXT: [[TMP4:%.*]] = extractelement <2 x ptr> [[TMP2]], i32 1
; THRESH-NEXT: [[TMP5:%.*]] = icmp ugt ptr [[TMP3]], [[TMP4]]
; THRESH-NEXT: [[TMP6:%.*]] = select i1 [[TMP5]], ptr [[TMP3]], ptr [[TMP4]]
; THRESH-NEXT: [[TMP7:%.*]] = load ptr, ptr getelementptr inbounds ([32 x ptr], ptr @arrp, i64 0, i64 2), align 8
>From 68a8a41f5237c2946454d28fb1d148f3da70d398 Mon Sep 17 00:00:00 2001
From: Ryan Buchner <buchner.ryan at gmail.com>
Date: Fri, 31 Jul 2026 17:00:43 -0700
Subject: [PATCH 08/22] Fix scheduling issues for deferred extracts
---
.../Transforms/Vectorize/SLPVectorizer.cpp | 36 ++++++++++---------
1 file changed, 19 insertions(+), 17 deletions(-)
diff --git a/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp b/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp
index 8500375cbba81..e559cb8fe3a9b 100644
--- a/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp
+++ b/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp
@@ -5679,6 +5679,17 @@ class slpvectorizer::BoUpSLP {
DecrUnschedForInst(BundleMember->getInst(), U.getOperandNo(), I);
}
}
+ if (auto *EI =
+ R.getCouldBeExtract().lookup(BundleMember->getInst())) {
+ for (Use &U : EI->operands()) {
+ if (auto *I = dyn_cast<Instruction>(U.get())) {
+ LLVM_DEBUG(dbgs() << "SLP: check for readiness (def): " << *I
+ << "\n");
+ DecrUnschedForInst(BundleMember->getInst(), U.getOperandNo(),
+ I);
+ }
+ }
+ }
}
// Handle the memory dependencies.
auto *SD = dyn_cast<ScheduleData>(BundleMember);
@@ -27180,25 +27191,16 @@ void BoUpSLP::BlockScheduling::initScheduleData(Instruction *FromI,
continue;
ScheduleData *SD = ScheduleDataMap.lookup(I);
if (!SD) {
- // Both an extract and its rematerialization ought to be scheduled
- // together
- if (auto *EI = R.getCouldBeExtract().lookup(I)) {
- SD = ScheduleDataMap.lookup(EI);
- if (!SD) {
- SD = allocateScheduleDataChunks();
- ScheduleDataMap[EI] = SD;
- }
- } else if (auto *RI = R.getCouldBeRemat().lookup(I)) {
- SD = ScheduleDataMap.lookup(RI);
- if (!SD) {
- SD = allocateScheduleDataChunks();
- ScheduleDataMap[RI] = SD;
- }
- } else {
- SD = allocateScheduleDataChunks();
- }
+ SD = allocateScheduleDataChunks();
ScheduleDataMap[I] = SD;
}
+ // Both an extract and its rematerialization ought to be scheduled together
+ if (auto *EI = R.getCouldBeExtract().lookup(I))
+ if (!ScheduleDataMap.lookup(EI))
+ ScheduleDataMap[EI] = SD;
+ else if (auto *RI = R.getCouldBeRemat().lookup(I))
+ if (!ScheduleDataMap.lookup(RI))
+ ScheduleDataMap[RI] = SD;
bool IsSharedNode =
R.getCouldBeExtract().contains(I) || R.getCouldBeRemat().contains(I);
assert((!isInSchedulingRegion(*SD) || IsSharedNode) &&
>From 539f0c82831f3fbb1132ee74e3d8db7c10b5804c Mon Sep 17 00:00:00 2001
From: Ryan Buchner <buchner.ryan at gmail.com>
Date: Fri, 31 Jul 2026 19:41:01 -0700
Subject: [PATCH 09/22] Hush compile warning
---
llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp | 3 ++-
1 file changed, 2 insertions(+), 1 deletion(-)
diff --git a/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp b/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp
index e559cb8fe3a9b..bf7651170bc03 100644
--- a/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp
+++ b/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp
@@ -5466,9 +5466,10 @@ class slpvectorizer::BoUpSLP {
// Track the operands from the extractelement copy
// as well to make sure the dependency on the vector
// is tracked
- if (auto *EI = R.getCouldBeExtract().lookup(In))
+ if (auto *EI = R.getCouldBeExtract().lookup(In)) {
for (const Use &U : EI->operands())
HandleOneOp(U);
+ }
}
// Decrement the unscheduled counter and insert to ready list if
// ready.
>From 08af91e042f47c88e690ab16e0558b098a1d0148 Mon Sep 17 00:00:00 2001
From: Ryan Buchner <buchner.ryan at gmail.com>
Date: Tue, 4 Aug 2026 08:59:25 -0700
Subject: [PATCH 10/22] [SLP] Pull out deferred extract logic into an inner
class
---
.../Transforms/Vectorize/SLPVectorizer.cpp | 461 ++++++++++--------
1 file changed, 249 insertions(+), 212 deletions(-)
diff --git a/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp b/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp
index bf7651170bc03..cec5e64c04342 100644
--- a/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp
+++ b/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp
@@ -1191,6 +1191,7 @@ class slpvectorizer::BoUpSLP {
class ScheduleBundle;
class ShuffleCostEstimator;
class ShuffleInstructionBuilder;
+ class DeferredExtractTracker;
public:
/// If we decide to generate strided load / store, this struct contains all
@@ -1480,9 +1481,7 @@ class slpvectorizer::BoUpSLP {
CompressEntryToData.clear();
ExternalUses.clear();
ExternalUsesAsOriginalScalar.clear();
- ExternalUsesAsExtract.clear();
- ExternalUsesAsRematCostTmp.clear();
- ExternalUsesAsExtractCostTmp.clear();
+ DE.clearForDeleteTree();
ExternalUsesWithNonUsers.clear();
RTChecks.clear();
HasRuntimeCheckableBlockers = false;
@@ -2993,26 +2992,7 @@ class slpvectorizer::BoUpSLP {
/// is delayed until BoUpSLP is destructed.
void eraseInstruction(Instruction *I) {
DeletedInstructions.insert(I);
- if (auto It = DeferredScalarsToExtract.find(I);
- It != DeferredScalarsToExtract.end()) {
- SmallPtrSet<Instruction *, 2> ProcessedExtracts;
- for (DeferredExtractType &DET : It->getSecond()) {
- auto *E = cast<Instruction>(DET.NewInst);
- if (!ProcessedExtracts.insert(E).second)
- continue;
- bool LiveUsers = false;
- for (Use &U : E->uses())
- if (!isDeleted(cast<Instruction>(U.getUser()))) {
- LiveUsers = true;
- break;
- }
- if (!LiveUsers) {
- LLVM_DEBUG(dbgs() << "SLP: \tErasing scalar:" << *E << ".\n");
- eraseInstruction(E);
- }
- }
- }
- DeferredScalarsToExtract.erase(I);
+ DE.eraseInstruction(I, *this);
}
/// Remove instructions from the parent function and clear the operands of \p
@@ -3175,15 +3155,11 @@ class slpvectorizer::BoUpSLP {
OrdersType &ReorderIndices) const;
// Create ExtractElement instructions that we deferred creating earlier
- // to allow for better vectorization of chains using those values
+ // to allow for better vectorization of chains using those values.
void emitDeferredExtracts();
- const DenseMap<Value *, ExtractElementInst *> &getCouldBeExtract() const {
- return CouldBeExtract;
- }
- const DenseMap<Value *, Instruction *> &getCouldBeRemat() const {
- return CouldBeRemat;
- }
+ // Get handle for the DeferredExtractTracker
+ const DeferredExtractTracker &getDE() { return DE; }
~BoUpSLP();
@@ -3313,22 +3289,195 @@ class slpvectorizer::BoUpSLP {
SmallVectorImpl<unsigned> &CurrentOrder,
bool ResizeAllowed = false) const;
- /// Vectorize a single entry in the tree.
- Value *vectorizeTree(TreeEntry *E);
+ /// Tracks deferred extract/rematerialization data and late cleanup actions.
+ class DeferredExtractTracker {
+ public:
+ struct DeferredExtractType {
+ Value *Scalar;
+ Value *NewInst;
+ llvm::User *User;
+ DeferredExtractType(Value *Scalar, Value *NewInst, llvm::User *User)
+ : Scalar(Scalar), NewInst(NewInst), User(User) {}
+ };
+
+ /// Map a scalar rematerialized form to the matching extractelement.
+ DenseMap<Value *, ExtractElementInst *> CouldBeExtract;
+ /// Reverse mapping used by scheduling: extractelement back to remat scalar.
+ DenseMap<Value *, Instruction *> CouldBeRemat;
+
+ /// Cases where extraction is estimated as more profitable but want to
+ /// delay extraction to allow for better vectorization in the interim.
+ SmallPtrSet<Value *, 4> ExternalUsesAsExtract;
+ /// Per-tree extract profitability cost for ExternalUsesAsExtract scalars.
+ DenseMap<const Value *, InstructionCost> ExternalUsesAsExtractCost;
+ DenseMap<const Value *, InstructionCost> ExternalUsesAsRematCost;
+ /// Cache the costs for the current tree since we may not keep around these
+ /// extracts if the tree is rejected.
+ DenseMap<const Value *, InstructionCost> ExternalUsesAsExtractCostTmp;
+ DenseMap<const Value *, InstructionCost> ExternalUsesAsRematCostTmp;
+
+ /// Track external uses that are more profitable as extracts. We
+ /// rematerialize the value initially, but on cleanup we need to swap back
+ /// in the extract instruction.
+ DenseMap<Value *, SmallVector<DeferredExtractType, 2>>
+ DeferredScalarsToExtract;
+
+ void clearForDeleteTree() {
+ ExternalUsesAsExtract.clear();
+ ExternalUsesAsRematCostTmp.clear();
+ ExternalUsesAsExtractCostTmp.clear();
+ }
+
+ void eraseInstruction(Instruction *I, BoUpSLP &R) {
+ if (auto It = DeferredScalarsToExtract.find(I);
+ It != DeferredScalarsToExtract.end()) {
+ SmallPtrSet<Instruction *, 2> ProcessedExtracts;
+ for (DeferredExtractType &DET : It->getSecond()) {
+ auto *E = cast<Instruction>(DET.NewInst);
+ if (!ProcessedExtracts.insert(E).second)
+ continue;
+ bool LiveUsers = false;
+ for (Use &U : E->uses())
+ if (!R.isDeleted(cast<Instruction>(U.getUser()))) {
+ LiveUsers = true;
+ break;
+ }
+ if (!LiveUsers) {
+ LLVM_DEBUG(dbgs() << "SLP: \tErasing scalar:" << *E << ".\n");
+ R.eraseInstruction(E);
+ }
+ }
+ }
+ DeferredScalarsToExtract.erase(I);
+ }
+
+ void emitDeferredExtracts(BoUpSLP &R) {
+ SmallVector<DeferredExtractType> DeferredExtracts;
+ for (const auto &Entry : DeferredScalarsToExtract)
+ append_range(DeferredExtracts, Entry.second);
+ for (const auto &DET : DeferredExtracts) {
+ auto *UI = cast_or_null<Instruction>(DET.User);
+ if (!DET.User) {
+ DET.Scalar->replaceAllUsesWith(DET.NewInst);
+ } else {
+ if (R.isDeleted(UI))
+ continue;
+ DET.User->replaceUsesOfWith(DET.Scalar, DET.NewInst);
+ LLVM_DEBUG(dbgs() << "SLP: Delayed replacement:" << *UI << ".\n");
+ }
+ }
+ DeferredScalarsToExtract.clear();
+ for (const auto &DET : DeferredExtracts) {
+ LLVM_DEBUG(dbgs() << "SLP: \tErasing scalar:" << *DET.Scalar << ".\n");
+ auto *I = cast<Instruction>(DET.Scalar);
+ if (R.isDeleted(I))
+ continue;
+ assert((I->use_empty() || all_of(I->uses(),
+ [&](Use &U) {
+ return R.isDeleted(
+ cast<Instruction>(U.getUser()));
+ })) &&
+ "trying to erase instruction with users.");
+ R.eraseInstruction(I);
+ }
+ for (const auto &P : CouldBeExtract) {
+ if (auto *Ext = P.second) {
+ if (!R.isDeleted(Ext)) {
+ unsigned NumUses = Ext->getNumUses();
+ if (!NumUses) {
+ R.eraseInstruction(Ext);
+ } else if (NumUses == 1) {
+ Value *User = Ext->uses().begin()->getUser();
+ if (auto *CI = dyn_cast<CastInst>(User);
+ CI && CI->getNumUses() == 0) {
+ R.eraseInstruction(CI);
+ R.eraseInstruction(Ext);
+ }
+ }
+ }
+ }
+ }
+ }
+
+ void transferExtractCost(Instruction *Inst, Instruction *Replacement) {
+ if (ExternalUsesAsExtractCostTmp.contains(Inst)) {
+ InstructionCost ExtractCost = ExternalUsesAsExtractCostTmp.lookup(Inst);
+ ExternalUsesAsExtractCost.try_emplace(Replacement, ExtractCost);
+ }
+ }
+
+ void transferRematCost(Instruction *Inst, Instruction *Replacement) {
+ if (ExternalUsesAsRematCostTmp.contains(Inst)) {
+ InstructionCost RematCost = ExternalUsesAsRematCostTmp.lookup(Inst);
+ ExternalUsesAsRematCost.try_emplace(Replacement, RematCost);
+ }
+ }
+
+ void logExtractRematPair(ExtractElementInst *EI, Instruction *RI) {
+ CouldBeExtract.try_emplace(RI, EI);
+ CouldBeRemat.try_emplace(EI, RI);
+ }
+
+ void replaceWithExtractCandidates(SmallVectorImpl<Value *> &VL) {
+ // Can this bundle be represented as extracts from the same vector
+ auto IsExtractLikeBundle = [&](ArrayRef<Value *> VL) -> bool {
+ if (!all_of(VL, [&](const Value *Scalar) -> bool {
+ return CouldBeExtract.contains(Scalar);
+ }))
+ return false;
+
+ auto *BaseEE = CouldBeExtract.lookup(VL[0]);
+ Value *BaseVec = BaseEE->getVectorOperand();
+ for (const Value *S : VL) {
+ auto *EE = CouldBeExtract.lookup(S);
+ // Only handle simple case where all elements come from the same
+ // vector If needs to be from multiple vectors, better off leaving in
+ // current form. Will handle replacing individual rematerializations
+ // with extracts during gather node creation.
+ Value *Vec = EE->getVectorOperand();
+ if (Vec != BaseVec)
+ return false;
+ std::optional<unsigned> Lane = getExtractIndex(EE);
+ if (!Lane)
+ return false;
+ }
+ return true;
+ };
+ if (!IsExtractLikeBundle(VL))
+ return;
+ (void)replaceWithExtracts(VL);
+ }
- struct DeferredExtractType {
- Value *Scalar;
- Value *NewInst;
- llvm::User *User;
- DeferredExtractType(Value *Scalar, Value *NewInst, llvm::User *User)
- : Scalar(Scalar), NewInst(NewInst), User(User) {}
+ bool replaceWithExtracts(SmallVectorImpl<Value *> &VL) {
+ bool Replaced = false;
+ for (Value *&VPtr : VL) {
+ if (auto *Extract = CouldBeExtract.lookup(VPtr)) {
+ VPtr = Extract;
+ Replaced = true;
+ }
+ }
+ return Replaced;
+ }
+
+ void trackExternalUseCost(Value *ExternalUser, Value *Key, bool AsRemat,
+ InstructionCost ExtractCost,
+ InstructionCost RematCost) {
+ if (isDeferredExtractable(ExternalUser)) {
+ if (!AsRemat)
+ ExternalUsesAsExtract.insert(ExternalUser);
+ InstructionCost Delta =
+ AsRemat ? ExtractCost - RematCost : RematCost - ExtractCost;
+ auto &CostStruct =
+ AsRemat ? ExternalUsesAsRematCostTmp : ExternalUsesAsExtractCostTmp;
+ auto [ItCost, Inserted] = CostStruct.try_emplace(Key, Delta);
+ if (!Inserted)
+ ItCost->second = std::min(ItCost->second, Delta);
+ }
+ }
};
- /// Track external uses that are more profitable as extracts
- /// We rematerialize the value initially, but on cleanup
- /// we need to swap back in the extract instruction
- DenseMap<Value *, SmallVector<DeferredExtractType, 2>>
- DeferredScalarsToExtract;
+ /// Vectorize a single entry in the tree.
+ Value *vectorizeTree(TreeEntry *E);
/// Vectorize a single entry in the tree, the \p Idx-th operand of the entry
/// \p E.
@@ -4037,11 +4186,7 @@ class slpvectorizer::BoUpSLP {
/// Update the parent nodes operands to match
bool ReplacedByExtractCandidate = false;
if (EntryState == TreeEntry::NeedToGather) {
- for (auto &VPtr : Last->Scalars)
- if (auto *NewVPtr = CouldBeExtract.lookup(VPtr)) {
- VPtr = NewVPtr;
- ReplacedByExtractCandidate = true;
- }
+ ReplacedByExtractCandidate = DE.replaceWithExtracts(Last->Scalars);
if (ReplacedByExtractCandidate) {
LoadEntriesToVectorize.remove(Last->Idx);
ScalarsVectorizationLegality Legality =
@@ -4052,7 +4197,7 @@ class slpvectorizer::BoUpSLP {
if (UserTreeIdx.UserTE)
for (auto &VPtr :
UserTreeIdx.UserTE->getOperand(UserTreeIdx.EdgeIdx))
- if (auto *NewVPtr = CouldBeExtract.lookup(VPtr))
+ if (auto *NewVPtr = DE.CouldBeExtract.lookup(VPtr))
VPtr = NewVPtr;
}
}
@@ -4487,23 +4632,8 @@ class slpvectorizer::BoUpSLP {
/// extractelement materialization.
SmallPtrSet<Value *, 4> ExternalUsesAsOriginalScalar;
- /// Map a scalar rematerialized form to the matching extractelement.
- DenseMap<Value *, ExtractElementInst *> CouldBeExtract;
- /// Reverse mapping used by scheduling: extractelement back to remat scalar.
- DenseMap<Value *, Instruction *> CouldBeRemat;
-
- /// Cases where extraction is estimated as more profitable but want to delay
- /// extraction to allow for better vectorization in the interim. These values
- /// are converted to extracts in a late cleanup step after primary SLP
- /// rewriting.
- SmallPtrSet<Value *, 4> ExternalUsesAsExtract;
- /// Per-tree extract profitability cost for ExternalUsesAsExtract scalars.
- DenseMap<const Value *, InstructionCost> ExternalUsesAsExtractCost;
- DenseMap<const Value *, InstructionCost> ExternalUsesAsRematCost;
- /// Cache the costs for the current tree since we may not keep around these
- /// extracts if the tree is rejected
- DenseMap<const Value *, InstructionCost> ExternalUsesAsExtractCostTmp;
- DenseMap<const Value *, InstructionCost> ExternalUsesAsRematCostTmp;
+ /// Deferred extract/rematerialization state for the current function pass.
+ DeferredExtractTracker DE;
/// A list of scalar to be extracted without specific user because of too many
/// uses.
@@ -5038,8 +5168,8 @@ class slpvectorizer::BoUpSLP {
/// extractelements/insertelements only or nodes with instructions, with
/// uses/operands outside of the block.
struct BlockScheduling {
- BlockScheduling(BasicBlock *BB, BoUpSLP &R)
- : BB(BB), ChunkSize(BB->size()), ChunkPos(ChunkSize), R(R) {}
+ BlockScheduling(BasicBlock *BB, const BoUpSLP::DeferredExtractTracker &DE)
+ : BB(BB), ChunkSize(BB->size()), ChunkPos(ChunkSize), DE(DE) {}
void clear() {
ScheduledBundles.clear();
@@ -5466,7 +5596,7 @@ class slpvectorizer::BoUpSLP {
// Track the operands from the extractelement copy
// as well to make sure the dependency on the vector
// is tracked
- if (auto *EI = R.getCouldBeExtract().lookup(In)) {
+ if (auto *EI = DE.CouldBeExtract.lookup(In)) {
for (const Use &U : EI->operands())
HandleOneOp(U);
}
@@ -5585,9 +5715,9 @@ class slpvectorizer::BoUpSLP {
OpIdx)[Lane])) {
LLVM_DEBUG(dbgs() << "SLP: check for readiness (def): "
<< *I << "\n");
- // The scheduling node works on the rematerialize version
+ // The scheduling node works on the rematerialized version
// of the extract
- if (auto *RI = R.getCouldBeRemat().lookup(I))
+ if (auto *RI = DE.CouldBeRemat.lookup(I))
I = RI;
DecrUnschedForInst(
I, Bundle->getTreeEntry(), OpIdx, Checked,
@@ -5680,8 +5810,7 @@ class slpvectorizer::BoUpSLP {
DecrUnschedForInst(BundleMember->getInst(), U.getOperandNo(), I);
}
}
- if (auto *EI =
- R.getCouldBeExtract().lookup(BundleMember->getInst())) {
+ if (auto *EI = DE.CouldBeExtract.lookup(BundleMember->getInst())) {
for (Use &U : EI->operands()) {
if (auto *I = dyn_cast<Instruction>(U.get())) {
LLVM_DEBUG(dbgs() << "SLP: check for readiness (def): " << *I
@@ -5921,7 +6050,7 @@ class slpvectorizer::BoUpSLP {
int ChunkPos;
/// Use to access information about deferred extracts
- BoUpSLP &R;
+ const BoUpSLP::DeferredExtractTracker &DE;
/// Attaches ScheduleData to Instruction.
/// Note that the mapping survives during all vectorization iterations, i.e.
@@ -12881,42 +13010,10 @@ void BoUpSLP::buildTreeRec(ArrayRef<Value *> VLRef, unsigned Depth,
SmallVector<int> ReuseShuffleIndices;
SmallVector<Value *> VL(VLRef);
- // Can this bundle be represented as extracts from the same vector
- auto IsExtractLikeBundle = [&](ArrayRef<Value *> Scalars) -> bool {
- if (!all_of(Scalars, [&](const Value *Scalar) -> bool {
- return CouldBeExtract.contains(Scalar);
- }))
- return false;
-
- auto *BaseEE = CouldBeExtract.lookup(Scalars[0]);
- Value *BaseVec = BaseEE->getVectorOperand();
- for (const Value *S : Scalars) {
- auto *EE = CouldBeExtract.lookup(S);
- // Only handle simple case where all elements come from the same vector
- // If needs to be from multiple vectors, better off leaving in current
- // form. Will handle replacing individual rematerializations with extracts
- // during gather node creation.
- Value *Vec = EE->getVectorOperand();
- if (Vec != BaseVec)
- return false;
- std::optional<unsigned> Lane = getExtractIndex(EE);
- if (!Lane)
- return false;
- }
- return true;
- };
- auto ReplaceWithExtractCandidates = [&](SmallVectorImpl<Value *> &Scalars) {
- if (!IsExtractLikeBundle(Scalars))
- return;
- for (Value *&VPtr : Scalars) {
- auto *Extract = CouldBeExtract.lookup(VPtr);
- VPtr = Extract;
- }
- };
// Normally we replace at the time of operand creation but if first
// Entry in tree these VL were never operands
if (Depth == 0)
- ReplaceWithExtractCandidates(VL);
+ DE.replaceWithExtractCandidates(VL);
// Tries to build split node.
auto TrySplitNode = [&](const InstructionsState &LocalState) {
@@ -13119,7 +13216,7 @@ void BoUpSLP::buildTreeRec(ArrayRef<Value *> VLRef, unsigned Depth,
BasicBlock *BB = VL0->getParent();
auto &BSRef = BlocksSchedules[BB];
if (!BSRef)
- BSRef = std::make_unique<BlockScheduling>(BB, *this);
+ BSRef = std::make_unique<BlockScheduling>(BB, DE);
BlockScheduling &BS = *BSRef;
@@ -13152,7 +13249,7 @@ void BoUpSLP::buildTreeRec(ArrayRef<Value *> VLRef, unsigned Depth,
InstructionsCompatibilityAnalysis Analysis(*DT, *DL, *TTI, *TLI);
SmallVector<ValueList> Operands = Analysis.buildOperands(S, VL);
for (auto &Ops : Operands)
- ReplaceWithExtractCandidates(Ops);
+ DE.replaceWithExtractCandidates(Ops);
// Flatten associative binary chains into operand columns. Only the peeled
// chain links are required to be single-use (they are erased); the root
@@ -13564,7 +13661,7 @@ void BoUpSLP::buildTreeRec(ArrayRef<Value *> VLRef, unsigned Depth,
// Try to replace again after shuffling operands
// TODO: Make VLOperands aware of deferred extracts
for (auto &Ops : Operands)
- ReplaceWithExtractCandidates(Ops);
+ DE.replaceWithExtractCandidates(Ops);
TE->setOperands(Operands);
for (unsigned I : seq<unsigned>(TE->getNumOperands()))
@@ -14014,7 +14111,7 @@ uint64_t BoUpSLP::getNumVectorInsts(bool HasTreeLoop) {
// entries and count once at the end.
if (all_of(TE.Scalars, [&](Value *V) {
return isa<ExtractElementInst, UndefValue, Constant>(V) ||
- CouldBeExtract.contains(V);
+ DE.CouldBeExtract.contains(V);
})) {
for (Value *V : TE.Scalars) {
if (auto *EE = dyn_cast<ExtractElementInst>(V)) {
@@ -14022,8 +14119,8 @@ uint64_t BoUpSLP::getNumVectorInsts(bool HasTreeLoop) {
GatherExtractSourceVecs.try_emplace(EE->getVectorOperand(), 0)
.first->second;
VecScale = std::max(VecScale, Scale);
- } else if (CouldBeExtract.contains(V)) {
- if (auto *EE = CouldBeExtract.lookup(V)) {
+ } else if (DE.CouldBeExtract.contains(V)) {
+ if (auto *EE = DE.CouldBeExtract.lookup(V)) {
uint64_t &VecScale =
GatherExtractSourceVecs.try_emplace(EE->getVectorOperand(), 0)
.first->second;
@@ -17243,13 +17340,13 @@ BoUpSLP::getEntryCost(const TreeEntry *E, ArrayRef<Value *> VectorizedVals,
if (UsedScalars.test(I))
continue;
if (auto *Inst = dyn_cast<Instruction>(UniqueValues[I])) {
- if (auto It = ExternalUsesAsExtractCost.find(Inst);
- It != ExternalUsesAsExtractCost.end()) {
+ if (auto It = DE.ExternalUsesAsExtractCost.find(Inst);
+ It != DE.ExternalUsesAsExtractCost.end()) {
RematAdjustment += It->second;
continue;
}
- if (auto It = ExternalUsesAsRematCost.find(Inst);
- It != ExternalUsesAsRematCost.end()) {
+ if (auto It = DE.ExternalUsesAsRematCost.find(Inst);
+ It != DE.ExternalUsesAsRematCost.end()) {
RematAdjustment += It->second;
continue;
}
@@ -18417,7 +18514,7 @@ bool BoUpSLP::isFullyVectorizableTinyTree(bool ForReduction) const {
(((TE->hasState() &&
TE->getOpcode() == Instruction::ExtractElement) ||
all_of(TE->Scalars, IsaPred<ExtractElementInst, UndefValue>)) &&
- isFixedVectorShuffle(TE->Scalars, Mask, AC, CouldBeExtract)) ||
+ isFixedVectorShuffle(TE->Scalars, Mask, AC, DE.CouldBeExtract)) ||
(TE->hasState() && TE->getOpcode() == Instruction::Load &&
!TE->isAltShuffle()) ||
any_of(TE->Scalars, IsaPred<LoadInst>));
@@ -20393,14 +20490,8 @@ InstructionCost BoUpSLP::getTreeCost(InstructionCost TreeCost,
}
if (KeepScalar) {
ExternalUsesAsOriginalScalar.insert(EU.Scalar);
- if (isDeferredExtractable(EU.Scalar)) {
- InstructionCost RematDelta = ExtraCost - ScalarCost;
- auto [ItCost, Inserted] =
- ExternalUsesAsRematCostTmp.try_emplace(EU.Scalar, RematDelta);
- if (!Inserted)
- ItCost->second = std::min(ItCost->second, RematDelta);
- }
-
+ DE.trackExternalUseCost(/*ExternalUser*/ EU.Scalar, /*Key*/ EU.Scalar,
+ /*AsRemat*/ true, ExtraCost, ScalarCost);
for (Value *V : Inst->operands()) {
// Struct operands cannot be rebuilt by the !User extraction
// path (it has no insertvalue chain), so leave their existing
@@ -20432,13 +20523,9 @@ InstructionCost BoUpSLP::getTreeCost(InstructionCost TreeCost,
}
}
}
- } else if (isDeferredExtractable(EU.Scalar)) {
- ExternalUsesAsExtract.insert(EU.Scalar);
- InstructionCost ExtractDelta = ScalarCost - ExtraCost;
- auto [ItCost, Inserted] =
- ExternalUsesAsExtractCostTmp.try_emplace(Inst, ExtractDelta);
- if (!Inserted)
- ItCost->second = std::min(ItCost->second, ExtractDelta);
+ } else {
+ DE.trackExternalUseCost(/*ExternalUser*/ EU.Scalar, /*Key*/ Inst,
+ /*AsRemat*/ false, ExtraCost, ScalarCost);
}
}
}
@@ -20844,7 +20931,7 @@ BoUpSLP::tryToGatherSingleRegisterExtractElements(
// Check that gather of extractelements can be represented as just a
// shuffle of a single/two vectors the scalars are extracted from.
std::optional<TTI::ShuffleKind> Res =
- isFixedVectorShuffle(GatheredExtracts, Mask, AC, CouldBeExtract);
+ isFixedVectorShuffle(GatheredExtracts, Mask, AC, DE.CouldBeExtract);
if (!Res || all_of(Mask, equal_to(PoisonMaskElem))) {
// TODO: try to check other subsets if possible.
// Restore the original VL if attempt was not successful.
@@ -22904,7 +22991,7 @@ ResTy BoUpSLP::processBuildVector(const TreeEntry *E, Type *ScalarTy,
auto GetGatheredExtract = [&](unsigned Idx) -> ExtractElementInst * {
if (auto *EI = dyn_cast<ExtractElementInst>(StoredGS[Idx]))
return EI;
- return CouldBeExtract.lookup(StoredGS[Idx]);
+ return DE.CouldBeExtract.lookup(StoredGS[Idx]);
};
if (!all_of(GatheredScalars, IsaPred<UndefValue>)) {
// Check for gathered extracts.
@@ -22932,7 +23019,7 @@ ResTy BoUpSLP::processBuildVector(const TreeEntry *E, Type *ScalarTy,
}
if (Value *VecBase = ShuffleBuilder.adjustExtracts(
E, ExtractMask, ExtractShuffles, NumParts, UseVecBaseAsInput,
- CouldBeRemat)) {
+ DE.CouldBeRemat)) {
ExtractVecBase = VecBase;
if (auto *VecBaseTy = dyn_cast<FixedVectorType>(VecBase->getType()))
if (VF == VecBaseTy->getNumElements() &&
@@ -25809,28 +25896,21 @@ Value *BoUpSLP::vectorizeTree(
auto *EI = dyn_cast<ExtractElementInst>(ReplacedExtract);
auto *RI = dyn_cast<Instruction>(Replacement);
assert(EI && RI && "Expected to find underlying instructions");
- if (CouldBeExtract.contains(RI))
+ if (DE.CouldBeExtract.contains(RI))
return;
- if (ExternalUsesAsRematCostTmp.contains(Inst)) {
- InstructionCost RematCost = ExternalUsesAsRematCostTmp.lookup(Inst);
- ExternalUsesAsRematCost.try_emplace(EI, RematCost);
- }
- CouldBeExtract.try_emplace(RI, EI);
- CouldBeRemat.try_emplace(EI, RI);
+ DE.transferRematCost(Inst, EI);
+ DE.logExtractRematPair(EI, RI);
ExtractAnyways = false;
return;
}
auto *EI = dyn_cast<ExtractElementInst>(Replacement);
auto *RI = Inst;
assert(EI && RI && "Expected to find underlying instructions");
- if (!ExternalUsesAsExtract.contains(RI) || CouldBeExtract.contains(RI))
+ if (!DE.ExternalUsesAsExtract.contains(RI) ||
+ DE.CouldBeExtract.contains(RI))
return;
- if (ExternalUsesAsExtractCostTmp.contains(Inst)) {
- InstructionCost ExtractCost = ExternalUsesAsExtractCostTmp.lookup(Inst);
- ExternalUsesAsExtractCost.try_emplace(RI, ExtractCost);
- }
- CouldBeExtract.try_emplace(RI, EI);
- CouldBeRemat.try_emplace(EI, RI);
+ DE.transferExtractCost(Inst, RI);
+ DE.logExtractRematPair(EI, RI);
if (User)
User->replaceUsesOfWith(EI, RI);
else
@@ -25902,8 +25982,8 @@ Value *BoUpSLP::vectorizeTree(
"Extractelements should not be replaced.");
Scalar->replaceAllUsesWith(NewInst);
}
- if (ExternalUsesAsExtract.contains(Scalar))
- DeferredScalarsToExtract[Scalar].emplace_back(Scalar, NewInst, User);
+ if (DE.ExternalUsesAsExtract.contains(Scalar))
+ DE.DeferredScalarsToExtract[Scalar].emplace_back(Scalar, NewInst, User);
if (IsDeferredScalar)
TrackDeferredExtract(dyn_cast<Instruction>(Scalar), NewInst, User);
@@ -26003,8 +26083,9 @@ Value *BoUpSLP::vectorizeTree(
!isa<StructType>(NewInst->getType())) {
User->replaceAllUsesWith(NewInst);
eraseInstruction(cast<Instruction>(User));
- } else if (ExternalUsesAsExtract.contains(Scalar)) {
- DeferredScalarsToExtract[Scalar].emplace_back(Scalar, NewInst, User);
+ } else if (DE.ExternalUsesAsExtract.contains(Scalar)) {
+ DE.DeferredScalarsToExtract[Scalar].emplace_back(Scalar, NewInst,
+ User);
} else {
User->replaceUsesOfWith(Scalar, NewInst);
}
@@ -26172,8 +26253,8 @@ Value *BoUpSLP::vectorizeTree(
continue;
if (!isa<Instruction>(Scalar) || Entry->isCopyableElement(Scalar))
continue;
- if (DeferredScalarsToExtract.contains(Scalar) ||
- CouldBeRemat.contains(Scalar))
+ if (DE.DeferredScalarsToExtract.contains(Scalar) ||
+ DE.CouldBeRemat.contains(Scalar))
continue;
#ifndef NDEBUG
Type *Ty = Scalar->getType();
@@ -26345,53 +26426,7 @@ Value *BoUpSLP::vectorizeTree(
return Vec;
}
-void BoUpSLP::emitDeferredExtracts() {
- SmallVector<DeferredExtractType> DeferredExtracts;
- for (const auto &Entry : DeferredScalarsToExtract)
- append_range(DeferredExtracts, Entry.second);
- for (const auto &DET : DeferredExtracts) {
- auto *UI = cast_or_null<Instruction>(DET.User);
- if (!DET.User) {
- DET.Scalar->replaceAllUsesWith(DET.NewInst);
- } else {
- if (isDeleted(UI))
- continue;
- DET.User->replaceUsesOfWith(DET.Scalar, DET.NewInst);
- LLVM_DEBUG(dbgs() << "SLP: Delayed replacement:" << *UI << ".\n");
- }
- }
- DeferredScalarsToExtract.clear();
- for (const auto &DET : DeferredExtracts) {
- LLVM_DEBUG(dbgs() << "SLP: \tErasing scalar:" << *DET.Scalar << ".\n");
- auto *I = cast<Instruction>(DET.Scalar);
- if (isDeleted(I))
- continue;
- assert((I->use_empty() || all_of(I->uses(),
- [&](Use &U) {
- return isDeleted(
- cast<Instruction>(U.getUser()));
- })) &&
- "trying to erase instruction with users.");
- eraseInstruction(I);
- }
- for (const auto &P : CouldBeExtract) {
- if (auto *Ext = P.second) {
- if (!isDeleted(Ext)) {
- unsigned NumUses = Ext->getNumUses();
- if (!NumUses) {
- eraseInstruction(Ext);
- } else if (NumUses == 1) {
- Value *User = Ext->uses().begin()->getUser();
- if (auto *CI = dyn_cast<CastInst>(User);
- CI && CI->getNumUses() == 0) {
- eraseInstruction(CI);
- eraseInstruction(Ext);
- }
- }
- }
- }
- }
-}
+void BoUpSLP::emitDeferredExtracts() { DE.emitDeferredExtracts(*this); }
void BoUpSLP::optimizeGatherSequence() {
LLVM_DEBUG(dbgs() << "SLP: Optimizing " << GatherShuffleExtractSeq.size()
@@ -27196,18 +27231,19 @@ void BoUpSLP::BlockScheduling::initScheduleData(Instruction *FromI,
ScheduleDataMap[I] = SD;
}
// Both an extract and its rematerialization ought to be scheduled together
- if (auto *EI = R.getCouldBeExtract().lookup(I))
+ if (auto *EI = DE.CouldBeExtract.lookup(I)) {
if (!ScheduleDataMap.lookup(EI))
ScheduleDataMap[EI] = SD;
- else if (auto *RI = R.getCouldBeRemat().lookup(I))
- if (!ScheduleDataMap.lookup(RI))
- ScheduleDataMap[RI] = SD;
+ } else if (auto *RI = DE.CouldBeRemat.lookup(I)) {
+ if (!ScheduleDataMap.lookup(RI))
+ ScheduleDataMap[RI] = SD;
+ }
bool IsSharedNode =
- R.getCouldBeExtract().contains(I) || R.getCouldBeRemat().contains(I);
+ DE.CouldBeExtract.contains(I) || DE.CouldBeRemat.contains(I);
assert((!isInSchedulingRegion(*SD) || IsSharedNode) &&
"new ScheduleData already in scheduling region");
if (!isInSchedulingRegion(*SD)) {
- if (auto *RI = R.getCouldBeRemat().lookup(I))
+ if (auto *RI = DE.CouldBeRemat.lookup(I))
SD->init(SchedulingRegionID, RI);
else
SD->init(SchedulingRegionID, I);
@@ -27804,7 +27840,7 @@ void BoUpSLP::scheduleBlock(const BoUpSLP &R, BlockScheduling *BS) {
if (ShouldMove)
PickedInst->moveAfter(LastScheduledInst->getPrevNode());
LastScheduledInst = PickedInst;
- if (auto *EI = CouldBeExtract.lookup(PickedInst)) {
+ if (auto *EI = DE.CouldBeExtract.lookup(PickedInst)) {
if (ShouldMove)
EI->moveAfter(LastScheduledInst->getPrevNode());
LastScheduledInst = EI;
@@ -31060,7 +31096,7 @@ class HorizontalReduction {
}
SmallVector<int> Mask;
if (isFixedVectorShuffle(CommonCandidates, Mask, AC,
- V.getCouldBeExtract())) {
+ V.getDE().CouldBeExtract)) {
++I;
Candidates.swap(CommonCandidates);
ShuffledExtracts = true;
@@ -33050,7 +33086,8 @@ bool SLPVectorizerPass::vectorizeInsertElementInst(InsertElementInst *IEI,
SmallVector<int> Mask;
if (!findBuildAggregate(IEI, TTI, BuildVectorOpds, BuildVectorInsts, R) ||
(all_of(BuildVectorOpds, IsaPred<ExtractElementInst, UndefValue>) &&
- isFixedVectorShuffle(BuildVectorOpds, Mask, AC, R.getCouldBeExtract())))
+ isFixedVectorShuffle(BuildVectorOpds, Mask, AC,
+ R.getDE().CouldBeExtract)))
return false;
if (MaxVFOnly && BuildVectorInsts.size() == 2) {
>From c53ad5f67af220063a64294907aff637683a1e42 Mon Sep 17 00:00:00 2001
From: Ryan Buchner <buchner.ryan at gmail.com>
Date: Tue, 4 Aug 2026 17:16:20 -0700
Subject: [PATCH 11/22] [SLP] Move isDeferredExtractable() into
DeferredExtractTracker
---
llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp | 16 ++++++++++++++--
.../SLPVectorizer/SLPCompatibilityAnalysis.cpp | 6 ------
.../SLPVectorizer/SLPCompatibilityAnalysis.h | 6 ------
3 files changed, 14 insertions(+), 14 deletions(-)
diff --git a/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp b/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp
index cec5e64c04342..4911b4fc82fd6 100644
--- a/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp
+++ b/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp
@@ -3474,6 +3474,16 @@ class slpvectorizer::BoUpSLP {
ItCost->second = std::min(ItCost->second, Delta);
}
}
+
+ /// \returns true if \p Scalar can stay rematerialized during vectorization
+ /// and be switched to an extractelement later. For now, only support load
+ /// and load-cast pairs since those are simpler to handle and are commonly
+ /// profitable to rematerialize
+ static bool isDeferredExtractable(Value *Scalar) {
+ if (isa<VectorType>(Scalar->getType()))
+ return false;
+ return isa<LoadInst>(Scalar);
+ }
};
/// Vectorize a single entry in the tree.
@@ -25973,7 +25983,8 @@ Value *BoUpSLP::vectorizeTree(
} else {
Builder.SetInsertPoint(&F->getEntryBlock(), F->getEntryBlock().begin());
}
- bool IsDeferredScalar = isDeferredExtractable(Scalar);
+ bool IsDeferredScalar =
+ DeferredExtractTracker::isDeferredExtractable(Scalar);
Value *NewInst = ExtractAndExtendIfNeeded(Vec);
// Required to update internally referenced instructions.
if (Scalar != NewInst) {
@@ -26076,7 +26087,8 @@ Value *BoUpSLP::vectorizeTree(
}
} else {
Builder.SetInsertPoint(cast<Instruction>(User));
- bool IsDeferredScalar = isDeferredExtractable(Scalar);
+ bool IsDeferredScalar =
+ DeferredExtractTracker::isDeferredExtractable(Scalar);
Value *NewInst = ExtractAndExtendIfNeeded(Vec);
if (isa<StructType>(Scalar->getType()) &&
isa_and_nonnull<ExtractValueInst>(User) &&
diff --git a/llvm/lib/Transforms/Vectorize/SLPVectorizer/SLPCompatibilityAnalysis.cpp b/llvm/lib/Transforms/Vectorize/SLPVectorizer/SLPCompatibilityAnalysis.cpp
index d588446b8e4d1..7a00f6d16037f 100644
--- a/llvm/lib/Transforms/Vectorize/SLPVectorizer/SLPCompatibilityAnalysis.cpp
+++ b/llvm/lib/Transforms/Vectorize/SLPVectorizer/SLPCompatibilityAnalysis.cpp
@@ -26,12 +26,6 @@ using namespace llvm;
namespace llvm::slpvectorizer {
-bool isDeferredExtractable(Value *Scalar) {
- if (isa<VectorType>(Scalar->getType()))
- return false;
- return isa<LoadInst>(Scalar);
-}
-
bool isValidForAlternation(unsigned Opcode) {
return !Instruction::isIntDivRem(Opcode);
}
diff --git a/llvm/lib/Transforms/Vectorize/SLPVectorizer/SLPCompatibilityAnalysis.h b/llvm/lib/Transforms/Vectorize/SLPVectorizer/SLPCompatibilityAnalysis.h
index c8a311103742f..156418fe39c44 100644
--- a/llvm/lib/Transforms/Vectorize/SLPVectorizer/SLPCompatibilityAnalysis.h
+++ b/llvm/lib/Transforms/Vectorize/SLPVectorizer/SLPCompatibilityAnalysis.h
@@ -31,12 +31,6 @@ class Value;
namespace llvm::slpvectorizer {
-/// \returns true if \p Scalar can stay rematerialized during vectorization and
-/// be switched to an extractelement later.
-/// For now, only support load and load-cast pairs since those are simpler to
-/// handle and are commonly profitable to rematerialize
-bool isDeferredExtractable(Value *Scalar);
-
/// \returns true if \p Opcode is allowed as part of the main/alternate
/// instruction for SLP vectorization.
///
>From 74d2f680ae0e7991e979abe128338041dbaf6c85 Mon Sep 17 00:00:00 2001
From: Ryan Buchner <buchner.ryan at gmail.com>
Date: Tue, 4 Aug 2026 14:59:17 -0700
Subject: [PATCH 12/22] [SLP] Abstract away CouldBeExtract when iterating over
operands of ScheduleData
---
.../Transforms/Vectorize/SLPVectorizer.cpp | 55 +++++++++++--------
1 file changed, 32 insertions(+), 23 deletions(-)
diff --git a/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp b/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp
index 4911b4fc82fd6..73c05c4fe43e7 100644
--- a/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp
+++ b/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp
@@ -4672,6 +4672,10 @@ class slpvectorizer::BoUpSLP {
friend class ScheduleData;
friend class ScheduleCopyableData;
+ using OpRange = llvm::User::op_range;
+ using operand_range =
+ llvm::detail::concat_range<llvm::Use, OpRange, OpRange>;
+
protected:
enum class Kind { ScheduleData, ScheduleBundle, ScheduleCopyableData };
Kind getKind() const { return K; }
@@ -4735,6 +4739,14 @@ class slpvectorizer::BoUpSLP {
return cast<ScheduleCopyableData>(this)->getInst();
}
+ operand_range operands() {
+ if (auto *SD = dyn_cast<ScheduleData>(this))
+ return SD->operands();
+ return llvm::concat<llvm::Use>(
+ getInst()->operands(),
+ OpRange{getInst()->op_end(), getInst()->op_end()});
+ }
+
/// Gets/sets if the bundle is scheduled.
bool isScheduled() const { return IsScheduled; }
void setScheduled(bool Scheduled) { IsScheduled = Scheduled; }
@@ -4786,6 +4798,7 @@ class slpvectorizer::BoUpSLP {
SchedulingRegionID = BlockSchedulingRegionID;
clearDependencies();
Inst = I;
+ ExtractInst = nullptr;
}
/// Verify basic self consistency properties
@@ -4859,6 +4872,15 @@ class slpvectorizer::BoUpSLP {
/// Gets the instruction.
Instruction *getInst() const { return Inst; }
+ operand_range operands() {
+ if (ExtractInst)
+ return llvm::concat<Use>(Inst->operands(), ExtractInst->operands());
+ return llvm::concat<llvm::Use>(Inst->operands(),
+ OpRange{Inst->op_end(), Inst->op_end()});
+ }
+
+ void setExtractInst(Instruction *I) { ExtractInst = I; }
+
/// Gets the list of memory dependencies.
ArrayRef<ScheduleData *> getMemoryDependencies() const {
return MemoryDependencies;
@@ -4918,6 +4940,8 @@ class slpvectorizer::BoUpSLP {
/// for scheduling.
/// Note that this is negative as long as Dependencies is not calculated.
int UnscheduledDeps = InvalidDeps;
+
+ Instruction *ExtractInst = nullptr;
};
#ifndef NDEBUG
@@ -5584,7 +5608,7 @@ class slpvectorizer::BoUpSLP {
// Copyable data is used only once (uses itself).
TotalOpCount = OperandsUses[In] = 1;
} else {
- auto HandleOneOp = [&](const Use &U) {
+ for (const Use &U : BundleMember->operands()) {
if (auto *I = dyn_cast<Instruction>(U.get())) {
auto Res = OperandsUses.try_emplace(I, 0);
unsigned ExtraDeps = 1;
@@ -5600,15 +5624,6 @@ class slpvectorizer::BoUpSLP {
Res.first->getSecond() += ExtraDeps;
TotalOpCount += ExtraDeps;
}
- };
- for (const Use &U : In->operands())
- HandleOneOp(U);
- // Track the operands from the extractelement copy
- // as well to make sure the dependency on the vector
- // is tracked
- if (auto *EI = DE.CouldBeExtract.lookup(In)) {
- for (const Use &U : EI->operands())
- HandleOneOp(U);
}
}
// Decrement the unscheduled counter and insert to ready list if
@@ -5813,23 +5828,13 @@ class slpvectorizer::BoUpSLP {
} else {
// If BundleMember is a stand-alone instruction, no operand reordering
// has taken place, so we directly access its operands.
- for (Use &U : BundleMember->getInst()->operands()) {
+ for (Use &U : BundleMember->operands()) {
if (auto *I = dyn_cast<Instruction>(U.get())) {
LLVM_DEBUG(dbgs()
<< "SLP: check for readiness (def): " << *I << "\n");
DecrUnschedForInst(BundleMember->getInst(), U.getOperandNo(), I);
}
}
- if (auto *EI = DE.CouldBeExtract.lookup(BundleMember->getInst())) {
- for (Use &U : EI->operands()) {
- if (auto *I = dyn_cast<Instruction>(U.get())) {
- LLVM_DEBUG(dbgs() << "SLP: check for readiness (def): " << *I
- << "\n");
- DecrUnschedForInst(BundleMember->getInst(), U.getOperandNo(),
- I);
- }
- }
- }
}
// Handle the memory dependencies.
auto *SD = dyn_cast<ScheduleData>(BundleMember);
@@ -27255,10 +27260,14 @@ void BoUpSLP::BlockScheduling::initScheduleData(Instruction *FromI,
assert((!isInSchedulingRegion(*SD) || IsSharedNode) &&
"new ScheduleData already in scheduling region");
if (!isInSchedulingRegion(*SD)) {
- if (auto *RI = DE.CouldBeRemat.lookup(I))
+ if (auto *RI = DE.CouldBeRemat.lookup(I)) {
SD->init(SchedulingRegionID, RI);
- else
+ SD->setExtractInst(I);
+ } else {
SD->init(SchedulingRegionID, I);
+ if (auto *EI = DE.CouldBeExtract.lookup(I))
+ SD->setExtractInst(EI);
+ }
}
auto CanIgnoreLoad = [](const Instruction *I) {
>From 380f0ac80dde767e786967cf2eb8819f8a7a4597 Mon Sep 17 00:00:00 2001
From: Ryan Buchner <buchner.ryan at gmail.com>
Date: Tue, 4 Aug 2026 16:33:36 -0700
Subject: [PATCH 13/22] [SLP] Cleanup code for DRY
---
llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp | 16 ++++++----------
1 file changed, 6 insertions(+), 10 deletions(-)
diff --git a/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp b/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp
index 73c05c4fe43e7..65d74ad244df8 100644
--- a/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp
+++ b/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp
@@ -4645,8 +4645,8 @@ class slpvectorizer::BoUpSLP {
/// Deferred extract/rematerialization state for the current function pass.
DeferredExtractTracker DE;
- /// A list of scalar to be extracted without specific user because of too many
- /// uses.
+ /// A list of scalars to be extracted without specific user because of too
+ /// many uses.
SmallPtrSet<Value *, 4> ExternalUsesWithNonUsers;
/// Values used only by @llvm.assume calls.
@@ -14129,18 +14129,14 @@ uint64_t BoUpSLP::getNumVectorInsts(bool HasTreeLoop) {
DE.CouldBeExtract.contains(V);
})) {
for (Value *V : TE.Scalars) {
- if (auto *EE = dyn_cast<ExtractElementInst>(V)) {
+ auto *EE = dyn_cast<ExtractElementInst>(V);
+ if (!EE)
+ EE = DE.CouldBeExtract.lookup(V);
+ if (EE) {
uint64_t &VecScale =
GatherExtractSourceVecs.try_emplace(EE->getVectorOperand(), 0)
.first->second;
VecScale = std::max(VecScale, Scale);
- } else if (DE.CouldBeExtract.contains(V)) {
- if (auto *EE = DE.CouldBeExtract.lookup(V)) {
- uint64_t &VecScale =
- GatherExtractSourceVecs.try_emplace(EE->getVectorOperand(), 0)
- .first->second;
- VecScale = std::max(VecScale, Scale);
- }
}
}
} else {
>From b6e09619e04b85964c48d04d42f882f6c9b2448d Mon Sep 17 00:00:00 2001
From: Ryan Buchner <buchner.ryan at gmail.com>
Date: Wed, 5 Aug 2026 11:51:23 -0700
Subject: [PATCH 14/22] Adjust ordering logic in final schedule
---
llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp | 7 ++++---
1 file changed, 4 insertions(+), 3 deletions(-)
diff --git a/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp b/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp
index 65d74ad244df8..da02a0eeffabc 100644
--- a/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp
+++ b/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp
@@ -27853,12 +27853,13 @@ void BoUpSLP::scheduleBlock(const BoUpSLP &R, BlockScheduling *BS) {
} else {
auto *SD = cast<ScheduleData>(Picked);
Instruction *PickedInst = SD->getInst();
- bool ShouldMove = PickedInst->getNextNode() != LastScheduledInst;
- if (ShouldMove)
+ if (PickedInst->getNextNode() != LastScheduledInst)
PickedInst->moveAfter(LastScheduledInst->getPrevNode());
LastScheduledInst = PickedInst;
if (auto *EI = DE.CouldBeExtract.lookup(PickedInst)) {
- if (ShouldMove)
+ // Keep deferred extract/remat instructions contiguous in the scheduled
+ // suffix so the scheduling frontier always points at a valid anchor.
+ if (EI->getNextNode() != LastScheduledInst)
EI->moveAfter(LastScheduledInst->getPrevNode());
LastScheduledInst = EI;
}
>From ada763fc5315731635bd21f4e28c4112a8f2442d Mon Sep 17 00:00:00 2001
From: Ryan Buchner <buchner.ryan at gmail.com>
Date: Wed, 5 Aug 2026 13:33:02 -0700
Subject: [PATCH 15/22] Add assertion
---
llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp | 1 +
1 file changed, 1 insertion(+)
diff --git a/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp b/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp
index da02a0eeffabc..3b5e70887c879 100644
--- a/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp
+++ b/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp
@@ -27857,6 +27857,7 @@ void BoUpSLP::scheduleBlock(const BoUpSLP &R, BlockScheduling *BS) {
PickedInst->moveAfter(LastScheduledInst->getPrevNode());
LastScheduledInst = PickedInst;
if (auto *EI = DE.CouldBeExtract.lookup(PickedInst)) {
+ assert(EI->getParent() == PickedInst->getParent() && "Expected extract to be in same block as rematerialize version");
// Keep deferred extract/remat instructions contiguous in the scheduled
// suffix so the scheduling frontier always points at a valid anchor.
if (EI->getNextNode() != LastScheduledInst)
>From 7ffd25bcb794861ddf8f1cfd62993dabbf633e4c Mon Sep 17 00:00:00 2001
From: Ryan Buchner <buchner.ryan at gmail.com>
Date: Wed, 5 Aug 2026 14:57:19 -0700
Subject: [PATCH 16/22] [SLP] Encapsulate deferred extract/remat map access
---
.../Transforms/Vectorize/SLPVectorizer.cpp | 85 +++++++++++--------
1 file changed, 49 insertions(+), 36 deletions(-)
diff --git a/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp b/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp
index 3b5e70887c879..5f9887731ff02 100644
--- a/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp
+++ b/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp
@@ -507,11 +507,11 @@ getMaskedDivRemCost(const TargetTransformInfo &TTI, unsigned Opcode,
/// ShuffleVectorInst/getShuffleCost?
static std::optional<TargetTransformInfo::ShuffleKind> isFixedVectorShuffle(
ArrayRef<Value *> VL, SmallVectorImpl<int> &Mask, AssumptionCache *AC,
- const DenseMap<Value *, ExtractElementInst *> &CouldBeExtract) {
- auto GetExtract = [&CouldBeExtract](Value *V) -> ExtractElementInst * {
+ function_ref<ExtractElementInst *(Value *)> LookupExtract) {
+ auto GetExtract = [&LookupExtract](Value *V) -> ExtractElementInst * {
if (auto *EI = dyn_cast<ExtractElementInst>(V))
return EI;
- return CouldBeExtract.lookup(V);
+ return LookupExtract(V);
};
const auto *It = find_if(VL, [&](Value *V) { return GetExtract(V); });
if (It == VL.end())
@@ -3159,7 +3159,7 @@ class slpvectorizer::BoUpSLP {
void emitDeferredExtracts();
// Get handle for the DeferredExtractTracker
- const DeferredExtractTracker &getDE() { return DE; }
+ const DeferredExtractTracker &getDE() const { return DE; }
~BoUpSLP();
@@ -3300,10 +3300,15 @@ class slpvectorizer::BoUpSLP {
: Scalar(Scalar), NewInst(NewInst), User(User) {}
};
- /// Map a scalar rematerialized form to the matching extractelement.
- DenseMap<Value *, ExtractElementInst *> CouldBeExtract;
- /// Reverse mapping used by scheduling: extractelement back to remat scalar.
- DenseMap<Value *, Instruction *> CouldBeRemat;
+ ExtractElementInst *lookupExtract(Value *V) const {
+ return CouldBeExtract.lookup(V);
+ }
+
+ Instruction *lookupRemat(Value *V) const { return CouldBeRemat.lookup(V); }
+
+ bool hasExtract(Value *V) const { return CouldBeExtract.contains(V); }
+
+ bool hasRemat(Value *V) const { return CouldBeRemat.contains(V); }
/// Cases where extraction is estimated as more profitable but want to
/// delay extraction to allow for better vectorization in the interim.
@@ -3484,6 +3489,12 @@ class slpvectorizer::BoUpSLP {
return false;
return isa<LoadInst>(Scalar);
}
+
+ private:
+ /// Map a scalar rematerialized form to the matching extractelement.
+ DenseMap<Value *, ExtractElementInst *> CouldBeExtract;
+ /// Reverse mapping used by scheduling: extractelement back to remat scalar.
+ DenseMap<Value *, Instruction *> CouldBeRemat;
};
/// Vectorize a single entry in the tree.
@@ -4207,7 +4218,7 @@ class slpvectorizer::BoUpSLP {
if (UserTreeIdx.UserTE)
for (auto &VPtr :
UserTreeIdx.UserTE->getOperand(UserTreeIdx.EdgeIdx))
- if (auto *NewVPtr = DE.CouldBeExtract.lookup(VPtr))
+ if (auto *NewVPtr = DE.lookupExtract(VPtr))
VPtr = NewVPtr;
}
}
@@ -5742,7 +5753,7 @@ class slpvectorizer::BoUpSLP {
<< *I << "\n");
// The scheduling node works on the rematerialized version
// of the extract
- if (auto *RI = DE.CouldBeRemat.lookup(I))
+ if (auto *RI = DE.lookupRemat(I))
I = RI;
DecrUnschedForInst(
I, Bundle->getTreeEntry(), OpIdx, Checked,
@@ -14126,12 +14137,12 @@ uint64_t BoUpSLP::getNumVectorInsts(bool HasTreeLoop) {
// entries and count once at the end.
if (all_of(TE.Scalars, [&](Value *V) {
return isa<ExtractElementInst, UndefValue, Constant>(V) ||
- DE.CouldBeExtract.contains(V);
+ DE.hasExtract(V);
})) {
for (Value *V : TE.Scalars) {
auto *EE = dyn_cast<ExtractElementInst>(V);
if (!EE)
- EE = DE.CouldBeExtract.lookup(V);
+ EE = DE.lookupExtract(V);
if (EE) {
uint64_t &VecScale =
GatherExtractSourceVecs.try_emplace(EE->getVectorOperand(), 0)
@@ -16387,7 +16398,7 @@ class BoUpSLP::ShuffleCostEstimator : public BaseShuffleAnalysis {
Value *adjustExtracts(const TreeEntry *E, MutableArrayRef<int> Mask,
ArrayRef<std::optional<TTI::ShuffleKind>> ShuffleKinds,
unsigned NumParts, bool &UseVecBaseAsInput,
- const DenseMap<Value *, Instruction *> &CouldBeRemat) {
+ function_ref<Instruction *(Value *)> LookupRemat) {
UseVecBaseAsInput = false;
if (Mask.empty())
return nullptr;
@@ -16464,13 +16475,13 @@ class BoUpSLP::ShuffleCostEstimator : public BaseShuffleAnalysis {
// rematerialized, the uses are stored by the rematerialized
// instruction.
bool OneUse;
- if (auto *RI = CouldBeRemat.lookup(EE))
+ if (auto *RI = LookupRemat(EE))
OneUse = RI->hasOneUse();
else
OneUse = EE->hasOneUse();
if (OneUse || !PrevNodeFound) {
Instruction *Ext;
- if (auto *RI = CouldBeRemat.lookup(EE)) {
+ if (auto *RI = LookupRemat(EE)) {
Ext = RI->user_back();
} else {
Ext = EE->user_back();
@@ -18525,7 +18536,9 @@ bool BoUpSLP::isFullyVectorizableTinyTree(bool ForReduction) const {
(((TE->hasState() &&
TE->getOpcode() == Instruction::ExtractElement) ||
all_of(TE->Scalars, IsaPred<ExtractElementInst, UndefValue>)) &&
- isFixedVectorShuffle(TE->Scalars, Mask, AC, DE.CouldBeExtract)) ||
+ isFixedVectorShuffle(
+ TE->Scalars, Mask, AC,
+ [&](Value *V) { return DE.lookupExtract(V); })) ||
(TE->hasState() && TE->getOpcode() == Instruction::Load &&
!TE->isAltShuffle()) ||
any_of(TE->Scalars, IsaPred<LoadInst>));
@@ -20942,7 +20955,8 @@ BoUpSLP::tryToGatherSingleRegisterExtractElements(
// Check that gather of extractelements can be represented as just a
// shuffle of a single/two vectors the scalars are extracted from.
std::optional<TTI::ShuffleKind> Res =
- isFixedVectorShuffle(GatheredExtracts, Mask, AC, DE.CouldBeExtract);
+ isFixedVectorShuffle(GatheredExtracts, Mask, AC,
+ [&](Value *V) { return DE.lookupExtract(V); });
if (!Res || all_of(Mask, equal_to(PoisonMaskElem))) {
// TODO: try to check other subsets if possible.
// Restore the original VL if attempt was not successful.
@@ -22489,7 +22503,7 @@ class BoUpSLP::ShuffleInstructionBuilder final : public BaseShuffleAnalysis {
Value *adjustExtracts(const TreeEntry *E, MutableArrayRef<int> Mask,
ArrayRef<std::optional<TTI::ShuffleKind>> ShuffleKinds,
unsigned NumParts, bool &UseVecBaseAsInput,
- const DenseMap<Value *, Instruction *> &CouldBeRemat) {
+ function_ref<Instruction *(Value *)>) {
UseVecBaseAsInput = false;
SmallPtrSet<Value *, 4> UniqueBases;
Value *VecBase = nullptr;
@@ -23002,7 +23016,7 @@ ResTy BoUpSLP::processBuildVector(const TreeEntry *E, Type *ScalarTy,
auto GetGatheredExtract = [&](unsigned Idx) -> ExtractElementInst * {
if (auto *EI = dyn_cast<ExtractElementInst>(StoredGS[Idx]))
return EI;
- return DE.CouldBeExtract.lookup(StoredGS[Idx]);
+ return DE.lookupExtract(StoredGS[Idx]);
};
if (!all_of(GatheredScalars, IsaPred<UndefValue>)) {
// Check for gathered extracts.
@@ -23030,7 +23044,7 @@ ResTy BoUpSLP::processBuildVector(const TreeEntry *E, Type *ScalarTy,
}
if (Value *VecBase = ShuffleBuilder.adjustExtracts(
E, ExtractMask, ExtractShuffles, NumParts, UseVecBaseAsInput,
- DE.CouldBeRemat)) {
+ [&](Value *V) { return DE.lookupRemat(V); })) {
ExtractVecBase = VecBase;
if (auto *VecBaseTy = dyn_cast<FixedVectorType>(VecBase->getType()))
if (VF == VecBaseTy->getNumElements() &&
@@ -25907,7 +25921,7 @@ Value *BoUpSLP::vectorizeTree(
auto *EI = dyn_cast<ExtractElementInst>(ReplacedExtract);
auto *RI = dyn_cast<Instruction>(Replacement);
assert(EI && RI && "Expected to find underlying instructions");
- if (DE.CouldBeExtract.contains(RI))
+ if (DE.hasExtract(RI))
return;
DE.transferRematCost(Inst, EI);
DE.logExtractRematPair(EI, RI);
@@ -25917,8 +25931,7 @@ Value *BoUpSLP::vectorizeTree(
auto *EI = dyn_cast<ExtractElementInst>(Replacement);
auto *RI = Inst;
assert(EI && RI && "Expected to find underlying instructions");
- if (!DE.ExternalUsesAsExtract.contains(RI) ||
- DE.CouldBeExtract.contains(RI))
+ if (!DE.ExternalUsesAsExtract.contains(RI) || DE.hasExtract(RI))
return;
DE.transferExtractCost(Inst, RI);
DE.logExtractRematPair(EI, RI);
@@ -26266,8 +26279,7 @@ Value *BoUpSLP::vectorizeTree(
continue;
if (!isa<Instruction>(Scalar) || Entry->isCopyableElement(Scalar))
continue;
- if (DE.DeferredScalarsToExtract.contains(Scalar) ||
- DE.CouldBeRemat.contains(Scalar))
+ if (DE.DeferredScalarsToExtract.contains(Scalar) || DE.hasRemat(Scalar))
continue;
#ifndef NDEBUG
Type *Ty = Scalar->getType();
@@ -27244,24 +27256,23 @@ void BoUpSLP::BlockScheduling::initScheduleData(Instruction *FromI,
ScheduleDataMap[I] = SD;
}
// Both an extract and its rematerialization ought to be scheduled together
- if (auto *EI = DE.CouldBeExtract.lookup(I)) {
+ if (auto *EI = DE.lookupExtract(I)) {
if (!ScheduleDataMap.lookup(EI))
ScheduleDataMap[EI] = SD;
- } else if (auto *RI = DE.CouldBeRemat.lookup(I)) {
+ } else if (auto *RI = DE.lookupRemat(I)) {
if (!ScheduleDataMap.lookup(RI))
ScheduleDataMap[RI] = SD;
}
- bool IsSharedNode =
- DE.CouldBeExtract.contains(I) || DE.CouldBeRemat.contains(I);
+ bool IsSharedNode = DE.hasExtract(I) || DE.hasRemat(I);
assert((!isInSchedulingRegion(*SD) || IsSharedNode) &&
"new ScheduleData already in scheduling region");
if (!isInSchedulingRegion(*SD)) {
- if (auto *RI = DE.CouldBeRemat.lookup(I)) {
+ if (auto *RI = DE.lookupRemat(I)) {
SD->init(SchedulingRegionID, RI);
SD->setExtractInst(I);
} else {
SD->init(SchedulingRegionID, I);
- if (auto *EI = DE.CouldBeExtract.lookup(I))
+ if (auto *EI = DE.lookupExtract(I))
SD->setExtractInst(EI);
}
}
@@ -27856,7 +27867,7 @@ void BoUpSLP::scheduleBlock(const BoUpSLP &R, BlockScheduling *BS) {
if (PickedInst->getNextNode() != LastScheduledInst)
PickedInst->moveAfter(LastScheduledInst->getPrevNode());
LastScheduledInst = PickedInst;
- if (auto *EI = DE.CouldBeExtract.lookup(PickedInst)) {
+ if (auto *EI = DE.lookupExtract(PickedInst)) {
assert(EI->getParent() == PickedInst->getParent() && "Expected extract to be in same block as rematerialize version");
// Keep deferred extract/remat instructions contiguous in the scheduled
// suffix so the scheduling frontier always points at a valid anchor.
@@ -31114,8 +31125,9 @@ class HorizontalReduction {
TrackedToOrig.push_back(RV);
}
SmallVector<int> Mask;
- if (isFixedVectorShuffle(CommonCandidates, Mask, AC,
- V.getDE().CouldBeExtract)) {
+ if (isFixedVectorShuffle(CommonCandidates, Mask, AC, [&](Value *Val) {
+ return V.getDE().lookupExtract(Val);
+ })) {
++I;
Candidates.swap(CommonCandidates);
ShuffledExtracts = true;
@@ -33105,8 +33117,9 @@ bool SLPVectorizerPass::vectorizeInsertElementInst(InsertElementInst *IEI,
SmallVector<int> Mask;
if (!findBuildAggregate(IEI, TTI, BuildVectorOpds, BuildVectorInsts, R) ||
(all_of(BuildVectorOpds, IsaPred<ExtractElementInst, UndefValue>) &&
- isFixedVectorShuffle(BuildVectorOpds, Mask, AC,
- R.getDE().CouldBeExtract)))
+ isFixedVectorShuffle(BuildVectorOpds, Mask, AC, [&](Value *V) {
+ return R.getDE().lookupExtract(V);
+ })))
return false;
if (MaxVFOnly && BuildVectorInsts.size() == 2) {
>From 5a465158f99928a7105874e4bab10f32a1014ebe Mon Sep 17 00:00:00 2001
From: Ryan Buchner <buchner.ryan at gmail.com>
Date: Thu, 6 Aug 2026 14:08:00 -0700
Subject: [PATCH 17/22] [SLP] Cleanup/refactor
---
.../Transforms/Vectorize/SLPVectorizer.cpp | 58 ++++++++++++++-----
1 file changed, 43 insertions(+), 15 deletions(-)
diff --git a/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp b/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp
index 5f9887731ff02..53657e3d6b566 100644
--- a/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp
+++ b/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp
@@ -1481,7 +1481,7 @@ class slpvectorizer::BoUpSLP {
CompressEntryToData.clear();
ExternalUses.clear();
ExternalUsesAsOriginalScalar.clear();
- DE.clearForDeleteTree();
+ DE.clearTreeState();
ExternalUsesWithNonUsers.clear();
RTChecks.clear();
HasRuntimeCheckableBlockers = false;
@@ -3327,12 +3327,41 @@ class slpvectorizer::BoUpSLP {
DenseMap<Value *, SmallVector<DeferredExtractType, 2>>
DeferredScalarsToExtract;
- void clearForDeleteTree() {
+ void clearTreeState() {
ExternalUsesAsExtract.clear();
ExternalUsesAsRematCostTmp.clear();
ExternalUsesAsExtractCostTmp.clear();
}
+ bool isExternalUseAsExtract(Value *V) const {
+ return ExternalUsesAsExtract.contains(V);
+ }
+
+ void addDeferredScalarToExtract(Value *Scalar, Value *NewInst,
+ llvm::User *User) {
+ DeferredScalarsToExtract[Scalar].emplace_back(Scalar, NewInst, User);
+ }
+
+ bool hasDeferredScalarToExtract(Value *Scalar) const {
+ return DeferredScalarsToExtract.contains(Scalar);
+ }
+
+ std::optional<InstructionCost>
+ getExternalUsesAsExtractCost(const Value *V) const {
+ auto It = ExternalUsesAsExtractCost.find(V);
+ if (It == ExternalUsesAsExtractCost.end())
+ return std::nullopt;
+ return It->second;
+ }
+
+ std::optional<InstructionCost>
+ getExternalUsesAsRematCost(const Value *V) const {
+ auto It = ExternalUsesAsRematCost.find(V);
+ if (It == ExternalUsesAsRematCost.end())
+ return std::nullopt;
+ return It->second;
+ }
+
void eraseInstruction(Instruction *I, BoUpSLP &R) {
if (auto It = DeferredScalarsToExtract.find(I);
It != DeferredScalarsToExtract.end()) {
@@ -17362,14 +17391,14 @@ BoUpSLP::getEntryCost(const TreeEntry *E, ArrayRef<Value *> VectorizedVals,
if (UsedScalars.test(I))
continue;
if (auto *Inst = dyn_cast<Instruction>(UniqueValues[I])) {
- if (auto It = DE.ExternalUsesAsExtractCost.find(Inst);
- It != DE.ExternalUsesAsExtractCost.end()) {
- RematAdjustment += It->second;
+ if (std::optional<InstructionCost> Cost =
+ DE.getExternalUsesAsExtractCost(Inst)) {
+ RematAdjustment += *Cost;
continue;
}
- if (auto It = DE.ExternalUsesAsRematCost.find(Inst);
- It != DE.ExternalUsesAsRematCost.end()) {
- RematAdjustment += It->second;
+ if (std::optional<InstructionCost> Cost =
+ DE.getExternalUsesAsRematCost(Inst)) {
+ RematAdjustment += *Cost;
continue;
}
}
@@ -25931,7 +25960,7 @@ Value *BoUpSLP::vectorizeTree(
auto *EI = dyn_cast<ExtractElementInst>(Replacement);
auto *RI = Inst;
assert(EI && RI && "Expected to find underlying instructions");
- if (!DE.ExternalUsesAsExtract.contains(RI) || DE.hasExtract(RI))
+ if (!DE.isExternalUseAsExtract(RI) || DE.hasExtract(RI))
return;
DE.transferExtractCost(Inst, RI);
DE.logExtractRematPair(EI, RI);
@@ -26007,8 +26036,8 @@ Value *BoUpSLP::vectorizeTree(
"Extractelements should not be replaced.");
Scalar->replaceAllUsesWith(NewInst);
}
- if (DE.ExternalUsesAsExtract.contains(Scalar))
- DE.DeferredScalarsToExtract[Scalar].emplace_back(Scalar, NewInst, User);
+ if (DE.isExternalUseAsExtract(Scalar))
+ DE.addDeferredScalarToExtract(Scalar, NewInst, User);
if (IsDeferredScalar)
TrackDeferredExtract(dyn_cast<Instruction>(Scalar), NewInst, User);
@@ -26109,9 +26138,8 @@ Value *BoUpSLP::vectorizeTree(
!isa<StructType>(NewInst->getType())) {
User->replaceAllUsesWith(NewInst);
eraseInstruction(cast<Instruction>(User));
- } else if (DE.ExternalUsesAsExtract.contains(Scalar)) {
- DE.DeferredScalarsToExtract[Scalar].emplace_back(Scalar, NewInst,
- User);
+ } else if (DE.isExternalUseAsExtract(Scalar)) {
+ DE.addDeferredScalarToExtract(Scalar, NewInst, User);
} else {
User->replaceUsesOfWith(Scalar, NewInst);
}
@@ -26279,7 +26307,7 @@ Value *BoUpSLP::vectorizeTree(
continue;
if (!isa<Instruction>(Scalar) || Entry->isCopyableElement(Scalar))
continue;
- if (DE.DeferredScalarsToExtract.contains(Scalar) || DE.hasRemat(Scalar))
+ if (DE.hasDeferredScalarToExtract(Scalar) || DE.hasRemat(Scalar))
continue;
#ifndef NDEBUG
Type *Ty = Scalar->getType();
>From d6c30bc67f0ec8de432bba0dc9d65ff972fcca9c Mon Sep 17 00:00:00 2001
From: Ryan Buchner <buchner.ryan at gmail.com>
Date: Thu, 6 Aug 2026 17:00:50 -0700
Subject: [PATCH 18/22] [SLP] Code cleanup
---
llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp | 7 ++-----
1 file changed, 2 insertions(+), 5 deletions(-)
diff --git a/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp b/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp
index 53657e3d6b566..d6cd70a0eb3d5 100644
--- a/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp
+++ b/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp
@@ -25778,8 +25778,7 @@ Value *BoUpSLP::vectorizeTree(
Value *Vec = E->VectorizedValue;
assert(Vec && "Can't find vectorizable value");
- bool ExtractAnyways = false;
- auto ExtractAndExtendIfNeeded = [&](Value *Vec) {
+ auto ExtractAndExtendIfNeeded = [&](Value *Vec, bool ExtractAnyways = false) {
if (isa<InsertValueInst>(Scalar))
return Vec;
if (Scalar->getType() != Vec->getType()) {
@@ -25945,8 +25944,7 @@ Value *BoUpSLP::vectorizeTree(
if (!Inst)
return;
if (ExternalUsesAsOriginalScalar.contains(Inst)) {
- ExtractAnyways = true;
- Value *ReplacedExtract = ExtractAndExtendIfNeeded(Vec);
+ Value *ReplacedExtract = ExtractAndExtendIfNeeded(Vec, /*ExtractAnyways*/true);
auto *EI = dyn_cast<ExtractElementInst>(ReplacedExtract);
auto *RI = dyn_cast<Instruction>(Replacement);
assert(EI && RI && "Expected to find underlying instructions");
@@ -25954,7 +25952,6 @@ Value *BoUpSLP::vectorizeTree(
return;
DE.transferRematCost(Inst, EI);
DE.logExtractRematPair(EI, RI);
- ExtractAnyways = false;
return;
}
auto *EI = dyn_cast<ExtractElementInst>(Replacement);
>From 95210a12b5130bf6850c6350b40c6c77776c4a2e Mon Sep 17 00:00:00 2001
From: Ryan Buchner <buchner.ryan at gmail.com>
Date: Thu, 6 Aug 2026 17:27:28 -0700
Subject: [PATCH 19/22] Add extra gaurdrails for co-locality
---
llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp | 4 ++++
1 file changed, 4 insertions(+)
diff --git a/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp b/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp
index d6cd70a0eb3d5..7fc9e7ae2cbc8 100644
--- a/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp
+++ b/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp
@@ -3448,6 +3448,10 @@ class slpvectorizer::BoUpSLP {
}
void logExtractRematPair(ExtractElementInst *EI, Instruction *RI) {
+ // The final scheduler may move the extract together with the remat
+ // instruction, so only track pairs that are already co-located.
+ if (EI->getParent() != RI->getParent())
+ return;
CouldBeExtract.try_emplace(RI, EI);
CouldBeRemat.try_emplace(EI, RI);
}
>From 9fc27a5bd2888c04a47d0205eb7c704a05b5587a Mon Sep 17 00:00:00 2001
From: Ryan Buchner <buchner.ryan at gmail.com>
Date: Fri, 7 Aug 2026 09:05:20 -0700
Subject: [PATCH 20/22] [SLP] Finish encapsulation of DeferredExtractTracker
---
.../Transforms/Vectorize/SLPVectorizer.cpp | 23 ++++++++++---------
1 file changed, 12 insertions(+), 11 deletions(-)
diff --git a/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp b/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp
index 7fc9e7ae2cbc8..c7bf3dfff3a9c 100644
--- a/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp
+++ b/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp
@@ -3291,7 +3291,7 @@ class slpvectorizer::BoUpSLP {
/// Tracks deferred extract/rematerialization data and late cleanup actions.
class DeferredExtractTracker {
- public:
+ private:
struct DeferredExtractType {
Value *Scalar;
Value *NewInst;
@@ -3300,16 +3300,6 @@ class slpvectorizer::BoUpSLP {
: Scalar(Scalar), NewInst(NewInst), User(User) {}
};
- ExtractElementInst *lookupExtract(Value *V) const {
- return CouldBeExtract.lookup(V);
- }
-
- Instruction *lookupRemat(Value *V) const { return CouldBeRemat.lookup(V); }
-
- bool hasExtract(Value *V) const { return CouldBeExtract.contains(V); }
-
- bool hasRemat(Value *V) const { return CouldBeRemat.contains(V); }
-
/// Cases where extraction is estimated as more profitable but want to
/// delay extraction to allow for better vectorization in the interim.
SmallPtrSet<Value *, 4> ExternalUsesAsExtract;
@@ -3327,6 +3317,17 @@ class slpvectorizer::BoUpSLP {
DenseMap<Value *, SmallVector<DeferredExtractType, 2>>
DeferredScalarsToExtract;
+ public:
+ ExtractElementInst *lookupExtract(Value *V) const {
+ return CouldBeExtract.lookup(V);
+ }
+
+ Instruction *lookupRemat(Value *V) const { return CouldBeRemat.lookup(V); }
+
+ bool hasExtract(Value *V) const { return CouldBeExtract.contains(V); }
+
+ bool hasRemat(Value *V) const { return CouldBeRemat.contains(V); }
+
void clearTreeState() {
ExternalUsesAsExtract.clear();
ExternalUsesAsRematCostTmp.clear();
>From fc3c1be9695a86bb8eaca2c82545f41933f2583f Mon Sep 17 00:00:00 2001
From: Ryan Buchner <buchner.ryan at gmail.com>
Date: Fri, 7 Aug 2026 09:05:55 -0700
Subject: [PATCH 21/22] Handle failing logExtractRematPair calls
---
llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp | 13 +++++++++----
1 file changed, 9 insertions(+), 4 deletions(-)
diff --git a/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp b/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp
index c7bf3dfff3a9c..339f0fa9f8d4c 100644
--- a/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp
+++ b/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp
@@ -3448,13 +3448,14 @@ class slpvectorizer::BoUpSLP {
}
}
- void logExtractRematPair(ExtractElementInst *EI, Instruction *RI) {
+ bool logExtractRematPair(ExtractElementInst *EI, Instruction *RI) {
// The final scheduler may move the extract together with the remat
// instruction, so only track pairs that are already co-located.
if (EI->getParent() != RI->getParent())
- return;
+ return false;
CouldBeExtract.try_emplace(RI, EI);
CouldBeRemat.try_emplace(EI, RI);
+ return true;
}
void replaceWithExtractCandidates(SmallVectorImpl<Value *> &VL) {
@@ -25956,7 +25957,8 @@ Value *BoUpSLP::vectorizeTree(
if (DE.hasExtract(RI))
return;
DE.transferRematCost(Inst, EI);
- DE.logExtractRematPair(EI, RI);
+ if (!DE.logExtractRematPair(EI, RI))
+ eraseInstruction(EI);
return;
}
auto *EI = dyn_cast<ExtractElementInst>(Replacement);
@@ -25965,7 +25967,10 @@ Value *BoUpSLP::vectorizeTree(
if (!DE.isExternalUseAsExtract(RI) || DE.hasExtract(RI))
return;
DE.transferExtractCost(Inst, RI);
- DE.logExtractRematPair(EI, RI);
+ if (!DE.logExtractRematPair(EI, RI)) {
+ eraseInstruction(RI);
+ return;
+ }
if (User)
User->replaceUsesOfWith(EI, RI);
else
>From 4553cd23b36f0432af2ea080e0f18c2d87ca0568 Mon Sep 17 00:00:00 2001
From: Ryan Buchner <buchner.ryan at gmail.com>
Date: Fri, 7 Aug 2026 17:52:17 -0700
Subject: [PATCH 22/22] [SLP] Fix bug in scheduling of deferred extracted
copyables
---
llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp b/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp
index 1a8e3f4c442c0..9f39c337dc78f 100644
--- a/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp
+++ b/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp
@@ -5335,13 +5335,13 @@ class slpvectorizer::BoUpSLP {
IsBlended ? In->getOperand(OpIdx)
: Bundle->getTreeEntry()->getOperand(
OpIdx)[Lane])) {
- FoundInOpColumns |= I == In;
LLVM_DEBUG(dbgs() << "SLP: check for readiness (def): "
<< *I << "\n");
// The scheduling node works on the rematerialized version
// of the extract
if (auto *RI = DE.lookupRemat(I))
I = RI;
+ FoundInOpColumns |= I == In;
DecrUnschedForInst(
I, Bundle->getTreeEntry(), OpIdx, Checked,
Bundle->getTreeEntry()->isExpandedOperand(In, OpIdx));
More information about the llvm-commits
mailing list