[llvm] [SLP] Gather operands of associative binary chains into one node (PR #208514)
Alexey Bataev via llvm-commits
llvm-commits at lists.llvm.org
Thu Jul 16 08:04:04 PDT 2026
https://github.com/alexey-bataev updated https://github.com/llvm/llvm-project/pull/208514
>From ee9a3a2288f121f2ae21e8194339eadb76acf0d8 Mon Sep 17 00:00:00 2001
From: Alexey Bataev <a.bataev at outlook.com>
Date: Thu, 9 Jul 2026 10:22:34 -0700
Subject: [PATCH] =?UTF-8?q?[=F0=9D=98=80=F0=9D=97=BD=F0=9D=97=BF]=20initia?=
=?UTF-8?q?l=20version?=
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit
Created using spr 1.3.7
---
.../Transforms/Vectorize/SLPVectorizer.cpp | 573 +++++++++++++++++-
.../AArch64/externally-used-copyables.ll | 116 ++--
.../SLPVectorizer/X86/bv-shuffle-mask.ll | 26 +-
.../X86/cast-operand-extracted.ll | 18 +-
.../SLPVectorizer/X86/reassociate-ops.ll | 53 +-
.../Transforms/SLPVectorizer/X86/supernode.ll | 14 +-
6 files changed, 656 insertions(+), 144 deletions(-)
diff --git a/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp b/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp
index 880cb87bd9923..6334570914b9b 100644
--- a/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp
+++ b/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp
@@ -271,6 +271,11 @@ static cl::opt<bool> VectorizeCopyableElements(
cl::desc("Try to replace values with the idempotent instructions for "
"better vectorization."));
+/// Gather operands of associative single-use binary chains into one node.
+static cl::opt<bool> VectorizeReassociatedOps(
+ "slp-reassociate-ops", cl::init(true), cl::Hidden,
+ cl::desc("Gather operands of associative binary chains into one node."));
+
static cl::opt<unsigned> LoopAwareTripCount(
"slp-cost-loop-trip-count", cl::init(2), cl::Hidden,
cl::desc("Loop trip count, considered by the cost model during "
@@ -2355,6 +2360,7 @@ class slpvectorizer::BoUpSLP {
OperandsToTreeEntry.clear();
ScalarsInSplitNodes.clear();
MustGather.clear();
+ ReassocScalarToTreeEntries.clear();
NonScheduledFirst.clear();
EntryToLastInstruction.clear();
LastInstructionToPos.clear();
@@ -3618,6 +3624,23 @@ class slpvectorizer::BoUpSLP {
appendOperands(RootVL, Operands, S);
}
+ /// Initialize with flattened operand columns of an associative node.
+ /// ArgSize is taken from \p Operands, APO is always false.
+ VLOperands(ArrayRef<ValueList> Operands, const BasicBlock *BB,
+ const BoUpSLP &R)
+ : TLI(*R.TLI), DL(*R.DL), SE(*R.SE), R(R), L(R.LI->getLoopFor(BB)) {
+ assert(!Operands.empty() && "Expected at least one operand column");
+ ArgSize = Operands.size();
+ OpsVec.resize(ArgSize);
+ unsigned NumLanes = Operands.front().size();
+ for (auto [OpIdx, Ops] : enumerate(OpsVec)) {
+ Ops.resize(NumLanes);
+ for (unsigned Lane : seq<unsigned>(NumLanes))
+ Ops[Lane] = OperandData(Operands[OpIdx][Lane], /*APO=*/false,
+ /*IsUsed=*/false);
+ }
+ }
+
/// \Returns a value vector with the operands across all lanes for the
/// opearnd at \p OpIdx.
ValueList getVL(unsigned OpIdx) const {
@@ -3985,10 +4008,14 @@ class slpvectorizer::BoUpSLP {
/// Check if the value is vectorized in the tree.
bool isVectorized(const Value *V) const {
assert(V && "V cannot be nullptr.");
- ArrayRef<TreeEntry *> Entries = getTreeEntries(V);
- return any_of(Entries, [&](const TreeEntry *E) {
+ auto IsLive = [&](const TreeEntry *E) {
return !DeletedNodes.contains(E) && !TransformedToGatherNodes.contains(E);
- });
+ };
+ if (auto It = ReassocScalarToTreeEntries.find(V);
+ It != ReassocScalarToTreeEntries.end() && any_of(It->second, IsLive))
+ return true;
+ ArrayRef<TreeEntry *> Entries = getTreeEntries(V);
+ return any_of(Entries, IsLive);
}
/// Checks if it is legal and profitable to build SplitVectorize node for the
@@ -4458,6 +4485,10 @@ class slpvectorizer::BoUpSLP {
/// Copyable elements of the entry node.
SmallPtrSet<const Value *, 4> CopyableElements;
+ /// Intermediate instructions peeled from an associative chain (e.g. the
+ /// inner add in add(add(v0,x),v1)). Not part of Scalars.
+ SmallVector<Value *, 4> ReassocScalars;
+
/// MainOp and AltOp are recorded inside. S should be obtained from
/// newTreeEntry.
InstructionsState S = InstructionsState::invalid();
@@ -4595,6 +4626,15 @@ class slpvectorizer::BoUpSLP {
/// Returns true if any scalar in the list is a copyable element.
bool hasCopyableElements() const { return !CopyableElements.empty(); }
+ /// Adds \p V to the peeled reassociated scalars.
+ void addReassocScalar(Value *V) { ReassocScalars.push_back(V); }
+
+ /// True if operands were gathered from an associative chain.
+ bool hasReassocScalars() const { return !ReassocScalars.empty(); }
+
+ /// Returns peeled reassociated scalars.
+ ArrayRef<Value *> getReassocScalars() const { return ReassocScalars; }
+
/// Returns the state of the operations.
const InstructionsState &getOperations() const { return S; }
@@ -5018,6 +5058,11 @@ class slpvectorizer::BoUpSLP {
/// A list of scalars that we found that we need to keep as scalars.
ValueSet MustGather;
+ /// Maps each peeled reassociated scalar to owning entries. Keeps them
+ /// treated as vectorized while an owner is live.
+ SmallDenseMap<const Value *, SmallVector<const TreeEntry *>>
+ ReassocScalarToTreeEntries;
+
/// A set of first non-schedulable values.
ValueSet NonScheduledFirst;
@@ -5886,9 +5931,13 @@ class slpvectorizer::BoUpSLP {
continue;
}
}
+ // Flattened nodes may place an operand in any column; scan all of
+ // them so copyable scheduling does not double-count.
for (unsigned OpIdx :
- seq<unsigned>(::getNumberOfPotentiallyCommutativeOps(
- P.first->getMainOp()))) {
+ seq<unsigned>(P.first->hasReassocScalars()
+ ? P.first->getNumOperands()
+ : ::getNumberOfPotentiallyCommutativeOps(
+ P.first->getMainOp()))) {
if (P.first->getOperand(OpIdx)[Lane] == Op &&
getScheduleCopyableData(EdgeInfo(P.first, OpIdx), Op))
--P.getSecond();
@@ -6119,7 +6168,14 @@ class slpvectorizer::BoUpSLP {
}
}
auto It = OperandsUses.find(I);
- assert(It != OperandsUses.end() && "Operand not found");
+ if (It == OperandsUses.end()) {
+ // Column value may be a peeled intermediate, not a direct
+ // operand of In; its deps are released when it is scheduled.
+ LLVM_DEBUG(dbgs() << "SLP: operand " << *I
+ << " not modeled as a direct operand of "
+ << *In << ", skipping.\n");
+ return;
+ }
if (It->second > 0) {
if (ScheduleData *OpSD = getScheduleData(I)) {
if (!IsExpandedOperand &&
@@ -6177,7 +6233,8 @@ class slpvectorizer::BoUpSLP {
Bundle->getTreeEntry()->getNumOperands() ||
(isa<ZExtInst>(In) && Bundle->getTreeEntry()->getOpcode() ==
Instruction::Select) ||
- Bundle->getTreeEntry()->isCopyableElement(In)) &&
+ Bundle->getTreeEntry()->isCopyableElement(In) ||
+ Bundle->getTreeEntry()->hasReassocScalars()) &&
"Missed TreeEntry operands?");
// Count the number of unique phi nodes, which are the parent for
@@ -6239,6 +6296,46 @@ class slpvectorizer::BoUpSLP {
}
}
}
+ // Peeled intermediates stay as direct operands but drop out of
+ // operand columns; release their scheduling deps here.
+ for (const ScheduleBundle *Bundle : Bundles) {
+ if (TotalOpCount == 0)
+ break;
+ const TreeEntry *TE = Bundle->getTreeEntry();
+ if (!TE->hasReassocScalars())
+ continue;
+ for (Value *V : TE->getReassocScalars()) {
+ auto *OpI = dyn_cast<Instruction>(V);
+ if (!OpI)
+ continue;
+ auto UseIt = OperandsUses.find(OpI);
+ if (UseIt == OperandsUses.end() || UseIt->second == 0)
+ continue;
+ LLVM_DEBUG(dbgs() << "SLP: check for readiness "
+ "(reassociated operand): "
+ << *OpI << "\n");
+ // Copyable deps may live on per-edge ScheduleCopyableData.
+ bool ReleasedAsCopyable = false;
+ if (!ScheduleCopyableDataMap.empty()) {
+ for (const Use &U : In->operands()) {
+ if (U.get() != OpI)
+ continue;
+ for (ScheduleCopyableData *CD :
+ getScheduleCopyableData(In, U.getOperandNo(), OpI)) {
+ DecrUnsched(CD, /*IsControl=*/false);
+ ReleasedAsCopyable = true;
+ }
+ }
+ }
+ if (!ReleasedAsCopyable) {
+ if (ScheduleData *OpSD = getScheduleData(OpI))
+ for (unsigned I = 0, E = UseIt->second; I != E; ++I)
+ DecrUnsched(OpSD, /*IsControl=*/false);
+ }
+ TotalOpCount -= UseIt->second;
+ UseIt->second = 0;
+ }
+ }
}
} else {
// If BundleMember is a stand-alone instruction, no operand reordering
@@ -6287,7 +6384,8 @@ class slpvectorizer::BoUpSLP {
if (!Entries.empty()) {
for (TreeEntry *TE : Entries) {
if (!isa<ExtractValueInst, ExtractElementInst, CallBase>(In) &&
- In->getNumOperands() != TE->getNumOperands())
+ In->getNumOperands() != TE->getNumOperands() &&
+ !TE->hasReassocScalars())
continue;
auto &BundlePtr =
PseudoBundles.emplace_back(std::make_unique<ScheduleBundle>());
@@ -6344,7 +6442,8 @@ class slpvectorizer::BoUpSLP {
if (TE->isCopyableElement(In))
continue;
if (!isa<ExtractValueInst, ExtractElementInst, CallBase>(In) &&
- In->getNumOperands() != TE->getNumOperands())
+ In->getNumOperands() != TE->getNumOperands() &&
+ !TE->hasReassocScalars())
continue;
if (any_of(SDBundles, [&](const ScheduleBundle *SDBundle) {
return SDBundle->getTreeEntry() == TE;
@@ -9855,6 +9954,20 @@ void BoUpSLP::buildExternalUses(
if (UserIgnoreList && UserIgnoreList->contains(UserInst))
continue;
+ // Peeled reassociated scalars are subsumed by the flattened node and
+ // erased during vectorization, not external users.
+ if (auto ReassocIt = ReassocScalarToTreeEntries.find(UserInst);
+ ReassocIt != ReassocScalarToTreeEntries.end() &&
+ any_of(ReassocIt->second, [&](const TreeEntry *E) {
+ return !DeletedNodes.contains(E) &&
+ !TransformedToGatherNodes.contains(E);
+ })) {
+ LLVM_DEBUG(dbgs() << "SLP: \tInternal (reassociated) user will be "
+ "removed:"
+ << *U << ".\n");
+ continue;
+ }
+
// Skip in-tree scalars that become vectors
if (ArrayRef<TreeEntry *> UseEntries = getTreeEntries(U);
any_of(UseEntries, [this](const TreeEntry *UseEntry) {
@@ -13145,6 +13258,208 @@ BoUpSLP::getScalarsVectorizationLegality(ArrayRef<Value *> VL, unsigned Depth,
return ScalarsVectorizationLegality(S, /*IsLegal=*/true);
}
+/// Peel associative single-use binary chains into operand columns. Peeled
+/// instructions go to \p ReassocScalars; sub-operands are inserted after the
+/// peeled column so real pairs stay adjacent for flag reuse.
+/// Poison lanes are allowed through; a lane that is not itself a matching
+/// real instruction can still be peeled as a copyable identity leaf, as
+/// long as some other lane anchors the opcode.
+static void
+scanAssociativeOperands(const InstructionsState &S, DominatorTree &DT,
+ const DataLayout &DL, const TargetTransformInfo &TTI,
+ const TargetLibraryInfo &TLI, const BoUpSLP &R,
+ SmallVectorImpl<BoUpSLP::ValueList> &Operands,
+ SmallVectorImpl<Value *> &ReassocScalars) {
+ assert(Operands.size() == 2 && "Expected the initial 2 operand columns.");
+ InstructionsCompatibilityAnalysis Analysis(DT, DL, TTI, TLI);
+ // A lane whose value is itself a single-use associative instruction with
+ // S's opcode; block placement does not matter here (buildTreeRec
+ // re-checks that later for whatever columns end up as real leaves).
+ auto GetRealLane = [&](Value *V) -> Instruction * {
+ auto *I = dyn_cast<Instruction>(V);
+ if (I && I->getOpcode() == S.getOpcode() && I->isAssociative() &&
+ I->hasOneUse())
+ return I;
+ return nullptr;
+ };
+ // Builds on InstructionsCompatibilityAnalysis's own opcode matching
+ // (rather than the more permissive getSameOpcode()) so a column that
+ // falls back to a copyable identity leaf only ever anchors on a MainOp
+ // that InstructionsCompatibilityAnalysis::isSupportedOpcode() accepts.
+ auto CanPeel = [&](ArrayRef<Value *> Column) {
+ InstructionsState ColS = Analysis.buildInstructionsState(Column, R);
+ if (!ColS || ColS.getOpcode() != S.getOpcode() || ColS.isAltShuffle())
+ return InstructionsState::invalid();
+ // Every lane must be poison, a genuine matching real lane, or (only for
+ // a copyable ColS) stand in as a copyable identity leaf
+ // (Opcode(V, identity) == V). A single-use load is left ungrouped
+ // rather than peeled as a copyable leaf: this is its only position, and
+ // peeling commits it to this column's shape, so the ordinary
+ // tree-building copyable path never gets a chance to discover its own
+ // consecutive-load grouping with sibling lanes instead. A multi-use
+ // load has no such single opportunity to protect.
+ if (!all_of(Column, [&](Value *V) {
+ if (isa<PoisonValue>(V) || GetRealLane(V))
+ return true;
+ if (!ColS.areInstructionsWithCopyableElements())
+ return false;
+ auto *LI = dyn_cast<LoadInst>(V);
+ return (!LI || !LI->hasOneUse()) && ColS.isCopyableElement(V);
+ }))
+ return InstructionsState::invalid();
+ return ColS;
+ };
+ // Skip columns whose instruction lanes use different opcodes; folding them
+ // blocks split/gather paths that may find a better structure.
+ auto HasMixedOpcodes = [](ArrayRef<Value *> Column) {
+ Instruction *First = nullptr;
+ for (Value *V : Column) {
+ auto *I = dyn_cast<Instruction>(V);
+ if (!I)
+ continue;
+ if (!First)
+ First = I;
+ else if (First->getOpcode() != I->getOpcode())
+ return true;
+ }
+ return false;
+ };
+ for (unsigned Idx = 0; Idx != Operands.size();) {
+ InstructionsState ColS = CanPeel(Operands[Idx]);
+ if (!ColS) {
+ ++Idx;
+ continue;
+ }
+ BoUpSLP::ValueList Column = std::move(Operands[Idx]);
+ SmallVector<BoUpSLP::ValueList> SubOperands =
+ Analysis.buildOperands(ColS, Column);
+ if (any_of(SubOperands, HasMixedOpcodes)) {
+ Operands[Idx] = std::move(Column);
+ ++Idx;
+ continue;
+ }
+ // Poison and copyable lanes have no real instruction left to erase
+ // later: a copyable V is used as-is, not subsumed by the flattened
+ // combine.
+ for (Value *V : Column)
+ if (!isa<PoisonValue>(V) && !ColS.isCopyableElement(V))
+ ReassocScalars.push_back(V);
+ Operands[Idx] = std::move(SubOperands.front());
+ Operands.insert(std::next(Operands.begin(), Idx + 1),
+ std::make_move_iterator(std::next(SubOperands.begin())),
+ std::make_move_iterator(SubOperands.end()));
+ // Do not advance Idx: re-examine the column that was just placed here.
+ }
+}
+
+/// Realign flattened operand columns so matching value families share a
+/// column across lanes. Lane 0 is canonical; other lanes match by key equality
+/// only (never compare hash values).
+static SmallVector<BoUpSLP::ValueList>
+alignReassociatedOperandsByKey(ArrayRef<BoUpSLP::ValueList> Operands,
+ const TargetLibraryInfo &TLI) {
+ const unsigned NumCols = Operands.size();
+ const unsigned NumLanes = Operands.front().size();
+ auto LoadsSubkey = [](size_t /*Key*/, LoadInst *LI) {
+ return hash_value(getUnderlyingObject(LI->getPointerOperand()));
+ };
+ auto GetKey = [&](Value *V) {
+ return generateKeySubkey(V, &TLI, LoadsSubkey, /*AllowAlternate=*/false);
+ };
+ SmallVector<std::pair<size_t, size_t>> Lane0Keys;
+ Lane0Keys.reserve(NumCols);
+ for (const BoUpSLP::ValueList &Col : Operands)
+ Lane0Keys.push_back(GetKey(Col[0]));
+
+ SmallVector<BoUpSLP::ValueList> Aligned(NumCols,
+ BoUpSLP::ValueList(NumLanes));
+ for (unsigned Col : seq<unsigned>(NumCols))
+ Aligned[Col][0] = Operands[Col][0];
+
+ for (unsigned Lane : seq<unsigned>(1, NumLanes)) {
+ SmallDenseMap<std::pair<size_t, size_t>, SmallVector<unsigned, 2>, 8>
+ Buckets;
+ for (unsigned Col : seq<unsigned>(NumCols))
+ Buckets[GetKey(Operands[Col][Lane])].push_back(Col);
+ SmallDenseMap<std::pair<size_t, size_t>, unsigned, 8> BucketCursor;
+ SmallVector<unsigned> SlotSrcCol(NumCols, NumCols);
+ SmallVector<bool> ColClaimed(NumCols, false);
+ for (unsigned Slot : seq<unsigned>(NumCols)) {
+ auto BucketIt = Buckets.find(Lane0Keys[Slot]);
+ if (BucketIt == Buckets.end())
+ continue;
+ unsigned &Cursor = BucketCursor[Lane0Keys[Slot]];
+ if (Cursor >= BucketIt->second.size())
+ continue;
+ unsigned SrcCol = BucketIt->second[Cursor++];
+ SlotSrcCol[Slot] = SrcCol;
+ ColClaimed[SrcCol] = true;
+ }
+ unsigned NextLeftover = 0;
+ for (unsigned Slot : seq<unsigned>(NumCols)) {
+ if (SlotSrcCol[Slot] != NumCols)
+ continue;
+ while (ColClaimed[NextLeftover])
+ ++NextLeftover;
+ SlotSrcCol[Slot] = NextLeftover;
+ ColClaimed[NextLeftover] = true;
+ }
+ for (unsigned Slot : seq<unsigned>(NumCols))
+ Aligned[Slot][Lane] = Operands[SlotSrcCol[Slot]][Lane];
+ }
+ return Aligned;
+}
+
+/// True if \p V is the integer identity for \p Opcode (0/1/all-ones). FP
+/// identities are ignored (fast-math may break 0.0/1.0 as identity): the
+/// integer-only ConstantInt cast below never matches the ConstantFP identity
+/// that getBinOpIdentity() returns for FAdd/FMul.
+static bool isReassocIdentityConstant(const Value *V, unsigned Opcode) {
+ const auto *CI = dyn_cast<ConstantInt>(V);
+ return CI && ConstantExpr::getBinOpIdentity(Opcode, CI->getType()) == CI;
+}
+
+/// Rank reassociated operand layouts by vectorizable load columns, other load
+/// patterns, broadcast/constant columns, then fewer unique values per column.
+static std::tuple<unsigned, unsigned, unsigned, int>
+getReassocColumnsQuality(ArrayRef<BoUpSLP::ValueList> Columns, const BoUpSLP &R,
+ unsigned Opcode) {
+ unsigned NumConsecutiveLoadCols = 0;
+ unsigned NumOtherVecLoadCols = 0;
+ unsigned NumBroadcastOrConstCols = 0;
+ int NumUniqueValues = 0;
+ for (ArrayRef<Value *> Col : Columns) {
+ if (all_of(Col, IsaPred<Constant>)) {
+ ++NumBroadcastOrConstCols;
+ continue;
+ }
+ // Identity constants are free beyond the base they are inserted into.
+ SmallPtrSet<Value *, 8> UniqueValues;
+ for (Value *V : Col)
+ if (!isReassocIdentityConstant(V, Opcode))
+ UniqueValues.insert(V);
+ NumUniqueValues += UniqueValues.size();
+ if (UniqueValues.size() <= 1) {
+ ++NumBroadcastOrConstCols;
+ continue;
+ }
+ if (all_of(Col, IsaPred<LoadInst>)) {
+ BoUpSLP::OrdersType Order;
+ SmallVector<Value *> PointerOps;
+ BoUpSLP::StridedPtrInfo SPtrInfo;
+ BoUpSLP::LoadsState LS =
+ R.canVectorizeLoads(Col, Col.front(), Order, PointerOps, SPtrInfo);
+ if (LS == BoUpSLP::LoadsState::Vectorize)
+ ++NumConsecutiveLoadCols;
+ else if (LS != BoUpSLP::LoadsState::Gather)
+ ++NumOtherVecLoadCols;
+ }
+ }
+ // Greater is better; negate unique count as a tie-breaker.
+ return std::make_tuple(NumConsecutiveLoadCols, NumOtherVecLoadCols,
+ NumBroadcastOrConstCols, -NumUniqueValues);
+}
+
void BoUpSLP::buildTreeRec(ArrayRef<Value *> VLRef, unsigned Depth,
const EdgeInfo &UserTreeIdx,
unsigned InterleaveFactor) {
@@ -13386,6 +13701,49 @@ void BoUpSLP::buildTreeRec(ArrayRef<Value *> VLRef, unsigned Depth,
}
InstructionsCompatibilityAnalysis Analysis(*DT, *DL, *TTI, *TLI);
SmallVector<ValueList> Operands = Analysis.buildOperands(S, VL);
+ // Flatten associative single-use binary chains into operand columns. Skip
+ // alt-shuffle, copyable, non-associative, and narrow sub-slices of a wider
+ // user. Restricted to BinaryOperator: isAssociative() is also true for
+ // associative intrinsics (e.g. smax/smin/umax/umin), which are CallInst,
+ // not BinaryOperator, and are not supported by the copyable-identity
+ // machinery used below (ConstantExpr::getBinOpIdentity, isSupportedOpcode).
+ bool IsNarrowerThanUser = UserTreeIdx.UserTE &&
+ UserTreeIdx.UserTE->hasState() &&
+ UserTreeIdx.UserTE->getVectorFactor() > VL.size();
+ SmallVector<Value *> ReassocScalars;
+ // Cached below (when the peel is kept) so the reorder step further down
+ // does not need to redo the aligning/scoring work.
+ SmallVector<ValueList> ReassocAlignedOperands;
+ std::tuple<unsigned, unsigned, unsigned, int> ReassocPeeledQuality;
+ if (VectorizeReassociatedOps && !S.isAltShuffle() &&
+ !S.areInstructionsWithCopyableElements() && Operands.size() == 2 &&
+ !IsNarrowerThanUser && all_of(VL, [](Value *V) {
+ auto *I = dyn_cast<BinaryOperator>(V);
+ return I && I->isAssociative() && I->hasOneUse();
+ })) {
+ SmallVector<ValueList> NaturalTwoColumns(Operands);
+ scanAssociativeOperands(S, *DT, *DL, *TTI, *TLI, *this, Operands,
+ ReassocScalars);
+ // Drop flattening unless realigning improves load or broadcast column
+ // structure; an unimproved peel ties and reverts to natural columns.
+ if (!ReassocScalars.empty()) {
+ ReassocAlignedOperands = alignReassociatedOperandsByKey(Operands, *TLI);
+ ReassocPeeledQuality =
+ getReassocColumnsQuality(Operands, *this, S.getOpcode());
+ // The unique-value count (4th field) is only a tie-break for the
+ // later reorder-or-not decision, not for this one.
+ auto DropUniqueCount = [](const auto &Quality) {
+ return std::make_tuple(std::get<0>(Quality), std::get<1>(Quality),
+ std::get<2>(Quality));
+ };
+ if (DropUniqueCount(getReassocColumnsQuality(ReassocAlignedOperands,
+ *this, S.getOpcode())) <=
+ DropUniqueCount(ReassocPeeledQuality)) {
+ Operands = std::move(NaturalTwoColumns);
+ ReassocScalars.clear();
+ }
+ }
+ }
ScheduleBundle Empty;
ScheduleBundle &Bundle = BundlePtr.value() ? *BundlePtr.value() : Empty;
LLVM_DEBUG(dbgs() << "SLP: We are able to schedule this bundle.\n");
@@ -13678,14 +14036,37 @@ void BoUpSLP::buildTreeRec(ArrayRef<Value *> VLRef, unsigned Depth,
"(SelectInst/UnaryOperator/BinaryOperator/FreezeInst).\n";
TE->dump());
- if (isa<BinaryOperator>(VL0) && isCommutative(VL0)) {
+ if (!ReassocScalars.empty()) {
+ // Reorder all flattened columns; realign by value family first
+ // (cached above as ReassocAlignedOperands). Keep the reorder only if
+ // column-quality score does not regress vs the peeled order.
+ VLOperands Ops(ReassocAlignedOperands, VL0->getParent(), *this);
+ Ops.reorder();
+ SmallVector<ValueList> Reordered(Operands.size());
+ for (unsigned I : seq<unsigned>(Operands.size()))
+ Reordered[I] = Ops.getVL(I);
+ if (getReassocColumnsQuality(Reordered, *this, S.getOpcode()) >=
+ ReassocPeeledQuality)
+ Operands = std::move(Reordered);
+ else
+ LLVM_DEBUG(dbgs() << "SLP: reassociation reorder loses column "
+ "structure, keeping the natural operand "
+ "order.\n");
+ for (Value *V : ReassocScalars) {
+ TE->addReassocScalar(V);
+ SmallVectorImpl<const TreeEntry *> &Owners =
+ ReassocScalarToTreeEntries.try_emplace(V).first->second;
+ if (!is_contained(Owners, TE))
+ Owners.push_back(TE);
+ }
+ } else if (isa<BinaryOperator>(VL0) && isCommutative(VL0)) {
VLOperands Ops(VL, Operands, S, *this);
Ops.reorder();
Operands[0] = Ops.getVL(0);
Operands[1] = Ops.getVL(1);
}
TE->setOperands(Operands);
- for (unsigned I : seq<unsigned>(VL0->getNumOperands()))
+ for (unsigned I : seq<unsigned>(TE->getNumOperands()))
buildTreeRec(TE->getOperand(I), Depth + 1, {TE, I});
return;
}
@@ -14048,6 +14429,8 @@ unsigned BoUpSLP::getNumScalarInsts() const {
++Count;
}
}
+ // Count peeled intermediates in the scalar baseline (not in Scalars).
+ Count += TE.getReassocScalars().size();
// Even when the whole node is not combined, individual scalar
// instructions may be fused by the backend. Each fused pair (e.g.
// fadd+fmul → fma, select+cmp → smin/smax) becomes a single scalar
@@ -14146,6 +14529,9 @@ unsigned BoUpSLP::getNumVectorInsts() const {
}
if (TE.State == TreeEntry::SplitVectorize)
Count += 2;
+ else if (TE.hasReassocScalars())
+ // N operand columns need N-1 binary combines.
+ Count += TE.getNumOperands() - 1;
else
++Count;
if (!TE.ReorderIndices.empty() || !TE.ReuseShuffleIndices.empty())
@@ -15730,10 +16116,11 @@ void BoUpSLP::transformNodes() {
}
case Instruction::FSub:
case Instruction::FAdd: {
- // Check if possible to convert (a*b)+c to fma.
+ // Check if possible to convert (a*b)+c to fma; fma takes exactly two
+ // operands, so bail out on reassociated (>2-operand) nodes.
if (E.State != TreeEntry::Vectorize ||
!E.getOperations().isAddSubLikeOp() ||
- E.getOperations().isAltShuffle())
+ E.getOperations().isAltShuffle() || E.getNumOperands() != 2)
break;
const TreeEntry *LHS = getOperandEntry(&E, 0);
const TreeEntry *RHS = getOperandEntry(&E, 1);
@@ -15765,6 +16152,9 @@ void BoUpSLP::transformNodes() {
break;
}
case Instruction::Shl: {
+ // Shl is not reassociated; guard since this case indexes operands 0/1.
+ if (E.getNumOperands() != 2)
+ break;
if (E.Idx != 0 || DL->isBigEndian())
break;
if (!UserIgnoreList)
@@ -18052,7 +18442,9 @@ BoUpSLP::getEntryCost(const TreeEntry *E, ArrayRef<Value *> VectorizedVals,
return ScalarCost;
};
auto GetVectorCost = [=](InstructionCost CommonCost) {
- if (ShuffleOrOp == Instruction::And && It != MinBWs.end()) {
+ // And peephole only applies to plain 2-operand nodes.
+ if (ShuffleOrOp == Instruction::And && It != MinBWs.end() &&
+ !E->hasReassocScalars()) {
for (unsigned I : seq<unsigned>(0, E->getNumOperands())) {
ArrayRef<Value *> Ops = E->getOperand(I);
if (all_of(Ops, [&](Value *Op) {
@@ -18065,11 +18457,30 @@ BoUpSLP::getEntryCost(const TreeEntry *E, ArrayRef<Value *> VectorizedVals,
unsigned OpIdx = isa<UnaryOperator>(VL0) ? 0 : 1;
TTI::OperandValueInfo Op1Info = getOperandInfo(E->getOperand(0));
TTI::OperandValueInfo Op2Info = getOperandInfo(E->getOperand(OpIdx));
- return TTI->getArithmeticInstrCost(ShuffleOrOp, VecTy, CostKind, Op1Info,
- Op2Info, {}, nullptr, TLI) +
- CommonCost;
+ InstructionCost Cost = TTI->getArithmeticInstrCost(
+ ShuffleOrOp, VecTy, CostKind, Op1Info, Op2Info, {}, nullptr, TLI);
+ // N columns need N-1 vector combines; price extra columns conservatively.
+ if (E->hasReassocScalars())
+ for (unsigned Idx : seq<unsigned>(2, E->getNumOperands()))
+ Cost += TTI->getArithmeticInstrCost(
+ ShuffleOrOp, VecTy, CostKind, {},
+ getOperandInfo(E->getOperand(Idx)), {}, nullptr, TLI);
+ return Cost + CommonCost;
};
- return GetCostDiff(GetScalarCost, GetVectorCost);
+ InstructionCost CostDiff = GetCostDiff(GetScalarCost, GetVectorCost);
+ if (E->hasReassocScalars()) {
+ // Also subtract cost of peeled intermediate instructions. These are
+ // always 2-operand associative binops (isAssociative() excludes
+ // UnaryOperator), so operand 1 is always the second operand.
+ for (Value *V : E->getReassocScalars()) {
+ auto *I = cast<Instruction>(V);
+ TTI::OperandValueInfo Op1Info = TTI::getOperandInfo(I->getOperand(0));
+ TTI::OperandValueInfo Op2Info = TTI::getOperandInfo(I->getOperand(1));
+ CostDiff -= TTI->getArithmeticInstrCost(ShuffleOrOp, OrigScalarTy,
+ CostKind, Op1Info, Op2Info);
+ }
+ }
+ return CostDiff;
}
case Instruction::GetElementPtr: {
return CommonCost + GetGEPCostDiff(VL, VL0);
@@ -24189,9 +24600,109 @@ Value *BoUpSLP::vectorizeTree(TreeEntry *E) {
case Instruction::Xor: {
setInsertPointAfterBundle(E);
+ if (E->hasReassocScalars()) {
+ // Vectorize operand columns, then combine pairwise in a balanced tree.
+ SmallVector<Value *> CombinedScalars(E->Scalars.begin(),
+ E->Scalars.end());
+ CombinedScalars.append(E->getReassocScalars().begin(),
+ E->getReassocScalars().end());
+ // Reuse flags when a combine exactly matches a subsumed scalar pair.
+ auto MakeOperandKey = [](Value *A, Value *B) {
+ return A <= B ? std::make_pair(A, B) : std::make_pair(B, A);
+ };
+ SmallDenseMap<std::pair<Value *, Value *>,
+ SmallVector<Instruction *, 1>>
+ ByOperands;
+ for (Value *V : CombinedScalars) {
+ auto *I = cast<Instruction>(V);
+ ByOperands[MakeOperandKey(I->getOperand(0), I->getOperand(1))]
+ .push_back(I);
+ }
+ // Cast operand columns to VecTy when bit-width demotion changed types.
+ auto GetCastOperand = [&](unsigned Idx, Value *Op) {
+ if (Op->getType() == VecTy)
+ return Op;
+ return Builder.CreateIntCast(Op, VecTy, GetOperandSignedness(Idx));
+ };
+ SmallVector<Value *> Ops(E->getNumOperands());
+ // Track which scalar each partial result still represents for flag
+ // reuse.
+ SmallVector<SmallVector<Value *>> ScalarOps(E->getNumOperands());
+ for (unsigned Idx : seq<unsigned>(E->getNumOperands())) {
+ Ops[Idx] = GetCastOperand(Idx, vectorizeOperand(E, Idx));
+ ScalarOps[Idx].assign(E->getOperand(Idx));
+ }
+ while (Ops.size() > 1) {
+ SmallVector<Value *> NextOps((Ops.size() + 1) / 2);
+ SmallVector<SmallVector<Value *>> NextScalarOps(NextOps.size());
+ for (unsigned Idx : seq<unsigned>(NextOps.size())) {
+ if (2 * Idx + 1 == Ops.size()) {
+ // Odd one out this round: carry it over unpaired.
+ NextOps[Idx] = Ops[2 * Idx];
+ NextScalarOps[Idx] = std::move(ScalarOps[2 * Idx]);
+ continue;
+ }
+ Value *Combined = Builder.CreateBinOp(
+ static_cast<Instruction::BinaryOps>(E->getOpcode()),
+ Ops[2 * Idx], Ops[2 * Idx + 1]);
+ // Check whether this combine exactly matches a subsumed scalar.
+ const ArrayRef<Value *> LHSScalars = ScalarOps[2 * Idx];
+ const ArrayRef<Value *> RHSScalars = ScalarOps[2 * Idx + 1];
+ SmallVector<Value *> ExactMatches;
+ // Cache per-lane representatives for the next combine round.
+ SmallVector<Value *> Representatives(LHSScalars.size());
+ // Either side may be empty after a non-exact combine.
+ bool IsExact =
+ !LHSScalars.empty() && LHSScalars.size() == RHSScalars.size() &&
+ all_of(seq<unsigned>(LHSScalars.size()), [&](unsigned Lane) {
+ if (!LHSScalars[Lane] || !RHSScalars[Lane])
+ return false;
+ auto It = ByOperands.find(
+ MakeOperandKey(LHSScalars[Lane], RHSScalars[Lane]));
+ if (It == ByOperands.end())
+ return false;
+ ExactMatches.append(It->second.begin(), It->second.end());
+ Representatives[Lane] = It->second.front();
+ return true;
+ });
+ if (IsExact) {
+ NextOps[Idx] =
+ PropagateIRFlags(Combined, E->getOpcode(), ExactMatches);
+ NextScalarOps[Idx] = std::move(Representatives);
+ continue;
+ }
+ NextOps[Idx] =
+ PropagateIRFlags(Combined, E->getOpcode(), CombinedScalars);
+ // Drop overflow/fast-math flags not proven exact; regrouping can
+ // invalidate nsw/nuw and nnan/ninf even when each step was safe.
+ if (auto *CombinedI = dyn_cast<Instruction>(NextOps[Idx])) {
+ if (E->getOpcode() == Instruction::Add ||
+ E->getOpcode() == Instruction::Mul) {
+ CombinedI->setHasNoSignedWrap(false);
+ if (E->getOpcode() != Instruction::Add)
+ CombinedI->setHasNoUnsignedWrap(false);
+ } else if (E->getOpcode() == Instruction::FAdd ||
+ E->getOpcode() == Instruction::FMul) {
+ CombinedI->setHasNoNaNs(false);
+ CombinedI->setHasNoInfs(false);
+ }
+ }
+ // No scalar match: deeper combines from this result cannot be
+ // exact.
+ }
+ Ops = std::move(NextOps);
+ ScalarOps = std::move(NextScalarOps);
+ }
+ Value *V = FinalShuffle(Ops.front(), E);
+ E->VectorizedValue = V;
+ ++NumVectorInstructions;
+ return V;
+ }
+
Value *LHS = vectorizeOperand(E, 0);
Value *RHS = vectorizeOperand(E, 1);
if (ShuffleOrOp == Instruction::And && It != MinBWs.end()) {
+ assert(E->getNumOperands() == 2 && "Expected exactly 2 operands.");
for (unsigned I : seq<unsigned>(0, E->getNumOperands())) {
ArrayRef<Value *> Ops = E->getOperand(I);
if (all_of(Ops, [&](Value *Op) {
@@ -25503,6 +26014,24 @@ Value *BoUpSLP::vectorizeTree(
auto *I = cast<Instruction>(Scalar);
RemovedInsts.push_back(I);
}
+
+ // Erase peeled intermediates (not listed in Scalars).
+ for (Value *V : Entry->getReassocScalars()) {
+#ifndef NDEBUG
+ for (User *U : V->users()) {
+ LLVM_DEBUG(dbgs() << "SLP: \tvalidating user:" << *U << ".\n");
+ // Single-use peeled scalars are consumed by this entry or a sibling
+ // in the same chain.
+ assert((isVectorized(U) ||
+ (UserIgnoreList && UserIgnoreList->contains(U)) ||
+ (isa_and_nonnull<Instruction>(U) &&
+ isDeleted(cast<Instruction>(U)))) &&
+ "Deleting out-of-tree value");
+ }
+#endif
+ LLVM_DEBUG(dbgs() << "SLP: \tErasing scalar:" << *V << ".\n");
+ RemovedInsts.push_back(cast<Instruction>(V));
+ }
}
// Collect tree-entry vector instructions that ended up without any non-dead
@@ -27357,8 +27886,7 @@ bool BoUpSLP::collectValuesToDemote(
IsProfitableToDemote = true;
return TryProcessInstruction(BitWidth);
- // We can demote certain binary operations if we can demote both of their
- // operands.
+ // Demote binary ops when all operands can be demoted (may be >2 operands).
case Instruction::Add:
case Instruction::Sub:
case Instruction::Mul:
@@ -27366,7 +27894,10 @@ bool BoUpSLP::collectValuesToDemote(
case Instruction::Or:
case Instruction::Xor: {
return TryProcessInstruction(
- BitWidth, {getOperandEntry(&E, 0), getOperandEntry(&E, 1)});
+ BitWidth, map_to_vector(seq<unsigned>(E.getNumOperands()),
+ [&](unsigned Idx) -> const TreeEntry * {
+ return getOperandEntry(&E, Idx);
+ }));
}
case Instruction::Freeze:
return TryProcessInstruction(BitWidth, getOperandEntry(&E, 0));
diff --git a/llvm/test/Transforms/SLPVectorizer/AArch64/externally-used-copyables.ll b/llvm/test/Transforms/SLPVectorizer/AArch64/externally-used-copyables.ll
index 2c9eea2571f07..6fc7135671b76 100644
--- a/llvm/test/Transforms/SLPVectorizer/AArch64/externally-used-copyables.ll
+++ b/llvm/test/Transforms/SLPVectorizer/AArch64/externally-used-copyables.ll
@@ -20,29 +20,29 @@ define void @test(i64 %0, i64 %1, i64 %2, i64 %3, i64 %.sroa.3341.0.copyload, i6
; CHECK-NEXT: [[TMP21:%.*]] = sub i64 1, [[TMP20]]
; CHECK-NEXT: [[TMP22:%.*]] = add <2 x i64> [[TMP25]], <i64 8, i64 1>
; CHECK-NEXT: [[TMP23:%.*]] = or <2 x i64> [[TMP25]], <i64 8, i64 1>
-; CHECK-NEXT: [[TMP33:%.*]] = shufflevector <2 x i64> [[TMP22]], <2 x i64> [[TMP23]], <2 x i32> <i32 0, i32 3>
-; CHECK-NEXT: [[TMP18:%.*]] = shl i64 [[TMP0]], 1
-; CHECK-NEXT: [[TMP19:%.*]] = or i64 [[TMP18]], [[TMP0]]
-; CHECK-NEXT: [[TMP34:%.*]] = or i64 [[TMP19]], 1
-; CHECK-NEXT: [[TMP55:%.*]] = mul i64 [[TMP0]], [[TMP0]]
-; CHECK-NEXT: [[TMP58:%.*]] = insertelement <8 x i64> poison, i64 [[TMP0]], i32 0
-; CHECK-NEXT: [[TMP30:%.*]] = shufflevector <8 x i64> [[TMP58]], <8 x i64> poison, <8 x i32> zeroinitializer
+; CHECK-NEXT: [[TMP24:%.*]] = shufflevector <2 x i64> [[TMP22]], <2 x i64> [[TMP23]], <2 x i32> <i32 0, i32 3>
+; CHECK-NEXT: [[TMP100:%.*]] = shl i64 [[TMP0]], 1
+; CHECK-NEXT: [[TMP26:%.*]] = or i64 [[TMP100]], [[TMP0]]
+; CHECK-NEXT: [[TMP27:%.*]] = or i64 [[TMP26]], 1
+; CHECK-NEXT: [[TMP28:%.*]] = mul i64 [[TMP0]], [[TMP0]]
+; CHECK-NEXT: [[TMP29:%.*]] = insertelement <8 x i64> poison, i64 [[TMP0]], i32 0
+; CHECK-NEXT: [[TMP30:%.*]] = shufflevector <8 x i64> [[TMP29]], <8 x i64> poison, <8 x i32> zeroinitializer
; CHECK-NEXT: [[TMP31:%.*]] = shufflevector <8 x i64> [[TMP30]], <8 x i64> <i64 poison, i64 1, i64 1, i64 1, i64 poison, i64 poison, i64 poison, i64 poison>, <6 x i32> <i32 0, i32 9, i32 10, i32 11, i32 poison, i32 poison>
; CHECK-NEXT: [[TMP32:%.*]] = shufflevector <8 x i64> [[TMP30]], <8 x i64> <i64 0, i64 0, i64 0, i64 0, i64 poison, i64 1, i64 1, i64 1>, <8 x i32> <i32 8, i32 9, i32 10, i32 11, i32 0, i32 13, i32 14, i32 15>
-; CHECK-NEXT: [[TMP27:%.*]] = shufflevector <8 x i64> [[TMP30]], <8 x i64> poison, <3 x i32> <i32 0, i32 poison, i32 poison>
-; CHECK-NEXT: [[TMP28:%.*]] = insertelement <3 x i64> [[TMP27]], i64 [[TMP1]], i32 1
-; CHECK-NEXT: [[TMP29:%.*]] = insertelement <3 x i64> [[TMP28]], i64 [[DOTNEG1]], i32 2
-; CHECK-NEXT: [[TMP36:%.*]] = shufflevector <3 x i64> [[TMP29]], <3 x i64> poison, <8 x i32> <i32 0, i32 1, i32 1, i32 1, i32 2, i32 0, i32 0, i32 0>
+; CHECK-NEXT: [[TMP33:%.*]] = shufflevector <8 x i64> [[TMP30]], <8 x i64> poison, <3 x i32> <i32 0, i32 poison, i32 poison>
+; CHECK-NEXT: [[TMP34:%.*]] = insertelement <3 x i64> [[TMP33]], i64 [[TMP1]], i32 1
+; CHECK-NEXT: [[TMP105:%.*]] = insertelement <3 x i64> [[TMP34]], i64 [[DOTNEG1]], i32 2
+; CHECK-NEXT: [[TMP36:%.*]] = shufflevector <3 x i64> [[TMP105]], <3 x i64> poison, <8 x i32> <i32 0, i32 1, i32 1, i32 1, i32 2, i32 0, i32 0, i32 0>
; CHECK-NEXT: [[TMP37:%.*]] = insertelement <4 x i64> poison, i64 [[TMP2]], i32 0
; CHECK-NEXT: [[TMP38:%.*]] = insertelement <4 x i64> [[TMP37]], i64 [[TMP8]], i32 1
; CHECK-NEXT: [[TMP39:%.*]] = insertelement <4 x i64> [[TMP38]], i64 [[DOTSROA_3308_0_COPYLOAD]], i32 2
; CHECK-NEXT: [[TMP40:%.*]] = insertelement <4 x i64> [[TMP39]], i64 [[TMP0]], i32 3
; CHECK-NEXT: [[TMP41:%.*]] = shufflevector <2 x i64> [[TMP17]], <2 x i64> poison, <8 x i32> <i32 0, i32 1, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
-; CHECK-NEXT: [[TMP74:%.*]] = insertelement <4 x i64> poison, i64 [[TMP0]], i32 1
+; CHECK-NEXT: [[TMP42:%.*]] = insertelement <4 x i64> poison, i64 [[TMP0]], i32 1
; CHECK-NEXT: [[TMP43:%.*]] = mul i64 [[TMP0]], [[TMP0]]
; CHECK-NEXT: [[TMP44:%.*]] = shufflevector <2 x i64> [[TMP13]], <2 x i64> poison, <32 x i32> <i32 0, i32 poison, i32 poison, i32 poison, i32 poison, i32 1, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
; CHECK-NEXT: [[TMP45:%.*]] = shufflevector <32 x i64> [[TMP44]], <32 x i64> poison, <32 x i32> <i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 0, i32 poison, i32 poison, i32 poison, i32 poison, i32 5, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
-; CHECK-NEXT: [[TMP46:%.*]] = shufflevector <2 x i64> [[TMP33]], <2 x i64> poison, <6 x i32> <i32 0, i32 1, i32 poison, i32 poison, i32 poison, i32 poison>
+; CHECK-NEXT: [[TMP46:%.*]] = shufflevector <2 x i64> [[TMP24]], <2 x i64> poison, <6 x i32> <i32 0, i32 1, i32 poison, i32 poison, i32 poison, i32 poison>
; CHECK-NEXT: [[TMP47:%.*]] = shufflevector <6 x i64> [[TMP31]], <6 x i64> [[TMP46]], <6 x i32> <i32 0, i32 1, i32 2, i32 3, i32 6, i32 7>
; CHECK-NEXT: [[TMP48:%.*]] = shufflevector <2 x i64> [[TMP16]], <2 x i64> poison, <4 x i32> <i32 0, i32 1, i32 poison, i32 poison>
; CHECK-NEXT: br label %[[DOTLR_PH1977_US:.*]]
@@ -52,73 +52,73 @@ define void @test(i64 %0, i64 %1, i64 %2, i64 %3, i64 %.sroa.3341.0.copyload, i6
; CHECK-NEXT: [[TMP50:%.*]] = mul <8 x i64> [[TMP30]], [[TMP49]]
; CHECK-NEXT: [[TMP51:%.*]] = or <8 x i64> [[TMP30]], [[TMP49]]
; CHECK-NEXT: [[TMP52:%.*]] = shufflevector <8 x i64> [[TMP50]], <8 x i64> [[TMP51]], <8 x i32> <i32 0, i32 9, i32 10, i32 11, i32 4, i32 5, i32 6, i32 7>
-; CHECK-NEXT: [[TMP53:%.*]] = mul i64 [[TMP34]], [[TMP0]]
+; CHECK-NEXT: [[TMP53:%.*]] = mul i64 [[TMP27]], [[TMP0]]
; CHECK-NEXT: [[TMP54:%.*]] = mul i64 [[TMP21]], [[TMP0]]
-; CHECK-NEXT: [[TMP26:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-NEXT: [[DIFF_CHECK3783:%.*]] = icmp ult i64 [[TMP11]], [[TMP26]]
+; CHECK-NEXT: [[TMP55:%.*]] = call i64 @llvm.vscale.i64()
+; CHECK-NEXT: [[DIFF_CHECK3783:%.*]] = icmp ult i64 [[TMP11]], [[TMP55]]
; CHECK-NEXT: [[DIFF_CHECK3790:%.*]] = icmp ult i64 [[INDVAR37888]], [[TMP0]]
-; CHECK-NEXT: [[DIFF_CHECK3805:%.*]] = icmp ugt i64 [[TMP26]], 1
+; CHECK-NEXT: [[DIFF_CHECK3805:%.*]] = icmp ugt i64 [[TMP55]], 1
; CHECK-NEXT: [[TMP56:%.*]] = add <8 x i64> [[TMP52]], [[TMP32]]
; CHECK-NEXT: [[TMP57:%.*]] = icmp ult <8 x i64> [[TMP56]], [[TMP36]]
-; CHECK-NEXT: [[TMP24:%.*]] = extractelement <8 x i64> [[TMP52]], i32 5
-; CHECK-NEXT: [[TMP42:%.*]] = add i64 [[TMP24]], 1
-; CHECK-NEXT: [[DIFF_CHECK3842:%.*]] = icmp ult i64 [[TMP42]], [[TMP0]]
+; CHECK-NEXT: [[TMP58:%.*]] = extractelement <8 x i64> [[TMP52]], i32 5
+; CHECK-NEXT: [[TMP59:%.*]] = add i64 [[TMP58]], 1
+; CHECK-NEXT: [[DIFF_CHECK3842:%.*]] = icmp ult i64 [[TMP59]], [[TMP0]]
; CHECK-NEXT: [[DIFF_CHECK3817:%.*]] = icmp ult i64 [[TMP1]], [[TMP2]]
-; CHECK-NEXT: [[TMP76:%.*]] = add i64 [[TMP53]], 1
-; CHECK-NEXT: [[DIFF_CHECK3820:%.*]] = icmp ult i64 [[TMP76]], [[TMP0]]
-; CHECK-NEXT: [[TMP98:%.*]] = shl <2 x i64> [[TMP12]], splat (i64 1)
-; CHECK-NEXT: [[TMP77:%.*]] = insertelement <4 x i64> [[TMP74]], i64 [[INDVAR37888]], i32 0
-; CHECK-NEXT: [[TMP78:%.*]] = shufflevector <4 x i64> [[TMP77]], <4 x i64> poison, <4 x i32> <i32 0, i32 1, i32 1, i32 1>
-; CHECK-NEXT: [[TMP80:%.*]] = shufflevector <4 x i64> [[TMP77]], <4 x i64> <i64 1, i64 poison, i64 poison, i64 poison>, <4 x i32> <i32 4, i32 1, i32 poison, i32 poison>
-; CHECK-NEXT: [[TMP81:%.*]] = shufflevector <4 x i64> [[TMP80]], <4 x i64> [[TMP48]], <4 x i32> <i32 0, i32 1, i32 4, i32 5>
-; CHECK-NEXT: [[TMP82:%.*]] = mul <4 x i64> [[TMP78]], [[TMP81]]
-; CHECK-NEXT: [[TMP84:%.*]] = add i64 [[TMP54]], 1
-; CHECK-NEXT: [[TMP85:%.*]] = insertelement <32 x i64> [[TMP45]], i64 [[TMP55]], i32 11
-; CHECK-NEXT: [[TMP86:%.*]] = shufflevector <4 x i64> [[TMP82]], <4 x i64> poison, <32 x i32> <i32 0, i32 1, i32 2, i32 3, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
-; CHECK-NEXT: [[TMP89:%.*]] = shufflevector <32 x i64> [[TMP85]], <32 x i64> [[TMP86]], <32 x i32> <i32 32, i32 33, i32 34, i32 35, i32 poison, i32 5, i32 poison, i32 poison, i32 poison, i32 poison, i32 10, i32 11, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
-; CHECK-NEXT: [[TMP90:%.*]] = insertelement <32 x i64> [[TMP89]], i64 [[TMP84]], i32 4
-; CHECK-NEXT: [[TMP100:%.*]] = shufflevector <8 x i64> [[TMP52]], <8 x i64> poison, <32 x i32> <i32 poison, i32 poison, i32 poison, i32 poison, i32 4, i32 5, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
+; CHECK-NEXT: [[TMP60:%.*]] = add i64 [[TMP53]], 1
+; CHECK-NEXT: [[DIFF_CHECK3820:%.*]] = icmp ult i64 [[TMP60]], [[TMP0]]
+; CHECK-NEXT: [[TMP61:%.*]] = shl <2 x i64> [[TMP12]], splat (i64 1)
+; CHECK-NEXT: [[TMP62:%.*]] = insertelement <4 x i64> [[TMP42]], i64 [[INDVAR37888]], i32 0
+; CHECK-NEXT: [[TMP63:%.*]] = shufflevector <4 x i64> [[TMP62]], <4 x i64> poison, <4 x i32> <i32 0, i32 1, i32 1, i32 1>
+; CHECK-NEXT: [[TMP64:%.*]] = shufflevector <4 x i64> [[TMP62]], <4 x i64> <i64 1, i64 poison, i64 poison, i64 poison>, <4 x i32> <i32 4, i32 1, i32 poison, i32 poison>
+; CHECK-NEXT: [[TMP65:%.*]] = shufflevector <4 x i64> [[TMP64]], <4 x i64> [[TMP48]], <4 x i32> <i32 0, i32 1, i32 4, i32 5>
+; CHECK-NEXT: [[TMP66:%.*]] = mul <4 x i64> [[TMP63]], [[TMP65]]
+; CHECK-NEXT: [[TMP67:%.*]] = add i64 [[TMP54]], 1
+; CHECK-NEXT: [[TMP68:%.*]] = insertelement <32 x i64> [[TMP45]], i64 [[TMP28]], i32 11
+; CHECK-NEXT: [[TMP69:%.*]] = shufflevector <4 x i64> [[TMP66]], <4 x i64> poison, <32 x i32> <i32 0, i32 1, i32 2, i32 3, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
+; CHECK-NEXT: [[TMP70:%.*]] = shufflevector <32 x i64> [[TMP68]], <32 x i64> [[TMP69]], <32 x i32> <i32 32, i32 33, i32 34, i32 35, i32 poison, i32 5, i32 poison, i32 poison, i32 poison, i32 poison, i32 10, i32 11, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
+; CHECK-NEXT: [[TMP71:%.*]] = insertelement <32 x i64> [[TMP70]], i64 [[TMP67]], i32 4
+; CHECK-NEXT: [[TMP72:%.*]] = shufflevector <8 x i64> [[TMP52]], <8 x i64> poison, <32 x i32> <i32 poison, i32 poison, i32 poison, i32 poison, i32 4, i32 5, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
; CHECK-NEXT: [[TMP73:%.*]] = shufflevector <8 x i64> [[TMP52]], <8 x i64> poison, <32 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
-; CHECK-NEXT: [[TMP87:%.*]] = shufflevector <32 x i64> [[TMP90]], <32 x i64> [[TMP73]], <32 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 36, i32 37, i32 poison, i32 poison, i32 10, i32 11, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
-; CHECK-NEXT: [[TMP125:%.*]] = shufflevector <2 x i64> [[TMP98]], <2 x i64> poison, <32 x i32> <i32 0, i32 1, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
-; CHECK-NEXT: [[TMP88:%.*]] = shufflevector <32 x i64> [[TMP87]], <32 x i64> [[TMP125]], <32 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 32, i32 33, i32 10, i32 11, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
-; CHECK-NEXT: [[TMP59:%.*]] = insertelement <32 x i64> [[TMP88]], i64 [[TMP54]], i32 12
-; CHECK-NEXT: [[TMP60:%.*]] = shufflevector <32 x i64> [[TMP59]], <32 x i64> poison, <32 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 5, i32 5, i32 7, i32 5, i32 8, i32 9, i32 10, i32 5, i32 0, i32 1, i32 11, i32 12, i32 5, i32 5, i32 6, i32 7, i32 5, i32 5, i32 7, i32 5, i32 9, i32 10, i32 0, i32 1>
-; CHECK-NEXT: [[TMP61:%.*]] = shufflevector <32 x i64> [[TMP59]], <32 x i64> poison, <10 x i32> <i32 poison, i32 poison, i32 5, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
-; CHECK-NEXT: [[TMP62:%.*]] = insertelement <10 x i64> [[TMP61]], i64 [[TMP0]], i32 0
-; CHECK-NEXT: [[TMP63:%.*]] = insertelement <10 x i64> [[TMP62]], i64 [[TMP3]], i32 1
-; CHECK-NEXT: [[TMP64:%.*]] = insertelement <10 x i64> [[TMP63]], i64 [[TMP4]], i32 3
-; CHECK-NEXT: [[TMP65:%.*]] = insertelement <10 x i64> [[TMP64]], i64 [[INDVAR3788]], i32 4
-; CHECK-NEXT: [[TMP66:%.*]] = insertelement <10 x i64> [[TMP65]], i64 [[TMP2]], i32 5
-; CHECK-NEXT: [[TMP67:%.*]] = insertelement <10 x i64> [[TMP66]], i64 [[TMP5]], i32 6
-; CHECK-NEXT: [[TMP68:%.*]] = insertelement <10 x i64> [[TMP67]], i64 [[TMP6]], i32 7
-; CHECK-NEXT: [[TMP69:%.*]] = insertelement <10 x i64> [[TMP68]], i64 [[TMP7]], i32 8
-; CHECK-NEXT: [[TMP70:%.*]] = insertelement <10 x i64> [[TMP69]], i64 [[DOTSROA_3341_0_COPYLOAD]], i32 9
-; CHECK-NEXT: [[TMP71:%.*]] = shufflevector <10 x i64> [[TMP70]], <10 x i64> poison, <32 x i32> <i32 0, i32 0, i32 0, i32 0, i32 0, i32 1, i32 0, i32 2, i32 3, i32 4, i32 0, i32 0, i32 2, i32 2, i32 0, i32 5, i32 0, i32 0, i32 0, i32 0, i32 6, i32 7, i32 0, i32 2, i32 8, i32 9, i32 0, i32 0, i32 2, i32 0, i32 0, i32 0>
-; CHECK-NEXT: [[TMP72:%.*]] = icmp ult <32 x i64> [[TMP60]], [[TMP71]]
+; CHECK-NEXT: [[TMP74:%.*]] = shufflevector <32 x i64> [[TMP71]], <32 x i64> [[TMP73]], <32 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 36, i32 37, i32 poison, i32 poison, i32 10, i32 11, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
+; CHECK-NEXT: [[TMP107:%.*]] = shufflevector <2 x i64> [[TMP61]], <2 x i64> poison, <32 x i32> <i32 0, i32 1, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
+; CHECK-NEXT: [[TMP76:%.*]] = shufflevector <32 x i64> [[TMP74]], <32 x i64> [[TMP107]], <32 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 32, i32 33, i32 10, i32 11, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
+; CHECK-NEXT: [[TMP77:%.*]] = insertelement <32 x i64> [[TMP76]], i64 [[TMP54]], i32 12
+; CHECK-NEXT: [[TMP78:%.*]] = shufflevector <32 x i64> [[TMP77]], <32 x i64> poison, <32 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 5, i32 5, i32 7, i32 5, i32 8, i32 9, i32 10, i32 5, i32 0, i32 1, i32 11, i32 12, i32 5, i32 5, i32 6, i32 7, i32 5, i32 5, i32 7, i32 5, i32 9, i32 10, i32 0, i32 1>
+; CHECK-NEXT: [[TMP109:%.*]] = shufflevector <32 x i64> [[TMP77]], <32 x i64> poison, <10 x i32> <i32 poison, i32 poison, i32 5, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
+; CHECK-NEXT: [[TMP80:%.*]] = insertelement <10 x i64> [[TMP109]], i64 [[TMP0]], i32 0
+; CHECK-NEXT: [[TMP81:%.*]] = insertelement <10 x i64> [[TMP80]], i64 [[TMP3]], i32 1
+; CHECK-NEXT: [[TMP82:%.*]] = insertelement <10 x i64> [[TMP81]], i64 [[TMP4]], i32 3
+; CHECK-NEXT: [[TMP110:%.*]] = insertelement <10 x i64> [[TMP82]], i64 [[INDVAR3788]], i32 4
+; CHECK-NEXT: [[TMP84:%.*]] = insertelement <10 x i64> [[TMP110]], i64 [[TMP2]], i32 5
+; CHECK-NEXT: [[TMP85:%.*]] = insertelement <10 x i64> [[TMP84]], i64 [[TMP5]], i32 6
+; CHECK-NEXT: [[TMP86:%.*]] = insertelement <10 x i64> [[TMP85]], i64 [[TMP6]], i32 7
+; CHECK-NEXT: [[TMP87:%.*]] = insertelement <10 x i64> [[TMP86]], i64 [[TMP7]], i32 8
+; CHECK-NEXT: [[TMP88:%.*]] = insertelement <10 x i64> [[TMP87]], i64 [[DOTSROA_3341_0_COPYLOAD]], i32 9
+; CHECK-NEXT: [[TMP89:%.*]] = shufflevector <10 x i64> [[TMP88]], <10 x i64> poison, <32 x i32> <i32 0, i32 0, i32 0, i32 0, i32 0, i32 1, i32 0, i32 2, i32 3, i32 4, i32 0, i32 0, i32 2, i32 2, i32 0, i32 5, i32 0, i32 0, i32 0, i32 0, i32 6, i32 7, i32 0, i32 2, i32 8, i32 9, i32 0, i32 0, i32 2, i32 0, i32 0, i32 0>
+; CHECK-NEXT: [[TMP90:%.*]] = icmp ult <32 x i64> [[TMP78]], [[TMP89]]
; CHECK-NEXT: [[TMP91:%.*]] = add i64 [[TMP54]], 1
; CHECK-NEXT: [[TMP92:%.*]] = shufflevector <8 x i64> [[TMP52]], <8 x i64> poison, <8 x i32> <i32 5, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 4, i32 4>
; CHECK-NEXT: [[TMP93:%.*]] = shufflevector <8 x i64> [[TMP92]], <8 x i64> [[TMP41]], <8 x i32> <i32 0, i32 1, i32 2, i32 8, i32 9, i32 5, i32 6, i32 7>
-; CHECK-NEXT: [[TMP94:%.*]] = shufflevector <2 x i64> [[TMP98]], <2 x i64> poison, <8 x i32> <i32 0, i32 1, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
+; CHECK-NEXT: [[TMP94:%.*]] = shufflevector <2 x i64> [[TMP61]], <2 x i64> poison, <8 x i32> <i32 0, i32 1, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
; CHECK-NEXT: [[TMP95:%.*]] = shufflevector <8 x i64> [[TMP93]], <8 x i64> [[TMP94]], <8 x i32> <i32 0, i32 8, i32 9, i32 3, i32 4, i32 5, i32 6, i32 7>
; CHECK-NEXT: [[TMP96:%.*]] = insertelement <8 x i64> [[TMP95]], i64 [[TMP91]], i32 5
-; CHECK-NEXT: [[TMP105:%.*]] = shufflevector <8 x i64> [[TMP96]], <8 x i64> poison, <2 x i32> <i32 poison, i32 3>
-; CHECK-NEXT: [[TMP107:%.*]] = insertelement <2 x i64> [[TMP105]], i64 [[TMP0]], i32 0
-; CHECK-NEXT: [[TMP99:%.*]] = shufflevector <2 x i64> [[TMP107]], <2 x i64> poison, <8 x i32> <i32 0, i32 1, i32 1, i32 0, i32 0, i32 0, i32 0, i32 1>
+; CHECK-NEXT: [[TMP111:%.*]] = shufflevector <8 x i64> [[TMP96]], <8 x i64> poison, <2 x i32> <i32 poison, i32 3>
+; CHECK-NEXT: [[TMP98:%.*]] = insertelement <2 x i64> [[TMP111]], i64 [[TMP0]], i32 0
+; CHECK-NEXT: [[TMP99:%.*]] = shufflevector <2 x i64> [[TMP98]], <2 x i64> poison, <8 x i32> <i32 0, i32 1, i32 1, i32 0, i32 0, i32 0, i32 0, i32 1>
; CHECK-NEXT: [[TMP83:%.*]] = icmp ult <8 x i64> [[TMP96]], [[TMP99]]
; CHECK-NEXT: [[TMP101:%.*]] = shufflevector <8 x i64> [[TMP52]], <8 x i64> poison, <4 x i32> <i32 4, i32 poison, i32 poison, i32 poison>
; CHECK-NEXT: [[TMP102:%.*]] = insertelement <4 x i64> [[TMP101]], i64 [[TMP1]], i32 1
; CHECK-NEXT: [[TMP103:%.*]] = insertelement <4 x i64> [[TMP102]], i64 [[TMP53]], i32 3
; CHECK-NEXT: [[TMP104:%.*]] = shufflevector <4 x i64> [[TMP103]], <4 x i64> poison, <4 x i32> <i32 0, i32 1, i32 1, i32 3>
; CHECK-NEXT: [[TMP97:%.*]] = icmp ult <4 x i64> [[TMP104]], [[TMP40]]
-; CHECK-NEXT: [[DIFF_CHECK3930:%.*]] = icmp ult i64 [[TMP24]], [[TMP0]]
-; CHECK-NEXT: [[TMP106:%.*]] = extractelement <2 x i64> [[TMP98]], i32 0
+; CHECK-NEXT: [[DIFF_CHECK3930:%.*]] = icmp ult i64 [[TMP58]], [[TMP0]]
+; CHECK-NEXT: [[TMP106:%.*]] = extractelement <2 x i64> [[TMP61]], i32 0
; CHECK-NEXT: [[DIFF_CHECK3932:%.*]] = icmp ult i64 [[TMP106]], [[TMP1]]
; CHECK-NEXT: [[OP_RDX50:%.*]] = icmp ult i64 [[TMP1]], [[TMP0]]
; CHECK-NEXT: [[TMP126:%.*]] = icmp ult i64 [[INDVAR37888]], [[TMP0]]
; CHECK-NEXT: [[DIFF_CHECK3938:%.*]] = icmp ult i64 [[TMP43]], [[TMP0]]
; CHECK-NEXT: [[DIFF_CHECK3950:%.*]] = icmp ult i64 [[TMP1]], [[TMP2]]
-; CHECK-NEXT: [[TMP75:%.*]] = call i1 @llvm.vector.reduce.or.v32i1(<32 x i1> [[TMP72]])
+; CHECK-NEXT: [[TMP75:%.*]] = call i1 @llvm.vector.reduce.or.v32i1(<32 x i1> [[TMP90]])
; CHECK-NEXT: [[TMP108:%.*]] = call i1 @llvm.vector.reduce.or.v8i1(<8 x i1> [[TMP57]])
; CHECK-NEXT: [[TMP127:%.*]] = shufflevector <8 x i1> [[TMP83]], <8 x i1> poison, <4 x i32> <i32 0, i32 1, i32 2, i32 3>
; CHECK-NEXT: [[RDX_OP:%.*]] = or <4 x i1> [[TMP127]], [[TMP97]]
diff --git a/llvm/test/Transforms/SLPVectorizer/X86/bv-shuffle-mask.ll b/llvm/test/Transforms/SLPVectorizer/X86/bv-shuffle-mask.ll
index b5d9915b690c0..24b2ddb1eb94f 100644
--- a/llvm/test/Transforms/SLPVectorizer/X86/bv-shuffle-mask.ll
+++ b/llvm/test/Transforms/SLPVectorizer/X86/bv-shuffle-mask.ll
@@ -5,21 +5,21 @@ define i16 @test(i16 %v1, i16 %v2) {
; CHECK-LABEL: define i16 @test(
; CHECK-SAME: i16 [[V1:%.*]], i16 [[V2:%.*]]) {
; CHECK-NEXT: [[ENTRY:.*:]]
-; CHECK-NEXT: [[TMP4:%.*]] = insertelement <4 x i16> poison, i16 [[V1]], i32 0
-; CHECK-NEXT: [[TMP5:%.*]] = insertelement <4 x i16> [[TMP4]], i16 [[V2]], i32 1
-; CHECK-NEXT: [[TMP2:%.*]] = shufflevector <4 x i16> [[TMP5]], <4 x i16> poison, <4 x i32> <i32 0, i32 0, i32 0, i32 1>
+; CHECK-NEXT: [[TMP0:%.*]] = insertelement <4 x i16> poison, i16 [[V1]], i32 0
+; CHECK-NEXT: [[TMP1:%.*]] = insertelement <4 x i16> [[TMP0]], i16 [[V2]], i32 1
+; CHECK-NEXT: [[TMP2:%.*]] = shufflevector <4 x i16> [[TMP1]], <4 x i16> poison, <4 x i32> <i32 0, i32 0, i32 0, i32 1>
; CHECK-NEXT: [[TMP8:%.*]] = or <4 x i16> [[TMP2]], zeroinitializer
-; CHECK-NEXT: [[TMP0:%.*]] = shufflevector <4 x i16> [[TMP2]], <4 x i16> <i16 -1, i16 -1, i16 -1, i16 poison>, <4 x i32> <i32 4, i32 5, i32 6, i32 3>
-; CHECK-NEXT: [[TMP1:%.*]] = insertelement <4 x i16> <i16 0, i16 0, i16 0, i16 poison>, i16 [[V1]], i32 3
-; CHECK-NEXT: [[TMP3:%.*]] = and <4 x i16> [[TMP0]], [[TMP1]]
-; CHECK-NEXT: [[TMP7:%.*]] = or <4 x i16> [[TMP3]], zeroinitializer
-; CHECK-NEXT: [[TMP9:%.*]] = and <4 x i16> [[TMP7]], zeroinitializer
-; CHECK-NEXT: [[TMP10:%.*]] = and <4 x i16> [[TMP9]], zeroinitializer
-; CHECK-NEXT: [[TMP11:%.*]] = icmp ne <4 x i16> [[TMP10]], zeroinitializer
+; CHECK-NEXT: [[TMP4:%.*]] = shufflevector <4 x i16> [[TMP2]], <4 x i16> <i16 0, i16 0, i16 0, i16 poison>, <4 x i32> <i32 4, i32 5, i32 6, i32 3>
+; CHECK-NEXT: [[TMP5:%.*]] = or <4 x i16> [[TMP4]], zeroinitializer
+; CHECK-NEXT: [[TMP6:%.*]] = insertelement <4 x i16> <i16 0, i16 0, i16 0, i16 poison>, i16 [[V1]], i32 3
+; CHECK-NEXT: [[TMP7:%.*]] = and <4 x i16> <i16 -1, i16 -1, i16 -1, i16 0>, [[TMP5]]
+; CHECK-NEXT: [[TMP13:%.*]] = and <4 x i16> [[TMP7]], [[TMP6]]
+; CHECK-NEXT: [[TMP9:%.*]] = and <4 x i16> [[TMP13]], zeroinitializer
+; CHECK-NEXT: [[TMP11:%.*]] = icmp ne <4 x i16> [[TMP9]], zeroinitializer
; CHECK-NEXT: [[TMP12:%.*]] = or <4 x i1> [[TMP11]], zeroinitializer
-; CHECK-NEXT: [[TMP13:%.*]] = or <4 x i16> [[TMP8]], zeroinitializer
-; CHECK-NEXT: [[TMP14:%.*]] = or <4 x i16> [[TMP13]], zeroinitializer
-; CHECK-NEXT: [[TMP15:%.*]] = or <4 x i16> [[TMP14]], zeroinitializer
+; CHECK-NEXT: [[TMP10:%.*]] = or <4 x i16> [[TMP8]], zeroinitializer
+; CHECK-NEXT: [[TMP28:%.*]] = or <4 x i16> [[TMP10]], zeroinitializer
+; CHECK-NEXT: [[TMP15:%.*]] = or <4 x i16> [[TMP28]], zeroinitializer
; CHECK-NEXT: [[TMP16:%.*]] = icmp ne <4 x i16> [[TMP15]], zeroinitializer
; CHECK-NEXT: [[TMP17:%.*]] = or <4 x i1> zeroinitializer, [[TMP16]]
; CHECK-NEXT: [[TMP18:%.*]] = or <4 x i1> [[TMP12]], [[TMP17]]
diff --git a/llvm/test/Transforms/SLPVectorizer/X86/cast-operand-extracted.ll b/llvm/test/Transforms/SLPVectorizer/X86/cast-operand-extracted.ll
index 2d4077f82621a..bd48f63c7be96 100644
--- a/llvm/test/Transforms/SLPVectorizer/X86/cast-operand-extracted.ll
+++ b/llvm/test/Transforms/SLPVectorizer/X86/cast-operand-extracted.ll
@@ -18,15 +18,15 @@ define void @test(ptr %0, i32 %add651) {
; CHECK-NEXT: [[TMP4:%.*]] = load ptr, ptr [[TMP0]], align 8
; CHECK-NEXT: [[ARRAYIDX660:%.*]] = getelementptr i8, ptr [[TMP4]], i64 7800
; CHECK-NEXT: [[ARRAYIDX689:%.*]] = getelementptr i8, ptr [[TMP4]], i64 7816
-; CHECK-NEXT: [[TMP6:%.*]] = add <2 x i32> [[TMP3]], splat (i32 1)
-; CHECK-NEXT: [[TMP10:%.*]] = add <2 x i32> [[TMP6]], [[TMP7]]
-; CHECK-NEXT: [[TMP11:%.*]] = insertelement <4 x i32> poison, i32 [[ADD651]], i32 0
-; CHECK-NEXT: [[TMP13:%.*]] = insertelement <4 x i32> [[TMP11]], i32 [[TMP2]], i32 1
-; CHECK-NEXT: [[TMP19:%.*]] = shufflevector <2 x i32> [[TMP10]], <2 x i32> poison, <4 x i32> <i32 0, i32 1, i32 poison, i32 poison>
-; CHECK-NEXT: [[TMP14:%.*]] = shufflevector <4 x i32> [[TMP13]], <4 x i32> [[TMP19]], <4 x i32> <i32 0, i32 1, i32 4, i32 5>
-; CHECK-NEXT: [[TMP20:%.*]] = insertelement <4 x i32> <i32 0, i32 0, i32 1, i32 poison>, i32 [[TMP5]], i32 3
-; CHECK-NEXT: [[TMP21:%.*]] = add <4 x i32> [[TMP14]], [[TMP20]]
-; CHECK-NEXT: [[TMP15:%.*]] = lshr <4 x i32> [[TMP21]], splat (i32 1)
+; CHECK-NEXT: [[TMP19:%.*]] = add <2 x i32> [[TMP3]], splat (i32 1)
+; CHECK-NEXT: [[TMP8:%.*]] = add <2 x i32> [[TMP19]], [[TMP7]]
+; CHECK-NEXT: [[TMP9:%.*]] = insertelement <4 x i32> poison, i32 [[ADD651]], i32 0
+; CHECK-NEXT: [[TMP10:%.*]] = insertelement <4 x i32> [[TMP9]], i32 [[TMP2]], i32 1
+; CHECK-NEXT: [[TMP11:%.*]] = shufflevector <2 x i32> [[TMP8]], <2 x i32> poison, <4 x i32> <i32 0, i32 1, i32 poison, i32 poison>
+; CHECK-NEXT: [[TMP12:%.*]] = shufflevector <4 x i32> [[TMP10]], <4 x i32> [[TMP11]], <4 x i32> <i32 0, i32 1, i32 4, i32 5>
+; CHECK-NEXT: [[TMP13:%.*]] = insertelement <4 x i32> <i32 0, i32 0, i32 1, i32 poison>, i32 [[TMP5]], i32 3
+; CHECK-NEXT: [[TMP14:%.*]] = add <4 x i32> [[TMP12]], [[TMP13]]
+; CHECK-NEXT: [[TMP15:%.*]] = lshr <4 x i32> [[TMP14]], splat (i32 1)
; CHECK-NEXT: [[SHR685:%.*]] = lshr i32 [[TMP2]], 1
; CHECK-NEXT: [[TMP16:%.*]] = trunc <4 x i32> [[TMP15]] to <4 x i16>
; CHECK-NEXT: [[CONV686:%.*]] = trunc i32 [[SHR685]] to i16
diff --git a/llvm/test/Transforms/SLPVectorizer/X86/reassociate-ops.ll b/llvm/test/Transforms/SLPVectorizer/X86/reassociate-ops.ll
index 7e298a9b2da68..673fa4d71c378 100644
--- a/llvm/test/Transforms/SLPVectorizer/X86/reassociate-ops.ll
+++ b/llvm/test/Transforms/SLPVectorizer/X86/reassociate-ops.ll
@@ -13,10 +13,8 @@ define void @test_reassoc_add(ptr %Aarray, ptr %Barray, ptr %Carray, ptr %Sarray
; CHECK-NEXT: [[TMP0:%.*]] = load <2 x double>, ptr [[AARRAY]], align 8
; CHECK-NEXT: [[TMP1:%.*]] = load <2 x double>, ptr [[BARRAY]], align 8
; CHECK-NEXT: [[TMP2:%.*]] = load <2 x double>, ptr [[CARRAY]], align 8
-; CHECK-NEXT: [[TMP3:%.*]] = shufflevector <2 x double> [[TMP0]], <2 x double> [[TMP2]], <2 x i32> <i32 0, i32 3>
-; CHECK-NEXT: [[TMP4:%.*]] = fadd fast <2 x double> [[TMP1]], [[TMP3]]
-; CHECK-NEXT: [[TMP5:%.*]] = shufflevector <2 x double> [[TMP2]], <2 x double> [[TMP0]], <2 x i32> <i32 0, i32 3>
-; CHECK-NEXT: [[TMP6:%.*]] = fadd fast <2 x double> [[TMP4]], [[TMP5]]
+; CHECK-NEXT: [[TMP3:%.*]] = fadd reassoc nsz arcp contract afn <2 x double> [[TMP0]], [[TMP1]]
+; CHECK-NEXT: [[TMP6:%.*]] = fadd reassoc nsz arcp contract afn <2 x double> [[TMP3]], [[TMP2]]
; CHECK-NEXT: store <2 x double> [[TMP6]], ptr [[SARRAY]], align 8
; CHECK-NEXT: ret void
;
@@ -63,10 +61,8 @@ define void @test_reassoc_add_wrapflags(ptr %Aarray, ptr %Barray, ptr %Carray, p
; CHECK-NEXT: [[TMP0:%.*]] = load <2 x i32>, ptr [[AARRAY]], align 4
; CHECK-NEXT: [[TMP1:%.*]] = load <2 x i32>, ptr [[BARRAY]], align 4
; CHECK-NEXT: [[TMP2:%.*]] = load <2 x i32>, ptr [[CARRAY]], align 4
-; CHECK-NEXT: [[TMP3:%.*]] = shufflevector <2 x i32> [[TMP0]], <2 x i32> [[TMP2]], <2 x i32> <i32 0, i32 3>
-; CHECK-NEXT: [[TMP4:%.*]] = add nuw nsw <2 x i32> [[TMP1]], [[TMP3]]
-; CHECK-NEXT: [[TMP5:%.*]] = shufflevector <2 x i32> [[TMP2]], <2 x i32> [[TMP0]], <2 x i32> <i32 0, i32 3>
-; CHECK-NEXT: [[TMP6:%.*]] = add nuw nsw <2 x i32> [[TMP4]], [[TMP5]]
+; CHECK-NEXT: [[TMP3:%.*]] = add nuw <2 x i32> [[TMP0]], [[TMP1]]
+; CHECK-NEXT: [[TMP6:%.*]] = add nuw <2 x i32> [[TMP3]], [[TMP2]]
; CHECK-NEXT: store <2 x i32> [[TMP6]], ptr [[SARRAY]], align 4
; CHECK-NEXT: ret void
;
@@ -160,10 +156,8 @@ define void @test_reassoc_mul_wrapflags(ptr %Aarray, ptr %Barray, ptr %Carray, p
; CHECK-NEXT: [[TMP0:%.*]] = load <2 x i32>, ptr [[AARRAY]], align 4
; CHECK-NEXT: [[TMP1:%.*]] = load <2 x i32>, ptr [[BARRAY]], align 4
; CHECK-NEXT: [[TMP2:%.*]] = load <2 x i32>, ptr [[CARRAY]], align 4
-; CHECK-NEXT: [[TMP3:%.*]] = shufflevector <2 x i32> [[TMP0]], <2 x i32> [[TMP2]], <2 x i32> <i32 0, i32 3>
-; CHECK-NEXT: [[TMP4:%.*]] = mul nuw nsw <2 x i32> [[TMP1]], [[TMP3]]
-; CHECK-NEXT: [[TMP5:%.*]] = shufflevector <2 x i32> [[TMP2]], <2 x i32> [[TMP0]], <2 x i32> <i32 0, i32 3>
-; CHECK-NEXT: [[TMP6:%.*]] = mul nuw nsw <2 x i32> [[TMP4]], [[TMP5]]
+; CHECK-NEXT: [[TMP3:%.*]] = mul <2 x i32> [[TMP0]], [[TMP1]]
+; CHECK-NEXT: [[TMP6:%.*]] = mul <2 x i32> [[TMP3]], [[TMP2]]
; CHECK-NEXT: store <2 x i32> [[TMP6]], ptr [[SARRAY]], align 4
; CHECK-NEXT: ret void
;
@@ -207,10 +201,8 @@ define void @test_reassoc_mul_fast(ptr %Aarray, ptr %Barray, ptr %Carray, ptr %S
; CHECK-NEXT: [[TMP0:%.*]] = load <2 x double>, ptr [[AARRAY]], align 8
; CHECK-NEXT: [[TMP1:%.*]] = load <2 x double>, ptr [[BARRAY]], align 8
; CHECK-NEXT: [[TMP2:%.*]] = load <2 x double>, ptr [[CARRAY]], align 8
-; CHECK-NEXT: [[TMP3:%.*]] = shufflevector <2 x double> [[TMP0]], <2 x double> [[TMP2]], <2 x i32> <i32 0, i32 3>
-; CHECK-NEXT: [[TMP4:%.*]] = fmul fast <2 x double> [[TMP1]], [[TMP3]]
-; CHECK-NEXT: [[TMP5:%.*]] = shufflevector <2 x double> [[TMP2]], <2 x double> [[TMP0]], <2 x i32> <i32 0, i32 3>
-; CHECK-NEXT: [[TMP6:%.*]] = fmul fast <2 x double> [[TMP4]], [[TMP5]]
+; CHECK-NEXT: [[TMP3:%.*]] = fmul reassoc nsz arcp contract afn <2 x double> [[TMP0]], [[TMP1]]
+; CHECK-NEXT: [[TMP6:%.*]] = fmul reassoc nsz arcp contract afn <2 x double> [[TMP3]], [[TMP2]]
; CHECK-NEXT: store <2 x double> [[TMP6]], ptr [[SARRAY]], align 8
; CHECK-NEXT: ret void
;
@@ -252,12 +244,9 @@ define void @test_reassoc_add_deep(ptr %Aarray, ptr %Barray, ptr %Carray, ptr %D
; CHECK-NEXT: [[TMP1:%.*]] = load <2 x double>, ptr [[BARRAY]], align 8
; CHECK-NEXT: [[TMP2:%.*]] = load <2 x double>, ptr [[CARRAY]], align 8
; CHECK-NEXT: [[TMP3:%.*]] = load <2 x double>, ptr [[DARRAY]], align 8
-; CHECK-NEXT: [[TMP4:%.*]] = shufflevector <2 x double> [[TMP0]], <2 x double> [[TMP2]], <2 x i32> <i32 0, i32 3>
-; CHECK-NEXT: [[TMP5:%.*]] = fadd fast <2 x double> [[TMP1]], [[TMP4]]
-; CHECK-NEXT: [[TMP6:%.*]] = shufflevector <2 x double> [[TMP2]], <2 x double> [[TMP3]], <2 x i32> <i32 0, i32 3>
-; CHECK-NEXT: [[TMP7:%.*]] = fadd fast <2 x double> [[TMP5]], [[TMP6]]
-; CHECK-NEXT: [[TMP8:%.*]] = shufflevector <2 x double> [[TMP3]], <2 x double> [[TMP0]], <2 x i32> <i32 0, i32 3>
-; CHECK-NEXT: [[TMP9:%.*]] = fadd fast <2 x double> [[TMP7]], [[TMP8]]
+; CHECK-NEXT: [[TMP4:%.*]] = fadd reassoc nsz arcp contract afn <2 x double> [[TMP0]], [[TMP1]]
+; CHECK-NEXT: [[TMP5:%.*]] = fadd reassoc nsz arcp contract afn <2 x double> [[TMP2]], [[TMP3]]
+; CHECK-NEXT: [[TMP9:%.*]] = fadd reassoc nsz arcp contract afn <2 x double> [[TMP4]], [[TMP5]]
; CHECK-NEXT: store <2 x double> [[TMP9]], ptr [[SARRAY]], align 8
; CHECK-NEXT: ret void
;
@@ -313,11 +302,9 @@ define void @test_reassoc_add_balanced_permuted(ptr %Aarray, ptr %Barray, ptr %C
; CHECK-NEXT: [[TMP1:%.*]] = load <2 x double>, ptr [[BARRAY]], align 8
; CHECK-NEXT: [[TMP2:%.*]] = load <2 x double>, ptr [[CARRAY]], align 8
; CHECK-NEXT: [[TMP3:%.*]] = load <2 x double>, ptr [[DARRAY]], align 8
-; CHECK-NEXT: [[TMP4:%.*]] = shufflevector <2 x double> [[TMP0]], <2 x double> [[TMP3]], <2 x i32> <i32 0, i32 3>
-; CHECK-NEXT: [[TMP5:%.*]] = fadd fast <2 x double> [[TMP1]], [[TMP4]]
-; CHECK-NEXT: [[TMP8:%.*]] = shufflevector <2 x double> [[TMP3]], <2 x double> [[TMP0]], <2 x i32> <i32 0, i32 3>
-; CHECK-NEXT: [[TMP7:%.*]] = fadd fast <2 x double> [[TMP8]], [[TMP2]]
-; CHECK-NEXT: [[TMP6:%.*]] = fadd fast <2 x double> [[TMP5]], [[TMP7]]
+; CHECK-NEXT: [[TMP4:%.*]] = fadd reassoc nsz arcp contract afn <2 x double> [[TMP0]], [[TMP1]]
+; CHECK-NEXT: [[TMP5:%.*]] = fadd reassoc nsz arcp contract afn <2 x double> [[TMP2]], [[TMP3]]
+; CHECK-NEXT: [[TMP6:%.*]] = fadd reassoc nsz arcp contract afn <2 x double> [[TMP4]], [[TMP5]]
; CHECK-NEXT: store <2 x double> [[TMP6]], ptr [[SARRAY]], align 8
; CHECK-NEXT: ret void
;
@@ -422,10 +409,8 @@ define void @test_reassoc_and_balanced_permuted(ptr %Aarray, ptr %Barray, ptr %C
; CHECK-NEXT: [[TMP1:%.*]] = load <2 x i32>, ptr [[BARRAY]], align 4
; CHECK-NEXT: [[TMP2:%.*]] = load <2 x i32>, ptr [[CARRAY]], align 4
; CHECK-NEXT: [[TMP3:%.*]] = load <2 x i32>, ptr [[DARRAY]], align 4
-; CHECK-NEXT: [[TMP7:%.*]] = shufflevector <2 x i32> [[TMP0]], <2 x i32> [[TMP3]], <2 x i32> <i32 0, i32 3>
-; CHECK-NEXT: [[TMP4:%.*]] = and <2 x i32> [[TMP1]], [[TMP7]]
-; CHECK-NEXT: [[TMP8:%.*]] = shufflevector <2 x i32> [[TMP3]], <2 x i32> [[TMP0]], <2 x i32> <i32 0, i32 3>
-; CHECK-NEXT: [[TMP5:%.*]] = and <2 x i32> [[TMP8]], [[TMP2]]
+; CHECK-NEXT: [[TMP4:%.*]] = and <2 x i32> [[TMP0]], [[TMP1]]
+; CHECK-NEXT: [[TMP5:%.*]] = and <2 x i32> [[TMP2]], [[TMP3]]
; CHECK-NEXT: [[TMP6:%.*]] = and <2 x i32> [[TMP4]], [[TMP5]]
; CHECK-NEXT: store <2 x i32> [[TMP6]], ptr [[SARRAY]], align 4
; CHECK-NEXT: ret void
@@ -482,10 +467,8 @@ define void @test_reassoc_add_identity(ptr %Aarray, ptr %Barray, ptr %Sarray) {
; CHECK-NEXT: [[ENTRY:.*:]]
; CHECK-NEXT: [[TMP0:%.*]] = load <2 x i32>, ptr [[AARRAY]], align 4
; CHECK-NEXT: [[TMP1:%.*]] = load <2 x i32>, ptr [[BARRAY]], align 4
-; CHECK-NEXT: [[TMP2:%.*]] = shufflevector <2 x i32> [[TMP0]], <2 x i32> <i32 poison, i32 0>, <2 x i32> <i32 0, i32 3>
-; CHECK-NEXT: [[TMP5:%.*]] = add <2 x i32> [[TMP1]], [[TMP2]]
-; CHECK-NEXT: [[TMP4:%.*]] = shufflevector <2 x i32> [[TMP0]], <2 x i32> <i32 0, i32 poison>, <2 x i32> <i32 2, i32 1>
-; CHECK-NEXT: [[TMP3:%.*]] = add <2 x i32> [[TMP5]], [[TMP4]]
+; CHECK-NEXT: [[TMP2:%.*]] = add <2 x i32> [[TMP0]], [[TMP1]]
+; CHECK-NEXT: [[TMP3:%.*]] = add <2 x i32> [[TMP2]], zeroinitializer
; CHECK-NEXT: store <2 x i32> [[TMP3]], ptr [[SARRAY]], align 4
; CHECK-NEXT: ret void
;
diff --git a/llvm/test/Transforms/SLPVectorizer/X86/supernode.ll b/llvm/test/Transforms/SLPVectorizer/X86/supernode.ll
index 02fad2f0e7d41..1018e93bfe421 100644
--- a/llvm/test/Transforms/SLPVectorizer/X86/supernode.ll
+++ b/llvm/test/Transforms/SLPVectorizer/X86/supernode.ll
@@ -10,10 +10,8 @@ define void @test_supernode_add(ptr %Aarray, ptr %Barray, ptr %Carray, ptr %Sarr
; ENABLED-NEXT: [[TMP0:%.*]] = load <2 x double>, ptr [[AARRAY:%.*]], align 8
; ENABLED-NEXT: [[TMP1:%.*]] = load <2 x double>, ptr [[BARRAY:%.*]], align 8
; ENABLED-NEXT: [[TMP2:%.*]] = load <2 x double>, ptr [[CARRAY:%.*]], align 8
-; ENABLED-NEXT: [[TMP3:%.*]] = shufflevector <2 x double> [[TMP0]], <2 x double> [[TMP2]], <2 x i32> <i32 0, i32 3>
-; ENABLED-NEXT: [[TMP4:%.*]] = fadd fast <2 x double> [[TMP1]], [[TMP3]]
-; ENABLED-NEXT: [[TMP5:%.*]] = shufflevector <2 x double> [[TMP2]], <2 x double> [[TMP0]], <2 x i32> <i32 0, i32 3>
-; ENABLED-NEXT: [[TMP6:%.*]] = fadd fast <2 x double> [[TMP4]], [[TMP5]]
+; ENABLED-NEXT: [[TMP3:%.*]] = fadd reassoc nsz arcp contract afn <2 x double> [[TMP0]], [[TMP1]]
+; ENABLED-NEXT: [[TMP6:%.*]] = fadd reassoc nsz arcp contract afn <2 x double> [[TMP3]], [[TMP2]]
; ENABLED-NEXT: store <2 x double> [[TMP6]], ptr [[SARRAY:%.*]], align 8
; ENABLED-NEXT: ret void
;
@@ -163,10 +161,10 @@ define void @supernode_scheduling(ptr %Aarray, ptr %Barray, ptr %Carray, ptr %Da
; ENABLED-NEXT: [[D:%.*]] = load double, ptr [[DARRAY:%.*]], align 8
; ENABLED-NEXT: [[TMP0:%.*]] = load <2 x double>, ptr [[AARRAY:%.*]], align 8
; ENABLED-NEXT: [[TMP1:%.*]] = load <2 x double>, ptr [[BARRAY:%.*]], align 8
-; ENABLED-NEXT: [[TMP2:%.*]] = insertelement <2 x double> [[TMP1]], double [[C]], i32 0
-; ENABLED-NEXT: [[TMP3:%.*]] = fadd fast <2 x double> [[TMP0]], [[TMP2]]
-; ENABLED-NEXT: [[TMP4:%.*]] = insertelement <2 x double> [[TMP1]], double [[D]], i32 1
-; ENABLED-NEXT: [[TMP5:%.*]] = fadd fast <2 x double> [[TMP3]], [[TMP4]]
+; ENABLED-NEXT: [[TMP2:%.*]] = insertelement <2 x double> poison, double [[C]], i32 0
+; ENABLED-NEXT: [[TMP3:%.*]] = insertelement <2 x double> [[TMP2]], double [[D]], i32 1
+; ENABLED-NEXT: [[TMP4:%.*]] = fadd reassoc nsz arcp contract afn <2 x double> [[TMP0]], [[TMP3]]
+; ENABLED-NEXT: [[TMP5:%.*]] = fadd reassoc nsz arcp contract afn <2 x double> [[TMP4]], [[TMP1]]
; ENABLED-NEXT: store <2 x double> [[TMP5]], ptr [[SARRAY:%.*]], align 8
; ENABLED-NEXT: ret void
;
More information about the llvm-commits
mailing list