[llvm] 4376b8d - [SLP]Flatten add/sub and fadd/fsub chains into one reassociated node
via llvm-commits
llvm-commits at lists.llvm.org
Tue Aug 4 11:57:45 PDT 2026
Author: Alexey Bataev
Date: 2026-08-04T14:57:40-04:00
New Revision: 4376b8d66961db360945db44abe510d9f13abf61
URL: https://github.com/llvm/llvm-project/commit/4376b8d66961db360945db44abe510d9f13abf61
DIFF: https://github.com/llvm/llvm-project/commit/4376b8d66961db360945db44abe510d9f13abf61.diff
LOG: [SLP]Flatten add/sub and fadd/fsub chains into one reassociated node
A peeled subtract keeps the column sign for its first operand and flips
it for the second, so mixed add/sub (fadd/fsub with reassoc) chains
linearize into positive and negative operand columns combining as
sum(positive) - sum(negative), with subtrahend columns reordered by
value family. nuw is now also dropped on non-exact combines when a
negated leaf exists, since a regrouped subtrahend sum can exceed the
original running total.
Reviewers: RKSimon, bababuck, hiraditya
Pull Request: https://github.com/llvm/llvm-project/pull/213544
Added:
Modified:
llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp
llvm/lib/Transforms/Vectorize/SLPVectorizer/SLPUtils.cpp
llvm/lib/Transforms/Vectorize/SLPVectorizer/SLPUtils.h
llvm/test/Transforms/SLPVectorizer/X86/reassociate-ops.ll
llvm/test/Transforms/SLPVectorizer/X86/supernode.ll
Removed:
################################################################################
diff --git a/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp b/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp
index e90f969cdf8d6..898ab88dd8ae0 100644
--- a/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp
+++ b/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp
@@ -3402,6 +3402,11 @@ class slpvectorizer::BoUpSLP {
/// inner add in add(add(v0,x),v1)). Not part of Scalars.
SmallVector<Value *, 4> ReassocScalars;
+ /// Sign of each flattened operand column of a reassociated add/sub
+ /// chain, parallel to the operand columns: a negated column is
+ /// subtracted from the positive total. Empty when no column is negated.
+ SmallBitVector ReassocNegatedOps;
+
/// MainOp and AltOp are recorded inside. S should be obtained from
/// newTreeEntry.
InstructionsState S = InstructionsState::invalid();
@@ -3548,6 +3553,18 @@ class slpvectorizer::BoUpSLP {
/// Returns peeled reassociated scalars.
ArrayRef<Value *> getReassocScalars() const { return ReassocScalars; }
+ /// Records the signs of the flattened operand columns.
+ void setReassocNegatedOps(const SmallBitVector &NegatedOps) {
+ assert(NegatedOps.size() == getNumOperands() &&
+ "Signs must cover all operand columns.");
+ ReassocNegatedOps = NegatedOps;
+ }
+
+ /// True if operand column \p Idx is subtracted rather than added.
+ bool isReassocNegatedOp(unsigned Idx) const {
+ return Idx < ReassocNegatedOps.size() && ReassocNegatedOps[Idx];
+ }
+
/// Returns the state of the operations.
const InstructionsState &getOperations() const { return S; }
@@ -12495,21 +12512,41 @@ BoUpSLP::getScalarsVectorizationLegality(ArrayRef<Value *> VL, unsigned Depth,
/// Poison lanes are allowed through; a lane that is not itself a matching
/// real instruction can still be peeled as a copyable identity leaf, as
/// long as some other lane anchors the opcode.
-static void
-scanAssociativeOperands(const InstructionsState &S, DominatorTree &DT,
- const DataLayout &DL, const TargetTransformInfo &TTI,
- const TargetLibraryInfo &TLI, const BoUpSLP &R,
- SmallVectorImpl<BoUpSLP::ValueList> &Operands,
- SmallVectorImpl<Value *> &ReassocScalars) {
- assert(Operands.size() == 2 && "Expected the initial 2 operand columns.");
+/// Add/sub (and fadd/fsub) chains peel together: a peeled subtract keeps
+/// the column sign for its first operand and flips it for the second one,
+/// recorded per column in \p NegatedColumns, so subtracted leaves stay out
+/// of the positive total.
+static void scanAssociativeOperands(
+ const InstructionsState &S, DominatorTree &DT, const DataLayout &DL,
+ const TargetTransformInfo &TTI, const TargetLibraryInfo &TLI,
+ const BoUpSLP &R, SmallVectorImpl<BoUpSLP::ValueList> &Operands,
+ SmallBitVector &NegatedColumns, SmallVectorImpl<Value *> &ReassocScalars) {
+ assert(Operands.size() == 2 && NegatedColumns.empty() &&
+ "Expected the initial 2 operand columns.");
+ // The subtract family flips the sign of its second operand column.
+ auto IsSub = [](unsigned Opcode) {
+ return Opcode == Instruction::Sub || Opcode == Instruction::FSub;
+ };
+ // Signs ride with the columns so the parallel lists cannot drift apart.
+ struct SignedColumn {
+ BoUpSLP::ValueList Col;
+ bool Negated;
+ };
+ SmallVector<SignedColumn, 4> Columns = {
+ {std::move(Operands[0]), false},
+ {std::move(Operands[1]), IsSub(S.getOpcode())}};
InstructionsCompatibilityAnalysis Analysis(DT, DL, TTI, TLI);
- // A lane whose value is itself a single-use associative instruction with
- // S's opcode; block placement does not matter here (buildTreeRec
- // re-checks that later for whatever columns end up as real leaves).
+ // The opcode family a column may peel into: subtracts flatten as adds of
+ // a negated leaf, other opcodes peel only into their own kind.
+ const unsigned RootFamily = getReassocCombineOpcode(S.getOpcode());
+ // A lane whose value is itself a single-use chain link; block placement
+ // does not matter here (buildTreeRec re-checks that later for whatever
+ // columns end up as real leaves).
auto IsRealLane = [&](Value *V) {
auto *I = dyn_cast<Instruction>(V);
- return I && I->getOpcode() == S.getOpcode() && I->isAssociative() &&
- I->hasOneUse();
+ return I && I->hasOneUse() &&
+ getReassocCombineOpcode(I->getOpcode()) == RootFamily &&
+ isReassocChainLink(I);
};
// Builds on InstructionsCompatibilityAnalysis's own opcode matching
// (rather than the more permissive getSameOpcode()) so a column that
@@ -12517,7 +12554,8 @@ scanAssociativeOperands(const InstructionsState &S, DominatorTree &DT,
// that InstructionsCompatibilityAnalysis::isSupportedOpcode() accepts.
auto CanPeel = [&](ArrayRef<Value *> Column) {
InstructionsState ColS = Analysis.buildInstructionsState(Column, R);
- if (!ColS || ColS.getOpcode() != S.getOpcode() || ColS.isAltShuffle())
+ if (!ColS || getReassocCombineOpcode(ColS.getOpcode()) != RootFamily ||
+ ColS.isAltShuffle())
return InstructionsState::invalid();
// Every lane must be poison, a genuine matching real lane, or (only for
// a copyable ColS) stand in as a copyable identity leaf
@@ -12530,27 +12568,35 @@ scanAssociativeOperands(const InstructionsState &S, DominatorTree &DT,
return InstructionsState::invalid();
return ColS;
};
- for (unsigned Idx = 0; Idx != Operands.size();) {
- InstructionsState ColS = CanPeel(Operands[Idx]);
+ for (unsigned Idx = 0; Idx != Columns.size();) {
+ InstructionsState ColS = CanPeel(Columns[Idx].Col);
if (!ColS) {
++Idx;
continue;
}
- BoUpSLP::ValueList Column = std::move(Operands[Idx]);
+ BoUpSLP::ValueList Column = std::move(Columns[Idx].Col);
SmallVector<BoUpSLP::ValueList> SubOperands =
Analysis.buildOperands(ColS, Column);
+ assert(SubOperands.size() == 2 && "Expected 2 operand columns.");
// Poison and copyable lanes have no real instruction left to erase
// later: a copyable V is used as-is, not subsumed by the flattened
// combine.
for (Value *V : Column)
if (!isa<PoisonValue>(V) && !ColS.isCopyableElement(V))
ReassocScalars.push_back(V);
- Operands[Idx] = std::move(SubOperands.front());
- Operands.insert(std::next(Operands.begin(), Idx + 1),
- std::make_move_iterator(std::next(SubOperands.begin())),
- std::make_move_iterator(SubOperands.end()));
+ // A peeled subtract negates its second operand column.
+ const bool Negated = Columns[Idx].Negated;
+ Columns[Idx].Col = std::move(SubOperands.front());
+ Columns.insert(
+ std::next(Columns.begin(), Idx + 1),
+ {std::move(SubOperands.back()), IsSub(ColS.getOpcode()) != Negated});
// Do not advance Idx: re-examine the column that was just placed here.
}
+ Operands.clear();
+ for (auto &[Col, Negated] : Columns) {
+ Operands.push_back(std::move(Col));
+ NegatedColumns.push_back(Negated);
+ }
}
/// Realign flattened operand columns so matching value families share a
@@ -12558,8 +12604,11 @@ scanAssociativeOperands(const InstructionsState &S, DominatorTree &DT,
/// only (never compare hash values). Columns sharing a key are paired by the
/// family of their first operand where available, so e.g. shifts fed by the
/// same load family land in one column instead of pairing by encounter order.
+/// Values move between columns only within the same sign: a subtracted leaf
+/// never lands in an added column.
static SmallVector<BoUpSLP::ValueList>
alignReassociatedOperandsByKey(ArrayRef<BoUpSLP::ValueList> Operands,
+ const SmallBitVector &NegatedColumns,
const TargetLibraryInfo &TLI) {
const unsigned NumCols = Operands.size();
const unsigned NumLanes = Operands.front().size();
@@ -12591,18 +12640,21 @@ alignReassociatedOperandsByKey(ArrayRef<BoUpSLP::ValueList> Operands,
Aligned[Col][0] = Operands[Col][0];
for (unsigned Lane : seq<unsigned>(1, NumLanes)) {
- SmallDenseMap<std::pair<size_t, size_t>, SmallVector<unsigned, 2>, 8>
- Buckets;
+ // Buckets are keyed by the value key and the column sign.
+ using Key = std::pair<std::pair<size_t, size_t>, unsigned>;
+ auto Sign = [&](unsigned Col) { return NegatedColumns[Col] ? 1U : 0U; };
+ SmallDenseMap<Key, SmallVector<unsigned, 2>, 8> Buckets;
for (unsigned Col : seq<unsigned>(NumCols))
- Buckets[GetKey(Operands[Col][Lane])].push_back(Col);
- SmallDenseMap<std::pair<size_t, size_t>, unsigned, 8> BucketCursor;
+ Buckets[{GetKey(Operands[Col][Lane]), Sign(Col)}].push_back(Col);
+ SmallDenseMap<Key, unsigned, 8> NextInBucket;
SmallVector<unsigned> SlotSrcCol(NumCols, NumCols);
SmallBitVector ColClaimed(NumCols, false);
for (unsigned Slot : seq<unsigned>(NumCols)) {
- auto BucketIt = Buckets.find(Lane0Keys[Slot]);
+ const auto SlotKey = std::make_pair(Lane0Keys[Slot], Sign(Slot));
+ auto BucketIt = Buckets.find(SlotKey);
if (BucketIt == Buckets.end())
continue;
- unsigned &Cursor = BucketCursor[Lane0Keys[Slot]];
+ unsigned &Cursor = NextInBucket[SlotKey];
if (Cursor >= BucketIt->second.size())
continue;
// Among the remaining same-key columns prefer the one whose operand
@@ -12620,14 +12672,18 @@ alignReassociatedOperandsByKey(ArrayRef<BoUpSLP::ValueList> Operands,
SlotSrcCol[Slot] = SrcCol;
ColClaimed[SrcCol] = true;
}
- unsigned NextLeftover = 0;
+ // Leftover slots take leftover columns of the same sign; per-lane sign
+ // counts match, so every slot finds one.
for (unsigned Slot : seq<unsigned>(NumCols)) {
if (SlotSrcCol[Slot] != NumCols)
continue;
- while (ColClaimed[NextLeftover])
- ++NextLeftover;
- SlotSrcCol[Slot] = NextLeftover;
- ColClaimed[NextLeftover] = true;
+ for (unsigned Col : seq<unsigned>(NumCols)) {
+ if (!ColClaimed[Col] && NegatedColumns[Col] == NegatedColumns[Slot]) {
+ SlotSrcCol[Slot] = Col;
+ ColClaimed[Col] = true;
+ break;
+ }
+ }
}
for (unsigned Slot : seq<unsigned>(NumCols))
Aligned[Slot][Lane] = Operands[SlotSrcCol[Slot]][Lane];
@@ -12637,6 +12693,7 @@ alignReassociatedOperandsByKey(ArrayRef<BoUpSLP::ValueList> Operands,
/// Rank reassociated operand layouts by vectorizable load columns, other load
/// patterns, broadcast/constant columns, then fewer unique values per column.
+/// Identity columns are scored by the opcode of the emitted combines.
static std::tuple<unsigned, unsigned, unsigned, int>
getReassocColumnsQuality(ArrayRef<BoUpSLP::ValueList> Columns, const BoUpSLP &R,
unsigned Opcode) {
@@ -12644,6 +12701,7 @@ getReassocColumnsQuality(ArrayRef<BoUpSLP::ValueList> Columns, const BoUpSLP &R,
unsigned NumOtherVecLoadCols = 0;
unsigned NumBroadcastOrConstCols = 0;
int NumUniqueValues = 0;
+ const unsigned CombineOpcode = getReassocCombineOpcode(Opcode);
for (ArrayRef<Value *> Col : Columns) {
if (all_of(Col, IsaPred<Constant>)) {
++NumBroadcastOrConstCols;
@@ -12652,7 +12710,7 @@ getReassocColumnsQuality(ArrayRef<BoUpSLP::ValueList> Columns, const BoUpSLP &R,
// Identity constants are free beyond the base they are inserted into.
SmallPtrSet<Value *, 8> UniqueValues;
for (Value *V : Col)
- if (!isBinOpIdentityConstant(V, Opcode))
+ if (!isBinOpIdentityConstant(V, CombineOpcode))
UniqueValues.insert(V);
NumUniqueValues += UniqueValues.size();
if (UniqueValues.size() <= 1) {
@@ -12919,32 +12977,35 @@ void BoUpSLP::buildTreeRec(ArrayRef<Value *> VLRef, unsigned Depth,
SmallVector<ValueList> Operands = Analysis.buildOperands(S, VL);
// Flatten associative binary chains into operand columns. Only the peeled
// chain links are required to be single-use (they are erased); the root
- // being flattened may have other uses. Skip alt-shuffle, copyable, and
- // non-associative nodes. Restricted to BinaryOperator: isAssociative() is
- // also true for associative intrinsics (e.g. smax/smin/umax/umin), which are
- // CallInst, not BinaryOperator, and are not supported by the
- // copyable-identity machinery used below (ConstantExpr::getBinOpIdentity,
- // isSupportedOpcode, isBinOpIdentityConstant).
+ // being flattened may have other uses. Skip alt-shuffle nodes and lanes
+ // that are neither chain links nor copyable identity leaves. Restricted to
+ // BinaryOperator: isAssociative() is also true for associative intrinsics
+ // (e.g. smax/smin/umax/umin), which are CallInst, not BinaryOperator, and
+ // are not supported by the copyable-identity machinery used below
+ // (ConstantExpr::getBinOpIdentity, isSupportedOpcode,
+ // isBinOpIdentityConstant).
SmallVector<Value *> ReassocScalars;
+ // Sign of each flattened operand column (a subtracted leaf is negated).
+ SmallBitVector NegatedColumns;
// Cached below (when the peel is kept) so the reorder step further down
// does not need to redo the aligning/scoring work.
SmallVector<ValueList> ReassocAlignedOperands;
// Snapshot of the pre-flatten operand columns, used by both revert points.
SmallVector<ValueList> NaturalTwoColumns;
std::tuple<unsigned, unsigned, unsigned, int> ReassocPeeledQuality;
- if (VectorizeReassociatedOps && !S.isAltShuffle() &&
- !S.areInstructionsWithCopyableElements() && Operands.size() == 2 &&
- all_of(VL, [](Value *V) {
+ if (VectorizeReassociatedOps && !S.isAltShuffle() && Operands.size() == 2 &&
+ all_of(VL, [&](Value *V) {
auto *I = dyn_cast<BinaryOperator>(V);
- return I && I->isAssociative();
+ return S.isCopyableElement(V) || (I && isReassocChainLink(I));
})) {
NaturalTwoColumns = Operands;
scanAssociativeOperands(S, *DT, *DL, *TTI, *TLI, *this, Operands,
- ReassocScalars);
+ NegatedColumns, ReassocScalars);
// Drop flattening unless realigning improves load or broadcast column
// structure; an unimproved peel ties and reverts to natural columns.
if (!ReassocScalars.empty()) {
- ReassocAlignedOperands = alignReassociatedOperandsByKey(Operands, *TLI);
+ ReassocAlignedOperands =
+ alignReassociatedOperandsByKey(Operands, NegatedColumns, *TLI);
ReassocPeeledQuality =
getReassocColumnsQuality(Operands, *this, S.getOpcode());
// The unique-value count (4th field) is only a tie-break for the
@@ -13278,13 +13339,24 @@ void BoUpSLP::buildTreeRec(ArrayRef<Value *> VLRef, unsigned Depth,
// keeping the polish only if the column-quality score does not regress
// vs the peeled order. That polish is quadratic in the column count, so
// past ReassocReorderColumnLimit keep the realigned seed, which already
- // groups the vectorizable columns.
+ // groups the vectorizable columns. The polish swaps values between
+ // columns per lane, which must not mix added and subtracted leaves in
+ // one column, so signed nodes polish each sign group separately.
if (Operands.size() <= ReassocReorderColumnLimit) {
- VLOperands Ops(ReassocAlignedOperands, VL0->getParent(), *this);
- Ops.reorder();
- SmallVector<ValueList> Reordered =
- map_to_vector(seq<unsigned>(Operands.size()),
- [&](unsigned I) { return Ops.getVL(I); });
+ SmallVector<ValueList> Reordered = ReassocAlignedOperands;
+ SmallVector<unsigned> GroupIds[2];
+ for (unsigned I : seq<unsigned>(NegatedColumns.size()))
+ GroupIds[NegatedColumns[I]].push_back(I);
+ for (ArrayRef<unsigned> Group : GroupIds) {
+ if (Group.size() <= 1)
+ continue;
+ SmallVector<ValueList> GroupCols = map_to_vector(
+ Group, [&](unsigned I) { return ReassocAlignedOperands[I]; });
+ VLOperands Ops(GroupCols, VL0->getParent(), *this);
+ Ops.reorder();
+ for (unsigned Pos : seq<unsigned>(Group.size()))
+ Reordered[Group[Pos]] = Ops.getVL(Pos);
+ }
if (getReassocColumnsQuality(Reordered, *this, S.getOpcode()) >=
ReassocPeeledQuality)
Operands = std::move(Reordered);
@@ -13324,6 +13396,8 @@ void BoUpSLP::buildTreeRec(ArrayRef<Value *> VLRef, unsigned Depth,
Operands[1] = Ops.getVL(1);
}
TE->setOperands(Operands);
+ if (!ReassocScalars.empty() && NegatedColumns.any())
+ TE->setReassocNegatedOps(NegatedColumns);
for (unsigned I : seq<unsigned>(TE->getNumOperands()))
buildTreeRec(TE->getOperand(I), Depth + 1, {TE, I});
return;
@@ -17668,22 +17742,24 @@ BoUpSLP::getEntryCost(const TreeEntry *E, ArrayRef<Value *> VectorizedVals,
// N columns need N-1 vector combines; price extra columns
// conservatively, skipping identity-only columns (not combined by
// codegen).
- if (E->hasReassocScalars())
+ if (E->hasReassocScalars()) {
+ const unsigned CombineOpcode = getReassocCombineOpcode(E->getOpcode());
for (unsigned Idx : seq<unsigned>(2, E->getNumOperands())) {
if (all_of(E->getOperand(Idx), [&](Value *V) {
- return isBinOpIdentityConstant(V, E->getOpcode());
+ return isBinOpIdentityConstant(V, CombineOpcode);
}))
continue;
Cost += TTI->getArithmeticInstrCost(
ShuffleOrOp, VecTy, CostKind, {},
getOperandInfo(E->getOperand(Idx)), {}, nullptr, TLI);
}
+ }
return Cost + CommonCost;
};
// Price peeled intermediate instructions on the scalar side: they are
// erased when the node vectorizes. Folded into the first scalar-cost
// query so the cost dump reports the full scalar cost. These are always
- // 2-operand associative binops (isAssociative() excludes UnaryOperator),
+ // 2-operand chain links (adds, subtracts, or other associative binops),
// so operand 1 is always the second operand.
InstructionCost PeeledScalarCost = 0;
for (Value *V : E->getReassocScalars()) {
@@ -17691,7 +17767,7 @@ BoUpSLP::getEntryCost(const TreeEntry *E, ArrayRef<Value *> VectorizedVals,
TTI::OperandValueInfo Op1Info = TTI::getOperandInfo(I->getOperand(0));
TTI::OperandValueInfo Op2Info = TTI::getOperandInfo(I->getOperand(1));
PeeledScalarCost += TTI->getArithmeticInstrCost(
- ShuffleOrOp, OrigScalarTy, CostKind, Op1Info, Op2Info);
+ I->getOpcode(), OrigScalarTy, CostKind, Op1Info, Op2Info);
}
bool PeeledCostAdded = false;
InstructionCost CostDiff = GetCostDiff(
@@ -23921,21 +23997,34 @@ Value *BoUpSLP::vectorizeTree(TreeEntry *E) {
setInsertPointAfterBundle(E);
if (E->hasReassocScalars()) {
- // Vectorize operand columns, then combine pairwise in a balanced tree.
+ // Vectorize operand columns, then combine pairwise in a balanced
+ // tree; negated columns (flattened subtracts) are subtracted from
+ // the positive total at the end.
SmallVector<Value *> CombinedScalars = to_vector(E->Scalars);
append_range(CombinedScalars, E->getReassocScalars());
// Reuse flags when a combine exactly matches a subsumed scalar pair.
- auto MakeOperandKey = [](Value *A, Value *B) {
- return A <= B ? std::make_pair(A, B) : std::make_pair(B, A);
+ // Commutative scalars are keyed operand-order independently,
+ // subtracts keep their operand order.
+ auto MakeOperandKey = [](Value *A, Value *B, unsigned Opcode) {
+ return Instruction::isCommutative(Opcode) && B < A
+ ? std::make_pair(B, A)
+ : std::make_pair(A, B);
};
- SmallDenseMap<std::pair<Value *, Value *>,
+ SmallDenseMap<std::pair<std::pair<Value *, Value *>, unsigned>,
SmallVector<Instruction *, 1>>
ByOperands;
for (Value *V : CombinedScalars) {
+ // Copyable leaves are used as-is, they are never combine targets.
+ if (E->isCopyableElement(V))
+ continue;
auto *I = cast<Instruction>(V);
- ByOperands[MakeOperandKey(I->getOperand(0), I->getOperand(1))]
+ const unsigned Opcode = I->getOpcode();
+ ByOperands[{MakeOperandKey(I->getOperand(0), I->getOperand(1),
+ Opcode),
+ Opcode}]
.push_back(I);
}
+ const unsigned CombineOpcode = getReassocCombineOpcode(E->getOpcode());
// Cast operand columns to VecTy when bit-width demotion changed types.
auto GetCastOperand = [&](unsigned Idx, Value *Op) {
if (Op->getType() == VecTy)
@@ -23943,83 +24032,155 @@ Value *BoUpSLP::vectorizeTree(TreeEntry *E) {
return Builder.CreateIntCast(Op, VecTy, GetOperandSignedness(Idx));
};
SmallVector<Value *> Ops;
+ SmallVector<Value *> NegOps;
// Track which scalar each partial result still represents for flag
// reuse.
SmallVector<SmallVector<Value *>> ScalarOps;
+ SmallVector<SmallVector<Value *>> NegScalarOps;
for (unsigned Idx : seq<unsigned>(E->getNumOperands())) {
// Identity-only columns combine to a no-op; skip them rather than
// emitting a real binop against an identity vector.
if (all_of(E->getOperand(Idx), [&](Value *V) {
- return isBinOpIdentityConstant(V, E->getOpcode());
+ return isBinOpIdentityConstant(V, CombineOpcode);
}))
continue;
- Ops.push_back(GetCastOperand(Idx, vectorizeOperand(E, Idx)));
- ScalarOps.emplace_back(E->getOperand(Idx).begin(),
- E->getOperand(Idx).end());
- }
- assert(!Ops.empty() && "Expected at least one non-identity column.");
- while (Ops.size() > 1) {
- SmallVector<Value *> NextOps((Ops.size() + 1) / 2);
- SmallVector<SmallVector<Value *>> NextScalarOps(NextOps.size());
- for (unsigned Idx : seq<unsigned>(NextOps.size())) {
- if (2 * Idx + 1 == Ops.size()) {
- // Odd one out this round: carry it over unpaired.
- NextOps[Idx] = Ops[2 * Idx];
- NextScalarOps[Idx] = std::move(ScalarOps[2 * Idx]);
- continue;
- }
- Value *Combined = Builder.CreateBinOp(
- static_cast<Instruction::BinaryOps>(E->getOpcode()),
- Ops[2 * Idx], Ops[2 * Idx + 1]);
- // Check whether this combine exactly matches a subsumed scalar.
- const ArrayRef<Value *> LHSScalars = ScalarOps[2 * Idx];
- const ArrayRef<Value *> RHSScalars = ScalarOps[2 * Idx + 1];
- SmallVector<Value *> ExactMatches;
- // Cache per-lane representatives for the next combine round.
- SmallVector<Value *> Representatives(LHSScalars.size());
- // Either side may be empty after a non-exact combine.
- bool IsExact =
- !LHSScalars.empty() && LHSScalars.size() == RHSScalars.size() &&
- all_of(seq<unsigned>(LHSScalars.size()), [&](unsigned Lane) {
- if (!LHSScalars[Lane] || !RHSScalars[Lane])
- return false;
- auto It = ByOperands.find(
- MakeOperandKey(LHSScalars[Lane], RHSScalars[Lane]));
- if (It == ByOperands.end())
- return false;
- append_range(ExactMatches, It->second);
- Representatives[Lane] = It->second.front();
- return true;
- });
- if (IsExact) {
- NextOps[Idx] =
- PropagateIRFlags(Combined, E->getOpcode(), ExactMatches);
- NextScalarOps[Idx] = std::move(Representatives);
- continue;
- }
- NextOps[Idx] =
- PropagateIRFlags(Combined, E->getOpcode(), CombinedScalars);
- // Drop overflow/fast-math flags not proven exact; regrouping can
- // invalidate nsw/nuw and nnan/ninf even when each step was safe.
- if (auto *CombinedI = dyn_cast<Instruction>(NextOps[Idx])) {
- if (E->getOpcode() == Instruction::Add ||
- E->getOpcode() == Instruction::Mul) {
- CombinedI->setHasNoSignedWrap(false);
- if (E->getOpcode() != Instruction::Add)
- CombinedI->setHasNoUnsignedWrap(false);
- } else if (E->getOpcode() == Instruction::FAdd ||
- E->getOpcode() == Instruction::FMul) {
- CombinedI->setHasNoNaNs(false);
- CombinedI->setHasNoInfs(false);
+ const bool Negated = E->isReassocNegatedOp(Idx);
+ (Negated ? NegOps : Ops)
+ .push_back(GetCastOperand(Idx, vectorizeOperand(E, Idx)));
+ (Negated ? NegScalarOps : ScalarOps)
+ .emplace_back(E->getOperand(Idx).begin(),
+ E->getOperand(Idx).end());
+ }
+ const bool HasNegatedOps = !NegOps.empty();
+ // Check whether combining the partial results of \p LHSScalars and
+ // \p RHSScalars reproduces a subsumed scalar pair lane by lane, so
+ // the matched scalars' flags can be reused. Fills per-lane
+ // representatives for the next combine round on an exact match.
+ auto TryExactCombine =
+ [&](Value *Combined, unsigned Opcode, ArrayRef<Value *> LHSScalars,
+ ArrayRef<Value *> RHSScalars,
+ SmallVectorImpl<Value *> &Representatives) -> Value * {
+ // Either side may be empty after a non-exact combine.
+ if (LHSScalars.empty() || LHSScalars.size() != RHSScalars.size())
+ return nullptr;
+ SmallVector<Value *> ExactMatches;
+ Representatives.resize(LHSScalars.size());
+ bool IsExact =
+ all_of(seq<unsigned>(LHSScalars.size()), [&](unsigned Lane) {
+ if (!LHSScalars[Lane] || !RHSScalars[Lane])
+ return false;
+ auto It = ByOperands.find(
+ {MakeOperandKey(LHSScalars[Lane], RHSScalars[Lane], Opcode),
+ Opcode});
+ if (It == ByOperands.end())
+ return false;
+ append_range(ExactMatches, It->second);
+ Representatives[Lane] = It->second.front();
+ return true;
+ });
+ if (!IsExact)
+ return nullptr;
+ return PropagateIRFlags(Combined, Opcode, ExactMatches);
+ };
+ auto CombineFallback = [&](Value *Combined, unsigned Opcode) {
+ Value *V = PropagateIRFlags(Combined, Opcode, CombinedScalars);
+ // Drop overflow/fast-math flags not proven exact; regrouping can
+ // invalidate nsw/nuw and nnan/ninf even when each step was safe.
+ // nuw is kept for pure add trees only: a negated leaf can make a
+ // re-paired partial sum exceed the original running total.
+ auto *CombinedI = dyn_cast<Instruction>(V);
+ if (!CombinedI)
+ return V;
+ if (isa<FPBinaryOperator>(CombinedI)) {
+ CombinedI->setHasNoNaNs(false);
+ CombinedI->setHasNoInfs(false);
+ } else if (isa<OverflowingBinaryOperator>(CombinedI)) {
+ CombinedI->setHasNoSignedWrap(false);
+ if (Opcode != Instruction::Add || HasNegatedOps)
+ CombinedI->setHasNoUnsignedWrap(false);
+ }
+ return V;
+ };
+ // Combine the group columns pairwise in a balanced tree and return
+ // the total with the per-lane scalars it still represents.
+ auto CombineColumns =
+ [&](SmallVector<Value *> GroupOps,
+ SmallVector<SmallVector<Value *>> GroupScalarOps,
+ unsigned Opcode) -> std::pair<Value *, SmallVector<Value *>> {
+ while (GroupOps.size() > 1) {
+ SmallVector<Value *> NextOps((GroupOps.size() + 1) / 2);
+ SmallVector<SmallVector<Value *>> NextScalarOps(NextOps.size());
+ for (unsigned Idx : seq<unsigned>(NextOps.size())) {
+ if (2 * Idx + 1 == GroupOps.size()) {
+ // Odd one out this round: carry it over unpaired.
+ NextOps[Idx] = GroupOps[2 * Idx];
+ NextScalarOps[Idx] = std::move(GroupScalarOps[2 * Idx]);
+ continue;
+ }
+ Value *Combined = Builder.CreateBinOp(
+ static_cast<Instruction::BinaryOps>(Opcode),
+ GroupOps[2 * Idx], GroupOps[2 * Idx + 1]);
+ // Check whether this combine exactly matches a subsumed
+ // scalar.
+ SmallVector<Value *> Representatives;
+ if (Value *Exact = TryExactCombine(
+ Combined, Opcode, GroupScalarOps[2 * Idx],
+ GroupScalarOps[2 * Idx + 1], Representatives)) {
+ NextOps[Idx] = Exact;
+ NextScalarOps[Idx] = std::move(Representatives);
+ continue;
}
+ NextOps[Idx] = CombineFallback(Combined, Opcode);
+ // No scalar match: deeper combines from this result cannot be
+ // exact.
}
- // No scalar match: deeper combines from this result cannot be
- // exact.
+ GroupOps = std::move(NextOps);
+ GroupScalarOps = std::move(NextScalarOps);
+ }
+ return {GroupOps.front(), GroupScalarOps.front()};
+ };
+ // -(a+b)
diff ers from -a + -b in the sign of a zero result, so the
+ // negated columns are summed only for integers or with nsz on every
+ // link; without it each is subtracted in turn.
+ const bool CanSumNegated =
+ CombineOpcode != Instruction::FAdd ||
+ all_of(CombinedScalars, [&](Value *V) {
+ auto *I = dyn_cast<Instruction>(V);
+ return !I || E->isCopyableElement(V) || I->hasNoSignedZeros();
+ });
+ const unsigned SubOpcode = CombineOpcode == Instruction::FAdd
+ ? Instruction::FSub
+ : Instruction::Sub;
+ Value *V;
+ SmallVector<Value *> PosReps;
+ if (Ops.empty()) {
+ // Every positive column was an identity: the total is the identity.
+ V = ConstantExpr::getBinOpIdentity(CombineOpcode, VecTy);
+ } else {
+ std::tie(V, PosReps) = CombineColumns(
+ std::move(Ops), std::move(ScalarOps), CombineOpcode);
+ }
+ if (CanSumNegated && !NegOps.empty()) {
+ auto [NegTotal, NegReps] = CombineColumns(
+ std::move(NegOps), std::move(NegScalarOps), CombineOpcode);
+ NegOps = {NegTotal};
+ NegScalarOps.clear();
+ NegScalarOps.push_back(std::move(NegReps));
+ }
+ for (auto [NegOp, NegScalars] : zip(NegOps, NegScalarOps)) {
+ Value *Combined = Builder.CreateBinOp(
+ static_cast<Instruction::BinaryOps>(SubOpcode), V, NegOp);
+ SmallVector<Value *> NextReps;
+ if (Value *Exact = TryExactCombine(Combined, SubOpcode, PosReps,
+ NegScalars, NextReps)) {
+ V = Exact;
+ PosReps = std::move(NextReps);
+ continue;
}
- Ops = std::move(NextOps);
- ScalarOps = std::move(NextScalarOps);
+ V = CombineFallback(Combined, SubOpcode);
+ PosReps.clear();
}
- Value *V = FinalShuffle(Ops.front(), E);
+ V = FinalShuffle(V, E);
E->VectorizedValue = V;
++NumVectorInstructions;
return V;
diff --git a/llvm/lib/Transforms/Vectorize/SLPVectorizer/SLPUtils.cpp b/llvm/lib/Transforms/Vectorize/SLPVectorizer/SLPUtils.cpp
index 1b8d393ed1a81..2889661006e1c 100644
--- a/llvm/lib/Transforms/Vectorize/SLPVectorizer/SLPUtils.cpp
+++ b/llvm/lib/Transforms/Vectorize/SLPVectorizer/SLPUtils.cpp
@@ -40,6 +40,25 @@ bool isBinOpIdentityConstant(const Value *V, unsigned Opcode) {
return CI && ConstantExpr::getBinOpIdentity(Opcode, CI->getType()) == CI;
}
+unsigned getReassocCombineOpcode(unsigned Opcode) {
+ switch (Opcode) {
+ case Instruction::Sub:
+ return Instruction::Add;
+ case Instruction::FSub:
+ return Instruction::FAdd;
+ default:
+ return Opcode;
+ }
+}
+
+bool isReassocChainLink(const Instruction *I) {
+ if (I->getOpcode() == Instruction::Sub)
+ return true;
+ if (I->getOpcode() == Instruction::FSub)
+ return I->hasAllowReassoc();
+ return I->isAssociative();
+}
+
bool isVectorLikeInstWithConstOps(Value *V) {
auto *I = dyn_cast<Instruction>(V);
// Non-instructions are vector-like only if they are undef.
diff --git a/llvm/lib/Transforms/Vectorize/SLPVectorizer/SLPUtils.h b/llvm/lib/Transforms/Vectorize/SLPVectorizer/SLPUtils.h
index 8d10f2b123dff..d826e197656b1 100644
--- a/llvm/lib/Transforms/Vectorize/SLPVectorizer/SLPUtils.h
+++ b/llvm/lib/Transforms/Vectorize/SLPVectorizer/SLPUtils.h
@@ -51,6 +51,16 @@ bool isConstant(Value *V);
/// returns for FAdd/FMul, whose identity fast-math may break anyway.
bool isBinOpIdentityConstant(const Value *V, unsigned Opcode);
+/// \returns the opcode of the combines emitted for a reassociated node:
+/// subtract chains regroup their positive and negative operand columns with
+/// plain adds.
+unsigned getReassocCombineOpcode(unsigned Opcode);
+
+/// \returns True if \p I can be a link of a flattenable binary chain:
+/// subtracts flatten as adds of a negated leaf, float subtracts need reassoc
+/// to allow the regrouping.
+bool isReassocChainLink(const Instruction *I);
+
/// Checks if \p V is one of vector-like instructions, i.e. undef,
/// insertelement/extractelement with constant indices for fixed vector type
/// or extractvalue instruction.
diff --git a/llvm/test/Transforms/SLPVectorizer/X86/reassociate-ops.ll b/llvm/test/Transforms/SLPVectorizer/X86/reassociate-ops.ll
index ca8d1a334ebae..5db2d0b3165fb 100644
--- a/llvm/test/Transforms/SLPVectorizer/X86/reassociate-ops.ll
+++ b/llvm/test/Transforms/SLPVectorizer/X86/reassociate-ops.ll
@@ -616,11 +616,9 @@ define void @test_reassoc_add_sub(ptr %Aarray, ptr %Barray, ptr %Carray, ptr %Da
; CHECK-NEXT: [[TMP1:%.*]] = load <2 x i32>, ptr [[BARRAY]], align 4
; CHECK-NEXT: [[TMP2:%.*]] = load <2 x i32>, ptr [[CARRAY]], align 4
; CHECK-NEXT: [[TMP3:%.*]] = load <2 x i32>, ptr [[DARRAY]], align 4
-; CHECK-NEXT: [[TMP4:%.*]] = shufflevector <2 x i32> [[TMP1]], <2 x i32> [[TMP3]], <2 x i32> <i32 0, i32 3>
-; CHECK-NEXT: [[TMP5:%.*]] = sub <2 x i32> [[TMP0]], [[TMP4]]
-; CHECK-NEXT: [[TMP8:%.*]] = shufflevector <2 x i32> [[TMP3]], <2 x i32> [[TMP1]], <2 x i32> <i32 0, i32 3>
-; CHECK-NEXT: [[TMP7:%.*]] = sub <2 x i32> [[TMP2]], [[TMP8]]
-; CHECK-NEXT: [[TMP6:%.*]] = add <2 x i32> [[TMP5]], [[TMP7]]
+; CHECK-NEXT: [[TMP5:%.*]] = add <2 x i32> [[TMP0]], [[TMP2]]
+; CHECK-NEXT: [[TMP4:%.*]] = add <2 x i32> [[TMP1]], [[TMP3]]
+; CHECK-NEXT: [[TMP6:%.*]] = sub <2 x i32> [[TMP5]], [[TMP4]]
; CHECK-NEXT: store <2 x i32> [[TMP6]], ptr [[SARRAY]], align 4
; CHECK-NEXT: ret void
;
@@ -670,10 +668,8 @@ define void @test_reassoc_sub_chain_wrapflags(ptr %Aarray, ptr %Barray, ptr %Car
; CHECK-NEXT: [[TMP0:%.*]] = load <2 x i32>, ptr [[AARRAY]], align 4
; CHECK-NEXT: [[TMP1:%.*]] = load <2 x i32>, ptr [[BARRAY]], align 4
; CHECK-NEXT: [[TMP2:%.*]] = load <2 x i32>, ptr [[CARRAY]], align 4
-; CHECK-NEXT: [[TMP3:%.*]] = shufflevector <2 x i32> [[TMP1]], <2 x i32> [[TMP2]], <2 x i32> <i32 0, i32 3>
-; CHECK-NEXT: [[TMP6:%.*]] = sub nuw nsw <2 x i32> [[TMP0]], [[TMP3]]
-; CHECK-NEXT: [[TMP5:%.*]] = shufflevector <2 x i32> [[TMP2]], <2 x i32> [[TMP1]], <2 x i32> <i32 0, i32 3>
-; CHECK-NEXT: [[TMP4:%.*]] = sub nuw nsw <2 x i32> [[TMP6]], [[TMP5]]
+; CHECK-NEXT: [[TMP3:%.*]] = add <2 x i32> [[TMP1]], [[TMP2]]
+; CHECK-NEXT: [[TMP4:%.*]] = sub <2 x i32> [[TMP0]], [[TMP3]]
; CHECK-NEXT: store <2 x i32> [[TMP4]], ptr [[SARRAY]], align 4
; CHECK-NEXT: ret void
;
@@ -718,7 +714,7 @@ define void @test_reassoc_sub_preserves_flags(ptr %Aarray, ptr %Barray, ptr %Car
; CHECK-NEXT: [[TMP0:%.*]] = load <2 x i32>, ptr [[AARRAY]], align 4
; CHECK-NEXT: [[TMP1:%.*]] = load <2 x i32>, ptr [[BARRAY]], align 4
; CHECK-NEXT: [[TMP2:%.*]] = load <2 x i32>, ptr [[CARRAY]], align 4
-; CHECK-NEXT: [[TMP3:%.*]] = add nuw nsw <2 x i32> [[TMP1]], [[TMP0]]
+; CHECK-NEXT: [[TMP3:%.*]] = add nuw nsw <2 x i32> [[TMP0]], [[TMP1]]
; CHECK-NEXT: [[TMP4:%.*]] = sub nuw nsw <2 x i32> [[TMP3]], [[TMP2]]
; CHECK-NEXT: store <2 x i32> [[TMP4]], ptr [[SARRAY]], align 4
; CHECK-NEXT: ret void
@@ -763,11 +759,9 @@ define void @test_reassoc_fadd_fsub(ptr %Aarray, ptr %Barray, ptr %Carray, ptr %
; CHECK-NEXT: [[TMP1:%.*]] = load <2 x double>, ptr [[BARRAY]], align 8
; CHECK-NEXT: [[TMP2:%.*]] = load <2 x double>, ptr [[CARRAY]], align 8
; CHECK-NEXT: [[TMP3:%.*]] = load <2 x double>, ptr [[DARRAY]], align 8
-; CHECK-NEXT: [[TMP4:%.*]] = shufflevector <2 x double> [[TMP1]], <2 x double> [[TMP3]], <2 x i32> <i32 0, i32 3>
-; CHECK-NEXT: [[TMP5:%.*]] = fsub fast <2 x double> [[TMP0]], [[TMP4]]
-; CHECK-NEXT: [[TMP8:%.*]] = shufflevector <2 x double> [[TMP3]], <2 x double> [[TMP1]], <2 x i32> <i32 0, i32 3>
-; CHECK-NEXT: [[TMP7:%.*]] = fsub fast <2 x double> [[TMP2]], [[TMP8]]
-; CHECK-NEXT: [[TMP6:%.*]] = fadd fast <2 x double> [[TMP5]], [[TMP7]]
+; CHECK-NEXT: [[TMP5:%.*]] = fadd reassoc nsz arcp contract afn <2 x double> [[TMP0]], [[TMP2]]
+; CHECK-NEXT: [[TMP4:%.*]] = fadd reassoc nsz arcp contract afn <2 x double> [[TMP1]], [[TMP3]]
+; CHECK-NEXT: [[TMP6:%.*]] = fsub reassoc nsz arcp contract afn <2 x double> [[TMP5]], [[TMP4]]
; CHECK-NEXT: store <2 x double> [[TMP6]], ptr [[SARRAY]], align 8
; CHECK-NEXT: ret void
;
@@ -858,10 +852,8 @@ define void @test_reassoc_fsub_chain(ptr %Aarray, ptr %Barray, ptr %Carray, ptr
; CHECK-NEXT: [[TMP0:%.*]] = load <2 x double>, ptr [[AARRAY]], align 8
; CHECK-NEXT: [[TMP1:%.*]] = load <2 x double>, ptr [[BARRAY]], align 8
; CHECK-NEXT: [[TMP2:%.*]] = load <2 x double>, ptr [[CARRAY]], align 8
-; CHECK-NEXT: [[TMP3:%.*]] = shufflevector <2 x double> [[TMP1]], <2 x double> [[TMP2]], <2 x i32> <i32 0, i32 3>
-; CHECK-NEXT: [[TMP4:%.*]] = fsub reassoc <2 x double> [[TMP0]], [[TMP3]]
-; CHECK-NEXT: [[TMP5:%.*]] = shufflevector <2 x double> [[TMP2]], <2 x double> [[TMP1]], <2 x i32> <i32 0, i32 3>
-; CHECK-NEXT: [[TMP6:%.*]] = fsub reassoc <2 x double> [[TMP4]], [[TMP5]]
+; CHECK-NEXT: [[TMP3:%.*]] = fsub reassoc <2 x double> [[TMP0]], [[TMP1]]
+; CHECK-NEXT: [[TMP6:%.*]] = fsub reassoc <2 x double> [[TMP3]], [[TMP2]]
; CHECK-NEXT: store <2 x double> [[TMP6]], ptr [[SARRAY]], align 8
; CHECK-NEXT: ret void
;
@@ -904,11 +896,9 @@ define void @test_reassoc_sub_nested_signs(ptr %Aarray, ptr %Barray, ptr %Carray
; CHECK-NEXT: [[TMP1:%.*]] = load <2 x i32>, ptr [[BARRAY]], align 4
; CHECK-NEXT: [[TMP2:%.*]] = load <2 x i32>, ptr [[CARRAY]], align 4
; CHECK-NEXT: [[TMP3:%.*]] = load <2 x i32>, ptr [[DARRAY]], align 4
-; CHECK-NEXT: [[TMP4:%.*]] = shufflevector <2 x i32> [[TMP1]], <2 x i32> [[TMP3]], <2 x i32> <i32 0, i32 3>
-; CHECK-NEXT: [[TMP5:%.*]] = add <2 x i32> [[TMP0]], [[TMP4]]
-; CHECK-NEXT: [[TMP8:%.*]] = shufflevector <2 x i32> [[TMP3]], <2 x i32> [[TMP1]], <2 x i32> <i32 0, i32 3>
-; CHECK-NEXT: [[TMP7:%.*]] = sub <2 x i32> [[TMP2]], [[TMP8]]
-; CHECK-NEXT: [[TMP6:%.*]] = sub <2 x i32> [[TMP5]], [[TMP7]]
+; CHECK-NEXT: [[TMP4:%.*]] = add <2 x i32> [[TMP0]], [[TMP1]]
+; CHECK-NEXT: [[TMP5:%.*]] = add <2 x i32> [[TMP4]], [[TMP3]]
+; CHECK-NEXT: [[TMP6:%.*]] = sub <2 x i32> [[TMP5]], [[TMP2]]
; CHECK-NEXT: store <2 x i32> [[TMP6]], ptr [[SARRAY]], align 4
; CHECK-NEXT: ret void
;
@@ -959,7 +949,7 @@ define void @test_reassoc_sub_neg_group_exact(ptr %Aarray, ptr %Barray, ptr %Car
; CHECK-NEXT: [[TMP0:%.*]] = load <2 x i32>, ptr [[AARRAY]], align 4
; CHECK-NEXT: [[TMP1:%.*]] = load <2 x i32>, ptr [[BARRAY]], align 4
; CHECK-NEXT: [[TMP2:%.*]] = load <2 x i32>, ptr [[CARRAY]], align 4
-; CHECK-NEXT: [[TMP3:%.*]] = add nuw nsw <2 x i32> [[TMP2]], [[TMP1]]
+; CHECK-NEXT: [[TMP3:%.*]] = add nuw nsw <2 x i32> [[TMP1]], [[TMP2]]
; CHECK-NEXT: [[TMP4:%.*]] = sub nuw nsw <2 x i32> [[TMP0]], [[TMP3]]
; CHECK-NEXT: store <2 x i32> [[TMP4]], ptr [[SARRAY]], align 4
; CHECK-NEXT: ret void
@@ -1003,7 +993,7 @@ define void @test_reassoc_sub_mixed_lane_flags(ptr %Aarray, ptr %Barray, ptr %Ca
; CHECK-NEXT: [[TMP0:%.*]] = load <2 x i32>, ptr [[AARRAY]], align 4
; CHECK-NEXT: [[TMP1:%.*]] = load <2 x i32>, ptr [[BARRAY]], align 4
; CHECK-NEXT: [[TMP2:%.*]] = load <2 x i32>, ptr [[CARRAY]], align 4
-; CHECK-NEXT: [[TMP3:%.*]] = add <2 x i32> [[TMP2]], [[TMP1]]
+; CHECK-NEXT: [[TMP3:%.*]] = add <2 x i32> [[TMP1]], [[TMP2]]
; CHECK-NEXT: [[TMP4:%.*]] = sub <2 x i32> [[TMP0]], [[TMP3]]
; CHECK-NEXT: store <2 x i32> [[TMP4]], ptr [[SARRAY]], align 4
; CHECK-NEXT: ret void
@@ -1125,11 +1115,9 @@ define void @test_reassoc_sub_external_uses(ptr %Aarray, ptr %Barray, ptr %Carra
; CHECK-NEXT: [[TMP1:%.*]] = load <4 x i32>, ptr [[BARRAY]], align 4
; CHECK-NEXT: [[TMP2:%.*]] = load <4 x i32>, ptr [[CARRAY]], align 4
; CHECK-NEXT: [[TMP3:%.*]] = load <4 x i32>, ptr [[DARRAY]], align 4
-; CHECK-NEXT: [[TMP4:%.*]] = shufflevector <4 x i32> [[TMP1]], <4 x i32> [[TMP3]], <4 x i32> <i32 0, i32 5, i32 2, i32 7>
-; CHECK-NEXT: [[TMP5:%.*]] = sub <4 x i32> [[TMP0]], [[TMP4]]
-; CHECK-NEXT: [[TMP8:%.*]] = shufflevector <4 x i32> [[TMP3]], <4 x i32> [[TMP1]], <4 x i32> <i32 0, i32 5, i32 2, i32 7>
-; CHECK-NEXT: [[TMP9:%.*]] = sub <4 x i32> [[TMP2]], [[TMP8]]
-; CHECK-NEXT: [[TMP6:%.*]] = add <4 x i32> [[TMP5]], [[TMP9]]
+; CHECK-NEXT: [[TMP5:%.*]] = add <4 x i32> [[TMP0]], [[TMP2]]
+; CHECK-NEXT: [[TMP4:%.*]] = add <4 x i32> [[TMP1]], [[TMP3]]
+; CHECK-NEXT: [[TMP6:%.*]] = sub <4 x i32> [[TMP5]], [[TMP4]]
; CHECK-NEXT: store <4 x i32> [[TMP6]], ptr [[SARRAY]], align 4
; CHECK-NEXT: [[TMP7:%.*]] = extractelement <4 x i32> [[TMP6]], i64 0
; CHECK-NEXT: [[EXT:%.*]] = add i32 [[TMP7]], [[A0]]
@@ -1221,10 +1209,10 @@ define void @test_reassoc_sub_copyable(ptr %Aarray, ptr %Barray, ptr %Xarray, pt
; CHECK-NEXT: [[TMP3:%.*]] = insertelement <4 x i32> <i32 poison, i32 poison, i32 poison, i32 0>, i32 [[X2]], i64 2
; CHECK-NEXT: [[TMP4:%.*]] = shufflevector <2 x i32> [[TMP2]], <2 x i32> poison, <4 x i32> <i32 0, i32 1, i32 poison, i32 poison>
; CHECK-NEXT: [[TMP5:%.*]] = shufflevector <4 x i32> [[TMP3]], <4 x i32> [[TMP4]], <4 x i32> <i32 4, i32 5, i32 2, i32 3>
-; CHECK-NEXT: [[TMP9:%.*]] = add <4 x i32> [[TMP0]], [[TMP5]]
; CHECK-NEXT: [[TMP6:%.*]] = insertelement <4 x i32> <i32 poison, i32 poison, i32 poison, i32 0>, i32 [[B2]], i64 2
; CHECK-NEXT: [[TMP7:%.*]] = shufflevector <2 x i32> [[TMP1]], <2 x i32> poison, <4 x i32> <i32 0, i32 1, i32 poison, i32 poison>
; CHECK-NEXT: [[TMP8:%.*]] = shufflevector <4 x i32> [[TMP6]], <4 x i32> [[TMP7]], <4 x i32> <i32 4, i32 5, i32 2, i32 3>
+; CHECK-NEXT: [[TMP9:%.*]] = add <4 x i32> [[TMP0]], [[TMP5]]
; CHECK-NEXT: [[TMP10:%.*]] = sub <4 x i32> [[TMP9]], [[TMP8]]
; CHECK-NEXT: store <4 x i32> [[TMP10]], ptr [[SARRAY]], align 4
; CHECK-NEXT: ret void
diff --git a/llvm/test/Transforms/SLPVectorizer/X86/supernode.ll b/llvm/test/Transforms/SLPVectorizer/X86/supernode.ll
index 1f97093b38180..8c3aac2d59c11 100644
--- a/llvm/test/Transforms/SLPVectorizer/X86/supernode.ll
+++ b/llvm/test/Transforms/SLPVectorizer/X86/supernode.ll
@@ -49,10 +49,8 @@ define void @test_supernode_addsub(ptr %Aarray, ptr %Barray, ptr %Carray, ptr %S
; ENABLED-NEXT: [[TMP0:%.*]] = load <2 x double>, ptr [[AARRAY:%.*]], align 8
; ENABLED-NEXT: [[TMP1:%.*]] = load <2 x double>, ptr [[BARRAY:%.*]], align 8
; ENABLED-NEXT: [[TMP2:%.*]] = load <2 x double>, ptr [[CARRAY:%.*]], align 8
-; ENABLED-NEXT: [[TMP3:%.*]] = shufflevector <2 x double> [[TMP0]], <2 x double> [[TMP2]], <2 x i32> <i32 0, i32 3>
-; ENABLED-NEXT: [[TMP4:%.*]] = fsub fast <2 x double> [[TMP3]], [[TMP1]]
-; ENABLED-NEXT: [[TMP5:%.*]] = shufflevector <2 x double> [[TMP2]], <2 x double> [[TMP0]], <2 x i32> <i32 0, i32 3>
-; ENABLED-NEXT: [[TMP6:%.*]] = fadd fast <2 x double> [[TMP4]], [[TMP5]]
+; ENABLED-NEXT: [[TMP3:%.*]] = fadd reassoc nsz arcp contract afn <2 x double> [[TMP0]], [[TMP2]]
+; ENABLED-NEXT: [[TMP6:%.*]] = fsub reassoc nsz arcp contract afn <2 x double> [[TMP3]], [[TMP1]]
; ENABLED-NEXT: store <2 x double> [[TMP6]], ptr [[SARRAY:%.*]], align 8
; ENABLED-NEXT: ret void
;
More information about the llvm-commits
mailing list