[llvm] b5cae89 - [SLP]Fix fmul/fadd fusion costs and retry FMA seeds after all blocks
via llvm-commits
llvm-commits at lists.llvm.org
Mon Sep 28 06:01:46 PDT 2026
Author: Alexey Bataev
Date: 2026-09-28T09:01:40-04:00
New Revision: b5cae89e8463dd397960f8a37bb877a06306c0f4
URL: https://github.com/llvm/llvm-project/commit/b5cae89e8463dd397960f8a37bb877a06306c0f4
DIFF: https://github.com/llvm/llvm-project/commit/b5cae89e8463dd397960f8a37bb877a06306c0f4.diff
LOG: [SLP]Fix fmul/fadd fusion costs and retry FMA seeds after all blocks
Fix the cost for the scalar of fmul+fadd pairs: fused fmul lanes add
the full fmul cost, fadd lanes with gathered fmuls were costed as
fmuladd, and the FMulAdd combine missed fmuls in the second operand,
c - a*b and constant lanes. Cost fused fmul lanes at (fmuladd - fadd),
such fadd lanes at (fmuladd - fmul), form the combine for either
operand, and treat a multi-use reduction root as fused. Extract nodes
take no credit for extracts the target folds into their users.
Reviewers: RKSimon, bababuck, MrSidims
Pull Request: https://github.com/llvm/llvm-project/pull/226117
Added:
Modified:
llvm/include/llvm/Transforms/Vectorize/SLPVectorizer.h
llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp
llvm/test/Transforms/PhaseOrdering/AArch64/reassociate-fma-pairs.ll
llvm/test/Transforms/SLPVectorizer/AArch64/extracts-folded-into-fmul-users-no-credit.ll
llvm/test/Transforms/SLPVectorizer/AArch64/fadd-with-gathered-fmul-operands.ll
llvm/test/Transforms/SLPVectorizer/AArch64/fma-candidates-after-store-chains.ll
llvm/test/Transforms/SLPVectorizer/AArch64/fma-chain-no-alt-node-reduction.ll
llvm/test/Transforms/SLPVectorizer/AArch64/fmul-constant-lane-fmuladd-combine.ll
llvm/test/Transforms/SLPVectorizer/AArch64/hor-fp-reduction-instcount.ll
llvm/test/Transforms/SLPVectorizer/AArch64/loop-accumulator-reduction.ll
llvm/test/Transforms/SLPVectorizer/AArch64/reduction-root-multi-use-fma.ll
llvm/test/Transforms/SLPVectorizer/AArch64/vec3-reorder-reshuffle.ll
llvm/test/Transforms/SLPVectorizer/AMDGPU/cross-block-fmul-fadd.ll
llvm/test/Transforms/SLPVectorizer/NVPTX/ordered-reduction-fma-fusion.ll
llvm/test/Transforms/SLPVectorizer/X86/fmul-fused-into-scalar-fadd.ll
llvm/test/Transforms/SLPVectorizer/X86/fsub-fmul-rhs-combine.ll
llvm/test/Transforms/SLPVectorizer/X86/slp-fma-loss-ordered.ll
llvm/test/Transforms/SLPVectorizer/consecutive-access.ll
Removed:
################################################################################
diff --git a/llvm/include/llvm/Transforms/Vectorize/SLPVectorizer.h b/llvm/include/llvm/Transforms/Vectorize/SLPVectorizer.h
index 468cec660343c..3d8ab0d1b7573 100644
--- a/llvm/include/llvm/Transforms/Vectorize/SLPVectorizer.h
+++ b/llvm/include/llvm/Transforms/Vectorize/SLPVectorizer.h
@@ -176,8 +176,10 @@ struct SLPVectorizerPass : public OptionalPassInfoMixin<SLPVectorizerPass> {
SmallSetVector<Instruction *, 8> &FMACandidates);
/// Scan the basic block and look for patterns that are likely to start
- /// a vectorization chain.
- bool vectorizeChainsInBlock(BasicBlock *BB, slpvectorizer::BoUpSLP &R);
+ /// a vectorization chain. The FMA candidates are collected in
+ /// \p FMACandidates for the retry after all the blocks of the function.
+ bool vectorizeChainsInBlock(BasicBlock *BB, slpvectorizer::BoUpSLP &R,
+ SmallSetVector<Instruction *, 8> &FMACandidates);
std::optional<bool> vectorizeStoreChain(ArrayRef<Value *> Chain,
slpvectorizer::BoUpSLP &R,
diff --git a/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp b/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp
index 442cc9a0895b2..c9e649c551b92 100644
--- a/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp
+++ b/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp
@@ -2508,6 +2508,23 @@ class slpvectorizer::BoUpSLP {
/// vector savings the count check does not model.
bool bypassesInstCountCheck(InstructionCost TreeCost) const;
+ /// \returns the fadd/fsub user of the single-use fmul \p I, which the
+ /// backend fuses with \p I into an fmuladd in the scalar code, and the cost
+ /// of that fmuladd, or {nullptr, invalid cost}.
+ std::pair<Instruction *, InstructionCost>
+ getFMulFusingUser(Instruction *I) const;
+
+ /// \returns true if the fadd/fsub \p U, fused with its fmul operand by the
+ /// backend, is a vectorized tree scalar whose lane is priced as fmuladd.
+ /// Peeled reassociated scalars are priced as plain fadd/fsub.
+ bool isFusedVectorizedFAddSub(Instruction *U) const;
+
+ /// \returns the scalar cost of the fmul lanes of \p TE that are priced as
+ /// fused into their vectorized fadd/fsub users. If the node is turned into a
+ /// gather, these fmuls stay scalar and lose the fusion, so the gather
+ /// alternative must pay their full price.
+ InstructionCost getUnfusedFMulsPenalty(const TreeEntry &TE) const;
+
/// Return information about the vector formed for the specified index
/// of a vector of (the same) instruction.
TargetTransformInfo::OperandValueInfo
@@ -13187,6 +13204,95 @@ static InstructionCost canConvertToFMA(ArrayRef<Value *> VL,
const TargetLibraryInfo &TLI,
const BoUpSLP &R);
+/// \returns true if \p Op is a contractable single-use fmul, which the backend
+/// fuses with its fadd/fsub user \p I: \p Op is in the block of \p I or the
+/// target sinks it there.
+static bool isFusableFMulOperand(Value *Op, Instruction *I,
+ const TargetTransformInfo &TTI) {
+ Instruction *FMul;
+ if (!match(Op, m_Instruction(FMul, m_OneUse(m_AllowContract(
+ m_FMul(m_Value(), m_Value()))))))
+ return false;
+ if (FMul->getParent() == I->getParent())
+ return true;
+ SmallVector<Use *> OpsToSink;
+ return TTI.isProfitableToSinkOperands(I, OpsToSink) &&
+ any_of(OpsToSink, [&](const Use *U) { return U->get() == FMul; });
+}
+
+/// \returns the fmul operand of the fadd/fsub \p I that the backend fuses with
+/// \p I into an fmuladd: the first fusable one from either operand, c - a*b
+/// included. \p I must allow contraction.
+static Instruction *getFusableFMulOperand(Instruction *I,
+ const TargetTransformInfo &TTI) {
+ if (!match(I, m_AllowContract(m_CombineOr(m_FAdd(m_Value(), m_Value()),
+ m_FSub(m_Value(), m_Value())))))
+ return nullptr;
+ for (Value *Op : I->operands())
+ if (isFusableFMulOperand(Op, I, TTI))
+ return cast<Instruction>(Op);
+ return nullptr;
+}
+
+/// \returns the cost of the binary operator \p I, priced as not fused. No
+/// context instruction is passed: targets that model the fmuladd fusion would
+/// discount the unfused side of the comparison as well.
+static InstructionCost getUnfusedBinOpCost(Instruction *I,
+ TargetTransformInfo &TTI,
+ const TargetLibraryInfo &TLI,
+ TTI::TargetCostKind CostKind) {
+ assert(match(I, m_BinOp()) && "Expected a binary operator.");
+ TTI::OperandValueInfo Op1Info = TTI::getOperandInfo(I->getOperand(0));
+ TTI::OperandValueInfo Op2Info = TTI::getOperandInfo(I->getOperand(1));
+ return TTI.getArithmeticInstrCost(
+ I->getOpcode(), I->getType(), CostKind, Op1Info, Op2Info,
+ {I->getOperand(0), I->getOperand(1)}, /*CxtI=*/nullptr, &TLI);
+}
+
+std::pair<Instruction *, InstructionCost>
+BoUpSLP::getFMulFusingUser(Instruction *I) const {
+ if (!match(I, m_OneUse(m_FMul(m_Value(), m_Value()))))
+ return {nullptr, InstructionCost::getInvalid()};
+ auto *U = cast<Instruction>(I->user_back());
+ // The reduction cost accounts for the fusion of the reduced values into the
+ // reduction operations itself. The lanes of the combined fmuladd node are
+ // fused with its own fmul operand node, not with the other operand.
+ if (getFusableFMulOperand(U, *TTI) != I ||
+ (UserIgnoreList && UserIgnoreList->contains(U)) ||
+ any_of(getTreeEntries(U), [](const TreeEntry *TE) {
+ return TE->CombinedOp == TreeEntry::FMulAdd;
+ }))
+ return {nullptr, InstructionCost::getInvalid()};
+ // The target must actually prefer the fused form.
+ InstructionCost FMACost =
+ canConvertToFMA(U, InstructionsState(U, U), *DT, *DL, *TTI, *TLI, *this);
+ return {FMACost.isValid() ? U : nullptr, FMACost};
+}
+
+bool BoUpSLP::isFusedVectorizedFAddSub(Instruction *U) const {
+ return any_of(getTreeEntries(U), [&](const TreeEntry *TE) {
+ return !DeletedNodes.contains(TE) && !TransformedToGatherNodes.contains(TE);
+ });
+}
+
+InstructionCost BoUpSLP::getUnfusedFMulsPenalty(const TreeEntry &TE) const {
+ if (!TE.hasState() || TE.isGather() ||
+ (TE.getOpcode() != Instruction::FMul &&
+ TE.getAltOpcode() != Instruction::FMul))
+ return TTI::TCC_Free;
+ InstructionCost Penalty = TTI::TCC_Free;
+ for (Value *V : TE.Scalars) {
+ Instruction *I;
+ if (!match(V, m_Instruction(I, m_FMul(m_Value(), m_Value()))) ||
+ (TE.hasCopyableElements() && TE.isCopyableElement(I)))
+ continue;
+ if (Instruction *U = getFMulFusingUser(I).first;
+ U && isFusedVectorizedFAddSub(U))
+ Penalty += getUnfusedBinOpCost(I, *TTI, *TLI, CostKind);
+ }
+ return Penalty;
+}
+
// A poor-throughput entry's real vector-vs-scalar savings (fdiv/frem/fsqrt)
// are already folded into TreeCost like any other entry, including all
// shuffle/insert/extract overhead elsewhere in the tree. So bypassing the
@@ -13276,19 +13382,30 @@ uint64_t BoUpSLP::getNumScalarInsts(bool HasTreeLoop) {
--Count;
}
}
- } else if (Opcode == Instruction::FAdd || Opcode == Instruction::FSub) {
+ } else {
for (Value *V : TE.Scalars) {
if (TE.hasCopyableElements() && TE.isCopyableElement(V))
continue;
auto *I = dyn_cast<Instruction>(V);
- if (!I || (TE.isAltShuffle() && I->getOpcode() != Instruction::FAdd &&
- I->getOpcode() != Instruction::FSub))
+ if (!I)
continue;
- if (canConvertToFMA(I, InstructionsState(I, I), *DT, *DL, *TTI, *TLI,
- *this)
- .isValid()) {
- assert(Count > 0 && "Underflow in scalar inst count (fma)");
- --Count;
+ if (match(I, m_CombineOr(m_FAdd(m_Value(), m_Value()),
+ m_FSub(m_Value(), m_Value())))) {
+ if (canConvertToFMA(I, InstructionsState(I, I), *DT, *DL, *TTI,
+ *TLI, *this)
+ .isValid()) {
+ assert(Count > 0 && "Underflow in scalar inst count (fma)");
+ --Count;
+ }
+ } else if (match(I, m_FMul(m_Value(), m_Value()))) {
+ // The fmul, fused into a scalar user, disappears into the user's
+ // fmuladd. A vectorized fadd/fsub user lane already accounts for
+ // the fusion.
+ Instruction *U = getFMulFusingUser(I).first;
+ if (U && !isFusedVectorizedFAddSub(U)) {
+ assert(Count > 0 && "Underflow in scalar inst count (fma)");
+ --Count;
+ }
}
}
}
@@ -13686,8 +13803,23 @@ static InstructionCost canConvertToFMA(ArrayRef<Value *> VL,
// fmul also should be contractable
InstructionsCompatibilityAnalysis Analysis(DT, DL, TTI, TLI);
SmallVector<BoUpSLP::ValueList> Operands = Analysis.buildOperands(S, VL, R);
+ // The operands follow the majority pattern of the lanes, while the backend
+ // fuses the fmul from either operand (c - a*b included); move it to the
+ // first column.
+ if (Operands.size() == 2) {
+ for (auto [Idx, V] : enumerate(VL)) {
+ auto *I = dyn_cast<Instruction>(V);
+ if (!I || S.isCopyableElement(I))
+ continue;
+ if (Instruction *FMul = getFusableFMulOperand(I, TTI);
+ FMul && Operands[1][Idx] == FMul)
+ std::swap(Operands[0][Idx], Operands[1][Idx]);
+ }
+ }
- InstructionsState OpS = getSameOpcode(Operands.front(), TLI);
+ // The lanes without the fmul, e.g. the constant lanes of the copyable nodes,
+ // are the copyable lanes of the fmul column.
+ InstructionsState OpS = Analysis.buildInstructionsState(Operands.front(), R);
if (!OpS.valid())
return InstructionCost::getInvalid();
@@ -13698,17 +13830,6 @@ static InstructionCost canConvertToFMA(ArrayRef<Value *> VL,
// Compare the costs.
InstructionCost FMulPlusFAddCost = 0;
InstructionCost FMACost = 0;
- // Price both sides of the fmul+fadd pair as not fused. Passing a context
- // instruction would let targets that model the fusion discount the unfused
- // side of the comparison as well.
- auto GetUnfusedFMulCost = [&](Instruction *I) {
- assert(I->getOpcode() == Instruction::FMul && "Expected an fmul");
- TTI::OperandValueInfo Op1Info = TTI::getOperandInfo(I->getOperand(0));
- TTI::OperandValueInfo Op2Info = TTI::getOperandInfo(I->getOperand(1));
- return TTI.getArithmeticInstrCost(I->getOpcode(), I->getType(), CostKind,
- Op1Info, Op2Info,
- {I->getOperand(0), I->getOperand(1)});
- };
FastMathFlags FMF;
FMF.set();
const bool IsArithmeticState = S.isAddSubLikeOp() || S.isMulDivLikeOp() ||
@@ -13721,11 +13842,7 @@ static InstructionCost canConvertToFMA(ArrayRef<Value *> VL,
(I->getOpcode() != S.getOpcode() &&
I->getOpcode() != S.getAltOpcode()))
return TTI.getInstructionCost(I, CostKind);
- TTI::OperandValueInfo Op1Info = TTI::getOperandInfo(I->getOperand(0));
- TTI::OperandValueInfo Op2Info = TTI::getOperandInfo(I->getOperand(1));
- return TTI.getArithmeticInstrCost(I->getOpcode(), I->getType(), CostKind,
- Op1Info, Op2Info,
- {I->getOperand(0), I->getOperand(1)});
+ return getUnfusedBinOpCost(I, TTI, TLI, CostKind);
}
return TTI.getArithmeticInstrCost(Instruction::FAdd, I->getType(), CostKind,
{},
@@ -13748,7 +13865,8 @@ static InstructionCost canConvertToFMA(ArrayRef<Value *> VL,
if (S.isCopyableElement(V))
continue;
auto *I = dyn_cast<Instruction>(Op);
- if (!I || !I->hasOneUse() || OpS.isCopyableElement(I)) {
+ if (!I || OpS.isCopyableElement(I) ||
+ !isFusableFMulOperand(I, cast<Instruction>(V), TTI)) {
if (auto *OpI = dyn_cast<Instruction>(V))
FMACost += GetLinkCost(OpI);
if (I)
@@ -13758,7 +13876,7 @@ static InstructionCost canConvertToFMA(ArrayRef<Value *> VL,
++NumOps;
if (auto *FPCI = dyn_cast<FPMathOperator>(I))
FMF &= FPCI->getFastMathFlags();
- FMulPlusFAddCost += GetUnfusedFMulCost(I);
+ FMulPlusFAddCost += getUnfusedBinOpCost(I, TTI, TLI, CostKind);
}
Type *Ty = VL.front()->getType();
IntrinsicCostAttributes ICA(Intrinsic::fmuladd, Ty, {Ty, Ty, Ty}, FMF);
@@ -14710,9 +14828,9 @@ void BoUpSLP::transformNodes() {
V->hasOneUse();
});
};
- if (!IsOneUseVectorFMulOperand(LHS) &&
- (E.getOpcode() == Instruction::FSub ||
- !IsOneUseVectorFMulOperand(RHS)))
+ // Both a*b - c and c - a*b are fused.
+ const bool IsLHSFMul = IsOneUseVectorFMulOperand(LHS);
+ if (!IsLHSFMul && !IsOneUseVectorFMulOperand(RHS))
break;
if (!canConvertToFMA(E.Scalars, E.getOperations(), *DT, *DL, *TTI, *TLI,
*this)
@@ -14720,7 +14838,7 @@ void BoUpSLP::transformNodes() {
break;
// This node is a fmuladd node.
E.CombinedOp = TreeEntry::FMulAdd;
- TreeEntry *FMulEntry = getOperandEntry(&E, 0);
+ TreeEntry *FMulEntry = getOperandEntry(&E, IsLHSFMul ? 0 : 1);
if (FMulEntry->UserTreeIndex &&
FMulEntry->State == TreeEntry::Vectorize) {
// The FMul node is part of the combined fmuladd node.
@@ -16354,9 +16472,12 @@ BoUpSLP::getEntryCost(const TreeEntry *E, ArrayRef<Value *> VectorizedVals,
return 0;
if (isa<InsertElementInst, InsertValueInst>(VL[0]))
return InstructionCost::getInvalid();
+ // The fmuls of the node turned into a gather stay scalar and lose the
+ // fusion with their vectorized fadd/fsub users.
return SpillsReloads +
processBuildVector<ShuffleCostEstimator, InstructionCost>(
- E, ScalarTy, *TTI, VectorizedVals, *this, CheckedExtracts);
+ E, ScalarTy, *TTI, VectorizedVals, *this, CheckedExtracts) +
+ getUnfusedFMulsPenalty(*E);
}
if (E->State == TreeEntry::SplitVectorize) {
assert(E->CombinedEntriesWithIndices.size() == 2 &&
@@ -16592,10 +16713,30 @@ BoUpSLP::getEntryCost(const TreeEntry *E, ArrayRef<Value *> VectorizedVals,
}
return IntrinsicCost;
};
+ // The fadd/fsub, fused with its fmul operand into an fmuladd in the scalar
+ // code. If the fmul is not vectorized, the vector code still emits it, so
+ // only the fusion delta is attributed to the fadd/fsub.
auto GetFMulAddCost = [&, &TTI = *TTI](const InstructionsState &S,
Instruction *VI) {
InstructionCost Cost = canConvertToFMA(VI, S, *DT, *DL, TTI, *TLI, *this);
- return Cost;
+ Instruction *FMul = getFusableFMulOperand(VI, TTI);
+ if (!Cost.isValid() || !FMul || isVectorized(FMul))
+ return Cost;
+ InstructionCost FMulCost = getUnfusedBinOpCost(FMul, TTI, *TLI, CostKind);
+ return Cost > FMulCost ? Cost - FMulCost : InstructionCost(TTI::TCC_Free);
+ };
+ // The fmul, fused into its fadd/fsub user, is free if the user lane is
+ // priced as fmuladd, otherwise the user stays scalar and only the fusion
+ // delta is attributed to the fmul.
+ auto GetFusedFMulCost = [&, &TTI = *TTI](Instruction *VI) {
+ auto [U, FMACost] = getFMulFusingUser(VI);
+ if (!U)
+ return InstructionCost::getInvalid();
+ if (isFusedVectorizedFAddSub(U))
+ return InstructionCost(TTI::TCC_Free);
+ InstructionCost FAddCost = getUnfusedBinOpCost(U, TTI, *TLI, CostKind);
+ return FMACost > FAddCost ? FMACost - FAddCost
+ : InstructionCost(TTI::TCC_Free);
};
switch (ShuffleOrOp) {
case Instruction::PHI: {
@@ -16682,6 +16823,13 @@ BoUpSLP::getEntryCost(const TreeEntry *E, ArrayRef<Value *> VectorizedVals,
return Cost;
}
}
+ // The extract, which the target folds into its users (e.g. the lane
+ // operand of a by-element fmul on AArch64), costs nothing in the scalar
+ // code; its removal saves nothing.
+ if (ShuffleOrOp == Instruction::ExtractElement &&
+ TTI->getVectorInstrCost(*I, SrcVecTy, CostKind, *ExtIdx,
+ TTI::getVectorInstrContextHint(I)) == 0)
+ return InstructionCost(TTI::TCC_Free);
if (DemandedElts.isZero())
DemandedElts = APInt::getZero(getNumElements(SrcVecTy));
DemandedElts.setBit(*ExtIdx);
@@ -17069,8 +17217,18 @@ BoUpSLP::getEntryCost(const TreeEntry *E, ArrayRef<Value *> VectorizedVals,
auto GetScalarCost = [&](unsigned Idx) {
if (isa<PoisonValue>(UniqueValues[Idx]))
return InstructionCost(TTI::TCC_Free);
- return GetFMulAddCost(E->getOperations(),
- cast<Instruction>(UniqueValues[Idx]));
+ auto *I = cast<Instruction>(UniqueValues[Idx]);
+ InstructionCost Cost = GetFMulAddCost(E->getOperations(), I);
+ if (Cost.isValid())
+ return Cost;
+ // The lanes without the fmul, e.g. the constant lanes of the fmul node,
+ // stay plain fadd/fsub.
+ unsigned Lane = UniqueIndexes[Idx];
+ Value *Op1 = E->getOperand(0)[Lane];
+ Value *Op2 = E->getOperand(1)[Lane];
+ return TTI->getArithmeticInstrCost(
+ E->getOpcode(), OrigScalarTy, CostKind, TTI::getOperandInfo(Op1),
+ TTI::getOperandInfo(Op2), {Op1, Op2}, I, TLI);
};
auto GetVectorCost = [&, &TTI = *TTI](InstructionCost CommonCost) {
FastMathFlags FMF;
@@ -17078,8 +17236,10 @@ BoUpSLP::getEntryCost(const TreeEntry *E, ArrayRef<Value *> VectorizedVals,
for (Value *V : E->Scalars) {
if (auto *FPCI = dyn_cast<FPMathOperator>(V)) {
FMF &= FPCI->getFastMathFlags();
- if (auto *FPCIOp = dyn_cast<FPMathOperator>(FPCI->getOperand(0)))
- FMF &= FPCIOp->getFastMathFlags();
+ // The fused fmul may sit in either operand (c - a*b included).
+ if (Instruction *FMul =
+ getFusableFMulOperand(cast<Instruction>(V), TTI))
+ FMF &= FMul->getFastMathFlags();
}
}
IntrinsicCostAttributes ICA(Intrinsic::fmuladd, VecTy,
@@ -17290,6 +17450,10 @@ BoUpSLP::getEntryCost(const TreeEntry *E, ArrayRef<Value *> VectorizedVals,
InstructionCost IntrinsicCost = GetFMulAddCost(E->getOperations(), I);
if (IntrinsicCost.isValid())
ScalarCost = IntrinsicCost;
+ } else if (I && ShuffleOrOp == Instruction::FMul) {
+ InstructionCost FusedCost = GetFusedFMulCost(I);
+ if (FusedCost.isValid())
+ ScalarCost = FusedCost;
}
return ScalarCost;
};
@@ -17604,7 +17768,43 @@ BoUpSLP::getEntryCost(const TreeEntry *E, ArrayRef<Value *> VectorizedVals,
assert(E->getMatchingMainOpOrAltOp(VI) &&
"Unexpected main/alternate opcode");
(void)E;
- return TTI->getInstructionCost(VI, CostKind);
+ if (auto *CI = dyn_cast<CmpInst>(VI))
+ return TTI->getCmpSelInstrCost(
+ CI->getOpcode(), CI->getOperand(0)->getType(), CI->getType(),
+ CI->getPredicate(), CostKind,
+ TTI::getOperandInfo(CI->getOperand(0)),
+ TTI::getOperandInfo(CI->getOperand(1)), CI);
+ if (auto *CI = dyn_cast<CastInst>(VI))
+ return TTI->getCastInstrCost(CI->getOpcode(), CI->getDestTy(),
+ CI->getSrcTy(),
+ TTI::getCastContextHint(CI), CostKind, CI);
+ if (auto *SV = dyn_cast<ShuffleVectorInst>(VI)) {
+ int Index;
+ [[maybe_unused]] bool IsExtractSubvectorMask =
+ SV->isExtractSubvectorMask(Index);
+ assert(IsExtractSubvectorMask && "Not supported shufflevector usage.");
+ auto *SubTy = cast<VectorType>(SV->getType());
+ return TTI->getShuffleCost(
+ TTI::SK_ExtractSubvector, SubTy,
+ cast<VectorType>(SV->getOperand(0)->getType()), CostKind,
+ SV->getShuffleMask(), Index, SubTy,
+ {SV->getOperand(0), SV->getOperand(1)}, SV,
+ TTI::getVectorInstrContextHint(SV));
+ }
+ // The backend fuses fmul+fadd/fsub pairs in the scalar code.
+ InstructionCost FusedCost = InstructionCost::getInvalid();
+ if (match(VI, m_CombineOr(m_FAdd(m_Value(), m_Value()),
+ m_FSub(m_Value(), m_Value()))))
+ FusedCost = GetFMulAddCost(InstructionsState(VI, VI), VI);
+ else if (match(VI, m_FMul(m_Value(), m_Value())))
+ FusedCost = GetFusedFMulCost(VI);
+ if (FusedCost.isValid())
+ return FusedCost;
+ return TTI->getArithmeticInstrCost(
+ VI->getOpcode(), VI->getType(), CostKind,
+ TTI::getOperandInfo(VI->getOperand(0)),
+ TTI::getOperandInfo(VI->getOperand(1)),
+ {VI->getOperand(0), VI->getOperand(1)}, VI, TLI);
};
// Need to clear CommonCost since the final shuffle cost is included into
// vector cost.
@@ -19626,6 +19826,9 @@ BoUpSLP::calculateTreeCostAndTrimNonProfitable(ArrayRef<Value *> VectorizedVals,
isConstant(V) || isGathered(V) || getTreeEntries(V).size() > 1;
}))
GatherCost *= 2;
+ // The fmuls, priced as fused into their vectorized fadd/fsub users, stay
+ // scalar and unfused in the gathered form.
+ GatherCost += getUnfusedFMulsPenalty(*TE);
// Erase subtree if it is non-profitable.
ArrayRef<unsigned> Nodes = std::get<2>(Worklist.top().second);
// Prefer trimming equal-cost alternate-shuffle subtrees rooted at binary
@@ -29062,6 +29265,7 @@ bool SLPVectorizerPass::runImpl(Function &F, ScalarEvolution *SE_,
// Update DFS numbers now so that we can use them for ordering.
DT->updateDFSNumbers();
+ SmallSetVector<Instruction *, 8> FMACandidates;
// Scan the blocks in the function in post order.
for (auto *BB : post_order(&F.getEntryBlock())) {
if (BB->isEHPad() || isa_and_nonnull<UnreachableInst>(BB->getTerminator()))
@@ -29084,7 +29288,7 @@ bool SLPVectorizerPass::runImpl(Function &F, ScalarEvolution *SE_,
}
// Vectorize trees that end at reductions.
- Changed |= vectorizeChainsInBlock(BB, R);
+ Changed |= vectorizeChainsInBlock(BB, R, FMACandidates);
// Vectorize the index computations of getelementptr instructions. This
// is primarily intended to catch gather-like idioms ending at
@@ -29096,6 +29300,20 @@ bool SLPVectorizerPass::runImpl(Function &F, ScalarEvolution *SE_,
}
}
+ // Vectorized on their own, the FMA candidates lose the fusion with their fmul
+ // operands. They are retried after all the blocks, not to preempt the trees
+ // of the blocks processed later, e.g. the store chains of a loop body,
+ // processed after the latch PHIs reaching the same instructions.
+ R.clearReductionData();
+ SmallSetVector<Instruction *, 8> Empty;
+ for (Instruction *I : FMACandidates) {
+ if (R.isDeleted(I))
+ continue;
+ Changed |= tryToVectorize(I, R, Empty, /*AllowFMACandidates=*/true);
+ }
+ assert(Empty.empty() &&
+ "No new FMA candidates expected during AllowFMACandidates retry.");
+
// Instructions with the single user require just one extract per lane, so
// they are used as the seeds for the very last attempt, after all the other
// roots in the function are exhausted.
@@ -33697,11 +33915,17 @@ class HorizontalReduction {
InstructionCost ScalarCost = 0;
for (User *U : RdxVal->users()) {
auto *RdxOp = cast<Instruction>(U);
- if (hasRequiredNumberOfUses(IsCmpSelMinMax, RdxOp)) {
+ // The reduction root is used outside of the reduction any
+ // number of times, its fmul operand is still fused into it.
+ if ((RdxKind == RecurKind::FAdd && RdxOp == ReductionRoot) ||
+ hasRequiredNumberOfUses(IsCmpSelMinMax, RdxOp)) {
InstructionsState RdxOpS = RdxKind == RecurKind::FAdd
? getSameOpcode(RdxOp, TLI)
: InstructionsState::invalid();
- if (RdxOpS && RdxOpS.isAddSubOrFNegLikeOp()) {
+ // The reduced value itself must be a fusable fmul, whose cost
+ // is excluded.
+ if (RdxOpS && RdxOpS.isAddSubOrFNegLikeOp() &&
+ isFusableFMulOperand(RdxVal, RdxOp, *TTI)) {
InstructionCost FMACost =
canConvertToFMA(RdxOp, RdxOpS, DT, DL, *TTI, TLI, R);
if (FMACost.isValid()) {
@@ -33860,7 +34084,11 @@ class HorizontalReduction {
FastMathFlags FMF;
FMF.set();
for (Value *RdxVal : ReducedVals) {
- if (!RdxVal->hasOneUse()) {
+ // The vector fmul fuses only if all the reduced values are
+ // fusable fmuls.
+ if (!RdxVal->hasOneUse() ||
+ !isFusableFMulOperand(
+ RdxVal, cast<Instruction>(RdxVal->user_back()), *TTI)) {
Ops.clear();
break;
}
@@ -34759,8 +34987,8 @@ bool SLPVectorizerPass::tryToVectorize(
if (!isa<BinaryOperator, CmpInst>(I) || isa<VectorType>(I->getType()))
return false;
- // Skip potential FMA candidates and collect them for a retry after all other
- // instructions in the block have been processed.
+ // Skip potential FMA candidates and collect them for a retry after all
+ // blocks of the function have been processed.
if (!AllowFMACandidates &&
(I->getOpcode() == Instruction::FAdd ||
I->getOpcode() == Instruction::FSub) &&
@@ -35535,7 +35763,9 @@ bool SLPVectorizerPass::vectorizeInserts(
return OpsChanged;
}
-bool SLPVectorizerPass::vectorizeChainsInBlock(BasicBlock *BB, BoUpSLP &R) {
+bool SLPVectorizerPass::vectorizeChainsInBlock(
+ BasicBlock *BB, BoUpSLP &R,
+ SmallSetVector<Instruction *, 8> &FMACandidates) {
bool Changed = false;
SmallVector<Value *, 4> Incoming;
SmallPtrSet<Value *, 16> VisitedInstrs;
@@ -35774,7 +36004,6 @@ bool SLPVectorizerPass::vectorizeChainsInBlock(BasicBlock *BB, BoUpSLP &R) {
SmallSetVector<Instruction *, 8> PostProcessInsts;
// Stores are processed after all other instructions/roots.
SmallSetVector<StoreInst *, 8> PostProcessStores;
- SmallSetVector<Instruction *, 8> FMACandidates;
SmallSetVector<Instruction *, 8> PoorThroughputSeeds;
PoorThroughputOpCache PoorThroughputCache;
auto VectorizeInsertsAndCmps = [&](bool AtTerminator) {
@@ -35978,14 +36207,6 @@ bool SLPVectorizerPass::vectorizeChainsInBlock(BasicBlock *BB, BoUpSLP &R) {
R, FMACandidates);
}
}
- SmallSetVector<Instruction *, 8> Empty;
- for (Instruction *I : FMACandidates) {
- if (R.isDeleted(I))
- continue;
- Changed |= tryToVectorize(I, R, Empty, /*AllowFMACandidates=*/true);
- }
- assert(Empty.empty() &&
- "No new FMA candidates expected during AllowFMACandidates retry.");
if (PoorThroughputSeeds.size() >= 2) {
SmallVector<Value *> Seeds;
diff --git a/llvm/test/Transforms/PhaseOrdering/AArch64/reassociate-fma-pairs.ll b/llvm/test/Transforms/PhaseOrdering/AArch64/reassociate-fma-pairs.ll
index 9631c6444cb2c..c4eab73309c54 100644
--- a/llvm/test/Transforms/PhaseOrdering/AArch64/reassociate-fma-pairs.ll
+++ b/llvm/test/Transforms/PhaseOrdering/AArch64/reassociate-fma-pairs.ll
@@ -18,49 +18,49 @@ define double @md_vdw_energy(ptr nocapture readonly %coeffs, double %energy, dou
; CHECK-NEXT: [[FACTOR_OP_FMUL2:%.*]] = fmul fast double [[TABLE_DELTA]], 5.000000e-01
; CHECK-NEXT: [[FACTOR_OP_FMUL3:%.*]] = fmul fast double [[FACTOR_OP_FMUL]], [[TABLE_DELTA]]
; CHECK-NEXT: [[FACTOR_OP_FMUL4:%.*]] = fmul fast double [[TMP0]], 2.500000e-01
-; CHECK-NEXT: [[TMP1:%.*]] = insertelement <2 x double> poison, double [[SCALE]], i64 0
-; CHECK-NEXT: [[TMP11:%.*]] = shufflevector <2 x double> [[TMP1]], <2 x double> poison, <2 x i32> zeroinitializer
; CHECK-NEXT: br label %[[LOOP:.*]]
; CHECK: [[LOOP]]:
; CHECK-NEXT: [[I:%.*]] = phi i64 [ 0, %[[ENTRY]] ], [ [[NEXT:%.*]], %[[LOOP]] ]
; CHECK-NEXT: [[ACC2:%.*]] = phi double [ [[ENERGY]], %[[ENTRY]] ], [ [[RESULT1:%.*]], %[[LOOP]] ]
; CHECK-NEXT: [[BASE:%.*]] = getelementptr inbounds [8 x i8], ptr [[COEFFS]], i64 [[I]]
+; CHECK-NEXT: [[A:%.*]] = load double, ptr [[BASE]], align 8
+; CHECK-NEXT: [[B_PTR:%.*]] = getelementptr inbounds nuw i8, ptr [[BASE]], i64 8
+; CHECK-NEXT: [[B:%.*]] = load double, ptr [[B_PTR]], align 8
+; CHECK-NEXT: [[TMP5:%.*]] = fmul fast double [[A]], [[SCALE]]
+; CHECK-NEXT: [[TMP6:%.*]] = fmul fast double [[B]], [[SCALE]]
; CHECK-NEXT: [[C0_PTR:%.*]] = getelementptr inbounds nuw i8, ptr [[BASE]], i64 16
; CHECK-NEXT: [[C0:%.*]] = load double, ptr [[C0_PTR]], align 8
; CHECK-NEXT: [[C1_PTR:%.*]] = getelementptr inbounds nuw i8, ptr [[BASE]], i64 24
; CHECK-NEXT: [[C1:%.*]] = load double, ptr [[C1_PTR]], align 8
+; CHECK-NEXT: [[P0:%.*]] = fmul fast double [[C0]], [[TMP5]]
+; CHECK-NEXT: [[Q0:%.*]] = fmul fast double [[C1]], [[TMP6]]
+; CHECK-NEXT: [[D3_NEG:%.*]] = fsub fast double [[P0]], [[Q0]]
; CHECK-NEXT: [[C2_PTR:%.*]] = getelementptr inbounds nuw i8, ptr [[BASE]], i64 32
; CHECK-NEXT: [[C2:%.*]] = load double, ptr [[C2_PTR]], align 8
; CHECK-NEXT: [[C3_PTR:%.*]] = getelementptr inbounds nuw i8, ptr [[BASE]], i64 40
; CHECK-NEXT: [[C3:%.*]] = load double, ptr [[C3_PTR]], align 8
+; CHECK-NEXT: [[P1:%.*]] = fmul fast double [[C2]], [[TMP5]]
+; CHECK-NEXT: [[Q1:%.*]] = fmul fast double [[C3]], [[TMP6]]
+; CHECK-NEXT: [[D1:%.*]] = fsub fast double [[P1]], [[Q1]]
; CHECK-NEXT: [[C4_PTR:%.*]] = getelementptr inbounds nuw i8, ptr [[BASE]], i64 48
; CHECK-NEXT: [[C4:%.*]] = load double, ptr [[C4_PTR]], align 8
; CHECK-NEXT: [[C5_PTR:%.*]] = getelementptr inbounds nuw i8, ptr [[BASE]], i64 56
; CHECK-NEXT: [[C5:%.*]] = load double, ptr [[C5_PTR]], align 8
-; CHECK-NEXT: [[C6_PTR:%.*]] = getelementptr inbounds nuw i8, ptr [[BASE]], i64 64
-; CHECK-NEXT: [[TMP3:%.*]] = load <2 x double>, ptr [[BASE]], align 8
-; CHECK-NEXT: [[TMP4:%.*]] = fmul fast <2 x double> [[TMP3]], [[TMP11]]
-; CHECK-NEXT: [[TMP5:%.*]] = extractelement <2 x double> [[TMP4]], i64 0
-; CHECK-NEXT: [[ACC:%.*]] = fmul fast double [[C0]], [[TMP5]]
-; CHECK-NEXT: [[TMP6:%.*]] = extractelement <2 x double> [[TMP4]], i64 1
-; CHECK-NEXT: [[TMP2:%.*]] = fmul fast double [[C1]], [[TMP6]]
-; CHECK-NEXT: [[D3_NEG:%.*]] = fsub fast double [[ACC]], [[TMP2]]
-; CHECK-NEXT: [[P1:%.*]] = fmul fast double [[C2]], [[TMP5]]
-; CHECK-NEXT: [[Q1:%.*]] = fmul fast double [[C3]], [[TMP6]]
-; CHECK-NEXT: [[D1:%.*]] = fsub fast double [[P1]], [[Q1]]
; CHECK-NEXT: [[P2:%.*]] = fmul fast double [[C4]], [[TMP5]]
; CHECK-NEXT: [[Q2:%.*]] = fmul fast double [[C5]], [[TMP6]]
; CHECK-NEXT: [[D2:%.*]] = fsub fast double [[P2]], [[Q2]]
-; CHECK-NEXT: [[TMP7:%.*]] = load <2 x double>, ptr [[C6_PTR]], align 8
+; CHECK-NEXT: [[C6_PTR:%.*]] = getelementptr inbounds nuw i8, ptr [[BASE]], i64 64
+; CHECK-NEXT: [[C6:%.*]] = load double, ptr [[C6_PTR]], align 8
+; CHECK-NEXT: [[C7_PTR:%.*]] = getelementptr inbounds nuw i8, ptr [[BASE]], i64 72
+; CHECK-NEXT: [[C7:%.*]] = load double, ptr [[C7_PTR]], align 8
; CHECK-NEXT: [[TMP9:%.*]] = fmul fast double [[FACTOR_OP_FMUL3]], [[D3_NEG]]
; CHECK-NEXT: [[RESULT:%.*]] = fmul fast double [[D1]], [[FACTOR_OP_FMUL4]]
; CHECK-NEXT: [[REASS_ADD:%.*]] = fadd fast double [[RESULT]], [[TMP9]]
; CHECK-NEXT: [[T4_REASS:%.*]] = fmul fast double [[D2]], [[FACTOR_OP_FMUL2]]
; CHECK-NEXT: [[ACC1:%.*]] = fadd fast double [[REASS_ADD]], [[T4_REASS]]
-; CHECK-NEXT: [[TMP8:%.*]] = fmul fast <2 x double> [[TMP7]], [[TMP4]]
-; CHECK-NEXT: [[TMP10:%.*]] = extractelement <2 x double> [[TMP8]], i64 0
+; CHECK-NEXT: [[TMP10:%.*]] = fmul fast double [[C6]], [[TMP5]]
+; CHECK-NEXT: [[TMP12:%.*]] = fmul fast double [[C7]], [[TMP6]]
; CHECK-NEXT: [[S2_NEG1:%.*]] = fadd fast double [[ACC1]], [[TMP10]]
-; CHECK-NEXT: [[TMP12:%.*]] = extractelement <2 x double> [[TMP8]], i64 1
; CHECK-NEXT: [[S2_NEG:%.*]] = fadd fast double [[ACC2]], [[TMP12]]
; CHECK-NEXT: [[RESULT1]] = fsub fast double [[S2_NEG]], [[S2_NEG1]]
; CHECK-NEXT: [[NEXT]] = add nuw i64 [[I]], 10
diff --git a/llvm/test/Transforms/SLPVectorizer/AArch64/extracts-folded-into-fmul-users-no-credit.ll b/llvm/test/Transforms/SLPVectorizer/AArch64/extracts-folded-into-fmul-users-no-credit.ll
index f93a7dc3abf83..21c709e57765f 100644
--- a/llvm/test/Transforms/SLPVectorizer/AArch64/extracts-folded-into-fmul-users-no-credit.ll
+++ b/llvm/test/Transforms/SLPVectorizer/AArch64/extracts-folded-into-fmul-users-no-credit.ll
@@ -14,8 +14,6 @@ define void @fmul_reduction_extracts_free_for_fmul_users(ptr %vp, ptr %sp, ptr %
; CHECK-NEXT: [[B:%.*]] = load double, ptr [[BP]], align 8
; CHECK-NEXT: [[C:%.*]] = load double, ptr [[CP]], align 8
; CHECK-NEXT: [[D:%.*]] = load double, ptr [[DP]], align 8
-; CHECK-NEXT: [[TMP0:%.*]] = insertelement <2 x double> poison, double [[A]], i64 0
-; CHECK-NEXT: [[TMP1:%.*]] = insertelement <2 x double> [[TMP0]], double [[B]], i64 1
; CHECK-NEXT: br label %[[LOOP:.*]]
; CHECK: [[LOOP]]:
; CHECK-NEXT: [[I:%.*]] = phi i64 [ 0, %[[ENTRY]] ], [ [[I_NEXT:%.*]], %[[LOOP]] ]
@@ -25,10 +23,11 @@ define void @fmul_reduction_extracts_free_for_fmul_users(ptr %vp, ptr %sp, ptr %
; CHECK-NEXT: [[S:%.*]] = load double, ptr [[SPTR]], align 8
; CHECK-NEXT: [[E0:%.*]] = extractelement <2 x double> [[V]], i64 0
; CHECK-NEXT: [[E1:%.*]] = extractelement <2 x double> [[V]], i64 1
-; CHECK-NEXT: [[TMP2:%.*]] = fmul fast <2 x double> [[V]], [[TMP1]]
-; CHECK-NEXT: [[M2:%.*]] = fmul fast double [[S]], [[C]]
-; CHECK-NEXT: [[R0:%.*]] = call fast double @llvm.vector.reduce.fadd.v2f64(double 0.000000e+00, <2 x double> [[TMP2]])
+; CHECK-NEXT: [[R0:%.*]] = fmul fast double [[E0]], [[A]]
+; CHECK-NEXT: [[M2:%.*]] = fmul fast double [[E1]], [[B]]
+; CHECK-NEXT: [[M3:%.*]] = fmul fast double [[S]], [[C]]
; CHECK-NEXT: [[R:%.*]] = fadd fast double [[R0]], [[M2]]
+; CHECK-NEXT: [[R1:%.*]] = fadd fast double [[R]], [[M3]]
; CHECK-NEXT: [[U0:%.*]] = fmul fast double [[E0]], [[S]]
; CHECK-NEXT: [[U1:%.*]] = fmul fast double [[E1]], [[D]]
; CHECK-NEXT: [[U2:%.*]] = fmul fast double [[E1]], [[S]]
@@ -42,7 +41,7 @@ define void @fmul_reduction_extracts_free_for_fmul_users(ptr %vp, ptr %sp, ptr %
; CHECK-NEXT: store double [[U2]], ptr [[O2]], align 8
; CHECK-NEXT: [[I_NEXT]] = add nuw i64 [[I]], 1
; CHECK-NEXT: [[CMP:%.*]] = icmp ult i64 [[I_NEXT]], [[N]]
-; CHECK-NEXT: [[LT:%.*]] = fcmp fast olt double [[R]], [[D]]
+; CHECK-NEXT: [[LT:%.*]] = fcmp fast olt double [[R1]], [[D]]
; CHECK-NEXT: [[CONT:%.*]] = and i1 [[CMP]], [[LT]]
; CHECK-NEXT: br i1 [[CONT]], label %[[LOOP]], label %[[EXIT:.*]]
; CHECK: [[EXIT]]:
diff --git a/llvm/test/Transforms/SLPVectorizer/AArch64/fadd-with-gathered-fmul-operands.ll b/llvm/test/Transforms/SLPVectorizer/AArch64/fadd-with-gathered-fmul-operands.ll
index 4385123ca8716..b19204af12b8b 100644
--- a/llvm/test/Transforms/SLPVectorizer/AArch64/fadd-with-gathered-fmul-operands.ll
+++ b/llvm/test/Transforms/SLPVectorizer/AArch64/fadd-with-gathered-fmul-operands.ll
@@ -22,13 +22,21 @@ define void @fadd_with_gathered_fmul_operands(double %a0, double %b0, double %a1
; CHECK-NEXT: br label %[[MUL3:.*]]
; CHECK: [[MUL3]]:
; CHECK-NEXT: [[M3:%.*]] = fmul contract double [[A3]], [[B3]]
-; CHECK-NEXT: [[TMP0:%.*]] = load <4 x double>, ptr [[X]], align 8
-; CHECK-NEXT: [[TMP1:%.*]] = insertelement <4 x double> poison, double [[M0]], i64 0
-; CHECK-NEXT: [[TMP2:%.*]] = insertelement <4 x double> [[TMP1]], double [[M1]], i64 1
-; CHECK-NEXT: [[TMP3:%.*]] = insertelement <4 x double> [[TMP2]], double [[M2]], i64 2
-; CHECK-NEXT: [[TMP4:%.*]] = insertelement <4 x double> [[TMP3]], double [[M3]], i64 3
-; CHECK-NEXT: [[TMP5:%.*]] = fadd contract <4 x double> [[TMP4]], [[TMP0]]
-; CHECK-NEXT: store <4 x double> [[TMP5]], ptr [[OUT]], align 8
+; CHECK-NEXT: [[X2P:%.*]] = getelementptr inbounds double, ptr [[X]], i64 2
+; CHECK-NEXT: [[X2:%.*]] = load double, ptr [[X2P]], align 8
+; CHECK-NEXT: [[X3P:%.*]] = getelementptr inbounds double, ptr [[X]], i64 3
+; CHECK-NEXT: [[X3:%.*]] = load double, ptr [[X3P]], align 8
+; CHECK-NEXT: [[S2:%.*]] = fadd contract double [[M2]], [[X2]]
+; CHECK-NEXT: [[S3:%.*]] = fadd contract double [[M3]], [[X3]]
+; CHECK-NEXT: [[TMP0:%.*]] = load <2 x double>, ptr [[X]], align 8
+; CHECK-NEXT: [[TMP1:%.*]] = insertelement <2 x double> poison, double [[M0]], i64 0
+; CHECK-NEXT: [[TMP2:%.*]] = insertelement <2 x double> [[TMP1]], double [[M1]], i64 1
+; CHECK-NEXT: [[TMP3:%.*]] = fadd contract <2 x double> [[TMP2]], [[TMP0]]
+; CHECK-NEXT: store <2 x double> [[TMP3]], ptr [[OUT]], align 8
+; CHECK-NEXT: [[O2:%.*]] = getelementptr inbounds double, ptr [[OUT]], i64 2
+; CHECK-NEXT: store double [[S2]], ptr [[O2]], align 8
+; CHECK-NEXT: [[O3:%.*]] = getelementptr inbounds double, ptr [[OUT]], i64 3
+; CHECK-NEXT: store double [[S3]], ptr [[O3]], align 8
; CHECK-NEXT: ret void
;
entry:
diff --git a/llvm/test/Transforms/SLPVectorizer/AArch64/fma-candidates-after-store-chains.ll b/llvm/test/Transforms/SLPVectorizer/AArch64/fma-candidates-after-store-chains.ll
index 9e63b6eb3dbce..ada85a41fb777 100644
--- a/llvm/test/Transforms/SLPVectorizer/AArch64/fma-candidates-after-store-chains.ll
+++ b/llvm/test/Transforms/SLPVectorizer/AArch64/fma-candidates-after-store-chains.ll
@@ -26,21 +26,21 @@ define double @test(ptr %a, ptr %h, ptr %out, ptr %q, i64 %n) {
; CHECK-NEXT: br i1 [[CMP]], label %[[BODY:.*]], label %[[LATCH]]
; CHECK: [[BODY]]:
; CHECK-NEXT: [[PA0:%.*]] = getelementptr double, ptr [[A]], i64 [[I]]
-; CHECK-NEXT: [[PA2:%.*]] = getelementptr double, ptr [[PA0]], i64 2
-; CHECK-NEXT: [[A0:%.*]] = load double, ptr [[PA2]], align 8
+; CHECK-NEXT: [[PA1:%.*]] = getelementptr double, ptr [[PA0]], i64 1
+; CHECK-NEXT: [[A0:%.*]] = load double, ptr [[PA0]], align 8
; CHECK-NEXT: [[X:%.*]] = fsub fast double [[A0]], 1.000000e+00
-; CHECK-NEXT: [[TMP0:%.*]] = load <2 x double>, ptr [[PA0]], align 8
+; CHECK-NEXT: [[TMP0:%.*]] = load <2 x double>, ptr [[PA1]], align 8
; CHECK-NEXT: [[TMP1:%.*]] = fsub fast <2 x double> [[TMP0]], splat (double 1.000000e+00)
; CHECK-NEXT: [[TMP2:%.*]] = load <2 x double>, ptr [[H]], align 8
-; CHECK-NEXT: [[TMP5:%.*]] = fmul fast <2 x double> [[TMP1]], [[TMP2]]
+; CHECK-NEXT: [[TMP3:%.*]] = shufflevector <2 x double> [[TMP1]], <2 x double> poison, <2 x i32> <i32 poison, i32 0>
+; CHECK-NEXT: [[TMP4:%.*]] = insertelement <2 x double> [[TMP3]], double [[X]], i64 0
+; CHECK-NEXT: [[TMP5:%.*]] = fmul fast <2 x double> [[TMP4]], [[TMP2]]
; CHECK-NEXT: [[TMP6:%.*]] = call fast double @llvm.vector.reduce.fadd.v2f64(double 0.000000e+00, <2 x double> [[TMP5]])
; CHECK-NEXT: [[ACC_BODY:%.*]] = fadd fast double [[TMP6]], [[ACC]]
-; CHECK-NEXT: [[TMP7:%.*]] = extractelement <2 x double> [[TMP1]], i64 1
-; CHECK-NEXT: [[FY:%.*]] = fmul fast double [[C]], [[TMP7]]
-; CHECK-NEXT: [[FZ:%.*]] = fmul fast double [[C]], [[X]]
-; CHECK-NEXT: [[POUT1:%.*]] = getelementptr double, ptr [[OUT]], i64 1
-; CHECK-NEXT: store double [[FY]], ptr [[OUT]], align 8
-; CHECK-NEXT: store double [[FZ]], ptr [[POUT1]], align 8
+; CHECK-NEXT: [[TMP7:%.*]] = insertelement <2 x double> poison, double [[C]], i64 0
+; CHECK-NEXT: [[TMP8:%.*]] = shufflevector <2 x double> [[TMP7]], <2 x double> poison, <2 x i32> zeroinitializer
+; CHECK-NEXT: [[TMP9:%.*]] = fmul fast <2 x double> [[TMP8]], [[TMP1]]
+; CHECK-NEXT: store <2 x double> [[TMP9]], ptr [[OUT]], align 8
; CHECK-NEXT: br label %[[LATCH]]
; CHECK: [[LATCH]]:
; CHECK-NEXT: [[ACC_NEXT]] = phi double [ [[ACC_BODY]], %[[BODY]] ], [ [[ACC]], %[[HEADER]] ]
diff --git a/llvm/test/Transforms/SLPVectorizer/AArch64/fma-chain-no-alt-node-reduction.ll b/llvm/test/Transforms/SLPVectorizer/AArch64/fma-chain-no-alt-node-reduction.ll
index 66c89d60e23d9..c08ce3e4dd298 100644
--- a/llvm/test/Transforms/SLPVectorizer/AArch64/fma-chain-no-alt-node-reduction.ll
+++ b/llvm/test/Transforms/SLPVectorizer/AArch64/fma-chain-no-alt-node-reduction.ll
@@ -33,6 +33,7 @@ define void @density(ptr %lcs, ptr %liscom, ptr %densi, ptr %walls, i64 %first,
; CHECK-NEXT: [[P72:%.*]] = getelementptr inbounds nuw i8, ptr [[WALLS]], i64 72
; CHECK-NEXT: [[G11:%.*]] = load double, ptr [[P72]], align 8
; CHECK-NEXT: [[P104:%.*]] = getelementptr inbounds nuw i8, ptr [[WALLS]], i64 104
+; CHECK-NEXT: [[G22:%.*]] = load double, ptr [[P104]], align 8
; CHECK-NEXT: [[P136:%.*]] = getelementptr inbounds nuw i8, ptr [[WALLS]], i64 136
; CHECK-NEXT: [[G33:%.*]] = load double, ptr [[P136]], align 8
; CHECK-NEXT: [[P96:%.*]] = getelementptr inbounds nuw i8, ptr [[WALLS]], i64 96
@@ -41,23 +42,25 @@ define void @density(ptr %lcs, ptr %liscom, ptr %densi, ptr %walls, i64 %first,
; CHECK-NEXT: [[W80:%.*]] = load double, ptr [[P80]], align 8
; CHECK-NEXT: [[G12D:%.*]] = fadd fast double [[W80]], [[W96]]
; CHECK-NEXT: [[P120:%.*]] = getelementptr inbounds nuw i8, ptr [[WALLS]], i64 120
+; CHECK-NEXT: [[W120:%.*]] = load double, ptr [[P120]], align 8
; CHECK-NEXT: [[P88:%.*]] = getelementptr inbounds nuw i8, ptr [[WALLS]], i64 88
; CHECK-NEXT: [[W88:%.*]] = load double, ptr [[P88]], align 8
-; CHECK-NEXT: [[TMP0:%.*]] = load <2 x double>, ptr [[P120]], align 8
-; CHECK-NEXT: [[TMP8:%.*]] = load <2 x double>, ptr [[P104]], align 8
-; CHECK-NEXT: [[TMP18:%.*]] = insertelement <2 x double> [[TMP8]], double [[W88]], i64 0
-; CHECK-NEXT: [[TMP19:%.*]] = fadd fast <2 x double> [[TMP18]], [[TMP0]]
-; CHECK-NEXT: [[TMP4:%.*]] = shufflevector <2 x double> [[TMP8]], <2 x double> poison, <2 x i32> <i32 poison, i32 0>
-; CHECK-NEXT: [[TMP21:%.*]] = insertelement <2 x double> poison, double [[XI]], i64 0
-; CHECK-NEXT: [[TMP1:%.*]] = insertelement <2 x double> [[TMP21]], double [[YI]], i64 1
+; CHECK-NEXT: [[G13D:%.*]] = fadd fast double [[W88]], [[W120]]
+; CHECK-NEXT: [[P128:%.*]] = getelementptr inbounds nuw i8, ptr [[WALLS]], i64 128
+; CHECK-NEXT: [[W128:%.*]] = load double, ptr [[P128]], align 8
+; CHECK-NEXT: [[P112:%.*]] = getelementptr inbounds nuw i8, ptr [[WALLS]], i64 112
+; CHECK-NEXT: [[W112:%.*]] = load double, ptr [[P112]], align 8
+; CHECK-NEXT: [[G23D:%.*]] = fadd fast double [[W112]], [[W128]]
+; CHECK-NEXT: [[TMP0:%.*]] = insertelement <2 x double> poison, double [[YI]], i64 0
+; CHECK-NEXT: [[TMP1:%.*]] = insertelement <2 x double> [[TMP0]], double [[ZI]], i64 1
; CHECK-NEXT: [[TMP2:%.*]] = insertelement <2 x double> poison, double [[PBC]], i64 0
; CHECK-NEXT: [[TMP3:%.*]] = shufflevector <2 x double> [[TMP2]], <2 x double> poison, <2 x i32> zeroinitializer
-; CHECK-NEXT: [[TMP22:%.*]] = insertelement <2 x double> poison, double [[BX]], i64 0
-; CHECK-NEXT: [[TMP5:%.*]] = insertelement <2 x double> [[TMP22]], double [[BY]], i64 1
+; CHECK-NEXT: [[TMP4:%.*]] = insertelement <2 x double> poison, double [[BY]], i64 0
+; CHECK-NEXT: [[TMP5:%.*]] = insertelement <2 x double> [[TMP4]], double [[BZ]], i64 1
; CHECK-NEXT: [[TMP6:%.*]] = insertelement <2 x double> poison, double [[NPBC]], i64 0
; CHECK-NEXT: [[TMP7:%.*]] = shufflevector <2 x double> [[TMP6]], <2 x double> poison, <2 x i32> zeroinitializer
-; CHECK-NEXT: [[TMP23:%.*]] = insertelement <2 x double> poison, double [[G11]], i64 0
-; CHECK-NEXT: [[TMP9:%.*]] = insertelement <2 x double> [[TMP23]], double [[G12D]], i64 1
+; CHECK-NEXT: [[TMP8:%.*]] = insertelement <2 x double> poison, double [[G22]], i64 0
+; CHECK-NEXT: [[TMP9:%.*]] = insertelement <2 x double> [[TMP8]], double [[G23D]], i64 1
; CHECK-NEXT: br label %[[LOOP:.*]]
; CHECK: [[LOOP]]:
; CHECK-NEXT: [[IV:%.*]] = phi i64 [ [[FIRST]], %[[ENTRY]] ], [ [[IV_NEXT:%.*]], %[[LATCH:.*]] ]
@@ -69,16 +72,17 @@ define void @density(ptr %lcs, ptr %liscom, ptr %densi, ptr %walls, i64 %first,
; CHECK-NEXT: [[J_IDX:%.*]] = mul nsw i64 [[J]], 24
; CHECK-NEXT: [[LCS_J:%.*]] = getelementptr i8, ptr [[LCS]], i64 [[J_IDX]]
; CHECK-NEXT: [[PX:%.*]] = getelementptr i8, ptr [[LCS_J]], i64 48
-; CHECK-NEXT: [[PZ:%.*]] = getelementptr i8, ptr [[LCS_J]], i64 64
-; CHECK-NEXT: [[ZJ:%.*]] = load double, ptr [[PZ]], align 8
-; CHECK-NEXT: [[DX:%.*]] = fsub fast double [[ZI]], [[ZJ]]
+; CHECK-NEXT: [[XJ:%.*]] = load double, ptr [[PX]], align 8
+; CHECK-NEXT: [[DX:%.*]] = fsub fast double [[XI]], [[XJ]]
; CHECK-NEXT: [[CX:%.*]] = fcmp fast ogt double [[DX]], [[PBC]]
-; CHECK-NEXT: [[SX:%.*]] = select ninf nsz i1 [[CX]], double [[BZ]], double 0.000000e+00
+; CHECK-NEXT: [[SX:%.*]] = select ninf nsz i1 [[CX]], double [[BX]], double 0.000000e+00
; CHECK-NEXT: [[DX1:%.*]] = fsub reassoc nsz arcp contract afn double [[DX]], [[SX]]
; CHECK-NEXT: [[CX2:%.*]] = fcmp fast olt double [[DX1]], [[NPBC]]
-; CHECK-NEXT: [[DX2:%.*]] = fadd fast double [[DX1]], [[BZ]]
+; CHECK-NEXT: [[DX2:%.*]] = fadd fast double [[DX1]], [[BX]]
; CHECK-NEXT: [[XIJ:%.*]] = select nsz i1 [[CX2]], double [[DX2]], double [[DX1]]
-; CHECK-NEXT: [[TMP10:%.*]] = load <2 x double>, ptr [[PX]], align 8
+; CHECK-NEXT: [[PY:%.*]] = getelementptr i8, ptr [[LCS_J]], i64 56
+; CHECK-NEXT: [[M1:%.*]] = fmul fast double [[XIJ]], [[G11]]
+; CHECK-NEXT: [[TMP10:%.*]] = load <2 x double>, ptr [[PY]], align 8
; CHECK-NEXT: [[TMP11:%.*]] = fsub fast <2 x double> [[TMP1]], [[TMP10]]
; CHECK-NEXT: [[TMP12:%.*]] = fcmp fast ogt <2 x double> [[TMP11]], [[TMP3]]
; CHECK-NEXT: [[TMP13:%.*]] = select <2 x i1> [[TMP12]], <2 x double> [[TMP5]], <2 x double> zeroinitializer
@@ -86,22 +90,19 @@ define void @density(ptr %lcs, ptr %liscom, ptr %densi, ptr %walls, i64 %first,
; CHECK-NEXT: [[TMP15:%.*]] = fcmp fast olt <2 x double> [[TMP14]], [[TMP7]]
; CHECK-NEXT: [[TMP16:%.*]] = fadd fast <2 x double> [[TMP14]], [[TMP5]]
; CHECK-NEXT: [[TMP17:%.*]] = select <2 x i1> [[TMP15]], <2 x double> [[TMP16]], <2 x double> [[TMP14]]
+; CHECK-NEXT: [[TMP18:%.*]] = extractelement <2 x double> [[TMP17]], i64 0
+; CHECK-NEXT: [[M2:%.*]] = fmul fast double [[TMP18]], [[G12D]]
+; CHECK-NEXT: [[A1:%.*]] = fadd fast double [[M2]], [[M1]]
+; CHECK-NEXT: [[TMP19:%.*]] = extractelement <2 x double> [[TMP17]], i64 1
+; CHECK-NEXT: [[M3:%.*]] = fmul fast double [[TMP19]], [[G13D]]
+; CHECK-NEXT: [[A2:%.*]] = fadd fast double [[A1]], [[M3]]
+; CHECK-NEXT: [[TX:%.*]] = fmul fast double [[A2]], [[XIJ]]
; CHECK-NEXT: [[TMP20:%.*]] = fmul fast <2 x double> [[TMP17]], [[TMP9]]
-; CHECK-NEXT: [[TMP24:%.*]] = shufflevector <2 x double> [[TMP17]], <2 x double> [[TMP20]], <2 x i32> <i32 3, i32 1>
-; CHECK-NEXT: [[TMP25:%.*]] = shufflevector <2 x double> [[TMP4]], <2 x double> [[TMP20]], <2 x i32> <i32 2, i32 1>
-; CHECK-NEXT: [[TMP26:%.*]] = fmul fast <2 x double> [[TMP24]], [[TMP25]]
-; CHECK-NEXT: [[TMP27:%.*]] = fadd fast <2 x double> [[TMP24]], [[TMP25]]
-; CHECK-NEXT: [[TMP28:%.*]] = shufflevector <2 x double> [[TMP26]], <2 x double> [[TMP27]], <2 x i32> <i32 2, i32 1>
-; CHECK-NEXT: [[TMP29:%.*]] = insertelement <2 x double> poison, double [[XIJ]], i64 0
-; CHECK-NEXT: [[TMP30:%.*]] = shufflevector <2 x double> [[TMP29]], <2 x double> poison, <2 x i32> zeroinitializer
-; CHECK-NEXT: [[TMP31:%.*]] = fmul fast <2 x double> [[TMP30]], [[TMP19]]
-; CHECK-NEXT: [[TMP32:%.*]] = fadd fast <2 x double> [[TMP31]], [[TMP28]]
-; CHECK-NEXT: [[TMP33:%.*]] = fmul fast <2 x double> [[TMP32]], [[TMP17]]
-; CHECK-NEXT: [[ZZ:%.*]] = fmul fast double [[XIJ]], [[XIJ]]
+; CHECK-NEXT: [[TMP21:%.*]] = call fast double @llvm.vector.reduce.fadd.v2f64(double 0.000000e+00, <2 x double> [[TMP20]])
+; CHECK-NEXT: [[TY:%.*]] = fmul fast double [[TMP21]], [[TMP18]]
+; CHECK-NEXT: [[ZZ:%.*]] = fmul fast double [[TMP19]], [[TMP19]]
; CHECK-NEXT: [[TZ:%.*]] = fmul fast double [[ZZ]], [[G33]]
-; CHECK-NEXT: [[TY:%.*]] = extractelement <2 x double> [[TMP33]], i64 1
; CHECK-NEXT: [[A4:%.*]] = fadd fast double [[TY]], [[TZ]]
-; CHECK-NEXT: [[TX:%.*]] = extractelement <2 x double> [[TMP33]], i64 0
; CHECK-NEXT: [[RSQ:%.*]] = fadd fast double [[A4]], [[TX]]
; CHECK-NEXT: [[CMP:%.*]] = fcmp fast olt double [[RSQ]], [[RCUT]]
; CHECK-NEXT: br i1 [[CMP]], label %[[ACCUM:.*]], label %[[LATCH]]
diff --git a/llvm/test/Transforms/SLPVectorizer/AArch64/fmul-constant-lane-fmuladd-combine.ll b/llvm/test/Transforms/SLPVectorizer/AArch64/fmul-constant-lane-fmuladd-combine.ll
index e96a940de1733..38730e816a824 100644
--- a/llvm/test/Transforms/SLPVectorizer/AArch64/fmul-constant-lane-fmuladd-combine.ll
+++ b/llvm/test/Transforms/SLPVectorizer/AArch64/fmul-constant-lane-fmuladd-combine.ll
@@ -15,31 +15,33 @@ define void @fmul_constant_lane_rows(ptr %p, ptr %out, double %c0, double %c1, d
; CHECK-LABEL: define void @fmul_constant_lane_rows(
; CHECK-SAME: ptr [[P:%.*]], ptr [[OUT:%.*]], double [[C0:%.*]], double [[C1:%.*]], double [[C2:%.*]], double [[C3:%.*]], double [[C4:%.*]], double [[C5:%.*]], double [[C6:%.*]], double [[C7:%.*]], double [[C8:%.*]]) #[[ATTR0:[0-9]+]] {
; CHECK-NEXT: [[ENTRY:.*:]]
+; CHECK-NEXT: [[L0:%.*]] = load double, ptr [[P]], align 8
+; CHECK-NEXT: [[P1:%.*]] = getelementptr double, ptr [[P]], i64 1
+; CHECK-NEXT: [[L1:%.*]] = load double, ptr [[P1]], align 8
; CHECK-NEXT: [[P2:%.*]] = getelementptr double, ptr [[P]], i64 2
; CHECK-NEXT: [[L2:%.*]] = load double, ptr [[P2]], align 8
-; CHECK-NEXT: [[TMP0:%.*]] = load <2 x double>, ptr [[P]], align 8
-; CHECK-NEXT: [[TMP1:%.*]] = shufflevector <2 x double> [[TMP0]], <2 x double> <double undef, double -0.000000e+00>, <4 x i32> <i32 0, i32 0, i32 0, i32 3>
-; CHECK-NEXT: [[TMP2:%.*]] = insertelement <4 x double> <double poison, double poison, double poison, double 1.000000e+00>, double [[C0]], i64 0
-; CHECK-NEXT: [[TMP3:%.*]] = insertelement <4 x double> [[TMP2]], double [[C3]], i64 1
-; CHECK-NEXT: [[TMP4:%.*]] = insertelement <4 x double> [[TMP3]], double [[C6]], i64 2
-; CHECK-NEXT: [[TMP5:%.*]] = fmul fast <4 x double> [[TMP1]], [[TMP4]]
-; CHECK-NEXT: [[TMP6:%.*]] = shufflevector <2 x double> [[TMP0]], <2 x double> poison, <4 x i32> <i32 poison, i32 1, i32 poison, i32 poison>
-; CHECK-NEXT: [[TMP7:%.*]] = shufflevector <2 x double> [[TMP0]], <2 x double> poison, <4 x i32> <i32 0, i32 1, i32 poison, i32 poison>
-; CHECK-NEXT: [[TMP8:%.*]] = shufflevector <4 x double> <double poison, double 0.000000e+00, double poison, double poison>, <4 x double> [[TMP7]], <4 x i32> <i32 5, i32 1, i32 poison, i32 poison>
-; CHECK-NEXT: [[TMP9:%.*]] = shufflevector <4 x double> [[TMP8]], <4 x double> poison, <4 x i32> <i32 0, i32 0, i32 0, i32 1>
-; CHECK-NEXT: [[TMP10:%.*]] = insertelement <4 x double> <double poison, double poison, double poison, double 1.000000e+00>, double [[C1]], i64 0
-; CHECK-NEXT: [[TMP11:%.*]] = insertelement <4 x double> [[TMP10]], double [[C4]], i64 1
-; CHECK-NEXT: [[TMP12:%.*]] = insertelement <4 x double> [[TMP11]], double [[C7]], i64 2
-; CHECK-NEXT: [[TMP13:%.*]] = fmul fast <4 x double> [[TMP9]], [[TMP12]]
-; CHECK-NEXT: [[TMP14:%.*]] = fadd fast <4 x double> [[TMP13]], [[TMP5]]
-; CHECK-NEXT: [[TMP15:%.*]] = insertelement <4 x double> <double poison, double -0.000000e+00, double poison, double poison>, double [[L2]], i64 0
-; CHECK-NEXT: [[TMP16:%.*]] = shufflevector <4 x double> [[TMP15]], <4 x double> poison, <4 x i32> <i32 0, i32 0, i32 0, i32 1>
-; CHECK-NEXT: [[TMP17:%.*]] = insertelement <4 x double> <double poison, double poison, double poison, double 1.000000e+00>, double [[C2]], i64 0
-; CHECK-NEXT: [[TMP18:%.*]] = insertelement <4 x double> [[TMP17]], double [[C5]], i64 1
-; CHECK-NEXT: [[TMP19:%.*]] = insertelement <4 x double> [[TMP18]], double [[C8]], i64 2
-; CHECK-NEXT: [[TMP20:%.*]] = fmul fast <4 x double> [[TMP16]], [[TMP19]]
-; CHECK-NEXT: [[TMP21:%.*]] = fadd fast <4 x double> [[TMP14]], [[TMP20]]
-; CHECK-NEXT: store <4 x double> [[TMP21]], ptr [[OUT]], align 8
+; CHECK-NEXT: [[A00:%.*]] = fmul fast double [[L0]], [[C0]]
+; CHECK-NEXT: [[A01:%.*]] = fmul fast double [[L1]], [[C1]]
+; CHECK-NEXT: [[A02:%.*]] = fadd fast double [[A01]], [[A00]]
+; CHECK-NEXT: [[A03:%.*]] = fmul fast double [[L2]], [[C2]]
+; CHECK-NEXT: [[A:%.*]] = fadd fast double [[A02]], [[A03]]
+; CHECK-NEXT: [[B00:%.*]] = fmul fast double [[L0]], [[C3]]
+; CHECK-NEXT: [[B01:%.*]] = fmul fast double [[L1]], [[C4]]
+; CHECK-NEXT: [[B02:%.*]] = fadd fast double [[B01]], [[B00]]
+; CHECK-NEXT: [[B03:%.*]] = fmul fast double [[L2]], [[C5]]
+; CHECK-NEXT: [[B:%.*]] = fadd fast double [[B02]], [[B03]]
+; CHECK-NEXT: [[D00:%.*]] = fmul fast double [[L0]], [[C6]]
+; CHECK-NEXT: [[D01:%.*]] = fmul fast double [[L1]], [[C7]]
+; CHECK-NEXT: [[D02:%.*]] = fadd fast double [[D01]], [[D00]]
+; CHECK-NEXT: [[D03:%.*]] = fmul fast double [[L2]], [[C8]]
+; CHECK-NEXT: [[D:%.*]] = fadd fast double [[D02]], [[D03]]
+; CHECK-NEXT: store double [[A]], ptr [[OUT]], align 8
+; CHECK-NEXT: [[O1:%.*]] = getelementptr double, ptr [[OUT]], i64 1
+; CHECK-NEXT: store double [[B]], ptr [[O1]], align 8
+; CHECK-NEXT: [[O2:%.*]] = getelementptr double, ptr [[OUT]], i64 2
+; CHECK-NEXT: store double [[D]], ptr [[O2]], align 8
+; CHECK-NEXT: [[O3:%.*]] = getelementptr double, ptr [[OUT]], i64 3
+; CHECK-NEXT: store double 0.000000e+00, ptr [[O3]], align 8
; CHECK-NEXT: ret void
;
entry:
diff --git a/llvm/test/Transforms/SLPVectorizer/AArch64/hor-fp-reduction-instcount.ll b/llvm/test/Transforms/SLPVectorizer/AArch64/hor-fp-reduction-instcount.ll
index 304db13bf6bb0..0dad46ec85a40 100644
--- a/llvm/test/Transforms/SLPVectorizer/AArch64/hor-fp-reduction-instcount.ll
+++ b/llvm/test/Transforms/SLPVectorizer/AArch64/hor-fp-reduction-instcount.ll
@@ -2,46 +2,59 @@
; RUN: opt < %s -S -passes=slp-vectorizer -mtriple=aarch64-unknown-linux -mcpu=neoverse-v2 | FileCheck %s --check-prefix=NARROW
; RUN: opt < %s -S -passes=slp-vectorizer -mtriple=aarch64-unknown-linux -mcpu=neoverse-v1 | FileCheck %s --check-prefix=WIDE
-; A horizontal double reduction with gathered operands is not profitable on
-; targets whose vector register holds only 2 doubles: the operand gathering and
-; the reduction epilogue produce more instructions than the scalar reduction.
-; On wider targets (4+ doubles per register) it stays profitable.
+; A horizontal double reduction with gathered operands is not profitable: the
+; operand gathering and the reduction epilogue cost more than the scalar
+; reduction, whose fadd links all fuse with their fmul operands into fmadds.
define double @f64_red_gather(ptr %a, ptr %w) {
; NARROW-LABEL: define double @f64_red_gather(
; NARROW-SAME: ptr [[A:%.*]], ptr [[W:%.*]]) #[[ATTR0:[0-9]+]] {
; NARROW-NEXT: [[A0:%.*]] = load double, ptr [[A]], align 8
+; NARROW-NEXT: [[W0:%.*]] = load double, ptr [[W]], align 8
+; NARROW-NEXT: [[P0:%.*]] = fmul reassoc contract double [[A0]], [[W0]]
; NARROW-NEXT: [[A1P:%.*]] = getelementptr double, ptr [[A]], i64 3
; NARROW-NEXT: [[A1:%.*]] = load double, ptr [[A1P]], align 8
+; NARROW-NEXT: [[W1P:%.*]] = getelementptr double, ptr [[W]], i64 1
+; NARROW-NEXT: [[W1:%.*]] = load double, ptr [[W1P]], align 8
+; NARROW-NEXT: [[P1:%.*]] = fmul reassoc contract double [[A1]], [[W1]]
; NARROW-NEXT: [[A2P:%.*]] = getelementptr double, ptr [[A]], i64 6
; NARROW-NEXT: [[A2:%.*]] = load double, ptr [[A2P]], align 8
+; NARROW-NEXT: [[W2P:%.*]] = getelementptr double, ptr [[W]], i64 2
+; NARROW-NEXT: [[W2:%.*]] = load double, ptr [[W2P]], align 8
+; NARROW-NEXT: [[P2:%.*]] = fmul reassoc contract double [[A2]], [[W2]]
; NARROW-NEXT: [[A3P:%.*]] = getelementptr double, ptr [[A]], i64 9
; NARROW-NEXT: [[A3:%.*]] = load double, ptr [[A3P]], align 8
-; NARROW-NEXT: [[TMP1:%.*]] = load <4 x double>, ptr [[W]], align 8
-; NARROW-NEXT: [[TMP2:%.*]] = insertelement <4 x double> poison, double [[A0]], i64 0
-; NARROW-NEXT: [[TMP3:%.*]] = insertelement <4 x double> [[TMP2]], double [[A1]], i64 1
-; NARROW-NEXT: [[TMP4:%.*]] = insertelement <4 x double> [[TMP3]], double [[A2]], i64 2
-; NARROW-NEXT: [[TMP5:%.*]] = insertelement <4 x double> [[TMP4]], double [[A3]], i64 3
-; NARROW-NEXT: [[TMP6:%.*]] = fmul reassoc contract <4 x double> [[TMP5]], [[TMP1]]
-; NARROW-NEXT: [[TMP7:%.*]] = call reassoc contract double @llvm.vector.reduce.fadd.v4f64(double -0.000000e+00, <4 x double> [[TMP6]])
+; NARROW-NEXT: [[W3P:%.*]] = getelementptr double, ptr [[W]], i64 3
+; NARROW-NEXT: [[W3:%.*]] = load double, ptr [[W3P]], align 8
+; NARROW-NEXT: [[P3:%.*]] = fmul reassoc contract double [[A3]], [[W3]]
+; NARROW-NEXT: [[R0:%.*]] = fadd reassoc contract double [[P0]], [[P1]]
+; NARROW-NEXT: [[R1:%.*]] = fadd reassoc contract double [[R0]], [[P2]]
+; NARROW-NEXT: [[TMP7:%.*]] = fadd reassoc contract double [[R1]], [[P3]]
; NARROW-NEXT: ret double [[TMP7]]
;
; WIDE-LABEL: define double @f64_red_gather(
; WIDE-SAME: ptr [[A:%.*]], ptr [[W:%.*]]) #[[ATTR0:[0-9]+]] {
; WIDE-NEXT: [[A0:%.*]] = load double, ptr [[A]], align 8
+; WIDE-NEXT: [[W0:%.*]] = load double, ptr [[W]], align 8
+; WIDE-NEXT: [[P0:%.*]] = fmul reassoc contract double [[A0]], [[W0]]
; WIDE-NEXT: [[A1P:%.*]] = getelementptr double, ptr [[A]], i64 3
; WIDE-NEXT: [[A1:%.*]] = load double, ptr [[A1P]], align 8
+; WIDE-NEXT: [[W1P:%.*]] = getelementptr double, ptr [[W]], i64 1
+; WIDE-NEXT: [[W1:%.*]] = load double, ptr [[W1P]], align 8
+; WIDE-NEXT: [[P1:%.*]] = fmul reassoc contract double [[A1]], [[W1]]
; WIDE-NEXT: [[A2P:%.*]] = getelementptr double, ptr [[A]], i64 6
; WIDE-NEXT: [[A2:%.*]] = load double, ptr [[A2P]], align 8
+; WIDE-NEXT: [[W2P:%.*]] = getelementptr double, ptr [[W]], i64 2
+; WIDE-NEXT: [[W2:%.*]] = load double, ptr [[W2P]], align 8
+; WIDE-NEXT: [[P2:%.*]] = fmul reassoc contract double [[A2]], [[W2]]
; WIDE-NEXT: [[A3P:%.*]] = getelementptr double, ptr [[A]], i64 9
; WIDE-NEXT: [[A3:%.*]] = load double, ptr [[A3P]], align 8
-; WIDE-NEXT: [[TMP1:%.*]] = load <4 x double>, ptr [[W]], align 8
-; WIDE-NEXT: [[TMP2:%.*]] = insertelement <4 x double> poison, double [[A0]], i64 0
-; WIDE-NEXT: [[TMP3:%.*]] = insertelement <4 x double> [[TMP2]], double [[A1]], i64 1
-; WIDE-NEXT: [[TMP4:%.*]] = insertelement <4 x double> [[TMP3]], double [[A2]], i64 2
-; WIDE-NEXT: [[TMP5:%.*]] = insertelement <4 x double> [[TMP4]], double [[A3]], i64 3
-; WIDE-NEXT: [[TMP6:%.*]] = fmul reassoc contract <4 x double> [[TMP5]], [[TMP1]]
-; WIDE-NEXT: [[TMP7:%.*]] = call reassoc contract double @llvm.vector.reduce.fadd.v4f64(double -0.000000e+00, <4 x double> [[TMP6]])
+; WIDE-NEXT: [[W3P:%.*]] = getelementptr double, ptr [[W]], i64 3
+; WIDE-NEXT: [[W3:%.*]] = load double, ptr [[W3P]], align 8
+; WIDE-NEXT: [[P3:%.*]] = fmul reassoc contract double [[A3]], [[W3]]
+; WIDE-NEXT: [[R0:%.*]] = fadd reassoc contract double [[P0]], [[P1]]
+; WIDE-NEXT: [[R1:%.*]] = fadd reassoc contract double [[R0]], [[P2]]
+; WIDE-NEXT: [[TMP7:%.*]] = fadd reassoc contract double [[R1]], [[P3]]
; WIDE-NEXT: ret double [[TMP7]]
;
%a0 = load double, ptr %a
diff --git a/llvm/test/Transforms/SLPVectorizer/AArch64/loop-accumulator-reduction.ll b/llvm/test/Transforms/SLPVectorizer/AArch64/loop-accumulator-reduction.ll
index 985f31d5f3090..8cdb1b4fccafe 100644
--- a/llvm/test/Transforms/SLPVectorizer/AArch64/loop-accumulator-reduction.ll
+++ b/llvm/test/Transforms/SLPVectorizer/AArch64/loop-accumulator-reduction.ll
@@ -123,15 +123,14 @@ define double @loop_acc_fadd_extra_use(ptr %p, i64 %n, ptr %o) {
; CHECK-NEXT: br label %[[LOOP:.*]]
; CHECK: [[LOOP]]:
; CHECK-NEXT: [[IV:%.*]] = phi i64 [ 1, %[[ENTRY]] ], [ [[IV_NEXT:%.*]], %[[LOOP]] ]
-; CHECK-NEXT: [[ACC:%.*]] = phi double [ 0.000000e+00, %[[ENTRY]] ], [ [[TMP24:%.*]], %[[LOOP]] ]
+; CHECK-NEXT: [[ACC:%.*]] = phi double [ 0.000000e+00, %[[ENTRY]] ], [ [[SUM:%.*]], %[[LOOP]] ]
; CHECK-NEXT: [[IV3:%.*]] = mul nuw i64 [[IV]], 3
; CHECK-NEXT: [[P0:%.*]] = getelementptr double, ptr [[P]], i64 [[IV3]]
+; CHECK-NEXT: [[L0:%.*]] = load double, ptr [[P0]], align 8
; CHECK-NEXT: [[P1:%.*]] = getelementptr double, ptr [[P0]], i64 1
+; CHECK-NEXT: [[L1:%.*]] = load double, ptr [[P1]], align 8
; CHECK-NEXT: [[P2:%.*]] = getelementptr double, ptr [[P0]], i64 2
; CHECK-NEXT: [[L2:%.*]] = load double, ptr [[P2]], align 8
-; CHECK-NEXT: [[C8:%.*]] = load double, ptr getelementptr (double, ptr @cst, i64 8), align 8
-; CHECK-NEXT: [[L0:%.*]] = load double, ptr [[P0]], align 8
-; CHECK-NEXT: [[L1:%.*]] = load double, ptr [[P1]], align 8
; CHECK-NEXT: [[C0:%.*]] = load double, ptr @cst, align 8
; CHECK-NEXT: [[C1:%.*]] = load double, ptr getelementptr (double, ptr @cst, i64 1), align 8
; CHECK-NEXT: [[C2:%.*]] = load double, ptr getelementptr (double, ptr @cst, i64 2), align 8
@@ -140,37 +139,34 @@ define double @loop_acc_fadd_extra_use(ptr %p, i64 %n, ptr %o) {
; CHECK-NEXT: [[C5:%.*]] = load double, ptr getelementptr (double, ptr @cst, i64 5), align 8
; CHECK-NEXT: [[C6:%.*]] = load double, ptr getelementptr (double, ptr @cst, i64 6), align 8
; CHECK-NEXT: [[C7:%.*]] = load double, ptr getelementptr (double, ptr @cst, i64 7), align 8
-; CHECK-NEXT: [[TMP0:%.*]] = insertelement <4 x double> <double poison, double -0.000000e+00, double poison, double poison>, double [[L0]], i64 0
-; CHECK-NEXT: [[TMP1:%.*]] = shufflevector <4 x double> [[TMP0]], <4 x double> poison, <4 x i32> <i32 0, i32 0, i32 0, i32 1>
-; CHECK-NEXT: [[TMP2:%.*]] = insertelement <4 x double> <double poison, double poison, double poison, double 1.000000e+00>, double [[C0]], i64 0
-; CHECK-NEXT: [[TMP3:%.*]] = insertelement <4 x double> [[TMP2]], double [[C3]], i64 1
-; CHECK-NEXT: [[TMP4:%.*]] = insertelement <4 x double> [[TMP3]], double [[C6]], i64 2
-; CHECK-NEXT: [[TMP5:%.*]] = fmul fast <4 x double> [[TMP1]], [[TMP4]]
-; CHECK-NEXT: [[TMP6:%.*]] = insertelement <4 x double> poison, double [[L1]], i64 0
-; CHECK-NEXT: [[TMP7:%.*]] = insertelement <4 x double> [[TMP6]], double [[ACC]], i64 1
-; CHECK-NEXT: [[TMP8:%.*]] = shufflevector <4 x double> [[TMP7]], <4 x double> poison, <4 x i32> <i32 0, i32 0, i32 0, i32 1>
-; CHECK-NEXT: [[TMP9:%.*]] = insertelement <4 x double> <double poison, double poison, double poison, double 1.000000e+00>, double [[C1]], i64 0
-; CHECK-NEXT: [[TMP10:%.*]] = insertelement <4 x double> [[TMP9]], double [[C4]], i64 1
-; CHECK-NEXT: [[TMP11:%.*]] = insertelement <4 x double> [[TMP10]], double [[C7]], i64 2
-; CHECK-NEXT: [[TMP12:%.*]] = fmul reassoc nsz arcp contract afn <4 x double> [[TMP8]], [[TMP11]]
-; CHECK-NEXT: [[TMP13:%.*]] = fadd reassoc nsz arcp contract afn <4 x double> [[TMP12]], [[TMP5]]
-; CHECK-NEXT: [[TMP14:%.*]] = insertelement <4 x double> <double poison, double -0.000000e+00, double poison, double poison>, double [[L2]], i64 0
-; CHECK-NEXT: [[TMP15:%.*]] = shufflevector <4 x double> [[TMP14]], <4 x double> poison, <4 x i32> <i32 0, i32 0, i32 0, i32 1>
-; CHECK-NEXT: [[TMP16:%.*]] = insertelement <4 x double> <double poison, double poison, double poison, double 1.000000e+00>, double [[C2]], i64 0
-; CHECK-NEXT: [[TMP17:%.*]] = insertelement <4 x double> [[TMP16]], double [[C5]], i64 1
-; CHECK-NEXT: [[TMP18:%.*]] = insertelement <4 x double> [[TMP17]], double [[C8]], i64 2
-; CHECK-NEXT: [[TMP19:%.*]] = fmul fast <4 x double> [[TMP15]], [[TMP18]]
-; CHECK-NEXT: [[TMP20:%.*]] = fadd reassoc nsz arcp contract afn <4 x double> [[TMP13]], [[TMP19]]
-; CHECK-NEXT: [[TMP21:%.*]] = shufflevector <4 x double> [[TMP20]], <4 x double> <double poison, double poison, double poison, double 1.000000e+00>, <4 x i32> <i32 0, i32 1, i32 poison, i32 7>
-; CHECK-NEXT: [[TMP22:%.*]] = shufflevector <4 x double> [[TMP21]], <4 x double> [[TMP20]], <4 x i32> <i32 0, i32 1, i32 6, i32 3>
-; CHECK-NEXT: [[TMP23:%.*]] = fmul <4 x double> [[TMP20]], [[TMP22]]
-; CHECK-NEXT: [[TMP24]] = call fast double @llvm.vector.reduce.fadd.v4f64(double 0.000000e+00, <4 x double> [[TMP23]])
+; CHECK-NEXT: [[C8:%.*]] = load double, ptr getelementptr (double, ptr @cst, i64 8), align 8
+; CHECK-NEXT: [[A00:%.*]] = fmul fast double [[L0]], [[C0]]
+; CHECK-NEXT: [[A01:%.*]] = fmul fast double [[L1]], [[C1]]
+; CHECK-NEXT: [[A02:%.*]] = fadd fast double [[A01]], [[A00]]
+; CHECK-NEXT: [[A03:%.*]] = fmul fast double [[L2]], [[C2]]
+; CHECK-NEXT: [[A:%.*]] = fadd fast double [[A02]], [[A03]]
+; CHECK-NEXT: [[B00:%.*]] = fmul fast double [[L0]], [[C3]]
+; CHECK-NEXT: [[B01:%.*]] = fmul fast double [[L1]], [[C4]]
+; CHECK-NEXT: [[B02:%.*]] = fadd fast double [[B01]], [[B00]]
+; CHECK-NEXT: [[B03:%.*]] = fmul fast double [[L2]], [[C5]]
+; CHECK-NEXT: [[B:%.*]] = fadd fast double [[B02]], [[B03]]
+; CHECK-NEXT: [[D00:%.*]] = fmul fast double [[L0]], [[C6]]
+; CHECK-NEXT: [[D01:%.*]] = fmul fast double [[L1]], [[C7]]
+; CHECK-NEXT: [[D02:%.*]] = fadd fast double [[D01]], [[D00]]
+; CHECK-NEXT: [[D03:%.*]] = fmul fast double [[L2]], [[C8]]
+; CHECK-NEXT: [[D:%.*]] = fadd fast double [[D02]], [[D03]]
+; CHECK-NEXT: [[SA:%.*]] = fmul double [[A]], [[A]]
+; CHECK-NEXT: [[T1:%.*]] = fadd fast double [[SA]], [[ACC]]
+; CHECK-NEXT: [[SB:%.*]] = fmul double [[B]], [[B]]
+; CHECK-NEXT: [[T2:%.*]] = fadd fast double [[T1]], [[SB]]
+; CHECK-NEXT: [[SD:%.*]] = fmul double [[D]], [[D]]
+; CHECK-NEXT: [[SUM]] = fadd fast double [[T2]], [[SD]]
; CHECK-NEXT: store double [[ACC]], ptr [[O]], align 8
; CHECK-NEXT: [[IV_NEXT]] = add nuw nsw i64 [[IV]], 1
; CHECK-NEXT: [[CMP:%.*]] = icmp eq i64 [[IV]], [[N]]
; CHECK-NEXT: br i1 [[CMP]], label %[[EXIT:.*]], label %[[LOOP]]
; CHECK: [[EXIT]]:
-; CHECK-NEXT: [[RES:%.*]] = phi double [ [[TMP24]], %[[LOOP]] ]
+; CHECK-NEXT: [[RES:%.*]] = phi double [ [[SUM]], %[[LOOP]] ]
; CHECK-NEXT: ret double [[RES]]
;
entry:
@@ -403,12 +399,11 @@ define double @root_stored(ptr %p, i64 %n, ptr %o) {
; CHECK-NEXT: [[ACC:%.*]] = phi double [ 0.000000e+00, %[[ENTRY]] ], [ [[TMP23:%.*]], %[[LOOP]] ]
; CHECK-NEXT: [[IV3:%.*]] = mul nuw i64 [[IV]], 3
; CHECK-NEXT: [[P0:%.*]] = getelementptr double, ptr [[P]], i64 [[IV3]]
+; CHECK-NEXT: [[L0:%.*]] = load double, ptr [[P0]], align 8
; CHECK-NEXT: [[P1:%.*]] = getelementptr double, ptr [[P0]], i64 1
+; CHECK-NEXT: [[L1:%.*]] = load double, ptr [[P1]], align 8
; CHECK-NEXT: [[P2:%.*]] = getelementptr double, ptr [[P0]], i64 2
; CHECK-NEXT: [[L2:%.*]] = load double, ptr [[P2]], align 8
-; CHECK-NEXT: [[C8:%.*]] = load double, ptr getelementptr (double, ptr @cst, i64 8), align 8
-; CHECK-NEXT: [[L0:%.*]] = load double, ptr [[P0]], align 8
-; CHECK-NEXT: [[L1:%.*]] = load double, ptr [[P1]], align 8
; CHECK-NEXT: [[C0:%.*]] = load double, ptr @cst, align 8
; CHECK-NEXT: [[C1:%.*]] = load double, ptr getelementptr (double, ptr @cst, i64 1), align 8
; CHECK-NEXT: [[C2:%.*]] = load double, ptr getelementptr (double, ptr @cst, i64 2), align 8
@@ -417,31 +412,28 @@ define double @root_stored(ptr %p, i64 %n, ptr %o) {
; CHECK-NEXT: [[C5:%.*]] = load double, ptr getelementptr (double, ptr @cst, i64 5), align 8
; CHECK-NEXT: [[C6:%.*]] = load double, ptr getelementptr (double, ptr @cst, i64 6), align 8
; CHECK-NEXT: [[C7:%.*]] = load double, ptr getelementptr (double, ptr @cst, i64 7), align 8
-; CHECK-NEXT: [[TMP0:%.*]] = insertelement <4 x double> <double poison, double -0.000000e+00, double poison, double poison>, double [[L0]], i64 0
-; CHECK-NEXT: [[TMP1:%.*]] = shufflevector <4 x double> [[TMP0]], <4 x double> poison, <4 x i32> <i32 0, i32 0, i32 0, i32 1>
-; CHECK-NEXT: [[TMP2:%.*]] = insertelement <4 x double> <double poison, double poison, double poison, double 1.000000e+00>, double [[C0]], i64 0
-; CHECK-NEXT: [[TMP3:%.*]] = insertelement <4 x double> [[TMP2]], double [[C3]], i64 1
-; CHECK-NEXT: [[TMP4:%.*]] = insertelement <4 x double> [[TMP3]], double [[C6]], i64 2
-; CHECK-NEXT: [[TMP5:%.*]] = fmul fast <4 x double> [[TMP1]], [[TMP4]]
-; CHECK-NEXT: [[TMP6:%.*]] = insertelement <4 x double> poison, double [[L1]], i64 0
-; CHECK-NEXT: [[TMP7:%.*]] = insertelement <4 x double> [[TMP6]], double [[ACC]], i64 1
-; CHECK-NEXT: [[TMP8:%.*]] = shufflevector <4 x double> [[TMP7]], <4 x double> poison, <4 x i32> <i32 0, i32 0, i32 0, i32 1>
-; CHECK-NEXT: [[TMP9:%.*]] = insertelement <4 x double> <double poison, double poison, double poison, double 1.000000e+00>, double [[C1]], i64 0
-; CHECK-NEXT: [[TMP10:%.*]] = insertelement <4 x double> [[TMP9]], double [[C4]], i64 1
-; CHECK-NEXT: [[TMP11:%.*]] = insertelement <4 x double> [[TMP10]], double [[C7]], i64 2
-; CHECK-NEXT: [[TMP12:%.*]] = fmul reassoc nsz arcp contract afn <4 x double> [[TMP8]], [[TMP11]]
-; CHECK-NEXT: [[TMP13:%.*]] = fadd reassoc nsz arcp contract afn <4 x double> [[TMP12]], [[TMP5]]
-; CHECK-NEXT: [[TMP14:%.*]] = insertelement <4 x double> <double poison, double -0.000000e+00, double poison, double poison>, double [[L2]], i64 0
-; CHECK-NEXT: [[TMP15:%.*]] = shufflevector <4 x double> [[TMP14]], <4 x double> poison, <4 x i32> <i32 0, i32 0, i32 0, i32 1>
-; CHECK-NEXT: [[TMP16:%.*]] = insertelement <4 x double> <double poison, double poison, double poison, double 1.000000e+00>, double [[C2]], i64 0
-; CHECK-NEXT: [[TMP17:%.*]] = insertelement <4 x double> [[TMP16]], double [[C5]], i64 1
-; CHECK-NEXT: [[TMP18:%.*]] = insertelement <4 x double> [[TMP17]], double [[C8]], i64 2
-; CHECK-NEXT: [[TMP19:%.*]] = fmul fast <4 x double> [[TMP15]], [[TMP18]]
-; CHECK-NEXT: [[TMP20:%.*]] = fadd reassoc nsz arcp contract afn <4 x double> [[TMP13]], [[TMP19]]
-; CHECK-NEXT: [[TMP21:%.*]] = shufflevector <4 x double> [[TMP20]], <4 x double> <double poison, double poison, double poison, double 1.000000e+00>, <4 x i32> <i32 0, i32 1, i32 poison, i32 7>
-; CHECK-NEXT: [[TMP22:%.*]] = shufflevector <4 x double> [[TMP21]], <4 x double> [[TMP20]], <4 x i32> <i32 0, i32 1, i32 6, i32 3>
-; CHECK-NEXT: [[TMP24:%.*]] = fmul <4 x double> [[TMP20]], [[TMP22]]
-; CHECK-NEXT: [[TMP23]] = call fast double @llvm.vector.reduce.fadd.v4f64(double 0.000000e+00, <4 x double> [[TMP24]])
+; CHECK-NEXT: [[C8:%.*]] = load double, ptr getelementptr (double, ptr @cst, i64 8), align 8
+; CHECK-NEXT: [[A00:%.*]] = fmul fast double [[L0]], [[C0]]
+; CHECK-NEXT: [[A01:%.*]] = fmul fast double [[L1]], [[C1]]
+; CHECK-NEXT: [[A02:%.*]] = fadd fast double [[A01]], [[A00]]
+; CHECK-NEXT: [[A03:%.*]] = fmul fast double [[L2]], [[C2]]
+; CHECK-NEXT: [[A:%.*]] = fadd fast double [[A02]], [[A03]]
+; CHECK-NEXT: [[B00:%.*]] = fmul fast double [[L0]], [[C3]]
+; CHECK-NEXT: [[B01:%.*]] = fmul fast double [[L1]], [[C4]]
+; CHECK-NEXT: [[B02:%.*]] = fadd fast double [[B01]], [[B00]]
+; CHECK-NEXT: [[B03:%.*]] = fmul fast double [[L2]], [[C5]]
+; CHECK-NEXT: [[B:%.*]] = fadd fast double [[B02]], [[B03]]
+; CHECK-NEXT: [[D00:%.*]] = fmul fast double [[L0]], [[C6]]
+; CHECK-NEXT: [[D01:%.*]] = fmul fast double [[L1]], [[C7]]
+; CHECK-NEXT: [[D02:%.*]] = fadd fast double [[D01]], [[D00]]
+; CHECK-NEXT: [[D03:%.*]] = fmul fast double [[L2]], [[C8]]
+; CHECK-NEXT: [[D:%.*]] = fadd fast double [[D02]], [[D03]]
+; CHECK-NEXT: [[SA:%.*]] = fmul double [[A]], [[A]]
+; CHECK-NEXT: [[T1:%.*]] = fadd fast double [[SA]], [[ACC]]
+; CHECK-NEXT: [[SB:%.*]] = fmul double [[B]], [[B]]
+; CHECK-NEXT: [[T2:%.*]] = fadd fast double [[T1]], [[SB]]
+; CHECK-NEXT: [[SD:%.*]] = fmul double [[D]], [[D]]
+; CHECK-NEXT: [[TMP23]] = fadd fast double [[T2]], [[SD]]
; CHECK-NEXT: store double [[TMP23]], ptr [[O]], align 8
; CHECK-NEXT: [[IV_NEXT]] = add nuw nsw i64 [[IV]], 1
; CHECK-NEXT: [[CMP:%.*]] = icmp eq i64 [[IV]], [[N]]
@@ -512,15 +504,14 @@ define double @two_exit_phis(ptr %p, i64 %n, i1 %c) {
; CHECK-NEXT: br label %[[LOOP:.*]]
; CHECK: [[LOOP]]:
; CHECK-NEXT: [[IV:%.*]] = phi i64 [ 1, %[[ENTRY]] ], [ [[IV_NEXT:%.*]], %[[LOOP]] ]
-; CHECK-NEXT: [[SD:%.*]] = phi double [ 0.000000e+00, %[[ENTRY]] ], [ [[SUM:%.*]], %[[LOOP]] ]
+; CHECK-NEXT: [[ACC:%.*]] = phi double [ 0.000000e+00, %[[ENTRY]] ], [ [[SUM1:%.*]], %[[LOOP]] ]
; CHECK-NEXT: [[IV3:%.*]] = mul nuw i64 [[IV]], 3
; CHECK-NEXT: [[P0:%.*]] = getelementptr double, ptr [[P]], i64 [[IV3]]
+; CHECK-NEXT: [[L0:%.*]] = load double, ptr [[P0]], align 8
; CHECK-NEXT: [[P1:%.*]] = getelementptr double, ptr [[P0]], i64 1
+; CHECK-NEXT: [[L1:%.*]] = load double, ptr [[P1]], align 8
; CHECK-NEXT: [[P2:%.*]] = getelementptr double, ptr [[P0]], i64 2
; CHECK-NEXT: [[L2:%.*]] = load double, ptr [[P2]], align 8
-; CHECK-NEXT: [[C8:%.*]] = load double, ptr getelementptr (double, ptr @cst, i64 8), align 8
-; CHECK-NEXT: [[L0:%.*]] = load double, ptr [[P0]], align 8
-; CHECK-NEXT: [[L1:%.*]] = load double, ptr [[P1]], align 8
; CHECK-NEXT: [[C0:%.*]] = load double, ptr @cst, align 8
; CHECK-NEXT: [[C1:%.*]] = load double, ptr getelementptr (double, ptr @cst, i64 1), align 8
; CHECK-NEXT: [[C2:%.*]] = load double, ptr getelementptr (double, ptr @cst, i64 2), align 8
@@ -529,37 +520,34 @@ define double @two_exit_phis(ptr %p, i64 %n, i1 %c) {
; CHECK-NEXT: [[C5:%.*]] = load double, ptr getelementptr (double, ptr @cst, i64 5), align 8
; CHECK-NEXT: [[C6:%.*]] = load double, ptr getelementptr (double, ptr @cst, i64 6), align 8
; CHECK-NEXT: [[C7:%.*]] = load double, ptr getelementptr (double, ptr @cst, i64 7), align 8
-; CHECK-NEXT: [[TMP0:%.*]] = insertelement <4 x double> <double poison, double -0.000000e+00, double poison, double poison>, double [[L0]], i64 0
-; CHECK-NEXT: [[TMP1:%.*]] = shufflevector <4 x double> [[TMP0]], <4 x double> poison, <4 x i32> <i32 0, i32 0, i32 0, i32 1>
-; CHECK-NEXT: [[TMP2:%.*]] = insertelement <4 x double> <double poison, double poison, double poison, double 1.000000e+00>, double [[C0]], i64 0
-; CHECK-NEXT: [[TMP3:%.*]] = insertelement <4 x double> [[TMP2]], double [[C3]], i64 1
-; CHECK-NEXT: [[TMP4:%.*]] = insertelement <4 x double> [[TMP3]], double [[C6]], i64 2
-; CHECK-NEXT: [[TMP5:%.*]] = fmul fast <4 x double> [[TMP1]], [[TMP4]]
-; CHECK-NEXT: [[TMP6:%.*]] = insertelement <4 x double> <double poison, double 0.000000e+00, double poison, double poison>, double [[L1]], i64 0
-; CHECK-NEXT: [[TMP7:%.*]] = shufflevector <4 x double> [[TMP6]], <4 x double> poison, <4 x i32> <i32 0, i32 0, i32 0, i32 1>
-; CHECK-NEXT: [[TMP8:%.*]] = insertelement <4 x double> <double poison, double poison, double poison, double 1.000000e+00>, double [[C1]], i64 0
-; CHECK-NEXT: [[TMP9:%.*]] = insertelement <4 x double> [[TMP8]], double [[C4]], i64 1
-; CHECK-NEXT: [[TMP10:%.*]] = insertelement <4 x double> [[TMP9]], double [[C7]], i64 2
-; CHECK-NEXT: [[TMP11:%.*]] = fmul fast <4 x double> [[TMP7]], [[TMP10]]
-; CHECK-NEXT: [[TMP12:%.*]] = fadd fast <4 x double> [[TMP11]], [[TMP5]]
-; CHECK-NEXT: [[TMP13:%.*]] = insertelement <4 x double> <double poison, double -0.000000e+00, double poison, double poison>, double [[L2]], i64 0
-; CHECK-NEXT: [[TMP14:%.*]] = shufflevector <4 x double> [[TMP13]], <4 x double> poison, <4 x i32> <i32 0, i32 0, i32 0, i32 1>
-; CHECK-NEXT: [[TMP15:%.*]] = insertelement <4 x double> <double poison, double poison, double poison, double 1.000000e+00>, double [[C2]], i64 0
-; CHECK-NEXT: [[TMP16:%.*]] = insertelement <4 x double> [[TMP15]], double [[C5]], i64 1
-; CHECK-NEXT: [[TMP17:%.*]] = insertelement <4 x double> [[TMP16]], double [[C8]], i64 2
-; CHECK-NEXT: [[TMP18:%.*]] = fmul fast <4 x double> [[TMP14]], [[TMP17]]
-; CHECK-NEXT: [[TMP19:%.*]] = fadd fast <4 x double> [[TMP12]], [[TMP18]]
-; CHECK-NEXT: [[TMP20:%.*]] = shufflevector <4 x double> [[TMP19]], <4 x double> <double poison, double poison, double poison, double 1.000000e+00>, <4 x i32> <i32 0, i32 1, i32 poison, i32 7>
-; CHECK-NEXT: [[TMP21:%.*]] = shufflevector <4 x double> [[TMP20]], <4 x double> [[TMP19]], <4 x i32> <i32 0, i32 1, i32 6, i32 3>
-; CHECK-NEXT: [[TMP22:%.*]] = fmul <4 x double> [[TMP19]], [[TMP21]]
-; CHECK-NEXT: [[T2:%.*]] = call fast double @llvm.vector.reduce.fadd.v4f64(double 0.000000e+00, <4 x double> [[TMP22]])
-; CHECK-NEXT: [[SUM]] = fadd fast double [[T2]], [[SD]]
+; CHECK-NEXT: [[C8:%.*]] = load double, ptr getelementptr (double, ptr @cst, i64 8), align 8
+; CHECK-NEXT: [[SD:%.*]] = fmul fast double [[L0]], [[C0]]
+; CHECK-NEXT: [[T2:%.*]] = fmul fast double [[L1]], [[C1]]
+; CHECK-NEXT: [[SUM:%.*]] = fadd fast double [[T2]], [[SD]]
+; CHECK-NEXT: [[A03:%.*]] = fmul fast double [[L2]], [[C2]]
+; CHECK-NEXT: [[A:%.*]] = fadd fast double [[SUM]], [[A03]]
+; CHECK-NEXT: [[B00:%.*]] = fmul fast double [[L0]], [[C3]]
+; CHECK-NEXT: [[B01:%.*]] = fmul fast double [[L1]], [[C4]]
+; CHECK-NEXT: [[B02:%.*]] = fadd fast double [[B01]], [[B00]]
+; CHECK-NEXT: [[B03:%.*]] = fmul fast double [[L2]], [[C5]]
+; CHECK-NEXT: [[B:%.*]] = fadd fast double [[B02]], [[B03]]
+; CHECK-NEXT: [[D00:%.*]] = fmul fast double [[L0]], [[C6]]
+; CHECK-NEXT: [[D01:%.*]] = fmul fast double [[L1]], [[C7]]
+; CHECK-NEXT: [[D02:%.*]] = fadd fast double [[D01]], [[D00]]
+; CHECK-NEXT: [[D03:%.*]] = fmul fast double [[L2]], [[C8]]
+; CHECK-NEXT: [[D:%.*]] = fadd fast double [[D02]], [[D03]]
+; CHECK-NEXT: [[SA:%.*]] = fmul double [[A]], [[A]]
+; CHECK-NEXT: [[T1:%.*]] = fadd fast double [[SA]], [[ACC]]
+; CHECK-NEXT: [[SB:%.*]] = fmul double [[B]], [[B]]
+; CHECK-NEXT: [[T3:%.*]] = fadd fast double [[T1]], [[SB]]
+; CHECK-NEXT: [[SD1:%.*]] = fmul double [[D]], [[D]]
+; CHECK-NEXT: [[SUM1]] = fadd fast double [[T3]], [[SD1]]
; CHECK-NEXT: [[IV_NEXT]] = add nuw nsw i64 [[IV]], 1
; CHECK-NEXT: [[CMP:%.*]] = icmp eq i64 [[IV]], [[N]]
; CHECK-NEXT: br i1 [[CMP]], label %[[EXIT:.*]], label %[[LOOP]]
; CHECK: [[EXIT]]:
-; CHECK-NEXT: [[TMP23:%.*]] = phi double [ [[SUM]], %[[LOOP]] ]
-; CHECK-NEXT: [[TMP24:%.*]] = phi double [ [[SUM]], %[[LOOP]] ]
+; CHECK-NEXT: [[TMP23:%.*]] = phi double [ [[SUM1]], %[[LOOP]] ]
+; CHECK-NEXT: [[TMP24:%.*]] = phi double [ [[SUM1]], %[[LOOP]] ]
; CHECK-NEXT: [[R:%.*]] = fadd double [[TMP23]], [[TMP24]]
; CHECK-NEXT: ret double [[R]]
;
diff --git a/llvm/test/Transforms/SLPVectorizer/AArch64/reduction-root-multi-use-fma.ll b/llvm/test/Transforms/SLPVectorizer/AArch64/reduction-root-multi-use-fma.ll
index 0b998f1efd678..cac4abf971d85 100644
--- a/llvm/test/Transforms/SLPVectorizer/AArch64/reduction-root-multi-use-fma.ll
+++ b/llvm/test/Transforms/SLPVectorizer/AArch64/reduction-root-multi-use-fma.ll
@@ -8,14 +8,15 @@
define double @fmul_reduction_root_multi_use(ptr %p, double %a, double %b) {
; CHECK-LABEL: define double @fmul_reduction_root_multi_use(
; CHECK-SAME: ptr [[P:%.*]], double [[A:%.*]], double [[B:%.*]]) #[[ATTR0:[0-9]+]] {
-; CHECK-NEXT: [[TMP1:%.*]] = load <2 x double>, ptr [[P]], align 8
-; CHECK-NEXT: [[TMP2:%.*]] = insertelement <2 x double> poison, double [[A]], i64 0
-; CHECK-NEXT: [[TMP3:%.*]] = insertelement <2 x double> [[TMP2]], double [[B]], i64 1
-; CHECK-NEXT: [[TMP4:%.*]] = fmul fast <2 x double> [[TMP1]], [[TMP3]]
-; CHECK-NEXT: [[R:%.*]] = call fast double @llvm.vector.reduce.fadd.v2f64(double 0.000000e+00, <2 x double> [[TMP4]])
+; CHECK-NEXT: [[P1:%.*]] = getelementptr inbounds double, ptr [[P]], i64 1
+; CHECK-NEXT: [[R:%.*]] = load double, ptr [[P]], align 8
+; CHECK-NEXT: [[L1:%.*]] = load double, ptr [[P1]], align 8
; CHECK-NEXT: [[U0:%.*]] = fmul fast double [[R]], [[A]]
-; CHECK-NEXT: [[U1:%.*]] = fmul fast double [[R]], [[B]]
-; CHECK-NEXT: [[RES:%.*]] = fdiv fast double [[U0]], [[U1]]
+; CHECK-NEXT: [[M1:%.*]] = fmul fast double [[L1]], [[B]]
+; CHECK-NEXT: [[R1:%.*]] = fadd fast double [[U0]], [[M1]]
+; CHECK-NEXT: [[U2:%.*]] = fmul fast double [[R1]], [[A]]
+; CHECK-NEXT: [[U1:%.*]] = fmul fast double [[R1]], [[B]]
+; CHECK-NEXT: [[RES:%.*]] = fdiv fast double [[U2]], [[U1]]
; CHECK-NEXT: ret double [[RES]]
;
%p1 = getelementptr inbounds double, ptr %p, i64 1
diff --git a/llvm/test/Transforms/SLPVectorizer/AArch64/vec3-reorder-reshuffle.ll b/llvm/test/Transforms/SLPVectorizer/AArch64/vec3-reorder-reshuffle.ll
index 5dbd7079be9ce..0d333ac022d1a 100644
--- a/llvm/test/Transforms/SLPVectorizer/AArch64/vec3-reorder-reshuffle.ll
+++ b/llvm/test/Transforms/SLPVectorizer/AArch64/vec3-reorder-reshuffle.ll
@@ -79,7 +79,9 @@ define void @extract_mask(ptr %object, double %conv503, double %conv520) {
; CHECK-NEXT: [[TMP6:%.*]] = select <2 x i1> [[TMP5]], <2 x double> [[TMP3]], <2 x double> <double 0.000000e+00, double -2.000000e+10>
; CHECK-NEXT: [[TMP7:%.*]] = fsub <2 x double> zeroinitializer, [[TMP6]]
; CHECK-NEXT: [[TMP8:%.*]] = fptrunc <2 x double> [[TMP7]] to <2 x float>
-; CHECK-NEXT: [[MUL646:%.*]] = call reassoc float @llvm.vector.reduce.fmul.v2f32(float 1.000000e+00, <2 x float> [[TMP8]])
+; CHECK-NEXT: [[TMP9:%.*]] = extractelement <2 x float> [[TMP8]], i64 0
+; CHECK-NEXT: [[TMP10:%.*]] = extractelement <2 x float> [[TMP8]], i64 1
+; CHECK-NEXT: [[MUL646:%.*]] = fmul float [[TMP9]], [[TMP10]]
; CHECK-NEXT: [[CMP663:%.*]] = fcmp olt float [[MUL646]], 0.000000e+00
; CHECK-NEXT: br i1 [[CMP663]], label [[IF_THEN665:%.*]], label [[IF_END668:%.*]]
; CHECK: if.then665:
diff --git a/llvm/test/Transforms/SLPVectorizer/AMDGPU/cross-block-fmul-fadd.ll b/llvm/test/Transforms/SLPVectorizer/AMDGPU/cross-block-fmul-fadd.ll
index 8af404e1ce3c8..8fdc1d71af190 100644
--- a/llvm/test/Transforms/SLPVectorizer/AMDGPU/cross-block-fmul-fadd.ll
+++ b/llvm/test/Transforms/SLPVectorizer/AMDGPU/cross-block-fmul-fadd.ll
@@ -75,19 +75,22 @@ define void @cross_block_fmul_rhs(ptr addrspace(1) %p, ptr addrspace(1) %q, ptr
; CHECK-NEXT: [[TID:%.*]] = call i32 @llvm.amdgcn.workitem.id.x()
; CHECK-NEXT: [[I0:%.*]] = shl i32 [[TID]], 1
; CHECK-NEXT: [[P0:%.*]] = getelementptr inbounds float, ptr addrspace(1) [[P]], i32 [[I0]]
+; CHECK-NEXT: [[P1:%.*]] = getelementptr inbounds float, ptr addrspace(1) [[P0]], i32 1
; CHECK-NEXT: [[Q0:%.*]] = getelementptr inbounds float, ptr addrspace(1) [[Q]], i32 [[I0]]
-; CHECK-NEXT: [[TMP0:%.*]] = load <2 x float>, ptr addrspace(1) [[P0]], align 4
-; CHECK-NEXT: [[TMP1:%.*]] = load <2 x float>, ptr addrspace(1) [[Q0]], align 4
-; CHECK-NEXT: [[TMP2:%.*]] = fmul contract <2 x float> [[TMP0]], [[TMP1]]
+; CHECK-NEXT: [[Q1:%.*]] = getelementptr inbounds float, ptr addrspace(1) [[Q0]], i32 1
+; CHECK-NEXT: [[A0:%.*]] = load float, ptr addrspace(1) [[P0]], align 4
+; CHECK-NEXT: [[A1:%.*]] = load float, ptr addrspace(1) [[P1]], align 4
+; CHECK-NEXT: [[B0:%.*]] = load float, ptr addrspace(1) [[Q0]], align 4
+; CHECK-NEXT: [[B1:%.*]] = load float, ptr addrspace(1) [[Q1]], align 4
+; CHECK-NEXT: [[TMP3:%.*]] = fmul contract float [[A0]], [[B0]]
+; CHECK-NEXT: [[TMP4:%.*]] = fmul contract float [[A1]], [[B1]]
; CHECK-NEXT: br i1 [[C]], label %[[T:.*]], label %[[F:.*]]
; CHECK: [[T]]:
-; CHECK-NEXT: [[TMP3:%.*]] = extractelement <2 x float> [[TMP2]], i64 0
; CHECK-NEXT: [[S0:%.*]] = fadd contract float [[X]], [[TMP3]]
; CHECK-NEXT: [[RT:%.*]] = getelementptr inbounds float, ptr addrspace(1) [[R]], i32 [[TID]]
; CHECK-NEXT: store float [[S0]], ptr addrspace(1) [[RT]], align 4
; CHECK-NEXT: ret void
; CHECK: [[F]]:
-; CHECK-NEXT: [[TMP4:%.*]] = extractelement <2 x float> [[TMP2]], i64 1
; CHECK-NEXT: [[S1:%.*]] = fadd contract float [[Y]], [[TMP4]]
; CHECK-NEXT: [[RF:%.*]] = getelementptr inbounds float, ptr addrspace(1) [[R2]], i32 [[TID]]
; CHECK-NEXT: store float [[S1]], ptr addrspace(1) [[RF]], align 4
diff --git a/llvm/test/Transforms/SLPVectorizer/NVPTX/ordered-reduction-fma-fusion.ll b/llvm/test/Transforms/SLPVectorizer/NVPTX/ordered-reduction-fma-fusion.ll
index a553fe126ee17..bad672c7743f6 100644
--- a/llvm/test/Transforms/SLPVectorizer/NVPTX/ordered-reduction-fma-fusion.ll
+++ b/llvm/test/Transforms/SLPVectorizer/NVPTX/ordered-reduction-fma-fusion.ll
@@ -9,10 +9,14 @@
define float @dot_contract(float %x) {
; CHECK-LABEL: @dot_contract(
-; CHECK-NEXT: [[TMP1:%.*]] = insertelement <4 x float> poison, float [[X:%.*]], i64 0
-; CHECK-NEXT: [[TMP2:%.*]] = shufflevector <4 x float> [[TMP1]], <4 x float> poison, <4 x i32> zeroinitializer
-; CHECK-NEXT: [[TMP3:%.*]] = fmul contract <4 x float> <float 7.000000e+00, float 3.000000e+00, float 5.000000e+00, float 9.000000e+00>, [[TMP2]]
-; CHECK-NEXT: [[TMP4:%.*]] = call contract float @llvm.vector.reduce.fadd.v4f32(float [[X]], <4 x float> [[TMP3]])
+; CHECK-NEXT: [[M0:%.*]] = fmul contract float 7.000000e+00, [[X:%.*]]
+; CHECK-NEXT: [[A0:%.*]] = fadd contract float [[M0]], [[X]]
+; CHECK-NEXT: [[M1:%.*]] = fmul contract float 3.000000e+00, [[X]]
+; CHECK-NEXT: [[A1:%.*]] = fadd contract float [[M1]], [[A0]]
+; CHECK-NEXT: [[M2:%.*]] = fmul contract float 5.000000e+00, [[X]]
+; CHECK-NEXT: [[A2:%.*]] = fadd contract float [[M2]], [[A1]]
+; CHECK-NEXT: [[M3:%.*]] = fmul contract float 9.000000e+00, [[X]]
+; CHECK-NEXT: [[TMP4:%.*]] = fadd contract float [[M3]], [[A2]]
; CHECK-NEXT: ret float [[TMP4]]
;
%m0 = fmul contract float 7.000000e+00, %x
diff --git a/llvm/test/Transforms/SLPVectorizer/X86/fmul-fused-into-scalar-fadd.ll b/llvm/test/Transforms/SLPVectorizer/X86/fmul-fused-into-scalar-fadd.ll
index edcfa0e997233..ab5bde661185b 100644
--- a/llvm/test/Transforms/SLPVectorizer/X86/fmul-fused-into-scalar-fadd.ll
+++ b/llvm/test/Transforms/SLPVectorizer/X86/fmul-fused-into-scalar-fadd.ll
@@ -12,19 +12,16 @@ define double @fmul_lhs_of_scalar_fadd(ptr %p, ptr %q, double %x, double %y, i1
; CHECK-LABEL: define double @fmul_lhs_of_scalar_fadd(
; CHECK-SAME: ptr [[P:%.*]], ptr [[Q:%.*]], double [[X:%.*]], double [[Y:%.*]], i1 [[C:%.*]]) #[[ATTR0:[0-9]+]] {
; CHECK-NEXT: [[ENTRY:.*:]]
-; CHECK-NEXT: [[A0:%.*]] = load double, ptr [[P]], align 8
-; CHECK-NEXT: [[P1:%.*]] = getelementptr inbounds double, ptr [[P]], i64 1
-; CHECK-NEXT: [[A1:%.*]] = load double, ptr [[P1]], align 8
-; CHECK-NEXT: [[B0:%.*]] = load double, ptr [[Q]], align 8
-; CHECK-NEXT: [[Q1:%.*]] = getelementptr inbounds double, ptr [[Q]], i64 1
-; CHECK-NEXT: [[B1:%.*]] = load double, ptr [[Q1]], align 8
-; CHECK-NEXT: [[M0:%.*]] = fmul contract double [[A0]], [[B0]]
-; CHECK-NEXT: [[M1:%.*]] = fmul contract double [[A1]], [[B1]]
+; CHECK-NEXT: [[TMP0:%.*]] = load <2 x double>, ptr [[P]], align 8
+; CHECK-NEXT: [[TMP1:%.*]] = load <2 x double>, ptr [[Q]], align 8
+; CHECK-NEXT: [[TMP2:%.*]] = fmul contract <2 x double> [[TMP0]], [[TMP1]]
; CHECK-NEXT: br i1 [[C]], label %[[T:.*]], label %[[F:.*]]
; CHECK: [[T]]:
+; CHECK-NEXT: [[M0:%.*]] = extractelement <2 x double> [[TMP2]], i64 0
; CHECK-NEXT: [[S0:%.*]] = fadd contract double [[M0]], [[X]]
; CHECK-NEXT: ret double [[S0]]
; CHECK: [[F]]:
+; CHECK-NEXT: [[M1:%.*]] = extractelement <2 x double> [[TMP2]], i64 1
; CHECK-NEXT: [[S1:%.*]] = fadd contract double [[M1]], [[Y]]
; CHECK-NEXT: ret double [[S1]]
;
@@ -126,12 +123,15 @@ define double @fmul_rhs_of_scalar_fadd_same_block(ptr %p, ptr %q, double %x, dou
; CHECK-LABEL: define double @fmul_rhs_of_scalar_fadd_same_block(
; CHECK-SAME: ptr [[P:%.*]], ptr [[Q:%.*]], double [[X:%.*]], double [[Y:%.*]], i1 [[C:%.*]]) #[[ATTR0]] {
; CHECK-NEXT: [[ENTRY:.*:]]
-; CHECK-NEXT: [[TMP0:%.*]] = load <2 x double>, ptr [[P]], align 8
-; CHECK-NEXT: [[TMP1:%.*]] = load <2 x double>, ptr [[Q]], align 8
-; CHECK-NEXT: [[TMP2:%.*]] = fmul contract <2 x double> [[TMP0]], [[TMP1]]
-; CHECK-NEXT: [[M0:%.*]] = extractelement <2 x double> [[TMP2]], i64 0
+; CHECK-NEXT: [[A0:%.*]] = load double, ptr [[P]], align 8
+; CHECK-NEXT: [[P1:%.*]] = getelementptr inbounds double, ptr [[P]], i64 1
+; CHECK-NEXT: [[A1:%.*]] = load double, ptr [[P1]], align 8
+; CHECK-NEXT: [[B0:%.*]] = load double, ptr [[Q]], align 8
+; CHECK-NEXT: [[Q1:%.*]] = getelementptr inbounds double, ptr [[Q]], i64 1
+; CHECK-NEXT: [[B1:%.*]] = load double, ptr [[Q1]], align 8
+; CHECK-NEXT: [[M0:%.*]] = fmul contract double [[A0]], [[B0]]
+; CHECK-NEXT: [[M1:%.*]] = fmul contract double [[A1]], [[B1]]
; CHECK-NEXT: [[S0:%.*]] = fadd contract double [[X]], [[M0]]
-; CHECK-NEXT: [[M1:%.*]] = extractelement <2 x double> [[TMP2]], i64 1
; CHECK-NEXT: [[S1:%.*]] = fadd contract double [[Y]], [[M1]]
; CHECK-NEXT: [[R:%.*]] = select i1 [[C]], double [[S0]], double [[S1]]
; CHECK-NEXT: ret double [[R]]
diff --git a/llvm/test/Transforms/SLPVectorizer/X86/fsub-fmul-rhs-combine.ll b/llvm/test/Transforms/SLPVectorizer/X86/fsub-fmul-rhs-combine.ll
index b766d92bd13fb..7ceb8784ef1e5 100644
--- a/llvm/test/Transforms/SLPVectorizer/X86/fsub-fmul-rhs-combine.ll
+++ b/llvm/test/Transforms/SLPVectorizer/X86/fsub-fmul-rhs-combine.ll
@@ -50,14 +50,16 @@ define double @fsub_fmul_rhs_gathered_c(ptr %a, ptr %b, double %c0, double %c1)
; CHECK-LABEL: define double @fsub_fmul_rhs_gathered_c(
; CHECK-SAME: ptr [[A:%.*]], ptr [[B:%.*]], double [[C0:%.*]], double [[C1:%.*]]) #[[ATTR0]] {
; CHECK-NEXT: [[ENTRY:.*:]]
-; CHECK-NEXT: [[TMP0:%.*]] = load <2 x double>, ptr [[A]], align 8
-; CHECK-NEXT: [[TMP1:%.*]] = load <2 x double>, ptr [[B]], align 8
-; CHECK-NEXT: [[TMP2:%.*]] = fmul contract <2 x double> [[TMP0]], [[TMP1]]
-; CHECK-NEXT: [[TMP3:%.*]] = insertelement <2 x double> poison, double [[C0]], i64 0
-; CHECK-NEXT: [[TMP4:%.*]] = insertelement <2 x double> [[TMP3]], double [[C1]], i64 1
-; CHECK-NEXT: [[TMP5:%.*]] = fsub contract <2 x double> [[TMP4]], [[TMP2]]
-; CHECK-NEXT: [[S0:%.*]] = extractelement <2 x double> [[TMP5]], i64 0
-; CHECK-NEXT: [[S1:%.*]] = extractelement <2 x double> [[TMP5]], i64 1
+; CHECK-NEXT: [[A0:%.*]] = load double, ptr [[A]], align 8
+; CHECK-NEXT: [[A1P:%.*]] = getelementptr inbounds double, ptr [[A]], i64 1
+; CHECK-NEXT: [[A1:%.*]] = load double, ptr [[A1P]], align 8
+; CHECK-NEXT: [[B0:%.*]] = load double, ptr [[B]], align 8
+; CHECK-NEXT: [[B1P:%.*]] = getelementptr inbounds double, ptr [[B]], i64 1
+; CHECK-NEXT: [[B1:%.*]] = load double, ptr [[B1P]], align 8
+; CHECK-NEXT: [[M0:%.*]] = fmul contract double [[A0]], [[B0]]
+; CHECK-NEXT: [[M1:%.*]] = fmul contract double [[A1]], [[B1]]
+; CHECK-NEXT: [[S0:%.*]] = fsub contract double [[C0]], [[M0]]
+; CHECK-NEXT: [[S1:%.*]] = fsub contract double [[C1]], [[M1]]
; CHECK-NEXT: [[R:%.*]] = fmul double [[S0]], [[S1]]
; CHECK-NEXT: ret double [[R]]
;
diff --git a/llvm/test/Transforms/SLPVectorizer/X86/slp-fma-loss-ordered.ll b/llvm/test/Transforms/SLPVectorizer/X86/slp-fma-loss-ordered.ll
index 01e953d7c250c..1304f5353c968 100644
--- a/llvm/test/Transforms/SLPVectorizer/X86/slp-fma-loss-ordered.ll
+++ b/llvm/test/Transforms/SLPVectorizer/X86/slp-fma-loss-ordered.ll
@@ -19,14 +19,11 @@ define double @mul_fun() {
; FMA-NEXT: [[CVT0:%.*]] = uitofp i16 3 to double
; FMA-NEXT: [[MUL0:%.*]] = fmul contract double 7.000000e+00, [[CVT0]]
; FMA-NEXT: [[ADD0:%.*]] = fadd contract double [[MUL0]], [[CVT0]]
-; FMA-NEXT: [[TMP1:%.*]] = insertelement <4 x double> poison, double [[CVT0]], i64 0
-; FMA-NEXT: [[TMP2:%.*]] = shufflevector <4 x double> [[TMP1]], <4 x double> poison, <4 x i32> zeroinitializer
-; FMA-NEXT: [[TMP3:%.*]] = fmul contract <4 x double> [[TMP2]], <double -4.300000e+01, double 2.200000e-02, double 9.500000e+00, double 1.000000e+00>
-; FMA-NEXT: [[MUL1:%.*]] = extractelement <4 x double> [[TMP3]], i64 0
+; FMA-NEXT: [[MUL1:%.*]] = fmul contract double -4.300000e+01, [[CVT0]]
; FMA-NEXT: [[ADD1:%.*]] = fadd contract double [[MUL1]], [[ADD0]]
-; FMA-NEXT: [[MUL2:%.*]] = extractelement <4 x double> [[TMP3]], i64 1
+; FMA-NEXT: [[MUL2:%.*]] = fmul contract double 2.200000e-02, [[CVT0]]
; FMA-NEXT: [[ADD2:%.*]] = fadd contract double [[MUL2]], [[ADD1]]
-; FMA-NEXT: [[MUL3:%.*]] = extractelement <4 x double> [[TMP3]], i64 2
+; FMA-NEXT: [[MUL3:%.*]] = fmul contract double 9.500000e+00, [[CVT0]]
; FMA-NEXT: [[ADD3:%.*]] = fadd contract double [[MUL3]], [[ADD2]]
; FMA-NEXT: ret double [[ADD3]]
;
@@ -59,15 +56,12 @@ define double @mul_fun_multiuse(ptr %dst) {
; FMA-NEXT: [[CVT0:%.*]] = uitofp i16 3 to double
; FMA-NEXT: [[TMP4:%.*]] = fmul contract double 7.000000e+00, [[CVT0]]
; FMA-NEXT: [[ADD0:%.*]] = fadd contract double [[TMP4]], [[CVT0]]
-; FMA-NEXT: [[TMP1:%.*]] = insertelement <4 x double> poison, double [[CVT0]], i64 0
-; FMA-NEXT: [[TMP2:%.*]] = shufflevector <4 x double> [[TMP1]], <4 x double> poison, <4 x i32> zeroinitializer
-; FMA-NEXT: [[TMP3:%.*]] = fmul contract <4 x double> [[TMP2]], <double -4.300000e+01, double 2.200000e-02, double 9.500000e+00, double 1.000000e+00>
-; FMA-NEXT: [[TMP5:%.*]] = extractelement <4 x double> [[TMP3]], i64 0
+; FMA-NEXT: [[TMP5:%.*]] = fmul contract double -4.300000e+01, [[CVT0]]
; FMA-NEXT: store double [[TMP5]], ptr [[DST:%.*]], align 8
; FMA-NEXT: [[ADD1:%.*]] = fadd contract double [[TMP5]], [[ADD0]]
-; FMA-NEXT: [[MUL2:%.*]] = extractelement <4 x double> [[TMP3]], i64 1
+; FMA-NEXT: [[MUL2:%.*]] = fmul contract double 2.200000e-02, [[CVT0]]
; FMA-NEXT: [[ADD2:%.*]] = fadd contract double [[MUL2]], [[ADD1]]
-; FMA-NEXT: [[MUL3:%.*]] = extractelement <4 x double> [[TMP3]], i64 2
+; FMA-NEXT: [[MUL3:%.*]] = fmul contract double 9.500000e+00, [[CVT0]]
; FMA-NEXT: [[ADD3:%.*]] = fadd contract double [[MUL3]], [[ADD2]]
; FMA-NEXT: ret double [[ADD3]]
;
diff --git a/llvm/test/Transforms/SLPVectorizer/consecutive-access.ll b/llvm/test/Transforms/SLPVectorizer/consecutive-access.ll
index 8347d5c8094b1..27e88a5b6d73e 100644
--- a/llvm/test/Transforms/SLPVectorizer/consecutive-access.ll
+++ b/llvm/test/Transforms/SLPVectorizer/consecutive-access.ll
@@ -1,6 +1,6 @@
; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
-; RUN: %if x86-registered-target %{ opt < %s -passes=slp-vectorizer -S -mtriple=x86_64-apple-macosx10.9.0 | FileCheck %s --check-prefixes=CHECK,X86 %}
-; RUN: %if aarch64-registered-target %{ opt < %s -passes=slp-vectorizer -S -mtriple=aarch64-unknown-linux-gnu | FileCheck %s --check-prefixes=CHECK,AARCH64 %}
+; RUN: %if x86-registered-target %{ opt < %s -passes=slp-vectorizer -S -mtriple=x86_64-apple-macosx10.9.0 | FileCheck %s %}
+; RUN: %if aarch64-registered-target %{ opt < %s -passes=slp-vectorizer -S -mtriple=aarch64-unknown-linux-gnu | FileCheck %s %}
@A = common global [2000 x double] zeroinitializer, align 16
@B = common global [2000 x double] zeroinitializer, align 16
@@ -439,45 +439,26 @@ for.end: ; preds = %for.cond.for.end_cr
; Make sure we are able to vectorize this from now on:
;
define double @bar(ptr nocapture readonly %a, i32 %n) local_unnamed_addr {
-; X86-LABEL: @bar(
-; X86-NEXT: entry:
-; X86-NEXT: [[CMP15:%.*]] = icmp eq i32 [[N:%.*]], 0
-; X86-NEXT: br i1 [[CMP15]], label [[FOR_COND_CLEANUP:%.*]], label [[FOR_BODY:%.*]]
-; X86: for.cond.cleanup:
-; X86-NEXT: [[TMP0:%.*]] = phi <2 x double> [ zeroinitializer, [[ENTRY:%.*]] ], [ [[TMP5:%.*]], [[FOR_BODY]] ]
-; X86-NEXT: [[TMP1:%.*]] = extractelement <2 x double> [[TMP0]], i64 0
-; X86-NEXT: [[TMP2:%.*]] = extractelement <2 x double> [[TMP0]], i64 1
-; X86-NEXT: [[MUL:%.*]] = fmul double [[TMP1]], [[TMP2]]
-; X86-NEXT: ret double [[MUL]]
-; X86: for.body:
-; X86-NEXT: [[I_018:%.*]] = phi i32 [ [[ADD5:%.*]], [[FOR_BODY]] ], [ 0, [[ENTRY]] ]
-; X86-NEXT: [[TMP3:%.*]] = phi <2 x double> [ [[TMP5]], [[FOR_BODY]] ], [ zeroinitializer, [[ENTRY]] ]
-; X86-NEXT: [[IDXPROM:%.*]] = zext i32 [[I_018]] to i64
-; X86-NEXT: [[ARRAYIDX:%.*]] = getelementptr inbounds double, ptr [[A:%.*]], i64 [[IDXPROM]]
-; X86-NEXT: [[TMP4:%.*]] = load <2 x double>, ptr [[ARRAYIDX]], align 8
-; X86-NEXT: [[TMP5]] = fadd <2 x double> [[TMP3]], [[TMP4]]
-; X86-NEXT: [[ADD5]] = add i32 [[I_018]], 2
-; X86-NEXT: [[CMP:%.*]] = icmp ult i32 [[ADD5]], [[N]]
-; X86-NEXT: br i1 [[CMP]], label [[FOR_BODY]], label [[FOR_COND_CLEANUP]]
-;
-; AARCH64-LABEL: @bar(
-; AARCH64-NEXT: entry:
-; AARCH64-NEXT: [[CMP15:%.*]] = icmp eq i32 [[N:%.*]], 0
-; AARCH64-NEXT: br i1 [[CMP15]], label [[FOR_COND_CLEANUP:%.*]], label [[FOR_BODY:%.*]]
-; AARCH64: for.cond.cleanup:
-; AARCH64-NEXT: [[TMP0:%.*]] = phi <2 x double> [ zeroinitializer, [[ENTRY:%.*]] ], [ [[TMP4:%.*]], [[FOR_BODY]] ]
-; AARCH64-NEXT: [[TMP1:%.*]] = call reassoc double @llvm.vector.reduce.fmul.v2f64(double 1.000000e+00, <2 x double> [[TMP0]])
-; AARCH64-NEXT: ret double [[TMP1]]
-; AARCH64: for.body:
-; AARCH64-NEXT: [[I_018:%.*]] = phi i32 [ [[ADD5:%.*]], [[FOR_BODY]] ], [ 0, [[ENTRY]] ]
-; AARCH64-NEXT: [[TMP2:%.*]] = phi <2 x double> [ [[TMP4]], [[FOR_BODY]] ], [ zeroinitializer, [[ENTRY]] ]
-; AARCH64-NEXT: [[IDXPROM:%.*]] = zext i32 [[I_018]] to i64
-; AARCH64-NEXT: [[ARRAYIDX:%.*]] = getelementptr inbounds double, ptr [[A:%.*]], i64 [[IDXPROM]]
-; AARCH64-NEXT: [[TMP3:%.*]] = load <2 x double>, ptr [[ARRAYIDX]], align 8
-; AARCH64-NEXT: [[TMP4]] = fadd <2 x double> [[TMP2]], [[TMP3]]
-; AARCH64-NEXT: [[ADD5]] = add i32 [[I_018]], 2
-; AARCH64-NEXT: [[CMP:%.*]] = icmp ult i32 [[ADD5]], [[N]]
-; AARCH64-NEXT: br i1 [[CMP]], label [[FOR_BODY]], label [[FOR_COND_CLEANUP]]
+; CHECK-LABEL: @bar(
+; CHECK-NEXT: entry:
+; CHECK-NEXT: [[CMP15:%.*]] = icmp eq i32 [[N:%.*]], 0
+; CHECK-NEXT: br i1 [[CMP15]], label [[FOR_COND_CLEANUP:%.*]], label [[FOR_BODY:%.*]]
+; CHECK: for.cond.cleanup:
+; CHECK-NEXT: [[TMP0:%.*]] = phi <2 x double> [ zeroinitializer, [[ENTRY:%.*]] ], [ [[TMP5:%.*]], [[FOR_BODY]] ]
+; CHECK-NEXT: [[TMP1:%.*]] = extractelement <2 x double> [[TMP0]], i64 0
+; CHECK-NEXT: [[TMP2:%.*]] = extractelement <2 x double> [[TMP0]], i64 1
+; CHECK-NEXT: [[MUL:%.*]] = fmul double [[TMP1]], [[TMP2]]
+; CHECK-NEXT: ret double [[MUL]]
+; CHECK: for.body:
+; CHECK-NEXT: [[I_018:%.*]] = phi i32 [ [[ADD5:%.*]], [[FOR_BODY]] ], [ 0, [[ENTRY]] ]
+; CHECK-NEXT: [[TMP3:%.*]] = phi <2 x double> [ [[TMP5]], [[FOR_BODY]] ], [ zeroinitializer, [[ENTRY]] ]
+; CHECK-NEXT: [[IDXPROM:%.*]] = zext i32 [[I_018]] to i64
+; CHECK-NEXT: [[ARRAYIDX:%.*]] = getelementptr inbounds double, ptr [[A:%.*]], i64 [[IDXPROM]]
+; CHECK-NEXT: [[TMP4:%.*]] = load <2 x double>, ptr [[ARRAYIDX]], align 8
+; CHECK-NEXT: [[TMP5]] = fadd <2 x double> [[TMP3]], [[TMP4]]
+; CHECK-NEXT: [[ADD5]] = add i32 [[I_018]], 2
+; CHECK-NEXT: [[CMP:%.*]] = icmp ult i32 [[ADD5]], [[N]]
+; CHECK-NEXT: br i1 [[CMP]], label [[FOR_BODY]], label [[FOR_COND_CLEANUP]]
;
entry:
%cmp15 = icmp eq i32 %n, 0
More information about the llvm-commits
mailing list