[llvm] [SLP]Fix fmul/fadd fusion costs and retry FMA seeds after all blocks (PR #226117)
Alexey Bataev via llvm-commits
llvm-commits at lists.llvm.org
Sat Sep 26 05:30:31 PDT 2026
https://github.com/alexey-bataev updated https://github.com/llvm/llvm-project/pull/226117
>From 61df043b55f38cdbacbf77a04b28dc5bda492925 Mon Sep 17 00:00:00 2001
From: Alexey Bataev <a.bataev at outlook.com>
Date: Thu, 24 Sep 2026 04:05:47 -0700
Subject: [PATCH] =?UTF-8?q?[=F0=9D=98=80=F0=9D=97=BD=F0=9D=97=BF]=20initia?=
=?UTF-8?q?l=20version?=
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit
Created using spr 1.3.7
---
.../llvm/Transforms/Vectorize/SLPVectorizer.h | 6 +-
.../Transforms/Vectorize/SLPVectorizer.cpp | 341 ++++++++++++++----
.../AArch64/reassociate-fma-pairs.ll | 34 +-
...tracts-folded-into-fmul-users-no-credit.ll | 11 +-
.../fadd-with-gathered-fmul-operands.ll | 22 +-
.../fma-candidates-after-store-chains.ll | 20 +-
.../fma-chain-no-alt-node-reduction.ll | 61 ++--
.../fmul-constant-lane-fmuladd-combine.ll | 48 +--
.../AArch64/loop-accumulator-reduction.ll | 208 +++++------
.../AArch64/reduction-root-multi-use-fma.ll | 15 +-
.../AArch64/vec3-reorder-reshuffle.ll | 4 +-
.../NVPTX/ordered-reduction-fma-fusion.ll | 12 +-
.../X86/fmul-fused-into-scalar-fadd.ll | 26 +-
.../X86/fsub-fmul-rhs-combine.ll | 18 +-
.../SLPVectorizer/X86/slp-fma-loss-ordered.ll | 18 +-
.../SLPVectorizer/consecutive-access.ll | 63 ++--
16 files changed, 549 insertions(+), 358 deletions(-)
diff --git a/llvm/include/llvm/Transforms/Vectorize/SLPVectorizer.h b/llvm/include/llvm/Transforms/Vectorize/SLPVectorizer.h
index 468cec660343c..3d8ab0d1b7573 100644
--- a/llvm/include/llvm/Transforms/Vectorize/SLPVectorizer.h
+++ b/llvm/include/llvm/Transforms/Vectorize/SLPVectorizer.h
@@ -176,8 +176,10 @@ struct SLPVectorizerPass : public OptionalPassInfoMixin<SLPVectorizerPass> {
SmallSetVector<Instruction *, 8> &FMACandidates);
/// Scan the basic block and look for patterns that are likely to start
- /// a vectorization chain.
- bool vectorizeChainsInBlock(BasicBlock *BB, slpvectorizer::BoUpSLP &R);
+ /// a vectorization chain. The FMA candidates are collected in
+ /// \p FMACandidates for the retry after all the blocks of the function.
+ bool vectorizeChainsInBlock(BasicBlock *BB, slpvectorizer::BoUpSLP &R,
+ SmallSetVector<Instruction *, 8> &FMACandidates);
std::optional<bool> vectorizeStoreChain(ArrayRef<Value *> Chain,
slpvectorizer::BoUpSLP &R,
diff --git a/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp b/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp
index f6c0f9e08a2af..e39e0cbb8dcd5 100644
--- a/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp
+++ b/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp
@@ -2494,6 +2494,23 @@ class slpvectorizer::BoUpSLP {
/// vector savings the count check does not model.
bool bypassesInstCountCheck(InstructionCost TreeCost) const;
+ /// \returns the fadd/fsub user of the single-use fmul \p I, which the
+ /// backend fuses with \p I into an fmuladd in the scalar code, and the cost
+ /// of that fmuladd, or {nullptr, invalid cost}.
+ std::pair<Instruction *, InstructionCost>
+ getFMulFusingUser(Instruction *I) const;
+
+ /// \returns true if the fadd/fsub \p U, fused with its fmul operand by the
+ /// backend, is a vectorized tree scalar whose lane is priced as fmuladd.
+ /// Peeled reassociated scalars are priced as plain fadd/fsub.
+ bool isFusedVectorizedFAddSub(Instruction *U) const;
+
+ /// \returns the scalar cost of the fmul lanes of \p TE that are priced as
+ /// fused into their vectorized fadd/fsub users. If the node is turned into a
+ /// gather, these fmuls stay scalar and lose the fusion, so the gather
+ /// alternative must pay their full price.
+ InstructionCost getUnfusedFMulsPenalty(const TreeEntry &TE) const;
+
/// Return information about the vector formed for the specified index
/// of a vector of (the same) instruction.
TargetTransformInfo::OperandValueInfo
@@ -13100,12 +13117,89 @@ bool BoUpSLP::areAllUsersVectorized(
});
}
-static InstructionCost canConvertToFMA(ArrayRef<Value *> VL,
- const InstructionsState &S,
- DominatorTree &DT, const DataLayout &DL,
- TargetTransformInfo &TTI,
- const TargetLibraryInfo &TLI,
- const BoUpSLP &R);
+static InstructionCost
+canConvertToFMA(ArrayRef<Value *> VL, const InstructionsState &S,
+ DominatorTree &DT, const DataLayout &DL,
+ TargetTransformInfo &TTI, const TargetLibraryInfo &TLI,
+ const BoUpSLP &R, bool FuseEitherOperand = true);
+
+/// \returns the fmul operand of the fadd/fsub \p I that the backend fuses with
+/// \p I into an fmuladd: a contractable single-use fmul from the same block,
+/// from either operand, the first one preferred, c - a*b included. \p I must
+/// allow contraction.
+static Instruction *getFusableFMulOperand(Instruction *I) {
+ if ((I->getOpcode() != Instruction::FAdd &&
+ I->getOpcode() != Instruction::FSub) ||
+ !I->hasAllowContract())
+ return nullptr;
+ for (Value *Op : I->operands()) {
+ auto *FMul = dyn_cast<Instruction>(Op);
+ if (FMul && FMul->getOpcode() == Instruction::FMul && FMul->hasOneUse() &&
+ FMul->hasAllowContract() && FMul->getParent() == I->getParent())
+ return FMul;
+ }
+ return nullptr;
+}
+
+/// \returns the cost of the binary operator \p I, priced as not fused. No
+/// context instruction is passed: targets that model the fmuladd fusion would
+/// discount the unfused side of the comparison as well.
+static InstructionCost getUnfusedBinOpCost(Instruction *I,
+ TargetTransformInfo &TTI,
+ const TargetLibraryInfo &TLI,
+ TTI::TargetCostKind CostKind) {
+ assert(I->isBinaryOp() && "Expected a binary operator.");
+ TTI::OperandValueInfo Op1Info = TTI::getOperandInfo(I->getOperand(0));
+ TTI::OperandValueInfo Op2Info = TTI::getOperandInfo(I->getOperand(1));
+ return TTI.getArithmeticInstrCost(
+ I->getOpcode(), I->getType(), CostKind, Op1Info, Op2Info,
+ {I->getOperand(0), I->getOperand(1)}, /*CxtI=*/nullptr, &TLI);
+}
+
+std::pair<Instruction *, InstructionCost>
+BoUpSLP::getFMulFusingUser(Instruction *I) const {
+ assert(I->getOpcode() == Instruction::FMul && "Expected fmul.");
+ if (!I->hasOneUse())
+ return {nullptr, InstructionCost::getInvalid()};
+ auto *U = cast<Instruction>(I->user_back());
+ // The reduction cost accounts for the fusion of the reduced values into the
+ // reduction operations itself. The lanes of the combined fmuladd node are
+ // fused with its own fmul operand node, not with the other operand.
+ if (getFusableFMulOperand(U) != I ||
+ (UserIgnoreList && UserIgnoreList->contains(U)) ||
+ any_of(getTreeEntries(U), [](const TreeEntry *TE) {
+ return TE->CombinedOp == TreeEntry::FMulAdd;
+ }))
+ return {nullptr, InstructionCost::getInvalid()};
+ // The target must actually prefer the fused form.
+ InstructionCost FMACost =
+ canConvertToFMA(U, InstructionsState(U, U), *DT, *DL, *TTI, *TLI, *this);
+ return {FMACost.isValid() ? U : nullptr, FMACost};
+}
+
+bool BoUpSLP::isFusedVectorizedFAddSub(Instruction *U) const {
+ return any_of(getTreeEntries(U), [&](const TreeEntry *TE) {
+ return !DeletedNodes.contains(TE) && !TransformedToGatherNodes.contains(TE);
+ });
+}
+
+InstructionCost BoUpSLP::getUnfusedFMulsPenalty(const TreeEntry &TE) const {
+ if (!TE.hasState() || TE.isGather() ||
+ (TE.getOpcode() != Instruction::FMul &&
+ TE.getAltOpcode() != Instruction::FMul))
+ return TTI::TCC_Free;
+ InstructionCost Penalty = TTI::TCC_Free;
+ for (Value *V : TE.Scalars) {
+ auto *I = dyn_cast<Instruction>(V);
+ if (!I || I->getOpcode() != Instruction::FMul ||
+ (TE.hasCopyableElements() && TE.isCopyableElement(I)))
+ continue;
+ if (Instruction *U = getFMulFusingUser(I).first;
+ U && isFusedVectorizedFAddSub(U))
+ Penalty += getUnfusedBinOpCost(I, *TTI, *TLI, CostKind);
+ }
+ return Penalty;
+}
// A poor-throughput entry's real vector-vs-scalar savings (fdiv/frem/fsqrt)
// are already folded into TreeCost like any other entry, including all
@@ -13196,19 +13290,30 @@ uint64_t BoUpSLP::getNumScalarInsts(bool HasTreeLoop) {
--Count;
}
}
- } else if (Opcode == Instruction::FAdd || Opcode == Instruction::FSub) {
+ } else {
for (Value *V : TE.Scalars) {
if (TE.hasCopyableElements() && TE.isCopyableElement(V))
continue;
auto *I = dyn_cast<Instruction>(V);
- if (!I || (TE.isAltShuffle() && I->getOpcode() != Instruction::FAdd &&
- I->getOpcode() != Instruction::FSub))
+ if (!I)
continue;
- if (canConvertToFMA(I, InstructionsState(I, I), *DT, *DL, *TTI, *TLI,
- *this)
- .isValid()) {
- assert(Count > 0 && "Underflow in scalar inst count (fma)");
- --Count;
+ if (I->getOpcode() == Instruction::FAdd ||
+ I->getOpcode() == Instruction::FSub) {
+ if (canConvertToFMA(I, InstructionsState(I, I), *DT, *DL, *TTI,
+ *TLI, *this)
+ .isValid()) {
+ assert(Count > 0 && "Underflow in scalar inst count (fma)");
+ --Count;
+ }
+ } else if (I->getOpcode() == Instruction::FMul) {
+ // The fmul, fused into a scalar user, disappears into the user's
+ // fmuladd. A vectorized fadd/fsub user lane already accounts for
+ // the fusion.
+ Instruction *U = getFMulFusingUser(I).first;
+ if (U && !isFusedVectorizedFAddSub(U)) {
+ assert(Count > 0 && "Underflow in scalar inst count (fma)");
+ --Count;
+ }
}
}
}
@@ -13565,15 +13670,18 @@ void BoUpSLP::reorderGatherNode(TreeEntry &TE) {
}
}
-/// Check if we can convert fadd/fsub sequence to FMAD.
+/// Check if we can convert fadd/fsub sequence to FMAD. If \p FuseEitherOperand
+/// is set, the fmul of the lane is taken from either operand, as the backend
+/// does, otherwise from the first operand only. The reduction cost uses the
+/// latter: the latency of the scalar reduction chain is not modeled, and
+/// pricing all its links as fused makes the vector reductions unprofitable.
/// \returns Cost of the FMAD, if conversion is possible, invalid cost
/// otherwise.
-static InstructionCost canConvertToFMA(ArrayRef<Value *> VL,
- const InstructionsState &S,
- DominatorTree &DT, const DataLayout &DL,
- TargetTransformInfo &TTI,
- const TargetLibraryInfo &TLI,
- const BoUpSLP &R) {
+static InstructionCost
+canConvertToFMA(ArrayRef<Value *> VL, const InstructionsState &S,
+ DominatorTree &DT, const DataLayout &DL,
+ TargetTransformInfo &TTI, const TargetLibraryInfo &TLI,
+ const BoUpSLP &R, bool FuseEitherOperand) {
assert(all_of(VL,
[](Value *V) {
return V->getType()->getScalarType()->isFloatingPointTy();
@@ -13606,8 +13714,23 @@ static InstructionCost canConvertToFMA(ArrayRef<Value *> VL,
// fmul also should be contractable
InstructionsCompatibilityAnalysis Analysis(DT, DL, TTI, TLI);
SmallVector<BoUpSLP::ValueList> Operands = Analysis.buildOperands(S, VL, R);
+ // The operands follow the majority pattern of the lanes, while the backend
+ // fuses the fmul from either operand (c - a*b included); move it to the
+ // first column.
+ if (FuseEitherOperand && Operands.size() == 2) {
+ for (auto [Idx, V] : enumerate(VL)) {
+ auto *I = dyn_cast<Instruction>(V);
+ if (!I || S.isCopyableElement(I))
+ continue;
+ if (Instruction *FMul = getFusableFMulOperand(I);
+ FMul && Operands[1][Idx] == FMul)
+ std::swap(Operands[0][Idx], Operands[1][Idx]);
+ }
+ }
- InstructionsState OpS = getSameOpcode(Operands.front(), TLI);
+ // The lanes without the fmul, e.g. the constant lanes of the copyable nodes,
+ // are the copyable lanes of the fmul column.
+ InstructionsState OpS = Analysis.buildInstructionsState(Operands.front(), R);
if (!OpS.valid())
return InstructionCost::getInvalid();
@@ -13618,17 +13741,6 @@ static InstructionCost canConvertToFMA(ArrayRef<Value *> VL,
// Compare the costs.
InstructionCost FMulPlusFAddCost = 0;
InstructionCost FMACost = 0;
- // Price both sides of the fmul+fadd pair as not fused. Passing a context
- // instruction would let targets that model the fusion discount the unfused
- // side of the comparison as well.
- auto GetUnfusedFMulCost = [&](Instruction *I) {
- assert(I->getOpcode() == Instruction::FMul && "Expected an fmul");
- TTI::OperandValueInfo Op1Info = TTI::getOperandInfo(I->getOperand(0));
- TTI::OperandValueInfo Op2Info = TTI::getOperandInfo(I->getOperand(1));
- return TTI.getArithmeticInstrCost(I->getOpcode(), I->getType(), CostKind,
- Op1Info, Op2Info,
- {I->getOperand(0), I->getOperand(1)});
- };
FastMathFlags FMF;
FMF.set();
const bool IsArithmeticState = S.isAddSubLikeOp() || S.isMulDivLikeOp() ||
@@ -13641,11 +13753,7 @@ static InstructionCost canConvertToFMA(ArrayRef<Value *> VL,
(I->getOpcode() != S.getOpcode() &&
I->getOpcode() != S.getAltOpcode()))
return TTI.getInstructionCost(I, CostKind);
- TTI::OperandValueInfo Op1Info = TTI::getOperandInfo(I->getOperand(0));
- TTI::OperandValueInfo Op2Info = TTI::getOperandInfo(I->getOperand(1));
- return TTI.getArithmeticInstrCost(I->getOpcode(), I->getType(), CostKind,
- Op1Info, Op2Info,
- {I->getOperand(0), I->getOperand(1)});
+ return getUnfusedBinOpCost(I, TTI, TLI, CostKind);
}
return TTI.getArithmeticInstrCost(Instruction::FAdd, I->getType(), CostKind,
{},
@@ -13668,7 +13776,9 @@ static InstructionCost canConvertToFMA(ArrayRef<Value *> VL,
if (S.isCopyableElement(V))
continue;
auto *I = dyn_cast<Instruction>(Op);
- if (!I || !I->hasOneUse() || OpS.isCopyableElement(I)) {
+ // The backend fuses the fmul with its user in the same block only.
+ if (!I || !I->hasOneUse() || OpS.isCopyableElement(I) ||
+ I->getParent() != cast<Instruction>(V)->getParent()) {
if (auto *OpI = dyn_cast<Instruction>(V))
FMACost += GetLinkCost(OpI);
if (I)
@@ -13678,7 +13788,7 @@ static InstructionCost canConvertToFMA(ArrayRef<Value *> VL,
++NumOps;
if (auto *FPCI = dyn_cast<FPMathOperator>(I))
FMF &= FPCI->getFastMathFlags();
- FMulPlusFAddCost += GetUnfusedFMulCost(I);
+ FMulPlusFAddCost += getUnfusedBinOpCost(I, TTI, TLI, CostKind);
}
Type *Ty = VL.front()->getType();
IntrinsicCostAttributes ICA(Intrinsic::fmuladd, Ty, {Ty, Ty, Ty}, FMF);
@@ -14613,9 +14723,9 @@ void BoUpSLP::transformNodes() {
V->hasOneUse();
});
};
- if (!IsOneUseVectorFMulOperand(LHS) &&
- (E.getOpcode() == Instruction::FSub ||
- !IsOneUseVectorFMulOperand(RHS)))
+ // Both a*b - c and c - a*b are fused.
+ const bool IsLHSFMul = IsOneUseVectorFMulOperand(LHS);
+ if (!IsLHSFMul && !IsOneUseVectorFMulOperand(RHS))
break;
if (!canConvertToFMA(E.Scalars, E.getOperations(), *DT, *DL, *TTI, *TLI,
*this)
@@ -14623,7 +14733,7 @@ void BoUpSLP::transformNodes() {
break;
// This node is a fmuladd node.
E.CombinedOp = TreeEntry::FMulAdd;
- TreeEntry *FMulEntry = getOperandEntry(&E, 0);
+ TreeEntry *FMulEntry = getOperandEntry(&E, IsLHSFMul ? 0 : 1);
if (FMulEntry->UserTreeIndex &&
FMulEntry->State == TreeEntry::Vectorize) {
// The FMul node is part of the combined fmuladd node.
@@ -16202,9 +16312,12 @@ BoUpSLP::getEntryCost(const TreeEntry *E, ArrayRef<Value *> VectorizedVals,
return 0;
if (isa<InsertElementInst, InsertValueInst>(VL[0]))
return InstructionCost::getInvalid();
+ // The fmuls of the node turned into a gather stay scalar and lose the
+ // fusion with their vectorized fadd/fsub users.
return SpillsReloads +
processBuildVector<ShuffleCostEstimator, InstructionCost>(
- E, ScalarTy, *TTI, VectorizedVals, *this, CheckedExtracts);
+ E, ScalarTy, *TTI, VectorizedVals, *this, CheckedExtracts) +
+ getUnfusedFMulsPenalty(*E);
}
if (E->State == TreeEntry::SplitVectorize) {
assert(E->CombinedEntriesWithIndices.size() == 2 &&
@@ -16440,10 +16553,30 @@ BoUpSLP::getEntryCost(const TreeEntry *E, ArrayRef<Value *> VectorizedVals,
}
return IntrinsicCost;
};
+ // The fadd/fsub, fused with its fmul operand into an fmuladd in the scalar
+ // code. If the fmul is not vectorized, the vector code still emits it, so
+ // only the fusion delta is attributed to the fadd/fsub.
auto GetFMulAddCost = [&, &TTI = *TTI](const InstructionsState &S,
Instruction *VI) {
InstructionCost Cost = canConvertToFMA(VI, S, *DT, *DL, TTI, *TLI, *this);
- return Cost;
+ Instruction *FMul = getFusableFMulOperand(VI);
+ if (!Cost.isValid() || !FMul || isVectorized(FMul))
+ return Cost;
+ InstructionCost FMulCost = getUnfusedBinOpCost(FMul, TTI, *TLI, CostKind);
+ return Cost > FMulCost ? Cost - FMulCost : InstructionCost(TTI::TCC_Free);
+ };
+ // The fmul, fused into its fadd/fsub user, is free if the user lane is
+ // priced as fmuladd, otherwise the user stays scalar and only the fusion
+ // delta is attributed to the fmul.
+ auto GetFusedFMulCost = [&, &TTI = *TTI](Instruction *VI) {
+ auto [U, FMACost] = getFMulFusingUser(VI);
+ if (!U)
+ return InstructionCost::getInvalid();
+ if (isFusedVectorizedFAddSub(U))
+ return InstructionCost(TTI::TCC_Free);
+ InstructionCost FAddCost = getUnfusedBinOpCost(U, TTI, *TLI, CostKind);
+ return FMACost > FAddCost ? FMACost - FAddCost
+ : InstructionCost(TTI::TCC_Free);
};
switch (ShuffleOrOp) {
case Instruction::PHI: {
@@ -16530,6 +16663,13 @@ BoUpSLP::getEntryCost(const TreeEntry *E, ArrayRef<Value *> VectorizedVals,
return Cost;
}
}
+ // The extract, which the target folds into its users (e.g. the lane
+ // operand of a by-element fmul on AArch64), costs nothing in the scalar
+ // code; its removal saves nothing.
+ if (ShuffleOrOp == Instruction::ExtractElement &&
+ TTI->getVectorInstrCost(*I, SrcVecTy, CostKind, *ExtIdx,
+ TTI::getVectorInstrContextHint(I)) == 0)
+ return InstructionCost(TTI::TCC_Free);
if (DemandedElts.isZero())
DemandedElts = APInt::getZero(getNumElements(SrcVecTy));
DemandedElts.setBit(*ExtIdx);
@@ -16917,8 +17057,18 @@ BoUpSLP::getEntryCost(const TreeEntry *E, ArrayRef<Value *> VectorizedVals,
auto GetScalarCost = [&](unsigned Idx) {
if (isa<PoisonValue>(UniqueValues[Idx]))
return InstructionCost(TTI::TCC_Free);
- return GetFMulAddCost(E->getOperations(),
- cast<Instruction>(UniqueValues[Idx]));
+ auto *I = cast<Instruction>(UniqueValues[Idx]);
+ InstructionCost Cost = GetFMulAddCost(E->getOperations(), I);
+ if (Cost.isValid())
+ return Cost;
+ // The lanes without the fmul, e.g. the constant lanes of the fmul node,
+ // stay plain fadd/fsub.
+ unsigned Lane = UniqueIndexes[Idx];
+ Value *Op1 = E->getOperand(0)[Lane];
+ Value *Op2 = E->getOperand(1)[Lane];
+ return TTI->getArithmeticInstrCost(
+ E->getOpcode(), OrigScalarTy, CostKind, TTI::getOperandInfo(Op1),
+ TTI::getOperandInfo(Op2), {Op1, Op2}, I, TLI);
};
auto GetVectorCost = [&, &TTI = *TTI](InstructionCost CommonCost) {
FastMathFlags FMF;
@@ -16926,8 +17076,9 @@ BoUpSLP::getEntryCost(const TreeEntry *E, ArrayRef<Value *> VectorizedVals,
for (Value *V : E->Scalars) {
if (auto *FPCI = dyn_cast<FPMathOperator>(V)) {
FMF &= FPCI->getFastMathFlags();
- if (auto *FPCIOp = dyn_cast<FPMathOperator>(FPCI->getOperand(0)))
- FMF &= FPCIOp->getFastMathFlags();
+ // The fused fmul may sit in either operand (c - a*b included).
+ if (Instruction *FMul = getFusableFMulOperand(cast<Instruction>(V)))
+ FMF &= FMul->getFastMathFlags();
}
}
IntrinsicCostAttributes ICA(Intrinsic::fmuladd, VecTy,
@@ -17138,6 +17289,10 @@ BoUpSLP::getEntryCost(const TreeEntry *E, ArrayRef<Value *> VectorizedVals,
InstructionCost IntrinsicCost = GetFMulAddCost(E->getOperations(), I);
if (IntrinsicCost.isValid())
ScalarCost = IntrinsicCost;
+ } else if (I && ShuffleOrOp == Instruction::FMul) {
+ InstructionCost FusedCost = GetFusedFMulCost(I);
+ if (FusedCost.isValid())
+ ScalarCost = FusedCost;
}
return ScalarCost;
};
@@ -17452,7 +17607,43 @@ BoUpSLP::getEntryCost(const TreeEntry *E, ArrayRef<Value *> VectorizedVals,
assert(E->getMatchingMainOpOrAltOp(VI) &&
"Unexpected main/alternate opcode");
(void)E;
- return TTI->getInstructionCost(VI, CostKind);
+ if (auto *CI = dyn_cast<CmpInst>(VI))
+ return TTI->getCmpSelInstrCost(
+ CI->getOpcode(), CI->getOperand(0)->getType(), CI->getType(),
+ CI->getPredicate(), CostKind,
+ TTI::getOperandInfo(CI->getOperand(0)),
+ TTI::getOperandInfo(CI->getOperand(1)), CI);
+ if (auto *CI = dyn_cast<CastInst>(VI))
+ return TTI->getCastInstrCost(CI->getOpcode(), CI->getDestTy(),
+ CI->getSrcTy(),
+ TTI::getCastContextHint(CI), CostKind, CI);
+ if (auto *SV = dyn_cast<ShuffleVectorInst>(VI)) {
+ int Index;
+ [[maybe_unused]] bool IsExtractSubvectorMask =
+ SV->isExtractSubvectorMask(Index);
+ assert(IsExtractSubvectorMask && "Not supported shufflevector usage.");
+ auto *SubTy = cast<VectorType>(SV->getType());
+ return TTI->getShuffleCost(
+ TTI::SK_ExtractSubvector, SubTy,
+ cast<VectorType>(SV->getOperand(0)->getType()), CostKind,
+ SV->getShuffleMask(), Index, SubTy,
+ {SV->getOperand(0), SV->getOperand(1)}, SV,
+ TTI::getVectorInstrContextHint(SV));
+ }
+ // The backend fuses fmul+fadd/fsub pairs in the scalar code.
+ InstructionCost FusedCost = InstructionCost::getInvalid();
+ if (VI->getOpcode() == Instruction::FAdd ||
+ VI->getOpcode() == Instruction::FSub)
+ FusedCost = GetFMulAddCost(InstructionsState(VI, VI), VI);
+ else if (VI->getOpcode() == Instruction::FMul)
+ FusedCost = GetFusedFMulCost(VI);
+ if (FusedCost.isValid())
+ return FusedCost;
+ return TTI->getArithmeticInstrCost(
+ VI->getOpcode(), VI->getType(), CostKind,
+ TTI::getOperandInfo(VI->getOperand(0)),
+ TTI::getOperandInfo(VI->getOperand(1)),
+ {VI->getOperand(0), VI->getOperand(1)}, VI, TLI);
};
// Need to clear CommonCost since the final shuffle cost is included into
// vector cost.
@@ -19473,6 +19664,9 @@ BoUpSLP::calculateTreeCostAndTrimNonProfitable(ArrayRef<Value *> VectorizedVals,
isConstant(V) || isGathered(V) || getTreeEntries(V).size() > 1;
}))
GatherCost *= 2;
+ // The fmuls, priced as fused into their vectorized fadd/fsub users, stay
+ // scalar and unfused in the gathered form.
+ GatherCost += getUnfusedFMulsPenalty(*TE);
// Erase subtree if it is non-profitable.
ArrayRef<unsigned> Nodes = std::get<2>(Worklist.top().second);
// Prefer trimming equal-cost alternate-shuffle subtrees rooted at binary
@@ -28876,6 +29070,7 @@ bool SLPVectorizerPass::runImpl(Function &F, ScalarEvolution *SE_,
// Update DFS numbers now so that we can use them for ordering.
DT->updateDFSNumbers();
+ SmallSetVector<Instruction *, 8> FMACandidates;
// Scan the blocks in the function in post order.
for (auto *BB : post_order(&F.getEntryBlock())) {
if (BB->isEHPad() || isa_and_nonnull<UnreachableInst>(BB->getTerminator()))
@@ -28898,7 +29093,7 @@ bool SLPVectorizerPass::runImpl(Function &F, ScalarEvolution *SE_,
}
// Vectorize trees that end at reductions.
- Changed |= vectorizeChainsInBlock(BB, R);
+ Changed |= vectorizeChainsInBlock(BB, R, FMACandidates);
// Vectorize the index computations of getelementptr instructions. This
// is primarily intended to catch gather-like idioms ending at
@@ -28910,6 +29105,20 @@ bool SLPVectorizerPass::runImpl(Function &F, ScalarEvolution *SE_,
}
}
+ // Vectorized on their own, the FMA candidates lose the fusion with their fmul
+ // operands. They are retried after all the blocks, not to preempt the trees
+ // of the blocks processed later, e.g. the store chains of a loop body,
+ // processed after the latch PHIs reaching the same instructions.
+ R.clearReductionData();
+ SmallSetVector<Instruction *, 8> Empty;
+ for (Instruction *I : FMACandidates) {
+ if (R.isDeleted(I))
+ continue;
+ Changed |= tryToVectorize(I, R, Empty, /*AllowFMACandidates=*/true);
+ }
+ assert(Empty.empty() &&
+ "No new FMA candidates expected during AllowFMACandidates retry.");
+
// Instructions with the single user require just one extract per lane, so
// they are used as the seeds for the very last attempt, after all the other
// roots in the function are exhausted.
@@ -33524,13 +33733,17 @@ class HorizontalReduction {
InstructionCost ScalarCost = 0;
for (User *U : RdxVal->users()) {
auto *RdxOp = cast<Instruction>(U);
- if (hasRequiredNumberOfUses(IsCmpSelMinMax, RdxOp)) {
+ // The reduction root is used outside of the reduction any
+ // number of times, its fmul operand is still fused into it.
+ if ((RdxKind == RecurKind::FAdd && RdxOp == ReductionRoot) ||
+ hasRequiredNumberOfUses(IsCmpSelMinMax, RdxOp)) {
InstructionsState RdxOpS = RdxKind == RecurKind::FAdd
? getSameOpcode(RdxOp, TLI)
: InstructionsState::invalid();
if (RdxOpS && RdxOpS.isAddSubOrFNegLikeOp()) {
InstructionCost FMACost =
- canConvertToFMA(RdxOp, RdxOpS, DT, DL, *TTI, TLI, R);
+ canConvertToFMA(RdxOp, RdxOpS, DT, DL, *TTI, TLI, R,
+ /*FuseEitherOperand=*/false);
if (FMACost.isValid()) {
LLVM_DEBUG(dbgs() << "FMA cost: " << FMACost << "\n");
if (auto *I = dyn_cast<Instruction>(RdxVal)) {
@@ -33698,7 +33911,8 @@ class HorizontalReduction {
if (!Ops.empty()) {
InstructionsState S = getSameOpcode(Ops, TLI);
if (S && S.isAddSubOrFNegLikeOp())
- FMACost = canConvertToFMA(Ops, S, DT, DL, *TTI, TLI, R);
+ FMACost = canConvertToFMA(Ops, S, DT, DL, *TTI, TLI, R,
+ /*FuseEitherOperand=*/false);
if (FMACost.isValid()) {
// Calculate actual FMAD cost.
IntrinsicCostAttributes ICA(Intrinsic::fmuladd, RVecTy,
@@ -34586,8 +34800,8 @@ bool SLPVectorizerPass::tryToVectorize(
if (!isa<BinaryOperator, CmpInst>(I) || isa<VectorType>(I->getType()))
return false;
- // Skip potential FMA candidates and collect them for a retry after all other
- // instructions in the block have been processed.
+ // Skip potential FMA candidates and collect them for a retry after all
+ // blocks of the function have been processed.
if (!AllowFMACandidates &&
(I->getOpcode() == Instruction::FAdd ||
I->getOpcode() == Instruction::FSub) &&
@@ -35362,7 +35576,9 @@ bool SLPVectorizerPass::vectorizeInserts(
return OpsChanged;
}
-bool SLPVectorizerPass::vectorizeChainsInBlock(BasicBlock *BB, BoUpSLP &R) {
+bool SLPVectorizerPass::vectorizeChainsInBlock(
+ BasicBlock *BB, BoUpSLP &R,
+ SmallSetVector<Instruction *, 8> &FMACandidates) {
bool Changed = false;
SmallVector<Value *, 4> Incoming;
SmallPtrSet<Value *, 16> VisitedInstrs;
@@ -35601,7 +35817,6 @@ bool SLPVectorizerPass::vectorizeChainsInBlock(BasicBlock *BB, BoUpSLP &R) {
SmallSetVector<Instruction *, 8> PostProcessInsts;
// Stores are processed after all other instructions/roots.
SmallSetVector<StoreInst *, 8> PostProcessStores;
- SmallSetVector<Instruction *, 8> FMACandidates;
SmallSetVector<Instruction *, 8> PoorThroughputSeeds;
PoorThroughputOpCache PoorThroughputCache;
auto VectorizeInsertsAndCmps = [&](bool AtTerminator) {
@@ -35805,14 +36020,6 @@ bool SLPVectorizerPass::vectorizeChainsInBlock(BasicBlock *BB, BoUpSLP &R) {
R, FMACandidates);
}
}
- SmallSetVector<Instruction *, 8> Empty;
- for (Instruction *I : FMACandidates) {
- if (R.isDeleted(I))
- continue;
- Changed |= tryToVectorize(I, R, Empty, /*AllowFMACandidates=*/true);
- }
- assert(Empty.empty() &&
- "No new FMA candidates expected during AllowFMACandidates retry.");
if (PoorThroughputSeeds.size() >= 2) {
SmallVector<Value *> Seeds;
diff --git a/llvm/test/Transforms/PhaseOrdering/AArch64/reassociate-fma-pairs.ll b/llvm/test/Transforms/PhaseOrdering/AArch64/reassociate-fma-pairs.ll
index 9631c6444cb2c..c4eab73309c54 100644
--- a/llvm/test/Transforms/PhaseOrdering/AArch64/reassociate-fma-pairs.ll
+++ b/llvm/test/Transforms/PhaseOrdering/AArch64/reassociate-fma-pairs.ll
@@ -18,49 +18,49 @@ define double @md_vdw_energy(ptr nocapture readonly %coeffs, double %energy, dou
; CHECK-NEXT: [[FACTOR_OP_FMUL2:%.*]] = fmul fast double [[TABLE_DELTA]], 5.000000e-01
; CHECK-NEXT: [[FACTOR_OP_FMUL3:%.*]] = fmul fast double [[FACTOR_OP_FMUL]], [[TABLE_DELTA]]
; CHECK-NEXT: [[FACTOR_OP_FMUL4:%.*]] = fmul fast double [[TMP0]], 2.500000e-01
-; CHECK-NEXT: [[TMP1:%.*]] = insertelement <2 x double> poison, double [[SCALE]], i64 0
-; CHECK-NEXT: [[TMP11:%.*]] = shufflevector <2 x double> [[TMP1]], <2 x double> poison, <2 x i32> zeroinitializer
; CHECK-NEXT: br label %[[LOOP:.*]]
; CHECK: [[LOOP]]:
; CHECK-NEXT: [[I:%.*]] = phi i64 [ 0, %[[ENTRY]] ], [ [[NEXT:%.*]], %[[LOOP]] ]
; CHECK-NEXT: [[ACC2:%.*]] = phi double [ [[ENERGY]], %[[ENTRY]] ], [ [[RESULT1:%.*]], %[[LOOP]] ]
; CHECK-NEXT: [[BASE:%.*]] = getelementptr inbounds [8 x i8], ptr [[COEFFS]], i64 [[I]]
+; CHECK-NEXT: [[A:%.*]] = load double, ptr [[BASE]], align 8
+; CHECK-NEXT: [[B_PTR:%.*]] = getelementptr inbounds nuw i8, ptr [[BASE]], i64 8
+; CHECK-NEXT: [[B:%.*]] = load double, ptr [[B_PTR]], align 8
+; CHECK-NEXT: [[TMP5:%.*]] = fmul fast double [[A]], [[SCALE]]
+; CHECK-NEXT: [[TMP6:%.*]] = fmul fast double [[B]], [[SCALE]]
; CHECK-NEXT: [[C0_PTR:%.*]] = getelementptr inbounds nuw i8, ptr [[BASE]], i64 16
; CHECK-NEXT: [[C0:%.*]] = load double, ptr [[C0_PTR]], align 8
; CHECK-NEXT: [[C1_PTR:%.*]] = getelementptr inbounds nuw i8, ptr [[BASE]], i64 24
; CHECK-NEXT: [[C1:%.*]] = load double, ptr [[C1_PTR]], align 8
+; CHECK-NEXT: [[P0:%.*]] = fmul fast double [[C0]], [[TMP5]]
+; CHECK-NEXT: [[Q0:%.*]] = fmul fast double [[C1]], [[TMP6]]
+; CHECK-NEXT: [[D3_NEG:%.*]] = fsub fast double [[P0]], [[Q0]]
; CHECK-NEXT: [[C2_PTR:%.*]] = getelementptr inbounds nuw i8, ptr [[BASE]], i64 32
; CHECK-NEXT: [[C2:%.*]] = load double, ptr [[C2_PTR]], align 8
; CHECK-NEXT: [[C3_PTR:%.*]] = getelementptr inbounds nuw i8, ptr [[BASE]], i64 40
; CHECK-NEXT: [[C3:%.*]] = load double, ptr [[C3_PTR]], align 8
+; CHECK-NEXT: [[P1:%.*]] = fmul fast double [[C2]], [[TMP5]]
+; CHECK-NEXT: [[Q1:%.*]] = fmul fast double [[C3]], [[TMP6]]
+; CHECK-NEXT: [[D1:%.*]] = fsub fast double [[P1]], [[Q1]]
; CHECK-NEXT: [[C4_PTR:%.*]] = getelementptr inbounds nuw i8, ptr [[BASE]], i64 48
; CHECK-NEXT: [[C4:%.*]] = load double, ptr [[C4_PTR]], align 8
; CHECK-NEXT: [[C5_PTR:%.*]] = getelementptr inbounds nuw i8, ptr [[BASE]], i64 56
; CHECK-NEXT: [[C5:%.*]] = load double, ptr [[C5_PTR]], align 8
-; CHECK-NEXT: [[C6_PTR:%.*]] = getelementptr inbounds nuw i8, ptr [[BASE]], i64 64
-; CHECK-NEXT: [[TMP3:%.*]] = load <2 x double>, ptr [[BASE]], align 8
-; CHECK-NEXT: [[TMP4:%.*]] = fmul fast <2 x double> [[TMP3]], [[TMP11]]
-; CHECK-NEXT: [[TMP5:%.*]] = extractelement <2 x double> [[TMP4]], i64 0
-; CHECK-NEXT: [[ACC:%.*]] = fmul fast double [[C0]], [[TMP5]]
-; CHECK-NEXT: [[TMP6:%.*]] = extractelement <2 x double> [[TMP4]], i64 1
-; CHECK-NEXT: [[TMP2:%.*]] = fmul fast double [[C1]], [[TMP6]]
-; CHECK-NEXT: [[D3_NEG:%.*]] = fsub fast double [[ACC]], [[TMP2]]
-; CHECK-NEXT: [[P1:%.*]] = fmul fast double [[C2]], [[TMP5]]
-; CHECK-NEXT: [[Q1:%.*]] = fmul fast double [[C3]], [[TMP6]]
-; CHECK-NEXT: [[D1:%.*]] = fsub fast double [[P1]], [[Q1]]
; CHECK-NEXT: [[P2:%.*]] = fmul fast double [[C4]], [[TMP5]]
; CHECK-NEXT: [[Q2:%.*]] = fmul fast double [[C5]], [[TMP6]]
; CHECK-NEXT: [[D2:%.*]] = fsub fast double [[P2]], [[Q2]]
-; CHECK-NEXT: [[TMP7:%.*]] = load <2 x double>, ptr [[C6_PTR]], align 8
+; CHECK-NEXT: [[C6_PTR:%.*]] = getelementptr inbounds nuw i8, ptr [[BASE]], i64 64
+; CHECK-NEXT: [[C6:%.*]] = load double, ptr [[C6_PTR]], align 8
+; CHECK-NEXT: [[C7_PTR:%.*]] = getelementptr inbounds nuw i8, ptr [[BASE]], i64 72
+; CHECK-NEXT: [[C7:%.*]] = load double, ptr [[C7_PTR]], align 8
; CHECK-NEXT: [[TMP9:%.*]] = fmul fast double [[FACTOR_OP_FMUL3]], [[D3_NEG]]
; CHECK-NEXT: [[RESULT:%.*]] = fmul fast double [[D1]], [[FACTOR_OP_FMUL4]]
; CHECK-NEXT: [[REASS_ADD:%.*]] = fadd fast double [[RESULT]], [[TMP9]]
; CHECK-NEXT: [[T4_REASS:%.*]] = fmul fast double [[D2]], [[FACTOR_OP_FMUL2]]
; CHECK-NEXT: [[ACC1:%.*]] = fadd fast double [[REASS_ADD]], [[T4_REASS]]
-; CHECK-NEXT: [[TMP8:%.*]] = fmul fast <2 x double> [[TMP7]], [[TMP4]]
-; CHECK-NEXT: [[TMP10:%.*]] = extractelement <2 x double> [[TMP8]], i64 0
+; CHECK-NEXT: [[TMP10:%.*]] = fmul fast double [[C6]], [[TMP5]]
+; CHECK-NEXT: [[TMP12:%.*]] = fmul fast double [[C7]], [[TMP6]]
; CHECK-NEXT: [[S2_NEG1:%.*]] = fadd fast double [[ACC1]], [[TMP10]]
-; CHECK-NEXT: [[TMP12:%.*]] = extractelement <2 x double> [[TMP8]], i64 1
; CHECK-NEXT: [[S2_NEG:%.*]] = fadd fast double [[ACC2]], [[TMP12]]
; CHECK-NEXT: [[RESULT1]] = fsub fast double [[S2_NEG]], [[S2_NEG1]]
; CHECK-NEXT: [[NEXT]] = add nuw i64 [[I]], 10
diff --git a/llvm/test/Transforms/SLPVectorizer/AArch64/extracts-folded-into-fmul-users-no-credit.ll b/llvm/test/Transforms/SLPVectorizer/AArch64/extracts-folded-into-fmul-users-no-credit.ll
index f93a7dc3abf83..21c709e57765f 100644
--- a/llvm/test/Transforms/SLPVectorizer/AArch64/extracts-folded-into-fmul-users-no-credit.ll
+++ b/llvm/test/Transforms/SLPVectorizer/AArch64/extracts-folded-into-fmul-users-no-credit.ll
@@ -14,8 +14,6 @@ define void @fmul_reduction_extracts_free_for_fmul_users(ptr %vp, ptr %sp, ptr %
; CHECK-NEXT: [[B:%.*]] = load double, ptr [[BP]], align 8
; CHECK-NEXT: [[C:%.*]] = load double, ptr [[CP]], align 8
; CHECK-NEXT: [[D:%.*]] = load double, ptr [[DP]], align 8
-; CHECK-NEXT: [[TMP0:%.*]] = insertelement <2 x double> poison, double [[A]], i64 0
-; CHECK-NEXT: [[TMP1:%.*]] = insertelement <2 x double> [[TMP0]], double [[B]], i64 1
; CHECK-NEXT: br label %[[LOOP:.*]]
; CHECK: [[LOOP]]:
; CHECK-NEXT: [[I:%.*]] = phi i64 [ 0, %[[ENTRY]] ], [ [[I_NEXT:%.*]], %[[LOOP]] ]
@@ -25,10 +23,11 @@ define void @fmul_reduction_extracts_free_for_fmul_users(ptr %vp, ptr %sp, ptr %
; CHECK-NEXT: [[S:%.*]] = load double, ptr [[SPTR]], align 8
; CHECK-NEXT: [[E0:%.*]] = extractelement <2 x double> [[V]], i64 0
; CHECK-NEXT: [[E1:%.*]] = extractelement <2 x double> [[V]], i64 1
-; CHECK-NEXT: [[TMP2:%.*]] = fmul fast <2 x double> [[V]], [[TMP1]]
-; CHECK-NEXT: [[M2:%.*]] = fmul fast double [[S]], [[C]]
-; CHECK-NEXT: [[R0:%.*]] = call fast double @llvm.vector.reduce.fadd.v2f64(double 0.000000e+00, <2 x double> [[TMP2]])
+; CHECK-NEXT: [[R0:%.*]] = fmul fast double [[E0]], [[A]]
+; CHECK-NEXT: [[M2:%.*]] = fmul fast double [[E1]], [[B]]
+; CHECK-NEXT: [[M3:%.*]] = fmul fast double [[S]], [[C]]
; CHECK-NEXT: [[R:%.*]] = fadd fast double [[R0]], [[M2]]
+; CHECK-NEXT: [[R1:%.*]] = fadd fast double [[R]], [[M3]]
; CHECK-NEXT: [[U0:%.*]] = fmul fast double [[E0]], [[S]]
; CHECK-NEXT: [[U1:%.*]] = fmul fast double [[E1]], [[D]]
; CHECK-NEXT: [[U2:%.*]] = fmul fast double [[E1]], [[S]]
@@ -42,7 +41,7 @@ define void @fmul_reduction_extracts_free_for_fmul_users(ptr %vp, ptr %sp, ptr %
; CHECK-NEXT: store double [[U2]], ptr [[O2]], align 8
; CHECK-NEXT: [[I_NEXT]] = add nuw i64 [[I]], 1
; CHECK-NEXT: [[CMP:%.*]] = icmp ult i64 [[I_NEXT]], [[N]]
-; CHECK-NEXT: [[LT:%.*]] = fcmp fast olt double [[R]], [[D]]
+; CHECK-NEXT: [[LT:%.*]] = fcmp fast olt double [[R1]], [[D]]
; CHECK-NEXT: [[CONT:%.*]] = and i1 [[CMP]], [[LT]]
; CHECK-NEXT: br i1 [[CONT]], label %[[LOOP]], label %[[EXIT:.*]]
; CHECK: [[EXIT]]:
diff --git a/llvm/test/Transforms/SLPVectorizer/AArch64/fadd-with-gathered-fmul-operands.ll b/llvm/test/Transforms/SLPVectorizer/AArch64/fadd-with-gathered-fmul-operands.ll
index 4385123ca8716..b19204af12b8b 100644
--- a/llvm/test/Transforms/SLPVectorizer/AArch64/fadd-with-gathered-fmul-operands.ll
+++ b/llvm/test/Transforms/SLPVectorizer/AArch64/fadd-with-gathered-fmul-operands.ll
@@ -22,13 +22,21 @@ define void @fadd_with_gathered_fmul_operands(double %a0, double %b0, double %a1
; CHECK-NEXT: br label %[[MUL3:.*]]
; CHECK: [[MUL3]]:
; CHECK-NEXT: [[M3:%.*]] = fmul contract double [[A3]], [[B3]]
-; CHECK-NEXT: [[TMP0:%.*]] = load <4 x double>, ptr [[X]], align 8
-; CHECK-NEXT: [[TMP1:%.*]] = insertelement <4 x double> poison, double [[M0]], i64 0
-; CHECK-NEXT: [[TMP2:%.*]] = insertelement <4 x double> [[TMP1]], double [[M1]], i64 1
-; CHECK-NEXT: [[TMP3:%.*]] = insertelement <4 x double> [[TMP2]], double [[M2]], i64 2
-; CHECK-NEXT: [[TMP4:%.*]] = insertelement <4 x double> [[TMP3]], double [[M3]], i64 3
-; CHECK-NEXT: [[TMP5:%.*]] = fadd contract <4 x double> [[TMP4]], [[TMP0]]
-; CHECK-NEXT: store <4 x double> [[TMP5]], ptr [[OUT]], align 8
+; CHECK-NEXT: [[X2P:%.*]] = getelementptr inbounds double, ptr [[X]], i64 2
+; CHECK-NEXT: [[X2:%.*]] = load double, ptr [[X2P]], align 8
+; CHECK-NEXT: [[X3P:%.*]] = getelementptr inbounds double, ptr [[X]], i64 3
+; CHECK-NEXT: [[X3:%.*]] = load double, ptr [[X3P]], align 8
+; CHECK-NEXT: [[S2:%.*]] = fadd contract double [[M2]], [[X2]]
+; CHECK-NEXT: [[S3:%.*]] = fadd contract double [[M3]], [[X3]]
+; CHECK-NEXT: [[TMP0:%.*]] = load <2 x double>, ptr [[X]], align 8
+; CHECK-NEXT: [[TMP1:%.*]] = insertelement <2 x double> poison, double [[M0]], i64 0
+; CHECK-NEXT: [[TMP2:%.*]] = insertelement <2 x double> [[TMP1]], double [[M1]], i64 1
+; CHECK-NEXT: [[TMP3:%.*]] = fadd contract <2 x double> [[TMP2]], [[TMP0]]
+; CHECK-NEXT: store <2 x double> [[TMP3]], ptr [[OUT]], align 8
+; CHECK-NEXT: [[O2:%.*]] = getelementptr inbounds double, ptr [[OUT]], i64 2
+; CHECK-NEXT: store double [[S2]], ptr [[O2]], align 8
+; CHECK-NEXT: [[O3:%.*]] = getelementptr inbounds double, ptr [[OUT]], i64 3
+; CHECK-NEXT: store double [[S3]], ptr [[O3]], align 8
; CHECK-NEXT: ret void
;
entry:
diff --git a/llvm/test/Transforms/SLPVectorizer/AArch64/fma-candidates-after-store-chains.ll b/llvm/test/Transforms/SLPVectorizer/AArch64/fma-candidates-after-store-chains.ll
index 9e63b6eb3dbce..ada85a41fb777 100644
--- a/llvm/test/Transforms/SLPVectorizer/AArch64/fma-candidates-after-store-chains.ll
+++ b/llvm/test/Transforms/SLPVectorizer/AArch64/fma-candidates-after-store-chains.ll
@@ -26,21 +26,21 @@ define double @test(ptr %a, ptr %h, ptr %out, ptr %q, i64 %n) {
; CHECK-NEXT: br i1 [[CMP]], label %[[BODY:.*]], label %[[LATCH]]
; CHECK: [[BODY]]:
; CHECK-NEXT: [[PA0:%.*]] = getelementptr double, ptr [[A]], i64 [[I]]
-; CHECK-NEXT: [[PA2:%.*]] = getelementptr double, ptr [[PA0]], i64 2
-; CHECK-NEXT: [[A0:%.*]] = load double, ptr [[PA2]], align 8
+; CHECK-NEXT: [[PA1:%.*]] = getelementptr double, ptr [[PA0]], i64 1
+; CHECK-NEXT: [[A0:%.*]] = load double, ptr [[PA0]], align 8
; CHECK-NEXT: [[X:%.*]] = fsub fast double [[A0]], 1.000000e+00
-; CHECK-NEXT: [[TMP0:%.*]] = load <2 x double>, ptr [[PA0]], align 8
+; CHECK-NEXT: [[TMP0:%.*]] = load <2 x double>, ptr [[PA1]], align 8
; CHECK-NEXT: [[TMP1:%.*]] = fsub fast <2 x double> [[TMP0]], splat (double 1.000000e+00)
; CHECK-NEXT: [[TMP2:%.*]] = load <2 x double>, ptr [[H]], align 8
-; CHECK-NEXT: [[TMP5:%.*]] = fmul fast <2 x double> [[TMP1]], [[TMP2]]
+; CHECK-NEXT: [[TMP3:%.*]] = shufflevector <2 x double> [[TMP1]], <2 x double> poison, <2 x i32> <i32 poison, i32 0>
+; CHECK-NEXT: [[TMP4:%.*]] = insertelement <2 x double> [[TMP3]], double [[X]], i64 0
+; CHECK-NEXT: [[TMP5:%.*]] = fmul fast <2 x double> [[TMP4]], [[TMP2]]
; CHECK-NEXT: [[TMP6:%.*]] = call fast double @llvm.vector.reduce.fadd.v2f64(double 0.000000e+00, <2 x double> [[TMP5]])
; CHECK-NEXT: [[ACC_BODY:%.*]] = fadd fast double [[TMP6]], [[ACC]]
-; CHECK-NEXT: [[TMP7:%.*]] = extractelement <2 x double> [[TMP1]], i64 1
-; CHECK-NEXT: [[FY:%.*]] = fmul fast double [[C]], [[TMP7]]
-; CHECK-NEXT: [[FZ:%.*]] = fmul fast double [[C]], [[X]]
-; CHECK-NEXT: [[POUT1:%.*]] = getelementptr double, ptr [[OUT]], i64 1
-; CHECK-NEXT: store double [[FY]], ptr [[OUT]], align 8
-; CHECK-NEXT: store double [[FZ]], ptr [[POUT1]], align 8
+; CHECK-NEXT: [[TMP7:%.*]] = insertelement <2 x double> poison, double [[C]], i64 0
+; CHECK-NEXT: [[TMP8:%.*]] = shufflevector <2 x double> [[TMP7]], <2 x double> poison, <2 x i32> zeroinitializer
+; CHECK-NEXT: [[TMP9:%.*]] = fmul fast <2 x double> [[TMP8]], [[TMP1]]
+; CHECK-NEXT: store <2 x double> [[TMP9]], ptr [[OUT]], align 8
; CHECK-NEXT: br label %[[LATCH]]
; CHECK: [[LATCH]]:
; CHECK-NEXT: [[ACC_NEXT]] = phi double [ [[ACC_BODY]], %[[BODY]] ], [ [[ACC]], %[[HEADER]] ]
diff --git a/llvm/test/Transforms/SLPVectorizer/AArch64/fma-chain-no-alt-node-reduction.ll b/llvm/test/Transforms/SLPVectorizer/AArch64/fma-chain-no-alt-node-reduction.ll
index 66c89d60e23d9..c08ce3e4dd298 100644
--- a/llvm/test/Transforms/SLPVectorizer/AArch64/fma-chain-no-alt-node-reduction.ll
+++ b/llvm/test/Transforms/SLPVectorizer/AArch64/fma-chain-no-alt-node-reduction.ll
@@ -33,6 +33,7 @@ define void @density(ptr %lcs, ptr %liscom, ptr %densi, ptr %walls, i64 %first,
; CHECK-NEXT: [[P72:%.*]] = getelementptr inbounds nuw i8, ptr [[WALLS]], i64 72
; CHECK-NEXT: [[G11:%.*]] = load double, ptr [[P72]], align 8
; CHECK-NEXT: [[P104:%.*]] = getelementptr inbounds nuw i8, ptr [[WALLS]], i64 104
+; CHECK-NEXT: [[G22:%.*]] = load double, ptr [[P104]], align 8
; CHECK-NEXT: [[P136:%.*]] = getelementptr inbounds nuw i8, ptr [[WALLS]], i64 136
; CHECK-NEXT: [[G33:%.*]] = load double, ptr [[P136]], align 8
; CHECK-NEXT: [[P96:%.*]] = getelementptr inbounds nuw i8, ptr [[WALLS]], i64 96
@@ -41,23 +42,25 @@ define void @density(ptr %lcs, ptr %liscom, ptr %densi, ptr %walls, i64 %first,
; CHECK-NEXT: [[W80:%.*]] = load double, ptr [[P80]], align 8
; CHECK-NEXT: [[G12D:%.*]] = fadd fast double [[W80]], [[W96]]
; CHECK-NEXT: [[P120:%.*]] = getelementptr inbounds nuw i8, ptr [[WALLS]], i64 120
+; CHECK-NEXT: [[W120:%.*]] = load double, ptr [[P120]], align 8
; CHECK-NEXT: [[P88:%.*]] = getelementptr inbounds nuw i8, ptr [[WALLS]], i64 88
; CHECK-NEXT: [[W88:%.*]] = load double, ptr [[P88]], align 8
-; CHECK-NEXT: [[TMP0:%.*]] = load <2 x double>, ptr [[P120]], align 8
-; CHECK-NEXT: [[TMP8:%.*]] = load <2 x double>, ptr [[P104]], align 8
-; CHECK-NEXT: [[TMP18:%.*]] = insertelement <2 x double> [[TMP8]], double [[W88]], i64 0
-; CHECK-NEXT: [[TMP19:%.*]] = fadd fast <2 x double> [[TMP18]], [[TMP0]]
-; CHECK-NEXT: [[TMP4:%.*]] = shufflevector <2 x double> [[TMP8]], <2 x double> poison, <2 x i32> <i32 poison, i32 0>
-; CHECK-NEXT: [[TMP21:%.*]] = insertelement <2 x double> poison, double [[XI]], i64 0
-; CHECK-NEXT: [[TMP1:%.*]] = insertelement <2 x double> [[TMP21]], double [[YI]], i64 1
+; CHECK-NEXT: [[G13D:%.*]] = fadd fast double [[W88]], [[W120]]
+; CHECK-NEXT: [[P128:%.*]] = getelementptr inbounds nuw i8, ptr [[WALLS]], i64 128
+; CHECK-NEXT: [[W128:%.*]] = load double, ptr [[P128]], align 8
+; CHECK-NEXT: [[P112:%.*]] = getelementptr inbounds nuw i8, ptr [[WALLS]], i64 112
+; CHECK-NEXT: [[W112:%.*]] = load double, ptr [[P112]], align 8
+; CHECK-NEXT: [[G23D:%.*]] = fadd fast double [[W112]], [[W128]]
+; CHECK-NEXT: [[TMP0:%.*]] = insertelement <2 x double> poison, double [[YI]], i64 0
+; CHECK-NEXT: [[TMP1:%.*]] = insertelement <2 x double> [[TMP0]], double [[ZI]], i64 1
; CHECK-NEXT: [[TMP2:%.*]] = insertelement <2 x double> poison, double [[PBC]], i64 0
; CHECK-NEXT: [[TMP3:%.*]] = shufflevector <2 x double> [[TMP2]], <2 x double> poison, <2 x i32> zeroinitializer
-; CHECK-NEXT: [[TMP22:%.*]] = insertelement <2 x double> poison, double [[BX]], i64 0
-; CHECK-NEXT: [[TMP5:%.*]] = insertelement <2 x double> [[TMP22]], double [[BY]], i64 1
+; CHECK-NEXT: [[TMP4:%.*]] = insertelement <2 x double> poison, double [[BY]], i64 0
+; CHECK-NEXT: [[TMP5:%.*]] = insertelement <2 x double> [[TMP4]], double [[BZ]], i64 1
; CHECK-NEXT: [[TMP6:%.*]] = insertelement <2 x double> poison, double [[NPBC]], i64 0
; CHECK-NEXT: [[TMP7:%.*]] = shufflevector <2 x double> [[TMP6]], <2 x double> poison, <2 x i32> zeroinitializer
-; CHECK-NEXT: [[TMP23:%.*]] = insertelement <2 x double> poison, double [[G11]], i64 0
-; CHECK-NEXT: [[TMP9:%.*]] = insertelement <2 x double> [[TMP23]], double [[G12D]], i64 1
+; CHECK-NEXT: [[TMP8:%.*]] = insertelement <2 x double> poison, double [[G22]], i64 0
+; CHECK-NEXT: [[TMP9:%.*]] = insertelement <2 x double> [[TMP8]], double [[G23D]], i64 1
; CHECK-NEXT: br label %[[LOOP:.*]]
; CHECK: [[LOOP]]:
; CHECK-NEXT: [[IV:%.*]] = phi i64 [ [[FIRST]], %[[ENTRY]] ], [ [[IV_NEXT:%.*]], %[[LATCH:.*]] ]
@@ -69,16 +72,17 @@ define void @density(ptr %lcs, ptr %liscom, ptr %densi, ptr %walls, i64 %first,
; CHECK-NEXT: [[J_IDX:%.*]] = mul nsw i64 [[J]], 24
; CHECK-NEXT: [[LCS_J:%.*]] = getelementptr i8, ptr [[LCS]], i64 [[J_IDX]]
; CHECK-NEXT: [[PX:%.*]] = getelementptr i8, ptr [[LCS_J]], i64 48
-; CHECK-NEXT: [[PZ:%.*]] = getelementptr i8, ptr [[LCS_J]], i64 64
-; CHECK-NEXT: [[ZJ:%.*]] = load double, ptr [[PZ]], align 8
-; CHECK-NEXT: [[DX:%.*]] = fsub fast double [[ZI]], [[ZJ]]
+; CHECK-NEXT: [[XJ:%.*]] = load double, ptr [[PX]], align 8
+; CHECK-NEXT: [[DX:%.*]] = fsub fast double [[XI]], [[XJ]]
; CHECK-NEXT: [[CX:%.*]] = fcmp fast ogt double [[DX]], [[PBC]]
-; CHECK-NEXT: [[SX:%.*]] = select ninf nsz i1 [[CX]], double [[BZ]], double 0.000000e+00
+; CHECK-NEXT: [[SX:%.*]] = select ninf nsz i1 [[CX]], double [[BX]], double 0.000000e+00
; CHECK-NEXT: [[DX1:%.*]] = fsub reassoc nsz arcp contract afn double [[DX]], [[SX]]
; CHECK-NEXT: [[CX2:%.*]] = fcmp fast olt double [[DX1]], [[NPBC]]
-; CHECK-NEXT: [[DX2:%.*]] = fadd fast double [[DX1]], [[BZ]]
+; CHECK-NEXT: [[DX2:%.*]] = fadd fast double [[DX1]], [[BX]]
; CHECK-NEXT: [[XIJ:%.*]] = select nsz i1 [[CX2]], double [[DX2]], double [[DX1]]
-; CHECK-NEXT: [[TMP10:%.*]] = load <2 x double>, ptr [[PX]], align 8
+; CHECK-NEXT: [[PY:%.*]] = getelementptr i8, ptr [[LCS_J]], i64 56
+; CHECK-NEXT: [[M1:%.*]] = fmul fast double [[XIJ]], [[G11]]
+; CHECK-NEXT: [[TMP10:%.*]] = load <2 x double>, ptr [[PY]], align 8
; CHECK-NEXT: [[TMP11:%.*]] = fsub fast <2 x double> [[TMP1]], [[TMP10]]
; CHECK-NEXT: [[TMP12:%.*]] = fcmp fast ogt <2 x double> [[TMP11]], [[TMP3]]
; CHECK-NEXT: [[TMP13:%.*]] = select <2 x i1> [[TMP12]], <2 x double> [[TMP5]], <2 x double> zeroinitializer
@@ -86,22 +90,19 @@ define void @density(ptr %lcs, ptr %liscom, ptr %densi, ptr %walls, i64 %first,
; CHECK-NEXT: [[TMP15:%.*]] = fcmp fast olt <2 x double> [[TMP14]], [[TMP7]]
; CHECK-NEXT: [[TMP16:%.*]] = fadd fast <2 x double> [[TMP14]], [[TMP5]]
; CHECK-NEXT: [[TMP17:%.*]] = select <2 x i1> [[TMP15]], <2 x double> [[TMP16]], <2 x double> [[TMP14]]
+; CHECK-NEXT: [[TMP18:%.*]] = extractelement <2 x double> [[TMP17]], i64 0
+; CHECK-NEXT: [[M2:%.*]] = fmul fast double [[TMP18]], [[G12D]]
+; CHECK-NEXT: [[A1:%.*]] = fadd fast double [[M2]], [[M1]]
+; CHECK-NEXT: [[TMP19:%.*]] = extractelement <2 x double> [[TMP17]], i64 1
+; CHECK-NEXT: [[M3:%.*]] = fmul fast double [[TMP19]], [[G13D]]
+; CHECK-NEXT: [[A2:%.*]] = fadd fast double [[A1]], [[M3]]
+; CHECK-NEXT: [[TX:%.*]] = fmul fast double [[A2]], [[XIJ]]
; CHECK-NEXT: [[TMP20:%.*]] = fmul fast <2 x double> [[TMP17]], [[TMP9]]
-; CHECK-NEXT: [[TMP24:%.*]] = shufflevector <2 x double> [[TMP17]], <2 x double> [[TMP20]], <2 x i32> <i32 3, i32 1>
-; CHECK-NEXT: [[TMP25:%.*]] = shufflevector <2 x double> [[TMP4]], <2 x double> [[TMP20]], <2 x i32> <i32 2, i32 1>
-; CHECK-NEXT: [[TMP26:%.*]] = fmul fast <2 x double> [[TMP24]], [[TMP25]]
-; CHECK-NEXT: [[TMP27:%.*]] = fadd fast <2 x double> [[TMP24]], [[TMP25]]
-; CHECK-NEXT: [[TMP28:%.*]] = shufflevector <2 x double> [[TMP26]], <2 x double> [[TMP27]], <2 x i32> <i32 2, i32 1>
-; CHECK-NEXT: [[TMP29:%.*]] = insertelement <2 x double> poison, double [[XIJ]], i64 0
-; CHECK-NEXT: [[TMP30:%.*]] = shufflevector <2 x double> [[TMP29]], <2 x double> poison, <2 x i32> zeroinitializer
-; CHECK-NEXT: [[TMP31:%.*]] = fmul fast <2 x double> [[TMP30]], [[TMP19]]
-; CHECK-NEXT: [[TMP32:%.*]] = fadd fast <2 x double> [[TMP31]], [[TMP28]]
-; CHECK-NEXT: [[TMP33:%.*]] = fmul fast <2 x double> [[TMP32]], [[TMP17]]
-; CHECK-NEXT: [[ZZ:%.*]] = fmul fast double [[XIJ]], [[XIJ]]
+; CHECK-NEXT: [[TMP21:%.*]] = call fast double @llvm.vector.reduce.fadd.v2f64(double 0.000000e+00, <2 x double> [[TMP20]])
+; CHECK-NEXT: [[TY:%.*]] = fmul fast double [[TMP21]], [[TMP18]]
+; CHECK-NEXT: [[ZZ:%.*]] = fmul fast double [[TMP19]], [[TMP19]]
; CHECK-NEXT: [[TZ:%.*]] = fmul fast double [[ZZ]], [[G33]]
-; CHECK-NEXT: [[TY:%.*]] = extractelement <2 x double> [[TMP33]], i64 1
; CHECK-NEXT: [[A4:%.*]] = fadd fast double [[TY]], [[TZ]]
-; CHECK-NEXT: [[TX:%.*]] = extractelement <2 x double> [[TMP33]], i64 0
; CHECK-NEXT: [[RSQ:%.*]] = fadd fast double [[A4]], [[TX]]
; CHECK-NEXT: [[CMP:%.*]] = fcmp fast olt double [[RSQ]], [[RCUT]]
; CHECK-NEXT: br i1 [[CMP]], label %[[ACCUM:.*]], label %[[LATCH]]
diff --git a/llvm/test/Transforms/SLPVectorizer/AArch64/fmul-constant-lane-fmuladd-combine.ll b/llvm/test/Transforms/SLPVectorizer/AArch64/fmul-constant-lane-fmuladd-combine.ll
index e96a940de1733..38730e816a824 100644
--- a/llvm/test/Transforms/SLPVectorizer/AArch64/fmul-constant-lane-fmuladd-combine.ll
+++ b/llvm/test/Transforms/SLPVectorizer/AArch64/fmul-constant-lane-fmuladd-combine.ll
@@ -15,31 +15,33 @@ define void @fmul_constant_lane_rows(ptr %p, ptr %out, double %c0, double %c1, d
; CHECK-LABEL: define void @fmul_constant_lane_rows(
; CHECK-SAME: ptr [[P:%.*]], ptr [[OUT:%.*]], double [[C0:%.*]], double [[C1:%.*]], double [[C2:%.*]], double [[C3:%.*]], double [[C4:%.*]], double [[C5:%.*]], double [[C6:%.*]], double [[C7:%.*]], double [[C8:%.*]]) #[[ATTR0:[0-9]+]] {
; CHECK-NEXT: [[ENTRY:.*:]]
+; CHECK-NEXT: [[L0:%.*]] = load double, ptr [[P]], align 8
+; CHECK-NEXT: [[P1:%.*]] = getelementptr double, ptr [[P]], i64 1
+; CHECK-NEXT: [[L1:%.*]] = load double, ptr [[P1]], align 8
; CHECK-NEXT: [[P2:%.*]] = getelementptr double, ptr [[P]], i64 2
; CHECK-NEXT: [[L2:%.*]] = load double, ptr [[P2]], align 8
-; CHECK-NEXT: [[TMP0:%.*]] = load <2 x double>, ptr [[P]], align 8
-; CHECK-NEXT: [[TMP1:%.*]] = shufflevector <2 x double> [[TMP0]], <2 x double> <double undef, double -0.000000e+00>, <4 x i32> <i32 0, i32 0, i32 0, i32 3>
-; CHECK-NEXT: [[TMP2:%.*]] = insertelement <4 x double> <double poison, double poison, double poison, double 1.000000e+00>, double [[C0]], i64 0
-; CHECK-NEXT: [[TMP3:%.*]] = insertelement <4 x double> [[TMP2]], double [[C3]], i64 1
-; CHECK-NEXT: [[TMP4:%.*]] = insertelement <4 x double> [[TMP3]], double [[C6]], i64 2
-; CHECK-NEXT: [[TMP5:%.*]] = fmul fast <4 x double> [[TMP1]], [[TMP4]]
-; CHECK-NEXT: [[TMP6:%.*]] = shufflevector <2 x double> [[TMP0]], <2 x double> poison, <4 x i32> <i32 poison, i32 1, i32 poison, i32 poison>
-; CHECK-NEXT: [[TMP7:%.*]] = shufflevector <2 x double> [[TMP0]], <2 x double> poison, <4 x i32> <i32 0, i32 1, i32 poison, i32 poison>
-; CHECK-NEXT: [[TMP8:%.*]] = shufflevector <4 x double> <double poison, double 0.000000e+00, double poison, double poison>, <4 x double> [[TMP7]], <4 x i32> <i32 5, i32 1, i32 poison, i32 poison>
-; CHECK-NEXT: [[TMP9:%.*]] = shufflevector <4 x double> [[TMP8]], <4 x double> poison, <4 x i32> <i32 0, i32 0, i32 0, i32 1>
-; CHECK-NEXT: [[TMP10:%.*]] = insertelement <4 x double> <double poison, double poison, double poison, double 1.000000e+00>, double [[C1]], i64 0
-; CHECK-NEXT: [[TMP11:%.*]] = insertelement <4 x double> [[TMP10]], double [[C4]], i64 1
-; CHECK-NEXT: [[TMP12:%.*]] = insertelement <4 x double> [[TMP11]], double [[C7]], i64 2
-; CHECK-NEXT: [[TMP13:%.*]] = fmul fast <4 x double> [[TMP9]], [[TMP12]]
-; CHECK-NEXT: [[TMP14:%.*]] = fadd fast <4 x double> [[TMP13]], [[TMP5]]
-; CHECK-NEXT: [[TMP15:%.*]] = insertelement <4 x double> <double poison, double -0.000000e+00, double poison, double poison>, double [[L2]], i64 0
-; CHECK-NEXT: [[TMP16:%.*]] = shufflevector <4 x double> [[TMP15]], <4 x double> poison, <4 x i32> <i32 0, i32 0, i32 0, i32 1>
-; CHECK-NEXT: [[TMP17:%.*]] = insertelement <4 x double> <double poison, double poison, double poison, double 1.000000e+00>, double [[C2]], i64 0
-; CHECK-NEXT: [[TMP18:%.*]] = insertelement <4 x double> [[TMP17]], double [[C5]], i64 1
-; CHECK-NEXT: [[TMP19:%.*]] = insertelement <4 x double> [[TMP18]], double [[C8]], i64 2
-; CHECK-NEXT: [[TMP20:%.*]] = fmul fast <4 x double> [[TMP16]], [[TMP19]]
-; CHECK-NEXT: [[TMP21:%.*]] = fadd fast <4 x double> [[TMP14]], [[TMP20]]
-; CHECK-NEXT: store <4 x double> [[TMP21]], ptr [[OUT]], align 8
+; CHECK-NEXT: [[A00:%.*]] = fmul fast double [[L0]], [[C0]]
+; CHECK-NEXT: [[A01:%.*]] = fmul fast double [[L1]], [[C1]]
+; CHECK-NEXT: [[A02:%.*]] = fadd fast double [[A01]], [[A00]]
+; CHECK-NEXT: [[A03:%.*]] = fmul fast double [[L2]], [[C2]]
+; CHECK-NEXT: [[A:%.*]] = fadd fast double [[A02]], [[A03]]
+; CHECK-NEXT: [[B00:%.*]] = fmul fast double [[L0]], [[C3]]
+; CHECK-NEXT: [[B01:%.*]] = fmul fast double [[L1]], [[C4]]
+; CHECK-NEXT: [[B02:%.*]] = fadd fast double [[B01]], [[B00]]
+; CHECK-NEXT: [[B03:%.*]] = fmul fast double [[L2]], [[C5]]
+; CHECK-NEXT: [[B:%.*]] = fadd fast double [[B02]], [[B03]]
+; CHECK-NEXT: [[D00:%.*]] = fmul fast double [[L0]], [[C6]]
+; CHECK-NEXT: [[D01:%.*]] = fmul fast double [[L1]], [[C7]]
+; CHECK-NEXT: [[D02:%.*]] = fadd fast double [[D01]], [[D00]]
+; CHECK-NEXT: [[D03:%.*]] = fmul fast double [[L2]], [[C8]]
+; CHECK-NEXT: [[D:%.*]] = fadd fast double [[D02]], [[D03]]
+; CHECK-NEXT: store double [[A]], ptr [[OUT]], align 8
+; CHECK-NEXT: [[O1:%.*]] = getelementptr double, ptr [[OUT]], i64 1
+; CHECK-NEXT: store double [[B]], ptr [[O1]], align 8
+; CHECK-NEXT: [[O2:%.*]] = getelementptr double, ptr [[OUT]], i64 2
+; CHECK-NEXT: store double [[D]], ptr [[O2]], align 8
+; CHECK-NEXT: [[O3:%.*]] = getelementptr double, ptr [[OUT]], i64 3
+; CHECK-NEXT: store double 0.000000e+00, ptr [[O3]], align 8
; CHECK-NEXT: ret void
;
entry:
diff --git a/llvm/test/Transforms/SLPVectorizer/AArch64/loop-accumulator-reduction.ll b/llvm/test/Transforms/SLPVectorizer/AArch64/loop-accumulator-reduction.ll
index e567fa141a32d..f40ba6b7d9644 100644
--- a/llvm/test/Transforms/SLPVectorizer/AArch64/loop-accumulator-reduction.ll
+++ b/llvm/test/Transforms/SLPVectorizer/AArch64/loop-accumulator-reduction.ll
@@ -123,54 +123,50 @@ define double @loop_acc_fadd_extra_use(ptr %p, i64 %n, ptr %o) {
; CHECK-NEXT: br label %[[LOOP:.*]]
; CHECK: [[LOOP]]:
; CHECK-NEXT: [[IV:%.*]] = phi i64 [ 1, %[[ENTRY]] ], [ [[IV_NEXT:%.*]], %[[LOOP]] ]
-; CHECK-NEXT: [[ACC:%.*]] = phi double [ 0.000000e+00, %[[ENTRY]] ], [ [[TMP24:%.*]], %[[LOOP]] ]
+; CHECK-NEXT: [[ACC:%.*]] = phi double [ 0.000000e+00, %[[ENTRY]] ], [ [[SUM:%.*]], %[[LOOP]] ]
; CHECK-NEXT: [[IV3:%.*]] = mul nuw i64 [[IV]], 3
; CHECK-NEXT: [[P0:%.*]] = getelementptr double, ptr [[P]], i64 [[IV3]]
+; CHECK-NEXT: [[L0:%.*]] = load double, ptr [[P0]], align 8
; CHECK-NEXT: [[P1:%.*]] = getelementptr double, ptr [[P0]], i64 1
+; CHECK-NEXT: [[L1:%.*]] = load double, ptr [[P1]], align 8
; CHECK-NEXT: [[P2:%.*]] = getelementptr double, ptr [[P0]], i64 2
; CHECK-NEXT: [[L2:%.*]] = load double, ptr [[P2]], align 8
-; CHECK-NEXT: [[C8:%.*]] = load double, ptr getelementptr (double, ptr @cst, i64 8), align 8
-; CHECK-NEXT: [[L1:%.*]] = load double, ptr [[P1]], align 8
-; CHECK-NEXT: [[L0:%.*]] = load double, ptr [[P0]], align 8
-; CHECK-NEXT: [[C7:%.*]] = load double, ptr getelementptr (double, ptr @cst, i64 7), align 8
-; CHECK-NEXT: [[C6:%.*]] = load double, ptr getelementptr (double, ptr @cst, i64 6), align 8
-; CHECK-NEXT: [[C5:%.*]] = load double, ptr getelementptr (double, ptr @cst, i64 5), align 8
-; CHECK-NEXT: [[C4:%.*]] = load double, ptr getelementptr (double, ptr @cst, i64 4), align 8
-; CHECK-NEXT: [[C3:%.*]] = load double, ptr getelementptr (double, ptr @cst, i64 3), align 8
-; CHECK-NEXT: [[C2:%.*]] = load double, ptr getelementptr (double, ptr @cst, i64 2), align 8
-; CHECK-NEXT: [[C1:%.*]] = load double, ptr getelementptr (double, ptr @cst, i64 1), align 8
; CHECK-NEXT: [[C0:%.*]] = load double, ptr @cst, align 8
-; CHECK-NEXT: [[TMP0:%.*]] = insertelement <4 x double> <double poison, double -0.000000e+00, double poison, double poison>, double [[L0]], i64 0
-; CHECK-NEXT: [[TMP1:%.*]] = shufflevector <4 x double> [[TMP0]], <4 x double> poison, <4 x i32> <i32 0, i32 0, i32 0, i32 1>
-; CHECK-NEXT: [[TMP2:%.*]] = insertelement <4 x double> <double poison, double poison, double poison, double 1.000000e+00>, double [[C0]], i64 0
-; CHECK-NEXT: [[TMP3:%.*]] = insertelement <4 x double> [[TMP2]], double [[C3]], i64 1
-; CHECK-NEXT: [[TMP4:%.*]] = insertelement <4 x double> [[TMP3]], double [[C6]], i64 2
-; CHECK-NEXT: [[TMP5:%.*]] = fmul fast <4 x double> [[TMP1]], [[TMP4]]
-; CHECK-NEXT: [[TMP6:%.*]] = insertelement <4 x double> poison, double [[L1]], i64 0
-; CHECK-NEXT: [[TMP7:%.*]] = insertelement <4 x double> [[TMP6]], double [[ACC]], i64 1
-; CHECK-NEXT: [[TMP8:%.*]] = shufflevector <4 x double> [[TMP7]], <4 x double> poison, <4 x i32> <i32 0, i32 0, i32 0, i32 1>
-; CHECK-NEXT: [[TMP9:%.*]] = insertelement <4 x double> <double poison, double poison, double poison, double 1.000000e+00>, double [[C1]], i64 0
-; CHECK-NEXT: [[TMP10:%.*]] = insertelement <4 x double> [[TMP9]], double [[C4]], i64 1
-; CHECK-NEXT: [[TMP11:%.*]] = insertelement <4 x double> [[TMP10]], double [[C7]], i64 2
-; CHECK-NEXT: [[TMP12:%.*]] = fmul reassoc nsz arcp contract afn <4 x double> [[TMP8]], [[TMP11]]
-; CHECK-NEXT: [[TMP13:%.*]] = fadd reassoc nsz arcp contract afn <4 x double> [[TMP12]], [[TMP5]]
-; CHECK-NEXT: [[TMP14:%.*]] = insertelement <4 x double> <double poison, double -0.000000e+00, double poison, double poison>, double [[L2]], i64 0
-; CHECK-NEXT: [[TMP15:%.*]] = shufflevector <4 x double> [[TMP14]], <4 x double> poison, <4 x i32> <i32 0, i32 0, i32 0, i32 1>
-; CHECK-NEXT: [[TMP16:%.*]] = insertelement <4 x double> <double poison, double poison, double poison, double 1.000000e+00>, double [[C2]], i64 0
-; CHECK-NEXT: [[TMP17:%.*]] = insertelement <4 x double> [[TMP16]], double [[C5]], i64 1
-; CHECK-NEXT: [[TMP18:%.*]] = insertelement <4 x double> [[TMP17]], double [[C8]], i64 2
-; CHECK-NEXT: [[TMP19:%.*]] = fmul fast <4 x double> [[TMP15]], [[TMP18]]
-; CHECK-NEXT: [[TMP20:%.*]] = fadd reassoc nsz arcp contract afn <4 x double> [[TMP13]], [[TMP19]]
-; CHECK-NEXT: [[TMP21:%.*]] = shufflevector <4 x double> [[TMP20]], <4 x double> <double poison, double poison, double poison, double 1.000000e+00>, <4 x i32> <i32 0, i32 1, i32 poison, i32 7>
-; CHECK-NEXT: [[TMP22:%.*]] = shufflevector <4 x double> [[TMP21]], <4 x double> [[TMP20]], <4 x i32> <i32 0, i32 1, i32 6, i32 3>
-; CHECK-NEXT: [[TMP23:%.*]] = fmul <4 x double> [[TMP20]], [[TMP22]]
-; CHECK-NEXT: [[TMP24]] = call fast double @llvm.vector.reduce.fadd.v4f64(double 0.000000e+00, <4 x double> [[TMP23]])
+; CHECK-NEXT: [[C1:%.*]] = load double, ptr getelementptr (double, ptr @cst, i64 1), align 8
+; CHECK-NEXT: [[C2:%.*]] = load double, ptr getelementptr (double, ptr @cst, i64 2), align 8
+; CHECK-NEXT: [[C3:%.*]] = load double, ptr getelementptr (double, ptr @cst, i64 3), align 8
+; CHECK-NEXT: [[C4:%.*]] = load double, ptr getelementptr (double, ptr @cst, i64 4), align 8
+; CHECK-NEXT: [[C5:%.*]] = load double, ptr getelementptr (double, ptr @cst, i64 5), align 8
+; CHECK-NEXT: [[C6:%.*]] = load double, ptr getelementptr (double, ptr @cst, i64 6), align 8
+; CHECK-NEXT: [[C7:%.*]] = load double, ptr getelementptr (double, ptr @cst, i64 7), align 8
+; CHECK-NEXT: [[C8:%.*]] = load double, ptr getelementptr (double, ptr @cst, i64 8), align 8
+; CHECK-NEXT: [[A00:%.*]] = fmul fast double [[L0]], [[C0]]
+; CHECK-NEXT: [[A01:%.*]] = fmul fast double [[L1]], [[C1]]
+; CHECK-NEXT: [[A02:%.*]] = fadd fast double [[A01]], [[A00]]
+; CHECK-NEXT: [[A03:%.*]] = fmul fast double [[L2]], [[C2]]
+; CHECK-NEXT: [[A:%.*]] = fadd fast double [[A02]], [[A03]]
+; CHECK-NEXT: [[B00:%.*]] = fmul fast double [[L0]], [[C3]]
+; CHECK-NEXT: [[B01:%.*]] = fmul fast double [[L1]], [[C4]]
+; CHECK-NEXT: [[B02:%.*]] = fadd fast double [[B01]], [[B00]]
+; CHECK-NEXT: [[B03:%.*]] = fmul fast double [[L2]], [[C5]]
+; CHECK-NEXT: [[B:%.*]] = fadd fast double [[B02]], [[B03]]
+; CHECK-NEXT: [[D00:%.*]] = fmul fast double [[L0]], [[C6]]
+; CHECK-NEXT: [[D01:%.*]] = fmul fast double [[L1]], [[C7]]
+; CHECK-NEXT: [[D02:%.*]] = fadd fast double [[D01]], [[D00]]
+; CHECK-NEXT: [[D03:%.*]] = fmul fast double [[L2]], [[C8]]
+; CHECK-NEXT: [[D:%.*]] = fadd fast double [[D02]], [[D03]]
+; CHECK-NEXT: [[SA:%.*]] = fmul double [[A]], [[A]]
+; CHECK-NEXT: [[T1:%.*]] = fadd fast double [[SA]], [[ACC]]
+; CHECK-NEXT: [[SB:%.*]] = fmul double [[B]], [[B]]
+; CHECK-NEXT: [[T2:%.*]] = fadd fast double [[T1]], [[SB]]
+; CHECK-NEXT: [[SD:%.*]] = fmul double [[D]], [[D]]
+; CHECK-NEXT: [[SUM]] = fadd fast double [[T2]], [[SD]]
; CHECK-NEXT: store double [[ACC]], ptr [[O]], align 8
; CHECK-NEXT: [[IV_NEXT]] = add nuw nsw i64 [[IV]], 1
; CHECK-NEXT: [[CMP:%.*]] = icmp eq i64 [[IV]], [[N]]
; CHECK-NEXT: br i1 [[CMP]], label %[[EXIT:.*]], label %[[LOOP]]
; CHECK: [[EXIT]]:
-; CHECK-NEXT: [[RES:%.*]] = phi double [ [[TMP24]], %[[LOOP]] ]
+; CHECK-NEXT: [[RES:%.*]] = phi double [ [[SUM]], %[[LOOP]] ]
; CHECK-NEXT: ret double [[RES]]
;
entry:
@@ -403,45 +399,41 @@ define double @root_stored(ptr %p, i64 %n, ptr %o) {
; CHECK-NEXT: [[ACC:%.*]] = phi double [ 0.000000e+00, %[[ENTRY]] ], [ [[TMP23:%.*]], %[[LOOP]] ]
; CHECK-NEXT: [[IV3:%.*]] = mul nuw i64 [[IV]], 3
; CHECK-NEXT: [[P0:%.*]] = getelementptr double, ptr [[P]], i64 [[IV3]]
+; CHECK-NEXT: [[L0:%.*]] = load double, ptr [[P0]], align 8
; CHECK-NEXT: [[P1:%.*]] = getelementptr double, ptr [[P0]], i64 1
+; CHECK-NEXT: [[L1:%.*]] = load double, ptr [[P1]], align 8
; CHECK-NEXT: [[P2:%.*]] = getelementptr double, ptr [[P0]], i64 2
; CHECK-NEXT: [[L2:%.*]] = load double, ptr [[P2]], align 8
-; CHECK-NEXT: [[C8:%.*]] = load double, ptr getelementptr (double, ptr @cst, i64 8), align 8
-; CHECK-NEXT: [[L1:%.*]] = load double, ptr [[P1]], align 8
-; CHECK-NEXT: [[L0:%.*]] = load double, ptr [[P0]], align 8
-; CHECK-NEXT: [[C7:%.*]] = load double, ptr getelementptr (double, ptr @cst, i64 7), align 8
-; CHECK-NEXT: [[C6:%.*]] = load double, ptr getelementptr (double, ptr @cst, i64 6), align 8
-; CHECK-NEXT: [[C5:%.*]] = load double, ptr getelementptr (double, ptr @cst, i64 5), align 8
-; CHECK-NEXT: [[C4:%.*]] = load double, ptr getelementptr (double, ptr @cst, i64 4), align 8
-; CHECK-NEXT: [[C3:%.*]] = load double, ptr getelementptr (double, ptr @cst, i64 3), align 8
-; CHECK-NEXT: [[C2:%.*]] = load double, ptr getelementptr (double, ptr @cst, i64 2), align 8
-; CHECK-NEXT: [[C1:%.*]] = load double, ptr getelementptr (double, ptr @cst, i64 1), align 8
; CHECK-NEXT: [[C0:%.*]] = load double, ptr @cst, align 8
-; CHECK-NEXT: [[TMP0:%.*]] = insertelement <4 x double> <double poison, double -0.000000e+00, double poison, double poison>, double [[L0]], i64 0
-; CHECK-NEXT: [[TMP1:%.*]] = shufflevector <4 x double> [[TMP0]], <4 x double> poison, <4 x i32> <i32 0, i32 0, i32 0, i32 1>
-; CHECK-NEXT: [[TMP2:%.*]] = insertelement <4 x double> <double poison, double poison, double poison, double 1.000000e+00>, double [[C0]], i64 0
-; CHECK-NEXT: [[TMP3:%.*]] = insertelement <4 x double> [[TMP2]], double [[C3]], i64 1
-; CHECK-NEXT: [[TMP4:%.*]] = insertelement <4 x double> [[TMP3]], double [[C6]], i64 2
-; CHECK-NEXT: [[TMP5:%.*]] = fmul fast <4 x double> [[TMP1]], [[TMP4]]
-; CHECK-NEXT: [[TMP6:%.*]] = insertelement <4 x double> poison, double [[L1]], i64 0
-; CHECK-NEXT: [[TMP7:%.*]] = insertelement <4 x double> [[TMP6]], double [[ACC]], i64 1
-; CHECK-NEXT: [[TMP8:%.*]] = shufflevector <4 x double> [[TMP7]], <4 x double> poison, <4 x i32> <i32 0, i32 0, i32 0, i32 1>
-; CHECK-NEXT: [[TMP9:%.*]] = insertelement <4 x double> <double poison, double poison, double poison, double 1.000000e+00>, double [[C1]], i64 0
-; CHECK-NEXT: [[TMP10:%.*]] = insertelement <4 x double> [[TMP9]], double [[C4]], i64 1
-; CHECK-NEXT: [[TMP11:%.*]] = insertelement <4 x double> [[TMP10]], double [[C7]], i64 2
-; CHECK-NEXT: [[TMP12:%.*]] = fmul reassoc nsz arcp contract afn <4 x double> [[TMP8]], [[TMP11]]
-; CHECK-NEXT: [[TMP13:%.*]] = fadd reassoc nsz arcp contract afn <4 x double> [[TMP12]], [[TMP5]]
-; CHECK-NEXT: [[TMP14:%.*]] = insertelement <4 x double> <double poison, double -0.000000e+00, double poison, double poison>, double [[L2]], i64 0
-; CHECK-NEXT: [[TMP15:%.*]] = shufflevector <4 x double> [[TMP14]], <4 x double> poison, <4 x i32> <i32 0, i32 0, i32 0, i32 1>
-; CHECK-NEXT: [[TMP16:%.*]] = insertelement <4 x double> <double poison, double poison, double poison, double 1.000000e+00>, double [[C2]], i64 0
-; CHECK-NEXT: [[TMP17:%.*]] = insertelement <4 x double> [[TMP16]], double [[C5]], i64 1
-; CHECK-NEXT: [[TMP18:%.*]] = insertelement <4 x double> [[TMP17]], double [[C8]], i64 2
-; CHECK-NEXT: [[TMP19:%.*]] = fmul fast <4 x double> [[TMP15]], [[TMP18]]
-; CHECK-NEXT: [[TMP20:%.*]] = fadd reassoc nsz arcp contract afn <4 x double> [[TMP13]], [[TMP19]]
-; CHECK-NEXT: [[TMP21:%.*]] = shufflevector <4 x double> [[TMP20]], <4 x double> <double poison, double poison, double poison, double 1.000000e+00>, <4 x i32> <i32 0, i32 1, i32 poison, i32 7>
-; CHECK-NEXT: [[TMP22:%.*]] = shufflevector <4 x double> [[TMP21]], <4 x double> [[TMP20]], <4 x i32> <i32 0, i32 1, i32 6, i32 3>
-; CHECK-NEXT: [[TMP24:%.*]] = fmul <4 x double> [[TMP20]], [[TMP22]]
-; CHECK-NEXT: [[TMP23]] = call fast double @llvm.vector.reduce.fadd.v4f64(double 0.000000e+00, <4 x double> [[TMP24]])
+; CHECK-NEXT: [[C1:%.*]] = load double, ptr getelementptr (double, ptr @cst, i64 1), align 8
+; CHECK-NEXT: [[C2:%.*]] = load double, ptr getelementptr (double, ptr @cst, i64 2), align 8
+; CHECK-NEXT: [[C3:%.*]] = load double, ptr getelementptr (double, ptr @cst, i64 3), align 8
+; CHECK-NEXT: [[C4:%.*]] = load double, ptr getelementptr (double, ptr @cst, i64 4), align 8
+; CHECK-NEXT: [[C5:%.*]] = load double, ptr getelementptr (double, ptr @cst, i64 5), align 8
+; CHECK-NEXT: [[C6:%.*]] = load double, ptr getelementptr (double, ptr @cst, i64 6), align 8
+; CHECK-NEXT: [[C7:%.*]] = load double, ptr getelementptr (double, ptr @cst, i64 7), align 8
+; CHECK-NEXT: [[C8:%.*]] = load double, ptr getelementptr (double, ptr @cst, i64 8), align 8
+; CHECK-NEXT: [[A00:%.*]] = fmul fast double [[L0]], [[C0]]
+; CHECK-NEXT: [[A01:%.*]] = fmul fast double [[L1]], [[C1]]
+; CHECK-NEXT: [[A02:%.*]] = fadd fast double [[A01]], [[A00]]
+; CHECK-NEXT: [[A03:%.*]] = fmul fast double [[L2]], [[C2]]
+; CHECK-NEXT: [[A:%.*]] = fadd fast double [[A02]], [[A03]]
+; CHECK-NEXT: [[B00:%.*]] = fmul fast double [[L0]], [[C3]]
+; CHECK-NEXT: [[B01:%.*]] = fmul fast double [[L1]], [[C4]]
+; CHECK-NEXT: [[B02:%.*]] = fadd fast double [[B01]], [[B00]]
+; CHECK-NEXT: [[B03:%.*]] = fmul fast double [[L2]], [[C5]]
+; CHECK-NEXT: [[B:%.*]] = fadd fast double [[B02]], [[B03]]
+; CHECK-NEXT: [[D00:%.*]] = fmul fast double [[L0]], [[C6]]
+; CHECK-NEXT: [[D01:%.*]] = fmul fast double [[L1]], [[C7]]
+; CHECK-NEXT: [[D02:%.*]] = fadd fast double [[D01]], [[D00]]
+; CHECK-NEXT: [[D03:%.*]] = fmul fast double [[L2]], [[C8]]
+; CHECK-NEXT: [[D:%.*]] = fadd fast double [[D02]], [[D03]]
+; CHECK-NEXT: [[SA:%.*]] = fmul double [[A]], [[A]]
+; CHECK-NEXT: [[T1:%.*]] = fadd fast double [[SA]], [[ACC]]
+; CHECK-NEXT: [[SB:%.*]] = fmul double [[B]], [[B]]
+; CHECK-NEXT: [[T2:%.*]] = fadd fast double [[T1]], [[SB]]
+; CHECK-NEXT: [[SD:%.*]] = fmul double [[D]], [[D]]
+; CHECK-NEXT: [[TMP23]] = fadd fast double [[T2]], [[SD]]
; CHECK-NEXT: store double [[TMP23]], ptr [[O]], align 8
; CHECK-NEXT: [[IV_NEXT]] = add nuw nsw i64 [[IV]], 1
; CHECK-NEXT: [[CMP:%.*]] = icmp eq i64 [[IV]], [[N]]
@@ -512,54 +504,50 @@ define double @two_exit_phis(ptr %p, i64 %n, i1 %c) {
; CHECK-NEXT: br label %[[LOOP:.*]]
; CHECK: [[LOOP]]:
; CHECK-NEXT: [[IV:%.*]] = phi i64 [ 1, %[[ENTRY]] ], [ [[IV_NEXT:%.*]], %[[LOOP]] ]
-; CHECK-NEXT: [[SD:%.*]] = phi double [ 0.000000e+00, %[[ENTRY]] ], [ [[SUM:%.*]], %[[LOOP]] ]
+; CHECK-NEXT: [[ACC:%.*]] = phi double [ 0.000000e+00, %[[ENTRY]] ], [ [[SUM1:%.*]], %[[LOOP]] ]
; CHECK-NEXT: [[IV3:%.*]] = mul nuw i64 [[IV]], 3
; CHECK-NEXT: [[P0:%.*]] = getelementptr double, ptr [[P]], i64 [[IV3]]
+; CHECK-NEXT: [[L0:%.*]] = load double, ptr [[P0]], align 8
; CHECK-NEXT: [[P1:%.*]] = getelementptr double, ptr [[P0]], i64 1
+; CHECK-NEXT: [[L1:%.*]] = load double, ptr [[P1]], align 8
; CHECK-NEXT: [[P2:%.*]] = getelementptr double, ptr [[P0]], i64 2
; CHECK-NEXT: [[L2:%.*]] = load double, ptr [[P2]], align 8
-; CHECK-NEXT: [[C8:%.*]] = load double, ptr getelementptr (double, ptr @cst, i64 8), align 8
-; CHECK-NEXT: [[L1:%.*]] = load double, ptr [[P1]], align 8
-; CHECK-NEXT: [[L0:%.*]] = load double, ptr [[P0]], align 8
-; CHECK-NEXT: [[C7:%.*]] = load double, ptr getelementptr (double, ptr @cst, i64 7), align 8
-; CHECK-NEXT: [[C6:%.*]] = load double, ptr getelementptr (double, ptr @cst, i64 6), align 8
-; CHECK-NEXT: [[C5:%.*]] = load double, ptr getelementptr (double, ptr @cst, i64 5), align 8
-; CHECK-NEXT: [[C4:%.*]] = load double, ptr getelementptr (double, ptr @cst, i64 4), align 8
-; CHECK-NEXT: [[C3:%.*]] = load double, ptr getelementptr (double, ptr @cst, i64 3), align 8
-; CHECK-NEXT: [[C2:%.*]] = load double, ptr getelementptr (double, ptr @cst, i64 2), align 8
-; CHECK-NEXT: [[C1:%.*]] = load double, ptr getelementptr (double, ptr @cst, i64 1), align 8
; CHECK-NEXT: [[C0:%.*]] = load double, ptr @cst, align 8
-; CHECK-NEXT: [[TMP0:%.*]] = insertelement <4 x double> <double poison, double -0.000000e+00, double poison, double poison>, double [[L0]], i64 0
-; CHECK-NEXT: [[TMP1:%.*]] = shufflevector <4 x double> [[TMP0]], <4 x double> poison, <4 x i32> <i32 0, i32 0, i32 0, i32 1>
-; CHECK-NEXT: [[TMP2:%.*]] = insertelement <4 x double> <double poison, double poison, double poison, double 1.000000e+00>, double [[C0]], i64 0
-; CHECK-NEXT: [[TMP3:%.*]] = insertelement <4 x double> [[TMP2]], double [[C3]], i64 1
-; CHECK-NEXT: [[TMP4:%.*]] = insertelement <4 x double> [[TMP3]], double [[C6]], i64 2
-; CHECK-NEXT: [[TMP5:%.*]] = fmul fast <4 x double> [[TMP1]], [[TMP4]]
-; CHECK-NEXT: [[TMP6:%.*]] = insertelement <4 x double> <double poison, double 0.000000e+00, double poison, double poison>, double [[L1]], i64 0
-; CHECK-NEXT: [[TMP7:%.*]] = shufflevector <4 x double> [[TMP6]], <4 x double> poison, <4 x i32> <i32 0, i32 0, i32 0, i32 1>
-; CHECK-NEXT: [[TMP8:%.*]] = insertelement <4 x double> <double poison, double poison, double poison, double 1.000000e+00>, double [[C1]], i64 0
-; CHECK-NEXT: [[TMP9:%.*]] = insertelement <4 x double> [[TMP8]], double [[C4]], i64 1
-; CHECK-NEXT: [[TMP10:%.*]] = insertelement <4 x double> [[TMP9]], double [[C7]], i64 2
-; CHECK-NEXT: [[TMP11:%.*]] = fmul fast <4 x double> [[TMP7]], [[TMP10]]
-; CHECK-NEXT: [[TMP12:%.*]] = fadd fast <4 x double> [[TMP11]], [[TMP5]]
-; CHECK-NEXT: [[TMP13:%.*]] = insertelement <4 x double> <double poison, double -0.000000e+00, double poison, double poison>, double [[L2]], i64 0
-; CHECK-NEXT: [[TMP14:%.*]] = shufflevector <4 x double> [[TMP13]], <4 x double> poison, <4 x i32> <i32 0, i32 0, i32 0, i32 1>
-; CHECK-NEXT: [[TMP15:%.*]] = insertelement <4 x double> <double poison, double poison, double poison, double 1.000000e+00>, double [[C2]], i64 0
-; CHECK-NEXT: [[TMP16:%.*]] = insertelement <4 x double> [[TMP15]], double [[C5]], i64 1
-; CHECK-NEXT: [[TMP17:%.*]] = insertelement <4 x double> [[TMP16]], double [[C8]], i64 2
-; CHECK-NEXT: [[TMP18:%.*]] = fmul fast <4 x double> [[TMP14]], [[TMP17]]
-; CHECK-NEXT: [[TMP19:%.*]] = fadd fast <4 x double> [[TMP12]], [[TMP18]]
-; CHECK-NEXT: [[TMP20:%.*]] = shufflevector <4 x double> [[TMP19]], <4 x double> <double poison, double poison, double poison, double 1.000000e+00>, <4 x i32> <i32 0, i32 1, i32 poison, i32 7>
-; CHECK-NEXT: [[TMP21:%.*]] = shufflevector <4 x double> [[TMP20]], <4 x double> [[TMP19]], <4 x i32> <i32 0, i32 1, i32 6, i32 3>
-; CHECK-NEXT: [[TMP22:%.*]] = fmul <4 x double> [[TMP19]], [[TMP21]]
-; CHECK-NEXT: [[T2:%.*]] = call fast double @llvm.vector.reduce.fadd.v4f64(double 0.000000e+00, <4 x double> [[TMP22]])
-; CHECK-NEXT: [[SUM]] = fadd fast double [[T2]], [[SD]]
+; CHECK-NEXT: [[C1:%.*]] = load double, ptr getelementptr (double, ptr @cst, i64 1), align 8
+; CHECK-NEXT: [[C2:%.*]] = load double, ptr getelementptr (double, ptr @cst, i64 2), align 8
+; CHECK-NEXT: [[C3:%.*]] = load double, ptr getelementptr (double, ptr @cst, i64 3), align 8
+; CHECK-NEXT: [[C4:%.*]] = load double, ptr getelementptr (double, ptr @cst, i64 4), align 8
+; CHECK-NEXT: [[C5:%.*]] = load double, ptr getelementptr (double, ptr @cst, i64 5), align 8
+; CHECK-NEXT: [[C6:%.*]] = load double, ptr getelementptr (double, ptr @cst, i64 6), align 8
+; CHECK-NEXT: [[C7:%.*]] = load double, ptr getelementptr (double, ptr @cst, i64 7), align 8
+; CHECK-NEXT: [[C8:%.*]] = load double, ptr getelementptr (double, ptr @cst, i64 8), align 8
+; CHECK-NEXT: [[SD:%.*]] = fmul fast double [[L0]], [[C0]]
+; CHECK-NEXT: [[T2:%.*]] = fmul fast double [[L1]], [[C1]]
+; CHECK-NEXT: [[SUM:%.*]] = fadd fast double [[T2]], [[SD]]
+; CHECK-NEXT: [[A03:%.*]] = fmul fast double [[L2]], [[C2]]
+; CHECK-NEXT: [[A:%.*]] = fadd fast double [[SUM]], [[A03]]
+; CHECK-NEXT: [[B00:%.*]] = fmul fast double [[L0]], [[C3]]
+; CHECK-NEXT: [[B01:%.*]] = fmul fast double [[L1]], [[C4]]
+; CHECK-NEXT: [[B02:%.*]] = fadd fast double [[B01]], [[B00]]
+; CHECK-NEXT: [[B03:%.*]] = fmul fast double [[L2]], [[C5]]
+; CHECK-NEXT: [[B:%.*]] = fadd fast double [[B02]], [[B03]]
+; CHECK-NEXT: [[D00:%.*]] = fmul fast double [[L0]], [[C6]]
+; CHECK-NEXT: [[D01:%.*]] = fmul fast double [[L1]], [[C7]]
+; CHECK-NEXT: [[D02:%.*]] = fadd fast double [[D01]], [[D00]]
+; CHECK-NEXT: [[D03:%.*]] = fmul fast double [[L2]], [[C8]]
+; CHECK-NEXT: [[D:%.*]] = fadd fast double [[D02]], [[D03]]
+; CHECK-NEXT: [[SA:%.*]] = fmul double [[A]], [[A]]
+; CHECK-NEXT: [[T1:%.*]] = fadd fast double [[SA]], [[ACC]]
+; CHECK-NEXT: [[SB:%.*]] = fmul double [[B]], [[B]]
+; CHECK-NEXT: [[T3:%.*]] = fadd fast double [[T1]], [[SB]]
+; CHECK-NEXT: [[SD1:%.*]] = fmul double [[D]], [[D]]
+; CHECK-NEXT: [[SUM1]] = fadd fast double [[T3]], [[SD1]]
; CHECK-NEXT: [[IV_NEXT]] = add nuw nsw i64 [[IV]], 1
; CHECK-NEXT: [[CMP:%.*]] = icmp eq i64 [[IV]], [[N]]
; CHECK-NEXT: br i1 [[CMP]], label %[[EXIT:.*]], label %[[LOOP]]
; CHECK: [[EXIT]]:
-; CHECK-NEXT: [[TMP23:%.*]] = phi double [ [[SUM]], %[[LOOP]] ]
-; CHECK-NEXT: [[TMP24:%.*]] = phi double [ [[SUM]], %[[LOOP]] ]
+; CHECK-NEXT: [[TMP23:%.*]] = phi double [ [[SUM1]], %[[LOOP]] ]
+; CHECK-NEXT: [[TMP24:%.*]] = phi double [ [[SUM1]], %[[LOOP]] ]
; CHECK-NEXT: [[R:%.*]] = fadd double [[TMP23]], [[TMP24]]
; CHECK-NEXT: ret double [[R]]
;
diff --git a/llvm/test/Transforms/SLPVectorizer/AArch64/reduction-root-multi-use-fma.ll b/llvm/test/Transforms/SLPVectorizer/AArch64/reduction-root-multi-use-fma.ll
index 0b998f1efd678..cac4abf971d85 100644
--- a/llvm/test/Transforms/SLPVectorizer/AArch64/reduction-root-multi-use-fma.ll
+++ b/llvm/test/Transforms/SLPVectorizer/AArch64/reduction-root-multi-use-fma.ll
@@ -8,14 +8,15 @@
define double @fmul_reduction_root_multi_use(ptr %p, double %a, double %b) {
; CHECK-LABEL: define double @fmul_reduction_root_multi_use(
; CHECK-SAME: ptr [[P:%.*]], double [[A:%.*]], double [[B:%.*]]) #[[ATTR0:[0-9]+]] {
-; CHECK-NEXT: [[TMP1:%.*]] = load <2 x double>, ptr [[P]], align 8
-; CHECK-NEXT: [[TMP2:%.*]] = insertelement <2 x double> poison, double [[A]], i64 0
-; CHECK-NEXT: [[TMP3:%.*]] = insertelement <2 x double> [[TMP2]], double [[B]], i64 1
-; CHECK-NEXT: [[TMP4:%.*]] = fmul fast <2 x double> [[TMP1]], [[TMP3]]
-; CHECK-NEXT: [[R:%.*]] = call fast double @llvm.vector.reduce.fadd.v2f64(double 0.000000e+00, <2 x double> [[TMP4]])
+; CHECK-NEXT: [[P1:%.*]] = getelementptr inbounds double, ptr [[P]], i64 1
+; CHECK-NEXT: [[R:%.*]] = load double, ptr [[P]], align 8
+; CHECK-NEXT: [[L1:%.*]] = load double, ptr [[P1]], align 8
; CHECK-NEXT: [[U0:%.*]] = fmul fast double [[R]], [[A]]
-; CHECK-NEXT: [[U1:%.*]] = fmul fast double [[R]], [[B]]
-; CHECK-NEXT: [[RES:%.*]] = fdiv fast double [[U0]], [[U1]]
+; CHECK-NEXT: [[M1:%.*]] = fmul fast double [[L1]], [[B]]
+; CHECK-NEXT: [[R1:%.*]] = fadd fast double [[U0]], [[M1]]
+; CHECK-NEXT: [[U2:%.*]] = fmul fast double [[R1]], [[A]]
+; CHECK-NEXT: [[U1:%.*]] = fmul fast double [[R1]], [[B]]
+; CHECK-NEXT: [[RES:%.*]] = fdiv fast double [[U2]], [[U1]]
; CHECK-NEXT: ret double [[RES]]
;
%p1 = getelementptr inbounds double, ptr %p, i64 1
diff --git a/llvm/test/Transforms/SLPVectorizer/AArch64/vec3-reorder-reshuffle.ll b/llvm/test/Transforms/SLPVectorizer/AArch64/vec3-reorder-reshuffle.ll
index 5dbd7079be9ce..0d333ac022d1a 100644
--- a/llvm/test/Transforms/SLPVectorizer/AArch64/vec3-reorder-reshuffle.ll
+++ b/llvm/test/Transforms/SLPVectorizer/AArch64/vec3-reorder-reshuffle.ll
@@ -79,7 +79,9 @@ define void @extract_mask(ptr %object, double %conv503, double %conv520) {
; CHECK-NEXT: [[TMP6:%.*]] = select <2 x i1> [[TMP5]], <2 x double> [[TMP3]], <2 x double> <double 0.000000e+00, double -2.000000e+10>
; CHECK-NEXT: [[TMP7:%.*]] = fsub <2 x double> zeroinitializer, [[TMP6]]
; CHECK-NEXT: [[TMP8:%.*]] = fptrunc <2 x double> [[TMP7]] to <2 x float>
-; CHECK-NEXT: [[MUL646:%.*]] = call reassoc float @llvm.vector.reduce.fmul.v2f32(float 1.000000e+00, <2 x float> [[TMP8]])
+; CHECK-NEXT: [[TMP9:%.*]] = extractelement <2 x float> [[TMP8]], i64 0
+; CHECK-NEXT: [[TMP10:%.*]] = extractelement <2 x float> [[TMP8]], i64 1
+; CHECK-NEXT: [[MUL646:%.*]] = fmul float [[TMP9]], [[TMP10]]
; CHECK-NEXT: [[CMP663:%.*]] = fcmp olt float [[MUL646]], 0.000000e+00
; CHECK-NEXT: br i1 [[CMP663]], label [[IF_THEN665:%.*]], label [[IF_END668:%.*]]
; CHECK: if.then665:
diff --git a/llvm/test/Transforms/SLPVectorizer/NVPTX/ordered-reduction-fma-fusion.ll b/llvm/test/Transforms/SLPVectorizer/NVPTX/ordered-reduction-fma-fusion.ll
index a553fe126ee17..bad672c7743f6 100644
--- a/llvm/test/Transforms/SLPVectorizer/NVPTX/ordered-reduction-fma-fusion.ll
+++ b/llvm/test/Transforms/SLPVectorizer/NVPTX/ordered-reduction-fma-fusion.ll
@@ -9,10 +9,14 @@
define float @dot_contract(float %x) {
; CHECK-LABEL: @dot_contract(
-; CHECK-NEXT: [[TMP1:%.*]] = insertelement <4 x float> poison, float [[X:%.*]], i64 0
-; CHECK-NEXT: [[TMP2:%.*]] = shufflevector <4 x float> [[TMP1]], <4 x float> poison, <4 x i32> zeroinitializer
-; CHECK-NEXT: [[TMP3:%.*]] = fmul contract <4 x float> <float 7.000000e+00, float 3.000000e+00, float 5.000000e+00, float 9.000000e+00>, [[TMP2]]
-; CHECK-NEXT: [[TMP4:%.*]] = call contract float @llvm.vector.reduce.fadd.v4f32(float [[X]], <4 x float> [[TMP3]])
+; CHECK-NEXT: [[M0:%.*]] = fmul contract float 7.000000e+00, [[X:%.*]]
+; CHECK-NEXT: [[A0:%.*]] = fadd contract float [[M0]], [[X]]
+; CHECK-NEXT: [[M1:%.*]] = fmul contract float 3.000000e+00, [[X]]
+; CHECK-NEXT: [[A1:%.*]] = fadd contract float [[M1]], [[A0]]
+; CHECK-NEXT: [[M2:%.*]] = fmul contract float 5.000000e+00, [[X]]
+; CHECK-NEXT: [[A2:%.*]] = fadd contract float [[M2]], [[A1]]
+; CHECK-NEXT: [[M3:%.*]] = fmul contract float 9.000000e+00, [[X]]
+; CHECK-NEXT: [[TMP4:%.*]] = fadd contract float [[M3]], [[A2]]
; CHECK-NEXT: ret float [[TMP4]]
;
%m0 = fmul contract float 7.000000e+00, %x
diff --git a/llvm/test/Transforms/SLPVectorizer/X86/fmul-fused-into-scalar-fadd.ll b/llvm/test/Transforms/SLPVectorizer/X86/fmul-fused-into-scalar-fadd.ll
index edcfa0e997233..ab5bde661185b 100644
--- a/llvm/test/Transforms/SLPVectorizer/X86/fmul-fused-into-scalar-fadd.ll
+++ b/llvm/test/Transforms/SLPVectorizer/X86/fmul-fused-into-scalar-fadd.ll
@@ -12,19 +12,16 @@ define double @fmul_lhs_of_scalar_fadd(ptr %p, ptr %q, double %x, double %y, i1
; CHECK-LABEL: define double @fmul_lhs_of_scalar_fadd(
; CHECK-SAME: ptr [[P:%.*]], ptr [[Q:%.*]], double [[X:%.*]], double [[Y:%.*]], i1 [[C:%.*]]) #[[ATTR0:[0-9]+]] {
; CHECK-NEXT: [[ENTRY:.*:]]
-; CHECK-NEXT: [[A0:%.*]] = load double, ptr [[P]], align 8
-; CHECK-NEXT: [[P1:%.*]] = getelementptr inbounds double, ptr [[P]], i64 1
-; CHECK-NEXT: [[A1:%.*]] = load double, ptr [[P1]], align 8
-; CHECK-NEXT: [[B0:%.*]] = load double, ptr [[Q]], align 8
-; CHECK-NEXT: [[Q1:%.*]] = getelementptr inbounds double, ptr [[Q]], i64 1
-; CHECK-NEXT: [[B1:%.*]] = load double, ptr [[Q1]], align 8
-; CHECK-NEXT: [[M0:%.*]] = fmul contract double [[A0]], [[B0]]
-; CHECK-NEXT: [[M1:%.*]] = fmul contract double [[A1]], [[B1]]
+; CHECK-NEXT: [[TMP0:%.*]] = load <2 x double>, ptr [[P]], align 8
+; CHECK-NEXT: [[TMP1:%.*]] = load <2 x double>, ptr [[Q]], align 8
+; CHECK-NEXT: [[TMP2:%.*]] = fmul contract <2 x double> [[TMP0]], [[TMP1]]
; CHECK-NEXT: br i1 [[C]], label %[[T:.*]], label %[[F:.*]]
; CHECK: [[T]]:
+; CHECK-NEXT: [[M0:%.*]] = extractelement <2 x double> [[TMP2]], i64 0
; CHECK-NEXT: [[S0:%.*]] = fadd contract double [[M0]], [[X]]
; CHECK-NEXT: ret double [[S0]]
; CHECK: [[F]]:
+; CHECK-NEXT: [[M1:%.*]] = extractelement <2 x double> [[TMP2]], i64 1
; CHECK-NEXT: [[S1:%.*]] = fadd contract double [[M1]], [[Y]]
; CHECK-NEXT: ret double [[S1]]
;
@@ -126,12 +123,15 @@ define double @fmul_rhs_of_scalar_fadd_same_block(ptr %p, ptr %q, double %x, dou
; CHECK-LABEL: define double @fmul_rhs_of_scalar_fadd_same_block(
; CHECK-SAME: ptr [[P:%.*]], ptr [[Q:%.*]], double [[X:%.*]], double [[Y:%.*]], i1 [[C:%.*]]) #[[ATTR0]] {
; CHECK-NEXT: [[ENTRY:.*:]]
-; CHECK-NEXT: [[TMP0:%.*]] = load <2 x double>, ptr [[P]], align 8
-; CHECK-NEXT: [[TMP1:%.*]] = load <2 x double>, ptr [[Q]], align 8
-; CHECK-NEXT: [[TMP2:%.*]] = fmul contract <2 x double> [[TMP0]], [[TMP1]]
-; CHECK-NEXT: [[M0:%.*]] = extractelement <2 x double> [[TMP2]], i64 0
+; CHECK-NEXT: [[A0:%.*]] = load double, ptr [[P]], align 8
+; CHECK-NEXT: [[P1:%.*]] = getelementptr inbounds double, ptr [[P]], i64 1
+; CHECK-NEXT: [[A1:%.*]] = load double, ptr [[P1]], align 8
+; CHECK-NEXT: [[B0:%.*]] = load double, ptr [[Q]], align 8
+; CHECK-NEXT: [[Q1:%.*]] = getelementptr inbounds double, ptr [[Q]], i64 1
+; CHECK-NEXT: [[B1:%.*]] = load double, ptr [[Q1]], align 8
+; CHECK-NEXT: [[M0:%.*]] = fmul contract double [[A0]], [[B0]]
+; CHECK-NEXT: [[M1:%.*]] = fmul contract double [[A1]], [[B1]]
; CHECK-NEXT: [[S0:%.*]] = fadd contract double [[X]], [[M0]]
-; CHECK-NEXT: [[M1:%.*]] = extractelement <2 x double> [[TMP2]], i64 1
; CHECK-NEXT: [[S1:%.*]] = fadd contract double [[Y]], [[M1]]
; CHECK-NEXT: [[R:%.*]] = select i1 [[C]], double [[S0]], double [[S1]]
; CHECK-NEXT: ret double [[R]]
diff --git a/llvm/test/Transforms/SLPVectorizer/X86/fsub-fmul-rhs-combine.ll b/llvm/test/Transforms/SLPVectorizer/X86/fsub-fmul-rhs-combine.ll
index b766d92bd13fb..7ceb8784ef1e5 100644
--- a/llvm/test/Transforms/SLPVectorizer/X86/fsub-fmul-rhs-combine.ll
+++ b/llvm/test/Transforms/SLPVectorizer/X86/fsub-fmul-rhs-combine.ll
@@ -50,14 +50,16 @@ define double @fsub_fmul_rhs_gathered_c(ptr %a, ptr %b, double %c0, double %c1)
; CHECK-LABEL: define double @fsub_fmul_rhs_gathered_c(
; CHECK-SAME: ptr [[A:%.*]], ptr [[B:%.*]], double [[C0:%.*]], double [[C1:%.*]]) #[[ATTR0]] {
; CHECK-NEXT: [[ENTRY:.*:]]
-; CHECK-NEXT: [[TMP0:%.*]] = load <2 x double>, ptr [[A]], align 8
-; CHECK-NEXT: [[TMP1:%.*]] = load <2 x double>, ptr [[B]], align 8
-; CHECK-NEXT: [[TMP2:%.*]] = fmul contract <2 x double> [[TMP0]], [[TMP1]]
-; CHECK-NEXT: [[TMP3:%.*]] = insertelement <2 x double> poison, double [[C0]], i64 0
-; CHECK-NEXT: [[TMP4:%.*]] = insertelement <2 x double> [[TMP3]], double [[C1]], i64 1
-; CHECK-NEXT: [[TMP5:%.*]] = fsub contract <2 x double> [[TMP4]], [[TMP2]]
-; CHECK-NEXT: [[S0:%.*]] = extractelement <2 x double> [[TMP5]], i64 0
-; CHECK-NEXT: [[S1:%.*]] = extractelement <2 x double> [[TMP5]], i64 1
+; CHECK-NEXT: [[A0:%.*]] = load double, ptr [[A]], align 8
+; CHECK-NEXT: [[A1P:%.*]] = getelementptr inbounds double, ptr [[A]], i64 1
+; CHECK-NEXT: [[A1:%.*]] = load double, ptr [[A1P]], align 8
+; CHECK-NEXT: [[B0:%.*]] = load double, ptr [[B]], align 8
+; CHECK-NEXT: [[B1P:%.*]] = getelementptr inbounds double, ptr [[B]], i64 1
+; CHECK-NEXT: [[B1:%.*]] = load double, ptr [[B1P]], align 8
+; CHECK-NEXT: [[M0:%.*]] = fmul contract double [[A0]], [[B0]]
+; CHECK-NEXT: [[M1:%.*]] = fmul contract double [[A1]], [[B1]]
+; CHECK-NEXT: [[S0:%.*]] = fsub contract double [[C0]], [[M0]]
+; CHECK-NEXT: [[S1:%.*]] = fsub contract double [[C1]], [[M1]]
; CHECK-NEXT: [[R:%.*]] = fmul double [[S0]], [[S1]]
; CHECK-NEXT: ret double [[R]]
;
diff --git a/llvm/test/Transforms/SLPVectorizer/X86/slp-fma-loss-ordered.ll b/llvm/test/Transforms/SLPVectorizer/X86/slp-fma-loss-ordered.ll
index 01e953d7c250c..1304f5353c968 100644
--- a/llvm/test/Transforms/SLPVectorizer/X86/slp-fma-loss-ordered.ll
+++ b/llvm/test/Transforms/SLPVectorizer/X86/slp-fma-loss-ordered.ll
@@ -19,14 +19,11 @@ define double @mul_fun() {
; FMA-NEXT: [[CVT0:%.*]] = uitofp i16 3 to double
; FMA-NEXT: [[MUL0:%.*]] = fmul contract double 7.000000e+00, [[CVT0]]
; FMA-NEXT: [[ADD0:%.*]] = fadd contract double [[MUL0]], [[CVT0]]
-; FMA-NEXT: [[TMP1:%.*]] = insertelement <4 x double> poison, double [[CVT0]], i64 0
-; FMA-NEXT: [[TMP2:%.*]] = shufflevector <4 x double> [[TMP1]], <4 x double> poison, <4 x i32> zeroinitializer
-; FMA-NEXT: [[TMP3:%.*]] = fmul contract <4 x double> [[TMP2]], <double -4.300000e+01, double 2.200000e-02, double 9.500000e+00, double 1.000000e+00>
-; FMA-NEXT: [[MUL1:%.*]] = extractelement <4 x double> [[TMP3]], i64 0
+; FMA-NEXT: [[MUL1:%.*]] = fmul contract double -4.300000e+01, [[CVT0]]
; FMA-NEXT: [[ADD1:%.*]] = fadd contract double [[MUL1]], [[ADD0]]
-; FMA-NEXT: [[MUL2:%.*]] = extractelement <4 x double> [[TMP3]], i64 1
+; FMA-NEXT: [[MUL2:%.*]] = fmul contract double 2.200000e-02, [[CVT0]]
; FMA-NEXT: [[ADD2:%.*]] = fadd contract double [[MUL2]], [[ADD1]]
-; FMA-NEXT: [[MUL3:%.*]] = extractelement <4 x double> [[TMP3]], i64 2
+; FMA-NEXT: [[MUL3:%.*]] = fmul contract double 9.500000e+00, [[CVT0]]
; FMA-NEXT: [[ADD3:%.*]] = fadd contract double [[MUL3]], [[ADD2]]
; FMA-NEXT: ret double [[ADD3]]
;
@@ -59,15 +56,12 @@ define double @mul_fun_multiuse(ptr %dst) {
; FMA-NEXT: [[CVT0:%.*]] = uitofp i16 3 to double
; FMA-NEXT: [[TMP4:%.*]] = fmul contract double 7.000000e+00, [[CVT0]]
; FMA-NEXT: [[ADD0:%.*]] = fadd contract double [[TMP4]], [[CVT0]]
-; FMA-NEXT: [[TMP1:%.*]] = insertelement <4 x double> poison, double [[CVT0]], i64 0
-; FMA-NEXT: [[TMP2:%.*]] = shufflevector <4 x double> [[TMP1]], <4 x double> poison, <4 x i32> zeroinitializer
-; FMA-NEXT: [[TMP3:%.*]] = fmul contract <4 x double> [[TMP2]], <double -4.300000e+01, double 2.200000e-02, double 9.500000e+00, double 1.000000e+00>
-; FMA-NEXT: [[TMP5:%.*]] = extractelement <4 x double> [[TMP3]], i64 0
+; FMA-NEXT: [[TMP5:%.*]] = fmul contract double -4.300000e+01, [[CVT0]]
; FMA-NEXT: store double [[TMP5]], ptr [[DST:%.*]], align 8
; FMA-NEXT: [[ADD1:%.*]] = fadd contract double [[TMP5]], [[ADD0]]
-; FMA-NEXT: [[MUL2:%.*]] = extractelement <4 x double> [[TMP3]], i64 1
+; FMA-NEXT: [[MUL2:%.*]] = fmul contract double 2.200000e-02, [[CVT0]]
; FMA-NEXT: [[ADD2:%.*]] = fadd contract double [[MUL2]], [[ADD1]]
-; FMA-NEXT: [[MUL3:%.*]] = extractelement <4 x double> [[TMP3]], i64 2
+; FMA-NEXT: [[MUL3:%.*]] = fmul contract double 9.500000e+00, [[CVT0]]
; FMA-NEXT: [[ADD3:%.*]] = fadd contract double [[MUL3]], [[ADD2]]
; FMA-NEXT: ret double [[ADD3]]
;
diff --git a/llvm/test/Transforms/SLPVectorizer/consecutive-access.ll b/llvm/test/Transforms/SLPVectorizer/consecutive-access.ll
index 8347d5c8094b1..27e88a5b6d73e 100644
--- a/llvm/test/Transforms/SLPVectorizer/consecutive-access.ll
+++ b/llvm/test/Transforms/SLPVectorizer/consecutive-access.ll
@@ -1,6 +1,6 @@
; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
-; RUN: %if x86-registered-target %{ opt < %s -passes=slp-vectorizer -S -mtriple=x86_64-apple-macosx10.9.0 | FileCheck %s --check-prefixes=CHECK,X86 %}
-; RUN: %if aarch64-registered-target %{ opt < %s -passes=slp-vectorizer -S -mtriple=aarch64-unknown-linux-gnu | FileCheck %s --check-prefixes=CHECK,AARCH64 %}
+; RUN: %if x86-registered-target %{ opt < %s -passes=slp-vectorizer -S -mtriple=x86_64-apple-macosx10.9.0 | FileCheck %s %}
+; RUN: %if aarch64-registered-target %{ opt < %s -passes=slp-vectorizer -S -mtriple=aarch64-unknown-linux-gnu | FileCheck %s %}
@A = common global [2000 x double] zeroinitializer, align 16
@B = common global [2000 x double] zeroinitializer, align 16
@@ -439,45 +439,26 @@ for.end: ; preds = %for.cond.for.end_cr
; Make sure we are able to vectorize this from now on:
;
define double @bar(ptr nocapture readonly %a, i32 %n) local_unnamed_addr {
-; X86-LABEL: @bar(
-; X86-NEXT: entry:
-; X86-NEXT: [[CMP15:%.*]] = icmp eq i32 [[N:%.*]], 0
-; X86-NEXT: br i1 [[CMP15]], label [[FOR_COND_CLEANUP:%.*]], label [[FOR_BODY:%.*]]
-; X86: for.cond.cleanup:
-; X86-NEXT: [[TMP0:%.*]] = phi <2 x double> [ zeroinitializer, [[ENTRY:%.*]] ], [ [[TMP5:%.*]], [[FOR_BODY]] ]
-; X86-NEXT: [[TMP1:%.*]] = extractelement <2 x double> [[TMP0]], i64 0
-; X86-NEXT: [[TMP2:%.*]] = extractelement <2 x double> [[TMP0]], i64 1
-; X86-NEXT: [[MUL:%.*]] = fmul double [[TMP1]], [[TMP2]]
-; X86-NEXT: ret double [[MUL]]
-; X86: for.body:
-; X86-NEXT: [[I_018:%.*]] = phi i32 [ [[ADD5:%.*]], [[FOR_BODY]] ], [ 0, [[ENTRY]] ]
-; X86-NEXT: [[TMP3:%.*]] = phi <2 x double> [ [[TMP5]], [[FOR_BODY]] ], [ zeroinitializer, [[ENTRY]] ]
-; X86-NEXT: [[IDXPROM:%.*]] = zext i32 [[I_018]] to i64
-; X86-NEXT: [[ARRAYIDX:%.*]] = getelementptr inbounds double, ptr [[A:%.*]], i64 [[IDXPROM]]
-; X86-NEXT: [[TMP4:%.*]] = load <2 x double>, ptr [[ARRAYIDX]], align 8
-; X86-NEXT: [[TMP5]] = fadd <2 x double> [[TMP3]], [[TMP4]]
-; X86-NEXT: [[ADD5]] = add i32 [[I_018]], 2
-; X86-NEXT: [[CMP:%.*]] = icmp ult i32 [[ADD5]], [[N]]
-; X86-NEXT: br i1 [[CMP]], label [[FOR_BODY]], label [[FOR_COND_CLEANUP]]
-;
-; AARCH64-LABEL: @bar(
-; AARCH64-NEXT: entry:
-; AARCH64-NEXT: [[CMP15:%.*]] = icmp eq i32 [[N:%.*]], 0
-; AARCH64-NEXT: br i1 [[CMP15]], label [[FOR_COND_CLEANUP:%.*]], label [[FOR_BODY:%.*]]
-; AARCH64: for.cond.cleanup:
-; AARCH64-NEXT: [[TMP0:%.*]] = phi <2 x double> [ zeroinitializer, [[ENTRY:%.*]] ], [ [[TMP4:%.*]], [[FOR_BODY]] ]
-; AARCH64-NEXT: [[TMP1:%.*]] = call reassoc double @llvm.vector.reduce.fmul.v2f64(double 1.000000e+00, <2 x double> [[TMP0]])
-; AARCH64-NEXT: ret double [[TMP1]]
-; AARCH64: for.body:
-; AARCH64-NEXT: [[I_018:%.*]] = phi i32 [ [[ADD5:%.*]], [[FOR_BODY]] ], [ 0, [[ENTRY]] ]
-; AARCH64-NEXT: [[TMP2:%.*]] = phi <2 x double> [ [[TMP4]], [[FOR_BODY]] ], [ zeroinitializer, [[ENTRY]] ]
-; AARCH64-NEXT: [[IDXPROM:%.*]] = zext i32 [[I_018]] to i64
-; AARCH64-NEXT: [[ARRAYIDX:%.*]] = getelementptr inbounds double, ptr [[A:%.*]], i64 [[IDXPROM]]
-; AARCH64-NEXT: [[TMP3:%.*]] = load <2 x double>, ptr [[ARRAYIDX]], align 8
-; AARCH64-NEXT: [[TMP4]] = fadd <2 x double> [[TMP2]], [[TMP3]]
-; AARCH64-NEXT: [[ADD5]] = add i32 [[I_018]], 2
-; AARCH64-NEXT: [[CMP:%.*]] = icmp ult i32 [[ADD5]], [[N]]
-; AARCH64-NEXT: br i1 [[CMP]], label [[FOR_BODY]], label [[FOR_COND_CLEANUP]]
+; CHECK-LABEL: @bar(
+; CHECK-NEXT: entry:
+; CHECK-NEXT: [[CMP15:%.*]] = icmp eq i32 [[N:%.*]], 0
+; CHECK-NEXT: br i1 [[CMP15]], label [[FOR_COND_CLEANUP:%.*]], label [[FOR_BODY:%.*]]
+; CHECK: for.cond.cleanup:
+; CHECK-NEXT: [[TMP0:%.*]] = phi <2 x double> [ zeroinitializer, [[ENTRY:%.*]] ], [ [[TMP5:%.*]], [[FOR_BODY]] ]
+; CHECK-NEXT: [[TMP1:%.*]] = extractelement <2 x double> [[TMP0]], i64 0
+; CHECK-NEXT: [[TMP2:%.*]] = extractelement <2 x double> [[TMP0]], i64 1
+; CHECK-NEXT: [[MUL:%.*]] = fmul double [[TMP1]], [[TMP2]]
+; CHECK-NEXT: ret double [[MUL]]
+; CHECK: for.body:
+; CHECK-NEXT: [[I_018:%.*]] = phi i32 [ [[ADD5:%.*]], [[FOR_BODY]] ], [ 0, [[ENTRY]] ]
+; CHECK-NEXT: [[TMP3:%.*]] = phi <2 x double> [ [[TMP5]], [[FOR_BODY]] ], [ zeroinitializer, [[ENTRY]] ]
+; CHECK-NEXT: [[IDXPROM:%.*]] = zext i32 [[I_018]] to i64
+; CHECK-NEXT: [[ARRAYIDX:%.*]] = getelementptr inbounds double, ptr [[A:%.*]], i64 [[IDXPROM]]
+; CHECK-NEXT: [[TMP4:%.*]] = load <2 x double>, ptr [[ARRAYIDX]], align 8
+; CHECK-NEXT: [[TMP5]] = fadd <2 x double> [[TMP3]], [[TMP4]]
+; CHECK-NEXT: [[ADD5]] = add i32 [[I_018]], 2
+; CHECK-NEXT: [[CMP:%.*]] = icmp ult i32 [[ADD5]], [[N]]
+; CHECK-NEXT: br i1 [[CMP]], label [[FOR_BODY]], label [[FOR_COND_CLEANUP]]
;
entry:
%cmp15 = icmp eq i32 %n, 0
More information about the llvm-commits
mailing list