[llvm] 34dff48 - [SLP]Flatten fsub/fneg chains into fadd reductions with per-operand signs
via llvm-commits
llvm-commits at lists.llvm.org
Mon Sep 7 05:05:44 PDT 2026
Author: Alexey Bataev
Date: 2026-09-07T08:05:39-04:00
New Revision: 34dff487dd711a6a6d6a22f474ce475bce757b47
URL: https://github.com/llvm/llvm-project/commit/34dff487dd711a6a6d6a22f474ce475bce757b47
DIFF: https://github.com/llvm/llvm-project/commit/34dff487dd711a6a6d6a22f474ce475bce757b47.diff
LOG: [SLP]Flatten fsub/fneg chains into fadd reductions with per-operand signs
For unordered fadd reductions, reassociable (reassoc+nsz) fsub/fneg
chain links are recursed into and their leaves tracked with a flipped
sign, which is part of the grouping key, so reduced-values groups are
sign-uniform by construction. If per-value signs cannot model the
chain, the analysis restarts once without the flattening. Negated
parts are combined with vector fsub (may form per-lane fma) and
subtracted once in the final combine; no extra negations are emitted.
Fixes #211521
Reviewers: hiraditya, RKSimon
Pull Request: https://github.com/llvm/llvm-project/pull/215840
Added:
Modified:
llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp
llvm/lib/Transforms/Vectorize/SLPVectorizer/SLPCompatibilityAnalysis.cpp
llvm/lib/Transforms/Vectorize/SLPVectorizer/SLPCompatibilityAnalysis.h
llvm/test/Transforms/SLPVectorizer/AArch64/loop-accumulator-reduction.ll
llvm/test/Transforms/SLPVectorizer/AArch64/many-uses-fma-candidate.ll
llvm/test/Transforms/SLPVectorizer/AMDGPU/reduction.ll
llvm/test/Transforms/SLPVectorizer/X86/fma-operand-index.ll
llvm/test/Transforms/SLPVectorizer/X86/horizontal-fadd-with-sub.ll
llvm/test/Transforms/SLPVectorizer/X86/horizontal-list.ll
llvm/test/Transforms/SLPVectorizer/X86/loop-accumulator-reduction.ll
llvm/test/Transforms/SLPVectorizer/X86/ordered-reduction-replaced.ll
llvm/test/Transforms/SLPVectorizer/X86/reduction-fadd-reassoc.ll
Removed:
################################################################################
diff --git a/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp b/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp
index 1a9766442856e..2b6eed20572fa 100644
--- a/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp
+++ b/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp
@@ -345,6 +345,27 @@ static const int MinScheduleRegionSize = 16;
/// Maximum allowed number of operands in the PHI nodes.
static const unsigned MaxPHINumOperands = 128;
+namespace {
+/// A vectorized part of a split reduction, combined into the final reduction
+/// result by the horizontal reduction emitter.
+struct ReductionVectorPart {
+ /// The vectorized value, tracked in case it is replaced while other parts
+ /// are vectorized.
+ WeakTrackingVH Vec;
+ /// The number of times each lane is repeated in the reduction (emitted as a
+ /// multiplication by the scale for add/fadd reductions).
+ unsigned Scale = 1;
+ /// Signedness of \p Vec for reductions, operating on truncated types.
+ bool IsSigned = false;
+ /// True if the value was already reduced in-tree.
+ bool ReducedInTree = false;
+ /// True if the part contribution is subtracted from (rather than added to)
+ /// the final reduction result. Used for reassociated fadd reductions,
+ /// flattened through fsub/fneg operations.
+ bool Negated = false;
+};
+} // namespace
+
/// Bottom Up SLP Vectorizer.
class slpvectorizer::BoUpSLP {
class TreeEntry;
@@ -422,8 +443,7 @@ class slpvectorizer::BoUpSLP {
Value *
vectorizeTree(const ExtraValueToDebugLocsMap &ExternallyUsedValues,
Instruction *ReductionRoot = nullptr,
- ArrayRef<std::tuple<WeakTrackingVH, unsigned, bool, bool>>
- VectorValuesAndScales = {});
+ ArrayRef<ReductionVectorPart> VectorValuesAndScales = {});
/// \returns the cost incurred by unwanted spills and fills, caused by
/// holding live values over call sites.
@@ -2193,8 +2213,7 @@ class slpvectorizer::BoUpSLP {
template <typename T>
void removeInstructionsAndOperands(
ArrayRef<T *> DeadVals,
- ArrayRef<std::tuple<WeakTrackingVH, unsigned, bool, bool>>
- VectorValuesAndScales) {
+ ArrayRef<ReductionVectorPart> VectorValuesAndScales) {
SmallVector<WeakTrackingVH> DeadInsts;
for (T *V : DeadVals) {
auto *I = cast<Instruction>(V);
@@ -2263,10 +2282,10 @@ class slpvectorizer::BoUpSLP {
if (!DeletedInstructions.contains(OpI) &&
!ExternalUseReplacements.contains(OpI) &&
(!OpI->getType()->isVectorTy() ||
- none_of(
- VectorValuesAndScales,
- [&](const std::tuple<WeakTrackingVH, unsigned, bool, bool>
- &V) { return std::get<0>(V) == OpI; })) &&
+ none_of(VectorValuesAndScales,
+ [&](const ReductionVectorPart &V) {
+ return V.Vec == OpI;
+ })) &&
isInstructionTriviallyDead(OpI, TLI))
DeadInsts.push_back(OpI);
}
@@ -14308,7 +14327,8 @@ void BoUpSLP::reorderGatherNode(TreeEntry &TE) {
}
/// Check if we can convert fadd/fsub sequence to FMAD.
-/// \returns Cost of the FMAD, if conversion is possible, invalid cost otherwise.
+/// \returns Cost of the FMAD, if conversion is possible, invalid cost
+/// otherwise.
static InstructionCost canConvertToFMA(ArrayRef<Value *> VL,
const InstructionsState &S,
DominatorTree &DT, const DataLayout &DL,
@@ -14320,7 +14340,8 @@ static InstructionCost canConvertToFMA(ArrayRef<Value *> VL,
return V->getType()->getScalarType()->isFloatingPointTy();
}) &&
"Can only convert to FMA for floating point types");
- assert(S.isAddSubLikeOp() && "Can only convert to FMA for add/sub");
+ assert(S.isAddSubOrFNegLikeOp() &&
+ "Can only convert to FMA for add/sub or fneg chain links");
auto CheckForContractable = [](ArrayRef<Value *> VL,
const InstructionsState &S) {
@@ -14372,6 +14393,26 @@ static InstructionCost canConvertToFMA(ArrayRef<Value *> VL,
FMF.set();
const bool IsArithmeticState = S.isAddSubLikeOp() || S.isMulDivLikeOp() ||
S.isShiftOp() || S.isBitwiseLogicOp();
+ // A dropped fneg link stands in for the sign-flipped combine its operand
+ // joins and is costed as an fadd plus the fneg itself (both are removed).
+ auto GetLinkCost = [&](Instruction *I, bool IsCopyable = false) {
+ if (S.getOpcode() != Instruction::FNeg) {
+ if (IsCopyable || !IsArithmeticState ||
+ (I->getOpcode() != S.getOpcode() &&
+ I->getOpcode() != S.getAltOpcode()))
+ return TTI.getInstructionCost(I, CostKind);
+ TTI::OperandValueInfo Op1Info = TTI::getOperandInfo(I->getOperand(0));
+ TTI::OperandValueInfo Op2Info = TTI::getOperandInfo(I->getOperand(1));
+ return TTI.getArithmeticInstrCost(I->getOpcode(), I->getType(), CostKind,
+ Op1Info, Op2Info,
+ {I->getOperand(0), I->getOperand(1)});
+ }
+ return TTI.getArithmeticInstrCost(Instruction::FAdd, I->getType(), CostKind,
+ {},
+ TTI::getOperandInfo(I->getOperand(0))) +
+ TTI.getArithmeticInstrCost(Instruction::FNeg, I->getType(), CostKind,
+ TTI::getOperandInfo(I->getOperand(0)));
+ };
for (Value *V : VL) {
auto *I = dyn_cast<Instruction>(V);
if (!I)
@@ -14380,17 +14421,7 @@ static InstructionCost canConvertToFMA(ArrayRef<Value *> VL,
if (!IsCopyable)
if (auto *FPCI = dyn_cast<FPMathOperator>(I))
FMF &= FPCI->getFastMathFlags();
- if (IsCopyable || !IsArithmeticState ||
- (I->getOpcode() != S.getOpcode() &&
- I->getOpcode() != S.getAltOpcode())) {
- FMulPlusFAddCost += TTI.getInstructionCost(I, CostKind);
- continue;
- }
- TTI::OperandValueInfo Op1Info = TTI::getOperandInfo(I->getOperand(0));
- TTI::OperandValueInfo Op2Info = TTI::getOperandInfo(I->getOperand(1));
- FMulPlusFAddCost += TTI.getArithmeticInstrCost(
- I->getOpcode(), I->getType(), CostKind, Op1Info, Op2Info,
- {I->getOperand(0), I->getOperand(1)});
+ FMulPlusFAddCost += GetLinkCost(I, IsCopyable);
}
unsigned NumOps = 0;
for (auto [V, Op] : zip(VL, Operands.front())) {
@@ -14399,7 +14430,7 @@ static InstructionCost canConvertToFMA(ArrayRef<Value *> VL,
auto *I = dyn_cast<Instruction>(Op);
if (!I || !I->hasOneUse() || OpS.isCopyableElement(I)) {
if (auto *OpI = dyn_cast<Instruction>(V))
- FMACost += TTI.getInstructionCost(OpI, CostKind);
+ FMACost += GetLinkCost(OpI);
if (I)
FMACost += TTI.getInstructionCost(I, CostKind);
continue;
@@ -25391,11 +25422,10 @@ Value *BoUpSLP::vectorizeTree() {
return vectorizeTree(ExternallyUsedValues);
}
-Value *BoUpSLP::vectorizeTree(
- const ExtraValueToDebugLocsMap &ExternallyUsedValues,
- Instruction *ReductionRoot,
- ArrayRef<std::tuple<WeakTrackingVH, unsigned, bool, bool>>
- VectorValuesAndScales) {
+Value *
+BoUpSLP::vectorizeTree(const ExtraValueToDebugLocsMap &ExternallyUsedValues,
+ Instruction *ReductionRoot,
+ ArrayRef<ReductionVectorPart> VectorValuesAndScales) {
// Clean Entry-to-LastInstruction table. It can be affected after scheduling,
// need to rebuild it.
EntryToLastInstruction.clear();
@@ -30083,6 +30113,9 @@ class HorizontalReduction {
ReductionOpsListType ReductionOps;
/// List of possibly reduced values.
SmallVector<SmallVector<Value *>> ReducedVals;
+ /// Reduced values that enter the reduction with a flipped sign (from an
+ /// fsub/fneg chain link), subtracted in the final combine.
+ SmallPtrSet<Value *, 8> NegatedReducedVals;
/// Maps reduced value to the corresponding reduction operation.
SmallDenseMap<Value *, SmallVector<Instruction *>, 16> ReducedValsToOps;
WeakTrackingVH ReductionRoot;
@@ -30098,10 +30131,9 @@ class HorizontalReduction {
bool IsSupportedHorRdxIdentityOp = false;
/// The minimum number of the reduced values.
const unsigned ReductionLimit = VectorizeNonPowerOf2 ? 3 : 4;
- /// Contains vector values for reduction including their scale factor and
- /// signedness. The last bool is true, if the value was reduced in-tree.
- SmallVector<std::tuple<WeakTrackingVH, unsigned, bool, bool>>
- VectorValuesAndScales;
+ /// Contains vector values for reduction including their scale factor,
+ /// signedness and sign of their contribution to the reduction result.
+ SmallVector<ReductionVectorPart> VectorValuesAndScales;
/// Narrow reduced values mapped to the shift and the mask applied after
/// widening.
@@ -30198,6 +30230,7 @@ class HorizontalReduction {
case RecurKind::Mul:
case RecurKind::Xor:
case RecurKind::FAdd:
+ case RecurKind::FSub:
case RecurKind::FMul: {
unsigned RdxOpcode = RecurrenceDescriptor::getOpcode(Kind);
return Builder.CreateBinOp((Instruction::BinaryOps)RdxOpcode, LHS, RHS,
@@ -30366,7 +30399,7 @@ class HorizontalReduction {
private:
/// Total number of operands in the reduction operation.
static unsigned getNumberOfOperands(Instruction *I) {
- return isCmpSelMinMax(I) ? 3 : 2;
+ return isCmpSelMinMax(I) ? 3 : (I->isUnaryOp() ? 1 : 2);
}
/// Checks if the instruction is in basic block \p BB.
@@ -30492,6 +30525,17 @@ class HorizontalReduction {
ReducedValsToOps[V].push_back(I);
}
+ /// Returns true if the matched reduction chain has flattened fsub/fneg
+ /// links (the leaves behind them may all be positive, e.g. with double
+ /// negations).
+ bool hasFlattenedNegations() const {
+ return any_of(ReductionOps, [](ArrayRef<Value *> Ops) {
+ return any_of(Ops, [](Value *Op) {
+ return Op && isFlattenableNegation(cast<Instruction>(Op));
+ });
+ });
+ }
+
bool matchReductionForOperands() {
// Analyze "regular" integer/FP types for reductions - no target-specific
// types or pointers.
@@ -30521,6 +30565,7 @@ class HorizontalReduction {
RoundedLinks.clear();
NarrowedLeafShifts.clear();
NarrowedChainInsts.clear();
+ NegatedReducedVals.clear();
RdxKind = getRdxKind(Root);
// Currently, only ordered fadd reductions are supported.
if (RdxKind != RecurKind::FAdd)
@@ -30618,13 +30663,34 @@ class HorizontalReduction {
return true;
}
- /// Try to find a reduction tree.
+ /// Checks if \p I is an fsub/fneg, which can be flattened into an fadd
+ /// reduction chain with a flipped sign of its subtracted operand. nsz is
+ /// required: subtracted leaves are regrouped and negated as a whole, and
+ /// -a + -b == -(a + b) may flip the sign of a zero result.
+ static bool isFlattenableNegation(Instruction *I) {
+ return (I->getOpcode() == Instruction::FSub ||
+ I->getOpcode() == Instruction::FNeg) &&
+ I->hasAllowReassoc() && I->hasNoSignedZeros();
+ }
+
+ /// Try to find a reduction tree. If \p FlattenNegations is false, the
+ /// reassociable fsub/fneg links of an fadd chain are not flattened (they are
+ /// leaves of the reduction).
bool matchAssociativeReduction(BoUpSLP &R, Instruction *Root,
ScalarEvolution &SE, DominatorTree &DT,
const DataLayout &DL,
const TargetTransformInfo &TTI,
- const TargetLibraryInfo &TLI) {
+ const TargetLibraryInfo &TLI,
+ bool FlattenNegations = true) {
RdxKind = HorizontalReduction::getRdxKind(Root);
+ // A reassociable fsub root is a flattened link of an fadd reduction: its
+ // subtracted operand enters with a flipped sign. Without the flattening
+ // there is no reduction to match. (An fneg is not a reduction candidate.)
+ const bool IsNegatedRoot = RdxKind == RecurKind::None &&
+ Root->getOpcode() == Instruction::FSub &&
+ isFlattenableNegation(Root);
+ if (IsNegatedRoot)
+ RdxKind = RecurKind::FAdd;
RK = isVectorizable(RdxKind, Root);
if (RK == ReductionOrdering::None)
return false;
@@ -30647,268 +30713,420 @@ class HorizontalReduction {
// gather all the reduced values, sorting them by their value id.
BasicBlock *BB = Root->getParent();
bool IsCmpSelMinMax = isCmpSelMinMax(Root);
- SmallVector<std::pair<Instruction *, unsigned>> Worklist(
- 1, std::make_pair(Root, 0));
- SmallPtrSet<Value *, 8> Operands;
- SmallVector<std::pair<Instruction *, unsigned>> PossibleOrderedReductionOps;
- // Checks if the operands of the \p TreeN instruction are also reduction
- // operations or should be treated as reduced values or an extra argument,
- // which is not part of the reduction.
- auto CheckOperands = [&](Instruction *TreeN,
- SmallVectorImpl<Value *> &PossibleReducedVals,
- SmallVectorImpl<Instruction *> &ReductionOps,
- ReductionOpsType &AllReductionOps,
- unsigned Level) {
- for (int I : reverse(seq<int>(getFirstOperandIndex(TreeN),
- getNumberOfOperands(TreeN)))) {
- Value *EdgeVal = getRdxOperand(TreeN, I);
- auto *EdgeInst = dyn_cast<Instruction>(EdgeVal);
- // The link may be an elidable cast round-trip; look through it to
- // continue the chain. It is dropped when the reduction is folded.
- if (EdgeInst && getRdxKind(EdgeInst) != RdxKind) {
- if (Instruction *NarrowCast =
- lookThroughCastRoundTrip(EdgeVal, /*MustBeElidable=*/true)) {
- auto *SrcI = cast<Instruction>(NarrowCast->getOperand(0));
- if (getRdxKind(SrcI) == RdxKind &&
- hasRequiredNumberOfUses(IsCmpSelMinMax, SrcI)) {
- AllReductionOps.push_back(EdgeVal);
- AllReductionOps.push_back(NarrowCast);
- EdgeVal = SrcI;
- EdgeInst = SrcI;
+ const ReductionOrdering InitialRK = RK;
+ // For unordered fadd reductions, reassociable fsub/fneg chain links are
+ // flattened with a flipped sign on the subtracted operand: the leaves,
+ // reduced through such links, are subtracted in the final combine.
+ // Ordered reductions keep their accumulation order and are excluded.
+ bool TrackSign = FlattenNegations && RdxKind == RecurKind::FAdd &&
+ RK == ReductionOrdering::Unordered;
+ // The leaf signs are modeled per value. If they cannot be modeled this way
+ // (the same value occurs both added and subtracted, or the reduction
+ // switches to ordered after some fsub/fneg links have been flattened
+ // already), the analysis restarts without the fsub/fneg flattening.
+ bool Restart = false;
+ do {
+ if (Restart) {
+ assert(TrackSign && "Expected a sign-tracking analysis to restart");
+ TrackSign = false;
+ }
+ // The root itself is a flattened link - there is no reduction without
+ // the flattening.
+ if (IsNegatedRoot && !TrackSign)
+ return false;
+ Restart = false;
+ RK = InitialRK;
+ ReducedVals.clear();
+ ReducedValsToOps.clear();
+ NegatedReducedVals.clear();
+ bool FlattenedNegations = false;
+ initReductionOps(Root);
+ DenseMap<std::pair<size_t, Value *>, SmallVector<LoadInst *>> LoadsMap;
+ SmallSet<size_t, 2> LoadKeyUsed;
+ auto GenerateLoadsSubkey = [&](size_t Key, LoadInst *LI) {
+ Key = hash_combine(hash_value(LI->getParent()->getNumber()), Key);
+ Value *Ptr =
+ getUnderlyingObject(LI->getPointerOperand(), RecursionMaxDepth);
+ if (!LoadKeyUsed.insert(Key).second) {
+ auto LIt = LoadsMap.find(std::make_pair(Key, Ptr));
+ if (LIt != LoadsMap.end()) {
+ for (LoadInst *RLI : LIt->second) {
+ if (getPointersDiff(RLI->getType(), RLI->getPointerOperand(),
+ LI->getType(), LI->getPointerOperand(), DL,
+ SE, /*StrictCheck=*/true))
+ return hash_value(RLI->getPointerOperand());
+ }
+ for (LoadInst *RLI : LIt->second) {
+ if (arePointersCompatible(RLI->getPointerOperand(),
+ LI->getPointerOperand(), TLI)) {
+ hash_code SubKey = hash_value(RLI->getPointerOperand());
+ return SubKey;
+ }
+ }
+ if (LIt->second.size() > 2) {
+ hash_code SubKey =
+ hash_value(LIt->second.back()->getPointerOperand());
+ return SubKey;
}
}
}
- ReducedValsToOps[EdgeVal].push_back(TreeN);
- // If the edge is not an instruction, or it is
diff erent from the main
- // reduction opcode or has too many uses - possible reduced value.
- // Also, do not try to reduce const values, if the operation is not
- // foldable.
- bool IsReducedVal = !EdgeInst || Level > RecursionMaxDepth ||
- getRdxKind(EdgeInst) != RdxKind ||
- IsCmpSelMinMax != isCmpSelMinMax(EdgeInst);
- ReductionOrdering CurrentRK = IsReducedVal
- ? ReductionOrdering::None
- : isVectorizable(RdxKind, EdgeInst);
- if (!IsReducedVal && CurrentRK == ReductionOrdering::Unordered &&
- RK == ReductionOrdering::Unordered &&
- !hasRequiredNumberOfUses(IsCmpSelMinMax, EdgeInst)) {
- IsReducedVal = true;
- CurrentRK = ReductionOrdering::None;
- if (PossibleReducedVals.size() < ReductionLimit &&
- !Operands.contains(EdgeInst))
- PossibleOrderedReductionOps.emplace_back(EdgeInst, Level);
- }
- if (CurrentRK == ReductionOrdering::None ||
- Operands.contains(EdgeInst) ||
- (R.isAnalyzedReductionRoot(EdgeInst) &&
- all_of(EdgeInst->operands(), IsaPred<Constant>))) {
- PossibleReducedVals.push_back(EdgeVal);
- if (EdgeInst && !isCmpSelMinMax(EdgeInst))
- Operands.insert_range(EdgeInst->operands());
+ LoadsMap.try_emplace(std::make_pair(Key, Ptr))
+ .first->second.push_back(LI);
+ return hash_value(LI->getPointerOperand());
+ };
+ SmallVector<Value *> ReducedValsCandidates;
+ bool AdjustedToOrdered = false;
+ // True if the chain has an opaque fsub/fneg leaf (not reassociable, extra
+ // uses, too deep).
+ bool HasOpaqueNegationLeaf = false;
+ SmallVector<std::tuple<Instruction *, unsigned, bool>> Worklist(
+ 1, std::make_tuple(Root, 0, /*Negated=*/false));
+ SmallPtrSet<Value *, 8> Operands;
+ SmallVector<std::pair<Instruction *, unsigned>>
+ PossibleOrderedReductionOps;
+ // Leaves that (also) occur with a non-flipped sign.
+ SmallPtrSet<Value *, 8> PositiveReducedVals;
+ // Checks if the instruction continues the reduction chain. An fsub/fneg
+ // chain link in an fadd reduction is recursed into like an fadd, but its
+ // subtracted operand enters with a flipped sign. Only reassociable links
+ // qualify.
+ auto IsChainLink = [&](Instruction *I) {
+ if (getRdxKind(I) == RdxKind)
+ return true;
+ return TrackSign && isFlattenableNegation(I);
+ };
+ // Checks if the operands of the \p TreeN instruction are also reduction
+ // operations or should be treated as reduced values or an extra
+ // argument, which is not part of the reduction.
+ auto CheckOperands =
+ [&](Instruction *TreeN, SmallVectorImpl<Value *> &PossibleReducedVals,
+ SmallVectorImpl<std::pair<Instruction *, bool>> &ReductionOps,
+ ReductionOpsType &AllReductionOps, unsigned Level, bool Negated) {
+ for (int I : reverse(seq<int>(getFirstOperandIndex(TreeN),
+ getNumberOfOperands(TreeN)))) {
+ Value *EdgeVal = getRdxOperand(TreeN, I);
+ auto *EdgeInst = dyn_cast<Instruction>(EdgeVal);
+ // The link may be an elidable cast round-trip; look through it to
+ // continue the chain. It is dropped when the reduction is folded.
+ if (EdgeInst && getRdxKind(EdgeInst) != RdxKind) {
+ if (Instruction *NarrowCast = lookThroughCastRoundTrip(
+ EdgeVal, /*MustBeElidable=*/true)) {
+ auto *SrcI = cast<Instruction>(NarrowCast->getOperand(0));
+ if (getRdxKind(SrcI) == RdxKind &&
+ hasRequiredNumberOfUses(IsCmpSelMinMax, SrcI)) {
+ AllReductionOps.push_back(EdgeVal);
+ AllReductionOps.push_back(NarrowCast);
+ EdgeVal = SrcI;
+ EdgeInst = SrcI;
+ }
+ }
+ }
+ ReducedValsToOps[EdgeVal].push_back(TreeN);
+ // fsub flips its second operand's sign, fneg its only
+ // operand's.
+ bool EdgeNegated =
+ Negated !=
+ (TreeN->getOpcode() == Instruction::FNeg ||
+ (TreeN->getOpcode() == Instruction::FSub && I == 1));
+ // If the edge is not an instruction, or it is
diff erent from
+ // the main reduction opcode or has too many uses - possible
+ // reduced value. Also, do not try to reduce const values, if
+ // the operation is not foldable.
+ bool IsReducedVal = !EdgeInst || Level > RecursionMaxDepth ||
+ !IsChainLink(EdgeInst) ||
+ IsCmpSelMinMax != isCmpSelMinMax(EdgeInst);
+ ReductionOrdering CurrentRK =
+ IsReducedVal ? ReductionOrdering::None
+ : isVectorizable(RdxKind, EdgeInst);
+ if (!IsReducedVal && CurrentRK == ReductionOrdering::Unordered &&
+ RK == ReductionOrdering::Unordered &&
+ !hasRequiredNumberOfUses(IsCmpSelMinMax, EdgeInst)) {
+ IsReducedVal = true;
+ CurrentRK = ReductionOrdering::None;
+ if (PossibleReducedVals.size() < ReductionLimit &&
+ !Operands.contains(EdgeInst))
+ PossibleOrderedReductionOps.emplace_back(EdgeInst, Level);
+ }
+ if (CurrentRK == ReductionOrdering::None ||
+ Operands.contains(EdgeInst) ||
+ (R.isAnalyzedReductionRoot(EdgeInst) &&
+ all_of(EdgeInst->operands(), IsaPred<Constant>))) {
+ // Keep the sign of the leaf. A leaf, occurring with both
+ // signs, cannot be modeled with a per-value sign - restart
+ // without sign tracking.
+ if (TrackSign) {
+ auto &Same =
+ EdgeNegated ? NegatedReducedVals : PositiveReducedVals;
+ auto &Other =
+ EdgeNegated ? PositiveReducedVals : NegatedReducedVals;
+ Same.insert(EdgeVal);
+ Restart |= Other.contains(EdgeVal);
+ HasOpaqueNegationLeaf |=
+ EdgeInst && (EdgeInst->getOpcode() == Instruction::FSub ||
+ EdgeInst->getOpcode() == Instruction::FNeg);
+ }
+ PossibleReducedVals.push_back(EdgeVal);
+ if (EdgeInst && !isCmpSelMinMax(EdgeInst))
+ Operands.insert_range(EdgeInst->operands());
+ continue;
+ }
+ if (CurrentRK == ReductionOrdering::Ordered)
+ RK = ReductionOrdering::Ordered;
+ ReductionOps.emplace_back(EdgeInst, EdgeNegated);
+ }
+ };
+ SmallPtrSet<Instruction *, 16> Visited;
+ while (!Worklist.empty()) {
+ auto [TreeN, Level, Negated] = Worklist.pop_back_val();
+ if (!Visited.insert(TreeN).second)
continue;
- }
- if (CurrentRK == ReductionOrdering::Ordered)
+ // A chain link other than the reduction operation is a flattened
+ // fsub/fneg.
+ if (getRdxKind(TreeN) != RdxKind)
+ FlattenedNegations = true;
+ SmallVector<Value *> PossibleRedVals;
+ SmallVector<std::pair<Instruction *, bool>> PossibleReductionOps;
+ CheckOperands(TreeN, PossibleRedVals, PossibleReductionOps,
+ ReductionOps[0], Level, Negated);
+ // A leaf occurred with both signs - stop, the analysis restarts
+ // without the fsub/fneg flattening.
+ if (Restart)
+ break;
+ addReductionOps(TreeN);
+ ReducedValsCandidates.append(PossibleRedVals.begin(),
+ PossibleRedVals.end());
+ for (auto [I, OpNegated] : reverse(PossibleReductionOps))
+ Worklist.emplace_back(I, I->getParent() == BB ? 0 : Level + 1,
+ OpNegated);
+ // If not enough elements for unordered vectorization, check if there
+ // are potential candidates for the ordered vectorization and try to
+ // add them to the worklist.
+ if (Worklist.empty() && ReducedValsCandidates.size() < ReductionLimit &&
+ !PossibleOrderedReductionOps.empty() &&
+ RK == ReductionOrdering::Unordered) {
RK = ReductionOrdering::Ordered;
- ReductionOps.push_back(EdgeInst);
- }
- };
- // Try to regroup reduced values so that it gets more profitable to try to
- // reduce them. Values are grouped by their value ids, instructions - by
- // instruction op id and/or alternate op id, plus do extra analysis for
- // loads (grouping them by the distance between pointers) and cmp
- // instructions (grouping them by the predicate).
- SmallMapVector<
- size_t, SmallMapVector<size_t, SmallMapVector<Value *, unsigned, 2>, 2>,
- 8>
- PossibleReducedVals;
- initReductionOps(Root);
- DenseMap<std::pair<size_t, Value *>, SmallVector<LoadInst *>> LoadsMap;
- SmallSet<size_t, 2> LoadKeyUsed;
-
- auto GenerateLoadsSubkey = [&](size_t Key, LoadInst *LI) {
- Key = hash_combine(hash_value(LI->getParent()->getNumber()), Key);
- Value *Ptr =
- getUnderlyingObject(LI->getPointerOperand(), RecursionMaxDepth);
- if (!LoadKeyUsed.insert(Key).second) {
- auto LIt = LoadsMap.find(std::make_pair(Key, Ptr));
- if (LIt != LoadsMap.end()) {
- for (LoadInst *RLI : LIt->second) {
- if (getPointersDiff(RLI->getType(), RLI->getPointerOperand(),
- LI->getType(), LI->getPointerOperand(), DL, SE,
- /*StrictCheck=*/true))
- return hash_value(RLI->getPointerOperand());
+ AdjustedToOrdered = true;
+ SmallPtrSet<const Instruction *, 4> Ops;
+ // The ordered switch discards the tracked signs (the analysis
+ // restarts without the fsub/fneg flattening, if any occurred), so
+ // the candidates re-enter the walk with the non-flipped sign.
+ for (const auto &[I, Level] : PossibleOrderedReductionOps) {
+ Ops.insert(I);
+ Worklist.emplace_back(I, Level, /*Negated=*/false);
}
- for (LoadInst *RLI : LIt->second) {
- if (arePointersCompatible(RLI->getPointerOperand(),
- LI->getPointerOperand(), TLI)) {
- hash_code SubKey = hash_value(RLI->getPointerOperand());
- return SubKey;
+ erase_if(ReducedValsCandidates, [&](Value *V) {
+ auto *I = dyn_cast<Instruction>(V);
+ return I && Ops.contains(I);
+ });
+ PossibleOrderedReductionOps.clear();
+ }
+ }
+ // The flattening reassociates the chain, which the ordered reduction
+ // must not do, and negating the regrouped subtracted leaves as a whole
+ // is not sign-of-zero-safe unless the whole flattened chain is nsz (this
+ // conservatively includes the elidable cast round-trips, collected into
+ // the reduction operations) - restart without the fsub/fneg flattening,
+ // if any occurred. Also, an opaque fsub/fneg leaf forms a homogeneous
+ // group with the flattened links when not flattened, which the
+ // sign-uniform regrouping of their operands may only fragment - keep the
+ // unflattened form.
+ if (FlattenedNegations &&
+ (RK == ReductionOrdering::Ordered || HasOpaqueNegationLeaf ||
+ (!NegatedReducedVals.empty() &&
+ any_of(ReductionOps.front(), [](Value *Op) {
+ return !cast<Instruction>(Op)->hasNoSignedZeros();
+ }))))
+ Restart = true;
+ if (Restart)
+ continue;
+ // InstCombine may pack same-kind binops under a zext, hiding the
+ // leaves. Analyze them in the narrow type; the vectorized result is
+ // widened back, shifted and masked per lane before the reduction.
+ if (Instruction::isBitwiseLogicOp(
+ RecurrenceDescriptor::getOpcode(RdxKind)) &&
+ RK == ReductionOrdering::Unordered && !IsCmpSelMinMax &&
+ Ty->isIntegerTy() && !Ty->isIntegerTy(1) &&
+ any_of(ReducedValsCandidates, [](Value *V) {
+ return isa<ZExtInst>(V) ||
+ match(V, m_Shl(m_ZExt(m_Value()), m_ConstantInt()));
+ })) {
+ unsigned WideBW = Ty->getIntegerBitWidth();
+ SmallVector<SmallVector<NarrowedLeafInfo>> AllLeaves;
+ SmallVector<Instruction *> ChainInsts;
+ Type *NarrowTy = nullptr;
+ bool Bail = false;
+ unsigned NumLeaves = 0;
+ SmallPtrSet<Value *, 8> UniqueLeaves;
+ for (Value *Cand : ReducedValsCandidates) {
+ SmallVector<NarrowedLeafInfo> &Leaves = AllLeaves.emplace_back();
+ collectNarrowedLeaves(Cand, RecurrenceDescriptor::getOpcode(RdxKind),
+ WideBW, RecursionMaxDepth, Leaves, ChainInsts);
+ for (const NarrowedLeafInfo &L : Leaves) {
+ if (isa<Constant>(L.V) || !L.V->getType()->isIntegerTy() ||
+ (NarrowTy && L.V->getType() != NarrowTy) ||
+ !UniqueLeaves.insert(L.V).second) {
+ Bail = true;
+ break;
}
+ if (!NarrowTy)
+ NarrowTy = L.V->getType();
}
- if (LIt->second.size() > 2) {
- hash_code SubKey =
- hash_value(LIt->second.back()->getPointerOperand());
- return SubKey;
+ if (Bail)
+ break;
+ NumLeaves += Leaves.size();
+ }
+ if (!Bail && NumLeaves > ReducedValsCandidates.size() &&
+ NarrowTy != Ty) {
+ // Collected user-first; dies with the reduction iff all users die.
+ SmallPtrSet<Value *, 8> Ignorable;
+ for (ArrayRef<Value *> RdxOps : ReductionOps)
+ Ignorable.insert_range(RdxOps);
+ for (Instruction *I : ChainInsts)
+ if (all_of(I->users(),
+ [&](User *U) { return Ignorable.contains(U); })) {
+ Ignorable.insert(I);
+ NarrowedChainInsts.push_back(I);
+ }
+ SmallVector<Value *> NewCandidates;
+ for (const auto &[Cand, Leaves] :
+ zip(ReducedValsCandidates, AllLeaves)) {
+ SmallVector<Instruction *> RdxOps = ReducedValsToOps.lookup(Cand);
+ for (const NarrowedLeafInfo &L : Leaves) {
+ NarrowedLeafShifts.try_emplace(L.V, L);
+ NewCandidates.push_back(L.V);
+ ReducedValsToOps[L.V].append(RdxOps);
+ }
}
+ ReducedValsCandidates = std::move(NewCandidates);
}
}
- LoadsMap.try_emplace(std::make_pair(Key, Ptr))
- .first->second.push_back(LI);
- return hash_value(LI->getPointerOperand());
- };
-
- SmallVector<Value *> ReducedValsCandidates;
- bool AdjustedToOrdered = false;
- SmallPtrSet<Instruction *, 16> Visited;
- while (!Worklist.empty()) {
- auto [TreeN, Level] = Worklist.pop_back_val();
- if (!Visited.insert(TreeN).second)
- continue;
- SmallVector<Value *> PossibleRedVals;
- SmallVector<Instruction *> PossibleReductionOps;
- CheckOperands(TreeN, PossibleRedVals, PossibleReductionOps,
- ReductionOps[0], Level);
- addReductionOps(TreeN);
- ReducedValsCandidates.append(PossibleRedVals.begin(),
- PossibleRedVals.end());
- for (Instruction *I : reverse(PossibleReductionOps))
- Worklist.emplace_back(I, I->getParent() == BB ? 0 : Level + 1);
- // If not enough elements for unordered vectorization, check if there are
- // potential candidates for the ordered vectorization and try to add them
- // to the worklist.
- if (Worklist.empty() && ReducedValsCandidates.size() < ReductionLimit &&
- !PossibleOrderedReductionOps.empty() &&
- RK == ReductionOrdering::Unordered) {
- RK = ReductionOrdering::Ordered;
- AdjustedToOrdered = true;
- SmallPtrSet<const Instruction *, 4> Ops;
- for (const auto &P : PossibleOrderedReductionOps)
- Ops.insert(P.first);
- erase_if(ReducedValsCandidates, [&](Value *V) {
- auto *I = dyn_cast<Instruction>(V);
- return I && Ops.contains(I);
+ // Too many integer reduced values candidates for the ordered
+ // reductions after adjustements - try to switch to unordered
+ // reductions instead.
+ constexpr unsigned ReducedValsLimit = 1024;
+ if (ReducedValsCandidates.size() > ReducedValsLimit &&
+ AdjustedToOrdered &&
+ ReducedValsCandidates.front()->getType()->isIntOrIntVectorTy())
+ return false;
+ // Try to regroup reduced values so that it gets more profitable to try
+ // to reduce them. Values are grouped by their value ids, instructions -
+ // by instruction op id and/or alternate op id, plus do extra analysis
+ // for loads (grouping them by the distance between pointers) and cmp
+ // instructions (grouping them by the predicate).
+ SmallMapVector<
+ size_t,
+ SmallMapVector<size_t, SmallMapVector<Value *, unsigned, 2>, 2>, 8>
+ PossibleReducedVals;
+ // Add reduction values. The values are sorted for better vectorization
+ // results.
+ for (Value *V : ReducedValsCandidates) {
+ if (RK == ReductionOrdering::Ordered && !isa<Instruction>(V))
+ continue;
+ size_t Key, Idx;
+ std::tie(Key, Idx) = generateKeySubkey(V, &TLI, GenerateLoadsSubkey,
+ /*AllowAlternate=*/false);
+ // The sign of the contribution is a part of the grouping key: a
+ // mixed-sign group cannot be negated as a whole.
+ Key = hash_combine(Key, NegatedReducedVals.contains(V));
+ ++PossibleReducedVals[Key][Idx].try_emplace(V, 0).first->second;
+ }
+ auto PossibleReducedValsVect = PossibleReducedVals.takeVector();
+ // Sort values by the total number of values kinds to start the
+ // reduction from the longest possible reduced values sequences.
+ for (auto &PossibleReducedVals : PossibleReducedValsVect) {
+ auto PossibleRedVals = PossibleReducedVals.second.takeVector();
+ SmallVector<SmallVector<Value *>> PossibleRedValsVect;
+ for (auto &Slice : PossibleRedVals) {
+ PossibleRedValsVect.emplace_back();
+ auto RedValsVect = Slice.second.takeVector();
+ stable_sort(RedValsVect, llvm::less_second());
+ for (const std::pair<Value *, unsigned> &Data : RedValsVect)
+ PossibleRedValsVect.back().append(Data.second, Data.first);
+ }
+ stable_sort(PossibleRedValsVect, [](const auto &P1, const auto &P2) {
+ return P1.size() > P2.size();
});
- Worklist.append(PossibleOrderedReductionOps.begin(),
- PossibleOrderedReductionOps.end());
- PossibleOrderedReductionOps.clear();
- }
- }
- // InstCombine may pack same-kind binops under a zext, hiding the leaves.
- // Analyze them in the narrow type; the vectorized result is widened back,
- // shifted and masked per lane before the reduction.
- if (Instruction::isBitwiseLogicOp(
- RecurrenceDescriptor::getOpcode(RdxKind)) &&
- RK == ReductionOrdering::Unordered && !IsCmpSelMinMax &&
- Ty->isIntegerTy() && !Ty->isIntegerTy(1) &&
- any_of(ReducedValsCandidates, [](Value *V) {
- return isa<ZExtInst>(V) ||
- match(V, m_Shl(m_ZExt(m_Value()), m_ConstantInt()));
- })) {
- unsigned WideBW = Ty->getIntegerBitWidth();
- SmallVector<SmallVector<NarrowedLeafInfo>> AllLeaves;
- SmallVector<Instruction *> ChainInsts;
- Type *NarrowTy = nullptr;
- bool Bail = false;
- unsigned NumLeaves = 0;
- SmallPtrSet<Value *, 8> UniqueLeaves;
- for (Value *Cand : ReducedValsCandidates) {
- SmallVector<NarrowedLeafInfo> &Leaves = AllLeaves.emplace_back();
- collectNarrowedLeaves(Cand, RecurrenceDescriptor::getOpcode(RdxKind),
- WideBW, RecursionMaxDepth, Leaves, ChainInsts);
- for (const NarrowedLeafInfo &L : Leaves) {
- if (isa<Constant>(L.V) || !L.V->getType()->isIntegerTy() ||
- (NarrowTy && L.V->getType() != NarrowTy) ||
- !UniqueLeaves.insert(L.V).second) {
- Bail = true;
- break;
- }
- if (!NarrowTy)
- NarrowTy = L.V->getType();
- }
- if (Bail)
- break;
- NumLeaves += Leaves.size();
- }
- if (!Bail && NumLeaves > ReducedValsCandidates.size() && NarrowTy != Ty) {
- // Collected user-first; dies with the reduction iff all users die.
- SmallPtrSet<Value *, 8> Ignorable;
- for (ArrayRef<Value *> RdxOps : ReductionOps)
- Ignorable.insert_range(RdxOps);
- for (Instruction *I : ChainInsts)
- if (all_of(I->users(),
- [&](User *U) { return Ignorable.contains(U); })) {
- Ignorable.insert(I);
- NarrowedChainInsts.push_back(I);
- }
- SmallVector<Value *> NewCandidates;
- for (const auto &[Cand, Leaves] :
- zip(ReducedValsCandidates, AllLeaves)) {
- SmallVector<Instruction *> RdxOps = ReducedValsToOps.lookup(Cand);
- for (const NarrowedLeafInfo &L : Leaves) {
- NarrowedLeafShifts.try_emplace(L.V, L);
- NewCandidates.push_back(L.V);
- ReducedValsToOps[L.V].append(RdxOps);
+ bool First = true;
+ for (ArrayRef<Value *> Data : PossibleRedValsVect) {
+ if (First) {
+ First = false;
+ ReducedVals.emplace_back();
+ } else if (!isGoodForReduction(Data)) {
+ auto *LI = dyn_cast<LoadInst>(Data.front());
+ auto *LastLI = dyn_cast<LoadInst>(ReducedVals.back().front());
+ if (!LI || !LastLI ||
+ getUnderlyingObject(LI->getPointerOperand()) !=
+ getUnderlyingObject(LastLI->getPointerOperand()))
+ ReducedVals.emplace_back();
}
- }
- ReducedValsCandidates = std::move(NewCandidates);
- }
- }
- // Too many integer reduced values candidates for the ordered reductions
- // after adjustements - try to switch to unordered reductions instead.
- constexpr unsigned ReducedValsLimit = 1024;
- if (ReducedValsCandidates.size() > ReducedValsLimit && AdjustedToOrdered &&
- ReducedValsCandidates.front()->getType()->isIntOrIntVectorTy())
- return false;
- // Add reduction values. The values are sorted for better vectorization
- // results.
- for (Value *V : ReducedValsCandidates) {
- if (RK == ReductionOrdering::Ordered && !isa<Instruction>(V))
- continue;
- size_t Key, Idx;
- std::tie(Key, Idx) = generateKeySubkey(V, &TLI, GenerateLoadsSubkey,
- /*AllowAlternate=*/false);
- ++PossibleReducedVals[Key][Idx].try_emplace(V, 0).first->second;
- }
- auto PossibleReducedValsVect = PossibleReducedVals.takeVector();
- // Sort values by the total number of values kinds to start the reduction
- // from the longest possible reduced values sequences.
- for (auto &PossibleReducedVals : PossibleReducedValsVect) {
- auto PossibleRedVals = PossibleReducedVals.second.takeVector();
- SmallVector<SmallVector<Value *>> PossibleRedValsVect;
- for (auto &Slice : PossibleRedVals) {
- PossibleRedValsVect.emplace_back();
- auto RedValsVect = Slice.second.takeVector();
- stable_sort(RedValsVect, llvm::less_second());
- for (const std::pair<Value *, unsigned> &Data : RedValsVect)
- PossibleRedValsVect.back().append(Data.second, Data.first);
- }
- stable_sort(PossibleRedValsVect, [](const auto &P1, const auto &P2) {
- return P1.size() > P2.size();
+ ReducedVals.back().append(Data.rbegin(), Data.rend());
+ }
+ }
+ // Post optimize reduced values to get better reduction sequences and
+ // sort them by size.
+ optimizeReducedVals(R, DT, DL, TTI, TLI);
+ // Sort the reduced values by number of same/alternate opcode and/or
+ // pointer operand. For the sign-aware reductions, same-size groups are
+ // ordered by their expected profitability: the first vectorized group
+ // is charged the cost of the reduction operation itself, which a group
+ // of loads or non-instructions rarely amortizes on its own, while it is
+ // a cheap addition (a single vector operation) to an already
+ // vectorized group - such groups go last. Among the rest, non-negated
+ // groups go first.
+ stable_sort(ReducedVals, [&](ArrayRef<Value *> P1, ArrayRef<Value *> P2) {
+ if (P1.size() != P2.size())
+ return P1.size() > P2.size();
+ if (NegatedReducedVals.empty())
+ return false;
+ auto IsCheapGroup = [](ArrayRef<Value *> P) {
+ return !isa<Instruction>(P.front()) || isa<LoadInst>(P.front());
+ };
+ bool Cheap1 = IsCheapGroup(P1);
+ bool Cheap2 = IsCheapGroup(P2);
+ if (Cheap1 != Cheap2)
+ return Cheap2;
+ return NegatedReducedVals.contains(P1.front()) <
+ NegatedReducedVals.contains(P2.front());
});
- bool First = true;
- for (ArrayRef<Value *> Data : PossibleRedValsVect) {
- if (First) {
- First = false;
- ReducedVals.emplace_back();
- } else if (!isGoodForReduction(Data)) {
- auto *LI = dyn_cast<LoadInst>(Data.front());
- auto *LastLI = dyn_cast<LoadInst>(ReducedVals.back().front());
- if (!LI || !LastLI ||
- getUnderlyingObject(LI->getPointerOperand()) !=
- getUnderlyingObject(LastLI->getPointerOperand()))
- ReducedVals.emplace_back();
+ // A group of subtracted loads, which are not consecutive (a gather):
+ // the flattening splits the fsub leaves over them into separate load
+ // groups of their operands, which the unflattened form vectorizes as a
+ // whole (a vector fsub with shuffled operands).
+ auto IsGatheredNegatedLoads = [&](ArrayRef<Value *> Vals) {
+ auto *LI0 = dyn_cast<LoadInst>(Vals.front());
+ if (!LI0 || Vals.size() < 2 || !NegatedReducedVals.contains(LI0))
+ return false;
+ SmallVector<int64_t> Diffs;
+ for (Value *V : Vals) {
+ auto *LI = dyn_cast<LoadInst>(V);
+ if (!LI)
+ return false;
+ std::optional<int64_t> Diff = getPointersDiff(
+ LI0->getType(), LI0->getPointerOperand(), LI->getType(),
+ LI->getPointerOperand(), DL, SE, /*StrictCheck=*/true);
+ if (!Diff)
+ return true;
+ Diffs.push_back(*Diff);
}
- ReducedVals.back().append(Data.rbegin(), Data.rend());
- }
- }
- // Post optimize reduced values to get better reduction sequences and sort
- // them by size.
- optimizeReducedVals(R, DT, DL, TTI, TLI);
- // Sort the reduced values by number of same/alternate opcode and/or pointer
- // operand.
- stable_sort(ReducedVals, [](ArrayRef<Value *> P1, ArrayRef<Value *> P2) {
- return P1.size() > P2.size();
- });
+ sort(Diffs);
+ Diffs.erase(llvm::unique(Diffs), Diffs.end());
+ return Diffs.back() - Diffs.front() + 1 !=
+ static_cast<int64_t>(Diffs.size());
+ };
+ // The flattening must not make the reduction unviable: the leaves,
+ // hidden behind the flattened fsub/fneg operations, may split into
+ // too small sign-uniform groups, while the unflattened form may
+ // still be vectorizable (e.g. with alternate fadd/fsub opcodes).
+ if (FlattenedNegations &&
+ (accumulate(ReducedVals, 0u,
+ [](unsigned Num, ArrayRef<Value *> Vals) -> unsigned {
+ if (!isGoodForReduction(Vals))
+ return Num;
+ return Num + Vals.size();
+ }) < ReductionLimit ||
+ any_of(ReducedVals, IsGatheredNegatedLoads)))
+ Restart = true;
+ } while (Restart);
return true;
}
@@ -30955,6 +31173,11 @@ class HorizontalReduction {
IRBuilder<TargetFolder> Builder(ReductionRoot->getContext(),
TargetFolder(DL));
Builder.SetInsertPoint(cast<Instruction>(ReductionRoot));
+ // The scalar parts (scaled reused values, their combines) are emitted right
+ // before the root; the instruction preceding them is remembered to drop
+ // them, if the sign-aware reduction is abandoned.
+ Instruction *InstBeforeRoot =
+ cast<Instruction>(ReductionRoot)->getPrevNode();
// Track the reduced values in case if they are replaced by extractelement
// because of the vectorization.
@@ -31025,6 +31248,13 @@ class HorizontalReduction {
for (Value *U : IgnoreList)
if (auto *FPMO = dyn_cast<FPMathOperator>(U))
RdxFMF &= FPMO->getFastMathFlags();
+ // Returns true if the original reduced value \p V is subtracted from the
+ // reduction result.
+ auto IsNegated = [&](Value *V) { return NegatedReducedVals.contains(V); };
+ // Scalar values subtracted from the reduction result: combined pairwise
+ // like the positive ones and subtracted in the final combine, so no
+ // extra negations are emitted.
+ SmallVector<std::pair<Instruction *, Value *>> NegExtraReductions;
// For ordered reductions here we need to generate extractelement
// instructions, so clear IgnoreList.
if (RK == ReductionOrdering::Ordered)
@@ -31049,6 +31279,27 @@ class HorizontalReduction {
// nodes and thus requiring extract if fully vectorized in other trees.
SmallPtrSet<Value *, 4> RequiredExtract;
WeakTrackingVH VectorizedTree = nullptr;
+ // The flattened reduction may be abandoned and retried without the
+ // fsub/fneg flattening, where the same lists of reduced values may be
+ // profitable - they are not marked as analyzed.
+ auto MarkAnalyzedVals = [&](ArrayRef<Value *> VL) {
+ if (!hasFlattenedNegations())
+ V.analyzedReductionVals(VL);
+ };
+ // Routes the emitted value by the sign of its group: negated parts are
+ // combined separately and subtracted in the final combine. \p OrigV is
+ // the original reduced value (null for the combined vector reduction
+ // result); its reduction operation provides the debug location.
+ auto AddReducedPart = [&](Value *OrigV, Value *Res, bool Negated) {
+ if (Negated) {
+ NegExtraReductions.emplace_back(OrigV
+ ? ReducedValsToOps.at(OrigV).front()
+ : cast<Instruction>(ReductionRoot),
+ Res);
+ return;
+ }
+ VectorizedTree = GetNewVectorizedTree(VectorizedTree, Res);
+ };
bool CheckForReusedReductionOps = false;
// Try to vectorize elements based on their type.
SmallVector<InstructionsState> States;
@@ -31061,6 +31312,14 @@ class HorizontalReduction {
ReducedVals.front().size() == ReducedVals.back().size() &&
ReducedVals.front().size() < ReductionLimit;
for (ArrayRef<Value *> RV : ReducedVals) {
+ // Groups with
diff erent signs of their contribution cannot be merged:
+ // the mixed group cannot be added/subtracted as a whole. The merge
+ // candidate is the last group of the same sign.
+ int MergeIdx = LocalReducedVals.size() - 1;
+ while (MergeIdx >= 0 && IsNegated(LocalReducedVals[MergeIdx].front()) !=
+ IsNegated(RV.front()))
+ --MergeIdx;
+ const bool SameSign = LocalReducedVals.empty() || MergeIdx >= 0;
// Loads are not very compatible with undefs.
if (isa<UndefValue>(RV.front()) &&
(States.empty() || !States.back() ||
@@ -31078,10 +31337,10 @@ class HorizontalReduction {
}
// Do some copyables analysis only if more than 2 groups exist or they
// are large enough.
- if (!TwoGroupsOfSameSmallSize) {
+ if (SameSign && !TwoGroupsOfSameSmallSize) {
SmallVector<Value *> Ops;
if (!LocalReducedVals.empty())
- Ops = LocalReducedVals.back();
+ Ops = LocalReducedVals[MergeIdx];
Ops.append(RV.begin(), RV.end());
InstructionsCompatibilityAnalysis Analysis(DT, DL, *TTI, TLI);
InstructionsState OpS = Analysis.buildInstructionsState(
@@ -31092,8 +31351,8 @@ class HorizontalReduction {
States.push_back(OpS);
continue;
}
- LocalReducedVals.back().swap(Ops);
- States.back() = OpS;
+ LocalReducedVals[MergeIdx].swap(Ops);
+ States[MergeIdx] = OpS;
continue;
}
// For safety, allow split vectorization only if 2 groups are available
@@ -31111,8 +31370,8 @@ class HorizontalReduction {
States.push_back(OpS);
continue;
}
- LocalReducedVals.back().swap(Ops);
- States.back() = OpS;
+ LocalReducedVals[MergeIdx].swap(Ops);
+ States[MergeIdx] = OpS;
continue;
}
}
@@ -31155,15 +31414,21 @@ class HorizontalReduction {
Candidates.push_back(RdxVal);
TrackedToOrig.push_back(ReducedVal);
}
+ // The groups are sign-uniform by matching, so the first original
+ // value's sign applies to all of them.
+ const bool GroupNegated =
+ !TrackedToOrig.empty() && IsNegated(TrackedToOrig.front());
bool ShuffledExtracts = false;
// Try to handle shuffled extractelements. Only pure extractelement
// groups can be merged: merged groups are skipped for external uses,
// and other values would be erased while still used by reduction ops.
if (S && S.getOpcode() == Instruction::ExtractElement &&
!S.isAltShuffle() && I + 1 < E &&
- all_of(ReducedVals[I + 1], [&](Value *RV) {
- return isa<ExtractElementInst>(TrackedVals.at(RV));
- })) {
+ all_of(ReducedVals[I + 1],
+ [&](Value *RV) {
+ return isa<ExtractElementInst>(TrackedVals.at(RV));
+ }) &&
+ GroupNegated == IsNegated(ReducedVals[I + 1].front())) {
SmallVector<Value *> CommonCandidates(Candidates);
for (Value *RV : ReducedVals[I + 1]) {
Value *RdxVal = TrackedVals.at(RV);
@@ -31183,6 +31448,9 @@ class HorizontalReduction {
ShuffledExtracts = true;
}
}
+ assert(all_of(TrackedToOrig,
+ [&](Value *V) { return IsNegated(V) == GroupNegated; }) &&
+ "Expected sign-uniform group of reduced values");
// Emit code for constant values.
if (Candidates.size() > 1 && allConstant(Candidates)) {
@@ -31199,13 +31467,16 @@ class HorizontalReduction {
if (auto *ResI = dyn_cast<Instruction>(Res))
V.analyzedReductionRoot(ResI);
}
- VectorizedTree = GetNewVectorizedTree(VectorizedTree, Res);
+ AddReducedPart(TrackedToOrig.front(), Res, GroupNegated);
continue;
}
unsigned NumReducedVals = Candidates.size();
+ // Sign-aware reductions pair small positive/negative groups: allow
+ // non-splat groups down to 2 elements.
if (NumReducedVals < ReductionLimit &&
- (NumReducedVals < 2 || !isSplat(Candidates)))
+ (NumReducedVals < 2 ||
+ (!isSplat(Candidates) && NegatedReducedVals.empty())))
continue;
// Check if we support repeated scalar values processing (optimization of
@@ -31256,9 +31527,24 @@ class HorizontalReduction {
if (NumReducedVals == 1) {
Value *OrigV = TrackedToOrig.front();
unsigned Cnt = At(SameValuesCounter, OrigV);
- Value *RedVal =
- emitScaleForReusedOps(Candidates.front(), Builder, Cnt);
- VectorizedTree = GetNewVectorizedTree(VectorizedTree, RedVal);
+ // A negated repeated value joins an already vectorized part of the
+ // same width as a splat operand of the vector fsub (a per-lane fnma
+ // with a multiply part) instead of the scalar scale and subtract.
+ if (GroupNegated &&
+ any_of(VectorValuesAndScales, [&](const ReductionVectorPart &P) {
+ return P.Vec && !P.ReducedInTree &&
+ getNumElements(P.Vec->getType()) == Cnt;
+ })) {
+ Value *Splat = Builder.CreateVectorSplat(Cnt, Candidates.front());
+ VectorValuesAndScales.push_back({Splat, /*Scale=*/1,
+ /*IsSigned=*/true,
+ /*ReducedInTree=*/false,
+ GroupNegated});
+ } else {
+ Value *RedVal =
+ emitScaleForReusedOps(Candidates.front(), Builder, Cnt);
+ AddReducedPart(OrigV, RedVal, GroupNegated);
+ }
VectorizedVals.try_emplace(OrigV, Cnt);
ExternallyUsedValues.insert(OrigV);
continue;
@@ -31324,8 +31610,11 @@ class HorizontalReduction {
};
bool AnyVectorized = false;
SmallDenseSet<std::pair<unsigned, unsigned>, 8> IgnoredCandidates;
+ // Same small-group allowance for the vector width.
+ const unsigned MinReduxWidth =
+ NegatedReducedVals.empty() ? ReductionLimit : 2;
while (Pos < NumReducedVals - ReduxWidth + 1 &&
- ReduxWidth >= ReductionLimit) {
+ ReduxWidth >= MinReduxWidth) {
// Dependency in tree of the reduction ops - drop this attempt, try
// later.
if (CheckForReusedReductionOpsLocal && PrevReduxWidth != ReduxWidth &&
@@ -31365,7 +31654,7 @@ class HorizontalReduction {
constexpr unsigned CandidatesLimit = 64;
if (!AdjustReducedVals(RK == ReductionOrdering::Ordered &&
Candidates.size() >= CandidatesLimit))
- V.analyzedReductionVals(VL);
+ MarkAnalyzedVals(VL);
continue;
}
V.reorderTopToBottom();
@@ -31466,7 +31755,7 @@ class HorizontalReduction {
<< ore::NV("Threshold", -SLPCostThreshold);
});
if (!AdjustReducedVals()) {
- V.analyzedReductionVals(VL);
+ MarkAnalyzedVals(VL);
unsigned Offset = Pos == Start ? Pos : Pos - 1;
if (ReduxWidth > ReductionLimit && V.isTreeNotExtendable()) {
// Add subvectors of VL to the list of the analyzed values.
@@ -31581,15 +31870,18 @@ class HorizontalReduction {
Type *ScalarTy = VL.front()->getType();
Type *VecTy = VectorizedRoot->getType();
Type *RedScalarTy = VecTy->getScalarType();
- VectorValuesAndScales.emplace_back(
- VectorizedRoot,
- OptReusedScalars && SameScaleFactor
- ? SameValuesCounter.front().second
- : 1,
- RedScalarTy != ScalarTy->getScalarType()
- ? NarrowedLeafShifts.empty() && V.isSignedMinBitwidthRootNode()
- : true,
- V.isReducedBitcastRoot() || V.isReducedCmpBitcastRoot());
+ // A group of negated reduced values (from an fsub/fneg in the chain)
+ // is subtracted in the final combine.
+ VectorValuesAndScales.push_back(
+ {VectorizedRoot,
+ OptReusedScalars && SameScaleFactor
+ ? SameValuesCounter.front().second
+ : 1,
+ RedScalarTy != ScalarTy->getScalarType()
+ ? NarrowedLeafShifts.empty() && V.isSignedMinBitwidthRootNode()
+ : true,
+ V.isReducedBitcastRoot() || V.isReducedCmpBitcastRoot(),
+ GroupNegated});
// Count vectorized reduced values to exclude them from final reduction.
for (const auto [Idx, RdxVal] : enumerate(VL)) {
@@ -31613,7 +31905,7 @@ class HorizontalReduction {
for (const std::pair<Value *, unsigned> &P : SameValuesCounter) {
Value *RdxVal = TrackedVals.at(P.first);
Value *RedVal = emitScaleForReusedOps(RdxVal, Builder, P.second);
- VectorizedTree = GetNewVectorizedTree(VectorizedTree, RedVal);
+ AddReducedPart(P.first, RedVal, GroupNegated);
VectorizedVals.try_emplace(P.first, P.second);
}
continue;
@@ -31621,15 +31913,37 @@ class HorizontalReduction {
}
// Early exit for the ordered reductions.
// No need to do anything else here, so we can just exit.
- if (RK == ReductionOrdering::Ordered)
+ if (RK == ReductionOrdering::Ordered) {
+ assert(NegatedReducedVals.empty() &&
+ "Unexpected negated reduced values in the ordered reduction");
return VectorizedTree;
+ }
- if (!VectorValuesAndScales.empty())
- VectorizedTree = GetNewVectorizedTree(
- VectorizedTree,
- emitReduction(Builder, *TTI, ReductionRoot->getType()));
+ if (!VectorValuesAndScales.empty()) {
+ auto [Res, ResNegated] =
+ emitReduction(Builder, *TTI, ReductionRoot->getType());
+ // The reduction result of the all-negated parts is subtracted in the
+ // final combine.
+ AddReducedPart(/*OrigV=*/nullptr, Res, ResNegated);
+ } else if (hasFlattenedNegations()) {
+ // The flattened reduction pays off only with a vector part. Without one
+ // the scalar chain is merely reshuffled (which may also break the scalar
+ // fma formation), while the original code may still be vectorizable as
+ // an operand tree - drop the scalar parts emitted so far and give up.
+ auto *Root = cast<Instruction>(ReductionRoot);
+ for (Instruction *I = Root->getPrevNode(); I && I != InstBeforeRoot;) {
+ Instruction *Prev = I->getPrevNode();
+ if (I->use_empty() && !V.isAnalyzedReductionRoot(I))
+ I->eraseFromParent();
+ I = Prev;
+ }
+ // The fsub/fneg leaves themselves may still form vectorizable groups
+ // and the chains behind them are matched as separate reductions - leave
+ // the reduction operations unmarked for the retry.
+ return nullptr;
+ }
- if (!VectorizedTree) {
+ if (!VectorizedTree && NegExtraReductions.empty()) {
if (!CheckForReusedReductionOps) {
for (ReductionOpsType &RdxOps : ReductionOps)
for (Value *RdxOp : RdxOps)
@@ -31736,8 +32050,9 @@ class HorizontalReduction {
return ExtraReds;
};
SmallVector<std::pair<Instruction *, Value *>> ExtraReductions;
- ExtraReductions.emplace_back(cast<Instruction>(ReductionRoot),
- VectorizedTree);
+ if (VectorizedTree)
+ ExtraReductions.emplace_back(cast<Instruction>(ReductionRoot),
+ VectorizedTree);
SmallPtrSet<Value *, 8> Visited;
for (ArrayRef<Value *> Candidates : ReducedVals) {
for (Value *RdxVal : Candidates) {
@@ -31746,18 +32061,44 @@ class HorizontalReduction {
unsigned NumOps = VectorizedVals.lookup(RdxVal);
for (Instruction *RedOp :
ArrayRef(ReducedValsToOps.at(RdxVal)).drop_back(NumOps))
- ExtraReductions.emplace_back(RedOp, RdxVal);
- }
- }
- // Iterate through all not-vectorized reduction values/extra arguments.
- bool InitStep = true;
- while (ExtraReductions.size() > 1) {
- SmallVector<std::pair<Instruction *, Value *>> NewReds =
- FinalGen(ExtraReductions, InitStep);
- ExtraReductions.swap(NewReds);
- InitStep = false;
+ (IsNegated(RdxVal) ? NegExtraReductions : ExtraReductions)
+ .emplace_back(RedOp, RdxVal);
+ }
+ }
+ // Iterate through all not-vectorized reduction values/extra arguments,
+ // combining them pairwise.
+ auto CombineReds =
+ [&](SmallVectorImpl<std::pair<Instruction *, Value *>> &Reds) {
+ bool InitStep = true;
+ while (Reds.size() > 1) {
+ SmallVector<std::pair<Instruction *, Value *>> NewReds =
+ FinalGen(Reds, InitStep);
+ Reds.swap(NewReds);
+ InitStep = false;
+ }
+ Value *RdxVal = Reds.front().second;
+ auto It = TrackedVals.find(RdxVal);
+ if (It != TrackedVals.end())
+ RdxVal = It->second;
+ return RdxVal;
+ };
+ VectorizedTree = nullptr;
+ if (!ExtraReductions.empty())
+ VectorizedTree = CombineReds(ExtraReductions);
+ // Combine the subtracted values pairwise as well and subtract the combined
+ // value from the positive part of the reduction (or just negate it, if
+ // there is no positive part).
+ if (!NegExtraReductions.empty()) {
+ Value *NegTree = CombineReds(NegExtraReductions);
+ Builder.SetCurrentDebugLocation(
+ cast<Instruction>(ReductionOps.front().front())->getDebugLoc());
+ Value *Op = VectorizedTree
+ ? Builder.CreateFSub(VectorizedTree, NegTree, "op.rdx")
+ : Builder.CreateFNeg(NegTree, "op.rdx");
+ propagateIRFlags(Op, ReductionOps.front(), nullptr,
+ /*IncludeWrapFlags=*/false);
+ VectorizedTree = Op;
}
- VectorizedTree = ExtraReductions.front().second;
ReductionRoot->replaceAllUsesWith(VectorizedTree);
@@ -31804,6 +32145,8 @@ class HorizontalReduction {
assert(RK == ReductionOrdering::Ordered && "Expected ordered reduction");
assert(ReducedVals.size() == 1 &&
"Expected single group from matchOrderedReduction");
+ assert(NegatedReducedVals.empty() &&
+ "Unexpected negated reduced values in the ordered reduction");
IRBuilder<TargetFolder> Builder(ReductionRoot->getContext(),
TargetFolder(DL));
@@ -32155,7 +32498,13 @@ class HorizontalReduction {
if (It != ReducedValsToOps.end())
return It->second.front();
for (User *U : RdxVal->users())
- if (getRdxKind(U) == RdxKind)
+ // For flattened fadd reductions the reduced value may be consumed by
+ // an fsub/fneg chain link instead of an fadd. No reassoc/nsz gating
+ // is needed: the result is only a cost context.
+ if (getRdxKind(U) == RdxKind ||
+ (RdxKind == RecurKind::FAdd &&
+ (match(U, m_FSub(m_Value(), m_Value())) ||
+ match(U, m_FNeg(m_Value())))))
return cast<Instruction>(U);
return cast<Instruction>(RdxVal);
};
@@ -32184,10 +32533,12 @@ class HorizontalReduction {
for (User *U : RdxVal->users()) {
auto *RdxOp = cast<Instruction>(U);
if (hasRequiredNumberOfUses(IsCmpSelMinMax, RdxOp)) {
- if (RdxKind == RecurKind::FAdd) {
- InstructionCost FMACost =
- canConvertToFMA(RdxOp, getSameOpcode(RdxOp, TLI), DT, DL,
- *TTI, TLI, CostKind);
+ InstructionsState RdxOpS = RdxKind == RecurKind::FAdd
+ ? getSameOpcode(RdxOp, TLI)
+ : InstructionsState::invalid();
+ if (RdxOpS && RdxOpS.isAddSubOrFNegLikeOp()) {
+ InstructionCost FMACost = canConvertToFMA(
+ RdxOp, RdxOpS, DT, DL, *TTI, TLI, CostKind);
if (FMACost.isValid()) {
LLVM_DEBUG(dbgs() << "FMA cost: " << FMACost << "\n");
if (auto *I = dyn_cast<Instruction>(RdxVal)) {
@@ -32312,8 +32663,9 @@ class HorizontalReduction {
Ops.push_back(RdxVal->user_back());
}
if (!Ops.empty()) {
- FMACost = canConvertToFMA(Ops, getSameOpcode(Ops, TLI), DT, DL,
- *TTI, TLI, CostKind);
+ InstructionsState S = getSameOpcode(Ops, TLI);
+ if (S && S.isAddSubOrFNegLikeOp())
+ FMACost = canConvertToFMA(Ops, S, DT, DL, *TTI, TLI, CostKind);
if (FMACost.isValid()) {
// Calculate actual FMAD cost.
IntrinsicCostAttributes ICA(Intrinsic::fmuladd, RVecTy,
@@ -32347,11 +32699,24 @@ class HorizontalReduction {
Type *ScalarCostTy =
!NarrowedLeafShifts.empty() ? ReductionRoot->getType() : ScalarTy;
ScalarCost = EvaluateScalarCost([&](Instruction *RdxOp) {
+ // A flattened fneg chain link is removed together with the fadd/fsub
+ // consuming it - cost both. Also, RdxOp may be the reduced value
+ // itself, if the actual reduction operation was not found - guard the
+ // second operand access.
+ if (RdxOp->isUnaryOp())
+ return TTI->getArithmeticInstrCost(
+ RdxOp->getOpcode(), ScalarCostTy, CostKind,
+ TTI::getOperandInfo(RdxOp->getOperand(0)), {}, {}, RdxOp) +
+ TTI->getArithmeticInstrCost(RdxOpcode, ScalarCostTy, CostKind);
+ TTI::OperandValueInfo Op2Info;
+ if (RdxOp->getNumOperands() > 1)
+ Op2Info = TTI::getOperandInfo(RdxOp->getOperand(1));
return TTI->getArithmeticInstrCost(
RdxOpcode, ScalarCostTy, CostKind,
- TTI::getOperandInfo(RdxOp->getOperand(0)),
- TTI::getOperandInfo(RdxOp->getOperand(1)), {}, RdxOp);
+ TTI::getOperandInfo(RdxOp->getOperand(0)), Op2Info, {}, RdxOp);
});
+ // Sign-flipped combines are emitted as fsubs but costed as fadds:
+ // same op count as the unflattened chain, same cost.
break;
}
case RecurKind::FMax:
@@ -32404,34 +32769,65 @@ class HorizontalReduction {
/// Splits the values, stored in VectorValuesAndScales, into registers/free
/// sub-registers, combines them with the given reduction operation as a
/// vector operation and then performs single (small enough) reduction.
- Value *emitReduction(IRBuilderBase &Builder, const TargetTransformInfo &TTI,
- Type *DestTy) {
+ /// The parts, marked as negated, are combined with the vector fsub operation
+ /// (which may form per-lane fma with the multiplications in the reduced
+ /// values). \returns the combined value and a flag set if it represents a
+ /// negated result (all parts negated) and must be subtracted in the final
+ /// combine.
+ std::pair<Value *, bool> emitReduction(IRBuilderBase &Builder,
+ const TargetTransformInfo &TTI,
+ Type *DestTy) {
Value *ReducedSubTree = nullptr;
- // Creates reduction and combines with the previous reduction.
+ bool ResNegated = false;
+ // Combines the accumulated value \p Acc (with the sign \p AccNegated)
+ // with \p V (with the sign \p VNegated). Values with
diff erent signs are
+ // combined via fsub, subtracting the negated one from the positive one, so
+ // the result is positive. If \p KeepAccSign is set (the result is inserted
+ // back into the wider \p Acc), the result keeps the sign of \p Acc
+ // instead: with the negated Acc it is Acc - V == -(V - Acc).
+ // \returns the combined value and its sign.
+ auto CombineOps = [&](Value *Acc, bool AccNegated, Value *V, bool VNegated,
+ const Twine &Name,
+ bool KeepAccSign) -> std::pair<Value *, bool> {
+ if (AccNegated == VNegated)
+ return {createOp(Builder, RdxKind, Acc, V, Name, ReductionOps),
+ AccNegated};
+ assert(RdxKind == RecurKind::FAdd &&
+ "Negated parts expected in the fadd reductions only");
+ if (AccNegated && !KeepAccSign)
+ std::swap(Acc, V);
+ return {createOp(Builder, RecurKind::FSub, Acc, V, Name, ReductionOps),
+ AccNegated && KeepAccSign};
+ };
+ // Creates reduction and combines with the previous reduction, respecting
+ // the signs of the operands.
auto CreateSingleOp = [&](Value *Vec, unsigned Scale, bool IsSigned,
- bool ReducedInTree) {
+ bool ReducedInTree, bool Negated) {
Value *Rdx = createSingleOp(Builder, TTI, Vec, Scale, IsSigned, DestTy,
ReducedInTree);
- if (ReducedSubTree)
- ReducedSubTree = createOp(Builder, RdxKind, ReducedSubTree, Rdx,
- "op.rdx", ReductionOps);
- else
+ if (!ReducedSubTree) {
ReducedSubTree = Rdx;
+ ResNegated = Negated;
+ return;
+ }
+ std::tie(ReducedSubTree, ResNegated) =
+ CombineOps(ReducedSubTree, ResNegated, Rdx, Negated, "op.rdx",
+ /*KeepAccSign=*/false);
};
if (VectorValuesAndScales.size() == 1) {
- const auto &[Vec, Scale, IsSigned, ReducedInTree] =
- VectorValuesAndScales.front();
- CreateSingleOp(Vec, Scale, IsSigned, ReducedInTree);
- return ReducedSubTree;
+ const ReductionVectorPart &P = VectorValuesAndScales.front();
+ CreateSingleOp(P.Vec, P.Scale, P.IsSigned, P.ReducedInTree, P.Negated);
+ return {ReducedSubTree, ResNegated};
}
// Scales Vec using given Cnt scale factor and then performs vector combine
// with previous value of VecOp.
Value *VecRes = nullptr;
bool VecResSignedness = false;
+ bool VecResNegated = false;
auto CreateVecOp = [&](Value *Vec, unsigned Cnt, bool IsSigned,
- bool ReducedInTree) {
+ bool ReducedInTree, bool Negated) {
if (ReducedInTree) {
- CreateSingleOp(Vec, Cnt, IsSigned, ReducedInTree);
+ CreateSingleOp(Vec, Cnt, IsSigned, ReducedInTree, Negated);
return;
}
Type *ScalarTy = Vec->getType()->getScalarType();
@@ -32517,6 +32913,7 @@ class HorizontalReduction {
if (!VecRes) {
VecRes = Vec;
VecResSignedness = IsSigned;
+ VecResNegated = Negated;
} else {
++NumVectorInstructions;
if (ScalarTy == Builder.getInt1Ty() && ScalarTy != DestTy &&
@@ -32562,23 +32959,30 @@ class HorizontalReduction {
if (VecResVF < VecVF) {
std::swap(VecRes, Vec);
std::swap(VecResVF, VecVF);
+ std::swap(VecResNegated, Negated);
}
// extract + op + insert
Value *Op = VecRes;
if (VecResVF != VecVF)
Op = createExtractVector(Builder, VecRes, VecVF, /*Index=*/0);
- Op = createOp(Builder, RdxKind, Op, Vec, "rdx.op", ReductionOps);
+ std::tie(Op, VecResNegated) =
+ CombineOps(Op, VecResNegated, Vec, Negated, "rdx.op",
+ /*KeepAccSign=*/VecResVF != VecVF);
if (VecResVF != VecVF)
Op = createInsertVector(Builder, VecRes, Op, /*Index=*/0);
VecRes = Op;
}
};
- for (auto [Vec, Scale, IsSigned, ReducedInTree] : VectorValuesAndScales)
- CreateVecOp(Vec, Scale, IsSigned, ReducedInTree);
+ // Emit the positive parts first, so the combined vector value stays
+ // positive when there is at least one positive part.
+ for (bool Negated : {false, true})
+ for (const ReductionVectorPart &P : VectorValuesAndScales)
+ if (P.Negated == Negated)
+ CreateVecOp(P.Vec, P.Scale, P.IsSigned, P.ReducedInTree, P.Negated);
CreateSingleOp(VecRes, /*Scale=*/1, /*IsSigned=*/false,
- /*ReducedInTree=*/false);
+ /*ReducedInTree=*/false, VecResNegated);
- return ReducedSubTree;
+ return {ReducedSubTree, ResNegated};
}
/// Emit a horizontal reduction of the vectorized value.
@@ -32639,7 +33043,15 @@ class HorizontalReduction {
Value *Scale = ConstantFP::get(VectorizedValue->getType(), Cnt);
LLVM_DEBUG(dbgs() << "SLP: FAdd (to-fmul) " << Cnt << "of "
<< VectorizedValue << ". (HorRdx)\n");
- return Builder.CreateFMul(VectorizedValue, Scale);
+ // The mul stands in for a run of the chain operations and joins the
+ // same combines, so it carries the flags of the whole chain, except
+ // contract: an fma candidate would only block the later vectorization
+ // of the scaled value's operands (fma candidates are deferred).
+ Value *Mul = createOp(Builder, RecurKind::FMul, VectorizedValue, Scale,
+ "", ReductionOps);
+ if (auto *MulI = dyn_cast<Instruction>(Mul))
+ MulI->setHasAllowContract(false);
+ return Mul;
}
case RecurKind::And:
case RecurKind::Or:
@@ -33031,12 +33443,23 @@ bool SLPVectorizerPass::vectorizeHorReduction(
return nullptr;
HorizontalReduction HorRdx;
Value *Res = nullptr;
- if (HorRdx.matchAssociativeReduction(R, Inst, *SE, *DT, *DL, *TTI, *TLI))
- if (Value *Red = HorRdx.tryToReduce(R, *DL, TTI, *TLI, AC, *DT)) {
+ if (HorRdx.matchAssociativeReduction(R, Inst, *SE, *DT, *DL, *TTI, *TLI)) {
+ Value *Red = HorRdx.tryToReduce(R, *DL, TTI, *TLI, AC, *DT);
+ // The chain with the flattened fsub/fneg links produced no vector part:
+ // retry with the fsub/fneg links as leaves, they may be vectorizable on
+ // their own.
+ if (!Red && HorRdx.hasFlattenedNegations()) {
+ HorizontalReduction UnflattenedHorRdx;
+ if (UnflattenedHorRdx.matchAssociativeReduction(
+ R, Inst, *SE, *DT, *DL, *TTI, *TLI, /*FlattenNegations=*/false))
+ Red = UnflattenedHorRdx.tryToReduce(R, *DL, TTI, *TLI, AC, *DT);
+ }
+ if (Red) {
if (Red != Inst)
return Red;
Res = Red;
}
+ }
if (HorizontalReduction::isSupportedOrderedReductionOp(Inst)) {
if (HorRdx.matchOrderedReduction(R, Inst, /*MatchLHS=*/true))
if (Value *Red =
diff --git a/llvm/lib/Transforms/Vectorize/SLPVectorizer/SLPCompatibilityAnalysis.cpp b/llvm/lib/Transforms/Vectorize/SLPVectorizer/SLPCompatibilityAnalysis.cpp
index 133abe65208c9..d72f5ad94b767 100644
--- a/llvm/lib/Transforms/Vectorize/SLPVectorizer/SLPCompatibilityAnalysis.cpp
+++ b/llvm/lib/Transforms/Vectorize/SLPVectorizer/SLPCompatibilityAnalysis.cpp
@@ -539,6 +539,10 @@ bool InstructionsState::isAddSubLikeOp() const {
is_contained(AddSub, getAltOpcode());
}
+bool InstructionsState::isAddSubOrFNegLikeOp() const {
+ return isAddSubLikeOp() || getOpcode() == Instruction::FNeg;
+}
+
bool InstructionsState::isCopyableElement(Value *V) const {
assert(valid() && "InstructionsState is invalid.");
if (!HasCopyables)
diff --git a/llvm/lib/Transforms/Vectorize/SLPVectorizer/SLPCompatibilityAnalysis.h b/llvm/lib/Transforms/Vectorize/SLPVectorizer/SLPCompatibilityAnalysis.h
index 2c8978f520e91..c29b760cea113 100644
--- a/llvm/lib/Transforms/Vectorize/SLPVectorizer/SLPCompatibilityAnalysis.h
+++ b/llvm/lib/Transforms/Vectorize/SLPVectorizer/SLPCompatibilityAnalysis.h
@@ -267,6 +267,11 @@ class InstructionsState {
/// Checks if main/alt instructions are add/sub/fadd/fsub operations.
bool isAddSubLikeOp() const;
+ /// Checks for an add/sub-like operation or an fneg, which models a
+ /// flattened fadd reduction chain link: the link is dropped and its
+ /// operand joins a sign-flipped combine, which may form an fma as well.
+ bool isAddSubOrFNegLikeOp() const;
+
/// Checks if main/alt instructions are cmp operations.
bool isCmpOp() const {
return (getOpcode() == Instruction::ICmp ||
diff --git a/llvm/test/Transforms/SLPVectorizer/AArch64/loop-accumulator-reduction.ll b/llvm/test/Transforms/SLPVectorizer/AArch64/loop-accumulator-reduction.ll
index 256d16254ea29..df153a38c0558 100644
--- a/llvm/test/Transforms/SLPVectorizer/AArch64/loop-accumulator-reduction.ll
+++ b/llvm/test/Transforms/SLPVectorizer/AArch64/loop-accumulator-reduction.ll
@@ -1132,7 +1132,7 @@ define double @acc_used_twice(ptr %p, i64 %n) {
; CHECK-NEXT: [[ACC:%.*]] = phi double [ 1.000000e+00, %[[ENTRY]] ], [ [[OP_RDX:%.*]], %[[LOOP]] ]
; CHECK-NEXT: [[P0:%.*]] = getelementptr double, ptr [[P]], i64 [[IV]]
; CHECK-NEXT: [[TMP0:%.*]] = load <4 x double>, ptr [[P0]], align 8
-; CHECK-NEXT: [[TMP1:%.*]] = fmul fast double [[ACC]], 2.000000e+00
+; CHECK-NEXT: [[TMP1:%.*]] = fmul reassoc nnan ninf nsz arcp afn double [[ACC]], 2.000000e+00
; CHECK-NEXT: [[TMP2:%.*]] = call fast double @llvm.vector.reduce.fadd.v4f64(double 0.000000e+00, <4 x double> [[TMP0]])
; CHECK-NEXT: [[OP_RDX]] = fadd fast double [[TMP1]], [[TMP2]]
; CHECK-NEXT: [[IV_NEXT]] = add nuw nsw i64 [[IV]], 1
diff --git a/llvm/test/Transforms/SLPVectorizer/AArch64/many-uses-fma-candidate.ll b/llvm/test/Transforms/SLPVectorizer/AArch64/many-uses-fma-candidate.ll
index fae8110ccf04f..d34acc74f492c 100644
--- a/llvm/test/Transforms/SLPVectorizer/AArch64/many-uses-fma-candidate.ll
+++ b/llvm/test/Transforms/SLPVectorizer/AArch64/many-uses-fma-candidate.ll
@@ -14,7 +14,7 @@ define double @test(ptr %0, ptr %1) {
; CHECK-NEXT: [[TMP8:%.*]] = load double, ptr [[TMP0]], align 8
; CHECK-NEXT: [[TMP9:%.*]] = fmul reassoc nsz <4 x double> [[TMP4]], splat (double 1.000000e+00)
; CHECK-NEXT: [[TMP10:%.*]] = call reassoc nsz double @llvm.vector.reduce.fadd.v4f64(double 0.000000e+00, <4 x double> [[TMP9]])
-; CHECK-NEXT: [[TMP11:%.*]] = fmul double [[TMP5]], 2.000000e+00
+; CHECK-NEXT: [[TMP11:%.*]] = fmul reassoc nsz double [[TMP5]], 2.000000e+00
; CHECK-NEXT: [[OP_RDX5:%.*]] = fadd reassoc nsz double [[TMP11]], [[TMP8]]
; CHECK-NEXT: [[OP_RDX6:%.*]] = fadd reassoc nsz double [[OP_RDX5]], [[TMP6]]
; CHECK-NEXT: [[OP_RDX7:%.*]] = fadd reassoc nsz double [[OP_RDX6]], [[TMP7]]
diff --git a/llvm/test/Transforms/SLPVectorizer/AMDGPU/reduction.ll b/llvm/test/Transforms/SLPVectorizer/AMDGPU/reduction.ll
index 4929073d931db..ce5871b094f8a 100644
--- a/llvm/test/Transforms/SLPVectorizer/AMDGPU/reduction.ll
+++ b/llvm/test/Transforms/SLPVectorizer/AMDGPU/reduction.ll
@@ -158,16 +158,26 @@ entry:
; FIXME: support vectorization;
define half @reduction_sub_half4(<4 x half> %a) {
-; GCN-LABEL: @reduction_sub_half4(
-; GCN-NEXT: entry:
-; GCN-NEXT: [[ELT0:%.*]] = extractelement <4 x half> [[A:%.*]], i64 0
-; GCN-NEXT: [[ELT1:%.*]] = extractelement <4 x half> [[A]], i64 1
-; GCN-NEXT: [[ELT2:%.*]] = extractelement <4 x half> [[A]], i64 2
-; GCN-NEXT: [[ELT3:%.*]] = extractelement <4 x half> [[A]], i64 3
-; GCN-NEXT: [[ADD1:%.*]] = fsub fast half [[ELT1]], [[ELT0]]
-; GCN-NEXT: [[ADD2:%.*]] = fsub fast half [[ELT2]], [[ADD1]]
-; GCN-NEXT: [[ADD3:%.*]] = fsub fast half [[ELT3]], [[ADD2]]
-; GCN-NEXT: ret half [[ADD3]]
+; GFX9-LABEL: @reduction_sub_half4(
+; GFX9-NEXT: entry:
+; GFX9-NEXT: [[ELT0:%.*]] = extractelement <4 x half> [[A:%.*]], i64 0
+; GFX9-NEXT: [[ELT2:%.*]] = extractelement <4 x half> [[A]], i64 2
+; GFX9-NEXT: [[TMP0:%.*]] = shufflevector <4 x half> [[A]], <4 x half> poison, <2 x i32> <i32 1, i32 3>
+; GFX9-NEXT: [[TMP1:%.*]] = call fast half @llvm.vector.reduce.fadd.v2f16(half 0.000000e+00, <2 x half> [[TMP0]])
+; GFX9-NEXT: [[OP_RDX:%.*]] = fadd fast half [[ELT0]], [[ELT2]]
+; GFX9-NEXT: [[OP_RDX1:%.*]] = fsub fast half [[TMP1]], [[OP_RDX]]
+; GFX9-NEXT: ret half [[OP_RDX1]]
+;
+; VI-LABEL: @reduction_sub_half4(
+; VI-NEXT: entry:
+; VI-NEXT: [[ELT0:%.*]] = extractelement <4 x half> [[A:%.*]], i64 0
+; VI-NEXT: [[ELT1:%.*]] = extractelement <4 x half> [[A]], i64 1
+; VI-NEXT: [[ELT2:%.*]] = extractelement <4 x half> [[A]], i64 2
+; VI-NEXT: [[ELT3:%.*]] = extractelement <4 x half> [[A]], i64 3
+; VI-NEXT: [[ADD1:%.*]] = fsub fast half [[ELT1]], [[ELT0]]
+; VI-NEXT: [[ADD2:%.*]] = fsub fast half [[ELT2]], [[ADD1]]
+; VI-NEXT: [[ADD3:%.*]] = fsub fast half [[ELT3]], [[ADD2]]
+; VI-NEXT: ret half [[ADD3]]
;
entry:
%elt0 = extractelement <4 x half> %a, i64 0
diff --git a/llvm/test/Transforms/SLPVectorizer/X86/fma-operand-index.ll b/llvm/test/Transforms/SLPVectorizer/X86/fma-operand-index.ll
index abf8abf106eee..0fa4058f874d2 100644
--- a/llvm/test/Transforms/SLPVectorizer/X86/fma-operand-index.ll
+++ b/llvm/test/Transforms/SLPVectorizer/X86/fma-operand-index.ll
@@ -46,17 +46,12 @@ define double @fmul_rhs_fsub(ptr %x, ptr %y, ptr %z) {
; CHECK-LABEL: define double @fmul_rhs_fsub(
; CHECK-SAME: ptr [[X:%.*]], ptr [[Y:%.*]], ptr [[Z:%.*]]) #[[ATTR0]] {
; CHECK-NEXT: [[ENTRY:.*:]]
-; CHECK-NEXT: [[Z8:%.*]] = getelementptr inbounds nuw i8, ptr [[Z]], i64 8
-; CHECK-NEXT: [[Z0:%.*]] = load double, ptr [[Z]], align 8
; CHECK-NEXT: [[TMP0:%.*]] = load <2 x double>, ptr [[X]], align 8
; CHECK-NEXT: [[TMP1:%.*]] = load <2 x double>, ptr [[Y]], align 8
; CHECK-NEXT: [[TMP2:%.*]] = fmul reassoc nsz contract <2 x double> [[TMP1]], [[TMP0]]
-; CHECK-NEXT: [[Z1:%.*]] = load double, ptr [[Z8]], align 8
-; CHECK-NEXT: [[ZSUM:%.*]] = fadd reassoc nsz contract double [[Z0]], [[Z1]]
-; CHECK-NEXT: [[MUL0:%.*]] = extractelement <2 x double> [[TMP2]], i64 0
-; CHECK-NEXT: [[SUB0:%.*]] = fsub reassoc nsz contract double [[ZSUM]], [[MUL0]]
-; CHECK-NEXT: [[MUL1:%.*]] = extractelement <2 x double> [[TMP2]], i64 1
-; CHECK-NEXT: [[SUB1:%.*]] = fsub reassoc nsz contract double [[SUB0]], [[MUL1]]
+; CHECK-NEXT: [[TMP3:%.*]] = load <2 x double>, ptr [[Z]], align 8
+; CHECK-NEXT: [[RDX_OP:%.*]] = fsub reassoc nsz contract <2 x double> [[TMP3]], [[TMP2]]
+; CHECK-NEXT: [[SUB1:%.*]] = call reassoc nsz contract double @llvm.vector.reduce.fadd.v2f64(double 0.000000e+00, <2 x double> [[RDX_OP]])
; CHECK-NEXT: ret double [[SUB1]]
;
entry:
diff --git a/llvm/test/Transforms/SLPVectorizer/X86/horizontal-fadd-with-sub.ll b/llvm/test/Transforms/SLPVectorizer/X86/horizontal-fadd-with-sub.ll
index 915bdbb01548e..e8c566ecdaabc 100644
--- a/llvm/test/Transforms/SLPVectorizer/X86/horizontal-fadd-with-sub.ll
+++ b/llvm/test/Transforms/SLPVectorizer/X86/horizontal-fadd-with-sub.ll
@@ -9,17 +9,12 @@ define double @fsub_fmul_2(ptr %x, ptr %y, ptr %z) {
; CHECK-LABEL: define double @fsub_fmul_2(
; CHECK-SAME: ptr [[X:%.*]], ptr [[Y:%.*]], ptr [[Z:%.*]]) #[[ATTR0:[0-9]+]] {
; CHECK-NEXT: [[ENTRY:.*:]]
-; CHECK-NEXT: [[Z8:%.*]] = getelementptr inbounds nuw i8, ptr [[Z]], i64 8
-; CHECK-NEXT: [[Z0:%.*]] = load double, ptr [[Z]], align 8
; CHECK-NEXT: [[TMP2:%.*]] = load <2 x double>, ptr [[X]], align 8
; CHECK-NEXT: [[TMP1:%.*]] = load <2 x double>, ptr [[Y]], align 8
; CHECK-NEXT: [[TMP3:%.*]] = fmul reassoc nsz contract <2 x double> [[TMP1]], [[TMP2]]
-; CHECK-NEXT: [[Z1:%.*]] = load double, ptr [[Z8]], align 8
-; CHECK-NEXT: [[ZSUM:%.*]] = fadd reassoc nsz contract double [[Z0]], [[Z1]]
-; CHECK-NEXT: [[TMP5:%.*]] = extractelement <2 x double> [[TMP3]], i64 0
-; CHECK-NEXT: [[SUB:%.*]] = fsub reassoc nsz contract double [[TMP5]], [[ZSUM]]
-; CHECK-NEXT: [[TMP4:%.*]] = extractelement <2 x double> [[TMP3]], i64 1
-; CHECK-NEXT: [[TMP6:%.*]] = fadd reassoc nsz contract double [[SUB]], [[TMP4]]
+; CHECK-NEXT: [[TMP4:%.*]] = load <2 x double>, ptr [[Z]], align 8
+; CHECK-NEXT: [[RDX_OP:%.*]] = fsub reassoc nsz contract <2 x double> [[TMP3]], [[TMP4]]
+; CHECK-NEXT: [[TMP6:%.*]] = call reassoc nsz contract double @llvm.vector.reduce.fadd.v2f64(double 0.000000e+00, <2 x double> [[RDX_OP]])
; CHECK-NEXT: ret double [[TMP6]]
;
entry:
@@ -44,27 +39,12 @@ define double @fsub_fmul_4(ptr %x, ptr %y, ptr %z) {
; CHECK-LABEL: define double @fsub_fmul_4(
; CHECK-SAME: ptr [[X:%.*]], ptr [[Y:%.*]], ptr [[Z:%.*]]) #[[ATTR0]] {
; CHECK-NEXT: [[ENTRY:.*:]]
-; CHECK-NEXT: [[X8:%.*]] = getelementptr inbounds nuw i8, ptr [[X]], i64 8
-; CHECK-NEXT: [[Y8:%.*]] = getelementptr inbounds nuw i8, ptr [[Y]], i64 8
-; CHECK-NEXT: [[X24:%.*]] = getelementptr inbounds nuw i8, ptr [[X]], i64 24
-; CHECK-NEXT: [[Y24:%.*]] = getelementptr inbounds nuw i8, ptr [[Y]], i64 24
-; CHECK-NEXT: [[X0:%.*]] = load double, ptr [[X]], align 8
-; CHECK-NEXT: [[Y0:%.*]] = load double, ptr [[Y]], align 8
-; CHECK-NEXT: [[MUL:%.*]] = fmul reassoc nsz contract double [[Y0]], [[X0]]
-; CHECK-NEXT: [[TMP0:%.*]] = load <2 x double>, ptr [[X8]], align 8
-; CHECK-NEXT: [[TMP1:%.*]] = load <2 x double>, ptr [[Y8]], align 8
-; CHECK-NEXT: [[TMP2:%.*]] = fmul reassoc nsz contract <2 x double> [[TMP1]], [[TMP0]]
-; CHECK-NEXT: [[X3:%.*]] = load double, ptr [[X24]], align 8
-; CHECK-NEXT: [[Y3:%.*]] = load double, ptr [[Y24]], align 8
-; CHECK-NEXT: [[MUL16:%.*]] = fmul reassoc nsz contract double [[Y3]], [[X3]]
+; CHECK-NEXT: [[TMP0:%.*]] = load <4 x double>, ptr [[X]], align 8
+; CHECK-NEXT: [[TMP1:%.*]] = load <4 x double>, ptr [[Y]], align 8
+; CHECK-NEXT: [[TMP2:%.*]] = fmul reassoc nsz contract <4 x double> [[TMP1]], [[TMP0]]
; CHECK-NEXT: [[TMP3:%.*]] = load <4 x double>, ptr [[Z]], align 8
-; CHECK-NEXT: [[TMP4:%.*]] = extractelement <2 x double> [[TMP2]], i64 0
-; CHECK-NEXT: [[T1:%.*]] = fadd reassoc nsz contract double [[MUL]], [[TMP4]]
-; CHECK-NEXT: [[TMP7:%.*]] = extractelement <2 x double> [[TMP2]], i64 1
-; CHECK-NEXT: [[T3:%.*]] = fadd reassoc nsz contract double [[T1]], [[TMP7]]
-; CHECK-NEXT: [[TMP6:%.*]] = call reassoc nsz contract double @llvm.vector.reduce.fadd.v4f64(double 0.000000e+00, <4 x double> [[TMP3]])
-; CHECK-NEXT: [[ADD13:%.*]] = fsub reassoc nsz contract double [[T3]], [[TMP6]]
-; CHECK-NEXT: [[TMP5:%.*]] = fadd reassoc nsz contract double [[ADD13]], [[MUL16]]
+; CHECK-NEXT: [[RDX_OP:%.*]] = fsub reassoc nsz contract <4 x double> [[TMP2]], [[TMP3]]
+; CHECK-NEXT: [[TMP5:%.*]] = call reassoc nsz contract double @llvm.vector.reduce.fadd.v4f64(double 0.000000e+00, <4 x double> [[RDX_OP]])
; CHECK-NEXT: ret double [[TMP5]]
;
entry:
@@ -108,21 +88,12 @@ define double @fneg_chain_2(ptr %x, ptr %y, ptr %z) {
; CHECK-LABEL: define double @fneg_chain_2(
; CHECK-SAME: ptr [[X:%.*]], ptr [[Y:%.*]], ptr [[Z:%.*]]) #[[ATTR0]] {
; CHECK-NEXT: [[ENTRY:.*:]]
-; CHECK-NEXT: [[X8:%.*]] = getelementptr inbounds nuw i8, ptr [[X]], i64 8
-; CHECK-NEXT: [[Y8:%.*]] = getelementptr inbounds nuw i8, ptr [[Y]], i64 8
-; CHECK-NEXT: [[X0:%.*]] = load double, ptr [[X]], align 8
-; CHECK-NEXT: [[Y0:%.*]] = load double, ptr [[Y]], align 8
-; CHECK-NEXT: [[MUL:%.*]] = fmul reassoc nsz contract double [[Y0]], [[X0]]
-; CHECK-NEXT: [[X1:%.*]] = load double, ptr [[X8]], align 8
-; CHECK-NEXT: [[Y1:%.*]] = load double, ptr [[Y8]], align 8
-; CHECK-NEXT: [[MUL5:%.*]] = fmul reassoc nsz contract double [[Y1]], [[X1]]
+; CHECK-NEXT: [[TMP0:%.*]] = load <2 x double>, ptr [[X]], align 8
+; CHECK-NEXT: [[TMP1:%.*]] = load <2 x double>, ptr [[Y]], align 8
+; CHECK-NEXT: [[TMP2:%.*]] = fmul reassoc nsz contract <2 x double> [[TMP1]], [[TMP0]]
; CHECK-NEXT: [[TMP3:%.*]] = load <2 x double>, ptr [[Z]], align 8
-; CHECK-NEXT: [[TMP1:%.*]] = fneg reassoc nsz contract <2 x double> [[TMP3]]
-; CHECK-NEXT: [[TMP2:%.*]] = extractelement <2 x double> [[TMP1]], i64 0
-; CHECK-NEXT: [[T:%.*]] = fadd reassoc nsz contract double [[MUL]], [[TMP2]]
-; CHECK-NEXT: [[TMP4:%.*]] = extractelement <2 x double> [[TMP1]], i64 1
-; CHECK-NEXT: [[T2:%.*]] = fadd reassoc nsz contract double [[T]], [[TMP4]]
-; CHECK-NEXT: [[TMP5:%.*]] = fadd reassoc nsz contract double [[T2]], [[MUL5]]
+; CHECK-NEXT: [[RDX_OP:%.*]] = fsub reassoc nsz contract <2 x double> [[TMP2]], [[TMP3]]
+; CHECK-NEXT: [[TMP5:%.*]] = call reassoc nsz contract double @llvm.vector.reduce.fadd.v2f64(double 0.000000e+00, <2 x double> [[RDX_OP]])
; CHECK-NEXT: ret double [[TMP5]]
;
entry:
@@ -150,18 +121,14 @@ define double @negated_reused(ptr %x, ptr %y, ptr %z) {
; CHECK-LABEL: define double @negated_reused(
; CHECK-SAME: ptr [[X:%.*]], ptr [[Y:%.*]], ptr [[Z:%.*]]) #[[ATTR0]] {
; CHECK-NEXT: [[ENTRY:.*:]]
-; CHECK-NEXT: [[X8:%.*]] = getelementptr inbounds nuw i8, ptr [[X]], i64 8
-; CHECK-NEXT: [[Y8:%.*]] = getelementptr inbounds nuw i8, ptr [[Y]], i64 8
-; CHECK-NEXT: [[X0:%.*]] = load double, ptr [[X]], align 8
-; CHECK-NEXT: [[Y0:%.*]] = load double, ptr [[Y]], align 8
-; CHECK-NEXT: [[MUL:%.*]] = fmul reassoc nsz contract double [[Y0]], [[X0]]
; CHECK-NEXT: [[Z0:%.*]] = load double, ptr [[Z]], align 8
-; CHECK-NEXT: [[X1:%.*]] = load double, ptr [[X8]], align 8
-; CHECK-NEXT: [[Y1:%.*]] = load double, ptr [[Y8]], align 8
-; CHECK-NEXT: [[MUL5:%.*]] = fmul reassoc nsz contract double [[Y1]], [[X1]]
-; CHECK-NEXT: [[SUB:%.*]] = fsub reassoc nsz contract double [[MUL]], [[Z0]]
-; CHECK-NEXT: [[SUB7:%.*]] = fsub reassoc nsz contract double [[MUL5]], [[Z0]]
-; CHECK-NEXT: [[OP_RDX:%.*]] = fadd reassoc nsz contract double [[SUB]], [[SUB7]]
+; CHECK-NEXT: [[TMP0:%.*]] = load <2 x double>, ptr [[X]], align 8
+; CHECK-NEXT: [[TMP1:%.*]] = load <2 x double>, ptr [[Y]], align 8
+; CHECK-NEXT: [[TMP2:%.*]] = fmul reassoc nsz contract <2 x double> [[TMP1]], [[TMP0]]
+; CHECK-NEXT: [[DOTSPLATINSERT:%.*]] = insertelement <2 x double> poison, double [[Z0]], i64 0
+; CHECK-NEXT: [[DOTSPLAT:%.*]] = shufflevector <2 x double> [[DOTSPLATINSERT]], <2 x double> poison, <2 x i32> zeroinitializer
+; CHECK-NEXT: [[RDX_OP:%.*]] = fsub reassoc nsz contract <2 x double> [[TMP2]], [[DOTSPLAT]]
+; CHECK-NEXT: [[OP_RDX:%.*]] = call reassoc nsz contract double @llvm.vector.reduce.fadd.v2f64(double 0.000000e+00, <2 x double> [[RDX_OP]])
; CHECK-NEXT: ret double [[OP_RDX]]
;
entry:
@@ -186,8 +153,8 @@ define double @all_negated(ptr %a) {
; CHECK-SAME: ptr [[A:%.*]]) #[[ATTR0]] {
; CHECK-NEXT: [[ENTRY:.*:]]
; CHECK-NEXT: [[TMP0:%.*]] = load <4 x double>, ptr [[A]], align 8
-; CHECK-NEXT: [[TMP1:%.*]] = fneg reassoc nsz contract <4 x double> [[TMP0]]
-; CHECK-NEXT: [[OP_RDX:%.*]] = call reassoc nsz contract double @llvm.vector.reduce.fadd.v4f64(double 0.000000e+00, <4 x double> [[TMP1]])
+; CHECK-NEXT: [[TMP1:%.*]] = call reassoc nsz contract double @llvm.vector.reduce.fadd.v4f64(double 0.000000e+00, <4 x double> [[TMP0]])
+; CHECK-NEXT: [[OP_RDX:%.*]] = fneg reassoc nsz contract double [[TMP1]]
; CHECK-NEXT: ret double [[OP_RDX]]
;
entry:
@@ -214,18 +181,9 @@ define double @negated_constants(ptr %a) {
; CHECK-LABEL: define double @negated_constants(
; CHECK-SAME: ptr [[A:%.*]]) #[[ATTR0]] {
; CHECK-NEXT: [[ENTRY:.*:]]
-; CHECK-NEXT: [[P1:%.*]] = getelementptr inbounds nuw i8, ptr [[A]], i64 8
-; CHECK-NEXT: [[P2:%.*]] = getelementptr inbounds nuw i8, ptr [[A]], i64 16
-; CHECK-NEXT: [[P3:%.*]] = getelementptr inbounds nuw i8, ptr [[A]], i64 24
-; CHECK-NEXT: [[A0:%.*]] = load double, ptr [[A]], align 8
-; CHECK-NEXT: [[A1:%.*]] = load double, ptr [[P1]], align 8
-; CHECK-NEXT: [[A2:%.*]] = load double, ptr [[P2]], align 8
-; CHECK-NEXT: [[A3:%.*]] = load double, ptr [[P3]], align 8
-; CHECK-NEXT: [[T0:%.*]] = fadd reassoc nsz contract double [[A0]], [[A1]]
-; CHECK-NEXT: [[T1:%.*]] = fsub reassoc nsz contract double [[T0]], 2.000000e+00
-; CHECK-NEXT: [[T2:%.*]] = fadd reassoc nsz contract double [[T1]], [[A2]]
-; CHECK-NEXT: [[T3:%.*]] = fsub reassoc nsz contract double [[T2]], 3.000000e+00
-; CHECK-NEXT: [[OP_RDX:%.*]] = fadd reassoc nsz contract double [[T3]], [[A3]]
+; CHECK-NEXT: [[TMP0:%.*]] = load <4 x double>, ptr [[A]], align 8
+; CHECK-NEXT: [[TMP1:%.*]] = call reassoc nsz contract double @llvm.vector.reduce.fadd.v4f64(double 0.000000e+00, <4 x double> [[TMP0]])
+; CHECK-NEXT: [[OP_RDX:%.*]] = fsub reassoc nsz contract double [[TMP1]], 5.000000e+00
; CHECK-NEXT: ret double [[OP_RDX]]
;
entry:
@@ -356,17 +314,8 @@ define double @double_negation_cancels(ptr %a) {
; CHECK-LABEL: define double @double_negation_cancels(
; CHECK-SAME: ptr [[A:%.*]]) #[[ATTR0]] {
; CHECK-NEXT: [[ENTRY:.*:]]
-; CHECK-NEXT: [[P1:%.*]] = getelementptr inbounds nuw i8, ptr [[A]], i64 8
-; CHECK-NEXT: [[P2:%.*]] = getelementptr inbounds nuw i8, ptr [[A]], i64 16
-; CHECK-NEXT: [[P3:%.*]] = getelementptr inbounds nuw i8, ptr [[A]], i64 24
-; CHECK-NEXT: [[A0:%.*]] = load double, ptr [[A]], align 8
-; CHECK-NEXT: [[A1:%.*]] = load double, ptr [[P1]], align 8
-; CHECK-NEXT: [[A2:%.*]] = load double, ptr [[P2]], align 8
-; CHECK-NEXT: [[A3:%.*]] = load double, ptr [[P3]], align 8
-; CHECK-NEXT: [[T0:%.*]] = fadd reassoc nsz contract double [[A0]], [[A1]]
-; CHECK-NEXT: [[NEG:%.*]] = fneg reassoc nsz contract double [[A3]]
-; CHECK-NEXT: [[T1:%.*]] = fsub reassoc nsz contract double [[T0]], [[NEG]]
-; CHECK-NEXT: [[T2:%.*]] = fadd reassoc nsz contract double [[T1]], [[A2]]
+; CHECK-NEXT: [[TMP0:%.*]] = load <4 x double>, ptr [[A]], align 8
+; CHECK-NEXT: [[T2:%.*]] = call reassoc nsz contract double @llvm.vector.reduce.fadd.v4f64(double 0.000000e+00, <4 x double> [[TMP0]])
; CHECK-NEXT: ret double [[T2]]
;
entry:
@@ -390,32 +339,12 @@ define double @fsub_fmul_subtrahend_4(ptr %x, ptr %y, ptr %z) {
; CHECK-LABEL: define double @fsub_fmul_subtrahend_4(
; CHECK-SAME: ptr [[X:%.*]], ptr [[Y:%.*]], ptr [[Z:%.*]]) #[[ATTR0]] {
; CHECK-NEXT: [[ENTRY:.*:]]
-; CHECK-NEXT: [[Z8:%.*]] = getelementptr inbounds nuw i8, ptr [[Z]], i64 8
-; CHECK-NEXT: [[X16:%.*]] = getelementptr inbounds nuw i8, ptr [[X]], i64 16
-; CHECK-NEXT: [[Y16:%.*]] = getelementptr inbounds nuw i8, ptr [[Y]], i64 16
-; CHECK-NEXT: [[Z16:%.*]] = getelementptr inbounds nuw i8, ptr [[Z]], i64 16
-; CHECK-NEXT: [[Z24:%.*]] = getelementptr inbounds nuw i8, ptr [[Z]], i64 24
-; CHECK-NEXT: [[Z0:%.*]] = load double, ptr [[Z]], align 8
-; CHECK-NEXT: [[TMP0:%.*]] = load <2 x double>, ptr [[X]], align 8
-; CHECK-NEXT: [[TMP1:%.*]] = load <2 x double>, ptr [[Y]], align 8
-; CHECK-NEXT: [[TMP2:%.*]] = fmul reassoc nsz contract <2 x double> [[TMP1]], [[TMP0]]
-; CHECK-NEXT: [[Z1:%.*]] = load double, ptr [[Z8]], align 8
-; CHECK-NEXT: [[Z2:%.*]] = load double, ptr [[Z16]], align 8
-; CHECK-NEXT: [[TMP3:%.*]] = load <2 x double>, ptr [[X16]], align 8
-; CHECK-NEXT: [[TMP4:%.*]] = load <2 x double>, ptr [[Y16]], align 8
-; CHECK-NEXT: [[TMP5:%.*]] = fmul reassoc nsz contract <2 x double> [[TMP4]], [[TMP3]]
-; CHECK-NEXT: [[Z3:%.*]] = load double, ptr [[Z24]], align 8
-; CHECK-NEXT: [[TMP6:%.*]] = extractelement <2 x double> [[TMP2]], i64 0
-; CHECK-NEXT: [[S0:%.*]] = fsub reassoc nsz contract double [[Z0]], [[TMP6]]
-; CHECK-NEXT: [[A1:%.*]] = fadd reassoc nsz contract double [[S0]], [[Z1]]
-; CHECK-NEXT: [[TMP7:%.*]] = extractelement <2 x double> [[TMP2]], i64 1
-; CHECK-NEXT: [[S1:%.*]] = fsub reassoc nsz contract double [[A1]], [[TMP7]]
-; CHECK-NEXT: [[A2:%.*]] = fadd reassoc nsz contract double [[S1]], [[Z2]]
-; CHECK-NEXT: [[TMP8:%.*]] = extractelement <2 x double> [[TMP5]], i64 0
-; CHECK-NEXT: [[S2:%.*]] = fsub reassoc nsz contract double [[A2]], [[TMP8]]
-; CHECK-NEXT: [[TMP9:%.*]] = extractelement <2 x double> [[TMP5]], i64 1
-; CHECK-NEXT: [[S3:%.*]] = fsub reassoc nsz contract double [[S2]], [[TMP9]]
-; CHECK-NEXT: [[R:%.*]] = fadd reassoc nsz contract double [[S3]], [[Z3]]
+; CHECK-NEXT: [[TMP0:%.*]] = load <4 x double>, ptr [[X]], align 8
+; CHECK-NEXT: [[TMP1:%.*]] = load <4 x double>, ptr [[Y]], align 8
+; CHECK-NEXT: [[TMP2:%.*]] = fmul reassoc nsz contract <4 x double> [[TMP1]], [[TMP0]]
+; CHECK-NEXT: [[TMP3:%.*]] = load <4 x double>, ptr [[Z]], align 8
+; CHECK-NEXT: [[RDX_OP:%.*]] = fsub reassoc nsz contract <4 x double> [[TMP3]], [[TMP2]]
+; CHECK-NEXT: [[R:%.*]] = call reassoc nsz contract double @llvm.vector.reduce.fadd.v4f64(double 0.000000e+00, <4 x double> [[RDX_OP]])
; CHECK-NEXT: ret double [[R]]
;
entry:
@@ -461,19 +390,10 @@ define double @all_negated_plus_scalar(ptr %a, double %b) {
; CHECK-LABEL: define double @all_negated_plus_scalar(
; CHECK-SAME: ptr [[A:%.*]], double [[B:%.*]]) #[[ATTR0]] {
; CHECK-NEXT: [[ENTRY:.*:]]
-; CHECK-NEXT: [[P1:%.*]] = getelementptr inbounds nuw i8, ptr [[A]], i64 8
-; CHECK-NEXT: [[P2:%.*]] = getelementptr inbounds nuw i8, ptr [[A]], i64 16
-; CHECK-NEXT: [[P3:%.*]] = getelementptr inbounds nuw i8, ptr [[A]], i64 24
-; CHECK-NEXT: [[A0:%.*]] = load double, ptr [[A]], align 8
-; CHECK-NEXT: [[A1:%.*]] = load double, ptr [[P1]], align 8
-; CHECK-NEXT: [[A2:%.*]] = load double, ptr [[P2]], align 8
-; CHECK-NEXT: [[A3:%.*]] = load double, ptr [[P3]], align 8
+; CHECK-NEXT: [[TMP0:%.*]] = load <4 x double>, ptr [[A]], align 8
+; CHECK-NEXT: [[A0:%.*]] = call reassoc nsz contract double @llvm.vector.reduce.fadd.v4f64(double 0.000000e+00, <4 x double> [[TMP0]])
; CHECK-NEXT: [[S0:%.*]] = fsub reassoc nsz contract double [[B]], [[A0]]
-; CHECK-NEXT: [[S1:%.*]] = fsub reassoc nsz contract double [[S0]], [[A1]]
-; CHECK-NEXT: [[S2:%.*]] = fsub reassoc nsz contract double [[S1]], [[A2]]
-; CHECK-NEXT: [[N3:%.*]] = fneg reassoc nsz contract double [[A3]]
-; CHECK-NEXT: [[R:%.*]] = fadd reassoc nsz contract double [[S2]], [[N3]]
-; CHECK-NEXT: ret double [[R]]
+; CHECK-NEXT: ret double [[S0]]
;
entry:
%p1 = getelementptr inbounds nuw i8, ptr %a, i64 8
@@ -498,51 +418,16 @@ define double @three_sign_groups(ptr %x, ptr %y, ptr %u, ptr %v, ptr %z) {
; CHECK-LABEL: define double @three_sign_groups(
; CHECK-SAME: ptr [[X:%.*]], ptr [[Y:%.*]], ptr [[U:%.*]], ptr [[V:%.*]], ptr [[Z:%.*]]) #[[ATTR0]] {
; CHECK-NEXT: [[ENTRY:.*:]]
-; CHECK-NEXT: [[X8:%.*]] = getelementptr inbounds nuw i8, ptr [[X]], i64 8
-; CHECK-NEXT: [[X24:%.*]] = getelementptr inbounds nuw i8, ptr [[X]], i64 24
-; CHECK-NEXT: [[Y8:%.*]] = getelementptr inbounds nuw i8, ptr [[Y]], i64 8
-; CHECK-NEXT: [[Y24:%.*]] = getelementptr inbounds nuw i8, ptr [[Y]], i64 24
-; CHECK-NEXT: [[U16:%.*]] = getelementptr inbounds nuw i8, ptr [[U]], i64 16
-; CHECK-NEXT: [[V16:%.*]] = getelementptr inbounds nuw i8, ptr [[V]], i64 16
-; CHECK-NEXT: [[Z8:%.*]] = getelementptr inbounds nuw i8, ptr [[Z]], i64 8
-; CHECK-NEXT: [[Z16:%.*]] = getelementptr inbounds nuw i8, ptr [[Z]], i64 16
-; CHECK-NEXT: [[Z24:%.*]] = getelementptr inbounds nuw i8, ptr [[Z]], i64 24
-; CHECK-NEXT: [[X0:%.*]] = load double, ptr [[X]], align 8
-; CHECK-NEXT: [[X3:%.*]] = load double, ptr [[X24]], align 8
-; CHECK-NEXT: [[Y0:%.*]] = load double, ptr [[Y]], align 8
-; CHECK-NEXT: [[Y3:%.*]] = load double, ptr [[Y24]], align 8
-; CHECK-NEXT: [[Z0:%.*]] = load double, ptr [[Z]], align 8
-; CHECK-NEXT: [[Z1:%.*]] = load double, ptr [[Z8]], align 8
-; CHECK-NEXT: [[Z2:%.*]] = load double, ptr [[Z16]], align 8
-; CHECK-NEXT: [[Z3:%.*]] = load double, ptr [[Z24]], align 8
-; CHECK-NEXT: [[MXY0:%.*]] = fmul reassoc nsz contract double [[X0]], [[Y0]]
-; CHECK-NEXT: [[TMP0:%.*]] = load <2 x double>, ptr [[X8]], align 8
-; CHECK-NEXT: [[TMP1:%.*]] = load <2 x double>, ptr [[Y8]], align 8
-; CHECK-NEXT: [[TMP2:%.*]] = fmul reassoc nsz contract <2 x double> [[TMP0]], [[TMP1]]
-; CHECK-NEXT: [[MXY3:%.*]] = fmul reassoc nsz contract double [[X3]], [[Y3]]
-; CHECK-NEXT: [[TMP3:%.*]] = load <2 x double>, ptr [[U]], align 8
-; CHECK-NEXT: [[TMP4:%.*]] = load <2 x double>, ptr [[V]], align 8
-; CHECK-NEXT: [[TMP5:%.*]] = fmul reassoc nsz contract <2 x double> [[TMP3]], [[TMP4]]
-; CHECK-NEXT: [[TMP6:%.*]] = load <2 x double>, ptr [[U16]], align 8
-; CHECK-NEXT: [[TMP7:%.*]] = load <2 x double>, ptr [[V16]], align 8
-; CHECK-NEXT: [[TMP8:%.*]] = fmul reassoc nsz contract <2 x double> [[TMP6]], [[TMP7]]
-; CHECK-NEXT: [[TMP9:%.*]] = extractelement <2 x double> [[TMP2]], i64 0
-; CHECK-NEXT: [[T0:%.*]] = fadd reassoc nsz contract double [[MXY0]], [[TMP9]]
-; CHECK-NEXT: [[TMP10:%.*]] = extractelement <2 x double> [[TMP2]], i64 1
-; CHECK-NEXT: [[T1:%.*]] = fadd reassoc nsz contract double [[T0]], [[TMP10]]
-; CHECK-NEXT: [[TMP11:%.*]] = extractelement <2 x double> [[TMP5]], i64 0
-; CHECK-NEXT: [[S0:%.*]] = fsub reassoc nsz contract double [[T1]], [[TMP11]]
-; CHECK-NEXT: [[TMP12:%.*]] = extractelement <2 x double> [[TMP5]], i64 1
-; CHECK-NEXT: [[S1:%.*]] = fsub reassoc nsz contract double [[S0]], [[TMP12]]
-; CHECK-NEXT: [[TMP13:%.*]] = extractelement <2 x double> [[TMP8]], i64 0
-; CHECK-NEXT: [[S2:%.*]] = fsub reassoc nsz contract double [[S1]], [[TMP13]]
-; CHECK-NEXT: [[TMP14:%.*]] = extractelement <2 x double> [[TMP8]], i64 1
-; CHECK-NEXT: [[S3:%.*]] = fsub reassoc nsz contract double [[S2]], [[TMP14]]
-; CHECK-NEXT: [[W0:%.*]] = fsub reassoc nsz contract double [[S3]], [[Z0]]
-; CHECK-NEXT: [[W1:%.*]] = fsub reassoc nsz contract double [[W0]], [[Z1]]
-; CHECK-NEXT: [[W2:%.*]] = fsub reassoc nsz contract double [[W1]], [[Z2]]
-; CHECK-NEXT: [[W3:%.*]] = fsub reassoc nsz contract double [[W2]], [[Z3]]
-; CHECK-NEXT: [[R:%.*]] = fadd reassoc nsz contract double [[W3]], [[MXY3]]
+; CHECK-NEXT: [[TMP0:%.*]] = load <4 x double>, ptr [[Z]], align 8
+; CHECK-NEXT: [[TMP1:%.*]] = load <4 x double>, ptr [[X]], align 8
+; CHECK-NEXT: [[TMP2:%.*]] = load <4 x double>, ptr [[Y]], align 8
+; CHECK-NEXT: [[TMP3:%.*]] = fmul reassoc nsz contract <4 x double> [[TMP1]], [[TMP2]]
+; CHECK-NEXT: [[TMP4:%.*]] = load <4 x double>, ptr [[U]], align 8
+; CHECK-NEXT: [[TMP5:%.*]] = load <4 x double>, ptr [[V]], align 8
+; CHECK-NEXT: [[TMP6:%.*]] = fmul reassoc nsz contract <4 x double> [[TMP4]], [[TMP5]]
+; CHECK-NEXT: [[RDX_OP:%.*]] = fsub reassoc nsz contract <4 x double> [[TMP3]], [[TMP6]]
+; CHECK-NEXT: [[RDX_OP1:%.*]] = fsub reassoc nsz contract <4 x double> [[RDX_OP]], [[TMP0]]
+; CHECK-NEXT: [[R:%.*]] = call reassoc nsz contract double @llvm.vector.reduce.fadd.v4f64(double 0.000000e+00, <4 x double> [[RDX_OP1]])
; CHECK-NEXT: ret double [[R]]
;
entry:
@@ -610,31 +495,16 @@ define double @negated_repeated_pair(ptr %x, ptr %y, ptr %z) {
; CHECK-LABEL: define double @negated_repeated_pair(
; CHECK-SAME: ptr [[X:%.*]], ptr [[Y:%.*]], ptr [[Z:%.*]]) #[[ATTR0]] {
; CHECK-NEXT: [[ENTRY:.*:]]
-; CHECK-NEXT: [[X8:%.*]] = getelementptr inbounds nuw i8, ptr [[X]], i64 8
-; CHECK-NEXT: [[X24:%.*]] = getelementptr inbounds nuw i8, ptr [[X]], i64 24
-; CHECK-NEXT: [[Y8:%.*]] = getelementptr inbounds nuw i8, ptr [[Y]], i64 8
-; CHECK-NEXT: [[Y24:%.*]] = getelementptr inbounds nuw i8, ptr [[Y]], i64 24
-; CHECK-NEXT: [[Z8:%.*]] = getelementptr inbounds nuw i8, ptr [[Z]], i64 8
-; CHECK-NEXT: [[X0:%.*]] = load double, ptr [[X]], align 8
-; CHECK-NEXT: [[X3:%.*]] = load double, ptr [[X24]], align 8
-; CHECK-NEXT: [[Y0:%.*]] = load double, ptr [[Y]], align 8
-; CHECK-NEXT: [[Y3:%.*]] = load double, ptr [[Y24]], align 8
-; CHECK-NEXT: [[ZA:%.*]] = load double, ptr [[Z]], align 8
-; CHECK-NEXT: [[ZB:%.*]] = load double, ptr [[Z8]], align 8
-; CHECK-NEXT: [[M0:%.*]] = fmul reassoc nsz contract double [[X0]], [[Y0]]
-; CHECK-NEXT: [[TMP0:%.*]] = load <2 x double>, ptr [[X8]], align 8
-; CHECK-NEXT: [[TMP1:%.*]] = load <2 x double>, ptr [[Y8]], align 8
-; CHECK-NEXT: [[TMP2:%.*]] = fmul reassoc nsz contract <2 x double> [[TMP0]], [[TMP1]]
-; CHECK-NEXT: [[M3:%.*]] = fmul reassoc nsz contract double [[X3]], [[Y3]]
-; CHECK-NEXT: [[TMP3:%.*]] = extractelement <2 x double> [[TMP2]], i64 0
-; CHECK-NEXT: [[T0:%.*]] = fadd reassoc nsz contract double [[M0]], [[TMP3]]
-; CHECK-NEXT: [[TMP4:%.*]] = extractelement <2 x double> [[TMP2]], i64 1
-; CHECK-NEXT: [[T1:%.*]] = fadd reassoc nsz contract double [[T0]], [[TMP4]]
-; CHECK-NEXT: [[S0:%.*]] = fsub reassoc nsz contract double [[T1]], [[ZA]]
-; CHECK-NEXT: [[S1:%.*]] = fsub reassoc nsz contract double [[S0]], [[ZB]]
-; CHECK-NEXT: [[S2:%.*]] = fsub reassoc nsz contract double [[S1]], [[ZA]]
-; CHECK-NEXT: [[S3:%.*]] = fsub reassoc nsz contract double [[S2]], [[ZB]]
-; CHECK-NEXT: [[R:%.*]] = fadd reassoc nsz contract double [[S3]], [[M3]]
+; CHECK-NEXT: [[TMP0:%.*]] = load <2 x double>, ptr [[Z]], align 8
+; CHECK-NEXT: [[TMP1:%.*]] = load <4 x double>, ptr [[X]], align 8
+; CHECK-NEXT: [[TMP2:%.*]] = load <4 x double>, ptr [[Y]], align 8
+; CHECK-NEXT: [[TMP3:%.*]] = fmul reassoc nsz contract <4 x double> [[TMP1]], [[TMP2]]
+; CHECK-NEXT: [[TMP4:%.*]] = fmul reassoc nsz contract <2 x double> [[TMP0]], splat (double 2.000000e+00)
+; CHECK-NEXT: [[TMP5:%.*]] = shufflevector <4 x double> [[TMP3]], <4 x double> poison, <2 x i32> <i32 0, i32 1>
+; CHECK-NEXT: [[RDX_OP:%.*]] = fsub reassoc nsz contract <2 x double> [[TMP5]], [[TMP4]]
+; CHECK-NEXT: [[TMP6:%.*]] = shufflevector <2 x double> [[RDX_OP]], <2 x double> poison, <4 x i32> <i32 0, i32 1, i32 poison, i32 poison>
+; CHECK-NEXT: [[TMP7:%.*]] = shufflevector <4 x double> [[TMP3]], <4 x double> [[TMP6]], <4 x i32> <i32 4, i32 5, i32 2, i32 3>
+; CHECK-NEXT: [[R:%.*]] = call reassoc nsz contract double @llvm.vector.reduce.fadd.v4f64(double 0.000000e+00, <4 x double> [[TMP7]])
; CHECK-NEXT: ret double [[R]]
;
entry:
@@ -746,20 +616,12 @@ define double @fsub_root_2(ptr %x, ptr %y, ptr %z) {
; CHECK-LABEL: define double @fsub_root_2(
; CHECK-SAME: ptr [[X:%.*]], ptr [[Y:%.*]], ptr [[Z:%.*]]) #[[ATTR0]] {
; CHECK-NEXT: [[ENTRY:.*:]]
-; CHECK-NEXT: [[X8:%.*]] = getelementptr inbounds nuw i8, ptr [[X]], i64 8
-; CHECK-NEXT: [[Y8:%.*]] = getelementptr inbounds nuw i8, ptr [[Y]], i64 8
-; CHECK-NEXT: [[Z8:%.*]] = getelementptr inbounds nuw i8, ptr [[Z]], i64 8
-; CHECK-NEXT: [[X0:%.*]] = load double, ptr [[X]], align 8
-; CHECK-NEXT: [[Y0:%.*]] = load double, ptr [[Y]], align 8
-; CHECK-NEXT: [[M0:%.*]] = fmul reassoc nsz contract double [[X0]], [[Y0]]
-; CHECK-NEXT: [[X1:%.*]] = load double, ptr [[X8]], align 8
-; CHECK-NEXT: [[Y1:%.*]] = load double, ptr [[Y8]], align 8
-; CHECK-NEXT: [[M1:%.*]] = fmul reassoc nsz contract double [[X1]], [[Y1]]
-; CHECK-NEXT: [[Z0:%.*]] = load double, ptr [[Z]], align 8
-; CHECK-NEXT: [[Z1:%.*]] = load double, ptr [[Z8]], align 8
-; CHECK-NEXT: [[ZS:%.*]] = fadd reassoc nsz contract double [[Z0]], [[Z1]]
-; CHECK-NEXT: [[MS:%.*]] = fadd reassoc nsz contract double [[M0]], [[M1]]
-; CHECK-NEXT: [[TMP4:%.*]] = fsub reassoc nsz contract double [[ZS]], [[MS]]
+; CHECK-NEXT: [[TMP0:%.*]] = load <2 x double>, ptr [[X]], align 8
+; CHECK-NEXT: [[TMP1:%.*]] = load <2 x double>, ptr [[Y]], align 8
+; CHECK-NEXT: [[TMP2:%.*]] = fmul reassoc nsz contract <2 x double> [[TMP0]], [[TMP1]]
+; CHECK-NEXT: [[TMP3:%.*]] = load <2 x double>, ptr [[Z]], align 8
+; CHECK-NEXT: [[RDX_OP:%.*]] = fsub reassoc nsz contract <2 x double> [[TMP3]], [[TMP2]]
+; CHECK-NEXT: [[TMP4:%.*]] = call reassoc nsz contract double @llvm.vector.reduce.fadd.v2f64(double 0.000000e+00, <2 x double> [[RDX_OP]])
; CHECK-NEXT: ret double [[TMP4]]
;
entry:
@@ -785,23 +647,12 @@ define double @fsub_root_chain_4(ptr %x, ptr %y, double %acc) {
; CHECK-LABEL: define double @fsub_root_chain_4(
; CHECK-SAME: ptr [[X:%.*]], ptr [[Y:%.*]], double [[ACC:%.*]]) #[[ATTR0]] {
; CHECK-NEXT: [[ENTRY:.*:]]
-; CHECK-NEXT: [[X16:%.*]] = getelementptr inbounds nuw i8, ptr [[X]], i64 16
-; CHECK-NEXT: [[Y16:%.*]] = getelementptr inbounds nuw i8, ptr [[Y]], i64 16
-; CHECK-NEXT: [[TMP0:%.*]] = load <2 x double>, ptr [[X]], align 8
-; CHECK-NEXT: [[TMP1:%.*]] = load <2 x double>, ptr [[Y]], align 8
-; CHECK-NEXT: [[TMP2:%.*]] = fmul reassoc nsz contract <2 x double> [[TMP0]], [[TMP1]]
-; CHECK-NEXT: [[TMP6:%.*]] = load <2 x double>, ptr [[X16]], align 8
-; CHECK-NEXT: [[TMP4:%.*]] = load <2 x double>, ptr [[Y16]], align 8
-; CHECK-NEXT: [[TMP5:%.*]] = fmul reassoc nsz contract <2 x double> [[TMP6]], [[TMP4]]
-; CHECK-NEXT: [[TMP3:%.*]] = extractelement <2 x double> [[TMP2]], i64 0
+; CHECK-NEXT: [[TMP0:%.*]] = load <4 x double>, ptr [[X]], align 8
+; CHECK-NEXT: [[TMP1:%.*]] = load <4 x double>, ptr [[Y]], align 8
+; CHECK-NEXT: [[TMP2:%.*]] = fmul reassoc nsz contract <4 x double> [[TMP0]], [[TMP1]]
+; CHECK-NEXT: [[TMP3:%.*]] = call reassoc nsz contract double @llvm.vector.reduce.fadd.v4f64(double 0.000000e+00, <4 x double> [[TMP2]])
; CHECK-NEXT: [[OP_RDX:%.*]] = fsub reassoc nsz contract double [[ACC]], [[TMP3]]
-; CHECK-NEXT: [[TMP7:%.*]] = extractelement <2 x double> [[TMP2]], i64 1
-; CHECK-NEXT: [[T1:%.*]] = fsub reassoc nsz contract double [[OP_RDX]], [[TMP7]]
-; CHECK-NEXT: [[TMP8:%.*]] = extractelement <2 x double> [[TMP5]], i64 0
-; CHECK-NEXT: [[T2:%.*]] = fsub reassoc nsz contract double [[T1]], [[TMP8]]
-; CHECK-NEXT: [[TMP9:%.*]] = extractelement <2 x double> [[TMP5]], i64 1
-; CHECK-NEXT: [[T3:%.*]] = fsub reassoc nsz contract double [[T2]], [[TMP9]]
-; CHECK-NEXT: ret double [[T3]]
+; CHECK-NEXT: ret double [[OP_RDX]]
;
entry:
%x8 = getelementptr inbounds nuw i8, ptr %x, i64 8
@@ -839,9 +690,8 @@ define double @fsub_root_4_4(ptr %x, ptr %y, ptr %z) {
; CHECK-NEXT: [[TMP1:%.*]] = load <4 x double>, ptr [[X]], align 8
; CHECK-NEXT: [[TMP2:%.*]] = load <4 x double>, ptr [[Y]], align 8
; CHECK-NEXT: [[TMP3:%.*]] = fmul reassoc nsz contract <4 x double> [[TMP1]], [[TMP2]]
-; CHECK-NEXT: [[TMP6:%.*]] = call reassoc nsz contract double @llvm.vector.reduce.fadd.v4f64(double 0.000000e+00, <4 x double> [[TMP3]])
-; CHECK-NEXT: [[TMP5:%.*]] = call reassoc nsz contract double @llvm.vector.reduce.fadd.v4f64(double 0.000000e+00, <4 x double> [[TMP0]])
-; CHECK-NEXT: [[TMP4:%.*]] = fsub reassoc nsz contract double [[TMP6]], [[TMP5]]
+; CHECK-NEXT: [[RDX_OP:%.*]] = fsub reassoc nsz contract <4 x double> [[TMP3]], [[TMP0]]
+; CHECK-NEXT: [[TMP4:%.*]] = call reassoc nsz contract double @llvm.vector.reduce.fadd.v4f64(double 0.000000e+00, <4 x double> [[RDX_OP]])
; CHECK-NEXT: ret double [[TMP4]]
;
entry:
@@ -971,27 +821,13 @@ define double @fsub_root_loop(ptr %x, ptr %y, i64 %n) {
; CHECK-NEXT: br label %[[LOOP:.*]]
; CHECK: [[LOOP]]:
; CHECK-NEXT: [[I:%.*]] = phi i64 [ 0, %[[ENTRY]] ], [ [[I_NEXT:%.*]], %[[LOOP]] ]
-; CHECK-NEXT: [[ACC:%.*]] = phi double [ 0.000000e+00, %[[ENTRY]] ], [ [[ACC_NEXT:%.*]], %[[LOOP]] ]
+; CHECK-NEXT: [[T2:%.*]] = phi double [ 0.000000e+00, %[[ENTRY]] ], [ [[ACC_NEXT:%.*]], %[[LOOP]] ]
; CHECK-NEXT: [[XP:%.*]] = getelementptr inbounds double, ptr [[X]], i64 [[I]]
; CHECK-NEXT: [[YP:%.*]] = getelementptr inbounds double, ptr [[Y]], i64 [[I]]
-; CHECK-NEXT: [[X8:%.*]] = getelementptr inbounds nuw i8, ptr [[XP]], i64 8
-; CHECK-NEXT: [[X24:%.*]] = getelementptr inbounds nuw i8, ptr [[XP]], i64 24
-; CHECK-NEXT: [[Y8:%.*]] = getelementptr inbounds nuw i8, ptr [[YP]], i64 8
-; CHECK-NEXT: [[Y24:%.*]] = getelementptr inbounds nuw i8, ptr [[YP]], i64 24
-; CHECK-NEXT: [[X0:%.*]] = load double, ptr [[XP]], align 8
-; CHECK-NEXT: [[X3:%.*]] = load double, ptr [[X24]], align 8
-; CHECK-NEXT: [[Y0:%.*]] = load double, ptr [[YP]], align 8
-; CHECK-NEXT: [[Y3:%.*]] = load double, ptr [[Y24]], align 8
-; CHECK-NEXT: [[TMP3:%.*]] = fmul reassoc nsz contract double [[X0]], [[Y0]]
-; CHECK-NEXT: [[TMP0:%.*]] = load <2 x double>, ptr [[X8]], align 8
-; CHECK-NEXT: [[TMP1:%.*]] = load <2 x double>, ptr [[Y8]], align 8
-; CHECK-NEXT: [[TMP2:%.*]] = fmul reassoc nsz contract <2 x double> [[TMP0]], [[TMP1]]
-; CHECK-NEXT: [[M3:%.*]] = fmul reassoc nsz contract double [[X3]], [[Y3]]
-; CHECK-NEXT: [[OP_RDX:%.*]] = fsub reassoc nsz contract double [[ACC]], [[TMP3]]
-; CHECK-NEXT: [[TMP5:%.*]] = extractelement <2 x double> [[TMP2]], i64 0
-; CHECK-NEXT: [[T1:%.*]] = fsub reassoc nsz contract double [[OP_RDX]], [[TMP5]]
-; CHECK-NEXT: [[TMP4:%.*]] = extractelement <2 x double> [[TMP2]], i64 1
-; CHECK-NEXT: [[T2:%.*]] = fsub reassoc nsz contract double [[T1]], [[TMP4]]
+; CHECK-NEXT: [[TMP0:%.*]] = load <4 x double>, ptr [[XP]], align 8
+; CHECK-NEXT: [[TMP1:%.*]] = load <4 x double>, ptr [[YP]], align 8
+; CHECK-NEXT: [[TMP2:%.*]] = fmul reassoc nsz contract <4 x double> [[TMP0]], [[TMP1]]
+; CHECK-NEXT: [[M3:%.*]] = call reassoc nsz contract double @llvm.vector.reduce.fadd.v4f64(double 0.000000e+00, <4 x double> [[TMP2]])
; CHECK-NEXT: [[ACC_NEXT]] = fsub reassoc nsz contract double [[T2]], [[M3]]
; CHECK-NEXT: [[I_NEXT]] = add nuw i64 [[I]], 4
; CHECK-NEXT: [[CMP:%.*]] = icmp ult i64 [[I_NEXT]], [[N]]
@@ -1044,17 +880,12 @@ define double @loads_pos_fmul_neg_2(ptr %x, ptr %y, ptr %z) {
; CHECK-LABEL: define double @loads_pos_fmul_neg_2(
; CHECK-SAME: ptr [[X:%.*]], ptr [[Y:%.*]], ptr [[Z:%.*]]) #[[ATTR0]] {
; CHECK-NEXT: [[ENTRY:.*:]]
-; CHECK-NEXT: [[Z8:%.*]] = getelementptr inbounds nuw i8, ptr [[Z]], i64 8
; CHECK-NEXT: [[TMP0:%.*]] = load <2 x double>, ptr [[X]], align 8
; CHECK-NEXT: [[TMP1:%.*]] = load <2 x double>, ptr [[Y]], align 8
; CHECK-NEXT: [[TMP2:%.*]] = fmul reassoc nsz contract <2 x double> [[TMP0]], [[TMP1]]
-; CHECK-NEXT: [[Z0:%.*]] = load double, ptr [[Z]], align 8
-; CHECK-NEXT: [[Z1:%.*]] = load double, ptr [[Z8]], align 8
-; CHECK-NEXT: [[TMP3:%.*]] = extractelement <2 x double> [[TMP2]], i64 0
-; CHECK-NEXT: [[S0:%.*]] = fsub reassoc nsz contract double [[Z0]], [[TMP3]]
-; CHECK-NEXT: [[TMP5:%.*]] = extractelement <2 x double> [[TMP2]], i64 1
-; CHECK-NEXT: [[S1:%.*]] = fsub reassoc nsz contract double [[S0]], [[TMP5]]
-; CHECK-NEXT: [[TMP4:%.*]] = fadd reassoc nsz contract double [[S1]], [[Z1]]
+; CHECK-NEXT: [[TMP3:%.*]] = load <2 x double>, ptr [[Z]], align 8
+; CHECK-NEXT: [[RDX_OP:%.*]] = fsub reassoc nsz contract <2 x double> [[TMP3]], [[TMP2]]
+; CHECK-NEXT: [[TMP4:%.*]] = call reassoc nsz contract double @llvm.vector.reduce.fadd.v2f64(double 0.000000e+00, <2 x double> [[RDX_OP]])
; CHECK-NEXT: ret double [[TMP4]]
;
entry:
@@ -1104,18 +935,14 @@ define double @negated_reused_splat_operand(ptr %x, ptr %y, ptr %z) {
; CHECK-LABEL: define double @negated_reused_splat_operand(
; CHECK-SAME: ptr [[X:%.*]], ptr [[Y:%.*]], ptr [[Z:%.*]]) #[[ATTR0]] {
; CHECK-NEXT: [[ENTRY:.*:]]
-; CHECK-NEXT: [[X8:%.*]] = getelementptr inbounds nuw i8, ptr [[X]], i64 8
-; CHECK-NEXT: [[Y8:%.*]] = getelementptr inbounds nuw i8, ptr [[Y]], i64 8
-; CHECK-NEXT: [[X0:%.*]] = load double, ptr [[X]], align 8
-; CHECK-NEXT: [[Y0:%.*]] = load double, ptr [[Y]], align 8
-; CHECK-NEXT: [[MUL:%.*]] = fmul reassoc nsz contract double [[Y0]], [[X0]]
; CHECK-NEXT: [[Z0:%.*]] = load double, ptr [[Z]], align 8
-; CHECK-NEXT: [[X1:%.*]] = load double, ptr [[X8]], align 8
-; CHECK-NEXT: [[Y1:%.*]] = load double, ptr [[Y8]], align 8
-; CHECK-NEXT: [[MUL5:%.*]] = fmul reassoc nsz contract double [[Y1]], [[X1]]
-; CHECK-NEXT: [[SUB:%.*]] = fsub reassoc nsz contract double [[MUL]], [[Z0]]
-; CHECK-NEXT: [[SUB7:%.*]] = fsub reassoc nsz contract double [[MUL5]], [[Z0]]
-; CHECK-NEXT: [[TMP3:%.*]] = fadd reassoc nsz contract double [[SUB]], [[SUB7]]
+; CHECK-NEXT: [[TMP0:%.*]] = load <2 x double>, ptr [[X]], align 8
+; CHECK-NEXT: [[TMP1:%.*]] = load <2 x double>, ptr [[Y]], align 8
+; CHECK-NEXT: [[TMP2:%.*]] = fmul reassoc nsz contract <2 x double> [[TMP1]], [[TMP0]]
+; CHECK-NEXT: [[DOTSPLATINSERT:%.*]] = insertelement <2 x double> poison, double [[Z0]], i64 0
+; CHECK-NEXT: [[DOTSPLAT:%.*]] = shufflevector <2 x double> [[DOTSPLATINSERT]], <2 x double> poison, <2 x i32> zeroinitializer
+; CHECK-NEXT: [[RDX_OP:%.*]] = fsub reassoc nsz contract <2 x double> [[TMP2]], [[DOTSPLAT]]
+; CHECK-NEXT: [[TMP3:%.*]] = call reassoc nsz contract double @llvm.vector.reduce.fadd.v2f64(double 0.000000e+00, <2 x double> [[RDX_OP]])
; CHECK-NEXT: ret double [[TMP3]]
;
entry:
@@ -1186,10 +1013,9 @@ define double @negated_reused_2_of_4(ptr %x, ptr %y, double %z) {
; CHECK-NEXT: [[TMP0:%.*]] = load <4 x double>, ptr [[X]], align 8
; CHECK-NEXT: [[TMP1:%.*]] = load <4 x double>, ptr [[Y]], align 8
; CHECK-NEXT: [[TMP2:%.*]] = fmul reassoc nsz contract <4 x double> [[TMP0]], [[TMP1]]
-; CHECK-NEXT: [[TMP3:%.*]] = insertelement <4 x double> <double poison, double poison, double 0.000000e+00, double 0.000000e+00>, double [[Z]], i64 0
-; CHECK-NEXT: [[TMP4:%.*]] = shufflevector <4 x double> [[TMP3]], <4 x double> poison, <4 x i32> <i32 0, i32 0, i32 2, i32 3>
-; CHECK-NEXT: [[TMP5:%.*]] = fsub reassoc nsz contract <4 x double> [[TMP2]], [[TMP4]]
-; CHECK-NEXT: [[OP_RDX:%.*]] = call nsz contract double @llvm.vector.reduce.fadd.v4f64(double 0.000000e+00, <4 x double> [[TMP5]])
+; CHECK-NEXT: [[TMP3:%.*]] = fmul reassoc nsz double [[Z]], 2.000000e+00
+; CHECK-NEXT: [[TMP4:%.*]] = call reassoc nsz contract double @llvm.vector.reduce.fadd.v4f64(double 0.000000e+00, <4 x double> [[TMP2]])
+; CHECK-NEXT: [[OP_RDX:%.*]] = fsub reassoc nsz contract double [[TMP4]], [[TMP3]]
; CHECK-NEXT: ret double [[OP_RDX]]
;
entry:
@@ -1451,7 +1277,7 @@ define double @scaled_value_no_contract(ptr %x, ptr %y, ptr %z, ptr %w) {
; CHECK-NEXT: [[W0:%.*]] = load double, ptr [[W]], align 8
; CHECK-NEXT: [[N:%.*]] = fneg reassoc nsz contract double [[W0]]
; CHECK-NEXT: [[TMP3:%.*]] = extractelement <2 x double> [[TMP2]], i64 0
-; CHECK-NEXT: [[TMP4:%.*]] = fmul double [[TMP3]], 2.000000e+00
+; CHECK-NEXT: [[TMP4:%.*]] = fmul reassoc nsz double [[TMP3]], 2.000000e+00
; CHECK-NEXT: [[TMP5:%.*]] = extractelement <2 x double> [[TMP2]], i64 1
; CHECK-NEXT: [[OP_RDX:%.*]] = fadd reassoc nsz contract double [[TMP4]], [[TMP5]]
; CHECK-NEXT: [[OP_RDX1:%.*]] = fadd reassoc nsz contract double [[OP_RDX]], [[N]]
@@ -1513,9 +1339,8 @@ define double @fneg_leaves_vs_fsub_chain_fneg(ptr %a, double %b) {
; CHECK-SAME: ptr [[A:%.*]], double [[B:%.*]]) #[[ATTR0]] {
; CHECK-NEXT: [[ENTRY:.*:]]
; CHECK-NEXT: [[TMP0:%.*]] = load <4 x double>, ptr [[A]], align 8
-; CHECK-NEXT: [[TMP1:%.*]] = fneg reassoc nsz contract <4 x double> [[TMP0]]
-; CHECK-NEXT: [[TMP2:%.*]] = call reassoc nsz contract double @llvm.vector.reduce.fadd.v4f64(double 0.000000e+00, <4 x double> [[TMP1]])
-; CHECK-NEXT: [[OP_RDX:%.*]] = fadd reassoc nsz contract double [[TMP2]], [[B]]
+; CHECK-NEXT: [[TMP1:%.*]] = call reassoc nsz contract double @llvm.vector.reduce.fadd.v4f64(double 0.000000e+00, <4 x double> [[TMP0]])
+; CHECK-NEXT: [[OP_RDX:%.*]] = fsub reassoc nsz contract double [[B]], [[TMP1]]
; CHECK-NEXT: ret double [[OP_RDX]]
;
entry:
@@ -1541,18 +1366,10 @@ define double @fneg_leaves_vs_fsub_chain_fsub(ptr %a, double %b) {
; CHECK-LABEL: define double @fneg_leaves_vs_fsub_chain_fsub(
; CHECK-SAME: ptr [[A:%.*]], double [[B:%.*]]) #[[ATTR0]] {
; CHECK-NEXT: [[ENTRY:.*:]]
-; CHECK-NEXT: [[P1:%.*]] = getelementptr inbounds nuw i8, ptr [[A]], i64 8
-; CHECK-NEXT: [[P2:%.*]] = getelementptr inbounds nuw i8, ptr [[A]], i64 16
-; CHECK-NEXT: [[P3:%.*]] = getelementptr inbounds nuw i8, ptr [[A]], i64 24
-; CHECK-NEXT: [[TMP1:%.*]] = load double, ptr [[A]], align 8
-; CHECK-NEXT: [[A1:%.*]] = load double, ptr [[P1]], align 8
-; CHECK-NEXT: [[A2:%.*]] = load double, ptr [[P2]], align 8
-; CHECK-NEXT: [[A3:%.*]] = load double, ptr [[P3]], align 8
+; CHECK-NEXT: [[TMP0:%.*]] = load <4 x double>, ptr [[A]], align 8
+; CHECK-NEXT: [[TMP1:%.*]] = call reassoc nsz contract double @llvm.vector.reduce.fadd.v4f64(double 0.000000e+00, <4 x double> [[TMP0]])
; CHECK-NEXT: [[OP_RDX:%.*]] = fsub reassoc nsz contract double [[B]], [[TMP1]]
-; CHECK-NEXT: [[T1:%.*]] = fsub reassoc nsz contract double [[OP_RDX]], [[A1]]
-; CHECK-NEXT: [[T2:%.*]] = fsub reassoc nsz contract double [[T1]], [[A2]]
-; CHECK-NEXT: [[T3:%.*]] = fsub reassoc nsz contract double [[T2]], [[A3]]
-; CHECK-NEXT: ret double [[T3]]
+; CHECK-NEXT: ret double [[OP_RDX]]
;
entry:
%p1 = getelementptr inbounds nuw i8, ptr %a, i64 8
diff --git a/llvm/test/Transforms/SLPVectorizer/X86/horizontal-list.ll b/llvm/test/Transforms/SLPVectorizer/X86/horizontal-list.ll
index e7e7e9aa1697d..69822351ffd57 100644
--- a/llvm/test/Transforms/SLPVectorizer/X86/horizontal-list.ll
+++ b/llvm/test/Transforms/SLPVectorizer/X86/horizontal-list.ll
@@ -17,7 +17,7 @@ define float @baz() {
; CHECK-NEXT: [[TMP2:%.*]] = load <4 x float>, ptr @arr1, align 16
; CHECK-NEXT: [[TMP3:%.*]] = fmul fast <4 x float> [[TMP2]], [[TMP1]]
; CHECK-NEXT: [[TMP4:%.*]] = call fast float @llvm.vector.reduce.fadd.v4f32(float 0.000000e+00, <4 x float> [[TMP3]])
-; CHECK-NEXT: [[TMP6:%.*]] = fmul fast float [[TMP4]], 2.000000e+00
+; CHECK-NEXT: [[TMP6:%.*]] = fmul reassoc nnan ninf nsz arcp afn float [[TMP4]], 2.000000e+00
; CHECK-NEXT: [[OP_RDX1:%.*]] = fadd fast float [[TMP6]], [[CONV]]
; CHECK-NEXT: [[OP_RDX:%.*]] = fadd fast float [[OP_RDX1]], [[CONV]]
; CHECK-NEXT: store float [[OP_RDX]], ptr @res, align 4
@@ -32,7 +32,7 @@ define float @baz() {
; THRESHOLD-NEXT: [[TMP2:%.*]] = load <4 x float>, ptr @arr1, align 16
; THRESHOLD-NEXT: [[TMP3:%.*]] = fmul fast <4 x float> [[TMP2]], [[TMP1]]
; THRESHOLD-NEXT: [[TMP4:%.*]] = call fast float @llvm.vector.reduce.fadd.v4f32(float 0.000000e+00, <4 x float> [[TMP3]])
-; THRESHOLD-NEXT: [[TMP9:%.*]] = fmul fast float [[TMP4]], 2.000000e+00
+; THRESHOLD-NEXT: [[TMP9:%.*]] = fmul reassoc nnan ninf nsz arcp afn float [[TMP4]], 2.000000e+00
; THRESHOLD-NEXT: [[OP_RDX1:%.*]] = fadd fast float [[TMP9]], [[CONV]]
; THRESHOLD-NEXT: [[OP_RDX:%.*]] = fadd fast float [[OP_RDX1]], [[CONV]]
; THRESHOLD-NEXT: store float [[OP_RDX]], ptr @res, align 4
@@ -731,7 +731,7 @@ define float @extra_args(ptr nocapture readonly %x, i32 %a, i32 %b) {
; CHECK-NEXT: [[MUL:%.*]] = mul nsw i32 [[B:%.*]], [[A:%.*]]
; CHECK-NEXT: [[CONV:%.*]] = sitofp i32 [[MUL]] to float
; CHECK-NEXT: [[TMP0:%.*]] = load <8 x float>, ptr [[X:%.*]], align 4
-; CHECK-NEXT: [[TMP2:%.*]] = fmul fast float [[CONV]], 2.000000e+00
+; CHECK-NEXT: [[TMP2:%.*]] = fmul reassoc nnan ninf nsz arcp afn float [[CONV]], 2.000000e+00
; CHECK-NEXT: [[TMP3:%.*]] = call fast float @llvm.vector.reduce.fadd.v8f32(float 0.000000e+00, <8 x float> [[TMP0]])
; CHECK-NEXT: [[OP_RDX:%.*]] = fadd fast float [[TMP2]], [[TMP3]]
; CHECK-NEXT: [[OP_RDX1:%.*]] = fadd fast float [[OP_RDX]], 3.000000e+00
@@ -742,7 +742,7 @@ define float @extra_args(ptr nocapture readonly %x, i32 %a, i32 %b) {
; THRESHOLD-NEXT: [[MUL:%.*]] = mul nsw i32 [[B:%.*]], [[A:%.*]]
; THRESHOLD-NEXT: [[CONV:%.*]] = sitofp i32 [[MUL]] to float
; THRESHOLD-NEXT: [[TMP0:%.*]] = load <8 x float>, ptr [[X:%.*]], align 4
-; THRESHOLD-NEXT: [[TMP2:%.*]] = fmul fast float [[CONV]], 2.000000e+00
+; THRESHOLD-NEXT: [[TMP2:%.*]] = fmul reassoc nnan ninf nsz arcp afn float [[CONV]], 2.000000e+00
; THRESHOLD-NEXT: [[TMP3:%.*]] = call fast float @llvm.vector.reduce.fadd.v8f32(float 0.000000e+00, <8 x float> [[TMP0]])
; THRESHOLD-NEXT: [[OP_RDX:%.*]] = fadd fast float [[TMP2]], [[TMP3]]
; THRESHOLD-NEXT: [[OP_RDX1:%.*]] = fadd fast float [[OP_RDX]], 3.000000e+00
@@ -785,7 +785,7 @@ define float @extra_args_same_several_times(ptr nocapture readonly %x, i32 %a, i
; CHECK-NEXT: [[MUL:%.*]] = mul nsw i32 [[B:%.*]], [[A:%.*]]
; CHECK-NEXT: [[CONV:%.*]] = sitofp i32 [[MUL]] to float
; CHECK-NEXT: [[TMP0:%.*]] = load <8 x float>, ptr [[X:%.*]], align 4
-; CHECK-NEXT: [[TMP2:%.*]] = fmul fast float [[CONV]], 2.000000e+00
+; CHECK-NEXT: [[TMP2:%.*]] = fmul reassoc nnan ninf nsz arcp afn float [[CONV]], 2.000000e+00
; CHECK-NEXT: [[OP_RDX:%.*]] = fadd fast float 1.300000e+01, [[TMP2]]
; CHECK-NEXT: [[TMP3:%.*]] = call fast float @llvm.vector.reduce.fadd.v8f32(float 0.000000e+00, <8 x float> [[TMP0]])
; CHECK-NEXT: [[OP_RDX1:%.*]] = fadd fast float [[OP_RDX]], [[TMP3]]
@@ -796,7 +796,7 @@ define float @extra_args_same_several_times(ptr nocapture readonly %x, i32 %a, i
; THRESHOLD-NEXT: [[MUL:%.*]] = mul nsw i32 [[B:%.*]], [[A:%.*]]
; THRESHOLD-NEXT: [[CONV:%.*]] = sitofp i32 [[MUL]] to float
; THRESHOLD-NEXT: [[TMP0:%.*]] = load <8 x float>, ptr [[X:%.*]], align 4
-; THRESHOLD-NEXT: [[TMP2:%.*]] = fmul fast float [[CONV]], 2.000000e+00
+; THRESHOLD-NEXT: [[TMP2:%.*]] = fmul reassoc nnan ninf nsz arcp afn float [[CONV]], 2.000000e+00
; THRESHOLD-NEXT: [[OP_RDX:%.*]] = fadd fast float 1.300000e+01, [[TMP2]]
; THRESHOLD-NEXT: [[TMP3:%.*]] = call fast float @llvm.vector.reduce.fadd.v8f32(float 0.000000e+00, <8 x float> [[TMP0]])
; THRESHOLD-NEXT: [[OP_RDX1:%.*]] = fadd fast float [[OP_RDX]], [[TMP3]]
@@ -842,7 +842,7 @@ define float @extra_args_no_replace(ptr nocapture readonly %x, i32 %a, i32 %b, i
; CHECK-NEXT: [[CONV:%.*]] = sitofp i32 [[MUL]] to float
; CHECK-NEXT: [[CONVC:%.*]] = sitofp i32 [[C:%.*]] to float
; CHECK-NEXT: [[TMP0:%.*]] = load <8 x float>, ptr [[X:%.*]], align 4
-; CHECK-NEXT: [[TMP2:%.*]] = fmul fast float [[CONV]], 2.000000e+00
+; CHECK-NEXT: [[TMP2:%.*]] = fmul reassoc nnan ninf nsz arcp afn float [[CONV]], 2.000000e+00
; CHECK-NEXT: [[TMP3:%.*]] = call fast float @llvm.vector.reduce.fadd.v8f32(float 0.000000e+00, <8 x float> [[TMP0]])
; CHECK-NEXT: [[OP_RDX:%.*]] = fadd fast float [[TMP2]], [[TMP3]]
; CHECK-NEXT: [[OP_RDX1:%.*]] = fadd fast float [[OP_RDX]], 3.000000e+00
@@ -855,7 +855,7 @@ define float @extra_args_no_replace(ptr nocapture readonly %x, i32 %a, i32 %b, i
; THRESHOLD-NEXT: [[CONV:%.*]] = sitofp i32 [[MUL]] to float
; THRESHOLD-NEXT: [[CONVC:%.*]] = sitofp i32 [[C:%.*]] to float
; THRESHOLD-NEXT: [[TMP0:%.*]] = load <8 x float>, ptr [[X:%.*]], align 4
-; THRESHOLD-NEXT: [[TMP2:%.*]] = fmul fast float [[CONV]], 2.000000e+00
+; THRESHOLD-NEXT: [[TMP2:%.*]] = fmul reassoc nnan ninf nsz arcp afn float [[CONV]], 2.000000e+00
; THRESHOLD-NEXT: [[TMP3:%.*]] = call fast float @llvm.vector.reduce.fadd.v8f32(float 0.000000e+00, <8 x float> [[TMP0]])
; THRESHOLD-NEXT: [[OP_RDX:%.*]] = fadd fast float [[TMP2]], [[TMP3]]
; THRESHOLD-NEXT: [[OP_RDX1:%.*]] = fadd fast float [[OP_RDX]], 3.000000e+00
diff --git a/llvm/test/Transforms/SLPVectorizer/X86/loop-accumulator-reduction.ll b/llvm/test/Transforms/SLPVectorizer/X86/loop-accumulator-reduction.ll
index f05cc887d7315..158f855151ed2 100644
--- a/llvm/test/Transforms/SLPVectorizer/X86/loop-accumulator-reduction.ll
+++ b/llvm/test/Transforms/SLPVectorizer/X86/loop-accumulator-reduction.ll
@@ -884,7 +884,7 @@ define double @acc_used_twice(ptr %p, i64 %n) {
; CHECK-NEXT: [[ACC:%.*]] = phi double [ 1.000000e+00, %[[ENTRY]] ], [ [[OP_RDX:%.*]], %[[LOOP]] ]
; CHECK-NEXT: [[P0:%.*]] = getelementptr double, ptr [[P]], i64 [[IV]]
; CHECK-NEXT: [[TMP0:%.*]] = load <4 x double>, ptr [[P0]], align 8
-; CHECK-NEXT: [[TMP1:%.*]] = fmul fast double [[ACC]], 2.000000e+00
+; CHECK-NEXT: [[TMP1:%.*]] = fmul reassoc nnan ninf nsz arcp afn double [[ACC]], 2.000000e+00
; CHECK-NEXT: [[TMP2:%.*]] = call fast double @llvm.vector.reduce.fadd.v4f64(double 0.000000e+00, <4 x double> [[TMP0]])
; CHECK-NEXT: [[OP_RDX]] = fadd fast double [[TMP1]], [[TMP2]]
; CHECK-NEXT: [[IV_NEXT]] = add nuw nsw i64 [[IV]], 1
diff --git a/llvm/test/Transforms/SLPVectorizer/X86/ordered-reduction-replaced.ll b/llvm/test/Transforms/SLPVectorizer/X86/ordered-reduction-replaced.ll
index 7b6c896154c7e..ca6e24fcae5ba 100644
--- a/llvm/test/Transforms/SLPVectorizer/X86/ordered-reduction-replaced.ll
+++ b/llvm/test/Transforms/SLPVectorizer/X86/ordered-reduction-replaced.ll
@@ -6,7 +6,7 @@ define double @test(double %0) {
; CHECK-SAME: double [[TMP0:%.*]]) {
; CHECK-NEXT: [[ENTRY:.*:]]
; CHECK-NEXT: [[TMP5:%.*]] = fmul double [[TMP0]], 0.000000e+00
-; CHECK-NEXT: [[TMP2:%.*]] = fmul double [[TMP5]], 4.000000e+00
+; CHECK-NEXT: [[TMP2:%.*]] = fmul reassoc double [[TMP5]], 4.000000e+00
; CHECK-NEXT: [[TMP7:%.*]] = fadd reassoc double [[TMP2]], 0.000000e+00
; CHECK-NEXT: ret double [[TMP7]]
;
diff --git a/llvm/test/Transforms/SLPVectorizer/X86/reduction-fadd-reassoc.ll b/llvm/test/Transforms/SLPVectorizer/X86/reduction-fadd-reassoc.ll
index 5c934d9e13087..bdb60bf481011 100644
--- a/llvm/test/Transforms/SLPVectorizer/X86/reduction-fadd-reassoc.ll
+++ b/llvm/test/Transforms/SLPVectorizer/X86/reduction-fadd-reassoc.ll
@@ -43,7 +43,7 @@ define double @reassoc_repeated_vals(double %x) {
; CHECK-LABEL: define double @reassoc_repeated_vals(
; CHECK-SAME: double [[X:%.*]]) #[[ATTR0]] {
; CHECK-NEXT: [[ENTRY:.*:]]
-; CHECK-NEXT: [[TMP0:%.*]] = fmul double [[X]], 9.000000e+00
+; CHECK-NEXT: [[TMP0:%.*]] = fmul reassoc double [[X]], 9.000000e+00
; CHECK-NEXT: ret double [[TMP0]]
;
entry:
More information about the llvm-commits
mailing list