[llvm] [SLP]Flatten fsub/fneg chains into fadd reductions with per-operand signs (PR #215840)
Ryan Buchner via llvm-commits
llvm-commits at lists.llvm.org
Sat Aug 15 23:33:36 PDT 2026
================
@@ -30498,96 +30444,263 @@ class HorizontalReduction {
SmallVector<Value *> ReducedValsCandidates;
bool AdjustedToOrdered = false;
- SmallPtrSet<Instruction *, 16> Visited;
- while (!Worklist.empty()) {
- auto [TreeN, Level] = Worklist.pop_back_val();
- if (!Visited.insert(TreeN).second)
- continue;
- SmallVector<Value *> PossibleRedVals;
- SmallVector<Instruction *> PossibleReductionOps;
- CheckOperands(TreeN, PossibleRedVals, PossibleReductionOps,
- ReductionOps[0], Level);
- addReductionOps(TreeN);
- ReducedValsCandidates.append(PossibleRedVals.begin(),
- PossibleRedVals.end());
- for (Instruction *I : reverse(PossibleReductionOps))
- Worklist.emplace_back(I, I->getParent() == BB ? 0 : Level + 1);
- // If not enough elements for unordered vectorization, check if there are
- // potential candidates for the ordered vectorization and try to add them
- // to the worklist.
- if (Worklist.empty() && ReducedValsCandidates.size() < ReductionLimit &&
- !PossibleOrderedReductionOps.empty() &&
- RK == ReductionOrdering::Unordered) {
- RK = ReductionOrdering::Ordered;
- AdjustedToOrdered = true;
- SmallPtrSet<const Instruction *, 4> Ops;
- for (const auto &P : PossibleOrderedReductionOps)
- Ops.insert(P.first);
- erase_if(ReducedValsCandidates, [&](Value *V) {
- auto *I = dyn_cast<Instruction>(V);
- return I && Ops.contains(I);
- });
- Worklist.append(PossibleOrderedReductionOps.begin(),
- PossibleOrderedReductionOps.end());
- PossibleOrderedReductionOps.clear();
+ const ReductionOrdering InitialRK = RK;
+ // For unordered fadd reductions, reassociable fsub/fneg chain links are
+ // flattened with a flipped sign on the subtracted operand: the leaves,
+ // reduced through such links, are subtracted in the final combine.
+ // Ordered reductions keep their accumulation order and are excluded.
+ bool TrackSign = RdxKind == RecurKind::FAdd && !IsCmpSelMinMax &&
+ RK == ReductionOrdering::Unordered;
+ // The leaf signs are modeled per value. If they cannot be modeled this way
+ // (the same value occurs both added and subtracted, or the reduction
+ // switches to ordered after some fsub/fneg links have been flattened
+ // already), the analysis restarts without the fsub/fneg flattening.
+ bool Restart;
+ do {
+ Restart = false;
+ // Try to regroup reduced values so that it gets more profitable to try
+ // to reduce them. Values are grouped by their value ids, instructions -
+ // by instruction op id and/or alternate op id, plus do extra analysis
+ // for loads (grouping them by the distance between pointers) and cmp
+ // instructions (grouping them by the predicate).
+ SmallMapVector<
+ size_t,
+ SmallMapVector<size_t, SmallMapVector<Value *, unsigned, 2>, 2>, 8>
+ PossibleReducedVals;
+ RK = InitialRK;
+ ReducedVals.clear();
+ ReducedValsToOps.clear();
+ NegatedReducedVals.clear();
+ ReducedValsCandidates.clear();
+ LoadsMap.clear();
+ LoadKeyUsed.clear();
+ AdjustedToOrdered = false;
+ initReductionOps(Root);
+ SmallVector<std::tuple<Instruction *, unsigned, bool>> Worklist(
+ 1, std::make_tuple(Root, 0, false));
+ SmallPtrSet<Value *, 8> Operands;
+ SmallVector<std::tuple<Instruction *, unsigned, bool>>
+ PossibleOrderedReductionOps;
+ // Leaves that (also) occur with a non-flipped sign.
+ SmallPtrSet<Value *, 8> PositiveReducedVals;
+ // Checks if the instruction continues the reduction chain. An fsub/fneg
+ // chain link in an fadd reduction is recursed into like an fadd, but its
+ // subtracted operand enters with a flipped sign. Only reassociable links
+ // qualify.
+ auto IsChainLink = [&](Instruction *I) {
+ if (getRdxKind(I) == RdxKind)
+ return true;
+ // nsz is required: subtracted leaves are regrouped and negated as a
+ // whole, and -a + -b == -(a + b) may flip the sign of a zero result.
+ return TrackSign &&
+ (I->getOpcode() == Instruction::FSub ||
+ I->getOpcode() == Instruction::FNeg) &&
+ I->hasAllowReassoc() && I->hasNoSignedZeros();
+ };
+ // Checks if the operands of the \p TreeN instruction are also reduction
+ // operations or should be treated as reduced values or an extra
+ // argument, which is not part of the reduction.
+ auto CheckOperands =
+ [&](Instruction *TreeN, SmallVectorImpl<Value *> &PossibleReducedVals,
+ SmallVectorImpl<std::pair<Instruction *, bool>> &ReductionOps,
+ ReductionOpsType &AllReductionOps, unsigned Level, bool Negated) {
+ for (int I : reverse(seq<int>(getFirstOperandIndex(TreeN),
+ getNumberOfOperands(TreeN)))) {
+ Value *EdgeVal = getRdxOperand(TreeN, I);
+ auto *EdgeInst = dyn_cast<Instruction>(EdgeVal);
+ // The link may be an elidable cast round-trip; look through it to
+ // continue the chain. It is dropped when the reduction is folded.
+ if (EdgeInst && getRdxKind(EdgeInst) != RdxKind) {
+ if (Instruction *NarrowCast = lookThroughCastRoundTrip(
+ EdgeVal, /*MustBeElidable=*/true)) {
+ auto *SrcI = cast<Instruction>(NarrowCast->getOperand(0));
+ if (getRdxKind(SrcI) == RdxKind &&
+ hasRequiredNumberOfUses(IsCmpSelMinMax, SrcI)) {
+ AllReductionOps.push_back(EdgeVal);
+ AllReductionOps.push_back(NarrowCast);
+ EdgeVal = SrcI;
+ EdgeInst = SrcI;
+ }
+ }
+ }
+ ReducedValsToOps[EdgeVal].push_back(TreeN);
+ // fsub flips its second operand's sign, fneg its only
+ // operand's.
+ bool EdgeNegated =
+ Negated !=
+ ((TreeN->getOpcode() == Instruction::FSub && I == 1) ||
+ TreeN->getOpcode() == Instruction::FNeg);
+ // If the edge is not an instruction, or it is different from
+ // the main reduction opcode or has too many uses - possible
+ // reduced value. Also, do not try to reduce const values, if
+ // the operation is not foldable.
+ bool IsReducedVal = !EdgeInst || Level > RecursionMaxDepth ||
+ !IsChainLink(EdgeInst) ||
+ IsCmpSelMinMax != isCmpSelMinMax(EdgeInst);
+ ReductionOrdering CurrentRK =
+ IsReducedVal ? ReductionOrdering::None
+ : isVectorizable(RdxKind, EdgeInst);
+ if (!IsReducedVal && CurrentRK == ReductionOrdering::Unordered &&
+ RK == ReductionOrdering::Unordered &&
+ !hasRequiredNumberOfUses(IsCmpSelMinMax, EdgeInst)) {
+ IsReducedVal = true;
+ CurrentRK = ReductionOrdering::None;
+ if (PossibleReducedVals.size() < ReductionLimit &&
+ !Operands.contains(EdgeInst))
+ PossibleOrderedReductionOps.emplace_back(EdgeInst, Level,
+ false);
+ }
+ if (CurrentRK == ReductionOrdering::None ||
+ Operands.contains(EdgeInst) ||
+ (R.isAnalyzedReductionRoot(EdgeInst) &&
+ all_of(EdgeInst->operands(), IsaPred<Constant>))) {
+ // Keep the sign of the leaf. A leaf, occurring with both
+ // signs, cannot be modeled with a per-value sign - restart
+ // without sign tracking.
+ if (TrackSign) {
+ auto &Same =
+ EdgeNegated ? NegatedReducedVals : PositiveReducedVals;
+ auto &Other =
+ EdgeNegated ? PositiveReducedVals : NegatedReducedVals;
+ Same.insert(EdgeVal);
+ Restart |= Other.contains(EdgeVal);
+ }
+ PossibleReducedVals.push_back(EdgeVal);
+ if (EdgeInst && !isCmpSelMinMax(EdgeInst))
+ Operands.insert_range(EdgeInst->operands());
+ continue;
+ }
+ if (CurrentRK == ReductionOrdering::Ordered)
+ RK = ReductionOrdering::Ordered;
+ ReductionOps.emplace_back(EdgeInst, EdgeNegated);
+ }
+ };
+ SmallPtrSet<Instruction *, 16> Visited;
+ while (!Worklist.empty() && !Restart) {
+ auto [TreeN, Level, Negated] = Worklist.pop_back_val();
+ if (!Visited.insert(TreeN).second)
+ continue;
+ SmallVector<Value *> PossibleRedVals;
+ SmallVector<std::pair<Instruction *, bool>> PossibleReductionOps;
+ CheckOperands(TreeN, PossibleRedVals, PossibleReductionOps,
+ ReductionOps[0], Level, Negated);
+ addReductionOps(TreeN);
+ ReducedValsCandidates.append(PossibleRedVals.begin(),
+ PossibleRedVals.end());
+ for (auto [I, OpNegated] : reverse(PossibleReductionOps))
+ Worklist.emplace_back(I, I->getParent() == BB ? 0 : Level + 1,
+ OpNegated);
+ // If not enough elements for unordered vectorization, check if there
+ // are potential candidates for the ordered vectorization and try to
+ // add them to the worklist.
+ if (Worklist.empty() && ReducedValsCandidates.size() < ReductionLimit &&
+ !PossibleOrderedReductionOps.empty() &&
+ RK == ReductionOrdering::Unordered) {
+ RK = ReductionOrdering::Ordered;
+ AdjustedToOrdered = true;
+ SmallPtrSet<const Instruction *, 4> Ops;
+ for (const auto &P : PossibleOrderedReductionOps)
+ Ops.insert(std::get<0>(P));
+ erase_if(ReducedValsCandidates, [&](Value *V) {
+ auto *I = dyn_cast<Instruction>(V);
+ return I && Ops.contains(I);
+ });
+ Worklist.append(PossibleOrderedReductionOps);
+ PossibleOrderedReductionOps.clear();
+ }
+ }
+ // The signs of the flattened leaves are meaningless for the ordered
+ // reduction - restart without the fsub/fneg flattening, if any occurred.
+ if (RK == ReductionOrdering::Ordered)
+ Restart |= !NegatedReducedVals.empty();
+ // Negating the regrouped subtracted leaves as a whole is not
+ // sign-of-zero-safe, so the whole flattened chain must be nsz.
+ if (!NegatedReducedVals.empty() &&
+ any_of(ReductionOps.front(), [](Value *Op) {
+ return !cast<Instruction>(Op)->hasNoSignedZeros();
+ }))
+ Restart = true;
+ if (!Restart) {
+ // Too many integer reduced values candidates for the ordered
+ // reductions after adjustements - try to switch to unordered
+ // reductions instead.
+ constexpr unsigned ReducedValsLimit = 1024;
----------------
bababuck wrote:
Do you have case which this limit is hit or was this added pre-emptively. I'm just curious as to the type of code that would hit this limit.
https://github.com/llvm/llvm-project/pull/215840
More information about the llvm-commits
mailing list