[llvm] [SLP] Support ordered fadd reduction via reduction intrinsics (PR #189451)
Ryan Buchner via llvm-commits
llvm-commits at lists.llvm.org
Sat May 9 11:27:16 PDT 2026
================
@@ -28030,6 +28035,93 @@ class HorizontalReduction {
return RK != ReductionOrdering::None;
}
+ /// Analyze whether \p Root forms a linearized ordered reduction chain.
+ /// If \p MatchLHS is true, analyzes LHS-associated (left-linearized) chains
+ /// where the chain recurses on the LHS and the RHS at each level is a leaf:
+ /// ((((v0 op v1) op v2) op v3) op v4)
+ /// If \p MatchLHS is false, analyzes RHS-associated (right-linearized) chains
+ /// where the chain recurses on the RHS and the LHS at each level is a leaf:
+ /// (v0 op (v1 op (v2 op (v3 op v4))))
+ /// Leaf values are stored in ReducedVals.back() in accumulation order
+ /// (innermost pair first, outermost last), e.g. [v0,v1,v2,v3,v4] for
+ /// LHS-associated and [v4,v3,v2,v1,v0] for RHS-associated chains.
+ /// \returns true if the chain is a valid ordered reduction.
+ bool matchOrderedReduction(BoUpSLP &R, Instruction *Root, bool MatchLHS) {
+ ReducedVals.clear();
+ ReducedValsToOps.clear();
+ ReductionOps.clear();
+ RdxKind = getRdxKind(Root);
+ // Currently, only ordered fadd reductions are supported.
+ if (RdxKind != RecurKind::FAdd)
+ return false;
+ if (isVectorizable(RdxKind, Root) != ReductionOrdering::Ordered)
+ return false;
+
+ // Ordered reductions only support simple binary ops, not min/max
+ // select(cmp) patterns or poison-safe bool logic ops.
+ if (isCmpSelMinMax(Root) || isBoolLogicOp(Root))
+ return false;
+
+ Type *Ty = Root->getType();
+ if (!isValidElementType(Ty) || Ty->isPointerTy())
+ return false;
+
+ // This is an ordered (linearized) reduction regardless of whether the
+ // individual operations are associative.
+ RK = ReductionOrdering::Ordered;
+ ReductionRoot = Root;
+ unsigned FirstOpIdx = getFirstOperandIndex(Root);
+ unsigned ChainOpIdx = MatchLHS ? FirstOpIdx : FirstOpIdx + 1;
+ unsigned LeafOpIdx = MatchLHS ? FirstOpIdx + 1 : FirstOpIdx;
+ initReductionOps(Root);
+ ReducedVals.emplace_back();
+ SmallPtrSet<Instruction *, 16> Visited;
+ Instruction *TreeN = Root;
+ unsigned Depth = 0;
+ bool ChainComplete = false;
+ constexpr unsigned MaxReducedVals = 1024;
+ while (TreeN) {
+ if (Depth++ > RecursionMaxDepth)
+ break;
+ if (ReducedVals.back().size() >= MaxReducedVals)
+ break;
+ if (!Visited.insert(TreeN).second)
+ break;
+ if (getRdxKind(TreeN) != RdxKind)
+ break;
+ addReductionOps(TreeN);
+ Value *EdgeVal = getRdxOperand(TreeN, LeafOpIdx);
+ Value *ChainVal = getRdxOperand(TreeN, ChainOpIdx);
+ ReducedValsToOps[EdgeVal].push_back(TreeN);
+ ReducedValsToOps[ChainVal].push_back(TreeN);
+ ReducedVals.back().push_back(EdgeVal);
+ auto *EdgeInst = dyn_cast<Instruction>(ChainVal);
----------------
bababuck wrote:
On that same note, why do we use `Edge` at all. Wouldn't leaf be more appropriate?
```
Value *EdgeVal = getRdxOperand(TreeN, LeafOpIdx);
```
to
```
Value *ChainVal = getRdxOperand(TreeN, LeafOpIdx);
```
https://github.com/llvm/llvm-project/pull/189451
More information about the llvm-commits
mailing list