[llvm] [SLP] Support ordered FAdd reductions in SLPVectorizer (PR #146570)
via llvm-commits
llvm-commits at lists.llvm.org
Wed Jul 9 09:10:50 PDT 2025
github-actions[bot] wrote:
<!--LLVM CODE FORMAT COMMENT: {clang-format}-->
:warning: C/C++ code formatter, clang-format found issues in your code. :warning:
<details>
<summary>
You can test this locally with the following command:
</summary>
``````````bash
git-clang-format --diff HEAD~1 HEAD --extensions cpp -- llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp
``````````
</details>
<details>
<summary>
View the diff from clang-format here.
</summary>
``````````diff
diff --git a/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp b/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp
index 5efd346c9..e003e800c 100644
--- a/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp
+++ b/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp
@@ -22076,32 +22076,33 @@ private:
// Checks if the operands of the \p TreeN instruction are also reduction
// operations or should be treated as reduced values or an extra argument,
// which is not part of the reduction.
- void CheckOperands(BoUpSLP &R, Instruction *TreeN, Instruction *Root, SmallVectorImpl<Value *> &PossibleReducedVals,
- SmallVectorImpl<Instruction *> &ReductionOps,
- unsigned Level) {
- bool IsCmpSelMinMax = isCmpSelMinMax(Root);
- for (int I : reverse(seq<int>(getFirstOperandIndex(TreeN),
- getNumberOfOperands(TreeN)))) {
- Value *EdgeVal = getRdxOperand(TreeN, I);
- ReducedValsToOps[EdgeVal].push_back(TreeN);
- auto *EdgeInst = dyn_cast<Instruction>(EdgeVal);
- // If the edge is not an instruction, or it is different from the main
- // reduction opcode or has too many uses - possible reduced value.
- // Also, do not try to reduce const values, if the operation is not
- // foldable.
- if (!EdgeInst || Level > RecursionMaxDepth ||
- getRdxKind(EdgeInst) != RdxKind ||
- IsCmpSelMinMax != isCmpSelMinMax(EdgeInst) ||
- !hasRequiredNumberOfUses(IsCmpSelMinMax, EdgeInst) ||
- !isVectorizable(RdxKind, EdgeInst) ||
- (R.isAnalyzedReductionRoot(EdgeInst) &&
- all_of(EdgeInst->operands(), IsaPred<Constant>))) {
- PossibleReducedVals.push_back(EdgeVal);
- continue;
- }
- ReductionOps.push_back(EdgeInst);
+ void CheckOperands(BoUpSLP &R, Instruction *TreeN, Instruction *Root,
+ SmallVectorImpl<Value *> &PossibleReducedVals,
+ SmallVectorImpl<Instruction *> &ReductionOps,
+ unsigned Level) {
+ bool IsCmpSelMinMax = isCmpSelMinMax(Root);
+ for (int I : reverse(seq<int>(getFirstOperandIndex(TreeN),
+ getNumberOfOperands(TreeN)))) {
+ Value *EdgeVal = getRdxOperand(TreeN, I);
+ ReducedValsToOps[EdgeVal].push_back(TreeN);
+ auto *EdgeInst = dyn_cast<Instruction>(EdgeVal);
+ // If the edge is not an instruction, or it is different from the main
+ // reduction opcode or has too many uses - possible reduced value.
+ // Also, do not try to reduce const values, if the operation is not
+ // foldable.
+ if (!EdgeInst || Level > RecursionMaxDepth ||
+ getRdxKind(EdgeInst) != RdxKind ||
+ IsCmpSelMinMax != isCmpSelMinMax(EdgeInst) ||
+ !hasRequiredNumberOfUses(IsCmpSelMinMax, EdgeInst) ||
+ !isVectorizable(RdxKind, EdgeInst) ||
+ (R.isAnalyzedReductionRoot(EdgeInst) &&
+ all_of(EdgeInst->operands(), IsaPred<Constant>))) {
+ PossibleReducedVals.push_back(EdgeVal);
+ continue;
}
- };
+ ReductionOps.push_back(EdgeInst);
+ }
+ };
public:
HorizontalReduction() = default;
@@ -22184,14 +22185,15 @@ public:
auto [TreeN, Level] = Worklist.pop_back_val();
SmallVector<Value *> PossibleRedVals;
SmallVector<Instruction *> PossibleReductionOps;
- CheckOperands(R, TreeN, Root, PossibleRedVals, PossibleReductionOps, Level);
+ CheckOperands(R, TreeN, Root, PossibleRedVals, PossibleReductionOps,
+ Level);
addReductionOps(TreeN);
// Add reduction values. The values are sorted for better vectorization
// results.
for (Value *V : PossibleRedVals) {
size_t Key = 0, Idx = 0;
std::tie(Key, Idx) = generateKeySubkey(V, &TLI, GenerateLoadsSubkey,
- /*AllowAlternate=*/false);
+ /*AllowAlternate=*/false);
++PossibleReducedVals[Key][Idx]
.insert(std::make_pair(V, 0))
.first->second;
@@ -22246,8 +22248,8 @@ public:
}
bool matchNonAssociativeReduction(BoUpSLP &R, Instruction *Root,
- ScalarEvolution &SE, const DataLayout &DL,
- const TargetLibraryInfo &TLI) {
+ ScalarEvolution &SE, const DataLayout &DL,
+ const TargetLibraryInfo &TLI) {
RdxKind = HorizontalReduction::getRdxKind(Root);
if (!isVectorizable(RdxKind, Root))
return false;
@@ -22274,7 +22276,8 @@ public:
auto [TreeN, Level] = Worklist.pop_back_val();
SmallVector<Value *> PossibleRedVals;
SmallVector<Instruction *> PossibleReductionOps;
- CheckOperands(R, TreeN, Root, PossibleRedVals, PossibleReductionOps, Level);
+ CheckOperands(R, TreeN, Root, PossibleRedVals, PossibleReductionOps,
+ Level);
addReductionOps(TreeN);
for (Value *V : PossibleRedVals)
++ReusedVals.insert(std::make_pair(V, 0)).first->second;
@@ -22287,7 +22290,8 @@ public:
std::reverse(ReducedVals[0].begin(), ReducedVals[0].end());
- if (ReducedVals[0].size() == 2 || !checkOperandsOrder() || !checkFastMathFlags())
+ if (ReducedVals[0].size() == 2 || !checkOperandsOrder() ||
+ !checkFastMathFlags())
return false;
return true;
@@ -22482,9 +22486,9 @@ public:
// Check if we support repeated scalar values processing (optimization of
// original scalar identity operations on matched horizontal reductions).
- IsSupportedHorRdxIdentityOp = RdxKind != RecurKind::Mul &&
- RdxKind != RecurKind::FMul &&
- RdxKind != RecurKind::FMulAdd && !isOrderedFaddReduction();
+ IsSupportedHorRdxIdentityOp =
+ RdxKind != RecurKind::Mul && RdxKind != RecurKind::FMul &&
+ RdxKind != RecurKind::FMulAdd && !isOrderedFaddReduction();
// Gather same values.
SmallMapVector<Value *, unsigned, 16> SameValuesCounter;
if (IsSupportedHorRdxIdentityOp)
@@ -22585,7 +22589,8 @@ public:
return IsAnyRedOpGathered;
};
bool AnyVectorized = false;
- Instruction *RdxRootInst = cast<Instruction>(ReductionRoot);;
+ Instruction *RdxRootInst = cast<Instruction>(ReductionRoot);
+ ;
Instruction *InsertPt = RdxRootInst;
SmallDenseSet<std::pair<unsigned, unsigned>, 8> IgnoredCandidates;
while (Pos < NumReducedVals - ReduxWidth + 1 &&
@@ -22859,17 +22864,18 @@ public:
} else {
for (auto V : VectorValuesAndScales) {
Value *InitialFAddValue = InitialFAddValues.back();
- VectorizedTree = Builder.CreateFAddReduce(InitialFAddValue, std::get<0>(V));
+ VectorizedTree =
+ Builder.CreateFAddReduce(InitialFAddValue, std::get<0>(V));
InitialFAddValues.push_back(VectorizedTree);
}
auto LastIt = find_if(reverse(ReducedVals[0]), [&](Value *RdxVal) {
return VectorizedVals.lookup(RdxVal);
});
for_each(reverse(make_range(LastIt.base(), ReducedVals[0].end())),
- [&](Value *V) {
- ReducedValsToOps.find(V)->second[0]->moveAfter(
- cast<Instruction>(VectorizedTree));
- });
+ [&](Value *V) {
+ ReducedValsToOps.find(V)->second[0]->moveAfter(
+ cast<Instruction>(VectorizedTree));
+ });
}
}
if (VectorizedTree) {
@@ -22986,17 +22992,15 @@ public:
for (ArrayRef<Value *> RdxOps : ReductionOps) {
SmallVector<Value *, 4> RdxOpsForDeletion;
for (Value *Ignore : RdxOps) {
- if (!Ignore || (isOrderedFaddReduction() && !Ignore->use_empty() &&
- !any_of(cast<Instruction>(Ignore)->operands(),
- [](const Value *Val) {
- return isa<PoisonValue>(Val);
- })))
+ if (!Ignore ||
+ (isOrderedFaddReduction() && !Ignore->use_empty() &&
+ !any_of(cast<Instruction>(Ignore)->operands(),
+ [](const Value *Val) { return isa<PoisonValue>(Val); })))
continue;
#ifndef NDEBUG
for (auto *U : Ignore->users()) {
- assert((IgnoreSet.count(U) ||
- isOrderedFaddReduction()) &&
- "All users must be either in the reduction ops list.");
+ assert((IgnoreSet.count(U) || isOrderedFaddReduction()) &&
+ "All users must be either in the reduction ops list.");
}
#endif
if (!Ignore->use_empty()) {
@@ -23005,7 +23009,8 @@ public:
}
RdxOpsForDeletion.push_back(Ignore);
}
- V.removeInstructionsAndOperands(ArrayRef(RdxOpsForDeletion), VectorValuesAndScales);
+ V.removeInstructionsAndOperands(ArrayRef(RdxOpsForDeletion),
+ VectorValuesAndScales);
}
} else if (!CheckForReusedReductionOps) {
for (ReductionOpsType &RdxOps : ReductionOps)
@@ -23808,7 +23813,8 @@ bool SLPVectorizerPass::vectorizeHorReduction(
if (!isReductionCandidate(Inst))
return nullptr;
HorizontalReduction HorRdx;
- if (!HorRdx.matchAssociativeReduction(R, Inst, *SE, *DL, *TLI) && !HorRdx.matchNonAssociativeReduction(R, Inst, *SE, *DL, *TLI))
+ if (!HorRdx.matchAssociativeReduction(R, Inst, *SE, *DL, *TLI) &&
+ !HorRdx.matchNonAssociativeReduction(R, Inst, *SE, *DL, *TLI))
return nullptr;
return HorRdx.tryToReduce(R, *DL, TTI, *TLI, AC);
};
``````````
</details>
https://github.com/llvm/llvm-project/pull/146570
More information about the llvm-commits
mailing list