[llvm] [SLP] Account for vectorizable reduction consumers (PR #227303)
via llvm-commits
llvm-commits at lists.llvm.org
Tue Sep 29 06:03:18 PDT 2026
llvmorg-github-actions[bot] wrote:
<!--LLVM PR SUMMARY COMMENT-->
@llvm/pr-subscribers-vectorizers
@llvm/pr-subscribers-llvm-transforms
Author: Dávid Bolvanský (davidbolvansky)
<details>
<summary>Changes</summary>
This fixes a missed SLP opportunity in Firefox's `LightPixel` code, visible in
the `rasterflood-svg` benchmark (https://github.com/llvm/llvm-project/issues/46312):
```c
void shade(uint8_t components[4], uint32_t scale) {
components[0] = umin((scale * components[0]) >> 15, 255U);
components[1] = umin((scale * components[1]) >> 15, 255U);
components[2] = umin((scale * components[2]) >> 15, 255U);
components[3] = umax(components[0],
umax(components[1], components[2]));
}
```
The three component computations form a non-power-of-two vector, but their
horizontal reduction is not profitable in isolation because its values need
to be extracted for the scalar stores. Conversely, starting from the four
stores hides the reduction behind a mixed scalar bundle.
Analyze such reduction roots before their store chains and account for a
compatible consecutive store consumer when checking profitability. This lets
SLP produce a `<3 x i32>` computation and reduction followed by one `<4 x i8>`
store on Haswell.
The non-power-of-two override is analysis-only. Emission still requires the
existing reduction transformation to be legal and both the consumer bundle
and their combined cost to pass the normal SLP threshold. Nonconsecutive or
interfering stores do not use the combined cost.
Fixes #<!-- -->46312.
---
Full diff: https://github.com/llvm/llvm-project/pull/227303.diff
3 Files Affected:
- (modified) llvm/include/llvm/Transforms/Vectorize/SLPVectorizer.h (+4-1)
- (modified) llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp (+205-7)
- (added) llvm/test/Transforms/SLPVectorizer/X86/reduction-external-store-bundle.ll (+70)
``````````diff
diff --git a/llvm/include/llvm/Transforms/Vectorize/SLPVectorizer.h b/llvm/include/llvm/Transforms/Vectorize/SLPVectorizer.h
index 3d8ab0d1b7573..db268423b200a 100644
--- a/llvm/include/llvm/Transforms/Vectorize/SLPVectorizer.h
+++ b/llvm/include/llvm/Transforms/Vectorize/SLPVectorizer.h
@@ -146,7 +146,8 @@ struct SLPVectorizerPass : public OptionalPassInfoMixin<SLPVectorizerPass> {
bool
vectorizeRootInstruction(PHINode *P, Instruction *Root, BasicBlock *BB,
slpvectorizer::BoUpSLP &R,
- SmallSetVector<Instruction *, 8> &FMACandidates);
+ SmallSetVector<Instruction *, 8> &FMACandidates,
+ bool AllowNonPowerOf2Analysis = false);
/// Try to vectorize trees that start at insertvalue instructions.
bool vectorizeInsertValueInst(InsertValueInst *IVI, BasicBlock *BB,
@@ -181,6 +182,8 @@ struct SLPVectorizerPass : public OptionalPassInfoMixin<SLPVectorizerPass> {
bool vectorizeChainsInBlock(BasicBlock *BB, slpvectorizer::BoUpSLP &R,
SmallSetVector<Instruction *, 8> &FMACandidates);
+ bool isReductionStoreChainRoot(Instruction *I) const;
+
std::optional<bool> vectorizeStoreChain(ArrayRef<Value *> Chain,
slpvectorizer::BoUpSLP &R,
unsigned Idx, unsigned MinVF,
diff --git a/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp b/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp
index e0dd13ce5c2a8..676b03c3b38e8 100644
--- a/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp
+++ b/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp
@@ -445,6 +445,17 @@ class slpvectorizer::BoUpSLP {
MinVecRegSize = TTI->getMinVectorRegisterBitWidth();
}
+ /// Permit building an NPOT reduction tree for combined-profitability
+ /// analysis. Emission remains guarded by the normal policy or a profitable
+ /// external-consumer bundle.
+ void setAllowReductionNonPowerOf2Analysis(bool Allow) {
+ AllowReductionNonPowerOf2Analysis = Allow;
+ }
+
+ bool allowsReductionNonPowerOf2Analysis() const {
+ return AllowReductionNonPowerOf2Analysis;
+ }
+
/// Vectorize the tree that starts with the elements in \p VL.
/// Returns the vectorized root.
Value *vectorizeTree();
@@ -2459,6 +2470,8 @@ class slpvectorizer::BoUpSLP {
~BoUpSLP();
private:
+ /// Temporary analysis-only override; never authorizes emission on its own.
+ bool AllowReductionNonPowerOf2Analysis = false;
/// Determine if a node \p E in can be demoted to a smaller type with a
/// truncation. We collect the entries that will be demoted in ToDemote.
/// \param E Node for analysis
@@ -8622,7 +8635,8 @@ void BoUpSLP::tryToVectorizeGatheredLoads(
SmallVector<std::pair<ArrayRef<Value *>, LoadsState>> Results;
unsigned StartIdx = 0;
SmallVector<int> CandidateVFs;
- if (isAllowedNonPowerOf2VF(MaxVF, VectorizeNonPowerOf2))
+ if (isAllowedNonPowerOf2VF(MaxVF, VectorizeNonPowerOf2 ||
+ AllowReductionNonPowerOf2Analysis))
CandidateVFs.push_back(MaxVF);
for (int NumElts = getFloorFullVectorNumberOfElements(
*TTI, Loads.front()->getType(), MaxVF, SLPReVec);
@@ -8630,7 +8644,8 @@ void BoUpSLP::tryToVectorizeGatheredLoads(
NumElts = getFloorFullVectorNumberOfElements(
*TTI, Loads.front()->getType(), NumElts - 1, SLPReVec)) {
CandidateVFs.push_back(NumElts);
- if (VectorizeNonPowerOf2 && NumElts > 2)
+ if ((VectorizeNonPowerOf2 || AllowReductionNonPowerOf2Analysis) &&
+ NumElts > 2)
CandidateVFs.push_back(NumElts - 1);
}
@@ -29193,6 +29208,18 @@ bool SLPVectorizerPass::runImpl(Function &F, ScalarEvolution *SE_,
R.clearReductionData();
collectSeedInstructions(BB);
+ // Analyze complete reduction chains before partial store trees can mark
+ // their leaves as failed reduction candidates.
+ SmallVector<WeakTrackingVH> ReductionRoots;
+ for (Instruction &I : reverse(*BB))
+ if (isReductionStoreChainRoot(&I))
+ ReductionRoots.emplace_back(&I);
+ for (Value *V : ReductionRoots)
+ if (auto *I = dyn_cast_or_null<Instruction>(V); I && !R.isDeleted(I))
+ Changed |= vectorizeRootInstruction(nullptr, I, BB, R, FMACandidates,
+ /*AllowNonPowerOf2Analysis=*/true);
+ collectSeedInstructions(BB);
+
// Vectorize trees that end at stores.
if (!Stores.empty()) {
LLVM_DEBUG(dbgs() << "SLP: Found stores for " << Stores.size()
@@ -30603,7 +30630,7 @@ class HorizontalReduction {
/// matched horizontal reductions is enabled and allowed.
bool IsSupportedHorRdxIdentityOp = false;
/// The minimum number of the reduced values.
- const unsigned ReductionLimit = VectorizeNonPowerOf2 ? 3 : 4;
+ unsigned ReductionLimit = VectorizeNonPowerOf2 ? 3 : 4;
/// Contains vector values for reduction including their scale factor,
/// signedness and sign of their contribution to the reduction result.
SmallVector<ReductionVectorPart> VectorValuesAndScales;
@@ -31150,6 +31177,8 @@ class HorizontalReduction {
const TargetTransformInfo &TTI,
const TargetLibraryInfo &TLI,
bool FlattenNegations = true) {
+ ReductionLimit =
+ VectorizeNonPowerOf2 || R.allowsReductionNonPowerOf2Analysis() ? 3 : 4;
RdxKind = HorizontalReduction::getRdxKind(Root);
// A reassociable fsub root is a flattened link of an fadd reduction: its
// subtracted operand enters with a flipped sign. Without the flattening
@@ -32691,7 +32720,9 @@ class HorizontalReduction {
ReduxWidth = bit_floor(ReduxWidth);
return ReduxWidth;
};
- if (!isAllowedNonPowerOf2VF(ReduxWidth, VectorizeNonPowerOf2))
+ if (!isAllowedNonPowerOf2VF(
+ ReduxWidth, VectorizeNonPowerOf2 ||
+ V.allowsReductionNonPowerOf2Analysis()))
ReduxWidth = GetVectorFactor(ReduxWidth);
ReduxWidth = std::min(ReduxWidth, MaxElts);
@@ -32878,7 +32909,138 @@ class HorizontalReduction {
<< " for reduction\n");
if (!Cost.isValid())
break;
- if (Cost >= -SLPCostThreshold) {
+ // A seed reduction's external-use extracts may be folded by a
+ // compatible consumer bundle ending in consecutive stores. Account
+ // for the store saving before deciding the reduction in isolation.
+ auto GetConsumerBundleBenefit = [&]() -> InstructionCost {
+ if (!IsSeedRoot || !ReductionRoot->hasOneUse())
+ return 0;
+ SmallVector<Instruction *> Consumers;
+ for (const BoUpSLP::ExternalUser &EU : V.ExternalUses) {
+ auto *User = dyn_cast_or_null<Instruction>(EU.User);
+ if (!User || !User->hasOneUse())
+ return 0;
+ Consumers.push_back(User);
+ }
+ if (Consumers.size() != VL.size())
+ return 0;
+ auto *RootUser = dyn_cast<Instruction>(*ReductionRoot->user_begin());
+ if (!RootUser || !RootUser->hasOneUse())
+ return 0;
+ Consumers.push_back(RootUser);
+ SmallVector<Value *> ConsumerVals(Consumers.begin(), Consumers.end());
+ InstructionsState ConsumerState = getSameOpcode(ConsumerVals, TLI);
+ if (!ConsumerState || !all_of(Consumers, IsaPred<CastInst>))
+ return 0;
+ SmallVector<StoreInst *> Sinks;
+ for (Instruction *Consumer : Consumers) {
+ auto *SI = dyn_cast<StoreInst>(*Consumer->user_begin());
+ if (!SI || SI->getParent() !=
+ cast<Instruction>(ReductionRoot)->getParent())
+ return 0;
+ Sinks.push_back(SI);
+ }
+ Instruction *FirstSink =
+ *llvm::min_element(Sinks, [](Instruction *L, Instruction *R) {
+ return L->comesBefore(R);
+ });
+ Instruction *LastSink =
+ *llvm::max_element(Sinks, [](Instruction *L, Instruction *R) {
+ return L->comesBefore(R);
+ });
+ for (Instruction *I = FirstSink; I != LastSink->getNextNode();
+ I = I->getNextNode())
+ if (I->mayReadOrWriteMemory() && !is_contained(Sinks, I) &&
+ !V.isVectorized(I))
+ return 0;
+ Type *ScalarTy = Sinks.front()->getValueOperand()->getType();
+ SmallVector<Value *> PointerOps;
+ for (StoreInst *SI : Sinks) {
+ if (SI->getValueOperand()->getType() != ScalarTy)
+ return 0;
+ PointerOps.push_back(SI->getPointerOperand());
+ }
+ BoUpSLP::OrdersType StoreOrder;
+ if (!sortPtrAccesses(PointerOps, ScalarTy, DL, *V.SE, StoreOrder))
+ return 0;
+ if (!V.canFormVector(Sinks, StoreOrder))
+ return 0;
+ if (StoreOrder.empty()) {
+ StoreOrder.resize(Sinks.size());
+ std::iota(StoreOrder.begin(), StoreOrder.end(), 0);
+ } else {
+ SmallVector<int> InverseOrder;
+ inversePermutation(StoreOrder, InverseOrder);
+ StoreOrder.assign(InverseOrder.begin(), InverseOrder.end());
+ }
+ TTI::TargetCostKind CK = V.getCostKind();
+ InstructionCost ScalarConsumerCost = 0, ScalarStoreCost = 0;
+ for (auto [Consumer, SI] : zip(Consumers, Sinks)) {
+ ScalarConsumerCost += TTI->getInstructionCost(Consumer, CK);
+ ScalarStoreCost += TTI->getMemoryOpCost(
+ Instruction::Store, ScalarTy, SI->getAlign(),
+ SI->getPointerAddressSpace(), CK,
+ TTI::getOperandInfo(SI->getValueOperand()), SI);
+ }
+ Type *SourceScalarTy = Consumers.front()->getOperand(0)->getType();
+ if (any_of(Consumers, [SourceScalarTy](Instruction *I) {
+ return I->getOperand(0)->getType() != SourceScalarTy;
+ }))
+ return 0;
+ auto *LeafVecTy = FixedVectorType::get(SourceScalarTy, VL.size());
+ // These extracts are included in the isolated reduction cost but
+ // disappear when the consumer bundle directly reuses its vector.
+ InstructionCost ExtractCost = TTI->getScalarizationOverhead(
+ LeafVecTy, APInt::getAllOnes(VL.size()), /*Insert=*/false,
+ /*Extract=*/true, CK);
+ auto *SourceVecTy =
+ FixedVectorType::get(SourceScalarTy, Sinks.size());
+ auto *VecTy = FixedVectorType::get(ScalarTy, Sinks.size());
+ StoreInst *BaseSI = Sinks.front();
+ InstructionCost VectorCastCost = TTI->getCastInstrCost(
+ Consumers.front()->getOpcode(), VecTy, SourceVecTy,
+ TTI::CastContextHint::None, CK);
+ SmallVector<int> ResizeMask(Sinks.size(), PoisonMaskElem);
+ auto RootLane = find(StoreOrder, Sinks.size() - 1);
+ assert(RootLane != StoreOrder.end() && "Expected root consumer lane");
+ for (auto [Lane, OriginalLane] : enumerate(StoreOrder))
+ if (OriginalLane + 1 != Sinks.size())
+ ResizeMask[Lane] = OriginalLane;
+ InstructionCost ShuffleCost = getShuffleCost(
+ *TTI, TTI::SK_PermuteSingleSrc, SourceVecTy, CK, ResizeMask);
+ InstructionCost InsertCost = TTI->getVectorInstrCost(
+ Instruction::InsertElement, SourceVecTy, CK,
+ std::distance(StoreOrder.begin(), RootLane), ReductionRoot);
+ InstructionCost VectorStoreCost = TTI->getMemoryOpCost(
+ Instruction::Store, VecTy, BaseSI->getAlign(),
+ BaseSI->getPointerAddressSpace(), CK);
+ InstructionCost ScalarCost =
+ ScalarConsumerCost + ScalarStoreCost + ExtractCost;
+ InstructionCost VectorCost =
+ VectorCastCost + ShuffleCost + InsertCost + VectorStoreCost;
+ LLVM_DEBUG(dbgs()
+ << "SLP: downstream bundle costs: scalar-users="
+ << ScalarConsumerCost << ", scalar-stores="
+ << ScalarStoreCost << ", extracts=" << ExtractCost
+ << ", vector-user=" << VectorCastCost
+ << ", shuffle=" << ShuffleCost << ", insert=" << InsertCost
+ << ", vector-store=" << VectorStoreCost << "\n");
+ return ScalarCost - VectorCost;
+ };
+ InstructionCost ConsumerBenefit =
+ V.allowsReductionNonPowerOf2Analysis()
+ ? GetConsumerBundleBenefit()
+ : InstructionCost(0);
+ bool UseCombinedCost =
+ ConsumerBenefit.isValid() &&
+ ConsumerBenefit > SLPCostThreshold &&
+ Cost - ConsumerBenefit < -SLPCostThreshold;
+ if (UseCombinedCost)
+ Cost -= ConsumerBenefit;
+ bool DisallowedNonPowerOf2 =
+ V.allowsReductionNonPowerOf2Analysis() && !VectorizeNonPowerOf2 &&
+ !has_single_bit(VL.size()) && !UseCombinedCost;
+ if (Cost >= -SLPCostThreshold || DisallowedNonPowerOf2) {
V.getORE()->emit([&]() {
return OptimizationRemarkMissed(SV_NAME, "HorSLPNotBeneficial",
ReducedValsToOps.at(VL[0]).front())
@@ -34584,6 +34746,34 @@ class HorizontalReduction {
static RecurKind getRdxKind(Value *V) {
return HorizontalReduction::getRdxKind(V);
}
+
+bool SLPVectorizerPass::isReductionStoreChainRoot(Instruction *I) const {
+ RecurKind Kind = getRdxKind(I);
+ if (!I->hasOneUse() || Kind == RecurKind::None)
+ return false;
+ auto *Cast = dyn_cast<CastInst>(*I->user_begin());
+ if (!Cast || !Cast->hasOneUse() || !isa<StoreInst>(*Cast->user_begin()))
+ return false;
+
+ unsigned NumLeaves = 0;
+ SmallVector<Instruction *> Worklist(1, I);
+ SmallPtrSet<Instruction *, 8> Visited;
+ while (!Worklist.empty()) {
+ Instruction *Op = Worklist.pop_back_val();
+ if (!Visited.insert(Op).second)
+ continue;
+ for (Value *V : Op->operand_values()) {
+ if (isa<Function>(V))
+ continue;
+ if (auto *Child = dyn_cast<Instruction>(V);
+ Child && getRdxKind(Child) == Kind)
+ Worklist.push_back(Child);
+ else
+ ++NumLeaves;
+ }
+ }
+ return NumLeaves >= 3 && !has_single_bit(NumLeaves);
+}
static std::optional<unsigned> getAggregateSize(Instruction *InsertInst) {
if (auto *IE = dyn_cast<InsertElementInst>(InsertInst))
return cast<FixedVectorType>(IE->getType())->getNumElements();
@@ -35029,9 +35219,17 @@ bool SLPVectorizerPass::tryToVectorize(
bool SLPVectorizerPass::vectorizeRootInstruction(
PHINode *P, Instruction *Root, BasicBlock *BB, BoUpSLP &R,
- SmallSetVector<Instruction *, 8> &FMACandidates) {
+ SmallSetVector<Instruction *, 8> &FMACandidates,
+ bool AllowNonPowerOf2Analysis) {
SmallVector<WeakTrackingVH> PostponedInsts;
- bool Res = vectorizeHorReduction(P, Root, BB, R, PostponedInsts);
+ bool Res;
+ if (AllowNonPowerOf2Analysis) {
+ R.setAllowReductionNonPowerOf2Analysis(true);
+ scope_exit ResetAnalysisOverride(
+ [&R] { R.setAllowReductionNonPowerOf2Analysis(false); });
+ Res = vectorizeHorReduction(P, Root, BB, R, PostponedInsts);
+ } else
+ Res = vectorizeHorReduction(P, Root, BB, R, PostponedInsts);
Res |= tryToVectorize(PostponedInsts, R, FMACandidates);
return Res;
}
diff --git a/llvm/test/Transforms/SLPVectorizer/X86/reduction-external-store-bundle.ll b/llvm/test/Transforms/SLPVectorizer/X86/reduction-external-store-bundle.ll
new file mode 100644
index 0000000000000..4b16a54dc2ef2
--- /dev/null
+++ b/llvm/test/Transforms/SLPVectorizer/X86/reduction-external-store-bundle.ll
@@ -0,0 +1,70 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --version 5
+; RUN: opt -passes=slp-vectorizer -mtriple=x86_64 -mcpu=haswell -S %s | FileCheck %s
+
+declare i32 @llvm.umin.i32(i32, i32)
+declare i32 @llvm.umax.i32(i32, i32)
+
+define void @three_values_and_reduction(ptr %p, i32 %scale) {
+; CHECK-LABEL: define void @three_values_and_reduction(
+; CHECK: %[[LOAD:.*]] = load <3 x i8>, ptr %p
+; CHECK: %[[MIN:.*]] = call <3 x i32> @llvm.umin.v3i32
+; CHECK: %[[MAX:.*]] = call i32 @llvm.vector.reduce.umax.v3i32(<3 x i32> %[[MIN]])
+; CHECK: %[[EXT:.*]] = shufflevector <3 x i32> %[[MIN]], <3 x i32> poison, <4 x i32> <i32 0, i32 1, i32 2, i32 poison>
+; CHECK: %[[INS:.*]] = insertelement <4 x i32> %[[EXT]], i32 %[[MAX]], i64 3
+; CHECK: %[[TRUNC:.*]] = trunc <4 x i32> %[[INS]] to <4 x i8>
+; CHECK: store <4 x i8> %[[TRUNC]], ptr %p
+entry:
+ %l0 = load i8, ptr %p
+ %z0 = zext i8 %l0 to i32
+ %m0 = mul i32 %z0, %scale
+ %s0 = lshr i32 %m0, 15
+ %n0 = call i32 @llvm.umin.i32(i32 %s0, i32 255)
+ %t0 = trunc i32 %n0 to i8
+ store i8 %t0, ptr %p
+ %p1 = getelementptr i8, ptr %p, i64 1
+ %l1 = load i8, ptr %p1
+ %z1 = zext i8 %l1 to i32
+ %m1 = mul i32 %z1, %scale
+ %s1 = lshr i32 %m1, 15
+ %n1 = call i32 @llvm.umin.i32(i32 %s1, i32 255)
+ %t1 = trunc i32 %n1 to i8
+ store i8 %t1, ptr %p1
+ %p2 = getelementptr i8, ptr %p, i64 2
+ %l2 = load i8, ptr %p2
+ %z2 = zext i8 %l2 to i32
+ %m2 = mul i32 %z2, %scale
+ %s2 = lshr i32 %m2, 15
+ %n2 = call i32 @llvm.umin.i32(i32 %s2, i32 255)
+ %t2 = trunc i32 %n2 to i8
+ store i8 %t2, ptr %p2
+ %max1 = call i32 @llvm.umax.i32(i32 %n1, i32 %n2)
+ %max0 = call i32 @llvm.umax.i32(i32 %n0, i32 %max1)
+ %ta = trunc i32 %max0 to i8
+ %p3 = getelementptr i8, ptr %p, i64 3
+ store i8 %ta, ptr %p3
+ ret void
+}
+
+; A gap in the sink bundle prevents combined profitability from being used.
+define void @nonconsecutive_sinks(ptr %p, i32 %x0, i32 %x1, i32 %x2) {
+; CHECK-LABEL: define void @nonconsecutive_sinks(
+; CHECK-NOT: call i32 @llvm.vector.reduce
+entry:
+ %p1 = getelementptr i8, ptr %p, i64 1
+ %p2 = getelementptr i8, ptr %p, i64 2
+ %p4 = getelementptr i8, ptr %p, i64 4
+ %n0 = call i32 @llvm.umin.i32(i32 %x0, i32 255)
+ %n1 = call i32 @llvm.umin.i32(i32 %x1, i32 255)
+ %n2 = call i32 @llvm.umin.i32(i32 %x2, i32 255)
+ %t0 = trunc i32 %n0 to i8
+ %t1 = trunc i32 %n1 to i8
+ %t2 = trunc i32 %n2 to i8
+ store i8 %t0, ptr %p
+ store i8 %t1, ptr %p1
+ store i8 %t2, ptr %p2
+ %max1 = call i32 @llvm.umax.i32(i32 %n1, i32 %n2)
+ %max0 = call i32 @llvm.umax.i32(i32 %n0, i32 %max1)
+ %ta = trunc i32 %max0 to i8
+ store i8 %ta, ptr %p4
+ ret void
+}
``````````
</details>
https://github.com/llvm/llvm-project/pull/227303
More information about the llvm-commits
mailing list