[llvm] [LV] Make partial-reduction naming consistent (NFC) (PR #222377)
Benjamin Maxwell via llvm-commits
llvm-commits at lists.llvm.org
Mon Oct 5 01:50:03 PDT 2026
https://github.com/MacDue updated https://github.com/llvm/llvm-project/pull/222377
>From 0812cfb5140c431035f20ceb3e6c4fdf6b2647db Mon Sep 17 00:00:00 2001
From: Benjamin Maxwell <benjamin.maxwell at arm.com>
Date: Wed, 9 Sep 2026 16:08:15 +0000
Subject: [PATCH 1/3] [LV] Make partial-reduction naming consistent (NFC)
The partial-reduction code used "Chain" to refer to two different
concepts:
1. A `VPPartialReductionChain`, which is a collection of recipes that
form a partial reduction.
2. A list of `VPPartialReductionChain` objects that forms a chained
reduction.
This patch renames `VPPartialReductionChain` to
`VPPartialReduction` and updates the terminology as follows: a `Link`
is a single `VPPartialReduction`, a `Chain` is a list of links, and
`Chains` refers to a collection of chains.
---
.../Transforms/Vectorize/VPlanTransforms.cpp | 117 +++++++++---------
1 file changed, 58 insertions(+), 59 deletions(-)
diff --git a/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp b/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp
index cc1c9a683e404..ed752c8c95279 100644
--- a/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp
+++ b/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp
@@ -4947,12 +4947,12 @@ struct ExtendedReductionOperand {
ReductionExtend ExtendA, ExtendB;
};
-/// A chain of recipes that form a partial reduction. Matches either
+/// A collection of recipes that form a partial reduction. Matches either
/// reduction_bin_op (extended op, accumulator), or
/// reduction_bin_op (accumulator, extended op).
/// The possible forms of the "extended op" are listed in
/// matchExtendedReductionOperand.
-struct VPPartialReductionChain {
+struct VPPartialReduction {
/// The top-level binary operation that forms the reduction to a scalar
/// after the loop body.
VPWidenRecipe *ReductionBinOp = nullptr;
@@ -5123,17 +5123,17 @@ createPartialReductionExpression(VPReductionRecipe *Red) {
llvm_unreachable("Unsupported expression");
}
-// Helper to transform a partial reduction chain into a partial reduction
-// recipe. Assumes profitability has been checked.
-static void transformToPartialReduction(const VPPartialReductionChain &Chain,
+// Helper to transform a VPPartialReduction into a partial reduction recipe.
+// Assumes profitability has been checked.
+static void transformToPartialReduction(const VPPartialReduction &Link,
VPlan &Plan,
VPReductionPHIRecipe *RdxPhi) {
- VPWidenRecipe *WidenRecipe = Chain.ReductionBinOp;
+ VPWidenRecipe *WidenRecipe = Link.ReductionBinOp;
assert(WidenRecipe->getNumOperands() == 2 && "Expected binary operation");
- VPValue *Accumulator = WidenRecipe->getOperand(Chain.AccumulatorOpIdx);
+ VPValue *Accumulator = WidenRecipe->getOperand(Link.AccumulatorOpIdx);
auto *ExtendedOp = cast<VPSingleDefRecipe>(
- WidenRecipe->getOperand(1 - Chain.AccumulatorOpIdx));
+ WidenRecipe->getOperand(1 - Link.AccumulatorOpIdx));
// FIXME: Do these transforms before invoking the cost-model.
ExtendedOp = optimizeExtendsForPartialReduction(ExtendedOp);
@@ -5153,9 +5153,9 @@ static void transformToPartialReduction(const VPPartialReductionChain &Chain,
// reduction is always positive (starting at '0') and to do a final
// subtract in the middle block.
if ((WidenRecipe->getOpcode() == Instruction::Sub &&
- Chain.RK != RecurKind::Sub) ||
+ Link.RK != RecurKind::Sub) ||
(WidenRecipe->getOpcode() == Instruction::FSub &&
- Chain.RK != RecurKind::FSub)) {
+ Link.RK != RecurKind::FSub)) {
VPBuilder Builder(WidenRecipe);
Type *ElemTy = ExtendedOp->getScalarType();
VPWidenRecipe *NegRecipe;
@@ -5179,20 +5179,20 @@ static void transformToPartialReduction(const VPPartialReductionChain &Chain,
// through the Select recipe introduced by tail-folding, otherwise look
// through any Blend recipe introduced by predication for the block.
VPValue *ExitSearch =
- Chain.Blend ? cast<VPValue>(Chain.Blend) : cast<VPValue>(WidenRecipe);
+ Link.Blend ? cast<VPValue>(Link.Blend) : cast<VPValue>(WidenRecipe);
VPValue *Cond = nullptr;
VPValue *ExitValue = cast_or_null<VPInstruction>(
findUserOf(ExitSearch, m_Select(m_VPValue(Cond), m_Specific(ExitSearch),
m_Specific(RdxPhi))));
- if (Chain.Blend) {
+ if (Link.Blend) {
std::optional<unsigned> BlendReductionIdx =
- getBlendReductionUpdateValueIdx(Chain.Blend);
+ getBlendReductionUpdateValueIdx(Link.Blend);
assert(BlendReductionIdx &&
- Chain.Blend->getIncomingValue(*BlendReductionIdx) == WidenRecipe &&
+ Link.Blend->getIncomingValue(*BlendReductionIdx) == WidenRecipe &&
"Expected blend to contain the reduction update");
- VPValue *BlendCond = Chain.Blend->getMask(*BlendReductionIdx);
+ VPValue *BlendCond = Link.Blend->getMask(*BlendReductionIdx);
Cond = ExitValue ? VPBuilder(WidenRecipe)
.createLogicalAnd(Cond, BlendCond,
WidenRecipe->getDebugLoc())
@@ -5207,7 +5207,7 @@ static void transformToPartialReduction(const VPPartialReductionChain &Chain,
bool IsLastInChain = RdxPhi->getBackedgeValue() == WidenRecipe ||
RdxPhi->getBackedgeValue() == ExitValue ||
- RdxPhi->getBackedgeValue() == Chain.Blend;
+ RdxPhi->getBackedgeValue() == Link.Blend;
assert((!ExitValue || IsLastInChain) &&
"if we found ExitValue, it must match RdxPhi's backedge value");
@@ -5219,13 +5219,13 @@ static void transformToPartialReduction(const VPPartialReductionChain &Chain,
RdxKind == RecurKind::FAdd ? WidenRecipe->getFastMathFlagsOrNone()
: FastMathFlags(),
WidenRecipe->getUnderlyingInstr(), Accumulator, ExtendedOp, Cond,
- RdxUnordered{/*VFScaleFactor=*/Chain.ScaleFactor});
+ RdxUnordered{/*VFScaleFactor=*/Link.ScaleFactor});
PartialRed->insertBefore(WidenRecipe);
if (ExitValue)
ExitValue->replaceAllUsesWith(PartialRed);
- if (Chain.Blend)
- Chain.Blend->replaceAllUsesWith(PartialRed);
+ if (Link.Blend)
+ Link.Blend->replaceAllUsesWith(PartialRed);
WidenRecipe->replaceAllUsesWith(PartialRed);
// For cost-model purposes, fold this into a VPExpression.
@@ -5240,17 +5240,17 @@ static void transformToPartialReduction(const VPPartialReductionChain &Chain,
// Scale the PHI and ReductionStartVector by the VFScaleFactor
assert(RdxPhi->getVFScaleFactor() == 1 && "scale factor must not be set");
- RdxPhi->setVFScaleFactor(Chain.ScaleFactor);
+ RdxPhi->setVFScaleFactor(Link.ScaleFactor);
auto *StartInst = cast<VPInstruction>(RdxPhi->getStartValue());
assert(StartInst->getOpcode() == VPInstruction::ReductionStartVector);
- auto *NewScaleFactor = Plan.getConstantInt(32, Chain.ScaleFactor);
+ auto *NewScaleFactor = Plan.getConstantInt(32, Link.ScaleFactor);
StartInst->setOperand(2, NewScaleFactor);
// If this is the last value in a sub-reduction chain, then update the PHI
// node to start at `0` and update the reduction-result to subtract from
// the PHI's start value.
- if (Chain.RK != RecurKind::Sub && Chain.RK != RecurKind::FSub)
+ if (Link.RK != RecurKind::Sub && Link.RK != RecurKind::FSub)
return;
VPValue *OldStartValue = StartInst->getOperand(0);
@@ -5261,8 +5261,8 @@ static void transformToPartialReduction(const VPPartialReductionChain &Chain,
assert(RdxResult && "Could not find reduction result");
VPBuilder Builder = VPBuilder::getToInsertAfter(RdxResult);
- unsigned SubOpc = Chain.RK == RecurKind::FSub ? Instruction::BinaryOps::FSub
- : Instruction::BinaryOps::Sub;
+ unsigned SubOpc = Link.RK == RecurKind::FSub ? Instruction::BinaryOps::FSub
+ : Instruction::BinaryOps::Sub;
VPInstruction *NewResult = Builder.createNaryOp(
SubOpc, {OldStartValue, RdxResult}, VPIRFlags::getDefaultFlags(SubOpc),
RdxPhi->getDebugLoc());
@@ -5274,8 +5274,7 @@ static void transformToPartialReduction(const VPPartialReductionChain &Chain,
/// Returns the cost of a link in a partial-reduction chain for a given VF.
static InstructionCost
getPartialReductionLinkCost(VPCostContext &CostCtx,
- const VPPartialReductionChain &Link,
- ElementCount VF) {
+ const VPPartialReduction &Link, ElementCount VF) {
Type *RdxType = Link.ReductionBinOp->getScalarType();
const ExtendedReductionOperand &ExtendedOp = Link.ExtendedOp;
std::optional<unsigned> BinOpc = std::nullopt;
@@ -5426,8 +5425,8 @@ matchExtendedReductionOperand(VPWidenRecipe *UpdateR, VPValue *Op) {
/// and determines if the target can use a cheaper operation with a wider
/// per-iteration input VF and narrower PHI VF. If successful, returns the chain
/// of operations in the reduction.
-static std::optional<SmallVector<VPPartialReductionChain>>
-getScaledReductions(VPReductionPHIRecipe *RedPhiR) {
+static std::optional<SmallVector<VPPartialReduction>>
+getScaledReductionChain(VPReductionPHIRecipe *RedPhiR) {
// Get the backedge value from the reduction PHI and find the
// ComputeReductionResult that uses it (directly or through a select for
// predicated reductions).
@@ -5437,7 +5436,7 @@ getScaledReductions(VPReductionPHIRecipe *RedPhiR) {
VPValue *ExitValue = RdxResult->getOperand(0);
match(ExitValue, m_Select(m_VPValue(), m_VPValue(ExitValue), m_VPValue()));
- SmallVector<VPPartialReductionChain> Chain;
+ SmallVector<VPPartialReduction> Chain;
RecurKind RK = RedPhiR->getRecurrenceKind();
Type *PhiType = RedPhiR->getScalarType();
TypeSize PHISize = PhiType->getPrimitiveSizeInBits();
@@ -5488,7 +5487,7 @@ getScaledReductions(VPReductionPHIRecipe *RedPhiR) {
if (!PHISize.hasKnownScalarFactor(ExtSrcSize))
return std::nullopt;
- VPPartialReductionChain Link(
+ VPPartialReduction Link(
{UpdateR, *ExtendedOp, RK,
PrevValue == UpdateR->getOperand(0) ? 0U : 1U,
static_cast<unsigned>(PHISize.getKnownScalarFactor(ExtSrcSize)),
@@ -5498,7 +5497,7 @@ getScaledReductions(VPReductionPHIRecipe *RedPhiR) {
}
// The chain links were collected by traversing backwards from the exit value.
- // Reverse the chains so they are in program order.
+ // Reverse the chain so the links follow program order.
std::reverse(Chain.begin(), Chain.end());
return Chain;
}
@@ -5510,14 +5509,14 @@ void VPlanTransforms::createPartialReductions(VPlan &Plan,
// Find all possible valid partial reductions, grouping chains by their PHI.
// This grouping allows invalidating the whole chain, if any link is not a
// valid partial reduction.
- MapVector<VPReductionPHIRecipe *, SmallVector<VPPartialReductionChain>>
+ MapVector<VPReductionPHIRecipe *, SmallVector<VPPartialReduction>>
ChainsByPhi;
VPBasicBlock *HeaderVPBB = Plan.getVectorLoopRegion()->getEntryBasicBlock();
SmallVector<VPReductionPHIRecipe *, 4> UnorderedReductions;
for (VPReductionPHIRecipe &RedPhiR :
make_isa_range<VPReductionPHIRecipe>(HeaderVPBB->phis())) {
- if (auto Chains = getScaledReductions(&RedPhiR))
- ChainsByPhi.try_emplace(&RedPhiR, std::move(*Chains));
+ if (auto Chain = getScaledReductionChain(&RedPhiR))
+ ChainsByPhi.try_emplace(&RedPhiR, std::move(*Chain));
else if (UsePartialReductionsByDefault &&
(RedPhiR.getRecurrenceKind() == RecurKind::Add ||
(RedPhiR.getRecurrenceKind() == RecurKind::FAdd &&
@@ -5574,12 +5573,12 @@ void VPlanTransforms::createPartialReductions(VPlan &Plan,
SmallPtrSet<VPRecipeBase *, 4> PartialReductionOps;
SmallPtrSet<VPBlendRecipe *, 4> PartialReductionBlends;
DenseMap<VPSingleDefRecipe *, unsigned> ScaledReductionMap;
- for (const auto &[_, Chains] : ChainsByPhi)
- for (const VPPartialReductionChain &Chain : Chains) {
- PartialReductionOps.insert(Chain.ExtendedOp.ExtendsUser);
- if (Chain.Blend)
- PartialReductionBlends.insert(Chain.Blend);
- ScaledReductionMap[Chain.ReductionBinOp] = Chain.ScaleFactor;
+ for (const auto &[_, Chain] : ChainsByPhi)
+ for (const VPPartialReduction &Link : Chain) {
+ PartialReductionOps.insert(Link.ExtendedOp.ExtendsUser);
+ if (Link.Blend)
+ PartialReductionBlends.insert(Link.Blend);
+ ScaledReductionMap[Link.ReductionBinOp] = Link.ScaleFactor;
}
// A partial reduction is invalid if any of its extends are used by
@@ -5592,13 +5591,13 @@ void VPlanTransforms::createPartialReductions(VPlan &Plan,
};
auto IsProfitablePartialReductionChainForVF =
- [&](ArrayRef<VPPartialReductionChain> Chain, ElementCount VF) -> bool {
+ [&](ArrayRef<VPPartialReduction> Chain, ElementCount VF) -> bool {
InstructionCost PartialCost = 0, RegularCost = 0;
// The chain is a profitable partial reduction chain if the cost of handling
// the entire chain is cheaper when using partial reductions than when
// handling the entire chain using regular reductions.
- for (const VPPartialReductionChain &Link : Chain) {
+ for (const VPPartialReduction &Link : Chain) {
const ExtendedReductionOperand &ExtendedOp = Link.ExtendedOp;
InstructionCost LinkCost = getPartialReductionLinkCost(CostCtx, Link, VF);
if (!LinkCost.isValid())
@@ -5621,10 +5620,10 @@ void VPlanTransforms::createPartialReductions(VPlan &Plan,
// matching scale factors, are outside the loop region or the select
// introduced by tail-folding. Otherwise we would create users of scaled
// reductions where the types of the other operands don't match.
- for (auto &[RedPhiR, Chains] : ChainsByPhi) {
- for (const VPPartialReductionChain &Chain : Chains) {
- if (!all_of(Chain.ExtendedOp.ExtendsUser->operands(), ExtendUsersValid)) {
- Chains.clear();
+ for (auto &[RedPhiR, Chain] : ChainsByPhi) {
+ for (const VPPartialReduction &Link : Chain) {
+ if (!all_of(Link.ExtendedOp.ExtendsUser->operands(), ExtendUsersValid)) {
+ Chain.clear();
break;
}
auto UseIsValid = [&, RedPhiR = RedPhiR](VPUser *U) {
@@ -5633,16 +5632,16 @@ void VPlanTransforms::createPartialReductions(VPlan &Plan,
auto *R = cast<VPSingleDefRecipe>(U);
if (auto *Blend = dyn_cast<VPBlendRecipe>(R))
- return Blend == Chain.Blend || PartialReductionBlends.contains(Blend);
+ return Blend == Link.Blend || PartialReductionBlends.contains(Blend);
- return Chain.ScaleFactor == ScaledReductionMap.lookup_or(R, 0) ||
+ return Link.ScaleFactor == ScaledReductionMap.lookup_or(R, 0) ||
match(R, m_ComputeReductionResult(
- m_Specific(Chain.ReductionBinOp))) ||
- match(R, m_Select(m_VPValue(), m_Specific(Chain.ReductionBinOp),
+ m_Specific(Link.ReductionBinOp))) ||
+ match(R, m_Select(m_VPValue(), m_Specific(Link.ReductionBinOp),
m_Specific(RedPhiR)));
};
- if (!all_of(Chain.ReductionBinOp->users(), UseIsValid)) {
- Chains.clear();
+ if (!all_of(Link.ReductionBinOp->users(), UseIsValid)) {
+ Chain.clear();
break;
}
@@ -5653,7 +5652,7 @@ void VPlanTransforms::createPartialReductions(VPlan &Plan,
auto *RepR = dyn_cast<VPReplicateRecipe>(U);
return RepR && RepR->getOpcode() == Instruction::Store;
})) {
- Chains.clear();
+ Chain.clear();
break;
}
}
@@ -5661,16 +5660,16 @@ void VPlanTransforms::createPartialReductions(VPlan &Plan,
// Clear the chain if it is not profitable.
if (!LoopVectorizationPlanner::getDecisionAndClampRange(
- [&, &Chains = Chains](ElementCount VF) {
- return IsProfitablePartialReductionChainForVF(Chains, VF);
+ [&, &Chain = Chain](ElementCount VF) {
+ return IsProfitablePartialReductionChainForVF(Chain, VF);
},
Range))
- Chains.clear();
+ Chain.clear();
}
- for (auto &[Phi, Chains] : ChainsByPhi)
- for (const VPPartialReductionChain &Chain : Chains)
- transformToPartialReduction(Chain, Plan, Phi);
+ for (auto &[Phi, Chain] : ChainsByPhi)
+ for (const VPPartialReduction &Link : Chain)
+ transformToPartialReduction(Link, Plan, Phi);
}
void VPlanTransforms::makeMemOpWideningDecisions(VPlan &Plan, VFRange &Range,
>From d3da633798fd4059a426407f4d44fae4a109d34d Mon Sep 17 00:00:00 2001
From: Benjamin Maxwell <benjamin.maxwell at arm.com>
Date: Fri, 2 Oct 2026 15:19:23 +0000
Subject: [PATCH 2/3] Fixups
---
.../Transforms/Vectorize/VPlanTransforms.cpp | 46 ++++++++++---------
1 file changed, 24 insertions(+), 22 deletions(-)
diff --git a/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp b/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp
index ed752c8c95279..4312ce7b0be39 100644
--- a/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp
+++ b/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp
@@ -4947,12 +4947,12 @@ struct ExtendedReductionOperand {
ReductionExtend ExtendA, ExtendB;
};
-/// A collection of recipes that form a partial reduction. Matches either
+/// A collection of recipes that describe a partial reduction. Matches either
/// reduction_bin_op (extended op, accumulator), or
/// reduction_bin_op (accumulator, extended op).
/// The possible forms of the "extended op" are listed in
/// matchExtendedReductionOperand.
-struct VPPartialReduction {
+struct PartialReductionDescriptor {
/// The top-level binary operation that forms the reduction to a scalar
/// after the loop body.
VPWidenRecipe *ReductionBinOp = nullptr;
@@ -5123,9 +5123,9 @@ createPartialReductionExpression(VPReductionRecipe *Red) {
llvm_unreachable("Unsupported expression");
}
-// Helper to transform a VPPartialReduction into a partial reduction recipe.
-// Assumes profitability has been checked.
-static void transformToPartialReduction(const VPPartialReduction &Link,
+// Helper to transform a PartialReductionDescriptor into a partial reduction
+// recipe. Assumes profitability has been checked.
+static void transformToPartialReduction(const PartialReductionDescriptor &Link,
VPlan &Plan,
VPReductionPHIRecipe *RdxPhi) {
VPWidenRecipe *WidenRecipe = Link.ReductionBinOp;
@@ -5274,7 +5274,8 @@ static void transformToPartialReduction(const VPPartialReduction &Link,
/// Returns the cost of a link in a partial-reduction chain for a given VF.
static InstructionCost
getPartialReductionLinkCost(VPCostContext &CostCtx,
- const VPPartialReduction &Link, ElementCount VF) {
+ const PartialReductionDescriptor &Link,
+ ElementCount VF) {
Type *RdxType = Link.ReductionBinOp->getScalarType();
const ExtendedReductionOperand &ExtendedOp = Link.ExtendedOp;
std::optional<unsigned> BinOpc = std::nullopt;
@@ -5424,8 +5425,8 @@ matchExtendedReductionOperand(VPWidenRecipe *UpdateR, VPValue *Op) {
/// Examines each operation in the reduction chain corresponding to \p RedPhiR,
/// and determines if the target can use a cheaper operation with a wider
/// per-iteration input VF and narrower PHI VF. If successful, returns the chain
-/// of operations in the reduction.
-static std::optional<SmallVector<VPPartialReduction>>
+/// of partial reduction descriptors (that are links in the reduction chain).
+static std::optional<SmallVector<PartialReductionDescriptor>>
getScaledReductionChain(VPReductionPHIRecipe *RedPhiR) {
// Get the backedge value from the reduction PHI and find the
// ComputeReductionResult that uses it (directly or through a select for
@@ -5436,7 +5437,7 @@ getScaledReductionChain(VPReductionPHIRecipe *RedPhiR) {
VPValue *ExitValue = RdxResult->getOperand(0);
match(ExitValue, m_Select(m_VPValue(), m_VPValue(ExitValue), m_VPValue()));
- SmallVector<VPPartialReduction> Chain;
+ SmallVector<PartialReductionDescriptor> Chain;
RecurKind RK = RedPhiR->getRecurrenceKind();
Type *PhiType = RedPhiR->getScalarType();
TypeSize PHISize = PhiType->getPrimitiveSizeInBits();
@@ -5487,7 +5488,7 @@ getScaledReductionChain(VPReductionPHIRecipe *RedPhiR) {
if (!PHISize.hasKnownScalarFactor(ExtSrcSize))
return std::nullopt;
- VPPartialReduction Link(
+ PartialReductionDescriptor Link(
{UpdateR, *ExtendedOp, RK,
PrevValue == UpdateR->getOperand(0) ? 0U : 1U,
static_cast<unsigned>(PHISize.getKnownScalarFactor(ExtSrcSize)),
@@ -5509,14 +5510,15 @@ void VPlanTransforms::createPartialReductions(VPlan &Plan,
// Find all possible valid partial reductions, grouping chains by their PHI.
// This grouping allows invalidating the whole chain, if any link is not a
// valid partial reduction.
- MapVector<VPReductionPHIRecipe *, SmallVector<VPPartialReduction>>
- ChainsByPhi;
+ MapVector<VPReductionPHIRecipe *, SmallVector<PartialReductionDescriptor>>
+ PhiToChain;
VPBasicBlock *HeaderVPBB = Plan.getVectorLoopRegion()->getEntryBasicBlock();
SmallVector<VPReductionPHIRecipe *, 4> UnorderedReductions;
+ SmallVector<VPReductionPHIRecipe *, 4> UnorderedReductions;
for (VPReductionPHIRecipe &RedPhiR :
make_isa_range<VPReductionPHIRecipe>(HeaderVPBB->phis())) {
if (auto Chain = getScaledReductionChain(&RedPhiR))
- ChainsByPhi.try_emplace(&RedPhiR, std::move(*Chain));
+ PhiToChain.try_emplace(&RedPhiR, std::move(*Chain));
else if (UsePartialReductionsByDefault &&
(RedPhiR.getRecurrenceKind() == RecurKind::Add ||
(RedPhiR.getRecurrenceKind() == RecurKind::FAdd &&
@@ -5565,7 +5567,7 @@ void VPlanTransforms::createPartialReductions(VPlan &Plan,
Backedge->eraseFromParent();
}
- if (ChainsByPhi.empty())
+ if (PhiToChain.empty())
return;
// Build set of partial reduction operations and blends for user validation
@@ -5573,8 +5575,8 @@ void VPlanTransforms::createPartialReductions(VPlan &Plan,
SmallPtrSet<VPRecipeBase *, 4> PartialReductionOps;
SmallPtrSet<VPBlendRecipe *, 4> PartialReductionBlends;
DenseMap<VPSingleDefRecipe *, unsigned> ScaledReductionMap;
- for (const auto &[_, Chain] : ChainsByPhi)
- for (const VPPartialReduction &Link : Chain) {
+ for (auto &[_, Chain] : PhiToChain)
+ for (const PartialReductionDescriptor &Link : Chain) {
PartialReductionOps.insert(Link.ExtendedOp.ExtendsUser);
if (Link.Blend)
PartialReductionBlends.insert(Link.Blend);
@@ -5591,13 +5593,13 @@ void VPlanTransforms::createPartialReductions(VPlan &Plan,
};
auto IsProfitablePartialReductionChainForVF =
- [&](ArrayRef<VPPartialReduction> Chain, ElementCount VF) -> bool {
+ [&](ArrayRef<PartialReductionDescriptor> Chain, ElementCount VF) -> bool {
InstructionCost PartialCost = 0, RegularCost = 0;
// The chain is a profitable partial reduction chain if the cost of handling
// the entire chain is cheaper when using partial reductions than when
// handling the entire chain using regular reductions.
- for (const VPPartialReduction &Link : Chain) {
+ for (const PartialReductionDescriptor &Link : Chain) {
const ExtendedReductionOperand &ExtendedOp = Link.ExtendedOp;
InstructionCost LinkCost = getPartialReductionLinkCost(CostCtx, Link, VF);
if (!LinkCost.isValid())
@@ -5620,8 +5622,8 @@ void VPlanTransforms::createPartialReductions(VPlan &Plan,
// matching scale factors, are outside the loop region or the select
// introduced by tail-folding. Otherwise we would create users of scaled
// reductions where the types of the other operands don't match.
- for (auto &[RedPhiR, Chain] : ChainsByPhi) {
- for (const VPPartialReduction &Link : Chain) {
+ for (auto &[RedPhiR, Chain] : PhiToChain) {
+ for (const PartialReductionDescriptor &Link : Chain) {
if (!all_of(Link.ExtendedOp.ExtendsUser->operands(), ExtendUsersValid)) {
Chain.clear();
break;
@@ -5667,8 +5669,8 @@ void VPlanTransforms::createPartialReductions(VPlan &Plan,
Chain.clear();
}
- for (auto &[Phi, Chain] : ChainsByPhi)
- for (const VPPartialReduction &Link : Chain)
+ for (auto &[Phi, Chain] : PhiToChain)
+ for (const PartialReductionDescriptor &Link : Chain)
transformToPartialReduction(Link, Plan, Phi);
}
>From 553ce2ca189122ed44c2c7ddbbce9cbc58fe4b48 Mon Sep 17 00:00:00 2001
From: Benjamin Maxwell <macdue at dueutil.tech>
Date: Fri, 2 Oct 2026 19:31:10 +0100
Subject: [PATCH 3/3] Update VPlanTransforms.cpp
---
llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp | 1 -
1 file changed, 1 deletion(-)
diff --git a/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp b/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp
index 4312ce7b0be39..4fb0707606d3c 100644
--- a/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp
+++ b/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp
@@ -5514,7 +5514,6 @@ void VPlanTransforms::createPartialReductions(VPlan &Plan,
PhiToChain;
VPBasicBlock *HeaderVPBB = Plan.getVectorLoopRegion()->getEntryBasicBlock();
SmallVector<VPReductionPHIRecipe *, 4> UnorderedReductions;
- SmallVector<VPReductionPHIRecipe *, 4> UnorderedReductions;
for (VPReductionPHIRecipe &RedPhiR :
make_isa_range<VPReductionPHIRecipe>(HeaderVPBB->phis())) {
if (auto Chain = getScaledReductionChain(&RedPhiR))
More information about the llvm-commits
mailing list