[llvm] [LV] NFCI: Create VPExpressions in transformToPartialReductions. (PR #182863)
Sander de Smalen via llvm-commits
llvm-commits at lists.llvm.org
Thu Apr 16 06:07:53 PDT 2026
https://github.com/sdesmalen-arm updated https://github.com/llvm/llvm-project/pull/182863
>From 2cb3d69c0cb6d57130d074afc6f2b57a49de9bb4 Mon Sep 17 00:00:00 2001
From: Sander de Smalen <sander.desmalen at arm.com>
Date: Mon, 23 Feb 2026 09:41:16 +0000
Subject: [PATCH 1/4] [LV] NFCI: Create VPExpressions in
transformToPartialReductions.
With this change, all logic to generate partial reductions and
recognising them as VPExpressions is contained in
`transformToPartialReductions`, without the need for a second
transform pass.
The PR intends to be a non-functional change.
---
.../Transforms/Vectorize/VPlanTransforms.cpp | 75 ++++++++++++++-----
1 file changed, 57 insertions(+), 18 deletions(-)
diff --git a/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp b/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp
index d1ff014907dc9..cde087175c9e4 100644
--- a/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp
+++ b/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp
@@ -4415,11 +4415,8 @@ tryToMatchAndCreateExtendedReduction(VPReductionRecipe *Red, VPCostContext &Ctx,
Type *RedTy = Ctx.Types.inferScalarType(Red);
VPValue *VecOp = Red->getVecOp();
- // For partial reductions, the decision has already been made at the point of
- // transforming reductions -> partial reductions for a given plan, based on
- // the cost-model.
- if (Red->isPartialReduction())
- return new VPExpressionRecipe(cast<VPWidenCastRecipe>(VecOp), Red);
+ assert(!Red->isPartialReduction() &&
+ "This path no longer supports partial reductions");
// Clamp the range if using extended-reduction is profitable.
auto IsExtendedRedValidAndClampRange =
@@ -4476,6 +4473,8 @@ tryToMatchAndCreateMulAccumulateReduction(VPReductionRecipe *Red,
Opcode != Instruction::FAdd)
return nullptr;
+ assert(!Red->isPartialReduction() &&
+ "This path no longer supports partial reductions");
Type *RedTy = Ctx.Types.inferScalarType(Red);
// Clamp the range if using multiply-accumulate-reduction is profitable.
@@ -4484,19 +4483,13 @@ tryToMatchAndCreateMulAccumulateReduction(VPReductionRecipe *Red,
VPWidenCastRecipe *OuterExt) -> bool {
return LoopVectorizationPlanner::getDecisionAndClampRange(
[&](ElementCount VF) {
- // For partial reductions, the decision has already been made at the
- // point of transforming reductions -> partial reductions for a given
- // plan, based on the cost-model.
- if (Red->isPartialReduction())
- return true;
-
TTI::TargetCostKind CostKind = TTI::TCK_RecipThroughput;
Type *SrcTy =
Ext0 ? Ctx.Types.inferScalarType(Ext0->getOperand(0)) : RedTy;
InstructionCost MulAccCost;
- // Only partial reductions support mixed or floating-point extends at
- // the moment.
+ // getMulAccReductionCost for in-loop reductions does not support
+ // mixed or floating-point extends.
if (Ext0 && Ext1 &&
(Ext0->getOpcode() != Ext1->getOpcode() ||
Ext0->getOpcode() == Instruction::CastOps::FPExt))
@@ -4560,11 +4553,10 @@ tryToMatchAndCreateMulAccumulateReduction(VPReductionRecipe *Red,
// creates two uniform extends that can more easily be matched by the rest of
// the bundling code. The ExtB reference, ValB and operand 1 of Mul are all
// replaced with the new extend of the constant.
- auto ExtendAndReplaceConstantOp = [&Ctx, &Red](VPWidenCastRecipe *ExtA,
- VPWidenCastRecipe *&ExtB,
- VPValue *&ValB,
- VPWidenRecipe *Mul) {
- if (!ExtA || ExtB || !isa<VPIRValue>(ValB) || Red->isPartialReduction())
+ auto ExtendAndReplaceConstantOp = [&Ctx](VPWidenCastRecipe *ExtA,
+ VPWidenCastRecipe *&ExtB,
+ VPValue *&ValB, VPWidenRecipe *Mul) {
+ if (!ExtA || ExtB || !isa<VPIRValue>(ValB))
return;
Type *NarrowTy = Ctx.Types.inferScalarType(ExtA->getOperand(0));
Instruction::CastOps ExtOpc = ExtA->getOpcode();
@@ -4661,6 +4653,11 @@ tryToMatchAndCreateMulAccumulateReduction(VPReductionRecipe *Red,
static void tryToCreateAbstractReductionRecipe(VPReductionRecipe *Red,
VPCostContext &Ctx,
VFRange &Range) {
+ // Creation of VPExpressions for partial reductions is entirely handled in
+ // `transformToPartialReduction`.
+ if (Red->isPartialReduction())
+ return;
+
VPExpressionRecipe *AbstractR = nullptr;
auto IP = std::next(Red->getIterator());
auto *VPBB = Red->getParent();
@@ -6098,6 +6095,42 @@ optimizeExtendsForPartialReduction(VPSingleDefRecipe *BinOp,
return BinOp;
}
+static VPExpressionRecipe *
+createPartialReductionExpression(VPReductionRecipe *Red) {
+ VPValue *VecOp = Red->getVecOp();
+
+ // reduce.[f]add(ext(op))
+ // -> VPExpressionRecipe(op, red)
+ if (isa<VPWidenCastRecipe>(VecOp) &&
+ (match(VecOp, m_ZExtOrSExt(m_VPValue())) ||
+ match(VecOp, m_FPExt(m_VPValue()))))
+ return new VPExpressionRecipe(cast<VPWidenCastRecipe>(VecOp), Red);
+
+ // reduce.[f]add([f]mul(ext(a), ext(b)))
+ // -> VPExpressionRecipe(a, b, mul, red)
+ if (match(VecOp, m_FMul(m_FPExt(m_VPValue()), m_FPExt(m_VPValue()))) ||
+ match(VecOp,
+ m_Mul(m_ZExtOrSExt(m_VPValue()), m_ZExtOrSExt(m_VPValue())))) {
+ auto *Mul = cast<VPWidenRecipe>(VecOp);
+ auto *ExtA = cast<VPWidenCastRecipe>(Mul->getOperand(0));
+ auto *ExtB = cast<VPWidenCastRecipe>(Mul->getOperand(1));
+ return new VPExpressionRecipe(ExtA, ExtB, Mul, Red);
+ }
+
+ // reduce.add(neg(mul(ext(a), ext(b))))
+ // -> VPExpressionRecipe(a, b, mul, sub, red)
+ if (match(VecOp, m_Sub(m_ZeroInt(), m_Mul(m_ZExtOrSExt(m_VPValue()),
+ m_ZExtOrSExt(m_VPValue()))))) {
+ auto *Sub = cast<VPWidenRecipe>(VecOp);
+ auto *Mul = cast<VPWidenRecipe>(Sub->getOperand(1));
+ auto *ExtA = cast<VPWidenCastRecipe>(Mul->getOperand(0));
+ auto *ExtB = cast<VPWidenCastRecipe>(Mul->getOperand(1));
+ return new VPExpressionRecipe(ExtA, ExtB, Mul, Sub, Red);
+ }
+
+ return nullptr;
+}
+
// Helper to transform a partial reduction chain into a partial reduction
// recipe. Assumes profitability has been checked.
static void transformToPartialReduction(const VPPartialReductionChain &Chain,
@@ -6170,6 +6203,12 @@ static void transformToPartialReduction(const VPPartialReductionChain &Chain,
ExitValue->replaceAllUsesWith(PartialRed);
WidenRecipe->replaceAllUsesWith(PartialRed);
+ // For cost-model purposes, see if we can fold this into a VPExpression.
+ if (VPExpressionRecipe *E = createPartialReductionExpression(PartialRed)) {
+ E->insertBefore(WidenRecipe);
+ PartialRed->replaceAllUsesWith(E);
+ }
+
// We only need to update the PHI node once, which is when we find the
// last reduction in the chain.
if (!IsLastInChain)
>From 2d62a2e272bf38dc9637ab5b2996e20e4619ff11 Mon Sep 17 00:00:00 2001
From: Sander de Smalen <sander.desmalen at arm.com>
Date: Fri, 10 Apr 2026 08:05:48 +0000
Subject: [PATCH 2/4] Address review comments
---
llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp | 9 ++++-----
1 file changed, 4 insertions(+), 5 deletions(-)
diff --git a/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp b/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp
index cde087175c9e4..b36fc5126d2f7 100644
--- a/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp
+++ b/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp
@@ -4606,8 +4606,7 @@ tryToMatchAndCreateMulAccumulateReduction(VPReductionRecipe *Red,
return nullptr;
// Match reduce.add(ext(mul(A, B))).
- if (!Red->isPartialReduction() &&
- match(VecOp, m_ZExtOrSExt(m_Mul(m_VPValue(A), m_VPValue(B))))) {
+ if (match(VecOp, m_ZExtOrSExt(m_Mul(m_VPValue(A), m_VPValue(B))))) {
auto *Ext = cast<VPWidenCastRecipe>(VecOp);
auto *Mul = cast<VPWidenRecipe>(Ext->getOperand(0));
auto *Ext0 = dyn_cast_if_present<VPWidenCastRecipe>(A);
@@ -4654,9 +4653,9 @@ static void tryToCreateAbstractReductionRecipe(VPReductionRecipe *Red,
VPCostContext &Ctx,
VFRange &Range) {
// Creation of VPExpressions for partial reductions is entirely handled in
- // `transformToPartialReduction`.
- if (Red->isPartialReduction())
- return;
+ // transformToPartialReduction.
+ assert(!Red->isPartialReduction() &&
+ "This path no longer supports partial reductions");
VPExpressionRecipe *AbstractR = nullptr;
auto IP = std::next(Red->getIterator());
>From b276ef7dce9c940baeedc397763a6dc5459fa86b Mon Sep 17 00:00:00 2001
From: Sander de Smalen <sander.desmalen at arm.com>
Date: Fri, 10 Apr 2026 12:09:19 +0000
Subject: [PATCH 3/4] Address review comments
---
llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp | 10 ++++------
1 file changed, 4 insertions(+), 6 deletions(-)
diff --git a/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp b/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp
index b36fc5126d2f7..fbe395d7bdaf4 100644
--- a/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp
+++ b/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp
@@ -4416,7 +4416,7 @@ tryToMatchAndCreateExtendedReduction(VPReductionRecipe *Red, VPCostContext &Ctx,
VPValue *VecOp = Red->getVecOp();
assert(!Red->isPartialReduction() &&
- "This path no longer supports partial reductions");
+ "This path does not support partial reductions");
// Clamp the range if using extended-reduction is profitable.
auto IsExtendedRedValidAndClampRange =
@@ -4474,7 +4474,7 @@ tryToMatchAndCreateMulAccumulateReduction(VPReductionRecipe *Red,
return nullptr;
assert(!Red->isPartialReduction() &&
- "This path no longer supports partial reductions");
+ "This path does not support partial reductions");
Type *RedTy = Ctx.Types.inferScalarType(Red);
// Clamp the range if using multiply-accumulate-reduction is profitable.
@@ -4655,7 +4655,7 @@ static void tryToCreateAbstractReductionRecipe(VPReductionRecipe *Red,
// Creation of VPExpressions for partial reductions is entirely handled in
// transformToPartialReduction.
assert(!Red->isPartialReduction() &&
- "This path no longer supports partial reductions");
+ "This path does not support partial reductions");
VPExpressionRecipe *AbstractR = nullptr;
auto IP = std::next(Red->getIterator());
@@ -6100,9 +6100,7 @@ createPartialReductionExpression(VPReductionRecipe *Red) {
// reduce.[f]add(ext(op))
// -> VPExpressionRecipe(op, red)
- if (isa<VPWidenCastRecipe>(VecOp) &&
- (match(VecOp, m_ZExtOrSExt(m_VPValue())) ||
- match(VecOp, m_FPExt(m_VPValue()))))
+ if (match(VecOp, m_WidenAnyExtend(m_VPValue())))
return new VPExpressionRecipe(cast<VPWidenCastRecipe>(VecOp), Red);
// reduce.[f]add([f]mul(ext(a), ext(b)))
>From d42c9d55fbaaa1ac052f6f364198d9e537bca624 Mon Sep 17 00:00:00 2001
From: Sander de Smalen <sander.desmalen at arm.com>
Date: Thu, 16 Apr 2026 12:26:47 +0000
Subject: [PATCH 4/4] Address comments
* Limit support to 'mul/fmul' binops.
* assert that a VPExpression can be used to represent a partial reduction.
* Remove dead code from tryToMatchAndCreateExtendedReduction and tryToMatchAndCreateMulAccumulateReduction.
---
.../Transforms/Vectorize/VPlanTransforms.cpp | 38 +++++--------------
1 file changed, 9 insertions(+), 29 deletions(-)
diff --git a/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp b/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp
index fbe395d7bdaf4..6d7c306fbfe06 100644
--- a/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp
+++ b/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp
@@ -4431,10 +4431,8 @@ tryToMatchAndCreateExtendedReduction(VPReductionRecipe *Red, VPCostContext &Ctx,
cast<VPWidenCastRecipe>(VecOp)->computeCost(VF, Ctx);
InstructionCost RedCost = Red->computeCost(VF, Ctx);
- // TTI::getExtendedReductionCost for in-loop reductions
- // only supports integer types.
- if (RedTy->isFloatingPointTy())
- return false;
+ assert(!RedTy->isFloatingPointTy() &&
+ "getExtendedReductionCost only supports integer types");
ExtRedCost = Ctx.TTI.getExtendedReductionCost(
Opcode, ExtOpc == Instruction::CastOps::ZExt, RedTy, SrcVecTy,
Red->getFastMathFlags(), CostKind);
@@ -4445,8 +4443,7 @@ tryToMatchAndCreateExtendedReduction(VPReductionRecipe *Red, VPCostContext &Ctx,
VPValue *A;
// Match reduce(ext)).
- if (match(VecOp, m_Isa<VPWidenCastRecipe>(m_CombineOr(
- m_ZExtOrSExt(m_VPValue(A)), m_FPExt(m_VPValue(A))))) &&
+ if (match(VecOp, m_Isa<VPWidenCastRecipe>(m_ZExtOrSExt(m_VPValue(A)))) &&
IsExtendedRedValidAndClampRange(
RecurrenceDescriptor::getOpcode(Red->getRecurrenceKind()),
cast<VPWidenCastRecipe>(VecOp)->getOpcode(),
@@ -4522,23 +4519,6 @@ tryToMatchAndCreateMulAccumulateReduction(VPReductionRecipe *Red,
VPValue *A, *B;
VPValue *Tmp = nullptr;
- // Try to match reduce.fadd(fmul(fpext(...), fpext(...))).
- if (match(VecOp, m_FMul(m_FPExt(m_VPValue()), m_FPExt(m_VPValue())))) {
- assert(Opcode == Instruction::FAdd &&
- "MulAccumulateReduction from an FMul must accumulate into an FAdd "
- "instruction");
- auto *FMul = dyn_cast<VPWidenRecipe>(VecOp);
- if (!FMul)
- return nullptr;
-
- auto *RecipeA = dyn_cast<VPWidenCastRecipe>(FMul->getOperand(0));
- auto *RecipeB = dyn_cast<VPWidenCastRecipe>(FMul->getOperand(1));
-
- if (RecipeA && RecipeB &&
- IsMulAccValidAndClampRange(FMul, RecipeA, RecipeB, nullptr)) {
- return new VPExpressionRecipe(RecipeA, RecipeB, FMul, Red);
- }
- }
if (RedTy->isFloatingPointTy())
return nullptr;
@@ -6125,7 +6105,7 @@ createPartialReductionExpression(VPReductionRecipe *Red) {
return new VPExpressionRecipe(ExtA, ExtB, Mul, Sub, Red);
}
- return nullptr;
+ llvm_unreachable("Unsupported expression");
}
// Helper to transform a partial reduction chain into a partial reduction
@@ -6201,10 +6181,9 @@ static void transformToPartialReduction(const VPPartialReductionChain &Chain,
WidenRecipe->replaceAllUsesWith(PartialRed);
// For cost-model purposes, see if we can fold this into a VPExpression.
- if (VPExpressionRecipe *E = createPartialReductionExpression(PartialRed)) {
- E->insertBefore(WidenRecipe);
- PartialRed->replaceAllUsesWith(E);
- }
+ VPExpressionRecipe *E = createPartialReductionExpression(PartialRed);
+ E->insertBefore(WidenRecipe);
+ PartialRed->replaceAllUsesWith(E);
// We only need to update the PHI node once, which is when we find the
// last reduction in the chain.
@@ -6354,7 +6333,8 @@ matchExtendedReductionOperand(VPWidenRecipe *UpdateR, VPValue *Op,
Op = NegatedOp;
VPWidenRecipe *BinOp = dyn_cast<VPWidenRecipe>(Op);
- if (!BinOp || !Instruction::isBinaryOp(BinOp->getOpcode()))
+ if (!BinOp ||
+ !is_contained({Instruction::Mul, Instruction::FMul}, BinOp->getOpcode()))
return std::nullopt;
// The rest of the matching assumes `Op` is a (possibly extended/negated)
More information about the llvm-commits
mailing list