[llvm] f4e8cc1 - [VPlan] Detect and create partial reductions in VPlan. (NFCI) (#167851)
via llvm-commits
llvm-commits at lists.llvm.org
Sat Jan 31 11:44:52 PST 2026
Author: Florian Hahn
Date: 2026-01-31T19:44:46Z
New Revision: f4e8cc1a2229dca76d21c8d37439c4c194b06b86
URL: https://github.com/llvm/llvm-project/commit/f4e8cc1a2229dca76d21c8d37439c4c194b06b86
DIFF: https://github.com/llvm/llvm-project/commit/f4e8cc1a2229dca76d21c8d37439c4c194b06b86.diff
LOG: [VPlan] Detect and create partial reductions in VPlan. (NFCI) (#167851)
As a first step, move the existing partial reduction detection logic to
VPlan, trying to preserve the existing code structure & behavior as
closely as possible.
With this, partial reductions are detected and created together in a
single step.
This allows forming partial reductions and bundling them up if
profitable together in a follow-up.
PR: https://github.com/llvm/llvm-project/pull/167851
Added:
Modified:
llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
llvm/lib/Transforms/Vectorize/VPRecipeBuilder.h
llvm/lib/Transforms/Vectorize/VPlan.h
llvm/lib/Transforms/Vectorize/VPlanConstruction.cpp
llvm/lib/Transforms/Vectorize/VPlanPatternMatch.h
llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp
llvm/lib/Transforms/Vectorize/VPlanTransforms.h
llvm/lib/Transforms/Vectorize/VPlanUtils.cpp
llvm/lib/Transforms/Vectorize/VPlanUtils.h
llvm/test/Transforms/LoopVectorize/vplan-print-after-all.ll
Removed:
################################################################################
diff --git a/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp b/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
index bb5fae95eda0d..383fc7bc50e72 100644
--- a/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
+++ b/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
@@ -8036,209 +8036,6 @@ VPReplicateRecipe *VPRecipeBuilder::handleReplication(VPInstruction *VPI,
return Recipe;
}
-/// Find all possible partial reductions in the loop and track all of those that
-/// are valid so recipes can be formed later.
-void VPRecipeBuilder::collectScaledReductions(VFRange &Range) {
- // Find all possible partial reductions, grouping chains by their PHI. This
- // grouping allows invalidating the whole chain, if any link is not a valid
- // partial reduction.
- MapVector<Instruction *,
- SmallVector<std::pair<PartialReductionChain, unsigned>>>
- ChainsByPhi;
- for (const auto &[Phi, RdxDesc] : Legal->getReductionVars()) {
- if (Instruction *RdxExitInstr = RdxDesc.getLoopExitInstr())
- getScaledReductions(Phi, RdxExitInstr, Range, ChainsByPhi[Phi]);
- }
-
- // A partial reduction is invalid if any of its extends are used by
- // something that isn't another partial reduction. This is because the
- // extends are intended to be lowered along with the reduction itself.
-
- // Build up a set of partial reduction ops for efficient use checking.
- SmallPtrSet<User *, 4> PartialReductionOps;
- for (const auto &[_, Chains] : ChainsByPhi)
- for (const auto &[PartialRdx, _] : Chains)
- PartialReductionOps.insert(PartialRdx.ExtendUser);
-
- auto ExtendIsOnlyUsedByPartialReductions =
- [&PartialReductionOps](Instruction *Extend) {
- return all_of(Extend->users(), [&](const User *U) {
- return PartialReductionOps.contains(U);
- });
- };
-
- // Check if each use of a chain's two extends is a partial reduction
- // and only add those that don't have non-partial reduction users.
- for (const auto &[_, Chains] : ChainsByPhi) {
- for (const auto &[Chain, Scale] : Chains) {
- if (ExtendIsOnlyUsedByPartialReductions(Chain.ExtendA) &&
- (!Chain.ExtendB ||
- ExtendIsOnlyUsedByPartialReductions(Chain.ExtendB)))
- ScaledReductionMap.try_emplace(Chain.Reduction, Scale);
- }
- }
-
- // Check that all partial reductions in a chain are only used by other
- // partial reductions with the same scale factor. Otherwise we end up creating
- // users of scaled reductions where the types of the other operands don't
- // match.
- for (const auto &[Phi, Chains] : ChainsByPhi) {
- for (const auto &[Chain, Scale] : Chains) {
- auto AllUsersPartialRdx = [ScaleVal = Scale, RdxPhi = Phi,
- this](const User *U) {
- auto *UI = cast<Instruction>(U);
- if (isa<PHINode>(UI) && UI->getParent() == OrigLoop->getHeader())
- return UI == RdxPhi;
- return ScaledReductionMap.lookup_or(UI, 0) == ScaleVal ||
- !OrigLoop->contains(UI->getParent());
- };
-
- // If any partial reduction entry for the phi is invalid, invalidate the
- // whole chain.
- if (!all_of(Chain.Reduction->users(), AllUsersPartialRdx)) {
- for (const auto &[Chain, _] : Chains)
- ScaledReductionMap.erase(Chain.Reduction);
- break;
- }
- }
- }
-}
-
-bool VPRecipeBuilder::getScaledReductions(
- Instruction *PHI, Instruction *RdxExitInstr, VFRange &Range,
- SmallVectorImpl<std::pair<PartialReductionChain, unsigned>> &Chains) {
- if (!CM.TheLoop->contains(RdxExitInstr))
- return false;
-
- auto *Update = dyn_cast<BinaryOperator>(RdxExitInstr);
- if (!Update)
- return false;
-
- Value *Op = Update->getOperand(0);
- Value *PhiOp = Update->getOperand(1);
- if (Op == PHI)
- std::swap(Op, PhiOp);
-
- using namespace llvm::PatternMatch;
- // If Op is an extend, then it's still a valid partial reduction if the
- // extended mul fulfills the other requirements.
- // For example, reduce.add(ext(mul(ext(A), ext(B)))) is still a valid partial
- // reduction since the inner extends will be widened. We already have oneUse
- // checks on the inner extends so widening them is safe.
- std::optional<TTI::PartialReductionExtendKind> OuterExtKind = std::nullopt;
- if (match(Op, m_ZExtOrSExt(m_Mul(m_Value(), m_Value())))) {
- auto *Cast = cast<CastInst>(Op);
- OuterExtKind = TTI::getPartialReductionExtendKind(Cast->getOpcode());
- Op = Cast->getOperand(0);
- }
-
- // Try and get a scaled reduction from the first non-phi operand.
- // If one is found, we use the discovered reduction instruction in
- // place of the accumulator for costing.
- if (auto *OpInst = dyn_cast<Instruction>(Op)) {
- if (getScaledReductions(PHI, OpInst, Range, Chains)) {
- PHI = Chains.rbegin()->first.Reduction;
-
- Op = Update->getOperand(0);
- PhiOp = Update->getOperand(1);
- if (Op == PHI)
- std::swap(Op, PhiOp);
- }
- }
- if (PhiOp != PHI)
- return false;
-
- // If the update is a binary operator, check both of its operands to see if
- // they are extends. Otherwise, see if the update comes directly from an
- // extend.
- Instruction *Exts[2] = {nullptr};
- BinaryOperator *ExtendUser = dyn_cast<BinaryOperator>(Op);
- std::optional<unsigned> BinOpc;
- Type *ExtOpTypes[2] = {nullptr};
- TTI::PartialReductionExtendKind ExtKinds[2] = {TTI::PR_None};
-
- auto CollectExtInfo = [this, OuterExtKind, &Exts, &ExtOpTypes,
- &ExtKinds](SmallVectorImpl<Value *> &Ops) -> bool {
- for (const auto &[I, OpI] : enumerate(Ops)) {
- const APInt *C;
- if (I > 0 && match(OpI, m_APInt(C)) &&
- canConstantBeExtended(C, ExtOpTypes[0], ExtKinds[0])) {
- ExtOpTypes[I] = ExtOpTypes[0];
- ExtKinds[I] = ExtKinds[0];
- continue;
- }
- Value *ExtOp;
- if (!match(OpI, m_ZExtOrSExt(m_Value(ExtOp))) &&
- !match(OpI, m_FPExt(m_Value(ExtOp))))
- return false;
- Exts[I] = cast<Instruction>(OpI);
-
- // TODO: We should be able to support live-ins.
- if (!CM.TheLoop->contains(Exts[I]))
- return false;
-
- ExtOpTypes[I] = ExtOp->getType();
- ExtKinds[I] = TTI::getPartialReductionExtendKind(Exts[I]);
- // The outer extend kind must be the same as the inner extends, so that
- // they can be folded together.
- if (OuterExtKind.has_value() && OuterExtKind.value() != ExtKinds[I])
- return false;
- }
- return true;
- };
-
- if (ExtendUser) {
- if (!ExtendUser->hasOneUse())
- return false;
-
- // Use the side-effect of match to replace BinOp only if the pattern is
- // matched, we don't care at this point whether it actually matched.
- match(ExtendUser, m_Neg(m_BinOp(ExtendUser)));
-
- SmallVector<Value *> Ops(ExtendUser->operands());
- if (!CollectExtInfo(Ops))
- return false;
-
- BinOpc = std::make_optional(ExtendUser->getOpcode());
- } else if (match(Update, m_Add(m_Value(), m_Value())) ||
- match(Update, m_FAdd(m_Value(), m_Value()))) {
- // We already know the operands for Update are Op and PhiOp.
- SmallVector<Value *> Ops({Op});
- if (!CollectExtInfo(Ops))
- return false;
-
- ExtendUser = Update;
- BinOpc = std::nullopt;
- } else
- return false;
-
- PartialReductionChain Chain(RdxExitInstr, Exts[0], Exts[1], ExtendUser);
-
- TypeSize PHISize = PHI->getType()->getPrimitiveSizeInBits();
- TypeSize ASize = ExtOpTypes[0]->getPrimitiveSizeInBits();
- if (!PHISize.hasKnownScalarFactor(ASize))
- return false;
- unsigned TargetScaleFactor = PHISize.getKnownScalarFactor(ASize);
-
- if (LoopVectorizationPlanner::getDecisionAndClampRange(
- [&](ElementCount VF) {
- std::optional<FastMathFlags> FMF = std::nullopt;
- if (Update->getOpcode() == Instruction::FAdd)
- FMF = Update->getFastMathFlags();
- InstructionCost Cost = TTI->getPartialReductionCost(
- Update->getOpcode(), ExtOpTypes[0], ExtOpTypes[1],
- PHI->getType(), VF, ExtKinds[0], ExtKinds[1], BinOpc,
- CM.CostKind, FMF);
- return Cost.isValid();
- },
- Range)) {
- Chains.emplace_back(Chain, TargetScaleFactor);
- return true;
- }
-
- return false;
-}
-
VPRecipeBase *
VPRecipeBuilder::tryToCreateWidenNonPhiRecipe(VPSingleDefRecipe *R,
VFRange &Range) {
@@ -8269,9 +8066,6 @@ VPRecipeBuilder::tryToCreateWidenNonPhiRecipe(VPSingleDefRecipe *R,
VPI->getOpcode() == Instruction::Store)
return tryToWidenMemory(VPI, Range);
- if (std::optional<unsigned> ScaleFactor = getScalingForReduction(Instr))
- return tryToCreatePartialReduction(VPI, ScaleFactor.value());
-
if (!shouldWiden(Instr, Range))
return nullptr;
@@ -8290,55 +8084,6 @@ VPRecipeBuilder::tryToCreateWidenNonPhiRecipe(VPSingleDefRecipe *R,
return tryToWiden(VPI);
}
-VPRecipeBase *
-VPRecipeBuilder::tryToCreatePartialReduction(VPInstruction *Reduction,
- unsigned ScaleFactor) {
- assert(Reduction->getNumOperands() == 2 &&
- "Unexpected number of operands for partial reduction");
-
- VPValue *BinOp = Reduction->getOperand(0);
- VPValue *Accumulator = Reduction->getOperand(1);
- VPRecipeBase *BinOpRecipe = BinOp->getDefiningRecipe();
- if (isa<VPReductionPHIRecipe>(BinOpRecipe) ||
- (isa<VPReductionRecipe>(BinOpRecipe) &&
- cast<VPReductionRecipe>(BinOpRecipe)->isPartialReduction()))
- std::swap(BinOp, Accumulator);
-
- if (auto *RedPhiR = dyn_cast<VPReductionPHIRecipe>(Accumulator))
- RedPhiR->setVFScaleFactor(ScaleFactor);
-
- assert(ScaleFactor ==
- vputils::getVFScaleFactor(Accumulator->getDefiningRecipe()) &&
- "all accumulators in chain must have same scale factor");
-
- auto *ReductionI = Reduction->getUnderlyingInstr();
- assert(
- Reduction->getOpcode() != Instruction::FAdd ||
- (ReductionI->hasAllowReassoc() && ReductionI->hasAllowContract()) &&
- "FAdd partial reduction requires allow-reassoc and allow-contract");
- if (Reduction->getOpcode() == Instruction::Sub) {
- SmallVector<VPValue *, 2> Ops;
- Ops.push_back(Plan.getConstantInt(ReductionI->getType(), 0));
- Ops.push_back(BinOp);
- BinOp = new VPWidenRecipe(*ReductionI, Ops, VPIRFlags(*ReductionI),
- VPIRMetadata(), ReductionI->getDebugLoc());
- Builder.insert(BinOp->getDefiningRecipe());
- }
-
- VPValue *Cond = nullptr;
- if (CM.blockNeedsPredicationForAnyReason(ReductionI->getParent()))
- Cond = getBlockInMask(Builder.getInsertBlock());
-
- return new VPReductionRecipe(
- Reduction->getOpcode() == Instruction::FAdd ? RecurKind::FAdd
- : RecurKind::Add,
- Reduction->getOpcode() == Instruction::FAdd
- ? Reduction->getFastMathFlags()
- : FastMathFlags(),
- ReductionI, Accumulator, BinOp, Cond,
- RdxUnordered{/*VFScaleFactor=*/ScaleFactor}, ReductionI->getDebugLoc());
-}
-
void LoopVectorizationPlanner::buildVPlansWithVPRecipes(ElementCount MinVF,
ElementCount MaxVF) {
if (ElementCount::isKnownGT(MinVF, MaxVF))
@@ -8475,11 +8220,7 @@ VPlanPtr LoopVectorizationPlanner::tryToBuildVPlanWithVPRecipes(
// Construct wide recipes and apply predication for original scalar
// VPInstructions in the loop.
// ---------------------------------------------------------------------------
- VPRecipeBuilder RecipeBuilder(*Plan, OrigLoop, TLI, &TTI, Legal, CM, Builder,
- BlockMaskCache);
- // TODO: Handle partial reductions with EVL tail folding.
- if (!CM.foldTailWithEVL())
- RecipeBuilder.collectScaledReductions(Range);
+ VPRecipeBuilder RecipeBuilder(*Plan, TLI, Legal, CM, Builder, BlockMaskCache);
// Scan the body of the loop in a topological order to visit each basic block
// after having visited its predecessor basic blocks.
@@ -8601,13 +8342,16 @@ VPlanPtr LoopVectorizationPlanner::tryToBuildVPlanWithVPRecipes(
if (!RUN_VPLAN_PASS(VPlanTransforms::handleFindLastReductions, *Plan))
return nullptr;
- // Transform recipes to abstract recipes if it is legal and beneficial and
- // clamp the range for better cost estimation.
+ // Create partial reduction recipes for scaled reductions and transform
+ // recipes to abstract recipes if it is legal and beneficial and clamp the
+ // range for better cost estimation.
// TODO: Enable following transform when the EVL-version of extended-reduction
// and mulacc-reduction are implemented.
if (!CM.foldTailWithEVL()) {
VPCostContext CostCtx(CM.TTI, *CM.TLI, *Plan, CM, CM.CostKind, CM.PSE,
OrigLoop);
+ RUN_VPLAN_PASS(VPlanTransforms::createPartialReductions, *Plan, CostCtx,
+ Range);
RUN_VPLAN_PASS(VPlanTransforms::convertToAbstractRecipes, *Plan, CostCtx,
Range);
}
@@ -8902,11 +8646,7 @@ void LoopVectorizationPlanner::addReductionResultComputation(
VPBuilder PHBuilder(Plan->getVectorPreheader());
VPValue *Iden = Plan->getOrAddLiveIn(
getRecurrenceIdentity(RK, PhiTy, RdxDesc.getFastMathFlags()));
- // If the PHI is used by a partial reduction, set the scale factor.
- unsigned ScaleFactor =
- RecipeBuilder.getScalingForReduction(RdxDesc.getLoopExitInstr())
- .value_or(1);
- auto *ScaleFactorVPV = Plan->getConstantInt(32, ScaleFactor);
+ auto *ScaleFactorVPV = Plan->getConstantInt(32, 1);
VPValue *StartV = PHBuilder.createNaryOp(
VPInstruction::ReductionStartVector,
{PhiR->getStartValue(), Iden, ScaleFactorVPV},
diff --git a/llvm/lib/Transforms/Vectorize/VPRecipeBuilder.h b/llvm/lib/Transforms/Vectorize/VPRecipeBuilder.h
index 534f51e86a9b7..3280fc4827239 100644
--- a/llvm/lib/Transforms/Vectorize/VPRecipeBuilder.h
+++ b/llvm/lib/Transforms/Vectorize/VPRecipeBuilder.h
@@ -19,44 +19,17 @@ namespace llvm {
class LoopVectorizationLegality;
class LoopVectorizationCostModel;
class TargetLibraryInfo;
-class TargetTransformInfo;
struct HistogramInfo;
struct VFRange;
-/// A chain of instructions that form a partial reduction.
-/// Designed to match either:
-/// reduction_bin_op (extend (A), accumulator), or
-/// reduction_bin_op (bin_op (extend (A), (extend (B))), accumulator).
-struct PartialReductionChain {
- PartialReductionChain(Instruction *Reduction, Instruction *ExtendA,
- Instruction *ExtendB, Instruction *ExtendUser)
- : Reduction(Reduction), ExtendA(ExtendA), ExtendB(ExtendB),
- ExtendUser(ExtendUser) {}
- /// The top-level binary operation that forms the reduction to a scalar
- /// after the loop body.
- Instruction *Reduction;
- /// The extension of each of the inner binary operation's operands.
- Instruction *ExtendA;
- Instruction *ExtendB;
-
- /// The user of the extends that is then reduced.
- Instruction *ExtendUser;
-};
-
/// Helper class to create VPRecipies from IR instructions.
class VPRecipeBuilder {
/// The VPlan new recipes are added to.
VPlan &Plan;
- /// The loop that we evaluate.
- Loop *OrigLoop;
-
/// Target Library Info.
const TargetLibraryInfo *TLI;
- // Target Transform Info.
- const TargetTransformInfo *TTI;
-
/// The legality analysis.
LoopVectorizationLegality *Legal;
@@ -79,9 +52,6 @@ class VPRecipeBuilder {
/// created.
SmallVector<VPHeaderPHIRecipe *, 4> PhisToFix;
- /// A mapping of partial reduction exit instructions to their scaling factor.
- DenseMap<const Instruction *, unsigned> ScaledReductionMap;
-
/// Check if \p I can be widened at the start of \p Range and possibly
/// decrease the range such that the returned value holds for the entire \p
/// Range. The function should not be called for memory instructions or calls.
@@ -114,48 +84,19 @@ class VPRecipeBuilder {
VPHistogramRecipe *tryToWidenHistogram(const HistogramInfo *HI,
VPInstruction *VPI);
- /// Examines reduction operations to see if the target can use a cheaper
- /// operation with a wider per-iteration input VF and narrower PHI VF.
- /// Each element within Chains is a pair with a struct containing reduction
- /// information and the scaling factor between the number of elements in
- /// the input and output.
- /// Recursively calls itself to identify chained scaled reductions.
- /// Returns true if this invocation added an entry to Chains, otherwise false.
- /// i.e. returns false in the case that a subcall adds an entry to Chains,
- /// but the top-level call does not.
- bool getScaledReductions(
- Instruction *PHI, Instruction *RdxExitInstr, VFRange &Range,
- SmallVectorImpl<std::pair<PartialReductionChain, unsigned>> &Chains);
-
public:
- VPRecipeBuilder(VPlan &Plan, Loop *OrigLoop, const TargetLibraryInfo *TLI,
- const TargetTransformInfo *TTI,
+ VPRecipeBuilder(VPlan &Plan, const TargetLibraryInfo *TLI,
LoopVectorizationLegality *Legal,
LoopVectorizationCostModel &CM, VPBuilder &Builder,
DenseMap<VPBasicBlock *, VPValue *> &BlockMaskCache)
- : Plan(Plan), OrigLoop(OrigLoop), TLI(TLI), TTI(TTI), Legal(Legal),
- CM(CM), Builder(Builder), BlockMaskCache(BlockMaskCache) {}
-
- std::optional<unsigned> getScalingForReduction(const Instruction *ExitInst) {
- auto It = ScaledReductionMap.find(ExitInst);
- return It == ScaledReductionMap.end() ? std::nullopt
- : std::make_optional(It->second);
- }
-
- /// Find all possible partial reductions in the loop and track all of those
- /// that are valid so recipes can be formed later.
- void collectScaledReductions(VFRange &Range);
+ : Plan(Plan), TLI(TLI), Legal(Legal), CM(CM), Builder(Builder),
+ BlockMaskCache(BlockMaskCache) {}
/// Create and return a widened recipe for a non-phi recipe \p R if one can be
/// created within the given VF \p Range.
VPRecipeBase *tryToCreateWidenNonPhiRecipe(VPSingleDefRecipe *R,
VFRange &Range);
- /// Create and return a partial reduction recipe for a reduction instruction
- /// along with binary operation and reduction phi operands.
- VPRecipeBase *tryToCreatePartialReduction(VPInstruction *Reduction,
- unsigned ScaleFactor);
-
/// Set the recipe created for given ingredient.
void setRecipe(Instruction *I, VPRecipeBase *R) {
assert(!Ingredient2Recipe.contains(I) &&
diff --git a/llvm/lib/Transforms/Vectorize/VPlan.h b/llvm/lib/Transforms/Vectorize/VPlan.h
index f650971c6b397..bccf3c61a3285 100644
--- a/llvm/lib/Transforms/Vectorize/VPlan.h
+++ b/llvm/lib/Transforms/Vectorize/VPlan.h
@@ -1641,11 +1641,19 @@ class LLVM_ABI_FOR_TEST VPWidenRecipe : public VPRecipeWithIRFlags,
setUnderlyingValue(&I);
}
+ VPWidenRecipe(unsigned Opcode, ArrayRef<VPValue *> Operands,
+ const VPIRFlags &Flags = {}, const VPIRMetadata &Metadata = {},
+ DebugLoc DL = {})
+ : VPRecipeWithIRFlags(VPRecipeBase::VPWidenSC, Operands, Flags, DL),
+ VPIRMetadata(Metadata), Opcode(Opcode) {}
+
~VPWidenRecipe() override = default;
VPWidenRecipe *clone() override {
- return new VPWidenRecipe(*getUnderlyingInstr(), operands(), *this, *this,
- getDebugLoc());
+ if (auto *UV = getUnderlyingValue())
+ return new VPWidenRecipe(*cast<Instruction>(UV), operands(), *this, *this,
+ getDebugLoc());
+ return new VPWidenRecipe(Opcode, operands(), *this, *this, getDebugLoc());
}
VP_CLASSOF_IMPL(VPRecipeBase::VPWidenSC)
diff --git a/llvm/lib/Transforms/Vectorize/VPlanConstruction.cpp b/llvm/lib/Transforms/Vectorize/VPlanConstruction.cpp
index bf02c548723de..d6b6d8a92cba7 100644
--- a/llvm/lib/Transforms/Vectorize/VPlanConstruction.cpp
+++ b/llvm/lib/Transforms/Vectorize/VPlanConstruction.cpp
@@ -16,12 +16,15 @@
#include "VPlanAnalysis.h"
#include "VPlanCFG.h"
#include "VPlanDominatorTree.h"
+#include "VPlanHelpers.h"
#include "VPlanPatternMatch.h"
#include "VPlanTransforms.h"
+#include "VPlanUtils.h"
#include "llvm/Analysis/LoopInfo.h"
#include "llvm/Analysis/LoopIterator.h"
#include "llvm/Analysis/ScalarEvolution.h"
#include "llvm/Analysis/ScalarEvolutionExpressions.h"
+#include "llvm/Analysis/TargetTransformInfo.h"
#include "llvm/IR/InstrTypes.h"
#include "llvm/IR/MDBuilder.h"
#include "llvm/Transforms/Utils/LoopUtils.h"
@@ -1127,37 +1130,6 @@ void VPlanTransforms::addMinimumVectorEpilogueIterationCheck(
Branch->setMetadata(LLVMContext::MD_prof, BranchWeights);
}
-/// If \p V is used by a recipe matching pattern \p P, return it. Otherwise
-/// return nullptr;
-template <typename MatchT>
-static VPRecipeBase *findUserOf(VPValue *V, const MatchT &P) {
- auto It = find_if(V->users(), match_fn(P));
- return It == V->user_end() ? nullptr : cast<VPRecipeBase>(*It);
-}
-
-/// If \p V is used by a VPInstruction with \p Opcode, return it. Otherwise
-/// return nullptr.
-template <unsigned Opcode> static VPInstruction *findUserOf(VPValue *V) {
- return cast_or_null<VPInstruction>(findUserOf(V, m_VPInstruction<Opcode>()));
-}
-
-/// Find the ComputeReductionResult recipe for \p PhiR, looking through selects
-/// inserted for predicated reductions or tail folding.
-static VPInstruction *findComputeReductionResult(VPReductionPHIRecipe *PhiR) {
- VPValue *BackedgeVal = PhiR->getBackedgeValue();
- if (auto *Res =
- findUserOf<VPInstruction::ComputeReductionResult>(BackedgeVal))
- return Res;
-
- // Look through selects inserted for tail folding or predicated reductions.
- VPRecipeBase *SelR =
- findUserOf(BackedgeVal, m_Select(m_VPValue(), m_VPValue(), m_VPValue()));
- if (!SelR)
- return nullptr;
- return findUserOf<VPInstruction::ComputeReductionResult>(
- cast<VPSingleDefRecipe>(SelR));
-}
-
/// Find and return the final select instruction of the FindIV result pattern
/// for the given \p BackedgeVal:
/// select(icmp ne ComputeReductionResult(ReducedIV), Sentinel),
@@ -1266,7 +1238,7 @@ bool VPlanTransforms::handleMaxMinNumReductions(VPlan &Plan) {
// If we exit early due to NaNs, compute the final reduction result based on
// the reduction phi at the beginning of the last vector iteration.
- auto *RdxResult = findComputeReductionResult(RedPhiR);
+ auto *RdxResult = vputils::findComputeReductionResult(RedPhiR);
assert(RdxResult && "must find a ComputeReductionResult");
auto *NewSel = MiddleBuilder.createSelect(AnyNaNLane, RedPhiR,
@@ -1394,7 +1366,7 @@ bool VPlanTransforms::handleFindLastReductions(VPlan &Plan) {
// Find final reduction computation and replace it with an
// extract.last.active intrinsic.
auto *RdxResult =
- findUserOf<VPInstruction::ComputeReductionResult>(DataSelect);
+ vputils::findUserOf<VPInstruction::ComputeReductionResult>(DataSelect);
// TODO: Handle tail-folding.
if (!RdxResult)
return false;
@@ -1453,7 +1425,7 @@ bool VPlanTransforms::handleMultiUseReductions(VPlan &Plan) {
VPValue *CmpOpA;
VPValue *CmpOpB;
CmpPredicate Pred;
- auto *Cmp = dyn_cast_or_null<VPRecipeWithIRFlags>(findUserOf(
+ auto *Cmp = dyn_cast_or_null<VPRecipeWithIRFlags>(vputils::findUserOf(
MinOrMaxPhiR, m_Cmp(Pred, m_VPValue(CmpOpA), m_VPValue(CmpOpB))));
if (!Cmp || Cmp->getNumUsers() != 1 ||
(CmpOpA != MinOrMaxOpValue && CmpOpB != MinOrMaxOpValue))
@@ -1469,7 +1441,7 @@ bool VPlanTransforms::handleMultiUseReductions(VPlan &Plan) {
return false;
VPInstruction *MinOrMaxResult =
- findUserOf<VPInstruction::ComputeReductionResult>(MinOrMaxOp);
+ vputils::findUserOf<VPInstruction::ComputeReductionResult>(MinOrMaxOp);
assert(is_contained(MinOrMaxPhiR->users(), MinOrMaxOp) &&
"one user must be MinOrMaxOp");
assert(MinOrMaxResult &&
diff --git a/llvm/lib/Transforms/Vectorize/VPlanPatternMatch.h b/llvm/lib/Transforms/Vectorize/VPlanPatternMatch.h
index 6f605ac877e1b..9428070b02f1a 100644
--- a/llvm/lib/Transforms/Vectorize/VPlanPatternMatch.h
+++ b/llvm/lib/Transforms/Vectorize/VPlanPatternMatch.h
@@ -597,6 +597,18 @@ m_FMul(const Op0_t &Op0, const Op1_t &Op1) {
return m_Binary<Instruction::FMul, Op0_t, Op1_t>(Op0, Op1);
}
+template <typename Op0_t, typename Op1_t>
+inline AllRecipe_match<Instruction::FAdd, Op0_t, Op1_t>
+m_FAdd(const Op0_t &Op0, const Op1_t &Op1) {
+ return m_Binary<Instruction::FAdd, Op0_t, Op1_t>(Op0, Op1);
+}
+
+template <typename Op0_t, typename Op1_t>
+inline AllRecipe_commutative_match<Instruction::FAdd, Op0_t, Op1_t>
+m_c_FAdd(const Op0_t &Op0, const Op1_t &Op1) {
+ return m_c_Binary<Instruction::FAdd, Op0_t, Op1_t>(Op0, Op1);
+}
+
template <typename Op0_t, typename Op1_t>
inline AllRecipe_match<Instruction::UDiv, Op0_t, Op1_t>
m_UDiv(const Op0_t &Op0, const Op1_t &Op1) {
diff --git a/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp b/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp
index 501a23478f66f..117337ec47c1a 100644
--- a/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp
+++ b/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp
@@ -908,8 +908,7 @@ getOptimizableIVOf(VPValue *VPV, PredicatedScalarEvolution &PSE) {
case Instruction::Add:
return match(VPV, m_c_Add(m_Specific(WideIV), m_Specific(IVStep)));
case Instruction::FAdd:
- return match(VPV, m_c_Binary<Instruction::FAdd>(m_Specific(WideIV),
- m_Specific(IVStep)));
+ return match(VPV, m_c_FAdd(m_Specific(WideIV), m_Specific(IVStep)));
case Instruction::FSub:
return match(VPV, m_Binary<Instruction::FSub>(m_Specific(WideIV),
m_Specific(IVStep)));
@@ -5389,3 +5388,360 @@ void VPlanTransforms::addExitUsersForFirstOrderRecurrences(VPlan &Plan,
}
}
}
+
+namespace {
+
+/// A chain of recipes that form a partial reduction. Matches either
+/// reduction_bin_op (extend (A), accumulator), or
+/// reduction_bin_op (bin_op (extend (A), (extend (B))), accumulator).
+struct VPPartialReductionChain {
+ /// The top-level binary operation that forms the reduction to a scalar
+ /// after the loop body.
+ VPWidenRecipe *ReductionBinOp;
+ /// The extension of each of the inner binary operation's operands.
+ VPWidenCastRecipe *ExtendA;
+ VPWidenCastRecipe *ExtendB;
+ /// The user of the extends that is then reduced.
+ VPWidenRecipe *BinOp;
+ unsigned ScaleFactor;
+};
+
+// Helper to transform a partial reduction chain into a partial reduction
+// recipe. Returns true if transformation succeeded. Checks profitability and
+// clamps VF range.
+static bool transformToPartialReduction(const VPPartialReductionChain &Chain,
+ VFRange &Range, VPCostContext &CostCtx,
+ VPlan &Plan) {
+ VPWidenRecipe *WidenRecipe = Chain.ReductionBinOp;
+ unsigned ScaleFactor = Chain.ScaleFactor;
+ assert(WidenRecipe->getNumOperands() == 2 && "Expected binary operation");
+
+ VPValue *BinOp = WidenRecipe->getOperand(0);
+ VPValue *Accumulator = WidenRecipe->getOperand(1);
+
+ // Swap if needed to ensure Accumulator is the PHI or partial reduction.
+ if (isa_and_present<VPReductionPHIRecipe, VPReductionRecipe>(BinOp))
+ std::swap(BinOp, Accumulator);
+
+ // For chained reductions, only transform if accumulator is already a PHI or
+ // partial reduction. Otherwise, it needs to be transformed first.
+ auto *AccumRecipe = Accumulator->getDefiningRecipe();
+ if (!isa_and_present<VPReductionPHIRecipe, VPReductionRecipe>(AccumRecipe))
+ return false;
+
+ // Check if the partial reduction is profitable for the VF range.
+ Type *PhiType = CostCtx.Types.inferScalarType(Accumulator);
+
+ // Derive extend info from the stored extends.
+ auto GetExtInfo = [&CostCtx](VPWidenCastRecipe *Ext)
+ -> std::pair<Type *, TargetTransformInfo::PartialReductionExtendKind> {
+ if (!Ext)
+ return {nullptr, TargetTransformInfo::PR_None};
+ Type *ExtOpType = CostCtx.Types.inferScalarType(Ext->getOperand(0));
+ auto ExtKind = TargetTransformInfo::getPartialReductionExtendKind(
+ static_cast<Instruction::CastOps>(Ext->getOpcode()));
+ return {ExtOpType, ExtKind};
+ };
+ auto ExtInfoA = GetExtInfo(Chain.ExtendA);
+ auto ExtInfoB = GetExtInfo(Chain.ExtendB);
+ Type *ExtOpTypeA = ExtInfoA.first;
+ Type *ExtOpTypeB = ExtInfoB.first;
+ auto ExtKindA = ExtInfoA.second;
+ auto ExtKindB = ExtInfoB.second;
+ // If ExtendB is nullptr but there's a separate BinOp, the second operand
+ // was a constant that can use the same extend kind as the first.
+ if (!Chain.ExtendB && Chain.BinOp && Chain.BinOp != Chain.ReductionBinOp) {
+ // Validate that the constant can be extended to the narrow type.
+ const APInt *Const = nullptr;
+ for (VPValue *Op : Chain.BinOp->operands()) {
+ if (match(Op, m_APInt(Const)))
+ break;
+ }
+ if (!Const || !canConstantBeExtended(Const, ExtOpTypeA, ExtKindA))
+ return false;
+ ExtOpTypeB = ExtOpTypeA;
+ ExtKindB = ExtKindA;
+ }
+
+ // BinOpc is only set when there's a separate binary op (not when BinOp is
+ // the reduction itself).
+ std::optional<unsigned> BinOpc =
+ (Chain.BinOp && Chain.BinOp != Chain.ReductionBinOp)
+ ? std::make_optional(Chain.BinOp->getOpcode())
+ : std::nullopt;
+
+ if (!LoopVectorizationPlanner::getDecisionAndClampRange(
+ [&](ElementCount VF) {
+ return CostCtx.TTI
+ .getPartialReductionCost(
+ WidenRecipe->getOpcode(), ExtOpTypeA, ExtOpTypeB, PhiType,
+ VF, ExtKindA, ExtKindB, BinOpc, CostCtx.CostKind,
+ PhiType->isFloatingPointTy()
+ ? std::optional{WidenRecipe->getFastMathFlags()}
+ : std::nullopt)
+ .isValid();
+ },
+ Range))
+ return false;
+
+ VPValue *Cond = nullptr;
+ VPValue *ExitValue = nullptr;
+ if (auto *RdxPhi = dyn_cast<VPReductionPHIRecipe>(AccumRecipe)) {
+ assert(RdxPhi->getVFScaleFactor() == 1 && "scale factor must not be set");
+ RdxPhi->setVFScaleFactor(ScaleFactor);
+
+ // Update ReductionStartVector instruction scale factor.
+ VPValue *StartValue = RdxPhi->getOperand(0);
+ auto *StartInst = cast<VPInstruction>(StartValue);
+ assert(StartInst->getOpcode() == VPInstruction::ReductionStartVector);
+ auto *NewScaleFactor = Plan.getConstantInt(32, ScaleFactor);
+ StartInst->setOperand(2, NewScaleFactor);
+
+ // Find the ComputeReductionResult that uses the WidenRecipe (the exit
+ // value). Look through selects for predicated reductions.
+ if (auto *RdxResult = vputils::findComputeReductionResult(RdxPhi)) {
+ ExitValue = RdxResult->getOperand(0);
+ match(ExitValue, m_Select(m_VPValue(Cond), m_VPValue(), m_VPValue()));
+ }
+ }
+
+ // Handle SUB by negating the operand and using ADD for the partial reduction.
+ if (WidenRecipe->getOpcode() == Instruction::Sub) {
+ VPBuilder Builder(WidenRecipe);
+ Type *ElemTy = CostCtx.Types.inferScalarType(BinOp);
+ auto *Zero = Plan.getConstantInt(ElemTy, 0);
+ VPIRFlags Flags = WidenRecipe->getUnderlyingInstr()
+ ? VPIRFlags(*WidenRecipe->getUnderlyingInstr())
+ : VPIRFlags();
+ auto *NegRecipe = new VPWidenRecipe(Instruction::Sub, {Zero, BinOp}, Flags,
+ VPIRMetadata(), DebugLoc::getUnknown());
+ Builder.insert(NegRecipe);
+ BinOp = NegRecipe;
+ }
+
+ RecurKind RdxKind =
+ PhiType->isFloatingPointTy() ? RecurKind::FAdd : RecurKind::Add;
+ auto *PartialRed = new VPReductionRecipe(
+ RdxKind,
+ RdxKind == RecurKind::FAdd ? WidenRecipe->getFastMathFlags()
+ : FastMathFlags(),
+ WidenRecipe->getUnderlyingInstr(), Accumulator, BinOp, Cond,
+ RdxUnordered{/*VFScaleFactor=*/ScaleFactor});
+ PartialRed->insertBefore(WidenRecipe);
+
+ if (Cond)
+ ExitValue->replaceAllUsesWith(PartialRed);
+ WidenRecipe->replaceAllUsesWith(PartialRed);
+ return true;
+}
+
+/// Examines reduction operations to see if the target can use a cheaper
+/// operation with a wider per-iteration input VF and narrower PHI VF.
+/// Recursively calls itself to identify chained scaled reductions.
+/// Returns true if this invocation added an entry to Chains, otherwise false.
+static bool
+getScaledReductions(VPSingleDefRecipe *RedPhiR, VPValue *PrevValue,
+ SmallVectorImpl<VPPartialReductionChain> &Chains,
+ VPTypeAnalysis &TypeInfo) {
+ auto *UpdateR = dyn_cast<VPWidenRecipe>(PrevValue);
+ if (!UpdateR || !Instruction::isBinaryOp(UpdateR->getOpcode()))
+ return false;
+
+ VPValue *Op = UpdateR->getOperand(0);
+ VPValue *PhiOp = UpdateR->getOperand(1);
+ if (Op == RedPhiR)
+ std::swap(Op, PhiOp);
+
+ // If Op is an extend, then it's still a valid partial reduction if the
+ // extended mul fulfills the other requirements.
+ // For example, reduce.add(ext(mul(ext(A), ext(B)))) is still a valid partial
+ // reduction since the inner extends will be widened. We already have oneUse
+ // checks on the inner extends so widening them is safe.
+ std::optional<TTI::PartialReductionExtendKind> OuterExtKind;
+ if (match(Op, m_ZExtOrSExt(m_Mul(m_VPValue(), m_VPValue()))) ||
+ match(Op, m_FPExt(m_FMul(m_VPValue(), m_VPValue())))) {
+ auto *CastRecipe = cast<VPWidenCastRecipe>(Op);
+ auto CastOp = static_cast<Instruction::CastOps>(CastRecipe->getOpcode());
+ OuterExtKind = TTI::getPartialReductionExtendKind(CastOp);
+ Op = CastRecipe->getOperand(0);
+ }
+
+ // Try and get a scaled reduction from the first non-phi operand.
+ // If one is found, we use the discovered reduction instruction in
+ // place of the accumulator for costing.
+ if (getScaledReductions(RedPhiR, Op, Chains, TypeInfo)) {
+ RedPhiR = Chains.rbegin()->ReductionBinOp;
+ Op = UpdateR->getOperand(0);
+ PhiOp = UpdateR->getOperand(1);
+ if (Op == RedPhiR)
+ std::swap(Op, PhiOp);
+ }
+ if (RedPhiR != PhiOp)
+ return false;
+
+ // If the update is a binary op, check both of its operands to see if
+ // they are extends. Otherwise, see if the update comes directly from an
+ // extend.
+ VPWidenCastRecipe *CastRecipes[2] = {nullptr};
+
+ // Match extends and populate CastRecipes. Returns false if matching fails.
+ auto MatchExtends = [OuterExtKind,
+ &CastRecipes](ArrayRef<VPValue *> Operands) {
+ assert(Operands.size() <= 2 && "expected at most 2 operands");
+
+ for (const auto &[I, OpVal] : enumerate(Operands)) {
+ // Allow constant as second operand - validation happens in transform.
+ const APInt *Unused;
+ if (I > 0 && CastRecipes[0] && match(OpVal, m_APInt(Unused)))
+ continue;
+
+ VPValue *ExtInput;
+ if (!match(OpVal, m_ZExtOrSExt(m_VPValue(ExtInput))) &&
+ !match(OpVal, m_FPExt(m_VPValue(ExtInput))))
+ return false;
+
+ CastRecipes[I] = cast<VPWidenCastRecipe>(OpVal);
+
+ // The outer extend kind must match the inner extends for folding.
+ if (OuterExtKind) {
+ auto CastOp =
+ static_cast<Instruction::CastOps>(CastRecipes[I]->getOpcode());
+ if (*OuterExtKind != TTI::getPartialReductionExtendKind(CastOp))
+ return false;
+ }
+ }
+ return CastRecipes[0] != nullptr;
+ };
+
+ // If Op is a binary operator, check both of its operands to see if they are
+ // extends. Otherwise, see if the update comes directly from an extend.
+ auto *BinOp = dyn_cast<VPWidenRecipe>(Op);
+ if (BinOp && Instruction::isBinaryOp(BinOp->getOpcode())) {
+ if (!BinOp->hasOneUse())
+ return false;
+
+ // Handle neg(binop(ext, ext)) pattern.
+ VPValue *OtherOp = nullptr;
+ if (match(BinOp, m_Sub(m_ZeroInt(), m_VPValue(OtherOp))))
+ BinOp = dyn_cast<VPWidenRecipe>(OtherOp);
+
+ if (!BinOp || !Instruction::isBinaryOp(BinOp->getOpcode()) ||
+ !MatchExtends(BinOp->operands()))
+ return false;
+ } else if (match(UpdateR, m_Add(m_VPValue(), m_VPValue())) ||
+ match(UpdateR, m_FAdd(m_VPValue(), m_VPValue()))) {
+ // We already know the operands for Update are Op and PhiOp.
+ if (!MatchExtends({Op}))
+ return false;
+ BinOp = UpdateR;
+ } else {
+ return false;
+ }
+
+ Type *PhiType = TypeInfo.inferScalarType(RedPhiR);
+ TypeSize PHISize = PhiType->getPrimitiveSizeInBits();
+ Type *ExtOpType = TypeInfo.inferScalarType(CastRecipes[0]->getOperand(0));
+ TypeSize ASize = ExtOpType->getPrimitiveSizeInBits();
+ if (!PHISize.hasKnownScalarFactor(ASize))
+ return false;
+
+ Chains.push_back(
+ {UpdateR, CastRecipes[0], CastRecipes[1], BinOp,
+ static_cast<unsigned>(PHISize.getKnownScalarFactor(ASize))});
+ return true;
+}
+} // namespace
+
+void VPlanTransforms::createPartialReductions(VPlan &Plan,
+ VPCostContext &CostCtx,
+ VFRange &Range) {
+ // Find all possible partial reductions, grouping chains by their PHI. This
+ // grouping allows invalidating the whole chain, if any link is not a valid
+ // partial reduction.
+ MapVector<VPReductionPHIRecipe *, SmallVector<VPPartialReductionChain>>
+ ChainsByPhi;
+ VPBasicBlock *HeaderVPBB = Plan.getVectorLoopRegion()->getEntryBasicBlock();
+ for (VPRecipeBase &R : HeaderVPBB->phis()) {
+ auto *RedPhiR = dyn_cast<VPReductionPHIRecipe>(&R);
+ if (!RedPhiR)
+ continue;
+
+ // Get the backedge value from the reduction PHI and find the
+ // ComputeReductionResult that uses it (directly or through a select for
+ // predicated reductions).
+ if (auto *RdxResult = vputils::findComputeReductionResult(RedPhiR)) {
+ VPValue *ExitValue = RdxResult->getOperand(0);
+ match(ExitValue,
+ m_Select(m_VPValue(), m_VPValue(ExitValue), m_VPValue()));
+ getScaledReductions(RedPhiR, ExitValue, ChainsByPhi[RedPhiR],
+ CostCtx.Types);
+ }
+ }
+
+ if (ChainsByPhi.empty())
+ return;
+
+ // Build set of partial reduction operations for extend user validation and
+ // a map of reduction bin ops to their scale factors for scale validation.
+ SmallPtrSet<VPRecipeBase *, 4> PartialReductionOps;
+ DenseMap<VPSingleDefRecipe *, unsigned> ScaledReductionMap;
+ for (const auto &[_, Chains] : ChainsByPhi)
+ for (const VPPartialReductionChain &Chain : Chains) {
+ PartialReductionOps.insert(Chain.BinOp);
+ ScaledReductionMap[Chain.ReductionBinOp] = Chain.ScaleFactor;
+ }
+
+ // A partial reduction is invalid if any of its extends are used by
+ // something that isn't another partial reduction. This is because the
+ // extends are intended to be lowered along with the reduction itself.
+ auto ExtendUsersValid = [&](VPWidenCastRecipe *Ext) {
+ return !Ext || all_of(Ext->users(), [&](VPUser *U) {
+ return PartialReductionOps.contains(cast<VPRecipeBase>(U));
+ });
+ };
+
+ // Validate chains: check that extends are only used by partial reductions,
+ // and that reduction bin ops are only used by other partial reductions with
+ // matching scale factors, are outside the loop region or the select
+ // introduced by tail-folding. Otherwise we would create users of scaled
+ // reductions where the types of the other operands don't match.
+ for (auto &[RedPhiR, Chains] : ChainsByPhi) {
+ for (const VPPartialReductionChain &Chain : Chains) {
+ if (!ExtendUsersValid(Chain.ExtendA) ||
+ !ExtendUsersValid(Chain.ExtendB)) {
+ Chains.clear();
+ break;
+ }
+ auto UseIsValid = [&, RedPhiR = RedPhiR](VPUser *U) {
+ if (auto *PhiR = dyn_cast<VPReductionPHIRecipe>(U))
+ return PhiR == RedPhiR;
+ auto *R = cast<VPSingleDefRecipe>(U);
+ return Chain.ScaleFactor == ScaledReductionMap.lookup_or(R, 0) ||
+ match(R, m_ComputeReductionResult(
+ m_Specific(Chain.ReductionBinOp))) ||
+ match(R, m_Select(m_VPValue(), m_Specific(Chain.ReductionBinOp),
+ m_Specific(RedPhiR)));
+ };
+ if (!all_of(Chain.ReductionBinOp->users(), UseIsValid)) {
+ Chains.clear();
+ break;
+ }
+
+ // Check if the compute-reduction-result is used by a sunk store.
+ // TODO: Also form partial reductions in those cases.
+ if (auto *RdxResult = vputils::findComputeReductionResult(RedPhiR)) {
+ if (any_of(RdxResult->users(), [](VPUser *U) {
+ auto *RepR = dyn_cast<VPReplicateRecipe>(U);
+ return RepR && isa<StoreInst>(RepR->getUnderlyingInstr());
+ })) {
+ Chains.clear();
+ break;
+ }
+ }
+ }
+ }
+
+ for (const auto &[_, Chains] : ChainsByPhi)
+ for (const VPPartialReductionChain &Chain : Chains)
+ transformToPartialReduction(Chain, Range, CostCtx, Plan);
+}
diff --git a/llvm/lib/Transforms/Vectorize/VPlanTransforms.h b/llvm/lib/Transforms/Vectorize/VPlanTransforms.h
index f33c672e9ea59..e5091b7329d34 100644
--- a/llvm/lib/Transforms/Vectorize/VPlanTransforms.h
+++ b/llvm/lib/Transforms/Vectorize/VPlanTransforms.h
@@ -29,6 +29,7 @@ class PHINode;
class ScalarEvolution;
class PredicatedScalarEvolution;
class TargetLibraryInfo;
+class TargetTransformInfo;
class VPBuilder;
class VPRecipeBuilder;
struct VFRange;
@@ -451,6 +452,12 @@ struct VPlanTransforms {
/// users in the original exit block using the VPIRInstruction wrapping to the
/// LCSSA phi.
static void addExitUsersForFirstOrderRecurrences(VPlan &Plan, VFRange &Range);
+
+ /// Detect and create partial reduction recipes for scaled reductions in
+ /// \p Plan. Must be called after recipe construction. If partial reductions
+ /// are only valid for a subset of VFs in Range, Range.End is updated.
+ static void createPartialReductions(VPlan &Plan, VPCostContext &CostCtx,
+ VFRange &Range);
};
} // namespace llvm
diff --git a/llvm/lib/Transforms/Vectorize/VPlanUtils.cpp b/llvm/lib/Transforms/Vectorize/VPlanUtils.cpp
index baf55c33194e3..c3f7c1117aa2a 100644
--- a/llvm/lib/Transforms/Vectorize/VPlanUtils.cpp
+++ b/llvm/lib/Transforms/Vectorize/VPlanUtils.cpp
@@ -586,3 +586,20 @@ vputils::getMemoryLocation(const VPRecipeBase &R) {
Loc.AATags.Scope = AliasScopeMD;
return Loc;
}
+
+/// Find the ComputeReductionResult recipe for \p PhiR, looking through selects
+/// inserted for predicated reductions or tail folding.
+VPInstruction *vputils::findComputeReductionResult(VPReductionPHIRecipe *PhiR) {
+ VPValue *BackedgeVal = PhiR->getBackedgeValue();
+ if (auto *Res = vputils::findUserOf<VPInstruction::ComputeReductionResult>(
+ BackedgeVal))
+ return Res;
+
+ // Look through selects inserted for tail folding or predicated reductions.
+ VPRecipeBase *SelR = vputils::findUserOf(
+ BackedgeVal, m_Select(m_VPValue(), m_VPValue(), m_VPValue()));
+ if (!SelR)
+ return nullptr;
+ return vputils::findUserOf<VPInstruction::ComputeReductionResult>(
+ cast<VPSingleDefRecipe>(SelR));
+}
diff --git a/llvm/lib/Transforms/Vectorize/VPlanUtils.h b/llvm/lib/Transforms/Vectorize/VPlanUtils.h
index 31248cca50543..a789f3cbb2815 100644
--- a/llvm/lib/Transforms/Vectorize/VPlanUtils.h
+++ b/llvm/lib/Transforms/Vectorize/VPlanUtils.h
@@ -10,6 +10,7 @@
#define LLVM_TRANSFORMS_VECTORIZE_VPLANUTILS_H
#include "VPlan.h"
+#include "VPlanPatternMatch.h"
#include "llvm/Support/Compiler.h"
namespace llvm {
@@ -125,6 +126,25 @@ inline VPRecipeBase *findRecipe(VPValue *Start, PredT Pred) {
return nullptr;
}
+/// If \p V is used by a recipe matching pattern \p P, return it. Otherwise
+/// return nullptr;
+template <typename MatchT>
+static VPRecipeBase *findUserOf(VPValue *V, const MatchT &P) {
+ auto It = find_if(V->users(), match_fn(P));
+ return It == V->user_end() ? nullptr : cast<VPRecipeBase>(*It);
+}
+
+/// If \p V is used by a VPInstruction with \p Opcode, return it. Otherwise
+/// return nullptr.
+template <unsigned Opcode> static VPInstruction *findUserOf(VPValue *V) {
+ using namespace llvm::VPlanPatternMatch;
+ return cast_or_null<VPInstruction>(findUserOf(V, m_VPInstruction<Opcode>()));
+}
+
+/// Find the ComputeReductionResult recipe for \p PhiR, looking through selects
+/// inserted for predicated reductions or tail folding.
+VPInstruction *findComputeReductionResult(VPReductionPHIRecipe *PhiR);
+
} // namespace vputils
//===----------------------------------------------------------------------===//
diff --git a/llvm/test/Transforms/LoopVectorize/vplan-print-after-all.ll b/llvm/test/Transforms/LoopVectorize/vplan-print-after-all.ll
index 96e15740cd413..9f0a528bfa889 100644
--- a/llvm/test/Transforms/LoopVectorize/vplan-print-after-all.ll
+++ b/llvm/test/Transforms/LoopVectorize/vplan-print-after-all.ll
@@ -9,6 +9,7 @@
; CHECK: VPlan after VPlanTransforms::handleMultiUseReductions
; CHECK: VPlan after VPlanTransforms::handleMaxMinNumReductions
; CHECK: VPlan after VPlanTransforms::handleFindLastReductions
+; CHECK: VPlan after VPlanTransforms::createPartialReductions
; CHECK: VPlan after VPlanTransforms::convertToAbstractRecipes
; CHECK: VPlan after VPlanTransforms::createInterleaveGroups
; CHECK: VPlan after VPlanTransforms::replaceSymbolicStrides
More information about the llvm-commits
mailing list