[llvm] d1e477b - Revert "[VPlan] Detect and create partial reductions in VPlan. (NFCI) (#167851)"
Martin Storsjö via llvm-commits
llvm-commits at lists.llvm.org
Sun Feb 1 04:36:58 PST 2026
Author: Martin Storsjö
Date: 2026-02-01T14:35:02+02:00
New Revision: d1e477b00b49c63ff4dd513eeb14a5b18bc055d7
URL: https://github.com/llvm/llvm-project/commit/d1e477b00b49c63ff4dd513eeb14a5b18bc055d7
DIFF: https://github.com/llvm/llvm-project/commit/d1e477b00b49c63ff4dd513eeb14a5b18bc055d7.diff
LOG: Revert "[VPlan] Detect and create partial reductions in VPlan. (NFCI) (#167851)"
This reverts commit f4e8cc1a2229dca76d21c8d37439c4c194b06b86.
This change wasn't NFC; it causes failed asserts when building
ffmpeg for i686 windows, see
https://github.com/llvm/llvm-project/pull/167851 for details.
Added:
Modified:
llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
llvm/lib/Transforms/Vectorize/VPRecipeBuilder.h
llvm/lib/Transforms/Vectorize/VPlan.h
llvm/lib/Transforms/Vectorize/VPlanConstruction.cpp
llvm/lib/Transforms/Vectorize/VPlanPatternMatch.h
llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp
llvm/lib/Transforms/Vectorize/VPlanTransforms.h
llvm/lib/Transforms/Vectorize/VPlanUtils.cpp
llvm/lib/Transforms/Vectorize/VPlanUtils.h
llvm/test/Transforms/LoopVectorize/vplan-print-after-all.ll
Removed:
################################################################################
diff --git a/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp b/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
index 383fc7bc50e72..bb5fae95eda0d 100644
--- a/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
+++ b/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
@@ -8036,6 +8036,209 @@ VPReplicateRecipe *VPRecipeBuilder::handleReplication(VPInstruction *VPI,
return Recipe;
}
+/// Find all possible partial reductions in the loop and track all of those that
+/// are valid so recipes can be formed later.
+void VPRecipeBuilder::collectScaledReductions(VFRange &Range) {
+ // Find all possible partial reductions, grouping chains by their PHI. This
+ // grouping allows invalidating the whole chain, if any link is not a valid
+ // partial reduction.
+ MapVector<Instruction *,
+ SmallVector<std::pair<PartialReductionChain, unsigned>>>
+ ChainsByPhi;
+ for (const auto &[Phi, RdxDesc] : Legal->getReductionVars()) {
+ if (Instruction *RdxExitInstr = RdxDesc.getLoopExitInstr())
+ getScaledReductions(Phi, RdxExitInstr, Range, ChainsByPhi[Phi]);
+ }
+
+ // A partial reduction is invalid if any of its extends are used by
+ // something that isn't another partial reduction. This is because the
+ // extends are intended to be lowered along with the reduction itself.
+
+ // Build up a set of partial reduction ops for efficient use checking.
+ SmallPtrSet<User *, 4> PartialReductionOps;
+ for (const auto &[_, Chains] : ChainsByPhi)
+ for (const auto &[PartialRdx, _] : Chains)
+ PartialReductionOps.insert(PartialRdx.ExtendUser);
+
+ auto ExtendIsOnlyUsedByPartialReductions =
+ [&PartialReductionOps](Instruction *Extend) {
+ return all_of(Extend->users(), [&](const User *U) {
+ return PartialReductionOps.contains(U);
+ });
+ };
+
+ // Check if each use of a chain's two extends is a partial reduction
+ // and only add those that don't have non-partial reduction users.
+ for (const auto &[_, Chains] : ChainsByPhi) {
+ for (const auto &[Chain, Scale] : Chains) {
+ if (ExtendIsOnlyUsedByPartialReductions(Chain.ExtendA) &&
+ (!Chain.ExtendB ||
+ ExtendIsOnlyUsedByPartialReductions(Chain.ExtendB)))
+ ScaledReductionMap.try_emplace(Chain.Reduction, Scale);
+ }
+ }
+
+ // Check that all partial reductions in a chain are only used by other
+ // partial reductions with the same scale factor. Otherwise we end up creating
+ // users of scaled reductions where the types of the other operands don't
+ // match.
+ for (const auto &[Phi, Chains] : ChainsByPhi) {
+ for (const auto &[Chain, Scale] : Chains) {
+ auto AllUsersPartialRdx = [ScaleVal = Scale, RdxPhi = Phi,
+ this](const User *U) {
+ auto *UI = cast<Instruction>(U);
+ if (isa<PHINode>(UI) && UI->getParent() == OrigLoop->getHeader())
+ return UI == RdxPhi;
+ return ScaledReductionMap.lookup_or(UI, 0) == ScaleVal ||
+ !OrigLoop->contains(UI->getParent());
+ };
+
+ // If any partial reduction entry for the phi is invalid, invalidate the
+ // whole chain.
+ if (!all_of(Chain.Reduction->users(), AllUsersPartialRdx)) {
+ for (const auto &[Chain, _] : Chains)
+ ScaledReductionMap.erase(Chain.Reduction);
+ break;
+ }
+ }
+ }
+}
+
+bool VPRecipeBuilder::getScaledReductions(
+ Instruction *PHI, Instruction *RdxExitInstr, VFRange &Range,
+ SmallVectorImpl<std::pair<PartialReductionChain, unsigned>> &Chains) {
+ if (!CM.TheLoop->contains(RdxExitInstr))
+ return false;
+
+ auto *Update = dyn_cast<BinaryOperator>(RdxExitInstr);
+ if (!Update)
+ return false;
+
+ Value *Op = Update->getOperand(0);
+ Value *PhiOp = Update->getOperand(1);
+ if (Op == PHI)
+ std::swap(Op, PhiOp);
+
+ using namespace llvm::PatternMatch;
+ // If Op is an extend, then it's still a valid partial reduction if the
+ // extended mul fulfills the other requirements.
+ // For example, reduce.add(ext(mul(ext(A), ext(B)))) is still a valid partial
+ // reduction since the inner extends will be widened. We already have oneUse
+ // checks on the inner extends so widening them is safe.
+ std::optional<TTI::PartialReductionExtendKind> OuterExtKind = std::nullopt;
+ if (match(Op, m_ZExtOrSExt(m_Mul(m_Value(), m_Value())))) {
+ auto *Cast = cast<CastInst>(Op);
+ OuterExtKind = TTI::getPartialReductionExtendKind(Cast->getOpcode());
+ Op = Cast->getOperand(0);
+ }
+
+ // Try and get a scaled reduction from the first non-phi operand.
+ // If one is found, we use the discovered reduction instruction in
+ // place of the accumulator for costing.
+ if (auto *OpInst = dyn_cast<Instruction>(Op)) {
+ if (getScaledReductions(PHI, OpInst, Range, Chains)) {
+ PHI = Chains.rbegin()->first.Reduction;
+
+ Op = Update->getOperand(0);
+ PhiOp = Update->getOperand(1);
+ if (Op == PHI)
+ std::swap(Op, PhiOp);
+ }
+ }
+ if (PhiOp != PHI)
+ return false;
+
+ // If the update is a binary operator, check both of its operands to see if
+ // they are extends. Otherwise, see if the update comes directly from an
+ // extend.
+ Instruction *Exts[2] = {nullptr};
+ BinaryOperator *ExtendUser = dyn_cast<BinaryOperator>(Op);
+ std::optional<unsigned> BinOpc;
+ Type *ExtOpTypes[2] = {nullptr};
+ TTI::PartialReductionExtendKind ExtKinds[2] = {TTI::PR_None};
+
+ auto CollectExtInfo = [this, OuterExtKind, &Exts, &ExtOpTypes,
+ &ExtKinds](SmallVectorImpl<Value *> &Ops) -> bool {
+ for (const auto &[I, OpI] : enumerate(Ops)) {
+ const APInt *C;
+ if (I > 0 && match(OpI, m_APInt(C)) &&
+ canConstantBeExtended(C, ExtOpTypes[0], ExtKinds[0])) {
+ ExtOpTypes[I] = ExtOpTypes[0];
+ ExtKinds[I] = ExtKinds[0];
+ continue;
+ }
+ Value *ExtOp;
+ if (!match(OpI, m_ZExtOrSExt(m_Value(ExtOp))) &&
+ !match(OpI, m_FPExt(m_Value(ExtOp))))
+ return false;
+ Exts[I] = cast<Instruction>(OpI);
+
+ // TODO: We should be able to support live-ins.
+ if (!CM.TheLoop->contains(Exts[I]))
+ return false;
+
+ ExtOpTypes[I] = ExtOp->getType();
+ ExtKinds[I] = TTI::getPartialReductionExtendKind(Exts[I]);
+ // The outer extend kind must be the same as the inner extends, so that
+ // they can be folded together.
+ if (OuterExtKind.has_value() && OuterExtKind.value() != ExtKinds[I])
+ return false;
+ }
+ return true;
+ };
+
+ if (ExtendUser) {
+ if (!ExtendUser->hasOneUse())
+ return false;
+
+ // Use the side-effect of match to replace BinOp only if the pattern is
+ // matched, we don't care at this point whether it actually matched.
+ match(ExtendUser, m_Neg(m_BinOp(ExtendUser)));
+
+ SmallVector<Value *> Ops(ExtendUser->operands());
+ if (!CollectExtInfo(Ops))
+ return false;
+
+ BinOpc = std::make_optional(ExtendUser->getOpcode());
+ } else if (match(Update, m_Add(m_Value(), m_Value())) ||
+ match(Update, m_FAdd(m_Value(), m_Value()))) {
+ // We already know the operands for Update are Op and PhiOp.
+ SmallVector<Value *> Ops({Op});
+ if (!CollectExtInfo(Ops))
+ return false;
+
+ ExtendUser = Update;
+ BinOpc = std::nullopt;
+ } else
+ return false;
+
+ PartialReductionChain Chain(RdxExitInstr, Exts[0], Exts[1], ExtendUser);
+
+ TypeSize PHISize = PHI->getType()->getPrimitiveSizeInBits();
+ TypeSize ASize = ExtOpTypes[0]->getPrimitiveSizeInBits();
+ if (!PHISize.hasKnownScalarFactor(ASize))
+ return false;
+ unsigned TargetScaleFactor = PHISize.getKnownScalarFactor(ASize);
+
+ if (LoopVectorizationPlanner::getDecisionAndClampRange(
+ [&](ElementCount VF) {
+ std::optional<FastMathFlags> FMF = std::nullopt;
+ if (Update->getOpcode() == Instruction::FAdd)
+ FMF = Update->getFastMathFlags();
+ InstructionCost Cost = TTI->getPartialReductionCost(
+ Update->getOpcode(), ExtOpTypes[0], ExtOpTypes[1],
+ PHI->getType(), VF, ExtKinds[0], ExtKinds[1], BinOpc,
+ CM.CostKind, FMF);
+ return Cost.isValid();
+ },
+ Range)) {
+ Chains.emplace_back(Chain, TargetScaleFactor);
+ return true;
+ }
+
+ return false;
+}
+
VPRecipeBase *
VPRecipeBuilder::tryToCreateWidenNonPhiRecipe(VPSingleDefRecipe *R,
VFRange &Range) {
@@ -8066,6 +8269,9 @@ VPRecipeBuilder::tryToCreateWidenNonPhiRecipe(VPSingleDefRecipe *R,
VPI->getOpcode() == Instruction::Store)
return tryToWidenMemory(VPI, Range);
+ if (std::optional<unsigned> ScaleFactor = getScalingForReduction(Instr))
+ return tryToCreatePartialReduction(VPI, ScaleFactor.value());
+
if (!shouldWiden(Instr, Range))
return nullptr;
@@ -8084,6 +8290,55 @@ VPRecipeBuilder::tryToCreateWidenNonPhiRecipe(VPSingleDefRecipe *R,
return tryToWiden(VPI);
}
+VPRecipeBase *
+VPRecipeBuilder::tryToCreatePartialReduction(VPInstruction *Reduction,
+ unsigned ScaleFactor) {
+ assert(Reduction->getNumOperands() == 2 &&
+ "Unexpected number of operands for partial reduction");
+
+ VPValue *BinOp = Reduction->getOperand(0);
+ VPValue *Accumulator = Reduction->getOperand(1);
+ VPRecipeBase *BinOpRecipe = BinOp->getDefiningRecipe();
+ if (isa<VPReductionPHIRecipe>(BinOpRecipe) ||
+ (isa<VPReductionRecipe>(BinOpRecipe) &&
+ cast<VPReductionRecipe>(BinOpRecipe)->isPartialReduction()))
+ std::swap(BinOp, Accumulator);
+
+ if (auto *RedPhiR = dyn_cast<VPReductionPHIRecipe>(Accumulator))
+ RedPhiR->setVFScaleFactor(ScaleFactor);
+
+ assert(ScaleFactor ==
+ vputils::getVFScaleFactor(Accumulator->getDefiningRecipe()) &&
+ "all accumulators in chain must have same scale factor");
+
+ auto *ReductionI = Reduction->getUnderlyingInstr();
+ assert(
+ Reduction->getOpcode() != Instruction::FAdd ||
+ (ReductionI->hasAllowReassoc() && ReductionI->hasAllowContract()) &&
+ "FAdd partial reduction requires allow-reassoc and allow-contract");
+ if (Reduction->getOpcode() == Instruction::Sub) {
+ SmallVector<VPValue *, 2> Ops;
+ Ops.push_back(Plan.getConstantInt(ReductionI->getType(), 0));
+ Ops.push_back(BinOp);
+ BinOp = new VPWidenRecipe(*ReductionI, Ops, VPIRFlags(*ReductionI),
+ VPIRMetadata(), ReductionI->getDebugLoc());
+ Builder.insert(BinOp->getDefiningRecipe());
+ }
+
+ VPValue *Cond = nullptr;
+ if (CM.blockNeedsPredicationForAnyReason(ReductionI->getParent()))
+ Cond = getBlockInMask(Builder.getInsertBlock());
+
+ return new VPReductionRecipe(
+ Reduction->getOpcode() == Instruction::FAdd ? RecurKind::FAdd
+ : RecurKind::Add,
+ Reduction->getOpcode() == Instruction::FAdd
+ ? Reduction->getFastMathFlags()
+ : FastMathFlags(),
+ ReductionI, Accumulator, BinOp, Cond,
+ RdxUnordered{/*VFScaleFactor=*/ScaleFactor}, ReductionI->getDebugLoc());
+}
+
void LoopVectorizationPlanner::buildVPlansWithVPRecipes(ElementCount MinVF,
ElementCount MaxVF) {
if (ElementCount::isKnownGT(MinVF, MaxVF))
@@ -8220,7 +8475,11 @@ VPlanPtr LoopVectorizationPlanner::tryToBuildVPlanWithVPRecipes(
// Construct wide recipes and apply predication for original scalar
// VPInstructions in the loop.
// ---------------------------------------------------------------------------
- VPRecipeBuilder RecipeBuilder(*Plan, TLI, Legal, CM, Builder, BlockMaskCache);
+ VPRecipeBuilder RecipeBuilder(*Plan, OrigLoop, TLI, &TTI, Legal, CM, Builder,
+ BlockMaskCache);
+ // TODO: Handle partial reductions with EVL tail folding.
+ if (!CM.foldTailWithEVL())
+ RecipeBuilder.collectScaledReductions(Range);
// Scan the body of the loop in a topological order to visit each basic block
// after having visited its predecessor basic blocks.
@@ -8342,16 +8601,13 @@ VPlanPtr LoopVectorizationPlanner::tryToBuildVPlanWithVPRecipes(
if (!RUN_VPLAN_PASS(VPlanTransforms::handleFindLastReductions, *Plan))
return nullptr;
- // Create partial reduction recipes for scaled reductions and transform
- // recipes to abstract recipes if it is legal and beneficial and clamp the
- // range for better cost estimation.
+ // Transform recipes to abstract recipes if it is legal and beneficial and
+ // clamp the range for better cost estimation.
// TODO: Enable following transform when the EVL-version of extended-reduction
// and mulacc-reduction are implemented.
if (!CM.foldTailWithEVL()) {
VPCostContext CostCtx(CM.TTI, *CM.TLI, *Plan, CM, CM.CostKind, CM.PSE,
OrigLoop);
- RUN_VPLAN_PASS(VPlanTransforms::createPartialReductions, *Plan, CostCtx,
- Range);
RUN_VPLAN_PASS(VPlanTransforms::convertToAbstractRecipes, *Plan, CostCtx,
Range);
}
@@ -8646,7 +8902,11 @@ void LoopVectorizationPlanner::addReductionResultComputation(
VPBuilder PHBuilder(Plan->getVectorPreheader());
VPValue *Iden = Plan->getOrAddLiveIn(
getRecurrenceIdentity(RK, PhiTy, RdxDesc.getFastMathFlags()));
- auto *ScaleFactorVPV = Plan->getConstantInt(32, 1);
+ // If the PHI is used by a partial reduction, set the scale factor.
+ unsigned ScaleFactor =
+ RecipeBuilder.getScalingForReduction(RdxDesc.getLoopExitInstr())
+ .value_or(1);
+ auto *ScaleFactorVPV = Plan->getConstantInt(32, ScaleFactor);
VPValue *StartV = PHBuilder.createNaryOp(
VPInstruction::ReductionStartVector,
{PhiR->getStartValue(), Iden, ScaleFactorVPV},
diff --git a/llvm/lib/Transforms/Vectorize/VPRecipeBuilder.h b/llvm/lib/Transforms/Vectorize/VPRecipeBuilder.h
index 3280fc4827239..534f51e86a9b7 100644
--- a/llvm/lib/Transforms/Vectorize/VPRecipeBuilder.h
+++ b/llvm/lib/Transforms/Vectorize/VPRecipeBuilder.h
@@ -19,17 +19,44 @@ namespace llvm {
class LoopVectorizationLegality;
class LoopVectorizationCostModel;
class TargetLibraryInfo;
+class TargetTransformInfo;
struct HistogramInfo;
struct VFRange;
+/// A chain of instructions that form a partial reduction.
+/// Designed to match either:
+/// reduction_bin_op (extend (A), accumulator), or
+/// reduction_bin_op (bin_op (extend (A), (extend (B))), accumulator).
+struct PartialReductionChain {
+ PartialReductionChain(Instruction *Reduction, Instruction *ExtendA,
+ Instruction *ExtendB, Instruction *ExtendUser)
+ : Reduction(Reduction), ExtendA(ExtendA), ExtendB(ExtendB),
+ ExtendUser(ExtendUser) {}
+ /// The top-level binary operation that forms the reduction to a scalar
+ /// after the loop body.
+ Instruction *Reduction;
+ /// The extension of each of the inner binary operation's operands.
+ Instruction *ExtendA;
+ Instruction *ExtendB;
+
+ /// The user of the extends that is then reduced.
+ Instruction *ExtendUser;
+};
+
/// Helper class to create VPRecipies from IR instructions.
class VPRecipeBuilder {
/// The VPlan new recipes are added to.
VPlan &Plan;
+ /// The loop that we evaluate.
+ Loop *OrigLoop;
+
/// Target Library Info.
const TargetLibraryInfo *TLI;
+ // Target Transform Info.
+ const TargetTransformInfo *TTI;
+
/// The legality analysis.
LoopVectorizationLegality *Legal;
@@ -52,6 +79,9 @@ class VPRecipeBuilder {
/// created.
SmallVector<VPHeaderPHIRecipe *, 4> PhisToFix;
+ /// A mapping of partial reduction exit instructions to their scaling factor.
+ DenseMap<const Instruction *, unsigned> ScaledReductionMap;
+
/// Check if \p I can be widened at the start of \p Range and possibly
/// decrease the range such that the returned value holds for the entire \p
/// Range. The function should not be called for memory instructions or calls.
@@ -84,19 +114,48 @@ class VPRecipeBuilder {
VPHistogramRecipe *tryToWidenHistogram(const HistogramInfo *HI,
VPInstruction *VPI);
+ /// Examines reduction operations to see if the target can use a cheaper
+ /// operation with a wider per-iteration input VF and narrower PHI VF.
+ /// Each element within Chains is a pair with a struct containing reduction
+ /// information and the scaling factor between the number of elements in
+ /// the input and output.
+ /// Recursively calls itself to identify chained scaled reductions.
+ /// Returns true if this invocation added an entry to Chains, otherwise false.
+ /// i.e. returns false in the case that a subcall adds an entry to Chains,
+ /// but the top-level call does not.
+ bool getScaledReductions(
+ Instruction *PHI, Instruction *RdxExitInstr, VFRange &Range,
+ SmallVectorImpl<std::pair<PartialReductionChain, unsigned>> &Chains);
+
public:
- VPRecipeBuilder(VPlan &Plan, const TargetLibraryInfo *TLI,
+ VPRecipeBuilder(VPlan &Plan, Loop *OrigLoop, const TargetLibraryInfo *TLI,
+ const TargetTransformInfo *TTI,
LoopVectorizationLegality *Legal,
LoopVectorizationCostModel &CM, VPBuilder &Builder,
DenseMap<VPBasicBlock *, VPValue *> &BlockMaskCache)
- : Plan(Plan), TLI(TLI), Legal(Legal), CM(CM), Builder(Builder),
- BlockMaskCache(BlockMaskCache) {}
+ : Plan(Plan), OrigLoop(OrigLoop), TLI(TLI), TTI(TTI), Legal(Legal),
+ CM(CM), Builder(Builder), BlockMaskCache(BlockMaskCache) {}
+
+ std::optional<unsigned> getScalingForReduction(const Instruction *ExitInst) {
+ auto It = ScaledReductionMap.find(ExitInst);
+ return It == ScaledReductionMap.end() ? std::nullopt
+ : std::make_optional(It->second);
+ }
+
+ /// Find all possible partial reductions in the loop and track all of those
+ /// that are valid so recipes can be formed later.
+ void collectScaledReductions(VFRange &Range);
/// Create and return a widened recipe for a non-phi recipe \p R if one can be
/// created within the given VF \p Range.
VPRecipeBase *tryToCreateWidenNonPhiRecipe(VPSingleDefRecipe *R,
VFRange &Range);
+ /// Create and return a partial reduction recipe for a reduction instruction
+ /// along with binary operation and reduction phi operands.
+ VPRecipeBase *tryToCreatePartialReduction(VPInstruction *Reduction,
+ unsigned ScaleFactor);
+
/// Set the recipe created for given ingredient.
void setRecipe(Instruction *I, VPRecipeBase *R) {
assert(!Ingredient2Recipe.contains(I) &&
diff --git a/llvm/lib/Transforms/Vectorize/VPlan.h b/llvm/lib/Transforms/Vectorize/VPlan.h
index ae6988d86e194..48c156a538d00 100644
--- a/llvm/lib/Transforms/Vectorize/VPlan.h
+++ b/llvm/lib/Transforms/Vectorize/VPlan.h
@@ -1641,19 +1641,11 @@ class LLVM_ABI_FOR_TEST VPWidenRecipe : public VPRecipeWithIRFlags,
setUnderlyingValue(&I);
}
- VPWidenRecipe(unsigned Opcode, ArrayRef<VPValue *> Operands,
- const VPIRFlags &Flags = {}, const VPIRMetadata &Metadata = {},
- DebugLoc DL = {})
- : VPRecipeWithIRFlags(VPRecipeBase::VPWidenSC, Operands, Flags, DL),
- VPIRMetadata(Metadata), Opcode(Opcode) {}
-
~VPWidenRecipe() override = default;
VPWidenRecipe *clone() override {
- if (auto *UV = getUnderlyingValue())
- return new VPWidenRecipe(*cast<Instruction>(UV), operands(), *this, *this,
- getDebugLoc());
- return new VPWidenRecipe(Opcode, operands(), *this, *this, getDebugLoc());
+ return new VPWidenRecipe(*getUnderlyingInstr(), operands(), *this, *this,
+ getDebugLoc());
}
VP_CLASSOF_IMPL(VPRecipeBase::VPWidenSC)
diff --git a/llvm/lib/Transforms/Vectorize/VPlanConstruction.cpp b/llvm/lib/Transforms/Vectorize/VPlanConstruction.cpp
index 4e7b1bbab5227..a84c400fd411b 100644
--- a/llvm/lib/Transforms/Vectorize/VPlanConstruction.cpp
+++ b/llvm/lib/Transforms/Vectorize/VPlanConstruction.cpp
@@ -16,15 +16,12 @@
#include "VPlanAnalysis.h"
#include "VPlanCFG.h"
#include "VPlanDominatorTree.h"
-#include "VPlanHelpers.h"
#include "VPlanPatternMatch.h"
#include "VPlanTransforms.h"
-#include "VPlanUtils.h"
#include "llvm/Analysis/LoopInfo.h"
#include "llvm/Analysis/LoopIterator.h"
#include "llvm/Analysis/ScalarEvolution.h"
#include "llvm/Analysis/ScalarEvolutionExpressions.h"
-#include "llvm/Analysis/TargetTransformInfo.h"
#include "llvm/IR/InstrTypes.h"
#include "llvm/IR/MDBuilder.h"
#include "llvm/Transforms/Utils/LoopUtils.h"
@@ -1138,6 +1135,37 @@ void VPlanTransforms::addMinimumVectorEpilogueIterationCheck(
Branch->setMetadata(LLVMContext::MD_prof, BranchWeights);
}
+/// If \p V is used by a recipe matching pattern \p P, return it. Otherwise
+/// return nullptr;
+template <typename MatchT>
+static VPRecipeBase *findUserOf(VPValue *V, const MatchT &P) {
+ auto It = find_if(V->users(), match_fn(P));
+ return It == V->user_end() ? nullptr : cast<VPRecipeBase>(*It);
+}
+
+/// If \p V is used by a VPInstruction with \p Opcode, return it. Otherwise
+/// return nullptr.
+template <unsigned Opcode> static VPInstruction *findUserOf(VPValue *V) {
+ return cast_or_null<VPInstruction>(findUserOf(V, m_VPInstruction<Opcode>()));
+}
+
+/// Find the ComputeReductionResult recipe for \p PhiR, looking through selects
+/// inserted for predicated reductions or tail folding.
+static VPInstruction *findComputeReductionResult(VPReductionPHIRecipe *PhiR) {
+ VPValue *BackedgeVal = PhiR->getBackedgeValue();
+ if (auto *Res =
+ findUserOf<VPInstruction::ComputeReductionResult>(BackedgeVal))
+ return Res;
+
+ // Look through selects inserted for tail folding or predicated reductions.
+ VPRecipeBase *SelR =
+ findUserOf(BackedgeVal, m_Select(m_VPValue(), m_VPValue(), m_VPValue()));
+ if (!SelR)
+ return nullptr;
+ return findUserOf<VPInstruction::ComputeReductionResult>(
+ cast<VPSingleDefRecipe>(SelR));
+}
+
/// Find and return the final select instruction of the FindIV result pattern
/// for the given \p BackedgeVal:
/// select(icmp ne ComputeReductionResult(ReducedIV), Sentinel),
@@ -1246,7 +1274,7 @@ bool VPlanTransforms::handleMaxMinNumReductions(VPlan &Plan) {
// If we exit early due to NaNs, compute the final reduction result based on
// the reduction phi at the beginning of the last vector iteration.
- auto *RdxResult = vputils::findComputeReductionResult(RedPhiR);
+ auto *RdxResult = findComputeReductionResult(RedPhiR);
assert(RdxResult && "must find a ComputeReductionResult");
auto *NewSel = MiddleBuilder.createSelect(AnyNaNLane, RedPhiR,
@@ -1374,7 +1402,7 @@ bool VPlanTransforms::handleFindLastReductions(VPlan &Plan) {
// Find final reduction computation and replace it with an
// extract.last.active intrinsic.
auto *RdxResult =
- vputils::findUserOf<VPInstruction::ComputeReductionResult>(DataSelect);
+ findUserOf<VPInstruction::ComputeReductionResult>(DataSelect);
// TODO: Handle tail-folding.
if (!RdxResult)
return false;
@@ -1433,7 +1461,7 @@ bool VPlanTransforms::handleMultiUseReductions(VPlan &Plan) {
VPValue *CmpOpA;
VPValue *CmpOpB;
CmpPredicate Pred;
- auto *Cmp = dyn_cast_or_null<VPRecipeWithIRFlags>(vputils::findUserOf(
+ auto *Cmp = dyn_cast_or_null<VPRecipeWithIRFlags>(findUserOf(
MinOrMaxPhiR, m_Cmp(Pred, m_VPValue(CmpOpA), m_VPValue(CmpOpB))));
if (!Cmp || Cmp->getNumUsers() != 1 ||
(CmpOpA != MinOrMaxOpValue && CmpOpB != MinOrMaxOpValue))
@@ -1449,7 +1477,7 @@ bool VPlanTransforms::handleMultiUseReductions(VPlan &Plan) {
return false;
VPInstruction *MinOrMaxResult =
- vputils::findUserOf<VPInstruction::ComputeReductionResult>(MinOrMaxOp);
+ findUserOf<VPInstruction::ComputeReductionResult>(MinOrMaxOp);
assert(is_contained(MinOrMaxPhiR->users(), MinOrMaxOp) &&
"one user must be MinOrMaxOp");
assert(MinOrMaxResult &&
diff --git a/llvm/lib/Transforms/Vectorize/VPlanPatternMatch.h b/llvm/lib/Transforms/Vectorize/VPlanPatternMatch.h
index 9428070b02f1a..6f605ac877e1b 100644
--- a/llvm/lib/Transforms/Vectorize/VPlanPatternMatch.h
+++ b/llvm/lib/Transforms/Vectorize/VPlanPatternMatch.h
@@ -597,18 +597,6 @@ m_FMul(const Op0_t &Op0, const Op1_t &Op1) {
return m_Binary<Instruction::FMul, Op0_t, Op1_t>(Op0, Op1);
}
-template <typename Op0_t, typename Op1_t>
-inline AllRecipe_match<Instruction::FAdd, Op0_t, Op1_t>
-m_FAdd(const Op0_t &Op0, const Op1_t &Op1) {
- return m_Binary<Instruction::FAdd, Op0_t, Op1_t>(Op0, Op1);
-}
-
-template <typename Op0_t, typename Op1_t>
-inline AllRecipe_commutative_match<Instruction::FAdd, Op0_t, Op1_t>
-m_c_FAdd(const Op0_t &Op0, const Op1_t &Op1) {
- return m_c_Binary<Instruction::FAdd, Op0_t, Op1_t>(Op0, Op1);
-}
-
template <typename Op0_t, typename Op1_t>
inline AllRecipe_match<Instruction::UDiv, Op0_t, Op1_t>
m_UDiv(const Op0_t &Op0, const Op1_t &Op1) {
diff --git a/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp b/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp
index 117337ec47c1a..501a23478f66f 100644
--- a/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp
+++ b/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp
@@ -908,7 +908,8 @@ getOptimizableIVOf(VPValue *VPV, PredicatedScalarEvolution &PSE) {
case Instruction::Add:
return match(VPV, m_c_Add(m_Specific(WideIV), m_Specific(IVStep)));
case Instruction::FAdd:
- return match(VPV, m_c_FAdd(m_Specific(WideIV), m_Specific(IVStep)));
+ return match(VPV, m_c_Binary<Instruction::FAdd>(m_Specific(WideIV),
+ m_Specific(IVStep)));
case Instruction::FSub:
return match(VPV, m_Binary<Instruction::FSub>(m_Specific(WideIV),
m_Specific(IVStep)));
@@ -5388,360 +5389,3 @@ void VPlanTransforms::addExitUsersForFirstOrderRecurrences(VPlan &Plan,
}
}
}
-
-namespace {
-
-/// A chain of recipes that form a partial reduction. Matches either
-/// reduction_bin_op (extend (A), accumulator), or
-/// reduction_bin_op (bin_op (extend (A), (extend (B))), accumulator).
-struct VPPartialReductionChain {
- /// The top-level binary operation that forms the reduction to a scalar
- /// after the loop body.
- VPWidenRecipe *ReductionBinOp;
- /// The extension of each of the inner binary operation's operands.
- VPWidenCastRecipe *ExtendA;
- VPWidenCastRecipe *ExtendB;
- /// The user of the extends that is then reduced.
- VPWidenRecipe *BinOp;
- unsigned ScaleFactor;
-};
-
-// Helper to transform a partial reduction chain into a partial reduction
-// recipe. Returns true if transformation succeeded. Checks profitability and
-// clamps VF range.
-static bool transformToPartialReduction(const VPPartialReductionChain &Chain,
- VFRange &Range, VPCostContext &CostCtx,
- VPlan &Plan) {
- VPWidenRecipe *WidenRecipe = Chain.ReductionBinOp;
- unsigned ScaleFactor = Chain.ScaleFactor;
- assert(WidenRecipe->getNumOperands() == 2 && "Expected binary operation");
-
- VPValue *BinOp = WidenRecipe->getOperand(0);
- VPValue *Accumulator = WidenRecipe->getOperand(1);
-
- // Swap if needed to ensure Accumulator is the PHI or partial reduction.
- if (isa_and_present<VPReductionPHIRecipe, VPReductionRecipe>(BinOp))
- std::swap(BinOp, Accumulator);
-
- // For chained reductions, only transform if accumulator is already a PHI or
- // partial reduction. Otherwise, it needs to be transformed first.
- auto *AccumRecipe = Accumulator->getDefiningRecipe();
- if (!isa_and_present<VPReductionPHIRecipe, VPReductionRecipe>(AccumRecipe))
- return false;
-
- // Check if the partial reduction is profitable for the VF range.
- Type *PhiType = CostCtx.Types.inferScalarType(Accumulator);
-
- // Derive extend info from the stored extends.
- auto GetExtInfo = [&CostCtx](VPWidenCastRecipe *Ext)
- -> std::pair<Type *, TargetTransformInfo::PartialReductionExtendKind> {
- if (!Ext)
- return {nullptr, TargetTransformInfo::PR_None};
- Type *ExtOpType = CostCtx.Types.inferScalarType(Ext->getOperand(0));
- auto ExtKind = TargetTransformInfo::getPartialReductionExtendKind(
- static_cast<Instruction::CastOps>(Ext->getOpcode()));
- return {ExtOpType, ExtKind};
- };
- auto ExtInfoA = GetExtInfo(Chain.ExtendA);
- auto ExtInfoB = GetExtInfo(Chain.ExtendB);
- Type *ExtOpTypeA = ExtInfoA.first;
- Type *ExtOpTypeB = ExtInfoB.first;
- auto ExtKindA = ExtInfoA.second;
- auto ExtKindB = ExtInfoB.second;
- // If ExtendB is nullptr but there's a separate BinOp, the second operand
- // was a constant that can use the same extend kind as the first.
- if (!Chain.ExtendB && Chain.BinOp && Chain.BinOp != Chain.ReductionBinOp) {
- // Validate that the constant can be extended to the narrow type.
- const APInt *Const = nullptr;
- for (VPValue *Op : Chain.BinOp->operands()) {
- if (match(Op, m_APInt(Const)))
- break;
- }
- if (!Const || !canConstantBeExtended(Const, ExtOpTypeA, ExtKindA))
- return false;
- ExtOpTypeB = ExtOpTypeA;
- ExtKindB = ExtKindA;
- }
-
- // BinOpc is only set when there's a separate binary op (not when BinOp is
- // the reduction itself).
- std::optional<unsigned> BinOpc =
- (Chain.BinOp && Chain.BinOp != Chain.ReductionBinOp)
- ? std::make_optional(Chain.BinOp->getOpcode())
- : std::nullopt;
-
- if (!LoopVectorizationPlanner::getDecisionAndClampRange(
- [&](ElementCount VF) {
- return CostCtx.TTI
- .getPartialReductionCost(
- WidenRecipe->getOpcode(), ExtOpTypeA, ExtOpTypeB, PhiType,
- VF, ExtKindA, ExtKindB, BinOpc, CostCtx.CostKind,
- PhiType->isFloatingPointTy()
- ? std::optional{WidenRecipe->getFastMathFlags()}
- : std::nullopt)
- .isValid();
- },
- Range))
- return false;
-
- VPValue *Cond = nullptr;
- VPValue *ExitValue = nullptr;
- if (auto *RdxPhi = dyn_cast<VPReductionPHIRecipe>(AccumRecipe)) {
- assert(RdxPhi->getVFScaleFactor() == 1 && "scale factor must not be set");
- RdxPhi->setVFScaleFactor(ScaleFactor);
-
- // Update ReductionStartVector instruction scale factor.
- VPValue *StartValue = RdxPhi->getOperand(0);
- auto *StartInst = cast<VPInstruction>(StartValue);
- assert(StartInst->getOpcode() == VPInstruction::ReductionStartVector);
- auto *NewScaleFactor = Plan.getConstantInt(32, ScaleFactor);
- StartInst->setOperand(2, NewScaleFactor);
-
- // Find the ComputeReductionResult that uses the WidenRecipe (the exit
- // value). Look through selects for predicated reductions.
- if (auto *RdxResult = vputils::findComputeReductionResult(RdxPhi)) {
- ExitValue = RdxResult->getOperand(0);
- match(ExitValue, m_Select(m_VPValue(Cond), m_VPValue(), m_VPValue()));
- }
- }
-
- // Handle SUB by negating the operand and using ADD for the partial reduction.
- if (WidenRecipe->getOpcode() == Instruction::Sub) {
- VPBuilder Builder(WidenRecipe);
- Type *ElemTy = CostCtx.Types.inferScalarType(BinOp);
- auto *Zero = Plan.getConstantInt(ElemTy, 0);
- VPIRFlags Flags = WidenRecipe->getUnderlyingInstr()
- ? VPIRFlags(*WidenRecipe->getUnderlyingInstr())
- : VPIRFlags();
- auto *NegRecipe = new VPWidenRecipe(Instruction::Sub, {Zero, BinOp}, Flags,
- VPIRMetadata(), DebugLoc::getUnknown());
- Builder.insert(NegRecipe);
- BinOp = NegRecipe;
- }
-
- RecurKind RdxKind =
- PhiType->isFloatingPointTy() ? RecurKind::FAdd : RecurKind::Add;
- auto *PartialRed = new VPReductionRecipe(
- RdxKind,
- RdxKind == RecurKind::FAdd ? WidenRecipe->getFastMathFlags()
- : FastMathFlags(),
- WidenRecipe->getUnderlyingInstr(), Accumulator, BinOp, Cond,
- RdxUnordered{/*VFScaleFactor=*/ScaleFactor});
- PartialRed->insertBefore(WidenRecipe);
-
- if (Cond)
- ExitValue->replaceAllUsesWith(PartialRed);
- WidenRecipe->replaceAllUsesWith(PartialRed);
- return true;
-}
-
-/// Examines reduction operations to see if the target can use a cheaper
-/// operation with a wider per-iteration input VF and narrower PHI VF.
-/// Recursively calls itself to identify chained scaled reductions.
-/// Returns true if this invocation added an entry to Chains, otherwise false.
-static bool
-getScaledReductions(VPSingleDefRecipe *RedPhiR, VPValue *PrevValue,
- SmallVectorImpl<VPPartialReductionChain> &Chains,
- VPTypeAnalysis &TypeInfo) {
- auto *UpdateR = dyn_cast<VPWidenRecipe>(PrevValue);
- if (!UpdateR || !Instruction::isBinaryOp(UpdateR->getOpcode()))
- return false;
-
- VPValue *Op = UpdateR->getOperand(0);
- VPValue *PhiOp = UpdateR->getOperand(1);
- if (Op == RedPhiR)
- std::swap(Op, PhiOp);
-
- // If Op is an extend, then it's still a valid partial reduction if the
- // extended mul fulfills the other requirements.
- // For example, reduce.add(ext(mul(ext(A), ext(B)))) is still a valid partial
- // reduction since the inner extends will be widened. We already have oneUse
- // checks on the inner extends so widening them is safe.
- std::optional<TTI::PartialReductionExtendKind> OuterExtKind;
- if (match(Op, m_ZExtOrSExt(m_Mul(m_VPValue(), m_VPValue()))) ||
- match(Op, m_FPExt(m_FMul(m_VPValue(), m_VPValue())))) {
- auto *CastRecipe = cast<VPWidenCastRecipe>(Op);
- auto CastOp = static_cast<Instruction::CastOps>(CastRecipe->getOpcode());
- OuterExtKind = TTI::getPartialReductionExtendKind(CastOp);
- Op = CastRecipe->getOperand(0);
- }
-
- // Try and get a scaled reduction from the first non-phi operand.
- // If one is found, we use the discovered reduction instruction in
- // place of the accumulator for costing.
- if (getScaledReductions(RedPhiR, Op, Chains, TypeInfo)) {
- RedPhiR = Chains.rbegin()->ReductionBinOp;
- Op = UpdateR->getOperand(0);
- PhiOp = UpdateR->getOperand(1);
- if (Op == RedPhiR)
- std::swap(Op, PhiOp);
- }
- if (RedPhiR != PhiOp)
- return false;
-
- // If the update is a binary op, check both of its operands to see if
- // they are extends. Otherwise, see if the update comes directly from an
- // extend.
- VPWidenCastRecipe *CastRecipes[2] = {nullptr};
-
- // Match extends and populate CastRecipes. Returns false if matching fails.
- auto MatchExtends = [OuterExtKind,
- &CastRecipes](ArrayRef<VPValue *> Operands) {
- assert(Operands.size() <= 2 && "expected at most 2 operands");
-
- for (const auto &[I, OpVal] : enumerate(Operands)) {
- // Allow constant as second operand - validation happens in transform.
- const APInt *Unused;
- if (I > 0 && CastRecipes[0] && match(OpVal, m_APInt(Unused)))
- continue;
-
- VPValue *ExtInput;
- if (!match(OpVal, m_ZExtOrSExt(m_VPValue(ExtInput))) &&
- !match(OpVal, m_FPExt(m_VPValue(ExtInput))))
- return false;
-
- CastRecipes[I] = cast<VPWidenCastRecipe>(OpVal);
-
- // The outer extend kind must match the inner extends for folding.
- if (OuterExtKind) {
- auto CastOp =
- static_cast<Instruction::CastOps>(CastRecipes[I]->getOpcode());
- if (*OuterExtKind != TTI::getPartialReductionExtendKind(CastOp))
- return false;
- }
- }
- return CastRecipes[0] != nullptr;
- };
-
- // If Op is a binary operator, check both of its operands to see if they are
- // extends. Otherwise, see if the update comes directly from an extend.
- auto *BinOp = dyn_cast<VPWidenRecipe>(Op);
- if (BinOp && Instruction::isBinaryOp(BinOp->getOpcode())) {
- if (!BinOp->hasOneUse())
- return false;
-
- // Handle neg(binop(ext, ext)) pattern.
- VPValue *OtherOp = nullptr;
- if (match(BinOp, m_Sub(m_ZeroInt(), m_VPValue(OtherOp))))
- BinOp = dyn_cast<VPWidenRecipe>(OtherOp);
-
- if (!BinOp || !Instruction::isBinaryOp(BinOp->getOpcode()) ||
- !MatchExtends(BinOp->operands()))
- return false;
- } else if (match(UpdateR, m_Add(m_VPValue(), m_VPValue())) ||
- match(UpdateR, m_FAdd(m_VPValue(), m_VPValue()))) {
- // We already know the operands for Update are Op and PhiOp.
- if (!MatchExtends({Op}))
- return false;
- BinOp = UpdateR;
- } else {
- return false;
- }
-
- Type *PhiType = TypeInfo.inferScalarType(RedPhiR);
- TypeSize PHISize = PhiType->getPrimitiveSizeInBits();
- Type *ExtOpType = TypeInfo.inferScalarType(CastRecipes[0]->getOperand(0));
- TypeSize ASize = ExtOpType->getPrimitiveSizeInBits();
- if (!PHISize.hasKnownScalarFactor(ASize))
- return false;
-
- Chains.push_back(
- {UpdateR, CastRecipes[0], CastRecipes[1], BinOp,
- static_cast<unsigned>(PHISize.getKnownScalarFactor(ASize))});
- return true;
-}
-} // namespace
-
-void VPlanTransforms::createPartialReductions(VPlan &Plan,
- VPCostContext &CostCtx,
- VFRange &Range) {
- // Find all possible partial reductions, grouping chains by their PHI. This
- // grouping allows invalidating the whole chain, if any link is not a valid
- // partial reduction.
- MapVector<VPReductionPHIRecipe *, SmallVector<VPPartialReductionChain>>
- ChainsByPhi;
- VPBasicBlock *HeaderVPBB = Plan.getVectorLoopRegion()->getEntryBasicBlock();
- for (VPRecipeBase &R : HeaderVPBB->phis()) {
- auto *RedPhiR = dyn_cast<VPReductionPHIRecipe>(&R);
- if (!RedPhiR)
- continue;
-
- // Get the backedge value from the reduction PHI and find the
- // ComputeReductionResult that uses it (directly or through a select for
- // predicated reductions).
- if (auto *RdxResult = vputils::findComputeReductionResult(RedPhiR)) {
- VPValue *ExitValue = RdxResult->getOperand(0);
- match(ExitValue,
- m_Select(m_VPValue(), m_VPValue(ExitValue), m_VPValue()));
- getScaledReductions(RedPhiR, ExitValue, ChainsByPhi[RedPhiR],
- CostCtx.Types);
- }
- }
-
- if (ChainsByPhi.empty())
- return;
-
- // Build set of partial reduction operations for extend user validation and
- // a map of reduction bin ops to their scale factors for scale validation.
- SmallPtrSet<VPRecipeBase *, 4> PartialReductionOps;
- DenseMap<VPSingleDefRecipe *, unsigned> ScaledReductionMap;
- for (const auto &[_, Chains] : ChainsByPhi)
- for (const VPPartialReductionChain &Chain : Chains) {
- PartialReductionOps.insert(Chain.BinOp);
- ScaledReductionMap[Chain.ReductionBinOp] = Chain.ScaleFactor;
- }
-
- // A partial reduction is invalid if any of its extends are used by
- // something that isn't another partial reduction. This is because the
- // extends are intended to be lowered along with the reduction itself.
- auto ExtendUsersValid = [&](VPWidenCastRecipe *Ext) {
- return !Ext || all_of(Ext->users(), [&](VPUser *U) {
- return PartialReductionOps.contains(cast<VPRecipeBase>(U));
- });
- };
-
- // Validate chains: check that extends are only used by partial reductions,
- // and that reduction bin ops are only used by other partial reductions with
- // matching scale factors, are outside the loop region or the select
- // introduced by tail-folding. Otherwise we would create users of scaled
- // reductions where the types of the other operands don't match.
- for (auto &[RedPhiR, Chains] : ChainsByPhi) {
- for (const VPPartialReductionChain &Chain : Chains) {
- if (!ExtendUsersValid(Chain.ExtendA) ||
- !ExtendUsersValid(Chain.ExtendB)) {
- Chains.clear();
- break;
- }
- auto UseIsValid = [&, RedPhiR = RedPhiR](VPUser *U) {
- if (auto *PhiR = dyn_cast<VPReductionPHIRecipe>(U))
- return PhiR == RedPhiR;
- auto *R = cast<VPSingleDefRecipe>(U);
- return Chain.ScaleFactor == ScaledReductionMap.lookup_or(R, 0) ||
- match(R, m_ComputeReductionResult(
- m_Specific(Chain.ReductionBinOp))) ||
- match(R, m_Select(m_VPValue(), m_Specific(Chain.ReductionBinOp),
- m_Specific(RedPhiR)));
- };
- if (!all_of(Chain.ReductionBinOp->users(), UseIsValid)) {
- Chains.clear();
- break;
- }
-
- // Check if the compute-reduction-result is used by a sunk store.
- // TODO: Also form partial reductions in those cases.
- if (auto *RdxResult = vputils::findComputeReductionResult(RedPhiR)) {
- if (any_of(RdxResult->users(), [](VPUser *U) {
- auto *RepR = dyn_cast<VPReplicateRecipe>(U);
- return RepR && isa<StoreInst>(RepR->getUnderlyingInstr());
- })) {
- Chains.clear();
- break;
- }
- }
- }
- }
-
- for (const auto &[_, Chains] : ChainsByPhi)
- for (const VPPartialReductionChain &Chain : Chains)
- transformToPartialReduction(Chain, Range, CostCtx, Plan);
-}
diff --git a/llvm/lib/Transforms/Vectorize/VPlanTransforms.h b/llvm/lib/Transforms/Vectorize/VPlanTransforms.h
index e5091b7329d34..f33c672e9ea59 100644
--- a/llvm/lib/Transforms/Vectorize/VPlanTransforms.h
+++ b/llvm/lib/Transforms/Vectorize/VPlanTransforms.h
@@ -29,7 +29,6 @@ class PHINode;
class ScalarEvolution;
class PredicatedScalarEvolution;
class TargetLibraryInfo;
-class TargetTransformInfo;
class VPBuilder;
class VPRecipeBuilder;
struct VFRange;
@@ -452,12 +451,6 @@ struct VPlanTransforms {
/// users in the original exit block using the VPIRInstruction wrapping to the
/// LCSSA phi.
static void addExitUsersForFirstOrderRecurrences(VPlan &Plan, VFRange &Range);
-
- /// Detect and create partial reduction recipes for scaled reductions in
- /// \p Plan. Must be called after recipe construction. If partial reductions
- /// are only valid for a subset of VFs in Range, Range.End is updated.
- static void createPartialReductions(VPlan &Plan, VPCostContext &CostCtx,
- VFRange &Range);
};
} // namespace llvm
diff --git a/llvm/lib/Transforms/Vectorize/VPlanUtils.cpp b/llvm/lib/Transforms/Vectorize/VPlanUtils.cpp
index c3f7c1117aa2a..baf55c33194e3 100644
--- a/llvm/lib/Transforms/Vectorize/VPlanUtils.cpp
+++ b/llvm/lib/Transforms/Vectorize/VPlanUtils.cpp
@@ -586,20 +586,3 @@ vputils::getMemoryLocation(const VPRecipeBase &R) {
Loc.AATags.Scope = AliasScopeMD;
return Loc;
}
-
-/// Find the ComputeReductionResult recipe for \p PhiR, looking through selects
-/// inserted for predicated reductions or tail folding.
-VPInstruction *vputils::findComputeReductionResult(VPReductionPHIRecipe *PhiR) {
- VPValue *BackedgeVal = PhiR->getBackedgeValue();
- if (auto *Res = vputils::findUserOf<VPInstruction::ComputeReductionResult>(
- BackedgeVal))
- return Res;
-
- // Look through selects inserted for tail folding or predicated reductions.
- VPRecipeBase *SelR = vputils::findUserOf(
- BackedgeVal, m_Select(m_VPValue(), m_VPValue(), m_VPValue()));
- if (!SelR)
- return nullptr;
- return vputils::findUserOf<VPInstruction::ComputeReductionResult>(
- cast<VPSingleDefRecipe>(SelR));
-}
diff --git a/llvm/lib/Transforms/Vectorize/VPlanUtils.h b/llvm/lib/Transforms/Vectorize/VPlanUtils.h
index a789f3cbb2815..31248cca50543 100644
--- a/llvm/lib/Transforms/Vectorize/VPlanUtils.h
+++ b/llvm/lib/Transforms/Vectorize/VPlanUtils.h
@@ -10,7 +10,6 @@
#define LLVM_TRANSFORMS_VECTORIZE_VPLANUTILS_H
#include "VPlan.h"
-#include "VPlanPatternMatch.h"
#include "llvm/Support/Compiler.h"
namespace llvm {
@@ -126,25 +125,6 @@ inline VPRecipeBase *findRecipe(VPValue *Start, PredT Pred) {
return nullptr;
}
-/// If \p V is used by a recipe matching pattern \p P, return it. Otherwise
-/// return nullptr;
-template <typename MatchT>
-static VPRecipeBase *findUserOf(VPValue *V, const MatchT &P) {
- auto It = find_if(V->users(), match_fn(P));
- return It == V->user_end() ? nullptr : cast<VPRecipeBase>(*It);
-}
-
-/// If \p V is used by a VPInstruction with \p Opcode, return it. Otherwise
-/// return nullptr.
-template <unsigned Opcode> static VPInstruction *findUserOf(VPValue *V) {
- using namespace llvm::VPlanPatternMatch;
- return cast_or_null<VPInstruction>(findUserOf(V, m_VPInstruction<Opcode>()));
-}
-
-/// Find the ComputeReductionResult recipe for \p PhiR, looking through selects
-/// inserted for predicated reductions or tail folding.
-VPInstruction *findComputeReductionResult(VPReductionPHIRecipe *PhiR);
-
} // namespace vputils
//===----------------------------------------------------------------------===//
diff --git a/llvm/test/Transforms/LoopVectorize/vplan-print-after-all.ll b/llvm/test/Transforms/LoopVectorize/vplan-print-after-all.ll
index 9f0a528bfa889..96e15740cd413 100644
--- a/llvm/test/Transforms/LoopVectorize/vplan-print-after-all.ll
+++ b/llvm/test/Transforms/LoopVectorize/vplan-print-after-all.ll
@@ -9,7 +9,6 @@
; CHECK: VPlan after VPlanTransforms::handleMultiUseReductions
; CHECK: VPlan after VPlanTransforms::handleMaxMinNumReductions
; CHECK: VPlan after VPlanTransforms::handleFindLastReductions
-; CHECK: VPlan after VPlanTransforms::createPartialReductions
; CHECK: VPlan after VPlanTransforms::convertToAbstractRecipes
; CHECK: VPlan after VPlanTransforms::createInterleaveGroups
; CHECK: VPlan after VPlanTransforms::replaceSymbolicStrides
More information about the llvm-commits
mailing list