[llvm] [VPlan] Split off VPReductionRecipe creation for in-loop reductions (NFC) (PR #168784)
Florian Hahn via llvm-commits
llvm-commits at lists.llvm.org
Mon Dec 22 08:46:30 PST 2025
================
@@ -8612,178 +8621,13 @@ VPlanPtr LoopVectorizationPlanner::tryToBuildVPlan(VFRange &Range) {
return Plan;
}
-// Adjust the recipes for reductions. For in-loop reductions the chain of
-// instructions leading from the loop exit instr to the phi need to be converted
-// to reductions, with one operand being vector and the other being the scalar
-// reduction chain. For other reductions, a select is introduced between the phi
-// and users outside the vector region when folding the tail.
-//
-// A ComputeReductionResult recipe is added to the middle block, also for
-// in-loop reductions which compute their result in-loop, because generating
-// the subsequent bc.merge.rdx phi is driven by ComputeReductionResult recipes.
-//
-// Adjust AnyOf reductions; replace the reduction phi for the selected value
-// with a boolean reduction phi node to check if the condition is true in any
-// iteration. The final value is selected by the final ComputeReductionResult.
-void LoopVectorizationPlanner::adjustRecipesForReductions(
+void LoopVectorizationPlanner::addReductionResultComputation(
VPlanPtr &Plan, VPRecipeBuilder &RecipeBuilder, ElementCount MinVF) {
using namespace VPlanPatternMatch;
- VPTypeAnalysis TypeInfo(*Plan);
VPRegionBlock *VectorLoopRegion = Plan->getVectorLoopRegion();
- VPBasicBlock *Header = VectorLoopRegion->getEntryBasicBlock();
VPBasicBlock *MiddleVPBB = Plan->getMiddleBlock();
+ VPTypeAnalysis TypeInfo(*Plan);
SmallVector<VPRecipeBase *> ToDelete;
-
- for (VPRecipeBase &R : Header->phis()) {
- auto *PhiR = dyn_cast<VPReductionPHIRecipe>(&R);
- if (!PhiR || !PhiR->isInLoop() || (MinVF.isScalar() && !PhiR->isOrdered()))
- continue;
-
- RecurKind Kind = PhiR->getRecurrenceKind();
- assert(
- !RecurrenceDescriptor::isAnyOfRecurrenceKind(Kind) &&
- !RecurrenceDescriptor::isFindIVRecurrenceKind(Kind) &&
- "AnyOf and FindIV reductions are not allowed for in-loop reductions");
-
- bool IsFPRecurrence =
- RecurrenceDescriptor::isFloatingPointRecurrenceKind(Kind);
- FastMathFlags FMFs =
- IsFPRecurrence ? FastMathFlags::getFast() : FastMathFlags();
-
- // Collect the chain of "link" recipes for the reduction starting at PhiR.
- SetVector<VPSingleDefRecipe *> Worklist;
- Worklist.insert(PhiR);
- for (unsigned I = 0; I != Worklist.size(); ++I) {
- VPSingleDefRecipe *Cur = Worklist[I];
- for (VPUser *U : Cur->users()) {
- auto *UserRecipe = cast<VPSingleDefRecipe>(U);
- if (!UserRecipe->getParent()->getEnclosingLoopRegion()) {
- assert((UserRecipe->getParent() == MiddleVPBB ||
- UserRecipe->getParent() == Plan->getScalarPreheader()) &&
- "U must be either in the loop region, the middle block or the "
- "scalar preheader.");
- continue;
- }
- Worklist.insert(UserRecipe);
- }
- }
-
- // Visit operation "Links" along the reduction chain top-down starting from
- // the phi until LoopExitValue. We keep track of the previous item
- // (PreviousLink) to tell which of the two operands of a Link will remain
- // scalar and which will be reduced. For minmax by select(cmp), Link will be
- // the select instructions. Blend recipes of in-loop reduction phi's will
- // get folded to their non-phi operand, as the reduction recipe handles the
- // condition directly.
- VPSingleDefRecipe *PreviousLink = PhiR; // Aka Worklist[0].
- for (VPSingleDefRecipe *CurrentLink : drop_begin(Worklist)) {
- if (auto *Blend = dyn_cast<VPBlendRecipe>(CurrentLink)) {
- assert(Blend->getNumIncomingValues() == 2 &&
- "Blend must have 2 incoming values");
- if (Blend->getIncomingValue(0) == PhiR) {
- Blend->replaceAllUsesWith(Blend->getIncomingValue(1));
- } else {
- assert(Blend->getIncomingValue(1) == PhiR &&
- "PhiR must be an operand of the blend");
- Blend->replaceAllUsesWith(Blend->getIncomingValue(0));
- }
- continue;
- }
-
- if (IsFPRecurrence) {
- FastMathFlags CurFMF =
- cast<VPRecipeWithIRFlags>(CurrentLink)->getFastMathFlags();
- if (match(CurrentLink, m_Select(m_VPValue(), m_VPValue(), m_VPValue())))
- CurFMF |= cast<VPRecipeWithIRFlags>(CurrentLink->getOperand(0))
- ->getFastMathFlags();
- FMFs &= CurFMF;
- }
-
- Instruction *CurrentLinkI = CurrentLink->getUnderlyingInstr();
-
- // Index of the first operand which holds a non-mask vector operand.
- unsigned IndexOfFirstOperand;
- // Recognize a call to the llvm.fmuladd intrinsic.
- bool IsFMulAdd = (Kind == RecurKind::FMulAdd);
- VPValue *VecOp;
- VPBasicBlock *LinkVPBB = CurrentLink->getParent();
- if (IsFMulAdd) {
- assert(
- RecurrenceDescriptor::isFMulAddIntrinsic(CurrentLinkI) &&
- "Expected instruction to be a call to the llvm.fmuladd intrinsic");
- assert(((MinVF.isScalar() && isa<VPReplicateRecipe>(CurrentLink)) ||
- isa<VPWidenIntrinsicRecipe>(CurrentLink)) &&
- CurrentLink->getOperand(2) == PreviousLink &&
- "expected a call where the previous link is the added operand");
-
- // If the instruction is a call to the llvm.fmuladd intrinsic then we
- // need to create an fmul recipe (multiplying the first two operands of
- // the fmuladd together) to use as the vector operand for the fadd
- // reduction.
- VPInstruction *FMulRecipe = new VPInstruction(
- Instruction::FMul,
- {CurrentLink->getOperand(0), CurrentLink->getOperand(1)},
- CurrentLinkI->getFastMathFlags());
- LinkVPBB->insert(FMulRecipe, CurrentLink->getIterator());
- VecOp = FMulRecipe;
- } else if (PhiR->isInLoop() && Kind == RecurKind::AddChainWithSubs &&
- match(CurrentLink, m_Sub(m_VPValue(), m_VPValue()))) {
- Type *PhiTy = TypeInfo.inferScalarType(PhiR);
- auto *Zero = Plan->getConstantInt(PhiTy, 0);
- auto *Sub = new VPInstruction(Instruction::Sub,
- {Zero, CurrentLink->getOperand(1)}, {},
- {}, CurrentLinkI->getDebugLoc());
- Sub->setUnderlyingValue(CurrentLinkI);
- LinkVPBB->insert(Sub, CurrentLink->getIterator());
- VecOp = Sub;
- } else {
- if (RecurrenceDescriptor::isMinMaxRecurrenceKind(Kind)) {
- if (match(CurrentLink, m_Cmp(m_VPValue(), m_VPValue())))
- continue;
- assert(isa<VPWidenSelectRecipe>(CurrentLink) &&
- "must be a select recipe");
- IndexOfFirstOperand = 1;
- } else {
- assert((MinVF.isScalar() || isa<VPWidenRecipe>(CurrentLink)) &&
- "Expected to replace a VPWidenSC");
- IndexOfFirstOperand = 0;
- }
- // Note that for non-commutable operands (cmp-selects), the semantics of
- // the cmp-select are captured in the recurrence kind.
- unsigned VecOpId =
- CurrentLink->getOperand(IndexOfFirstOperand) == PreviousLink
- ? IndexOfFirstOperand + 1
- : IndexOfFirstOperand;
- VecOp = CurrentLink->getOperand(VecOpId);
- assert(VecOp != PreviousLink &&
- CurrentLink->getOperand(CurrentLink->getNumOperands() - 1 -
- (VecOpId - IndexOfFirstOperand)) ==
- PreviousLink &&
- "PreviousLink must be the operand other than VecOp");
- }
-
- VPValue *CondOp = nullptr;
- if (CM.blockNeedsPredicationForAnyReason(CurrentLinkI->getParent()))
- CondOp = RecipeBuilder.getBlockInMask(CurrentLink->getParent());
-
- ReductionStyle Style = getReductionStyle(true, PhiR->isOrdered(), 1);
- auto *RedRecipe =
- new VPReductionRecipe(Kind, FMFs, CurrentLinkI, PreviousLink, VecOp,
- CondOp, Style, CurrentLinkI->getDebugLoc());
- // Append the recipe to the end of the VPBasicBlock because we need to
- // ensure that it comes after all of it's inputs, including CondOp.
- // Delete CurrentLink as it will be invalid if its operand is replaced
- // with a reduction defined at the bottom of the block in the next link.
- if (LinkVPBB->getNumSuccessors() == 0)
- RedRecipe->insertBefore(&*std::prev(std::prev(LinkVPBB->end())));
- else
- LinkVPBB->appendRecipe(RedRecipe);
-
- CurrentLink->replaceAllUsesWith(RedRecipe);
- ToDelete.push_back(CurrentLink);
- PreviousLink = RedRecipe;
- }
- }
VPBasicBlock *LatchVPBB = VectorLoopRegion->getExitingBasicBlock();
Builder.setInsertPoint(&*std::prev(std::prev(LatchVPBB->end())));
VPBasicBlock::iterator IP = MiddleVPBB->getFirstNonPhi();
----------------
fhahn wrote:
Thanks, will check
https://github.com/llvm/llvm-project/pull/168784
More information about the llvm-commits
mailing list