[llvm] c2a8739 - [VPlan] Split off VPReductionRecipe creation for in-loop reductions (NFC) (#168784)
via llvm-commits
llvm-commits at lists.llvm.org
Thu Dec 25 06:03:03 PST 2025
Author: Florian Hahn
Date: 2025-12-25T14:02:58Z
New Revision: c2a8739cd1fbdafa2e16b006d35ad94cc6995163
URL: https://github.com/llvm/llvm-project/commit/c2a8739cd1fbdafa2e16b006d35ad94cc6995163
DIFF: https://github.com/llvm/llvm-project/commit/c2a8739cd1fbdafa2e16b006d35ad94cc6995163.diff
LOG: [VPlan] Split off VPReductionRecipe creation for in-loop reductions (NFC) (#168784)
This patch splits off VPReductionRecipe creation for in-loop reductions
to a separate transform from adjustInLoopReductions, which has been
renamed.
The new transform has been updated to work directly on VPInstructions,
and gets applied after header phis have been processed, once on VPlan0.
Builds on top of https://github.com/llvm/llvm-project/pull/168291 and
https://github.com/llvm/llvm-project/pull/166099 which should be
reviewed first.
PR: https://github.com/llvm/llvm-project/pull/168784
Added:
Modified:
llvm/lib/Transforms/Vectorize/LoopVectorizationPlanner.h
llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
llvm/lib/Transforms/Vectorize/VPlanConstruction.cpp
llvm/lib/Transforms/Vectorize/VPlanTransforms.h
Removed:
################################################################################
diff --git a/llvm/lib/Transforms/Vectorize/LoopVectorizationPlanner.h b/llvm/lib/Transforms/Vectorize/LoopVectorizationPlanner.h
index 741392247c0d6..430d9469c7698 100644
--- a/llvm/lib/Transforms/Vectorize/LoopVectorizationPlanner.h
+++ b/llvm/lib/Transforms/Vectorize/LoopVectorizationPlanner.h
@@ -620,14 +620,13 @@ class LoopVectorizationPlanner {
/// legal to vectorize the loop. This method creates VPlans using VPRecipes.
void buildVPlansWithVPRecipes(ElementCount MinVF, ElementCount MaxVF);
- // Adjust the recipes for reductions. For in-loop reductions the chain of
- // instructions leading from the loop exit instr to the phi need to be
- // converted to reductions, with one operand being vector and the other being
- // the scalar reduction chain. For other reductions, a select is introduced
- // between the phi and users outside the vector region when folding the tail.
- void adjustRecipesForReductions(VPlanPtr &Plan,
- VPRecipeBuilder &RecipeBuilder,
- ElementCount MinVF);
+ /// Add recipes to compute the final reduction result (ComputeFindIVResult,
+ /// ComputeAnyOfResult, ComputeReductionResult depending on the reduction) in
+ /// the middle block. Selects are introduced for reductions between the phi
+ /// and users outside the vector region when folding the tail.
+ void addReductionResultComputation(VPlanPtr &Plan,
+ VPRecipeBuilder &RecipeBuilder,
+ ElementCount MinVF);
/// Attach the runtime checks of \p RTChecks to \p Plan.
void attachRuntimeChecks(VPlan &Plan, GeneratedRTChecks &RTChecks,
diff --git a/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp b/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
index de9968f78ad8a..2c62f64be1c9b 100644
--- a/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
+++ b/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
@@ -8445,6 +8445,16 @@ VPlanPtr LoopVectorizationPlanner::tryToBuildVPlanWithVPRecipes(
// temporarily to update created block masks.
DenseMap<VPValue *, VPValue *> Old2New;
+ // Collect blocks that need predication for in-loop reduction recipes.
+ DenseSet<BasicBlock *> BlocksNeedingPredication;
+ for (BasicBlock *BB : OrigLoop->blocks())
+ if (CM.blockNeedsPredicationForAnyReason(BB))
+ BlocksNeedingPredication.insert(BB);
+
+ VPlanTransforms::createInLoopReductionRecipes(
+ *Plan, BlockMaskCache, BlocksNeedingPredication, Range.Start);
+
+ // Now process all other blocks and instructions.
for (VPBasicBlock *VPBB : VPBlockUtils::blocksOnly<VPBasicBlock>(RPOT)) {
// Convert input VPInstructions to widened recipes.
for (VPRecipeBase &R : make_early_inc_range(*VPBB)) {
@@ -8527,8 +8537,7 @@ VPlanPtr LoopVectorizationPlanner::tryToBuildVPlanWithVPRecipes(
// bring the VPlan to its final state.
// ---------------------------------------------------------------------------
- // Adjust the recipes for any inloop reductions.
- adjustRecipesForReductions(Plan, RecipeBuilder, Range.Start);
+ addReductionResultComputation(Plan, RecipeBuilder, Range.Start);
// Apply mandatory transformation to handle reductions with multiple in-loop
// uses if possible, bail out otherwise.
@@ -8634,178 +8643,13 @@ VPlanPtr LoopVectorizationPlanner::tryToBuildVPlan(VFRange &Range) {
return Plan;
}
-// Adjust the recipes for reductions. For in-loop reductions the chain of
-// instructions leading from the loop exit instr to the phi need to be converted
-// to reductions, with one operand being vector and the other being the scalar
-// reduction chain. For other reductions, a select is introduced between the phi
-// and users outside the vector region when folding the tail.
-//
-// A ComputeReductionResult recipe is added to the middle block, also for
-// in-loop reductions which compute their result in-loop, because generating
-// the subsequent bc.merge.rdx phi is driven by ComputeReductionResult recipes.
-//
-// Adjust AnyOf reductions; replace the reduction phi for the selected value
-// with a boolean reduction phi node to check if the condition is true in any
-// iteration. The final value is selected by the final ComputeReductionResult.
-void LoopVectorizationPlanner::adjustRecipesForReductions(
+void LoopVectorizationPlanner::addReductionResultComputation(
VPlanPtr &Plan, VPRecipeBuilder &RecipeBuilder, ElementCount MinVF) {
using namespace VPlanPatternMatch;
VPTypeAnalysis TypeInfo(*Plan);
VPRegionBlock *VectorLoopRegion = Plan->getVectorLoopRegion();
- VPBasicBlock *Header = VectorLoopRegion->getEntryBasicBlock();
VPBasicBlock *MiddleVPBB = Plan->getMiddleBlock();
SmallVector<VPRecipeBase *> ToDelete;
-
- for (VPRecipeBase &R : Header->phis()) {
- auto *PhiR = dyn_cast<VPReductionPHIRecipe>(&R);
- if (!PhiR || !PhiR->isInLoop() || (MinVF.isScalar() && !PhiR->isOrdered()))
- continue;
-
- RecurKind Kind = PhiR->getRecurrenceKind();
- assert(
- !RecurrenceDescriptor::isAnyOfRecurrenceKind(Kind) &&
- !RecurrenceDescriptor::isFindIVRecurrenceKind(Kind) &&
- "AnyOf and FindIV reductions are not allowed for in-loop reductions");
-
- bool IsFPRecurrence =
- RecurrenceDescriptor::isFloatingPointRecurrenceKind(Kind);
- FastMathFlags FMFs =
- IsFPRecurrence ? FastMathFlags::getFast() : FastMathFlags();
-
- // Collect the chain of "link" recipes for the reduction starting at PhiR.
- SetVector<VPSingleDefRecipe *> Worklist;
- Worklist.insert(PhiR);
- for (unsigned I = 0; I != Worklist.size(); ++I) {
- VPSingleDefRecipe *Cur = Worklist[I];
- for (VPUser *U : Cur->users()) {
- auto *UserRecipe = cast<VPSingleDefRecipe>(U);
- if (!UserRecipe->getParent()->getEnclosingLoopRegion()) {
- assert((UserRecipe->getParent() == MiddleVPBB ||
- UserRecipe->getParent() == Plan->getScalarPreheader()) &&
- "U must be either in the loop region, the middle block or the "
- "scalar preheader.");
- continue;
- }
- Worklist.insert(UserRecipe);
- }
- }
-
- // Visit operation "Links" along the reduction chain top-down starting from
- // the phi until LoopExitValue. We keep track of the previous item
- // (PreviousLink) to tell which of the two operands of a Link will remain
- // scalar and which will be reduced. For minmax by select(cmp), Link will be
- // the select instructions. Blend recipes of in-loop reduction phi's will
- // get folded to their non-phi operand, as the reduction recipe handles the
- // condition directly.
- VPSingleDefRecipe *PreviousLink = PhiR; // Aka Worklist[0].
- for (VPSingleDefRecipe *CurrentLink : drop_begin(Worklist)) {
- if (auto *Blend = dyn_cast<VPBlendRecipe>(CurrentLink)) {
- assert(Blend->getNumIncomingValues() == 2 &&
- "Blend must have 2 incoming values");
- if (Blend->getIncomingValue(0) == PhiR) {
- Blend->replaceAllUsesWith(Blend->getIncomingValue(1));
- } else {
- assert(Blend->getIncomingValue(1) == PhiR &&
- "PhiR must be an operand of the blend");
- Blend->replaceAllUsesWith(Blend->getIncomingValue(0));
- }
- continue;
- }
-
- if (IsFPRecurrence) {
- FastMathFlags CurFMF =
- cast<VPRecipeWithIRFlags>(CurrentLink)->getFastMathFlags();
- if (match(CurrentLink, m_Select(m_VPValue(), m_VPValue(), m_VPValue())))
- CurFMF |= cast<VPRecipeWithIRFlags>(CurrentLink->getOperand(0))
- ->getFastMathFlags();
- FMFs &= CurFMF;
- }
-
- Instruction *CurrentLinkI = CurrentLink->getUnderlyingInstr();
-
- // Index of the first operand which holds a non-mask vector operand.
- unsigned IndexOfFirstOperand;
- // Recognize a call to the llvm.fmuladd intrinsic.
- bool IsFMulAdd = (Kind == RecurKind::FMulAdd);
- VPValue *VecOp;
- VPBasicBlock *LinkVPBB = CurrentLink->getParent();
- if (IsFMulAdd) {
- assert(
- RecurrenceDescriptor::isFMulAddIntrinsic(CurrentLinkI) &&
- "Expected instruction to be a call to the llvm.fmuladd intrinsic");
- assert(((MinVF.isScalar() && isa<VPReplicateRecipe>(CurrentLink)) ||
- isa<VPWidenIntrinsicRecipe>(CurrentLink)) &&
- CurrentLink->getOperand(2) == PreviousLink &&
- "expected a call where the previous link is the added operand");
-
- // If the instruction is a call to the llvm.fmuladd intrinsic then we
- // need to create an fmul recipe (multiplying the first two operands of
- // the fmuladd together) to use as the vector operand for the fadd
- // reduction.
- VPInstruction *FMulRecipe = new VPInstruction(
- Instruction::FMul,
- {CurrentLink->getOperand(0), CurrentLink->getOperand(1)},
- CurrentLinkI->getFastMathFlags());
- LinkVPBB->insert(FMulRecipe, CurrentLink->getIterator());
- VecOp = FMulRecipe;
- } else if (PhiR->isInLoop() && Kind == RecurKind::AddChainWithSubs &&
- match(CurrentLink, m_Sub(m_VPValue(), m_VPValue()))) {
- Type *PhiTy = TypeInfo.inferScalarType(PhiR);
- auto *Zero = Plan->getConstantInt(PhiTy, 0);
- auto *Sub = new VPInstruction(Instruction::Sub,
- {Zero, CurrentLink->getOperand(1)}, {},
- {}, CurrentLinkI->getDebugLoc());
- Sub->setUnderlyingValue(CurrentLinkI);
- LinkVPBB->insert(Sub, CurrentLink->getIterator());
- VecOp = Sub;
- } else {
- if (RecurrenceDescriptor::isMinMaxRecurrenceKind(Kind)) {
- if (match(CurrentLink, m_Cmp(m_VPValue(), m_VPValue())))
- continue;
- assert(isa<VPWidenSelectRecipe>(CurrentLink) &&
- "must be a select recipe");
- IndexOfFirstOperand = 1;
- } else {
- assert((MinVF.isScalar() || isa<VPWidenRecipe>(CurrentLink)) &&
- "Expected to replace a VPWidenSC");
- IndexOfFirstOperand = 0;
- }
- // Note that for non-commutable operands (cmp-selects), the semantics of
- // the cmp-select are captured in the recurrence kind.
- unsigned VecOpId =
- CurrentLink->getOperand(IndexOfFirstOperand) == PreviousLink
- ? IndexOfFirstOperand + 1
- : IndexOfFirstOperand;
- VecOp = CurrentLink->getOperand(VecOpId);
- assert(VecOp != PreviousLink &&
- CurrentLink->getOperand(CurrentLink->getNumOperands() - 1 -
- (VecOpId - IndexOfFirstOperand)) ==
- PreviousLink &&
- "PreviousLink must be the operand other than VecOp");
- }
-
- VPValue *CondOp = nullptr;
- if (CM.blockNeedsPredicationForAnyReason(CurrentLinkI->getParent()))
- CondOp = RecipeBuilder.getBlockInMask(CurrentLink->getParent());
-
- ReductionStyle Style = getReductionStyle(true, PhiR->isOrdered(), 1);
- auto *RedRecipe =
- new VPReductionRecipe(Kind, FMFs, CurrentLinkI, PreviousLink, VecOp,
- CondOp, Style, CurrentLinkI->getDebugLoc());
- // Append the recipe to the end of the VPBasicBlock because we need to
- // ensure that it comes after all of it's inputs, including CondOp.
- // Delete CurrentLink as it will be invalid if its operand is replaced
- // with a reduction defined at the bottom of the block in the next link.
- if (LinkVPBB->getNumSuccessors() == 0)
- RedRecipe->insertBefore(&*std::prev(std::prev(LinkVPBB->end())));
- else
- LinkVPBB->appendRecipe(RedRecipe);
-
- CurrentLink->replaceAllUsesWith(RedRecipe);
- ToDelete.push_back(CurrentLink);
- PreviousLink = RedRecipe;
- }
- }
VPBasicBlock *LatchVPBB = VectorLoopRegion->getExitingBasicBlock();
Builder.setInsertPoint(&*std::prev(std::prev(LatchVPBB->end())));
VPBasicBlock::iterator IP = MiddleVPBB->getFirstNonPhi();
diff --git a/llvm/lib/Transforms/Vectorize/VPlanConstruction.cpp b/llvm/lib/Transforms/Vectorize/VPlanConstruction.cpp
index 4d1532eff74e8..a087322fe862f 100644
--- a/llvm/lib/Transforms/Vectorize/VPlanConstruction.cpp
+++ b/llvm/lib/Transforms/Vectorize/VPlanConstruction.cpp
@@ -13,6 +13,7 @@
#include "LoopVectorizationPlanner.h"
#include "VPlan.h"
+#include "VPlanAnalysis.h"
#include "VPlanCFG.h"
#include "VPlanDominatorTree.h"
#include "VPlanPatternMatch.h"
@@ -702,6 +703,168 @@ void VPlanTransforms::createHeaderPhiRecipes(
}
}
+void VPlanTransforms::createInLoopReductionRecipes(
+ VPlan &Plan, const DenseMap<VPBasicBlock *, VPValue *> &BlockMaskCache,
+ const DenseSet<BasicBlock *> &BlocksNeedingPredication,
+ ElementCount MinVF) {
+ VPTypeAnalysis TypeInfo(Plan);
+ VPBasicBlock *Header = Plan.getVectorLoopRegion()->getEntryBasicBlock();
+ SmallVector<VPRecipeBase *> ToDelete;
+
+ for (VPRecipeBase &R : Header->phis()) {
+ auto *PhiR = dyn_cast<VPReductionPHIRecipe>(&R);
+ if (!PhiR || !PhiR->isInLoop() || (MinVF.isScalar() && !PhiR->isOrdered()))
+ continue;
+
+ RecurKind Kind = PhiR->getRecurrenceKind();
+ assert(
+ !RecurrenceDescriptor::isAnyOfRecurrenceKind(Kind) &&
+ !RecurrenceDescriptor::isFindIVRecurrenceKind(Kind) &&
+ "AnyOf and FindIV reductions are not allowed for in-loop reductions");
+
+ bool IsFPRecurrence =
+ RecurrenceDescriptor::isFloatingPointRecurrenceKind(Kind);
+ FastMathFlags FMFs =
+ IsFPRecurrence ? FastMathFlags::getFast() : FastMathFlags();
+
+ // Collect the chain of "link" recipes for the reduction starting at PhiR.
+ SetVector<VPSingleDefRecipe *> Worklist;
+ Worklist.insert(PhiR);
+ for (unsigned I = 0; I != Worklist.size(); ++I) {
+ VPSingleDefRecipe *Cur = Worklist[I];
+ for (VPUser *U : Cur->users()) {
+ auto *UserRecipe = cast<VPSingleDefRecipe>(U);
+ if (!UserRecipe->getParent()->getEnclosingLoopRegion()) {
+ assert((UserRecipe->getParent() == Plan.getMiddleBlock() ||
+ UserRecipe->getParent() == Plan.getScalarPreheader()) &&
+ "U must be either in the loop region, the middle block or the "
+ "scalar preheader.");
+ continue;
+ }
+
+ // Stores using instructions will be sunk later.
+ if (match(UserRecipe, m_VPInstruction<Instruction::Store>()))
+ continue;
+ Worklist.insert(UserRecipe);
+ }
+ }
+
+ // Visit operation "Links" along the reduction chain top-down starting from
+ // the phi until LoopExitValue. We keep track of the previous item
+ // (PreviousLink) to tell which of the two operands of a Link will remain
+ // scalar and which will be reduced. For minmax by select(cmp), Link will be
+ // the select instructions. Blend recipes of in-loop reduction phi's will
+ // get folded to their non-phi operand, as the reduction recipe handles the
+ // condition directly.
+ VPSingleDefRecipe *PreviousLink = PhiR; // Aka Worklist[0].
+ for (VPSingleDefRecipe *CurrentLink : drop_begin(Worklist)) {
+ if (auto *Blend = dyn_cast<VPBlendRecipe>(CurrentLink)) {
+ assert(Blend->getNumIncomingValues() == 2 &&
+ "Blend must have 2 incoming values");
+ unsigned PhiRIdx = Blend->getIncomingValue(0) == PhiR ? 0 : 1;
+ assert(Blend->getIncomingValue(PhiRIdx) == PhiR &&
+ "PhiR must be an operand of the blend");
+ Blend->replaceAllUsesWith(Blend->getIncomingValue(1 - PhiRIdx));
+ continue;
+ }
+
+ if (IsFPRecurrence) {
+ FastMathFlags CurFMF =
+ cast<VPRecipeWithIRFlags>(CurrentLink)->getFastMathFlags();
+ if (match(CurrentLink, m_Select(m_VPValue(), m_VPValue(), m_VPValue())))
+ CurFMF |= cast<VPRecipeWithIRFlags>(CurrentLink->getOperand(0))
+ ->getFastMathFlags();
+ FMFs &= CurFMF;
+ }
+
+ Instruction *CurrentLinkI = CurrentLink->getUnderlyingInstr();
+
+ // Recognize a call to the llvm.fmuladd intrinsic.
+ bool IsFMulAdd = Kind == RecurKind::FMulAdd;
+ VPValue *VecOp;
+ VPBasicBlock *LinkVPBB = CurrentLink->getParent();
+ if (IsFMulAdd) {
+ assert(RecurrenceDescriptor::isFMulAddIntrinsic(CurrentLinkI) &&
+ "Expected current VPInstruction to be a call to the "
+ "llvm.fmuladd intrinsic");
+ assert(CurrentLink->getOperand(2) == PreviousLink &&
+ "expected a call where the previous link is the added operand");
+
+ // If the instruction is a call to the llvm.fmuladd intrinsic then we
+ // need to create an fmul recipe (multiplying the first two operands of
+ // the fmuladd together) to use as the vector operand for the fadd
+ // reduction.
+ auto *FMulRecipe = new VPInstruction(
+ Instruction::FMul,
+ {CurrentLink->getOperand(0), CurrentLink->getOperand(1)},
+ CurrentLinkI->getFastMathFlags());
+ LinkVPBB->insert(FMulRecipe, CurrentLink->getIterator());
+ VecOp = FMulRecipe;
+ } else if (Kind == RecurKind::AddChainWithSubs &&
+ match(CurrentLink, m_Sub(m_VPValue(), m_VPValue()))) {
+ Type *PhiTy = TypeInfo.inferScalarType(PhiR);
+ auto *Zero = Plan.getConstantInt(PhiTy, 0);
+ auto *Sub = new VPInstruction(Instruction::Sub,
+ {Zero, CurrentLink->getOperand(1)}, {},
+ {}, CurrentLinkI->getDebugLoc());
+ Sub->setUnderlyingValue(CurrentLinkI);
+ LinkVPBB->insert(Sub, CurrentLink->getIterator());
+ VecOp = Sub;
+ } else {
+ // Index of the first operand which holds a non-mask vector operand.
+ unsigned IndexOfFirstOperand = 0;
+ if (RecurrenceDescriptor::isMinMaxRecurrenceKind(Kind)) {
+ if (match(CurrentLink, m_Cmp(m_VPValue(), m_VPValue())))
+ continue;
+ assert(match(CurrentLink,
+ m_Select(m_VPValue(), m_VPValue(), m_VPValue())) &&
+ "must be a select recipe");
+ IndexOfFirstOperand = 1;
+ }
+ // Note that for non-commutable operands (cmp-selects), the semantics of
+ // the cmp-select are captured in the recurrence kind.
+ unsigned VecOpId =
+ CurrentLink->getOperand(IndexOfFirstOperand) == PreviousLink
+ ? IndexOfFirstOperand + 1
+ : IndexOfFirstOperand;
+ VecOp = CurrentLink->getOperand(VecOpId);
+ assert(VecOp != PreviousLink &&
+ CurrentLink->getOperand(CurrentLink->getNumOperands() - 1 -
+ (VecOpId - IndexOfFirstOperand)) ==
+ PreviousLink &&
+ "PreviousLink must be the operand other than VecOp");
+ }
+
+ // Get block mask from BlockMaskCache if the block needs predication.
+ VPValue *CondOp = nullptr;
+ if (BlocksNeedingPredication.contains(CurrentLinkI->getParent()))
+ CondOp = BlockMaskCache.lookup(LinkVPBB);
+
+ assert(PhiR->getVFScaleFactor() == 1 &&
+ "inloop reductions must be unscaled");
+ auto *RedRecipe = new VPReductionRecipe(
+ Kind, FMFs, CurrentLinkI, PreviousLink, VecOp, CondOp,
+ getReductionStyle(/*IsInLoop=*/true, PhiR->isOrdered(), 1),
+ CurrentLinkI->getDebugLoc());
+ // Append the recipe to the end of the VPBasicBlock because we need to
+ // ensure that it comes after all of it's inputs, including CondOp.
+ // Delete CurrentLink as it will be invalid if its operand is replaced
+ // with a reduction defined at the bottom of the block in the next link.
+ if (LinkVPBB->getNumSuccessors() == 0)
+ RedRecipe->insertBefore(&*std::prev(std::prev(LinkVPBB->end())));
+ else
+ LinkVPBB->appendRecipe(RedRecipe);
+
+ CurrentLink->replaceAllUsesWith(RedRecipe);
+ ToDelete.push_back(CurrentLink);
+ PreviousLink = RedRecipe;
+ }
+ }
+
+ for (VPRecipeBase *R : ToDelete)
+ R->eraseFromParent();
+}
+
void VPlanTransforms::handleEarlyExits(VPlan &Plan,
bool HasUncountableEarlyExit) {
auto *MiddleVPBB = cast<VPBasicBlock>(
diff --git a/llvm/lib/Transforms/Vectorize/VPlanTransforms.h b/llvm/lib/Transforms/Vectorize/VPlanTransforms.h
index ba208549af9d3..1eebedcb0feb2 100644
--- a/llvm/lib/Transforms/Vectorize/VPlanTransforms.h
+++ b/llvm/lib/Transforms/Vectorize/VPlanTransforms.h
@@ -113,6 +113,15 @@ struct VPlanTransforms {
const SmallPtrSetImpl<const PHINode *> &FixedOrderRecurrences,
const SmallPtrSetImpl<PHINode *> &InLoopReductions, bool AllowReordering);
+ /// Create VPReductionRecipes for in-loop reductions. This processes chains
+ /// of operations contributing to in-loop reductions and creates appropriate
+ /// VPReductionRecipe instances. Block masks from \p BlockMaskCache are used
+ /// to add predication for blocks in \p BlocksNeedingPredication.
+ static void createInLoopReductionRecipes(
+ VPlan &Plan, const DenseMap<VPBasicBlock *, VPValue *> &BlockMaskCache,
+ const DenseSet<BasicBlock *> &BlocksNeedingPredication,
+ ElementCount MinVF);
+
/// Update \p Plan to account for all early exits.
LLVM_ABI_FOR_TEST static void handleEarlyExits(VPlan &Plan,
bool HasUncountableExit);
More information about the llvm-commits
mailing list