[llvm] [VPlan] Split off VPReductionRecipe creation for in-loop reductions (NFC) (PR #168784)
Florian Hahn via llvm-commits
llvm-commits at lists.llvm.org
Mon Dec 22 08:46:08 PST 2025
https://github.com/fhahn updated https://github.com/llvm/llvm-project/pull/168784
>From 49a6783dbc9314ee1b2df07cacf3b709f4e71514 Mon Sep 17 00:00:00 2001
From: Florian Hahn <flo at fhahn.com>
Date: Wed, 19 Nov 2025 22:24:05 +0000
Subject: [PATCH 1/2] [VPlan] Split off VPReductionRecipe creation for in-loop
reductions (NFC)
This patch splits off VPReductionRecipe creation for in-loop reductions
to a separate transform from adjustInLoopReductions, which has been
renamed.
The new transform has been updated to work directly on VPInstructions,
and gets applied after header phis have been processed, once on VPlan0.
Builds on top of https://github.com/llvm/llvm-project/pull/168291 and
https://github.com/llvm/llvm-project/pull/166099 which should be
reviewed first.
---
.../Vectorize/LoopVectorizationPlanner.h | 15 +-
.../Transforms/Vectorize/LoopVectorize.cpp | 182 ++----------------
.../Vectorize/VPlanConstruction.cpp | 159 +++++++++++++++
.../Transforms/Vectorize/VPlanTransforms.h | 9 +
4 files changed, 188 insertions(+), 177 deletions(-)
diff --git a/llvm/lib/Transforms/Vectorize/LoopVectorizationPlanner.h b/llvm/lib/Transforms/Vectorize/LoopVectorizationPlanner.h
index 741392247c0d6..2ae81c1b7b3d9 100644
--- a/llvm/lib/Transforms/Vectorize/LoopVectorizationPlanner.h
+++ b/llvm/lib/Transforms/Vectorize/LoopVectorizationPlanner.h
@@ -620,14 +620,13 @@ class LoopVectorizationPlanner {
/// legal to vectorize the loop. This method creates VPlans using VPRecipes.
void buildVPlansWithVPRecipes(ElementCount MinVF, ElementCount MaxVF);
- // Adjust the recipes for reductions. For in-loop reductions the chain of
- // instructions leading from the loop exit instr to the phi need to be
- // converted to reductions, with one operand being vector and the other being
- // the scalar reduction chain. For other reductions, a select is introduced
- // between the phi and users outside the vector region when folding the tail.
- void adjustRecipesForReductions(VPlanPtr &Plan,
- VPRecipeBuilder &RecipeBuilder,
- ElementCount MinVF);
+ /// Add recipes to compute the final reduction result (ComputeFindIVResult,
+ /// ComputeAnyOfResult, ComputeReductionResult depending on the reduction) in
+ /// the middle block. Selects are introduced for regular reductions between
+ /// the phi and users outside the vector region when folding the tail.
+ void addReductionResultComputation(VPlanPtr &Plan,
+ VPRecipeBuilder &RecipeBuilder,
+ ElementCount MinVF);
/// Attach the runtime checks of \p RTChecks to \p Plan.
void attachRuntimeChecks(VPlan &Plan, GeneratedRTChecks &RTChecks,
diff --git a/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp b/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
index e236928ee2ae6..bd7b084d8d82e 100644
--- a/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
+++ b/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
@@ -8423,6 +8423,16 @@ VPlanPtr LoopVectorizationPlanner::tryToBuildVPlanWithVPRecipes(
// temporarily to update created block masks.
DenseMap<VPValue *, VPValue *> Old2New;
+ // Collect blocks that need predication for in-loop reduction recipes.
+ DenseSet<BasicBlock *> BlocksNeedingPredication;
+ for (BasicBlock *BB : OrigLoop->blocks())
+ if (CM.blockNeedsPredicationForAnyReason(BB))
+ BlocksNeedingPredication.insert(BB);
+
+ VPlanTransforms::createVPReductionRecipesForInLoopReductions(
+ *Plan, BlockMaskCache, BlocksNeedingPredication, Range.Start);
+
+ // Now process all other blocks and instructions.
for (VPBasicBlock *VPBB : VPBlockUtils::blocksOnly<VPBasicBlock>(RPOT)) {
// Convert input VPInstructions to widened recipes.
for (VPRecipeBase &R : make_early_inc_range(*VPBB)) {
@@ -8505,8 +8515,7 @@ VPlanPtr LoopVectorizationPlanner::tryToBuildVPlanWithVPRecipes(
// bring the VPlan to its final state.
// ---------------------------------------------------------------------------
- // Adjust the recipes for any inloop reductions.
- adjustRecipesForReductions(Plan, RecipeBuilder, Range.Start);
+ addReductionResultComputation(Plan, RecipeBuilder, Range.Start);
// Apply mandatory transformation to handle reductions with multiple in-loop
// uses if possible, bail out otherwise.
@@ -8612,178 +8621,13 @@ VPlanPtr LoopVectorizationPlanner::tryToBuildVPlan(VFRange &Range) {
return Plan;
}
-// Adjust the recipes for reductions. For in-loop reductions the chain of
-// instructions leading from the loop exit instr to the phi need to be converted
-// to reductions, with one operand being vector and the other being the scalar
-// reduction chain. For other reductions, a select is introduced between the phi
-// and users outside the vector region when folding the tail.
-//
-// A ComputeReductionResult recipe is added to the middle block, also for
-// in-loop reductions which compute their result in-loop, because generating
-// the subsequent bc.merge.rdx phi is driven by ComputeReductionResult recipes.
-//
-// Adjust AnyOf reductions; replace the reduction phi for the selected value
-// with a boolean reduction phi node to check if the condition is true in any
-// iteration. The final value is selected by the final ComputeReductionResult.
-void LoopVectorizationPlanner::adjustRecipesForReductions(
+void LoopVectorizationPlanner::addReductionResultComputation(
VPlanPtr &Plan, VPRecipeBuilder &RecipeBuilder, ElementCount MinVF) {
using namespace VPlanPatternMatch;
- VPTypeAnalysis TypeInfo(*Plan);
VPRegionBlock *VectorLoopRegion = Plan->getVectorLoopRegion();
- VPBasicBlock *Header = VectorLoopRegion->getEntryBasicBlock();
VPBasicBlock *MiddleVPBB = Plan->getMiddleBlock();
+ VPTypeAnalysis TypeInfo(*Plan);
SmallVector<VPRecipeBase *> ToDelete;
-
- for (VPRecipeBase &R : Header->phis()) {
- auto *PhiR = dyn_cast<VPReductionPHIRecipe>(&R);
- if (!PhiR || !PhiR->isInLoop() || (MinVF.isScalar() && !PhiR->isOrdered()))
- continue;
-
- RecurKind Kind = PhiR->getRecurrenceKind();
- assert(
- !RecurrenceDescriptor::isAnyOfRecurrenceKind(Kind) &&
- !RecurrenceDescriptor::isFindIVRecurrenceKind(Kind) &&
- "AnyOf and FindIV reductions are not allowed for in-loop reductions");
-
- bool IsFPRecurrence =
- RecurrenceDescriptor::isFloatingPointRecurrenceKind(Kind);
- FastMathFlags FMFs =
- IsFPRecurrence ? FastMathFlags::getFast() : FastMathFlags();
-
- // Collect the chain of "link" recipes for the reduction starting at PhiR.
- SetVector<VPSingleDefRecipe *> Worklist;
- Worklist.insert(PhiR);
- for (unsigned I = 0; I != Worklist.size(); ++I) {
- VPSingleDefRecipe *Cur = Worklist[I];
- for (VPUser *U : Cur->users()) {
- auto *UserRecipe = cast<VPSingleDefRecipe>(U);
- if (!UserRecipe->getParent()->getEnclosingLoopRegion()) {
- assert((UserRecipe->getParent() == MiddleVPBB ||
- UserRecipe->getParent() == Plan->getScalarPreheader()) &&
- "U must be either in the loop region, the middle block or the "
- "scalar preheader.");
- continue;
- }
- Worklist.insert(UserRecipe);
- }
- }
-
- // Visit operation "Links" along the reduction chain top-down starting from
- // the phi until LoopExitValue. We keep track of the previous item
- // (PreviousLink) to tell which of the two operands of a Link will remain
- // scalar and which will be reduced. For minmax by select(cmp), Link will be
- // the select instructions. Blend recipes of in-loop reduction phi's will
- // get folded to their non-phi operand, as the reduction recipe handles the
- // condition directly.
- VPSingleDefRecipe *PreviousLink = PhiR; // Aka Worklist[0].
- for (VPSingleDefRecipe *CurrentLink : drop_begin(Worklist)) {
- if (auto *Blend = dyn_cast<VPBlendRecipe>(CurrentLink)) {
- assert(Blend->getNumIncomingValues() == 2 &&
- "Blend must have 2 incoming values");
- if (Blend->getIncomingValue(0) == PhiR) {
- Blend->replaceAllUsesWith(Blend->getIncomingValue(1));
- } else {
- assert(Blend->getIncomingValue(1) == PhiR &&
- "PhiR must be an operand of the blend");
- Blend->replaceAllUsesWith(Blend->getIncomingValue(0));
- }
- continue;
- }
-
- if (IsFPRecurrence) {
- FastMathFlags CurFMF =
- cast<VPRecipeWithIRFlags>(CurrentLink)->getFastMathFlags();
- if (match(CurrentLink, m_Select(m_VPValue(), m_VPValue(), m_VPValue())))
- CurFMF |= cast<VPRecipeWithIRFlags>(CurrentLink->getOperand(0))
- ->getFastMathFlags();
- FMFs &= CurFMF;
- }
-
- Instruction *CurrentLinkI = CurrentLink->getUnderlyingInstr();
-
- // Index of the first operand which holds a non-mask vector operand.
- unsigned IndexOfFirstOperand;
- // Recognize a call to the llvm.fmuladd intrinsic.
- bool IsFMulAdd = (Kind == RecurKind::FMulAdd);
- VPValue *VecOp;
- VPBasicBlock *LinkVPBB = CurrentLink->getParent();
- if (IsFMulAdd) {
- assert(
- RecurrenceDescriptor::isFMulAddIntrinsic(CurrentLinkI) &&
- "Expected instruction to be a call to the llvm.fmuladd intrinsic");
- assert(((MinVF.isScalar() && isa<VPReplicateRecipe>(CurrentLink)) ||
- isa<VPWidenIntrinsicRecipe>(CurrentLink)) &&
- CurrentLink->getOperand(2) == PreviousLink &&
- "expected a call where the previous link is the added operand");
-
- // If the instruction is a call to the llvm.fmuladd intrinsic then we
- // need to create an fmul recipe (multiplying the first two operands of
- // the fmuladd together) to use as the vector operand for the fadd
- // reduction.
- VPInstruction *FMulRecipe = new VPInstruction(
- Instruction::FMul,
- {CurrentLink->getOperand(0), CurrentLink->getOperand(1)},
- CurrentLinkI->getFastMathFlags());
- LinkVPBB->insert(FMulRecipe, CurrentLink->getIterator());
- VecOp = FMulRecipe;
- } else if (PhiR->isInLoop() && Kind == RecurKind::AddChainWithSubs &&
- match(CurrentLink, m_Sub(m_VPValue(), m_VPValue()))) {
- Type *PhiTy = TypeInfo.inferScalarType(PhiR);
- auto *Zero = Plan->getConstantInt(PhiTy, 0);
- auto *Sub = new VPInstruction(Instruction::Sub,
- {Zero, CurrentLink->getOperand(1)}, {},
- {}, CurrentLinkI->getDebugLoc());
- Sub->setUnderlyingValue(CurrentLinkI);
- LinkVPBB->insert(Sub, CurrentLink->getIterator());
- VecOp = Sub;
- } else {
- if (RecurrenceDescriptor::isMinMaxRecurrenceKind(Kind)) {
- if (match(CurrentLink, m_Cmp(m_VPValue(), m_VPValue())))
- continue;
- assert(isa<VPWidenSelectRecipe>(CurrentLink) &&
- "must be a select recipe");
- IndexOfFirstOperand = 1;
- } else {
- assert((MinVF.isScalar() || isa<VPWidenRecipe>(CurrentLink)) &&
- "Expected to replace a VPWidenSC");
- IndexOfFirstOperand = 0;
- }
- // Note that for non-commutable operands (cmp-selects), the semantics of
- // the cmp-select are captured in the recurrence kind.
- unsigned VecOpId =
- CurrentLink->getOperand(IndexOfFirstOperand) == PreviousLink
- ? IndexOfFirstOperand + 1
- : IndexOfFirstOperand;
- VecOp = CurrentLink->getOperand(VecOpId);
- assert(VecOp != PreviousLink &&
- CurrentLink->getOperand(CurrentLink->getNumOperands() - 1 -
- (VecOpId - IndexOfFirstOperand)) ==
- PreviousLink &&
- "PreviousLink must be the operand other than VecOp");
- }
-
- VPValue *CondOp = nullptr;
- if (CM.blockNeedsPredicationForAnyReason(CurrentLinkI->getParent()))
- CondOp = RecipeBuilder.getBlockInMask(CurrentLink->getParent());
-
- ReductionStyle Style = getReductionStyle(true, PhiR->isOrdered(), 1);
- auto *RedRecipe =
- new VPReductionRecipe(Kind, FMFs, CurrentLinkI, PreviousLink, VecOp,
- CondOp, Style, CurrentLinkI->getDebugLoc());
- // Append the recipe to the end of the VPBasicBlock because we need to
- // ensure that it comes after all of it's inputs, including CondOp.
- // Delete CurrentLink as it will be invalid if its operand is replaced
- // with a reduction defined at the bottom of the block in the next link.
- if (LinkVPBB->getNumSuccessors() == 0)
- RedRecipe->insertBefore(&*std::prev(std::prev(LinkVPBB->end())));
- else
- LinkVPBB->appendRecipe(RedRecipe);
-
- CurrentLink->replaceAllUsesWith(RedRecipe);
- ToDelete.push_back(CurrentLink);
- PreviousLink = RedRecipe;
- }
- }
VPBasicBlock *LatchVPBB = VectorLoopRegion->getExitingBasicBlock();
Builder.setInsertPoint(&*std::prev(std::prev(LatchVPBB->end())));
VPBasicBlock::iterator IP = MiddleVPBB->getFirstNonPhi();
diff --git a/llvm/lib/Transforms/Vectorize/VPlanConstruction.cpp b/llvm/lib/Transforms/Vectorize/VPlanConstruction.cpp
index 318c05d8ef7c5..2e16e5d8553a6 100644
--- a/llvm/lib/Transforms/Vectorize/VPlanConstruction.cpp
+++ b/llvm/lib/Transforms/Vectorize/VPlanConstruction.cpp
@@ -13,6 +13,7 @@
#include "LoopVectorizationPlanner.h"
#include "VPlan.h"
+#include "VPlanAnalysis.h"
#include "VPlanCFG.h"
#include "VPlanDominatorTree.h"
#include "VPlanPatternMatch.h"
@@ -699,6 +700,164 @@ void VPlanTransforms::createHeaderPhiRecipes(
}
}
+void VPlanTransforms::createVPReductionRecipesForInLoopReductions(
+ VPlan &Plan, const DenseMap<VPBasicBlock *, VPValue *> &BlockMaskCache,
+ const DenseSet<BasicBlock *> &BlocksNeedingPredication,
+ ElementCount MinVF) {
+ VPTypeAnalysis TypeInfo(Plan);
+ VPBasicBlock *Header = Plan.getVectorLoopRegion()->getEntryBasicBlock();
+ SmallVector<VPRecipeBase *> ToDelete;
+
+ for (VPRecipeBase &R : Header->phis()) {
+ auto *PhiR = dyn_cast<VPReductionPHIRecipe>(&R);
+ if (!PhiR || !PhiR->isInLoop() || (MinVF.isScalar() && !PhiR->isOrdered()))
+ continue;
+
+ RecurKind Kind = PhiR->getRecurrenceKind();
+ assert(
+ !RecurrenceDescriptor::isAnyOfRecurrenceKind(Kind) &&
+ !RecurrenceDescriptor::isFindIVRecurrenceKind(Kind) &&
+ "AnyOf and FindIV reductions are not allowed for in-loop reductions");
+
+ bool IsFPRecurrence =
+ RecurrenceDescriptor::isFloatingPointRecurrenceKind(Kind);
+ FastMathFlags FMFs =
+ IsFPRecurrence ? FastMathFlags::getFast() : FastMathFlags();
+
+ // Collect the chain of "link" recipes for the reduction starting at PhiR.
+ SetVector<VPSingleDefRecipe *> Worklist;
+ Worklist.insert(PhiR);
+ for (unsigned I = 0; I != Worklist.size(); ++I) {
+ VPSingleDefRecipe *Cur = Worklist[I];
+ for (VPUser *U : Cur->users()) {
+ auto *UserRecipe = cast<VPSingleDefRecipe>(U);
+ if (!UserRecipe->getParent()->getEnclosingLoopRegion()) {
+ assert((UserRecipe->getParent() == Plan.getMiddleBlock() ||
+ UserRecipe->getParent() == Plan.getScalarPreheader()) &&
+ "U must be either in the loop region, the middle block or the "
+ "scalar preheader.");
+ continue;
+ }
+
+ // Stores using instructions will be sunk later.
+ if (match(UserRecipe, m_VPInstruction<Instruction::Store>()))
+ continue;
+ Worklist.insert(UserRecipe);
+ }
+ }
+
+ // Visit operation "Links" along the reduction chain top-down starting from
+ // the phi until LoopExitValue. We keep track of the previous item
+ // (PreviousLink) to tell which of the two operands of a Link will remain
+ // scalar and which will be reduced. For minmax by select(cmp), Link will be
+ // the select instructions. Blend recipes of in-loop reduction phi's will
+ // get folded to their non-phi operand, as the reduction recipe handles the
+ // condition directly.
+ VPSingleDefRecipe *PreviousLink = PhiR; // Aka Worklist[0].
+ for (VPSingleDefRecipe *CurrentLink : drop_begin(Worklist)) {
+ if (auto *Blend = dyn_cast<VPBlendRecipe>(CurrentLink)) {
+ assert(Blend->getNumIncomingValues() == 2 &&
+ "Blend must have 2 incoming values");
+ unsigned Idx = Blend->getIncomingValue(0) == PhiR ? 1 : 0;
+ assert(Blend->getIncomingValue(1 - Idx) == PhiR &&
+ "PhiR must be an operand of the blend");
+ Blend->replaceAllUsesWith(Blend->getIncomingValue(Idx));
+ continue;
+ }
+
+ if (IsFPRecurrence) {
+ FastMathFlags CurFMF =
+ cast<VPRecipeWithIRFlags>(CurrentLink)->getFastMathFlags();
+ if (match(CurrentLink, m_Select(m_VPValue(), m_VPValue(), m_VPValue())))
+ CurFMF |= cast<VPRecipeWithIRFlags>(CurrentLink->getOperand(0))
+ ->getFastMathFlags();
+ FMFs &= CurFMF;
+ }
+
+ Instruction *CurrentLinkI = CurrentLink->getUnderlyingInstr();
+
+ // Recognize a call to the llvm.fmuladd intrinsic.
+ bool IsFMulAdd = Kind == RecurKind::FMulAdd &&
+ RecurrenceDescriptor::isFMulAddIntrinsic(CurrentLinkI);
+ VPValue *VecOp;
+ VPBasicBlock *LinkVPBB = CurrentLink->getParent();
+ if (IsFMulAdd) {
+ assert(CurrentLink->getOperand(2) == PreviousLink &&
+ "expected a call where the previous link is the added operand");
+
+ // If the instruction is a call to the llvm.fmuladd intrinsic then we
+ // need to create an fmul recipe (multiplying the first two operands of
+ // the fmuladd together) to use as the vector operand for the fadd
+ // reduction.
+ auto *FMulRecipe = new VPInstruction(
+ Instruction::FMul,
+ {CurrentLink->getOperand(0), CurrentLink->getOperand(1)},
+ CurrentLinkI->getFastMathFlags());
+ LinkVPBB->insert(FMulRecipe, CurrentLink->getIterator());
+ VecOp = FMulRecipe;
+ } else if (Kind == RecurKind::AddChainWithSubs &&
+ match(CurrentLink, m_Sub(m_VPValue(), m_VPValue()))) {
+ Type *PhiTy = TypeInfo.inferScalarType(PhiR);
+ auto *Zero = Plan.getConstantInt(PhiTy, 0);
+ auto *Sub = new VPInstruction(Instruction::Sub,
+ {Zero, CurrentLink->getOperand(1)}, {},
+ {}, CurrentLinkI->getDebugLoc());
+ Sub->setUnderlyingValue(CurrentLinkI);
+ LinkVPBB->insert(Sub, CurrentLink->getIterator());
+ VecOp = Sub;
+ } else {
+ // Index of the first operand which holds a non-mask vector operand.
+ unsigned IndexOfFirstOperand = 0;
+ if (RecurrenceDescriptor::isMinMaxRecurrenceKind(Kind)) {
+ if (match(CurrentLink, m_Cmp(m_VPValue(), m_VPValue())))
+ continue;
+ assert(match(CurrentLink,
+ m_Select(m_VPValue(), m_VPValue(), m_VPValue())) &&
+ "must be a select recipe");
+ IndexOfFirstOperand = 1;
+ }
+ // Note that for non-commutable operands (cmp-selects), the semantics of
+ // the cmp-select are captured in the recurrence kind.
+ unsigned VecOpId =
+ CurrentLink->getOperand(IndexOfFirstOperand) == PreviousLink
+ ? IndexOfFirstOperand + 1
+ : IndexOfFirstOperand;
+ VecOp = CurrentLink->getOperand(VecOpId);
+ assert(VecOp != PreviousLink &&
+ CurrentLink->getOperand(CurrentLink->getNumOperands() - 1 -
+ (VecOpId - IndexOfFirstOperand)) ==
+ PreviousLink &&
+ "PreviousLink must be the operand other than VecOp");
+ }
+
+ // Get block mask from BlockMaskCache if the block needs predication.
+ VPValue *CondOp = nullptr;
+ if (BlocksNeedingPredication.contains(CurrentLinkI->getParent()))
+ CondOp = BlockMaskCache.lookup(LinkVPBB);
+
+ auto *RedRecipe = new VPReductionRecipe(
+ Kind, FMFs, CurrentLinkI, PreviousLink, VecOp, CondOp,
+ getReductionStyle(/*IsInLoop=*/true, PhiR->isOrdered(), 1),
+ CurrentLinkI->getDebugLoc());
+ // Append the recipe to the end of the VPBasicBlock because we need to
+ // ensure that it comes after all of it's inputs, including CondOp.
+ // Delete CurrentLink as it will be invalid if its operand is replaced
+ // with a reduction defined at the bottom of the block in the next link.
+ if (LinkVPBB->getNumSuccessors() == 0)
+ RedRecipe->insertBefore(&*std::prev(std::prev(LinkVPBB->end())));
+ else
+ LinkVPBB->appendRecipe(RedRecipe);
+
+ CurrentLink->replaceAllUsesWith(RedRecipe);
+ ToDelete.push_back(CurrentLink);
+ PreviousLink = RedRecipe;
+ }
+ }
+
+ for (VPRecipeBase *R : ToDelete)
+ R->eraseFromParent();
+}
+
void VPlanTransforms::handleEarlyExits(VPlan &Plan,
bool HasUncountableEarlyExit) {
auto *MiddleVPBB = cast<VPBasicBlock>(
diff --git a/llvm/lib/Transforms/Vectorize/VPlanTransforms.h b/llvm/lib/Transforms/Vectorize/VPlanTransforms.h
index 1a3ff4f9b9bbc..d3a4e1b70fc93 100644
--- a/llvm/lib/Transforms/Vectorize/VPlanTransforms.h
+++ b/llvm/lib/Transforms/Vectorize/VPlanTransforms.h
@@ -113,6 +113,15 @@ struct VPlanTransforms {
const SmallPtrSetImpl<const PHINode *> &FixedOrderRecurrences,
const SmallPtrSetImpl<PHINode *> &InLoopReductions, bool AllowReordering);
+ /// Create VPReductionRecipes for in-loop reductions. This processes chains
+ /// of operations contributing to in-loop reductions and creates appropriate
+ /// VPReductionRecipe instances. Block masks from \p BlockMaskCache are used
+ /// to add predication for blocks in \p BlocksNeedingPredication.
+ static void createVPReductionRecipesForInLoopReductions(
+ VPlan &Plan, const DenseMap<VPBasicBlock *, VPValue *> &BlockMaskCache,
+ const DenseSet<BasicBlock *> &BlocksNeedingPredication,
+ ElementCount MinVF);
+
/// Update \p Plan to account for all early exits.
LLVM_ABI_FOR_TEST static void handleEarlyExits(VPlan &Plan,
bool HasUncountableExit);
>From 7c5cb874f01a3d808926102b6742e34c1b069f11 Mon Sep 17 00:00:00 2001
From: Florian Hahn <flo at fhahn.com>
Date: Mon, 22 Dec 2025 16:45:33 +0000
Subject: [PATCH 2/2] !fixup address comments, thanks
---
.../lib/Transforms/Vectorize/LoopVectorize.cpp | 2 +-
.../Transforms/Vectorize/VPlanConstruction.cpp | 18 +++++++++++-------
.../lib/Transforms/Vectorize/VPlanTransforms.h | 2 +-
3 files changed, 13 insertions(+), 9 deletions(-)
diff --git a/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp b/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
index 47d6c05f2a6b4..045373d980c80 100644
--- a/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
+++ b/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
@@ -8451,7 +8451,7 @@ VPlanPtr LoopVectorizationPlanner::tryToBuildVPlanWithVPRecipes(
if (CM.blockNeedsPredicationForAnyReason(BB))
BlocksNeedingPredication.insert(BB);
- VPlanTransforms::createVPReductionRecipesForInLoopReductions(
+ VPlanTransforms::createInLoopReductionRecipes(
*Plan, BlockMaskCache, BlocksNeedingPredication, Range.Start);
// Now process all other blocks and instructions.
diff --git a/llvm/lib/Transforms/Vectorize/VPlanConstruction.cpp b/llvm/lib/Transforms/Vectorize/VPlanConstruction.cpp
index 1d87a313cf442..1455ec185d50c 100644
--- a/llvm/lib/Transforms/Vectorize/VPlanConstruction.cpp
+++ b/llvm/lib/Transforms/Vectorize/VPlanConstruction.cpp
@@ -703,7 +703,7 @@ void VPlanTransforms::createHeaderPhiRecipes(
}
}
-void VPlanTransforms::createVPReductionRecipesForInLoopReductions(
+void VPlanTransforms::createInLoopReductionRecipes(
VPlan &Plan, const DenseMap<VPBasicBlock *, VPValue *> &BlockMaskCache,
const DenseSet<BasicBlock *> &BlocksNeedingPredication,
ElementCount MinVF) {
@@ -753,7 +753,7 @@ void VPlanTransforms::createVPReductionRecipesForInLoopReductions(
// the phi until LoopExitValue. We keep track of the previous item
// (PreviousLink) to tell which of the two operands of a Link will remain
// scalar and which will be reduced. For minmax by select(cmp), Link will be
- // the select instructions. Blend recipes of in-loop reduction phi's will
+ // the select instructions. Blend recipes of in-loop reduction phi's will
// get folded to their non-phi operand, as the reduction recipe handles the
// condition directly.
VPSingleDefRecipe *PreviousLink = PhiR; // Aka Worklist[0].
@@ -761,10 +761,10 @@ void VPlanTransforms::createVPReductionRecipesForInLoopReductions(
if (auto *Blend = dyn_cast<VPBlendRecipe>(CurrentLink)) {
assert(Blend->getNumIncomingValues() == 2 &&
"Blend must have 2 incoming values");
- unsigned Idx = Blend->getIncomingValue(0) == PhiR ? 1 : 0;
- assert(Blend->getIncomingValue(1 - Idx) == PhiR &&
+ unsigned PhiRIdx = Blend->getIncomingValue(0) == PhiR ? 1 : 0;
+ assert(Blend->getIncomingValue(1 - PhiRIdx) == PhiR &&
"PhiR must be an operand of the blend");
- Blend->replaceAllUsesWith(Blend->getIncomingValue(Idx));
+ Blend->replaceAllUsesWith(Blend->getIncomingValue(PhiRIdx));
continue;
}
@@ -780,11 +780,13 @@ void VPlanTransforms::createVPReductionRecipesForInLoopReductions(
Instruction *CurrentLinkI = CurrentLink->getUnderlyingInstr();
// Recognize a call to the llvm.fmuladd intrinsic.
- bool IsFMulAdd = Kind == RecurKind::FMulAdd &&
- RecurrenceDescriptor::isFMulAddIntrinsic(CurrentLinkI);
+ bool IsFMulAdd = Kind == RecurKind::FMulAdd;
VPValue *VecOp;
VPBasicBlock *LinkVPBB = CurrentLink->getParent();
if (IsFMulAdd) {
+ assert(RecurrenceDescriptor::isFMulAddIntrinsic(CurrentLinkI) &&
+ "Expected current VPInstruction to be a call to the "
+ "llvm.fmuladd intrinsic");
assert(CurrentLink->getOperand(2) == PreviousLink &&
"expected a call where the previous link is the added operand");
@@ -838,6 +840,8 @@ void VPlanTransforms::createVPReductionRecipesForInLoopReductions(
if (BlocksNeedingPredication.contains(CurrentLinkI->getParent()))
CondOp = BlockMaskCache.lookup(LinkVPBB);
+ assert(PhiR->getVFScaleFactor() == 1 &&
+ "inloop reductions must be unscaled");
auto *RedRecipe = new VPReductionRecipe(
Kind, FMFs, CurrentLinkI, PreviousLink, VecOp, CondOp,
getReductionStyle(/*IsInLoop=*/true, PhiR->isOrdered(), 1),
diff --git a/llvm/lib/Transforms/Vectorize/VPlanTransforms.h b/llvm/lib/Transforms/Vectorize/VPlanTransforms.h
index ad6842fb75b53..1eebedcb0feb2 100644
--- a/llvm/lib/Transforms/Vectorize/VPlanTransforms.h
+++ b/llvm/lib/Transforms/Vectorize/VPlanTransforms.h
@@ -117,7 +117,7 @@ struct VPlanTransforms {
/// of operations contributing to in-loop reductions and creates appropriate
/// VPReductionRecipe instances. Block masks from \p BlockMaskCache are used
/// to add predication for blocks in \p BlocksNeedingPredication.
- static void createVPReductionRecipesForInLoopReductions(
+ static void createInLoopReductionRecipes(
VPlan &Plan, const DenseMap<VPBasicBlock *, VPValue *> &BlockMaskCache,
const DenseSet<BasicBlock *> &BlocksNeedingPredication,
ElementCount MinVF);
More information about the llvm-commits
mailing list