[llvm] f612211 - [VPlan] Introduce reduction selects for tail folding in foldTailByMasking. NFCI (#192987)
via llvm-commits
llvm-commits at lists.llvm.org
Tue May 12 07:30:32 PDT 2026
Author: Luke Lau
Date: 2026-05-12T14:30:26Z
New Revision: f612211116e1234a1ffc36c64aa662c0a462eac4
URL: https://github.com/llvm/llvm-project/commit/f612211116e1234a1ffc36c64aa662c0a462eac4
DIFF: https://github.com/llvm/llvm-project/commit/f612211116e1234a1ffc36c64aa662c0a462eac4.diff
LOG: [VPlan] Introduce reduction selects for tail folding in foldTailByMasking. NFCI (#192987)
Currently addComputeReductionResult has to check the cost model to see
if the loop is tail folded, and if so then manually fix up the backedge
value so any tail elements are ignored.
This PR moves this handling into foldTailByMasking itself so the plan
doesn't requiring fixing up. We do this by setting the incoming value
for the latch phi to the reduction phi instead of poison. A blend will
be created for this automatically.
The main benefits of this are that the reduction is correct when tail
folding is applied, and we don't need to worry about tail folding in as
many places.
In order to preserve some of the optimizations that we get on
VPInstruction::Select we need to convert the VPBlendRecipe to a select.
Added:
Modified:
llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
llvm/lib/Transforms/Vectorize/VPlanConstruction.cpp
llvm/test/Transforms/LoopVectorize/VPlan/tail-folding.ll
Removed:
################################################################################
diff --git a/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp b/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
index 61c2d3cd228ec..9fec3b74d9630 100644
--- a/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
+++ b/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
@@ -6620,9 +6620,19 @@ bool VPRecipeBuilder::replaceWithFinalIfReductionStore(
Legal->isInvariantAddressOfReduction(SI->getPointerOperand())) {
// Only create recipe for the final invariant store of the reduction.
if (Legal->isInvariantStoreOfReduction(SI)) {
+ VPValue *Val = VPI->getOperand(0);
+ VPValue *Addr = VPI->getOperand(1);
+ // We need to store the exiting value of the reduction, so use the blend
+ // if tail folded.
+ if (auto *Blend = vputils::findUserOf<VPBlendRecipe>(Val))
+ Val = Blend;
+ assert(
+ vputils::findUserOf<VPReductionPHIRecipe>(Val)->getBackedgeValue() ==
+ Val &&
+ "Store isn't backedge value?");
auto *Recipe = new VPReplicateRecipe(
- SI, VPI->operandsWithoutMask(), true /* IsUniform */,
- nullptr /*Mask*/, *VPI, *VPI, VPI->getDebugLoc());
+ SI, {Val, Addr}, true /* IsUniform */, nullptr /*Mask*/, *VPI, *VPI,
+ VPI->getDebugLoc());
FinalRedStoresBuilder.insert(Recipe);
}
VPI->eraseFromParent();
@@ -7087,6 +7097,7 @@ void LoopVectorizationPlanner::addReductionResultComputation(
VPBasicBlock *LatchVPBB = VectorLoopRegion->getExitingBasicBlock();
Builder.setInsertPoint(&*std::prev(std::prev(LatchVPBB->end())));
VPBasicBlock::iterator IP = MiddleVPBB->getFirstNonPhi();
+ VPValue *HeaderMask = vputils::findHeaderMask(*Plan);
for (VPRecipeBase &R :
Plan->getVectorLoopRegion()->getEntryBasicBlock()->phis()) {
VPReductionPHIRecipe *PhiR = dyn_cast<VPReductionPHIRecipe>(&R);
@@ -7097,23 +7108,28 @@ void LoopVectorizationPlanner::addReductionResultComputation(
const RecurrenceDescriptor &RdxDesc = Legal->getRecurrenceDescriptor(
cast<PHINode>(PhiR->getUnderlyingInstr()));
Type *PhiTy = TypeInfo.inferScalarType(PhiR);
- // If tail is folded by masking, introduce selects between the phi
- // and the users outside the vector region of each reduction, at the
- // beginning of the dedicated latch block.
+
+ // Convert a VPBlendRecipe backedge to a select.
+ if (auto *Blend = dyn_cast<VPBlendRecipe>(PhiR->getBackedgeValue())) {
+ if (Blend->getNumIncomingValues() == 2 &&
+ Blend->getMask(0) == HeaderMask) {
+ auto *Sel = VPBuilder(Blend).createSelect(
+ Blend->getMask(0), Blend->getIncomingValue(0),
+ Blend->getIncomingValue(1), {}, "", *Blend);
+ Blend->replaceAllUsesWith(Sel);
+ Blend->eraseFromParent();
+ }
+ }
+
auto *OrigExitingVPV = PhiR->getBackedgeValue();
auto *NewExitingVPV = PhiR->getBackedgeValue();
- if (!PhiR->isInLoop() && CM.foldTailByMasking()) {
- VPValue *Cond = vputils::findHeaderMask(*Plan);
- NewExitingVPV =
- Builder.createSelect(Cond, OrigExitingVPV, PhiR, {}, "", *PhiR);
- OrigExitingVPV->replaceUsesWithIf(NewExitingVPV, [](VPUser &U, unsigned) {
- return match(&U,
- m_VPInstruction<VPInstruction::ComputeReductionResult>());
- });
- if (CM.usePredicatedReductionSelect(RecurrenceKind))
- PhiR->setOperand(1, NewExitingVPV);
- }
+ // Remove the predicated select if the target doesn't want it.
+ VPValue *V;
+ if (!CM.usePredicatedReductionSelect(RecurrenceKind) &&
+ match(PhiR->getBackedgeValue(),
+ m_Select(m_Specific(HeaderMask), m_VPValue(V), m_Specific(PhiR))))
+ PhiR->setBackedgeValue(V);
// We want code in the middle block to appear to execute on the location of
// the scalar loop's latch terminator because: (a) it is all compiler
diff --git a/llvm/lib/Transforms/Vectorize/VPlanConstruction.cpp b/llvm/lib/Transforms/Vectorize/VPlanConstruction.cpp
index c2b05984db272..7c54a223f9793 100644
--- a/llvm/lib/Transforms/Vectorize/VPlanConstruction.cpp
+++ b/llvm/lib/Transforms/Vectorize/VPlanConstruction.cpp
@@ -1359,18 +1359,26 @@ void VPlanTransforms::foldTailByMasking(VPlan &Plan) {
if (match(&R, m_ExtractLastPart(m_VPValue(V))))
NeedsPhi[V].push_back(&R);
- // Insert phis with a poison incoming value for past the end of the tail.
+ // Insert phis for values coming past the end of the tail.
Builder.setInsertPoint(Latch, Latch->begin());
VPTypeAnalysis TypeInfo(Plan);
for (const auto &[V, Users] : NeedsPhi) {
if (isa<VPIRValue>(V))
continue;
- // TODO: For reduction phis, use phi value instead of poison so we can
- // remove the special casing for tail folding in
- // LoopVectorizationPlanner::addReductionResultComputation
- VPValue *Poison =
+ VPValue *TailVal =
Plan.getOrAddLiveIn(PoisonValue::get(TypeInfo.inferScalarType(V)));
- VPInstruction *Phi = Builder.createScalarPhi({V, Poison});
+ VPIRFlags Flags;
+ assert(llvm::count_if(Users, IsaPred<VPReductionPHIRecipe>) <= 1 &&
+ "Value used by more than two reduction phis?");
+ auto *RedIt = find_if(Users, IsaPred<VPReductionPHIRecipe>);
+ auto *RdxPhi =
+ RedIt != Users.end() ? cast<VPReductionPHIRecipe>(*RedIt) : nullptr;
+ if (RdxPhi && !RdxPhi->isInLoop()) {
+ TailVal = RdxPhi;
+ Flags = *RdxPhi;
+ }
+
+ VPInstruction *Phi = Builder.createScalarPhi({V, TailVal}, {}, "", Flags);
for (VPUser *U : Users)
U->replaceUsesOfWith(V, Phi);
}
diff --git a/llvm/test/Transforms/LoopVectorize/VPlan/tail-folding.ll b/llvm/test/Transforms/LoopVectorize/VPlan/tail-folding.ll
index 062ee47a43444..04b7c612f4587 100644
--- a/llvm/test/Transforms/LoopVectorize/VPlan/tail-folding.ll
+++ b/llvm/test/Transforms/LoopVectorize/VPlan/tail-folding.ll
@@ -284,7 +284,7 @@ define i32 @reduction(ptr noalias %p, i32 %n) {
; CHECK-NEXT: Successor(s): vector.latch
; CHECK-EMPTY:
; CHECK-NEXT: vector.latch:
-; CHECK-NEXT: EMIT-SCALAR vp<[[VP8]]> = phi [ ir<%rdx.next>, vector.body.split ], [ ir<poison>, vector.body ]
+; CHECK-NEXT: EMIT-SCALAR vp<[[VP8]]> = phi [ ir<%rdx.next>, vector.body.split ], [ ir<%rdx>, vector.body ]
; CHECK-NEXT: EMIT vp<%index.next> = add nuw vp<[[VP4]]>, vp<[[VP1]]>
; CHECK-NEXT: EMIT branch-on-count vp<%index.next>, vp<[[VP2]]>
; CHECK-NEXT: No successors
More information about the llvm-commits
mailing list