[llvm] [VPlan] Replace PhiR operand of ComputeAnyOfResult with VPIRFlags. (PR #175657)
Florian Hahn via llvm-commits
llvm-commits at lists.llvm.org
Sun Jan 18 01:59:32 PST 2026
https://github.com/fhahn updated https://github.com/llvm/llvm-project/pull/175657
>From eb542abf71112317d78f2b0a72df9bccdfac97e2 Mon Sep 17 00:00:00 2001
From: Florian Hahn <flo at fhahn.com>
Date: Mon, 12 Jan 2026 21:57:43 +0000
Subject: [PATCH 1/2] [VPlan] Replace PhiR operand of ComputeAnyOfResult with
VPIRFlags.
Replace the Phi recipe operand of ComputeAnyOfVResult with VPIRFlags, building on top of #174026.
---
.../Transforms/Vectorize/LoopVectorize.cpp | 24 ++++++++++++++-----
.../lib/Transforms/Vectorize/VPlanRecipes.cpp | 21 +++++++++++-----
2 files changed, 33 insertions(+), 12 deletions(-)
diff --git a/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp b/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
index ce0aab1e2bde8..bd284e46d92a9 100644
--- a/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
+++ b/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
@@ -7361,7 +7361,7 @@ static void fixReductionScalarResumeWhenVectorizingEpilog(
MainResumeValue = EpiRedHeaderPhi->getStartValue()->getUnderlyingValue();
if (RecurrenceDescriptor::isAnyOfRecurrenceKind(Kind)) {
[[maybe_unused]] Value *StartV =
- EpiRedResult->getOperand(1)->getLiveInIRValue();
+ EpiRedResult->getOperand(0)->getLiveInIRValue();
auto *Cmp = cast<ICmpInst>(MainResumeValue);
assert(Cmp->getPredicate() == CmpInst::ICMP_NE &&
"AnyOf expected to start with ICMP_NE");
@@ -8770,9 +8770,22 @@ void LoopVectorizationPlanner::addReductionResultComputation(
{PhiR, Start, Sentinel, NewExitingVPV}, ExitDL);
} else if (RecurrenceDescriptor::isAnyOfRecurrenceKind(RecurrenceKind)) {
VPValue *Start = PhiR->getStartValue();
+ // Find the NewVal from the original scalar select. NewVal is the value
+ // selected when the condition is true (i.e., the non-phi operand).
+ // We need to find the SelectInst among the original phi's users.
+ auto *OrigPhi = cast<PHINode>(PhiR->getUnderlyingInstr());
+ SelectInst *SI = nullptr;
+ for (auto *U : OrigPhi->users()) {
+ if ((SI = dyn_cast<SelectInst>(U)))
+ break;
+ }
+ assert(SI && "AnyOf reduction must have a select user");
+ Value *NewValIR = SI->getTrueValue() == OrigPhi ? SI->getFalseValue()
+ : SI->getTrueValue();
+ VPValue *NewVal = Plan->getOrAddLiveIn(NewValIR);
FinalReductionResult =
Builder.createNaryOp(VPInstruction::ComputeAnyOfResult,
- {PhiR, Start, NewExitingVPV}, ExitDL);
+ {Start, NewVal, NewExitingVPV}, ExitDL);
} else {
FastMathFlags FMFs =
RecurrenceDescriptor::isFloatingPointRecurrenceKind(RecurrenceKind)
@@ -9449,9 +9462,8 @@ static SmallVector<Instruction *> preparePlanForEpilogueVectorLoop(
ResumeV = cast<PHINode>(ReductionPhi->getUnderlyingInstr())
->getIncomingValueForBlock(L->getLoopPreheader());
- RecurKind RK = ReductionPhi->getRecurrenceKind();
- if (RecurrenceDescriptor::isAnyOfRecurrenceKind(RK)) {
- Value *StartV = RdxResult->getOperand(1)->getLiveInIRValue();
+ if (RdxResult->getOpcode() == VPInstruction::ComputeAnyOfResult) {
+ Value *StartV = RdxResult->getOperand(0)->getLiveInIRValue();
// VPReductionPHIRecipes for AnyOf reductions expect a boolean as
// start value; compare the final value from the main vector loop
// to the start value.
@@ -9460,7 +9472,7 @@ static SmallVector<Instruction *> preparePlanForEpilogueVectorLoop(
ResumeV = Builder.CreateICmpNE(ResumeV, StartV);
if (auto *I = dyn_cast<Instruction>(ResumeV))
InstsToMove.push_back(I);
- } else if (RecurrenceDescriptor::isFindIVRecurrenceKind(RK)) {
+ } else if (RdxResult->getOpcode() == VPInstruction::ComputeFindIVResult) {
Value *StartV = getStartValueFromReductionResult(RdxResult);
ToFrozen[StartV] = cast<PHINode>(ResumeV)->getIncomingValueForBlock(
EPI.MainLoopIterationCountCheck);
diff --git a/llvm/lib/Transforms/Vectorize/VPlanRecipes.cpp b/llvm/lib/Transforms/Vectorize/VPlanRecipes.cpp
index 701235de8b377..f9d9f2541ceda 100644
--- a/llvm/lib/Transforms/Vectorize/VPlanRecipes.cpp
+++ b/llvm/lib/Transforms/Vectorize/VPlanRecipes.cpp
@@ -711,17 +711,22 @@ Value *VPInstruction::generate(VPTransformState &State) {
Builder.getInt32(0));
}
case VPInstruction::ComputeAnyOfResult: {
- // FIXME: The cross-recipe dependency on VPReductionPHIRecipe is temporary
- // and will be removed by breaking up the recipe further.
- auto *PhiR = cast<VPReductionPHIRecipe>(getOperand(0));
- auto *OrigPhi = cast<PHINode>(PhiR->getUnderlyingValue());
+ // Operands: Start, NewVal, followed by reduction parts.
+ Value *InitVal = State.get(getOperand(0), VPLane(0));
+ Value *NewVal = State.get(getOperand(1), VPLane(0));
Value *ReducedPartRdx = State.get(getOperand(2));
for (unsigned Idx = 3; Idx < getNumOperands(); ++Idx)
ReducedPartRdx =
Builder.CreateBinOp(Instruction::Or, State.get(getOperand(Idx)),
ReducedPartRdx, "bin.rdx");
- return createAnyOfReduction(Builder, ReducedPartRdx,
- State.get(getOperand(1), VPLane(0)), OrigPhi);
+ // If any predicate is true it means that we want to select the new value.
+ Value *AnyOf = ReducedPartRdx->getType()->isVectorTy()
+ ? Builder.CreateOrReduce(ReducedPartRdx)
+ : ReducedPartRdx;
+ // The compares in the loop may yield poison, which propagates through the
+ // bitwise ORs. Freeze it here before the condition is used.
+ AnyOf = Builder.CreateFreeze(AnyOf);
+ return Builder.CreateSelect(AnyOf, NewVal, InitVal, "rdx.select");
}
case VPInstruction::ComputeFindIVResult: {
// FIXME: The cross-recipe dependency on VPReductionPHIRecipe is temporary
@@ -1410,6 +1415,7 @@ bool VPInstruction::usesFirstLaneOnly(const VPValue *Op) const {
// WidePtrAdd supports scalar and vector base addresses.
return false;
case VPInstruction::ComputeAnyOfResult:
+ return Op == getOperand(0) || Op == getOperand(1);
case VPInstruction::ComputeFindIVResult:
return Op == getOperand(1);
case VPInstruction::ExtractLane:
@@ -2176,6 +2182,9 @@ void VPIRFlags::printFlags(raw_ostream &O) const {
RecurKind RK = getRecurKind();
O << " (";
switch (RK) {
+ case RecurKind::AnyOf:
+ O << "any-of";
+ break;
case RecurKind::SMax:
O << "smax";
break;
>From 47678b93604496ff1bb74c9fe98362f880a343e5 Mon Sep 17 00:00:00 2001
From: Florian Hahn <flo at fhahn.com>
Date: Fri, 16 Jan 2026 13:08:12 +0000
Subject: [PATCH 2/2] !fixup address latest comments, thanks
---
.../Transforms/Vectorize/LoopVectorize.cpp | 49 +++++++++----------
1 file changed, 23 insertions(+), 26 deletions(-)
diff --git a/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp b/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
index bd284e46d92a9..bf95a5ff70845 100644
--- a/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
+++ b/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
@@ -8762,27 +8762,26 @@ void LoopVectorizationPlanner::addReductionResultComputation(
VPBuilder::InsertPointGuard Guard(Builder);
Builder.setInsertPoint(MiddleVPBB, IP);
RecurKind RecurrenceKind = PhiR->getRecurrenceKind();
+ // For AnyOf reductions, find the select among PhiR's users. This is used
+ // both to find NewVal for ComputeAnyOfResult and to adjust the reduction.
+ VPRecipeBase *AnyOfSelect = nullptr;
+ if (RecurrenceDescriptor::isAnyOfRecurrenceKind(RecurrenceKind)) {
+ AnyOfSelect = cast<VPRecipeBase>(*find_if(PhiR->users(), [](VPUser *U) {
+ return match(U, m_Select(m_VPValue(), m_VPValue(), m_VPValue()));
+ }));
+ }
if (RecurrenceDescriptor::isFindIVRecurrenceKind(RecurrenceKind)) {
VPValue *Start = PhiR->getStartValue();
VPValue *Sentinel = Plan->getOrAddLiveIn(RdxDesc.getSentinelValue());
FinalReductionResult =
Builder.createNaryOp(VPInstruction::ComputeFindIVResult,
{PhiR, Start, Sentinel, NewExitingVPV}, ExitDL);
- } else if (RecurrenceDescriptor::isAnyOfRecurrenceKind(RecurrenceKind)) {
+ } else if (AnyOfSelect) {
VPValue *Start = PhiR->getStartValue();
- // Find the NewVal from the original scalar select. NewVal is the value
- // selected when the condition is true (i.e., the non-phi operand).
- // We need to find the SelectInst among the original phi's users.
- auto *OrigPhi = cast<PHINode>(PhiR->getUnderlyingInstr());
- SelectInst *SI = nullptr;
- for (auto *U : OrigPhi->users()) {
- if ((SI = dyn_cast<SelectInst>(U)))
- break;
- }
- assert(SI && "AnyOf reduction must have a select user");
- Value *NewValIR = SI->getTrueValue() == OrigPhi ? SI->getFalseValue()
- : SI->getTrueValue();
- VPValue *NewVal = Plan->getOrAddLiveIn(NewValIR);
+ // NewVal is the non-phi operand of the select.
+ VPValue *NewVal = AnyOfSelect->getOperand(1) == PhiR
+ ? AnyOfSelect->getOperand(2)
+ : AnyOfSelect->getOperand(1);
FinalReductionResult =
Builder.createNaryOp(VPInstruction::ComputeAnyOfResult,
{Start, NewVal, NewExitingVPV}, ExitDL);
@@ -8850,25 +8849,22 @@ void LoopVectorizationPlanner::addReductionResultComputation(
// with a boolean reduction phi node to check if the condition is true in
// any iteration. The final value is selected by the final
// ComputeReductionResult.
- if (RecurrenceDescriptor::isAnyOfRecurrenceKind(RecurrenceKind)) {
- auto *Select = cast<VPRecipeBase>(*find_if(PhiR->users(), [](VPUser *U) {
- return match(U, m_Select(m_VPValue(), m_VPValue(), m_VPValue()));
- }));
- VPValue *Cmp = Select->getOperand(0);
+ if (AnyOfSelect) {
+ VPValue *Cmp = AnyOfSelect->getOperand(0);
// If the compare is checking the reduction PHI node, adjust it to check
// the start value.
if (VPRecipeBase *CmpR = Cmp->getDefiningRecipe())
CmpR->replaceUsesOfWith(PhiR, PhiR->getStartValue());
- Builder.setInsertPoint(Select);
+ Builder.setInsertPoint(AnyOfSelect);
// If the true value of the select is the reduction phi, the new value is
// selected if the negated condition is true in any iteration.
- if (Select->getOperand(1) == PhiR)
+ if (AnyOfSelect->getOperand(1) == PhiR)
Cmp = Builder.createNot(Cmp);
VPValue *Or = Builder.createOr(PhiR, Cmp);
- Select->getVPSingleValue()->replaceAllUsesWith(Or);
- // Delete Select now that it has invalid types.
- ToDelete.push_back(Select);
+ AnyOfSelect->getVPSingleValue()->replaceAllUsesWith(Or);
+ // Delete AnyOfSelect now that it has invalid types.
+ ToDelete.push_back(AnyOfSelect);
// Convert the reduction phi to operate on bools.
PhiR->setOperand(0, Plan->getFalse());
@@ -9462,7 +9458,8 @@ static SmallVector<Instruction *> preparePlanForEpilogueVectorLoop(
ResumeV = cast<PHINode>(ReductionPhi->getUnderlyingInstr())
->getIncomingValueForBlock(L->getLoopPreheader());
- if (RdxResult->getOpcode() == VPInstruction::ComputeAnyOfResult) {
+ RecurKind RK = ReductionPhi->getRecurrenceKind();
+ if (RecurrenceDescriptor::isAnyOfRecurrenceKind(RK)) {
Value *StartV = RdxResult->getOperand(0)->getLiveInIRValue();
// VPReductionPHIRecipes for AnyOf reductions expect a boolean as
// start value; compare the final value from the main vector loop
@@ -9472,7 +9469,7 @@ static SmallVector<Instruction *> preparePlanForEpilogueVectorLoop(
ResumeV = Builder.CreateICmpNE(ResumeV, StartV);
if (auto *I = dyn_cast<Instruction>(ResumeV))
InstsToMove.push_back(I);
- } else if (RdxResult->getOpcode() == VPInstruction::ComputeFindIVResult) {
+ } else if (RecurrenceDescriptor::isFindIVRecurrenceKind(RK)) {
Value *StartV = getStartValueFromReductionResult(RdxResult);
ToFrozen[StartV] = cast<PHINode>(ResumeV)->getIncomingValueForBlock(
EPI.MainLoopIterationCountCheck);
More information about the llvm-commits
mailing list