[llvm] [VPlan] Handle AnyOf Or reduction via ComputeReductionResult. (PR #191049)
via llvm-commits
llvm-commits at lists.llvm.org
Wed Apr 8 13:56:10 PDT 2026
llvmbot wrote:
<!--LLVM PR SUMMARY COMMENT-->
@llvm/pr-subscribers-vectorizers
Author: Florian Hahn (fhahn)
<details>
<summary>Changes</summary>
Instead of having ComputeAnyOfResult handle the Or reduction of unrolled parts inline, route it through ComputeReductionResult with RecurKind::Or. ComputeAnyOfResult now takes a pre-reduced scalar and only performs the freeze + select.
This is a preparatory step towards removing ComputeAnyOfResult entirely in https://github.com/llvm/llvm-project/pull/190039.
---
Full diff: https://github.com/llvm/llvm-project/pull/191049.diff
5 Files Affected:
- (modified) llvm/lib/Transforms/Vectorize/LoopVectorize.cpp (+22-10)
- (modified) llvm/lib/Transforms/Vectorize/VPlanRecipes.cpp (+1-8)
- (modified) llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp (+5-1)
- (modified) llvm/lib/Transforms/Vectorize/VPlanUnroll.cpp (+5-2)
- (modified) llvm/test/Transforms/LoopVectorize/epilog-vectorization-reductions.ll (+2-2)
``````````diff
diff --git a/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp b/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
index 1fc880208ebf7..10cca6377bb56 100644
--- a/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
+++ b/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
@@ -8368,9 +8368,13 @@ void LoopVectorizationPlanner::addReductionResultComputation(
VPValue *NewVal = AnyOfSelect->getOperand(1) == PhiR
? AnyOfSelect->getOperand(2)
: AnyOfSelect->getOperand(1);
- FinalReductionResult =
- Builder.createNaryOp(VPInstruction::ComputeAnyOfResult,
- {Start, NewVal, NewExitingVPV}, ExitDL);
+ VPIRFlags OrFlags(RecurKind::Or, /*IsOrdered=*/false,
+ /*IsInLoop=*/false, FastMathFlags());
+ auto *OrReduce =
+ Builder.createNaryOp(VPInstruction::ComputeReductionResult,
+ {NewExitingVPV}, OrFlags, ExitDL);
+ FinalReductionResult = Builder.createNaryOp(
+ VPInstruction::ComputeAnyOfResult, {Start, NewVal, OrReduce}, ExitDL);
} else {
VPIRFlags Flags(RecurrenceKind, PhiR->isOrdered(), PhiR->isInLoop(),
PhiR->getFastMathFlags());
@@ -8416,11 +8420,11 @@ void LoopVectorizationPlanner::addReductionResultComputation(
auto *Parent = cast<VPRecipeBase>(U)->getParent();
if (FinalReductionResult == U || Parent->getParent())
continue;
- // Skip FindIV reduction chain recipes (ComputeReductionResult, icmp).
- if (RecurrenceDescriptor::isFindIVRecurrenceKind(RecurrenceKind) &&
- match(U, m_CombineOr(
- m_VPInstruction<VPInstruction::ComputeReductionResult>(),
- m_VPInstruction<Instruction::ICmp>())))
+ // Skip ComputeReductionResult and FindIV reductions when they are not the
+ // final result.
+ if (match(U, m_VPInstruction<VPInstruction::ComputeReductionResult>()) ||
+ (RecurrenceDescriptor::isFindIVRecurrenceKind(RecurrenceKind) &&
+ match(U, m_VPInstruction<Instruction::ICmp>())))
continue;
U->replaceUsesOfWith(OrigExitingVPV, FinalReductionResult);
@@ -8995,12 +8999,20 @@ static SmallVector<Instruction *> preparePlanForEpilogueVectorLoop(
if (auto *ReductionPhi = dyn_cast<VPReductionPHIRecipe>(&R)) {
// Find the reduction result by searching users of the phi or its backedge
// value.
+ using namespace VPlanPatternMatch;
auto IsReductionResult = [](VPRecipeBase *R) {
auto *VPI = dyn_cast<VPInstruction>(R);
if (!VPI)
return false;
- return VPI->getOpcode() == VPInstruction::ComputeAnyOfResult ||
- VPI->getOpcode() == VPInstruction::ComputeReductionResult;
+ // Prefer ComputeAnyOfResult over intermediate ComputeReductionResult,
+ // which is used for the Or reduction in AnyOf reductions.
+ if (VPI->getOpcode() == VPInstruction::ComputeAnyOfResult)
+ return true;
+ if (VPI->getOpcode() != VPInstruction::ComputeReductionResult)
+ return false;
+ return !any_of(VPI->users(), [](VPUser *U) {
+ return match(U, m_VPInstruction<VPInstruction::ComputeAnyOfResult>());
+ });
};
auto *RdxResult = cast<VPInstruction>(
vputils::findRecipe(ReductionPhi->getBackedgeValue(), IsReductionResult));
diff --git a/llvm/lib/Transforms/Vectorize/VPlanRecipes.cpp b/llvm/lib/Transforms/Vectorize/VPlanRecipes.cpp
index bc442c22d1256..2c2f3915c41d4 100644
--- a/llvm/lib/Transforms/Vectorize/VPlanRecipes.cpp
+++ b/llvm/lib/Transforms/Vectorize/VPlanRecipes.cpp
@@ -723,14 +723,7 @@ Value *VPInstruction::generate(VPTransformState &State) {
case VPInstruction::ComputeAnyOfResult: {
Value *Start = State.get(getOperand(0), VPLane(0));
Value *NewVal = State.get(getOperand(1), VPLane(0));
- Value *ReducedResult = State.get(getOperand(2));
- for (unsigned Idx = 3; Idx < getNumOperands(); ++Idx)
- ReducedResult =
- Builder.CreateBinOp(Instruction::Or, State.get(getOperand(Idx)),
- ReducedResult, "bin.rdx");
- // If any predicate is true it means that we want to select the new value.
- if (ReducedResult->getType()->isVectorTy())
- ReducedResult = Builder.CreateOrReduce(ReducedResult);
+ Value *ReducedResult = State.get(getOperand(2), VPLane(0));
// The compares in the loop may yield poison, which propagates through the
// bitwise ORs. Freeze it here before the condition is used.
ReducedResult = Builder.CreateFreeze(ReducedResult);
diff --git a/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp b/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp
index 4264a216e2455..ef8c88ad81785 100644
--- a/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp
+++ b/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp
@@ -5947,9 +5947,13 @@ void VPlanTransforms::optimizeFindIVReductions(VPlan &Plan,
VPValue *OrVal = LoopBuilder.createOr(AnyOfPhi, AnyOfCond);
AnyOfPhi->setOperand(1, OrVal);
+ VPIRFlags OrFlags(RecurKind::Or, /*IsOrdered=*/false,
+ /*IsInLoop=*/false, FastMathFlags());
+ auto *OrReduce = MiddleBuilder.createNaryOp(
+ VPInstruction::ComputeReductionResult, {OrVal}, OrFlags, ExitDL);
NewRdxResult = MiddleBuilder.createNaryOp(
VPInstruction::ComputeAnyOfResult,
- {StartVPV, VectorRegionExitingVal, OrVal}, {}, ExitDL);
+ {StartVPV, VectorRegionExitingVal, OrReduce}, {}, ExitDL);
// Initialize the IV reduction phi with the neutral element, not the
// original start value, to ensure correct min/max reduction results.
diff --git a/llvm/lib/Transforms/Vectorize/VPlanUnroll.cpp b/llvm/lib/Transforms/Vectorize/VPlanUnroll.cpp
index 50b05a9e9c0fd..ef5fef7a75fe0 100644
--- a/llvm/lib/Transforms/Vectorize/VPlanUnroll.cpp
+++ b/llvm/lib/Transforms/Vectorize/VPlanUnroll.cpp
@@ -428,14 +428,17 @@ void UnrollState::unrollBlock(VPBlockBase *VPB) {
if (match(&R, m_VPInstruction<VPInstruction::AnyOf>(m_VPValue(Op1))) ||
match(&R, m_FirstActiveLane(m_VPValue(Op1))) ||
match(&R, m_LastActiveLane(m_VPValue(Op1))) ||
- match(&R,
- m_ComputeAnyOfResult(m_VPValue(), m_VPValue(), m_VPValue(Op1))) ||
match(&R, m_ComputeReductionResult(m_VPValue(Op1)))) {
addUniformForAllParts(cast<VPInstruction>(&R));
for (unsigned Part = 1; Part != UF; ++Part)
R.addOperand(getValueForPart(Op1, Part));
continue;
}
+ if (match(&R,
+ m_ComputeAnyOfResult(m_VPValue(), m_VPValue(), m_VPValue()))) {
+ addUniformForAllParts(cast<VPInstruction>(&R));
+ continue;
+ }
VPValue *Op0;
if (match(&R, m_ExtractLane(m_VPValue(Op0), m_VPValue(Op1)))) {
addUniformForAllParts(cast<VPInstruction>(&R));
diff --git a/llvm/test/Transforms/LoopVectorize/epilog-vectorization-reductions.ll b/llvm/test/Transforms/LoopVectorize/epilog-vectorization-reductions.ll
index f08f7ac0fd4e2..43487c0c87e04 100644
--- a/llvm/test/Transforms/LoopVectorize/epilog-vectorization-reductions.ll
+++ b/llvm/test/Transforms/LoopVectorize/epilog-vectorization-reductions.ll
@@ -1140,7 +1140,7 @@ define i32 @test_foldable_reduction(i64 %N) {
; CHECK: [[MIDDLE_BLOCK]]:
; CHECK-NEXT: [[TMP2:%.*]] = call i32 @llvm.vector.reduce.or.v4i32(<4 x i32> [[VEC_PHI]])
; CHECK-NEXT: [[TMP3:%.*]] = call i1 @llvm.vector.reduce.or.v4i1(<4 x i1> splat (i1 true))
-; CHECK-NEXT: [[TMP4:%.*]] = freeze i1 [[TMP3]]
+; CHECK-NEXT: [[TMP4:%.*]] = freeze i1 true
; CHECK-NEXT: [[RDX_SELECT:%.*]] = select i1 [[TMP4]], i32 0, i32 0
; CHECK-NEXT: [[CMP_N:%.*]] = icmp eq i64 [[TMP0]], [[N_VEC]]
; CHECK-NEXT: br i1 [[CMP_N]], label %[[EXIT:.*]], label %[[VEC_EPILOG_ITER_CHECK:.*]]
@@ -1163,7 +1163,7 @@ define i32 @test_foldable_reduction(i64 %N) {
; CHECK: [[VEC_EPILOG_MIDDLE_BLOCK]]:
; CHECK-NEXT: [[TMP7:%.*]] = call i32 @llvm.vector.reduce.or.v4i32(<4 x i32> [[VEC_PHI6]])
; CHECK-NEXT: [[TMP8:%.*]] = call i1 @llvm.vector.reduce.or.v4i1(<4 x i1> splat (i1 true))
-; CHECK-NEXT: [[TMP9:%.*]] = freeze i1 [[TMP8]]
+; CHECK-NEXT: [[TMP9:%.*]] = freeze i1 true
; CHECK-NEXT: [[RDX_SELECT8:%.*]] = select i1 [[TMP9]], i32 0, i32 0
; CHECK-NEXT: [[CMP_N9:%.*]] = icmp eq i64 [[TMP0]], [[N_VEC4]]
; CHECK-NEXT: br i1 [[CMP_N9]], label %[[EXIT]], label %[[VEC_EPILOG_SCALAR_PH]]
``````````
</details>
https://github.com/llvm/llvm-project/pull/191049
More information about the llvm-commits
mailing list