[llvm] [VPlan] Replace ComputeFindIVRes with ComputeRdxRes + cmp + sel (NFC) (PR #176672)
Florian Hahn via llvm-commits
llvm-commits at lists.llvm.org
Mon Jan 19 02:34:03 PST 2026
https://github.com/fhahn updated https://github.com/llvm/llvm-project/pull/176672
>From 83cee391403d4ae6434831096d63494c1e88d85c Mon Sep 17 00:00:00 2001
From: Florian Hahn <flo at fhahn.com>
Date: Sun, 18 Jan 2026 22:24:19 +0000
Subject: [PATCH] [VPlan] Replace ComputeFindIVRes with ComputeRdxRes + cmp +
sel (NFC)
Replace ComputeFindIVResult with ComputeReductionResult + explicit
compare + select, to more explicitly and simpler model computing finding
the first/last induction, which boils down to a min/max reduction +
compare and select of the sentinel value.
---
.../Transforms/Vectorize/LoopVectorize.cpp | 134 +++++++++---------
llvm/lib/Transforms/Vectorize/VPlan.h | 1 -
.../Transforms/Vectorize/VPlanAnalysis.cpp | 1 -
.../Vectorize/VPlanConstruction.cpp | 39 ++---
.../Transforms/Vectorize/VPlanPatternMatch.h | 7 -
.../lib/Transforms/Vectorize/VPlanRecipes.cpp | 42 +-----
llvm/lib/Transforms/Vectorize/VPlanUnroll.cpp | 4 +-
llvm/lib/Transforms/Vectorize/VPlanUtils.cpp | 1 +
.../vplan-printing-reductions.ll | 4 +-
9 files changed, 99 insertions(+), 134 deletions(-)
diff --git a/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp b/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
index 0b4546636e584..6c2be05e459a0 100644
--- a/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
+++ b/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
@@ -7303,13 +7303,15 @@ static VPRecipeBase *findRecipe(VPValue *Start, PredT Pred) {
return nullptr;
}
-static Value *getStartValueFromReductionResult(VPInstruction *RdxResult) {
+/// Match FindIV result: select(icmp ne ReducedIV, Sentinel), ReducedIV, Start.
+template <typename Op0_t, typename Op1_t>
+static bool matchFindIVResult(VPInstruction *VPI, Op0_t ReducedIV,
+ Op1_t Start) {
using namespace VPlanPatternMatch;
- assert(RdxResult->getOpcode() == VPInstruction::ComputeFindIVResult &&
- "RdxResult must be ComputeFindIVResult");
- VPValue *StartVPV = RdxResult->getOperand(0);
- match(StartVPV, m_Freeze(m_VPValue(StartVPV)));
- return StartVPV->getLiveInIRValue();
+ return match(VPI, m_Select(m_SpecificICmp(ICmpInst::ICMP_NE,
+ m_ComputeReductionResult(ReducedIV),
+ m_VPValue()),
+ m_ComputeReductionResult(ReducedIV), Start));
}
// If \p EpiResumePhiR is resume VPPhi for a reduction when vectorizing the
@@ -7317,6 +7319,7 @@ static Value *getStartValueFromReductionResult(VPInstruction *RdxResult) {
// from the main vector loop.
static void fixReductionScalarResumeWhenVectorizingEpilog(
VPPhi *EpiResumePhiR, PHINode &EpiResumePhi, BasicBlock *BypassBlock) {
+ using namespace VPlanPatternMatch;
// Get the VPInstruction computing the reduction result in the middle block.
// The first operand may not be from the middle block if it is not connected
// to the scalar preheader. In that case, there's nothing to fix.
@@ -7324,15 +7327,17 @@ static void fixReductionScalarResumeWhenVectorizingEpilog(
match(Incoming, VPlanPatternMatch::m_ZExtOrSExt(
VPlanPatternMatch::m_VPValue(Incoming)));
auto *EpiRedResult = dyn_cast<VPInstruction>(Incoming);
- if (!EpiRedResult ||
- (EpiRedResult->getOpcode() != VPInstruction::ComputeAnyOfResult &&
- EpiRedResult->getOpcode() != VPInstruction::ComputeReductionResult &&
- EpiRedResult->getOpcode() != VPInstruction::ComputeFindIVResult))
+ if (!EpiRedResult)
+ return;
+
+ VPValue *BackedgeVal;
+ if (EpiRedResult->getOpcode() == VPInstruction::ComputeAnyOfResult ||
+ EpiRedResult->getOpcode() == VPInstruction::ComputeReductionResult)
+ BackedgeVal = EpiRedResult->getOperand(EpiRedResult->getNumOperands() - 1);
+ else if (!matchFindIVResult(EpiRedResult, m_VPValue(BackedgeVal),
+ m_VPValue()))
return;
- // Find the reduction phi by searching users of the backedge value.
- VPValue *BackedgeVal =
- EpiRedResult->getOperand(EpiRedResult->getNumOperands() - 1);
auto *EpiRedHeaderPhi = cast_if_present<VPReductionPHIRecipe>(
findRecipe(BackedgeVal, IsaPred<VPReductionPHIRecipe>));
if (!EpiRedHeaderPhi) {
@@ -7364,19 +7369,8 @@ static void fixReductionScalarResumeWhenVectorizingEpilog(
"start value");
MainResumeValue = Cmp->getOperand(0);
} else if (RecurrenceDescriptor::isFindIVRecurrenceKind(Kind)) {
- Value *StartV = getStartValueFromReductionResult(EpiRedResult);
- Value *SentinelV = EpiRedResult->getOperand(1)->getLiveInIRValue();
- using namespace llvm::PatternMatch;
- Value *Cmp, *OrigResumeV, *CmpOp;
- [[maybe_unused]] bool IsExpectedPattern =
- match(MainResumeValue,
- m_Select(m_OneUse(m_Value(Cmp)), m_Specific(SentinelV),
- m_Value(OrigResumeV))) &&
- (match(Cmp, m_SpecificICmp(ICmpInst::ICMP_EQ, m_Specific(OrigResumeV),
- m_Value(CmpOp))) &&
- ((CmpOp == StartV && isGuaranteedNotToBeUndefOrPoison(CmpOp))));
- assert(IsExpectedPattern && "Unexpected reduction resume pattern");
- MainResumeValue = OrigResumeV;
+ auto *Sel = cast<SelectInst>(MainResumeValue);
+ MainResumeValue = Sel->getFalseValue();
}
PHINode *MainResumePhi = cast<PHINode>(MainResumeValue);
@@ -8727,13 +8721,11 @@ void LoopVectorizationPlanner::addReductionResultComputation(
NewExitingVPV =
Builder.createSelect(Cond, OrigExitingVPV, PhiR, {}, "", FMFs);
OrigExitingVPV->replaceUsesWithIf(NewExitingVPV, [](VPUser &U, unsigned) {
- return isa<VPInstruction>(&U) &&
- (cast<VPInstruction>(&U)->getOpcode() ==
- VPInstruction::ComputeAnyOfResult ||
- cast<VPInstruction>(&U)->getOpcode() ==
- VPInstruction::ComputeReductionResult ||
- cast<VPInstruction>(&U)->getOpcode() ==
- VPInstruction::ComputeFindIVResult);
+ using namespace VPlanPatternMatch;
+ return match(
+ &U, m_CombineOr(
+ m_VPInstruction<VPInstruction::ComputeAnyOfResult>(),
+ m_VPInstruction<VPInstruction::ComputeReductionResult>()));
});
if (CM.usePredicatedReductionSelect())
PhiR->setOperand(1, NewExitingVPV);
@@ -8776,9 +8768,13 @@ void LoopVectorizationPlanner::addReductionResultComputation(
MinMaxKind = IsSigned ? RecurKind::SMin : RecurKind::UMin;
VPIRFlags Flags(MinMaxKind, /*IsOrdered=*/false, /*IsInLoop=*/false,
FastMathFlags());
- FinalReductionResult =
- Builder.createNaryOp(VPInstruction::ComputeFindIVResult,
- {Start, Sentinel, NewExitingVPV}, Flags, ExitDL);
+ auto *ReducedIV =
+ Builder.createNaryOp(VPInstruction::ComputeReductionResult,
+ {NewExitingVPV}, Flags, ExitDL);
+ auto *Cmp =
+ Builder.createICmp(CmpInst::ICMP_NE, ReducedIV, Sentinel, ExitDL);
+ FinalReductionResult = cast<VPInstruction>(
+ Builder.createSelect(Cmp, ReducedIV, Start, ExitDL));
} else if (AnyOfSelect) {
VPValue *Start = PhiR->getStartValue();
// NewVal is the non-phi operand of the select.
@@ -8837,6 +8833,12 @@ void LoopVectorizationPlanner::addReductionResultComputation(
auto *Parent = cast<VPRecipeBase>(U)->getParent();
if (FinalReductionResult == U || Parent->getParent())
continue;
+ // Skip FindIV reduction chain recipes (ComputeReductionResult, icmp).
+ if (RecurrenceDescriptor::isFindIVRecurrenceKind(RecurrenceKind) &&
+ match(U, m_CombineOr(
+ m_VPInstruction<VPInstruction::ComputeReductionResult>(),
+ m_VPInstruction<Instruction::ICmp>())))
+ continue;
U->replaceUsesOfWith(OrigExitingVPV, FinalReductionResult);
// Look through ExtractLastPart.
@@ -9324,14 +9326,16 @@ static void preparePlanForMainVectorLoop(VPlan &MainPlan, VPlan &EpiPlan) {
VPBuilder Builder(Plan.getEntry());
for (VPRecipeBase &R : *Plan.getMiddleBlock()) {
auto *VPI = dyn_cast<VPInstruction>(&R);
- if (!VPI || VPI->getOpcode() != VPInstruction::ComputeFindIVResult)
+ if (!VPI)
+ continue;
+ VPValue *OrigStart;
+ if (!matchFindIVResult(VPI, m_VPValue(), m_VPValue(OrigStart)))
continue;
- VPValue *OrigStart = VPI->getOperand(0);
if (isGuaranteedNotToBeUndefOrPoison(OrigStart->getLiveInIRValue()))
continue;
VPInstruction *Freeze =
Builder.createNaryOp(Instruction::Freeze, {OrigStart}, {}, "fr");
- VPI->setOperand(0, Freeze);
+ VPI->setOperand(2, Freeze);
if (UpdateResumePhis)
OrigStart->replaceUsesWithIf(Freeze, [Freeze](VPUser &U, unsigned) {
return Freeze != &U && isa<VPPhi>(&U);
@@ -9438,30 +9442,22 @@ static SmallVector<Instruction *> preparePlanForEpilogueVectorLoop(
Value *ResumeV = nullptr;
// TODO: Move setting of resume values to prepareToExecute.
if (auto *ReductionPhi = dyn_cast<VPReductionPHIRecipe>(&R)) {
+ RecurKind RK = ReductionPhi->getRecurrenceKind();
// Find the reduction result by searching users of the phi or its backedge
// value.
auto IsReductionResult = [](VPRecipeBase *R) {
auto *VPI = dyn_cast<VPInstruction>(R);
- return VPI &&
- (VPI->getOpcode() == VPInstruction::ComputeAnyOfResult ||
- VPI->getOpcode() == VPInstruction::ComputeReductionResult ||
- VPI->getOpcode() == VPInstruction::ComputeFindIVResult);
+ if (!VPI)
+ return false;
+ return VPI->getOpcode() == VPInstruction::ComputeAnyOfResult ||
+ VPI->getOpcode() == VPInstruction::ComputeReductionResult;
};
auto *RdxResult = cast<VPInstruction>(
findRecipe(ReductionPhi->getBackedgeValue(), IsReductionResult));
- assert(
- (is_contained(RdxResult->operands(),
- ReductionPhi->getBackedgeValue()) ||
- (isa<VPWidenCastRecipe>(ReductionPhi->getBackedgeValue()) &&
- is_contained(RdxResult->operands(), ReductionPhi->getBackedgeValue()
- ->getDefiningRecipe()
- ->getOperand(0))) ||
- RdxResult->getOpcode() == VPInstruction::ComputeFindIVResult) &&
- "expected to find reduction result via backedge");
+ assert(RdxResult && "expected to find reduction result");
ResumeV = cast<PHINode>(ReductionPhi->getUnderlyingInstr())
->getIncomingValueForBlock(L->getLoopPreheader());
- RecurKind RK = ReductionPhi->getRecurrenceKind();
if (RecurrenceDescriptor::isAnyOfRecurrenceKind(RK)) {
Value *StartV = RdxResult->getOperand(0)->getLiveInIRValue();
// VPReductionPHIRecipes for AnyOf reductions expect a boolean as
@@ -9473,23 +9469,33 @@ static SmallVector<Instruction *> preparePlanForEpilogueVectorLoop(
if (auto *I = dyn_cast<Instruction>(ResumeV))
InstsToMove.push_back(I);
} else if (RecurrenceDescriptor::isFindIVRecurrenceKind(RK)) {
- Value *StartV = getStartValueFromReductionResult(RdxResult);
- ToFrozen[StartV] = cast<PHINode>(ResumeV)->getIncomingValueForBlock(
+ using namespace VPlanPatternMatch;
+ // Find the icmp user of RdxResult to get the sentinel value.
+ // The pattern is: select(icmp ne RdxResult, Sentinel), RdxResult, Start
+ VPValue *SentinelVPV = nullptr;
+ for (VPUser *U : RdxResult->users()) {
+ if (match(U, VPlanPatternMatch::m_SpecificICmp(
+ ICmpInst::ICMP_NE, m_Specific(RdxResult),
+ m_VPValue(SentinelVPV))))
+ break;
+ }
+ assert(SentinelVPV && "expected to find icmp using RdxResult");
+
+ // Get the frozen start value from the main loop.
+ Value *FrozenStartV = cast<PHINode>(ResumeV)->getIncomingValueForBlock(
EPI.MainLoopIterationCountCheck);
+ if (auto *FreezeI = dyn_cast<FreezeInst>(FrozenStartV))
+ ToFrozen[FreezeI->getOperand(0)] = FrozenStartV;
- // VPReductionPHIRecipe for FindFirstIV/FindLastIV reductions requires
- // an adjustment to the resume value. The resume value is adjusted to
- // the sentinel value when the final value from the main vector loop
- // equals the start value. This ensures correctness when the start value
- // might not be less than the minimum value of a monotonically
- // increasing induction variable.
+ // Adjust resume: select(icmp eq ResumeV, FrozenStartV), Sentinel,
+ // ResumeV
BasicBlock *ResumeBB = cast<Instruction>(ResumeV)->getParent();
IRBuilder<> Builder(ResumeBB, ResumeBB->getFirstNonPHIIt());
- Value *Cmp = Builder.CreateICmpEQ(ResumeV, ToFrozen[StartV]);
+ Value *Cmp = Builder.CreateICmpEQ(ResumeV, FrozenStartV);
if (auto *I = dyn_cast<Instruction>(Cmp))
InstsToMove.push_back(I);
- Value *Sentinel = RdxResult->getOperand(1)->getLiveInIRValue();
- ResumeV = Builder.CreateSelect(Cmp, Sentinel, ResumeV);
+ ResumeV =
+ Builder.CreateSelect(Cmp, SentinelVPV->getLiveInIRValue(), ResumeV);
if (auto *I = dyn_cast<Instruction>(ResumeV))
InstsToMove.push_back(I);
} else {
diff --git a/llvm/lib/Transforms/Vectorize/VPlan.h b/llvm/lib/Transforms/Vectorize/VPlan.h
index 94a19beb75a8f..099d5d06b3e92 100644
--- a/llvm/lib/Transforms/Vectorize/VPlan.h
+++ b/llvm/lib/Transforms/Vectorize/VPlan.h
@@ -1131,7 +1131,6 @@ class LLVM_ABI_FOR_TEST VPInstruction : public VPRecipeWithIRFlags,
/// Compute the final result of a AnyOf reduction with select(cmp(),x,y),
/// where one of (x,y) is loop invariant, and both x and y are integer type.
ComputeAnyOfResult,
- ComputeFindIVResult,
ComputeReductionResult,
// Extracts the last part of its operand. Removed during unrolling.
ExtractLastPart,
diff --git a/llvm/lib/Transforms/Vectorize/VPlanAnalysis.cpp b/llvm/lib/Transforms/Vectorize/VPlanAnalysis.cpp
index 39e7a044dfff5..ba255313283d1 100644
--- a/llvm/lib/Transforms/Vectorize/VPlanAnalysis.cpp
+++ b/llvm/lib/Transforms/Vectorize/VPlanAnalysis.cpp
@@ -77,7 +77,6 @@ Type *VPTypeAnalysis::inferScalarTypeForRecipe(const VPInstruction *R) {
case Instruction::Freeze:
case Instruction::PHI:
case VPInstruction::Broadcast:
- case VPInstruction::ComputeFindIVResult:
case VPInstruction::ComputeReductionResult:
case VPInstruction::ExtractLastLane:
case VPInstruction::ExtractPenultimateElement:
diff --git a/llvm/lib/Transforms/Vectorize/VPlanConstruction.cpp b/llvm/lib/Transforms/Vectorize/VPlanConstruction.cpp
index 1f8243d5f6c72..2196501f4cd3a 100644
--- a/llvm/lib/Transforms/Vectorize/VPlanConstruction.cpp
+++ b/llvm/lib/Transforms/Vectorize/VPlanConstruction.cpp
@@ -1511,35 +1511,40 @@ bool VPlanTransforms::handleMultiUseReductions(VPlan &Plan) {
//
// For example, this transforms
// vp<%min.result> = compute-reduction-result ir<%min.val.next>
- // vp<%find.iv.result = compute-find-iv-result ir<0>, SENTINEL,
- // vp<%min.idx.next>
+ // vp<%iv.rdx> = compute-reduction-result (smax) vp<%min.idx.next>
+ // vp<%cmp> = icmp ne vp<%iv.rdx>, SENTINEL
+ // vp<%find.iv.result> = select vp<%cmp>, vp<%iv.rdx>, ir<0>
//
// into:
//
// vp<min.result> = compute-reduction-result ir<%min.val.next>
// vp<%final.min.cmp> = icmp eq ir<%min.val.next>, vp<min.result>
- // vp<%final.iv> = select vp<%final.min.cmp>, ir<%min.idx.next>, SENTINEL
- // vp<%find.iv.result> = compute-find-iv-result ir<0>, SENTINEL,
- // vp<%final.iv>
- VPInstruction *FindIVResult =
- findUserOf<VPInstruction::ComputeFindIVResult>(
- FindIVPhiR->getBackedgeValue());
- assert(FindIVResult && "Backedge value feeding FindIVPhiR expected to also "
- "feed a ComputeFindIVResult");
- assert(FindIVResult->getParent() == MinOrMaxResult->getParent() &&
+ // vp<%final.iv> = select vp<%final.min.cmp>, vp<%min.idx.next>, SENTINEL
+ // vp<%iv.rdx> = compute-reduction-result (smax) vp<%final.iv>
+ // vp<%cmp> = icmp ne vp<%iv.rdx>, SENTINEL
+ // vp<%find.iv.result> = select vp<%cmp>, vp<%iv.rdx>, ir<0>
+ //
+ // Find the ComputeReductionResult with minmax kind for FindIV.
+ auto *FindIVRdxResult = cast<VPInstruction>(findUserOf(
+ FindIVPhiR->getBackedgeValue(),
+ m_VPInstruction<VPInstruction::ComputeReductionResult>(m_VPValue())));
+ // Find the icmp -> select pattern wrapping the reduction result.
+ auto *FindIVCmp = findUserOf<Instruction::ICmp>(FindIVRdxResult);
+ auto *FindIVSelect = findUserOf<Instruction::Select>(FindIVCmp);
+ assert(FindIVSelect->getParent() == MinOrMaxResult->getParent() &&
"both results must be computed in the same block");
- MinOrMaxResult->moveBefore(*FindIVResult->getParent(),
- FindIVResult->getIterator());
+ MinOrMaxResult->moveBefore(*FindIVRdxResult->getParent(),
+ FindIVRdxResult->getIterator());
- VPBuilder B(FindIVResult);
+ VPBuilder B(FindIVRdxResult);
VPValue *MinOrMaxExiting = MinOrMaxResult->getOperand(0);
auto *FinalMinOrMaxCmp =
B.createICmp(CmpInst::ICMP_EQ, MinOrMaxExiting, MinOrMaxResult);
- VPValue *Sentinel = FindIVResult->getOperand(1);
- VPValue *LastIVExiting = FindIVResult->getOperand(2);
+ VPValue *Sentinel = FindIVCmp->getOperand(1);
+ VPValue *LastIVExiting = FindIVRdxResult->getOperand(0);
auto *FinalIVSelect =
B.createSelect(FinalMinOrMaxCmp, LastIVExiting, Sentinel);
- FindIVResult->setOperand(2, FinalIVSelect);
+ FindIVRdxResult->setOperand(0, FinalIVSelect);
}
return true;
}
diff --git a/llvm/lib/Transforms/Vectorize/VPlanPatternMatch.h b/llvm/lib/Transforms/Vectorize/VPlanPatternMatch.h
index 09c3722961246..5d05d0dffc570 100644
--- a/llvm/lib/Transforms/Vectorize/VPlanPatternMatch.h
+++ b/llvm/lib/Transforms/Vectorize/VPlanPatternMatch.h
@@ -479,13 +479,6 @@ m_ComputeAnyOfResult(const Op0_t &Op0, const Op1_t &Op1, const Op2_t &Op2) {
return m_VPInstruction<VPInstruction::ComputeAnyOfResult>(Op0, Op1, Op2);
}
-template <typename Op0_t, typename Op1_t, typename Op2_t>
-inline VPInstruction_match<VPInstruction::ComputeFindIVResult, Op0_t, Op1_t,
- Op2_t>
-m_ComputeFindIVResult(const Op0_t &Op0, const Op1_t &Op1, const Op2_t &Op2) {
- return m_VPInstruction<VPInstruction::ComputeFindIVResult>(Op0, Op1, Op2);
-}
-
template <typename Op0_t>
inline VPInstruction_match<VPInstruction::Reverse, Op0_t>
m_Reverse(const Op0_t &Op0) {
diff --git a/llvm/lib/Transforms/Vectorize/VPlanRecipes.cpp b/llvm/lib/Transforms/Vectorize/VPlanRecipes.cpp
index b10dd17fbfc89..4b6de80cc8f7d 100644
--- a/llvm/lib/Transforms/Vectorize/VPlanRecipes.cpp
+++ b/llvm/lib/Transforms/Vectorize/VPlanRecipes.cpp
@@ -473,7 +473,6 @@ unsigned VPInstruction::getNumOperandsForOpcode(unsigned Opcode) {
case Instruction::Select:
case VPInstruction::ActiveLaneMask:
case VPInstruction::ComputeAnyOfResult:
- case VPInstruction::ComputeFindIVResult:
case VPInstruction::ReductionStartVector:
case VPInstruction::ExtractLastActive:
return 3;
@@ -725,42 +724,12 @@ Value *VPInstruction::generate(VPTransformState &State) {
ReducedResult = Builder.CreateFreeze(ReducedResult);
return Builder.CreateSelect(ReducedResult, NewVal, Start, "rdx.select");
}
- case VPInstruction::ComputeFindIVResult: {
- // The recipe's operands are the start value, the sentinel value, followed
- // by one operand for each part of the reduction.
- unsigned UF = getNumOperands() - 2;
- Value *ReducedResult = State.get(getOperand(2));
- RecurKind MinMaxKind = getRecurKind();
- assert((MinMaxKind == RecurKind::SMin || MinMaxKind == RecurKind::SMax ||
- MinMaxKind == RecurKind::UMin || MinMaxKind == RecurKind::UMax) &&
- "unexpected recurrence kind for ComputeFindIVResult");
- for (unsigned Part = 1; Part < UF; ++Part)
- ReducedResult = createMinMaxOp(Builder, MinMaxKind, ReducedResult,
- State.get(getOperand(2 + Part)));
-
- // Reduce the vector to a scalar.
- bool IsMaxRdx =
- MinMaxKind == RecurKind::SMax || MinMaxKind == RecurKind::UMax;
- bool IsSigned =
- MinMaxKind == RecurKind::SMin || MinMaxKind == RecurKind::SMax;
- if (ReducedResult->getType()->isVectorTy())
- ReducedResult = IsMaxRdx
- ? Builder.CreateIntMaxReduce(ReducedResult, IsSigned)
- : Builder.CreateIntMinReduce(ReducedResult, IsSigned);
- // Correct the final reduction result back to the start value if the
- // reduction result is the sentinel value.
- Value *Start = State.get(getOperand(0), true);
- Value *Sentinel = getOperand(1)->getLiveInIRValue();
- Value *Cmp =
- Builder.CreateICmpNE(ReducedResult, Sentinel, "rdx.select.cmp");
- return Builder.CreateSelect(Cmp, ReducedResult, Start, "rdx.select");
- }
case VPInstruction::ComputeReductionResult: {
RecurKind RK = getRecurKind();
bool IsOrdered = isReductionOrdered();
bool IsInLoop = isReductionInLoop();
assert(!RecurrenceDescriptor::isFindIVRecurrenceKind(RK) &&
- "should be handled by ComputeFindIVResult");
+ "FindIV should use min/max reduction kinds");
// The recipe may have multiple operands to be reduced together.
unsigned NumOperandsToReduce = getNumOperands();
@@ -1280,7 +1249,6 @@ bool VPInstruction::isVectorToScalar() const {
getOpcode() == VPInstruction::FirstActiveLane ||
getOpcode() == VPInstruction::LastActiveLane ||
getOpcode() == VPInstruction::ComputeAnyOfResult ||
- getOpcode() == VPInstruction::ComputeFindIVResult ||
getOpcode() == VPInstruction::ExtractLastActive ||
getOpcode() == VPInstruction::ComputeReductionResult ||
getOpcode() == VPInstruction::AnyOf;
@@ -1409,8 +1377,6 @@ bool VPInstruction::usesFirstLaneOnly(const VPValue *Op) const {
return false;
case VPInstruction::ComputeAnyOfResult:
return Op == getOperand(0) || Op == getOperand(1);
- case VPInstruction::ComputeFindIVResult:
- return Op == getOperand(0);
case VPInstruction::ExtractLane:
return Op == getOperand(0);
};
@@ -1511,9 +1477,6 @@ void VPInstruction::printRecipe(raw_ostream &O, const Twine &Indent,
case VPInstruction::ComputeAnyOfResult:
O << "compute-anyof-result";
break;
- case VPInstruction::ComputeFindIVResult:
- O << "compute-find-iv-result";
- break;
case VPInstruction::ComputeReductionResult:
O << "compute-reduction-result";
break;
@@ -2118,8 +2081,7 @@ bool VPIRFlags::flagsValidForOpcode(unsigned Opcode) const {
case OperationType::Cmp:
return Opcode == Instruction::FCmp || Opcode == Instruction::ICmp;
case OperationType::ReductionOp:
- return Opcode == VPInstruction::ComputeReductionResult ||
- Opcode == VPInstruction::ComputeFindIVResult;
+ return Opcode == VPInstruction::ComputeReductionResult;
case OperationType::Other:
return true;
}
diff --git a/llvm/lib/Transforms/Vectorize/VPlanUnroll.cpp b/llvm/lib/Transforms/Vectorize/VPlanUnroll.cpp
index d87ef7e9927c1..9cba23582d198 100644
--- a/llvm/lib/Transforms/Vectorize/VPlanUnroll.cpp
+++ b/llvm/lib/Transforms/Vectorize/VPlanUnroll.cpp
@@ -370,9 +370,7 @@ void UnrollState::unrollBlock(VPBlockBase *VPB) {
match(&R, m_LastActiveLane(m_VPValue(Op1))) ||
match(&R,
m_ComputeAnyOfResult(m_VPValue(), m_VPValue(), m_VPValue(Op1))) ||
- match(&R, m_ComputeReductionResult(m_VPValue(Op1))) ||
- match(&R, m_ComputeFindIVResult(m_VPValue(), m_VPValue(),
- m_VPValue(Op1)))) {
+ match(&R, m_ComputeReductionResult(m_VPValue(Op1)))) {
addUniformForAllParts(cast<VPInstruction>(&R));
for (unsigned Part = 1; Part != UF; ++Part)
R.addOperand(getValueForPart(Op1, Part));
diff --git a/llvm/lib/Transforms/Vectorize/VPlanUtils.cpp b/llvm/lib/Transforms/Vectorize/VPlanUtils.cpp
index 4873075a771c4..812d974ca0308 100644
--- a/llvm/lib/Transforms/Vectorize/VPlanUtils.cpp
+++ b/llvm/lib/Transforms/Vectorize/VPlanUtils.cpp
@@ -317,6 +317,7 @@ static bool preservesUniformity(unsigned Opcode) {
if (Instruction::isBinaryOp(Opcode) || Instruction::isCast(Opcode))
return true;
switch (Opcode) {
+ case Instruction::Freeze:
case Instruction::GetElementPtr:
case Instruction::ICmp:
case Instruction::FCmp:
diff --git a/llvm/test/Transforms/LoopVectorize/vplan-printing-reductions.ll b/llvm/test/Transforms/LoopVectorize/vplan-printing-reductions.ll
index 088a40ca8fcc3..4d026daca115c 100644
--- a/llvm/test/Transforms/LoopVectorize/vplan-printing-reductions.ll
+++ b/llvm/test/Transforms/LoopVectorize/vplan-printing-reductions.ll
@@ -238,7 +238,9 @@ define i64 @find_last_iv(ptr %a, i64 %n, i64 %start) {
; CHECK-NEXT: Successor(s): middle.block
; CHECK-EMPTY:
; CHECK-NEXT: middle.block:
-; CHECK-NEXT: EMIT vp<[[RDX_RES:%.+]]> = compute-find-iv-result (smax) ir<%start>, ir<-9223372036854775808>, ir<%cond>
+; CHECK-NEXT: EMIT vp<[[RDX_VAL:%.+]]> = compute-reduction-result (smax) ir<%cond>
+; CHECK-NEXT: EMIT vp<[[RDX_CMP:%.+]]> = icmp ne vp<[[RDX_VAL]]>, ir<-9223372036854775808>
+; CHECK-NEXT: EMIT vp<[[RDX_RES:%.+]]> = select vp<[[RDX_CMP]]>, vp<[[RDX_VAL]]>, ir<%start>
; CHECK-NEXT: EMIT vp<%cmp.n> = icmp eq ir<%n>, vp<{{.+}}>
; CHECK-NEXT: EMIT branch-on-cond vp<%cmp.n>
; CHECK-NEXT: Successor(s): ir-bb<exit>, scalar.ph
More information about the llvm-commits
mailing list