[llvm] [LV] Optimize FindLast recurrences to FindIV (NFCI). (PR #177870)
Florian Hahn via llvm-commits
llvm-commits at lists.llvm.org
Mon Feb 2 04:07:53 PST 2026
https://github.com/fhahn updated https://github.com/llvm/llvm-project/pull/177870
>From 2f806e0be10f59ebde7537551d2aa34c0f14f68b Mon Sep 17 00:00:00 2001
From: Florian Hahn <flo at fhahn.com>
Date: Sun, 25 Jan 2026 19:00:12 +0000
Subject: [PATCH 1/2] [LV] Optimize FindLast recurrences to FindIV (NFCI).
This patch restructures Find(First|Last)IV handling. Instead of
differentiating between FindLast, FindFirstIV and FindLastIV up front,
this patch simplifies the logic in IVDescriptor to just identify the
FindLast pattern up-front.
It then adds a new VPlan transformation to optimize FindLast
reductions to FindIV reductions if there is a suitable sentinel
value. Find(Last|First)IV recurrence kinds to a single FindIV kind.
This is simpler and more accurate, given selecting the first/last
induction of the final IV reduction is directly controlled by the
corresponding recurrence kind of the ComputeReductionResult.
The new structure also allows further optimizations, like vectorizing
FindLastIV with another boolean reduction that tracks if the condition
in the loop was ever true, if there is no suitable sentinel value.
---
llvm/include/llvm/Analysis/IVDescriptors.h | 67 ++-----------
llvm/lib/Analysis/IVDescriptors.cpp | 93 +------------------
.../Vectorize/LoopVectorizationPlanner.h | 4 +-
.../Transforms/Vectorize/LoopVectorize.cpp | 32 +------
.../Transforms/Vectorize/SLPVectorizer.cpp | 15 +--
llvm/lib/Transforms/Vectorize/VPlan.h | 2 +-
.../Vectorize/VPlanConstruction.cpp | 14 ++-
.../Transforms/Vectorize/VPlanTransforms.cpp | 93 +++++++++++++++++++
.../Transforms/Vectorize/VPlanTransforms.h | 5 +
9 files changed, 134 insertions(+), 191 deletions(-)
diff --git a/llvm/include/llvm/Analysis/IVDescriptors.h b/llvm/include/llvm/Analysis/IVDescriptors.h
index 05c17632e0e49..377e60c76e560 100644
--- a/llvm/include/llvm/Analysis/IVDescriptors.h
+++ b/llvm/include/llvm/Analysis/IVDescriptors.h
@@ -58,19 +58,11 @@ enum class RecurKind {
FMulAdd, ///< Sum of float products with llvm.fmuladd(a * b + sum).
AnyOf, ///< AnyOf reduction with select(cmp(),x,y) where one of (x,y) is
///< loop invariant, and both x and y are integer type.
- FindFirstIVSMin, /// FindFirst reduction with select(icmp(),x,y) where one of
- ///< (x,y) is a decreasing loop induction, and both x and y
- ///< are integer type, producing a SMin reduction.
- FindFirstIVUMin, /// FindFirst reduction with select(icmp(),x,y) where one of
- ///< (x,y) is a decreasing loop induction, and both x and y
- ///< are integer type, producing a UMin reduction.
- FindLastIVSMax, ///< FindLast reduction with select(cmp(),x,y) where one of
- ///< (x,y) is increasing loop induction, and both x and y
- ///< are integer type, producing a SMax reduction.
- FindLastIVUMax, ///< FindLast reduction with select(cmp(),x,y) where one of
- ///< (x,y) is increasing loop induction, and both x and y
- ///< are integer type, producing a UMax reduction.
- FindLast, ///< FindLast reduction with select(cmp(),x,y) where x and y
+ FindIV, ///< FindIV reduction with select(icmp(),x,y) where one of (x,y) is
+ ///< a loop induction variable (increasing or decreasing), and both
+ ///< x and y are integer type. The signedness and direction are
+ ///< stored separately.
+ FindLast, ///< FindLast reduction with select(cmp(),x,y) where x and y
///< are an integer type, one is the current recurrence value,
///< and the other is an arbitrary value.
// clang-format on
@@ -283,32 +275,9 @@ class RecurrenceDescriptor {
}
/// Returns true if the recurrence kind is of the form
- /// select(cmp(),x,y) where one of (x,y) is decreasing loop induction.
- static bool isFindFirstIVRecurrenceKind(RecurKind Kind) {
- return Kind == RecurKind::FindFirstIVSMin ||
- Kind == RecurKind::FindFirstIVUMin;
- }
-
- /// Returns true if the recurrence kind is of the form
- /// select(cmp(),x,y) where one of (x,y) is increasing loop induction.
- static bool isFindLastIVRecurrenceKind(RecurKind Kind) {
- return Kind == RecurKind::FindLastIVSMax ||
- Kind == RecurKind::FindLastIVUMax;
- }
-
- /// Returns true if recurrece kind is a signed redux kind.
- static bool isSignedRecurrenceKind(RecurKind Kind) {
- return Kind == RecurKind::SMax || Kind == RecurKind::SMin ||
- Kind == RecurKind::FindFirstIVSMin ||
- Kind == RecurKind::FindLastIVSMax;
- }
-
- /// Returns true if the recurrence kind is of the form
- /// select(cmp(),x,y) where one of (x,y) is an increasing or decreasing loop
- /// induction.
+ /// select(cmp(),x,y) where one of (x,y) is a loop induction variable.
static bool isFindIVRecurrenceKind(RecurKind Kind) {
- return isFindFirstIVRecurrenceKind(Kind) ||
- isFindLastIVRecurrenceKind(Kind);
+ return Kind == RecurKind::FindIV;
}
/// Returns true if the recurrence kind is of the form
@@ -326,21 +295,6 @@ class RecurrenceDescriptor {
/// actual type of the Phi if the recurrence has been type-promoted.
Type *getRecurrenceType() const { return RecurrenceType; }
- /// Returns the sentinel value for FindFirstIV & FindLastIV recurrences to
- /// replace the start value.
- Value *getSentinelValue() const {
- Type *Ty = StartValue->getType();
- unsigned BW = Ty->getIntegerBitWidth();
- if (isFindLastIVRecurrenceKind(Kind)) {
- return ConstantInt::get(Ty, isSignedRecurrenceKind(Kind)
- ? APInt::getSignedMinValue(BW)
- : APInt::getMinValue(BW));
- }
- return ConstantInt::get(Ty, isSignedRecurrenceKind(Kind)
- ? APInt::getSignedMaxValue(BW)
- : APInt::getMaxValue(BW));
- }
-
/// Returns a reference to the instructions used for type-promoting the
/// recurrence.
const SmallPtrSet<Instruction *, 8> &getCastInsts() const { return CastInsts; }
@@ -357,8 +311,8 @@ class RecurrenceDescriptor {
bool isOrdered() const { return IsOrdered; }
/// Returns true if the reduction PHI has any uses outside the reduction
- /// chain. This is relevant for min/max reductions that are part of a
- /// FindLastIV pattern.
+ /// chain. This is relevant for min/max reductions that are part of a FindIV
+ /// pattern.
bool hasUsesOutsideReductionChain() const {
return PhiHasUsesOutsideReductionChain;
}
@@ -401,8 +355,7 @@ class RecurrenceDescriptor {
// if it is also the only FAdd in the PHI's use chain.
bool IsOrdered = false;
// True if the reduction PHI has in-loop users outside the reduction chain.
- // This is relevant for min/max reductions that are part of a FindLastIV
- // pattern.
+ // This is relevant for min/max reductions that are part of a FindIV pattern.
bool PhiHasUsesOutsideReductionChain = false;
// Instructions used for type-promoting the recurrence.
SmallPtrSet<Instruction *, 8> CastInsts;
diff --git a/llvm/lib/Analysis/IVDescriptors.cpp b/llvm/lib/Analysis/IVDescriptors.cpp
index 3946a7ce8b8f6..f5c1730e50fab 100644
--- a/llvm/lib/Analysis/IVDescriptors.cpp
+++ b/llvm/lib/Analysis/IVDescriptors.cpp
@@ -54,10 +54,7 @@ bool RecurrenceDescriptor::isIntegerRecurrenceKind(RecurKind Kind) {
case RecurKind::UMax:
case RecurKind::UMin:
case RecurKind::AnyOf:
- case RecurKind::FindFirstIVSMin:
- case RecurKind::FindFirstIVUMin:
- case RecurKind::FindLastIVSMax:
- case RecurKind::FindLastIVUMax:
+ case RecurKind::FindIV:
// TODO: Make type-agnostic.
case RecurKind::FindLast:
return true;
@@ -740,19 +737,6 @@ RecurrenceDescriptor::isAnyOfPattern(Loop *Loop, PHINode *OrigPhi,
// %spec.select = select i1 %cmp, i32 %i, i32 %r
// %inc = add nsw i32 %i, 1
// ...
-// When searching for an induction variable (i), the reduction value after the
-// loop will be the maximum (increasing induction) or minimum (decreasing
-// induction) value of 'i' that the condition (src[i] > 3) is satisfied, or the
-// start value (0 in the example above). When the start value of the induction
-// variable 'i' is greater than the minimum (increasing induction) or maximum
-// (decreasing induction) value of the data type, we can use the minimum
-// (increasing induction) or maximum (decreasing induction) value of the data
-// type as a sentinel value to replace the start value. This allows us to
-// perform a single reduction max (increasing induction) or min (decreasing
-// induction) operation to obtain the final reduction result.
-// TODO: It is possible to solve the case where the start value is the minimum
-// value of the data type or a non-constant value by using mask and multiple
-// reduction operations.
//
// When searching for an arbitrary loop-varying value, the reduction value will
// either be the initial value (0) if the condition was never met, or the value
@@ -771,8 +755,6 @@ RecurrenceDescriptor::isFindPattern(Loop *TheLoop, PHINode *OrigPhi,
// We are looking for selects of the form:
// select(cmp(), phi, value) or
// select(cmp(), value, phi)
- // where 'value' must be a loop induction variable
- // (for FindFirstIV/FindLastIV) or an arbitrary value (for FindLast).
// TODO: Match selects with multi-use cmp conditions.
Value *NonRdxPhi = nullptr;
if (!match(I, m_CombineOr(m_Select(m_OneUse(m_Cmp()), m_Value(NonRdxPhi),
@@ -781,74 +763,6 @@ RecurrenceDescriptor::isFindPattern(Loop *TheLoop, PHINode *OrigPhi,
m_Value(NonRdxPhi)))))
return InstDesc(false, I);
- // Returns either FindFirstIV/FindLastIV, if such a pattern is found, or
- // std::nullopt.
- auto GetFindFirstLastIVRecurKind = [&](Value *V) -> std::optional<RecurKind> {
- Type *Ty = V->getType();
- if (!SE.isSCEVable(Ty))
- return std::nullopt;
-
- auto *AR = SE.getSCEV(V);
- const SCEV *Step;
- if (!match(AR, m_scev_AffineAddRec(m_SCEV(), m_SCEV(Step),
- m_SpecificLoop(TheLoop))))
- return std::nullopt;
-
- // We must have a known positive or negative step for FindIV
- const bool PositiveStep = SE.isKnownPositive(Step);
- if (!SE.isKnownNonZero(Step))
- return std::nullopt;
-
- // Check if the minimum (FindLast) or maximum (FindFirst) value of the
- // recurrence type can be used as a sentinel value. The maximum acceptable
- // range for the induction variable, called the valid range will exclude
- // <sentinel value>, where <sentinel value> is
- // [Signed|Unsigned]Min(<recurrence type>) for FindLastIV or
- // [Signed|Unsigned]Max(<recurrence type>) for FindFirstIV.
- // TODO: This range restriction can be lifted by adding an additional
- // virtual OR reduction.
- auto CheckRange = [&](bool IsSigned) {
- const ConstantRange IVRange =
- IsSigned ? SE.getSignedRange(AR) : SE.getUnsignedRange(AR);
- unsigned NumBits = Ty->getIntegerBitWidth();
- APInt Sentinel;
- if (PositiveStep) {
- Sentinel = IsSigned ? APInt::getSignedMinValue(NumBits)
- : APInt::getMinValue(NumBits);
- } else {
- Sentinel = IsSigned ? APInt::getSignedMaxValue(NumBits)
- : APInt::getMaxValue(NumBits);
- }
- ConstantRange ValidRange = ConstantRange(Sentinel).inverse();
-
- LLVM_DEBUG(
- dbgs() << "LV: " << (PositiveStep ? "FindLastIV" : "FindFirstIV")
- << " valid range is " << ValidRange << ", and the range of "
- << *AR << " is " << IVRange << "\n");
-
- // Ensure the induction variable does not wrap around by verifying that
- // its range is fully contained within the valid range.
- return ValidRange.contains(IVRange);
- };
- if (PositiveStep) {
- if (CheckRange(true))
- return RecurKind::FindLastIVSMax;
- if (CheckRange(false))
- return RecurKind::FindLastIVUMax;
- return std::nullopt;
- }
-
- if (CheckRange(true))
- return RecurKind::FindFirstIVSMin;
- if (CheckRange(false))
- return RecurKind::FindFirstIVUMin;
- return std::nullopt;
- };
-
- if (auto RK = GetFindFirstLastIVRecurKind(NonRdxPhi))
- return InstDesc(I, *RK);
-
- // If the recurrence is not specific to an IV, return a generic FindLast.
return InstDesc(I, RecurKind::FindLast);
}
@@ -1302,10 +1216,7 @@ unsigned RecurrenceDescriptor::getOpcode(RecurKind Kind) {
return Instruction::FCmp;
case RecurKind::FindLast:
case RecurKind::AnyOf:
- case RecurKind::FindFirstIVSMin:
- case RecurKind::FindFirstIVUMin:
- case RecurKind::FindLastIVSMax:
- case RecurKind::FindLastIVUMax:
+ case RecurKind::FindIV:
// TODO: Set AnyOf and FindIV to Instruction::Select once in-loop reductions
// are supported.
default:
diff --git a/llvm/lib/Transforms/Vectorize/LoopVectorizationPlanner.h b/llvm/lib/Transforms/Vectorize/LoopVectorizationPlanner.h
index 44d4d92d4a7e2..0091f025afe2d 100644
--- a/llvm/lib/Transforms/Vectorize/LoopVectorizationPlanner.h
+++ b/llvm/lib/Transforms/Vectorize/LoopVectorizationPlanner.h
@@ -631,8 +631,8 @@ class LoopVectorizationPlanner {
/// legal to vectorize the loop. This method creates VPlans using VPRecipes.
void buildVPlansWithVPRecipes(ElementCount MinVF, ElementCount MaxVF);
- /// Add recipes to compute the final reduction result (ComputeFindIVResult,
- /// ComputeAnyOfResult, ComputeReductionResult depending on the reduction) in
+ /// Add recipes to compute the final reduction result (ComputeAnyOfResult,
+ /// ComputeReductionResult depending on the reduction) in
/// the middle block. Selects are introduced for reductions between the phi
/// and users outside the vector region when folding the tail.
void addReductionResultComputation(VPlanPtr &Plan,
diff --git a/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp b/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
index 68e61c01c65d9..879d562d9a5d3 100644
--- a/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
+++ b/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
@@ -8560,6 +8560,10 @@ VPlanPtr LoopVectorizationPlanner::tryToBuildVPlanWithVPRecipes(
addReductionResultComputation(Plan, RecipeBuilder, Range.Start);
+ // Optimize FindIV reductions to use sentinel-based approach when possible.
+ VPlanTransforms::runPass(VPlanTransforms::optimizeFindIVReductions, *Plan,
+ PSE, *OrigLoop);
+
// Apply mandatory transformation to handle reductions with multiple in-loop
// uses if possible, bail out otherwise.
if (!VPlanTransforms::runPass(VPlanTransforms::handleMultiUseReductions,
@@ -8743,26 +8747,7 @@ void LoopVectorizationPlanner::addReductionResultComputation(
return match(U, m_Select(m_VPValue(), m_VPValue(), m_VPValue()));
}));
}
- if (RecurrenceDescriptor::isFindIVRecurrenceKind(RecurrenceKind)) {
- VPValue *Start = PhiR->getStartValue();
- VPValue *Sentinel = Plan->getOrAddLiveIn(RdxDesc.getSentinelValue());
- RecurKind MinMaxKind;
- bool IsSigned =
- RecurrenceDescriptor::isSignedRecurrenceKind(RecurrenceKind);
- if (RecurrenceDescriptor::isFindLastIVRecurrenceKind(RecurrenceKind))
- MinMaxKind = IsSigned ? RecurKind::SMax : RecurKind::UMax;
- else
- MinMaxKind = IsSigned ? RecurKind::SMin : RecurKind::UMin;
- VPIRFlags Flags(MinMaxKind, /*IsOrdered=*/false, /*IsInLoop=*/false,
- FastMathFlags());
- auto *ReducedIV =
- Builder.createNaryOp(VPInstruction::ComputeReductionResult,
- {NewExitingVPV}, Flags, ExitDL);
- auto *Cmp =
- Builder.createICmp(CmpInst::ICMP_NE, ReducedIV, Sentinel, ExitDL);
- FinalReductionResult = cast<VPInstruction>(
- Builder.createSelect(Cmp, ReducedIV, Start, ExitDL));
- } else if (AnyOfSelect) {
+ if (AnyOfSelect) {
VPValue *Start = PhiR->getStartValue();
// NewVal is the non-phi operand of the select.
VPValue *NewVal = AnyOfSelect->getOperand(1) == PhiR
@@ -8863,13 +8848,6 @@ void LoopVectorizationPlanner::addReductionResultComputation(
continue;
}
- if (RecurrenceDescriptor::isFindIVRecurrenceKind(
- RdxDesc.getRecurrenceKind())) {
- // Adjust the start value for FindFirstIV/FindLastIV recurrences to use
- // the sentinel value after generating the ResumePhi recipe, which uses
- // the original start value.
- PhiR->setOperand(0, Plan->getOrAddLiveIn(RdxDesc.getSentinelValue()));
- }
RecurKind RK = RdxDesc.getRecurrenceKind();
if ((!RecurrenceDescriptor::isAnyOfRecurrenceKind(RK) &&
!RecurrenceDescriptor::isFindIVRecurrenceKind(RK) &&
diff --git a/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp b/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp
index cede1081e0260..7e432817feba7 100644
--- a/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp
+++ b/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp
@@ -26130,10 +26130,7 @@ class HorizontalReduction {
case RecurKind::FMul:
case RecurKind::FMulAdd:
case RecurKind::AnyOf:
- case RecurKind::FindFirstIVSMin:
- case RecurKind::FindFirstIVUMin:
- case RecurKind::FindLastIVSMax:
- case RecurKind::FindLastIVUMax:
+ case RecurKind::FindIV:
case RecurKind::FindLast:
case RecurKind::FMaxNum:
case RecurKind::FMinNum:
@@ -26272,10 +26269,7 @@ class HorizontalReduction {
case RecurKind::FMul:
case RecurKind::FMulAdd:
case RecurKind::AnyOf:
- case RecurKind::FindFirstIVSMin:
- case RecurKind::FindFirstIVUMin:
- case RecurKind::FindLastIVSMax:
- case RecurKind::FindLastIVUMax:
+ case RecurKind::FindIV:
case RecurKind::FindLast:
case RecurKind::FMaxNum:
case RecurKind::FMinNum:
@@ -26379,10 +26373,7 @@ class HorizontalReduction {
case RecurKind::FMul:
case RecurKind::FMulAdd:
case RecurKind::AnyOf:
- case RecurKind::FindFirstIVSMin:
- case RecurKind::FindFirstIVUMin:
- case RecurKind::FindLastIVSMax:
- case RecurKind::FindLastIVUMax:
+ case RecurKind::FindIV:
case RecurKind::FindLast:
case RecurKind::FMaxNum:
case RecurKind::FMinNum:
diff --git a/llvm/lib/Transforms/Vectorize/VPlan.h b/llvm/lib/Transforms/Vectorize/VPlan.h
index f650971c6b397..b7543ded70252 100644
--- a/llvm/lib/Transforms/Vectorize/VPlan.h
+++ b/llvm/lib/Transforms/Vectorize/VPlan.h
@@ -2527,7 +2527,7 @@ class VPReductionPHIRecipe : public VPHeaderPHIRecipe,
ReductionStyle Style;
- /// The phi is part of a multi-use reduction (e.g., used in FindLastIV
+ /// The phi is part of a multi-use reduction (e.g., used in FindIV
/// patterns for argmin/argmax).
/// TODO: Also support cases where the phi itself has a single use, but its
/// compare has multiple uses.
diff --git a/llvm/lib/Transforms/Vectorize/VPlanConstruction.cpp b/llvm/lib/Transforms/Vectorize/VPlanConstruction.cpp
index 4fc7b2fe1c2de..7197f78ea930e 100644
--- a/llvm/lib/Transforms/Vectorize/VPlanConstruction.cpp
+++ b/llvm/lib/Transforms/Vectorize/VPlanConstruction.cpp
@@ -1482,10 +1482,22 @@ bool VPlanTransforms::handleMultiUseReductions(VPlan &Plan) {
}
auto *FindIVPhiR = dyn_cast<VPReductionPHIRecipe>(FindIV);
- if (!FindIVPhiR || !RecurrenceDescriptor::isFindLastIVRecurrenceKind(
+ if (!FindIVPhiR || !RecurrenceDescriptor::isFindIVRecurrenceKind(
FindIVPhiR->getRecurrenceKind()))
return false;
+ // Check if FindIVPhiR is a FindLast pattern by checking the MinMaxKind
+ // on its ComputeReductionResult. SMax/UMax indicates FindLast.
+ VPInstruction *FindIVResult =
+ findUserOf<VPInstruction::ComputeReductionResult>(
+ FindIVPhiR->getBackedgeValue());
+ if (!FindIVResult)
+ return false;
+ RecurKind FindIVMinMaxKind = FindIVResult->getRecurKind();
+ if (FindIVMinMaxKind != RecurKind::SMax &&
+ FindIVMinMaxKind != RecurKind::UMax)
+ return false;
+
// TODO: Support cases where IVOp is the IV increment.
if (!match(IVOp, m_TruncOrSelf(m_VPValue(IVOp))) ||
!isa<VPWidenIntOrFpInductionRecipe>(IVOp))
diff --git a/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp b/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp
index d1d5870d78a03..2497defc1b9bc 100644
--- a/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp
+++ b/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp
@@ -5349,3 +5349,96 @@ void VPlanTransforms::addExitUsersForFirstOrderRecurrences(VPlan &Plan,
}
}
}
+
+void VPlanTransforms::optimizeFindIVReductions(VPlan &Plan,
+ PredicatedScalarEvolution &PSE,
+ Loop &L) {
+ ScalarEvolution &SE = *PSE.getSE();
+ VPRegionBlock *VectorLoopRegion = Plan.getVectorLoopRegion();
+
+ // Helper lambda to check if the IV range excludes the sentinel value.
+ auto CheckSentinel = [&SE](const SCEV *IVSCEV, bool IsFindLast,
+ bool Signed) -> std::optional<APInt> {
+ unsigned BW = IVSCEV->getType()->getScalarSizeInBits();
+ APInt Sentinel =
+ IsFindLast
+ ? (Signed ? APInt::getSignedMinValue(BW) : APInt::getMinValue(BW))
+ : (Signed ? APInt::getSignedMaxValue(BW) : APInt::getMaxValue(BW));
+
+ ConstantRange IVRange =
+ Signed ? SE.getSignedRange(IVSCEV) : SE.getUnsignedRange(IVSCEV);
+ if (!IVRange.contains(Sentinel))
+ return Sentinel;
+ return std::nullopt;
+ };
+
+ for (VPRecipeBase &Phi :
+ make_early_inc_range(VectorLoopRegion->getEntryBasicBlock()->phis())) {
+ auto *PhiR = dyn_cast<VPReductionPHIRecipe>(&Phi);
+ if (!PhiR || !RecurrenceDescriptor::isFindLastRecurrenceKind(
+ PhiR->getRecurrenceKind()))
+ continue;
+
+ // Get the IV from the backedge value of the reduction phi.
+ // The backedge value should be a select between the phi and the IV.
+ VPValue *BackedgeVal = PhiR->getBackedgeValue();
+ VPValue *TrueVal, *FalseVal;
+ if (!match(BackedgeVal,
+ m_Select(m_VPValue(), m_VPValue(TrueVal), m_VPValue(FalseVal))))
+ continue;
+
+ // The non-phi operand of the select is the IV.
+ VPValue *IV = (TrueVal == PhiR) ? FalseVal : TrueVal;
+
+ const SCEV *IVSCEV = vputils::getSCEVExprForVPValue(IV, PSE, &L);
+ const SCEV *Step;
+ if (!match(IVSCEV, m_scev_AffineAddRec(m_SCEV(), m_SCEV(Step))))
+ continue;
+
+ // Determine direction from SCEV step.
+ // Positive step means FindLast (ascending), negative means FindFirst.
+ if (!SE.isKnownNonZero(Step))
+ continue;
+
+ bool IsFindLast = SE.isKnownPositive(Step);
+ bool UseSigned = true;
+ std::optional<APInt> SentinelVal = CheckSentinel(IVSCEV, IsFindLast, true);
+ if (!SentinelVal) {
+ SentinelVal = CheckSentinel(IVSCEV, IsFindLast, false);
+ if (!SentinelVal)
+ continue;
+ UseSigned = false;
+ }
+
+ VPInstruction *RdxResult = cast<VPInstruction>(vputils::findRecipe(
+ BackedgeVal,
+ match_fn(m_VPInstruction<VPInstruction::ComputeReductionResult>())));
+
+ // Create the reduction result in the middle block using sentinel directly.
+ RecurKind MinMaxKind =
+ IsFindLast ? (UseSigned ? RecurKind::SMax : RecurKind::UMax)
+ : (UseSigned ? RecurKind::SMin : RecurKind::UMin);
+ VPIRFlags Flags(MinMaxKind, /*IsOrdered=*/false, /*IsInLoop=*/false,
+ FastMathFlags());
+ VPValue *Sentinel = Plan.getConstantInt(*SentinelVal);
+ DebugLoc ExitDL = RdxResult->getDebugLoc();
+ VPBuilder MiddleBuilder(RdxResult);
+ VPValue *ReducedIV =
+ MiddleBuilder.createNaryOp(VPInstruction::ComputeReductionResult,
+ RdxResult->getOperand(0), Flags, ExitDL);
+ auto *Cmp =
+ MiddleBuilder.createICmp(CmpInst::ICMP_NE, ReducedIV, Sentinel, ExitDL);
+ VPInstruction *NewRdxResult =
+ cast<VPInstruction>(MiddleBuilder.createSelect(
+ Cmp, ReducedIV, PhiR->getStartValue(), ExitDL));
+ RdxResult->replaceAllUsesWith(NewRdxResult);
+ RdxResult->eraseFromParent();
+
+ auto *NewPhiR = new VPReductionPHIRecipe(
+ cast<PHINode>(PhiR->getUnderlyingInstr()), RecurKind::FindIV, *Sentinel,
+ *BackedgeVal, RdxUnordered{1}, PhiR->hasUsesOutsideReductionChain());
+ NewPhiR->insertBefore(PhiR);
+ PhiR->replaceAllUsesWith(NewPhiR);
+ PhiR->eraseFromParent();
+ }
+}
diff --git a/llvm/lib/Transforms/Vectorize/VPlanTransforms.h b/llvm/lib/Transforms/Vectorize/VPlanTransforms.h
index e0d09a099647a..e1de34dc17a47 100644
--- a/llvm/lib/Transforms/Vectorize/VPlanTransforms.h
+++ b/llvm/lib/Transforms/Vectorize/VPlanTransforms.h
@@ -440,6 +440,11 @@ struct VPlanTransforms {
/// users in the original exit block using the VPIRInstruction wrapping to the
/// LCSSA phi.
static void addExitUsersForFirstOrderRecurrences(VPlan &Plan, VFRange &Range);
+
+ /// Optimize FindLast reductions selecting IVs by converting them to FindIV
+ /// reductions, if their IV range excludes a suitable sentinel value.
+ static void optimizeFindIVReductions(VPlan &Plan,
+ PredicatedScalarEvolution &PSE, Loop &L);
};
} // namespace llvm
>From b5e809c0e2f61d1c82e6304ecca951d60ef9140d Mon Sep 17 00:00:00 2001
From: Florian Hahn <flo at fhahn.com>
Date: Mon, 2 Feb 2026 11:49:51 +0000
Subject: [PATCH 2/2] !fixup address comments, fix after merge
---
.../Transforms/Vectorize/LoopVectorize.cpp | 4 +--
.../Vectorize/VPlanConstruction.cpp | 6 ++--
.../Transforms/Vectorize/VPlanTransforms.cpp | 29 ++++++++++---------
.../LoopVectorize/vplan-print-after-all.ll | 3 +-
4 files changed, 22 insertions(+), 20 deletions(-)
diff --git a/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp b/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
index b170a6b43a09c..51ba55e8c4d2b 100644
--- a/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
+++ b/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
@@ -8332,8 +8332,8 @@ VPlanPtr LoopVectorizationPlanner::tryToBuildVPlanWithVPRecipes(
addReductionResultComputation(Plan, RecipeBuilder, Range.Start);
// Optimize FindIV reductions to use sentinel-based approach when possible.
- VPlanTransforms::runPass(VPlanTransforms::optimizeFindIVReductions, *Plan,
- PSE, *OrigLoop);
+ RUN_VPLAN_PASS(VPlanTransforms::optimizeFindIVReductions, *Plan, PSE,
+ *OrigLoop);
// Apply mandatory transformation to handle reductions with multiple in-loop
// uses if possible, bail out otherwise.
diff --git a/llvm/lib/Transforms/Vectorize/VPlanConstruction.cpp b/llvm/lib/Transforms/Vectorize/VPlanConstruction.cpp
index 407b74b2acdef..3695de3a47d12 100644
--- a/llvm/lib/Transforms/Vectorize/VPlanConstruction.cpp
+++ b/llvm/lib/Transforms/Vectorize/VPlanConstruction.cpp
@@ -1476,10 +1476,10 @@ bool VPlanTransforms::handleMultiUseReductions(VPlan &Plan) {
// Check if FindIVPhiR is a FindLast pattern by checking the MinMaxKind
// on its ComputeReductionResult. SMax/UMax indicates FindLast.
VPInstruction *FindIVResult =
- findUserOf<VPInstruction::ComputeReductionResult>(
+ vputils::findUserOf<VPInstruction::ComputeReductionResult>(
FindIVPhiR->getBackedgeValue());
- if (!FindIVResult)
- return false;
+ assert(FindIVResult &&
+ "must be able to retrieve the FindIVResult VPInstruction");
RecurKind FindIVMinMaxKind = FindIVResult->getRecurKind();
if (FindIVMinMaxKind != RecurKind::SMax &&
FindIVMinMaxKind != RecurKind::UMax)
diff --git a/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp b/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp
index 72a59304ee53b..d787efd785467 100644
--- a/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp
+++ b/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp
@@ -5408,7 +5408,6 @@ void VPlanTransforms::addExitUsersForFirstOrderRecurrences(VPlan &Plan,
}
}
}
-<<<<<<< HEAD
void VPlanTransforms::optimizeFindIVReductions(VPlan &Plan,
PredicatedScalarEvolution &PSE,
@@ -5417,11 +5416,11 @@ void VPlanTransforms::optimizeFindIVReductions(VPlan &Plan,
VPRegionBlock *VectorLoopRegion = Plan.getVectorLoopRegion();
// Helper lambda to check if the IV range excludes the sentinel value.
- auto CheckSentinel = [&SE](const SCEV *IVSCEV, bool IsFindLast,
+ auto CheckSentinel = [&SE](const SCEV *IVSCEV, bool UseMax,
bool Signed) -> std::optional<APInt> {
unsigned BW = IVSCEV->getType()->getScalarSizeInBits();
APInt Sentinel =
- IsFindLast
+ UseMax
? (Signed ? APInt::getSignedMinValue(BW) : APInt::getMinValue(BW))
: (Signed ? APInt::getSignedMaxValue(BW) : APInt::getMaxValue(BW));
@@ -5448,7 +5447,8 @@ void VPlanTransforms::optimizeFindIVReductions(VPlan &Plan,
continue;
// The non-phi operand of the select is the IV.
- VPValue *IV = (TrueVal == PhiR) ? FalseVal : TrueVal;
+ assert(is_contained(BackedgeVal->getDefiningRecipe()->operands(), PhiR));
+ VPValue *IV = TrueVal == PhiR ? FalseVal : TrueVal;
const SCEV *IVSCEV = vputils::getSCEVExprForVPValue(IV, PSE, &L);
const SCEV *Step;
@@ -5456,28 +5456,30 @@ void VPlanTransforms::optimizeFindIVReductions(VPlan &Plan,
continue;
// Determine direction from SCEV step.
- // Positive step means FindLast (ascending), negative means FindFirst.
if (!SE.isKnownNonZero(Step))
continue;
- bool IsFindLast = SE.isKnownPositive(Step);
+ // Positive step means we need UMax/SMax to find the last IV value, and
+ // UMin/SMin otherwise.
+ bool UseMax = SE.isKnownPositive(Step);
bool UseSigned = true;
- std::optional<APInt> SentinelVal = CheckSentinel(IVSCEV, IsFindLast, true);
+ std::optional<APInt> SentinelVal =
+ CheckSentinel(IVSCEV, UseMax, /*IsSigned=*/true);
if (!SentinelVal) {
- SentinelVal = CheckSentinel(IVSCEV, IsFindLast, false);
+ SentinelVal = CheckSentinel(IVSCEV, UseMax, /*IsSigned=*/false);
if (!SentinelVal)
continue;
UseSigned = false;
}
- VPInstruction *RdxResult = cast<VPInstruction>(vputils::findRecipe(
+ auto *RdxResult = cast<VPInstruction>(vputils::findRecipe(
BackedgeVal,
match_fn(m_VPInstruction<VPInstruction::ComputeReductionResult>())));
// Create the reduction result in the middle block using sentinel directly.
RecurKind MinMaxKind =
- IsFindLast ? (UseSigned ? RecurKind::SMax : RecurKind::UMax)
- : (UseSigned ? RecurKind::SMin : RecurKind::UMin);
+ UseMax ? (UseSigned ? RecurKind::SMax : RecurKind::UMax)
+ : (UseSigned ? RecurKind::SMin : RecurKind::UMin);
VPIRFlags Flags(MinMaxKind, /*IsOrdered=*/false, /*IsInLoop=*/false,
FastMathFlags());
VPValue *Sentinel = Plan.getConstantInt(*SentinelVal);
@@ -5488,9 +5490,8 @@ void VPlanTransforms::optimizeFindIVReductions(VPlan &Plan,
RdxResult->getOperand(0), Flags, ExitDL);
auto *Cmp =
MiddleBuilder.createICmp(CmpInst::ICMP_NE, ReducedIV, Sentinel, ExitDL);
- VPInstruction *NewRdxResult =
- cast<VPInstruction>(MiddleBuilder.createSelect(
- Cmp, ReducedIV, PhiR->getStartValue(), ExitDL));
+ VPInstruction *NewRdxResult = MiddleBuilder.createSelect(
+ Cmp, ReducedIV, PhiR->getStartValue(), ExitDL);
RdxResult->replaceAllUsesWith(NewRdxResult);
RdxResult->eraseFromParent();
diff --git a/llvm/test/Transforms/LoopVectorize/vplan-print-after-all.ll b/llvm/test/Transforms/LoopVectorize/vplan-print-after-all.ll
index 9f0a528bfa889..5b68887f0f7da 100644
--- a/llvm/test/Transforms/LoopVectorize/vplan-print-after-all.ll
+++ b/llvm/test/Transforms/LoopVectorize/vplan-print-after-all.ll
@@ -6,6 +6,7 @@
; CHECK: VPlan after printAfterInitialConstruction
; CHECK: VPlan after VPlanTransforms::clearReductionWrapFlags
+; CHECK: VPlan after VPlanTransforms::optimizeFindIVReductions
; CHECK: VPlan after VPlanTransforms::handleMultiUseReductions
; CHECK: VPlan after VPlanTransforms::handleMaxMinNumReductions
; CHECK: VPlan after VPlanTransforms::handleFindLastReductions
@@ -45,7 +46,7 @@
; CHECK-DUMP: VPlan after printAfterInitialConstruction
; CHECK-DUMP-NEXT: VPlan ' for UF>=1' {
;
-; CHECK-DUMP: VPlan after VPlanTransforms::optimize
+; CHECK-DUMP: VPlan after VPlanTransforms::optimize{{$}}
; CHECK-DUMP-NEXT: VPlan 'Initial VPlan for VF={4},UF>=1' {
;
; CHECK-DUMP: VPlan after printFinalVPlan
More information about the llvm-commits
mailing list