[llvm] [VPlan] Manage Sentinel value for FindLastIV in VPlan. (PR #142291)
Florian Hahn via llvm-commits
llvm-commits at lists.llvm.org
Sun Jun 8 04:59:29 PDT 2025
https://github.com/fhahn updated https://github.com/llvm/llvm-project/pull/142291
>From fb8a70b45c2e25a384be38c460678b0834cac499 Mon Sep 17 00:00:00 2001
From: Florian Hahn <flo at fhahn.com>
Date: Thu, 29 May 2025 20:41:57 +0100
Subject: [PATCH] [VPlan] Manage Sentinel value for FindLastIV in VPlan.
Similar to modeling the start value as operand, also model the sentinel
value as operand explicitly. This makes all require information for
code-gen available directly in VPlan.
---
.../Transforms/Vectorize/LoopVectorize.cpp | 20 +++++++++++--------
.../Transforms/Vectorize/VPlanPatternMatch.h | 19 ++++++++++++++++++
.../lib/Transforms/Vectorize/VPlanRecipes.cpp | 8 ++++----
llvm/lib/Transforms/Vectorize/VPlanUnroll.cpp | 2 +-
.../vplan-printing-reductions.ll | 2 +-
5 files changed, 37 insertions(+), 14 deletions(-)
diff --git a/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp b/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
index fc8ebebcf21b7..2adf6efacf25b 100644
--- a/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
+++ b/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
@@ -7254,9 +7254,11 @@ static void fixReductionScalarResumeWhenVectorizingEpilog(
using namespace llvm::PatternMatch;
Value *Cmp, *OrigResumeV, *CmpOp;
bool IsExpectedPattern =
- match(MainResumeValue, m_Select(m_OneUse(m_Value(Cmp)),
- m_Specific(RdxDesc.getSentinelValue()),
- m_Value(OrigResumeV))) &&
+ match(MainResumeValue,
+ m_Select(
+ m_OneUse(m_Value(Cmp)),
+ m_Specific(EpiRedResult->getOperand(2)->getLiveInIRValue()),
+ m_Value(OrigResumeV))) &&
(match(Cmp, m_SpecificICmp(ICmpInst::ICMP_EQ, m_Specific(OrigResumeV),
m_Value(CmpOp))) &&
((CmpOp == StartV && isGuaranteedNotToBeUndefOrPoison(CmpOp))));
@@ -9245,9 +9247,11 @@ void LoopVectorizationPlanner::adjustRecipesForReductions(
if (RecurrenceDescriptor::isFindLastIVRecurrenceKind(
RdxDesc.getRecurrenceKind())) {
VPValue *Start = PhiR->getStartValue();
- FinalReductionResult =
- Builder.createNaryOp(VPInstruction::ComputeFindLastIVResult,
- {PhiR, Start, NewExitingVPV}, ExitDL);
+ FinalReductionResult = Builder.createNaryOp(
+ VPInstruction::ComputeFindLastIVResult,
+ {PhiR, Start, Plan->getOrAddLiveIn(RdxDesc.getSentinelValue()),
+ NewExitingVPV},
+ ExitDL);
} else if (RecurrenceDescriptor::isAnyOfRecurrenceKind(
RdxDesc.getRecurrenceKind())) {
VPValue *Start = PhiR->getStartValue();
@@ -9823,8 +9827,8 @@ preparePlanForEpilogueVectorLoop(VPlan &Plan, Loop *L,
BasicBlock *ResumeBB = cast<Instruction>(ResumeV)->getParent();
IRBuilder<> Builder(ResumeBB, ResumeBB->getFirstNonPHIIt());
Value *Cmp = Builder.CreateICmpEQ(ResumeV, ToFrozen[StartV]);
- ResumeV =
- Builder.CreateSelect(Cmp, RdxDesc.getSentinelValue(), ResumeV);
+ ResumeV = Builder.CreateSelect(
+ Cmp, RdxResult->getOperand(2)->getLiveInIRValue(), ResumeV);
}
} else {
// Retrieve the induction resume values for wide inductions from
diff --git a/llvm/lib/Transforms/Vectorize/VPlanPatternMatch.h b/llvm/lib/Transforms/Vectorize/VPlanPatternMatch.h
index dfd9fc3d4d719..b2535fe3aa578 100644
--- a/llvm/lib/Transforms/Vectorize/VPlanPatternMatch.h
+++ b/llvm/lib/Transforms/Vectorize/VPlanPatternMatch.h
@@ -318,6 +318,25 @@ m_VPInstruction(const Op0_t &Op0, const Op1_t &Op1, const Op2_t &Op2) {
{Op0, Op1, Op2});
}
+template <typename Op0_t, typename Op1_t, typename Op2_t, typename Op3_t,
+ unsigned Opcode, bool Commutative, typename... RecipeTys>
+using Recipe4Op_match = Recipe_match<std::tuple<Op0_t, Op1_t, Op2_t, Op3_t>,
+ Opcode, Commutative, RecipeTys...>;
+
+template <typename Op0_t, typename Op1_t, typename Op2_t, typename Op3_t,
+ unsigned Opcode>
+using VPInstruction4Op_match =
+ Recipe4Op_match<Op0_t, Op1_t, Op2_t, Op3_t, Opcode, /*Commutative*/ false,
+ VPInstruction>;
+
+template <unsigned Opcode, typename Op0_t, typename Op1_t, typename Op2_t,
+ typename Op3_t>
+inline VPInstruction4Op_match<Op0_t, Op1_t, Op2_t, Op3_t, Opcode>
+m_VPInstruction(const Op0_t &Op0, const Op1_t &Op1, const Op2_t &Op2,
+ const Op3_t &Op3) {
+ return VPInstruction4Op_match<Op0_t, Op1_t, Op2_t, Op3_t, Opcode>(
+ {Op0, Op1, Op2, Op3});
+}
template <typename Op0_t>
inline UnaryVPInstruction_match<Op0_t, Instruction::Freeze>
m_Freeze(const Op0_t &Op0) {
diff --git a/llvm/lib/Transforms/Vectorize/VPlanRecipes.cpp b/llvm/lib/Transforms/Vectorize/VPlanRecipes.cpp
index 90a04af60e3d8..aa0b245437f01 100644
--- a/llvm/lib/Transforms/Vectorize/VPlanRecipes.cpp
+++ b/llvm/lib/Transforms/Vectorize/VPlanRecipes.cpp
@@ -632,16 +632,16 @@ Value *VPInstruction::generate(VPTransformState &State) {
// The recipe's operands are the reduction phi, followed by one operand for
// each part of the reduction.
- unsigned UF = getNumOperands() - 2;
- Value *ReducedPartRdx = State.get(getOperand(2));
+ unsigned UF = getNumOperands() - 3;
+ Value *ReducedPartRdx = State.get(getOperand(3));
for (unsigned Part = 1; Part < UF; ++Part) {
ReducedPartRdx = createMinMaxOp(Builder, RecurKind::SMax, ReducedPartRdx,
- State.get(getOperand(2 + Part)));
+ State.get(getOperand(3 + Part)));
}
return createFindLastIVReduction(Builder, ReducedPartRdx,
State.get(getOperand(1), true),
- RdxDesc.getSentinelValue());
+ getOperand(2)->getLiveInIRValue());
}
case VPInstruction::ComputeReductionResult: {
// FIXME: The cross-recipe dependency on VPReductionPHIRecipe is temporary
diff --git a/llvm/lib/Transforms/Vectorize/VPlanUnroll.cpp b/llvm/lib/Transforms/Vectorize/VPlanUnroll.cpp
index 335301a927ceb..ea08c83285fa1 100644
--- a/llvm/lib/Transforms/Vectorize/VPlanUnroll.cpp
+++ b/llvm/lib/Transforms/Vectorize/VPlanUnroll.cpp
@@ -332,7 +332,7 @@ void UnrollState::unrollBlock(VPBlockBase *VPB) {
match(&R, m_VPInstruction<VPInstruction::ComputeReductionResult>(
m_VPValue(), m_VPValue(Op1))) ||
match(&R, m_VPInstruction<VPInstruction::ComputeFindLastIVResult>(
- m_VPValue(), m_VPValue(), m_VPValue(Op1)))) {
+ m_VPValue(), m_VPValue(), m_VPValue(), m_VPValue(Op1)))) {
addUniformForAllParts(cast<VPInstruction>(&R));
for (unsigned Part = 1; Part != UF; ++Part)
R.addOperand(getValueForPart(Op1, Part));
diff --git a/llvm/test/Transforms/LoopVectorize/vplan-printing-reductions.ll b/llvm/test/Transforms/LoopVectorize/vplan-printing-reductions.ll
index d57e7fa526c94..ed0513679152d 100644
--- a/llvm/test/Transforms/LoopVectorize/vplan-printing-reductions.ll
+++ b/llvm/test/Transforms/LoopVectorize/vplan-printing-reductions.ll
@@ -237,7 +237,7 @@ define i64 @find_last_iv(ptr %a, i64 %n, i64 %start) {
; CHECK-NEXT: Successor(s): middle.block
; CHECK-EMPTY:
; CHECK-NEXT: middle.block:
-; CHECK-NEXT: EMIT vp<[[RDX_RES:%.+]]> = compute-find-last-iv-result ir<%rdx>, ir<%start>, ir<%cond>
+; CHECK-NEXT: EMIT vp<[[RDX_RES:%.+]]> = compute-find-last-iv-result ir<%rdx>, ir<%start>, ir<-9223372036854775808>, ir<%cond>
; CHECK-NEXT: EMIT vp<[[EXT:%.+]]> = extract-last-element vp<[[RDX_RES]]>
; CHECK-NEXT: EMIT vp<%cmp.n> = icmp eq ir<%n>, vp<{{.+}}>
; CHECK-NEXT: EMIT branch-on-cond vp<%cmp.n>
More information about the llvm-commits
mailing list