[llvm] [VPlan] Untie tail folding from optimizeInductionLiveOutUsers. NFC (PR #207984)
Luke Lau via llvm-commits
llvm-commits at lists.llvm.org
Tue Jul 28 07:01:22 PDT 2026
https://github.com/lukel97 updated https://github.com/llvm/llvm-project/pull/207984
>From 2334c2a0b5720d7b0a4990b71174999aea7b1a3c Mon Sep 17 00:00:00 2001
From: Luke Lau <luke at igalia.com>
Date: Tue, 30 Jun 2026 23:53:55 +0800
Subject: [PATCH 1/4] [VPlan] Untie tail folding from
optimizeInductionLiveOutUsers. NFC
---
.../Transforms/Vectorize/LoopVectorize.cpp | 6 +--
.../Transforms/Vectorize/VPlanTransforms.cpp | 48 ++++++++++++++-----
.../Transforms/Vectorize/VPlanTransforms.h | 3 +-
...fmax-without-fast-math-flags-interleave.ll | 2 +-
4 files changed, 40 insertions(+), 19 deletions(-)
diff --git a/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp b/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
index 6bb68d1f7bb19..4e7fb170526be 100644
--- a/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
+++ b/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
@@ -6626,8 +6626,7 @@ VPlanPtr LoopVectorizationPlanner::tryToBuildVPlan(VPlanPtr Plan,
if (!RUN_VPLAN_PASS(VPlanTransforms::tryToConvertVPInstructionsToVPRecipes,
*Plan, *TLI))
return nullptr;
- RUN_VPLAN_PASS(VPlanTransforms::optimizeInductionLiveOutUsers, *Plan, PSE,
- /*FoldTail=*/false);
+ RUN_VPLAN_PASS(VPlanTransforms::optimizeInductionLiveOutUsers, *Plan, PSE);
return Plan;
}
@@ -6796,8 +6795,7 @@ VPlanPtr LoopVectorizationPlanner::tryToBuildVPlan(VPlanPtr Plan,
// Optimize FindIV reductions to use sentinel-based approach when possible.
RUN_VPLAN_PASS(VPlanTransforms::optimizeFindIVReductions, *Plan, PSE,
*OrigLoop);
- RUN_VPLAN_PASS(VPlanTransforms::optimizeInductionLiveOutUsers, *Plan, PSE,
- CM.foldTailByMasking());
+ RUN_VPLAN_PASS(VPlanTransforms::optimizeInductionLiveOutUsers, *Plan, PSE);
// Apply mandatory transformation to handle reductions with multiple in-loop
// uses if possible, bail out otherwise.
diff --git a/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp b/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp
index 39e8ae5dd3b8c..574f2a345b14a 100644
--- a/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp
+++ b/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp
@@ -1153,32 +1153,56 @@ optimizeLatchExitInductionUser(VPlan &Plan, VPValue *Op,
}
void VPlanTransforms::optimizeInductionLiveOutUsers(
- VPlan &Plan, PredicatedScalarEvolution &PSE, bool FoldTail) {
- // Compute end values for all inductions.
+ VPlan &Plan, PredicatedScalarEvolution &PSE) {
+ // Compute the value inductions at two points:
+ // - when exiting the loop from the latch, at trip-count iterations.
+ // - when resuming at the scalar preheader, at vector trip-count iterations.
VPRegionBlock *VectorRegion = Plan.getVectorLoopRegion();
auto *VectorPH = cast<VPBasicBlock>(VectorRegion->getSinglePredecessor());
VPBuilder VectorPHBuilder(VectorPH, VectorPH->begin());
- DenseMap<VPValue *, VPValue *> EndValues;
- VPValue *ResumeTC =
- FoldTail ? Plan.getTripCount() : &Plan.getVectorTripCount();
+ DenseMap<VPValue *, VPValue *> LatchExitValues, ResumeValues;
for (auto &Phi : VectorRegion->getEntryBasicBlock()->phis()) {
auto *WideIV = dyn_cast<VPWidenInductionRecipe>(&Phi);
if (!WideIV)
continue;
- if (VPValue *EndValue =
- tryToComputeEndValueForInduction(WideIV, VectorPHBuilder, ResumeTC))
- EndValues[WideIV] = EndValue;
+ if (VPValue *EndValue = tryToComputeEndValueForInduction(
+ WideIV, VectorPHBuilder, Plan.getTripCount()))
+ LatchExitValues[WideIV] = EndValue;
+ if (VPValue *EndValue = tryToComputeEndValueForInduction(
+ WideIV, VectorPHBuilder, &Plan.getVectorTripCount()))
+ ResumeValues[WideIV] = EndValue;
}
VPBasicBlock *MiddleVPBB = Plan.getMiddleBlock();
+ VPBasicBlock *LatchExitVPBB = nullptr;
+ // Try and find the latch exit from MiddleVPBB.
+ if (Plan.isExitBlock(MiddleVPBB->getSuccessors()[0])) {
+ LatchExitVPBB = cast<VPBasicBlock>(MiddleVPBB->getSuccessors()[0]);
+ // If we branch to the latch exit on TC == VTC, then the IVs at the latch
+ // exit are equal to the IVs at the scalar preheader. Reuse ResumeValues
+ // to avoid computing the same value in two different ways.
+ if (match(MiddleVPBB->getTerminator(),
+ m_BranchOnCond(m_SpecificICmp(
+ CmpInst::ICMP_EQ, m_Specific(Plan.getTripCount()),
+ m_Specific(&Plan.getVectorTripCount())))))
+ LatchExitValues = ResumeValues;
+ }
+
+ // Optimize users in the latch exit and in the scalar preheader.
for (VPRecipeBase &R : make_early_inc_range(*MiddleVPBB)) {
VPValue *Op;
if (!match(&R, m_ExitingIVValue(m_VPValue(Op))))
continue;
auto *WideIV = cast<VPWidenInductionRecipe>(Op);
- if (VPValue *EndValue = EndValues.lookup(WideIV)) {
- R.getVPSingleValue()->replaceAllUsesWith(EndValue);
- R.eraseFromParent();
+ for (VPUser *U : to_vector(R.getVPSingleValue()->users())) {
+ auto *UR = cast<VPRecipeBase>(U);
+ if (UR->getParent() == LatchExitVPBB) {
+ if (VPValue *EndValue = LatchExitValues.lookup(WideIV))
+ UR->replaceUsesOfWith(R.getVPSingleValue(), EndValue);
+ } else if (UR->getParent() == Plan.getScalarPreheader()) {
+ if (VPValue *EndValue = ResumeValues.lookup(WideIV))
+ UR->replaceUsesOfWith(R.getVPSingleValue(), EndValue);
+ }
}
}
@@ -1191,7 +1215,7 @@ void VPlanTransforms::optimizeInductionLiveOutUsers(
VPValue *Escape = nullptr;
if (PredVPBB == MiddleVPBB)
Escape = optimizeLatchExitInductionUser(
- Plan, ExitIRI->getOperand(Idx), EndValues, PSE);
+ Plan, ExitIRI->getOperand(Idx), LatchExitValues, PSE);
else
Escape = optimizeEarlyExitInductionUser(
Plan, ExitIRI->getOperand(Idx), PSE);
diff --git a/llvm/lib/Transforms/Vectorize/VPlanTransforms.h b/llvm/lib/Transforms/Vectorize/VPlanTransforms.h
index 3260526552281..05fe7591a6fe1 100644
--- a/llvm/lib/Transforms/Vectorize/VPlanTransforms.h
+++ b/llvm/lib/Transforms/Vectorize/VPlanTransforms.h
@@ -417,8 +417,7 @@ struct VPlanTransforms {
/// IV values by feeding them precomputed end values instead, possibly taken
/// one step backwards.
static void optimizeInductionLiveOutUsers(VPlan &Plan,
- PredicatedScalarEvolution &PSE,
- bool FoldTail);
+ PredicatedScalarEvolution &PSE);
/// Add explicit broadcasts for live-ins and VPValues defined in \p Plan's entry block if they are used as vectors.
static void materializeBroadcasts(VPlan &Plan);
diff --git a/llvm/test/Transforms/LoopVectorize/fmax-without-fast-math-flags-interleave.ll b/llvm/test/Transforms/LoopVectorize/fmax-without-fast-math-flags-interleave.ll
index b8a15df2886e5..8206afea5b0d8 100644
--- a/llvm/test/Transforms/LoopVectorize/fmax-without-fast-math-flags-interleave.ll
+++ b/llvm/test/Transforms/LoopVectorize/fmax-without-fast-math-flags-interleave.ll
@@ -324,7 +324,7 @@ define float @fmaxnum_tailfold(ptr %src, i64 %n) #0 {
; CHECK-NEXT: [[TMP54:%.*]] = select <4 x i1> [[TMP2]], <4 x float> [[TMP52]], <4 x float> [[VEC_PHI1]]
; CHECK-NEXT: [[TMP60:%.*]] = select i1 [[TMP57]], <4 x float> [[VEC_PHI]], <4 x float> [[TMP53]]
; CHECK-NEXT: [[TMP61:%.*]] = select i1 [[TMP57]], <4 x float> [[VEC_PHI1]], <4 x float> [[TMP54]]
-; CHECK-NEXT: [[TMP62:%.*]] = select i1 [[TMP57]], i64 [[INDEX]], i64 [[TMP0]]
+; CHECK-NEXT: [[TMP62:%.*]] = select i1 [[TMP57]], i64 [[INDEX]], i64 [[N_VEC]]
; CHECK-NEXT: [[RDX_MINMAX:%.*]] = call <4 x float> @llvm.maxnum.v4f32(<4 x float> [[TMP60]], <4 x float> [[TMP61]])
; CHECK-NEXT: [[TMP63:%.*]] = call float @llvm.vector.reduce.fmax.v4f32(<4 x float> [[RDX_MINMAX]])
; CHECK-NEXT: [[TMP64:%.*]] = xor i1 [[TMP57]], true
>From 4ba07161c4fad05e4e4e620bd613a65a3eacdf91 Mon Sep 17 00:00:00 2001
From: Luke Lau <luke at igalia.com>
Date: Thu, 23 Jul 2026 14:04:44 +0800
Subject: [PATCH 2/4] Update test after merge
---
.../Transforms/LoopVectorize/fmax-without-fast-math-flags.ll | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/llvm/test/Transforms/LoopVectorize/fmax-without-fast-math-flags.ll b/llvm/test/Transforms/LoopVectorize/fmax-without-fast-math-flags.ll
index e04caf61609f4..9c07104e81ebf 100644
--- a/llvm/test/Transforms/LoopVectorize/fmax-without-fast-math-flags.ll
+++ b/llvm/test/Transforms/LoopVectorize/fmax-without-fast-math-flags.ll
@@ -826,7 +826,7 @@ define float @fmaxnum_constant_trip_count_tailfold(ptr %src) #0 {
; CHECK: [[MIDDLE_BLOCK]]:
; CHECK-NEXT: [[TMP30:%.*]] = select <4 x i1> [[TMP0]], <4 x float> [[TMP24]], <4 x float> [[VEC_PHI]]
; CHECK-NEXT: [[TMP31:%.*]] = select i1 [[TMP27]], <4 x float> [[VEC_PHI]], <4 x float> [[TMP30]]
-; CHECK-NEXT: [[TMP32:%.*]] = select i1 [[TMP27]], i64 [[INDEX]], i64 30
+; CHECK-NEXT: [[TMP32:%.*]] = select i1 [[TMP27]], i64 [[INDEX]], i64 32
; CHECK-NEXT: [[TMP33:%.*]] = call float @llvm.vector.reduce.fmax.v4f32(<4 x float> [[TMP31]])
; CHECK-NEXT: [[TMP34:%.*]] = xor i1 [[TMP27]], true
; CHECK-NEXT: br i1 [[TMP34]], label %[[EXIT:.*]], label %[[SCALAR_PH:.*]]
>From 2ea48057ce48f88b52b2bb047703e913586e6100 Mon Sep 17 00:00:00 2001
From: Luke Lau <luke at igalia.com>
Date: Thu, 23 Jul 2026 23:15:54 +0800
Subject: [PATCH 3/4] Rework to compute end values lazily
---
.../Transforms/Vectorize/VPlanTransforms.cpp | 86 +++++++++----------
1 file changed, 42 insertions(+), 44 deletions(-)
diff --git a/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp b/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp
index 4cf0422ffa83b..38494361c5f40 100644
--- a/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp
+++ b/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp
@@ -962,10 +962,10 @@ static VPValue *tryToComputeEndValueForInduction(VPWidenInductionRecipe *WideIV,
/// Attempts to optimize the induction variable exit values for users in the
/// exit block coming from the latch in the original scalar loop.
-static VPValue *
-optimizeLatchExitInductionUser(VPlan &Plan, VPValue *Op,
- DenseMap<VPValue *, VPValue *> &EndValues,
- PredicatedScalarEvolution &PSE) {
+template <typename EndValueFn>
+static VPValue *optimizeLatchExitInductionUser(VPlan &Plan, VPValue *Op,
+ EndValueFn GetEndValue,
+ PredicatedScalarEvolution &PSE) {
VPValue *Incoming;
if (!match(Op, m_CombineOr(m_ExtractLastLaneOfLastPart(m_VPValue(Incoming)),
m_ExtractLane(m_LastActiveLane(m_HeaderMask()),
@@ -976,7 +976,7 @@ optimizeLatchExitInductionUser(VPlan &Plan, VPValue *Op,
if (!WideIV)
return nullptr;
- VPValue *EndValue = EndValues.lookup(WideIV);
+ VPValue *EndValue = GetEndValue(WideIV);
assert(EndValue && "Must have computed the end value up front");
// `getOptimizableIVOf()` always returns the pre-incremented IV, so if it
@@ -1012,41 +1012,34 @@ optimizeLatchExitInductionUser(VPlan &Plan, VPValue *Op,
void VPlanTransforms::optimizeInductionLiveOutUsers(
VPlan &Plan, PredicatedScalarEvolution &PSE) {
- // Compute the value inductions at two points:
- // - when exiting the loop from the latch, at trip-count iterations.
- // - when resuming at the scalar preheader, at vector trip-count iterations.
VPRegionBlock *VectorRegion = Plan.getVectorLoopRegion();
auto *VectorPH = cast<VPBasicBlock>(VectorRegion->getSinglePredecessor());
- VPBuilder VectorPHBuilder(VectorPH, VectorPH->begin());
- DenseMap<VPValue *, VPValue *> LatchExitValues, ResumeValues;
- for (auto &Phi : VectorRegion->getEntryBasicBlock()->phis()) {
- auto *WideIV = dyn_cast<VPWidenInductionRecipe>(&Phi);
- if (!WideIV)
- continue;
- if (VPValue *EndValue = tryToComputeEndValueForInduction(
- WideIV, VectorPHBuilder, Plan.getTripCount()))
- LatchExitValues[WideIV] = EndValue;
- if (VPValue *EndValue = tryToComputeEndValueForInduction(
- WideIV, VectorPHBuilder, &Plan.getVectorTripCount()))
- ResumeValues[WideIV] = EndValue;
- }
+ auto GetEndValue =
+ [VectorPHBuilder = VPBuilder(VectorPH, VectorPH->begin()),
+ EndValues = DenseMap<std::pair<VPValue *, VPValue *>, VPValue *>()](
+ VPWidenInductionRecipe *WideIV, VPValue *IVEnd) mutable -> VPValue * {
+ if (EndValues.contains({WideIV, IVEnd}))
+ return EndValues[{WideIV, IVEnd}];
+ VPValue *EndValue =
+ tryToComputeEndValueForInduction(WideIV, VectorPHBuilder, IVEnd);
+ if (!EndValue)
+ return nullptr;
+ EndValues[{WideIV, IVEnd}] = EndValue;
+ return EndValue;
+ };
VPBasicBlock *MiddleVPBB = Plan.getMiddleBlock();
- VPBasicBlock *LatchExitVPBB = nullptr;
- // Try and find the latch exit from MiddleVPBB.
- if (Plan.isExitBlock(MiddleVPBB->getSuccessors()[0])) {
- LatchExitVPBB = cast<VPBasicBlock>(MiddleVPBB->getSuccessors()[0]);
- // If we branch to the latch exit on TC == VTC, then the IVs at the latch
- // exit are equal to the IVs at the scalar preheader. Reuse ResumeValues
- // to avoid computing the same value in two different ways.
- if (match(MiddleVPBB->getTerminator(),
- m_BranchOnCond(m_SpecificICmp(
- CmpInst::ICMP_EQ, m_Specific(Plan.getTripCount()),
- m_Specific(&Plan.getVectorTripCount())))))
- LatchExitValues = ResumeValues;
- }
-
- // Optimize users in the latch exit and in the scalar preheader.
+ // If we branch to the latch exit on TC == VTC, then the IVs at the latch
+ // exit are equal to the IVs at the scalar preheader. Reuse ResumeValues
+ // to avoid computing the same value in two different ways.
+ VPValue *TC = Plan.getTripCount();
+ VPValue *VTC = &Plan.getVectorTripCount();
+ if (MiddleVPBB->getTerminator() &&
+ match(MiddleVPBB->getTerminator(),
+ m_BranchOnCond(m_SpecificICmp(CmpInst::ICMP_EQ, m_Specific(TC),
+ m_Specific(VTC)))))
+ TC = VTC;
+
for (VPRecipeBase &R : make_early_inc_range(*MiddleVPBB)) {
VPValue *Op;
if (!match(&R, m_ExitingIVValue(m_VPValue(Op))))
@@ -1054,13 +1047,18 @@ void VPlanTransforms::optimizeInductionLiveOutUsers(
auto *WideIV = cast<VPWidenInductionRecipe>(Op);
for (VPUser *U : to_vector(R.getVPSingleValue()->users())) {
auto *UR = cast<VPRecipeBase>(U);
- if (UR->getParent() == LatchExitVPBB) {
- if (VPValue *EndValue = LatchExitValues.lookup(WideIV))
- UR->replaceUsesOfWith(R.getVPSingleValue(), EndValue);
- } else if (UR->getParent() == Plan.getScalarPreheader()) {
- if (VPValue *EndValue = ResumeValues.lookup(WideIV))
- UR->replaceUsesOfWith(R.getVPSingleValue(), EndValue);
- }
+ VPValue *EndIV;
+ // Optimize latch exit users at TC iterations.
+ if (Plan.isExitBlock(UR->getParent()) &&
+ is_contained(UR->getParent()->predecessors(), MiddleVPBB))
+ EndIV = TC;
+ // Optimize scalar preheader users at VTC iterations.
+ else if (UR->getParent() == Plan.getScalarPreheader())
+ EndIV = VTC;
+ else
+ continue;
+ if (VPValue *EndValue = GetEndValue(WideIV, EndIV))
+ UR->replaceUsesOfWith(R.getVPSingleValue(), EndValue);
}
}
@@ -1073,7 +1071,7 @@ void VPlanTransforms::optimizeInductionLiveOutUsers(
VPValue *Escape = nullptr;
if (PredVPBB == MiddleVPBB)
Escape = optimizeLatchExitInductionUser(
- Plan, ExitIRI->getOperand(Idx), LatchExitValues, PSE);
+ Plan, ExitIRI->getOperand(Idx), bind_back(GetEndValue, TC), PSE);
else
Escape = optimizeEarlyExitInductionUser(
Plan, ExitIRI->getOperand(Idx), PSE);
>From 1030f65ae4d3a2d47a5ff803fb9bbcc62c47d4f4 Mon Sep 17 00:00:00 2001
From: Luke Lau <luke at igalia.com>
Date: Fri, 24 Jul 2026 11:04:09 +0800
Subject: [PATCH 4/4] Use try_emplace do avoid double hashing
---
llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp | 13 +++++--------
1 file changed, 5 insertions(+), 8 deletions(-)
diff --git a/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp b/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp
index 38494361c5f40..e2b80164b03f8 100644
--- a/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp
+++ b/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp
@@ -1018,14 +1018,11 @@ void VPlanTransforms::optimizeInductionLiveOutUsers(
[VectorPHBuilder = VPBuilder(VectorPH, VectorPH->begin()),
EndValues = DenseMap<std::pair<VPValue *, VPValue *>, VPValue *>()](
VPWidenInductionRecipe *WideIV, VPValue *IVEnd) mutable -> VPValue * {
- if (EndValues.contains({WideIV, IVEnd}))
- return EndValues[{WideIV, IVEnd}];
- VPValue *EndValue =
- tryToComputeEndValueForInduction(WideIV, VectorPHBuilder, IVEnd);
- if (!EndValue)
- return nullptr;
- EndValues[{WideIV, IVEnd}] = EndValue;
- return EndValue;
+ auto [It, Inserted] = EndValues.try_emplace({WideIV, IVEnd});
+ if (Inserted)
+ It->second =
+ tryToComputeEndValueForInduction(WideIV, VectorPHBuilder, IVEnd);
+ return It->second;
};
VPBasicBlock *MiddleVPBB = Plan.getMiddleBlock();
More information about the llvm-commits
mailing list