[llvm] [VPlan] Create resume phis in scalar preheader early. (NFC) (PR #166099)
Florian Hahn via llvm-commits
llvm-commits at lists.llvm.org
Fri Nov 21 06:44:23 PST 2025
https://github.com/fhahn updated https://github.com/llvm/llvm-project/pull/166099
>From 67c16bd69dc1969a2771517a0f8dfdbc9be48723 Mon Sep 17 00:00:00 2001
From: Florian Hahn <flo at fhahn.com>
Date: Sun, 2 Nov 2025 21:10:19 +0000
Subject: [PATCH 1/2] [VPlan] Create resume phis in scalar preheader early.
(NFC)
Create phi recipes for scalar resume value up front in
addInitialSkeleton during initial construction. This will allow moving
the remaining code dealing with resume values to VPlan
transforms/construction.
---
llvm/lib/Transforms/Vectorize/VPlan.h | 5 ++-
.../Vectorize/VPlanConstruction.cpp | 9 +++++
.../Transforms/Vectorize/VPlanTransforms.cpp | 36 ++++++++-----------
.../Transforms/Vectorize/VPlanHCFGTest.cpp | 6 ++--
.../Vectorize/VPlanVerifierTest.cpp | 4 +--
5 files changed, 34 insertions(+), 26 deletions(-)
diff --git a/llvm/lib/Transforms/Vectorize/VPlan.h b/llvm/lib/Transforms/Vectorize/VPlan.h
index 0fc527e07e634..405f83a6ce8e5 100644
--- a/llvm/lib/Transforms/Vectorize/VPlan.h
+++ b/llvm/lib/Transforms/Vectorize/VPlan.h
@@ -1134,7 +1134,7 @@ class LLVM_ABI_FOR_TEST VPInstruction : public VPRecipeWithIRFlags,
OpcodeTy Opcode;
/// An optional name that can be used for the generated IR instruction.
- const std::string Name;
+ std::string Name;
/// Returns true if we can generate a scalar for the first lane only if
/// needed.
@@ -1225,6 +1225,9 @@ class LLVM_ABI_FOR_TEST VPInstruction : public VPRecipeWithIRFlags,
/// Returns the symbolic name assigned to the VPInstruction.
StringRef getName() const { return Name; }
+ /// Set the symbolic name for the VPInstruction.
+ void setName(StringRef NewName) { Name = NewName.str(); }
+
protected:
#if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP)
/// Print the VPInstruction to \p O.
diff --git a/llvm/lib/Transforms/Vectorize/VPlanConstruction.cpp b/llvm/lib/Transforms/Vectorize/VPlanConstruction.cpp
index dbbde1cafa9f2..5fbd61a929fe2 100644
--- a/llvm/lib/Transforms/Vectorize/VPlanConstruction.cpp
+++ b/llvm/lib/Transforms/Vectorize/VPlanConstruction.cpp
@@ -554,6 +554,15 @@ static void addInitialSkeleton(VPlan &Plan, Type *InductionTy, DebugLoc IVDL,
Plan.getEntry()->swapSuccessors();
createExtractsForLiveOuts(Plan, MiddleVPBB);
+
+ VPBuilder ScalarPHBuilder(ScalarPH);
+ for (const auto &[PhiR, ScalarPhiR] : zip_equal(
+ drop_begin(HeaderVPBB->phis()), Plan.getScalarHeader()->phis())) {
+ auto *VectorPhiR = cast<VPPhi>(&PhiR);
+ auto *ResumePhiR = ScalarPHBuilder.createScalarPhi(
+ {VectorPhiR, VectorPhiR->getOperand(0)}, VectorPhiR->getDebugLoc());
+ cast<VPIRPhi>(&ScalarPhiR)->addOperand(ResumePhiR);
+ }
}
std::unique_ptr<VPlan>
diff --git a/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp b/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp
index 25557f1d5d651..916e8b1cea28e 100644
--- a/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp
+++ b/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp
@@ -4527,9 +4527,10 @@ void VPlanTransforms::addBranchWeightToMiddleTerminator(
/// Create and return a ResumePhi for \p WideIV, unless it is truncated. If the
/// induction recipe is not canonical, creates a VPDerivedIVRecipe to compute
/// the end value of the induction.
-static VPInstruction *addResumePhiRecipeForInduction(
- VPWidenInductionRecipe *WideIV, VPBuilder &VectorPHBuilder,
- VPBuilder &ScalarPHBuilder, VPTypeAnalysis &TypeInfo, VPValue *VectorTC) {
+static VPValue *addResumePhiRecipeForInduction(VPWidenInductionRecipe *WideIV,
+ VPBuilder &VectorPHBuilder,
+ VPTypeAnalysis &TypeInfo,
+ VPValue *VectorTC) {
auto *WideIntOrFp = dyn_cast<VPWidenIntOrFpInductionRecipe>(WideIV);
// Truncated wide inductions resume from the last lane of their vector value
// in the last vector iteration which is handled elsewhere.
@@ -4555,9 +4556,7 @@ static VPInstruction *addResumePhiRecipeForInduction(
WideIV->getDebugLoc());
}
- auto *ResumePhiRecipe = ScalarPHBuilder.createScalarPhi(
- {EndValue, Start}, WideIV->getDebugLoc(), "bc.resume.val");
- return ResumePhiRecipe;
+ return EndValue;
}
void VPlanTransforms::addScalarResumePhis(
@@ -4570,21 +4569,18 @@ void VPlanTransforms::addScalarResumePhis(
VPBuilder VectorPHBuilder(
cast<VPBasicBlock>(VectorRegion->getSinglePredecessor()));
VPBuilder MiddleBuilder(MiddleVPBB, MiddleVPBB->getFirstNonPhi());
- VPBuilder ScalarPHBuilder(ScalarPH);
- for (VPRecipeBase &ScalarPhiR : Plan.getScalarHeader()->phis()) {
- auto *ScalarPhiIRI = cast<VPIRPhi>(&ScalarPhiR);
+ for (VPRecipeBase &PhiR : Plan.getScalarPreheader()->phis()) {
+ auto *ResumePhiR = cast<VPPhi>(&PhiR);
// TODO: Extract final value from induction recipe initially, optimize to
// pre-computed end value together in optimizeInductionExitUsers.
- auto *VectorPhiR =
- cast<VPHeaderPHIRecipe>(Builder.getRecipe(&ScalarPhiIRI->getIRPhi()));
+ auto *VectorPhiR = cast<VPHeaderPHIRecipe>(ResumePhiR->getOperand(0));
if (auto *WideIVR = dyn_cast<VPWidenInductionRecipe>(VectorPhiR)) {
- if (VPInstruction *ResumePhi = addResumePhiRecipeForInduction(
- WideIVR, VectorPHBuilder, ScalarPHBuilder, TypeInfo,
- &Plan.getVectorTripCount())) {
- assert(isa<VPPhi>(ResumePhi) && "Expected a phi");
- IVEndValues[WideIVR] = ResumePhi->getOperand(0);
- ScalarPhiIRI->addOperand(ResumePhi);
+ if (VPValue *ResumeV = addResumePhiRecipeForInduction(
+ WideIVR, VectorPHBuilder, TypeInfo, &Plan.getVectorTripCount())) {
+ IVEndValues[WideIVR] = ResumeV;
+ ResumePhiR->setOperand(0, ResumeV);
+ ResumePhiR->setName("bc.resume.val");
continue;
}
// TODO: Also handle truncated inductions here. Computing end-values
@@ -4606,10 +4602,8 @@ void VPlanTransforms::addScalarResumePhis(
ResumeFromVectorLoop = MiddleBuilder.createNaryOp(
VPInstruction::ExtractLastElement, {ResumeFromVectorLoop}, {},
"vector.recur.extract");
- StringRef Name = IsFOR ? "scalar.recur.init" : "bc.merge.rdx";
- auto *ResumePhiR = ScalarPHBuilder.createScalarPhi(
- {ResumeFromVectorLoop, VectorPhiR->getStartValue()}, {}, Name);
- ScalarPhiIRI->addOperand(ResumePhiR);
+ ResumePhiR->setName(IsFOR ? "scalar.recur.init" : "bc.merge.rdx");
+ ResumePhiR->setOperand(0, ResumeFromVectorLoop);
}
}
diff --git a/llvm/unittests/Transforms/Vectorize/VPlanHCFGTest.cpp b/llvm/unittests/Transforms/Vectorize/VPlanHCFGTest.cpp
index 5742df2aa3c53..127e830b1bc14 100644
--- a/llvm/unittests/Transforms/Vectorize/VPlanHCFGTest.cpp
+++ b/llvm/unittests/Transforms/Vectorize/VPlanHCFGTest.cpp
@@ -113,12 +113,13 @@ compound=true
N0 -> N2 [ label="F"]
N1 [label =
"scalar.ph:\l" +
+ " EMIT-SCALAR vp\<%6\> = phi [ ir\<%indvars.iv\>, middle.block ], [ ir\<0\>, ir-bb\<entry\> ]\l" +
"Successor(s): ir-bb\<for.body\>\l"
]
N1 -> N3 [ label=""]
N3 [label =
"ir-bb\<for.body\>:\l" +
- " IR %indvars.iv = phi i64 [ 0, %entry ], [ %indvars.iv.next, %for.body ]\l" +
+ " IR %indvars.iv = phi i64 [ 0, %entry ], [ %indvars.iv.next, %for.body ] (extra operand: vp\<%6\> from scalar.ph)\l" +
" IR %arr.idx = getelementptr inbounds i32, ptr %A, i64 %indvars.iv\l" +
" IR %l1 = load i32, ptr %arr.idx, align 4\l" +
" IR %res = add i32 %l1, 10\l" +
@@ -282,12 +283,13 @@ compound=true
N0 -> N2 [ label="F"]
N1 [label =
"scalar.ph:\l" +
+ " EMIT-SCALAR vp\<%6\> = phi [ ir\<%iv\>, middle.block ], [ ir\<0\>, ir-bb\<entry\> ]\l" +
"Successor(s): ir-bb\<loop.header\>\l"
]
N1 -> N3 [ label=""]
N3 [label =
"ir-bb\<loop.header\>:\l" +
- " IR %iv = phi i64 [ 0, %entry ], [ %iv.next, %loop.latch ]\l" +
+ " IR %iv = phi i64 [ 0, %entry ], [ %iv.next, %loop.latch ] (extra operand: vp\<%6\> from scalar.ph)\l" +
" IR %arr.idx = getelementptr inbounds i32, ptr %A, i64 %iv\l" +
" IR %l1 = load i32, ptr %arr.idx, align 4\l" +
" IR %c = icmp eq i32 %l1, 0\l" +
diff --git a/llvm/unittests/Transforms/Vectorize/VPlanVerifierTest.cpp b/llvm/unittests/Transforms/Vectorize/VPlanVerifierTest.cpp
index 1b0cd1074c1b5..ec8f06a1eb755 100644
--- a/llvm/unittests/Transforms/Vectorize/VPlanVerifierTest.cpp
+++ b/llvm/unittests/Transforms/Vectorize/VPlanVerifierTest.cpp
@@ -346,6 +346,8 @@ TEST_F(VPIRVerifierTest, testVerifyIRPhiInScalarHeaderVPIRBB) {
Function *F = M.getFunction("f");
BasicBlock *LoopHeader = F->getEntryBlock().getSingleSuccessor();
auto Plan = buildVPlan(LoopHeader);
+ VPValue *Zero = Plan->getConstantInt(32, 0);
+ Plan->getScalarHeader()->front().addOperand(Zero);
#if GTEST_HAS_STREAM_REDIRECTION
::testing::internal::CaptureStderr();
@@ -387,8 +389,6 @@ TEST_F(VPIRVerifierTest, testVerifyIRPhiInExitVPIRBB) {
{HeaderBlock->front().getVPSingleValue()});
DefI->insertBefore(Plan->getMiddleBlock()->getTerminator());
Plan->getExitBlocks()[0]->front().addOperand(DefI);
- VPValue *Zero = Plan->getConstantInt(32, 0);
- Plan->getScalarHeader()->front().addOperand(Zero);
#if GTEST_HAS_STREAM_REDIRECTION
::testing::internal::CaptureStderr();
>From 28596f9396584feaed95730c899046c297ebfda6 Mon Sep 17 00:00:00 2001
From: Florian Hahn <flo at fhahn.com>
Date: Fri, 21 Nov 2025 14:43:31 +0000
Subject: [PATCH 2/2] !fixup rename and remove some dead code.
---
llvm/lib/Transforms/Vectorize/LoopVectorize.cpp | 17 +++--------------
.../Transforms/Vectorize/VPlanTransforms.cpp | 5 ++---
llvm/lib/Transforms/Vectorize/VPlanTransforms.h | 12 ++++++------
3 files changed, 11 insertions(+), 23 deletions(-)
diff --git a/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp b/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
index 5b8725178521e..277e43a38018e 100644
--- a/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
+++ b/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
@@ -8532,7 +8532,7 @@ VPlanPtr LoopVectorizationPlanner::tryToBuildVPlanWithVPRecipes(
// failures.
VPlanTransforms::addExitUsersForFirstOrderRecurrences(*Plan, Range);
DenseMap<VPValue *, VPValue *> IVEndValues;
- VPlanTransforms::addScalarResumePhis(*Plan, RecipeBuilder, IVEndValues);
+ VPlanTransforms::updateScalarResumePhis(*Plan, IVEndValues);
// ---------------------------------------------------------------------------
// Transform initial VPlan: Apply previously taken decisions, in order, to
@@ -8630,23 +8630,12 @@ VPlanPtr LoopVectorizationPlanner::tryToBuildVPlan(VFRange &Range) {
*TLI))
return nullptr;
- // Collect mapping of IR header phis to header phi recipes, to be used in
- // addScalarResumePhis.
- DenseMap<VPBasicBlock *, VPValue *> BlockMaskCache;
- VPRecipeBuilder RecipeBuilder(*Plan, OrigLoop, TLI, &TTI, Legal, CM, PSE,
- Builder, BlockMaskCache);
- for (auto &R : Plan->getVectorLoopRegion()->getEntryBasicBlock()->phis()) {
- if (isa<VPCanonicalIVPHIRecipe>(&R))
- continue;
- auto *HeaderR = cast<VPHeaderPHIRecipe>(&R);
- RecipeBuilder.setRecipe(HeaderR->getUnderlyingInstr(), HeaderR);
- }
- DenseMap<VPValue *, VPValue *> IVEndValues;
// TODO: IVEndValues are not used yet in the native path, to optimize exit
// values.
// TODO: We can't call runPass on the transform yet, due to verifier
// failures.
- VPlanTransforms::addScalarResumePhis(*Plan, RecipeBuilder, IVEndValues);
+ DenseMap<VPValue *, VPValue *> IVEndValues;
+ VPlanTransforms::updateScalarResumePhis(*Plan, IVEndValues);
assert(verifyVPlanIsValid(*Plan) && "VPlan is invalid");
return Plan;
diff --git a/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp b/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp
index c5f4787b2c3ba..c589abd2e18e8 100644
--- a/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp
+++ b/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp
@@ -4561,9 +4561,8 @@ static VPValue *addResumePhiRecipeForInduction(VPWidenInductionRecipe *WideIV,
return EndValue;
}
-void VPlanTransforms::addScalarResumePhis(
- VPlan &Plan, VPRecipeBuilder &Builder,
- DenseMap<VPValue *, VPValue *> &IVEndValues) {
+void VPlanTransforms::updateScalarResumePhis(
+ VPlan &Plan, DenseMap<VPValue *, VPValue *> &IVEndValues) {
VPTypeAnalysis TypeInfo(Plan);
auto *ScalarPH = Plan.getScalarPreheader();
auto *MiddleVPBB = cast<VPBasicBlock>(ScalarPH->getPredecessors()[0]);
diff --git a/llvm/lib/Transforms/Vectorize/VPlanTransforms.h b/llvm/lib/Transforms/Vectorize/VPlanTransforms.h
index 708ea4185e1cb..17416c2fd00e4 100644
--- a/llvm/lib/Transforms/Vectorize/VPlanTransforms.h
+++ b/llvm/lib/Transforms/Vectorize/VPlanTransforms.h
@@ -372,12 +372,12 @@ struct VPlanTransforms {
addBranchWeightToMiddleTerminator(VPlan &Plan, ElementCount VF,
std::optional<unsigned> VScaleForTuning);
- /// Create resume phis in the scalar preheader for first-order recurrences,
- /// reductions and inductions, and update the VPIRInstructions wrapping the
- /// original phis in the scalar header. End values for inductions are added to
- /// \p IVEndValues.
- static void addScalarResumePhis(VPlan &Plan, VPRecipeBuilder &Builder,
- DenseMap<VPValue *, VPValue *> &IVEndValues);
+ /// Update the resume phis in the scalar preheader after creating wide recipes
+ /// for first-order recurrences, reductions and inductions, End values for
+ /// inductions are added to \p IVEndValues.
+ static void
+ updateScalarResumePhis(VPlan &Plan,
+ DenseMap<VPValue *, VPValue *> &IVEndValues);
/// Handle users in the exit block for first order reductions in the original
/// exit block. The penultimate value of recurrences is fed to their LCSSA phi
More information about the llvm-commits
mailing list