[llvm] 6766eea - [VPlan] Create header phi recipes after initial scalar optimizations. (#200920)
via llvm-commits
llvm-commits at lists.llvm.org
Wed Jun 10 12:54:07 PDT 2026
Author: Florian Hahn
Date: 2026-06-10T21:54:02+02:00
New Revision: 6766eea39e4b362066f1e2678a09a78ad98c2183
URL: https://github.com/llvm/llvm-project/commit/6766eea39e4b362066f1e2678a09a78ad98c2183
DIFF: https://github.com/llvm/llvm-project/commit/6766eea39e4b362066f1e2678a09a78ad98c2183.diff
LOG: [VPlan] Create header phi recipes after initial scalar optimizations. (#200920)
Move createHeaderPhiRecipes after initial scalar transformations.
Header phi creation effectively is the first step of the widening
process. Running scalar opimizations before it can clean up dead header
phis and also also results in a cleaned up scalar VPlan that can be used
to compute the cost of the scalar loop accurately
(https://github.com/llvm/llvm-project/pull/196845).
Now that creating header phis happens after replaceSymbolicStrides we
are looking up the stride from the VPlan directly if possible.
PR: https://github.com/llvm/llvm-project/pull/200920
Added:
Modified:
llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
llvm/lib/Transforms/Vectorize/VPlanConstruction.cpp
llvm/test/Transforms/LoopVectorize/VPlan/X86/scalarize-wide-load-for-address-use.ll
llvm/test/Transforms/LoopVectorize/VPlan/constant-fold.ll
llvm/test/Transforms/LoopVectorize/VPlan/for-phi-ordering.ll
llvm/test/Transforms/LoopVectorize/VPlan/vplan-print-after-all.ll
Removed:
################################################################################
diff --git a/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp b/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
index 9d3bc2ab52391..f9b325a5233e6 100644
--- a/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
+++ b/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
@@ -6508,7 +6508,14 @@ VPlanPtr LoopVectorizationPlanner::tryToBuildVPlan1() {
auto VPlan0 = VPlanTransforms::buildVPlan0(OrigLoop, *LI,
Legal->getWidestInductionType(),
PSE, LVer ? &*LVer : nullptr);
+
VPDominatorTree VPDT(*VPlan0);
+ if (const LoopAccessInfo *LAI = Legal->getLAI())
+ RUN_VPLAN_PASS(VPlanTransforms::replaceSymbolicStrides, *VPlan0, PSE,
+ LAI->getSymbolicStrides(), VPDT);
+ RUN_VPLAN_PASS(VPlanTransforms::simplifyRecipes, *VPlan0);
+ RUN_VPLAN_PASS(VPlanTransforms::removeDeadRecipes, *VPlan0);
+
// Create recipes for header phis. For outer loops, reductions, recurrences
// and in-loop reductions are empty since legality doesn't detect them.
if (!RUN_VPLAN_PASS(VPlanTransforms::createHeaderPhiRecipes, *VPlan0, PSE,
@@ -6522,8 +6529,6 @@ VPlanPtr LoopVectorizationPlanner::tryToBuildVPlan1() {
if (const LoopAccessInfo *LAI = Legal->getLAI())
RUN_VPLAN_PASS(VPlanTransforms::replaceSymbolicStrides, *VPlan0, PSE,
LAI->getSymbolicStrides(), VPDT);
- RUN_VPLAN_PASS(VPlanTransforms::simplifyRecipes, *VPlan0);
- RUN_VPLAN_PASS(VPlanTransforms::removeDeadRecipes, *VPlan0);
// Add surviving induction predicates to PSE and check constraints.
bool ForceVectorization = Hints.getForce() == LoopVectorizeHints::FK_Enabled;
diff --git a/llvm/lib/Transforms/Vectorize/VPlanConstruction.cpp b/llvm/lib/Transforms/Vectorize/VPlanConstruction.cpp
index 79713a2ee4a98..3e41133ff7463 100644
--- a/llvm/lib/Transforms/Vectorize/VPlanConstruction.cpp
+++ b/llvm/lib/Transforms/Vectorize/VPlanConstruction.cpp
@@ -640,7 +640,7 @@ createWidenInductionRecipe(PHINode *Phi, VPPhi *PhiR, VPIRValue *Start,
"step must be loop invariant");
assert((Plan.getLiveIn(IndDesc.getStartValue()) == Start ||
(SE.isSCEVable(IndDesc.getStartValue()->getType()) &&
- SE.getSCEV(IndDesc.getStartValue()) ==
+ PSE.getSCEV(IndDesc.getStartValue()) ==
vputils::getSCEVExprForVPValue(Start, PSE))) &&
"Start VPValue must match IndDesc's start value");
@@ -874,17 +874,20 @@ static bool tryToSinkOrHoistRecurrenceUsers(VPBasicBlock *HeaderVPBB,
Previous = PrevPhi->getBackedgeValue()->getDefiningRecipe();
}
- assert(Previous && "Previous must be a recipe");
- // Sink FOR users after Previous or hoist Previous before FOR users.
- if (!sinkRecurrenceUsersAfterPrevious(FOR, Previous, VPDT) &&
- !hoistPreviousBeforeFORUsers(FOR, Previous, VPDT))
- return false;
+ VPBasicBlock *InsertBlock = FOR->getParent();
+ VPBasicBlock::iterator InsertPt = InsertBlock->getFirstNonPhi();
+ if (Previous) {
+ // Sink FOR users after Previous or hoist Previous before FOR users.
+ if (!sinkRecurrenceUsersAfterPrevious(FOR, Previous, VPDT) &&
+ !hoistPreviousBeforeFORUsers(FOR, Previous, VPDT))
+ return false;
+ InsertBlock = Previous->getParent();
+ InsertPt = isa<VPHeaderPHIRecipe>(Previous)
+ ? InsertBlock->getFirstNonPhi()
+ : std::next(Previous->getIterator());
+ }
// Create FirstOrderRecurrenceSplice and replace FOR uses.
- VPBasicBlock *InsertBlock = Previous->getParent();
- auto InsertPt = isa<VPHeaderPHIRecipe>(Previous)
- ? InsertBlock->getFirstNonPhi()
- : std::next(Previous->getIterator());
VPBuilder LoopBuilder(InsertBlock, InsertPt);
auto *RecurSplice =
LoopBuilder.createNaryOp(VPInstruction::FirstOrderRecurrenceSplice,
@@ -962,6 +965,10 @@ bool VPlanTransforms::createHeaderPhiRecipes(
if (!tryToSinkOrHoistRecurrenceUsers(HeaderVPBB, VPDT))
return false;
+ // Skip renaming resume phi recipes, if any header phi has been removed.
+ if (range_size(HeaderVPBB->phis()) !=
+ range_size(Plan.getScalarPreheader()->phis()))
+ return true;
for (const auto &[HeaderPhiR, ScalarPhiR] :
zip_equal(HeaderVPBB->phis(), Plan.getScalarPreheader()->phis())) {
auto *ResumePhiR = cast<VPPhi>(&ScalarPhiR);
diff --git a/llvm/test/Transforms/LoopVectorize/VPlan/X86/scalarize-wide-load-for-address-use.ll b/llvm/test/Transforms/LoopVectorize/VPlan/X86/scalarize-wide-load-for-address-use.ll
index 7c9edb0a8e283..4039e8f235884 100644
--- a/llvm/test/Transforms/LoopVectorize/VPlan/X86/scalarize-wide-load-for-address-use.ll
+++ b/llvm/test/Transforms/LoopVectorize/VPlan/X86/scalarize-wide-load-for-address-use.ll
@@ -33,6 +33,7 @@ define void @reverse_unmasked_load_feeds_address(ptr noalias %src, i64 %n) {
; CHECK-NEXT: EMIT ir<%cmp> = fcmp oeq ir<%val>, ir<0.000000e+00>
; CHECK-NEXT: EMIT ir<%ptr.sel> = select ir<%cmp>, ir<@tbl.a>, ir<@tbl.b>
; CHECK-NEXT: REPLICATE store ir<1.000000e+00>, ir<%ptr.sel>
+; CHECK-NEXT: EMIT ir<%iv.next> = add ir<%iv>, ir<-1>
; CHECK-NEXT: EMIT ir<%ec> = icmp eq ir<%iv>, ir<0>
; CHECK-NEXT: EMIT vp<%index.next> = add nuw vp<[[VP3]]>, vp<[[VP1]]>
; CHECK-NEXT: EMIT branch-on-count vp<%index.next>, vp<[[VP2]]>
diff --git a/llvm/test/Transforms/LoopVectorize/VPlan/constant-fold.ll b/llvm/test/Transforms/LoopVectorize/VPlan/constant-fold.ll
index a2c71c4aa6a76..45543942b344d 100644
--- a/llvm/test/Transforms/LoopVectorize/VPlan/constant-fold.ll
+++ b/llvm/test/Transforms/LoopVectorize/VPlan/constant-fold.ll
@@ -15,7 +15,6 @@
define void @f1() {
; CHECK-LABEL: VPlan for loop in 'f1' after VPlanTransforms::simplifyRecipes
; CHECK-NEXT: VPlan ' for UF>=1' {
-; CHECK-NEXT: Live-in vp<[[VF:%[0-9]+]]> = VF
; CHECK-EMPTY:
; CHECK-NEXT: ir-bb<bb1>:
; CHECK-NEXT: Successor(s): scalar.ph, vector.ph
@@ -24,7 +23,7 @@ define void @f1() {
; CHECK-NEXT: Successor(s): bb2
; CHECK-EMPTY:
; CHECK-NEXT: bb2:
-; CHECK-NEXT: ir<%c.1.0> = WIDEN-INDUCTION nsw ir<0>, ir<1>, vp<[[VF]]>
+; CHECK-NEXT: EMIT-SCALAR ir<%c.1.0> = phi [ ir<0>, vector.ph ], [ ir<%_tmp9>, bb2 ]
; CHECK-NEXT: EMIT-SCALAR ir<%_tmp1> = zext ir<0> to i64
; CHECK-NEXT: EMIT ir<%_tmp2> = getelementptr ir<@a>, ir<0>, ir<0>
; CHECK-NEXT: EMIT-SCALAR ir<%_tmp6> = sext ir<%c.1.0> to i64
@@ -60,7 +59,6 @@ bb3:
define void @redundant_or_1(ptr %dst, i1 %c.0, i1 %c.1) {
; CHECK-LABEL: VPlan for loop in 'redundant_or_1' after VPlanTransforms::simplifyRecipes
; CHECK-NEXT: VPlan ' for UF>=1' {
-; CHECK-NEXT: Live-in vp<[[VF:%[0-9]+]]> = VF
; CHECK-NEXT: Live-in ir<3> = original trip-count
; CHECK-EMPTY:
; CHECK-NEXT: ir-bb<entry>:
@@ -70,7 +68,7 @@ define void @redundant_or_1(ptr %dst, i1 %c.0, i1 %c.1) {
; CHECK-NEXT: Successor(s): loop.header
; CHECK-EMPTY:
; CHECK-NEXT: loop.header:
-; CHECK-NEXT: ir<%iv> = WIDEN-INDUCTION nuw nsw ir<0>, ir<1>, vp<[[VF]]>
+; CHECK-NEXT: EMIT-SCALAR ir<%iv> = phi [ ir<0>, vector.ph ], [ ir<%iv.next>, loop.latch ]
; CHECK-NEXT: EMIT branch-on-cond ir<%c.0>
; CHECK-NEXT: Successor(s): loop.latch, then.1
; CHECK-EMPTY:
@@ -112,7 +110,6 @@ exit:
define void @redundant_or_2(ptr %dst, i1 %c.0, i1 %c.1) {
; CHECK-LABEL: VPlan for loop in 'redundant_or_2' after VPlanTransforms::simplifyRecipes
; CHECK-NEXT: VPlan ' for UF>=1' {
-; CHECK-NEXT: Live-in vp<[[VF:%[0-9]+]]> = VF
; CHECK-NEXT: Live-in ir<3> = original trip-count
; CHECK-EMPTY:
; CHECK-NEXT: ir-bb<entry>:
@@ -122,7 +119,7 @@ define void @redundant_or_2(ptr %dst, i1 %c.0, i1 %c.1) {
; CHECK-NEXT: Successor(s): loop.header
; CHECK-EMPTY:
; CHECK-NEXT: loop.header:
-; CHECK-NEXT: ir<%iv> = WIDEN-INDUCTION nuw nsw ir<0>, ir<1>, vp<[[VF]]>
+; CHECK-NEXT: EMIT-SCALAR ir<%iv> = phi [ ir<0>, vector.ph ], [ ir<%iv.next>, loop.latch ]
; CHECK-NEXT: EMIT branch-on-cond ir<%c.0>
; CHECK-NEXT: Successor(s): loop.latch, then.1
; CHECK-EMPTY:
@@ -164,7 +161,6 @@ exit:
define void @redundant_and_1(ptr %dst, i1 %c.0, i1 %c.1) {
; CHECK-LABEL: VPlan for loop in 'redundant_and_1' after VPlanTransforms::simplifyRecipes
; CHECK-NEXT: VPlan ' for UF>=1' {
-; CHECK-NEXT: Live-in vp<[[VF:%[0-9]+]]> = VF
; CHECK-NEXT: Live-in ir<3> = original trip-count
; CHECK-EMPTY:
; CHECK-NEXT: ir-bb<entry>:
@@ -174,7 +170,7 @@ define void @redundant_and_1(ptr %dst, i1 %c.0, i1 %c.1) {
; CHECK-NEXT: Successor(s): loop.header
; CHECK-EMPTY:
; CHECK-NEXT: loop.header:
-; CHECK-NEXT: ir<%iv> = WIDEN-INDUCTION nuw nsw ir<0>, ir<1>, vp<[[VF]]>
+; CHECK-NEXT: EMIT-SCALAR ir<%iv> = phi [ ir<0>, vector.ph ], [ ir<%iv.next>, loop.latch ]
; CHECK-NEXT: EMIT branch-on-cond ir<%c.0>
; CHECK-NEXT: Successor(s): loop.latch, then.1
; CHECK-EMPTY:
@@ -217,7 +213,6 @@ exit:
define void @redundant_and_2(ptr %dst, i1 %c.0, i1 %c.1) {
; CHECK-LABEL: VPlan for loop in 'redundant_and_2' after VPlanTransforms::simplifyRecipes
; CHECK-NEXT: VPlan ' for UF>=1' {
-; CHECK-NEXT: Live-in vp<[[VF:%[0-9]+]]> = VF
; CHECK-NEXT: Live-in ir<3> = original trip-count
; CHECK-EMPTY:
; CHECK-NEXT: ir-bb<entry>:
@@ -227,7 +222,7 @@ define void @redundant_and_2(ptr %dst, i1 %c.0, i1 %c.1) {
; CHECK-NEXT: Successor(s): loop.header
; CHECK-EMPTY:
; CHECK-NEXT: loop.header:
-; CHECK-NEXT: ir<%iv> = WIDEN-INDUCTION nuw nsw ir<0>, ir<1>, vp<[[VF]]>
+; CHECK-NEXT: EMIT-SCALAR ir<%iv> = phi [ ir<0>, vector.ph ], [ ir<%iv.next>, loop.latch ]
; CHECK-NEXT: EMIT branch-on-cond ir<%c.0>
; CHECK-NEXT: Successor(s): loop.latch, then.1
; CHECK-EMPTY:
diff --git a/llvm/test/Transforms/LoopVectorize/VPlan/for-phi-ordering.ll b/llvm/test/Transforms/LoopVectorize/VPlan/for-phi-ordering.ll
index 190aff228cf25..18cd5129f73da 100644
--- a/llvm/test/Transforms/LoopVectorize/VPlan/for-phi-ordering.ll
+++ b/llvm/test/Transforms/LoopVectorize/VPlan/for-phi-ordering.ll
@@ -33,7 +33,7 @@ define i32 @test(i32 %limit, i1 %guard, i32 %divisor) {
; CHECK-NEXT: WIDEN ir<%prev.iv> = add nsw ir<%iv>, ir<-1>
; CHECK-NEXT: EMIT vp<[[VP7:%[0-9]+]]> = compute-reduction-result (add) ir<%sum>
; CHECK-NEXT: EMIT vp<[[VP8:%[0-9]+]]> = extract-last-part ir<%prev.iv>
-; CHECK-NEXT: EMIT vp<%vector.recur.extract> = extract-last-lane vp<[[VP8]]>
+; CHECK-NEXT: EMIT vp<[[VP9:%[0-9]+]]> = extract-last-lane vp<[[VP8]]>
; CHECK-NEXT: EMIT vp<%cmp.n> = icmp eq vp<[[VP3]]>, vp<[[VP2]]>
; CHECK-NEXT: EMIT branch-on-cond vp<%cmp.n>
; CHECK-NEXT: Successor(s): ir-bb<exit>, scalar.ph
@@ -43,9 +43,9 @@ define i32 @test(i32 %limit, i1 %guard, i32 %divisor) {
; CHECK-NEXT: No successors
; CHECK-EMPTY:
; CHECK-NEXT: scalar.ph:
-; CHECK-NEXT: EMIT-SCALAR vp<%bc.resume.val> = phi [ vp<[[VP2]]>, middle.block ], [ ir<0>, ir-bb<entry> ]
-; CHECK-NEXT: EMIT-SCALAR vp<%scalar.recur.init> = phi [ vp<%vector.recur.extract>, middle.block ], [ ir<0>, ir-bb<entry> ]
-; CHECK-NEXT: EMIT-SCALAR vp<%bc.merge.rdx> = phi [ vp<[[VP7]]>, middle.block ], [ ir<0>, ir-bb<entry> ]
+; CHECK-NEXT: EMIT-SCALAR vp<[[VP11:%[0-9]+]]> = phi [ vp<[[VP2]]>, middle.block ], [ ir<0>, ir-bb<entry> ]
+; CHECK-NEXT: EMIT-SCALAR vp<[[VP12:%[0-9]+]]> = phi [ vp<[[VP9]]>, middle.block ], [ ir<0>, ir-bb<entry> ]
+; CHECK-NEXT: EMIT-SCALAR vp<[[VP13:%[0-9]+]]> = phi [ vp<[[VP7]]>, middle.block ], [ ir<0>, ir-bb<entry> ]
; CHECK-NEXT: Successor(s): ir-bb<loop.header>
; CHECK-EMPTY:
; CHECK-NEXT: ir-bb<loop.header>:
diff --git a/llvm/test/Transforms/LoopVectorize/VPlan/vplan-print-after-all.ll b/llvm/test/Transforms/LoopVectorize/VPlan/vplan-print-after-all.ll
index 366d4c3e159f6..0610af3710f37 100644
--- a/llvm/test/Transforms/LoopVectorize/VPlan/vplan-print-after-all.ll
+++ b/llvm/test/Transforms/LoopVectorize/VPlan/vplan-print-after-all.ll
@@ -4,10 +4,11 @@
; Verify that `-vplan-print-after-all` option works.
; CHECK: VPlan for loop in 'foo' after printAfterInitialConstruction
-; CHECK: VPlan for loop in 'foo' after VPlanTransforms::createHeaderPhiRecipes
; CHECK: VPlan for loop in 'foo' after VPlanTransforms::replaceSymbolicStrides
; CHECK: VPlan for loop in 'foo' after VPlanTransforms::simplifyRecipes
; CHECK: VPlan for loop in 'foo' after VPlanTransforms::removeDeadRecipes
+; CHECK: VPlan for loop in 'foo' after VPlanTransforms::createHeaderPhiRecipes
+; CHECK: VPlan for loop in 'foo' after VPlanTransforms::replaceSymbolicStrides
; CHECK: VPlan for loop in 'foo' after VPlanTransforms::finalizeSCEVPredicates
; CHECK: VPlan for loop in 'foo' after VPlanTransforms::handleEarlyExits
; CHECK: VPlan for loop in 'foo' after VPlanTransforms::addMiddleCheck
More information about the llvm-commits
mailing list