[llvm] [VPlan] Create header phi recipes after initial scalar optimizations. (PR #200920)
Florian Hahn via llvm-commits
llvm-commits at lists.llvm.org
Wed Jun 10 08:03:07 PDT 2026
https://github.com/fhahn updated https://github.com/llvm/llvm-project/pull/200920
>From 3ad1c27908df13f62eea8b1548701100431734bf Mon Sep 17 00:00:00 2001
From: Florian Hahn <flo at fhahn.com>
Date: Fri, 29 May 2026 11:53:10 +0100
Subject: [PATCH 1/3] [VPlan] Create header phi recipes after initial scalar
optimizations.
Move createHeaderPhiRecipes after initial scalar transformations.
Header phi creation effectively is the first step of the widening
process. Running scalar opimizations before it can clean up dead
header phis and also also results in a cleaned up scalar VPlan that
can be used to compute the cost of the scalar loop accurately
(https://github.com/llvm/llvm-project/pull/196845).
Now that creating header phis happens after replaceSymbolicStrides we
are looking up the stride from the VPlan directly if possible.
---
.../Transforms/Vectorize/LoopVectorize.cpp | 11 ++--
.../Vectorize/VPlanConstruction.cpp | 60 ++++++++++++-------
.../scalarize-wide-load-for-address-use.ll | 1 +
.../LoopVectorize/VPlan/constant-fold.ll | 15 ++---
.../LoopVectorize/VPlan/for-phi-ordering.ll | 8 +--
.../VPlan/vplan-print-after-all.ll | 2 +-
.../LoopVectorize/VPlan/vplan-printing.ll | 6 +-
.../LoopVectorize/pointer-induction.ll | 4 +-
8 files changed, 61 insertions(+), 46 deletions(-)
diff --git a/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp b/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
index f0855b3dbf86c..4afc730fa6edc 100644
--- a/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
+++ b/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
@@ -6535,6 +6535,12 @@ VPlanPtr LoopVectorizationPlanner::tryToBuildVPlan1() {
Legal->getWidestInductionType(),
PSE, LVer ? &*LVer : nullptr);
+ if (const LoopAccessInfo *LAI = Legal->getLAI())
+ RUN_VPLAN_PASS(VPlanTransforms::replaceSymbolicStrides, *VPlan0, PSE,
+ LAI->getSymbolicStrides());
+ RUN_VPLAN_PASS(VPlanTransforms::simplifyRecipes, *VPlan0);
+ RUN_VPLAN_PASS(VPlanTransforms::removeDeadRecipes, *VPlan0);
+
// Create recipes for header phis. For outer loops, reductions, recurrences
// and in-loop reductions are empty since legality doesn't detect them.
if (!RUN_VPLAN_PASS(VPlanTransforms::createHeaderPhiRecipes, *VPlan0, PSE,
@@ -6545,11 +6551,6 @@ VPlanPtr LoopVectorizationPlanner::tryToBuildVPlan1() {
return nullptr;
}
- if (const LoopAccessInfo *LAI = Legal->getLAI())
- RUN_VPLAN_PASS(VPlanTransforms::replaceSymbolicStrides, *VPlan0, PSE,
- LAI->getSymbolicStrides());
- RUN_VPLAN_PASS(VPlanTransforms::simplifyRecipes, *VPlan0);
- RUN_VPLAN_PASS(VPlanTransforms::removeDeadRecipes, *VPlan0);
// If we're vectorizing a loop with an uncountable exit, make sure that the
// recipes are safe to handle.
// TODO: Remove this once we can properly check the VPlan itself for both
diff --git a/llvm/lib/Transforms/Vectorize/VPlanConstruction.cpp b/llvm/lib/Transforms/Vectorize/VPlanConstruction.cpp
index 80cdf9db2814c..717a1edea4431 100644
--- a/llvm/lib/Transforms/Vectorize/VPlanConstruction.cpp
+++ b/llvm/lib/Transforms/Vectorize/VPlanConstruction.cpp
@@ -632,19 +632,38 @@ createWidenInductionRecipe(PHINode *Phi, VPPhi *PhiR, VPIRValue *Start,
const InductionDescriptor &IndDesc, VPlan &Plan,
PredicatedScalarEvolution &PSE, Loop &OrigLoop,
DebugLoc DL) {
- [[maybe_unused]] ScalarEvolution &SE = *PSE.getSE();
+ ScalarEvolution &SE = *PSE.getSE();
assert(SE.isLoopInvariant(IndDesc.getStep(), &OrigLoop) &&
"step must be loop invariant");
assert((Plan.getLiveIn(IndDesc.getStartValue()) == Start ||
(SE.isSCEVable(IndDesc.getStartValue()->getType()) &&
- SE.getSCEV(IndDesc.getStartValue()) ==
+ PSE.getSCEV(IndDesc.getStartValue()) ==
vputils::getSCEVExprForVPValue(Start, PSE))) &&
"Start VPValue must match IndDesc's start value");
- VPValue *Step =
- vputils::getOrCreateVPValueForSCEVExpr(Plan, IndDesc.getStep());
-
VPValue *BackedgeVal = PhiR->getOperand(1);
+ // Try to get the versioned step from the induction increment directly.
+ auto GetStep = [&]() -> VPValue * {
+ VPValue *Step;
+ if (match(BackedgeVal, m_Add(m_Specific(PhiR), m_VPValue(Step))))
+ return Step;
+
+ // Pointer induction: scale the GEP index by the source element size.
+ Type *GEPSrcElemTy;
+ ArrayRef<VPValue *> GEPOps;
+ if (match(BackedgeVal, m_GetElementPtr(GEPSrcElemTy, GEPOps)) &&
+ GEPOps.size() == 2 && GEPOps[0] == PhiR) {
+ Type *StepTy = IndDesc.getStep()->getType();
+ const SCEV *Idx = vputils::getSCEVExprForVPValue(GEPOps[1], PSE);
+ if (!isa<SCEVCouldNotCompute>(Idx))
+ return vputils::getOrCreateVPValueForSCEVExpr(
+ Plan, SE.getMulExpr(SE.getTruncateOrSignExtend(Idx, StepTy),
+ SE.getSizeOfExpr(StepTy, GEPSrcElemTy)));
+ }
+ return vputils::getOrCreateVPValueForSCEVExpr(Plan, IndDesc.getStep());
+ };
+ VPValue *Step = GetStep();
+
// Replace live-out extracts of WideIV's backedge value by ExitingIVValue
// recipes. optimizeInductionLiveOutUsers will later compute the proper
// DerivedIV.
@@ -680,12 +699,6 @@ createWidenInductionRecipe(PHINode *Phi, VPPhi *PhiR, VPIRValue *Start,
IndDesc.getKind() == InductionDescriptor::IK_FpInduction) &&
"must have an integer or float induction at this point");
- // Update wide induction increments to use the same step as the corresponding
- // wide induction. This enables detecting induction increments directly in
- // VPlan and removes redundant splats.
- if (match(BackedgeVal, m_Add(m_Specific(PhiR), m_VPValue())))
- BackedgeVal->getDefiningRecipe()->setOperand(1, Step);
-
// It is always safe to copy over the NoWrap and FastMath flags. In
// particular, when folding tail by masking, the masked-off lanes are never
// used, so it is safe.
@@ -871,17 +884,20 @@ static bool tryToSinkOrHoistRecurrenceUsers(VPBasicBlock *HeaderVPBB,
Previous = PrevPhi->getBackedgeValue()->getDefiningRecipe();
}
- assert(Previous && "Previous must be a recipe");
- // Sink FOR users after Previous or hoist Previous before FOR users.
- if (!sinkRecurrenceUsersAfterPrevious(FOR, Previous, VPDT) &&
- !hoistPreviousBeforeFORUsers(FOR, Previous, VPDT))
- return false;
+ VPBasicBlock *InsertBlock = FOR->getParent();
+ VPBasicBlock::iterator InsertPt = InsertBlock->getFirstNonPhi();
+ if (Previous) {
+ // Sink FOR users after Previous or hoist Previous before FOR users.
+ if (!sinkRecurrenceUsersAfterPrevious(FOR, Previous, VPDT) &&
+ !hoistPreviousBeforeFORUsers(FOR, Previous, VPDT))
+ return false;
+ InsertBlock = Previous->getParent();
+ InsertPt = isa<VPHeaderPHIRecipe>(Previous)
+ ? InsertBlock->getFirstNonPhi()
+ : std::next(Previous->getIterator());
+ }
// Create FirstOrderRecurrenceSplice and replace FOR uses.
- VPBasicBlock *InsertBlock = Previous->getParent();
- auto InsertPt = isa<VPHeaderPHIRecipe>(Previous)
- ? InsertBlock->getFirstNonPhi()
- : std::next(Previous->getIterator());
VPBuilder LoopBuilder(InsertBlock, InsertPt);
auto *RecurSplice =
LoopBuilder.createNaryOp(VPInstruction::FirstOrderRecurrenceSplice,
@@ -948,6 +964,7 @@ bool VPlanTransforms::createHeaderPhiRecipes(
RdxDesc.hasUsesOutsideReductionChain());
};
+ SmallVector<VPValue *> BackedgeVals;
for (VPRecipeBase &R : make_early_inc_range(HeaderVPBB->phis())) {
auto *PhiR = cast<VPPhi>(&R);
VPHeaderPHIRecipe *HeaderPhiR = CreateHeaderPhiRecipe(PhiR);
@@ -959,6 +976,9 @@ bool VPlanTransforms::createHeaderPhiRecipes(
if (!tryToSinkOrHoistRecurrenceUsers(HeaderVPBB, VPDT))
return false;
+ if (range_size(HeaderVPBB->phis()) !=
+ range_size(Plan.getScalarPreheader()->phis()))
+ return true;
for (const auto &[HeaderPhiR, ScalarPhiR] :
zip_equal(HeaderVPBB->phis(), Plan.getScalarPreheader()->phis())) {
auto *ResumePhiR = cast<VPPhi>(&ScalarPhiR);
diff --git a/llvm/test/Transforms/LoopVectorize/VPlan/X86/scalarize-wide-load-for-address-use.ll b/llvm/test/Transforms/LoopVectorize/VPlan/X86/scalarize-wide-load-for-address-use.ll
index 7c9edb0a8e283..4039e8f235884 100644
--- a/llvm/test/Transforms/LoopVectorize/VPlan/X86/scalarize-wide-load-for-address-use.ll
+++ b/llvm/test/Transforms/LoopVectorize/VPlan/X86/scalarize-wide-load-for-address-use.ll
@@ -33,6 +33,7 @@ define void @reverse_unmasked_load_feeds_address(ptr noalias %src, i64 %n) {
; CHECK-NEXT: EMIT ir<%cmp> = fcmp oeq ir<%val>, ir<0.000000e+00>
; CHECK-NEXT: EMIT ir<%ptr.sel> = select ir<%cmp>, ir<@tbl.a>, ir<@tbl.b>
; CHECK-NEXT: REPLICATE store ir<1.000000e+00>, ir<%ptr.sel>
+; CHECK-NEXT: EMIT ir<%iv.next> = add ir<%iv>, ir<-1>
; CHECK-NEXT: EMIT ir<%ec> = icmp eq ir<%iv>, ir<0>
; CHECK-NEXT: EMIT vp<%index.next> = add nuw vp<[[VP3]]>, vp<[[VP1]]>
; CHECK-NEXT: EMIT branch-on-count vp<%index.next>, vp<[[VP2]]>
diff --git a/llvm/test/Transforms/LoopVectorize/VPlan/constant-fold.ll b/llvm/test/Transforms/LoopVectorize/VPlan/constant-fold.ll
index fbde96d2115dd..febfa24b9920c 100644
--- a/llvm/test/Transforms/LoopVectorize/VPlan/constant-fold.ll
+++ b/llvm/test/Transforms/LoopVectorize/VPlan/constant-fold.ll
@@ -15,7 +15,6 @@
define void @f1() {
; CHECK-LABEL: VPlan for loop in 'f1' after VPlanTransforms::simplifyRecipes
; CHECK-NEXT: VPlan ' for UF>=1' {
-; CHECK-NEXT: Live-in vp<[[VF:%[0-9]+]]> = VF
; CHECK-NEXT: Live-in ir<2> = original trip-count
; CHECK-EMPTY:
; CHECK-NEXT: ir-bb<bb1>:
@@ -25,7 +24,7 @@ define void @f1() {
; CHECK-NEXT: Successor(s): bb2
; CHECK-EMPTY:
; CHECK-NEXT: bb2:
-; CHECK-NEXT: ir<%c.1.0> = WIDEN-INDUCTION nsw ir<0>, ir<1>, vp<[[VF]]>
+; CHECK-NEXT: EMIT-SCALAR ir<%c.1.0> = phi [ ir<0>, vector.ph ], [ ir<%_tmp9>, bb2 ]
; CHECK-NEXT: EMIT-SCALAR ir<%_tmp1> = zext ir<0> to i64
; CHECK-NEXT: EMIT ir<%_tmp2> = getelementptr ir<@a>, ir<0>, ir<0>
; CHECK-NEXT: EMIT-SCALAR ir<%_tmp6> = sext ir<%c.1.0> to i64
@@ -61,7 +60,6 @@ bb3:
define void @redundant_or_1(ptr %dst, i1 %c.0, i1 %c.1) {
; CHECK-LABEL: VPlan for loop in 'redundant_or_1' after VPlanTransforms::simplifyRecipes
; CHECK-NEXT: VPlan ' for UF>=1' {
-; CHECK-NEXT: Live-in vp<[[VF:%[0-9]+]]> = VF
; CHECK-NEXT: Live-in ir<3> = original trip-count
; CHECK-EMPTY:
; CHECK-NEXT: ir-bb<entry>:
@@ -71,7 +69,7 @@ define void @redundant_or_1(ptr %dst, i1 %c.0, i1 %c.1) {
; CHECK-NEXT: Successor(s): loop.header
; CHECK-EMPTY:
; CHECK-NEXT: loop.header:
-; CHECK-NEXT: ir<%iv> = WIDEN-INDUCTION nuw nsw ir<0>, ir<1>, vp<[[VF]]>
+; CHECK-NEXT: EMIT-SCALAR ir<%iv> = phi [ ir<0>, vector.ph ], [ ir<%iv.next>, loop.latch ]
; CHECK-NEXT: EMIT branch-on-cond ir<%c.0>
; CHECK-NEXT: Successor(s): loop.latch, then.1
; CHECK-EMPTY:
@@ -113,7 +111,6 @@ exit:
define void @redundant_or_2(ptr %dst, i1 %c.0, i1 %c.1) {
; CHECK-LABEL: VPlan for loop in 'redundant_or_2' after VPlanTransforms::simplifyRecipes
; CHECK-NEXT: VPlan ' for UF>=1' {
-; CHECK-NEXT: Live-in vp<[[VF:%[0-9]+]]> = VF
; CHECK-NEXT: Live-in ir<3> = original trip-count
; CHECK-EMPTY:
; CHECK-NEXT: ir-bb<entry>:
@@ -123,7 +120,7 @@ define void @redundant_or_2(ptr %dst, i1 %c.0, i1 %c.1) {
; CHECK-NEXT: Successor(s): loop.header
; CHECK-EMPTY:
; CHECK-NEXT: loop.header:
-; CHECK-NEXT: ir<%iv> = WIDEN-INDUCTION nuw nsw ir<0>, ir<1>, vp<[[VF]]>
+; CHECK-NEXT: EMIT-SCALAR ir<%iv> = phi [ ir<0>, vector.ph ], [ ir<%iv.next>, loop.latch ]
; CHECK-NEXT: EMIT branch-on-cond ir<%c.0>
; CHECK-NEXT: Successor(s): loop.latch, then.1
; CHECK-EMPTY:
@@ -165,7 +162,6 @@ exit:
define void @redundant_and_1(ptr %dst, i1 %c.0, i1 %c.1) {
; CHECK-LABEL: VPlan for loop in 'redundant_and_1' after VPlanTransforms::simplifyRecipes
; CHECK-NEXT: VPlan ' for UF>=1' {
-; CHECK-NEXT: Live-in vp<[[VF:%[0-9]+]]> = VF
; CHECK-NEXT: Live-in ir<3> = original trip-count
; CHECK-EMPTY:
; CHECK-NEXT: ir-bb<entry>:
@@ -175,7 +171,7 @@ define void @redundant_and_1(ptr %dst, i1 %c.0, i1 %c.1) {
; CHECK-NEXT: Successor(s): loop.header
; CHECK-EMPTY:
; CHECK-NEXT: loop.header:
-; CHECK-NEXT: ir<%iv> = WIDEN-INDUCTION nuw nsw ir<0>, ir<1>, vp<[[VF]]>
+; CHECK-NEXT: EMIT-SCALAR ir<%iv> = phi [ ir<0>, vector.ph ], [ ir<%iv.next>, loop.latch ]
; CHECK-NEXT: EMIT branch-on-cond ir<%c.0>
; CHECK-NEXT: Successor(s): loop.latch, then.1
; CHECK-EMPTY:
@@ -218,7 +214,6 @@ exit:
define void @redundant_and_2(ptr %dst, i1 %c.0, i1 %c.1) {
; CHECK-LABEL: VPlan for loop in 'redundant_and_2' after VPlanTransforms::simplifyRecipes
; CHECK-NEXT: VPlan ' for UF>=1' {
-; CHECK-NEXT: Live-in vp<[[VF:%[0-9]+]]> = VF
; CHECK-NEXT: Live-in ir<3> = original trip-count
; CHECK-EMPTY:
; CHECK-NEXT: ir-bb<entry>:
@@ -228,7 +223,7 @@ define void @redundant_and_2(ptr %dst, i1 %c.0, i1 %c.1) {
; CHECK-NEXT: Successor(s): loop.header
; CHECK-EMPTY:
; CHECK-NEXT: loop.header:
-; CHECK-NEXT: ir<%iv> = WIDEN-INDUCTION nuw nsw ir<0>, ir<1>, vp<[[VF]]>
+; CHECK-NEXT: EMIT-SCALAR ir<%iv> = phi [ ir<0>, vector.ph ], [ ir<%iv.next>, loop.latch ]
; CHECK-NEXT: EMIT branch-on-cond ir<%c.0>
; CHECK-NEXT: Successor(s): loop.latch, then.1
; CHECK-EMPTY:
diff --git a/llvm/test/Transforms/LoopVectorize/VPlan/for-phi-ordering.ll b/llvm/test/Transforms/LoopVectorize/VPlan/for-phi-ordering.ll
index 190aff228cf25..18cd5129f73da 100644
--- a/llvm/test/Transforms/LoopVectorize/VPlan/for-phi-ordering.ll
+++ b/llvm/test/Transforms/LoopVectorize/VPlan/for-phi-ordering.ll
@@ -33,7 +33,7 @@ define i32 @test(i32 %limit, i1 %guard, i32 %divisor) {
; CHECK-NEXT: WIDEN ir<%prev.iv> = add nsw ir<%iv>, ir<-1>
; CHECK-NEXT: EMIT vp<[[VP7:%[0-9]+]]> = compute-reduction-result (add) ir<%sum>
; CHECK-NEXT: EMIT vp<[[VP8:%[0-9]+]]> = extract-last-part ir<%prev.iv>
-; CHECK-NEXT: EMIT vp<%vector.recur.extract> = extract-last-lane vp<[[VP8]]>
+; CHECK-NEXT: EMIT vp<[[VP9:%[0-9]+]]> = extract-last-lane vp<[[VP8]]>
; CHECK-NEXT: EMIT vp<%cmp.n> = icmp eq vp<[[VP3]]>, vp<[[VP2]]>
; CHECK-NEXT: EMIT branch-on-cond vp<%cmp.n>
; CHECK-NEXT: Successor(s): ir-bb<exit>, scalar.ph
@@ -43,9 +43,9 @@ define i32 @test(i32 %limit, i1 %guard, i32 %divisor) {
; CHECK-NEXT: No successors
; CHECK-EMPTY:
; CHECK-NEXT: scalar.ph:
-; CHECK-NEXT: EMIT-SCALAR vp<%bc.resume.val> = phi [ vp<[[VP2]]>, middle.block ], [ ir<0>, ir-bb<entry> ]
-; CHECK-NEXT: EMIT-SCALAR vp<%scalar.recur.init> = phi [ vp<%vector.recur.extract>, middle.block ], [ ir<0>, ir-bb<entry> ]
-; CHECK-NEXT: EMIT-SCALAR vp<%bc.merge.rdx> = phi [ vp<[[VP7]]>, middle.block ], [ ir<0>, ir-bb<entry> ]
+; CHECK-NEXT: EMIT-SCALAR vp<[[VP11:%[0-9]+]]> = phi [ vp<[[VP2]]>, middle.block ], [ ir<0>, ir-bb<entry> ]
+; CHECK-NEXT: EMIT-SCALAR vp<[[VP12:%[0-9]+]]> = phi [ vp<[[VP9]]>, middle.block ], [ ir<0>, ir-bb<entry> ]
+; CHECK-NEXT: EMIT-SCALAR vp<[[VP13:%[0-9]+]]> = phi [ vp<[[VP7]]>, middle.block ], [ ir<0>, ir-bb<entry> ]
; CHECK-NEXT: Successor(s): ir-bb<loop.header>
; CHECK-EMPTY:
; CHECK-NEXT: ir-bb<loop.header>:
diff --git a/llvm/test/Transforms/LoopVectorize/VPlan/vplan-print-after-all.ll b/llvm/test/Transforms/LoopVectorize/VPlan/vplan-print-after-all.ll
index fecf708368eab..3989caadbd792 100644
--- a/llvm/test/Transforms/LoopVectorize/VPlan/vplan-print-after-all.ll
+++ b/llvm/test/Transforms/LoopVectorize/VPlan/vplan-print-after-all.ll
@@ -4,10 +4,10 @@
; Verify that `-vplan-print-after-all` option works.
; CHECK: VPlan for loop in 'foo' after printAfterInitialConstruction
-; CHECK: VPlan for loop in 'foo' after VPlanTransforms::createHeaderPhiRecipes
; CHECK: VPlan for loop in 'foo' after VPlanTransforms::replaceSymbolicStrides
; CHECK: VPlan for loop in 'foo' after VPlanTransforms::simplifyRecipes
; CHECK: VPlan for loop in 'foo' after VPlanTransforms::removeDeadRecipes
+; CHECK: VPlan for loop in 'foo' after VPlanTransforms::createHeaderPhiRecipes
; CHECK: VPlan for loop in 'foo' after VPlanTransforms::handleEarlyExits
; CHECK: VPlan for loop in 'foo' after VPlanTransforms::addMiddleCheck
; CHECK: VPlan for loop in 'foo' after VPlanTransforms::createLoopRegions
diff --git a/llvm/test/Transforms/LoopVectorize/VPlan/vplan-printing.ll b/llvm/test/Transforms/LoopVectorize/VPlan/vplan-printing.ll
index 19d67ecc210d0..19154083be222 100644
--- a/llvm/test/Transforms/LoopVectorize/VPlan/vplan-printing.ll
+++ b/llvm/test/Transforms/LoopVectorize/VPlan/vplan-printing.ll
@@ -489,7 +489,7 @@ define void @print_expand_scev(i64 %y, ptr %ptr) {
; CHECK-NEXT: Successor(s): scalar.ph, vector.ph
; CHECK-EMPTY:
; CHECK-NEXT: vector.ph:
-; CHECK-NEXT: vp<[[VP5:%[0-9]+]]> = DERIVED-IV ir<0> + vp<[[VP2]]> * vp<[[VP4]]>
+; CHECK-NEXT: vp<[[VP5:%[0-9]+]]> = DERIVED-IV ir<0> + vp<[[VP2]]> * ir<%inc>
; CHECK-NEXT: Successor(s): vector loop
; CHECK-EMPTY:
; CHECK-NEXT: <x1> vector loop: {
@@ -497,8 +497,8 @@ define void @print_expand_scev(i64 %y, ptr %ptr) {
; CHECK-EMPTY:
; CHECK-NEXT: vector.body:
; CHECK-NEXT: ir<%iv> = WIDEN-INDUCTION ir<0>, vp<[[VP4]]>, vp<[[VP0]]> (truncated to i8)
-; CHECK-NEXT: vp<[[VP7:%[0-9]+]]> = DERIVED-IV ir<0> + vp<[[VP6]]> * vp<[[VP4]]>
-; CHECK-NEXT: vp<[[VP8:%[0-9]+]]> = SCALAR-STEPS vp<[[VP7]]>, vp<[[VP4]]>, vp<[[VP0]]>
+; CHECK-NEXT: vp<[[VP7:%[0-9]+]]> = DERIVED-IV ir<0> + vp<[[VP6]]> * ir<%inc>
+; CHECK-NEXT: vp<[[VP8:%[0-9]+]]> = SCALAR-STEPS vp<[[VP7]]>, ir<%inc>, vp<[[VP0]]>
; CHECK-NEXT: WIDEN ir<%v3> = add nuw ir<%iv>, ir<1>
; CHECK-NEXT: REPLICATE ir<%gep> = getelementptr inbounds ir<%ptr>, vp<[[VP8]]>
; CHECK-NEXT: REPLICATE store ir<%v3>, ir<%gep>
diff --git a/llvm/test/Transforms/LoopVectorize/pointer-induction.ll b/llvm/test/Transforms/LoopVectorize/pointer-induction.ll
index 0b312ea259349..a5462f2c78163 100644
--- a/llvm/test/Transforms/LoopVectorize/pointer-induction.ll
+++ b/llvm/test/Transforms/LoopVectorize/pointer-induction.ll
@@ -702,14 +702,12 @@ define void @strided_ptr_iv_runtime_stride(ptr %pIn, ptr %pOut, i32 %nCols, i32
; STRIDED-NEXT: entry:
; STRIDED-NEXT: [[PIN2:%.*]] = ptrtoaddr ptr [[PIN:%.*]] to i64
; STRIDED-NEXT: [[POUT1:%.*]] = ptrtoaddr ptr [[POUT:%.*]] to i64
-; STRIDED-NEXT: [[TMP1:%.*]] = sext i32 [[STRIDE:%.*]] to i64
-; STRIDED-NEXT: [[TMP2:%.*]] = shl nsw i64 [[TMP1]], 2
; STRIDED-NEXT: [[TMP10:%.*]] = zext i32 [[NCOLS:%.*]] to i64
; STRIDED-NEXT: [[UMAX:%.*]] = call i64 @llvm.umax.i64(i64 [[TMP10]], i64 1)
; STRIDED-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[UMAX]], 4
; STRIDED-NEXT: br i1 [[MIN_ITERS_CHECK]], label [[SCALAR_PH:%.*]], label [[VECTOR_SCEVCHECK:%.*]]
; STRIDED: vector.scevcheck:
-; STRIDED-NEXT: [[IDENT_CHECK:%.*]] = icmp ne i32 [[STRIDE]], 1
+; STRIDED-NEXT: [[IDENT_CHECK:%.*]] = icmp ne i32 [[STRIDE:%.*]], 1
; STRIDED-NEXT: br i1 [[IDENT_CHECK]], label [[SCALAR_PH]], label [[VECTOR_MEMCHECK:%.*]]
; STRIDED: vector.memcheck:
; STRIDED-NEXT: [[TMP3:%.*]] = sub i64 [[POUT1]], [[PIN2]]
>From ecab6643e8c7af403f6d650cb4e6d933256011f8 Mon Sep 17 00:00:00 2001
From: Florian Hahn <flo at fhahn.com>
Date: Wed, 10 Jun 2026 14:01:22 +0200
Subject: [PATCH 2/3] !fixup run replaceSymbolicStrides again
---
.../Transforms/Vectorize/LoopVectorize.cpp | 4 +++
.../Vectorize/VPlanConstruction.cpp | 31 ++++++-------------
.../VPlan/vplan-print-after-all.ll | 1 +
.../LoopVectorize/VPlan/vplan-printing.ll | 6 ++--
.../LoopVectorize/pointer-induction.ll | 4 ++-
5 files changed, 20 insertions(+), 26 deletions(-)
diff --git a/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp b/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
index 6556d4b1b0296..596ac89a206a7 100644
--- a/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
+++ b/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
@@ -6520,6 +6520,10 @@ VPlanPtr LoopVectorizationPlanner::tryToBuildVPlan1() {
return nullptr;
}
+ if (const LoopAccessInfo *LAI = Legal->getLAI())
+ RUN_VPLAN_PASS(VPlanTransforms::replaceSymbolicStrides, *VPlan0, PSE,
+ LAI->getSymbolicStrides());
+
// Add surviving induction predicates to PSE and check constraints.
bool ForceVectorization = Hints.getForce() == LoopVectorizeHints::FK_Enabled;
bool OptForSize =
diff --git a/llvm/lib/Transforms/Vectorize/VPlanConstruction.cpp b/llvm/lib/Transforms/Vectorize/VPlanConstruction.cpp
index db373dd3e5395..b98aa7e4cf79d 100644
--- a/llvm/lib/Transforms/Vectorize/VPlanConstruction.cpp
+++ b/llvm/lib/Transforms/Vectorize/VPlanConstruction.cpp
@@ -644,28 +644,10 @@ createWidenInductionRecipe(PHINode *Phi, VPPhi *PhiR, VPIRValue *Start,
vputils::getSCEVExprForVPValue(Start, PSE))) &&
"Start VPValue must match IndDesc's start value");
+ VPValue *Step =
+ vputils::getOrCreateVPValueForSCEVExpr(Plan, IndDesc.getStep());
+
VPValue *BackedgeVal = PhiR->getOperand(1);
- // Try to get the versioned step from the induction increment directly.
- auto GetStep = [&]() -> VPValue * {
- VPValue *Step;
- if (match(BackedgeVal, m_Add(m_Specific(PhiR), m_VPValue(Step))))
- return Step;
-
- // Pointer induction: scale the GEP index by the source element size.
- Type *GEPSrcElemTy;
- ArrayRef<VPValue *> GEPOps;
- if (match(BackedgeVal, m_GetElementPtr(GEPSrcElemTy, GEPOps)) &&
- GEPOps.size() == 2 && GEPOps[0] == PhiR) {
- Type *StepTy = IndDesc.getStep()->getType();
- const SCEV *Idx = vputils::getSCEVExprForVPValue(GEPOps[1], PSE);
- if (!isa<SCEVCouldNotCompute>(Idx))
- return vputils::getOrCreateVPValueForSCEVExpr(
- Plan, SE.getMulExpr(SE.getTruncateOrSignExtend(Idx, StepTy),
- SE.getSizeOfExpr(StepTy, GEPSrcElemTy)));
- }
- return vputils::getOrCreateVPValueForSCEVExpr(Plan, IndDesc.getStep());
- };
- VPValue *Step = GetStep();
// Replace live-out extracts of WideIV's backedge value by ExitingIVValue
// recipes. optimizeInductionLiveOutUsers will later compute the proper
@@ -702,6 +684,12 @@ createWidenInductionRecipe(PHINode *Phi, VPPhi *PhiR, VPIRValue *Start,
IndDesc.getKind() == InductionDescriptor::IK_FpInduction) &&
"must have an integer or float induction at this point");
+ // Update wide induction increments to use the same step as the corresponding
+ // wide induction. This enables detecting induction increments directly in
+ // VPlan and removes redundant splats.
+ if (match(BackedgeVal, m_Add(m_Specific(PhiR), m_VPValue())))
+ BackedgeVal->getDefiningRecipe()->setOperand(1, Step);
+
// It is always safe to copy over the NoWrap and FastMath flags. In
// particular, when folding tail by masking, the masked-off lanes are never
// used, so it is safe.
@@ -967,7 +955,6 @@ bool VPlanTransforms::createHeaderPhiRecipes(
RdxDesc.hasUsesOutsideReductionChain());
};
- SmallVector<VPValue *> BackedgeVals;
for (VPRecipeBase &R : make_early_inc_range(HeaderVPBB->phis())) {
auto *PhiR = cast<VPPhi>(&R);
VPHeaderPHIRecipe *HeaderPhiR = CreateHeaderPhiRecipe(PhiR);
diff --git a/llvm/test/Transforms/LoopVectorize/VPlan/vplan-print-after-all.ll b/llvm/test/Transforms/LoopVectorize/VPlan/vplan-print-after-all.ll
index 2df25d874bd4a..0610af3710f37 100644
--- a/llvm/test/Transforms/LoopVectorize/VPlan/vplan-print-after-all.ll
+++ b/llvm/test/Transforms/LoopVectorize/VPlan/vplan-print-after-all.ll
@@ -8,6 +8,7 @@
; CHECK: VPlan for loop in 'foo' after VPlanTransforms::simplifyRecipes
; CHECK: VPlan for loop in 'foo' after VPlanTransforms::removeDeadRecipes
; CHECK: VPlan for loop in 'foo' after VPlanTransforms::createHeaderPhiRecipes
+; CHECK: VPlan for loop in 'foo' after VPlanTransforms::replaceSymbolicStrides
; CHECK: VPlan for loop in 'foo' after VPlanTransforms::finalizeSCEVPredicates
; CHECK: VPlan for loop in 'foo' after VPlanTransforms::handleEarlyExits
; CHECK: VPlan for loop in 'foo' after VPlanTransforms::addMiddleCheck
diff --git a/llvm/test/Transforms/LoopVectorize/VPlan/vplan-printing.ll b/llvm/test/Transforms/LoopVectorize/VPlan/vplan-printing.ll
index 327a126c45590..7514250c82627 100644
--- a/llvm/test/Transforms/LoopVectorize/VPlan/vplan-printing.ll
+++ b/llvm/test/Transforms/LoopVectorize/VPlan/vplan-printing.ll
@@ -489,7 +489,7 @@ define void @print_expand_scev(i64 %y, ptr %ptr) {
; CHECK-NEXT: Successor(s): scalar.ph, vector.ph
; CHECK-EMPTY:
; CHECK-NEXT: vector.ph:
-; CHECK-NEXT: vp<[[VP5:%[0-9]+]]> = DERIVED-IV ir<0> + vp<[[VP2]]> * ir<%inc>
+; CHECK-NEXT: vp<[[VP5:%[0-9]+]]> = DERIVED-IV ir<0> + vp<[[VP2]]> * vp<[[VP4]]>
; CHECK-NEXT: Successor(s): vector loop
; CHECK-EMPTY:
; CHECK-NEXT: <x1> vector loop: {
@@ -497,8 +497,8 @@ define void @print_expand_scev(i64 %y, ptr %ptr) {
; CHECK-EMPTY:
; CHECK-NEXT: vector.body:
; CHECK-NEXT: ir<%iv> = WIDEN-INDUCTION ir<0>, vp<[[VP4]]>, vp<[[VP0]]> (truncated to i8)
-; CHECK-NEXT: vp<[[VP7:%[0-9]+]]> = DERIVED-IV ir<0> + vp<[[VP6]]> * ir<%inc>
-; CHECK-NEXT: vp<[[VP8:%[0-9]+]]> = SCALAR-STEPS vp<[[VP7]]>, ir<%inc>, vp<[[VP0]]>
+; CHECK-NEXT: vp<[[VP7:%[0-9]+]]> = DERIVED-IV ir<0> + vp<[[VP6]]> * vp<[[VP4]]>
+; CHECK-NEXT: vp<[[VP8:%[0-9]+]]> = SCALAR-STEPS vp<[[VP7]]>, vp<[[VP4]]>, vp<[[VP0]]>
; CHECK-NEXT: WIDEN ir<%v3> = add nuw ir<%iv>, ir<1>
; CHECK-NEXT: REPLICATE ir<%gep> = getelementptr inbounds ir<%ptr>, vp<[[VP8]]>
; CHECK-NEXT: REPLICATE store ir<%v3>, ir<%gep>
diff --git a/llvm/test/Transforms/LoopVectorize/pointer-induction.ll b/llvm/test/Transforms/LoopVectorize/pointer-induction.ll
index d40f2388881e2..760d8e4d1a465 100644
--- a/llvm/test/Transforms/LoopVectorize/pointer-induction.ll
+++ b/llvm/test/Transforms/LoopVectorize/pointer-induction.ll
@@ -702,12 +702,14 @@ define void @strided_ptr_iv_runtime_stride(ptr %pIn, ptr %pOut, i32 %nCols, i32
; STRIDED-NEXT: entry:
; STRIDED-NEXT: [[PIN2:%.*]] = ptrtoaddr ptr [[PIN:%.*]] to i64
; STRIDED-NEXT: [[POUT1:%.*]] = ptrtoaddr ptr [[POUT:%.*]] to i64
+; STRIDED-NEXT: [[TMP1:%.*]] = sext i32 [[STRIDE:%.*]] to i64
+; STRIDED-NEXT: [[TMP2:%.*]] = shl nsw i64 [[TMP1]], 2
; STRIDED-NEXT: [[TMP10:%.*]] = zext i32 [[NCOLS:%.*]] to i64
; STRIDED-NEXT: [[UMAX:%.*]] = call i64 @llvm.umax.i64(i64 [[TMP10]], i64 1)
; STRIDED-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[UMAX]], 4
; STRIDED-NEXT: br i1 [[MIN_ITERS_CHECK]], label [[SCALAR_PH:%.*]], label [[VECTOR_SCEVCHECK:%.*]]
; STRIDED: vector.scevcheck:
-; STRIDED-NEXT: [[IDENT_CHECK:%.*]] = icmp ne i32 [[STRIDE:%.*]], 1
+; STRIDED-NEXT: [[IDENT_CHECK:%.*]] = icmp ne i32 [[STRIDE]], 1
; STRIDED-NEXT: br i1 [[IDENT_CHECK]], label [[SCALAR_PH]], label [[VECTOR_MEMCHECK:%.*]]
; STRIDED: vector.memcheck:
; STRIDED-NEXT: [[TMP3:%.*]] = sub i64 [[POUT1]], [[PIN2]]
>From dd808755f282ff485c8eafd4feb4ea7182e9290d Mon Sep 17 00:00:00 2001
From: Florian Hahn <flo at fhahn.com>
Date: Wed, 10 Jun 2026 17:02:34 +0200
Subject: [PATCH 3/3] !fixup remove some unneeded changes, add comment, thanks
---
llvm/lib/Transforms/Vectorize/VPlanConstruction.cpp | 4 ++--
1 file changed, 2 insertions(+), 2 deletions(-)
diff --git a/llvm/lib/Transforms/Vectorize/VPlanConstruction.cpp b/llvm/lib/Transforms/Vectorize/VPlanConstruction.cpp
index b98aa7e4cf79d..b0a92c795798f 100644
--- a/llvm/lib/Transforms/Vectorize/VPlanConstruction.cpp
+++ b/llvm/lib/Transforms/Vectorize/VPlanConstruction.cpp
@@ -635,7 +635,7 @@ createWidenInductionRecipe(PHINode *Phi, VPPhi *PhiR, VPIRValue *Start,
const InductionDescriptor &IndDesc, VPlan &Plan,
PredicatedScalarEvolution &PSE, Loop &OrigLoop,
DebugLoc DL) {
- ScalarEvolution &SE = *PSE.getSE();
+ [[maybe_unused] ScalarEvolution &SE = *PSE.getSE();
assert(SE.isLoopInvariant(IndDesc.getStep(), &OrigLoop) &&
"step must be loop invariant");
assert((Plan.getLiveIn(IndDesc.getStartValue()) == Start ||
@@ -648,7 +648,6 @@ createWidenInductionRecipe(PHINode *Phi, VPPhi *PhiR, VPIRValue *Start,
vputils::getOrCreateVPValueForSCEVExpr(Plan, IndDesc.getStep());
VPValue *BackedgeVal = PhiR->getOperand(1);
-
// Replace live-out extracts of WideIV's backedge value by ExitingIVValue
// recipes. optimizeInductionLiveOutUsers will later compute the proper
// DerivedIV.
@@ -966,6 +965,7 @@ bool VPlanTransforms::createHeaderPhiRecipes(
if (!tryToSinkOrHoistRecurrenceUsers(HeaderVPBB, VPDT))
return false;
+ // Skip renaming resume phi recipes, if any header phi as been removed.
if (range_size(HeaderVPBB->phis()) !=
range_size(Plan.getScalarPreheader()->phis()))
return true;
More information about the llvm-commits
mailing list