[llvm] [VPlan] Use ExitingIVValue for pointer inductions as well. (PR #180925)
Florian Hahn via llvm-commits
llvm-commits at lists.llvm.org
Fri Feb 13 05:12:56 PST 2026
https://github.com/fhahn updated https://github.com/llvm/llvm-project/pull/180925
>From 6a0afc517c77ec7dbfd14c9ceac3ea5e56a4f0fe Mon Sep 17 00:00:00 2001
From: Florian Hahn <flo at fhahn.com>
Date: Sat, 7 Feb 2026 21:09:06 +0000
Subject: [PATCH 1/2] [VPlan] Use ExitingIVValue for pointer inductions as
well.
Use ExitingIVValue from https://github.com/llvm/llvm-project/pull/175651
also for pointer inductions.
---
.../Vectorize/VPlanConstruction.cpp | 61 +++++++++++--------
.../LoopVectorize/pointer-induction.ll | 13 +---
2 files changed, 35 insertions(+), 39 deletions(-)
diff --git a/llvm/lib/Transforms/Vectorize/VPlanConstruction.cpp b/llvm/lib/Transforms/Vectorize/VPlanConstruction.cpp
index 9b4ae56e7f175..94bf0d5664b7b 100644
--- a/llvm/lib/Transforms/Vectorize/VPlanConstruction.cpp
+++ b/llvm/lib/Transforms/Vectorize/VPlanConstruction.cpp
@@ -624,9 +624,37 @@ createWidenInductionRecipe(PHINode *Phi, VPPhi *PhiR, VPIRValue *Start,
VPValue *Step =
vputils::getOrCreateVPValueForSCEVExpr(Plan, IndDesc.getStep());
- if (IndDesc.getKind() == InductionDescriptor::IK_PtrInduction)
- return new VPWidenPointerInductionRecipe(Phi, Start, Step, &Plan.getVFxUF(),
- IndDesc, DL);
+ VPValue *BackedgeVal = PhiR->getOperand(1);
+ // Replace live-out extracts of WideIV's backedge value by ExitingIVValue
+ // recipes. optimizeInductionExitUsers will later compute the proper
+ // DerivedIV.
+ auto ReplaceExtractsWithExitingIVValue = [&](VPHeaderPHIRecipe *WideIV) {
+ for (VPUser *U : to_vector(BackedgeVal->users())) {
+ if (!match(U, m_ExtractLastPart(m_VPValue())))
+ continue;
+ auto *ExtractLastPart = cast<VPInstruction>(U);
+ if (!match(ExtractLastPart->getSingleUser(),
+ m_ExtractLastLane(m_VPValue())))
+ continue;
+ auto *ExtractLastLane =
+ cast<VPInstruction>(ExtractLastPart->getSingleUser());
+ assert(is_contained(ExtractLastLane->getParent()->successors(),
+ Plan.getScalarPreheader()) &&
+ "last lane must be extracted in the middle block");
+ VPBuilder Builder(ExtractLastLane);
+ ExtractLastLane->replaceAllUsesWith(Builder.createNaryOp(
+ VPInstruction::ExitingIVValue, {WideIV, BackedgeVal}));
+ ExtractLastLane->eraseFromParent();
+ ExtractLastPart->eraseFromParent();
+ }
+ };
+
+ if (IndDesc.getKind() == InductionDescriptor::IK_PtrInduction) {
+ auto *WideIV = new VPWidenPointerInductionRecipe(
+ Phi, Start, Step, &Plan.getVFxUF(), IndDesc, DL);
+ ReplaceExtractsWithExitingIVValue(WideIV);
+ return WideIV;
+ }
assert((IndDesc.getKind() == InductionDescriptor::IK_IntInduction ||
IndDesc.getKind() == InductionDescriptor::IK_FpInduction) &&
@@ -635,9 +663,8 @@ createWidenInductionRecipe(PHINode *Phi, VPPhi *PhiR, VPIRValue *Start,
// Update wide induction increments to use the same step as the corresponding
// wide induction. This enables detecting induction increments directly in
// VPlan and removes redundant splats.
- using namespace llvm::VPlanPatternMatch;
- if (match(PhiR->getOperand(1), m_Add(m_Specific(PhiR), m_VPValue())))
- PhiR->getOperand(1)->getDefiningRecipe()->setOperand(1, Step);
+ if (match(BackedgeVal, m_Add(m_Specific(PhiR), m_VPValue())))
+ BackedgeVal->getDefiningRecipe()->setOperand(1, Step);
// It is always safe to copy over the NoWrap and FastMath flags. In
// particular, when folding tail by masking, the masked-off lanes are never
@@ -647,27 +674,7 @@ createWidenInductionRecipe(PHINode *Phi, VPPhi *PhiR, VPIRValue *Start,
auto *WideIV = new VPWidenIntOrFpInductionRecipe(
Phi, Start, Step, &Plan.getVF(), IndDesc, Flags, DL);
- // Replace live-out extracts of WideIV's backedge value by ExitingIVValue
- // recipes.
- VPValue *BackedgeVal = PhiR->getOperand(1);
- for (VPUser *U : to_vector(BackedgeVal->users())) {
- if (!match(U, m_ExtractLastPart(m_VPValue())))
- continue;
- auto *ExtractLastPart = cast<VPInstruction>(U);
- if (!match(ExtractLastPart->getSingleUser(),
- m_ExtractLastLane(m_VPValue())))
- continue;
- auto *ExtractLastLane =
- cast<VPInstruction>(ExtractLastPart->getSingleUser());
- assert(is_contained(ExtractLastLane->getParent()->successors(),
- Plan.getScalarPreheader()) &&
- "last lane must be extracted in the middle block");
- VPBuilder Builder(ExtractLastLane);
- ExtractLastLane->replaceAllUsesWith(Builder.createNaryOp(
- VPInstruction::ExitingIVValue, {WideIV, BackedgeVal}));
- ExtractLastLane->eraseFromParent();
- ExtractLastPart->eraseFromParent();
- }
+ ReplaceExtractsWithExitingIVValue(WideIV);
return WideIV;
}
diff --git a/llvm/test/Transforms/LoopVectorize/pointer-induction.ll b/llvm/test/Transforms/LoopVectorize/pointer-induction.ll
index 995c2016339a4..82a92e51291b0 100644
--- a/llvm/test/Transforms/LoopVectorize/pointer-induction.ll
+++ b/llvm/test/Transforms/LoopVectorize/pointer-induction.ll
@@ -642,19 +642,8 @@ define i64 @ivopt_widen_ptr_indvar_3(ptr noalias %a, i64 %stride, i64 %n) {
; STRIDED-NEXT: br label [[VECTOR_BODY:%.*]]
; STRIDED: vector.body:
; STRIDED-NEXT: [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ]
-; STRIDED-NEXT: [[OFFSET_IDX:%.*]] = mul i64 [[INDEX]], [[TMP1]]
-; STRIDED-NEXT: [[TMP4:%.*]] = mul i64 0, [[TMP1]]
-; STRIDED-NEXT: [[TMP5:%.*]] = add i64 [[OFFSET_IDX]], [[TMP4]]
-; STRIDED-NEXT: [[TMP6:%.*]] = mul i64 1, [[TMP1]]
-; STRIDED-NEXT: [[TMP7:%.*]] = add i64 [[OFFSET_IDX]], [[TMP6]]
-; STRIDED-NEXT: [[TMP8:%.*]] = mul i64 2, [[TMP1]]
-; STRIDED-NEXT: [[TMP9:%.*]] = add i64 [[OFFSET_IDX]], [[TMP8]]
-; STRIDED-NEXT: [[TMP10:%.*]] = mul i64 3, [[TMP1]]
-; STRIDED-NEXT: [[TMP11:%.*]] = add i64 [[OFFSET_IDX]], [[TMP10]]
-; STRIDED-NEXT: [[NEXT_GEP3:%.*]] = getelementptr i8, ptr null, i64 [[TMP11]]
; STRIDED-NEXT: [[TMP12:%.*]] = getelementptr i64, ptr [[A:%.*]], i64 [[INDEX]]
; STRIDED-NEXT: [[WIDE_LOAD:%.*]] = load <4 x i64>, ptr [[TMP12]], align 8
-; STRIDED-NEXT: [[TMP14:%.*]] = getelementptr i64, ptr [[NEXT_GEP3]], i64 [[STRIDE]]
; STRIDED-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 4
; STRIDED-NEXT: [[TMP15:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
; STRIDED-NEXT: br i1 [[TMP15]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP14:![0-9]+]]
@@ -677,7 +666,7 @@ define i64 @ivopt_widen_ptr_indvar_3(ptr noalias %a, i64 %stride, i64 %n) {
; STRIDED-NEXT: br i1 [[EXITCOND_NOT]], label [[EXIT]], label [[FOR_BODY]], !llvm.loop [[LOOP15:![0-9]+]]
; STRIDED: exit:
; STRIDED-NEXT: [[DOTLCSSA:%.*]] = phi i64 [ [[TMP17]], [[FOR_BODY]] ], [ [[TMP16]], [[MIDDLE_BLOCK]] ]
-; STRIDED-NEXT: [[PTR_IV_NEXT_LCSSA:%.*]] = phi ptr [ [[PTR_IV_NEXT]], [[FOR_BODY]] ], [ [[TMP14]], [[MIDDLE_BLOCK]] ]
+; STRIDED-NEXT: [[PTR_IV_NEXT_LCSSA:%.*]] = phi ptr [ [[PTR_IV_NEXT]], [[FOR_BODY]] ], [ [[TMP3]], [[MIDDLE_BLOCK]] ]
; STRIDED-NEXT: [[CAST_PTR:%.*]] = ptrtoint ptr [[PTR_IV_NEXT_LCSSA]] to i64
; STRIDED-NEXT: [[RESULT:%.*]] = add i64 [[CAST_PTR]], [[DOTLCSSA]]
; STRIDED-NEXT: ret i64 [[RESULT]]
>From 07e05e030d44f2d9ec7d8abfd52dd4b422367a35 Mon Sep 17 00:00:00 2001
From: Florian Hahn <flo at fhahn.com>
Date: Fri, 13 Feb 2026 13:03:28 +0000
Subject: [PATCH 2/2] !fixup add assert, thanks
---
llvm/lib/Transforms/Vectorize/VPlanConstruction.cpp | 8 ++++----
1 file changed, 4 insertions(+), 4 deletions(-)
diff --git a/llvm/lib/Transforms/Vectorize/VPlanConstruction.cpp b/llvm/lib/Transforms/Vectorize/VPlanConstruction.cpp
index 94bf0d5664b7b..1accec8743d01 100644
--- a/llvm/lib/Transforms/Vectorize/VPlanConstruction.cpp
+++ b/llvm/lib/Transforms/Vectorize/VPlanConstruction.cpp
@@ -633,11 +633,11 @@ createWidenInductionRecipe(PHINode *Phi, VPPhi *PhiR, VPIRValue *Start,
if (!match(U, m_ExtractLastPart(m_VPValue())))
continue;
auto *ExtractLastPart = cast<VPInstruction>(U);
- if (!match(ExtractLastPart->getSingleUser(),
- m_ExtractLastLane(m_VPValue())))
+ VPUser *ExtractLastPartUser = ExtractLastPart->getSingleUser();
+ assert(ExtractLastPartUser && "must have a single user");
+ if (!match(ExtractLastPartUser, m_ExtractLastLane(m_VPValue())))
continue;
- auto *ExtractLastLane =
- cast<VPInstruction>(ExtractLastPart->getSingleUser());
+ auto *ExtractLastLane = cast<VPInstruction>(ExtractLastPartUser);
assert(is_contained(ExtractLastLane->getParent()->successors(),
Plan.getScalarPreheader()) &&
"last lane must be extracted in the middle block");
More information about the llvm-commits
mailing list