[llvm] d154ec7 - [VPlan] Use ExitingIVValue for pointer inductions as well. (#180925)
via llvm-commits
llvm-commits at lists.llvm.org
Wed Feb 18 04:47:11 PST 2026
Author: Florian Hahn
Date: 2026-02-18T12:47:06Z
New Revision: d154ec7bb02a7b6b84c4481a58495baf08542541
URL: https://github.com/llvm/llvm-project/commit/d154ec7bb02a7b6b84c4481a58495baf08542541
DIFF: https://github.com/llvm/llvm-project/commit/d154ec7bb02a7b6b84c4481a58495baf08542541.diff
LOG: [VPlan] Use ExitingIVValue for pointer inductions as well. (#180925)
Use ExitingIVValue from https://github.com/llvm/llvm-project/pull/175651
also for pointer inductions.
PR: https://github.com/llvm/llvm-project/pull/180925
Added:
Modified:
llvm/lib/Transforms/Vectorize/VPlanConstruction.cpp
llvm/test/Transforms/LoopVectorize/pointer-induction.ll
Removed:
################################################################################
diff --git a/llvm/lib/Transforms/Vectorize/VPlanConstruction.cpp b/llvm/lib/Transforms/Vectorize/VPlanConstruction.cpp
index 80847fcbb77fb..e56691b1c960e 100644
--- a/llvm/lib/Transforms/Vectorize/VPlanConstruction.cpp
+++ b/llvm/lib/Transforms/Vectorize/VPlanConstruction.cpp
@@ -623,9 +623,37 @@ createWidenInductionRecipe(PHINode *Phi, VPPhi *PhiR, VPIRValue *Start,
VPValue *Step =
vputils::getOrCreateVPValueForSCEVExpr(Plan, IndDesc.getStep());
- if (IndDesc.getKind() == InductionDescriptor::IK_PtrInduction)
- return new VPWidenPointerInductionRecipe(Phi, Start, Step, &Plan.getVFxUF(),
- IndDesc, DL);
+ VPValue *BackedgeVal = PhiR->getOperand(1);
+ // Replace live-out extracts of WideIV's backedge value by ExitingIVValue
+ // recipes. optimizeInductionExitUsers will later compute the proper
+ // DerivedIV.
+ auto ReplaceExtractsWithExitingIVValue = [&](VPHeaderPHIRecipe *WideIV) {
+ for (VPUser *U : to_vector(BackedgeVal->users())) {
+ if (!match(U, m_ExtractLastPart(m_VPValue())))
+ continue;
+ auto *ExtractLastPart = cast<VPInstruction>(U);
+ VPUser *ExtractLastPartUser = ExtractLastPart->getSingleUser();
+ assert(ExtractLastPartUser && "must have a single user");
+ if (!match(ExtractLastPartUser, m_ExtractLastLane(m_VPValue())))
+ continue;
+ auto *ExtractLastLane = cast<VPInstruction>(ExtractLastPartUser);
+ assert(is_contained(ExtractLastLane->getParent()->successors(),
+ Plan.getScalarPreheader()) &&
+ "last lane must be extracted in the middle block");
+ VPBuilder Builder(ExtractLastLane);
+ ExtractLastLane->replaceAllUsesWith(Builder.createNaryOp(
+ VPInstruction::ExitingIVValue, {WideIV, BackedgeVal}));
+ ExtractLastLane->eraseFromParent();
+ ExtractLastPart->eraseFromParent();
+ }
+ };
+
+ if (IndDesc.getKind() == InductionDescriptor::IK_PtrInduction) {
+ auto *WideIV = new VPWidenPointerInductionRecipe(
+ Phi, Start, Step, &Plan.getVFxUF(), IndDesc, DL);
+ ReplaceExtractsWithExitingIVValue(WideIV);
+ return WideIV;
+ }
assert((IndDesc.getKind() == InductionDescriptor::IK_IntInduction ||
IndDesc.getKind() == InductionDescriptor::IK_FpInduction) &&
@@ -634,9 +662,8 @@ createWidenInductionRecipe(PHINode *Phi, VPPhi *PhiR, VPIRValue *Start,
// Update wide induction increments to use the same step as the corresponding
// wide induction. This enables detecting induction increments directly in
// VPlan and removes redundant splats.
- using namespace llvm::VPlanPatternMatch;
- if (match(PhiR->getOperand(1), m_Add(m_Specific(PhiR), m_VPValue())))
- PhiR->getOperand(1)->getDefiningRecipe()->setOperand(1, Step);
+ if (match(BackedgeVal, m_Add(m_Specific(PhiR), m_VPValue())))
+ BackedgeVal->getDefiningRecipe()->setOperand(1, Step);
// It is always safe to copy over the NoWrap and FastMath flags. In
// particular, when folding tail by masking, the masked-off lanes are never
@@ -646,27 +673,7 @@ createWidenInductionRecipe(PHINode *Phi, VPPhi *PhiR, VPIRValue *Start,
auto *WideIV = new VPWidenIntOrFpInductionRecipe(
Phi, Start, Step, &Plan.getVF(), IndDesc, Flags, DL);
- // Replace live-out extracts of WideIV's backedge value by ExitingIVValue
- // recipes.
- VPValue *BackedgeVal = PhiR->getOperand(1);
- for (VPUser *U : to_vector(BackedgeVal->users())) {
- if (!match(U, m_ExtractLastPart(m_VPValue())))
- continue;
- auto *ExtractLastPart = cast<VPInstruction>(U);
- if (!match(ExtractLastPart->getSingleUser(),
- m_ExtractLastLane(m_VPValue())))
- continue;
- auto *ExtractLastLane =
- cast<VPInstruction>(ExtractLastPart->getSingleUser());
- assert(is_contained(ExtractLastLane->getParent()->successors(),
- Plan.getScalarPreheader()) &&
- "last lane must be extracted in the middle block");
- VPBuilder Builder(ExtractLastLane);
- ExtractLastLane->replaceAllUsesWith(Builder.createNaryOp(
- VPInstruction::ExitingIVValue, {WideIV, BackedgeVal}));
- ExtractLastLane->eraseFromParent();
- ExtractLastPart->eraseFromParent();
- }
+ ReplaceExtractsWithExitingIVValue(WideIV);
return WideIV;
}
diff --git a/llvm/test/Transforms/LoopVectorize/pointer-induction.ll b/llvm/test/Transforms/LoopVectorize/pointer-induction.ll
index fe0c839f3f030..6c3e7575c6c32 100644
--- a/llvm/test/Transforms/LoopVectorize/pointer-induction.ll
+++ b/llvm/test/Transforms/LoopVectorize/pointer-induction.ll
@@ -641,19 +641,8 @@ define i64 @ivopt_widen_ptr_indvar_3(ptr noalias %a, i64 %stride, i64 %n) {
; STRIDED-NEXT: br label [[VECTOR_BODY:%.*]]
; STRIDED: vector.body:
; STRIDED-NEXT: [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ]
-; STRIDED-NEXT: [[OFFSET_IDX:%.*]] = mul i64 [[INDEX]], [[TMP1]]
-; STRIDED-NEXT: [[TMP4:%.*]] = mul i64 0, [[TMP1]]
-; STRIDED-NEXT: [[TMP5:%.*]] = add i64 [[OFFSET_IDX]], [[TMP4]]
-; STRIDED-NEXT: [[TMP6:%.*]] = mul i64 1, [[TMP1]]
-; STRIDED-NEXT: [[TMP7:%.*]] = add i64 [[OFFSET_IDX]], [[TMP6]]
-; STRIDED-NEXT: [[TMP8:%.*]] = mul i64 2, [[TMP1]]
-; STRIDED-NEXT: [[TMP9:%.*]] = add i64 [[OFFSET_IDX]], [[TMP8]]
-; STRIDED-NEXT: [[TMP10:%.*]] = mul i64 3, [[TMP1]]
-; STRIDED-NEXT: [[TMP11:%.*]] = add i64 [[OFFSET_IDX]], [[TMP10]]
-; STRIDED-NEXT: [[NEXT_GEP3:%.*]] = getelementptr i8, ptr null, i64 [[TMP11]]
; STRIDED-NEXT: [[TMP12:%.*]] = getelementptr i64, ptr [[A:%.*]], i64 [[INDEX]]
; STRIDED-NEXT: [[WIDE_LOAD:%.*]] = load <4 x i64>, ptr [[TMP12]], align 8
-; STRIDED-NEXT: [[TMP14:%.*]] = getelementptr i64, ptr [[NEXT_GEP3]], i64 [[STRIDE]]
; STRIDED-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 4
; STRIDED-NEXT: [[TMP15:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
; STRIDED-NEXT: br i1 [[TMP15]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP14:![0-9]+]]
@@ -676,7 +665,7 @@ define i64 @ivopt_widen_ptr_indvar_3(ptr noalias %a, i64 %stride, i64 %n) {
; STRIDED-NEXT: br i1 [[EXITCOND_NOT]], label [[EXIT]], label [[FOR_BODY]], !llvm.loop [[LOOP15:![0-9]+]]
; STRIDED: exit:
; STRIDED-NEXT: [[DOTLCSSA:%.*]] = phi i64 [ [[TMP17]], [[FOR_BODY]] ], [ [[TMP16]], [[MIDDLE_BLOCK]] ]
-; STRIDED-NEXT: [[PTR_IV_NEXT_LCSSA:%.*]] = phi ptr [ [[PTR_IV_NEXT]], [[FOR_BODY]] ], [ [[TMP14]], [[MIDDLE_BLOCK]] ]
+; STRIDED-NEXT: [[PTR_IV_NEXT_LCSSA:%.*]] = phi ptr [ [[PTR_IV_NEXT]], [[FOR_BODY]] ], [ [[TMP3]], [[MIDDLE_BLOCK]] ]
; STRIDED-NEXT: [[CAST_PTR:%.*]] = ptrtoint ptr [[PTR_IV_NEXT_LCSSA]] to i64
; STRIDED-NEXT: [[RESULT:%.*]] = add i64 [[CAST_PTR]], [[DOTLCSSA]]
; STRIDED-NEXT: ret i64 [[RESULT]]
More information about the llvm-commits
mailing list