[llvm] [LV] Use SCEV to compute final value of complex induction variables (PR #195059)
Mel Chen via llvm-commits
llvm-commits at lists.llvm.org
Thu Jun 11 00:44:38 PDT 2026
https://github.com/Mel-Chen updated https://github.com/llvm/llvm-project/pull/195059
>From 28957bde02adb70cdb4003ffb0daf4e82ffe5186 Mon Sep 17 00:00:00 2001
From: Mel Chen <mel.chen at sifive.com>
Date: Mon, 23 Mar 2026 20:49:03 -0700
Subject: [PATCH 1/5] [First candidate] Use SCEV to compute final value of
complex IV
---
.../Transforms/Vectorize/LoopVectorize.cpp | 4 +-
.../Transforms/Vectorize/VPlanTransforms.cpp | 106 ++++++++++-------
.../Transforms/Vectorize/VPlanTransforms.h | 2 +-
.../LoopVectorize/AArch64/reduction-cost.ll | 3 -
.../LoopVectorize/VPlan/vplan-printing.ll | 6 +-
.../LoopVectorize/X86/gep-use-outside-loop.ll | 2 +-
...6-sunk-instruction-used-outside-of-loop.ll | 15 +--
.../instruction-only-used-outside-of-loop.ll | 5 +-
.../LoopVectorize/iv_outside_user.ll | 109 ++++++------------
.../LoopVectorize/no_outside_user.ll | 5 +-
10 files changed, 120 insertions(+), 137 deletions(-)
diff --git a/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp b/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
index f9b325a5233e6..d90988c5cadeb 100644
--- a/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
+++ b/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
@@ -6625,7 +6625,7 @@ VPlanPtr LoopVectorizationPlanner::tryToBuildVPlan(VPlanPtr Plan,
*Plan, *TLI))
return nullptr;
RUN_VPLAN_PASS(VPlanTransforms::optimizeInductionLiveOutUsers, *Plan, PSE,
- /*FoldTail=*/false);
+ OrigLoop, /*FoldTail=*/false);
return Plan;
}
@@ -6795,7 +6795,7 @@ VPlanPtr LoopVectorizationPlanner::tryToBuildVPlan(VPlanPtr Plan,
RUN_VPLAN_PASS(VPlanTransforms::optimizeFindIVReductions, *Plan, PSE,
*OrigLoop);
RUN_VPLAN_PASS(VPlanTransforms::optimizeInductionLiveOutUsers, *Plan, PSE,
- CM.foldTailByMasking());
+ OrigLoop, CM.foldTailByMasking());
// Apply mandatory transformation to handle reductions with multiple in-loop
// uses if possible, bail out otherwise.
diff --git a/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp b/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp
index 3fe8c84658c8f..c278bcbf2146e 100644
--- a/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp
+++ b/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp
@@ -1098,54 +1098,82 @@ static VPValue *tryToComputeEndValueForInduction(VPWidenInductionRecipe *WideIV,
/// Attempts to optimize the induction variable exit values for users in the
/// exit block coming from the latch in the original scalar loop.
-static VPValue *
-optimizeLatchExitInductionUser(VPlan &Plan, VPValue *Op,
- DenseMap<VPValue *, VPValue *> &EndValues,
- PredicatedScalarEvolution &PSE) {
+static VPValue *optimizeLatchExitInductionUser(
+ VPlan &Plan, VPValue *Op, DenseMap<VPValue *, VPValue *> &EndValues,
+ PredicatedScalarEvolution &PSE, VPValue *ResumeTC, const Loop *L) {
VPValue *Incoming;
if (!match(Op, m_ExtractLastLaneOfLastPart(m_VPValue(Incoming))))
return nullptr;
- VPWidenInductionRecipe *WideIV = getOptimizableIVOf(Incoming, PSE);
- if (!WideIV)
- return nullptr;
+ auto *ExtractR = cast<VPInstruction>(Op);
+ if (VPWidenInductionRecipe *WideIV = getOptimizableIVOf(Incoming, PSE)) {
+ VPValue *EndValue = EndValues.lookup(WideIV);
+ assert(EndValue && "Must have computed the end value up front");
+
+ // `getOptimizableIVOf()` always returns the pre-incremented IV, so if it
+ // changed it means the exit is using the incremented value, so we don't
+ // need to subtract the step.
+ if (Incoming != WideIV)
+ return EndValue;
+
+ // Otherwise, subtract the step from the EndValue.
+ VPBuilder B(ExtractR);
+ VPValue *Step = WideIV->getStepValue();
+ Type *ScalarTy = WideIV->getScalarType();
+ if (ScalarTy->isIntegerTy())
+ return B.createSub(EndValue, Step, DebugLoc::getUnknown(), "ind.escape");
+ if (ScalarTy->isPointerTy()) {
+ Type *StepTy = Step->getScalarType();
+ auto *Zero = Plan.getZero(StepTy);
+ return B.createPtrAdd(EndValue, B.createSub(Zero, Step),
+ DebugLoc::getUnknown(), "ind.escape");
+ }
+ if (ScalarTy->isFloatingPointTy()) {
+ const auto &ID = WideIV->getInductionDescriptor();
+ return B.createNaryOp(
+ ID.getInductionBinOp()->getOpcode() == Instruction::FAdd
+ ? Instruction::FSub
+ : Instruction::FAdd,
+ {EndValue, Step}, {ID.getInductionBinOp()->getFastMathFlags()});
+ }
+ llvm_unreachable("all possible induction types must be handled");
+ }
- VPValue *EndValue = EndValues.lookup(WideIV);
- assert(EndValue && "Must have computed the end value up front");
+ const SCEV *IncomingSCEV = vputils::getSCEVExprForVPValue(Incoming, PSE, L);
+ const SCEV *Start, *Step;
+ if (!match(IncomingSCEV, m_scev_AffineAddRec(m_SCEV(Start), m_SCEV(Step),
+ m_SpecificLoop(L))))
+ return nullptr;
- // `getOptimizableIVOf()` always returns the pre-incremented IV, so if it
- // changed it means the exit is using the incremented value, so we don't
- // need to subtract the step.
- if (Incoming != WideIV)
- return EndValue;
+ VPValue *StartVPV = vputils::getOrCreateVPValueForSCEVExpr(Plan, Start);
+ auto *StartIRV = dyn_cast<VPIRValue>(StartVPV);
+ if (!StartIRV) {
+ VPRecipeBase *Def = StartVPV->getDefiningRecipe();
+ assert(Def && "The value must be defined by VPExpandSCEVRecipe");
+ assert(StartVPV->getNumUsers() == 0 &&
+ "Newly created VPExpandSCEVRecipe should have no users");
+ Def->eraseFromParent();
+ return nullptr;
+ }
- // Otherwise, subtract the step from the EndValue.
- auto *ExtractR = cast<VPInstruction>(Op);
- VPBuilder B(ExtractR);
- VPValue *Step = WideIV->getStepValue();
- Type *ScalarTy = WideIV->getScalarType();
- if (ScalarTy->isIntegerTy())
- return B.createSub(EndValue, Step, DebugLoc::getUnknown(), "ind.escape");
- if (ScalarTy->isPointerTy()) {
- Type *StepTy = Step->getScalarType();
- auto *Zero = Plan.getZero(StepTy);
- return B.createPtrAdd(EndValue, B.createSub(Zero, Step),
- DebugLoc::getUnknown(), "ind.escape");
- }
- if (ScalarTy->isFloatingPointTy()) {
- const auto &ID = WideIV->getInductionDescriptor();
- return B.createNaryOp(
- ID.getInductionBinOp()->getOpcode() == Instruction::FAdd
- ? Instruction::FSub
- : Instruction::FAdd,
- {EndValue, Step}, {ID.getInductionBinOp()->getFastMathFlags()});
- }
- llvm_unreachable("all possible induction types must be handled");
- return nullptr;
+ Type *StartTy = StartIRV->getType();
+ assert(StartTy->isIntOrPtrTy() && "The type must be SCEVable");
+ InductionDescriptor::InductionKind Kind =
+ StartTy->isPointerTy() ? InductionDescriptor::IK_PtrInduction
+ : InductionDescriptor::IK_IntInduction;
+ VPValue *StepVPV = vputils::getOrCreateVPValueForSCEVExpr(Plan, Step);
+ VPBuilder Builder(ExtractR);
+ Type *TCTy = ResumeTC->getScalarType();
+ VPValue *It = Builder.createSub(ResumeTC, Plan.getConstantInt(TCTy, 1),
+ DebugLoc::getUnknown());
+ It = Builder.createScalarZExtOrTrunc(It, StepVPV->getScalarType(), TCTy,
+ DebugLoc::getUnknown());
+ return Builder.createDerivedIV(Kind, /*FPBinOp=*/nullptr, StartIRV, It,
+ StepVPV);
}
void VPlanTransforms::optimizeInductionLiveOutUsers(
- VPlan &Plan, PredicatedScalarEvolution &PSE, bool FoldTail) {
+ VPlan &Plan, PredicatedScalarEvolution &PSE, const Loop *L, bool FoldTail) {
// Compute end values for all inductions.
VPRegionBlock *VectorRegion = Plan.getVectorLoopRegion();
auto *VectorPH = cast<VPBasicBlock>(VectorRegion->getSinglePredecessor());
@@ -1183,7 +1211,7 @@ void VPlanTransforms::optimizeInductionLiveOutUsers(
VPValue *Escape = nullptr;
if (PredVPBB == MiddleVPBB)
Escape = optimizeLatchExitInductionUser(
- Plan, ExitIRI->getOperand(Idx), EndValues, PSE);
+ Plan, ExitIRI->getOperand(Idx), EndValues, PSE, ResumeTC, L);
else
Escape = optimizeEarlyExitInductionUser(
Plan, ExitIRI->getOperand(Idx), PSE);
diff --git a/llvm/lib/Transforms/Vectorize/VPlanTransforms.h b/llvm/lib/Transforms/Vectorize/VPlanTransforms.h
index b186594cd5b1b..325adfa3ec643 100644
--- a/llvm/lib/Transforms/Vectorize/VPlanTransforms.h
+++ b/llvm/lib/Transforms/Vectorize/VPlanTransforms.h
@@ -401,7 +401,7 @@ struct VPlanTransforms {
/// one step backwards.
static void optimizeInductionLiveOutUsers(VPlan &Plan,
PredicatedScalarEvolution &PSE,
- bool FoldTail);
+ const Loop *L, bool FoldTail);
/// Add explicit broadcasts for live-ins and VPValues defined in \p Plan's entry block if they are used as vectors.
static void materializeBroadcasts(VPlan &Plan);
diff --git a/llvm/test/Transforms/LoopVectorize/AArch64/reduction-cost.ll b/llvm/test/Transforms/LoopVectorize/AArch64/reduction-cost.ll
index 4bb6bf39413d0..adf7adb09b94c 100644
--- a/llvm/test/Transforms/LoopVectorize/AArch64/reduction-cost.ll
+++ b/llvm/test/Transforms/LoopVectorize/AArch64/reduction-cost.ll
@@ -10,14 +10,11 @@ define i64 @reduction(i64 %arg) #0 {
; CHECK-NEXT: br label %[[VECTOR_BODY:.*]]
; CHECK: [[VECTOR_BODY]]:
; CHECK-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
-; CHECK-NEXT: [[VEC_IND:%.*]] = phi <4 x i64> [ <i64 0, i64 1, i64 2, i64 3>, %[[VECTOR_PH]] ], [ [[VEC_IND_NEXT:%.*]], %[[VECTOR_BODY]] ]
; CHECK-NEXT: [[VEC_PHI:%.*]] = phi <4 x i32> [ zeroinitializer, %[[VECTOR_PH]] ], [ [[TMP5:%.*]], %[[VECTOR_BODY]] ]
; CHECK-NEXT: [[VEC_PHI2:%.*]] = phi <4 x i32> [ zeroinitializer, %[[VECTOR_PH]] ], [ [[TMP1:%.*]], %[[VECTOR_BODY]] ]
-; CHECK-NEXT: [[STEP_ADD:%.*]] = add nuw <4 x i64> [[VEC_IND]], splat (i64 4)
; CHECK-NEXT: [[TMP5]] = or <4 x i32> [[VEC_PHI]], splat (i32 1)
; CHECK-NEXT: [[TMP1]] = or <4 x i32> [[VEC_PHI2]], splat (i32 1)
; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 8
-; CHECK-NEXT: [[VEC_IND_NEXT]] = add nsw <4 x i64> [[STEP_ADD]], splat (i64 4)
; CHECK-NEXT: [[TMP6:%.*]] = icmp eq i64 [[INDEX_NEXT]], 96
; CHECK-NEXT: br i1 [[TMP6]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP0:![0-9]+]]
; CHECK: [[MIDDLE_BLOCK]]:
diff --git a/llvm/test/Transforms/LoopVectorize/VPlan/vplan-printing.ll b/llvm/test/Transforms/LoopVectorize/VPlan/vplan-printing.ll
index 2e588784eaf00..f3d3fef6b7a74 100644
--- a/llvm/test/Transforms/LoopVectorize/VPlan/vplan-printing.ll
+++ b/llvm/test/Transforms/LoopVectorize/VPlan/vplan-printing.ll
@@ -569,7 +569,6 @@ define i32 @print_exit_value(ptr %ptr, i32 %off) {
; CHECK-NEXT: vp<[[VP3:%[0-9]+]]> = CANONICAL-IV
; CHECK-EMPTY:
; CHECK-NEXT: vector.body:
-; CHECK-NEXT: ir<%iv> = WIDEN-INDUCTION nsw ir<0>, ir<1>, vp<[[VP0]]>
; CHECK-NEXT: vp<[[VP4:%[0-9]+]]> = SCALAR-STEPS vp<[[VP3]]>, ir<1>, vp<[[VP0]]>
; CHECK-NEXT: CLONE ir<%gep> = getelementptr inbounds ir<%ptr>, vp<[[VP4]]>
; CHECK-NEXT: vp<[[VP5:%[0-9]+]]> = vector-pointer inbounds ir<%gep>, ir<1>
@@ -581,9 +580,8 @@ define i32 @print_exit_value(ptr %ptr, i32 %off) {
; CHECK-NEXT: Successor(s): middle.block
; CHECK-EMPTY:
; CHECK-NEXT: middle.block:
-; CHECK-NEXT: WIDEN ir<%add> = add ir<%iv>, ir<%off>
-; CHECK-NEXT: EMIT vp<[[VP7:%[0-9]+]]> = extract-last-part ir<%add>
-; CHECK-NEXT: EMIT vp<[[VP8:%[0-9]+]]> = extract-last-lane vp<[[VP7]]>
+; CHECK-NEXT: EMIT vp<[[VP7:%[0-9]+]]> = sub vp<[[VP2]]>, ir<1>
+; CHECK-NEXT: vp<[[VP8:%[0-9]+]]> = DERIVED-IV ir<%off> + vp<[[VP7]]> * ir<1>
; CHECK-NEXT: EMIT vp<%cmp.n> = icmp eq ir<1000>, vp<[[VP2]]>
; CHECK-NEXT: EMIT branch-on-cond vp<%cmp.n>
; CHECK-NEXT: Successor(s): ir-bb<exit>, scalar.ph
diff --git a/llvm/test/Transforms/LoopVectorize/X86/gep-use-outside-loop.ll b/llvm/test/Transforms/LoopVectorize/X86/gep-use-outside-loop.ll
index 8fdd60f2dc1b1..c3ea2a3f4fd76 100644
--- a/llvm/test/Transforms/LoopVectorize/X86/gep-use-outside-loop.ll
+++ b/llvm/test/Transforms/LoopVectorize/X86/gep-use-outside-loop.ll
@@ -82,10 +82,10 @@ define void @gep_use_outside_loop(ptr noalias %dst, ptr %src) {
; CHECK-NEXT: [[TMP0:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
; CHECK-NEXT: [[VEC_IND:%.*]] = phi <4 x i64> [ <i64 0, i64 1, i64 2, i64 3>, %[[VECTOR_PH]] ], [ [[VEC_IND_NEXT:%.*]], %[[VECTOR_BODY]] ]
; CHECK-NEXT: [[TMP1:%.*]] = getelementptr i16, ptr [[DST]], <4 x i64> [[VEC_IND]]
-; CHECK-NEXT: [[TMP6:%.*]] = extractelement <4 x ptr> [[TMP1]], i64 0
; CHECK-NEXT: [[TMP2:%.*]] = getelementptr i16, ptr [[SRC]], i64 [[TMP0]]
; CHECK-NEXT: [[WIDE_LOAD:%.*]] = load <4 x i16>, ptr [[TMP2]], align 2
; CHECK-NEXT: [[TMP5:%.*]] = icmp ne <4 x i16> [[WIDE_LOAD]], splat (i16 10)
+; CHECK-NEXT: [[TMP6:%.*]] = extractelement <4 x ptr> [[TMP1]], i64 0
; CHECK-NEXT: call void @llvm.masked.store.v4i16.p0(<4 x i16> zeroinitializer, ptr align 2 [[TMP6]], <4 x i1> [[TMP5]])
; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[TMP0]], 4
; CHECK-NEXT: [[VEC_IND_NEXT]] = add <4 x i64> [[VEC_IND]], splat (i64 4)
diff --git a/llvm/test/Transforms/LoopVectorize/X86/pr51366-sunk-instruction-used-outside-of-loop.ll b/llvm/test/Transforms/LoopVectorize/X86/pr51366-sunk-instruction-used-outside-of-loop.ll
index 3d31aaab1341c..29cd22c0944f1 100644
--- a/llvm/test/Transforms/LoopVectorize/X86/pr51366-sunk-instruction-used-outside-of-loop.ll
+++ b/llvm/test/Transforms/LoopVectorize/X86/pr51366-sunk-instruction-used-outside-of-loop.ll
@@ -9,15 +9,13 @@ define ptr @test(ptr noalias %src, ptr noalias %dst) {
; CHECK: [[VECTOR_PH]]:
; CHECK-NEXT: br label %[[VECTOR_BODY:.*]]
; CHECK: [[VECTOR_BODY]]:
-; CHECK-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[PRED_LOAD_CONTINUE2:.*]] ]
+; CHECK-NEXT: [[TMP3:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[PRED_LOAD_CONTINUE2:.*]] ]
; CHECK-NEXT: [[VEC_IND:%.*]] = phi <2 x i64> [ <i64 0, i64 1>, %[[VECTOR_PH]] ], [ [[VEC_IND_NEXT:%.*]], %[[PRED_LOAD_CONTINUE2]] ]
-; CHECK-NEXT: [[TMP1:%.*]] = add i64 [[INDEX]], 1
-; CHECK-NEXT: [[TMP6:%.*]] = getelementptr inbounds i32, ptr [[SRC]], i64 [[INDEX]]
-; CHECK-NEXT: [[TMP2:%.*]] = getelementptr inbounds i32, ptr [[SRC]], i64 [[TMP1]]
; CHECK-NEXT: [[TMP4:%.*]] = icmp ne <2 x i64> [[VEC_IND]], zeroinitializer
; CHECK-NEXT: [[TMP5:%.*]] = extractelement <2 x i1> [[TMP4]], i64 0
; CHECK-NEXT: br i1 [[TMP5]], label %[[PRED_LOAD_IF:.*]], label %[[PRED_LOAD_CONTINUE:.*]]
; CHECK: [[PRED_LOAD_IF]]:
+; CHECK-NEXT: [[TMP6:%.*]] = getelementptr inbounds i32, ptr [[SRC]], i64 [[TMP3]]
; CHECK-NEXT: [[TMP7:%.*]] = load i32, ptr [[TMP6]], align 4
; CHECK-NEXT: [[TMP8:%.*]] = insertelement <2 x i32> poison, i32 [[TMP7]], i64 0
; CHECK-NEXT: br label %[[PRED_LOAD_CONTINUE]]
@@ -26,22 +24,25 @@ define ptr @test(ptr noalias %src, ptr noalias %dst) {
; CHECK-NEXT: [[TMP10:%.*]] = extractelement <2 x i1> [[TMP4]], i64 1
; CHECK-NEXT: br i1 [[TMP10]], label %[[PRED_LOAD_IF1:.*]], label %[[PRED_LOAD_CONTINUE2]]
; CHECK: [[PRED_LOAD_IF1]]:
+; CHECK-NEXT: [[TMP13:%.*]] = add i64 [[TMP3]], 1
+; CHECK-NEXT: [[TMP2:%.*]] = getelementptr inbounds i32, ptr [[SRC]], i64 [[TMP13]]
; CHECK-NEXT: [[TMP11:%.*]] = load i32, ptr [[TMP2]], align 4
; CHECK-NEXT: [[TMP12:%.*]] = insertelement <2 x i32> [[TMP9]], i32 [[TMP11]], i64 1
; CHECK-NEXT: br label %[[PRED_LOAD_CONTINUE2]]
; CHECK: [[PRED_LOAD_CONTINUE2]]:
; CHECK-NEXT: [[TMP15:%.*]] = phi <2 x i32> [ [[TMP9]], %[[PRED_LOAD_CONTINUE]] ], [ [[TMP12]], %[[PRED_LOAD_IF1]] ]
; CHECK-NEXT: [[PREDPHI:%.*]] = select <2 x i1> [[TMP4]], <2 x i32> [[TMP15]], <2 x i32> zeroinitializer
-; CHECK-NEXT: [[TMP14:%.*]] = getelementptr inbounds i32, ptr [[DST]], i64 [[INDEX]]
+; CHECK-NEXT: [[TMP14:%.*]] = getelementptr inbounds i32, ptr [[DST]], i64 [[TMP3]]
; CHECK-NEXT: store <2 x i32> [[PREDPHI]], ptr [[TMP14]], align 4
-; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 2
+; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[TMP3]], 2
; CHECK-NEXT: [[VEC_IND_NEXT]] = add nsw <2 x i64> [[VEC_IND]], splat (i64 2)
; CHECK-NEXT: [[TMP17:%.*]] = icmp eq i64 [[INDEX_NEXT]], 1000
; CHECK-NEXT: br i1 [[TMP17]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP0:![0-9]+]]
; CHECK: [[MIDDLE_BLOCK]]:
+; CHECK-NEXT: [[TMP16:%.*]] = getelementptr i8, ptr [[SRC]], i64 3996
; CHECK-NEXT: br label %[[EXIT:.*]]
; CHECK: [[EXIT]]:
-; CHECK-NEXT: ret ptr [[TMP2]]
+; CHECK-NEXT: ret ptr [[TMP16]]
;
entry:
br label %loop.header
diff --git a/llvm/test/Transforms/LoopVectorize/instruction-only-used-outside-of-loop.ll b/llvm/test/Transforms/LoopVectorize/instruction-only-used-outside-of-loop.ll
index 1bec39fbae92f..7ffeb8b3c2908 100644
--- a/llvm/test/Transforms/LoopVectorize/instruction-only-used-outside-of-loop.ll
+++ b/llvm/test/Transforms/LoopVectorize/instruction-only-used-outside-of-loop.ll
@@ -150,16 +150,13 @@ define i32 @optimizable_trunc_used_outside() {
; CHECK-NEXT: br label [[VECTOR_BODY:%.*]]
; CHECK: vector.body:
; CHECK-NEXT: [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ]
-; CHECK-NEXT: [[VEC_IND:%.*]] = phi <4 x i32> [ <i32 0, i32 1, i32 2, i32 3>, [[VECTOR_PH]] ], [ [[VEC_IND_NEXT:%.*]], [[VECTOR_BODY]] ]
; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 4
-; CHECK-NEXT: [[VEC_IND_NEXT]] = add <4 x i32> [[VEC_IND]], splat (i32 4)
; CHECK-NEXT: [[TMP0:%.*]] = icmp eq i64 [[INDEX_NEXT]], 1000
; CHECK-NEXT: br i1 [[TMP0]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP5:![0-9]+]]
; CHECK: middle.block:
-; CHECK-NEXT: [[TMP1:%.*]] = extractelement <4 x i32> [[VEC_IND]], i64 3
; CHECK-NEXT: br label [[LOOP:%.*]]
; CHECK: exit:
-; CHECK-NEXT: ret i32 [[TMP1]]
+; CHECK-NEXT: ret i32 999
;
entry:
br label %loop
diff --git a/llvm/test/Transforms/LoopVectorize/iv_outside_user.ll b/llvm/test/Transforms/LoopVectorize/iv_outside_user.ll
index e1b1f34d69fcf..ce07497fd0d34 100644
--- a/llvm/test/Transforms/LoopVectorize/iv_outside_user.ll
+++ b/llvm/test/Transforms/LoopVectorize/iv_outside_user.ll
@@ -631,71 +631,37 @@ for.end:
}
define i32 @postinc_not_iv_backedge_value(i32 %k) {
-; VEC-LABEL: define i32 @postinc_not_iv_backedge_value(
-; VEC-SAME: i32 [[K:%.*]]) {
-; VEC-NEXT: [[ENTRY:.*]]:
-; VEC-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ult i32 [[K]], 2
-; VEC-NEXT: br i1 [[MIN_ITERS_CHECK]], label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]]
-; VEC: [[VECTOR_PH]]:
-; VEC-NEXT: [[N_MOD_VF:%.*]] = urem i32 [[K]], 2
-; VEC-NEXT: [[N_VEC:%.*]] = sub i32 [[K]], [[N_MOD_VF]]
-; VEC-NEXT: br label %[[VECTOR_BODY:.*]]
-; VEC: [[VECTOR_BODY]]:
-; VEC-NEXT: [[INDEX:%.*]] = phi i32 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
-; VEC-NEXT: [[VEC_IND:%.*]] = phi <2 x i32> [ <i32 0, i32 1>, %[[VECTOR_PH]] ], [ [[VEC_IND_NEXT:%.*]], %[[VECTOR_BODY]] ]
-; VEC-NEXT: [[INDEX_NEXT]] = add nuw i32 [[INDEX]], 2
-; VEC-NEXT: [[VEC_IND_NEXT]] = add nsw <2 x i32> [[VEC_IND]], splat (i32 2)
-; VEC-NEXT: [[TMP1:%.*]] = icmp eq i32 [[INDEX_NEXT]], [[N_VEC]]
-; VEC-NEXT: br i1 [[TMP1]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], {{!llvm.loop ![0-9]+}}
-; VEC: [[MIDDLE_BLOCK]]:
-; VEC-NEXT: [[TMP0:%.*]] = add <2 x i32> [[VEC_IND]], splat (i32 2)
-; VEC-NEXT: [[TMP2:%.*]] = extractelement <2 x i32> [[TMP0]], i64 1
-; VEC-NEXT: [[CMP_N:%.*]] = icmp eq i32 [[K]], [[N_VEC]]
-; VEC-NEXT: br i1 [[CMP_N]], label %[[FOR_END:.*]], label %[[SCALAR_PH]]
-; VEC: [[SCALAR_PH]]:
-; VEC-NEXT: [[BC_RESUME_VAL:%.*]] = phi i32 [ [[N_VEC]], %[[MIDDLE_BLOCK]] ], [ 0, %[[ENTRY]] ]
-; VEC-NEXT: br label %[[FOR_BODY:.*]]
-; VEC: [[FOR_BODY]]:
-; VEC-NEXT: [[INC_PHI:%.*]] = phi i32 [ [[BC_RESUME_VAL]], %[[SCALAR_PH]] ], [ [[INC:%.*]], %[[FOR_BODY]] ]
-; VEC-NEXT: [[INC]] = add nsw i32 [[INC_PHI]], 1
-; VEC-NEXT: [[INC_2:%.*]] = add i32 [[INC_PHI]], 2
-; VEC-NEXT: [[CMP:%.*]] = icmp eq i32 [[INC]], [[K]]
-; VEC-NEXT: br i1 [[CMP]], label %[[FOR_END]], label %[[FOR_BODY]], {{!llvm.loop ![0-9]+}}
-; VEC: [[FOR_END]]:
-; VEC-NEXT: [[INC_2_LCSSA:%.*]] = phi i32 [ [[INC_2]], %[[FOR_BODY]] ], [ [[TMP2]], %[[MIDDLE_BLOCK]] ]
-; VEC-NEXT: ret i32 [[INC_2_LCSSA]]
-;
-; INTERLEAVE-LABEL: define i32 @postinc_not_iv_backedge_value(
-; INTERLEAVE-SAME: i32 [[K:%.*]]) {
-; INTERLEAVE-NEXT: [[ENTRY:.*]]:
-; INTERLEAVE-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ult i32 [[K]], 2
-; INTERLEAVE-NEXT: br i1 [[MIN_ITERS_CHECK]], label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]]
-; INTERLEAVE: [[VECTOR_PH]]:
-; INTERLEAVE-NEXT: [[N_MOD_VF:%.*]] = urem i32 [[K]], 2
-; INTERLEAVE-NEXT: [[N_VEC:%.*]] = sub i32 [[K]], [[N_MOD_VF]]
-; INTERLEAVE-NEXT: br label %[[VECTOR_BODY:.*]]
-; INTERLEAVE: [[VECTOR_BODY]]:
-; INTERLEAVE-NEXT: [[INDEX:%.*]] = phi i32 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
-; INTERLEAVE-NEXT: [[INDEX_NEXT]] = add nuw i32 [[INDEX]], 2
-; INTERLEAVE-NEXT: [[TMP2:%.*]] = icmp eq i32 [[INDEX_NEXT]], [[N_VEC]]
-; INTERLEAVE-NEXT: br i1 [[TMP2]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], {{!llvm.loop ![0-9]+}}
-; INTERLEAVE: [[MIDDLE_BLOCK]]:
-; INTERLEAVE-NEXT: [[TMP0:%.*]] = add i32 [[INDEX]], 1
-; INTERLEAVE-NEXT: [[TMP1:%.*]] = add i32 [[TMP0]], 2
-; INTERLEAVE-NEXT: [[CMP_N:%.*]] = icmp eq i32 [[K]], [[N_VEC]]
-; INTERLEAVE-NEXT: br i1 [[CMP_N]], label %[[FOR_END:.*]], label %[[SCALAR_PH]]
-; INTERLEAVE: [[SCALAR_PH]]:
-; INTERLEAVE-NEXT: [[BC_RESUME_VAL:%.*]] = phi i32 [ [[N_VEC]], %[[MIDDLE_BLOCK]] ], [ 0, %[[ENTRY]] ]
-; INTERLEAVE-NEXT: br label %[[FOR_BODY:.*]]
-; INTERLEAVE: [[FOR_BODY]]:
-; INTERLEAVE-NEXT: [[INC_PHI:%.*]] = phi i32 [ [[BC_RESUME_VAL]], %[[SCALAR_PH]] ], [ [[INC:%.*]], %[[FOR_BODY]] ]
-; INTERLEAVE-NEXT: [[INC]] = add nsw i32 [[INC_PHI]], 1
-; INTERLEAVE-NEXT: [[INC_2:%.*]] = add i32 [[INC_PHI]], 2
-; INTERLEAVE-NEXT: [[CMP:%.*]] = icmp eq i32 [[INC]], [[K]]
-; INTERLEAVE-NEXT: br i1 [[CMP]], label %[[FOR_END]], label %[[FOR_BODY]], {{!llvm.loop ![0-9]+}}
-; INTERLEAVE: [[FOR_END]]:
-; INTERLEAVE-NEXT: [[INC_2_LCSSA:%.*]] = phi i32 [ [[INC_2]], %[[FOR_BODY]] ], [ [[TMP1]], %[[MIDDLE_BLOCK]] ]
-; INTERLEAVE-NEXT: ret i32 [[INC_2_LCSSA]]
+; CHECK-LABEL: define i32 @postinc_not_iv_backedge_value(
+; CHECK-SAME: i32 [[K:%.*]]) {
+; CHECK-NEXT: [[ENTRY:.*]]:
+; CHECK-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ult i32 [[K]], 2
+; CHECK-NEXT: br i1 [[MIN_ITERS_CHECK]], label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]]
+; CHECK: [[VECTOR_PH]]:
+; CHECK-NEXT: [[N_MOD_VF:%.*]] = urem i32 [[K]], 2
+; CHECK-NEXT: [[N_VEC:%.*]] = sub i32 [[K]], [[N_MOD_VF]]
+; CHECK-NEXT: br label %[[VECTOR_BODY:.*]]
+; CHECK: [[VECTOR_BODY]]:
+; CHECK-NEXT: [[INDEX:%.*]] = phi i32 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i32 [[INDEX]], 2
+; CHECK-NEXT: [[TMP0:%.*]] = icmp eq i32 [[INDEX_NEXT]], [[N_VEC]]
+; CHECK-NEXT: br i1 [[TMP0]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], {{!llvm.loop ![0-9]+}}
+; CHECK: [[MIDDLE_BLOCK]]:
+; CHECK-NEXT: [[TMP1:%.*]] = sub i32 [[N_VEC]], 1
+; CHECK-NEXT: [[TMP2:%.*]] = add i32 2, [[TMP1]]
+; CHECK-NEXT: [[CMP_N:%.*]] = icmp eq i32 [[K]], [[N_VEC]]
+; CHECK-NEXT: br i1 [[CMP_N]], label %[[FOR_END:.*]], label %[[SCALAR_PH]]
+; CHECK: [[SCALAR_PH]]:
+; CHECK-NEXT: [[BC_RESUME_VAL:%.*]] = phi i32 [ [[N_VEC]], %[[MIDDLE_BLOCK]] ], [ 0, %[[ENTRY]] ]
+; CHECK-NEXT: br label %[[FOR_BODY:.*]]
+; CHECK: [[FOR_BODY]]:
+; CHECK-NEXT: [[INC_PHI:%.*]] = phi i32 [ [[BC_RESUME_VAL]], %[[SCALAR_PH]] ], [ [[INC:%.*]], %[[FOR_BODY]] ]
+; CHECK-NEXT: [[INC]] = add nsw i32 [[INC_PHI]], 1
+; CHECK-NEXT: [[INC_2:%.*]] = add i32 [[INC_PHI]], 2
+; CHECK-NEXT: [[CMP:%.*]] = icmp eq i32 [[INC]], [[K]]
+; CHECK-NEXT: br i1 [[CMP]], label %[[FOR_END]], label %[[FOR_BODY]], {{!llvm.loop ![0-9]+}}
+; CHECK: [[FOR_END]]:
+; CHECK-NEXT: [[INC_2_LCSSA:%.*]] = phi i32 [ [[INC_2]], %[[FOR_BODY]] ], [ [[TMP2]], %[[MIDDLE_BLOCK]] ]
+; CHECK-NEXT: ret i32 [[INC_2_LCSSA]]
;
entry:
br label %for.body
@@ -1320,19 +1286,17 @@ define i32 @cast_incremented_iv_live_out(ptr %arr, i32 %n) {
; VEC-NEXT: br label %[[VECTOR_BODY:.*]]
; VEC: [[VECTOR_BODY]]:
; VEC-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
-; VEC-NEXT: [[VEC_IND:%.*]] = phi <2 x i64> [ <i64 0, i64 1>, %[[VECTOR_PH]] ], [ [[VEC_IND_NEXT:%.*]], %[[VECTOR_BODY]] ]
; VEC-NEXT: [[TMP1:%.*]] = getelementptr i8, ptr [[ARR]], i64 [[INDEX]]
; VEC-NEXT: [[WIDE_LOAD:%.*]] = load <2 x i8>, ptr [[TMP1]], align 1
; VEC-NEXT: [[TMP2:%.*]] = add <2 x i8> [[WIDE_LOAD]], splat (i8 1)
; VEC-NEXT: store <2 x i8> [[TMP2]], ptr [[TMP1]], align 1
; VEC-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 2
-; VEC-NEXT: [[VEC_IND_NEXT]] = add <2 x i64> [[VEC_IND]], splat (i64 2)
; VEC-NEXT: [[TMP3:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
; VEC-NEXT: br i1 [[TMP3]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], {{!llvm.loop ![0-9]+}}
; VEC: [[MIDDLE_BLOCK]]:
-; VEC-NEXT: [[TMP4:%.*]] = add <2 x i64> [[VEC_IND]], splat (i64 1)
-; VEC-NEXT: [[TMP5:%.*]] = trunc <2 x i64> [[TMP4]] to <2 x i32>
-; VEC-NEXT: [[TMP6:%.*]] = extractelement <2 x i32> [[TMP5]], i64 1
+; VEC-NEXT: [[TMP4:%.*]] = sub i64 [[N_VEC]], 1
+; VEC-NEXT: [[TMP5:%.*]] = trunc i64 [[TMP4]] to i32
+; VEC-NEXT: [[TMP6:%.*]] = add i32 1, [[TMP5]]
; VEC-NEXT: [[CMP_N:%.*]] = icmp eq i64 [[UMAX]], [[N_VEC]]
; VEC-NEXT: br i1 [[CMP_N]], label %[[EXIT:.*]], label %[[SCALAR_PH]]
; VEC: [[SCALAR_PH]]:
@@ -1378,8 +1342,9 @@ define i32 @cast_incremented_iv_live_out(ptr %arr, i32 %n) {
; INTERLEAVE-NEXT: [[TMP8:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
; INTERLEAVE-NEXT: br i1 [[TMP8]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], {{!llvm.loop ![0-9]+}}
; INTERLEAVE: [[MIDDLE_BLOCK]]:
-; INTERLEAVE-NEXT: [[TMP9:%.*]] = add i64 [[TMP1]], 1
+; INTERLEAVE-NEXT: [[TMP9:%.*]] = sub i64 [[N_VEC]], 1
; INTERLEAVE-NEXT: [[TMP10:%.*]] = trunc i64 [[TMP9]] to i32
+; INTERLEAVE-NEXT: [[TMP11:%.*]] = add i32 1, [[TMP10]]
; INTERLEAVE-NEXT: [[CMP_N:%.*]] = icmp eq i64 [[UMAX]], [[N_VEC]]
; INTERLEAVE-NEXT: br i1 [[CMP_N]], label %[[EXIT:.*]], label %[[SCALAR_PH]]
; INTERLEAVE: [[SCALAR_PH]]:
@@ -1396,7 +1361,7 @@ define i32 @cast_incremented_iv_live_out(ptr %arr, i32 %n) {
; INTERLEAVE-NEXT: [[COND:%.*]] = icmp ult i32 [[IV_TRUNC]], [[N]]
; INTERLEAVE-NEXT: br i1 [[COND]], label %[[LOOP]], label %[[EXIT]], {{!llvm.loop ![0-9]+}}
; INTERLEAVE: [[EXIT]]:
-; INTERLEAVE-NEXT: [[IV_TRUNC_LCSSA:%.*]] = phi i32 [ [[IV_TRUNC]], %[[LOOP]] ], [ [[TMP10]], %[[MIDDLE_BLOCK]] ]
+; INTERLEAVE-NEXT: [[IV_TRUNC_LCSSA:%.*]] = phi i32 [ [[IV_TRUNC]], %[[LOOP]] ], [ [[TMP11]], %[[MIDDLE_BLOCK]] ]
; INTERLEAVE-NEXT: ret i32 [[IV_TRUNC_LCSSA]]
;
entry:
diff --git a/llvm/test/Transforms/LoopVectorize/no_outside_user.ll b/llvm/test/Transforms/LoopVectorize/no_outside_user.ll
index 8b303f9c640e2..c80ae85d18fed 100644
--- a/llvm/test/Transforms/LoopVectorize/no_outside_user.ll
+++ b/llvm/test/Transforms/LoopVectorize/no_outside_user.ll
@@ -734,15 +734,12 @@ define i32 @non_uniform_live_out() {
; CHECK-NEXT: br label %[[VECTOR_BODY:.*]]
; CHECK: [[VECTOR_BODY]]:
; CHECK-NEXT: [[INDEX:%.*]] = phi i32 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
-; CHECK-NEXT: [[VEC_IND:%.*]] = phi <2 x i32> [ <i32 0, i32 1>, %[[VECTOR_PH]] ], [ [[VEC_IND_NEXT:%.*]], %[[VECTOR_BODY]] ]
-; CHECK-NEXT: [[TMP0:%.*]] = add <2 x i32> [[VEC_IND]], splat (i32 7)
-; CHECK-NEXT: [[TMP1:%.*]] = extractelement <2 x i32> [[TMP0]], i64 0
+; CHECK-NEXT: [[TMP1:%.*]] = add i32 [[INDEX]], 7
; CHECK-NEXT: [[TMP2:%.*]] = getelementptr inbounds [32 x i8], ptr @tab, i32 0, i32 [[TMP1]]
; CHECK-NEXT: [[WIDE_LOAD:%.*]] = load <2 x i8>, ptr [[TMP2]], align 1
; CHECK-NEXT: [[TMP4:%.*]] = add <2 x i8> [[WIDE_LOAD]], splat (i8 1)
; CHECK-NEXT: store <2 x i8> [[TMP4]], ptr [[TMP2]], align 1
; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i32 [[INDEX]], 2
-; CHECK-NEXT: [[VEC_IND_NEXT]] = add nsw <2 x i32> [[VEC_IND]], splat (i32 2)
; CHECK-NEXT: [[TMP5:%.*]] = icmp eq i32 [[INDEX_NEXT]], 20000
; CHECK-NEXT: br i1 [[TMP5]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP14:![0-9]+]]
; CHECK: [[MIDDLE_BLOCK]]:
>From 8fa619556bcf34892c5549600f53ab8836c54d39 Mon Sep 17 00:00:00 2001
From: Mel Chen <mel.chen at sifive.com>
Date: Thu, 14 May 2026 01:39:58 -0700
Subject: [PATCH 2/5] create helper vputils::getVPIRValueForSCEVExpr
---
.../Transforms/Vectorize/VPlanTransforms.cpp | 11 ++---------
llvm/lib/Transforms/Vectorize/VPlanUtils.cpp | 18 +++++++++++++-----
llvm/lib/Transforms/Vectorize/VPlanUtils.h | 4 ++++
3 files changed, 19 insertions(+), 14 deletions(-)
diff --git a/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp b/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp
index c278bcbf2146e..962a4f39e7815 100644
--- a/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp
+++ b/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp
@@ -1145,16 +1145,9 @@ static VPValue *optimizeLatchExitInductionUser(
m_SpecificLoop(L))))
return nullptr;
- VPValue *StartVPV = vputils::getOrCreateVPValueForSCEVExpr(Plan, Start);
- auto *StartIRV = dyn_cast<VPIRValue>(StartVPV);
- if (!StartIRV) {
- VPRecipeBase *Def = StartVPV->getDefiningRecipe();
- assert(Def && "The value must be defined by VPExpandSCEVRecipe");
- assert(StartVPV->getNumUsers() == 0 &&
- "Newly created VPExpandSCEVRecipe should have no users");
- Def->eraseFromParent();
+ auto *StartIRV = vputils::getVPIRValueForSCEVExpr(Plan, Start);
+ if (!StartIRV)
return nullptr;
- }
Type *StartTy = StartIRV->getType();
assert(StartTy->isIntOrPtrTy() && "The type must be SCEVable");
diff --git a/llvm/lib/Transforms/Vectorize/VPlanUtils.cpp b/llvm/lib/Transforms/Vectorize/VPlanUtils.cpp
index 442ddbdde66ca..538b2b022c209 100644
--- a/llvm/lib/Transforms/Vectorize/VPlanUtils.cpp
+++ b/llvm/lib/Transforms/Vectorize/VPlanUtils.cpp
@@ -38,16 +38,24 @@ bool vputils::onlyScalarValuesUsed(const VPValue *Def) {
[Def](const VPUser *U) { return U->usesScalars(Def); });
}
-VPValue *vputils::getOrCreateVPValueForSCEVExpr(VPlan &Plan, const SCEV *Expr) {
+VPIRValue *vputils::getVPIRValueForSCEVExpr(VPlan &Plan, const SCEV *Expr) {
if (auto *E = dyn_cast<SCEVConstant>(Expr))
return Plan.getOrAddLiveIn(E->getValue());
- // Skip SCEV expansion if Expr is a SCEVUnknown wrapping a non-instruction
- // value. Otherwise the value may be defined in a loop and using it directly
- // will break LCSSA form. The SCEV expansion takes care of preserving LCSSA
- // form.
+ // For a SCEVUnknown wrapping an instruction, return nullptr since the value
+ // may be defined in a loop and using it directly will break LCSSA form.
auto *U = dyn_cast<SCEVUnknown>(Expr);
if (U && !isa<Instruction>(U->getValue()))
return Plan.getOrAddLiveIn(U->getValue());
+ return nullptr;
+}
+
+VPValue *vputils::getOrCreateVPValueForSCEVExpr(VPlan &Plan, const SCEV *Expr) {
+ // If SCEV expression can be represented as a live-in VPIRValue, use it
+ // directly without expanding.
+ if (auto *IRV = getVPIRValueForSCEVExpr(Plan, Expr))
+ return IRV;
+ // Otherwise use VPExpandSCEVRecipe to expand expression, which preserves
+ // LCSSA form for values defined in a loop.
auto *Expanded = new VPExpandSCEVRecipe(Expr);
VPBasicBlock *EntryVPBB = Plan.getEntry();
auto Iter = EntryVPBB->getFirstNonPhi();
diff --git a/llvm/lib/Transforms/Vectorize/VPlanUtils.h b/llvm/lib/Transforms/Vectorize/VPlanUtils.h
index deb025992c718..2e7bec9f1e64b 100644
--- a/llvm/lib/Transforms/Vectorize/VPlanUtils.h
+++ b/llvm/lib/Transforms/Vectorize/VPlanUtils.h
@@ -32,6 +32,10 @@ bool onlyFirstPartUsed(const VPValue *Def);
/// Returns true if only scalar values of \p Def are used by all users.
bool onlyScalarValuesUsed(const VPValue *Def);
+/// If \p Expr is a SCEVConstant or a SCEVUnknown wrapping a non-instruction
+/// value, return the corresponding VPIRValue live-in. Otherwise return nullptr.
+VPIRValue *getVPIRValueForSCEVExpr(VPlan &Plan, const SCEV *Expr);
+
/// Get or create a VPValue that corresponds to the expansion of \p Expr. If \p
/// Expr is a SCEVConstant or SCEVUnknown, return a VPValue wrapping the live-in
/// value. Otherwise return a VPExpandSCEVRecipe to expand \p Expr. If \p Plan's
>From 2ad53e77cc001125ba240ad860584705f2d978ae Mon Sep 17 00:00:00 2001
From: Mel Chen <mel.chen at sifive.com>
Date: Thu, 11 Jun 2026 00:24:27 -0700
Subject: [PATCH 3/5] Rename It to ExitCount
---
llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp | 10 +++++-----
1 file changed, 5 insertions(+), 5 deletions(-)
diff --git a/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp b/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp
index 962a4f39e7815..cfea2d5a7031c 100644
--- a/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp
+++ b/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp
@@ -1157,11 +1157,11 @@ static VPValue *optimizeLatchExitInductionUser(
VPValue *StepVPV = vputils::getOrCreateVPValueForSCEVExpr(Plan, Step);
VPBuilder Builder(ExtractR);
Type *TCTy = ResumeTC->getScalarType();
- VPValue *It = Builder.createSub(ResumeTC, Plan.getConstantInt(TCTy, 1),
- DebugLoc::getUnknown());
- It = Builder.createScalarZExtOrTrunc(It, StepVPV->getScalarType(), TCTy,
- DebugLoc::getUnknown());
- return Builder.createDerivedIV(Kind, /*FPBinOp=*/nullptr, StartIRV, It,
+ VPValue *ExitCount = Builder.createSub(ResumeTC, Plan.getConstantInt(TCTy, 1),
+ DebugLoc::getUnknown());
+ ExitCount = Builder.createScalarZExtOrTrunc(
+ ExitCount, StepVPV->getScalarType(), TCTy, DebugLoc::getUnknown());
+ return Builder.createDerivedIV(Kind, /*FPBinOp=*/nullptr, StartIRV, ExitCount,
StepVPV);
}
>From 3da8131026a5500d4cbc0f64aa6fc806e4a1bf89 Mon Sep 17 00:00:00 2001
From: Mel Chen <mel.chen at sifive.com>
Date: Thu, 11 Jun 2026 00:34:14 -0700
Subject: [PATCH 4/5] Add TODO comment
---
llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp | 4 +++-
1 file changed, 3 insertions(+), 1 deletion(-)
diff --git a/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp b/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp
index cfea2d5a7031c..b01dec7df986e 100644
--- a/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp
+++ b/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp
@@ -1145,7 +1145,9 @@ static VPValue *optimizeLatchExitInductionUser(
m_SpecificLoop(L))))
return nullptr;
- auto *StartIRV = vputils::getVPIRValueForSCEVExpr(Plan, Start);
+ // TODO: Start value can be defined be a VPExpandSCEVRecipe after
+ // VPDerivedIVRecipe supports a general VPValue as the start value.
+ VPIRValue *StartIRV = vputils::getVPIRValueForSCEVExpr(Plan, Start);
if (!StartIRV)
return nullptr;
>From 99177a1f29c76c85c6622d4c4525e509effadd6c Mon Sep 17 00:00:00 2001
From: Mel Chen <mel.chen at sifive.com>
Date: Thu, 11 Jun 2026 00:43:42 -0700
Subject: [PATCH 5/5] Remove redundant cast
---
llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp | 2 --
1 file changed, 2 deletions(-)
diff --git a/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp b/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp
index b01dec7df986e..07de5717d6fe4 100644
--- a/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp
+++ b/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp
@@ -1161,8 +1161,6 @@ static VPValue *optimizeLatchExitInductionUser(
Type *TCTy = ResumeTC->getScalarType();
VPValue *ExitCount = Builder.createSub(ResumeTC, Plan.getConstantInt(TCTy, 1),
DebugLoc::getUnknown());
- ExitCount = Builder.createScalarZExtOrTrunc(
- ExitCount, StepVPV->getScalarType(), TCTy, DebugLoc::getUnknown());
return Builder.createDerivedIV(Kind, /*FPBinOp=*/nullptr, StartIRV, ExitCount,
StepVPV);
}
More information about the llvm-commits
mailing list