[llvm] [LV] Enable the exit value optimization via SCEV for tail folding. (PR #210928)
via llvm-commits
llvm-commits at lists.llvm.org
Tue Jul 21 03:11:17 PDT 2026
llvmorg-github-actions[bot] wrote:
<!--LLVM PR SUMMARY COMMENT-->
@llvm/pr-subscribers-llvm-transforms
Author: Mel Chen (Mel-Chen)
<details>
<summary>Changes</summary>
Based on #<!-- -->195059
---
Patch is 85.09 KiB, truncated to 20.00 KiB below, full version: https://github.com/llvm/llvm-project/pull/210928.diff
14 Files Affected:
- (modified) llvm/lib/Transforms/Vectorize/LoopVectorize.cpp (+4-2)
- (modified) llvm/lib/Transforms/Vectorize/VPlanRecipes.cpp (+1-1)
- (modified) llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp (+49-4)
- (modified) llvm/lib/Transforms/Vectorize/VPlanTransforms.h (+2-1)
- (modified) llvm/test/Transforms/LoopVectorize/AArch64/reduction-cost.ll (+7-23)
- (modified) llvm/test/Transforms/LoopVectorize/AArch64/scalable-avoid-scalarization.ll (+7-4)
- (modified) llvm/test/Transforms/LoopVectorize/VPlan/buildvector-first-lane-only.ll (+2-11)
- (modified) llvm/test/Transforms/LoopVectorize/VPlan/vplan-printing.ll (+14-16)
- (modified) llvm/test/Transforms/LoopVectorize/X86/gep-use-outside-loop.ll (+1-1)
- (modified) llvm/test/Transforms/LoopVectorize/X86/pr51366-sunk-instruction-used-outside-of-loop.ll (+8-7)
- (modified) llvm/test/Transforms/LoopVectorize/instruction-only-used-outside-of-loop.ll (+1-4)
- (modified) llvm/test/Transforms/LoopVectorize/iv_outside_user.ll (+695-74)
- (modified) llvm/test/Transforms/LoopVectorize/no_outside_user.ll (+1-4)
- (modified) llvm/test/Transforms/LoopVectorize/predicated-inductions.ll (+5-5)
``````````diff
diff --git a/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp b/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
index 53ab36ccea2ce..e8826ed46c7e8 100644
--- a/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
+++ b/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
@@ -6636,7 +6636,8 @@ VPlanPtr LoopVectorizationPlanner::tryToBuildVPlan(VPlanPtr Plan,
if (!RUN_VPLAN_PASS(VPlanTransforms::tryToConvertVPInstructionsToVPRecipes,
*Plan, *TLI))
return nullptr;
- RUN_VPLAN_PASS(VPlanTransforms::optimizeInductionLiveOutUsers, *Plan, PSE);
+ RUN_VPLAN_PASS(VPlanTransforms::optimizeInductionLiveOutUsers, *Plan, PSE,
+ OrigLoop);
return Plan;
}
@@ -6805,7 +6806,8 @@ VPlanPtr LoopVectorizationPlanner::tryToBuildVPlan(VPlanPtr Plan,
// Optimize FindIV reductions to use sentinel-based approach when possible.
RUN_VPLAN_PASS(VPlanTransforms::optimizeFindIVReductions, *Plan, PSE,
*OrigLoop);
- RUN_VPLAN_PASS(VPlanTransforms::optimizeInductionLiveOutUsers, *Plan, PSE);
+ RUN_VPLAN_PASS(VPlanTransforms::optimizeInductionLiveOutUsers, *Plan, PSE,
+ OrigLoop);
// Apply mandatory transformation to handle reductions with multiple in-loop
// uses if possible, bail out otherwise.
diff --git a/llvm/lib/Transforms/Vectorize/VPlanRecipes.cpp b/llvm/lib/Transforms/Vectorize/VPlanRecipes.cpp
index f545c63d691d9..6385a9c0b5a13 100644
--- a/llvm/lib/Transforms/Vectorize/VPlanRecipes.cpp
+++ b/llvm/lib/Transforms/Vectorize/VPlanRecipes.cpp
@@ -3010,7 +3010,7 @@ InstructionCost VPDerivedIVRecipe::computeCost(ElementCount VF,
unsigned IndexTySize = IndexTy->getScalarSizeInBits();
if ((NeedsAdd || NeedsMul || NeedsShl) && StepTySize != IndexTySize) {
unsigned CastOpc =
- StepTySize < IndexTySize ? Instruction::Trunc : Instruction::SExt;
+ StepTySize < IndexTySize ? Instruction::Trunc : Instruction::ZExt;
Cost += Ctx.TTI.getCastInstrCost(
CastOpc, StepTy, IndexTy, TTI::CastContextHint::None, Ctx.CostKind);
}
diff --git a/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp b/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp
index 32ce8392ed08d..224511f2d26eb 100644
--- a/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp
+++ b/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp
@@ -1125,8 +1125,49 @@ optimizeLatchExitInductionUser(VPlan &Plan, VPValue *Op,
return nullptr;
}
+static VPValue *optimizeLatchExitIVUserViaSCEV(VPlan &Plan, VPValue *Op,
+ PredicatedScalarEvolution &PSE,
+ VPValue *ResumeTC,
+ const Loop *L) {
+ VPValue *Incoming;
+ if (!match(Op, m_ExtractLastLaneOfLastPart(m_VPValue(Incoming)))) {
+ VPValue *Mask;
+ if (!match(Op, m_ExtractLane(m_LastActiveLane(m_VPValue(Mask)),
+ m_VPValue(Incoming))) ||
+ !match(Mask, m_HeaderMask()))
+ return nullptr;
+ }
+
+ const SCEV *IncomingSCEV = vputils::getSCEVExprForVPValue(Incoming, PSE, L);
+ const SCEV *Start, *Step;
+ if (!match(IncomingSCEV, m_scev_AffineAddRec(m_SCEV(Start), m_SCEV(Step),
+ m_SpecificLoop(L))))
+ return nullptr;
+
+ auto *ExtractR = cast<VPInstruction>(Op);
+ DebugLoc DL = ExtractR->getDebugLoc();
+ VPBuilder Builder(ExtractR);
+ VPSCEVExpander Expander(Builder, *PSE.getSE(), DL);
+ VPValue *StartVPV = Expander.tryToExpand(Start);
+ VPValue *StepVPV = Expander.tryToExpand(Step);
+ if (!StartVPV || !StepVPV)
+ return nullptr;
+
+ Type *StartTy = StartVPV->getScalarType();
+ assert(StartTy->isIntOrPtrTy() && "The type must be SCEVable");
+ InductionDescriptor::InductionKind Kind =
+ StartTy->isPointerTy() ? InductionDescriptor::IK_PtrInduction
+ : InductionDescriptor::IK_IntInduction;
+ Type *TCTy = ResumeTC->getScalarType();
+ VPValue *ExitCount = Builder.createOverflowingOp(
+ Instruction::Sub, {ResumeTC, Plan.getConstantInt(TCTy, 1)},
+ {/*HasNUW=*/true, /*HasNSW=*/false}, DebugLoc::getUnknown());
+ return Builder.createDerivedIV(Kind, /*FPBinOp=*/nullptr, StartVPV, ExitCount,
+ StepVPV);
+}
+
void VPlanTransforms::optimizeInductionLiveOutUsers(
- VPlan &Plan, PredicatedScalarEvolution &PSE) {
+ VPlan &Plan, PredicatedScalarEvolution &PSE, const Loop *L) {
// Compute end values for all inductions.
VPRegionBlock *VectorRegion = Plan.getVectorLoopRegion();
auto *VectorPH = cast<VPBasicBlock>(VectorRegion->getSinglePredecessor());
@@ -1162,12 +1203,16 @@ void VPlanTransforms::optimizeInductionLiveOutUsers(
for (auto [Idx, PredVPBB] : enumerate(ExitVPBB->getPredecessors())) {
VPValue *Escape = nullptr;
- if (PredVPBB == MiddleVPBB)
+ if (PredVPBB == MiddleVPBB) {
Escape = optimizeLatchExitInductionUser(
Plan, ExitIRI->getOperand(Idx), EndValues, PSE);
- else
+ if (!Escape)
+ Escape = optimizeLatchExitIVUserViaSCEV(
+ Plan, ExitIRI->getOperand(Idx), PSE, ResumeTC, L);
+ } else {
Escape = optimizeEarlyExitInductionUser(
Plan, ExitIRI->getOperand(Idx), PSE);
+ }
if (Escape)
ExitIRI->setOperand(Idx, Escape);
}
@@ -3303,7 +3348,7 @@ static void expandVPDerivedIV(VPDerivedIVRecipe *R) {
VPValue *Index = R->getIndex();
Type *StepTy = Step->getScalarType();
Index = StepTy->isIntegerTy()
- ? Builder.createScalarSExtOrTrunc(
+ ? Builder.createScalarZExtOrTrunc(
Index, StepTy, DebugLoc::getCompilerGenerated())
: Builder.createScalarCast(Instruction::SIToFP, Index, StepTy,
DebugLoc::getCompilerGenerated());
diff --git a/llvm/lib/Transforms/Vectorize/VPlanTransforms.h b/llvm/lib/Transforms/Vectorize/VPlanTransforms.h
index 85375625d34b5..2eb5c1fd5b34b 100644
--- a/llvm/lib/Transforms/Vectorize/VPlanTransforms.h
+++ b/llvm/lib/Transforms/Vectorize/VPlanTransforms.h
@@ -416,7 +416,8 @@ struct VPlanTransforms {
/// IV values by feeding them precomputed end values instead, possibly taken
/// one step backwards.
static void optimizeInductionLiveOutUsers(VPlan &Plan,
- PredicatedScalarEvolution &PSE);
+ PredicatedScalarEvolution &PSE,
+ const Loop *L);
/// Add explicit broadcasts for live-ins and VPValues defined in \p Plan's entry block if they are used as vectors.
static void materializeBroadcasts(VPlan &Plan);
diff --git a/llvm/test/Transforms/LoopVectorize/AArch64/reduction-cost.ll b/llvm/test/Transforms/LoopVectorize/AArch64/reduction-cost.ll
index 230a9d7325bc5..7f21fdeb2b766 100644
--- a/llvm/test/Transforms/LoopVectorize/AArch64/reduction-cost.ll
+++ b/llvm/test/Transforms/LoopVectorize/AArch64/reduction-cost.ll
@@ -9,35 +9,27 @@ define i64 @reduction(i64 %arg) #0 {
; CHECK: [[VECTOR_PH]]:
; CHECK-NEXT: br i1 false, label %[[VEC_EPILOG_PH:.*]], label %[[VECTOR_PH1:.*]]
; CHECK: [[VECTOR_PH1]]:
-; CHECK-NEXT: [[BROADCAST_SPLATINSERT:%.*]] = insertelement <4 x i64> poison, i64 [[ARG]], i64 0
-; CHECK-NEXT: [[BROADCAST_SPLAT:%.*]] = shufflevector <4 x i64> [[BROADCAST_SPLATINSERT]], <4 x i64> poison, <4 x i32> zeroinitializer
; CHECK-NEXT: br label %[[VECTOR_BODY:.*]]
; CHECK: [[VECTOR_BODY]]:
; CHECK-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH1]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
-; CHECK-NEXT: [[VEC_IND:%.*]] = phi <4 x i64> [ <i64 0, i64 1, i64 2, i64 3>, %[[VECTOR_PH1]] ], [ [[VEC_IND_NEXT:%.*]], %[[VECTOR_BODY]] ]
; CHECK-NEXT: [[VEC_PHI:%.*]] = phi <4 x i32> [ zeroinitializer, %[[VECTOR_PH1]] ], [ [[TMP5:%.*]], %[[VECTOR_BODY]] ]
; CHECK-NEXT: [[VEC_PHI2:%.*]] = phi <4 x i32> [ zeroinitializer, %[[VECTOR_PH1]] ], [ [[TMP1:%.*]], %[[VECTOR_BODY]] ]
; CHECK-NEXT: [[VEC_PHI4:%.*]] = phi <4 x i32> [ zeroinitializer, %[[VECTOR_PH1]] ], [ [[TMP16:%.*]], %[[VECTOR_BODY]] ]
; CHECK-NEXT: [[VEC_PHI3:%.*]] = phi <4 x i32> [ zeroinitializer, %[[VECTOR_PH1]] ], [ [[TMP17:%.*]], %[[VECTOR_BODY]] ]
-; CHECK-NEXT: [[STEP_ADD:%.*]] = add nuw <4 x i64> [[VEC_IND]], splat (i64 4)
-; CHECK-NEXT: [[STEP_ADD_2:%.*]] = add nuw <4 x i64> [[STEP_ADD]], splat (i64 4)
-; CHECK-NEXT: [[STEP_ADD_3:%.*]] = add nuw <4 x i64> [[STEP_ADD_2]], splat (i64 4)
; CHECK-NEXT: [[TMP5]] = or <4 x i32> [[VEC_PHI]], splat (i32 1)
; CHECK-NEXT: [[TMP1]] = or <4 x i32> [[VEC_PHI2]], splat (i32 1)
; CHECK-NEXT: [[TMP16]] = or <4 x i32> [[VEC_PHI4]], splat (i32 1)
; CHECK-NEXT: [[TMP17]] = or <4 x i32> [[VEC_PHI3]], splat (i32 1)
; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 16
-; CHECK-NEXT: [[VEC_IND_NEXT]] = add nsw <4 x i64> [[STEP_ADD_3]], splat (i64 4)
; CHECK-NEXT: [[TMP6:%.*]] = icmp eq i64 [[INDEX_NEXT]], 96
; CHECK-NEXT: br i1 [[TMP6]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP0:![0-9]+]]
; CHECK: [[MIDDLE_BLOCK]]:
-; CHECK-NEXT: [[TMP18:%.*]] = add nsw <4 x i64> [[STEP_ADD_3]], splat (i64 1)
-; CHECK-NEXT: [[TMP19:%.*]] = mul nsw <4 x i64> [[TMP18]], [[BROADCAST_SPLAT]]
; CHECK-NEXT: [[BIN_RDX1:%.*]] = or <4 x i32> [[TMP1]], [[TMP5]]
; CHECK-NEXT: [[BIN_RDX4:%.*]] = or <4 x i32> [[TMP16]], [[BIN_RDX1]]
; CHECK-NEXT: [[BIN_RDX5:%.*]] = or <4 x i32> [[TMP17]], [[BIN_RDX4]]
; CHECK-NEXT: [[TMP7:%.*]] = call i32 @llvm.vector.reduce.or.v4i32(<4 x i32> [[BIN_RDX5]])
-; CHECK-NEXT: [[TMP8:%.*]] = extractelement <4 x i64> [[TMP19]], i64 3
+; CHECK-NEXT: [[TMP8:%.*]] = mul i64 95, [[ARG]]
+; CHECK-NEXT: [[TMP15:%.*]] = add i64 [[ARG]], [[TMP8]]
; CHECK-NEXT: br i1 false, label %[[EXIT:.*]], label %[[VEC_EPILOG_ITER_CHECK:.*]]
; CHECK: [[VEC_EPILOG_ITER_CHECK]]:
; CHECK-NEXT: br i1 false, label %[[VEC_EPILOG_SCALAR_PH]], label %[[VEC_EPILOG_PH]], !prof [[PROF3:![0-9]+]]
@@ -45,41 +37,33 @@ define i64 @reduction(i64 %arg) #0 {
; CHECK-NEXT: [[VEC_EPILOG_RESUME_VAL:%.*]] = phi i64 [ 96, %[[VEC_EPILOG_ITER_CHECK]] ], [ 0, %[[VECTOR_PH]] ]
; CHECK-NEXT: [[BC_MERGE_RDX:%.*]] = phi i32 [ [[TMP7]], %[[VEC_EPILOG_ITER_CHECK]] ], [ 0, %[[VECTOR_PH]] ]
; CHECK-NEXT: [[TMP9:%.*]] = insertelement <4 x i32> zeroinitializer, i32 [[BC_MERGE_RDX]], i32 0
-; CHECK-NEXT: [[BROADCAST_SPLATINSERT6:%.*]] = insertelement <4 x i64> poison, i64 [[ARG]], i64 0
-; CHECK-NEXT: [[BROADCAST_SPLAT7:%.*]] = shufflevector <4 x i64> [[BROADCAST_SPLATINSERT6]], <4 x i64> poison, <4 x i32> zeroinitializer
-; CHECK-NEXT: [[BROADCAST_SPLATINSERT8:%.*]] = insertelement <4 x i64> poison, i64 [[VEC_EPILOG_RESUME_VAL]], i64 0
-; CHECK-NEXT: [[BROADCAST_SPLAT9:%.*]] = shufflevector <4 x i64> [[BROADCAST_SPLATINSERT8]], <4 x i64> poison, <4 x i32> zeroinitializer
-; CHECK-NEXT: [[INDUCTION:%.*]] = add nsw <4 x i64> [[BROADCAST_SPLAT9]], <i64 0, i64 1, i64 2, i64 3>
; CHECK-NEXT: br label %[[SCALAR_PH:.*]]
; CHECK: [[SCALAR_PH]]:
; CHECK-NEXT: [[INDEX10:%.*]] = phi i64 [ [[VEC_EPILOG_RESUME_VAL]], %[[VEC_EPILOG_PH]] ], [ [[INDEX_NEXT13:%.*]], %[[SCALAR_PH]] ]
-; CHECK-NEXT: [[VEC_IND11:%.*]] = phi <4 x i64> [ [[INDUCTION]], %[[VEC_EPILOG_PH]] ], [ [[VEC_IND_NEXT14:%.*]], %[[SCALAR_PH]] ]
; CHECK-NEXT: [[VEC_PHI12:%.*]] = phi <4 x i32> [ [[TMP9]], %[[VEC_EPILOG_PH]] ], [ [[TMP10:%.*]], %[[SCALAR_PH]] ]
; CHECK-NEXT: [[TMP10]] = or <4 x i32> [[VEC_PHI12]], splat (i32 1)
; CHECK-NEXT: [[INDEX_NEXT13]] = add nuw i64 [[INDEX10]], 4
-; CHECK-NEXT: [[VEC_IND_NEXT14]] = add nsw <4 x i64> [[VEC_IND11]], splat (i64 4)
; CHECK-NEXT: [[TMP11:%.*]] = icmp eq i64 [[INDEX_NEXT13]], 100
; CHECK-NEXT: br i1 [[TMP11]], label %[[VEC_EPILOG_MIDDLE_BLOCK:.*]], label %[[SCALAR_PH]], !llvm.loop [[LOOP4:![0-9]+]]
; CHECK: [[VEC_EPILOG_MIDDLE_BLOCK]]:
-; CHECK-NEXT: [[TMP12:%.*]] = add nsw <4 x i64> [[VEC_IND11]], splat (i64 1)
-; CHECK-NEXT: [[TMP13:%.*]] = mul nsw <4 x i64> [[TMP12]], [[BROADCAST_SPLAT7]]
; CHECK-NEXT: [[TMP14:%.*]] = call i32 @llvm.vector.reduce.or.v4i32(<4 x i32> [[TMP10]])
-; CHECK-NEXT: [[TMP15:%.*]] = extractelement <4 x i64> [[TMP13]], i64 3
+; CHECK-NEXT: [[TMP12:%.*]] = mul i64 99, [[ARG]]
+; CHECK-NEXT: [[TMP13:%.*]] = add i64 [[ARG]], [[TMP12]]
; CHECK-NEXT: br i1 true, label %[[EXIT]], label %[[VEC_EPILOG_SCALAR_PH]]
; CHECK: [[VEC_EPILOG_SCALAR_PH]]:
; CHECK-NEXT: [[BC_RESUME_VAL:%.*]] = phi i64 [ 100, %[[VEC_EPILOG_MIDDLE_BLOCK]] ], [ 96, %[[VEC_EPILOG_ITER_CHECK]] ], [ 0, %[[ENTRY]] ]
-; CHECK-NEXT: [[BC_MERGE_RDX15:%.*]] = phi i32 [ [[TMP14]], %[[VEC_EPILOG_MIDDLE_BLOCK]] ], [ [[TMP7]], %[[VEC_EPILOG_ITER_CHECK]] ], [ 0, %[[ENTRY]] ]
+; CHECK-NEXT: [[BC_MERGE_RDX9:%.*]] = phi i32 [ [[TMP14]], %[[VEC_EPILOG_MIDDLE_BLOCK]] ], [ [[TMP7]], %[[VEC_EPILOG_ITER_CHECK]] ], [ 0, %[[ENTRY]] ]
; CHECK-NEXT: br label %[[LOOP:.*]]
; CHECK: [[LOOP]]:
; CHECK-NEXT: [[TMP0:%.*]] = phi i64 [ [[BC_RESUME_VAL]], %[[VEC_EPILOG_SCALAR_PH]] ], [ [[TMP3:%.*]], %[[LOOP]] ]
-; CHECK-NEXT: [[VEC_PHI1:%.*]] = phi i32 [ [[BC_MERGE_RDX15]], %[[VEC_EPILOG_SCALAR_PH]] ], [ [[TMP2:%.*]], %[[LOOP]] ]
+; CHECK-NEXT: [[VEC_PHI1:%.*]] = phi i32 [ [[BC_MERGE_RDX9]], %[[VEC_EPILOG_SCALAR_PH]] ], [ [[TMP2:%.*]], %[[LOOP]] ]
; CHECK-NEXT: [[TMP2]] = or i32 [[VEC_PHI1]], 1
; CHECK-NEXT: [[TMP3]] = add nsw i64 [[TMP0]], 1
; CHECK-NEXT: [[TMP4:%.*]] = mul nsw i64 [[TMP3]], [[ARG]]
; CHECK-NEXT: [[CMP:%.*]] = icmp ne i64 [[TMP3]], 100
; CHECK-NEXT: br i1 [[CMP]], label %[[LOOP]], label %[[EXIT]], !llvm.loop [[LOOP5:![0-9]+]]
; CHECK: [[EXIT]]:
-; CHECK-NEXT: [[MUL_LCSSA:%.*]] = phi i64 [ [[TMP4]], %[[LOOP]] ], [ [[TMP8]], %[[MIDDLE_BLOCK]] ], [ [[TMP15]], %[[VEC_EPILOG_MIDDLE_BLOCK]] ]
+; CHECK-NEXT: [[MUL_LCSSA:%.*]] = phi i64 [ [[TMP4]], %[[LOOP]] ], [ [[TMP15]], %[[MIDDLE_BLOCK]] ], [ [[TMP13]], %[[VEC_EPILOG_MIDDLE_BLOCK]] ]
; CHECK-NEXT: [[BIN_RDX:%.*]] = phi i32 [ [[TMP2]], %[[LOOP]] ], [ [[TMP7]], %[[MIDDLE_BLOCK]] ], [ [[TMP14]], %[[VEC_EPILOG_MIDDLE_BLOCK]] ]
; CHECK-NEXT: [[EXT:%.*]] = zext i32 [[BIN_RDX]] to i64
; CHECK-NEXT: [[RES:%.*]] = add i64 [[EXT]], [[MUL_LCSSA]]
diff --git a/llvm/test/Transforms/LoopVectorize/AArch64/scalable-avoid-scalarization.ll b/llvm/test/Transforms/LoopVectorize/AArch64/scalable-avoid-scalarization.ll
index 856afaaa5c5fb..9e9274b17148d 100644
--- a/llvm/test/Transforms/LoopVectorize/AArch64/scalable-avoid-scalarization.ll
+++ b/llvm/test/Transforms/LoopVectorize/AArch64/scalable-avoid-scalarization.ll
@@ -44,10 +44,13 @@ define void @test_no_scalarization(ptr %a, ptr noalias %b, i32 %idx, i32 %n) #0
; CHECK-NEXT: [[TMP20:%.*]] = icmp eq i32 [[INDEX_NEXT]], [[N_VEC]]
; CHECK-NEXT: br i1 [[TMP20]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP0:![0-9]+]]
; CHECK: middle.block:
-; CHECK-NEXT: [[TMP12:%.*]] = call i32 @llvm.vscale.i32()
-; CHECK-NEXT: [[TMP13:%.*]] = mul nuw i32 [[TMP12]], 2
-; CHECK-NEXT: [[TMP14:%.*]] = sub i32 [[TMP13]], 1
-; CHECK-NEXT: [[TMP24:%.*]] = extractelement <vscale x 2 x ptr> [[TMP15]], i32 [[TMP14]]
+; CHECK-NEXT: [[TMP11:%.*]] = sext i32 [[IDX]] to i64
+; CHECK-NEXT: [[TMP12:%.*]] = shl nsw i64 [[TMP11]], 3
+; CHECK-NEXT: [[TMP13:%.*]] = getelementptr i8, ptr [[A]], i64 [[TMP12]]
+; CHECK-NEXT: [[TMP14:%.*]] = sub nuw i32 [[N_VEC]], 1
+; CHECK-NEXT: [[TMP17:%.*]] = zext i32 [[TMP14]] to i64
+; CHECK-NEXT: [[TMP19:%.*]] = shl i64 [[TMP17]], 3
+; CHECK-NEXT: [[TMP24:%.*]] = getelementptr i8, ptr [[TMP13]], i64 [[TMP19]]
; CHECK-NEXT: [[CMP_N:%.*]] = icmp eq i32 [[TMP1]], [[N_VEC]]
; CHECK-NEXT: br i1 [[CMP_N]], label [[L_EXIT:%.*]], label [[SCALAR_PH]]
; CHECK: scalar.ph:
diff --git a/llvm/test/Transforms/LoopVectorize/VPlan/buildvector-first-lane-only.ll b/llvm/test/Transforms/LoopVectorize/VPlan/buildvector-first-lane-only.ll
index ded3bc66016fe..6d15c43baac10 100644
--- a/llvm/test/Transforms/LoopVectorize/VPlan/buildvector-first-lane-only.ll
+++ b/llvm/test/Transforms/LoopVectorize/VPlan/buildvector-first-lane-only.ll
@@ -4,31 +4,22 @@
define i16 @last_active_lane_live_out(i32 %x) {
; CHECK-LABEL: VPlan for loop in 'last_active_lane_live_out'
; CHECK: VPlan 'Final VPlan for VF={4},UF={1}' {
-; CHECK-NEXT: Live-in ir<2> = original trip-count
; CHECK-EMPTY:
; CHECK-NEXT: ir-bb<entry>:
; CHECK-NEXT: Successor(s): vector.ph
; CHECK-EMPTY:
; CHECK-NEXT: vector.ph:
-; CHECK-NEXT: EMIT-SCALAR vp<[[VP2:%[0-9]+]]> = trunc ir<%x> to i16
-; CHECK-NEXT: EMIT vp<[[VP3:%[0-9]+]]> = step-vector i16
-; CHECK-NEXT: EMIT vp<[[VP4:%[0-9]+]]> = broadcast vp<[[VP2]]>
-; CHECK-NEXT: EMIT vp<[[VP5:%[0-9]+]]> = mul vp<[[VP3]]>, vp<[[VP4]]>
; CHECK-NEXT: Successor(s): vector.body
; CHECK-EMPTY:
; CHECK-NEXT: vector.body:
-; CHECK-NEXT: EMIT vp<%active.lane.mask> = active lane mask ir<0>, ir<2>, ir<1>
; CHECK-NEXT: Successor(s): middle.block
; CHECK-EMPTY:
; CHECK-NEXT: middle.block:
-; CHECK-NEXT: EMIT vp<[[VP6:%[0-9]+]]> = not vp<%active.lane.mask>
-; CHECK-NEXT: EMIT vp<%first.inactive.lane> = first-active-lane vp<[[VP6]]>
-; CHECK-NEXT: EMIT vp<%last.active.lane> = sub vp<%first.inactive.lane>, ir<1>
-; CHECK-NEXT: EMIT vp<[[VP7:%[0-9]+]]> = extractelement vp<[[VP5]]>, vp<%last.active.lane>
+; CHECK-NEXT: EMIT-SCALAR vp<[[VP2:%[0-9]+]]> = trunc ir<%x> to i16
; CHECK-NEXT: Successor(s): ir-bb<exit>
; CHECK-EMPTY:
; CHECK-NEXT: ir-bb<exit>:
-; CHECK-NEXT: IR %t.lcssa = phi i16 [ %t, %loop ] (extra operand: vp<[[VP7]]> from middle.block)
+; CHECK-NEXT: IR %t.lcssa = phi i16 [ %t, %loop ] (extra operand: vp<[[VP2]]> from middle.block)
; CHECK-NEXT: No successors
; CHECK-NEXT: }
;
diff --git a/llvm/test/Transforms/LoopVectorize/VPlan/vplan-printing.ll b/llvm/test/Transforms/LoopVectorize/VPlan/vplan-printing.ll
index 2e588784eaf00..54a8f4e068fee 100644
--- a/llvm/test/Transforms/LoopVectorize/VPlan/vplan-printing.ll
+++ b/llvm/test/Transforms/LoopVectorize/VPlan/vplan-printing.ll
@@ -479,26 +479,26 @@ define void @print_expand_scev(i64 %y, ptr %ptr) {
; CHECK-NEXT: Live-in vp<[[VP0:%[0-9]+]]> = VF
; CHECK-NEXT: Live-in vp<[[VP1:%[0-9]+]]> = VF * UF
; CHECK-NEXT: Live-in vp<[[VP2:%[0-9]+]]> = vector-trip-count
-; CHECK-NEXT: vp<[[VP3:%[0-9]+]]> = original trip-count
+; CHECK-NEXT: vp<[[VP4:%[0-9]+]]> = original trip-count
; CHECK-EMPTY:
-; CHECK-NEXT: ir-bb<entry>:
-; CHECK-NEXT: IR %div = udiv i64 %y, 492802768830814060
-; CHECK-NEXT: IR %inc = add i64 %div, 1
-; CHECK-NEXT: EMIT vp<[[VP4:%.+]]> = EXPAND SCEV (1 + (%y /u 492802768830814060))<nuw><nsw>
-; CHECK-NEXT: EMIT vp<[[VP3]]> = EXPAND SCEV (1 + ((15 + (%y /u 492802768830814060))<nuw><nsw> /u (1 + (%y /u 492802768830814060))<nuw><nsw>))<nuw><nsw>
-; CHECK-NEXT: Successor(s): scalar.ph, vector.ph
+; CHECK-NEXT: ir-bb<entry>:
+; CHECK-NEXT: IR %div = udiv i64 %y, 492802768830814060
+; CHECK-NEXT: IR %inc = add i64 %div, 1
+; CHECK-NEXT: EMIT vp<[[VP3:%[0-9]+]]> = EXPAND SCEV (1 + (%y /u 492802768830814060))<nuw><nsw>
+; CHECK-NEXT: EMIT vp<[[VP4]]> = EXPAND SCEV (1 + ((15 + (%y /u 492802768830814060))<nuw><nsw> /u (1 + (%y /u 492802768830814060))<nuw><nsw>))<nuw><nsw>
+; CHECK-NEXT: Successor(s): scalar.ph, vector.ph
; CHECK-EMPTY:
; CHECK-NEXT: vector.ph:
-; CHECK-NEXT: vp<[[VP5:%[0-9]+]]> = DERIVED-IV ir<0> + vp<[[VP2]]> * vp<[[VP4]]>
+; CHECK-NEXT: vp<[[VP5:%[0-9]+]]> = DERIVED-IV ir<0> + vp<[[VP2]]> * vp<[[VP3]]>
; CHECK-NEXT: Successor(s): vector loop
; CHECK-EMPTY:
; CHECK-NEXT: <x1> vector loop: {
; CHECK-NEXT: vp<[[VP6:%[0-9]+]]> = CANONICAL-IV
; CHECK-EMPTY:
; CHECK-NEXT: vector.body:
-; CHECK-NEXT: ir<%iv> = WIDEN-INDUCTION ir<0>, vp<[[VP4]]>, vp<[[VP0]]> (truncated to i8)
-; CHECK-NEXT: vp<[[VP7:%[0-9]+]]> = DERIVED-IV ir<0> + vp<[[VP6]]> * vp<[[VP4]]>
-; CHECK-NEXT: vp<[[VP8:%[0-9]+]]> = SCALAR-STEPS vp<[[VP7]]>, vp<[[VP4]]>, vp<[[VP0]]>
+; CHECK-NEXT: ir<%iv> = WIDEN-INDUCTION ir<0>, vp<[[VP3]]>, vp<[[VP0]]> (truncated to i8)
+; CHECK-NEXT: vp<[[VP7:%[0-9]+]]> = DERIVED-IV ir<0> + vp<[[VP6]]> * vp<[[VP3]]>
+; CHECK-...
[truncated]
``````````
</details>
https://github.com/llvm/llvm-project/pull/210928
More information about the llvm-commits
mailing list