[llvm] 4cdb033 - [VPlan] Optimize pre-increment IV latch users with tail folding (#206499)
via llvm-commits
llvm-commits at lists.llvm.org
Fri Jul 3 06:29:49 PDT 2026
Author: Luke Lau
Date: 2026-07-03T13:29:43Z
New Revision: 4cdb0335647089e12cb7b8d675871bb7086ed189
URL: https://github.com/llvm/llvm-project/commit/4cdb0335647089e12cb7b8d675871bb7086ed189
DIFF: https://github.com/llvm/llvm-project/commit/4cdb0335647089e12cb7b8d675871bb7086ed189.diff
LOG: [VPlan] Optimize pre-increment IV latch users with tail folding (#206499)
This was noticed after #204089 caused IndVarsSimplify to convert some
live out IV users to use the pre-incremented IV, not the
post-incremented.
Tail folded live-outs don't have the `(extract-last-lane
(extract-last-part foo))` form, but instead have the form `(extract-lane
(last-active-lane header-mask), foo)`.
For post-incremented IVs in tail folding, these are converted to
VPInstruction::ExitingIVValue which are handled separately. But
ExitingIVValue can't be used for the pre-incremented IV. So this teaches
optimizeLatchExitInductionUser to detect the last-active-lane of the
header mask form.
Added:
llvm/test/Transforms/LoopVectorize/RISCV/tail-folding-iv-outside-user.ll
Modified:
llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp
llvm/test/Transforms/LoopVectorize/VPlan/buildvector-first-lane-only.ll
llvm/test/Transforms/LoopVectorize/no-fold-tail-by-masking-iv-external-uses.ll
llvm/test/Transforms/LoopVectorize/tail-folding-iv-outside-user.ll
llvm/test/Transforms/LoopVectorize/vector-loop-backedge-elimination-tail-folding.ll
Removed:
################################################################################
diff --git a/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp b/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp
index 3e6761a161ad0..e5d16ccbec434 100644
--- a/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp
+++ b/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp
@@ -1105,8 +1105,14 @@ optimizeLatchExitInductionUser(VPlan &Plan, VPValue *Op,
DenseMap<VPValue *, VPValue *> &EndValues,
PredicatedScalarEvolution &PSE) {
VPValue *Incoming;
- if (!match(Op, m_ExtractLastLaneOfLastPart(m_VPValue(Incoming))))
- return nullptr;
+ if (!match(Op,
+ m_CombineOr(m_ExtractLastLaneOfLastPart(m_VPValue(Incoming))))) {
+ VPValue *Mask;
+ if (!match(Op, m_ExtractLane(m_LastActiveLane(m_VPValue(Mask)),
+ m_VPValue(Incoming))) ||
+ Mask != vputils::findHeaderMask(Plan))
+ return nullptr;
+ }
VPWidenInductionRecipe *WideIV = getOptimizableIVOf(Incoming, PSE);
if (!WideIV)
diff --git a/llvm/test/Transforms/LoopVectorize/RISCV/tail-folding-iv-outside-user.ll b/llvm/test/Transforms/LoopVectorize/RISCV/tail-folding-iv-outside-user.ll
new file mode 100644
index 0000000000000..4e083dd9ffe76
--- /dev/null
+++ b/llvm/test/Transforms/LoopVectorize/RISCV/tail-folding-iv-outside-user.ll
@@ -0,0 +1,66 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --check-globals none --version 6
+; RUN: opt < %s -S -p loop-vectorize -mtriple riscv64 -mattr=+v | FileCheck %s
+
+define i32 @f(ptr noalias %p, i32 %start, i32 %step, i32 %n) {
+; CHECK-LABEL: define i32 @f(
+; CHECK-SAME: ptr noalias [[P:%.*]], i32 [[START:%.*]], i32 [[STEP:%.*]], i32 [[N:%.*]]) #[[ATTR0:[0-9]+]] {
+; CHECK-NEXT: [[ENTRY:.*:]]
+; CHECK-NEXT: [[TMP0:%.*]] = add i32 [[N]], 1
+; CHECK-NEXT: br label %[[LOOP:.*]]
+; CHECK: [[LOOP]]:
+; CHECK-NEXT: [[TMP1:%.*]] = icmp slt i32 [[N]], 0
+; CHECK-NEXT: br i1 [[TMP1]], label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]]
+; CHECK: [[VECTOR_PH]]:
+; CHECK-NEXT: [[TMP2:%.*]] = mul i32 [[TMP0]], [[STEP]]
+; CHECK-NEXT: [[TMP5:%.*]] = add i32 [[START]], [[TMP2]]
+; CHECK-NEXT: br label %[[EXIT:.*]]
+; CHECK: [[EXIT]]:
+; CHECK-NEXT: [[INDEX:%.*]] = phi i32 [ 0, %[[VECTOR_PH]] ], [ [[CURRENT_ITERATION_NEXT:%.*]], %[[EXIT]] ]
+; CHECK-NEXT: [[AVL:%.*]] = phi i32 [ [[TMP0]], %[[VECTOR_PH]] ], [ [[AVL_NEXT:%.*]], %[[EXIT]] ]
+; CHECK-NEXT: [[TMP3:%.*]] = call i32 @llvm.experimental.get.vector.length.i32(i32 [[AVL]], i32 4, i1 true)
+; CHECK-NEXT: [[TMP6:%.*]] = getelementptr i32, ptr [[P]], i32 [[INDEX]]
+; CHECK-NEXT: [[VP_OP_LOAD:%.*]] = call <vscale x 4 x i32> @llvm.vp.load.nxv4i32.p0(ptr align 4 [[TMP6]], <vscale x 4 x i1> splat (i1 true), i32 [[TMP3]])
+; CHECK-NEXT: [[TMP7:%.*]] = add <vscale x 4 x i32> [[VP_OP_LOAD]], splat (i32 1)
+; CHECK-NEXT: call void @llvm.vp.store.nxv4i32.p0(<vscale x 4 x i32> [[TMP7]], ptr align 4 [[TMP6]], <vscale x 4 x i1> splat (i1 true), i32 [[TMP3]])
+; CHECK-NEXT: [[CURRENT_ITERATION_NEXT]] = add i32 [[TMP3]], [[INDEX]]
+; CHECK-NEXT: [[AVL_NEXT]] = sub nuw i32 [[AVL]], [[TMP3]]
+; CHECK-NEXT: [[TMP4:%.*]] = icmp eq i32 [[AVL_NEXT]], 0
+; CHECK-NEXT: br i1 [[TMP4]], label %[[MIDDLE_BLOCK:.*]], label %[[EXIT]], !llvm.loop [[LOOP0:![0-9]+]]
+; CHECK: [[MIDDLE_BLOCK]]:
+; CHECK-NEXT: [[IND_ESCAPE:%.*]] = sub i32 [[TMP5]], [[STEP]]
+; CHECK-NEXT: br label %[[EXIT1:.*]]
+; CHECK: [[SCALAR_PH]]:
+; CHECK-NEXT: br label %[[LOOP1:.*]]
+; CHECK: [[LOOP1]]:
+; CHECK-NEXT: [[IV:%.*]] = phi i32 [ 0, %[[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], %[[LOOP1]] ]
+; CHECK-NEXT: [[IV2:%.*]] = phi i32 [ [[START]], %[[SCALAR_PH]] ], [ [[IV2_NEXT:%.*]], %[[LOOP1]] ]
+; CHECK-NEXT: [[GEP:%.*]] = getelementptr i32, ptr [[P]], i32 [[IV]]
+; CHECK-NEXT: [[X:%.*]] = load i32, ptr [[GEP]], align 4
+; CHECK-NEXT: [[Y:%.*]] = add i32 [[X]], 1
+; CHECK-NEXT: store i32 [[Y]], ptr [[GEP]], align 4
+; CHECK-NEXT: [[IV_NEXT]] = add i32 [[IV]], 1
+; CHECK-NEXT: [[IV2_NEXT]] = add i32 [[IV2]], [[STEP]]
+; CHECK-NEXT: [[EC:%.*]] = icmp eq i32 [[IV]], [[N]]
+; CHECK-NEXT: br i1 [[EC]], label %[[EXIT1]], label %[[LOOP1]], !llvm.loop [[LOOP3:![0-9]+]]
+; CHECK: [[EXIT1]]:
+; CHECK-NEXT: [[IV2_LCSSA:%.*]] = phi i32 [ [[IV2]], %[[LOOP1]] ], [ [[IND_ESCAPE]], %[[MIDDLE_BLOCK]] ]
+; CHECK-NEXT: ret i32 [[IV2_LCSSA]]
+;
+entry:
+ br label %loop
+
+loop:
+ %iv = phi i32 [0, %entry], [%iv.next, %loop]
+ %iv2 = phi i32 [%start, %entry], [%iv2.next, %loop]
+ %gep = getelementptr i32, ptr %p, i32 %iv
+ %x = load i32, ptr %gep
+ %y = add i32 %x, 1
+ store i32 %y, ptr %gep
+ %iv.next = add i32 %iv, 1
+ %iv2.next = add i32 %iv2, %step
+ %ec = icmp eq i32 %iv, %n
+ br i1 %ec, label %exit, label %loop
+
+exit:
+ ret i32 %iv2
+}
diff --git a/llvm/test/Transforms/LoopVectorize/VPlan/buildvector-first-lane-only.ll b/llvm/test/Transforms/LoopVectorize/VPlan/buildvector-first-lane-only.ll
index 54614ad7c2035..04f3aa3accbc8 100644
--- a/llvm/test/Transforms/LoopVectorize/VPlan/buildvector-first-lane-only.ll
+++ b/llvm/test/Transforms/LoopVectorize/VPlan/buildvector-first-lane-only.ll
@@ -1,7 +1,7 @@
; NOTE: Assertions have been autogenerated by utils/update_analyze_test_checks.py UTC_ARGS: --version 6
; RUN: opt < %s -p loop-vectorize -force-vector-width=4 -tail-folding-policy=must-fold-tail -force-tail-folding-style=data -vplan-print-after=printFinalVPlan -disable-output 2>&1 | FileCheck %s
-define i32 @last_active_lane_live_out(i32 %x) {
+define i16 @last_active_lane_live_out(i32 %x) {
; CHECK-LABEL: VPlan for loop in 'last_active_lane_live_out'
; CHECK: VPlan 'Final VPlan for VF={4},UF={1}' {
; CHECK-NEXT: Live-in ir<2> = original trip-count
@@ -10,28 +10,29 @@ define i32 @last_active_lane_live_out(i32 %x) {
; CHECK-NEXT: Successor(s): vector.ph
; CHECK-EMPTY:
; CHECK-NEXT: vector.ph:
-; CHECK-NEXT: EMIT vp<[[VP2:%[0-9]+]]> = step-vector i32
-; CHECK-NEXT: EMIT vp<[[VP3:%[0-9]+]]> = broadcast ir<%x>
-; CHECK-NEXT: EMIT vp<[[VP4:%[0-9]+]]> = mul vp<[[VP2]]>, vp<[[VP3]]>
+; CHECK-NEXT: EMIT-SCALAR vp<[[VP2:%[0-9]+]]> = trunc ir<%x> to i16
+; CHECK-NEXT: EMIT vp<[[VP3:%[0-9]+]]> = step-vector i16
+; CHECK-NEXT: EMIT vp<[[VP4:%[0-9]+]]> = broadcast vp<[[VP2]]>
+; CHECK-NEXT: EMIT vp<[[VP5:%[0-9]+]]> = mul vp<[[VP3]]>, vp<[[VP4]]>
; CHECK-NEXT: Successor(s): vector.body
; CHECK-EMPTY:
; CHECK-NEXT: vector.body:
; CHECK-NEXT: Successor(s): middle.block
; CHECK-EMPTY:
; CHECK-NEXT: middle.block:
-; CHECK-NEXT: vp<[[VP5:%[0-9]+]]> = SCALAR-STEPS ir<0>, ir<1>, ir<4>, ir<1>
-; CHECK-NEXT: vp<[[VP6:%[0-9]+]]> = SCALAR-STEPS ir<0>, ir<1>, ir<4>, ir<2>
-; CHECK-NEXT: vp<[[VP7:%[0-9]+]]> = SCALAR-STEPS ir<0>, ir<1>, ir<4>, ir<3>
-; CHECK-NEXT: EMIT vp<[[VP8:%[0-9]+]]> = buildvector ir<0>, vp<[[VP5]]>, vp<[[VP6]]>, vp<[[VP7]]>
-; CHECK-NEXT: EMIT vp<%active.lane.mask> = active lane mask vp<[[VP8]]>, ir<2>, ir<1>
-; CHECK-NEXT: EMIT vp<[[VP9:%[0-9]+]]> = not vp<%active.lane.mask>
-; CHECK-NEXT: EMIT vp<%first.inactive.lane> = first-active-lane vp<[[VP9]]>
+; CHECK-NEXT: vp<[[VP6:%[0-9]+]]> = SCALAR-STEPS ir<0>, ir<1>, ir<4>, ir<1>
+; CHECK-NEXT: vp<[[VP7:%[0-9]+]]> = SCALAR-STEPS ir<0>, ir<1>, ir<4>, ir<2>
+; CHECK-NEXT: vp<[[VP8:%[0-9]+]]> = SCALAR-STEPS ir<0>, ir<1>, ir<4>, ir<3>
+; CHECK-NEXT: EMIT vp<[[VP9:%[0-9]+]]> = buildvector ir<0>, vp<[[VP6]]>, vp<[[VP7]]>, vp<[[VP8]]>
+; CHECK-NEXT: EMIT vp<%active.lane.mask> = active lane mask vp<[[VP9]]>, ir<2>, ir<1>
+; CHECK-NEXT: EMIT vp<[[VP10:%[0-9]+]]> = not vp<%active.lane.mask>
+; CHECK-NEXT: EMIT vp<%first.inactive.lane> = first-active-lane vp<[[VP10]]>
; CHECK-NEXT: EMIT vp<%last.active.lane> = sub vp<%first.inactive.lane>, ir<1>
-; CHECK-NEXT: EMIT vp<[[VP10:%[0-9]+]]> = extractelement vp<[[VP4]]>, vp<%last.active.lane>
+; CHECK-NEXT: EMIT vp<[[VP11:%[0-9]+]]> = extractelement vp<[[VP5]]>, vp<%last.active.lane>
; CHECK-NEXT: Successor(s): ir-bb<exit>
; CHECK-EMPTY:
; CHECK-NEXT: ir-bb<exit>:
-; CHECK-NEXT: IR %red.lcssa = phi i32 [ %red, %loop ] (extra operand: vp<[[VP10]]> from middle.block)
+; CHECK-NEXT: IR %t.lcssa = phi i16 [ %t, %loop ] (extra operand: vp<[[VP11]]> from middle.block)
; CHECK-NEXT: No successors
; CHECK-NEXT: }
;
@@ -41,11 +42,12 @@ entry:
loop:
%iv = phi i64 [ 0, %entry ], [ %iv.next, %loop ]
%red = phi i32 [ 0, %entry ], [ %add, %loop ]
+ %t = trunc i32 %red to i16 ; stops optimizeInductionLiveOutUsers from folding buildvector
%add = add i32 %red, %x
%iv.next = add i64 %iv, 1
%ec = icmp eq i64 %iv, 1
br i1 %ec, label %exit, label %loop
exit:
- ret i32 %red
+ ret i16 %t
}
diff --git a/llvm/test/Transforms/LoopVectorize/no-fold-tail-by-masking-iv-external-uses.ll b/llvm/test/Transforms/LoopVectorize/no-fold-tail-by-masking-iv-external-uses.ll
index 9da8f897ed9e5..93c81c02d4e5d 100644
--- a/llvm/test/Transforms/LoopVectorize/no-fold-tail-by-masking-iv-external-uses.ll
+++ b/llvm/test/Transforms/LoopVectorize/no-fold-tail-by-masking-iv-external-uses.ll
@@ -25,6 +25,7 @@ define i32 @test(ptr %arr, i64 %n) {
; CHECK-NEXT: [[N_MOD_VF:%.*]] = urem i64 [[N_RND_UP]], 4
; CHECK-NEXT: [[N_VEC:%.*]] = sub i64 [[N_RND_UP]], [[N_MOD_VF]]
; CHECK-NEXT: [[TRIP_COUNT_MINUS_1:%.*]] = sub i64 [[TMP0]], 1
+; CHECK-NEXT: [[IND_END:%.*]] = add i64 1, [[TMP0]]
; CHECK-NEXT: [[BROADCAST_SPLATINSERT:%.*]] = insertelement <4 x i64> poison, i64 [[TRIP_COUNT_MINUS_1]], i64 0
; CHECK-NEXT: [[BROADCAST_SPLAT:%.*]] = shufflevector <4 x i64> [[BROADCAST_SPLATINSERT]], <4 x i64> poison, <4 x i32> zeroinitializer
; CHECK-NEXT: br label [[VECTOR_BODY:%.*]]
@@ -72,10 +73,7 @@ define i32 @test(ptr %arr, i64 %n) {
; CHECK-NEXT: [[TMP20:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
; CHECK-NEXT: br i1 [[TMP20]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP0:![0-9]+]]
; CHECK: middle.block:
-; CHECK-NEXT: [[TMP22:%.*]] = xor <4 x i1> [[TMP11]], splat (i1 true)
-; CHECK-NEXT: [[IND_END:%.*]] = call i64 @llvm.experimental.cttz.elts.i64.v4i1(<4 x i1> [[TMP22]], i1 false)
; CHECK-NEXT: [[IND_ESCAPE:%.*]] = sub i64 [[IND_END]], 1
-; CHECK-NEXT: [[TMP23:%.*]] = extractelement <4 x i64> [[VEC_IND]], i64 [[IND_ESCAPE]]
; CHECK-NEXT: br label [[LOAD_VAL:%.*]]
; CHECK: scalar.ph:
; CHECK-NEXT: br label [[LOOP:%.*]]
@@ -90,7 +88,7 @@ define i32 @test(ptr %arr, i64 %n) {
; CHECK-NEXT: [[CMP2:%.*]] = icmp ult i64 [[CONV2]], [[N]]
; CHECK-NEXT: br i1 [[CMP2]], label [[LOOP]], label [[LOAD_VAL]], !llvm.loop [[LOOP4:![0-9]+]]
; CHECK: load_val:
-; CHECK-NEXT: [[FINAL:%.*]] = phi i64 [ [[CONV]], [[LOOP]] ], [ [[TMP23]], [[MIDDLE_BLOCK]] ]
+; CHECK-NEXT: [[FINAL:%.*]] = phi i64 [ [[CONV]], [[LOOP]] ], [ [[IND_ESCAPE]], [[MIDDLE_BLOCK]] ]
; CHECK-NEXT: [[PTR2:%.*]] = getelementptr inbounds i32, ptr [[ARR]], i64 [[FINAL]]
; CHECK-NEXT: [[VAL:%.*]] = load i32, ptr [[PTR2]], align 4
; CHECK-NEXT: br label [[DONE]]
diff --git a/llvm/test/Transforms/LoopVectorize/tail-folding-iv-outside-user.ll b/llvm/test/Transforms/LoopVectorize/tail-folding-iv-outside-user.ll
index e2e936216ea8f..35748bb01d394 100644
--- a/llvm/test/Transforms/LoopVectorize/tail-folding-iv-outside-user.ll
+++ b/llvm/test/Transforms/LoopVectorize/tail-folding-iv-outside-user.ll
@@ -11,33 +11,16 @@ define i32 @f(ptr noalias %p, i32 %start, i32 %step, i32 %n) {
; CHECK-NEXT: [[N_RND_UP:%.*]] = add i32 [[TMP0]], 3
; CHECK-NEXT: [[N_MOD_VF:%.*]] = urem i32 [[N_RND_UP]], 4
; CHECK-NEXT: [[N_VEC:%.*]] = sub i32 [[N_RND_UP]], [[N_MOD_VF]]
-; CHECK-NEXT: [[TRIP_COUNT_MINUS_1:%.*]] = sub i32 [[TMP0]], 1
-; CHECK-NEXT: [[BROADCAST_SPLATINSERT:%.*]] = insertelement <4 x i32> poison, i32 [[TRIP_COUNT_MINUS_1]], i64 0
-; CHECK-NEXT: [[BROADCAST_SPLAT:%.*]] = shufflevector <4 x i32> [[BROADCAST_SPLATINSERT]], <4 x i32> poison, <4 x i32> zeroinitializer
-; CHECK-NEXT: [[BROADCAST_SPLATINSERT1:%.*]] = insertelement <4 x i32> poison, i32 [[START]], i64 0
-; CHECK-NEXT: [[BROADCAST_SPLAT2:%.*]] = shufflevector <4 x i32> [[BROADCAST_SPLATINSERT1]], <4 x i32> poison, <4 x i32> zeroinitializer
-; CHECK-NEXT: [[BROADCAST_SPLATINSERT3:%.*]] = insertelement <4 x i32> poison, i32 [[STEP]], i64 0
-; CHECK-NEXT: [[BROADCAST_SPLAT4:%.*]] = shufflevector <4 x i32> [[BROADCAST_SPLATINSERT3]], <4 x i32> poison, <4 x i32> zeroinitializer
-; CHECK-NEXT: [[TMP1:%.*]] = mul <4 x i32> <i32 0, i32 1, i32 2, i32 3>, [[BROADCAST_SPLAT4]]
-; CHECK-NEXT: [[INDUCTION:%.*]] = add <4 x i32> [[BROADCAST_SPLAT2]], [[TMP1]]
-; CHECK-NEXT: [[TMP2:%.*]] = shl i32 [[STEP]], 2
-; CHECK-NEXT: [[BROADCAST_SPLATINSERT5:%.*]] = insertelement <4 x i32> poison, i32 [[TMP2]], i64 0
-; CHECK-NEXT: [[BROADCAST_SPLAT6:%.*]] = shufflevector <4 x i32> [[BROADCAST_SPLATINSERT5]], <4 x i32> poison, <4 x i32> zeroinitializer
+; CHECK-NEXT: [[TMP1:%.*]] = mul i32 [[TMP0]], [[STEP]]
+; CHECK-NEXT: [[TMP2:%.*]] = add i32 [[START]], [[TMP1]]
; CHECK-NEXT: br label %[[VECTOR_BODY:.*]]
; CHECK: [[VECTOR_BODY]]:
; CHECK-NEXT: [[INDEX:%.*]] = phi i32 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
-; CHECK-NEXT: [[VEC_IND:%.*]] = phi <4 x i32> [ [[INDUCTION]], %[[VECTOR_PH]] ], [ [[VEC_IND_NEXT:%.*]], %[[VECTOR_BODY]] ]
-; CHECK-NEXT: [[VEC_IV:%.*]] = phi <4 x i32> [ <i32 0, i32 1, i32 2, i32 3>, %[[VECTOR_PH]] ], [ [[VEC_IND_NEXT8:%.*]], %[[VECTOR_BODY]] ]
; CHECK-NEXT: [[INDEX_NEXT]] = add i32 [[INDEX]], 4
-; CHECK-NEXT: [[VEC_IND_NEXT]] = add <4 x i32> [[VEC_IND]], [[BROADCAST_SPLAT6]]
-; CHECK-NEXT: [[VEC_IND_NEXT8]] = add nuw <4 x i32> [[VEC_IV]], splat (i32 4)
; CHECK-NEXT: [[TMP3:%.*]] = icmp eq i32 [[INDEX_NEXT]], [[N_VEC]]
; CHECK-NEXT: br i1 [[TMP3]], label %[[SCALAR_PH:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP0:![0-9]+]]
; CHECK: [[SCALAR_PH]]:
-; CHECK-NEXT: [[TMP4:%.*]] = icmp ugt <4 x i32> [[VEC_IV]], [[BROADCAST_SPLAT]]
-; CHECK-NEXT: [[FIRST_INACTIVE_LANE:%.*]] = call i64 @llvm.experimental.cttz.elts.i64.v4i1(<4 x i1> [[TMP4]], i1 false)
-; CHECK-NEXT: [[LAST_ACTIVE_LANE:%.*]] = sub i64 [[FIRST_INACTIVE_LANE]], 1
-; CHECK-NEXT: [[IV2_LCSSA:%.*]] = extractelement <4 x i32> [[VEC_IND]], i64 [[LAST_ACTIVE_LANE]]
+; CHECK-NEXT: [[IV2_LCSSA:%.*]] = sub i32 [[TMP2]], [[STEP]]
; CHECK-NEXT: br label %[[LOOP:.*]]
; CHECK: [[LOOP]]:
; CHECK-NEXT: ret i32 [[IV2_LCSSA]]
diff --git a/llvm/test/Transforms/LoopVectorize/vector-loop-backedge-elimination-tail-folding.ll b/llvm/test/Transforms/LoopVectorize/vector-loop-backedge-elimination-tail-folding.ll
index bdaf9b37da10c..09a11a0f7fcc2 100644
--- a/llvm/test/Transforms/LoopVectorize/vector-loop-backedge-elimination-tail-folding.ll
+++ b/llvm/test/Transforms/LoopVectorize/vector-loop-backedge-elimination-tail-folding.ll
@@ -9,18 +9,12 @@ define i32 @last_active_lane_live_out(i32 %x) {
; CHECK-NEXT: [[ENTRY:.*:]]
; CHECK-NEXT: br label %[[VECTOR_PH:.*]]
; CHECK: [[VECTOR_PH]]:
-; CHECK-NEXT: [[BROADCAST_SPLATINSERT:%.*]] = insertelement <4 x i32> poison, i32 [[X]], i64 0
-; CHECK-NEXT: [[BROADCAST_SPLAT:%.*]] = shufflevector <4 x i32> [[BROADCAST_SPLATINSERT]], <4 x i32> poison, <4 x i32> zeroinitializer
-; CHECK-NEXT: [[TMP0:%.*]] = mul <4 x i32> <i32 0, i32 1, i32 2, i32 3>, [[BROADCAST_SPLAT]]
+; CHECK-NEXT: [[TMP0:%.*]] = shl i32 [[X]], 1
; CHECK-NEXT: br label %[[VECTOR_BODY:.*]]
; CHECK: [[VECTOR_BODY]]:
; CHECK-NEXT: br label %[[MIDDLE_BLOCK:.*]]
; CHECK: [[MIDDLE_BLOCK]]:
-; CHECK-NEXT: [[ACTIVE_LANE_MASK:%.*]] = call <4 x i1> @llvm.get.active.lane.mask.v4i1.i64(i64 0, i64 2)
-; CHECK-NEXT: [[TMP1:%.*]] = xor <4 x i1> [[ACTIVE_LANE_MASK]], splat (i1 true)
-; CHECK-NEXT: [[FIRST_INACTIVE_LANE:%.*]] = call i64 @llvm.experimental.cttz.elts.i64.v4i1(<4 x i1> [[TMP1]], i1 false)
-; CHECK-NEXT: [[LAST_ACTIVE_LANE:%.*]] = sub i64 [[FIRST_INACTIVE_LANE]], 1
-; CHECK-NEXT: [[TMP2:%.*]] = extractelement <4 x i32> [[TMP0]], i64 [[LAST_ACTIVE_LANE]]
+; CHECK-NEXT: [[TMP2:%.*]] = sub i32 [[TMP0]], [[X]]
; CHECK-NEXT: br label %[[EXIT:.*]]
; CHECK: [[EXIT]]:
; CHECK-NEXT: ret i32 [[TMP2]]
More information about the llvm-commits
mailing list