[llvm] efd8a91 - [VPlan] Split sinkable FindLastIV tail folding tests into new test. NFC (#191195)
via llvm-commits
llvm-commits at lists.llvm.org
Fri Apr 17 06:10:30 PDT 2026
Author: Luke Lau
Date: 2026-04-17T13:10:24Z
New Revision: efd8a91ee1e10e10fa7eeab06abbee91c5c5232d
URL: https://github.com/llvm/llvm-project/commit/efd8a91ee1e10e10fa7eeab06abbee91c5c5232d
DIFF: https://github.com/llvm/llvm-project/commit/efd8a91ee1e10e10fa7eeab06abbee91c5c5232d.diff
LOG: [VPlan] Split sinkable FindLastIV tail folding tests into new test. NFC (#191195)
Follow up to
https://github.com/llvm/llvm-project/pull/191176#pullrequestreview-4082422318
Added:
llvm/test/Transforms/LoopVectorize/find-last-iv-sinkable-expr-tail-folding.ll
Modified:
llvm/test/Transforms/LoopVectorize/find-last-iv-sinkable-expr.ll
Removed:
################################################################################
diff --git a/llvm/test/Transforms/LoopVectorize/find-last-iv-sinkable-expr-tail-folding.ll b/llvm/test/Transforms/LoopVectorize/find-last-iv-sinkable-expr-tail-folding.ll
new file mode 100644
index 0000000000000..13d40704d2a87
--- /dev/null
+++ b/llvm/test/Transforms/LoopVectorize/find-last-iv-sinkable-expr-tail-folding.ll
@@ -0,0 +1,827 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --check-globals none --version 6
+; RUN: opt -passes=loop-vectorize -force-vector-interleave=1 -force-vector-width=4 -prefer-predicate-over-epilogue=predicate-else-scalar-epilogue -force-target-supports-masked-memory-ops -S < %s | FileCheck %s
+
+; Tests for FindLastIV reductions where the selected value is an expression
+; of the induction variable (e.g., IV * const, IV << const, IV | const).
+
+define i64 @findlast_shl_expr(ptr %a, i64 %n) {
+; CHECK-LABEL: define i64 @findlast_shl_expr(
+; CHECK-SAME: ptr [[A:%.*]], i64 [[N:%.*]]) {
+; CHECK-NEXT: [[ENTRY:.*:]]
+; CHECK-NEXT: br label %[[VECTOR_PH:.*]]
+; CHECK: [[VECTOR_PH]]:
+; CHECK-NEXT: [[N_RND_UP:%.*]] = add i64 [[N]], 3
+; CHECK-NEXT: [[N_MOD_VF:%.*]] = urem i64 [[N_RND_UP]], 4
+; CHECK-NEXT: [[N_VEC:%.*]] = sub i64 [[N_RND_UP]], [[N_MOD_VF]]
+; CHECK-NEXT: [[TRIP_COUNT_MINUS_1:%.*]] = sub i64 [[N]], 1
+; CHECK-NEXT: [[BROADCAST_SPLATINSERT:%.*]] = insertelement <4 x i64> poison, i64 [[TRIP_COUNT_MINUS_1]], i64 0
+; CHECK-NEXT: [[BROADCAST_SPLAT:%.*]] = shufflevector <4 x i64> [[BROADCAST_SPLATINSERT]], <4 x i64> poison, <4 x i32> zeroinitializer
+; CHECK-NEXT: br label %[[VECTOR_BODY:.*]]
+; CHECK: [[VECTOR_BODY]]:
+; CHECK-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-NEXT: [[VEC_IND:%.*]] = phi <4 x i64> [ <i64 0, i64 1, i64 2, i64 3>, %[[VECTOR_PH]] ], [ [[VEC_IND_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-NEXT: [[VEC_PHI:%.*]] = phi <4 x i64> [ splat (i64 -9223372036854775808), %[[VECTOR_PH]] ], [ [[TMP3:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-NEXT: [[TMP0:%.*]] = icmp ule <4 x i64> [[VEC_IND]], [[BROADCAST_SPLAT]]
+; CHECK-NEXT: [[TMP1:%.*]] = getelementptr inbounds i64, ptr [[A]], i64 [[INDEX]]
+; CHECK-NEXT: [[WIDE_MASKED_LOAD:%.*]] = call <4 x i64> @llvm.masked.load.v4i64.p0(ptr align 8 [[TMP1]], <4 x i1> [[TMP0]], <4 x i64> poison)
+; CHECK-NEXT: [[TMP2:%.*]] = icmp eq <4 x i64> [[WIDE_MASKED_LOAD]], splat (i64 42)
+; CHECK-NEXT: [[TMP9:%.*]] = select <4 x i1> [[TMP0]], <4 x i1> [[TMP2]], <4 x i1> zeroinitializer
+; CHECK-NEXT: [[TMP3]] = select <4 x i1> [[TMP9]], <4 x i64> [[VEC_IND]], <4 x i64> [[VEC_PHI]]
+; CHECK-NEXT: [[INDEX_NEXT]] = add i64 [[INDEX]], 4
+; CHECK-NEXT: [[VEC_IND_NEXT]] = add <4 x i64> [[VEC_IND]], splat (i64 4)
+; CHECK-NEXT: [[TMP4:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
+; CHECK-NEXT: br i1 [[TMP4]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP0:![0-9]+]]
+; CHECK: [[MIDDLE_BLOCK]]:
+; CHECK-NEXT: [[TMP5:%.*]] = call i64 @llvm.vector.reduce.smax.v4i64(<4 x i64> [[TMP3]])
+; CHECK-NEXT: [[TMP6:%.*]] = shl i64 [[TMP5]], 2
+; CHECK-NEXT: [[TMP7:%.*]] = icmp ne i64 [[TMP5]], -9223372036854775808
+; CHECK-NEXT: [[TMP8:%.*]] = select i1 [[TMP7]], i64 [[TMP6]], i64 -1
+; CHECK-NEXT: br label %[[DONE:.*]]
+; CHECK: [[DONE]]:
+; CHECK-NEXT: ret i64 [[TMP8]]
+;
+entry:
+ br label %loop
+
+loop:
+ %iv = phi i64 [ 0, %entry ], [ %iv.next, %loop ]
+ %rdx = phi i64 [ -1, %entry ], [ %sel, %loop ]
+ %gep = getelementptr inbounds i64, ptr %a, i64 %iv
+ %load = load i64, ptr %gep, align 8
+ %cmp = icmp eq i64 %load, 42
+ %expr = shl i64 %iv, 2
+ %sel = select i1 %cmp, i64 %expr, i64 %rdx
+ %iv.next = add nuw nsw i64 %iv, 1
+ %exit = icmp eq i64 %iv.next, %n
+ br i1 %exit, label %done, label %loop
+
+done:
+ ret i64 %sel
+}
+
+define i64 @findlast_mul_expr(ptr %a, i64 %n) {
+; CHECK-LABEL: define i64 @findlast_mul_expr(
+; CHECK-SAME: ptr [[A:%.*]], i64 [[N:%.*]]) {
+; CHECK-NEXT: [[ENTRY:.*:]]
+; CHECK-NEXT: br label %[[VECTOR_PH:.*]]
+; CHECK: [[VECTOR_PH]]:
+; CHECK-NEXT: [[N_RND_UP:%.*]] = add i64 [[N]], 3
+; CHECK-NEXT: [[N_MOD_VF:%.*]] = urem i64 [[N_RND_UP]], 4
+; CHECK-NEXT: [[N_VEC:%.*]] = sub i64 [[N_RND_UP]], [[N_MOD_VF]]
+; CHECK-NEXT: [[TRIP_COUNT_MINUS_1:%.*]] = sub i64 [[N]], 1
+; CHECK-NEXT: [[BROADCAST_SPLATINSERT:%.*]] = insertelement <4 x i64> poison, i64 [[TRIP_COUNT_MINUS_1]], i64 0
+; CHECK-NEXT: [[BROADCAST_SPLAT:%.*]] = shufflevector <4 x i64> [[BROADCAST_SPLATINSERT]], <4 x i64> poison, <4 x i32> zeroinitializer
+; CHECK-NEXT: br label %[[VECTOR_BODY:.*]]
+; CHECK: [[VECTOR_BODY]]:
+; CHECK-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-NEXT: [[VEC_IND:%.*]] = phi <4 x i64> [ <i64 0, i64 1, i64 2, i64 3>, %[[VECTOR_PH]] ], [ [[VEC_IND_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-NEXT: [[VEC_PHI:%.*]] = phi <4 x i64> [ splat (i64 -9223372036854775808), %[[VECTOR_PH]] ], [ [[TMP3:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-NEXT: [[TMP0:%.*]] = icmp ule <4 x i64> [[VEC_IND]], [[BROADCAST_SPLAT]]
+; CHECK-NEXT: [[TMP1:%.*]] = getelementptr inbounds i64, ptr [[A]], i64 [[INDEX]]
+; CHECK-NEXT: [[WIDE_MASKED_LOAD:%.*]] = call <4 x i64> @llvm.masked.load.v4i64.p0(ptr align 8 [[TMP1]], <4 x i1> [[TMP0]], <4 x i64> poison)
+; CHECK-NEXT: [[TMP2:%.*]] = icmp eq <4 x i64> [[WIDE_MASKED_LOAD]], splat (i64 42)
+; CHECK-NEXT: [[TMP9:%.*]] = select <4 x i1> [[TMP0]], <4 x i1> [[TMP2]], <4 x i1> zeroinitializer
+; CHECK-NEXT: [[TMP3]] = select <4 x i1> [[TMP9]], <4 x i64> [[VEC_IND]], <4 x i64> [[VEC_PHI]]
+; CHECK-NEXT: [[INDEX_NEXT]] = add i64 [[INDEX]], 4
+; CHECK-NEXT: [[VEC_IND_NEXT]] = add <4 x i64> [[VEC_IND]], splat (i64 4)
+; CHECK-NEXT: [[TMP4:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
+; CHECK-NEXT: br i1 [[TMP4]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP3:![0-9]+]]
+; CHECK: [[MIDDLE_BLOCK]]:
+; CHECK-NEXT: [[TMP5:%.*]] = call i64 @llvm.vector.reduce.smax.v4i64(<4 x i64> [[TMP3]])
+; CHECK-NEXT: [[TMP6:%.*]] = mul i64 [[TMP5]], 3
+; CHECK-NEXT: [[TMP7:%.*]] = icmp ne i64 [[TMP5]], -9223372036854775808
+; CHECK-NEXT: [[TMP8:%.*]] = select i1 [[TMP7]], i64 [[TMP6]], i64 -1
+; CHECK-NEXT: br label %[[DONE:.*]]
+; CHECK: [[DONE]]:
+; CHECK-NEXT: ret i64 [[TMP8]]
+;
+entry:
+ br label %loop
+
+loop:
+ %iv = phi i64 [ 0, %entry ], [ %iv.next, %loop ]
+ %rdx = phi i64 [ -1, %entry ], [ %sel, %loop ]
+ %gep = getelementptr inbounds i64, ptr %a, i64 %iv
+ %load = load i64, ptr %gep, align 8
+ %cmp = icmp eq i64 %load, 42
+ %expr = mul i64 %iv, 3
+ %sel = select i1 %cmp, i64 %expr, i64 %rdx
+ %iv.next = add nuw nsw i64 %iv, 1
+ %exit = icmp eq i64 %iv.next, %n
+ br i1 %exit, label %done, label %loop
+
+done:
+ ret i64 %sel
+}
+
+define i64 @findlast_or_expr(ptr %a, i64 %n) {
+; CHECK-LABEL: define i64 @findlast_or_expr(
+; CHECK-SAME: ptr [[A:%.*]], i64 [[N:%.*]]) {
+; CHECK-NEXT: [[ENTRY:.*:]]
+; CHECK-NEXT: br label %[[VECTOR_PH:.*]]
+; CHECK: [[VECTOR_PH]]:
+; CHECK-NEXT: [[N_RND_UP:%.*]] = add i64 [[N]], 3
+; CHECK-NEXT: [[N_MOD_VF:%.*]] = urem i64 [[N_RND_UP]], 4
+; CHECK-NEXT: [[N_VEC:%.*]] = sub i64 [[N_RND_UP]], [[N_MOD_VF]]
+; CHECK-NEXT: [[TRIP_COUNT_MINUS_1:%.*]] = sub i64 [[N]], 1
+; CHECK-NEXT: [[BROADCAST_SPLATINSERT:%.*]] = insertelement <4 x i64> poison, i64 [[TRIP_COUNT_MINUS_1]], i64 0
+; CHECK-NEXT: [[BROADCAST_SPLAT:%.*]] = shufflevector <4 x i64> [[BROADCAST_SPLATINSERT]], <4 x i64> poison, <4 x i32> zeroinitializer
+; CHECK-NEXT: br label %[[VECTOR_BODY:.*]]
+; CHECK: [[VECTOR_BODY]]:
+; CHECK-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-NEXT: [[VEC_IND:%.*]] = phi <4 x i64> [ <i64 0, i64 1, i64 2, i64 3>, %[[VECTOR_PH]] ], [ [[VEC_IND_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-NEXT: [[VEC_PHI:%.*]] = phi <4 x i64> [ splat (i64 -9223372036854775808), %[[VECTOR_PH]] ], [ [[TMP3:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-NEXT: [[TMP0:%.*]] = icmp ule <4 x i64> [[VEC_IND]], [[BROADCAST_SPLAT]]
+; CHECK-NEXT: [[TMP1:%.*]] = getelementptr inbounds i64, ptr [[A]], i64 [[INDEX]]
+; CHECK-NEXT: [[WIDE_MASKED_LOAD:%.*]] = call <4 x i64> @llvm.masked.load.v4i64.p0(ptr align 8 [[TMP1]], <4 x i1> [[TMP0]], <4 x i64> poison)
+; CHECK-NEXT: [[TMP2:%.*]] = icmp eq <4 x i64> [[WIDE_MASKED_LOAD]], splat (i64 42)
+; CHECK-NEXT: [[TMP9:%.*]] = select <4 x i1> [[TMP0]], <4 x i1> [[TMP2]], <4 x i1> zeroinitializer
+; CHECK-NEXT: [[TMP3]] = select <4 x i1> [[TMP9]], <4 x i64> [[VEC_IND]], <4 x i64> [[VEC_PHI]]
+; CHECK-NEXT: [[INDEX_NEXT]] = add i64 [[INDEX]], 4
+; CHECK-NEXT: [[VEC_IND_NEXT]] = add <4 x i64> [[VEC_IND]], splat (i64 4)
+; CHECK-NEXT: [[TMP4:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
+; CHECK-NEXT: br i1 [[TMP4]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP4:![0-9]+]]
+; CHECK: [[MIDDLE_BLOCK]]:
+; CHECK-NEXT: [[TMP5:%.*]] = call i64 @llvm.vector.reduce.smax.v4i64(<4 x i64> [[TMP3]])
+; CHECK-NEXT: [[TMP6:%.*]] = or i64 [[TMP5]], 1
+; CHECK-NEXT: [[TMP7:%.*]] = icmp ne i64 [[TMP5]], -9223372036854775808
+; CHECK-NEXT: [[TMP8:%.*]] = select i1 [[TMP7]], i64 [[TMP6]], i64 -1
+; CHECK-NEXT: br label %[[DONE:.*]]
+; CHECK: [[DONE]]:
+; CHECK-NEXT: ret i64 [[TMP8]]
+;
+entry:
+ br label %loop
+
+loop:
+ %iv = phi i64 [ 0, %entry ], [ %iv.next, %loop ]
+ %rdx = phi i64 [ -1, %entry ], [ %sel, %loop ]
+ %gep = getelementptr inbounds i64, ptr %a, i64 %iv
+ %load = load i64, ptr %gep, align 8
+ %cmp = icmp eq i64 %load, 42
+ %expr = or i64 %iv, 1
+ %sel = select i1 %cmp, i64 %expr, i64 %rdx
+ %iv.next = add nuw nsw i64 %iv, 1
+ %exit = icmp eq i64 %iv.next, %n
+ br i1 %exit, label %done, label %loop
+
+done:
+ ret i64 %sel
+}
+
+; Test non-canonical IV (starting at 10, step 2) with expression sinking.
+; The IV is non-canonical but expression sinking should still apply.
+define i64 @findlast_non_canonical_iv_with_expr(ptr %a, i64 %n) {
+; CHECK-LABEL: define i64 @findlast_non_canonical_iv_with_expr(
+; CHECK-SAME: ptr [[A:%.*]], i64 [[N:%.*]]) {
+; CHECK-NEXT: [[ENTRY:.*:]]
+; CHECK-NEXT: [[TMP0:%.*]] = add i64 [[N]], -12
+; CHECK-NEXT: [[TMP1:%.*]] = lshr i64 [[TMP0]], 1
+; CHECK-NEXT: [[TMP2:%.*]] = add nuw i64 [[TMP1]], 1
+; CHECK-NEXT: br label %[[VECTOR_PH:.*]]
+; CHECK: [[VECTOR_PH]]:
+; CHECK-NEXT: [[N_RND_UP:%.*]] = add i64 [[TMP2]], 3
+; CHECK-NEXT: [[N_MOD_VF:%.*]] = urem i64 [[N_RND_UP]], 4
+; CHECK-NEXT: [[N_VEC:%.*]] = sub i64 [[N_RND_UP]], [[N_MOD_VF]]
+; CHECK-NEXT: [[TRIP_COUNT_MINUS_1:%.*]] = sub i64 [[TMP2]], 1
+; CHECK-NEXT: [[BROADCAST_SPLATINSERT:%.*]] = insertelement <4 x i64> poison, i64 [[TRIP_COUNT_MINUS_1]], i64 0
+; CHECK-NEXT: [[BROADCAST_SPLAT:%.*]] = shufflevector <4 x i64> [[BROADCAST_SPLATINSERT]], <4 x i64> poison, <4 x i32> zeroinitializer
+; CHECK-NEXT: br label %[[VECTOR_BODY:.*]]
+; CHECK: [[VECTOR_BODY]]:
+; CHECK-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[PRED_LOAD_CONTINUE8:.*]] ]
+; CHECK-NEXT: [[VEC_IND:%.*]] = phi <4 x i64> [ <i64 10, i64 12, i64 14, i64 16>, %[[VECTOR_PH]] ], [ [[VEC_IND_NEXT:%.*]], %[[PRED_LOAD_CONTINUE8]] ]
+; CHECK-NEXT: [[VEC_PHI:%.*]] = phi <4 x i64> [ zeroinitializer, %[[VECTOR_PH]] ], [ [[TMP36:%.*]], %[[PRED_LOAD_CONTINUE8]] ]
+; CHECK-NEXT: [[TMP3:%.*]] = mul i64 [[INDEX]], 2
+; CHECK-NEXT: [[OFFSET_IDX:%.*]] = add i64 10, [[TMP3]]
+; CHECK-NEXT: [[BROADCAST_SPLATINSERT1:%.*]] = insertelement <4 x i64> poison, i64 [[INDEX]], i64 0
+; CHECK-NEXT: [[BROADCAST_SPLAT2:%.*]] = shufflevector <4 x i64> [[BROADCAST_SPLATINSERT1]], <4 x i64> poison, <4 x i32> zeroinitializer
+; CHECK-NEXT: [[VEC_IV:%.*]] = add <4 x i64> [[BROADCAST_SPLAT2]], <i64 0, i64 1, i64 2, i64 3>
+; CHECK-NEXT: [[TMP4:%.*]] = icmp ule <4 x i64> [[VEC_IV]], [[BROADCAST_SPLAT]]
+; CHECK-NEXT: [[TMP5:%.*]] = extractelement <4 x i1> [[TMP4]], i32 0
+; CHECK-NEXT: br i1 [[TMP5]], label %[[PRED_LOAD_IF:.*]], label %[[PRED_LOAD_CONTINUE:.*]]
+; CHECK: [[PRED_LOAD_IF]]:
+; CHECK-NEXT: [[TMP6:%.*]] = add i64 [[OFFSET_IDX]], 0
+; CHECK-NEXT: [[TMP7:%.*]] = getelementptr inbounds i64, ptr [[A]], i64 [[TMP6]]
+; CHECK-NEXT: [[TMP8:%.*]] = load i64, ptr [[TMP7]], align 8
+; CHECK-NEXT: [[TMP9:%.*]] = insertelement <4 x i64> poison, i64 [[TMP8]], i32 0
+; CHECK-NEXT: br label %[[PRED_LOAD_CONTINUE]]
+; CHECK: [[PRED_LOAD_CONTINUE]]:
+; CHECK-NEXT: [[TMP10:%.*]] = phi <4 x i64> [ poison, %[[VECTOR_BODY]] ], [ [[TMP9]], %[[PRED_LOAD_IF]] ]
+; CHECK-NEXT: [[TMP11:%.*]] = extractelement <4 x i1> [[TMP4]], i32 1
+; CHECK-NEXT: br i1 [[TMP11]], label %[[PRED_LOAD_IF3:.*]], label %[[PRED_LOAD_CONTINUE4:.*]]
+; CHECK: [[PRED_LOAD_IF3]]:
+; CHECK-NEXT: [[TMP12:%.*]] = add i64 [[OFFSET_IDX]], 2
+; CHECK-NEXT: [[TMP13:%.*]] = getelementptr inbounds i64, ptr [[A]], i64 [[TMP12]]
+; CHECK-NEXT: [[TMP14:%.*]] = load i64, ptr [[TMP13]], align 8
+; CHECK-NEXT: [[TMP15:%.*]] = insertelement <4 x i64> [[TMP10]], i64 [[TMP14]], i32 1
+; CHECK-NEXT: br label %[[PRED_LOAD_CONTINUE4]]
+; CHECK: [[PRED_LOAD_CONTINUE4]]:
+; CHECK-NEXT: [[TMP16:%.*]] = phi <4 x i64> [ [[TMP10]], %[[PRED_LOAD_CONTINUE]] ], [ [[TMP15]], %[[PRED_LOAD_IF3]] ]
+; CHECK-NEXT: [[TMP17:%.*]] = extractelement <4 x i1> [[TMP4]], i32 2
+; CHECK-NEXT: br i1 [[TMP17]], label %[[PRED_LOAD_IF5:.*]], label %[[PRED_LOAD_CONTINUE6:.*]]
+; CHECK: [[PRED_LOAD_IF5]]:
+; CHECK-NEXT: [[TMP18:%.*]] = add i64 [[OFFSET_IDX]], 4
+; CHECK-NEXT: [[TMP19:%.*]] = getelementptr inbounds i64, ptr [[A]], i64 [[TMP18]]
+; CHECK-NEXT: [[TMP20:%.*]] = load i64, ptr [[TMP19]], align 8
+; CHECK-NEXT: [[TMP21:%.*]] = insertelement <4 x i64> [[TMP16]], i64 [[TMP20]], i32 2
+; CHECK-NEXT: br label %[[PRED_LOAD_CONTINUE6]]
+; CHECK: [[PRED_LOAD_CONTINUE6]]:
+; CHECK-NEXT: [[TMP22:%.*]] = phi <4 x i64> [ [[TMP16]], %[[PRED_LOAD_CONTINUE4]] ], [ [[TMP21]], %[[PRED_LOAD_IF5]] ]
+; CHECK-NEXT: [[TMP23:%.*]] = extractelement <4 x i1> [[TMP4]], i32 3
+; CHECK-NEXT: br i1 [[TMP23]], label %[[PRED_LOAD_IF7:.*]], label %[[PRED_LOAD_CONTINUE8]]
+; CHECK: [[PRED_LOAD_IF7]]:
+; CHECK-NEXT: [[TMP24:%.*]] = add i64 [[OFFSET_IDX]], 6
+; CHECK-NEXT: [[TMP25:%.*]] = getelementptr inbounds i64, ptr [[A]], i64 [[TMP24]]
+; CHECK-NEXT: [[TMP26:%.*]] = load i64, ptr [[TMP25]], align 8
+; CHECK-NEXT: [[TMP27:%.*]] = insertelement <4 x i64> [[TMP22]], i64 [[TMP26]], i32 3
+; CHECK-NEXT: br label %[[PRED_LOAD_CONTINUE8]]
+; CHECK: [[PRED_LOAD_CONTINUE8]]:
+; CHECK-NEXT: [[TMP28:%.*]] = phi <4 x i64> [ [[TMP22]], %[[PRED_LOAD_CONTINUE6]] ], [ [[TMP27]], %[[PRED_LOAD_IF7]] ]
+; CHECK-NEXT: [[TMP29:%.*]] = icmp eq <4 x i64> [[TMP28]], splat (i64 42)
+; CHECK-NEXT: [[TMP30:%.*]] = add <4 x i64> [[VEC_IND]], splat (i64 100)
+; CHECK-NEXT: [[TMP31:%.*]] = select <4 x i1> [[TMP29]], <4 x i64> [[TMP30]], <4 x i64> [[VEC_PHI]]
+; CHECK-NEXT: [[TMP36]] = select <4 x i1> [[TMP4]], <4 x i64> [[TMP31]], <4 x i64> [[VEC_PHI]]
+; CHECK-NEXT: [[INDEX_NEXT]] = add i64 [[INDEX]], 4
+; CHECK-NEXT: [[VEC_IND_NEXT]] = add nuw nsw <4 x i64> [[VEC_IND]], splat (i64 8)
+; CHECK-NEXT: [[TMP32:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
+; CHECK-NEXT: br i1 [[TMP32]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP5:![0-9]+]]
+; CHECK: [[MIDDLE_BLOCK]]:
+; CHECK-NEXT: [[TMP33:%.*]] = call i64 @llvm.vector.reduce.umax.v4i64(<4 x i64> [[TMP36]])
+; CHECK-NEXT: [[TMP34:%.*]] = icmp ne i64 [[TMP33]], 0
+; CHECK-NEXT: [[TMP35:%.*]] = select i1 [[TMP34]], i64 [[TMP33]], i64 -1
+; CHECK-NEXT: br label %[[DONE:.*]]
+; CHECK: [[DONE]]:
+; CHECK-NEXT: ret i64 [[TMP35]]
+;
+entry:
+ br label %loop
+
+loop:
+ %iv = phi i64 [ 10, %entry ], [ %iv.next, %loop ]
+ %rdx = phi i64 [ -1, %entry ], [ %sel, %loop ]
+ %gep = getelementptr inbounds i64, ptr %a, i64 %iv
+ %load = load i64, ptr %gep, align 8
+ %cmp = icmp eq i64 %load, 42
+ %expr = add i64 %iv, 100
+ %sel = select i1 %cmp, i64 %expr, i64 %rdx
+ %iv.next = add nuw nsw i64 %iv, 2
+ %exit = icmp eq i64 %iv.next, %n
+ br i1 %exit, label %done, label %loop
+
+done:
+ ret i64 %sel
+}
+
+; Test wider IV (i64) with narrower reduction (i32), only scalar users.
+; Expression sinking should NOT apply because introducing wider computations
+; in the middle block would be slower when the wide IV can be eliminated.
+define i32 @findlast_wider_iv_only_scalar_users(ptr %a, i64 %n) {
+; CHECK-LABEL: define i32 @findlast_wider_iv_only_scalar_users(
+; CHECK-SAME: ptr [[A:%.*]], i64 [[N:%.*]]) {
+; CHECK-NEXT: [[ENTRY:.*:]]
+; CHECK-NEXT: br label %[[VECTOR_PH:.*]]
+; CHECK: [[VECTOR_PH]]:
+; CHECK-NEXT: [[N_RND_UP:%.*]] = add i64 [[N]], 3
+; CHECK-NEXT: [[N_MOD_VF:%.*]] = urem i64 [[N_RND_UP]], 4
+; CHECK-NEXT: [[N_VEC:%.*]] = sub i64 [[N_RND_UP]], [[N_MOD_VF]]
+; CHECK-NEXT: [[TRIP_COUNT_MINUS_1:%.*]] = sub i64 [[N]], 1
+; CHECK-NEXT: [[BROADCAST_SPLATINSERT:%.*]] = insertelement <4 x i64> poison, i64 [[TRIP_COUNT_MINUS_1]], i64 0
+; CHECK-NEXT: [[BROADCAST_SPLAT:%.*]] = shufflevector <4 x i64> [[BROADCAST_SPLATINSERT]], <4 x i64> poison, <4 x i32> zeroinitializer
+; CHECK-NEXT: br label %[[VECTOR_BODY:.*]]
+; CHECK: [[VECTOR_BODY]]:
+; CHECK-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-NEXT: [[VEC_IND:%.*]] = phi <4 x i64> [ <i64 0, i64 1, i64 2, i64 3>, %[[VECTOR_PH]] ], [ [[VEC_IND_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-NEXT: [[VEC_PHI:%.*]] = phi <4 x i32> [ splat (i32 -1), %[[VECTOR_PH]] ], [ [[TMP9:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-NEXT: [[TMP0:%.*]] = phi <4 x i1> [ zeroinitializer, %[[VECTOR_PH]] ], [ [[TMP8:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-NEXT: [[VEC_IND1:%.*]] = phi <4 x i32> [ <i32 0, i32 1, i32 2, i32 3>, %[[VECTOR_PH]] ], [ [[VEC_IND_NEXT2:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-NEXT: [[TMP1:%.*]] = icmp ule <4 x i64> [[VEC_IND]], [[BROADCAST_SPLAT]]
+; CHECK-NEXT: [[TMP2:%.*]] = getelementptr inbounds i32, ptr [[A]], i64 [[INDEX]]
+; CHECK-NEXT: [[WIDE_MASKED_LOAD:%.*]] = call <4 x i32> @llvm.masked.load.v4i32.p0(ptr align 4 [[TMP2]], <4 x i1> [[TMP1]], <4 x i32> poison)
+; CHECK-NEXT: [[TMP3:%.*]] = icmp eq <4 x i32> [[WIDE_MASKED_LOAD]], splat (i32 42)
+; CHECK-NEXT: [[TMP4:%.*]] = shl <4 x i32> [[VEC_IND1]], splat (i32 2)
+; CHECK-NEXT: [[TMP5:%.*]] = select <4 x i1> [[TMP1]], <4 x i1> [[TMP3]], <4 x i1> zeroinitializer
+; CHECK-NEXT: [[TMP6:%.*]] = freeze <4 x i1> [[TMP5]]
+; CHECK-NEXT: [[TMP7:%.*]] = call i1 @llvm.vector.reduce.or.v4i1(<4 x i1> [[TMP6]])
+; CHECK-NEXT: [[TMP8]] = select i1 [[TMP7]], <4 x i1> [[TMP5]], <4 x i1> [[TMP0]]
+; CHECK-NEXT: [[TMP9]] = select i1 [[TMP7]], <4 x i32> [[TMP4]], <4 x i32> [[VEC_PHI]]
+; CHECK-NEXT: [[INDEX_NEXT]] = add i64 [[INDEX]], 4
+; CHECK-NEXT: [[VEC_IND_NEXT]] = add <4 x i64> [[VEC_IND]], splat (i64 4)
+; CHECK-NEXT: [[VEC_IND_NEXT2]] = add <4 x i32> [[VEC_IND1]], splat (i32 4)
+; CHECK-NEXT: [[TMP10:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
+; CHECK-NEXT: br i1 [[TMP10]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP6:![0-9]+]]
+; CHECK: [[MIDDLE_BLOCK]]:
+; CHECK-NEXT: [[TMP11:%.*]] = call i32 @llvm.experimental.vector.extract.last.active.v4i32(<4 x i32> [[TMP9]], <4 x i1> [[TMP8]], i32 -1)
+; CHECK-NEXT: br label %[[DONE:.*]]
+; CHECK: [[DONE]]:
+; CHECK-NEXT: ret i32 [[TMP11]]
+;
+entry:
+ br label %loop
+
+loop:
+ %iv = phi i64 [ 0, %entry ], [ %iv.next, %loop ]
+ %rdx = phi i32 [ -1, %entry ], [ %sel, %loop ]
+ %gep = getelementptr inbounds i32, ptr %a, i64 %iv
+ %load = load i32, ptr %gep, align 4
+ %cmp = icmp eq i32 %load, 42
+ %iv.trunc = trunc i64 %iv to i32
+ %expr = shl i32 %iv.trunc, 2
+ %sel = select i1 %cmp, i32 %expr, i32 %rdx
+ %iv.next = add nuw nsw i64 %iv, 1
+ %exit = icmp eq i64 %iv.next, %n
+ br i1 %exit, label %done, label %loop
+
+done:
+ ret i32 %sel
+}
+
+; Test wider IV (i64) with narrower reduction (i32), with vector users.
+; Expression sinking SHOULD apply because the wide IV has vector users
+; (the mul that feeds into both the store and the reduction), so it will
+; stay wide anyway.
+define i32 @findlast_wider_iv_with_vector_users(ptr %a, ptr %b, i64 %n) {
+; CHECK-LABEL: define i32 @findlast_wider_iv_with_vector_users(
+; CHECK-SAME: ptr [[A:%.*]], ptr [[B:%.*]], i64 [[N:%.*]]) {
+; CHECK-NEXT: [[ENTRY:.*:]]
+; CHECK-NEXT: br label %[[VECTOR_MEMCHECK:.*]]
+; CHECK: [[VECTOR_MEMCHECK]]:
+; CHECK-NEXT: [[TMP0:%.*]] = shl i64 [[N]], 3
+; CHECK-NEXT: [[SCEVGEP:%.*]] = getelementptr i8, ptr [[B]], i64 [[TMP0]]
+; CHECK-NEXT: [[TMP1:%.*]] = shl i64 [[N]], 2
+; CHECK-NEXT: [[SCEVGEP1:%.*]] = getelementptr i8, ptr [[A]], i64 [[TMP1]]
+; CHECK-NEXT: [[BOUND0:%.*]] = icmp ult ptr [[B]], [[SCEVGEP1]]
+; CHECK-NEXT: [[BOUND1:%.*]] = icmp ult ptr [[A]], [[SCEVGEP]]
+; CHECK-NEXT: [[FOUND_CONFLICT:%.*]] = and i1 [[BOUND0]], [[BOUND1]]
+; CHECK-NEXT: br i1 [[FOUND_CONFLICT]], label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]]
+; CHECK: [[VECTOR_PH]]:
+; CHECK-NEXT: [[N_RND_UP:%.*]] = add i64 [[N]], 3
+; CHECK-NEXT: [[N_MOD_VF:%.*]] = urem i64 [[N_RND_UP]], 4
+; CHECK-NEXT: [[N_VEC:%.*]] = sub i64 [[N_RND_UP]], [[N_MOD_VF]]
+; CHECK-NEXT: [[TRIP_COUNT_MINUS_1:%.*]] = sub i64 [[N]], 1
+; CHECK-NEXT: [[BROADCAST_SPLATINSERT:%.*]] = insertelement <4 x i64> poison, i64 [[TRIP_COUNT_MINUS_1]], i64 0
+; CHECK-NEXT: [[BROADCAST_SPLAT:%.*]] = shufflevector <4 x i64> [[BROADCAST_SPLATINSERT]], <4 x i64> poison, <4 x i32> zeroinitializer
+; CHECK-NEXT: br label %[[VECTOR_BODY:.*]]
+; CHECK: [[VECTOR_BODY]]:
+; CHECK-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-NEXT: [[VEC_IND:%.*]] = phi <4 x i64> [ <i64 0, i64 1, i64 2, i64 3>, %[[VECTOR_PH]] ], [ [[VEC_IND_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-NEXT: [[VEC_PHI:%.*]] = phi <4 x i32> [ splat (i32 -1), %[[VECTOR_PH]] ], [ [[TMP13:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-NEXT: [[TMP2:%.*]] = phi <4 x i1> [ zeroinitializer, %[[VECTOR_PH]] ], [ [[TMP12:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-NEXT: [[VEC_IND2:%.*]] = phi <4 x i32> [ <i32 0, i32 1, i32 2, i32 3>, %[[VECTOR_PH]] ], [ [[VEC_IND_NEXT3:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-NEXT: [[TMP3:%.*]] = icmp ule <4 x i64> [[VEC_IND]], [[BROADCAST_SPLAT]]
+; CHECK-NEXT: [[TMP4:%.*]] = getelementptr inbounds i32, ptr [[A]], i64 [[INDEX]]
+; CHECK-NEXT: [[WIDE_MASKED_LOAD:%.*]] = call <4 x i32> @llvm.masked.load.v4i32.p0(ptr align 4 [[TMP4]], <4 x i1> [[TMP3]], <4 x i32> poison), !alias.scope [[META7:![0-9]+]]
+; CHECK-NEXT: [[TMP5:%.*]] = icmp eq <4 x i32> [[WIDE_MASKED_LOAD]], splat (i32 42)
+; CHECK-NEXT: [[TMP6:%.*]] = mul <4 x i64> [[VEC_IND]], splat (i64 3)
+; CHECK-NEXT: [[TMP7:%.*]] = getelementptr inbounds i64, ptr [[B]], i64 [[INDEX]]
+; CHECK-NEXT: call void @llvm.masked.store.v4i64.p0(<4 x i64> [[TMP6]], ptr align 8 [[TMP7]], <4 x i1> [[TMP3]]), !alias.scope [[META10:![0-9]+]], !noalias [[META7]]
+; CHECK-NEXT: [[TMP8:%.*]] = add <4 x i32> [[VEC_IND2]], splat (i32 100)
+; CHECK-NEXT: [[TMP9:%.*]] = select <4 x i1> [[TMP3]], <4 x i1> [[TMP5]], <4 x i1> zeroinitializer
+; CHECK-NEXT: [[TMP10:%.*]] = freeze <4 x i1> [[TMP9]]
+; CHECK-NEXT: [[TMP11:%.*]] = call i1 @llvm.vector.reduce.or.v4i1(<4 x i1> [[TMP10]])
+; CHECK-NEXT: [[TMP12]] = select i1 [[TMP11]], <4 x i1> [[TMP9]], <4 x i1> [[TMP2]]
+; CHECK-NEXT: [[TMP13]] = select i1 [[TMP11]], <4 x i32> [[TMP8]], <4 x i32> [[VEC_PHI]]
+; CHECK-NEXT: [[INDEX_NEXT]] = add i64 [[INDEX]], 4
+; CHECK-NEXT: [[VEC_IND_NEXT]] = add <4 x i64> [[VEC_IND]], splat (i64 4)
+; CHECK-NEXT: [[VEC_IND_NEXT3]] = add <4 x i32> [[VEC_IND2]], splat (i32 4)
+; CHECK-NEXT: [[TMP14:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
+; CHECK-NEXT: br i1 [[TMP14]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP12:![0-9]+]]
+; CHECK: [[MIDDLE_BLOCK]]:
+; CHECK-NEXT: [[TMP15:%.*]] = call i32 @llvm.experimental.vector.extract.last.active.v4i32(<4 x i32> [[TMP13]], <4 x i1> [[TMP12]], i32 -1)
+; CHECK-NEXT: br label %[[DONE:.*]]
+; CHECK: [[SCALAR_PH]]:
+; CHECK-NEXT: br label %[[LOOP:.*]]
+; CHECK: [[LOOP]]:
+; CHECK-NEXT: [[IV:%.*]] = phi i64 [ 0, %[[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], %[[LOOP]] ]
+; CHECK-NEXT: [[RDX:%.*]] = phi i32 [ -1, %[[SCALAR_PH]] ], [ [[SEL:%.*]], %[[LOOP]] ]
+; CHECK-NEXT: [[GEP_A:%.*]] = getelementptr inbounds i32, ptr [[A]], i64 [[IV]]
+; CHECK-NEXT: [[LOAD:%.*]] = load i32, ptr [[GEP_A]], align 4
+; CHECK-NEXT: [[CMP:%.*]] = icmp eq i32 [[LOAD]], 42
+; CHECK-NEXT: [[IV_MUL:%.*]] = mul i64 [[IV]], 3
+; CHECK-NEXT: [[GEP_B:%.*]] = getelementptr inbounds i64, ptr [[B]], i64 [[IV]]
+; CHECK-NEXT: store i64 [[IV_MUL]], ptr [[GEP_B]], align 8
+; CHECK-NEXT: [[IV_TRUNC:%.*]] = trunc i64 [[IV]] to i32
+; CHECK-NEXT: [[EXPR:%.*]] = add i32 [[IV_TRUNC]], 100
+; CHECK-NEXT: [[SEL]] = select i1 [[CMP]], i32 [[EXPR]], i32 [[RDX]]
+; CHECK-NEXT: [[IV_NEXT]] = add nuw nsw i64 [[IV]], 1
+; CHECK-NEXT: [[EXIT:%.*]] = icmp eq i64 [[IV_NEXT]], [[N]]
+; CHECK-NEXT: br i1 [[EXIT]], label %[[DONE]], label %[[LOOP]], !llvm.loop [[LOOP13:![0-9]+]]
+; CHECK: [[DONE]]:
+; CHECK-NEXT: [[SEL_LCSSA:%.*]] = phi i32 [ [[SEL]], %[[LOOP]] ], [ [[TMP15]], %[[MIDDLE_BLOCK]] ]
+; CHECK-NEXT: ret i32 [[SEL_LCSSA]]
+;
+entry:
+ br label %loop
+
+loop:
+ %iv = phi i64 [ 0, %entry ], [ %iv.next, %loop ]
+ %rdx = phi i32 [ -1, %entry ], [ %sel, %loop ]
+ %gep.a = getelementptr inbounds i32, ptr %a, i64 %iv
+ %load = load i32, ptr %gep.a, align 4
+ %cmp = icmp eq i32 %load, 42
+ ; This mul uses the wide IV and feeds into a store (vector user)
+ %iv.mul = mul i64 %iv, 3
+ %gep.b = getelementptr inbounds i64, ptr %b, i64 %iv
+ store i64 %iv.mul, ptr %gep.b, align 8
+ %iv.trunc = trunc i64 %iv to i32
+ %expr = add i32 %iv.trunc, 100
+ %sel = select i1 %cmp, i32 %expr, i32 %rdx
+ %iv.next = add nuw nsw i64 %iv, 1
+ %exit = icmp eq i64 %iv.next, %n
+ br i1 %exit, label %done, label %loop
+
+done:
+ ret i32 %sel
+}
+
+; Test sdiv where the IV is the second operand (divisor). Verifies that
+; operand order is preserved when sinking the expression to the middle block.
+; IV starts at 1 to avoid division by zero.
+define i64 @findlast_sdiv_iv_as_divisor(ptr %a, i64 %n) {
+; CHECK-LABEL: define i64 @findlast_sdiv_iv_as_divisor(
+; CHECK-SAME: ptr [[A:%.*]], i64 [[N:%.*]]) {
+; CHECK-NEXT: [[ENTRY:.*:]]
+; CHECK-NEXT: [[TMP0:%.*]] = add i64 [[N]], -1
+; CHECK-NEXT: br label %[[VECTOR_PH:.*]]
+; CHECK: [[VECTOR_PH]]:
+; CHECK-NEXT: [[N_RND_UP:%.*]] = add i64 [[TMP0]], 3
+; CHECK-NEXT: [[N_MOD_VF:%.*]] = urem i64 [[N_RND_UP]], 4
+; CHECK-NEXT: [[N_VEC:%.*]] = sub i64 [[N_RND_UP]], [[N_MOD_VF]]
+; CHECK-NEXT: [[TRIP_COUNT_MINUS_1:%.*]] = sub i64 [[TMP0]], 1
+; CHECK-NEXT: [[BROADCAST_SPLATINSERT:%.*]] = insertelement <4 x i64> poison, i64 [[TRIP_COUNT_MINUS_1]], i64 0
+; CHECK-NEXT: [[BROADCAST_SPLAT:%.*]] = shufflevector <4 x i64> [[BROADCAST_SPLATINSERT]], <4 x i64> poison, <4 x i32> zeroinitializer
+; CHECK-NEXT: br label %[[VECTOR_BODY:.*]]
+; CHECK: [[VECTOR_BODY]]:
+; CHECK-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-NEXT: [[VEC_IND:%.*]] = phi <4 x i64> [ <i64 1, i64 2, i64 3, i64 4>, %[[VECTOR_PH]] ], [ [[VEC_IND_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-NEXT: [[VEC_PHI:%.*]] = phi <4 x i64> [ splat (i64 -1), %[[VECTOR_PH]] ], [ [[TMP11:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-NEXT: [[TMP1:%.*]] = phi <4 x i1> [ zeroinitializer, %[[VECTOR_PH]] ], [ [[TMP10:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-NEXT: [[OFFSET_IDX:%.*]] = add i64 1, [[INDEX]]
+; CHECK-NEXT: [[BROADCAST_SPLATINSERT1:%.*]] = insertelement <4 x i64> poison, i64 [[INDEX]], i64 0
+; CHECK-NEXT: [[BROADCAST_SPLAT2:%.*]] = shufflevector <4 x i64> [[BROADCAST_SPLATINSERT1]], <4 x i64> poison, <4 x i32> zeroinitializer
+; CHECK-NEXT: [[VEC_IV:%.*]] = add <4 x i64> [[BROADCAST_SPLAT2]], <i64 0, i64 1, i64 2, i64 3>
+; CHECK-NEXT: [[TMP2:%.*]] = icmp ule <4 x i64> [[VEC_IV]], [[BROADCAST_SPLAT]]
+; CHECK-NEXT: [[TMP3:%.*]] = getelementptr inbounds i64, ptr [[A]], i64 [[OFFSET_IDX]]
+; CHECK-NEXT: [[WIDE_MASKED_LOAD:%.*]] = call <4 x i64> @llvm.masked.load.v4i64.p0(ptr align 8 [[TMP3]], <4 x i1> [[TMP2]], <4 x i64> poison)
+; CHECK-NEXT: [[TMP4:%.*]] = icmp eq <4 x i64> [[WIDE_MASKED_LOAD]], splat (i64 42)
+; CHECK-NEXT: [[TMP5:%.*]] = select <4 x i1> [[TMP2]], <4 x i64> [[VEC_IND]], <4 x i64> splat (i64 1)
+; CHECK-NEXT: [[TMP6:%.*]] = sdiv <4 x i64> splat (i64 100), [[TMP5]]
+; CHECK-NEXT: [[TMP7:%.*]] = select <4 x i1> [[TMP2]], <4 x i1> [[TMP4]], <4 x i1> zeroinitializer
+; CHECK-NEXT: [[TMP8:%.*]] = freeze <4 x i1> [[TMP7]]
+; CHECK-NEXT: [[TMP9:%.*]] = call i1 @llvm.vector.reduce.or.v4i1(<4 x i1> [[TMP8]])
+; CHECK-NEXT: [[TMP10]] = select i1 [[TMP9]], <4 x i1> [[TMP7]], <4 x i1> [[TMP1]]
+; CHECK-NEXT: [[TMP11]] = select i1 [[TMP9]], <4 x i64> [[TMP6]], <4 x i64> [[VEC_PHI]]
+; CHECK-NEXT: [[INDEX_NEXT]] = add i64 [[INDEX]], 4
+; CHECK-NEXT: [[VEC_IND_NEXT]] = add nuw nsw <4 x i64> [[VEC_IND]], splat (i64 4)
+; CHECK-NEXT: [[TMP12:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
+; CHECK-NEXT: br i1 [[TMP12]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP14:![0-9]+]]
+; CHECK: [[MIDDLE_BLOCK]]:
+; CHECK-NEXT: [[TMP13:%.*]] = call i64 @llvm.experimental.vector.extract.last.active.v4i64(<4 x i64> [[TMP11]], <4 x i1> [[TMP10]], i64 -1)
+; CHECK-NEXT: br label %[[DONE:.*]]
+; CHECK: [[DONE]]:
+; CHECK-NEXT: ret i64 [[TMP13]]
+;
+entry:
+ br label %loop
+
+loop:
+ %iv = phi i64 [ 1, %entry ], [ %iv.next, %loop ]
+ %rdx = phi i64 [ -1, %entry ], [ %sel, %loop ]
+ %gep = getelementptr inbounds i64, ptr %a, i64 %iv
+ %load = load i64, ptr %gep, align 8
+ %cmp = icmp eq i64 %load, 42
+ %expr = sdiv i64 100, %iv
+ %sel = select i1 %cmp, i64 %expr, i64 %rdx
+ %iv.next = add nuw nsw i64 %iv, 1
+ %exit = icmp eq i64 %iv.next, %n
+ br i1 %exit, label %done, label %loop
+
+done:
+ ret i64 %sel
+}
+
+; Test expression sinking with flipped select polarity (TrueVal == PhiR) on the
+; AnyOf path. Based on @select_decreasing_induction_icmp_non_const_start from
+; iv-select-cmp-decreasing.ll but with the select operands swapped to exercise
+; the TrueVal == PhiR case, where the condition must be inverted without
+; corrupting the original Cond used later by the AnyOf tracking.
+define i64 @findlast_expr_flipped_select_anyof(ptr %a, ptr %b, i64 %rdx.start, i64 %n) {
+; CHECK-LABEL: define i64 @findlast_expr_flipped_select_anyof(
+; CHECK-SAME: ptr [[A:%.*]], ptr [[B:%.*]], i64 [[RDX_START:%.*]], i64 [[N:%.*]]) {
+; CHECK-NEXT: [[ENTRY:.*:]]
+; CHECK-NEXT: [[TMP0:%.*]] = add i64 [[N]], 1
+; CHECK-NEXT: [[UMIN:%.*]] = call i64 @llvm.umin.i64(i64 [[N]], i64 1)
+; CHECK-NEXT: [[TMP1:%.*]] = sub i64 [[TMP0]], [[UMIN]]
+; CHECK-NEXT: br label %[[VECTOR_PH:.*]]
+; CHECK: [[VECTOR_PH]]:
+; CHECK-NEXT: [[N_RND_UP:%.*]] = add i64 [[TMP1]], 3
+; CHECK-NEXT: [[N_MOD_VF:%.*]] = urem i64 [[N_RND_UP]], 4
+; CHECK-NEXT: [[N_VEC:%.*]] = sub i64 [[N_RND_UP]], [[N_MOD_VF]]
+; CHECK-NEXT: [[TRIP_COUNT_MINUS_1:%.*]] = sub i64 [[TMP1]], 1
+; CHECK-NEXT: [[BROADCAST_SPLATINSERT:%.*]] = insertelement <4 x i64> poison, i64 [[TRIP_COUNT_MINUS_1]], i64 0
+; CHECK-NEXT: [[BROADCAST_SPLAT:%.*]] = shufflevector <4 x i64> [[BROADCAST_SPLATINSERT]], <4 x i64> poison, <4 x i32> zeroinitializer
+; CHECK-NEXT: [[BROADCAST_SPLATINSERT1:%.*]] = insertelement <4 x i64> poison, i64 [[N]], i64 0
+; CHECK-NEXT: [[BROADCAST_SPLAT2:%.*]] = shufflevector <4 x i64> [[BROADCAST_SPLATINSERT1]], <4 x i64> poison, <4 x i32> zeroinitializer
+; CHECK-NEXT: [[INDUCTION:%.*]] = add nsw <4 x i64> [[BROADCAST_SPLAT2]], <i64 0, i64 -1, i64 -2, i64 -3>
+; CHECK-NEXT: br label %[[VECTOR_BODY:.*]]
+; CHECK: [[VECTOR_BODY]]:
+; CHECK-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-NEXT: [[VEC_IND:%.*]] = phi <4 x i64> [ [[INDUCTION]], %[[VECTOR_PH]] ], [ [[VEC_IND_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-NEXT: [[VEC_PHI:%.*]] = phi <4 x i64> [ splat (i64 9223372036854775807), %[[VECTOR_PH]] ], [ [[TMP9:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-NEXT: [[VEC_PHI3:%.*]] = phi <4 x i1> [ zeroinitializer, %[[VECTOR_PH]] ], [ [[TMP10:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-NEXT: [[OFFSET_IDX:%.*]] = sub i64 [[N]], [[INDEX]]
+; CHECK-NEXT: [[BROADCAST_SPLATINSERT4:%.*]] = insertelement <4 x i64> poison, i64 [[INDEX]], i64 0
+; CHECK-NEXT: [[BROADCAST_SPLAT5:%.*]] = shufflevector <4 x i64> [[BROADCAST_SPLATINSERT4]], <4 x i64> poison, <4 x i32> zeroinitializer
+; CHECK-NEXT: [[VEC_IV:%.*]] = add <4 x i64> [[BROADCAST_SPLAT5]], <i64 0, i64 1, i64 2, i64 3>
+; CHECK-NEXT: [[TMP2:%.*]] = icmp ule <4 x i64> [[VEC_IV]], [[BROADCAST_SPLAT]]
+; CHECK-NEXT: [[TMP3:%.*]] = add nsw i64 [[OFFSET_IDX]], -1
+; CHECK-NEXT: [[TMP4:%.*]] = getelementptr inbounds i64, ptr [[A]], i64 [[TMP3]]
+; CHECK-NEXT: [[TMP5:%.*]] = getelementptr i64, ptr [[TMP4]], i64 -3
+; CHECK-NEXT: [[REVERSE:%.*]] = shufflevector <4 x i1> [[TMP2]], <4 x i1> poison, <4 x i32> <i32 3, i32 2, i32 1, i32 0>
+; CHECK-NEXT: [[WIDE_MASKED_LOAD:%.*]] = call <4 x i64> @llvm.masked.load.v4i64.p0(ptr align 8 [[TMP5]], <4 x i1> [[REVERSE]], <4 x i64> poison)
+; CHECK-NEXT: [[REVERSE6:%.*]] = shufflevector <4 x i64> [[WIDE_MASKED_LOAD]], <4 x i64> poison, <4 x i32> <i32 3, i32 2, i32 1, i32 0>
+; CHECK-NEXT: [[TMP6:%.*]] = getelementptr inbounds i64, ptr [[B]], i64 [[TMP3]]
+; CHECK-NEXT: [[TMP7:%.*]] = getelementptr i64, ptr [[TMP6]], i64 -3
+; CHECK-NEXT: [[WIDE_MASKED_LOAD8:%.*]] = call <4 x i64> @llvm.masked.load.v4i64.p0(ptr align 8 [[TMP7]], <4 x i1> [[REVERSE]], <4 x i64> poison)
+; CHECK-NEXT: [[REVERSE9:%.*]] = shufflevector <4 x i64> [[WIDE_MASKED_LOAD8]], <4 x i64> poison, <4 x i32> <i32 3, i32 2, i32 1, i32 0>
+; CHECK-NEXT: [[TMP8:%.*]] = icmp sle <4 x i64> [[REVERSE6]], [[REVERSE9]]
+; CHECK-NEXT: [[TMP16:%.*]] = select <4 x i1> [[TMP2]], <4 x i1> [[TMP8]], <4 x i1> zeroinitializer
+; CHECK-NEXT: [[TMP9]] = select <4 x i1> [[TMP16]], <4 x i64> [[VEC_IND]], <4 x i64> [[VEC_PHI]]
+; CHECK-NEXT: [[TMP10]] = or <4 x i1> [[VEC_PHI3]], [[TMP16]]
+; CHECK-NEXT: [[INDEX_NEXT]] = add i64 [[INDEX]], 4
+; CHECK-NEXT: [[VEC_IND_NEXT]] = add nsw <4 x i64> [[VEC_IND]], splat (i64 -4)
+; CHECK-NEXT: [[TMP11:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
+; CHECK-NEXT: br i1 [[TMP11]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP15:![0-9]+]]
+; CHECK: [[MIDDLE_BLOCK]]:
+; CHECK-NEXT: [[TMP12:%.*]] = call i64 @llvm.vector.reduce.smin.v4i64(<4 x i64> [[TMP9]])
+; CHECK-NEXT: [[TMP13:%.*]] = add nsw i64 [[TMP12]], -1
+; CHECK-NEXT: [[TMP14:%.*]] = call i1 @llvm.vector.reduce.or.v4i1(<4 x i1> [[TMP10]])
+; CHECK-NEXT: [[TMP15:%.*]] = freeze i1 [[TMP14]]
+; CHECK-NEXT: [[RDX_SELECT:%.*]] = select i1 [[TMP15]], i64 [[TMP13]], i64 [[RDX_START]]
+; CHECK-NEXT: br label %[[EXIT:.*]]
+; CHECK: [[EXIT]]:
+; CHECK-NEXT: ret i64 [[RDX_SELECT]]
+;
+entry:
+ br label %loop
+
+loop:
+ %iv = phi i64 [ %iv.next, %loop ], [ %n, %entry ]
+ %rdx = phi i64 [ %cond, %loop ], [ %rdx.start, %entry ]
+ %iv.next = add nsw i64 %iv, -1
+ %gep.a.iv = getelementptr inbounds i64, ptr %a, i64 %iv.next
+ %ld.a = load i64, ptr %gep.a.iv, align 8
+ %gep.b.iv = getelementptr inbounds i64, ptr %b, i64 %iv.next
+ %ld.b = load i64, ptr %gep.b.iv, align 8
+ %cmp.a.b = icmp sgt i64 %ld.a, %ld.b
+ ; Flipped: TrueVal is rdx (PhiR), FalseVal is iv.next (expression of IV)
+ %cond = select i1 %cmp.a.b, i64 %rdx, i64 %iv.next
+ %exit.cond = icmp ugt i64 %iv, 1
+ br i1 %exit.cond, label %loop, label %exit
+
+exit:
+ ret i64 %cond
+}
+
+; Test case where the expression %iv steps by one, but %expr is an AddRec
+; stepping by -1.
+define i64 @findlast_sub_n_expr(ptr %a, i64 %n) {
+; CHECK-LABEL: define i64 @findlast_sub_n_expr(
+; CHECK-SAME: ptr [[A:%.*]], i64 [[N:%.*]]) {
+; CHECK-NEXT: [[ENTRY:.*:]]
+; CHECK-NEXT: br label %[[VECTOR_PH:.*]]
+; CHECK: [[VECTOR_PH]]:
+; CHECK-NEXT: [[N_RND_UP:%.*]] = add i64 [[N]], 3
+; CHECK-NEXT: [[N_MOD_VF:%.*]] = urem i64 [[N_RND_UP]], 4
+; CHECK-NEXT: [[N_VEC:%.*]] = sub i64 [[N_RND_UP]], [[N_MOD_VF]]
+; CHECK-NEXT: [[TRIP_COUNT_MINUS_1:%.*]] = sub i64 [[N]], 1
+; CHECK-NEXT: [[BROADCAST_SPLATINSERT:%.*]] = insertelement <4 x i64> poison, i64 [[TRIP_COUNT_MINUS_1]], i64 0
+; CHECK-NEXT: [[BROADCAST_SPLAT:%.*]] = shufflevector <4 x i64> [[BROADCAST_SPLATINSERT]], <4 x i64> poison, <4 x i32> zeroinitializer
+; CHECK-NEXT: br label %[[VECTOR_BODY:.*]]
+; CHECK: [[VECTOR_BODY]]:
+; CHECK-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-NEXT: [[VEC_IND:%.*]] = phi <4 x i64> [ <i64 0, i64 1, i64 2, i64 3>, %[[VECTOR_PH]] ], [ [[VEC_IND_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-NEXT: [[VEC_PHI:%.*]] = phi <4 x i64> [ splat (i64 -9223372036854775808), %[[VECTOR_PH]] ], [ [[TMP3:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-NEXT: [[TMP0:%.*]] = icmp ule <4 x i64> [[VEC_IND]], [[BROADCAST_SPLAT]]
+; CHECK-NEXT: [[TMP1:%.*]] = getelementptr inbounds i64, ptr [[A]], i64 [[INDEX]]
+; CHECK-NEXT: [[WIDE_MASKED_LOAD:%.*]] = call <4 x i64> @llvm.masked.load.v4i64.p0(ptr align 8 [[TMP1]], <4 x i1> [[TMP0]], <4 x i64> poison)
+; CHECK-NEXT: [[TMP2:%.*]] = icmp eq <4 x i64> [[WIDE_MASKED_LOAD]], splat (i64 42)
+; CHECK-NEXT: [[TMP9:%.*]] = select <4 x i1> [[TMP0]], <4 x i1> [[TMP2]], <4 x i1> zeroinitializer
+; CHECK-NEXT: [[TMP3]] = select <4 x i1> [[TMP9]], <4 x i64> [[VEC_IND]], <4 x i64> [[VEC_PHI]]
+; CHECK-NEXT: [[INDEX_NEXT]] = add i64 [[INDEX]], 4
+; CHECK-NEXT: [[VEC_IND_NEXT]] = add <4 x i64> [[VEC_IND]], splat (i64 4)
+; CHECK-NEXT: [[TMP4:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
+; CHECK-NEXT: br i1 [[TMP4]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP16:![0-9]+]]
+; CHECK: [[MIDDLE_BLOCK]]:
+; CHECK-NEXT: [[TMP5:%.*]] = call i64 @llvm.vector.reduce.smax.v4i64(<4 x i64> [[TMP3]])
+; CHECK-NEXT: [[TMP6:%.*]] = sub i64 [[N]], [[TMP5]]
+; CHECK-NEXT: [[TMP7:%.*]] = icmp ne i64 [[TMP5]], -9223372036854775808
+; CHECK-NEXT: [[TMP8:%.*]] = select i1 [[TMP7]], i64 [[TMP6]], i64 -1
+; CHECK-NEXT: br label %[[EXIT:.*]]
+; CHECK: [[EXIT]]:
+; CHECK-NEXT: ret i64 [[TMP8]]
+;
+entry:
+ br label %loop
+
+loop:
+ %iv = phi i64 [ 0, %entry ], [ %iv.next, %loop ]
+ %rdx = phi i64 [ -1, %entry ], [ %sel, %loop ]
+ %gep = getelementptr inbounds i64, ptr %a, i64 %iv
+ %load = load i64, ptr %gep, align 8
+ %cmp = icmp eq i64 %load, 42
+ %expr = sub i64 %n, %iv
+ %sel = select i1 %cmp, i64 %expr, i64 %rdx
+ %iv.next = add nuw nsw i64 %iv, 1
+ %exitcond = icmp eq i64 %iv.next, %n
+ br i1 %exitcond, label %exit, label %loop
+
+exit:
+ ret i64 %sel
+}
+
+; Test sub where IV is the first operand (non-commutative). Verifies that
+; operand order is preserved when sinking the expression to the middle block.
+define i64 @findlast_sub_iv_as_lhs(ptr %a, i64 %n) {
+; CHECK-LABEL: define i64 @findlast_sub_iv_as_lhs(
+; CHECK-SAME: ptr [[A:%.*]], i64 [[N:%.*]]) {
+; CHECK-NEXT: [[ENTRY:.*:]]
+; CHECK-NEXT: br label %[[VECTOR_PH:.*]]
+; CHECK: [[VECTOR_PH]]:
+; CHECK-NEXT: [[N_RND_UP:%.*]] = add i64 [[N]], 3
+; CHECK-NEXT: [[N_MOD_VF:%.*]] = urem i64 [[N_RND_UP]], 4
+; CHECK-NEXT: [[N_VEC:%.*]] = sub i64 [[N_RND_UP]], [[N_MOD_VF]]
+; CHECK-NEXT: [[TRIP_COUNT_MINUS_1:%.*]] = sub i64 [[N]], 1
+; CHECK-NEXT: [[BROADCAST_SPLATINSERT:%.*]] = insertelement <4 x i64> poison, i64 [[TRIP_COUNT_MINUS_1]], i64 0
+; CHECK-NEXT: [[BROADCAST_SPLAT:%.*]] = shufflevector <4 x i64> [[BROADCAST_SPLATINSERT]], <4 x i64> poison, <4 x i32> zeroinitializer
+; CHECK-NEXT: br label %[[VECTOR_BODY:.*]]
+; CHECK: [[VECTOR_BODY]]:
+; CHECK-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-NEXT: [[VEC_IND:%.*]] = phi <4 x i64> [ <i64 0, i64 1, i64 2, i64 3>, %[[VECTOR_PH]] ], [ [[VEC_IND_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-NEXT: [[VEC_PHI:%.*]] = phi <4 x i64> [ splat (i64 -9223372036854775808), %[[VECTOR_PH]] ], [ [[TMP3:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-NEXT: [[TMP0:%.*]] = icmp ule <4 x i64> [[VEC_IND]], [[BROADCAST_SPLAT]]
+; CHECK-NEXT: [[TMP1:%.*]] = getelementptr inbounds i64, ptr [[A]], i64 [[INDEX]]
+; CHECK-NEXT: [[WIDE_MASKED_LOAD:%.*]] = call <4 x i64> @llvm.masked.load.v4i64.p0(ptr align 8 [[TMP1]], <4 x i1> [[TMP0]], <4 x i64> poison)
+; CHECK-NEXT: [[TMP2:%.*]] = icmp eq <4 x i64> [[WIDE_MASKED_LOAD]], splat (i64 42)
+; CHECK-NEXT: [[TMP9:%.*]] = select <4 x i1> [[TMP0]], <4 x i1> [[TMP2]], <4 x i1> zeroinitializer
+; CHECK-NEXT: [[TMP3]] = select <4 x i1> [[TMP9]], <4 x i64> [[VEC_IND]], <4 x i64> [[VEC_PHI]]
+; CHECK-NEXT: [[INDEX_NEXT]] = add i64 [[INDEX]], 4
+; CHECK-NEXT: [[VEC_IND_NEXT]] = add <4 x i64> [[VEC_IND]], splat (i64 4)
+; CHECK-NEXT: [[TMP4:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
+; CHECK-NEXT: br i1 [[TMP4]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP17:![0-9]+]]
+; CHECK: [[MIDDLE_BLOCK]]:
+; CHECK-NEXT: [[TMP5:%.*]] = call i64 @llvm.vector.reduce.smax.v4i64(<4 x i64> [[TMP3]])
+; CHECK-NEXT: [[TMP6:%.*]] = sub i64 [[TMP5]], [[N]]
+; CHECK-NEXT: [[TMP7:%.*]] = icmp ne i64 [[TMP5]], -9223372036854775808
+; CHECK-NEXT: [[TMP8:%.*]] = select i1 [[TMP7]], i64 [[TMP6]], i64 -1
+; CHECK-NEXT: br label %[[EXIT:.*]]
+; CHECK: [[EXIT]]:
+; CHECK-NEXT: ret i64 [[TMP8]]
+;
+entry:
+ br label %loop
+
+loop:
+ %iv = phi i64 [ 0, %entry ], [ %iv.next, %loop ]
+ %rdx = phi i64 [ -1, %entry ], [ %sel, %loop ]
+ %gep = getelementptr inbounds i64, ptr %a, i64 %iv
+ %load = load i64, ptr %gep, align 8
+ %cmp = icmp eq i64 %load, 42
+ %expr = sub i64 %iv, %n
+ %sel = select i1 %cmp, i64 %expr, i64 %rdx
+ %iv.next = add nuw nsw i64 %iv, 1
+ %exitcond = icmp eq i64 %iv.next, %n
+ br i1 %exitcond, label %exit, label %loop
+
+exit:
+ ret i64 %sel
+}
+
+; The IV starts at SMIN, so the signed range includes SMIN; signed sentinel
+; fails and an unsigned sentinel = 0 is used. The sunk udiv in the middle block
+; would divide by 0 if the condition is never true.
+define i64 @findlast_udiv_may_trap_due_to_sentinel(ptr %a, i64 %n) {
+; CHECK-LABEL: define i64 @findlast_udiv_may_trap_due_to_sentinel(
+; CHECK-SAME: ptr [[A:%.*]], i64 [[N:%.*]]) {
+; CHECK-NEXT: [[ENTRY:.*:]]
+; CHECK-NEXT: [[TMP0:%.*]] = add i64 [[N]], -9223372036854775808
+; CHECK-NEXT: br label %[[VECTOR_PH:.*]]
+; CHECK: [[VECTOR_PH]]:
+; CHECK-NEXT: [[N_RND_UP:%.*]] = add i64 [[TMP0]], 3
+; CHECK-NEXT: [[N_MOD_VF:%.*]] = urem i64 [[N_RND_UP]], 4
+; CHECK-NEXT: [[N_VEC:%.*]] = sub i64 [[N_RND_UP]], [[N_MOD_VF]]
+; CHECK-NEXT: [[TRIP_COUNT_MINUS_1:%.*]] = sub i64 [[TMP0]], 1
+; CHECK-NEXT: [[BROADCAST_SPLATINSERT:%.*]] = insertelement <4 x i64> poison, i64 [[TRIP_COUNT_MINUS_1]], i64 0
+; CHECK-NEXT: [[BROADCAST_SPLAT:%.*]] = shufflevector <4 x i64> [[BROADCAST_SPLATINSERT]], <4 x i64> poison, <4 x i32> zeroinitializer
+; CHECK-NEXT: br label %[[VECTOR_BODY:.*]]
+; CHECK: [[VECTOR_BODY]]:
+; CHECK-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-NEXT: [[VEC_IND:%.*]] = phi <4 x i64> [ <i64 -9223372036854775808, i64 -9223372036854775807, i64 -9223372036854775806, i64 -9223372036854775805>, %[[VECTOR_PH]] ], [ [[VEC_IND_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-NEXT: [[VEC_PHI:%.*]] = phi <4 x i64> [ splat (i64 -1), %[[VECTOR_PH]] ], [ [[TMP11:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-NEXT: [[TMP1:%.*]] = phi <4 x i1> [ zeroinitializer, %[[VECTOR_PH]] ], [ [[TMP10:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-NEXT: [[OFFSET_IDX:%.*]] = add i64 -9223372036854775808, [[INDEX]]
+; CHECK-NEXT: [[BROADCAST_SPLATINSERT1:%.*]] = insertelement <4 x i64> poison, i64 [[INDEX]], i64 0
+; CHECK-NEXT: [[BROADCAST_SPLAT2:%.*]] = shufflevector <4 x i64> [[BROADCAST_SPLATINSERT1]], <4 x i64> poison, <4 x i32> zeroinitializer
+; CHECK-NEXT: [[VEC_IV:%.*]] = add <4 x i64> [[BROADCAST_SPLAT2]], <i64 0, i64 1, i64 2, i64 3>
+; CHECK-NEXT: [[TMP2:%.*]] = icmp ule <4 x i64> [[VEC_IV]], [[BROADCAST_SPLAT]]
+; CHECK-NEXT: [[TMP3:%.*]] = getelementptr inbounds i64, ptr [[A]], i64 [[OFFSET_IDX]]
+; CHECK-NEXT: [[WIDE_MASKED_LOAD:%.*]] = call <4 x i64> @llvm.masked.load.v4i64.p0(ptr align 8 [[TMP3]], <4 x i1> [[TMP2]], <4 x i64> poison)
+; CHECK-NEXT: [[TMP4:%.*]] = icmp eq <4 x i64> [[WIDE_MASKED_LOAD]], splat (i64 42)
+; CHECK-NEXT: [[TMP5:%.*]] = select <4 x i1> [[TMP2]], <4 x i64> [[VEC_IND]], <4 x i64> splat (i64 1)
+; CHECK-NEXT: [[TMP6:%.*]] = udiv <4 x i64> splat (i64 100), [[TMP5]]
+; CHECK-NEXT: [[TMP7:%.*]] = select <4 x i1> [[TMP2]], <4 x i1> [[TMP4]], <4 x i1> zeroinitializer
+; CHECK-NEXT: [[TMP8:%.*]] = freeze <4 x i1> [[TMP7]]
+; CHECK-NEXT: [[TMP9:%.*]] = call i1 @llvm.vector.reduce.or.v4i1(<4 x i1> [[TMP8]])
+; CHECK-NEXT: [[TMP10]] = select i1 [[TMP9]], <4 x i1> [[TMP7]], <4 x i1> [[TMP1]]
+; CHECK-NEXT: [[TMP11]] = select i1 [[TMP9]], <4 x i64> [[TMP6]], <4 x i64> [[VEC_PHI]]
+; CHECK-NEXT: [[INDEX_NEXT]] = add i64 [[INDEX]], 4
+; CHECK-NEXT: [[VEC_IND_NEXT]] = add nuw <4 x i64> [[VEC_IND]], splat (i64 4)
+; CHECK-NEXT: [[TMP12:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
+; CHECK-NEXT: br i1 [[TMP12]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP18:![0-9]+]]
+; CHECK: [[MIDDLE_BLOCK]]:
+; CHECK-NEXT: [[TMP13:%.*]] = call i64 @llvm.experimental.vector.extract.last.active.v4i64(<4 x i64> [[TMP11]], <4 x i1> [[TMP10]], i64 -1)
+; CHECK-NEXT: br label %[[DONE:.*]]
+; CHECK: [[DONE]]:
+; CHECK-NEXT: ret i64 [[TMP13]]
+;
+entry:
+ br label %loop
+
+loop:
+ %iv = phi i64 [ -9223372036854775808, %entry ], [ %iv.next, %loop ]
+ %rdx = phi i64 [ -1, %entry ], [ %sel, %loop ]
+ %gep = getelementptr inbounds i64, ptr %a, i64 %iv
+ %load = load i64, ptr %gep, align 8
+ %cmp = icmp eq i64 %load, 42
+ %expr = udiv i64 100, %iv
+ %sel = select i1 %cmp, i64 %expr, i64 %rdx
+ %iv.next = add nuw i64 %iv, 1
+ %exit = icmp eq i64 %iv.next, %n
+ br i1 %exit, label %done, label %loop
+
+done:
+ ret i64 %sel
+}
+
+; srem is not sinkable (may trap), so the expression should not be sunk.
+define i64 @findlast_srem_iv_as_divisor(ptr %a, i64 %n) {
+; CHECK-LABEL: define i64 @findlast_srem_iv_as_divisor(
+; CHECK-SAME: ptr [[A:%.*]], i64 [[N:%.*]]) {
+; CHECK-NEXT: [[ENTRY:.*:]]
+; CHECK-NEXT: [[TMP0:%.*]] = add i64 [[N]], -1
+; CHECK-NEXT: br label %[[VECTOR_PH:.*]]
+; CHECK: [[VECTOR_PH]]:
+; CHECK-NEXT: [[N_RND_UP:%.*]] = add i64 [[TMP0]], 3
+; CHECK-NEXT: [[N_MOD_VF:%.*]] = urem i64 [[N_RND_UP]], 4
+; CHECK-NEXT: [[N_VEC:%.*]] = sub i64 [[N_RND_UP]], [[N_MOD_VF]]
+; CHECK-NEXT: [[TRIP_COUNT_MINUS_1:%.*]] = sub i64 [[TMP0]], 1
+; CHECK-NEXT: [[BROADCAST_SPLATINSERT:%.*]] = insertelement <4 x i64> poison, i64 [[TRIP_COUNT_MINUS_1]], i64 0
+; CHECK-NEXT: [[BROADCAST_SPLAT:%.*]] = shufflevector <4 x i64> [[BROADCAST_SPLATINSERT]], <4 x i64> poison, <4 x i32> zeroinitializer
+; CHECK-NEXT: br label %[[VECTOR_BODY:.*]]
+; CHECK: [[VECTOR_BODY]]:
+; CHECK-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-NEXT: [[VEC_IND:%.*]] = phi <4 x i64> [ <i64 1, i64 2, i64 3, i64 4>, %[[VECTOR_PH]] ], [ [[VEC_IND_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-NEXT: [[VEC_PHI:%.*]] = phi <4 x i64> [ splat (i64 -1), %[[VECTOR_PH]] ], [ [[TMP11:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-NEXT: [[TMP1:%.*]] = phi <4 x i1> [ zeroinitializer, %[[VECTOR_PH]] ], [ [[TMP10:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-NEXT: [[OFFSET_IDX:%.*]] = add i64 1, [[INDEX]]
+; CHECK-NEXT: [[BROADCAST_SPLATINSERT1:%.*]] = insertelement <4 x i64> poison, i64 [[INDEX]], i64 0
+; CHECK-NEXT: [[BROADCAST_SPLAT2:%.*]] = shufflevector <4 x i64> [[BROADCAST_SPLATINSERT1]], <4 x i64> poison, <4 x i32> zeroinitializer
+; CHECK-NEXT: [[VEC_IV:%.*]] = add <4 x i64> [[BROADCAST_SPLAT2]], <i64 0, i64 1, i64 2, i64 3>
+; CHECK-NEXT: [[TMP2:%.*]] = icmp ule <4 x i64> [[VEC_IV]], [[BROADCAST_SPLAT]]
+; CHECK-NEXT: [[TMP3:%.*]] = getelementptr inbounds i64, ptr [[A]], i64 [[OFFSET_IDX]]
+; CHECK-NEXT: [[WIDE_MASKED_LOAD:%.*]] = call <4 x i64> @llvm.masked.load.v4i64.p0(ptr align 8 [[TMP3]], <4 x i1> [[TMP2]], <4 x i64> poison)
+; CHECK-NEXT: [[TMP4:%.*]] = icmp eq <4 x i64> [[WIDE_MASKED_LOAD]], splat (i64 42)
+; CHECK-NEXT: [[TMP5:%.*]] = select <4 x i1> [[TMP2]], <4 x i64> [[VEC_IND]], <4 x i64> splat (i64 1)
+; CHECK-NEXT: [[TMP6:%.*]] = srem <4 x i64> splat (i64 100), [[TMP5]]
+; CHECK-NEXT: [[TMP7:%.*]] = select <4 x i1> [[TMP2]], <4 x i1> [[TMP4]], <4 x i1> zeroinitializer
+; CHECK-NEXT: [[TMP8:%.*]] = freeze <4 x i1> [[TMP7]]
+; CHECK-NEXT: [[TMP9:%.*]] = call i1 @llvm.vector.reduce.or.v4i1(<4 x i1> [[TMP8]])
+; CHECK-NEXT: [[TMP10]] = select i1 [[TMP9]], <4 x i1> [[TMP7]], <4 x i1> [[TMP1]]
+; CHECK-NEXT: [[TMP11]] = select i1 [[TMP9]], <4 x i64> [[TMP6]], <4 x i64> [[VEC_PHI]]
+; CHECK-NEXT: [[INDEX_NEXT]] = add i64 [[INDEX]], 4
+; CHECK-NEXT: [[VEC_IND_NEXT]] = add nuw nsw <4 x i64> [[VEC_IND]], splat (i64 4)
+; CHECK-NEXT: [[TMP12:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
+; CHECK-NEXT: br i1 [[TMP12]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP19:![0-9]+]]
+; CHECK: [[MIDDLE_BLOCK]]:
+; CHECK-NEXT: [[TMP13:%.*]] = call i64 @llvm.experimental.vector.extract.last.active.v4i64(<4 x i64> [[TMP11]], <4 x i1> [[TMP10]], i64 -1)
+; CHECK-NEXT: br label %[[DONE:.*]]
+; CHECK: [[DONE]]:
+; CHECK-NEXT: ret i64 [[TMP13]]
+;
+entry:
+ br label %loop
+
+loop:
+ %iv = phi i64 [ 1, %entry ], [ %iv.next, %loop ]
+ %rdx = phi i64 [ -1, %entry ], [ %sel, %loop ]
+ %gep = getelementptr inbounds i64, ptr %a, i64 %iv
+ %load = load i64, ptr %gep, align 8
+ %cmp = icmp eq i64 %load, 42
+ %expr = srem i64 100, %iv
+ %sel = select i1 %cmp, i64 %expr, i64 %rdx
+ %iv.next = add nuw nsw i64 %iv, 1
+ %exit = icmp eq i64 %iv.next, %n
+ br i1 %exit, label %done, label %loop
+
+done:
+ ret i64 %sel
+}
diff --git a/llvm/test/Transforms/LoopVectorize/find-last-iv-sinkable-expr.ll b/llvm/test/Transforms/LoopVectorize/find-last-iv-sinkable-expr.ll
index 0554be0b0cc70..eaf53139c3ff0 100644
--- a/llvm/test/Transforms/LoopVectorize/find-last-iv-sinkable-expr.ll
+++ b/llvm/test/Transforms/LoopVectorize/find-last-iv-sinkable-expr.ll
@@ -1,92 +1,56 @@
; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --check-globals none --version 6
-; RUN: opt -passes=loop-vectorize -force-vector-interleave=1 -force-vector-width=4 -S %s | FileCheck %s --check-prefix=NO-TF
-; RUN: opt -passes=loop-vectorize -force-vector-interleave=1 -force-vector-width=4 -prefer-predicate-over-epilogue=predicate-else-scalar-epilogue -force-target-supports-masked-memory-ops -S %s | FileCheck %s --check-prefix=TF
+; RUN: opt -passes=loop-vectorize -force-vector-interleave=1 -force-vector-width=4 -S < %s | FileCheck %s
; Tests for FindLastIV reductions where the selected value is an expression
; of the induction variable (e.g., IV * const, IV << const, IV | const).
define i64 @findlast_shl_expr(ptr %a, i64 %n) {
-; NO-TF-LABEL: define i64 @findlast_shl_expr(
-; NO-TF-SAME: ptr [[A:%.*]], i64 [[N:%.*]]) {
-; NO-TF-NEXT: [[ENTRY:.*]]:
-; NO-TF-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[N]], 4
-; NO-TF-NEXT: br i1 [[MIN_ITERS_CHECK]], label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]]
-; NO-TF: [[VECTOR_PH]]:
-; NO-TF-NEXT: [[N_MOD_VF:%.*]] = urem i64 [[N]], 4
-; NO-TF-NEXT: [[N_VEC:%.*]] = sub i64 [[N]], [[N_MOD_VF]]
-; NO-TF-NEXT: br label %[[VECTOR_BODY:.*]]
-; NO-TF: [[VECTOR_BODY]]:
-; NO-TF-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
-; NO-TF-NEXT: [[VEC_IND:%.*]] = phi <4 x i64> [ <i64 0, i64 1, i64 2, i64 3>, %[[VECTOR_PH]] ], [ [[VEC_IND_NEXT:%.*]], %[[VECTOR_BODY]] ]
-; NO-TF-NEXT: [[VEC_PHI:%.*]] = phi <4 x i64> [ splat (i64 -9223372036854775808), %[[VECTOR_PH]] ], [ [[TMP2:%.*]], %[[VECTOR_BODY]] ]
-; NO-TF-NEXT: [[TMP0:%.*]] = getelementptr inbounds i64, ptr [[A]], i64 [[INDEX]]
-; NO-TF-NEXT: [[WIDE_LOAD:%.*]] = load <4 x i64>, ptr [[TMP0]], align 8
-; NO-TF-NEXT: [[TMP1:%.*]] = icmp eq <4 x i64> [[WIDE_LOAD]], splat (i64 42)
-; NO-TF-NEXT: [[TMP2]] = select <4 x i1> [[TMP1]], <4 x i64> [[VEC_IND]], <4 x i64> [[VEC_PHI]]
-; NO-TF-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 4
-; NO-TF-NEXT: [[VEC_IND_NEXT]] = add nuw nsw <4 x i64> [[VEC_IND]], splat (i64 4)
-; NO-TF-NEXT: [[TMP3:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
-; NO-TF-NEXT: br i1 [[TMP3]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP0:![0-9]+]]
-; NO-TF: [[MIDDLE_BLOCK]]:
-; NO-TF-NEXT: [[TMP4:%.*]] = call i64 @llvm.vector.reduce.smax.v4i64(<4 x i64> [[TMP2]])
-; NO-TF-NEXT: [[TMP5:%.*]] = shl i64 [[TMP4]], 2
-; NO-TF-NEXT: [[TMP6:%.*]] = icmp ne i64 [[TMP4]], -9223372036854775808
-; NO-TF-NEXT: [[TMP7:%.*]] = select i1 [[TMP6]], i64 [[TMP5]], i64 -1
-; NO-TF-NEXT: [[CMP_N:%.*]] = icmp eq i64 [[N]], [[N_VEC]]
-; NO-TF-NEXT: br i1 [[CMP_N]], label %[[DONE:.*]], label %[[SCALAR_PH]]
-; NO-TF: [[SCALAR_PH]]:
-; NO-TF-NEXT: [[BC_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC]], %[[MIDDLE_BLOCK]] ], [ 0, %[[ENTRY]] ]
-; NO-TF-NEXT: [[BC_MERGE_RDX:%.*]] = phi i64 [ [[TMP7]], %[[MIDDLE_BLOCK]] ], [ -1, %[[ENTRY]] ]
-; NO-TF-NEXT: br label %[[LOOP:.*]]
-; NO-TF: [[LOOP]]:
-; NO-TF-NEXT: [[IV:%.*]] = phi i64 [ [[BC_RESUME_VAL]], %[[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], %[[LOOP]] ]
-; NO-TF-NEXT: [[RDX:%.*]] = phi i64 [ [[BC_MERGE_RDX]], %[[SCALAR_PH]] ], [ [[SEL:%.*]], %[[LOOP]] ]
-; NO-TF-NEXT: [[GEP:%.*]] = getelementptr inbounds i64, ptr [[A]], i64 [[IV]]
-; NO-TF-NEXT: [[LOAD:%.*]] = load i64, ptr [[GEP]], align 8
-; NO-TF-NEXT: [[CMP:%.*]] = icmp eq i64 [[LOAD]], 42
-; NO-TF-NEXT: [[EXPR:%.*]] = shl i64 [[IV]], 2
-; NO-TF-NEXT: [[SEL]] = select i1 [[CMP]], i64 [[EXPR]], i64 [[RDX]]
-; NO-TF-NEXT: [[IV_NEXT]] = add nuw nsw i64 [[IV]], 1
-; NO-TF-NEXT: [[EXIT:%.*]] = icmp eq i64 [[IV_NEXT]], [[N]]
-; NO-TF-NEXT: br i1 [[EXIT]], label %[[DONE]], label %[[LOOP]], !llvm.loop [[LOOP3:![0-9]+]]
-; NO-TF: [[DONE]]:
-; NO-TF-NEXT: [[SEL_LCSSA:%.*]] = phi i64 [ [[SEL]], %[[LOOP]] ], [ [[TMP7]], %[[MIDDLE_BLOCK]] ]
-; NO-TF-NEXT: ret i64 [[SEL_LCSSA]]
-;
-; TF-LABEL: define i64 @findlast_shl_expr(
-; TF-SAME: ptr [[A:%.*]], i64 [[N:%.*]]) {
-; TF-NEXT: [[ENTRY:.*:]]
-; TF-NEXT: br label %[[VECTOR_PH:.*]]
-; TF: [[VECTOR_PH]]:
-; TF-NEXT: [[N_RND_UP:%.*]] = add i64 [[N]], 3
-; TF-NEXT: [[N_MOD_VF:%.*]] = urem i64 [[N_RND_UP]], 4
-; TF-NEXT: [[N_VEC:%.*]] = sub i64 [[N_RND_UP]], [[N_MOD_VF]]
-; TF-NEXT: [[TRIP_COUNT_MINUS_1:%.*]] = sub i64 [[N]], 1
-; TF-NEXT: [[BROADCAST_SPLATINSERT:%.*]] = insertelement <4 x i64> poison, i64 [[TRIP_COUNT_MINUS_1]], i64 0
-; TF-NEXT: [[BROADCAST_SPLAT:%.*]] = shufflevector <4 x i64> [[BROADCAST_SPLATINSERT]], <4 x i64> poison, <4 x i32> zeroinitializer
-; TF-NEXT: br label %[[VECTOR_BODY:.*]]
-; TF: [[VECTOR_BODY]]:
-; TF-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
-; TF-NEXT: [[VEC_IND:%.*]] = phi <4 x i64> [ <i64 0, i64 1, i64 2, i64 3>, %[[VECTOR_PH]] ], [ [[VEC_IND_NEXT:%.*]], %[[VECTOR_BODY]] ]
-; TF-NEXT: [[VEC_PHI:%.*]] = phi <4 x i64> [ splat (i64 -9223372036854775808), %[[VECTOR_PH]] ], [ [[TMP3:%.*]], %[[VECTOR_BODY]] ]
-; TF-NEXT: [[TMP0:%.*]] = icmp ule <4 x i64> [[VEC_IND]], [[BROADCAST_SPLAT]]
-; TF-NEXT: [[TMP1:%.*]] = getelementptr inbounds i64, ptr [[A]], i64 [[INDEX]]
-; TF-NEXT: [[WIDE_MASKED_LOAD:%.*]] = call <4 x i64> @llvm.masked.load.v4i64.p0(ptr align 8 [[TMP1]], <4 x i1> [[TMP0]], <4 x i64> poison)
-; TF-NEXT: [[TMP2:%.*]] = icmp eq <4 x i64> [[WIDE_MASKED_LOAD]], splat (i64 42)
-; TF-NEXT: [[TMP9:%.*]] = select <4 x i1> [[TMP0]], <4 x i1> [[TMP2]], <4 x i1> zeroinitializer
-; TF-NEXT: [[TMP3]] = select <4 x i1> [[TMP9]], <4 x i64> [[VEC_IND]], <4 x i64> [[VEC_PHI]]
-; TF-NEXT: [[INDEX_NEXT]] = add i64 [[INDEX]], 4
-; TF-NEXT: [[VEC_IND_NEXT]] = add <4 x i64> [[VEC_IND]], splat (i64 4)
-; TF-NEXT: [[TMP4:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
-; TF-NEXT: br i1 [[TMP4]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP0:![0-9]+]]
-; TF: [[MIDDLE_BLOCK]]:
-; TF-NEXT: [[TMP5:%.*]] = call i64 @llvm.vector.reduce.smax.v4i64(<4 x i64> [[TMP3]])
-; TF-NEXT: [[TMP6:%.*]] = shl i64 [[TMP5]], 2
-; TF-NEXT: [[TMP7:%.*]] = icmp ne i64 [[TMP5]], -9223372036854775808
-; TF-NEXT: [[TMP8:%.*]] = select i1 [[TMP7]], i64 [[TMP6]], i64 -1
-; TF-NEXT: br label %[[DONE:.*]]
-; TF: [[DONE]]:
-; TF-NEXT: ret i64 [[TMP8]]
+; CHECK-LABEL: define i64 @findlast_shl_expr(
+; CHECK-SAME: ptr [[A:%.*]], i64 [[N:%.*]]) {
+; CHECK-NEXT: [[ENTRY:.*]]:
+; CHECK-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[N]], 4
+; CHECK-NEXT: br i1 [[MIN_ITERS_CHECK]], label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]]
+; CHECK: [[VECTOR_PH]]:
+; CHECK-NEXT: [[N_MOD_VF:%.*]] = urem i64 [[N]], 4
+; CHECK-NEXT: [[N_VEC:%.*]] = sub i64 [[N]], [[N_MOD_VF]]
+; CHECK-NEXT: br label %[[VECTOR_BODY:.*]]
+; CHECK: [[VECTOR_BODY]]:
+; CHECK-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-NEXT: [[VEC_IND:%.*]] = phi <4 x i64> [ <i64 0, i64 1, i64 2, i64 3>, %[[VECTOR_PH]] ], [ [[VEC_IND_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-NEXT: [[VEC_PHI:%.*]] = phi <4 x i64> [ splat (i64 -9223372036854775808), %[[VECTOR_PH]] ], [ [[TMP3:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-NEXT: [[TMP1:%.*]] = getelementptr inbounds i64, ptr [[A]], i64 [[INDEX]]
+; CHECK-NEXT: [[WIDE_LOAD:%.*]] = load <4 x i64>, ptr [[TMP1]], align 8
+; CHECK-NEXT: [[TMP2:%.*]] = icmp eq <4 x i64> [[WIDE_LOAD]], splat (i64 42)
+; CHECK-NEXT: [[TMP3]] = select <4 x i1> [[TMP2]], <4 x i64> [[VEC_IND]], <4 x i64> [[VEC_PHI]]
+; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 4
+; CHECK-NEXT: [[VEC_IND_NEXT]] = add nuw nsw <4 x i64> [[VEC_IND]], splat (i64 4)
+; CHECK-NEXT: [[TMP8:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
+; CHECK-NEXT: br i1 [[TMP8]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP0:![0-9]+]]
+; CHECK: [[MIDDLE_BLOCK]]:
+; CHECK-NEXT: [[TMP4:%.*]] = call i64 @llvm.vector.reduce.smax.v4i64(<4 x i64> [[TMP3]])
+; CHECK-NEXT: [[TMP5:%.*]] = shl i64 [[TMP4]], 2
+; CHECK-NEXT: [[TMP6:%.*]] = icmp ne i64 [[TMP4]], -9223372036854775808
+; CHECK-NEXT: [[TMP7:%.*]] = select i1 [[TMP6]], i64 [[TMP5]], i64 -1
+; CHECK-NEXT: [[CMP_N:%.*]] = icmp eq i64 [[N]], [[N_VEC]]
+; CHECK-NEXT: br i1 [[CMP_N]], label %[[DONE:.*]], label %[[SCALAR_PH]]
+; CHECK: [[SCALAR_PH]]:
+; CHECK-NEXT: [[BC_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC]], %[[MIDDLE_BLOCK]] ], [ 0, %[[ENTRY]] ]
+; CHECK-NEXT: [[BC_MERGE_RDX:%.*]] = phi i64 [ [[TMP7]], %[[MIDDLE_BLOCK]] ], [ -1, %[[ENTRY]] ]
+; CHECK-NEXT: br label %[[LOOP:.*]]
+; CHECK: [[LOOP]]:
+; CHECK-NEXT: [[IV:%.*]] = phi i64 [ [[BC_RESUME_VAL]], %[[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], %[[LOOP]] ]
+; CHECK-NEXT: [[RDX:%.*]] = phi i64 [ [[BC_MERGE_RDX]], %[[SCALAR_PH]] ], [ [[SEL:%.*]], %[[LOOP]] ]
+; CHECK-NEXT: [[GEP:%.*]] = getelementptr inbounds i64, ptr [[A]], i64 [[IV]]
+; CHECK-NEXT: [[LOAD:%.*]] = load i64, ptr [[GEP]], align 8
+; CHECK-NEXT: [[CMP:%.*]] = icmp eq i64 [[LOAD]], 42
+; CHECK-NEXT: [[EXPR:%.*]] = shl i64 [[IV]], 2
+; CHECK-NEXT: [[SEL]] = select i1 [[CMP]], i64 [[EXPR]], i64 [[RDX]]
+; CHECK-NEXT: [[IV_NEXT]] = add nuw nsw i64 [[IV]], 1
+; CHECK-NEXT: [[EXIT:%.*]] = icmp eq i64 [[IV_NEXT]], [[N]]
+; CHECK-NEXT: br i1 [[EXIT]], label %[[DONE]], label %[[LOOP]], !llvm.loop [[LOOP3:![0-9]+]]
+; CHECK: [[DONE]]:
+; CHECK-NEXT: [[SEL_LCSSA:%.*]] = phi i64 [ [[SEL]], %[[LOOP]] ], [ [[TMP7]], %[[MIDDLE_BLOCK]] ]
+; CHECK-NEXT: ret i64 [[SEL_LCSSA]]
;
entry:
br label %loop
@@ -108,87 +72,52 @@ done:
}
define i64 @findlast_mul_expr(ptr %a, i64 %n) {
-; NO-TF-LABEL: define i64 @findlast_mul_expr(
-; NO-TF-SAME: ptr [[A:%.*]], i64 [[N:%.*]]) {
-; NO-TF-NEXT: [[ENTRY:.*]]:
-; NO-TF-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[N]], 4
-; NO-TF-NEXT: br i1 [[MIN_ITERS_CHECK]], label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]]
-; NO-TF: [[VECTOR_PH]]:
-; NO-TF-NEXT: [[N_MOD_VF:%.*]] = urem i64 [[N]], 4
-; NO-TF-NEXT: [[N_VEC:%.*]] = sub i64 [[N]], [[N_MOD_VF]]
-; NO-TF-NEXT: br label %[[VECTOR_BODY:.*]]
-; NO-TF: [[VECTOR_BODY]]:
-; NO-TF-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
-; NO-TF-NEXT: [[VEC_IND:%.*]] = phi <4 x i64> [ <i64 0, i64 1, i64 2, i64 3>, %[[VECTOR_PH]] ], [ [[VEC_IND_NEXT:%.*]], %[[VECTOR_BODY]] ]
-; NO-TF-NEXT: [[VEC_PHI:%.*]] = phi <4 x i64> [ splat (i64 -9223372036854775808), %[[VECTOR_PH]] ], [ [[TMP2:%.*]], %[[VECTOR_BODY]] ]
-; NO-TF-NEXT: [[TMP0:%.*]] = getelementptr inbounds i64, ptr [[A]], i64 [[INDEX]]
-; NO-TF-NEXT: [[WIDE_LOAD:%.*]] = load <4 x i64>, ptr [[TMP0]], align 8
-; NO-TF-NEXT: [[TMP1:%.*]] = icmp eq <4 x i64> [[WIDE_LOAD]], splat (i64 42)
-; NO-TF-NEXT: [[TMP2]] = select <4 x i1> [[TMP1]], <4 x i64> [[VEC_IND]], <4 x i64> [[VEC_PHI]]
-; NO-TF-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 4
-; NO-TF-NEXT: [[VEC_IND_NEXT]] = add nuw nsw <4 x i64> [[VEC_IND]], splat (i64 4)
-; NO-TF-NEXT: [[TMP3:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
-; NO-TF-NEXT: br i1 [[TMP3]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP4:![0-9]+]]
-; NO-TF: [[MIDDLE_BLOCK]]:
-; NO-TF-NEXT: [[TMP4:%.*]] = call i64 @llvm.vector.reduce.smax.v4i64(<4 x i64> [[TMP2]])
-; NO-TF-NEXT: [[TMP5:%.*]] = mul i64 [[TMP4]], 3
-; NO-TF-NEXT: [[TMP6:%.*]] = icmp ne i64 [[TMP4]], -9223372036854775808
-; NO-TF-NEXT: [[TMP7:%.*]] = select i1 [[TMP6]], i64 [[TMP5]], i64 -1
-; NO-TF-NEXT: [[CMP_N:%.*]] = icmp eq i64 [[N]], [[N_VEC]]
-; NO-TF-NEXT: br i1 [[CMP_N]], label %[[DONE:.*]], label %[[SCALAR_PH]]
-; NO-TF: [[SCALAR_PH]]:
-; NO-TF-NEXT: [[BC_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC]], %[[MIDDLE_BLOCK]] ], [ 0, %[[ENTRY]] ]
-; NO-TF-NEXT: [[BC_MERGE_RDX:%.*]] = phi i64 [ [[TMP7]], %[[MIDDLE_BLOCK]] ], [ -1, %[[ENTRY]] ]
-; NO-TF-NEXT: br label %[[LOOP:.*]]
-; NO-TF: [[LOOP]]:
-; NO-TF-NEXT: [[IV:%.*]] = phi i64 [ [[BC_RESUME_VAL]], %[[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], %[[LOOP]] ]
-; NO-TF-NEXT: [[RDX:%.*]] = phi i64 [ [[BC_MERGE_RDX]], %[[SCALAR_PH]] ], [ [[SEL:%.*]], %[[LOOP]] ]
-; NO-TF-NEXT: [[GEP:%.*]] = getelementptr inbounds i64, ptr [[A]], i64 [[IV]]
-; NO-TF-NEXT: [[LOAD:%.*]] = load i64, ptr [[GEP]], align 8
-; NO-TF-NEXT: [[CMP:%.*]] = icmp eq i64 [[LOAD]], 42
-; NO-TF-NEXT: [[EXPR:%.*]] = mul i64 [[IV]], 3
-; NO-TF-NEXT: [[SEL]] = select i1 [[CMP]], i64 [[EXPR]], i64 [[RDX]]
-; NO-TF-NEXT: [[IV_NEXT]] = add nuw nsw i64 [[IV]], 1
-; NO-TF-NEXT: [[EXIT:%.*]] = icmp eq i64 [[IV_NEXT]], [[N]]
-; NO-TF-NEXT: br i1 [[EXIT]], label %[[DONE]], label %[[LOOP]], !llvm.loop [[LOOP5:![0-9]+]]
-; NO-TF: [[DONE]]:
-; NO-TF-NEXT: [[SEL_LCSSA:%.*]] = phi i64 [ [[SEL]], %[[LOOP]] ], [ [[TMP7]], %[[MIDDLE_BLOCK]] ]
-; NO-TF-NEXT: ret i64 [[SEL_LCSSA]]
-;
-; TF-LABEL: define i64 @findlast_mul_expr(
-; TF-SAME: ptr [[A:%.*]], i64 [[N:%.*]]) {
-; TF-NEXT: [[ENTRY:.*:]]
-; TF-NEXT: br label %[[VECTOR_PH:.*]]
-; TF: [[VECTOR_PH]]:
-; TF-NEXT: [[N_RND_UP:%.*]] = add i64 [[N]], 3
-; TF-NEXT: [[N_MOD_VF:%.*]] = urem i64 [[N_RND_UP]], 4
-; TF-NEXT: [[N_VEC:%.*]] = sub i64 [[N_RND_UP]], [[N_MOD_VF]]
-; TF-NEXT: [[TRIP_COUNT_MINUS_1:%.*]] = sub i64 [[N]], 1
-; TF-NEXT: [[BROADCAST_SPLATINSERT:%.*]] = insertelement <4 x i64> poison, i64 [[TRIP_COUNT_MINUS_1]], i64 0
-; TF-NEXT: [[BROADCAST_SPLAT:%.*]] = shufflevector <4 x i64> [[BROADCAST_SPLATINSERT]], <4 x i64> poison, <4 x i32> zeroinitializer
-; TF-NEXT: br label %[[VECTOR_BODY:.*]]
-; TF: [[VECTOR_BODY]]:
-; TF-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
-; TF-NEXT: [[VEC_IND:%.*]] = phi <4 x i64> [ <i64 0, i64 1, i64 2, i64 3>, %[[VECTOR_PH]] ], [ [[VEC_IND_NEXT:%.*]], %[[VECTOR_BODY]] ]
-; TF-NEXT: [[VEC_PHI:%.*]] = phi <4 x i64> [ splat (i64 -9223372036854775808), %[[VECTOR_PH]] ], [ [[TMP3:%.*]], %[[VECTOR_BODY]] ]
-; TF-NEXT: [[TMP0:%.*]] = icmp ule <4 x i64> [[VEC_IND]], [[BROADCAST_SPLAT]]
-; TF-NEXT: [[TMP1:%.*]] = getelementptr inbounds i64, ptr [[A]], i64 [[INDEX]]
-; TF-NEXT: [[WIDE_MASKED_LOAD:%.*]] = call <4 x i64> @llvm.masked.load.v4i64.p0(ptr align 8 [[TMP1]], <4 x i1> [[TMP0]], <4 x i64> poison)
-; TF-NEXT: [[TMP2:%.*]] = icmp eq <4 x i64> [[WIDE_MASKED_LOAD]], splat (i64 42)
-; TF-NEXT: [[TMP9:%.*]] = select <4 x i1> [[TMP0]], <4 x i1> [[TMP2]], <4 x i1> zeroinitializer
-; TF-NEXT: [[TMP3]] = select <4 x i1> [[TMP9]], <4 x i64> [[VEC_IND]], <4 x i64> [[VEC_PHI]]
-; TF-NEXT: [[INDEX_NEXT]] = add i64 [[INDEX]], 4
-; TF-NEXT: [[VEC_IND_NEXT]] = add <4 x i64> [[VEC_IND]], splat (i64 4)
-; TF-NEXT: [[TMP4:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
-; TF-NEXT: br i1 [[TMP4]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP3:![0-9]+]]
-; TF: [[MIDDLE_BLOCK]]:
-; TF-NEXT: [[TMP5:%.*]] = call i64 @llvm.vector.reduce.smax.v4i64(<4 x i64> [[TMP3]])
-; TF-NEXT: [[TMP6:%.*]] = mul i64 [[TMP5]], 3
-; TF-NEXT: [[TMP7:%.*]] = icmp ne i64 [[TMP5]], -9223372036854775808
-; TF-NEXT: [[TMP8:%.*]] = select i1 [[TMP7]], i64 [[TMP6]], i64 -1
-; TF-NEXT: br label %[[DONE:.*]]
-; TF: [[DONE]]:
-; TF-NEXT: ret i64 [[TMP8]]
+; CHECK-LABEL: define i64 @findlast_mul_expr(
+; CHECK-SAME: ptr [[A:%.*]], i64 [[N:%.*]]) {
+; CHECK-NEXT: [[ENTRY:.*]]:
+; CHECK-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[N]], 4
+; CHECK-NEXT: br i1 [[MIN_ITERS_CHECK]], label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]]
+; CHECK: [[VECTOR_PH]]:
+; CHECK-NEXT: [[N_MOD_VF:%.*]] = urem i64 [[N]], 4
+; CHECK-NEXT: [[N_VEC:%.*]] = sub i64 [[N]], [[N_MOD_VF]]
+; CHECK-NEXT: br label %[[VECTOR_BODY:.*]]
+; CHECK: [[VECTOR_BODY]]:
+; CHECK-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-NEXT: [[VEC_IND:%.*]] = phi <4 x i64> [ <i64 0, i64 1, i64 2, i64 3>, %[[VECTOR_PH]] ], [ [[VEC_IND_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-NEXT: [[VEC_PHI:%.*]] = phi <4 x i64> [ splat (i64 -9223372036854775808), %[[VECTOR_PH]] ], [ [[TMP3:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-NEXT: [[TMP1:%.*]] = getelementptr inbounds i64, ptr [[A]], i64 [[INDEX]]
+; CHECK-NEXT: [[WIDE_LOAD:%.*]] = load <4 x i64>, ptr [[TMP1]], align 8
+; CHECK-NEXT: [[TMP2:%.*]] = icmp eq <4 x i64> [[WIDE_LOAD]], splat (i64 42)
+; CHECK-NEXT: [[TMP3]] = select <4 x i1> [[TMP2]], <4 x i64> [[VEC_IND]], <4 x i64> [[VEC_PHI]]
+; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 4
+; CHECK-NEXT: [[VEC_IND_NEXT]] = add nuw nsw <4 x i64> [[VEC_IND]], splat (i64 4)
+; CHECK-NEXT: [[TMP8:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
+; CHECK-NEXT: br i1 [[TMP8]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP4:![0-9]+]]
+; CHECK: [[MIDDLE_BLOCK]]:
+; CHECK-NEXT: [[TMP4:%.*]] = call i64 @llvm.vector.reduce.smax.v4i64(<4 x i64> [[TMP3]])
+; CHECK-NEXT: [[TMP5:%.*]] = mul i64 [[TMP4]], 3
+; CHECK-NEXT: [[TMP6:%.*]] = icmp ne i64 [[TMP4]], -9223372036854775808
+; CHECK-NEXT: [[TMP7:%.*]] = select i1 [[TMP6]], i64 [[TMP5]], i64 -1
+; CHECK-NEXT: [[CMP_N:%.*]] = icmp eq i64 [[N]], [[N_VEC]]
+; CHECK-NEXT: br i1 [[CMP_N]], label %[[DONE:.*]], label %[[SCALAR_PH]]
+; CHECK: [[SCALAR_PH]]:
+; CHECK-NEXT: [[BC_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC]], %[[MIDDLE_BLOCK]] ], [ 0, %[[ENTRY]] ]
+; CHECK-NEXT: [[BC_MERGE_RDX:%.*]] = phi i64 [ [[TMP7]], %[[MIDDLE_BLOCK]] ], [ -1, %[[ENTRY]] ]
+; CHECK-NEXT: br label %[[LOOP:.*]]
+; CHECK: [[LOOP]]:
+; CHECK-NEXT: [[IV:%.*]] = phi i64 [ [[BC_RESUME_VAL]], %[[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], %[[LOOP]] ]
+; CHECK-NEXT: [[RDX:%.*]] = phi i64 [ [[BC_MERGE_RDX]], %[[SCALAR_PH]] ], [ [[SEL:%.*]], %[[LOOP]] ]
+; CHECK-NEXT: [[GEP:%.*]] = getelementptr inbounds i64, ptr [[A]], i64 [[IV]]
+; CHECK-NEXT: [[LOAD:%.*]] = load i64, ptr [[GEP]], align 8
+; CHECK-NEXT: [[CMP:%.*]] = icmp eq i64 [[LOAD]], 42
+; CHECK-NEXT: [[EXPR:%.*]] = mul i64 [[IV]], 3
+; CHECK-NEXT: [[SEL]] = select i1 [[CMP]], i64 [[EXPR]], i64 [[RDX]]
+; CHECK-NEXT: [[IV_NEXT]] = add nuw nsw i64 [[IV]], 1
+; CHECK-NEXT: [[EXIT:%.*]] = icmp eq i64 [[IV_NEXT]], [[N]]
+; CHECK-NEXT: br i1 [[EXIT]], label %[[DONE]], label %[[LOOP]], !llvm.loop [[LOOP5:![0-9]+]]
+; CHECK: [[DONE]]:
+; CHECK-NEXT: [[SEL_LCSSA:%.*]] = phi i64 [ [[SEL]], %[[LOOP]] ], [ [[TMP7]], %[[MIDDLE_BLOCK]] ]
+; CHECK-NEXT: ret i64 [[SEL_LCSSA]]
;
entry:
br label %loop
@@ -212,97 +141,58 @@ done:
; Negative test: the mul operand is loop-varying (a load), so expression
; sinking should NOT apply. Uses extract-last-active instead.
define i64 @findlast_mul_loop_varying_operand(ptr %a, ptr %b, i64 %n) {
-; NO-TF-LABEL: define i64 @findlast_mul_loop_varying_operand(
-; NO-TF-SAME: ptr [[A:%.*]], ptr [[B:%.*]], i64 [[N:%.*]]) {
-; NO-TF-NEXT: [[ENTRY:.*]]:
-; NO-TF-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[N]], 4
-; NO-TF-NEXT: br i1 [[MIN_ITERS_CHECK]], label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]]
-; NO-TF: [[VECTOR_PH]]:
-; NO-TF-NEXT: [[N_MOD_VF:%.*]] = urem i64 [[N]], 4
-; NO-TF-NEXT: [[N_VEC:%.*]] = sub i64 [[N]], [[N_MOD_VF]]
-; NO-TF-NEXT: br label %[[VECTOR_BODY:.*]]
-; NO-TF: [[VECTOR_BODY]]:
-; NO-TF-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
-; NO-TF-NEXT: [[VEC_IND:%.*]] = phi <4 x i64> [ <i64 0, i64 1, i64 2, i64 3>, %[[VECTOR_PH]] ], [ [[VEC_IND_NEXT:%.*]], %[[VECTOR_BODY]] ]
-; NO-TF-NEXT: [[VEC_PHI:%.*]] = phi <4 x i64> [ splat (i64 -1), %[[VECTOR_PH]] ], [ [[TMP8:%.*]], %[[VECTOR_BODY]] ]
-; NO-TF-NEXT: [[TMP0:%.*]] = phi <4 x i1> [ zeroinitializer, %[[VECTOR_PH]] ], [ [[TMP7:%.*]], %[[VECTOR_BODY]] ]
-; NO-TF-NEXT: [[TMP1:%.*]] = getelementptr inbounds i64, ptr [[A]], i64 [[INDEX]]
-; NO-TF-NEXT: [[WIDE_LOAD:%.*]] = load <4 x i64>, ptr [[TMP1]], align 8
-; NO-TF-NEXT: [[TMP2:%.*]] = icmp eq <4 x i64> [[WIDE_LOAD]], splat (i64 42)
-; NO-TF-NEXT: [[TMP3:%.*]] = getelementptr inbounds i64, ptr [[B]], i64 [[INDEX]]
-; NO-TF-NEXT: [[WIDE_LOAD1:%.*]] = load <4 x i64>, ptr [[TMP3]], align 8
-; NO-TF-NEXT: [[TMP4:%.*]] = mul <4 x i64> [[VEC_IND]], [[WIDE_LOAD1]]
-; NO-TF-NEXT: [[TMP5:%.*]] = freeze <4 x i1> [[TMP2]]
-; NO-TF-NEXT: [[TMP6:%.*]] = call i1 @llvm.vector.reduce.or.v4i1(<4 x i1> [[TMP5]])
-; NO-TF-NEXT: [[TMP7]] = select i1 [[TMP6]], <4 x i1> [[TMP2]], <4 x i1> [[TMP0]]
-; NO-TF-NEXT: [[TMP8]] = select i1 [[TMP6]], <4 x i64> [[TMP4]], <4 x i64> [[VEC_PHI]]
-; NO-TF-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 4
-; NO-TF-NEXT: [[VEC_IND_NEXT]] = add nuw nsw <4 x i64> [[VEC_IND]], splat (i64 4)
-; NO-TF-NEXT: [[TMP9:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
-; NO-TF-NEXT: br i1 [[TMP9]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP6:![0-9]+]]
-; NO-TF: [[MIDDLE_BLOCK]]:
-; NO-TF-NEXT: [[TMP10:%.*]] = call i64 @llvm.experimental.vector.extract.last.active.v4i64(<4 x i64> [[TMP8]], <4 x i1> [[TMP7]], i64 -1)
-; NO-TF-NEXT: [[CMP_N:%.*]] = icmp eq i64 [[N]], [[N_VEC]]
-; NO-TF-NEXT: br i1 [[CMP_N]], label %[[DONE:.*]], label %[[SCALAR_PH]]
-; NO-TF: [[SCALAR_PH]]:
-; NO-TF-NEXT: [[BC_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC]], %[[MIDDLE_BLOCK]] ], [ 0, %[[ENTRY]] ]
-; NO-TF-NEXT: [[BC_MERGE_RDX:%.*]] = phi i64 [ [[TMP10]], %[[MIDDLE_BLOCK]] ], [ -1, %[[ENTRY]] ]
-; NO-TF-NEXT: br label %[[LOOP:.*]]
-; NO-TF: [[LOOP]]:
-; NO-TF-NEXT: [[IV:%.*]] = phi i64 [ [[BC_RESUME_VAL]], %[[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], %[[LOOP]] ]
-; NO-TF-NEXT: [[RDX:%.*]] = phi i64 [ [[BC_MERGE_RDX]], %[[SCALAR_PH]] ], [ [[SEL:%.*]], %[[LOOP]] ]
-; NO-TF-NEXT: [[GEP_A:%.*]] = getelementptr inbounds i64, ptr [[A]], i64 [[IV]]
-; NO-TF-NEXT: [[LOAD_A:%.*]] = load i64, ptr [[GEP_A]], align 8
-; NO-TF-NEXT: [[CMP:%.*]] = icmp eq i64 [[LOAD_A]], 42
-; NO-TF-NEXT: [[GEP_B:%.*]] = getelementptr inbounds i64, ptr [[B]], i64 [[IV]]
-; NO-TF-NEXT: [[LOAD_B:%.*]] = load i64, ptr [[GEP_B]], align 8
-; NO-TF-NEXT: [[EXPR:%.*]] = mul i64 [[IV]], [[LOAD_B]]
-; NO-TF-NEXT: [[SEL]] = select i1 [[CMP]], i64 [[EXPR]], i64 [[RDX]]
-; NO-TF-NEXT: [[IV_NEXT]] = add nuw nsw i64 [[IV]], 1
-; NO-TF-NEXT: [[EXIT:%.*]] = icmp eq i64 [[IV_NEXT]], [[N]]
-; NO-TF-NEXT: br i1 [[EXIT]], label %[[DONE]], label %[[LOOP]], !llvm.loop [[LOOP7:![0-9]+]]
-; NO-TF: [[DONE]]:
-; NO-TF-NEXT: [[SEL_LCSSA:%.*]] = phi i64 [ [[SEL]], %[[LOOP]] ], [ [[TMP10]], %[[MIDDLE_BLOCK]] ]
-; NO-TF-NEXT: ret i64 [[SEL_LCSSA]]
-;
-; TF-LABEL: define i64 @findlast_mul_loop_varying_operand(
-; TF-SAME: ptr [[A:%.*]], ptr [[B:%.*]], i64 [[N:%.*]]) {
-; TF-NEXT: [[ENTRY:.*:]]
-; TF-NEXT: br label %[[VECTOR_PH:.*]]
-; TF: [[VECTOR_PH]]:
-; TF-NEXT: [[N_RND_UP:%.*]] = add i64 [[N]], 3
-; TF-NEXT: [[N_MOD_VF:%.*]] = urem i64 [[N_RND_UP]], 4
-; TF-NEXT: [[N_VEC:%.*]] = sub i64 [[N_RND_UP]], [[N_MOD_VF]]
-; TF-NEXT: [[TRIP_COUNT_MINUS_1:%.*]] = sub i64 [[N]], 1
-; TF-NEXT: [[BROADCAST_SPLATINSERT:%.*]] = insertelement <4 x i64> poison, i64 [[TRIP_COUNT_MINUS_1]], i64 0
-; TF-NEXT: [[BROADCAST_SPLAT:%.*]] = shufflevector <4 x i64> [[BROADCAST_SPLATINSERT]], <4 x i64> poison, <4 x i32> zeroinitializer
-; TF-NEXT: br label %[[VECTOR_BODY:.*]]
-; TF: [[VECTOR_BODY]]:
-; TF-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
-; TF-NEXT: [[VEC_IND:%.*]] = phi <4 x i64> [ <i64 0, i64 1, i64 2, i64 3>, %[[VECTOR_PH]] ], [ [[VEC_IND_NEXT:%.*]], %[[VECTOR_BODY]] ]
-; TF-NEXT: [[VEC_PHI:%.*]] = phi <4 x i64> [ splat (i64 -1), %[[VECTOR_PH]] ], [ [[TMP10:%.*]], %[[VECTOR_BODY]] ]
-; TF-NEXT: [[TMP0:%.*]] = phi <4 x i1> [ zeroinitializer, %[[VECTOR_PH]] ], [ [[TMP9:%.*]], %[[VECTOR_BODY]] ]
-; TF-NEXT: [[TMP1:%.*]] = icmp ule <4 x i64> [[VEC_IND]], [[BROADCAST_SPLAT]]
-; TF-NEXT: [[TMP2:%.*]] = getelementptr inbounds i64, ptr [[A]], i64 [[INDEX]]
-; TF-NEXT: [[WIDE_MASKED_LOAD:%.*]] = call <4 x i64> @llvm.masked.load.v4i64.p0(ptr align 8 [[TMP2]], <4 x i1> [[TMP1]], <4 x i64> poison)
-; TF-NEXT: [[TMP3:%.*]] = icmp eq <4 x i64> [[WIDE_MASKED_LOAD]], splat (i64 42)
-; TF-NEXT: [[TMP4:%.*]] = getelementptr inbounds i64, ptr [[B]], i64 [[INDEX]]
-; TF-NEXT: [[WIDE_MASKED_LOAD1:%.*]] = call <4 x i64> @llvm.masked.load.v4i64.p0(ptr align 8 [[TMP4]], <4 x i1> [[TMP1]], <4 x i64> poison)
-; TF-NEXT: [[TMP5:%.*]] = mul <4 x i64> [[VEC_IND]], [[WIDE_MASKED_LOAD1]]
-; TF-NEXT: [[TMP6:%.*]] = select <4 x i1> [[TMP1]], <4 x i1> [[TMP3]], <4 x i1> zeroinitializer
-; TF-NEXT: [[TMP7:%.*]] = freeze <4 x i1> [[TMP6]]
-; TF-NEXT: [[TMP8:%.*]] = call i1 @llvm.vector.reduce.or.v4i1(<4 x i1> [[TMP7]])
-; TF-NEXT: [[TMP9]] = select i1 [[TMP8]], <4 x i1> [[TMP6]], <4 x i1> [[TMP0]]
-; TF-NEXT: [[TMP10]] = select i1 [[TMP8]], <4 x i64> [[TMP5]], <4 x i64> [[VEC_PHI]]
-; TF-NEXT: [[INDEX_NEXT]] = add i64 [[INDEX]], 4
-; TF-NEXT: [[VEC_IND_NEXT]] = add <4 x i64> [[VEC_IND]], splat (i64 4)
-; TF-NEXT: [[TMP11:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
-; TF-NEXT: br i1 [[TMP11]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP4:![0-9]+]]
-; TF: [[MIDDLE_BLOCK]]:
-; TF-NEXT: [[TMP12:%.*]] = call i64 @llvm.experimental.vector.extract.last.active.v4i64(<4 x i64> [[TMP10]], <4 x i1> [[TMP9]], i64 -1)
-; TF-NEXT: br label %[[DONE:.*]]
-; TF: [[DONE]]:
-; TF-NEXT: ret i64 [[TMP12]]
+; CHECK-LABEL: define i64 @findlast_mul_loop_varying_operand(
+; CHECK-SAME: ptr [[A:%.*]], ptr [[B:%.*]], i64 [[N:%.*]]) {
+; CHECK-NEXT: [[ENTRY:.*]]:
+; CHECK-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[N]], 4
+; CHECK-NEXT: br i1 [[MIN_ITERS_CHECK]], label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]]
+; CHECK: [[VECTOR_PH]]:
+; CHECK-NEXT: [[N_MOD_VF:%.*]] = urem i64 [[N]], 4
+; CHECK-NEXT: [[N_VEC:%.*]] = sub i64 [[N]], [[N_MOD_VF]]
+; CHECK-NEXT: br label %[[VECTOR_BODY:.*]]
+; CHECK: [[VECTOR_BODY]]:
+; CHECK-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-NEXT: [[VEC_IND:%.*]] = phi <4 x i64> [ <i64 0, i64 1, i64 2, i64 3>, %[[VECTOR_PH]] ], [ [[VEC_IND_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-NEXT: [[VEC_PHI:%.*]] = phi <4 x i64> [ splat (i64 -1), %[[VECTOR_PH]] ], [ [[TMP7:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-NEXT: [[TMP0:%.*]] = phi <4 x i1> [ zeroinitializer, %[[VECTOR_PH]] ], [ [[TMP6:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-NEXT: [[TMP1:%.*]] = getelementptr inbounds i64, ptr [[A]], i64 [[INDEX]]
+; CHECK-NEXT: [[WIDE_LOAD:%.*]] = load <4 x i64>, ptr [[TMP1]], align 8
+; CHECK-NEXT: [[TMP2:%.*]] = icmp eq <4 x i64> [[WIDE_LOAD]], splat (i64 42)
+; CHECK-NEXT: [[TMP3:%.*]] = getelementptr inbounds i64, ptr [[B]], i64 [[INDEX]]
+; CHECK-NEXT: [[WIDE_LOAD1:%.*]] = load <4 x i64>, ptr [[TMP3]], align 8
+; CHECK-NEXT: [[EXPR:%.*]] = mul <4 x i64> [[VEC_IND]], [[WIDE_LOAD1]]
+; CHECK-NEXT: [[TMP4:%.*]] = freeze <4 x i1> [[TMP2]]
+; CHECK-NEXT: [[TMP5:%.*]] = call i1 @llvm.vector.reduce.or.v4i1(<4 x i1> [[TMP4]])
+; CHECK-NEXT: [[TMP6]] = select i1 [[TMP5]], <4 x i1> [[TMP2]], <4 x i1> [[TMP0]]
+; CHECK-NEXT: [[TMP7]] = select i1 [[TMP5]], <4 x i64> [[EXPR]], <4 x i64> [[VEC_PHI]]
+; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 4
+; CHECK-NEXT: [[VEC_IND_NEXT]] = add nuw nsw <4 x i64> [[VEC_IND]], splat (i64 4)
+; CHECK-NEXT: [[TMP8:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
+; CHECK-NEXT: br i1 [[TMP8]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP6:![0-9]+]]
+; CHECK: [[MIDDLE_BLOCK]]:
+; CHECK-NEXT: [[TMP9:%.*]] = call i64 @llvm.experimental.vector.extract.last.active.v4i64(<4 x i64> [[TMP7]], <4 x i1> [[TMP6]], i64 -1)
+; CHECK-NEXT: [[CMP_N:%.*]] = icmp eq i64 [[N]], [[N_VEC]]
+; CHECK-NEXT: br i1 [[CMP_N]], label %[[DONE:.*]], label %[[SCALAR_PH]]
+; CHECK: [[SCALAR_PH]]:
+; CHECK-NEXT: [[BC_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC]], %[[MIDDLE_BLOCK]] ], [ 0, %[[ENTRY]] ]
+; CHECK-NEXT: [[BC_MERGE_RDX:%.*]] = phi i64 [ [[TMP9]], %[[MIDDLE_BLOCK]] ], [ -1, %[[ENTRY]] ]
+; CHECK-NEXT: br label %[[LOOP:.*]]
+; CHECK: [[LOOP]]:
+; CHECK-NEXT: [[IV:%.*]] = phi i64 [ [[BC_RESUME_VAL]], %[[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], %[[LOOP]] ]
+; CHECK-NEXT: [[RDX:%.*]] = phi i64 [ [[BC_MERGE_RDX]], %[[SCALAR_PH]] ], [ [[SEL:%.*]], %[[LOOP]] ]
+; CHECK-NEXT: [[GEP_A:%.*]] = getelementptr inbounds i64, ptr [[A]], i64 [[IV]]
+; CHECK-NEXT: [[LOAD_A:%.*]] = load i64, ptr [[GEP_A]], align 8
+; CHECK-NEXT: [[CMP:%.*]] = icmp eq i64 [[LOAD_A]], 42
+; CHECK-NEXT: [[GEP_B:%.*]] = getelementptr inbounds i64, ptr [[B]], i64 [[IV]]
+; CHECK-NEXT: [[LOAD_B:%.*]] = load i64, ptr [[GEP_B]], align 8
+; CHECK-NEXT: [[EXPR1:%.*]] = mul i64 [[IV]], [[LOAD_B]]
+; CHECK-NEXT: [[SEL]] = select i1 [[CMP]], i64 [[EXPR1]], i64 [[RDX]]
+; CHECK-NEXT: [[IV_NEXT]] = add nuw nsw i64 [[IV]], 1
+; CHECK-NEXT: [[EXIT:%.*]] = icmp eq i64 [[IV_NEXT]], [[N]]
+; CHECK-NEXT: br i1 [[EXIT]], label %[[DONE]], label %[[LOOP]], !llvm.loop [[LOOP7:![0-9]+]]
+; CHECK: [[DONE]]:
+; CHECK-NEXT: [[SEL_LCSSA:%.*]] = phi i64 [ [[SEL]], %[[LOOP]] ], [ [[TMP9]], %[[MIDDLE_BLOCK]] ]
+; CHECK-NEXT: ret i64 [[SEL_LCSSA]]
;
entry:
br label %loop
@@ -327,87 +217,52 @@ done:
}
define i64 @findlast_or_expr(ptr %a, i64 %n) {
-; NO-TF-LABEL: define i64 @findlast_or_expr(
-; NO-TF-SAME: ptr [[A:%.*]], i64 [[N:%.*]]) {
-; NO-TF-NEXT: [[ENTRY:.*]]:
-; NO-TF-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[N]], 4
-; NO-TF-NEXT: br i1 [[MIN_ITERS_CHECK]], label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]]
-; NO-TF: [[VECTOR_PH]]:
-; NO-TF-NEXT: [[N_MOD_VF:%.*]] = urem i64 [[N]], 4
-; NO-TF-NEXT: [[N_VEC:%.*]] = sub i64 [[N]], [[N_MOD_VF]]
-; NO-TF-NEXT: br label %[[VECTOR_BODY:.*]]
-; NO-TF: [[VECTOR_BODY]]:
-; NO-TF-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
-; NO-TF-NEXT: [[VEC_IND:%.*]] = phi <4 x i64> [ <i64 0, i64 1, i64 2, i64 3>, %[[VECTOR_PH]] ], [ [[VEC_IND_NEXT:%.*]], %[[VECTOR_BODY]] ]
-; NO-TF-NEXT: [[VEC_PHI:%.*]] = phi <4 x i64> [ splat (i64 -9223372036854775808), %[[VECTOR_PH]] ], [ [[TMP2:%.*]], %[[VECTOR_BODY]] ]
-; NO-TF-NEXT: [[TMP0:%.*]] = getelementptr inbounds i64, ptr [[A]], i64 [[INDEX]]
-; NO-TF-NEXT: [[WIDE_LOAD:%.*]] = load <4 x i64>, ptr [[TMP0]], align 8
-; NO-TF-NEXT: [[TMP1:%.*]] = icmp eq <4 x i64> [[WIDE_LOAD]], splat (i64 42)
-; NO-TF-NEXT: [[TMP2]] = select <4 x i1> [[TMP1]], <4 x i64> [[VEC_IND]], <4 x i64> [[VEC_PHI]]
-; NO-TF-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 4
-; NO-TF-NEXT: [[VEC_IND_NEXT]] = add nuw nsw <4 x i64> [[VEC_IND]], splat (i64 4)
-; NO-TF-NEXT: [[TMP3:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
-; NO-TF-NEXT: br i1 [[TMP3]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP8:![0-9]+]]
-; NO-TF: [[MIDDLE_BLOCK]]:
-; NO-TF-NEXT: [[TMP4:%.*]] = call i64 @llvm.vector.reduce.smax.v4i64(<4 x i64> [[TMP2]])
-; NO-TF-NEXT: [[TMP5:%.*]] = or i64 [[TMP4]], 1
-; NO-TF-NEXT: [[TMP6:%.*]] = icmp ne i64 [[TMP4]], -9223372036854775808
-; NO-TF-NEXT: [[TMP7:%.*]] = select i1 [[TMP6]], i64 [[TMP5]], i64 -1
-; NO-TF-NEXT: [[CMP_N:%.*]] = icmp eq i64 [[N]], [[N_VEC]]
-; NO-TF-NEXT: br i1 [[CMP_N]], label %[[DONE:.*]], label %[[SCALAR_PH]]
-; NO-TF: [[SCALAR_PH]]:
-; NO-TF-NEXT: [[BC_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC]], %[[MIDDLE_BLOCK]] ], [ 0, %[[ENTRY]] ]
-; NO-TF-NEXT: [[BC_MERGE_RDX:%.*]] = phi i64 [ [[TMP7]], %[[MIDDLE_BLOCK]] ], [ -1, %[[ENTRY]] ]
-; NO-TF-NEXT: br label %[[LOOP:.*]]
-; NO-TF: [[LOOP]]:
-; NO-TF-NEXT: [[IV:%.*]] = phi i64 [ [[BC_RESUME_VAL]], %[[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], %[[LOOP]] ]
-; NO-TF-NEXT: [[RDX:%.*]] = phi i64 [ [[BC_MERGE_RDX]], %[[SCALAR_PH]] ], [ [[SEL:%.*]], %[[LOOP]] ]
-; NO-TF-NEXT: [[GEP:%.*]] = getelementptr inbounds i64, ptr [[A]], i64 [[IV]]
-; NO-TF-NEXT: [[LOAD:%.*]] = load i64, ptr [[GEP]], align 8
-; NO-TF-NEXT: [[CMP:%.*]] = icmp eq i64 [[LOAD]], 42
-; NO-TF-NEXT: [[EXPR:%.*]] = or i64 [[IV]], 1
-; NO-TF-NEXT: [[SEL]] = select i1 [[CMP]], i64 [[EXPR]], i64 [[RDX]]
-; NO-TF-NEXT: [[IV_NEXT]] = add nuw nsw i64 [[IV]], 1
-; NO-TF-NEXT: [[EXIT:%.*]] = icmp eq i64 [[IV_NEXT]], [[N]]
-; NO-TF-NEXT: br i1 [[EXIT]], label %[[DONE]], label %[[LOOP]], !llvm.loop [[LOOP9:![0-9]+]]
-; NO-TF: [[DONE]]:
-; NO-TF-NEXT: [[SEL_LCSSA:%.*]] = phi i64 [ [[SEL]], %[[LOOP]] ], [ [[TMP7]], %[[MIDDLE_BLOCK]] ]
-; NO-TF-NEXT: ret i64 [[SEL_LCSSA]]
-;
-; TF-LABEL: define i64 @findlast_or_expr(
-; TF-SAME: ptr [[A:%.*]], i64 [[N:%.*]]) {
-; TF-NEXT: [[ENTRY:.*:]]
-; TF-NEXT: br label %[[VECTOR_PH:.*]]
-; TF: [[VECTOR_PH]]:
-; TF-NEXT: [[N_RND_UP:%.*]] = add i64 [[N]], 3
-; TF-NEXT: [[N_MOD_VF:%.*]] = urem i64 [[N_RND_UP]], 4
-; TF-NEXT: [[N_VEC:%.*]] = sub i64 [[N_RND_UP]], [[N_MOD_VF]]
-; TF-NEXT: [[TRIP_COUNT_MINUS_1:%.*]] = sub i64 [[N]], 1
-; TF-NEXT: [[BROADCAST_SPLATINSERT:%.*]] = insertelement <4 x i64> poison, i64 [[TRIP_COUNT_MINUS_1]], i64 0
-; TF-NEXT: [[BROADCAST_SPLAT:%.*]] = shufflevector <4 x i64> [[BROADCAST_SPLATINSERT]], <4 x i64> poison, <4 x i32> zeroinitializer
-; TF-NEXT: br label %[[VECTOR_BODY:.*]]
-; TF: [[VECTOR_BODY]]:
-; TF-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
-; TF-NEXT: [[VEC_IND:%.*]] = phi <4 x i64> [ <i64 0, i64 1, i64 2, i64 3>, %[[VECTOR_PH]] ], [ [[VEC_IND_NEXT:%.*]], %[[VECTOR_BODY]] ]
-; TF-NEXT: [[VEC_PHI:%.*]] = phi <4 x i64> [ splat (i64 -9223372036854775808), %[[VECTOR_PH]] ], [ [[TMP3:%.*]], %[[VECTOR_BODY]] ]
-; TF-NEXT: [[TMP0:%.*]] = icmp ule <4 x i64> [[VEC_IND]], [[BROADCAST_SPLAT]]
-; TF-NEXT: [[TMP1:%.*]] = getelementptr inbounds i64, ptr [[A]], i64 [[INDEX]]
-; TF-NEXT: [[WIDE_MASKED_LOAD:%.*]] = call <4 x i64> @llvm.masked.load.v4i64.p0(ptr align 8 [[TMP1]], <4 x i1> [[TMP0]], <4 x i64> poison)
-; TF-NEXT: [[TMP2:%.*]] = icmp eq <4 x i64> [[WIDE_MASKED_LOAD]], splat (i64 42)
-; TF-NEXT: [[TMP9:%.*]] = select <4 x i1> [[TMP0]], <4 x i1> [[TMP2]], <4 x i1> zeroinitializer
-; TF-NEXT: [[TMP3]] = select <4 x i1> [[TMP9]], <4 x i64> [[VEC_IND]], <4 x i64> [[VEC_PHI]]
-; TF-NEXT: [[INDEX_NEXT]] = add i64 [[INDEX]], 4
-; TF-NEXT: [[VEC_IND_NEXT]] = add <4 x i64> [[VEC_IND]], splat (i64 4)
-; TF-NEXT: [[TMP4:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
-; TF-NEXT: br i1 [[TMP4]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP5:![0-9]+]]
-; TF: [[MIDDLE_BLOCK]]:
-; TF-NEXT: [[TMP5:%.*]] = call i64 @llvm.vector.reduce.smax.v4i64(<4 x i64> [[TMP3]])
-; TF-NEXT: [[TMP6:%.*]] = or i64 [[TMP5]], 1
-; TF-NEXT: [[TMP7:%.*]] = icmp ne i64 [[TMP5]], -9223372036854775808
-; TF-NEXT: [[TMP8:%.*]] = select i1 [[TMP7]], i64 [[TMP6]], i64 -1
-; TF-NEXT: br label %[[DONE:.*]]
-; TF: [[DONE]]:
-; TF-NEXT: ret i64 [[TMP8]]
+; CHECK-LABEL: define i64 @findlast_or_expr(
+; CHECK-SAME: ptr [[A:%.*]], i64 [[N:%.*]]) {
+; CHECK-NEXT: [[ENTRY:.*]]:
+; CHECK-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[N]], 4
+; CHECK-NEXT: br i1 [[MIN_ITERS_CHECK]], label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]]
+; CHECK: [[VECTOR_PH]]:
+; CHECK-NEXT: [[N_MOD_VF:%.*]] = urem i64 [[N]], 4
+; CHECK-NEXT: [[N_VEC:%.*]] = sub i64 [[N]], [[N_MOD_VF]]
+; CHECK-NEXT: br label %[[VECTOR_BODY:.*]]
+; CHECK: [[VECTOR_BODY]]:
+; CHECK-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-NEXT: [[VEC_IND:%.*]] = phi <4 x i64> [ <i64 0, i64 1, i64 2, i64 3>, %[[VECTOR_PH]] ], [ [[VEC_IND_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-NEXT: [[VEC_PHI:%.*]] = phi <4 x i64> [ splat (i64 -9223372036854775808), %[[VECTOR_PH]] ], [ [[TMP3:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-NEXT: [[TMP1:%.*]] = getelementptr inbounds i64, ptr [[A]], i64 [[INDEX]]
+; CHECK-NEXT: [[WIDE_LOAD:%.*]] = load <4 x i64>, ptr [[TMP1]], align 8
+; CHECK-NEXT: [[TMP2:%.*]] = icmp eq <4 x i64> [[WIDE_LOAD]], splat (i64 42)
+; CHECK-NEXT: [[TMP3]] = select <4 x i1> [[TMP2]], <4 x i64> [[VEC_IND]], <4 x i64> [[VEC_PHI]]
+; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 4
+; CHECK-NEXT: [[VEC_IND_NEXT]] = add nuw nsw <4 x i64> [[VEC_IND]], splat (i64 4)
+; CHECK-NEXT: [[TMP8:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
+; CHECK-NEXT: br i1 [[TMP8]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP8:![0-9]+]]
+; CHECK: [[MIDDLE_BLOCK]]:
+; CHECK-NEXT: [[TMP4:%.*]] = call i64 @llvm.vector.reduce.smax.v4i64(<4 x i64> [[TMP3]])
+; CHECK-NEXT: [[TMP5:%.*]] = or i64 [[TMP4]], 1
+; CHECK-NEXT: [[TMP6:%.*]] = icmp ne i64 [[TMP4]], -9223372036854775808
+; CHECK-NEXT: [[TMP7:%.*]] = select i1 [[TMP6]], i64 [[TMP5]], i64 -1
+; CHECK-NEXT: [[CMP_N:%.*]] = icmp eq i64 [[N]], [[N_VEC]]
+; CHECK-NEXT: br i1 [[CMP_N]], label %[[DONE:.*]], label %[[SCALAR_PH]]
+; CHECK: [[SCALAR_PH]]:
+; CHECK-NEXT: [[BC_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC]], %[[MIDDLE_BLOCK]] ], [ 0, %[[ENTRY]] ]
+; CHECK-NEXT: [[BC_MERGE_RDX:%.*]] = phi i64 [ [[TMP7]], %[[MIDDLE_BLOCK]] ], [ -1, %[[ENTRY]] ]
+; CHECK-NEXT: br label %[[LOOP:.*]]
+; CHECK: [[LOOP]]:
+; CHECK-NEXT: [[IV:%.*]] = phi i64 [ [[BC_RESUME_VAL]], %[[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], %[[LOOP]] ]
+; CHECK-NEXT: [[RDX:%.*]] = phi i64 [ [[BC_MERGE_RDX]], %[[SCALAR_PH]] ], [ [[SEL:%.*]], %[[LOOP]] ]
+; CHECK-NEXT: [[GEP:%.*]] = getelementptr inbounds i64, ptr [[A]], i64 [[IV]]
+; CHECK-NEXT: [[LOAD:%.*]] = load i64, ptr [[GEP]], align 8
+; CHECK-NEXT: [[CMP:%.*]] = icmp eq i64 [[LOAD]], 42
+; CHECK-NEXT: [[EXPR:%.*]] = or i64 [[IV]], 1
+; CHECK-NEXT: [[SEL]] = select i1 [[CMP]], i64 [[EXPR]], i64 [[RDX]]
+; CHECK-NEXT: [[IV_NEXT]] = add nuw nsw i64 [[IV]], 1
+; CHECK-NEXT: [[EXIT:%.*]] = icmp eq i64 [[IV_NEXT]], [[N]]
+; CHECK-NEXT: br i1 [[EXIT]], label %[[DONE]], label %[[LOOP]], !llvm.loop [[LOOP9:![0-9]+]]
+; CHECK: [[DONE]]:
+; CHECK-NEXT: [[SEL_LCSSA:%.*]] = phi i64 [ [[SEL]], %[[LOOP]] ], [ [[TMP7]], %[[MIDDLE_BLOCK]] ]
+; CHECK-NEXT: ret i64 [[SEL_LCSSA]]
;
entry:
br label %loop
@@ -431,153 +286,72 @@ done:
; Test non-canonical IV (starting at 10, step 2) with expression sinking.
; The IV is non-canonical but expression sinking should still apply.
define i64 @findlast_non_canonical_iv_with_expr(ptr %a, i64 %n) {
-; NO-TF-LABEL: define i64 @findlast_non_canonical_iv_with_expr(
-; NO-TF-SAME: ptr [[A:%.*]], i64 [[N:%.*]]) {
-; NO-TF-NEXT: [[ENTRY:.*]]:
-; NO-TF-NEXT: [[TMP0:%.*]] = add i64 [[N]], -12
-; NO-TF-NEXT: [[TMP1:%.*]] = lshr i64 [[TMP0]], 1
-; NO-TF-NEXT: [[TMP2:%.*]] = add nuw i64 [[TMP1]], 1
-; NO-TF-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[TMP2]], 4
-; NO-TF-NEXT: br i1 [[MIN_ITERS_CHECK]], label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]]
-; NO-TF: [[VECTOR_PH]]:
-; NO-TF-NEXT: [[N_MOD_VF:%.*]] = urem i64 [[TMP2]], 4
-; NO-TF-NEXT: [[N_VEC:%.*]] = sub i64 [[TMP2]], [[N_MOD_VF]]
-; NO-TF-NEXT: [[TMP3:%.*]] = mul i64 [[N_VEC]], 2
-; NO-TF-NEXT: [[TMP4:%.*]] = add i64 10, [[TMP3]]
-; NO-TF-NEXT: br label %[[VECTOR_BODY:.*]]
-; NO-TF: [[VECTOR_BODY]]:
-; NO-TF-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
-; NO-TF-NEXT: [[VEC_IND:%.*]] = phi <4 x i64> [ <i64 10, i64 12, i64 14, i64 16>, %[[VECTOR_PH]] ], [ [[VEC_IND_NEXT:%.*]], %[[VECTOR_BODY]] ]
-; NO-TF-NEXT: [[VEC_PHI:%.*]] = phi <4 x i64> [ zeroinitializer, %[[VECTOR_PH]] ], [ [[TMP23:%.*]], %[[VECTOR_BODY]] ]
-; NO-TF-NEXT: [[TMP5:%.*]] = mul i64 [[INDEX]], 2
-; NO-TF-NEXT: [[OFFSET_IDX:%.*]] = add i64 10, [[TMP5]]
-; NO-TF-NEXT: [[TMP6:%.*]] = add i64 [[OFFSET_IDX]], 2
-; NO-TF-NEXT: [[TMP7:%.*]] = add i64 [[OFFSET_IDX]], 4
-; NO-TF-NEXT: [[TMP8:%.*]] = add i64 [[OFFSET_IDX]], 6
-; NO-TF-NEXT: [[TMP9:%.*]] = getelementptr inbounds i64, ptr [[A]], i64 [[OFFSET_IDX]]
-; NO-TF-NEXT: [[TMP10:%.*]] = getelementptr inbounds i64, ptr [[A]], i64 [[TMP6]]
-; NO-TF-NEXT: [[TMP11:%.*]] = getelementptr inbounds i64, ptr [[A]], i64 [[TMP7]]
-; NO-TF-NEXT: [[TMP12:%.*]] = getelementptr inbounds i64, ptr [[A]], i64 [[TMP8]]
-; NO-TF-NEXT: [[TMP13:%.*]] = load i64, ptr [[TMP9]], align 8
-; NO-TF-NEXT: [[TMP14:%.*]] = load i64, ptr [[TMP10]], align 8
-; NO-TF-NEXT: [[TMP15:%.*]] = load i64, ptr [[TMP11]], align 8
-; NO-TF-NEXT: [[TMP16:%.*]] = load i64, ptr [[TMP12]], align 8
-; NO-TF-NEXT: [[TMP17:%.*]] = insertelement <4 x i64> poison, i64 [[TMP13]], i32 0
-; NO-TF-NEXT: [[TMP18:%.*]] = insertelement <4 x i64> [[TMP17]], i64 [[TMP14]], i32 1
-; NO-TF-NEXT: [[TMP19:%.*]] = insertelement <4 x i64> [[TMP18]], i64 [[TMP15]], i32 2
-; NO-TF-NEXT: [[TMP20:%.*]] = insertelement <4 x i64> [[TMP19]], i64 [[TMP16]], i32 3
-; NO-TF-NEXT: [[TMP21:%.*]] = icmp eq <4 x i64> [[TMP20]], splat (i64 42)
-; NO-TF-NEXT: [[TMP22:%.*]] = add <4 x i64> [[VEC_IND]], splat (i64 100)
-; NO-TF-NEXT: [[TMP23]] = select <4 x i1> [[TMP21]], <4 x i64> [[TMP22]], <4 x i64> [[VEC_PHI]]
-; NO-TF-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 4
-; NO-TF-NEXT: [[VEC_IND_NEXT]] = add nuw nsw <4 x i64> [[VEC_IND]], splat (i64 8)
-; NO-TF-NEXT: [[TMP24:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
-; NO-TF-NEXT: br i1 [[TMP24]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP10:![0-9]+]]
-; NO-TF: [[MIDDLE_BLOCK]]:
-; NO-TF-NEXT: [[TMP25:%.*]] = call i64 @llvm.vector.reduce.umax.v4i64(<4 x i64> [[TMP23]])
-; NO-TF-NEXT: [[TMP26:%.*]] = icmp ne i64 [[TMP25]], 0
-; NO-TF-NEXT: [[TMP27:%.*]] = select i1 [[TMP26]], i64 [[TMP25]], i64 -1
-; NO-TF-NEXT: [[CMP_N:%.*]] = icmp eq i64 [[TMP2]], [[N_VEC]]
-; NO-TF-NEXT: br i1 [[CMP_N]], label %[[DONE:.*]], label %[[SCALAR_PH]]
-; NO-TF: [[SCALAR_PH]]:
-; NO-TF-NEXT: [[BC_RESUME_VAL:%.*]] = phi i64 [ [[TMP4]], %[[MIDDLE_BLOCK]] ], [ 10, %[[ENTRY]] ]
-; NO-TF-NEXT: [[BC_MERGE_RDX:%.*]] = phi i64 [ [[TMP27]], %[[MIDDLE_BLOCK]] ], [ -1, %[[ENTRY]] ]
-; NO-TF-NEXT: br label %[[LOOP:.*]]
-; NO-TF: [[LOOP]]:
-; NO-TF-NEXT: [[IV:%.*]] = phi i64 [ [[BC_RESUME_VAL]], %[[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], %[[LOOP]] ]
-; NO-TF-NEXT: [[RDX:%.*]] = phi i64 [ [[BC_MERGE_RDX]], %[[SCALAR_PH]] ], [ [[SEL:%.*]], %[[LOOP]] ]
-; NO-TF-NEXT: [[GEP:%.*]] = getelementptr inbounds i64, ptr [[A]], i64 [[IV]]
-; NO-TF-NEXT: [[LOAD:%.*]] = load i64, ptr [[GEP]], align 8
-; NO-TF-NEXT: [[CMP:%.*]] = icmp eq i64 [[LOAD]], 42
-; NO-TF-NEXT: [[EXPR:%.*]] = add i64 [[IV]], 100
-; NO-TF-NEXT: [[SEL]] = select i1 [[CMP]], i64 [[EXPR]], i64 [[RDX]]
-; NO-TF-NEXT: [[IV_NEXT]] = add nuw nsw i64 [[IV]], 2
-; NO-TF-NEXT: [[EXIT:%.*]] = icmp eq i64 [[IV_NEXT]], [[N]]
-; NO-TF-NEXT: br i1 [[EXIT]], label %[[DONE]], label %[[LOOP]], !llvm.loop [[LOOP11:![0-9]+]]
-; NO-TF: [[DONE]]:
-; NO-TF-NEXT: [[SEL_LCSSA:%.*]] = phi i64 [ [[SEL]], %[[LOOP]] ], [ [[TMP27]], %[[MIDDLE_BLOCK]] ]
-; NO-TF-NEXT: ret i64 [[SEL_LCSSA]]
-;
-; TF-LABEL: define i64 @findlast_non_canonical_iv_with_expr(
-; TF-SAME: ptr [[A:%.*]], i64 [[N:%.*]]) {
-; TF-NEXT: [[ENTRY:.*:]]
-; TF-NEXT: [[TMP0:%.*]] = add i64 [[N]], -12
-; TF-NEXT: [[TMP1:%.*]] = lshr i64 [[TMP0]], 1
-; TF-NEXT: [[TMP2:%.*]] = add nuw i64 [[TMP1]], 1
-; TF-NEXT: br label %[[VECTOR_PH:.*]]
-; TF: [[VECTOR_PH]]:
-; TF-NEXT: [[N_RND_UP:%.*]] = add i64 [[TMP2]], 3
-; TF-NEXT: [[N_MOD_VF:%.*]] = urem i64 [[N_RND_UP]], 4
-; TF-NEXT: [[N_VEC:%.*]] = sub i64 [[N_RND_UP]], [[N_MOD_VF]]
-; TF-NEXT: [[TRIP_COUNT_MINUS_1:%.*]] = sub i64 [[TMP2]], 1
-; TF-NEXT: [[BROADCAST_SPLATINSERT:%.*]] = insertelement <4 x i64> poison, i64 [[TRIP_COUNT_MINUS_1]], i64 0
-; TF-NEXT: [[BROADCAST_SPLAT:%.*]] = shufflevector <4 x i64> [[BROADCAST_SPLATINSERT]], <4 x i64> poison, <4 x i32> zeroinitializer
-; TF-NEXT: br label %[[VECTOR_BODY:.*]]
-; TF: [[VECTOR_BODY]]:
-; TF-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[PRED_LOAD_CONTINUE8:.*]] ]
-; TF-NEXT: [[VEC_IND:%.*]] = phi <4 x i64> [ <i64 10, i64 12, i64 14, i64 16>, %[[VECTOR_PH]] ], [ [[VEC_IND_NEXT:%.*]], %[[PRED_LOAD_CONTINUE8]] ]
-; TF-NEXT: [[VEC_PHI:%.*]] = phi <4 x i64> [ zeroinitializer, %[[VECTOR_PH]] ], [ [[TMP36:%.*]], %[[PRED_LOAD_CONTINUE8]] ]
-; TF-NEXT: [[TMP3:%.*]] = mul i64 [[INDEX]], 2
-; TF-NEXT: [[OFFSET_IDX:%.*]] = add i64 10, [[TMP3]]
-; TF-NEXT: [[BROADCAST_SPLATINSERT1:%.*]] = insertelement <4 x i64> poison, i64 [[INDEX]], i64 0
-; TF-NEXT: [[BROADCAST_SPLAT2:%.*]] = shufflevector <4 x i64> [[BROADCAST_SPLATINSERT1]], <4 x i64> poison, <4 x i32> zeroinitializer
-; TF-NEXT: [[VEC_IV:%.*]] = add <4 x i64> [[BROADCAST_SPLAT2]], <i64 0, i64 1, i64 2, i64 3>
-; TF-NEXT: [[TMP4:%.*]] = icmp ule <4 x i64> [[VEC_IV]], [[BROADCAST_SPLAT]]
-; TF-NEXT: [[TMP5:%.*]] = extractelement <4 x i1> [[TMP4]], i32 0
-; TF-NEXT: br i1 [[TMP5]], label %[[PRED_LOAD_IF:.*]], label %[[PRED_LOAD_CONTINUE:.*]]
-; TF: [[PRED_LOAD_IF]]:
-; TF-NEXT: [[TMP6:%.*]] = add i64 [[OFFSET_IDX]], 0
-; TF-NEXT: [[TMP7:%.*]] = getelementptr inbounds i64, ptr [[A]], i64 [[TMP6]]
-; TF-NEXT: [[TMP8:%.*]] = load i64, ptr [[TMP7]], align 8
-; TF-NEXT: [[TMP9:%.*]] = insertelement <4 x i64> poison, i64 [[TMP8]], i32 0
-; TF-NEXT: br label %[[PRED_LOAD_CONTINUE]]
-; TF: [[PRED_LOAD_CONTINUE]]:
-; TF-NEXT: [[TMP10:%.*]] = phi <4 x i64> [ poison, %[[VECTOR_BODY]] ], [ [[TMP9]], %[[PRED_LOAD_IF]] ]
-; TF-NEXT: [[TMP11:%.*]] = extractelement <4 x i1> [[TMP4]], i32 1
-; TF-NEXT: br i1 [[TMP11]], label %[[PRED_LOAD_IF3:.*]], label %[[PRED_LOAD_CONTINUE4:.*]]
-; TF: [[PRED_LOAD_IF3]]:
-; TF-NEXT: [[TMP12:%.*]] = add i64 [[OFFSET_IDX]], 2
-; TF-NEXT: [[TMP13:%.*]] = getelementptr inbounds i64, ptr [[A]], i64 [[TMP12]]
-; TF-NEXT: [[TMP14:%.*]] = load i64, ptr [[TMP13]], align 8
-; TF-NEXT: [[TMP15:%.*]] = insertelement <4 x i64> [[TMP10]], i64 [[TMP14]], i32 1
-; TF-NEXT: br label %[[PRED_LOAD_CONTINUE4]]
-; TF: [[PRED_LOAD_CONTINUE4]]:
-; TF-NEXT: [[TMP16:%.*]] = phi <4 x i64> [ [[TMP10]], %[[PRED_LOAD_CONTINUE]] ], [ [[TMP15]], %[[PRED_LOAD_IF3]] ]
-; TF-NEXT: [[TMP17:%.*]] = extractelement <4 x i1> [[TMP4]], i32 2
-; TF-NEXT: br i1 [[TMP17]], label %[[PRED_LOAD_IF5:.*]], label %[[PRED_LOAD_CONTINUE6:.*]]
-; TF: [[PRED_LOAD_IF5]]:
-; TF-NEXT: [[TMP18:%.*]] = add i64 [[OFFSET_IDX]], 4
-; TF-NEXT: [[TMP19:%.*]] = getelementptr inbounds i64, ptr [[A]], i64 [[TMP18]]
-; TF-NEXT: [[TMP20:%.*]] = load i64, ptr [[TMP19]], align 8
-; TF-NEXT: [[TMP21:%.*]] = insertelement <4 x i64> [[TMP16]], i64 [[TMP20]], i32 2
-; TF-NEXT: br label %[[PRED_LOAD_CONTINUE6]]
-; TF: [[PRED_LOAD_CONTINUE6]]:
-; TF-NEXT: [[TMP22:%.*]] = phi <4 x i64> [ [[TMP16]], %[[PRED_LOAD_CONTINUE4]] ], [ [[TMP21]], %[[PRED_LOAD_IF5]] ]
-; TF-NEXT: [[TMP23:%.*]] = extractelement <4 x i1> [[TMP4]], i32 3
-; TF-NEXT: br i1 [[TMP23]], label %[[PRED_LOAD_IF7:.*]], label %[[PRED_LOAD_CONTINUE8]]
-; TF: [[PRED_LOAD_IF7]]:
-; TF-NEXT: [[TMP24:%.*]] = add i64 [[OFFSET_IDX]], 6
-; TF-NEXT: [[TMP25:%.*]] = getelementptr inbounds i64, ptr [[A]], i64 [[TMP24]]
-; TF-NEXT: [[TMP26:%.*]] = load i64, ptr [[TMP25]], align 8
-; TF-NEXT: [[TMP27:%.*]] = insertelement <4 x i64> [[TMP22]], i64 [[TMP26]], i32 3
-; TF-NEXT: br label %[[PRED_LOAD_CONTINUE8]]
-; TF: [[PRED_LOAD_CONTINUE8]]:
-; TF-NEXT: [[TMP28:%.*]] = phi <4 x i64> [ [[TMP22]], %[[PRED_LOAD_CONTINUE6]] ], [ [[TMP27]], %[[PRED_LOAD_IF7]] ]
-; TF-NEXT: [[TMP29:%.*]] = icmp eq <4 x i64> [[TMP28]], splat (i64 42)
-; TF-NEXT: [[TMP30:%.*]] = add <4 x i64> [[VEC_IND]], splat (i64 100)
-; TF-NEXT: [[TMP31:%.*]] = select <4 x i1> [[TMP29]], <4 x i64> [[TMP30]], <4 x i64> [[VEC_PHI]]
-; TF-NEXT: [[TMP36]] = select <4 x i1> [[TMP4]], <4 x i64> [[TMP31]], <4 x i64> [[VEC_PHI]]
-; TF-NEXT: [[INDEX_NEXT]] = add i64 [[INDEX]], 4
-; TF-NEXT: [[VEC_IND_NEXT]] = add nuw nsw <4 x i64> [[VEC_IND]], splat (i64 8)
-; TF-NEXT: [[TMP32:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
-; TF-NEXT: br i1 [[TMP32]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP6:![0-9]+]]
-; TF: [[MIDDLE_BLOCK]]:
-; TF-NEXT: [[TMP33:%.*]] = call i64 @llvm.vector.reduce.umax.v4i64(<4 x i64> [[TMP36]])
-; TF-NEXT: [[TMP34:%.*]] = icmp ne i64 [[TMP33]], 0
-; TF-NEXT: [[TMP35:%.*]] = select i1 [[TMP34]], i64 [[TMP33]], i64 -1
-; TF-NEXT: br label %[[DONE:.*]]
-; TF: [[DONE]]:
-; TF-NEXT: ret i64 [[TMP35]]
+; CHECK-LABEL: define i64 @findlast_non_canonical_iv_with_expr(
+; CHECK-SAME: ptr [[A:%.*]], i64 [[N:%.*]]) {
+; CHECK-NEXT: [[ENTRY:.*]]:
+; CHECK-NEXT: [[TMP0:%.*]] = add i64 [[N]], -12
+; CHECK-NEXT: [[TMP1:%.*]] = lshr i64 [[TMP0]], 1
+; CHECK-NEXT: [[TMP2:%.*]] = add nuw i64 [[TMP1]], 1
+; CHECK-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[TMP2]], 4
+; CHECK-NEXT: br i1 [[MIN_ITERS_CHECK]], label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]]
+; CHECK: [[VECTOR_PH]]:
+; CHECK-NEXT: [[N_MOD_VF:%.*]] = urem i64 [[TMP2]], 4
+; CHECK-NEXT: [[N_VEC:%.*]] = sub i64 [[TMP2]], [[N_MOD_VF]]
+; CHECK-NEXT: [[TMP3:%.*]] = mul i64 [[N_VEC]], 2
+; CHECK-NEXT: [[TMP4:%.*]] = add i64 10, [[TMP3]]
+; CHECK-NEXT: br label %[[VECTOR_BODY:.*]]
+; CHECK: [[VECTOR_BODY]]:
+; CHECK-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-NEXT: [[VEC_IND1:%.*]] = phi <4 x i64> [ <i64 10, i64 12, i64 14, i64 16>, %[[VECTOR_PH]] ], [ [[VEC_IND_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-NEXT: [[VEC_PHI:%.*]] = phi <4 x i64> [ zeroinitializer, %[[VECTOR_PH]] ], [ [[TMP23:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-NEXT: [[TMP5:%.*]] = mul i64 [[INDEX]], 2
+; CHECK-NEXT: [[OFFSET_IDX:%.*]] = add i64 10, [[TMP5]]
+; CHECK-NEXT: [[TMP7:%.*]] = add i64 [[OFFSET_IDX]], 2
+; CHECK-NEXT: [[TMP8:%.*]] = add i64 [[OFFSET_IDX]], 4
+; CHECK-NEXT: [[TMP9:%.*]] = add i64 [[OFFSET_IDX]], 6
+; CHECK-NEXT: [[TMP10:%.*]] = getelementptr inbounds i64, ptr [[A]], i64 [[OFFSET_IDX]]
+; CHECK-NEXT: [[TMP11:%.*]] = getelementptr inbounds i64, ptr [[A]], i64 [[TMP7]]
+; CHECK-NEXT: [[TMP12:%.*]] = getelementptr inbounds i64, ptr [[A]], i64 [[TMP8]]
+; CHECK-NEXT: [[TMP13:%.*]] = getelementptr inbounds i64, ptr [[A]], i64 [[TMP9]]
+; CHECK-NEXT: [[TMP14:%.*]] = load i64, ptr [[TMP10]], align 8
+; CHECK-NEXT: [[TMP15:%.*]] = load i64, ptr [[TMP11]], align 8
+; CHECK-NEXT: [[TMP16:%.*]] = load i64, ptr [[TMP12]], align 8
+; CHECK-NEXT: [[TMP17:%.*]] = load i64, ptr [[TMP13]], align 8
+; CHECK-NEXT: [[TMP18:%.*]] = insertelement <4 x i64> poison, i64 [[TMP14]], i32 0
+; CHECK-NEXT: [[TMP19:%.*]] = insertelement <4 x i64> [[TMP18]], i64 [[TMP15]], i32 1
+; CHECK-NEXT: [[TMP20:%.*]] = insertelement <4 x i64> [[TMP19]], i64 [[TMP16]], i32 2
+; CHECK-NEXT: [[TMP21:%.*]] = insertelement <4 x i64> [[TMP20]], i64 [[TMP17]], i32 3
+; CHECK-NEXT: [[TMP22:%.*]] = icmp eq <4 x i64> [[TMP21]], splat (i64 42)
+; CHECK-NEXT: [[VEC_IND:%.*]] = add <4 x i64> [[VEC_IND1]], splat (i64 100)
+; CHECK-NEXT: [[TMP23]] = select <4 x i1> [[TMP22]], <4 x i64> [[VEC_IND]], <4 x i64> [[VEC_PHI]]
+; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 4
+; CHECK-NEXT: [[VEC_IND_NEXT]] = add nuw nsw <4 x i64> [[VEC_IND1]], splat (i64 8)
+; CHECK-NEXT: [[TMP24:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
+; CHECK-NEXT: br i1 [[TMP24]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP10:![0-9]+]]
+; CHECK: [[MIDDLE_BLOCK]]:
+; CHECK-NEXT: [[TMP27:%.*]] = call i64 @llvm.vector.reduce.umax.v4i64(<4 x i64> [[TMP23]])
+; CHECK-NEXT: [[TMP26:%.*]] = icmp ne i64 [[TMP27]], 0
+; CHECK-NEXT: [[TMP28:%.*]] = select i1 [[TMP26]], i64 [[TMP27]], i64 -1
+; CHECK-NEXT: [[CMP_N:%.*]] = icmp eq i64 [[TMP2]], [[N_VEC]]
+; CHECK-NEXT: br i1 [[CMP_N]], label %[[DONE:.*]], label %[[SCALAR_PH]]
+; CHECK: [[SCALAR_PH]]:
+; CHECK-NEXT: [[BC_RESUME_VAL:%.*]] = phi i64 [ [[TMP4]], %[[MIDDLE_BLOCK]] ], [ 10, %[[ENTRY]] ]
+; CHECK-NEXT: [[BC_MERGE_RDX:%.*]] = phi i64 [ [[TMP28]], %[[MIDDLE_BLOCK]] ], [ -1, %[[ENTRY]] ]
+; CHECK-NEXT: br label %[[LOOP:.*]]
+; CHECK: [[LOOP]]:
+; CHECK-NEXT: [[IV:%.*]] = phi i64 [ [[BC_RESUME_VAL]], %[[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], %[[LOOP]] ]
+; CHECK-NEXT: [[RDX:%.*]] = phi i64 [ [[BC_MERGE_RDX]], %[[SCALAR_PH]] ], [ [[SEL:%.*]], %[[LOOP]] ]
+; CHECK-NEXT: [[GEP:%.*]] = getelementptr inbounds i64, ptr [[A]], i64 [[IV]]
+; CHECK-NEXT: [[LOAD:%.*]] = load i64, ptr [[GEP]], align 8
+; CHECK-NEXT: [[CMP:%.*]] = icmp eq i64 [[LOAD]], 42
+; CHECK-NEXT: [[EXPR:%.*]] = add i64 [[IV]], 100
+; CHECK-NEXT: [[SEL]] = select i1 [[CMP]], i64 [[EXPR]], i64 [[RDX]]
+; CHECK-NEXT: [[IV_NEXT]] = add nuw nsw i64 [[IV]], 2
+; CHECK-NEXT: [[EXIT:%.*]] = icmp eq i64 [[IV_NEXT]], [[N]]
+; CHECK-NEXT: br i1 [[EXIT]], label %[[DONE]], label %[[LOOP]], !llvm.loop [[LOOP11:![0-9]+]]
+; CHECK: [[DONE]]:
+; CHECK-NEXT: [[SEL_LCSSA:%.*]] = phi i64 [ [[SEL]], %[[LOOP]] ], [ [[TMP28]], %[[MIDDLE_BLOCK]] ]
+; CHECK-NEXT: ret i64 [[SEL_LCSSA]]
;
entry:
br label %loop
@@ -602,94 +376,55 @@ done:
; Expression sinking should NOT apply because introducing wider computations
; in the middle block would be slower when the wide IV can be eliminated.
define i32 @findlast_wider_iv_only_scalar_users(ptr %a, i64 %n) {
-; NO-TF-LABEL: define i32 @findlast_wider_iv_only_scalar_users(
-; NO-TF-SAME: ptr [[A:%.*]], i64 [[N:%.*]]) {
-; NO-TF-NEXT: [[ENTRY:.*]]:
-; NO-TF-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[N]], 4
-; NO-TF-NEXT: br i1 [[MIN_ITERS_CHECK]], label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]]
-; NO-TF: [[VECTOR_PH]]:
-; NO-TF-NEXT: [[N_MOD_VF:%.*]] = urem i64 [[N]], 4
-; NO-TF-NEXT: [[N_VEC:%.*]] = sub i64 [[N]], [[N_MOD_VF]]
-; NO-TF-NEXT: br label %[[VECTOR_BODY:.*]]
-; NO-TF: [[VECTOR_BODY]]:
-; NO-TF-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
-; NO-TF-NEXT: [[VEC_PHI:%.*]] = phi <4 x i32> [ splat (i32 -1), %[[VECTOR_PH]] ], [ [[TMP7:%.*]], %[[VECTOR_BODY]] ]
-; NO-TF-NEXT: [[TMP0:%.*]] = phi <4 x i1> [ zeroinitializer, %[[VECTOR_PH]] ], [ [[TMP6:%.*]], %[[VECTOR_BODY]] ]
-; NO-TF-NEXT: [[VEC_IND:%.*]] = phi <4 x i32> [ <i32 0, i32 1, i32 2, i32 3>, %[[VECTOR_PH]] ], [ [[VEC_IND_NEXT:%.*]], %[[VECTOR_BODY]] ]
-; NO-TF-NEXT: [[TMP1:%.*]] = getelementptr inbounds i32, ptr [[A]], i64 [[INDEX]]
-; NO-TF-NEXT: [[WIDE_LOAD:%.*]] = load <4 x i32>, ptr [[TMP1]], align 4
-; NO-TF-NEXT: [[TMP2:%.*]] = icmp eq <4 x i32> [[WIDE_LOAD]], splat (i32 42)
-; NO-TF-NEXT: [[TMP3:%.*]] = shl <4 x i32> [[VEC_IND]], splat (i32 2)
-; NO-TF-NEXT: [[TMP4:%.*]] = freeze <4 x i1> [[TMP2]]
-; NO-TF-NEXT: [[TMP5:%.*]] = call i1 @llvm.vector.reduce.or.v4i1(<4 x i1> [[TMP4]])
-; NO-TF-NEXT: [[TMP6]] = select i1 [[TMP5]], <4 x i1> [[TMP2]], <4 x i1> [[TMP0]]
-; NO-TF-NEXT: [[TMP7]] = select i1 [[TMP5]], <4 x i32> [[TMP3]], <4 x i32> [[VEC_PHI]]
-; NO-TF-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 4
-; NO-TF-NEXT: [[VEC_IND_NEXT]] = add <4 x i32> [[VEC_IND]], splat (i32 4)
-; NO-TF-NEXT: [[TMP8:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
-; NO-TF-NEXT: br i1 [[TMP8]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP12:![0-9]+]]
-; NO-TF: [[MIDDLE_BLOCK]]:
-; NO-TF-NEXT: [[TMP9:%.*]] = call i32 @llvm.experimental.vector.extract.last.active.v4i32(<4 x i32> [[TMP7]], <4 x i1> [[TMP6]], i32 -1)
-; NO-TF-NEXT: [[CMP_N:%.*]] = icmp eq i64 [[N]], [[N_VEC]]
-; NO-TF-NEXT: br i1 [[CMP_N]], label %[[DONE:.*]], label %[[SCALAR_PH]]
-; NO-TF: [[SCALAR_PH]]:
-; NO-TF-NEXT: [[BC_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC]], %[[MIDDLE_BLOCK]] ], [ 0, %[[ENTRY]] ]
-; NO-TF-NEXT: [[BC_MERGE_RDX:%.*]] = phi i32 [ [[TMP9]], %[[MIDDLE_BLOCK]] ], [ -1, %[[ENTRY]] ]
-; NO-TF-NEXT: br label %[[LOOP:.*]]
-; NO-TF: [[LOOP]]:
-; NO-TF-NEXT: [[IV:%.*]] = phi i64 [ [[BC_RESUME_VAL]], %[[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], %[[LOOP]] ]
-; NO-TF-NEXT: [[RDX:%.*]] = phi i32 [ [[BC_MERGE_RDX]], %[[SCALAR_PH]] ], [ [[SEL:%.*]], %[[LOOP]] ]
-; NO-TF-NEXT: [[GEP:%.*]] = getelementptr inbounds i32, ptr [[A]], i64 [[IV]]
-; NO-TF-NEXT: [[LOAD:%.*]] = load i32, ptr [[GEP]], align 4
-; NO-TF-NEXT: [[CMP:%.*]] = icmp eq i32 [[LOAD]], 42
-; NO-TF-NEXT: [[IV_TRUNC:%.*]] = trunc i64 [[IV]] to i32
-; NO-TF-NEXT: [[EXPR:%.*]] = shl i32 [[IV_TRUNC]], 2
-; NO-TF-NEXT: [[SEL]] = select i1 [[CMP]], i32 [[EXPR]], i32 [[RDX]]
-; NO-TF-NEXT: [[IV_NEXT]] = add nuw nsw i64 [[IV]], 1
-; NO-TF-NEXT: [[EXIT:%.*]] = icmp eq i64 [[IV_NEXT]], [[N]]
-; NO-TF-NEXT: br i1 [[EXIT]], label %[[DONE]], label %[[LOOP]], !llvm.loop [[LOOP13:![0-9]+]]
-; NO-TF: [[DONE]]:
-; NO-TF-NEXT: [[SEL_LCSSA:%.*]] = phi i32 [ [[SEL]], %[[LOOP]] ], [ [[TMP9]], %[[MIDDLE_BLOCK]] ]
-; NO-TF-NEXT: ret i32 [[SEL_LCSSA]]
-;
-; TF-LABEL: define i32 @findlast_wider_iv_only_scalar_users(
-; TF-SAME: ptr [[A:%.*]], i64 [[N:%.*]]) {
-; TF-NEXT: [[ENTRY:.*:]]
-; TF-NEXT: br label %[[VECTOR_PH:.*]]
-; TF: [[VECTOR_PH]]:
-; TF-NEXT: [[N_RND_UP:%.*]] = add i64 [[N]], 3
-; TF-NEXT: [[N_MOD_VF:%.*]] = urem i64 [[N_RND_UP]], 4
-; TF-NEXT: [[N_VEC:%.*]] = sub i64 [[N_RND_UP]], [[N_MOD_VF]]
-; TF-NEXT: [[TRIP_COUNT_MINUS_1:%.*]] = sub i64 [[N]], 1
-; TF-NEXT: [[BROADCAST_SPLATINSERT:%.*]] = insertelement <4 x i64> poison, i64 [[TRIP_COUNT_MINUS_1]], i64 0
-; TF-NEXT: [[BROADCAST_SPLAT:%.*]] = shufflevector <4 x i64> [[BROADCAST_SPLATINSERT]], <4 x i64> poison, <4 x i32> zeroinitializer
-; TF-NEXT: br label %[[VECTOR_BODY:.*]]
-; TF: [[VECTOR_BODY]]:
-; TF-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
-; TF-NEXT: [[VEC_IND:%.*]] = phi <4 x i64> [ <i64 0, i64 1, i64 2, i64 3>, %[[VECTOR_PH]] ], [ [[VEC_IND_NEXT:%.*]], %[[VECTOR_BODY]] ]
-; TF-NEXT: [[VEC_PHI:%.*]] = phi <4 x i32> [ splat (i32 -1), %[[VECTOR_PH]] ], [ [[TMP9:%.*]], %[[VECTOR_BODY]] ]
-; TF-NEXT: [[TMP0:%.*]] = phi <4 x i1> [ zeroinitializer, %[[VECTOR_PH]] ], [ [[TMP8:%.*]], %[[VECTOR_BODY]] ]
-; TF-NEXT: [[VEC_IND1:%.*]] = phi <4 x i32> [ <i32 0, i32 1, i32 2, i32 3>, %[[VECTOR_PH]] ], [ [[VEC_IND_NEXT2:%.*]], %[[VECTOR_BODY]] ]
-; TF-NEXT: [[TMP1:%.*]] = icmp ule <4 x i64> [[VEC_IND]], [[BROADCAST_SPLAT]]
-; TF-NEXT: [[TMP2:%.*]] = getelementptr inbounds i32, ptr [[A]], i64 [[INDEX]]
-; TF-NEXT: [[WIDE_MASKED_LOAD:%.*]] = call <4 x i32> @llvm.masked.load.v4i32.p0(ptr align 4 [[TMP2]], <4 x i1> [[TMP1]], <4 x i32> poison)
-; TF-NEXT: [[TMP3:%.*]] = icmp eq <4 x i32> [[WIDE_MASKED_LOAD]], splat (i32 42)
-; TF-NEXT: [[TMP4:%.*]] = shl <4 x i32> [[VEC_IND1]], splat (i32 2)
-; TF-NEXT: [[TMP5:%.*]] = select <4 x i1> [[TMP1]], <4 x i1> [[TMP3]], <4 x i1> zeroinitializer
-; TF-NEXT: [[TMP6:%.*]] = freeze <4 x i1> [[TMP5]]
-; TF-NEXT: [[TMP7:%.*]] = call i1 @llvm.vector.reduce.or.v4i1(<4 x i1> [[TMP6]])
-; TF-NEXT: [[TMP8]] = select i1 [[TMP7]], <4 x i1> [[TMP5]], <4 x i1> [[TMP0]]
-; TF-NEXT: [[TMP9]] = select i1 [[TMP7]], <4 x i32> [[TMP4]], <4 x i32> [[VEC_PHI]]
-; TF-NEXT: [[INDEX_NEXT]] = add i64 [[INDEX]], 4
-; TF-NEXT: [[VEC_IND_NEXT]] = add <4 x i64> [[VEC_IND]], splat (i64 4)
-; TF-NEXT: [[VEC_IND_NEXT2]] = add <4 x i32> [[VEC_IND1]], splat (i32 4)
-; TF-NEXT: [[TMP10:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
-; TF-NEXT: br i1 [[TMP10]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP7:![0-9]+]]
-; TF: [[MIDDLE_BLOCK]]:
-; TF-NEXT: [[TMP11:%.*]] = call i32 @llvm.experimental.vector.extract.last.active.v4i32(<4 x i32> [[TMP9]], <4 x i1> [[TMP8]], i32 -1)
-; TF-NEXT: br label %[[DONE:.*]]
-; TF: [[DONE]]:
-; TF-NEXT: ret i32 [[TMP11]]
+; CHECK-LABEL: define i32 @findlast_wider_iv_only_scalar_users(
+; CHECK-SAME: ptr [[A:%.*]], i64 [[N:%.*]]) {
+; CHECK-NEXT: [[ENTRY:.*]]:
+; CHECK-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[N]], 4
+; CHECK-NEXT: br i1 [[MIN_ITERS_CHECK]], label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]]
+; CHECK: [[VECTOR_PH]]:
+; CHECK-NEXT: [[N_MOD_VF:%.*]] = urem i64 [[N]], 4
+; CHECK-NEXT: [[N_VEC:%.*]] = sub i64 [[N]], [[N_MOD_VF]]
+; CHECK-NEXT: br label %[[VECTOR_BODY:.*]]
+; CHECK: [[VECTOR_BODY]]:
+; CHECK-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-NEXT: [[VEC_PHI:%.*]] = phi <4 x i32> [ splat (i32 -1), %[[VECTOR_PH]] ], [ [[TMP7:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-NEXT: [[TMP0:%.*]] = phi <4 x i1> [ zeroinitializer, %[[VECTOR_PH]] ], [ [[TMP6:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-NEXT: [[VEC_IND:%.*]] = phi <4 x i32> [ <i32 0, i32 1, i32 2, i32 3>, %[[VECTOR_PH]] ], [ [[VEC_IND_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-NEXT: [[TMP1:%.*]] = getelementptr inbounds i32, ptr [[A]], i64 [[INDEX]]
+; CHECK-NEXT: [[WIDE_LOAD:%.*]] = load <4 x i32>, ptr [[TMP1]], align 4
+; CHECK-NEXT: [[TMP2:%.*]] = icmp eq <4 x i32> [[WIDE_LOAD]], splat (i32 42)
+; CHECK-NEXT: [[TMP3:%.*]] = shl <4 x i32> [[VEC_IND]], splat (i32 2)
+; CHECK-NEXT: [[TMP4:%.*]] = freeze <4 x i1> [[TMP2]]
+; CHECK-NEXT: [[TMP5:%.*]] = call i1 @llvm.vector.reduce.or.v4i1(<4 x i1> [[TMP4]])
+; CHECK-NEXT: [[TMP6]] = select i1 [[TMP5]], <4 x i1> [[TMP2]], <4 x i1> [[TMP0]]
+; CHECK-NEXT: [[TMP7]] = select i1 [[TMP5]], <4 x i32> [[TMP3]], <4 x i32> [[VEC_PHI]]
+; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 4
+; CHECK-NEXT: [[VEC_IND_NEXT]] = add <4 x i32> [[VEC_IND]], splat (i32 4)
+; CHECK-NEXT: [[TMP8:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
+; CHECK-NEXT: br i1 [[TMP8]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP12:![0-9]+]]
+; CHECK: [[MIDDLE_BLOCK]]:
+; CHECK-NEXT: [[TMP9:%.*]] = call i32 @llvm.experimental.vector.extract.last.active.v4i32(<4 x i32> [[TMP7]], <4 x i1> [[TMP6]], i32 -1)
+; CHECK-NEXT: [[CMP_N:%.*]] = icmp eq i64 [[N]], [[N_VEC]]
+; CHECK-NEXT: br i1 [[CMP_N]], label %[[DONE:.*]], label %[[SCALAR_PH]]
+; CHECK: [[SCALAR_PH]]:
+; CHECK-NEXT: [[BC_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC]], %[[MIDDLE_BLOCK]] ], [ 0, %[[ENTRY]] ]
+; CHECK-NEXT: [[BC_MERGE_RDX:%.*]] = phi i32 [ [[TMP9]], %[[MIDDLE_BLOCK]] ], [ -1, %[[ENTRY]] ]
+; CHECK-NEXT: br label %[[LOOP:.*]]
+; CHECK: [[LOOP]]:
+; CHECK-NEXT: [[IV:%.*]] = phi i64 [ [[BC_RESUME_VAL]], %[[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], %[[LOOP]] ]
+; CHECK-NEXT: [[RDX:%.*]] = phi i32 [ [[BC_MERGE_RDX]], %[[SCALAR_PH]] ], [ [[SEL:%.*]], %[[LOOP]] ]
+; CHECK-NEXT: [[GEP:%.*]] = getelementptr inbounds i32, ptr [[A]], i64 [[IV]]
+; CHECK-NEXT: [[LOAD:%.*]] = load i32, ptr [[GEP]], align 4
+; CHECK-NEXT: [[CMP:%.*]] = icmp eq i32 [[LOAD]], 42
+; CHECK-NEXT: [[IV_TRUNC:%.*]] = trunc i64 [[IV]] to i32
+; CHECK-NEXT: [[EXPR:%.*]] = shl i32 [[IV_TRUNC]], 2
+; CHECK-NEXT: [[SEL]] = select i1 [[CMP]], i32 [[EXPR]], i32 [[RDX]]
+; CHECK-NEXT: [[IV_NEXT]] = add nuw nsw i64 [[IV]], 1
+; CHECK-NEXT: [[EXIT:%.*]] = icmp eq i64 [[IV_NEXT]], [[N]]
+; CHECK-NEXT: br i1 [[EXIT]], label %[[DONE]], label %[[LOOP]], !llvm.loop [[LOOP13:![0-9]+]]
+; CHECK: [[DONE]]:
+; CHECK-NEXT: [[SEL_LCSSA:%.*]] = phi i32 [ [[SEL]], %[[LOOP]] ], [ [[TMP9]], %[[MIDDLE_BLOCK]] ]
+; CHECK-NEXT: ret i32 [[SEL_LCSSA]]
;
entry:
br label %loop
@@ -716,141 +451,72 @@ done:
; (the mul that feeds into both the store and the reduction), so it will
; stay wide anyway.
define i32 @findlast_wider_iv_with_vector_users(ptr %a, ptr %b, i64 %n) {
-; NO-TF-LABEL: define i32 @findlast_wider_iv_with_vector_users(
-; NO-TF-SAME: ptr [[A:%.*]], ptr [[B:%.*]], i64 [[N:%.*]]) {
-; NO-TF-NEXT: [[ENTRY:.*]]:
-; NO-TF-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[N]], 4
-; NO-TF-NEXT: br i1 [[MIN_ITERS_CHECK]], label %[[SCALAR_PH:.*]], label %[[VECTOR_MEMCHECK:.*]]
-; NO-TF: [[VECTOR_MEMCHECK]]:
-; NO-TF-NEXT: [[TMP0:%.*]] = shl i64 [[N]], 3
-; NO-TF-NEXT: [[SCEVGEP:%.*]] = getelementptr i8, ptr [[B]], i64 [[TMP0]]
-; NO-TF-NEXT: [[TMP1:%.*]] = shl i64 [[N]], 2
-; NO-TF-NEXT: [[SCEVGEP1:%.*]] = getelementptr i8, ptr [[A]], i64 [[TMP1]]
-; NO-TF-NEXT: [[BOUND0:%.*]] = icmp ult ptr [[B]], [[SCEVGEP1]]
-; NO-TF-NEXT: [[BOUND1:%.*]] = icmp ult ptr [[A]], [[SCEVGEP]]
-; NO-TF-NEXT: [[FOUND_CONFLICT:%.*]] = and i1 [[BOUND0]], [[BOUND1]]
-; NO-TF-NEXT: br i1 [[FOUND_CONFLICT]], label %[[SCALAR_PH]], label %[[VECTOR_PH:.*]]
-; NO-TF: [[VECTOR_PH]]:
-; NO-TF-NEXT: [[N_MOD_VF:%.*]] = urem i64 [[N]], 4
-; NO-TF-NEXT: [[N_VEC:%.*]] = sub i64 [[N]], [[N_MOD_VF]]
-; NO-TF-NEXT: br label %[[VECTOR_BODY:.*]]
-; NO-TF: [[VECTOR_BODY]]:
-; NO-TF-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
-; NO-TF-NEXT: [[VEC_IND:%.*]] = phi <4 x i64> [ <i64 0, i64 1, i64 2, i64 3>, %[[VECTOR_PH]] ], [ [[VEC_IND_NEXT:%.*]], %[[VECTOR_BODY]] ]
-; NO-TF-NEXT: [[VEC_PHI:%.*]] = phi <4 x i32> [ splat (i32 -1), %[[VECTOR_PH]] ], [ [[TMP11:%.*]], %[[VECTOR_BODY]] ]
-; NO-TF-NEXT: [[TMP2:%.*]] = phi <4 x i1> [ zeroinitializer, %[[VECTOR_PH]] ], [ [[TMP10:%.*]], %[[VECTOR_BODY]] ]
-; NO-TF-NEXT: [[VEC_IND2:%.*]] = phi <4 x i32> [ <i32 0, i32 1, i32 2, i32 3>, %[[VECTOR_PH]] ], [ [[VEC_IND_NEXT3:%.*]], %[[VECTOR_BODY]] ]
-; NO-TF-NEXT: [[TMP3:%.*]] = getelementptr inbounds i32, ptr [[A]], i64 [[INDEX]]
-; NO-TF-NEXT: [[WIDE_LOAD:%.*]] = load <4 x i32>, ptr [[TMP3]], align 4, !alias.scope [[META14:![0-9]+]]
-; NO-TF-NEXT: [[TMP4:%.*]] = icmp eq <4 x i32> [[WIDE_LOAD]], splat (i32 42)
-; NO-TF-NEXT: [[TMP5:%.*]] = mul <4 x i64> [[VEC_IND]], splat (i64 3)
-; NO-TF-NEXT: [[TMP6:%.*]] = getelementptr inbounds i64, ptr [[B]], i64 [[INDEX]]
-; NO-TF-NEXT: store <4 x i64> [[TMP5]], ptr [[TMP6]], align 8, !alias.scope [[META17:![0-9]+]], !noalias [[META14]]
-; NO-TF-NEXT: [[TMP7:%.*]] = add <4 x i32> [[VEC_IND2]], splat (i32 100)
-; NO-TF-NEXT: [[TMP8:%.*]] = freeze <4 x i1> [[TMP4]]
-; NO-TF-NEXT: [[TMP9:%.*]] = call i1 @llvm.vector.reduce.or.v4i1(<4 x i1> [[TMP8]])
-; NO-TF-NEXT: [[TMP10]] = select i1 [[TMP9]], <4 x i1> [[TMP4]], <4 x i1> [[TMP2]]
-; NO-TF-NEXT: [[TMP11]] = select i1 [[TMP9]], <4 x i32> [[TMP7]], <4 x i32> [[VEC_PHI]]
-; NO-TF-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 4
-; NO-TF-NEXT: [[VEC_IND_NEXT]] = add nuw nsw <4 x i64> [[VEC_IND]], splat (i64 4)
-; NO-TF-NEXT: [[VEC_IND_NEXT3]] = add <4 x i32> [[VEC_IND2]], splat (i32 4)
-; NO-TF-NEXT: [[TMP12:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
-; NO-TF-NEXT: br i1 [[TMP12]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP19:![0-9]+]]
-; NO-TF: [[MIDDLE_BLOCK]]:
-; NO-TF-NEXT: [[TMP13:%.*]] = call i32 @llvm.experimental.vector.extract.last.active.v4i32(<4 x i32> [[TMP11]], <4 x i1> [[TMP10]], i32 -1)
-; NO-TF-NEXT: [[CMP_N:%.*]] = icmp eq i64 [[N]], [[N_VEC]]
-; NO-TF-NEXT: br i1 [[CMP_N]], label %[[DONE:.*]], label %[[SCALAR_PH]]
-; NO-TF: [[SCALAR_PH]]:
-; NO-TF-NEXT: [[BC_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC]], %[[MIDDLE_BLOCK]] ], [ 0, %[[ENTRY]] ], [ 0, %[[VECTOR_MEMCHECK]] ]
-; NO-TF-NEXT: [[BC_MERGE_RDX:%.*]] = phi i32 [ [[TMP13]], %[[MIDDLE_BLOCK]] ], [ -1, %[[ENTRY]] ], [ -1, %[[VECTOR_MEMCHECK]] ]
-; NO-TF-NEXT: br label %[[LOOP:.*]]
-; NO-TF: [[LOOP]]:
-; NO-TF-NEXT: [[IV:%.*]] = phi i64 [ [[BC_RESUME_VAL]], %[[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], %[[LOOP]] ]
-; NO-TF-NEXT: [[RDX:%.*]] = phi i32 [ [[BC_MERGE_RDX]], %[[SCALAR_PH]] ], [ [[SEL:%.*]], %[[LOOP]] ]
-; NO-TF-NEXT: [[GEP_A:%.*]] = getelementptr inbounds i32, ptr [[A]], i64 [[IV]]
-; NO-TF-NEXT: [[LOAD:%.*]] = load i32, ptr [[GEP_A]], align 4
-; NO-TF-NEXT: [[CMP:%.*]] = icmp eq i32 [[LOAD]], 42
-; NO-TF-NEXT: [[IV_MUL:%.*]] = mul i64 [[IV]], 3
-; NO-TF-NEXT: [[GEP_B:%.*]] = getelementptr inbounds i64, ptr [[B]], i64 [[IV]]
-; NO-TF-NEXT: store i64 [[IV_MUL]], ptr [[GEP_B]], align 8
-; NO-TF-NEXT: [[IV_TRUNC:%.*]] = trunc i64 [[IV]] to i32
-; NO-TF-NEXT: [[EXPR:%.*]] = add i32 [[IV_TRUNC]], 100
-; NO-TF-NEXT: [[SEL]] = select i1 [[CMP]], i32 [[EXPR]], i32 [[RDX]]
-; NO-TF-NEXT: [[IV_NEXT]] = add nuw nsw i64 [[IV]], 1
-; NO-TF-NEXT: [[EXIT:%.*]] = icmp eq i64 [[IV_NEXT]], [[N]]
-; NO-TF-NEXT: br i1 [[EXIT]], label %[[DONE]], label %[[LOOP]], !llvm.loop [[LOOP20:![0-9]+]]
-; NO-TF: [[DONE]]:
-; NO-TF-NEXT: [[SEL_LCSSA:%.*]] = phi i32 [ [[SEL]], %[[LOOP]] ], [ [[TMP13]], %[[MIDDLE_BLOCK]] ]
-; NO-TF-NEXT: ret i32 [[SEL_LCSSA]]
-;
-; TF-LABEL: define i32 @findlast_wider_iv_with_vector_users(
-; TF-SAME: ptr [[A:%.*]], ptr [[B:%.*]], i64 [[N:%.*]]) {
-; TF-NEXT: [[ENTRY:.*:]]
-; TF-NEXT: br label %[[VECTOR_MEMCHECK:.*]]
-; TF: [[VECTOR_MEMCHECK]]:
-; TF-NEXT: [[TMP0:%.*]] = shl i64 [[N]], 3
-; TF-NEXT: [[SCEVGEP:%.*]] = getelementptr i8, ptr [[B]], i64 [[TMP0]]
-; TF-NEXT: [[TMP1:%.*]] = shl i64 [[N]], 2
-; TF-NEXT: [[SCEVGEP1:%.*]] = getelementptr i8, ptr [[A]], i64 [[TMP1]]
-; TF-NEXT: [[BOUND0:%.*]] = icmp ult ptr [[B]], [[SCEVGEP1]]
-; TF-NEXT: [[BOUND1:%.*]] = icmp ult ptr [[A]], [[SCEVGEP]]
-; TF-NEXT: [[FOUND_CONFLICT:%.*]] = and i1 [[BOUND0]], [[BOUND1]]
-; TF-NEXT: br i1 [[FOUND_CONFLICT]], label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]]
-; TF: [[VECTOR_PH]]:
-; TF-NEXT: [[N_RND_UP:%.*]] = add i64 [[N]], 3
-; TF-NEXT: [[N_MOD_VF:%.*]] = urem i64 [[N_RND_UP]], 4
-; TF-NEXT: [[N_VEC:%.*]] = sub i64 [[N_RND_UP]], [[N_MOD_VF]]
-; TF-NEXT: [[TRIP_COUNT_MINUS_1:%.*]] = sub i64 [[N]], 1
-; TF-NEXT: [[BROADCAST_SPLATINSERT:%.*]] = insertelement <4 x i64> poison, i64 [[TRIP_COUNT_MINUS_1]], i64 0
-; TF-NEXT: [[BROADCAST_SPLAT:%.*]] = shufflevector <4 x i64> [[BROADCAST_SPLATINSERT]], <4 x i64> poison, <4 x i32> zeroinitializer
-; TF-NEXT: br label %[[VECTOR_BODY:.*]]
-; TF: [[VECTOR_BODY]]:
-; TF-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
-; TF-NEXT: [[VEC_IND:%.*]] = phi <4 x i64> [ <i64 0, i64 1, i64 2, i64 3>, %[[VECTOR_PH]] ], [ [[VEC_IND_NEXT:%.*]], %[[VECTOR_BODY]] ]
-; TF-NEXT: [[VEC_PHI:%.*]] = phi <4 x i32> [ splat (i32 -1), %[[VECTOR_PH]] ], [ [[TMP13:%.*]], %[[VECTOR_BODY]] ]
-; TF-NEXT: [[TMP2:%.*]] = phi <4 x i1> [ zeroinitializer, %[[VECTOR_PH]] ], [ [[TMP12:%.*]], %[[VECTOR_BODY]] ]
-; TF-NEXT: [[VEC_IND2:%.*]] = phi <4 x i32> [ <i32 0, i32 1, i32 2, i32 3>, %[[VECTOR_PH]] ], [ [[VEC_IND_NEXT3:%.*]], %[[VECTOR_BODY]] ]
-; TF-NEXT: [[TMP3:%.*]] = icmp ule <4 x i64> [[VEC_IND]], [[BROADCAST_SPLAT]]
-; TF-NEXT: [[TMP4:%.*]] = getelementptr inbounds i32, ptr [[A]], i64 [[INDEX]]
-; TF-NEXT: [[WIDE_MASKED_LOAD:%.*]] = call <4 x i32> @llvm.masked.load.v4i32.p0(ptr align 4 [[TMP4]], <4 x i1> [[TMP3]], <4 x i32> poison), !alias.scope [[META8:![0-9]+]]
-; TF-NEXT: [[TMP5:%.*]] = icmp eq <4 x i32> [[WIDE_MASKED_LOAD]], splat (i32 42)
-; TF-NEXT: [[TMP6:%.*]] = mul <4 x i64> [[VEC_IND]], splat (i64 3)
-; TF-NEXT: [[TMP7:%.*]] = getelementptr inbounds i64, ptr [[B]], i64 [[INDEX]]
-; TF-NEXT: call void @llvm.masked.store.v4i64.p0(<4 x i64> [[TMP6]], ptr align 8 [[TMP7]], <4 x i1> [[TMP3]]), !alias.scope [[META11:![0-9]+]], !noalias [[META8]]
-; TF-NEXT: [[TMP8:%.*]] = add <4 x i32> [[VEC_IND2]], splat (i32 100)
-; TF-NEXT: [[TMP9:%.*]] = select <4 x i1> [[TMP3]], <4 x i1> [[TMP5]], <4 x i1> zeroinitializer
-; TF-NEXT: [[TMP10:%.*]] = freeze <4 x i1> [[TMP9]]
-; TF-NEXT: [[TMP11:%.*]] = call i1 @llvm.vector.reduce.or.v4i1(<4 x i1> [[TMP10]])
-; TF-NEXT: [[TMP12]] = select i1 [[TMP11]], <4 x i1> [[TMP9]], <4 x i1> [[TMP2]]
-; TF-NEXT: [[TMP13]] = select i1 [[TMP11]], <4 x i32> [[TMP8]], <4 x i32> [[VEC_PHI]]
-; TF-NEXT: [[INDEX_NEXT]] = add i64 [[INDEX]], 4
-; TF-NEXT: [[VEC_IND_NEXT]] = add <4 x i64> [[VEC_IND]], splat (i64 4)
-; TF-NEXT: [[VEC_IND_NEXT3]] = add <4 x i32> [[VEC_IND2]], splat (i32 4)
-; TF-NEXT: [[TMP14:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
-; TF-NEXT: br i1 [[TMP14]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP13:![0-9]+]]
-; TF: [[MIDDLE_BLOCK]]:
-; TF-NEXT: [[TMP15:%.*]] = call i32 @llvm.experimental.vector.extract.last.active.v4i32(<4 x i32> [[TMP13]], <4 x i1> [[TMP12]], i32 -1)
-; TF-NEXT: br label %[[DONE:.*]]
-; TF: [[SCALAR_PH]]:
-; TF-NEXT: br label %[[LOOP:.*]]
-; TF: [[LOOP]]:
-; TF-NEXT: [[IV:%.*]] = phi i64 [ 0, %[[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], %[[LOOP]] ]
-; TF-NEXT: [[RDX:%.*]] = phi i32 [ -1, %[[SCALAR_PH]] ], [ [[SEL:%.*]], %[[LOOP]] ]
-; TF-NEXT: [[GEP_A:%.*]] = getelementptr inbounds i32, ptr [[A]], i64 [[IV]]
-; TF-NEXT: [[LOAD:%.*]] = load i32, ptr [[GEP_A]], align 4
-; TF-NEXT: [[CMP:%.*]] = icmp eq i32 [[LOAD]], 42
-; TF-NEXT: [[IV_MUL:%.*]] = mul i64 [[IV]], 3
-; TF-NEXT: [[GEP_B:%.*]] = getelementptr inbounds i64, ptr [[B]], i64 [[IV]]
-; TF-NEXT: store i64 [[IV_MUL]], ptr [[GEP_B]], align 8
-; TF-NEXT: [[IV_TRUNC:%.*]] = trunc i64 [[IV]] to i32
-; TF-NEXT: [[EXPR:%.*]] = add i32 [[IV_TRUNC]], 100
-; TF-NEXT: [[SEL]] = select i1 [[CMP]], i32 [[EXPR]], i32 [[RDX]]
-; TF-NEXT: [[IV_NEXT]] = add nuw nsw i64 [[IV]], 1
-; TF-NEXT: [[EXIT:%.*]] = icmp eq i64 [[IV_NEXT]], [[N]]
-; TF-NEXT: br i1 [[EXIT]], label %[[DONE]], label %[[LOOP]], !llvm.loop [[LOOP14:![0-9]+]]
-; TF: [[DONE]]:
-; TF-NEXT: [[SEL_LCSSA:%.*]] = phi i32 [ [[SEL]], %[[LOOP]] ], [ [[TMP15]], %[[MIDDLE_BLOCK]] ]
-; TF-NEXT: ret i32 [[SEL_LCSSA]]
+; CHECK-LABEL: define i32 @findlast_wider_iv_with_vector_users(
+; CHECK-SAME: ptr [[A:%.*]], ptr [[B:%.*]], i64 [[N:%.*]]) {
+; CHECK-NEXT: [[ENTRY:.*]]:
+; CHECK-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[N]], 4
+; CHECK-NEXT: br i1 [[MIN_ITERS_CHECK]], label %[[SCALAR_PH:.*]], label %[[VECTOR_MEMCHECK:.*]]
+; CHECK: [[VECTOR_MEMCHECK]]:
+; CHECK-NEXT: [[TMP0:%.*]] = shl i64 [[N]], 3
+; CHECK-NEXT: [[SCEVGEP:%.*]] = getelementptr i8, ptr [[B]], i64 [[TMP0]]
+; CHECK-NEXT: [[TMP1:%.*]] = shl i64 [[N]], 2
+; CHECK-NEXT: [[SCEVGEP1:%.*]] = getelementptr i8, ptr [[A]], i64 [[TMP1]]
+; CHECK-NEXT: [[BOUND0:%.*]] = icmp ult ptr [[B]], [[SCEVGEP1]]
+; CHECK-NEXT: [[BOUND1:%.*]] = icmp ult ptr [[A]], [[SCEVGEP]]
+; CHECK-NEXT: [[FOUND_CONFLICT:%.*]] = and i1 [[BOUND0]], [[BOUND1]]
+; CHECK-NEXT: br i1 [[FOUND_CONFLICT]], label %[[SCALAR_PH]], label %[[VECTOR_PH:.*]]
+; CHECK: [[VECTOR_PH]]:
+; CHECK-NEXT: [[N_MOD_VF:%.*]] = urem i64 [[N]], 4
+; CHECK-NEXT: [[N_VEC:%.*]] = sub i64 [[N]], [[N_MOD_VF]]
+; CHECK-NEXT: br label %[[VECTOR_BODY:.*]]
+; CHECK: [[VECTOR_BODY]]:
+; CHECK-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-NEXT: [[VEC_IND:%.*]] = phi <4 x i64> [ <i64 0, i64 1, i64 2, i64 3>, %[[VECTOR_PH]] ], [ [[VEC_IND_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-NEXT: [[VEC_PHI:%.*]] = phi <4 x i32> [ splat (i32 -1), %[[VECTOR_PH]] ], [ [[TMP11:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-NEXT: [[TMP2:%.*]] = phi <4 x i1> [ zeroinitializer, %[[VECTOR_PH]] ], [ [[TMP10:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-NEXT: [[VEC_IND2:%.*]] = phi <4 x i32> [ <i32 0, i32 1, i32 2, i32 3>, %[[VECTOR_PH]] ], [ [[VEC_IND_NEXT3:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-NEXT: [[TMP3:%.*]] = getelementptr inbounds i32, ptr [[A]], i64 [[INDEX]]
+; CHECK-NEXT: [[WIDE_LOAD:%.*]] = load <4 x i32>, ptr [[TMP3]], align 4, !alias.scope [[META14:![0-9]+]]
+; CHECK-NEXT: [[TMP4:%.*]] = icmp eq <4 x i32> [[WIDE_LOAD]], splat (i32 42)
+; CHECK-NEXT: [[TMP5:%.*]] = mul <4 x i64> [[VEC_IND]], splat (i64 3)
+; CHECK-NEXT: [[TMP6:%.*]] = getelementptr inbounds i64, ptr [[B]], i64 [[INDEX]]
+; CHECK-NEXT: store <4 x i64> [[TMP5]], ptr [[TMP6]], align 8, !alias.scope [[META17:![0-9]+]], !noalias [[META14]]
+; CHECK-NEXT: [[TMP7:%.*]] = add <4 x i32> [[VEC_IND2]], splat (i32 100)
+; CHECK-NEXT: [[TMP8:%.*]] = freeze <4 x i1> [[TMP4]]
+; CHECK-NEXT: [[TMP9:%.*]] = call i1 @llvm.vector.reduce.or.v4i1(<4 x i1> [[TMP8]])
+; CHECK-NEXT: [[TMP10]] = select i1 [[TMP9]], <4 x i1> [[TMP4]], <4 x i1> [[TMP2]]
+; CHECK-NEXT: [[TMP11]] = select i1 [[TMP9]], <4 x i32> [[TMP7]], <4 x i32> [[VEC_PHI]]
+; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 4
+; CHECK-NEXT: [[VEC_IND_NEXT]] = add nuw nsw <4 x i64> [[VEC_IND]], splat (i64 4)
+; CHECK-NEXT: [[VEC_IND_NEXT3]] = add <4 x i32> [[VEC_IND2]], splat (i32 4)
+; CHECK-NEXT: [[TMP12:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
+; CHECK-NEXT: br i1 [[TMP12]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP19:![0-9]+]]
+; CHECK: [[MIDDLE_BLOCK]]:
+; CHECK-NEXT: [[TMP13:%.*]] = call i32 @llvm.experimental.vector.extract.last.active.v4i32(<4 x i32> [[TMP11]], <4 x i1> [[TMP10]], i32 -1)
+; CHECK-NEXT: [[CMP_N:%.*]] = icmp eq i64 [[N]], [[N_VEC]]
+; CHECK-NEXT: br i1 [[CMP_N]], label %[[DONE:.*]], label %[[SCALAR_PH]]
+; CHECK: [[SCALAR_PH]]:
+; CHECK-NEXT: [[BC_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC]], %[[MIDDLE_BLOCK]] ], [ 0, %[[ENTRY]] ], [ 0, %[[VECTOR_MEMCHECK]] ]
+; CHECK-NEXT: [[BC_MERGE_RDX:%.*]] = phi i32 [ [[TMP13]], %[[MIDDLE_BLOCK]] ], [ -1, %[[ENTRY]] ], [ -1, %[[VECTOR_MEMCHECK]] ]
+; CHECK-NEXT: br label %[[LOOP:.*]]
+; CHECK: [[LOOP]]:
+; CHECK-NEXT: [[IV:%.*]] = phi i64 [ [[BC_RESUME_VAL]], %[[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], %[[LOOP]] ]
+; CHECK-NEXT: [[RDX:%.*]] = phi i32 [ [[BC_MERGE_RDX]], %[[SCALAR_PH]] ], [ [[SEL:%.*]], %[[LOOP]] ]
+; CHECK-NEXT: [[GEP_A:%.*]] = getelementptr inbounds i32, ptr [[A]], i64 [[IV]]
+; CHECK-NEXT: [[LOAD:%.*]] = load i32, ptr [[GEP_A]], align 4
+; CHECK-NEXT: [[CMP:%.*]] = icmp eq i32 [[LOAD]], 42
+; CHECK-NEXT: [[IV_MUL:%.*]] = mul i64 [[IV]], 3
+; CHECK-NEXT: [[GEP_B:%.*]] = getelementptr inbounds i64, ptr [[B]], i64 [[IV]]
+; CHECK-NEXT: store i64 [[IV_MUL]], ptr [[GEP_B]], align 8
+; CHECK-NEXT: [[IV_TRUNC:%.*]] = trunc i64 [[IV]] to i32
+; CHECK-NEXT: [[EXPR:%.*]] = add i32 [[IV_TRUNC]], 100
+; CHECK-NEXT: [[SEL]] = select i1 [[CMP]], i32 [[EXPR]], i32 [[RDX]]
+; CHECK-NEXT: [[IV_NEXT]] = add nuw nsw i64 [[IV]], 1
+; CHECK-NEXT: [[EXIT:%.*]] = icmp eq i64 [[IV_NEXT]], [[N]]
+; CHECK-NEXT: br i1 [[EXIT]], label %[[DONE]], label %[[LOOP]], !llvm.loop [[LOOP20:![0-9]+]]
+; CHECK: [[DONE]]:
+; CHECK-NEXT: [[SEL_LCSSA:%.*]] = phi i32 [ [[SEL]], %[[LOOP]] ], [ [[TMP13]], %[[MIDDLE_BLOCK]] ]
+; CHECK-NEXT: ret i32 [[SEL_LCSSA]]
;
entry:
br label %loop
@@ -880,100 +546,57 @@ done:
; operand order is preserved when sinking the expression to the middle block.
; IV starts at 1 to avoid division by zero.
define i64 @findlast_sdiv_iv_as_divisor(ptr %a, i64 %n) {
-; NO-TF-LABEL: define i64 @findlast_sdiv_iv_as_divisor(
-; NO-TF-SAME: ptr [[A:%.*]], i64 [[N:%.*]]) {
-; NO-TF-NEXT: [[ENTRY:.*]]:
-; NO-TF-NEXT: [[TMP0:%.*]] = add i64 [[N]], -1
-; NO-TF-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[TMP0]], 4
-; NO-TF-NEXT: br i1 [[MIN_ITERS_CHECK]], label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]]
-; NO-TF: [[VECTOR_PH]]:
-; NO-TF-NEXT: [[N_MOD_VF:%.*]] = urem i64 [[TMP0]], 4
-; NO-TF-NEXT: [[N_VEC:%.*]] = sub i64 [[TMP0]], [[N_MOD_VF]]
-; NO-TF-NEXT: [[TMP1:%.*]] = add i64 1, [[N_VEC]]
-; NO-TF-NEXT: br label %[[VECTOR_BODY:.*]]
-; NO-TF: [[VECTOR_BODY]]:
-; NO-TF-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
-; NO-TF-NEXT: [[VEC_IND:%.*]] = phi <4 x i64> [ <i64 1, i64 2, i64 3, i64 4>, %[[VECTOR_PH]] ], [ [[VEC_IND_NEXT:%.*]], %[[VECTOR_BODY]] ]
-; NO-TF-NEXT: [[VEC_PHI:%.*]] = phi <4 x i64> [ splat (i64 -1), %[[VECTOR_PH]] ], [ [[TMP9:%.*]], %[[VECTOR_BODY]] ]
-; NO-TF-NEXT: [[TMP2:%.*]] = phi <4 x i1> [ zeroinitializer, %[[VECTOR_PH]] ], [ [[TMP8:%.*]], %[[VECTOR_BODY]] ]
-; NO-TF-NEXT: [[OFFSET_IDX:%.*]] = add i64 1, [[INDEX]]
-; NO-TF-NEXT: [[TMP3:%.*]] = getelementptr inbounds i64, ptr [[A]], i64 [[OFFSET_IDX]]
-; NO-TF-NEXT: [[WIDE_LOAD:%.*]] = load <4 x i64>, ptr [[TMP3]], align 8
-; NO-TF-NEXT: [[TMP4:%.*]] = icmp eq <4 x i64> [[WIDE_LOAD]], splat (i64 42)
-; NO-TF-NEXT: [[TMP5:%.*]] = sdiv <4 x i64> splat (i64 100), [[VEC_IND]]
-; NO-TF-NEXT: [[TMP6:%.*]] = freeze <4 x i1> [[TMP4]]
-; NO-TF-NEXT: [[TMP7:%.*]] = call i1 @llvm.vector.reduce.or.v4i1(<4 x i1> [[TMP6]])
-; NO-TF-NEXT: [[TMP8]] = select i1 [[TMP7]], <4 x i1> [[TMP4]], <4 x i1> [[TMP2]]
-; NO-TF-NEXT: [[TMP9]] = select i1 [[TMP7]], <4 x i64> [[TMP5]], <4 x i64> [[VEC_PHI]]
-; NO-TF-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 4
-; NO-TF-NEXT: [[VEC_IND_NEXT]] = add nuw nsw <4 x i64> [[VEC_IND]], splat (i64 4)
-; NO-TF-NEXT: [[TMP10:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
-; NO-TF-NEXT: br i1 [[TMP10]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP21:![0-9]+]]
-; NO-TF: [[MIDDLE_BLOCK]]:
-; NO-TF-NEXT: [[TMP11:%.*]] = call i64 @llvm.experimental.vector.extract.last.active.v4i64(<4 x i64> [[TMP9]], <4 x i1> [[TMP8]], i64 -1)
-; NO-TF-NEXT: [[CMP_N:%.*]] = icmp eq i64 [[TMP0]], [[N_VEC]]
-; NO-TF-NEXT: br i1 [[CMP_N]], label %[[DONE:.*]], label %[[SCALAR_PH]]
-; NO-TF: [[SCALAR_PH]]:
-; NO-TF-NEXT: [[BC_RESUME_VAL:%.*]] = phi i64 [ [[TMP1]], %[[MIDDLE_BLOCK]] ], [ 1, %[[ENTRY]] ]
-; NO-TF-NEXT: [[BC_MERGE_RDX:%.*]] = phi i64 [ [[TMP11]], %[[MIDDLE_BLOCK]] ], [ -1, %[[ENTRY]] ]
-; NO-TF-NEXT: br label %[[LOOP:.*]]
-; NO-TF: [[LOOP]]:
-; NO-TF-NEXT: [[IV:%.*]] = phi i64 [ [[BC_RESUME_VAL]], %[[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], %[[LOOP]] ]
-; NO-TF-NEXT: [[RDX:%.*]] = phi i64 [ [[BC_MERGE_RDX]], %[[SCALAR_PH]] ], [ [[SEL:%.*]], %[[LOOP]] ]
-; NO-TF-NEXT: [[GEP:%.*]] = getelementptr inbounds i64, ptr [[A]], i64 [[IV]]
-; NO-TF-NEXT: [[LOAD:%.*]] = load i64, ptr [[GEP]], align 8
-; NO-TF-NEXT: [[CMP:%.*]] = icmp eq i64 [[LOAD]], 42
-; NO-TF-NEXT: [[EXPR:%.*]] = sdiv i64 100, [[IV]]
-; NO-TF-NEXT: [[SEL]] = select i1 [[CMP]], i64 [[EXPR]], i64 [[RDX]]
-; NO-TF-NEXT: [[IV_NEXT]] = add nuw nsw i64 [[IV]], 1
-; NO-TF-NEXT: [[EXIT:%.*]] = icmp eq i64 [[IV_NEXT]], [[N]]
-; NO-TF-NEXT: br i1 [[EXIT]], label %[[DONE]], label %[[LOOP]], !llvm.loop [[LOOP22:![0-9]+]]
-; NO-TF: [[DONE]]:
-; NO-TF-NEXT: [[SEL_LCSSA:%.*]] = phi i64 [ [[SEL]], %[[LOOP]] ], [ [[TMP11]], %[[MIDDLE_BLOCK]] ]
-; NO-TF-NEXT: ret i64 [[SEL_LCSSA]]
-;
-; TF-LABEL: define i64 @findlast_sdiv_iv_as_divisor(
-; TF-SAME: ptr [[A:%.*]], i64 [[N:%.*]]) {
-; TF-NEXT: [[ENTRY:.*:]]
-; TF-NEXT: [[TMP0:%.*]] = add i64 [[N]], -1
-; TF-NEXT: br label %[[VECTOR_PH:.*]]
-; TF: [[VECTOR_PH]]:
-; TF-NEXT: [[N_RND_UP:%.*]] = add i64 [[TMP0]], 3
-; TF-NEXT: [[N_MOD_VF:%.*]] = urem i64 [[N_RND_UP]], 4
-; TF-NEXT: [[N_VEC:%.*]] = sub i64 [[N_RND_UP]], [[N_MOD_VF]]
-; TF-NEXT: [[TRIP_COUNT_MINUS_1:%.*]] = sub i64 [[TMP0]], 1
-; TF-NEXT: [[BROADCAST_SPLATINSERT:%.*]] = insertelement <4 x i64> poison, i64 [[TRIP_COUNT_MINUS_1]], i64 0
-; TF-NEXT: [[BROADCAST_SPLAT:%.*]] = shufflevector <4 x i64> [[BROADCAST_SPLATINSERT]], <4 x i64> poison, <4 x i32> zeroinitializer
-; TF-NEXT: br label %[[VECTOR_BODY:.*]]
-; TF: [[VECTOR_BODY]]:
-; TF-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
-; TF-NEXT: [[VEC_IND:%.*]] = phi <4 x i64> [ <i64 1, i64 2, i64 3, i64 4>, %[[VECTOR_PH]] ], [ [[VEC_IND_NEXT:%.*]], %[[VECTOR_BODY]] ]
-; TF-NEXT: [[VEC_PHI:%.*]] = phi <4 x i64> [ splat (i64 -1), %[[VECTOR_PH]] ], [ [[TMP11:%.*]], %[[VECTOR_BODY]] ]
-; TF-NEXT: [[TMP1:%.*]] = phi <4 x i1> [ zeroinitializer, %[[VECTOR_PH]] ], [ [[TMP10:%.*]], %[[VECTOR_BODY]] ]
-; TF-NEXT: [[OFFSET_IDX:%.*]] = add i64 1, [[INDEX]]
-; TF-NEXT: [[BROADCAST_SPLATINSERT1:%.*]] = insertelement <4 x i64> poison, i64 [[INDEX]], i64 0
-; TF-NEXT: [[BROADCAST_SPLAT2:%.*]] = shufflevector <4 x i64> [[BROADCAST_SPLATINSERT1]], <4 x i64> poison, <4 x i32> zeroinitializer
-; TF-NEXT: [[VEC_IV:%.*]] = add <4 x i64> [[BROADCAST_SPLAT2]], <i64 0, i64 1, i64 2, i64 3>
-; TF-NEXT: [[TMP2:%.*]] = icmp ule <4 x i64> [[VEC_IV]], [[BROADCAST_SPLAT]]
-; TF-NEXT: [[TMP3:%.*]] = getelementptr inbounds i64, ptr [[A]], i64 [[OFFSET_IDX]]
-; TF-NEXT: [[WIDE_MASKED_LOAD:%.*]] = call <4 x i64> @llvm.masked.load.v4i64.p0(ptr align 8 [[TMP3]], <4 x i1> [[TMP2]], <4 x i64> poison)
-; TF-NEXT: [[TMP4:%.*]] = icmp eq <4 x i64> [[WIDE_MASKED_LOAD]], splat (i64 42)
-; TF-NEXT: [[TMP5:%.*]] = select <4 x i1> [[TMP2]], <4 x i64> [[VEC_IND]], <4 x i64> splat (i64 1)
-; TF-NEXT: [[TMP6:%.*]] = sdiv <4 x i64> splat (i64 100), [[TMP5]]
-; TF-NEXT: [[TMP7:%.*]] = select <4 x i1> [[TMP2]], <4 x i1> [[TMP4]], <4 x i1> zeroinitializer
-; TF-NEXT: [[TMP8:%.*]] = freeze <4 x i1> [[TMP7]]
-; TF-NEXT: [[TMP9:%.*]] = call i1 @llvm.vector.reduce.or.v4i1(<4 x i1> [[TMP8]])
-; TF-NEXT: [[TMP10]] = select i1 [[TMP9]], <4 x i1> [[TMP7]], <4 x i1> [[TMP1]]
-; TF-NEXT: [[TMP11]] = select i1 [[TMP9]], <4 x i64> [[TMP6]], <4 x i64> [[VEC_PHI]]
-; TF-NEXT: [[INDEX_NEXT]] = add i64 [[INDEX]], 4
-; TF-NEXT: [[VEC_IND_NEXT]] = add nuw nsw <4 x i64> [[VEC_IND]], splat (i64 4)
-; TF-NEXT: [[TMP12:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
-; TF-NEXT: br i1 [[TMP12]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP15:![0-9]+]]
-; TF: [[MIDDLE_BLOCK]]:
-; TF-NEXT: [[TMP13:%.*]] = call i64 @llvm.experimental.vector.extract.last.active.v4i64(<4 x i64> [[TMP11]], <4 x i1> [[TMP10]], i64 -1)
-; TF-NEXT: br label %[[DONE:.*]]
-; TF: [[DONE]]:
-; TF-NEXT: ret i64 [[TMP13]]
+; CHECK-LABEL: define i64 @findlast_sdiv_iv_as_divisor(
+; CHECK-SAME: ptr [[A:%.*]], i64 [[N:%.*]]) {
+; CHECK-NEXT: [[ENTRY:.*]]:
+; CHECK-NEXT: [[TMP0:%.*]] = add i64 [[N]], -1
+; CHECK-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[TMP0]], 4
+; CHECK-NEXT: br i1 [[MIN_ITERS_CHECK]], label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]]
+; CHECK: [[VECTOR_PH]]:
+; CHECK-NEXT: [[N_MOD_VF:%.*]] = urem i64 [[TMP0]], 4
+; CHECK-NEXT: [[N_VEC:%.*]] = sub i64 [[TMP0]], [[N_MOD_VF]]
+; CHECK-NEXT: [[TMP1:%.*]] = add i64 1, [[N_VEC]]
+; CHECK-NEXT: br label %[[VECTOR_BODY:.*]]
+; CHECK: [[VECTOR_BODY]]:
+; CHECK-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-NEXT: [[VEC_IND:%.*]] = phi <4 x i64> [ <i64 1, i64 2, i64 3, i64 4>, %[[VECTOR_PH]] ], [ [[VEC_IND_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-NEXT: [[VEC_PHI:%.*]] = phi <4 x i64> [ splat (i64 -1), %[[VECTOR_PH]] ], [ [[TMP9:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-NEXT: [[TMP4:%.*]] = phi <4 x i1> [ zeroinitializer, %[[VECTOR_PH]] ], [ [[TMP8:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-NEXT: [[OFFSET_IDX:%.*]] = add i64 1, [[INDEX]]
+; CHECK-NEXT: [[TMP2:%.*]] = getelementptr inbounds i64, ptr [[A]], i64 [[OFFSET_IDX]]
+; CHECK-NEXT: [[WIDE_LOAD:%.*]] = load <4 x i64>, ptr [[TMP2]], align 8
+; CHECK-NEXT: [[TMP3:%.*]] = icmp eq <4 x i64> [[WIDE_LOAD]], splat (i64 42)
+; CHECK-NEXT: [[TMP10:%.*]] = sdiv <4 x i64> splat (i64 100), [[VEC_IND]]
+; CHECK-NEXT: [[TMP6:%.*]] = freeze <4 x i1> [[TMP3]]
+; CHECK-NEXT: [[TMP7:%.*]] = call i1 @llvm.vector.reduce.or.v4i1(<4 x i1> [[TMP6]])
+; CHECK-NEXT: [[TMP8]] = select i1 [[TMP7]], <4 x i1> [[TMP3]], <4 x i1> [[TMP4]]
+; CHECK-NEXT: [[TMP9]] = select i1 [[TMP7]], <4 x i64> [[TMP10]], <4 x i64> [[VEC_PHI]]
+; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 4
+; CHECK-NEXT: [[VEC_IND_NEXT]] = add nuw nsw <4 x i64> [[VEC_IND]], splat (i64 4)
+; CHECK-NEXT: [[TMP5:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
+; CHECK-NEXT: br i1 [[TMP5]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP21:![0-9]+]]
+; CHECK: [[MIDDLE_BLOCK]]:
+; CHECK-NEXT: [[TMP11:%.*]] = call i64 @llvm.experimental.vector.extract.last.active.v4i64(<4 x i64> [[TMP9]], <4 x i1> [[TMP8]], i64 -1)
+; CHECK-NEXT: [[CMP_N:%.*]] = icmp eq i64 [[TMP0]], [[N_VEC]]
+; CHECK-NEXT: br i1 [[CMP_N]], label %[[DONE:.*]], label %[[SCALAR_PH]]
+; CHECK: [[SCALAR_PH]]:
+; CHECK-NEXT: [[BC_RESUME_VAL:%.*]] = phi i64 [ [[TMP1]], %[[MIDDLE_BLOCK]] ], [ 1, %[[ENTRY]] ]
+; CHECK-NEXT: [[BC_MERGE_RDX:%.*]] = phi i64 [ [[TMP11]], %[[MIDDLE_BLOCK]] ], [ -1, %[[ENTRY]] ]
+; CHECK-NEXT: br label %[[LOOP:.*]]
+; CHECK: [[LOOP]]:
+; CHECK-NEXT: [[IV:%.*]] = phi i64 [ [[BC_RESUME_VAL]], %[[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], %[[LOOP]] ]
+; CHECK-NEXT: [[RDX:%.*]] = phi i64 [ [[BC_MERGE_RDX]], %[[SCALAR_PH]] ], [ [[SEL:%.*]], %[[LOOP]] ]
+; CHECK-NEXT: [[GEP:%.*]] = getelementptr inbounds i64, ptr [[A]], i64 [[IV]]
+; CHECK-NEXT: [[LOAD:%.*]] = load i64, ptr [[GEP]], align 8
+; CHECK-NEXT: [[CMP:%.*]] = icmp eq i64 [[LOAD]], 42
+; CHECK-NEXT: [[EXPR:%.*]] = sdiv i64 100, [[IV]]
+; CHECK-NEXT: [[SEL]] = select i1 [[CMP]], i64 [[EXPR]], i64 [[RDX]]
+; CHECK-NEXT: [[IV_NEXT]] = add nuw nsw i64 [[IV]], 1
+; CHECK-NEXT: [[EXIT:%.*]] = icmp eq i64 [[IV_NEXT]], [[N]]
+; CHECK-NEXT: br i1 [[EXIT]], label %[[DONE]], label %[[LOOP]], !llvm.loop [[LOOP22:![0-9]+]]
+; CHECK: [[DONE]]:
+; CHECK-NEXT: [[SEL_LCSSA:%.*]] = phi i64 [ [[SEL]], %[[LOOP]] ], [ [[TMP11]], %[[MIDDLE_BLOCK]] ]
+; CHECK-NEXT: ret i64 [[SEL_LCSSA]]
;
entry:
br label %loop
@@ -998,94 +621,55 @@ done:
; IV in the vectorized code - the IV should be narrowed to i8 since only scalar
; i8 values are needed from it.
define i8 @findlast_narrow_i8_reduction(ptr %a, i64 %n) {
-; NO-TF-LABEL: define i8 @findlast_narrow_i8_reduction(
-; NO-TF-SAME: ptr [[A:%.*]], i64 [[N:%.*]]) {
-; NO-TF-NEXT: [[ENTRY:.*]]:
-; NO-TF-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[N]], 4
-; NO-TF-NEXT: br i1 [[MIN_ITERS_CHECK]], label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]]
-; NO-TF: [[VECTOR_PH]]:
-; NO-TF-NEXT: [[N_MOD_VF:%.*]] = urem i64 [[N]], 4
-; NO-TF-NEXT: [[N_VEC:%.*]] = sub i64 [[N]], [[N_MOD_VF]]
-; NO-TF-NEXT: br label %[[VECTOR_BODY:.*]]
-; NO-TF: [[VECTOR_BODY]]:
-; NO-TF-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
-; NO-TF-NEXT: [[VEC_PHI:%.*]] = phi <4 x i8> [ splat (i8 -1), %[[VECTOR_PH]] ], [ [[TMP7:%.*]], %[[VECTOR_BODY]] ]
-; NO-TF-NEXT: [[TMP0:%.*]] = phi <4 x i1> [ zeroinitializer, %[[VECTOR_PH]] ], [ [[TMP6:%.*]], %[[VECTOR_BODY]] ]
-; NO-TF-NEXT: [[VEC_IND:%.*]] = phi <4 x i8> [ <i8 0, i8 1, i8 2, i8 3>, %[[VECTOR_PH]] ], [ [[VEC_IND_NEXT:%.*]], %[[VECTOR_BODY]] ]
-; NO-TF-NEXT: [[TMP1:%.*]] = getelementptr inbounds i8, ptr [[A]], i64 [[INDEX]]
-; NO-TF-NEXT: [[WIDE_LOAD:%.*]] = load <4 x i8>, ptr [[TMP1]], align 1
-; NO-TF-NEXT: [[TMP2:%.*]] = icmp eq <4 x i8> [[WIDE_LOAD]], splat (i8 42)
-; NO-TF-NEXT: [[TMP3:%.*]] = add <4 x i8> [[VEC_IND]], splat (i8 10)
-; NO-TF-NEXT: [[TMP4:%.*]] = freeze <4 x i1> [[TMP2]]
-; NO-TF-NEXT: [[TMP5:%.*]] = call i1 @llvm.vector.reduce.or.v4i1(<4 x i1> [[TMP4]])
-; NO-TF-NEXT: [[TMP6]] = select i1 [[TMP5]], <4 x i1> [[TMP2]], <4 x i1> [[TMP0]]
-; NO-TF-NEXT: [[TMP7]] = select i1 [[TMP5]], <4 x i8> [[TMP3]], <4 x i8> [[VEC_PHI]]
-; NO-TF-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 4
-; NO-TF-NEXT: [[VEC_IND_NEXT]] = add <4 x i8> [[VEC_IND]], splat (i8 4)
-; NO-TF-NEXT: [[TMP8:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
-; NO-TF-NEXT: br i1 [[TMP8]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP23:![0-9]+]]
-; NO-TF: [[MIDDLE_BLOCK]]:
-; NO-TF-NEXT: [[TMP9:%.*]] = call i8 @llvm.experimental.vector.extract.last.active.v4i8(<4 x i8> [[TMP7]], <4 x i1> [[TMP6]], i8 -1)
-; NO-TF-NEXT: [[CMP_N:%.*]] = icmp eq i64 [[N]], [[N_VEC]]
-; NO-TF-NEXT: br i1 [[CMP_N]], label %[[DONE:.*]], label %[[SCALAR_PH]]
-; NO-TF: [[SCALAR_PH]]:
-; NO-TF-NEXT: [[BC_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC]], %[[MIDDLE_BLOCK]] ], [ 0, %[[ENTRY]] ]
-; NO-TF-NEXT: [[BC_MERGE_RDX:%.*]] = phi i8 [ [[TMP9]], %[[MIDDLE_BLOCK]] ], [ -1, %[[ENTRY]] ]
-; NO-TF-NEXT: br label %[[LOOP:.*]]
-; NO-TF: [[LOOP]]:
-; NO-TF-NEXT: [[IV:%.*]] = phi i64 [ [[BC_RESUME_VAL]], %[[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], %[[LOOP]] ]
-; NO-TF-NEXT: [[RDX:%.*]] = phi i8 [ [[BC_MERGE_RDX]], %[[SCALAR_PH]] ], [ [[SEL:%.*]], %[[LOOP]] ]
-; NO-TF-NEXT: [[GEP:%.*]] = getelementptr inbounds i8, ptr [[A]], i64 [[IV]]
-; NO-TF-NEXT: [[LOAD:%.*]] = load i8, ptr [[GEP]], align 1
-; NO-TF-NEXT: [[CMP:%.*]] = icmp eq i8 [[LOAD]], 42
-; NO-TF-NEXT: [[IV_TRUNC:%.*]] = trunc i64 [[IV]] to i8
-; NO-TF-NEXT: [[EXPR:%.*]] = add i8 [[IV_TRUNC]], 10
-; NO-TF-NEXT: [[SEL]] = select i1 [[CMP]], i8 [[EXPR]], i8 [[RDX]]
-; NO-TF-NEXT: [[IV_NEXT]] = add nuw nsw i64 [[IV]], 1
-; NO-TF-NEXT: [[EXIT:%.*]] = icmp eq i64 [[IV_NEXT]], [[N]]
-; NO-TF-NEXT: br i1 [[EXIT]], label %[[DONE]], label %[[LOOP]], !llvm.loop [[LOOP24:![0-9]+]]
-; NO-TF: [[DONE]]:
-; NO-TF-NEXT: [[SEL_LCSSA:%.*]] = phi i8 [ [[SEL]], %[[LOOP]] ], [ [[TMP9]], %[[MIDDLE_BLOCK]] ]
-; NO-TF-NEXT: ret i8 [[SEL_LCSSA]]
-;
-; TF-LABEL: define i8 @findlast_narrow_i8_reduction(
-; TF-SAME: ptr [[A:%.*]], i64 [[N:%.*]]) {
-; TF-NEXT: [[ENTRY:.*:]]
-; TF-NEXT: br label %[[VECTOR_PH:.*]]
-; TF: [[VECTOR_PH]]:
-; TF-NEXT: [[N_RND_UP:%.*]] = add i64 [[N]], 3
-; TF-NEXT: [[N_MOD_VF:%.*]] = urem i64 [[N_RND_UP]], 4
-; TF-NEXT: [[N_VEC:%.*]] = sub i64 [[N_RND_UP]], [[N_MOD_VF]]
-; TF-NEXT: [[TRIP_COUNT_MINUS_1:%.*]] = sub i64 [[N]], 1
-; TF-NEXT: [[BROADCAST_SPLATINSERT:%.*]] = insertelement <4 x i64> poison, i64 [[TRIP_COUNT_MINUS_1]], i64 0
-; TF-NEXT: [[BROADCAST_SPLAT:%.*]] = shufflevector <4 x i64> [[BROADCAST_SPLATINSERT]], <4 x i64> poison, <4 x i32> zeroinitializer
-; TF-NEXT: br label %[[VECTOR_BODY:.*]]
-; TF: [[VECTOR_BODY]]:
-; TF-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
-; TF-NEXT: [[VEC_IND:%.*]] = phi <4 x i64> [ <i64 0, i64 1, i64 2, i64 3>, %[[VECTOR_PH]] ], [ [[VEC_IND_NEXT:%.*]], %[[VECTOR_BODY]] ]
-; TF-NEXT: [[VEC_PHI:%.*]] = phi <4 x i8> [ splat (i8 -1), %[[VECTOR_PH]] ], [ [[TMP9:%.*]], %[[VECTOR_BODY]] ]
-; TF-NEXT: [[TMP0:%.*]] = phi <4 x i1> [ zeroinitializer, %[[VECTOR_PH]] ], [ [[TMP8:%.*]], %[[VECTOR_BODY]] ]
-; TF-NEXT: [[VEC_IND1:%.*]] = phi <4 x i8> [ <i8 0, i8 1, i8 2, i8 3>, %[[VECTOR_PH]] ], [ [[VEC_IND_NEXT2:%.*]], %[[VECTOR_BODY]] ]
-; TF-NEXT: [[TMP1:%.*]] = icmp ule <4 x i64> [[VEC_IND]], [[BROADCAST_SPLAT]]
-; TF-NEXT: [[TMP2:%.*]] = getelementptr inbounds i8, ptr [[A]], i64 [[INDEX]]
-; TF-NEXT: [[WIDE_MASKED_LOAD:%.*]] = call <4 x i8> @llvm.masked.load.v4i8.p0(ptr align 1 [[TMP2]], <4 x i1> [[TMP1]], <4 x i8> poison)
-; TF-NEXT: [[TMP3:%.*]] = icmp eq <4 x i8> [[WIDE_MASKED_LOAD]], splat (i8 42)
-; TF-NEXT: [[TMP4:%.*]] = add <4 x i8> [[VEC_IND1]], splat (i8 10)
-; TF-NEXT: [[TMP5:%.*]] = select <4 x i1> [[TMP1]], <4 x i1> [[TMP3]], <4 x i1> zeroinitializer
-; TF-NEXT: [[TMP6:%.*]] = freeze <4 x i1> [[TMP5]]
-; TF-NEXT: [[TMP7:%.*]] = call i1 @llvm.vector.reduce.or.v4i1(<4 x i1> [[TMP6]])
-; TF-NEXT: [[TMP8]] = select i1 [[TMP7]], <4 x i1> [[TMP5]], <4 x i1> [[TMP0]]
-; TF-NEXT: [[TMP9]] = select i1 [[TMP7]], <4 x i8> [[TMP4]], <4 x i8> [[VEC_PHI]]
-; TF-NEXT: [[INDEX_NEXT]] = add i64 [[INDEX]], 4
-; TF-NEXT: [[VEC_IND_NEXT]] = add <4 x i64> [[VEC_IND]], splat (i64 4)
-; TF-NEXT: [[VEC_IND_NEXT2]] = add <4 x i8> [[VEC_IND1]], splat (i8 4)
-; TF-NEXT: [[TMP10:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
-; TF-NEXT: br i1 [[TMP10]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP16:![0-9]+]]
-; TF: [[MIDDLE_BLOCK]]:
-; TF-NEXT: [[TMP11:%.*]] = call i8 @llvm.experimental.vector.extract.last.active.v4i8(<4 x i8> [[TMP9]], <4 x i1> [[TMP8]], i8 -1)
-; TF-NEXT: br label %[[DONE:.*]]
-; TF: [[DONE]]:
-; TF-NEXT: ret i8 [[TMP11]]
+; CHECK-LABEL: define i8 @findlast_narrow_i8_reduction(
+; CHECK-SAME: ptr [[A:%.*]], i64 [[N:%.*]]) {
+; CHECK-NEXT: [[ENTRY:.*]]:
+; CHECK-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[N]], 4
+; CHECK-NEXT: br i1 [[MIN_ITERS_CHECK]], label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]]
+; CHECK: [[VECTOR_PH]]:
+; CHECK-NEXT: [[N_MOD_VF:%.*]] = urem i64 [[N]], 4
+; CHECK-NEXT: [[N_VEC:%.*]] = sub i64 [[N]], [[N_MOD_VF]]
+; CHECK-NEXT: br label %[[VECTOR_BODY:.*]]
+; CHECK: [[VECTOR_BODY]]:
+; CHECK-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-NEXT: [[VEC_PHI:%.*]] = phi <4 x i8> [ splat (i8 -1), %[[VECTOR_PH]] ], [ [[TMP7:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-NEXT: [[TMP0:%.*]] = phi <4 x i1> [ zeroinitializer, %[[VECTOR_PH]] ], [ [[TMP6:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-NEXT: [[VEC_IND:%.*]] = phi <4 x i8> [ <i8 0, i8 1, i8 2, i8 3>, %[[VECTOR_PH]] ], [ [[VEC_IND_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-NEXT: [[TMP1:%.*]] = getelementptr inbounds i8, ptr [[A]], i64 [[INDEX]]
+; CHECK-NEXT: [[WIDE_LOAD:%.*]] = load <4 x i8>, ptr [[TMP1]], align 1
+; CHECK-NEXT: [[TMP2:%.*]] = icmp eq <4 x i8> [[WIDE_LOAD]], splat (i8 42)
+; CHECK-NEXT: [[TMP3:%.*]] = add <4 x i8> [[VEC_IND]], splat (i8 10)
+; CHECK-NEXT: [[TMP4:%.*]] = freeze <4 x i1> [[TMP2]]
+; CHECK-NEXT: [[TMP5:%.*]] = call i1 @llvm.vector.reduce.or.v4i1(<4 x i1> [[TMP4]])
+; CHECK-NEXT: [[TMP6]] = select i1 [[TMP5]], <4 x i1> [[TMP2]], <4 x i1> [[TMP0]]
+; CHECK-NEXT: [[TMP7]] = select i1 [[TMP5]], <4 x i8> [[TMP3]], <4 x i8> [[VEC_PHI]]
+; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 4
+; CHECK-NEXT: [[VEC_IND_NEXT]] = add <4 x i8> [[VEC_IND]], splat (i8 4)
+; CHECK-NEXT: [[TMP8:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
+; CHECK-NEXT: br i1 [[TMP8]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP23:![0-9]+]]
+; CHECK: [[MIDDLE_BLOCK]]:
+; CHECK-NEXT: [[TMP9:%.*]] = call i8 @llvm.experimental.vector.extract.last.active.v4i8(<4 x i8> [[TMP7]], <4 x i1> [[TMP6]], i8 -1)
+; CHECK-NEXT: [[CMP_N:%.*]] = icmp eq i64 [[N]], [[N_VEC]]
+; CHECK-NEXT: br i1 [[CMP_N]], label %[[DONE:.*]], label %[[SCALAR_PH]]
+; CHECK: [[SCALAR_PH]]:
+; CHECK-NEXT: [[BC_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC]], %[[MIDDLE_BLOCK]] ], [ 0, %[[ENTRY]] ]
+; CHECK-NEXT: [[BC_MERGE_RDX:%.*]] = phi i8 [ [[TMP9]], %[[MIDDLE_BLOCK]] ], [ -1, %[[ENTRY]] ]
+; CHECK-NEXT: br label %[[LOOP:.*]]
+; CHECK: [[LOOP]]:
+; CHECK-NEXT: [[IV:%.*]] = phi i64 [ [[BC_RESUME_VAL]], %[[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], %[[LOOP]] ]
+; CHECK-NEXT: [[RDX:%.*]] = phi i8 [ [[BC_MERGE_RDX]], %[[SCALAR_PH]] ], [ [[SEL:%.*]], %[[LOOP]] ]
+; CHECK-NEXT: [[GEP:%.*]] = getelementptr inbounds i8, ptr [[A]], i64 [[IV]]
+; CHECK-NEXT: [[LOAD:%.*]] = load i8, ptr [[GEP]], align 1
+; CHECK-NEXT: [[CMP:%.*]] = icmp eq i8 [[LOAD]], 42
+; CHECK-NEXT: [[IV_TRUNC:%.*]] = trunc i64 [[IV]] to i8
+; CHECK-NEXT: [[EXPR:%.*]] = add i8 [[IV_TRUNC]], 10
+; CHECK-NEXT: [[SEL]] = select i1 [[CMP]], i8 [[EXPR]], i8 [[RDX]]
+; CHECK-NEXT: [[IV_NEXT]] = add nuw nsw i64 [[IV]], 1
+; CHECK-NEXT: [[EXIT:%.*]] = icmp eq i64 [[IV_NEXT]], [[N]]
+; CHECK-NEXT: br i1 [[EXIT]], label %[[DONE]], label %[[LOOP]], !llvm.loop [[LOOP24:![0-9]+]]
+; CHECK: [[DONE]]:
+; CHECK-NEXT: [[SEL_LCSSA:%.*]] = phi i8 [ [[SEL]], %[[LOOP]] ], [ [[TMP9]], %[[MIDDLE_BLOCK]] ]
+; CHECK-NEXT: ret i8 [[SEL_LCSSA]]
;
entry:
br label %loop
@@ -1113,127 +697,71 @@ done:
; the TrueVal == PhiR case, where the condition must be inverted without
; corrupting the original Cond used later by the AnyOf tracking.
define i64 @findlast_expr_flipped_select_anyof(ptr %a, ptr %b, i64 %rdx.start, i64 %n) {
-; NO-TF-LABEL: define i64 @findlast_expr_flipped_select_anyof(
-; NO-TF-SAME: ptr [[A:%.*]], ptr [[B:%.*]], i64 [[RDX_START:%.*]], i64 [[N:%.*]]) {
-; NO-TF-NEXT: [[ENTRY:.*]]:
-; NO-TF-NEXT: [[TMP0:%.*]] = add i64 [[N]], 1
-; NO-TF-NEXT: [[UMIN:%.*]] = call i64 @llvm.umin.i64(i64 [[N]], i64 1)
-; NO-TF-NEXT: [[TMP1:%.*]] = sub i64 [[TMP0]], [[UMIN]]
-; NO-TF-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[TMP1]], 4
-; NO-TF-NEXT: br i1 [[MIN_ITERS_CHECK]], label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]]
-; NO-TF: [[VECTOR_PH]]:
-; NO-TF-NEXT: [[N_MOD_VF:%.*]] = urem i64 [[TMP1]], 4
-; NO-TF-NEXT: [[N_VEC:%.*]] = sub i64 [[TMP1]], [[N_MOD_VF]]
-; NO-TF-NEXT: [[TMP2:%.*]] = sub i64 [[N]], [[N_VEC]]
-; NO-TF-NEXT: [[BROADCAST_SPLATINSERT:%.*]] = insertelement <4 x i64> poison, i64 [[N]], i64 0
-; NO-TF-NEXT: [[BROADCAST_SPLAT:%.*]] = shufflevector <4 x i64> [[BROADCAST_SPLATINSERT]], <4 x i64> poison, <4 x i32> zeroinitializer
-; NO-TF-NEXT: [[INDUCTION:%.*]] = add nsw <4 x i64> [[BROADCAST_SPLAT]], <i64 0, i64 -1, i64 -2, i64 -3>
-; NO-TF-NEXT: br label %[[VECTOR_BODY:.*]]
-; NO-TF: [[VECTOR_BODY]]:
-; NO-TF-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
-; NO-TF-NEXT: [[VEC_IND:%.*]] = phi <4 x i64> [ [[INDUCTION]], %[[VECTOR_PH]] ], [ [[VEC_IND_NEXT:%.*]], %[[VECTOR_BODY]] ]
-; NO-TF-NEXT: [[VEC_PHI:%.*]] = phi <4 x i64> [ splat (i64 9223372036854775807), %[[VECTOR_PH]] ], [ [[TMP9:%.*]], %[[VECTOR_BODY]] ]
-; NO-TF-NEXT: [[VEC_PHI1:%.*]] = phi <4 x i1> [ zeroinitializer, %[[VECTOR_PH]] ], [ [[TMP10:%.*]], %[[VECTOR_BODY]] ]
-; NO-TF-NEXT: [[OFFSET_IDX:%.*]] = sub i64 [[N]], [[INDEX]]
-; NO-TF-NEXT: [[TMP3:%.*]] = add nsw i64 [[OFFSET_IDX]], -1
-; NO-TF-NEXT: [[TMP4:%.*]] = getelementptr inbounds i64, ptr [[A]], i64 [[TMP3]]
-; NO-TF-NEXT: [[TMP5:%.*]] = getelementptr inbounds i64, ptr [[TMP4]], i64 -3
-; NO-TF-NEXT: [[WIDE_LOAD:%.*]] = load <4 x i64>, ptr [[TMP5]], align 8
-; NO-TF-NEXT: [[REVERSE:%.*]] = shufflevector <4 x i64> [[WIDE_LOAD]], <4 x i64> poison, <4 x i32> <i32 3, i32 2, i32 1, i32 0>
-; NO-TF-NEXT: [[TMP6:%.*]] = getelementptr inbounds i64, ptr [[B]], i64 [[TMP3]]
-; NO-TF-NEXT: [[TMP7:%.*]] = getelementptr inbounds i64, ptr [[TMP6]], i64 -3
-; NO-TF-NEXT: [[WIDE_LOAD2:%.*]] = load <4 x i64>, ptr [[TMP7]], align 8
-; NO-TF-NEXT: [[REVERSE3:%.*]] = shufflevector <4 x i64> [[WIDE_LOAD2]], <4 x i64> poison, <4 x i32> <i32 3, i32 2, i32 1, i32 0>
-; NO-TF-NEXT: [[TMP8:%.*]] = icmp sle <4 x i64> [[REVERSE]], [[REVERSE3]]
-; NO-TF-NEXT: [[TMP9]] = select <4 x i1> [[TMP8]], <4 x i64> [[VEC_IND]], <4 x i64> [[VEC_PHI]]
-; NO-TF-NEXT: [[TMP10]] = or <4 x i1> [[VEC_PHI1]], [[TMP8]]
-; NO-TF-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 4
-; NO-TF-NEXT: [[VEC_IND_NEXT]] = add nsw <4 x i64> [[VEC_IND]], splat (i64 -4)
-; NO-TF-NEXT: [[TMP11:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
-; NO-TF-NEXT: br i1 [[TMP11]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP25:![0-9]+]]
-; NO-TF: [[MIDDLE_BLOCK]]:
-; NO-TF-NEXT: [[TMP12:%.*]] = call i64 @llvm.vector.reduce.smin.v4i64(<4 x i64> [[TMP9]])
-; NO-TF-NEXT: [[TMP13:%.*]] = add nsw i64 [[TMP12]], -1
-; NO-TF-NEXT: [[TMP14:%.*]] = call i1 @llvm.vector.reduce.or.v4i1(<4 x i1> [[TMP10]])
-; NO-TF-NEXT: [[TMP15:%.*]] = freeze i1 [[TMP14]]
-; NO-TF-NEXT: [[RDX_SELECT:%.*]] = select i1 [[TMP15]], i64 [[TMP13]], i64 [[RDX_START]]
-; NO-TF-NEXT: [[CMP_N:%.*]] = icmp eq i64 [[TMP1]], [[N_VEC]]
-; NO-TF-NEXT: br i1 [[CMP_N]], label %[[EXIT:.*]], label %[[SCALAR_PH]]
-; NO-TF: [[SCALAR_PH]]:
-; NO-TF-NEXT: [[BC_RESUME_VAL:%.*]] = phi i64 [ [[TMP2]], %[[MIDDLE_BLOCK]] ], [ [[N]], %[[ENTRY]] ]
-; NO-TF-NEXT: [[BC_MERGE_RDX:%.*]] = phi i64 [ [[RDX_SELECT]], %[[MIDDLE_BLOCK]] ], [ [[RDX_START]], %[[ENTRY]] ]
-; NO-TF-NEXT: br label %[[LOOP:.*]]
-; NO-TF: [[LOOP]]:
-; NO-TF-NEXT: [[IV:%.*]] = phi i64 [ [[IV_NEXT:%.*]], %[[LOOP]] ], [ [[BC_RESUME_VAL]], %[[SCALAR_PH]] ]
-; NO-TF-NEXT: [[RDX:%.*]] = phi i64 [ [[COND:%.*]], %[[LOOP]] ], [ [[BC_MERGE_RDX]], %[[SCALAR_PH]] ]
-; NO-TF-NEXT: [[IV_NEXT]] = add nsw i64 [[IV]], -1
-; NO-TF-NEXT: [[GEP_A_IV:%.*]] = getelementptr inbounds i64, ptr [[A]], i64 [[IV_NEXT]]
-; NO-TF-NEXT: [[LD_A:%.*]] = load i64, ptr [[GEP_A_IV]], align 8
-; NO-TF-NEXT: [[GEP_B_IV:%.*]] = getelementptr inbounds i64, ptr [[B]], i64 [[IV_NEXT]]
-; NO-TF-NEXT: [[LD_B:%.*]] = load i64, ptr [[GEP_B_IV]], align 8
-; NO-TF-NEXT: [[CMP_A_B:%.*]] = icmp sgt i64 [[LD_A]], [[LD_B]]
-; NO-TF-NEXT: [[COND]] = select i1 [[CMP_A_B]], i64 [[RDX]], i64 [[IV_NEXT]]
-; NO-TF-NEXT: [[EXIT_COND:%.*]] = icmp ugt i64 [[IV]], 1
-; NO-TF-NEXT: br i1 [[EXIT_COND]], label %[[LOOP]], label %[[EXIT]], !llvm.loop [[LOOP26:![0-9]+]]
-; NO-TF: [[EXIT]]:
-; NO-TF-NEXT: [[COND_LCSSA:%.*]] = phi i64 [ [[COND]], %[[LOOP]] ], [ [[RDX_SELECT]], %[[MIDDLE_BLOCK]] ]
-; NO-TF-NEXT: ret i64 [[COND_LCSSA]]
-;
-; TF-LABEL: define i64 @findlast_expr_flipped_select_anyof(
-; TF-SAME: ptr [[A:%.*]], ptr [[B:%.*]], i64 [[RDX_START:%.*]], i64 [[N:%.*]]) {
-; TF-NEXT: [[ENTRY:.*:]]
-; TF-NEXT: [[TMP0:%.*]] = add i64 [[N]], 1
-; TF-NEXT: [[UMIN:%.*]] = call i64 @llvm.umin.i64(i64 [[N]], i64 1)
-; TF-NEXT: [[TMP1:%.*]] = sub i64 [[TMP0]], [[UMIN]]
-; TF-NEXT: br label %[[VECTOR_PH:.*]]
-; TF: [[VECTOR_PH]]:
-; TF-NEXT: [[N_RND_UP:%.*]] = add i64 [[TMP1]], 3
-; TF-NEXT: [[N_MOD_VF:%.*]] = urem i64 [[N_RND_UP]], 4
-; TF-NEXT: [[N_VEC:%.*]] = sub i64 [[N_RND_UP]], [[N_MOD_VF]]
-; TF-NEXT: [[TRIP_COUNT_MINUS_1:%.*]] = sub i64 [[TMP1]], 1
-; TF-NEXT: [[BROADCAST_SPLATINSERT:%.*]] = insertelement <4 x i64> poison, i64 [[TRIP_COUNT_MINUS_1]], i64 0
-; TF-NEXT: [[BROADCAST_SPLAT:%.*]] = shufflevector <4 x i64> [[BROADCAST_SPLATINSERT]], <4 x i64> poison, <4 x i32> zeroinitializer
-; TF-NEXT: [[BROADCAST_SPLATINSERT1:%.*]] = insertelement <4 x i64> poison, i64 [[N]], i64 0
-; TF-NEXT: [[BROADCAST_SPLAT2:%.*]] = shufflevector <4 x i64> [[BROADCAST_SPLATINSERT1]], <4 x i64> poison, <4 x i32> zeroinitializer
-; TF-NEXT: [[INDUCTION:%.*]] = add nsw <4 x i64> [[BROADCAST_SPLAT2]], <i64 0, i64 -1, i64 -2, i64 -3>
-; TF-NEXT: br label %[[VECTOR_BODY:.*]]
-; TF: [[VECTOR_BODY]]:
-; TF-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
-; TF-NEXT: [[VEC_IND:%.*]] = phi <4 x i64> [ [[INDUCTION]], %[[VECTOR_PH]] ], [ [[VEC_IND_NEXT:%.*]], %[[VECTOR_BODY]] ]
-; TF-NEXT: [[VEC_PHI:%.*]] = phi <4 x i64> [ splat (i64 9223372036854775807), %[[VECTOR_PH]] ], [ [[TMP9:%.*]], %[[VECTOR_BODY]] ]
-; TF-NEXT: [[VEC_PHI3:%.*]] = phi <4 x i1> [ zeroinitializer, %[[VECTOR_PH]] ], [ [[TMP10:%.*]], %[[VECTOR_BODY]] ]
-; TF-NEXT: [[OFFSET_IDX:%.*]] = sub i64 [[N]], [[INDEX]]
-; TF-NEXT: [[BROADCAST_SPLATINSERT4:%.*]] = insertelement <4 x i64> poison, i64 [[INDEX]], i64 0
-; TF-NEXT: [[BROADCAST_SPLAT5:%.*]] = shufflevector <4 x i64> [[BROADCAST_SPLATINSERT4]], <4 x i64> poison, <4 x i32> zeroinitializer
-; TF-NEXT: [[VEC_IV:%.*]] = add <4 x i64> [[BROADCAST_SPLAT5]], <i64 0, i64 1, i64 2, i64 3>
-; TF-NEXT: [[TMP2:%.*]] = icmp ule <4 x i64> [[VEC_IV]], [[BROADCAST_SPLAT]]
-; TF-NEXT: [[TMP3:%.*]] = add nsw i64 [[OFFSET_IDX]], -1
-; TF-NEXT: [[TMP4:%.*]] = getelementptr inbounds i64, ptr [[A]], i64 [[TMP3]]
-; TF-NEXT: [[TMP5:%.*]] = getelementptr i64, ptr [[TMP4]], i64 -3
-; TF-NEXT: [[REVERSE:%.*]] = shufflevector <4 x i1> [[TMP2]], <4 x i1> poison, <4 x i32> <i32 3, i32 2, i32 1, i32 0>
-; TF-NEXT: [[WIDE_MASKED_LOAD:%.*]] = call <4 x i64> @llvm.masked.load.v4i64.p0(ptr align 8 [[TMP5]], <4 x i1> [[REVERSE]], <4 x i64> poison)
-; TF-NEXT: [[REVERSE6:%.*]] = shufflevector <4 x i64> [[WIDE_MASKED_LOAD]], <4 x i64> poison, <4 x i32> <i32 3, i32 2, i32 1, i32 0>
-; TF-NEXT: [[TMP6:%.*]] = getelementptr inbounds i64, ptr [[B]], i64 [[TMP3]]
-; TF-NEXT: [[TMP7:%.*]] = getelementptr i64, ptr [[TMP6]], i64 -3
-; TF-NEXT: [[WIDE_MASKED_LOAD8:%.*]] = call <4 x i64> @llvm.masked.load.v4i64.p0(ptr align 8 [[TMP7]], <4 x i1> [[REVERSE]], <4 x i64> poison)
-; TF-NEXT: [[REVERSE9:%.*]] = shufflevector <4 x i64> [[WIDE_MASKED_LOAD8]], <4 x i64> poison, <4 x i32> <i32 3, i32 2, i32 1, i32 0>
-; TF-NEXT: [[TMP16:%.*]] = icmp sle <4 x i64> [[REVERSE6]], [[REVERSE9]]
-; TF-NEXT: [[TMP8:%.*]] = select <4 x i1> [[TMP2]], <4 x i1> [[TMP16]], <4 x i1> zeroinitializer
-; TF-NEXT: [[TMP9]] = select <4 x i1> [[TMP8]], <4 x i64> [[VEC_IND]], <4 x i64> [[VEC_PHI]]
-; TF-NEXT: [[TMP10]] = or <4 x i1> [[VEC_PHI3]], [[TMP8]]
-; TF-NEXT: [[INDEX_NEXT]] = add i64 [[INDEX]], 4
-; TF-NEXT: [[VEC_IND_NEXT]] = add nsw <4 x i64> [[VEC_IND]], splat (i64 -4)
-; TF-NEXT: [[TMP11:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
-; TF-NEXT: br i1 [[TMP11]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP17:![0-9]+]]
-; TF: [[MIDDLE_BLOCK]]:
-; TF-NEXT: [[TMP12:%.*]] = call i64 @llvm.vector.reduce.smin.v4i64(<4 x i64> [[TMP9]])
-; TF-NEXT: [[TMP13:%.*]] = add nsw i64 [[TMP12]], -1
-; TF-NEXT: [[TMP14:%.*]] = call i1 @llvm.vector.reduce.or.v4i1(<4 x i1> [[TMP10]])
-; TF-NEXT: [[TMP15:%.*]] = freeze i1 [[TMP14]]
-; TF-NEXT: [[RDX_SELECT:%.*]] = select i1 [[TMP15]], i64 [[TMP13]], i64 [[RDX_START]]
-; TF-NEXT: br label %[[EXIT:.*]]
-; TF: [[EXIT]]:
-; TF-NEXT: ret i64 [[RDX_SELECT]]
+; CHECK-LABEL: define i64 @findlast_expr_flipped_select_anyof(
+; CHECK-SAME: ptr [[A:%.*]], ptr [[B:%.*]], i64 [[RDX_START:%.*]], i64 [[N:%.*]]) {
+; CHECK-NEXT: [[ENTRY:.*]]:
+; CHECK-NEXT: [[TMP0:%.*]] = add i64 [[N]], 1
+; CHECK-NEXT: [[UMIN:%.*]] = call i64 @llvm.umin.i64(i64 [[N]], i64 1)
+; CHECK-NEXT: [[TMP1:%.*]] = sub i64 [[TMP0]], [[UMIN]]
+; CHECK-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[TMP1]], 4
+; CHECK-NEXT: br i1 [[MIN_ITERS_CHECK]], label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]]
+; CHECK: [[VECTOR_PH]]:
+; CHECK-NEXT: [[N_MOD_VF:%.*]] = urem i64 [[TMP1]], 4
+; CHECK-NEXT: [[N_VEC:%.*]] = sub i64 [[TMP1]], [[N_MOD_VF]]
+; CHECK-NEXT: [[TMP2:%.*]] = sub i64 [[N]], [[N_VEC]]
+; CHECK-NEXT: [[BROADCAST_SPLATINSERT:%.*]] = insertelement <4 x i64> poison, i64 [[N]], i64 0
+; CHECK-NEXT: [[BROADCAST_SPLAT:%.*]] = shufflevector <4 x i64> [[BROADCAST_SPLATINSERT]], <4 x i64> poison, <4 x i32> zeroinitializer
+; CHECK-NEXT: [[INDUCTION:%.*]] = add nsw <4 x i64> [[BROADCAST_SPLAT]], <i64 0, i64 -1, i64 -2, i64 -3>
+; CHECK-NEXT: br label %[[VECTOR_BODY:.*]]
+; CHECK: [[VECTOR_BODY]]:
+; CHECK-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-NEXT: [[TMP13:%.*]] = phi <4 x i64> [ [[INDUCTION]], %[[VECTOR_PH]] ], [ [[VEC_IND_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-NEXT: [[VEC_PHI:%.*]] = phi <4 x i64> [ splat (i64 9223372036854775807), %[[VECTOR_PH]] ], [ [[TMP9:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-NEXT: [[VEC_PHI1:%.*]] = phi <4 x i1> [ zeroinitializer, %[[VECTOR_PH]] ], [ [[TMP10:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-NEXT: [[OFFSET_IDX:%.*]] = sub i64 [[N]], [[INDEX]]
+; CHECK-NEXT: [[TMP3:%.*]] = add nsw i64 [[OFFSET_IDX]], -1
+; CHECK-NEXT: [[TMP4:%.*]] = getelementptr inbounds i64, ptr [[A]], i64 [[TMP3]]
+; CHECK-NEXT: [[TMP5:%.*]] = getelementptr inbounds i64, ptr [[TMP4]], i64 -3
+; CHECK-NEXT: [[WIDE_LOAD:%.*]] = load <4 x i64>, ptr [[TMP5]], align 8
+; CHECK-NEXT: [[REVERSE:%.*]] = shufflevector <4 x i64> [[WIDE_LOAD]], <4 x i64> poison, <4 x i32> <i32 3, i32 2, i32 1, i32 0>
+; CHECK-NEXT: [[TMP6:%.*]] = getelementptr inbounds i64, ptr [[B]], i64 [[TMP3]]
+; CHECK-NEXT: [[TMP7:%.*]] = getelementptr inbounds i64, ptr [[TMP6]], i64 -3
+; CHECK-NEXT: [[WIDE_LOAD2:%.*]] = load <4 x i64>, ptr [[TMP7]], align 8
+; CHECK-NEXT: [[REVERSE3:%.*]] = shufflevector <4 x i64> [[WIDE_LOAD2]], <4 x i64> poison, <4 x i32> <i32 3, i32 2, i32 1, i32 0>
+; CHECK-NEXT: [[TMP8:%.*]] = icmp sle <4 x i64> [[REVERSE]], [[REVERSE3]]
+; CHECK-NEXT: [[TMP9]] = select <4 x i1> [[TMP8]], <4 x i64> [[TMP13]], <4 x i64> [[VEC_PHI]]
+; CHECK-NEXT: [[TMP10]] = or <4 x i1> [[VEC_PHI1]], [[TMP8]]
+; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 4
+; CHECK-NEXT: [[VEC_IND_NEXT]] = add nsw <4 x i64> [[TMP13]], splat (i64 -4)
+; CHECK-NEXT: [[TMP11:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
+; CHECK-NEXT: br i1 [[TMP11]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP25:![0-9]+]]
+; CHECK: [[MIDDLE_BLOCK]]:
+; CHECK-NEXT: [[TMP12:%.*]] = call i64 @llvm.vector.reduce.smin.v4i64(<4 x i64> [[TMP9]])
+; CHECK-NEXT: [[TMP16:%.*]] = add nsw i64 [[TMP12]], -1
+; CHECK-NEXT: [[TMP14:%.*]] = call i1 @llvm.vector.reduce.or.v4i1(<4 x i1> [[TMP10]])
+; CHECK-NEXT: [[TMP15:%.*]] = freeze i1 [[TMP14]]
+; CHECK-NEXT: [[RDX_SELECT:%.*]] = select i1 [[TMP15]], i64 [[TMP16]], i64 [[RDX_START]]
+; CHECK-NEXT: [[CMP_N:%.*]] = icmp eq i64 [[TMP1]], [[N_VEC]]
+; CHECK-NEXT: br i1 [[CMP_N]], label %[[EXIT:.*]], label %[[SCALAR_PH]]
+; CHECK: [[SCALAR_PH]]:
+; CHECK-NEXT: [[BC_RESUME_VAL:%.*]] = phi i64 [ [[TMP2]], %[[MIDDLE_BLOCK]] ], [ [[N]], %[[ENTRY]] ]
+; CHECK-NEXT: [[BC_MERGE_RDX:%.*]] = phi i64 [ [[RDX_SELECT]], %[[MIDDLE_BLOCK]] ], [ [[RDX_START]], %[[ENTRY]] ]
+; CHECK-NEXT: br label %[[LOOP:.*]]
+; CHECK: [[LOOP]]:
+; CHECK-NEXT: [[IV:%.*]] = phi i64 [ [[IV_NEXT:%.*]], %[[LOOP]] ], [ [[BC_RESUME_VAL]], %[[SCALAR_PH]] ]
+; CHECK-NEXT: [[RDX:%.*]] = phi i64 [ [[COND:%.*]], %[[LOOP]] ], [ [[BC_MERGE_RDX]], %[[SCALAR_PH]] ]
+; CHECK-NEXT: [[IV_NEXT]] = add nsw i64 [[IV]], -1
+; CHECK-NEXT: [[GEP_A_IV:%.*]] = getelementptr inbounds i64, ptr [[A]], i64 [[IV_NEXT]]
+; CHECK-NEXT: [[LD_A:%.*]] = load i64, ptr [[GEP_A_IV]], align 8
+; CHECK-NEXT: [[GEP_B_IV:%.*]] = getelementptr inbounds i64, ptr [[B]], i64 [[IV_NEXT]]
+; CHECK-NEXT: [[LD_B:%.*]] = load i64, ptr [[GEP_B_IV]], align 8
+; CHECK-NEXT: [[CMP_A_B:%.*]] = icmp sgt i64 [[LD_A]], [[LD_B]]
+; CHECK-NEXT: [[COND]] = select i1 [[CMP_A_B]], i64 [[RDX]], i64 [[IV_NEXT]]
+; CHECK-NEXT: [[EXIT_COND:%.*]] = icmp ugt i64 [[IV]], 1
+; CHECK-NEXT: br i1 [[EXIT_COND]], label %[[LOOP]], label %[[EXIT]], !llvm.loop [[LOOP26:![0-9]+]]
+; CHECK: [[EXIT]]:
+; CHECK-NEXT: [[COND_LCSSA:%.*]] = phi i64 [ [[COND]], %[[LOOP]] ], [ [[RDX_SELECT]], %[[MIDDLE_BLOCK]] ]
+; CHECK-NEXT: ret i64 [[COND_LCSSA]]
;
entry:
br label %loop
@@ -1259,87 +787,52 @@ exit:
; Test case where the expression %iv steps by one, but %expr is an AddRec
; stepping by -1.
define i64 @findlast_sub_n_expr(ptr %a, i64 %n) {
-; NO-TF-LABEL: define i64 @findlast_sub_n_expr(
-; NO-TF-SAME: ptr [[A:%.*]], i64 [[N:%.*]]) {
-; NO-TF-NEXT: [[ENTRY:.*]]:
-; NO-TF-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[N]], 4
-; NO-TF-NEXT: br i1 [[MIN_ITERS_CHECK]], label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]]
-; NO-TF: [[VECTOR_PH]]:
-; NO-TF-NEXT: [[N_MOD_VF:%.*]] = urem i64 [[N]], 4
-; NO-TF-NEXT: [[N_VEC:%.*]] = sub i64 [[N]], [[N_MOD_VF]]
-; NO-TF-NEXT: br label %[[VECTOR_BODY:.*]]
-; NO-TF: [[VECTOR_BODY]]:
-; NO-TF-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
-; NO-TF-NEXT: [[VEC_IND:%.*]] = phi <4 x i64> [ <i64 0, i64 1, i64 2, i64 3>, %[[VECTOR_PH]] ], [ [[VEC_IND_NEXT:%.*]], %[[VECTOR_BODY]] ]
-; NO-TF-NEXT: [[VEC_PHI:%.*]] = phi <4 x i64> [ splat (i64 -9223372036854775808), %[[VECTOR_PH]] ], [ [[TMP2:%.*]], %[[VECTOR_BODY]] ]
-; NO-TF-NEXT: [[TMP0:%.*]] = getelementptr inbounds i64, ptr [[A]], i64 [[INDEX]]
-; NO-TF-NEXT: [[WIDE_LOAD:%.*]] = load <4 x i64>, ptr [[TMP0]], align 8
-; NO-TF-NEXT: [[TMP1:%.*]] = icmp eq <4 x i64> [[WIDE_LOAD]], splat (i64 42)
-; NO-TF-NEXT: [[TMP2]] = select <4 x i1> [[TMP1]], <4 x i64> [[VEC_IND]], <4 x i64> [[VEC_PHI]]
-; NO-TF-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 4
-; NO-TF-NEXT: [[VEC_IND_NEXT]] = add nuw nsw <4 x i64> [[VEC_IND]], splat (i64 4)
-; NO-TF-NEXT: [[TMP3:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
-; NO-TF-NEXT: br i1 [[TMP3]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP27:![0-9]+]]
-; NO-TF: [[MIDDLE_BLOCK]]:
-; NO-TF-NEXT: [[TMP4:%.*]] = call i64 @llvm.vector.reduce.smax.v4i64(<4 x i64> [[TMP2]])
-; NO-TF-NEXT: [[TMP5:%.*]] = sub i64 [[N]], [[TMP4]]
-; NO-TF-NEXT: [[TMP6:%.*]] = icmp ne i64 [[TMP4]], -9223372036854775808
-; NO-TF-NEXT: [[TMP7:%.*]] = select i1 [[TMP6]], i64 [[TMP5]], i64 -1
-; NO-TF-NEXT: [[CMP_N:%.*]] = icmp eq i64 [[N]], [[N_VEC]]
-; NO-TF-NEXT: br i1 [[CMP_N]], label %[[EXIT:.*]], label %[[SCALAR_PH]]
-; NO-TF: [[SCALAR_PH]]:
-; NO-TF-NEXT: [[BC_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC]], %[[MIDDLE_BLOCK]] ], [ 0, %[[ENTRY]] ]
-; NO-TF-NEXT: [[BC_MERGE_RDX:%.*]] = phi i64 [ [[TMP7]], %[[MIDDLE_BLOCK]] ], [ -1, %[[ENTRY]] ]
-; NO-TF-NEXT: br label %[[LOOP:.*]]
-; NO-TF: [[LOOP]]:
-; NO-TF-NEXT: [[IV:%.*]] = phi i64 [ [[BC_RESUME_VAL]], %[[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], %[[LOOP]] ]
-; NO-TF-NEXT: [[RDX:%.*]] = phi i64 [ [[BC_MERGE_RDX]], %[[SCALAR_PH]] ], [ [[SEL:%.*]], %[[LOOP]] ]
-; NO-TF-NEXT: [[GEP:%.*]] = getelementptr inbounds i64, ptr [[A]], i64 [[IV]]
-; NO-TF-NEXT: [[LOAD:%.*]] = load i64, ptr [[GEP]], align 8
-; NO-TF-NEXT: [[CMP:%.*]] = icmp eq i64 [[LOAD]], 42
-; NO-TF-NEXT: [[EXPR:%.*]] = sub i64 [[N]], [[IV]]
-; NO-TF-NEXT: [[SEL]] = select i1 [[CMP]], i64 [[EXPR]], i64 [[RDX]]
-; NO-TF-NEXT: [[IV_NEXT]] = add nuw nsw i64 [[IV]], 1
-; NO-TF-NEXT: [[EXITCOND:%.*]] = icmp eq i64 [[IV_NEXT]], [[N]]
-; NO-TF-NEXT: br i1 [[EXITCOND]], label %[[EXIT]], label %[[LOOP]], !llvm.loop [[LOOP28:![0-9]+]]
-; NO-TF: [[EXIT]]:
-; NO-TF-NEXT: [[SEL_LCSSA:%.*]] = phi i64 [ [[SEL]], %[[LOOP]] ], [ [[TMP7]], %[[MIDDLE_BLOCK]] ]
-; NO-TF-NEXT: ret i64 [[SEL_LCSSA]]
-;
-; TF-LABEL: define i64 @findlast_sub_n_expr(
-; TF-SAME: ptr [[A:%.*]], i64 [[N:%.*]]) {
-; TF-NEXT: [[ENTRY:.*:]]
-; TF-NEXT: br label %[[VECTOR_PH:.*]]
-; TF: [[VECTOR_PH]]:
-; TF-NEXT: [[N_RND_UP:%.*]] = add i64 [[N]], 3
-; TF-NEXT: [[N_MOD_VF:%.*]] = urem i64 [[N_RND_UP]], 4
-; TF-NEXT: [[N_VEC:%.*]] = sub i64 [[N_RND_UP]], [[N_MOD_VF]]
-; TF-NEXT: [[TRIP_COUNT_MINUS_1:%.*]] = sub i64 [[N]], 1
-; TF-NEXT: [[BROADCAST_SPLATINSERT:%.*]] = insertelement <4 x i64> poison, i64 [[TRIP_COUNT_MINUS_1]], i64 0
-; TF-NEXT: [[BROADCAST_SPLAT:%.*]] = shufflevector <4 x i64> [[BROADCAST_SPLATINSERT]], <4 x i64> poison, <4 x i32> zeroinitializer
-; TF-NEXT: br label %[[VECTOR_BODY:.*]]
-; TF: [[VECTOR_BODY]]:
-; TF-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
-; TF-NEXT: [[VEC_IND:%.*]] = phi <4 x i64> [ <i64 0, i64 1, i64 2, i64 3>, %[[VECTOR_PH]] ], [ [[VEC_IND_NEXT:%.*]], %[[VECTOR_BODY]] ]
-; TF-NEXT: [[VEC_PHI:%.*]] = phi <4 x i64> [ splat (i64 -9223372036854775808), %[[VECTOR_PH]] ], [ [[TMP3:%.*]], %[[VECTOR_BODY]] ]
-; TF-NEXT: [[TMP0:%.*]] = icmp ule <4 x i64> [[VEC_IND]], [[BROADCAST_SPLAT]]
-; TF-NEXT: [[TMP1:%.*]] = getelementptr inbounds i64, ptr [[A]], i64 [[INDEX]]
-; TF-NEXT: [[WIDE_MASKED_LOAD:%.*]] = call <4 x i64> @llvm.masked.load.v4i64.p0(ptr align 8 [[TMP1]], <4 x i1> [[TMP0]], <4 x i64> poison)
-; TF-NEXT: [[TMP2:%.*]] = icmp eq <4 x i64> [[WIDE_MASKED_LOAD]], splat (i64 42)
-; TF-NEXT: [[TMP9:%.*]] = select <4 x i1> [[TMP0]], <4 x i1> [[TMP2]], <4 x i1> zeroinitializer
-; TF-NEXT: [[TMP3]] = select <4 x i1> [[TMP9]], <4 x i64> [[VEC_IND]], <4 x i64> [[VEC_PHI]]
-; TF-NEXT: [[INDEX_NEXT]] = add i64 [[INDEX]], 4
-; TF-NEXT: [[VEC_IND_NEXT]] = add <4 x i64> [[VEC_IND]], splat (i64 4)
-; TF-NEXT: [[TMP4:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
-; TF-NEXT: br i1 [[TMP4]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP18:![0-9]+]]
-; TF: [[MIDDLE_BLOCK]]:
-; TF-NEXT: [[TMP5:%.*]] = call i64 @llvm.vector.reduce.smax.v4i64(<4 x i64> [[TMP3]])
-; TF-NEXT: [[TMP6:%.*]] = sub i64 [[N]], [[TMP5]]
-; TF-NEXT: [[TMP7:%.*]] = icmp ne i64 [[TMP5]], -9223372036854775808
-; TF-NEXT: [[TMP8:%.*]] = select i1 [[TMP7]], i64 [[TMP6]], i64 -1
-; TF-NEXT: br label %[[EXIT:.*]]
-; TF: [[EXIT]]:
-; TF-NEXT: ret i64 [[TMP8]]
+; CHECK-LABEL: define i64 @findlast_sub_n_expr(
+; CHECK-SAME: ptr [[A:%.*]], i64 [[N:%.*]]) {
+; CHECK-NEXT: [[ENTRY:.*]]:
+; CHECK-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[N]], 4
+; CHECK-NEXT: br i1 [[MIN_ITERS_CHECK]], label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]]
+; CHECK: [[VECTOR_PH]]:
+; CHECK-NEXT: [[N_MOD_VF:%.*]] = urem i64 [[N]], 4
+; CHECK-NEXT: [[N_VEC:%.*]] = sub i64 [[N]], [[N_MOD_VF]]
+; CHECK-NEXT: br label %[[VECTOR_BODY:.*]]
+; CHECK: [[VECTOR_BODY]]:
+; CHECK-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-NEXT: [[VEC_IND:%.*]] = phi <4 x i64> [ <i64 0, i64 1, i64 2, i64 3>, %[[VECTOR_PH]] ], [ [[VEC_IND_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-NEXT: [[VEC_PHI:%.*]] = phi <4 x i64> [ splat (i64 -9223372036854775808), %[[VECTOR_PH]] ], [ [[TMP2:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-NEXT: [[TMP0:%.*]] = getelementptr inbounds i64, ptr [[A]], i64 [[INDEX]]
+; CHECK-NEXT: [[WIDE_LOAD:%.*]] = load <4 x i64>, ptr [[TMP0]], align 8
+; CHECK-NEXT: [[TMP1:%.*]] = icmp eq <4 x i64> [[WIDE_LOAD]], splat (i64 42)
+; CHECK-NEXT: [[TMP2]] = select <4 x i1> [[TMP1]], <4 x i64> [[VEC_IND]], <4 x i64> [[VEC_PHI]]
+; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 4
+; CHECK-NEXT: [[VEC_IND_NEXT]] = add nuw nsw <4 x i64> [[VEC_IND]], splat (i64 4)
+; CHECK-NEXT: [[TMP3:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
+; CHECK-NEXT: br i1 [[TMP3]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP27:![0-9]+]]
+; CHECK: [[MIDDLE_BLOCK]]:
+; CHECK-NEXT: [[TMP4:%.*]] = call i64 @llvm.vector.reduce.smax.v4i64(<4 x i64> [[TMP2]])
+; CHECK-NEXT: [[TMP5:%.*]] = sub i64 [[N]], [[TMP4]]
+; CHECK-NEXT: [[TMP6:%.*]] = icmp ne i64 [[TMP4]], -9223372036854775808
+; CHECK-NEXT: [[TMP7:%.*]] = select i1 [[TMP6]], i64 [[TMP5]], i64 -1
+; CHECK-NEXT: [[CMP_N:%.*]] = icmp eq i64 [[N]], [[N_VEC]]
+; CHECK-NEXT: br i1 [[CMP_N]], label %[[EXIT:.*]], label %[[SCALAR_PH]]
+; CHECK: [[SCALAR_PH]]:
+; CHECK-NEXT: [[BC_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC]], %[[MIDDLE_BLOCK]] ], [ 0, %[[ENTRY]] ]
+; CHECK-NEXT: [[BC_MERGE_RDX:%.*]] = phi i64 [ [[TMP7]], %[[MIDDLE_BLOCK]] ], [ -1, %[[ENTRY]] ]
+; CHECK-NEXT: br label %[[LOOP:.*]]
+; CHECK: [[LOOP]]:
+; CHECK-NEXT: [[IV:%.*]] = phi i64 [ [[BC_RESUME_VAL]], %[[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], %[[LOOP]] ]
+; CHECK-NEXT: [[RDX:%.*]] = phi i64 [ [[BC_MERGE_RDX]], %[[SCALAR_PH]] ], [ [[SEL:%.*]], %[[LOOP]] ]
+; CHECK-NEXT: [[GEP:%.*]] = getelementptr inbounds i64, ptr [[A]], i64 [[IV]]
+; CHECK-NEXT: [[LOAD:%.*]] = load i64, ptr [[GEP]], align 8
+; CHECK-NEXT: [[CMP:%.*]] = icmp eq i64 [[LOAD]], 42
+; CHECK-NEXT: [[EXPR:%.*]] = sub i64 [[N]], [[IV]]
+; CHECK-NEXT: [[SEL]] = select i1 [[CMP]], i64 [[EXPR]], i64 [[RDX]]
+; CHECK-NEXT: [[IV_NEXT]] = add nuw nsw i64 [[IV]], 1
+; CHECK-NEXT: [[EXITCOND:%.*]] = icmp eq i64 [[IV_NEXT]], [[N]]
+; CHECK-NEXT: br i1 [[EXITCOND]], label %[[EXIT]], label %[[LOOP]], !llvm.loop [[LOOP28:![0-9]+]]
+; CHECK: [[EXIT]]:
+; CHECK-NEXT: [[SEL_LCSSA:%.*]] = phi i64 [ [[SEL]], %[[LOOP]] ], [ [[TMP7]], %[[MIDDLE_BLOCK]] ]
+; CHECK-NEXT: ret i64 [[SEL_LCSSA]]
;
entry:
br label %loop
@@ -1363,87 +856,52 @@ exit:
; Test sub where IV is the first operand (non-commutative). Verifies that
; operand order is preserved when sinking the expression to the middle block.
define i64 @findlast_sub_iv_as_lhs(ptr %a, i64 %n) {
-; NO-TF-LABEL: define i64 @findlast_sub_iv_as_lhs(
-; NO-TF-SAME: ptr [[A:%.*]], i64 [[N:%.*]]) {
-; NO-TF-NEXT: [[ENTRY:.*]]:
-; NO-TF-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[N]], 4
-; NO-TF-NEXT: br i1 [[MIN_ITERS_CHECK]], label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]]
-; NO-TF: [[VECTOR_PH]]:
-; NO-TF-NEXT: [[N_MOD_VF:%.*]] = urem i64 [[N]], 4
-; NO-TF-NEXT: [[N_VEC:%.*]] = sub i64 [[N]], [[N_MOD_VF]]
-; NO-TF-NEXT: br label %[[VECTOR_BODY:.*]]
-; NO-TF: [[VECTOR_BODY]]:
-; NO-TF-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
-; NO-TF-NEXT: [[VEC_IND:%.*]] = phi <4 x i64> [ <i64 0, i64 1, i64 2, i64 3>, %[[VECTOR_PH]] ], [ [[VEC_IND_NEXT:%.*]], %[[VECTOR_BODY]] ]
-; NO-TF-NEXT: [[VEC_PHI:%.*]] = phi <4 x i64> [ splat (i64 -9223372036854775808), %[[VECTOR_PH]] ], [ [[TMP2:%.*]], %[[VECTOR_BODY]] ]
-; NO-TF-NEXT: [[TMP0:%.*]] = getelementptr inbounds i64, ptr [[A]], i64 [[INDEX]]
-; NO-TF-NEXT: [[WIDE_LOAD:%.*]] = load <4 x i64>, ptr [[TMP0]], align 8
-; NO-TF-NEXT: [[TMP1:%.*]] = icmp eq <4 x i64> [[WIDE_LOAD]], splat (i64 42)
-; NO-TF-NEXT: [[TMP2]] = select <4 x i1> [[TMP1]], <4 x i64> [[VEC_IND]], <4 x i64> [[VEC_PHI]]
-; NO-TF-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 4
-; NO-TF-NEXT: [[VEC_IND_NEXT]] = add nuw nsw <4 x i64> [[VEC_IND]], splat (i64 4)
-; NO-TF-NEXT: [[TMP3:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
-; NO-TF-NEXT: br i1 [[TMP3]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP29:![0-9]+]]
-; NO-TF: [[MIDDLE_BLOCK]]:
-; NO-TF-NEXT: [[TMP4:%.*]] = call i64 @llvm.vector.reduce.smax.v4i64(<4 x i64> [[TMP2]])
-; NO-TF-NEXT: [[TMP5:%.*]] = sub i64 [[TMP4]], [[N]]
-; NO-TF-NEXT: [[TMP6:%.*]] = icmp ne i64 [[TMP4]], -9223372036854775808
-; NO-TF-NEXT: [[TMP7:%.*]] = select i1 [[TMP6]], i64 [[TMP5]], i64 -1
-; NO-TF-NEXT: [[CMP_N:%.*]] = icmp eq i64 [[N]], [[N_VEC]]
-; NO-TF-NEXT: br i1 [[CMP_N]], label %[[EXIT:.*]], label %[[SCALAR_PH]]
-; NO-TF: [[SCALAR_PH]]:
-; NO-TF-NEXT: [[BC_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC]], %[[MIDDLE_BLOCK]] ], [ 0, %[[ENTRY]] ]
-; NO-TF-NEXT: [[BC_MERGE_RDX:%.*]] = phi i64 [ [[TMP7]], %[[MIDDLE_BLOCK]] ], [ -1, %[[ENTRY]] ]
-; NO-TF-NEXT: br label %[[LOOP:.*]]
-; NO-TF: [[LOOP]]:
-; NO-TF-NEXT: [[IV:%.*]] = phi i64 [ [[BC_RESUME_VAL]], %[[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], %[[LOOP]] ]
-; NO-TF-NEXT: [[RDX:%.*]] = phi i64 [ [[BC_MERGE_RDX]], %[[SCALAR_PH]] ], [ [[SEL:%.*]], %[[LOOP]] ]
-; NO-TF-NEXT: [[GEP:%.*]] = getelementptr inbounds i64, ptr [[A]], i64 [[IV]]
-; NO-TF-NEXT: [[LOAD:%.*]] = load i64, ptr [[GEP]], align 8
-; NO-TF-NEXT: [[CMP:%.*]] = icmp eq i64 [[LOAD]], 42
-; NO-TF-NEXT: [[EXPR:%.*]] = sub i64 [[IV]], [[N]]
-; NO-TF-NEXT: [[SEL]] = select i1 [[CMP]], i64 [[EXPR]], i64 [[RDX]]
-; NO-TF-NEXT: [[IV_NEXT]] = add nuw nsw i64 [[IV]], 1
-; NO-TF-NEXT: [[EXITCOND:%.*]] = icmp eq i64 [[IV_NEXT]], [[N]]
-; NO-TF-NEXT: br i1 [[EXITCOND]], label %[[EXIT]], label %[[LOOP]], !llvm.loop [[LOOP30:![0-9]+]]
-; NO-TF: [[EXIT]]:
-; NO-TF-NEXT: [[SEL_LCSSA:%.*]] = phi i64 [ [[SEL]], %[[LOOP]] ], [ [[TMP7]], %[[MIDDLE_BLOCK]] ]
-; NO-TF-NEXT: ret i64 [[SEL_LCSSA]]
-;
-; TF-LABEL: define i64 @findlast_sub_iv_as_lhs(
-; TF-SAME: ptr [[A:%.*]], i64 [[N:%.*]]) {
-; TF-NEXT: [[ENTRY:.*:]]
-; TF-NEXT: br label %[[VECTOR_PH:.*]]
-; TF: [[VECTOR_PH]]:
-; TF-NEXT: [[N_RND_UP:%.*]] = add i64 [[N]], 3
-; TF-NEXT: [[N_MOD_VF:%.*]] = urem i64 [[N_RND_UP]], 4
-; TF-NEXT: [[N_VEC:%.*]] = sub i64 [[N_RND_UP]], [[N_MOD_VF]]
-; TF-NEXT: [[TRIP_COUNT_MINUS_1:%.*]] = sub i64 [[N]], 1
-; TF-NEXT: [[BROADCAST_SPLATINSERT:%.*]] = insertelement <4 x i64> poison, i64 [[TRIP_COUNT_MINUS_1]], i64 0
-; TF-NEXT: [[BROADCAST_SPLAT:%.*]] = shufflevector <4 x i64> [[BROADCAST_SPLATINSERT]], <4 x i64> poison, <4 x i32> zeroinitializer
-; TF-NEXT: br label %[[VECTOR_BODY:.*]]
-; TF: [[VECTOR_BODY]]:
-; TF-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
-; TF-NEXT: [[VEC_IND:%.*]] = phi <4 x i64> [ <i64 0, i64 1, i64 2, i64 3>, %[[VECTOR_PH]] ], [ [[VEC_IND_NEXT:%.*]], %[[VECTOR_BODY]] ]
-; TF-NEXT: [[VEC_PHI:%.*]] = phi <4 x i64> [ splat (i64 -9223372036854775808), %[[VECTOR_PH]] ], [ [[TMP3:%.*]], %[[VECTOR_BODY]] ]
-; TF-NEXT: [[TMP0:%.*]] = icmp ule <4 x i64> [[VEC_IND]], [[BROADCAST_SPLAT]]
-; TF-NEXT: [[TMP1:%.*]] = getelementptr inbounds i64, ptr [[A]], i64 [[INDEX]]
-; TF-NEXT: [[WIDE_MASKED_LOAD:%.*]] = call <4 x i64> @llvm.masked.load.v4i64.p0(ptr align 8 [[TMP1]], <4 x i1> [[TMP0]], <4 x i64> poison)
-; TF-NEXT: [[TMP2:%.*]] = icmp eq <4 x i64> [[WIDE_MASKED_LOAD]], splat (i64 42)
-; TF-NEXT: [[TMP9:%.*]] = select <4 x i1> [[TMP0]], <4 x i1> [[TMP2]], <4 x i1> zeroinitializer
-; TF-NEXT: [[TMP3]] = select <4 x i1> [[TMP9]], <4 x i64> [[VEC_IND]], <4 x i64> [[VEC_PHI]]
-; TF-NEXT: [[INDEX_NEXT]] = add i64 [[INDEX]], 4
-; TF-NEXT: [[VEC_IND_NEXT]] = add <4 x i64> [[VEC_IND]], splat (i64 4)
-; TF-NEXT: [[TMP4:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
-; TF-NEXT: br i1 [[TMP4]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP19:![0-9]+]]
-; TF: [[MIDDLE_BLOCK]]:
-; TF-NEXT: [[TMP5:%.*]] = call i64 @llvm.vector.reduce.smax.v4i64(<4 x i64> [[TMP3]])
-; TF-NEXT: [[TMP6:%.*]] = sub i64 [[TMP5]], [[N]]
-; TF-NEXT: [[TMP7:%.*]] = icmp ne i64 [[TMP5]], -9223372036854775808
-; TF-NEXT: [[TMP8:%.*]] = select i1 [[TMP7]], i64 [[TMP6]], i64 -1
-; TF-NEXT: br label %[[EXIT:.*]]
-; TF: [[EXIT]]:
-; TF-NEXT: ret i64 [[TMP8]]
+; CHECK-LABEL: define i64 @findlast_sub_iv_as_lhs(
+; CHECK-SAME: ptr [[A:%.*]], i64 [[N:%.*]]) {
+; CHECK-NEXT: [[ENTRY:.*]]:
+; CHECK-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[N]], 4
+; CHECK-NEXT: br i1 [[MIN_ITERS_CHECK]], label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]]
+; CHECK: [[VECTOR_PH]]:
+; CHECK-NEXT: [[N_MOD_VF:%.*]] = urem i64 [[N]], 4
+; CHECK-NEXT: [[N_VEC:%.*]] = sub i64 [[N]], [[N_MOD_VF]]
+; CHECK-NEXT: br label %[[VECTOR_BODY:.*]]
+; CHECK: [[VECTOR_BODY]]:
+; CHECK-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-NEXT: [[VEC_IND:%.*]] = phi <4 x i64> [ <i64 0, i64 1, i64 2, i64 3>, %[[VECTOR_PH]] ], [ [[VEC_IND_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-NEXT: [[VEC_PHI:%.*]] = phi <4 x i64> [ splat (i64 -9223372036854775808), %[[VECTOR_PH]] ], [ [[TMP2:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-NEXT: [[TMP0:%.*]] = getelementptr inbounds i64, ptr [[A]], i64 [[INDEX]]
+; CHECK-NEXT: [[WIDE_LOAD:%.*]] = load <4 x i64>, ptr [[TMP0]], align 8
+; CHECK-NEXT: [[TMP1:%.*]] = icmp eq <4 x i64> [[WIDE_LOAD]], splat (i64 42)
+; CHECK-NEXT: [[TMP2]] = select <4 x i1> [[TMP1]], <4 x i64> [[VEC_IND]], <4 x i64> [[VEC_PHI]]
+; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 4
+; CHECK-NEXT: [[VEC_IND_NEXT]] = add nuw nsw <4 x i64> [[VEC_IND]], splat (i64 4)
+; CHECK-NEXT: [[TMP3:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
+; CHECK-NEXT: br i1 [[TMP3]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP29:![0-9]+]]
+; CHECK: [[MIDDLE_BLOCK]]:
+; CHECK-NEXT: [[TMP4:%.*]] = call i64 @llvm.vector.reduce.smax.v4i64(<4 x i64> [[TMP2]])
+; CHECK-NEXT: [[TMP5:%.*]] = sub i64 [[TMP4]], [[N]]
+; CHECK-NEXT: [[TMP6:%.*]] = icmp ne i64 [[TMP4]], -9223372036854775808
+; CHECK-NEXT: [[TMP7:%.*]] = select i1 [[TMP6]], i64 [[TMP5]], i64 -1
+; CHECK-NEXT: [[CMP_N:%.*]] = icmp eq i64 [[N]], [[N_VEC]]
+; CHECK-NEXT: br i1 [[CMP_N]], label %[[EXIT:.*]], label %[[SCALAR_PH]]
+; CHECK: [[SCALAR_PH]]:
+; CHECK-NEXT: [[BC_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC]], %[[MIDDLE_BLOCK]] ], [ 0, %[[ENTRY]] ]
+; CHECK-NEXT: [[BC_MERGE_RDX:%.*]] = phi i64 [ [[TMP7]], %[[MIDDLE_BLOCK]] ], [ -1, %[[ENTRY]] ]
+; CHECK-NEXT: br label %[[LOOP:.*]]
+; CHECK: [[LOOP]]:
+; CHECK-NEXT: [[IV:%.*]] = phi i64 [ [[BC_RESUME_VAL]], %[[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], %[[LOOP]] ]
+; CHECK-NEXT: [[RDX:%.*]] = phi i64 [ [[BC_MERGE_RDX]], %[[SCALAR_PH]] ], [ [[SEL:%.*]], %[[LOOP]] ]
+; CHECK-NEXT: [[GEP:%.*]] = getelementptr inbounds i64, ptr [[A]], i64 [[IV]]
+; CHECK-NEXT: [[LOAD:%.*]] = load i64, ptr [[GEP]], align 8
+; CHECK-NEXT: [[CMP:%.*]] = icmp eq i64 [[LOAD]], 42
+; CHECK-NEXT: [[EXPR:%.*]] = sub i64 [[IV]], [[N]]
+; CHECK-NEXT: [[SEL]] = select i1 [[CMP]], i64 [[EXPR]], i64 [[RDX]]
+; CHECK-NEXT: [[IV_NEXT]] = add nuw nsw i64 [[IV]], 1
+; CHECK-NEXT: [[EXITCOND:%.*]] = icmp eq i64 [[IV_NEXT]], [[N]]
+; CHECK-NEXT: br i1 [[EXITCOND]], label %[[EXIT]], label %[[LOOP]], !llvm.loop [[LOOP30:![0-9]+]]
+; CHECK: [[EXIT]]:
+; CHECK-NEXT: [[SEL_LCSSA:%.*]] = phi i64 [ [[SEL]], %[[LOOP]] ], [ [[TMP7]], %[[MIDDLE_BLOCK]] ]
+; CHECK-NEXT: ret i64 [[SEL_LCSSA]]
;
entry:
br label %loop
@@ -1468,100 +926,57 @@ exit:
; fails and an unsigned sentinel = 0 is used. The sunk udiv in the middle block
; would divide by 0 if the condition is never true.
define i64 @findlast_udiv_may_trap_due_to_sentinel(ptr %a, i64 %n) {
-; NO-TF-LABEL: define i64 @findlast_udiv_may_trap_due_to_sentinel(
-; NO-TF-SAME: ptr [[A:%.*]], i64 [[N:%.*]]) {
-; NO-TF-NEXT: [[ENTRY:.*]]:
-; NO-TF-NEXT: [[TMP0:%.*]] = add i64 [[N]], -9223372036854775808
-; NO-TF-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[TMP0]], 4
-; NO-TF-NEXT: br i1 [[MIN_ITERS_CHECK]], label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]]
-; NO-TF: [[VECTOR_PH]]:
-; NO-TF-NEXT: [[N_MOD_VF:%.*]] = urem i64 [[TMP0]], 4
-; NO-TF-NEXT: [[N_VEC:%.*]] = sub i64 [[TMP0]], [[N_MOD_VF]]
-; NO-TF-NEXT: [[TMP1:%.*]] = add i64 -9223372036854775808, [[N_VEC]]
-; NO-TF-NEXT: br label %[[VECTOR_BODY:.*]]
-; NO-TF: [[VECTOR_BODY]]:
-; NO-TF-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
-; NO-TF-NEXT: [[VEC_IND:%.*]] = phi <4 x i64> [ <i64 -9223372036854775808, i64 -9223372036854775807, i64 -9223372036854775806, i64 -9223372036854775805>, %[[VECTOR_PH]] ], [ [[VEC_IND_NEXT:%.*]], %[[VECTOR_BODY]] ]
-; NO-TF-NEXT: [[VEC_PHI:%.*]] = phi <4 x i64> [ splat (i64 -1), %[[VECTOR_PH]] ], [ [[TMP9:%.*]], %[[VECTOR_BODY]] ]
-; NO-TF-NEXT: [[TMP2:%.*]] = phi <4 x i1> [ zeroinitializer, %[[VECTOR_PH]] ], [ [[TMP8:%.*]], %[[VECTOR_BODY]] ]
-; NO-TF-NEXT: [[OFFSET_IDX:%.*]] = add i64 -9223372036854775808, [[INDEX]]
-; NO-TF-NEXT: [[TMP3:%.*]] = getelementptr inbounds i64, ptr [[A]], i64 [[OFFSET_IDX]]
-; NO-TF-NEXT: [[WIDE_LOAD:%.*]] = load <4 x i64>, ptr [[TMP3]], align 8
-; NO-TF-NEXT: [[TMP4:%.*]] = icmp eq <4 x i64> [[WIDE_LOAD]], splat (i64 42)
-; NO-TF-NEXT: [[TMP5:%.*]] = udiv <4 x i64> splat (i64 100), [[VEC_IND]]
-; NO-TF-NEXT: [[TMP6:%.*]] = freeze <4 x i1> [[TMP4]]
-; NO-TF-NEXT: [[TMP7:%.*]] = call i1 @llvm.vector.reduce.or.v4i1(<4 x i1> [[TMP6]])
-; NO-TF-NEXT: [[TMP8]] = select i1 [[TMP7]], <4 x i1> [[TMP4]], <4 x i1> [[TMP2]]
-; NO-TF-NEXT: [[TMP9]] = select i1 [[TMP7]], <4 x i64> [[TMP5]], <4 x i64> [[VEC_PHI]]
-; NO-TF-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 4
-; NO-TF-NEXT: [[VEC_IND_NEXT]] = add nuw <4 x i64> [[VEC_IND]], splat (i64 4)
-; NO-TF-NEXT: [[TMP10:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
-; NO-TF-NEXT: br i1 [[TMP10]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP31:![0-9]+]]
-; NO-TF: [[MIDDLE_BLOCK]]:
-; NO-TF-NEXT: [[TMP11:%.*]] = call i64 @llvm.experimental.vector.extract.last.active.v4i64(<4 x i64> [[TMP9]], <4 x i1> [[TMP8]], i64 -1)
-; NO-TF-NEXT: [[CMP_N:%.*]] = icmp eq i64 [[TMP0]], [[N_VEC]]
-; NO-TF-NEXT: br i1 [[CMP_N]], label %[[DONE:.*]], label %[[SCALAR_PH]]
-; NO-TF: [[SCALAR_PH]]:
-; NO-TF-NEXT: [[BC_RESUME_VAL:%.*]] = phi i64 [ [[TMP1]], %[[MIDDLE_BLOCK]] ], [ -9223372036854775808, %[[ENTRY]] ]
-; NO-TF-NEXT: [[BC_MERGE_RDX:%.*]] = phi i64 [ [[TMP11]], %[[MIDDLE_BLOCK]] ], [ -1, %[[ENTRY]] ]
-; NO-TF-NEXT: br label %[[LOOP:.*]]
-; NO-TF: [[LOOP]]:
-; NO-TF-NEXT: [[IV:%.*]] = phi i64 [ [[BC_RESUME_VAL]], %[[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], %[[LOOP]] ]
-; NO-TF-NEXT: [[RDX:%.*]] = phi i64 [ [[BC_MERGE_RDX]], %[[SCALAR_PH]] ], [ [[SEL:%.*]], %[[LOOP]] ]
-; NO-TF-NEXT: [[GEP:%.*]] = getelementptr inbounds i64, ptr [[A]], i64 [[IV]]
-; NO-TF-NEXT: [[LOAD:%.*]] = load i64, ptr [[GEP]], align 8
-; NO-TF-NEXT: [[CMP:%.*]] = icmp eq i64 [[LOAD]], 42
-; NO-TF-NEXT: [[EXPR:%.*]] = udiv i64 100, [[IV]]
-; NO-TF-NEXT: [[SEL]] = select i1 [[CMP]], i64 [[EXPR]], i64 [[RDX]]
-; NO-TF-NEXT: [[IV_NEXT]] = add nuw i64 [[IV]], 1
-; NO-TF-NEXT: [[EXIT:%.*]] = icmp eq i64 [[IV_NEXT]], [[N]]
-; NO-TF-NEXT: br i1 [[EXIT]], label %[[DONE]], label %[[LOOP]], !llvm.loop [[LOOP32:![0-9]+]]
-; NO-TF: [[DONE]]:
-; NO-TF-NEXT: [[SEL_LCSSA:%.*]] = phi i64 [ [[SEL]], %[[LOOP]] ], [ [[TMP11]], %[[MIDDLE_BLOCK]] ]
-; NO-TF-NEXT: ret i64 [[SEL_LCSSA]]
-;
-; TF-LABEL: define i64 @findlast_udiv_may_trap_due_to_sentinel(
-; TF-SAME: ptr [[A:%.*]], i64 [[N:%.*]]) {
-; TF-NEXT: [[ENTRY:.*:]]
-; TF-NEXT: [[TMP0:%.*]] = add i64 [[N]], -9223372036854775808
-; TF-NEXT: br label %[[VECTOR_PH:.*]]
-; TF: [[VECTOR_PH]]:
-; TF-NEXT: [[N_RND_UP:%.*]] = add i64 [[TMP0]], 3
-; TF-NEXT: [[N_MOD_VF:%.*]] = urem i64 [[N_RND_UP]], 4
-; TF-NEXT: [[N_VEC:%.*]] = sub i64 [[N_RND_UP]], [[N_MOD_VF]]
-; TF-NEXT: [[TRIP_COUNT_MINUS_1:%.*]] = sub i64 [[TMP0]], 1
-; TF-NEXT: [[BROADCAST_SPLATINSERT:%.*]] = insertelement <4 x i64> poison, i64 [[TRIP_COUNT_MINUS_1]], i64 0
-; TF-NEXT: [[BROADCAST_SPLAT:%.*]] = shufflevector <4 x i64> [[BROADCAST_SPLATINSERT]], <4 x i64> poison, <4 x i32> zeroinitializer
-; TF-NEXT: br label %[[VECTOR_BODY:.*]]
-; TF: [[VECTOR_BODY]]:
-; TF-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
-; TF-NEXT: [[VEC_IND:%.*]] = phi <4 x i64> [ <i64 -9223372036854775808, i64 -9223372036854775807, i64 -9223372036854775806, i64 -9223372036854775805>, %[[VECTOR_PH]] ], [ [[VEC_IND_NEXT:%.*]], %[[VECTOR_BODY]] ]
-; TF-NEXT: [[VEC_PHI:%.*]] = phi <4 x i64> [ splat (i64 -1), %[[VECTOR_PH]] ], [ [[TMP11:%.*]], %[[VECTOR_BODY]] ]
-; TF-NEXT: [[TMP1:%.*]] = phi <4 x i1> [ zeroinitializer, %[[VECTOR_PH]] ], [ [[TMP10:%.*]], %[[VECTOR_BODY]] ]
-; TF-NEXT: [[OFFSET_IDX:%.*]] = add i64 -9223372036854775808, [[INDEX]]
-; TF-NEXT: [[BROADCAST_SPLATINSERT1:%.*]] = insertelement <4 x i64> poison, i64 [[INDEX]], i64 0
-; TF-NEXT: [[BROADCAST_SPLAT2:%.*]] = shufflevector <4 x i64> [[BROADCAST_SPLATINSERT1]], <4 x i64> poison, <4 x i32> zeroinitializer
-; TF-NEXT: [[VEC_IV:%.*]] = add <4 x i64> [[BROADCAST_SPLAT2]], <i64 0, i64 1, i64 2, i64 3>
-; TF-NEXT: [[TMP2:%.*]] = icmp ule <4 x i64> [[VEC_IV]], [[BROADCAST_SPLAT]]
-; TF-NEXT: [[TMP3:%.*]] = getelementptr inbounds i64, ptr [[A]], i64 [[OFFSET_IDX]]
-; TF-NEXT: [[WIDE_MASKED_LOAD:%.*]] = call <4 x i64> @llvm.masked.load.v4i64.p0(ptr align 8 [[TMP3]], <4 x i1> [[TMP2]], <4 x i64> poison)
-; TF-NEXT: [[TMP4:%.*]] = icmp eq <4 x i64> [[WIDE_MASKED_LOAD]], splat (i64 42)
-; TF-NEXT: [[TMP5:%.*]] = select <4 x i1> [[TMP2]], <4 x i64> [[VEC_IND]], <4 x i64> splat (i64 1)
-; TF-NEXT: [[TMP6:%.*]] = udiv <4 x i64> splat (i64 100), [[TMP5]]
-; TF-NEXT: [[TMP7:%.*]] = select <4 x i1> [[TMP2]], <4 x i1> [[TMP4]], <4 x i1> zeroinitializer
-; TF-NEXT: [[TMP8:%.*]] = freeze <4 x i1> [[TMP7]]
-; TF-NEXT: [[TMP9:%.*]] = call i1 @llvm.vector.reduce.or.v4i1(<4 x i1> [[TMP8]])
-; TF-NEXT: [[TMP10]] = select i1 [[TMP9]], <4 x i1> [[TMP7]], <4 x i1> [[TMP1]]
-; TF-NEXT: [[TMP11]] = select i1 [[TMP9]], <4 x i64> [[TMP6]], <4 x i64> [[VEC_PHI]]
-; TF-NEXT: [[INDEX_NEXT]] = add i64 [[INDEX]], 4
-; TF-NEXT: [[VEC_IND_NEXT]] = add nuw <4 x i64> [[VEC_IND]], splat (i64 4)
-; TF-NEXT: [[TMP12:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
-; TF-NEXT: br i1 [[TMP12]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP20:![0-9]+]]
-; TF: [[MIDDLE_BLOCK]]:
-; TF-NEXT: [[TMP13:%.*]] = call i64 @llvm.experimental.vector.extract.last.active.v4i64(<4 x i64> [[TMP11]], <4 x i1> [[TMP10]], i64 -1)
-; TF-NEXT: br label %[[DONE:.*]]
-; TF: [[DONE]]:
-; TF-NEXT: ret i64 [[TMP13]]
+; CHECK-LABEL: define i64 @findlast_udiv_may_trap_due_to_sentinel(
+; CHECK-SAME: ptr [[A:%.*]], i64 [[N:%.*]]) {
+; CHECK-NEXT: [[ENTRY:.*]]:
+; CHECK-NEXT: [[TMP0:%.*]] = add i64 [[N]], -9223372036854775808
+; CHECK-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[TMP0]], 4
+; CHECK-NEXT: br i1 [[MIN_ITERS_CHECK]], label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]]
+; CHECK: [[VECTOR_PH]]:
+; CHECK-NEXT: [[N_MOD_VF:%.*]] = urem i64 [[TMP0]], 4
+; CHECK-NEXT: [[N_VEC:%.*]] = sub i64 [[TMP0]], [[N_MOD_VF]]
+; CHECK-NEXT: [[TMP1:%.*]] = add i64 -9223372036854775808, [[N_VEC]]
+; CHECK-NEXT: br label %[[VECTOR_BODY:.*]]
+; CHECK: [[VECTOR_BODY]]:
+; CHECK-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-NEXT: [[VEC_IND:%.*]] = phi <4 x i64> [ <i64 -9223372036854775808, i64 -9223372036854775807, i64 -9223372036854775806, i64 -9223372036854775805>, %[[VECTOR_PH]] ], [ [[VEC_IND_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-NEXT: [[VEC_PHI:%.*]] = phi <4 x i64> [ splat (i64 -1), %[[VECTOR_PH]] ], [ [[TMP9:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-NEXT: [[TMP4:%.*]] = phi <4 x i1> [ zeroinitializer, %[[VECTOR_PH]] ], [ [[TMP8:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-NEXT: [[OFFSET_IDX:%.*]] = add i64 -9223372036854775808, [[INDEX]]
+; CHECK-NEXT: [[TMP2:%.*]] = getelementptr inbounds i64, ptr [[A]], i64 [[OFFSET_IDX]]
+; CHECK-NEXT: [[WIDE_LOAD:%.*]] = load <4 x i64>, ptr [[TMP2]], align 8
+; CHECK-NEXT: [[TMP3:%.*]] = icmp eq <4 x i64> [[WIDE_LOAD]], splat (i64 42)
+; CHECK-NEXT: [[TMP10:%.*]] = udiv <4 x i64> splat (i64 100), [[VEC_IND]]
+; CHECK-NEXT: [[TMP6:%.*]] = freeze <4 x i1> [[TMP3]]
+; CHECK-NEXT: [[TMP7:%.*]] = call i1 @llvm.vector.reduce.or.v4i1(<4 x i1> [[TMP6]])
+; CHECK-NEXT: [[TMP8]] = select i1 [[TMP7]], <4 x i1> [[TMP3]], <4 x i1> [[TMP4]]
+; CHECK-NEXT: [[TMP9]] = select i1 [[TMP7]], <4 x i64> [[TMP10]], <4 x i64> [[VEC_PHI]]
+; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 4
+; CHECK-NEXT: [[VEC_IND_NEXT]] = add nuw <4 x i64> [[VEC_IND]], splat (i64 4)
+; CHECK-NEXT: [[TMP5:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
+; CHECK-NEXT: br i1 [[TMP5]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP31:![0-9]+]]
+; CHECK: [[MIDDLE_BLOCK]]:
+; CHECK-NEXT: [[TMP11:%.*]] = call i64 @llvm.experimental.vector.extract.last.active.v4i64(<4 x i64> [[TMP9]], <4 x i1> [[TMP8]], i64 -1)
+; CHECK-NEXT: [[CMP_N:%.*]] = icmp eq i64 [[TMP0]], [[N_VEC]]
+; CHECK-NEXT: br i1 [[CMP_N]], label %[[DONE:.*]], label %[[SCALAR_PH]]
+; CHECK: [[SCALAR_PH]]:
+; CHECK-NEXT: [[BC_RESUME_VAL:%.*]] = phi i64 [ [[TMP1]], %[[MIDDLE_BLOCK]] ], [ -9223372036854775808, %[[ENTRY]] ]
+; CHECK-NEXT: [[BC_MERGE_RDX:%.*]] = phi i64 [ [[TMP11]], %[[MIDDLE_BLOCK]] ], [ -1, %[[ENTRY]] ]
+; CHECK-NEXT: br label %[[LOOP:.*]]
+; CHECK: [[LOOP]]:
+; CHECK-NEXT: [[IV:%.*]] = phi i64 [ [[BC_RESUME_VAL]], %[[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], %[[LOOP]] ]
+; CHECK-NEXT: [[RDX:%.*]] = phi i64 [ [[BC_MERGE_RDX]], %[[SCALAR_PH]] ], [ [[SEL:%.*]], %[[LOOP]] ]
+; CHECK-NEXT: [[GEP:%.*]] = getelementptr inbounds i64, ptr [[A]], i64 [[IV]]
+; CHECK-NEXT: [[LOAD:%.*]] = load i64, ptr [[GEP]], align 8
+; CHECK-NEXT: [[CMP:%.*]] = icmp eq i64 [[LOAD]], 42
+; CHECK-NEXT: [[EXPR:%.*]] = udiv i64 100, [[IV]]
+; CHECK-NEXT: [[SEL]] = select i1 [[CMP]], i64 [[EXPR]], i64 [[RDX]]
+; CHECK-NEXT: [[IV_NEXT]] = add nuw i64 [[IV]], 1
+; CHECK-NEXT: [[EXIT:%.*]] = icmp eq i64 [[IV_NEXT]], [[N]]
+; CHECK-NEXT: br i1 [[EXIT]], label %[[DONE]], label %[[LOOP]], !llvm.loop [[LOOP32:![0-9]+]]
+; CHECK: [[DONE]]:
+; CHECK-NEXT: [[SEL_LCSSA:%.*]] = phi i64 [ [[SEL]], %[[LOOP]] ], [ [[TMP11]], %[[MIDDLE_BLOCK]] ]
+; CHECK-NEXT: ret i64 [[SEL_LCSSA]]
;
entry:
br label %loop
@@ -1584,100 +999,57 @@ done:
; srem is not sinkable (may trap), so the expression should not be sunk.
define i64 @findlast_srem_iv_as_divisor(ptr %a, i64 %n) {
-; NO-TF-LABEL: define i64 @findlast_srem_iv_as_divisor(
-; NO-TF-SAME: ptr [[A:%.*]], i64 [[N:%.*]]) {
-; NO-TF-NEXT: [[ENTRY:.*]]:
-; NO-TF-NEXT: [[TMP0:%.*]] = add i64 [[N]], -1
-; NO-TF-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[TMP0]], 4
-; NO-TF-NEXT: br i1 [[MIN_ITERS_CHECK]], label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]]
-; NO-TF: [[VECTOR_PH]]:
-; NO-TF-NEXT: [[N_MOD_VF:%.*]] = urem i64 [[TMP0]], 4
-; NO-TF-NEXT: [[N_VEC:%.*]] = sub i64 [[TMP0]], [[N_MOD_VF]]
-; NO-TF-NEXT: [[TMP1:%.*]] = add i64 1, [[N_VEC]]
-; NO-TF-NEXT: br label %[[VECTOR_BODY:.*]]
-; NO-TF: [[VECTOR_BODY]]:
-; NO-TF-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
-; NO-TF-NEXT: [[VEC_IND:%.*]] = phi <4 x i64> [ <i64 1, i64 2, i64 3, i64 4>, %[[VECTOR_PH]] ], [ [[VEC_IND_NEXT:%.*]], %[[VECTOR_BODY]] ]
-; NO-TF-NEXT: [[VEC_PHI:%.*]] = phi <4 x i64> [ splat (i64 -1), %[[VECTOR_PH]] ], [ [[TMP9:%.*]], %[[VECTOR_BODY]] ]
-; NO-TF-NEXT: [[TMP2:%.*]] = phi <4 x i1> [ zeroinitializer, %[[VECTOR_PH]] ], [ [[TMP8:%.*]], %[[VECTOR_BODY]] ]
-; NO-TF-NEXT: [[OFFSET_IDX:%.*]] = add i64 1, [[INDEX]]
-; NO-TF-NEXT: [[TMP3:%.*]] = getelementptr inbounds i64, ptr [[A]], i64 [[OFFSET_IDX]]
-; NO-TF-NEXT: [[WIDE_LOAD:%.*]] = load <4 x i64>, ptr [[TMP3]], align 8
-; NO-TF-NEXT: [[TMP4:%.*]] = icmp eq <4 x i64> [[WIDE_LOAD]], splat (i64 42)
-; NO-TF-NEXT: [[TMP5:%.*]] = srem <4 x i64> splat (i64 100), [[VEC_IND]]
-; NO-TF-NEXT: [[TMP6:%.*]] = freeze <4 x i1> [[TMP4]]
-; NO-TF-NEXT: [[TMP7:%.*]] = call i1 @llvm.vector.reduce.or.v4i1(<4 x i1> [[TMP6]])
-; NO-TF-NEXT: [[TMP8]] = select i1 [[TMP7]], <4 x i1> [[TMP4]], <4 x i1> [[TMP2]]
-; NO-TF-NEXT: [[TMP9]] = select i1 [[TMP7]], <4 x i64> [[TMP5]], <4 x i64> [[VEC_PHI]]
-; NO-TF-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 4
-; NO-TF-NEXT: [[VEC_IND_NEXT]] = add nuw nsw <4 x i64> [[VEC_IND]], splat (i64 4)
-; NO-TF-NEXT: [[TMP10:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
-; NO-TF-NEXT: br i1 [[TMP10]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP33:![0-9]+]]
-; NO-TF: [[MIDDLE_BLOCK]]:
-; NO-TF-NEXT: [[TMP11:%.*]] = call i64 @llvm.experimental.vector.extract.last.active.v4i64(<4 x i64> [[TMP9]], <4 x i1> [[TMP8]], i64 -1)
-; NO-TF-NEXT: [[CMP_N:%.*]] = icmp eq i64 [[TMP0]], [[N_VEC]]
-; NO-TF-NEXT: br i1 [[CMP_N]], label %[[DONE:.*]], label %[[SCALAR_PH]]
-; NO-TF: [[SCALAR_PH]]:
-; NO-TF-NEXT: [[BC_RESUME_VAL:%.*]] = phi i64 [ [[TMP1]], %[[MIDDLE_BLOCK]] ], [ 1, %[[ENTRY]] ]
-; NO-TF-NEXT: [[BC_MERGE_RDX:%.*]] = phi i64 [ [[TMP11]], %[[MIDDLE_BLOCK]] ], [ -1, %[[ENTRY]] ]
-; NO-TF-NEXT: br label %[[LOOP:.*]]
-; NO-TF: [[LOOP]]:
-; NO-TF-NEXT: [[IV:%.*]] = phi i64 [ [[BC_RESUME_VAL]], %[[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], %[[LOOP]] ]
-; NO-TF-NEXT: [[RDX:%.*]] = phi i64 [ [[BC_MERGE_RDX]], %[[SCALAR_PH]] ], [ [[SEL:%.*]], %[[LOOP]] ]
-; NO-TF-NEXT: [[GEP:%.*]] = getelementptr inbounds i64, ptr [[A]], i64 [[IV]]
-; NO-TF-NEXT: [[LOAD:%.*]] = load i64, ptr [[GEP]], align 8
-; NO-TF-NEXT: [[CMP:%.*]] = icmp eq i64 [[LOAD]], 42
-; NO-TF-NEXT: [[EXPR:%.*]] = srem i64 100, [[IV]]
-; NO-TF-NEXT: [[SEL]] = select i1 [[CMP]], i64 [[EXPR]], i64 [[RDX]]
-; NO-TF-NEXT: [[IV_NEXT]] = add nuw nsw i64 [[IV]], 1
-; NO-TF-NEXT: [[EXIT:%.*]] = icmp eq i64 [[IV_NEXT]], [[N]]
-; NO-TF-NEXT: br i1 [[EXIT]], label %[[DONE]], label %[[LOOP]], !llvm.loop [[LOOP34:![0-9]+]]
-; NO-TF: [[DONE]]:
-; NO-TF-NEXT: [[SEL_LCSSA:%.*]] = phi i64 [ [[SEL]], %[[LOOP]] ], [ [[TMP11]], %[[MIDDLE_BLOCK]] ]
-; NO-TF-NEXT: ret i64 [[SEL_LCSSA]]
-;
-; TF-LABEL: define i64 @findlast_srem_iv_as_divisor(
-; TF-SAME: ptr [[A:%.*]], i64 [[N:%.*]]) {
-; TF-NEXT: [[ENTRY:.*:]]
-; TF-NEXT: [[TMP0:%.*]] = add i64 [[N]], -1
-; TF-NEXT: br label %[[VECTOR_PH:.*]]
-; TF: [[VECTOR_PH]]:
-; TF-NEXT: [[N_RND_UP:%.*]] = add i64 [[TMP0]], 3
-; TF-NEXT: [[N_MOD_VF:%.*]] = urem i64 [[N_RND_UP]], 4
-; TF-NEXT: [[N_VEC:%.*]] = sub i64 [[N_RND_UP]], [[N_MOD_VF]]
-; TF-NEXT: [[TRIP_COUNT_MINUS_1:%.*]] = sub i64 [[TMP0]], 1
-; TF-NEXT: [[BROADCAST_SPLATINSERT:%.*]] = insertelement <4 x i64> poison, i64 [[TRIP_COUNT_MINUS_1]], i64 0
-; TF-NEXT: [[BROADCAST_SPLAT:%.*]] = shufflevector <4 x i64> [[BROADCAST_SPLATINSERT]], <4 x i64> poison, <4 x i32> zeroinitializer
-; TF-NEXT: br label %[[VECTOR_BODY:.*]]
-; TF: [[VECTOR_BODY]]:
-; TF-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
-; TF-NEXT: [[VEC_IND:%.*]] = phi <4 x i64> [ <i64 1, i64 2, i64 3, i64 4>, %[[VECTOR_PH]] ], [ [[VEC_IND_NEXT:%.*]], %[[VECTOR_BODY]] ]
-; TF-NEXT: [[VEC_PHI:%.*]] = phi <4 x i64> [ splat (i64 -1), %[[VECTOR_PH]] ], [ [[TMP11:%.*]], %[[VECTOR_BODY]] ]
-; TF-NEXT: [[TMP1:%.*]] = phi <4 x i1> [ zeroinitializer, %[[VECTOR_PH]] ], [ [[TMP10:%.*]], %[[VECTOR_BODY]] ]
-; TF-NEXT: [[OFFSET_IDX:%.*]] = add i64 1, [[INDEX]]
-; TF-NEXT: [[BROADCAST_SPLATINSERT1:%.*]] = insertelement <4 x i64> poison, i64 [[INDEX]], i64 0
-; TF-NEXT: [[BROADCAST_SPLAT2:%.*]] = shufflevector <4 x i64> [[BROADCAST_SPLATINSERT1]], <4 x i64> poison, <4 x i32> zeroinitializer
-; TF-NEXT: [[VEC_IV:%.*]] = add <4 x i64> [[BROADCAST_SPLAT2]], <i64 0, i64 1, i64 2, i64 3>
-; TF-NEXT: [[TMP2:%.*]] = icmp ule <4 x i64> [[VEC_IV]], [[BROADCAST_SPLAT]]
-; TF-NEXT: [[TMP3:%.*]] = getelementptr inbounds i64, ptr [[A]], i64 [[OFFSET_IDX]]
-; TF-NEXT: [[WIDE_MASKED_LOAD:%.*]] = call <4 x i64> @llvm.masked.load.v4i64.p0(ptr align 8 [[TMP3]], <4 x i1> [[TMP2]], <4 x i64> poison)
-; TF-NEXT: [[TMP4:%.*]] = icmp eq <4 x i64> [[WIDE_MASKED_LOAD]], splat (i64 42)
-; TF-NEXT: [[TMP5:%.*]] = select <4 x i1> [[TMP2]], <4 x i64> [[VEC_IND]], <4 x i64> splat (i64 1)
-; TF-NEXT: [[TMP6:%.*]] = srem <4 x i64> splat (i64 100), [[TMP5]]
-; TF-NEXT: [[TMP7:%.*]] = select <4 x i1> [[TMP2]], <4 x i1> [[TMP4]], <4 x i1> zeroinitializer
-; TF-NEXT: [[TMP8:%.*]] = freeze <4 x i1> [[TMP7]]
-; TF-NEXT: [[TMP9:%.*]] = call i1 @llvm.vector.reduce.or.v4i1(<4 x i1> [[TMP8]])
-; TF-NEXT: [[TMP10]] = select i1 [[TMP9]], <4 x i1> [[TMP7]], <4 x i1> [[TMP1]]
-; TF-NEXT: [[TMP11]] = select i1 [[TMP9]], <4 x i64> [[TMP6]], <4 x i64> [[VEC_PHI]]
-; TF-NEXT: [[INDEX_NEXT]] = add i64 [[INDEX]], 4
-; TF-NEXT: [[VEC_IND_NEXT]] = add nuw nsw <4 x i64> [[VEC_IND]], splat (i64 4)
-; TF-NEXT: [[TMP12:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
-; TF-NEXT: br i1 [[TMP12]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP21:![0-9]+]]
-; TF: [[MIDDLE_BLOCK]]:
-; TF-NEXT: [[TMP13:%.*]] = call i64 @llvm.experimental.vector.extract.last.active.v4i64(<4 x i64> [[TMP11]], <4 x i1> [[TMP10]], i64 -1)
-; TF-NEXT: br label %[[DONE:.*]]
-; TF: [[DONE]]:
-; TF-NEXT: ret i64 [[TMP13]]
+; CHECK-LABEL: define i64 @findlast_srem_iv_as_divisor(
+; CHECK-SAME: ptr [[A:%.*]], i64 [[N:%.*]]) {
+; CHECK-NEXT: [[ENTRY:.*]]:
+; CHECK-NEXT: [[TMP0:%.*]] = add i64 [[N]], -1
+; CHECK-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[TMP0]], 4
+; CHECK-NEXT: br i1 [[MIN_ITERS_CHECK]], label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]]
+; CHECK: [[VECTOR_PH]]:
+; CHECK-NEXT: [[N_MOD_VF:%.*]] = urem i64 [[TMP0]], 4
+; CHECK-NEXT: [[N_VEC:%.*]] = sub i64 [[TMP0]], [[N_MOD_VF]]
+; CHECK-NEXT: [[TMP1:%.*]] = add i64 1, [[N_VEC]]
+; CHECK-NEXT: br label %[[VECTOR_BODY:.*]]
+; CHECK: [[VECTOR_BODY]]:
+; CHECK-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-NEXT: [[VEC_IND:%.*]] = phi <4 x i64> [ <i64 1, i64 2, i64 3, i64 4>, %[[VECTOR_PH]] ], [ [[VEC_IND_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-NEXT: [[VEC_PHI:%.*]] = phi <4 x i64> [ splat (i64 -1), %[[VECTOR_PH]] ], [ [[TMP9:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-NEXT: [[TMP2:%.*]] = phi <4 x i1> [ zeroinitializer, %[[VECTOR_PH]] ], [ [[TMP8:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-NEXT: [[OFFSET_IDX:%.*]] = add i64 1, [[INDEX]]
+; CHECK-NEXT: [[TMP3:%.*]] = getelementptr inbounds i64, ptr [[A]], i64 [[OFFSET_IDX]]
+; CHECK-NEXT: [[WIDE_LOAD:%.*]] = load <4 x i64>, ptr [[TMP3]], align 8
+; CHECK-NEXT: [[TMP4:%.*]] = icmp eq <4 x i64> [[WIDE_LOAD]], splat (i64 42)
+; CHECK-NEXT: [[TMP5:%.*]] = srem <4 x i64> splat (i64 100), [[VEC_IND]]
+; CHECK-NEXT: [[TMP6:%.*]] = freeze <4 x i1> [[TMP4]]
+; CHECK-NEXT: [[TMP7:%.*]] = call i1 @llvm.vector.reduce.or.v4i1(<4 x i1> [[TMP6]])
+; CHECK-NEXT: [[TMP8]] = select i1 [[TMP7]], <4 x i1> [[TMP4]], <4 x i1> [[TMP2]]
+; CHECK-NEXT: [[TMP9]] = select i1 [[TMP7]], <4 x i64> [[TMP5]], <4 x i64> [[VEC_PHI]]
+; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 4
+; CHECK-NEXT: [[VEC_IND_NEXT]] = add nuw nsw <4 x i64> [[VEC_IND]], splat (i64 4)
+; CHECK-NEXT: [[TMP10:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
+; CHECK-NEXT: br i1 [[TMP10]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP33:![0-9]+]]
+; CHECK: [[MIDDLE_BLOCK]]:
+; CHECK-NEXT: [[TMP11:%.*]] = call i64 @llvm.experimental.vector.extract.last.active.v4i64(<4 x i64> [[TMP9]], <4 x i1> [[TMP8]], i64 -1)
+; CHECK-NEXT: [[CMP_N:%.*]] = icmp eq i64 [[TMP0]], [[N_VEC]]
+; CHECK-NEXT: br i1 [[CMP_N]], label %[[DONE:.*]], label %[[SCALAR_PH]]
+; CHECK: [[SCALAR_PH]]:
+; CHECK-NEXT: [[BC_RESUME_VAL:%.*]] = phi i64 [ [[TMP1]], %[[MIDDLE_BLOCK]] ], [ 1, %[[ENTRY]] ]
+; CHECK-NEXT: [[BC_MERGE_RDX:%.*]] = phi i64 [ [[TMP11]], %[[MIDDLE_BLOCK]] ], [ -1, %[[ENTRY]] ]
+; CHECK-NEXT: br label %[[LOOP:.*]]
+; CHECK: [[LOOP]]:
+; CHECK-NEXT: [[IV:%.*]] = phi i64 [ [[BC_RESUME_VAL]], %[[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], %[[LOOP]] ]
+; CHECK-NEXT: [[RDX:%.*]] = phi i64 [ [[BC_MERGE_RDX]], %[[SCALAR_PH]] ], [ [[SEL:%.*]], %[[LOOP]] ]
+; CHECK-NEXT: [[GEP:%.*]] = getelementptr inbounds i64, ptr [[A]], i64 [[IV]]
+; CHECK-NEXT: [[LOAD:%.*]] = load i64, ptr [[GEP]], align 8
+; CHECK-NEXT: [[CMP:%.*]] = icmp eq i64 [[LOAD]], 42
+; CHECK-NEXT: [[EXPR:%.*]] = srem i64 100, [[IV]]
+; CHECK-NEXT: [[SEL]] = select i1 [[CMP]], i64 [[EXPR]], i64 [[RDX]]
+; CHECK-NEXT: [[IV_NEXT]] = add nuw nsw i64 [[IV]], 1
+; CHECK-NEXT: [[EXIT:%.*]] = icmp eq i64 [[IV_NEXT]], [[N]]
+; CHECK-NEXT: br i1 [[EXIT]], label %[[DONE]], label %[[LOOP]], !llvm.loop [[LOOP34:![0-9]+]]
+; CHECK: [[DONE]]:
+; CHECK-NEXT: [[SEL_LCSSA:%.*]] = phi i64 [ [[SEL]], %[[LOOP]] ], [ [[TMP11]], %[[MIDDLE_BLOCK]] ]
+; CHECK-NEXT: ret i64 [[SEL_LCSSA]]
;
entry:
br label %loop
More information about the llvm-commits
mailing list