[llvm-branch-commits] [llvm] [VPlan] Support tail folding uncountable early exits (PR #238024)
Luke Lau via llvm-branch-commits
llvm-branch-commits at lists.llvm.org
Sun Nov 8 21:48:36 PST 2026
https://github.com/lukel97 updated https://github.com/llvm/llvm-project/pull/238024
>From 86866e47868996926e08903ccd1bb238f62e02fa Mon Sep 17 00:00:00 2001
From: Luke Lau <luke at igalia.com>
Date: Mon, 28 Sep 2026 15:34:17 +0100
Subject: [PATCH 1/2] Precommit tests
---
.../LoopVectorize/RISCV/early-exit.ll | 552 ++++++++++++++++++
...arly-exit-tail-folding-active-lane-mask.ll | 65 +++
.../LoopVectorize/early-exit-tail-folding.ll | 524 +++++++++++++++++
3 files changed, 1141 insertions(+)
create mode 100644 llvm/test/Transforms/LoopVectorize/RISCV/early-exit.ll
create mode 100644 llvm/test/Transforms/LoopVectorize/early-exit-tail-folding-active-lane-mask.ll
create mode 100644 llvm/test/Transforms/LoopVectorize/early-exit-tail-folding.ll
diff --git a/llvm/test/Transforms/LoopVectorize/RISCV/early-exit.ll b/llvm/test/Transforms/LoopVectorize/RISCV/early-exit.ll
new file mode 100644
index 00000000000000..1ab2e3461477c8
--- /dev/null
+++ b/llvm/test/Transforms/LoopVectorize/RISCV/early-exit.ll
@@ -0,0 +1,552 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --check-globals none --version 6
+; RUN: opt -S < %s -p loop-vectorize -mtriple riscv64 -mattr=+v -enable-early-exit-vectorization-with-side-effects | FileCheck %s
+
+define i32 @single_early_exit(ptr align 4 dereferenceable(4096) %p) {
+; CHECK-LABEL: define i32 @single_early_exit(
+; CHECK-SAME: ptr align 4 dereferenceable(4096) [[P:%.*]]) #[[ATTR0:[0-9]+]] {
+; CHECK-NEXT: [[ENTRY:.*]]:
+; CHECK-NEXT: [[TMP0:%.*]] = call i32 @llvm.vscale.i32()
+; CHECK-NEXT: [[TMP1:%.*]] = shl nuw i32 [[TMP0]], 2
+; CHECK-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ult i32 123, [[TMP1]]
+; CHECK-NEXT: br i1 [[MIN_ITERS_CHECK]], label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]]
+; CHECK: [[VECTOR_PH]]:
+; CHECK-NEXT: [[N_MOD_VF:%.*]] = urem i32 123, [[TMP1]]
+; CHECK-NEXT: [[N_VEC:%.*]] = sub i32 123, [[N_MOD_VF]]
+; CHECK-NEXT: br label %[[VECTOR_BODY:.*]]
+; CHECK: [[VECTOR_BODY]]:
+; CHECK-NEXT: [[INDEX:%.*]] = phi i32 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY_INTERIM:.*]] ]
+; CHECK-NEXT: [[TMP2:%.*]] = getelementptr inbounds i32, ptr [[P]], i32 [[INDEX]]
+; CHECK-NEXT: [[WIDE_LOAD:%.*]] = load <vscale x 4 x i32>, ptr [[TMP2]], align 4
+; CHECK-NEXT: [[TMP3:%.*]] = icmp eq <vscale x 4 x i32> [[WIDE_LOAD]], zeroinitializer
+; CHECK-NEXT: [[TMP4:%.*]] = freeze <vscale x 4 x i1> [[TMP3]]
+; CHECK-NEXT: [[TMP5:%.*]] = call i1 @llvm.vector.reduce.or.nxv4i1(<vscale x 4 x i1> [[TMP4]])
+; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i32 [[INDEX]], [[TMP1]]
+; CHECK-NEXT: [[TMP6:%.*]] = icmp eq i32 [[INDEX_NEXT]], [[N_VEC]]
+; CHECK-NEXT: br i1 [[TMP5]], label %[[VECTOR_EARLY_EXIT:.*]], label %[[VECTOR_BODY_INTERIM]]
+; CHECK: [[VECTOR_BODY_INTERIM]]:
+; CHECK-NEXT: br i1 [[TMP6]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP0:![0-9]+]]
+; CHECK: [[MIDDLE_BLOCK]]:
+; CHECK-NEXT: [[CMP_N:%.*]] = icmp eq i32 123, [[N_VEC]]
+; CHECK-NEXT: br i1 [[CMP_N]], label %[[EXIT:.*]], label %[[SCALAR_PH]]
+; CHECK: [[VECTOR_EARLY_EXIT]]:
+; CHECK-NEXT: [[TMP7:%.*]] = call i64 @llvm.experimental.cttz.elts.i64.nxv4i1(<vscale x 4 x i1> [[TMP4]], i1 false)
+; CHECK-NEXT: [[TMP8:%.*]] = trunc i64 [[TMP7]] to i32
+; CHECK-NEXT: [[TMP9:%.*]] = add i32 [[INDEX]], [[TMP8]]
+; CHECK-NEXT: br label %[[EXIT]]
+; CHECK: [[SCALAR_PH]]:
+; CHECK-NEXT: [[BC_RESUME_VAL:%.*]] = phi i32 [ [[N_VEC]], %[[MIDDLE_BLOCK]] ], [ 0, %[[ENTRY]] ]
+; CHECK-NEXT: br label %[[LOOP:.*]]
+; CHECK: [[LOOP]]:
+; CHECK-NEXT: [[IV:%.*]] = phi i32 [ [[BC_RESUME_VAL]], %[[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], %[[LATCH:.*]] ]
+; CHECK-NEXT: [[GEP:%.*]] = getelementptr inbounds i32, ptr [[P]], i32 [[IV]]
+; CHECK-NEXT: [[LD:%.*]] = load i32, ptr [[GEP]], align 4
+; CHECK-NEXT: [[C:%.*]] = icmp eq i32 [[LD]], 0
+; CHECK-NEXT: br i1 [[C]], label %[[EXIT]], label %[[LATCH]]
+; CHECK: [[LATCH]]:
+; CHECK-NEXT: [[IV_NEXT]] = add nuw nsw i32 [[IV]], 1
+; CHECK-NEXT: [[EC:%.*]] = icmp eq i32 [[IV_NEXT]], 123
+; CHECK-NEXT: br i1 [[EC]], label %[[EXIT]], label %[[LOOP]], !llvm.loop [[LOOP3:![0-9]+]]
+; CHECK: [[EXIT]]:
+; CHECK-NEXT: [[RES:%.*]] = phi i32 [ [[IV]], %[[LOOP]] ], [ -1, %[[LATCH]] ], [ -1, %[[MIDDLE_BLOCK]] ], [ [[TMP9]], %[[VECTOR_EARLY_EXIT]] ]
+; CHECK-NEXT: ret i32 [[RES]]
+;
+entry:
+ br label %loop
+
+loop:
+ %iv = phi i32 [ 0, %entry ], [ %iv.next, %latch ]
+ %gep = getelementptr inbounds i32, ptr %p, i32 %iv
+ %ld = load i32, ptr %gep
+ %c = icmp eq i32 %ld, 0
+ br i1 %c, label %exit, label %latch
+
+latch:
+ %iv.next = add nuw nsw i32 %iv, 1
+ %ec = icmp eq i32 %iv.next, 123
+ br i1 %ec, label %exit, label %loop
+
+exit:
+ %res = phi i32 [ %iv, %loop ], [ -1, %latch ]
+ ret i32 %res
+}
+
+define i32 @multi_early_exits_same_exits(ptr align 4 dereferenceable(4096) %p, ptr align 4 dereferenceable(4096) %q) {
+; CHECK-LABEL: define i32 @multi_early_exits_same_exits(
+; CHECK-SAME: ptr align 4 dereferenceable(4096) [[P:%.*]], ptr align 4 dereferenceable(4096) [[Q:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT: [[ENTRY:.*]]:
+; CHECK-NEXT: [[TMP0:%.*]] = call i32 @llvm.vscale.i32()
+; CHECK-NEXT: [[TMP1:%.*]] = shl nuw i32 [[TMP0]], 2
+; CHECK-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ult i32 123, [[TMP1]]
+; CHECK-NEXT: br i1 [[MIN_ITERS_CHECK]], label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]]
+; CHECK: [[VECTOR_PH]]:
+; CHECK-NEXT: [[N_MOD_VF:%.*]] = urem i32 123, [[TMP1]]
+; CHECK-NEXT: [[N_VEC:%.*]] = sub i32 123, [[N_MOD_VF]]
+; CHECK-NEXT: [[TMP2:%.*]] = call <vscale x 4 x i32> @llvm.stepvector.nxv4i32()
+; CHECK-NEXT: [[BROADCAST_SPLATINSERT:%.*]] = insertelement <vscale x 4 x i32> poison, i32 [[TMP1]], i64 0
+; CHECK-NEXT: [[BROADCAST_SPLAT:%.*]] = shufflevector <vscale x 4 x i32> [[BROADCAST_SPLATINSERT]], <vscale x 4 x i32> poison, <vscale x 4 x i32> zeroinitializer
+; CHECK-NEXT: br label %[[VECTOR_BODY:.*]]
+; CHECK: [[VECTOR_BODY]]:
+; CHECK-NEXT: [[INDEX:%.*]] = phi i32 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY_INTERIM:.*]] ]
+; CHECK-NEXT: [[VEC_IND:%.*]] = phi <vscale x 4 x i32> [ [[TMP2]], %[[VECTOR_PH]] ], [ [[VEC_IND_NEXT:%.*]], %[[VECTOR_BODY_INTERIM]] ]
+; CHECK-NEXT: [[TMP3:%.*]] = getelementptr inbounds i32, ptr [[P]], i32 [[INDEX]]
+; CHECK-NEXT: [[WIDE_LOAD:%.*]] = load <vscale x 4 x i32>, ptr [[TMP3]], align 4
+; CHECK-NEXT: [[TMP4:%.*]] = icmp eq <vscale x 4 x i32> [[WIDE_LOAD]], zeroinitializer
+; CHECK-NEXT: [[TMP5:%.*]] = getelementptr inbounds i32, ptr [[Q]], i32 [[INDEX]]
+; CHECK-NEXT: [[WIDE_LOAD1:%.*]] = load <vscale x 4 x i32>, ptr [[TMP5]], align 4
+; CHECK-NEXT: [[TMP6:%.*]] = icmp eq <vscale x 4 x i32> [[WIDE_LOAD1]], zeroinitializer
+; CHECK-NEXT: [[TMP7:%.*]] = add <vscale x 4 x i32> [[VEC_IND]], splat (i32 1000)
+; CHECK-NEXT: [[TMP8:%.*]] = select <vscale x 4 x i1> [[TMP4]], <vscale x 4 x i1> splat (i1 true), <vscale x 4 x i1> [[TMP6]]
+; CHECK-NEXT: [[TMP9:%.*]] = freeze <vscale x 4 x i1> [[TMP8]]
+; CHECK-NEXT: [[TMP10:%.*]] = call i1 @llvm.vector.reduce.or.nxv4i1(<vscale x 4 x i1> [[TMP9]])
+; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i32 [[INDEX]], [[TMP1]]
+; CHECK-NEXT: [[TMP11:%.*]] = icmp eq i32 [[INDEX_NEXT]], [[N_VEC]]
+; CHECK-NEXT: [[VEC_IND_NEXT]] = add nuw nsw <vscale x 4 x i32> [[VEC_IND]], [[BROADCAST_SPLAT]]
+; CHECK-NEXT: br i1 [[TMP10]], label %[[VECTOR_EARLY_EXIT_CHECK:.*]], label %[[VECTOR_BODY_INTERIM]]
+; CHECK: [[VECTOR_BODY_INTERIM]]:
+; CHECK-NEXT: br i1 [[TMP11]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP4:![0-9]+]]
+; CHECK: [[MIDDLE_BLOCK]]:
+; CHECK-NEXT: [[CMP_N:%.*]] = icmp eq i32 123, [[N_VEC]]
+; CHECK-NEXT: br i1 [[CMP_N]], label %[[EXIT:.*]], label %[[SCALAR_PH]]
+; CHECK: [[VECTOR_EARLY_EXIT_CHECK]]:
+; CHECK-NEXT: [[FIRST_ACTIVE_LANE:%.*]] = call i64 @llvm.experimental.cttz.elts.i64.nxv4i1(<vscale x 4 x i1> [[TMP9]], i1 false)
+; CHECK-NEXT: [[TMP12:%.*]] = extractelement <vscale x 4 x i1> [[TMP4]], i64 [[FIRST_ACTIVE_LANE]]
+; CHECK-NEXT: br i1 [[TMP12]], label %[[VECTOR_EARLY_EXIT_0:.*]], label %[[VECTOR_EARLY_EXIT_1:.*]]
+; CHECK: [[VECTOR_EARLY_EXIT_1]]:
+; CHECK-NEXT: [[TMP13:%.*]] = extractelement <vscale x 4 x i32> [[TMP7]], i64 [[FIRST_ACTIVE_LANE]]
+; CHECK-NEXT: br label %[[EXIT]]
+; CHECK: [[VECTOR_EARLY_EXIT_0]]:
+; CHECK-NEXT: [[TMP14:%.*]] = trunc i64 [[FIRST_ACTIVE_LANE]] to i32
+; CHECK-NEXT: [[TMP15:%.*]] = add i32 [[INDEX]], [[TMP14]]
+; CHECK-NEXT: br label %[[EXIT]]
+; CHECK: [[SCALAR_PH]]:
+; CHECK-NEXT: [[BC_RESUME_VAL:%.*]] = phi i32 [ [[N_VEC]], %[[MIDDLE_BLOCK]] ], [ 0, %[[ENTRY]] ]
+; CHECK-NEXT: br label %[[LOOP:.*]]
+; CHECK: [[LOOP]]:
+; CHECK-NEXT: [[IV:%.*]] = phi i32 [ [[BC_RESUME_VAL]], %[[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], %[[LATCH:.*]] ]
+; CHECK-NEXT: [[GEP_P:%.*]] = getelementptr inbounds i32, ptr [[P]], i32 [[IV]]
+; CHECK-NEXT: [[X:%.*]] = load i32, ptr [[GEP_P]], align 4
+; CHECK-NEXT: [[C1:%.*]] = icmp eq i32 [[X]], 0
+; CHECK-NEXT: br i1 [[C1]], label %[[EXIT]], label %[[NEXT:.*]]
+; CHECK: [[NEXT]]:
+; CHECK-NEXT: [[GEP_Q:%.*]] = getelementptr inbounds i32, ptr [[Q]], i32 [[IV]]
+; CHECK-NEXT: [[Y:%.*]] = load i32, ptr [[GEP_Q]], align 4
+; CHECK-NEXT: [[C2:%.*]] = icmp eq i32 [[Y]], 0
+; CHECK-NEXT: [[IV_Q:%.*]] = add i32 [[IV]], 1000
+; CHECK-NEXT: br i1 [[C2]], label %[[EXIT]], label %[[LATCH]]
+; CHECK: [[LATCH]]:
+; CHECK-NEXT: [[IV_NEXT]] = add nuw nsw i32 [[IV]], 1
+; CHECK-NEXT: [[EC:%.*]] = icmp eq i32 [[IV_NEXT]], 123
+; CHECK-NEXT: br i1 [[EC]], label %[[EXIT]], label %[[LOOP]], !llvm.loop [[LOOP5:![0-9]+]]
+; CHECK: [[EXIT]]:
+; CHECK-NEXT: [[RES:%.*]] = phi i32 [ [[IV]], %[[LOOP]] ], [ [[IV_Q]], %[[NEXT]] ], [ -1, %[[LATCH]] ], [ -1, %[[MIDDLE_BLOCK]] ], [ [[TMP15]], %[[VECTOR_EARLY_EXIT_0]] ], [ [[TMP13]], %[[VECTOR_EARLY_EXIT_1]] ]
+; CHECK-NEXT: ret i32 [[RES]]
+;
+entry:
+ br label %loop
+
+loop:
+ %iv = phi i32 [ 0, %entry ], [ %iv.next, %latch ]
+ %gep.p = getelementptr inbounds i32, ptr %p, i32 %iv
+ %x = load i32, ptr %gep.p, align 4
+ %c1 = icmp eq i32 %x, 0
+ br i1 %c1, label %exit, label %next
+
+next:
+ %gep.q = getelementptr inbounds i32, ptr %q, i32 %iv
+ %y = load i32, ptr %gep.q, align 4
+ %c2 = icmp eq i32 %y, 0
+ %iv.q = add i32 %iv, 1000
+ br i1 %c2, label %exit, label %latch
+
+latch:
+ %iv.next = add nuw nsw i32 %iv, 1
+ %ec = icmp eq i32 %iv.next, 123
+ br i1 %ec, label %exit, label %loop
+
+exit:
+ %res = phi i32 [ %iv, %loop ], [ %iv.q, %next ], [ -1, %latch ]
+ ret i32 %res
+}
+
+define i32 @multi_early_exits_different_exits(ptr align 4 dereferenceable(4096) %p, ptr align 4 dereferenceable(4096) %q) {
+; CHECK-LABEL: define i32 @multi_early_exits_different_exits(
+; CHECK-SAME: ptr align 4 dereferenceable(4096) [[P:%.*]], ptr align 4 dereferenceable(4096) [[Q:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT: [[ENTRY:.*]]:
+; CHECK-NEXT: [[TMP0:%.*]] = call i32 @llvm.vscale.i32()
+; CHECK-NEXT: [[TMP1:%.*]] = shl nuw i32 [[TMP0]], 2
+; CHECK-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ult i32 123, [[TMP1]]
+; CHECK-NEXT: br i1 [[MIN_ITERS_CHECK]], label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]]
+; CHECK: [[VECTOR_PH]]:
+; CHECK-NEXT: [[N_MOD_VF:%.*]] = urem i32 123, [[TMP1]]
+; CHECK-NEXT: [[N_VEC:%.*]] = sub i32 123, [[N_MOD_VF]]
+; CHECK-NEXT: [[TMP2:%.*]] = call <vscale x 4 x i32> @llvm.stepvector.nxv4i32()
+; CHECK-NEXT: [[BROADCAST_SPLATINSERT:%.*]] = insertelement <vscale x 4 x i32> poison, i32 [[TMP1]], i64 0
+; CHECK-NEXT: [[BROADCAST_SPLAT:%.*]] = shufflevector <vscale x 4 x i32> [[BROADCAST_SPLATINSERT]], <vscale x 4 x i32> poison, <vscale x 4 x i32> zeroinitializer
+; CHECK-NEXT: br label %[[VECTOR_BODY:.*]]
+; CHECK: [[VECTOR_BODY]]:
+; CHECK-NEXT: [[INDEX:%.*]] = phi i32 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY_INTERIM:.*]] ]
+; CHECK-NEXT: [[VEC_IND:%.*]] = phi <vscale x 4 x i32> [ [[TMP2]], %[[VECTOR_PH]] ], [ [[VEC_IND_NEXT:%.*]], %[[VECTOR_BODY_INTERIM]] ]
+; CHECK-NEXT: [[TMP3:%.*]] = getelementptr inbounds i32, ptr [[P]], i32 [[INDEX]]
+; CHECK-NEXT: [[WIDE_LOAD:%.*]] = load <vscale x 4 x i32>, ptr [[TMP3]], align 4
+; CHECK-NEXT: [[TMP4:%.*]] = icmp eq <vscale x 4 x i32> [[WIDE_LOAD]], zeroinitializer
+; CHECK-NEXT: [[TMP5:%.*]] = getelementptr inbounds i32, ptr [[Q]], i32 [[INDEX]]
+; CHECK-NEXT: [[WIDE_LOAD1:%.*]] = load <vscale x 4 x i32>, ptr [[TMP5]], align 4
+; CHECK-NEXT: [[TMP6:%.*]] = icmp eq <vscale x 4 x i32> [[WIDE_LOAD1]], zeroinitializer
+; CHECK-NEXT: [[TMP7:%.*]] = add <vscale x 4 x i32> [[VEC_IND]], splat (i32 1000)
+; CHECK-NEXT: [[TMP8:%.*]] = select <vscale x 4 x i1> [[TMP4]], <vscale x 4 x i1> splat (i1 true), <vscale x 4 x i1> [[TMP6]]
+; CHECK-NEXT: [[TMP9:%.*]] = freeze <vscale x 4 x i1> [[TMP8]]
+; CHECK-NEXT: [[TMP10:%.*]] = call i1 @llvm.vector.reduce.or.nxv4i1(<vscale x 4 x i1> [[TMP9]])
+; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i32 [[INDEX]], [[TMP1]]
+; CHECK-NEXT: [[TMP11:%.*]] = icmp eq i32 [[INDEX_NEXT]], [[N_VEC]]
+; CHECK-NEXT: [[VEC_IND_NEXT]] = add nuw nsw <vscale x 4 x i32> [[VEC_IND]], [[BROADCAST_SPLAT]]
+; CHECK-NEXT: br i1 [[TMP10]], label %[[VECTOR_EARLY_EXIT_CHECK:.*]], label %[[VECTOR_BODY_INTERIM]]
+; CHECK: [[VECTOR_BODY_INTERIM]]:
+; CHECK-NEXT: br i1 [[TMP11]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP6:![0-9]+]]
+; CHECK: [[MIDDLE_BLOCK]]:
+; CHECK-NEXT: [[CMP_N:%.*]] = icmp eq i32 123, [[N_VEC]]
+; CHECK-NEXT: br i1 [[CMP_N]], label %[[EXIT_LATCH:.*]], label %[[SCALAR_PH]]
+; CHECK: [[VECTOR_EARLY_EXIT_CHECK]]:
+; CHECK-NEXT: [[FIRST_ACTIVE_LANE:%.*]] = call i64 @llvm.experimental.cttz.elts.i64.nxv4i1(<vscale x 4 x i1> [[TMP9]], i1 false)
+; CHECK-NEXT: [[TMP12:%.*]] = extractelement <vscale x 4 x i1> [[TMP4]], i64 [[FIRST_ACTIVE_LANE]]
+; CHECK-NEXT: br i1 [[TMP12]], label %[[VECTOR_EARLY_EXIT_0:.*]], label %[[VECTOR_EARLY_EXIT_1:.*]]
+; CHECK: [[VECTOR_EARLY_EXIT_1]]:
+; CHECK-NEXT: [[TMP13:%.*]] = extractelement <vscale x 4 x i32> [[TMP7]], i64 [[FIRST_ACTIVE_LANE]]
+; CHECK-NEXT: br label %[[EXIT_Q:.*]]
+; CHECK: [[VECTOR_EARLY_EXIT_0]]:
+; CHECK-NEXT: [[TMP14:%.*]] = trunc i64 [[FIRST_ACTIVE_LANE]] to i32
+; CHECK-NEXT: [[TMP15:%.*]] = add i32 [[INDEX]], [[TMP14]]
+; CHECK-NEXT: br label %[[EXIT_P:.*]]
+; CHECK: [[SCALAR_PH]]:
+; CHECK-NEXT: [[BC_RESUME_VAL:%.*]] = phi i32 [ [[N_VEC]], %[[MIDDLE_BLOCK]] ], [ 0, %[[ENTRY]] ]
+; CHECK-NEXT: br label %[[LOOP:.*]]
+; CHECK: [[LOOP]]:
+; CHECK-NEXT: [[IV:%.*]] = phi i32 [ [[BC_RESUME_VAL]], %[[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], %[[LATCH:.*]] ]
+; CHECK-NEXT: [[GEP_P:%.*]] = getelementptr inbounds i32, ptr [[P]], i32 [[IV]]
+; CHECK-NEXT: [[X:%.*]] = load i32, ptr [[GEP_P]], align 4
+; CHECK-NEXT: [[C1:%.*]] = icmp eq i32 [[X]], 0
+; CHECK-NEXT: br i1 [[C1]], label %[[EXIT_P]], label %[[NEXT:.*]]
+; CHECK: [[NEXT]]:
+; CHECK-NEXT: [[GEP_Q:%.*]] = getelementptr inbounds i32, ptr [[Q]], i32 [[IV]]
+; CHECK-NEXT: [[Y:%.*]] = load i32, ptr [[GEP_Q]], align 4
+; CHECK-NEXT: [[C2:%.*]] = icmp eq i32 [[Y]], 0
+; CHECK-NEXT: [[IV_Q:%.*]] = add i32 [[IV]], 1000
+; CHECK-NEXT: br i1 [[C2]], label %[[EXIT_Q]], label %[[LATCH]]
+; CHECK: [[LATCH]]:
+; CHECK-NEXT: [[IV_NEXT]] = add nuw nsw i32 [[IV]], 1
+; CHECK-NEXT: [[EC:%.*]] = icmp eq i32 [[IV_NEXT]], 123
+; CHECK-NEXT: br i1 [[EC]], label %[[EXIT_LATCH]], label %[[LOOP]], !llvm.loop [[LOOP7:![0-9]+]]
+; CHECK: [[EXIT_P]]:
+; CHECK-NEXT: [[RES_P:%.*]] = phi i32 [ [[IV]], %[[LOOP]] ], [ [[TMP15]], %[[VECTOR_EARLY_EXIT_0]] ]
+; CHECK-NEXT: ret i32 [[RES_P]]
+; CHECK: [[EXIT_Q]]:
+; CHECK-NEXT: [[RES_Q:%.*]] = phi i32 [ [[IV_Q]], %[[NEXT]] ], [ [[TMP13]], %[[VECTOR_EARLY_EXIT_1]] ]
+; CHECK-NEXT: ret i32 [[RES_Q]]
+; CHECK: [[EXIT_LATCH]]:
+; CHECK-NEXT: ret i32 -1
+;
+entry:
+ br label %loop
+
+loop:
+ %iv = phi i32 [ 0, %entry ], [ %iv.next, %latch ]
+ %gep.p = getelementptr inbounds i32, ptr %p, i32 %iv
+ %x = load i32, ptr %gep.p, align 4
+ %c1 = icmp eq i32 %x, 0
+ br i1 %c1, label %exit.p, label %next
+
+next:
+ %gep.q = getelementptr inbounds i32, ptr %q, i32 %iv
+ %y = load i32, ptr %gep.q, align 4
+ %c2 = icmp eq i32 %y, 0
+ %iv.q = add i32 %iv, 1000
+ br i1 %c2, label %exit.q, label %latch
+
+latch:
+ %iv.next = add nuw nsw i32 %iv, 1
+ %ec = icmp eq i32 %iv.next, 123
+ br i1 %ec, label %exit.latch, label %loop
+
+exit.p:
+ %res.p = phi i32 [ %iv, %loop ]
+ ret i32 %res.p
+
+exit.q:
+ %res.q = phi i32 [ %iv.q, %next ]
+ ret i32 %res.q
+
+exit.latch:
+ ret i32 -1
+}
+
+define i32 @predicated_early_exit(i1 %c0, ptr align 4 dereferenceable(4096) %p) {
+; CHECK-LABEL: define i32 @predicated_early_exit(
+; CHECK-SAME: i1 [[C0:%.*]], ptr align 4 dereferenceable(4096) [[P:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT: [[ENTRY:.*]]:
+; CHECK-NEXT: [[TMP0:%.*]] = call i32 @llvm.vscale.i32()
+; CHECK-NEXT: [[TMP1:%.*]] = shl nuw i32 [[TMP0]], 2
+; CHECK-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ult i32 123, [[TMP1]]
+; CHECK-NEXT: br i1 [[MIN_ITERS_CHECK]], label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]]
+; CHECK: [[VECTOR_PH]]:
+; CHECK-NEXT: [[N_MOD_VF:%.*]] = urem i32 123, [[TMP1]]
+; CHECK-NEXT: [[N_VEC:%.*]] = sub i32 123, [[N_MOD_VF]]
+; CHECK-NEXT: [[BROADCAST_SPLATINSERT:%.*]] = insertelement <vscale x 4 x i1> poison, i1 [[C0]], i64 0
+; CHECK-NEXT: [[BROADCAST_SPLAT:%.*]] = shufflevector <vscale x 4 x i1> [[BROADCAST_SPLATINSERT]], <vscale x 4 x i1> poison, <vscale x 4 x i32> zeroinitializer
+; CHECK-NEXT: [[TMP11:%.*]] = freeze <vscale x 4 x i1> [[BROADCAST_SPLAT]]
+; CHECK-NEXT: br label %[[VECTOR_BODY:.*]]
+; CHECK: [[VECTOR_BODY]]:
+; CHECK-NEXT: [[INDEX:%.*]] = phi i32 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY_INTERIM:.*]] ]
+; CHECK-NEXT: [[TMP2:%.*]] = getelementptr inbounds i32, ptr [[P]], i32 [[INDEX]]
+; CHECK-NEXT: [[WIDE_LOAD:%.*]] = load <vscale x 4 x i32>, ptr [[TMP2]], align 4
+; CHECK-NEXT: [[TMP3:%.*]] = icmp eq <vscale x 4 x i32> [[WIDE_LOAD]], zeroinitializer
+; CHECK-NEXT: [[TMP4:%.*]] = select <vscale x 4 x i1> [[TMP11]], <vscale x 4 x i1> [[TMP3]], <vscale x 4 x i1> zeroinitializer
+; CHECK-NEXT: [[TMP5:%.*]] = freeze <vscale x 4 x i1> [[TMP4]]
+; CHECK-NEXT: [[TMP6:%.*]] = call i1 @llvm.vector.reduce.or.nxv4i1(<vscale x 4 x i1> [[TMP5]])
+; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i32 [[INDEX]], [[TMP1]]
+; CHECK-NEXT: [[TMP7:%.*]] = icmp eq i32 [[INDEX_NEXT]], [[N_VEC]]
+; CHECK-NEXT: br i1 [[TMP6]], label %[[VECTOR_EARLY_EXIT:.*]], label %[[VECTOR_BODY_INTERIM]]
+; CHECK: [[VECTOR_BODY_INTERIM]]:
+; CHECK-NEXT: br i1 [[TMP7]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP8:![0-9]+]]
+; CHECK: [[MIDDLE_BLOCK]]:
+; CHECK-NEXT: [[CMP_N:%.*]] = icmp eq i32 123, [[N_VEC]]
+; CHECK-NEXT: br i1 [[CMP_N]], label %[[EXIT:.*]], label %[[SCALAR_PH]]
+; CHECK: [[VECTOR_EARLY_EXIT]]:
+; CHECK-NEXT: [[TMP8:%.*]] = call i64 @llvm.experimental.cttz.elts.i64.nxv4i1(<vscale x 4 x i1> [[TMP5]], i1 false)
+; CHECK-NEXT: [[TMP9:%.*]] = trunc i64 [[TMP8]] to i32
+; CHECK-NEXT: [[TMP10:%.*]] = add i32 [[INDEX]], [[TMP9]]
+; CHECK-NEXT: br label %[[EXIT]]
+; CHECK: [[SCALAR_PH]]:
+; CHECK-NEXT: [[BC_RESUME_VAL:%.*]] = phi i32 [ [[N_VEC]], %[[MIDDLE_BLOCK]] ], [ 0, %[[ENTRY]] ]
+; CHECK-NEXT: br label %[[LOOP:.*]]
+; CHECK: [[LOOP]]:
+; CHECK-NEXT: [[IV:%.*]] = phi i32 [ [[BC_RESUME_VAL]], %[[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], %[[LATCH:.*]] ]
+; CHECK-NEXT: br i1 [[C0]], label %[[THEN:.*]], label %[[LATCH]]
+; CHECK: [[THEN]]:
+; CHECK-NEXT: [[GEP:%.*]] = getelementptr inbounds i32, ptr [[P]], i32 [[IV]]
+; CHECK-NEXT: [[LD:%.*]] = load i32, ptr [[GEP]], align 4
+; CHECK-NEXT: [[C1:%.*]] = icmp eq i32 [[LD]], 0
+; CHECK-NEXT: br i1 [[C1]], label %[[EXIT]], label %[[LATCH]]
+; CHECK: [[LATCH]]:
+; CHECK-NEXT: [[IV_NEXT]] = add nuw nsw i32 [[IV]], 1
+; CHECK-NEXT: [[EC:%.*]] = icmp eq i32 [[IV_NEXT]], 123
+; CHECK-NEXT: br i1 [[EC]], label %[[EXIT]], label %[[LOOP]], !llvm.loop [[LOOP9:![0-9]+]]
+; CHECK: [[EXIT]]:
+; CHECK-NEXT: [[RES:%.*]] = phi i32 [ [[IV]], %[[THEN]] ], [ -1, %[[LATCH]] ], [ -1, %[[MIDDLE_BLOCK]] ], [ [[TMP10]], %[[VECTOR_EARLY_EXIT]] ]
+; CHECK-NEXT: ret i32 [[RES]]
+;
+entry:
+ br label %loop
+
+loop:
+ %iv = phi i32 [ 0, %entry ], [ %iv.next, %latch ]
+ br i1 %c0, label %then, label %latch
+
+then:
+ %gep = getelementptr inbounds i32, ptr %p, i32 %iv
+ %ld = load i32, ptr %gep
+ %c1 = icmp eq i32 %ld, 0
+ br i1 %c1, label %exit, label %latch
+
+latch:
+ %iv.next = add nuw nsw i32 %iv, 1
+ %ec = icmp eq i32 %iv.next, 123
+ br i1 %ec, label %exit, label %loop
+
+exit:
+ %res = phi i32 [ %iv, %then ], [ -1, %latch ]
+ ret i32 %res
+}
+
+; TODO: Cannot tail fold yet.
+define i32 @countable_early_exit(ptr align 4 dereferenceable(4096) %p, i32 %n) {
+; CHECK-LABEL: define i32 @countable_early_exit(
+; CHECK-SAME: ptr align 4 dereferenceable(4096) [[P:%.*]], i32 [[N:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT: [[ENTRY:.*]]:
+; CHECK-NEXT: br label %[[LOOP:.*]]
+; CHECK: [[LOOP]]:
+; CHECK-NEXT: [[IV:%.*]] = phi i32 [ 0, %[[ENTRY]] ], [ [[IV_NEXT:%.*]], %[[LATCH:.*]] ]
+; CHECK-NEXT: [[C:%.*]] = icmp eq i32 [[IV]], [[N]]
+; CHECK-NEXT: br i1 [[C]], label %[[EXIT:.*]], label %[[LATCH]]
+; CHECK: [[LATCH]]:
+; CHECK-NEXT: [[GEP:%.*]] = getelementptr inbounds i32, ptr [[P]], i32 [[IV]]
+; CHECK-NEXT: [[LD:%.*]] = load i32, ptr [[GEP]], align 4
+; CHECK-NEXT: [[IV_NEXT]] = add nuw nsw i32 [[IV]], 1
+; CHECK-NEXT: [[EC:%.*]] = icmp eq i32 [[IV_NEXT]], 123
+; CHECK-NEXT: br i1 [[EC]], label %[[EXIT]], label %[[LOOP]]
+; CHECK: [[EXIT]]:
+; CHECK-NEXT: [[RES:%.*]] = phi i32 [ 0, %[[LOOP]] ], [ [[LD]], %[[LATCH]] ]
+; CHECK-NEXT: ret i32 [[RES]]
+;
+entry:
+ br label %loop
+
+loop:
+ %iv = phi i32 [ 0, %entry ], [ %iv.next, %latch ]
+ %c = icmp eq i32 %iv, %n
+ br i1 %c, label %exit, label %latch
+
+latch:
+ %gep = getelementptr inbounds i32, ptr %p, i32 %iv
+ %ld = load i32, ptr %gep, align 4
+ %iv.next = add nuw nsw i32 %iv, 1
+ %ec = icmp eq i32 %iv.next, 123
+ br i1 %ec, label %exit, label %loop
+
+exit:
+ %res = phi i32 [ 0, %loop ], [ %ld, %latch ]
+ ret i32 %res
+}
+
+; TODO: Cannot tail fold yet.
+define void @early_exit_with_store(ptr align 4 dereferenceable(4096) noalias %dst, ptr align 4 dereferenceable(4096) %pred) {
+; CHECK-LABEL: define void @early_exit_with_store(
+; CHECK-SAME: ptr noalias align 4 dereferenceable(4096) [[DST:%.*]], ptr align 4 dereferenceable(4096) [[PRED:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT: [[ENTRY:.*]]:
+; CHECK-NEXT: [[TMP0:%.*]] = call i32 @llvm.vscale.i32()
+; CHECK-NEXT: [[TMP1:%.*]] = shl nuw i32 [[TMP0]], 2
+; CHECK-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ult i32 123, [[TMP1]]
+; CHECK-NEXT: br i1 [[MIN_ITERS_CHECK]], label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]]
+; CHECK: [[VECTOR_PH]]:
+; CHECK-NEXT: [[N_MOD_VF:%.*]] = urem i32 123, [[TMP1]]
+; CHECK-NEXT: [[N_VEC:%.*]] = sub i32 123, [[N_MOD_VF]]
+; CHECK-NEXT: br label %[[VECTOR_BODY:.*]]
+; CHECK: [[VECTOR_BODY]]:
+; CHECK-NEXT: [[INDEX:%.*]] = phi i32 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-NEXT: [[TMP2:%.*]] = getelementptr inbounds nuw i32, ptr [[PRED]], i32 [[INDEX]]
+; CHECK-NEXT: [[WIDE_LOAD:%.*]] = load <vscale x 4 x i32>, ptr [[TMP2]], align 4
+; CHECK-NEXT: [[TMP3:%.*]] = icmp sgt <vscale x 4 x i32> [[WIDE_LOAD]], splat (i32 500)
+; CHECK-NEXT: [[TMP4:%.*]] = freeze <vscale x 4 x i1> [[TMP3]]
+; CHECK-NEXT: [[UNCOUNTABLE_EXIT_MASK:%.*]] = call <vscale x 4 x i1> @llvm.mask.beforefirst.nxv4i1(<vscale x 4 x i1> [[TMP4]])
+; CHECK-NEXT: [[TMP7:%.*]] = getelementptr i32, ptr [[DST]], i32 [[INDEX]]
+; CHECK-NEXT: [[WIDE_MASKED_LOAD:%.*]] = call <vscale x 4 x i32> @llvm.masked.load.nxv4i32.p0(ptr align 4 [[TMP7]], <vscale x 4 x i1> [[UNCOUNTABLE_EXIT_MASK]], <vscale x 4 x i32> poison)
+; CHECK-NEXT: [[TMP8:%.*]] = add nsw <vscale x 4 x i32> [[WIDE_MASKED_LOAD]], splat (i32 1)
+; CHECK-NEXT: call void @llvm.masked.store.nxv4i32.p0(<vscale x 4 x i32> [[TMP8]], ptr align 4 [[TMP7]], <vscale x 4 x i1> [[UNCOUNTABLE_EXIT_MASK]])
+; CHECK-NEXT: [[TMP9:%.*]] = call i1 @llvm.vector.reduce.or.nxv4i1(<vscale x 4 x i1> [[TMP4]])
+; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i32 [[INDEX]], [[TMP1]]
+; CHECK-NEXT: [[TMP10:%.*]] = icmp eq i32 [[INDEX_NEXT]], [[N_VEC]]
+; CHECK-NEXT: [[TMP11:%.*]] = or i1 [[TMP9]], [[TMP10]]
+; CHECK-NEXT: br i1 [[TMP11]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP10:![0-9]+]]
+; CHECK: [[MIDDLE_BLOCK]]:
+; CHECK-NEXT: [[TMP14:%.*]] = call i64 @llvm.experimental.cttz.elts.i64.nxv4i1(<vscale x 4 x i1> [[TMP4]], i1 false)
+; CHECK-NEXT: [[TMP6:%.*]] = trunc i64 [[TMP14]] to i32
+; CHECK-NEXT: [[TMP12:%.*]] = add i32 [[INDEX]], [[TMP6]]
+; CHECK-NEXT: [[TMP13:%.*]] = icmp eq i32 [[TMP12]], 123
+; CHECK-NEXT: br i1 [[TMP13]], label %[[EXIT:.*]], label %[[SCALAR_PH]]
+; CHECK: [[SCALAR_PH]]:
+; CHECK-NEXT: [[BC_RESUME_VAL:%.*]] = phi i32 [ [[TMP12]], %[[MIDDLE_BLOCK]] ], [ 0, %[[ENTRY]] ]
+; CHECK-NEXT: br label %[[LOOP:.*]]
+; CHECK: [[LOOP]]:
+; CHECK-NEXT: [[IV:%.*]] = phi i32 [ [[BC_RESUME_VAL]], %[[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], %[[LATCH:.*]] ]
+; CHECK-NEXT: [[ST_ADDR:%.*]] = getelementptr inbounds nuw i32, ptr [[DST]], i32 [[IV]]
+; CHECK-NEXT: [[DATA:%.*]] = load i32, ptr [[ST_ADDR]], align 4
+; CHECK-NEXT: [[INC:%.*]] = add nsw i32 [[DATA]], 1
+; CHECK-NEXT: store i32 [[INC]], ptr [[ST_ADDR]], align 4
+; CHECK-NEXT: [[EE_ADDR:%.*]] = getelementptr inbounds nuw i32, ptr [[PRED]], i32 [[IV]]
+; CHECK-NEXT: [[EE_VAL:%.*]] = load i32, ptr [[EE_ADDR]], align 4
+; CHECK-NEXT: [[EE_COND:%.*]] = icmp sgt i32 [[EE_VAL]], 500
+; CHECK-NEXT: br i1 [[EE_COND]], label %[[EXIT]], label %[[LATCH]]
+; CHECK: [[LATCH]]:
+; CHECK-NEXT: [[IV_NEXT]] = add nuw nsw i32 [[IV]], 1
+; CHECK-NEXT: [[EC:%.*]] = icmp eq i32 [[IV_NEXT]], 123
+; CHECK-NEXT: br i1 [[EC]], label %[[EXIT]], label %[[LOOP]], !llvm.loop [[LOOP11:![0-9]+]]
+; CHECK: [[EXIT]]:
+; CHECK-NEXT: ret void
+;
+entry:
+ br label %loop
+
+loop:
+ %iv = phi i32 [ 0, %entry ], [ %iv.next, %latch ]
+ %st.addr = getelementptr inbounds nuw i32, ptr %dst, i32 %iv
+ %data = load i32, ptr %st.addr, align 4
+ %inc = add nsw i32 %data, 1
+ store i32 %inc, ptr %st.addr, align 4
+ %ee.addr = getelementptr inbounds nuw i32, ptr %pred, i32 %iv
+ %ee.val = load i32, ptr %ee.addr, align 4
+ %ee.cond = icmp sgt i32 %ee.val, 500
+ br i1 %ee.cond, label %exit, label %latch
+
+latch:
+ %iv.next = add nuw nsw i32 %iv, 1
+ %ec = icmp eq i32 %iv.next, 123
+ br i1 %ec, label %exit, label %loop
+
+exit:
+ ret void
+}
+
+define void @early_exit_with_store_after_exit(ptr align 4 dereferenceable(4096) noalias %dst, ptr align 4 dereferenceable(4096) %pred) {
+; CHECK-LABEL: define void @early_exit_with_store_after_exit(
+; CHECK-SAME: ptr noalias align 4 dereferenceable(4096) [[DST:%.*]], ptr align 4 dereferenceable(4096) [[PRED:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT: [[ENTRY:.*]]:
+; CHECK-NEXT: [[TMP0:%.*]] = call i32 @llvm.vscale.i32()
+; CHECK-NEXT: [[TMP1:%.*]] = shl nuw i32 [[TMP0]], 2
+; CHECK-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ult i32 123, [[TMP1]]
+; CHECK-NEXT: br i1 [[MIN_ITERS_CHECK]], label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]]
+; CHECK: [[VECTOR_PH]]:
+; CHECK-NEXT: [[N_MOD_VF:%.*]] = urem i32 123, [[TMP1]]
+; CHECK-NEXT: [[N_VEC:%.*]] = sub i32 123, [[N_MOD_VF]]
+; CHECK-NEXT: br label %[[VECTOR_BODY:.*]]
+; CHECK: [[VECTOR_BODY]]:
+; CHECK-NEXT: [[INDEX:%.*]] = phi i32 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY_INTERIM:.*]] ]
+; CHECK-NEXT: [[TMP2:%.*]] = getelementptr inbounds nuw i32, ptr [[PRED]], i32 [[INDEX]]
+; CHECK-NEXT: [[WIDE_LOAD:%.*]] = load <vscale x 4 x i32>, ptr [[TMP2]], align 4
+; CHECK-NEXT: [[TMP3:%.*]] = icmp sgt <vscale x 4 x i32> [[WIDE_LOAD]], splat (i32 500)
+; CHECK-NEXT: [[TMP4:%.*]] = freeze <vscale x 4 x i1> [[TMP3]]
+; CHECK-NEXT: [[TMP5:%.*]] = call <vscale x 4 x i1> @llvm.mask.beforefirst.nxv4i1(<vscale x 4 x i1> [[TMP4]])
+; CHECK-NEXT: [[TMP6:%.*]] = getelementptr i32, ptr [[DST]], i32 [[INDEX]]
+; CHECK-NEXT: [[WIDE_MASKED_LOAD:%.*]] = call <vscale x 4 x i32> @llvm.masked.load.nxv4i32.p0(ptr align 4 [[TMP6]], <vscale x 4 x i1> [[TMP5]], <vscale x 4 x i32> poison)
+; CHECK-NEXT: [[TMP7:%.*]] = add nsw <vscale x 4 x i32> [[WIDE_MASKED_LOAD]], splat (i32 1)
+; CHECK-NEXT: call void @llvm.masked.store.nxv4i32.p0(<vscale x 4 x i32> [[TMP7]], ptr align 4 [[TMP6]], <vscale x 4 x i1> [[TMP5]])
+; CHECK-NEXT: [[TMP8:%.*]] = call i1 @llvm.vector.reduce.or.nxv4i1(<vscale x 4 x i1> [[TMP4]])
+; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i32 [[INDEX]], [[TMP1]]
+; CHECK-NEXT: [[TMP9:%.*]] = icmp eq i32 [[INDEX_NEXT]], [[N_VEC]]
+; CHECK-NEXT: br i1 [[TMP8]], label %[[VECTOR_EARLY_EXIT:.*]], label %[[VECTOR_BODY_INTERIM]]
+; CHECK: [[VECTOR_BODY_INTERIM]]:
+; CHECK-NEXT: br i1 [[TMP9]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP12:![0-9]+]]
+; CHECK: [[MIDDLE_BLOCK]]:
+; CHECK-NEXT: [[CMP_N:%.*]] = icmp eq i32 123, [[N_VEC]]
+; CHECK-NEXT: br i1 [[CMP_N]], label %[[EXIT:.*]], label %[[SCALAR_PH]]
+; CHECK: [[VECTOR_EARLY_EXIT]]:
+; CHECK-NEXT: br label %[[EXIT]]
+; CHECK: [[SCALAR_PH]]:
+; CHECK-NEXT: [[BC_RESUME_VAL:%.*]] = phi i32 [ [[N_VEC]], %[[MIDDLE_BLOCK]] ], [ 0, %[[ENTRY]] ]
+; CHECK-NEXT: br label %[[LOOP:.*]]
+; CHECK: [[LOOP]]:
+; CHECK-NEXT: [[IV:%.*]] = phi i32 [ [[BC_RESUME_VAL]], %[[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], %[[LATCH:.*]] ]
+; CHECK-NEXT: [[EE_ADDR:%.*]] = getelementptr inbounds nuw i32, ptr [[PRED]], i32 [[IV]]
+; CHECK-NEXT: [[EE_VAL:%.*]] = load i32, ptr [[EE_ADDR]], align 4
+; CHECK-NEXT: [[EE_COND:%.*]] = icmp sgt i32 [[EE_VAL]], 500
+; CHECK-NEXT: br i1 [[EE_COND]], label %[[EXIT]], label %[[LATCH]]
+; CHECK: [[LATCH]]:
+; CHECK-NEXT: [[ST_ADDR:%.*]] = getelementptr inbounds nuw i32, ptr [[DST]], i32 [[IV]]
+; CHECK-NEXT: [[DATA:%.*]] = load i32, ptr [[ST_ADDR]], align 4
+; CHECK-NEXT: [[INC:%.*]] = add nsw i32 [[DATA]], 1
+; CHECK-NEXT: store i32 [[INC]], ptr [[ST_ADDR]], align 4
+; CHECK-NEXT: [[IV_NEXT]] = add nuw nsw i32 [[IV]], 1
+; CHECK-NEXT: [[EC:%.*]] = icmp eq i32 [[IV_NEXT]], 123
+; CHECK-NEXT: br i1 [[EC]], label %[[EXIT]], label %[[LOOP]], !llvm.loop [[LOOP13:![0-9]+]]
+; CHECK: [[EXIT]]:
+; CHECK-NEXT: ret void
+;
+entry:
+ br label %loop
+
+loop:
+ %iv = phi i32 [ 0, %entry ], [ %iv.next, %latch ]
+ %ee.addr = getelementptr inbounds nuw i32, ptr %pred, i32 %iv
+ %ee.val = load i32, ptr %ee.addr
+ %ee.cond = icmp sgt i32 %ee.val, 500
+ br i1 %ee.cond, label %exit, label %latch
+
+latch:
+ %st.addr = getelementptr inbounds nuw i32, ptr %dst, i32 %iv
+ %data = load i32, ptr %st.addr
+ %inc = add nsw i32 %data, 1
+ store i32 %inc, ptr %st.addr
+ %iv.next = add nuw nsw i32 %iv, 1
+ %ec = icmp eq i32 %iv.next, 123
+ br i1 %ec, label %exit, label %loop
+
+exit:
+ ret void
+}
diff --git a/llvm/test/Transforms/LoopVectorize/early-exit-tail-folding-active-lane-mask.ll b/llvm/test/Transforms/LoopVectorize/early-exit-tail-folding-active-lane-mask.ll
new file mode 100644
index 00000000000000..a5c7c2a214335a
--- /dev/null
+++ b/llvm/test/Transforms/LoopVectorize/early-exit-tail-folding-active-lane-mask.ll
@@ -0,0 +1,65 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --check-globals none --version 6
+; RUN: opt -S < %s -p loop-vectorize -force-vector-width=4 -force-target-supports-masked-memory-ops -tail-folding-policy=prefer-fold-tail -force-tail-folding-style=data-and-control | FileCheck %s
+
+; TODO: Handle active lane mask style
+define i32 @single_early_exit(ptr align 4 dereferenceable(4096) %p) {
+; CHECK-LABEL: define i32 @single_early_exit(
+; CHECK-SAME: ptr align 4 dereferenceable(4096) [[P:%.*]]) {
+; CHECK-NEXT: [[ENTRY:.*:]]
+; CHECK-NEXT: br label %[[VECTOR_PH:.*]]
+; CHECK: [[VECTOR_PH]]:
+; CHECK-NEXT: br label %[[VECTOR_BODY:.*]]
+; CHECK: [[VECTOR_BODY]]:
+; CHECK-NEXT: [[INDEX:%.*]] = phi i32 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY_INTERIM:.*]] ]
+; CHECK-NEXT: [[TMP0:%.*]] = getelementptr inbounds i32, ptr [[P]], i32 [[INDEX]]
+; CHECK-NEXT: [[WIDE_LOAD:%.*]] = load <4 x i32>, ptr [[TMP0]], align 4
+; CHECK-NEXT: [[TMP1:%.*]] = icmp eq <4 x i32> [[WIDE_LOAD]], zeroinitializer
+; CHECK-NEXT: [[TMP2:%.*]] = freeze <4 x i1> [[TMP1]]
+; CHECK-NEXT: [[TMP3:%.*]] = call i1 @llvm.vector.reduce.or.v4i1(<4 x i1> [[TMP2]])
+; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i32 [[INDEX]], 4
+; CHECK-NEXT: [[TMP4:%.*]] = icmp eq i32 [[INDEX_NEXT]], 120
+; CHECK-NEXT: br i1 [[TMP3]], label %[[VECTOR_EARLY_EXIT:.*]], label %[[VECTOR_BODY_INTERIM]]
+; CHECK: [[VECTOR_BODY_INTERIM]]:
+; CHECK-NEXT: br i1 [[TMP4]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP0:![0-9]+]]
+; CHECK: [[MIDDLE_BLOCK]]:
+; CHECK-NEXT: br label %[[EXIT:.*]]
+; CHECK: [[VECTOR_EARLY_EXIT]]:
+; CHECK-NEXT: [[TMP5:%.*]] = call i64 @llvm.experimental.cttz.elts.i64.v4i1(<4 x i1> [[TMP2]], i1 false)
+; CHECK-NEXT: [[TMP6:%.*]] = trunc i64 [[TMP5]] to i32
+; CHECK-NEXT: [[TMP7:%.*]] = add i32 [[INDEX]], [[TMP6]]
+; CHECK-NEXT: br label %[[EXIT1:.*]]
+; CHECK: [[EXIT]]:
+; CHECK-NEXT: br label %[[LOOP:.*]]
+; CHECK: [[LOOP]]:
+; CHECK-NEXT: [[IV:%.*]] = phi i32 [ 120, %[[EXIT]] ], [ [[IV_NEXT:%.*]], %[[LATCH:.*]] ]
+; CHECK-NEXT: [[GEP:%.*]] = getelementptr inbounds i32, ptr [[P]], i32 [[IV]]
+; CHECK-NEXT: [[LD:%.*]] = load i32, ptr [[GEP]], align 4
+; CHECK-NEXT: [[C:%.*]] = icmp eq i32 [[LD]], 0
+; CHECK-NEXT: br i1 [[C]], label %[[EXIT1]], label %[[LATCH]]
+; CHECK: [[LATCH]]:
+; CHECK-NEXT: [[IV_NEXT]] = add nuw nsw i32 [[IV]], 1
+; CHECK-NEXT: [[EC:%.*]] = icmp eq i32 [[IV_NEXT]], 123
+; CHECK-NEXT: br i1 [[EC]], label %[[EXIT1]], label %[[LOOP]], !llvm.loop [[LOOP3:![0-9]+]]
+; CHECK: [[EXIT1]]:
+; CHECK-NEXT: [[RES:%.*]] = phi i32 [ [[IV]], %[[LOOP]] ], [ -1, %[[LATCH]] ], [ [[TMP7]], %[[VECTOR_EARLY_EXIT]] ]
+; CHECK-NEXT: ret i32 [[RES]]
+;
+entry:
+ br label %loop
+
+loop:
+ %iv = phi i32 [ 0, %entry ], [ %iv.next, %latch ]
+ %gep = getelementptr inbounds i32, ptr %p, i32 %iv
+ %ld = load i32, ptr %gep
+ %c = icmp eq i32 %ld, 0
+ br i1 %c, label %exit, label %latch
+
+latch:
+ %iv.next = add nuw nsw i32 %iv, 1
+ %ec = icmp eq i32 %iv.next, 123
+ br i1 %ec, label %exit, label %loop
+
+exit:
+ %res = phi i32 [ %iv, %loop ], [ -1, %latch ]
+ ret i32 %res
+}
diff --git a/llvm/test/Transforms/LoopVectorize/early-exit-tail-folding.ll b/llvm/test/Transforms/LoopVectorize/early-exit-tail-folding.ll
new file mode 100644
index 00000000000000..9026514723ae5d
--- /dev/null
+++ b/llvm/test/Transforms/LoopVectorize/early-exit-tail-folding.ll
@@ -0,0 +1,524 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --check-globals none --version 6
+; RUN: opt -S < %s -p loop-vectorize -force-vector-width=4 -force-target-supports-masked-memory-ops -enable-early-exit-vectorization-with-side-effects -tail-folding-policy=prefer-fold-tail | FileCheck %s
+
+define i32 @single_early_exit(ptr align 4 dereferenceable(4096) %p) {
+; CHECK-LABEL: define i32 @single_early_exit(
+; CHECK-SAME: ptr align 4 dereferenceable(4096) [[P:%.*]]) {
+; CHECK-NEXT: [[ENTRY:.*:]]
+; CHECK-NEXT: br label %[[VECTOR_PH:.*]]
+; CHECK: [[VECTOR_PH]]:
+; CHECK-NEXT: br label %[[VECTOR_BODY:.*]]
+; CHECK: [[VECTOR_BODY]]:
+; CHECK-NEXT: [[INDEX:%.*]] = phi i32 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY_INTERIM:.*]] ]
+; CHECK-NEXT: [[TMP0:%.*]] = getelementptr inbounds i32, ptr [[P]], i32 [[INDEX]]
+; CHECK-NEXT: [[WIDE_LOAD:%.*]] = load <4 x i32>, ptr [[TMP0]], align 4
+; CHECK-NEXT: [[TMP1:%.*]] = icmp eq <4 x i32> [[WIDE_LOAD]], zeroinitializer
+; CHECK-NEXT: [[TMP2:%.*]] = freeze <4 x i1> [[TMP1]]
+; CHECK-NEXT: [[TMP3:%.*]] = call i1 @llvm.vector.reduce.or.v4i1(<4 x i1> [[TMP2]])
+; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i32 [[INDEX]], 4
+; CHECK-NEXT: [[TMP4:%.*]] = icmp eq i32 [[INDEX_NEXT]], 120
+; CHECK-NEXT: br i1 [[TMP3]], label %[[VECTOR_EARLY_EXIT:.*]], label %[[VECTOR_BODY_INTERIM]]
+; CHECK: [[VECTOR_BODY_INTERIM]]:
+; CHECK-NEXT: br i1 [[TMP4]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP0:![0-9]+]]
+; CHECK: [[MIDDLE_BLOCK]]:
+; CHECK-NEXT: br label %[[EXIT:.*]]
+; CHECK: [[VECTOR_EARLY_EXIT]]:
+; CHECK-NEXT: [[TMP5:%.*]] = call i64 @llvm.experimental.cttz.elts.i64.v4i1(<4 x i1> [[TMP2]], i1 false)
+; CHECK-NEXT: [[TMP6:%.*]] = trunc i64 [[TMP5]] to i32
+; CHECK-NEXT: [[TMP7:%.*]] = add i32 [[INDEX]], [[TMP6]]
+; CHECK-NEXT: br label %[[EXIT1:.*]]
+; CHECK: [[EXIT]]:
+; CHECK-NEXT: br label %[[LOOP:.*]]
+; CHECK: [[LOOP]]:
+; CHECK-NEXT: [[IV:%.*]] = phi i32 [ 120, %[[EXIT]] ], [ [[IV_NEXT:%.*]], %[[LATCH:.*]] ]
+; CHECK-NEXT: [[GEP:%.*]] = getelementptr inbounds i32, ptr [[P]], i32 [[IV]]
+; CHECK-NEXT: [[LD:%.*]] = load i32, ptr [[GEP]], align 4
+; CHECK-NEXT: [[C:%.*]] = icmp eq i32 [[LD]], 0
+; CHECK-NEXT: br i1 [[C]], label %[[EXIT1]], label %[[LATCH]]
+; CHECK: [[LATCH]]:
+; CHECK-NEXT: [[IV_NEXT]] = add nuw nsw i32 [[IV]], 1
+; CHECK-NEXT: [[EC:%.*]] = icmp eq i32 [[IV_NEXT]], 123
+; CHECK-NEXT: br i1 [[EC]], label %[[EXIT1]], label %[[LOOP]], !llvm.loop [[LOOP3:![0-9]+]]
+; CHECK: [[EXIT1]]:
+; CHECK-NEXT: [[RES:%.*]] = phi i32 [ [[IV]], %[[LOOP]] ], [ -1, %[[LATCH]] ], [ [[TMP7]], %[[VECTOR_EARLY_EXIT]] ]
+; CHECK-NEXT: ret i32 [[RES]]
+;
+entry:
+ br label %loop
+
+loop:
+ %iv = phi i32 [ 0, %entry ], [ %iv.next, %latch ]
+ %gep = getelementptr inbounds i32, ptr %p, i32 %iv
+ %ld = load i32, ptr %gep
+ %c = icmp eq i32 %ld, 0
+ br i1 %c, label %exit, label %latch
+
+latch:
+ %iv.next = add nuw nsw i32 %iv, 1
+ %ec = icmp eq i32 %iv.next, 123
+ br i1 %ec, label %exit, label %loop
+
+exit:
+ %res = phi i32 [ %iv, %loop ], [ -1, %latch ]
+ ret i32 %res
+}
+
+define i32 @multi_early_exits_same_exits(ptr align 4 dereferenceable(4096) %p, ptr align 4 dereferenceable(4096) %q) {
+; CHECK-LABEL: define i32 @multi_early_exits_same_exits(
+; CHECK-SAME: ptr align 4 dereferenceable(4096) [[P:%.*]], ptr align 4 dereferenceable(4096) [[Q:%.*]]) {
+; CHECK-NEXT: [[ENTRY:.*:]]
+; CHECK-NEXT: br label %[[VECTOR_PH:.*]]
+; CHECK: [[VECTOR_PH]]:
+; CHECK-NEXT: br label %[[VECTOR_BODY:.*]]
+; CHECK: [[VECTOR_BODY]]:
+; CHECK-NEXT: [[INDEX:%.*]] = phi i32 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY_INTERIM:.*]] ]
+; CHECK-NEXT: [[VEC_IND:%.*]] = phi <4 x i32> [ <i32 0, i32 1, i32 2, i32 3>, %[[VECTOR_PH]] ], [ [[VEC_IND_NEXT:%.*]], %[[VECTOR_BODY_INTERIM]] ]
+; CHECK-NEXT: [[TMP0:%.*]] = getelementptr inbounds i32, ptr [[P]], i32 [[INDEX]]
+; CHECK-NEXT: [[WIDE_LOAD:%.*]] = load <4 x i32>, ptr [[TMP0]], align 4
+; CHECK-NEXT: [[TMP1:%.*]] = icmp eq <4 x i32> [[WIDE_LOAD]], zeroinitializer
+; CHECK-NEXT: [[TMP2:%.*]] = getelementptr inbounds i32, ptr [[Q]], i32 [[INDEX]]
+; CHECK-NEXT: [[WIDE_LOAD1:%.*]] = load <4 x i32>, ptr [[TMP2]], align 4
+; CHECK-NEXT: [[TMP3:%.*]] = icmp eq <4 x i32> [[WIDE_LOAD1]], zeroinitializer
+; CHECK-NEXT: [[TMP4:%.*]] = add <4 x i32> [[VEC_IND]], splat (i32 1000)
+; CHECK-NEXT: [[TMP5:%.*]] = select <4 x i1> [[TMP1]], <4 x i1> splat (i1 true), <4 x i1> [[TMP3]]
+; CHECK-NEXT: [[TMP6:%.*]] = freeze <4 x i1> [[TMP5]]
+; CHECK-NEXT: [[TMP7:%.*]] = call i1 @llvm.vector.reduce.or.v4i1(<4 x i1> [[TMP6]])
+; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i32 [[INDEX]], 4
+; CHECK-NEXT: [[TMP8:%.*]] = icmp eq i32 [[INDEX_NEXT]], 120
+; CHECK-NEXT: [[VEC_IND_NEXT]] = add nuw nsw <4 x i32> [[VEC_IND]], splat (i32 4)
+; CHECK-NEXT: br i1 [[TMP7]], label %[[VECTOR_EARLY_EXIT_CHECK:.*]], label %[[VECTOR_BODY_INTERIM]]
+; CHECK: [[VECTOR_BODY_INTERIM]]:
+; CHECK-NEXT: br i1 [[TMP8]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP4:![0-9]+]]
+; CHECK: [[MIDDLE_BLOCK]]:
+; CHECK-NEXT: br label %[[EXIT:.*]]
+; CHECK: [[VECTOR_EARLY_EXIT_CHECK]]:
+; CHECK-NEXT: [[FIRST_ACTIVE_LANE:%.*]] = call i64 @llvm.experimental.cttz.elts.i64.v4i1(<4 x i1> [[TMP6]], i1 false)
+; CHECK-NEXT: [[TMP9:%.*]] = extractelement <4 x i1> [[TMP1]], i64 [[FIRST_ACTIVE_LANE]]
+; CHECK-NEXT: br i1 [[TMP9]], label %[[VECTOR_EARLY_EXIT_0:.*]], label %[[VECTOR_EARLY_EXIT_1:.*]]
+; CHECK: [[VECTOR_EARLY_EXIT_1]]:
+; CHECK-NEXT: [[TMP10:%.*]] = extractelement <4 x i32> [[TMP4]], i64 [[FIRST_ACTIVE_LANE]]
+; CHECK-NEXT: br label %[[EXIT1:.*]]
+; CHECK: [[VECTOR_EARLY_EXIT_0]]:
+; CHECK-NEXT: [[TMP11:%.*]] = trunc i64 [[FIRST_ACTIVE_LANE]] to i32
+; CHECK-NEXT: [[TMP12:%.*]] = add i32 [[INDEX]], [[TMP11]]
+; CHECK-NEXT: br label %[[EXIT1]]
+; CHECK: [[EXIT]]:
+; CHECK-NEXT: br label %[[LOOP:.*]]
+; CHECK: [[LOOP]]:
+; CHECK-NEXT: [[IV:%.*]] = phi i32 [ 120, %[[EXIT]] ], [ [[IV_NEXT:%.*]], %[[LATCH:.*]] ]
+; CHECK-NEXT: [[GEP_P:%.*]] = getelementptr inbounds i32, ptr [[P]], i32 [[IV]]
+; CHECK-NEXT: [[X:%.*]] = load i32, ptr [[GEP_P]], align 4
+; CHECK-NEXT: [[C1:%.*]] = icmp eq i32 [[X]], 0
+; CHECK-NEXT: br i1 [[C1]], label %[[EXIT1]], label %[[NEXT:.*]]
+; CHECK: [[NEXT]]:
+; CHECK-NEXT: [[GEP_Q:%.*]] = getelementptr inbounds i32, ptr [[Q]], i32 [[IV]]
+; CHECK-NEXT: [[Y:%.*]] = load i32, ptr [[GEP_Q]], align 4
+; CHECK-NEXT: [[C2:%.*]] = icmp eq i32 [[Y]], 0
+; CHECK-NEXT: [[IV_Q:%.*]] = add i32 [[IV]], 1000
+; CHECK-NEXT: br i1 [[C2]], label %[[EXIT1]], label %[[LATCH]]
+; CHECK: [[LATCH]]:
+; CHECK-NEXT: [[IV_NEXT]] = add nuw nsw i32 [[IV]], 1
+; CHECK-NEXT: [[EC:%.*]] = icmp eq i32 [[IV_NEXT]], 123
+; CHECK-NEXT: br i1 [[EC]], label %[[EXIT1]], label %[[LOOP]], !llvm.loop [[LOOP5:![0-9]+]]
+; CHECK: [[EXIT1]]:
+; CHECK-NEXT: [[RES:%.*]] = phi i32 [ [[IV]], %[[LOOP]] ], [ [[IV_Q]], %[[NEXT]] ], [ -1, %[[LATCH]] ], [ [[TMP12]], %[[VECTOR_EARLY_EXIT_0]] ], [ [[TMP10]], %[[VECTOR_EARLY_EXIT_1]] ]
+; CHECK-NEXT: ret i32 [[RES]]
+;
+entry:
+ br label %loop
+
+loop:
+ %iv = phi i32 [ 0, %entry ], [ %iv.next, %latch ]
+ %gep.p = getelementptr inbounds i32, ptr %p, i32 %iv
+ %x = load i32, ptr %gep.p, align 4
+ %c1 = icmp eq i32 %x, 0
+ br i1 %c1, label %exit, label %next
+
+next:
+ %gep.q = getelementptr inbounds i32, ptr %q, i32 %iv
+ %y = load i32, ptr %gep.q, align 4
+ %c2 = icmp eq i32 %y, 0
+ %iv.q = add i32 %iv, 1000
+ br i1 %c2, label %exit, label %latch
+
+latch:
+ %iv.next = add nuw nsw i32 %iv, 1
+ %ec = icmp eq i32 %iv.next, 123
+ br i1 %ec, label %exit, label %loop
+
+exit:
+ %res = phi i32 [ %iv, %loop ], [ %iv.q, %next ], [ -1, %latch ]
+ ret i32 %res
+}
+
+define i32 @multi_early_exits_different_exits(ptr align 4 dereferenceable(4096) %p, ptr align 4 dereferenceable(4096) %q) {
+; CHECK-LABEL: define i32 @multi_early_exits_different_exits(
+; CHECK-SAME: ptr align 4 dereferenceable(4096) [[P:%.*]], ptr align 4 dereferenceable(4096) [[Q:%.*]]) {
+; CHECK-NEXT: [[ENTRY:.*:]]
+; CHECK-NEXT: br label %[[VECTOR_PH:.*]]
+; CHECK: [[VECTOR_PH]]:
+; CHECK-NEXT: br label %[[VECTOR_BODY:.*]]
+; CHECK: [[VECTOR_BODY]]:
+; CHECK-NEXT: [[INDEX:%.*]] = phi i32 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY_INTERIM:.*]] ]
+; CHECK-NEXT: [[VEC_IND:%.*]] = phi <4 x i32> [ <i32 0, i32 1, i32 2, i32 3>, %[[VECTOR_PH]] ], [ [[VEC_IND_NEXT:%.*]], %[[VECTOR_BODY_INTERIM]] ]
+; CHECK-NEXT: [[TMP0:%.*]] = getelementptr inbounds i32, ptr [[P]], i32 [[INDEX]]
+; CHECK-NEXT: [[WIDE_LOAD:%.*]] = load <4 x i32>, ptr [[TMP0]], align 4
+; CHECK-NEXT: [[TMP1:%.*]] = icmp eq <4 x i32> [[WIDE_LOAD]], zeroinitializer
+; CHECK-NEXT: [[TMP2:%.*]] = getelementptr inbounds i32, ptr [[Q]], i32 [[INDEX]]
+; CHECK-NEXT: [[WIDE_LOAD1:%.*]] = load <4 x i32>, ptr [[TMP2]], align 4
+; CHECK-NEXT: [[TMP3:%.*]] = icmp eq <4 x i32> [[WIDE_LOAD1]], zeroinitializer
+; CHECK-NEXT: [[TMP4:%.*]] = add <4 x i32> [[VEC_IND]], splat (i32 1000)
+; CHECK-NEXT: [[TMP5:%.*]] = select <4 x i1> [[TMP1]], <4 x i1> splat (i1 true), <4 x i1> [[TMP3]]
+; CHECK-NEXT: [[TMP6:%.*]] = freeze <4 x i1> [[TMP5]]
+; CHECK-NEXT: [[TMP7:%.*]] = call i1 @llvm.vector.reduce.or.v4i1(<4 x i1> [[TMP6]])
+; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i32 [[INDEX]], 4
+; CHECK-NEXT: [[TMP8:%.*]] = icmp eq i32 [[INDEX_NEXT]], 120
+; CHECK-NEXT: [[VEC_IND_NEXT]] = add nuw nsw <4 x i32> [[VEC_IND]], splat (i32 4)
+; CHECK-NEXT: br i1 [[TMP7]], label %[[VECTOR_EARLY_EXIT_CHECK:.*]], label %[[VECTOR_BODY_INTERIM]]
+; CHECK: [[VECTOR_BODY_INTERIM]]:
+; CHECK-NEXT: br i1 [[TMP8]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP6:![0-9]+]]
+; CHECK: [[MIDDLE_BLOCK]]:
+; CHECK-NEXT: br label %[[EXIT_LATCH:.*]]
+; CHECK: [[VECTOR_EARLY_EXIT_CHECK]]:
+; CHECK-NEXT: [[FIRST_ACTIVE_LANE:%.*]] = call i64 @llvm.experimental.cttz.elts.i64.v4i1(<4 x i1> [[TMP6]], i1 false)
+; CHECK-NEXT: [[TMP9:%.*]] = extractelement <4 x i1> [[TMP1]], i64 [[FIRST_ACTIVE_LANE]]
+; CHECK-NEXT: br i1 [[TMP9]], label %[[VECTOR_EARLY_EXIT_0:.*]], label %[[VECTOR_EARLY_EXIT_1:.*]]
+; CHECK: [[VECTOR_EARLY_EXIT_1]]:
+; CHECK-NEXT: [[TMP10:%.*]] = extractelement <4 x i32> [[TMP4]], i64 [[FIRST_ACTIVE_LANE]]
+; CHECK-NEXT: br label %[[EXIT_Q:.*]]
+; CHECK: [[VECTOR_EARLY_EXIT_0]]:
+; CHECK-NEXT: [[TMP11:%.*]] = trunc i64 [[FIRST_ACTIVE_LANE]] to i32
+; CHECK-NEXT: [[TMP12:%.*]] = add i32 [[INDEX]], [[TMP11]]
+; CHECK-NEXT: br label %[[EXIT_P:.*]]
+; CHECK: [[EXIT_LATCH]]:
+; CHECK-NEXT: br label %[[LOOP:.*]]
+; CHECK: [[LOOP]]:
+; CHECK-NEXT: [[IV:%.*]] = phi i32 [ 120, %[[EXIT_LATCH]] ], [ [[IV_NEXT:%.*]], %[[LATCH:.*]] ]
+; CHECK-NEXT: [[GEP_P:%.*]] = getelementptr inbounds i32, ptr [[P]], i32 [[IV]]
+; CHECK-NEXT: [[X:%.*]] = load i32, ptr [[GEP_P]], align 4
+; CHECK-NEXT: [[C1:%.*]] = icmp eq i32 [[X]], 0
+; CHECK-NEXT: br i1 [[C1]], label %[[EXIT_P]], label %[[NEXT:.*]]
+; CHECK: [[NEXT]]:
+; CHECK-NEXT: [[GEP_Q:%.*]] = getelementptr inbounds i32, ptr [[Q]], i32 [[IV]]
+; CHECK-NEXT: [[Y:%.*]] = load i32, ptr [[GEP_Q]], align 4
+; CHECK-NEXT: [[C2:%.*]] = icmp eq i32 [[Y]], 0
+; CHECK-NEXT: [[IV_Q:%.*]] = add i32 [[IV]], 1000
+; CHECK-NEXT: br i1 [[C2]], label %[[EXIT_Q]], label %[[LATCH]]
+; CHECK: [[LATCH]]:
+; CHECK-NEXT: [[IV_NEXT]] = add nuw nsw i32 [[IV]], 1
+; CHECK-NEXT: [[EC:%.*]] = icmp eq i32 [[IV_NEXT]], 123
+; CHECK-NEXT: br i1 [[EC]], label %[[EXIT_LATCH1:.*]], label %[[LOOP]], !llvm.loop [[LOOP7:![0-9]+]]
+; CHECK: [[EXIT_P]]:
+; CHECK-NEXT: [[RES_P:%.*]] = phi i32 [ [[IV]], %[[LOOP]] ], [ [[TMP12]], %[[VECTOR_EARLY_EXIT_0]] ]
+; CHECK-NEXT: ret i32 [[RES_P]]
+; CHECK: [[EXIT_Q]]:
+; CHECK-NEXT: [[RES_Q:%.*]] = phi i32 [ [[IV_Q]], %[[NEXT]] ], [ [[TMP10]], %[[VECTOR_EARLY_EXIT_1]] ]
+; CHECK-NEXT: ret i32 [[RES_Q]]
+; CHECK: [[EXIT_LATCH1]]:
+; CHECK-NEXT: ret i32 -1
+;
+entry:
+ br label %loop
+
+loop:
+ %iv = phi i32 [ 0, %entry ], [ %iv.next, %latch ]
+ %gep.p = getelementptr inbounds i32, ptr %p, i32 %iv
+ %x = load i32, ptr %gep.p, align 4
+ %c1 = icmp eq i32 %x, 0
+ br i1 %c1, label %exit.p, label %next
+
+next:
+ %gep.q = getelementptr inbounds i32, ptr %q, i32 %iv
+ %y = load i32, ptr %gep.q, align 4
+ %c2 = icmp eq i32 %y, 0
+ %iv.q = add i32 %iv, 1000
+ br i1 %c2, label %exit.q, label %latch
+
+latch:
+ %iv.next = add nuw nsw i32 %iv, 1
+ %ec = icmp eq i32 %iv.next, 123
+ br i1 %ec, label %exit.latch, label %loop
+
+exit.p:
+ %res.p = phi i32 [ %iv, %loop ]
+ ret i32 %res.p
+
+exit.q:
+ %res.q = phi i32 [ %iv.q, %next ]
+ ret i32 %res.q
+
+exit.latch:
+ ret i32 -1
+}
+
+define i32 @predicated_early_exit(i1 %c0, ptr align 4 dereferenceable(4096) %p) {
+; CHECK-LABEL: define i32 @predicated_early_exit(
+; CHECK-SAME: i1 [[C0:%.*]], ptr align 4 dereferenceable(4096) [[P:%.*]]) {
+; CHECK-NEXT: [[ENTRY:.*:]]
+; CHECK-NEXT: br label %[[VECTOR_PH:.*]]
+; CHECK: [[VECTOR_PH]]:
+; CHECK-NEXT: [[BROADCAST_SPLATINSERT:%.*]] = insertelement <4 x i1> poison, i1 [[C0]], i64 0
+; CHECK-NEXT: [[BROADCAST_SPLAT:%.*]] = shufflevector <4 x i1> [[BROADCAST_SPLATINSERT]], <4 x i1> poison, <4 x i32> zeroinitializer
+; CHECK-NEXT: [[TMP9:%.*]] = freeze <4 x i1> [[BROADCAST_SPLAT]]
+; CHECK-NEXT: br label %[[VECTOR_BODY:.*]]
+; CHECK: [[VECTOR_BODY]]:
+; CHECK-NEXT: [[INDEX:%.*]] = phi i32 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY_INTERIM:.*]] ]
+; CHECK-NEXT: [[TMP0:%.*]] = getelementptr inbounds i32, ptr [[P]], i32 [[INDEX]]
+; CHECK-NEXT: [[WIDE_LOAD:%.*]] = load <4 x i32>, ptr [[TMP0]], align 4
+; CHECK-NEXT: [[TMP1:%.*]] = icmp eq <4 x i32> [[WIDE_LOAD]], zeroinitializer
+; CHECK-NEXT: [[TMP2:%.*]] = select <4 x i1> [[TMP9]], <4 x i1> [[TMP1]], <4 x i1> zeroinitializer
+; CHECK-NEXT: [[TMP3:%.*]] = freeze <4 x i1> [[TMP2]]
+; CHECK-NEXT: [[TMP4:%.*]] = call i1 @llvm.vector.reduce.or.v4i1(<4 x i1> [[TMP3]])
+; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i32 [[INDEX]], 4
+; CHECK-NEXT: [[TMP5:%.*]] = icmp eq i32 [[INDEX_NEXT]], 120
+; CHECK-NEXT: br i1 [[TMP4]], label %[[VECTOR_EARLY_EXIT:.*]], label %[[VECTOR_BODY_INTERIM]]
+; CHECK: [[VECTOR_BODY_INTERIM]]:
+; CHECK-NEXT: br i1 [[TMP5]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP8:![0-9]+]]
+; CHECK: [[MIDDLE_BLOCK]]:
+; CHECK-NEXT: br label %[[EXIT:.*]]
+; CHECK: [[VECTOR_EARLY_EXIT]]:
+; CHECK-NEXT: [[TMP6:%.*]] = call i64 @llvm.experimental.cttz.elts.i64.v4i1(<4 x i1> [[TMP3]], i1 false)
+; CHECK-NEXT: [[TMP7:%.*]] = trunc i64 [[TMP6]] to i32
+; CHECK-NEXT: [[TMP8:%.*]] = add i32 [[INDEX]], [[TMP7]]
+; CHECK-NEXT: br label %[[EXIT1:.*]]
+; CHECK: [[EXIT]]:
+; CHECK-NEXT: br label %[[LOOP:.*]]
+; CHECK: [[LOOP]]:
+; CHECK-NEXT: [[IV:%.*]] = phi i32 [ 120, %[[EXIT]] ], [ [[IV_NEXT:%.*]], %[[LATCH:.*]] ]
+; CHECK-NEXT: br i1 [[C0]], label %[[THEN:.*]], label %[[LATCH]]
+; CHECK: [[THEN]]:
+; CHECK-NEXT: [[GEP:%.*]] = getelementptr inbounds i32, ptr [[P]], i32 [[IV]]
+; CHECK-NEXT: [[LD:%.*]] = load i32, ptr [[GEP]], align 4
+; CHECK-NEXT: [[C1:%.*]] = icmp eq i32 [[LD]], 0
+; CHECK-NEXT: br i1 [[C1]], label %[[EXIT1]], label %[[LATCH]]
+; CHECK: [[LATCH]]:
+; CHECK-NEXT: [[IV_NEXT]] = add nuw nsw i32 [[IV]], 1
+; CHECK-NEXT: [[EC:%.*]] = icmp eq i32 [[IV_NEXT]], 123
+; CHECK-NEXT: br i1 [[EC]], label %[[EXIT1]], label %[[LOOP]], !llvm.loop [[LOOP9:![0-9]+]]
+; CHECK: [[EXIT1]]:
+; CHECK-NEXT: [[RES:%.*]] = phi i32 [ [[IV]], %[[THEN]] ], [ -1, %[[LATCH]] ], [ [[TMP8]], %[[VECTOR_EARLY_EXIT]] ]
+; CHECK-NEXT: ret i32 [[RES]]
+;
+entry:
+ br label %loop
+
+loop:
+ %iv = phi i32 [ 0, %entry ], [ %iv.next, %latch ]
+ br i1 %c0, label %then, label %latch
+
+then:
+ %gep = getelementptr inbounds i32, ptr %p, i32 %iv
+ %ld = load i32, ptr %gep
+ %c1 = icmp eq i32 %ld, 0
+ br i1 %c1, label %exit, label %latch
+
+latch:
+ %iv.next = add nuw nsw i32 %iv, 1
+ %ec = icmp eq i32 %iv.next, 123
+ br i1 %ec, label %exit, label %loop
+
+exit:
+ %res = phi i32 [ %iv, %then ], [ -1, %latch ]
+ ret i32 %res
+}
+
+; TODO: Cannot tail fold yet.
+define i32 @countable_early_exit(ptr align 4 dereferenceable(4096) %p, i32 %n) {
+; CHECK-LABEL: define i32 @countable_early_exit(
+; CHECK-SAME: ptr align 4 dereferenceable(4096) [[P:%.*]], i32 [[N:%.*]]) {
+; CHECK-NEXT: [[ENTRY:.*]]:
+; CHECK-NEXT: [[TMP0:%.*]] = call i32 @llvm.umin.i32(i32 [[N]], i32 122)
+; CHECK-NEXT: [[TMP1:%.*]] = add nuw nsw i32 [[TMP0]], 1
+; CHECK-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ule i32 [[TMP1]], 4
+; CHECK-NEXT: br i1 [[MIN_ITERS_CHECK]], label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]]
+; CHECK: [[VECTOR_PH]]:
+; CHECK-NEXT: [[TMP2:%.*]] = and i32 [[TMP1]], 3
+; CHECK-NEXT: [[TMP3:%.*]] = icmp eq i32 [[TMP2]], 0
+; CHECK-NEXT: [[TMP4:%.*]] = select i1 [[TMP3]], i32 4, i32 [[TMP2]]
+; CHECK-NEXT: [[N_VEC:%.*]] = sub i32 [[TMP1]], [[TMP4]]
+; CHECK-NEXT: br label %[[VECTOR_BODY:.*]]
+; CHECK: [[VECTOR_BODY]]:
+; CHECK-NEXT: [[INDEX:%.*]] = phi i32 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i32 [[INDEX]], 4
+; CHECK-NEXT: [[TMP5:%.*]] = icmp eq i32 [[INDEX_NEXT]], [[N_VEC]]
+; CHECK-NEXT: br i1 [[TMP5]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP10:![0-9]+]]
+; CHECK: [[MIDDLE_BLOCK]]:
+; CHECK-NEXT: br label %[[SCALAR_PH]]
+; CHECK: [[SCALAR_PH]]:
+; CHECK-NEXT: [[BC_RESUME_VAL:%.*]] = phi i32 [ [[N_VEC]], %[[MIDDLE_BLOCK]] ], [ 0, %[[ENTRY]] ]
+; CHECK-NEXT: br label %[[LOOP:.*]]
+; CHECK: [[LOOP]]:
+; CHECK-NEXT: [[IV:%.*]] = phi i32 [ [[BC_RESUME_VAL]], %[[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], %[[LATCH:.*]] ]
+; CHECK-NEXT: [[C:%.*]] = icmp eq i32 [[IV]], [[N]]
+; CHECK-NEXT: br i1 [[C]], label %[[EXIT:.*]], label %[[LATCH]]
+; CHECK: [[LATCH]]:
+; CHECK-NEXT: [[GEP:%.*]] = getelementptr inbounds i32, ptr [[P]], i32 [[IV]]
+; CHECK-NEXT: [[LD:%.*]] = load i32, ptr [[GEP]], align 4
+; CHECK-NEXT: [[IV_NEXT]] = add nuw nsw i32 [[IV]], 1
+; CHECK-NEXT: [[EC:%.*]] = icmp eq i32 [[IV_NEXT]], 123
+; CHECK-NEXT: br i1 [[EC]], label %[[EXIT]], label %[[LOOP]], !llvm.loop [[LOOP11:![0-9]+]]
+; CHECK: [[EXIT]]:
+; CHECK-NEXT: [[RES:%.*]] = phi i32 [ 0, %[[LOOP]] ], [ [[LD]], %[[LATCH]] ]
+; CHECK-NEXT: ret i32 [[RES]]
+;
+entry:
+ br label %loop
+
+loop:
+ %iv = phi i32 [ 0, %entry ], [ %iv.next, %latch ]
+ %c = icmp eq i32 %iv, %n
+ br i1 %c, label %exit, label %latch
+
+latch:
+ %gep = getelementptr inbounds i32, ptr %p, i32 %iv
+ %ld = load i32, ptr %gep, align 4
+ %iv.next = add nuw nsw i32 %iv, 1
+ %ec = icmp eq i32 %iv.next, 123
+ br i1 %ec, label %exit, label %loop
+
+exit:
+ %res = phi i32 [ 0, %loop ], [ %ld, %latch ]
+ ret i32 %res
+}
+
+; TODO: Cannot tail fold yet.
+define void @early_exit_with_store(ptr align 4 dereferenceable(4096) noalias %dst, ptr align 4 dereferenceable(4096) %pred) {
+; CHECK-LABEL: define void @early_exit_with_store(
+; CHECK-SAME: ptr noalias align 4 dereferenceable(4096) [[DST:%.*]], ptr align 4 dereferenceable(4096) [[PRED:%.*]]) {
+; CHECK-NEXT: [[ENTRY:.*:]]
+; CHECK-NEXT: br label %[[VECTOR_PH:.*]]
+; CHECK: [[VECTOR_PH]]:
+; CHECK-NEXT: br label %[[VECTOR_BODY:.*]]
+; CHECK: [[VECTOR_BODY]]:
+; CHECK-NEXT: [[INDEX:%.*]] = phi i32 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-NEXT: [[TMP0:%.*]] = getelementptr inbounds nuw i32, ptr [[PRED]], i32 [[INDEX]]
+; CHECK-NEXT: [[WIDE_LOAD:%.*]] = load <4 x i32>, ptr [[TMP0]], align 4
+; CHECK-NEXT: [[TMP1:%.*]] = icmp sgt <4 x i32> [[WIDE_LOAD]], splat (i32 500)
+; CHECK-NEXT: [[TMP2:%.*]] = freeze <4 x i1> [[TMP1]]
+; CHECK-NEXT: [[UNCOUNTABLE_EXIT_MASK:%.*]] = call <4 x i1> @llvm.mask.beforefirst.v4i1(<4 x i1> [[TMP2]])
+; CHECK-NEXT: [[TMP5:%.*]] = getelementptr i32, ptr [[DST]], i32 [[INDEX]]
+; CHECK-NEXT: [[WIDE_MASKED_LOAD:%.*]] = call <4 x i32> @llvm.masked.load.v4i32.p0(ptr align 4 [[TMP5]], <4 x i1> [[UNCOUNTABLE_EXIT_MASK]], <4 x i32> poison)
+; CHECK-NEXT: [[TMP6:%.*]] = add nsw <4 x i32> [[WIDE_MASKED_LOAD]], splat (i32 1)
+; CHECK-NEXT: call void @llvm.masked.store.v4i32.p0(<4 x i32> [[TMP6]], ptr align 4 [[TMP5]], <4 x i1> [[UNCOUNTABLE_EXIT_MASK]])
+; CHECK-NEXT: [[TMP7:%.*]] = call i1 @llvm.vector.reduce.or.v4i1(<4 x i1> [[TMP2]])
+; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i32 [[INDEX]], 4
+; CHECK-NEXT: [[TMP8:%.*]] = icmp eq i32 [[INDEX_NEXT]], 120
+; CHECK-NEXT: [[TMP9:%.*]] = or i1 [[TMP7]], [[TMP8]]
+; CHECK-NEXT: br i1 [[TMP9]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP12:![0-9]+]]
+; CHECK: [[MIDDLE_BLOCK]]:
+; CHECK-NEXT: [[TMP12:%.*]] = call i64 @llvm.experimental.cttz.elts.i64.v4i1(<4 x i1> [[TMP2]], i1 false)
+; CHECK-NEXT: [[TMP4:%.*]] = trunc i64 [[TMP12]] to i32
+; CHECK-NEXT: [[TMP10:%.*]] = add i32 [[INDEX]], [[TMP4]]
+; CHECK-NEXT: [[TMP11:%.*]] = icmp eq i32 [[TMP10]], 123
+; CHECK-NEXT: br i1 [[TMP11]], label %[[EXIT:.*]], label %[[SCALAR_PH:.*]]
+; CHECK: [[SCALAR_PH]]:
+; CHECK-NEXT: br label %[[LOOP:.*]]
+; CHECK: [[LOOP]]:
+; CHECK-NEXT: [[IV:%.*]] = phi i32 [ [[TMP10]], %[[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], %[[LATCH:.*]] ]
+; CHECK-NEXT: [[ST_ADDR:%.*]] = getelementptr inbounds nuw i32, ptr [[DST]], i32 [[IV]]
+; CHECK-NEXT: [[DATA:%.*]] = load i32, ptr [[ST_ADDR]], align 4
+; CHECK-NEXT: [[INC:%.*]] = add nsw i32 [[DATA]], 1
+; CHECK-NEXT: store i32 [[INC]], ptr [[ST_ADDR]], align 4
+; CHECK-NEXT: [[EE_ADDR:%.*]] = getelementptr inbounds nuw i32, ptr [[PRED]], i32 [[IV]]
+; CHECK-NEXT: [[EE_VAL:%.*]] = load i32, ptr [[EE_ADDR]], align 4
+; CHECK-NEXT: [[EE_COND:%.*]] = icmp sgt i32 [[EE_VAL]], 500
+; CHECK-NEXT: br i1 [[EE_COND]], label %[[EXIT]], label %[[LATCH]]
+; CHECK: [[LATCH]]:
+; CHECK-NEXT: [[IV_NEXT]] = add nuw nsw i32 [[IV]], 1
+; CHECK-NEXT: [[EC:%.*]] = icmp eq i32 [[IV_NEXT]], 123
+; CHECK-NEXT: br i1 [[EC]], label %[[EXIT]], label %[[LOOP]], !llvm.loop [[LOOP13:![0-9]+]]
+; CHECK: [[EXIT]]:
+; CHECK-NEXT: ret void
+;
+entry:
+ br label %loop
+
+loop:
+ %iv = phi i32 [ 0, %entry ], [ %iv.next, %latch ]
+ %st.addr = getelementptr inbounds nuw i32, ptr %dst, i32 %iv
+ %data = load i32, ptr %st.addr, align 4
+ %inc = add nsw i32 %data, 1
+ store i32 %inc, ptr %st.addr, align 4
+ %ee.addr = getelementptr inbounds nuw i32, ptr %pred, i32 %iv
+ %ee.val = load i32, ptr %ee.addr, align 4
+ %ee.cond = icmp sgt i32 %ee.val, 500
+ br i1 %ee.cond, label %exit, label %latch
+
+latch:
+ %iv.next = add nuw nsw i32 %iv, 1
+ %ec = icmp eq i32 %iv.next, 123
+ br i1 %ec, label %exit, label %loop
+
+exit:
+ ret void
+}
+
+define void @early_exit_with_store_after_exit(ptr align 4 dereferenceable(4096) noalias %dst, ptr align 4 dereferenceable(4096) %pred) {
+; CHECK-LABEL: define void @early_exit_with_store_after_exit(
+; CHECK-SAME: ptr noalias align 4 dereferenceable(4096) [[DST:%.*]], ptr align 4 dereferenceable(4096) [[PRED:%.*]]) {
+; CHECK-NEXT: [[ENTRY:.*:]]
+; CHECK-NEXT: br label %[[VECTOR_PH:.*]]
+; CHECK: [[VECTOR_PH]]:
+; CHECK-NEXT: br label %[[VECTOR_BODY:.*]]
+; CHECK: [[VECTOR_BODY]]:
+; CHECK-NEXT: [[INDEX:%.*]] = phi i32 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY_INTERIM:.*]] ]
+; CHECK-NEXT: [[TMP0:%.*]] = getelementptr inbounds nuw i32, ptr [[PRED]], i32 [[INDEX]]
+; CHECK-NEXT: [[WIDE_LOAD:%.*]] = load <4 x i32>, ptr [[TMP0]], align 4
+; CHECK-NEXT: [[TMP1:%.*]] = icmp sgt <4 x i32> [[WIDE_LOAD]], splat (i32 500)
+; CHECK-NEXT: [[TMP2:%.*]] = freeze <4 x i1> [[TMP1]]
+; CHECK-NEXT: [[TMP3:%.*]] = call <4 x i1> @llvm.mask.beforefirst.v4i1(<4 x i1> [[TMP2]])
+; CHECK-NEXT: [[TMP4:%.*]] = getelementptr i32, ptr [[DST]], i32 [[INDEX]]
+; CHECK-NEXT: [[WIDE_MASKED_LOAD:%.*]] = call <4 x i32> @llvm.masked.load.v4i32.p0(ptr align 4 [[TMP4]], <4 x i1> [[TMP3]], <4 x i32> poison)
+; CHECK-NEXT: [[TMP5:%.*]] = add nsw <4 x i32> [[WIDE_MASKED_LOAD]], splat (i32 1)
+; CHECK-NEXT: call void @llvm.masked.store.v4i32.p0(<4 x i32> [[TMP5]], ptr align 4 [[TMP4]], <4 x i1> [[TMP3]])
+; CHECK-NEXT: [[TMP6:%.*]] = call i1 @llvm.vector.reduce.or.v4i1(<4 x i1> [[TMP2]])
+; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i32 [[INDEX]], 4
+; CHECK-NEXT: [[TMP7:%.*]] = icmp eq i32 [[INDEX_NEXT]], 120
+; CHECK-NEXT: br i1 [[TMP6]], label %[[VECTOR_EARLY_EXIT:.*]], label %[[VECTOR_BODY_INTERIM]]
+; CHECK: [[VECTOR_BODY_INTERIM]]:
+; CHECK-NEXT: br i1 [[TMP7]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP14:![0-9]+]]
+; CHECK: [[MIDDLE_BLOCK]]:
+; CHECK-NEXT: br label %[[SCALAR_PH:.*]]
+; CHECK: [[VECTOR_EARLY_EXIT]]:
+; CHECK-NEXT: br label %[[EXIT:.*]]
+; CHECK: [[SCALAR_PH]]:
+; CHECK-NEXT: br label %[[LOOP:.*]]
+; CHECK: [[LOOP]]:
+; CHECK-NEXT: [[IV:%.*]] = phi i32 [ 120, %[[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], %[[LATCH:.*]] ]
+; CHECK-NEXT: [[EE_ADDR:%.*]] = getelementptr inbounds nuw i32, ptr [[PRED]], i32 [[IV]]
+; CHECK-NEXT: [[EE_VAL:%.*]] = load i32, ptr [[EE_ADDR]], align 4
+; CHECK-NEXT: [[EE_COND:%.*]] = icmp sgt i32 [[EE_VAL]], 500
+; CHECK-NEXT: br i1 [[EE_COND]], label %[[EXIT]], label %[[LATCH]]
+; CHECK: [[LATCH]]:
+; CHECK-NEXT: [[ST_ADDR:%.*]] = getelementptr inbounds nuw i32, ptr [[DST]], i32 [[IV]]
+; CHECK-NEXT: [[DATA:%.*]] = load i32, ptr [[ST_ADDR]], align 4
+; CHECK-NEXT: [[INC:%.*]] = add nsw i32 [[DATA]], 1
+; CHECK-NEXT: store i32 [[INC]], ptr [[ST_ADDR]], align 4
+; CHECK-NEXT: [[IV_NEXT]] = add nuw nsw i32 [[IV]], 1
+; CHECK-NEXT: [[EC:%.*]] = icmp eq i32 [[IV_NEXT]], 123
+; CHECK-NEXT: br i1 [[EC]], label %[[EXIT]], label %[[LOOP]], !llvm.loop [[LOOP15:![0-9]+]]
+; CHECK: [[EXIT]]:
+; CHECK-NEXT: ret void
+;
+entry:
+ br label %loop
+
+loop:
+ %iv = phi i32 [ 0, %entry ], [ %iv.next, %latch ]
+ %ee.addr = getelementptr inbounds nuw i32, ptr %pred, i32 %iv
+ %ee.val = load i32, ptr %ee.addr
+ %ee.cond = icmp sgt i32 %ee.val, 500
+ br i1 %ee.cond, label %exit, label %latch
+
+latch:
+ %st.addr = getelementptr inbounds nuw i32, ptr %dst, i32 %iv
+ %data = load i32, ptr %st.addr
+ %inc = add nsw i32 %data, 1
+ store i32 %inc, ptr %st.addr
+ %iv.next = add nuw nsw i32 %iv, 1
+ %ec = icmp eq i32 %iv.next, 123
+ br i1 %ec, label %exit, label %loop
+
+exit:
+ ret void
+}
>From 3df680bb2646e9d3d1a8b88051f28067efa7fed0 Mon Sep 17 00:00:00 2001
From: Luke Lau <luke at igalia.com>
Date: Sat, 31 Oct 2026 02:24:34 +0800
Subject: [PATCH 2/2] [VPlan] Support tail folding uncountable early exits
Enables the first step towards #179595. After #237254, the MaskedHandleExitInVectorLoop early exit style uses the CFG to handle masking for early exit loops. This allows it to compose with tail folding which is also handled via the CFG in #176143.
This allows us to tail fold the following loop:
```c
extern char x[256];
int f(char * restrict y) {
for (int i = 0; i < 256; i++) {
if (x[i] == 0)
return i;
}
return -1;
}
```
Into this on RISC-V:
```asm
.LBB0_1:
vsetvli a4, a1, e8, m2, ta, ma
add a5, a2, a0
vle8.v v10, (a5)
vmseq.vi v8, v10, 0
vcpop.m a5, v8
bnez a5, .LBB0_4
sub a1, a1, a4
add a0, a0, a4
bnez a1, .LBB0_1
```
foldTailByMasking is updated to make sure the combined early exit condition is false on the lanes past the header mask.
And a fold was added to turn the VPInstruction::AnyOf(header-mask && exit-cond) in the vector body into a vp.reduce.or intrinsic.
This PR only enables tail folding for early exit loops with no side effects. Side effects will be enabled in the next PR. data-and-control-flow tail folding styles are also not handled in this PR.
---
.../Vectorize/LoopVectorizationLegality.cpp | 20 +-
.../Transforms/Vectorize/LoopVectorize.cpp | 8 +
llvm/lib/Transforms/Vectorize/VPlan.h | 7 +
.../Vectorize/VPlanConstruction.cpp | 50 ++--
.../Vectorize/VPlanEVLTailFolding.cpp | 5 +
.../RISCV/early-exit-live-out.ll | 249 +++++++-----------
.../LoopVectorize/RISCV/early-exit.ll | 226 ++++++----------
.../LoopVectorize/RISCV/sink-to-early-exit.ll | 57 ++--
.../LoopVectorize/early-exit-tail-folding.ll | 159 ++++-------
.../LoopVectorize/multiple-early-exits.ll | 118 ++++++++-
10 files changed, 408 insertions(+), 491 deletions(-)
diff --git a/llvm/lib/Transforms/Vectorize/LoopVectorizationLegality.cpp b/llvm/lib/Transforms/Vectorize/LoopVectorizationLegality.cpp
index 6e7a7feff12e29..0cd4b7ec2ffb45 100644
--- a/llvm/lib/Transforms/Vectorize/LoopVectorizationLegality.cpp
+++ b/llvm/lib/Transforms/Vectorize/LoopVectorizationLegality.cpp
@@ -2104,17 +2104,21 @@ bool LoopVectorizationLegality::canFoldTailByMasking() const {
// a bottom-test and a single exiting block. We'd have to handle the fact
// that not every instruction executes on the last iteration. This will
// require a lane mask which varies through the vector loop body. (TODO)
- if (TheLoop->getExitingBlock() != TheLoop->getLoopLatch()) {
- LLVM_DEBUG(
- dbgs()
- << "LV: Cannot fold tail by masking. Requires a singe latch exit\n");
+ bool LatchIsOnlyCountableExit =
+ hasUncountableEarlyExit()
+ ? all_of(getCountableExitingBlocks(),
+ equal_to(TheLoop->getLoopLatch()))
+ : TheLoop->getExitingBlock() == TheLoop->getLoopLatch();
+ if (!LatchIsOnlyCountableExit) {
+ LLVM_DEBUG(dbgs() << "LV: Cannot fold tail by masking. Requires the latch "
+ "to be the only countable exit\n");
return false;
}
- // TODO: Support tail folding with uncountable exits.
- if (hasUncountableEarlyExit()) {
- LLVM_DEBUG(dbgs() << "LV: Cannot tail fold by masking. Loop contains an "
- "uncountable early exit.\n");
+ // TODO: Handle early exits with side effects without a scalar tail.
+ if (hasUncountableExitWithSideEffects()) {
+ LLVM_DEBUG(dbgs() << "LV: Cannot fold tail by masking. Uncountable exit "
+ "with side effects requires a scalar epilogue\n");
return false;
}
diff --git a/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp b/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
index fb082825a3fc2b..e38995f42ff5b4 100644
--- a/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
+++ b/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
@@ -715,6 +715,14 @@ class LoopVectorizationCostModel {
ChosenTailFoldingStyle = Opts.force_tail_folding_style.value_or(
TTI.getPreferredTailFoldingStyle());
+ // TODO: Support uncountable early exits w/ DataAndControlFlow. Both
+ // adjust the latch terminator structure.
+ if (ChosenTailFoldingStyle == TailFoldingStyle::DataAndControlFlow &&
+ Legal->hasUncountableEarlyExit()) {
+ ChosenTailFoldingStyle = TailFoldingStyle::None;
+ return;
+ }
+
if (ChosenTailFoldingStyle != TailFoldingStyle::DataWithEVL)
return;
// Override EVL styles if needed.
diff --git a/llvm/lib/Transforms/Vectorize/VPlan.h b/llvm/lib/Transforms/Vectorize/VPlan.h
index 0e322a13054370..867d85ac72c662 100644
--- a/llvm/lib/Transforms/Vectorize/VPlan.h
+++ b/llvm/lib/Transforms/Vectorize/VPlan.h
@@ -3448,6 +3448,13 @@ class LLVM_ABI_FOR_TEST VPReductionEVLRecipe : public VPReductionRecipe {
R.getVFScaleFactor()),
DL) {}
+ VPReductionEVLRecipe(const RecurKind RdxKind, FastMathFlags FMFs,
+ VPValue *ChainOp, VPValue *VecOp, VPValue *CondOp,
+ VPValue *EVL, ReductionStyle Style,
+ DebugLoc DL = DebugLoc::getUnknown())
+ : VPReductionRecipe(VPRecipeBase::VPReductionSC, RdxKind, FMFs, nullptr,
+ {ChainOp, VecOp, EVL}, CondOp, Style, DL) {}
+
~VPReductionEVLRecipe() override = default;
VPReductionEVLRecipe *clone() override {
diff --git a/llvm/lib/Transforms/Vectorize/VPlanConstruction.cpp b/llvm/lib/Transforms/Vectorize/VPlanConstruction.cpp
index ff009b49a79fa6..7c944c224f8080 100644
--- a/llvm/lib/Transforms/Vectorize/VPlanConstruction.cpp
+++ b/llvm/lib/Transforms/Vectorize/VPlanConstruction.cpp
@@ -1373,15 +1373,9 @@ void VPlanTransforms::createLoopRegions(VPlan &Plan, DebugLoc DL) {
}
void VPlanTransforms::foldTailByMasking(VPlan &Plan) {
- assert(Plan.getExitBlocks().size() == 1 &&
- "only a single-exit block is supported currently");
- assert(Plan.getExitBlocks().front()->getSinglePredecessor() ==
- Plan.getMiddleBlock() &&
- "the exit block must have middle block as single predecessor");
-
VPRegionBlock *LoopRegion = Plan.getVectorLoopRegion();
- assert(LoopRegion->getSingleSuccessor() == Plan.getMiddleBlock() &&
- "The vector loop region must have the middle block as its single "
+ assert(LoopRegion->getSuccessors().back() == Plan.getMiddleBlock() &&
+ "The vector loop region must have the middle block as its "
"successor for now");
VPBasicBlock *Header = LoopRegion->getEntryBasicBlock();
@@ -1393,21 +1387,39 @@ void VPlanTransforms::foldTailByMasking(VPlan &Plan) {
Builder.createNaryOp(VPInstruction::BranchOnCond, HeaderMask);
VPBasicBlock *OrigLatch = LoopRegion->getExitingBasicBlock();
- VPValue *IVInc = vputils::findCanonicalIVIncrement(Plan);
- assert(IVInc &&
- std::next(IVInc->getDefiningRecipe()->getIterator()) ==
- OrigLatch->getTerminator()->getIterator() &&
+ VPInstruction *IVInc = vputils::findCanonicalIVIncrement(Plan);
+ assert(IVInc && "Expected a canonical iv increment");
+ [[maybe_unused]] auto NextIt = std::next(IVInc->getIterator());
+ assert((NextIt == OrigLatch->getTerminator()->getIterator() ||
+ (match(&*NextIt,
+ m_SpecificICmp(CmpInst::ICMP_EQ, m_Specific(IVInc),
+ m_Specific(&Plan.getVectorTripCount()))) &&
+ match(&*std::next(NextIt),
+ m_BranchOnTwoConds(
+ m_VPValue(), m_Specific(NextIt->getVPSingleValue()))))) &&
"Unexpected canonical iv increment");
// Split the latch at the IV update, and branch to it from the header mask.
- VPBasicBlock *Latch =
- OrigLatch->splitAt(IVInc->getDefiningRecipe()->getIterator());
+ VPBasicBlock *Latch = OrigLatch->splitAt(IVInc->getIterator());
Latch->setName("vector.latch");
VPBlockUtils::connectBlocks(Header, Latch);
+ // The early exit condition is false on the tail lanes. Sink the any-of to
+ // the new latch, operating on a phi that is false when the header mask is.
+ VPSingleDefRecipe *AnyOf;
+ VPValue *AllExitConds;
+ if (match(Latch->getTerminator(),
+ m_BranchOnTwoConds(m_CombineAnd(m_VPSingleDefRecipe(AnyOf),
+ m_AnyOf(m_VPValue(AllExitConds))),
+ m_VPValue()))) {
+ Builder.setInsertPoint(Latch->begin());
+ VPPhi *Phi = Builder.createScalarPhi({AllExitConds, Plan.getFalse()});
+ AllExitConds->replaceUsesWithIf(Phi, not_equal_to(Phi));
+ AnyOf->moveBefore(IVInc->getIterator());
+ }
+
// Collect any values defined in the loop that need a phi, that is header phis
- // and any value defined n the loop and used in the middle block.
- // TODO: Handle early exits via Plan.getExitBlocks()
+ // and any value defined in the loop and used after the loop region.
MapVector<VPValue *, SmallVector<VPUser *>> NeedsPhi;
auto CollectBypassedOperands = [&](VPRecipeBase &R) {
for (VPValue *Op : R.operands()) {
@@ -1419,8 +1431,10 @@ void VPlanTransforms::foldTailByMasking(VPlan &Plan) {
};
for (VPRecipeBase &R : Header->phis())
CollectBypassedOperands(R);
- for (VPRecipeBase &R : *Plan.getMiddleBlock())
- CollectBypassedOperands(R);
+ for (VPBasicBlock *VPBB : VPBlockUtils::blocksOnly<VPBasicBlock>(
+ vp_depth_first_shallow(LoopRegion)))
+ for (VPRecipeBase &R : *VPBB)
+ CollectBypassedOperands(R);
// Insert phis for values coming past the end of the tail.
Builder.setInsertPoint(Latch->begin());
diff --git a/llvm/lib/Transforms/Vectorize/VPlanEVLTailFolding.cpp b/llvm/lib/Transforms/Vectorize/VPlanEVLTailFolding.cpp
index 62ef733d2899dc..bd6543c6f3f7e7 100644
--- a/llvm/lib/Transforms/Vectorize/VPlanEVLTailFolding.cpp
+++ b/llvm/lib/Transforms/Vectorize/VPlanEVLTailFolding.cpp
@@ -178,6 +178,11 @@ static VPRecipeBase *optimizeMaskToEVL(VPValue *HeaderMask,
match(Rdx->getCondOp(), m_RemoveMask(HeaderMask, Mask)))
return new VPReductionEVLRecipe(*Rdx, EVL, Mask);
+ if (match(&CurRecipe, m_AnyOf(m_RemoveMask(HeaderMask, Mask))))
+ return new VPReductionEVLRecipe(
+ RecurKind::Or, {}, Plan->getFalse(), Mask ? Mask : Plan->getTrue(),
+ nullptr, &EVL, getReductionStyle(true, false, 1), DL);
+
if (auto *Interleave = dyn_cast<VPInterleaveRecipe>(&CurRecipe))
if (Interleave->getMask() &&
match(Interleave->getMask(), m_RemoveMask(HeaderMask, Mask)))
diff --git a/llvm/test/Transforms/LoopVectorize/RISCV/early-exit-live-out.ll b/llvm/test/Transforms/LoopVectorize/RISCV/early-exit-live-out.ll
index 6878c499c94137..005f52876257b2 100644
--- a/llvm/test/Transforms/LoopVectorize/RISCV/early-exit-live-out.ll
+++ b/llvm/test/Transforms/LoopVectorize/RISCV/early-exit-live-out.ll
@@ -10,143 +10,104 @@
define i32 @early_exit_live_out(ptr align 4 dereferenceable(4096) %p) {
; RV64-LABEL: define i32 @early_exit_live_out(
; RV64-SAME: ptr align 4 dereferenceable(4096) [[P:%.*]]) #[[ATTR0:[0-9]+]] {
-; RV64-NEXT: [[ENTRY:.*]]:
-; RV64-NEXT: [[TMP0:%.*]] = call i64 @llvm.vscale.i64()
-; RV64-NEXT: [[TMP1:%.*]] = shl nuw i64 [[TMP0]], 2
-; RV64-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 1024, [[TMP1]]
-; RV64-NEXT: br i1 [[MIN_ITERS_CHECK]], label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]]
+; RV64-NEXT: [[ENTRY:.*:]]
+; RV64-NEXT: br label %[[VECTOR_PH:.*]]
; RV64: [[VECTOR_PH]]:
-; RV64-NEXT: [[N_MOD_VF:%.*]] = urem i64 1024, [[TMP1]]
-; RV64-NEXT: [[N_VEC:%.*]] = sub i64 1024, [[N_MOD_VF]]
; RV64-NEXT: br label %[[VECTOR_BODY:.*]]
; RV64: [[VECTOR_BODY]]:
-; RV64-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY_INTERIM:.*]] ]
+; RV64-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[CURRENT_ITERATION_NEXT:%.*]], %[[VECTOR_BODY_INTERIM:.*]] ]
+; RV64-NEXT: [[AVL:%.*]] = phi i64 [ 1024, %[[VECTOR_PH]] ], [ [[AVL_NEXT:%.*]], %[[VECTOR_BODY_INTERIM]] ]
+; RV64-NEXT: [[TMP0:%.*]] = call i32 @llvm.experimental.get.vector.length.i64(i64 [[AVL]], i32 4, i1 true)
; RV64-NEXT: [[TMP4:%.*]] = getelementptr i32, ptr [[P]], i64 [[INDEX]]
-; RV64-NEXT: [[WIDE_LOAD:%.*]] = load <vscale x 4 x i32>, ptr [[TMP4]], align 4
+; RV64-NEXT: [[WIDE_LOAD:%.*]] = call <vscale x 4 x i32> @llvm.vp.load.nxv4i32.p0(ptr align 4 [[TMP4]], <vscale x 4 x i1> splat (i1 true), i32 [[TMP0]])
; RV64-NEXT: [[TMP5:%.*]] = icmp ne <vscale x 4 x i32> [[WIDE_LOAD]], zeroinitializer
; RV64-NEXT: [[TMP6:%.*]] = freeze <vscale x 4 x i1> [[TMP5]]
-; RV64-NEXT: [[TMP7:%.*]] = call i1 @llvm.vector.reduce.or.nxv4i1(<vscale x 4 x i1> [[TMP6]])
-; RV64-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], [[TMP1]]
-; RV64-NEXT: [[TMP8:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
+; RV64-NEXT: [[TMP8:%.*]] = call <vscale x 4 x i1> @llvm.vp.merge.nxv4i1(<vscale x 4 x i1> splat (i1 true), <vscale x 4 x i1> [[TMP6]], <vscale x 4 x i1> zeroinitializer, i32 [[TMP0]])
+; RV64-NEXT: [[TMP12:%.*]] = call i1 @llvm.vp.reduce.or.nxv4i1(i1 false, <vscale x 4 x i1> [[TMP6]], <vscale x 4 x i1> splat (i1 true), i32 [[TMP0]])
+; RV64-NEXT: [[TMP7:%.*]] = or i1 [[TMP12]], false
+; RV64-NEXT: [[TMP10:%.*]] = zext i32 [[TMP0]] to i64
+; RV64-NEXT: [[CURRENT_ITERATION_NEXT]] = add i64 [[TMP10]], [[INDEX]]
+; RV64-NEXT: [[AVL_NEXT]] = sub nuw i64 [[AVL]], [[TMP10]]
+; RV64-NEXT: [[TMP11:%.*]] = icmp eq i64 [[AVL_NEXT]], 0
; RV64-NEXT: br i1 [[TMP7]], label %[[VECTOR_EARLY_EXIT:.*]], label %[[VECTOR_BODY_INTERIM]]
; RV64: [[VECTOR_BODY_INTERIM]]:
-; RV64-NEXT: br i1 [[TMP8]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP0:![0-9]+]]
+; RV64-NEXT: br i1 [[TMP11]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP0:![0-9]+]]
; RV64: [[MIDDLE_BLOCK]]:
-; RV64-NEXT: [[CMP_N:%.*]] = icmp eq i64 1024, [[N_VEC]]
-; RV64-NEXT: br i1 [[CMP_N]], label %[[EXIT:.*]], label %[[SCALAR_PH]]
+; RV64-NEXT: br label %[[LOOP_HEADER:.*]]
; RV64: [[VECTOR_EARLY_EXIT]]:
-; RV64-NEXT: [[FIRST_ACTIVE_LANE:%.*]] = call i64 @llvm.experimental.cttz.elts.i64.nxv4i1(<vscale x 4 x i1> [[TMP6]], i1 false)
+; RV64-NEXT: [[FIRST_ACTIVE_LANE:%.*]] = call i64 @llvm.experimental.cttz.elts.i64.nxv4i1(<vscale x 4 x i1> [[TMP8]], i1 false)
; RV64-NEXT: [[TMP9:%.*]] = extractelement <vscale x 4 x i32> [[WIDE_LOAD]], i64 [[FIRST_ACTIVE_LANE]]
-; RV64-NEXT: br label %[[EXIT]]
-; RV64: [[SCALAR_PH]]:
-; RV64-NEXT: [[BC_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC]], %[[MIDDLE_BLOCK]] ], [ 0, %[[ENTRY]] ]
-; RV64-NEXT: br label %[[LOOP_HEADER:.*]]
+; RV64-NEXT: br label %[[LOOP_HEADER]]
; RV64: [[LOOP_HEADER]]:
-; RV64-NEXT: [[IV:%.*]] = phi i64 [ [[BC_RESUME_VAL]], %[[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], %[[LATCH:.*]] ]
-; RV64-NEXT: [[GEP:%.*]] = getelementptr i32, ptr [[P]], i64 [[IV]]
-; RV64-NEXT: [[LD:%.*]] = load i32, ptr [[GEP]], align 4
-; RV64-NEXT: [[C:%.*]] = icmp eq i32 [[LD]], 0
-; RV64-NEXT: br i1 [[C]], label %[[LATCH]], label %[[EXIT]]
-; RV64: [[LATCH]]:
-; RV64-NEXT: [[IV_NEXT]] = add i64 [[IV]], 1
-; RV64-NEXT: [[EC:%.*]] = icmp eq i64 [[IV_NEXT]], 1024
-; RV64-NEXT: br i1 [[EC]], label %[[EXIT]], label %[[LOOP_HEADER]], !llvm.loop [[LOOP3:![0-9]+]]
-; RV64: [[EXIT]]:
-; RV64-NEXT: [[RET:%.*]] = phi i32 [ [[LD]], %[[LOOP_HEADER]] ], [ 0, %[[LATCH]] ], [ 0, %[[MIDDLE_BLOCK]] ], [ [[TMP9]], %[[VECTOR_EARLY_EXIT]] ]
+; RV64-NEXT: [[RET:%.*]] = phi i32 [ [[TMP9]], %[[VECTOR_EARLY_EXIT]] ], [ 0, %[[MIDDLE_BLOCK]] ]
; RV64-NEXT: ret i32 [[RET]]
;
; RV32-LABEL: define i32 @early_exit_live_out(
; RV32-SAME: ptr align 4 dereferenceable(4096) [[P:%.*]]) #[[ATTR0:[0-9]+]] {
-; RV32-NEXT: [[ENTRY:.*]]:
-; RV32-NEXT: [[TMP0:%.*]] = call i64 @llvm.vscale.i64()
-; RV32-NEXT: [[TMP1:%.*]] = shl nuw i64 [[TMP0]], 2
-; RV32-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 1024, [[TMP1]]
-; RV32-NEXT: br i1 [[MIN_ITERS_CHECK]], label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]]
+; RV32-NEXT: [[ENTRY:.*:]]
+; RV32-NEXT: br label %[[VECTOR_PH:.*]]
; RV32: [[VECTOR_PH]]:
-; RV32-NEXT: [[N_MOD_VF:%.*]] = urem i64 1024, [[TMP1]]
-; RV32-NEXT: [[N_VEC:%.*]] = sub i64 1024, [[N_MOD_VF]]
; RV32-NEXT: br label %[[VECTOR_BODY:.*]]
; RV32: [[VECTOR_BODY]]:
-; RV32-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY_INTERIM:.*]] ]
+; RV32-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[CURRENT_ITERATION_NEXT:%.*]], %[[VECTOR_BODY_INTERIM:.*]] ]
+; RV32-NEXT: [[AVL:%.*]] = phi i64 [ 1024, %[[VECTOR_PH]] ], [ [[AVL_NEXT:%.*]], %[[VECTOR_BODY_INTERIM]] ]
+; RV32-NEXT: [[TMP0:%.*]] = call i32 @llvm.experimental.get.vector.length.i64(i64 [[AVL]], i32 4, i1 true)
; RV32-NEXT: [[TMP4:%.*]] = getelementptr i32, ptr [[P]], i64 [[INDEX]]
-; RV32-NEXT: [[WIDE_LOAD:%.*]] = load <vscale x 4 x i32>, ptr [[TMP4]], align 4
+; RV32-NEXT: [[WIDE_LOAD:%.*]] = call <vscale x 4 x i32> @llvm.vp.load.nxv4i32.p0(ptr align 4 [[TMP4]], <vscale x 4 x i1> splat (i1 true), i32 [[TMP0]])
; RV32-NEXT: [[TMP5:%.*]] = icmp ne <vscale x 4 x i32> [[WIDE_LOAD]], zeroinitializer
; RV32-NEXT: [[TMP6:%.*]] = freeze <vscale x 4 x i1> [[TMP5]]
-; RV32-NEXT: [[TMP7:%.*]] = call i1 @llvm.vector.reduce.or.nxv4i1(<vscale x 4 x i1> [[TMP6]])
-; RV32-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], [[TMP1]]
-; RV32-NEXT: [[TMP8:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
+; RV32-NEXT: [[TMP8:%.*]] = call <vscale x 4 x i1> @llvm.vp.merge.nxv4i1(<vscale x 4 x i1> splat (i1 true), <vscale x 4 x i1> [[TMP6]], <vscale x 4 x i1> zeroinitializer, i32 [[TMP0]])
+; RV32-NEXT: [[TMP12:%.*]] = call i1 @llvm.vp.reduce.or.nxv4i1(i1 false, <vscale x 4 x i1> [[TMP6]], <vscale x 4 x i1> splat (i1 true), i32 [[TMP0]])
+; RV32-NEXT: [[TMP7:%.*]] = or i1 [[TMP12]], false
+; RV32-NEXT: [[TMP10:%.*]] = zext i32 [[TMP0]] to i64
+; RV32-NEXT: [[CURRENT_ITERATION_NEXT]] = add i64 [[TMP10]], [[INDEX]]
+; RV32-NEXT: [[AVL_NEXT]] = sub nuw i64 [[AVL]], [[TMP10]]
+; RV32-NEXT: [[TMP11:%.*]] = icmp eq i64 [[AVL_NEXT]], 0
; RV32-NEXT: br i1 [[TMP7]], label %[[VECTOR_EARLY_EXIT:.*]], label %[[VECTOR_BODY_INTERIM]]
; RV32: [[VECTOR_BODY_INTERIM]]:
-; RV32-NEXT: br i1 [[TMP8]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP0:![0-9]+]]
+; RV32-NEXT: br i1 [[TMP11]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP0:![0-9]+]]
; RV32: [[MIDDLE_BLOCK]]:
-; RV32-NEXT: [[CMP_N:%.*]] = icmp eq i64 1024, [[N_VEC]]
-; RV32-NEXT: br i1 [[CMP_N]], label %[[EXIT:.*]], label %[[SCALAR_PH]]
+; RV32-NEXT: br label %[[LOOP_HEADER:.*]]
; RV32: [[VECTOR_EARLY_EXIT]]:
-; RV32-NEXT: [[FIRST_ACTIVE_LANE:%.*]] = call i32 @llvm.experimental.cttz.elts.i32.nxv4i1(<vscale x 4 x i1> [[TMP6]], i1 false)
+; RV32-NEXT: [[FIRST_ACTIVE_LANE:%.*]] = call i32 @llvm.experimental.cttz.elts.i32.nxv4i1(<vscale x 4 x i1> [[TMP8]], i1 false)
; RV32-NEXT: [[TMP9:%.*]] = extractelement <vscale x 4 x i32> [[WIDE_LOAD]], i32 [[FIRST_ACTIVE_LANE]]
-; RV32-NEXT: br label %[[EXIT]]
-; RV32: [[SCALAR_PH]]:
-; RV32-NEXT: [[BC_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC]], %[[MIDDLE_BLOCK]] ], [ 0, %[[ENTRY]] ]
-; RV32-NEXT: br label %[[LOOP_HEADER:.*]]
+; RV32-NEXT: br label %[[LOOP_HEADER]]
; RV32: [[LOOP_HEADER]]:
-; RV32-NEXT: [[IV:%.*]] = phi i64 [ [[BC_RESUME_VAL]], %[[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], %[[LATCH:.*]] ]
-; RV32-NEXT: [[GEP:%.*]] = getelementptr i32, ptr [[P]], i64 [[IV]]
-; RV32-NEXT: [[LD:%.*]] = load i32, ptr [[GEP]], align 4
-; RV32-NEXT: [[C:%.*]] = icmp eq i32 [[LD]], 0
-; RV32-NEXT: br i1 [[C]], label %[[LATCH]], label %[[EXIT]]
-; RV32: [[LATCH]]:
-; RV32-NEXT: [[IV_NEXT]] = add i64 [[IV]], 1
-; RV32-NEXT: [[EC:%.*]] = icmp eq i64 [[IV_NEXT]], 1024
-; RV32-NEXT: br i1 [[EC]], label %[[EXIT]], label %[[LOOP_HEADER]], !llvm.loop [[LOOP3:![0-9]+]]
-; RV32: [[EXIT]]:
-; RV32-NEXT: [[RET:%.*]] = phi i32 [ [[LD]], %[[LOOP_HEADER]] ], [ 0, %[[LATCH]] ], [ 0, %[[MIDDLE_BLOCK]] ], [ [[TMP9]], %[[VECTOR_EARLY_EXIT]] ]
+; RV32-NEXT: [[RET:%.*]] = phi i32 [ [[TMP9]], %[[VECTOR_EARLY_EXIT]] ], [ 0, %[[MIDDLE_BLOCK]] ]
; RV32-NEXT: ret i32 [[RET]]
;
; ZVE32X-LABEL: define i32 @early_exit_live_out(
; ZVE32X-SAME: ptr align 4 dereferenceable(4096) [[P:%.*]]) #[[ATTR0:[0-9]+]] {
-; ZVE32X-NEXT: [[ENTRY:.*]]:
-; ZVE32X-NEXT: [[TMP0:%.*]] = call i64 @llvm.vscale.i64()
-; ZVE32X-NEXT: [[TMP1:%.*]] = shl nuw i64 [[TMP0]], 2
-; ZVE32X-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 1024, [[TMP1]]
-; ZVE32X-NEXT: br i1 [[MIN_ITERS_CHECK]], label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]]
+; ZVE32X-NEXT: [[ENTRY:.*:]]
+; ZVE32X-NEXT: br label %[[VECTOR_PH:.*]]
; ZVE32X: [[VECTOR_PH]]:
-; ZVE32X-NEXT: [[N_MOD_VF:%.*]] = urem i64 1024, [[TMP1]]
-; ZVE32X-NEXT: [[N_VEC:%.*]] = sub i64 1024, [[N_MOD_VF]]
; ZVE32X-NEXT: br label %[[VECTOR_BODY:.*]]
; ZVE32X: [[VECTOR_BODY]]:
-; ZVE32X-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY_INTERIM:.*]] ]
+; ZVE32X-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[CURRENT_ITERATION_NEXT:%.*]], %[[VECTOR_BODY_INTERIM:.*]] ]
+; ZVE32X-NEXT: [[AVL:%.*]] = phi i64 [ 1024, %[[VECTOR_PH]] ], [ [[AVL_NEXT:%.*]], %[[VECTOR_BODY_INTERIM]] ]
+; ZVE32X-NEXT: [[TMP0:%.*]] = call i32 @llvm.experimental.get.vector.length.i64(i64 [[AVL]], i32 4, i1 true)
; ZVE32X-NEXT: [[TMP4:%.*]] = getelementptr i32, ptr [[P]], i64 [[INDEX]]
-; ZVE32X-NEXT: [[WIDE_LOAD:%.*]] = load <vscale x 4 x i32>, ptr [[TMP4]], align 4
+; ZVE32X-NEXT: [[WIDE_LOAD:%.*]] = call <vscale x 4 x i32> @llvm.vp.load.nxv4i32.p0(ptr align 4 [[TMP4]], <vscale x 4 x i1> splat (i1 true), i32 [[TMP0]])
; ZVE32X-NEXT: [[TMP5:%.*]] = icmp ne <vscale x 4 x i32> [[WIDE_LOAD]], zeroinitializer
; ZVE32X-NEXT: [[TMP6:%.*]] = freeze <vscale x 4 x i1> [[TMP5]]
-; ZVE32X-NEXT: [[TMP7:%.*]] = call i1 @llvm.vector.reduce.or.nxv4i1(<vscale x 4 x i1> [[TMP6]])
-; ZVE32X-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], [[TMP1]]
-; ZVE32X-NEXT: [[TMP8:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
+; ZVE32X-NEXT: [[TMP8:%.*]] = call <vscale x 4 x i1> @llvm.vp.merge.nxv4i1(<vscale x 4 x i1> splat (i1 true), <vscale x 4 x i1> [[TMP6]], <vscale x 4 x i1> zeroinitializer, i32 [[TMP0]])
+; ZVE32X-NEXT: [[TMP12:%.*]] = call i1 @llvm.vp.reduce.or.nxv4i1(i1 false, <vscale x 4 x i1> [[TMP6]], <vscale x 4 x i1> splat (i1 true), i32 [[TMP0]])
+; ZVE32X-NEXT: [[TMP7:%.*]] = or i1 [[TMP12]], false
+; ZVE32X-NEXT: [[TMP10:%.*]] = zext i32 [[TMP0]] to i64
+; ZVE32X-NEXT: [[CURRENT_ITERATION_NEXT]] = add i64 [[TMP10]], [[INDEX]]
+; ZVE32X-NEXT: [[AVL_NEXT]] = sub nuw i64 [[AVL]], [[TMP10]]
+; ZVE32X-NEXT: [[TMP11:%.*]] = icmp eq i64 [[AVL_NEXT]], 0
; ZVE32X-NEXT: br i1 [[TMP7]], label %[[VECTOR_EARLY_EXIT:.*]], label %[[VECTOR_BODY_INTERIM]]
; ZVE32X: [[VECTOR_BODY_INTERIM]]:
-; ZVE32X-NEXT: br i1 [[TMP8]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP0:![0-9]+]]
+; ZVE32X-NEXT: br i1 [[TMP11]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP0:![0-9]+]]
; ZVE32X: [[MIDDLE_BLOCK]]:
-; ZVE32X-NEXT: [[CMP_N:%.*]] = icmp eq i64 1024, [[N_VEC]]
-; ZVE32X-NEXT: br i1 [[CMP_N]], label %[[EXIT:.*]], label %[[SCALAR_PH]]
+; ZVE32X-NEXT: br label %[[LOOP_HEADER:.*]]
; ZVE32X: [[VECTOR_EARLY_EXIT]]:
-; ZVE32X-NEXT: [[FIRST_ACTIVE_LANE:%.*]] = call i32 @llvm.experimental.cttz.elts.i32.nxv4i1(<vscale x 4 x i1> [[TMP6]], i1 false)
+; ZVE32X-NEXT: [[FIRST_ACTIVE_LANE:%.*]] = call i32 @llvm.experimental.cttz.elts.i32.nxv4i1(<vscale x 4 x i1> [[TMP8]], i1 false)
; ZVE32X-NEXT: [[TMP9:%.*]] = extractelement <vscale x 4 x i32> [[WIDE_LOAD]], i32 [[FIRST_ACTIVE_LANE]]
-; ZVE32X-NEXT: br label %[[EXIT]]
-; ZVE32X: [[SCALAR_PH]]:
-; ZVE32X-NEXT: [[BC_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC]], %[[MIDDLE_BLOCK]] ], [ 0, %[[ENTRY]] ]
-; ZVE32X-NEXT: br label %[[LOOP_HEADER:.*]]
+; ZVE32X-NEXT: br label %[[LOOP_HEADER]]
; ZVE32X: [[LOOP_HEADER]]:
-; ZVE32X-NEXT: [[IV:%.*]] = phi i64 [ [[BC_RESUME_VAL]], %[[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], %[[LATCH:.*]] ]
-; ZVE32X-NEXT: [[GEP:%.*]] = getelementptr i32, ptr [[P]], i64 [[IV]]
-; ZVE32X-NEXT: [[LD:%.*]] = load i32, ptr [[GEP]], align 4
-; ZVE32X-NEXT: [[C:%.*]] = icmp eq i32 [[LD]], 0
-; ZVE32X-NEXT: br i1 [[C]], label %[[LATCH]], label %[[EXIT]]
-; ZVE32X: [[LATCH]]:
-; ZVE32X-NEXT: [[IV_NEXT]] = add i64 [[IV]], 1
-; ZVE32X-NEXT: [[EC:%.*]] = icmp eq i64 [[IV_NEXT]], 1024
-; ZVE32X-NEXT: br i1 [[EC]], label %[[EXIT]], label %[[LOOP_HEADER]], !llvm.loop [[LOOP3:![0-9]+]]
-; ZVE32X: [[EXIT]]:
-; ZVE32X-NEXT: [[RET:%.*]] = phi i32 [ [[LD]], %[[LOOP_HEADER]] ], [ 0, %[[LATCH]] ], [ 0, %[[MIDDLE_BLOCK]] ], [ [[TMP9]], %[[VECTOR_EARLY_EXIT]] ]
+; ZVE32X-NEXT: [[RET:%.*]] = phi i32 [ [[TMP9]], %[[VECTOR_EARLY_EXIT]] ], [ 0, %[[MIDDLE_BLOCK]] ]
; ZVE32X-NEXT: ret i32 [[RET]]
;
entry:
@@ -172,112 +133,78 @@ exit:
define i64 @strided_search(ptr align 8 dereferenceable(14784) %p) {
; RV64-LABEL: define i64 @strided_search(
; RV64-SAME: ptr align 8 dereferenceable(14784) [[P:%.*]]) #[[ATTR0]] {
-; RV64-NEXT: [[ENTRY:.*]]:
-; RV64-NEXT: [[TMP0:%.*]] = call i64 @llvm.vscale.i64()
-; RV64-NEXT: [[TMP1:%.*]] = shl nuw i64 [[TMP0]], 1
-; RV64-NEXT: [[TMP2:%.*]] = call i64 @llvm.umax.i64(i64 [[TMP1]], i64 4)
-; RV64-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 132, [[TMP2]]
-; RV64-NEXT: br i1 [[MIN_ITERS_CHECK]], label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]]
+; RV64-NEXT: [[ENTRY:.*:]]
+; RV64-NEXT: br label %[[VECTOR_PH:.*]]
; RV64: [[VECTOR_PH]]:
-; RV64-NEXT: [[N_MOD_VF:%.*]] = urem i64 132, [[TMP1]]
-; RV64-NEXT: [[N_VEC:%.*]] = sub i64 132, [[N_MOD_VF]]
-; RV64-NEXT: [[TMP4:%.*]] = mul i64 [[N_VEC]], 112
-; RV64-NEXT: [[TMP7:%.*]] = trunc i64 [[TMP1]] to i32
; RV64-NEXT: [[SCEVGEP:%.*]] = getelementptr nuw i8, ptr [[P]], i64 88
; RV64-NEXT: br label %[[VECTOR_BODY:.*]]
; RV64: [[VECTOR_BODY]]:
-; RV64-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY_INTERIM:.*]] ]
+; RV64-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[CURRENT_ITERATION_NEXT:%.*]], %[[VECTOR_BODY_INTERIM:.*]] ]
+; RV64-NEXT: [[AVL:%.*]] = phi i64 [ 132, %[[VECTOR_PH]] ], [ [[AVL_NEXT:%.*]], %[[VECTOR_BODY_INTERIM]] ]
+; RV64-NEXT: [[TMP7:%.*]] = call i32 @llvm.experimental.get.vector.length.i64(i64 [[AVL]], i32 2, i1 true)
; RV64-NEXT: [[TMP5:%.*]] = mul nuw i64 [[INDEX]], 112
; RV64-NEXT: [[TMP6:%.*]] = getelementptr nuw i8, ptr [[SCEVGEP]], i64 [[TMP5]]
; RV64-NEXT: [[WIDE_MASKED_GATHER:%.*]] = call <vscale x 2 x i64> @llvm.experimental.vp.strided.load.nxv2i64.p0.i64(ptr align 8 [[TMP6]], i64 112, <vscale x 2 x i1> splat (i1 true), i32 [[TMP7]])
; RV64-NEXT: [[TMP10:%.*]] = icmp eq <vscale x 2 x i64> [[WIDE_MASKED_GATHER]], zeroinitializer
; RV64-NEXT: [[TMP11:%.*]] = freeze <vscale x 2 x i1> [[TMP10]]
-; RV64-NEXT: [[TMP12:%.*]] = call i1 @llvm.vector.reduce.or.nxv2i1(<vscale x 2 x i1> [[TMP11]])
-; RV64-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], [[TMP1]]
-; RV64-NEXT: [[TMP13:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
+; RV64-NEXT: [[TMP8:%.*]] = call <vscale x 2 x i1> @llvm.vp.merge.nxv2i1(<vscale x 2 x i1> splat (i1 true), <vscale x 2 x i1> [[TMP11]], <vscale x 2 x i1> zeroinitializer, i32 [[TMP7]])
+; RV64-NEXT: [[TMP17:%.*]] = call i1 @llvm.vp.reduce.or.nxv2i1(i1 false, <vscale x 2 x i1> [[TMP11]], <vscale x 2 x i1> splat (i1 true), i32 [[TMP7]])
+; RV64-NEXT: [[TMP12:%.*]] = or i1 [[TMP17]], false
+; RV64-NEXT: [[TMP9:%.*]] = zext i32 [[TMP7]] to i64
+; RV64-NEXT: [[CURRENT_ITERATION_NEXT]] = add i64 [[TMP9]], [[INDEX]]
+; RV64-NEXT: [[AVL_NEXT]] = sub nuw i64 [[AVL]], [[TMP9]]
+; RV64-NEXT: [[TMP13:%.*]] = icmp eq i64 [[AVL_NEXT]], 0
; RV64-NEXT: br i1 [[TMP12]], label %[[VECTOR_EARLY_EXIT:.*]], label %[[VECTOR_BODY_INTERIM]]
; RV64: [[VECTOR_BODY_INTERIM]]:
-; RV64-NEXT: br i1 [[TMP13]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP4:![0-9]+]]
+; RV64-NEXT: br i1 [[TMP13]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP3:![0-9]+]]
; RV64: [[MIDDLE_BLOCK]]:
-; RV64-NEXT: [[CMP_N:%.*]] = icmp eq i64 132, [[N_VEC]]
-; RV64-NEXT: br i1 [[CMP_N]], label %[[EXIT:.*]], label %[[SCALAR_PH]]
+; RV64-NEXT: br label %[[LOOP_HEADER:.*]]
; RV64: [[VECTOR_EARLY_EXIT]]:
-; RV64-NEXT: [[TMP14:%.*]] = call i64 @llvm.experimental.cttz.elts.i64.nxv2i1(<vscale x 2 x i1> [[TMP11]], i1 false)
+; RV64-NEXT: [[TMP14:%.*]] = call i64 @llvm.experimental.cttz.elts.i64.nxv2i1(<vscale x 2 x i1> [[TMP8]], i1 false)
; RV64-NEXT: [[TMP15:%.*]] = add i64 [[INDEX]], [[TMP14]]
; RV64-NEXT: [[TMP16:%.*]] = mul i64 [[TMP15]], 112
-; RV64-NEXT: br label %[[EXIT]]
-; RV64: [[SCALAR_PH]]:
-; RV64-NEXT: [[BC_RESUME_VAL:%.*]] = phi i64 [ [[TMP4]], %[[MIDDLE_BLOCK]] ], [ 0, %[[ENTRY]] ]
-; RV64-NEXT: br label %[[LOOP_HEADER:.*]]
+; RV64-NEXT: br label %[[LOOP_HEADER]]
; RV64: [[LOOP_HEADER]]:
-; RV64-NEXT: [[IDX:%.*]] = phi i64 [ [[BC_RESUME_VAL]], %[[SCALAR_PH]] ], [ [[IDX_NEXT:%.*]], %[[LATCH:.*]] ]
-; RV64-NEXT: [[PTR:%.*]] = getelementptr inbounds nuw i8, ptr [[P]], i64 [[IDX]]
-; RV64-NEXT: [[FIELDP:%.*]] = getelementptr inbounds nuw i8, ptr [[PTR]], i64 88
-; RV64-NEXT: [[V:%.*]] = load i64, ptr [[FIELDP]], align 8
-; RV64-NEXT: [[HIT:%.*]] = icmp eq i64 [[V]], 0
-; RV64-NEXT: br i1 [[HIT]], label %[[EXIT]], label %[[LATCH]]
-; RV64: [[LATCH]]:
-; RV64-NEXT: [[IDX_NEXT]] = add nuw nsw i64 [[IDX]], 112
-; RV64-NEXT: [[DONE:%.*]] = icmp eq i64 [[IDX_NEXT]], 14784
-; RV64-NEXT: br i1 [[DONE]], label %[[EXIT]], label %[[LOOP_HEADER]], !llvm.loop [[LOOP5:![0-9]+]]
-; RV64: [[EXIT]]:
-; RV64-NEXT: [[RET:%.*]] = phi i64 [ [[IDX]], %[[LOOP_HEADER]] ], [ -1, %[[LATCH]] ], [ -1, %[[MIDDLE_BLOCK]] ], [ [[TMP16]], %[[VECTOR_EARLY_EXIT]] ]
+; RV64-NEXT: [[RET:%.*]] = phi i64 [ [[TMP16]], %[[VECTOR_EARLY_EXIT]] ], [ -1, %[[MIDDLE_BLOCK]] ]
; RV64-NEXT: ret i64 [[RET]]
;
; RV32-LABEL: define i64 @strided_search(
; RV32-SAME: ptr align 8 dereferenceable(14784) [[P:%.*]]) #[[ATTR0]] {
-; RV32-NEXT: [[ENTRY:.*]]:
-; RV32-NEXT: [[TMP0:%.*]] = call i64 @llvm.vscale.i64()
-; RV32-NEXT: [[TMP1:%.*]] = shl nuw i64 [[TMP0]], 1
-; RV32-NEXT: [[TMP3:%.*]] = call i64 @llvm.umax.i64(i64 [[TMP1]], i64 3)
-; RV32-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 132, [[TMP3]]
-; RV32-NEXT: br i1 [[MIN_ITERS_CHECK]], label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]]
+; RV32-NEXT: [[ENTRY:.*:]]
+; RV32-NEXT: br label %[[VECTOR_PH:.*]]
; RV32: [[VECTOR_PH]]:
-; RV32-NEXT: [[N_MOD_VF:%.*]] = urem i64 132, [[TMP1]]
-; RV32-NEXT: [[N_VEC:%.*]] = sub i64 132, [[N_MOD_VF]]
-; RV32-NEXT: [[TMP4:%.*]] = mul i64 [[N_VEC]], 112
-; RV32-NEXT: [[TMP5:%.*]] = trunc i64 [[TMP1]] to i32
; RV32-NEXT: [[TMP2:%.*]] = getelementptr nuw i8, ptr [[P]], i32 88
; RV32-NEXT: br label %[[VECTOR_BODY:.*]]
; RV32: [[VECTOR_BODY]]:
-; RV32-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY_INTERIM:.*]] ]
+; RV32-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[CURRENT_ITERATION_NEXT:%.*]], %[[VECTOR_BODY_INTERIM:.*]] ]
+; RV32-NEXT: [[AVL:%.*]] = phi i64 [ 132, %[[VECTOR_PH]] ], [ [[AVL_NEXT:%.*]], %[[VECTOR_BODY_INTERIM]] ]
+; RV32-NEXT: [[TMP5:%.*]] = call i32 @llvm.experimental.get.vector.length.i64(i64 [[AVL]], i32 2, i1 true)
; RV32-NEXT: [[TMP6:%.*]] = trunc i64 [[INDEX]] to i32
; RV32-NEXT: [[TMP7:%.*]] = mul nuw i32 [[TMP6]], 112
; RV32-NEXT: [[TMP8:%.*]] = getelementptr nuw i8, ptr [[TMP2]], i32 [[TMP7]]
; RV32-NEXT: [[WIDE_MASKED_GATHER:%.*]] = call <vscale x 2 x i64> @llvm.experimental.vp.strided.load.nxv2i64.p0.i32(ptr align 8 [[TMP8]], i32 112, <vscale x 2 x i1> splat (i1 true), i32 [[TMP5]])
; RV32-NEXT: [[TMP10:%.*]] = icmp eq <vscale x 2 x i64> [[WIDE_MASKED_GATHER]], zeroinitializer
; RV32-NEXT: [[TMP11:%.*]] = freeze <vscale x 2 x i1> [[TMP10]]
-; RV32-NEXT: [[TMP12:%.*]] = call i1 @llvm.vector.reduce.or.nxv2i1(<vscale x 2 x i1> [[TMP11]])
-; RV32-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], [[TMP1]]
-; RV32-NEXT: [[TMP13:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
+; RV32-NEXT: [[TMP9:%.*]] = call <vscale x 2 x i1> @llvm.vp.merge.nxv2i1(<vscale x 2 x i1> splat (i1 true), <vscale x 2 x i1> [[TMP11]], <vscale x 2 x i1> zeroinitializer, i32 [[TMP5]])
+; RV32-NEXT: [[TMP19:%.*]] = call i1 @llvm.vp.reduce.or.nxv2i1(i1 false, <vscale x 2 x i1> [[TMP11]], <vscale x 2 x i1> splat (i1 true), i32 [[TMP5]])
+; RV32-NEXT: [[TMP12:%.*]] = or i1 [[TMP19]], false
+; RV32-NEXT: [[TMP13:%.*]] = zext i32 [[TMP5]] to i64
+; RV32-NEXT: [[CURRENT_ITERATION_NEXT]] = add i64 [[TMP13]], [[INDEX]]
+; RV32-NEXT: [[AVL_NEXT]] = sub nuw i64 [[AVL]], [[TMP13]]
+; RV32-NEXT: [[TMP18:%.*]] = icmp eq i64 [[AVL_NEXT]], 0
; RV32-NEXT: br i1 [[TMP12]], label %[[VECTOR_EARLY_EXIT:.*]], label %[[VECTOR_BODY_INTERIM]]
; RV32: [[VECTOR_BODY_INTERIM]]:
-; RV32-NEXT: br i1 [[TMP13]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP4:![0-9]+]]
+; RV32-NEXT: br i1 [[TMP18]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP3:![0-9]+]]
; RV32: [[MIDDLE_BLOCK]]:
-; RV32-NEXT: [[CMP_N:%.*]] = icmp eq i64 132, [[N_VEC]]
-; RV32-NEXT: br i1 [[CMP_N]], label %[[EXIT:.*]], label %[[SCALAR_PH]]
+; RV32-NEXT: br label %[[LOOP_HEADER:.*]]
; RV32: [[VECTOR_EARLY_EXIT]]:
-; RV32-NEXT: [[TMP14:%.*]] = call i32 @llvm.experimental.cttz.elts.i32.nxv2i1(<vscale x 2 x i1> [[TMP11]], i1 false)
+; RV32-NEXT: [[TMP14:%.*]] = call i32 @llvm.experimental.cttz.elts.i32.nxv2i1(<vscale x 2 x i1> [[TMP9]], i1 false)
; RV32-NEXT: [[TMP15:%.*]] = zext i32 [[TMP14]] to i64
; RV32-NEXT: [[TMP16:%.*]] = add i64 [[INDEX]], [[TMP15]]
; RV32-NEXT: [[TMP17:%.*]] = mul i64 [[TMP16]], 112
-; RV32-NEXT: br label %[[EXIT]]
-; RV32: [[SCALAR_PH]]:
-; RV32-NEXT: [[BC_RESUME_VAL:%.*]] = phi i64 [ [[TMP4]], %[[MIDDLE_BLOCK]] ], [ 0, %[[ENTRY]] ]
-; RV32-NEXT: br label %[[LOOP_HEADER:.*]]
+; RV32-NEXT: br label %[[LOOP_HEADER]]
; RV32: [[LOOP_HEADER]]:
-; RV32-NEXT: [[IDX:%.*]] = phi i64 [ [[BC_RESUME_VAL]], %[[SCALAR_PH]] ], [ [[IDX_NEXT:%.*]], %[[LATCH:.*]] ]
-; RV32-NEXT: [[PTR:%.*]] = getelementptr inbounds nuw i8, ptr [[P]], i64 [[IDX]]
-; RV32-NEXT: [[FIELDP:%.*]] = getelementptr inbounds nuw i8, ptr [[PTR]], i64 88
-; RV32-NEXT: [[V:%.*]] = load i64, ptr [[FIELDP]], align 8
-; RV32-NEXT: [[HIT:%.*]] = icmp eq i64 [[V]], 0
-; RV32-NEXT: br i1 [[HIT]], label %[[EXIT]], label %[[LATCH]]
-; RV32: [[LATCH]]:
-; RV32-NEXT: [[IDX_NEXT]] = add nuw nsw i64 [[IDX]], 112
-; RV32-NEXT: [[DONE:%.*]] = icmp eq i64 [[IDX_NEXT]], 14784
-; RV32-NEXT: br i1 [[DONE]], label %[[EXIT]], label %[[LOOP_HEADER]], !llvm.loop [[LOOP5:![0-9]+]]
-; RV32: [[EXIT]]:
-; RV32-NEXT: [[RET:%.*]] = phi i64 [ [[IDX]], %[[LOOP_HEADER]] ], [ -1, %[[LATCH]] ], [ -1, %[[MIDDLE_BLOCK]] ], [ [[TMP17]], %[[VECTOR_EARLY_EXIT]] ]
+; RV32-NEXT: [[RET:%.*]] = phi i64 [ [[TMP17]], %[[VECTOR_EARLY_EXIT]] ], [ -1, %[[MIDDLE_BLOCK]] ]
; RV32-NEXT: ret i64 [[RET]]
;
; ZVE32X-LABEL: define i64 @strided_search(
diff --git a/llvm/test/Transforms/LoopVectorize/RISCV/early-exit.ll b/llvm/test/Transforms/LoopVectorize/RISCV/early-exit.ll
index 1ab2e3461477c8..f8563ff31a417f 100644
--- a/llvm/test/Transforms/LoopVectorize/RISCV/early-exit.ll
+++ b/llvm/test/Transforms/LoopVectorize/RISCV/early-exit.ll
@@ -4,50 +4,36 @@
define i32 @single_early_exit(ptr align 4 dereferenceable(4096) %p) {
; CHECK-LABEL: define i32 @single_early_exit(
; CHECK-SAME: ptr align 4 dereferenceable(4096) [[P:%.*]]) #[[ATTR0:[0-9]+]] {
-; CHECK-NEXT: [[ENTRY:.*]]:
-; CHECK-NEXT: [[TMP0:%.*]] = call i32 @llvm.vscale.i32()
-; CHECK-NEXT: [[TMP1:%.*]] = shl nuw i32 [[TMP0]], 2
-; CHECK-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ult i32 123, [[TMP1]]
-; CHECK-NEXT: br i1 [[MIN_ITERS_CHECK]], label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]]
+; CHECK-NEXT: [[ENTRY:.*:]]
+; CHECK-NEXT: br label %[[VECTOR_PH:.*]]
; CHECK: [[VECTOR_PH]]:
-; CHECK-NEXT: [[N_MOD_VF:%.*]] = urem i32 123, [[TMP1]]
-; CHECK-NEXT: [[N_VEC:%.*]] = sub i32 123, [[N_MOD_VF]]
; CHECK-NEXT: br label %[[VECTOR_BODY:.*]]
; CHECK: [[VECTOR_BODY]]:
-; CHECK-NEXT: [[INDEX:%.*]] = phi i32 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY_INTERIM:.*]] ]
+; CHECK-NEXT: [[INDEX:%.*]] = phi i32 [ 0, %[[VECTOR_PH]] ], [ [[CURRENT_ITERATION_NEXT:%.*]], %[[VECTOR_BODY_INTERIM:.*]] ]
+; CHECK-NEXT: [[AVL:%.*]] = phi i32 [ 123, %[[VECTOR_PH]] ], [ [[AVL_NEXT:%.*]], %[[VECTOR_BODY_INTERIM]] ]
+; CHECK-NEXT: [[TMP0:%.*]] = call i32 @llvm.experimental.get.vector.length.i32(i32 [[AVL]], i32 4, i1 true)
; CHECK-NEXT: [[TMP2:%.*]] = getelementptr inbounds i32, ptr [[P]], i32 [[INDEX]]
-; CHECK-NEXT: [[WIDE_LOAD:%.*]] = load <vscale x 4 x i32>, ptr [[TMP2]], align 4
+; CHECK-NEXT: [[WIDE_LOAD:%.*]] = call <vscale x 4 x i32> @llvm.vp.load.nxv4i32.p0(ptr align 4 [[TMP2]], <vscale x 4 x i1> splat (i1 true), i32 [[TMP0]])
; CHECK-NEXT: [[TMP3:%.*]] = icmp eq <vscale x 4 x i32> [[WIDE_LOAD]], zeroinitializer
; CHECK-NEXT: [[TMP4:%.*]] = freeze <vscale x 4 x i1> [[TMP3]]
-; CHECK-NEXT: [[TMP5:%.*]] = call i1 @llvm.vector.reduce.or.nxv4i1(<vscale x 4 x i1> [[TMP4]])
-; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i32 [[INDEX]], [[TMP1]]
-; CHECK-NEXT: [[TMP6:%.*]] = icmp eq i32 [[INDEX_NEXT]], [[N_VEC]]
+; CHECK-NEXT: [[TMP10:%.*]] = call <vscale x 4 x i1> @llvm.vp.merge.nxv4i1(<vscale x 4 x i1> splat (i1 true), <vscale x 4 x i1> [[TMP4]], <vscale x 4 x i1> zeroinitializer, i32 [[TMP0]])
+; CHECK-NEXT: [[TMP11:%.*]] = call i1 @llvm.vp.reduce.or.nxv4i1(i1 false, <vscale x 4 x i1> [[TMP4]], <vscale x 4 x i1> splat (i1 true), i32 [[TMP0]])
+; CHECK-NEXT: [[TMP5:%.*]] = or i1 [[TMP11]], false
+; CHECK-NEXT: [[CURRENT_ITERATION_NEXT]] = add i32 [[TMP0]], [[INDEX]]
+; CHECK-NEXT: [[AVL_NEXT]] = sub nuw i32 [[AVL]], [[TMP0]]
+; CHECK-NEXT: [[TMP6:%.*]] = icmp eq i32 [[AVL_NEXT]], 0
; CHECK-NEXT: br i1 [[TMP5]], label %[[VECTOR_EARLY_EXIT:.*]], label %[[VECTOR_BODY_INTERIM]]
; CHECK: [[VECTOR_BODY_INTERIM]]:
; CHECK-NEXT: br i1 [[TMP6]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP0:![0-9]+]]
; CHECK: [[MIDDLE_BLOCK]]:
-; CHECK-NEXT: [[CMP_N:%.*]] = icmp eq i32 123, [[N_VEC]]
-; CHECK-NEXT: br i1 [[CMP_N]], label %[[EXIT:.*]], label %[[SCALAR_PH]]
+; CHECK-NEXT: br label %[[LOOP:.*]]
; CHECK: [[VECTOR_EARLY_EXIT]]:
-; CHECK-NEXT: [[TMP7:%.*]] = call i64 @llvm.experimental.cttz.elts.i64.nxv4i1(<vscale x 4 x i1> [[TMP4]], i1 false)
+; CHECK-NEXT: [[TMP7:%.*]] = call i64 @llvm.experimental.cttz.elts.i64.nxv4i1(<vscale x 4 x i1> [[TMP10]], i1 false)
; CHECK-NEXT: [[TMP8:%.*]] = trunc i64 [[TMP7]] to i32
; CHECK-NEXT: [[TMP9:%.*]] = add i32 [[INDEX]], [[TMP8]]
-; CHECK-NEXT: br label %[[EXIT]]
-; CHECK: [[SCALAR_PH]]:
-; CHECK-NEXT: [[BC_RESUME_VAL:%.*]] = phi i32 [ [[N_VEC]], %[[MIDDLE_BLOCK]] ], [ 0, %[[ENTRY]] ]
-; CHECK-NEXT: br label %[[LOOP:.*]]
+; CHECK-NEXT: br label %[[LOOP]]
; CHECK: [[LOOP]]:
-; CHECK-NEXT: [[IV:%.*]] = phi i32 [ [[BC_RESUME_VAL]], %[[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], %[[LATCH:.*]] ]
-; CHECK-NEXT: [[GEP:%.*]] = getelementptr inbounds i32, ptr [[P]], i32 [[IV]]
-; CHECK-NEXT: [[LD:%.*]] = load i32, ptr [[GEP]], align 4
-; CHECK-NEXT: [[C:%.*]] = icmp eq i32 [[LD]], 0
-; CHECK-NEXT: br i1 [[C]], label %[[EXIT]], label %[[LATCH]]
-; CHECK: [[LATCH]]:
-; CHECK-NEXT: [[IV_NEXT]] = add nuw nsw i32 [[IV]], 1
-; CHECK-NEXT: [[EC:%.*]] = icmp eq i32 [[IV_NEXT]], 123
-; CHECK-NEXT: br i1 [[EC]], label %[[EXIT]], label %[[LOOP]], !llvm.loop [[LOOP3:![0-9]+]]
-; CHECK: [[EXIT]]:
-; CHECK-NEXT: [[RES:%.*]] = phi i32 [ [[IV]], %[[LOOP]] ], [ -1, %[[LATCH]] ], [ -1, %[[MIDDLE_BLOCK]] ], [ [[TMP9]], %[[VECTOR_EARLY_EXIT]] ]
+; CHECK-NEXT: [[RES:%.*]] = phi i32 [ [[TMP9]], %[[VECTOR_EARLY_EXIT]] ], [ -1, %[[MIDDLE_BLOCK]] ]
; CHECK-NEXT: ret i32 [[RES]]
;
entry:
@@ -73,42 +59,41 @@ exit:
define i32 @multi_early_exits_same_exits(ptr align 4 dereferenceable(4096) %p, ptr align 4 dereferenceable(4096) %q) {
; CHECK-LABEL: define i32 @multi_early_exits_same_exits(
; CHECK-SAME: ptr align 4 dereferenceable(4096) [[P:%.*]], ptr align 4 dereferenceable(4096) [[Q:%.*]]) #[[ATTR0]] {
-; CHECK-NEXT: [[ENTRY:.*]]:
-; CHECK-NEXT: [[TMP0:%.*]] = call i32 @llvm.vscale.i32()
-; CHECK-NEXT: [[TMP1:%.*]] = shl nuw i32 [[TMP0]], 2
-; CHECK-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ult i32 123, [[TMP1]]
-; CHECK-NEXT: br i1 [[MIN_ITERS_CHECK]], label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]]
+; CHECK-NEXT: [[ENTRY:.*:]]
+; CHECK-NEXT: br label %[[VECTOR_PH:.*]]
; CHECK: [[VECTOR_PH]]:
-; CHECK-NEXT: [[N_MOD_VF:%.*]] = urem i32 123, [[TMP1]]
-; CHECK-NEXT: [[N_VEC:%.*]] = sub i32 123, [[N_MOD_VF]]
; CHECK-NEXT: [[TMP2:%.*]] = call <vscale x 4 x i32> @llvm.stepvector.nxv4i32()
-; CHECK-NEXT: [[BROADCAST_SPLATINSERT:%.*]] = insertelement <vscale x 4 x i32> poison, i32 [[TMP1]], i64 0
-; CHECK-NEXT: [[BROADCAST_SPLAT:%.*]] = shufflevector <vscale x 4 x i32> [[BROADCAST_SPLATINSERT]], <vscale x 4 x i32> poison, <vscale x 4 x i32> zeroinitializer
; CHECK-NEXT: br label %[[VECTOR_BODY:.*]]
; CHECK: [[VECTOR_BODY]]:
-; CHECK-NEXT: [[INDEX:%.*]] = phi i32 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY_INTERIM:.*]] ]
+; CHECK-NEXT: [[INDEX:%.*]] = phi i32 [ 0, %[[VECTOR_PH]] ], [ [[CURRENT_ITERATION_NEXT:%.*]], %[[VECTOR_BODY_INTERIM:.*]] ]
; CHECK-NEXT: [[VEC_IND:%.*]] = phi <vscale x 4 x i32> [ [[TMP2]], %[[VECTOR_PH]] ], [ [[VEC_IND_NEXT:%.*]], %[[VECTOR_BODY_INTERIM]] ]
+; CHECK-NEXT: [[AVL:%.*]] = phi i32 [ 123, %[[VECTOR_PH]] ], [ [[AVL_NEXT:%.*]], %[[VECTOR_BODY_INTERIM]] ]
+; CHECK-NEXT: [[TMP1:%.*]] = call i32 @llvm.experimental.get.vector.length.i32(i32 [[AVL]], i32 4, i1 true)
+; CHECK-NEXT: [[BROADCAST_SPLATINSERT:%.*]] = insertelement <vscale x 4 x i32> poison, i32 [[TMP1]], i64 0
+; CHECK-NEXT: [[BROADCAST_SPLAT:%.*]] = shufflevector <vscale x 4 x i32> [[BROADCAST_SPLATINSERT]], <vscale x 4 x i32> poison, <vscale x 4 x i32> zeroinitializer
; CHECK-NEXT: [[TMP3:%.*]] = getelementptr inbounds i32, ptr [[P]], i32 [[INDEX]]
-; CHECK-NEXT: [[WIDE_LOAD:%.*]] = load <vscale x 4 x i32>, ptr [[TMP3]], align 4
+; CHECK-NEXT: [[WIDE_LOAD:%.*]] = call <vscale x 4 x i32> @llvm.vp.load.nxv4i32.p0(ptr align 4 [[TMP3]], <vscale x 4 x i1> splat (i1 true), i32 [[TMP1]])
; CHECK-NEXT: [[TMP4:%.*]] = icmp eq <vscale x 4 x i32> [[WIDE_LOAD]], zeroinitializer
; CHECK-NEXT: [[TMP5:%.*]] = getelementptr inbounds i32, ptr [[Q]], i32 [[INDEX]]
-; CHECK-NEXT: [[WIDE_LOAD1:%.*]] = load <vscale x 4 x i32>, ptr [[TMP5]], align 4
+; CHECK-NEXT: [[WIDE_LOAD1:%.*]] = call <vscale x 4 x i32> @llvm.vp.load.nxv4i32.p0(ptr align 4 [[TMP5]], <vscale x 4 x i1> splat (i1 true), i32 [[TMP1]])
; CHECK-NEXT: [[TMP6:%.*]] = icmp eq <vscale x 4 x i32> [[WIDE_LOAD1]], zeroinitializer
; CHECK-NEXT: [[TMP7:%.*]] = add <vscale x 4 x i32> [[VEC_IND]], splat (i32 1000)
; CHECK-NEXT: [[TMP8:%.*]] = select <vscale x 4 x i1> [[TMP4]], <vscale x 4 x i1> splat (i1 true), <vscale x 4 x i1> [[TMP6]]
; CHECK-NEXT: [[TMP9:%.*]] = freeze <vscale x 4 x i1> [[TMP8]]
-; CHECK-NEXT: [[TMP10:%.*]] = call i1 @llvm.vector.reduce.or.nxv4i1(<vscale x 4 x i1> [[TMP9]])
-; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i32 [[INDEX]], [[TMP1]]
-; CHECK-NEXT: [[TMP11:%.*]] = icmp eq i32 [[INDEX_NEXT]], [[N_VEC]]
+; CHECK-NEXT: [[TMP16:%.*]] = call <vscale x 4 x i1> @llvm.vp.merge.nxv4i1(<vscale x 4 x i1> splat (i1 true), <vscale x 4 x i1> [[TMP9]], <vscale x 4 x i1> zeroinitializer, i32 [[TMP1]])
+; CHECK-NEXT: [[TMP17:%.*]] = call i1 @llvm.vp.reduce.or.nxv4i1(i1 false, <vscale x 4 x i1> [[TMP9]], <vscale x 4 x i1> splat (i1 true), i32 [[TMP1]])
+; CHECK-NEXT: [[TMP10:%.*]] = or i1 [[TMP17]], false
+; CHECK-NEXT: [[CURRENT_ITERATION_NEXT]] = add i32 [[TMP1]], [[INDEX]]
+; CHECK-NEXT: [[AVL_NEXT]] = sub nuw i32 [[AVL]], [[TMP1]]
+; CHECK-NEXT: [[TMP11:%.*]] = icmp eq i32 [[AVL_NEXT]], 0
; CHECK-NEXT: [[VEC_IND_NEXT]] = add nuw nsw <vscale x 4 x i32> [[VEC_IND]], [[BROADCAST_SPLAT]]
; CHECK-NEXT: br i1 [[TMP10]], label %[[VECTOR_EARLY_EXIT_CHECK:.*]], label %[[VECTOR_BODY_INTERIM]]
; CHECK: [[VECTOR_BODY_INTERIM]]:
-; CHECK-NEXT: br i1 [[TMP11]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP4:![0-9]+]]
+; CHECK-NEXT: br i1 [[TMP11]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP3:![0-9]+]]
; CHECK: [[MIDDLE_BLOCK]]:
-; CHECK-NEXT: [[CMP_N:%.*]] = icmp eq i32 123, [[N_VEC]]
-; CHECK-NEXT: br i1 [[CMP_N]], label %[[EXIT:.*]], label %[[SCALAR_PH]]
+; CHECK-NEXT: br label %[[EXIT:.*]]
; CHECK: [[VECTOR_EARLY_EXIT_CHECK]]:
-; CHECK-NEXT: [[FIRST_ACTIVE_LANE:%.*]] = call i64 @llvm.experimental.cttz.elts.i64.nxv4i1(<vscale x 4 x i1> [[TMP9]], i1 false)
+; CHECK-NEXT: [[FIRST_ACTIVE_LANE:%.*]] = call i64 @llvm.experimental.cttz.elts.i64.nxv4i1(<vscale x 4 x i1> [[TMP16]], i1 false)
; CHECK-NEXT: [[TMP12:%.*]] = extractelement <vscale x 4 x i1> [[TMP4]], i64 [[FIRST_ACTIVE_LANE]]
; CHECK-NEXT: br i1 [[TMP12]], label %[[VECTOR_EARLY_EXIT_0:.*]], label %[[VECTOR_EARLY_EXIT_1:.*]]
; CHECK: [[VECTOR_EARLY_EXIT_1]]:
@@ -118,27 +103,8 @@ define i32 @multi_early_exits_same_exits(ptr align 4 dereferenceable(4096) %p, p
; CHECK-NEXT: [[TMP14:%.*]] = trunc i64 [[FIRST_ACTIVE_LANE]] to i32
; CHECK-NEXT: [[TMP15:%.*]] = add i32 [[INDEX]], [[TMP14]]
; CHECK-NEXT: br label %[[EXIT]]
-; CHECK: [[SCALAR_PH]]:
-; CHECK-NEXT: [[BC_RESUME_VAL:%.*]] = phi i32 [ [[N_VEC]], %[[MIDDLE_BLOCK]] ], [ 0, %[[ENTRY]] ]
-; CHECK-NEXT: br label %[[LOOP:.*]]
-; CHECK: [[LOOP]]:
-; CHECK-NEXT: [[IV:%.*]] = phi i32 [ [[BC_RESUME_VAL]], %[[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], %[[LATCH:.*]] ]
-; CHECK-NEXT: [[GEP_P:%.*]] = getelementptr inbounds i32, ptr [[P]], i32 [[IV]]
-; CHECK-NEXT: [[X:%.*]] = load i32, ptr [[GEP_P]], align 4
-; CHECK-NEXT: [[C1:%.*]] = icmp eq i32 [[X]], 0
-; CHECK-NEXT: br i1 [[C1]], label %[[EXIT]], label %[[NEXT:.*]]
-; CHECK: [[NEXT]]:
-; CHECK-NEXT: [[GEP_Q:%.*]] = getelementptr inbounds i32, ptr [[Q]], i32 [[IV]]
-; CHECK-NEXT: [[Y:%.*]] = load i32, ptr [[GEP_Q]], align 4
-; CHECK-NEXT: [[C2:%.*]] = icmp eq i32 [[Y]], 0
-; CHECK-NEXT: [[IV_Q:%.*]] = add i32 [[IV]], 1000
-; CHECK-NEXT: br i1 [[C2]], label %[[EXIT]], label %[[LATCH]]
-; CHECK: [[LATCH]]:
-; CHECK-NEXT: [[IV_NEXT]] = add nuw nsw i32 [[IV]], 1
-; CHECK-NEXT: [[EC:%.*]] = icmp eq i32 [[IV_NEXT]], 123
-; CHECK-NEXT: br i1 [[EC]], label %[[EXIT]], label %[[LOOP]], !llvm.loop [[LOOP5:![0-9]+]]
; CHECK: [[EXIT]]:
-; CHECK-NEXT: [[RES:%.*]] = phi i32 [ [[IV]], %[[LOOP]] ], [ [[IV_Q]], %[[NEXT]] ], [ -1, %[[LATCH]] ], [ -1, %[[MIDDLE_BLOCK]] ], [ [[TMP15]], %[[VECTOR_EARLY_EXIT_0]] ], [ [[TMP13]], %[[VECTOR_EARLY_EXIT_1]] ]
+; CHECK-NEXT: [[RES:%.*]] = phi i32 [ [[TMP13]], %[[VECTOR_EARLY_EXIT_1]] ], [ [[TMP15]], %[[VECTOR_EARLY_EXIT_0]] ], [ -1, %[[MIDDLE_BLOCK]] ]
; CHECK-NEXT: ret i32 [[RES]]
;
entry:
@@ -171,42 +137,41 @@ exit:
define i32 @multi_early_exits_different_exits(ptr align 4 dereferenceable(4096) %p, ptr align 4 dereferenceable(4096) %q) {
; CHECK-LABEL: define i32 @multi_early_exits_different_exits(
; CHECK-SAME: ptr align 4 dereferenceable(4096) [[P:%.*]], ptr align 4 dereferenceable(4096) [[Q:%.*]]) #[[ATTR0]] {
-; CHECK-NEXT: [[ENTRY:.*]]:
-; CHECK-NEXT: [[TMP0:%.*]] = call i32 @llvm.vscale.i32()
-; CHECK-NEXT: [[TMP1:%.*]] = shl nuw i32 [[TMP0]], 2
-; CHECK-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ult i32 123, [[TMP1]]
-; CHECK-NEXT: br i1 [[MIN_ITERS_CHECK]], label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]]
+; CHECK-NEXT: [[ENTRY:.*:]]
+; CHECK-NEXT: br label %[[VECTOR_PH:.*]]
; CHECK: [[VECTOR_PH]]:
-; CHECK-NEXT: [[N_MOD_VF:%.*]] = urem i32 123, [[TMP1]]
-; CHECK-NEXT: [[N_VEC:%.*]] = sub i32 123, [[N_MOD_VF]]
; CHECK-NEXT: [[TMP2:%.*]] = call <vscale x 4 x i32> @llvm.stepvector.nxv4i32()
-; CHECK-NEXT: [[BROADCAST_SPLATINSERT:%.*]] = insertelement <vscale x 4 x i32> poison, i32 [[TMP1]], i64 0
-; CHECK-NEXT: [[BROADCAST_SPLAT:%.*]] = shufflevector <vscale x 4 x i32> [[BROADCAST_SPLATINSERT]], <vscale x 4 x i32> poison, <vscale x 4 x i32> zeroinitializer
; CHECK-NEXT: br label %[[VECTOR_BODY:.*]]
; CHECK: [[VECTOR_BODY]]:
-; CHECK-NEXT: [[INDEX:%.*]] = phi i32 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY_INTERIM:.*]] ]
+; CHECK-NEXT: [[INDEX:%.*]] = phi i32 [ 0, %[[VECTOR_PH]] ], [ [[CURRENT_ITERATION_NEXT:%.*]], %[[VECTOR_BODY_INTERIM:.*]] ]
; CHECK-NEXT: [[VEC_IND:%.*]] = phi <vscale x 4 x i32> [ [[TMP2]], %[[VECTOR_PH]] ], [ [[VEC_IND_NEXT:%.*]], %[[VECTOR_BODY_INTERIM]] ]
+; CHECK-NEXT: [[AVL:%.*]] = phi i32 [ 123, %[[VECTOR_PH]] ], [ [[AVL_NEXT:%.*]], %[[VECTOR_BODY_INTERIM]] ]
+; CHECK-NEXT: [[TMP1:%.*]] = call i32 @llvm.experimental.get.vector.length.i32(i32 [[AVL]], i32 4, i1 true)
+; CHECK-NEXT: [[BROADCAST_SPLATINSERT:%.*]] = insertelement <vscale x 4 x i32> poison, i32 [[TMP1]], i64 0
+; CHECK-NEXT: [[BROADCAST_SPLAT:%.*]] = shufflevector <vscale x 4 x i32> [[BROADCAST_SPLATINSERT]], <vscale x 4 x i32> poison, <vscale x 4 x i32> zeroinitializer
; CHECK-NEXT: [[TMP3:%.*]] = getelementptr inbounds i32, ptr [[P]], i32 [[INDEX]]
-; CHECK-NEXT: [[WIDE_LOAD:%.*]] = load <vscale x 4 x i32>, ptr [[TMP3]], align 4
+; CHECK-NEXT: [[WIDE_LOAD:%.*]] = call <vscale x 4 x i32> @llvm.vp.load.nxv4i32.p0(ptr align 4 [[TMP3]], <vscale x 4 x i1> splat (i1 true), i32 [[TMP1]])
; CHECK-NEXT: [[TMP4:%.*]] = icmp eq <vscale x 4 x i32> [[WIDE_LOAD]], zeroinitializer
; CHECK-NEXT: [[TMP5:%.*]] = getelementptr inbounds i32, ptr [[Q]], i32 [[INDEX]]
-; CHECK-NEXT: [[WIDE_LOAD1:%.*]] = load <vscale x 4 x i32>, ptr [[TMP5]], align 4
+; CHECK-NEXT: [[WIDE_LOAD1:%.*]] = call <vscale x 4 x i32> @llvm.vp.load.nxv4i32.p0(ptr align 4 [[TMP5]], <vscale x 4 x i1> splat (i1 true), i32 [[TMP1]])
; CHECK-NEXT: [[TMP6:%.*]] = icmp eq <vscale x 4 x i32> [[WIDE_LOAD1]], zeroinitializer
; CHECK-NEXT: [[TMP7:%.*]] = add <vscale x 4 x i32> [[VEC_IND]], splat (i32 1000)
; CHECK-NEXT: [[TMP8:%.*]] = select <vscale x 4 x i1> [[TMP4]], <vscale x 4 x i1> splat (i1 true), <vscale x 4 x i1> [[TMP6]]
; CHECK-NEXT: [[TMP9:%.*]] = freeze <vscale x 4 x i1> [[TMP8]]
-; CHECK-NEXT: [[TMP10:%.*]] = call i1 @llvm.vector.reduce.or.nxv4i1(<vscale x 4 x i1> [[TMP9]])
-; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i32 [[INDEX]], [[TMP1]]
-; CHECK-NEXT: [[TMP11:%.*]] = icmp eq i32 [[INDEX_NEXT]], [[N_VEC]]
+; CHECK-NEXT: [[TMP16:%.*]] = call <vscale x 4 x i1> @llvm.vp.merge.nxv4i1(<vscale x 4 x i1> splat (i1 true), <vscale x 4 x i1> [[TMP9]], <vscale x 4 x i1> zeroinitializer, i32 [[TMP1]])
+; CHECK-NEXT: [[TMP17:%.*]] = call i1 @llvm.vp.reduce.or.nxv4i1(i1 false, <vscale x 4 x i1> [[TMP9]], <vscale x 4 x i1> splat (i1 true), i32 [[TMP1]])
+; CHECK-NEXT: [[TMP10:%.*]] = or i1 [[TMP17]], false
+; CHECK-NEXT: [[CURRENT_ITERATION_NEXT]] = add i32 [[TMP1]], [[INDEX]]
+; CHECK-NEXT: [[AVL_NEXT]] = sub nuw i32 [[AVL]], [[TMP1]]
+; CHECK-NEXT: [[TMP11:%.*]] = icmp eq i32 [[AVL_NEXT]], 0
; CHECK-NEXT: [[VEC_IND_NEXT]] = add nuw nsw <vscale x 4 x i32> [[VEC_IND]], [[BROADCAST_SPLAT]]
; CHECK-NEXT: br i1 [[TMP10]], label %[[VECTOR_EARLY_EXIT_CHECK:.*]], label %[[VECTOR_BODY_INTERIM]]
; CHECK: [[VECTOR_BODY_INTERIM]]:
-; CHECK-NEXT: br i1 [[TMP11]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP6:![0-9]+]]
+; CHECK-NEXT: br i1 [[TMP11]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP4:![0-9]+]]
; CHECK: [[MIDDLE_BLOCK]]:
-; CHECK-NEXT: [[CMP_N:%.*]] = icmp eq i32 123, [[N_VEC]]
-; CHECK-NEXT: br i1 [[CMP_N]], label %[[EXIT_LATCH:.*]], label %[[SCALAR_PH]]
+; CHECK-NEXT: br label %[[EXIT_LATCH:.*]]
; CHECK: [[VECTOR_EARLY_EXIT_CHECK]]:
-; CHECK-NEXT: [[FIRST_ACTIVE_LANE:%.*]] = call i64 @llvm.experimental.cttz.elts.i64.nxv4i1(<vscale x 4 x i1> [[TMP9]], i1 false)
+; CHECK-NEXT: [[FIRST_ACTIVE_LANE:%.*]] = call i64 @llvm.experimental.cttz.elts.i64.nxv4i1(<vscale x 4 x i1> [[TMP16]], i1 false)
; CHECK-NEXT: [[TMP12:%.*]] = extractelement <vscale x 4 x i1> [[TMP4]], i64 [[FIRST_ACTIVE_LANE]]
; CHECK-NEXT: br i1 [[TMP12]], label %[[VECTOR_EARLY_EXIT_0:.*]], label %[[VECTOR_EARLY_EXIT_1:.*]]
; CHECK: [[VECTOR_EARLY_EXIT_1]]:
@@ -215,32 +180,11 @@ define i32 @multi_early_exits_different_exits(ptr align 4 dereferenceable(4096)
; CHECK: [[VECTOR_EARLY_EXIT_0]]:
; CHECK-NEXT: [[TMP14:%.*]] = trunc i64 [[FIRST_ACTIVE_LANE]] to i32
; CHECK-NEXT: [[TMP15:%.*]] = add i32 [[INDEX]], [[TMP14]]
-; CHECK-NEXT: br label %[[EXIT_P:.*]]
-; CHECK: [[SCALAR_PH]]:
-; CHECK-NEXT: [[BC_RESUME_VAL:%.*]] = phi i32 [ [[N_VEC]], %[[MIDDLE_BLOCK]] ], [ 0, %[[ENTRY]] ]
; CHECK-NEXT: br label %[[LOOP:.*]]
; CHECK: [[LOOP]]:
-; CHECK-NEXT: [[IV:%.*]] = phi i32 [ [[BC_RESUME_VAL]], %[[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], %[[LATCH:.*]] ]
-; CHECK-NEXT: [[GEP_P:%.*]] = getelementptr inbounds i32, ptr [[P]], i32 [[IV]]
-; CHECK-NEXT: [[X:%.*]] = load i32, ptr [[GEP_P]], align 4
-; CHECK-NEXT: [[C1:%.*]] = icmp eq i32 [[X]], 0
-; CHECK-NEXT: br i1 [[C1]], label %[[EXIT_P]], label %[[NEXT:.*]]
-; CHECK: [[NEXT]]:
-; CHECK-NEXT: [[GEP_Q:%.*]] = getelementptr inbounds i32, ptr [[Q]], i32 [[IV]]
-; CHECK-NEXT: [[Y:%.*]] = load i32, ptr [[GEP_Q]], align 4
-; CHECK-NEXT: [[C2:%.*]] = icmp eq i32 [[Y]], 0
-; CHECK-NEXT: [[IV_Q:%.*]] = add i32 [[IV]], 1000
-; CHECK-NEXT: br i1 [[C2]], label %[[EXIT_Q]], label %[[LATCH]]
-; CHECK: [[LATCH]]:
-; CHECK-NEXT: [[IV_NEXT]] = add nuw nsw i32 [[IV]], 1
-; CHECK-NEXT: [[EC:%.*]] = icmp eq i32 [[IV_NEXT]], 123
-; CHECK-NEXT: br i1 [[EC]], label %[[EXIT_LATCH]], label %[[LOOP]], !llvm.loop [[LOOP7:![0-9]+]]
-; CHECK: [[EXIT_P]]:
-; CHECK-NEXT: [[RES_P:%.*]] = phi i32 [ [[IV]], %[[LOOP]] ], [ [[TMP15]], %[[VECTOR_EARLY_EXIT_0]] ]
-; CHECK-NEXT: ret i32 [[RES_P]]
+; CHECK-NEXT: ret i32 [[TMP15]]
; CHECK: [[EXIT_Q]]:
-; CHECK-NEXT: [[RES_Q:%.*]] = phi i32 [ [[IV_Q]], %[[NEXT]] ], [ [[TMP13]], %[[VECTOR_EARLY_EXIT_1]] ]
-; CHECK-NEXT: ret i32 [[RES_Q]]
+; CHECK-NEXT: ret i32 [[TMP13]]
; CHECK: [[EXIT_LATCH]]:
; CHECK-NEXT: ret i32 -1
;
@@ -281,56 +225,40 @@ exit.latch:
define i32 @predicated_early_exit(i1 %c0, ptr align 4 dereferenceable(4096) %p) {
; CHECK-LABEL: define i32 @predicated_early_exit(
; CHECK-SAME: i1 [[C0:%.*]], ptr align 4 dereferenceable(4096) [[P:%.*]]) #[[ATTR0]] {
-; CHECK-NEXT: [[ENTRY:.*]]:
-; CHECK-NEXT: [[TMP0:%.*]] = call i32 @llvm.vscale.i32()
-; CHECK-NEXT: [[TMP1:%.*]] = shl nuw i32 [[TMP0]], 2
-; CHECK-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ult i32 123, [[TMP1]]
-; CHECK-NEXT: br i1 [[MIN_ITERS_CHECK]], label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]]
+; CHECK-NEXT: [[ENTRY:.*:]]
+; CHECK-NEXT: br label %[[VECTOR_PH:.*]]
; CHECK: [[VECTOR_PH]]:
-; CHECK-NEXT: [[N_MOD_VF:%.*]] = urem i32 123, [[TMP1]]
-; CHECK-NEXT: [[N_VEC:%.*]] = sub i32 123, [[N_MOD_VF]]
; CHECK-NEXT: [[BROADCAST_SPLATINSERT:%.*]] = insertelement <vscale x 4 x i1> poison, i1 [[C0]], i64 0
; CHECK-NEXT: [[BROADCAST_SPLAT:%.*]] = shufflevector <vscale x 4 x i1> [[BROADCAST_SPLATINSERT]], <vscale x 4 x i1> poison, <vscale x 4 x i32> zeroinitializer
; CHECK-NEXT: [[TMP11:%.*]] = freeze <vscale x 4 x i1> [[BROADCAST_SPLAT]]
; CHECK-NEXT: br label %[[VECTOR_BODY:.*]]
; CHECK: [[VECTOR_BODY]]:
-; CHECK-NEXT: [[INDEX:%.*]] = phi i32 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY_INTERIM:.*]] ]
-; CHECK-NEXT: [[TMP2:%.*]] = getelementptr inbounds i32, ptr [[P]], i32 [[INDEX]]
-; CHECK-NEXT: [[WIDE_LOAD:%.*]] = load <vscale x 4 x i32>, ptr [[TMP2]], align 4
+; CHECK-NEXT: [[INDEX:%.*]] = phi i32 [ 0, %[[VECTOR_PH]] ], [ [[CURRENT_ITERATION_NEXT:%.*]], %[[VECTOR_BODY_INTERIM:.*]] ]
+; CHECK-NEXT: [[AVL:%.*]] = phi i32 [ 123, %[[VECTOR_PH]] ], [ [[AVL_NEXT:%.*]], %[[VECTOR_BODY_INTERIM]] ]
+; CHECK-NEXT: [[TMP1:%.*]] = call i32 @llvm.experimental.get.vector.length.i32(i32 [[AVL]], i32 4, i1 true)
+; CHECK-NEXT: [[TMP2:%.*]] = getelementptr i32, ptr [[P]], i32 [[INDEX]]
+; CHECK-NEXT: [[WIDE_LOAD:%.*]] = call <vscale x 4 x i32> @llvm.vp.load.nxv4i32.p0(ptr align 4 [[TMP2]], <vscale x 4 x i1> [[TMP11]], i32 [[TMP1]])
; CHECK-NEXT: [[TMP3:%.*]] = icmp eq <vscale x 4 x i32> [[WIDE_LOAD]], zeroinitializer
; CHECK-NEXT: [[TMP4:%.*]] = select <vscale x 4 x i1> [[TMP11]], <vscale x 4 x i1> [[TMP3]], <vscale x 4 x i1> zeroinitializer
; CHECK-NEXT: [[TMP5:%.*]] = freeze <vscale x 4 x i1> [[TMP4]]
-; CHECK-NEXT: [[TMP6:%.*]] = call i1 @llvm.vector.reduce.or.nxv4i1(<vscale x 4 x i1> [[TMP5]])
-; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i32 [[INDEX]], [[TMP1]]
-; CHECK-NEXT: [[TMP7:%.*]] = icmp eq i32 [[INDEX_NEXT]], [[N_VEC]]
+; CHECK-NEXT: [[TMP7:%.*]] = call <vscale x 4 x i1> @llvm.vp.merge.nxv4i1(<vscale x 4 x i1> splat (i1 true), <vscale x 4 x i1> [[TMP5]], <vscale x 4 x i1> zeroinitializer, i32 [[TMP1]])
+; CHECK-NEXT: [[TMP13:%.*]] = call i1 @llvm.vp.reduce.or.nxv4i1(i1 false, <vscale x 4 x i1> [[TMP5]], <vscale x 4 x i1> splat (i1 true), i32 [[TMP1]])
+; CHECK-NEXT: [[TMP6:%.*]] = or i1 [[TMP13]], false
+; CHECK-NEXT: [[CURRENT_ITERATION_NEXT]] = add i32 [[TMP1]], [[INDEX]]
+; CHECK-NEXT: [[AVL_NEXT]] = sub nuw i32 [[AVL]], [[TMP1]]
+; CHECK-NEXT: [[TMP12:%.*]] = icmp eq i32 [[AVL_NEXT]], 0
; CHECK-NEXT: br i1 [[TMP6]], label %[[VECTOR_EARLY_EXIT:.*]], label %[[VECTOR_BODY_INTERIM]]
; CHECK: [[VECTOR_BODY_INTERIM]]:
-; CHECK-NEXT: br i1 [[TMP7]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP8:![0-9]+]]
+; CHECK-NEXT: br i1 [[TMP12]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP5:![0-9]+]]
; CHECK: [[MIDDLE_BLOCK]]:
-; CHECK-NEXT: [[CMP_N:%.*]] = icmp eq i32 123, [[N_VEC]]
-; CHECK-NEXT: br i1 [[CMP_N]], label %[[EXIT:.*]], label %[[SCALAR_PH]]
+; CHECK-NEXT: br label %[[LOOP:.*]]
; CHECK: [[VECTOR_EARLY_EXIT]]:
-; CHECK-NEXT: [[TMP8:%.*]] = call i64 @llvm.experimental.cttz.elts.i64.nxv4i1(<vscale x 4 x i1> [[TMP5]], i1 false)
+; CHECK-NEXT: [[TMP8:%.*]] = call i64 @llvm.experimental.cttz.elts.i64.nxv4i1(<vscale x 4 x i1> [[TMP7]], i1 false)
; CHECK-NEXT: [[TMP9:%.*]] = trunc i64 [[TMP8]] to i32
; CHECK-NEXT: [[TMP10:%.*]] = add i32 [[INDEX]], [[TMP9]]
-; CHECK-NEXT: br label %[[EXIT]]
-; CHECK: [[SCALAR_PH]]:
-; CHECK-NEXT: [[BC_RESUME_VAL:%.*]] = phi i32 [ [[N_VEC]], %[[MIDDLE_BLOCK]] ], [ 0, %[[ENTRY]] ]
-; CHECK-NEXT: br label %[[LOOP:.*]]
+; CHECK-NEXT: br label %[[LOOP]]
; CHECK: [[LOOP]]:
-; CHECK-NEXT: [[IV:%.*]] = phi i32 [ [[BC_RESUME_VAL]], %[[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], %[[LATCH:.*]] ]
-; CHECK-NEXT: br i1 [[C0]], label %[[THEN:.*]], label %[[LATCH]]
-; CHECK: [[THEN]]:
-; CHECK-NEXT: [[GEP:%.*]] = getelementptr inbounds i32, ptr [[P]], i32 [[IV]]
-; CHECK-NEXT: [[LD:%.*]] = load i32, ptr [[GEP]], align 4
-; CHECK-NEXT: [[C1:%.*]] = icmp eq i32 [[LD]], 0
-; CHECK-NEXT: br i1 [[C1]], label %[[EXIT]], label %[[LATCH]]
-; CHECK: [[LATCH]]:
-; CHECK-NEXT: [[IV_NEXT]] = add nuw nsw i32 [[IV]], 1
-; CHECK-NEXT: [[EC:%.*]] = icmp eq i32 [[IV_NEXT]], 123
-; CHECK-NEXT: br i1 [[EC]], label %[[EXIT]], label %[[LOOP]], !llvm.loop [[LOOP9:![0-9]+]]
-; CHECK: [[EXIT]]:
-; CHECK-NEXT: [[RES:%.*]] = phi i32 [ [[IV]], %[[THEN]] ], [ -1, %[[LATCH]] ], [ -1, %[[MIDDLE_BLOCK]] ], [ [[TMP10]], %[[VECTOR_EARLY_EXIT]] ]
+; CHECK-NEXT: [[RES:%.*]] = phi i32 [ [[TMP10]], %[[VECTOR_EARLY_EXIT]] ], [ -1, %[[MIDDLE_BLOCK]] ]
; CHECK-NEXT: ret i32 [[RES]]
;
entry:
@@ -424,7 +352,7 @@ define void @early_exit_with_store(ptr align 4 dereferenceable(4096) noalias %ds
; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i32 [[INDEX]], [[TMP1]]
; CHECK-NEXT: [[TMP10:%.*]] = icmp eq i32 [[INDEX_NEXT]], [[N_VEC]]
; CHECK-NEXT: [[TMP11:%.*]] = or i1 [[TMP9]], [[TMP10]]
-; CHECK-NEXT: br i1 [[TMP11]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP10:![0-9]+]]
+; CHECK-NEXT: br i1 [[TMP11]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP6:![0-9]+]]
; CHECK: [[MIDDLE_BLOCK]]:
; CHECK-NEXT: [[TMP14:%.*]] = call i64 @llvm.experimental.cttz.elts.i64.nxv4i1(<vscale x 4 x i1> [[TMP4]], i1 false)
; CHECK-NEXT: [[TMP6:%.*]] = trunc i64 [[TMP14]] to i32
@@ -447,7 +375,7 @@ define void @early_exit_with_store(ptr align 4 dereferenceable(4096) noalias %ds
; CHECK: [[LATCH]]:
; CHECK-NEXT: [[IV_NEXT]] = add nuw nsw i32 [[IV]], 1
; CHECK-NEXT: [[EC:%.*]] = icmp eq i32 [[IV_NEXT]], 123
-; CHECK-NEXT: br i1 [[EC]], label %[[EXIT]], label %[[LOOP]], !llvm.loop [[LOOP11:![0-9]+]]
+; CHECK-NEXT: br i1 [[EC]], label %[[EXIT]], label %[[LOOP]], !llvm.loop [[LOOP7:![0-9]+]]
; CHECK: [[EXIT]]:
; CHECK-NEXT: ret void
;
@@ -502,7 +430,7 @@ define void @early_exit_with_store_after_exit(ptr align 4 dereferenceable(4096)
; CHECK-NEXT: [[TMP9:%.*]] = icmp eq i32 [[INDEX_NEXT]], [[N_VEC]]
; CHECK-NEXT: br i1 [[TMP8]], label %[[VECTOR_EARLY_EXIT:.*]], label %[[VECTOR_BODY_INTERIM]]
; CHECK: [[VECTOR_BODY_INTERIM]]:
-; CHECK-NEXT: br i1 [[TMP9]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP12:![0-9]+]]
+; CHECK-NEXT: br i1 [[TMP9]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP8:![0-9]+]]
; CHECK: [[MIDDLE_BLOCK]]:
; CHECK-NEXT: [[CMP_N:%.*]] = icmp eq i32 123, [[N_VEC]]
; CHECK-NEXT: br i1 [[CMP_N]], label %[[EXIT:.*]], label %[[SCALAR_PH]]
@@ -524,7 +452,7 @@ define void @early_exit_with_store_after_exit(ptr align 4 dereferenceable(4096)
; CHECK-NEXT: store i32 [[INC]], ptr [[ST_ADDR]], align 4
; CHECK-NEXT: [[IV_NEXT]] = add nuw nsw i32 [[IV]], 1
; CHECK-NEXT: [[EC:%.*]] = icmp eq i32 [[IV_NEXT]], 123
-; CHECK-NEXT: br i1 [[EC]], label %[[EXIT]], label %[[LOOP]], !llvm.loop [[LOOP13:![0-9]+]]
+; CHECK-NEXT: br i1 [[EC]], label %[[EXIT]], label %[[LOOP]], !llvm.loop [[LOOP9:![0-9]+]]
; CHECK: [[EXIT]]:
; CHECK-NEXT: ret void
;
diff --git a/llvm/test/Transforms/LoopVectorize/RISCV/sink-to-early-exit.ll b/llvm/test/Transforms/LoopVectorize/RISCV/sink-to-early-exit.ll
index c812ab30381f0d..67d1adeaf12f8f 100644
--- a/llvm/test/Transforms/LoopVectorize/RISCV/sink-to-early-exit.ll
+++ b/llvm/test/Transforms/LoopVectorize/RISCV/sink-to-early-exit.ll
@@ -8,67 +8,50 @@ declare void @init_mem(ptr, i64)
define i64 @sink_to_early_exit(i64 %offset) {
; CHECK-LABEL: define i64 @sink_to_early_exit(
; CHECK-SAME: i64 [[OFFSET:%.*]]) #[[ATTR0:[0-9]+]] {
-; CHECK-NEXT: [[ENTRY:.*]]:
+; CHECK-NEXT: [[ENTRY:.*:]]
; CHECK-NEXT: [[P1:%.*]] = alloca [1024 x i8], align 4
; CHECK-NEXT: [[P2:%.*]] = alloca [1024 x i8], align 4
; CHECK-NEXT: call void @init_mem(ptr [[P1]], i64 1024)
; CHECK-NEXT: call void @init_mem(ptr [[P2]], i64 1024)
-; CHECK-NEXT: [[TMP0:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-NEXT: [[TMP1:%.*]] = shl nuw i64 [[TMP0]], 4
-; CHECK-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 1024, [[TMP1]]
-; CHECK-NEXT: br i1 [[MIN_ITERS_CHECK]], label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]]
+; CHECK-NEXT: br label %[[VECTOR_PH:.*]]
; CHECK: [[VECTOR_PH]]:
-; CHECK-NEXT: [[N_MOD_VF:%.*]] = urem i64 1024, [[TMP1]]
-; CHECK-NEXT: [[N_VEC:%.*]] = sub i64 1024, [[N_MOD_VF]]
; CHECK-NEXT: [[BROADCAST_SPLATINSERT:%.*]] = insertelement <vscale x 16 x i64> poison, i64 [[OFFSET]], i64 0
; CHECK-NEXT: [[BROADCAST_SPLAT:%.*]] = shufflevector <vscale x 16 x i64> [[BROADCAST_SPLATINSERT]], <vscale x 16 x i64> poison, <vscale x 16 x i32> zeroinitializer
; CHECK-NEXT: [[TMP4:%.*]] = call <vscale x 16 x i64> @llvm.stepvector.nxv16i64()
-; CHECK-NEXT: [[BROADCAST_SPLATINSERT2:%.*]] = insertelement <vscale x 16 x i64> poison, i64 [[TMP1]], i64 0
-; CHECK-NEXT: [[BROADCAST_SPLAT3:%.*]] = shufflevector <vscale x 16 x i64> [[BROADCAST_SPLATINSERT2]], <vscale x 16 x i64> poison, <vscale x 16 x i32> zeroinitializer
; CHECK-NEXT: br label %[[VECTOR_BODY:.*]]
; CHECK: [[VECTOR_BODY]]:
-; CHECK-NEXT: [[INDEX4:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT6:%.*]], %[[VECTOR_BODY_INTERIM:.*]] ]
+; CHECK-NEXT: [[INDEX4:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[CURRENT_ITERATION_NEXT:%.*]], %[[VECTOR_BODY_INTERIM:.*]] ]
; CHECK-NEXT: [[VEC_IND:%.*]] = phi <vscale x 16 x i64> [ [[TMP4]], %[[VECTOR_PH]] ], [ [[VEC_IND_NEXT:%.*]], %[[VECTOR_BODY_INTERIM]] ]
+; CHECK-NEXT: [[AVL:%.*]] = phi i64 [ 1024, %[[VECTOR_PH]] ], [ [[AVL_NEXT:%.*]], %[[VECTOR_BODY_INTERIM]] ]
+; CHECK-NEXT: [[TMP1:%.*]] = call i32 @llvm.experimental.get.vector.length.i64(i64 [[AVL]], i32 16, i1 true)
+; CHECK-NEXT: [[TMP2:%.*]] = zext i32 [[TMP1]] to i64
+; CHECK-NEXT: [[BROADCAST_SPLATINSERT3:%.*]] = insertelement <vscale x 16 x i64> poison, i64 [[TMP2]], i64 0
+; CHECK-NEXT: [[BROADCAST_SPLAT3:%.*]] = shufflevector <vscale x 16 x i64> [[BROADCAST_SPLATINSERT3]], <vscale x 16 x i64> poison, <vscale x 16 x i32> zeroinitializer
; CHECK-NEXT: [[TMP5:%.*]] = getelementptr inbounds i8, ptr [[P1]], i64 [[INDEX4]]
-; CHECK-NEXT: [[WIDE_LOAD:%.*]] = load <vscale x 16 x i8>, ptr [[TMP5]], align 1
+; CHECK-NEXT: [[WIDE_LOAD:%.*]] = call <vscale x 16 x i8> @llvm.vp.load.nxv16i8.p0(ptr align 1 [[TMP5]], <vscale x 16 x i1> splat (i1 true), i32 [[TMP1]])
; CHECK-NEXT: [[TMP6:%.*]] = getelementptr inbounds i8, ptr [[P2]], i64 [[INDEX4]]
-; CHECK-NEXT: [[WIDE_LOAD5:%.*]] = load <vscale x 16 x i8>, ptr [[TMP6]], align 1
+; CHECK-NEXT: [[WIDE_LOAD5:%.*]] = call <vscale x 16 x i8> @llvm.vp.load.nxv16i8.p0(ptr align 1 [[TMP6]], <vscale x 16 x i1> splat (i1 true), i32 [[TMP1]])
; CHECK-NEXT: [[TMP8:%.*]] = icmp ne <vscale x 16 x i8> [[WIDE_LOAD]], [[WIDE_LOAD5]]
; CHECK-NEXT: [[TMP9:%.*]] = freeze <vscale x 16 x i1> [[TMP8]]
-; CHECK-NEXT: [[TMP10:%.*]] = call i1 @llvm.vector.reduce.or.nxv16i1(<vscale x 16 x i1> [[TMP9]])
-; CHECK-NEXT: [[INDEX_NEXT6]] = add nuw i64 [[INDEX4]], [[TMP1]]
-; CHECK-NEXT: [[TMP11:%.*]] = icmp eq i64 [[INDEX_NEXT6]], [[N_VEC]]
+; CHECK-NEXT: [[TMP11:%.*]] = call <vscale x 16 x i1> @llvm.vp.merge.nxv16i1(<vscale x 16 x i1> splat (i1 true), <vscale x 16 x i1> [[TMP9]], <vscale x 16 x i1> zeroinitializer, i32 [[TMP1]])
+; CHECK-NEXT: [[TMP14:%.*]] = call i1 @llvm.vp.reduce.or.nxv16i1(i1 false, <vscale x 16 x i1> [[TMP9]], <vscale x 16 x i1> splat (i1 true), i32 [[TMP1]])
+; CHECK-NEXT: [[TMP10:%.*]] = or i1 [[TMP14]], false
+; CHECK-NEXT: [[CURRENT_ITERATION_NEXT]] = add i64 [[TMP2]], [[INDEX4]]
+; CHECK-NEXT: [[AVL_NEXT]] = sub nuw i64 [[AVL]], [[TMP2]]
+; CHECK-NEXT: [[TMP13:%.*]] = icmp eq i64 [[AVL_NEXT]], 0
; CHECK-NEXT: [[VEC_IND_NEXT]] = add <vscale x 16 x i64> [[VEC_IND]], [[BROADCAST_SPLAT3]]
; CHECK-NEXT: br i1 [[TMP10]], label %[[VECTOR_EARLY_EXIT:.*]], label %[[VECTOR_BODY_INTERIM]]
; CHECK: [[VECTOR_BODY_INTERIM]]:
-; CHECK-NEXT: br i1 [[TMP11]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP0:![0-9]+]]
+; CHECK-NEXT: br i1 [[TMP13]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP0:![0-9]+]]
; CHECK: [[MIDDLE_BLOCK]]:
-; CHECK-NEXT: [[CMP_N:%.*]] = icmp eq i64 1024, [[N_VEC]]
-; CHECK-NEXT: br i1 [[CMP_N]], label %[[LOOP_END:.*]], label %[[SCALAR_PH]]
+; CHECK-NEXT: br label %[[LOOP_END:.*]]
; CHECK: [[VECTOR_EARLY_EXIT]]:
; CHECK-NEXT: [[TMP7:%.*]] = add <vscale x 16 x i64> [[VEC_IND]], [[BROADCAST_SPLAT]]
-; CHECK-NEXT: [[FIRST_ACTIVE_LANE:%.*]] = call i64 @llvm.experimental.cttz.elts.i64.nxv16i1(<vscale x 16 x i1> [[TMP9]], i1 false)
+; CHECK-NEXT: [[FIRST_ACTIVE_LANE:%.*]] = call i64 @llvm.experimental.cttz.elts.i64.nxv16i1(<vscale x 16 x i1> [[TMP11]], i1 false)
; CHECK-NEXT: [[TMP12:%.*]] = extractelement <vscale x 16 x i64> [[TMP7]], i64 [[FIRST_ACTIVE_LANE]]
-; CHECK-NEXT: br label %[[LOOP_EARLY_EXIT:.*]]
-; CHECK: [[SCALAR_PH]]:
-; CHECK-NEXT: [[BC_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC]], %[[MIDDLE_BLOCK]] ], [ 0, %[[ENTRY]] ]
; CHECK-NEXT: br label %[[LOOP:.*]]
; CHECK: [[LOOP]]:
-; CHECK-NEXT: [[INDEX:%.*]] = phi i64 [ [[INDEX_NEXT:%.*]], %[[LOOP_INC:.*]] ], [ [[BC_RESUME_VAL]], %[[SCALAR_PH]] ]
-; CHECK-NEXT: [[ARRAYIDX:%.*]] = getelementptr inbounds i8, ptr [[P1]], i64 [[INDEX]]
-; CHECK-NEXT: [[LD1:%.*]] = load i8, ptr [[ARRAYIDX]], align 1
-; CHECK-NEXT: [[ARRAYIDX1:%.*]] = getelementptr inbounds i8, ptr [[P2]], i64 [[INDEX]]
-; CHECK-NEXT: [[LD2:%.*]] = load i8, ptr [[ARRAYIDX1]], align 1
-; CHECK-NEXT: [[RESULT:%.*]] = add i64 [[INDEX]], [[OFFSET]]
-; CHECK-NEXT: [[CMP:%.*]] = icmp eq i8 [[LD1]], [[LD2]]
-; CHECK-NEXT: br i1 [[CMP]], label %[[LOOP_INC]], label %[[LOOP_EARLY_EXIT]]
-; CHECK: [[LOOP_INC]]:
-; CHECK-NEXT: [[INDEX_NEXT]] = add i64 [[INDEX]], 1
-; CHECK-NEXT: [[EXITCOND:%.*]] = icmp ne i64 [[INDEX_NEXT]], 1024
-; CHECK-NEXT: br i1 [[EXITCOND]], label %[[LOOP]], label %[[LOOP_END]], !llvm.loop [[LOOP3:![0-9]+]]
-; CHECK: [[LOOP_EARLY_EXIT]]:
-; CHECK-NEXT: [[RESULT_LCSSA:%.*]] = phi i64 [ [[RESULT]], %[[LOOP]] ], [ [[TMP12]], %[[VECTOR_EARLY_EXIT]] ]
-; CHECK-NEXT: ret i64 [[RESULT_LCSSA]]
+; CHECK-NEXT: ret i64 [[TMP12]]
; CHECK: [[LOOP_END]]:
; CHECK-NEXT: ret i64 0
;
diff --git a/llvm/test/Transforms/LoopVectorize/early-exit-tail-folding.ll b/llvm/test/Transforms/LoopVectorize/early-exit-tail-folding.ll
index 9026514723ae5d..cff27685627005 100644
--- a/llvm/test/Transforms/LoopVectorize/early-exit-tail-folding.ll
+++ b/llvm/test/Transforms/LoopVectorize/early-exit-tail-folding.ll
@@ -10,37 +10,29 @@ define i32 @single_early_exit(ptr align 4 dereferenceable(4096) %p) {
; CHECK-NEXT: br label %[[VECTOR_BODY:.*]]
; CHECK: [[VECTOR_BODY]]:
; CHECK-NEXT: [[INDEX:%.*]] = phi i32 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY_INTERIM:.*]] ]
+; CHECK-NEXT: [[VEC_IND:%.*]] = phi <4 x i8> [ <i8 0, i8 1, i8 2, i8 3>, %[[VECTOR_PH]] ], [ [[VEC_IND_NEXT:%.*]], %[[VECTOR_BODY_INTERIM]] ]
+; CHECK-NEXT: [[TMP8:%.*]] = icmp ule <4 x i8> [[VEC_IND]], splat (i8 122)
; CHECK-NEXT: [[TMP0:%.*]] = getelementptr inbounds i32, ptr [[P]], i32 [[INDEX]]
-; CHECK-NEXT: [[WIDE_LOAD:%.*]] = load <4 x i32>, ptr [[TMP0]], align 4
+; CHECK-NEXT: [[WIDE_LOAD:%.*]] = call <4 x i32> @llvm.masked.load.v4i32.p0(ptr align 4 [[TMP0]], <4 x i1> [[TMP8]], <4 x i32> poison)
; CHECK-NEXT: [[TMP1:%.*]] = icmp eq <4 x i32> [[WIDE_LOAD]], zeroinitializer
; CHECK-NEXT: [[TMP2:%.*]] = freeze <4 x i1> [[TMP1]]
-; CHECK-NEXT: [[TMP3:%.*]] = call i1 @llvm.vector.reduce.or.v4i1(<4 x i1> [[TMP2]])
+; CHECK-NEXT: [[TMP4:%.*]] = select <4 x i1> [[TMP8]], <4 x i1> [[TMP2]], <4 x i1> zeroinitializer
+; CHECK-NEXT: [[TMP3:%.*]] = call i1 @llvm.vector.reduce.or.v4i1(<4 x i1> [[TMP4]])
; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i32 [[INDEX]], 4
-; CHECK-NEXT: [[TMP4:%.*]] = icmp eq i32 [[INDEX_NEXT]], 120
+; CHECK-NEXT: [[TMP9:%.*]] = icmp eq i32 [[INDEX_NEXT]], 124
+; CHECK-NEXT: [[VEC_IND_NEXT]] = add nuw <4 x i8> [[VEC_IND]], splat (i8 4)
; CHECK-NEXT: br i1 [[TMP3]], label %[[VECTOR_EARLY_EXIT:.*]], label %[[VECTOR_BODY_INTERIM]]
; CHECK: [[VECTOR_BODY_INTERIM]]:
-; CHECK-NEXT: br i1 [[TMP4]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP0:![0-9]+]]
+; CHECK-NEXT: br i1 [[TMP9]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP0:![0-9]+]]
; CHECK: [[MIDDLE_BLOCK]]:
; CHECK-NEXT: br label %[[EXIT:.*]]
; CHECK: [[VECTOR_EARLY_EXIT]]:
-; CHECK-NEXT: [[TMP5:%.*]] = call i64 @llvm.experimental.cttz.elts.i64.v4i1(<4 x i1> [[TMP2]], i1 false)
+; CHECK-NEXT: [[TMP5:%.*]] = call i64 @llvm.experimental.cttz.elts.i64.v4i1(<4 x i1> [[TMP4]], i1 false)
; CHECK-NEXT: [[TMP6:%.*]] = trunc i64 [[TMP5]] to i32
; CHECK-NEXT: [[TMP7:%.*]] = add i32 [[INDEX]], [[TMP6]]
-; CHECK-NEXT: br label %[[EXIT1:.*]]
+; CHECK-NEXT: br label %[[EXIT]]
; CHECK: [[EXIT]]:
-; CHECK-NEXT: br label %[[LOOP:.*]]
-; CHECK: [[LOOP]]:
-; CHECK-NEXT: [[IV:%.*]] = phi i32 [ 120, %[[EXIT]] ], [ [[IV_NEXT:%.*]], %[[LATCH:.*]] ]
-; CHECK-NEXT: [[GEP:%.*]] = getelementptr inbounds i32, ptr [[P]], i32 [[IV]]
-; CHECK-NEXT: [[LD:%.*]] = load i32, ptr [[GEP]], align 4
-; CHECK-NEXT: [[C:%.*]] = icmp eq i32 [[LD]], 0
-; CHECK-NEXT: br i1 [[C]], label %[[EXIT1]], label %[[LATCH]]
-; CHECK: [[LATCH]]:
-; CHECK-NEXT: [[IV_NEXT]] = add nuw nsw i32 [[IV]], 1
-; CHECK-NEXT: [[EC:%.*]] = icmp eq i32 [[IV_NEXT]], 123
-; CHECK-NEXT: br i1 [[EC]], label %[[EXIT1]], label %[[LOOP]], !llvm.loop [[LOOP3:![0-9]+]]
-; CHECK: [[EXIT1]]:
-; CHECK-NEXT: [[RES:%.*]] = phi i32 [ [[IV]], %[[LOOP]] ], [ -1, %[[LATCH]] ], [ [[TMP7]], %[[VECTOR_EARLY_EXIT]] ]
+; CHECK-NEXT: [[RES:%.*]] = phi i32 [ [[TMP7]], %[[VECTOR_EARLY_EXIT]] ], [ -1, %[[MIDDLE_BLOCK]] ]
; CHECK-NEXT: ret i32 [[RES]]
;
entry:
@@ -73,55 +65,39 @@ define i32 @multi_early_exits_same_exits(ptr align 4 dereferenceable(4096) %p, p
; CHECK: [[VECTOR_BODY]]:
; CHECK-NEXT: [[INDEX:%.*]] = phi i32 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY_INTERIM:.*]] ]
; CHECK-NEXT: [[VEC_IND:%.*]] = phi <4 x i32> [ <i32 0, i32 1, i32 2, i32 3>, %[[VECTOR_PH]] ], [ [[VEC_IND_NEXT:%.*]], %[[VECTOR_BODY_INTERIM]] ]
+; CHECK-NEXT: [[TMP13:%.*]] = icmp ule <4 x i32> [[VEC_IND]], splat (i32 122)
; CHECK-NEXT: [[TMP0:%.*]] = getelementptr inbounds i32, ptr [[P]], i32 [[INDEX]]
-; CHECK-NEXT: [[WIDE_LOAD:%.*]] = load <4 x i32>, ptr [[TMP0]], align 4
+; CHECK-NEXT: [[WIDE_LOAD:%.*]] = call <4 x i32> @llvm.masked.load.v4i32.p0(ptr align 4 [[TMP0]], <4 x i1> [[TMP13]], <4 x i32> poison)
; CHECK-NEXT: [[TMP1:%.*]] = icmp eq <4 x i32> [[WIDE_LOAD]], zeroinitializer
; CHECK-NEXT: [[TMP2:%.*]] = getelementptr inbounds i32, ptr [[Q]], i32 [[INDEX]]
-; CHECK-NEXT: [[WIDE_LOAD1:%.*]] = load <4 x i32>, ptr [[TMP2]], align 4
+; CHECK-NEXT: [[WIDE_LOAD1:%.*]] = call <4 x i32> @llvm.masked.load.v4i32.p0(ptr align 4 [[TMP2]], <4 x i1> [[TMP13]], <4 x i32> poison)
; CHECK-NEXT: [[TMP3:%.*]] = icmp eq <4 x i32> [[WIDE_LOAD1]], zeroinitializer
; CHECK-NEXT: [[TMP4:%.*]] = add <4 x i32> [[VEC_IND]], splat (i32 1000)
; CHECK-NEXT: [[TMP5:%.*]] = select <4 x i1> [[TMP1]], <4 x i1> splat (i1 true), <4 x i1> [[TMP3]]
; CHECK-NEXT: [[TMP6:%.*]] = freeze <4 x i1> [[TMP5]]
-; CHECK-NEXT: [[TMP7:%.*]] = call i1 @llvm.vector.reduce.or.v4i1(<4 x i1> [[TMP6]])
+; CHECK-NEXT: [[TMP8:%.*]] = select <4 x i1> [[TMP13]], <4 x i1> [[TMP6]], <4 x i1> zeroinitializer
+; CHECK-NEXT: [[TMP7:%.*]] = call i1 @llvm.vector.reduce.or.v4i1(<4 x i1> [[TMP8]])
; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i32 [[INDEX]], 4
-; CHECK-NEXT: [[TMP8:%.*]] = icmp eq i32 [[INDEX_NEXT]], 120
-; CHECK-NEXT: [[VEC_IND_NEXT]] = add nuw nsw <4 x i32> [[VEC_IND]], splat (i32 4)
+; CHECK-NEXT: [[TMP14:%.*]] = icmp eq i32 [[INDEX_NEXT]], 124
+; CHECK-NEXT: [[VEC_IND_NEXT]] = add <4 x i32> [[VEC_IND]], splat (i32 4)
; CHECK-NEXT: br i1 [[TMP7]], label %[[VECTOR_EARLY_EXIT_CHECK:.*]], label %[[VECTOR_BODY_INTERIM]]
; CHECK: [[VECTOR_BODY_INTERIM]]:
-; CHECK-NEXT: br i1 [[TMP8]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP4:![0-9]+]]
+; CHECK-NEXT: br i1 [[TMP14]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP3:![0-9]+]]
; CHECK: [[MIDDLE_BLOCK]]:
; CHECK-NEXT: br label %[[EXIT:.*]]
; CHECK: [[VECTOR_EARLY_EXIT_CHECK]]:
-; CHECK-NEXT: [[FIRST_ACTIVE_LANE:%.*]] = call i64 @llvm.experimental.cttz.elts.i64.v4i1(<4 x i1> [[TMP6]], i1 false)
+; CHECK-NEXT: [[FIRST_ACTIVE_LANE:%.*]] = call i64 @llvm.experimental.cttz.elts.i64.v4i1(<4 x i1> [[TMP8]], i1 false)
; CHECK-NEXT: [[TMP9:%.*]] = extractelement <4 x i1> [[TMP1]], i64 [[FIRST_ACTIVE_LANE]]
; CHECK-NEXT: br i1 [[TMP9]], label %[[VECTOR_EARLY_EXIT_0:.*]], label %[[VECTOR_EARLY_EXIT_1:.*]]
; CHECK: [[VECTOR_EARLY_EXIT_1]]:
; CHECK-NEXT: [[TMP10:%.*]] = extractelement <4 x i32> [[TMP4]], i64 [[FIRST_ACTIVE_LANE]]
-; CHECK-NEXT: br label %[[EXIT1:.*]]
+; CHECK-NEXT: br label %[[EXIT]]
; CHECK: [[VECTOR_EARLY_EXIT_0]]:
; CHECK-NEXT: [[TMP11:%.*]] = trunc i64 [[FIRST_ACTIVE_LANE]] to i32
; CHECK-NEXT: [[TMP12:%.*]] = add i32 [[INDEX]], [[TMP11]]
-; CHECK-NEXT: br label %[[EXIT1]]
+; CHECK-NEXT: br label %[[EXIT]]
; CHECK: [[EXIT]]:
-; CHECK-NEXT: br label %[[LOOP:.*]]
-; CHECK: [[LOOP]]:
-; CHECK-NEXT: [[IV:%.*]] = phi i32 [ 120, %[[EXIT]] ], [ [[IV_NEXT:%.*]], %[[LATCH:.*]] ]
-; CHECK-NEXT: [[GEP_P:%.*]] = getelementptr inbounds i32, ptr [[P]], i32 [[IV]]
-; CHECK-NEXT: [[X:%.*]] = load i32, ptr [[GEP_P]], align 4
-; CHECK-NEXT: [[C1:%.*]] = icmp eq i32 [[X]], 0
-; CHECK-NEXT: br i1 [[C1]], label %[[EXIT1]], label %[[NEXT:.*]]
-; CHECK: [[NEXT]]:
-; CHECK-NEXT: [[GEP_Q:%.*]] = getelementptr inbounds i32, ptr [[Q]], i32 [[IV]]
-; CHECK-NEXT: [[Y:%.*]] = load i32, ptr [[GEP_Q]], align 4
-; CHECK-NEXT: [[C2:%.*]] = icmp eq i32 [[Y]], 0
-; CHECK-NEXT: [[IV_Q:%.*]] = add i32 [[IV]], 1000
-; CHECK-NEXT: br i1 [[C2]], label %[[EXIT1]], label %[[LATCH]]
-; CHECK: [[LATCH]]:
-; CHECK-NEXT: [[IV_NEXT]] = add nuw nsw i32 [[IV]], 1
-; CHECK-NEXT: [[EC:%.*]] = icmp eq i32 [[IV_NEXT]], 123
-; CHECK-NEXT: br i1 [[EC]], label %[[EXIT1]], label %[[LOOP]], !llvm.loop [[LOOP5:![0-9]+]]
-; CHECK: [[EXIT1]]:
-; CHECK-NEXT: [[RES:%.*]] = phi i32 [ [[IV]], %[[LOOP]] ], [ [[IV_Q]], %[[NEXT]] ], [ -1, %[[LATCH]] ], [ [[TMP12]], %[[VECTOR_EARLY_EXIT_0]] ], [ [[TMP10]], %[[VECTOR_EARLY_EXIT_1]] ]
+; CHECK-NEXT: [[RES:%.*]] = phi i32 [ [[TMP10]], %[[VECTOR_EARLY_EXIT_1]] ], [ [[TMP12]], %[[VECTOR_EARLY_EXIT_0]] ], [ -1, %[[MIDDLE_BLOCK]] ]
; CHECK-NEXT: ret i32 [[RES]]
;
entry:
@@ -161,26 +137,28 @@ define i32 @multi_early_exits_different_exits(ptr align 4 dereferenceable(4096)
; CHECK: [[VECTOR_BODY]]:
; CHECK-NEXT: [[INDEX:%.*]] = phi i32 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY_INTERIM:.*]] ]
; CHECK-NEXT: [[VEC_IND:%.*]] = phi <4 x i32> [ <i32 0, i32 1, i32 2, i32 3>, %[[VECTOR_PH]] ], [ [[VEC_IND_NEXT:%.*]], %[[VECTOR_BODY_INTERIM]] ]
+; CHECK-NEXT: [[TMP13:%.*]] = icmp ule <4 x i32> [[VEC_IND]], splat (i32 122)
; CHECK-NEXT: [[TMP0:%.*]] = getelementptr inbounds i32, ptr [[P]], i32 [[INDEX]]
-; CHECK-NEXT: [[WIDE_LOAD:%.*]] = load <4 x i32>, ptr [[TMP0]], align 4
+; CHECK-NEXT: [[WIDE_LOAD:%.*]] = call <4 x i32> @llvm.masked.load.v4i32.p0(ptr align 4 [[TMP0]], <4 x i1> [[TMP13]], <4 x i32> poison)
; CHECK-NEXT: [[TMP1:%.*]] = icmp eq <4 x i32> [[WIDE_LOAD]], zeroinitializer
; CHECK-NEXT: [[TMP2:%.*]] = getelementptr inbounds i32, ptr [[Q]], i32 [[INDEX]]
-; CHECK-NEXT: [[WIDE_LOAD1:%.*]] = load <4 x i32>, ptr [[TMP2]], align 4
+; CHECK-NEXT: [[WIDE_LOAD1:%.*]] = call <4 x i32> @llvm.masked.load.v4i32.p0(ptr align 4 [[TMP2]], <4 x i1> [[TMP13]], <4 x i32> poison)
; CHECK-NEXT: [[TMP3:%.*]] = icmp eq <4 x i32> [[WIDE_LOAD1]], zeroinitializer
; CHECK-NEXT: [[TMP4:%.*]] = add <4 x i32> [[VEC_IND]], splat (i32 1000)
; CHECK-NEXT: [[TMP5:%.*]] = select <4 x i1> [[TMP1]], <4 x i1> splat (i1 true), <4 x i1> [[TMP3]]
; CHECK-NEXT: [[TMP6:%.*]] = freeze <4 x i1> [[TMP5]]
-; CHECK-NEXT: [[TMP7:%.*]] = call i1 @llvm.vector.reduce.or.v4i1(<4 x i1> [[TMP6]])
+; CHECK-NEXT: [[TMP8:%.*]] = select <4 x i1> [[TMP13]], <4 x i1> [[TMP6]], <4 x i1> zeroinitializer
+; CHECK-NEXT: [[TMP7:%.*]] = call i1 @llvm.vector.reduce.or.v4i1(<4 x i1> [[TMP8]])
; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i32 [[INDEX]], 4
-; CHECK-NEXT: [[TMP8:%.*]] = icmp eq i32 [[INDEX_NEXT]], 120
-; CHECK-NEXT: [[VEC_IND_NEXT]] = add nuw nsw <4 x i32> [[VEC_IND]], splat (i32 4)
+; CHECK-NEXT: [[TMP14:%.*]] = icmp eq i32 [[INDEX_NEXT]], 124
+; CHECK-NEXT: [[VEC_IND_NEXT]] = add <4 x i32> [[VEC_IND]], splat (i32 4)
; CHECK-NEXT: br i1 [[TMP7]], label %[[VECTOR_EARLY_EXIT_CHECK:.*]], label %[[VECTOR_BODY_INTERIM]]
; CHECK: [[VECTOR_BODY_INTERIM]]:
-; CHECK-NEXT: br i1 [[TMP8]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP6:![0-9]+]]
+; CHECK-NEXT: br i1 [[TMP14]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP4:![0-9]+]]
; CHECK: [[MIDDLE_BLOCK]]:
; CHECK-NEXT: br label %[[EXIT_LATCH:.*]]
; CHECK: [[VECTOR_EARLY_EXIT_CHECK]]:
-; CHECK-NEXT: [[FIRST_ACTIVE_LANE:%.*]] = call i64 @llvm.experimental.cttz.elts.i64.v4i1(<4 x i1> [[TMP6]], i1 false)
+; CHECK-NEXT: [[FIRST_ACTIVE_LANE:%.*]] = call i64 @llvm.experimental.cttz.elts.i64.v4i1(<4 x i1> [[TMP8]], i1 false)
; CHECK-NEXT: [[TMP9:%.*]] = extractelement <4 x i1> [[TMP1]], i64 [[FIRST_ACTIVE_LANE]]
; CHECK-NEXT: br i1 [[TMP9]], label %[[VECTOR_EARLY_EXIT_0:.*]], label %[[VECTOR_EARLY_EXIT_1:.*]]
; CHECK: [[VECTOR_EARLY_EXIT_1]]:
@@ -189,32 +167,12 @@ define i32 @multi_early_exits_different_exits(ptr align 4 dereferenceable(4096)
; CHECK: [[VECTOR_EARLY_EXIT_0]]:
; CHECK-NEXT: [[TMP11:%.*]] = trunc i64 [[FIRST_ACTIVE_LANE]] to i32
; CHECK-NEXT: [[TMP12:%.*]] = add i32 [[INDEX]], [[TMP11]]
-; CHECK-NEXT: br label %[[EXIT_P:.*]]
-; CHECK: [[EXIT_LATCH]]:
; CHECK-NEXT: br label %[[LOOP:.*]]
; CHECK: [[LOOP]]:
-; CHECK-NEXT: [[IV:%.*]] = phi i32 [ 120, %[[EXIT_LATCH]] ], [ [[IV_NEXT:%.*]], %[[LATCH:.*]] ]
-; CHECK-NEXT: [[GEP_P:%.*]] = getelementptr inbounds i32, ptr [[P]], i32 [[IV]]
-; CHECK-NEXT: [[X:%.*]] = load i32, ptr [[GEP_P]], align 4
-; CHECK-NEXT: [[C1:%.*]] = icmp eq i32 [[X]], 0
-; CHECK-NEXT: br i1 [[C1]], label %[[EXIT_P]], label %[[NEXT:.*]]
-; CHECK: [[NEXT]]:
-; CHECK-NEXT: [[GEP_Q:%.*]] = getelementptr inbounds i32, ptr [[Q]], i32 [[IV]]
-; CHECK-NEXT: [[Y:%.*]] = load i32, ptr [[GEP_Q]], align 4
-; CHECK-NEXT: [[C2:%.*]] = icmp eq i32 [[Y]], 0
-; CHECK-NEXT: [[IV_Q:%.*]] = add i32 [[IV]], 1000
-; CHECK-NEXT: br i1 [[C2]], label %[[EXIT_Q]], label %[[LATCH]]
-; CHECK: [[LATCH]]:
-; CHECK-NEXT: [[IV_NEXT]] = add nuw nsw i32 [[IV]], 1
-; CHECK-NEXT: [[EC:%.*]] = icmp eq i32 [[IV_NEXT]], 123
-; CHECK-NEXT: br i1 [[EC]], label %[[EXIT_LATCH1:.*]], label %[[LOOP]], !llvm.loop [[LOOP7:![0-9]+]]
-; CHECK: [[EXIT_P]]:
-; CHECK-NEXT: [[RES_P:%.*]] = phi i32 [ [[IV]], %[[LOOP]] ], [ [[TMP12]], %[[VECTOR_EARLY_EXIT_0]] ]
-; CHECK-NEXT: ret i32 [[RES_P]]
+; CHECK-NEXT: ret i32 [[TMP12]]
; CHECK: [[EXIT_Q]]:
-; CHECK-NEXT: [[RES_Q:%.*]] = phi i32 [ [[IV_Q]], %[[NEXT]] ], [ [[TMP10]], %[[VECTOR_EARLY_EXIT_1]] ]
-; CHECK-NEXT: ret i32 [[RES_Q]]
-; CHECK: [[EXIT_LATCH1]]:
+; CHECK-NEXT: ret i32 [[TMP10]]
+; CHECK: [[EXIT_LATCH]]:
; CHECK-NEXT: ret i32 -1
;
entry:
@@ -263,40 +221,31 @@ define i32 @predicated_early_exit(i1 %c0, ptr align 4 dereferenceable(4096) %p)
; CHECK-NEXT: br label %[[VECTOR_BODY:.*]]
; CHECK: [[VECTOR_BODY]]:
; CHECK-NEXT: [[INDEX:%.*]] = phi i32 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY_INTERIM:.*]] ]
-; CHECK-NEXT: [[TMP0:%.*]] = getelementptr inbounds i32, ptr [[P]], i32 [[INDEX]]
-; CHECK-NEXT: [[WIDE_LOAD:%.*]] = load <4 x i32>, ptr [[TMP0]], align 4
+; CHECK-NEXT: [[VEC_IND:%.*]] = phi <4 x i8> [ <i8 0, i8 1, i8 2, i8 3>, %[[VECTOR_PH]] ], [ [[VEC_IND_NEXT:%.*]], %[[VECTOR_BODY_INTERIM]] ]
+; CHECK-NEXT: [[TMP5:%.*]] = icmp ule <4 x i8> [[VEC_IND]], splat (i8 122)
+; CHECK-NEXT: [[TMP10:%.*]] = select <4 x i1> [[TMP5]], <4 x i1> [[TMP9]], <4 x i1> zeroinitializer
+; CHECK-NEXT: [[TMP11:%.*]] = getelementptr i32, ptr [[P]], i32 [[INDEX]]
+; CHECK-NEXT: [[WIDE_LOAD:%.*]] = call <4 x i32> @llvm.masked.load.v4i32.p0(ptr align 4 [[TMP11]], <4 x i1> [[TMP10]], <4 x i32> poison)
; CHECK-NEXT: [[TMP1:%.*]] = icmp eq <4 x i32> [[WIDE_LOAD]], zeroinitializer
; CHECK-NEXT: [[TMP2:%.*]] = select <4 x i1> [[TMP9]], <4 x i1> [[TMP1]], <4 x i1> zeroinitializer
; CHECK-NEXT: [[TMP3:%.*]] = freeze <4 x i1> [[TMP2]]
-; CHECK-NEXT: [[TMP4:%.*]] = call i1 @llvm.vector.reduce.or.v4i1(<4 x i1> [[TMP3]])
+; CHECK-NEXT: [[TMP12:%.*]] = select <4 x i1> [[TMP5]], <4 x i1> [[TMP3]], <4 x i1> zeroinitializer
+; CHECK-NEXT: [[TMP4:%.*]] = call i1 @llvm.vector.reduce.or.v4i1(<4 x i1> [[TMP12]])
; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i32 [[INDEX]], 4
-; CHECK-NEXT: [[TMP5:%.*]] = icmp eq i32 [[INDEX_NEXT]], 120
+; CHECK-NEXT: [[TMP13:%.*]] = icmp eq i32 [[INDEX_NEXT]], 124
+; CHECK-NEXT: [[VEC_IND_NEXT]] = add nuw <4 x i8> [[VEC_IND]], splat (i8 4)
; CHECK-NEXT: br i1 [[TMP4]], label %[[VECTOR_EARLY_EXIT:.*]], label %[[VECTOR_BODY_INTERIM]]
; CHECK: [[VECTOR_BODY_INTERIM]]:
-; CHECK-NEXT: br i1 [[TMP5]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP8:![0-9]+]]
+; CHECK-NEXT: br i1 [[TMP13]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP5:![0-9]+]]
; CHECK: [[MIDDLE_BLOCK]]:
; CHECK-NEXT: br label %[[EXIT:.*]]
; CHECK: [[VECTOR_EARLY_EXIT]]:
-; CHECK-NEXT: [[TMP6:%.*]] = call i64 @llvm.experimental.cttz.elts.i64.v4i1(<4 x i1> [[TMP3]], i1 false)
+; CHECK-NEXT: [[TMP6:%.*]] = call i64 @llvm.experimental.cttz.elts.i64.v4i1(<4 x i1> [[TMP12]], i1 false)
; CHECK-NEXT: [[TMP7:%.*]] = trunc i64 [[TMP6]] to i32
; CHECK-NEXT: [[TMP8:%.*]] = add i32 [[INDEX]], [[TMP7]]
-; CHECK-NEXT: br label %[[EXIT1:.*]]
+; CHECK-NEXT: br label %[[EXIT]]
; CHECK: [[EXIT]]:
-; CHECK-NEXT: br label %[[LOOP:.*]]
-; CHECK: [[LOOP]]:
-; CHECK-NEXT: [[IV:%.*]] = phi i32 [ 120, %[[EXIT]] ], [ [[IV_NEXT:%.*]], %[[LATCH:.*]] ]
-; CHECK-NEXT: br i1 [[C0]], label %[[THEN:.*]], label %[[LATCH]]
-; CHECK: [[THEN]]:
-; CHECK-NEXT: [[GEP:%.*]] = getelementptr inbounds i32, ptr [[P]], i32 [[IV]]
-; CHECK-NEXT: [[LD:%.*]] = load i32, ptr [[GEP]], align 4
-; CHECK-NEXT: [[C1:%.*]] = icmp eq i32 [[LD]], 0
-; CHECK-NEXT: br i1 [[C1]], label %[[EXIT1]], label %[[LATCH]]
-; CHECK: [[LATCH]]:
-; CHECK-NEXT: [[IV_NEXT]] = add nuw nsw i32 [[IV]], 1
-; CHECK-NEXT: [[EC:%.*]] = icmp eq i32 [[IV_NEXT]], 123
-; CHECK-NEXT: br i1 [[EC]], label %[[EXIT1]], label %[[LOOP]], !llvm.loop [[LOOP9:![0-9]+]]
-; CHECK: [[EXIT1]]:
-; CHECK-NEXT: [[RES:%.*]] = phi i32 [ [[IV]], %[[THEN]] ], [ -1, %[[LATCH]] ], [ [[TMP8]], %[[VECTOR_EARLY_EXIT]] ]
+; CHECK-NEXT: [[RES:%.*]] = phi i32 [ [[TMP8]], %[[VECTOR_EARLY_EXIT]] ], [ -1, %[[MIDDLE_BLOCK]] ]
; CHECK-NEXT: ret i32 [[RES]]
;
entry:
@@ -341,7 +290,7 @@ define i32 @countable_early_exit(ptr align 4 dereferenceable(4096) %p, i32 %n) {
; CHECK-NEXT: [[INDEX:%.*]] = phi i32 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i32 [[INDEX]], 4
; CHECK-NEXT: [[TMP5:%.*]] = icmp eq i32 [[INDEX_NEXT]], [[N_VEC]]
-; CHECK-NEXT: br i1 [[TMP5]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP10:![0-9]+]]
+; CHECK-NEXT: br i1 [[TMP5]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP6:![0-9]+]]
; CHECK: [[MIDDLE_BLOCK]]:
; CHECK-NEXT: br label %[[SCALAR_PH]]
; CHECK: [[SCALAR_PH]]:
@@ -356,7 +305,7 @@ define i32 @countable_early_exit(ptr align 4 dereferenceable(4096) %p, i32 %n) {
; CHECK-NEXT: [[LD:%.*]] = load i32, ptr [[GEP]], align 4
; CHECK-NEXT: [[IV_NEXT]] = add nuw nsw i32 [[IV]], 1
; CHECK-NEXT: [[EC:%.*]] = icmp eq i32 [[IV_NEXT]], 123
-; CHECK-NEXT: br i1 [[EC]], label %[[EXIT]], label %[[LOOP]], !llvm.loop [[LOOP11:![0-9]+]]
+; CHECK-NEXT: br i1 [[EC]], label %[[EXIT]], label %[[LOOP]], !llvm.loop [[LOOP7:![0-9]+]]
; CHECK: [[EXIT]]:
; CHECK-NEXT: [[RES:%.*]] = phi i32 [ 0, %[[LOOP]] ], [ [[LD]], %[[LATCH]] ]
; CHECK-NEXT: ret i32 [[RES]]
@@ -404,7 +353,7 @@ define void @early_exit_with_store(ptr align 4 dereferenceable(4096) noalias %ds
; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i32 [[INDEX]], 4
; CHECK-NEXT: [[TMP8:%.*]] = icmp eq i32 [[INDEX_NEXT]], 120
; CHECK-NEXT: [[TMP9:%.*]] = or i1 [[TMP7]], [[TMP8]]
-; CHECK-NEXT: br i1 [[TMP9]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP12:![0-9]+]]
+; CHECK-NEXT: br i1 [[TMP9]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP8:![0-9]+]]
; CHECK: [[MIDDLE_BLOCK]]:
; CHECK-NEXT: [[TMP12:%.*]] = call i64 @llvm.experimental.cttz.elts.i64.v4i1(<4 x i1> [[TMP2]], i1 false)
; CHECK-NEXT: [[TMP4:%.*]] = trunc i64 [[TMP12]] to i32
@@ -426,7 +375,7 @@ define void @early_exit_with_store(ptr align 4 dereferenceable(4096) noalias %ds
; CHECK: [[LATCH]]:
; CHECK-NEXT: [[IV_NEXT]] = add nuw nsw i32 [[IV]], 1
; CHECK-NEXT: [[EC:%.*]] = icmp eq i32 [[IV_NEXT]], 123
-; CHECK-NEXT: br i1 [[EC]], label %[[EXIT]], label %[[LOOP]], !llvm.loop [[LOOP13:![0-9]+]]
+; CHECK-NEXT: br i1 [[EC]], label %[[EXIT]], label %[[LOOP]], !llvm.loop [[LOOP9:![0-9]+]]
; CHECK: [[EXIT]]:
; CHECK-NEXT: ret void
;
@@ -476,7 +425,7 @@ define void @early_exit_with_store_after_exit(ptr align 4 dereferenceable(4096)
; CHECK-NEXT: [[TMP7:%.*]] = icmp eq i32 [[INDEX_NEXT]], 120
; CHECK-NEXT: br i1 [[TMP6]], label %[[VECTOR_EARLY_EXIT:.*]], label %[[VECTOR_BODY_INTERIM]]
; CHECK: [[VECTOR_BODY_INTERIM]]:
-; CHECK-NEXT: br i1 [[TMP7]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP14:![0-9]+]]
+; CHECK-NEXT: br i1 [[TMP7]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP10:![0-9]+]]
; CHECK: [[MIDDLE_BLOCK]]:
; CHECK-NEXT: br label %[[SCALAR_PH:.*]]
; CHECK: [[VECTOR_EARLY_EXIT]]:
@@ -496,7 +445,7 @@ define void @early_exit_with_store_after_exit(ptr align 4 dereferenceable(4096)
; CHECK-NEXT: store i32 [[INC]], ptr [[ST_ADDR]], align 4
; CHECK-NEXT: [[IV_NEXT]] = add nuw nsw i32 [[IV]], 1
; CHECK-NEXT: [[EC:%.*]] = icmp eq i32 [[IV_NEXT]], 123
-; CHECK-NEXT: br i1 [[EC]], label %[[EXIT]], label %[[LOOP]], !llvm.loop [[LOOP15:![0-9]+]]
+; CHECK-NEXT: br i1 [[EC]], label %[[EXIT]], label %[[LOOP]], !llvm.loop [[LOOP11:![0-9]+]]
; CHECK: [[EXIT]]:
; CHECK-NEXT: ret void
;
diff --git a/llvm/test/Transforms/LoopVectorize/multiple-early-exits.ll b/llvm/test/Transforms/LoopVectorize/multiple-early-exits.ll
index c3bbdc5adf214e..08f49a2560d8d9 100644
--- a/llvm/test/Transforms/LoopVectorize/multiple-early-exits.ll
+++ b/llvm/test/Transforms/LoopVectorize/multiple-early-exits.ll
@@ -1491,29 +1491,121 @@ exit:
define i64 @three_early_exits_same_block_no_latch_exit(i1 %cmp0) {
; CHECK-LABEL: define i64 @three_early_exits_same_block_no_latch_exit(
; CHECK-SAME: i1 [[CMP0:%.*]]) {
-; CHECK-NEXT: [[ENTRY:.*]]:
+; CHECK-NEXT: [[ENTRY:.*:]]
; CHECK-NEXT: br label %[[LOOP_HEADER:.*]]
; CHECK: [[LOOP_HEADER]]:
-; CHECK-NEXT: [[IV:%.*]] = phi i64 [ 0, %[[ENTRY]] ], [ [[IV_NEXT:%.*]], %[[LOOP_LATCH:.*]] ]
-; CHECK-NEXT: br i1 [[CMP0]], label %[[EARLY_EXIT:.*]], label %[[LOOP_BODY_0:.*]]
+; CHECK-NEXT: [[BROADCAST_SPLATINSERT:%.*]] = insertelement <4 x i1> poison, i1 [[CMP0]], i64 0
+; CHECK-NEXT: [[BROADCAST_SPLAT:%.*]] = shufflevector <4 x i1> [[BROADCAST_SPLATINSERT]], <4 x i1> poison, <4 x i32> zeroinitializer
+; CHECK-NEXT: br label %[[LOOP_BODY_0:.*]]
; CHECK: [[LOOP_BODY_0]]:
+; CHECK-NEXT: [[IV:%.*]] = phi i64 [ 0, %[[LOOP_HEADER]] ], [ [[INDEX_NEXT:%.*]], %[[PRED_LOAD_CONTINUE_INTERIM:.*]] ]
+; CHECK-NEXT: [[VEC_IND:%.*]] = phi <4 x i8> [ <i8 0, i8 1, i8 2, i8 3>, %[[LOOP_HEADER]] ], [ [[VEC_IND_NEXT:%.*]], %[[PRED_LOAD_CONTINUE_INTERIM]] ]
+; CHECK-NEXT: [[TMP0:%.*]] = icmp ule <4 x i8> [[VEC_IND]], zeroinitializer
+; CHECK-NEXT: [[TMP1:%.*]] = add i64 [[IV]], 1
+; CHECK-NEXT: [[TMP2:%.*]] = add i64 [[IV]], 2
+; CHECK-NEXT: [[TMP3:%.*]] = add i64 [[IV]], 3
; CHECK-NEXT: [[GEP:%.*]] = getelementptr [24 x i8], ptr @GlobA, i64 [[IV]]
+; CHECK-NEXT: [[TMP5:%.*]] = getelementptr [24 x i8], ptr @GlobA, i64 [[TMP1]]
+; CHECK-NEXT: [[TMP6:%.*]] = getelementptr [24 x i8], ptr @GlobA, i64 [[TMP2]]
+; CHECK-NEXT: [[TMP7:%.*]] = getelementptr [24 x i8], ptr @GlobA, i64 [[TMP3]]
+; CHECK-NEXT: [[TMP8:%.*]] = extractelement <4 x i1> [[TMP0]], i64 0
+; CHECK-NEXT: br i1 [[TMP8]], label %[[PRED_LOAD_IF:.*]], label %[[PRED_LOAD_CONTINUE:.*]]
+; CHECK: [[PRED_LOAD_IF]]:
; CHECK-NEXT: [[GEP_1:%.*]] = getelementptr i8, ptr [[GEP]], i64 4
; CHECK-NEXT: [[L_0:%.*]] = load i8, ptr [[GEP_1]], align 1
-; CHECK-NEXT: [[CMP1:%.*]] = icmp eq i8 [[L_0]], 0
-; CHECK-NEXT: br i1 [[CMP1]], label %[[EARLY_EXIT]], label %[[LOOP_BODY_1:.*]]
+; CHECK-NEXT: [[TMP11:%.*]] = insertelement <4 x i8> poison, i8 [[L_0]], i64 0
+; CHECK-NEXT: br label %[[PRED_LOAD_CONTINUE]]
+; CHECK: [[PRED_LOAD_CONTINUE]]:
+; CHECK-NEXT: [[TMP12:%.*]] = phi <4 x i8> [ poison, %[[LOOP_BODY_0]] ], [ [[TMP11]], %[[PRED_LOAD_IF]] ]
+; CHECK-NEXT: [[CMP1:%.*]] = extractelement <4 x i1> [[TMP0]], i64 1
+; CHECK-NEXT: br i1 [[CMP1]], label %[[EARLY_EXIT:.*]], label %[[LOOP_BODY_1:.*]]
+; CHECK: [[EARLY_EXIT]]:
+; CHECK-NEXT: [[TMP14:%.*]] = getelementptr i8, ptr [[TMP5]], i64 4
+; CHECK-NEXT: [[TMP15:%.*]] = load i8, ptr [[TMP14]], align 1
+; CHECK-NEXT: [[TMP16:%.*]] = insertelement <4 x i8> [[TMP12]], i8 [[TMP15]], i64 1
+; CHECK-NEXT: br label %[[LOOP_BODY_1]]
; CHECK: [[LOOP_BODY_1]]:
-; CHECK-NEXT: [[L_1:%.*]] = load i8, ptr [[GEP]], align 1
-; CHECK-NEXT: [[CMP2:%.*]] = icmp eq i8 [[L_1]], 0
-; CHECK-NEXT: br i1 [[CMP2]], label %[[EARLY_EXIT]], label %[[LOOP_LATCH]]
+; CHECK-NEXT: [[TMP17:%.*]] = phi <4 x i8> [ [[TMP12]], %[[PRED_LOAD_CONTINUE]] ], [ [[TMP16]], %[[EARLY_EXIT]] ]
+; CHECK-NEXT: [[TMP18:%.*]] = extractelement <4 x i1> [[TMP0]], i64 2
+; CHECK-NEXT: br i1 [[TMP18]], label %[[LOOP_LATCH:.*]], label %[[PRED_LOAD_CONTINUE5:.*]]
; CHECK: [[LOOP_LATCH]]:
-; CHECK-NEXT: [[IV_NEXT]] = add nuw nsw i64 [[IV]], 1
-; CHECK-NEXT: [[EC:%.*]] = icmp eq i64 [[IV]], 0
-; CHECK-NEXT: br i1 [[EC]], label %[[LATCH_EXIT:.*]], label %[[LOOP_HEADER]]
+; CHECK-NEXT: [[TMP19:%.*]] = getelementptr i8, ptr [[TMP6]], i64 4
+; CHECK-NEXT: [[TMP20:%.*]] = load i8, ptr [[TMP19]], align 1
+; CHECK-NEXT: [[TMP21:%.*]] = insertelement <4 x i8> [[TMP17]], i8 [[TMP20]], i64 2
+; CHECK-NEXT: br label %[[PRED_LOAD_CONTINUE5]]
+; CHECK: [[PRED_LOAD_CONTINUE5]]:
+; CHECK-NEXT: [[TMP22:%.*]] = phi <4 x i8> [ [[TMP17]], %[[LOOP_BODY_1]] ], [ [[TMP21]], %[[LOOP_LATCH]] ]
+; CHECK-NEXT: [[TMP23:%.*]] = extractelement <4 x i1> [[TMP0]], i64 3
+; CHECK-NEXT: br i1 [[TMP23]], label %[[LATCH_EXIT:.*]], label %[[PRED_LOAD_CONTINUE7:.*]]
; CHECK: [[LATCH_EXIT]]:
+; CHECK-NEXT: [[TMP24:%.*]] = getelementptr i8, ptr [[TMP7]], i64 4
+; CHECK-NEXT: [[TMP25:%.*]] = load i8, ptr [[TMP24]], align 1
+; CHECK-NEXT: [[TMP26:%.*]] = insertelement <4 x i8> [[TMP22]], i8 [[TMP25]], i64 3
+; CHECK-NEXT: br label %[[PRED_LOAD_CONTINUE7]]
+; CHECK: [[PRED_LOAD_CONTINUE7]]:
+; CHECK-NEXT: [[TMP27:%.*]] = phi <4 x i8> [ [[TMP22]], %[[PRED_LOAD_CONTINUE5]] ], [ [[TMP26]], %[[LATCH_EXIT]] ]
+; CHECK-NEXT: [[TMP28:%.*]] = icmp eq <4 x i8> [[TMP27]], zeroinitializer
+; CHECK-NEXT: br i1 [[TMP8]], label %[[PRED_LOAD_IF8:.*]], label %[[PRED_LOAD_CONTINUE9:.*]]
+; CHECK: [[PRED_LOAD_IF8]]:
+; CHECK-NEXT: [[TMP29:%.*]] = load i8, ptr [[GEP]], align 1
+; CHECK-NEXT: [[TMP30:%.*]] = insertelement <4 x i8> poison, i8 [[TMP29]], i64 0
+; CHECK-NEXT: br label %[[PRED_LOAD_CONTINUE9]]
+; CHECK: [[PRED_LOAD_CONTINUE9]]:
+; CHECK-NEXT: [[TMP31:%.*]] = phi <4 x i8> [ poison, %[[PRED_LOAD_CONTINUE7]] ], [ [[TMP30]], %[[PRED_LOAD_IF8]] ]
+; CHECK-NEXT: br i1 [[CMP1]], label %[[PRED_LOAD_IF10:.*]], label %[[PRED_LOAD_CONTINUE11:.*]]
+; CHECK: [[PRED_LOAD_IF10]]:
+; CHECK-NEXT: [[TMP32:%.*]] = load i8, ptr [[TMP5]], align 1
+; CHECK-NEXT: [[TMP33:%.*]] = insertelement <4 x i8> [[TMP31]], i8 [[TMP32]], i64 1
+; CHECK-NEXT: br label %[[PRED_LOAD_CONTINUE11]]
+; CHECK: [[PRED_LOAD_CONTINUE11]]:
+; CHECK-NEXT: [[TMP34:%.*]] = phi <4 x i8> [ [[TMP31]], %[[PRED_LOAD_CONTINUE9]] ], [ [[TMP33]], %[[PRED_LOAD_IF10]] ]
+; CHECK-NEXT: br i1 [[TMP18]], label %[[PRED_LOAD_IF12:.*]], label %[[PRED_LOAD_CONTINUE13:.*]]
+; CHECK: [[PRED_LOAD_IF12]]:
+; CHECK-NEXT: [[TMP35:%.*]] = load i8, ptr [[TMP6]], align 1
+; CHECK-NEXT: [[TMP36:%.*]] = insertelement <4 x i8> [[TMP34]], i8 [[TMP35]], i64 2
+; CHECK-NEXT: br label %[[PRED_LOAD_CONTINUE13]]
+; CHECK: [[PRED_LOAD_CONTINUE13]]:
+; CHECK-NEXT: [[TMP37:%.*]] = phi <4 x i8> [ [[TMP34]], %[[PRED_LOAD_CONTINUE11]] ], [ [[TMP36]], %[[PRED_LOAD_IF12]] ]
+; CHECK-NEXT: br i1 [[TMP23]], label %[[PRED_LOAD_IF14:.*]], label %[[PRED_LOAD_CONTINUE15:.*]]
+; CHECK: [[PRED_LOAD_IF14]]:
+; CHECK-NEXT: [[TMP38:%.*]] = load i8, ptr [[TMP7]], align 1
+; CHECK-NEXT: [[TMP39:%.*]] = insertelement <4 x i8> [[TMP37]], i8 [[TMP38]], i64 3
+; CHECK-NEXT: br label %[[PRED_LOAD_CONTINUE15]]
+; CHECK: [[PRED_LOAD_CONTINUE15]]:
+; CHECK-NEXT: [[TMP40:%.*]] = phi <4 x i8> [ [[TMP37]], %[[PRED_LOAD_CONTINUE13]] ], [ [[TMP39]], %[[PRED_LOAD_IF14]] ]
+; CHECK-NEXT: [[TMP41:%.*]] = icmp eq <4 x i8> [[TMP40]], zeroinitializer
+; CHECK-NEXT: [[TMP42:%.*]] = select <4 x i1> [[BROADCAST_SPLAT]], <4 x i1> splat (i1 true), <4 x i1> [[TMP28]]
+; CHECK-NEXT: [[TMP43:%.*]] = select <4 x i1> [[TMP42]], <4 x i1> splat (i1 true), <4 x i1> [[TMP41]]
+; CHECK-NEXT: [[TMP44:%.*]] = freeze <4 x i1> [[TMP43]]
+; CHECK-NEXT: [[TMP45:%.*]] = select <4 x i1> [[TMP0]], <4 x i1> [[TMP44]], <4 x i1> zeroinitializer
+; CHECK-NEXT: [[TMP46:%.*]] = call i1 @llvm.vector.reduce.or.v4i1(<4 x i1> [[TMP45]])
+; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[IV]], 4
+; CHECK-NEXT: [[TMP47:%.*]] = icmp eq i64 [[INDEX_NEXT]], 4
+; CHECK-NEXT: [[VEC_IND_NEXT]] = add nuw <4 x i8> [[VEC_IND]], splat (i8 4)
+; CHECK-NEXT: br i1 [[TMP46]], label %[[VECTOR_EARLY_EXIT_CHECK:.*]], label %[[PRED_LOAD_CONTINUE_INTERIM]]
+; CHECK: [[PRED_LOAD_CONTINUE_INTERIM]]:
+; CHECK-NEXT: br i1 [[TMP47]], label %[[MIDDLE_BLOCK:.*]], label %[[LOOP_BODY_0]], !llvm.loop [[LOOP19:![0-9]+]]
+; CHECK: [[MIDDLE_BLOCK]]:
+; CHECK-NEXT: br label %[[LATCH_EXIT1:.*]]
+; CHECK: [[VECTOR_EARLY_EXIT_CHECK]]:
+; CHECK-NEXT: [[FIRST_ACTIVE_LANE:%.*]] = call i64 @llvm.experimental.cttz.elts.i64.v4i1(<4 x i1> [[TMP45]], i1 false)
+; CHECK-NEXT: br i1 [[CMP0]], label %[[VECTOR_EARLY_EXIT_0:.*]], label %[[VECTOR_EARLY_EXIT_CHECK_0:.*]]
+; CHECK: [[VECTOR_EARLY_EXIT_CHECK_0]]:
+; CHECK-NEXT: [[TMP48:%.*]] = extractelement <4 x i1> [[TMP28]], i64 [[FIRST_ACTIVE_LANE]]
+; CHECK-NEXT: br i1 [[TMP48]], label %[[VECTOR_EARLY_EXIT_1:.*]], label %[[VECTOR_EARLY_EXIT_2:.*]]
+; CHECK: [[VECTOR_EARLY_EXIT_2]]:
+; CHECK-NEXT: [[TMP49:%.*]] = add i64 [[IV]], [[FIRST_ACTIVE_LANE]]
+; CHECK-NEXT: br label %[[EARLY_EXIT1:.*]]
+; CHECK: [[VECTOR_EARLY_EXIT_1]]:
+; CHECK-NEXT: [[TMP50:%.*]] = add i64 [[IV]], [[FIRST_ACTIVE_LANE]]
+; CHECK-NEXT: br label %[[EARLY_EXIT1]]
+; CHECK: [[VECTOR_EARLY_EXIT_0]]:
+; CHECK-NEXT: [[TMP51:%.*]] = add i64 [[IV]], [[FIRST_ACTIVE_LANE]]
+; CHECK-NEXT: br label %[[EARLY_EXIT1]]
+; CHECK: [[LATCH_EXIT1]]:
; CHECK-NEXT: ret i64 0
-; CHECK: [[EARLY_EXIT]]:
-; CHECK-NEXT: [[IV_LCSSA:%.*]] = phi i64 [ [[IV]], %[[LOOP_BODY_1]] ], [ [[IV]], %[[LOOP_BODY_0]] ], [ [[IV]], %[[LOOP_HEADER]] ]
+; CHECK: [[EARLY_EXIT1]]:
+; CHECK-NEXT: [[IV_LCSSA:%.*]] = phi i64 [ [[TMP51]], %[[VECTOR_EARLY_EXIT_0]] ], [ [[TMP50]], %[[VECTOR_EARLY_EXIT_1]] ], [ [[TMP49]], %[[VECTOR_EARLY_EXIT_2]] ]
; CHECK-NEXT: ret i64 [[IV_LCSSA]]
;
entry:
More information about the llvm-branch-commits
mailing list