[llvm-branch-commits] [llvm] [VPlan] Allow tail folding early exits with side effects (PR #238045)

Luke Lau via llvm-branch-commits llvm-branch-commits at lists.llvm.org
Mon Nov 16 20:32:37 PST 2026


https://github.com/lukel97 updated https://github.com/llvm/llvm-project/pull/238045

>From 5f0dc0ff2d527a40bccc812d7f365be8a8a9f372 Mon Sep 17 00:00:00 2001
From: Luke Lau <luke at igalia.com>
Date: Tue, 3 Nov 2026 01:15:40 +0800
Subject: [PATCH 1/2] Precommit tests

---
 .../LoopVectorize/early-exit-tail-folding.ll  | 365 ++++++++++--------
 1 file changed, 213 insertions(+), 152 deletions(-)

diff --git a/llvm/test/Transforms/LoopVectorize/early-exit-tail-folding.ll b/llvm/test/Transforms/LoopVectorize/early-exit-tail-folding.ll
index cff27685627005..dfad0a260c0251 100644
--- a/llvm/test/Transforms/LoopVectorize/early-exit-tail-folding.ll
+++ b/llvm/test/Transforms/LoopVectorize/early-exit-tail-folding.ll
@@ -1,5 +1,6 @@
 ; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --check-globals none --version 6
-; RUN: opt -S < %s -p loop-vectorize -force-vector-width=4 -force-target-supports-masked-memory-ops -enable-early-exit-vectorization-with-side-effects -tail-folding-policy=prefer-fold-tail | FileCheck %s
+; RUN: opt -S < %s -p loop-vectorize -force-vector-width=4 -force-target-supports-masked-memory-ops -enable-early-exit-vectorization-with-side-effects -tail-folding-policy=prefer-fold-tail | FileCheck %s --check-prefixes=CHECK,PREFER
+; RUN: opt -S < %s -p loop-vectorize -force-vector-width=4 -force-target-supports-masked-memory-ops -enable-early-exit-vectorization-with-side-effects -tail-folding-policy=must-fold-tail | FileCheck %s --check-prefixes=CHECK,MUST
 
 define i32 @single_early_exit(ptr align 4 dereferenceable(4096) %p) {
 ; CHECK-LABEL: define i32 @single_early_exit(
@@ -271,199 +272,259 @@ exit:
   ret i32 %res
 }
 
-; TODO: Cannot tail fold yet.
-define i32 @countable_early_exit(ptr align 4 dereferenceable(4096) %p, i32 %n) {
-; CHECK-LABEL: define i32 @countable_early_exit(
-; CHECK-SAME: ptr align 4 dereferenceable(4096) [[P:%.*]], i32 [[N:%.*]]) {
-; CHECK-NEXT:  [[ENTRY:.*]]:
-; CHECK-NEXT:    [[TMP0:%.*]] = call i32 @llvm.umin.i32(i32 [[N]], i32 122)
-; CHECK-NEXT:    [[TMP1:%.*]] = add nuw nsw i32 [[TMP0]], 1
-; CHECK-NEXT:    [[MIN_ITERS_CHECK:%.*]] = icmp ule i32 [[TMP1]], 4
-; CHECK-NEXT:    br i1 [[MIN_ITERS_CHECK]], label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]]
-; CHECK:       [[VECTOR_PH]]:
-; CHECK-NEXT:    [[TMP2:%.*]] = and i32 [[TMP1]], 3
-; CHECK-NEXT:    [[TMP3:%.*]] = icmp eq i32 [[TMP2]], 0
-; CHECK-NEXT:    [[TMP4:%.*]] = select i1 [[TMP3]], i32 4, i32 [[TMP2]]
-; CHECK-NEXT:    [[N_VEC:%.*]] = sub i32 [[TMP1]], [[TMP4]]
-; CHECK-NEXT:    br label %[[VECTOR_BODY:.*]]
-; CHECK:       [[VECTOR_BODY]]:
-; CHECK-NEXT:    [[INDEX:%.*]] = phi i32 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
-; CHECK-NEXT:    [[INDEX_NEXT]] = add nuw i32 [[INDEX]], 4
-; CHECK-NEXT:    [[TMP5:%.*]] = icmp eq i32 [[INDEX_NEXT]], [[N_VEC]]
-; CHECK-NEXT:    br i1 [[TMP5]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP6:![0-9]+]]
-; CHECK:       [[MIDDLE_BLOCK]]:
-; CHECK-NEXT:    br label %[[SCALAR_PH]]
-; CHECK:       [[SCALAR_PH]]:
-; CHECK-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i32 [ [[N_VEC]], %[[MIDDLE_BLOCK]] ], [ 0, %[[ENTRY]] ]
-; CHECK-NEXT:    br label %[[LOOP:.*]]
-; CHECK:       [[LOOP]]:
-; CHECK-NEXT:    [[IV:%.*]] = phi i32 [ [[BC_RESUME_VAL]], %[[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], %[[LATCH:.*]] ]
-; CHECK-NEXT:    [[C:%.*]] = icmp eq i32 [[IV]], [[N]]
-; CHECK-NEXT:    br i1 [[C]], label %[[EXIT:.*]], label %[[LATCH]]
-; CHECK:       [[LATCH]]:
-; CHECK-NEXT:    [[GEP:%.*]] = getelementptr inbounds i32, ptr [[P]], i32 [[IV]]
-; CHECK-NEXT:    [[LD:%.*]] = load i32, ptr [[GEP]], align 4
-; CHECK-NEXT:    [[IV_NEXT]] = add nuw nsw i32 [[IV]], 1
-; CHECK-NEXT:    [[EC:%.*]] = icmp eq i32 [[IV_NEXT]], 123
-; CHECK-NEXT:    br i1 [[EC]], label %[[EXIT]], label %[[LOOP]], !llvm.loop [[LOOP7:![0-9]+]]
-; CHECK:       [[EXIT]]:
-; CHECK-NEXT:    [[RES:%.*]] = phi i32 [ 0, %[[LOOP]] ], [ [[LD]], %[[LATCH]] ]
-; CHECK-NEXT:    ret i32 [[RES]]
+define void @early_exit_with_store_after_exit(ptr align 4 dereferenceable(4096) noalias %dst, ptr align 4 dereferenceable(4096) %pred) {
+; PREFER-LABEL: define void @early_exit_with_store_after_exit(
+; PREFER-SAME: ptr noalias align 4 dereferenceable(4096) [[DST:%.*]], ptr align 4 dereferenceable(4096) [[PRED:%.*]]) {
+; PREFER-NEXT:  [[ENTRY:.*:]]
+; PREFER-NEXT:    br label %[[VECTOR_PH:.*]]
+; PREFER:       [[VECTOR_PH]]:
+; PREFER-NEXT:    br label %[[VECTOR_BODY:.*]]
+; PREFER:       [[VECTOR_BODY]]:
+; PREFER-NEXT:    [[INDEX:%.*]] = phi i32 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY_INTERIM:.*]] ]
+; PREFER-NEXT:    [[TMP0:%.*]] = getelementptr inbounds nuw i32, ptr [[PRED]], i32 [[INDEX]]
+; PREFER-NEXT:    [[WIDE_LOAD:%.*]] = load <4 x i32>, ptr [[TMP0]], align 4
+; PREFER-NEXT:    [[TMP1:%.*]] = icmp sgt <4 x i32> [[WIDE_LOAD]], splat (i32 500)
+; PREFER-NEXT:    [[TMP2:%.*]] = freeze <4 x i1> [[TMP1]]
+; PREFER-NEXT:    [[TMP3:%.*]] = call <4 x i1> @llvm.mask.beforefirst.v4i1(<4 x i1> [[TMP2]])
+; PREFER-NEXT:    [[TMP4:%.*]] = getelementptr i32, ptr [[DST]], i32 [[INDEX]]
+; PREFER-NEXT:    [[WIDE_MASKED_LOAD:%.*]] = call <4 x i32> @llvm.masked.load.v4i32.p0(ptr align 4 [[TMP4]], <4 x i1> [[TMP3]], <4 x i32> poison)
+; PREFER-NEXT:    [[TMP5:%.*]] = add nsw <4 x i32> [[WIDE_MASKED_LOAD]], splat (i32 1)
+; PREFER-NEXT:    call void @llvm.masked.store.v4i32.p0(<4 x i32> [[TMP5]], ptr align 4 [[TMP4]], <4 x i1> [[TMP3]])
+; PREFER-NEXT:    [[TMP6:%.*]] = call i1 @llvm.vector.reduce.or.v4i1(<4 x i1> [[TMP2]])
+; PREFER-NEXT:    [[INDEX_NEXT]] = add nuw i32 [[INDEX]], 4
+; PREFER-NEXT:    [[TMP7:%.*]] = icmp eq i32 [[INDEX_NEXT]], 120
+; PREFER-NEXT:    br i1 [[TMP6]], label %[[VECTOR_EARLY_EXIT:.*]], label %[[VECTOR_BODY_INTERIM]]
+; PREFER:       [[VECTOR_BODY_INTERIM]]:
+; PREFER-NEXT:    br i1 [[TMP7]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP6:![0-9]+]]
+; PREFER:       [[MIDDLE_BLOCK]]:
+; PREFER-NEXT:    br label %[[SCALAR_PH:.*]]
+; PREFER:       [[VECTOR_EARLY_EXIT]]:
+; PREFER-NEXT:    br label %[[EXIT:.*]]
+; PREFER:       [[SCALAR_PH]]:
+; PREFER-NEXT:    br label %[[LOOP:.*]]
+; PREFER:       [[LOOP]]:
+; PREFER-NEXT:    [[IV:%.*]] = phi i32 [ 120, %[[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], %[[LATCH:.*]] ]
+; PREFER-NEXT:    [[EE_ADDR:%.*]] = getelementptr inbounds nuw i32, ptr [[PRED]], i32 [[IV]]
+; PREFER-NEXT:    [[EE_VAL:%.*]] = load i32, ptr [[EE_ADDR]], align 4
+; PREFER-NEXT:    [[EE_COND:%.*]] = icmp sgt i32 [[EE_VAL]], 500
+; PREFER-NEXT:    br i1 [[EE_COND]], label %[[EXIT]], label %[[LATCH]]
+; PREFER:       [[LATCH]]:
+; PREFER-NEXT:    [[ST_ADDR:%.*]] = getelementptr inbounds nuw i32, ptr [[DST]], i32 [[IV]]
+; PREFER-NEXT:    [[DATA:%.*]] = load i32, ptr [[ST_ADDR]], align 4
+; PREFER-NEXT:    [[INC:%.*]] = add nsw i32 [[DATA]], 1
+; PREFER-NEXT:    store i32 [[INC]], ptr [[ST_ADDR]], align 4
+; PREFER-NEXT:    [[IV_NEXT]] = add nuw nsw i32 [[IV]], 1
+; PREFER-NEXT:    [[EC:%.*]] = icmp eq i32 [[IV_NEXT]], 123
+; PREFER-NEXT:    br i1 [[EC]], label %[[EXIT]], label %[[LOOP]], !llvm.loop [[LOOP7:![0-9]+]]
+; PREFER:       [[EXIT]]:
+; PREFER-NEXT:    ret void
+;
+; MUST-LABEL: define void @early_exit_with_store_after_exit(
+; MUST-SAME: ptr noalias align 4 dereferenceable(4096) [[DST:%.*]], ptr align 4 dereferenceable(4096) [[PRED:%.*]]) {
+; MUST-NEXT:  [[ENTRY:.*]]:
+; MUST-NEXT:    br label %[[LOOP:.*]]
+; MUST:       [[LOOP]]:
+; MUST-NEXT:    [[IV:%.*]] = phi i32 [ 0, %[[ENTRY]] ], [ [[IV_NEXT:%.*]], %[[LATCH:.*]] ]
+; MUST-NEXT:    [[EE_ADDR:%.*]] = getelementptr inbounds nuw i32, ptr [[PRED]], i32 [[IV]]
+; MUST-NEXT:    [[EE_VAL:%.*]] = load i32, ptr [[EE_ADDR]], align 4
+; MUST-NEXT:    [[EE_COND:%.*]] = icmp sgt i32 [[EE_VAL]], 500
+; MUST-NEXT:    br i1 [[EE_COND]], label %[[EXIT:.*]], label %[[LATCH]]
+; MUST:       [[LATCH]]:
+; MUST-NEXT:    [[ST_ADDR:%.*]] = getelementptr inbounds nuw i32, ptr [[DST]], i32 [[IV]]
+; MUST-NEXT:    [[DATA:%.*]] = load i32, ptr [[ST_ADDR]], align 4
+; MUST-NEXT:    [[INC:%.*]] = add nsw i32 [[DATA]], 1
+; MUST-NEXT:    store i32 [[INC]], ptr [[ST_ADDR]], align 4
+; MUST-NEXT:    [[IV_NEXT]] = add nuw nsw i32 [[IV]], 1
+; MUST-NEXT:    [[EC:%.*]] = icmp eq i32 [[IV_NEXT]], 123
+; MUST-NEXT:    br i1 [[EC]], label %[[EXIT]], label %[[LOOP]]
+; MUST:       [[EXIT]]:
+; MUST-NEXT:    ret void
 ;
 entry:
   br label %loop
 
 loop:
   %iv = phi i32 [ 0, %entry ], [ %iv.next, %latch ]
-  %c = icmp eq i32 %iv, %n
-  br i1 %c, label %exit, label %latch
+  %ee.addr = getelementptr inbounds nuw i32, ptr %pred, i32 %iv
+  %ee.val = load i32, ptr %ee.addr
+  %ee.cond = icmp sgt i32 %ee.val, 500
+  br i1 %ee.cond, label %exit, label %latch
 
 latch:
-  %gep = getelementptr inbounds i32, ptr %p, i32 %iv
-  %ld = load i32, ptr %gep, align 4
+  %st.addr = getelementptr inbounds nuw i32, ptr %dst, i32 %iv
+  %data = load i32, ptr %st.addr
+  %inc = add nsw i32 %data, 1
+  store i32 %inc, ptr %st.addr
   %iv.next = add nuw nsw i32 %iv, 1
   %ec = icmp eq i32 %iv.next, 123
   br i1 %ec, label %exit, label %loop
 
 exit:
-  %res = phi i32 [ 0, %loop ], [ %ld, %latch ]
-  ret i32 %res
+  ret void
 }
 
 ; TODO: Cannot tail fold yet.
-define void @early_exit_with_store(ptr align 4 dereferenceable(4096) noalias %dst, ptr align 4 dereferenceable(4096) %pred) {
-; CHECK-LABEL: define void @early_exit_with_store(
-; CHECK-SAME: ptr noalias align 4 dereferenceable(4096) [[DST:%.*]], ptr align 4 dereferenceable(4096) [[PRED:%.*]]) {
-; CHECK-NEXT:  [[ENTRY:.*:]]
-; CHECK-NEXT:    br label %[[VECTOR_PH:.*]]
-; CHECK:       [[VECTOR_PH]]:
-; CHECK-NEXT:    br label %[[VECTOR_BODY:.*]]
-; CHECK:       [[VECTOR_BODY]]:
-; CHECK-NEXT:    [[INDEX:%.*]] = phi i32 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
-; CHECK-NEXT:    [[TMP0:%.*]] = getelementptr inbounds nuw i32, ptr [[PRED]], i32 [[INDEX]]
-; CHECK-NEXT:    [[WIDE_LOAD:%.*]] = load <4 x i32>, ptr [[TMP0]], align 4
-; CHECK-NEXT:    [[TMP1:%.*]] = icmp sgt <4 x i32> [[WIDE_LOAD]], splat (i32 500)
-; CHECK-NEXT:    [[TMP2:%.*]] = freeze <4 x i1> [[TMP1]]
-; CHECK-NEXT:    [[UNCOUNTABLE_EXIT_MASK:%.*]] = call <4 x i1> @llvm.mask.beforefirst.v4i1(<4 x i1> [[TMP2]])
-; CHECK-NEXT:    [[TMP5:%.*]] = getelementptr i32, ptr [[DST]], i32 [[INDEX]]
-; CHECK-NEXT:    [[WIDE_MASKED_LOAD:%.*]] = call <4 x i32> @llvm.masked.load.v4i32.p0(ptr align 4 [[TMP5]], <4 x i1> [[UNCOUNTABLE_EXIT_MASK]], <4 x i32> poison)
-; CHECK-NEXT:    [[TMP6:%.*]] = add nsw <4 x i32> [[WIDE_MASKED_LOAD]], splat (i32 1)
-; CHECK-NEXT:    call void @llvm.masked.store.v4i32.p0(<4 x i32> [[TMP6]], ptr align 4 [[TMP5]], <4 x i1> [[UNCOUNTABLE_EXIT_MASK]])
-; CHECK-NEXT:    [[TMP7:%.*]] = call i1 @llvm.vector.reduce.or.v4i1(<4 x i1> [[TMP2]])
-; CHECK-NEXT:    [[INDEX_NEXT]] = add nuw i32 [[INDEX]], 4
-; CHECK-NEXT:    [[TMP8:%.*]] = icmp eq i32 [[INDEX_NEXT]], 120
-; CHECK-NEXT:    [[TMP9:%.*]] = or i1 [[TMP7]], [[TMP8]]
-; CHECK-NEXT:    br i1 [[TMP9]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP8:![0-9]+]]
-; CHECK:       [[MIDDLE_BLOCK]]:
-; CHECK-NEXT:    [[TMP12:%.*]] = call i64 @llvm.experimental.cttz.elts.i64.v4i1(<4 x i1> [[TMP2]], i1 false)
-; CHECK-NEXT:    [[TMP4:%.*]] = trunc i64 [[TMP12]] to i32
-; CHECK-NEXT:    [[TMP10:%.*]] = add i32 [[INDEX]], [[TMP4]]
-; CHECK-NEXT:    [[TMP11:%.*]] = icmp eq i32 [[TMP10]], 123
-; CHECK-NEXT:    br i1 [[TMP11]], label %[[EXIT:.*]], label %[[SCALAR_PH:.*]]
-; CHECK:       [[SCALAR_PH]]:
-; CHECK-NEXT:    br label %[[LOOP:.*]]
-; CHECK:       [[LOOP]]:
-; CHECK-NEXT:    [[IV:%.*]] = phi i32 [ [[TMP10]], %[[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], %[[LATCH:.*]] ]
-; CHECK-NEXT:    [[ST_ADDR:%.*]] = getelementptr inbounds nuw i32, ptr [[DST]], i32 [[IV]]
-; CHECK-NEXT:    [[DATA:%.*]] = load i32, ptr [[ST_ADDR]], align 4
-; CHECK-NEXT:    [[INC:%.*]] = add nsw i32 [[DATA]], 1
-; CHECK-NEXT:    store i32 [[INC]], ptr [[ST_ADDR]], align 4
-; CHECK-NEXT:    [[EE_ADDR:%.*]] = getelementptr inbounds nuw i32, ptr [[PRED]], i32 [[IV]]
-; CHECK-NEXT:    [[EE_VAL:%.*]] = load i32, ptr [[EE_ADDR]], align 4
-; CHECK-NEXT:    [[EE_COND:%.*]] = icmp sgt i32 [[EE_VAL]], 500
-; CHECK-NEXT:    br i1 [[EE_COND]], label %[[EXIT]], label %[[LATCH]]
-; CHECK:       [[LATCH]]:
-; CHECK-NEXT:    [[IV_NEXT]] = add nuw nsw i32 [[IV]], 1
-; CHECK-NEXT:    [[EC:%.*]] = icmp eq i32 [[IV_NEXT]], 123
-; CHECK-NEXT:    br i1 [[EC]], label %[[EXIT]], label %[[LOOP]], !llvm.loop [[LOOP9:![0-9]+]]
-; CHECK:       [[EXIT]]:
-; CHECK-NEXT:    ret void
+define i32 @countable_early_exit(ptr align 4 dereferenceable(4096) %p, i32 %n) {
+; PREFER-LABEL: define i32 @countable_early_exit(
+; PREFER-SAME: ptr align 4 dereferenceable(4096) [[P:%.*]], i32 [[N:%.*]]) {
+; PREFER-NEXT:  [[ENTRY:.*]]:
+; PREFER-NEXT:    [[TMP0:%.*]] = call i32 @llvm.umin.i32(i32 [[N]], i32 122)
+; PREFER-NEXT:    [[TMP1:%.*]] = add nuw nsw i32 [[TMP0]], 1
+; PREFER-NEXT:    [[MIN_ITERS_CHECK:%.*]] = icmp ule i32 [[TMP1]], 4
+; PREFER-NEXT:    br i1 [[MIN_ITERS_CHECK]], label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]]
+; PREFER:       [[VECTOR_PH]]:
+; PREFER-NEXT:    [[TMP2:%.*]] = and i32 [[TMP1]], 3
+; PREFER-NEXT:    [[TMP3:%.*]] = icmp eq i32 [[TMP2]], 0
+; PREFER-NEXT:    [[TMP4:%.*]] = select i1 [[TMP3]], i32 4, i32 [[TMP2]]
+; PREFER-NEXT:    [[N_VEC:%.*]] = sub i32 [[TMP1]], [[TMP4]]
+; PREFER-NEXT:    br label %[[VECTOR_BODY:.*]]
+; PREFER:       [[VECTOR_BODY]]:
+; PREFER-NEXT:    [[INDEX:%.*]] = phi i32 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; PREFER-NEXT:    [[INDEX_NEXT]] = add nuw i32 [[INDEX]], 4
+; PREFER-NEXT:    [[TMP5:%.*]] = icmp eq i32 [[INDEX_NEXT]], [[N_VEC]]
+; PREFER-NEXT:    br i1 [[TMP5]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP8:![0-9]+]]
+; PREFER:       [[MIDDLE_BLOCK]]:
+; PREFER-NEXT:    br label %[[SCALAR_PH]]
+; PREFER:       [[SCALAR_PH]]:
+; PREFER-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i32 [ [[N_VEC]], %[[MIDDLE_BLOCK]] ], [ 0, %[[ENTRY]] ]
+; PREFER-NEXT:    br label %[[LOOP:.*]]
+; PREFER:       [[LOOP]]:
+; PREFER-NEXT:    [[IV:%.*]] = phi i32 [ [[BC_RESUME_VAL]], %[[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], %[[LATCH:.*]] ]
+; PREFER-NEXT:    [[C:%.*]] = icmp eq i32 [[IV]], [[N]]
+; PREFER-NEXT:    br i1 [[C]], label %[[EXIT:.*]], label %[[LATCH]]
+; PREFER:       [[LATCH]]:
+; PREFER-NEXT:    [[GEP:%.*]] = getelementptr inbounds i32, ptr [[P]], i32 [[IV]]
+; PREFER-NEXT:    [[LD:%.*]] = load i32, ptr [[GEP]], align 4
+; PREFER-NEXT:    [[IV_NEXT]] = add nuw nsw i32 [[IV]], 1
+; PREFER-NEXT:    [[EC:%.*]] = icmp eq i32 [[IV_NEXT]], 123
+; PREFER-NEXT:    br i1 [[EC]], label %[[EXIT]], label %[[LOOP]], !llvm.loop [[LOOP9:![0-9]+]]
+; PREFER:       [[EXIT]]:
+; PREFER-NEXT:    [[RES:%.*]] = phi i32 [ 0, %[[LOOP]] ], [ [[LD]], %[[LATCH]] ]
+; PREFER-NEXT:    ret i32 [[RES]]
+;
+; MUST-LABEL: define i32 @countable_early_exit(
+; MUST-SAME: ptr align 4 dereferenceable(4096) [[P:%.*]], i32 [[N:%.*]]) {
+; MUST-NEXT:  [[ENTRY:.*]]:
+; MUST-NEXT:    br label %[[LOOP:.*]]
+; MUST:       [[LOOP]]:
+; MUST-NEXT:    [[IV:%.*]] = phi i32 [ 0, %[[ENTRY]] ], [ [[IV_NEXT:%.*]], %[[LATCH:.*]] ]
+; MUST-NEXT:    [[C:%.*]] = icmp eq i32 [[IV]], [[N]]
+; MUST-NEXT:    br i1 [[C]], label %[[EXIT:.*]], label %[[LATCH]]
+; MUST:       [[LATCH]]:
+; MUST-NEXT:    [[GEP:%.*]] = getelementptr inbounds i32, ptr [[P]], i32 [[IV]]
+; MUST-NEXT:    [[LD:%.*]] = load i32, ptr [[GEP]], align 4
+; MUST-NEXT:    [[IV_NEXT]] = add nuw nsw i32 [[IV]], 1
+; MUST-NEXT:    [[EC:%.*]] = icmp eq i32 [[IV_NEXT]], 123
+; MUST-NEXT:    br i1 [[EC]], label %[[EXIT]], label %[[LOOP]]
+; MUST:       [[EXIT]]:
+; MUST-NEXT:    [[RES:%.*]] = phi i32 [ 0, %[[LOOP]] ], [ [[LD]], %[[LATCH]] ]
+; MUST-NEXT:    ret i32 [[RES]]
 ;
 entry:
   br label %loop
 
 loop:
   %iv = phi i32 [ 0, %entry ], [ %iv.next, %latch ]
-  %st.addr = getelementptr inbounds nuw i32, ptr %dst, i32 %iv
-  %data = load i32, ptr %st.addr, align 4
-  %inc = add nsw i32 %data, 1
-  store i32 %inc, ptr %st.addr, align 4
-  %ee.addr = getelementptr inbounds nuw i32, ptr %pred, i32 %iv
-  %ee.val = load i32, ptr %ee.addr, align 4
-  %ee.cond = icmp sgt i32 %ee.val, 500
-  br i1 %ee.cond, label %exit, label %latch
+  %c = icmp eq i32 %iv, %n
+  br i1 %c, label %exit, label %latch
 
 latch:
+  %gep = getelementptr inbounds i32, ptr %p, i32 %iv
+  %ld = load i32, ptr %gep, align 4
   %iv.next = add nuw nsw i32 %iv, 1
   %ec = icmp eq i32 %iv.next, 123
   br i1 %ec, label %exit, label %loop
 
 exit:
-  ret void
+  %res = phi i32 [ 0, %loop ], [ %ld, %latch ]
+  ret i32 %res
 }
 
-define void @early_exit_with_store_after_exit(ptr align 4 dereferenceable(4096) noalias %dst, ptr align 4 dereferenceable(4096) %pred) {
-; CHECK-LABEL: define void @early_exit_with_store_after_exit(
-; CHECK-SAME: ptr noalias align 4 dereferenceable(4096) [[DST:%.*]], ptr align 4 dereferenceable(4096) [[PRED:%.*]]) {
-; CHECK-NEXT:  [[ENTRY:.*:]]
-; CHECK-NEXT:    br label %[[VECTOR_PH:.*]]
-; CHECK:       [[VECTOR_PH]]:
-; CHECK-NEXT:    br label %[[VECTOR_BODY:.*]]
-; CHECK:       [[VECTOR_BODY]]:
-; CHECK-NEXT:    [[INDEX:%.*]] = phi i32 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY_INTERIM:.*]] ]
-; CHECK-NEXT:    [[TMP0:%.*]] = getelementptr inbounds nuw i32, ptr [[PRED]], i32 [[INDEX]]
-; CHECK-NEXT:    [[WIDE_LOAD:%.*]] = load <4 x i32>, ptr [[TMP0]], align 4
-; CHECK-NEXT:    [[TMP1:%.*]] = icmp sgt <4 x i32> [[WIDE_LOAD]], splat (i32 500)
-; CHECK-NEXT:    [[TMP2:%.*]] = freeze <4 x i1> [[TMP1]]
-; CHECK-NEXT:    [[TMP3:%.*]] = call <4 x i1> @llvm.mask.beforefirst.v4i1(<4 x i1> [[TMP2]])
-; CHECK-NEXT:    [[TMP4:%.*]] = getelementptr i32, ptr [[DST]], i32 [[INDEX]]
-; CHECK-NEXT:    [[WIDE_MASKED_LOAD:%.*]] = call <4 x i32> @llvm.masked.load.v4i32.p0(ptr align 4 [[TMP4]], <4 x i1> [[TMP3]], <4 x i32> poison)
-; CHECK-NEXT:    [[TMP5:%.*]] = add nsw <4 x i32> [[WIDE_MASKED_LOAD]], splat (i32 1)
-; CHECK-NEXT:    call void @llvm.masked.store.v4i32.p0(<4 x i32> [[TMP5]], ptr align 4 [[TMP4]], <4 x i1> [[TMP3]])
-; CHECK-NEXT:    [[TMP6:%.*]] = call i1 @llvm.vector.reduce.or.v4i1(<4 x i1> [[TMP2]])
-; CHECK-NEXT:    [[INDEX_NEXT]] = add nuw i32 [[INDEX]], 4
-; CHECK-NEXT:    [[TMP7:%.*]] = icmp eq i32 [[INDEX_NEXT]], 120
-; CHECK-NEXT:    br i1 [[TMP6]], label %[[VECTOR_EARLY_EXIT:.*]], label %[[VECTOR_BODY_INTERIM]]
-; CHECK:       [[VECTOR_BODY_INTERIM]]:
-; CHECK-NEXT:    br i1 [[TMP7]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP10:![0-9]+]]
-; CHECK:       [[MIDDLE_BLOCK]]:
-; CHECK-NEXT:    br label %[[SCALAR_PH:.*]]
-; CHECK:       [[VECTOR_EARLY_EXIT]]:
-; CHECK-NEXT:    br label %[[EXIT:.*]]
-; CHECK:       [[SCALAR_PH]]:
-; CHECK-NEXT:    br label %[[LOOP:.*]]
-; CHECK:       [[LOOP]]:
-; CHECK-NEXT:    [[IV:%.*]] = phi i32 [ 120, %[[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], %[[LATCH:.*]] ]
-; CHECK-NEXT:    [[EE_ADDR:%.*]] = getelementptr inbounds nuw i32, ptr [[PRED]], i32 [[IV]]
-; CHECK-NEXT:    [[EE_VAL:%.*]] = load i32, ptr [[EE_ADDR]], align 4
-; CHECK-NEXT:    [[EE_COND:%.*]] = icmp sgt i32 [[EE_VAL]], 500
-; CHECK-NEXT:    br i1 [[EE_COND]], label %[[EXIT]], label %[[LATCH]]
-; CHECK:       [[LATCH]]:
-; CHECK-NEXT:    [[ST_ADDR:%.*]] = getelementptr inbounds nuw i32, ptr [[DST]], i32 [[IV]]
-; CHECK-NEXT:    [[DATA:%.*]] = load i32, ptr [[ST_ADDR]], align 4
-; CHECK-NEXT:    [[INC:%.*]] = add nsw i32 [[DATA]], 1
-; CHECK-NEXT:    store i32 [[INC]], ptr [[ST_ADDR]], align 4
-; CHECK-NEXT:    [[IV_NEXT]] = add nuw nsw i32 [[IV]], 1
-; CHECK-NEXT:    [[EC:%.*]] = icmp eq i32 [[IV_NEXT]], 123
-; CHECK-NEXT:    br i1 [[EC]], label %[[EXIT]], label %[[LOOP]], !llvm.loop [[LOOP11:![0-9]+]]
-; CHECK:       [[EXIT]]:
-; CHECK-NEXT:    ret void
+; TODO: Cannot tail fold yet.
+define void @early_exit_with_store(ptr align 4 dereferenceable(4096) noalias %dst, ptr align 4 dereferenceable(4096) %pred) {
+; PREFER-LABEL: define void @early_exit_with_store(
+; PREFER-SAME: ptr noalias align 4 dereferenceable(4096) [[DST:%.*]], ptr align 4 dereferenceable(4096) [[PRED:%.*]]) {
+; PREFER-NEXT:  [[ENTRY:.*:]]
+; PREFER-NEXT:    br label %[[VECTOR_PH:.*]]
+; PREFER:       [[VECTOR_PH]]:
+; PREFER-NEXT:    br label %[[VECTOR_BODY:.*]]
+; PREFER:       [[VECTOR_BODY]]:
+; PREFER-NEXT:    [[INDEX:%.*]] = phi i32 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; PREFER-NEXT:    [[TMP0:%.*]] = getelementptr inbounds nuw i32, ptr [[PRED]], i32 [[INDEX]]
+; PREFER-NEXT:    [[WIDE_LOAD:%.*]] = load <4 x i32>, ptr [[TMP0]], align 4
+; PREFER-NEXT:    [[TMP1:%.*]] = icmp sgt <4 x i32> [[WIDE_LOAD]], splat (i32 500)
+; PREFER-NEXT:    [[TMP2:%.*]] = freeze <4 x i1> [[TMP1]]
+; PREFER-NEXT:    [[TMP3:%.*]] = call <4 x i1> @llvm.mask.beforefirst.v4i1(<4 x i1> [[TMP2]])
+; PREFER-NEXT:    [[TMP4:%.*]] = getelementptr i32, ptr [[DST]], i32 [[INDEX]]
+; PREFER-NEXT:    [[WIDE_MASKED_LOAD:%.*]] = call <4 x i32> @llvm.masked.load.v4i32.p0(ptr align 4 [[TMP4]], <4 x i1> [[TMP3]], <4 x i32> poison)
+; PREFER-NEXT:    [[TMP5:%.*]] = add nsw <4 x i32> [[WIDE_MASKED_LOAD]], splat (i32 1)
+; PREFER-NEXT:    call void @llvm.masked.store.v4i32.p0(<4 x i32> [[TMP5]], ptr align 4 [[TMP4]], <4 x i1> [[TMP3]])
+; PREFER-NEXT:    [[TMP6:%.*]] = call i1 @llvm.vector.reduce.or.v4i1(<4 x i1> [[TMP2]])
+; PREFER-NEXT:    [[INDEX_NEXT]] = add nuw i32 [[INDEX]], 4
+; PREFER-NEXT:    [[TMP7:%.*]] = icmp eq i32 [[INDEX_NEXT]], 120
+; PREFER-NEXT:    [[TMP8:%.*]] = or i1 [[TMP6]], [[TMP7]]
+; PREFER-NEXT:    br i1 [[TMP8]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP10:![0-9]+]]
+; PREFER:       [[MIDDLE_BLOCK]]:
+; PREFER-NEXT:    [[TMP9:%.*]] = call i64 @llvm.experimental.cttz.elts.i64.v4i1(<4 x i1> [[TMP2]], i1 false)
+; PREFER-NEXT:    [[TMP10:%.*]] = trunc i64 [[TMP9]] to i32
+; PREFER-NEXT:    [[TMP11:%.*]] = add i32 [[INDEX]], [[TMP10]]
+; PREFER-NEXT:    [[TMP12:%.*]] = icmp eq i32 [[TMP11]], 123
+; PREFER-NEXT:    br i1 [[TMP12]], label %[[EXIT:.*]], label %[[SCALAR_PH:.*]]
+; PREFER:       [[SCALAR_PH]]:
+; PREFER-NEXT:    br label %[[LOOP:.*]]
+; PREFER:       [[LOOP]]:
+; PREFER-NEXT:    [[IV:%.*]] = phi i32 [ [[TMP11]], %[[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], %[[LATCH:.*]] ]
+; PREFER-NEXT:    [[ST_ADDR:%.*]] = getelementptr inbounds nuw i32, ptr [[DST]], i32 [[IV]]
+; PREFER-NEXT:    [[DATA:%.*]] = load i32, ptr [[ST_ADDR]], align 4
+; PREFER-NEXT:    [[INC:%.*]] = add nsw i32 [[DATA]], 1
+; PREFER-NEXT:    store i32 [[INC]], ptr [[ST_ADDR]], align 4
+; PREFER-NEXT:    [[EE_ADDR:%.*]] = getelementptr inbounds nuw i32, ptr [[PRED]], i32 [[IV]]
+; PREFER-NEXT:    [[EE_VAL:%.*]] = load i32, ptr [[EE_ADDR]], align 4
+; PREFER-NEXT:    [[EE_COND:%.*]] = icmp sgt i32 [[EE_VAL]], 500
+; PREFER-NEXT:    br i1 [[EE_COND]], label %[[EXIT]], label %[[LATCH]]
+; PREFER:       [[LATCH]]:
+; PREFER-NEXT:    [[IV_NEXT]] = add nuw nsw i32 [[IV]], 1
+; PREFER-NEXT:    [[EC:%.*]] = icmp eq i32 [[IV_NEXT]], 123
+; PREFER-NEXT:    br i1 [[EC]], label %[[EXIT]], label %[[LOOP]], !llvm.loop [[LOOP11:![0-9]+]]
+; PREFER:       [[EXIT]]:
+; PREFER-NEXT:    ret void
+;
+; MUST-LABEL: define void @early_exit_with_store(
+; MUST-SAME: ptr noalias align 4 dereferenceable(4096) [[DST:%.*]], ptr align 4 dereferenceable(4096) [[PRED:%.*]]) {
+; MUST-NEXT:  [[ENTRY:.*]]:
+; MUST-NEXT:    br label %[[LOOP:.*]]
+; MUST:       [[LOOP]]:
+; MUST-NEXT:    [[IV:%.*]] = phi i32 [ 0, %[[ENTRY]] ], [ [[IV_NEXT:%.*]], %[[LATCH:.*]] ]
+; MUST-NEXT:    [[ST_ADDR:%.*]] = getelementptr inbounds nuw i32, ptr [[DST]], i32 [[IV]]
+; MUST-NEXT:    [[DATA:%.*]] = load i32, ptr [[ST_ADDR]], align 4
+; MUST-NEXT:    [[INC:%.*]] = add nsw i32 [[DATA]], 1
+; MUST-NEXT:    store i32 [[INC]], ptr [[ST_ADDR]], align 4
+; MUST-NEXT:    [[EE_ADDR:%.*]] = getelementptr inbounds nuw i32, ptr [[PRED]], i32 [[IV]]
+; MUST-NEXT:    [[EE_VAL:%.*]] = load i32, ptr [[EE_ADDR]], align 4
+; MUST-NEXT:    [[EE_COND:%.*]] = icmp sgt i32 [[EE_VAL]], 500
+; MUST-NEXT:    br i1 [[EE_COND]], label %[[EXIT:.*]], label %[[LATCH]]
+; MUST:       [[LATCH]]:
+; MUST-NEXT:    [[IV_NEXT]] = add nuw nsw i32 [[IV]], 1
+; MUST-NEXT:    [[EC:%.*]] = icmp eq i32 [[IV_NEXT]], 123
+; MUST-NEXT:    br i1 [[EC]], label %[[EXIT]], label %[[LOOP]]
+; MUST:       [[EXIT]]:
+; MUST-NEXT:    ret void
 ;
 entry:
   br label %loop
 
 loop:
   %iv = phi i32 [ 0, %entry ], [ %iv.next, %latch ]
+  %st.addr = getelementptr inbounds nuw i32, ptr %dst, i32 %iv
+  %data = load i32, ptr %st.addr, align 4
+  %inc = add nsw i32 %data, 1
+  store i32 %inc, ptr %st.addr, align 4
   %ee.addr = getelementptr inbounds nuw i32, ptr %pred, i32 %iv
-  %ee.val = load i32, ptr %ee.addr
+  %ee.val = load i32, ptr %ee.addr, align 4
   %ee.cond = icmp sgt i32 %ee.val, 500
   br i1 %ee.cond, label %exit, label %latch
 
 latch:
-  %st.addr = getelementptr inbounds nuw i32, ptr %dst, i32 %iv
-  %data = load i32, ptr %st.addr
-  %inc = add nsw i32 %data, 1
-  store i32 %inc, ptr %st.addr
   %iv.next = add nuw nsw i32 %iv, 1
   %ec = icmp eq i32 %iv.next, 123
   br i1 %ec, label %exit, label %loop

>From 160d2a30e8989db06dcbed369c4fab8e253e8fa8 Mon Sep 17 00:00:00 2001
From: Luke Lau <luke at igalia.com>
Date: Sat, 31 Oct 2026 02:56:48 +0800
Subject: [PATCH 2/2] [VPlan] Allow tail folding early exits with side effects

Continues the work in #179595.

This allows us to tail fold e.g. the following loop:

```c
extern char a[1024], b[1024], c[1024], y[1024];
void two_guards(void) {
  for (int i = 0; i < 1000; i++) {
    if (a[i] == 0)
      break;
    if (b[i] > 3) {
      if (c[i] < 9)
        y[i] = 1;
    }
  }
}
```

Into this on RISC-V:

```asm
.LBB0_1:                                # %vector.body
                                        # =>This Inner Loop Header: Depth=1
	vsetvli	a6, a1, e8, m2, ta, ma
	add	a7, a2, a0
	vle8.v	v10, (a7)
	vmseq.vi	v12, v10, 0
	vmsbf.m	v0, v12
	add	a7, a3, a0
	vle8.v	v10, (a7), v0.t
	vmsgtu.vi	v13, v10, 3
	vmand.mm	v0, v0, v13
	add	a7, a4, a0
	vle8.v	v10, (a7), v0.t
	vmsleu.vi	v13, v10, 8
	vmand.mm	v0, v0, v13
	vcpop.m	a7, v12
	add	t0, a5, a0
	vse8.v	v8, (t0), v0.t
	bnez	a7, .LBB0_3
                                        #   in Loop: Header=BB0_1 Depth=1
	sub	a1, a1, a6
	add	a0, a0, a6
	bnez	a1, .LBB0_1
```

For early exit loops with side effects, we may be able to handle the exits by masking the vector body, or we may have to handle them in the scalar loop. If it's the latter we can't tail fold the plan just yet, we need to fixup a lot of assumptions around the middle block etc. For now just disable tail folding in the cost model. It's safe to change the tail folding here because foldTailByMasking hasn't been called yet and there hasn't been any costing done.
---
 .../Vectorize/LoopVectorizationLegality.cpp   |   7 --
 .../Transforms/Vectorize/LoopVectorize.cpp    |  24 ++++
 .../Transforms/Vectorize/VPlanTransforms.cpp  |   2 +-
 .../Transforms/Vectorize/VPlanTransforms.h    |   3 +-
 .../LoopVectorize/RISCV/early-exit.ll         |  53 +++------
 .../RISCV/early_exit_with_stores.ll           |  65 ++++-------
 .../LoopVectorize/early-exit-tail-folding.ll  | 110 +++++++-----------
 .../early_exit_store_legality.ll              |   4 +-
 8 files changed, 111 insertions(+), 157 deletions(-)

diff --git a/llvm/lib/Transforms/Vectorize/LoopVectorizationLegality.cpp b/llvm/lib/Transforms/Vectorize/LoopVectorizationLegality.cpp
index 0cd4b7ec2ffb45..1271fe6de706a8 100644
--- a/llvm/lib/Transforms/Vectorize/LoopVectorizationLegality.cpp
+++ b/llvm/lib/Transforms/Vectorize/LoopVectorizationLegality.cpp
@@ -2115,13 +2115,6 @@ bool LoopVectorizationLegality::canFoldTailByMasking() const {
     return false;
   }
 
-  // TODO: Handle early exits with side effects without a scalar tail.
-  if (hasUncountableExitWithSideEffects()) {
-    LLVM_DEBUG(dbgs() << "LV: Cannot fold tail by masking. Uncountable exit "
-                         "with side effects requires a scalar epilogue\n");
-    return false;
-  }
-
   LLVM_DEBUG(dbgs() << "LV: checking if tail can be folded by masking.\n");
 
   // No accesses are known to be safe in inactive lanes of the folded tail.
diff --git a/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp b/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
index c59fa092a4a906..acdb5d4f6de309 100644
--- a/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
+++ b/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
@@ -748,6 +748,20 @@ class LoopVectorizationCostModel {
                        : "due to non-interleaving reasons.\n"));
   }
 
+  /// Disable all tail folding styles. Returns false if an epilogue isn't
+  /// allowed. Must be called before the plan is tail folded.
+  bool tryToDisableTailFolding() {
+    assert(foldTailByMasking() && "Expected tail folding to be enabled!");
+    if (EpilogueLoweringStatus != CM_EpilogueNotNeededFoldTail)
+      return false;
+    LLVM_DEBUG(dbgs() << "LV: Disabling tail folding: vectorize with an "
+                         "epilogue instead.\n");
+    ChosenTailFoldingStyle = TailFoldingStyle::None;
+    EpilogueLoweringStatus = CM_EpilogueAllowed;
+    PartialAliasMaskingStatus = AliasMaskingStatus::Disabled;
+    return true;
+  }
+
   /// Returns true if all loop blocks should be masked to fold tail loop.
   bool foldTailByMasking() const {
     return getTailFoldingStyle() != TailFoldingStyle::None;
@@ -6283,6 +6297,16 @@ LoopVectorizationPlanner::tryToBuildVPlan1(LoopVectorizationCostModel &PlanCM) {
                         EEStyle)) {
       return nullptr;
     }
+
+    // Exiting in the scalar loop requires a scalar epilogue for now.
+    if (EEStyle == UncountableExitStyle::MaskedHandleExitInScalarLoop &&
+        PlanCM.foldTailByMasking() && !PlanCM.tryToDisableTailFolding()) {
+      reportVectorizationFailure(
+          "Cannot fold tail by masking when early exits are handled in the "
+          "scalar loop",
+          "EarlyExitInScalarLoopCannotFoldTail", ORE, OrigLoop);
+      return nullptr;
+    }
   } else {
     RUN_VPLAN_PASS(VPlanTransforms::handleCountableEarlyExits, *VPlan0);
   }
diff --git a/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp b/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp
index b09bfe4095a5a7..f538142ebf341e 100644
--- a/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp
+++ b/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp
@@ -3578,7 +3578,7 @@ static SmallVector<VPBasicBlock *> blocksWithStores(VPBasicBlock *HeaderVPBB) {
 bool VPlanTransforms::handleUncountableEarlyExits(
     VPlan &Plan, OptimizationRemarkEmitter *ORE, Loop *TheLoop,
     PredicatedScalarEvolution &PSE, DominatorTree &DT, AssumptionCache *AC,
-    UncountableExitStyle Style) {
+    UncountableExitStyle &Style) {
   VPDominatorTree VPDT(Plan);
 
   auto *MiddleVPBB = VPBlockUtils::getPlainCFGMiddleBlock(Plan);
diff --git a/llvm/lib/Transforms/Vectorize/VPlanTransforms.h b/llvm/lib/Transforms/Vectorize/VPlanTransforms.h
index 52e29e8926a0da..368f2f8c0da301 100644
--- a/llvm/lib/Transforms/Vectorize/VPlanTransforms.h
+++ b/llvm/lib/Transforms/Vectorize/VPlanTransforms.h
@@ -409,11 +409,12 @@ struct VPlanTransforms {
   /// appropriate branching logic in the latch that handles early exits and the
   /// latch exit condition. Multiple exits are handled with a dispatch block
   /// that determines which exit to take based on lane-by-lane semantics.
+  /// \p Style is the preferred style, and is updated to the style used.
   LLVM_ABI_FOR_TEST static bool
   handleUncountableEarlyExits(VPlan &Plan, OptimizationRemarkEmitter *ORE,
                               Loop *TheLoop, PredicatedScalarEvolution &PSE,
                               DominatorTree &DT, AssumptionCache *AC,
-                              UncountableExitStyle Style);
+                              UncountableExitStyle &Style);
 
   /// Disconnect countable early exits from the loop.
   LLVM_ABI_FOR_TEST static void handleCountableEarlyExits(VPlan &Plan);
diff --git a/llvm/test/Transforms/LoopVectorize/RISCV/early-exit.ll b/llvm/test/Transforms/LoopVectorize/RISCV/early-exit.ll
index f8563ff31a417f..06afcc1d4e1c8b 100644
--- a/llvm/test/Transforms/LoopVectorize/RISCV/early-exit.ll
+++ b/llvm/test/Transforms/LoopVectorize/RISCV/early-exit.ll
@@ -405,54 +405,37 @@ exit:
 define void @early_exit_with_store_after_exit(ptr align 4 dereferenceable(4096) noalias %dst, ptr align 4 dereferenceable(4096) %pred) {
 ; CHECK-LABEL: define void @early_exit_with_store_after_exit(
 ; CHECK-SAME: ptr noalias align 4 dereferenceable(4096) [[DST:%.*]], ptr align 4 dereferenceable(4096) [[PRED:%.*]]) #[[ATTR0]] {
-; CHECK-NEXT:  [[ENTRY:.*]]:
-; CHECK-NEXT:    [[TMP0:%.*]] = call i32 @llvm.vscale.i32()
-; CHECK-NEXT:    [[TMP1:%.*]] = shl nuw i32 [[TMP0]], 2
-; CHECK-NEXT:    [[MIN_ITERS_CHECK:%.*]] = icmp ult i32 123, [[TMP1]]
-; CHECK-NEXT:    br i1 [[MIN_ITERS_CHECK]], label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]]
+; CHECK-NEXT:  [[ENTRY:.*:]]
+; CHECK-NEXT:    br label %[[VECTOR_PH:.*]]
 ; CHECK:       [[VECTOR_PH]]:
-; CHECK-NEXT:    [[N_MOD_VF:%.*]] = urem i32 123, [[TMP1]]
-; CHECK-NEXT:    [[N_VEC:%.*]] = sub i32 123, [[N_MOD_VF]]
 ; CHECK-NEXT:    br label %[[VECTOR_BODY:.*]]
 ; CHECK:       [[VECTOR_BODY]]:
-; CHECK-NEXT:    [[INDEX:%.*]] = phi i32 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY_INTERIM:.*]] ]
+; CHECK-NEXT:    [[INDEX:%.*]] = phi i32 [ 0, %[[VECTOR_PH]] ], [ [[CURRENT_ITERATION_NEXT:%.*]], %[[VECTOR_BODY_INTERIM:.*]] ]
+; CHECK-NEXT:    [[AVL:%.*]] = phi i32 [ 123, %[[VECTOR_PH]] ], [ [[AVL_NEXT:%.*]], %[[VECTOR_BODY_INTERIM]] ]
+; CHECK-NEXT:    [[TMP0:%.*]] = call i32 @llvm.experimental.get.vector.length.i32(i32 [[AVL]], i32 4, i1 true)
 ; CHECK-NEXT:    [[TMP2:%.*]] = getelementptr inbounds nuw i32, ptr [[PRED]], i32 [[INDEX]]
-; CHECK-NEXT:    [[WIDE_LOAD:%.*]] = load <vscale x 4 x i32>, ptr [[TMP2]], align 4
+; CHECK-NEXT:    [[WIDE_LOAD:%.*]] = call <vscale x 4 x i32> @llvm.vp.load.nxv4i32.p0(ptr align 4 [[TMP2]], <vscale x 4 x i1> splat (i1 true), i32 [[TMP0]])
 ; CHECK-NEXT:    [[TMP3:%.*]] = icmp sgt <vscale x 4 x i32> [[WIDE_LOAD]], splat (i32 500)
-; CHECK-NEXT:    [[TMP4:%.*]] = freeze <vscale x 4 x i1> [[TMP3]]
+; CHECK-NEXT:    [[TMP10:%.*]] = call <vscale x 4 x i1> @llvm.vp.merge.nxv4i1(<vscale x 4 x i1> splat (i1 true), <vscale x 4 x i1> [[TMP3]], <vscale x 4 x i1> zeroinitializer, i32 [[TMP0]])
+; CHECK-NEXT:    [[TMP4:%.*]] = freeze <vscale x 4 x i1> [[TMP10]]
 ; CHECK-NEXT:    [[TMP5:%.*]] = call <vscale x 4 x i1> @llvm.mask.beforefirst.nxv4i1(<vscale x 4 x i1> [[TMP4]])
 ; CHECK-NEXT:    [[TMP6:%.*]] = getelementptr i32, ptr [[DST]], i32 [[INDEX]]
-; CHECK-NEXT:    [[WIDE_MASKED_LOAD:%.*]] = call <vscale x 4 x i32> @llvm.masked.load.nxv4i32.p0(ptr align 4 [[TMP6]], <vscale x 4 x i1> [[TMP5]], <vscale x 4 x i32> poison)
+; CHECK-NEXT:    [[WIDE_MASKED_LOAD:%.*]] = call <vscale x 4 x i32> @llvm.vp.load.nxv4i32.p0(ptr align 4 [[TMP6]], <vscale x 4 x i1> [[TMP5]], i32 [[TMP0]])
 ; CHECK-NEXT:    [[TMP7:%.*]] = add nsw <vscale x 4 x i32> [[WIDE_MASKED_LOAD]], splat (i32 1)
-; CHECK-NEXT:    call void @llvm.masked.store.nxv4i32.p0(<vscale x 4 x i32> [[TMP7]], ptr align 4 [[TMP6]], <vscale x 4 x i1> [[TMP5]])
-; CHECK-NEXT:    [[TMP8:%.*]] = call i1 @llvm.vector.reduce.or.nxv4i1(<vscale x 4 x i1> [[TMP4]])
-; CHECK-NEXT:    [[INDEX_NEXT]] = add nuw i32 [[INDEX]], [[TMP1]]
-; CHECK-NEXT:    [[TMP9:%.*]] = icmp eq i32 [[INDEX_NEXT]], [[N_VEC]]
+; CHECK-NEXT:    call void @llvm.vp.store.nxv4i32.p0(<vscale x 4 x i32> [[TMP7]], ptr align 4 [[TMP6]], <vscale x 4 x i1> [[TMP5]], i32 [[TMP0]])
+; CHECK-NEXT:    [[TMP12:%.*]] = freeze <vscale x 4 x i1> [[TMP3]]
+; CHECK-NEXT:    [[TMP9:%.*]] = call i1 @llvm.vp.reduce.or.nxv4i1(i1 false, <vscale x 4 x i1> [[TMP12]], <vscale x 4 x i1> splat (i1 true), i32 [[TMP0]])
+; CHECK-NEXT:    [[TMP8:%.*]] = or i1 [[TMP9]], false
+; CHECK-NEXT:    [[CURRENT_ITERATION_NEXT]] = add i32 [[TMP0]], [[INDEX]]
+; CHECK-NEXT:    [[AVL_NEXT]] = sub nuw i32 [[AVL]], [[TMP0]]
+; CHECK-NEXT:    [[TMP11:%.*]] = icmp eq i32 [[AVL_NEXT]], 0
 ; CHECK-NEXT:    br i1 [[TMP8]], label %[[VECTOR_EARLY_EXIT:.*]], label %[[VECTOR_BODY_INTERIM]]
 ; CHECK:       [[VECTOR_BODY_INTERIM]]:
-; CHECK-NEXT:    br i1 [[TMP9]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP8:![0-9]+]]
+; CHECK-NEXT:    br i1 [[TMP11]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP8:![0-9]+]]
 ; CHECK:       [[MIDDLE_BLOCK]]:
-; CHECK-NEXT:    [[CMP_N:%.*]] = icmp eq i32 123, [[N_VEC]]
-; CHECK-NEXT:    br i1 [[CMP_N]], label %[[EXIT:.*]], label %[[SCALAR_PH]]
+; CHECK-NEXT:    br label %[[EXIT:.*]]
 ; CHECK:       [[VECTOR_EARLY_EXIT]]:
 ; CHECK-NEXT:    br label %[[EXIT]]
-; CHECK:       [[SCALAR_PH]]:
-; CHECK-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i32 [ [[N_VEC]], %[[MIDDLE_BLOCK]] ], [ 0, %[[ENTRY]] ]
-; CHECK-NEXT:    br label %[[LOOP:.*]]
-; CHECK:       [[LOOP]]:
-; CHECK-NEXT:    [[IV:%.*]] = phi i32 [ [[BC_RESUME_VAL]], %[[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], %[[LATCH:.*]] ]
-; CHECK-NEXT:    [[EE_ADDR:%.*]] = getelementptr inbounds nuw i32, ptr [[PRED]], i32 [[IV]]
-; CHECK-NEXT:    [[EE_VAL:%.*]] = load i32, ptr [[EE_ADDR]], align 4
-; CHECK-NEXT:    [[EE_COND:%.*]] = icmp sgt i32 [[EE_VAL]], 500
-; CHECK-NEXT:    br i1 [[EE_COND]], label %[[EXIT]], label %[[LATCH]]
-; CHECK:       [[LATCH]]:
-; CHECK-NEXT:    [[ST_ADDR:%.*]] = getelementptr inbounds nuw i32, ptr [[DST]], i32 [[IV]]
-; CHECK-NEXT:    [[DATA:%.*]] = load i32, ptr [[ST_ADDR]], align 4
-; CHECK-NEXT:    [[INC:%.*]] = add nsw i32 [[DATA]], 1
-; CHECK-NEXT:    store i32 [[INC]], ptr [[ST_ADDR]], align 4
-; CHECK-NEXT:    [[IV_NEXT]] = add nuw nsw i32 [[IV]], 1
-; CHECK-NEXT:    [[EC:%.*]] = icmp eq i32 [[IV_NEXT]], 123
-; CHECK-NEXT:    br i1 [[EC]], label %[[EXIT]], label %[[LOOP]], !llvm.loop [[LOOP9:![0-9]+]]
 ; CHECK:       [[EXIT]]:
 ; CHECK-NEXT:    ret void
 ;
diff --git a/llvm/test/Transforms/LoopVectorize/RISCV/early_exit_with_stores.ll b/llvm/test/Transforms/LoopVectorize/RISCV/early_exit_with_stores.ll
index 8ba6d5f81679e5..a566df5db77019 100644
--- a/llvm/test/Transforms/LoopVectorize/RISCV/early_exit_with_stores.ll
+++ b/llvm/test/Transforms/LoopVectorize/RISCV/early_exit_with_stores.ll
@@ -1267,57 +1267,40 @@ exit.uncountable:
 define void @interleaved_stores_after_early_exit(ptr noalias %dst, ptr align 2 dereferenceable(40) %src) {
 ; CHECK-LABEL: define void @interleaved_stores_after_early_exit(
 ; CHECK-SAME: ptr noalias [[DST:%.*]], ptr align 2 dereferenceable(40) [[SRC:%.*]]) #[[ATTR0]] {
-; CHECK-NEXT:  [[ENTRY:.*]]:
-; CHECK-NEXT:    [[TMP0:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-NEXT:    [[TMP1:%.*]] = shl nuw i64 [[TMP0]], 3
-; CHECK-NEXT:    [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 20, [[TMP1]]
-; CHECK-NEXT:    br i1 [[MIN_ITERS_CHECK]], label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]]
+; CHECK-NEXT:  [[ENTRY:.*:]]
+; CHECK-NEXT:    br label %[[VECTOR_PH:.*]]
 ; CHECK:       [[VECTOR_PH]]:
-; CHECK-NEXT:    [[N_MOD_VF:%.*]] = urem i64 20, [[TMP1]]
-; CHECK-NEXT:    [[N_VEC:%.*]] = sub i64 20, [[N_MOD_VF]]
 ; CHECK-NEXT:    br label %[[VECTOR_BODY:.*]]
 ; CHECK:       [[VECTOR_BODY]]:
-; CHECK-NEXT:    [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[MIDDLE_BLOCK:.*]] ]
+; CHECK-NEXT:    [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[CURRENT_ITERATION_NEXT:%.*]], %[[MIDDLE_BLOCK:.*]] ]
+; CHECK-NEXT:    [[AVL:%.*]] = phi i64 [ 20, %[[VECTOR_PH]] ], [ [[AVL_NEXT:%.*]], %[[MIDDLE_BLOCK]] ]
+; CHECK-NEXT:    [[TMP0:%.*]] = call i32 @llvm.experimental.get.vector.length.i64(i64 [[AVL]], i32 8, i1 true)
 ; CHECK-NEXT:    [[TMP2:%.*]] = getelementptr i16, ptr [[SRC]], i64 [[INDEX]]
-; CHECK-NEXT:    [[WIDE_LOAD:%.*]] = load <vscale x 8 x i16>, ptr [[TMP2]], align 2
+; CHECK-NEXT:    [[WIDE_LOAD:%.*]] = call <vscale x 8 x i16> @llvm.vp.load.nxv8i16.p0(ptr align 2 [[TMP2]], <vscale x 8 x i1> splat (i1 true), i32 [[TMP0]])
 ; CHECK-NEXT:    [[TMP3:%.*]] = icmp sgt <vscale x 8 x i16> [[WIDE_LOAD]], splat (i16 500)
-; CHECK-NEXT:    [[TMP4:%.*]] = freeze <vscale x 8 x i1> [[TMP3]]
+; CHECK-NEXT:    [[TMP5:%.*]] = call <vscale x 8 x i1> @llvm.vp.merge.nxv8i1(<vscale x 8 x i1> splat (i1 true), <vscale x 8 x i1> [[TMP3]], <vscale x 8 x i1> zeroinitializer, i32 [[TMP0]])
+; CHECK-NEXT:    [[TMP4:%.*]] = freeze <vscale x 8 x i1> [[TMP5]]
 ; CHECK-NEXT:    [[UNCOUNTABLE_EXIT_MASK:%.*]] = call <vscale x 8 x i1> @llvm.mask.beforefirst.nxv8i1(<vscale x 8 x i1> [[TMP4]])
 ; CHECK-NEXT:    [[TMP6:%.*]] = shl i64 [[INDEX]], 1
 ; CHECK-NEXT:    [[TMP7:%.*]] = getelementptr i16, ptr [[DST]], i64 [[TMP6]]
-; CHECK-NEXT:    [[INTERLEAVED_VEC:%.*]] = call <vscale x 16 x i16> @llvm.vector.interleave2.nxv16i16(<vscale x 8 x i16> splat (i16 1), <vscale x 8 x i16> splat (i16 2))
+; CHECK-NEXT:    [[INTERLEAVE_EVL:%.*]] = mul nuw nsw i32 [[TMP0]], 2
 ; CHECK-NEXT:    [[INTERLEAVED_MASK:%.*]] = call <vscale x 16 x i1> @llvm.vector.interleave2.nxv16i1(<vscale x 8 x i1> [[UNCOUNTABLE_EXIT_MASK]], <vscale x 8 x i1> [[UNCOUNTABLE_EXIT_MASK]])
-; CHECK-NEXT:    call void @llvm.masked.store.nxv16i16.p0(<vscale x 16 x i16> [[INTERLEAVED_VEC]], ptr align 2 [[TMP7]], <vscale x 16 x i1> [[INTERLEAVED_MASK]])
-; CHECK-NEXT:    [[TMP8:%.*]] = call i1 @llvm.vector.reduce.or.nxv8i1(<vscale x 8 x i1> [[TMP4]])
-; CHECK-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[INDEX]], [[TMP1]]
-; CHECK-NEXT:    [[TMP9:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
+; CHECK-NEXT:    [[INTERLEAVED_VEC:%.*]] = call <vscale x 16 x i16> @llvm.vector.interleave2.nxv16i16(<vscale x 8 x i16> splat (i16 1), <vscale x 8 x i16> splat (i16 2))
+; CHECK-NEXT:    call void @llvm.vp.store.nxv16i16.p0(<vscale x 16 x i16> [[INTERLEAVED_VEC]], ptr align 2 [[TMP7]], <vscale x 16 x i1> [[INTERLEAVED_MASK]], i32 [[INTERLEAVE_EVL]])
+; CHECK-NEXT:    [[TMP10:%.*]] = freeze <vscale x 8 x i1> [[TMP3]]
+; CHECK-NEXT:    [[TMP9:%.*]] = call i1 @llvm.vp.reduce.or.nxv8i1(i1 false, <vscale x 8 x i1> [[TMP10]], <vscale x 8 x i1> splat (i1 true), i32 [[TMP0]])
+; CHECK-NEXT:    [[TMP8:%.*]] = or i1 [[TMP9]], false
+; CHECK-NEXT:    [[TMP11:%.*]] = zext i32 [[TMP0]] to i64
+; CHECK-NEXT:    [[CURRENT_ITERATION_NEXT]] = add i64 [[TMP11]], [[INDEX]]
+; CHECK-NEXT:    [[AVL_NEXT]] = sub nuw i64 [[AVL]], [[TMP11]]
+; CHECK-NEXT:    [[TMP12:%.*]] = icmp eq i64 [[AVL_NEXT]], 0
 ; CHECK-NEXT:    br i1 [[TMP8]], label %[[VECTOR_EARLY_EXIT:.*]], label %[[MIDDLE_BLOCK]]
 ; CHECK:       [[MIDDLE_BLOCK]]:
-; CHECK-NEXT:    br i1 [[TMP9]], label %[[MIDDLE_BLOCK1:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP11:![0-9]+]]
+; CHECK-NEXT:    br i1 [[TMP12]], label %[[MIDDLE_BLOCK1:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP11:![0-9]+]]
 ; CHECK:       [[MIDDLE_BLOCK1]]:
-; CHECK-NEXT:    [[TMP12:%.*]] = icmp eq i64 20, [[N_VEC]]
-; CHECK-NEXT:    br i1 [[TMP12]], label %[[EXIT:.*]], label %[[SCALAR_PH]]
+; CHECK-NEXT:    br label %[[EXIT:.*]]
 ; CHECK:       [[VECTOR_EARLY_EXIT]]:
 ; CHECK-NEXT:    br label %[[EXIT]]
-; CHECK:       [[SCALAR_PH]]:
-; CHECK-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC]], %[[MIDDLE_BLOCK1]] ], [ 0, %[[ENTRY]] ]
-; CHECK-NEXT:    br label %[[LOOP_HEADER:.*]]
-; CHECK:       [[LOOP_HEADER]]:
-; CHECK-NEXT:    [[IV:%.*]] = phi i64 [ [[BC_RESUME_VAL]], %[[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], %[[LOOP_LATCH:.*]] ]
-; CHECK-NEXT:    [[GEP_SRC:%.*]] = getelementptr i16, ptr [[SRC]], i64 [[IV]]
-; CHECK-NEXT:    [[L:%.*]] = load i16, ptr [[GEP_SRC]], align 2
-; CHECK-NEXT:    [[C:%.*]] = icmp sgt i16 [[L]], 500
-; CHECK-NEXT:    br i1 [[C]], label %[[EXIT]], label %[[LOOP_LATCH]]
-; CHECK:       [[LOOP_LATCH]]:
-; CHECK-NEXT:    [[OFF_0:%.*]] = shl i64 [[IV]], 1
-; CHECK-NEXT:    [[GEP_DST_0:%.*]] = getelementptr i16, ptr [[DST]], i64 [[OFF_0]]
-; CHECK-NEXT:    store i16 1, ptr [[GEP_DST_0]], align 2
-; CHECK-NEXT:    [[OFF_1:%.*]] = or disjoint i64 [[OFF_0]], 1
-; CHECK-NEXT:    [[GEP_DST_1:%.*]] = getelementptr i16, ptr [[DST]], i64 [[OFF_1]]
-; CHECK-NEXT:    store i16 2, ptr [[GEP_DST_1]], align 2
-; CHECK-NEXT:    [[IV_NEXT]] = add nuw nsw i64 [[IV]], 1
-; CHECK-NEXT:    [[EC:%.*]] = icmp eq i64 [[IV_NEXT]], 20
-; CHECK-NEXT:    br i1 [[EC]], label %[[EXIT]], label %[[LOOP_HEADER]], !llvm.loop [[LOOP12:![0-9]+]]
 ; CHECK:       [[EXIT]]:
 ; CHECK-NEXT:    ret void
 ;
@@ -1374,7 +1357,7 @@ define void @interleaved_stores_before_early_exit(ptr noalias %dst, ptr align 2
 ; CHECK-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[INDEX]], [[TMP1]]
 ; CHECK-NEXT:    [[TMP9:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
 ; CHECK-NEXT:    [[TMP10:%.*]] = or i1 [[TMP8]], [[TMP9]]
-; CHECK-NEXT:    br i1 [[TMP10]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP13:![0-9]+]]
+; CHECK-NEXT:    br i1 [[TMP10]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP12:![0-9]+]]
 ; CHECK:       [[MIDDLE_BLOCK]]:
 ; CHECK-NEXT:    [[TMP5:%.*]] = call i64 @llvm.experimental.cttz.elts.i64.nxv8i1(<vscale x 8 x i1> [[TMP4]], i1 false)
 ; CHECK-NEXT:    [[TMP11:%.*]] = add i64 [[INDEX]], [[TMP5]]
@@ -1398,7 +1381,7 @@ define void @interleaved_stores_before_early_exit(ptr noalias %dst, ptr align 2
 ; CHECK:       [[LOOP_LATCH]]:
 ; CHECK-NEXT:    [[IV_NEXT]] = add nuw nsw i64 [[IV]], 1
 ; CHECK-NEXT:    [[EC:%.*]] = icmp eq i64 [[IV_NEXT]], 20
-; CHECK-NEXT:    br i1 [[EC]], label %[[EXIT]], label %[[LOOP_HEADER]], !llvm.loop [[LOOP14:![0-9]+]]
+; CHECK-NEXT:    br i1 [[EC]], label %[[EXIT]], label %[[LOOP_HEADER]], !llvm.loop [[LOOP13:![0-9]+]]
 ; CHECK:       [[EXIT]]:
 ; CHECK-NEXT:    ret void
 ;
@@ -1458,7 +1441,7 @@ define void @reverse_interleaved_stores_before_early_exit(ptr noalias %dst, ptr
 ; CHECK-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[INDEX]], [[TMP1]]
 ; CHECK-NEXT:    [[TMP13:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
 ; CHECK-NEXT:    [[TMP14:%.*]] = or i1 [[TMP12]], [[TMP13]]
-; CHECK-NEXT:    br i1 [[TMP14]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP15:![0-9]+]]
+; CHECK-NEXT:    br i1 [[TMP14]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP14:![0-9]+]]
 ; CHECK:       [[MIDDLE_BLOCK]]:
 ; CHECK-NEXT:    [[TMP8:%.*]] = call i64 @llvm.experimental.cttz.elts.i64.nxv2i1(<vscale x 2 x i1> [[TMP7]], i1 false)
 ; CHECK-NEXT:    [[TMP15:%.*]] = add i64 [[INDEX]], [[TMP8]]
@@ -1483,7 +1466,7 @@ define void @reverse_interleaved_stores_before_early_exit(ptr noalias %dst, ptr
 ; CHECK:       [[LOOP_LATCH]]:
 ; CHECK-NEXT:    [[IV_NEXT]] = add nuw nsw i64 [[IV]], 1
 ; CHECK-NEXT:    [[EC:%.*]] = icmp eq i64 [[IV_NEXT]], 20
-; CHECK-NEXT:    br i1 [[EC]], label %[[EXIT]], label %[[LOOP_HEADER]], !llvm.loop [[LOOP16:![0-9]+]]
+; CHECK-NEXT:    br i1 [[EC]], label %[[EXIT]], label %[[LOOP_HEADER]], !llvm.loop [[LOOP15:![0-9]+]]
 ; CHECK:       [[EXIT]]:
 ; CHECK-NEXT:    ret void
 ;
diff --git a/llvm/test/Transforms/LoopVectorize/early-exit-tail-folding.ll b/llvm/test/Transforms/LoopVectorize/early-exit-tail-folding.ll
index dfad0a260c0251..74cd834636c7c3 100644
--- a/llvm/test/Transforms/LoopVectorize/early-exit-tail-folding.ll
+++ b/llvm/test/Transforms/LoopVectorize/early-exit-tail-folding.ll
@@ -273,72 +273,42 @@ exit:
 }
 
 define void @early_exit_with_store_after_exit(ptr align 4 dereferenceable(4096) noalias %dst, ptr align 4 dereferenceable(4096) %pred) {
-; PREFER-LABEL: define void @early_exit_with_store_after_exit(
-; PREFER-SAME: ptr noalias align 4 dereferenceable(4096) [[DST:%.*]], ptr align 4 dereferenceable(4096) [[PRED:%.*]]) {
-; PREFER-NEXT:  [[ENTRY:.*:]]
-; PREFER-NEXT:    br label %[[VECTOR_PH:.*]]
-; PREFER:       [[VECTOR_PH]]:
-; PREFER-NEXT:    br label %[[VECTOR_BODY:.*]]
-; PREFER:       [[VECTOR_BODY]]:
-; PREFER-NEXT:    [[INDEX:%.*]] = phi i32 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY_INTERIM:.*]] ]
-; PREFER-NEXT:    [[TMP0:%.*]] = getelementptr inbounds nuw i32, ptr [[PRED]], i32 [[INDEX]]
-; PREFER-NEXT:    [[WIDE_LOAD:%.*]] = load <4 x i32>, ptr [[TMP0]], align 4
-; PREFER-NEXT:    [[TMP1:%.*]] = icmp sgt <4 x i32> [[WIDE_LOAD]], splat (i32 500)
-; PREFER-NEXT:    [[TMP2:%.*]] = freeze <4 x i1> [[TMP1]]
-; PREFER-NEXT:    [[TMP3:%.*]] = call <4 x i1> @llvm.mask.beforefirst.v4i1(<4 x i1> [[TMP2]])
-; PREFER-NEXT:    [[TMP4:%.*]] = getelementptr i32, ptr [[DST]], i32 [[INDEX]]
-; PREFER-NEXT:    [[WIDE_MASKED_LOAD:%.*]] = call <4 x i32> @llvm.masked.load.v4i32.p0(ptr align 4 [[TMP4]], <4 x i1> [[TMP3]], <4 x i32> poison)
-; PREFER-NEXT:    [[TMP5:%.*]] = add nsw <4 x i32> [[WIDE_MASKED_LOAD]], splat (i32 1)
-; PREFER-NEXT:    call void @llvm.masked.store.v4i32.p0(<4 x i32> [[TMP5]], ptr align 4 [[TMP4]], <4 x i1> [[TMP3]])
-; PREFER-NEXT:    [[TMP6:%.*]] = call i1 @llvm.vector.reduce.or.v4i1(<4 x i1> [[TMP2]])
-; PREFER-NEXT:    [[INDEX_NEXT]] = add nuw i32 [[INDEX]], 4
-; PREFER-NEXT:    [[TMP7:%.*]] = icmp eq i32 [[INDEX_NEXT]], 120
-; PREFER-NEXT:    br i1 [[TMP6]], label %[[VECTOR_EARLY_EXIT:.*]], label %[[VECTOR_BODY_INTERIM]]
-; PREFER:       [[VECTOR_BODY_INTERIM]]:
-; PREFER-NEXT:    br i1 [[TMP7]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP6:![0-9]+]]
-; PREFER:       [[MIDDLE_BLOCK]]:
-; PREFER-NEXT:    br label %[[SCALAR_PH:.*]]
-; PREFER:       [[VECTOR_EARLY_EXIT]]:
-; PREFER-NEXT:    br label %[[EXIT:.*]]
-; PREFER:       [[SCALAR_PH]]:
-; PREFER-NEXT:    br label %[[LOOP:.*]]
-; PREFER:       [[LOOP]]:
-; PREFER-NEXT:    [[IV:%.*]] = phi i32 [ 120, %[[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], %[[LATCH:.*]] ]
-; PREFER-NEXT:    [[EE_ADDR:%.*]] = getelementptr inbounds nuw i32, ptr [[PRED]], i32 [[IV]]
-; PREFER-NEXT:    [[EE_VAL:%.*]] = load i32, ptr [[EE_ADDR]], align 4
-; PREFER-NEXT:    [[EE_COND:%.*]] = icmp sgt i32 [[EE_VAL]], 500
-; PREFER-NEXT:    br i1 [[EE_COND]], label %[[EXIT]], label %[[LATCH]]
-; PREFER:       [[LATCH]]:
-; PREFER-NEXT:    [[ST_ADDR:%.*]] = getelementptr inbounds nuw i32, ptr [[DST]], i32 [[IV]]
-; PREFER-NEXT:    [[DATA:%.*]] = load i32, ptr [[ST_ADDR]], align 4
-; PREFER-NEXT:    [[INC:%.*]] = add nsw i32 [[DATA]], 1
-; PREFER-NEXT:    store i32 [[INC]], ptr [[ST_ADDR]], align 4
-; PREFER-NEXT:    [[IV_NEXT]] = add nuw nsw i32 [[IV]], 1
-; PREFER-NEXT:    [[EC:%.*]] = icmp eq i32 [[IV_NEXT]], 123
-; PREFER-NEXT:    br i1 [[EC]], label %[[EXIT]], label %[[LOOP]], !llvm.loop [[LOOP7:![0-9]+]]
-; PREFER:       [[EXIT]]:
-; PREFER-NEXT:    ret void
-;
-; MUST-LABEL: define void @early_exit_with_store_after_exit(
-; MUST-SAME: ptr noalias align 4 dereferenceable(4096) [[DST:%.*]], ptr align 4 dereferenceable(4096) [[PRED:%.*]]) {
-; MUST-NEXT:  [[ENTRY:.*]]:
-; MUST-NEXT:    br label %[[LOOP:.*]]
-; MUST:       [[LOOP]]:
-; MUST-NEXT:    [[IV:%.*]] = phi i32 [ 0, %[[ENTRY]] ], [ [[IV_NEXT:%.*]], %[[LATCH:.*]] ]
-; MUST-NEXT:    [[EE_ADDR:%.*]] = getelementptr inbounds nuw i32, ptr [[PRED]], i32 [[IV]]
-; MUST-NEXT:    [[EE_VAL:%.*]] = load i32, ptr [[EE_ADDR]], align 4
-; MUST-NEXT:    [[EE_COND:%.*]] = icmp sgt i32 [[EE_VAL]], 500
-; MUST-NEXT:    br i1 [[EE_COND]], label %[[EXIT:.*]], label %[[LATCH]]
-; MUST:       [[LATCH]]:
-; MUST-NEXT:    [[ST_ADDR:%.*]] = getelementptr inbounds nuw i32, ptr [[DST]], i32 [[IV]]
-; MUST-NEXT:    [[DATA:%.*]] = load i32, ptr [[ST_ADDR]], align 4
-; MUST-NEXT:    [[INC:%.*]] = add nsw i32 [[DATA]], 1
-; MUST-NEXT:    store i32 [[INC]], ptr [[ST_ADDR]], align 4
-; MUST-NEXT:    [[IV_NEXT]] = add nuw nsw i32 [[IV]], 1
-; MUST-NEXT:    [[EC:%.*]] = icmp eq i32 [[IV_NEXT]], 123
-; MUST-NEXT:    br i1 [[EC]], label %[[EXIT]], label %[[LOOP]]
-; MUST:       [[EXIT]]:
-; MUST-NEXT:    ret void
+; CHECK-LABEL: define void @early_exit_with_store_after_exit(
+; CHECK-SAME: ptr noalias align 4 dereferenceable(4096) [[DST:%.*]], ptr align 4 dereferenceable(4096) [[PRED:%.*]]) {
+; CHECK-NEXT:  [[ENTRY:.*:]]
+; CHECK-NEXT:    br label %[[VECTOR_PH:.*]]
+; CHECK:       [[VECTOR_PH]]:
+; CHECK-NEXT:    br label %[[VECTOR_BODY:.*]]
+; CHECK:       [[VECTOR_BODY]]:
+; CHECK-NEXT:    [[INDEX:%.*]] = phi i32 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY_INTERIM:.*]] ]
+; CHECK-NEXT:    [[VEC_IND:%.*]] = phi <4 x i8> [ <i8 0, i8 1, i8 2, i8 3>, %[[VECTOR_PH]] ], [ [[VEC_IND_NEXT:%.*]], %[[VECTOR_BODY_INTERIM]] ]
+; CHECK-NEXT:    [[TMP0:%.*]] = icmp ule <4 x i8> [[VEC_IND]], splat (i8 122)
+; CHECK-NEXT:    [[TMP1:%.*]] = getelementptr inbounds nuw i32, ptr [[PRED]], i32 [[INDEX]]
+; CHECK-NEXT:    [[WIDE_MASKED_LOAD:%.*]] = call <4 x i32> @llvm.masked.load.v4i32.p0(ptr align 4 [[TMP1]], <4 x i1> [[TMP0]], <4 x i32> poison)
+; CHECK-NEXT:    [[TMP2:%.*]] = icmp sgt <4 x i32> [[WIDE_MASKED_LOAD]], splat (i32 500)
+; CHECK-NEXT:    [[TMP3:%.*]] = select <4 x i1> [[TMP0]], <4 x i1> [[TMP2]], <4 x i1> zeroinitializer
+; CHECK-NEXT:    [[TMP4:%.*]] = freeze <4 x i1> [[TMP3]]
+; CHECK-NEXT:    [[TMP5:%.*]] = call <4 x i1> @llvm.mask.beforefirst.v4i1(<4 x i1> [[TMP4]])
+; CHECK-NEXT:    [[TMP6:%.*]] = select <4 x i1> [[TMP0]], <4 x i1> [[TMP5]], <4 x i1> zeroinitializer
+; CHECK-NEXT:    [[TMP7:%.*]] = getelementptr i32, ptr [[DST]], i32 [[INDEX]]
+; CHECK-NEXT:    [[WIDE_MASKED_LOAD1:%.*]] = call <4 x i32> @llvm.masked.load.v4i32.p0(ptr align 4 [[TMP7]], <4 x i1> [[TMP6]], <4 x i32> poison)
+; CHECK-NEXT:    [[TMP8:%.*]] = add nsw <4 x i32> [[WIDE_MASKED_LOAD1]], splat (i32 1)
+; CHECK-NEXT:    call void @llvm.masked.store.v4i32.p0(<4 x i32> [[TMP8]], ptr align 4 [[TMP7]], <4 x i1> [[TMP6]])
+; CHECK-NEXT:    [[TMP9:%.*]] = freeze <4 x i1> [[TMP2]]
+; CHECK-NEXT:    [[TMP10:%.*]] = select <4 x i1> [[TMP0]], <4 x i1> [[TMP9]], <4 x i1> zeroinitializer
+; CHECK-NEXT:    [[TMP11:%.*]] = call i1 @llvm.vector.reduce.or.v4i1(<4 x i1> [[TMP10]])
+; CHECK-NEXT:    [[INDEX_NEXT]] = add nuw i32 [[INDEX]], 4
+; CHECK-NEXT:    [[TMP12:%.*]] = icmp eq i32 [[INDEX_NEXT]], 124
+; CHECK-NEXT:    [[VEC_IND_NEXT]] = add nuw <4 x i8> [[VEC_IND]], splat (i8 4)
+; CHECK-NEXT:    br i1 [[TMP11]], label %[[VECTOR_EARLY_EXIT:.*]], label %[[VECTOR_BODY_INTERIM]]
+; CHECK:       [[VECTOR_BODY_INTERIM]]:
+; CHECK-NEXT:    br i1 [[TMP12]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP6:![0-9]+]]
+; CHECK:       [[MIDDLE_BLOCK]]:
+; CHECK-NEXT:    br label %[[EXIT:.*]]
+; CHECK:       [[VECTOR_EARLY_EXIT]]:
+; CHECK-NEXT:    br label %[[EXIT]]
+; CHECK:       [[EXIT]]:
+; CHECK-NEXT:    ret void
 ;
 entry:
   br label %loop
@@ -382,7 +352,7 @@ define i32 @countable_early_exit(ptr align 4 dereferenceable(4096) %p, i32 %n) {
 ; PREFER-NEXT:    [[INDEX:%.*]] = phi i32 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
 ; PREFER-NEXT:    [[INDEX_NEXT]] = add nuw i32 [[INDEX]], 4
 ; PREFER-NEXT:    [[TMP5:%.*]] = icmp eq i32 [[INDEX_NEXT]], [[N_VEC]]
-; PREFER-NEXT:    br i1 [[TMP5]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP8:![0-9]+]]
+; PREFER-NEXT:    br i1 [[TMP5]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP7:![0-9]+]]
 ; PREFER:       [[MIDDLE_BLOCK]]:
 ; PREFER-NEXT:    br label %[[SCALAR_PH]]
 ; PREFER:       [[SCALAR_PH]]:
@@ -397,7 +367,7 @@ define i32 @countable_early_exit(ptr align 4 dereferenceable(4096) %p, i32 %n) {
 ; PREFER-NEXT:    [[LD:%.*]] = load i32, ptr [[GEP]], align 4
 ; PREFER-NEXT:    [[IV_NEXT]] = add nuw nsw i32 [[IV]], 1
 ; PREFER-NEXT:    [[EC:%.*]] = icmp eq i32 [[IV_NEXT]], 123
-; PREFER-NEXT:    br i1 [[EC]], label %[[EXIT]], label %[[LOOP]], !llvm.loop [[LOOP9:![0-9]+]]
+; PREFER-NEXT:    br i1 [[EC]], label %[[EXIT]], label %[[LOOP]], !llvm.loop [[LOOP8:![0-9]+]]
 ; PREFER:       [[EXIT]]:
 ; PREFER-NEXT:    [[RES:%.*]] = phi i32 [ 0, %[[LOOP]] ], [ [[LD]], %[[LATCH]] ]
 ; PREFER-NEXT:    ret i32 [[RES]]
@@ -463,7 +433,7 @@ define void @early_exit_with_store(ptr align 4 dereferenceable(4096) noalias %ds
 ; PREFER-NEXT:    [[INDEX_NEXT]] = add nuw i32 [[INDEX]], 4
 ; PREFER-NEXT:    [[TMP7:%.*]] = icmp eq i32 [[INDEX_NEXT]], 120
 ; PREFER-NEXT:    [[TMP8:%.*]] = or i1 [[TMP6]], [[TMP7]]
-; PREFER-NEXT:    br i1 [[TMP8]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP10:![0-9]+]]
+; PREFER-NEXT:    br i1 [[TMP8]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP9:![0-9]+]]
 ; PREFER:       [[MIDDLE_BLOCK]]:
 ; PREFER-NEXT:    [[TMP9:%.*]] = call i64 @llvm.experimental.cttz.elts.i64.v4i1(<4 x i1> [[TMP2]], i1 false)
 ; PREFER-NEXT:    [[TMP10:%.*]] = trunc i64 [[TMP9]] to i32
@@ -485,7 +455,7 @@ define void @early_exit_with_store(ptr align 4 dereferenceable(4096) noalias %ds
 ; PREFER:       [[LATCH]]:
 ; PREFER-NEXT:    [[IV_NEXT]] = add nuw nsw i32 [[IV]], 1
 ; PREFER-NEXT:    [[EC:%.*]] = icmp eq i32 [[IV_NEXT]], 123
-; PREFER-NEXT:    br i1 [[EC]], label %[[EXIT]], label %[[LOOP]], !llvm.loop [[LOOP11:![0-9]+]]
+; PREFER-NEXT:    br i1 [[EC]], label %[[EXIT]], label %[[LOOP]], !llvm.loop [[LOOP10:![0-9]+]]
 ; PREFER:       [[EXIT]]:
 ; PREFER-NEXT:    ret void
 ;
diff --git a/llvm/test/Transforms/LoopVectorize/early_exit_store_legality.ll b/llvm/test/Transforms/LoopVectorize/early_exit_store_legality.ll
index 40e869e94857ef..67ba1b29406561 100644
--- a/llvm/test/Transforms/LoopVectorize/early_exit_store_legality.ll
+++ b/llvm/test/Transforms/LoopVectorize/early_exit_store_legality.ll
@@ -777,8 +777,8 @@ exit:
 define void @uncountable_exit_with_constant_nonunit_stride(ptr dereferenceable(4000) noalias %array, ptr align 2 dereferenceable(4000) readonly %pred) !dbg !60 {
 ; CHECK-DEBUG-LABEL: LV: Checking a loop in 'uncountable_exit_with_constant_nonunit_stride'
 ; CHECK-DEBUG:       LV: Loop passed LoopVectorizationLegality checks!
-; CHECK-DEBUG:       LV: Not vectorizing: unable to calculate the loop count due to complex control flow.
-; CHECK-REMARK:      foo.c:260:3: loop not vectorized: unable to calculate the loop count due to complex control flow
+; CHECK-DEBUG:       LV: Not vectorizing: Early exit loop with side effects contains load used by the exit condition with an unsupported memory access pattern.
+; CHECK-REMARK:      foo.c:260:3: loop not vectorized: Early exit loop with side effects contains load used by the exit condition with an unsupported memory access pattern
 entry:
   br label %for.body, !dbg !61
 



More information about the llvm-branch-commits mailing list