[llvm-branch-commits] [llvm] [VPlan] Support interleaving early exit loops with side effects (PR #235021)
Luke Lau via llvm-branch-commits
llvm-branch-commits at lists.llvm.org
Fri Oct 23 19:47:00 PDT 2026
https://github.com/lukel97 updated https://github.com/llvm/llvm-project/pull/235021
>From 0fd85a456d14ef7be5a68cdf74a0aadef6c39eaf Mon Sep 17 00:00:00 2001
From: Luke Lau <luke at igalia.com>
Date: Fri, 23 Oct 2026 13:54:55 +0800
Subject: [PATCH] [VPlan] Support interleaving early exit loops with side
effects
We couldn't do this on main previously because the exit mask `active-lane-mask(0, FirstActiveLane(Cond))` doesn't get unrolled properly across parts. Now that we use mask.beforefirst we can unroll it properly in a chain. The unrolled form is:
unroll(mask.beforefirst, P) = mask.beforefirst(cond[P] | broadcast(anyof(not(unroll(mask.beforefirst, P-1)))))
On AArch64 this can be combined to brkpb in another patch.
---
.../Transforms/Vectorize/LoopVectorize.cpp | 9 -
llvm/lib/Transforms/Vectorize/VPlanUnroll.cpp | 11 +
.../AArch64/early_exit_with_stores.ll | 231 ++++++++++++++++--
.../interleave_uncountable_exits.ll | 38 ++-
...ountable-and-uncountable-exits-combined.ll | 39 ++-
5 files changed, 293 insertions(+), 35 deletions(-)
diff --git a/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp b/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
index 49d5c5631e8a8e..8619a136d08426 100644
--- a/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
+++ b/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
@@ -7641,15 +7641,6 @@ bool LoopVectorizePass::processLoop(Loop *L) {
IC = 1;
}
- // FIXME: Enable interleaving for EE-with-side-effects.
- if (InterleaveLoop && LVL.hasUncountableExitWithSideEffects()) {
- LLVM_DEBUG(dbgs() << "LV: Not interleaving due to EE with side effects.\n");
- IntDiagMsg = {"EEWithSideEffectsPreventsInterleaving",
- "Unable to interleave due to early exit with side effects."};
- InterleaveLoop = false;
- IC = 1;
- }
-
// Emit diagnostic messages, if any.
if (!VectorizeLoop && !InterleaveLoop) {
// Do not vectorize or interleaving the loop.
diff --git a/llvm/lib/Transforms/Vectorize/VPlanUnroll.cpp b/llvm/lib/Transforms/Vectorize/VPlanUnroll.cpp
index 8858f0fad801ad..fb39b0029c2800 100644
--- a/llvm/lib/Transforms/Vectorize/VPlanUnroll.cpp
+++ b/llvm/lib/Transforms/Vectorize/VPlanUnroll.cpp
@@ -334,6 +334,17 @@ void UnrollState::unrollRecipeByUF(VPRecipeBase &R) {
Copy->setOperand(1, getValueForPart(Op, Part));
continue;
}
+ if (match(&R,
+ m_VPInstruction<VPInstruction::MaskBeforeFirst>(m_VPValue(Op)))) {
+ VPBuilder Builder(Copy);
+ VPValue *Chain = Builder.createNaryOp(
+ VPInstruction::AnyOf,
+ Builder.createNot(getValueForPart(R.getVPSingleValue(), Part - 1)));
+ if (!Plan.hasScalarVFOnly())
+ Chain = Builder.createNaryOp(VPInstruction::Broadcast, Chain);
+ Copy->setOperand(0, Builder.createOr(getValueForPart(Op, Part), Chain));
+ continue;
+ }
if (match(&R, m_VPInstruction<VPInstruction::ExtractVectorForPart>(
m_VPValue(Op), m_VPValue()))) {
Copy->setOperand(0, Op);
diff --git a/llvm/test/Transforms/LoopVectorize/AArch64/early_exit_with_stores.ll b/llvm/test/Transforms/LoopVectorize/AArch64/early_exit_with_stores.ll
index 9eb18183098010..5064e4197e5023 100644
--- a/llvm/test/Transforms/LoopVectorize/AArch64/early_exit_with_stores.ll
+++ b/llvm/test/Transforms/LoopVectorize/AArch64/early_exit_with_stores.ll
@@ -54,32 +54,72 @@ define void @loop_contains_store_condition_load_has_single_user(ptr dereferencea
; CHECK-SAME: ptr noalias dereferenceable(40) [[ARRAY:%.*]], ptr readonly align 2 dereferenceable(40) [[PRED:%.*]]) #[[ATTR0]] {
; CHECK-NEXT: [[SCALAR_PH1:.*]]:
; CHECK-NEXT: [[TMP0:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-NEXT: [[TMP1:%.*]] = shl nuw i64 [[TMP0]], 2
+; CHECK-NEXT: [[TMP1:%.*]] = shl nuw i64 [[TMP0]], 3
; CHECK-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 20, [[TMP1]]
; CHECK-NEXT: br i1 [[MIN_ITERS_CHECK]], label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]]
; CHECK: [[VECTOR_PH]]:
+; CHECK-NEXT: [[TMP2:%.*]] = shl nuw i64 [[TMP0]], 2
+; CHECK-NEXT: [[BROADCAST_SPLATINSERT:%.*]] = insertelement <vscale x 4 x i64> poison, i64 [[TMP2]], i64 0
+; CHECK-NEXT: [[BROADCAST_SPLAT:%.*]] = shufflevector <vscale x 4 x i64> [[BROADCAST_SPLATINSERT]], <vscale x 4 x i64> poison, <vscale x 4 x i32> zeroinitializer
; CHECK-NEXT: [[N_MOD_VF:%.*]] = urem i64 20, [[TMP1]]
; CHECK-NEXT: [[N_VEC:%.*]] = sub i64 20, [[N_MOD_VF]]
+; CHECK-NEXT: [[TMP17:%.*]] = call <vscale x 4 x i64> @llvm.stepvector.nxv4i64()
; CHECK-NEXT: br label %[[FOR_BODY1:.*]]
; CHECK: [[FOR_BODY1]]:
; CHECK-NEXT: [[IV1:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[FOR_BODY1]] ]
+; CHECK-NEXT: [[VEC_IND:%.*]] = phi <vscale x 4 x i64> [ [[TMP17]], %[[VECTOR_PH]] ], [ [[VEC_IND_NEXT:%.*]], %[[FOR_BODY1]] ]
+; CHECK-NEXT: [[STEP_ADD:%.*]] = add nuw <vscale x 4 x i64> [[VEC_IND]], [[BROADCAST_SPLAT]]
; CHECK-NEXT: [[TMP5:%.*]] = getelementptr inbounds nuw i16, ptr [[PRED]], i64 [[IV1]]
+; CHECK-NEXT: [[TMP18:%.*]] = getelementptr inbounds nuw i16, ptr [[TMP5]], i64 [[TMP2]]
; CHECK-NEXT: [[WIDE_LOAD:%.*]] = load <vscale x 4 x i16>, ptr [[TMP5]], align 2
+; CHECK-NEXT: [[WIDE_LOAD1:%.*]] = load <vscale x 4 x i16>, ptr [[TMP18]], align 2
; CHECK-NEXT: [[TMP3:%.*]] = icmp sgt <vscale x 4 x i16> [[WIDE_LOAD]], splat (i16 500)
+; CHECK-NEXT: [[TMP19:%.*]] = icmp sgt <vscale x 4 x i16> [[WIDE_LOAD1]], splat (i16 500)
; CHECK-NEXT: [[TMP4:%.*]] = freeze <vscale x 4 x i1> [[TMP3]]
+; CHECK-NEXT: [[TMP9:%.*]] = freeze <vscale x 4 x i1> [[TMP19]]
; CHECK-NEXT: [[TMP6:%.*]] = call <vscale x 4 x i1> @llvm.mask.beforefirst.nxv4i1(<vscale x 4 x i1> [[TMP4]])
+; CHECK-NEXT: [[TMP25:%.*]] = xor <vscale x 4 x i1> [[TMP6]], splat (i1 true)
+; CHECK-NEXT: [[TMP30:%.*]] = call i1 @llvm.vector.reduce.or.nxv4i1(<vscale x 4 x i1> [[TMP25]])
+; CHECK-NEXT: [[BROADCAST_SPLATINSERT2:%.*]] = insertelement <vscale x 4 x i1> poison, i1 [[TMP30]], i64 0
+; CHECK-NEXT: [[BROADCAST_SPLAT3:%.*]] = shufflevector <vscale x 4 x i1> [[BROADCAST_SPLATINSERT2]], <vscale x 4 x i1> poison, <vscale x 4 x i32> zeroinitializer
+; CHECK-NEXT: [[TMP32:%.*]] = or <vscale x 4 x i1> [[TMP9]], [[BROADCAST_SPLAT3]]
+; CHECK-NEXT: [[UNCOUNTABLE_EXIT_MASK2:%.*]] = call <vscale x 4 x i1> @llvm.mask.beforefirst.nxv4i1(<vscale x 4 x i1> [[TMP32]])
; CHECK-NEXT: [[ST_ADDR1:%.*]] = getelementptr i16, ptr [[ARRAY]], i64 [[IV1]]
+; CHECK-NEXT: [[TMP31:%.*]] = getelementptr i16, ptr [[ST_ADDR1]], i64 [[TMP2]]
; CHECK-NEXT: [[WIDE_MASKED_LOAD:%.*]] = call <vscale x 4 x i16> @llvm.masked.load.nxv4i16.p0(ptr align 2 [[ST_ADDR1]], <vscale x 4 x i1> [[TMP6]], <vscale x 4 x i16> poison)
+; CHECK-NEXT: [[WIDE_MASKED_LOAD3:%.*]] = call <vscale x 4 x i16> @llvm.masked.load.nxv4i16.p0(ptr align 2 [[TMP31]], <vscale x 4 x i1> [[UNCOUNTABLE_EXIT_MASK2]], <vscale x 4 x i16> poison)
; CHECK-NEXT: [[TMP8:%.*]] = add nsw <vscale x 4 x i16> [[WIDE_MASKED_LOAD]], splat (i16 1)
+; CHECK-NEXT: [[TMP15:%.*]] = add nsw <vscale x 4 x i16> [[WIDE_MASKED_LOAD3]], splat (i16 1)
; CHECK-NEXT: call void @llvm.masked.store.nxv4i16.p0(<vscale x 4 x i16> [[TMP8]], ptr align 2 [[ST_ADDR1]], <vscale x 4 x i1> [[TMP6]])
-; CHECK-NEXT: [[TMP10:%.*]] = call i1 @llvm.vector.reduce.or.nxv4i1(<vscale x 4 x i1> [[TMP4]])
+; CHECK-NEXT: call void @llvm.masked.store.nxv4i16.p0(<vscale x 4 x i16> [[TMP15]], ptr align 2 [[TMP31]], <vscale x 4 x i1> [[UNCOUNTABLE_EXIT_MASK2]])
+; CHECK-NEXT: [[TMP16:%.*]] = or <vscale x 4 x i1> [[TMP4]], [[TMP9]]
+; CHECK-NEXT: [[TMP10:%.*]] = call i1 @llvm.vector.reduce.or.nxv4i1(<vscale x 4 x i1> [[TMP16]])
; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[IV1]], [[TMP1]]
; CHECK-NEXT: [[TMP11:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
+; CHECK-NEXT: [[VEC_IND_NEXT]] = add nuw nsw <vscale x 4 x i64> [[STEP_ADD]], [[BROADCAST_SPLAT]]
; CHECK-NEXT: [[TMP12:%.*]] = or i1 [[TMP10]], [[TMP11]]
; CHECK-NEXT: br i1 [[TMP12]], label %[[MIDDLE_BLOCK:.*]], label %[[FOR_BODY1]], !llvm.loop [[LOOP0:![0-9]+]]
; CHECK: [[MIDDLE_BLOCK]]:
+; CHECK-NEXT: [[TMP20:%.*]] = call i64 @llvm.vscale.i64()
+; CHECK-NEXT: [[TMP21:%.*]] = mul nuw i64 [[TMP20]], 4
+; CHECK-NEXT: [[TMP22:%.*]] = call i64 @llvm.experimental.cttz.elts.i64.nxv4i1(<vscale x 4 x i1> [[TMP9]], i1 false)
+; CHECK-NEXT: [[TMP23:%.*]] = mul i64 [[TMP21]], 1
+; CHECK-NEXT: [[TMP24:%.*]] = add i64 [[TMP23]], [[TMP22]]
; CHECK-NEXT: [[TMP7:%.*]] = call i64 @llvm.experimental.cttz.elts.i64.nxv4i1(<vscale x 4 x i1> [[TMP4]], i1 false)
-; CHECK-NEXT: [[TMP13:%.*]] = add i64 [[IV1]], [[TMP7]]
+; CHECK-NEXT: [[TMP26:%.*]] = mul i64 [[TMP21]], 0
+; CHECK-NEXT: [[TMP27:%.*]] = add i64 [[TMP26]], [[TMP7]]
+; CHECK-NEXT: [[TMP28:%.*]] = icmp ne i64 [[TMP7]], [[TMP21]]
+; CHECK-NEXT: [[TMP29:%.*]] = select i1 [[TMP28]], i64 [[TMP27]], i64 [[TMP24]]
+; CHECK-NEXT: [[TMP40:%.*]] = call i64 @llvm.vscale.i64()
+; CHECK-NEXT: [[TMP41:%.*]] = mul nuw i64 [[TMP40]], 4
+; CHECK-NEXT: [[TMP33:%.*]] = mul i64 [[TMP41]], 0
+; CHECK-NEXT: [[TMP34:%.*]] = extractelement <vscale x 4 x i64> [[VEC_IND]], i64 0
+; CHECK-NEXT: [[TMP35:%.*]] = mul i64 [[TMP41]], 1
+; CHECK-NEXT: [[TMP36:%.*]] = sub i64 0, [[TMP35]]
+; CHECK-NEXT: [[TMP37:%.*]] = extractelement <vscale x 4 x i64> [[STEP_ADD]], i64 [[TMP36]]
+; CHECK-NEXT: [[TMP38:%.*]] = icmp uge i64 0, [[TMP35]]
+; CHECK-NEXT: [[TMP39:%.*]] = select i1 [[TMP38]], i64 [[TMP37]], i64 [[TMP34]]
+; CHECK-NEXT: [[TMP13:%.*]] = add i64 [[TMP39]], [[TMP29]]
; CHECK-NEXT: [[TMP14:%.*]] = icmp eq i64 [[TMP13]], 20
; CHECK-NEXT: br i1 [[TMP14]], label %[[EXIT:.*]], label %[[SCALAR_PH]]
; CHECK: [[SCALAR_PH]]:
@@ -130,32 +170,72 @@ define void @loop_contains_store_fcmp_condition(ptr dereferenceable(40) noalias
; CHECK-SAME: ptr noalias dereferenceable(40) [[ARRAY:%.*]], ptr readonly align 2 dereferenceable(40) [[PRED:%.*]]) #[[ATTR0]] {
; CHECK-NEXT: [[ENTRY:.*]]:
; CHECK-NEXT: [[TMP0:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-NEXT: [[TMP1:%.*]] = shl nuw i64 [[TMP0]], 2
+; CHECK-NEXT: [[TMP1:%.*]] = shl nuw i64 [[TMP0]], 3
; CHECK-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 20, [[TMP1]]
; CHECK-NEXT: br i1 [[MIN_ITERS_CHECK]], label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]]
; CHECK: [[VECTOR_PH]]:
+; CHECK-NEXT: [[TMP14:%.*]] = shl nuw i64 [[TMP0]], 2
+; CHECK-NEXT: [[BROADCAST_SPLATINSERT:%.*]] = insertelement <vscale x 4 x i64> poison, i64 [[TMP14]], i64 0
+; CHECK-NEXT: [[BROADCAST_SPLAT:%.*]] = shufflevector <vscale x 4 x i64> [[BROADCAST_SPLATINSERT]], <vscale x 4 x i64> poison, <vscale x 4 x i32> zeroinitializer
; CHECK-NEXT: [[N_MOD_VF:%.*]] = urem i64 20, [[TMP1]]
; CHECK-NEXT: [[N_VEC:%.*]] = sub i64 20, [[N_MOD_VF]]
+; CHECK-NEXT: [[TMP17:%.*]] = call <vscale x 4 x i64> @llvm.stepvector.nxv4i64()
; CHECK-NEXT: br label %[[FOR_BODY:.*]]
; CHECK: [[FOR_BODY]]:
; CHECK-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[FOR_BODY]] ]
+; CHECK-NEXT: [[VEC_IND:%.*]] = phi <vscale x 4 x i64> [ [[TMP17]], %[[VECTOR_PH]] ], [ [[VEC_IND_NEXT:%.*]], %[[FOR_BODY]] ]
+; CHECK-NEXT: [[STEP_ADD:%.*]] = add nuw <vscale x 4 x i64> [[VEC_IND]], [[BROADCAST_SPLAT]]
; CHECK-NEXT: [[TMP2:%.*]] = getelementptr inbounds nuw half, ptr [[PRED]], i64 [[INDEX]]
+; CHECK-NEXT: [[TMP18:%.*]] = getelementptr inbounds nuw half, ptr [[TMP2]], i64 [[TMP14]]
; CHECK-NEXT: [[WIDE_LOAD:%.*]] = load <vscale x 4 x half>, ptr [[TMP2]], align 2
+; CHECK-NEXT: [[WIDE_LOAD1:%.*]] = load <vscale x 4 x half>, ptr [[TMP18]], align 2
; CHECK-NEXT: [[TMP3:%.*]] = fcmp ugt <vscale x 4 x half> [[WIDE_LOAD]], splat (half 5.000000e+02)
+; CHECK-NEXT: [[TMP19:%.*]] = fcmp ugt <vscale x 4 x half> [[WIDE_LOAD1]], splat (half 5.000000e+02)
; CHECK-NEXT: [[TMP4:%.*]] = freeze <vscale x 4 x i1> [[TMP3]]
+; CHECK-NEXT: [[TMP25:%.*]] = freeze <vscale x 4 x i1> [[TMP19]]
; CHECK-NEXT: [[TMP13:%.*]] = call <vscale x 4 x i1> @llvm.mask.beforefirst.nxv4i1(<vscale x 4 x i1> [[TMP4]])
+; CHECK-NEXT: [[TMP30:%.*]] = xor <vscale x 4 x i1> [[TMP13]], splat (i1 true)
+; CHECK-NEXT: [[TMP31:%.*]] = call i1 @llvm.vector.reduce.or.nxv4i1(<vscale x 4 x i1> [[TMP30]])
+; CHECK-NEXT: [[BROADCAST_SPLATINSERT2:%.*]] = insertelement <vscale x 4 x i1> poison, i1 [[TMP31]], i64 0
+; CHECK-NEXT: [[BROADCAST_SPLAT3:%.*]] = shufflevector <vscale x 4 x i1> [[BROADCAST_SPLATINSERT2]], <vscale x 4 x i1> poison, <vscale x 4 x i32> zeroinitializer
+; CHECK-NEXT: [[TMP33:%.*]] = or <vscale x 4 x i1> [[TMP25]], [[BROADCAST_SPLAT3]]
+; CHECK-NEXT: [[UNCOUNTABLE_EXIT_MASK2:%.*]] = call <vscale x 4 x i1> @llvm.mask.beforefirst.nxv4i1(<vscale x 4 x i1> [[TMP33]])
; CHECK-NEXT: [[TMP6:%.*]] = getelementptr i16, ptr [[ARRAY]], i64 [[INDEX]]
+; CHECK-NEXT: [[TMP32:%.*]] = getelementptr i16, ptr [[TMP6]], i64 [[TMP14]]
; CHECK-NEXT: [[WIDE_MASKED_LOAD:%.*]] = call <vscale x 4 x i16> @llvm.masked.load.nxv4i16.p0(ptr align 2 [[TMP6]], <vscale x 4 x i1> [[TMP13]], <vscale x 4 x i16> poison)
+; CHECK-NEXT: [[WIDE_MASKED_LOAD3:%.*]] = call <vscale x 4 x i16> @llvm.masked.load.nxv4i16.p0(ptr align 2 [[TMP32]], <vscale x 4 x i1> [[UNCOUNTABLE_EXIT_MASK2]], <vscale x 4 x i16> poison)
; CHECK-NEXT: [[TMP7:%.*]] = add nsw <vscale x 4 x i16> [[WIDE_MASKED_LOAD]], splat (i16 1)
+; CHECK-NEXT: [[TMP15:%.*]] = add nsw <vscale x 4 x i16> [[WIDE_MASKED_LOAD3]], splat (i16 1)
; CHECK-NEXT: call void @llvm.masked.store.nxv4i16.p0(<vscale x 4 x i16> [[TMP7]], ptr align 2 [[TMP6]], <vscale x 4 x i1> [[TMP13]])
-; CHECK-NEXT: [[TMP8:%.*]] = call i1 @llvm.vector.reduce.or.nxv4i1(<vscale x 4 x i1> [[TMP4]])
+; CHECK-NEXT: call void @llvm.masked.store.nxv4i16.p0(<vscale x 4 x i16> [[TMP15]], ptr align 2 [[TMP32]], <vscale x 4 x i1> [[UNCOUNTABLE_EXIT_MASK2]])
+; CHECK-NEXT: [[TMP16:%.*]] = or <vscale x 4 x i1> [[TMP4]], [[TMP25]]
+; CHECK-NEXT: [[TMP8:%.*]] = call i1 @llvm.vector.reduce.or.nxv4i1(<vscale x 4 x i1> [[TMP16]])
; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], [[TMP1]]
; CHECK-NEXT: [[TMP9:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
+; CHECK-NEXT: [[VEC_IND_NEXT]] = add nuw nsw <vscale x 4 x i64> [[STEP_ADD]], [[BROADCAST_SPLAT]]
; CHECK-NEXT: [[TMP10:%.*]] = or i1 [[TMP8]], [[TMP9]]
; CHECK-NEXT: br i1 [[TMP10]], label %[[MIDDLE_BLOCK:.*]], label %[[FOR_BODY]], !llvm.loop [[LOOP4:![0-9]+]]
; CHECK: [[MIDDLE_BLOCK]]:
+; CHECK-NEXT: [[TMP20:%.*]] = call i64 @llvm.vscale.i64()
+; CHECK-NEXT: [[TMP21:%.*]] = mul nuw i64 [[TMP20]], 4
+; CHECK-NEXT: [[TMP22:%.*]] = call i64 @llvm.experimental.cttz.elts.i64.nxv4i1(<vscale x 4 x i1> [[TMP25]], i1 false)
+; CHECK-NEXT: [[TMP23:%.*]] = mul i64 [[TMP21]], 1
+; CHECK-NEXT: [[TMP24:%.*]] = add i64 [[TMP23]], [[TMP22]]
; CHECK-NEXT: [[TMP5:%.*]] = call i64 @llvm.experimental.cttz.elts.i64.nxv4i1(<vscale x 4 x i1> [[TMP4]], i1 false)
-; CHECK-NEXT: [[TMP11:%.*]] = add i64 [[INDEX]], [[TMP5]]
+; CHECK-NEXT: [[TMP26:%.*]] = mul i64 [[TMP21]], 0
+; CHECK-NEXT: [[TMP27:%.*]] = add i64 [[TMP26]], [[TMP5]]
+; CHECK-NEXT: [[TMP28:%.*]] = icmp ne i64 [[TMP5]], [[TMP21]]
+; CHECK-NEXT: [[TMP29:%.*]] = select i1 [[TMP28]], i64 [[TMP27]], i64 [[TMP24]]
+; CHECK-NEXT: [[TMP40:%.*]] = call i64 @llvm.vscale.i64()
+; CHECK-NEXT: [[TMP41:%.*]] = mul nuw i64 [[TMP40]], 4
+; CHECK-NEXT: [[TMP42:%.*]] = mul i64 [[TMP41]], 0
+; CHECK-NEXT: [[TMP34:%.*]] = extractelement <vscale x 4 x i64> [[VEC_IND]], i64 0
+; CHECK-NEXT: [[TMP35:%.*]] = mul i64 [[TMP41]], 1
+; CHECK-NEXT: [[TMP36:%.*]] = sub i64 0, [[TMP35]]
+; CHECK-NEXT: [[TMP37:%.*]] = extractelement <vscale x 4 x i64> [[STEP_ADD]], i64 [[TMP36]]
+; CHECK-NEXT: [[TMP38:%.*]] = icmp uge i64 0, [[TMP35]]
+; CHECK-NEXT: [[TMP39:%.*]] = select i1 [[TMP38]], i64 [[TMP37]], i64 [[TMP34]]
+; CHECK-NEXT: [[TMP11:%.*]] = add i64 [[TMP39]], [[TMP29]]
; CHECK-NEXT: [[TMP12:%.*]] = icmp eq i64 [[TMP11]], 20
; CHECK-NEXT: br i1 [[TMP12]], label %[[EXIT:.*]], label %[[SCALAR_PH]]
; CHECK: [[SCALAR_PH]]:
@@ -363,33 +443,74 @@ define void @loop_contains_store_to_pointer_with_no_deref_info(ptr align 2 deref
; CHECK-SAME: ptr readonly align 2 dereferenceable(40) [[LOAD_ARRAY:%.*]], ptr noalias align 2 [[ARRAY:%.*]], ptr readonly align 2 dereferenceable(40) [[PRED:%.*]]) #[[ATTR0]] {
; CHECK-NEXT: [[SCALAR_PH:.*]]:
; CHECK-NEXT: [[TMP0:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-NEXT: [[TMP1:%.*]] = shl nuw i64 [[TMP0]], 2
+; CHECK-NEXT: [[TMP1:%.*]] = shl nuw i64 [[TMP0]], 3
; CHECK-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 20, [[TMP1]]
; CHECK-NEXT: br i1 [[MIN_ITERS_CHECK]], label %[[SCALAR_PH1:.*]], label %[[VECTOR_PH:.*]]
; CHECK: [[VECTOR_PH]]:
+; CHECK-NEXT: [[TMP2:%.*]] = shl nuw i64 [[TMP0]], 2
+; CHECK-NEXT: [[BROADCAST_SPLATINSERT:%.*]] = insertelement <vscale x 4 x i64> poison, i64 [[TMP2]], i64 0
+; CHECK-NEXT: [[BROADCAST_SPLAT:%.*]] = shufflevector <vscale x 4 x i64> [[BROADCAST_SPLATINSERT]], <vscale x 4 x i64> poison, <vscale x 4 x i32> zeroinitializer
; CHECK-NEXT: [[N_MOD_VF:%.*]] = urem i64 20, [[TMP1]]
; CHECK-NEXT: [[N_VEC:%.*]] = sub i64 20, [[N_MOD_VF]]
+; CHECK-NEXT: [[TMP16:%.*]] = call <vscale x 4 x i64> @llvm.stepvector.nxv4i64()
; CHECK-NEXT: br label %[[FOR_BODY:.*]]
; CHECK: [[FOR_BODY]]:
; CHECK-NEXT: [[IV:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[FOR_BODY]] ]
+; CHECK-NEXT: [[VEC_IND:%.*]] = phi <vscale x 4 x i64> [ [[TMP16]], %[[VECTOR_PH]] ], [ [[VEC_IND_NEXT:%.*]], %[[FOR_BODY]] ]
+; CHECK-NEXT: [[STEP_ADD:%.*]] = add nuw <vscale x 4 x i64> [[VEC_IND]], [[BROADCAST_SPLAT]]
; CHECK-NEXT: [[TMP5:%.*]] = getelementptr inbounds nuw i16, ptr [[PRED]], i64 [[IV]]
+; CHECK-NEXT: [[TMP19:%.*]] = getelementptr inbounds nuw i16, ptr [[TMP5]], i64 [[TMP2]]
; CHECK-NEXT: [[WIDE_LOAD:%.*]] = load <vscale x 4 x i16>, ptr [[TMP5]], align 2
+; CHECK-NEXT: [[WIDE_LOAD1:%.*]] = load <vscale x 4 x i16>, ptr [[TMP19]], align 2
; CHECK-NEXT: [[TMP3:%.*]] = icmp sgt <vscale x 4 x i16> [[WIDE_LOAD]], splat (i16 500)
+; CHECK-NEXT: [[TMP20:%.*]] = icmp sgt <vscale x 4 x i16> [[WIDE_LOAD1]], splat (i16 500)
; CHECK-NEXT: [[TMP4:%.*]] = freeze <vscale x 4 x i1> [[TMP3]]
+; CHECK-NEXT: [[TMP21:%.*]] = freeze <vscale x 4 x i1> [[TMP20]]
; CHECK-NEXT: [[TMP6:%.*]] = call <vscale x 4 x i1> @llvm.mask.beforefirst.nxv4i1(<vscale x 4 x i1> [[TMP4]])
+; CHECK-NEXT: [[TMP10:%.*]] = xor <vscale x 4 x i1> [[TMP6]], splat (i1 true)
+; CHECK-NEXT: [[TMP27:%.*]] = call i1 @llvm.vector.reduce.or.nxv4i1(<vscale x 4 x i1> [[TMP10]])
+; CHECK-NEXT: [[BROADCAST_SPLATINSERT2:%.*]] = insertelement <vscale x 4 x i1> poison, i1 [[TMP27]], i64 0
+; CHECK-NEXT: [[BROADCAST_SPLAT3:%.*]] = shufflevector <vscale x 4 x i1> [[BROADCAST_SPLATINSERT2]], <vscale x 4 x i1> poison, <vscale x 4 x i32> zeroinitializer
+; CHECK-NEXT: [[TMP34:%.*]] = or <vscale x 4 x i1> [[TMP21]], [[BROADCAST_SPLAT3]]
+; CHECK-NEXT: [[UNCOUNTABLE_EXIT_MASK2:%.*]] = call <vscale x 4 x i1> @llvm.mask.beforefirst.nxv4i1(<vscale x 4 x i1> [[TMP34]])
; CHECK-NEXT: [[LD_ADDR:%.*]] = getelementptr i16, ptr [[LOAD_ARRAY]], i64 [[IV]]
+; CHECK-NEXT: [[TMP32:%.*]] = getelementptr i16, ptr [[LD_ADDR]], i64 [[TMP2]]
; CHECK-NEXT: [[WIDE_MASKED_LOAD:%.*]] = call <vscale x 4 x i16> @llvm.masked.load.nxv4i16.p0(ptr align 2 [[LD_ADDR]], <vscale x 4 x i1> [[TMP6]], <vscale x 4 x i16> poison)
+; CHECK-NEXT: [[WIDE_MASKED_LOAD3:%.*]] = call <vscale x 4 x i16> @llvm.masked.load.nxv4i16.p0(ptr align 2 [[TMP32]], <vscale x 4 x i1> [[UNCOUNTABLE_EXIT_MASK2]], <vscale x 4 x i16> poison)
; CHECK-NEXT: [[TMP8:%.*]] = add nsw <vscale x 4 x i16> [[WIDE_MASKED_LOAD]], splat (i16 1)
+; CHECK-NEXT: [[TMP33:%.*]] = add nsw <vscale x 4 x i16> [[WIDE_MASKED_LOAD3]], splat (i16 1)
; CHECK-NEXT: [[TMP9:%.*]] = getelementptr i16, ptr [[ARRAY]], i64 [[IV]]
+; CHECK-NEXT: [[TMP17:%.*]] = getelementptr i16, ptr [[TMP9]], i64 [[TMP2]]
; CHECK-NEXT: call void @llvm.masked.store.nxv4i16.p0(<vscale x 4 x i16> [[TMP8]], ptr align 2 [[TMP9]], <vscale x 4 x i1> [[TMP6]])
-; CHECK-NEXT: [[TMP11:%.*]] = call i1 @llvm.vector.reduce.or.nxv4i1(<vscale x 4 x i1> [[TMP4]])
+; CHECK-NEXT: call void @llvm.masked.store.nxv4i16.p0(<vscale x 4 x i16> [[TMP33]], ptr align 2 [[TMP17]], <vscale x 4 x i1> [[UNCOUNTABLE_EXIT_MASK2]])
+; CHECK-NEXT: [[TMP18:%.*]] = or <vscale x 4 x i1> [[TMP4]], [[TMP21]]
+; CHECK-NEXT: [[TMP11:%.*]] = call i1 @llvm.vector.reduce.or.nxv4i1(<vscale x 4 x i1> [[TMP18]])
; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[IV]], [[TMP1]]
; CHECK-NEXT: [[TMP12:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
+; CHECK-NEXT: [[VEC_IND_NEXT]] = add nuw nsw <vscale x 4 x i64> [[STEP_ADD]], [[BROADCAST_SPLAT]]
; CHECK-NEXT: [[TMP13:%.*]] = or i1 [[TMP11]], [[TMP12]]
; CHECK-NEXT: br i1 [[TMP13]], label %[[MIDDLE_BLOCK:.*]], label %[[FOR_BODY]], !llvm.loop [[LOOP6:![0-9]+]]
; CHECK: [[MIDDLE_BLOCK]]:
+; CHECK-NEXT: [[TMP22:%.*]] = call i64 @llvm.vscale.i64()
+; CHECK-NEXT: [[TMP23:%.*]] = mul nuw i64 [[TMP22]], 4
+; CHECK-NEXT: [[TMP24:%.*]] = call i64 @llvm.experimental.cttz.elts.i64.nxv4i1(<vscale x 4 x i1> [[TMP21]], i1 false)
+; CHECK-NEXT: [[TMP25:%.*]] = mul i64 [[TMP23]], 1
+; CHECK-NEXT: [[TMP26:%.*]] = add i64 [[TMP25]], [[TMP24]]
; CHECK-NEXT: [[TMP7:%.*]] = call i64 @llvm.experimental.cttz.elts.i64.nxv4i1(<vscale x 4 x i1> [[TMP4]], i1 false)
-; CHECK-NEXT: [[TMP14:%.*]] = add i64 [[IV]], [[TMP7]]
+; CHECK-NEXT: [[TMP28:%.*]] = mul i64 [[TMP23]], 0
+; CHECK-NEXT: [[TMP29:%.*]] = add i64 [[TMP28]], [[TMP7]]
+; CHECK-NEXT: [[TMP30:%.*]] = icmp ne i64 [[TMP7]], [[TMP23]]
+; CHECK-NEXT: [[TMP31:%.*]] = select i1 [[TMP30]], i64 [[TMP29]], i64 [[TMP26]]
+; CHECK-NEXT: [[TMP42:%.*]] = call i64 @llvm.vscale.i64()
+; CHECK-NEXT: [[TMP43:%.*]] = mul nuw i64 [[TMP42]], 4
+; CHECK-NEXT: [[TMP35:%.*]] = mul i64 [[TMP43]], 0
+; CHECK-NEXT: [[TMP36:%.*]] = extractelement <vscale x 4 x i64> [[VEC_IND]], i64 0
+; CHECK-NEXT: [[TMP37:%.*]] = mul i64 [[TMP43]], 1
+; CHECK-NEXT: [[TMP38:%.*]] = sub i64 0, [[TMP37]]
+; CHECK-NEXT: [[TMP39:%.*]] = extractelement <vscale x 4 x i64> [[STEP_ADD]], i64 [[TMP38]]
+; CHECK-NEXT: [[TMP40:%.*]] = icmp uge i64 0, [[TMP37]]
+; CHECK-NEXT: [[TMP41:%.*]] = select i1 [[TMP40]], i64 [[TMP39]], i64 [[TMP36]]
+; CHECK-NEXT: [[TMP14:%.*]] = add i64 [[TMP41]], [[TMP31]]
; CHECK-NEXT: [[TMP15:%.*]] = icmp eq i64 [[TMP14]], 20
; CHECK-NEXT: br i1 [[TMP15]], label %[[EXIT:.*]], label %[[SCALAR_PH1]]
; CHECK: [[SCALAR_PH1]]:
@@ -487,32 +608,72 @@ define void @loop_contains_store_in_latch_block(ptr dereferenceable(40) noalias
; CHECK-SAME: ptr noalias dereferenceable(40) [[ARRAY:%.*]], ptr readonly align 2 dereferenceable(40) [[PRED:%.*]]) #[[ATTR0]] {
; CHECK-NEXT: [[SCALAR_PH1:.*]]:
; CHECK-NEXT: [[TMP0:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-NEXT: [[TMP1:%.*]] = shl nuw i64 [[TMP0]], 2
+; CHECK-NEXT: [[TMP1:%.*]] = shl nuw i64 [[TMP0]], 3
; CHECK-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 20, [[TMP1]]
; CHECK-NEXT: br i1 [[MIN_ITERS_CHECK]], label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]]
; CHECK: [[VECTOR_PH]]:
+; CHECK-NEXT: [[TMP2:%.*]] = shl nuw i64 [[TMP0]], 2
+; CHECK-NEXT: [[BROADCAST_SPLATINSERT:%.*]] = insertelement <vscale x 4 x i64> poison, i64 [[TMP2]], i64 0
+; CHECK-NEXT: [[BROADCAST_SPLAT:%.*]] = shufflevector <vscale x 4 x i64> [[BROADCAST_SPLATINSERT]], <vscale x 4 x i64> poison, <vscale x 4 x i32> zeroinitializer
; CHECK-NEXT: [[N_MOD_VF:%.*]] = urem i64 20, [[TMP1]]
; CHECK-NEXT: [[N_VEC:%.*]] = sub i64 20, [[N_MOD_VF]]
+; CHECK-NEXT: [[TMP17:%.*]] = call <vscale x 4 x i64> @llvm.stepvector.nxv4i64()
; CHECK-NEXT: br label %[[FOR_BODY1:.*]]
; CHECK: [[FOR_BODY1]]:
; CHECK-NEXT: [[IV1:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[FOR_BODY1]] ]
+; CHECK-NEXT: [[VEC_IND:%.*]] = phi <vscale x 4 x i64> [ [[TMP17]], %[[VECTOR_PH]] ], [ [[VEC_IND_NEXT:%.*]], %[[FOR_BODY1]] ]
+; CHECK-NEXT: [[STEP_ADD:%.*]] = add nuw <vscale x 4 x i64> [[VEC_IND]], [[BROADCAST_SPLAT]]
; CHECK-NEXT: [[EE_ADDR1:%.*]] = getelementptr inbounds nuw i16, ptr [[PRED]], i64 [[IV1]]
+; CHECK-NEXT: [[TMP18:%.*]] = getelementptr inbounds nuw i16, ptr [[EE_ADDR1]], i64 [[TMP2]]
; CHECK-NEXT: [[WIDE_LOAD:%.*]] = load <vscale x 4 x i16>, ptr [[EE_ADDR1]], align 2
+; CHECK-NEXT: [[WIDE_LOAD1:%.*]] = load <vscale x 4 x i16>, ptr [[TMP18]], align 2
; CHECK-NEXT: [[TMP3:%.*]] = icmp sgt <vscale x 4 x i16> [[WIDE_LOAD]], splat (i16 500)
+; CHECK-NEXT: [[TMP19:%.*]] = icmp sgt <vscale x 4 x i16> [[WIDE_LOAD1]], splat (i16 500)
; CHECK-NEXT: [[TMP4:%.*]] = freeze <vscale x 4 x i1> [[TMP3]]
+; CHECK-NEXT: [[TMP9:%.*]] = freeze <vscale x 4 x i1> [[TMP19]]
; CHECK-NEXT: [[TMP5:%.*]] = call <vscale x 4 x i1> @llvm.mask.beforefirst.nxv4i1(<vscale x 4 x i1> [[TMP4]])
+; CHECK-NEXT: [[TMP25:%.*]] = xor <vscale x 4 x i1> [[TMP5]], splat (i1 true)
+; CHECK-NEXT: [[TMP30:%.*]] = call i1 @llvm.vector.reduce.or.nxv4i1(<vscale x 4 x i1> [[TMP25]])
+; CHECK-NEXT: [[BROADCAST_SPLATINSERT2:%.*]] = insertelement <vscale x 4 x i1> poison, i1 [[TMP30]], i64 0
+; CHECK-NEXT: [[BROADCAST_SPLAT3:%.*]] = shufflevector <vscale x 4 x i1> [[BROADCAST_SPLATINSERT2]], <vscale x 4 x i1> poison, <vscale x 4 x i32> zeroinitializer
+; CHECK-NEXT: [[TMP32:%.*]] = or <vscale x 4 x i1> [[TMP9]], [[BROADCAST_SPLAT3]]
+; CHECK-NEXT: [[UNCOUNTABLE_EXIT_MASK2:%.*]] = call <vscale x 4 x i1> @llvm.mask.beforefirst.nxv4i1(<vscale x 4 x i1> [[TMP32]])
; CHECK-NEXT: [[TMP7:%.*]] = getelementptr i16, ptr [[ARRAY]], i64 [[IV1]]
+; CHECK-NEXT: [[TMP31:%.*]] = getelementptr i16, ptr [[TMP7]], i64 [[TMP2]]
; CHECK-NEXT: [[WIDE_MASKED_LOAD:%.*]] = call <vscale x 4 x i16> @llvm.masked.load.nxv4i16.p0(ptr align 2 [[TMP7]], <vscale x 4 x i1> [[TMP5]], <vscale x 4 x i16> poison)
+; CHECK-NEXT: [[WIDE_MASKED_LOAD3:%.*]] = call <vscale x 4 x i16> @llvm.masked.load.nxv4i16.p0(ptr align 2 [[TMP31]], <vscale x 4 x i1> [[UNCOUNTABLE_EXIT_MASK2]], <vscale x 4 x i16> poison)
; CHECK-NEXT: [[TMP8:%.*]] = add nsw <vscale x 4 x i16> [[WIDE_MASKED_LOAD]], splat (i16 1)
+; CHECK-NEXT: [[TMP15:%.*]] = add nsw <vscale x 4 x i16> [[WIDE_MASKED_LOAD3]], splat (i16 1)
; CHECK-NEXT: call void @llvm.masked.store.nxv4i16.p0(<vscale x 4 x i16> [[TMP8]], ptr align 2 [[TMP7]], <vscale x 4 x i1> [[TMP5]])
-; CHECK-NEXT: [[TMP10:%.*]] = call i1 @llvm.vector.reduce.or.nxv4i1(<vscale x 4 x i1> [[TMP4]])
+; CHECK-NEXT: call void @llvm.masked.store.nxv4i16.p0(<vscale x 4 x i16> [[TMP15]], ptr align 2 [[TMP31]], <vscale x 4 x i1> [[UNCOUNTABLE_EXIT_MASK2]])
+; CHECK-NEXT: [[TMP16:%.*]] = or <vscale x 4 x i1> [[TMP4]], [[TMP9]]
+; CHECK-NEXT: [[TMP10:%.*]] = call i1 @llvm.vector.reduce.or.nxv4i1(<vscale x 4 x i1> [[TMP16]])
; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[IV1]], [[TMP1]]
; CHECK-NEXT: [[TMP11:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
+; CHECK-NEXT: [[VEC_IND_NEXT]] = add nuw nsw <vscale x 4 x i64> [[STEP_ADD]], [[BROADCAST_SPLAT]]
; CHECK-NEXT: [[TMP12:%.*]] = or i1 [[TMP10]], [[TMP11]]
; CHECK-NEXT: br i1 [[TMP12]], label %[[MIDDLE_BLOCK:.*]], label %[[FOR_BODY1]], !llvm.loop [[LOOP8:![0-9]+]]
; CHECK: [[MIDDLE_BLOCK]]:
+; CHECK-NEXT: [[TMP20:%.*]] = call i64 @llvm.vscale.i64()
+; CHECK-NEXT: [[TMP21:%.*]] = mul nuw i64 [[TMP20]], 4
+; CHECK-NEXT: [[TMP22:%.*]] = call i64 @llvm.experimental.cttz.elts.i64.nxv4i1(<vscale x 4 x i1> [[TMP9]], i1 false)
+; CHECK-NEXT: [[TMP23:%.*]] = mul i64 [[TMP21]], 1
+; CHECK-NEXT: [[TMP24:%.*]] = add i64 [[TMP23]], [[TMP22]]
; CHECK-NEXT: [[TMP6:%.*]] = call i64 @llvm.experimental.cttz.elts.i64.nxv4i1(<vscale x 4 x i1> [[TMP4]], i1 false)
-; CHECK-NEXT: [[TMP13:%.*]] = add i64 [[IV1]], [[TMP6]]
+; CHECK-NEXT: [[TMP26:%.*]] = mul i64 [[TMP21]], 0
+; CHECK-NEXT: [[TMP27:%.*]] = add i64 [[TMP26]], [[TMP6]]
+; CHECK-NEXT: [[TMP28:%.*]] = icmp ne i64 [[TMP6]], [[TMP21]]
+; CHECK-NEXT: [[TMP29:%.*]] = select i1 [[TMP28]], i64 [[TMP27]], i64 [[TMP24]]
+; CHECK-NEXT: [[TMP40:%.*]] = call i64 @llvm.vscale.i64()
+; CHECK-NEXT: [[TMP41:%.*]] = mul nuw i64 [[TMP40]], 4
+; CHECK-NEXT: [[TMP33:%.*]] = mul i64 [[TMP41]], 0
+; CHECK-NEXT: [[TMP34:%.*]] = extractelement <vscale x 4 x i64> [[VEC_IND]], i64 0
+; CHECK-NEXT: [[TMP35:%.*]] = mul i64 [[TMP41]], 1
+; CHECK-NEXT: [[TMP36:%.*]] = sub i64 0, [[TMP35]]
+; CHECK-NEXT: [[TMP37:%.*]] = extractelement <vscale x 4 x i64> [[STEP_ADD]], i64 [[TMP36]]
+; CHECK-NEXT: [[TMP38:%.*]] = icmp uge i64 0, [[TMP35]]
+; CHECK-NEXT: [[TMP39:%.*]] = select i1 [[TMP38]], i64 [[TMP37]], i64 [[TMP34]]
+; CHECK-NEXT: [[TMP13:%.*]] = add i64 [[TMP39]], [[TMP29]]
; CHECK-NEXT: [[TMP14:%.*]] = icmp eq i64 [[TMP13]], 20
; CHECK-NEXT: br i1 [[TMP14]], label %[[EXIT:.*]], label %[[SCALAR_PH]]
; CHECK: [[SCALAR_PH]]:
@@ -1159,32 +1320,72 @@ define i32 @uncountable_exit_with_separate_exit_block(ptr dereferenceable(40) no
; CHECK-SAME: ptr noalias dereferenceable(40) [[ARRAY:%.*]], ptr readonly align 2 dereferenceable(40) [[PRED:%.*]]) #[[ATTR0]] {
; CHECK-NEXT: [[SCALAR_PH:.*]]:
; CHECK-NEXT: [[TMP0:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-NEXT: [[TMP1:%.*]] = shl nuw i64 [[TMP0]], 2
+; CHECK-NEXT: [[TMP1:%.*]] = shl nuw i64 [[TMP0]], 3
; CHECK-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 20, [[TMP1]]
; CHECK-NEXT: br i1 [[MIN_ITERS_CHECK]], label %[[SCALAR_PH1:.*]], label %[[VECTOR_PH:.*]]
; CHECK: [[VECTOR_PH]]:
+; CHECK-NEXT: [[TMP2:%.*]] = shl nuw i64 [[TMP0]], 2
+; CHECK-NEXT: [[BROADCAST_SPLATINSERT:%.*]] = insertelement <vscale x 4 x i64> poison, i64 [[TMP2]], i64 0
+; CHECK-NEXT: [[BROADCAST_SPLAT:%.*]] = shufflevector <vscale x 4 x i64> [[BROADCAST_SPLATINSERT]], <vscale x 4 x i64> poison, <vscale x 4 x i32> zeroinitializer
; CHECK-NEXT: [[N_MOD_VF:%.*]] = urem i64 20, [[TMP1]]
; CHECK-NEXT: [[N_VEC:%.*]] = sub i64 20, [[N_MOD_VF]]
+; CHECK-NEXT: [[TMP17:%.*]] = call <vscale x 4 x i64> @llvm.stepvector.nxv4i64()
; CHECK-NEXT: br label %[[FOR_BODY1:.*]]
; CHECK: [[FOR_BODY1]]:
; CHECK-NEXT: [[IV1:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[FOR_BODY1]] ]
+; CHECK-NEXT: [[VEC_IND:%.*]] = phi <vscale x 4 x i64> [ [[TMP17]], %[[VECTOR_PH]] ], [ [[VEC_IND_NEXT:%.*]], %[[FOR_BODY1]] ]
+; CHECK-NEXT: [[STEP_ADD:%.*]] = add nuw <vscale x 4 x i64> [[VEC_IND]], [[BROADCAST_SPLAT]]
; CHECK-NEXT: [[TMP5:%.*]] = getelementptr inbounds nuw i16, ptr [[PRED]], i64 [[IV1]]
+; CHECK-NEXT: [[TMP18:%.*]] = getelementptr inbounds nuw i16, ptr [[TMP5]], i64 [[TMP2]]
; CHECK-NEXT: [[WIDE_LOAD:%.*]] = load <vscale x 4 x i16>, ptr [[TMP5]], align 2
+; CHECK-NEXT: [[WIDE_LOAD1:%.*]] = load <vscale x 4 x i16>, ptr [[TMP18]], align 2
; CHECK-NEXT: [[TMP3:%.*]] = icmp sgt <vscale x 4 x i16> [[WIDE_LOAD]], splat (i16 500)
+; CHECK-NEXT: [[TMP19:%.*]] = icmp sgt <vscale x 4 x i16> [[WIDE_LOAD1]], splat (i16 500)
; CHECK-NEXT: [[TMP4:%.*]] = freeze <vscale x 4 x i1> [[TMP3]]
+; CHECK-NEXT: [[TMP9:%.*]] = freeze <vscale x 4 x i1> [[TMP19]]
; CHECK-NEXT: [[TMP6:%.*]] = call <vscale x 4 x i1> @llvm.mask.beforefirst.nxv4i1(<vscale x 4 x i1> [[TMP4]])
+; CHECK-NEXT: [[TMP25:%.*]] = xor <vscale x 4 x i1> [[TMP6]], splat (i1 true)
+; CHECK-NEXT: [[TMP30:%.*]] = call i1 @llvm.vector.reduce.or.nxv4i1(<vscale x 4 x i1> [[TMP25]])
+; CHECK-NEXT: [[BROADCAST_SPLATINSERT2:%.*]] = insertelement <vscale x 4 x i1> poison, i1 [[TMP30]], i64 0
+; CHECK-NEXT: [[BROADCAST_SPLAT3:%.*]] = shufflevector <vscale x 4 x i1> [[BROADCAST_SPLATINSERT2]], <vscale x 4 x i1> poison, <vscale x 4 x i32> zeroinitializer
+; CHECK-NEXT: [[TMP32:%.*]] = or <vscale x 4 x i1> [[TMP9]], [[BROADCAST_SPLAT3]]
+; CHECK-NEXT: [[UNCOUNTABLE_EXIT_MASK2:%.*]] = call <vscale x 4 x i1> @llvm.mask.beforefirst.nxv4i1(<vscale x 4 x i1> [[TMP32]])
; CHECK-NEXT: [[ST_ADDR1:%.*]] = getelementptr i16, ptr [[ARRAY]], i64 [[IV1]]
+; CHECK-NEXT: [[TMP31:%.*]] = getelementptr i16, ptr [[ST_ADDR1]], i64 [[TMP2]]
; CHECK-NEXT: [[WIDE_MASKED_LOAD:%.*]] = call <vscale x 4 x i16> @llvm.masked.load.nxv4i16.p0(ptr align 2 [[ST_ADDR1]], <vscale x 4 x i1> [[TMP6]], <vscale x 4 x i16> poison)
+; CHECK-NEXT: [[WIDE_MASKED_LOAD3:%.*]] = call <vscale x 4 x i16> @llvm.masked.load.nxv4i16.p0(ptr align 2 [[TMP31]], <vscale x 4 x i1> [[UNCOUNTABLE_EXIT_MASK2]], <vscale x 4 x i16> poison)
; CHECK-NEXT: [[TMP8:%.*]] = add nsw <vscale x 4 x i16> [[WIDE_MASKED_LOAD]], splat (i16 1)
+; CHECK-NEXT: [[TMP15:%.*]] = add nsw <vscale x 4 x i16> [[WIDE_MASKED_LOAD3]], splat (i16 1)
; CHECK-NEXT: call void @llvm.masked.store.nxv4i16.p0(<vscale x 4 x i16> [[TMP8]], ptr align 2 [[ST_ADDR1]], <vscale x 4 x i1> [[TMP6]])
-; CHECK-NEXT: [[TMP10:%.*]] = call i1 @llvm.vector.reduce.or.nxv4i1(<vscale x 4 x i1> [[TMP4]])
+; CHECK-NEXT: call void @llvm.masked.store.nxv4i16.p0(<vscale x 4 x i16> [[TMP15]], ptr align 2 [[TMP31]], <vscale x 4 x i1> [[UNCOUNTABLE_EXIT_MASK2]])
+; CHECK-NEXT: [[TMP16:%.*]] = or <vscale x 4 x i1> [[TMP4]], [[TMP9]]
+; CHECK-NEXT: [[TMP10:%.*]] = call i1 @llvm.vector.reduce.or.nxv4i1(<vscale x 4 x i1> [[TMP16]])
; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[IV1]], [[TMP1]]
; CHECK-NEXT: [[TMP11:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
+; CHECK-NEXT: [[VEC_IND_NEXT]] = add nuw nsw <vscale x 4 x i64> [[STEP_ADD]], [[BROADCAST_SPLAT]]
; CHECK-NEXT: [[TMP12:%.*]] = or i1 [[TMP10]], [[TMP11]]
; CHECK-NEXT: br i1 [[TMP12]], label %[[MIDDLE_BLOCK:.*]], label %[[FOR_BODY1]], !llvm.loop [[LOOP10:![0-9]+]]
; CHECK: [[MIDDLE_BLOCK]]:
+; CHECK-NEXT: [[TMP20:%.*]] = call i64 @llvm.vscale.i64()
+; CHECK-NEXT: [[TMP21:%.*]] = mul nuw i64 [[TMP20]], 4
+; CHECK-NEXT: [[TMP22:%.*]] = call i64 @llvm.experimental.cttz.elts.i64.nxv4i1(<vscale x 4 x i1> [[TMP9]], i1 false)
+; CHECK-NEXT: [[TMP23:%.*]] = mul i64 [[TMP21]], 1
+; CHECK-NEXT: [[TMP24:%.*]] = add i64 [[TMP23]], [[TMP22]]
; CHECK-NEXT: [[TMP7:%.*]] = call i64 @llvm.experimental.cttz.elts.i64.nxv4i1(<vscale x 4 x i1> [[TMP4]], i1 false)
-; CHECK-NEXT: [[TMP13:%.*]] = add i64 [[IV1]], [[TMP7]]
+; CHECK-NEXT: [[TMP26:%.*]] = mul i64 [[TMP21]], 0
+; CHECK-NEXT: [[TMP27:%.*]] = add i64 [[TMP26]], [[TMP7]]
+; CHECK-NEXT: [[TMP28:%.*]] = icmp ne i64 [[TMP7]], [[TMP21]]
+; CHECK-NEXT: [[TMP29:%.*]] = select i1 [[TMP28]], i64 [[TMP27]], i64 [[TMP24]]
+; CHECK-NEXT: [[TMP40:%.*]] = call i64 @llvm.vscale.i64()
+; CHECK-NEXT: [[TMP41:%.*]] = mul nuw i64 [[TMP40]], 4
+; CHECK-NEXT: [[TMP33:%.*]] = mul i64 [[TMP41]], 0
+; CHECK-NEXT: [[TMP34:%.*]] = extractelement <vscale x 4 x i64> [[VEC_IND]], i64 0
+; CHECK-NEXT: [[TMP35:%.*]] = mul i64 [[TMP41]], 1
+; CHECK-NEXT: [[TMP36:%.*]] = sub i64 0, [[TMP35]]
+; CHECK-NEXT: [[TMP37:%.*]] = extractelement <vscale x 4 x i64> [[STEP_ADD]], i64 [[TMP36]]
+; CHECK-NEXT: [[TMP38:%.*]] = icmp uge i64 0, [[TMP35]]
+; CHECK-NEXT: [[TMP39:%.*]] = select i1 [[TMP38]], i64 [[TMP37]], i64 [[TMP34]]
+; CHECK-NEXT: [[TMP13:%.*]] = add i64 [[TMP39]], [[TMP29]]
; CHECK-NEXT: [[TMP14:%.*]] = icmp eq i64 [[TMP13]], 20
; CHECK-NEXT: br i1 [[TMP14]], label %[[EXIT_COUNTABLE:.*]], label %[[SCALAR_PH1]]
; CHECK: [[SCALAR_PH1]]:
diff --git a/llvm/test/Transforms/LoopVectorize/interleave_uncountable_exits.ll b/llvm/test/Transforms/LoopVectorize/interleave_uncountable_exits.ll
index d3dae6a8c9651f..0fbca21c9d31da 100644
--- a/llvm/test/Transforms/LoopVectorize/interleave_uncountable_exits.ll
+++ b/llvm/test/Transforms/LoopVectorize/interleave_uncountable_exits.ll
@@ -1,8 +1,6 @@
; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --check-globals none --version 6
; RUN: opt -S < %s -p loop-vectorize -force-vector-width=4 -force-vector-interleave=2 -force-target-max-vector-interleave=2 -force-target-supports-masked-memory-ops -enable-early-exit-vectorization-with-side-effects | FileCheck %s
-; We do not support interleaving loops with early exits yet, so make sure we don't
-; even with a flag to force it.
define void @loop_contains_store_condition_load_has_single_user(ptr dereferenceable(40) noalias %array, ptr align 2 dereferenceable(40) readonly %pred) {
; CHECK-LABEL: define void @loop_contains_store_condition_load_has_single_user(
; CHECK-SAME: ptr noalias dereferenceable(40) [[ARRAY:%.*]], ptr readonly align 2 dereferenceable(40) [[PRED:%.*]]) {
@@ -12,23 +10,49 @@ define void @loop_contains_store_condition_load_has_single_user(ptr dereferencea
; CHECK-NEXT: br label %[[VECTOR_BODY:.*]]
; CHECK: [[VECTOR_BODY]]:
; CHECK-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-NEXT: [[VEC_IND:%.*]] = phi <4 x i64> [ <i64 0, i64 1, i64 2, i64 3>, %[[VECTOR_PH]] ], [ [[VEC_IND_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-NEXT: [[STEP_ADD:%.*]] = add nuw <4 x i64> [[VEC_IND]], splat (i64 4)
; CHECK-NEXT: [[TMP1:%.*]] = getelementptr inbounds nuw i16, ptr [[PRED]], i64 [[INDEX]]
+; CHECK-NEXT: [[TMP13:%.*]] = getelementptr inbounds nuw i16, ptr [[TMP1]], i64 4
; CHECK-NEXT: [[WIDE_LOAD:%.*]] = load <4 x i16>, ptr [[TMP1]], align 2
+; CHECK-NEXT: [[WIDE_LOAD1:%.*]] = load <4 x i16>, ptr [[TMP13]], align 2
; CHECK-NEXT: [[TMP2:%.*]] = icmp sgt <4 x i16> [[WIDE_LOAD]], splat (i16 500)
+; CHECK-NEXT: [[TMP14:%.*]] = icmp sgt <4 x i16> [[WIDE_LOAD1]], splat (i16 500)
; CHECK-NEXT: [[TMP5:%.*]] = freeze <4 x i1> [[TMP2]]
+; CHECK-NEXT: [[TMP15:%.*]] = freeze <4 x i1> [[TMP14]]
; CHECK-NEXT: [[UNCOUNTABLE_EXIT_MASK:%.*]] = call <4 x i1> @llvm.mask.beforefirst.v4i1(<4 x i1> [[TMP5]])
+; CHECK-NEXT: [[TMP21:%.*]] = xor <4 x i1> [[UNCOUNTABLE_EXIT_MASK]], splat (i1 true)
+; CHECK-NEXT: [[TMP22:%.*]] = call i1 @llvm.vector.reduce.or.v4i1(<4 x i1> [[TMP21]])
+; CHECK-NEXT: [[BROADCAST_SPLATINSERT:%.*]] = insertelement <4 x i1> poison, i1 [[TMP22]], i64 0
+; CHECK-NEXT: [[BROADCAST_SPLAT:%.*]] = shufflevector <4 x i1> [[BROADCAST_SPLATINSERT]], <4 x i1> poison, <4 x i32> zeroinitializer
+; CHECK-NEXT: [[TMP24:%.*]] = or <4 x i1> [[TMP15]], [[BROADCAST_SPLAT]]
+; CHECK-NEXT: [[UNCOUNTABLE_EXIT_MASK2:%.*]] = call <4 x i1> @llvm.mask.beforefirst.v4i1(<4 x i1> [[TMP24]])
; CHECK-NEXT: [[TMP0:%.*]] = getelementptr i16, ptr [[ARRAY]], i64 [[INDEX]]
+; CHECK-NEXT: [[TMP23:%.*]] = getelementptr i16, ptr [[TMP0]], i64 4
; CHECK-NEXT: [[WIDE_MASKED_LOAD:%.*]] = call <4 x i16> @llvm.masked.load.v4i16.p0(ptr align 2 [[TMP0]], <4 x i1> [[UNCOUNTABLE_EXIT_MASK]], <4 x i16> poison)
+; CHECK-NEXT: [[WIDE_MASKED_LOAD3:%.*]] = call <4 x i16> @llvm.masked.load.v4i16.p0(ptr align 2 [[TMP23]], <4 x i1> [[UNCOUNTABLE_EXIT_MASK2]], <4 x i16> poison)
; CHECK-NEXT: [[TMP4:%.*]] = add nsw <4 x i16> [[WIDE_MASKED_LOAD]], splat (i16 1)
+; CHECK-NEXT: [[TMP11:%.*]] = add nsw <4 x i16> [[WIDE_MASKED_LOAD3]], splat (i16 1)
; CHECK-NEXT: call void @llvm.masked.store.v4i16.p0(<4 x i16> [[TMP4]], ptr align 2 [[TMP0]], <4 x i1> [[UNCOUNTABLE_EXIT_MASK]])
-; CHECK-NEXT: [[TMP6:%.*]] = call i1 @llvm.vector.reduce.or.v4i1(<4 x i1> [[TMP5]])
-; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 4
-; CHECK-NEXT: [[TMP7:%.*]] = icmp eq i64 [[INDEX_NEXT]], 20
+; CHECK-NEXT: call void @llvm.masked.store.v4i16.p0(<4 x i16> [[TMP11]], ptr align 2 [[TMP23]], <4 x i1> [[UNCOUNTABLE_EXIT_MASK2]])
+; CHECK-NEXT: [[TMP12:%.*]] = or <4 x i1> [[TMP5]], [[TMP15]]
+; CHECK-NEXT: [[TMP6:%.*]] = call i1 @llvm.vector.reduce.or.v4i1(<4 x i1> [[TMP12]])
+; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 8
+; CHECK-NEXT: [[TMP7:%.*]] = icmp eq i64 [[INDEX_NEXT]], 16
+; CHECK-NEXT: [[VEC_IND_NEXT]] = add nuw nsw <4 x i64> [[STEP_ADD]], splat (i64 4)
; CHECK-NEXT: [[TMP8:%.*]] = or i1 [[TMP6]], [[TMP7]]
; CHECK-NEXT: br i1 [[TMP8]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP0:![0-9]+]]
; CHECK: [[MIDDLE_BLOCK]]:
-; CHECK-NEXT: [[TMP3:%.*]] = call i64 @llvm.experimental.cttz.elts.i64.v4i1(<4 x i1> [[TMP5]], i1 false)
-; CHECK-NEXT: [[TMP9:%.*]] = add i64 [[INDEX]], [[TMP3]]
+; CHECK-NEXT: [[TMP16:%.*]] = call i64 @llvm.experimental.cttz.elts.i64.v4i1(<4 x i1> [[TMP15]], i1 false)
+; CHECK-NEXT: [[TMP17:%.*]] = add i64 4, [[TMP16]]
+; CHECK-NEXT: [[TMP18:%.*]] = call i64 @llvm.experimental.cttz.elts.i64.v4i1(<4 x i1> [[TMP5]], i1 false)
+; CHECK-NEXT: [[TMP19:%.*]] = add i64 0, [[TMP18]]
+; CHECK-NEXT: [[TMP20:%.*]] = icmp ne i64 [[TMP18]], 4
+; CHECK-NEXT: [[TMP3:%.*]] = select i1 [[TMP20]], i64 [[TMP19]], i64 [[TMP17]]
+; CHECK-NEXT: [[TMP26:%.*]] = extractelement <4 x i64> [[VEC_IND]], i64 0
+; CHECK-NEXT: [[TMP27:%.*]] = extractelement <4 x i64> [[STEP_ADD]], i64 -4
+; CHECK-NEXT: [[TMP25:%.*]] = select i1 false, i64 [[TMP27]], i64 [[TMP26]]
+; CHECK-NEXT: [[TMP9:%.*]] = add i64 [[TMP25]], [[TMP3]]
; CHECK-NEXT: [[TMP10:%.*]] = icmp eq i64 [[TMP9]], 20
; CHECK-NEXT: br i1 [[TMP10]], label %[[EXIT:.*]], label %[[SCALAR_PH:.*]]
; CHECK: [[SCALAR_PH]]:
diff --git a/llvm/test/Transforms/PhaseOrdering/AArch64/countable-and-uncountable-exits-combined.ll b/llvm/test/Transforms/PhaseOrdering/AArch64/countable-and-uncountable-exits-combined.ll
index 9d5804468f12b7..af05a83af92318 100644
--- a/llvm/test/Transforms/PhaseOrdering/AArch64/countable-and-uncountable-exits-combined.ll
+++ b/llvm/test/Transforms/PhaseOrdering/AArch64/countable-and-uncountable-exits-combined.ll
@@ -27,33 +27,64 @@ define void @foo() #0 {
; CHECK-SAME: ) local_unnamed_addr #[[ATTR0:[0-9]+]] {
; CHECK-NEXT: [[ENTRY:.*]]:
; CHECK-NEXT: [[TMP0:%.*]] = tail call i64 @llvm.vscale.i64()
-; CHECK-NEXT: [[TMP1:%.*]] = shl nuw i64 [[TMP0]], 2
-; CHECK-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ugt i64 [[TMP0]], 2500
+; CHECK-NEXT: [[TMP1:%.*]] = shl nuw i64 [[TMP0]], 3
+; CHECK-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ugt i64 [[TMP0]], 1250
; CHECK-NEXT: br i1 [[MIN_ITERS_CHECK]], label %[[FOR_BODY_PREHEADER:.*]], label %[[VECTOR_PH:.*]]
; CHECK: [[VECTOR_PH]]:
+; CHECK-NEXT: [[TMP13:%.*]] = shl nuw nsw i64 [[TMP0]], 2
; CHECK-NEXT: [[N_MOD_VF:%.*]] = urem i64 10000, [[TMP1]]
; CHECK-NEXT: [[N_VEC:%.*]] = sub nuw nsw i64 10000, [[N_MOD_VF]]
+; CHECK-NEXT: [[TMP30:%.*]] = tail call <vscale x 4 x i64> @llvm.stepvector.nxv4i64()
+; CHECK-NEXT: [[DOTSCALAR:%.*]] = shl nuw nsw i64 [[TMP0]], 3
+; CHECK-NEXT: [[TMP31:%.*]] = insertelement <vscale x 4 x i64> poison, i64 [[DOTSCALAR]], i64 0
+; CHECK-NEXT: [[REASS_ADD:%.*]] = shufflevector <vscale x 4 x i64> [[TMP31]], <vscale x 4 x i64> poison, <vscale x 4 x i32> zeroinitializer
; CHECK-NEXT: br label %[[FOR_BODY:.*]]
; CHECK: [[FOR_BODY]]:
; CHECK-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[FOR_BODY]] ]
+; CHECK-NEXT: [[VEC_IND:%.*]] = phi <vscale x 4 x i64> [ [[TMP30]], %[[VECTOR_PH]] ], [ [[VEC_IND_NEXT:%.*]], %[[FOR_BODY]] ]
; CHECK-NEXT: [[TMP2:%.*]] = getelementptr inbounds nuw [4 x i8], ptr @c, i64 [[INDEX]]
+; CHECK-NEXT: [[TMP17:%.*]] = getelementptr inbounds nuw [4 x i8], ptr [[TMP2]], i64 [[TMP13]]
; CHECK-NEXT: [[WIDE_LOAD:%.*]] = load <vscale x 4 x i32>, ptr [[TMP2]], align 4
+; CHECK-NEXT: [[WIDE_LOAD3:%.*]] = load <vscale x 4 x i32>, ptr [[TMP17]], align 4
; CHECK-NEXT: [[WIDE_LOAD_FR:%.*]] = freeze <vscale x 4 x i32> [[WIDE_LOAD]]
; CHECK-NEXT: [[TMP3:%.*]] = icmp ne <vscale x 4 x i32> [[WIDE_LOAD_FR]], zeroinitializer
+; CHECK-NEXT: [[WIDE_LOAD3_FR:%.*]] = freeze <vscale x 4 x i32> [[WIDE_LOAD3]]
+; CHECK-NEXT: [[TMP18:%.*]] = icmp ne <vscale x 4 x i32> [[WIDE_LOAD3_FR]], zeroinitializer
; CHECK-NEXT: [[UNCOUNTABLE_EXIT_MASK:%.*]] = tail call <vscale x 4 x i1> @llvm.mask.beforefirst.nxv4i1(<vscale x 4 x i1> [[TMP3]])
+; CHECK-NEXT: [[TMP19:%.*]] = xor <vscale x 4 x i1> [[UNCOUNTABLE_EXIT_MASK]], splat (i1 true)
+; CHECK-NEXT: [[TMP24:%.*]] = tail call i1 @llvm.vector.reduce.or.nxv4i1(<vscale x 4 x i1> [[TMP19]])
+; CHECK-NEXT: [[BROADCAST_SPLATINSERT4:%.*]] = insertelement <vscale x 4 x i1> poison, i1 [[TMP24]], i64 0
+; CHECK-NEXT: [[BROADCAST_SPLAT5:%.*]] = shufflevector <vscale x 4 x i1> [[BROADCAST_SPLATINSERT4]], <vscale x 4 x i1> poison, <vscale x 4 x i32> zeroinitializer
+; CHECK-NEXT: [[TMP28:%.*]] = or <vscale x 4 x i1> [[TMP18]], [[BROADCAST_SPLAT5]]
+; CHECK-NEXT: [[UNCOUNTABLE_EXIT_MASK4:%.*]] = tail call <vscale x 4 x i1> @llvm.mask.beforefirst.nxv4i1(<vscale x 4 x i1> [[TMP28]])
; CHECK-NEXT: [[TMP5:%.*]] = getelementptr [4 x i8], ptr @src, i64 [[INDEX]]
+; CHECK-NEXT: [[TMP26:%.*]] = getelementptr [4 x i8], ptr [[TMP5]], i64 [[TMP13]]
; CHECK-NEXT: [[WIDE_MASKED_LOAD:%.*]] = tail call <vscale x 4 x i32> @llvm.masked.load.nxv4i32.p0(ptr align 4 [[TMP5]], <vscale x 4 x i1> [[UNCOUNTABLE_EXIT_MASK]], <vscale x 4 x i32> poison)
+; CHECK-NEXT: [[WIDE_MASKED_LOAD5:%.*]] = tail call <vscale x 4 x i32> @llvm.masked.load.nxv4i32.p0(ptr align 4 [[TMP26]], <vscale x 4 x i1> [[UNCOUNTABLE_EXIT_MASK4]], <vscale x 4 x i32> poison)
; CHECK-NEXT: [[TMP6:%.*]] = add nsw <vscale x 4 x i32> [[WIDE_MASKED_LOAD]], splat (i32 42)
+; CHECK-NEXT: [[TMP27:%.*]] = add nsw <vscale x 4 x i32> [[WIDE_MASKED_LOAD5]], splat (i32 42)
; CHECK-NEXT: [[TMP7:%.*]] = getelementptr [4 x i8], ptr @dst, i64 [[INDEX]]
+; CHECK-NEXT: [[TMP14:%.*]] = getelementptr [4 x i8], ptr [[TMP7]], i64 [[TMP13]]
; CHECK-NEXT: tail call void @llvm.masked.store.nxv4i32.p0(<vscale x 4 x i32> [[TMP6]], ptr align 4 [[TMP7]], <vscale x 4 x i1> [[UNCOUNTABLE_EXIT_MASK]])
-; CHECK-NEXT: [[TMP8:%.*]] = tail call i1 @llvm.vector.reduce.or.nxv4i1(<vscale x 4 x i1> [[TMP3]])
+; CHECK-NEXT: tail call void @llvm.masked.store.nxv4i32.p0(<vscale x 4 x i32> [[TMP27]], ptr align 4 [[TMP14]], <vscale x 4 x i1> [[UNCOUNTABLE_EXIT_MASK4]])
+; CHECK-NEXT: [[TMP15:%.*]] = or <vscale x 4 x i32> [[WIDE_LOAD_FR]], [[WIDE_LOAD3_FR]]
+; CHECK-NEXT: [[TMP16:%.*]] = icmp ne <vscale x 4 x i32> [[TMP15]], zeroinitializer
+; CHECK-NEXT: [[TMP8:%.*]] = tail call i1 @llvm.vector.reduce.or.nxv4i1(<vscale x 4 x i1> [[TMP16]])
; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], [[TMP1]]
; CHECK-NEXT: [[TMP9:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
+; CHECK-NEXT: [[VEC_IND_NEXT]] = add <vscale x 4 x i64> [[VEC_IND]], [[REASS_ADD]]
; CHECK-NEXT: [[TMP10:%.*]] = or i1 [[TMP8]], [[TMP9]]
; CHECK-NEXT: br i1 [[TMP10]], label %[[MIDDLE_BLOCK:.*]], label %[[FOR_BODY]], !llvm.loop [[LOOP0:![0-9]+]]
; CHECK: [[MIDDLE_BLOCK]]:
+; CHECK-NEXT: [[TMP20:%.*]] = tail call i64 @llvm.vscale.i64()
+; CHECK-NEXT: [[TMP21:%.*]] = shl nuw i64 [[TMP20]], 2
+; CHECK-NEXT: [[TMP22:%.*]] = tail call i64 @llvm.experimental.cttz.elts.i64.nxv4i1(<vscale x 4 x i1> [[TMP18]], i1 false)
+; CHECK-NEXT: [[TMP23:%.*]] = add i64 [[TMP21]], [[TMP22]]
; CHECK-NEXT: [[TMP4:%.*]] = tail call i64 @llvm.experimental.cttz.elts.i64.nxv4i1(<vscale x 4 x i1> [[TMP3]], i1 false)
-; CHECK-NEXT: [[TMP11:%.*]] = add i64 [[INDEX]], [[TMP4]]
+; CHECK-NEXT: [[DOTNOT:%.*]] = icmp eq i64 [[TMP4]], [[TMP21]]
+; CHECK-NEXT: [[TMP25:%.*]] = select i1 [[DOTNOT]], i64 [[TMP23]], i64 [[TMP4]]
+; CHECK-NEXT: [[TMP29:%.*]] = extractelement <vscale x 4 x i64> [[VEC_IND]], i64 0
+; CHECK-NEXT: [[TMP11:%.*]] = add i64 [[TMP29]], [[TMP25]]
; CHECK-NEXT: [[TMP12:%.*]] = icmp eq i64 [[TMP11]], 10000
; CHECK-NEXT: br i1 [[TMP12]], label %[[EXIT:.*]], label %[[FOR_BODY_PREHEADER]]
; CHECK: [[FOR_BODY_PREHEADER]]:
More information about the llvm-branch-commits
mailing list