[llvm-branch-commits] [llvm] [VPlan] Support tail folding uncountable early exits (PR #238024)
Luke Lau via llvm-branch-commits
llvm-branch-commits at lists.llvm.org
Thu Nov 12 01:26:44 PST 2026
https://github.com/lukel97 updated https://github.com/llvm/llvm-project/pull/238024
>From ba2c36d0f105840152313b2241b6ee2fe0cede80 Mon Sep 17 00:00:00 2001
From: Luke Lau <luke at igalia.com>
Date: Mon, 27 Jul 2026 22:58:50 +0800
Subject: [PATCH 01/15] [VPlan] Use llvm.mask.beforefirst for computing early
exit mask
This makes the cost more accurate and improves codegen for RISC-V. The codegen for an early exit loop with side effects then goes from:
.LBB0_3: # %vector.body
# =>This Inner Loop Header: Depth=1
vl1r.v v16, (a3)
vsetvli zero, zero, e8, m1, ta, ma
vmseq.vi v16, v16, 0
vfirst.m a7, v16
srli t0, a7, 63
czero.nez a7, a7, t0
czero.eqz t0, a1, t0
or a7, t0, a7
vsetvli zero, zero, e64, m8, ta, ma
vmsltu.vx v0, v8, a7
vle8.v v17, (a5), v0.t
vcpop.m t1, v16
vsetvli zero, zero, e8, m1, ta, ma
vadd.vi v16, v17, 1
vse8.v v16, (a5), v0.t
sub a4, a4, a2
bnez t1, .LBB0_5
To
.LBB0_3: # %vector.body
# =>This Inner Loop Header: Depth=1
vl1r.v v8, (a4)
vmseq.vi v8, v8, 0
vmsbf.m v0, v8
vle8.v v9, (a5), v0.t
vcpop.m t0, v8
vadd.vi v9, v9, 1
vse8.v v9, (a5), v0.t
sub a2, a2, a3
bnez t0, .LBB0_5
Some AArch64 loops now use a smaller VF because the early exit mask is now more accurately costed as cheaper.
This adds a dedicated VPInstruction instead of VPWidenIntrinsic so that it can be unrolled later on.
---
llvm/lib/Transforms/Vectorize/VPlan.h | 3 +
.../lib/Transforms/Vectorize/VPlanRecipes.cpp | 21 ++++
.../Transforms/Vectorize/VPlanTransforms.cpp | 14 ++-
.../AArch64/early_exit_with_stores.ll | 100 +++++++++---------
.../RISCV/early_exit_with_stores.ll | 46 ++++----
.../VPlan/early_exit_with_stores_vplan.ll | 80 +++++++-------
.../X86/early_exit_with_stores.ll | 8 +-
.../early_exit_combined_exits.ll | 16 +--
.../early_exit_combined_exits_epilogue.ll | 4 +-
.../LoopVectorize/early_exit_with_stores.ll | 36 +++----
.../interleave_uncountable_exits.ll | 4 +-
...rized_conditional_ops_uncountable_exits.ll | 4 +-
.../tail_fold_uncountable_exits.ll | 4 +-
...ountable-and-uncountable-exits-combined.ll | 4 +-
14 files changed, 183 insertions(+), 161 deletions(-)
diff --git a/llvm/lib/Transforms/Vectorize/VPlan.h b/llvm/lib/Transforms/Vectorize/VPlan.h
index d27473af8a7ca5..b9c2e7e7f88739 100644
--- a/llvm/lib/Transforms/Vectorize/VPlan.h
+++ b/llvm/lib/Transforms/Vectorize/VPlan.h
@@ -1390,6 +1390,9 @@ class LLVM_ABI_FOR_TEST VPInstruction : public VPRecipeWithIRFlags,
LogicalAnd, // Non-poison propagating logical And.
LogicalOr, // Non-poison propagating logical Or.
NumActiveLanes, // Counts the number of active lanes in a mask.
+ // Returns a mask of all the lanes before the first active lane, i.e.
+ // @llvm.mask.beforefirst.
+ MaskBeforeFirst,
// Add an offset in bytes (second operand) to a base pointer (first
// operand). Only generates scalar values (either for the first lane only or
// for all lanes, depending on its uses).
diff --git a/llvm/lib/Transforms/Vectorize/VPlanRecipes.cpp b/llvm/lib/Transforms/Vectorize/VPlanRecipes.cpp
index 0b89ff55803edd..63d4fbeb1c753c 100644
--- a/llvm/lib/Transforms/Vectorize/VPlanRecipes.cpp
+++ b/llvm/lib/Transforms/Vectorize/VPlanRecipes.cpp
@@ -538,6 +538,7 @@ Type *llvm::computeScalarTypeForInstruction(unsigned Opcode,
AssertOperandType(1, Op0Ty);
return IntegerType::get(Ctx, 1);
case VPInstruction::MaskedCond:
+ case VPInstruction::MaskBeforeFirst:
assert(Op0Ty->isIntegerTy(1) && "expected bool operand");
return IntegerType::get(Ctx, 1);
case VPInstruction::LogicalAnd:
@@ -679,6 +680,7 @@ unsigned VPInstruction::getNumOperandsForOpcode() const {
case VPInstruction::ExtractLastLane:
case VPInstruction::ExtractLastPart:
case VPInstruction::ExtractPenultimateElement:
+ case VPInstruction::MaskBeforeFirst:
case VPInstruction::MaskedCond:
case VPInstruction::Not:
case VPInstruction::Reverse:
@@ -894,6 +896,13 @@ Value *VPInstruction::generate(VPTransformState &State,
Builder.CreateUnaryIntrinsic(Intrinsic::vector_reduce_add, ZExt);
return NumActive;
}
+ case VPInstruction::MaskBeforeFirst: {
+ Value *Cond = State.get(getOperand(0));
+ if (State.VF.isScalar())
+ return Builder.CreateNot(Cond, Name);
+ return Builder.CreateUnaryIntrinsic(Intrinsic::mask_beforefirst, Cond, {},
+ Name);
+ }
case VPInstruction::FirstOrderRecurrenceSplice: {
// Generate code to combine the previous and current values in vector v3.
//
@@ -1578,6 +1587,14 @@ InstructionCost VPInstruction::computeCost(ElementCount VF,
{ArgTy, ArgTy});
return Ctx.TTI.getIntrinsicInstrCost(Attrs, Ctx.CostKind);
}
+ case VPInstruction::MaskBeforeFirst: {
+ if (VF.isScalar())
+ return Ctx.TTI.getArithmeticInstrCost(
+ Instruction::Xor, Type::getInt1Ty(Ctx.LLVMCtx), Ctx.CostKind);
+ Type *MaskTy = toVectorTy(Type::getInt1Ty(Ctx.LLVMCtx), VF);
+ IntrinsicCostAttributes Attrs(Intrinsic::mask_beforefirst, MaskTy, MaskTy);
+ return Ctx.TTI.getIntrinsicInstrCost(Attrs, Ctx.CostKind);
+ }
case VPInstruction::ExplicitVectorLength: {
Type *Arg0Ty = getOperand(0)->getScalarType();
Type *I32Ty = Type::getInt32Ty(Ctx.LLVMCtx);
@@ -1802,6 +1819,7 @@ bool VPInstruction::opcodeMayReadOrWriteFromMemory() const {
case VPInstruction::FirstOrderRecurrenceSplice:
case VPInstruction::LogicalAnd:
case VPInstruction::LogicalOr:
+ case VPInstruction::MaskBeforeFirst:
case VPInstruction::MaskedCond:
case VPInstruction::Not:
case VPInstruction::PtrAdd:
@@ -2005,6 +2023,9 @@ void VPInstruction::printRecipe(raw_ostream &O, const Twine &Indent,
case VPInstruction::NumActiveLanes:
O << "num-active-lanes";
break;
+ case VPInstruction::MaskBeforeFirst:
+ O << "mask-before-first";
+ break;
case VPInstruction::WideIVStep:
O << "wide-iv-step";
break;
diff --git a/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp b/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp
index 9ec861e4416226..6dafb81a0237bb 100644
--- a/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp
+++ b/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp
@@ -3485,14 +3485,8 @@ static bool handleUncountableExitsWithSideEffects(
// Create a mask to represent all lanes that fully execute in the vector loop,
// stopping short of any early exit.
VPBuilder MaskBuilder(InsertIt);
- VPValue *FirstActive = MaskBuilder.createFirstActiveLane(Cond);
- Type *IVScalarTy = IV->getScalarType();
- VPValue *Zero = Plan.getZero(IVScalarTy);
- FirstActive =
- MaskBuilder.createScalarZExtOrTrunc(FirstActive, IVScalarTy, DebugLoc());
- VPValue *Mask = MaskBuilder.createNaryOp(VPInstruction::ActiveLaneMask,
- {Zero, FirstActive}, DebugLoc(),
- "uncountable.exit.mask");
+ VPValue *Mask = MaskBuilder.createNaryOp(VPInstruction::MaskBeforeFirst, Cond,
+ DebugLoc(), "uncountable.exit.mask");
// Convert all other memory operations to use the mask.
for (VPBasicBlock *VPBB : vp_rpo_plain_cfg_loop_body(HeaderVPBB))
@@ -3516,6 +3510,10 @@ static bool handleUncountableExitsWithSideEffects(
assert(match(MiddleVPBB->getTerminator(), m_BranchOnCond()) &&
"Expected BranchOnCond terminator for MiddleVPBB");
VPBuilder MiddleBuilder(MiddleVPBB->getTerminator());
+ Type *IVScalarTy = IV->getScalarType();
+ VPValue *Zero = Plan.getZero(IVScalarTy);
+ VPValue *FirstActive = MiddleBuilder.createScalarZExtOrTrunc(
+ MiddleBuilder.createFirstActiveLane(Cond), IVScalarTy, DebugLoc());
VPValue *ScalarIV = MiddleBuilder.createNaryOp(VPInstruction::ExtractLane,
{Zero, IV}, DebugLoc());
VPValue *ExitIV = MiddleBuilder.createAdd(ScalarIV, FirstActive);
diff --git a/llvm/test/Transforms/LoopVectorize/AArch64/early_exit_with_stores.ll b/llvm/test/Transforms/LoopVectorize/AArch64/early_exit_with_stores.ll
index e57fe0d7883f60..616c2a95ff1fd8 100644
--- a/llvm/test/Transforms/LoopVectorize/AArch64/early_exit_with_stores.ll
+++ b/llvm/test/Transforms/LoopVectorize/AArch64/early_exit_with_stores.ll
@@ -54,7 +54,7 @@ define void @loop_contains_store_condition_load_has_single_user(ptr dereferencea
; CHECK-SAME: ptr noalias dereferenceable(40) [[ARRAY:%.*]], ptr readonly align 2 dereferenceable(40) [[PRED:%.*]]) #[[ATTR0]] {
; CHECK-NEXT: [[SCALAR_PH1:.*]]:
; CHECK-NEXT: [[TMP0:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-NEXT: [[TMP1:%.*]] = shl nuw i64 [[TMP0]], 3
+; CHECK-NEXT: [[TMP1:%.*]] = shl nuw i64 [[TMP0]], 2
; CHECK-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 20, [[TMP1]]
; CHECK-NEXT: br i1 [[MIN_ITERS_CHECK]], label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]]
; CHECK: [[VECTOR_PH]]:
@@ -64,21 +64,21 @@ define void @loop_contains_store_condition_load_has_single_user(ptr dereferencea
; CHECK: [[FOR_BODY1]]:
; CHECK-NEXT: [[IV1:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[FOR_BODY1]] ]
; CHECK-NEXT: [[TMP5:%.*]] = getelementptr inbounds nuw i16, ptr [[PRED]], i64 [[IV1]]
-; CHECK-NEXT: [[WIDE_LOAD:%.*]] = load <vscale x 8 x i16>, ptr [[TMP5]], align 2
-; CHECK-NEXT: [[TMP6:%.*]] = icmp sgt <vscale x 8 x i16> [[WIDE_LOAD]], splat (i16 500)
-; CHECK-NEXT: [[TMP9:%.*]] = freeze <vscale x 8 x i1> [[TMP6]]
-; CHECK-NEXT: [[TMP7:%.*]] = call i64 @llvm.experimental.cttz.elts.i64.nxv8i1(<vscale x 8 x i1> [[TMP9]], i1 false)
-; CHECK-NEXT: [[UNCOUNTABLE_EXIT_MASK:%.*]] = call <vscale x 8 x i1> @llvm.get.active.lane.mask.nxv8i1.i64(i64 0, i64 [[TMP7]])
+; CHECK-NEXT: [[WIDE_LOAD:%.*]] = load <vscale x 4 x i16>, ptr [[TMP5]], align 2
+; CHECK-NEXT: [[TMP3:%.*]] = icmp sgt <vscale x 4 x i16> [[WIDE_LOAD]], splat (i16 500)
+; CHECK-NEXT: [[TMP4:%.*]] = freeze <vscale x 4 x i1> [[TMP3]]
+; CHECK-NEXT: [[TMP6:%.*]] = call <vscale x 4 x i1> @llvm.mask.beforefirst.nxv4i1(<vscale x 4 x i1> [[TMP4]])
; CHECK-NEXT: [[ST_ADDR1:%.*]] = getelementptr i16, ptr [[ARRAY]], i64 [[IV1]]
-; CHECK-NEXT: [[WIDE_MASKED_LOAD:%.*]] = call <vscale x 8 x i16> @llvm.masked.load.nxv8i16.p0(ptr align 2 [[ST_ADDR1]], <vscale x 8 x i1> [[UNCOUNTABLE_EXIT_MASK]], <vscale x 8 x i16> poison)
-; CHECK-NEXT: [[TMP8:%.*]] = add nsw <vscale x 8 x i16> [[WIDE_MASKED_LOAD]], splat (i16 1)
-; CHECK-NEXT: call void @llvm.masked.store.nxv8i16.p0(<vscale x 8 x i16> [[TMP8]], ptr align 2 [[ST_ADDR1]], <vscale x 8 x i1> [[UNCOUNTABLE_EXIT_MASK]])
-; CHECK-NEXT: [[TMP10:%.*]] = call i1 @llvm.vector.reduce.or.nxv8i1(<vscale x 8 x i1> [[TMP9]])
+; CHECK-NEXT: [[WIDE_MASKED_LOAD:%.*]] = call <vscale x 4 x i16> @llvm.masked.load.nxv4i16.p0(ptr align 2 [[ST_ADDR1]], <vscale x 4 x i1> [[TMP6]], <vscale x 4 x i16> poison)
+; CHECK-NEXT: [[TMP8:%.*]] = add nsw <vscale x 4 x i16> [[WIDE_MASKED_LOAD]], splat (i16 1)
+; CHECK-NEXT: call void @llvm.masked.store.nxv4i16.p0(<vscale x 4 x i16> [[TMP8]], ptr align 2 [[ST_ADDR1]], <vscale x 4 x i1> [[TMP6]])
+; CHECK-NEXT: [[TMP10:%.*]] = call i1 @llvm.vector.reduce.or.nxv4i1(<vscale x 4 x i1> [[TMP4]])
; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[IV1]], [[TMP1]]
; CHECK-NEXT: [[TMP11:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
; CHECK-NEXT: [[TMP12:%.*]] = or i1 [[TMP10]], [[TMP11]]
; CHECK-NEXT: br i1 [[TMP12]], label %[[MIDDLE_BLOCK:.*]], label %[[FOR_BODY1]], !llvm.loop [[LOOP0:![0-9]+]]
; CHECK: [[MIDDLE_BLOCK]]:
+; CHECK-NEXT: [[TMP7:%.*]] = call i64 @llvm.experimental.cttz.elts.i64.nxv4i1(<vscale x 4 x i1> [[TMP4]], i1 false)
; CHECK-NEXT: [[TMP13:%.*]] = add i64 [[IV1]], [[TMP7]]
; CHECK-NEXT: [[TMP14:%.*]] = icmp eq i64 [[TMP13]], 20
; CHECK-NEXT: br i1 [[TMP14]], label %[[EXIT:.*]], label %[[SCALAR_PH]]
@@ -130,7 +130,7 @@ define void @loop_contains_store_fcmp_condition(ptr dereferenceable(40) noalias
; CHECK-SAME: ptr noalias dereferenceable(40) [[ARRAY:%.*]], ptr readonly align 2 dereferenceable(40) [[PRED:%.*]]) #[[ATTR0]] {
; CHECK-NEXT: [[ENTRY:.*]]:
; CHECK-NEXT: [[TMP0:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-NEXT: [[TMP1:%.*]] = shl nuw i64 [[TMP0]], 3
+; CHECK-NEXT: [[TMP1:%.*]] = shl nuw i64 [[TMP0]], 2
; CHECK-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 20, [[TMP1]]
; CHECK-NEXT: br i1 [[MIN_ITERS_CHECK]], label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]]
; CHECK: [[VECTOR_PH]]:
@@ -140,21 +140,21 @@ define void @loop_contains_store_fcmp_condition(ptr dereferenceable(40) noalias
; CHECK: [[FOR_BODY]]:
; CHECK-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[FOR_BODY]] ]
; CHECK-NEXT: [[TMP2:%.*]] = getelementptr inbounds nuw half, ptr [[PRED]], i64 [[INDEX]]
-; CHECK-NEXT: [[WIDE_LOAD:%.*]] = load <vscale x 8 x half>, ptr [[TMP2]], align 2
-; CHECK-NEXT: [[TMP3:%.*]] = fcmp ugt <vscale x 8 x half> [[WIDE_LOAD]], splat (half 5.000000e+02)
-; CHECK-NEXT: [[TMP4:%.*]] = freeze <vscale x 8 x i1> [[TMP3]]
-; CHECK-NEXT: [[TMP5:%.*]] = call i64 @llvm.experimental.cttz.elts.i64.nxv8i1(<vscale x 8 x i1> [[TMP4]], i1 false)
-; CHECK-NEXT: [[UNCOUNTABLE_EXIT_MASK:%.*]] = call <vscale x 8 x i1> @llvm.get.active.lane.mask.nxv8i1.i64(i64 0, i64 [[TMP5]])
+; CHECK-NEXT: [[WIDE_LOAD:%.*]] = load <vscale x 4 x half>, ptr [[TMP2]], align 2
+; CHECK-NEXT: [[TMP3:%.*]] = fcmp ugt <vscale x 4 x half> [[WIDE_LOAD]], splat (half 5.000000e+02)
+; CHECK-NEXT: [[TMP4:%.*]] = freeze <vscale x 4 x i1> [[TMP3]]
+; CHECK-NEXT: [[TMP13:%.*]] = call <vscale x 4 x i1> @llvm.mask.beforefirst.nxv4i1(<vscale x 4 x i1> [[TMP4]])
; CHECK-NEXT: [[TMP6:%.*]] = getelementptr i16, ptr [[ARRAY]], i64 [[INDEX]]
-; CHECK-NEXT: [[WIDE_MASKED_LOAD:%.*]] = call <vscale x 8 x i16> @llvm.masked.load.nxv8i16.p0(ptr align 2 [[TMP6]], <vscale x 8 x i1> [[UNCOUNTABLE_EXIT_MASK]], <vscale x 8 x i16> poison)
-; CHECK-NEXT: [[TMP7:%.*]] = add nsw <vscale x 8 x i16> [[WIDE_MASKED_LOAD]], splat (i16 1)
-; CHECK-NEXT: call void @llvm.masked.store.nxv8i16.p0(<vscale x 8 x i16> [[TMP7]], ptr align 2 [[TMP6]], <vscale x 8 x i1> [[UNCOUNTABLE_EXIT_MASK]])
-; CHECK-NEXT: [[TMP8:%.*]] = call i1 @llvm.vector.reduce.or.nxv8i1(<vscale x 8 x i1> [[TMP4]])
+; CHECK-NEXT: [[WIDE_MASKED_LOAD:%.*]] = call <vscale x 4 x i16> @llvm.masked.load.nxv4i16.p0(ptr align 2 [[TMP6]], <vscale x 4 x i1> [[TMP13]], <vscale x 4 x i16> poison)
+; CHECK-NEXT: [[TMP7:%.*]] = add nsw <vscale x 4 x i16> [[WIDE_MASKED_LOAD]], splat (i16 1)
+; CHECK-NEXT: call void @llvm.masked.store.nxv4i16.p0(<vscale x 4 x i16> [[TMP7]], ptr align 2 [[TMP6]], <vscale x 4 x i1> [[TMP13]])
+; CHECK-NEXT: [[TMP8:%.*]] = call i1 @llvm.vector.reduce.or.nxv4i1(<vscale x 4 x i1> [[TMP4]])
; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], [[TMP1]]
; CHECK-NEXT: [[TMP9:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
; CHECK-NEXT: [[TMP10:%.*]] = or i1 [[TMP8]], [[TMP9]]
; CHECK-NEXT: br i1 [[TMP10]], label %[[MIDDLE_BLOCK:.*]], label %[[FOR_BODY]], !llvm.loop [[LOOP4:![0-9]+]]
; CHECK: [[MIDDLE_BLOCK]]:
+; CHECK-NEXT: [[TMP5:%.*]] = call i64 @llvm.experimental.cttz.elts.i64.nxv4i1(<vscale x 4 x i1> [[TMP4]], i1 false)
; CHECK-NEXT: [[TMP11:%.*]] = add i64 [[INDEX]], [[TMP5]]
; CHECK-NEXT: [[TMP12:%.*]] = icmp eq i64 [[TMP11]], 20
; CHECK-NEXT: br i1 [[TMP12]], label %[[EXIT:.*]], label %[[SCALAR_PH]]
@@ -363,7 +363,7 @@ define void @loop_contains_store_to_pointer_with_no_deref_info(ptr align 2 deref
; CHECK-SAME: ptr readonly align 2 dereferenceable(40) [[LOAD_ARRAY:%.*]], ptr noalias align 2 [[ARRAY:%.*]], ptr readonly align 2 dereferenceable(40) [[PRED:%.*]]) #[[ATTR0]] {
; CHECK-NEXT: [[SCALAR_PH:.*]]:
; CHECK-NEXT: [[TMP0:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-NEXT: [[TMP1:%.*]] = shl nuw i64 [[TMP0]], 3
+; CHECK-NEXT: [[TMP1:%.*]] = shl nuw i64 [[TMP0]], 2
; CHECK-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 20, [[TMP1]]
; CHECK-NEXT: br i1 [[MIN_ITERS_CHECK]], label %[[SCALAR_PH1:.*]], label %[[VECTOR_PH:.*]]
; CHECK: [[VECTOR_PH]]:
@@ -373,22 +373,22 @@ define void @loop_contains_store_to_pointer_with_no_deref_info(ptr align 2 deref
; CHECK: [[FOR_BODY]]:
; CHECK-NEXT: [[IV:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[FOR_BODY]] ]
; CHECK-NEXT: [[TMP5:%.*]] = getelementptr inbounds nuw i16, ptr [[PRED]], i64 [[IV]]
-; CHECK-NEXT: [[WIDE_LOAD:%.*]] = load <vscale x 8 x i16>, ptr [[TMP5]], align 2
-; CHECK-NEXT: [[TMP6:%.*]] = icmp sgt <vscale x 8 x i16> [[WIDE_LOAD]], splat (i16 500)
-; CHECK-NEXT: [[TMP10:%.*]] = freeze <vscale x 8 x i1> [[TMP6]]
-; CHECK-NEXT: [[TMP7:%.*]] = call i64 @llvm.experimental.cttz.elts.i64.nxv8i1(<vscale x 8 x i1> [[TMP10]], i1 false)
-; CHECK-NEXT: [[UNCOUNTABLE_EXIT_MASK:%.*]] = call <vscale x 8 x i1> @llvm.get.active.lane.mask.nxv8i1.i64(i64 0, i64 [[TMP7]])
+; CHECK-NEXT: [[WIDE_LOAD:%.*]] = load <vscale x 4 x i16>, ptr [[TMP5]], align 2
+; CHECK-NEXT: [[TMP3:%.*]] = icmp sgt <vscale x 4 x i16> [[WIDE_LOAD]], splat (i16 500)
+; CHECK-NEXT: [[TMP4:%.*]] = freeze <vscale x 4 x i1> [[TMP3]]
+; CHECK-NEXT: [[TMP6:%.*]] = call <vscale x 4 x i1> @llvm.mask.beforefirst.nxv4i1(<vscale x 4 x i1> [[TMP4]])
; CHECK-NEXT: [[LD_ADDR:%.*]] = getelementptr i16, ptr [[LOAD_ARRAY]], i64 [[IV]]
-; CHECK-NEXT: [[WIDE_MASKED_LOAD:%.*]] = call <vscale x 8 x i16> @llvm.masked.load.nxv8i16.p0(ptr align 2 [[LD_ADDR]], <vscale x 8 x i1> [[UNCOUNTABLE_EXIT_MASK]], <vscale x 8 x i16> poison)
-; CHECK-NEXT: [[TMP8:%.*]] = add nsw <vscale x 8 x i16> [[WIDE_MASKED_LOAD]], splat (i16 1)
+; CHECK-NEXT: [[WIDE_MASKED_LOAD:%.*]] = call <vscale x 4 x i16> @llvm.masked.load.nxv4i16.p0(ptr align 2 [[LD_ADDR]], <vscale x 4 x i1> [[TMP6]], <vscale x 4 x i16> poison)
+; CHECK-NEXT: [[TMP8:%.*]] = add nsw <vscale x 4 x i16> [[WIDE_MASKED_LOAD]], splat (i16 1)
; CHECK-NEXT: [[TMP9:%.*]] = getelementptr i16, ptr [[ARRAY]], i64 [[IV]]
-; CHECK-NEXT: call void @llvm.masked.store.nxv8i16.p0(<vscale x 8 x i16> [[TMP8]], ptr align 2 [[TMP9]], <vscale x 8 x i1> [[UNCOUNTABLE_EXIT_MASK]])
-; CHECK-NEXT: [[TMP11:%.*]] = call i1 @llvm.vector.reduce.or.nxv8i1(<vscale x 8 x i1> [[TMP10]])
+; CHECK-NEXT: call void @llvm.masked.store.nxv4i16.p0(<vscale x 4 x i16> [[TMP8]], ptr align 2 [[TMP9]], <vscale x 4 x i1> [[TMP6]])
+; CHECK-NEXT: [[TMP11:%.*]] = call i1 @llvm.vector.reduce.or.nxv4i1(<vscale x 4 x i1> [[TMP4]])
; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[IV]], [[TMP1]]
; CHECK-NEXT: [[TMP12:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
; CHECK-NEXT: [[TMP13:%.*]] = or i1 [[TMP11]], [[TMP12]]
; CHECK-NEXT: br i1 [[TMP13]], label %[[MIDDLE_BLOCK:.*]], label %[[FOR_BODY]], !llvm.loop [[LOOP6:![0-9]+]]
; CHECK: [[MIDDLE_BLOCK]]:
+; CHECK-NEXT: [[TMP7:%.*]] = call i64 @llvm.experimental.cttz.elts.i64.nxv4i1(<vscale x 4 x i1> [[TMP4]], i1 false)
; CHECK-NEXT: [[TMP14:%.*]] = add i64 [[IV]], [[TMP7]]
; CHECK-NEXT: [[TMP15:%.*]] = icmp eq i64 [[TMP14]], 20
; CHECK-NEXT: br i1 [[TMP15]], label %[[EXIT:.*]], label %[[SCALAR_PH1]]
@@ -487,7 +487,7 @@ define void @loop_contains_store_in_latch_block(ptr dereferenceable(40) noalias
; CHECK-SAME: ptr noalias dereferenceable(40) [[ARRAY:%.*]], ptr readonly align 2 dereferenceable(40) [[PRED:%.*]]) #[[ATTR0]] {
; CHECK-NEXT: [[SCALAR_PH1:.*]]:
; CHECK-NEXT: [[TMP0:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-NEXT: [[TMP1:%.*]] = shl nuw i64 [[TMP0]], 3
+; CHECK-NEXT: [[TMP1:%.*]] = shl nuw i64 [[TMP0]], 2
; CHECK-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 20, [[TMP1]]
; CHECK-NEXT: br i1 [[MIN_ITERS_CHECK]], label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]]
; CHECK: [[VECTOR_PH]]:
@@ -497,21 +497,21 @@ define void @loop_contains_store_in_latch_block(ptr dereferenceable(40) noalias
; CHECK: [[FOR_BODY1]]:
; CHECK-NEXT: [[IV1:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[FOR_BODY1]] ]
; CHECK-NEXT: [[EE_ADDR1:%.*]] = getelementptr inbounds nuw i16, ptr [[PRED]], i64 [[IV1]]
-; CHECK-NEXT: [[WIDE_LOAD:%.*]] = load <vscale x 8 x i16>, ptr [[EE_ADDR1]], align 2
-; CHECK-NEXT: [[TMP5:%.*]] = icmp sgt <vscale x 8 x i16> [[WIDE_LOAD]], splat (i16 500)
-; CHECK-NEXT: [[TMP9:%.*]] = freeze <vscale x 8 x i1> [[TMP5]]
-; CHECK-NEXT: [[TMP6:%.*]] = call i64 @llvm.experimental.cttz.elts.i64.nxv8i1(<vscale x 8 x i1> [[TMP9]], i1 false)
-; CHECK-NEXT: [[UNCOUNTABLE_EXIT_MASK:%.*]] = call <vscale x 8 x i1> @llvm.get.active.lane.mask.nxv8i1.i64(i64 0, i64 [[TMP6]])
+; CHECK-NEXT: [[WIDE_LOAD:%.*]] = load <vscale x 4 x i16>, ptr [[EE_ADDR1]], align 2
+; CHECK-NEXT: [[TMP3:%.*]] = icmp sgt <vscale x 4 x i16> [[WIDE_LOAD]], splat (i16 500)
+; CHECK-NEXT: [[TMP4:%.*]] = freeze <vscale x 4 x i1> [[TMP3]]
+; CHECK-NEXT: [[TMP5:%.*]] = call <vscale x 4 x i1> @llvm.mask.beforefirst.nxv4i1(<vscale x 4 x i1> [[TMP4]])
; CHECK-NEXT: [[TMP7:%.*]] = getelementptr i16, ptr [[ARRAY]], i64 [[IV1]]
-; CHECK-NEXT: [[WIDE_MASKED_LOAD:%.*]] = call <vscale x 8 x i16> @llvm.masked.load.nxv8i16.p0(ptr align 2 [[TMP7]], <vscale x 8 x i1> [[UNCOUNTABLE_EXIT_MASK]], <vscale x 8 x i16> poison)
-; CHECK-NEXT: [[TMP8:%.*]] = add nsw <vscale x 8 x i16> [[WIDE_MASKED_LOAD]], splat (i16 1)
-; CHECK-NEXT: call void @llvm.masked.store.nxv8i16.p0(<vscale x 8 x i16> [[TMP8]], ptr align 2 [[TMP7]], <vscale x 8 x i1> [[UNCOUNTABLE_EXIT_MASK]])
-; CHECK-NEXT: [[TMP10:%.*]] = call i1 @llvm.vector.reduce.or.nxv8i1(<vscale x 8 x i1> [[TMP9]])
+; CHECK-NEXT: [[WIDE_MASKED_LOAD:%.*]] = call <vscale x 4 x i16> @llvm.masked.load.nxv4i16.p0(ptr align 2 [[TMP7]], <vscale x 4 x i1> [[TMP5]], <vscale x 4 x i16> poison)
+; CHECK-NEXT: [[TMP8:%.*]] = add nsw <vscale x 4 x i16> [[WIDE_MASKED_LOAD]], splat (i16 1)
+; CHECK-NEXT: call void @llvm.masked.store.nxv4i16.p0(<vscale x 4 x i16> [[TMP8]], ptr align 2 [[TMP7]], <vscale x 4 x i1> [[TMP5]])
+; CHECK-NEXT: [[TMP10:%.*]] = call i1 @llvm.vector.reduce.or.nxv4i1(<vscale x 4 x i1> [[TMP4]])
; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[IV1]], [[TMP1]]
; CHECK-NEXT: [[TMP11:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
; CHECK-NEXT: [[TMP12:%.*]] = or i1 [[TMP10]], [[TMP11]]
; CHECK-NEXT: br i1 [[TMP12]], label %[[MIDDLE_BLOCK:.*]], label %[[FOR_BODY1]], !llvm.loop [[LOOP8:![0-9]+]]
; CHECK: [[MIDDLE_BLOCK]]:
+; CHECK-NEXT: [[TMP6:%.*]] = call i64 @llvm.experimental.cttz.elts.i64.nxv4i1(<vscale x 4 x i1> [[TMP4]], i1 false)
; CHECK-NEXT: [[TMP13:%.*]] = add i64 [[IV1]], [[TMP6]]
; CHECK-NEXT: [[TMP14:%.*]] = icmp eq i64 [[TMP13]], 20
; CHECK-NEXT: br i1 [[TMP14]], label %[[EXIT:.*]], label %[[SCALAR_PH]]
@@ -1159,7 +1159,7 @@ define i32 @uncountable_exit_with_separate_exit_block(ptr dereferenceable(40) no
; CHECK-SAME: ptr noalias dereferenceable(40) [[ARRAY:%.*]], ptr readonly align 2 dereferenceable(40) [[PRED:%.*]]) #[[ATTR0]] {
; CHECK-NEXT: [[SCALAR_PH:.*]]:
; CHECK-NEXT: [[TMP0:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-NEXT: [[TMP1:%.*]] = shl nuw i64 [[TMP0]], 3
+; CHECK-NEXT: [[TMP1:%.*]] = shl nuw i64 [[TMP0]], 2
; CHECK-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 20, [[TMP1]]
; CHECK-NEXT: br i1 [[MIN_ITERS_CHECK]], label %[[SCALAR_PH1:.*]], label %[[VECTOR_PH:.*]]
; CHECK: [[VECTOR_PH]]:
@@ -1169,21 +1169,21 @@ define i32 @uncountable_exit_with_separate_exit_block(ptr dereferenceable(40) no
; CHECK: [[FOR_BODY1]]:
; CHECK-NEXT: [[IV1:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[FOR_BODY1]] ]
; CHECK-NEXT: [[TMP5:%.*]] = getelementptr inbounds nuw i16, ptr [[PRED]], i64 [[IV1]]
-; CHECK-NEXT: [[WIDE_LOAD:%.*]] = load <vscale x 8 x i16>, ptr [[TMP5]], align 2
-; CHECK-NEXT: [[TMP6:%.*]] = icmp sgt <vscale x 8 x i16> [[WIDE_LOAD]], splat (i16 500)
-; CHECK-NEXT: [[TMP9:%.*]] = freeze <vscale x 8 x i1> [[TMP6]]
-; CHECK-NEXT: [[TMP7:%.*]] = call i64 @llvm.experimental.cttz.elts.i64.nxv8i1(<vscale x 8 x i1> [[TMP9]], i1 false)
-; CHECK-NEXT: [[UNCOUNTABLE_EXIT_MASK:%.*]] = call <vscale x 8 x i1> @llvm.get.active.lane.mask.nxv8i1.i64(i64 0, i64 [[TMP7]])
+; CHECK-NEXT: [[WIDE_LOAD:%.*]] = load <vscale x 4 x i16>, ptr [[TMP5]], align 2
+; CHECK-NEXT: [[TMP3:%.*]] = icmp sgt <vscale x 4 x i16> [[WIDE_LOAD]], splat (i16 500)
+; CHECK-NEXT: [[TMP4:%.*]] = freeze <vscale x 4 x i1> [[TMP3]]
+; CHECK-NEXT: [[TMP6:%.*]] = call <vscale x 4 x i1> @llvm.mask.beforefirst.nxv4i1(<vscale x 4 x i1> [[TMP4]])
; CHECK-NEXT: [[ST_ADDR1:%.*]] = getelementptr i16, ptr [[ARRAY]], i64 [[IV1]]
-; CHECK-NEXT: [[WIDE_MASKED_LOAD:%.*]] = call <vscale x 8 x i16> @llvm.masked.load.nxv8i16.p0(ptr align 2 [[ST_ADDR1]], <vscale x 8 x i1> [[UNCOUNTABLE_EXIT_MASK]], <vscale x 8 x i16> poison)
-; CHECK-NEXT: [[TMP8:%.*]] = add nsw <vscale x 8 x i16> [[WIDE_MASKED_LOAD]], splat (i16 1)
-; CHECK-NEXT: call void @llvm.masked.store.nxv8i16.p0(<vscale x 8 x i16> [[TMP8]], ptr align 2 [[ST_ADDR1]], <vscale x 8 x i1> [[UNCOUNTABLE_EXIT_MASK]])
-; CHECK-NEXT: [[TMP10:%.*]] = call i1 @llvm.vector.reduce.or.nxv8i1(<vscale x 8 x i1> [[TMP9]])
+; CHECK-NEXT: [[WIDE_MASKED_LOAD:%.*]] = call <vscale x 4 x i16> @llvm.masked.load.nxv4i16.p0(ptr align 2 [[ST_ADDR1]], <vscale x 4 x i1> [[TMP6]], <vscale x 4 x i16> poison)
+; CHECK-NEXT: [[TMP8:%.*]] = add nsw <vscale x 4 x i16> [[WIDE_MASKED_LOAD]], splat (i16 1)
+; CHECK-NEXT: call void @llvm.masked.store.nxv4i16.p0(<vscale x 4 x i16> [[TMP8]], ptr align 2 [[ST_ADDR1]], <vscale x 4 x i1> [[TMP6]])
+; CHECK-NEXT: [[TMP10:%.*]] = call i1 @llvm.vector.reduce.or.nxv4i1(<vscale x 4 x i1> [[TMP4]])
; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[IV1]], [[TMP1]]
; CHECK-NEXT: [[TMP11:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
; CHECK-NEXT: [[TMP12:%.*]] = or i1 [[TMP10]], [[TMP11]]
; CHECK-NEXT: br i1 [[TMP12]], label %[[MIDDLE_BLOCK:.*]], label %[[FOR_BODY1]], !llvm.loop [[LOOP10:![0-9]+]]
; CHECK: [[MIDDLE_BLOCK]]:
+; CHECK-NEXT: [[TMP7:%.*]] = call i64 @llvm.experimental.cttz.elts.i64.nxv4i1(<vscale x 4 x i1> [[TMP4]], i1 false)
; CHECK-NEXT: [[TMP13:%.*]] = add i64 [[IV1]], [[TMP7]]
; CHECK-NEXT: [[TMP14:%.*]] = icmp eq i64 [[TMP13]], 20
; CHECK-NEXT: br i1 [[TMP14]], label %[[EXIT_COUNTABLE:.*]], label %[[SCALAR_PH1]]
diff --git a/llvm/test/Transforms/LoopVectorize/RISCV/early_exit_with_stores.ll b/llvm/test/Transforms/LoopVectorize/RISCV/early_exit_with_stores.ll
index 461ad36dc2a604..573ee73e76a439 100644
--- a/llvm/test/Transforms/LoopVectorize/RISCV/early_exit_with_stores.ll
+++ b/llvm/test/Transforms/LoopVectorize/RISCV/early_exit_with_stores.ll
@@ -65,8 +65,7 @@ define void @loop_contains_store_condition_load_has_single_user(ptr dereferencea
; CHECK-NEXT: [[WIDE_LOAD:%.*]] = load <vscale x 8 x i16>, ptr [[TMP5]], align 2
; CHECK-NEXT: [[TMP6:%.*]] = icmp sgt <vscale x 8 x i16> [[WIDE_LOAD]], splat (i16 500)
; CHECK-NEXT: [[TMP9:%.*]] = freeze <vscale x 8 x i1> [[TMP6]]
-; CHECK-NEXT: [[TMP7:%.*]] = call i64 @llvm.experimental.cttz.elts.i64.nxv8i1(<vscale x 8 x i1> [[TMP9]], i1 false)
-; CHECK-NEXT: [[UNCOUNTABLE_EXIT_MASK:%.*]] = call <vscale x 8 x i1> @llvm.get.active.lane.mask.nxv8i1.i64(i64 0, i64 [[TMP7]])
+; CHECK-NEXT: [[UNCOUNTABLE_EXIT_MASK:%.*]] = call <vscale x 8 x i1> @llvm.mask.beforefirst.nxv8i1(<vscale x 8 x i1> [[TMP9]])
; CHECK-NEXT: [[ST_ADDR:%.*]] = getelementptr i16, ptr [[ARRAY]], i64 [[IV]]
; CHECK-NEXT: [[WIDE_MASKED_LOAD:%.*]] = call <vscale x 8 x i16> @llvm.masked.load.nxv8i16.p0(ptr align 2 [[ST_ADDR]], <vscale x 8 x i1> [[UNCOUNTABLE_EXIT_MASK]], <vscale x 8 x i16> poison)
; CHECK-NEXT: [[TMP8:%.*]] = add nsw <vscale x 8 x i16> [[WIDE_MASKED_LOAD]], splat (i16 1)
@@ -77,6 +76,7 @@ define void @loop_contains_store_condition_load_has_single_user(ptr dereferencea
; CHECK-NEXT: [[TMP12:%.*]] = or i1 [[TMP10]], [[TMP11]]
; CHECK-NEXT: br i1 [[TMP12]], label %[[MIDDLE_BLOCK:.*]], label %[[FOR_BODY]], !llvm.loop [[LOOP0:![0-9]+]]
; CHECK: [[MIDDLE_BLOCK]]:
+; CHECK-NEXT: [[TMP7:%.*]] = call i64 @llvm.experimental.cttz.elts.i64.nxv8i1(<vscale x 8 x i1> [[TMP9]], i1 false)
; CHECK-NEXT: [[TMP13:%.*]] = add i64 [[IV]], [[TMP7]]
; CHECK-NEXT: [[TMP14:%.*]] = icmp eq i64 [[TMP13]], 20
; CHECK-NEXT: br i1 [[TMP14]], label %[[EXIT:.*]], label %[[SCALAR_PH1]]
@@ -134,13 +134,13 @@ define void @loop_contains_store_fcmp_condition(ptr dereferenceable(40) noalias
; CHECK-NEXT: [[WIDE_LOAD:%.*]] = load <16 x half>, ptr [[PRED]], align 2
; CHECK-NEXT: [[TMP0:%.*]] = fcmp ugt <16 x half> [[WIDE_LOAD]], splat (half 5.000000e+02)
; CHECK-NEXT: [[TMP1:%.*]] = freeze <16 x i1> [[TMP0]]
-; CHECK-NEXT: [[TMP2:%.*]] = call i64 @llvm.experimental.cttz.elts.i64.v16i1(<16 x i1> [[TMP1]], i1 false)
-; CHECK-NEXT: [[UNCOUNTABLE_EXIT_MASK:%.*]] = call <16 x i1> @llvm.get.active.lane.mask.v16i1.i64(i64 0, i64 [[TMP2]])
+; CHECK-NEXT: [[UNCOUNTABLE_EXIT_MASK:%.*]] = call <16 x i1> @llvm.mask.beforefirst.v16i1(<16 x i1> [[TMP1]])
; CHECK-NEXT: [[WIDE_MASKED_LOAD:%.*]] = call <16 x i16> @llvm.masked.load.v16i16.p0(ptr align 2 [[ARRAY]], <16 x i1> [[UNCOUNTABLE_EXIT_MASK]], <16 x i16> poison)
; CHECK-NEXT: [[TMP3:%.*]] = add nsw <16 x i16> [[WIDE_MASKED_LOAD]], splat (i16 1)
; CHECK-NEXT: call void @llvm.masked.store.v16i16.p0(<16 x i16> [[TMP3]], ptr align 2 [[ARRAY]], <16 x i1> [[UNCOUNTABLE_EXIT_MASK]])
; CHECK-NEXT: br label %[[MIDDLE_BLOCK:.*]]
; CHECK: [[MIDDLE_BLOCK]]:
+; CHECK-NEXT: [[TMP2:%.*]] = call i64 @llvm.experimental.cttz.elts.i64.v16i1(<16 x i1> [[TMP1]], i1 false)
; CHECK-NEXT: [[TMP4:%.*]] = icmp eq i64 [[TMP2]], 20
; CHECK-NEXT: br i1 [[TMP4]], label %[[EXIT:.*]], label %[[SCALAR_PH:.*]]
; CHECK: [[SCALAR_PH]]:
@@ -360,8 +360,7 @@ define void @loop_contains_store_to_pointer_with_no_deref_info(ptr align 2 deref
; CHECK-NEXT: [[WIDE_LOAD:%.*]] = load <vscale x 8 x i16>, ptr [[TMP5]], align 2
; CHECK-NEXT: [[TMP6:%.*]] = icmp sgt <vscale x 8 x i16> [[WIDE_LOAD]], splat (i16 500)
; CHECK-NEXT: [[TMP10:%.*]] = freeze <vscale x 8 x i1> [[TMP6]]
-; CHECK-NEXT: [[TMP7:%.*]] = call i64 @llvm.experimental.cttz.elts.i64.nxv8i1(<vscale x 8 x i1> [[TMP10]], i1 false)
-; CHECK-NEXT: [[UNCOUNTABLE_EXIT_MASK:%.*]] = call <vscale x 8 x i1> @llvm.get.active.lane.mask.nxv8i1.i64(i64 0, i64 [[TMP7]])
+; CHECK-NEXT: [[UNCOUNTABLE_EXIT_MASK:%.*]] = call <vscale x 8 x i1> @llvm.mask.beforefirst.nxv8i1(<vscale x 8 x i1> [[TMP10]])
; CHECK-NEXT: [[LD_ADDR:%.*]] = getelementptr i16, ptr [[LOAD_ARRAY]], i64 [[IV]]
; CHECK-NEXT: [[WIDE_MASKED_LOAD:%.*]] = call <vscale x 8 x i16> @llvm.masked.load.nxv8i16.p0(ptr align 2 [[LD_ADDR]], <vscale x 8 x i1> [[UNCOUNTABLE_EXIT_MASK]], <vscale x 8 x i16> poison)
; CHECK-NEXT: [[TMP8:%.*]] = add nsw <vscale x 8 x i16> [[WIDE_MASKED_LOAD]], splat (i16 1)
@@ -373,6 +372,7 @@ define void @loop_contains_store_to_pointer_with_no_deref_info(ptr align 2 deref
; CHECK-NEXT: [[TMP13:%.*]] = or i1 [[TMP11]], [[TMP12]]
; CHECK-NEXT: br i1 [[TMP13]], label %[[MIDDLE_BLOCK:.*]], label %[[FOR_BODY]], !llvm.loop [[LOOP5:![0-9]+]]
; CHECK: [[MIDDLE_BLOCK]]:
+; CHECK-NEXT: [[TMP7:%.*]] = call i64 @llvm.experimental.cttz.elts.i64.nxv8i1(<vscale x 8 x i1> [[TMP10]], i1 false)
; CHECK-NEXT: [[TMP14:%.*]] = add i64 [[IV]], [[TMP7]]
; CHECK-NEXT: [[TMP15:%.*]] = icmp eq i64 [[TMP14]], 20
; CHECK-NEXT: br i1 [[TMP15]], label %[[EXIT:.*]], label %[[SCALAR_PH1]]
@@ -529,8 +529,7 @@ define void @loop_contains_store_in_latch_block(ptr dereferenceable(40) noalias
; CHECK-NEXT: [[WIDE_LOAD:%.*]] = load <vscale x 8 x i16>, ptr [[EE_ADDR]], align 2
; CHECK-NEXT: [[TMP5:%.*]] = icmp sgt <vscale x 8 x i16> [[WIDE_LOAD]], splat (i16 500)
; CHECK-NEXT: [[TMP9:%.*]] = freeze <vscale x 8 x i1> [[TMP5]]
-; CHECK-NEXT: [[TMP6:%.*]] = call i64 @llvm.experimental.cttz.elts.i64.nxv8i1(<vscale x 8 x i1> [[TMP9]], i1 false)
-; CHECK-NEXT: [[UNCOUNTABLE_EXIT_MASK:%.*]] = call <vscale x 8 x i1> @llvm.get.active.lane.mask.nxv8i1.i64(i64 0, i64 [[TMP6]])
+; CHECK-NEXT: [[UNCOUNTABLE_EXIT_MASK:%.*]] = call <vscale x 8 x i1> @llvm.mask.beforefirst.nxv8i1(<vscale x 8 x i1> [[TMP9]])
; CHECK-NEXT: [[TMP7:%.*]] = getelementptr i16, ptr [[ARRAY]], i64 [[IV]]
; CHECK-NEXT: [[WIDE_MASKED_LOAD:%.*]] = call <vscale x 8 x i16> @llvm.masked.load.nxv8i16.p0(ptr align 2 [[TMP7]], <vscale x 8 x i1> [[UNCOUNTABLE_EXIT_MASK]], <vscale x 8 x i16> poison)
; CHECK-NEXT: [[TMP8:%.*]] = add nsw <vscale x 8 x i16> [[WIDE_MASKED_LOAD]], splat (i16 1)
@@ -541,6 +540,7 @@ define void @loop_contains_store_in_latch_block(ptr dereferenceable(40) noalias
; CHECK-NEXT: [[TMP12:%.*]] = or i1 [[TMP10]], [[TMP11]]
; CHECK-NEXT: br i1 [[TMP12]], label %[[MIDDLE_BLOCK:.*]], label %[[FOR_BODY]], !llvm.loop [[LOOP7:![0-9]+]]
; CHECK: [[MIDDLE_BLOCK]]:
+; CHECK-NEXT: [[TMP6:%.*]] = call i64 @llvm.experimental.cttz.elts.i64.nxv8i1(<vscale x 8 x i1> [[TMP9]], i1 false)
; CHECK-NEXT: [[TMP13:%.*]] = add i64 [[IV]], [[TMP6]]
; CHECK-NEXT: [[TMP14:%.*]] = icmp eq i64 [[TMP13]], 20
; CHECK-NEXT: br i1 [[TMP14]], label %[[EXIT:.*]], label %[[SCALAR_PH1]]
@@ -1201,8 +1201,7 @@ define i32 @uncountable_exit_with_separate_exit_block(ptr dereferenceable(40) no
; CHECK-NEXT: [[WIDE_LOAD:%.*]] = load <vscale x 8 x i16>, ptr [[TMP5]], align 2
; CHECK-NEXT: [[TMP6:%.*]] = icmp sgt <vscale x 8 x i16> [[WIDE_LOAD]], splat (i16 500)
; CHECK-NEXT: [[TMP9:%.*]] = freeze <vscale x 8 x i1> [[TMP6]]
-; CHECK-NEXT: [[TMP7:%.*]] = call i64 @llvm.experimental.cttz.elts.i64.nxv8i1(<vscale x 8 x i1> [[TMP9]], i1 false)
-; CHECK-NEXT: [[UNCOUNTABLE_EXIT_MASK:%.*]] = call <vscale x 8 x i1> @llvm.get.active.lane.mask.nxv8i1.i64(i64 0, i64 [[TMP7]])
+; CHECK-NEXT: [[UNCOUNTABLE_EXIT_MASK:%.*]] = call <vscale x 8 x i1> @llvm.mask.beforefirst.nxv8i1(<vscale x 8 x i1> [[TMP9]])
; CHECK-NEXT: [[ST_ADDR:%.*]] = getelementptr i16, ptr [[ARRAY]], i64 [[IV]]
; CHECK-NEXT: [[WIDE_MASKED_LOAD:%.*]] = call <vscale x 8 x i16> @llvm.masked.load.nxv8i16.p0(ptr align 2 [[ST_ADDR]], <vscale x 8 x i1> [[UNCOUNTABLE_EXIT_MASK]], <vscale x 8 x i16> poison)
; CHECK-NEXT: [[TMP8:%.*]] = add nsw <vscale x 8 x i16> [[WIDE_MASKED_LOAD]], splat (i16 1)
@@ -1213,6 +1212,7 @@ define i32 @uncountable_exit_with_separate_exit_block(ptr dereferenceable(40) no
; CHECK-NEXT: [[TMP12:%.*]] = or i1 [[TMP10]], [[TMP11]]
; CHECK-NEXT: br i1 [[TMP12]], label %[[MIDDLE_BLOCK:.*]], label %[[FOR_BODY]], !llvm.loop [[LOOP9:![0-9]+]]
; CHECK: [[MIDDLE_BLOCK]]:
+; CHECK-NEXT: [[TMP7:%.*]] = call i64 @llvm.experimental.cttz.elts.i64.nxv8i1(<vscale x 8 x i1> [[TMP9]], i1 false)
; CHECK-NEXT: [[TMP13:%.*]] = add i64 [[IV]], [[TMP7]]
; CHECK-NEXT: [[TMP14:%.*]] = icmp eq i64 [[TMP13]], 20
; CHECK-NEXT: br i1 [[TMP14]], label %[[EXIT_COUNTABLE:.*]], label %[[SCALAR_PH]]
@@ -1282,8 +1282,7 @@ define void @interleaved_stores_after_early_exit(ptr noalias %dst, ptr align 2 d
; CHECK-NEXT: [[WIDE_LOAD:%.*]] = load <vscale x 8 x i16>, ptr [[TMP2]], align 2
; CHECK-NEXT: [[TMP3:%.*]] = icmp sgt <vscale x 8 x i16> [[WIDE_LOAD]], splat (i16 500)
; CHECK-NEXT: [[TMP4:%.*]] = freeze <vscale x 8 x i1> [[TMP3]]
-; CHECK-NEXT: [[TMP5:%.*]] = call i64 @llvm.experimental.cttz.elts.i64.nxv8i1(<vscale x 8 x i1> [[TMP4]], i1 false)
-; CHECK-NEXT: [[UNCOUNTABLE_EXIT_MASK:%.*]] = call <vscale x 8 x i1> @llvm.get.active.lane.mask.nxv8i1.i64(i64 0, i64 [[TMP5]])
+; CHECK-NEXT: [[UNCOUNTABLE_EXIT_MASK:%.*]] = call <vscale x 8 x i1> @llvm.mask.beforefirst.nxv8i1(<vscale x 8 x i1> [[TMP4]])
; CHECK-NEXT: [[TMP6:%.*]] = shl i64 [[INDEX]], 1
; CHECK-NEXT: [[TMP7:%.*]] = getelementptr i16, ptr [[DST]], i64 [[TMP6]]
; CHECK-NEXT: [[INTERLEAVED_VEC:%.*]] = call <vscale x 16 x i16> @llvm.vector.interleave2.nxv16i16(<vscale x 8 x i16> splat (i16 1), <vscale x 8 x i16> splat (i16 2))
@@ -1295,6 +1294,7 @@ define void @interleaved_stores_after_early_exit(ptr noalias %dst, ptr align 2 d
; CHECK-NEXT: [[TMP10:%.*]] = or i1 [[TMP8]], [[TMP9]]
; CHECK-NEXT: br i1 [[TMP10]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP11:![0-9]+]]
; CHECK: [[MIDDLE_BLOCK]]:
+; CHECK-NEXT: [[TMP5:%.*]] = call i64 @llvm.experimental.cttz.elts.i64.nxv8i1(<vscale x 8 x i1> [[TMP4]], i1 false)
; CHECK-NEXT: [[TMP11:%.*]] = add i64 [[INDEX]], [[TMP5]]
; CHECK-NEXT: [[TMP12:%.*]] = icmp eq i64 [[TMP11]], 20
; CHECK-NEXT: br i1 [[TMP12]], label %[[EXIT:.*]], label %[[SCALAR_PH]]
@@ -1363,8 +1363,7 @@ define void @interleaved_stores_before_early_exit(ptr noalias %dst, ptr align 2
; CHECK-NEXT: [[WIDE_LOAD:%.*]] = load <vscale x 8 x i16>, ptr [[TMP2]], align 2
; CHECK-NEXT: [[TMP3:%.*]] = icmp sgt <vscale x 8 x i16> [[WIDE_LOAD]], splat (i16 500)
; CHECK-NEXT: [[TMP4:%.*]] = freeze <vscale x 8 x i1> [[TMP3]]
-; CHECK-NEXT: [[TMP5:%.*]] = call i64 @llvm.experimental.cttz.elts.i64.nxv8i1(<vscale x 8 x i1> [[TMP4]], i1 false)
-; CHECK-NEXT: [[UNCOUNTABLE_EXIT_MASK:%.*]] = call <vscale x 8 x i1> @llvm.get.active.lane.mask.nxv8i1.i64(i64 0, i64 [[TMP5]])
+; CHECK-NEXT: [[UNCOUNTABLE_EXIT_MASK:%.*]] = call <vscale x 8 x i1> @llvm.mask.beforefirst.nxv8i1(<vscale x 8 x i1> [[TMP4]])
; CHECK-NEXT: [[TMP6:%.*]] = shl i64 [[INDEX]], 1
; CHECK-NEXT: [[TMP7:%.*]] = getelementptr i16, ptr [[DST]], i64 [[TMP6]]
; CHECK-NEXT: [[INTERLEAVED_VEC:%.*]] = call <vscale x 16 x i16> @llvm.vector.interleave2.nxv16i16(<vscale x 8 x i16> splat (i16 1), <vscale x 8 x i16> splat (i16 2))
@@ -1376,6 +1375,7 @@ define void @interleaved_stores_before_early_exit(ptr noalias %dst, ptr align 2
; CHECK-NEXT: [[TMP10:%.*]] = or i1 [[TMP8]], [[TMP9]]
; CHECK-NEXT: br i1 [[TMP10]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP13:![0-9]+]]
; CHECK: [[MIDDLE_BLOCK]]:
+; CHECK-NEXT: [[TMP5:%.*]] = call i64 @llvm.experimental.cttz.elts.i64.nxv8i1(<vscale x 8 x i1> [[TMP4]], i1 false)
; CHECK-NEXT: [[TMP11:%.*]] = add i64 [[INDEX]], [[TMP5]]
; CHECK-NEXT: [[TMP12:%.*]] = icmp eq i64 [[TMP11]], 20
; CHECK-NEXT: br i1 [[TMP12]], label %[[EXIT:.*]], label %[[SCALAR_PH]]
@@ -1431,7 +1431,7 @@ define void @reverse_interleaved_stores_before_early_exit(ptr noalias %dst, ptr
; CHECK-SAME: ptr noalias [[DST:%.*]], ptr align 2 dereferenceable(40) [[SRC:%.*]]) #[[ATTR0]] {
; CHECK-NEXT: [[ENTRY:.*]]:
; CHECK-NEXT: [[TMP0:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-NEXT: [[TMP1:%.*]] = shl nuw i64 [[TMP0]], 3
+; CHECK-NEXT: [[TMP1:%.*]] = shl nuw i64 [[TMP0]], 1
; CHECK-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 20, [[TMP1]]
; CHECK-NEXT: br i1 [[MIN_ITERS_CHECK]], label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]]
; CHECK: [[VECTOR_PH]]:
@@ -1444,22 +1444,22 @@ define void @reverse_interleaved_stores_before_early_exit(ptr noalias %dst, ptr
; CHECK: [[VECTOR_BODY]]:
; CHECK-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
; CHECK-NEXT: [[TMP5:%.*]] = getelementptr i16, ptr [[SRC]], i64 [[INDEX]]
-; CHECK-NEXT: [[WIDE_LOAD:%.*]] = load <vscale x 8 x i16>, ptr [[TMP5]], align 2
-; CHECK-NEXT: [[TMP6:%.*]] = icmp sgt <vscale x 8 x i16> [[WIDE_LOAD]], splat (i16 500)
-; CHECK-NEXT: [[TMP7:%.*]] = freeze <vscale x 8 x i1> [[TMP6]]
-; CHECK-NEXT: [[TMP8:%.*]] = call i64 @llvm.experimental.cttz.elts.i64.nxv8i1(<vscale x 8 x i1> [[TMP7]], i1 false)
-; CHECK-NEXT: [[UNCOUNTABLE_EXIT_MASK:%.*]] = call <vscale x 8 x i1> @llvm.get.active.lane.mask.nxv8i1.i64(i64 0, i64 [[TMP8]])
+; CHECK-NEXT: [[WIDE_LOAD:%.*]] = load <vscale x 2 x i16>, ptr [[TMP5]], align 2
+; CHECK-NEXT: [[TMP6:%.*]] = icmp sgt <vscale x 2 x i16> [[WIDE_LOAD]], splat (i16 500)
+; CHECK-NEXT: [[TMP7:%.*]] = freeze <vscale x 2 x i1> [[TMP6]]
+; CHECK-NEXT: [[UNCOUNTABLE_EXIT_MASK:%.*]] = call <vscale x 2 x i1> @llvm.mask.beforefirst.nxv2i1(<vscale x 2 x i1> [[TMP7]])
; CHECK-NEXT: [[TMP9:%.*]] = mul i64 [[INDEX]], -4
; CHECK-NEXT: [[TMP10:%.*]] = getelementptr i8, ptr [[TMP3]], i64 [[TMP9]]
-; CHECK-NEXT: call void @llvm.experimental.vp.strided.store.nxv8i16.p0.i64(<vscale x 8 x i16> splat (i16 1), ptr align 2 [[TMP10]], i64 -4, <vscale x 8 x i1> [[UNCOUNTABLE_EXIT_MASK]], i32 [[TMP2]])
+; CHECK-NEXT: call void @llvm.experimental.vp.strided.store.nxv2i16.p0.i64(<vscale x 2 x i16> splat (i16 1), ptr align 2 [[TMP10]], i64 -4, <vscale x 2 x i1> [[UNCOUNTABLE_EXIT_MASK]], i32 [[TMP2]])
; CHECK-NEXT: [[TMP11:%.*]] = getelementptr i8, ptr [[TMP4]], i64 [[TMP9]]
-; CHECK-NEXT: call void @llvm.experimental.vp.strided.store.nxv8i16.p0.i64(<vscale x 8 x i16> splat (i16 2), ptr align 2 [[TMP11]], i64 -4, <vscale x 8 x i1> [[UNCOUNTABLE_EXIT_MASK]], i32 [[TMP2]])
-; CHECK-NEXT: [[TMP12:%.*]] = call i1 @llvm.vector.reduce.or.nxv8i1(<vscale x 8 x i1> [[TMP7]])
+; CHECK-NEXT: call void @llvm.experimental.vp.strided.store.nxv2i16.p0.i64(<vscale x 2 x i16> splat (i16 2), ptr align 2 [[TMP11]], i64 -4, <vscale x 2 x i1> [[UNCOUNTABLE_EXIT_MASK]], i32 [[TMP2]])
+; CHECK-NEXT: [[TMP12:%.*]] = call i1 @llvm.vector.reduce.or.nxv2i1(<vscale x 2 x i1> [[TMP7]])
; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], [[TMP1]]
; CHECK-NEXT: [[TMP13:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
; CHECK-NEXT: [[TMP14:%.*]] = or i1 [[TMP12]], [[TMP13]]
; CHECK-NEXT: br i1 [[TMP14]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP15:![0-9]+]]
; CHECK: [[MIDDLE_BLOCK]]:
+; CHECK-NEXT: [[TMP8:%.*]] = call i64 @llvm.experimental.cttz.elts.i64.nxv2i1(<vscale x 2 x i1> [[TMP7]], i1 false)
; CHECK-NEXT: [[TMP15:%.*]] = add i64 [[INDEX]], [[TMP8]]
; CHECK-NEXT: [[TMP16:%.*]] = icmp eq i64 [[TMP15]], 20
; CHECK-NEXT: br i1 [[TMP16]], label %[[EXIT:.*]], label %[[SCALAR_PH]]
diff --git a/llvm/test/Transforms/LoopVectorize/VPlan/early_exit_with_stores_vplan.ll b/llvm/test/Transforms/LoopVectorize/VPlan/early_exit_with_stores_vplan.ll
index b3c45c9f07362e..1dd06049564be4 100644
--- a/llvm/test/Transforms/LoopVectorize/VPlan/early_exit_with_stores_vplan.ll
+++ b/llvm/test/Transforms/LoopVectorize/VPlan/early_exit_with_stores_vplan.ll
@@ -28,24 +28,24 @@ define void @loop_contains_store_condition_load_has_single_user(ptr dereferencea
; CHECK-NEXT: WIDEN ir<%ee.val> = load vp<[[VP5]]>, align 2
; CHECK-NEXT: WIDEN ir<%ee.cond> = icmp sgt ir<%ee.val>, ir<500>
; CHECK-NEXT: EMIT vp<[[VP6:%[0-9]+]]> = freeze ir<%ee.cond>
-; CHECK-NEXT: EMIT vp<[[VP7:%[0-9]+]]> = first-active-lane vp<[[VP6]]>
-; CHECK-NEXT: EMIT vp<%uncountable.exit.mask> = active lane mask ir<0>, vp<[[VP7]]>
+; CHECK-NEXT: EMIT vp<%uncountable.exit.mask> = mask-before-first vp<[[VP6]]>
; CHECK-NEXT: CLONE ir<%st.addr> = getelementptr ir<%array>, vp<[[VP4]]>
-; CHECK-NEXT: vp<[[VP8:%[0-9]+]]> = vector-pointer i16, ir<%st.addr>, ir<1>
-; CHECK-NEXT: WIDEN ir<%data> = load vp<[[VP8]]>, vp<%uncountable.exit.mask>, align 2
+; CHECK-NEXT: vp<[[VP7:%[0-9]+]]> = vector-pointer i16, ir<%st.addr>, ir<1>
+; CHECK-NEXT: WIDEN ir<%data> = load vp<[[VP7]]>, vp<%uncountable.exit.mask>, align 2
; CHECK-NEXT: WIDEN ir<%inc> = add nsw ir<%data>, ir<1>
-; CHECK-NEXT: WIDEN store vp<[[VP8]]>, ir<%inc>, vp<%uncountable.exit.mask>, align 2
-; CHECK-NEXT: EMIT vp<[[VP9:%[0-9]+]]> = any-of vp<[[VP6]]>
+; CHECK-NEXT: WIDEN store vp<[[VP7]]>, ir<%inc>, vp<%uncountable.exit.mask>, align 2
+; CHECK-NEXT: EMIT vp<[[VP8:%[0-9]+]]> = any-of vp<[[VP6]]>
; CHECK-NEXT: EMIT vp<%index.next> = add nuw vp<[[VP3]]>, vp<[[VP1]]>
-; CHECK-NEXT: EMIT vp<[[VP10:%[0-9]+]]> = icmp eq vp<%index.next>, vp<[[VP2]]>
-; CHECK-NEXT: EMIT branch-on-two-conds vp<[[VP9]]>, vp<[[VP10]]>
+; CHECK-NEXT: EMIT vp<[[VP9:%[0-9]+]]> = icmp eq vp<%index.next>, vp<[[VP2]]>
+; CHECK-NEXT: EMIT branch-on-two-conds vp<[[VP8]]>, vp<[[VP9]]>
; CHECK-NEXT: No successors
; CHECK-NEXT: }
; CHECK-NEXT: Successor(s): middle.block, middle.block
; CHECK-EMPTY:
; CHECK-NEXT: middle.block:
+; CHECK-NEXT: EMIT vp<[[VP11:%[0-9]+]]> = first-active-lane vp<[[VP6]]>
; CHECK-NEXT: EMIT vp<[[VP12:%[0-9]+]]> = extract-lane ir<0>, ir<%iv>
-; CHECK-NEXT: EMIT vp<[[VP13:%[0-9]+]]> = add vp<[[VP12]]>, vp<[[VP7]]>
+; CHECK-NEXT: EMIT vp<[[VP13:%[0-9]+]]> = add vp<[[VP12]]>, vp<[[VP11]]>
; CHECK-NEXT: EMIT vp<[[VP14:%[0-9]+]]> = icmp eq vp<[[VP13]]>, ir<20>
; CHECK-NEXT: EMIT branch-on-cond vp<[[VP14]]>
; CHECK-NEXT: Successor(s): ir-bb<exit>, scalar.ph
@@ -117,24 +117,24 @@ define void @loop_contains_store_after_uncountable_exit(ptr dereferenceable(40)
; CHECK-NEXT: WIDEN ir<%ee.val> = load vp<[[VP5]]>, align 2
; CHECK-NEXT: WIDEN ir<%ee.cond> = icmp sgt ir<%ee.val>, ir<500>
; CHECK-NEXT: EMIT vp<[[VP6:%[0-9]+]]> = freeze ir<%ee.cond>
-; CHECK-NEXT: EMIT vp<[[VP7:%[0-9]+]]> = first-active-lane vp<[[VP6]]>
-; CHECK-NEXT: EMIT vp<%uncountable.exit.mask> = active lane mask ir<0>, vp<[[VP7]]>
+; CHECK-NEXT: EMIT vp<%uncountable.exit.mask> = mask-before-first vp<[[VP6]]>
; CHECK-NEXT: CLONE ir<%st.addr> = getelementptr ir<%array>, vp<[[VP4]]>
-; CHECK-NEXT: vp<[[VP8:%[0-9]+]]> = vector-pointer i16, ir<%st.addr>, ir<1>
-; CHECK-NEXT: WIDEN ir<%data> = load vp<[[VP8]]>, vp<%uncountable.exit.mask>, align 2 (!vplan.execution.frequency 17870283321406128127 (96.88%, estimated))
+; CHECK-NEXT: vp<[[VP7:%[0-9]+]]> = vector-pointer i16, ir<%st.addr>, ir<1>
+; CHECK-NEXT: WIDEN ir<%data> = load vp<[[VP7]]>, vp<%uncountable.exit.mask>, align 2 (!vplan.execution.frequency 17870283321406128127 (96.88%, estimated))
; CHECK-NEXT: WIDEN ir<%inc> = add nsw ir<%data>, ir<1>
-; CHECK-NEXT: WIDEN store vp<[[VP8]]>, ir<%inc>, vp<%uncountable.exit.mask>, align 2 (!vplan.execution.frequency 17870283321406128127 (96.88%, estimated))
-; CHECK-NEXT: EMIT vp<[[VP9:%[0-9]+]]> = any-of vp<[[VP6]]>
+; CHECK-NEXT: WIDEN store vp<[[VP7]]>, ir<%inc>, vp<%uncountable.exit.mask>, align 2 (!vplan.execution.frequency 17870283321406128127 (96.88%, estimated))
+; CHECK-NEXT: EMIT vp<[[VP8:%[0-9]+]]> = any-of vp<[[VP6]]>
; CHECK-NEXT: EMIT vp<%index.next> = add nuw vp<[[VP3]]>, vp<[[VP1]]>
-; CHECK-NEXT: EMIT vp<[[VP10:%[0-9]+]]> = icmp eq vp<%index.next>, vp<[[VP2]]>
-; CHECK-NEXT: EMIT branch-on-two-conds vp<[[VP9]]>, vp<[[VP10]]>
+; CHECK-NEXT: EMIT vp<[[VP9:%[0-9]+]]> = icmp eq vp<%index.next>, vp<[[VP2]]>
+; CHECK-NEXT: EMIT branch-on-two-conds vp<[[VP8]]>, vp<[[VP9]]>
; CHECK-NEXT: No successors
; CHECK-NEXT: }
; CHECK-NEXT: Successor(s): middle.block, middle.block
; CHECK-EMPTY:
; CHECK-NEXT: middle.block:
+; CHECK-NEXT: EMIT vp<[[VP11:%[0-9]+]]> = first-active-lane vp<[[VP6]]>
; CHECK-NEXT: EMIT vp<[[VP12:%[0-9]+]]> = extract-lane ir<0>, ir<%iv>
-; CHECK-NEXT: EMIT vp<[[VP13:%[0-9]+]]> = add vp<[[VP12]]>, vp<[[VP7]]>
+; CHECK-NEXT: EMIT vp<[[VP13:%[0-9]+]]> = add vp<[[VP12]]>, vp<[[VP11]]>
; CHECK-NEXT: EMIT vp<[[VP14:%[0-9]+]]> = icmp eq vp<[[VP13]]>, ir<20>
; CHECK-NEXT: EMIT branch-on-cond vp<[[VP14]]>
; CHECK-NEXT: Successor(s): ir-bb<exit>, scalar.ph
@@ -202,32 +202,32 @@ define i16 @uncountable_exit_with_live_out(ptr dereferenceable(40) noalias %arra
; CHECK-NEXT: WIDEN ir<%ee.val> = load vp<[[VP5]]>, align 2
; CHECK-NEXT: WIDEN ir<%ee.cond> = icmp sgt ir<%ee.val>, ir<500>
; CHECK-NEXT: EMIT vp<[[VP6:%[0-9]+]]> = freeze ir<%ee.cond>
-; CHECK-NEXT: EMIT vp<[[VP7:%[0-9]+]]> = first-active-lane vp<[[VP6]]>
-; CHECK-NEXT: EMIT vp<%uncountable.exit.mask> = active lane mask ir<0>, vp<[[VP7]]>
+; CHECK-NEXT: EMIT vp<%uncountable.exit.mask> = mask-before-first vp<[[VP6]]>
; CHECK-NEXT: CLONE ir<%st.addr> = getelementptr ir<%array>, vp<[[VP4]]>
-; CHECK-NEXT: vp<[[VP8:%[0-9]+]]> = vector-pointer i16, ir<%st.addr>, ir<1>
-; CHECK-NEXT: WIDEN ir<%data> = load vp<[[VP8]]>, vp<%uncountable.exit.mask>, align 2
+; CHECK-NEXT: vp<[[VP7:%[0-9]+]]> = vector-pointer i16, ir<%st.addr>, ir<1>
+; CHECK-NEXT: WIDEN ir<%data> = load vp<[[VP7]]>, vp<%uncountable.exit.mask>, align 2
; CHECK-NEXT: WIDEN ir<%inc> = add nsw ir<%data>, ir<1>
-; CHECK-NEXT: WIDEN store vp<[[VP8]]>, ir<%inc>, vp<%uncountable.exit.mask>, align 2
-; CHECK-NEXT: EMIT vp<[[VP9:%[0-9]+]]> = any-of vp<[[VP6]]>
+; CHECK-NEXT: WIDEN store vp<[[VP7]]>, ir<%inc>, vp<%uncountable.exit.mask>, align 2
+; CHECK-NEXT: EMIT vp<[[VP8:%[0-9]+]]> = any-of vp<[[VP6]]>
; CHECK-NEXT: EMIT vp<%index.next> = add nuw vp<[[VP3]]>, vp<[[VP1]]>
-; CHECK-NEXT: EMIT vp<[[VP10:%[0-9]+]]> = icmp eq vp<%index.next>, vp<[[VP2]]>
-; CHECK-NEXT: EMIT branch-on-two-conds vp<[[VP9]]>, vp<[[VP10]]>
+; CHECK-NEXT: EMIT vp<[[VP9:%[0-9]+]]> = icmp eq vp<%index.next>, vp<[[VP2]]>
+; CHECK-NEXT: EMIT branch-on-two-conds vp<[[VP8]]>, vp<[[VP9]]>
; CHECK-NEXT: No successors
; CHECK-NEXT: }
; CHECK-NEXT: Successor(s): middle.block, middle.block
; CHECK-EMPTY:
; CHECK-NEXT: middle.block:
-; CHECK-NEXT: EMIT vp<[[VP12:%[0-9]+]]> = extract-last-part ir<%data>
-; CHECK-NEXT: EMIT vp<[[VP13:%[0-9]+]]> = extract-last-lane vp<[[VP12]]>
+; CHECK-NEXT: EMIT vp<[[VP11:%[0-9]+]]> = extract-last-part ir<%data>
+; CHECK-NEXT: EMIT vp<[[VP12:%[0-9]+]]> = extract-last-lane vp<[[VP11]]>
+; CHECK-NEXT: EMIT vp<[[VP13:%[0-9]+]]> = first-active-lane vp<[[VP6]]>
; CHECK-NEXT: EMIT vp<[[VP14:%[0-9]+]]> = extract-lane ir<0>, ir<%iv>
-; CHECK-NEXT: EMIT vp<[[VP15:%[0-9]+]]> = add vp<[[VP14]]>, vp<[[VP7]]>
+; CHECK-NEXT: EMIT vp<[[VP15:%[0-9]+]]> = add vp<[[VP14]]>, vp<[[VP13]]>
; CHECK-NEXT: EMIT vp<[[VP16:%[0-9]+]]> = icmp eq vp<[[VP15]]>, ir<20>
; CHECK-NEXT: EMIT branch-on-cond vp<[[VP16]]>
; CHECK-NEXT: Successor(s): ir-bb<exit>, scalar.ph
; CHECK-EMPTY:
; CHECK-NEXT: ir-bb<exit>:
-; CHECK-NEXT: IR %data.lcssa = phi i16 [ %data, %for.inc ], [ %data, %for.body ] (extra operand: vp<[[VP13]]> from middle.block)
+; CHECK-NEXT: IR %data.lcssa = phi i16 [ %data, %for.inc ], [ %data, %for.body ] (extra operand: vp<[[VP12]]> from middle.block)
; CHECK-NEXT: No successors
; CHECK-EMPTY:
; CHECK-NEXT: scalar.ph:
@@ -295,26 +295,26 @@ define void @combined_exit_conditions(ptr align 4 dereferenceable(80) readonly %
; CHECK-NEXT: WIDEN ir<%ee.cmp> = icmp ne ir<%ee.val>, ir<0>
; CHECK-NEXT: EMIT vp<[[VP6:%[0-9]+]]> = freeze ir<%ee.cmp>
; CHECK-NEXT: EMIT vp<[[VP7:%[0-9]+]]> = freeze vp<[[VP6]]>
-; CHECK-NEXT: EMIT vp<[[VP8:%[0-9]+]]> = first-active-lane vp<[[VP7]]>
-; CHECK-NEXT: EMIT vp<%uncountable.exit.mask> = active lane mask ir<0>, vp<[[VP8]]>
+; CHECK-NEXT: EMIT vp<%uncountable.exit.mask> = mask-before-first vp<[[VP7]]>
; CHECK-NEXT: CLONE ir<%src.ptr> = getelementptr ir<%src>, vp<[[VP4]]>
-; CHECK-NEXT: vp<[[VP9:%[0-9]+]]> = vector-pointer i32, ir<%src.ptr>, ir<1>
-; CHECK-NEXT: WIDEN ir<%data> = load vp<[[VP9]]>, vp<%uncountable.exit.mask>, align 4
+; CHECK-NEXT: vp<[[VP8:%[0-9]+]]> = vector-pointer i32, ir<%src.ptr>, ir<1>
+; CHECK-NEXT: WIDEN ir<%data> = load vp<[[VP8]]>, vp<%uncountable.exit.mask>, align 4
; CHECK-NEXT: WIDEN ir<%add> = add nsw ir<%data>, ir<1>
; CHECK-NEXT: CLONE ir<%dst.ptr> = getelementptr ir<%dst>, vp<[[VP4]]>
-; CHECK-NEXT: vp<[[VP10:%[0-9]+]]> = vector-pointer i32, ir<%dst.ptr>, ir<1>
-; CHECK-NEXT: WIDEN store vp<[[VP10]]>, ir<%add>, vp<%uncountable.exit.mask>, align 4
-; CHECK-NEXT: EMIT vp<[[VP11:%[0-9]+]]> = any-of vp<[[VP7]]>
+; CHECK-NEXT: vp<[[VP9:%[0-9]+]]> = vector-pointer i32, ir<%dst.ptr>, ir<1>
+; CHECK-NEXT: WIDEN store vp<[[VP9]]>, ir<%add>, vp<%uncountable.exit.mask>, align 4
+; CHECK-NEXT: EMIT vp<[[VP10:%[0-9]+]]> = any-of vp<[[VP7]]>
; CHECK-NEXT: EMIT vp<%index.next> = add nuw vp<[[VP3]]>, vp<[[VP1]]>
-; CHECK-NEXT: EMIT vp<[[VP12:%[0-9]+]]> = icmp eq vp<%index.next>, vp<[[VP2]]>
-; CHECK-NEXT: EMIT branch-on-two-conds vp<[[VP11]]>, vp<[[VP12]]>
+; CHECK-NEXT: EMIT vp<[[VP11:%[0-9]+]]> = icmp eq vp<%index.next>, vp<[[VP2]]>
+; CHECK-NEXT: EMIT branch-on-two-conds vp<[[VP10]]>, vp<[[VP11]]>
; CHECK-NEXT: No successors
; CHECK-NEXT: }
; CHECK-NEXT: Successor(s): middle.block, middle.block
; CHECK-EMPTY:
; CHECK-NEXT: middle.block:
+; CHECK-NEXT: EMIT vp<[[VP13:%[0-9]+]]> = first-active-lane vp<[[VP7]]>
; CHECK-NEXT: EMIT vp<[[VP14:%[0-9]+]]> = extract-lane ir<0>, ir<%iv>
-; CHECK-NEXT: EMIT vp<[[VP15:%[0-9]+]]> = add vp<[[VP14]]>, vp<[[VP8]]>
+; CHECK-NEXT: EMIT vp<[[VP15:%[0-9]+]]> = add vp<[[VP14]]>, vp<[[VP13]]>
; CHECK-NEXT: EMIT vp<[[VP16:%[0-9]+]]> = icmp eq vp<[[VP15]]>, ir<20>
; CHECK-NEXT: EMIT branch-on-cond vp<[[VP16]]>
; CHECK-NEXT: Successor(s): ir-bb<exit>, scalar.ph
diff --git a/llvm/test/Transforms/LoopVectorize/X86/early_exit_with_stores.ll b/llvm/test/Transforms/LoopVectorize/X86/early_exit_with_stores.ll
index 3635a51a925dad..6fcbe461084bfb 100644
--- a/llvm/test/Transforms/LoopVectorize/X86/early_exit_with_stores.ll
+++ b/llvm/test/Transforms/LoopVectorize/X86/early_exit_with_stores.ll
@@ -17,8 +17,7 @@ define void @interleaved_stores_after_early_exit(ptr noalias %dst, ptr align 2 d
; CHECK-NEXT: [[WIDE_LOAD:%.*]] = load <4 x i16>, ptr [[GEP_SRC]], align 2
; CHECK-NEXT: [[TMP1:%.*]] = icmp sgt <4 x i16> [[WIDE_LOAD]], splat (i16 500)
; CHECK-NEXT: [[TMP2:%.*]] = freeze <4 x i1> [[TMP1]]
-; CHECK-NEXT: [[TMP3:%.*]] = call i64 @llvm.experimental.cttz.elts.i64.v4i1(<4 x i1> [[TMP2]], i1 false)
-; CHECK-NEXT: [[UNCOUNTABLE_EXIT_MASK:%.*]] = call <4 x i1> @llvm.get.active.lane.mask.v4i1.i64(i64 0, i64 [[TMP3]])
+; CHECK-NEXT: [[UNCOUNTABLE_EXIT_MASK:%.*]] = call <4 x i1> @llvm.mask.beforefirst.v4i1(<4 x i1> [[TMP2]])
; CHECK-NEXT: [[TMP4:%.*]] = extractelement <4 x i1> [[UNCOUNTABLE_EXIT_MASK]], i64 0
; CHECK-NEXT: br i1 [[TMP4]], label %[[PRED_STORE_IF:.*]], label %[[PRED_STORE_CONTINUE:.*]]
; CHECK: [[PRED_STORE_IF]]:
@@ -72,6 +71,7 @@ define void @interleaved_stores_after_early_exit(ptr noalias %dst, ptr align 2 d
; CHECK-NEXT: [[TMP29:%.*]] = or i1 [[TMP27]], [[TMP28]]
; CHECK-NEXT: br i1 [[TMP29]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP0:![0-9]+]]
; CHECK: [[MIDDLE_BLOCK]]:
+; CHECK-NEXT: [[TMP3:%.*]] = call i64 @llvm.experimental.cttz.elts.i64.v4i1(<4 x i1> [[TMP2]], i1 false)
; CHECK-NEXT: [[TMP30:%.*]] = add i64 [[IV]], [[TMP3]]
; CHECK-NEXT: [[TMP31:%.*]] = icmp eq i64 [[TMP30]], 20
; CHECK-NEXT: br i1 [[TMP31]], [[EXIT:label %.*]], label %[[SCALAR_PH1:.*]]
@@ -116,8 +116,7 @@ define void @interleaved_stores_before_early_exit(ptr noalias %dst, ptr align 2
; CHECK-NEXT: [[WIDE_LOAD:%.*]] = load <4 x i16>, ptr [[TMP0]], align 2
; CHECK-NEXT: [[TMP1:%.*]] = icmp sgt <4 x i16> [[WIDE_LOAD]], splat (i16 500)
; CHECK-NEXT: [[TMP2:%.*]] = freeze <4 x i1> [[TMP1]]
-; CHECK-NEXT: [[TMP3:%.*]] = call i64 @llvm.experimental.cttz.elts.i64.v4i1(<4 x i1> [[TMP2]], i1 false)
-; CHECK-NEXT: [[UNCOUNTABLE_EXIT_MASK:%.*]] = call <4 x i1> @llvm.get.active.lane.mask.v4i1.i64(i64 0, i64 [[TMP3]])
+; CHECK-NEXT: [[UNCOUNTABLE_EXIT_MASK:%.*]] = call <4 x i1> @llvm.mask.beforefirst.v4i1(<4 x i1> [[TMP2]])
; CHECK-NEXT: [[TMP4:%.*]] = extractelement <4 x i1> [[UNCOUNTABLE_EXIT_MASK]], i64 0
; CHECK-NEXT: br i1 [[TMP4]], label %[[PRED_STORE_IF:.*]], label %[[PRED_STORE_CONTINUE:.*]]
; CHECK: [[PRED_STORE_IF]]:
@@ -171,6 +170,7 @@ define void @interleaved_stores_before_early_exit(ptr noalias %dst, ptr align 2
; CHECK-NEXT: [[TMP29:%.*]] = or i1 [[TMP27]], [[TMP28]]
; CHECK-NEXT: br i1 [[TMP29]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP4:![0-9]+]]
; CHECK: [[MIDDLE_BLOCK]]:
+; CHECK-NEXT: [[TMP3:%.*]] = call i64 @llvm.experimental.cttz.elts.i64.v4i1(<4 x i1> [[TMP2]], i1 false)
; CHECK-NEXT: [[TMP30:%.*]] = add i64 [[IV]], [[TMP3]]
; CHECK-NEXT: [[TMP31:%.*]] = icmp eq i64 [[TMP30]], 20
; CHECK-NEXT: br i1 [[TMP31]], [[EXIT:label %.*]], label %[[SCALAR_PH1:.*]]
diff --git a/llvm/test/Transforms/LoopVectorize/early_exit_combined_exits.ll b/llvm/test/Transforms/LoopVectorize/early_exit_combined_exits.ll
index 385dcc78a5807a..88dfa28a4a7e4d 100644
--- a/llvm/test/Transforms/LoopVectorize/early_exit_combined_exits.ll
+++ b/llvm/test/Transforms/LoopVectorize/early_exit_combined_exits.ll
@@ -15,8 +15,7 @@ define void @combined_exit_conditions(ptr align 4 dereferenceable(80) readonly %
; CHECK-NEXT: [[TMP2:%.*]] = icmp ne <4 x i32> [[WIDE_LOAD]], zeroinitializer
; CHECK-NEXT: [[TMP12:%.*]] = freeze <4 x i1> [[TMP2]]
; CHECK-NEXT: [[TMP6:%.*]] = freeze <4 x i1> [[TMP12]]
-; CHECK-NEXT: [[TMP3:%.*]] = call i64 @llvm.experimental.cttz.elts.i64.v4i1(<4 x i1> [[TMP6]], i1 false)
-; CHECK-NEXT: [[UNCOUNTABLE_EXIT_MASK:%.*]] = call <4 x i1> @llvm.get.active.lane.mask.v4i1.i64(i64 0, i64 [[TMP3]])
+; CHECK-NEXT: [[UNCOUNTABLE_EXIT_MASK:%.*]] = call <4 x i1> @llvm.mask.beforefirst.v4i1(<4 x i1> [[TMP6]])
; CHECK-NEXT: [[TMP0:%.*]] = getelementptr [4 x i8], ptr [[SRC]], i64 [[INDEX]]
; CHECK-NEXT: [[WIDE_MASKED_LOAD:%.*]] = call <4 x i32> @llvm.masked.load.v4i32.p0(ptr align 4 [[TMP0]], <4 x i1> [[UNCOUNTABLE_EXIT_MASK]], <4 x i32> poison)
; CHECK-NEXT: [[TMP4:%.*]] = add nsw <4 x i32> [[WIDE_MASKED_LOAD]], splat (i32 1)
@@ -28,6 +27,7 @@ define void @combined_exit_conditions(ptr align 4 dereferenceable(80) readonly %
; CHECK-NEXT: [[TMP9:%.*]] = or i1 [[TMP7]], [[TMP8]]
; CHECK-NEXT: br i1 [[TMP9]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP0:![0-9]+]]
; CHECK: [[MIDDLE_BLOCK]]:
+; CHECK-NEXT: [[TMP3:%.*]] = call i64 @llvm.experimental.cttz.elts.i64.v4i1(<4 x i1> [[TMP6]], i1 false)
; CHECK-NEXT: [[TMP10:%.*]] = add i64 [[INDEX]], [[TMP3]]
; CHECK-NEXT: [[TMP11:%.*]] = icmp eq i64 [[TMP10]], 20
; CHECK-NEXT: br i1 [[TMP11]], label %[[EXIT:.*]], label %[[SCALAR_PH:.*]]
@@ -86,8 +86,7 @@ define void @combined_exit_conditions_swap_comparison_order(ptr align 4 derefere
; CHECK-NEXT: [[TMP1:%.*]] = icmp ne <4 x i32> [[WIDE_LOAD]], zeroinitializer
; CHECK-NEXT: [[TMP12:%.*]] = freeze <4 x i1> [[TMP1]]
; CHECK-NEXT: [[TMP6:%.*]] = freeze <4 x i1> [[TMP12]]
-; CHECK-NEXT: [[TMP2:%.*]] = call i64 @llvm.experimental.cttz.elts.i64.v4i1(<4 x i1> [[TMP6]], i1 false)
-; CHECK-NEXT: [[UNCOUNTABLE_EXIT_MASK:%.*]] = call <4 x i1> @llvm.get.active.lane.mask.v4i1.i64(i64 0, i64 [[TMP2]])
+; CHECK-NEXT: [[UNCOUNTABLE_EXIT_MASK:%.*]] = call <4 x i1> @llvm.mask.beforefirst.v4i1(<4 x i1> [[TMP6]])
; CHECK-NEXT: [[TMP3:%.*]] = getelementptr [4 x i8], ptr [[SRC]], i64 [[INDEX]]
; CHECK-NEXT: [[WIDE_MASKED_LOAD:%.*]] = call <4 x i32> @llvm.masked.load.v4i32.p0(ptr align 4 [[TMP3]], <4 x i1> [[UNCOUNTABLE_EXIT_MASK]], <4 x i32> poison)
; CHECK-NEXT: [[TMP4:%.*]] = add nsw <4 x i32> [[WIDE_MASKED_LOAD]], splat (i32 1)
@@ -99,6 +98,7 @@ define void @combined_exit_conditions_swap_comparison_order(ptr align 4 derefere
; CHECK-NEXT: [[TMP9:%.*]] = or i1 [[TMP7]], [[TMP8]]
; CHECK-NEXT: br i1 [[TMP9]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP4:![0-9]+]]
; CHECK: [[MIDDLE_BLOCK]]:
+; CHECK-NEXT: [[TMP2:%.*]] = call i64 @llvm.experimental.cttz.elts.i64.v4i1(<4 x i1> [[TMP6]], i1 false)
; CHECK-NEXT: [[TMP10:%.*]] = add i64 [[INDEX]], [[TMP2]]
; CHECK-NEXT: [[TMP11:%.*]] = icmp eq i64 [[TMP10]], 20
; CHECK-NEXT: br i1 [[TMP11]], label %[[EXIT:.*]], label %[[SCALAR_PH:.*]]
@@ -386,8 +386,7 @@ define void @combined_exit_conditions_binary_or(ptr align 4 dereferenceable(80)
; CHECK-NEXT: [[WIDE_LOAD:%.*]] = load <4 x i32>, ptr [[TMP0]], align 4
; CHECK-NEXT: [[TMP1:%.*]] = icmp ne <4 x i32> [[WIDE_LOAD]], zeroinitializer
; CHECK-NEXT: [[TMP6:%.*]] = freeze <4 x i1> [[TMP1]]
-; CHECK-NEXT: [[TMP2:%.*]] = call i64 @llvm.experimental.cttz.elts.i64.v4i1(<4 x i1> [[TMP6]], i1 false)
-; CHECK-NEXT: [[UNCOUNTABLE_EXIT_MASK:%.*]] = call <4 x i1> @llvm.get.active.lane.mask.v4i1.i64(i64 0, i64 [[TMP2]])
+; CHECK-NEXT: [[UNCOUNTABLE_EXIT_MASK:%.*]] = call <4 x i1> @llvm.mask.beforefirst.v4i1(<4 x i1> [[TMP6]])
; CHECK-NEXT: [[TMP3:%.*]] = getelementptr [4 x i8], ptr [[SRC]], i64 [[INDEX]]
; CHECK-NEXT: [[WIDE_MASKED_LOAD:%.*]] = call <4 x i32> @llvm.masked.load.v4i32.p0(ptr align 4 [[TMP3]], <4 x i1> [[UNCOUNTABLE_EXIT_MASK]], <4 x i32> poison)
; CHECK-NEXT: [[TMP4:%.*]] = add nsw <4 x i32> [[WIDE_MASKED_LOAD]], splat (i32 1)
@@ -399,6 +398,7 @@ define void @combined_exit_conditions_binary_or(ptr align 4 dereferenceable(80)
; CHECK-NEXT: [[TMP9:%.*]] = or i1 [[TMP7]], [[TMP8]]
; CHECK-NEXT: br i1 [[TMP9]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP6:![0-9]+]]
; CHECK: [[MIDDLE_BLOCK]]:
+; CHECK-NEXT: [[TMP2:%.*]] = call i64 @llvm.experimental.cttz.elts.i64.v4i1(<4 x i1> [[TMP6]], i1 false)
; CHECK-NEXT: [[TMP10:%.*]] = add i64 [[INDEX]], [[TMP2]]
; CHECK-NEXT: [[TMP11:%.*]] = icmp eq i64 [[TMP10]], 20
; CHECK-NEXT: br i1 [[TMP11]], label %[[EXIT:.*]], label %[[SCALAR_PH:.*]]
@@ -819,8 +819,7 @@ define void @combined_exit_condition_has_two_users(ptr align 4 dereferenceable(8
; CHECK-NEXT: [[TMP1:%.*]] = icmp ne <4 x i32> [[WIDE_LOAD]], zeroinitializer
; CHECK-NEXT: [[TMP2:%.*]] = freeze <4 x i1> [[TMP1]]
; CHECK-NEXT: [[TMP3:%.*]] = freeze <4 x i1> [[TMP2]]
-; CHECK-NEXT: [[TMP4:%.*]] = call i64 @llvm.experimental.cttz.elts.i64.v4i1(<4 x i1> [[TMP3]], i1 false)
-; CHECK-NEXT: [[UNCOUNTABLE_EXIT_MASK:%.*]] = call <4 x i1> @llvm.get.active.lane.mask.v4i1.i64(i64 0, i64 [[TMP4]])
+; CHECK-NEXT: [[UNCOUNTABLE_EXIT_MASK:%.*]] = call <4 x i1> @llvm.mask.beforefirst.v4i1(<4 x i1> [[TMP3]])
; CHECK-NEXT: [[TMP5:%.*]] = add nuw nsw <4 x i64> [[VEC_IND]], splat (i64 1)
; CHECK-NEXT: [[TMP6:%.*]] = icmp eq <4 x i64> [[TMP5]], splat (i64 20)
; CHECK-NEXT: [[TMP7:%.*]] = select <4 x i1> [[TMP1]], <4 x i1> splat (i1 true), <4 x i1> [[TMP6]]
@@ -834,6 +833,7 @@ define void @combined_exit_condition_has_two_users(ptr align 4 dereferenceable(8
; CHECK-NEXT: [[TMP12:%.*]] = or i1 [[TMP10]], [[TMP11]]
; CHECK-NEXT: br i1 [[TMP12]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP8:![0-9]+]]
; CHECK: [[MIDDLE_BLOCK]]:
+; CHECK-NEXT: [[TMP4:%.*]] = call i64 @llvm.experimental.cttz.elts.i64.v4i1(<4 x i1> [[TMP3]], i1 false)
; CHECK-NEXT: [[TMP13:%.*]] = add i64 [[INDEX]], [[TMP4]]
; CHECK-NEXT: [[TMP14:%.*]] = icmp eq i64 [[TMP13]], 20
; CHECK-NEXT: br i1 [[TMP14]], label %[[EXIT:.*]], label %[[SCALAR_PH:.*]]
diff --git a/llvm/test/Transforms/LoopVectorize/early_exit_combined_exits_epilogue.ll b/llvm/test/Transforms/LoopVectorize/early_exit_combined_exits_epilogue.ll
index c7d7e4637da1f6..be4afea9ab902d 100644
--- a/llvm/test/Transforms/LoopVectorize/early_exit_combined_exits_epilogue.ll
+++ b/llvm/test/Transforms/LoopVectorize/early_exit_combined_exits_epilogue.ll
@@ -15,8 +15,7 @@ define void @combined_exit_conditions(ptr align 4 dereferenceable(80) readonly %
; CHECK-NEXT: [[TMP1:%.*]] = icmp ne <8 x i32> [[WIDE_LOAD]], zeroinitializer
; CHECK-NEXT: [[TMP2:%.*]] = freeze <8 x i1> [[TMP1]]
; CHECK-NEXT: [[TMP7:%.*]] = freeze <8 x i1> [[TMP2]]
-; CHECK-NEXT: [[TMP3:%.*]] = call i64 @llvm.experimental.cttz.elts.i64.v8i1(<8 x i1> [[TMP7]], i1 false)
-; CHECK-NEXT: [[UNCOUNTABLE_EXIT_MASK:%.*]] = call <8 x i1> @llvm.get.active.lane.mask.v8i1.i64(i64 0, i64 [[TMP3]])
+; CHECK-NEXT: [[UNCOUNTABLE_EXIT_MASK:%.*]] = call <8 x i1> @llvm.mask.beforefirst.v8i1(<8 x i1> [[TMP7]])
; CHECK-NEXT: [[TMP4:%.*]] = getelementptr [4 x i8], ptr [[SRC]], i64 [[INDEX]]
; CHECK-NEXT: [[WIDE_MASKED_LOAD:%.*]] = call <8 x i32> @llvm.masked.load.v8i32.p0(ptr align 4 [[TMP4]], <8 x i1> [[UNCOUNTABLE_EXIT_MASK]], <8 x i32> poison)
; CHECK-NEXT: [[TMP5:%.*]] = add nsw <8 x i32> [[WIDE_MASKED_LOAD]], splat (i32 1)
@@ -28,6 +27,7 @@ define void @combined_exit_conditions(ptr align 4 dereferenceable(80) readonly %
; CHECK-NEXT: [[TMP10:%.*]] = or i1 [[TMP8]], [[TMP9]]
; CHECK-NEXT: br i1 [[TMP10]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP0:![0-9]+]]
; CHECK: [[MIDDLE_BLOCK]]:
+; CHECK-NEXT: [[TMP3:%.*]] = call i64 @llvm.experimental.cttz.elts.i64.v8i1(<8 x i1> [[TMP7]], i1 false)
; CHECK-NEXT: [[TMP11:%.*]] = add i64 [[INDEX]], [[TMP3]]
; CHECK-NEXT: [[TMP12:%.*]] = icmp eq i64 [[TMP11]], 20
; CHECK-NEXT: br i1 [[TMP12]], label %[[EXIT:.*]], label %[[SCALAR_PH:.*]]
diff --git a/llvm/test/Transforms/LoopVectorize/early_exit_with_stores.ll b/llvm/test/Transforms/LoopVectorize/early_exit_with_stores.ll
index 87716393117700..ad1269aa3c8523 100644
--- a/llvm/test/Transforms/LoopVectorize/early_exit_with_stores.ll
+++ b/llvm/test/Transforms/LoopVectorize/early_exit_with_stores.ll
@@ -60,8 +60,7 @@ define void @loop_contains_store_condition_load_has_single_user(ptr dereferencea
; CHECK-NEXT: [[WIDE_LOAD:%.*]] = load <4 x i16>, ptr [[TMP12]], align 2
; CHECK-NEXT: [[TMP13:%.*]] = icmp sgt <4 x i16> [[WIDE_LOAD]], splat (i16 500)
; CHECK-NEXT: [[TMP40:%.*]] = freeze <4 x i1> [[TMP13]]
-; CHECK-NEXT: [[TMP14:%.*]] = call i64 @llvm.experimental.cttz.elts.i64.v4i1(<4 x i1> [[TMP40]], i1 false)
-; CHECK-NEXT: [[UNCOUNTABLE_EXIT_MASK:%.*]] = call <4 x i1> @llvm.get.active.lane.mask.v4i1.i64(i64 0, i64 [[TMP14]])
+; CHECK-NEXT: [[UNCOUNTABLE_EXIT_MASK:%.*]] = call <4 x i1> @llvm.mask.beforefirst.v4i1(<4 x i1> [[TMP40]])
; CHECK-NEXT: [[TMP7:%.*]] = getelementptr i16, ptr [[ARRAY]], i64 [[TMP3]]
; CHECK-NEXT: [[TMP30:%.*]] = call <4 x i16> @llvm.masked.load.v4i16.p0(ptr align 2 [[TMP7]], <4 x i1> [[UNCOUNTABLE_EXIT_MASK]], <4 x i16> poison)
; CHECK-NEXT: [[TMP31:%.*]] = add nsw <4 x i16> [[TMP30]], splat (i16 1)
@@ -72,6 +71,7 @@ define void @loop_contains_store_condition_load_has_single_user(ptr dereferencea
; CHECK-NEXT: [[TMP43:%.*]] = or i1 [[TMP41]], [[TMP42]]
; CHECK-NEXT: br i1 [[TMP43]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP0:![0-9]+]]
; CHECK: [[MIDDLE_BLOCK]]:
+; CHECK-NEXT: [[TMP14:%.*]] = call i64 @llvm.experimental.cttz.elts.i64.v4i1(<4 x i1> [[TMP40]], i1 false)
; CHECK-NEXT: [[TMP44:%.*]] = add i64 [[TMP3]], [[TMP14]]
; CHECK-NEXT: [[TMP45:%.*]] = icmp eq i64 [[TMP44]], 20
; CHECK-NEXT: br i1 [[TMP45]], label %[[EXIT:.*]], label %[[SCALAR_PH1:.*]]
@@ -130,8 +130,7 @@ define void @loop_contains_store_fcmp_condition(ptr dereferenceable(40) noalias
; CHECK-NEXT: [[WIDE_LOAD:%.*]] = load <4 x half>, ptr [[TMP0]], align 2
; CHECK-NEXT: [[TMP1:%.*]] = fcmp ugt <4 x half> [[WIDE_LOAD]], splat (half 5.000000e+02)
; CHECK-NEXT: [[TMP2:%.*]] = freeze <4 x i1> [[TMP1]]
-; CHECK-NEXT: [[TMP3:%.*]] = call i64 @llvm.experimental.cttz.elts.i64.v4i1(<4 x i1> [[TMP2]], i1 false)
-; CHECK-NEXT: [[UNCOUNTABLE_EXIT_MASK:%.*]] = call <4 x i1> @llvm.get.active.lane.mask.v4i1.i64(i64 0, i64 [[TMP3]])
+; CHECK-NEXT: [[UNCOUNTABLE_EXIT_MASK:%.*]] = call <4 x i1> @llvm.mask.beforefirst.v4i1(<4 x i1> [[TMP2]])
; CHECK-NEXT: [[TMP4:%.*]] = getelementptr i16, ptr [[ARRAY]], i64 [[INDEX]]
; CHECK-NEXT: [[WIDE_MASKED_LOAD:%.*]] = call <4 x i16> @llvm.masked.load.v4i16.p0(ptr align 2 [[TMP4]], <4 x i1> [[UNCOUNTABLE_EXIT_MASK]], <4 x i16> poison)
; CHECK-NEXT: [[TMP5:%.*]] = add nsw <4 x i16> [[WIDE_MASKED_LOAD]], splat (i16 1)
@@ -142,6 +141,7 @@ define void @loop_contains_store_fcmp_condition(ptr dereferenceable(40) noalias
; CHECK-NEXT: [[TMP8:%.*]] = or i1 [[TMP6]], [[TMP7]]
; CHECK-NEXT: br i1 [[TMP8]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP4:![0-9]+]]
; CHECK: [[MIDDLE_BLOCK]]:
+; CHECK-NEXT: [[TMP3:%.*]] = call i64 @llvm.experimental.cttz.elts.i64.v4i1(<4 x i1> [[TMP2]], i1 false)
; CHECK-NEXT: [[TMP9:%.*]] = add i64 [[INDEX]], [[TMP3]]
; CHECK-NEXT: [[TMP10:%.*]] = icmp eq i64 [[TMP9]], 20
; CHECK-NEXT: br i1 [[TMP10]], label %[[EXIT:.*]], label %[[SCALAR_PH:.*]]
@@ -357,8 +357,7 @@ define void @loop_contains_store_to_pointer_with_no_deref_info(ptr align 2 deref
; CHECK-NEXT: [[WIDE_LOAD:%.*]] = load <4 x i16>, ptr [[TMP4]], align 2
; CHECK-NEXT: [[TMP5:%.*]] = icmp sgt <4 x i16> [[WIDE_LOAD]], splat (i16 500)
; CHECK-NEXT: [[TMP40:%.*]] = freeze <4 x i1> [[TMP5]]
-; CHECK-NEXT: [[TMP6:%.*]] = call i64 @llvm.experimental.cttz.elts.i64.v4i1(<4 x i1> [[TMP40]], i1 false)
-; CHECK-NEXT: [[UNCOUNTABLE_EXIT_MASK:%.*]] = call <4 x i1> @llvm.get.active.lane.mask.v4i1.i64(i64 0, i64 [[TMP6]])
+; CHECK-NEXT: [[UNCOUNTABLE_EXIT_MASK:%.*]] = call <4 x i1> @llvm.mask.beforefirst.v4i1(<4 x i1> [[TMP40]])
; CHECK-NEXT: [[TMP0:%.*]] = getelementptr i16, ptr [[LOAD_ARRAY]], i64 [[INDEX]]
; CHECK-NEXT: [[TMP26:%.*]] = call <4 x i16> @llvm.masked.load.v4i16.p0(ptr align 2 [[TMP0]], <4 x i1> [[UNCOUNTABLE_EXIT_MASK]], <4 x i16> poison)
; CHECK-NEXT: [[TMP27:%.*]] = add nsw <4 x i16> [[TMP26]], splat (i16 1)
@@ -370,6 +369,7 @@ define void @loop_contains_store_to_pointer_with_no_deref_info(ptr align 2 deref
; CHECK-NEXT: [[TMP43:%.*]] = or i1 [[TMP41]], [[TMP42]]
; CHECK-NEXT: br i1 [[TMP43]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP6:![0-9]+]]
; CHECK: [[MIDDLE_BLOCK]]:
+; CHECK-NEXT: [[TMP6:%.*]] = call i64 @llvm.experimental.cttz.elts.i64.v4i1(<4 x i1> [[TMP40]], i1 false)
; CHECK-NEXT: [[TMP44:%.*]] = add i64 [[INDEX]], [[TMP6]]
; CHECK-NEXT: [[TMP45:%.*]] = icmp eq i64 [[TMP44]], 20
; CHECK-NEXT: br i1 [[TMP45]], label %[[EXIT:.*]], label %[[SCALAR_PH1:.*]]
@@ -475,8 +475,7 @@ define void @loop_contains_store_in_latch_block(ptr dereferenceable(40) noalias
; CHECK-NEXT: [[WIDE_LOAD:%.*]] = load <4 x i16>, ptr [[EE_ADDR]], align 2
; CHECK-NEXT: [[TMP5:%.*]] = icmp sgt <4 x i16> [[WIDE_LOAD]], splat (i16 500)
; CHECK-NEXT: [[TMP40:%.*]] = freeze <4 x i1> [[TMP5]]
-; CHECK-NEXT: [[TMP6:%.*]] = call i64 @llvm.experimental.cttz.elts.i64.v4i1(<4 x i1> [[TMP40]], i1 false)
-; CHECK-NEXT: [[UNCOUNTABLE_EXIT_MASK:%.*]] = call <4 x i1> @llvm.get.active.lane.mask.v4i1.i64(i64 0, i64 [[TMP6]])
+; CHECK-NEXT: [[UNCOUNTABLE_EXIT_MASK:%.*]] = call <4 x i1> @llvm.mask.beforefirst.v4i1(<4 x i1> [[TMP40]])
; CHECK-NEXT: [[TMP3:%.*]] = getelementptr i16, ptr [[ARRAY]], i64 [[INDEX]]
; CHECK-NEXT: [[TMP30:%.*]] = call <4 x i16> @llvm.masked.load.v4i16.p0(ptr align 2 [[TMP3]], <4 x i1> [[UNCOUNTABLE_EXIT_MASK]], <4 x i16> poison)
; CHECK-NEXT: [[TMP31:%.*]] = add nsw <4 x i16> [[TMP30]], splat (i16 1)
@@ -487,6 +486,7 @@ define void @loop_contains_store_in_latch_block(ptr dereferenceable(40) noalias
; CHECK-NEXT: [[TMP43:%.*]] = or i1 [[TMP41]], [[TMP42]]
; CHECK-NEXT: br i1 [[TMP43]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP8:![0-9]+]]
; CHECK: [[MIDDLE_BLOCK]]:
+; CHECK-NEXT: [[TMP6:%.*]] = call i64 @llvm.experimental.cttz.elts.i64.v4i1(<4 x i1> [[TMP40]], i1 false)
; CHECK-NEXT: [[TMP44:%.*]] = add i64 [[INDEX]], [[TMP6]]
; CHECK-NEXT: [[TMP45:%.*]] = icmp eq i64 [[TMP44]], 20
; CHECK-NEXT: br i1 [[TMP45]], label %[[EXIT:.*]], label %[[SCALAR_PH1:.*]]
@@ -1051,8 +1051,7 @@ define i16 @uncountable_exit_with_live_out(ptr dereferenceable(40) noalias %arra
; CHECK-NEXT: [[WIDE_LOAD:%.*]] = load <4 x i16>, ptr [[TMP12]], align 2
; CHECK-NEXT: [[TMP13:%.*]] = icmp sgt <4 x i16> [[WIDE_LOAD]], splat (i16 500)
; CHECK-NEXT: [[TMP40:%.*]] = freeze <4 x i1> [[TMP13]]
-; CHECK-NEXT: [[TMP14:%.*]] = call i64 @llvm.experimental.cttz.elts.i64.v4i1(<4 x i1> [[TMP40]], i1 false)
-; CHECK-NEXT: [[UNCOUNTABLE_EXIT_MASK:%.*]] = call <4 x i1> @llvm.get.active.lane.mask.v4i1.i64(i64 0, i64 [[TMP14]])
+; CHECK-NEXT: [[UNCOUNTABLE_EXIT_MASK:%.*]] = call <4 x i1> @llvm.mask.beforefirst.v4i1(<4 x i1> [[TMP40]])
; CHECK-NEXT: [[TMP7:%.*]] = getelementptr i16, ptr [[ARRAY]], i64 [[TMP3]]
; CHECK-NEXT: [[TMP30:%.*]] = call <4 x i16> @llvm.masked.load.v4i16.p0(ptr align 2 [[TMP7]], <4 x i1> [[UNCOUNTABLE_EXIT_MASK]], <4 x i16> poison)
; CHECK-NEXT: [[TMP31:%.*]] = add nsw <4 x i16> [[TMP30]], splat (i16 1)
@@ -1064,6 +1063,7 @@ define i16 @uncountable_exit_with_live_out(ptr dereferenceable(40) noalias %arra
; CHECK-NEXT: br i1 [[TMP43]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP10:![0-9]+]]
; CHECK: [[MIDDLE_BLOCK]]:
; CHECK-NEXT: [[TMP9:%.*]] = extractelement <4 x i16> [[TMP30]], i64 3
+; CHECK-NEXT: [[TMP14:%.*]] = call i64 @llvm.experimental.cttz.elts.i64.v4i1(<4 x i1> [[TMP40]], i1 false)
; CHECK-NEXT: [[TMP45:%.*]] = add i64 [[TMP3]], [[TMP14]]
; CHECK-NEXT: [[TMP46:%.*]] = icmp eq i64 [[TMP45]], 20
; CHECK-NEXT: br i1 [[TMP46]], label %[[EXIT:.*]], label %[[SCALAR_PH:.*]]
@@ -1123,8 +1123,7 @@ define i64 @uncountable_exit_with_live_out_iv(ptr dereferenceable(40) noalias %a
; CHECK-NEXT: [[WIDE_LOAD:%.*]] = load <4 x i16>, ptr [[TMP0]], align 2
; CHECK-NEXT: [[TMP1:%.*]] = icmp sgt <4 x i16> [[WIDE_LOAD]], splat (i16 500)
; CHECK-NEXT: [[TMP5:%.*]] = freeze <4 x i1> [[TMP1]]
-; CHECK-NEXT: [[TMP2:%.*]] = call i64 @llvm.experimental.cttz.elts.i64.v4i1(<4 x i1> [[TMP5]], i1 false)
-; CHECK-NEXT: [[UNCOUNTABLE_EXIT_MASK:%.*]] = call <4 x i1> @llvm.get.active.lane.mask.v4i1.i64(i64 0, i64 [[TMP2]])
+; CHECK-NEXT: [[UNCOUNTABLE_EXIT_MASK:%.*]] = call <4 x i1> @llvm.mask.beforefirst.v4i1(<4 x i1> [[TMP5]])
; CHECK-NEXT: [[TMP3:%.*]] = getelementptr i16, ptr [[ARRAY]], i64 [[INDEX]]
; CHECK-NEXT: [[WIDE_MASKED_LOAD:%.*]] = call <4 x i16> @llvm.masked.load.v4i16.p0(ptr align 2 [[TMP3]], <4 x i1> [[UNCOUNTABLE_EXIT_MASK]], <4 x i16> poison)
; CHECK-NEXT: [[TMP4:%.*]] = add nsw <4 x i16> [[WIDE_MASKED_LOAD]], splat (i16 1)
@@ -1135,6 +1134,7 @@ define i64 @uncountable_exit_with_live_out_iv(ptr dereferenceable(40) noalias %a
; CHECK-NEXT: [[TMP8:%.*]] = or i1 [[TMP6]], [[TMP7]]
; CHECK-NEXT: br i1 [[TMP8]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP12:![0-9]+]]
; CHECK: [[MIDDLE_BLOCK]]:
+; CHECK-NEXT: [[TMP2:%.*]] = call i64 @llvm.experimental.cttz.elts.i64.v4i1(<4 x i1> [[TMP5]], i1 false)
; CHECK-NEXT: [[TMP9:%.*]] = add i64 [[INDEX]], [[TMP2]]
; CHECK-NEXT: [[TMP10:%.*]] = icmp eq i64 [[TMP9]], 20
; CHECK-NEXT: br i1 [[TMP10]], label %[[EXIT:.*]], label %[[SCALAR_PH:.*]]
@@ -1284,8 +1284,7 @@ define i32 @uncountable_exit_with_separate_exit_block(ptr dereferenceable(40) no
; CHECK-NEXT: [[WIDE_LOAD:%.*]] = load <4 x i16>, ptr [[TMP12]], align 2
; CHECK-NEXT: [[TMP13:%.*]] = icmp sgt <4 x i16> [[WIDE_LOAD]], splat (i16 500)
; CHECK-NEXT: [[TMP40:%.*]] = freeze <4 x i1> [[TMP13]]
-; CHECK-NEXT: [[TMP14:%.*]] = call i64 @llvm.experimental.cttz.elts.i64.v4i1(<4 x i1> [[TMP40]], i1 false)
-; CHECK-NEXT: [[UNCOUNTABLE_EXIT_MASK:%.*]] = call <4 x i1> @llvm.get.active.lane.mask.v4i1.i64(i64 0, i64 [[TMP14]])
+; CHECK-NEXT: [[UNCOUNTABLE_EXIT_MASK:%.*]] = call <4 x i1> @llvm.mask.beforefirst.v4i1(<4 x i1> [[TMP40]])
; CHECK-NEXT: [[TMP7:%.*]] = getelementptr i16, ptr [[ARRAY]], i64 [[TMP3]]
; CHECK-NEXT: [[TMP30:%.*]] = call <4 x i16> @llvm.masked.load.v4i16.p0(ptr align 2 [[TMP7]], <4 x i1> [[UNCOUNTABLE_EXIT_MASK]], <4 x i16> poison)
; CHECK-NEXT: [[TMP31:%.*]] = add nsw <4 x i16> [[TMP30]], splat (i16 1)
@@ -1296,6 +1295,7 @@ define i32 @uncountable_exit_with_separate_exit_block(ptr dereferenceable(40) no
; CHECK-NEXT: [[TMP43:%.*]] = or i1 [[TMP41]], [[TMP42]]
; CHECK-NEXT: br i1 [[TMP43]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP14:![0-9]+]]
; CHECK: [[MIDDLE_BLOCK]]:
+; CHECK-NEXT: [[TMP14:%.*]] = call i64 @llvm.experimental.cttz.elts.i64.v4i1(<4 x i1> [[TMP40]], i1 false)
; CHECK-NEXT: [[TMP44:%.*]] = add i64 [[TMP3]], [[TMP14]]
; CHECK-NEXT: [[TMP45:%.*]] = icmp eq i64 [[TMP44]], 20
; CHECK-NEXT: br i1 [[TMP45]], label %[[EXIT_COUNTABLE:.*]], label %[[SCALAR_PH:.*]]
@@ -1427,8 +1427,7 @@ define void @swapped_cmp_operands(ptr dereferenceable(40) noalias %array, ptr al
; CHECK-NEXT: [[WIDE_LOAD:%.*]] = load <4 x i16>, ptr [[TMP1]], align 2
; CHECK-NEXT: [[TMP2:%.*]] = icmp slt <4 x i16> splat (i16 500), [[WIDE_LOAD]]
; CHECK-NEXT: [[TMP5:%.*]] = freeze <4 x i1> [[TMP2]]
-; CHECK-NEXT: [[TMP3:%.*]] = call i64 @llvm.experimental.cttz.elts.i64.v4i1(<4 x i1> [[TMP5]], i1 false)
-; CHECK-NEXT: [[UNCOUNTABLE_EXIT_MASK:%.*]] = call <4 x i1> @llvm.get.active.lane.mask.v4i1.i64(i64 0, i64 [[TMP3]])
+; CHECK-NEXT: [[UNCOUNTABLE_EXIT_MASK:%.*]] = call <4 x i1> @llvm.mask.beforefirst.v4i1(<4 x i1> [[TMP5]])
; CHECK-NEXT: [[TMP0:%.*]] = getelementptr i16, ptr [[ARRAY]], i64 [[INDEX]]
; CHECK-NEXT: [[WIDE_MASKED_LOAD:%.*]] = call <4 x i16> @llvm.masked.load.v4i16.p0(ptr align 2 [[TMP0]], <4 x i1> [[UNCOUNTABLE_EXIT_MASK]], <4 x i16> poison)
; CHECK-NEXT: [[TMP4:%.*]] = add nsw <4 x i16> [[WIDE_MASKED_LOAD]], splat (i16 1)
@@ -1439,6 +1438,7 @@ define void @swapped_cmp_operands(ptr dereferenceable(40) noalias %array, ptr al
; CHECK-NEXT: [[TMP8:%.*]] = or i1 [[TMP6]], [[TMP7]]
; CHECK-NEXT: br i1 [[TMP8]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP16:![0-9]+]]
; CHECK: [[MIDDLE_BLOCK]]:
+; CHECK-NEXT: [[TMP3:%.*]] = call i64 @llvm.experimental.cttz.elts.i64.v4i1(<4 x i1> [[TMP5]], i1 false)
; CHECK-NEXT: [[TMP9:%.*]] = add i64 [[INDEX]], [[TMP3]]
; CHECK-NEXT: [[TMP10:%.*]] = icmp eq i64 [[TMP9]], 20
; CHECK-NEXT: br i1 [[TMP10]], label %[[EXIT:.*]], label %[[SCALAR_PH:.*]]
@@ -1497,8 +1497,7 @@ define void @loop_contains_store_condition_load_has_two_users(ptr dereferenceabl
; CHECK-NEXT: [[WIDE_LOAD:%.*]] = load <4 x i16>, ptr [[TMP0]], align 2
; CHECK-NEXT: [[TMP1:%.*]] = icmp sgt <4 x i16> [[WIDE_LOAD]], splat (i16 500)
; CHECK-NEXT: [[TMP2:%.*]] = freeze <4 x i1> [[TMP1]]
-; CHECK-NEXT: [[TMP3:%.*]] = call i64 @llvm.experimental.cttz.elts.i64.v4i1(<4 x i1> [[TMP2]], i1 false)
-; CHECK-NEXT: [[UNCOUNTABLE_EXIT_MASK:%.*]] = call <4 x i1> @llvm.get.active.lane.mask.v4i1.i64(i64 0, i64 [[TMP3]])
+; CHECK-NEXT: [[UNCOUNTABLE_EXIT_MASK:%.*]] = call <4 x i1> @llvm.mask.beforefirst.v4i1(<4 x i1> [[TMP2]])
; CHECK-NEXT: [[TMP4:%.*]] = getelementptr i16, ptr [[ARRAY]], i64 [[INDEX]]
; CHECK-NEXT: [[WIDE_MASKED_LOAD:%.*]] = call <4 x i16> @llvm.masked.load.v4i16.p0(ptr align 2 [[TMP4]], <4 x i1> [[UNCOUNTABLE_EXIT_MASK]], <4 x i16> poison)
; CHECK-NEXT: [[TMP5:%.*]] = add nsw <4 x i16> [[WIDE_MASKED_LOAD]], [[WIDE_LOAD]]
@@ -1509,6 +1508,7 @@ define void @loop_contains_store_condition_load_has_two_users(ptr dereferenceabl
; CHECK-NEXT: [[TMP8:%.*]] = or i1 [[TMP6]], [[TMP7]]
; CHECK-NEXT: br i1 [[TMP8]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP18:![0-9]+]]
; CHECK: [[MIDDLE_BLOCK]]:
+; CHECK-NEXT: [[TMP3:%.*]] = call i64 @llvm.experimental.cttz.elts.i64.v4i1(<4 x i1> [[TMP2]], i1 false)
; CHECK-NEXT: [[TMP9:%.*]] = add i64 [[INDEX]], [[TMP3]]
; CHECK-NEXT: [[TMP10:%.*]] = icmp eq i64 [[TMP9]], 20
; CHECK-NEXT: br i1 [[TMP10]], label %[[EXIT:.*]], label %[[SCALAR_PH:.*]]
diff --git a/llvm/test/Transforms/LoopVectorize/interleave_uncountable_exits.ll b/llvm/test/Transforms/LoopVectorize/interleave_uncountable_exits.ll
index 8637897644caee..d3dae6a8c9651f 100644
--- a/llvm/test/Transforms/LoopVectorize/interleave_uncountable_exits.ll
+++ b/llvm/test/Transforms/LoopVectorize/interleave_uncountable_exits.ll
@@ -16,8 +16,7 @@ define void @loop_contains_store_condition_load_has_single_user(ptr dereferencea
; CHECK-NEXT: [[WIDE_LOAD:%.*]] = load <4 x i16>, ptr [[TMP1]], align 2
; CHECK-NEXT: [[TMP2:%.*]] = icmp sgt <4 x i16> [[WIDE_LOAD]], splat (i16 500)
; CHECK-NEXT: [[TMP5:%.*]] = freeze <4 x i1> [[TMP2]]
-; CHECK-NEXT: [[TMP3:%.*]] = call i64 @llvm.experimental.cttz.elts.i64.v4i1(<4 x i1> [[TMP5]], i1 false)
-; CHECK-NEXT: [[UNCOUNTABLE_EXIT_MASK:%.*]] = call <4 x i1> @llvm.get.active.lane.mask.v4i1.i64(i64 0, i64 [[TMP3]])
+; CHECK-NEXT: [[UNCOUNTABLE_EXIT_MASK:%.*]] = call <4 x i1> @llvm.mask.beforefirst.v4i1(<4 x i1> [[TMP5]])
; CHECK-NEXT: [[TMP0:%.*]] = getelementptr i16, ptr [[ARRAY]], i64 [[INDEX]]
; CHECK-NEXT: [[WIDE_MASKED_LOAD:%.*]] = call <4 x i16> @llvm.masked.load.v4i16.p0(ptr align 2 [[TMP0]], <4 x i1> [[UNCOUNTABLE_EXIT_MASK]], <4 x i16> poison)
; CHECK-NEXT: [[TMP4:%.*]] = add nsw <4 x i16> [[WIDE_MASKED_LOAD]], splat (i16 1)
@@ -28,6 +27,7 @@ define void @loop_contains_store_condition_load_has_single_user(ptr dereferencea
; CHECK-NEXT: [[TMP8:%.*]] = or i1 [[TMP6]], [[TMP7]]
; CHECK-NEXT: br i1 [[TMP8]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP0:![0-9]+]]
; CHECK: [[MIDDLE_BLOCK]]:
+; CHECK-NEXT: [[TMP3:%.*]] = call i64 @llvm.experimental.cttz.elts.i64.v4i1(<4 x i1> [[TMP5]], i1 false)
; CHECK-NEXT: [[TMP9:%.*]] = add i64 [[INDEX]], [[TMP3]]
; CHECK-NEXT: [[TMP10:%.*]] = icmp eq i64 [[TMP9]], 20
; CHECK-NEXT: br i1 [[TMP10]], label %[[EXIT:.*]], label %[[SCALAR_PH:.*]]
diff --git a/llvm/test/Transforms/LoopVectorize/scalarized_conditional_ops_uncountable_exits.ll b/llvm/test/Transforms/LoopVectorize/scalarized_conditional_ops_uncountable_exits.ll
index 4523ba2d7d3e7b..c9f296894b997c 100644
--- a/llvm/test/Transforms/LoopVectorize/scalarized_conditional_ops_uncountable_exits.ll
+++ b/llvm/test/Transforms/LoopVectorize/scalarized_conditional_ops_uncountable_exits.ll
@@ -17,8 +17,7 @@ define void @loop_contains_store_condition_load_has_single_user(ptr dereferencea
; CHECK-NEXT: [[WIDE_LOAD:%.*]] = load <4 x i16>, ptr [[TMP1]], align 2
; CHECK-NEXT: [[TMP2:%.*]] = icmp sgt <4 x i16> [[WIDE_LOAD]], splat (i16 500)
; CHECK-NEXT: [[TMP5:%.*]] = freeze <4 x i1> [[TMP2]]
-; CHECK-NEXT: [[TMP3:%.*]] = call i64 @llvm.experimental.cttz.elts.i64.v4i1(<4 x i1> [[TMP5]], i1 false)
-; CHECK-NEXT: [[UNCOUNTABLE_EXIT_MASK:%.*]] = call <4 x i1> @llvm.get.active.lane.mask.v4i1.i64(i64 0, i64 [[TMP3]])
+; CHECK-NEXT: [[UNCOUNTABLE_EXIT_MASK:%.*]] = call <4 x i1> @llvm.mask.beforefirst.v4i1(<4 x i1> [[TMP5]])
; CHECK-NEXT: [[TMP31:%.*]] = getelementptr inbounds nuw i16, ptr [[ARRAY]], i64 [[INDEX]]
; CHECK-NEXT: [[TMP32:%.*]] = getelementptr inbounds nuw i16, ptr [[ARRAY]], i64 [[TMP0]]
; CHECK-NEXT: [[TMP33:%.*]] = getelementptr inbounds nuw i16, ptr [[ARRAY]], i64 [[TMP25]]
@@ -86,6 +85,7 @@ define void @loop_contains_store_condition_load_has_single_user(ptr dereferencea
; CHECK-NEXT: [[TMP8:%.*]] = or i1 [[TMP6]], [[TMP7]]
; CHECK-NEXT: br i1 [[TMP8]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP0:![0-9]+]]
; CHECK: [[MIDDLE_BLOCK]]:
+; CHECK-NEXT: [[TMP3:%.*]] = call i64 @llvm.experimental.cttz.elts.i64.v4i1(<4 x i1> [[TMP5]], i1 false)
; CHECK-NEXT: [[TMP9:%.*]] = add i64 [[INDEX]], [[TMP3]]
; CHECK-NEXT: [[TMP10:%.*]] = icmp eq i64 [[TMP9]], 20
; CHECK-NEXT: br i1 [[TMP10]], label %[[EXIT:.*]], label %[[SCALAR_PH:.*]]
diff --git a/llvm/test/Transforms/LoopVectorize/tail_fold_uncountable_exits.ll b/llvm/test/Transforms/LoopVectorize/tail_fold_uncountable_exits.ll
index e4633bcb5568a0..35ea7acbaa6d8b 100644
--- a/llvm/test/Transforms/LoopVectorize/tail_fold_uncountable_exits.ll
+++ b/llvm/test/Transforms/LoopVectorize/tail_fold_uncountable_exits.ll
@@ -19,8 +19,7 @@ define void @loop_contains_store_condition_load_has_single_user(ptr dereferencea
; CHECK-NEXT: [[WIDE_LOAD:%.*]] = load <4 x i16>, ptr [[TMP1]], align 2
; CHECK-NEXT: [[TMP2:%.*]] = icmp sgt <4 x i16> [[WIDE_LOAD]], splat (i16 500)
; CHECK-NEXT: [[TMP5:%.*]] = freeze <4 x i1> [[TMP2]]
-; CHECK-NEXT: [[TMP3:%.*]] = call i64 @llvm.experimental.cttz.elts.i64.v4i1(<4 x i1> [[TMP5]], i1 false)
-; CHECK-NEXT: [[UNCOUNTABLE_EXIT_MASK:%.*]] = call <4 x i1> @llvm.get.active.lane.mask.v4i1.i64(i64 0, i64 [[TMP3]])
+; CHECK-NEXT: [[UNCOUNTABLE_EXIT_MASK:%.*]] = call <4 x i1> @llvm.mask.beforefirst.v4i1(<4 x i1> [[TMP5]])
; CHECK-NEXT: [[TMP0:%.*]] = getelementptr i16, ptr [[ARRAY]], i64 [[INDEX]]
; CHECK-NEXT: [[WIDE_MASKED_LOAD:%.*]] = call <4 x i16> @llvm.masked.load.v4i16.p0(ptr align 2 [[TMP0]], <4 x i1> [[UNCOUNTABLE_EXIT_MASK]], <4 x i16> poison)
; CHECK-NEXT: [[TMP4:%.*]] = add nsw <4 x i16> [[WIDE_MASKED_LOAD]], splat (i16 1)
@@ -31,6 +30,7 @@ define void @loop_contains_store_condition_load_has_single_user(ptr dereferencea
; CHECK-NEXT: [[TMP8:%.*]] = or i1 [[TMP6]], [[TMP7]]
; CHECK-NEXT: br i1 [[TMP8]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP0:![0-9]+]]
; CHECK: [[MIDDLE_BLOCK]]:
+; CHECK-NEXT: [[TMP3:%.*]] = call i64 @llvm.experimental.cttz.elts.i64.v4i1(<4 x i1> [[TMP5]], i1 false)
; CHECK-NEXT: [[TMP9:%.*]] = add i64 [[INDEX]], [[TMP3]]
; CHECK-NEXT: [[TMP10:%.*]] = icmp eq i64 [[TMP9]], 20
; CHECK-NEXT: br i1 [[TMP10]], label %[[EXIT:.*]], label %[[SCALAR_PH:.*]]
diff --git a/llvm/test/Transforms/PhaseOrdering/AArch64/countable-and-uncountable-exits-combined.ll b/llvm/test/Transforms/PhaseOrdering/AArch64/countable-and-uncountable-exits-combined.ll
index e7ed25c7954d72..9d5804468f12b7 100644
--- a/llvm/test/Transforms/PhaseOrdering/AArch64/countable-and-uncountable-exits-combined.ll
+++ b/llvm/test/Transforms/PhaseOrdering/AArch64/countable-and-uncountable-exits-combined.ll
@@ -40,8 +40,7 @@ define void @foo() #0 {
; CHECK-NEXT: [[WIDE_LOAD:%.*]] = load <vscale x 4 x i32>, ptr [[TMP2]], align 4
; CHECK-NEXT: [[WIDE_LOAD_FR:%.*]] = freeze <vscale x 4 x i32> [[WIDE_LOAD]]
; CHECK-NEXT: [[TMP3:%.*]] = icmp ne <vscale x 4 x i32> [[WIDE_LOAD_FR]], zeroinitializer
-; CHECK-NEXT: [[TMP4:%.*]] = tail call i64 @llvm.experimental.cttz.elts.i64.nxv4i1(<vscale x 4 x i1> [[TMP3]], i1 false)
-; CHECK-NEXT: [[UNCOUNTABLE_EXIT_MASK:%.*]] = tail call <vscale x 4 x i1> @llvm.get.active.lane.mask.nxv4i1.i64(i64 0, i64 [[TMP4]])
+; CHECK-NEXT: [[UNCOUNTABLE_EXIT_MASK:%.*]] = tail call <vscale x 4 x i1> @llvm.mask.beforefirst.nxv4i1(<vscale x 4 x i1> [[TMP3]])
; CHECK-NEXT: [[TMP5:%.*]] = getelementptr [4 x i8], ptr @src, i64 [[INDEX]]
; CHECK-NEXT: [[WIDE_MASKED_LOAD:%.*]] = tail call <vscale x 4 x i32> @llvm.masked.load.nxv4i32.p0(ptr align 4 [[TMP5]], <vscale x 4 x i1> [[UNCOUNTABLE_EXIT_MASK]], <vscale x 4 x i32> poison)
; CHECK-NEXT: [[TMP6:%.*]] = add nsw <vscale x 4 x i32> [[WIDE_MASKED_LOAD]], splat (i32 42)
@@ -53,6 +52,7 @@ define void @foo() #0 {
; CHECK-NEXT: [[TMP10:%.*]] = or i1 [[TMP8]], [[TMP9]]
; CHECK-NEXT: br i1 [[TMP10]], label %[[MIDDLE_BLOCK:.*]], label %[[FOR_BODY]], !llvm.loop [[LOOP0:![0-9]+]]
; CHECK: [[MIDDLE_BLOCK]]:
+; CHECK-NEXT: [[TMP4:%.*]] = tail call i64 @llvm.experimental.cttz.elts.i64.nxv4i1(<vscale x 4 x i1> [[TMP3]], i1 false)
; CHECK-NEXT: [[TMP11:%.*]] = add i64 [[INDEX]], [[TMP4]]
; CHECK-NEXT: [[TMP12:%.*]] = icmp eq i64 [[TMP11]], 10000
; CHECK-NEXT: br i1 [[TMP12]], label %[[EXIT:.*]], label %[[FOR_BODY_PREHEADER]]
>From 28af280de09e98f15212def736653bd92acf0975 Mon Sep 17 00:00:00 2001
From: Luke Lau <luke at igalia.com>
Date: Fri, 30 Oct 2026 00:37:22 +0800
Subject: [PATCH 02/15] Emit VPWidenIntrinsicRecipe
---
.../Vectorize/LoopVectorizationPlanner.h | 9 ++
llvm/lib/Transforms/Vectorize/VPlan.h | 3 -
.../lib/Transforms/Vectorize/VPlanRecipes.cpp | 21 ----
.../Transforms/Vectorize/VPlanTransforms.cpp | 4 +-
.../VPlan/early_exit_with_stores_vplan.ll | 112 +++++++++---------
5 files changed, 67 insertions(+), 82 deletions(-)
diff --git a/llvm/lib/Transforms/Vectorize/LoopVectorizationPlanner.h b/llvm/lib/Transforms/Vectorize/LoopVectorizationPlanner.h
index e380e5a0e78be8..4b92710038f78f 100644
--- a/llvm/lib/Transforms/Vectorize/LoopVectorizationPlanner.h
+++ b/llvm/lib/Transforms/Vectorize/LoopVectorizationPlanner.h
@@ -554,6 +554,15 @@ template <typename InserterTy> class VPBuilderBase : public InserterTy {
return createVectorPointer(Ptr, SourceElementTy, StrideOne, Flags, DL);
}
+ VPWidenIntrinsicRecipe *
+ createWidenIntrinsic(Intrinsic::ID VectorIntrinsicID,
+ ArrayRef<VPValue *> CallArguments, Type *Ty,
+ const VPIRFlags &Flags = {}, const VPIRMetadata &MD = {},
+ DebugLoc DL = DebugLoc::getUnknown()) {
+ return tryInsertInstruction(new VPWidenIntrinsicRecipe(
+ VectorIntrinsicID, CallArguments, Ty, Flags, MD, DL));
+ }
+
VPWidenMemIntrinsicRecipe *createWidenMemIntrinsic(
Intrinsic::ID VectorIntrinsicID, ArrayRef<VPValue *> CallArguments,
Type *Ty, Align Alignment, const VPIRMetadata &MD, DebugLoc DL) {
diff --git a/llvm/lib/Transforms/Vectorize/VPlan.h b/llvm/lib/Transforms/Vectorize/VPlan.h
index b9c2e7e7f88739..d27473af8a7ca5 100644
--- a/llvm/lib/Transforms/Vectorize/VPlan.h
+++ b/llvm/lib/Transforms/Vectorize/VPlan.h
@@ -1390,9 +1390,6 @@ class LLVM_ABI_FOR_TEST VPInstruction : public VPRecipeWithIRFlags,
LogicalAnd, // Non-poison propagating logical And.
LogicalOr, // Non-poison propagating logical Or.
NumActiveLanes, // Counts the number of active lanes in a mask.
- // Returns a mask of all the lanes before the first active lane, i.e.
- // @llvm.mask.beforefirst.
- MaskBeforeFirst,
// Add an offset in bytes (second operand) to a base pointer (first
// operand). Only generates scalar values (either for the first lane only or
// for all lanes, depending on its uses).
diff --git a/llvm/lib/Transforms/Vectorize/VPlanRecipes.cpp b/llvm/lib/Transforms/Vectorize/VPlanRecipes.cpp
index 63d4fbeb1c753c..0b89ff55803edd 100644
--- a/llvm/lib/Transforms/Vectorize/VPlanRecipes.cpp
+++ b/llvm/lib/Transforms/Vectorize/VPlanRecipes.cpp
@@ -538,7 +538,6 @@ Type *llvm::computeScalarTypeForInstruction(unsigned Opcode,
AssertOperandType(1, Op0Ty);
return IntegerType::get(Ctx, 1);
case VPInstruction::MaskedCond:
- case VPInstruction::MaskBeforeFirst:
assert(Op0Ty->isIntegerTy(1) && "expected bool operand");
return IntegerType::get(Ctx, 1);
case VPInstruction::LogicalAnd:
@@ -680,7 +679,6 @@ unsigned VPInstruction::getNumOperandsForOpcode() const {
case VPInstruction::ExtractLastLane:
case VPInstruction::ExtractLastPart:
case VPInstruction::ExtractPenultimateElement:
- case VPInstruction::MaskBeforeFirst:
case VPInstruction::MaskedCond:
case VPInstruction::Not:
case VPInstruction::Reverse:
@@ -896,13 +894,6 @@ Value *VPInstruction::generate(VPTransformState &State,
Builder.CreateUnaryIntrinsic(Intrinsic::vector_reduce_add, ZExt);
return NumActive;
}
- case VPInstruction::MaskBeforeFirst: {
- Value *Cond = State.get(getOperand(0));
- if (State.VF.isScalar())
- return Builder.CreateNot(Cond, Name);
- return Builder.CreateUnaryIntrinsic(Intrinsic::mask_beforefirst, Cond, {},
- Name);
- }
case VPInstruction::FirstOrderRecurrenceSplice: {
// Generate code to combine the previous and current values in vector v3.
//
@@ -1587,14 +1578,6 @@ InstructionCost VPInstruction::computeCost(ElementCount VF,
{ArgTy, ArgTy});
return Ctx.TTI.getIntrinsicInstrCost(Attrs, Ctx.CostKind);
}
- case VPInstruction::MaskBeforeFirst: {
- if (VF.isScalar())
- return Ctx.TTI.getArithmeticInstrCost(
- Instruction::Xor, Type::getInt1Ty(Ctx.LLVMCtx), Ctx.CostKind);
- Type *MaskTy = toVectorTy(Type::getInt1Ty(Ctx.LLVMCtx), VF);
- IntrinsicCostAttributes Attrs(Intrinsic::mask_beforefirst, MaskTy, MaskTy);
- return Ctx.TTI.getIntrinsicInstrCost(Attrs, Ctx.CostKind);
- }
case VPInstruction::ExplicitVectorLength: {
Type *Arg0Ty = getOperand(0)->getScalarType();
Type *I32Ty = Type::getInt32Ty(Ctx.LLVMCtx);
@@ -1819,7 +1802,6 @@ bool VPInstruction::opcodeMayReadOrWriteFromMemory() const {
case VPInstruction::FirstOrderRecurrenceSplice:
case VPInstruction::LogicalAnd:
case VPInstruction::LogicalOr:
- case VPInstruction::MaskBeforeFirst:
case VPInstruction::MaskedCond:
case VPInstruction::Not:
case VPInstruction::PtrAdd:
@@ -2023,9 +2005,6 @@ void VPInstruction::printRecipe(raw_ostream &O, const Twine &Indent,
case VPInstruction::NumActiveLanes:
O << "num-active-lanes";
break;
- case VPInstruction::MaskBeforeFirst:
- O << "mask-before-first";
- break;
case VPInstruction::WideIVStep:
O << "wide-iv-step";
break;
diff --git a/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp b/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp
index 6dafb81a0237bb..fd4479f7cc4763 100644
--- a/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp
+++ b/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp
@@ -3485,8 +3485,8 @@ static bool handleUncountableExitsWithSideEffects(
// Create a mask to represent all lanes that fully execute in the vector loop,
// stopping short of any early exit.
VPBuilder MaskBuilder(InsertIt);
- VPValue *Mask = MaskBuilder.createNaryOp(VPInstruction::MaskBeforeFirst, Cond,
- DebugLoc(), "uncountable.exit.mask");
+ VPValue *Mask = MaskBuilder.createWidenIntrinsic(Intrinsic::mask_beforefirst,
+ Cond, Cond->getScalarType());
// Convert all other memory operations to use the mask.
for (VPBasicBlock *VPBB : vp_rpo_plain_cfg_loop_body(HeaderVPBB))
diff --git a/llvm/test/Transforms/LoopVectorize/VPlan/early_exit_with_stores_vplan.ll b/llvm/test/Transforms/LoopVectorize/VPlan/early_exit_with_stores_vplan.ll
index 1dd06049564be4..ed3ead1afd1bcd 100644
--- a/llvm/test/Transforms/LoopVectorize/VPlan/early_exit_with_stores_vplan.ll
+++ b/llvm/test/Transforms/LoopVectorize/VPlan/early_exit_with_stores_vplan.ll
@@ -28,33 +28,33 @@ define void @loop_contains_store_condition_load_has_single_user(ptr dereferencea
; CHECK-NEXT: WIDEN ir<%ee.val> = load vp<[[VP5]]>, align 2
; CHECK-NEXT: WIDEN ir<%ee.cond> = icmp sgt ir<%ee.val>, ir<500>
; CHECK-NEXT: EMIT vp<[[VP6:%[0-9]+]]> = freeze ir<%ee.cond>
-; CHECK-NEXT: EMIT vp<%uncountable.exit.mask> = mask-before-first vp<[[VP6]]>
+; CHECK-NEXT: WIDEN-INTRINSIC vp<[[VP7:%[0-9]+]]> = call llvm.mask.beforefirst(vp<[[VP6]]>)
; CHECK-NEXT: CLONE ir<%st.addr> = getelementptr ir<%array>, vp<[[VP4]]>
-; CHECK-NEXT: vp<[[VP7:%[0-9]+]]> = vector-pointer i16, ir<%st.addr>, ir<1>
-; CHECK-NEXT: WIDEN ir<%data> = load vp<[[VP7]]>, vp<%uncountable.exit.mask>, align 2
+; CHECK-NEXT: vp<[[VP8:%[0-9]+]]> = vector-pointer i16, ir<%st.addr>, ir<1>
+; CHECK-NEXT: WIDEN ir<%data> = load vp<[[VP8]]>, vp<[[VP7]]>, align 2
; CHECK-NEXT: WIDEN ir<%inc> = add nsw ir<%data>, ir<1>
-; CHECK-NEXT: WIDEN store vp<[[VP7]]>, ir<%inc>, vp<%uncountable.exit.mask>, align 2
-; CHECK-NEXT: EMIT vp<[[VP8:%[0-9]+]]> = any-of vp<[[VP6]]>
+; CHECK-NEXT: WIDEN store vp<[[VP8]]>, ir<%inc>, vp<[[VP7]]>, align 2
+; CHECK-NEXT: EMIT vp<[[VP9:%[0-9]+]]> = any-of vp<[[VP6]]>
; CHECK-NEXT: EMIT vp<%index.next> = add nuw vp<[[VP3]]>, vp<[[VP1]]>
-; CHECK-NEXT: EMIT vp<[[VP9:%[0-9]+]]> = icmp eq vp<%index.next>, vp<[[VP2]]>
-; CHECK-NEXT: EMIT branch-on-two-conds vp<[[VP8]]>, vp<[[VP9]]>
+; CHECK-NEXT: EMIT vp<[[VP10:%[0-9]+]]> = icmp eq vp<%index.next>, vp<[[VP2]]>
+; CHECK-NEXT: EMIT branch-on-two-conds vp<[[VP9]]>, vp<[[VP10]]>
; CHECK-NEXT: No successors
; CHECK-NEXT: }
; CHECK-NEXT: Successor(s): middle.block, middle.block
; CHECK-EMPTY:
; CHECK-NEXT: middle.block:
-; CHECK-NEXT: EMIT vp<[[VP11:%[0-9]+]]> = first-active-lane vp<[[VP6]]>
-; CHECK-NEXT: EMIT vp<[[VP12:%[0-9]+]]> = extract-lane ir<0>, ir<%iv>
-; CHECK-NEXT: EMIT vp<[[VP13:%[0-9]+]]> = add vp<[[VP12]]>, vp<[[VP11]]>
-; CHECK-NEXT: EMIT vp<[[VP14:%[0-9]+]]> = icmp eq vp<[[VP13]]>, ir<20>
-; CHECK-NEXT: EMIT branch-on-cond vp<[[VP14]]>
+; CHECK-NEXT: EMIT vp<[[VP12:%[0-9]+]]> = first-active-lane vp<[[VP6]]>
+; CHECK-NEXT: EMIT vp<[[VP13:%[0-9]+]]> = extract-lane ir<0>, ir<%iv>
+; CHECK-NEXT: EMIT vp<[[VP14:%[0-9]+]]> = add vp<[[VP13]]>, vp<[[VP12]]>
+; CHECK-NEXT: EMIT vp<[[VP15:%[0-9]+]]> = icmp eq vp<[[VP14]]>, ir<20>
+; CHECK-NEXT: EMIT branch-on-cond vp<[[VP15]]>
; CHECK-NEXT: Successor(s): ir-bb<exit>, scalar.ph
; CHECK-EMPTY:
; CHECK-NEXT: ir-bb<exit>:
; CHECK-NEXT: No successors
; CHECK-EMPTY:
; CHECK-NEXT: scalar.ph:
-; CHECK-NEXT: EMIT-SCALAR vp<%bc.resume.val> = phi [ vp<[[VP13]]>, middle.block ], [ ir<0>, ir-bb<entry> ]
+; CHECK-NEXT: EMIT-SCALAR vp<%bc.resume.val> = phi [ vp<[[VP14]]>, middle.block ], [ ir<0>, ir-bb<entry> ]
; CHECK-NEXT: Successor(s): ir-bb<for.body>
; CHECK-EMPTY:
; CHECK-NEXT: ir-bb<for.body>:
@@ -117,33 +117,33 @@ define void @loop_contains_store_after_uncountable_exit(ptr dereferenceable(40)
; CHECK-NEXT: WIDEN ir<%ee.val> = load vp<[[VP5]]>, align 2
; CHECK-NEXT: WIDEN ir<%ee.cond> = icmp sgt ir<%ee.val>, ir<500>
; CHECK-NEXT: EMIT vp<[[VP6:%[0-9]+]]> = freeze ir<%ee.cond>
-; CHECK-NEXT: EMIT vp<%uncountable.exit.mask> = mask-before-first vp<[[VP6]]>
+; CHECK-NEXT: WIDEN-INTRINSIC vp<[[VP7:%[0-9]+]]> = call llvm.mask.beforefirst(vp<[[VP6]]>)
; CHECK-NEXT: CLONE ir<%st.addr> = getelementptr ir<%array>, vp<[[VP4]]>
-; CHECK-NEXT: vp<[[VP7:%[0-9]+]]> = vector-pointer i16, ir<%st.addr>, ir<1>
-; CHECK-NEXT: WIDEN ir<%data> = load vp<[[VP7]]>, vp<%uncountable.exit.mask>, align 2 (!vplan.execution.frequency 17870283321406128127 (96.88%, estimated))
+; CHECK-NEXT: vp<[[VP8:%[0-9]+]]> = vector-pointer i16, ir<%st.addr>, ir<1>
+; CHECK-NEXT: WIDEN ir<%data> = load vp<[[VP8]]>, vp<[[VP7]]>, align 2 (!vplan.execution.frequency 17870283321406128127 (96.88%, estimated))
; CHECK-NEXT: WIDEN ir<%inc> = add nsw ir<%data>, ir<1>
-; CHECK-NEXT: WIDEN store vp<[[VP7]]>, ir<%inc>, vp<%uncountable.exit.mask>, align 2 (!vplan.execution.frequency 17870283321406128127 (96.88%, estimated))
-; CHECK-NEXT: EMIT vp<[[VP8:%[0-9]+]]> = any-of vp<[[VP6]]>
+; CHECK-NEXT: WIDEN store vp<[[VP8]]>, ir<%inc>, vp<[[VP7]]>, align 2 (!vplan.execution.frequency 17870283321406128127 (96.88%, estimated))
+; CHECK-NEXT: EMIT vp<[[VP9:%[0-9]+]]> = any-of vp<[[VP6]]>
; CHECK-NEXT: EMIT vp<%index.next> = add nuw vp<[[VP3]]>, vp<[[VP1]]>
-; CHECK-NEXT: EMIT vp<[[VP9:%[0-9]+]]> = icmp eq vp<%index.next>, vp<[[VP2]]>
-; CHECK-NEXT: EMIT branch-on-two-conds vp<[[VP8]]>, vp<[[VP9]]>
+; CHECK-NEXT: EMIT vp<[[VP10:%[0-9]+]]> = icmp eq vp<%index.next>, vp<[[VP2]]>
+; CHECK-NEXT: EMIT branch-on-two-conds vp<[[VP9]]>, vp<[[VP10]]>
; CHECK-NEXT: No successors
; CHECK-NEXT: }
; CHECK-NEXT: Successor(s): middle.block, middle.block
; CHECK-EMPTY:
; CHECK-NEXT: middle.block:
-; CHECK-NEXT: EMIT vp<[[VP11:%[0-9]+]]> = first-active-lane vp<[[VP6]]>
-; CHECK-NEXT: EMIT vp<[[VP12:%[0-9]+]]> = extract-lane ir<0>, ir<%iv>
-; CHECK-NEXT: EMIT vp<[[VP13:%[0-9]+]]> = add vp<[[VP12]]>, vp<[[VP11]]>
-; CHECK-NEXT: EMIT vp<[[VP14:%[0-9]+]]> = icmp eq vp<[[VP13]]>, ir<20>
-; CHECK-NEXT: EMIT branch-on-cond vp<[[VP14]]>
+; CHECK-NEXT: EMIT vp<[[VP12:%[0-9]+]]> = first-active-lane vp<[[VP6]]>
+; CHECK-NEXT: EMIT vp<[[VP13:%[0-9]+]]> = extract-lane ir<0>, ir<%iv>
+; CHECK-NEXT: EMIT vp<[[VP14:%[0-9]+]]> = add vp<[[VP13]]>, vp<[[VP12]]>
+; CHECK-NEXT: EMIT vp<[[VP15:%[0-9]+]]> = icmp eq vp<[[VP14]]>, ir<20>
+; CHECK-NEXT: EMIT branch-on-cond vp<[[VP15]]>
; CHECK-NEXT: Successor(s): ir-bb<exit>, scalar.ph
; CHECK-EMPTY:
; CHECK-NEXT: ir-bb<exit>:
; CHECK-NEXT: No successors
; CHECK-EMPTY:
; CHECK-NEXT: scalar.ph:
-; CHECK-NEXT: EMIT-SCALAR vp<%bc.resume.val> = phi [ vp<[[VP13]]>, middle.block ], [ ir<0>, ir-bb<entry> ]
+; CHECK-NEXT: EMIT-SCALAR vp<%bc.resume.val> = phi [ vp<[[VP14]]>, middle.block ], [ ir<0>, ir-bb<entry> ]
; CHECK-NEXT: Successor(s): ir-bb<for.body>
; CHECK-EMPTY:
; CHECK-NEXT: ir-bb<for.body>:
@@ -202,36 +202,36 @@ define i16 @uncountable_exit_with_live_out(ptr dereferenceable(40) noalias %arra
; CHECK-NEXT: WIDEN ir<%ee.val> = load vp<[[VP5]]>, align 2
; CHECK-NEXT: WIDEN ir<%ee.cond> = icmp sgt ir<%ee.val>, ir<500>
; CHECK-NEXT: EMIT vp<[[VP6:%[0-9]+]]> = freeze ir<%ee.cond>
-; CHECK-NEXT: EMIT vp<%uncountable.exit.mask> = mask-before-first vp<[[VP6]]>
+; CHECK-NEXT: WIDEN-INTRINSIC vp<[[VP7:%[0-9]+]]> = call llvm.mask.beforefirst(vp<[[VP6]]>)
; CHECK-NEXT: CLONE ir<%st.addr> = getelementptr ir<%array>, vp<[[VP4]]>
-; CHECK-NEXT: vp<[[VP7:%[0-9]+]]> = vector-pointer i16, ir<%st.addr>, ir<1>
-; CHECK-NEXT: WIDEN ir<%data> = load vp<[[VP7]]>, vp<%uncountable.exit.mask>, align 2
+; CHECK-NEXT: vp<[[VP8:%[0-9]+]]> = vector-pointer i16, ir<%st.addr>, ir<1>
+; CHECK-NEXT: WIDEN ir<%data> = load vp<[[VP8]]>, vp<[[VP7]]>, align 2
; CHECK-NEXT: WIDEN ir<%inc> = add nsw ir<%data>, ir<1>
-; CHECK-NEXT: WIDEN store vp<[[VP7]]>, ir<%inc>, vp<%uncountable.exit.mask>, align 2
-; CHECK-NEXT: EMIT vp<[[VP8:%[0-9]+]]> = any-of vp<[[VP6]]>
+; CHECK-NEXT: WIDEN store vp<[[VP8]]>, ir<%inc>, vp<[[VP7]]>, align 2
+; CHECK-NEXT: EMIT vp<[[VP9:%[0-9]+]]> = any-of vp<[[VP6]]>
; CHECK-NEXT: EMIT vp<%index.next> = add nuw vp<[[VP3]]>, vp<[[VP1]]>
-; CHECK-NEXT: EMIT vp<[[VP9:%[0-9]+]]> = icmp eq vp<%index.next>, vp<[[VP2]]>
-; CHECK-NEXT: EMIT branch-on-two-conds vp<[[VP8]]>, vp<[[VP9]]>
+; CHECK-NEXT: EMIT vp<[[VP10:%[0-9]+]]> = icmp eq vp<%index.next>, vp<[[VP2]]>
+; CHECK-NEXT: EMIT branch-on-two-conds vp<[[VP9]]>, vp<[[VP10]]>
; CHECK-NEXT: No successors
; CHECK-NEXT: }
; CHECK-NEXT: Successor(s): middle.block, middle.block
; CHECK-EMPTY:
; CHECK-NEXT: middle.block:
-; CHECK-NEXT: EMIT vp<[[VP11:%[0-9]+]]> = extract-last-part ir<%data>
-; CHECK-NEXT: EMIT vp<[[VP12:%[0-9]+]]> = extract-last-lane vp<[[VP11]]>
-; CHECK-NEXT: EMIT vp<[[VP13:%[0-9]+]]> = first-active-lane vp<[[VP6]]>
-; CHECK-NEXT: EMIT vp<[[VP14:%[0-9]+]]> = extract-lane ir<0>, ir<%iv>
-; CHECK-NEXT: EMIT vp<[[VP15:%[0-9]+]]> = add vp<[[VP14]]>, vp<[[VP13]]>
-; CHECK-NEXT: EMIT vp<[[VP16:%[0-9]+]]> = icmp eq vp<[[VP15]]>, ir<20>
-; CHECK-NEXT: EMIT branch-on-cond vp<[[VP16]]>
+; CHECK-NEXT: EMIT vp<[[VP12:%[0-9]+]]> = extract-last-part ir<%data>
+; CHECK-NEXT: EMIT vp<[[VP13:%[0-9]+]]> = extract-last-lane vp<[[VP12]]>
+; CHECK-NEXT: EMIT vp<[[VP14:%[0-9]+]]> = first-active-lane vp<[[VP6]]>
+; CHECK-NEXT: EMIT vp<[[VP15:%[0-9]+]]> = extract-lane ir<0>, ir<%iv>
+; CHECK-NEXT: EMIT vp<[[VP16:%[0-9]+]]> = add vp<[[VP15]]>, vp<[[VP14]]>
+; CHECK-NEXT: EMIT vp<[[VP17:%[0-9]+]]> = icmp eq vp<[[VP16]]>, ir<20>
+; CHECK-NEXT: EMIT branch-on-cond vp<[[VP17]]>
; CHECK-NEXT: Successor(s): ir-bb<exit>, scalar.ph
; CHECK-EMPTY:
; CHECK-NEXT: ir-bb<exit>:
-; CHECK-NEXT: IR %data.lcssa = phi i16 [ %data, %for.inc ], [ %data, %for.body ] (extra operand: vp<[[VP12]]> from middle.block)
+; CHECK-NEXT: IR %data.lcssa = phi i16 [ %data, %for.inc ], [ %data, %for.body ] (extra operand: vp<[[VP13]]> from middle.block)
; CHECK-NEXT: No successors
; CHECK-EMPTY:
; CHECK-NEXT: scalar.ph:
-; CHECK-NEXT: EMIT-SCALAR vp<%bc.resume.val> = phi [ vp<[[VP15]]>, middle.block ], [ ir<0>, ir-bb<entry> ]
+; CHECK-NEXT: EMIT-SCALAR vp<%bc.resume.val> = phi [ vp<[[VP16]]>, middle.block ], [ ir<0>, ir-bb<entry> ]
; CHECK-NEXT: Successor(s): ir-bb<for.body>
; CHECK-EMPTY:
; CHECK-NEXT: ir-bb<for.body>:
@@ -295,35 +295,35 @@ define void @combined_exit_conditions(ptr align 4 dereferenceable(80) readonly %
; CHECK-NEXT: WIDEN ir<%ee.cmp> = icmp ne ir<%ee.val>, ir<0>
; CHECK-NEXT: EMIT vp<[[VP6:%[0-9]+]]> = freeze ir<%ee.cmp>
; CHECK-NEXT: EMIT vp<[[VP7:%[0-9]+]]> = freeze vp<[[VP6]]>
-; CHECK-NEXT: EMIT vp<%uncountable.exit.mask> = mask-before-first vp<[[VP7]]>
+; CHECK-NEXT: WIDEN-INTRINSIC vp<[[VP8:%[0-9]+]]> = call llvm.mask.beforefirst(vp<[[VP7]]>)
; CHECK-NEXT: CLONE ir<%src.ptr> = getelementptr ir<%src>, vp<[[VP4]]>
-; CHECK-NEXT: vp<[[VP8:%[0-9]+]]> = vector-pointer i32, ir<%src.ptr>, ir<1>
-; CHECK-NEXT: WIDEN ir<%data> = load vp<[[VP8]]>, vp<%uncountable.exit.mask>, align 4
+; CHECK-NEXT: vp<[[VP9:%[0-9]+]]> = vector-pointer i32, ir<%src.ptr>, ir<1>
+; CHECK-NEXT: WIDEN ir<%data> = load vp<[[VP9]]>, vp<[[VP8]]>, align 4
; CHECK-NEXT: WIDEN ir<%add> = add nsw ir<%data>, ir<1>
; CHECK-NEXT: CLONE ir<%dst.ptr> = getelementptr ir<%dst>, vp<[[VP4]]>
-; CHECK-NEXT: vp<[[VP9:%[0-9]+]]> = vector-pointer i32, ir<%dst.ptr>, ir<1>
-; CHECK-NEXT: WIDEN store vp<[[VP9]]>, ir<%add>, vp<%uncountable.exit.mask>, align 4
-; CHECK-NEXT: EMIT vp<[[VP10:%[0-9]+]]> = any-of vp<[[VP7]]>
+; CHECK-NEXT: vp<[[VP10:%[0-9]+]]> = vector-pointer i32, ir<%dst.ptr>, ir<1>
+; CHECK-NEXT: WIDEN store vp<[[VP10]]>, ir<%add>, vp<[[VP8]]>, align 4
+; CHECK-NEXT: EMIT vp<[[VP11:%[0-9]+]]> = any-of vp<[[VP7]]>
; CHECK-NEXT: EMIT vp<%index.next> = add nuw vp<[[VP3]]>, vp<[[VP1]]>
-; CHECK-NEXT: EMIT vp<[[VP11:%[0-9]+]]> = icmp eq vp<%index.next>, vp<[[VP2]]>
-; CHECK-NEXT: EMIT branch-on-two-conds vp<[[VP10]]>, vp<[[VP11]]>
+; CHECK-NEXT: EMIT vp<[[VP12:%[0-9]+]]> = icmp eq vp<%index.next>, vp<[[VP2]]>
+; CHECK-NEXT: EMIT branch-on-two-conds vp<[[VP11]]>, vp<[[VP12]]>
; CHECK-NEXT: No successors
; CHECK-NEXT: }
; CHECK-NEXT: Successor(s): middle.block, middle.block
; CHECK-EMPTY:
; CHECK-NEXT: middle.block:
-; CHECK-NEXT: EMIT vp<[[VP13:%[0-9]+]]> = first-active-lane vp<[[VP7]]>
-; CHECK-NEXT: EMIT vp<[[VP14:%[0-9]+]]> = extract-lane ir<0>, ir<%iv>
-; CHECK-NEXT: EMIT vp<[[VP15:%[0-9]+]]> = add vp<[[VP14]]>, vp<[[VP13]]>
-; CHECK-NEXT: EMIT vp<[[VP16:%[0-9]+]]> = icmp eq vp<[[VP15]]>, ir<20>
-; CHECK-NEXT: EMIT branch-on-cond vp<[[VP16]]>
+; CHECK-NEXT: EMIT vp<[[VP14:%[0-9]+]]> = first-active-lane vp<[[VP7]]>
+; CHECK-NEXT: EMIT vp<[[VP15:%[0-9]+]]> = extract-lane ir<0>, ir<%iv>
+; CHECK-NEXT: EMIT vp<[[VP16:%[0-9]+]]> = add vp<[[VP15]]>, vp<[[VP14]]>
+; CHECK-NEXT: EMIT vp<[[VP17:%[0-9]+]]> = icmp eq vp<[[VP16]]>, ir<20>
+; CHECK-NEXT: EMIT branch-on-cond vp<[[VP17]]>
; CHECK-NEXT: Successor(s): ir-bb<exit>, scalar.ph
; CHECK-EMPTY:
; CHECK-NEXT: ir-bb<exit>:
; CHECK-NEXT: No successors
; CHECK-EMPTY:
; CHECK-NEXT: scalar.ph:
-; CHECK-NEXT: EMIT-SCALAR vp<%bc.resume.val> = phi [ vp<[[VP15]]>, middle.block ], [ ir<0>, ir-bb<entry> ]
+; CHECK-NEXT: EMIT-SCALAR vp<%bc.resume.val> = phi [ vp<[[VP16]]>, middle.block ], [ ir<0>, ir-bb<entry> ]
; CHECK-NEXT: Successor(s): ir-bb<for.body>
; CHECK-EMPTY:
; CHECK-NEXT: ir-bb<for.body>:
>From 4c975ea9ba426bea9376ab43f0c5bc408cb13fad Mon Sep 17 00:00:00 2001
From: Luke Lau <luke at igalia.com>
Date: Thu, 12 Nov 2026 16:53:18 +0800
Subject: [PATCH 03/15] Expand mask.beforefirst if not cheap
---
.../Transforms/Vectorize/LoopVectorize.cpp | 4 +++
.../Transforms/Vectorize/VPlanTransforms.cpp | 34 +++++++++++++++++++
.../Transforms/Vectorize/VPlanTransforms.h | 6 ++++
.../X86/early_exit_with_stores.ll | 8 ++---
4 files changed, 48 insertions(+), 4 deletions(-)
diff --git a/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp b/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
index f814d734ec5642..b609265aeed570 100644
--- a/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
+++ b/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
@@ -6553,6 +6553,10 @@ LoopVectorizationPlanner::tryToBuildVPlan(VPlanPtr Plan, VFRange &Range,
RUN_VPLAN_PASS(VPlanTransforms::convertToStridedAccesses, *Plan, PSE,
*OrigLoop, CostCtx, Range);
+ if (Legal->hasUncountableExitWithSideEffects())
+ RUN_VPLAN_PASS(VPlanTransforms::expandMaskBeforeFirst, *Plan, CostCtx,
+ Range);
+
// Ensure scalar VF plans only contain VF=1, as required by hasScalarVFOnly.
if (Range.Start.isScalar())
Range.End = Range.Start * 2;
diff --git a/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp b/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp
index fd4479f7cc4763..6145fd8caef15e 100644
--- a/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp
+++ b/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp
@@ -6524,6 +6524,40 @@ void VPlanTransforms::convertToStridedAccesses(VPlan &Plan,
}
}
+void VPlanTransforms::expandMaskBeforeFirst(VPlan &Plan, VPCostContext &Ctx,
+ VFRange &Range) {
+ if (Range.Start.isScalar())
+ return;
+
+ for (VPBasicBlock *VPBB : VPBlockUtils::blocksOnly<VPBasicBlock>(
+ vp_depth_first_deep(Plan.getVectorLoopRegion()->getEntry()))) {
+ for (VPRecipeBase &R : make_early_inc_range(*VPBB)) {
+ VPValue *X;
+ if (!match(&R, m_Intrinsic<Intrinsic::mask_beforefirst>(m_VPValue(X))))
+ continue;
+ auto *BeforeFirst = cast<VPWidenIntrinsicRecipe>(&R);
+ auto IsCheap = [&](ElementCount VF) {
+ return BeforeFirst->computeCost(VF, Ctx) <=
+ TargetTransformInfo::TCC_Expensive;
+ };
+ if (LoopVectorizationPlanner::getDecisionAndClampRange(IsCheap, Range))
+ continue;
+
+ VPBuilder Builder(BeforeFirst);
+ Type *Ty = Plan.getVF().getScalarType();
+ DebugLoc DL = BeforeFirst->getDebugLoc();
+ VPValue *FirstActive = Builder.createScalarZExtOrTrunc(
+ Builder.createFirstActiveLane(X, DL), Ty, DL);
+ VPValue *PartStart = Builder.createOverflowingOp(
+ VPInstruction::CanonicalIVIncrementForPart,
+ {Plan.getZero(Ty), &Plan.getVF()});
+ BeforeFirst->replaceAllUsesWith(Builder.createNaryOp(
+ VPInstruction::ActiveLaneMask, {PartStart, FirstActive}, DL));
+ BeforeFirst->eraseFromParent();
+ }
+ }
+}
+
bool VPlanTransforms::widenCompressingPatterns(VPlan &Plan,
VPRecipeBuilder &RecipeBuilder,
ScalarEvolution &SE) {
diff --git a/llvm/lib/Transforms/Vectorize/VPlanTransforms.h b/llvm/lib/Transforms/Vectorize/VPlanTransforms.h
index 7f69f6e904d0f5..f38fa15d0a16c0 100644
--- a/llvm/lib/Transforms/Vectorize/VPlanTransforms.h
+++ b/llvm/lib/Transforms/Vectorize/VPlanTransforms.h
@@ -381,6 +381,12 @@ struct VPlanTransforms {
PredicatedScalarEvolution &PSE, Loop &L,
VPCostContext &Ctx, VFRange &Range);
+ /// Expand mask.beforefirst into an active-lane-mask of its first active lane
+ /// for the VFs in \p Range where it is expensive, clamping \p Range so the
+ /// decision is the same for all of its VFs.
+ static void expandMaskBeforeFirst(VPlan &Plan, VPCostContext &Ctx,
+ VFRange &Range);
+
/// Remove dead recipes from \p Plan.
static void removeDeadRecipes(VPlan &Plan);
diff --git a/llvm/test/Transforms/LoopVectorize/X86/early_exit_with_stores.ll b/llvm/test/Transforms/LoopVectorize/X86/early_exit_with_stores.ll
index 6fcbe461084bfb..3635a51a925dad 100644
--- a/llvm/test/Transforms/LoopVectorize/X86/early_exit_with_stores.ll
+++ b/llvm/test/Transforms/LoopVectorize/X86/early_exit_with_stores.ll
@@ -17,7 +17,8 @@ define void @interleaved_stores_after_early_exit(ptr noalias %dst, ptr align 2 d
; CHECK-NEXT: [[WIDE_LOAD:%.*]] = load <4 x i16>, ptr [[GEP_SRC]], align 2
; CHECK-NEXT: [[TMP1:%.*]] = icmp sgt <4 x i16> [[WIDE_LOAD]], splat (i16 500)
; CHECK-NEXT: [[TMP2:%.*]] = freeze <4 x i1> [[TMP1]]
-; CHECK-NEXT: [[UNCOUNTABLE_EXIT_MASK:%.*]] = call <4 x i1> @llvm.mask.beforefirst.v4i1(<4 x i1> [[TMP2]])
+; CHECK-NEXT: [[TMP3:%.*]] = call i64 @llvm.experimental.cttz.elts.i64.v4i1(<4 x i1> [[TMP2]], i1 false)
+; CHECK-NEXT: [[UNCOUNTABLE_EXIT_MASK:%.*]] = call <4 x i1> @llvm.get.active.lane.mask.v4i1.i64(i64 0, i64 [[TMP3]])
; CHECK-NEXT: [[TMP4:%.*]] = extractelement <4 x i1> [[UNCOUNTABLE_EXIT_MASK]], i64 0
; CHECK-NEXT: br i1 [[TMP4]], label %[[PRED_STORE_IF:.*]], label %[[PRED_STORE_CONTINUE:.*]]
; CHECK: [[PRED_STORE_IF]]:
@@ -71,7 +72,6 @@ define void @interleaved_stores_after_early_exit(ptr noalias %dst, ptr align 2 d
; CHECK-NEXT: [[TMP29:%.*]] = or i1 [[TMP27]], [[TMP28]]
; CHECK-NEXT: br i1 [[TMP29]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP0:![0-9]+]]
; CHECK: [[MIDDLE_BLOCK]]:
-; CHECK-NEXT: [[TMP3:%.*]] = call i64 @llvm.experimental.cttz.elts.i64.v4i1(<4 x i1> [[TMP2]], i1 false)
; CHECK-NEXT: [[TMP30:%.*]] = add i64 [[IV]], [[TMP3]]
; CHECK-NEXT: [[TMP31:%.*]] = icmp eq i64 [[TMP30]], 20
; CHECK-NEXT: br i1 [[TMP31]], [[EXIT:label %.*]], label %[[SCALAR_PH1:.*]]
@@ -116,7 +116,8 @@ define void @interleaved_stores_before_early_exit(ptr noalias %dst, ptr align 2
; CHECK-NEXT: [[WIDE_LOAD:%.*]] = load <4 x i16>, ptr [[TMP0]], align 2
; CHECK-NEXT: [[TMP1:%.*]] = icmp sgt <4 x i16> [[WIDE_LOAD]], splat (i16 500)
; CHECK-NEXT: [[TMP2:%.*]] = freeze <4 x i1> [[TMP1]]
-; CHECK-NEXT: [[UNCOUNTABLE_EXIT_MASK:%.*]] = call <4 x i1> @llvm.mask.beforefirst.v4i1(<4 x i1> [[TMP2]])
+; CHECK-NEXT: [[TMP3:%.*]] = call i64 @llvm.experimental.cttz.elts.i64.v4i1(<4 x i1> [[TMP2]], i1 false)
+; CHECK-NEXT: [[UNCOUNTABLE_EXIT_MASK:%.*]] = call <4 x i1> @llvm.get.active.lane.mask.v4i1.i64(i64 0, i64 [[TMP3]])
; CHECK-NEXT: [[TMP4:%.*]] = extractelement <4 x i1> [[UNCOUNTABLE_EXIT_MASK]], i64 0
; CHECK-NEXT: br i1 [[TMP4]], label %[[PRED_STORE_IF:.*]], label %[[PRED_STORE_CONTINUE:.*]]
; CHECK: [[PRED_STORE_IF]]:
@@ -170,7 +171,6 @@ define void @interleaved_stores_before_early_exit(ptr noalias %dst, ptr align 2
; CHECK-NEXT: [[TMP29:%.*]] = or i1 [[TMP27]], [[TMP28]]
; CHECK-NEXT: br i1 [[TMP29]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP4:![0-9]+]]
; CHECK: [[MIDDLE_BLOCK]]:
-; CHECK-NEXT: [[TMP3:%.*]] = call i64 @llvm.experimental.cttz.elts.i64.v4i1(<4 x i1> [[TMP2]], i1 false)
; CHECK-NEXT: [[TMP30:%.*]] = add i64 [[IV]], [[TMP3]]
; CHECK-NEXT: [[TMP31:%.*]] = icmp eq i64 [[TMP30]], 20
; CHECK-NEXT: br i1 [[TMP31]], [[EXIT:label %.*]], label %[[SCALAR_PH1:.*]]
>From 7996cf89436832edd9fc90616fa04bc3ec8c6823 Mon Sep 17 00:00:00 2001
From: Luke Lau <luke at igalia.com>
Date: Fri, 23 Oct 2026 13:54:55 +0800
Subject: [PATCH 04/15] [VPlan] Support interleaving early exit loops with side
effects
We couldn't do this on main previously because the exit mask `active-lane-mask(0, FirstActiveLane(Cond))` doesn't get unrolled properly across parts. Now that we use mask.beforefirst we can unroll it properly in a chain. The unrolled form is:
unroll(mask.beforefirst, P) = mask.beforefirst(cond[P] | broadcast(anyof(not(unroll(mask.beforefirst, P-1)))))
On AArch64 this can be combined to brkpb in another patch.
---
.../Transforms/Vectorize/LoopVectorize.cpp | 9 -
.../Transforms/Vectorize/VPlanLowering.cpp | 10 +
llvm/lib/Transforms/Vectorize/VPlanUnroll.cpp | 10 +
.../AArch64/early_exit_with_stores.ll | 231 ++++++++++++++++--
.../interleave_uncountable_exits.ll | 113 ++++++++-
...ountable-and-uncountable-exits-combined.ll | 39 ++-
6 files changed, 377 insertions(+), 35 deletions(-)
diff --git a/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp b/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
index b609265aeed570..128adbf60f1f22 100644
--- a/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
+++ b/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
@@ -7785,15 +7785,6 @@ bool LoopVectorizePass::processLoop(Loop *L) {
IC = 1;
}
- // FIXME: Enable interleaving for EE-with-side-effects.
- if (InterleaveLoop && LVL.hasUncountableExitWithSideEffects()) {
- LLVM_DEBUG(dbgs() << "LV: Not interleaving due to EE with side effects.\n");
- IntDiagMsg = {"EEWithSideEffectsPreventsInterleaving",
- "Unable to interleave due to early exit with side effects."};
- InterleaveLoop = false;
- IC = 1;
- }
-
if (InterleaveLoop && LVL.hasConditionalInductions()) {
LLVM_DEBUG(
dbgs() << "LV: Not interleaving due to conditional inductions.\n");
diff --git a/llvm/lib/Transforms/Vectorize/VPlanLowering.cpp b/llvm/lib/Transforms/Vectorize/VPlanLowering.cpp
index ce6d6d40d7ab48..93fb105d145839 100644
--- a/llvm/lib/Transforms/Vectorize/VPlanLowering.cpp
+++ b/llvm/lib/Transforms/Vectorize/VPlanLowering.cpp
@@ -675,6 +675,16 @@ void VPlanTransforms::convertToConcreteRecipes(VPlan &Plan) {
continue;
}
+ VPValue *Mask;
+ if (Plan.hasScalarVFOnly() &&
+ match(&R,
+ m_Intrinsic<Intrinsic::mask_beforefirst>(m_VPValue(Mask)))) {
+ R.getVPSingleValue()->replaceAllUsesWith(
+ Builder.createNot(Mask, R.getDebugLoc()));
+ R.eraseFromParent();
+ continue;
+ }
+
// Lower BranchOnCount to ICmp + BranchOnCond.
VPValue *IV, *TC;
if (match(&R, m_BranchOnCount(m_VPValue(IV), m_VPValue(TC)))) {
diff --git a/llvm/lib/Transforms/Vectorize/VPlanUnroll.cpp b/llvm/lib/Transforms/Vectorize/VPlanUnroll.cpp
index 53ae651de231fa..bcfdb3e1acc1b2 100644
--- a/llvm/lib/Transforms/Vectorize/VPlanUnroll.cpp
+++ b/llvm/lib/Transforms/Vectorize/VPlanUnroll.cpp
@@ -333,6 +333,16 @@ void UnrollState::unrollRecipeByUF(VPRecipeBase &R) {
Copy->setOperand(1, getValueForPart(Op, Part));
continue;
}
+ if (match(&R, m_Intrinsic<Intrinsic::mask_beforefirst>(m_VPValue(Op)))) {
+ VPBuilder Builder(Copy);
+ VPValue *Chain = Builder.createNaryOp(
+ VPInstruction::AnyOf,
+ Builder.createNot(getValueForPart(R.getVPSingleValue(), Part - 1)));
+ if (!Plan.hasScalarVFOnly())
+ Chain = Builder.createNaryOp(VPInstruction::Broadcast, Chain);
+ Copy->setOperand(0, Builder.createOr(getValueForPart(Op, Part), Chain));
+ continue;
+ }
if (match(&R, m_VPInstruction<VPInstruction::ExtractVectorForPart>(
m_VPValue(Op), m_VPValue()))) {
Copy->setOperand(0, Op);
diff --git a/llvm/test/Transforms/LoopVectorize/AArch64/early_exit_with_stores.ll b/llvm/test/Transforms/LoopVectorize/AArch64/early_exit_with_stores.ll
index 616c2a95ff1fd8..aa4c9abbf0208b 100644
--- a/llvm/test/Transforms/LoopVectorize/AArch64/early_exit_with_stores.ll
+++ b/llvm/test/Transforms/LoopVectorize/AArch64/early_exit_with_stores.ll
@@ -54,32 +54,72 @@ define void @loop_contains_store_condition_load_has_single_user(ptr dereferencea
; CHECK-SAME: ptr noalias dereferenceable(40) [[ARRAY:%.*]], ptr readonly align 2 dereferenceable(40) [[PRED:%.*]]) #[[ATTR0]] {
; CHECK-NEXT: [[SCALAR_PH1:.*]]:
; CHECK-NEXT: [[TMP0:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-NEXT: [[TMP1:%.*]] = shl nuw i64 [[TMP0]], 2
+; CHECK-NEXT: [[TMP1:%.*]] = shl nuw i64 [[TMP0]], 3
; CHECK-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 20, [[TMP1]]
; CHECK-NEXT: br i1 [[MIN_ITERS_CHECK]], label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]]
; CHECK: [[VECTOR_PH]]:
+; CHECK-NEXT: [[TMP2:%.*]] = shl nuw i64 [[TMP0]], 2
+; CHECK-NEXT: [[BROADCAST_SPLATINSERT:%.*]] = insertelement <vscale x 4 x i64> poison, i64 [[TMP2]], i64 0
+; CHECK-NEXT: [[BROADCAST_SPLAT:%.*]] = shufflevector <vscale x 4 x i64> [[BROADCAST_SPLATINSERT]], <vscale x 4 x i64> poison, <vscale x 4 x i32> zeroinitializer
; CHECK-NEXT: [[N_MOD_VF:%.*]] = urem i64 20, [[TMP1]]
; CHECK-NEXT: [[N_VEC:%.*]] = sub i64 20, [[N_MOD_VF]]
+; CHECK-NEXT: [[TMP17:%.*]] = call <vscale x 4 x i64> @llvm.stepvector.nxv4i64()
; CHECK-NEXT: br label %[[FOR_BODY1:.*]]
; CHECK: [[FOR_BODY1]]:
; CHECK-NEXT: [[IV1:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[FOR_BODY1]] ]
+; CHECK-NEXT: [[VEC_IND:%.*]] = phi <vscale x 4 x i64> [ [[TMP17]], %[[VECTOR_PH]] ], [ [[VEC_IND_NEXT:%.*]], %[[FOR_BODY1]] ]
+; CHECK-NEXT: [[STEP_ADD:%.*]] = add nuw <vscale x 4 x i64> [[VEC_IND]], [[BROADCAST_SPLAT]]
; CHECK-NEXT: [[TMP5:%.*]] = getelementptr inbounds nuw i16, ptr [[PRED]], i64 [[IV1]]
+; CHECK-NEXT: [[TMP18:%.*]] = getelementptr inbounds nuw i16, ptr [[TMP5]], i64 [[TMP2]]
; CHECK-NEXT: [[WIDE_LOAD:%.*]] = load <vscale x 4 x i16>, ptr [[TMP5]], align 2
+; CHECK-NEXT: [[WIDE_LOAD1:%.*]] = load <vscale x 4 x i16>, ptr [[TMP18]], align 2
; CHECK-NEXT: [[TMP3:%.*]] = icmp sgt <vscale x 4 x i16> [[WIDE_LOAD]], splat (i16 500)
+; CHECK-NEXT: [[TMP19:%.*]] = icmp sgt <vscale x 4 x i16> [[WIDE_LOAD1]], splat (i16 500)
; CHECK-NEXT: [[TMP4:%.*]] = freeze <vscale x 4 x i1> [[TMP3]]
+; CHECK-NEXT: [[TMP9:%.*]] = freeze <vscale x 4 x i1> [[TMP19]]
; CHECK-NEXT: [[TMP6:%.*]] = call <vscale x 4 x i1> @llvm.mask.beforefirst.nxv4i1(<vscale x 4 x i1> [[TMP4]])
+; CHECK-NEXT: [[TMP25:%.*]] = xor <vscale x 4 x i1> [[TMP6]], splat (i1 true)
+; CHECK-NEXT: [[TMP30:%.*]] = call i1 @llvm.vector.reduce.or.nxv4i1(<vscale x 4 x i1> [[TMP25]])
+; CHECK-NEXT: [[BROADCAST_SPLATINSERT2:%.*]] = insertelement <vscale x 4 x i1> poison, i1 [[TMP30]], i64 0
+; CHECK-NEXT: [[BROADCAST_SPLAT3:%.*]] = shufflevector <vscale x 4 x i1> [[BROADCAST_SPLATINSERT2]], <vscale x 4 x i1> poison, <vscale x 4 x i32> zeroinitializer
+; CHECK-NEXT: [[TMP32:%.*]] = or <vscale x 4 x i1> [[TMP9]], [[BROADCAST_SPLAT3]]
+; CHECK-NEXT: [[UNCOUNTABLE_EXIT_MASK2:%.*]] = call <vscale x 4 x i1> @llvm.mask.beforefirst.nxv4i1(<vscale x 4 x i1> [[TMP32]])
; CHECK-NEXT: [[ST_ADDR1:%.*]] = getelementptr i16, ptr [[ARRAY]], i64 [[IV1]]
+; CHECK-NEXT: [[TMP31:%.*]] = getelementptr i16, ptr [[ST_ADDR1]], i64 [[TMP2]]
; CHECK-NEXT: [[WIDE_MASKED_LOAD:%.*]] = call <vscale x 4 x i16> @llvm.masked.load.nxv4i16.p0(ptr align 2 [[ST_ADDR1]], <vscale x 4 x i1> [[TMP6]], <vscale x 4 x i16> poison)
+; CHECK-NEXT: [[WIDE_MASKED_LOAD3:%.*]] = call <vscale x 4 x i16> @llvm.masked.load.nxv4i16.p0(ptr align 2 [[TMP31]], <vscale x 4 x i1> [[UNCOUNTABLE_EXIT_MASK2]], <vscale x 4 x i16> poison)
; CHECK-NEXT: [[TMP8:%.*]] = add nsw <vscale x 4 x i16> [[WIDE_MASKED_LOAD]], splat (i16 1)
+; CHECK-NEXT: [[TMP15:%.*]] = add nsw <vscale x 4 x i16> [[WIDE_MASKED_LOAD3]], splat (i16 1)
; CHECK-NEXT: call void @llvm.masked.store.nxv4i16.p0(<vscale x 4 x i16> [[TMP8]], ptr align 2 [[ST_ADDR1]], <vscale x 4 x i1> [[TMP6]])
-; CHECK-NEXT: [[TMP10:%.*]] = call i1 @llvm.vector.reduce.or.nxv4i1(<vscale x 4 x i1> [[TMP4]])
+; CHECK-NEXT: call void @llvm.masked.store.nxv4i16.p0(<vscale x 4 x i16> [[TMP15]], ptr align 2 [[TMP31]], <vscale x 4 x i1> [[UNCOUNTABLE_EXIT_MASK2]])
+; CHECK-NEXT: [[TMP16:%.*]] = or <vscale x 4 x i1> [[TMP4]], [[TMP9]]
+; CHECK-NEXT: [[TMP10:%.*]] = call i1 @llvm.vector.reduce.or.nxv4i1(<vscale x 4 x i1> [[TMP16]])
; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[IV1]], [[TMP1]]
; CHECK-NEXT: [[TMP11:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
+; CHECK-NEXT: [[VEC_IND_NEXT]] = add nuw nsw <vscale x 4 x i64> [[STEP_ADD]], [[BROADCAST_SPLAT]]
; CHECK-NEXT: [[TMP12:%.*]] = or i1 [[TMP10]], [[TMP11]]
; CHECK-NEXT: br i1 [[TMP12]], label %[[MIDDLE_BLOCK:.*]], label %[[FOR_BODY1]], !llvm.loop [[LOOP0:![0-9]+]]
; CHECK: [[MIDDLE_BLOCK]]:
+; CHECK-NEXT: [[TMP20:%.*]] = call i64 @llvm.vscale.i64()
+; CHECK-NEXT: [[TMP21:%.*]] = mul nuw i64 [[TMP20]], 4
+; CHECK-NEXT: [[TMP22:%.*]] = call i64 @llvm.experimental.cttz.elts.i64.nxv4i1(<vscale x 4 x i1> [[TMP9]], i1 false)
+; CHECK-NEXT: [[TMP23:%.*]] = mul i64 [[TMP21]], 1
+; CHECK-NEXT: [[TMP24:%.*]] = add i64 [[TMP23]], [[TMP22]]
; CHECK-NEXT: [[TMP7:%.*]] = call i64 @llvm.experimental.cttz.elts.i64.nxv4i1(<vscale x 4 x i1> [[TMP4]], i1 false)
-; CHECK-NEXT: [[TMP13:%.*]] = add i64 [[IV1]], [[TMP7]]
+; CHECK-NEXT: [[TMP26:%.*]] = mul i64 [[TMP21]], 0
+; CHECK-NEXT: [[TMP27:%.*]] = add i64 [[TMP26]], [[TMP7]]
+; CHECK-NEXT: [[TMP28:%.*]] = icmp ne i64 [[TMP7]], [[TMP21]]
+; CHECK-NEXT: [[TMP29:%.*]] = select i1 [[TMP28]], i64 [[TMP27]], i64 [[TMP24]]
+; CHECK-NEXT: [[TMP40:%.*]] = call i64 @llvm.vscale.i64()
+; CHECK-NEXT: [[TMP41:%.*]] = mul nuw i64 [[TMP40]], 4
+; CHECK-NEXT: [[TMP33:%.*]] = mul i64 [[TMP41]], 0
+; CHECK-NEXT: [[TMP34:%.*]] = extractelement <vscale x 4 x i64> [[VEC_IND]], i64 0
+; CHECK-NEXT: [[TMP35:%.*]] = mul i64 [[TMP41]], 1
+; CHECK-NEXT: [[TMP36:%.*]] = sub i64 0, [[TMP35]]
+; CHECK-NEXT: [[TMP37:%.*]] = extractelement <vscale x 4 x i64> [[STEP_ADD]], i64 [[TMP36]]
+; CHECK-NEXT: [[TMP38:%.*]] = icmp uge i64 0, [[TMP35]]
+; CHECK-NEXT: [[TMP39:%.*]] = select i1 [[TMP38]], i64 [[TMP37]], i64 [[TMP34]]
+; CHECK-NEXT: [[TMP13:%.*]] = add i64 [[TMP39]], [[TMP29]]
; CHECK-NEXT: [[TMP14:%.*]] = icmp eq i64 [[TMP13]], 20
; CHECK-NEXT: br i1 [[TMP14]], label %[[EXIT:.*]], label %[[SCALAR_PH]]
; CHECK: [[SCALAR_PH]]:
@@ -130,32 +170,72 @@ define void @loop_contains_store_fcmp_condition(ptr dereferenceable(40) noalias
; CHECK-SAME: ptr noalias dereferenceable(40) [[ARRAY:%.*]], ptr readonly align 2 dereferenceable(40) [[PRED:%.*]]) #[[ATTR0]] {
; CHECK-NEXT: [[ENTRY:.*]]:
; CHECK-NEXT: [[TMP0:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-NEXT: [[TMP1:%.*]] = shl nuw i64 [[TMP0]], 2
+; CHECK-NEXT: [[TMP1:%.*]] = shl nuw i64 [[TMP0]], 3
; CHECK-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 20, [[TMP1]]
; CHECK-NEXT: br i1 [[MIN_ITERS_CHECK]], label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]]
; CHECK: [[VECTOR_PH]]:
+; CHECK-NEXT: [[TMP14:%.*]] = shl nuw i64 [[TMP0]], 2
+; CHECK-NEXT: [[BROADCAST_SPLATINSERT:%.*]] = insertelement <vscale x 4 x i64> poison, i64 [[TMP14]], i64 0
+; CHECK-NEXT: [[BROADCAST_SPLAT:%.*]] = shufflevector <vscale x 4 x i64> [[BROADCAST_SPLATINSERT]], <vscale x 4 x i64> poison, <vscale x 4 x i32> zeroinitializer
; CHECK-NEXT: [[N_MOD_VF:%.*]] = urem i64 20, [[TMP1]]
; CHECK-NEXT: [[N_VEC:%.*]] = sub i64 20, [[N_MOD_VF]]
+; CHECK-NEXT: [[TMP17:%.*]] = call <vscale x 4 x i64> @llvm.stepvector.nxv4i64()
; CHECK-NEXT: br label %[[FOR_BODY:.*]]
; CHECK: [[FOR_BODY]]:
; CHECK-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[FOR_BODY]] ]
+; CHECK-NEXT: [[VEC_IND:%.*]] = phi <vscale x 4 x i64> [ [[TMP17]], %[[VECTOR_PH]] ], [ [[VEC_IND_NEXT:%.*]], %[[FOR_BODY]] ]
+; CHECK-NEXT: [[STEP_ADD:%.*]] = add nuw <vscale x 4 x i64> [[VEC_IND]], [[BROADCAST_SPLAT]]
; CHECK-NEXT: [[TMP2:%.*]] = getelementptr inbounds nuw half, ptr [[PRED]], i64 [[INDEX]]
+; CHECK-NEXT: [[TMP18:%.*]] = getelementptr inbounds nuw half, ptr [[TMP2]], i64 [[TMP14]]
; CHECK-NEXT: [[WIDE_LOAD:%.*]] = load <vscale x 4 x half>, ptr [[TMP2]], align 2
+; CHECK-NEXT: [[WIDE_LOAD1:%.*]] = load <vscale x 4 x half>, ptr [[TMP18]], align 2
; CHECK-NEXT: [[TMP3:%.*]] = fcmp ugt <vscale x 4 x half> [[WIDE_LOAD]], splat (half 5.000000e+02)
+; CHECK-NEXT: [[TMP19:%.*]] = fcmp ugt <vscale x 4 x half> [[WIDE_LOAD1]], splat (half 5.000000e+02)
; CHECK-NEXT: [[TMP4:%.*]] = freeze <vscale x 4 x i1> [[TMP3]]
+; CHECK-NEXT: [[TMP25:%.*]] = freeze <vscale x 4 x i1> [[TMP19]]
; CHECK-NEXT: [[TMP13:%.*]] = call <vscale x 4 x i1> @llvm.mask.beforefirst.nxv4i1(<vscale x 4 x i1> [[TMP4]])
+; CHECK-NEXT: [[TMP30:%.*]] = xor <vscale x 4 x i1> [[TMP13]], splat (i1 true)
+; CHECK-NEXT: [[TMP31:%.*]] = call i1 @llvm.vector.reduce.or.nxv4i1(<vscale x 4 x i1> [[TMP30]])
+; CHECK-NEXT: [[BROADCAST_SPLATINSERT2:%.*]] = insertelement <vscale x 4 x i1> poison, i1 [[TMP31]], i64 0
+; CHECK-NEXT: [[BROADCAST_SPLAT3:%.*]] = shufflevector <vscale x 4 x i1> [[BROADCAST_SPLATINSERT2]], <vscale x 4 x i1> poison, <vscale x 4 x i32> zeroinitializer
+; CHECK-NEXT: [[TMP33:%.*]] = or <vscale x 4 x i1> [[TMP25]], [[BROADCAST_SPLAT3]]
+; CHECK-NEXT: [[UNCOUNTABLE_EXIT_MASK2:%.*]] = call <vscale x 4 x i1> @llvm.mask.beforefirst.nxv4i1(<vscale x 4 x i1> [[TMP33]])
; CHECK-NEXT: [[TMP6:%.*]] = getelementptr i16, ptr [[ARRAY]], i64 [[INDEX]]
+; CHECK-NEXT: [[TMP32:%.*]] = getelementptr i16, ptr [[TMP6]], i64 [[TMP14]]
; CHECK-NEXT: [[WIDE_MASKED_LOAD:%.*]] = call <vscale x 4 x i16> @llvm.masked.load.nxv4i16.p0(ptr align 2 [[TMP6]], <vscale x 4 x i1> [[TMP13]], <vscale x 4 x i16> poison)
+; CHECK-NEXT: [[WIDE_MASKED_LOAD3:%.*]] = call <vscale x 4 x i16> @llvm.masked.load.nxv4i16.p0(ptr align 2 [[TMP32]], <vscale x 4 x i1> [[UNCOUNTABLE_EXIT_MASK2]], <vscale x 4 x i16> poison)
; CHECK-NEXT: [[TMP7:%.*]] = add nsw <vscale x 4 x i16> [[WIDE_MASKED_LOAD]], splat (i16 1)
+; CHECK-NEXT: [[TMP15:%.*]] = add nsw <vscale x 4 x i16> [[WIDE_MASKED_LOAD3]], splat (i16 1)
; CHECK-NEXT: call void @llvm.masked.store.nxv4i16.p0(<vscale x 4 x i16> [[TMP7]], ptr align 2 [[TMP6]], <vscale x 4 x i1> [[TMP13]])
-; CHECK-NEXT: [[TMP8:%.*]] = call i1 @llvm.vector.reduce.or.nxv4i1(<vscale x 4 x i1> [[TMP4]])
+; CHECK-NEXT: call void @llvm.masked.store.nxv4i16.p0(<vscale x 4 x i16> [[TMP15]], ptr align 2 [[TMP32]], <vscale x 4 x i1> [[UNCOUNTABLE_EXIT_MASK2]])
+; CHECK-NEXT: [[TMP16:%.*]] = or <vscale x 4 x i1> [[TMP4]], [[TMP25]]
+; CHECK-NEXT: [[TMP8:%.*]] = call i1 @llvm.vector.reduce.or.nxv4i1(<vscale x 4 x i1> [[TMP16]])
; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], [[TMP1]]
; CHECK-NEXT: [[TMP9:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
+; CHECK-NEXT: [[VEC_IND_NEXT]] = add nuw nsw <vscale x 4 x i64> [[STEP_ADD]], [[BROADCAST_SPLAT]]
; CHECK-NEXT: [[TMP10:%.*]] = or i1 [[TMP8]], [[TMP9]]
; CHECK-NEXT: br i1 [[TMP10]], label %[[MIDDLE_BLOCK:.*]], label %[[FOR_BODY]], !llvm.loop [[LOOP4:![0-9]+]]
; CHECK: [[MIDDLE_BLOCK]]:
+; CHECK-NEXT: [[TMP20:%.*]] = call i64 @llvm.vscale.i64()
+; CHECK-NEXT: [[TMP21:%.*]] = mul nuw i64 [[TMP20]], 4
+; CHECK-NEXT: [[TMP22:%.*]] = call i64 @llvm.experimental.cttz.elts.i64.nxv4i1(<vscale x 4 x i1> [[TMP25]], i1 false)
+; CHECK-NEXT: [[TMP23:%.*]] = mul i64 [[TMP21]], 1
+; CHECK-NEXT: [[TMP24:%.*]] = add i64 [[TMP23]], [[TMP22]]
; CHECK-NEXT: [[TMP5:%.*]] = call i64 @llvm.experimental.cttz.elts.i64.nxv4i1(<vscale x 4 x i1> [[TMP4]], i1 false)
-; CHECK-NEXT: [[TMP11:%.*]] = add i64 [[INDEX]], [[TMP5]]
+; CHECK-NEXT: [[TMP26:%.*]] = mul i64 [[TMP21]], 0
+; CHECK-NEXT: [[TMP27:%.*]] = add i64 [[TMP26]], [[TMP5]]
+; CHECK-NEXT: [[TMP28:%.*]] = icmp ne i64 [[TMP5]], [[TMP21]]
+; CHECK-NEXT: [[TMP29:%.*]] = select i1 [[TMP28]], i64 [[TMP27]], i64 [[TMP24]]
+; CHECK-NEXT: [[TMP40:%.*]] = call i64 @llvm.vscale.i64()
+; CHECK-NEXT: [[TMP41:%.*]] = mul nuw i64 [[TMP40]], 4
+; CHECK-NEXT: [[TMP42:%.*]] = mul i64 [[TMP41]], 0
+; CHECK-NEXT: [[TMP34:%.*]] = extractelement <vscale x 4 x i64> [[VEC_IND]], i64 0
+; CHECK-NEXT: [[TMP35:%.*]] = mul i64 [[TMP41]], 1
+; CHECK-NEXT: [[TMP36:%.*]] = sub i64 0, [[TMP35]]
+; CHECK-NEXT: [[TMP37:%.*]] = extractelement <vscale x 4 x i64> [[STEP_ADD]], i64 [[TMP36]]
+; CHECK-NEXT: [[TMP38:%.*]] = icmp uge i64 0, [[TMP35]]
+; CHECK-NEXT: [[TMP39:%.*]] = select i1 [[TMP38]], i64 [[TMP37]], i64 [[TMP34]]
+; CHECK-NEXT: [[TMP11:%.*]] = add i64 [[TMP39]], [[TMP29]]
; CHECK-NEXT: [[TMP12:%.*]] = icmp eq i64 [[TMP11]], 20
; CHECK-NEXT: br i1 [[TMP12]], label %[[EXIT:.*]], label %[[SCALAR_PH]]
; CHECK: [[SCALAR_PH]]:
@@ -363,33 +443,74 @@ define void @loop_contains_store_to_pointer_with_no_deref_info(ptr align 2 deref
; CHECK-SAME: ptr readonly align 2 dereferenceable(40) [[LOAD_ARRAY:%.*]], ptr noalias align 2 [[ARRAY:%.*]], ptr readonly align 2 dereferenceable(40) [[PRED:%.*]]) #[[ATTR0]] {
; CHECK-NEXT: [[SCALAR_PH:.*]]:
; CHECK-NEXT: [[TMP0:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-NEXT: [[TMP1:%.*]] = shl nuw i64 [[TMP0]], 2
+; CHECK-NEXT: [[TMP1:%.*]] = shl nuw i64 [[TMP0]], 3
; CHECK-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 20, [[TMP1]]
; CHECK-NEXT: br i1 [[MIN_ITERS_CHECK]], label %[[SCALAR_PH1:.*]], label %[[VECTOR_PH:.*]]
; CHECK: [[VECTOR_PH]]:
+; CHECK-NEXT: [[TMP2:%.*]] = shl nuw i64 [[TMP0]], 2
+; CHECK-NEXT: [[BROADCAST_SPLATINSERT:%.*]] = insertelement <vscale x 4 x i64> poison, i64 [[TMP2]], i64 0
+; CHECK-NEXT: [[BROADCAST_SPLAT:%.*]] = shufflevector <vscale x 4 x i64> [[BROADCAST_SPLATINSERT]], <vscale x 4 x i64> poison, <vscale x 4 x i32> zeroinitializer
; CHECK-NEXT: [[N_MOD_VF:%.*]] = urem i64 20, [[TMP1]]
; CHECK-NEXT: [[N_VEC:%.*]] = sub i64 20, [[N_MOD_VF]]
+; CHECK-NEXT: [[TMP16:%.*]] = call <vscale x 4 x i64> @llvm.stepvector.nxv4i64()
; CHECK-NEXT: br label %[[FOR_BODY:.*]]
; CHECK: [[FOR_BODY]]:
; CHECK-NEXT: [[IV:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[FOR_BODY]] ]
+; CHECK-NEXT: [[VEC_IND:%.*]] = phi <vscale x 4 x i64> [ [[TMP16]], %[[VECTOR_PH]] ], [ [[VEC_IND_NEXT:%.*]], %[[FOR_BODY]] ]
+; CHECK-NEXT: [[STEP_ADD:%.*]] = add nuw <vscale x 4 x i64> [[VEC_IND]], [[BROADCAST_SPLAT]]
; CHECK-NEXT: [[TMP5:%.*]] = getelementptr inbounds nuw i16, ptr [[PRED]], i64 [[IV]]
+; CHECK-NEXT: [[TMP19:%.*]] = getelementptr inbounds nuw i16, ptr [[TMP5]], i64 [[TMP2]]
; CHECK-NEXT: [[WIDE_LOAD:%.*]] = load <vscale x 4 x i16>, ptr [[TMP5]], align 2
+; CHECK-NEXT: [[WIDE_LOAD1:%.*]] = load <vscale x 4 x i16>, ptr [[TMP19]], align 2
; CHECK-NEXT: [[TMP3:%.*]] = icmp sgt <vscale x 4 x i16> [[WIDE_LOAD]], splat (i16 500)
+; CHECK-NEXT: [[TMP20:%.*]] = icmp sgt <vscale x 4 x i16> [[WIDE_LOAD1]], splat (i16 500)
; CHECK-NEXT: [[TMP4:%.*]] = freeze <vscale x 4 x i1> [[TMP3]]
+; CHECK-NEXT: [[TMP21:%.*]] = freeze <vscale x 4 x i1> [[TMP20]]
; CHECK-NEXT: [[TMP6:%.*]] = call <vscale x 4 x i1> @llvm.mask.beforefirst.nxv4i1(<vscale x 4 x i1> [[TMP4]])
+; CHECK-NEXT: [[TMP10:%.*]] = xor <vscale x 4 x i1> [[TMP6]], splat (i1 true)
+; CHECK-NEXT: [[TMP27:%.*]] = call i1 @llvm.vector.reduce.or.nxv4i1(<vscale x 4 x i1> [[TMP10]])
+; CHECK-NEXT: [[BROADCAST_SPLATINSERT2:%.*]] = insertelement <vscale x 4 x i1> poison, i1 [[TMP27]], i64 0
+; CHECK-NEXT: [[BROADCAST_SPLAT3:%.*]] = shufflevector <vscale x 4 x i1> [[BROADCAST_SPLATINSERT2]], <vscale x 4 x i1> poison, <vscale x 4 x i32> zeroinitializer
+; CHECK-NEXT: [[TMP34:%.*]] = or <vscale x 4 x i1> [[TMP21]], [[BROADCAST_SPLAT3]]
+; CHECK-NEXT: [[UNCOUNTABLE_EXIT_MASK2:%.*]] = call <vscale x 4 x i1> @llvm.mask.beforefirst.nxv4i1(<vscale x 4 x i1> [[TMP34]])
; CHECK-NEXT: [[LD_ADDR:%.*]] = getelementptr i16, ptr [[LOAD_ARRAY]], i64 [[IV]]
+; CHECK-NEXT: [[TMP32:%.*]] = getelementptr i16, ptr [[LD_ADDR]], i64 [[TMP2]]
; CHECK-NEXT: [[WIDE_MASKED_LOAD:%.*]] = call <vscale x 4 x i16> @llvm.masked.load.nxv4i16.p0(ptr align 2 [[LD_ADDR]], <vscale x 4 x i1> [[TMP6]], <vscale x 4 x i16> poison)
+; CHECK-NEXT: [[WIDE_MASKED_LOAD3:%.*]] = call <vscale x 4 x i16> @llvm.masked.load.nxv4i16.p0(ptr align 2 [[TMP32]], <vscale x 4 x i1> [[UNCOUNTABLE_EXIT_MASK2]], <vscale x 4 x i16> poison)
; CHECK-NEXT: [[TMP8:%.*]] = add nsw <vscale x 4 x i16> [[WIDE_MASKED_LOAD]], splat (i16 1)
+; CHECK-NEXT: [[TMP33:%.*]] = add nsw <vscale x 4 x i16> [[WIDE_MASKED_LOAD3]], splat (i16 1)
; CHECK-NEXT: [[TMP9:%.*]] = getelementptr i16, ptr [[ARRAY]], i64 [[IV]]
+; CHECK-NEXT: [[TMP17:%.*]] = getelementptr i16, ptr [[TMP9]], i64 [[TMP2]]
; CHECK-NEXT: call void @llvm.masked.store.nxv4i16.p0(<vscale x 4 x i16> [[TMP8]], ptr align 2 [[TMP9]], <vscale x 4 x i1> [[TMP6]])
-; CHECK-NEXT: [[TMP11:%.*]] = call i1 @llvm.vector.reduce.or.nxv4i1(<vscale x 4 x i1> [[TMP4]])
+; CHECK-NEXT: call void @llvm.masked.store.nxv4i16.p0(<vscale x 4 x i16> [[TMP33]], ptr align 2 [[TMP17]], <vscale x 4 x i1> [[UNCOUNTABLE_EXIT_MASK2]])
+; CHECK-NEXT: [[TMP18:%.*]] = or <vscale x 4 x i1> [[TMP4]], [[TMP21]]
+; CHECK-NEXT: [[TMP11:%.*]] = call i1 @llvm.vector.reduce.or.nxv4i1(<vscale x 4 x i1> [[TMP18]])
; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[IV]], [[TMP1]]
; CHECK-NEXT: [[TMP12:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
+; CHECK-NEXT: [[VEC_IND_NEXT]] = add nuw nsw <vscale x 4 x i64> [[STEP_ADD]], [[BROADCAST_SPLAT]]
; CHECK-NEXT: [[TMP13:%.*]] = or i1 [[TMP11]], [[TMP12]]
; CHECK-NEXT: br i1 [[TMP13]], label %[[MIDDLE_BLOCK:.*]], label %[[FOR_BODY]], !llvm.loop [[LOOP6:![0-9]+]]
; CHECK: [[MIDDLE_BLOCK]]:
+; CHECK-NEXT: [[TMP22:%.*]] = call i64 @llvm.vscale.i64()
+; CHECK-NEXT: [[TMP23:%.*]] = mul nuw i64 [[TMP22]], 4
+; CHECK-NEXT: [[TMP24:%.*]] = call i64 @llvm.experimental.cttz.elts.i64.nxv4i1(<vscale x 4 x i1> [[TMP21]], i1 false)
+; CHECK-NEXT: [[TMP25:%.*]] = mul i64 [[TMP23]], 1
+; CHECK-NEXT: [[TMP26:%.*]] = add i64 [[TMP25]], [[TMP24]]
; CHECK-NEXT: [[TMP7:%.*]] = call i64 @llvm.experimental.cttz.elts.i64.nxv4i1(<vscale x 4 x i1> [[TMP4]], i1 false)
-; CHECK-NEXT: [[TMP14:%.*]] = add i64 [[IV]], [[TMP7]]
+; CHECK-NEXT: [[TMP28:%.*]] = mul i64 [[TMP23]], 0
+; CHECK-NEXT: [[TMP29:%.*]] = add i64 [[TMP28]], [[TMP7]]
+; CHECK-NEXT: [[TMP30:%.*]] = icmp ne i64 [[TMP7]], [[TMP23]]
+; CHECK-NEXT: [[TMP31:%.*]] = select i1 [[TMP30]], i64 [[TMP29]], i64 [[TMP26]]
+; CHECK-NEXT: [[TMP42:%.*]] = call i64 @llvm.vscale.i64()
+; CHECK-NEXT: [[TMP43:%.*]] = mul nuw i64 [[TMP42]], 4
+; CHECK-NEXT: [[TMP35:%.*]] = mul i64 [[TMP43]], 0
+; CHECK-NEXT: [[TMP36:%.*]] = extractelement <vscale x 4 x i64> [[VEC_IND]], i64 0
+; CHECK-NEXT: [[TMP37:%.*]] = mul i64 [[TMP43]], 1
+; CHECK-NEXT: [[TMP38:%.*]] = sub i64 0, [[TMP37]]
+; CHECK-NEXT: [[TMP39:%.*]] = extractelement <vscale x 4 x i64> [[STEP_ADD]], i64 [[TMP38]]
+; CHECK-NEXT: [[TMP40:%.*]] = icmp uge i64 0, [[TMP37]]
+; CHECK-NEXT: [[TMP41:%.*]] = select i1 [[TMP40]], i64 [[TMP39]], i64 [[TMP36]]
+; CHECK-NEXT: [[TMP14:%.*]] = add i64 [[TMP41]], [[TMP31]]
; CHECK-NEXT: [[TMP15:%.*]] = icmp eq i64 [[TMP14]], 20
; CHECK-NEXT: br i1 [[TMP15]], label %[[EXIT:.*]], label %[[SCALAR_PH1]]
; CHECK: [[SCALAR_PH1]]:
@@ -487,32 +608,72 @@ define void @loop_contains_store_in_latch_block(ptr dereferenceable(40) noalias
; CHECK-SAME: ptr noalias dereferenceable(40) [[ARRAY:%.*]], ptr readonly align 2 dereferenceable(40) [[PRED:%.*]]) #[[ATTR0]] {
; CHECK-NEXT: [[SCALAR_PH1:.*]]:
; CHECK-NEXT: [[TMP0:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-NEXT: [[TMP1:%.*]] = shl nuw i64 [[TMP0]], 2
+; CHECK-NEXT: [[TMP1:%.*]] = shl nuw i64 [[TMP0]], 3
; CHECK-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 20, [[TMP1]]
; CHECK-NEXT: br i1 [[MIN_ITERS_CHECK]], label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]]
; CHECK: [[VECTOR_PH]]:
+; CHECK-NEXT: [[TMP2:%.*]] = shl nuw i64 [[TMP0]], 2
+; CHECK-NEXT: [[BROADCAST_SPLATINSERT:%.*]] = insertelement <vscale x 4 x i64> poison, i64 [[TMP2]], i64 0
+; CHECK-NEXT: [[BROADCAST_SPLAT:%.*]] = shufflevector <vscale x 4 x i64> [[BROADCAST_SPLATINSERT]], <vscale x 4 x i64> poison, <vscale x 4 x i32> zeroinitializer
; CHECK-NEXT: [[N_MOD_VF:%.*]] = urem i64 20, [[TMP1]]
; CHECK-NEXT: [[N_VEC:%.*]] = sub i64 20, [[N_MOD_VF]]
+; CHECK-NEXT: [[TMP17:%.*]] = call <vscale x 4 x i64> @llvm.stepvector.nxv4i64()
; CHECK-NEXT: br label %[[FOR_BODY1:.*]]
; CHECK: [[FOR_BODY1]]:
; CHECK-NEXT: [[IV1:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[FOR_BODY1]] ]
+; CHECK-NEXT: [[VEC_IND:%.*]] = phi <vscale x 4 x i64> [ [[TMP17]], %[[VECTOR_PH]] ], [ [[VEC_IND_NEXT:%.*]], %[[FOR_BODY1]] ]
+; CHECK-NEXT: [[STEP_ADD:%.*]] = add nuw <vscale x 4 x i64> [[VEC_IND]], [[BROADCAST_SPLAT]]
; CHECK-NEXT: [[EE_ADDR1:%.*]] = getelementptr inbounds nuw i16, ptr [[PRED]], i64 [[IV1]]
+; CHECK-NEXT: [[TMP18:%.*]] = getelementptr inbounds nuw i16, ptr [[EE_ADDR1]], i64 [[TMP2]]
; CHECK-NEXT: [[WIDE_LOAD:%.*]] = load <vscale x 4 x i16>, ptr [[EE_ADDR1]], align 2
+; CHECK-NEXT: [[WIDE_LOAD1:%.*]] = load <vscale x 4 x i16>, ptr [[TMP18]], align 2
; CHECK-NEXT: [[TMP3:%.*]] = icmp sgt <vscale x 4 x i16> [[WIDE_LOAD]], splat (i16 500)
+; CHECK-NEXT: [[TMP19:%.*]] = icmp sgt <vscale x 4 x i16> [[WIDE_LOAD1]], splat (i16 500)
; CHECK-NEXT: [[TMP4:%.*]] = freeze <vscale x 4 x i1> [[TMP3]]
+; CHECK-NEXT: [[TMP9:%.*]] = freeze <vscale x 4 x i1> [[TMP19]]
; CHECK-NEXT: [[TMP5:%.*]] = call <vscale x 4 x i1> @llvm.mask.beforefirst.nxv4i1(<vscale x 4 x i1> [[TMP4]])
+; CHECK-NEXT: [[TMP25:%.*]] = xor <vscale x 4 x i1> [[TMP5]], splat (i1 true)
+; CHECK-NEXT: [[TMP30:%.*]] = call i1 @llvm.vector.reduce.or.nxv4i1(<vscale x 4 x i1> [[TMP25]])
+; CHECK-NEXT: [[BROADCAST_SPLATINSERT2:%.*]] = insertelement <vscale x 4 x i1> poison, i1 [[TMP30]], i64 0
+; CHECK-NEXT: [[BROADCAST_SPLAT3:%.*]] = shufflevector <vscale x 4 x i1> [[BROADCAST_SPLATINSERT2]], <vscale x 4 x i1> poison, <vscale x 4 x i32> zeroinitializer
+; CHECK-NEXT: [[TMP32:%.*]] = or <vscale x 4 x i1> [[TMP9]], [[BROADCAST_SPLAT3]]
+; CHECK-NEXT: [[UNCOUNTABLE_EXIT_MASK2:%.*]] = call <vscale x 4 x i1> @llvm.mask.beforefirst.nxv4i1(<vscale x 4 x i1> [[TMP32]])
; CHECK-NEXT: [[TMP7:%.*]] = getelementptr i16, ptr [[ARRAY]], i64 [[IV1]]
+; CHECK-NEXT: [[TMP31:%.*]] = getelementptr i16, ptr [[TMP7]], i64 [[TMP2]]
; CHECK-NEXT: [[WIDE_MASKED_LOAD:%.*]] = call <vscale x 4 x i16> @llvm.masked.load.nxv4i16.p0(ptr align 2 [[TMP7]], <vscale x 4 x i1> [[TMP5]], <vscale x 4 x i16> poison)
+; CHECK-NEXT: [[WIDE_MASKED_LOAD3:%.*]] = call <vscale x 4 x i16> @llvm.masked.load.nxv4i16.p0(ptr align 2 [[TMP31]], <vscale x 4 x i1> [[UNCOUNTABLE_EXIT_MASK2]], <vscale x 4 x i16> poison)
; CHECK-NEXT: [[TMP8:%.*]] = add nsw <vscale x 4 x i16> [[WIDE_MASKED_LOAD]], splat (i16 1)
+; CHECK-NEXT: [[TMP15:%.*]] = add nsw <vscale x 4 x i16> [[WIDE_MASKED_LOAD3]], splat (i16 1)
; CHECK-NEXT: call void @llvm.masked.store.nxv4i16.p0(<vscale x 4 x i16> [[TMP8]], ptr align 2 [[TMP7]], <vscale x 4 x i1> [[TMP5]])
-; CHECK-NEXT: [[TMP10:%.*]] = call i1 @llvm.vector.reduce.or.nxv4i1(<vscale x 4 x i1> [[TMP4]])
+; CHECK-NEXT: call void @llvm.masked.store.nxv4i16.p0(<vscale x 4 x i16> [[TMP15]], ptr align 2 [[TMP31]], <vscale x 4 x i1> [[UNCOUNTABLE_EXIT_MASK2]])
+; CHECK-NEXT: [[TMP16:%.*]] = or <vscale x 4 x i1> [[TMP4]], [[TMP9]]
+; CHECK-NEXT: [[TMP10:%.*]] = call i1 @llvm.vector.reduce.or.nxv4i1(<vscale x 4 x i1> [[TMP16]])
; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[IV1]], [[TMP1]]
; CHECK-NEXT: [[TMP11:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
+; CHECK-NEXT: [[VEC_IND_NEXT]] = add nuw nsw <vscale x 4 x i64> [[STEP_ADD]], [[BROADCAST_SPLAT]]
; CHECK-NEXT: [[TMP12:%.*]] = or i1 [[TMP10]], [[TMP11]]
; CHECK-NEXT: br i1 [[TMP12]], label %[[MIDDLE_BLOCK:.*]], label %[[FOR_BODY1]], !llvm.loop [[LOOP8:![0-9]+]]
; CHECK: [[MIDDLE_BLOCK]]:
+; CHECK-NEXT: [[TMP20:%.*]] = call i64 @llvm.vscale.i64()
+; CHECK-NEXT: [[TMP21:%.*]] = mul nuw i64 [[TMP20]], 4
+; CHECK-NEXT: [[TMP22:%.*]] = call i64 @llvm.experimental.cttz.elts.i64.nxv4i1(<vscale x 4 x i1> [[TMP9]], i1 false)
+; CHECK-NEXT: [[TMP23:%.*]] = mul i64 [[TMP21]], 1
+; CHECK-NEXT: [[TMP24:%.*]] = add i64 [[TMP23]], [[TMP22]]
; CHECK-NEXT: [[TMP6:%.*]] = call i64 @llvm.experimental.cttz.elts.i64.nxv4i1(<vscale x 4 x i1> [[TMP4]], i1 false)
-; CHECK-NEXT: [[TMP13:%.*]] = add i64 [[IV1]], [[TMP6]]
+; CHECK-NEXT: [[TMP26:%.*]] = mul i64 [[TMP21]], 0
+; CHECK-NEXT: [[TMP27:%.*]] = add i64 [[TMP26]], [[TMP6]]
+; CHECK-NEXT: [[TMP28:%.*]] = icmp ne i64 [[TMP6]], [[TMP21]]
+; CHECK-NEXT: [[TMP29:%.*]] = select i1 [[TMP28]], i64 [[TMP27]], i64 [[TMP24]]
+; CHECK-NEXT: [[TMP40:%.*]] = call i64 @llvm.vscale.i64()
+; CHECK-NEXT: [[TMP41:%.*]] = mul nuw i64 [[TMP40]], 4
+; CHECK-NEXT: [[TMP33:%.*]] = mul i64 [[TMP41]], 0
+; CHECK-NEXT: [[TMP34:%.*]] = extractelement <vscale x 4 x i64> [[VEC_IND]], i64 0
+; CHECK-NEXT: [[TMP35:%.*]] = mul i64 [[TMP41]], 1
+; CHECK-NEXT: [[TMP36:%.*]] = sub i64 0, [[TMP35]]
+; CHECK-NEXT: [[TMP37:%.*]] = extractelement <vscale x 4 x i64> [[STEP_ADD]], i64 [[TMP36]]
+; CHECK-NEXT: [[TMP38:%.*]] = icmp uge i64 0, [[TMP35]]
+; CHECK-NEXT: [[TMP39:%.*]] = select i1 [[TMP38]], i64 [[TMP37]], i64 [[TMP34]]
+; CHECK-NEXT: [[TMP13:%.*]] = add i64 [[TMP39]], [[TMP29]]
; CHECK-NEXT: [[TMP14:%.*]] = icmp eq i64 [[TMP13]], 20
; CHECK-NEXT: br i1 [[TMP14]], label %[[EXIT:.*]], label %[[SCALAR_PH]]
; CHECK: [[SCALAR_PH]]:
@@ -1159,32 +1320,72 @@ define i32 @uncountable_exit_with_separate_exit_block(ptr dereferenceable(40) no
; CHECK-SAME: ptr noalias dereferenceable(40) [[ARRAY:%.*]], ptr readonly align 2 dereferenceable(40) [[PRED:%.*]]) #[[ATTR0]] {
; CHECK-NEXT: [[SCALAR_PH:.*]]:
; CHECK-NEXT: [[TMP0:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-NEXT: [[TMP1:%.*]] = shl nuw i64 [[TMP0]], 2
+; CHECK-NEXT: [[TMP1:%.*]] = shl nuw i64 [[TMP0]], 3
; CHECK-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 20, [[TMP1]]
; CHECK-NEXT: br i1 [[MIN_ITERS_CHECK]], label %[[SCALAR_PH1:.*]], label %[[VECTOR_PH:.*]]
; CHECK: [[VECTOR_PH]]:
+; CHECK-NEXT: [[TMP2:%.*]] = shl nuw i64 [[TMP0]], 2
+; CHECK-NEXT: [[BROADCAST_SPLATINSERT:%.*]] = insertelement <vscale x 4 x i64> poison, i64 [[TMP2]], i64 0
+; CHECK-NEXT: [[BROADCAST_SPLAT:%.*]] = shufflevector <vscale x 4 x i64> [[BROADCAST_SPLATINSERT]], <vscale x 4 x i64> poison, <vscale x 4 x i32> zeroinitializer
; CHECK-NEXT: [[N_MOD_VF:%.*]] = urem i64 20, [[TMP1]]
; CHECK-NEXT: [[N_VEC:%.*]] = sub i64 20, [[N_MOD_VF]]
+; CHECK-NEXT: [[TMP17:%.*]] = call <vscale x 4 x i64> @llvm.stepvector.nxv4i64()
; CHECK-NEXT: br label %[[FOR_BODY1:.*]]
; CHECK: [[FOR_BODY1]]:
; CHECK-NEXT: [[IV1:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[FOR_BODY1]] ]
+; CHECK-NEXT: [[VEC_IND:%.*]] = phi <vscale x 4 x i64> [ [[TMP17]], %[[VECTOR_PH]] ], [ [[VEC_IND_NEXT:%.*]], %[[FOR_BODY1]] ]
+; CHECK-NEXT: [[STEP_ADD:%.*]] = add nuw <vscale x 4 x i64> [[VEC_IND]], [[BROADCAST_SPLAT]]
; CHECK-NEXT: [[TMP5:%.*]] = getelementptr inbounds nuw i16, ptr [[PRED]], i64 [[IV1]]
+; CHECK-NEXT: [[TMP18:%.*]] = getelementptr inbounds nuw i16, ptr [[TMP5]], i64 [[TMP2]]
; CHECK-NEXT: [[WIDE_LOAD:%.*]] = load <vscale x 4 x i16>, ptr [[TMP5]], align 2
+; CHECK-NEXT: [[WIDE_LOAD1:%.*]] = load <vscale x 4 x i16>, ptr [[TMP18]], align 2
; CHECK-NEXT: [[TMP3:%.*]] = icmp sgt <vscale x 4 x i16> [[WIDE_LOAD]], splat (i16 500)
+; CHECK-NEXT: [[TMP19:%.*]] = icmp sgt <vscale x 4 x i16> [[WIDE_LOAD1]], splat (i16 500)
; CHECK-NEXT: [[TMP4:%.*]] = freeze <vscale x 4 x i1> [[TMP3]]
+; CHECK-NEXT: [[TMP9:%.*]] = freeze <vscale x 4 x i1> [[TMP19]]
; CHECK-NEXT: [[TMP6:%.*]] = call <vscale x 4 x i1> @llvm.mask.beforefirst.nxv4i1(<vscale x 4 x i1> [[TMP4]])
+; CHECK-NEXT: [[TMP25:%.*]] = xor <vscale x 4 x i1> [[TMP6]], splat (i1 true)
+; CHECK-NEXT: [[TMP30:%.*]] = call i1 @llvm.vector.reduce.or.nxv4i1(<vscale x 4 x i1> [[TMP25]])
+; CHECK-NEXT: [[BROADCAST_SPLATINSERT2:%.*]] = insertelement <vscale x 4 x i1> poison, i1 [[TMP30]], i64 0
+; CHECK-NEXT: [[BROADCAST_SPLAT3:%.*]] = shufflevector <vscale x 4 x i1> [[BROADCAST_SPLATINSERT2]], <vscale x 4 x i1> poison, <vscale x 4 x i32> zeroinitializer
+; CHECK-NEXT: [[TMP32:%.*]] = or <vscale x 4 x i1> [[TMP9]], [[BROADCAST_SPLAT3]]
+; CHECK-NEXT: [[UNCOUNTABLE_EXIT_MASK2:%.*]] = call <vscale x 4 x i1> @llvm.mask.beforefirst.nxv4i1(<vscale x 4 x i1> [[TMP32]])
; CHECK-NEXT: [[ST_ADDR1:%.*]] = getelementptr i16, ptr [[ARRAY]], i64 [[IV1]]
+; CHECK-NEXT: [[TMP31:%.*]] = getelementptr i16, ptr [[ST_ADDR1]], i64 [[TMP2]]
; CHECK-NEXT: [[WIDE_MASKED_LOAD:%.*]] = call <vscale x 4 x i16> @llvm.masked.load.nxv4i16.p0(ptr align 2 [[ST_ADDR1]], <vscale x 4 x i1> [[TMP6]], <vscale x 4 x i16> poison)
+; CHECK-NEXT: [[WIDE_MASKED_LOAD3:%.*]] = call <vscale x 4 x i16> @llvm.masked.load.nxv4i16.p0(ptr align 2 [[TMP31]], <vscale x 4 x i1> [[UNCOUNTABLE_EXIT_MASK2]], <vscale x 4 x i16> poison)
; CHECK-NEXT: [[TMP8:%.*]] = add nsw <vscale x 4 x i16> [[WIDE_MASKED_LOAD]], splat (i16 1)
+; CHECK-NEXT: [[TMP15:%.*]] = add nsw <vscale x 4 x i16> [[WIDE_MASKED_LOAD3]], splat (i16 1)
; CHECK-NEXT: call void @llvm.masked.store.nxv4i16.p0(<vscale x 4 x i16> [[TMP8]], ptr align 2 [[ST_ADDR1]], <vscale x 4 x i1> [[TMP6]])
-; CHECK-NEXT: [[TMP10:%.*]] = call i1 @llvm.vector.reduce.or.nxv4i1(<vscale x 4 x i1> [[TMP4]])
+; CHECK-NEXT: call void @llvm.masked.store.nxv4i16.p0(<vscale x 4 x i16> [[TMP15]], ptr align 2 [[TMP31]], <vscale x 4 x i1> [[UNCOUNTABLE_EXIT_MASK2]])
+; CHECK-NEXT: [[TMP16:%.*]] = or <vscale x 4 x i1> [[TMP4]], [[TMP9]]
+; CHECK-NEXT: [[TMP10:%.*]] = call i1 @llvm.vector.reduce.or.nxv4i1(<vscale x 4 x i1> [[TMP16]])
; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[IV1]], [[TMP1]]
; CHECK-NEXT: [[TMP11:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
+; CHECK-NEXT: [[VEC_IND_NEXT]] = add nuw nsw <vscale x 4 x i64> [[STEP_ADD]], [[BROADCAST_SPLAT]]
; CHECK-NEXT: [[TMP12:%.*]] = or i1 [[TMP10]], [[TMP11]]
; CHECK-NEXT: br i1 [[TMP12]], label %[[MIDDLE_BLOCK:.*]], label %[[FOR_BODY1]], !llvm.loop [[LOOP10:![0-9]+]]
; CHECK: [[MIDDLE_BLOCK]]:
+; CHECK-NEXT: [[TMP20:%.*]] = call i64 @llvm.vscale.i64()
+; CHECK-NEXT: [[TMP21:%.*]] = mul nuw i64 [[TMP20]], 4
+; CHECK-NEXT: [[TMP22:%.*]] = call i64 @llvm.experimental.cttz.elts.i64.nxv4i1(<vscale x 4 x i1> [[TMP9]], i1 false)
+; CHECK-NEXT: [[TMP23:%.*]] = mul i64 [[TMP21]], 1
+; CHECK-NEXT: [[TMP24:%.*]] = add i64 [[TMP23]], [[TMP22]]
; CHECK-NEXT: [[TMP7:%.*]] = call i64 @llvm.experimental.cttz.elts.i64.nxv4i1(<vscale x 4 x i1> [[TMP4]], i1 false)
-; CHECK-NEXT: [[TMP13:%.*]] = add i64 [[IV1]], [[TMP7]]
+; CHECK-NEXT: [[TMP26:%.*]] = mul i64 [[TMP21]], 0
+; CHECK-NEXT: [[TMP27:%.*]] = add i64 [[TMP26]], [[TMP7]]
+; CHECK-NEXT: [[TMP28:%.*]] = icmp ne i64 [[TMP7]], [[TMP21]]
+; CHECK-NEXT: [[TMP29:%.*]] = select i1 [[TMP28]], i64 [[TMP27]], i64 [[TMP24]]
+; CHECK-NEXT: [[TMP40:%.*]] = call i64 @llvm.vscale.i64()
+; CHECK-NEXT: [[TMP41:%.*]] = mul nuw i64 [[TMP40]], 4
+; CHECK-NEXT: [[TMP33:%.*]] = mul i64 [[TMP41]], 0
+; CHECK-NEXT: [[TMP34:%.*]] = extractelement <vscale x 4 x i64> [[VEC_IND]], i64 0
+; CHECK-NEXT: [[TMP35:%.*]] = mul i64 [[TMP41]], 1
+; CHECK-NEXT: [[TMP36:%.*]] = sub i64 0, [[TMP35]]
+; CHECK-NEXT: [[TMP37:%.*]] = extractelement <vscale x 4 x i64> [[STEP_ADD]], i64 [[TMP36]]
+; CHECK-NEXT: [[TMP38:%.*]] = icmp uge i64 0, [[TMP35]]
+; CHECK-NEXT: [[TMP39:%.*]] = select i1 [[TMP38]], i64 [[TMP37]], i64 [[TMP34]]
+; CHECK-NEXT: [[TMP13:%.*]] = add i64 [[TMP39]], [[TMP29]]
; CHECK-NEXT: [[TMP14:%.*]] = icmp eq i64 [[TMP13]], 20
; CHECK-NEXT: br i1 [[TMP14]], label %[[EXIT_COUNTABLE:.*]], label %[[SCALAR_PH1]]
; CHECK: [[SCALAR_PH1]]:
diff --git a/llvm/test/Transforms/LoopVectorize/interleave_uncountable_exits.ll b/llvm/test/Transforms/LoopVectorize/interleave_uncountable_exits.ll
index d3dae6a8c9651f..f81ad5ee2f6d3d 100644
--- a/llvm/test/Transforms/LoopVectorize/interleave_uncountable_exits.ll
+++ b/llvm/test/Transforms/LoopVectorize/interleave_uncountable_exits.ll
@@ -1,8 +1,7 @@
; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --check-globals none --version 6
; RUN: opt -S < %s -p loop-vectorize -force-vector-width=4 -force-vector-interleave=2 -force-target-max-vector-interleave=2 -force-target-supports-masked-memory-ops -enable-early-exit-vectorization-with-side-effects | FileCheck %s
+; RUN: opt -S < %s -p loop-vectorize -force-vector-width=1 -force-vector-interleave=2 -force-target-max-vector-interleave=2 -force-target-supports-masked-memory-ops -enable-early-exit-vectorization-with-side-effects | FileCheck %s --check-prefix=VF1IC2
-; We do not support interleaving loops with early exits yet, so make sure we don't
-; even with a flag to force it.
define void @loop_contains_store_condition_load_has_single_user(ptr dereferenceable(40) noalias %array, ptr align 2 dereferenceable(40) readonly %pred) {
; CHECK-LABEL: define void @loop_contains_store_condition_load_has_single_user(
; CHECK-SAME: ptr noalias dereferenceable(40) [[ARRAY:%.*]], ptr readonly align 2 dereferenceable(40) [[PRED:%.*]]) {
@@ -12,23 +11,49 @@ define void @loop_contains_store_condition_load_has_single_user(ptr dereferencea
; CHECK-NEXT: br label %[[VECTOR_BODY:.*]]
; CHECK: [[VECTOR_BODY]]:
; CHECK-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-NEXT: [[VEC_IND:%.*]] = phi <4 x i64> [ <i64 0, i64 1, i64 2, i64 3>, %[[VECTOR_PH]] ], [ [[VEC_IND_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-NEXT: [[STEP_ADD:%.*]] = add nuw <4 x i64> [[VEC_IND]], splat (i64 4)
; CHECK-NEXT: [[TMP1:%.*]] = getelementptr inbounds nuw i16, ptr [[PRED]], i64 [[INDEX]]
+; CHECK-NEXT: [[TMP13:%.*]] = getelementptr inbounds nuw i16, ptr [[TMP1]], i64 4
; CHECK-NEXT: [[WIDE_LOAD:%.*]] = load <4 x i16>, ptr [[TMP1]], align 2
+; CHECK-NEXT: [[WIDE_LOAD1:%.*]] = load <4 x i16>, ptr [[TMP13]], align 2
; CHECK-NEXT: [[TMP2:%.*]] = icmp sgt <4 x i16> [[WIDE_LOAD]], splat (i16 500)
+; CHECK-NEXT: [[TMP14:%.*]] = icmp sgt <4 x i16> [[WIDE_LOAD1]], splat (i16 500)
; CHECK-NEXT: [[TMP5:%.*]] = freeze <4 x i1> [[TMP2]]
+; CHECK-NEXT: [[TMP15:%.*]] = freeze <4 x i1> [[TMP14]]
; CHECK-NEXT: [[UNCOUNTABLE_EXIT_MASK:%.*]] = call <4 x i1> @llvm.mask.beforefirst.v4i1(<4 x i1> [[TMP5]])
+; CHECK-NEXT: [[TMP21:%.*]] = xor <4 x i1> [[UNCOUNTABLE_EXIT_MASK]], splat (i1 true)
+; CHECK-NEXT: [[TMP22:%.*]] = call i1 @llvm.vector.reduce.or.v4i1(<4 x i1> [[TMP21]])
+; CHECK-NEXT: [[BROADCAST_SPLATINSERT:%.*]] = insertelement <4 x i1> poison, i1 [[TMP22]], i64 0
+; CHECK-NEXT: [[BROADCAST_SPLAT:%.*]] = shufflevector <4 x i1> [[BROADCAST_SPLATINSERT]], <4 x i1> poison, <4 x i32> zeroinitializer
+; CHECK-NEXT: [[TMP24:%.*]] = or <4 x i1> [[TMP15]], [[BROADCAST_SPLAT]]
+; CHECK-NEXT: [[UNCOUNTABLE_EXIT_MASK2:%.*]] = call <4 x i1> @llvm.mask.beforefirst.v4i1(<4 x i1> [[TMP24]])
; CHECK-NEXT: [[TMP0:%.*]] = getelementptr i16, ptr [[ARRAY]], i64 [[INDEX]]
+; CHECK-NEXT: [[TMP23:%.*]] = getelementptr i16, ptr [[TMP0]], i64 4
; CHECK-NEXT: [[WIDE_MASKED_LOAD:%.*]] = call <4 x i16> @llvm.masked.load.v4i16.p0(ptr align 2 [[TMP0]], <4 x i1> [[UNCOUNTABLE_EXIT_MASK]], <4 x i16> poison)
+; CHECK-NEXT: [[WIDE_MASKED_LOAD3:%.*]] = call <4 x i16> @llvm.masked.load.v4i16.p0(ptr align 2 [[TMP23]], <4 x i1> [[UNCOUNTABLE_EXIT_MASK2]], <4 x i16> poison)
; CHECK-NEXT: [[TMP4:%.*]] = add nsw <4 x i16> [[WIDE_MASKED_LOAD]], splat (i16 1)
+; CHECK-NEXT: [[TMP11:%.*]] = add nsw <4 x i16> [[WIDE_MASKED_LOAD3]], splat (i16 1)
; CHECK-NEXT: call void @llvm.masked.store.v4i16.p0(<4 x i16> [[TMP4]], ptr align 2 [[TMP0]], <4 x i1> [[UNCOUNTABLE_EXIT_MASK]])
-; CHECK-NEXT: [[TMP6:%.*]] = call i1 @llvm.vector.reduce.or.v4i1(<4 x i1> [[TMP5]])
-; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 4
-; CHECK-NEXT: [[TMP7:%.*]] = icmp eq i64 [[INDEX_NEXT]], 20
+; CHECK-NEXT: call void @llvm.masked.store.v4i16.p0(<4 x i16> [[TMP11]], ptr align 2 [[TMP23]], <4 x i1> [[UNCOUNTABLE_EXIT_MASK2]])
+; CHECK-NEXT: [[TMP12:%.*]] = or <4 x i1> [[TMP5]], [[TMP15]]
+; CHECK-NEXT: [[TMP6:%.*]] = call i1 @llvm.vector.reduce.or.v4i1(<4 x i1> [[TMP12]])
+; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 8
+; CHECK-NEXT: [[TMP7:%.*]] = icmp eq i64 [[INDEX_NEXT]], 16
+; CHECK-NEXT: [[VEC_IND_NEXT]] = add nuw nsw <4 x i64> [[STEP_ADD]], splat (i64 4)
; CHECK-NEXT: [[TMP8:%.*]] = or i1 [[TMP6]], [[TMP7]]
; CHECK-NEXT: br i1 [[TMP8]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP0:![0-9]+]]
; CHECK: [[MIDDLE_BLOCK]]:
-; CHECK-NEXT: [[TMP3:%.*]] = call i64 @llvm.experimental.cttz.elts.i64.v4i1(<4 x i1> [[TMP5]], i1 false)
-; CHECK-NEXT: [[TMP9:%.*]] = add i64 [[INDEX]], [[TMP3]]
+; CHECK-NEXT: [[TMP16:%.*]] = call i64 @llvm.experimental.cttz.elts.i64.v4i1(<4 x i1> [[TMP15]], i1 false)
+; CHECK-NEXT: [[TMP17:%.*]] = add i64 4, [[TMP16]]
+; CHECK-NEXT: [[TMP18:%.*]] = call i64 @llvm.experimental.cttz.elts.i64.v4i1(<4 x i1> [[TMP5]], i1 false)
+; CHECK-NEXT: [[TMP19:%.*]] = add i64 0, [[TMP18]]
+; CHECK-NEXT: [[TMP20:%.*]] = icmp ne i64 [[TMP18]], 4
+; CHECK-NEXT: [[TMP3:%.*]] = select i1 [[TMP20]], i64 [[TMP19]], i64 [[TMP17]]
+; CHECK-NEXT: [[TMP26:%.*]] = extractelement <4 x i64> [[VEC_IND]], i64 0
+; CHECK-NEXT: [[TMP27:%.*]] = extractelement <4 x i64> [[STEP_ADD]], i64 -4
+; CHECK-NEXT: [[TMP25:%.*]] = select i1 false, i64 [[TMP27]], i64 [[TMP26]]
+; CHECK-NEXT: [[TMP9:%.*]] = add i64 [[TMP25]], [[TMP3]]
; CHECK-NEXT: [[TMP10:%.*]] = icmp eq i64 [[TMP9]], 20
; CHECK-NEXT: br i1 [[TMP10]], label %[[EXIT:.*]], label %[[SCALAR_PH:.*]]
; CHECK: [[SCALAR_PH]]:
@@ -50,6 +75,80 @@ define void @loop_contains_store_condition_load_has_single_user(ptr dereferencea
; CHECK: [[EXIT]]:
; CHECK-NEXT: ret void
;
+; VF1IC2-LABEL: define void @loop_contains_store_condition_load_has_single_user(
+; VF1IC2-SAME: ptr noalias dereferenceable(40) [[ARRAY:%.*]], ptr readonly align 2 dereferenceable(40) [[PRED:%.*]]) {
+; VF1IC2-NEXT: [[ENTRY:.*:]]
+; VF1IC2-NEXT: br label %[[VECTOR_PH:.*]]
+; VF1IC2: [[VECTOR_PH]]:
+; VF1IC2-NEXT: br label %[[VECTOR_BODY:.*]]
+; VF1IC2: [[VECTOR_BODY]]:
+; VF1IC2-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[PRED_STORE_CONTINUE2:.*]] ]
+; VF1IC2-NEXT: [[TMP0:%.*]] = add i64 [[INDEX]], 0
+; VF1IC2-NEXT: [[TMP1:%.*]] = add i64 [[INDEX]], 1
+; VF1IC2-NEXT: [[TMP2:%.*]] = getelementptr inbounds nuw i16, ptr [[PRED]], i64 [[TMP0]]
+; VF1IC2-NEXT: [[TMP3:%.*]] = getelementptr inbounds nuw i16, ptr [[PRED]], i64 [[TMP1]]
+; VF1IC2-NEXT: [[TMP4:%.*]] = load i16, ptr [[TMP2]], align 2
+; VF1IC2-NEXT: [[TMP5:%.*]] = load i16, ptr [[TMP3]], align 2
+; VF1IC2-NEXT: [[TMP6:%.*]] = icmp sgt i16 [[TMP4]], 500
+; VF1IC2-NEXT: [[TMP7:%.*]] = icmp sgt i16 [[TMP5]], 500
+; VF1IC2-NEXT: [[TMP8:%.*]] = freeze i1 [[TMP6]]
+; VF1IC2-NEXT: [[TMP9:%.*]] = freeze i1 [[TMP7]]
+; VF1IC2-NEXT: [[TMP10:%.*]] = xor i1 [[TMP8]], true
+; VF1IC2-NEXT: [[TMP11:%.*]] = or i1 [[TMP9]], [[TMP8]]
+; VF1IC2-NEXT: [[TMP12:%.*]] = xor i1 [[TMP11]], true
+; VF1IC2-NEXT: br i1 [[TMP10]], label %[[PRED_STORE_IF:.*]], label %[[PRED_STORE_CONTINUE:.*]]
+; VF1IC2: [[PRED_STORE_IF]]:
+; VF1IC2-NEXT: [[TMP13:%.*]] = getelementptr inbounds nuw i16, ptr [[ARRAY]], i64 [[TMP0]]
+; VF1IC2-NEXT: [[TMP14:%.*]] = load i16, ptr [[TMP13]], align 2
+; VF1IC2-NEXT: [[TMP15:%.*]] = add nsw i16 [[TMP14]], 1
+; VF1IC2-NEXT: store i16 [[TMP15]], ptr [[TMP13]], align 2
+; VF1IC2-NEXT: br label %[[PRED_STORE_CONTINUE]]
+; VF1IC2: [[PRED_STORE_CONTINUE]]:
+; VF1IC2-NEXT: br i1 [[TMP12]], label %[[PRED_STORE_IF1:.*]], label %[[PRED_STORE_CONTINUE2]]
+; VF1IC2: [[PRED_STORE_IF1]]:
+; VF1IC2-NEXT: [[TMP16:%.*]] = getelementptr inbounds nuw i16, ptr [[ARRAY]], i64 [[TMP1]]
+; VF1IC2-NEXT: [[TMP17:%.*]] = load i16, ptr [[TMP16]], align 2
+; VF1IC2-NEXT: [[TMP18:%.*]] = add nsw i16 [[TMP17]], 1
+; VF1IC2-NEXT: store i16 [[TMP18]], ptr [[TMP16]], align 2
+; VF1IC2-NEXT: br label %[[PRED_STORE_CONTINUE2]]
+; VF1IC2: [[PRED_STORE_CONTINUE2]]:
+; VF1IC2-NEXT: [[TMP19:%.*]] = or i1 [[TMP8]], [[TMP9]]
+; VF1IC2-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 2
+; VF1IC2-NEXT: [[TMP20:%.*]] = icmp eq i64 [[INDEX_NEXT]], 20
+; VF1IC2-NEXT: [[TMP21:%.*]] = or i1 [[TMP19]], [[TMP20]]
+; VF1IC2-NEXT: br i1 [[TMP21]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP0:![0-9]+]]
+; VF1IC2: [[MIDDLE_BLOCK]]:
+; VF1IC2-NEXT: [[TMP22:%.*]] = icmp eq i1 [[TMP9]], false
+; VF1IC2-NEXT: [[TMP23:%.*]] = zext i1 [[TMP22]] to i64
+; VF1IC2-NEXT: [[TMP24:%.*]] = add i64 1, [[TMP23]]
+; VF1IC2-NEXT: [[TMP25:%.*]] = icmp eq i1 [[TMP8]], false
+; VF1IC2-NEXT: [[TMP26:%.*]] = zext i1 [[TMP25]] to i64
+; VF1IC2-NEXT: [[TMP27:%.*]] = add i64 0, [[TMP26]]
+; VF1IC2-NEXT: [[TMP28:%.*]] = icmp ne i64 [[TMP26]], 1
+; VF1IC2-NEXT: [[TMP29:%.*]] = select i1 [[TMP28]], i64 [[TMP27]], i64 [[TMP24]]
+; VF1IC2-NEXT: [[TMP30:%.*]] = select i1 false, i64 [[TMP1]], i64 [[TMP0]]
+; VF1IC2-NEXT: [[TMP31:%.*]] = add i64 [[TMP30]], [[TMP29]]
+; VF1IC2-NEXT: [[TMP32:%.*]] = icmp eq i64 [[TMP31]], 20
+; VF1IC2-NEXT: br i1 [[TMP32]], label %[[EXIT:.*]], label %[[SCALAR_PH:.*]]
+; VF1IC2: [[SCALAR_PH]]:
+; VF1IC2-NEXT: br label %[[FOR_BODY:.*]]
+; VF1IC2: [[FOR_BODY]]:
+; VF1IC2-NEXT: [[IV:%.*]] = phi i64 [ [[TMP31]], %[[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], %[[FOR_INC:.*]] ]
+; VF1IC2-NEXT: [[ST_ADDR:%.*]] = getelementptr inbounds nuw i16, ptr [[ARRAY]], i64 [[IV]]
+; VF1IC2-NEXT: [[DATA:%.*]] = load i16, ptr [[ST_ADDR]], align 2
+; VF1IC2-NEXT: [[INC:%.*]] = add nsw i16 [[DATA]], 1
+; VF1IC2-NEXT: store i16 [[INC]], ptr [[ST_ADDR]], align 2
+; VF1IC2-NEXT: [[EE_ADDR:%.*]] = getelementptr inbounds nuw i16, ptr [[PRED]], i64 [[IV]]
+; VF1IC2-NEXT: [[EE_VAL:%.*]] = load i16, ptr [[EE_ADDR]], align 2
+; VF1IC2-NEXT: [[EE_COND:%.*]] = icmp sgt i16 [[EE_VAL]], 500
+; VF1IC2-NEXT: br i1 [[EE_COND]], label %[[EXIT]], label %[[FOR_INC]]
+; VF1IC2: [[FOR_INC]]:
+; VF1IC2-NEXT: [[IV_NEXT]] = add nuw nsw i64 [[IV]], 1
+; VF1IC2-NEXT: [[COUNTED_COND:%.*]] = icmp eq i64 [[IV_NEXT]], 20
+; VF1IC2-NEXT: br i1 [[COUNTED_COND]], label %[[EXIT]], label %[[FOR_BODY]], !llvm.loop [[LOOP3:![0-9]+]]
+; VF1IC2: [[EXIT]]:
+; VF1IC2-NEXT: ret void
+;
entry:
br label %for.body
diff --git a/llvm/test/Transforms/PhaseOrdering/AArch64/countable-and-uncountable-exits-combined.ll b/llvm/test/Transforms/PhaseOrdering/AArch64/countable-and-uncountable-exits-combined.ll
index 9d5804468f12b7..af05a83af92318 100644
--- a/llvm/test/Transforms/PhaseOrdering/AArch64/countable-and-uncountable-exits-combined.ll
+++ b/llvm/test/Transforms/PhaseOrdering/AArch64/countable-and-uncountable-exits-combined.ll
@@ -27,33 +27,64 @@ define void @foo() #0 {
; CHECK-SAME: ) local_unnamed_addr #[[ATTR0:[0-9]+]] {
; CHECK-NEXT: [[ENTRY:.*]]:
; CHECK-NEXT: [[TMP0:%.*]] = tail call i64 @llvm.vscale.i64()
-; CHECK-NEXT: [[TMP1:%.*]] = shl nuw i64 [[TMP0]], 2
-; CHECK-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ugt i64 [[TMP0]], 2500
+; CHECK-NEXT: [[TMP1:%.*]] = shl nuw i64 [[TMP0]], 3
+; CHECK-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ugt i64 [[TMP0]], 1250
; CHECK-NEXT: br i1 [[MIN_ITERS_CHECK]], label %[[FOR_BODY_PREHEADER:.*]], label %[[VECTOR_PH:.*]]
; CHECK: [[VECTOR_PH]]:
+; CHECK-NEXT: [[TMP13:%.*]] = shl nuw nsw i64 [[TMP0]], 2
; CHECK-NEXT: [[N_MOD_VF:%.*]] = urem i64 10000, [[TMP1]]
; CHECK-NEXT: [[N_VEC:%.*]] = sub nuw nsw i64 10000, [[N_MOD_VF]]
+; CHECK-NEXT: [[TMP30:%.*]] = tail call <vscale x 4 x i64> @llvm.stepvector.nxv4i64()
+; CHECK-NEXT: [[DOTSCALAR:%.*]] = shl nuw nsw i64 [[TMP0]], 3
+; CHECK-NEXT: [[TMP31:%.*]] = insertelement <vscale x 4 x i64> poison, i64 [[DOTSCALAR]], i64 0
+; CHECK-NEXT: [[REASS_ADD:%.*]] = shufflevector <vscale x 4 x i64> [[TMP31]], <vscale x 4 x i64> poison, <vscale x 4 x i32> zeroinitializer
; CHECK-NEXT: br label %[[FOR_BODY:.*]]
; CHECK: [[FOR_BODY]]:
; CHECK-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[FOR_BODY]] ]
+; CHECK-NEXT: [[VEC_IND:%.*]] = phi <vscale x 4 x i64> [ [[TMP30]], %[[VECTOR_PH]] ], [ [[VEC_IND_NEXT:%.*]], %[[FOR_BODY]] ]
; CHECK-NEXT: [[TMP2:%.*]] = getelementptr inbounds nuw [4 x i8], ptr @c, i64 [[INDEX]]
+; CHECK-NEXT: [[TMP17:%.*]] = getelementptr inbounds nuw [4 x i8], ptr [[TMP2]], i64 [[TMP13]]
; CHECK-NEXT: [[WIDE_LOAD:%.*]] = load <vscale x 4 x i32>, ptr [[TMP2]], align 4
+; CHECK-NEXT: [[WIDE_LOAD3:%.*]] = load <vscale x 4 x i32>, ptr [[TMP17]], align 4
; CHECK-NEXT: [[WIDE_LOAD_FR:%.*]] = freeze <vscale x 4 x i32> [[WIDE_LOAD]]
; CHECK-NEXT: [[TMP3:%.*]] = icmp ne <vscale x 4 x i32> [[WIDE_LOAD_FR]], zeroinitializer
+; CHECK-NEXT: [[WIDE_LOAD3_FR:%.*]] = freeze <vscale x 4 x i32> [[WIDE_LOAD3]]
+; CHECK-NEXT: [[TMP18:%.*]] = icmp ne <vscale x 4 x i32> [[WIDE_LOAD3_FR]], zeroinitializer
; CHECK-NEXT: [[UNCOUNTABLE_EXIT_MASK:%.*]] = tail call <vscale x 4 x i1> @llvm.mask.beforefirst.nxv4i1(<vscale x 4 x i1> [[TMP3]])
+; CHECK-NEXT: [[TMP19:%.*]] = xor <vscale x 4 x i1> [[UNCOUNTABLE_EXIT_MASK]], splat (i1 true)
+; CHECK-NEXT: [[TMP24:%.*]] = tail call i1 @llvm.vector.reduce.or.nxv4i1(<vscale x 4 x i1> [[TMP19]])
+; CHECK-NEXT: [[BROADCAST_SPLATINSERT4:%.*]] = insertelement <vscale x 4 x i1> poison, i1 [[TMP24]], i64 0
+; CHECK-NEXT: [[BROADCAST_SPLAT5:%.*]] = shufflevector <vscale x 4 x i1> [[BROADCAST_SPLATINSERT4]], <vscale x 4 x i1> poison, <vscale x 4 x i32> zeroinitializer
+; CHECK-NEXT: [[TMP28:%.*]] = or <vscale x 4 x i1> [[TMP18]], [[BROADCAST_SPLAT5]]
+; CHECK-NEXT: [[UNCOUNTABLE_EXIT_MASK4:%.*]] = tail call <vscale x 4 x i1> @llvm.mask.beforefirst.nxv4i1(<vscale x 4 x i1> [[TMP28]])
; CHECK-NEXT: [[TMP5:%.*]] = getelementptr [4 x i8], ptr @src, i64 [[INDEX]]
+; CHECK-NEXT: [[TMP26:%.*]] = getelementptr [4 x i8], ptr [[TMP5]], i64 [[TMP13]]
; CHECK-NEXT: [[WIDE_MASKED_LOAD:%.*]] = tail call <vscale x 4 x i32> @llvm.masked.load.nxv4i32.p0(ptr align 4 [[TMP5]], <vscale x 4 x i1> [[UNCOUNTABLE_EXIT_MASK]], <vscale x 4 x i32> poison)
+; CHECK-NEXT: [[WIDE_MASKED_LOAD5:%.*]] = tail call <vscale x 4 x i32> @llvm.masked.load.nxv4i32.p0(ptr align 4 [[TMP26]], <vscale x 4 x i1> [[UNCOUNTABLE_EXIT_MASK4]], <vscale x 4 x i32> poison)
; CHECK-NEXT: [[TMP6:%.*]] = add nsw <vscale x 4 x i32> [[WIDE_MASKED_LOAD]], splat (i32 42)
+; CHECK-NEXT: [[TMP27:%.*]] = add nsw <vscale x 4 x i32> [[WIDE_MASKED_LOAD5]], splat (i32 42)
; CHECK-NEXT: [[TMP7:%.*]] = getelementptr [4 x i8], ptr @dst, i64 [[INDEX]]
+; CHECK-NEXT: [[TMP14:%.*]] = getelementptr [4 x i8], ptr [[TMP7]], i64 [[TMP13]]
; CHECK-NEXT: tail call void @llvm.masked.store.nxv4i32.p0(<vscale x 4 x i32> [[TMP6]], ptr align 4 [[TMP7]], <vscale x 4 x i1> [[UNCOUNTABLE_EXIT_MASK]])
-; CHECK-NEXT: [[TMP8:%.*]] = tail call i1 @llvm.vector.reduce.or.nxv4i1(<vscale x 4 x i1> [[TMP3]])
+; CHECK-NEXT: tail call void @llvm.masked.store.nxv4i32.p0(<vscale x 4 x i32> [[TMP27]], ptr align 4 [[TMP14]], <vscale x 4 x i1> [[UNCOUNTABLE_EXIT_MASK4]])
+; CHECK-NEXT: [[TMP15:%.*]] = or <vscale x 4 x i32> [[WIDE_LOAD_FR]], [[WIDE_LOAD3_FR]]
+; CHECK-NEXT: [[TMP16:%.*]] = icmp ne <vscale x 4 x i32> [[TMP15]], zeroinitializer
+; CHECK-NEXT: [[TMP8:%.*]] = tail call i1 @llvm.vector.reduce.or.nxv4i1(<vscale x 4 x i1> [[TMP16]])
; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], [[TMP1]]
; CHECK-NEXT: [[TMP9:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
+; CHECK-NEXT: [[VEC_IND_NEXT]] = add <vscale x 4 x i64> [[VEC_IND]], [[REASS_ADD]]
; CHECK-NEXT: [[TMP10:%.*]] = or i1 [[TMP8]], [[TMP9]]
; CHECK-NEXT: br i1 [[TMP10]], label %[[MIDDLE_BLOCK:.*]], label %[[FOR_BODY]], !llvm.loop [[LOOP0:![0-9]+]]
; CHECK: [[MIDDLE_BLOCK]]:
+; CHECK-NEXT: [[TMP20:%.*]] = tail call i64 @llvm.vscale.i64()
+; CHECK-NEXT: [[TMP21:%.*]] = shl nuw i64 [[TMP20]], 2
+; CHECK-NEXT: [[TMP22:%.*]] = tail call i64 @llvm.experimental.cttz.elts.i64.nxv4i1(<vscale x 4 x i1> [[TMP18]], i1 false)
+; CHECK-NEXT: [[TMP23:%.*]] = add i64 [[TMP21]], [[TMP22]]
; CHECK-NEXT: [[TMP4:%.*]] = tail call i64 @llvm.experimental.cttz.elts.i64.nxv4i1(<vscale x 4 x i1> [[TMP3]], i1 false)
-; CHECK-NEXT: [[TMP11:%.*]] = add i64 [[INDEX]], [[TMP4]]
+; CHECK-NEXT: [[DOTNOT:%.*]] = icmp eq i64 [[TMP4]], [[TMP21]]
+; CHECK-NEXT: [[TMP25:%.*]] = select i1 [[DOTNOT]], i64 [[TMP23]], i64 [[TMP4]]
+; CHECK-NEXT: [[TMP29:%.*]] = extractelement <vscale x 4 x i64> [[VEC_IND]], i64 0
+; CHECK-NEXT: [[TMP11:%.*]] = add i64 [[TMP29]], [[TMP25]]
; CHECK-NEXT: [[TMP12:%.*]] = icmp eq i64 [[TMP11]], 10000
; CHECK-NEXT: br i1 [[TMP12]], label %[[EXIT:.*]], label %[[FOR_BODY_PREHEADER]]
; CHECK: [[FOR_BODY_PREHEADER]]:
>From aab33498897d21c9a9ae835c7dd7c61ddfee5dc8 Mon Sep 17 00:00:00 2001
From: Luke Lau <luke at igalia.com>
Date: Fri, 23 Oct 2026 20:43:23 +0800
Subject: [PATCH 05/15] [VPlan] Move extract-lane(0, canWideIV) combine before
unrolling
Some extract-lane combines can't be performed before unrolling, e.g. the one that combines it to Instruction::ExtractElement can end up extracting from a different part. But the extract-lane(0, canWideIV) can because at index 0 you're always extracting from the first part. Do this before unrolling, which improves codegen of unrolled code.
---
.../Transforms/Vectorize/VPlanTransforms.cpp | 12 +--
.../AArch64/early_exit_with_stores.ll | 85 ++-----------------
.../VPlan/early_exit_with_stores_vplan.ll | 40 ++++-----
.../interleave_uncountable_exits.ll | 16 +---
...ountable-and-uncountable-exits-combined.ll | 9 +-
5 files changed, 32 insertions(+), 130 deletions(-)
diff --git a/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp b/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp
index 6145fd8caef15e..66c4da2f5ebd5d 100644
--- a/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp
+++ b/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp
@@ -1360,6 +1360,12 @@ static VPValue *simplifyRecipe(VPlan &Plan, VPSingleDefRecipe *Def) {
A->getScalarType() == Def->getScalarType())
return A;
+ // Replace extract-lane(0, canonical-WIDEN-INDUCTION) with the region's
+ // scalar canonical IV.
+ VPWidenIntOrFpInductionRecipe *WidenIV;
+ if (match(Def, m_ExtractLane(m_ZeroInt(), m_CanonicalWidenIV(WidenIV))))
+ return WidenIV->getRegion()->getCanonicalIV();
+
// Some simplifications can only be applied after unrolling. Perform them
// below.
if (!Plan.isUnrolled())
@@ -1371,12 +1377,6 @@ static VPValue *simplifyRecipe(VPlan &Plan, VPSingleDefRecipe *Def) {
vputils::isSingleScalar(Def->getOperand(1)))
return Def->getOperand(1);
- // Replace extract-lane(0, canonical-WIDEN-INDUCTION) with the region's
- // scalar canonical IV.
- VPWidenIntOrFpInductionRecipe *WidenIV;
- if (match(Def, m_ExtractLane(m_ZeroInt(), m_CanonicalWidenIV(WidenIV))))
- return WidenIV->getRegion()->getCanonicalIV();
-
// Simplify unrolled VectorPointer without offset, or with zero offset, to
// just the pointer operand.
if (auto *VPR = dyn_cast<VPVectorPointerRecipe>(Def))
diff --git a/llvm/test/Transforms/LoopVectorize/AArch64/early_exit_with_stores.ll b/llvm/test/Transforms/LoopVectorize/AArch64/early_exit_with_stores.ll
index aa4c9abbf0208b..faf4e185e5edbb 100644
--- a/llvm/test/Transforms/LoopVectorize/AArch64/early_exit_with_stores.ll
+++ b/llvm/test/Transforms/LoopVectorize/AArch64/early_exit_with_stores.ll
@@ -59,16 +59,11 @@ define void @loop_contains_store_condition_load_has_single_user(ptr dereferencea
; CHECK-NEXT: br i1 [[MIN_ITERS_CHECK]], label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]]
; CHECK: [[VECTOR_PH]]:
; CHECK-NEXT: [[TMP2:%.*]] = shl nuw i64 [[TMP0]], 2
-; CHECK-NEXT: [[BROADCAST_SPLATINSERT:%.*]] = insertelement <vscale x 4 x i64> poison, i64 [[TMP2]], i64 0
-; CHECK-NEXT: [[BROADCAST_SPLAT:%.*]] = shufflevector <vscale x 4 x i64> [[BROADCAST_SPLATINSERT]], <vscale x 4 x i64> poison, <vscale x 4 x i32> zeroinitializer
; CHECK-NEXT: [[N_MOD_VF:%.*]] = urem i64 20, [[TMP1]]
; CHECK-NEXT: [[N_VEC:%.*]] = sub i64 20, [[N_MOD_VF]]
-; CHECK-NEXT: [[TMP17:%.*]] = call <vscale x 4 x i64> @llvm.stepvector.nxv4i64()
; CHECK-NEXT: br label %[[FOR_BODY1:.*]]
; CHECK: [[FOR_BODY1]]:
; CHECK-NEXT: [[IV1:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[FOR_BODY1]] ]
-; CHECK-NEXT: [[VEC_IND:%.*]] = phi <vscale x 4 x i64> [ [[TMP17]], %[[VECTOR_PH]] ], [ [[VEC_IND_NEXT:%.*]], %[[FOR_BODY1]] ]
-; CHECK-NEXT: [[STEP_ADD:%.*]] = add nuw <vscale x 4 x i64> [[VEC_IND]], [[BROADCAST_SPLAT]]
; CHECK-NEXT: [[TMP5:%.*]] = getelementptr inbounds nuw i16, ptr [[PRED]], i64 [[IV1]]
; CHECK-NEXT: [[TMP18:%.*]] = getelementptr inbounds nuw i16, ptr [[TMP5]], i64 [[TMP2]]
; CHECK-NEXT: [[WIDE_LOAD:%.*]] = load <vscale x 4 x i16>, ptr [[TMP5]], align 2
@@ -96,7 +91,6 @@ define void @loop_contains_store_condition_load_has_single_user(ptr dereferencea
; CHECK-NEXT: [[TMP10:%.*]] = call i1 @llvm.vector.reduce.or.nxv4i1(<vscale x 4 x i1> [[TMP16]])
; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[IV1]], [[TMP1]]
; CHECK-NEXT: [[TMP11:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
-; CHECK-NEXT: [[VEC_IND_NEXT]] = add nuw nsw <vscale x 4 x i64> [[STEP_ADD]], [[BROADCAST_SPLAT]]
; CHECK-NEXT: [[TMP12:%.*]] = or i1 [[TMP10]], [[TMP11]]
; CHECK-NEXT: br i1 [[TMP12]], label %[[MIDDLE_BLOCK:.*]], label %[[FOR_BODY1]], !llvm.loop [[LOOP0:![0-9]+]]
; CHECK: [[MIDDLE_BLOCK]]:
@@ -110,16 +104,7 @@ define void @loop_contains_store_condition_load_has_single_user(ptr dereferencea
; CHECK-NEXT: [[TMP27:%.*]] = add i64 [[TMP26]], [[TMP7]]
; CHECK-NEXT: [[TMP28:%.*]] = icmp ne i64 [[TMP7]], [[TMP21]]
; CHECK-NEXT: [[TMP29:%.*]] = select i1 [[TMP28]], i64 [[TMP27]], i64 [[TMP24]]
-; CHECK-NEXT: [[TMP40:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-NEXT: [[TMP41:%.*]] = mul nuw i64 [[TMP40]], 4
-; CHECK-NEXT: [[TMP33:%.*]] = mul i64 [[TMP41]], 0
-; CHECK-NEXT: [[TMP34:%.*]] = extractelement <vscale x 4 x i64> [[VEC_IND]], i64 0
-; CHECK-NEXT: [[TMP35:%.*]] = mul i64 [[TMP41]], 1
-; CHECK-NEXT: [[TMP36:%.*]] = sub i64 0, [[TMP35]]
-; CHECK-NEXT: [[TMP37:%.*]] = extractelement <vscale x 4 x i64> [[STEP_ADD]], i64 [[TMP36]]
-; CHECK-NEXT: [[TMP38:%.*]] = icmp uge i64 0, [[TMP35]]
-; CHECK-NEXT: [[TMP39:%.*]] = select i1 [[TMP38]], i64 [[TMP37]], i64 [[TMP34]]
-; CHECK-NEXT: [[TMP13:%.*]] = add i64 [[TMP39]], [[TMP29]]
+; CHECK-NEXT: [[TMP13:%.*]] = add i64 [[IV1]], [[TMP29]]
; CHECK-NEXT: [[TMP14:%.*]] = icmp eq i64 [[TMP13]], 20
; CHECK-NEXT: br i1 [[TMP14]], label %[[EXIT:.*]], label %[[SCALAR_PH]]
; CHECK: [[SCALAR_PH]]:
@@ -175,16 +160,11 @@ define void @loop_contains_store_fcmp_condition(ptr dereferenceable(40) noalias
; CHECK-NEXT: br i1 [[MIN_ITERS_CHECK]], label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]]
; CHECK: [[VECTOR_PH]]:
; CHECK-NEXT: [[TMP14:%.*]] = shl nuw i64 [[TMP0]], 2
-; CHECK-NEXT: [[BROADCAST_SPLATINSERT:%.*]] = insertelement <vscale x 4 x i64> poison, i64 [[TMP14]], i64 0
-; CHECK-NEXT: [[BROADCAST_SPLAT:%.*]] = shufflevector <vscale x 4 x i64> [[BROADCAST_SPLATINSERT]], <vscale x 4 x i64> poison, <vscale x 4 x i32> zeroinitializer
; CHECK-NEXT: [[N_MOD_VF:%.*]] = urem i64 20, [[TMP1]]
; CHECK-NEXT: [[N_VEC:%.*]] = sub i64 20, [[N_MOD_VF]]
-; CHECK-NEXT: [[TMP17:%.*]] = call <vscale x 4 x i64> @llvm.stepvector.nxv4i64()
; CHECK-NEXT: br label %[[FOR_BODY:.*]]
; CHECK: [[FOR_BODY]]:
; CHECK-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[FOR_BODY]] ]
-; CHECK-NEXT: [[VEC_IND:%.*]] = phi <vscale x 4 x i64> [ [[TMP17]], %[[VECTOR_PH]] ], [ [[VEC_IND_NEXT:%.*]], %[[FOR_BODY]] ]
-; CHECK-NEXT: [[STEP_ADD:%.*]] = add nuw <vscale x 4 x i64> [[VEC_IND]], [[BROADCAST_SPLAT]]
; CHECK-NEXT: [[TMP2:%.*]] = getelementptr inbounds nuw half, ptr [[PRED]], i64 [[INDEX]]
; CHECK-NEXT: [[TMP18:%.*]] = getelementptr inbounds nuw half, ptr [[TMP2]], i64 [[TMP14]]
; CHECK-NEXT: [[WIDE_LOAD:%.*]] = load <vscale x 4 x half>, ptr [[TMP2]], align 2
@@ -212,7 +192,6 @@ define void @loop_contains_store_fcmp_condition(ptr dereferenceable(40) noalias
; CHECK-NEXT: [[TMP8:%.*]] = call i1 @llvm.vector.reduce.or.nxv4i1(<vscale x 4 x i1> [[TMP16]])
; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], [[TMP1]]
; CHECK-NEXT: [[TMP9:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
-; CHECK-NEXT: [[VEC_IND_NEXT]] = add nuw nsw <vscale x 4 x i64> [[STEP_ADD]], [[BROADCAST_SPLAT]]
; CHECK-NEXT: [[TMP10:%.*]] = or i1 [[TMP8]], [[TMP9]]
; CHECK-NEXT: br i1 [[TMP10]], label %[[MIDDLE_BLOCK:.*]], label %[[FOR_BODY]], !llvm.loop [[LOOP4:![0-9]+]]
; CHECK: [[MIDDLE_BLOCK]]:
@@ -226,16 +205,7 @@ define void @loop_contains_store_fcmp_condition(ptr dereferenceable(40) noalias
; CHECK-NEXT: [[TMP27:%.*]] = add i64 [[TMP26]], [[TMP5]]
; CHECK-NEXT: [[TMP28:%.*]] = icmp ne i64 [[TMP5]], [[TMP21]]
; CHECK-NEXT: [[TMP29:%.*]] = select i1 [[TMP28]], i64 [[TMP27]], i64 [[TMP24]]
-; CHECK-NEXT: [[TMP40:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-NEXT: [[TMP41:%.*]] = mul nuw i64 [[TMP40]], 4
-; CHECK-NEXT: [[TMP42:%.*]] = mul i64 [[TMP41]], 0
-; CHECK-NEXT: [[TMP34:%.*]] = extractelement <vscale x 4 x i64> [[VEC_IND]], i64 0
-; CHECK-NEXT: [[TMP35:%.*]] = mul i64 [[TMP41]], 1
-; CHECK-NEXT: [[TMP36:%.*]] = sub i64 0, [[TMP35]]
-; CHECK-NEXT: [[TMP37:%.*]] = extractelement <vscale x 4 x i64> [[STEP_ADD]], i64 [[TMP36]]
-; CHECK-NEXT: [[TMP38:%.*]] = icmp uge i64 0, [[TMP35]]
-; CHECK-NEXT: [[TMP39:%.*]] = select i1 [[TMP38]], i64 [[TMP37]], i64 [[TMP34]]
-; CHECK-NEXT: [[TMP11:%.*]] = add i64 [[TMP39]], [[TMP29]]
+; CHECK-NEXT: [[TMP11:%.*]] = add i64 [[INDEX]], [[TMP29]]
; CHECK-NEXT: [[TMP12:%.*]] = icmp eq i64 [[TMP11]], 20
; CHECK-NEXT: br i1 [[TMP12]], label %[[EXIT:.*]], label %[[SCALAR_PH]]
; CHECK: [[SCALAR_PH]]:
@@ -448,16 +418,11 @@ define void @loop_contains_store_to_pointer_with_no_deref_info(ptr align 2 deref
; CHECK-NEXT: br i1 [[MIN_ITERS_CHECK]], label %[[SCALAR_PH1:.*]], label %[[VECTOR_PH:.*]]
; CHECK: [[VECTOR_PH]]:
; CHECK-NEXT: [[TMP2:%.*]] = shl nuw i64 [[TMP0]], 2
-; CHECK-NEXT: [[BROADCAST_SPLATINSERT:%.*]] = insertelement <vscale x 4 x i64> poison, i64 [[TMP2]], i64 0
-; CHECK-NEXT: [[BROADCAST_SPLAT:%.*]] = shufflevector <vscale x 4 x i64> [[BROADCAST_SPLATINSERT]], <vscale x 4 x i64> poison, <vscale x 4 x i32> zeroinitializer
; CHECK-NEXT: [[N_MOD_VF:%.*]] = urem i64 20, [[TMP1]]
; CHECK-NEXT: [[N_VEC:%.*]] = sub i64 20, [[N_MOD_VF]]
-; CHECK-NEXT: [[TMP16:%.*]] = call <vscale x 4 x i64> @llvm.stepvector.nxv4i64()
; CHECK-NEXT: br label %[[FOR_BODY:.*]]
; CHECK: [[FOR_BODY]]:
; CHECK-NEXT: [[IV:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[FOR_BODY]] ]
-; CHECK-NEXT: [[VEC_IND:%.*]] = phi <vscale x 4 x i64> [ [[TMP16]], %[[VECTOR_PH]] ], [ [[VEC_IND_NEXT:%.*]], %[[FOR_BODY]] ]
-; CHECK-NEXT: [[STEP_ADD:%.*]] = add nuw <vscale x 4 x i64> [[VEC_IND]], [[BROADCAST_SPLAT]]
; CHECK-NEXT: [[TMP5:%.*]] = getelementptr inbounds nuw i16, ptr [[PRED]], i64 [[IV]]
; CHECK-NEXT: [[TMP19:%.*]] = getelementptr inbounds nuw i16, ptr [[TMP5]], i64 [[TMP2]]
; CHECK-NEXT: [[WIDE_LOAD:%.*]] = load <vscale x 4 x i16>, ptr [[TMP5]], align 2
@@ -487,7 +452,6 @@ define void @loop_contains_store_to_pointer_with_no_deref_info(ptr align 2 deref
; CHECK-NEXT: [[TMP11:%.*]] = call i1 @llvm.vector.reduce.or.nxv4i1(<vscale x 4 x i1> [[TMP18]])
; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[IV]], [[TMP1]]
; CHECK-NEXT: [[TMP12:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
-; CHECK-NEXT: [[VEC_IND_NEXT]] = add nuw nsw <vscale x 4 x i64> [[STEP_ADD]], [[BROADCAST_SPLAT]]
; CHECK-NEXT: [[TMP13:%.*]] = or i1 [[TMP11]], [[TMP12]]
; CHECK-NEXT: br i1 [[TMP13]], label %[[MIDDLE_BLOCK:.*]], label %[[FOR_BODY]], !llvm.loop [[LOOP6:![0-9]+]]
; CHECK: [[MIDDLE_BLOCK]]:
@@ -501,16 +465,7 @@ define void @loop_contains_store_to_pointer_with_no_deref_info(ptr align 2 deref
; CHECK-NEXT: [[TMP29:%.*]] = add i64 [[TMP28]], [[TMP7]]
; CHECK-NEXT: [[TMP30:%.*]] = icmp ne i64 [[TMP7]], [[TMP23]]
; CHECK-NEXT: [[TMP31:%.*]] = select i1 [[TMP30]], i64 [[TMP29]], i64 [[TMP26]]
-; CHECK-NEXT: [[TMP42:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-NEXT: [[TMP43:%.*]] = mul nuw i64 [[TMP42]], 4
-; CHECK-NEXT: [[TMP35:%.*]] = mul i64 [[TMP43]], 0
-; CHECK-NEXT: [[TMP36:%.*]] = extractelement <vscale x 4 x i64> [[VEC_IND]], i64 0
-; CHECK-NEXT: [[TMP37:%.*]] = mul i64 [[TMP43]], 1
-; CHECK-NEXT: [[TMP38:%.*]] = sub i64 0, [[TMP37]]
-; CHECK-NEXT: [[TMP39:%.*]] = extractelement <vscale x 4 x i64> [[STEP_ADD]], i64 [[TMP38]]
-; CHECK-NEXT: [[TMP40:%.*]] = icmp uge i64 0, [[TMP37]]
-; CHECK-NEXT: [[TMP41:%.*]] = select i1 [[TMP40]], i64 [[TMP39]], i64 [[TMP36]]
-; CHECK-NEXT: [[TMP14:%.*]] = add i64 [[TMP41]], [[TMP31]]
+; CHECK-NEXT: [[TMP14:%.*]] = add i64 [[IV]], [[TMP31]]
; CHECK-NEXT: [[TMP15:%.*]] = icmp eq i64 [[TMP14]], 20
; CHECK-NEXT: br i1 [[TMP15]], label %[[EXIT:.*]], label %[[SCALAR_PH1]]
; CHECK: [[SCALAR_PH1]]:
@@ -613,16 +568,11 @@ define void @loop_contains_store_in_latch_block(ptr dereferenceable(40) noalias
; CHECK-NEXT: br i1 [[MIN_ITERS_CHECK]], label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]]
; CHECK: [[VECTOR_PH]]:
; CHECK-NEXT: [[TMP2:%.*]] = shl nuw i64 [[TMP0]], 2
-; CHECK-NEXT: [[BROADCAST_SPLATINSERT:%.*]] = insertelement <vscale x 4 x i64> poison, i64 [[TMP2]], i64 0
-; CHECK-NEXT: [[BROADCAST_SPLAT:%.*]] = shufflevector <vscale x 4 x i64> [[BROADCAST_SPLATINSERT]], <vscale x 4 x i64> poison, <vscale x 4 x i32> zeroinitializer
; CHECK-NEXT: [[N_MOD_VF:%.*]] = urem i64 20, [[TMP1]]
; CHECK-NEXT: [[N_VEC:%.*]] = sub i64 20, [[N_MOD_VF]]
-; CHECK-NEXT: [[TMP17:%.*]] = call <vscale x 4 x i64> @llvm.stepvector.nxv4i64()
; CHECK-NEXT: br label %[[FOR_BODY1:.*]]
; CHECK: [[FOR_BODY1]]:
; CHECK-NEXT: [[IV1:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[FOR_BODY1]] ]
-; CHECK-NEXT: [[VEC_IND:%.*]] = phi <vscale x 4 x i64> [ [[TMP17]], %[[VECTOR_PH]] ], [ [[VEC_IND_NEXT:%.*]], %[[FOR_BODY1]] ]
-; CHECK-NEXT: [[STEP_ADD:%.*]] = add nuw <vscale x 4 x i64> [[VEC_IND]], [[BROADCAST_SPLAT]]
; CHECK-NEXT: [[EE_ADDR1:%.*]] = getelementptr inbounds nuw i16, ptr [[PRED]], i64 [[IV1]]
; CHECK-NEXT: [[TMP18:%.*]] = getelementptr inbounds nuw i16, ptr [[EE_ADDR1]], i64 [[TMP2]]
; CHECK-NEXT: [[WIDE_LOAD:%.*]] = load <vscale x 4 x i16>, ptr [[EE_ADDR1]], align 2
@@ -650,7 +600,6 @@ define void @loop_contains_store_in_latch_block(ptr dereferenceable(40) noalias
; CHECK-NEXT: [[TMP10:%.*]] = call i1 @llvm.vector.reduce.or.nxv4i1(<vscale x 4 x i1> [[TMP16]])
; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[IV1]], [[TMP1]]
; CHECK-NEXT: [[TMP11:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
-; CHECK-NEXT: [[VEC_IND_NEXT]] = add nuw nsw <vscale x 4 x i64> [[STEP_ADD]], [[BROADCAST_SPLAT]]
; CHECK-NEXT: [[TMP12:%.*]] = or i1 [[TMP10]], [[TMP11]]
; CHECK-NEXT: br i1 [[TMP12]], label %[[MIDDLE_BLOCK:.*]], label %[[FOR_BODY1]], !llvm.loop [[LOOP8:![0-9]+]]
; CHECK: [[MIDDLE_BLOCK]]:
@@ -664,16 +613,7 @@ define void @loop_contains_store_in_latch_block(ptr dereferenceable(40) noalias
; CHECK-NEXT: [[TMP27:%.*]] = add i64 [[TMP26]], [[TMP6]]
; CHECK-NEXT: [[TMP28:%.*]] = icmp ne i64 [[TMP6]], [[TMP21]]
; CHECK-NEXT: [[TMP29:%.*]] = select i1 [[TMP28]], i64 [[TMP27]], i64 [[TMP24]]
-; CHECK-NEXT: [[TMP40:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-NEXT: [[TMP41:%.*]] = mul nuw i64 [[TMP40]], 4
-; CHECK-NEXT: [[TMP33:%.*]] = mul i64 [[TMP41]], 0
-; CHECK-NEXT: [[TMP34:%.*]] = extractelement <vscale x 4 x i64> [[VEC_IND]], i64 0
-; CHECK-NEXT: [[TMP35:%.*]] = mul i64 [[TMP41]], 1
-; CHECK-NEXT: [[TMP36:%.*]] = sub i64 0, [[TMP35]]
-; CHECK-NEXT: [[TMP37:%.*]] = extractelement <vscale x 4 x i64> [[STEP_ADD]], i64 [[TMP36]]
-; CHECK-NEXT: [[TMP38:%.*]] = icmp uge i64 0, [[TMP35]]
-; CHECK-NEXT: [[TMP39:%.*]] = select i1 [[TMP38]], i64 [[TMP37]], i64 [[TMP34]]
-; CHECK-NEXT: [[TMP13:%.*]] = add i64 [[TMP39]], [[TMP29]]
+; CHECK-NEXT: [[TMP13:%.*]] = add i64 [[IV1]], [[TMP29]]
; CHECK-NEXT: [[TMP14:%.*]] = icmp eq i64 [[TMP13]], 20
; CHECK-NEXT: br i1 [[TMP14]], label %[[EXIT:.*]], label %[[SCALAR_PH]]
; CHECK: [[SCALAR_PH]]:
@@ -1325,16 +1265,11 @@ define i32 @uncountable_exit_with_separate_exit_block(ptr dereferenceable(40) no
; CHECK-NEXT: br i1 [[MIN_ITERS_CHECK]], label %[[SCALAR_PH1:.*]], label %[[VECTOR_PH:.*]]
; CHECK: [[VECTOR_PH]]:
; CHECK-NEXT: [[TMP2:%.*]] = shl nuw i64 [[TMP0]], 2
-; CHECK-NEXT: [[BROADCAST_SPLATINSERT:%.*]] = insertelement <vscale x 4 x i64> poison, i64 [[TMP2]], i64 0
-; CHECK-NEXT: [[BROADCAST_SPLAT:%.*]] = shufflevector <vscale x 4 x i64> [[BROADCAST_SPLATINSERT]], <vscale x 4 x i64> poison, <vscale x 4 x i32> zeroinitializer
; CHECK-NEXT: [[N_MOD_VF:%.*]] = urem i64 20, [[TMP1]]
; CHECK-NEXT: [[N_VEC:%.*]] = sub i64 20, [[N_MOD_VF]]
-; CHECK-NEXT: [[TMP17:%.*]] = call <vscale x 4 x i64> @llvm.stepvector.nxv4i64()
; CHECK-NEXT: br label %[[FOR_BODY1:.*]]
; CHECK: [[FOR_BODY1]]:
; CHECK-NEXT: [[IV1:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[FOR_BODY1]] ]
-; CHECK-NEXT: [[VEC_IND:%.*]] = phi <vscale x 4 x i64> [ [[TMP17]], %[[VECTOR_PH]] ], [ [[VEC_IND_NEXT:%.*]], %[[FOR_BODY1]] ]
-; CHECK-NEXT: [[STEP_ADD:%.*]] = add nuw <vscale x 4 x i64> [[VEC_IND]], [[BROADCAST_SPLAT]]
; CHECK-NEXT: [[TMP5:%.*]] = getelementptr inbounds nuw i16, ptr [[PRED]], i64 [[IV1]]
; CHECK-NEXT: [[TMP18:%.*]] = getelementptr inbounds nuw i16, ptr [[TMP5]], i64 [[TMP2]]
; CHECK-NEXT: [[WIDE_LOAD:%.*]] = load <vscale x 4 x i16>, ptr [[TMP5]], align 2
@@ -1362,7 +1297,6 @@ define i32 @uncountable_exit_with_separate_exit_block(ptr dereferenceable(40) no
; CHECK-NEXT: [[TMP10:%.*]] = call i1 @llvm.vector.reduce.or.nxv4i1(<vscale x 4 x i1> [[TMP16]])
; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[IV1]], [[TMP1]]
; CHECK-NEXT: [[TMP11:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
-; CHECK-NEXT: [[VEC_IND_NEXT]] = add nuw nsw <vscale x 4 x i64> [[STEP_ADD]], [[BROADCAST_SPLAT]]
; CHECK-NEXT: [[TMP12:%.*]] = or i1 [[TMP10]], [[TMP11]]
; CHECK-NEXT: br i1 [[TMP12]], label %[[MIDDLE_BLOCK:.*]], label %[[FOR_BODY1]], !llvm.loop [[LOOP10:![0-9]+]]
; CHECK: [[MIDDLE_BLOCK]]:
@@ -1376,16 +1310,7 @@ define i32 @uncountable_exit_with_separate_exit_block(ptr dereferenceable(40) no
; CHECK-NEXT: [[TMP27:%.*]] = add i64 [[TMP26]], [[TMP7]]
; CHECK-NEXT: [[TMP28:%.*]] = icmp ne i64 [[TMP7]], [[TMP21]]
; CHECK-NEXT: [[TMP29:%.*]] = select i1 [[TMP28]], i64 [[TMP27]], i64 [[TMP24]]
-; CHECK-NEXT: [[TMP40:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-NEXT: [[TMP41:%.*]] = mul nuw i64 [[TMP40]], 4
-; CHECK-NEXT: [[TMP33:%.*]] = mul i64 [[TMP41]], 0
-; CHECK-NEXT: [[TMP34:%.*]] = extractelement <vscale x 4 x i64> [[VEC_IND]], i64 0
-; CHECK-NEXT: [[TMP35:%.*]] = mul i64 [[TMP41]], 1
-; CHECK-NEXT: [[TMP36:%.*]] = sub i64 0, [[TMP35]]
-; CHECK-NEXT: [[TMP37:%.*]] = extractelement <vscale x 4 x i64> [[STEP_ADD]], i64 [[TMP36]]
-; CHECK-NEXT: [[TMP38:%.*]] = icmp uge i64 0, [[TMP35]]
-; CHECK-NEXT: [[TMP39:%.*]] = select i1 [[TMP38]], i64 [[TMP37]], i64 [[TMP34]]
-; CHECK-NEXT: [[TMP13:%.*]] = add i64 [[TMP39]], [[TMP29]]
+; CHECK-NEXT: [[TMP13:%.*]] = add i64 [[IV1]], [[TMP29]]
; CHECK-NEXT: [[TMP14:%.*]] = icmp eq i64 [[TMP13]], 20
; CHECK-NEXT: br i1 [[TMP14]], label %[[EXIT_COUNTABLE:.*]], label %[[SCALAR_PH1]]
; CHECK: [[SCALAR_PH1]]:
diff --git a/llvm/test/Transforms/LoopVectorize/VPlan/early_exit_with_stores_vplan.ll b/llvm/test/Transforms/LoopVectorize/VPlan/early_exit_with_stores_vplan.ll
index ed3ead1afd1bcd..80abe86a071c10 100644
--- a/llvm/test/Transforms/LoopVectorize/VPlan/early_exit_with_stores_vplan.ll
+++ b/llvm/test/Transforms/LoopVectorize/VPlan/early_exit_with_stores_vplan.ll
@@ -21,7 +21,6 @@ define void @loop_contains_store_condition_load_has_single_user(ptr dereferencea
; CHECK-NEXT: vp<[[VP3:%[0-9]+]]> = CANONICAL-IV
; CHECK-EMPTY:
; CHECK-NEXT: vector.body:
-; CHECK-NEXT: ir<%iv> = WIDEN-INDUCTION nuw nsw ir<0>, ir<1>, vp<[[VP0]]>
; CHECK-NEXT: vp<[[VP4:%[0-9]+]]> = SCALAR-STEPS vp<[[VP3]]>, ir<1>, vp<[[VP0]]>
; CHECK-NEXT: CLONE ir<%ee.addr> = getelementptr inbounds nuw ir<%pred>, vp<[[VP4]]>
; CHECK-NEXT: vp<[[VP5:%[0-9]+]]> = vector-pointer inbounds nuw i16, ir<%ee.addr>, ir<1>
@@ -44,17 +43,16 @@ define void @loop_contains_store_condition_load_has_single_user(ptr dereferencea
; CHECK-EMPTY:
; CHECK-NEXT: middle.block:
; CHECK-NEXT: EMIT vp<[[VP12:%[0-9]+]]> = first-active-lane vp<[[VP6]]>
-; CHECK-NEXT: EMIT vp<[[VP13:%[0-9]+]]> = extract-lane ir<0>, ir<%iv>
-; CHECK-NEXT: EMIT vp<[[VP14:%[0-9]+]]> = add vp<[[VP13]]>, vp<[[VP12]]>
-; CHECK-NEXT: EMIT vp<[[VP15:%[0-9]+]]> = icmp eq vp<[[VP14]]>, ir<20>
-; CHECK-NEXT: EMIT branch-on-cond vp<[[VP15]]>
+; CHECK-NEXT: EMIT vp<[[VP13:%[0-9]+]]> = add vp<[[VP3]]>, vp<[[VP12]]>
+; CHECK-NEXT: EMIT vp<[[VP14:%[0-9]+]]> = icmp eq vp<[[VP13]]>, ir<20>
+; CHECK-NEXT: EMIT branch-on-cond vp<[[VP14]]>
; CHECK-NEXT: Successor(s): ir-bb<exit>, scalar.ph
; CHECK-EMPTY:
; CHECK-NEXT: ir-bb<exit>:
; CHECK-NEXT: No successors
; CHECK-EMPTY:
; CHECK-NEXT: scalar.ph:
-; CHECK-NEXT: EMIT-SCALAR vp<%bc.resume.val> = phi [ vp<[[VP14]]>, middle.block ], [ ir<0>, ir-bb<entry> ]
+; CHECK-NEXT: EMIT-SCALAR vp<%bc.resume.val> = phi [ vp<[[VP13]]>, middle.block ], [ ir<0>, ir-bb<entry> ]
; CHECK-NEXT: Successor(s): ir-bb<for.body>
; CHECK-EMPTY:
; CHECK-NEXT: ir-bb<for.body>:
@@ -110,7 +108,6 @@ define void @loop_contains_store_after_uncountable_exit(ptr dereferenceable(40)
; CHECK-NEXT: vp<[[VP3:%[0-9]+]]> = CANONICAL-IV
; CHECK-EMPTY:
; CHECK-NEXT: vector.body:
-; CHECK-NEXT: ir<%iv> = WIDEN-INDUCTION nuw nsw ir<0>, ir<1>, vp<[[VP0]]>
; CHECK-NEXT: vp<[[VP4:%[0-9]+]]> = SCALAR-STEPS vp<[[VP3]]>, ir<1>, vp<[[VP0]]>
; CHECK-NEXT: CLONE ir<%ee.addr> = getelementptr inbounds nuw ir<%pred>, vp<[[VP4]]>
; CHECK-NEXT: vp<[[VP5:%[0-9]+]]> = vector-pointer inbounds nuw i16, ir<%ee.addr>, ir<1>
@@ -133,17 +130,16 @@ define void @loop_contains_store_after_uncountable_exit(ptr dereferenceable(40)
; CHECK-EMPTY:
; CHECK-NEXT: middle.block:
; CHECK-NEXT: EMIT vp<[[VP12:%[0-9]+]]> = first-active-lane vp<[[VP6]]>
-; CHECK-NEXT: EMIT vp<[[VP13:%[0-9]+]]> = extract-lane ir<0>, ir<%iv>
-; CHECK-NEXT: EMIT vp<[[VP14:%[0-9]+]]> = add vp<[[VP13]]>, vp<[[VP12]]>
-; CHECK-NEXT: EMIT vp<[[VP15:%[0-9]+]]> = icmp eq vp<[[VP14]]>, ir<20>
-; CHECK-NEXT: EMIT branch-on-cond vp<[[VP15]]>
+; CHECK-NEXT: EMIT vp<[[VP13:%[0-9]+]]> = add vp<[[VP3]]>, vp<[[VP12]]>
+; CHECK-NEXT: EMIT vp<[[VP14:%[0-9]+]]> = icmp eq vp<[[VP13]]>, ir<20>
+; CHECK-NEXT: EMIT branch-on-cond vp<[[VP14]]>
; CHECK-NEXT: Successor(s): ir-bb<exit>, scalar.ph
; CHECK-EMPTY:
; CHECK-NEXT: ir-bb<exit>:
; CHECK-NEXT: No successors
; CHECK-EMPTY:
; CHECK-NEXT: scalar.ph:
-; CHECK-NEXT: EMIT-SCALAR vp<%bc.resume.val> = phi [ vp<[[VP14]]>, middle.block ], [ ir<0>, ir-bb<entry> ]
+; CHECK-NEXT: EMIT-SCALAR vp<%bc.resume.val> = phi [ vp<[[VP13]]>, middle.block ], [ ir<0>, ir-bb<entry> ]
; CHECK-NEXT: Successor(s): ir-bb<for.body>
; CHECK-EMPTY:
; CHECK-NEXT: ir-bb<for.body>:
@@ -195,7 +191,6 @@ define i16 @uncountable_exit_with_live_out(ptr dereferenceable(40) noalias %arra
; CHECK-NEXT: vp<[[VP3:%[0-9]+]]> = CANONICAL-IV
; CHECK-EMPTY:
; CHECK-NEXT: vector.body:
-; CHECK-NEXT: ir<%iv> = WIDEN-INDUCTION nuw nsw ir<0>, ir<1>, vp<[[VP0]]>
; CHECK-NEXT: vp<[[VP4:%[0-9]+]]> = SCALAR-STEPS vp<[[VP3]]>, ir<1>, vp<[[VP0]]>
; CHECK-NEXT: CLONE ir<%ee.addr> = getelementptr inbounds nuw ir<%pred>, vp<[[VP4]]>
; CHECK-NEXT: vp<[[VP5:%[0-9]+]]> = vector-pointer inbounds nuw i16, ir<%ee.addr>, ir<1>
@@ -220,10 +215,9 @@ define i16 @uncountable_exit_with_live_out(ptr dereferenceable(40) noalias %arra
; CHECK-NEXT: EMIT vp<[[VP12:%[0-9]+]]> = extract-last-part ir<%data>
; CHECK-NEXT: EMIT vp<[[VP13:%[0-9]+]]> = extract-last-lane vp<[[VP12]]>
; CHECK-NEXT: EMIT vp<[[VP14:%[0-9]+]]> = first-active-lane vp<[[VP6]]>
-; CHECK-NEXT: EMIT vp<[[VP15:%[0-9]+]]> = extract-lane ir<0>, ir<%iv>
-; CHECK-NEXT: EMIT vp<[[VP16:%[0-9]+]]> = add vp<[[VP15]]>, vp<[[VP14]]>
-; CHECK-NEXT: EMIT vp<[[VP17:%[0-9]+]]> = icmp eq vp<[[VP16]]>, ir<20>
-; CHECK-NEXT: EMIT branch-on-cond vp<[[VP17]]>
+; CHECK-NEXT: EMIT vp<[[VP15:%[0-9]+]]> = add vp<[[VP3]]>, vp<[[VP14]]>
+; CHECK-NEXT: EMIT vp<[[VP16:%[0-9]+]]> = icmp eq vp<[[VP15]]>, ir<20>
+; CHECK-NEXT: EMIT branch-on-cond vp<[[VP16]]>
; CHECK-NEXT: Successor(s): ir-bb<exit>, scalar.ph
; CHECK-EMPTY:
; CHECK-NEXT: ir-bb<exit>:
@@ -231,7 +225,7 @@ define i16 @uncountable_exit_with_live_out(ptr dereferenceable(40) noalias %arra
; CHECK-NEXT: No successors
; CHECK-EMPTY:
; CHECK-NEXT: scalar.ph:
-; CHECK-NEXT: EMIT-SCALAR vp<%bc.resume.val> = phi [ vp<[[VP16]]>, middle.block ], [ ir<0>, ir-bb<entry> ]
+; CHECK-NEXT: EMIT-SCALAR vp<%bc.resume.val> = phi [ vp<[[VP15]]>, middle.block ], [ ir<0>, ir-bb<entry> ]
; CHECK-NEXT: Successor(s): ir-bb<for.body>
; CHECK-EMPTY:
; CHECK-NEXT: ir-bb<for.body>:
@@ -287,7 +281,6 @@ define void @combined_exit_conditions(ptr align 4 dereferenceable(80) readonly %
; CHECK-NEXT: vp<[[VP3:%[0-9]+]]> = CANONICAL-IV
; CHECK-EMPTY:
; CHECK-NEXT: vector.body:
-; CHECK-NEXT: ir<%iv> = WIDEN-INDUCTION nuw nsw ir<0>, ir<1>, vp<[[VP0]]>
; CHECK-NEXT: vp<[[VP4:%[0-9]+]]> = SCALAR-STEPS vp<[[VP3]]>, ir<1>, vp<[[VP0]]>
; CHECK-NEXT: CLONE ir<%ee.ptr> = getelementptr inbounds nuw ir<%pred>, vp<[[VP4]]>
; CHECK-NEXT: vp<[[VP5:%[0-9]+]]> = vector-pointer inbounds nuw i32, ir<%ee.ptr>, ir<1>
@@ -313,17 +306,16 @@ define void @combined_exit_conditions(ptr align 4 dereferenceable(80) readonly %
; CHECK-EMPTY:
; CHECK-NEXT: middle.block:
; CHECK-NEXT: EMIT vp<[[VP14:%[0-9]+]]> = first-active-lane vp<[[VP7]]>
-; CHECK-NEXT: EMIT vp<[[VP15:%[0-9]+]]> = extract-lane ir<0>, ir<%iv>
-; CHECK-NEXT: EMIT vp<[[VP16:%[0-9]+]]> = add vp<[[VP15]]>, vp<[[VP14]]>
-; CHECK-NEXT: EMIT vp<[[VP17:%[0-9]+]]> = icmp eq vp<[[VP16]]>, ir<20>
-; CHECK-NEXT: EMIT branch-on-cond vp<[[VP17]]>
+; CHECK-NEXT: EMIT vp<[[VP15:%[0-9]+]]> = add vp<[[VP3]]>, vp<[[VP14]]>
+; CHECK-NEXT: EMIT vp<[[VP16:%[0-9]+]]> = icmp eq vp<[[VP15]]>, ir<20>
+; CHECK-NEXT: EMIT branch-on-cond vp<[[VP16]]>
; CHECK-NEXT: Successor(s): ir-bb<exit>, scalar.ph
; CHECK-EMPTY:
; CHECK-NEXT: ir-bb<exit>:
; CHECK-NEXT: No successors
; CHECK-EMPTY:
; CHECK-NEXT: scalar.ph:
-; CHECK-NEXT: EMIT-SCALAR vp<%bc.resume.val> = phi [ vp<[[VP16]]>, middle.block ], [ ir<0>, ir-bb<entry> ]
+; CHECK-NEXT: EMIT-SCALAR vp<%bc.resume.val> = phi [ vp<[[VP15]]>, middle.block ], [ ir<0>, ir-bb<entry> ]
; CHECK-NEXT: Successor(s): ir-bb<for.body>
; CHECK-EMPTY:
; CHECK-NEXT: ir-bb<for.body>:
diff --git a/llvm/test/Transforms/LoopVectorize/interleave_uncountable_exits.ll b/llvm/test/Transforms/LoopVectorize/interleave_uncountable_exits.ll
index f81ad5ee2f6d3d..e0a8ddbeda75c1 100644
--- a/llvm/test/Transforms/LoopVectorize/interleave_uncountable_exits.ll
+++ b/llvm/test/Transforms/LoopVectorize/interleave_uncountable_exits.ll
@@ -11,8 +11,6 @@ define void @loop_contains_store_condition_load_has_single_user(ptr dereferencea
; CHECK-NEXT: br label %[[VECTOR_BODY:.*]]
; CHECK: [[VECTOR_BODY]]:
; CHECK-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
-; CHECK-NEXT: [[VEC_IND:%.*]] = phi <4 x i64> [ <i64 0, i64 1, i64 2, i64 3>, %[[VECTOR_PH]] ], [ [[VEC_IND_NEXT:%.*]], %[[VECTOR_BODY]] ]
-; CHECK-NEXT: [[STEP_ADD:%.*]] = add nuw <4 x i64> [[VEC_IND]], splat (i64 4)
; CHECK-NEXT: [[TMP1:%.*]] = getelementptr inbounds nuw i16, ptr [[PRED]], i64 [[INDEX]]
; CHECK-NEXT: [[TMP13:%.*]] = getelementptr inbounds nuw i16, ptr [[TMP1]], i64 4
; CHECK-NEXT: [[WIDE_LOAD:%.*]] = load <4 x i16>, ptr [[TMP1]], align 2
@@ -40,7 +38,6 @@ define void @loop_contains_store_condition_load_has_single_user(ptr dereferencea
; CHECK-NEXT: [[TMP6:%.*]] = call i1 @llvm.vector.reduce.or.v4i1(<4 x i1> [[TMP12]])
; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 8
; CHECK-NEXT: [[TMP7:%.*]] = icmp eq i64 [[INDEX_NEXT]], 16
-; CHECK-NEXT: [[VEC_IND_NEXT]] = add nuw nsw <4 x i64> [[STEP_ADD]], splat (i64 4)
; CHECK-NEXT: [[TMP8:%.*]] = or i1 [[TMP6]], [[TMP7]]
; CHECK-NEXT: br i1 [[TMP8]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP0:![0-9]+]]
; CHECK: [[MIDDLE_BLOCK]]:
@@ -50,10 +47,7 @@ define void @loop_contains_store_condition_load_has_single_user(ptr dereferencea
; CHECK-NEXT: [[TMP19:%.*]] = add i64 0, [[TMP18]]
; CHECK-NEXT: [[TMP20:%.*]] = icmp ne i64 [[TMP18]], 4
; CHECK-NEXT: [[TMP3:%.*]] = select i1 [[TMP20]], i64 [[TMP19]], i64 [[TMP17]]
-; CHECK-NEXT: [[TMP26:%.*]] = extractelement <4 x i64> [[VEC_IND]], i64 0
-; CHECK-NEXT: [[TMP27:%.*]] = extractelement <4 x i64> [[STEP_ADD]], i64 -4
-; CHECK-NEXT: [[TMP25:%.*]] = select i1 false, i64 [[TMP27]], i64 [[TMP26]]
-; CHECK-NEXT: [[TMP9:%.*]] = add i64 [[TMP25]], [[TMP3]]
+; CHECK-NEXT: [[TMP9:%.*]] = add i64 [[INDEX]], [[TMP3]]
; CHECK-NEXT: [[TMP10:%.*]] = icmp eq i64 [[TMP9]], 20
; CHECK-NEXT: br i1 [[TMP10]], label %[[EXIT:.*]], label %[[SCALAR_PH:.*]]
; CHECK: [[SCALAR_PH]]:
@@ -83,9 +77,8 @@ define void @loop_contains_store_condition_load_has_single_user(ptr dereferencea
; VF1IC2-NEXT: br label %[[VECTOR_BODY:.*]]
; VF1IC2: [[VECTOR_BODY]]:
; VF1IC2-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[PRED_STORE_CONTINUE2:.*]] ]
-; VF1IC2-NEXT: [[TMP0:%.*]] = add i64 [[INDEX]], 0
; VF1IC2-NEXT: [[TMP1:%.*]] = add i64 [[INDEX]], 1
-; VF1IC2-NEXT: [[TMP2:%.*]] = getelementptr inbounds nuw i16, ptr [[PRED]], i64 [[TMP0]]
+; VF1IC2-NEXT: [[TMP2:%.*]] = getelementptr inbounds nuw i16, ptr [[PRED]], i64 [[INDEX]]
; VF1IC2-NEXT: [[TMP3:%.*]] = getelementptr inbounds nuw i16, ptr [[PRED]], i64 [[TMP1]]
; VF1IC2-NEXT: [[TMP4:%.*]] = load i16, ptr [[TMP2]], align 2
; VF1IC2-NEXT: [[TMP5:%.*]] = load i16, ptr [[TMP3]], align 2
@@ -98,7 +91,7 @@ define void @loop_contains_store_condition_load_has_single_user(ptr dereferencea
; VF1IC2-NEXT: [[TMP12:%.*]] = xor i1 [[TMP11]], true
; VF1IC2-NEXT: br i1 [[TMP10]], label %[[PRED_STORE_IF:.*]], label %[[PRED_STORE_CONTINUE:.*]]
; VF1IC2: [[PRED_STORE_IF]]:
-; VF1IC2-NEXT: [[TMP13:%.*]] = getelementptr inbounds nuw i16, ptr [[ARRAY]], i64 [[TMP0]]
+; VF1IC2-NEXT: [[TMP13:%.*]] = getelementptr inbounds nuw i16, ptr [[ARRAY]], i64 [[INDEX]]
; VF1IC2-NEXT: [[TMP14:%.*]] = load i16, ptr [[TMP13]], align 2
; VF1IC2-NEXT: [[TMP15:%.*]] = add nsw i16 [[TMP14]], 1
; VF1IC2-NEXT: store i16 [[TMP15]], ptr [[TMP13]], align 2
@@ -126,8 +119,7 @@ define void @loop_contains_store_condition_load_has_single_user(ptr dereferencea
; VF1IC2-NEXT: [[TMP27:%.*]] = add i64 0, [[TMP26]]
; VF1IC2-NEXT: [[TMP28:%.*]] = icmp ne i64 [[TMP26]], 1
; VF1IC2-NEXT: [[TMP29:%.*]] = select i1 [[TMP28]], i64 [[TMP27]], i64 [[TMP24]]
-; VF1IC2-NEXT: [[TMP30:%.*]] = select i1 false, i64 [[TMP1]], i64 [[TMP0]]
-; VF1IC2-NEXT: [[TMP31:%.*]] = add i64 [[TMP30]], [[TMP29]]
+; VF1IC2-NEXT: [[TMP31:%.*]] = add i64 [[INDEX]], [[TMP29]]
; VF1IC2-NEXT: [[TMP32:%.*]] = icmp eq i64 [[TMP31]], 20
; VF1IC2-NEXT: br i1 [[TMP32]], label %[[EXIT:.*]], label %[[SCALAR_PH:.*]]
; VF1IC2: [[SCALAR_PH]]:
diff --git a/llvm/test/Transforms/PhaseOrdering/AArch64/countable-and-uncountable-exits-combined.ll b/llvm/test/Transforms/PhaseOrdering/AArch64/countable-and-uncountable-exits-combined.ll
index af05a83af92318..aa7939535c9344 100644
--- a/llvm/test/Transforms/PhaseOrdering/AArch64/countable-and-uncountable-exits-combined.ll
+++ b/llvm/test/Transforms/PhaseOrdering/AArch64/countable-and-uncountable-exits-combined.ll
@@ -34,14 +34,9 @@ define void @foo() #0 {
; CHECK-NEXT: [[TMP13:%.*]] = shl nuw nsw i64 [[TMP0]], 2
; CHECK-NEXT: [[N_MOD_VF:%.*]] = urem i64 10000, [[TMP1]]
; CHECK-NEXT: [[N_VEC:%.*]] = sub nuw nsw i64 10000, [[N_MOD_VF]]
-; CHECK-NEXT: [[TMP30:%.*]] = tail call <vscale x 4 x i64> @llvm.stepvector.nxv4i64()
-; CHECK-NEXT: [[DOTSCALAR:%.*]] = shl nuw nsw i64 [[TMP0]], 3
-; CHECK-NEXT: [[TMP31:%.*]] = insertelement <vscale x 4 x i64> poison, i64 [[DOTSCALAR]], i64 0
-; CHECK-NEXT: [[REASS_ADD:%.*]] = shufflevector <vscale x 4 x i64> [[TMP31]], <vscale x 4 x i64> poison, <vscale x 4 x i32> zeroinitializer
; CHECK-NEXT: br label %[[FOR_BODY:.*]]
; CHECK: [[FOR_BODY]]:
; CHECK-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[FOR_BODY]] ]
-; CHECK-NEXT: [[VEC_IND:%.*]] = phi <vscale x 4 x i64> [ [[TMP30]], %[[VECTOR_PH]] ], [ [[VEC_IND_NEXT:%.*]], %[[FOR_BODY]] ]
; CHECK-NEXT: [[TMP2:%.*]] = getelementptr inbounds nuw [4 x i8], ptr @c, i64 [[INDEX]]
; CHECK-NEXT: [[TMP17:%.*]] = getelementptr inbounds nuw [4 x i8], ptr [[TMP2]], i64 [[TMP13]]
; CHECK-NEXT: [[WIDE_LOAD:%.*]] = load <vscale x 4 x i32>, ptr [[TMP2]], align 4
@@ -72,7 +67,6 @@ define void @foo() #0 {
; CHECK-NEXT: [[TMP8:%.*]] = tail call i1 @llvm.vector.reduce.or.nxv4i1(<vscale x 4 x i1> [[TMP16]])
; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], [[TMP1]]
; CHECK-NEXT: [[TMP9:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
-; CHECK-NEXT: [[VEC_IND_NEXT]] = add <vscale x 4 x i64> [[VEC_IND]], [[REASS_ADD]]
; CHECK-NEXT: [[TMP10:%.*]] = or i1 [[TMP8]], [[TMP9]]
; CHECK-NEXT: br i1 [[TMP10]], label %[[MIDDLE_BLOCK:.*]], label %[[FOR_BODY]], !llvm.loop [[LOOP0:![0-9]+]]
; CHECK: [[MIDDLE_BLOCK]]:
@@ -83,8 +77,7 @@ define void @foo() #0 {
; CHECK-NEXT: [[TMP4:%.*]] = tail call i64 @llvm.experimental.cttz.elts.i64.nxv4i1(<vscale x 4 x i1> [[TMP3]], i1 false)
; CHECK-NEXT: [[DOTNOT:%.*]] = icmp eq i64 [[TMP4]], [[TMP21]]
; CHECK-NEXT: [[TMP25:%.*]] = select i1 [[DOTNOT]], i64 [[TMP23]], i64 [[TMP4]]
-; CHECK-NEXT: [[TMP29:%.*]] = extractelement <vscale x 4 x i64> [[VEC_IND]], i64 0
-; CHECK-NEXT: [[TMP11:%.*]] = add i64 [[TMP29]], [[TMP25]]
+; CHECK-NEXT: [[TMP11:%.*]] = add i64 [[INDEX]], [[TMP25]]
; CHECK-NEXT: [[TMP12:%.*]] = icmp eq i64 [[TMP11]], 10000
; CHECK-NEXT: br i1 [[TMP12]], label %[[EXIT:.*]], label %[[FOR_BODY_PREHEADER]]
; CHECK: [[FOR_BODY_PREHEADER]]:
>From 3c56faa59c73133673a31c3e672a661ac84c23e2 Mon Sep 17 00:00:00 2001
From: Luke Lau <luke at igalia.com>
Date: Fri, 30 Oct 2026 14:08:45 +0800
Subject: [PATCH 06/15] Precommit tests
---
.../test/Transforms/LoopVectorize/blend-i1.ll | 69 +++++++++++++++++++
1 file changed, 69 insertions(+)
diff --git a/llvm/test/Transforms/LoopVectorize/blend-i1.ll b/llvm/test/Transforms/LoopVectorize/blend-i1.ll
index 25bfbb3167fc28..84c8bc2a24426c 100644
--- a/llvm/test/Transforms/LoopVectorize/blend-i1.ll
+++ b/llvm/test/Transforms/LoopVectorize/blend-i1.ll
@@ -55,3 +55,72 @@ latch:
exit:
ret void
}
+
+define void @blend_of_i1_false_nested(ptr noalias %a, ptr noalias %dst) {
+; CHECK-LABEL: define void @blend_of_i1_false_nested(
+; CHECK-SAME: ptr noalias [[A:%.*]], ptr noalias [[DST:%.*]]) {
+; CHECK-NEXT: [[ENTRY:.*:]]
+; CHECK-NEXT: br label %[[VECTOR_PH:.*]]
+; CHECK: [[VECTOR_PH]]:
+; CHECK-NEXT: br label %[[VECTOR_BODY:.*]]
+; CHECK: [[VECTOR_BODY]]:
+; CHECK-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-NEXT: [[TMP0:%.*]] = getelementptr i32, ptr [[A]], i64 [[INDEX]]
+; CHECK-NEXT: [[WIDE_LOAD:%.*]] = load <4 x i32>, ptr [[TMP0]], align 4
+; CHECK-NEXT: [[TMP1:%.*]] = icmp sgt <4 x i32> [[WIDE_LOAD]], zeroinitializer
+; CHECK-NEXT: [[TMP2:%.*]] = icmp slt <4 x i32> [[WIDE_LOAD]], splat (i32 100)
+; CHECK-NEXT: [[TMP3:%.*]] = select <4 x i1> [[TMP1]], <4 x i1> [[TMP2]], <4 x i1> zeroinitializer
+; CHECK-NEXT: [[TMP4:%.*]] = icmp sge <4 x i32> [[WIDE_LOAD]], splat (i32 50)
+; CHECK-NEXT: [[TMP5:%.*]] = icmp eq <4 x i32> [[WIDE_LOAD]], splat (i32 42)
+; CHECK-NEXT: [[TMP6:%.*]] = xor <4 x i1> [[TMP1]], splat (i1 true)
+; CHECK-NEXT: [[TMP7:%.*]] = xor <4 x i1> [[TMP2]], splat (i1 true)
+; CHECK-NEXT: [[TMP8:%.*]] = select <4 x i1> [[TMP1]], <4 x i1> [[TMP7]], <4 x i1> zeroinitializer
+; CHECK-NEXT: [[TMP9:%.*]] = select <4 x i1> [[TMP3]], <4 x i1> [[TMP4]], <4 x i1> zeroinitializer
+; CHECK-NEXT: [[TMP10:%.*]] = or <4 x i1> [[TMP9]], [[TMP8]]
+; CHECK-NEXT: [[TMP11:%.*]] = or <4 x i1> [[TMP10]], [[TMP6]]
+; CHECK-NEXT: [[PREDPHI:%.*]] = select <4 x i1> [[TMP11]], <4 x i1> zeroinitializer, <4 x i1> [[TMP5]]
+; CHECK-NEXT: [[TMP12:%.*]] = zext <4 x i1> [[PREDPHI]] to <4 x i8>
+; CHECK-NEXT: [[TMP13:%.*]] = getelementptr i8, ptr [[DST]], i64 [[INDEX]]
+; CHECK-NEXT: store <4 x i8> [[TMP12]], ptr [[TMP13]], align 1
+; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 4
+; CHECK-NEXT: [[TMP14:%.*]] = icmp eq i64 [[INDEX_NEXT]], 128
+; CHECK-NEXT: br i1 [[TMP14]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP4:![0-9]+]]
+; CHECK: [[MIDDLE_BLOCK]]:
+; CHECK-NEXT: br label %[[EXIT:.*]]
+; CHECK: [[EXIT]]:
+; CHECK-NEXT: ret void
+;
+entry:
+ br label %loop
+
+loop:
+ %iv = phi i64 [ 0, %entry ], [ %iv.next, %latch ]
+ %gep.a = getelementptr i32, ptr %a, i64 %iv
+ %x = load i32, ptr %gep.a
+ %c0 = icmp sgt i32 %x, 0
+ br i1 %c0, label %if.0, label %latch
+
+if.0:
+ %c1 = icmp slt i32 %x, 100
+ br i1 %c1, label %if.1, label %latch
+
+if.1:
+ %c2 = icmp slt i32 %x, 50
+ br i1 %c2, label %then, label %latch
+
+then:
+ %c3 = icmp eq i32 %x, 42
+ br label %latch
+
+latch:
+ %p = phi i1 [ %c3, %then ], [ false, %loop ], [ false, %if.0 ], [ false, %if.1 ]
+ %zext = zext i1 %p to i8
+ %gep.dst = getelementptr i8, ptr %dst, i64 %iv
+ store i8 %zext, ptr %gep.dst
+ %iv.next = add nuw nsw i64 %iv, 1
+ %ec = icmp eq i64 %iv.next, 128
+ br i1 %ec, label %exit, label %loop
+
+exit:
+ ret void
+}
>From 9f51888e507269cda084a0e3c4d2e9df525d7beb Mon Sep 17 00:00:00 2001
From: Luke Lau <luke at igalia.com>
Date: Tue, 27 Oct 2026 22:35:39 +0800
Subject: [PATCH 07/15] [VPlan] Simplify BLEND X/M, false, ..., false -> M && X
If an i1 blend has false for all but one incoming value, then we can simplify this to the one non-false value. Blends by construction never have any overlapping mask values, so we don't need to worry about the order as the masks can't "shortcut" each other.
---
llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp | 13 +++++++++++++
llvm/test/Transforms/LoopVectorize/blend-i1.ll | 11 +++--------
2 files changed, 16 insertions(+), 8 deletions(-)
diff --git a/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp b/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp
index 66c4da2f5ebd5d..dd4504effe7080 100644
--- a/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp
+++ b/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp
@@ -1993,6 +1993,19 @@ static void simplifyBlends(VPlan &Plan) {
if (Blend.isNormalized())
continue;
+ // BLEND X/M, false, ..., false -> M && X
+ auto NonFalse = make_filter_range(
+ seq<unsigned>(Blend.getNumIncomingValues()), [&Blend](unsigned I) {
+ return !match(Blend.getIncomingValue(I), m_False());
+ });
+ if (hasSingleElement(NonFalse)) {
+ unsigned I = *NonFalse.begin();
+ Blend.replaceAllUsesWith(VPBuilder(&Blend).createLogicalAnd(
+ Blend.getMask(I), Blend.getIncomingValue(I)));
+ Blend.eraseFromParent();
+ continue;
+ }
+
// Normalize the blend so its first incoming value is used as the initial
// value with the others blended into it.
diff --git a/llvm/test/Transforms/LoopVectorize/blend-i1.ll b/llvm/test/Transforms/LoopVectorize/blend-i1.ll
index 84c8bc2a24426c..e728d712911e0b 100644
--- a/llvm/test/Transforms/LoopVectorize/blend-i1.ll
+++ b/llvm/test/Transforms/LoopVectorize/blend-i1.ll
@@ -70,15 +70,10 @@ define void @blend_of_i1_false_nested(ptr noalias %a, ptr noalias %dst) {
; CHECK-NEXT: [[TMP1:%.*]] = icmp sgt <4 x i32> [[WIDE_LOAD]], zeroinitializer
; CHECK-NEXT: [[TMP2:%.*]] = icmp slt <4 x i32> [[WIDE_LOAD]], splat (i32 100)
; CHECK-NEXT: [[TMP3:%.*]] = select <4 x i1> [[TMP1]], <4 x i1> [[TMP2]], <4 x i1> zeroinitializer
-; CHECK-NEXT: [[TMP4:%.*]] = icmp sge <4 x i32> [[WIDE_LOAD]], splat (i32 50)
-; CHECK-NEXT: [[TMP5:%.*]] = icmp eq <4 x i32> [[WIDE_LOAD]], splat (i32 42)
-; CHECK-NEXT: [[TMP6:%.*]] = xor <4 x i1> [[TMP1]], splat (i1 true)
-; CHECK-NEXT: [[TMP7:%.*]] = xor <4 x i1> [[TMP2]], splat (i1 true)
-; CHECK-NEXT: [[TMP8:%.*]] = select <4 x i1> [[TMP1]], <4 x i1> [[TMP7]], <4 x i1> zeroinitializer
+; CHECK-NEXT: [[TMP4:%.*]] = icmp slt <4 x i32> [[WIDE_LOAD]], splat (i32 50)
; CHECK-NEXT: [[TMP9:%.*]] = select <4 x i1> [[TMP3]], <4 x i1> [[TMP4]], <4 x i1> zeroinitializer
-; CHECK-NEXT: [[TMP10:%.*]] = or <4 x i1> [[TMP9]], [[TMP8]]
-; CHECK-NEXT: [[TMP11:%.*]] = or <4 x i1> [[TMP10]], [[TMP6]]
-; CHECK-NEXT: [[PREDPHI:%.*]] = select <4 x i1> [[TMP11]], <4 x i1> zeroinitializer, <4 x i1> [[TMP5]]
+; CHECK-NEXT: [[TMP6:%.*]] = icmp eq <4 x i32> [[WIDE_LOAD]], splat (i32 42)
+; CHECK-NEXT: [[PREDPHI:%.*]] = select <4 x i1> [[TMP9]], <4 x i1> [[TMP6]], <4 x i1> zeroinitializer
; CHECK-NEXT: [[TMP12:%.*]] = zext <4 x i1> [[PREDPHI]] to <4 x i8>
; CHECK-NEXT: [[TMP13:%.*]] = getelementptr i8, ptr [[DST]], i64 [[INDEX]]
; CHECK-NEXT: store <4 x i8> [[TMP12]], ptr [[TMP13]], align 1
>From 3135b1137892c74c2bd98b5001a8f072333d589b Mon Sep 17 00:00:00 2001
From: Luke Lau <luke at igalia.com>
Date: Mon, 26 Oct 2026 17:51:51 +0800
Subject: [PATCH 08/15] [VPlan] Peek through nested phi incoming values in
computeBlendMasks
---
.../Transforms/Vectorize/VPlanPredicator.cpp | 91 +++-
.../LoopVectorize/VPlan/predicator.ll | 395 +++++++++++++-
.../Transforms/LoopVectorize/find-last.ll | 81 +++
.../Transforms/LoopVectorize/predicator.ll | 495 ++++++++++++++++++
.../LoopVectorize/select-cmp-blend-chain.ll | 16 +-
.../Transforms/LoopVectorize/uniform-blend.ll | 15 +-
6 files changed, 1061 insertions(+), 32 deletions(-)
diff --git a/llvm/lib/Transforms/Vectorize/VPlanPredicator.cpp b/llvm/lib/Transforms/Vectorize/VPlanPredicator.cpp
index a12527775aa1a3..e0dbc41063921e 100644
--- a/llvm/lib/Transforms/Vectorize/VPlanPredicator.cpp
+++ b/llvm/lib/Transforms/Vectorize/VPlanPredicator.cpp
@@ -83,15 +83,37 @@ class VPPredicator {
return VPBB->getFirstNonPhi();
}
+ /// Returns where to insert new blends in \p VPBB.
+ VPBasicBlock::iterator getBlendInsertPoint(VPBasicBlock *VPBB) {
+ VPBasicBlock::iterator It = getMaskInsertPoint(VPBB);
+ if (It == VPBB->end())
+ return It;
+ // Insert after any incoming edge masks if they were already inserted.
+ for (VPBlockBase *Pred : VPBB->predecessors())
+ if (VPValue *EdgeMask = getEdgeMask(cast<VPBasicBlock>(Pred), VPBB))
+ if (VPRecipeBase *EdgeR = EdgeMask->getDefiningRecipe())
+ if (EdgeR->getParent() == VPBB &&
+ !VPDT.properlyDominates(EdgeR, &*It)) {
+ It = std::next(EdgeR->getIterator());
+ if (It == VPBB->end())
+ return It;
+ }
+ return It;
+ }
+
using EdgeTy = std::pair<const VPBasicBlock *, const VPBasicBlock *>;
/// Compute the set of edges that are "furthest up" in the CFG for each
/// incoming value of \p Phi.
MapVector<EdgeTy, VPValue *> computeBlendEdges(VPPhi *Phi);
- /// Given a set of \p Edges that each can reach \p VPBB, return the OR of all
- /// edges, or an equivalent block in-mask.
- VPValue *createBlendMaskForEdges(ArrayRef<EdgeTy> Edges, VPBasicBlock *VPBB);
+ /// Given a set of \p Edges that each can reach \p VPBB, try to see if there
+ /// is a block in-mask that can be used for its blend mask. Only valid when
+ /// there's exactly one unique incoming value on the paths from Edges to VPBB.
+ VPValue *tryFindBlockInMask(ArrayRef<EdgeTy> Edges, VPBasicBlock *VPBB);
+
+ /// Return the OR of the edge masks of \p Edges.
+ VPValue *createBlendMaskForEdges(ArrayRef<EdgeTy> Edges);
public:
VPPredicator(VPlan &Plan)
@@ -281,6 +303,15 @@ VPPredicator::computeBlendEdges(VPPhi *Phi) {
for (auto [InVal, InVPBB] : Phi->incoming_values_and_blocks())
AddEdge(InVPBB, Phi->getParent(), InVal);
+ // Don't optimize any phis used to mask tail values of reductions during tail
+ // folding until handleFindLastReductions can match the new form.
+ VPlan *Plan = Phi->getParent()->getPlan();
+ if (Phi->getParent() == Plan->getVectorLoopRegion()->getExiting() &&
+ any_of(Phi->incoming_values(), IsaPred<VPReductionPHIRecipe>))
+ return Edges;
+
+ // Phis whose incoming values have been added to Edges.
+ SmallPtrSet<const VPUser *, 4> Expanded = {Phi};
SetVector<const VPBlockBase *> Worklist(from_range, Phi->incoming_blocks());
while (!Worklist.empty()) {
auto *VPBB = cast<VPBasicBlock>(Worklist.pop_back_val());
@@ -299,6 +330,24 @@ VPPredicator::computeBlendEdges(VPPhi *Phi) {
for (EdgeTy Edge : OutEdges)
Edges.erase(Edge);
+ // If the value is a phi postdominated by VPBB which shares an incoming
+ // value in Edges, then look through the inner incoming values instead of
+ // propagating the phi. This deduplicates a select.
+ if (auto *InnerPhi = dyn_cast<VPPhi>(Common))
+ if (all_of(InnerPhi->users(),
+ [&Expanded](VPUser *U) { return Expanded.contains(U); }) &&
+ VPPDT.dominates(VPBB, InnerPhi->getParent()) &&
+ any_of(InnerPhi->incoming_values(), [&Edges](VPValue *InV) {
+ return is_contained(make_second_range(Edges), InV);
+ })) {
+ Expanded.insert(InnerPhi);
+ for (auto [InV, InVPBB] : InnerPhi->incoming_values_and_blocks()) {
+ AddEdge(InVPBB, InnerPhi->getParent(), InV);
+ Worklist.insert(InVPBB);
+ }
+ continue;
+ }
+
// Iterate up through the post dominance frontier.
assert(VPPDF.find(VPBB) != VPPDF.end() &&
"VPBB must have a post-dominance frontier entry");
@@ -315,8 +364,8 @@ VPPredicator::computeBlendEdges(VPPhi *Phi) {
return Edges;
}
-VPValue *VPPredicator::createBlendMaskForEdges(ArrayRef<EdgeTy> Edges,
- VPBasicBlock *VPBB) {
+VPValue *VPPredicator::tryFindBlockInMask(ArrayRef<EdgeTy> Edges,
+ VPBasicBlock *VPBB) {
// If the nearest common postdominator to all of Edges destinations isn't VPBB
// then we can use its block in-mask. E.g:
//
@@ -329,16 +378,19 @@ VPValue *VPPredicator::createBlendMaskForEdges(ArrayRef<EdgeTy> Edges,
// VPBB
//
// If the edges are A->D and B->C, PostDom will be D. We can reuse Ds block
- // in-mask.
+ // in-mask. Only valid when there is only one unique incoming value flowing
+ // from the edges to VPBB.
const VPBasicBlock *PostDom = Edges[0].second;
for (auto [_, DstVPBB] : drop_begin(Edges))
PostDom =
cast<VPBasicBlock>(VPPDT.findNearestCommonDominator(PostDom, DstVPBB));
assert(VPPDT.dominates(VPBB, PostDom) && "VPBB doesn't postdominate edges");
- if (PostDom != VPBB)
- return getBlockInMask(PostDom);
+ if (PostDom == VPBB)
+ return nullptr;
+ return getBlockInMask(PostDom);
+}
- // Otherwise, compute the disjunction of edges.
+VPValue *VPPredicator::createBlendMaskForEdges(ArrayRef<EdgeTy> Edges) {
VPValue *Mask = nullptr;
for (auto [Src, ConstDst] : Edges) {
auto *Dst = const_cast<VPBasicBlock *>(ConstDst);
@@ -354,7 +406,7 @@ VPValue *VPPredicator::createBlendMaskForEdges(ArrayRef<EdgeTy> Edges,
}
void VPPredicator::convertPhisToBlends(VPBasicBlock *VPBB) {
- Builder.setInsertPoint(getMaskInsertPoint(VPBB));
+ Builder.setInsertPoint(getBlendInsertPoint(VPBB));
SmallVector<VPPhi *> Phis;
for (VPRecipeBase &R : VPBB->phis())
@@ -379,8 +431,19 @@ void VPPredicator::convertPhisToBlends(VPBasicBlock *VPBB) {
MapVector<VPValue *, SmallVector<EdgeTy>> InValEdgesMap;
for (auto [Edge, Val] : computeBlendEdges(PhiR))
InValEdgesMap[Val].push_back(Edge);
+ bool IncomingEdgesUnique =
+ all_of(PhiR->incoming_values(), [&](VPValue *InV) {
+ return is_contained(InValEdgesMap.keys(), InV);
+ });
+
auto InValEdges = InValEdgesMap.takeVector();
+ if (InValEdges.size() == 1) {
+ PhiR->replaceAllUsesWith(InValEdges[0].first);
+ PhiR->eraseFromParent();
+ continue;
+ }
+
// Sort the incoming value order to match PhiR as much as possible.
llvm::stable_sort(InValEdges, [&PhiR](auto &L, auto &R) {
auto InVs = PhiR->incoming_values();
@@ -391,7 +454,13 @@ void VPPredicator::convertPhisToBlends(VPBasicBlock *VPBB) {
SmallVector<VPValue *, 2> OperandsWithMask;
for (const auto &[InVPV, Edges] : InValEdges) {
OperandsWithMask.push_back(InVPV);
- OperandsWithMask.push_back(createBlendMaskForEdges(Edges, VPBB));
+ if (IncomingEdgesUnique) {
+ if (auto *Mask = tryFindBlockInMask(Edges, VPBB)) {
+ OperandsWithMask.push_back(Mask);
+ continue;
+ }
+ }
+ OperandsWithMask.push_back(createBlendMaskForEdges(Edges));
}
PHINode *IRPhi = cast_or_null<PHINode>(PhiR->getUnderlyingValue());
auto *Blend =
diff --git a/llvm/test/Transforms/LoopVectorize/VPlan/predicator.ll b/llvm/test/Transforms/LoopVectorize/VPlan/predicator.ll
index 46887ec305304c..e8629499763db8 100644
--- a/llvm/test/Transforms/LoopVectorize/VPlan/predicator.ll
+++ b/llvm/test/Transforms/LoopVectorize/VPlan/predicator.ll
@@ -665,8 +665,6 @@ define void @blend_chain_non_trivial(ptr noalias %a, ptr noalias %b) {
; CHECK-NEXT: Successor(s): merge.a
; CHECK-EMPTY:
; CHECK-NEXT: merge.a:
-; CHECK-NEXT: EMIT vp<[[VP5:%[0-9]+]]> = not ir<%c0>
-; CHECK-NEXT: BLEND ir<%blend.a> = ir<%v1>/ir<%c0> ir<%v1>/vp<[[VP5]]>
; CHECK-NEXT: EMIT ir<%d0> = icmp sgt ir<%iv>, ir<0>
; CHECK-NEXT: Successor(s): if.b
; CHECK-EMPTY:
@@ -675,16 +673,14 @@ define void @blend_chain_non_trivial(ptr noalias %a, ptr noalias %b) {
; CHECK-NEXT: Successor(s): if.b.inner
; CHECK-EMPTY:
; CHECK-NEXT: if.b.inner:
-; CHECK-NEXT: EMIT vp<[[VP6:%[0-9]+]]> = logical-and ir<%d0>, ir<%cb>
+; CHECK-NEXT: EMIT vp<[[VP5:%[0-9]+]]> = logical-and ir<%d0>, ir<%cb>
; CHECK-NEXT: Successor(s): merge.b.inner
; CHECK-EMPTY:
; CHECK-NEXT: merge.b.inner:
; CHECK-NEXT: Successor(s): merge.b
; CHECK-EMPTY:
; CHECK-NEXT: merge.b:
-; CHECK-NEXT: EMIT vp<[[VP7:%[0-9]+]]> = not ir<%d0>
-; CHECK-NEXT: BLEND ir<%blend.b> = ir<%v2>/ir<%d0> ir<%v2>/vp<[[VP7]]>
-; CHECK-NEXT: EMIT ir<%sum> = add ir<%blend.a>, ir<%blend.b>
+; CHECK-NEXT: EMIT ir<%sum> = add ir<%v1>, ir<%v2>
; CHECK-NEXT: EMIT store ir<%sum>, ir<%gep>
; CHECK-NEXT: Successor(s): loop.latch
; CHECK-EMPTY:
@@ -1042,3 +1038,390 @@ latch:
exit:
ret void
}
+
+define void @look_thru_phi(i1 %c0, i1 %c1, i32 %x, i32 %y, ptr %p) {
+; CHECK-LABEL: VPlan for loop in 'look_thru_phi'
+; CHECK-NEXT: <x1> vector loop: {
+; CHECK-NEXT: vp<[[VP3:%[0-9]+]]> = CANONICAL-IV
+; CHECK-EMPTY:
+; CHECK-NEXT: vector.body:
+; CHECK-NEXT: ir<%iv> = WIDEN-INDUCTION ir<0>, ir<1>, vp<[[VP0:%[0-9]+]]>
+; CHECK-NEXT: Successor(s): bb2
+; CHECK-EMPTY:
+; CHECK-NEXT: bb2:
+; CHECK-NEXT: EMIT vp<[[VP4:%[0-9]+]]> = not ir<%c0>
+; CHECK-NEXT: Successor(s): bb1
+; CHECK-EMPTY:
+; CHECK-NEXT: bb1:
+; CHECK-NEXT: Successor(s): bb4
+; CHECK-EMPTY:
+; CHECK-NEXT: bb4:
+; CHECK-NEXT: EMIT vp<[[VP5:%[0-9]+]]> = not ir<%c1>
+; CHECK-NEXT: EMIT vp<[[VP6:%[0-9]+]]> = logical-and ir<%c0>, vp<[[VP5]]>
+; CHECK-NEXT: EMIT vp<[[VP7:%[0-9]+]]> = or vp<[[VP4]]>, vp<[[VP6]]>
+; CHECK-NEXT: BLEND ir<%phi4> = ir<%y>/vp<[[VP4]]> ir<%x>/vp<[[VP6]]>
+; CHECK-NEXT: Successor(s): bb3
+; CHECK-EMPTY:
+; CHECK-NEXT: bb3:
+; CHECK-NEXT: EMIT vp<[[VP8:%[0-9]+]]> = logical-and ir<%c0>, ir<%c1>
+; CHECK-NEXT: Successor(s): bb5
+; CHECK-EMPTY:
+; CHECK-NEXT: bb5:
+; CHECK-NEXT: BLEND ir<%phi5> = ir<%x>/ir<%c0> ir<%y>/vp<[[VP4]]>
+; CHECK-NEXT: EMIT ir<%gep> = getelementptr ir<%p>, ir<%iv>
+; CHECK-NEXT: EMIT store ir<%phi5>, ir<%gep>
+; CHECK-NEXT: EMIT ir<%iv.next> = add ir<%iv>, ir<1>
+; CHECK-NEXT: EMIT ir<%ec> = icmp eq ir<%iv.next>, ir<128>
+; CHECK-NEXT: EMIT vp<%index.next> = add nuw vp<[[VP3]]>, vp<[[VP1:%[0-9]+]]>
+; CHECK-NEXT: EMIT branch-on-count vp<%index.next>, vp<[[VP2:%[0-9]+]]>
+; CHECK-NEXT: No successors
+; CHECK-NEXT: }
+; CHECK-NEXT: Successor(s): middle.block
+;
+; bb0
+; / \
+; bb1 \
+; / | bb2
+; \ | /
+; bb3 bb4
+; \/
+; bb5
+entry:
+ br label %bb0
+
+bb0:
+ %iv = phi i32 [0, %entry], [%iv.next, %bb5]
+ br i1 %c0, label %bb1, label %bb2
+
+bb1:
+ br i1 %c1, label %bb3, label %bb4
+
+bb2:
+ br label %bb4
+
+bb3:
+ br label %bb5
+
+bb4:
+ %phi4 = phi i32 [%x, %bb1], [%y, %bb2]
+ br label %bb5
+
+bb5:
+ %phi5 = phi i32 [ %x, %bb3 ], [ %phi4, %bb4 ]
+ %gep = getelementptr i32, ptr %p, i32 %iv
+ store i32 %phi5, ptr %gep
+ %iv.next = add i32 %iv, 1
+ %ec = icmp eq i32 %iv.next, 128
+ br i1 %ec, label %exit, label %bb0
+
+exit:
+ ret void
+}
+
+; Same as the test above, but shouldn't look thru the phi because it has
+; multiple uses.
+define void @look_thru_phi_multi_use(i1 %c0, i1 %c1, i32 %x, i32 %y, ptr %p) {
+; CHECK-LABEL: VPlan for loop in 'look_thru_phi_multi_use'
+; CHECK-NEXT: <x1> vector loop: {
+; CHECK-NEXT: vp<[[VP3:%[0-9]+]]> = CANONICAL-IV
+; CHECK-EMPTY:
+; CHECK-NEXT: vector.body:
+; CHECK-NEXT: ir<%iv> = WIDEN-INDUCTION ir<0>, ir<1>, vp<[[VP0:%[0-9]+]]>
+; CHECK-NEXT: EMIT ir<%gep> = getelementptr ir<%p>, ir<%iv>
+; CHECK-NEXT: Successor(s): bb2
+; CHECK-EMPTY:
+; CHECK-NEXT: bb2:
+; CHECK-NEXT: EMIT vp<[[VP4:%[0-9]+]]> = not ir<%c0>
+; CHECK-NEXT: Successor(s): bb1
+; CHECK-EMPTY:
+; CHECK-NEXT: bb1:
+; CHECK-NEXT: Successor(s): bb4
+; CHECK-EMPTY:
+; CHECK-NEXT: bb4:
+; CHECK-NEXT: EMIT vp<[[VP5:%[0-9]+]]> = not ir<%c1>
+; CHECK-NEXT: EMIT vp<[[VP6:%[0-9]+]]> = logical-and ir<%c0>, vp<[[VP5]]>
+; CHECK-NEXT: EMIT vp<[[VP7:%[0-9]+]]> = or vp<[[VP4]]>, vp<[[VP6]]>
+; CHECK-NEXT: BLEND ir<%phi4> = ir<%y>/vp<[[VP4]]> ir<%x>/vp<[[VP6]]>
+; CHECK-NEXT: EMIT store ir<%phi4>, ir<%gep>, vp<[[VP7]]>
+; CHECK-NEXT: Successor(s): bb3
+; CHECK-EMPTY:
+; CHECK-NEXT: bb3:
+; CHECK-NEXT: EMIT vp<[[VP8:%[0-9]+]]> = logical-and ir<%c0>, ir<%c1>
+; CHECK-NEXT: Successor(s): bb5
+; CHECK-EMPTY:
+; CHECK-NEXT: bb5:
+; CHECK-NEXT: BLEND ir<%phi5> = ir<%phi4>/vp<[[VP7]]> ir<%x>/vp<[[VP8]]>
+; CHECK-NEXT: EMIT store ir<%phi5>, ir<%gep>
+; CHECK-NEXT: EMIT ir<%iv.next> = add ir<%iv>, ir<1>
+; CHECK-NEXT: EMIT ir<%ec> = icmp eq ir<%iv.next>, ir<128>
+; CHECK-NEXT: EMIT vp<%index.next> = add nuw vp<[[VP3]]>, vp<[[VP1:%[0-9]+]]>
+; CHECK-NEXT: EMIT branch-on-count vp<%index.next>, vp<[[VP2:%[0-9]+]]>
+; CHECK-NEXT: No successors
+; CHECK-NEXT: }
+; CHECK-NEXT: Successor(s): middle.block
+;
+; bb0
+; / \
+; bb1 \
+; / | bb2
+; \ | /
+; bb3 bb4
+; \/
+; bb5
+entry:
+ br label %bb0
+
+bb0:
+ %iv = phi i32 [0, %entry], [%iv.next, %bb5]
+ %gep = getelementptr i32, ptr %p, i32 %iv
+ br i1 %c0, label %bb1, label %bb2
+
+bb1:
+ br i1 %c1, label %bb3, label %bb4
+
+bb2:
+ br label %bb4
+
+bb3:
+ br label %bb5
+
+bb4:
+ %phi4 = phi i32 [%x, %bb1], [%y, %bb2]
+ store i32 %phi4, ptr %gep
+ br label %bb5
+
+bb5:
+ %phi5 = phi i32 [ %x, %bb3 ], [ %phi4, %bb4 ]
+ store i32 %phi5, ptr %gep
+ %iv.next = add i32 %iv, 1
+ %ec = icmp eq i32 %iv.next, 128
+ br i1 %ec, label %exit, label %bb0
+
+exit:
+ ret void
+}
+
+; Don't look through %phi3 as it doesn't share any incoming values with %phi4.
+define void @look_thru_phi_no_shared_values(i1 %c0, i1 %c1, i32 %x, i32 %y, i32 %z, ptr %p) {
+; CHECK-LABEL: VPlan for loop in 'look_thru_phi_no_shared_values'
+; CHECK-NEXT: <x1> vector loop: {
+; CHECK-NEXT: vp<[[VP3:%[0-9]+]]> = CANONICAL-IV
+; CHECK-EMPTY:
+; CHECK-NEXT: vector.body:
+; CHECK-NEXT: ir<%iv> = WIDEN-INDUCTION ir<0>, ir<1>, vp<[[VP0:%[0-9]+]]>
+; CHECK-NEXT: Successor(s): bb1
+; CHECK-EMPTY:
+; CHECK-NEXT: bb1:
+; CHECK-NEXT: EMIT vp<[[VP4:%[0-9]+]]> = not ir<%c0>
+; CHECK-NEXT: Successor(s): bb2
+; CHECK-EMPTY:
+; CHECK-NEXT: bb2:
+; CHECK-NEXT: EMIT vp<[[VP5:%[0-9]+]]> = logical-and vp<[[VP4]]>, ir<%c1>
+; CHECK-NEXT: Successor(s): bb3
+; CHECK-EMPTY:
+; CHECK-NEXT: bb3:
+; CHECK-NEXT: EMIT vp<[[VP6:%[0-9]+]]> = not ir<%c1>
+; CHECK-NEXT: EMIT vp<[[VP7:%[0-9]+]]> = logical-and vp<[[VP4]]>, vp<[[VP6]]>
+; CHECK-NEXT: BLEND ir<%phi3> = ir<%z>/vp<[[VP5]]> ir<%y>/vp<[[VP7]]>
+; CHECK-NEXT: Successor(s): bb4
+; CHECK-EMPTY:
+; CHECK-NEXT: bb4:
+; CHECK-NEXT: BLEND ir<%phi4> = ir<%phi3>/vp<[[VP4]]> ir<%x>/ir<%c0>
+; CHECK-NEXT: EMIT ir<%gep> = getelementptr ir<%p>, ir<%iv>
+; CHECK-NEXT: EMIT store ir<%phi4>, ir<%gep>
+; CHECK-NEXT: EMIT ir<%iv.next> = add ir<%iv>, ir<1>
+; CHECK-NEXT: EMIT ir<%ec> = icmp eq ir<%iv.next>, ir<128>
+; CHECK-NEXT: EMIT vp<%index.next> = add nuw vp<[[VP3]]>, vp<[[VP1:%[0-9]+]]>
+; CHECK-NEXT: EMIT branch-on-count vp<%index.next>, vp<[[VP2:%[0-9]+]]>
+; CHECK-NEXT: No successors
+; CHECK-NEXT: }
+; CHECK-NEXT: Successor(s): middle.block
+;
+; bb0
+; | \
+; | bb1
+; | | \
+; | | bb2
+; | | /
+; | bb3
+; | /
+; bb4
+entry:
+ br label %bb0
+
+bb0:
+ %iv = phi i32 [0, %entry], [%iv.next, %bb4]
+ br i1 %c0, label %bb4, label %bb1
+
+bb1:
+ br i1 %c1, label %bb2, label %bb3
+
+bb2:
+ br label %bb3
+
+bb3:
+ %phi3 = phi i32 [%y, %bb1], [%z, %bb2]
+ br label %bb4
+
+bb4:
+ %phi4 = phi i32 [%x, %bb0], [%phi3, %bb3]
+ %gep = getelementptr i32, ptr %p, i32 %iv
+ store i32 %phi4, ptr %gep
+ %iv.next = add i32 %iv, 1
+ %ec = icmp eq i32 %iv.next, 128
+ br i1 %ec, label %exit, label %bb0
+
+exit:
+ ret void
+}
+
+; bb4 will create the edge mask bb1->bb3 before bb3. Make sure bb4
+; inserts its blend after the already created edge mask.
+define i32 @look_thru_phi_blend_insert_pos(i1 %c0, i1 %c1) {
+; CHECK-LABEL: VPlan for loop in 'look_thru_phi_blend_insert_pos'
+; CHECK-NEXT: <x1> vector loop: {
+; CHECK-NEXT: vp<[[VP3:%[0-9]+]]> = CANONICAL-IV
+; CHECK-EMPTY:
+; CHECK-NEXT: vector.body:
+; CHECK-NEXT: ir<%iv> = WIDEN-INDUCTION ir<0>, ir<1>, vp<[[VP0:%[0-9]+]]>
+; CHECK-NEXT: Successor(s): bb1
+; CHECK-EMPTY:
+; CHECK-NEXT: bb1:
+; CHECK-NEXT: EMIT vp<[[VP4:%[0-9]+]]> = not ir<%c0>
+; CHECK-NEXT: Successor(s): bb2
+; CHECK-EMPTY:
+; CHECK-NEXT: bb2:
+; CHECK-NEXT: EMIT vp<[[VP5:%[0-9]+]]> = not ir<%c1>
+; CHECK-NEXT: EMIT vp<[[VP6:%[0-9]+]]> = logical-and vp<[[VP4]]>, vp<[[VP5]]>
+; CHECK-NEXT: Successor(s): bb3
+; CHECK-EMPTY:
+; CHECK-NEXT: bb3:
+; CHECK-NEXT: EMIT vp<[[VP7:%[0-9]+]]> = logical-and vp<[[VP4]]>, ir<%c1>
+; CHECK-NEXT: BLEND ir<%phi3> = ir<2>/vp<[[VP6]]> ir<1>/vp<[[VP7]]>
+; CHECK-NEXT: Successor(s): bb4
+; CHECK-EMPTY:
+; CHECK-NEXT: bb4:
+; CHECK-NEXT: EMIT vp<[[VP8:%[0-9]+]]> = or ir<%c0>, vp<[[VP7]]>
+; CHECK-NEXT: BLEND ir<%phi4> = ir<1>/vp<[[VP8]]> ir<2>/vp<[[VP6]]>
+; CHECK-NEXT: EMIT ir<%iv.next> = add ir<%iv>, ir<1>
+; CHECK-NEXT: EMIT ir<%ec> = icmp eq ir<%iv.next>, ir<128>
+; CHECK-NEXT: EMIT vp<%index.next> = add nuw vp<[[VP3]]>, vp<[[VP1:%[0-9]+]]>
+; CHECK-NEXT: EMIT branch-on-count vp<%index.next>, vp<[[VP2:%[0-9]+]]>
+; CHECK-NEXT: No successors
+; CHECK-NEXT: }
+; CHECK-NEXT: Successor(s): middle.block
+;
+; bb0
+; / \
+; bb1 \
+; / \ \
+; bb2 | /
+; \ / /
+; bb3 /
+; \ /
+; bb4
+entry:
+ br label %bb0
+
+bb0:
+ %iv = phi i64 [ 0, %entry ], [ %iv.next, %bb4 ]
+ br i1 %c0, label %bb4, label %bb1
+
+bb1:
+ br i1 %c1, label %bb3, label %bb2
+
+bb2:
+ br label %bb3
+
+bb3:
+ %phi3 = phi i32 [ 1, %bb1 ], [ 2, %bb2 ]
+ br label %bb4
+
+bb4:
+ %phi4 = phi i32 [ 1, %bb0 ], [ %phi3, %bb3 ]
+ %iv.next = add i64 %iv, 1
+ %ec = icmp eq i64 %iv.next, 128
+ br i1 %ec, label %exit, label %bb0
+
+exit:
+ ret i32 %phi4
+}
+
+define void @diamond_phi2_look_thru_phi(ptr %a, i1 %c1, i1 %c2) {
+; CHECK-LABEL: VPlan for loop in 'diamond_phi2_look_thru_phi'
+; CHECK-NEXT: <x1> vector loop: {
+; CHECK-NEXT: vp<[[VP3:%[0-9]+]]> = CANONICAL-IV
+; CHECK-EMPTY:
+; CHECK-NEXT: vector.body:
+; CHECK-NEXT: ir<%iv> = WIDEN-INDUCTION nuw nsw ir<0>, ir<1>, vp<[[VP0:%[0-9]+]]>
+; CHECK-NEXT: EMIT ir<%c0> = icmp sle ir<%iv>, ir<0>
+; CHECK-NEXT: Successor(s): bb2
+; CHECK-EMPTY:
+; CHECK-NEXT: bb2:
+; CHECK-NEXT: EMIT vp<[[VP4:%[0-9]+]]> = not ir<%c0>
+; CHECK-NEXT: Successor(s): bb1
+; CHECK-EMPTY:
+; CHECK-NEXT: bb1:
+; CHECK-NEXT: Successor(s): bb4
+; CHECK-EMPTY:
+; CHECK-NEXT: bb4:
+; CHECK-NEXT: EMIT vp<[[VP5:%[0-9]+]]> = logical-and vp<[[VP4]]>, ir<%c2>
+; CHECK-NEXT: EMIT vp<[[VP6:%[0-9]+]]> = logical-and ir<%c0>, ir<%c1>
+; CHECK-NEXT: EMIT vp<[[VP7:%[0-9]+]]> = or vp<[[VP5]]>, vp<[[VP6]]>
+; CHECK-NEXT: BLEND ir<%phi4> = ir<1>/vp<[[VP5]]> ir<0>/vp<[[VP6]]>
+; CHECK-NEXT: Successor(s): bb5
+; CHECK-EMPTY:
+; CHECK-NEXT: bb5:
+; CHECK-NEXT: EMIT vp<[[VP8:%[0-9]+]]> = not ir<%c1>
+; CHECK-NEXT: EMIT vp<[[VP9:%[0-9]+]]> = logical-and ir<%c0>, vp<[[VP8]]>
+; CHECK-NEXT: EMIT vp<[[VP10:%[0-9]+]]> = not ir<%c2>
+; CHECK-NEXT: EMIT vp<[[VP11:%[0-9]+]]> = logical-and vp<[[VP4]]>, vp<[[VP10]]>
+; CHECK-NEXT: BLEND ir<%phi5> = ir<%phi4>/vp<[[VP7]]> ir<3>/vp<[[VP11]]> ir<2>/vp<[[VP9]]>
+; CHECK-NEXT: EMIT ir<%gep> = getelementptr ir<%a>, ir<%iv>
+; CHECK-NEXT: EMIT store ir<%phi5>, ir<%gep>
+; CHECK-NEXT: EMIT ir<%iv.next> = add nuw nsw ir<%iv>, ir<1>
+; CHECK-NEXT: EMIT ir<%ec> = icmp eq ir<%iv.next>, ir<128>
+; CHECK-NEXT: EMIT vp<%index.next> = add nuw vp<[[VP3]]>, vp<[[VP1:%[0-9]+]]>
+; CHECK-NEXT: EMIT branch-on-count vp<%index.next>, vp<[[VP2:%[0-9]+]]>
+; CHECK-NEXT: No successors
+; CHECK-NEXT: }
+; CHECK-NEXT: Successor(s): middle.block
+;
+; bb0
+; / \
+; bb1 bb2
+; | \ / |
+; | bb4 |
+; \ | /
+; bb5
+entry:
+ br label %bb0
+
+bb0:
+ %iv = phi i64 [0, %entry], [%iv.next, %bb5]
+ %c0 = icmp sle i64 %iv, 0
+ br i1 %c0, label %bb1, label %bb2
+
+bb1:
+ %add1 = add i64 %iv, 1
+ br i1 %c1, label %bb4, label %bb5
+
+bb2:
+ %add2 = add i64 %iv, 2
+ br i1 %c2, label %bb4, label %bb5
+
+bb4:
+ %phi4 = phi i64 [0, %bb1], [1, %bb2]
+ br label %bb5
+
+bb5:
+ %phi5 = phi i64 [2, %bb1], [3, %bb2], [%phi4, %bb4]
+ %gep = getelementptr i64, ptr %a, i64 %iv
+ store i64 %phi5, ptr %gep
+ %iv.next = add nsw nuw i64 %iv, 1
+ %ec = icmp eq i64 %iv.next, 128
+ br i1 %ec, label %exit, label %bb0
+
+exit:
+ ret void
+}
diff --git a/llvm/test/Transforms/LoopVectorize/find-last.ll b/llvm/test/Transforms/LoopVectorize/find-last.ll
index 9948f34e4571b6..35cf71099b101c 100644
--- a/llvm/test/Transforms/LoopVectorize/find-last.ll
+++ b/llvm/test/Transforms/LoopVectorize/find-last.ll
@@ -445,3 +445,84 @@ loop.latch:
exit:
ret i32 %rdx.next
}
+
+define i32 @find_last_phi_of_two_updates(ptr noalias %a, ptr noalias %b, i64 %n) {
+; CHECK-LABEL: define i32 @find_last_phi_of_two_updates(
+; CHECK-SAME: ptr noalias [[A:%.*]], ptr noalias [[B:%.*]], i64 [[N:%.*]]) {
+; CHECK-NEXT: [[ENTRY:.*]]:
+; CHECK-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[N]], 4
+; CHECK-NEXT: br i1 [[MIN_ITERS_CHECK]], label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]]
+; CHECK: [[VECTOR_PH]]:
+; CHECK-NEXT: [[TMP0:%.*]] = and i64 [[N]], 3
+; CHECK-NEXT: [[N_VEC:%.*]] = sub i64 [[N]], [[TMP0]]
+; CHECK-NEXT: br label %[[VECTOR_BODY:.*]]
+; CHECK: [[VECTOR_BODY]]:
+; CHECK-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-NEXT: [[VEC_PHI:%.*]] = phi <4 x i32> [ splat (i32 -1), %[[VECTOR_PH]] ], [ [[TMP14:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-NEXT: [[TMP1:%.*]] = phi <4 x i1> [ zeroinitializer, %[[VECTOR_PH]] ], [ [[TMP13:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-NEXT: [[TMP2:%.*]] = getelementptr i32, ptr [[A]], i64 [[INDEX]]
+; CHECK-NEXT: [[WIDE_LOAD:%.*]] = load <4 x i32>, ptr [[TMP2]], align 4
+; CHECK-NEXT: [[TMP3:%.*]] = getelementptr i32, ptr [[B]], i64 [[INDEX]]
+; CHECK-NEXT: [[WIDE_LOAD1:%.*]] = load <4 x i32>, ptr [[TMP3]], align 4
+; CHECK-NEXT: [[TMP4:%.*]] = icmp sgt <4 x i32> [[WIDE_LOAD]], splat (i32 5)
+; CHECK-NEXT: [[TMP5:%.*]] = xor <4 x i1> [[TMP4]], splat (i1 true)
+; CHECK-NEXT: [[TMP6:%.*]] = icmp eq <4 x i32> [[WIDE_LOAD]], splat (i32 2)
+; CHECK-NEXT: [[TMP7:%.*]] = icmp eq <4 x i32> [[WIDE_LOAD1]], zeroinitializer
+; CHECK-NEXT: [[TMP8:%.*]] = select <4 x i1> [[TMP5]], <4 x i1> [[TMP6]], <4 x i1> zeroinitializer
+; CHECK-NEXT: [[TMP9:%.*]] = select <4 x i1> [[TMP4]], <4 x i1> [[TMP7]], <4 x i1> zeroinitializer
+; CHECK-NEXT: [[TMP10:%.*]] = or <4 x i1> [[TMP8]], [[TMP9]]
+; CHECK-NEXT: [[PREDPHI:%.*]] = select <4 x i1> [[TMP9]], <4 x i32> [[WIDE_LOAD1]], <4 x i32> [[WIDE_LOAD]]
+; CHECK-NEXT: [[TMP11:%.*]] = freeze <4 x i1> [[TMP10]]
+; CHECK-NEXT: [[TMP12:%.*]] = call i1 @llvm.vector.reduce.or.v4i1(<4 x i1> [[TMP11]])
+; CHECK-NEXT: [[TMP13]] = select i1 [[TMP12]], <4 x i1> [[TMP11]], <4 x i1> [[TMP1]]
+; CHECK-NEXT: [[TMP14]] = select i1 [[TMP12]], <4 x i32> [[PREDPHI]], <4 x i32> [[VEC_PHI]]
+; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 4
+; CHECK-NEXT: [[TMP15:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
+; CHECK-NEXT: br i1 [[TMP15]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP10:![0-9]+]]
+; CHECK: [[MIDDLE_BLOCK]]:
+; CHECK-NEXT: [[TMP16:%.*]] = call i32 @llvm.experimental.vector.extract.last.active.v4i32(<4 x i32> [[TMP14]], <4 x i1> [[TMP13]], i32 -1)
+; CHECK-NEXT: [[CMP_N:%.*]] = icmp eq i64 [[N]], [[N_VEC]]
+; CHECK-NEXT: br i1 [[CMP_N]], [[EXIT:label %.*]], label %[[SCALAR_PH]]
+; CHECK: [[SCALAR_PH]]:
+; CHECK-NEXT: [[BC_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC]], %[[MIDDLE_BLOCK]] ], [ 0, %[[ENTRY]] ]
+; CHECK-NEXT: [[BC_MERGE_RDX:%.*]] = phi i32 [ [[TMP16]], %[[MIDDLE_BLOCK]] ], [ -1, %[[ENTRY]] ]
+; CHECK-NEXT: br label %[[LOOP_HEADER:.*]]
+; CHECK: [[LOOP_HEADER]]:
+;
+entry:
+ br label %loop.header
+
+loop.header:
+ %iv = phi i64 [ 0, %entry ], [ %iv.next, %loop.latch ]
+ %rdx = phi i32 [ -1, %entry ], [ %rdx.next, %loop.latch ]
+ %gep.a = getelementptr i32, ptr %a, i64 %iv
+ %ld.a = load i32, ptr %gep.a
+ %gep.b = getelementptr i32, ptr %b, i64 %iv
+ %ld.b = load i32, ptr %gep.b
+ %c.0 = icmp sgt i32 %ld.a, 5
+ br i1 %c.0, label %then, label %else
+
+then:
+ %c.1 = icmp ne i32 %ld.b, 0
+ br i1 %c.1, label %merge, label %update
+
+else:
+ %c.2 = icmp ne i32 %ld.a, 2
+ br i1 %c.2, label %merge, label %update
+
+update:
+ %val = phi i32 [ %ld.b, %then ], [ %ld.a, %else ]
+ br label %merge
+
+merge:
+ %rdx.next = phi i32 [ %rdx, %then ], [ %rdx, %else ], [ %val, %update ]
+ br label %loop.latch
+
+loop.latch:
+ %iv.next = add i64 %iv, 1
+ %ec = icmp eq i64 %iv.next, %n
+ br i1 %ec, label %exit, label %loop.header
+
+exit:
+ ret i32 %rdx.next
+}
diff --git a/llvm/test/Transforms/LoopVectorize/predicator.ll b/llvm/test/Transforms/LoopVectorize/predicator.ll
index 8a7b4fb2f71816..53d853dd86dff9 100644
--- a/llvm/test/Transforms/LoopVectorize/predicator.ll
+++ b/llvm/test/Transforms/LoopVectorize/predicator.ll
@@ -362,3 +362,498 @@ latch:
exit:
ret void
}
+
+define void @look_thru_phi(i1 %c0, i1 %c1, i32 %x, i32 %y, ptr %p) {
+; CHECK-LABEL: define void @look_thru_phi(
+; CHECK-SAME: i1 [[C0:%.*]], i1 [[C1:%.*]], i32 [[X:%.*]], i32 [[Y:%.*]], ptr [[P:%.*]]) {
+; CHECK-NEXT: [[ENTRY:.*:]]
+; CHECK-NEXT: br label %[[VECTOR_PH:.*]]
+; CHECK: [[VECTOR_PH]]:
+; CHECK-NEXT: [[BROADCAST_SPLATINSERT:%.*]] = insertelement <4 x i32> poison, i32 [[Y]], i64 0
+; CHECK-NEXT: [[BROADCAST_SPLAT:%.*]] = shufflevector <4 x i32> [[BROADCAST_SPLATINSERT]], <4 x i32> poison, <4 x i32> zeroinitializer
+; CHECK-NEXT: [[BROADCAST_SPLATINSERT1:%.*]] = insertelement <4 x i32> poison, i32 [[X]], i64 0
+; CHECK-NEXT: [[BROADCAST_SPLAT2:%.*]] = shufflevector <4 x i32> [[BROADCAST_SPLATINSERT1]], <4 x i32> poison, <4 x i32> zeroinitializer
+; CHECK-NEXT: [[PREDPHI7:%.*]] = select i1 [[C0]], <4 x i32> [[BROADCAST_SPLAT2]], <4 x i32> [[BROADCAST_SPLAT]]
+; CHECK-NEXT: br label %[[VECTOR_BODY:.*]]
+; CHECK: [[VECTOR_BODY]]:
+; CHECK-NEXT: [[INDEX:%.*]] = phi i32 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-NEXT: [[TMP3:%.*]] = getelementptr i32, ptr [[P]], i32 [[INDEX]]
+; CHECK-NEXT: store <4 x i32> [[PREDPHI7]], ptr [[TMP3]], align 4
+; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i32 [[INDEX]], 4
+; CHECK-NEXT: [[TMP4:%.*]] = icmp eq i32 [[INDEX_NEXT]], 128
+; CHECK-NEXT: br i1 [[TMP4]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP6:![0-9]+]]
+; CHECK: [[MIDDLE_BLOCK]]:
+; CHECK-NEXT: br label %[[EXIT:.*]]
+; CHECK: [[EXIT]]:
+; CHECK-NEXT: ret void
+;
+; bb0
+; / \
+; bb1 \
+; / | bb2
+; \ | /
+; bb3 bb4
+; \/
+; bb5
+entry:
+ br label %bb0
+
+bb0:
+ %iv = phi i32 [0, %entry], [%iv.next, %bb5]
+ br i1 %c0, label %bb1, label %bb2
+
+bb1:
+ br i1 %c1, label %bb3, label %bb4
+
+bb2:
+ br label %bb4
+
+bb3:
+ br label %bb5
+
+bb4:
+ %phi4 = phi i32 [%x, %bb1], [%y, %bb2]
+ br label %bb5
+
+bb5:
+ %phi5 = phi i32 [ %x, %bb3 ], [ %phi4, %bb4 ]
+ %gep = getelementptr i32, ptr %p, i32 %iv
+ store i32 %phi5, ptr %gep
+ %iv.next = add i32 %iv, 1
+ %ec = icmp eq i32 %iv.next, 128
+ br i1 %ec, label %exit, label %bb0
+
+exit:
+ ret void
+}
+
+; Same as the test above, but shouldn't look thru the phi because it has
+; multiple uses.
+define void @look_thru_phi_multi_use(i1 %c0, i1 %c1, i32 %x, i32 %y, ptr %p) {
+; CHECK-LABEL: define void @look_thru_phi_multi_use(
+; CHECK-SAME: i1 [[C0:%.*]], i1 [[C1:%.*]], i32 [[X:%.*]], i32 [[Y:%.*]], ptr [[P:%.*]]) {
+; CHECK-NEXT: [[ENTRY:.*:]]
+; CHECK-NEXT: br label %[[VECTOR_PH:.*]]
+; CHECK: [[VECTOR_PH]]:
+; CHECK-NEXT: [[BROADCAST_SPLATINSERT:%.*]] = insertelement <4 x i32> poison, i32 [[Y]], i64 0
+; CHECK-NEXT: [[BROADCAST_SPLAT:%.*]] = shufflevector <4 x i32> [[BROADCAST_SPLATINSERT]], <4 x i32> poison, <4 x i32> zeroinitializer
+; CHECK-NEXT: [[BROADCAST_SPLATINSERT1:%.*]] = insertelement <4 x i32> poison, i32 [[X]], i64 0
+; CHECK-NEXT: [[BROADCAST_SPLAT2:%.*]] = shufflevector <4 x i32> [[BROADCAST_SPLATINSERT1]], <4 x i32> poison, <4 x i32> zeroinitializer
+; CHECK-NEXT: [[BROADCAST_SPLATINSERT5:%.*]] = insertelement <4 x i1> poison, i1 [[C1]], i64 0
+; CHECK-NEXT: [[BROADCAST_SPLAT6:%.*]] = shufflevector <4 x i1> [[BROADCAST_SPLATINSERT5]], <4 x i1> poison, <4 x i32> zeroinitializer
+; CHECK-NEXT: [[BROADCAST_SPLATINSERT6:%.*]] = insertelement <4 x i1> poison, i1 [[C0]], i64 0
+; CHECK-NEXT: [[BROADCAST_SPLAT7:%.*]] = shufflevector <4 x i1> [[BROADCAST_SPLATINSERT6]], <4 x i1> poison, <4 x i32> zeroinitializer
+; CHECK-NEXT: [[TMP0:%.*]] = xor <4 x i1> [[BROADCAST_SPLAT7]], splat (i1 true)
+; CHECK-NEXT: [[TMP1:%.*]] = xor <4 x i1> [[BROADCAST_SPLAT6]], splat (i1 true)
+; CHECK-NEXT: [[TMP2:%.*]] = select <4 x i1> [[BROADCAST_SPLAT7]], <4 x i1> [[TMP1]], <4 x i1> zeroinitializer
+; CHECK-NEXT: [[TMP3:%.*]] = or <4 x i1> [[TMP0]], [[TMP2]]
+; CHECK-NEXT: [[PREDPHI:%.*]] = select <4 x i1> [[TMP2]], <4 x i32> [[BROADCAST_SPLAT2]], <4 x i32> [[BROADCAST_SPLAT]]
+; CHECK-NEXT: [[PREDPHI7:%.*]] = select <4 x i1> [[TMP3]], <4 x i32> [[PREDPHI]], <4 x i32> [[BROADCAST_SPLAT2]]
+; CHECK-NEXT: br label %[[VECTOR_BODY:.*]]
+; CHECK: [[VECTOR_BODY]]:
+; CHECK-NEXT: [[INDEX:%.*]] = phi i32 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-NEXT: [[TMP4:%.*]] = getelementptr i32, ptr [[P]], i32 [[INDEX]]
+; CHECK-NEXT: call void @llvm.masked.store.v4i32.p0(<4 x i32> [[PREDPHI]], ptr align 4 [[TMP4]], <4 x i1> [[TMP3]])
+; CHECK-NEXT: store <4 x i32> [[PREDPHI7]], ptr [[TMP4]], align 4
+; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i32 [[INDEX]], 4
+; CHECK-NEXT: [[TMP5:%.*]] = icmp eq i32 [[INDEX_NEXT]], 128
+; CHECK-NEXT: br i1 [[TMP5]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP7:![0-9]+]]
+; CHECK: [[MIDDLE_BLOCK]]:
+; CHECK-NEXT: br label %[[EXIT:.*]]
+; CHECK: [[EXIT]]:
+; CHECK-NEXT: ret void
+;
+; bb0
+; / \
+; bb1 \
+; / | bb2
+; \ | /
+; bb3 bb4
+; \/
+; bb5
+entry:
+ br label %bb0
+
+bb0:
+ %iv = phi i32 [0, %entry], [%iv.next, %bb5]
+ %gep = getelementptr i32, ptr %p, i32 %iv
+ br i1 %c0, label %bb1, label %bb2
+
+bb1:
+ br i1 %c1, label %bb3, label %bb4
+
+bb2:
+ br label %bb4
+
+bb3:
+ br label %bb5
+
+bb4:
+ %phi4 = phi i32 [%x, %bb1], [%y, %bb2]
+ store i32 %phi4, ptr %gep
+ br label %bb5
+
+bb5:
+ %phi5 = phi i32 [ %x, %bb3 ], [ %phi4, %bb4 ]
+ store i32 %phi5, ptr %gep
+ %iv.next = add i32 %iv, 1
+ %ec = icmp eq i32 %iv.next, 128
+ br i1 %ec, label %exit, label %bb0
+
+exit:
+ ret void
+}
+
+; Don't look through %phi3 as it doesn't share any incoming values with %phi4.
+define void @look_thru_phi_no_shared_values(i1 %c0, i1 %c1, i32 %x, i32 %y, i32 %z, ptr %p) {
+; CHECK-LABEL: define void @look_thru_phi_no_shared_values(
+; CHECK-SAME: i1 [[C0:%.*]], i1 [[C1:%.*]], i32 [[X:%.*]], i32 [[Y:%.*]], i32 [[Z:%.*]], ptr [[P:%.*]]) {
+; CHECK-NEXT: [[ENTRY:.*:]]
+; CHECK-NEXT: br label %[[VECTOR_PH:.*]]
+; CHECK: [[VECTOR_PH]]:
+; CHECK-NEXT: [[BROADCAST_SPLATINSERT:%.*]] = insertelement <4 x i32> poison, i32 [[X]], i64 0
+; CHECK-NEXT: [[BROADCAST_SPLAT:%.*]] = shufflevector <4 x i32> [[BROADCAST_SPLATINSERT]], <4 x i32> poison, <4 x i32> zeroinitializer
+; CHECK-NEXT: [[BROADCAST_SPLATINSERT1:%.*]] = insertelement <4 x i32> poison, i32 [[Z]], i64 0
+; CHECK-NEXT: [[BROADCAST_SPLAT2:%.*]] = shufflevector <4 x i32> [[BROADCAST_SPLATINSERT1]], <4 x i32> poison, <4 x i32> zeroinitializer
+; CHECK-NEXT: [[BROADCAST_SPLATINSERT3:%.*]] = insertelement <4 x i32> poison, i32 [[Y]], i64 0
+; CHECK-NEXT: [[BROADCAST_SPLAT4:%.*]] = shufflevector <4 x i32> [[BROADCAST_SPLATINSERT3]], <4 x i32> poison, <4 x i32> zeroinitializer
+; CHECK-NEXT: [[PREDPHI:%.*]] = select i1 [[C1]], <4 x i32> [[BROADCAST_SPLAT2]], <4 x i32> [[BROADCAST_SPLAT4]]
+; CHECK-NEXT: [[PREDPHI5:%.*]] = select i1 [[C0]], <4 x i32> [[BROADCAST_SPLAT]], <4 x i32> [[PREDPHI]]
+; CHECK-NEXT: br label %[[VECTOR_BODY:.*]]
+; CHECK: [[VECTOR_BODY]]:
+; CHECK-NEXT: [[INDEX:%.*]] = phi i32 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-NEXT: [[TMP0:%.*]] = getelementptr i32, ptr [[P]], i32 [[INDEX]]
+; CHECK-NEXT: store <4 x i32> [[PREDPHI5]], ptr [[TMP0]], align 4
+; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i32 [[INDEX]], 4
+; CHECK-NEXT: [[TMP1:%.*]] = icmp eq i32 [[INDEX_NEXT]], 128
+; CHECK-NEXT: br i1 [[TMP1]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP8:![0-9]+]]
+; CHECK: [[MIDDLE_BLOCK]]:
+; CHECK-NEXT: br label %[[EXIT:.*]]
+; CHECK: [[EXIT]]:
+; CHECK-NEXT: ret void
+;
+; bb0
+; | \
+; | bb1
+; | | \
+; | | bb2
+; | | /
+; | bb3
+; | /
+; bb4
+entry:
+ br label %bb0
+
+bb0:
+ %iv = phi i32 [0, %entry], [%iv.next, %bb4]
+ br i1 %c0, label %bb4, label %bb1
+
+bb1:
+ br i1 %c1, label %bb2, label %bb3
+
+bb2:
+ br label %bb3
+
+bb3:
+ %phi3 = phi i32 [%y, %bb1], [%z, %bb2]
+ br label %bb4
+
+bb4:
+ %phi4 = phi i32 [%x, %bb0], [%phi3, %bb3]
+ %gep = getelementptr i32, ptr %p, i32 %iv
+ store i32 %phi4, ptr %gep
+ %iv.next = add i32 %iv, 1
+ %ec = icmp eq i32 %iv.next, 128
+ br i1 %ec, label %exit, label %bb0
+
+exit:
+ ret void
+}
+
+; bb4 will create the edge mask bb1->bb3 before bb3. Make sure bb4
+; inserts its blend after the already created edge mask.
+define i32 @look_thru_phi_blend_insert_pos(i1 %c0, i1 %c1) {
+; CHECK-LABEL: define i32 @look_thru_phi_blend_insert_pos(
+; CHECK-SAME: i1 [[C0:%.*]], i1 [[C1:%.*]]) {
+; CHECK-NEXT: [[ENTRY:.*:]]
+; CHECK-NEXT: br label %[[VECTOR_PH:.*]]
+; CHECK: [[VECTOR_PH]]:
+; CHECK-NEXT: [[BROADCAST_SPLATINSERT:%.*]] = insertelement <4 x i1> poison, i1 [[C1]], i64 0
+; CHECK-NEXT: [[BROADCAST_SPLAT:%.*]] = shufflevector <4 x i1> [[BROADCAST_SPLATINSERT]], <4 x i1> poison, <4 x i32> zeroinitializer
+; CHECK-NEXT: [[BROADCAST_SPLATINSERT1:%.*]] = insertelement <4 x i1> poison, i1 [[C0]], i64 0
+; CHECK-NEXT: [[BROADCAST_SPLAT2:%.*]] = shufflevector <4 x i1> [[BROADCAST_SPLATINSERT1]], <4 x i1> poison, <4 x i32> zeroinitializer
+; CHECK-NEXT: [[TMP0:%.*]] = xor <4 x i1> [[BROADCAST_SPLAT2]], splat (i1 true)
+; CHECK-NEXT: [[TMP1:%.*]] = xor <4 x i1> [[BROADCAST_SPLAT]], splat (i1 true)
+; CHECK-NEXT: [[TMP2:%.*]] = select <4 x i1> [[TMP0]], <4 x i1> [[TMP1]], <4 x i1> zeroinitializer
+; CHECK-NEXT: [[PREDPHI:%.*]] = select <4 x i1> [[TMP2]], <4 x i32> splat (i32 2), <4 x i32> splat (i32 1)
+; CHECK-NEXT: br label %[[VECTOR_BODY:.*]]
+; CHECK: [[VECTOR_BODY]]:
+; CHECK-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 4
+; CHECK-NEXT: [[TMP3:%.*]] = icmp eq i64 [[INDEX_NEXT]], 128
+; CHECK-NEXT: br i1 [[TMP3]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP9:![0-9]+]]
+; CHECK: [[MIDDLE_BLOCK]]:
+; CHECK-NEXT: [[TMP4:%.*]] = extractelement <4 x i32> [[PREDPHI]], i64 3
+; CHECK-NEXT: br label %[[EXIT:.*]]
+; CHECK: [[EXIT]]:
+; CHECK-NEXT: ret i32 [[TMP4]]
+;
+; bb0
+; / \
+; bb1 \
+; / \ \
+; bb2 | /
+; \ / /
+; bb3 /
+; \ /
+; bb4
+entry:
+ br label %bb0
+
+bb0:
+ %iv = phi i64 [ 0, %entry ], [ %iv.next, %bb4 ]
+ br i1 %c0, label %bb4, label %bb1
+
+bb1:
+ br i1 %c1, label %bb3, label %bb2
+
+bb2:
+ br label %bb3
+
+bb3:
+ %phi3 = phi i32 [ 1, %bb1 ], [ 2, %bb2 ]
+ br label %bb4
+
+bb4:
+ %phi4 = phi i32 [ 1, %bb0 ], [ %phi3, %bb3 ]
+ %iv.next = add i64 %iv, 1
+ %ec = icmp eq i64 %iv.next, 128
+ br i1 %ec, label %exit, label %bb0
+
+exit:
+ ret i32 %phi4
+}
+
+define void @diamond_phi2_look_thru_phi(ptr %a, i1 %c1, i1 %c2) {
+; CHECK-LABEL: define void @diamond_phi2_look_thru_phi(
+; CHECK-SAME: ptr [[A:%.*]], i1 [[C1:%.*]], i1 [[C2:%.*]]) {
+; CHECK-NEXT: [[ENTRY:.*:]]
+; CHECK-NEXT: br label %[[VECTOR_PH:.*]]
+; CHECK: [[VECTOR_PH]]:
+; CHECK-NEXT: [[BROADCAST_SPLATINSERT:%.*]] = insertelement <4 x i1> poison, i1 [[C1]], i64 0
+; CHECK-NEXT: [[BROADCAST_SPLAT:%.*]] = shufflevector <4 x i1> [[BROADCAST_SPLATINSERT]], <4 x i1> poison, <4 x i32> zeroinitializer
+; CHECK-NEXT: [[BROADCAST_SPLATINSERT1:%.*]] = insertelement <4 x i1> poison, i1 [[C2]], i64 0
+; CHECK-NEXT: [[BROADCAST_SPLAT2:%.*]] = shufflevector <4 x i1> [[BROADCAST_SPLATINSERT1]], <4 x i1> poison, <4 x i32> zeroinitializer
+; CHECK-NEXT: [[TMP0:%.*]] = xor <4 x i1> [[BROADCAST_SPLAT]], splat (i1 true)
+; CHECK-NEXT: [[TMP3:%.*]] = xor <4 x i1> [[BROADCAST_SPLAT2]], splat (i1 true)
+; CHECK-NEXT: br label %[[VECTOR_BODY:.*]]
+; CHECK: [[VECTOR_BODY]]:
+; CHECK-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-NEXT: [[VEC_IND:%.*]] = phi <4 x i64> [ <i64 0, i64 1, i64 2, i64 3>, %[[VECTOR_PH]] ], [ [[VEC_IND_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-NEXT: [[TMP1:%.*]] = icmp sle <4 x i64> [[VEC_IND]], zeroinitializer
+; CHECK-NEXT: [[TMP2:%.*]] = xor <4 x i1> [[TMP1]], splat (i1 true)
+; CHECK-NEXT: [[TMP4:%.*]] = select <4 x i1> [[TMP1]], <4 x i1> [[BROADCAST_SPLAT]], <4 x i1> zeroinitializer
+; CHECK-NEXT: [[PREDPHI:%.*]] = select <4 x i1> [[TMP4]], <4 x i64> zeroinitializer, <4 x i64> splat (i64 1)
+; CHECK-NEXT: [[TMP6:%.*]] = select <4 x i1> [[TMP1]], <4 x i1> [[TMP0]], <4 x i1> zeroinitializer
+; CHECK-NEXT: [[TMP9:%.*]] = select <4 x i1> [[TMP2]], <4 x i1> [[TMP3]], <4 x i1> zeroinitializer
+; CHECK-NEXT: [[PREDPHI3:%.*]] = select <4 x i1> [[TMP9]], <4 x i64> splat (i64 3), <4 x i64> [[PREDPHI]]
+; CHECK-NEXT: [[PREDPHI4:%.*]] = select <4 x i1> [[TMP6]], <4 x i64> splat (i64 2), <4 x i64> [[PREDPHI3]]
+; CHECK-NEXT: [[TMP7:%.*]] = getelementptr i64, ptr [[A]], i64 [[INDEX]]
+; CHECK-NEXT: store <4 x i64> [[PREDPHI4]], ptr [[TMP7]], align 4
+; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 4
+; CHECK-NEXT: [[VEC_IND_NEXT]] = add nuw nsw <4 x i64> [[VEC_IND]], splat (i64 4)
+; CHECK-NEXT: [[TMP8:%.*]] = icmp eq i64 [[INDEX_NEXT]], 128
+; CHECK-NEXT: br i1 [[TMP8]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP10:![0-9]+]]
+; CHECK: [[MIDDLE_BLOCK]]:
+; CHECK-NEXT: br label %[[EXIT:.*]]
+; CHECK: [[EXIT]]:
+; CHECK-NEXT: ret void
+;
+; bb0
+; / \
+; bb1 bb2
+; | \ / |
+; | bb4 |
+; \ | /
+; bb5
+entry:
+ br label %bb0
+
+bb0:
+ %iv = phi i64 [0, %entry], [%iv.next, %bb5]
+ %c0 = icmp sle i64 %iv, 0
+ br i1 %c0, label %bb1, label %bb2
+
+bb1:
+ %add1 = add i64 %iv, 1
+ br i1 %c1, label %bb4, label %bb5
+
+bb2:
+ %add2 = add i64 %iv, 2
+ br i1 %c2, label %bb4, label %bb5
+
+bb4:
+ %phi4 = phi i64 [0, %bb1], [1, %bb2]
+ br label %bb5
+
+bb5:
+ %phi5 = phi i64 [2, %bb1], [3, %bb2], [%phi4, %bb4]
+ %gep = getelementptr i64, ptr %a, i64 %iv
+ store i64 %phi5, ptr %gep
+ %iv.next = add nsw nuw i64 %iv, 1
+ %ec = icmp eq i64 %iv.next, 128
+ br i1 %ec, label %exit, label %bb0
+
+exit:
+ ret void
+}
+
+define void @blend_look_through_no_shared_values(ptr noalias %dst, ptr noalias %conds, i32 %x, i32 %y, i32 %u, i32 %w, i64 %n) {
+; CHECK-LABEL: define void @blend_look_through_no_shared_values(
+; CHECK-SAME: ptr noalias [[DST:%.*]], ptr noalias [[CONDS:%.*]], i32 [[X:%.*]], i32 [[Y:%.*]], i32 [[U:%.*]], i32 [[W:%.*]], i64 [[N:%.*]]) {
+; CHECK-NEXT: [[ENTRY:.*]]:
+; CHECK-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[N]], 4
+; CHECK-NEXT: br i1 [[MIN_ITERS_CHECK]], label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]]
+; CHECK: [[VECTOR_PH]]:
+; CHECK-NEXT: [[TMP0:%.*]] = and i64 [[N]], 3
+; CHECK-NEXT: [[N_VEC:%.*]] = sub i64 [[N]], [[TMP0]]
+; CHECK-NEXT: [[BROADCAST_SPLATINSERT:%.*]] = insertelement <4 x i32> poison, i32 [[U]], i64 0
+; CHECK-NEXT: [[BROADCAST_SPLAT:%.*]] = shufflevector <4 x i32> [[BROADCAST_SPLATINSERT]], <4 x i32> poison, <4 x i32> zeroinitializer
+; CHECK-NEXT: [[BROADCAST_SPLATINSERT1:%.*]] = insertelement <4 x i32> poison, i32 [[W]], i64 0
+; CHECK-NEXT: [[BROADCAST_SPLAT2:%.*]] = shufflevector <4 x i32> [[BROADCAST_SPLATINSERT1]], <4 x i32> poison, <4 x i32> zeroinitializer
+; CHECK-NEXT: [[BROADCAST_SPLATINSERT3:%.*]] = insertelement <4 x i32> poison, i32 [[X]], i64 0
+; CHECK-NEXT: [[BROADCAST_SPLAT4:%.*]] = shufflevector <4 x i32> [[BROADCAST_SPLATINSERT3]], <4 x i32> poison, <4 x i32> zeroinitializer
+; CHECK-NEXT: [[BROADCAST_SPLATINSERT5:%.*]] = insertelement <4 x i32> poison, i32 [[Y]], i64 0
+; CHECK-NEXT: [[BROADCAST_SPLAT6:%.*]] = shufflevector <4 x i32> [[BROADCAST_SPLATINSERT5]], <4 x i32> poison, <4 x i32> zeroinitializer
+; CHECK-NEXT: br label %[[VECTOR_BODY:.*]]
+; CHECK: [[VECTOR_BODY]]:
+; CHECK-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-NEXT: [[TMP1:%.*]] = getelementptr i8, ptr [[CONDS]], i64 [[INDEX]]
+; CHECK-NEXT: [[WIDE_LOAD:%.*]] = load <4 x i8>, ptr [[TMP1]], align 1
+; CHECK-NEXT: [[TMP2:%.*]] = trunc <4 x i8> [[WIDE_LOAD]] to <4 x i1>
+; CHECK-NEXT: [[TMP3:%.*]] = lshr <4 x i8> [[WIDE_LOAD]], splat (i8 1)
+; CHECK-NEXT: [[TMP4:%.*]] = trunc <4 x i8> [[TMP3]] to <4 x i1>
+; CHECK-NEXT: [[TMP5:%.*]] = lshr <4 x i8> [[WIDE_LOAD]], splat (i8 2)
+; CHECK-NEXT: [[TMP6:%.*]] = trunc <4 x i8> [[TMP5]] to <4 x i1>
+; CHECK-NEXT: [[TMP7:%.*]] = lshr <4 x i8> [[WIDE_LOAD]], splat (i8 3)
+; CHECK-NEXT: [[TMP8:%.*]] = trunc <4 x i8> [[TMP7]] to <4 x i1>
+; CHECK-NEXT: [[PREDPHI:%.*]] = select <4 x i1> [[TMP8]], <4 x i32> [[BROADCAST_SPLAT]], <4 x i32> [[BROADCAST_SPLAT2]]
+; CHECK-NEXT: [[TMP9:%.*]] = xor <4 x i1> [[TMP4]], splat (i1 true)
+; CHECK-NEXT: [[TMP10:%.*]] = select <4 x i1> [[TMP2]], <4 x i1> [[TMP9]], <4 x i1> zeroinitializer
+; CHECK-NEXT: [[TMP11:%.*]] = select <4 x i1> [[TMP2]], <4 x i1> [[TMP4]], <4 x i1> zeroinitializer
+; CHECK-NEXT: [[TMP12:%.*]] = select <4 x i1> [[TMP11]], <4 x i1> [[TMP6]], <4 x i1> zeroinitializer
+; CHECK-NEXT: [[TMP13:%.*]] = or <4 x i1> [[TMP10]], [[TMP12]]
+; CHECK-NEXT: [[PREDPHI7:%.*]] = select <4 x i1> [[TMP2]], <4 x i32> [[BROADCAST_SPLAT6]], <4 x i32> [[PREDPHI]]
+; CHECK-NEXT: [[PREDPHI8:%.*]] = select <4 x i1> [[TMP13]], <4 x i32> [[BROADCAST_SPLAT4]], <4 x i32> [[PREDPHI7]]
+; CHECK-NEXT: [[TMP14:%.*]] = getelementptr i32, ptr [[DST]], i64 [[INDEX]]
+; CHECK-NEXT: store <4 x i32> [[PREDPHI8]], ptr [[TMP14]], align 4
+; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 4
+; CHECK-NEXT: [[TMP15:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
+; CHECK-NEXT: br i1 [[TMP15]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP11:![0-9]+]]
+; CHECK: [[MIDDLE_BLOCK]]:
+; CHECK-NEXT: [[CMP_N:%.*]] = icmp eq i64 [[N]], [[N_VEC]]
+; CHECK-NEXT: br i1 [[CMP_N]], label %[[EXIT:.*]], label %[[SCALAR_PH]]
+; CHECK: [[SCALAR_PH]]:
+; CHECK-NEXT: [[BC_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC]], %[[MIDDLE_BLOCK]] ], [ 0, %[[ENTRY]] ]
+; CHECK-NEXT: br label %[[LOOP_HEADER:.*]]
+; CHECK: [[LOOP_HEADER]]:
+; CHECK-NEXT: [[IV:%.*]] = phi i64 [ [[BC_RESUME_VAL]], %[[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], %[[LOOP_LATCH:.*]] ]
+; CHECK-NEXT: [[GEP_CONDS:%.*]] = getelementptr i8, ptr [[CONDS]], i64 [[IV]]
+; CHECK-NEXT: [[LD:%.*]] = load i8, ptr [[GEP_CONDS]], align 1
+; CHECK-NEXT: [[C_0:%.*]] = trunc i8 [[LD]] to i1
+; CHECK-NEXT: [[SHR_1:%.*]] = lshr i8 [[LD]], 1
+; CHECK-NEXT: [[C_1:%.*]] = trunc i8 [[SHR_1]] to i1
+; CHECK-NEXT: [[SHR_2:%.*]] = lshr i8 [[LD]], 2
+; CHECK-NEXT: [[C_2:%.*]] = trunc i8 [[SHR_2]] to i1
+; CHECK-NEXT: [[SHR_3:%.*]] = lshr i8 [[LD]], 3
+; CHECK-NEXT: [[C_3:%.*]] = trunc i8 [[SHR_3]] to i1
+; CHECK-NEXT: br i1 [[C_0]], label %[[THEN:.*]], label %[[ELSE:.*]]
+; CHECK: [[THEN]]:
+; CHECK-NEXT: br i1 [[C_1]], label %[[THEN_THEN:.*]], label %[[THEN_ELSE:.*]]
+; CHECK: [[THEN_THEN]]:
+; CHECK-NEXT: br i1 [[C_2]], label %[[USE_X:.*]], label %[[USE_Y:.*]]
+; CHECK: [[THEN_ELSE]]:
+; CHECK-NEXT: br label %[[THEN_ELSE_CONT:.*]]
+; CHECK: [[THEN_ELSE_CONT]]:
+; CHECK-NEXT: br label %[[USE_X]]
+; CHECK: [[USE_X]]:
+; CHECK-NEXT: br label %[[LOOP_LATCH]]
+; CHECK: [[USE_Y]]:
+; CHECK-NEXT: br label %[[LOOP_LATCH]]
+; CHECK: [[ELSE]]:
+; CHECK-NEXT: br i1 [[C_3]], label %[[ELSE_THEN:.*]], label %[[ELSE_ELSE:.*]]
+; CHECK: [[ELSE_THEN]]:
+; CHECK-NEXT: br label %[[ELSE_MERGE:.*]]
+; CHECK: [[ELSE_ELSE]]:
+; CHECK-NEXT: br label %[[ELSE_MERGE]]
+; CHECK: [[ELSE_MERGE]]:
+; CHECK-NEXT: [[P:%.*]] = phi i32 [ [[U]], %[[ELSE_THEN]] ], [ [[W]], %[[ELSE_ELSE]] ]
+; CHECK-NEXT: br label %[[LOOP_LATCH]]
+; CHECK: [[LOOP_LATCH]]:
+; CHECK-NEXT: [[MERGE:%.*]] = phi i32 [ [[X]], %[[USE_X]] ], [ [[Y]], %[[USE_Y]] ], [ [[P]], %[[ELSE_MERGE]] ]
+; CHECK-NEXT: [[GEP_DST:%.*]] = getelementptr i32, ptr [[DST]], i64 [[IV]]
+; CHECK-NEXT: store i32 [[MERGE]], ptr [[GEP_DST]], align 4
+; CHECK-NEXT: [[IV_NEXT]] = add i64 [[IV]], 1
+; CHECK-NEXT: [[EC:%.*]] = icmp eq i64 [[IV_NEXT]], [[N]]
+; CHECK-NEXT: br i1 [[EC]], label %[[EXIT]], label %[[LOOP_HEADER]], !llvm.loop [[LOOP12:![0-9]+]]
+; CHECK: [[EXIT]]:
+; CHECK-NEXT: ret void
+;
+entry:
+ br label %loop.header
+
+loop.header:
+ %iv = phi i64 [ 0, %entry ], [ %iv.next, %loop.latch ]
+ %gep.conds = getelementptr i8, ptr %conds, i64 %iv
+ %ld = load i8, ptr %gep.conds
+ %c.0 = trunc i8 %ld to i1
+ %shr.1 = lshr i8 %ld, 1
+ %c.1 = trunc i8 %shr.1 to i1
+ %shr.2 = lshr i8 %ld, 2
+ %c.2 = trunc i8 %shr.2 to i1
+ %shr.3 = lshr i8 %ld, 3
+ %c.3 = trunc i8 %shr.3 to i1
+ br i1 %c.0, label %then, label %else
+
+then:
+ br i1 %c.1, label %then.then, label %then.else
+
+then.then:
+ br i1 %c.2, label %use.x, label %use.y
+
+then.else:
+ br label %then.else.cont
+
+then.else.cont:
+ br label %use.x
+
+use.x:
+ br label %loop.latch
+
+use.y:
+ br label %loop.latch
+
+else:
+ br i1 %c.3, label %else.then, label %else.else
+
+else.then:
+ br label %else.merge
+
+else.else:
+ br label %else.merge
+
+else.merge:
+ %p = phi i32 [ %u, %else.then ], [ %w, %else.else ]
+ br label %loop.latch
+
+loop.latch:
+ %merge = phi i32 [ %x, %use.x ], [ %y, %use.y ], [ %p, %else.merge ]
+ %gep.dst = getelementptr i32, ptr %dst, i64 %iv
+ store i32 %merge, ptr %gep.dst
+ %iv.next = add i64 %iv, 1
+ %ec = icmp eq i64 %iv.next, %n
+ br i1 %ec, label %exit, label %loop.header
+
+exit:
+ ret void
+}
diff --git a/llvm/test/Transforms/LoopVectorize/select-cmp-blend-chain.ll b/llvm/test/Transforms/LoopVectorize/select-cmp-blend-chain.ll
index 6cae52e2a8f283..e1fa8237cb9633 100644
--- a/llvm/test/Transforms/LoopVectorize/select-cmp-blend-chain.ll
+++ b/llvm/test/Transforms/LoopVectorize/select-cmp-blend-chain.ll
@@ -97,15 +97,25 @@ define i32 @anyof_three_blend_chain(i1 %c0, i1 %c1, i1 %c2, i32 %n) {
; CHECK: [[VECTOR_PH]]:
; CHECK-NEXT: [[N_MOD_VF:%.*]] = and i32 [[SMAX]], 3
; CHECK-NEXT: [[N_VEC:%.*]] = sub i32 [[SMAX]], [[N_MOD_VF]]
+; CHECK-NEXT: [[BROADCAST_SPLATINSERT:%.*]] = insertelement <4 x i1> poison, i1 [[C2]], i64 0
+; CHECK-NEXT: [[BROADCAST_SPLAT:%.*]] = shufflevector <4 x i1> [[BROADCAST_SPLATINSERT]], <4 x i1> poison, <4 x i32> zeroinitializer
+; CHECK-NEXT: [[BROADCAST_SPLATINSERT1:%.*]] = insertelement <4 x i1> poison, i1 [[C1]], i64 0
+; CHECK-NEXT: [[BROADCAST_SPLAT2:%.*]] = shufflevector <4 x i1> [[BROADCAST_SPLATINSERT1]], <4 x i1> poison, <4 x i32> zeroinitializer
+; CHECK-NEXT: [[BROADCAST_SPLATINSERT3:%.*]] = insertelement <4 x i1> poison, i1 [[C0]], i64 0
+; CHECK-NEXT: [[BROADCAST_SPLAT4:%.*]] = shufflevector <4 x i1> [[BROADCAST_SPLATINSERT3]], <4 x i1> poison, <4 x i32> zeroinitializer
+; CHECK-NEXT: [[TMP7:%.*]] = xor <4 x i1> [[BROADCAST_SPLAT4]], splat (i1 true)
+; CHECK-NEXT: [[TMP3:%.*]] = xor <4 x i1> [[BROADCAST_SPLAT2]], splat (i1 true)
+; CHECK-NEXT: [[TMP9:%.*]] = select <4 x i1> [[TMP7]], <4 x i1> [[TMP3]], <4 x i1> zeroinitializer
+; CHECK-NEXT: [[TMP10:%.*]] = xor <4 x i1> [[BROADCAST_SPLAT]], splat (i1 true)
+; CHECK-NEXT: [[TMP11:%.*]] = select <4 x i1> [[TMP9]], <4 x i1> [[TMP10]], <4 x i1> zeroinitializer
; CHECK-NEXT: br label %[[VECTOR_BODY:.*]]
; CHECK: [[VECTOR_BODY]]:
; CHECK-NEXT: [[INDEX:%.*]] = phi i32 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
; CHECK-NEXT: [[VEC_IND:%.*]] = phi <4 x i32> [ <i32 0, i32 1, i32 2, i32 3>, %[[VECTOR_PH]] ], [ [[VEC_IND_NEXT:%.*]], %[[VECTOR_BODY]] ]
; CHECK-NEXT: [[VEC_PHI:%.*]] = phi <4 x i1> [ zeroinitializer, %[[VECTOR_PH]] ], [ [[PREDPHI2:%.*]], %[[VECTOR_BODY]] ]
; CHECK-NEXT: [[TMP4:%.*]] = icmp eq <4 x i32> [[VEC_IND]], zeroinitializer
-; CHECK-NEXT: [[TMP3:%.*]] = or <4 x i1> [[VEC_PHI]], [[TMP4]]
-; CHECK-NEXT: [[PREDPHI:%.*]] = select i1 [[C2]], <4 x i1> [[VEC_PHI]], <4 x i1> [[TMP3]]
-; CHECK-NEXT: [[PREDPHI1:%.*]] = select i1 [[C1]], <4 x i1> [[VEC_PHI]], <4 x i1> [[PREDPHI]]
+; CHECK-NEXT: [[TMP8:%.*]] = select <4 x i1> [[TMP11]], <4 x i1> [[TMP4]], <4 x i1> zeroinitializer
+; CHECK-NEXT: [[PREDPHI1:%.*]] = or <4 x i1> [[VEC_PHI]], [[TMP8]]
; CHECK-NEXT: [[PREDPHI2]] = select i1 [[C0]], <4 x i1> [[VEC_PHI]], <4 x i1> [[PREDPHI1]]
; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i32 [[INDEX]], 4
; CHECK-NEXT: [[VEC_IND_NEXT]] = add <4 x i32> [[VEC_IND]], splat (i32 4)
diff --git a/llvm/test/Transforms/LoopVectorize/uniform-blend.ll b/llvm/test/Transforms/LoopVectorize/uniform-blend.ll
index 8d86c8ca2bd9d7..a016a26d479ec7 100644
--- a/llvm/test/Transforms/LoopVectorize/uniform-blend.ll
+++ b/llvm/test/Transforms/LoopVectorize/uniform-blend.ll
@@ -96,19 +96,10 @@ define void @blend_chain_iv(i1 %c) {
; CHECK: [[VECTOR_PH]]:
; CHECK-NEXT: br label %[[VECTOR_BODY:.*]]
; CHECK: [[VECTOR_BODY]]:
-; CHECK-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
-; CHECK-NEXT: [[TMP3:%.*]] = add i64 [[INDEX]], 1
-; CHECK-NEXT: [[TMP5:%.*]] = add i64 [[INDEX]], 2
-; CHECK-NEXT: [[TMP7:%.*]] = add i64 [[INDEX]], 3
-; CHECK-NEXT: [[TMP2:%.*]] = getelementptr inbounds [32 x i16], ptr @dst, i16 0, i64 [[INDEX]]
-; CHECK-NEXT: [[TMP4:%.*]] = getelementptr inbounds [32 x i16], ptr @dst, i16 0, i64 [[TMP3]]
-; CHECK-NEXT: [[TMP6:%.*]] = getelementptr inbounds [32 x i16], ptr @dst, i16 0, i64 [[TMP5]]
+; CHECK-NEXT: [[TMP7:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
; CHECK-NEXT: [[TMP8:%.*]] = getelementptr inbounds [32 x i16], ptr @dst, i16 0, i64 [[TMP7]]
-; CHECK-NEXT: store i16 0, ptr [[TMP2]], align 2
-; CHECK-NEXT: store i16 0, ptr [[TMP4]], align 2
-; CHECK-NEXT: store i16 0, ptr [[TMP6]], align 2
-; CHECK-NEXT: store i16 0, ptr [[TMP8]], align 2
-; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 4
+; CHECK-NEXT: store <4 x i16> zeroinitializer, ptr [[TMP8]], align 2
+; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[TMP7]], 4
; CHECK-NEXT: [[TMP9:%.*]] = icmp eq i64 [[INDEX_NEXT]], 32
; CHECK-NEXT: br i1 [[TMP9]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP4:![0-9]+]]
; CHECK: [[MIDDLE_BLOCK]]:
>From 94529588019cc4864e7ed181313b3634b48faf01 Mon Sep 17 00:00:00 2001
From: Luke Lau <luke at igalia.com>
Date: Wed, 28 Oct 2026 22:32:02 +0800
Subject: [PATCH 09/15] [VPlan] Preserve SSA in early exit loops
MaskedCond is used to logically-and the condition to early exit with the current block in-mask. However because we use it in the latch block where the condition isn't defined on all paths, we have to workaround it not being in SSA in the verifier.
This fixes this by using reconstructSSA added in #212209 to preserve SSA in both the exit conditions and live outs in the exit blocks. It also means we get the right exit condition without MaskedCond and allows us to remove the MaskedCond exceptions in the verifier. MaskedCond is still kept since a followup PR will still use it.
One subtle point is that we need to freeze any branch conditions. Any lanes after and including the early exiting lane might contain poison and we can have branches on those poison lanes that didn't we didn't previously branch on in the scalar loop.
There's no miscompile on main today because we coincidentally don't keep SSA so exit conditions are read directly without being polluted by poison stemming from a branch past the early exit. But in this PR the exit conditions are polluted and we can end up with poison in the condition operand of a select, which causes an undefined result for AnyOf even if it freezes its operand. The poison in the branch will also cause problems in future if we decide to preserve uniform control flow and emit branches instead of blends. The exit_cond_merged_after_poison_branch test illustrates this.
---
.../Transforms/Vectorize/VPlanTransforms.cpp | 35 +++-
.../Transforms/Vectorize/VPlanVerifier.cpp | 19 +-
.../predicated-early-exits-interleave.ll | 40 ++--
.../predicated-multiple-exits.ll | 192 ++++++++++++++----
.../LoopVectorize/predicated-single-exit.ll | 18 +-
5 files changed, 219 insertions(+), 85 deletions(-)
diff --git a/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp b/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp
index dd4504effe7080..c75441a3ed2729 100644
--- a/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp
+++ b/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp
@@ -3322,10 +3322,6 @@ getRecipesForUncountableExit(SmallVectorImpl<VPInstruction *> &Recipes,
} else if (match(V, m_Freeze(m_VPValue(Op1)))) {
Worklist.push_back(Op1);
Recipes.push_back(cast<VPInstruction>(V->getDefiningRecipe()));
- } else if (match(V, m_VPInstruction<VPInstruction::MaskedCond>(
- m_VPValue(Op1)))) {
- Worklist.push_back(Op1);
- Recipes.push_back(cast<VPInstruction>(V->getDefiningRecipe()));
} else
return nullptr;
}
@@ -3635,14 +3631,11 @@ bool VPlanTransforms::handleUncountableEarlyExits(
m_BranchOnCond(m_VPValue(CondOfEarlyExitingVPBB)));
assert(Matched && "Terminator must be BranchOnCond");
- // Insert the MaskedCond in the EarlyExitingVPBB so the predicator adds
- // the correct block mask.
VPBuilder EarlyExitingBuilder(EarlyExitingVPBB->getTerminator());
- auto *CondToEarlyExit = EarlyExitingBuilder.createNaryOp(
- VPInstruction::MaskedCond,
+ VPValue *CondToEarlyExit =
TrueSucc == ExitBlock
? CondOfEarlyExitingVPBB
- : EarlyExitingBuilder.createNot(CondOfEarlyExitingVPBB));
+ : EarlyExitingBuilder.createNot(CondOfEarlyExitingVPBB);
assert((isa<VPIRValue>(CondOfEarlyExitingVPBB) ||
!VPDT.properlyDominates(EarlyExitingVPBB, LatchVPBB) ||
VPDT.properlyDominates(
@@ -3656,6 +3649,26 @@ bool VPlanTransforms::handleUncountableEarlyExits(
});
}
+ if (Style == UncountableExitStyle::ReadOnly) {
+ // Lanes past the exit can be poison that the scalar loop never branched on,
+ // so freeze them to prevent branching on poison.
+ for (VPBasicBlock *VPBB : vp_rpo_plain_cfg_loop_body(HeaderVPBB)) {
+ VPRecipeBase *Term = VPBB->getTerminator();
+ if (!Term || VPBB == LatchVPBB)
+ continue;
+ Term->setOperand(0, VPBuilder(Term).createFreeze(Term->getOperand(0)));
+ }
+
+ // An exit's condition is false on any path that didn't go through its
+ // exiting block.
+ for (EarlyExitInfo &Exit : Exits) {
+ DenseMap<VPBasicBlock *, VPValue *> Defs = {
+ {HeaderVPBB, Plan.getFalse()}};
+ Defs[Exit.EarlyExitingVPBB] = Exit.CondToExit;
+ Exit.CondToExit = vputils::reconstructSSA(LatchVPBB, Defs);
+ }
+ }
+
assert(!Exits.empty() && "must have at least one early exit");
// Sort exits by RPO order to get correct program order. RPO gives a
// topological ordering of the CFG, ensuring upstream exits are checked
@@ -3790,6 +3803,10 @@ bool VPlanTransforms::handleUncountableEarlyExits(
ExitIRI->getIncomingValueForBlock(EarlyExitingVPBB);
VPValue *NewIncoming = IncomingVal;
if (!isa<VPIRValue>(IncomingVal)) {
+ DenseMap<VPBasicBlock *, VPValue *> Defs = {
+ {HeaderVPBB, Plan.getPoison(IncomingVal->getScalarType())}};
+ Defs[EarlyExitingVPBB] = IncomingVal;
+ IncomingVal = vputils::reconstructSSA(LatchVPBB, Defs);
VPBuilder EarlyExitBuilder(VectorEarlyExitVPBB);
NewIncoming = EarlyExitBuilder.createNaryOp(
VPInstruction::ExtractLane, {FirstActiveLane, IncomingVal},
diff --git a/llvm/lib/Transforms/Vectorize/VPlanVerifier.cpp b/llvm/lib/Transforms/Vectorize/VPlanVerifier.cpp
index 6fbd53d5293807..66509831d08e69 100644
--- a/llvm/lib/Transforms/Vectorize/VPlanVerifier.cpp
+++ b/llvm/lib/Transforms/Vectorize/VPlanVerifier.cpp
@@ -288,27 +288,10 @@ bool VPlanVerifier::verifyVPBasicBlock(const VPBasicBlock *VPBB) {
if (RecipeNumbering[UI] >= RecipeNumbering[&R])
continue;
} else {
- // MaskedCond may be used from blocks it don't dominate; the block
- // will be linearized and it will dominate its users after
- // linearization.
- if (match(&R, m_VPInstruction<VPInstruction::MaskedCond>()) ||
- VPDT.dominates(VPBB, UI->getParent()))
+ if (VPDT.dominates(VPBB, UI->getParent()))
continue;
}
- // Recipes in blocks with a MaskedCond may be used in exit blocks; the
- // block will be linearized and its recipes will dominate their users
- // after linearization.
- bool BlockHasMaskedCond = any_of(*VPBB, [](const VPRecipeBase &R) {
- return match(&R, m_VPInstruction<VPInstruction::MaskedCond>());
- });
- if (BlockHasMaskedCond &&
- any_of(VPBB->getPlan()->getExitBlocks(), [UI](VPIRBasicBlock *EB) {
- return is_contained(EB->getPredecessors(), UI->getParent());
- })) {
- continue;
- }
-
errs() << "Use before def!\n";
#if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP)
VPSlotTracker Tracker(VPBB->getPlan());
diff --git a/llvm/test/Transforms/LoopVectorize/predicated-early-exits-interleave.ll b/llvm/test/Transforms/LoopVectorize/predicated-early-exits-interleave.ll
index 402994588fbe6c..28742b1f36f523 100644
--- a/llvm/test/Transforms/LoopVectorize/predicated-early-exits-interleave.ll
+++ b/llvm/test/Transforms/LoopVectorize/predicated-early-exits-interleave.ll
@@ -20,14 +20,16 @@ define i64 @single_exit_in_conditional_block() {
; CHECK-NEXT: [[WIDE_LOAD2:%.*]] = load <4 x i8>, ptr [[TMP2]], align 1
; CHECK-NEXT: [[TMP1:%.*]] = icmp slt <4 x i8> [[WIDE_LOAD]], zeroinitializer
; CHECK-NEXT: [[TMP6:%.*]] = icmp slt <4 x i8> [[WIDE_LOAD2]], zeroinitializer
+; CHECK-NEXT: [[TMP20:%.*]] = freeze <4 x i1> [[TMP1]]
+; CHECK-NEXT: [[TMP21:%.*]] = freeze <4 x i1> [[TMP6]]
; CHECK-NEXT: [[GEP_B:%.*]] = getelementptr inbounds i8, ptr @B, i64 [[IV]]
; CHECK-NEXT: [[TMP10:%.*]] = getelementptr inbounds i8, ptr [[GEP_B]], i64 4
; CHECK-NEXT: [[WIDE_LOAD1:%.*]] = load <4 x i8>, ptr [[GEP_B]], align 1
; CHECK-NEXT: [[WIDE_LOAD3:%.*]] = load <4 x i8>, ptr [[TMP10]], align 1
; CHECK-NEXT: [[TMP3:%.*]] = icmp eq <4 x i8> [[WIDE_LOAD]], [[WIDE_LOAD1]]
; CHECK-NEXT: [[TMP13:%.*]] = icmp eq <4 x i8> [[WIDE_LOAD2]], [[WIDE_LOAD3]]
-; CHECK-NEXT: [[TMP4:%.*]] = select <4 x i1> [[TMP1]], <4 x i1> [[TMP3]], <4 x i1> zeroinitializer
-; CHECK-NEXT: [[TMP14:%.*]] = select <4 x i1> [[TMP6]], <4 x i1> [[TMP13]], <4 x i1> zeroinitializer
+; CHECK-NEXT: [[TMP4:%.*]] = select <4 x i1> [[TMP20]], <4 x i1> [[TMP3]], <4 x i1> zeroinitializer
+; CHECK-NEXT: [[TMP14:%.*]] = select <4 x i1> [[TMP21]], <4 x i1> [[TMP13]], <4 x i1> zeroinitializer
; CHECK-NEXT: [[TMP5:%.*]] = freeze <4 x i1> [[TMP4]]
; CHECK-NEXT: [[TMP11:%.*]] = freeze <4 x i1> [[TMP14]]
; CHECK-NEXT: [[TMP12:%.*]] = or <4 x i1> [[TMP5]], [[TMP11]]
@@ -92,6 +94,8 @@ define i64 @diamond_with_join_then_exit() {
; CHECK-NEXT: [[WIDE_LOAD1:%.*]] = load <4 x i8>, ptr [[TMP1]], align 1
; CHECK-NEXT: [[TMP14:%.*]] = icmp slt <4 x i8> [[WIDE_LOAD4]], zeroinitializer
; CHECK-NEXT: [[TMP3:%.*]] = icmp slt <4 x i8> [[WIDE_LOAD1]], zeroinitializer
+; CHECK-NEXT: [[TMP6:%.*]] = freeze <4 x i1> [[TMP14]]
+; CHECK-NEXT: [[TMP23:%.*]] = freeze <4 x i1> [[TMP3]]
; CHECK-NEXT: [[TMP2:%.*]] = getelementptr inbounds i8, ptr @C, i64 [[INDEX]]
; CHECK-NEXT: [[TMP5:%.*]] = getelementptr inbounds i8, ptr [[TMP2]], i64 4
; CHECK-NEXT: [[WIDE_LOAD3:%.*]] = load <4 x i8>, ptr [[TMP2]], align 1
@@ -104,8 +108,8 @@ define i64 @diamond_with_join_then_exit() {
; CHECK-NEXT: [[WIDE_LOAD7:%.*]] = load <4 x i8>, ptr [[TMP10]], align 1
; CHECK-NEXT: [[TMP11:%.*]] = zext <4 x i8> [[WIDE_LOAD5]] to <4 x i64>
; CHECK-NEXT: [[TMP12:%.*]] = zext <4 x i8> [[WIDE_LOAD7]] to <4 x i64>
-; CHECK-NEXT: [[PREDPHI6:%.*]] = select <4 x i1> [[TMP14]], <4 x i64> [[TMP11]], <4 x i64> [[TMP18]]
-; CHECK-NEXT: [[PREDPHI7:%.*]] = select <4 x i1> [[TMP3]], <4 x i64> [[TMP12]], <4 x i64> [[TMP7]]
+; CHECK-NEXT: [[PREDPHI6:%.*]] = select <4 x i1> [[TMP6]], <4 x i64> [[TMP11]], <4 x i64> [[TMP18]]
+; CHECK-NEXT: [[PREDPHI7:%.*]] = select <4 x i1> [[TMP23]], <4 x i64> [[TMP12]], <4 x i64> [[TMP7]]
; CHECK-NEXT: [[TMP13:%.*]] = icmp eq <4 x i64> [[PREDPHI6]], splat (i64 42)
; CHECK-NEXT: [[TMP17:%.*]] = icmp eq <4 x i64> [[PREDPHI7]], splat (i64 42)
; CHECK-NEXT: [[TMP15:%.*]] = freeze <4 x i1> [[TMP13]]
@@ -187,12 +191,16 @@ define i64 @three_early_exits() {
; CHECK-NEXT: [[WIDE_LOAD1:%.*]] = load <4 x i8>, ptr [[TMP1]], align 1
; CHECK-NEXT: [[TMP2:%.*]] = icmp slt <4 x i8> [[WIDE_LOAD]], splat (i8 -42)
; CHECK-NEXT: [[TMP3:%.*]] = icmp slt <4 x i8> [[WIDE_LOAD1]], splat (i8 -42)
-; CHECK-NEXT: [[TMP4:%.*]] = xor <4 x i1> [[TMP2]], splat (i1 true)
-; CHECK-NEXT: [[TMP5:%.*]] = xor <4 x i1> [[TMP3]], splat (i1 true)
+; CHECK-NEXT: [[TMP18:%.*]] = freeze <4 x i1> [[TMP2]]
+; CHECK-NEXT: [[TMP19:%.*]] = freeze <4 x i1> [[TMP3]]
+; CHECK-NEXT: [[TMP4:%.*]] = xor <4 x i1> [[TMP18]], splat (i1 true)
+; CHECK-NEXT: [[TMP5:%.*]] = xor <4 x i1> [[TMP19]], splat (i1 true)
; CHECK-NEXT: [[TMP6:%.*]] = icmp slt <4 x i8> [[WIDE_LOAD]], splat (i8 42)
; CHECK-NEXT: [[TMP7:%.*]] = icmp slt <4 x i8> [[WIDE_LOAD1]], splat (i8 42)
-; CHECK-NEXT: [[TMP8:%.*]] = xor <4 x i1> [[TMP6]], splat (i1 true)
-; CHECK-NEXT: [[TMP9:%.*]] = xor <4 x i1> [[TMP7]], splat (i1 true)
+; CHECK-NEXT: [[TMP39:%.*]] = freeze <4 x i1> [[TMP6]]
+; CHECK-NEXT: [[TMP49:%.*]] = freeze <4 x i1> [[TMP7]]
+; CHECK-NEXT: [[TMP8:%.*]] = xor <4 x i1> [[TMP39]], splat (i1 true)
+; CHECK-NEXT: [[TMP9:%.*]] = xor <4 x i1> [[TMP49]], splat (i1 true)
; CHECK-NEXT: [[TMP10:%.*]] = select <4 x i1> [[TMP4]], <4 x i1> [[TMP8]], <4 x i1> zeroinitializer
; CHECK-NEXT: [[TMP11:%.*]] = select <4 x i1> [[TMP5]], <4 x i1> [[TMP9]], <4 x i1> zeroinitializer
; CHECK-NEXT: [[TMP12:%.*]] = getelementptr inbounds i8, ptr @D, i64 [[IV]]
@@ -201,26 +209,26 @@ define i64 @three_early_exits() {
; CHECK-NEXT: [[WIDE_LOAD3:%.*]] = load <4 x i8>, ptr [[TMP13]], align 1
; CHECK-NEXT: [[TMP14:%.*]] = icmp eq <4 x i8> [[WIDE_LOAD]], [[WIDE_LOAD2]]
; CHECK-NEXT: [[TMP15:%.*]] = icmp eq <4 x i8> [[WIDE_LOAD1]], [[WIDE_LOAD3]]
-; CHECK-NEXT: [[TMP16:%.*]] = select <4 x i1> [[TMP10]], <4 x i1> [[TMP14]], <4 x i1> zeroinitializer
-; CHECK-NEXT: [[TMP17:%.*]] = select <4 x i1> [[TMP11]], <4 x i1> [[TMP15]], <4 x i1> zeroinitializer
-; CHECK-NEXT: [[TMP18:%.*]] = select <4 x i1> [[TMP4]], <4 x i1> [[TMP6]], <4 x i1> zeroinitializer
-; CHECK-NEXT: [[TMP19:%.*]] = select <4 x i1> [[TMP5]], <4 x i1> [[TMP7]], <4 x i1> zeroinitializer
+; CHECK-NEXT: [[TMP55:%.*]] = select <4 x i1> [[TMP4]], <4 x i1> [[TMP39]], <4 x i1> zeroinitializer
+; CHECK-NEXT: [[TMP56:%.*]] = select <4 x i1> [[TMP5]], <4 x i1> [[TMP49]], <4 x i1> zeroinitializer
; CHECK-NEXT: [[TMP20:%.*]] = getelementptr inbounds i8, ptr @C, i64 [[IV]]
; CHECK-NEXT: [[TMP21:%.*]] = getelementptr inbounds i8, ptr [[TMP20]], i64 4
; CHECK-NEXT: [[WIDE_LOAD4:%.*]] = load <4 x i8>, ptr [[TMP20]], align 1
; CHECK-NEXT: [[WIDE_LOAD5:%.*]] = load <4 x i8>, ptr [[TMP21]], align 1
; CHECK-NEXT: [[TMP22:%.*]] = icmp eq <4 x i8> [[WIDE_LOAD]], [[WIDE_LOAD4]]
; CHECK-NEXT: [[TMP23:%.*]] = icmp eq <4 x i8> [[WIDE_LOAD1]], [[WIDE_LOAD5]]
-; CHECK-NEXT: [[TMP24:%.*]] = select <4 x i1> [[TMP18]], <4 x i1> [[TMP22]], <4 x i1> zeroinitializer
-; CHECK-NEXT: [[TMP25:%.*]] = select <4 x i1> [[TMP19]], <4 x i1> [[TMP23]], <4 x i1> zeroinitializer
; CHECK-NEXT: [[TMP26:%.*]] = getelementptr inbounds i8, ptr @B, i64 [[IV]]
; CHECK-NEXT: [[TMP27:%.*]] = getelementptr inbounds i8, ptr [[TMP26]], i64 4
; CHECK-NEXT: [[WIDE_LOAD6:%.*]] = load <4 x i8>, ptr [[TMP26]], align 1
; CHECK-NEXT: [[WIDE_LOAD7:%.*]] = load <4 x i8>, ptr [[TMP27]], align 1
; CHECK-NEXT: [[TMP28:%.*]] = icmp eq <4 x i8> [[WIDE_LOAD]], [[WIDE_LOAD6]]
; CHECK-NEXT: [[TMP29:%.*]] = icmp eq <4 x i8> [[WIDE_LOAD1]], [[WIDE_LOAD7]]
-; CHECK-NEXT: [[TMP30:%.*]] = select <4 x i1> [[TMP2]], <4 x i1> [[TMP28]], <4 x i1> zeroinitializer
-; CHECK-NEXT: [[TMP31:%.*]] = select <4 x i1> [[TMP3]], <4 x i1> [[TMP29]], <4 x i1> zeroinitializer
+; CHECK-NEXT: [[TMP16:%.*]] = select <4 x i1> [[TMP10]], <4 x i1> [[TMP14]], <4 x i1> zeroinitializer
+; CHECK-NEXT: [[TMP17:%.*]] = select <4 x i1> [[TMP11]], <4 x i1> [[TMP15]], <4 x i1> zeroinitializer
+; CHECK-NEXT: [[TMP24:%.*]] = select <4 x i1> [[TMP55]], <4 x i1> [[TMP22]], <4 x i1> zeroinitializer
+; CHECK-NEXT: [[TMP25:%.*]] = select <4 x i1> [[TMP56]], <4 x i1> [[TMP23]], <4 x i1> zeroinitializer
+; CHECK-NEXT: [[TMP30:%.*]] = select <4 x i1> [[TMP18]], <4 x i1> [[TMP28]], <4 x i1> zeroinitializer
+; CHECK-NEXT: [[TMP31:%.*]] = select <4 x i1> [[TMP19]], <4 x i1> [[TMP29]], <4 x i1> zeroinitializer
; CHECK-NEXT: [[TMP32:%.*]] = select <4 x i1> [[TMP16]], <4 x i1> splat (i1 true), <4 x i1> [[TMP24]]
; CHECK-NEXT: [[TMP33:%.*]] = select <4 x i1> [[TMP17]], <4 x i1> splat (i1 true), <4 x i1> [[TMP25]]
; CHECK-NEXT: [[TMP34:%.*]] = select <4 x i1> [[TMP32]], <4 x i1> splat (i1 true), <4 x i1> [[TMP30]]
diff --git a/llvm/test/Transforms/LoopVectorize/predicated-multiple-exits.ll b/llvm/test/Transforms/LoopVectorize/predicated-multiple-exits.ll
index f00858dd1e6be3..8dfeb6e383e689 100644
--- a/llvm/test/Transforms/LoopVectorize/predicated-multiple-exits.ll
+++ b/llvm/test/Transforms/LoopVectorize/predicated-multiple-exits.ll
@@ -17,16 +17,17 @@ define i64 @diamond_with_2_early_exits() {
; CHECK-NEXT: [[TMP0:%.*]] = getelementptr inbounds i8, ptr @A, i64 [[IV]]
; CHECK-NEXT: [[WIDE_LOAD:%.*]] = load <4 x i8>, ptr [[TMP0]], align 1
; CHECK-NEXT: [[TMP1:%.*]] = icmp slt <4 x i8> [[WIDE_LOAD]], zeroinitializer
-; CHECK-NEXT: [[TMP2:%.*]] = xor <4 x i1> [[TMP1]], splat (i1 true)
+; CHECK-NEXT: [[TMP2:%.*]] = freeze <4 x i1> [[TMP1]]
+; CHECK-NEXT: [[TMP6:%.*]] = xor <4 x i1> [[TMP2]], splat (i1 true)
; CHECK-NEXT: [[TMP3:%.*]] = getelementptr inbounds i8, ptr @C, i64 [[IV]]
; CHECK-NEXT: [[WIDE_LOAD1:%.*]] = load <4 x i8>, ptr [[TMP3]], align 1
; CHECK-NEXT: [[TMP4:%.*]] = icmp eq <4 x i8> [[WIDE_LOAD]], [[WIDE_LOAD1]]
-; CHECK-NEXT: [[TMP5:%.*]] = select <4 x i1> [[TMP2]], <4 x i1> [[TMP4]], <4 x i1> zeroinitializer
; CHECK-NEXT: [[GEP_B:%.*]] = getelementptr inbounds i8, ptr @B, i64 [[IV]]
; CHECK-NEXT: [[WIDE_LOAD2:%.*]] = load <4 x i8>, ptr [[GEP_B]], align 1
; CHECK-NEXT: [[TMP7:%.*]] = zext <4 x i8> [[WIDE_LOAD2]] to <4 x i64>
; CHECK-NEXT: [[TMP8:%.*]] = icmp eq <4 x i8> [[WIDE_LOAD]], [[WIDE_LOAD2]]
-; CHECK-NEXT: [[TMP9:%.*]] = select <4 x i1> [[TMP1]], <4 x i1> [[TMP8]], <4 x i1> zeroinitializer
+; CHECK-NEXT: [[TMP5:%.*]] = select <4 x i1> [[TMP6]], <4 x i1> [[TMP4]], <4 x i1> zeroinitializer
+; CHECK-NEXT: [[TMP9:%.*]] = select <4 x i1> [[TMP2]], <4 x i1> [[TMP8]], <4 x i1> zeroinitializer
; CHECK-NEXT: [[TMP10:%.*]] = select <4 x i1> [[TMP5]], <4 x i1> splat (i1 true), <4 x i1> [[TMP9]]
; CHECK-NEXT: [[TMP11:%.*]] = freeze <4 x i1> [[TMP10]]
; CHECK-NEXT: [[CMP_A:%.*]] = call i1 @llvm.vector.reduce.or.v4i1(<4 x i1> [[TMP11]])
@@ -95,23 +96,25 @@ define i64 @three_early_exits() {
; CHECK-NEXT: [[GEP_A:%.*]] = getelementptr inbounds i8, ptr @A, i64 [[IV]]
; CHECK-NEXT: [[WIDE_LOAD:%.*]] = load <4 x i8>, ptr [[GEP_A]], align 1
; CHECK-NEXT: [[TMP1:%.*]] = icmp slt <4 x i8> [[WIDE_LOAD]], splat (i8 -42)
-; CHECK-NEXT: [[TMP2:%.*]] = xor <4 x i1> [[TMP1]], splat (i1 true)
+; CHECK-NEXT: [[TMP9:%.*]] = freeze <4 x i1> [[TMP1]]
+; CHECK-NEXT: [[TMP2:%.*]] = xor <4 x i1> [[TMP9]], splat (i1 true)
; CHECK-NEXT: [[TMP3:%.*]] = icmp slt <4 x i8> [[WIDE_LOAD]], splat (i8 42)
-; CHECK-NEXT: [[TMP4:%.*]] = xor <4 x i1> [[TMP3]], splat (i1 true)
+; CHECK-NEXT: [[TMP19:%.*]] = freeze <4 x i1> [[TMP3]]
+; CHECK-NEXT: [[TMP4:%.*]] = xor <4 x i1> [[TMP19]], splat (i1 true)
; CHECK-NEXT: [[TMP5:%.*]] = select <4 x i1> [[TMP2]], <4 x i1> [[TMP4]], <4 x i1> zeroinitializer
; CHECK-NEXT: [[TMP6:%.*]] = getelementptr inbounds i8, ptr @D, i64 [[IV]]
; CHECK-NEXT: [[WIDE_LOAD1:%.*]] = load <4 x i8>, ptr [[TMP6]], align 1
; CHECK-NEXT: [[TMP7:%.*]] = icmp eq <4 x i8> [[WIDE_LOAD]], [[WIDE_LOAD1]]
-; CHECK-NEXT: [[TMP8:%.*]] = select <4 x i1> [[TMP5]], <4 x i1> [[TMP7]], <4 x i1> zeroinitializer
-; CHECK-NEXT: [[TMP9:%.*]] = select <4 x i1> [[TMP2]], <4 x i1> [[TMP3]], <4 x i1> zeroinitializer
+; CHECK-NEXT: [[TMP21:%.*]] = select <4 x i1> [[TMP2]], <4 x i1> [[TMP19]], <4 x i1> zeroinitializer
; CHECK-NEXT: [[TMP10:%.*]] = getelementptr inbounds i8, ptr @C, i64 [[IV]]
; CHECK-NEXT: [[WIDE_LOAD2:%.*]] = load <4 x i8>, ptr [[TMP10]], align 1
; CHECK-NEXT: [[TMP11:%.*]] = icmp eq <4 x i8> [[WIDE_LOAD]], [[WIDE_LOAD2]]
-; CHECK-NEXT: [[TMP12:%.*]] = select <4 x i1> [[TMP9]], <4 x i1> [[TMP11]], <4 x i1> zeroinitializer
; CHECK-NEXT: [[TMP13:%.*]] = getelementptr inbounds i8, ptr @B, i64 [[IV]]
; CHECK-NEXT: [[WIDE_LOAD3:%.*]] = load <4 x i8>, ptr [[TMP13]], align 1
; CHECK-NEXT: [[TMP14:%.*]] = icmp eq <4 x i8> [[WIDE_LOAD]], [[WIDE_LOAD3]]
-; CHECK-NEXT: [[TMP15:%.*]] = select <4 x i1> [[TMP1]], <4 x i1> [[TMP14]], <4 x i1> zeroinitializer
+; CHECK-NEXT: [[TMP8:%.*]] = select <4 x i1> [[TMP5]], <4 x i1> [[TMP7]], <4 x i1> zeroinitializer
+; CHECK-NEXT: [[TMP12:%.*]] = select <4 x i1> [[TMP21]], <4 x i1> [[TMP11]], <4 x i1> zeroinitializer
+; CHECK-NEXT: [[TMP15:%.*]] = select <4 x i1> [[TMP9]], <4 x i1> [[TMP14]], <4 x i1> zeroinitializer
; CHECK-NEXT: [[TMP16:%.*]] = select <4 x i1> [[TMP8]], <4 x i1> splat (i1 true), <4 x i1> [[TMP12]]
; CHECK-NEXT: [[TMP17:%.*]] = select <4 x i1> [[TMP16]], <4 x i1> splat (i1 true), <4 x i1> [[TMP15]]
; CHECK-NEXT: [[TMP18:%.*]] = freeze <4 x i1> [[TMP17]]
@@ -193,22 +196,24 @@ define i64 @nested_diamond_inner_exits() {
; CHECK-NEXT: [[TMP0:%.*]] = getelementptr inbounds i8, ptr @A, i64 [[IV]]
; CHECK-NEXT: [[WIDE_LOAD:%.*]] = load <4 x i8>, ptr [[TMP0]], align 1
; CHECK-NEXT: [[TMP1:%.*]] = icmp slt <4 x i8> [[WIDE_LOAD]], zeroinitializer
-; CHECK-NEXT: [[TMP2:%.*]] = xor <4 x i1> [[TMP1]], splat (i1 true)
+; CHECK-NEXT: [[TMP2:%.*]] = freeze <4 x i1> [[TMP1]]
+; CHECK-NEXT: [[TMP6:%.*]] = xor <4 x i1> [[TMP2]], splat (i1 true)
; CHECK-NEXT: [[TMP3:%.*]] = getelementptr inbounds i8, ptr @D, i64 [[IV]]
; CHECK-NEXT: [[WIDE_LOAD1:%.*]] = load <4 x i8>, ptr [[TMP3]], align 1
; CHECK-NEXT: [[TMP4:%.*]] = icmp eq <4 x i8> [[WIDE_LOAD]], [[WIDE_LOAD1]]
-; CHECK-NEXT: [[TMP5:%.*]] = select <4 x i1> [[TMP2]], <4 x i1> [[TMP4]], <4 x i1> zeroinitializer
; CHECK-NEXT: [[GEP_B:%.*]] = getelementptr inbounds i8, ptr @B, i64 [[IV]]
; CHECK-NEXT: [[WIDE_LOAD2:%.*]] = load <4 x i8>, ptr [[GEP_B]], align 1
; CHECK-NEXT: [[TMP7:%.*]] = icmp slt <4 x i8> [[WIDE_LOAD2]], zeroinitializer
-; CHECK-NEXT: [[TMP8:%.*]] = xor <4 x i1> [[TMP7]], splat (i1 true)
-; CHECK-NEXT: [[TMP9:%.*]] = select <4 x i1> [[TMP1]], <4 x i1> [[TMP8]], <4 x i1> zeroinitializer
+; CHECK-NEXT: [[TMP8:%.*]] = freeze <4 x i1> [[TMP7]]
+; CHECK-NEXT: [[TMP9:%.*]] = xor <4 x i1> [[TMP8]], splat (i1 true)
+; CHECK-NEXT: [[TMP19:%.*]] = select <4 x i1> [[TMP2]], <4 x i1> [[TMP9]], <4 x i1> zeroinitializer
; CHECK-NEXT: [[TMP10:%.*]] = getelementptr inbounds i8, ptr @C, i64 [[IV]]
; CHECK-NEXT: [[WIDE_LOAD3:%.*]] = load <4 x i8>, ptr [[TMP10]], align 1
; CHECK-NEXT: [[TMP11:%.*]] = icmp eq <4 x i8> [[WIDE_LOAD]], [[WIDE_LOAD3]]
-; CHECK-NEXT: [[TMP12:%.*]] = select <4 x i1> [[TMP9]], <4 x i1> [[TMP11]], <4 x i1> zeroinitializer
-; CHECK-NEXT: [[TMP13:%.*]] = select <4 x i1> [[TMP1]], <4 x i1> [[TMP7]], <4 x i1> zeroinitializer
+; CHECK-NEXT: [[TMP13:%.*]] = select <4 x i1> [[TMP2]], <4 x i1> [[TMP8]], <4 x i1> zeroinitializer
; CHECK-NEXT: [[TMP14:%.*]] = icmp eq <4 x i8> [[WIDE_LOAD]], [[WIDE_LOAD2]]
+; CHECK-NEXT: [[TMP5:%.*]] = select <4 x i1> [[TMP6]], <4 x i1> [[TMP4]], <4 x i1> zeroinitializer
+; CHECK-NEXT: [[TMP12:%.*]] = select <4 x i1> [[TMP19]], <4 x i1> [[TMP11]], <4 x i1> zeroinitializer
; CHECK-NEXT: [[TMP15:%.*]] = select <4 x i1> [[TMP13]], <4 x i1> [[TMP14]], <4 x i1> zeroinitializer
; CHECK-NEXT: [[TMP16:%.*]] = select <4 x i1> [[TMP5]], <4 x i1> splat (i1 true), <4 x i1> [[TMP12]]
; CHECK-NEXT: [[TMP17:%.*]] = select <4 x i1> [[TMP16]], <4 x i1> splat (i1 true), <4 x i1> [[TMP15]]
@@ -294,18 +299,19 @@ define i64 @chain_of_3_exits() {
; CHECK-NEXT: [[TMP0:%.*]] = getelementptr inbounds i8, ptr @A, i64 [[IV]]
; CHECK-NEXT: [[WIDE_LOAD:%.*]] = load <4 x i8>, ptr [[TMP0]], align 1
; CHECK-NEXT: [[TMP1:%.*]] = icmp slt <4 x i8> [[WIDE_LOAD]], zeroinitializer
+; CHECK-NEXT: [[TMP2:%.*]] = freeze <4 x i1> [[TMP1]]
; CHECK-NEXT: [[GEP_B:%.*]] = getelementptr inbounds i8, ptr @B, i64 [[IV]]
; CHECK-NEXT: [[WIDE_LOAD1:%.*]] = load <4 x i8>, ptr [[GEP_B]], align 1
; CHECK-NEXT: [[TMP3:%.*]] = icmp eq <4 x i8> [[WIDE_LOAD]], [[WIDE_LOAD1]]
-; CHECK-NEXT: [[TMP4:%.*]] = select <4 x i1> [[TMP1]], <4 x i1> [[TMP3]], <4 x i1> zeroinitializer
; CHECK-NEXT: [[GEP_C:%.*]] = getelementptr inbounds i8, ptr @C, i64 [[IV]]
; CHECK-NEXT: [[WIDE_LOAD2:%.*]] = load <4 x i8>, ptr [[GEP_C]], align 1
; CHECK-NEXT: [[TMP6:%.*]] = icmp eq <4 x i8> [[WIDE_LOAD]], [[WIDE_LOAD2]]
-; CHECK-NEXT: [[TMP7:%.*]] = select <4 x i1> [[TMP1]], <4 x i1> [[TMP6]], <4 x i1> zeroinitializer
; CHECK-NEXT: [[TMP8:%.*]] = getelementptr inbounds i8, ptr @D, i64 [[IV]]
; CHECK-NEXT: [[WIDE_LOAD3:%.*]] = load <4 x i8>, ptr [[TMP8]], align 1
; CHECK-NEXT: [[TMP9:%.*]] = icmp eq <4 x i8> [[WIDE_LOAD]], [[WIDE_LOAD3]]
-; CHECK-NEXT: [[TMP10:%.*]] = select <4 x i1> [[TMP1]], <4 x i1> [[TMP9]], <4 x i1> zeroinitializer
+; CHECK-NEXT: [[TMP10:%.*]] = select <4 x i1> [[TMP2]], <4 x i1> [[TMP9]], <4 x i1> zeroinitializer
+; CHECK-NEXT: [[TMP7:%.*]] = select <4 x i1> [[TMP2]], <4 x i1> [[TMP6]], <4 x i1> zeroinitializer
+; CHECK-NEXT: [[TMP4:%.*]] = select <4 x i1> [[TMP2]], <4 x i1> [[TMP3]], <4 x i1> zeroinitializer
; CHECK-NEXT: [[TMP11:%.*]] = select <4 x i1> [[TMP4]], <4 x i1> splat (i1 true), <4 x i1> [[TMP7]]
; CHECK-NEXT: [[TMP12:%.*]] = select <4 x i1> [[TMP11]], <4 x i1> splat (i1 true), <4 x i1> [[TMP10]]
; CHECK-NEXT: [[TMP13:%.*]] = freeze <4 x i1> [[TMP12]]
@@ -383,23 +389,25 @@ define i64 @four_exits_2x2_diamond() {
; CHECK-NEXT: [[TMP0:%.*]] = getelementptr inbounds i8, ptr @A, i64 [[IV]]
; CHECK-NEXT: [[WIDE_LOAD:%.*]] = load <4 x i8>, ptr [[TMP0]], align 1
; CHECK-NEXT: [[TMP1:%.*]] = icmp slt <4 x i8> [[WIDE_LOAD]], zeroinitializer
-; CHECK-NEXT: [[TMP2:%.*]] = xor <4 x i1> [[TMP1]], splat (i1 true)
+; CHECK-NEXT: [[TMP2:%.*]] = freeze <4 x i1> [[TMP1]]
+; CHECK-NEXT: [[TMP6:%.*]] = xor <4 x i1> [[TMP2]], splat (i1 true)
; CHECK-NEXT: [[TMP3:%.*]] = getelementptr inbounds i8, ptr @C, i64 [[IV]]
; CHECK-NEXT: [[WIDE_LOAD1:%.*]] = load <4 x i8>, ptr [[TMP3]], align 1
; CHECK-NEXT: [[TMP4:%.*]] = icmp eq <4 x i8> [[WIDE_LOAD]], [[WIDE_LOAD1]]
-; CHECK-NEXT: [[TMP5:%.*]] = select <4 x i1> [[TMP2]], <4 x i1> [[TMP4]], <4 x i1> zeroinitializer
; CHECK-NEXT: [[GEP_B:%.*]] = getelementptr inbounds i8, ptr @B, i64 [[IV]]
; CHECK-NEXT: [[WIDE_LOAD2:%.*]] = load <4 x i8>, ptr [[GEP_B]], align 1
; CHECK-NEXT: [[TMP7:%.*]] = icmp eq <4 x i8> [[WIDE_LOAD]], [[WIDE_LOAD2]]
-; CHECK-NEXT: [[TMP8:%.*]] = select <4 x i1> [[TMP1]], <4 x i1> [[TMP7]], <4 x i1> zeroinitializer
+; CHECK-NEXT: [[TMP5:%.*]] = select <4 x i1> [[TMP6]], <4 x i1> [[TMP4]], <4 x i1> zeroinitializer
+; CHECK-NEXT: [[TMP8:%.*]] = select <4 x i1> [[TMP2]], <4 x i1> [[TMP7]], <4 x i1> zeroinitializer
; CHECK-NEXT: [[TMP9:%.*]] = getelementptr inbounds i8, ptr @D, i64 [[IV]]
; CHECK-NEXT: [[WIDE_LOAD3:%.*]] = load <4 x i8>, ptr [[TMP9]], align 1
; CHECK-NEXT: [[TMP10:%.*]] = icmp slt <4 x i8> [[WIDE_LOAD3]], zeroinitializer
-; CHECK-NEXT: [[TMP11:%.*]] = xor <4 x i1> [[TMP10]], splat (i1 true)
+; CHECK-NEXT: [[TMP20:%.*]] = freeze <4 x i1> [[TMP10]]
+; CHECK-NEXT: [[TMP25:%.*]] = xor <4 x i1> [[TMP20]], splat (i1 true)
; CHECK-NEXT: [[TMP12:%.*]] = icmp ne <4 x i8> [[WIDE_LOAD]], [[WIDE_LOAD3]]
-; CHECK-NEXT: [[TMP13:%.*]] = select <4 x i1> [[TMP11]], <4 x i1> [[TMP12]], <4 x i1> zeroinitializer
; CHECK-NEXT: [[TMP14:%.*]] = icmp eq <4 x i8> [[WIDE_LOAD]], [[WIDE_LOAD3]]
-; CHECK-NEXT: [[TMP15:%.*]] = select <4 x i1> [[TMP10]], <4 x i1> [[TMP14]], <4 x i1> zeroinitializer
+; CHECK-NEXT: [[TMP13:%.*]] = select <4 x i1> [[TMP25]], <4 x i1> [[TMP12]], <4 x i1> zeroinitializer
+; CHECK-NEXT: [[TMP15:%.*]] = select <4 x i1> [[TMP20]], <4 x i1> [[TMP14]], <4 x i1> zeroinitializer
; CHECK-NEXT: [[TMP16:%.*]] = select <4 x i1> [[TMP5]], <4 x i1> splat (i1 true), <4 x i1> [[TMP8]]
; CHECK-NEXT: [[TMP17:%.*]] = select <4 x i1> [[TMP16]], <4 x i1> splat (i1 true), <4 x i1> [[TMP13]]
; CHECK-NEXT: [[TMP18:%.*]] = select <4 x i1> [[TMP17]], <4 x i1> splat (i1 true), <4 x i1> [[TMP15]]
@@ -493,11 +501,12 @@ define i64 @diamond_merge_then_exit_with_phi_liveout() {
; CHECK-NEXT: [[TMP0:%.*]] = getelementptr inbounds i8, ptr @A, i64 [[INDEX]]
; CHECK-NEXT: [[WIDE_LOAD:%.*]] = load <4 x i8>, ptr [[TMP0]], align 1
; CHECK-NEXT: [[TMP1:%.*]] = icmp slt <4 x i8> [[WIDE_LOAD]], zeroinitializer
+; CHECK-NEXT: [[TMP11:%.*]] = freeze <4 x i1> [[TMP1]]
; CHECK-NEXT: [[TMP2:%.*]] = getelementptr inbounds i8, ptr @C, i64 [[INDEX]]
; CHECK-NEXT: [[WIDE_LOAD1:%.*]] = load <4 x i8>, ptr [[TMP2]], align 1
; CHECK-NEXT: [[TMP3:%.*]] = getelementptr inbounds i8, ptr @B, i64 [[INDEX]]
; CHECK-NEXT: [[WIDE_LOAD2:%.*]] = load <4 x i8>, ptr [[TMP3]], align 1
-; CHECK-NEXT: [[PREDPHI:%.*]] = select <4 x i1> [[TMP1]], <4 x i8> [[WIDE_LOAD2]], <4 x i8> [[WIDE_LOAD1]]
+; CHECK-NEXT: [[PREDPHI:%.*]] = select <4 x i1> [[TMP11]], <4 x i8> [[WIDE_LOAD2]], <4 x i8> [[WIDE_LOAD1]]
; CHECK-NEXT: [[TMP4:%.*]] = icmp eq <4 x i8> [[WIDE_LOAD]], [[PREDPHI]]
; CHECK-NEXT: [[TMP5:%.*]] = freeze <4 x i1> [[TMP4]]
; CHECK-NEXT: [[TMP6:%.*]] = call i1 @llvm.vector.reduce.or.v4i1(<4 x i1> [[TMP5]])
@@ -511,7 +520,7 @@ define i64 @diamond_merge_then_exit_with_phi_liveout() {
; CHECK: [[VECTOR_EARLY_EXIT]]:
; CHECK-NEXT: [[TMP8:%.*]] = zext <4 x i8> [[WIDE_LOAD1]] to <4 x i64>
; CHECK-NEXT: [[TMP9:%.*]] = zext <4 x i8> [[WIDE_LOAD2]] to <4 x i64>
-; CHECK-NEXT: [[PREDPHI3:%.*]] = select <4 x i1> [[TMP1]], <4 x i64> [[TMP9]], <4 x i64> [[TMP8]]
+; CHECK-NEXT: [[PREDPHI3:%.*]] = select <4 x i1> [[TMP11]], <4 x i64> [[TMP9]], <4 x i64> [[TMP8]]
; CHECK-NEXT: [[FIRST_ACTIVE_LANE:%.*]] = call i64 @llvm.experimental.cttz.elts.i64.v4i1(<4 x i1> [[TMP5]], i1 false)
; CHECK-NEXT: [[TMP10:%.*]] = extractelement <4 x i64> [[PREDPHI3]], i64 [[FIRST_ACTIVE_LANE]]
; CHECK-NEXT: br label %[[LOOP_END]]
@@ -577,11 +586,12 @@ define i64 @diamond_exits_overlapping_conditions() {
; CHECK-NEXT: [[GEP_C:%.*]] = getelementptr inbounds i8, ptr @C, i64 [[IV]]
; CHECK-NEXT: [[WIDE_LOAD2:%.*]] = load <4 x i8>, ptr [[GEP_C]], align 1
; CHECK-NEXT: [[TMP3:%.*]] = icmp slt <4 x i8> [[WIDE_LOAD]], zeroinitializer
-; CHECK-NEXT: [[TMP4:%.*]] = xor <4 x i1> [[TMP3]], splat (i1 true)
+; CHECK-NEXT: [[TMP4:%.*]] = freeze <4 x i1> [[TMP3]]
+; CHECK-NEXT: [[TMP11:%.*]] = xor <4 x i1> [[TMP4]], splat (i1 true)
; CHECK-NEXT: [[TMP5:%.*]] = icmp eq <4 x i8> [[WIDE_LOAD]], [[WIDE_LOAD2]]
-; CHECK-NEXT: [[TMP6:%.*]] = select <4 x i1> [[TMP4]], <4 x i1> [[TMP5]], <4 x i1> zeroinitializer
; CHECK-NEXT: [[TMP7:%.*]] = icmp eq <4 x i8> [[WIDE_LOAD]], [[WIDE_LOAD1]]
-; CHECK-NEXT: [[TMP8:%.*]] = select <4 x i1> [[TMP3]], <4 x i1> [[TMP7]], <4 x i1> zeroinitializer
+; CHECK-NEXT: [[TMP6:%.*]] = select <4 x i1> [[TMP11]], <4 x i1> [[TMP5]], <4 x i1> zeroinitializer
+; CHECK-NEXT: [[TMP8:%.*]] = select <4 x i1> [[TMP4]], <4 x i1> [[TMP7]], <4 x i1> zeroinitializer
; CHECK-NEXT: [[TMP9:%.*]] = select <4 x i1> [[TMP6]], <4 x i1> splat (i1 true), <4 x i1> [[TMP8]]
; CHECK-NEXT: [[TMP10:%.*]] = freeze <4 x i1> [[TMP9]]
; CHECK-NEXT: [[COND:%.*]] = call i1 @llvm.vector.reduce.or.v4i1(<4 x i1> [[TMP10]])
@@ -653,10 +663,11 @@ define i64 @exit_from_merge_of_exit_fallthrough_and_bypass() {
; CHECK-NEXT: [[TMP0:%.*]] = getelementptr inbounds i8, ptr @A, i64 [[IV]]
; CHECK-NEXT: [[WIDE_LOAD:%.*]] = load <4 x i8>, ptr [[TMP0]], align 1
; CHECK-NEXT: [[TMP1:%.*]] = icmp slt <4 x i8> [[WIDE_LOAD]], zeroinitializer
+; CHECK-NEXT: [[TMP2:%.*]] = freeze <4 x i1> [[TMP1]]
; CHECK-NEXT: [[GEP_B:%.*]] = getelementptr inbounds i8, ptr @B, i64 [[IV]]
; CHECK-NEXT: [[WIDE_LOAD1:%.*]] = load <4 x i8>, ptr [[GEP_B]], align 1
; CHECK-NEXT: [[TMP3:%.*]] = icmp eq <4 x i8> [[WIDE_LOAD]], [[WIDE_LOAD1]]
-; CHECK-NEXT: [[TMP4:%.*]] = select <4 x i1> [[TMP1]], <4 x i1> [[TMP3]], <4 x i1> zeroinitializer
+; CHECK-NEXT: [[TMP4:%.*]] = select <4 x i1> [[TMP2]], <4 x i1> [[TMP3]], <4 x i1> zeroinitializer
; CHECK-NEXT: [[GEP_C:%.*]] = getelementptr inbounds i8, ptr @C, i64 [[IV]]
; CHECK-NEXT: [[WIDE_LOAD2:%.*]] = load <4 x i8>, ptr [[GEP_C]], align 1
; CHECK-NEXT: [[TMP6:%.*]] = icmp eq <4 x i8> [[WIDE_LOAD]], [[WIDE_LOAD2]]
@@ -724,7 +735,13 @@ define i32 @diamond_exit_poison_from_speculated_branch() {
; CHECK: [[VECTOR_PH]]:
; CHECK-NEXT: br label %[[VECTOR_BODY:.*]]
; CHECK: [[VECTOR_BODY]]:
-; CHECK-NEXT: [[TMP0:%.*]] = freeze <4 x i1> splat (i1 true)
+; CHECK-NEXT: [[TMP5:%.*]] = freeze <4 x i1> <i1 true, i1 true, i1 false, i1 false>
+; CHECK-NEXT: [[TMP6:%.*]] = xor <4 x i1> [[TMP5]], splat (i1 true)
+; CHECK-NEXT: [[PREDPHI:%.*]] = select <4 x i1> [[TMP5]], <4 x i32> <i32 10, i32 11, i32 12, i32 13>, <4 x i32> <i32 poison, i32 poison, i32 1, i32 2>
+; CHECK-NEXT: [[TMP7:%.*]] = select <4 x i1> [[TMP6]], <4 x i1> <i1 poison, i1 poison, i1 false, i1 false>, <4 x i1> zeroinitializer
+; CHECK-NEXT: [[TMP3:%.*]] = icmp ult <4 x i32> [[PREDPHI]], splat (i32 12)
+; CHECK-NEXT: [[TMP4:%.*]] = select <4 x i1> [[TMP7]], <4 x i1> splat (i1 true), <4 x i1> [[TMP3]]
+; CHECK-NEXT: [[TMP0:%.*]] = freeze <4 x i1> [[TMP4]]
; CHECK-NEXT: [[TMP1:%.*]] = call i1 @llvm.vector.reduce.or.v4i1(<4 x i1> [[TMP0]])
; CHECK-NEXT: br i1 [[TMP1]], label %[[VECTOR_EARLY_EXIT_CHECK:.*]], label %[[MIDDLE_BLOCK:.*]]
; CHECK: [[MIDDLE_BLOCK]]:
@@ -733,10 +750,10 @@ define i32 @diamond_exit_poison_from_speculated_branch() {
; CHECK-NEXT: br label %[[LOOP_END1:.*]]
; CHECK: [[VECTOR_EARLY_EXIT_CHECK]]:
; CHECK-NEXT: [[FIRST_ACTIVE_LANE:%.*]] = call i64 @llvm.experimental.cttz.elts.i64.v4i1(<4 x i1> [[TMP0]], i1 false)
-; CHECK-NEXT: [[TMP2:%.*]] = extractelement <4 x i1> zeroinitializer, i64 [[FIRST_ACTIVE_LANE]]
+; CHECK-NEXT: [[TMP2:%.*]] = extractelement <4 x i1> [[TMP7]], i64 [[FIRST_ACTIVE_LANE]]
; CHECK-NEXT: br i1 [[TMP2]], label %[[VECTOR_EARLY_EXIT_0:.*]], label %[[VECTOR_EARLY_EXIT_1:.*]]
; CHECK: [[VECTOR_EARLY_EXIT_1]]:
-; CHECK-NEXT: [[TMP3:%.*]] = extractelement <4 x i32> <i32 10, i32 11, i32 1, i32 2>, i64 [[FIRST_ACTIVE_LANE]]
+; CHECK-NEXT: [[TMP8:%.*]] = extractelement <4 x i32> [[PREDPHI]], i64 [[FIRST_ACTIVE_LANE]]
; CHECK-NEXT: br label %[[LOOP_END1]]
; CHECK: [[VECTOR_EARLY_EXIT_0]]:
; CHECK-NEXT: br label %[[UNREACHABLE_EXIT:.*]]
@@ -744,7 +761,7 @@ define i32 @diamond_exit_poison_from_speculated_branch() {
; CHECK-NEXT: call void @llvm.trap()
; CHECK-NEXT: unreachable
; CHECK: [[LOOP_END1]]:
-; CHECK-NEXT: [[RETVAL:%.*]] = phi i32 [ [[TMP3]], %[[VECTOR_EARLY_EXIT_1]] ], [ -1, %[[LOOP_END]] ]
+; CHECK-NEXT: [[RETVAL:%.*]] = phi i32 [ [[TMP8]], %[[VECTOR_EARLY_EXIT_1]] ], [ -1, %[[LOOP_END]] ]
; CHECK-NEXT: ret i32 [[RETVAL]]
;
entry:
@@ -794,7 +811,12 @@ define i32 @diamond_exit_poison_cond_second() {
; CHECK: [[VECTOR_PH]]:
; CHECK-NEXT: br label %[[VECTOR_BODY:.*]]
; CHECK: [[VECTOR_BODY]]:
-; CHECK-NEXT: [[TMP0:%.*]] = freeze <4 x i1> <i1 true, i1 false, i1 false, i1 false>
+; CHECK-NEXT: [[TMP5:%.*]] = freeze <4 x i1> <i1 true, i1 true, i1 false, i1 false>
+; CHECK-NEXT: [[TMP6:%.*]] = xor <4 x i1> [[TMP5]], splat (i1 true)
+; CHECK-NEXT: [[TMP7:%.*]] = select <4 x i1> [[TMP6]], <4 x i1> <i1 poison, i1 poison, i1 false, i1 false>, <4 x i1> zeroinitializer
+; CHECK-NEXT: [[TMP8:%.*]] = select <4 x i1> [[TMP5]], <4 x i1> <i1 true, i1 false, i1 false, i1 false>, <4 x i1> zeroinitializer
+; CHECK-NEXT: [[TMP4:%.*]] = select <4 x i1> [[TMP7]], <4 x i1> splat (i1 true), <4 x i1> [[TMP8]]
+; CHECK-NEXT: [[TMP0:%.*]] = freeze <4 x i1> [[TMP4]]
; CHECK-NEXT: [[TMP1:%.*]] = call i1 @llvm.vector.reduce.or.v4i1(<4 x i1> [[TMP0]])
; CHECK-NEXT: br i1 [[TMP1]], label %[[VECTOR_EARLY_EXIT_CHECK:.*]], label %[[MIDDLE_BLOCK:.*]]
; CHECK: [[MIDDLE_BLOCK]]:
@@ -803,7 +825,7 @@ define i32 @diamond_exit_poison_cond_second() {
; CHECK-NEXT: br label %[[LOOP_END1:.*]]
; CHECK: [[VECTOR_EARLY_EXIT_CHECK]]:
; CHECK-NEXT: [[FIRST_ACTIVE_LANE:%.*]] = call i64 @llvm.experimental.cttz.elts.i64.v4i1(<4 x i1> [[TMP0]], i1 false)
-; CHECK-NEXT: [[TMP2:%.*]] = extractelement <4 x i1> zeroinitializer, i64 [[FIRST_ACTIVE_LANE]]
+; CHECK-NEXT: [[TMP2:%.*]] = extractelement <4 x i1> [[TMP7]], i64 [[FIRST_ACTIVE_LANE]]
; CHECK-NEXT: br i1 [[TMP2]], label %[[VECTOR_EARLY_EXIT_0:.*]], label %[[VECTOR_EARLY_EXIT_1:.*]]
; CHECK: [[VECTOR_EARLY_EXIT_1]]:
; CHECK-NEXT: [[TMP3:%.*]] = extractelement <4 x i32> <i32 10, i32 11, i32 12, i32 13>, i64 [[FIRST_ACTIVE_LANE]]
@@ -851,3 +873,101 @@ loop.end:
ret i32 %retval
}
+
+; The lane that takes the early exit also goes through after.exit in the
+; vector loop, where %c is poison if %l.C is out of range. The exit condition
+; reaches the latch through blends in %d and %e, which also merge in %other
+; and select on %c, so %c must be frozen or the exit condition is poison on
+; the exiting lane.
+define i64 @exit_cond_merged_after_poison_branch() {
+; CHECK-LABEL: define i64 @exit_cond_merged_after_poison_branch() {
+; CHECK-NEXT: [[ENTRY:.*:]]
+; CHECK-NEXT: br label %[[VECTOR_PH:.*]]
+; CHECK: [[VECTOR_PH]]:
+; CHECK-NEXT: br label %[[VECTOR_BODY:.*]]
+; CHECK: [[VECTOR_BODY]]:
+; CHECK-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY_INTERIM:.*]] ]
+; CHECK-NEXT: [[VEC_IND:%.*]] = phi <4 x i64> [ <i64 0, i64 1, i64 2, i64 3>, %[[VECTOR_PH]] ], [ [[VEC_IND_NEXT:%.*]], %[[VECTOR_BODY_INTERIM]] ]
+; CHECK-NEXT: [[TMP0:%.*]] = getelementptr inbounds i8, ptr @A, i64 [[INDEX]]
+; CHECK-NEXT: [[WIDE_LOAD:%.*]] = load <4 x i8>, ptr [[TMP0]], align 1
+; CHECK-NEXT: [[TMP1:%.*]] = icmp slt <4 x i8> [[WIDE_LOAD]], zeroinitializer
+; CHECK-NEXT: [[TMP19:%.*]] = freeze <4 x i1> [[TMP1]]
+; CHECK-NEXT: [[TMP20:%.*]] = xor <4 x i1> [[TMP19]], splat (i1 true)
+; CHECK-NEXT: [[TMP2:%.*]] = getelementptr inbounds i8, ptr @C, i64 [[INDEX]]
+; CHECK-NEXT: [[WIDE_LOAD1:%.*]] = load <4 x i8>, ptr [[TMP2]], align 1
+; CHECK-NEXT: [[TMP3:%.*]] = icmp eq <4 x i8> [[WIDE_LOAD]], [[WIDE_LOAD1]]
+; CHECK-NEXT: [[TMP21:%.*]] = getelementptr inbounds i8, ptr @B, i64 [[INDEX]]
+; CHECK-NEXT: [[WIDE_LOAD2:%.*]] = load <4 x i8>, ptr [[TMP21]], align 1
+; CHECK-NEXT: [[TMP23:%.*]] = lshr <4 x i8> [[WIDE_LOAD2]], [[WIDE_LOAD1]]
+; CHECK-NEXT: [[TMP8:%.*]] = icmp eq <4 x i8> [[TMP23]], [[WIDE_LOAD]]
+; CHECK-NEXT: [[TMP9:%.*]] = freeze <4 x i1> [[TMP8]]
+; CHECK-NEXT: [[TMP10:%.*]] = icmp eq <4 x i8> [[WIDE_LOAD]], splat (i8 -7)
+; CHECK-NEXT: [[TMP11:%.*]] = freeze <4 x i1> [[TMP10]]
+; CHECK-NEXT: [[TMP12:%.*]] = xor <4 x i1> [[TMP9]], splat (i1 true)
+; CHECK-NEXT: [[TMP13:%.*]] = select <4 x i1> [[TMP20]], <4 x i1> [[TMP12]], <4 x i1> zeroinitializer
+; CHECK-NEXT: [[TMP14:%.*]] = select <4 x i1> [[TMP13]], <4 x i1> [[TMP3]], <4 x i1> zeroinitializer
+; CHECK-NEXT: [[TMP15:%.*]] = select <4 x i1> [[TMP19]], <4 x i1> [[TMP11]], <4 x i1> zeroinitializer
+; CHECK-NEXT: [[TMP16:%.*]] = select <4 x i1> [[TMP20]], <4 x i1> [[TMP9]], <4 x i1> zeroinitializer
+; CHECK-NEXT: [[TMP17:%.*]] = or <4 x i1> [[TMP15]], [[TMP16]]
+; CHECK-NEXT: [[TMP18:%.*]] = select <4 x i1> [[TMP16]], <4 x i1> [[TMP3]], <4 x i1> zeroinitializer
+; CHECK-NEXT: [[PREDPHI:%.*]] = select <4 x i1> [[TMP17]], <4 x i1> [[TMP18]], <4 x i1> [[TMP14]]
+; CHECK-NEXT: [[TMP4:%.*]] = freeze <4 x i1> [[PREDPHI]]
+; CHECK-NEXT: [[TMP5:%.*]] = call i1 @llvm.vector.reduce.or.v4i1(<4 x i1> [[TMP4]])
+; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 4
+; CHECK-NEXT: [[TMP6:%.*]] = icmp eq i64 [[INDEX_NEXT]], 64
+; CHECK-NEXT: [[VEC_IND_NEXT]] = add <4 x i64> [[VEC_IND]], splat (i64 4)
+; CHECK-NEXT: br i1 [[TMP5]], label %[[VECTOR_EARLY_EXIT:.*]], label %[[VECTOR_BODY_INTERIM]]
+; CHECK: [[VECTOR_BODY_INTERIM]]:
+; CHECK-NEXT: br i1 [[TMP6]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP10:![0-9]+]]
+; CHECK: [[MIDDLE_BLOCK]]:
+; CHECK-NEXT: br label %[[LOOP_END:.*]]
+; CHECK: [[VECTOR_EARLY_EXIT]]:
+; CHECK-NEXT: [[TMP7:%.*]] = call i64 @llvm.experimental.cttz.elts.i64.v4i1(<4 x i1> [[TMP4]], i1 false)
+; CHECK-NEXT: [[TMP22:%.*]] = extractelement <4 x i64> [[VEC_IND]], i64 [[TMP7]]
+; CHECK-NEXT: br label %[[LOOP_END]]
+; CHECK: [[LOOP_END]]:
+; CHECK-NEXT: [[RETVAL:%.*]] = phi i64 [ [[TMP22]], %[[VECTOR_EARLY_EXIT]] ], [ -1, %[[MIDDLE_BLOCK]] ]
+; CHECK-NEXT: ret i64 [[RETVAL]]
+;
+entry:
+ br label %loop.header
+
+loop.header:
+ %iv = phi i64 [ %iv.next, %loop.latch ], [ 0, %entry ]
+ %gep.A = getelementptr inbounds i8, ptr @A, i64 %iv
+ %l.A = load i8, ptr %gep.A
+ %c.a = icmp slt i8 %l.A, 0
+ br i1 %c.a, label %other, label %exiting
+
+exiting:
+ %gep.C = getelementptr inbounds i8, ptr @C, i64 %iv
+ %l.C = load i8, ptr %gep.C
+ %cmp = icmp eq i8 %l.A, %l.C
+ br i1 %cmp, label %loop.end, label %after.exit
+
+after.exit:
+ %gep.B = getelementptr inbounds i8, ptr @B, i64 %iv
+ %l.B = load i8, ptr %gep.B
+ %shr = lshr i8 %l.B, %l.C
+ %c = icmp eq i8 %shr, %l.A
+ br i1 %c, label %d, label %e
+
+other:
+ %x = icmp eq i8 %l.A, -7
+ br i1 %x, label %d, label %e
+
+d:
+ br label %loop.latch
+
+e:
+ br label %loop.latch
+
+loop.latch:
+ %iv.next = add i64 %iv, 1
+ %exitcond = icmp ne i64 %iv.next, 64
+ br i1 %exitcond, label %loop.header, label %loop.end
+
+loop.end:
+ %retval = phi i64 [ %iv, %exiting ], [ -1, %loop.latch ]
+ ret i64 %retval
+}
diff --git a/llvm/test/Transforms/LoopVectorize/predicated-single-exit.ll b/llvm/test/Transforms/LoopVectorize/predicated-single-exit.ll
index 91531c49cdff6c..592cb66a4d1231 100644
--- a/llvm/test/Transforms/LoopVectorize/predicated-single-exit.ll
+++ b/llvm/test/Transforms/LoopVectorize/predicated-single-exit.ll
@@ -16,10 +16,11 @@ define i64 @single_exit_in_conditional_block() {
; CHECK-NEXT: [[TMP0:%.*]] = getelementptr inbounds i8, ptr @A, i64 [[IV]]
; CHECK-NEXT: [[WIDE_LOAD:%.*]] = load <4 x i8>, ptr [[TMP0]], align 1
; CHECK-NEXT: [[TMP1:%.*]] = icmp slt <4 x i8> [[WIDE_LOAD]], zeroinitializer
+; CHECK-NEXT: [[TMP2:%.*]] = freeze <4 x i1> [[TMP1]]
; CHECK-NEXT: [[GEP_B:%.*]] = getelementptr inbounds i8, ptr @B, i64 [[IV]]
; CHECK-NEXT: [[WIDE_LOAD1:%.*]] = load <4 x i8>, ptr [[GEP_B]], align 1
; CHECK-NEXT: [[TMP3:%.*]] = icmp eq <4 x i8> [[WIDE_LOAD]], [[WIDE_LOAD1]]
-; CHECK-NEXT: [[TMP4:%.*]] = select <4 x i1> [[TMP1]], <4 x i1> [[TMP3]], <4 x i1> zeroinitializer
+; CHECK-NEXT: [[TMP4:%.*]] = select <4 x i1> [[TMP2]], <4 x i1> [[TMP3]], <4 x i1> zeroinitializer
; CHECK-NEXT: [[TMP5:%.*]] = freeze <4 x i1> [[TMP4]]
; CHECK-NEXT: [[CMP:%.*]] = call i1 @llvm.vector.reduce.or.v4i1(<4 x i1> [[TMP5]])
; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[IV]], 4
@@ -131,10 +132,11 @@ define i64 @single_exit_in_conditional_block2() {
; CHECK-NEXT: [[TMP0:%.*]] = getelementptr inbounds i8, ptr @A, i64 [[IV]]
; CHECK-NEXT: [[WIDE_LOAD:%.*]] = load <4 x i8>, ptr [[TMP0]], align 1
; CHECK-NEXT: [[TMP1:%.*]] = icmp slt <4 x i8> [[WIDE_LOAD]], zeroinitializer
+; CHECK-NEXT: [[TMP2:%.*]] = freeze <4 x i1> [[TMP1]]
; CHECK-NEXT: [[GEP_B:%.*]] = getelementptr inbounds i8, ptr @B, i64 [[IV]]
; CHECK-NEXT: [[WIDE_LOAD1:%.*]] = load <4 x i8>, ptr [[GEP_B]], align 1
; CHECK-NEXT: [[TMP3:%.*]] = icmp eq <4 x i8> [[WIDE_LOAD]], [[WIDE_LOAD1]]
-; CHECK-NEXT: [[TMP4:%.*]] = select <4 x i1> [[TMP1]], <4 x i1> [[TMP3]], <4 x i1> zeroinitializer
+; CHECK-NEXT: [[TMP4:%.*]] = select <4 x i1> [[TMP2]], <4 x i1> [[TMP3]], <4 x i1> zeroinitializer
; CHECK-NEXT: [[TMP5:%.*]] = freeze <4 x i1> [[TMP4]]
; CHECK-NEXT: [[CMP:%.*]] = call i1 @llvm.vector.reduce.or.v4i1(<4 x i1> [[TMP5]])
; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[IV]], 4
@@ -194,7 +196,8 @@ define i64 @exit_cond_defined_in_header() {
; CHECK-NEXT: [[WIDE_LOAD:%.*]] = load <4 x i8>, ptr [[GEP_A]], align 1
; CHECK-NEXT: [[TMP1:%.*]] = icmp sgt <4 x i8> [[WIDE_LOAD]], zeroinitializer
; CHECK-NEXT: [[TMP2:%.*]] = icmp slt <4 x i8> [[WIDE_LOAD]], splat (i8 42)
-; CHECK-NEXT: [[TMP3:%.*]] = select <4 x i1> [[TMP1]], <4 x i1> [[TMP2]], <4 x i1> zeroinitializer
+; CHECK-NEXT: [[TMP5:%.*]] = freeze <4 x i1> [[TMP1]]
+; CHECK-NEXT: [[TMP3:%.*]] = select <4 x i1> [[TMP5]], <4 x i1> [[TMP2]], <4 x i1> zeroinitializer
; CHECK-NEXT: [[TMP4:%.*]] = freeze <4 x i1> [[TMP3]]
; CHECK-NEXT: [[BRANCH_COND:%.*]] = call i1 @llvm.vector.reduce.or.v4i1(<4 x i1> [[TMP4]])
; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[IV]], 4
@@ -248,7 +251,8 @@ define i64 @livein_exit_cond_in_conditional(i1 %exit.cond) {
; CHECK-NEXT: [[GEP_A:%.*]] = getelementptr inbounds i8, ptr @A, i64 [[IV]]
; CHECK-NEXT: [[WIDE_LOAD:%.*]] = load <4 x i8>, ptr [[GEP_A]], align 1
; CHECK-NEXT: [[TMP1:%.*]] = icmp slt <4 x i8> [[WIDE_LOAD]], zeroinitializer
-; CHECK-NEXT: [[TMP2:%.*]] = select <4 x i1> [[TMP1]], <4 x i1> [[BROADCAST_SPLAT]], <4 x i1> zeroinitializer
+; CHECK-NEXT: [[TMP4:%.*]] = freeze <4 x i1> [[TMP1]]
+; CHECK-NEXT: [[TMP2:%.*]] = select <4 x i1> [[TMP4]], <4 x i1> [[BROADCAST_SPLAT]], <4 x i1> zeroinitializer
; CHECK-NEXT: [[TMP3:%.*]] = freeze <4 x i1> [[TMP2]]
; CHECK-NEXT: [[BRANCH_COND:%.*]] = call i1 @llvm.vector.reduce.or.v4i1(<4 x i1> [[TMP3]])
; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[IV]], 4
@@ -303,7 +307,8 @@ define i64 @livein_exit_cond_in_conditional2(i1 %exit.cond) {
; CHECK-NEXT: [[GEP_A:%.*]] = getelementptr inbounds i8, ptr @A, i64 [[IV]]
; CHECK-NEXT: [[WIDE_LOAD:%.*]] = load <4 x i8>, ptr [[GEP_A]], align 1
; CHECK-NEXT: [[TMP1:%.*]] = icmp slt <4 x i8> [[WIDE_LOAD]], zeroinitializer
-; CHECK-NEXT: [[TMP2:%.*]] = select <4 x i1> [[TMP1]], <4 x i1> [[BROADCAST_SPLAT]], <4 x i1> zeroinitializer
+; CHECK-NEXT: [[TMP4:%.*]] = freeze <4 x i1> [[TMP1]]
+; CHECK-NEXT: [[TMP2:%.*]] = select <4 x i1> [[TMP4]], <4 x i1> [[BROADCAST_SPLAT]], <4 x i1> zeroinitializer
; CHECK-NEXT: [[TMP3:%.*]] = freeze <4 x i1> [[TMP2]]
; CHECK-NEXT: [[BRANCH_COND:%.*]] = call i1 @llvm.vector.reduce.or.v4i1(<4 x i1> [[TMP3]])
; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[IV]], 4
@@ -353,13 +358,14 @@ define i64 @diamond_with_join_then_exit() {
; CHECK-NEXT: [[TMP0:%.*]] = getelementptr inbounds i8, ptr @A, i64 [[INDEX]]
; CHECK-NEXT: [[WIDE_LOAD:%.*]] = load <4 x i8>, ptr [[TMP0]], align 1
; CHECK-NEXT: [[TMP1:%.*]] = icmp slt <4 x i8> [[WIDE_LOAD]], zeroinitializer
+; CHECK-NEXT: [[TMP11:%.*]] = freeze <4 x i1> [[TMP1]]
; CHECK-NEXT: [[TMP2:%.*]] = getelementptr inbounds i8, ptr @C, i64 [[INDEX]]
; CHECK-NEXT: [[WIDE_LOAD1:%.*]] = load <4 x i8>, ptr [[TMP2]], align 1
; CHECK-NEXT: [[TMP3:%.*]] = zext <4 x i8> [[WIDE_LOAD1]] to <4 x i64>
; CHECK-NEXT: [[TMP4:%.*]] = getelementptr inbounds i8, ptr @B, i64 [[INDEX]]
; CHECK-NEXT: [[WIDE_LOAD2:%.*]] = load <4 x i8>, ptr [[TMP4]], align 1
; CHECK-NEXT: [[TMP5:%.*]] = zext <4 x i8> [[WIDE_LOAD2]] to <4 x i64>
-; CHECK-NEXT: [[PREDPHI:%.*]] = select <4 x i1> [[TMP1]], <4 x i64> [[TMP5]], <4 x i64> [[TMP3]]
+; CHECK-NEXT: [[PREDPHI:%.*]] = select <4 x i1> [[TMP11]], <4 x i64> [[TMP5]], <4 x i64> [[TMP3]]
; CHECK-NEXT: [[TMP6:%.*]] = icmp eq <4 x i64> [[PREDPHI]], splat (i64 42)
; CHECK-NEXT: [[TMP7:%.*]] = freeze <4 x i1> [[TMP6]]
; CHECK-NEXT: [[TMP8:%.*]] = call i1 @llvm.vector.reduce.or.v4i1(<4 x i1> [[TMP7]])
>From 8118837ea1ee6ca1e0e45eeb12b65b0e47daf793 Mon Sep 17 00:00:00 2001
From: Luke Lau <luke at igalia.com>
Date: Wed, 11 Nov 2026 03:13:33 +0800
Subject: [PATCH 10/15] Remove old dominance unit test
---
.../Transforms/Vectorize/VPlanVerifierTest.cpp | 18 ------------------
1 file changed, 18 deletions(-)
diff --git a/llvm/unittests/Transforms/Vectorize/VPlanVerifierTest.cpp b/llvm/unittests/Transforms/Vectorize/VPlanVerifierTest.cpp
index 6eed6c30538198..e006ed7e4bd124 100644
--- a/llvm/unittests/Transforms/Vectorize/VPlanVerifierTest.cpp
+++ b/llvm/unittests/Transforms/Vectorize/VPlanVerifierTest.cpp
@@ -63,24 +63,6 @@ TEST_F(VPVerifierTest, VPInstructionUseBeforeDefSameBB) {
#endif
}
-TEST_F(VPVerifierTest, MaskedCondUseBeforeDefSameBB) {
- VPlan &Plan = getPlan();
- auto *DefI = new VPInstruction(VPInstruction::MaskedCond, {Plan.getFalse()});
- VPBasicBlock *Entry = Plan.getEntry();
- Entry->appendRecipe(new VPInstruction(Instruction::Freeze, {DefI}));
- Entry->appendRecipe(DefI);
- VPBlockUtils::connectBlocks(Entry, Plan.getScalarHeader());
-
-#if GTEST_HAS_STREAM_REDIRECTION
- ::testing::internal::CaptureStderr();
-#endif
- EXPECT_FALSE(verifyVPlanIsValid(Plan));
-#if GTEST_HAS_STREAM_REDIRECTION
- EXPECT_TRUE(StringRef(::testing::internal::GetCapturedStderr())
- .starts_with("Use before def!\n"));
-#endif
-}
-
TEST_F(VPVerifierTest, VPInstructionUseBeforeDefDifferentBB) {
VPlan &Plan = getPlan();
VPIRValue *Zero = Plan.getConstantInt(32, 0);
>From 5ab2d738db44f54083ff14557016ab416b142159 Mon Sep 17 00:00:00 2001
From: Luke Lau <luke at igalia.com>
Date: Wed, 28 Oct 2026 22:33:08 +0800
Subject: [PATCH 11/15] Precommit tests
---
.../LoopVectorize/early_exit_with_stores.ll | 221 ++++++++++++++++++
1 file changed, 221 insertions(+)
diff --git a/llvm/test/Transforms/LoopVectorize/early_exit_with_stores.ll b/llvm/test/Transforms/LoopVectorize/early_exit_with_stores.ll
index ad1269aa3c8523..139c9992cc38d7 100644
--- a/llvm/test/Transforms/LoopVectorize/early_exit_with_stores.ll
+++ b/llvm/test/Transforms/LoopVectorize/early_exit_with_stores.ll
@@ -1555,3 +1555,224 @@ exit:
}
declare i64 @get_an_unknown_offset();
+
+define void @store_in_latch_after_exit(ptr align 2 dereferenceable(40) noalias %array, ptr align 2 dereferenceable(40) readonly %pred) {
+; CHECK-LABEL: define void @store_in_latch_after_exit(
+; CHECK-SAME: ptr noalias align 2 dereferenceable(40) [[ARRAY:%.*]], ptr readonly align 2 dereferenceable(40) [[PRED:%.*]]) {
+; CHECK-NEXT: [[ENTRY:.*:]]
+; CHECK-NEXT: br label %[[VECTOR_PH:.*]]
+; CHECK: [[VECTOR_PH]]:
+; CHECK-NEXT: br label %[[VECTOR_BODY:.*]]
+; CHECK: [[VECTOR_BODY]]:
+; CHECK-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-NEXT: [[TMP0:%.*]] = getelementptr inbounds nuw i16, ptr [[PRED]], i64 [[INDEX]]
+; CHECK-NEXT: [[WIDE_LOAD:%.*]] = load <4 x i16>, ptr [[TMP0]], align 2
+; CHECK-NEXT: [[TMP1:%.*]] = icmp sgt <4 x i16> [[WIDE_LOAD]], splat (i16 500)
+; CHECK-NEXT: [[TMP2:%.*]] = freeze <4 x i1> [[TMP1]]
+; CHECK-NEXT: [[TMP3:%.*]] = call <4 x i1> @llvm.mask.beforefirst.v4i1(<4 x i1> [[TMP2]])
+; CHECK-NEXT: [[TMP4:%.*]] = getelementptr i16, ptr [[ARRAY]], i64 [[INDEX]]
+; CHECK-NEXT: [[WIDE_MASKED_LOAD:%.*]] = call <4 x i16> @llvm.masked.load.v4i16.p0(ptr align 2 [[TMP4]], <4 x i1> [[TMP3]], <4 x i16> poison)
+; CHECK-NEXT: [[TMP5:%.*]] = add nsw <4 x i16> [[WIDE_MASKED_LOAD]], splat (i16 1)
+; CHECK-NEXT: call void @llvm.masked.store.v4i16.p0(<4 x i16> [[TMP5]], ptr align 2 [[TMP4]], <4 x i1> [[TMP3]])
+; CHECK-NEXT: [[TMP6:%.*]] = call i1 @llvm.vector.reduce.or.v4i1(<4 x i1> [[TMP2]])
+; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 4
+; CHECK-NEXT: [[TMP7:%.*]] = icmp eq i64 [[INDEX_NEXT]], 20
+; CHECK-NEXT: [[TMP8:%.*]] = or i1 [[TMP6]], [[TMP7]]
+; CHECK-NEXT: br i1 [[TMP8]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP20:![0-9]+]]
+; CHECK: [[MIDDLE_BLOCK]]:
+; CHECK-NEXT: [[TMP9:%.*]] = call i64 @llvm.experimental.cttz.elts.i64.v4i1(<4 x i1> [[TMP2]], i1 false)
+; CHECK-NEXT: [[TMP10:%.*]] = add i64 [[INDEX]], [[TMP9]]
+; CHECK-NEXT: [[TMP11:%.*]] = icmp eq i64 [[TMP10]], 20
+; CHECK-NEXT: br i1 [[TMP11]], label %[[EXIT:.*]], label %[[SCALAR_PH:.*]]
+; CHECK: [[SCALAR_PH]]:
+; CHECK-NEXT: br label %[[LOOP:.*]]
+; CHECK: [[LOOP]]:
+; CHECK-NEXT: [[IV:%.*]] = phi i64 [ [[TMP10]], %[[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], %[[LOOP_LATCH:.*]] ]
+; CHECK-NEXT: [[EE_ADDR:%.*]] = getelementptr inbounds nuw i16, ptr [[PRED]], i64 [[IV]]
+; CHECK-NEXT: [[EE_VAL:%.*]] = load i16, ptr [[EE_ADDR]], align 2
+; CHECK-NEXT: [[EE_COND:%.*]] = icmp sgt i16 [[EE_VAL]], 500
+; CHECK-NEXT: br i1 [[EE_COND]], label %[[EXIT]], label %[[LOOP_LATCH]]
+; CHECK: [[LOOP_LATCH]]:
+; CHECK-NEXT: [[ST_ADDR:%.*]] = getelementptr inbounds nuw i16, ptr [[ARRAY]], i64 [[IV]]
+; CHECK-NEXT: [[DATA:%.*]] = load i16, ptr [[ST_ADDR]], align 2
+; CHECK-NEXT: [[INC:%.*]] = add nsw i16 [[DATA]], 1
+; CHECK-NEXT: store i16 [[INC]], ptr [[ST_ADDR]], align 2
+; CHECK-NEXT: [[IV_NEXT]] = add nuw nsw i64 [[IV]], 1
+; CHECK-NEXT: [[COUNTED_COND:%.*]] = icmp eq i64 [[IV_NEXT]], 20
+; CHECK-NEXT: br i1 [[COUNTED_COND]], label %[[EXIT]], label %[[LOOP]], !llvm.loop [[LOOP21:![0-9]+]]
+; CHECK: [[EXIT]]:
+; CHECK-NEXT: ret void
+;
+entry:
+ br label %loop
+
+loop:
+ %iv = phi i64 [ 0, %entry ], [ %iv.next, %loop.latch ]
+ %ee.addr = getelementptr inbounds nuw i16, ptr %pred, i64 %iv
+ %ee.val = load i16, ptr %ee.addr
+ %ee.cond = icmp sgt i16 %ee.val, 500
+ br i1 %ee.cond, label %exit, label %loop.latch
+
+loop.latch:
+ %st.addr = getelementptr inbounds nuw i16, ptr %array, i64 %iv
+ %data = load i16, ptr %st.addr
+ %inc = add nsw i16 %data, 1
+ store i16 %inc, ptr %st.addr
+ %iv.next = add nuw nsw i64 %iv, 1
+ %counted.cond = icmp eq i64 %iv.next, 20
+ br i1 %counted.cond, label %exit, label %loop
+
+exit:
+ ret void
+}
+
+define void @conditional_store_after_two_exits(ptr align 2 dereferenceable(40) noalias %array, ptr align 2 dereferenceable(40) readonly %pred, ptr align 2 dereferenceable(40) readonly %pred2) {
+; CHECK-LABEL: define void @conditional_store_after_two_exits(
+; CHECK-SAME: ptr noalias align 2 dereferenceable(40) [[ARRAY:%.*]], ptr readonly align 2 dereferenceable(40) [[PRED:%.*]], ptr readonly align 2 dereferenceable(40) [[PRED2:%.*]]) {
+; CHECK-NEXT: [[ENTRY:.*]]:
+; CHECK-NEXT: br label %[[LOOP:.*]]
+; CHECK: [[LOOP]]:
+; CHECK-NEXT: [[IV:%.*]] = phi i64 [ 0, %[[ENTRY]] ], [ [[IV_NEXT:%.*]], %[[LOOP_LATCH:.*]] ]
+; CHECK-NEXT: [[EE_ADDR:%.*]] = getelementptr inbounds nuw i16, ptr [[PRED]], i64 [[IV]]
+; CHECK-NEXT: [[EE_VAL:%.*]] = load i16, ptr [[EE_ADDR]], align 2
+; CHECK-NEXT: [[EE2_ADDR:%.*]] = getelementptr inbounds nuw i16, ptr [[PRED2]], i64 [[IV]]
+; CHECK-NEXT: [[EE2_VAL:%.*]] = load i16, ptr [[EE2_ADDR]], align 2
+; CHECK-NEXT: [[EE_COND:%.*]] = icmp sgt i16 [[EE_VAL]], 500
+; CHECK-NEXT: br i1 [[EE_COND]], label %[[EXIT:.*]], label %[[LOOP_2:.*]]
+; CHECK: [[LOOP_2]]:
+; CHECK-NEXT: [[EE2_COND:%.*]] = icmp eq i16 [[EE2_VAL]], 0
+; CHECK-NEXT: br i1 [[EE2_COND]], label %[[EXIT_2:.*]], label %[[LOOP_3:.*]]
+; CHECK: [[LOOP_3]]:
+; CHECK-NEXT: [[ST_ADDR:%.*]] = getelementptr inbounds nuw i16, ptr [[ARRAY]], i64 [[IV]]
+; CHECK-NEXT: [[DATA:%.*]] = load i16, ptr [[ST_ADDR]], align 2
+; CHECK-NEXT: [[ST_COND:%.*]] = icmp slt i16 [[DATA]], 0
+; CHECK-NEXT: br i1 [[ST_COND]], label %[[STORE:.*]], label %[[LOOP_LATCH]]
+; CHECK: [[STORE]]:
+; CHECK-NEXT: [[INC:%.*]] = add i16 [[DATA]], 1
+; CHECK-NEXT: store i16 [[INC]], ptr [[ST_ADDR]], align 2
+; CHECK-NEXT: br label %[[LOOP_LATCH]]
+; CHECK: [[LOOP_LATCH]]:
+; CHECK-NEXT: [[IV_NEXT]] = add nuw nsw i64 [[IV]], 1
+; CHECK-NEXT: [[COUNTED_COND:%.*]] = icmp eq i64 [[IV_NEXT]], 20
+; CHECK-NEXT: br i1 [[COUNTED_COND]], label %[[EXIT]], label %[[LOOP]]
+; CHECK: [[EXIT]]:
+; CHECK-NEXT: ret void
+; CHECK: [[EXIT_2]]:
+; CHECK-NEXT: ret void
+;
+entry:
+ br label %loop
+
+loop:
+ %iv = phi i64 [ 0, %entry ], [ %iv.next, %loop.latch ]
+ %ee.addr = getelementptr inbounds nuw i16, ptr %pred, i64 %iv
+ %ee.val = load i16, ptr %ee.addr
+ %ee2.addr = getelementptr inbounds nuw i16, ptr %pred2, i64 %iv
+ %ee2.val = load i16, ptr %ee2.addr
+ %ee.cond = icmp sgt i16 %ee.val, 500
+ br i1 %ee.cond, label %exit, label %loop.2
+
+loop.2:
+ %ee2.cond = icmp eq i16 %ee2.val, 0
+ br i1 %ee2.cond, label %exit.2, label %loop.3
+
+loop.3:
+ %st.addr = getelementptr inbounds nuw i16, ptr %array, i64 %iv
+ %data = load i16, ptr %st.addr
+ %st.cond = icmp slt i16 %data, 0
+ br i1 %st.cond, label %store, label %loop.latch
+
+store:
+ %inc = add i16 %data, 1
+ store i16 %inc, ptr %st.addr
+ br label %loop.latch
+
+loop.latch:
+ %iv.next = add nuw nsw i64 %iv, 1
+ %counted.cond = icmp eq i64 %iv.next, 20
+ br i1 %counted.cond, label %exit, label %loop
+
+exit:
+ ret void
+
+exit.2:
+ ret void
+}
+
+define i16 @store_after_exit_live_out(ptr align 2 dereferenceable(40) noalias %array, ptr align 2 dereferenceable(40) readonly %pred) {
+; CHECK-LABEL: define i16 @store_after_exit_live_out(
+; CHECK-SAME: ptr noalias align 2 dereferenceable(40) [[ARRAY:%.*]], ptr readonly align 2 dereferenceable(40) [[PRED:%.*]]) {
+; CHECK-NEXT: [[ENTRY:.*:]]
+; CHECK-NEXT: br label %[[VECTOR_PH:.*]]
+; CHECK: [[VECTOR_PH]]:
+; CHECK-NEXT: br label %[[VECTOR_BODY:.*]]
+; CHECK: [[VECTOR_BODY]]:
+; CHECK-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-NEXT: [[TMP0:%.*]] = getelementptr inbounds nuw i16, ptr [[PRED]], i64 [[INDEX]]
+; CHECK-NEXT: [[WIDE_LOAD:%.*]] = load <4 x i16>, ptr [[TMP0]], align 2
+; CHECK-NEXT: [[TMP1:%.*]] = icmp sgt <4 x i16> [[WIDE_LOAD]], splat (i16 500)
+; CHECK-NEXT: [[TMP2:%.*]] = freeze <4 x i1> [[TMP1]]
+; CHECK-NEXT: [[TMP3:%.*]] = call <4 x i1> @llvm.mask.beforefirst.v4i1(<4 x i1> [[TMP2]])
+; CHECK-NEXT: [[TMP4:%.*]] = getelementptr i16, ptr [[ARRAY]], i64 [[INDEX]]
+; CHECK-NEXT: [[WIDE_MASKED_LOAD:%.*]] = call <4 x i16> @llvm.masked.load.v4i16.p0(ptr align 2 [[TMP4]], <4 x i1> [[TMP3]], <4 x i16> poison)
+; CHECK-NEXT: [[TMP5:%.*]] = add nsw <4 x i16> [[WIDE_MASKED_LOAD]], splat (i16 1)
+; CHECK-NEXT: call void @llvm.masked.store.v4i16.p0(<4 x i16> [[TMP5]], ptr align 2 [[TMP4]], <4 x i1> [[TMP3]])
+; CHECK-NEXT: [[TMP6:%.*]] = call i1 @llvm.vector.reduce.or.v4i1(<4 x i1> [[TMP2]])
+; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 4
+; CHECK-NEXT: [[TMP7:%.*]] = icmp eq i64 [[INDEX_NEXT]], 20
+; CHECK-NEXT: [[TMP8:%.*]] = or i1 [[TMP6]], [[TMP7]]
+; CHECK-NEXT: br i1 [[TMP8]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP22:![0-9]+]]
+; CHECK: [[MIDDLE_BLOCK]]:
+; CHECK-NEXT: [[TMP9:%.*]] = extractelement <4 x i16> [[TMP5]], i64 3
+; CHECK-NEXT: [[TMP10:%.*]] = call i64 @llvm.experimental.cttz.elts.i64.v4i1(<4 x i1> [[TMP2]], i1 false)
+; CHECK-NEXT: [[TMP11:%.*]] = add i64 [[INDEX]], [[TMP10]]
+; CHECK-NEXT: [[TMP12:%.*]] = icmp eq i64 [[TMP11]], 20
+; CHECK-NEXT: br i1 [[TMP12]], label %[[EXIT:.*]], label %[[SCALAR_PH:.*]]
+; CHECK: [[SCALAR_PH]]:
+; CHECK-NEXT: br label %[[LOOP:.*]]
+; CHECK: [[LOOP]]:
+; CHECK-NEXT: [[IV:%.*]] = phi i64 [ [[TMP11]], %[[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], %[[LOOP_LATCH:.*]] ]
+; CHECK-NEXT: [[EE_ADDR:%.*]] = getelementptr inbounds nuw i16, ptr [[PRED]], i64 [[IV]]
+; CHECK-NEXT: [[EE_VAL:%.*]] = load i16, ptr [[EE_ADDR]], align 2
+; CHECK-NEXT: [[EE_COND:%.*]] = icmp sgt i16 [[EE_VAL]], 500
+; CHECK-NEXT: br i1 [[EE_COND]], label %[[EXIT]], label %[[LOOP_STORE:.*]]
+; CHECK: [[LOOP_STORE]]:
+; CHECK-NEXT: [[ST_ADDR:%.*]] = getelementptr inbounds nuw i16, ptr [[ARRAY]], i64 [[IV]]
+; CHECK-NEXT: [[DATA:%.*]] = load i16, ptr [[ST_ADDR]], align 2
+; CHECK-NEXT: [[INC:%.*]] = add nsw i16 [[DATA]], 1
+; CHECK-NEXT: store i16 [[INC]], ptr [[ST_ADDR]], align 2
+; CHECK-NEXT: br label %[[LOOP_LATCH]]
+; CHECK: [[LOOP_LATCH]]:
+; CHECK-NEXT: [[IV_NEXT]] = add nuw nsw i64 [[IV]], 1
+; CHECK-NEXT: [[COUNTED_COND:%.*]] = icmp eq i64 [[IV_NEXT]], 20
+; CHECK-NEXT: br i1 [[COUNTED_COND]], label %[[EXIT]], label %[[LOOP]], !llvm.loop [[LOOP23:![0-9]+]]
+; CHECK: [[EXIT]]:
+; CHECK-NEXT: [[RES:%.*]] = phi i16 [ -1, %[[LOOP]] ], [ [[INC]], %[[LOOP_LATCH]] ], [ [[TMP9]], %[[MIDDLE_BLOCK]] ]
+; CHECK-NEXT: ret i16 [[RES]]
+;
+entry:
+ br label %loop
+
+loop:
+ %iv = phi i64 [ 0, %entry ], [ %iv.next, %loop.latch ]
+ %ee.addr = getelementptr inbounds nuw i16, ptr %pred, i64 %iv
+ %ee.val = load i16, ptr %ee.addr
+ %ee.cond = icmp sgt i16 %ee.val, 500
+ br i1 %ee.cond, label %exit, label %loop.store
+
+loop.store:
+ %st.addr = getelementptr inbounds nuw i16, ptr %array, i64 %iv
+ %data = load i16, ptr %st.addr
+ %inc = add nsw i16 %data, 1
+ store i16 %inc, ptr %st.addr
+ br label %loop.latch
+
+loop.latch:
+ %iv.next = add nuw nsw i64 %iv, 1
+ %counted.cond = icmp eq i64 %iv.next, 20
+ br i1 %counted.cond, label %exit, label %loop
+
+exit:
+ %res = phi i16 [ -1, %loop ], [ %inc, %loop.latch ]
+ ret i16 %res
+}
>From f7cf97a2d75c1c45ea1c63f48eb125af1ee43f25 Mon Sep 17 00:00:00 2001
From: Luke Lau <luke at igalia.com>
Date: Fri, 30 Oct 2026 00:16:00 +0800
Subject: [PATCH 12/15] [VPlan] Allow early exit loops with side effects by
masking in the vector loop.
When all blocks with side effects are dominated by every early exit, we can vectorize the loop if we mask off any lanes past the exit. We can do this with control flow:
```mermaid
flowchart TD
header --> exiting
exiting -->| exit mask | sideeffects --> latch --> latch.split
exiting -->| !exit mask | latch.split
```
For every early exiting block we add a branch to the latch on `!mask.beforefirst(exitcond)`. This is implemented in UncountableExitStyle::ReadOnly which is now renamed to MaskedHandleExitInVectorLoop.
This is a complementary exit style to MaskedHandleExitInScalarLoop. If any of the side effects aren't dominated by the early exit it still falls back to the scalar loop style, which is why none of the existing early exit side effect tests changed. Masking in loop is profitable for RISC-V and I'm assuming it's also profitable for AArch64 too, but if not then we can add a TTI hook for a preferred exit style.
---
.../Transforms/Vectorize/LoopVectorize.cpp | 3 +-
llvm/lib/Transforms/Vectorize/VPlan.h | 9 +-
.../Vectorize/VPlanConstruction.cpp | 4 +-
.../Transforms/Vectorize/VPlanTransforms.cpp | 119 ++++++++++-----
.../RISCV/early_exit_with_stores.ll | 15 +-
.../X86/early_exit_with_stores.ll | 18 +--
.../LoopVectorize/early_exit_with_stores.ll | 135 ++++++++----------
.../Vectorize/VPlanVerifierTest.cpp | 5 +-
8 files changed, 176 insertions(+), 132 deletions(-)
diff --git a/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp b/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
index 128adbf60f1f22..8a4800a80a22f4 100644
--- a/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
+++ b/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
@@ -6268,7 +6268,8 @@ LoopVectorizationPlanner::tryToBuildVPlan1(LoopVectorizationCostModel &PlanCM) {
return nullptr;
// TODO: Check target preference for style.
- UncountableExitStyle EEStyle = UncountableExitStyle::ReadOnly;
+ UncountableExitStyle EEStyle =
+ UncountableExitStyle::MaskedHandleExitInVectorLoop;
if (!RUN_VPLAN_PASS(VPlanTransforms::handleUncountableEarlyExits, *VPlan0,
ORE, OrigLoop, PSE, *DT, Legal->getAssumptionCache(),
EEStyle)) {
diff --git a/llvm/lib/Transforms/Vectorize/VPlan.h b/llvm/lib/Transforms/Vectorize/VPlan.h
index d27473af8a7ca5..d2354f58aba100 100644
--- a/llvm/lib/Transforms/Vectorize/VPlan.h
+++ b/llvm/lib/Transforms/Vectorize/VPlan.h
@@ -79,10 +79,11 @@ using VPlanPtr = std::unique_ptr<VPlan>;
/// Different methods of handling early exits.
///
enum class UncountableExitStyle {
- /// No side effects to worry about, so we can process any uncountable exits
- /// in the loop and branch either to the middle block if the trip count was
- /// reached, or an early exitblock to determine which exit was taken.
- ReadOnly,
+ /// Any side effects come after all the uncountable exits and are masked off,
+ /// so we can process any uncountable exits in the loop and branch either to
+ /// the middle block if the trip count was reached, or an early exitblock to
+ /// determine which exit was taken.
+ MaskedHandleExitInVectorLoop,
/// All memory operations other than the load(s) required to determine whether
/// an uncountable exit occurre will be masked based on that condition. If an
/// uncountable exit is taken, then all lanes before the exiting lane will
diff --git a/llvm/lib/Transforms/Vectorize/VPlanConstruction.cpp b/llvm/lib/Transforms/Vectorize/VPlanConstruction.cpp
index 8a026aba613855..db47293ca8476d 100644
--- a/llvm/lib/Transforms/Vectorize/VPlanConstruction.cpp
+++ b/llvm/lib/Transforms/Vectorize/VPlanConstruction.cpp
@@ -1272,7 +1272,9 @@ bool VPlanTransforms::areAllLoadsDereferenceable(VPBasicBlock *HeaderVPBB,
continue;
auto *VPI = dyn_cast<VPInstruction>(&R);
if (!VPI || VPI->getOpcode() != Instruction::Load) {
- assert(!R.mayReadFromMemory() && "unexpected recipe reading memory");
+ assert((!R.mayReadFromMemory() ||
+ (VPI && VPI->getOpcode() == Instruction::Store)) &&
+ "unexpected recipe reading memory");
continue;
}
diff --git a/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp b/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp
index c75441a3ed2729..548ddc68f4f398 100644
--- a/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp
+++ b/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp
@@ -3554,50 +3554,59 @@ static bool handleUncountableExitsWithSideEffects(
// Returns true if the loop contains stores, the only writes legality accepts in
// early-exit loops.
-static bool loopHasStores(VPBasicBlock *HeaderVPBB) {
- return any_of(vp_rpo_plain_cfg_loop_body(HeaderVPBB), [](VPBasicBlock *VPBB) {
- return any_of(*VPBB, [VPBB](VPRecipeBase &R) {
- (void)VPBB;
- if (match(&R, m_Intrinsic<Intrinsic::pseudoprobe>()))
- return false;
- if (!R.mayWriteToMemory()) {
- assert((!R.mayHaveSideEffects() || &R == VPBB->getTerminator() ||
- R.isSafeToSpeculativelyExecute()) &&
- "legality only accepts recipes with side effects that are "
- "safe to speculate");
- return false;
- }
- assert(match(&R, m_VPInstruction<Instruction::Store>()) &&
- "legality only accepts stores as writes in early-exit loops");
- return true;
- });
- });
+static SmallVector<VPBasicBlock *> blocksWithStores(VPBasicBlock *HeaderVPBB) {
+ return to_vector(make_filter_range(
+ vp_rpo_plain_cfg_loop_body(HeaderVPBB), [](VPBasicBlock *VPBB) {
+ return any_of(*VPBB, [VPBB](VPRecipeBase &R) {
+ (void)VPBB;
+ if (match(&R, m_Intrinsic<Intrinsic::pseudoprobe>()))
+ return false;
+ if (!R.mayWriteToMemory()) {
+ assert((!R.mayHaveSideEffects() || &R == VPBB->getTerminator() ||
+ R.isSafeToSpeculativelyExecute()) &&
+ "legality only accepts recipes with side effects that are "
+ "safe to speculate");
+ return false;
+ }
+ assert(match(&R, m_VPInstruction<Instruction::Store>()) &&
+ "legality only accepts stores as writes in early-exit loops");
+ return true;
+ });
+ }));
}
bool VPlanTransforms::handleUncountableEarlyExits(
VPlan &Plan, OptimizationRemarkEmitter *ORE, Loop *TheLoop,
PredicatedScalarEvolution &PSE, DominatorTree &DT, AssumptionCache *AC,
UncountableExitStyle Style) {
-#ifndef NDEBUG
VPDominatorTree VPDT(Plan);
-#endif
auto *MiddleVPBB = VPBlockUtils::getPlainCFGMiddleBlock(Plan);
auto [HeaderVPBB, LatchVPBB] = VPBlockUtils::getPlainCFGHeaderAndLatch(Plan);
- if (loopHasStores(HeaderVPBB))
- Style = UncountableExitStyle::MaskedHandleExitInScalarLoop;
+ // If all side effects are dominated by the early exits, we can mask in the
+ // vector loop. Otherwise we need to handle side effects in the scalar loop.
+ // TODO: Legality only allows side effects that the scalar loop style can
+ // handle, relax it for side effects after all the early exits.
+ SmallVector<VPBasicBlock *> SideEffectVPBBs = blocksWithStores(HeaderVPBB);
+ if (!SideEffectVPBBs.empty()) {
+ auto EarlyExits = vputils::getEarlyExits(Plan, MiddleVPBB);
+ for (auto [EarlyExitingVPBB, _] : EarlyExits)
+ for (VPBasicBlock *VPBB : SideEffectVPBBs)
+ if (!VPDT.properlyDominates(EarlyExitingVPBB, VPBB))
+ Style = UncountableExitStyle::MaskedHandleExitInScalarLoop;
+ }
- // Dereferenceability is checked separately for uncountable exit loops with
- // stores, as only the loads contributing to the exit condition need to
- // be checked.
- if (Style == UncountableExitStyle::ReadOnly &&
+ if (Style == UncountableExitStyle::MaskedHandleExitInVectorLoop &&
!areAllLoadsDereferenceable(HeaderVPBB, TheLoop, PSE, DT, AC)) {
- reportVectorizationFailure(
- "Auto-vectorization of early exit loops with potentially "
- "faulting loads is not supported",
- "EarlyExitFaultingLoads", ORE, TheLoop);
- return false;
+ if (SideEffectVPBBs.empty()) {
+ reportVectorizationFailure(
+ "Auto-vectorization of early exit loops with potentially "
+ "faulting loads is not supported",
+ "EarlyExitFaultingLoads", ORE, TheLoop);
+ return false;
+ }
+ Style = UncountableExitStyle::MaskedHandleExitInScalarLoop;
}
VPBuilder LatchBuilder(LatchVPBB->getTerminator());
@@ -3649,7 +3658,7 @@ bool VPlanTransforms::handleUncountableEarlyExits(
});
}
- if (Style == UncountableExitStyle::ReadOnly) {
+ if (Style == UncountableExitStyle::MaskedHandleExitInVectorLoop) {
// Lanes past the exit can be poison that the scalar loop never branched on,
// so freeze them to prevent branching on poison.
for (VPBasicBlock *VPBB : vp_rpo_plain_cfg_loop_body(HeaderVPBB)) {
@@ -3659,6 +3668,49 @@ bool VPlanTransforms::handleUncountableEarlyExits(
Term->setOperand(0, VPBuilder(Term).createFreeze(Term->getOperand(0)));
}
+ if (!SideEffectVPBBs.empty()) {
+ // Split the latch in case it has recipes with side effects.
+ LatchVPBB = LatchVPBB->splitAt(LatchVPBB->getTerminator()->getIterator());
+ LatchBuilder.setInsertPoint(LatchVPBB->getTerminator());
+
+ // Branch to the latch to skip any side effects past the exiting lane.
+ for (EarlyExitInfo &Exit : Exits) {
+ VPBasicBlock *EarlyExitingVPBB = Exit.EarlyExitingVPBB;
+ VPBuilder ExitingBuilder(EarlyExitingVPBB->getTerminator());
+ VPValue *ExitMask = ExitingBuilder.createWidenIntrinsic(
+ Intrinsic::mask_beforefirst,
+ ExitingBuilder.createFreeze(ExitingBuilder.createNaryOp(
+ VPInstruction::MaskedCond, Exit.CondToExit)),
+ Type::getInt1Ty(Plan.getContext()));
+ EarlyExitingVPBB->getTerminator()->setOperand(0, ExitMask);
+ VPBlockUtils::connectBlocks(EarlyExitingVPBB, LatchVPBB);
+ }
+
+ // The latch's exit condition may be poison on lanes past the exit, freeze
+ // it.
+ VPRecipeBase *LatchTerm = LatchVPBB->getTerminator();
+ LatchTerm->setOperand(
+ 0, VPBuilder(LatchTerm).createFreeze(LatchTerm->getOperand(0)));
+
+ // If we took the exiting branch then any uses in the latch and beyond are
+ // poison.
+ VPDT.recalculate(Plan);
+ for (VPBasicBlock *VPBB : {LatchVPBB, MiddleVPBB}) {
+ for (VPRecipeBase &R :
+ make_range(VPBB->getFirstNonPhi(), VPBB->end())) {
+ for (auto [I, Op] : enumerate(R.operands())) {
+ VPRecipeBase *DefR = Op->getDefiningRecipe();
+ if (!DefR || VPDT.dominates(DefR->getParent(), VPBB))
+ continue;
+ DenseMap<VPBasicBlock *, VPValue *> Defs = {
+ {HeaderVPBB, Plan.getPoison(Op->getScalarType())}};
+ Defs[DefR->getParent()] = Op;
+ R.setOperand(I, vputils::reconstructSSA(LatchVPBB, Defs));
+ }
+ }
+ }
+ }
+
// An exit's condition is false on any path that didn't go through its
// exiting block.
for (EarlyExitInfo &Exit : Exits) {
@@ -3816,7 +3868,8 @@ bool VPlanTransforms::handleUncountableEarlyExits(
ExitIRI->addIncoming(NewIncoming);
}
- EarlyExitingVPBB->getTerminator()->eraseFromParent();
+ if (EarlyExitingVPBB->getNumSuccessors() == 2)
+ EarlyExitingVPBB->getTerminator()->eraseFromParent();
VPBlockUtils::disconnectBlocks(EarlyExitingVPBB, EarlyExitVPBB);
VPBlockUtils::connectBlocks(VectorEarlyExitVPBB, EarlyExitVPBB);
}
diff --git a/llvm/test/Transforms/LoopVectorize/RISCV/early_exit_with_stores.ll b/llvm/test/Transforms/LoopVectorize/RISCV/early_exit_with_stores.ll
index 573ee73e76a439..8ba6d5f81679e5 100644
--- a/llvm/test/Transforms/LoopVectorize/RISCV/early_exit_with_stores.ll
+++ b/llvm/test/Transforms/LoopVectorize/RISCV/early_exit_with_stores.ll
@@ -1277,7 +1277,7 @@ define void @interleaved_stores_after_early_exit(ptr noalias %dst, ptr align 2 d
; CHECK-NEXT: [[N_VEC:%.*]] = sub i64 20, [[N_MOD_VF]]
; CHECK-NEXT: br label %[[VECTOR_BODY:.*]]
; CHECK: [[VECTOR_BODY]]:
-; CHECK-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[MIDDLE_BLOCK:.*]] ]
; CHECK-NEXT: [[TMP2:%.*]] = getelementptr i16, ptr [[SRC]], i64 [[INDEX]]
; CHECK-NEXT: [[WIDE_LOAD:%.*]] = load <vscale x 8 x i16>, ptr [[TMP2]], align 2
; CHECK-NEXT: [[TMP3:%.*]] = icmp sgt <vscale x 8 x i16> [[WIDE_LOAD]], splat (i16 500)
@@ -1291,15 +1291,16 @@ define void @interleaved_stores_after_early_exit(ptr noalias %dst, ptr align 2 d
; CHECK-NEXT: [[TMP8:%.*]] = call i1 @llvm.vector.reduce.or.nxv8i1(<vscale x 8 x i1> [[TMP4]])
; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], [[TMP1]]
; CHECK-NEXT: [[TMP9:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
-; CHECK-NEXT: [[TMP10:%.*]] = or i1 [[TMP8]], [[TMP9]]
-; CHECK-NEXT: br i1 [[TMP10]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP11:![0-9]+]]
+; CHECK-NEXT: br i1 [[TMP8]], label %[[VECTOR_EARLY_EXIT:.*]], label %[[MIDDLE_BLOCK]]
; CHECK: [[MIDDLE_BLOCK]]:
-; CHECK-NEXT: [[TMP5:%.*]] = call i64 @llvm.experimental.cttz.elts.i64.nxv8i1(<vscale x 8 x i1> [[TMP4]], i1 false)
-; CHECK-NEXT: [[TMP11:%.*]] = add i64 [[INDEX]], [[TMP5]]
-; CHECK-NEXT: [[TMP12:%.*]] = icmp eq i64 [[TMP11]], 20
+; CHECK-NEXT: br i1 [[TMP9]], label %[[MIDDLE_BLOCK1:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP11:![0-9]+]]
+; CHECK: [[MIDDLE_BLOCK1]]:
+; CHECK-NEXT: [[TMP12:%.*]] = icmp eq i64 20, [[N_VEC]]
; CHECK-NEXT: br i1 [[TMP12]], label %[[EXIT:.*]], label %[[SCALAR_PH]]
+; CHECK: [[VECTOR_EARLY_EXIT]]:
+; CHECK-NEXT: br label %[[EXIT]]
; CHECK: [[SCALAR_PH]]:
-; CHECK-NEXT: [[BC_RESUME_VAL:%.*]] = phi i64 [ [[TMP11]], %[[MIDDLE_BLOCK]] ], [ 0, %[[ENTRY]] ]
+; CHECK-NEXT: [[BC_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC]], %[[MIDDLE_BLOCK1]] ], [ 0, %[[ENTRY]] ]
; CHECK-NEXT: br label %[[LOOP_HEADER:.*]]
; CHECK: [[LOOP_HEADER]]:
; CHECK-NEXT: [[IV:%.*]] = phi i64 [ [[BC_RESUME_VAL]], %[[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], %[[LOOP_LATCH:.*]] ]
diff --git a/llvm/test/Transforms/LoopVectorize/X86/early_exit_with_stores.ll b/llvm/test/Transforms/LoopVectorize/X86/early_exit_with_stores.ll
index 3635a51a925dad..8ffc6ba186b301 100644
--- a/llvm/test/Transforms/LoopVectorize/X86/early_exit_with_stores.ll
+++ b/llvm/test/Transforms/LoopVectorize/X86/early_exit_with_stores.ll
@@ -12,7 +12,7 @@ define void @interleaved_stores_after_early_exit(ptr noalias %dst, ptr align 2 d
; CHECK: [[LOOP_HEADER]]:
; CHECK-NEXT: br label %[[VECTOR_BODY:.*]]
; CHECK: [[VECTOR_BODY]]:
-; CHECK-NEXT: [[IV:%.*]] = phi i64 [ 0, %[[LOOP_HEADER]] ], [ [[INDEX_NEXT:%.*]], %[[PRED_STORE_CONTINUE6:.*]] ]
+; CHECK-NEXT: [[IV:%.*]] = phi i64 [ 0, %[[LOOP_HEADER]] ], [ [[INDEX_NEXT:%.*]], %[[MIDDLE_BLOCK:.*]] ]
; CHECK-NEXT: [[GEP_SRC:%.*]] = getelementptr i16, ptr [[SRC]], i64 [[IV]]
; CHECK-NEXT: [[WIDE_LOAD:%.*]] = load <4 x i16>, ptr [[GEP_SRC]], align 2
; CHECK-NEXT: [[TMP1:%.*]] = icmp sgt <4 x i16> [[WIDE_LOAD]], splat (i16 500)
@@ -55,7 +55,7 @@ define void @interleaved_stores_after_early_exit(ptr noalias %dst, ptr align 2 d
; CHECK-NEXT: br label %[[PRED_STORE_CONTINUE4]]
; CHECK: [[PRED_STORE_CONTINUE4]]:
; CHECK-NEXT: [[TMP21:%.*]] = extractelement <4 x i1> [[UNCOUNTABLE_EXIT_MASK]], i64 3
-; CHECK-NEXT: br i1 [[TMP21]], label %[[PRED_STORE_IF5:.*]], label %[[PRED_STORE_CONTINUE6]]
+; CHECK-NEXT: br i1 [[TMP21]], label %[[PRED_STORE_IF5:.*]], label %[[PRED_STORE_CONTINUE6:.*]]
; CHECK: [[PRED_STORE_IF5]]:
; CHECK-NEXT: [[TMP22:%.*]] = add i64 [[IV]], 3
; CHECK-NEXT: [[TMP23:%.*]] = shl i64 [[TMP22]], 1
@@ -69,13 +69,15 @@ define void @interleaved_stores_after_early_exit(ptr noalias %dst, ptr align 2 d
; CHECK-NEXT: [[TMP27:%.*]] = call i1 @llvm.vector.reduce.or.v4i1(<4 x i1> [[TMP2]])
; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[IV]], 4
; CHECK-NEXT: [[TMP28:%.*]] = icmp eq i64 [[INDEX_NEXT]], 20
-; CHECK-NEXT: [[TMP29:%.*]] = or i1 [[TMP27]], [[TMP28]]
-; CHECK-NEXT: br i1 [[TMP29]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP0:![0-9]+]]
+; CHECK-NEXT: br i1 [[TMP27]], label %[[VECTOR_EARLY_EXIT:.*]], label %[[MIDDLE_BLOCK]]
; CHECK: [[MIDDLE_BLOCK]]:
-; CHECK-NEXT: [[TMP30:%.*]] = add i64 [[IV]], [[TMP3]]
-; CHECK-NEXT: [[TMP31:%.*]] = icmp eq i64 [[TMP30]], 20
-; CHECK-NEXT: br i1 [[TMP31]], [[EXIT:label %.*]], label %[[SCALAR_PH1:.*]]
+; CHECK-NEXT: br i1 [[TMP28]], label %[[SCALAR_PH1:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP0:![0-9]+]]
; CHECK: [[SCALAR_PH1]]:
+; CHECK-NEXT: br label %[[EXIT:.*]]
+; CHECK: [[VECTOR_EARLY_EXIT]]:
+; CHECK-NEXT: br label %[[EXIT]]
+; CHECK: [[EXIT]]:
+; CHECK-NEXT: ret void
;
entry:
br label %loop.header
@@ -169,7 +171,7 @@ define void @interleaved_stores_before_early_exit(ptr noalias %dst, ptr align 2
; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[IV]], 4
; CHECK-NEXT: [[TMP28:%.*]] = icmp eq i64 [[INDEX_NEXT]], 20
; CHECK-NEXT: [[TMP29:%.*]] = or i1 [[TMP27]], [[TMP28]]
-; CHECK-NEXT: br i1 [[TMP29]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP4:![0-9]+]]
+; CHECK-NEXT: br i1 [[TMP29]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP3:![0-9]+]]
; CHECK: [[MIDDLE_BLOCK]]:
; CHECK-NEXT: [[TMP30:%.*]] = add i64 [[IV]], [[TMP3]]
; CHECK-NEXT: [[TMP31:%.*]] = icmp eq i64 [[TMP30]], 20
diff --git a/llvm/test/Transforms/LoopVectorize/early_exit_with_stores.ll b/llvm/test/Transforms/LoopVectorize/early_exit_with_stores.ll
index 139c9992cc38d7..d4f5ef95e5da73 100644
--- a/llvm/test/Transforms/LoopVectorize/early_exit_with_stores.ll
+++ b/llvm/test/Transforms/LoopVectorize/early_exit_with_stores.ll
@@ -1564,7 +1564,7 @@ define void @store_in_latch_after_exit(ptr align 2 dereferenceable(40) noalias %
; CHECK: [[VECTOR_PH]]:
; CHECK-NEXT: br label %[[VECTOR_BODY:.*]]
; CHECK: [[VECTOR_BODY]]:
-; CHECK-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY_INTERIM:.*]] ]
; CHECK-NEXT: [[TMP0:%.*]] = getelementptr inbounds nuw i16, ptr [[PRED]], i64 [[INDEX]]
; CHECK-NEXT: [[WIDE_LOAD:%.*]] = load <4 x i16>, ptr [[TMP0]], align 2
; CHECK-NEXT: [[TMP1:%.*]] = icmp sgt <4 x i16> [[WIDE_LOAD]], splat (i16 500)
@@ -1577,29 +1577,13 @@ define void @store_in_latch_after_exit(ptr align 2 dereferenceable(40) noalias %
; CHECK-NEXT: [[TMP6:%.*]] = call i1 @llvm.vector.reduce.or.v4i1(<4 x i1> [[TMP2]])
; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 4
; CHECK-NEXT: [[TMP7:%.*]] = icmp eq i64 [[INDEX_NEXT]], 20
-; CHECK-NEXT: [[TMP8:%.*]] = or i1 [[TMP6]], [[TMP7]]
-; CHECK-NEXT: br i1 [[TMP8]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP20:![0-9]+]]
+; CHECK-NEXT: br i1 [[TMP6]], label %[[VECTOR_EARLY_EXIT:.*]], label %[[VECTOR_BODY_INTERIM]]
+; CHECK: [[VECTOR_BODY_INTERIM]]:
+; CHECK-NEXT: br i1 [[TMP7]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP20:![0-9]+]]
; CHECK: [[MIDDLE_BLOCK]]:
-; CHECK-NEXT: [[TMP9:%.*]] = call i64 @llvm.experimental.cttz.elts.i64.v4i1(<4 x i1> [[TMP2]], i1 false)
-; CHECK-NEXT: [[TMP10:%.*]] = add i64 [[INDEX]], [[TMP9]]
-; CHECK-NEXT: [[TMP11:%.*]] = icmp eq i64 [[TMP10]], 20
-; CHECK-NEXT: br i1 [[TMP11]], label %[[EXIT:.*]], label %[[SCALAR_PH:.*]]
-; CHECK: [[SCALAR_PH]]:
-; CHECK-NEXT: br label %[[LOOP:.*]]
-; CHECK: [[LOOP]]:
-; CHECK-NEXT: [[IV:%.*]] = phi i64 [ [[TMP10]], %[[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], %[[LOOP_LATCH:.*]] ]
-; CHECK-NEXT: [[EE_ADDR:%.*]] = getelementptr inbounds nuw i16, ptr [[PRED]], i64 [[IV]]
-; CHECK-NEXT: [[EE_VAL:%.*]] = load i16, ptr [[EE_ADDR]], align 2
-; CHECK-NEXT: [[EE_COND:%.*]] = icmp sgt i16 [[EE_VAL]], 500
-; CHECK-NEXT: br i1 [[EE_COND]], label %[[EXIT]], label %[[LOOP_LATCH]]
-; CHECK: [[LOOP_LATCH]]:
-; CHECK-NEXT: [[ST_ADDR:%.*]] = getelementptr inbounds nuw i16, ptr [[ARRAY]], i64 [[IV]]
-; CHECK-NEXT: [[DATA:%.*]] = load i16, ptr [[ST_ADDR]], align 2
-; CHECK-NEXT: [[INC:%.*]] = add nsw i16 [[DATA]], 1
-; CHECK-NEXT: store i16 [[INC]], ptr [[ST_ADDR]], align 2
-; CHECK-NEXT: [[IV_NEXT]] = add nuw nsw i64 [[IV]], 1
-; CHECK-NEXT: [[COUNTED_COND:%.*]] = icmp eq i64 [[IV_NEXT]], 20
-; CHECK-NEXT: br i1 [[COUNTED_COND]], label %[[EXIT]], label %[[LOOP]], !llvm.loop [[LOOP21:![0-9]+]]
+; CHECK-NEXT: br label %[[EXIT:.*]]
+; CHECK: [[VECTOR_EARLY_EXIT]]:
+; CHECK-NEXT: br label %[[EXIT]]
; CHECK: [[EXIT]]:
; CHECK-NEXT: ret void
;
@@ -1629,32 +1613,49 @@ exit:
define void @conditional_store_after_two_exits(ptr align 2 dereferenceable(40) noalias %array, ptr align 2 dereferenceable(40) readonly %pred, ptr align 2 dereferenceable(40) readonly %pred2) {
; CHECK-LABEL: define void @conditional_store_after_two_exits(
; CHECK-SAME: ptr noalias align 2 dereferenceable(40) [[ARRAY:%.*]], ptr readonly align 2 dereferenceable(40) [[PRED:%.*]], ptr readonly align 2 dereferenceable(40) [[PRED2:%.*]]) {
-; CHECK-NEXT: [[ENTRY:.*]]:
-; CHECK-NEXT: br label %[[LOOP:.*]]
-; CHECK: [[LOOP]]:
-; CHECK-NEXT: [[IV:%.*]] = phi i64 [ 0, %[[ENTRY]] ], [ [[IV_NEXT:%.*]], %[[LOOP_LATCH:.*]] ]
-; CHECK-NEXT: [[EE_ADDR:%.*]] = getelementptr inbounds nuw i16, ptr [[PRED]], i64 [[IV]]
-; CHECK-NEXT: [[EE_VAL:%.*]] = load i16, ptr [[EE_ADDR]], align 2
-; CHECK-NEXT: [[EE2_ADDR:%.*]] = getelementptr inbounds nuw i16, ptr [[PRED2]], i64 [[IV]]
-; CHECK-NEXT: [[EE2_VAL:%.*]] = load i16, ptr [[EE2_ADDR]], align 2
-; CHECK-NEXT: [[EE_COND:%.*]] = icmp sgt i16 [[EE_VAL]], 500
-; CHECK-NEXT: br i1 [[EE_COND]], label %[[EXIT:.*]], label %[[LOOP_2:.*]]
-; CHECK: [[LOOP_2]]:
-; CHECK-NEXT: [[EE2_COND:%.*]] = icmp eq i16 [[EE2_VAL]], 0
-; CHECK-NEXT: br i1 [[EE2_COND]], label %[[EXIT_2:.*]], label %[[LOOP_3:.*]]
-; CHECK: [[LOOP_3]]:
-; CHECK-NEXT: [[ST_ADDR:%.*]] = getelementptr inbounds nuw i16, ptr [[ARRAY]], i64 [[IV]]
-; CHECK-NEXT: [[DATA:%.*]] = load i16, ptr [[ST_ADDR]], align 2
-; CHECK-NEXT: [[ST_COND:%.*]] = icmp slt i16 [[DATA]], 0
-; CHECK-NEXT: br i1 [[ST_COND]], label %[[STORE:.*]], label %[[LOOP_LATCH]]
-; CHECK: [[STORE]]:
-; CHECK-NEXT: [[INC:%.*]] = add i16 [[DATA]], 1
-; CHECK-NEXT: store i16 [[INC]], ptr [[ST_ADDR]], align 2
-; CHECK-NEXT: br label %[[LOOP_LATCH]]
-; CHECK: [[LOOP_LATCH]]:
-; CHECK-NEXT: [[IV_NEXT]] = add nuw nsw i64 [[IV]], 1
-; CHECK-NEXT: [[COUNTED_COND:%.*]] = icmp eq i64 [[IV_NEXT]], 20
-; CHECK-NEXT: br i1 [[COUNTED_COND]], label %[[EXIT]], label %[[LOOP]]
+; CHECK-NEXT: [[ENTRY:.*:]]
+; CHECK-NEXT: br label %[[VECTOR_PH:.*]]
+; CHECK: [[VECTOR_PH]]:
+; CHECK-NEXT: br label %[[VECTOR_BODY:.*]]
+; CHECK: [[VECTOR_BODY]]:
+; CHECK-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY_INTERIM:.*]] ]
+; CHECK-NEXT: [[TMP0:%.*]] = getelementptr inbounds nuw i16, ptr [[PRED]], i64 [[INDEX]]
+; CHECK-NEXT: [[WIDE_LOAD:%.*]] = load <4 x i16>, ptr [[TMP0]], align 2
+; CHECK-NEXT: [[TMP1:%.*]] = getelementptr inbounds nuw i16, ptr [[PRED2]], i64 [[INDEX]]
+; CHECK-NEXT: [[WIDE_LOAD1:%.*]] = load <4 x i16>, ptr [[TMP1]], align 2
+; CHECK-NEXT: [[TMP2:%.*]] = icmp sgt <4 x i16> [[WIDE_LOAD]], splat (i16 500)
+; CHECK-NEXT: [[TMP3:%.*]] = freeze <4 x i1> [[TMP2]]
+; CHECK-NEXT: [[TMP4:%.*]] = call <4 x i1> @llvm.mask.beforefirst.v4i1(<4 x i1> [[TMP3]])
+; CHECK-NEXT: [[TMP5:%.*]] = icmp eq <4 x i16> [[WIDE_LOAD1]], zeroinitializer
+; CHECK-NEXT: [[TMP6:%.*]] = select <4 x i1> [[TMP4]], <4 x i1> [[TMP5]], <4 x i1> zeroinitializer
+; CHECK-NEXT: [[TMP7:%.*]] = freeze <4 x i1> [[TMP6]]
+; CHECK-NEXT: [[TMP8:%.*]] = call <4 x i1> @llvm.mask.beforefirst.v4i1(<4 x i1> [[TMP7]])
+; CHECK-NEXT: [[TMP9:%.*]] = select <4 x i1> [[TMP4]], <4 x i1> [[TMP8]], <4 x i1> zeroinitializer
+; CHECK-NEXT: [[TMP10:%.*]] = getelementptr i16, ptr [[ARRAY]], i64 [[INDEX]]
+; CHECK-NEXT: [[WIDE_MASKED_LOAD:%.*]] = call <4 x i16> @llvm.masked.load.v4i16.p0(ptr align 2 [[TMP10]], <4 x i1> [[TMP9]], <4 x i16> poison)
+; CHECK-NEXT: [[TMP11:%.*]] = icmp slt <4 x i16> [[WIDE_MASKED_LOAD]], zeroinitializer
+; CHECK-NEXT: [[TMP12:%.*]] = freeze <4 x i1> [[TMP11]]
+; CHECK-NEXT: [[TMP13:%.*]] = select <4 x i1> [[TMP9]], <4 x i1> [[TMP12]], <4 x i1> zeroinitializer
+; CHECK-NEXT: [[TMP14:%.*]] = add <4 x i16> [[WIDE_MASKED_LOAD]], splat (i16 1)
+; CHECK-NEXT: call void @llvm.masked.store.v4i16.p0(<4 x i16> [[TMP14]], ptr align 2 [[TMP10]], <4 x i1> [[TMP13]])
+; CHECK-NEXT: [[TMP15:%.*]] = select <4 x i1> [[TMP2]], <4 x i1> splat (i1 true), <4 x i1> [[TMP6]]
+; CHECK-NEXT: [[TMP16:%.*]] = freeze <4 x i1> [[TMP15]]
+; CHECK-NEXT: [[TMP17:%.*]] = call i1 @llvm.vector.reduce.or.v4i1(<4 x i1> [[TMP16]])
+; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 4
+; CHECK-NEXT: [[TMP18:%.*]] = icmp eq i64 [[INDEX_NEXT]], 20
+; CHECK-NEXT: br i1 [[TMP17]], label %[[VECTOR_EARLY_EXIT_CHECK:.*]], label %[[VECTOR_BODY_INTERIM]]
+; CHECK: [[VECTOR_BODY_INTERIM]]:
+; CHECK-NEXT: br i1 [[TMP18]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP21:![0-9]+]]
+; CHECK: [[MIDDLE_BLOCK]]:
+; CHECK-NEXT: br label %[[EXIT:.*]]
+; CHECK: [[VECTOR_EARLY_EXIT_CHECK]]:
+; CHECK-NEXT: [[FIRST_ACTIVE_LANE:%.*]] = call i64 @llvm.experimental.cttz.elts.i64.v4i1(<4 x i1> [[TMP16]], i1 false)
+; CHECK-NEXT: [[TMP19:%.*]] = extractelement <4 x i1> [[TMP2]], i64 [[FIRST_ACTIVE_LANE]]
+; CHECK-NEXT: br i1 [[TMP19]], label %[[VECTOR_EARLY_EXIT_0:.*]], label %[[VECTOR_EARLY_EXIT_1:.*]]
+; CHECK: [[VECTOR_EARLY_EXIT_1]]:
+; CHECK-NEXT: br label %[[EXIT_2:.*]]
+; CHECK: [[VECTOR_EARLY_EXIT_0]]:
+; CHECK-NEXT: br label %[[EXIT]]
; CHECK: [[EXIT]]:
; CHECK-NEXT: ret void
; CHECK: [[EXIT_2]]:
@@ -1707,7 +1708,7 @@ define i16 @store_after_exit_live_out(ptr align 2 dereferenceable(40) noalias %a
; CHECK: [[VECTOR_PH]]:
; CHECK-NEXT: br label %[[VECTOR_BODY:.*]]
; CHECK: [[VECTOR_BODY]]:
-; CHECK-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY_INTERIM:.*]] ]
; CHECK-NEXT: [[TMP0:%.*]] = getelementptr inbounds nuw i16, ptr [[PRED]], i64 [[INDEX]]
; CHECK-NEXT: [[WIDE_LOAD:%.*]] = load <4 x i16>, ptr [[TMP0]], align 2
; CHECK-NEXT: [[TMP1:%.*]] = icmp sgt <4 x i16> [[WIDE_LOAD]], splat (i16 500)
@@ -1720,34 +1721,16 @@ define i16 @store_after_exit_live_out(ptr align 2 dereferenceable(40) noalias %a
; CHECK-NEXT: [[TMP6:%.*]] = call i1 @llvm.vector.reduce.or.v4i1(<4 x i1> [[TMP2]])
; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 4
; CHECK-NEXT: [[TMP7:%.*]] = icmp eq i64 [[INDEX_NEXT]], 20
-; CHECK-NEXT: [[TMP8:%.*]] = or i1 [[TMP6]], [[TMP7]]
-; CHECK-NEXT: br i1 [[TMP8]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP22:![0-9]+]]
+; CHECK-NEXT: br i1 [[TMP6]], label %[[VECTOR_EARLY_EXIT:.*]], label %[[VECTOR_BODY_INTERIM]]
+; CHECK: [[VECTOR_BODY_INTERIM]]:
+; CHECK-NEXT: br i1 [[TMP7]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP22:![0-9]+]]
; CHECK: [[MIDDLE_BLOCK]]:
-; CHECK-NEXT: [[TMP9:%.*]] = extractelement <4 x i16> [[TMP5]], i64 3
-; CHECK-NEXT: [[TMP10:%.*]] = call i64 @llvm.experimental.cttz.elts.i64.v4i1(<4 x i1> [[TMP2]], i1 false)
-; CHECK-NEXT: [[TMP11:%.*]] = add i64 [[INDEX]], [[TMP10]]
-; CHECK-NEXT: [[TMP12:%.*]] = icmp eq i64 [[TMP11]], 20
-; CHECK-NEXT: br i1 [[TMP12]], label %[[EXIT:.*]], label %[[SCALAR_PH:.*]]
-; CHECK: [[SCALAR_PH]]:
-; CHECK-NEXT: br label %[[LOOP:.*]]
-; CHECK: [[LOOP]]:
-; CHECK-NEXT: [[IV:%.*]] = phi i64 [ [[TMP11]], %[[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], %[[LOOP_LATCH:.*]] ]
-; CHECK-NEXT: [[EE_ADDR:%.*]] = getelementptr inbounds nuw i16, ptr [[PRED]], i64 [[IV]]
-; CHECK-NEXT: [[EE_VAL:%.*]] = load i16, ptr [[EE_ADDR]], align 2
-; CHECK-NEXT: [[EE_COND:%.*]] = icmp sgt i16 [[EE_VAL]], 500
-; CHECK-NEXT: br i1 [[EE_COND]], label %[[EXIT]], label %[[LOOP_STORE:.*]]
-; CHECK: [[LOOP_STORE]]:
-; CHECK-NEXT: [[ST_ADDR:%.*]] = getelementptr inbounds nuw i16, ptr [[ARRAY]], i64 [[IV]]
-; CHECK-NEXT: [[DATA:%.*]] = load i16, ptr [[ST_ADDR]], align 2
-; CHECK-NEXT: [[INC:%.*]] = add nsw i16 [[DATA]], 1
-; CHECK-NEXT: store i16 [[INC]], ptr [[ST_ADDR]], align 2
-; CHECK-NEXT: br label %[[LOOP_LATCH]]
-; CHECK: [[LOOP_LATCH]]:
-; CHECK-NEXT: [[IV_NEXT]] = add nuw nsw i64 [[IV]], 1
-; CHECK-NEXT: [[COUNTED_COND:%.*]] = icmp eq i64 [[IV_NEXT]], 20
-; CHECK-NEXT: br i1 [[COUNTED_COND]], label %[[EXIT]], label %[[LOOP]], !llvm.loop [[LOOP23:![0-9]+]]
+; CHECK-NEXT: [[TMP8:%.*]] = extractelement <4 x i16> [[TMP5]], i64 3
+; CHECK-NEXT: br label %[[EXIT:.*]]
+; CHECK: [[VECTOR_EARLY_EXIT]]:
+; CHECK-NEXT: br label %[[EXIT]]
; CHECK: [[EXIT]]:
-; CHECK-NEXT: [[RES:%.*]] = phi i16 [ -1, %[[LOOP]] ], [ [[INC]], %[[LOOP_LATCH]] ], [ [[TMP9]], %[[MIDDLE_BLOCK]] ]
+; CHECK-NEXT: [[RES:%.*]] = phi i16 [ -1, %[[VECTOR_EARLY_EXIT]] ], [ [[TMP8]], %[[MIDDLE_BLOCK]] ]
; CHECK-NEXT: ret i16 [[RES]]
;
entry:
diff --git a/llvm/unittests/Transforms/Vectorize/VPlanVerifierTest.cpp b/llvm/unittests/Transforms/Vectorize/VPlanVerifierTest.cpp
index e006ed7e4bd124..aaf677b894d2a5 100644
--- a/llvm/unittests/Transforms/Vectorize/VPlanVerifierTest.cpp
+++ b/llvm/unittests/Transforms/Vectorize/VPlanVerifierTest.cpp
@@ -578,8 +578,9 @@ TEST_F(VPIRVerifierTest, BranchOnTwoCondsLatchHeaderVerification) {
BasicBlock *LoopHeader = F->getEntryBlock().getSingleSuccessor();
// Build a plain CFG VPlan with BranchOnTwoConds as the latch terminator
// (3 successors), without wrapping blocks in loop regions.
- auto Plan = buildVPlan(LoopHeader, UncountableExitStyle::ReadOnly,
- /*CreateLoopRegions=*/false);
+ auto Plan =
+ buildVPlan(LoopHeader, UncountableExitStyle::MaskedHandleExitInVectorLoop,
+ /*CreateLoopRegions=*/false);
auto *MiddleVPBB =
cast<VPBasicBlock>(Plan->getScalarPreheader()->getPredecessors()[0]);
>From 1952746c4aaee48b0f2d7b828366c37d3a6606d2 Mon Sep 17 00:00:00 2001
From: Luke Lau <luke at igalia.com>
Date: Wed, 23 Sep 2026 16:58:44 +0100
Subject: [PATCH 13/15] [VPlan] Use vputils::findCanonicalIVIncrement in more
places. NFC
In a future patch we want to enable more scenarios with early exits and tail folding. In these scenarios the terminator will no longer be a BranchOnCount/BranchOnCond but a BranchOnTwoConds, so a bunch of canonical IV matchers break. Use vputils::findCanonicalIVIncrement instead since it searches the users of the canonical IV instead of looking at the terminator.
---
.../Transforms/Vectorize/LoopVectorize.cpp | 11 ++--------
.../Vectorize/VPlanConstruction.cpp | 10 ++--------
.../Vectorize/VPlanEVLTailFolding.cpp | 20 +++++++------------
3 files changed, 11 insertions(+), 30 deletions(-)
diff --git a/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp b/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
index 8a4800a80a22f4..9e35bc510892bb 100644
--- a/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
+++ b/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
@@ -6389,15 +6389,8 @@ LoopVectorizationPlanner::tryToBuildVPlan(VPlanPtr Plan, VFRange &Range,
// count is >= increment and a multiple of the increment.
VPRegionBlock *LoopRegion = Plan->getVectorLoopRegion();
bool HasNUW = !IVUpdateMayOverflow || Style == TailFoldingStyle::None;
- if (!HasNUW) {
- auto *IVInc =
- LoopRegion->getExitingBasicBlock()->getTerminator()->getOperand(0);
- assert(match(IVInc,
- m_VPInstruction<Instruction::Add>(
- m_Specific(LoopRegion->getCanonicalIV()), m_VPValue())) &&
- "Did not find the canonical IV increment");
- LoopRegion->clearCanonicalIVNUW(cast<VPInstruction>(IVInc));
- }
+ if (!HasNUW)
+ LoopRegion->clearCanonicalIVNUW(vputils::findCanonicalIVIncrement(*Plan));
// ---------------------------------------------------------------------------
// Pre-construction: record ingredients whose recipes we'll need to further
diff --git a/llvm/lib/Transforms/Vectorize/VPlanConstruction.cpp b/llvm/lib/Transforms/Vectorize/VPlanConstruction.cpp
index db47293ca8476d..ff009b49a79fa6 100644
--- a/llvm/lib/Transforms/Vectorize/VPlanConstruction.cpp
+++ b/llvm/lib/Transforms/Vectorize/VPlanConstruction.cpp
@@ -1393,14 +1393,8 @@ void VPlanTransforms::foldTailByMasking(VPlan &Plan) {
Builder.createNaryOp(VPInstruction::BranchOnCond, HeaderMask);
VPBasicBlock *OrigLatch = LoopRegion->getExitingBasicBlock();
- VPValue *IVInc;
- [[maybe_unused]] bool TermBranchOnCount =
- match(OrigLatch->getTerminator(),
- m_BranchOnCount(m_VPValue(IVInc),
- m_Specific(&Plan.getVectorTripCount())));
- assert(TermBranchOnCount &&
- match(IVInc, m_Add(m_Specific(LoopRegion->getCanonicalIV()),
- m_Specific(&Plan.getVFxUF()))) &&
+ VPValue *IVInc = vputils::findCanonicalIVIncrement(Plan);
+ assert(IVInc &&
std::next(IVInc->getDefiningRecipe()->getIterator()) ==
OrigLatch->getTerminator()->getIterator() &&
"Unexpected canonical iv increment");
diff --git a/llvm/lib/Transforms/Vectorize/VPlanEVLTailFolding.cpp b/llvm/lib/Transforms/Vectorize/VPlanEVLTailFolding.cpp
index 956b92eb07a21e..62ef733d2899dc 100644
--- a/llvm/lib/Transforms/Vectorize/VPlanEVLTailFolding.cpp
+++ b/llvm/lib/Transforms/Vectorize/VPlanEVLTailFolding.cpp
@@ -632,19 +632,13 @@ void VPlanTransforms::convertEVLExitCond(VPlan &Plan) {
if (match(LatchBr, m_BranchOnCond(m_True())))
return;
- VPValue *CanIVInc;
- [[maybe_unused]] bool FoundIncrement = match(
- LatchBr,
- m_BranchOnCond(m_SpecificCmp(CmpInst::ICMP_EQ, m_VPValue(CanIVInc),
- m_Specific(&Plan.getVectorTripCount()))));
- assert(FoundIncrement &&
- match(CanIVInc, m_Add(m_Specific(LoopRegion->getCanonicalIV()),
- m_Specific(&Plan.getVFxUF()))) &&
- "Expected BranchOnCond with ICmp comparing CanIV + VFxUF with vector "
- "trip count");
+ VPValue *CanIVInc = vputils::findCanonicalIVIncrement(Plan);
+ auto *ExitCond = findSingleUser<VPInstruction>(
+ CanIVInc, m_SpecificICmp(CmpInst::ICMP_EQ, m_Specific(CanIVInc),
+ m_Specific(&Plan.getVectorTripCount())));
Type *AVLTy = AVLNext->getScalarType();
- VPBuilder Builder(LatchBr);
- LatchBr->setOperand(
- 0, Builder.createICmp(CmpInst::ICMP_EQ, AVLNext, Plan.getZero(AVLTy)));
+ VPBuilder Builder(ExitCond);
+ ExitCond->replaceAllUsesWith(
+ Builder.createICmp(CmpInst::ICMP_EQ, AVLNext, Plan.getZero(AVLTy)));
}
>From ff2b0965d0011b1ac4cc1f599de5822fdf3da4cc Mon Sep 17 00:00:00 2001
From: Luke Lau <luke at igalia.com>
Date: Mon, 28 Sep 2026 15:34:17 +0100
Subject: [PATCH 14/15] Precommit tests
---
.../LoopVectorize/RISCV/early-exit.ll | 552 ++++++++++++++++++
...arly-exit-tail-folding-active-lane-mask.ll | 65 +++
.../LoopVectorize/early-exit-tail-folding.ll | 524 +++++++++++++++++
3 files changed, 1141 insertions(+)
create mode 100644 llvm/test/Transforms/LoopVectorize/RISCV/early-exit.ll
create mode 100644 llvm/test/Transforms/LoopVectorize/early-exit-tail-folding-active-lane-mask.ll
create mode 100644 llvm/test/Transforms/LoopVectorize/early-exit-tail-folding.ll
diff --git a/llvm/test/Transforms/LoopVectorize/RISCV/early-exit.ll b/llvm/test/Transforms/LoopVectorize/RISCV/early-exit.ll
new file mode 100644
index 00000000000000..1ab2e3461477c8
--- /dev/null
+++ b/llvm/test/Transforms/LoopVectorize/RISCV/early-exit.ll
@@ -0,0 +1,552 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --check-globals none --version 6
+; RUN: opt -S < %s -p loop-vectorize -mtriple riscv64 -mattr=+v -enable-early-exit-vectorization-with-side-effects | FileCheck %s
+
+define i32 @single_early_exit(ptr align 4 dereferenceable(4096) %p) {
+; CHECK-LABEL: define i32 @single_early_exit(
+; CHECK-SAME: ptr align 4 dereferenceable(4096) [[P:%.*]]) #[[ATTR0:[0-9]+]] {
+; CHECK-NEXT: [[ENTRY:.*]]:
+; CHECK-NEXT: [[TMP0:%.*]] = call i32 @llvm.vscale.i32()
+; CHECK-NEXT: [[TMP1:%.*]] = shl nuw i32 [[TMP0]], 2
+; CHECK-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ult i32 123, [[TMP1]]
+; CHECK-NEXT: br i1 [[MIN_ITERS_CHECK]], label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]]
+; CHECK: [[VECTOR_PH]]:
+; CHECK-NEXT: [[N_MOD_VF:%.*]] = urem i32 123, [[TMP1]]
+; CHECK-NEXT: [[N_VEC:%.*]] = sub i32 123, [[N_MOD_VF]]
+; CHECK-NEXT: br label %[[VECTOR_BODY:.*]]
+; CHECK: [[VECTOR_BODY]]:
+; CHECK-NEXT: [[INDEX:%.*]] = phi i32 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY_INTERIM:.*]] ]
+; CHECK-NEXT: [[TMP2:%.*]] = getelementptr inbounds i32, ptr [[P]], i32 [[INDEX]]
+; CHECK-NEXT: [[WIDE_LOAD:%.*]] = load <vscale x 4 x i32>, ptr [[TMP2]], align 4
+; CHECK-NEXT: [[TMP3:%.*]] = icmp eq <vscale x 4 x i32> [[WIDE_LOAD]], zeroinitializer
+; CHECK-NEXT: [[TMP4:%.*]] = freeze <vscale x 4 x i1> [[TMP3]]
+; CHECK-NEXT: [[TMP5:%.*]] = call i1 @llvm.vector.reduce.or.nxv4i1(<vscale x 4 x i1> [[TMP4]])
+; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i32 [[INDEX]], [[TMP1]]
+; CHECK-NEXT: [[TMP6:%.*]] = icmp eq i32 [[INDEX_NEXT]], [[N_VEC]]
+; CHECK-NEXT: br i1 [[TMP5]], label %[[VECTOR_EARLY_EXIT:.*]], label %[[VECTOR_BODY_INTERIM]]
+; CHECK: [[VECTOR_BODY_INTERIM]]:
+; CHECK-NEXT: br i1 [[TMP6]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP0:![0-9]+]]
+; CHECK: [[MIDDLE_BLOCK]]:
+; CHECK-NEXT: [[CMP_N:%.*]] = icmp eq i32 123, [[N_VEC]]
+; CHECK-NEXT: br i1 [[CMP_N]], label %[[EXIT:.*]], label %[[SCALAR_PH]]
+; CHECK: [[VECTOR_EARLY_EXIT]]:
+; CHECK-NEXT: [[TMP7:%.*]] = call i64 @llvm.experimental.cttz.elts.i64.nxv4i1(<vscale x 4 x i1> [[TMP4]], i1 false)
+; CHECK-NEXT: [[TMP8:%.*]] = trunc i64 [[TMP7]] to i32
+; CHECK-NEXT: [[TMP9:%.*]] = add i32 [[INDEX]], [[TMP8]]
+; CHECK-NEXT: br label %[[EXIT]]
+; CHECK: [[SCALAR_PH]]:
+; CHECK-NEXT: [[BC_RESUME_VAL:%.*]] = phi i32 [ [[N_VEC]], %[[MIDDLE_BLOCK]] ], [ 0, %[[ENTRY]] ]
+; CHECK-NEXT: br label %[[LOOP:.*]]
+; CHECK: [[LOOP]]:
+; CHECK-NEXT: [[IV:%.*]] = phi i32 [ [[BC_RESUME_VAL]], %[[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], %[[LATCH:.*]] ]
+; CHECK-NEXT: [[GEP:%.*]] = getelementptr inbounds i32, ptr [[P]], i32 [[IV]]
+; CHECK-NEXT: [[LD:%.*]] = load i32, ptr [[GEP]], align 4
+; CHECK-NEXT: [[C:%.*]] = icmp eq i32 [[LD]], 0
+; CHECK-NEXT: br i1 [[C]], label %[[EXIT]], label %[[LATCH]]
+; CHECK: [[LATCH]]:
+; CHECK-NEXT: [[IV_NEXT]] = add nuw nsw i32 [[IV]], 1
+; CHECK-NEXT: [[EC:%.*]] = icmp eq i32 [[IV_NEXT]], 123
+; CHECK-NEXT: br i1 [[EC]], label %[[EXIT]], label %[[LOOP]], !llvm.loop [[LOOP3:![0-9]+]]
+; CHECK: [[EXIT]]:
+; CHECK-NEXT: [[RES:%.*]] = phi i32 [ [[IV]], %[[LOOP]] ], [ -1, %[[LATCH]] ], [ -1, %[[MIDDLE_BLOCK]] ], [ [[TMP9]], %[[VECTOR_EARLY_EXIT]] ]
+; CHECK-NEXT: ret i32 [[RES]]
+;
+entry:
+ br label %loop
+
+loop:
+ %iv = phi i32 [ 0, %entry ], [ %iv.next, %latch ]
+ %gep = getelementptr inbounds i32, ptr %p, i32 %iv
+ %ld = load i32, ptr %gep
+ %c = icmp eq i32 %ld, 0
+ br i1 %c, label %exit, label %latch
+
+latch:
+ %iv.next = add nuw nsw i32 %iv, 1
+ %ec = icmp eq i32 %iv.next, 123
+ br i1 %ec, label %exit, label %loop
+
+exit:
+ %res = phi i32 [ %iv, %loop ], [ -1, %latch ]
+ ret i32 %res
+}
+
+define i32 @multi_early_exits_same_exits(ptr align 4 dereferenceable(4096) %p, ptr align 4 dereferenceable(4096) %q) {
+; CHECK-LABEL: define i32 @multi_early_exits_same_exits(
+; CHECK-SAME: ptr align 4 dereferenceable(4096) [[P:%.*]], ptr align 4 dereferenceable(4096) [[Q:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT: [[ENTRY:.*]]:
+; CHECK-NEXT: [[TMP0:%.*]] = call i32 @llvm.vscale.i32()
+; CHECK-NEXT: [[TMP1:%.*]] = shl nuw i32 [[TMP0]], 2
+; CHECK-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ult i32 123, [[TMP1]]
+; CHECK-NEXT: br i1 [[MIN_ITERS_CHECK]], label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]]
+; CHECK: [[VECTOR_PH]]:
+; CHECK-NEXT: [[N_MOD_VF:%.*]] = urem i32 123, [[TMP1]]
+; CHECK-NEXT: [[N_VEC:%.*]] = sub i32 123, [[N_MOD_VF]]
+; CHECK-NEXT: [[TMP2:%.*]] = call <vscale x 4 x i32> @llvm.stepvector.nxv4i32()
+; CHECK-NEXT: [[BROADCAST_SPLATINSERT:%.*]] = insertelement <vscale x 4 x i32> poison, i32 [[TMP1]], i64 0
+; CHECK-NEXT: [[BROADCAST_SPLAT:%.*]] = shufflevector <vscale x 4 x i32> [[BROADCAST_SPLATINSERT]], <vscale x 4 x i32> poison, <vscale x 4 x i32> zeroinitializer
+; CHECK-NEXT: br label %[[VECTOR_BODY:.*]]
+; CHECK: [[VECTOR_BODY]]:
+; CHECK-NEXT: [[INDEX:%.*]] = phi i32 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY_INTERIM:.*]] ]
+; CHECK-NEXT: [[VEC_IND:%.*]] = phi <vscale x 4 x i32> [ [[TMP2]], %[[VECTOR_PH]] ], [ [[VEC_IND_NEXT:%.*]], %[[VECTOR_BODY_INTERIM]] ]
+; CHECK-NEXT: [[TMP3:%.*]] = getelementptr inbounds i32, ptr [[P]], i32 [[INDEX]]
+; CHECK-NEXT: [[WIDE_LOAD:%.*]] = load <vscale x 4 x i32>, ptr [[TMP3]], align 4
+; CHECK-NEXT: [[TMP4:%.*]] = icmp eq <vscale x 4 x i32> [[WIDE_LOAD]], zeroinitializer
+; CHECK-NEXT: [[TMP5:%.*]] = getelementptr inbounds i32, ptr [[Q]], i32 [[INDEX]]
+; CHECK-NEXT: [[WIDE_LOAD1:%.*]] = load <vscale x 4 x i32>, ptr [[TMP5]], align 4
+; CHECK-NEXT: [[TMP6:%.*]] = icmp eq <vscale x 4 x i32> [[WIDE_LOAD1]], zeroinitializer
+; CHECK-NEXT: [[TMP7:%.*]] = add <vscale x 4 x i32> [[VEC_IND]], splat (i32 1000)
+; CHECK-NEXT: [[TMP8:%.*]] = select <vscale x 4 x i1> [[TMP4]], <vscale x 4 x i1> splat (i1 true), <vscale x 4 x i1> [[TMP6]]
+; CHECK-NEXT: [[TMP9:%.*]] = freeze <vscale x 4 x i1> [[TMP8]]
+; CHECK-NEXT: [[TMP10:%.*]] = call i1 @llvm.vector.reduce.or.nxv4i1(<vscale x 4 x i1> [[TMP9]])
+; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i32 [[INDEX]], [[TMP1]]
+; CHECK-NEXT: [[TMP11:%.*]] = icmp eq i32 [[INDEX_NEXT]], [[N_VEC]]
+; CHECK-NEXT: [[VEC_IND_NEXT]] = add nuw nsw <vscale x 4 x i32> [[VEC_IND]], [[BROADCAST_SPLAT]]
+; CHECK-NEXT: br i1 [[TMP10]], label %[[VECTOR_EARLY_EXIT_CHECK:.*]], label %[[VECTOR_BODY_INTERIM]]
+; CHECK: [[VECTOR_BODY_INTERIM]]:
+; CHECK-NEXT: br i1 [[TMP11]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP4:![0-9]+]]
+; CHECK: [[MIDDLE_BLOCK]]:
+; CHECK-NEXT: [[CMP_N:%.*]] = icmp eq i32 123, [[N_VEC]]
+; CHECK-NEXT: br i1 [[CMP_N]], label %[[EXIT:.*]], label %[[SCALAR_PH]]
+; CHECK: [[VECTOR_EARLY_EXIT_CHECK]]:
+; CHECK-NEXT: [[FIRST_ACTIVE_LANE:%.*]] = call i64 @llvm.experimental.cttz.elts.i64.nxv4i1(<vscale x 4 x i1> [[TMP9]], i1 false)
+; CHECK-NEXT: [[TMP12:%.*]] = extractelement <vscale x 4 x i1> [[TMP4]], i64 [[FIRST_ACTIVE_LANE]]
+; CHECK-NEXT: br i1 [[TMP12]], label %[[VECTOR_EARLY_EXIT_0:.*]], label %[[VECTOR_EARLY_EXIT_1:.*]]
+; CHECK: [[VECTOR_EARLY_EXIT_1]]:
+; CHECK-NEXT: [[TMP13:%.*]] = extractelement <vscale x 4 x i32> [[TMP7]], i64 [[FIRST_ACTIVE_LANE]]
+; CHECK-NEXT: br label %[[EXIT]]
+; CHECK: [[VECTOR_EARLY_EXIT_0]]:
+; CHECK-NEXT: [[TMP14:%.*]] = trunc i64 [[FIRST_ACTIVE_LANE]] to i32
+; CHECK-NEXT: [[TMP15:%.*]] = add i32 [[INDEX]], [[TMP14]]
+; CHECK-NEXT: br label %[[EXIT]]
+; CHECK: [[SCALAR_PH]]:
+; CHECK-NEXT: [[BC_RESUME_VAL:%.*]] = phi i32 [ [[N_VEC]], %[[MIDDLE_BLOCK]] ], [ 0, %[[ENTRY]] ]
+; CHECK-NEXT: br label %[[LOOP:.*]]
+; CHECK: [[LOOP]]:
+; CHECK-NEXT: [[IV:%.*]] = phi i32 [ [[BC_RESUME_VAL]], %[[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], %[[LATCH:.*]] ]
+; CHECK-NEXT: [[GEP_P:%.*]] = getelementptr inbounds i32, ptr [[P]], i32 [[IV]]
+; CHECK-NEXT: [[X:%.*]] = load i32, ptr [[GEP_P]], align 4
+; CHECK-NEXT: [[C1:%.*]] = icmp eq i32 [[X]], 0
+; CHECK-NEXT: br i1 [[C1]], label %[[EXIT]], label %[[NEXT:.*]]
+; CHECK: [[NEXT]]:
+; CHECK-NEXT: [[GEP_Q:%.*]] = getelementptr inbounds i32, ptr [[Q]], i32 [[IV]]
+; CHECK-NEXT: [[Y:%.*]] = load i32, ptr [[GEP_Q]], align 4
+; CHECK-NEXT: [[C2:%.*]] = icmp eq i32 [[Y]], 0
+; CHECK-NEXT: [[IV_Q:%.*]] = add i32 [[IV]], 1000
+; CHECK-NEXT: br i1 [[C2]], label %[[EXIT]], label %[[LATCH]]
+; CHECK: [[LATCH]]:
+; CHECK-NEXT: [[IV_NEXT]] = add nuw nsw i32 [[IV]], 1
+; CHECK-NEXT: [[EC:%.*]] = icmp eq i32 [[IV_NEXT]], 123
+; CHECK-NEXT: br i1 [[EC]], label %[[EXIT]], label %[[LOOP]], !llvm.loop [[LOOP5:![0-9]+]]
+; CHECK: [[EXIT]]:
+; CHECK-NEXT: [[RES:%.*]] = phi i32 [ [[IV]], %[[LOOP]] ], [ [[IV_Q]], %[[NEXT]] ], [ -1, %[[LATCH]] ], [ -1, %[[MIDDLE_BLOCK]] ], [ [[TMP15]], %[[VECTOR_EARLY_EXIT_0]] ], [ [[TMP13]], %[[VECTOR_EARLY_EXIT_1]] ]
+; CHECK-NEXT: ret i32 [[RES]]
+;
+entry:
+ br label %loop
+
+loop:
+ %iv = phi i32 [ 0, %entry ], [ %iv.next, %latch ]
+ %gep.p = getelementptr inbounds i32, ptr %p, i32 %iv
+ %x = load i32, ptr %gep.p, align 4
+ %c1 = icmp eq i32 %x, 0
+ br i1 %c1, label %exit, label %next
+
+next:
+ %gep.q = getelementptr inbounds i32, ptr %q, i32 %iv
+ %y = load i32, ptr %gep.q, align 4
+ %c2 = icmp eq i32 %y, 0
+ %iv.q = add i32 %iv, 1000
+ br i1 %c2, label %exit, label %latch
+
+latch:
+ %iv.next = add nuw nsw i32 %iv, 1
+ %ec = icmp eq i32 %iv.next, 123
+ br i1 %ec, label %exit, label %loop
+
+exit:
+ %res = phi i32 [ %iv, %loop ], [ %iv.q, %next ], [ -1, %latch ]
+ ret i32 %res
+}
+
+define i32 @multi_early_exits_different_exits(ptr align 4 dereferenceable(4096) %p, ptr align 4 dereferenceable(4096) %q) {
+; CHECK-LABEL: define i32 @multi_early_exits_different_exits(
+; CHECK-SAME: ptr align 4 dereferenceable(4096) [[P:%.*]], ptr align 4 dereferenceable(4096) [[Q:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT: [[ENTRY:.*]]:
+; CHECK-NEXT: [[TMP0:%.*]] = call i32 @llvm.vscale.i32()
+; CHECK-NEXT: [[TMP1:%.*]] = shl nuw i32 [[TMP0]], 2
+; CHECK-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ult i32 123, [[TMP1]]
+; CHECK-NEXT: br i1 [[MIN_ITERS_CHECK]], label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]]
+; CHECK: [[VECTOR_PH]]:
+; CHECK-NEXT: [[N_MOD_VF:%.*]] = urem i32 123, [[TMP1]]
+; CHECK-NEXT: [[N_VEC:%.*]] = sub i32 123, [[N_MOD_VF]]
+; CHECK-NEXT: [[TMP2:%.*]] = call <vscale x 4 x i32> @llvm.stepvector.nxv4i32()
+; CHECK-NEXT: [[BROADCAST_SPLATINSERT:%.*]] = insertelement <vscale x 4 x i32> poison, i32 [[TMP1]], i64 0
+; CHECK-NEXT: [[BROADCAST_SPLAT:%.*]] = shufflevector <vscale x 4 x i32> [[BROADCAST_SPLATINSERT]], <vscale x 4 x i32> poison, <vscale x 4 x i32> zeroinitializer
+; CHECK-NEXT: br label %[[VECTOR_BODY:.*]]
+; CHECK: [[VECTOR_BODY]]:
+; CHECK-NEXT: [[INDEX:%.*]] = phi i32 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY_INTERIM:.*]] ]
+; CHECK-NEXT: [[VEC_IND:%.*]] = phi <vscale x 4 x i32> [ [[TMP2]], %[[VECTOR_PH]] ], [ [[VEC_IND_NEXT:%.*]], %[[VECTOR_BODY_INTERIM]] ]
+; CHECK-NEXT: [[TMP3:%.*]] = getelementptr inbounds i32, ptr [[P]], i32 [[INDEX]]
+; CHECK-NEXT: [[WIDE_LOAD:%.*]] = load <vscale x 4 x i32>, ptr [[TMP3]], align 4
+; CHECK-NEXT: [[TMP4:%.*]] = icmp eq <vscale x 4 x i32> [[WIDE_LOAD]], zeroinitializer
+; CHECK-NEXT: [[TMP5:%.*]] = getelementptr inbounds i32, ptr [[Q]], i32 [[INDEX]]
+; CHECK-NEXT: [[WIDE_LOAD1:%.*]] = load <vscale x 4 x i32>, ptr [[TMP5]], align 4
+; CHECK-NEXT: [[TMP6:%.*]] = icmp eq <vscale x 4 x i32> [[WIDE_LOAD1]], zeroinitializer
+; CHECK-NEXT: [[TMP7:%.*]] = add <vscale x 4 x i32> [[VEC_IND]], splat (i32 1000)
+; CHECK-NEXT: [[TMP8:%.*]] = select <vscale x 4 x i1> [[TMP4]], <vscale x 4 x i1> splat (i1 true), <vscale x 4 x i1> [[TMP6]]
+; CHECK-NEXT: [[TMP9:%.*]] = freeze <vscale x 4 x i1> [[TMP8]]
+; CHECK-NEXT: [[TMP10:%.*]] = call i1 @llvm.vector.reduce.or.nxv4i1(<vscale x 4 x i1> [[TMP9]])
+; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i32 [[INDEX]], [[TMP1]]
+; CHECK-NEXT: [[TMP11:%.*]] = icmp eq i32 [[INDEX_NEXT]], [[N_VEC]]
+; CHECK-NEXT: [[VEC_IND_NEXT]] = add nuw nsw <vscale x 4 x i32> [[VEC_IND]], [[BROADCAST_SPLAT]]
+; CHECK-NEXT: br i1 [[TMP10]], label %[[VECTOR_EARLY_EXIT_CHECK:.*]], label %[[VECTOR_BODY_INTERIM]]
+; CHECK: [[VECTOR_BODY_INTERIM]]:
+; CHECK-NEXT: br i1 [[TMP11]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP6:![0-9]+]]
+; CHECK: [[MIDDLE_BLOCK]]:
+; CHECK-NEXT: [[CMP_N:%.*]] = icmp eq i32 123, [[N_VEC]]
+; CHECK-NEXT: br i1 [[CMP_N]], label %[[EXIT_LATCH:.*]], label %[[SCALAR_PH]]
+; CHECK: [[VECTOR_EARLY_EXIT_CHECK]]:
+; CHECK-NEXT: [[FIRST_ACTIVE_LANE:%.*]] = call i64 @llvm.experimental.cttz.elts.i64.nxv4i1(<vscale x 4 x i1> [[TMP9]], i1 false)
+; CHECK-NEXT: [[TMP12:%.*]] = extractelement <vscale x 4 x i1> [[TMP4]], i64 [[FIRST_ACTIVE_LANE]]
+; CHECK-NEXT: br i1 [[TMP12]], label %[[VECTOR_EARLY_EXIT_0:.*]], label %[[VECTOR_EARLY_EXIT_1:.*]]
+; CHECK: [[VECTOR_EARLY_EXIT_1]]:
+; CHECK-NEXT: [[TMP13:%.*]] = extractelement <vscale x 4 x i32> [[TMP7]], i64 [[FIRST_ACTIVE_LANE]]
+; CHECK-NEXT: br label %[[EXIT_Q:.*]]
+; CHECK: [[VECTOR_EARLY_EXIT_0]]:
+; CHECK-NEXT: [[TMP14:%.*]] = trunc i64 [[FIRST_ACTIVE_LANE]] to i32
+; CHECK-NEXT: [[TMP15:%.*]] = add i32 [[INDEX]], [[TMP14]]
+; CHECK-NEXT: br label %[[EXIT_P:.*]]
+; CHECK: [[SCALAR_PH]]:
+; CHECK-NEXT: [[BC_RESUME_VAL:%.*]] = phi i32 [ [[N_VEC]], %[[MIDDLE_BLOCK]] ], [ 0, %[[ENTRY]] ]
+; CHECK-NEXT: br label %[[LOOP:.*]]
+; CHECK: [[LOOP]]:
+; CHECK-NEXT: [[IV:%.*]] = phi i32 [ [[BC_RESUME_VAL]], %[[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], %[[LATCH:.*]] ]
+; CHECK-NEXT: [[GEP_P:%.*]] = getelementptr inbounds i32, ptr [[P]], i32 [[IV]]
+; CHECK-NEXT: [[X:%.*]] = load i32, ptr [[GEP_P]], align 4
+; CHECK-NEXT: [[C1:%.*]] = icmp eq i32 [[X]], 0
+; CHECK-NEXT: br i1 [[C1]], label %[[EXIT_P]], label %[[NEXT:.*]]
+; CHECK: [[NEXT]]:
+; CHECK-NEXT: [[GEP_Q:%.*]] = getelementptr inbounds i32, ptr [[Q]], i32 [[IV]]
+; CHECK-NEXT: [[Y:%.*]] = load i32, ptr [[GEP_Q]], align 4
+; CHECK-NEXT: [[C2:%.*]] = icmp eq i32 [[Y]], 0
+; CHECK-NEXT: [[IV_Q:%.*]] = add i32 [[IV]], 1000
+; CHECK-NEXT: br i1 [[C2]], label %[[EXIT_Q]], label %[[LATCH]]
+; CHECK: [[LATCH]]:
+; CHECK-NEXT: [[IV_NEXT]] = add nuw nsw i32 [[IV]], 1
+; CHECK-NEXT: [[EC:%.*]] = icmp eq i32 [[IV_NEXT]], 123
+; CHECK-NEXT: br i1 [[EC]], label %[[EXIT_LATCH]], label %[[LOOP]], !llvm.loop [[LOOP7:![0-9]+]]
+; CHECK: [[EXIT_P]]:
+; CHECK-NEXT: [[RES_P:%.*]] = phi i32 [ [[IV]], %[[LOOP]] ], [ [[TMP15]], %[[VECTOR_EARLY_EXIT_0]] ]
+; CHECK-NEXT: ret i32 [[RES_P]]
+; CHECK: [[EXIT_Q]]:
+; CHECK-NEXT: [[RES_Q:%.*]] = phi i32 [ [[IV_Q]], %[[NEXT]] ], [ [[TMP13]], %[[VECTOR_EARLY_EXIT_1]] ]
+; CHECK-NEXT: ret i32 [[RES_Q]]
+; CHECK: [[EXIT_LATCH]]:
+; CHECK-NEXT: ret i32 -1
+;
+entry:
+ br label %loop
+
+loop:
+ %iv = phi i32 [ 0, %entry ], [ %iv.next, %latch ]
+ %gep.p = getelementptr inbounds i32, ptr %p, i32 %iv
+ %x = load i32, ptr %gep.p, align 4
+ %c1 = icmp eq i32 %x, 0
+ br i1 %c1, label %exit.p, label %next
+
+next:
+ %gep.q = getelementptr inbounds i32, ptr %q, i32 %iv
+ %y = load i32, ptr %gep.q, align 4
+ %c2 = icmp eq i32 %y, 0
+ %iv.q = add i32 %iv, 1000
+ br i1 %c2, label %exit.q, label %latch
+
+latch:
+ %iv.next = add nuw nsw i32 %iv, 1
+ %ec = icmp eq i32 %iv.next, 123
+ br i1 %ec, label %exit.latch, label %loop
+
+exit.p:
+ %res.p = phi i32 [ %iv, %loop ]
+ ret i32 %res.p
+
+exit.q:
+ %res.q = phi i32 [ %iv.q, %next ]
+ ret i32 %res.q
+
+exit.latch:
+ ret i32 -1
+}
+
+define i32 @predicated_early_exit(i1 %c0, ptr align 4 dereferenceable(4096) %p) {
+; CHECK-LABEL: define i32 @predicated_early_exit(
+; CHECK-SAME: i1 [[C0:%.*]], ptr align 4 dereferenceable(4096) [[P:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT: [[ENTRY:.*]]:
+; CHECK-NEXT: [[TMP0:%.*]] = call i32 @llvm.vscale.i32()
+; CHECK-NEXT: [[TMP1:%.*]] = shl nuw i32 [[TMP0]], 2
+; CHECK-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ult i32 123, [[TMP1]]
+; CHECK-NEXT: br i1 [[MIN_ITERS_CHECK]], label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]]
+; CHECK: [[VECTOR_PH]]:
+; CHECK-NEXT: [[N_MOD_VF:%.*]] = urem i32 123, [[TMP1]]
+; CHECK-NEXT: [[N_VEC:%.*]] = sub i32 123, [[N_MOD_VF]]
+; CHECK-NEXT: [[BROADCAST_SPLATINSERT:%.*]] = insertelement <vscale x 4 x i1> poison, i1 [[C0]], i64 0
+; CHECK-NEXT: [[BROADCAST_SPLAT:%.*]] = shufflevector <vscale x 4 x i1> [[BROADCAST_SPLATINSERT]], <vscale x 4 x i1> poison, <vscale x 4 x i32> zeroinitializer
+; CHECK-NEXT: [[TMP11:%.*]] = freeze <vscale x 4 x i1> [[BROADCAST_SPLAT]]
+; CHECK-NEXT: br label %[[VECTOR_BODY:.*]]
+; CHECK: [[VECTOR_BODY]]:
+; CHECK-NEXT: [[INDEX:%.*]] = phi i32 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY_INTERIM:.*]] ]
+; CHECK-NEXT: [[TMP2:%.*]] = getelementptr inbounds i32, ptr [[P]], i32 [[INDEX]]
+; CHECK-NEXT: [[WIDE_LOAD:%.*]] = load <vscale x 4 x i32>, ptr [[TMP2]], align 4
+; CHECK-NEXT: [[TMP3:%.*]] = icmp eq <vscale x 4 x i32> [[WIDE_LOAD]], zeroinitializer
+; CHECK-NEXT: [[TMP4:%.*]] = select <vscale x 4 x i1> [[TMP11]], <vscale x 4 x i1> [[TMP3]], <vscale x 4 x i1> zeroinitializer
+; CHECK-NEXT: [[TMP5:%.*]] = freeze <vscale x 4 x i1> [[TMP4]]
+; CHECK-NEXT: [[TMP6:%.*]] = call i1 @llvm.vector.reduce.or.nxv4i1(<vscale x 4 x i1> [[TMP5]])
+; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i32 [[INDEX]], [[TMP1]]
+; CHECK-NEXT: [[TMP7:%.*]] = icmp eq i32 [[INDEX_NEXT]], [[N_VEC]]
+; CHECK-NEXT: br i1 [[TMP6]], label %[[VECTOR_EARLY_EXIT:.*]], label %[[VECTOR_BODY_INTERIM]]
+; CHECK: [[VECTOR_BODY_INTERIM]]:
+; CHECK-NEXT: br i1 [[TMP7]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP8:![0-9]+]]
+; CHECK: [[MIDDLE_BLOCK]]:
+; CHECK-NEXT: [[CMP_N:%.*]] = icmp eq i32 123, [[N_VEC]]
+; CHECK-NEXT: br i1 [[CMP_N]], label %[[EXIT:.*]], label %[[SCALAR_PH]]
+; CHECK: [[VECTOR_EARLY_EXIT]]:
+; CHECK-NEXT: [[TMP8:%.*]] = call i64 @llvm.experimental.cttz.elts.i64.nxv4i1(<vscale x 4 x i1> [[TMP5]], i1 false)
+; CHECK-NEXT: [[TMP9:%.*]] = trunc i64 [[TMP8]] to i32
+; CHECK-NEXT: [[TMP10:%.*]] = add i32 [[INDEX]], [[TMP9]]
+; CHECK-NEXT: br label %[[EXIT]]
+; CHECK: [[SCALAR_PH]]:
+; CHECK-NEXT: [[BC_RESUME_VAL:%.*]] = phi i32 [ [[N_VEC]], %[[MIDDLE_BLOCK]] ], [ 0, %[[ENTRY]] ]
+; CHECK-NEXT: br label %[[LOOP:.*]]
+; CHECK: [[LOOP]]:
+; CHECK-NEXT: [[IV:%.*]] = phi i32 [ [[BC_RESUME_VAL]], %[[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], %[[LATCH:.*]] ]
+; CHECK-NEXT: br i1 [[C0]], label %[[THEN:.*]], label %[[LATCH]]
+; CHECK: [[THEN]]:
+; CHECK-NEXT: [[GEP:%.*]] = getelementptr inbounds i32, ptr [[P]], i32 [[IV]]
+; CHECK-NEXT: [[LD:%.*]] = load i32, ptr [[GEP]], align 4
+; CHECK-NEXT: [[C1:%.*]] = icmp eq i32 [[LD]], 0
+; CHECK-NEXT: br i1 [[C1]], label %[[EXIT]], label %[[LATCH]]
+; CHECK: [[LATCH]]:
+; CHECK-NEXT: [[IV_NEXT]] = add nuw nsw i32 [[IV]], 1
+; CHECK-NEXT: [[EC:%.*]] = icmp eq i32 [[IV_NEXT]], 123
+; CHECK-NEXT: br i1 [[EC]], label %[[EXIT]], label %[[LOOP]], !llvm.loop [[LOOP9:![0-9]+]]
+; CHECK: [[EXIT]]:
+; CHECK-NEXT: [[RES:%.*]] = phi i32 [ [[IV]], %[[THEN]] ], [ -1, %[[LATCH]] ], [ -1, %[[MIDDLE_BLOCK]] ], [ [[TMP10]], %[[VECTOR_EARLY_EXIT]] ]
+; CHECK-NEXT: ret i32 [[RES]]
+;
+entry:
+ br label %loop
+
+loop:
+ %iv = phi i32 [ 0, %entry ], [ %iv.next, %latch ]
+ br i1 %c0, label %then, label %latch
+
+then:
+ %gep = getelementptr inbounds i32, ptr %p, i32 %iv
+ %ld = load i32, ptr %gep
+ %c1 = icmp eq i32 %ld, 0
+ br i1 %c1, label %exit, label %latch
+
+latch:
+ %iv.next = add nuw nsw i32 %iv, 1
+ %ec = icmp eq i32 %iv.next, 123
+ br i1 %ec, label %exit, label %loop
+
+exit:
+ %res = phi i32 [ %iv, %then ], [ -1, %latch ]
+ ret i32 %res
+}
+
+; TODO: Cannot tail fold yet.
+define i32 @countable_early_exit(ptr align 4 dereferenceable(4096) %p, i32 %n) {
+; CHECK-LABEL: define i32 @countable_early_exit(
+; CHECK-SAME: ptr align 4 dereferenceable(4096) [[P:%.*]], i32 [[N:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT: [[ENTRY:.*]]:
+; CHECK-NEXT: br label %[[LOOP:.*]]
+; CHECK: [[LOOP]]:
+; CHECK-NEXT: [[IV:%.*]] = phi i32 [ 0, %[[ENTRY]] ], [ [[IV_NEXT:%.*]], %[[LATCH:.*]] ]
+; CHECK-NEXT: [[C:%.*]] = icmp eq i32 [[IV]], [[N]]
+; CHECK-NEXT: br i1 [[C]], label %[[EXIT:.*]], label %[[LATCH]]
+; CHECK: [[LATCH]]:
+; CHECK-NEXT: [[GEP:%.*]] = getelementptr inbounds i32, ptr [[P]], i32 [[IV]]
+; CHECK-NEXT: [[LD:%.*]] = load i32, ptr [[GEP]], align 4
+; CHECK-NEXT: [[IV_NEXT]] = add nuw nsw i32 [[IV]], 1
+; CHECK-NEXT: [[EC:%.*]] = icmp eq i32 [[IV_NEXT]], 123
+; CHECK-NEXT: br i1 [[EC]], label %[[EXIT]], label %[[LOOP]]
+; CHECK: [[EXIT]]:
+; CHECK-NEXT: [[RES:%.*]] = phi i32 [ 0, %[[LOOP]] ], [ [[LD]], %[[LATCH]] ]
+; CHECK-NEXT: ret i32 [[RES]]
+;
+entry:
+ br label %loop
+
+loop:
+ %iv = phi i32 [ 0, %entry ], [ %iv.next, %latch ]
+ %c = icmp eq i32 %iv, %n
+ br i1 %c, label %exit, label %latch
+
+latch:
+ %gep = getelementptr inbounds i32, ptr %p, i32 %iv
+ %ld = load i32, ptr %gep, align 4
+ %iv.next = add nuw nsw i32 %iv, 1
+ %ec = icmp eq i32 %iv.next, 123
+ br i1 %ec, label %exit, label %loop
+
+exit:
+ %res = phi i32 [ 0, %loop ], [ %ld, %latch ]
+ ret i32 %res
+}
+
+; TODO: Cannot tail fold yet.
+define void @early_exit_with_store(ptr align 4 dereferenceable(4096) noalias %dst, ptr align 4 dereferenceable(4096) %pred) {
+; CHECK-LABEL: define void @early_exit_with_store(
+; CHECK-SAME: ptr noalias align 4 dereferenceable(4096) [[DST:%.*]], ptr align 4 dereferenceable(4096) [[PRED:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT: [[ENTRY:.*]]:
+; CHECK-NEXT: [[TMP0:%.*]] = call i32 @llvm.vscale.i32()
+; CHECK-NEXT: [[TMP1:%.*]] = shl nuw i32 [[TMP0]], 2
+; CHECK-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ult i32 123, [[TMP1]]
+; CHECK-NEXT: br i1 [[MIN_ITERS_CHECK]], label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]]
+; CHECK: [[VECTOR_PH]]:
+; CHECK-NEXT: [[N_MOD_VF:%.*]] = urem i32 123, [[TMP1]]
+; CHECK-NEXT: [[N_VEC:%.*]] = sub i32 123, [[N_MOD_VF]]
+; CHECK-NEXT: br label %[[VECTOR_BODY:.*]]
+; CHECK: [[VECTOR_BODY]]:
+; CHECK-NEXT: [[INDEX:%.*]] = phi i32 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-NEXT: [[TMP2:%.*]] = getelementptr inbounds nuw i32, ptr [[PRED]], i32 [[INDEX]]
+; CHECK-NEXT: [[WIDE_LOAD:%.*]] = load <vscale x 4 x i32>, ptr [[TMP2]], align 4
+; CHECK-NEXT: [[TMP3:%.*]] = icmp sgt <vscale x 4 x i32> [[WIDE_LOAD]], splat (i32 500)
+; CHECK-NEXT: [[TMP4:%.*]] = freeze <vscale x 4 x i1> [[TMP3]]
+; CHECK-NEXT: [[UNCOUNTABLE_EXIT_MASK:%.*]] = call <vscale x 4 x i1> @llvm.mask.beforefirst.nxv4i1(<vscale x 4 x i1> [[TMP4]])
+; CHECK-NEXT: [[TMP7:%.*]] = getelementptr i32, ptr [[DST]], i32 [[INDEX]]
+; CHECK-NEXT: [[WIDE_MASKED_LOAD:%.*]] = call <vscale x 4 x i32> @llvm.masked.load.nxv4i32.p0(ptr align 4 [[TMP7]], <vscale x 4 x i1> [[UNCOUNTABLE_EXIT_MASK]], <vscale x 4 x i32> poison)
+; CHECK-NEXT: [[TMP8:%.*]] = add nsw <vscale x 4 x i32> [[WIDE_MASKED_LOAD]], splat (i32 1)
+; CHECK-NEXT: call void @llvm.masked.store.nxv4i32.p0(<vscale x 4 x i32> [[TMP8]], ptr align 4 [[TMP7]], <vscale x 4 x i1> [[UNCOUNTABLE_EXIT_MASK]])
+; CHECK-NEXT: [[TMP9:%.*]] = call i1 @llvm.vector.reduce.or.nxv4i1(<vscale x 4 x i1> [[TMP4]])
+; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i32 [[INDEX]], [[TMP1]]
+; CHECK-NEXT: [[TMP10:%.*]] = icmp eq i32 [[INDEX_NEXT]], [[N_VEC]]
+; CHECK-NEXT: [[TMP11:%.*]] = or i1 [[TMP9]], [[TMP10]]
+; CHECK-NEXT: br i1 [[TMP11]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP10:![0-9]+]]
+; CHECK: [[MIDDLE_BLOCK]]:
+; CHECK-NEXT: [[TMP14:%.*]] = call i64 @llvm.experimental.cttz.elts.i64.nxv4i1(<vscale x 4 x i1> [[TMP4]], i1 false)
+; CHECK-NEXT: [[TMP6:%.*]] = trunc i64 [[TMP14]] to i32
+; CHECK-NEXT: [[TMP12:%.*]] = add i32 [[INDEX]], [[TMP6]]
+; CHECK-NEXT: [[TMP13:%.*]] = icmp eq i32 [[TMP12]], 123
+; CHECK-NEXT: br i1 [[TMP13]], label %[[EXIT:.*]], label %[[SCALAR_PH]]
+; CHECK: [[SCALAR_PH]]:
+; CHECK-NEXT: [[BC_RESUME_VAL:%.*]] = phi i32 [ [[TMP12]], %[[MIDDLE_BLOCK]] ], [ 0, %[[ENTRY]] ]
+; CHECK-NEXT: br label %[[LOOP:.*]]
+; CHECK: [[LOOP]]:
+; CHECK-NEXT: [[IV:%.*]] = phi i32 [ [[BC_RESUME_VAL]], %[[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], %[[LATCH:.*]] ]
+; CHECK-NEXT: [[ST_ADDR:%.*]] = getelementptr inbounds nuw i32, ptr [[DST]], i32 [[IV]]
+; CHECK-NEXT: [[DATA:%.*]] = load i32, ptr [[ST_ADDR]], align 4
+; CHECK-NEXT: [[INC:%.*]] = add nsw i32 [[DATA]], 1
+; CHECK-NEXT: store i32 [[INC]], ptr [[ST_ADDR]], align 4
+; CHECK-NEXT: [[EE_ADDR:%.*]] = getelementptr inbounds nuw i32, ptr [[PRED]], i32 [[IV]]
+; CHECK-NEXT: [[EE_VAL:%.*]] = load i32, ptr [[EE_ADDR]], align 4
+; CHECK-NEXT: [[EE_COND:%.*]] = icmp sgt i32 [[EE_VAL]], 500
+; CHECK-NEXT: br i1 [[EE_COND]], label %[[EXIT]], label %[[LATCH]]
+; CHECK: [[LATCH]]:
+; CHECK-NEXT: [[IV_NEXT]] = add nuw nsw i32 [[IV]], 1
+; CHECK-NEXT: [[EC:%.*]] = icmp eq i32 [[IV_NEXT]], 123
+; CHECK-NEXT: br i1 [[EC]], label %[[EXIT]], label %[[LOOP]], !llvm.loop [[LOOP11:![0-9]+]]
+; CHECK: [[EXIT]]:
+; CHECK-NEXT: ret void
+;
+entry:
+ br label %loop
+
+loop:
+ %iv = phi i32 [ 0, %entry ], [ %iv.next, %latch ]
+ %st.addr = getelementptr inbounds nuw i32, ptr %dst, i32 %iv
+ %data = load i32, ptr %st.addr, align 4
+ %inc = add nsw i32 %data, 1
+ store i32 %inc, ptr %st.addr, align 4
+ %ee.addr = getelementptr inbounds nuw i32, ptr %pred, i32 %iv
+ %ee.val = load i32, ptr %ee.addr, align 4
+ %ee.cond = icmp sgt i32 %ee.val, 500
+ br i1 %ee.cond, label %exit, label %latch
+
+latch:
+ %iv.next = add nuw nsw i32 %iv, 1
+ %ec = icmp eq i32 %iv.next, 123
+ br i1 %ec, label %exit, label %loop
+
+exit:
+ ret void
+}
+
+define void @early_exit_with_store_after_exit(ptr align 4 dereferenceable(4096) noalias %dst, ptr align 4 dereferenceable(4096) %pred) {
+; CHECK-LABEL: define void @early_exit_with_store_after_exit(
+; CHECK-SAME: ptr noalias align 4 dereferenceable(4096) [[DST:%.*]], ptr align 4 dereferenceable(4096) [[PRED:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT: [[ENTRY:.*]]:
+; CHECK-NEXT: [[TMP0:%.*]] = call i32 @llvm.vscale.i32()
+; CHECK-NEXT: [[TMP1:%.*]] = shl nuw i32 [[TMP0]], 2
+; CHECK-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ult i32 123, [[TMP1]]
+; CHECK-NEXT: br i1 [[MIN_ITERS_CHECK]], label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]]
+; CHECK: [[VECTOR_PH]]:
+; CHECK-NEXT: [[N_MOD_VF:%.*]] = urem i32 123, [[TMP1]]
+; CHECK-NEXT: [[N_VEC:%.*]] = sub i32 123, [[N_MOD_VF]]
+; CHECK-NEXT: br label %[[VECTOR_BODY:.*]]
+; CHECK: [[VECTOR_BODY]]:
+; CHECK-NEXT: [[INDEX:%.*]] = phi i32 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY_INTERIM:.*]] ]
+; CHECK-NEXT: [[TMP2:%.*]] = getelementptr inbounds nuw i32, ptr [[PRED]], i32 [[INDEX]]
+; CHECK-NEXT: [[WIDE_LOAD:%.*]] = load <vscale x 4 x i32>, ptr [[TMP2]], align 4
+; CHECK-NEXT: [[TMP3:%.*]] = icmp sgt <vscale x 4 x i32> [[WIDE_LOAD]], splat (i32 500)
+; CHECK-NEXT: [[TMP4:%.*]] = freeze <vscale x 4 x i1> [[TMP3]]
+; CHECK-NEXT: [[TMP5:%.*]] = call <vscale x 4 x i1> @llvm.mask.beforefirst.nxv4i1(<vscale x 4 x i1> [[TMP4]])
+; CHECK-NEXT: [[TMP6:%.*]] = getelementptr i32, ptr [[DST]], i32 [[INDEX]]
+; CHECK-NEXT: [[WIDE_MASKED_LOAD:%.*]] = call <vscale x 4 x i32> @llvm.masked.load.nxv4i32.p0(ptr align 4 [[TMP6]], <vscale x 4 x i1> [[TMP5]], <vscale x 4 x i32> poison)
+; CHECK-NEXT: [[TMP7:%.*]] = add nsw <vscale x 4 x i32> [[WIDE_MASKED_LOAD]], splat (i32 1)
+; CHECK-NEXT: call void @llvm.masked.store.nxv4i32.p0(<vscale x 4 x i32> [[TMP7]], ptr align 4 [[TMP6]], <vscale x 4 x i1> [[TMP5]])
+; CHECK-NEXT: [[TMP8:%.*]] = call i1 @llvm.vector.reduce.or.nxv4i1(<vscale x 4 x i1> [[TMP4]])
+; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i32 [[INDEX]], [[TMP1]]
+; CHECK-NEXT: [[TMP9:%.*]] = icmp eq i32 [[INDEX_NEXT]], [[N_VEC]]
+; CHECK-NEXT: br i1 [[TMP8]], label %[[VECTOR_EARLY_EXIT:.*]], label %[[VECTOR_BODY_INTERIM]]
+; CHECK: [[VECTOR_BODY_INTERIM]]:
+; CHECK-NEXT: br i1 [[TMP9]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP12:![0-9]+]]
+; CHECK: [[MIDDLE_BLOCK]]:
+; CHECK-NEXT: [[CMP_N:%.*]] = icmp eq i32 123, [[N_VEC]]
+; CHECK-NEXT: br i1 [[CMP_N]], label %[[EXIT:.*]], label %[[SCALAR_PH]]
+; CHECK: [[VECTOR_EARLY_EXIT]]:
+; CHECK-NEXT: br label %[[EXIT]]
+; CHECK: [[SCALAR_PH]]:
+; CHECK-NEXT: [[BC_RESUME_VAL:%.*]] = phi i32 [ [[N_VEC]], %[[MIDDLE_BLOCK]] ], [ 0, %[[ENTRY]] ]
+; CHECK-NEXT: br label %[[LOOP:.*]]
+; CHECK: [[LOOP]]:
+; CHECK-NEXT: [[IV:%.*]] = phi i32 [ [[BC_RESUME_VAL]], %[[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], %[[LATCH:.*]] ]
+; CHECK-NEXT: [[EE_ADDR:%.*]] = getelementptr inbounds nuw i32, ptr [[PRED]], i32 [[IV]]
+; CHECK-NEXT: [[EE_VAL:%.*]] = load i32, ptr [[EE_ADDR]], align 4
+; CHECK-NEXT: [[EE_COND:%.*]] = icmp sgt i32 [[EE_VAL]], 500
+; CHECK-NEXT: br i1 [[EE_COND]], label %[[EXIT]], label %[[LATCH]]
+; CHECK: [[LATCH]]:
+; CHECK-NEXT: [[ST_ADDR:%.*]] = getelementptr inbounds nuw i32, ptr [[DST]], i32 [[IV]]
+; CHECK-NEXT: [[DATA:%.*]] = load i32, ptr [[ST_ADDR]], align 4
+; CHECK-NEXT: [[INC:%.*]] = add nsw i32 [[DATA]], 1
+; CHECK-NEXT: store i32 [[INC]], ptr [[ST_ADDR]], align 4
+; CHECK-NEXT: [[IV_NEXT]] = add nuw nsw i32 [[IV]], 1
+; CHECK-NEXT: [[EC:%.*]] = icmp eq i32 [[IV_NEXT]], 123
+; CHECK-NEXT: br i1 [[EC]], label %[[EXIT]], label %[[LOOP]], !llvm.loop [[LOOP13:![0-9]+]]
+; CHECK: [[EXIT]]:
+; CHECK-NEXT: ret void
+;
+entry:
+ br label %loop
+
+loop:
+ %iv = phi i32 [ 0, %entry ], [ %iv.next, %latch ]
+ %ee.addr = getelementptr inbounds nuw i32, ptr %pred, i32 %iv
+ %ee.val = load i32, ptr %ee.addr
+ %ee.cond = icmp sgt i32 %ee.val, 500
+ br i1 %ee.cond, label %exit, label %latch
+
+latch:
+ %st.addr = getelementptr inbounds nuw i32, ptr %dst, i32 %iv
+ %data = load i32, ptr %st.addr
+ %inc = add nsw i32 %data, 1
+ store i32 %inc, ptr %st.addr
+ %iv.next = add nuw nsw i32 %iv, 1
+ %ec = icmp eq i32 %iv.next, 123
+ br i1 %ec, label %exit, label %loop
+
+exit:
+ ret void
+}
diff --git a/llvm/test/Transforms/LoopVectorize/early-exit-tail-folding-active-lane-mask.ll b/llvm/test/Transforms/LoopVectorize/early-exit-tail-folding-active-lane-mask.ll
new file mode 100644
index 00000000000000..a5c7c2a214335a
--- /dev/null
+++ b/llvm/test/Transforms/LoopVectorize/early-exit-tail-folding-active-lane-mask.ll
@@ -0,0 +1,65 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --check-globals none --version 6
+; RUN: opt -S < %s -p loop-vectorize -force-vector-width=4 -force-target-supports-masked-memory-ops -tail-folding-policy=prefer-fold-tail -force-tail-folding-style=data-and-control | FileCheck %s
+
+; TODO: Handle active lane mask style
+define i32 @single_early_exit(ptr align 4 dereferenceable(4096) %p) {
+; CHECK-LABEL: define i32 @single_early_exit(
+; CHECK-SAME: ptr align 4 dereferenceable(4096) [[P:%.*]]) {
+; CHECK-NEXT: [[ENTRY:.*:]]
+; CHECK-NEXT: br label %[[VECTOR_PH:.*]]
+; CHECK: [[VECTOR_PH]]:
+; CHECK-NEXT: br label %[[VECTOR_BODY:.*]]
+; CHECK: [[VECTOR_BODY]]:
+; CHECK-NEXT: [[INDEX:%.*]] = phi i32 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY_INTERIM:.*]] ]
+; CHECK-NEXT: [[TMP0:%.*]] = getelementptr inbounds i32, ptr [[P]], i32 [[INDEX]]
+; CHECK-NEXT: [[WIDE_LOAD:%.*]] = load <4 x i32>, ptr [[TMP0]], align 4
+; CHECK-NEXT: [[TMP1:%.*]] = icmp eq <4 x i32> [[WIDE_LOAD]], zeroinitializer
+; CHECK-NEXT: [[TMP2:%.*]] = freeze <4 x i1> [[TMP1]]
+; CHECK-NEXT: [[TMP3:%.*]] = call i1 @llvm.vector.reduce.or.v4i1(<4 x i1> [[TMP2]])
+; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i32 [[INDEX]], 4
+; CHECK-NEXT: [[TMP4:%.*]] = icmp eq i32 [[INDEX_NEXT]], 120
+; CHECK-NEXT: br i1 [[TMP3]], label %[[VECTOR_EARLY_EXIT:.*]], label %[[VECTOR_BODY_INTERIM]]
+; CHECK: [[VECTOR_BODY_INTERIM]]:
+; CHECK-NEXT: br i1 [[TMP4]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP0:![0-9]+]]
+; CHECK: [[MIDDLE_BLOCK]]:
+; CHECK-NEXT: br label %[[EXIT:.*]]
+; CHECK: [[VECTOR_EARLY_EXIT]]:
+; CHECK-NEXT: [[TMP5:%.*]] = call i64 @llvm.experimental.cttz.elts.i64.v4i1(<4 x i1> [[TMP2]], i1 false)
+; CHECK-NEXT: [[TMP6:%.*]] = trunc i64 [[TMP5]] to i32
+; CHECK-NEXT: [[TMP7:%.*]] = add i32 [[INDEX]], [[TMP6]]
+; CHECK-NEXT: br label %[[EXIT1:.*]]
+; CHECK: [[EXIT]]:
+; CHECK-NEXT: br label %[[LOOP:.*]]
+; CHECK: [[LOOP]]:
+; CHECK-NEXT: [[IV:%.*]] = phi i32 [ 120, %[[EXIT]] ], [ [[IV_NEXT:%.*]], %[[LATCH:.*]] ]
+; CHECK-NEXT: [[GEP:%.*]] = getelementptr inbounds i32, ptr [[P]], i32 [[IV]]
+; CHECK-NEXT: [[LD:%.*]] = load i32, ptr [[GEP]], align 4
+; CHECK-NEXT: [[C:%.*]] = icmp eq i32 [[LD]], 0
+; CHECK-NEXT: br i1 [[C]], label %[[EXIT1]], label %[[LATCH]]
+; CHECK: [[LATCH]]:
+; CHECK-NEXT: [[IV_NEXT]] = add nuw nsw i32 [[IV]], 1
+; CHECK-NEXT: [[EC:%.*]] = icmp eq i32 [[IV_NEXT]], 123
+; CHECK-NEXT: br i1 [[EC]], label %[[EXIT1]], label %[[LOOP]], !llvm.loop [[LOOP3:![0-9]+]]
+; CHECK: [[EXIT1]]:
+; CHECK-NEXT: [[RES:%.*]] = phi i32 [ [[IV]], %[[LOOP]] ], [ -1, %[[LATCH]] ], [ [[TMP7]], %[[VECTOR_EARLY_EXIT]] ]
+; CHECK-NEXT: ret i32 [[RES]]
+;
+entry:
+ br label %loop
+
+loop:
+ %iv = phi i32 [ 0, %entry ], [ %iv.next, %latch ]
+ %gep = getelementptr inbounds i32, ptr %p, i32 %iv
+ %ld = load i32, ptr %gep
+ %c = icmp eq i32 %ld, 0
+ br i1 %c, label %exit, label %latch
+
+latch:
+ %iv.next = add nuw nsw i32 %iv, 1
+ %ec = icmp eq i32 %iv.next, 123
+ br i1 %ec, label %exit, label %loop
+
+exit:
+ %res = phi i32 [ %iv, %loop ], [ -1, %latch ]
+ ret i32 %res
+}
diff --git a/llvm/test/Transforms/LoopVectorize/early-exit-tail-folding.ll b/llvm/test/Transforms/LoopVectorize/early-exit-tail-folding.ll
new file mode 100644
index 00000000000000..9026514723ae5d
--- /dev/null
+++ b/llvm/test/Transforms/LoopVectorize/early-exit-tail-folding.ll
@@ -0,0 +1,524 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --check-globals none --version 6
+; RUN: opt -S < %s -p loop-vectorize -force-vector-width=4 -force-target-supports-masked-memory-ops -enable-early-exit-vectorization-with-side-effects -tail-folding-policy=prefer-fold-tail | FileCheck %s
+
+define i32 @single_early_exit(ptr align 4 dereferenceable(4096) %p) {
+; CHECK-LABEL: define i32 @single_early_exit(
+; CHECK-SAME: ptr align 4 dereferenceable(4096) [[P:%.*]]) {
+; CHECK-NEXT: [[ENTRY:.*:]]
+; CHECK-NEXT: br label %[[VECTOR_PH:.*]]
+; CHECK: [[VECTOR_PH]]:
+; CHECK-NEXT: br label %[[VECTOR_BODY:.*]]
+; CHECK: [[VECTOR_BODY]]:
+; CHECK-NEXT: [[INDEX:%.*]] = phi i32 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY_INTERIM:.*]] ]
+; CHECK-NEXT: [[TMP0:%.*]] = getelementptr inbounds i32, ptr [[P]], i32 [[INDEX]]
+; CHECK-NEXT: [[WIDE_LOAD:%.*]] = load <4 x i32>, ptr [[TMP0]], align 4
+; CHECK-NEXT: [[TMP1:%.*]] = icmp eq <4 x i32> [[WIDE_LOAD]], zeroinitializer
+; CHECK-NEXT: [[TMP2:%.*]] = freeze <4 x i1> [[TMP1]]
+; CHECK-NEXT: [[TMP3:%.*]] = call i1 @llvm.vector.reduce.or.v4i1(<4 x i1> [[TMP2]])
+; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i32 [[INDEX]], 4
+; CHECK-NEXT: [[TMP4:%.*]] = icmp eq i32 [[INDEX_NEXT]], 120
+; CHECK-NEXT: br i1 [[TMP3]], label %[[VECTOR_EARLY_EXIT:.*]], label %[[VECTOR_BODY_INTERIM]]
+; CHECK: [[VECTOR_BODY_INTERIM]]:
+; CHECK-NEXT: br i1 [[TMP4]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP0:![0-9]+]]
+; CHECK: [[MIDDLE_BLOCK]]:
+; CHECK-NEXT: br label %[[EXIT:.*]]
+; CHECK: [[VECTOR_EARLY_EXIT]]:
+; CHECK-NEXT: [[TMP5:%.*]] = call i64 @llvm.experimental.cttz.elts.i64.v4i1(<4 x i1> [[TMP2]], i1 false)
+; CHECK-NEXT: [[TMP6:%.*]] = trunc i64 [[TMP5]] to i32
+; CHECK-NEXT: [[TMP7:%.*]] = add i32 [[INDEX]], [[TMP6]]
+; CHECK-NEXT: br label %[[EXIT1:.*]]
+; CHECK: [[EXIT]]:
+; CHECK-NEXT: br label %[[LOOP:.*]]
+; CHECK: [[LOOP]]:
+; CHECK-NEXT: [[IV:%.*]] = phi i32 [ 120, %[[EXIT]] ], [ [[IV_NEXT:%.*]], %[[LATCH:.*]] ]
+; CHECK-NEXT: [[GEP:%.*]] = getelementptr inbounds i32, ptr [[P]], i32 [[IV]]
+; CHECK-NEXT: [[LD:%.*]] = load i32, ptr [[GEP]], align 4
+; CHECK-NEXT: [[C:%.*]] = icmp eq i32 [[LD]], 0
+; CHECK-NEXT: br i1 [[C]], label %[[EXIT1]], label %[[LATCH]]
+; CHECK: [[LATCH]]:
+; CHECK-NEXT: [[IV_NEXT]] = add nuw nsw i32 [[IV]], 1
+; CHECK-NEXT: [[EC:%.*]] = icmp eq i32 [[IV_NEXT]], 123
+; CHECK-NEXT: br i1 [[EC]], label %[[EXIT1]], label %[[LOOP]], !llvm.loop [[LOOP3:![0-9]+]]
+; CHECK: [[EXIT1]]:
+; CHECK-NEXT: [[RES:%.*]] = phi i32 [ [[IV]], %[[LOOP]] ], [ -1, %[[LATCH]] ], [ [[TMP7]], %[[VECTOR_EARLY_EXIT]] ]
+; CHECK-NEXT: ret i32 [[RES]]
+;
+entry:
+ br label %loop
+
+loop:
+ %iv = phi i32 [ 0, %entry ], [ %iv.next, %latch ]
+ %gep = getelementptr inbounds i32, ptr %p, i32 %iv
+ %ld = load i32, ptr %gep
+ %c = icmp eq i32 %ld, 0
+ br i1 %c, label %exit, label %latch
+
+latch:
+ %iv.next = add nuw nsw i32 %iv, 1
+ %ec = icmp eq i32 %iv.next, 123
+ br i1 %ec, label %exit, label %loop
+
+exit:
+ %res = phi i32 [ %iv, %loop ], [ -1, %latch ]
+ ret i32 %res
+}
+
+define i32 @multi_early_exits_same_exits(ptr align 4 dereferenceable(4096) %p, ptr align 4 dereferenceable(4096) %q) {
+; CHECK-LABEL: define i32 @multi_early_exits_same_exits(
+; CHECK-SAME: ptr align 4 dereferenceable(4096) [[P:%.*]], ptr align 4 dereferenceable(4096) [[Q:%.*]]) {
+; CHECK-NEXT: [[ENTRY:.*:]]
+; CHECK-NEXT: br label %[[VECTOR_PH:.*]]
+; CHECK: [[VECTOR_PH]]:
+; CHECK-NEXT: br label %[[VECTOR_BODY:.*]]
+; CHECK: [[VECTOR_BODY]]:
+; CHECK-NEXT: [[INDEX:%.*]] = phi i32 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY_INTERIM:.*]] ]
+; CHECK-NEXT: [[VEC_IND:%.*]] = phi <4 x i32> [ <i32 0, i32 1, i32 2, i32 3>, %[[VECTOR_PH]] ], [ [[VEC_IND_NEXT:%.*]], %[[VECTOR_BODY_INTERIM]] ]
+; CHECK-NEXT: [[TMP0:%.*]] = getelementptr inbounds i32, ptr [[P]], i32 [[INDEX]]
+; CHECK-NEXT: [[WIDE_LOAD:%.*]] = load <4 x i32>, ptr [[TMP0]], align 4
+; CHECK-NEXT: [[TMP1:%.*]] = icmp eq <4 x i32> [[WIDE_LOAD]], zeroinitializer
+; CHECK-NEXT: [[TMP2:%.*]] = getelementptr inbounds i32, ptr [[Q]], i32 [[INDEX]]
+; CHECK-NEXT: [[WIDE_LOAD1:%.*]] = load <4 x i32>, ptr [[TMP2]], align 4
+; CHECK-NEXT: [[TMP3:%.*]] = icmp eq <4 x i32> [[WIDE_LOAD1]], zeroinitializer
+; CHECK-NEXT: [[TMP4:%.*]] = add <4 x i32> [[VEC_IND]], splat (i32 1000)
+; CHECK-NEXT: [[TMP5:%.*]] = select <4 x i1> [[TMP1]], <4 x i1> splat (i1 true), <4 x i1> [[TMP3]]
+; CHECK-NEXT: [[TMP6:%.*]] = freeze <4 x i1> [[TMP5]]
+; CHECK-NEXT: [[TMP7:%.*]] = call i1 @llvm.vector.reduce.or.v4i1(<4 x i1> [[TMP6]])
+; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i32 [[INDEX]], 4
+; CHECK-NEXT: [[TMP8:%.*]] = icmp eq i32 [[INDEX_NEXT]], 120
+; CHECK-NEXT: [[VEC_IND_NEXT]] = add nuw nsw <4 x i32> [[VEC_IND]], splat (i32 4)
+; CHECK-NEXT: br i1 [[TMP7]], label %[[VECTOR_EARLY_EXIT_CHECK:.*]], label %[[VECTOR_BODY_INTERIM]]
+; CHECK: [[VECTOR_BODY_INTERIM]]:
+; CHECK-NEXT: br i1 [[TMP8]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP4:![0-9]+]]
+; CHECK: [[MIDDLE_BLOCK]]:
+; CHECK-NEXT: br label %[[EXIT:.*]]
+; CHECK: [[VECTOR_EARLY_EXIT_CHECK]]:
+; CHECK-NEXT: [[FIRST_ACTIVE_LANE:%.*]] = call i64 @llvm.experimental.cttz.elts.i64.v4i1(<4 x i1> [[TMP6]], i1 false)
+; CHECK-NEXT: [[TMP9:%.*]] = extractelement <4 x i1> [[TMP1]], i64 [[FIRST_ACTIVE_LANE]]
+; CHECK-NEXT: br i1 [[TMP9]], label %[[VECTOR_EARLY_EXIT_0:.*]], label %[[VECTOR_EARLY_EXIT_1:.*]]
+; CHECK: [[VECTOR_EARLY_EXIT_1]]:
+; CHECK-NEXT: [[TMP10:%.*]] = extractelement <4 x i32> [[TMP4]], i64 [[FIRST_ACTIVE_LANE]]
+; CHECK-NEXT: br label %[[EXIT1:.*]]
+; CHECK: [[VECTOR_EARLY_EXIT_0]]:
+; CHECK-NEXT: [[TMP11:%.*]] = trunc i64 [[FIRST_ACTIVE_LANE]] to i32
+; CHECK-NEXT: [[TMP12:%.*]] = add i32 [[INDEX]], [[TMP11]]
+; CHECK-NEXT: br label %[[EXIT1]]
+; CHECK: [[EXIT]]:
+; CHECK-NEXT: br label %[[LOOP:.*]]
+; CHECK: [[LOOP]]:
+; CHECK-NEXT: [[IV:%.*]] = phi i32 [ 120, %[[EXIT]] ], [ [[IV_NEXT:%.*]], %[[LATCH:.*]] ]
+; CHECK-NEXT: [[GEP_P:%.*]] = getelementptr inbounds i32, ptr [[P]], i32 [[IV]]
+; CHECK-NEXT: [[X:%.*]] = load i32, ptr [[GEP_P]], align 4
+; CHECK-NEXT: [[C1:%.*]] = icmp eq i32 [[X]], 0
+; CHECK-NEXT: br i1 [[C1]], label %[[EXIT1]], label %[[NEXT:.*]]
+; CHECK: [[NEXT]]:
+; CHECK-NEXT: [[GEP_Q:%.*]] = getelementptr inbounds i32, ptr [[Q]], i32 [[IV]]
+; CHECK-NEXT: [[Y:%.*]] = load i32, ptr [[GEP_Q]], align 4
+; CHECK-NEXT: [[C2:%.*]] = icmp eq i32 [[Y]], 0
+; CHECK-NEXT: [[IV_Q:%.*]] = add i32 [[IV]], 1000
+; CHECK-NEXT: br i1 [[C2]], label %[[EXIT1]], label %[[LATCH]]
+; CHECK: [[LATCH]]:
+; CHECK-NEXT: [[IV_NEXT]] = add nuw nsw i32 [[IV]], 1
+; CHECK-NEXT: [[EC:%.*]] = icmp eq i32 [[IV_NEXT]], 123
+; CHECK-NEXT: br i1 [[EC]], label %[[EXIT1]], label %[[LOOP]], !llvm.loop [[LOOP5:![0-9]+]]
+; CHECK: [[EXIT1]]:
+; CHECK-NEXT: [[RES:%.*]] = phi i32 [ [[IV]], %[[LOOP]] ], [ [[IV_Q]], %[[NEXT]] ], [ -1, %[[LATCH]] ], [ [[TMP12]], %[[VECTOR_EARLY_EXIT_0]] ], [ [[TMP10]], %[[VECTOR_EARLY_EXIT_1]] ]
+; CHECK-NEXT: ret i32 [[RES]]
+;
+entry:
+ br label %loop
+
+loop:
+ %iv = phi i32 [ 0, %entry ], [ %iv.next, %latch ]
+ %gep.p = getelementptr inbounds i32, ptr %p, i32 %iv
+ %x = load i32, ptr %gep.p, align 4
+ %c1 = icmp eq i32 %x, 0
+ br i1 %c1, label %exit, label %next
+
+next:
+ %gep.q = getelementptr inbounds i32, ptr %q, i32 %iv
+ %y = load i32, ptr %gep.q, align 4
+ %c2 = icmp eq i32 %y, 0
+ %iv.q = add i32 %iv, 1000
+ br i1 %c2, label %exit, label %latch
+
+latch:
+ %iv.next = add nuw nsw i32 %iv, 1
+ %ec = icmp eq i32 %iv.next, 123
+ br i1 %ec, label %exit, label %loop
+
+exit:
+ %res = phi i32 [ %iv, %loop ], [ %iv.q, %next ], [ -1, %latch ]
+ ret i32 %res
+}
+
+define i32 @multi_early_exits_different_exits(ptr align 4 dereferenceable(4096) %p, ptr align 4 dereferenceable(4096) %q) {
+; CHECK-LABEL: define i32 @multi_early_exits_different_exits(
+; CHECK-SAME: ptr align 4 dereferenceable(4096) [[P:%.*]], ptr align 4 dereferenceable(4096) [[Q:%.*]]) {
+; CHECK-NEXT: [[ENTRY:.*:]]
+; CHECK-NEXT: br label %[[VECTOR_PH:.*]]
+; CHECK: [[VECTOR_PH]]:
+; CHECK-NEXT: br label %[[VECTOR_BODY:.*]]
+; CHECK: [[VECTOR_BODY]]:
+; CHECK-NEXT: [[INDEX:%.*]] = phi i32 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY_INTERIM:.*]] ]
+; CHECK-NEXT: [[VEC_IND:%.*]] = phi <4 x i32> [ <i32 0, i32 1, i32 2, i32 3>, %[[VECTOR_PH]] ], [ [[VEC_IND_NEXT:%.*]], %[[VECTOR_BODY_INTERIM]] ]
+; CHECK-NEXT: [[TMP0:%.*]] = getelementptr inbounds i32, ptr [[P]], i32 [[INDEX]]
+; CHECK-NEXT: [[WIDE_LOAD:%.*]] = load <4 x i32>, ptr [[TMP0]], align 4
+; CHECK-NEXT: [[TMP1:%.*]] = icmp eq <4 x i32> [[WIDE_LOAD]], zeroinitializer
+; CHECK-NEXT: [[TMP2:%.*]] = getelementptr inbounds i32, ptr [[Q]], i32 [[INDEX]]
+; CHECK-NEXT: [[WIDE_LOAD1:%.*]] = load <4 x i32>, ptr [[TMP2]], align 4
+; CHECK-NEXT: [[TMP3:%.*]] = icmp eq <4 x i32> [[WIDE_LOAD1]], zeroinitializer
+; CHECK-NEXT: [[TMP4:%.*]] = add <4 x i32> [[VEC_IND]], splat (i32 1000)
+; CHECK-NEXT: [[TMP5:%.*]] = select <4 x i1> [[TMP1]], <4 x i1> splat (i1 true), <4 x i1> [[TMP3]]
+; CHECK-NEXT: [[TMP6:%.*]] = freeze <4 x i1> [[TMP5]]
+; CHECK-NEXT: [[TMP7:%.*]] = call i1 @llvm.vector.reduce.or.v4i1(<4 x i1> [[TMP6]])
+; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i32 [[INDEX]], 4
+; CHECK-NEXT: [[TMP8:%.*]] = icmp eq i32 [[INDEX_NEXT]], 120
+; CHECK-NEXT: [[VEC_IND_NEXT]] = add nuw nsw <4 x i32> [[VEC_IND]], splat (i32 4)
+; CHECK-NEXT: br i1 [[TMP7]], label %[[VECTOR_EARLY_EXIT_CHECK:.*]], label %[[VECTOR_BODY_INTERIM]]
+; CHECK: [[VECTOR_BODY_INTERIM]]:
+; CHECK-NEXT: br i1 [[TMP8]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP6:![0-9]+]]
+; CHECK: [[MIDDLE_BLOCK]]:
+; CHECK-NEXT: br label %[[EXIT_LATCH:.*]]
+; CHECK: [[VECTOR_EARLY_EXIT_CHECK]]:
+; CHECK-NEXT: [[FIRST_ACTIVE_LANE:%.*]] = call i64 @llvm.experimental.cttz.elts.i64.v4i1(<4 x i1> [[TMP6]], i1 false)
+; CHECK-NEXT: [[TMP9:%.*]] = extractelement <4 x i1> [[TMP1]], i64 [[FIRST_ACTIVE_LANE]]
+; CHECK-NEXT: br i1 [[TMP9]], label %[[VECTOR_EARLY_EXIT_0:.*]], label %[[VECTOR_EARLY_EXIT_1:.*]]
+; CHECK: [[VECTOR_EARLY_EXIT_1]]:
+; CHECK-NEXT: [[TMP10:%.*]] = extractelement <4 x i32> [[TMP4]], i64 [[FIRST_ACTIVE_LANE]]
+; CHECK-NEXT: br label %[[EXIT_Q:.*]]
+; CHECK: [[VECTOR_EARLY_EXIT_0]]:
+; CHECK-NEXT: [[TMP11:%.*]] = trunc i64 [[FIRST_ACTIVE_LANE]] to i32
+; CHECK-NEXT: [[TMP12:%.*]] = add i32 [[INDEX]], [[TMP11]]
+; CHECK-NEXT: br label %[[EXIT_P:.*]]
+; CHECK: [[EXIT_LATCH]]:
+; CHECK-NEXT: br label %[[LOOP:.*]]
+; CHECK: [[LOOP]]:
+; CHECK-NEXT: [[IV:%.*]] = phi i32 [ 120, %[[EXIT_LATCH]] ], [ [[IV_NEXT:%.*]], %[[LATCH:.*]] ]
+; CHECK-NEXT: [[GEP_P:%.*]] = getelementptr inbounds i32, ptr [[P]], i32 [[IV]]
+; CHECK-NEXT: [[X:%.*]] = load i32, ptr [[GEP_P]], align 4
+; CHECK-NEXT: [[C1:%.*]] = icmp eq i32 [[X]], 0
+; CHECK-NEXT: br i1 [[C1]], label %[[EXIT_P]], label %[[NEXT:.*]]
+; CHECK: [[NEXT]]:
+; CHECK-NEXT: [[GEP_Q:%.*]] = getelementptr inbounds i32, ptr [[Q]], i32 [[IV]]
+; CHECK-NEXT: [[Y:%.*]] = load i32, ptr [[GEP_Q]], align 4
+; CHECK-NEXT: [[C2:%.*]] = icmp eq i32 [[Y]], 0
+; CHECK-NEXT: [[IV_Q:%.*]] = add i32 [[IV]], 1000
+; CHECK-NEXT: br i1 [[C2]], label %[[EXIT_Q]], label %[[LATCH]]
+; CHECK: [[LATCH]]:
+; CHECK-NEXT: [[IV_NEXT]] = add nuw nsw i32 [[IV]], 1
+; CHECK-NEXT: [[EC:%.*]] = icmp eq i32 [[IV_NEXT]], 123
+; CHECK-NEXT: br i1 [[EC]], label %[[EXIT_LATCH1:.*]], label %[[LOOP]], !llvm.loop [[LOOP7:![0-9]+]]
+; CHECK: [[EXIT_P]]:
+; CHECK-NEXT: [[RES_P:%.*]] = phi i32 [ [[IV]], %[[LOOP]] ], [ [[TMP12]], %[[VECTOR_EARLY_EXIT_0]] ]
+; CHECK-NEXT: ret i32 [[RES_P]]
+; CHECK: [[EXIT_Q]]:
+; CHECK-NEXT: [[RES_Q:%.*]] = phi i32 [ [[IV_Q]], %[[NEXT]] ], [ [[TMP10]], %[[VECTOR_EARLY_EXIT_1]] ]
+; CHECK-NEXT: ret i32 [[RES_Q]]
+; CHECK: [[EXIT_LATCH1]]:
+; CHECK-NEXT: ret i32 -1
+;
+entry:
+ br label %loop
+
+loop:
+ %iv = phi i32 [ 0, %entry ], [ %iv.next, %latch ]
+ %gep.p = getelementptr inbounds i32, ptr %p, i32 %iv
+ %x = load i32, ptr %gep.p, align 4
+ %c1 = icmp eq i32 %x, 0
+ br i1 %c1, label %exit.p, label %next
+
+next:
+ %gep.q = getelementptr inbounds i32, ptr %q, i32 %iv
+ %y = load i32, ptr %gep.q, align 4
+ %c2 = icmp eq i32 %y, 0
+ %iv.q = add i32 %iv, 1000
+ br i1 %c2, label %exit.q, label %latch
+
+latch:
+ %iv.next = add nuw nsw i32 %iv, 1
+ %ec = icmp eq i32 %iv.next, 123
+ br i1 %ec, label %exit.latch, label %loop
+
+exit.p:
+ %res.p = phi i32 [ %iv, %loop ]
+ ret i32 %res.p
+
+exit.q:
+ %res.q = phi i32 [ %iv.q, %next ]
+ ret i32 %res.q
+
+exit.latch:
+ ret i32 -1
+}
+
+define i32 @predicated_early_exit(i1 %c0, ptr align 4 dereferenceable(4096) %p) {
+; CHECK-LABEL: define i32 @predicated_early_exit(
+; CHECK-SAME: i1 [[C0:%.*]], ptr align 4 dereferenceable(4096) [[P:%.*]]) {
+; CHECK-NEXT: [[ENTRY:.*:]]
+; CHECK-NEXT: br label %[[VECTOR_PH:.*]]
+; CHECK: [[VECTOR_PH]]:
+; CHECK-NEXT: [[BROADCAST_SPLATINSERT:%.*]] = insertelement <4 x i1> poison, i1 [[C0]], i64 0
+; CHECK-NEXT: [[BROADCAST_SPLAT:%.*]] = shufflevector <4 x i1> [[BROADCAST_SPLATINSERT]], <4 x i1> poison, <4 x i32> zeroinitializer
+; CHECK-NEXT: [[TMP9:%.*]] = freeze <4 x i1> [[BROADCAST_SPLAT]]
+; CHECK-NEXT: br label %[[VECTOR_BODY:.*]]
+; CHECK: [[VECTOR_BODY]]:
+; CHECK-NEXT: [[INDEX:%.*]] = phi i32 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY_INTERIM:.*]] ]
+; CHECK-NEXT: [[TMP0:%.*]] = getelementptr inbounds i32, ptr [[P]], i32 [[INDEX]]
+; CHECK-NEXT: [[WIDE_LOAD:%.*]] = load <4 x i32>, ptr [[TMP0]], align 4
+; CHECK-NEXT: [[TMP1:%.*]] = icmp eq <4 x i32> [[WIDE_LOAD]], zeroinitializer
+; CHECK-NEXT: [[TMP2:%.*]] = select <4 x i1> [[TMP9]], <4 x i1> [[TMP1]], <4 x i1> zeroinitializer
+; CHECK-NEXT: [[TMP3:%.*]] = freeze <4 x i1> [[TMP2]]
+; CHECK-NEXT: [[TMP4:%.*]] = call i1 @llvm.vector.reduce.or.v4i1(<4 x i1> [[TMP3]])
+; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i32 [[INDEX]], 4
+; CHECK-NEXT: [[TMP5:%.*]] = icmp eq i32 [[INDEX_NEXT]], 120
+; CHECK-NEXT: br i1 [[TMP4]], label %[[VECTOR_EARLY_EXIT:.*]], label %[[VECTOR_BODY_INTERIM]]
+; CHECK: [[VECTOR_BODY_INTERIM]]:
+; CHECK-NEXT: br i1 [[TMP5]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP8:![0-9]+]]
+; CHECK: [[MIDDLE_BLOCK]]:
+; CHECK-NEXT: br label %[[EXIT:.*]]
+; CHECK: [[VECTOR_EARLY_EXIT]]:
+; CHECK-NEXT: [[TMP6:%.*]] = call i64 @llvm.experimental.cttz.elts.i64.v4i1(<4 x i1> [[TMP3]], i1 false)
+; CHECK-NEXT: [[TMP7:%.*]] = trunc i64 [[TMP6]] to i32
+; CHECK-NEXT: [[TMP8:%.*]] = add i32 [[INDEX]], [[TMP7]]
+; CHECK-NEXT: br label %[[EXIT1:.*]]
+; CHECK: [[EXIT]]:
+; CHECK-NEXT: br label %[[LOOP:.*]]
+; CHECK: [[LOOP]]:
+; CHECK-NEXT: [[IV:%.*]] = phi i32 [ 120, %[[EXIT]] ], [ [[IV_NEXT:%.*]], %[[LATCH:.*]] ]
+; CHECK-NEXT: br i1 [[C0]], label %[[THEN:.*]], label %[[LATCH]]
+; CHECK: [[THEN]]:
+; CHECK-NEXT: [[GEP:%.*]] = getelementptr inbounds i32, ptr [[P]], i32 [[IV]]
+; CHECK-NEXT: [[LD:%.*]] = load i32, ptr [[GEP]], align 4
+; CHECK-NEXT: [[C1:%.*]] = icmp eq i32 [[LD]], 0
+; CHECK-NEXT: br i1 [[C1]], label %[[EXIT1]], label %[[LATCH]]
+; CHECK: [[LATCH]]:
+; CHECK-NEXT: [[IV_NEXT]] = add nuw nsw i32 [[IV]], 1
+; CHECK-NEXT: [[EC:%.*]] = icmp eq i32 [[IV_NEXT]], 123
+; CHECK-NEXT: br i1 [[EC]], label %[[EXIT1]], label %[[LOOP]], !llvm.loop [[LOOP9:![0-9]+]]
+; CHECK: [[EXIT1]]:
+; CHECK-NEXT: [[RES:%.*]] = phi i32 [ [[IV]], %[[THEN]] ], [ -1, %[[LATCH]] ], [ [[TMP8]], %[[VECTOR_EARLY_EXIT]] ]
+; CHECK-NEXT: ret i32 [[RES]]
+;
+entry:
+ br label %loop
+
+loop:
+ %iv = phi i32 [ 0, %entry ], [ %iv.next, %latch ]
+ br i1 %c0, label %then, label %latch
+
+then:
+ %gep = getelementptr inbounds i32, ptr %p, i32 %iv
+ %ld = load i32, ptr %gep
+ %c1 = icmp eq i32 %ld, 0
+ br i1 %c1, label %exit, label %latch
+
+latch:
+ %iv.next = add nuw nsw i32 %iv, 1
+ %ec = icmp eq i32 %iv.next, 123
+ br i1 %ec, label %exit, label %loop
+
+exit:
+ %res = phi i32 [ %iv, %then ], [ -1, %latch ]
+ ret i32 %res
+}
+
+; TODO: Cannot tail fold yet.
+define i32 @countable_early_exit(ptr align 4 dereferenceable(4096) %p, i32 %n) {
+; CHECK-LABEL: define i32 @countable_early_exit(
+; CHECK-SAME: ptr align 4 dereferenceable(4096) [[P:%.*]], i32 [[N:%.*]]) {
+; CHECK-NEXT: [[ENTRY:.*]]:
+; CHECK-NEXT: [[TMP0:%.*]] = call i32 @llvm.umin.i32(i32 [[N]], i32 122)
+; CHECK-NEXT: [[TMP1:%.*]] = add nuw nsw i32 [[TMP0]], 1
+; CHECK-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ule i32 [[TMP1]], 4
+; CHECK-NEXT: br i1 [[MIN_ITERS_CHECK]], label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]]
+; CHECK: [[VECTOR_PH]]:
+; CHECK-NEXT: [[TMP2:%.*]] = and i32 [[TMP1]], 3
+; CHECK-NEXT: [[TMP3:%.*]] = icmp eq i32 [[TMP2]], 0
+; CHECK-NEXT: [[TMP4:%.*]] = select i1 [[TMP3]], i32 4, i32 [[TMP2]]
+; CHECK-NEXT: [[N_VEC:%.*]] = sub i32 [[TMP1]], [[TMP4]]
+; CHECK-NEXT: br label %[[VECTOR_BODY:.*]]
+; CHECK: [[VECTOR_BODY]]:
+; CHECK-NEXT: [[INDEX:%.*]] = phi i32 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i32 [[INDEX]], 4
+; CHECK-NEXT: [[TMP5:%.*]] = icmp eq i32 [[INDEX_NEXT]], [[N_VEC]]
+; CHECK-NEXT: br i1 [[TMP5]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP10:![0-9]+]]
+; CHECK: [[MIDDLE_BLOCK]]:
+; CHECK-NEXT: br label %[[SCALAR_PH]]
+; CHECK: [[SCALAR_PH]]:
+; CHECK-NEXT: [[BC_RESUME_VAL:%.*]] = phi i32 [ [[N_VEC]], %[[MIDDLE_BLOCK]] ], [ 0, %[[ENTRY]] ]
+; CHECK-NEXT: br label %[[LOOP:.*]]
+; CHECK: [[LOOP]]:
+; CHECK-NEXT: [[IV:%.*]] = phi i32 [ [[BC_RESUME_VAL]], %[[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], %[[LATCH:.*]] ]
+; CHECK-NEXT: [[C:%.*]] = icmp eq i32 [[IV]], [[N]]
+; CHECK-NEXT: br i1 [[C]], label %[[EXIT:.*]], label %[[LATCH]]
+; CHECK: [[LATCH]]:
+; CHECK-NEXT: [[GEP:%.*]] = getelementptr inbounds i32, ptr [[P]], i32 [[IV]]
+; CHECK-NEXT: [[LD:%.*]] = load i32, ptr [[GEP]], align 4
+; CHECK-NEXT: [[IV_NEXT]] = add nuw nsw i32 [[IV]], 1
+; CHECK-NEXT: [[EC:%.*]] = icmp eq i32 [[IV_NEXT]], 123
+; CHECK-NEXT: br i1 [[EC]], label %[[EXIT]], label %[[LOOP]], !llvm.loop [[LOOP11:![0-9]+]]
+; CHECK: [[EXIT]]:
+; CHECK-NEXT: [[RES:%.*]] = phi i32 [ 0, %[[LOOP]] ], [ [[LD]], %[[LATCH]] ]
+; CHECK-NEXT: ret i32 [[RES]]
+;
+entry:
+ br label %loop
+
+loop:
+ %iv = phi i32 [ 0, %entry ], [ %iv.next, %latch ]
+ %c = icmp eq i32 %iv, %n
+ br i1 %c, label %exit, label %latch
+
+latch:
+ %gep = getelementptr inbounds i32, ptr %p, i32 %iv
+ %ld = load i32, ptr %gep, align 4
+ %iv.next = add nuw nsw i32 %iv, 1
+ %ec = icmp eq i32 %iv.next, 123
+ br i1 %ec, label %exit, label %loop
+
+exit:
+ %res = phi i32 [ 0, %loop ], [ %ld, %latch ]
+ ret i32 %res
+}
+
+; TODO: Cannot tail fold yet.
+define void @early_exit_with_store(ptr align 4 dereferenceable(4096) noalias %dst, ptr align 4 dereferenceable(4096) %pred) {
+; CHECK-LABEL: define void @early_exit_with_store(
+; CHECK-SAME: ptr noalias align 4 dereferenceable(4096) [[DST:%.*]], ptr align 4 dereferenceable(4096) [[PRED:%.*]]) {
+; CHECK-NEXT: [[ENTRY:.*:]]
+; CHECK-NEXT: br label %[[VECTOR_PH:.*]]
+; CHECK: [[VECTOR_PH]]:
+; CHECK-NEXT: br label %[[VECTOR_BODY:.*]]
+; CHECK: [[VECTOR_BODY]]:
+; CHECK-NEXT: [[INDEX:%.*]] = phi i32 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-NEXT: [[TMP0:%.*]] = getelementptr inbounds nuw i32, ptr [[PRED]], i32 [[INDEX]]
+; CHECK-NEXT: [[WIDE_LOAD:%.*]] = load <4 x i32>, ptr [[TMP0]], align 4
+; CHECK-NEXT: [[TMP1:%.*]] = icmp sgt <4 x i32> [[WIDE_LOAD]], splat (i32 500)
+; CHECK-NEXT: [[TMP2:%.*]] = freeze <4 x i1> [[TMP1]]
+; CHECK-NEXT: [[UNCOUNTABLE_EXIT_MASK:%.*]] = call <4 x i1> @llvm.mask.beforefirst.v4i1(<4 x i1> [[TMP2]])
+; CHECK-NEXT: [[TMP5:%.*]] = getelementptr i32, ptr [[DST]], i32 [[INDEX]]
+; CHECK-NEXT: [[WIDE_MASKED_LOAD:%.*]] = call <4 x i32> @llvm.masked.load.v4i32.p0(ptr align 4 [[TMP5]], <4 x i1> [[UNCOUNTABLE_EXIT_MASK]], <4 x i32> poison)
+; CHECK-NEXT: [[TMP6:%.*]] = add nsw <4 x i32> [[WIDE_MASKED_LOAD]], splat (i32 1)
+; CHECK-NEXT: call void @llvm.masked.store.v4i32.p0(<4 x i32> [[TMP6]], ptr align 4 [[TMP5]], <4 x i1> [[UNCOUNTABLE_EXIT_MASK]])
+; CHECK-NEXT: [[TMP7:%.*]] = call i1 @llvm.vector.reduce.or.v4i1(<4 x i1> [[TMP2]])
+; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i32 [[INDEX]], 4
+; CHECK-NEXT: [[TMP8:%.*]] = icmp eq i32 [[INDEX_NEXT]], 120
+; CHECK-NEXT: [[TMP9:%.*]] = or i1 [[TMP7]], [[TMP8]]
+; CHECK-NEXT: br i1 [[TMP9]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP12:![0-9]+]]
+; CHECK: [[MIDDLE_BLOCK]]:
+; CHECK-NEXT: [[TMP12:%.*]] = call i64 @llvm.experimental.cttz.elts.i64.v4i1(<4 x i1> [[TMP2]], i1 false)
+; CHECK-NEXT: [[TMP4:%.*]] = trunc i64 [[TMP12]] to i32
+; CHECK-NEXT: [[TMP10:%.*]] = add i32 [[INDEX]], [[TMP4]]
+; CHECK-NEXT: [[TMP11:%.*]] = icmp eq i32 [[TMP10]], 123
+; CHECK-NEXT: br i1 [[TMP11]], label %[[EXIT:.*]], label %[[SCALAR_PH:.*]]
+; CHECK: [[SCALAR_PH]]:
+; CHECK-NEXT: br label %[[LOOP:.*]]
+; CHECK: [[LOOP]]:
+; CHECK-NEXT: [[IV:%.*]] = phi i32 [ [[TMP10]], %[[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], %[[LATCH:.*]] ]
+; CHECK-NEXT: [[ST_ADDR:%.*]] = getelementptr inbounds nuw i32, ptr [[DST]], i32 [[IV]]
+; CHECK-NEXT: [[DATA:%.*]] = load i32, ptr [[ST_ADDR]], align 4
+; CHECK-NEXT: [[INC:%.*]] = add nsw i32 [[DATA]], 1
+; CHECK-NEXT: store i32 [[INC]], ptr [[ST_ADDR]], align 4
+; CHECK-NEXT: [[EE_ADDR:%.*]] = getelementptr inbounds nuw i32, ptr [[PRED]], i32 [[IV]]
+; CHECK-NEXT: [[EE_VAL:%.*]] = load i32, ptr [[EE_ADDR]], align 4
+; CHECK-NEXT: [[EE_COND:%.*]] = icmp sgt i32 [[EE_VAL]], 500
+; CHECK-NEXT: br i1 [[EE_COND]], label %[[EXIT]], label %[[LATCH]]
+; CHECK: [[LATCH]]:
+; CHECK-NEXT: [[IV_NEXT]] = add nuw nsw i32 [[IV]], 1
+; CHECK-NEXT: [[EC:%.*]] = icmp eq i32 [[IV_NEXT]], 123
+; CHECK-NEXT: br i1 [[EC]], label %[[EXIT]], label %[[LOOP]], !llvm.loop [[LOOP13:![0-9]+]]
+; CHECK: [[EXIT]]:
+; CHECK-NEXT: ret void
+;
+entry:
+ br label %loop
+
+loop:
+ %iv = phi i32 [ 0, %entry ], [ %iv.next, %latch ]
+ %st.addr = getelementptr inbounds nuw i32, ptr %dst, i32 %iv
+ %data = load i32, ptr %st.addr, align 4
+ %inc = add nsw i32 %data, 1
+ store i32 %inc, ptr %st.addr, align 4
+ %ee.addr = getelementptr inbounds nuw i32, ptr %pred, i32 %iv
+ %ee.val = load i32, ptr %ee.addr, align 4
+ %ee.cond = icmp sgt i32 %ee.val, 500
+ br i1 %ee.cond, label %exit, label %latch
+
+latch:
+ %iv.next = add nuw nsw i32 %iv, 1
+ %ec = icmp eq i32 %iv.next, 123
+ br i1 %ec, label %exit, label %loop
+
+exit:
+ ret void
+}
+
+define void @early_exit_with_store_after_exit(ptr align 4 dereferenceable(4096) noalias %dst, ptr align 4 dereferenceable(4096) %pred) {
+; CHECK-LABEL: define void @early_exit_with_store_after_exit(
+; CHECK-SAME: ptr noalias align 4 dereferenceable(4096) [[DST:%.*]], ptr align 4 dereferenceable(4096) [[PRED:%.*]]) {
+; CHECK-NEXT: [[ENTRY:.*:]]
+; CHECK-NEXT: br label %[[VECTOR_PH:.*]]
+; CHECK: [[VECTOR_PH]]:
+; CHECK-NEXT: br label %[[VECTOR_BODY:.*]]
+; CHECK: [[VECTOR_BODY]]:
+; CHECK-NEXT: [[INDEX:%.*]] = phi i32 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY_INTERIM:.*]] ]
+; CHECK-NEXT: [[TMP0:%.*]] = getelementptr inbounds nuw i32, ptr [[PRED]], i32 [[INDEX]]
+; CHECK-NEXT: [[WIDE_LOAD:%.*]] = load <4 x i32>, ptr [[TMP0]], align 4
+; CHECK-NEXT: [[TMP1:%.*]] = icmp sgt <4 x i32> [[WIDE_LOAD]], splat (i32 500)
+; CHECK-NEXT: [[TMP2:%.*]] = freeze <4 x i1> [[TMP1]]
+; CHECK-NEXT: [[TMP3:%.*]] = call <4 x i1> @llvm.mask.beforefirst.v4i1(<4 x i1> [[TMP2]])
+; CHECK-NEXT: [[TMP4:%.*]] = getelementptr i32, ptr [[DST]], i32 [[INDEX]]
+; CHECK-NEXT: [[WIDE_MASKED_LOAD:%.*]] = call <4 x i32> @llvm.masked.load.v4i32.p0(ptr align 4 [[TMP4]], <4 x i1> [[TMP3]], <4 x i32> poison)
+; CHECK-NEXT: [[TMP5:%.*]] = add nsw <4 x i32> [[WIDE_MASKED_LOAD]], splat (i32 1)
+; CHECK-NEXT: call void @llvm.masked.store.v4i32.p0(<4 x i32> [[TMP5]], ptr align 4 [[TMP4]], <4 x i1> [[TMP3]])
+; CHECK-NEXT: [[TMP6:%.*]] = call i1 @llvm.vector.reduce.or.v4i1(<4 x i1> [[TMP2]])
+; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i32 [[INDEX]], 4
+; CHECK-NEXT: [[TMP7:%.*]] = icmp eq i32 [[INDEX_NEXT]], 120
+; CHECK-NEXT: br i1 [[TMP6]], label %[[VECTOR_EARLY_EXIT:.*]], label %[[VECTOR_BODY_INTERIM]]
+; CHECK: [[VECTOR_BODY_INTERIM]]:
+; CHECK-NEXT: br i1 [[TMP7]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP14:![0-9]+]]
+; CHECK: [[MIDDLE_BLOCK]]:
+; CHECK-NEXT: br label %[[SCALAR_PH:.*]]
+; CHECK: [[VECTOR_EARLY_EXIT]]:
+; CHECK-NEXT: br label %[[EXIT:.*]]
+; CHECK: [[SCALAR_PH]]:
+; CHECK-NEXT: br label %[[LOOP:.*]]
+; CHECK: [[LOOP]]:
+; CHECK-NEXT: [[IV:%.*]] = phi i32 [ 120, %[[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], %[[LATCH:.*]] ]
+; CHECK-NEXT: [[EE_ADDR:%.*]] = getelementptr inbounds nuw i32, ptr [[PRED]], i32 [[IV]]
+; CHECK-NEXT: [[EE_VAL:%.*]] = load i32, ptr [[EE_ADDR]], align 4
+; CHECK-NEXT: [[EE_COND:%.*]] = icmp sgt i32 [[EE_VAL]], 500
+; CHECK-NEXT: br i1 [[EE_COND]], label %[[EXIT]], label %[[LATCH]]
+; CHECK: [[LATCH]]:
+; CHECK-NEXT: [[ST_ADDR:%.*]] = getelementptr inbounds nuw i32, ptr [[DST]], i32 [[IV]]
+; CHECK-NEXT: [[DATA:%.*]] = load i32, ptr [[ST_ADDR]], align 4
+; CHECK-NEXT: [[INC:%.*]] = add nsw i32 [[DATA]], 1
+; CHECK-NEXT: store i32 [[INC]], ptr [[ST_ADDR]], align 4
+; CHECK-NEXT: [[IV_NEXT]] = add nuw nsw i32 [[IV]], 1
+; CHECK-NEXT: [[EC:%.*]] = icmp eq i32 [[IV_NEXT]], 123
+; CHECK-NEXT: br i1 [[EC]], label %[[EXIT]], label %[[LOOP]], !llvm.loop [[LOOP15:![0-9]+]]
+; CHECK: [[EXIT]]:
+; CHECK-NEXT: ret void
+;
+entry:
+ br label %loop
+
+loop:
+ %iv = phi i32 [ 0, %entry ], [ %iv.next, %latch ]
+ %ee.addr = getelementptr inbounds nuw i32, ptr %pred, i32 %iv
+ %ee.val = load i32, ptr %ee.addr
+ %ee.cond = icmp sgt i32 %ee.val, 500
+ br i1 %ee.cond, label %exit, label %latch
+
+latch:
+ %st.addr = getelementptr inbounds nuw i32, ptr %dst, i32 %iv
+ %data = load i32, ptr %st.addr
+ %inc = add nsw i32 %data, 1
+ store i32 %inc, ptr %st.addr
+ %iv.next = add nuw nsw i32 %iv, 1
+ %ec = icmp eq i32 %iv.next, 123
+ br i1 %ec, label %exit, label %loop
+
+exit:
+ ret void
+}
>From ba0ec3c1ba2fe234229e22f1b4a221024e01ea15 Mon Sep 17 00:00:00 2001
From: Luke Lau <luke at igalia.com>
Date: Sat, 31 Oct 2026 02:24:34 +0800
Subject: [PATCH 15/15] [VPlan] Support tail folding uncountable early exits
Enables the first step towards #179595. After #237254, the MaskedHandleExitInVectorLoop early exit style uses the CFG to handle masking for early exit loops. This allows it to compose with tail folding which is also handled via the CFG in #176143.
This allows us to tail fold the following loop:
```c
extern char x[256];
int f(char * restrict y) {
for (int i = 0; i < 256; i++) {
if (x[i] == 0)
return i;
}
return -1;
}
```
Into this on RISC-V:
```asm
.LBB0_1:
vsetvli a4, a1, e8, m2, ta, ma
add a5, a2, a0
vle8.v v10, (a5)
vmseq.vi v8, v10, 0
vcpop.m a5, v8
bnez a5, .LBB0_4
sub a1, a1, a4
add a0, a0, a4
bnez a1, .LBB0_1
```
foldTailByMasking is updated to make sure the combined early exit condition is false on the lanes past the header mask.
And a fold was added to turn the VPInstruction::AnyOf(header-mask && exit-cond) in the vector body into a vp.reduce.or intrinsic.
This PR only enables tail folding for early exit loops with no side effects. Side effects will be enabled in the next PR. data-and-control-flow tail folding styles are also not handled in this PR.
---
.../Vectorize/LoopVectorizationLegality.cpp | 20 +-
.../Transforms/Vectorize/LoopVectorize.cpp | 8 +
llvm/lib/Transforms/Vectorize/VPlan.h | 7 +
.../Vectorize/VPlanConstruction.cpp | 50 ++--
.../Vectorize/VPlanEVLTailFolding.cpp | 5 +
.../RISCV/early-exit-live-out.ll | 249 +++++++-----------
.../LoopVectorize/RISCV/early-exit.ll | 226 ++++++----------
.../LoopVectorize/RISCV/sink-to-early-exit.ll | 57 ++--
.../LoopVectorize/early-exit-tail-folding.ll | 159 ++++-------
.../LoopVectorize/multiple-early-exits.ll | 118 ++++++++-
10 files changed, 408 insertions(+), 491 deletions(-)
diff --git a/llvm/lib/Transforms/Vectorize/LoopVectorizationLegality.cpp b/llvm/lib/Transforms/Vectorize/LoopVectorizationLegality.cpp
index 6e7a7feff12e29..0cd4b7ec2ffb45 100644
--- a/llvm/lib/Transforms/Vectorize/LoopVectorizationLegality.cpp
+++ b/llvm/lib/Transforms/Vectorize/LoopVectorizationLegality.cpp
@@ -2104,17 +2104,21 @@ bool LoopVectorizationLegality::canFoldTailByMasking() const {
// a bottom-test and a single exiting block. We'd have to handle the fact
// that not every instruction executes on the last iteration. This will
// require a lane mask which varies through the vector loop body. (TODO)
- if (TheLoop->getExitingBlock() != TheLoop->getLoopLatch()) {
- LLVM_DEBUG(
- dbgs()
- << "LV: Cannot fold tail by masking. Requires a singe latch exit\n");
+ bool LatchIsOnlyCountableExit =
+ hasUncountableEarlyExit()
+ ? all_of(getCountableExitingBlocks(),
+ equal_to(TheLoop->getLoopLatch()))
+ : TheLoop->getExitingBlock() == TheLoop->getLoopLatch();
+ if (!LatchIsOnlyCountableExit) {
+ LLVM_DEBUG(dbgs() << "LV: Cannot fold tail by masking. Requires the latch "
+ "to be the only countable exit\n");
return false;
}
- // TODO: Support tail folding with uncountable exits.
- if (hasUncountableEarlyExit()) {
- LLVM_DEBUG(dbgs() << "LV: Cannot tail fold by masking. Loop contains an "
- "uncountable early exit.\n");
+ // TODO: Handle early exits with side effects without a scalar tail.
+ if (hasUncountableExitWithSideEffects()) {
+ LLVM_DEBUG(dbgs() << "LV: Cannot fold tail by masking. Uncountable exit "
+ "with side effects requires a scalar epilogue\n");
return false;
}
diff --git a/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp b/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
index 9e35bc510892bb..70a11792f76fd9 100644
--- a/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
+++ b/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
@@ -715,6 +715,14 @@ class LoopVectorizationCostModel {
ChosenTailFoldingStyle = Opts.force_tail_folding_style.value_or(
TTI.getPreferredTailFoldingStyle());
+ // TODO: Support uncountable early exits w/ DataAndControlFlow. Both
+ // adjust the latch terminator structure.
+ if (ChosenTailFoldingStyle == TailFoldingStyle::DataAndControlFlow &&
+ Legal->hasUncountableEarlyExit()) {
+ ChosenTailFoldingStyle = TailFoldingStyle::None;
+ return;
+ }
+
if (ChosenTailFoldingStyle != TailFoldingStyle::DataWithEVL)
return;
// Override EVL styles if needed.
diff --git a/llvm/lib/Transforms/Vectorize/VPlan.h b/llvm/lib/Transforms/Vectorize/VPlan.h
index d2354f58aba100..34036cf0ae2b6c 100644
--- a/llvm/lib/Transforms/Vectorize/VPlan.h
+++ b/llvm/lib/Transforms/Vectorize/VPlan.h
@@ -3459,6 +3459,13 @@ class LLVM_ABI_FOR_TEST VPReductionEVLRecipe : public VPReductionRecipe {
R.getVFScaleFactor()),
DL) {}
+ VPReductionEVLRecipe(const RecurKind RdxKind, FastMathFlags FMFs,
+ VPValue *ChainOp, VPValue *VecOp, VPValue *CondOp,
+ VPValue *EVL, ReductionStyle Style,
+ DebugLoc DL = DebugLoc::getUnknown())
+ : VPReductionRecipe(VPRecipeBase::VPReductionSC, RdxKind, FMFs, nullptr,
+ {ChainOp, VecOp, EVL}, CondOp, Style, DL) {}
+
~VPReductionEVLRecipe() override = default;
VPReductionEVLRecipe *clone() override {
diff --git a/llvm/lib/Transforms/Vectorize/VPlanConstruction.cpp b/llvm/lib/Transforms/Vectorize/VPlanConstruction.cpp
index ff009b49a79fa6..7c944c224f8080 100644
--- a/llvm/lib/Transforms/Vectorize/VPlanConstruction.cpp
+++ b/llvm/lib/Transforms/Vectorize/VPlanConstruction.cpp
@@ -1373,15 +1373,9 @@ void VPlanTransforms::createLoopRegions(VPlan &Plan, DebugLoc DL) {
}
void VPlanTransforms::foldTailByMasking(VPlan &Plan) {
- assert(Plan.getExitBlocks().size() == 1 &&
- "only a single-exit block is supported currently");
- assert(Plan.getExitBlocks().front()->getSinglePredecessor() ==
- Plan.getMiddleBlock() &&
- "the exit block must have middle block as single predecessor");
-
VPRegionBlock *LoopRegion = Plan.getVectorLoopRegion();
- assert(LoopRegion->getSingleSuccessor() == Plan.getMiddleBlock() &&
- "The vector loop region must have the middle block as its single "
+ assert(LoopRegion->getSuccessors().back() == Plan.getMiddleBlock() &&
+ "The vector loop region must have the middle block as its "
"successor for now");
VPBasicBlock *Header = LoopRegion->getEntryBasicBlock();
@@ -1393,21 +1387,39 @@ void VPlanTransforms::foldTailByMasking(VPlan &Plan) {
Builder.createNaryOp(VPInstruction::BranchOnCond, HeaderMask);
VPBasicBlock *OrigLatch = LoopRegion->getExitingBasicBlock();
- VPValue *IVInc = vputils::findCanonicalIVIncrement(Plan);
- assert(IVInc &&
- std::next(IVInc->getDefiningRecipe()->getIterator()) ==
- OrigLatch->getTerminator()->getIterator() &&
+ VPInstruction *IVInc = vputils::findCanonicalIVIncrement(Plan);
+ assert(IVInc && "Expected a canonical iv increment");
+ [[maybe_unused]] auto NextIt = std::next(IVInc->getIterator());
+ assert((NextIt == OrigLatch->getTerminator()->getIterator() ||
+ (match(&*NextIt,
+ m_SpecificICmp(CmpInst::ICMP_EQ, m_Specific(IVInc),
+ m_Specific(&Plan.getVectorTripCount()))) &&
+ match(&*std::next(NextIt),
+ m_BranchOnTwoConds(
+ m_VPValue(), m_Specific(NextIt->getVPSingleValue()))))) &&
"Unexpected canonical iv increment");
// Split the latch at the IV update, and branch to it from the header mask.
- VPBasicBlock *Latch =
- OrigLatch->splitAt(IVInc->getDefiningRecipe()->getIterator());
+ VPBasicBlock *Latch = OrigLatch->splitAt(IVInc->getIterator());
Latch->setName("vector.latch");
VPBlockUtils::connectBlocks(Header, Latch);
+ // The early exit condition is false on the tail lanes. Sink the any-of to
+ // the new latch, operating on a phi that is false when the header mask is.
+ VPSingleDefRecipe *AnyOf;
+ VPValue *AllExitConds;
+ if (match(Latch->getTerminator(),
+ m_BranchOnTwoConds(m_CombineAnd(m_VPSingleDefRecipe(AnyOf),
+ m_AnyOf(m_VPValue(AllExitConds))),
+ m_VPValue()))) {
+ Builder.setInsertPoint(Latch->begin());
+ VPPhi *Phi = Builder.createScalarPhi({AllExitConds, Plan.getFalse()});
+ AllExitConds->replaceUsesWithIf(Phi, not_equal_to(Phi));
+ AnyOf->moveBefore(IVInc->getIterator());
+ }
+
// Collect any values defined in the loop that need a phi, that is header phis
- // and any value defined n the loop and used in the middle block.
- // TODO: Handle early exits via Plan.getExitBlocks()
+ // and any value defined in the loop and used after the loop region.
MapVector<VPValue *, SmallVector<VPUser *>> NeedsPhi;
auto CollectBypassedOperands = [&](VPRecipeBase &R) {
for (VPValue *Op : R.operands()) {
@@ -1419,8 +1431,10 @@ void VPlanTransforms::foldTailByMasking(VPlan &Plan) {
};
for (VPRecipeBase &R : Header->phis())
CollectBypassedOperands(R);
- for (VPRecipeBase &R : *Plan.getMiddleBlock())
- CollectBypassedOperands(R);
+ for (VPBasicBlock *VPBB : VPBlockUtils::blocksOnly<VPBasicBlock>(
+ vp_depth_first_shallow(LoopRegion)))
+ for (VPRecipeBase &R : *VPBB)
+ CollectBypassedOperands(R);
// Insert phis for values coming past the end of the tail.
Builder.setInsertPoint(Latch->begin());
diff --git a/llvm/lib/Transforms/Vectorize/VPlanEVLTailFolding.cpp b/llvm/lib/Transforms/Vectorize/VPlanEVLTailFolding.cpp
index 62ef733d2899dc..bd6543c6f3f7e7 100644
--- a/llvm/lib/Transforms/Vectorize/VPlanEVLTailFolding.cpp
+++ b/llvm/lib/Transforms/Vectorize/VPlanEVLTailFolding.cpp
@@ -178,6 +178,11 @@ static VPRecipeBase *optimizeMaskToEVL(VPValue *HeaderMask,
match(Rdx->getCondOp(), m_RemoveMask(HeaderMask, Mask)))
return new VPReductionEVLRecipe(*Rdx, EVL, Mask);
+ if (match(&CurRecipe, m_AnyOf(m_RemoveMask(HeaderMask, Mask))))
+ return new VPReductionEVLRecipe(
+ RecurKind::Or, {}, Plan->getFalse(), Mask ? Mask : Plan->getTrue(),
+ nullptr, &EVL, getReductionStyle(true, false, 1), DL);
+
if (auto *Interleave = dyn_cast<VPInterleaveRecipe>(&CurRecipe))
if (Interleave->getMask() &&
match(Interleave->getMask(), m_RemoveMask(HeaderMask, Mask)))
diff --git a/llvm/test/Transforms/LoopVectorize/RISCV/early-exit-live-out.ll b/llvm/test/Transforms/LoopVectorize/RISCV/early-exit-live-out.ll
index 6878c499c94137..005f52876257b2 100644
--- a/llvm/test/Transforms/LoopVectorize/RISCV/early-exit-live-out.ll
+++ b/llvm/test/Transforms/LoopVectorize/RISCV/early-exit-live-out.ll
@@ -10,143 +10,104 @@
define i32 @early_exit_live_out(ptr align 4 dereferenceable(4096) %p) {
; RV64-LABEL: define i32 @early_exit_live_out(
; RV64-SAME: ptr align 4 dereferenceable(4096) [[P:%.*]]) #[[ATTR0:[0-9]+]] {
-; RV64-NEXT: [[ENTRY:.*]]:
-; RV64-NEXT: [[TMP0:%.*]] = call i64 @llvm.vscale.i64()
-; RV64-NEXT: [[TMP1:%.*]] = shl nuw i64 [[TMP0]], 2
-; RV64-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 1024, [[TMP1]]
-; RV64-NEXT: br i1 [[MIN_ITERS_CHECK]], label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]]
+; RV64-NEXT: [[ENTRY:.*:]]
+; RV64-NEXT: br label %[[VECTOR_PH:.*]]
; RV64: [[VECTOR_PH]]:
-; RV64-NEXT: [[N_MOD_VF:%.*]] = urem i64 1024, [[TMP1]]
-; RV64-NEXT: [[N_VEC:%.*]] = sub i64 1024, [[N_MOD_VF]]
; RV64-NEXT: br label %[[VECTOR_BODY:.*]]
; RV64: [[VECTOR_BODY]]:
-; RV64-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY_INTERIM:.*]] ]
+; RV64-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[CURRENT_ITERATION_NEXT:%.*]], %[[VECTOR_BODY_INTERIM:.*]] ]
+; RV64-NEXT: [[AVL:%.*]] = phi i64 [ 1024, %[[VECTOR_PH]] ], [ [[AVL_NEXT:%.*]], %[[VECTOR_BODY_INTERIM]] ]
+; RV64-NEXT: [[TMP0:%.*]] = call i32 @llvm.experimental.get.vector.length.i64(i64 [[AVL]], i32 4, i1 true)
; RV64-NEXT: [[TMP4:%.*]] = getelementptr i32, ptr [[P]], i64 [[INDEX]]
-; RV64-NEXT: [[WIDE_LOAD:%.*]] = load <vscale x 4 x i32>, ptr [[TMP4]], align 4
+; RV64-NEXT: [[WIDE_LOAD:%.*]] = call <vscale x 4 x i32> @llvm.vp.load.nxv4i32.p0(ptr align 4 [[TMP4]], <vscale x 4 x i1> splat (i1 true), i32 [[TMP0]])
; RV64-NEXT: [[TMP5:%.*]] = icmp ne <vscale x 4 x i32> [[WIDE_LOAD]], zeroinitializer
; RV64-NEXT: [[TMP6:%.*]] = freeze <vscale x 4 x i1> [[TMP5]]
-; RV64-NEXT: [[TMP7:%.*]] = call i1 @llvm.vector.reduce.or.nxv4i1(<vscale x 4 x i1> [[TMP6]])
-; RV64-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], [[TMP1]]
-; RV64-NEXT: [[TMP8:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
+; RV64-NEXT: [[TMP8:%.*]] = call <vscale x 4 x i1> @llvm.vp.merge.nxv4i1(<vscale x 4 x i1> splat (i1 true), <vscale x 4 x i1> [[TMP6]], <vscale x 4 x i1> zeroinitializer, i32 [[TMP0]])
+; RV64-NEXT: [[TMP12:%.*]] = call i1 @llvm.vp.reduce.or.nxv4i1(i1 false, <vscale x 4 x i1> [[TMP6]], <vscale x 4 x i1> splat (i1 true), i32 [[TMP0]])
+; RV64-NEXT: [[TMP7:%.*]] = or i1 [[TMP12]], false
+; RV64-NEXT: [[TMP10:%.*]] = zext i32 [[TMP0]] to i64
+; RV64-NEXT: [[CURRENT_ITERATION_NEXT]] = add i64 [[TMP10]], [[INDEX]]
+; RV64-NEXT: [[AVL_NEXT]] = sub nuw i64 [[AVL]], [[TMP10]]
+; RV64-NEXT: [[TMP11:%.*]] = icmp eq i64 [[AVL_NEXT]], 0
; RV64-NEXT: br i1 [[TMP7]], label %[[VECTOR_EARLY_EXIT:.*]], label %[[VECTOR_BODY_INTERIM]]
; RV64: [[VECTOR_BODY_INTERIM]]:
-; RV64-NEXT: br i1 [[TMP8]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP0:![0-9]+]]
+; RV64-NEXT: br i1 [[TMP11]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP0:![0-9]+]]
; RV64: [[MIDDLE_BLOCK]]:
-; RV64-NEXT: [[CMP_N:%.*]] = icmp eq i64 1024, [[N_VEC]]
-; RV64-NEXT: br i1 [[CMP_N]], label %[[EXIT:.*]], label %[[SCALAR_PH]]
+; RV64-NEXT: br label %[[LOOP_HEADER:.*]]
; RV64: [[VECTOR_EARLY_EXIT]]:
-; RV64-NEXT: [[FIRST_ACTIVE_LANE:%.*]] = call i64 @llvm.experimental.cttz.elts.i64.nxv4i1(<vscale x 4 x i1> [[TMP6]], i1 false)
+; RV64-NEXT: [[FIRST_ACTIVE_LANE:%.*]] = call i64 @llvm.experimental.cttz.elts.i64.nxv4i1(<vscale x 4 x i1> [[TMP8]], i1 false)
; RV64-NEXT: [[TMP9:%.*]] = extractelement <vscale x 4 x i32> [[WIDE_LOAD]], i64 [[FIRST_ACTIVE_LANE]]
-; RV64-NEXT: br label %[[EXIT]]
-; RV64: [[SCALAR_PH]]:
-; RV64-NEXT: [[BC_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC]], %[[MIDDLE_BLOCK]] ], [ 0, %[[ENTRY]] ]
-; RV64-NEXT: br label %[[LOOP_HEADER:.*]]
+; RV64-NEXT: br label %[[LOOP_HEADER]]
; RV64: [[LOOP_HEADER]]:
-; RV64-NEXT: [[IV:%.*]] = phi i64 [ [[BC_RESUME_VAL]], %[[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], %[[LATCH:.*]] ]
-; RV64-NEXT: [[GEP:%.*]] = getelementptr i32, ptr [[P]], i64 [[IV]]
-; RV64-NEXT: [[LD:%.*]] = load i32, ptr [[GEP]], align 4
-; RV64-NEXT: [[C:%.*]] = icmp eq i32 [[LD]], 0
-; RV64-NEXT: br i1 [[C]], label %[[LATCH]], label %[[EXIT]]
-; RV64: [[LATCH]]:
-; RV64-NEXT: [[IV_NEXT]] = add i64 [[IV]], 1
-; RV64-NEXT: [[EC:%.*]] = icmp eq i64 [[IV_NEXT]], 1024
-; RV64-NEXT: br i1 [[EC]], label %[[EXIT]], label %[[LOOP_HEADER]], !llvm.loop [[LOOP3:![0-9]+]]
-; RV64: [[EXIT]]:
-; RV64-NEXT: [[RET:%.*]] = phi i32 [ [[LD]], %[[LOOP_HEADER]] ], [ 0, %[[LATCH]] ], [ 0, %[[MIDDLE_BLOCK]] ], [ [[TMP9]], %[[VECTOR_EARLY_EXIT]] ]
+; RV64-NEXT: [[RET:%.*]] = phi i32 [ [[TMP9]], %[[VECTOR_EARLY_EXIT]] ], [ 0, %[[MIDDLE_BLOCK]] ]
; RV64-NEXT: ret i32 [[RET]]
;
; RV32-LABEL: define i32 @early_exit_live_out(
; RV32-SAME: ptr align 4 dereferenceable(4096) [[P:%.*]]) #[[ATTR0:[0-9]+]] {
-; RV32-NEXT: [[ENTRY:.*]]:
-; RV32-NEXT: [[TMP0:%.*]] = call i64 @llvm.vscale.i64()
-; RV32-NEXT: [[TMP1:%.*]] = shl nuw i64 [[TMP0]], 2
-; RV32-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 1024, [[TMP1]]
-; RV32-NEXT: br i1 [[MIN_ITERS_CHECK]], label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]]
+; RV32-NEXT: [[ENTRY:.*:]]
+; RV32-NEXT: br label %[[VECTOR_PH:.*]]
; RV32: [[VECTOR_PH]]:
-; RV32-NEXT: [[N_MOD_VF:%.*]] = urem i64 1024, [[TMP1]]
-; RV32-NEXT: [[N_VEC:%.*]] = sub i64 1024, [[N_MOD_VF]]
; RV32-NEXT: br label %[[VECTOR_BODY:.*]]
; RV32: [[VECTOR_BODY]]:
-; RV32-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY_INTERIM:.*]] ]
+; RV32-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[CURRENT_ITERATION_NEXT:%.*]], %[[VECTOR_BODY_INTERIM:.*]] ]
+; RV32-NEXT: [[AVL:%.*]] = phi i64 [ 1024, %[[VECTOR_PH]] ], [ [[AVL_NEXT:%.*]], %[[VECTOR_BODY_INTERIM]] ]
+; RV32-NEXT: [[TMP0:%.*]] = call i32 @llvm.experimental.get.vector.length.i64(i64 [[AVL]], i32 4, i1 true)
; RV32-NEXT: [[TMP4:%.*]] = getelementptr i32, ptr [[P]], i64 [[INDEX]]
-; RV32-NEXT: [[WIDE_LOAD:%.*]] = load <vscale x 4 x i32>, ptr [[TMP4]], align 4
+; RV32-NEXT: [[WIDE_LOAD:%.*]] = call <vscale x 4 x i32> @llvm.vp.load.nxv4i32.p0(ptr align 4 [[TMP4]], <vscale x 4 x i1> splat (i1 true), i32 [[TMP0]])
; RV32-NEXT: [[TMP5:%.*]] = icmp ne <vscale x 4 x i32> [[WIDE_LOAD]], zeroinitializer
; RV32-NEXT: [[TMP6:%.*]] = freeze <vscale x 4 x i1> [[TMP5]]
-; RV32-NEXT: [[TMP7:%.*]] = call i1 @llvm.vector.reduce.or.nxv4i1(<vscale x 4 x i1> [[TMP6]])
-; RV32-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], [[TMP1]]
-; RV32-NEXT: [[TMP8:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
+; RV32-NEXT: [[TMP8:%.*]] = call <vscale x 4 x i1> @llvm.vp.merge.nxv4i1(<vscale x 4 x i1> splat (i1 true), <vscale x 4 x i1> [[TMP6]], <vscale x 4 x i1> zeroinitializer, i32 [[TMP0]])
+; RV32-NEXT: [[TMP12:%.*]] = call i1 @llvm.vp.reduce.or.nxv4i1(i1 false, <vscale x 4 x i1> [[TMP6]], <vscale x 4 x i1> splat (i1 true), i32 [[TMP0]])
+; RV32-NEXT: [[TMP7:%.*]] = or i1 [[TMP12]], false
+; RV32-NEXT: [[TMP10:%.*]] = zext i32 [[TMP0]] to i64
+; RV32-NEXT: [[CURRENT_ITERATION_NEXT]] = add i64 [[TMP10]], [[INDEX]]
+; RV32-NEXT: [[AVL_NEXT]] = sub nuw i64 [[AVL]], [[TMP10]]
+; RV32-NEXT: [[TMP11:%.*]] = icmp eq i64 [[AVL_NEXT]], 0
; RV32-NEXT: br i1 [[TMP7]], label %[[VECTOR_EARLY_EXIT:.*]], label %[[VECTOR_BODY_INTERIM]]
; RV32: [[VECTOR_BODY_INTERIM]]:
-; RV32-NEXT: br i1 [[TMP8]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP0:![0-9]+]]
+; RV32-NEXT: br i1 [[TMP11]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP0:![0-9]+]]
; RV32: [[MIDDLE_BLOCK]]:
-; RV32-NEXT: [[CMP_N:%.*]] = icmp eq i64 1024, [[N_VEC]]
-; RV32-NEXT: br i1 [[CMP_N]], label %[[EXIT:.*]], label %[[SCALAR_PH]]
+; RV32-NEXT: br label %[[LOOP_HEADER:.*]]
; RV32: [[VECTOR_EARLY_EXIT]]:
-; RV32-NEXT: [[FIRST_ACTIVE_LANE:%.*]] = call i32 @llvm.experimental.cttz.elts.i32.nxv4i1(<vscale x 4 x i1> [[TMP6]], i1 false)
+; RV32-NEXT: [[FIRST_ACTIVE_LANE:%.*]] = call i32 @llvm.experimental.cttz.elts.i32.nxv4i1(<vscale x 4 x i1> [[TMP8]], i1 false)
; RV32-NEXT: [[TMP9:%.*]] = extractelement <vscale x 4 x i32> [[WIDE_LOAD]], i32 [[FIRST_ACTIVE_LANE]]
-; RV32-NEXT: br label %[[EXIT]]
-; RV32: [[SCALAR_PH]]:
-; RV32-NEXT: [[BC_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC]], %[[MIDDLE_BLOCK]] ], [ 0, %[[ENTRY]] ]
-; RV32-NEXT: br label %[[LOOP_HEADER:.*]]
+; RV32-NEXT: br label %[[LOOP_HEADER]]
; RV32: [[LOOP_HEADER]]:
-; RV32-NEXT: [[IV:%.*]] = phi i64 [ [[BC_RESUME_VAL]], %[[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], %[[LATCH:.*]] ]
-; RV32-NEXT: [[GEP:%.*]] = getelementptr i32, ptr [[P]], i64 [[IV]]
-; RV32-NEXT: [[LD:%.*]] = load i32, ptr [[GEP]], align 4
-; RV32-NEXT: [[C:%.*]] = icmp eq i32 [[LD]], 0
-; RV32-NEXT: br i1 [[C]], label %[[LATCH]], label %[[EXIT]]
-; RV32: [[LATCH]]:
-; RV32-NEXT: [[IV_NEXT]] = add i64 [[IV]], 1
-; RV32-NEXT: [[EC:%.*]] = icmp eq i64 [[IV_NEXT]], 1024
-; RV32-NEXT: br i1 [[EC]], label %[[EXIT]], label %[[LOOP_HEADER]], !llvm.loop [[LOOP3:![0-9]+]]
-; RV32: [[EXIT]]:
-; RV32-NEXT: [[RET:%.*]] = phi i32 [ [[LD]], %[[LOOP_HEADER]] ], [ 0, %[[LATCH]] ], [ 0, %[[MIDDLE_BLOCK]] ], [ [[TMP9]], %[[VECTOR_EARLY_EXIT]] ]
+; RV32-NEXT: [[RET:%.*]] = phi i32 [ [[TMP9]], %[[VECTOR_EARLY_EXIT]] ], [ 0, %[[MIDDLE_BLOCK]] ]
; RV32-NEXT: ret i32 [[RET]]
;
; ZVE32X-LABEL: define i32 @early_exit_live_out(
; ZVE32X-SAME: ptr align 4 dereferenceable(4096) [[P:%.*]]) #[[ATTR0:[0-9]+]] {
-; ZVE32X-NEXT: [[ENTRY:.*]]:
-; ZVE32X-NEXT: [[TMP0:%.*]] = call i64 @llvm.vscale.i64()
-; ZVE32X-NEXT: [[TMP1:%.*]] = shl nuw i64 [[TMP0]], 2
-; ZVE32X-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 1024, [[TMP1]]
-; ZVE32X-NEXT: br i1 [[MIN_ITERS_CHECK]], label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]]
+; ZVE32X-NEXT: [[ENTRY:.*:]]
+; ZVE32X-NEXT: br label %[[VECTOR_PH:.*]]
; ZVE32X: [[VECTOR_PH]]:
-; ZVE32X-NEXT: [[N_MOD_VF:%.*]] = urem i64 1024, [[TMP1]]
-; ZVE32X-NEXT: [[N_VEC:%.*]] = sub i64 1024, [[N_MOD_VF]]
; ZVE32X-NEXT: br label %[[VECTOR_BODY:.*]]
; ZVE32X: [[VECTOR_BODY]]:
-; ZVE32X-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY_INTERIM:.*]] ]
+; ZVE32X-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[CURRENT_ITERATION_NEXT:%.*]], %[[VECTOR_BODY_INTERIM:.*]] ]
+; ZVE32X-NEXT: [[AVL:%.*]] = phi i64 [ 1024, %[[VECTOR_PH]] ], [ [[AVL_NEXT:%.*]], %[[VECTOR_BODY_INTERIM]] ]
+; ZVE32X-NEXT: [[TMP0:%.*]] = call i32 @llvm.experimental.get.vector.length.i64(i64 [[AVL]], i32 4, i1 true)
; ZVE32X-NEXT: [[TMP4:%.*]] = getelementptr i32, ptr [[P]], i64 [[INDEX]]
-; ZVE32X-NEXT: [[WIDE_LOAD:%.*]] = load <vscale x 4 x i32>, ptr [[TMP4]], align 4
+; ZVE32X-NEXT: [[WIDE_LOAD:%.*]] = call <vscale x 4 x i32> @llvm.vp.load.nxv4i32.p0(ptr align 4 [[TMP4]], <vscale x 4 x i1> splat (i1 true), i32 [[TMP0]])
; ZVE32X-NEXT: [[TMP5:%.*]] = icmp ne <vscale x 4 x i32> [[WIDE_LOAD]], zeroinitializer
; ZVE32X-NEXT: [[TMP6:%.*]] = freeze <vscale x 4 x i1> [[TMP5]]
-; ZVE32X-NEXT: [[TMP7:%.*]] = call i1 @llvm.vector.reduce.or.nxv4i1(<vscale x 4 x i1> [[TMP6]])
-; ZVE32X-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], [[TMP1]]
-; ZVE32X-NEXT: [[TMP8:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
+; ZVE32X-NEXT: [[TMP8:%.*]] = call <vscale x 4 x i1> @llvm.vp.merge.nxv4i1(<vscale x 4 x i1> splat (i1 true), <vscale x 4 x i1> [[TMP6]], <vscale x 4 x i1> zeroinitializer, i32 [[TMP0]])
+; ZVE32X-NEXT: [[TMP12:%.*]] = call i1 @llvm.vp.reduce.or.nxv4i1(i1 false, <vscale x 4 x i1> [[TMP6]], <vscale x 4 x i1> splat (i1 true), i32 [[TMP0]])
+; ZVE32X-NEXT: [[TMP7:%.*]] = or i1 [[TMP12]], false
+; ZVE32X-NEXT: [[TMP10:%.*]] = zext i32 [[TMP0]] to i64
+; ZVE32X-NEXT: [[CURRENT_ITERATION_NEXT]] = add i64 [[TMP10]], [[INDEX]]
+; ZVE32X-NEXT: [[AVL_NEXT]] = sub nuw i64 [[AVL]], [[TMP10]]
+; ZVE32X-NEXT: [[TMP11:%.*]] = icmp eq i64 [[AVL_NEXT]], 0
; ZVE32X-NEXT: br i1 [[TMP7]], label %[[VECTOR_EARLY_EXIT:.*]], label %[[VECTOR_BODY_INTERIM]]
; ZVE32X: [[VECTOR_BODY_INTERIM]]:
-; ZVE32X-NEXT: br i1 [[TMP8]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP0:![0-9]+]]
+; ZVE32X-NEXT: br i1 [[TMP11]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP0:![0-9]+]]
; ZVE32X: [[MIDDLE_BLOCK]]:
-; ZVE32X-NEXT: [[CMP_N:%.*]] = icmp eq i64 1024, [[N_VEC]]
-; ZVE32X-NEXT: br i1 [[CMP_N]], label %[[EXIT:.*]], label %[[SCALAR_PH]]
+; ZVE32X-NEXT: br label %[[LOOP_HEADER:.*]]
; ZVE32X: [[VECTOR_EARLY_EXIT]]:
-; ZVE32X-NEXT: [[FIRST_ACTIVE_LANE:%.*]] = call i32 @llvm.experimental.cttz.elts.i32.nxv4i1(<vscale x 4 x i1> [[TMP6]], i1 false)
+; ZVE32X-NEXT: [[FIRST_ACTIVE_LANE:%.*]] = call i32 @llvm.experimental.cttz.elts.i32.nxv4i1(<vscale x 4 x i1> [[TMP8]], i1 false)
; ZVE32X-NEXT: [[TMP9:%.*]] = extractelement <vscale x 4 x i32> [[WIDE_LOAD]], i32 [[FIRST_ACTIVE_LANE]]
-; ZVE32X-NEXT: br label %[[EXIT]]
-; ZVE32X: [[SCALAR_PH]]:
-; ZVE32X-NEXT: [[BC_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC]], %[[MIDDLE_BLOCK]] ], [ 0, %[[ENTRY]] ]
-; ZVE32X-NEXT: br label %[[LOOP_HEADER:.*]]
+; ZVE32X-NEXT: br label %[[LOOP_HEADER]]
; ZVE32X: [[LOOP_HEADER]]:
-; ZVE32X-NEXT: [[IV:%.*]] = phi i64 [ [[BC_RESUME_VAL]], %[[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], %[[LATCH:.*]] ]
-; ZVE32X-NEXT: [[GEP:%.*]] = getelementptr i32, ptr [[P]], i64 [[IV]]
-; ZVE32X-NEXT: [[LD:%.*]] = load i32, ptr [[GEP]], align 4
-; ZVE32X-NEXT: [[C:%.*]] = icmp eq i32 [[LD]], 0
-; ZVE32X-NEXT: br i1 [[C]], label %[[LATCH]], label %[[EXIT]]
-; ZVE32X: [[LATCH]]:
-; ZVE32X-NEXT: [[IV_NEXT]] = add i64 [[IV]], 1
-; ZVE32X-NEXT: [[EC:%.*]] = icmp eq i64 [[IV_NEXT]], 1024
-; ZVE32X-NEXT: br i1 [[EC]], label %[[EXIT]], label %[[LOOP_HEADER]], !llvm.loop [[LOOP3:![0-9]+]]
-; ZVE32X: [[EXIT]]:
-; ZVE32X-NEXT: [[RET:%.*]] = phi i32 [ [[LD]], %[[LOOP_HEADER]] ], [ 0, %[[LATCH]] ], [ 0, %[[MIDDLE_BLOCK]] ], [ [[TMP9]], %[[VECTOR_EARLY_EXIT]] ]
+; ZVE32X-NEXT: [[RET:%.*]] = phi i32 [ [[TMP9]], %[[VECTOR_EARLY_EXIT]] ], [ 0, %[[MIDDLE_BLOCK]] ]
; ZVE32X-NEXT: ret i32 [[RET]]
;
entry:
@@ -172,112 +133,78 @@ exit:
define i64 @strided_search(ptr align 8 dereferenceable(14784) %p) {
; RV64-LABEL: define i64 @strided_search(
; RV64-SAME: ptr align 8 dereferenceable(14784) [[P:%.*]]) #[[ATTR0]] {
-; RV64-NEXT: [[ENTRY:.*]]:
-; RV64-NEXT: [[TMP0:%.*]] = call i64 @llvm.vscale.i64()
-; RV64-NEXT: [[TMP1:%.*]] = shl nuw i64 [[TMP0]], 1
-; RV64-NEXT: [[TMP2:%.*]] = call i64 @llvm.umax.i64(i64 [[TMP1]], i64 4)
-; RV64-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 132, [[TMP2]]
-; RV64-NEXT: br i1 [[MIN_ITERS_CHECK]], label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]]
+; RV64-NEXT: [[ENTRY:.*:]]
+; RV64-NEXT: br label %[[VECTOR_PH:.*]]
; RV64: [[VECTOR_PH]]:
-; RV64-NEXT: [[N_MOD_VF:%.*]] = urem i64 132, [[TMP1]]
-; RV64-NEXT: [[N_VEC:%.*]] = sub i64 132, [[N_MOD_VF]]
-; RV64-NEXT: [[TMP4:%.*]] = mul i64 [[N_VEC]], 112
-; RV64-NEXT: [[TMP7:%.*]] = trunc i64 [[TMP1]] to i32
; RV64-NEXT: [[SCEVGEP:%.*]] = getelementptr nuw i8, ptr [[P]], i64 88
; RV64-NEXT: br label %[[VECTOR_BODY:.*]]
; RV64: [[VECTOR_BODY]]:
-; RV64-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY_INTERIM:.*]] ]
+; RV64-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[CURRENT_ITERATION_NEXT:%.*]], %[[VECTOR_BODY_INTERIM:.*]] ]
+; RV64-NEXT: [[AVL:%.*]] = phi i64 [ 132, %[[VECTOR_PH]] ], [ [[AVL_NEXT:%.*]], %[[VECTOR_BODY_INTERIM]] ]
+; RV64-NEXT: [[TMP7:%.*]] = call i32 @llvm.experimental.get.vector.length.i64(i64 [[AVL]], i32 2, i1 true)
; RV64-NEXT: [[TMP5:%.*]] = mul nuw i64 [[INDEX]], 112
; RV64-NEXT: [[TMP6:%.*]] = getelementptr nuw i8, ptr [[SCEVGEP]], i64 [[TMP5]]
; RV64-NEXT: [[WIDE_MASKED_GATHER:%.*]] = call <vscale x 2 x i64> @llvm.experimental.vp.strided.load.nxv2i64.p0.i64(ptr align 8 [[TMP6]], i64 112, <vscale x 2 x i1> splat (i1 true), i32 [[TMP7]])
; RV64-NEXT: [[TMP10:%.*]] = icmp eq <vscale x 2 x i64> [[WIDE_MASKED_GATHER]], zeroinitializer
; RV64-NEXT: [[TMP11:%.*]] = freeze <vscale x 2 x i1> [[TMP10]]
-; RV64-NEXT: [[TMP12:%.*]] = call i1 @llvm.vector.reduce.or.nxv2i1(<vscale x 2 x i1> [[TMP11]])
-; RV64-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], [[TMP1]]
-; RV64-NEXT: [[TMP13:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
+; RV64-NEXT: [[TMP8:%.*]] = call <vscale x 2 x i1> @llvm.vp.merge.nxv2i1(<vscale x 2 x i1> splat (i1 true), <vscale x 2 x i1> [[TMP11]], <vscale x 2 x i1> zeroinitializer, i32 [[TMP7]])
+; RV64-NEXT: [[TMP17:%.*]] = call i1 @llvm.vp.reduce.or.nxv2i1(i1 false, <vscale x 2 x i1> [[TMP11]], <vscale x 2 x i1> splat (i1 true), i32 [[TMP7]])
+; RV64-NEXT: [[TMP12:%.*]] = or i1 [[TMP17]], false
+; RV64-NEXT: [[TMP9:%.*]] = zext i32 [[TMP7]] to i64
+; RV64-NEXT: [[CURRENT_ITERATION_NEXT]] = add i64 [[TMP9]], [[INDEX]]
+; RV64-NEXT: [[AVL_NEXT]] = sub nuw i64 [[AVL]], [[TMP9]]
+; RV64-NEXT: [[TMP13:%.*]] = icmp eq i64 [[AVL_NEXT]], 0
; RV64-NEXT: br i1 [[TMP12]], label %[[VECTOR_EARLY_EXIT:.*]], label %[[VECTOR_BODY_INTERIM]]
; RV64: [[VECTOR_BODY_INTERIM]]:
-; RV64-NEXT: br i1 [[TMP13]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP4:![0-9]+]]
+; RV64-NEXT: br i1 [[TMP13]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP3:![0-9]+]]
; RV64: [[MIDDLE_BLOCK]]:
-; RV64-NEXT: [[CMP_N:%.*]] = icmp eq i64 132, [[N_VEC]]
-; RV64-NEXT: br i1 [[CMP_N]], label %[[EXIT:.*]], label %[[SCALAR_PH]]
+; RV64-NEXT: br label %[[LOOP_HEADER:.*]]
; RV64: [[VECTOR_EARLY_EXIT]]:
-; RV64-NEXT: [[TMP14:%.*]] = call i64 @llvm.experimental.cttz.elts.i64.nxv2i1(<vscale x 2 x i1> [[TMP11]], i1 false)
+; RV64-NEXT: [[TMP14:%.*]] = call i64 @llvm.experimental.cttz.elts.i64.nxv2i1(<vscale x 2 x i1> [[TMP8]], i1 false)
; RV64-NEXT: [[TMP15:%.*]] = add i64 [[INDEX]], [[TMP14]]
; RV64-NEXT: [[TMP16:%.*]] = mul i64 [[TMP15]], 112
-; RV64-NEXT: br label %[[EXIT]]
-; RV64: [[SCALAR_PH]]:
-; RV64-NEXT: [[BC_RESUME_VAL:%.*]] = phi i64 [ [[TMP4]], %[[MIDDLE_BLOCK]] ], [ 0, %[[ENTRY]] ]
-; RV64-NEXT: br label %[[LOOP_HEADER:.*]]
+; RV64-NEXT: br label %[[LOOP_HEADER]]
; RV64: [[LOOP_HEADER]]:
-; RV64-NEXT: [[IDX:%.*]] = phi i64 [ [[BC_RESUME_VAL]], %[[SCALAR_PH]] ], [ [[IDX_NEXT:%.*]], %[[LATCH:.*]] ]
-; RV64-NEXT: [[PTR:%.*]] = getelementptr inbounds nuw i8, ptr [[P]], i64 [[IDX]]
-; RV64-NEXT: [[FIELDP:%.*]] = getelementptr inbounds nuw i8, ptr [[PTR]], i64 88
-; RV64-NEXT: [[V:%.*]] = load i64, ptr [[FIELDP]], align 8
-; RV64-NEXT: [[HIT:%.*]] = icmp eq i64 [[V]], 0
-; RV64-NEXT: br i1 [[HIT]], label %[[EXIT]], label %[[LATCH]]
-; RV64: [[LATCH]]:
-; RV64-NEXT: [[IDX_NEXT]] = add nuw nsw i64 [[IDX]], 112
-; RV64-NEXT: [[DONE:%.*]] = icmp eq i64 [[IDX_NEXT]], 14784
-; RV64-NEXT: br i1 [[DONE]], label %[[EXIT]], label %[[LOOP_HEADER]], !llvm.loop [[LOOP5:![0-9]+]]
-; RV64: [[EXIT]]:
-; RV64-NEXT: [[RET:%.*]] = phi i64 [ [[IDX]], %[[LOOP_HEADER]] ], [ -1, %[[LATCH]] ], [ -1, %[[MIDDLE_BLOCK]] ], [ [[TMP16]], %[[VECTOR_EARLY_EXIT]] ]
+; RV64-NEXT: [[RET:%.*]] = phi i64 [ [[TMP16]], %[[VECTOR_EARLY_EXIT]] ], [ -1, %[[MIDDLE_BLOCK]] ]
; RV64-NEXT: ret i64 [[RET]]
;
; RV32-LABEL: define i64 @strided_search(
; RV32-SAME: ptr align 8 dereferenceable(14784) [[P:%.*]]) #[[ATTR0]] {
-; RV32-NEXT: [[ENTRY:.*]]:
-; RV32-NEXT: [[TMP0:%.*]] = call i64 @llvm.vscale.i64()
-; RV32-NEXT: [[TMP1:%.*]] = shl nuw i64 [[TMP0]], 1
-; RV32-NEXT: [[TMP3:%.*]] = call i64 @llvm.umax.i64(i64 [[TMP1]], i64 3)
-; RV32-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 132, [[TMP3]]
-; RV32-NEXT: br i1 [[MIN_ITERS_CHECK]], label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]]
+; RV32-NEXT: [[ENTRY:.*:]]
+; RV32-NEXT: br label %[[VECTOR_PH:.*]]
; RV32: [[VECTOR_PH]]:
-; RV32-NEXT: [[N_MOD_VF:%.*]] = urem i64 132, [[TMP1]]
-; RV32-NEXT: [[N_VEC:%.*]] = sub i64 132, [[N_MOD_VF]]
-; RV32-NEXT: [[TMP4:%.*]] = mul i64 [[N_VEC]], 112
-; RV32-NEXT: [[TMP5:%.*]] = trunc i64 [[TMP1]] to i32
; RV32-NEXT: [[TMP2:%.*]] = getelementptr nuw i8, ptr [[P]], i32 88
; RV32-NEXT: br label %[[VECTOR_BODY:.*]]
; RV32: [[VECTOR_BODY]]:
-; RV32-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY_INTERIM:.*]] ]
+; RV32-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[CURRENT_ITERATION_NEXT:%.*]], %[[VECTOR_BODY_INTERIM:.*]] ]
+; RV32-NEXT: [[AVL:%.*]] = phi i64 [ 132, %[[VECTOR_PH]] ], [ [[AVL_NEXT:%.*]], %[[VECTOR_BODY_INTERIM]] ]
+; RV32-NEXT: [[TMP5:%.*]] = call i32 @llvm.experimental.get.vector.length.i64(i64 [[AVL]], i32 2, i1 true)
; RV32-NEXT: [[TMP6:%.*]] = trunc i64 [[INDEX]] to i32
; RV32-NEXT: [[TMP7:%.*]] = mul nuw i32 [[TMP6]], 112
; RV32-NEXT: [[TMP8:%.*]] = getelementptr nuw i8, ptr [[TMP2]], i32 [[TMP7]]
; RV32-NEXT: [[WIDE_MASKED_GATHER:%.*]] = call <vscale x 2 x i64> @llvm.experimental.vp.strided.load.nxv2i64.p0.i32(ptr align 8 [[TMP8]], i32 112, <vscale x 2 x i1> splat (i1 true), i32 [[TMP5]])
; RV32-NEXT: [[TMP10:%.*]] = icmp eq <vscale x 2 x i64> [[WIDE_MASKED_GATHER]], zeroinitializer
; RV32-NEXT: [[TMP11:%.*]] = freeze <vscale x 2 x i1> [[TMP10]]
-; RV32-NEXT: [[TMP12:%.*]] = call i1 @llvm.vector.reduce.or.nxv2i1(<vscale x 2 x i1> [[TMP11]])
-; RV32-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], [[TMP1]]
-; RV32-NEXT: [[TMP13:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
+; RV32-NEXT: [[TMP9:%.*]] = call <vscale x 2 x i1> @llvm.vp.merge.nxv2i1(<vscale x 2 x i1> splat (i1 true), <vscale x 2 x i1> [[TMP11]], <vscale x 2 x i1> zeroinitializer, i32 [[TMP5]])
+; RV32-NEXT: [[TMP19:%.*]] = call i1 @llvm.vp.reduce.or.nxv2i1(i1 false, <vscale x 2 x i1> [[TMP11]], <vscale x 2 x i1> splat (i1 true), i32 [[TMP5]])
+; RV32-NEXT: [[TMP12:%.*]] = or i1 [[TMP19]], false
+; RV32-NEXT: [[TMP13:%.*]] = zext i32 [[TMP5]] to i64
+; RV32-NEXT: [[CURRENT_ITERATION_NEXT]] = add i64 [[TMP13]], [[INDEX]]
+; RV32-NEXT: [[AVL_NEXT]] = sub nuw i64 [[AVL]], [[TMP13]]
+; RV32-NEXT: [[TMP18:%.*]] = icmp eq i64 [[AVL_NEXT]], 0
; RV32-NEXT: br i1 [[TMP12]], label %[[VECTOR_EARLY_EXIT:.*]], label %[[VECTOR_BODY_INTERIM]]
; RV32: [[VECTOR_BODY_INTERIM]]:
-; RV32-NEXT: br i1 [[TMP13]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP4:![0-9]+]]
+; RV32-NEXT: br i1 [[TMP18]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP3:![0-9]+]]
; RV32: [[MIDDLE_BLOCK]]:
-; RV32-NEXT: [[CMP_N:%.*]] = icmp eq i64 132, [[N_VEC]]
-; RV32-NEXT: br i1 [[CMP_N]], label %[[EXIT:.*]], label %[[SCALAR_PH]]
+; RV32-NEXT: br label %[[LOOP_HEADER:.*]]
; RV32: [[VECTOR_EARLY_EXIT]]:
-; RV32-NEXT: [[TMP14:%.*]] = call i32 @llvm.experimental.cttz.elts.i32.nxv2i1(<vscale x 2 x i1> [[TMP11]], i1 false)
+; RV32-NEXT: [[TMP14:%.*]] = call i32 @llvm.experimental.cttz.elts.i32.nxv2i1(<vscale x 2 x i1> [[TMP9]], i1 false)
; RV32-NEXT: [[TMP15:%.*]] = zext i32 [[TMP14]] to i64
; RV32-NEXT: [[TMP16:%.*]] = add i64 [[INDEX]], [[TMP15]]
; RV32-NEXT: [[TMP17:%.*]] = mul i64 [[TMP16]], 112
-; RV32-NEXT: br label %[[EXIT]]
-; RV32: [[SCALAR_PH]]:
-; RV32-NEXT: [[BC_RESUME_VAL:%.*]] = phi i64 [ [[TMP4]], %[[MIDDLE_BLOCK]] ], [ 0, %[[ENTRY]] ]
-; RV32-NEXT: br label %[[LOOP_HEADER:.*]]
+; RV32-NEXT: br label %[[LOOP_HEADER]]
; RV32: [[LOOP_HEADER]]:
-; RV32-NEXT: [[IDX:%.*]] = phi i64 [ [[BC_RESUME_VAL]], %[[SCALAR_PH]] ], [ [[IDX_NEXT:%.*]], %[[LATCH:.*]] ]
-; RV32-NEXT: [[PTR:%.*]] = getelementptr inbounds nuw i8, ptr [[P]], i64 [[IDX]]
-; RV32-NEXT: [[FIELDP:%.*]] = getelementptr inbounds nuw i8, ptr [[PTR]], i64 88
-; RV32-NEXT: [[V:%.*]] = load i64, ptr [[FIELDP]], align 8
-; RV32-NEXT: [[HIT:%.*]] = icmp eq i64 [[V]], 0
-; RV32-NEXT: br i1 [[HIT]], label %[[EXIT]], label %[[LATCH]]
-; RV32: [[LATCH]]:
-; RV32-NEXT: [[IDX_NEXT]] = add nuw nsw i64 [[IDX]], 112
-; RV32-NEXT: [[DONE:%.*]] = icmp eq i64 [[IDX_NEXT]], 14784
-; RV32-NEXT: br i1 [[DONE]], label %[[EXIT]], label %[[LOOP_HEADER]], !llvm.loop [[LOOP5:![0-9]+]]
-; RV32: [[EXIT]]:
-; RV32-NEXT: [[RET:%.*]] = phi i64 [ [[IDX]], %[[LOOP_HEADER]] ], [ -1, %[[LATCH]] ], [ -1, %[[MIDDLE_BLOCK]] ], [ [[TMP17]], %[[VECTOR_EARLY_EXIT]] ]
+; RV32-NEXT: [[RET:%.*]] = phi i64 [ [[TMP17]], %[[VECTOR_EARLY_EXIT]] ], [ -1, %[[MIDDLE_BLOCK]] ]
; RV32-NEXT: ret i64 [[RET]]
;
; ZVE32X-LABEL: define i64 @strided_search(
diff --git a/llvm/test/Transforms/LoopVectorize/RISCV/early-exit.ll b/llvm/test/Transforms/LoopVectorize/RISCV/early-exit.ll
index 1ab2e3461477c8..f8563ff31a417f 100644
--- a/llvm/test/Transforms/LoopVectorize/RISCV/early-exit.ll
+++ b/llvm/test/Transforms/LoopVectorize/RISCV/early-exit.ll
@@ -4,50 +4,36 @@
define i32 @single_early_exit(ptr align 4 dereferenceable(4096) %p) {
; CHECK-LABEL: define i32 @single_early_exit(
; CHECK-SAME: ptr align 4 dereferenceable(4096) [[P:%.*]]) #[[ATTR0:[0-9]+]] {
-; CHECK-NEXT: [[ENTRY:.*]]:
-; CHECK-NEXT: [[TMP0:%.*]] = call i32 @llvm.vscale.i32()
-; CHECK-NEXT: [[TMP1:%.*]] = shl nuw i32 [[TMP0]], 2
-; CHECK-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ult i32 123, [[TMP1]]
-; CHECK-NEXT: br i1 [[MIN_ITERS_CHECK]], label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]]
+; CHECK-NEXT: [[ENTRY:.*:]]
+; CHECK-NEXT: br label %[[VECTOR_PH:.*]]
; CHECK: [[VECTOR_PH]]:
-; CHECK-NEXT: [[N_MOD_VF:%.*]] = urem i32 123, [[TMP1]]
-; CHECK-NEXT: [[N_VEC:%.*]] = sub i32 123, [[N_MOD_VF]]
; CHECK-NEXT: br label %[[VECTOR_BODY:.*]]
; CHECK: [[VECTOR_BODY]]:
-; CHECK-NEXT: [[INDEX:%.*]] = phi i32 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY_INTERIM:.*]] ]
+; CHECK-NEXT: [[INDEX:%.*]] = phi i32 [ 0, %[[VECTOR_PH]] ], [ [[CURRENT_ITERATION_NEXT:%.*]], %[[VECTOR_BODY_INTERIM:.*]] ]
+; CHECK-NEXT: [[AVL:%.*]] = phi i32 [ 123, %[[VECTOR_PH]] ], [ [[AVL_NEXT:%.*]], %[[VECTOR_BODY_INTERIM]] ]
+; CHECK-NEXT: [[TMP0:%.*]] = call i32 @llvm.experimental.get.vector.length.i32(i32 [[AVL]], i32 4, i1 true)
; CHECK-NEXT: [[TMP2:%.*]] = getelementptr inbounds i32, ptr [[P]], i32 [[INDEX]]
-; CHECK-NEXT: [[WIDE_LOAD:%.*]] = load <vscale x 4 x i32>, ptr [[TMP2]], align 4
+; CHECK-NEXT: [[WIDE_LOAD:%.*]] = call <vscale x 4 x i32> @llvm.vp.load.nxv4i32.p0(ptr align 4 [[TMP2]], <vscale x 4 x i1> splat (i1 true), i32 [[TMP0]])
; CHECK-NEXT: [[TMP3:%.*]] = icmp eq <vscale x 4 x i32> [[WIDE_LOAD]], zeroinitializer
; CHECK-NEXT: [[TMP4:%.*]] = freeze <vscale x 4 x i1> [[TMP3]]
-; CHECK-NEXT: [[TMP5:%.*]] = call i1 @llvm.vector.reduce.or.nxv4i1(<vscale x 4 x i1> [[TMP4]])
-; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i32 [[INDEX]], [[TMP1]]
-; CHECK-NEXT: [[TMP6:%.*]] = icmp eq i32 [[INDEX_NEXT]], [[N_VEC]]
+; CHECK-NEXT: [[TMP10:%.*]] = call <vscale x 4 x i1> @llvm.vp.merge.nxv4i1(<vscale x 4 x i1> splat (i1 true), <vscale x 4 x i1> [[TMP4]], <vscale x 4 x i1> zeroinitializer, i32 [[TMP0]])
+; CHECK-NEXT: [[TMP11:%.*]] = call i1 @llvm.vp.reduce.or.nxv4i1(i1 false, <vscale x 4 x i1> [[TMP4]], <vscale x 4 x i1> splat (i1 true), i32 [[TMP0]])
+; CHECK-NEXT: [[TMP5:%.*]] = or i1 [[TMP11]], false
+; CHECK-NEXT: [[CURRENT_ITERATION_NEXT]] = add i32 [[TMP0]], [[INDEX]]
+; CHECK-NEXT: [[AVL_NEXT]] = sub nuw i32 [[AVL]], [[TMP0]]
+; CHECK-NEXT: [[TMP6:%.*]] = icmp eq i32 [[AVL_NEXT]], 0
; CHECK-NEXT: br i1 [[TMP5]], label %[[VECTOR_EARLY_EXIT:.*]], label %[[VECTOR_BODY_INTERIM]]
; CHECK: [[VECTOR_BODY_INTERIM]]:
; CHECK-NEXT: br i1 [[TMP6]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP0:![0-9]+]]
; CHECK: [[MIDDLE_BLOCK]]:
-; CHECK-NEXT: [[CMP_N:%.*]] = icmp eq i32 123, [[N_VEC]]
-; CHECK-NEXT: br i1 [[CMP_N]], label %[[EXIT:.*]], label %[[SCALAR_PH]]
+; CHECK-NEXT: br label %[[LOOP:.*]]
; CHECK: [[VECTOR_EARLY_EXIT]]:
-; CHECK-NEXT: [[TMP7:%.*]] = call i64 @llvm.experimental.cttz.elts.i64.nxv4i1(<vscale x 4 x i1> [[TMP4]], i1 false)
+; CHECK-NEXT: [[TMP7:%.*]] = call i64 @llvm.experimental.cttz.elts.i64.nxv4i1(<vscale x 4 x i1> [[TMP10]], i1 false)
; CHECK-NEXT: [[TMP8:%.*]] = trunc i64 [[TMP7]] to i32
; CHECK-NEXT: [[TMP9:%.*]] = add i32 [[INDEX]], [[TMP8]]
-; CHECK-NEXT: br label %[[EXIT]]
-; CHECK: [[SCALAR_PH]]:
-; CHECK-NEXT: [[BC_RESUME_VAL:%.*]] = phi i32 [ [[N_VEC]], %[[MIDDLE_BLOCK]] ], [ 0, %[[ENTRY]] ]
-; CHECK-NEXT: br label %[[LOOP:.*]]
+; CHECK-NEXT: br label %[[LOOP]]
; CHECK: [[LOOP]]:
-; CHECK-NEXT: [[IV:%.*]] = phi i32 [ [[BC_RESUME_VAL]], %[[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], %[[LATCH:.*]] ]
-; CHECK-NEXT: [[GEP:%.*]] = getelementptr inbounds i32, ptr [[P]], i32 [[IV]]
-; CHECK-NEXT: [[LD:%.*]] = load i32, ptr [[GEP]], align 4
-; CHECK-NEXT: [[C:%.*]] = icmp eq i32 [[LD]], 0
-; CHECK-NEXT: br i1 [[C]], label %[[EXIT]], label %[[LATCH]]
-; CHECK: [[LATCH]]:
-; CHECK-NEXT: [[IV_NEXT]] = add nuw nsw i32 [[IV]], 1
-; CHECK-NEXT: [[EC:%.*]] = icmp eq i32 [[IV_NEXT]], 123
-; CHECK-NEXT: br i1 [[EC]], label %[[EXIT]], label %[[LOOP]], !llvm.loop [[LOOP3:![0-9]+]]
-; CHECK: [[EXIT]]:
-; CHECK-NEXT: [[RES:%.*]] = phi i32 [ [[IV]], %[[LOOP]] ], [ -1, %[[LATCH]] ], [ -1, %[[MIDDLE_BLOCK]] ], [ [[TMP9]], %[[VECTOR_EARLY_EXIT]] ]
+; CHECK-NEXT: [[RES:%.*]] = phi i32 [ [[TMP9]], %[[VECTOR_EARLY_EXIT]] ], [ -1, %[[MIDDLE_BLOCK]] ]
; CHECK-NEXT: ret i32 [[RES]]
;
entry:
@@ -73,42 +59,41 @@ exit:
define i32 @multi_early_exits_same_exits(ptr align 4 dereferenceable(4096) %p, ptr align 4 dereferenceable(4096) %q) {
; CHECK-LABEL: define i32 @multi_early_exits_same_exits(
; CHECK-SAME: ptr align 4 dereferenceable(4096) [[P:%.*]], ptr align 4 dereferenceable(4096) [[Q:%.*]]) #[[ATTR0]] {
-; CHECK-NEXT: [[ENTRY:.*]]:
-; CHECK-NEXT: [[TMP0:%.*]] = call i32 @llvm.vscale.i32()
-; CHECK-NEXT: [[TMP1:%.*]] = shl nuw i32 [[TMP0]], 2
-; CHECK-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ult i32 123, [[TMP1]]
-; CHECK-NEXT: br i1 [[MIN_ITERS_CHECK]], label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]]
+; CHECK-NEXT: [[ENTRY:.*:]]
+; CHECK-NEXT: br label %[[VECTOR_PH:.*]]
; CHECK: [[VECTOR_PH]]:
-; CHECK-NEXT: [[N_MOD_VF:%.*]] = urem i32 123, [[TMP1]]
-; CHECK-NEXT: [[N_VEC:%.*]] = sub i32 123, [[N_MOD_VF]]
; CHECK-NEXT: [[TMP2:%.*]] = call <vscale x 4 x i32> @llvm.stepvector.nxv4i32()
-; CHECK-NEXT: [[BROADCAST_SPLATINSERT:%.*]] = insertelement <vscale x 4 x i32> poison, i32 [[TMP1]], i64 0
-; CHECK-NEXT: [[BROADCAST_SPLAT:%.*]] = shufflevector <vscale x 4 x i32> [[BROADCAST_SPLATINSERT]], <vscale x 4 x i32> poison, <vscale x 4 x i32> zeroinitializer
; CHECK-NEXT: br label %[[VECTOR_BODY:.*]]
; CHECK: [[VECTOR_BODY]]:
-; CHECK-NEXT: [[INDEX:%.*]] = phi i32 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY_INTERIM:.*]] ]
+; CHECK-NEXT: [[INDEX:%.*]] = phi i32 [ 0, %[[VECTOR_PH]] ], [ [[CURRENT_ITERATION_NEXT:%.*]], %[[VECTOR_BODY_INTERIM:.*]] ]
; CHECK-NEXT: [[VEC_IND:%.*]] = phi <vscale x 4 x i32> [ [[TMP2]], %[[VECTOR_PH]] ], [ [[VEC_IND_NEXT:%.*]], %[[VECTOR_BODY_INTERIM]] ]
+; CHECK-NEXT: [[AVL:%.*]] = phi i32 [ 123, %[[VECTOR_PH]] ], [ [[AVL_NEXT:%.*]], %[[VECTOR_BODY_INTERIM]] ]
+; CHECK-NEXT: [[TMP1:%.*]] = call i32 @llvm.experimental.get.vector.length.i32(i32 [[AVL]], i32 4, i1 true)
+; CHECK-NEXT: [[BROADCAST_SPLATINSERT:%.*]] = insertelement <vscale x 4 x i32> poison, i32 [[TMP1]], i64 0
+; CHECK-NEXT: [[BROADCAST_SPLAT:%.*]] = shufflevector <vscale x 4 x i32> [[BROADCAST_SPLATINSERT]], <vscale x 4 x i32> poison, <vscale x 4 x i32> zeroinitializer
; CHECK-NEXT: [[TMP3:%.*]] = getelementptr inbounds i32, ptr [[P]], i32 [[INDEX]]
-; CHECK-NEXT: [[WIDE_LOAD:%.*]] = load <vscale x 4 x i32>, ptr [[TMP3]], align 4
+; CHECK-NEXT: [[WIDE_LOAD:%.*]] = call <vscale x 4 x i32> @llvm.vp.load.nxv4i32.p0(ptr align 4 [[TMP3]], <vscale x 4 x i1> splat (i1 true), i32 [[TMP1]])
; CHECK-NEXT: [[TMP4:%.*]] = icmp eq <vscale x 4 x i32> [[WIDE_LOAD]], zeroinitializer
; CHECK-NEXT: [[TMP5:%.*]] = getelementptr inbounds i32, ptr [[Q]], i32 [[INDEX]]
-; CHECK-NEXT: [[WIDE_LOAD1:%.*]] = load <vscale x 4 x i32>, ptr [[TMP5]], align 4
+; CHECK-NEXT: [[WIDE_LOAD1:%.*]] = call <vscale x 4 x i32> @llvm.vp.load.nxv4i32.p0(ptr align 4 [[TMP5]], <vscale x 4 x i1> splat (i1 true), i32 [[TMP1]])
; CHECK-NEXT: [[TMP6:%.*]] = icmp eq <vscale x 4 x i32> [[WIDE_LOAD1]], zeroinitializer
; CHECK-NEXT: [[TMP7:%.*]] = add <vscale x 4 x i32> [[VEC_IND]], splat (i32 1000)
; CHECK-NEXT: [[TMP8:%.*]] = select <vscale x 4 x i1> [[TMP4]], <vscale x 4 x i1> splat (i1 true), <vscale x 4 x i1> [[TMP6]]
; CHECK-NEXT: [[TMP9:%.*]] = freeze <vscale x 4 x i1> [[TMP8]]
-; CHECK-NEXT: [[TMP10:%.*]] = call i1 @llvm.vector.reduce.or.nxv4i1(<vscale x 4 x i1> [[TMP9]])
-; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i32 [[INDEX]], [[TMP1]]
-; CHECK-NEXT: [[TMP11:%.*]] = icmp eq i32 [[INDEX_NEXT]], [[N_VEC]]
+; CHECK-NEXT: [[TMP16:%.*]] = call <vscale x 4 x i1> @llvm.vp.merge.nxv4i1(<vscale x 4 x i1> splat (i1 true), <vscale x 4 x i1> [[TMP9]], <vscale x 4 x i1> zeroinitializer, i32 [[TMP1]])
+; CHECK-NEXT: [[TMP17:%.*]] = call i1 @llvm.vp.reduce.or.nxv4i1(i1 false, <vscale x 4 x i1> [[TMP9]], <vscale x 4 x i1> splat (i1 true), i32 [[TMP1]])
+; CHECK-NEXT: [[TMP10:%.*]] = or i1 [[TMP17]], false
+; CHECK-NEXT: [[CURRENT_ITERATION_NEXT]] = add i32 [[TMP1]], [[INDEX]]
+; CHECK-NEXT: [[AVL_NEXT]] = sub nuw i32 [[AVL]], [[TMP1]]
+; CHECK-NEXT: [[TMP11:%.*]] = icmp eq i32 [[AVL_NEXT]], 0
; CHECK-NEXT: [[VEC_IND_NEXT]] = add nuw nsw <vscale x 4 x i32> [[VEC_IND]], [[BROADCAST_SPLAT]]
; CHECK-NEXT: br i1 [[TMP10]], label %[[VECTOR_EARLY_EXIT_CHECK:.*]], label %[[VECTOR_BODY_INTERIM]]
; CHECK: [[VECTOR_BODY_INTERIM]]:
-; CHECK-NEXT: br i1 [[TMP11]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP4:![0-9]+]]
+; CHECK-NEXT: br i1 [[TMP11]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP3:![0-9]+]]
; CHECK: [[MIDDLE_BLOCK]]:
-; CHECK-NEXT: [[CMP_N:%.*]] = icmp eq i32 123, [[N_VEC]]
-; CHECK-NEXT: br i1 [[CMP_N]], label %[[EXIT:.*]], label %[[SCALAR_PH]]
+; CHECK-NEXT: br label %[[EXIT:.*]]
; CHECK: [[VECTOR_EARLY_EXIT_CHECK]]:
-; CHECK-NEXT: [[FIRST_ACTIVE_LANE:%.*]] = call i64 @llvm.experimental.cttz.elts.i64.nxv4i1(<vscale x 4 x i1> [[TMP9]], i1 false)
+; CHECK-NEXT: [[FIRST_ACTIVE_LANE:%.*]] = call i64 @llvm.experimental.cttz.elts.i64.nxv4i1(<vscale x 4 x i1> [[TMP16]], i1 false)
; CHECK-NEXT: [[TMP12:%.*]] = extractelement <vscale x 4 x i1> [[TMP4]], i64 [[FIRST_ACTIVE_LANE]]
; CHECK-NEXT: br i1 [[TMP12]], label %[[VECTOR_EARLY_EXIT_0:.*]], label %[[VECTOR_EARLY_EXIT_1:.*]]
; CHECK: [[VECTOR_EARLY_EXIT_1]]:
@@ -118,27 +103,8 @@ define i32 @multi_early_exits_same_exits(ptr align 4 dereferenceable(4096) %p, p
; CHECK-NEXT: [[TMP14:%.*]] = trunc i64 [[FIRST_ACTIVE_LANE]] to i32
; CHECK-NEXT: [[TMP15:%.*]] = add i32 [[INDEX]], [[TMP14]]
; CHECK-NEXT: br label %[[EXIT]]
-; CHECK: [[SCALAR_PH]]:
-; CHECK-NEXT: [[BC_RESUME_VAL:%.*]] = phi i32 [ [[N_VEC]], %[[MIDDLE_BLOCK]] ], [ 0, %[[ENTRY]] ]
-; CHECK-NEXT: br label %[[LOOP:.*]]
-; CHECK: [[LOOP]]:
-; CHECK-NEXT: [[IV:%.*]] = phi i32 [ [[BC_RESUME_VAL]], %[[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], %[[LATCH:.*]] ]
-; CHECK-NEXT: [[GEP_P:%.*]] = getelementptr inbounds i32, ptr [[P]], i32 [[IV]]
-; CHECK-NEXT: [[X:%.*]] = load i32, ptr [[GEP_P]], align 4
-; CHECK-NEXT: [[C1:%.*]] = icmp eq i32 [[X]], 0
-; CHECK-NEXT: br i1 [[C1]], label %[[EXIT]], label %[[NEXT:.*]]
-; CHECK: [[NEXT]]:
-; CHECK-NEXT: [[GEP_Q:%.*]] = getelementptr inbounds i32, ptr [[Q]], i32 [[IV]]
-; CHECK-NEXT: [[Y:%.*]] = load i32, ptr [[GEP_Q]], align 4
-; CHECK-NEXT: [[C2:%.*]] = icmp eq i32 [[Y]], 0
-; CHECK-NEXT: [[IV_Q:%.*]] = add i32 [[IV]], 1000
-; CHECK-NEXT: br i1 [[C2]], label %[[EXIT]], label %[[LATCH]]
-; CHECK: [[LATCH]]:
-; CHECK-NEXT: [[IV_NEXT]] = add nuw nsw i32 [[IV]], 1
-; CHECK-NEXT: [[EC:%.*]] = icmp eq i32 [[IV_NEXT]], 123
-; CHECK-NEXT: br i1 [[EC]], label %[[EXIT]], label %[[LOOP]], !llvm.loop [[LOOP5:![0-9]+]]
; CHECK: [[EXIT]]:
-; CHECK-NEXT: [[RES:%.*]] = phi i32 [ [[IV]], %[[LOOP]] ], [ [[IV_Q]], %[[NEXT]] ], [ -1, %[[LATCH]] ], [ -1, %[[MIDDLE_BLOCK]] ], [ [[TMP15]], %[[VECTOR_EARLY_EXIT_0]] ], [ [[TMP13]], %[[VECTOR_EARLY_EXIT_1]] ]
+; CHECK-NEXT: [[RES:%.*]] = phi i32 [ [[TMP13]], %[[VECTOR_EARLY_EXIT_1]] ], [ [[TMP15]], %[[VECTOR_EARLY_EXIT_0]] ], [ -1, %[[MIDDLE_BLOCK]] ]
; CHECK-NEXT: ret i32 [[RES]]
;
entry:
@@ -171,42 +137,41 @@ exit:
define i32 @multi_early_exits_different_exits(ptr align 4 dereferenceable(4096) %p, ptr align 4 dereferenceable(4096) %q) {
; CHECK-LABEL: define i32 @multi_early_exits_different_exits(
; CHECK-SAME: ptr align 4 dereferenceable(4096) [[P:%.*]], ptr align 4 dereferenceable(4096) [[Q:%.*]]) #[[ATTR0]] {
-; CHECK-NEXT: [[ENTRY:.*]]:
-; CHECK-NEXT: [[TMP0:%.*]] = call i32 @llvm.vscale.i32()
-; CHECK-NEXT: [[TMP1:%.*]] = shl nuw i32 [[TMP0]], 2
-; CHECK-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ult i32 123, [[TMP1]]
-; CHECK-NEXT: br i1 [[MIN_ITERS_CHECK]], label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]]
+; CHECK-NEXT: [[ENTRY:.*:]]
+; CHECK-NEXT: br label %[[VECTOR_PH:.*]]
; CHECK: [[VECTOR_PH]]:
-; CHECK-NEXT: [[N_MOD_VF:%.*]] = urem i32 123, [[TMP1]]
-; CHECK-NEXT: [[N_VEC:%.*]] = sub i32 123, [[N_MOD_VF]]
; CHECK-NEXT: [[TMP2:%.*]] = call <vscale x 4 x i32> @llvm.stepvector.nxv4i32()
-; CHECK-NEXT: [[BROADCAST_SPLATINSERT:%.*]] = insertelement <vscale x 4 x i32> poison, i32 [[TMP1]], i64 0
-; CHECK-NEXT: [[BROADCAST_SPLAT:%.*]] = shufflevector <vscale x 4 x i32> [[BROADCAST_SPLATINSERT]], <vscale x 4 x i32> poison, <vscale x 4 x i32> zeroinitializer
; CHECK-NEXT: br label %[[VECTOR_BODY:.*]]
; CHECK: [[VECTOR_BODY]]:
-; CHECK-NEXT: [[INDEX:%.*]] = phi i32 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY_INTERIM:.*]] ]
+; CHECK-NEXT: [[INDEX:%.*]] = phi i32 [ 0, %[[VECTOR_PH]] ], [ [[CURRENT_ITERATION_NEXT:%.*]], %[[VECTOR_BODY_INTERIM:.*]] ]
; CHECK-NEXT: [[VEC_IND:%.*]] = phi <vscale x 4 x i32> [ [[TMP2]], %[[VECTOR_PH]] ], [ [[VEC_IND_NEXT:%.*]], %[[VECTOR_BODY_INTERIM]] ]
+; CHECK-NEXT: [[AVL:%.*]] = phi i32 [ 123, %[[VECTOR_PH]] ], [ [[AVL_NEXT:%.*]], %[[VECTOR_BODY_INTERIM]] ]
+; CHECK-NEXT: [[TMP1:%.*]] = call i32 @llvm.experimental.get.vector.length.i32(i32 [[AVL]], i32 4, i1 true)
+; CHECK-NEXT: [[BROADCAST_SPLATINSERT:%.*]] = insertelement <vscale x 4 x i32> poison, i32 [[TMP1]], i64 0
+; CHECK-NEXT: [[BROADCAST_SPLAT:%.*]] = shufflevector <vscale x 4 x i32> [[BROADCAST_SPLATINSERT]], <vscale x 4 x i32> poison, <vscale x 4 x i32> zeroinitializer
; CHECK-NEXT: [[TMP3:%.*]] = getelementptr inbounds i32, ptr [[P]], i32 [[INDEX]]
-; CHECK-NEXT: [[WIDE_LOAD:%.*]] = load <vscale x 4 x i32>, ptr [[TMP3]], align 4
+; CHECK-NEXT: [[WIDE_LOAD:%.*]] = call <vscale x 4 x i32> @llvm.vp.load.nxv4i32.p0(ptr align 4 [[TMP3]], <vscale x 4 x i1> splat (i1 true), i32 [[TMP1]])
; CHECK-NEXT: [[TMP4:%.*]] = icmp eq <vscale x 4 x i32> [[WIDE_LOAD]], zeroinitializer
; CHECK-NEXT: [[TMP5:%.*]] = getelementptr inbounds i32, ptr [[Q]], i32 [[INDEX]]
-; CHECK-NEXT: [[WIDE_LOAD1:%.*]] = load <vscale x 4 x i32>, ptr [[TMP5]], align 4
+; CHECK-NEXT: [[WIDE_LOAD1:%.*]] = call <vscale x 4 x i32> @llvm.vp.load.nxv4i32.p0(ptr align 4 [[TMP5]], <vscale x 4 x i1> splat (i1 true), i32 [[TMP1]])
; CHECK-NEXT: [[TMP6:%.*]] = icmp eq <vscale x 4 x i32> [[WIDE_LOAD1]], zeroinitializer
; CHECK-NEXT: [[TMP7:%.*]] = add <vscale x 4 x i32> [[VEC_IND]], splat (i32 1000)
; CHECK-NEXT: [[TMP8:%.*]] = select <vscale x 4 x i1> [[TMP4]], <vscale x 4 x i1> splat (i1 true), <vscale x 4 x i1> [[TMP6]]
; CHECK-NEXT: [[TMP9:%.*]] = freeze <vscale x 4 x i1> [[TMP8]]
-; CHECK-NEXT: [[TMP10:%.*]] = call i1 @llvm.vector.reduce.or.nxv4i1(<vscale x 4 x i1> [[TMP9]])
-; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i32 [[INDEX]], [[TMP1]]
-; CHECK-NEXT: [[TMP11:%.*]] = icmp eq i32 [[INDEX_NEXT]], [[N_VEC]]
+; CHECK-NEXT: [[TMP16:%.*]] = call <vscale x 4 x i1> @llvm.vp.merge.nxv4i1(<vscale x 4 x i1> splat (i1 true), <vscale x 4 x i1> [[TMP9]], <vscale x 4 x i1> zeroinitializer, i32 [[TMP1]])
+; CHECK-NEXT: [[TMP17:%.*]] = call i1 @llvm.vp.reduce.or.nxv4i1(i1 false, <vscale x 4 x i1> [[TMP9]], <vscale x 4 x i1> splat (i1 true), i32 [[TMP1]])
+; CHECK-NEXT: [[TMP10:%.*]] = or i1 [[TMP17]], false
+; CHECK-NEXT: [[CURRENT_ITERATION_NEXT]] = add i32 [[TMP1]], [[INDEX]]
+; CHECK-NEXT: [[AVL_NEXT]] = sub nuw i32 [[AVL]], [[TMP1]]
+; CHECK-NEXT: [[TMP11:%.*]] = icmp eq i32 [[AVL_NEXT]], 0
; CHECK-NEXT: [[VEC_IND_NEXT]] = add nuw nsw <vscale x 4 x i32> [[VEC_IND]], [[BROADCAST_SPLAT]]
; CHECK-NEXT: br i1 [[TMP10]], label %[[VECTOR_EARLY_EXIT_CHECK:.*]], label %[[VECTOR_BODY_INTERIM]]
; CHECK: [[VECTOR_BODY_INTERIM]]:
-; CHECK-NEXT: br i1 [[TMP11]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP6:![0-9]+]]
+; CHECK-NEXT: br i1 [[TMP11]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP4:![0-9]+]]
; CHECK: [[MIDDLE_BLOCK]]:
-; CHECK-NEXT: [[CMP_N:%.*]] = icmp eq i32 123, [[N_VEC]]
-; CHECK-NEXT: br i1 [[CMP_N]], label %[[EXIT_LATCH:.*]], label %[[SCALAR_PH]]
+; CHECK-NEXT: br label %[[EXIT_LATCH:.*]]
; CHECK: [[VECTOR_EARLY_EXIT_CHECK]]:
-; CHECK-NEXT: [[FIRST_ACTIVE_LANE:%.*]] = call i64 @llvm.experimental.cttz.elts.i64.nxv4i1(<vscale x 4 x i1> [[TMP9]], i1 false)
+; CHECK-NEXT: [[FIRST_ACTIVE_LANE:%.*]] = call i64 @llvm.experimental.cttz.elts.i64.nxv4i1(<vscale x 4 x i1> [[TMP16]], i1 false)
; CHECK-NEXT: [[TMP12:%.*]] = extractelement <vscale x 4 x i1> [[TMP4]], i64 [[FIRST_ACTIVE_LANE]]
; CHECK-NEXT: br i1 [[TMP12]], label %[[VECTOR_EARLY_EXIT_0:.*]], label %[[VECTOR_EARLY_EXIT_1:.*]]
; CHECK: [[VECTOR_EARLY_EXIT_1]]:
@@ -215,32 +180,11 @@ define i32 @multi_early_exits_different_exits(ptr align 4 dereferenceable(4096)
; CHECK: [[VECTOR_EARLY_EXIT_0]]:
; CHECK-NEXT: [[TMP14:%.*]] = trunc i64 [[FIRST_ACTIVE_LANE]] to i32
; CHECK-NEXT: [[TMP15:%.*]] = add i32 [[INDEX]], [[TMP14]]
-; CHECK-NEXT: br label %[[EXIT_P:.*]]
-; CHECK: [[SCALAR_PH]]:
-; CHECK-NEXT: [[BC_RESUME_VAL:%.*]] = phi i32 [ [[N_VEC]], %[[MIDDLE_BLOCK]] ], [ 0, %[[ENTRY]] ]
; CHECK-NEXT: br label %[[LOOP:.*]]
; CHECK: [[LOOP]]:
-; CHECK-NEXT: [[IV:%.*]] = phi i32 [ [[BC_RESUME_VAL]], %[[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], %[[LATCH:.*]] ]
-; CHECK-NEXT: [[GEP_P:%.*]] = getelementptr inbounds i32, ptr [[P]], i32 [[IV]]
-; CHECK-NEXT: [[X:%.*]] = load i32, ptr [[GEP_P]], align 4
-; CHECK-NEXT: [[C1:%.*]] = icmp eq i32 [[X]], 0
-; CHECK-NEXT: br i1 [[C1]], label %[[EXIT_P]], label %[[NEXT:.*]]
-; CHECK: [[NEXT]]:
-; CHECK-NEXT: [[GEP_Q:%.*]] = getelementptr inbounds i32, ptr [[Q]], i32 [[IV]]
-; CHECK-NEXT: [[Y:%.*]] = load i32, ptr [[GEP_Q]], align 4
-; CHECK-NEXT: [[C2:%.*]] = icmp eq i32 [[Y]], 0
-; CHECK-NEXT: [[IV_Q:%.*]] = add i32 [[IV]], 1000
-; CHECK-NEXT: br i1 [[C2]], label %[[EXIT_Q]], label %[[LATCH]]
-; CHECK: [[LATCH]]:
-; CHECK-NEXT: [[IV_NEXT]] = add nuw nsw i32 [[IV]], 1
-; CHECK-NEXT: [[EC:%.*]] = icmp eq i32 [[IV_NEXT]], 123
-; CHECK-NEXT: br i1 [[EC]], label %[[EXIT_LATCH]], label %[[LOOP]], !llvm.loop [[LOOP7:![0-9]+]]
-; CHECK: [[EXIT_P]]:
-; CHECK-NEXT: [[RES_P:%.*]] = phi i32 [ [[IV]], %[[LOOP]] ], [ [[TMP15]], %[[VECTOR_EARLY_EXIT_0]] ]
-; CHECK-NEXT: ret i32 [[RES_P]]
+; CHECK-NEXT: ret i32 [[TMP15]]
; CHECK: [[EXIT_Q]]:
-; CHECK-NEXT: [[RES_Q:%.*]] = phi i32 [ [[IV_Q]], %[[NEXT]] ], [ [[TMP13]], %[[VECTOR_EARLY_EXIT_1]] ]
-; CHECK-NEXT: ret i32 [[RES_Q]]
+; CHECK-NEXT: ret i32 [[TMP13]]
; CHECK: [[EXIT_LATCH]]:
; CHECK-NEXT: ret i32 -1
;
@@ -281,56 +225,40 @@ exit.latch:
define i32 @predicated_early_exit(i1 %c0, ptr align 4 dereferenceable(4096) %p) {
; CHECK-LABEL: define i32 @predicated_early_exit(
; CHECK-SAME: i1 [[C0:%.*]], ptr align 4 dereferenceable(4096) [[P:%.*]]) #[[ATTR0]] {
-; CHECK-NEXT: [[ENTRY:.*]]:
-; CHECK-NEXT: [[TMP0:%.*]] = call i32 @llvm.vscale.i32()
-; CHECK-NEXT: [[TMP1:%.*]] = shl nuw i32 [[TMP0]], 2
-; CHECK-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ult i32 123, [[TMP1]]
-; CHECK-NEXT: br i1 [[MIN_ITERS_CHECK]], label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]]
+; CHECK-NEXT: [[ENTRY:.*:]]
+; CHECK-NEXT: br label %[[VECTOR_PH:.*]]
; CHECK: [[VECTOR_PH]]:
-; CHECK-NEXT: [[N_MOD_VF:%.*]] = urem i32 123, [[TMP1]]
-; CHECK-NEXT: [[N_VEC:%.*]] = sub i32 123, [[N_MOD_VF]]
; CHECK-NEXT: [[BROADCAST_SPLATINSERT:%.*]] = insertelement <vscale x 4 x i1> poison, i1 [[C0]], i64 0
; CHECK-NEXT: [[BROADCAST_SPLAT:%.*]] = shufflevector <vscale x 4 x i1> [[BROADCAST_SPLATINSERT]], <vscale x 4 x i1> poison, <vscale x 4 x i32> zeroinitializer
; CHECK-NEXT: [[TMP11:%.*]] = freeze <vscale x 4 x i1> [[BROADCAST_SPLAT]]
; CHECK-NEXT: br label %[[VECTOR_BODY:.*]]
; CHECK: [[VECTOR_BODY]]:
-; CHECK-NEXT: [[INDEX:%.*]] = phi i32 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY_INTERIM:.*]] ]
-; CHECK-NEXT: [[TMP2:%.*]] = getelementptr inbounds i32, ptr [[P]], i32 [[INDEX]]
-; CHECK-NEXT: [[WIDE_LOAD:%.*]] = load <vscale x 4 x i32>, ptr [[TMP2]], align 4
+; CHECK-NEXT: [[INDEX:%.*]] = phi i32 [ 0, %[[VECTOR_PH]] ], [ [[CURRENT_ITERATION_NEXT:%.*]], %[[VECTOR_BODY_INTERIM:.*]] ]
+; CHECK-NEXT: [[AVL:%.*]] = phi i32 [ 123, %[[VECTOR_PH]] ], [ [[AVL_NEXT:%.*]], %[[VECTOR_BODY_INTERIM]] ]
+; CHECK-NEXT: [[TMP1:%.*]] = call i32 @llvm.experimental.get.vector.length.i32(i32 [[AVL]], i32 4, i1 true)
+; CHECK-NEXT: [[TMP2:%.*]] = getelementptr i32, ptr [[P]], i32 [[INDEX]]
+; CHECK-NEXT: [[WIDE_LOAD:%.*]] = call <vscale x 4 x i32> @llvm.vp.load.nxv4i32.p0(ptr align 4 [[TMP2]], <vscale x 4 x i1> [[TMP11]], i32 [[TMP1]])
; CHECK-NEXT: [[TMP3:%.*]] = icmp eq <vscale x 4 x i32> [[WIDE_LOAD]], zeroinitializer
; CHECK-NEXT: [[TMP4:%.*]] = select <vscale x 4 x i1> [[TMP11]], <vscale x 4 x i1> [[TMP3]], <vscale x 4 x i1> zeroinitializer
; CHECK-NEXT: [[TMP5:%.*]] = freeze <vscale x 4 x i1> [[TMP4]]
-; CHECK-NEXT: [[TMP6:%.*]] = call i1 @llvm.vector.reduce.or.nxv4i1(<vscale x 4 x i1> [[TMP5]])
-; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i32 [[INDEX]], [[TMP1]]
-; CHECK-NEXT: [[TMP7:%.*]] = icmp eq i32 [[INDEX_NEXT]], [[N_VEC]]
+; CHECK-NEXT: [[TMP7:%.*]] = call <vscale x 4 x i1> @llvm.vp.merge.nxv4i1(<vscale x 4 x i1> splat (i1 true), <vscale x 4 x i1> [[TMP5]], <vscale x 4 x i1> zeroinitializer, i32 [[TMP1]])
+; CHECK-NEXT: [[TMP13:%.*]] = call i1 @llvm.vp.reduce.or.nxv4i1(i1 false, <vscale x 4 x i1> [[TMP5]], <vscale x 4 x i1> splat (i1 true), i32 [[TMP1]])
+; CHECK-NEXT: [[TMP6:%.*]] = or i1 [[TMP13]], false
+; CHECK-NEXT: [[CURRENT_ITERATION_NEXT]] = add i32 [[TMP1]], [[INDEX]]
+; CHECK-NEXT: [[AVL_NEXT]] = sub nuw i32 [[AVL]], [[TMP1]]
+; CHECK-NEXT: [[TMP12:%.*]] = icmp eq i32 [[AVL_NEXT]], 0
; CHECK-NEXT: br i1 [[TMP6]], label %[[VECTOR_EARLY_EXIT:.*]], label %[[VECTOR_BODY_INTERIM]]
; CHECK: [[VECTOR_BODY_INTERIM]]:
-; CHECK-NEXT: br i1 [[TMP7]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP8:![0-9]+]]
+; CHECK-NEXT: br i1 [[TMP12]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP5:![0-9]+]]
; CHECK: [[MIDDLE_BLOCK]]:
-; CHECK-NEXT: [[CMP_N:%.*]] = icmp eq i32 123, [[N_VEC]]
-; CHECK-NEXT: br i1 [[CMP_N]], label %[[EXIT:.*]], label %[[SCALAR_PH]]
+; CHECK-NEXT: br label %[[LOOP:.*]]
; CHECK: [[VECTOR_EARLY_EXIT]]:
-; CHECK-NEXT: [[TMP8:%.*]] = call i64 @llvm.experimental.cttz.elts.i64.nxv4i1(<vscale x 4 x i1> [[TMP5]], i1 false)
+; CHECK-NEXT: [[TMP8:%.*]] = call i64 @llvm.experimental.cttz.elts.i64.nxv4i1(<vscale x 4 x i1> [[TMP7]], i1 false)
; CHECK-NEXT: [[TMP9:%.*]] = trunc i64 [[TMP8]] to i32
; CHECK-NEXT: [[TMP10:%.*]] = add i32 [[INDEX]], [[TMP9]]
-; CHECK-NEXT: br label %[[EXIT]]
-; CHECK: [[SCALAR_PH]]:
-; CHECK-NEXT: [[BC_RESUME_VAL:%.*]] = phi i32 [ [[N_VEC]], %[[MIDDLE_BLOCK]] ], [ 0, %[[ENTRY]] ]
-; CHECK-NEXT: br label %[[LOOP:.*]]
+; CHECK-NEXT: br label %[[LOOP]]
; CHECK: [[LOOP]]:
-; CHECK-NEXT: [[IV:%.*]] = phi i32 [ [[BC_RESUME_VAL]], %[[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], %[[LATCH:.*]] ]
-; CHECK-NEXT: br i1 [[C0]], label %[[THEN:.*]], label %[[LATCH]]
-; CHECK: [[THEN]]:
-; CHECK-NEXT: [[GEP:%.*]] = getelementptr inbounds i32, ptr [[P]], i32 [[IV]]
-; CHECK-NEXT: [[LD:%.*]] = load i32, ptr [[GEP]], align 4
-; CHECK-NEXT: [[C1:%.*]] = icmp eq i32 [[LD]], 0
-; CHECK-NEXT: br i1 [[C1]], label %[[EXIT]], label %[[LATCH]]
-; CHECK: [[LATCH]]:
-; CHECK-NEXT: [[IV_NEXT]] = add nuw nsw i32 [[IV]], 1
-; CHECK-NEXT: [[EC:%.*]] = icmp eq i32 [[IV_NEXT]], 123
-; CHECK-NEXT: br i1 [[EC]], label %[[EXIT]], label %[[LOOP]], !llvm.loop [[LOOP9:![0-9]+]]
-; CHECK: [[EXIT]]:
-; CHECK-NEXT: [[RES:%.*]] = phi i32 [ [[IV]], %[[THEN]] ], [ -1, %[[LATCH]] ], [ -1, %[[MIDDLE_BLOCK]] ], [ [[TMP10]], %[[VECTOR_EARLY_EXIT]] ]
+; CHECK-NEXT: [[RES:%.*]] = phi i32 [ [[TMP10]], %[[VECTOR_EARLY_EXIT]] ], [ -1, %[[MIDDLE_BLOCK]] ]
; CHECK-NEXT: ret i32 [[RES]]
;
entry:
@@ -424,7 +352,7 @@ define void @early_exit_with_store(ptr align 4 dereferenceable(4096) noalias %ds
; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i32 [[INDEX]], [[TMP1]]
; CHECK-NEXT: [[TMP10:%.*]] = icmp eq i32 [[INDEX_NEXT]], [[N_VEC]]
; CHECK-NEXT: [[TMP11:%.*]] = or i1 [[TMP9]], [[TMP10]]
-; CHECK-NEXT: br i1 [[TMP11]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP10:![0-9]+]]
+; CHECK-NEXT: br i1 [[TMP11]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP6:![0-9]+]]
; CHECK: [[MIDDLE_BLOCK]]:
; CHECK-NEXT: [[TMP14:%.*]] = call i64 @llvm.experimental.cttz.elts.i64.nxv4i1(<vscale x 4 x i1> [[TMP4]], i1 false)
; CHECK-NEXT: [[TMP6:%.*]] = trunc i64 [[TMP14]] to i32
@@ -447,7 +375,7 @@ define void @early_exit_with_store(ptr align 4 dereferenceable(4096) noalias %ds
; CHECK: [[LATCH]]:
; CHECK-NEXT: [[IV_NEXT]] = add nuw nsw i32 [[IV]], 1
; CHECK-NEXT: [[EC:%.*]] = icmp eq i32 [[IV_NEXT]], 123
-; CHECK-NEXT: br i1 [[EC]], label %[[EXIT]], label %[[LOOP]], !llvm.loop [[LOOP11:![0-9]+]]
+; CHECK-NEXT: br i1 [[EC]], label %[[EXIT]], label %[[LOOP]], !llvm.loop [[LOOP7:![0-9]+]]
; CHECK: [[EXIT]]:
; CHECK-NEXT: ret void
;
@@ -502,7 +430,7 @@ define void @early_exit_with_store_after_exit(ptr align 4 dereferenceable(4096)
; CHECK-NEXT: [[TMP9:%.*]] = icmp eq i32 [[INDEX_NEXT]], [[N_VEC]]
; CHECK-NEXT: br i1 [[TMP8]], label %[[VECTOR_EARLY_EXIT:.*]], label %[[VECTOR_BODY_INTERIM]]
; CHECK: [[VECTOR_BODY_INTERIM]]:
-; CHECK-NEXT: br i1 [[TMP9]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP12:![0-9]+]]
+; CHECK-NEXT: br i1 [[TMP9]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP8:![0-9]+]]
; CHECK: [[MIDDLE_BLOCK]]:
; CHECK-NEXT: [[CMP_N:%.*]] = icmp eq i32 123, [[N_VEC]]
; CHECK-NEXT: br i1 [[CMP_N]], label %[[EXIT:.*]], label %[[SCALAR_PH]]
@@ -524,7 +452,7 @@ define void @early_exit_with_store_after_exit(ptr align 4 dereferenceable(4096)
; CHECK-NEXT: store i32 [[INC]], ptr [[ST_ADDR]], align 4
; CHECK-NEXT: [[IV_NEXT]] = add nuw nsw i32 [[IV]], 1
; CHECK-NEXT: [[EC:%.*]] = icmp eq i32 [[IV_NEXT]], 123
-; CHECK-NEXT: br i1 [[EC]], label %[[EXIT]], label %[[LOOP]], !llvm.loop [[LOOP13:![0-9]+]]
+; CHECK-NEXT: br i1 [[EC]], label %[[EXIT]], label %[[LOOP]], !llvm.loop [[LOOP9:![0-9]+]]
; CHECK: [[EXIT]]:
; CHECK-NEXT: ret void
;
diff --git a/llvm/test/Transforms/LoopVectorize/RISCV/sink-to-early-exit.ll b/llvm/test/Transforms/LoopVectorize/RISCV/sink-to-early-exit.ll
index c812ab30381f0d..67d1adeaf12f8f 100644
--- a/llvm/test/Transforms/LoopVectorize/RISCV/sink-to-early-exit.ll
+++ b/llvm/test/Transforms/LoopVectorize/RISCV/sink-to-early-exit.ll
@@ -8,67 +8,50 @@ declare void @init_mem(ptr, i64)
define i64 @sink_to_early_exit(i64 %offset) {
; CHECK-LABEL: define i64 @sink_to_early_exit(
; CHECK-SAME: i64 [[OFFSET:%.*]]) #[[ATTR0:[0-9]+]] {
-; CHECK-NEXT: [[ENTRY:.*]]:
+; CHECK-NEXT: [[ENTRY:.*:]]
; CHECK-NEXT: [[P1:%.*]] = alloca [1024 x i8], align 4
; CHECK-NEXT: [[P2:%.*]] = alloca [1024 x i8], align 4
; CHECK-NEXT: call void @init_mem(ptr [[P1]], i64 1024)
; CHECK-NEXT: call void @init_mem(ptr [[P2]], i64 1024)
-; CHECK-NEXT: [[TMP0:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-NEXT: [[TMP1:%.*]] = shl nuw i64 [[TMP0]], 4
-; CHECK-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 1024, [[TMP1]]
-; CHECK-NEXT: br i1 [[MIN_ITERS_CHECK]], label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]]
+; CHECK-NEXT: br label %[[VECTOR_PH:.*]]
; CHECK: [[VECTOR_PH]]:
-; CHECK-NEXT: [[N_MOD_VF:%.*]] = urem i64 1024, [[TMP1]]
-; CHECK-NEXT: [[N_VEC:%.*]] = sub i64 1024, [[N_MOD_VF]]
; CHECK-NEXT: [[BROADCAST_SPLATINSERT:%.*]] = insertelement <vscale x 16 x i64> poison, i64 [[OFFSET]], i64 0
; CHECK-NEXT: [[BROADCAST_SPLAT:%.*]] = shufflevector <vscale x 16 x i64> [[BROADCAST_SPLATINSERT]], <vscale x 16 x i64> poison, <vscale x 16 x i32> zeroinitializer
; CHECK-NEXT: [[TMP4:%.*]] = call <vscale x 16 x i64> @llvm.stepvector.nxv16i64()
-; CHECK-NEXT: [[BROADCAST_SPLATINSERT2:%.*]] = insertelement <vscale x 16 x i64> poison, i64 [[TMP1]], i64 0
-; CHECK-NEXT: [[BROADCAST_SPLAT3:%.*]] = shufflevector <vscale x 16 x i64> [[BROADCAST_SPLATINSERT2]], <vscale x 16 x i64> poison, <vscale x 16 x i32> zeroinitializer
; CHECK-NEXT: br label %[[VECTOR_BODY:.*]]
; CHECK: [[VECTOR_BODY]]:
-; CHECK-NEXT: [[INDEX4:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT6:%.*]], %[[VECTOR_BODY_INTERIM:.*]] ]
+; CHECK-NEXT: [[INDEX4:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[CURRENT_ITERATION_NEXT:%.*]], %[[VECTOR_BODY_INTERIM:.*]] ]
; CHECK-NEXT: [[VEC_IND:%.*]] = phi <vscale x 16 x i64> [ [[TMP4]], %[[VECTOR_PH]] ], [ [[VEC_IND_NEXT:%.*]], %[[VECTOR_BODY_INTERIM]] ]
+; CHECK-NEXT: [[AVL:%.*]] = phi i64 [ 1024, %[[VECTOR_PH]] ], [ [[AVL_NEXT:%.*]], %[[VECTOR_BODY_INTERIM]] ]
+; CHECK-NEXT: [[TMP1:%.*]] = call i32 @llvm.experimental.get.vector.length.i64(i64 [[AVL]], i32 16, i1 true)
+; CHECK-NEXT: [[TMP2:%.*]] = zext i32 [[TMP1]] to i64
+; CHECK-NEXT: [[BROADCAST_SPLATINSERT3:%.*]] = insertelement <vscale x 16 x i64> poison, i64 [[TMP2]], i64 0
+; CHECK-NEXT: [[BROADCAST_SPLAT3:%.*]] = shufflevector <vscale x 16 x i64> [[BROADCAST_SPLATINSERT3]], <vscale x 16 x i64> poison, <vscale x 16 x i32> zeroinitializer
; CHECK-NEXT: [[TMP5:%.*]] = getelementptr inbounds i8, ptr [[P1]], i64 [[INDEX4]]
-; CHECK-NEXT: [[WIDE_LOAD:%.*]] = load <vscale x 16 x i8>, ptr [[TMP5]], align 1
+; CHECK-NEXT: [[WIDE_LOAD:%.*]] = call <vscale x 16 x i8> @llvm.vp.load.nxv16i8.p0(ptr align 1 [[TMP5]], <vscale x 16 x i1> splat (i1 true), i32 [[TMP1]])
; CHECK-NEXT: [[TMP6:%.*]] = getelementptr inbounds i8, ptr [[P2]], i64 [[INDEX4]]
-; CHECK-NEXT: [[WIDE_LOAD5:%.*]] = load <vscale x 16 x i8>, ptr [[TMP6]], align 1
+; CHECK-NEXT: [[WIDE_LOAD5:%.*]] = call <vscale x 16 x i8> @llvm.vp.load.nxv16i8.p0(ptr align 1 [[TMP6]], <vscale x 16 x i1> splat (i1 true), i32 [[TMP1]])
; CHECK-NEXT: [[TMP8:%.*]] = icmp ne <vscale x 16 x i8> [[WIDE_LOAD]], [[WIDE_LOAD5]]
; CHECK-NEXT: [[TMP9:%.*]] = freeze <vscale x 16 x i1> [[TMP8]]
-; CHECK-NEXT: [[TMP10:%.*]] = call i1 @llvm.vector.reduce.or.nxv16i1(<vscale x 16 x i1> [[TMP9]])
-; CHECK-NEXT: [[INDEX_NEXT6]] = add nuw i64 [[INDEX4]], [[TMP1]]
-; CHECK-NEXT: [[TMP11:%.*]] = icmp eq i64 [[INDEX_NEXT6]], [[N_VEC]]
+; CHECK-NEXT: [[TMP11:%.*]] = call <vscale x 16 x i1> @llvm.vp.merge.nxv16i1(<vscale x 16 x i1> splat (i1 true), <vscale x 16 x i1> [[TMP9]], <vscale x 16 x i1> zeroinitializer, i32 [[TMP1]])
+; CHECK-NEXT: [[TMP14:%.*]] = call i1 @llvm.vp.reduce.or.nxv16i1(i1 false, <vscale x 16 x i1> [[TMP9]], <vscale x 16 x i1> splat (i1 true), i32 [[TMP1]])
+; CHECK-NEXT: [[TMP10:%.*]] = or i1 [[TMP14]], false
+; CHECK-NEXT: [[CURRENT_ITERATION_NEXT]] = add i64 [[TMP2]], [[INDEX4]]
+; CHECK-NEXT: [[AVL_NEXT]] = sub nuw i64 [[AVL]], [[TMP2]]
+; CHECK-NEXT: [[TMP13:%.*]] = icmp eq i64 [[AVL_NEXT]], 0
; CHECK-NEXT: [[VEC_IND_NEXT]] = add <vscale x 16 x i64> [[VEC_IND]], [[BROADCAST_SPLAT3]]
; CHECK-NEXT: br i1 [[TMP10]], label %[[VECTOR_EARLY_EXIT:.*]], label %[[VECTOR_BODY_INTERIM]]
; CHECK: [[VECTOR_BODY_INTERIM]]:
-; CHECK-NEXT: br i1 [[TMP11]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP0:![0-9]+]]
+; CHECK-NEXT: br i1 [[TMP13]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP0:![0-9]+]]
; CHECK: [[MIDDLE_BLOCK]]:
-; CHECK-NEXT: [[CMP_N:%.*]] = icmp eq i64 1024, [[N_VEC]]
-; CHECK-NEXT: br i1 [[CMP_N]], label %[[LOOP_END:.*]], label %[[SCALAR_PH]]
+; CHECK-NEXT: br label %[[LOOP_END:.*]]
; CHECK: [[VECTOR_EARLY_EXIT]]:
; CHECK-NEXT: [[TMP7:%.*]] = add <vscale x 16 x i64> [[VEC_IND]], [[BROADCAST_SPLAT]]
-; CHECK-NEXT: [[FIRST_ACTIVE_LANE:%.*]] = call i64 @llvm.experimental.cttz.elts.i64.nxv16i1(<vscale x 16 x i1> [[TMP9]], i1 false)
+; CHECK-NEXT: [[FIRST_ACTIVE_LANE:%.*]] = call i64 @llvm.experimental.cttz.elts.i64.nxv16i1(<vscale x 16 x i1> [[TMP11]], i1 false)
; CHECK-NEXT: [[TMP12:%.*]] = extractelement <vscale x 16 x i64> [[TMP7]], i64 [[FIRST_ACTIVE_LANE]]
-; CHECK-NEXT: br label %[[LOOP_EARLY_EXIT:.*]]
-; CHECK: [[SCALAR_PH]]:
-; CHECK-NEXT: [[BC_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC]], %[[MIDDLE_BLOCK]] ], [ 0, %[[ENTRY]] ]
; CHECK-NEXT: br label %[[LOOP:.*]]
; CHECK: [[LOOP]]:
-; CHECK-NEXT: [[INDEX:%.*]] = phi i64 [ [[INDEX_NEXT:%.*]], %[[LOOP_INC:.*]] ], [ [[BC_RESUME_VAL]], %[[SCALAR_PH]] ]
-; CHECK-NEXT: [[ARRAYIDX:%.*]] = getelementptr inbounds i8, ptr [[P1]], i64 [[INDEX]]
-; CHECK-NEXT: [[LD1:%.*]] = load i8, ptr [[ARRAYIDX]], align 1
-; CHECK-NEXT: [[ARRAYIDX1:%.*]] = getelementptr inbounds i8, ptr [[P2]], i64 [[INDEX]]
-; CHECK-NEXT: [[LD2:%.*]] = load i8, ptr [[ARRAYIDX1]], align 1
-; CHECK-NEXT: [[RESULT:%.*]] = add i64 [[INDEX]], [[OFFSET]]
-; CHECK-NEXT: [[CMP:%.*]] = icmp eq i8 [[LD1]], [[LD2]]
-; CHECK-NEXT: br i1 [[CMP]], label %[[LOOP_INC]], label %[[LOOP_EARLY_EXIT]]
-; CHECK: [[LOOP_INC]]:
-; CHECK-NEXT: [[INDEX_NEXT]] = add i64 [[INDEX]], 1
-; CHECK-NEXT: [[EXITCOND:%.*]] = icmp ne i64 [[INDEX_NEXT]], 1024
-; CHECK-NEXT: br i1 [[EXITCOND]], label %[[LOOP]], label %[[LOOP_END]], !llvm.loop [[LOOP3:![0-9]+]]
-; CHECK: [[LOOP_EARLY_EXIT]]:
-; CHECK-NEXT: [[RESULT_LCSSA:%.*]] = phi i64 [ [[RESULT]], %[[LOOP]] ], [ [[TMP12]], %[[VECTOR_EARLY_EXIT]] ]
-; CHECK-NEXT: ret i64 [[RESULT_LCSSA]]
+; CHECK-NEXT: ret i64 [[TMP12]]
; CHECK: [[LOOP_END]]:
; CHECK-NEXT: ret i64 0
;
diff --git a/llvm/test/Transforms/LoopVectorize/early-exit-tail-folding.ll b/llvm/test/Transforms/LoopVectorize/early-exit-tail-folding.ll
index 9026514723ae5d..cff27685627005 100644
--- a/llvm/test/Transforms/LoopVectorize/early-exit-tail-folding.ll
+++ b/llvm/test/Transforms/LoopVectorize/early-exit-tail-folding.ll
@@ -10,37 +10,29 @@ define i32 @single_early_exit(ptr align 4 dereferenceable(4096) %p) {
; CHECK-NEXT: br label %[[VECTOR_BODY:.*]]
; CHECK: [[VECTOR_BODY]]:
; CHECK-NEXT: [[INDEX:%.*]] = phi i32 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY_INTERIM:.*]] ]
+; CHECK-NEXT: [[VEC_IND:%.*]] = phi <4 x i8> [ <i8 0, i8 1, i8 2, i8 3>, %[[VECTOR_PH]] ], [ [[VEC_IND_NEXT:%.*]], %[[VECTOR_BODY_INTERIM]] ]
+; CHECK-NEXT: [[TMP8:%.*]] = icmp ule <4 x i8> [[VEC_IND]], splat (i8 122)
; CHECK-NEXT: [[TMP0:%.*]] = getelementptr inbounds i32, ptr [[P]], i32 [[INDEX]]
-; CHECK-NEXT: [[WIDE_LOAD:%.*]] = load <4 x i32>, ptr [[TMP0]], align 4
+; CHECK-NEXT: [[WIDE_LOAD:%.*]] = call <4 x i32> @llvm.masked.load.v4i32.p0(ptr align 4 [[TMP0]], <4 x i1> [[TMP8]], <4 x i32> poison)
; CHECK-NEXT: [[TMP1:%.*]] = icmp eq <4 x i32> [[WIDE_LOAD]], zeroinitializer
; CHECK-NEXT: [[TMP2:%.*]] = freeze <4 x i1> [[TMP1]]
-; CHECK-NEXT: [[TMP3:%.*]] = call i1 @llvm.vector.reduce.or.v4i1(<4 x i1> [[TMP2]])
+; CHECK-NEXT: [[TMP4:%.*]] = select <4 x i1> [[TMP8]], <4 x i1> [[TMP2]], <4 x i1> zeroinitializer
+; CHECK-NEXT: [[TMP3:%.*]] = call i1 @llvm.vector.reduce.or.v4i1(<4 x i1> [[TMP4]])
; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i32 [[INDEX]], 4
-; CHECK-NEXT: [[TMP4:%.*]] = icmp eq i32 [[INDEX_NEXT]], 120
+; CHECK-NEXT: [[TMP9:%.*]] = icmp eq i32 [[INDEX_NEXT]], 124
+; CHECK-NEXT: [[VEC_IND_NEXT]] = add nuw <4 x i8> [[VEC_IND]], splat (i8 4)
; CHECK-NEXT: br i1 [[TMP3]], label %[[VECTOR_EARLY_EXIT:.*]], label %[[VECTOR_BODY_INTERIM]]
; CHECK: [[VECTOR_BODY_INTERIM]]:
-; CHECK-NEXT: br i1 [[TMP4]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP0:![0-9]+]]
+; CHECK-NEXT: br i1 [[TMP9]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP0:![0-9]+]]
; CHECK: [[MIDDLE_BLOCK]]:
; CHECK-NEXT: br label %[[EXIT:.*]]
; CHECK: [[VECTOR_EARLY_EXIT]]:
-; CHECK-NEXT: [[TMP5:%.*]] = call i64 @llvm.experimental.cttz.elts.i64.v4i1(<4 x i1> [[TMP2]], i1 false)
+; CHECK-NEXT: [[TMP5:%.*]] = call i64 @llvm.experimental.cttz.elts.i64.v4i1(<4 x i1> [[TMP4]], i1 false)
; CHECK-NEXT: [[TMP6:%.*]] = trunc i64 [[TMP5]] to i32
; CHECK-NEXT: [[TMP7:%.*]] = add i32 [[INDEX]], [[TMP6]]
-; CHECK-NEXT: br label %[[EXIT1:.*]]
+; CHECK-NEXT: br label %[[EXIT]]
; CHECK: [[EXIT]]:
-; CHECK-NEXT: br label %[[LOOP:.*]]
-; CHECK: [[LOOP]]:
-; CHECK-NEXT: [[IV:%.*]] = phi i32 [ 120, %[[EXIT]] ], [ [[IV_NEXT:%.*]], %[[LATCH:.*]] ]
-; CHECK-NEXT: [[GEP:%.*]] = getelementptr inbounds i32, ptr [[P]], i32 [[IV]]
-; CHECK-NEXT: [[LD:%.*]] = load i32, ptr [[GEP]], align 4
-; CHECK-NEXT: [[C:%.*]] = icmp eq i32 [[LD]], 0
-; CHECK-NEXT: br i1 [[C]], label %[[EXIT1]], label %[[LATCH]]
-; CHECK: [[LATCH]]:
-; CHECK-NEXT: [[IV_NEXT]] = add nuw nsw i32 [[IV]], 1
-; CHECK-NEXT: [[EC:%.*]] = icmp eq i32 [[IV_NEXT]], 123
-; CHECK-NEXT: br i1 [[EC]], label %[[EXIT1]], label %[[LOOP]], !llvm.loop [[LOOP3:![0-9]+]]
-; CHECK: [[EXIT1]]:
-; CHECK-NEXT: [[RES:%.*]] = phi i32 [ [[IV]], %[[LOOP]] ], [ -1, %[[LATCH]] ], [ [[TMP7]], %[[VECTOR_EARLY_EXIT]] ]
+; CHECK-NEXT: [[RES:%.*]] = phi i32 [ [[TMP7]], %[[VECTOR_EARLY_EXIT]] ], [ -1, %[[MIDDLE_BLOCK]] ]
; CHECK-NEXT: ret i32 [[RES]]
;
entry:
@@ -73,55 +65,39 @@ define i32 @multi_early_exits_same_exits(ptr align 4 dereferenceable(4096) %p, p
; CHECK: [[VECTOR_BODY]]:
; CHECK-NEXT: [[INDEX:%.*]] = phi i32 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY_INTERIM:.*]] ]
; CHECK-NEXT: [[VEC_IND:%.*]] = phi <4 x i32> [ <i32 0, i32 1, i32 2, i32 3>, %[[VECTOR_PH]] ], [ [[VEC_IND_NEXT:%.*]], %[[VECTOR_BODY_INTERIM]] ]
+; CHECK-NEXT: [[TMP13:%.*]] = icmp ule <4 x i32> [[VEC_IND]], splat (i32 122)
; CHECK-NEXT: [[TMP0:%.*]] = getelementptr inbounds i32, ptr [[P]], i32 [[INDEX]]
-; CHECK-NEXT: [[WIDE_LOAD:%.*]] = load <4 x i32>, ptr [[TMP0]], align 4
+; CHECK-NEXT: [[WIDE_LOAD:%.*]] = call <4 x i32> @llvm.masked.load.v4i32.p0(ptr align 4 [[TMP0]], <4 x i1> [[TMP13]], <4 x i32> poison)
; CHECK-NEXT: [[TMP1:%.*]] = icmp eq <4 x i32> [[WIDE_LOAD]], zeroinitializer
; CHECK-NEXT: [[TMP2:%.*]] = getelementptr inbounds i32, ptr [[Q]], i32 [[INDEX]]
-; CHECK-NEXT: [[WIDE_LOAD1:%.*]] = load <4 x i32>, ptr [[TMP2]], align 4
+; CHECK-NEXT: [[WIDE_LOAD1:%.*]] = call <4 x i32> @llvm.masked.load.v4i32.p0(ptr align 4 [[TMP2]], <4 x i1> [[TMP13]], <4 x i32> poison)
; CHECK-NEXT: [[TMP3:%.*]] = icmp eq <4 x i32> [[WIDE_LOAD1]], zeroinitializer
; CHECK-NEXT: [[TMP4:%.*]] = add <4 x i32> [[VEC_IND]], splat (i32 1000)
; CHECK-NEXT: [[TMP5:%.*]] = select <4 x i1> [[TMP1]], <4 x i1> splat (i1 true), <4 x i1> [[TMP3]]
; CHECK-NEXT: [[TMP6:%.*]] = freeze <4 x i1> [[TMP5]]
-; CHECK-NEXT: [[TMP7:%.*]] = call i1 @llvm.vector.reduce.or.v4i1(<4 x i1> [[TMP6]])
+; CHECK-NEXT: [[TMP8:%.*]] = select <4 x i1> [[TMP13]], <4 x i1> [[TMP6]], <4 x i1> zeroinitializer
+; CHECK-NEXT: [[TMP7:%.*]] = call i1 @llvm.vector.reduce.or.v4i1(<4 x i1> [[TMP8]])
; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i32 [[INDEX]], 4
-; CHECK-NEXT: [[TMP8:%.*]] = icmp eq i32 [[INDEX_NEXT]], 120
-; CHECK-NEXT: [[VEC_IND_NEXT]] = add nuw nsw <4 x i32> [[VEC_IND]], splat (i32 4)
+; CHECK-NEXT: [[TMP14:%.*]] = icmp eq i32 [[INDEX_NEXT]], 124
+; CHECK-NEXT: [[VEC_IND_NEXT]] = add <4 x i32> [[VEC_IND]], splat (i32 4)
; CHECK-NEXT: br i1 [[TMP7]], label %[[VECTOR_EARLY_EXIT_CHECK:.*]], label %[[VECTOR_BODY_INTERIM]]
; CHECK: [[VECTOR_BODY_INTERIM]]:
-; CHECK-NEXT: br i1 [[TMP8]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP4:![0-9]+]]
+; CHECK-NEXT: br i1 [[TMP14]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP3:![0-9]+]]
; CHECK: [[MIDDLE_BLOCK]]:
; CHECK-NEXT: br label %[[EXIT:.*]]
; CHECK: [[VECTOR_EARLY_EXIT_CHECK]]:
-; CHECK-NEXT: [[FIRST_ACTIVE_LANE:%.*]] = call i64 @llvm.experimental.cttz.elts.i64.v4i1(<4 x i1> [[TMP6]], i1 false)
+; CHECK-NEXT: [[FIRST_ACTIVE_LANE:%.*]] = call i64 @llvm.experimental.cttz.elts.i64.v4i1(<4 x i1> [[TMP8]], i1 false)
; CHECK-NEXT: [[TMP9:%.*]] = extractelement <4 x i1> [[TMP1]], i64 [[FIRST_ACTIVE_LANE]]
; CHECK-NEXT: br i1 [[TMP9]], label %[[VECTOR_EARLY_EXIT_0:.*]], label %[[VECTOR_EARLY_EXIT_1:.*]]
; CHECK: [[VECTOR_EARLY_EXIT_1]]:
; CHECK-NEXT: [[TMP10:%.*]] = extractelement <4 x i32> [[TMP4]], i64 [[FIRST_ACTIVE_LANE]]
-; CHECK-NEXT: br label %[[EXIT1:.*]]
+; CHECK-NEXT: br label %[[EXIT]]
; CHECK: [[VECTOR_EARLY_EXIT_0]]:
; CHECK-NEXT: [[TMP11:%.*]] = trunc i64 [[FIRST_ACTIVE_LANE]] to i32
; CHECK-NEXT: [[TMP12:%.*]] = add i32 [[INDEX]], [[TMP11]]
-; CHECK-NEXT: br label %[[EXIT1]]
+; CHECK-NEXT: br label %[[EXIT]]
; CHECK: [[EXIT]]:
-; CHECK-NEXT: br label %[[LOOP:.*]]
-; CHECK: [[LOOP]]:
-; CHECK-NEXT: [[IV:%.*]] = phi i32 [ 120, %[[EXIT]] ], [ [[IV_NEXT:%.*]], %[[LATCH:.*]] ]
-; CHECK-NEXT: [[GEP_P:%.*]] = getelementptr inbounds i32, ptr [[P]], i32 [[IV]]
-; CHECK-NEXT: [[X:%.*]] = load i32, ptr [[GEP_P]], align 4
-; CHECK-NEXT: [[C1:%.*]] = icmp eq i32 [[X]], 0
-; CHECK-NEXT: br i1 [[C1]], label %[[EXIT1]], label %[[NEXT:.*]]
-; CHECK: [[NEXT]]:
-; CHECK-NEXT: [[GEP_Q:%.*]] = getelementptr inbounds i32, ptr [[Q]], i32 [[IV]]
-; CHECK-NEXT: [[Y:%.*]] = load i32, ptr [[GEP_Q]], align 4
-; CHECK-NEXT: [[C2:%.*]] = icmp eq i32 [[Y]], 0
-; CHECK-NEXT: [[IV_Q:%.*]] = add i32 [[IV]], 1000
-; CHECK-NEXT: br i1 [[C2]], label %[[EXIT1]], label %[[LATCH]]
-; CHECK: [[LATCH]]:
-; CHECK-NEXT: [[IV_NEXT]] = add nuw nsw i32 [[IV]], 1
-; CHECK-NEXT: [[EC:%.*]] = icmp eq i32 [[IV_NEXT]], 123
-; CHECK-NEXT: br i1 [[EC]], label %[[EXIT1]], label %[[LOOP]], !llvm.loop [[LOOP5:![0-9]+]]
-; CHECK: [[EXIT1]]:
-; CHECK-NEXT: [[RES:%.*]] = phi i32 [ [[IV]], %[[LOOP]] ], [ [[IV_Q]], %[[NEXT]] ], [ -1, %[[LATCH]] ], [ [[TMP12]], %[[VECTOR_EARLY_EXIT_0]] ], [ [[TMP10]], %[[VECTOR_EARLY_EXIT_1]] ]
+; CHECK-NEXT: [[RES:%.*]] = phi i32 [ [[TMP10]], %[[VECTOR_EARLY_EXIT_1]] ], [ [[TMP12]], %[[VECTOR_EARLY_EXIT_0]] ], [ -1, %[[MIDDLE_BLOCK]] ]
; CHECK-NEXT: ret i32 [[RES]]
;
entry:
@@ -161,26 +137,28 @@ define i32 @multi_early_exits_different_exits(ptr align 4 dereferenceable(4096)
; CHECK: [[VECTOR_BODY]]:
; CHECK-NEXT: [[INDEX:%.*]] = phi i32 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY_INTERIM:.*]] ]
; CHECK-NEXT: [[VEC_IND:%.*]] = phi <4 x i32> [ <i32 0, i32 1, i32 2, i32 3>, %[[VECTOR_PH]] ], [ [[VEC_IND_NEXT:%.*]], %[[VECTOR_BODY_INTERIM]] ]
+; CHECK-NEXT: [[TMP13:%.*]] = icmp ule <4 x i32> [[VEC_IND]], splat (i32 122)
; CHECK-NEXT: [[TMP0:%.*]] = getelementptr inbounds i32, ptr [[P]], i32 [[INDEX]]
-; CHECK-NEXT: [[WIDE_LOAD:%.*]] = load <4 x i32>, ptr [[TMP0]], align 4
+; CHECK-NEXT: [[WIDE_LOAD:%.*]] = call <4 x i32> @llvm.masked.load.v4i32.p0(ptr align 4 [[TMP0]], <4 x i1> [[TMP13]], <4 x i32> poison)
; CHECK-NEXT: [[TMP1:%.*]] = icmp eq <4 x i32> [[WIDE_LOAD]], zeroinitializer
; CHECK-NEXT: [[TMP2:%.*]] = getelementptr inbounds i32, ptr [[Q]], i32 [[INDEX]]
-; CHECK-NEXT: [[WIDE_LOAD1:%.*]] = load <4 x i32>, ptr [[TMP2]], align 4
+; CHECK-NEXT: [[WIDE_LOAD1:%.*]] = call <4 x i32> @llvm.masked.load.v4i32.p0(ptr align 4 [[TMP2]], <4 x i1> [[TMP13]], <4 x i32> poison)
; CHECK-NEXT: [[TMP3:%.*]] = icmp eq <4 x i32> [[WIDE_LOAD1]], zeroinitializer
; CHECK-NEXT: [[TMP4:%.*]] = add <4 x i32> [[VEC_IND]], splat (i32 1000)
; CHECK-NEXT: [[TMP5:%.*]] = select <4 x i1> [[TMP1]], <4 x i1> splat (i1 true), <4 x i1> [[TMP3]]
; CHECK-NEXT: [[TMP6:%.*]] = freeze <4 x i1> [[TMP5]]
-; CHECK-NEXT: [[TMP7:%.*]] = call i1 @llvm.vector.reduce.or.v4i1(<4 x i1> [[TMP6]])
+; CHECK-NEXT: [[TMP8:%.*]] = select <4 x i1> [[TMP13]], <4 x i1> [[TMP6]], <4 x i1> zeroinitializer
+; CHECK-NEXT: [[TMP7:%.*]] = call i1 @llvm.vector.reduce.or.v4i1(<4 x i1> [[TMP8]])
; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i32 [[INDEX]], 4
-; CHECK-NEXT: [[TMP8:%.*]] = icmp eq i32 [[INDEX_NEXT]], 120
-; CHECK-NEXT: [[VEC_IND_NEXT]] = add nuw nsw <4 x i32> [[VEC_IND]], splat (i32 4)
+; CHECK-NEXT: [[TMP14:%.*]] = icmp eq i32 [[INDEX_NEXT]], 124
+; CHECK-NEXT: [[VEC_IND_NEXT]] = add <4 x i32> [[VEC_IND]], splat (i32 4)
; CHECK-NEXT: br i1 [[TMP7]], label %[[VECTOR_EARLY_EXIT_CHECK:.*]], label %[[VECTOR_BODY_INTERIM]]
; CHECK: [[VECTOR_BODY_INTERIM]]:
-; CHECK-NEXT: br i1 [[TMP8]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP6:![0-9]+]]
+; CHECK-NEXT: br i1 [[TMP14]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP4:![0-9]+]]
; CHECK: [[MIDDLE_BLOCK]]:
; CHECK-NEXT: br label %[[EXIT_LATCH:.*]]
; CHECK: [[VECTOR_EARLY_EXIT_CHECK]]:
-; CHECK-NEXT: [[FIRST_ACTIVE_LANE:%.*]] = call i64 @llvm.experimental.cttz.elts.i64.v4i1(<4 x i1> [[TMP6]], i1 false)
+; CHECK-NEXT: [[FIRST_ACTIVE_LANE:%.*]] = call i64 @llvm.experimental.cttz.elts.i64.v4i1(<4 x i1> [[TMP8]], i1 false)
; CHECK-NEXT: [[TMP9:%.*]] = extractelement <4 x i1> [[TMP1]], i64 [[FIRST_ACTIVE_LANE]]
; CHECK-NEXT: br i1 [[TMP9]], label %[[VECTOR_EARLY_EXIT_0:.*]], label %[[VECTOR_EARLY_EXIT_1:.*]]
; CHECK: [[VECTOR_EARLY_EXIT_1]]:
@@ -189,32 +167,12 @@ define i32 @multi_early_exits_different_exits(ptr align 4 dereferenceable(4096)
; CHECK: [[VECTOR_EARLY_EXIT_0]]:
; CHECK-NEXT: [[TMP11:%.*]] = trunc i64 [[FIRST_ACTIVE_LANE]] to i32
; CHECK-NEXT: [[TMP12:%.*]] = add i32 [[INDEX]], [[TMP11]]
-; CHECK-NEXT: br label %[[EXIT_P:.*]]
-; CHECK: [[EXIT_LATCH]]:
; CHECK-NEXT: br label %[[LOOP:.*]]
; CHECK: [[LOOP]]:
-; CHECK-NEXT: [[IV:%.*]] = phi i32 [ 120, %[[EXIT_LATCH]] ], [ [[IV_NEXT:%.*]], %[[LATCH:.*]] ]
-; CHECK-NEXT: [[GEP_P:%.*]] = getelementptr inbounds i32, ptr [[P]], i32 [[IV]]
-; CHECK-NEXT: [[X:%.*]] = load i32, ptr [[GEP_P]], align 4
-; CHECK-NEXT: [[C1:%.*]] = icmp eq i32 [[X]], 0
-; CHECK-NEXT: br i1 [[C1]], label %[[EXIT_P]], label %[[NEXT:.*]]
-; CHECK: [[NEXT]]:
-; CHECK-NEXT: [[GEP_Q:%.*]] = getelementptr inbounds i32, ptr [[Q]], i32 [[IV]]
-; CHECK-NEXT: [[Y:%.*]] = load i32, ptr [[GEP_Q]], align 4
-; CHECK-NEXT: [[C2:%.*]] = icmp eq i32 [[Y]], 0
-; CHECK-NEXT: [[IV_Q:%.*]] = add i32 [[IV]], 1000
-; CHECK-NEXT: br i1 [[C2]], label %[[EXIT_Q]], label %[[LATCH]]
-; CHECK: [[LATCH]]:
-; CHECK-NEXT: [[IV_NEXT]] = add nuw nsw i32 [[IV]], 1
-; CHECK-NEXT: [[EC:%.*]] = icmp eq i32 [[IV_NEXT]], 123
-; CHECK-NEXT: br i1 [[EC]], label %[[EXIT_LATCH1:.*]], label %[[LOOP]], !llvm.loop [[LOOP7:![0-9]+]]
-; CHECK: [[EXIT_P]]:
-; CHECK-NEXT: [[RES_P:%.*]] = phi i32 [ [[IV]], %[[LOOP]] ], [ [[TMP12]], %[[VECTOR_EARLY_EXIT_0]] ]
-; CHECK-NEXT: ret i32 [[RES_P]]
+; CHECK-NEXT: ret i32 [[TMP12]]
; CHECK: [[EXIT_Q]]:
-; CHECK-NEXT: [[RES_Q:%.*]] = phi i32 [ [[IV_Q]], %[[NEXT]] ], [ [[TMP10]], %[[VECTOR_EARLY_EXIT_1]] ]
-; CHECK-NEXT: ret i32 [[RES_Q]]
-; CHECK: [[EXIT_LATCH1]]:
+; CHECK-NEXT: ret i32 [[TMP10]]
+; CHECK: [[EXIT_LATCH]]:
; CHECK-NEXT: ret i32 -1
;
entry:
@@ -263,40 +221,31 @@ define i32 @predicated_early_exit(i1 %c0, ptr align 4 dereferenceable(4096) %p)
; CHECK-NEXT: br label %[[VECTOR_BODY:.*]]
; CHECK: [[VECTOR_BODY]]:
; CHECK-NEXT: [[INDEX:%.*]] = phi i32 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY_INTERIM:.*]] ]
-; CHECK-NEXT: [[TMP0:%.*]] = getelementptr inbounds i32, ptr [[P]], i32 [[INDEX]]
-; CHECK-NEXT: [[WIDE_LOAD:%.*]] = load <4 x i32>, ptr [[TMP0]], align 4
+; CHECK-NEXT: [[VEC_IND:%.*]] = phi <4 x i8> [ <i8 0, i8 1, i8 2, i8 3>, %[[VECTOR_PH]] ], [ [[VEC_IND_NEXT:%.*]], %[[VECTOR_BODY_INTERIM]] ]
+; CHECK-NEXT: [[TMP5:%.*]] = icmp ule <4 x i8> [[VEC_IND]], splat (i8 122)
+; CHECK-NEXT: [[TMP10:%.*]] = select <4 x i1> [[TMP5]], <4 x i1> [[TMP9]], <4 x i1> zeroinitializer
+; CHECK-NEXT: [[TMP11:%.*]] = getelementptr i32, ptr [[P]], i32 [[INDEX]]
+; CHECK-NEXT: [[WIDE_LOAD:%.*]] = call <4 x i32> @llvm.masked.load.v4i32.p0(ptr align 4 [[TMP11]], <4 x i1> [[TMP10]], <4 x i32> poison)
; CHECK-NEXT: [[TMP1:%.*]] = icmp eq <4 x i32> [[WIDE_LOAD]], zeroinitializer
; CHECK-NEXT: [[TMP2:%.*]] = select <4 x i1> [[TMP9]], <4 x i1> [[TMP1]], <4 x i1> zeroinitializer
; CHECK-NEXT: [[TMP3:%.*]] = freeze <4 x i1> [[TMP2]]
-; CHECK-NEXT: [[TMP4:%.*]] = call i1 @llvm.vector.reduce.or.v4i1(<4 x i1> [[TMP3]])
+; CHECK-NEXT: [[TMP12:%.*]] = select <4 x i1> [[TMP5]], <4 x i1> [[TMP3]], <4 x i1> zeroinitializer
+; CHECK-NEXT: [[TMP4:%.*]] = call i1 @llvm.vector.reduce.or.v4i1(<4 x i1> [[TMP12]])
; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i32 [[INDEX]], 4
-; CHECK-NEXT: [[TMP5:%.*]] = icmp eq i32 [[INDEX_NEXT]], 120
+; CHECK-NEXT: [[TMP13:%.*]] = icmp eq i32 [[INDEX_NEXT]], 124
+; CHECK-NEXT: [[VEC_IND_NEXT]] = add nuw <4 x i8> [[VEC_IND]], splat (i8 4)
; CHECK-NEXT: br i1 [[TMP4]], label %[[VECTOR_EARLY_EXIT:.*]], label %[[VECTOR_BODY_INTERIM]]
; CHECK: [[VECTOR_BODY_INTERIM]]:
-; CHECK-NEXT: br i1 [[TMP5]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP8:![0-9]+]]
+; CHECK-NEXT: br i1 [[TMP13]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP5:![0-9]+]]
; CHECK: [[MIDDLE_BLOCK]]:
; CHECK-NEXT: br label %[[EXIT:.*]]
; CHECK: [[VECTOR_EARLY_EXIT]]:
-; CHECK-NEXT: [[TMP6:%.*]] = call i64 @llvm.experimental.cttz.elts.i64.v4i1(<4 x i1> [[TMP3]], i1 false)
+; CHECK-NEXT: [[TMP6:%.*]] = call i64 @llvm.experimental.cttz.elts.i64.v4i1(<4 x i1> [[TMP12]], i1 false)
; CHECK-NEXT: [[TMP7:%.*]] = trunc i64 [[TMP6]] to i32
; CHECK-NEXT: [[TMP8:%.*]] = add i32 [[INDEX]], [[TMP7]]
-; CHECK-NEXT: br label %[[EXIT1:.*]]
+; CHECK-NEXT: br label %[[EXIT]]
; CHECK: [[EXIT]]:
-; CHECK-NEXT: br label %[[LOOP:.*]]
-; CHECK: [[LOOP]]:
-; CHECK-NEXT: [[IV:%.*]] = phi i32 [ 120, %[[EXIT]] ], [ [[IV_NEXT:%.*]], %[[LATCH:.*]] ]
-; CHECK-NEXT: br i1 [[C0]], label %[[THEN:.*]], label %[[LATCH]]
-; CHECK: [[THEN]]:
-; CHECK-NEXT: [[GEP:%.*]] = getelementptr inbounds i32, ptr [[P]], i32 [[IV]]
-; CHECK-NEXT: [[LD:%.*]] = load i32, ptr [[GEP]], align 4
-; CHECK-NEXT: [[C1:%.*]] = icmp eq i32 [[LD]], 0
-; CHECK-NEXT: br i1 [[C1]], label %[[EXIT1]], label %[[LATCH]]
-; CHECK: [[LATCH]]:
-; CHECK-NEXT: [[IV_NEXT]] = add nuw nsw i32 [[IV]], 1
-; CHECK-NEXT: [[EC:%.*]] = icmp eq i32 [[IV_NEXT]], 123
-; CHECK-NEXT: br i1 [[EC]], label %[[EXIT1]], label %[[LOOP]], !llvm.loop [[LOOP9:![0-9]+]]
-; CHECK: [[EXIT1]]:
-; CHECK-NEXT: [[RES:%.*]] = phi i32 [ [[IV]], %[[THEN]] ], [ -1, %[[LATCH]] ], [ [[TMP8]], %[[VECTOR_EARLY_EXIT]] ]
+; CHECK-NEXT: [[RES:%.*]] = phi i32 [ [[TMP8]], %[[VECTOR_EARLY_EXIT]] ], [ -1, %[[MIDDLE_BLOCK]] ]
; CHECK-NEXT: ret i32 [[RES]]
;
entry:
@@ -341,7 +290,7 @@ define i32 @countable_early_exit(ptr align 4 dereferenceable(4096) %p, i32 %n) {
; CHECK-NEXT: [[INDEX:%.*]] = phi i32 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i32 [[INDEX]], 4
; CHECK-NEXT: [[TMP5:%.*]] = icmp eq i32 [[INDEX_NEXT]], [[N_VEC]]
-; CHECK-NEXT: br i1 [[TMP5]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP10:![0-9]+]]
+; CHECK-NEXT: br i1 [[TMP5]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP6:![0-9]+]]
; CHECK: [[MIDDLE_BLOCK]]:
; CHECK-NEXT: br label %[[SCALAR_PH]]
; CHECK: [[SCALAR_PH]]:
@@ -356,7 +305,7 @@ define i32 @countable_early_exit(ptr align 4 dereferenceable(4096) %p, i32 %n) {
; CHECK-NEXT: [[LD:%.*]] = load i32, ptr [[GEP]], align 4
; CHECK-NEXT: [[IV_NEXT]] = add nuw nsw i32 [[IV]], 1
; CHECK-NEXT: [[EC:%.*]] = icmp eq i32 [[IV_NEXT]], 123
-; CHECK-NEXT: br i1 [[EC]], label %[[EXIT]], label %[[LOOP]], !llvm.loop [[LOOP11:![0-9]+]]
+; CHECK-NEXT: br i1 [[EC]], label %[[EXIT]], label %[[LOOP]], !llvm.loop [[LOOP7:![0-9]+]]
; CHECK: [[EXIT]]:
; CHECK-NEXT: [[RES:%.*]] = phi i32 [ 0, %[[LOOP]] ], [ [[LD]], %[[LATCH]] ]
; CHECK-NEXT: ret i32 [[RES]]
@@ -404,7 +353,7 @@ define void @early_exit_with_store(ptr align 4 dereferenceable(4096) noalias %ds
; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i32 [[INDEX]], 4
; CHECK-NEXT: [[TMP8:%.*]] = icmp eq i32 [[INDEX_NEXT]], 120
; CHECK-NEXT: [[TMP9:%.*]] = or i1 [[TMP7]], [[TMP8]]
-; CHECK-NEXT: br i1 [[TMP9]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP12:![0-9]+]]
+; CHECK-NEXT: br i1 [[TMP9]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP8:![0-9]+]]
; CHECK: [[MIDDLE_BLOCK]]:
; CHECK-NEXT: [[TMP12:%.*]] = call i64 @llvm.experimental.cttz.elts.i64.v4i1(<4 x i1> [[TMP2]], i1 false)
; CHECK-NEXT: [[TMP4:%.*]] = trunc i64 [[TMP12]] to i32
@@ -426,7 +375,7 @@ define void @early_exit_with_store(ptr align 4 dereferenceable(4096) noalias %ds
; CHECK: [[LATCH]]:
; CHECK-NEXT: [[IV_NEXT]] = add nuw nsw i32 [[IV]], 1
; CHECK-NEXT: [[EC:%.*]] = icmp eq i32 [[IV_NEXT]], 123
-; CHECK-NEXT: br i1 [[EC]], label %[[EXIT]], label %[[LOOP]], !llvm.loop [[LOOP13:![0-9]+]]
+; CHECK-NEXT: br i1 [[EC]], label %[[EXIT]], label %[[LOOP]], !llvm.loop [[LOOP9:![0-9]+]]
; CHECK: [[EXIT]]:
; CHECK-NEXT: ret void
;
@@ -476,7 +425,7 @@ define void @early_exit_with_store_after_exit(ptr align 4 dereferenceable(4096)
; CHECK-NEXT: [[TMP7:%.*]] = icmp eq i32 [[INDEX_NEXT]], 120
; CHECK-NEXT: br i1 [[TMP6]], label %[[VECTOR_EARLY_EXIT:.*]], label %[[VECTOR_BODY_INTERIM]]
; CHECK: [[VECTOR_BODY_INTERIM]]:
-; CHECK-NEXT: br i1 [[TMP7]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP14:![0-9]+]]
+; CHECK-NEXT: br i1 [[TMP7]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP10:![0-9]+]]
; CHECK: [[MIDDLE_BLOCK]]:
; CHECK-NEXT: br label %[[SCALAR_PH:.*]]
; CHECK: [[VECTOR_EARLY_EXIT]]:
@@ -496,7 +445,7 @@ define void @early_exit_with_store_after_exit(ptr align 4 dereferenceable(4096)
; CHECK-NEXT: store i32 [[INC]], ptr [[ST_ADDR]], align 4
; CHECK-NEXT: [[IV_NEXT]] = add nuw nsw i32 [[IV]], 1
; CHECK-NEXT: [[EC:%.*]] = icmp eq i32 [[IV_NEXT]], 123
-; CHECK-NEXT: br i1 [[EC]], label %[[EXIT]], label %[[LOOP]], !llvm.loop [[LOOP15:![0-9]+]]
+; CHECK-NEXT: br i1 [[EC]], label %[[EXIT]], label %[[LOOP]], !llvm.loop [[LOOP11:![0-9]+]]
; CHECK: [[EXIT]]:
; CHECK-NEXT: ret void
;
diff --git a/llvm/test/Transforms/LoopVectorize/multiple-early-exits.ll b/llvm/test/Transforms/LoopVectorize/multiple-early-exits.ll
index c3bbdc5adf214e..08f49a2560d8d9 100644
--- a/llvm/test/Transforms/LoopVectorize/multiple-early-exits.ll
+++ b/llvm/test/Transforms/LoopVectorize/multiple-early-exits.ll
@@ -1491,29 +1491,121 @@ exit:
define i64 @three_early_exits_same_block_no_latch_exit(i1 %cmp0) {
; CHECK-LABEL: define i64 @three_early_exits_same_block_no_latch_exit(
; CHECK-SAME: i1 [[CMP0:%.*]]) {
-; CHECK-NEXT: [[ENTRY:.*]]:
+; CHECK-NEXT: [[ENTRY:.*:]]
; CHECK-NEXT: br label %[[LOOP_HEADER:.*]]
; CHECK: [[LOOP_HEADER]]:
-; CHECK-NEXT: [[IV:%.*]] = phi i64 [ 0, %[[ENTRY]] ], [ [[IV_NEXT:%.*]], %[[LOOP_LATCH:.*]] ]
-; CHECK-NEXT: br i1 [[CMP0]], label %[[EARLY_EXIT:.*]], label %[[LOOP_BODY_0:.*]]
+; CHECK-NEXT: [[BROADCAST_SPLATINSERT:%.*]] = insertelement <4 x i1> poison, i1 [[CMP0]], i64 0
+; CHECK-NEXT: [[BROADCAST_SPLAT:%.*]] = shufflevector <4 x i1> [[BROADCAST_SPLATINSERT]], <4 x i1> poison, <4 x i32> zeroinitializer
+; CHECK-NEXT: br label %[[LOOP_BODY_0:.*]]
; CHECK: [[LOOP_BODY_0]]:
+; CHECK-NEXT: [[IV:%.*]] = phi i64 [ 0, %[[LOOP_HEADER]] ], [ [[INDEX_NEXT:%.*]], %[[PRED_LOAD_CONTINUE_INTERIM:.*]] ]
+; CHECK-NEXT: [[VEC_IND:%.*]] = phi <4 x i8> [ <i8 0, i8 1, i8 2, i8 3>, %[[LOOP_HEADER]] ], [ [[VEC_IND_NEXT:%.*]], %[[PRED_LOAD_CONTINUE_INTERIM]] ]
+; CHECK-NEXT: [[TMP0:%.*]] = icmp ule <4 x i8> [[VEC_IND]], zeroinitializer
+; CHECK-NEXT: [[TMP1:%.*]] = add i64 [[IV]], 1
+; CHECK-NEXT: [[TMP2:%.*]] = add i64 [[IV]], 2
+; CHECK-NEXT: [[TMP3:%.*]] = add i64 [[IV]], 3
; CHECK-NEXT: [[GEP:%.*]] = getelementptr [24 x i8], ptr @GlobA, i64 [[IV]]
+; CHECK-NEXT: [[TMP5:%.*]] = getelementptr [24 x i8], ptr @GlobA, i64 [[TMP1]]
+; CHECK-NEXT: [[TMP6:%.*]] = getelementptr [24 x i8], ptr @GlobA, i64 [[TMP2]]
+; CHECK-NEXT: [[TMP7:%.*]] = getelementptr [24 x i8], ptr @GlobA, i64 [[TMP3]]
+; CHECK-NEXT: [[TMP8:%.*]] = extractelement <4 x i1> [[TMP0]], i64 0
+; CHECK-NEXT: br i1 [[TMP8]], label %[[PRED_LOAD_IF:.*]], label %[[PRED_LOAD_CONTINUE:.*]]
+; CHECK: [[PRED_LOAD_IF]]:
; CHECK-NEXT: [[GEP_1:%.*]] = getelementptr i8, ptr [[GEP]], i64 4
; CHECK-NEXT: [[L_0:%.*]] = load i8, ptr [[GEP_1]], align 1
-; CHECK-NEXT: [[CMP1:%.*]] = icmp eq i8 [[L_0]], 0
-; CHECK-NEXT: br i1 [[CMP1]], label %[[EARLY_EXIT]], label %[[LOOP_BODY_1:.*]]
+; CHECK-NEXT: [[TMP11:%.*]] = insertelement <4 x i8> poison, i8 [[L_0]], i64 0
+; CHECK-NEXT: br label %[[PRED_LOAD_CONTINUE]]
+; CHECK: [[PRED_LOAD_CONTINUE]]:
+; CHECK-NEXT: [[TMP12:%.*]] = phi <4 x i8> [ poison, %[[LOOP_BODY_0]] ], [ [[TMP11]], %[[PRED_LOAD_IF]] ]
+; CHECK-NEXT: [[CMP1:%.*]] = extractelement <4 x i1> [[TMP0]], i64 1
+; CHECK-NEXT: br i1 [[CMP1]], label %[[EARLY_EXIT:.*]], label %[[LOOP_BODY_1:.*]]
+; CHECK: [[EARLY_EXIT]]:
+; CHECK-NEXT: [[TMP14:%.*]] = getelementptr i8, ptr [[TMP5]], i64 4
+; CHECK-NEXT: [[TMP15:%.*]] = load i8, ptr [[TMP14]], align 1
+; CHECK-NEXT: [[TMP16:%.*]] = insertelement <4 x i8> [[TMP12]], i8 [[TMP15]], i64 1
+; CHECK-NEXT: br label %[[LOOP_BODY_1]]
; CHECK: [[LOOP_BODY_1]]:
-; CHECK-NEXT: [[L_1:%.*]] = load i8, ptr [[GEP]], align 1
-; CHECK-NEXT: [[CMP2:%.*]] = icmp eq i8 [[L_1]], 0
-; CHECK-NEXT: br i1 [[CMP2]], label %[[EARLY_EXIT]], label %[[LOOP_LATCH]]
+; CHECK-NEXT: [[TMP17:%.*]] = phi <4 x i8> [ [[TMP12]], %[[PRED_LOAD_CONTINUE]] ], [ [[TMP16]], %[[EARLY_EXIT]] ]
+; CHECK-NEXT: [[TMP18:%.*]] = extractelement <4 x i1> [[TMP0]], i64 2
+; CHECK-NEXT: br i1 [[TMP18]], label %[[LOOP_LATCH:.*]], label %[[PRED_LOAD_CONTINUE5:.*]]
; CHECK: [[LOOP_LATCH]]:
-; CHECK-NEXT: [[IV_NEXT]] = add nuw nsw i64 [[IV]], 1
-; CHECK-NEXT: [[EC:%.*]] = icmp eq i64 [[IV]], 0
-; CHECK-NEXT: br i1 [[EC]], label %[[LATCH_EXIT:.*]], label %[[LOOP_HEADER]]
+; CHECK-NEXT: [[TMP19:%.*]] = getelementptr i8, ptr [[TMP6]], i64 4
+; CHECK-NEXT: [[TMP20:%.*]] = load i8, ptr [[TMP19]], align 1
+; CHECK-NEXT: [[TMP21:%.*]] = insertelement <4 x i8> [[TMP17]], i8 [[TMP20]], i64 2
+; CHECK-NEXT: br label %[[PRED_LOAD_CONTINUE5]]
+; CHECK: [[PRED_LOAD_CONTINUE5]]:
+; CHECK-NEXT: [[TMP22:%.*]] = phi <4 x i8> [ [[TMP17]], %[[LOOP_BODY_1]] ], [ [[TMP21]], %[[LOOP_LATCH]] ]
+; CHECK-NEXT: [[TMP23:%.*]] = extractelement <4 x i1> [[TMP0]], i64 3
+; CHECK-NEXT: br i1 [[TMP23]], label %[[LATCH_EXIT:.*]], label %[[PRED_LOAD_CONTINUE7:.*]]
; CHECK: [[LATCH_EXIT]]:
+; CHECK-NEXT: [[TMP24:%.*]] = getelementptr i8, ptr [[TMP7]], i64 4
+; CHECK-NEXT: [[TMP25:%.*]] = load i8, ptr [[TMP24]], align 1
+; CHECK-NEXT: [[TMP26:%.*]] = insertelement <4 x i8> [[TMP22]], i8 [[TMP25]], i64 3
+; CHECK-NEXT: br label %[[PRED_LOAD_CONTINUE7]]
+; CHECK: [[PRED_LOAD_CONTINUE7]]:
+; CHECK-NEXT: [[TMP27:%.*]] = phi <4 x i8> [ [[TMP22]], %[[PRED_LOAD_CONTINUE5]] ], [ [[TMP26]], %[[LATCH_EXIT]] ]
+; CHECK-NEXT: [[TMP28:%.*]] = icmp eq <4 x i8> [[TMP27]], zeroinitializer
+; CHECK-NEXT: br i1 [[TMP8]], label %[[PRED_LOAD_IF8:.*]], label %[[PRED_LOAD_CONTINUE9:.*]]
+; CHECK: [[PRED_LOAD_IF8]]:
+; CHECK-NEXT: [[TMP29:%.*]] = load i8, ptr [[GEP]], align 1
+; CHECK-NEXT: [[TMP30:%.*]] = insertelement <4 x i8> poison, i8 [[TMP29]], i64 0
+; CHECK-NEXT: br label %[[PRED_LOAD_CONTINUE9]]
+; CHECK: [[PRED_LOAD_CONTINUE9]]:
+; CHECK-NEXT: [[TMP31:%.*]] = phi <4 x i8> [ poison, %[[PRED_LOAD_CONTINUE7]] ], [ [[TMP30]], %[[PRED_LOAD_IF8]] ]
+; CHECK-NEXT: br i1 [[CMP1]], label %[[PRED_LOAD_IF10:.*]], label %[[PRED_LOAD_CONTINUE11:.*]]
+; CHECK: [[PRED_LOAD_IF10]]:
+; CHECK-NEXT: [[TMP32:%.*]] = load i8, ptr [[TMP5]], align 1
+; CHECK-NEXT: [[TMP33:%.*]] = insertelement <4 x i8> [[TMP31]], i8 [[TMP32]], i64 1
+; CHECK-NEXT: br label %[[PRED_LOAD_CONTINUE11]]
+; CHECK: [[PRED_LOAD_CONTINUE11]]:
+; CHECK-NEXT: [[TMP34:%.*]] = phi <4 x i8> [ [[TMP31]], %[[PRED_LOAD_CONTINUE9]] ], [ [[TMP33]], %[[PRED_LOAD_IF10]] ]
+; CHECK-NEXT: br i1 [[TMP18]], label %[[PRED_LOAD_IF12:.*]], label %[[PRED_LOAD_CONTINUE13:.*]]
+; CHECK: [[PRED_LOAD_IF12]]:
+; CHECK-NEXT: [[TMP35:%.*]] = load i8, ptr [[TMP6]], align 1
+; CHECK-NEXT: [[TMP36:%.*]] = insertelement <4 x i8> [[TMP34]], i8 [[TMP35]], i64 2
+; CHECK-NEXT: br label %[[PRED_LOAD_CONTINUE13]]
+; CHECK: [[PRED_LOAD_CONTINUE13]]:
+; CHECK-NEXT: [[TMP37:%.*]] = phi <4 x i8> [ [[TMP34]], %[[PRED_LOAD_CONTINUE11]] ], [ [[TMP36]], %[[PRED_LOAD_IF12]] ]
+; CHECK-NEXT: br i1 [[TMP23]], label %[[PRED_LOAD_IF14:.*]], label %[[PRED_LOAD_CONTINUE15:.*]]
+; CHECK: [[PRED_LOAD_IF14]]:
+; CHECK-NEXT: [[TMP38:%.*]] = load i8, ptr [[TMP7]], align 1
+; CHECK-NEXT: [[TMP39:%.*]] = insertelement <4 x i8> [[TMP37]], i8 [[TMP38]], i64 3
+; CHECK-NEXT: br label %[[PRED_LOAD_CONTINUE15]]
+; CHECK: [[PRED_LOAD_CONTINUE15]]:
+; CHECK-NEXT: [[TMP40:%.*]] = phi <4 x i8> [ [[TMP37]], %[[PRED_LOAD_CONTINUE13]] ], [ [[TMP39]], %[[PRED_LOAD_IF14]] ]
+; CHECK-NEXT: [[TMP41:%.*]] = icmp eq <4 x i8> [[TMP40]], zeroinitializer
+; CHECK-NEXT: [[TMP42:%.*]] = select <4 x i1> [[BROADCAST_SPLAT]], <4 x i1> splat (i1 true), <4 x i1> [[TMP28]]
+; CHECK-NEXT: [[TMP43:%.*]] = select <4 x i1> [[TMP42]], <4 x i1> splat (i1 true), <4 x i1> [[TMP41]]
+; CHECK-NEXT: [[TMP44:%.*]] = freeze <4 x i1> [[TMP43]]
+; CHECK-NEXT: [[TMP45:%.*]] = select <4 x i1> [[TMP0]], <4 x i1> [[TMP44]], <4 x i1> zeroinitializer
+; CHECK-NEXT: [[TMP46:%.*]] = call i1 @llvm.vector.reduce.or.v4i1(<4 x i1> [[TMP45]])
+; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[IV]], 4
+; CHECK-NEXT: [[TMP47:%.*]] = icmp eq i64 [[INDEX_NEXT]], 4
+; CHECK-NEXT: [[VEC_IND_NEXT]] = add nuw <4 x i8> [[VEC_IND]], splat (i8 4)
+; CHECK-NEXT: br i1 [[TMP46]], label %[[VECTOR_EARLY_EXIT_CHECK:.*]], label %[[PRED_LOAD_CONTINUE_INTERIM]]
+; CHECK: [[PRED_LOAD_CONTINUE_INTERIM]]:
+; CHECK-NEXT: br i1 [[TMP47]], label %[[MIDDLE_BLOCK:.*]], label %[[LOOP_BODY_0]], !llvm.loop [[LOOP19:![0-9]+]]
+; CHECK: [[MIDDLE_BLOCK]]:
+; CHECK-NEXT: br label %[[LATCH_EXIT1:.*]]
+; CHECK: [[VECTOR_EARLY_EXIT_CHECK]]:
+; CHECK-NEXT: [[FIRST_ACTIVE_LANE:%.*]] = call i64 @llvm.experimental.cttz.elts.i64.v4i1(<4 x i1> [[TMP45]], i1 false)
+; CHECK-NEXT: br i1 [[CMP0]], label %[[VECTOR_EARLY_EXIT_0:.*]], label %[[VECTOR_EARLY_EXIT_CHECK_0:.*]]
+; CHECK: [[VECTOR_EARLY_EXIT_CHECK_0]]:
+; CHECK-NEXT: [[TMP48:%.*]] = extractelement <4 x i1> [[TMP28]], i64 [[FIRST_ACTIVE_LANE]]
+; CHECK-NEXT: br i1 [[TMP48]], label %[[VECTOR_EARLY_EXIT_1:.*]], label %[[VECTOR_EARLY_EXIT_2:.*]]
+; CHECK: [[VECTOR_EARLY_EXIT_2]]:
+; CHECK-NEXT: [[TMP49:%.*]] = add i64 [[IV]], [[FIRST_ACTIVE_LANE]]
+; CHECK-NEXT: br label %[[EARLY_EXIT1:.*]]
+; CHECK: [[VECTOR_EARLY_EXIT_1]]:
+; CHECK-NEXT: [[TMP50:%.*]] = add i64 [[IV]], [[FIRST_ACTIVE_LANE]]
+; CHECK-NEXT: br label %[[EARLY_EXIT1]]
+; CHECK: [[VECTOR_EARLY_EXIT_0]]:
+; CHECK-NEXT: [[TMP51:%.*]] = add i64 [[IV]], [[FIRST_ACTIVE_LANE]]
+; CHECK-NEXT: br label %[[EARLY_EXIT1]]
+; CHECK: [[LATCH_EXIT1]]:
; CHECK-NEXT: ret i64 0
-; CHECK: [[EARLY_EXIT]]:
-; CHECK-NEXT: [[IV_LCSSA:%.*]] = phi i64 [ [[IV]], %[[LOOP_BODY_1]] ], [ [[IV]], %[[LOOP_BODY_0]] ], [ [[IV]], %[[LOOP_HEADER]] ]
+; CHECK: [[EARLY_EXIT1]]:
+; CHECK-NEXT: [[IV_LCSSA:%.*]] = phi i64 [ [[TMP51]], %[[VECTOR_EARLY_EXIT_0]] ], [ [[TMP50]], %[[VECTOR_EARLY_EXIT_1]] ], [ [[TMP49]], %[[VECTOR_EARLY_EXIT_2]] ]
; CHECK-NEXT: ret i64 [[IV_LCSSA]]
;
entry:
More information about the llvm-branch-commits
mailing list