[llvm-branch-commits] [llvm] [VPlan] Use llvm.mask.beforefirst for computing early exit mask (PR #212717)
Luke Lau via llvm-branch-commits
llvm-branch-commits at lists.llvm.org
Fri Oct 23 05:56:33 PDT 2026
https://github.com/lukel97 updated https://github.com/llvm/llvm-project/pull/212717
>From cbf15ef9b6a5e1f47eaf58af76085c637be7261e Mon Sep 17 00:00:00 2001
From: Luke Lau <luke at igalia.com>
Date: Mon, 27 Jul 2026 22:58:50 +0800
Subject: [PATCH] [VPlan] Use llvm.mask.beforefirst for computing early exit
mask
This makes the cost more accurate and improves codegen for RISC-V. The codegen for an early exit loop with side effects then goes from:
.LBB0_3: # %vector.body
# =>This Inner Loop Header: Depth=1
vl1r.v v16, (a3)
vsetvli zero, zero, e8, m1, ta, ma
vmseq.vi v16, v16, 0
vfirst.m a7, v16
srli t0, a7, 63
czero.nez a7, a7, t0
czero.eqz t0, a1, t0
or a7, t0, a7
vsetvli zero, zero, e64, m8, ta, ma
vmsltu.vx v0, v8, a7
vle8.v v17, (a5), v0.t
vcpop.m t1, v16
vsetvli zero, zero, e8, m1, ta, ma
vadd.vi v16, v17, 1
vse8.v v16, (a5), v0.t
sub a4, a4, a2
bnez t1, .LBB0_5
To
.LBB0_3: # %vector.body
# =>This Inner Loop Header: Depth=1
vl1r.v v8, (a4)
vmseq.vi v8, v8, 0
vmsbf.m v0, v8
vle8.v v9, (a5), v0.t
vcpop.m t0, v8
vadd.vi v9, v9, 1
vse8.v v9, (a5), v0.t
sub a2, a2, a3
bnez t0, .LBB0_5
Some AArch64 loops now use a smaller VF because the early exit mask is now more accurately costed as cheaper.
This adds a dedicated VPInstruction instead of VPWidenIntrinsic so that it can be unrolled later on.
---
llvm/lib/Transforms/Vectorize/VPlan.h | 3 +
.../lib/Transforms/Vectorize/VPlanRecipes.cpp | 21 ++++
.../Transforms/Vectorize/VPlanTransforms.cpp | 14 ++-
.../AArch64/early_exit_with_stores.ll | 100 +++++++++---------
.../RISCV/early_exit_with_stores.ll | 20 ++--
.../VPlan/early_exit_with_stores_vplan.ll | 80 +++++++-------
.../early_exit_combined_exits.ll | 16 +--
.../early_exit_combined_exits_epilogue.ll | 4 +-
.../LoopVectorize/early_exit_with_stores.ll | 36 +++----
.../interleave_uncountable_exits.ll | 4 +-
...rized_conditional_ops_uncountable_exits.ll | 4 +-
.../tail_fold_uncountable_exits.ll | 4 +-
...ountable-and-uncountable-exits-combined.ll | 4 +-
13 files changed, 166 insertions(+), 144 deletions(-)
diff --git a/llvm/lib/Transforms/Vectorize/VPlan.h b/llvm/lib/Transforms/Vectorize/VPlan.h
index 1757b913192be5..95edb9ae975d51 100644
--- a/llvm/lib/Transforms/Vectorize/VPlan.h
+++ b/llvm/lib/Transforms/Vectorize/VPlan.h
@@ -1383,6 +1383,9 @@ class LLVM_ABI_FOR_TEST VPInstruction : public VPRecipeWithIRFlags,
LogicalAnd, // Non-poison propagating logical And.
LogicalOr, // Non-poison propagating logical Or.
NumActiveLanes, // Counts the number of active lanes in a mask.
+ // Returns a mask of all the lanes before the first active lane, i.e.
+ // @llvm.mask.beforefirst.
+ MaskBeforeFirst,
// Add an offset in bytes (second operand) to a base pointer (first
// operand). Only generates scalar values (either for the first lane only or
// for all lanes, depending on its uses).
diff --git a/llvm/lib/Transforms/Vectorize/VPlanRecipes.cpp b/llvm/lib/Transforms/Vectorize/VPlanRecipes.cpp
index 295cb47def8b70..fca850f17eb217 100644
--- a/llvm/lib/Transforms/Vectorize/VPlanRecipes.cpp
+++ b/llvm/lib/Transforms/Vectorize/VPlanRecipes.cpp
@@ -519,6 +519,7 @@ Type *llvm::computeScalarTypeForInstruction(unsigned Opcode,
AssertOperandType(1, Op0Ty);
return IntegerType::get(Ctx, 1);
case VPInstruction::MaskedCond:
+ case VPInstruction::MaskBeforeFirst:
assert(Op0Ty->isIntegerTy(1) && "expected bool operand");
return IntegerType::get(Ctx, 1);
case VPInstruction::LogicalAnd:
@@ -658,6 +659,7 @@ unsigned VPInstruction::getNumOperandsForOpcode() const {
case VPInstruction::ExtractLastLane:
case VPInstruction::ExtractLastPart:
case VPInstruction::ExtractPenultimateElement:
+ case VPInstruction::MaskBeforeFirst:
case VPInstruction::MaskedCond:
case VPInstruction::Not:
case VPInstruction::Reverse:
@@ -873,6 +875,13 @@ Value *VPInstruction::generate(VPTransformState &State,
Builder.CreateUnaryIntrinsic(Intrinsic::vector_reduce_add, ZExt);
return NumActive;
}
+ case VPInstruction::MaskBeforeFirst: {
+ Value *Cond = State.get(getOperand(0));
+ if (State.VF.isScalar())
+ return Builder.CreateNot(Cond, Name);
+ return Builder.CreateUnaryIntrinsic(Intrinsic::mask_beforefirst, Cond, {},
+ Name);
+ }
case VPInstruction::FirstOrderRecurrenceSplice: {
// Generate code to combine the previous and current values in vector v3.
//
@@ -1540,6 +1549,14 @@ InstructionCost VPInstruction::computeCost(ElementCount VF,
{ArgTy, ArgTy});
return Ctx.TTI.getIntrinsicInstrCost(Attrs, Ctx.CostKind);
}
+ case VPInstruction::MaskBeforeFirst: {
+ if (VF.isScalar())
+ return Ctx.TTI.getArithmeticInstrCost(
+ Instruction::Xor, Type::getInt1Ty(Ctx.LLVMCtx), Ctx.CostKind);
+ Type *MaskTy = toVectorTy(Type::getInt1Ty(Ctx.LLVMCtx), VF);
+ IntrinsicCostAttributes Attrs(Intrinsic::mask_beforefirst, MaskTy, MaskTy);
+ return Ctx.TTI.getIntrinsicInstrCost(Attrs, Ctx.CostKind);
+ }
case VPInstruction::ExplicitVectorLength: {
Type *Arg0Ty = getOperand(0)->getScalarType();
Type *I32Ty = Type::getInt32Ty(Ctx.LLVMCtx);
@@ -1764,6 +1781,7 @@ bool VPInstruction::opcodeMayReadOrWriteFromMemory() const {
case VPInstruction::FirstOrderRecurrenceSplice:
case VPInstruction::LogicalAnd:
case VPInstruction::LogicalOr:
+ case VPInstruction::MaskBeforeFirst:
case VPInstruction::MaskedCond:
case VPInstruction::Not:
case VPInstruction::PtrAdd:
@@ -1971,6 +1989,9 @@ void VPInstruction::printRecipe(raw_ostream &O, const Twine &Indent,
case VPInstruction::NumActiveLanes:
O << "num-active-lanes";
break;
+ case VPInstruction::MaskBeforeFirst:
+ O << "mask-before-first";
+ break;
case VPInstruction::WideIVStep:
O << "wide-iv-step";
break;
diff --git a/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp b/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp
index 8c9907adf64029..3dfdddbb91d578 100644
--- a/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp
+++ b/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp
@@ -3466,14 +3466,8 @@ static bool handleUncountableExitsWithSideEffects(
// Create a mask to represent all lanes that fully execute in the vector loop,
// stopping short of any early exit.
VPBuilder MaskBuilder(HeaderVPBB, InsertIt);
- VPValue *FirstActive = MaskBuilder.createFirstActiveLane(Cond);
- Type *IVScalarTy = IV->getScalarType();
- VPValue *Zero = Plan.getZero(IVScalarTy);
- FirstActive =
- MaskBuilder.createScalarZExtOrTrunc(FirstActive, IVScalarTy, DebugLoc());
- VPValue *Mask = MaskBuilder.createNaryOp(VPInstruction::ActiveLaneMask,
- {Zero, FirstActive}, DebugLoc(),
- "uncountable.exit.mask");
+ VPValue *Mask = MaskBuilder.createNaryOp(VPInstruction::MaskBeforeFirst, Cond,
+ DebugLoc(), "uncountable.exit.mask");
// Convert all other memory operations to use the mask.
for (VPBasicBlock *VPBB : vp_rpo_plain_cfg_loop_body(HeaderVPBB))
@@ -3497,6 +3491,10 @@ static bool handleUncountableExitsWithSideEffects(
assert(match(MiddleVPBB->getTerminator(), m_BranchOnCond()) &&
"Expected BranchOnCond terminator for MiddleVPBB");
VPBuilder MiddleBuilder(MiddleVPBB->getTerminator());
+ Type *IVScalarTy = IV->getScalarType();
+ VPValue *Zero = Plan.getZero(IVScalarTy);
+ VPValue *FirstActive = MiddleBuilder.createScalarZExtOrTrunc(
+ MiddleBuilder.createFirstActiveLane(Cond), IVScalarTy, DebugLoc());
VPValue *ScalarIV = MiddleBuilder.createNaryOp(VPInstruction::ExtractLane,
{Zero, IV}, DebugLoc());
VPValue *ExitIV = MiddleBuilder.createAdd(ScalarIV, FirstActive);
diff --git a/llvm/test/Transforms/LoopVectorize/AArch64/early_exit_with_stores.ll b/llvm/test/Transforms/LoopVectorize/AArch64/early_exit_with_stores.ll
index 09fb90912a608f..9eb18183098010 100644
--- a/llvm/test/Transforms/LoopVectorize/AArch64/early_exit_with_stores.ll
+++ b/llvm/test/Transforms/LoopVectorize/AArch64/early_exit_with_stores.ll
@@ -54,7 +54,7 @@ define void @loop_contains_store_condition_load_has_single_user(ptr dereferencea
; CHECK-SAME: ptr noalias dereferenceable(40) [[ARRAY:%.*]], ptr readonly align 2 dereferenceable(40) [[PRED:%.*]]) #[[ATTR0]] {
; CHECK-NEXT: [[SCALAR_PH1:.*]]:
; CHECK-NEXT: [[TMP0:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-NEXT: [[TMP1:%.*]] = shl nuw i64 [[TMP0]], 3
+; CHECK-NEXT: [[TMP1:%.*]] = shl nuw i64 [[TMP0]], 2
; CHECK-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 20, [[TMP1]]
; CHECK-NEXT: br i1 [[MIN_ITERS_CHECK]], label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]]
; CHECK: [[VECTOR_PH]]:
@@ -64,21 +64,21 @@ define void @loop_contains_store_condition_load_has_single_user(ptr dereferencea
; CHECK: [[FOR_BODY1]]:
; CHECK-NEXT: [[IV1:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[FOR_BODY1]] ]
; CHECK-NEXT: [[TMP5:%.*]] = getelementptr inbounds nuw i16, ptr [[PRED]], i64 [[IV1]]
-; CHECK-NEXT: [[WIDE_LOAD:%.*]] = load <vscale x 8 x i16>, ptr [[TMP5]], align 2
-; CHECK-NEXT: [[TMP6:%.*]] = icmp sgt <vscale x 8 x i16> [[WIDE_LOAD]], splat (i16 500)
-; CHECK-NEXT: [[TMP9:%.*]] = freeze <vscale x 8 x i1> [[TMP6]]
-; CHECK-NEXT: [[TMP7:%.*]] = call i64 @llvm.experimental.cttz.elts.i64.nxv8i1(<vscale x 8 x i1> [[TMP9]], i1 false)
-; CHECK-NEXT: [[UNCOUNTABLE_EXIT_MASK:%.*]] = call <vscale x 8 x i1> @llvm.get.active.lane.mask.nxv8i1.i64(i64 0, i64 [[TMP7]])
+; CHECK-NEXT: [[WIDE_LOAD:%.*]] = load <vscale x 4 x i16>, ptr [[TMP5]], align 2
+; CHECK-NEXT: [[TMP3:%.*]] = icmp sgt <vscale x 4 x i16> [[WIDE_LOAD]], splat (i16 500)
+; CHECK-NEXT: [[TMP4:%.*]] = freeze <vscale x 4 x i1> [[TMP3]]
+; CHECK-NEXT: [[TMP6:%.*]] = call <vscale x 4 x i1> @llvm.mask.beforefirst.nxv4i1(<vscale x 4 x i1> [[TMP4]])
; CHECK-NEXT: [[ST_ADDR1:%.*]] = getelementptr i16, ptr [[ARRAY]], i64 [[IV1]]
-; CHECK-NEXT: [[WIDE_MASKED_LOAD:%.*]] = call <vscale x 8 x i16> @llvm.masked.load.nxv8i16.p0(ptr align 2 [[ST_ADDR1]], <vscale x 8 x i1> [[UNCOUNTABLE_EXIT_MASK]], <vscale x 8 x i16> poison)
-; CHECK-NEXT: [[TMP8:%.*]] = add nsw <vscale x 8 x i16> [[WIDE_MASKED_LOAD]], splat (i16 1)
-; CHECK-NEXT: call void @llvm.masked.store.nxv8i16.p0(<vscale x 8 x i16> [[TMP8]], ptr align 2 [[ST_ADDR1]], <vscale x 8 x i1> [[UNCOUNTABLE_EXIT_MASK]])
-; CHECK-NEXT: [[TMP10:%.*]] = call i1 @llvm.vector.reduce.or.nxv8i1(<vscale x 8 x i1> [[TMP9]])
+; CHECK-NEXT: [[WIDE_MASKED_LOAD:%.*]] = call <vscale x 4 x i16> @llvm.masked.load.nxv4i16.p0(ptr align 2 [[ST_ADDR1]], <vscale x 4 x i1> [[TMP6]], <vscale x 4 x i16> poison)
+; CHECK-NEXT: [[TMP8:%.*]] = add nsw <vscale x 4 x i16> [[WIDE_MASKED_LOAD]], splat (i16 1)
+; CHECK-NEXT: call void @llvm.masked.store.nxv4i16.p0(<vscale x 4 x i16> [[TMP8]], ptr align 2 [[ST_ADDR1]], <vscale x 4 x i1> [[TMP6]])
+; CHECK-NEXT: [[TMP10:%.*]] = call i1 @llvm.vector.reduce.or.nxv4i1(<vscale x 4 x i1> [[TMP4]])
; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[IV1]], [[TMP1]]
; CHECK-NEXT: [[TMP11:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
; CHECK-NEXT: [[TMP12:%.*]] = or i1 [[TMP10]], [[TMP11]]
; CHECK-NEXT: br i1 [[TMP12]], label %[[MIDDLE_BLOCK:.*]], label %[[FOR_BODY1]], !llvm.loop [[LOOP0:![0-9]+]]
; CHECK: [[MIDDLE_BLOCK]]:
+; CHECK-NEXT: [[TMP7:%.*]] = call i64 @llvm.experimental.cttz.elts.i64.nxv4i1(<vscale x 4 x i1> [[TMP4]], i1 false)
; CHECK-NEXT: [[TMP13:%.*]] = add i64 [[IV1]], [[TMP7]]
; CHECK-NEXT: [[TMP14:%.*]] = icmp eq i64 [[TMP13]], 20
; CHECK-NEXT: br i1 [[TMP14]], label %[[EXIT:.*]], label %[[SCALAR_PH]]
@@ -130,7 +130,7 @@ define void @loop_contains_store_fcmp_condition(ptr dereferenceable(40) noalias
; CHECK-SAME: ptr noalias dereferenceable(40) [[ARRAY:%.*]], ptr readonly align 2 dereferenceable(40) [[PRED:%.*]]) #[[ATTR0]] {
; CHECK-NEXT: [[ENTRY:.*]]:
; CHECK-NEXT: [[TMP0:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-NEXT: [[TMP1:%.*]] = shl nuw i64 [[TMP0]], 3
+; CHECK-NEXT: [[TMP1:%.*]] = shl nuw i64 [[TMP0]], 2
; CHECK-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 20, [[TMP1]]
; CHECK-NEXT: br i1 [[MIN_ITERS_CHECK]], label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]]
; CHECK: [[VECTOR_PH]]:
@@ -140,21 +140,21 @@ define void @loop_contains_store_fcmp_condition(ptr dereferenceable(40) noalias
; CHECK: [[FOR_BODY]]:
; CHECK-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[FOR_BODY]] ]
; CHECK-NEXT: [[TMP2:%.*]] = getelementptr inbounds nuw half, ptr [[PRED]], i64 [[INDEX]]
-; CHECK-NEXT: [[WIDE_LOAD:%.*]] = load <vscale x 8 x half>, ptr [[TMP2]], align 2
-; CHECK-NEXT: [[TMP3:%.*]] = fcmp ugt <vscale x 8 x half> [[WIDE_LOAD]], splat (half 5.000000e+02)
-; CHECK-NEXT: [[TMP4:%.*]] = freeze <vscale x 8 x i1> [[TMP3]]
-; CHECK-NEXT: [[TMP5:%.*]] = call i64 @llvm.experimental.cttz.elts.i64.nxv8i1(<vscale x 8 x i1> [[TMP4]], i1 false)
-; CHECK-NEXT: [[UNCOUNTABLE_EXIT_MASK:%.*]] = call <vscale x 8 x i1> @llvm.get.active.lane.mask.nxv8i1.i64(i64 0, i64 [[TMP5]])
+; CHECK-NEXT: [[WIDE_LOAD:%.*]] = load <vscale x 4 x half>, ptr [[TMP2]], align 2
+; CHECK-NEXT: [[TMP3:%.*]] = fcmp ugt <vscale x 4 x half> [[WIDE_LOAD]], splat (half 5.000000e+02)
+; CHECK-NEXT: [[TMP4:%.*]] = freeze <vscale x 4 x i1> [[TMP3]]
+; CHECK-NEXT: [[TMP13:%.*]] = call <vscale x 4 x i1> @llvm.mask.beforefirst.nxv4i1(<vscale x 4 x i1> [[TMP4]])
; CHECK-NEXT: [[TMP6:%.*]] = getelementptr i16, ptr [[ARRAY]], i64 [[INDEX]]
-; CHECK-NEXT: [[WIDE_MASKED_LOAD:%.*]] = call <vscale x 8 x i16> @llvm.masked.load.nxv8i16.p0(ptr align 2 [[TMP6]], <vscale x 8 x i1> [[UNCOUNTABLE_EXIT_MASK]], <vscale x 8 x i16> poison)
-; CHECK-NEXT: [[TMP7:%.*]] = add nsw <vscale x 8 x i16> [[WIDE_MASKED_LOAD]], splat (i16 1)
-; CHECK-NEXT: call void @llvm.masked.store.nxv8i16.p0(<vscale x 8 x i16> [[TMP7]], ptr align 2 [[TMP6]], <vscale x 8 x i1> [[UNCOUNTABLE_EXIT_MASK]])
-; CHECK-NEXT: [[TMP8:%.*]] = call i1 @llvm.vector.reduce.or.nxv8i1(<vscale x 8 x i1> [[TMP4]])
+; CHECK-NEXT: [[WIDE_MASKED_LOAD:%.*]] = call <vscale x 4 x i16> @llvm.masked.load.nxv4i16.p0(ptr align 2 [[TMP6]], <vscale x 4 x i1> [[TMP13]], <vscale x 4 x i16> poison)
+; CHECK-NEXT: [[TMP7:%.*]] = add nsw <vscale x 4 x i16> [[WIDE_MASKED_LOAD]], splat (i16 1)
+; CHECK-NEXT: call void @llvm.masked.store.nxv4i16.p0(<vscale x 4 x i16> [[TMP7]], ptr align 2 [[TMP6]], <vscale x 4 x i1> [[TMP13]])
+; CHECK-NEXT: [[TMP8:%.*]] = call i1 @llvm.vector.reduce.or.nxv4i1(<vscale x 4 x i1> [[TMP4]])
; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], [[TMP1]]
; CHECK-NEXT: [[TMP9:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
; CHECK-NEXT: [[TMP10:%.*]] = or i1 [[TMP8]], [[TMP9]]
; CHECK-NEXT: br i1 [[TMP10]], label %[[MIDDLE_BLOCK:.*]], label %[[FOR_BODY]], !llvm.loop [[LOOP4:![0-9]+]]
; CHECK: [[MIDDLE_BLOCK]]:
+; CHECK-NEXT: [[TMP5:%.*]] = call i64 @llvm.experimental.cttz.elts.i64.nxv4i1(<vscale x 4 x i1> [[TMP4]], i1 false)
; CHECK-NEXT: [[TMP11:%.*]] = add i64 [[INDEX]], [[TMP5]]
; CHECK-NEXT: [[TMP12:%.*]] = icmp eq i64 [[TMP11]], 20
; CHECK-NEXT: br i1 [[TMP12]], label %[[EXIT:.*]], label %[[SCALAR_PH]]
@@ -363,7 +363,7 @@ define void @loop_contains_store_to_pointer_with_no_deref_info(ptr align 2 deref
; CHECK-SAME: ptr readonly align 2 dereferenceable(40) [[LOAD_ARRAY:%.*]], ptr noalias align 2 [[ARRAY:%.*]], ptr readonly align 2 dereferenceable(40) [[PRED:%.*]]) #[[ATTR0]] {
; CHECK-NEXT: [[SCALAR_PH:.*]]:
; CHECK-NEXT: [[TMP0:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-NEXT: [[TMP1:%.*]] = shl nuw i64 [[TMP0]], 3
+; CHECK-NEXT: [[TMP1:%.*]] = shl nuw i64 [[TMP0]], 2
; CHECK-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 20, [[TMP1]]
; CHECK-NEXT: br i1 [[MIN_ITERS_CHECK]], label %[[SCALAR_PH1:.*]], label %[[VECTOR_PH:.*]]
; CHECK: [[VECTOR_PH]]:
@@ -373,22 +373,22 @@ define void @loop_contains_store_to_pointer_with_no_deref_info(ptr align 2 deref
; CHECK: [[FOR_BODY]]:
; CHECK-NEXT: [[IV:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[FOR_BODY]] ]
; CHECK-NEXT: [[TMP5:%.*]] = getelementptr inbounds nuw i16, ptr [[PRED]], i64 [[IV]]
-; CHECK-NEXT: [[WIDE_LOAD:%.*]] = load <vscale x 8 x i16>, ptr [[TMP5]], align 2
-; CHECK-NEXT: [[TMP6:%.*]] = icmp sgt <vscale x 8 x i16> [[WIDE_LOAD]], splat (i16 500)
-; CHECK-NEXT: [[TMP10:%.*]] = freeze <vscale x 8 x i1> [[TMP6]]
-; CHECK-NEXT: [[TMP7:%.*]] = call i64 @llvm.experimental.cttz.elts.i64.nxv8i1(<vscale x 8 x i1> [[TMP10]], i1 false)
-; CHECK-NEXT: [[UNCOUNTABLE_EXIT_MASK:%.*]] = call <vscale x 8 x i1> @llvm.get.active.lane.mask.nxv8i1.i64(i64 0, i64 [[TMP7]])
+; CHECK-NEXT: [[WIDE_LOAD:%.*]] = load <vscale x 4 x i16>, ptr [[TMP5]], align 2
+; CHECK-NEXT: [[TMP3:%.*]] = icmp sgt <vscale x 4 x i16> [[WIDE_LOAD]], splat (i16 500)
+; CHECK-NEXT: [[TMP4:%.*]] = freeze <vscale x 4 x i1> [[TMP3]]
+; CHECK-NEXT: [[TMP6:%.*]] = call <vscale x 4 x i1> @llvm.mask.beforefirst.nxv4i1(<vscale x 4 x i1> [[TMP4]])
; CHECK-NEXT: [[LD_ADDR:%.*]] = getelementptr i16, ptr [[LOAD_ARRAY]], i64 [[IV]]
-; CHECK-NEXT: [[WIDE_MASKED_LOAD:%.*]] = call <vscale x 8 x i16> @llvm.masked.load.nxv8i16.p0(ptr align 2 [[LD_ADDR]], <vscale x 8 x i1> [[UNCOUNTABLE_EXIT_MASK]], <vscale x 8 x i16> poison)
-; CHECK-NEXT: [[TMP8:%.*]] = add nsw <vscale x 8 x i16> [[WIDE_MASKED_LOAD]], splat (i16 1)
+; CHECK-NEXT: [[WIDE_MASKED_LOAD:%.*]] = call <vscale x 4 x i16> @llvm.masked.load.nxv4i16.p0(ptr align 2 [[LD_ADDR]], <vscale x 4 x i1> [[TMP6]], <vscale x 4 x i16> poison)
+; CHECK-NEXT: [[TMP8:%.*]] = add nsw <vscale x 4 x i16> [[WIDE_MASKED_LOAD]], splat (i16 1)
; CHECK-NEXT: [[TMP9:%.*]] = getelementptr i16, ptr [[ARRAY]], i64 [[IV]]
-; CHECK-NEXT: call void @llvm.masked.store.nxv8i16.p0(<vscale x 8 x i16> [[TMP8]], ptr align 2 [[TMP9]], <vscale x 8 x i1> [[UNCOUNTABLE_EXIT_MASK]])
-; CHECK-NEXT: [[TMP11:%.*]] = call i1 @llvm.vector.reduce.or.nxv8i1(<vscale x 8 x i1> [[TMP10]])
+; CHECK-NEXT: call void @llvm.masked.store.nxv4i16.p0(<vscale x 4 x i16> [[TMP8]], ptr align 2 [[TMP9]], <vscale x 4 x i1> [[TMP6]])
+; CHECK-NEXT: [[TMP11:%.*]] = call i1 @llvm.vector.reduce.or.nxv4i1(<vscale x 4 x i1> [[TMP4]])
; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[IV]], [[TMP1]]
; CHECK-NEXT: [[TMP12:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
; CHECK-NEXT: [[TMP13:%.*]] = or i1 [[TMP11]], [[TMP12]]
; CHECK-NEXT: br i1 [[TMP13]], label %[[MIDDLE_BLOCK:.*]], label %[[FOR_BODY]], !llvm.loop [[LOOP6:![0-9]+]]
; CHECK: [[MIDDLE_BLOCK]]:
+; CHECK-NEXT: [[TMP7:%.*]] = call i64 @llvm.experimental.cttz.elts.i64.nxv4i1(<vscale x 4 x i1> [[TMP4]], i1 false)
; CHECK-NEXT: [[TMP14:%.*]] = add i64 [[IV]], [[TMP7]]
; CHECK-NEXT: [[TMP15:%.*]] = icmp eq i64 [[TMP14]], 20
; CHECK-NEXT: br i1 [[TMP15]], label %[[EXIT:.*]], label %[[SCALAR_PH1]]
@@ -487,7 +487,7 @@ define void @loop_contains_store_in_latch_block(ptr dereferenceable(40) noalias
; CHECK-SAME: ptr noalias dereferenceable(40) [[ARRAY:%.*]], ptr readonly align 2 dereferenceable(40) [[PRED:%.*]]) #[[ATTR0]] {
; CHECK-NEXT: [[SCALAR_PH1:.*]]:
; CHECK-NEXT: [[TMP0:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-NEXT: [[TMP1:%.*]] = shl nuw i64 [[TMP0]], 3
+; CHECK-NEXT: [[TMP1:%.*]] = shl nuw i64 [[TMP0]], 2
; CHECK-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 20, [[TMP1]]
; CHECK-NEXT: br i1 [[MIN_ITERS_CHECK]], label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]]
; CHECK: [[VECTOR_PH]]:
@@ -497,21 +497,21 @@ define void @loop_contains_store_in_latch_block(ptr dereferenceable(40) noalias
; CHECK: [[FOR_BODY1]]:
; CHECK-NEXT: [[IV1:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[FOR_BODY1]] ]
; CHECK-NEXT: [[EE_ADDR1:%.*]] = getelementptr inbounds nuw i16, ptr [[PRED]], i64 [[IV1]]
-; CHECK-NEXT: [[WIDE_LOAD:%.*]] = load <vscale x 8 x i16>, ptr [[EE_ADDR1]], align 2
-; CHECK-NEXT: [[TMP5:%.*]] = icmp sgt <vscale x 8 x i16> [[WIDE_LOAD]], splat (i16 500)
-; CHECK-NEXT: [[TMP9:%.*]] = freeze <vscale x 8 x i1> [[TMP5]]
-; CHECK-NEXT: [[TMP6:%.*]] = call i64 @llvm.experimental.cttz.elts.i64.nxv8i1(<vscale x 8 x i1> [[TMP9]], i1 false)
-; CHECK-NEXT: [[UNCOUNTABLE_EXIT_MASK:%.*]] = call <vscale x 8 x i1> @llvm.get.active.lane.mask.nxv8i1.i64(i64 0, i64 [[TMP6]])
+; CHECK-NEXT: [[WIDE_LOAD:%.*]] = load <vscale x 4 x i16>, ptr [[EE_ADDR1]], align 2
+; CHECK-NEXT: [[TMP3:%.*]] = icmp sgt <vscale x 4 x i16> [[WIDE_LOAD]], splat (i16 500)
+; CHECK-NEXT: [[TMP4:%.*]] = freeze <vscale x 4 x i1> [[TMP3]]
+; CHECK-NEXT: [[TMP5:%.*]] = call <vscale x 4 x i1> @llvm.mask.beforefirst.nxv4i1(<vscale x 4 x i1> [[TMP4]])
; CHECK-NEXT: [[TMP7:%.*]] = getelementptr i16, ptr [[ARRAY]], i64 [[IV1]]
-; CHECK-NEXT: [[WIDE_MASKED_LOAD:%.*]] = call <vscale x 8 x i16> @llvm.masked.load.nxv8i16.p0(ptr align 2 [[TMP7]], <vscale x 8 x i1> [[UNCOUNTABLE_EXIT_MASK]], <vscale x 8 x i16> poison)
-; CHECK-NEXT: [[TMP8:%.*]] = add nsw <vscale x 8 x i16> [[WIDE_MASKED_LOAD]], splat (i16 1)
-; CHECK-NEXT: call void @llvm.masked.store.nxv8i16.p0(<vscale x 8 x i16> [[TMP8]], ptr align 2 [[TMP7]], <vscale x 8 x i1> [[UNCOUNTABLE_EXIT_MASK]])
-; CHECK-NEXT: [[TMP10:%.*]] = call i1 @llvm.vector.reduce.or.nxv8i1(<vscale x 8 x i1> [[TMP9]])
+; CHECK-NEXT: [[WIDE_MASKED_LOAD:%.*]] = call <vscale x 4 x i16> @llvm.masked.load.nxv4i16.p0(ptr align 2 [[TMP7]], <vscale x 4 x i1> [[TMP5]], <vscale x 4 x i16> poison)
+; CHECK-NEXT: [[TMP8:%.*]] = add nsw <vscale x 4 x i16> [[WIDE_MASKED_LOAD]], splat (i16 1)
+; CHECK-NEXT: call void @llvm.masked.store.nxv4i16.p0(<vscale x 4 x i16> [[TMP8]], ptr align 2 [[TMP7]], <vscale x 4 x i1> [[TMP5]])
+; CHECK-NEXT: [[TMP10:%.*]] = call i1 @llvm.vector.reduce.or.nxv4i1(<vscale x 4 x i1> [[TMP4]])
; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[IV1]], [[TMP1]]
; CHECK-NEXT: [[TMP11:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
; CHECK-NEXT: [[TMP12:%.*]] = or i1 [[TMP10]], [[TMP11]]
; CHECK-NEXT: br i1 [[TMP12]], label %[[MIDDLE_BLOCK:.*]], label %[[FOR_BODY1]], !llvm.loop [[LOOP8:![0-9]+]]
; CHECK: [[MIDDLE_BLOCK]]:
+; CHECK-NEXT: [[TMP6:%.*]] = call i64 @llvm.experimental.cttz.elts.i64.nxv4i1(<vscale x 4 x i1> [[TMP4]], i1 false)
; CHECK-NEXT: [[TMP13:%.*]] = add i64 [[IV1]], [[TMP6]]
; CHECK-NEXT: [[TMP14:%.*]] = icmp eq i64 [[TMP13]], 20
; CHECK-NEXT: br i1 [[TMP14]], label %[[EXIT:.*]], label %[[SCALAR_PH]]
@@ -1159,7 +1159,7 @@ define i32 @uncountable_exit_with_separate_exit_block(ptr dereferenceable(40) no
; CHECK-SAME: ptr noalias dereferenceable(40) [[ARRAY:%.*]], ptr readonly align 2 dereferenceable(40) [[PRED:%.*]]) #[[ATTR0]] {
; CHECK-NEXT: [[SCALAR_PH:.*]]:
; CHECK-NEXT: [[TMP0:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-NEXT: [[TMP1:%.*]] = shl nuw i64 [[TMP0]], 3
+; CHECK-NEXT: [[TMP1:%.*]] = shl nuw i64 [[TMP0]], 2
; CHECK-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 20, [[TMP1]]
; CHECK-NEXT: br i1 [[MIN_ITERS_CHECK]], label %[[SCALAR_PH1:.*]], label %[[VECTOR_PH:.*]]
; CHECK: [[VECTOR_PH]]:
@@ -1169,21 +1169,21 @@ define i32 @uncountable_exit_with_separate_exit_block(ptr dereferenceable(40) no
; CHECK: [[FOR_BODY1]]:
; CHECK-NEXT: [[IV1:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[FOR_BODY1]] ]
; CHECK-NEXT: [[TMP5:%.*]] = getelementptr inbounds nuw i16, ptr [[PRED]], i64 [[IV1]]
-; CHECK-NEXT: [[WIDE_LOAD:%.*]] = load <vscale x 8 x i16>, ptr [[TMP5]], align 2
-; CHECK-NEXT: [[TMP6:%.*]] = icmp sgt <vscale x 8 x i16> [[WIDE_LOAD]], splat (i16 500)
-; CHECK-NEXT: [[TMP9:%.*]] = freeze <vscale x 8 x i1> [[TMP6]]
-; CHECK-NEXT: [[TMP7:%.*]] = call i64 @llvm.experimental.cttz.elts.i64.nxv8i1(<vscale x 8 x i1> [[TMP9]], i1 false)
-; CHECK-NEXT: [[UNCOUNTABLE_EXIT_MASK:%.*]] = call <vscale x 8 x i1> @llvm.get.active.lane.mask.nxv8i1.i64(i64 0, i64 [[TMP7]])
+; CHECK-NEXT: [[WIDE_LOAD:%.*]] = load <vscale x 4 x i16>, ptr [[TMP5]], align 2
+; CHECK-NEXT: [[TMP3:%.*]] = icmp sgt <vscale x 4 x i16> [[WIDE_LOAD]], splat (i16 500)
+; CHECK-NEXT: [[TMP4:%.*]] = freeze <vscale x 4 x i1> [[TMP3]]
+; CHECK-NEXT: [[TMP6:%.*]] = call <vscale x 4 x i1> @llvm.mask.beforefirst.nxv4i1(<vscale x 4 x i1> [[TMP4]])
; CHECK-NEXT: [[ST_ADDR1:%.*]] = getelementptr i16, ptr [[ARRAY]], i64 [[IV1]]
-; CHECK-NEXT: [[WIDE_MASKED_LOAD:%.*]] = call <vscale x 8 x i16> @llvm.masked.load.nxv8i16.p0(ptr align 2 [[ST_ADDR1]], <vscale x 8 x i1> [[UNCOUNTABLE_EXIT_MASK]], <vscale x 8 x i16> poison)
-; CHECK-NEXT: [[TMP8:%.*]] = add nsw <vscale x 8 x i16> [[WIDE_MASKED_LOAD]], splat (i16 1)
-; CHECK-NEXT: call void @llvm.masked.store.nxv8i16.p0(<vscale x 8 x i16> [[TMP8]], ptr align 2 [[ST_ADDR1]], <vscale x 8 x i1> [[UNCOUNTABLE_EXIT_MASK]])
-; CHECK-NEXT: [[TMP10:%.*]] = call i1 @llvm.vector.reduce.or.nxv8i1(<vscale x 8 x i1> [[TMP9]])
+; CHECK-NEXT: [[WIDE_MASKED_LOAD:%.*]] = call <vscale x 4 x i16> @llvm.masked.load.nxv4i16.p0(ptr align 2 [[ST_ADDR1]], <vscale x 4 x i1> [[TMP6]], <vscale x 4 x i16> poison)
+; CHECK-NEXT: [[TMP8:%.*]] = add nsw <vscale x 4 x i16> [[WIDE_MASKED_LOAD]], splat (i16 1)
+; CHECK-NEXT: call void @llvm.masked.store.nxv4i16.p0(<vscale x 4 x i16> [[TMP8]], ptr align 2 [[ST_ADDR1]], <vscale x 4 x i1> [[TMP6]])
+; CHECK-NEXT: [[TMP10:%.*]] = call i1 @llvm.vector.reduce.or.nxv4i1(<vscale x 4 x i1> [[TMP4]])
; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[IV1]], [[TMP1]]
; CHECK-NEXT: [[TMP11:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
; CHECK-NEXT: [[TMP12:%.*]] = or i1 [[TMP10]], [[TMP11]]
; CHECK-NEXT: br i1 [[TMP12]], label %[[MIDDLE_BLOCK:.*]], label %[[FOR_BODY1]], !llvm.loop [[LOOP10:![0-9]+]]
; CHECK: [[MIDDLE_BLOCK]]:
+; CHECK-NEXT: [[TMP7:%.*]] = call i64 @llvm.experimental.cttz.elts.i64.nxv4i1(<vscale x 4 x i1> [[TMP4]], i1 false)
; CHECK-NEXT: [[TMP13:%.*]] = add i64 [[IV1]], [[TMP7]]
; CHECK-NEXT: [[TMP14:%.*]] = icmp eq i64 [[TMP13]], 20
; CHECK-NEXT: br i1 [[TMP14]], label %[[EXIT_COUNTABLE:.*]], label %[[SCALAR_PH1]]
diff --git a/llvm/test/Transforms/LoopVectorize/RISCV/early_exit_with_stores.ll b/llvm/test/Transforms/LoopVectorize/RISCV/early_exit_with_stores.ll
index ca4d4f5fc770f4..70d757841f606e 100644
--- a/llvm/test/Transforms/LoopVectorize/RISCV/early_exit_with_stores.ll
+++ b/llvm/test/Transforms/LoopVectorize/RISCV/early_exit_with_stores.ll
@@ -65,8 +65,7 @@ define void @loop_contains_store_condition_load_has_single_user(ptr dereferencea
; CHECK-NEXT: [[WIDE_LOAD:%.*]] = load <vscale x 8 x i16>, ptr [[TMP5]], align 2
; CHECK-NEXT: [[TMP6:%.*]] = icmp sgt <vscale x 8 x i16> [[WIDE_LOAD]], splat (i16 500)
; CHECK-NEXT: [[TMP9:%.*]] = freeze <vscale x 8 x i1> [[TMP6]]
-; CHECK-NEXT: [[TMP7:%.*]] = call i64 @llvm.experimental.cttz.elts.i64.nxv8i1(<vscale x 8 x i1> [[TMP9]], i1 false)
-; CHECK-NEXT: [[UNCOUNTABLE_EXIT_MASK:%.*]] = call <vscale x 8 x i1> @llvm.get.active.lane.mask.nxv8i1.i64(i64 0, i64 [[TMP7]])
+; CHECK-NEXT: [[UNCOUNTABLE_EXIT_MASK:%.*]] = call <vscale x 8 x i1> @llvm.mask.beforefirst.nxv8i1(<vscale x 8 x i1> [[TMP9]])
; CHECK-NEXT: [[ST_ADDR:%.*]] = getelementptr i16, ptr [[ARRAY]], i64 [[IV]]
; CHECK-NEXT: [[WIDE_MASKED_LOAD:%.*]] = call <vscale x 8 x i16> @llvm.masked.load.nxv8i16.p0(ptr align 2 [[ST_ADDR]], <vscale x 8 x i1> [[UNCOUNTABLE_EXIT_MASK]], <vscale x 8 x i16> poison)
; CHECK-NEXT: [[TMP8:%.*]] = add nsw <vscale x 8 x i16> [[WIDE_MASKED_LOAD]], splat (i16 1)
@@ -77,6 +76,7 @@ define void @loop_contains_store_condition_load_has_single_user(ptr dereferencea
; CHECK-NEXT: [[TMP12:%.*]] = or i1 [[TMP10]], [[TMP11]]
; CHECK-NEXT: br i1 [[TMP12]], label %[[MIDDLE_BLOCK:.*]], label %[[FOR_BODY]], !llvm.loop [[LOOP0:![0-9]+]]
; CHECK: [[MIDDLE_BLOCK]]:
+; CHECK-NEXT: [[TMP7:%.*]] = call i64 @llvm.experimental.cttz.elts.i64.nxv8i1(<vscale x 8 x i1> [[TMP9]], i1 false)
; CHECK-NEXT: [[TMP13:%.*]] = add i64 [[IV]], [[TMP7]]
; CHECK-NEXT: [[TMP14:%.*]] = icmp eq i64 [[TMP13]], 20
; CHECK-NEXT: br i1 [[TMP14]], label %[[EXIT:.*]], label %[[SCALAR_PH1]]
@@ -134,13 +134,13 @@ define void @loop_contains_store_fcmp_condition(ptr dereferenceable(40) noalias
; CHECK-NEXT: [[WIDE_LOAD:%.*]] = load <16 x half>, ptr [[PRED]], align 2
; CHECK-NEXT: [[TMP0:%.*]] = fcmp ugt <16 x half> [[WIDE_LOAD]], splat (half 5.000000e+02)
; CHECK-NEXT: [[TMP1:%.*]] = freeze <16 x i1> [[TMP0]]
-; CHECK-NEXT: [[TMP2:%.*]] = call i64 @llvm.experimental.cttz.elts.i64.v16i1(<16 x i1> [[TMP1]], i1 false)
-; CHECK-NEXT: [[UNCOUNTABLE_EXIT_MASK:%.*]] = call <16 x i1> @llvm.get.active.lane.mask.v16i1.i64(i64 0, i64 [[TMP2]])
+; CHECK-NEXT: [[UNCOUNTABLE_EXIT_MASK:%.*]] = call <16 x i1> @llvm.mask.beforefirst.v16i1(<16 x i1> [[TMP1]])
; CHECK-NEXT: [[WIDE_MASKED_LOAD:%.*]] = call <16 x i16> @llvm.masked.load.v16i16.p0(ptr align 2 [[ARRAY]], <16 x i1> [[UNCOUNTABLE_EXIT_MASK]], <16 x i16> poison)
; CHECK-NEXT: [[TMP3:%.*]] = add nsw <16 x i16> [[WIDE_MASKED_LOAD]], splat (i16 1)
; CHECK-NEXT: call void @llvm.masked.store.v16i16.p0(<16 x i16> [[TMP3]], ptr align 2 [[ARRAY]], <16 x i1> [[UNCOUNTABLE_EXIT_MASK]])
; CHECK-NEXT: br label %[[MIDDLE_BLOCK:.*]]
; CHECK: [[MIDDLE_BLOCK]]:
+; CHECK-NEXT: [[TMP2:%.*]] = call i64 @llvm.experimental.cttz.elts.i64.v16i1(<16 x i1> [[TMP1]], i1 false)
; CHECK-NEXT: [[TMP4:%.*]] = icmp eq i64 [[TMP2]], 20
; CHECK-NEXT: br i1 [[TMP4]], label %[[EXIT:.*]], label %[[SCALAR_PH:.*]]
; CHECK: [[SCALAR_PH]]:
@@ -360,8 +360,7 @@ define void @loop_contains_store_to_pointer_with_no_deref_info(ptr align 2 deref
; CHECK-NEXT: [[WIDE_LOAD:%.*]] = load <vscale x 8 x i16>, ptr [[TMP5]], align 2
; CHECK-NEXT: [[TMP6:%.*]] = icmp sgt <vscale x 8 x i16> [[WIDE_LOAD]], splat (i16 500)
; CHECK-NEXT: [[TMP10:%.*]] = freeze <vscale x 8 x i1> [[TMP6]]
-; CHECK-NEXT: [[TMP7:%.*]] = call i64 @llvm.experimental.cttz.elts.i64.nxv8i1(<vscale x 8 x i1> [[TMP10]], i1 false)
-; CHECK-NEXT: [[UNCOUNTABLE_EXIT_MASK:%.*]] = call <vscale x 8 x i1> @llvm.get.active.lane.mask.nxv8i1.i64(i64 0, i64 [[TMP7]])
+; CHECK-NEXT: [[UNCOUNTABLE_EXIT_MASK:%.*]] = call <vscale x 8 x i1> @llvm.mask.beforefirst.nxv8i1(<vscale x 8 x i1> [[TMP10]])
; CHECK-NEXT: [[LD_ADDR:%.*]] = getelementptr i16, ptr [[LOAD_ARRAY]], i64 [[IV]]
; CHECK-NEXT: [[WIDE_MASKED_LOAD:%.*]] = call <vscale x 8 x i16> @llvm.masked.load.nxv8i16.p0(ptr align 2 [[LD_ADDR]], <vscale x 8 x i1> [[UNCOUNTABLE_EXIT_MASK]], <vscale x 8 x i16> poison)
; CHECK-NEXT: [[TMP8:%.*]] = add nsw <vscale x 8 x i16> [[WIDE_MASKED_LOAD]], splat (i16 1)
@@ -373,6 +372,7 @@ define void @loop_contains_store_to_pointer_with_no_deref_info(ptr align 2 deref
; CHECK-NEXT: [[TMP13:%.*]] = or i1 [[TMP11]], [[TMP12]]
; CHECK-NEXT: br i1 [[TMP13]], label %[[MIDDLE_BLOCK:.*]], label %[[FOR_BODY]], !llvm.loop [[LOOP5:![0-9]+]]
; CHECK: [[MIDDLE_BLOCK]]:
+; CHECK-NEXT: [[TMP7:%.*]] = call i64 @llvm.experimental.cttz.elts.i64.nxv8i1(<vscale x 8 x i1> [[TMP10]], i1 false)
; CHECK-NEXT: [[TMP14:%.*]] = add i64 [[IV]], [[TMP7]]
; CHECK-NEXT: [[TMP15:%.*]] = icmp eq i64 [[TMP14]], 20
; CHECK-NEXT: br i1 [[TMP15]], label %[[EXIT:.*]], label %[[SCALAR_PH1]]
@@ -529,8 +529,7 @@ define void @loop_contains_store_in_latch_block(ptr dereferenceable(40) noalias
; CHECK-NEXT: [[WIDE_LOAD:%.*]] = load <vscale x 8 x i16>, ptr [[EE_ADDR]], align 2
; CHECK-NEXT: [[TMP5:%.*]] = icmp sgt <vscale x 8 x i16> [[WIDE_LOAD]], splat (i16 500)
; CHECK-NEXT: [[TMP9:%.*]] = freeze <vscale x 8 x i1> [[TMP5]]
-; CHECK-NEXT: [[TMP6:%.*]] = call i64 @llvm.experimental.cttz.elts.i64.nxv8i1(<vscale x 8 x i1> [[TMP9]], i1 false)
-; CHECK-NEXT: [[UNCOUNTABLE_EXIT_MASK:%.*]] = call <vscale x 8 x i1> @llvm.get.active.lane.mask.nxv8i1.i64(i64 0, i64 [[TMP6]])
+; CHECK-NEXT: [[UNCOUNTABLE_EXIT_MASK:%.*]] = call <vscale x 8 x i1> @llvm.mask.beforefirst.nxv8i1(<vscale x 8 x i1> [[TMP9]])
; CHECK-NEXT: [[TMP7:%.*]] = getelementptr i16, ptr [[ARRAY]], i64 [[IV]]
; CHECK-NEXT: [[WIDE_MASKED_LOAD:%.*]] = call <vscale x 8 x i16> @llvm.masked.load.nxv8i16.p0(ptr align 2 [[TMP7]], <vscale x 8 x i1> [[UNCOUNTABLE_EXIT_MASK]], <vscale x 8 x i16> poison)
; CHECK-NEXT: [[TMP8:%.*]] = add nsw <vscale x 8 x i16> [[WIDE_MASKED_LOAD]], splat (i16 1)
@@ -541,6 +540,7 @@ define void @loop_contains_store_in_latch_block(ptr dereferenceable(40) noalias
; CHECK-NEXT: [[TMP12:%.*]] = or i1 [[TMP10]], [[TMP11]]
; CHECK-NEXT: br i1 [[TMP12]], label %[[MIDDLE_BLOCK:.*]], label %[[FOR_BODY]], !llvm.loop [[LOOP7:![0-9]+]]
; CHECK: [[MIDDLE_BLOCK]]:
+; CHECK-NEXT: [[TMP6:%.*]] = call i64 @llvm.experimental.cttz.elts.i64.nxv8i1(<vscale x 8 x i1> [[TMP9]], i1 false)
; CHECK-NEXT: [[TMP13:%.*]] = add i64 [[IV]], [[TMP6]]
; CHECK-NEXT: [[TMP14:%.*]] = icmp eq i64 [[TMP13]], 20
; CHECK-NEXT: br i1 [[TMP14]], label %[[EXIT:.*]], label %[[SCALAR_PH1]]
@@ -1201,8 +1201,7 @@ define i32 @uncountable_exit_with_separate_exit_block(ptr dereferenceable(40) no
; CHECK-NEXT: [[WIDE_LOAD:%.*]] = load <vscale x 8 x i16>, ptr [[TMP5]], align 2
; CHECK-NEXT: [[TMP6:%.*]] = icmp sgt <vscale x 8 x i16> [[WIDE_LOAD]], splat (i16 500)
; CHECK-NEXT: [[TMP9:%.*]] = freeze <vscale x 8 x i1> [[TMP6]]
-; CHECK-NEXT: [[TMP7:%.*]] = call i64 @llvm.experimental.cttz.elts.i64.nxv8i1(<vscale x 8 x i1> [[TMP9]], i1 false)
-; CHECK-NEXT: [[UNCOUNTABLE_EXIT_MASK:%.*]] = call <vscale x 8 x i1> @llvm.get.active.lane.mask.nxv8i1.i64(i64 0, i64 [[TMP7]])
+; CHECK-NEXT: [[UNCOUNTABLE_EXIT_MASK:%.*]] = call <vscale x 8 x i1> @llvm.mask.beforefirst.nxv8i1(<vscale x 8 x i1> [[TMP9]])
; CHECK-NEXT: [[ST_ADDR:%.*]] = getelementptr i16, ptr [[ARRAY]], i64 [[IV]]
; CHECK-NEXT: [[WIDE_MASKED_LOAD:%.*]] = call <vscale x 8 x i16> @llvm.masked.load.nxv8i16.p0(ptr align 2 [[ST_ADDR]], <vscale x 8 x i1> [[UNCOUNTABLE_EXIT_MASK]], <vscale x 8 x i16> poison)
; CHECK-NEXT: [[TMP8:%.*]] = add nsw <vscale x 8 x i16> [[WIDE_MASKED_LOAD]], splat (i16 1)
@@ -1213,6 +1212,7 @@ define i32 @uncountable_exit_with_separate_exit_block(ptr dereferenceable(40) no
; CHECK-NEXT: [[TMP12:%.*]] = or i1 [[TMP10]], [[TMP11]]
; CHECK-NEXT: br i1 [[TMP12]], label %[[MIDDLE_BLOCK:.*]], label %[[FOR_BODY]], !llvm.loop [[LOOP9:![0-9]+]]
; CHECK: [[MIDDLE_BLOCK]]:
+; CHECK-NEXT: [[TMP7:%.*]] = call i64 @llvm.experimental.cttz.elts.i64.nxv8i1(<vscale x 8 x i1> [[TMP9]], i1 false)
; CHECK-NEXT: [[TMP13:%.*]] = add i64 [[IV]], [[TMP7]]
; CHECK-NEXT: [[TMP14:%.*]] = icmp eq i64 [[TMP13]], 20
; CHECK-NEXT: br i1 [[TMP14]], label %[[EXIT_COUNTABLE:.*]], label %[[SCALAR_PH]]
diff --git a/llvm/test/Transforms/LoopVectorize/VPlan/early_exit_with_stores_vplan.ll b/llvm/test/Transforms/LoopVectorize/VPlan/early_exit_with_stores_vplan.ll
index bc814cf618c6e9..7b403e6835b11f 100644
--- a/llvm/test/Transforms/LoopVectorize/VPlan/early_exit_with_stores_vplan.ll
+++ b/llvm/test/Transforms/LoopVectorize/VPlan/early_exit_with_stores_vplan.ll
@@ -28,24 +28,24 @@ define void @loop_contains_store_condition_load_has_single_user(ptr dereferencea
; CHECK-NEXT: WIDEN ir<%ee.val> = load vp<[[VP5]]>
; CHECK-NEXT: WIDEN ir<%ee.cond> = icmp sgt ir<%ee.val>, ir<500>
; CHECK-NEXT: EMIT vp<[[VP6:%[0-9]+]]> = freeze ir<%ee.cond>
-; CHECK-NEXT: EMIT vp<[[VP7:%[0-9]+]]> = first-active-lane vp<[[VP6]]>
-; CHECK-NEXT: EMIT vp<%uncountable.exit.mask> = active lane mask ir<0>, vp<[[VP7]]>
+; CHECK-NEXT: EMIT vp<%uncountable.exit.mask> = mask-before-first vp<[[VP6]]>
; CHECK-NEXT: CLONE ir<%st.addr> = getelementptr ir<%array>, vp<[[VP4]]>
-; CHECK-NEXT: vp<[[VP8:%[0-9]+]]> = vector-pointer i16, ir<%st.addr>, ir<1>
-; CHECK-NEXT: WIDEN ir<%data> = load vp<[[VP8]]>, vp<%uncountable.exit.mask>
+; CHECK-NEXT: vp<[[VP7:%[0-9]+]]> = vector-pointer i16, ir<%st.addr>, ir<1>
+; CHECK-NEXT: WIDEN ir<%data> = load vp<[[VP7]]>, vp<%uncountable.exit.mask>
; CHECK-NEXT: WIDEN ir<%inc> = add nsw ir<%data>, ir<1>
-; CHECK-NEXT: WIDEN store vp<[[VP8]]>, ir<%inc>, vp<%uncountable.exit.mask>
-; CHECK-NEXT: EMIT vp<[[VP9:%[0-9]+]]> = any-of vp<[[VP6]]>
+; CHECK-NEXT: WIDEN store vp<[[VP7]]>, ir<%inc>, vp<%uncountable.exit.mask>
+; CHECK-NEXT: EMIT vp<[[VP8:%[0-9]+]]> = any-of vp<[[VP6]]>
; CHECK-NEXT: EMIT vp<%index.next> = add nuw vp<[[VP3]]>, vp<[[VP1]]>
-; CHECK-NEXT: EMIT vp<[[VP10:%[0-9]+]]> = icmp eq vp<%index.next>, vp<[[VP2]]>
-; CHECK-NEXT: EMIT branch-on-two-conds vp<[[VP9]]>, vp<[[VP10]]>
+; CHECK-NEXT: EMIT vp<[[VP9:%[0-9]+]]> = icmp eq vp<%index.next>, vp<[[VP2]]>
+; CHECK-NEXT: EMIT branch-on-two-conds vp<[[VP8]]>, vp<[[VP9]]>
; CHECK-NEXT: No successors
; CHECK-NEXT: }
; CHECK-NEXT: Successor(s): middle.block, middle.block
; CHECK-EMPTY:
; CHECK-NEXT: middle.block:
+; CHECK-NEXT: EMIT vp<[[VP11:%[0-9]+]]> = first-active-lane vp<[[VP6]]>
; CHECK-NEXT: EMIT vp<[[VP12:%[0-9]+]]> = extract-lane ir<0>, ir<%iv>
-; CHECK-NEXT: EMIT vp<[[VP13:%[0-9]+]]> = add vp<[[VP12]]>, vp<[[VP7]]>
+; CHECK-NEXT: EMIT vp<[[VP13:%[0-9]+]]> = add vp<[[VP12]]>, vp<[[VP11]]>
; CHECK-NEXT: EMIT vp<[[VP14:%[0-9]+]]> = icmp eq vp<[[VP13]]>, ir<20>
; CHECK-NEXT: EMIT branch-on-cond vp<[[VP14]]>
; CHECK-NEXT: Successor(s): ir-bb<exit>, scalar.ph
@@ -117,24 +117,24 @@ define void @loop_contains_store_after_uncountable_exit(ptr dereferenceable(40)
; CHECK-NEXT: WIDEN ir<%ee.val> = load vp<[[VP5]]>
; CHECK-NEXT: WIDEN ir<%ee.cond> = icmp sgt ir<%ee.val>, ir<500>
; CHECK-NEXT: EMIT vp<[[VP6:%[0-9]+]]> = freeze ir<%ee.cond>
-; CHECK-NEXT: EMIT vp<[[VP7:%[0-9]+]]> = first-active-lane vp<[[VP6]]>
-; CHECK-NEXT: EMIT vp<%uncountable.exit.mask> = active lane mask ir<0>, vp<[[VP7]]>
+; CHECK-NEXT: EMIT vp<%uncountable.exit.mask> = mask-before-first vp<[[VP6]]>
; CHECK-NEXT: CLONE ir<%st.addr> = getelementptr ir<%array>, vp<[[VP4]]>
-; CHECK-NEXT: vp<[[VP8:%[0-9]+]]> = vector-pointer i16, ir<%st.addr>, ir<1>
-; CHECK-NEXT: WIDEN ir<%data> = load vp<[[VP8]]>, vp<%uncountable.exit.mask> (!vplan.execution.frequency 17870283321406128127 (96.88%, estimated))
+; CHECK-NEXT: vp<[[VP7:%[0-9]+]]> = vector-pointer i16, ir<%st.addr>, ir<1>
+; CHECK-NEXT: WIDEN ir<%data> = load vp<[[VP7]]>, vp<%uncountable.exit.mask> (!vplan.execution.frequency 17870283321406128127 (96.88%, estimated))
; CHECK-NEXT: WIDEN ir<%inc> = add nsw ir<%data>, ir<1>
-; CHECK-NEXT: WIDEN store vp<[[VP8]]>, ir<%inc>, vp<%uncountable.exit.mask> (!vplan.execution.frequency 17870283321406128127 (96.88%, estimated))
-; CHECK-NEXT: EMIT vp<[[VP9:%[0-9]+]]> = any-of vp<[[VP6]]>
+; CHECK-NEXT: WIDEN store vp<[[VP7]]>, ir<%inc>, vp<%uncountable.exit.mask> (!vplan.execution.frequency 17870283321406128127 (96.88%, estimated))
+; CHECK-NEXT: EMIT vp<[[VP8:%[0-9]+]]> = any-of vp<[[VP6]]>
; CHECK-NEXT: EMIT vp<%index.next> = add nuw vp<[[VP3]]>, vp<[[VP1]]>
-; CHECK-NEXT: EMIT vp<[[VP10:%[0-9]+]]> = icmp eq vp<%index.next>, vp<[[VP2]]>
-; CHECK-NEXT: EMIT branch-on-two-conds vp<[[VP9]]>, vp<[[VP10]]>
+; CHECK-NEXT: EMIT vp<[[VP9:%[0-9]+]]> = icmp eq vp<%index.next>, vp<[[VP2]]>
+; CHECK-NEXT: EMIT branch-on-two-conds vp<[[VP8]]>, vp<[[VP9]]>
; CHECK-NEXT: No successors
; CHECK-NEXT: }
; CHECK-NEXT: Successor(s): middle.block, middle.block
; CHECK-EMPTY:
; CHECK-NEXT: middle.block:
+; CHECK-NEXT: EMIT vp<[[VP11:%[0-9]+]]> = first-active-lane vp<[[VP6]]>
; CHECK-NEXT: EMIT vp<[[VP12:%[0-9]+]]> = extract-lane ir<0>, ir<%iv>
-; CHECK-NEXT: EMIT vp<[[VP13:%[0-9]+]]> = add vp<[[VP12]]>, vp<[[VP7]]>
+; CHECK-NEXT: EMIT vp<[[VP13:%[0-9]+]]> = add vp<[[VP12]]>, vp<[[VP11]]>
; CHECK-NEXT: EMIT vp<[[VP14:%[0-9]+]]> = icmp eq vp<[[VP13]]>, ir<20>
; CHECK-NEXT: EMIT branch-on-cond vp<[[VP14]]>
; CHECK-NEXT: Successor(s): ir-bb<exit>, scalar.ph
@@ -202,32 +202,32 @@ define i16 @uncountable_exit_with_live_out(ptr dereferenceable(40) noalias %arra
; CHECK-NEXT: WIDEN ir<%ee.val> = load vp<[[VP5]]>
; CHECK-NEXT: WIDEN ir<%ee.cond> = icmp sgt ir<%ee.val>, ir<500>
; CHECK-NEXT: EMIT vp<[[VP6:%[0-9]+]]> = freeze ir<%ee.cond>
-; CHECK-NEXT: EMIT vp<[[VP7:%[0-9]+]]> = first-active-lane vp<[[VP6]]>
-; CHECK-NEXT: EMIT vp<%uncountable.exit.mask> = active lane mask ir<0>, vp<[[VP7]]>
+; CHECK-NEXT: EMIT vp<%uncountable.exit.mask> = mask-before-first vp<[[VP6]]>
; CHECK-NEXT: CLONE ir<%st.addr> = getelementptr ir<%array>, vp<[[VP4]]>
-; CHECK-NEXT: vp<[[VP8:%[0-9]+]]> = vector-pointer i16, ir<%st.addr>, ir<1>
-; CHECK-NEXT: WIDEN ir<%data> = load vp<[[VP8]]>, vp<%uncountable.exit.mask>
+; CHECK-NEXT: vp<[[VP7:%[0-9]+]]> = vector-pointer i16, ir<%st.addr>, ir<1>
+; CHECK-NEXT: WIDEN ir<%data> = load vp<[[VP7]]>, vp<%uncountable.exit.mask>
; CHECK-NEXT: WIDEN ir<%inc> = add nsw ir<%data>, ir<1>
-; CHECK-NEXT: WIDEN store vp<[[VP8]]>, ir<%inc>, vp<%uncountable.exit.mask>
-; CHECK-NEXT: EMIT vp<[[VP9:%[0-9]+]]> = any-of vp<[[VP6]]>
+; CHECK-NEXT: WIDEN store vp<[[VP7]]>, ir<%inc>, vp<%uncountable.exit.mask>
+; CHECK-NEXT: EMIT vp<[[VP8:%[0-9]+]]> = any-of vp<[[VP6]]>
; CHECK-NEXT: EMIT vp<%index.next> = add nuw vp<[[VP3]]>, vp<[[VP1]]>
-; CHECK-NEXT: EMIT vp<[[VP10:%[0-9]+]]> = icmp eq vp<%index.next>, vp<[[VP2]]>
-; CHECK-NEXT: EMIT branch-on-two-conds vp<[[VP9]]>, vp<[[VP10]]>
+; CHECK-NEXT: EMIT vp<[[VP9:%[0-9]+]]> = icmp eq vp<%index.next>, vp<[[VP2]]>
+; CHECK-NEXT: EMIT branch-on-two-conds vp<[[VP8]]>, vp<[[VP9]]>
; CHECK-NEXT: No successors
; CHECK-NEXT: }
; CHECK-NEXT: Successor(s): middle.block, middle.block
; CHECK-EMPTY:
; CHECK-NEXT: middle.block:
-; CHECK-NEXT: EMIT vp<[[VP12:%[0-9]+]]> = extract-last-part ir<%data>
-; CHECK-NEXT: EMIT vp<[[VP13:%[0-9]+]]> = extract-last-lane vp<[[VP12]]>
+; CHECK-NEXT: EMIT vp<[[VP11:%[0-9]+]]> = extract-last-part ir<%data>
+; CHECK-NEXT: EMIT vp<[[VP12:%[0-9]+]]> = extract-last-lane vp<[[VP11]]>
+; CHECK-NEXT: EMIT vp<[[VP13:%[0-9]+]]> = first-active-lane vp<[[VP6]]>
; CHECK-NEXT: EMIT vp<[[VP14:%[0-9]+]]> = extract-lane ir<0>, ir<%iv>
-; CHECK-NEXT: EMIT vp<[[VP15:%[0-9]+]]> = add vp<[[VP14]]>, vp<[[VP7]]>
+; CHECK-NEXT: EMIT vp<[[VP15:%[0-9]+]]> = add vp<[[VP14]]>, vp<[[VP13]]>
; CHECK-NEXT: EMIT vp<[[VP16:%[0-9]+]]> = icmp eq vp<[[VP15]]>, ir<20>
; CHECK-NEXT: EMIT branch-on-cond vp<[[VP16]]>
; CHECK-NEXT: Successor(s): ir-bb<exit>, scalar.ph
; CHECK-EMPTY:
; CHECK-NEXT: ir-bb<exit>:
-; CHECK-NEXT: IR %data.lcssa = phi i16 [ %data, %for.inc ], [ %data, %for.body ] (extra operand: vp<[[VP13]]> from middle.block)
+; CHECK-NEXT: IR %data.lcssa = phi i16 [ %data, %for.inc ], [ %data, %for.body ] (extra operand: vp<[[VP12]]> from middle.block)
; CHECK-NEXT: No successors
; CHECK-EMPTY:
; CHECK-NEXT: scalar.ph:
@@ -295,26 +295,26 @@ define void @combined_exit_conditions(ptr align 4 dereferenceable(80) readonly %
; CHECK-NEXT: WIDEN ir<%ee.cmp> = icmp ne ir<%ee.val>, ir<0>
; CHECK-NEXT: EMIT vp<[[VP6:%[0-9]+]]> = freeze ir<%ee.cmp>
; CHECK-NEXT: EMIT vp<[[VP7:%[0-9]+]]> = freeze vp<[[VP6]]>
-; CHECK-NEXT: EMIT vp<[[VP8:%[0-9]+]]> = first-active-lane vp<[[VP7]]>
-; CHECK-NEXT: EMIT vp<%uncountable.exit.mask> = active lane mask ir<0>, vp<[[VP8]]>
+; CHECK-NEXT: EMIT vp<%uncountable.exit.mask> = mask-before-first vp<[[VP7]]>
; CHECK-NEXT: CLONE ir<%src.ptr> = getelementptr ir<%src>, vp<[[VP4]]>
-; CHECK-NEXT: vp<[[VP9:%[0-9]+]]> = vector-pointer i32, ir<%src.ptr>, ir<1>
-; CHECK-NEXT: WIDEN ir<%data> = load vp<[[VP9]]>, vp<%uncountable.exit.mask>
+; CHECK-NEXT: vp<[[VP8:%[0-9]+]]> = vector-pointer i32, ir<%src.ptr>, ir<1>
+; CHECK-NEXT: WIDEN ir<%data> = load vp<[[VP8]]>, vp<%uncountable.exit.mask>
; CHECK-NEXT: WIDEN ir<%add> = add nsw ir<%data>, ir<1>
; CHECK-NEXT: CLONE ir<%dst.ptr> = getelementptr ir<%dst>, vp<[[VP4]]>
-; CHECK-NEXT: vp<[[VP10:%[0-9]+]]> = vector-pointer i32, ir<%dst.ptr>, ir<1>
-; CHECK-NEXT: WIDEN store vp<[[VP10]]>, ir<%add>, vp<%uncountable.exit.mask>
-; CHECK-NEXT: EMIT vp<[[VP11:%[0-9]+]]> = any-of vp<[[VP7]]>
+; CHECK-NEXT: vp<[[VP9:%[0-9]+]]> = vector-pointer i32, ir<%dst.ptr>, ir<1>
+; CHECK-NEXT: WIDEN store vp<[[VP9]]>, ir<%add>, vp<%uncountable.exit.mask>
+; CHECK-NEXT: EMIT vp<[[VP10:%[0-9]+]]> = any-of vp<[[VP7]]>
; CHECK-NEXT: EMIT vp<%index.next> = add nuw vp<[[VP3]]>, vp<[[VP1]]>
-; CHECK-NEXT: EMIT vp<[[VP12:%[0-9]+]]> = icmp eq vp<%index.next>, vp<[[VP2]]>
-; CHECK-NEXT: EMIT branch-on-two-conds vp<[[VP11]]>, vp<[[VP12]]>
+; CHECK-NEXT: EMIT vp<[[VP11:%[0-9]+]]> = icmp eq vp<%index.next>, vp<[[VP2]]>
+; CHECK-NEXT: EMIT branch-on-two-conds vp<[[VP10]]>, vp<[[VP11]]>
; CHECK-NEXT: No successors
; CHECK-NEXT: }
; CHECK-NEXT: Successor(s): middle.block, middle.block
; CHECK-EMPTY:
; CHECK-NEXT: middle.block:
+; CHECK-NEXT: EMIT vp<[[VP13:%[0-9]+]]> = first-active-lane vp<[[VP7]]>
; CHECK-NEXT: EMIT vp<[[VP14:%[0-9]+]]> = extract-lane ir<0>, ir<%iv>
-; CHECK-NEXT: EMIT vp<[[VP15:%[0-9]+]]> = add vp<[[VP14]]>, vp<[[VP8]]>
+; CHECK-NEXT: EMIT vp<[[VP15:%[0-9]+]]> = add vp<[[VP14]]>, vp<[[VP13]]>
; CHECK-NEXT: EMIT vp<[[VP16:%[0-9]+]]> = icmp eq vp<[[VP15]]>, ir<20>
; CHECK-NEXT: EMIT branch-on-cond vp<[[VP16]]>
; CHECK-NEXT: Successor(s): ir-bb<exit>, scalar.ph
diff --git a/llvm/test/Transforms/LoopVectorize/early_exit_combined_exits.ll b/llvm/test/Transforms/LoopVectorize/early_exit_combined_exits.ll
index 385dcc78a5807a..88dfa28a4a7e4d 100644
--- a/llvm/test/Transforms/LoopVectorize/early_exit_combined_exits.ll
+++ b/llvm/test/Transforms/LoopVectorize/early_exit_combined_exits.ll
@@ -15,8 +15,7 @@ define void @combined_exit_conditions(ptr align 4 dereferenceable(80) readonly %
; CHECK-NEXT: [[TMP2:%.*]] = icmp ne <4 x i32> [[WIDE_LOAD]], zeroinitializer
; CHECK-NEXT: [[TMP12:%.*]] = freeze <4 x i1> [[TMP2]]
; CHECK-NEXT: [[TMP6:%.*]] = freeze <4 x i1> [[TMP12]]
-; CHECK-NEXT: [[TMP3:%.*]] = call i64 @llvm.experimental.cttz.elts.i64.v4i1(<4 x i1> [[TMP6]], i1 false)
-; CHECK-NEXT: [[UNCOUNTABLE_EXIT_MASK:%.*]] = call <4 x i1> @llvm.get.active.lane.mask.v4i1.i64(i64 0, i64 [[TMP3]])
+; CHECK-NEXT: [[UNCOUNTABLE_EXIT_MASK:%.*]] = call <4 x i1> @llvm.mask.beforefirst.v4i1(<4 x i1> [[TMP6]])
; CHECK-NEXT: [[TMP0:%.*]] = getelementptr [4 x i8], ptr [[SRC]], i64 [[INDEX]]
; CHECK-NEXT: [[WIDE_MASKED_LOAD:%.*]] = call <4 x i32> @llvm.masked.load.v4i32.p0(ptr align 4 [[TMP0]], <4 x i1> [[UNCOUNTABLE_EXIT_MASK]], <4 x i32> poison)
; CHECK-NEXT: [[TMP4:%.*]] = add nsw <4 x i32> [[WIDE_MASKED_LOAD]], splat (i32 1)
@@ -28,6 +27,7 @@ define void @combined_exit_conditions(ptr align 4 dereferenceable(80) readonly %
; CHECK-NEXT: [[TMP9:%.*]] = or i1 [[TMP7]], [[TMP8]]
; CHECK-NEXT: br i1 [[TMP9]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP0:![0-9]+]]
; CHECK: [[MIDDLE_BLOCK]]:
+; CHECK-NEXT: [[TMP3:%.*]] = call i64 @llvm.experimental.cttz.elts.i64.v4i1(<4 x i1> [[TMP6]], i1 false)
; CHECK-NEXT: [[TMP10:%.*]] = add i64 [[INDEX]], [[TMP3]]
; CHECK-NEXT: [[TMP11:%.*]] = icmp eq i64 [[TMP10]], 20
; CHECK-NEXT: br i1 [[TMP11]], label %[[EXIT:.*]], label %[[SCALAR_PH:.*]]
@@ -86,8 +86,7 @@ define void @combined_exit_conditions_swap_comparison_order(ptr align 4 derefere
; CHECK-NEXT: [[TMP1:%.*]] = icmp ne <4 x i32> [[WIDE_LOAD]], zeroinitializer
; CHECK-NEXT: [[TMP12:%.*]] = freeze <4 x i1> [[TMP1]]
; CHECK-NEXT: [[TMP6:%.*]] = freeze <4 x i1> [[TMP12]]
-; CHECK-NEXT: [[TMP2:%.*]] = call i64 @llvm.experimental.cttz.elts.i64.v4i1(<4 x i1> [[TMP6]], i1 false)
-; CHECK-NEXT: [[UNCOUNTABLE_EXIT_MASK:%.*]] = call <4 x i1> @llvm.get.active.lane.mask.v4i1.i64(i64 0, i64 [[TMP2]])
+; CHECK-NEXT: [[UNCOUNTABLE_EXIT_MASK:%.*]] = call <4 x i1> @llvm.mask.beforefirst.v4i1(<4 x i1> [[TMP6]])
; CHECK-NEXT: [[TMP3:%.*]] = getelementptr [4 x i8], ptr [[SRC]], i64 [[INDEX]]
; CHECK-NEXT: [[WIDE_MASKED_LOAD:%.*]] = call <4 x i32> @llvm.masked.load.v4i32.p0(ptr align 4 [[TMP3]], <4 x i1> [[UNCOUNTABLE_EXIT_MASK]], <4 x i32> poison)
; CHECK-NEXT: [[TMP4:%.*]] = add nsw <4 x i32> [[WIDE_MASKED_LOAD]], splat (i32 1)
@@ -99,6 +98,7 @@ define void @combined_exit_conditions_swap_comparison_order(ptr align 4 derefere
; CHECK-NEXT: [[TMP9:%.*]] = or i1 [[TMP7]], [[TMP8]]
; CHECK-NEXT: br i1 [[TMP9]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP4:![0-9]+]]
; CHECK: [[MIDDLE_BLOCK]]:
+; CHECK-NEXT: [[TMP2:%.*]] = call i64 @llvm.experimental.cttz.elts.i64.v4i1(<4 x i1> [[TMP6]], i1 false)
; CHECK-NEXT: [[TMP10:%.*]] = add i64 [[INDEX]], [[TMP2]]
; CHECK-NEXT: [[TMP11:%.*]] = icmp eq i64 [[TMP10]], 20
; CHECK-NEXT: br i1 [[TMP11]], label %[[EXIT:.*]], label %[[SCALAR_PH:.*]]
@@ -386,8 +386,7 @@ define void @combined_exit_conditions_binary_or(ptr align 4 dereferenceable(80)
; CHECK-NEXT: [[WIDE_LOAD:%.*]] = load <4 x i32>, ptr [[TMP0]], align 4
; CHECK-NEXT: [[TMP1:%.*]] = icmp ne <4 x i32> [[WIDE_LOAD]], zeroinitializer
; CHECK-NEXT: [[TMP6:%.*]] = freeze <4 x i1> [[TMP1]]
-; CHECK-NEXT: [[TMP2:%.*]] = call i64 @llvm.experimental.cttz.elts.i64.v4i1(<4 x i1> [[TMP6]], i1 false)
-; CHECK-NEXT: [[UNCOUNTABLE_EXIT_MASK:%.*]] = call <4 x i1> @llvm.get.active.lane.mask.v4i1.i64(i64 0, i64 [[TMP2]])
+; CHECK-NEXT: [[UNCOUNTABLE_EXIT_MASK:%.*]] = call <4 x i1> @llvm.mask.beforefirst.v4i1(<4 x i1> [[TMP6]])
; CHECK-NEXT: [[TMP3:%.*]] = getelementptr [4 x i8], ptr [[SRC]], i64 [[INDEX]]
; CHECK-NEXT: [[WIDE_MASKED_LOAD:%.*]] = call <4 x i32> @llvm.masked.load.v4i32.p0(ptr align 4 [[TMP3]], <4 x i1> [[UNCOUNTABLE_EXIT_MASK]], <4 x i32> poison)
; CHECK-NEXT: [[TMP4:%.*]] = add nsw <4 x i32> [[WIDE_MASKED_LOAD]], splat (i32 1)
@@ -399,6 +398,7 @@ define void @combined_exit_conditions_binary_or(ptr align 4 dereferenceable(80)
; CHECK-NEXT: [[TMP9:%.*]] = or i1 [[TMP7]], [[TMP8]]
; CHECK-NEXT: br i1 [[TMP9]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP6:![0-9]+]]
; CHECK: [[MIDDLE_BLOCK]]:
+; CHECK-NEXT: [[TMP2:%.*]] = call i64 @llvm.experimental.cttz.elts.i64.v4i1(<4 x i1> [[TMP6]], i1 false)
; CHECK-NEXT: [[TMP10:%.*]] = add i64 [[INDEX]], [[TMP2]]
; CHECK-NEXT: [[TMP11:%.*]] = icmp eq i64 [[TMP10]], 20
; CHECK-NEXT: br i1 [[TMP11]], label %[[EXIT:.*]], label %[[SCALAR_PH:.*]]
@@ -819,8 +819,7 @@ define void @combined_exit_condition_has_two_users(ptr align 4 dereferenceable(8
; CHECK-NEXT: [[TMP1:%.*]] = icmp ne <4 x i32> [[WIDE_LOAD]], zeroinitializer
; CHECK-NEXT: [[TMP2:%.*]] = freeze <4 x i1> [[TMP1]]
; CHECK-NEXT: [[TMP3:%.*]] = freeze <4 x i1> [[TMP2]]
-; CHECK-NEXT: [[TMP4:%.*]] = call i64 @llvm.experimental.cttz.elts.i64.v4i1(<4 x i1> [[TMP3]], i1 false)
-; CHECK-NEXT: [[UNCOUNTABLE_EXIT_MASK:%.*]] = call <4 x i1> @llvm.get.active.lane.mask.v4i1.i64(i64 0, i64 [[TMP4]])
+; CHECK-NEXT: [[UNCOUNTABLE_EXIT_MASK:%.*]] = call <4 x i1> @llvm.mask.beforefirst.v4i1(<4 x i1> [[TMP3]])
; CHECK-NEXT: [[TMP5:%.*]] = add nuw nsw <4 x i64> [[VEC_IND]], splat (i64 1)
; CHECK-NEXT: [[TMP6:%.*]] = icmp eq <4 x i64> [[TMP5]], splat (i64 20)
; CHECK-NEXT: [[TMP7:%.*]] = select <4 x i1> [[TMP1]], <4 x i1> splat (i1 true), <4 x i1> [[TMP6]]
@@ -834,6 +833,7 @@ define void @combined_exit_condition_has_two_users(ptr align 4 dereferenceable(8
; CHECK-NEXT: [[TMP12:%.*]] = or i1 [[TMP10]], [[TMP11]]
; CHECK-NEXT: br i1 [[TMP12]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP8:![0-9]+]]
; CHECK: [[MIDDLE_BLOCK]]:
+; CHECK-NEXT: [[TMP4:%.*]] = call i64 @llvm.experimental.cttz.elts.i64.v4i1(<4 x i1> [[TMP3]], i1 false)
; CHECK-NEXT: [[TMP13:%.*]] = add i64 [[INDEX]], [[TMP4]]
; CHECK-NEXT: [[TMP14:%.*]] = icmp eq i64 [[TMP13]], 20
; CHECK-NEXT: br i1 [[TMP14]], label %[[EXIT:.*]], label %[[SCALAR_PH:.*]]
diff --git a/llvm/test/Transforms/LoopVectorize/early_exit_combined_exits_epilogue.ll b/llvm/test/Transforms/LoopVectorize/early_exit_combined_exits_epilogue.ll
index c7d7e4637da1f6..be4afea9ab902d 100644
--- a/llvm/test/Transforms/LoopVectorize/early_exit_combined_exits_epilogue.ll
+++ b/llvm/test/Transforms/LoopVectorize/early_exit_combined_exits_epilogue.ll
@@ -15,8 +15,7 @@ define void @combined_exit_conditions(ptr align 4 dereferenceable(80) readonly %
; CHECK-NEXT: [[TMP1:%.*]] = icmp ne <8 x i32> [[WIDE_LOAD]], zeroinitializer
; CHECK-NEXT: [[TMP2:%.*]] = freeze <8 x i1> [[TMP1]]
; CHECK-NEXT: [[TMP7:%.*]] = freeze <8 x i1> [[TMP2]]
-; CHECK-NEXT: [[TMP3:%.*]] = call i64 @llvm.experimental.cttz.elts.i64.v8i1(<8 x i1> [[TMP7]], i1 false)
-; CHECK-NEXT: [[UNCOUNTABLE_EXIT_MASK:%.*]] = call <8 x i1> @llvm.get.active.lane.mask.v8i1.i64(i64 0, i64 [[TMP3]])
+; CHECK-NEXT: [[UNCOUNTABLE_EXIT_MASK:%.*]] = call <8 x i1> @llvm.mask.beforefirst.v8i1(<8 x i1> [[TMP7]])
; CHECK-NEXT: [[TMP4:%.*]] = getelementptr [4 x i8], ptr [[SRC]], i64 [[INDEX]]
; CHECK-NEXT: [[WIDE_MASKED_LOAD:%.*]] = call <8 x i32> @llvm.masked.load.v8i32.p0(ptr align 4 [[TMP4]], <8 x i1> [[UNCOUNTABLE_EXIT_MASK]], <8 x i32> poison)
; CHECK-NEXT: [[TMP5:%.*]] = add nsw <8 x i32> [[WIDE_MASKED_LOAD]], splat (i32 1)
@@ -28,6 +27,7 @@ define void @combined_exit_conditions(ptr align 4 dereferenceable(80) readonly %
; CHECK-NEXT: [[TMP10:%.*]] = or i1 [[TMP8]], [[TMP9]]
; CHECK-NEXT: br i1 [[TMP10]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP0:![0-9]+]]
; CHECK: [[MIDDLE_BLOCK]]:
+; CHECK-NEXT: [[TMP3:%.*]] = call i64 @llvm.experimental.cttz.elts.i64.v8i1(<8 x i1> [[TMP7]], i1 false)
; CHECK-NEXT: [[TMP11:%.*]] = add i64 [[INDEX]], [[TMP3]]
; CHECK-NEXT: [[TMP12:%.*]] = icmp eq i64 [[TMP11]], 20
; CHECK-NEXT: br i1 [[TMP12]], label %[[EXIT:.*]], label %[[SCALAR_PH:.*]]
diff --git a/llvm/test/Transforms/LoopVectorize/early_exit_with_stores.ll b/llvm/test/Transforms/LoopVectorize/early_exit_with_stores.ll
index 87716393117700..ad1269aa3c8523 100644
--- a/llvm/test/Transforms/LoopVectorize/early_exit_with_stores.ll
+++ b/llvm/test/Transforms/LoopVectorize/early_exit_with_stores.ll
@@ -60,8 +60,7 @@ define void @loop_contains_store_condition_load_has_single_user(ptr dereferencea
; CHECK-NEXT: [[WIDE_LOAD:%.*]] = load <4 x i16>, ptr [[TMP12]], align 2
; CHECK-NEXT: [[TMP13:%.*]] = icmp sgt <4 x i16> [[WIDE_LOAD]], splat (i16 500)
; CHECK-NEXT: [[TMP40:%.*]] = freeze <4 x i1> [[TMP13]]
-; CHECK-NEXT: [[TMP14:%.*]] = call i64 @llvm.experimental.cttz.elts.i64.v4i1(<4 x i1> [[TMP40]], i1 false)
-; CHECK-NEXT: [[UNCOUNTABLE_EXIT_MASK:%.*]] = call <4 x i1> @llvm.get.active.lane.mask.v4i1.i64(i64 0, i64 [[TMP14]])
+; CHECK-NEXT: [[UNCOUNTABLE_EXIT_MASK:%.*]] = call <4 x i1> @llvm.mask.beforefirst.v4i1(<4 x i1> [[TMP40]])
; CHECK-NEXT: [[TMP7:%.*]] = getelementptr i16, ptr [[ARRAY]], i64 [[TMP3]]
; CHECK-NEXT: [[TMP30:%.*]] = call <4 x i16> @llvm.masked.load.v4i16.p0(ptr align 2 [[TMP7]], <4 x i1> [[UNCOUNTABLE_EXIT_MASK]], <4 x i16> poison)
; CHECK-NEXT: [[TMP31:%.*]] = add nsw <4 x i16> [[TMP30]], splat (i16 1)
@@ -72,6 +71,7 @@ define void @loop_contains_store_condition_load_has_single_user(ptr dereferencea
; CHECK-NEXT: [[TMP43:%.*]] = or i1 [[TMP41]], [[TMP42]]
; CHECK-NEXT: br i1 [[TMP43]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP0:![0-9]+]]
; CHECK: [[MIDDLE_BLOCK]]:
+; CHECK-NEXT: [[TMP14:%.*]] = call i64 @llvm.experimental.cttz.elts.i64.v4i1(<4 x i1> [[TMP40]], i1 false)
; CHECK-NEXT: [[TMP44:%.*]] = add i64 [[TMP3]], [[TMP14]]
; CHECK-NEXT: [[TMP45:%.*]] = icmp eq i64 [[TMP44]], 20
; CHECK-NEXT: br i1 [[TMP45]], label %[[EXIT:.*]], label %[[SCALAR_PH1:.*]]
@@ -130,8 +130,7 @@ define void @loop_contains_store_fcmp_condition(ptr dereferenceable(40) noalias
; CHECK-NEXT: [[WIDE_LOAD:%.*]] = load <4 x half>, ptr [[TMP0]], align 2
; CHECK-NEXT: [[TMP1:%.*]] = fcmp ugt <4 x half> [[WIDE_LOAD]], splat (half 5.000000e+02)
; CHECK-NEXT: [[TMP2:%.*]] = freeze <4 x i1> [[TMP1]]
-; CHECK-NEXT: [[TMP3:%.*]] = call i64 @llvm.experimental.cttz.elts.i64.v4i1(<4 x i1> [[TMP2]], i1 false)
-; CHECK-NEXT: [[UNCOUNTABLE_EXIT_MASK:%.*]] = call <4 x i1> @llvm.get.active.lane.mask.v4i1.i64(i64 0, i64 [[TMP3]])
+; CHECK-NEXT: [[UNCOUNTABLE_EXIT_MASK:%.*]] = call <4 x i1> @llvm.mask.beforefirst.v4i1(<4 x i1> [[TMP2]])
; CHECK-NEXT: [[TMP4:%.*]] = getelementptr i16, ptr [[ARRAY]], i64 [[INDEX]]
; CHECK-NEXT: [[WIDE_MASKED_LOAD:%.*]] = call <4 x i16> @llvm.masked.load.v4i16.p0(ptr align 2 [[TMP4]], <4 x i1> [[UNCOUNTABLE_EXIT_MASK]], <4 x i16> poison)
; CHECK-NEXT: [[TMP5:%.*]] = add nsw <4 x i16> [[WIDE_MASKED_LOAD]], splat (i16 1)
@@ -142,6 +141,7 @@ define void @loop_contains_store_fcmp_condition(ptr dereferenceable(40) noalias
; CHECK-NEXT: [[TMP8:%.*]] = or i1 [[TMP6]], [[TMP7]]
; CHECK-NEXT: br i1 [[TMP8]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP4:![0-9]+]]
; CHECK: [[MIDDLE_BLOCK]]:
+; CHECK-NEXT: [[TMP3:%.*]] = call i64 @llvm.experimental.cttz.elts.i64.v4i1(<4 x i1> [[TMP2]], i1 false)
; CHECK-NEXT: [[TMP9:%.*]] = add i64 [[INDEX]], [[TMP3]]
; CHECK-NEXT: [[TMP10:%.*]] = icmp eq i64 [[TMP9]], 20
; CHECK-NEXT: br i1 [[TMP10]], label %[[EXIT:.*]], label %[[SCALAR_PH:.*]]
@@ -357,8 +357,7 @@ define void @loop_contains_store_to_pointer_with_no_deref_info(ptr align 2 deref
; CHECK-NEXT: [[WIDE_LOAD:%.*]] = load <4 x i16>, ptr [[TMP4]], align 2
; CHECK-NEXT: [[TMP5:%.*]] = icmp sgt <4 x i16> [[WIDE_LOAD]], splat (i16 500)
; CHECK-NEXT: [[TMP40:%.*]] = freeze <4 x i1> [[TMP5]]
-; CHECK-NEXT: [[TMP6:%.*]] = call i64 @llvm.experimental.cttz.elts.i64.v4i1(<4 x i1> [[TMP40]], i1 false)
-; CHECK-NEXT: [[UNCOUNTABLE_EXIT_MASK:%.*]] = call <4 x i1> @llvm.get.active.lane.mask.v4i1.i64(i64 0, i64 [[TMP6]])
+; CHECK-NEXT: [[UNCOUNTABLE_EXIT_MASK:%.*]] = call <4 x i1> @llvm.mask.beforefirst.v4i1(<4 x i1> [[TMP40]])
; CHECK-NEXT: [[TMP0:%.*]] = getelementptr i16, ptr [[LOAD_ARRAY]], i64 [[INDEX]]
; CHECK-NEXT: [[TMP26:%.*]] = call <4 x i16> @llvm.masked.load.v4i16.p0(ptr align 2 [[TMP0]], <4 x i1> [[UNCOUNTABLE_EXIT_MASK]], <4 x i16> poison)
; CHECK-NEXT: [[TMP27:%.*]] = add nsw <4 x i16> [[TMP26]], splat (i16 1)
@@ -370,6 +369,7 @@ define void @loop_contains_store_to_pointer_with_no_deref_info(ptr align 2 deref
; CHECK-NEXT: [[TMP43:%.*]] = or i1 [[TMP41]], [[TMP42]]
; CHECK-NEXT: br i1 [[TMP43]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP6:![0-9]+]]
; CHECK: [[MIDDLE_BLOCK]]:
+; CHECK-NEXT: [[TMP6:%.*]] = call i64 @llvm.experimental.cttz.elts.i64.v4i1(<4 x i1> [[TMP40]], i1 false)
; CHECK-NEXT: [[TMP44:%.*]] = add i64 [[INDEX]], [[TMP6]]
; CHECK-NEXT: [[TMP45:%.*]] = icmp eq i64 [[TMP44]], 20
; CHECK-NEXT: br i1 [[TMP45]], label %[[EXIT:.*]], label %[[SCALAR_PH1:.*]]
@@ -475,8 +475,7 @@ define void @loop_contains_store_in_latch_block(ptr dereferenceable(40) noalias
; CHECK-NEXT: [[WIDE_LOAD:%.*]] = load <4 x i16>, ptr [[EE_ADDR]], align 2
; CHECK-NEXT: [[TMP5:%.*]] = icmp sgt <4 x i16> [[WIDE_LOAD]], splat (i16 500)
; CHECK-NEXT: [[TMP40:%.*]] = freeze <4 x i1> [[TMP5]]
-; CHECK-NEXT: [[TMP6:%.*]] = call i64 @llvm.experimental.cttz.elts.i64.v4i1(<4 x i1> [[TMP40]], i1 false)
-; CHECK-NEXT: [[UNCOUNTABLE_EXIT_MASK:%.*]] = call <4 x i1> @llvm.get.active.lane.mask.v4i1.i64(i64 0, i64 [[TMP6]])
+; CHECK-NEXT: [[UNCOUNTABLE_EXIT_MASK:%.*]] = call <4 x i1> @llvm.mask.beforefirst.v4i1(<4 x i1> [[TMP40]])
; CHECK-NEXT: [[TMP3:%.*]] = getelementptr i16, ptr [[ARRAY]], i64 [[INDEX]]
; CHECK-NEXT: [[TMP30:%.*]] = call <4 x i16> @llvm.masked.load.v4i16.p0(ptr align 2 [[TMP3]], <4 x i1> [[UNCOUNTABLE_EXIT_MASK]], <4 x i16> poison)
; CHECK-NEXT: [[TMP31:%.*]] = add nsw <4 x i16> [[TMP30]], splat (i16 1)
@@ -487,6 +486,7 @@ define void @loop_contains_store_in_latch_block(ptr dereferenceable(40) noalias
; CHECK-NEXT: [[TMP43:%.*]] = or i1 [[TMP41]], [[TMP42]]
; CHECK-NEXT: br i1 [[TMP43]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP8:![0-9]+]]
; CHECK: [[MIDDLE_BLOCK]]:
+; CHECK-NEXT: [[TMP6:%.*]] = call i64 @llvm.experimental.cttz.elts.i64.v4i1(<4 x i1> [[TMP40]], i1 false)
; CHECK-NEXT: [[TMP44:%.*]] = add i64 [[INDEX]], [[TMP6]]
; CHECK-NEXT: [[TMP45:%.*]] = icmp eq i64 [[TMP44]], 20
; CHECK-NEXT: br i1 [[TMP45]], label %[[EXIT:.*]], label %[[SCALAR_PH1:.*]]
@@ -1051,8 +1051,7 @@ define i16 @uncountable_exit_with_live_out(ptr dereferenceable(40) noalias %arra
; CHECK-NEXT: [[WIDE_LOAD:%.*]] = load <4 x i16>, ptr [[TMP12]], align 2
; CHECK-NEXT: [[TMP13:%.*]] = icmp sgt <4 x i16> [[WIDE_LOAD]], splat (i16 500)
; CHECK-NEXT: [[TMP40:%.*]] = freeze <4 x i1> [[TMP13]]
-; CHECK-NEXT: [[TMP14:%.*]] = call i64 @llvm.experimental.cttz.elts.i64.v4i1(<4 x i1> [[TMP40]], i1 false)
-; CHECK-NEXT: [[UNCOUNTABLE_EXIT_MASK:%.*]] = call <4 x i1> @llvm.get.active.lane.mask.v4i1.i64(i64 0, i64 [[TMP14]])
+; CHECK-NEXT: [[UNCOUNTABLE_EXIT_MASK:%.*]] = call <4 x i1> @llvm.mask.beforefirst.v4i1(<4 x i1> [[TMP40]])
; CHECK-NEXT: [[TMP7:%.*]] = getelementptr i16, ptr [[ARRAY]], i64 [[TMP3]]
; CHECK-NEXT: [[TMP30:%.*]] = call <4 x i16> @llvm.masked.load.v4i16.p0(ptr align 2 [[TMP7]], <4 x i1> [[UNCOUNTABLE_EXIT_MASK]], <4 x i16> poison)
; CHECK-NEXT: [[TMP31:%.*]] = add nsw <4 x i16> [[TMP30]], splat (i16 1)
@@ -1064,6 +1063,7 @@ define i16 @uncountable_exit_with_live_out(ptr dereferenceable(40) noalias %arra
; CHECK-NEXT: br i1 [[TMP43]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP10:![0-9]+]]
; CHECK: [[MIDDLE_BLOCK]]:
; CHECK-NEXT: [[TMP9:%.*]] = extractelement <4 x i16> [[TMP30]], i64 3
+; CHECK-NEXT: [[TMP14:%.*]] = call i64 @llvm.experimental.cttz.elts.i64.v4i1(<4 x i1> [[TMP40]], i1 false)
; CHECK-NEXT: [[TMP45:%.*]] = add i64 [[TMP3]], [[TMP14]]
; CHECK-NEXT: [[TMP46:%.*]] = icmp eq i64 [[TMP45]], 20
; CHECK-NEXT: br i1 [[TMP46]], label %[[EXIT:.*]], label %[[SCALAR_PH:.*]]
@@ -1123,8 +1123,7 @@ define i64 @uncountable_exit_with_live_out_iv(ptr dereferenceable(40) noalias %a
; CHECK-NEXT: [[WIDE_LOAD:%.*]] = load <4 x i16>, ptr [[TMP0]], align 2
; CHECK-NEXT: [[TMP1:%.*]] = icmp sgt <4 x i16> [[WIDE_LOAD]], splat (i16 500)
; CHECK-NEXT: [[TMP5:%.*]] = freeze <4 x i1> [[TMP1]]
-; CHECK-NEXT: [[TMP2:%.*]] = call i64 @llvm.experimental.cttz.elts.i64.v4i1(<4 x i1> [[TMP5]], i1 false)
-; CHECK-NEXT: [[UNCOUNTABLE_EXIT_MASK:%.*]] = call <4 x i1> @llvm.get.active.lane.mask.v4i1.i64(i64 0, i64 [[TMP2]])
+; CHECK-NEXT: [[UNCOUNTABLE_EXIT_MASK:%.*]] = call <4 x i1> @llvm.mask.beforefirst.v4i1(<4 x i1> [[TMP5]])
; CHECK-NEXT: [[TMP3:%.*]] = getelementptr i16, ptr [[ARRAY]], i64 [[INDEX]]
; CHECK-NEXT: [[WIDE_MASKED_LOAD:%.*]] = call <4 x i16> @llvm.masked.load.v4i16.p0(ptr align 2 [[TMP3]], <4 x i1> [[UNCOUNTABLE_EXIT_MASK]], <4 x i16> poison)
; CHECK-NEXT: [[TMP4:%.*]] = add nsw <4 x i16> [[WIDE_MASKED_LOAD]], splat (i16 1)
@@ -1135,6 +1134,7 @@ define i64 @uncountable_exit_with_live_out_iv(ptr dereferenceable(40) noalias %a
; CHECK-NEXT: [[TMP8:%.*]] = or i1 [[TMP6]], [[TMP7]]
; CHECK-NEXT: br i1 [[TMP8]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP12:![0-9]+]]
; CHECK: [[MIDDLE_BLOCK]]:
+; CHECK-NEXT: [[TMP2:%.*]] = call i64 @llvm.experimental.cttz.elts.i64.v4i1(<4 x i1> [[TMP5]], i1 false)
; CHECK-NEXT: [[TMP9:%.*]] = add i64 [[INDEX]], [[TMP2]]
; CHECK-NEXT: [[TMP10:%.*]] = icmp eq i64 [[TMP9]], 20
; CHECK-NEXT: br i1 [[TMP10]], label %[[EXIT:.*]], label %[[SCALAR_PH:.*]]
@@ -1284,8 +1284,7 @@ define i32 @uncountable_exit_with_separate_exit_block(ptr dereferenceable(40) no
; CHECK-NEXT: [[WIDE_LOAD:%.*]] = load <4 x i16>, ptr [[TMP12]], align 2
; CHECK-NEXT: [[TMP13:%.*]] = icmp sgt <4 x i16> [[WIDE_LOAD]], splat (i16 500)
; CHECK-NEXT: [[TMP40:%.*]] = freeze <4 x i1> [[TMP13]]
-; CHECK-NEXT: [[TMP14:%.*]] = call i64 @llvm.experimental.cttz.elts.i64.v4i1(<4 x i1> [[TMP40]], i1 false)
-; CHECK-NEXT: [[UNCOUNTABLE_EXIT_MASK:%.*]] = call <4 x i1> @llvm.get.active.lane.mask.v4i1.i64(i64 0, i64 [[TMP14]])
+; CHECK-NEXT: [[UNCOUNTABLE_EXIT_MASK:%.*]] = call <4 x i1> @llvm.mask.beforefirst.v4i1(<4 x i1> [[TMP40]])
; CHECK-NEXT: [[TMP7:%.*]] = getelementptr i16, ptr [[ARRAY]], i64 [[TMP3]]
; CHECK-NEXT: [[TMP30:%.*]] = call <4 x i16> @llvm.masked.load.v4i16.p0(ptr align 2 [[TMP7]], <4 x i1> [[UNCOUNTABLE_EXIT_MASK]], <4 x i16> poison)
; CHECK-NEXT: [[TMP31:%.*]] = add nsw <4 x i16> [[TMP30]], splat (i16 1)
@@ -1296,6 +1295,7 @@ define i32 @uncountable_exit_with_separate_exit_block(ptr dereferenceable(40) no
; CHECK-NEXT: [[TMP43:%.*]] = or i1 [[TMP41]], [[TMP42]]
; CHECK-NEXT: br i1 [[TMP43]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP14:![0-9]+]]
; CHECK: [[MIDDLE_BLOCK]]:
+; CHECK-NEXT: [[TMP14:%.*]] = call i64 @llvm.experimental.cttz.elts.i64.v4i1(<4 x i1> [[TMP40]], i1 false)
; CHECK-NEXT: [[TMP44:%.*]] = add i64 [[TMP3]], [[TMP14]]
; CHECK-NEXT: [[TMP45:%.*]] = icmp eq i64 [[TMP44]], 20
; CHECK-NEXT: br i1 [[TMP45]], label %[[EXIT_COUNTABLE:.*]], label %[[SCALAR_PH:.*]]
@@ -1427,8 +1427,7 @@ define void @swapped_cmp_operands(ptr dereferenceable(40) noalias %array, ptr al
; CHECK-NEXT: [[WIDE_LOAD:%.*]] = load <4 x i16>, ptr [[TMP1]], align 2
; CHECK-NEXT: [[TMP2:%.*]] = icmp slt <4 x i16> splat (i16 500), [[WIDE_LOAD]]
; CHECK-NEXT: [[TMP5:%.*]] = freeze <4 x i1> [[TMP2]]
-; CHECK-NEXT: [[TMP3:%.*]] = call i64 @llvm.experimental.cttz.elts.i64.v4i1(<4 x i1> [[TMP5]], i1 false)
-; CHECK-NEXT: [[UNCOUNTABLE_EXIT_MASK:%.*]] = call <4 x i1> @llvm.get.active.lane.mask.v4i1.i64(i64 0, i64 [[TMP3]])
+; CHECK-NEXT: [[UNCOUNTABLE_EXIT_MASK:%.*]] = call <4 x i1> @llvm.mask.beforefirst.v4i1(<4 x i1> [[TMP5]])
; CHECK-NEXT: [[TMP0:%.*]] = getelementptr i16, ptr [[ARRAY]], i64 [[INDEX]]
; CHECK-NEXT: [[WIDE_MASKED_LOAD:%.*]] = call <4 x i16> @llvm.masked.load.v4i16.p0(ptr align 2 [[TMP0]], <4 x i1> [[UNCOUNTABLE_EXIT_MASK]], <4 x i16> poison)
; CHECK-NEXT: [[TMP4:%.*]] = add nsw <4 x i16> [[WIDE_MASKED_LOAD]], splat (i16 1)
@@ -1439,6 +1438,7 @@ define void @swapped_cmp_operands(ptr dereferenceable(40) noalias %array, ptr al
; CHECK-NEXT: [[TMP8:%.*]] = or i1 [[TMP6]], [[TMP7]]
; CHECK-NEXT: br i1 [[TMP8]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP16:![0-9]+]]
; CHECK: [[MIDDLE_BLOCK]]:
+; CHECK-NEXT: [[TMP3:%.*]] = call i64 @llvm.experimental.cttz.elts.i64.v4i1(<4 x i1> [[TMP5]], i1 false)
; CHECK-NEXT: [[TMP9:%.*]] = add i64 [[INDEX]], [[TMP3]]
; CHECK-NEXT: [[TMP10:%.*]] = icmp eq i64 [[TMP9]], 20
; CHECK-NEXT: br i1 [[TMP10]], label %[[EXIT:.*]], label %[[SCALAR_PH:.*]]
@@ -1497,8 +1497,7 @@ define void @loop_contains_store_condition_load_has_two_users(ptr dereferenceabl
; CHECK-NEXT: [[WIDE_LOAD:%.*]] = load <4 x i16>, ptr [[TMP0]], align 2
; CHECK-NEXT: [[TMP1:%.*]] = icmp sgt <4 x i16> [[WIDE_LOAD]], splat (i16 500)
; CHECK-NEXT: [[TMP2:%.*]] = freeze <4 x i1> [[TMP1]]
-; CHECK-NEXT: [[TMP3:%.*]] = call i64 @llvm.experimental.cttz.elts.i64.v4i1(<4 x i1> [[TMP2]], i1 false)
-; CHECK-NEXT: [[UNCOUNTABLE_EXIT_MASK:%.*]] = call <4 x i1> @llvm.get.active.lane.mask.v4i1.i64(i64 0, i64 [[TMP3]])
+; CHECK-NEXT: [[UNCOUNTABLE_EXIT_MASK:%.*]] = call <4 x i1> @llvm.mask.beforefirst.v4i1(<4 x i1> [[TMP2]])
; CHECK-NEXT: [[TMP4:%.*]] = getelementptr i16, ptr [[ARRAY]], i64 [[INDEX]]
; CHECK-NEXT: [[WIDE_MASKED_LOAD:%.*]] = call <4 x i16> @llvm.masked.load.v4i16.p0(ptr align 2 [[TMP4]], <4 x i1> [[UNCOUNTABLE_EXIT_MASK]], <4 x i16> poison)
; CHECK-NEXT: [[TMP5:%.*]] = add nsw <4 x i16> [[WIDE_MASKED_LOAD]], [[WIDE_LOAD]]
@@ -1509,6 +1508,7 @@ define void @loop_contains_store_condition_load_has_two_users(ptr dereferenceabl
; CHECK-NEXT: [[TMP8:%.*]] = or i1 [[TMP6]], [[TMP7]]
; CHECK-NEXT: br i1 [[TMP8]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP18:![0-9]+]]
; CHECK: [[MIDDLE_BLOCK]]:
+; CHECK-NEXT: [[TMP3:%.*]] = call i64 @llvm.experimental.cttz.elts.i64.v4i1(<4 x i1> [[TMP2]], i1 false)
; CHECK-NEXT: [[TMP9:%.*]] = add i64 [[INDEX]], [[TMP3]]
; CHECK-NEXT: [[TMP10:%.*]] = icmp eq i64 [[TMP9]], 20
; CHECK-NEXT: br i1 [[TMP10]], label %[[EXIT:.*]], label %[[SCALAR_PH:.*]]
diff --git a/llvm/test/Transforms/LoopVectorize/interleave_uncountable_exits.ll b/llvm/test/Transforms/LoopVectorize/interleave_uncountable_exits.ll
index 8637897644caee..d3dae6a8c9651f 100644
--- a/llvm/test/Transforms/LoopVectorize/interleave_uncountable_exits.ll
+++ b/llvm/test/Transforms/LoopVectorize/interleave_uncountable_exits.ll
@@ -16,8 +16,7 @@ define void @loop_contains_store_condition_load_has_single_user(ptr dereferencea
; CHECK-NEXT: [[WIDE_LOAD:%.*]] = load <4 x i16>, ptr [[TMP1]], align 2
; CHECK-NEXT: [[TMP2:%.*]] = icmp sgt <4 x i16> [[WIDE_LOAD]], splat (i16 500)
; CHECK-NEXT: [[TMP5:%.*]] = freeze <4 x i1> [[TMP2]]
-; CHECK-NEXT: [[TMP3:%.*]] = call i64 @llvm.experimental.cttz.elts.i64.v4i1(<4 x i1> [[TMP5]], i1 false)
-; CHECK-NEXT: [[UNCOUNTABLE_EXIT_MASK:%.*]] = call <4 x i1> @llvm.get.active.lane.mask.v4i1.i64(i64 0, i64 [[TMP3]])
+; CHECK-NEXT: [[UNCOUNTABLE_EXIT_MASK:%.*]] = call <4 x i1> @llvm.mask.beforefirst.v4i1(<4 x i1> [[TMP5]])
; CHECK-NEXT: [[TMP0:%.*]] = getelementptr i16, ptr [[ARRAY]], i64 [[INDEX]]
; CHECK-NEXT: [[WIDE_MASKED_LOAD:%.*]] = call <4 x i16> @llvm.masked.load.v4i16.p0(ptr align 2 [[TMP0]], <4 x i1> [[UNCOUNTABLE_EXIT_MASK]], <4 x i16> poison)
; CHECK-NEXT: [[TMP4:%.*]] = add nsw <4 x i16> [[WIDE_MASKED_LOAD]], splat (i16 1)
@@ -28,6 +27,7 @@ define void @loop_contains_store_condition_load_has_single_user(ptr dereferencea
; CHECK-NEXT: [[TMP8:%.*]] = or i1 [[TMP6]], [[TMP7]]
; CHECK-NEXT: br i1 [[TMP8]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP0:![0-9]+]]
; CHECK: [[MIDDLE_BLOCK]]:
+; CHECK-NEXT: [[TMP3:%.*]] = call i64 @llvm.experimental.cttz.elts.i64.v4i1(<4 x i1> [[TMP5]], i1 false)
; CHECK-NEXT: [[TMP9:%.*]] = add i64 [[INDEX]], [[TMP3]]
; CHECK-NEXT: [[TMP10:%.*]] = icmp eq i64 [[TMP9]], 20
; CHECK-NEXT: br i1 [[TMP10]], label %[[EXIT:.*]], label %[[SCALAR_PH:.*]]
diff --git a/llvm/test/Transforms/LoopVectorize/scalarized_conditional_ops_uncountable_exits.ll b/llvm/test/Transforms/LoopVectorize/scalarized_conditional_ops_uncountable_exits.ll
index 4523ba2d7d3e7b..c9f296894b997c 100644
--- a/llvm/test/Transforms/LoopVectorize/scalarized_conditional_ops_uncountable_exits.ll
+++ b/llvm/test/Transforms/LoopVectorize/scalarized_conditional_ops_uncountable_exits.ll
@@ -17,8 +17,7 @@ define void @loop_contains_store_condition_load_has_single_user(ptr dereferencea
; CHECK-NEXT: [[WIDE_LOAD:%.*]] = load <4 x i16>, ptr [[TMP1]], align 2
; CHECK-NEXT: [[TMP2:%.*]] = icmp sgt <4 x i16> [[WIDE_LOAD]], splat (i16 500)
; CHECK-NEXT: [[TMP5:%.*]] = freeze <4 x i1> [[TMP2]]
-; CHECK-NEXT: [[TMP3:%.*]] = call i64 @llvm.experimental.cttz.elts.i64.v4i1(<4 x i1> [[TMP5]], i1 false)
-; CHECK-NEXT: [[UNCOUNTABLE_EXIT_MASK:%.*]] = call <4 x i1> @llvm.get.active.lane.mask.v4i1.i64(i64 0, i64 [[TMP3]])
+; CHECK-NEXT: [[UNCOUNTABLE_EXIT_MASK:%.*]] = call <4 x i1> @llvm.mask.beforefirst.v4i1(<4 x i1> [[TMP5]])
; CHECK-NEXT: [[TMP31:%.*]] = getelementptr inbounds nuw i16, ptr [[ARRAY]], i64 [[INDEX]]
; CHECK-NEXT: [[TMP32:%.*]] = getelementptr inbounds nuw i16, ptr [[ARRAY]], i64 [[TMP0]]
; CHECK-NEXT: [[TMP33:%.*]] = getelementptr inbounds nuw i16, ptr [[ARRAY]], i64 [[TMP25]]
@@ -86,6 +85,7 @@ define void @loop_contains_store_condition_load_has_single_user(ptr dereferencea
; CHECK-NEXT: [[TMP8:%.*]] = or i1 [[TMP6]], [[TMP7]]
; CHECK-NEXT: br i1 [[TMP8]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP0:![0-9]+]]
; CHECK: [[MIDDLE_BLOCK]]:
+; CHECK-NEXT: [[TMP3:%.*]] = call i64 @llvm.experimental.cttz.elts.i64.v4i1(<4 x i1> [[TMP5]], i1 false)
; CHECK-NEXT: [[TMP9:%.*]] = add i64 [[INDEX]], [[TMP3]]
; CHECK-NEXT: [[TMP10:%.*]] = icmp eq i64 [[TMP9]], 20
; CHECK-NEXT: br i1 [[TMP10]], label %[[EXIT:.*]], label %[[SCALAR_PH:.*]]
diff --git a/llvm/test/Transforms/LoopVectorize/tail_fold_uncountable_exits.ll b/llvm/test/Transforms/LoopVectorize/tail_fold_uncountable_exits.ll
index e4633bcb5568a0..35ea7acbaa6d8b 100644
--- a/llvm/test/Transforms/LoopVectorize/tail_fold_uncountable_exits.ll
+++ b/llvm/test/Transforms/LoopVectorize/tail_fold_uncountable_exits.ll
@@ -19,8 +19,7 @@ define void @loop_contains_store_condition_load_has_single_user(ptr dereferencea
; CHECK-NEXT: [[WIDE_LOAD:%.*]] = load <4 x i16>, ptr [[TMP1]], align 2
; CHECK-NEXT: [[TMP2:%.*]] = icmp sgt <4 x i16> [[WIDE_LOAD]], splat (i16 500)
; CHECK-NEXT: [[TMP5:%.*]] = freeze <4 x i1> [[TMP2]]
-; CHECK-NEXT: [[TMP3:%.*]] = call i64 @llvm.experimental.cttz.elts.i64.v4i1(<4 x i1> [[TMP5]], i1 false)
-; CHECK-NEXT: [[UNCOUNTABLE_EXIT_MASK:%.*]] = call <4 x i1> @llvm.get.active.lane.mask.v4i1.i64(i64 0, i64 [[TMP3]])
+; CHECK-NEXT: [[UNCOUNTABLE_EXIT_MASK:%.*]] = call <4 x i1> @llvm.mask.beforefirst.v4i1(<4 x i1> [[TMP5]])
; CHECK-NEXT: [[TMP0:%.*]] = getelementptr i16, ptr [[ARRAY]], i64 [[INDEX]]
; CHECK-NEXT: [[WIDE_MASKED_LOAD:%.*]] = call <4 x i16> @llvm.masked.load.v4i16.p0(ptr align 2 [[TMP0]], <4 x i1> [[UNCOUNTABLE_EXIT_MASK]], <4 x i16> poison)
; CHECK-NEXT: [[TMP4:%.*]] = add nsw <4 x i16> [[WIDE_MASKED_LOAD]], splat (i16 1)
@@ -31,6 +30,7 @@ define void @loop_contains_store_condition_load_has_single_user(ptr dereferencea
; CHECK-NEXT: [[TMP8:%.*]] = or i1 [[TMP6]], [[TMP7]]
; CHECK-NEXT: br i1 [[TMP8]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP0:![0-9]+]]
; CHECK: [[MIDDLE_BLOCK]]:
+; CHECK-NEXT: [[TMP3:%.*]] = call i64 @llvm.experimental.cttz.elts.i64.v4i1(<4 x i1> [[TMP5]], i1 false)
; CHECK-NEXT: [[TMP9:%.*]] = add i64 [[INDEX]], [[TMP3]]
; CHECK-NEXT: [[TMP10:%.*]] = icmp eq i64 [[TMP9]], 20
; CHECK-NEXT: br i1 [[TMP10]], label %[[EXIT:.*]], label %[[SCALAR_PH:.*]]
diff --git a/llvm/test/Transforms/PhaseOrdering/AArch64/countable-and-uncountable-exits-combined.ll b/llvm/test/Transforms/PhaseOrdering/AArch64/countable-and-uncountable-exits-combined.ll
index e7ed25c7954d72..9d5804468f12b7 100644
--- a/llvm/test/Transforms/PhaseOrdering/AArch64/countable-and-uncountable-exits-combined.ll
+++ b/llvm/test/Transforms/PhaseOrdering/AArch64/countable-and-uncountable-exits-combined.ll
@@ -40,8 +40,7 @@ define void @foo() #0 {
; CHECK-NEXT: [[WIDE_LOAD:%.*]] = load <vscale x 4 x i32>, ptr [[TMP2]], align 4
; CHECK-NEXT: [[WIDE_LOAD_FR:%.*]] = freeze <vscale x 4 x i32> [[WIDE_LOAD]]
; CHECK-NEXT: [[TMP3:%.*]] = icmp ne <vscale x 4 x i32> [[WIDE_LOAD_FR]], zeroinitializer
-; CHECK-NEXT: [[TMP4:%.*]] = tail call i64 @llvm.experimental.cttz.elts.i64.nxv4i1(<vscale x 4 x i1> [[TMP3]], i1 false)
-; CHECK-NEXT: [[UNCOUNTABLE_EXIT_MASK:%.*]] = tail call <vscale x 4 x i1> @llvm.get.active.lane.mask.nxv4i1.i64(i64 0, i64 [[TMP4]])
+; CHECK-NEXT: [[UNCOUNTABLE_EXIT_MASK:%.*]] = tail call <vscale x 4 x i1> @llvm.mask.beforefirst.nxv4i1(<vscale x 4 x i1> [[TMP3]])
; CHECK-NEXT: [[TMP5:%.*]] = getelementptr [4 x i8], ptr @src, i64 [[INDEX]]
; CHECK-NEXT: [[WIDE_MASKED_LOAD:%.*]] = tail call <vscale x 4 x i32> @llvm.masked.load.nxv4i32.p0(ptr align 4 [[TMP5]], <vscale x 4 x i1> [[UNCOUNTABLE_EXIT_MASK]], <vscale x 4 x i32> poison)
; CHECK-NEXT: [[TMP6:%.*]] = add nsw <vscale x 4 x i32> [[WIDE_MASKED_LOAD]], splat (i32 42)
@@ -53,6 +52,7 @@ define void @foo() #0 {
; CHECK-NEXT: [[TMP10:%.*]] = or i1 [[TMP8]], [[TMP9]]
; CHECK-NEXT: br i1 [[TMP10]], label %[[MIDDLE_BLOCK:.*]], label %[[FOR_BODY]], !llvm.loop [[LOOP0:![0-9]+]]
; CHECK: [[MIDDLE_BLOCK]]:
+; CHECK-NEXT: [[TMP4:%.*]] = tail call i64 @llvm.experimental.cttz.elts.i64.nxv4i1(<vscale x 4 x i1> [[TMP3]], i1 false)
; CHECK-NEXT: [[TMP11:%.*]] = add i64 [[INDEX]], [[TMP4]]
; CHECK-NEXT: [[TMP12:%.*]] = icmp eq i64 [[TMP11]], 10000
; CHECK-NEXT: br i1 [[TMP12]], label %[[EXIT:.*]], label %[[FOR_BODY_PREHEADER]]
More information about the llvm-branch-commits
mailing list