[llvm-branch-commits] [llvm] [VPlan] Skip branch term in masks for some preserved uniform edges (PR #218552)
Andrei Elovikov via llvm-branch-commits
llvm-branch-commits at lists.llvm.org
Thu Sep 3 14:03:38 PDT 2026
https://github.com/eas updated https://github.com/llvm/llvm-project/pull/218552
>From 63273fb25d0dd39b5bb477a01ea1c20e95cf8d9a Mon Sep 17 00:00:00 2001
From: Andrei Elovikov <andrei.elovikov at sifive.com>
Date: Tue, 1 Sep 2026 15:36:45 -0700
Subject: [PATCH] [VPlan] Skip branch term in masks for some preserved uniform
edges
---
.../Transforms/Vectorize/LoopVectorize.cpp | 6 +-
.../Transforms/Vectorize/VPlanPredicator.cpp | 51 ++-
.../Transforms/Vectorize/VPlanTransforms.cpp | 3 +-
.../AArch64/invariant-replicate-region.ll | 49 +-
.../AArch64/predication_costs.ll | 38 +-
.../AArch64/sve-predicated-costs.ll | 5 +-
...-interleave-to-widen-memory-multi-block.ll | 36 +-
.../LoopVectorize/RISCV/dead-ops-cost.ll | 55 +--
.../Transforms/LoopVectorize/RISCV/divrem.ll | 30 +-
.../LoopVectorize/RISCV/predicated-costs.ll | 10 +-
.../RISCV/predicated-reverse-store.ll | 7 +-
.../RISCV/tail-folding-complex-mask.ll | 16 +-
.../VPlan/dissolve-replicate-regions.ll | 101 +---
.../VPlan/phi-with-fastflags-vplan.ll | 2 +-
.../LoopVectorize/VPlan/predicator.ll | 54 +--
.../VPlan/vplan-sink-scalars-and-merge-vf1.ll | 18 +-
.../VPlan/vplan-sink-scalars-and-merge.ll | 28 +-
.../X86/cost-conditional-branches.ll | 431 +-----------------
.../LoopVectorize/X86/masked-store-cost.ll | 4 +-
.../X86/pr141968-instsimplifyfolder.ll | 5 +-
.../X86/predicated-instruction-cost.ll | 43 +-
.../X86/predicated-replicate-feeding-cast.ll | 99 +---
.../constant-fold-commutative-and.ll | 8 +-
.../Transforms/LoopVectorize/div-exact.ll | 132 +-----
.../LoopVectorize/if-pred-stores.ll | 165 ++-----
.../multiple-result-intrinsics.ll | 10 -
.../LoopVectorize/phi-with-fastflags.ll | 4 +-
llvm/test/Transforms/LoopVectorize/pr37248.ll | 10 -
llvm/test/Transforms/LoopVectorize/pr45525.ll | 5 +-
.../Transforms/LoopVectorize/predicator.ll | 8 +-
.../LoopVectorize/scalarize-masked-call.ll | 22 +-
.../LoopVectorize/struct-return-replicate.ll | 66 +--
.../Transforms/LoopVectorize/uniform-blend.ll | 4 +-
.../PhaseOrdering/loop-vectorize-bfi.ll | 7 +-
34 files changed, 318 insertions(+), 1214 deletions(-)
diff --git a/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp b/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
index d85ab84e44bdf..18ed904b40f3f 100644
--- a/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
+++ b/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
@@ -6251,8 +6251,10 @@ VPRecipeWithIRFlags *VPRecipeBuilder::tryToWiden(VPInstruction *VPI) {
case Instruction::UDiv:
case Instruction::SRem:
case Instruction::URem:
- // If not provably safe, use a masked intrinsic.
- if (CM.isPredicatedInst(I))
+ // If not provably safe, use a masked intrinsic. Predicator might have
+ // preserved uniform control flow making this unmasked even if CM expected
+ // it to be masked.
+ if (CM.isPredicatedInst(I) && VPI->isMasked())
return new VPWidenIntrinsicRecipe(
getMaskedDivRemIntrinsic(VPI->getOpcode()), VPI->operands(),
I->getType(), {}, {}, VPI->getDebugLoc());
diff --git a/llvm/lib/Transforms/Vectorize/VPlanPredicator.cpp b/llvm/lib/Transforms/Vectorize/VPlanPredicator.cpp
index da9b2c42984e2..9899b4cd5ee81 100644
--- a/llvm/lib/Transforms/Vectorize/VPlanPredicator.cpp
+++ b/llvm/lib/Transforms/Vectorize/VPlanPredicator.cpp
@@ -376,9 +376,54 @@ void VPPredicator::createBlockInMask(VPBasicBlock *VPBB) {
// This is the block mask. We OR all unique incoming edges.
for (auto *Predecessor : SetVector<VPBlockBase *>(
VPBB->getPredecessors().begin(), VPBB->getPredecessors().end())) {
- VPValue *EdgeMask = createEdgeMask(cast<VPBasicBlock>(Predecessor), VPBB);
- if (!EdgeMask) { // Mask of predecessor is all-one so mask of block is
- // too.
+ auto *Pred = cast<VPBasicBlock>(Predecessor);
+ bool CanUseBlockMaskOnly = [&] {
+ if (!shouldPreserveTerminator(Pred))
+ return false;
+
+ LLVM_DEBUG(
+ dbgs()
+ << "Checking if can skip branch condition for a uniform branch at "
+ << Pred->getName() << "\n");
+
+ auto False = []([[maybe_unused]] StringRef Reason = "") {
+ LLVM_DEBUG(dbgs() << " can't skip"
+ << (Reason.empty() ? Twine() : (": " + Reason))
+ << "\n");
+ return false;
+ };
+
+ // Detect patterns like
+ //
+ // BB (uniform branch)
+ // / \
+ // / +-------------------------+
+ // | | no other incoming edges |
+ // | +-------------------------+
+ // \ / /
+ // \ / /
+ // PostDomBB <-- <possible other edges originated before BB
+ //
+ // where one of those uniform edges is a "bypass". We can use BB's mask
+ // instead of the edge mask for the masks inside the "box" region above.
+ auto *IPostDomNode = VPPDT.getNode(Pred)->getIDom();
+ if (!IPostDomNode)
+ return False("No post-dom");
+ auto *IPostDom = dyn_cast<VPBasicBlock>(IPostDomNode->getBlock());
+ LLVM_DEBUG(dbgs() << " post-dom: " << IPostDom->getName() << "\n");
+ if (!IPostDom || !VPDT.properlyDominates(Pred, IPostDom))
+ return False("doesn't dominate its post-dom");
+
+ if (!is_contained(Pred->getSuccessors(), IPostDom))
+ return False("not a bypass branch");
+
+ LLVM_DEBUG(dbgs() << " can skip\n");
+ return true;
+ }();
+ VPValue *EdgeMask =
+ CanUseBlockMaskOnly ? getBlockInMask(Pred) : createEdgeMask(Pred, VPBB);
+ if (!EdgeMask) {
+ // Mask of predecessor is all-one so mask of block is too.
setBlockInMask(VPBB, EdgeMask);
return;
}
diff --git a/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp b/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp
index b8311f5a77493..f5f8555cb3470 100644
--- a/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp
+++ b/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp
@@ -5433,7 +5433,8 @@ void VPlanTransforms::makeMemOpWideningDecisions(VPlan &Plan, VFRange &Range,
// A predicated access can only be widened (rather than scalarized) if
// the target supports a masked load/store for it.
// TODO: Determine if a load/store needs predication directly in VPlan.
- bool IsPredicated = RecipeBuilder.isPredicatedInst(I);
+ bool IsPredicated =
+ RecipeBuilder.isPredicatedInst(I) && VPI->isMasked();
if (IsPredicated && !CostCtx.Config.isLegalMaskedLoadOrStore(
IsLoad, ScalarTy, getLoadStoreAlignment(I),
getLoadStoreAddressSpace(I)))
diff --git a/llvm/test/Transforms/LoopVectorize/AArch64/invariant-replicate-region.ll b/llvm/test/Transforms/LoopVectorize/AArch64/invariant-replicate-region.ll
index 78eaa586735d8..d05002391d357 100644
--- a/llvm/test/Transforms/LoopVectorize/AArch64/invariant-replicate-region.ll
+++ b/llvm/test/Transforms/LoopVectorize/AArch64/invariant-replicate-region.ll
@@ -11,53 +11,33 @@ define i32 @test_invariant_replicate_region(i32 %x, i1 %c) {
; CHECK-NEXT: [[ENTRY:.*:]]
; CHECK-NEXT: br label %[[VECTOR_PH:.*]]
; CHECK: [[VECTOR_PH]]:
-; CHECK-NEXT: [[BROADCAST_SPLATINSERT:%.*]] = insertelement <4 x i1> poison, i1 [[C]], i64 0
-; CHECK-NEXT: [[BROADCAST_SPLAT:%.*]] = shufflevector <4 x i1> [[BROADCAST_SPLATINSERT]], <4 x i1> poison, <4 x i32> zeroinitializer
; CHECK-NEXT: br label %[[VECTOR_BODY:.*]]
; CHECK: [[VECTOR_BODY]]:
-; CHECK-NEXT: [[INDEX:%.*]] = phi i32 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[PRED_UREM_CONTINUE:.*]] ]
-; CHECK-NEXT: br i1 [[C]], label %[[PRED_UREM_IF:.*]], label %[[PRED_UREM_CONTINUE]]
-; CHECK: [[PRED_UREM_IF]]:
-; CHECK-NEXT: br i1 [[C]], label %[[PRED_UREM_IF2:.*]], label %[[PRED_UREM_CONTINUE1:.*]]
+; CHECK-NEXT: [[INDEX:%.*]] = phi i32 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[PRED_UREM_CONTINUE1:.*]] ]
+; CHECK-NEXT: br i1 [[C]], label %[[PRED_UREM_IF2:.*]], label %[[PRED_UREM_CONTINUE1]]
; CHECK: [[PRED_UREM_IF2]]:
-; CHECK-NEXT: [[TMP1:%.*]] = urem i32 10, [[X]]
-; CHECK-NEXT: [[TMP3:%.*]] = insertelement <4 x i32> poison, i32 [[TMP1]], i64 0
; CHECK-NEXT: br label %[[PRED_UREM_CONTINUE1]]
; CHECK: [[PRED_UREM_CONTINUE1]]:
-; CHECK-NEXT: [[TMP2:%.*]] = phi <4 x i32> [ poison, %[[PRED_UREM_IF]] ], [ [[TMP3]], %[[PRED_UREM_IF2]] ]
-; CHECK-NEXT: br i1 [[C]], label %[[PRED_UREM_IF1:.*]], label %[[PRED_UREM_CONTINUE2:.*]]
+; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i32 [[INDEX]], 8
+; CHECK-NEXT: [[TMP0:%.*]] = icmp eq i32 [[INDEX_NEXT]], 96
+; CHECK-NEXT: br i1 [[TMP0]], label %[[PRED_UREM_IF1:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP0:![0-9]+]]
; CHECK: [[PRED_UREM_IF1]]:
-; CHECK-NEXT: [[TMP5:%.*]] = urem i32 10, [[X]]
-; CHECK-NEXT: [[TMP4:%.*]] = insertelement <4 x i32> [[TMP2]], i32 [[TMP5]], i64 1
-; CHECK-NEXT: br label %[[PRED_UREM_CONTINUE2]]
+; CHECK-NEXT: br label %[[PRED_UREM_CONTINUE2:.*]]
; CHECK: [[PRED_UREM_CONTINUE2]]:
-; CHECK-NEXT: [[TMP6:%.*]] = phi <4 x i32> [ [[TMP2]], %[[PRED_UREM_CONTINUE1]] ], [ [[TMP4]], %[[PRED_UREM_IF1]] ]
-; CHECK-NEXT: br i1 [[C]], label %[[PRED_UREM_IF3:.*]], label %[[PRED_UREM_CONTINUE4:.*]]
-; CHECK: [[PRED_UREM_IF3]]:
-; CHECK-NEXT: [[TMP9:%.*]] = urem i32 10, [[X]]
-; CHECK-NEXT: [[TMP7:%.*]] = insertelement <4 x i32> [[TMP6]], i32 [[TMP9]], i64 2
-; CHECK-NEXT: br label %[[PRED_UREM_CONTINUE4]]
+; CHECK-NEXT: br label %[[PRED_UREM_CONTINUE4:.*]]
; CHECK: [[PRED_UREM_CONTINUE4]]:
-; CHECK-NEXT: [[TMP8:%.*]] = phi <4 x i32> [ [[TMP6]], %[[PRED_UREM_CONTINUE2]] ], [ [[TMP7]], %[[PRED_UREM_IF3]] ]
-; CHECK-NEXT: br i1 [[C]], label %[[PRED_UREM_IF5:.*]], label %[[PRED_UREM_CONTINUE6:.*]]
+; CHECK-NEXT: [[IV:%.*]] = phi i32 [ 96, %[[PRED_UREM_CONTINUE2]] ], [ [[IV_NEXT:%.*]], %[[PRED_UREM_CONTINUE6:.*]] ]
+; CHECK-NEXT: br i1 [[C]], label %[[PRED_UREM_IF5:.*]], label %[[PRED_UREM_CONTINUE6]]
; CHECK: [[PRED_UREM_IF5]]:
; CHECK-NEXT: [[TMP13:%.*]] = urem i32 10, [[X]]
-; CHECK-NEXT: [[TMP10:%.*]] = insertelement <4 x i32> [[TMP8]], i32 [[TMP13]], i64 3
; CHECK-NEXT: br label %[[PRED_UREM_CONTINUE6]]
; CHECK: [[PRED_UREM_CONTINUE6]]:
-; CHECK-NEXT: [[TMP11:%.*]] = phi <4 x i32> [ [[TMP8]], %[[PRED_UREM_CONTINUE4]] ], [ [[TMP10]], %[[PRED_UREM_IF5]] ]
-; CHECK-NEXT: br label %[[PRED_UREM_CONTINUE]]
-; CHECK: [[PRED_UREM_CONTINUE]]:
-; CHECK-NEXT: [[TMP12:%.*]] = phi <4 x i32> [ poison, %[[VECTOR_BODY]] ], [ [[TMP11]], %[[PRED_UREM_CONTINUE6]] ]
-; CHECK-NEXT: [[TMP14:%.*]] = phi <4 x i1> [ zeroinitializer, %[[VECTOR_BODY]] ], [ [[BROADCAST_SPLAT]], %[[PRED_UREM_CONTINUE6]] ]
-; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i32 [[INDEX]], 4
-; CHECK-NEXT: [[TMP16:%.*]] = icmp eq i32 [[INDEX_NEXT]], 100
-; CHECK-NEXT: br i1 [[TMP16]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP0:![0-9]+]]
-; CHECK: [[MIDDLE_BLOCK]]:
-; CHECK-NEXT: [[PREDPHI:%.*]] = select <4 x i1> [[TMP14]], <4 x i32> [[TMP12]], <4 x i32> zeroinitializer
-; CHECK-NEXT: [[TMP17:%.*]] = extractelement <4 x i32> [[PREDPHI]], i64 3
-; CHECK-NEXT: br label %[[EXIT:.*]]
+; CHECK-NEXT: [[RES:%.*]] = phi i32 [ 0, %[[PRED_UREM_CONTINUE4]] ], [ [[TMP13]], %[[PRED_UREM_IF5]] ]
+; CHECK-NEXT: [[IV_NEXT]] = add i32 [[IV]], 1
+; CHECK-NEXT: [[EC:%.*]] = icmp eq i32 [[IV]], 99
+; CHECK-NEXT: br i1 [[EC]], label %[[EXIT:.*]], label %[[PRED_UREM_CONTINUE4]], !llvm.loop [[LOOP3:![0-9]+]]
; CHECK: [[EXIT]]:
+; CHECK-NEXT: [[TMP17:%.*]] = phi i32 [ [[RES]], %[[PRED_UREM_CONTINUE6]] ]
; CHECK-NEXT: ret i32 [[TMP17]]
;
entry:
@@ -84,4 +64,5 @@ exit:
; CHECK: [[LOOP0]] = distinct !{[[LOOP0]], [[META1:![0-9]+]], [[META2:![0-9]+]]}
; CHECK: [[META1]] = !{!"llvm.loop.isvectorized", i32 1}
; CHECK: [[META2]] = !{!"llvm.loop.unroll.runtime.disable"}
+; CHECK: [[LOOP3]] = distinct !{[[LOOP3]], [[META2]], [[META1]]}
;.
diff --git a/llvm/test/Transforms/LoopVectorize/AArch64/predication_costs.ll b/llvm/test/Transforms/LoopVectorize/AArch64/predication_costs.ll
index fdb22beb19695..e7b718dc0d483 100644
--- a/llvm/test/Transforms/LoopVectorize/AArch64/predication_costs.ll
+++ b/llvm/test/Transforms/LoopVectorize/AArch64/predication_costs.ll
@@ -62,7 +62,34 @@ for.end:
; CHECK: Scalarizing and predicating: store i32 %tmp2, ptr %tmp0, align 4
; CHECK: Cost of 4 for VF 2: profitable to scalarize store i32 %tmp2, ptr %tmp0, align 4
;
-define void @predicated_store(ptr %a, i1 %c, i32 %x, i64 %n) {
+define void @predicated_store(ptr %a, i32 %x, i64 %n) {
+entry:
+ br label %for.body
+
+for.body:
+ %i = phi i64 [ 0, %entry ], [ %i.next, %for.inc ]
+ %tmp0 = getelementptr inbounds i32, ptr %a, i64 %i
+ %tmp1 = load i32, ptr %tmp0, align 4
+ %tmp2 = add nsw i32 %tmp1, %x
+ %c = icmp eq i32 %tmp1, 42
+ br i1 %c, label %if.then, label %for.inc
+
+if.then:
+ store i32 %tmp2, ptr %tmp0, align 4
+ br label %for.inc
+
+for.inc:
+ %i.next = add nuw nsw i64 %i, 1
+ %cond = icmp slt i64 %i.next, %n
+ br i1 %cond, label %for.body, label %for.end
+
+for.end:
+ ret void
+}
+
+; CHECK-LABEL: store_in_preserved_uniform_control_flow
+; CHECK: Cost of 1 for VF 2: WIDEN store {{.*}}, ir<%tmp2>
+define void @store_in_preserved_uniform_control_flow(ptr %a, i1 %c, i32 %x, i64 %n) {
entry:
br label %for.body
@@ -96,7 +123,7 @@ for.end:
; CHECK: Cost of 0 for VF 2: induction instruction %addr = phi ptr [ %a, %entry ], [ %addr.next, %for.inc ]
; CHECK: Cost of 4 for VF 2: profitable to scalarize store i32 %tmp2, ptr %addr, align 4
;
-define void @predicated_store_phi(ptr %a, i1 %c, i32 %x, i64 %n) {
+define void @predicated_store_phi(ptr %a, i32 %x, i64 %n) {
entry:
br label %for.body
@@ -105,6 +132,7 @@ for.body:
%addr = phi ptr [ %a, %entry ], [ %addr.next, %for.inc ]
%tmp1 = load i32, ptr %addr, align 4
%tmp2 = add nsw i32 %tmp1, %x
+ %c = icmp eq i32 %tmp1, 42
br i1 %c, label %if.then, label %for.inc
if.then:
@@ -184,7 +212,7 @@ for.end:
; CHECK: Cost of 2 for VF 2: profitable to scalarize store i32 %tmp2, ptr %tmp0, align 4
; CHECK: Cost of 3 for VF 2: profitable to scalarize %tmp2 = add nsw i32 %tmp1, %x
;
-define void @predicated_store_scalarized_operand(ptr %a, i1 %c, i32 %x, i64 %n) {
+define void @predicated_store_scalarized_operand(ptr %a, i32 %x, i64 %n) {
entry:
br label %for.body
@@ -192,6 +220,7 @@ for.body:
%i = phi i64 [ 0, %entry ], [ %i.next, %for.inc ]
%tmp0 = getelementptr inbounds i32, ptr %a, i64 %i
%tmp1 = load i32, ptr %tmp0, align 4
+ %c = icmp eq i32 %tmp1, 42
br i1 %c, label %if.then, label %for.inc
if.then:
@@ -240,7 +269,7 @@ for.end:
; CHECK: Cost of 7 for VF 2: REPLICATE ir<%tmp3> = sdiv ir<%tmp1>, ir<%tmp2>
; CHECK: Cost of 5 for VF 2: REPLICATE ir<%tmp4> = udiv ir<%tmp3>, ir<%tmp2>
;
-define void @predication_multi_context(ptr %a, i1 %c, i32 %x, i64 %n) {
+define void @predication_multi_context(ptr %a, i32 %x, i64 %n) {
entry:
br label %for.body
@@ -248,6 +277,7 @@ for.body:
%i = phi i64 [ 0, %entry ], [ %i.next, %for.inc ]
%tmp0 = getelementptr inbounds i32, ptr %a, i64 %i
%tmp1 = load i32, ptr %tmp0, align 4
+ %c = icmp eq i32 %tmp1, 42
br i1 %c, label %if.then, label %for.inc
if.then:
diff --git a/llvm/test/Transforms/LoopVectorize/AArch64/sve-predicated-costs.ll b/llvm/test/Transforms/LoopVectorize/AArch64/sve-predicated-costs.ll
index 76aa72c80580b..bc4394d3218ed 100644
--- a/llvm/test/Transforms/LoopVectorize/AArch64/sve-predicated-costs.ll
+++ b/llvm/test/Transforms/LoopVectorize/AArch64/sve-predicated-costs.ll
@@ -81,10 +81,7 @@ define void @always_taken(ptr noalias %p0, ptr noalias %p1, i1 %c0, i1 %c1, i1 %
; CHECK-NEXT: [[BROADCAST_SPLATINSERT2:%.*]] = insertelement <vscale x 2 x i1> poison, i1 [[C2]], i64 0
; CHECK-NEXT: [[BROADCAST_SPLAT1:%.*]] = shufflevector <vscale x 2 x i1> [[BROADCAST_SPLATINSERT2]], <vscale x 2 x i1> poison, <vscale x 2 x i32> zeroinitializer
; CHECK-NEXT: [[BROADCAST_SPLATINSERT:%.*]] = insertelement <vscale x 2 x i1> poison, i1 [[C1]], i64 0
-; CHECK-NEXT: [[BROADCAST_SPLAT:%.*]] = shufflevector <vscale x 2 x i1> [[BROADCAST_SPLATINSERT]], <vscale x 2 x i1> poison, <vscale x 2 x i32> zeroinitializer
-; CHECK-NEXT: [[BROADCAST_SPLATINSERT1:%.*]] = insertelement <vscale x 2 x i1> poison, i1 [[C0]], i64 0
-; CHECK-NEXT: [[BROADCAST_SPLAT2:%.*]] = shufflevector <vscale x 2 x i1> [[BROADCAST_SPLATINSERT1]], <vscale x 2 x i1> poison, <vscale x 2 x i32> zeroinitializer
-; CHECK-NEXT: [[TMP6:%.*]] = select <vscale x 2 x i1> [[BROADCAST_SPLAT2]], <vscale x 2 x i1> [[BROADCAST_SPLAT]], <vscale x 2 x i1> zeroinitializer
+; CHECK-NEXT: [[TMP6:%.*]] = shufflevector <vscale x 2 x i1> [[BROADCAST_SPLATINSERT]], <vscale x 2 x i1> poison, <vscale x 2 x i32> zeroinitializer
; CHECK-NEXT: [[TMP8:%.*]] = select <vscale x 2 x i1> [[TMP6]], <vscale x 2 x i1> [[BROADCAST_SPLAT1]], <vscale x 2 x i1> zeroinitializer
; CHECK-NEXT: br label %[[VECTOR_BODY:.*]]
; CHECK: [[VECTOR_BODY]]:
diff --git a/llvm/test/Transforms/LoopVectorize/AArch64/transform-narrow-interleave-to-widen-memory-multi-block.ll b/llvm/test/Transforms/LoopVectorize/AArch64/transform-narrow-interleave-to-widen-memory-multi-block.ll
index 9efff93543685..3d69698888f16 100644
--- a/llvm/test/Transforms/LoopVectorize/AArch64/transform-narrow-interleave-to-widen-memory-multi-block.ll
+++ b/llvm/test/Transforms/LoopVectorize/AArch64/transform-narrow-interleave-to-widen-memory-multi-block.ll
@@ -13,7 +13,7 @@ define void @load_store_interleave_group_block_invar_cond(ptr noalias %data, ptr
; VF2IC1: [[VECTOR_PH]]:
; VF2IC1-NEXT: br label %[[VECTOR_BODY:.*]]
; VF2IC1: [[VECTOR_BODY]]:
-; VF2IC1-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[PRED_STORE_CONTINUE:.*]] ]
+; VF2IC1-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[PRED_STORE_CONTINUE3:.*]] ]
; VF2IC1-NEXT: [[TMP0:%.*]] = shl nsw i64 [[INDEX]], 1
; VF2IC1-NEXT: [[TMP1:%.*]] = getelementptr inbounds i64, ptr [[DATA]], i64 [[TMP0]]
; VF2IC1-NEXT: [[WIDE_VEC:%.*]] = load <4 x i64>, ptr [[TMP1]], align 8
@@ -22,20 +22,11 @@ define void @load_store_interleave_group_block_invar_cond(ptr noalias %data, ptr
; VF2IC1-NEXT: [[TMP4:%.*]] = shufflevector <2 x i64> [[STRIDED_VEC]], <2 x i64> [[STRIDED_VEC1]], <4 x i32> <i32 0, i32 1, i32 2, i32 3>
; VF2IC1-NEXT: [[INTERLEAVED_VEC:%.*]] = shufflevector <4 x i64> [[TMP4]], <4 x i64> poison, <4 x i32> <i32 0, i32 2, i32 1, i32 3>
; VF2IC1-NEXT: store <4 x i64> [[INTERLEAVED_VEC]], ptr [[TMP1]], align 8
-; VF2IC1-NEXT: br i1 [[C]], label %[[PRED_STORE_IF:.*]], label %[[PRED_STORE_CONTINUE]]
-; VF2IC1: [[PRED_STORE_IF]]:
-; VF2IC1-NEXT: br i1 [[C]], label %[[PRED_STORE_IF1:.*]], label %[[PRED_STORE_CONTINUE1:.*]]
-; VF2IC1: [[PRED_STORE_IF1]]:
-; VF2IC1-NEXT: store i8 1, ptr [[DST_0]], align 1
-; VF2IC1-NEXT: br label %[[PRED_STORE_CONTINUE1]]
-; VF2IC1: [[PRED_STORE_CONTINUE1]]:
-; VF2IC1-NEXT: br i1 [[C]], label %[[PRED_STORE_IF2:.*]], label %[[PRED_STORE_CONTINUE3:.*]]
+; VF2IC1-NEXT: br i1 [[C]], label %[[PRED_STORE_IF2:.*]], label %[[PRED_STORE_CONTINUE3]]
; VF2IC1: [[PRED_STORE_IF2]]:
; VF2IC1-NEXT: store i8 1, ptr [[DST_0]], align 1
; VF2IC1-NEXT: br label %[[PRED_STORE_CONTINUE3]]
; VF2IC1: [[PRED_STORE_CONTINUE3]]:
-; VF2IC1-NEXT: br label %[[PRED_STORE_CONTINUE]]
-; VF2IC1: [[PRED_STORE_CONTINUE]]:
; VF2IC1-NEXT: [[TMP2:%.*]] = getelementptr inbounds i8, ptr [[DST_1]], i64 [[INDEX]]
; VF2IC1-NEXT: store <2 x i8> zeroinitializer, ptr [[TMP2]], align 1
; VF2IC1-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 2
@@ -53,7 +44,7 @@ define void @load_store_interleave_group_block_invar_cond(ptr noalias %data, ptr
; VF2IC2: [[VECTOR_PH]]:
; VF2IC2-NEXT: br label %[[VECTOR_BODY:.*]]
; VF2IC2: [[VECTOR_BODY]]:
-; VF2IC2-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[PRED_STORE_CONTINUE:.*]] ]
+; VF2IC2-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[PRED_STORE_CONTINUE11:.*]] ]
; VF2IC2-NEXT: [[TMP0:%.*]] = add i64 [[INDEX]], 2
; VF2IC2-NEXT: [[TMP1:%.*]] = shl nsw i64 [[INDEX]], 1
; VF2IC2-NEXT: [[TMP2:%.*]] = shl nsw i64 [[TMP0]], 1
@@ -71,30 +62,11 @@ define void @load_store_interleave_group_block_invar_cond(ptr noalias %data, ptr
; VF2IC2-NEXT: [[TMP9:%.*]] = shufflevector <2 x i64> [[STRIDED_VEC3]], <2 x i64> [[STRIDED_VEC4]], <4 x i32> <i32 0, i32 1, i32 2, i32 3>
; VF2IC2-NEXT: [[INTERLEAVED_VEC5:%.*]] = shufflevector <4 x i64> [[TMP9]], <4 x i64> poison, <4 x i32> <i32 0, i32 2, i32 1, i32 3>
; VF2IC2-NEXT: store <4 x i64> [[INTERLEAVED_VEC5]], ptr [[TMP4]], align 8
-; VF2IC2-NEXT: br i1 [[C]], label %[[PRED_STORE_IF:.*]], label %[[PRED_STORE_CONTINUE]]
-; VF2IC2: [[PRED_STORE_IF]]:
-; VF2IC2-NEXT: br i1 [[C]], label %[[PRED_STORE_IF1:.*]], label %[[PRED_STORE_CONTINUE1:.*]]
-; VF2IC2: [[PRED_STORE_IF1]]:
-; VF2IC2-NEXT: store i8 1, ptr [[DST_0]], align 1
-; VF2IC2-NEXT: br label %[[PRED_STORE_CONTINUE1]]
-; VF2IC2: [[PRED_STORE_CONTINUE1]]:
-; VF2IC2-NEXT: br i1 [[C]], label %[[PRED_STORE_IF6:.*]], label %[[PRED_STORE_CONTINUE7:.*]]
-; VF2IC2: [[PRED_STORE_IF6]]:
-; VF2IC2-NEXT: store i8 1, ptr [[DST_0]], align 1
-; VF2IC2-NEXT: br label %[[PRED_STORE_CONTINUE7]]
-; VF2IC2: [[PRED_STORE_CONTINUE7]]:
-; VF2IC2-NEXT: br i1 [[C]], label %[[PRED_STORE_IF8:.*]], label %[[PRED_STORE_CONTINUE9:.*]]
-; VF2IC2: [[PRED_STORE_IF8]]:
-; VF2IC2-NEXT: store i8 1, ptr [[DST_0]], align 1
-; VF2IC2-NEXT: br label %[[PRED_STORE_CONTINUE9]]
-; VF2IC2: [[PRED_STORE_CONTINUE9]]:
-; VF2IC2-NEXT: br i1 [[C]], label %[[PRED_STORE_IF10:.*]], label %[[PRED_STORE_CONTINUE11:.*]]
+; VF2IC2-NEXT: br i1 [[C]], label %[[PRED_STORE_IF10:.*]], label %[[PRED_STORE_CONTINUE11]]
; VF2IC2: [[PRED_STORE_IF10]]:
; VF2IC2-NEXT: store i8 1, ptr [[DST_0]], align 1
; VF2IC2-NEXT: br label %[[PRED_STORE_CONTINUE11]]
; VF2IC2: [[PRED_STORE_CONTINUE11]]:
-; VF2IC2-NEXT: br label %[[PRED_STORE_CONTINUE]]
-; VF2IC2: [[PRED_STORE_CONTINUE]]:
; VF2IC2-NEXT: [[TMP5:%.*]] = getelementptr inbounds i8, ptr [[DST_1]], i64 [[INDEX]]
; VF2IC2-NEXT: [[TMP6:%.*]] = getelementptr inbounds i8, ptr [[TMP5]], i64 2
; VF2IC2-NEXT: store <2 x i8> zeroinitializer, ptr [[TMP5]], align 1
diff --git a/llvm/test/Transforms/LoopVectorize/RISCV/dead-ops-cost.ll b/llvm/test/Transforms/LoopVectorize/RISCV/dead-ops-cost.ll
index c4ca7d1d47673..913eba75abf17 100644
--- a/llvm/test/Transforms/LoopVectorize/RISCV/dead-ops-cost.ll
+++ b/llvm/test/Transforms/LoopVectorize/RISCV/dead-ops-cost.ll
@@ -158,60 +158,19 @@ define i32 @cost_of_exit_branch_and_cond_insts(ptr %a, ptr %b, i1 %c, i16 %x) #0
; CHECK-NEXT: [[TMP8:%.*]] = icmp eq i32 [[N_MOD_VF]], 0
; CHECK-NEXT: [[TMP9:%.*]] = select i1 [[TMP8]], i32 8, i32 [[N_MOD_VF]]
; CHECK-NEXT: [[N_VEC:%.*]] = sub i32 [[TMP2]], [[TMP9]]
-; CHECK-NEXT: [[BROADCAST_SPLATINSERT:%.*]] = insertelement <8 x i1> poison, i1 [[C]], i64 0
-; CHECK-NEXT: [[BROADCAST_SPLAT:%.*]] = shufflevector <8 x i1> [[BROADCAST_SPLATINSERT]], <8 x i1> poison, <8 x i32> zeroinitializer
-; CHECK-NEXT: br label %[[VECTOR_BODY:.*]]
-; CHECK: [[VECTOR_BODY]]:
-; CHECK-NEXT: [[INDEX:%.*]] = phi i32 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[LOOP_EXITING18:.*]] ]
-; CHECK-NEXT: br i1 [[C]], label %[[THEN3:.*]], label %[[LOOP_EXITING18]]
-; CHECK: [[THEN3]]:
-; CHECK-NEXT: [[TMP11:%.*]] = getelementptr i32, ptr [[B]], i32 [[INDEX]]
-; CHECK-NEXT: br i1 [[C]], label %[[PRED_STORE_IF:.*]], label %[[PRED_STORE_CONTINUE:.*]]
-; CHECK: [[PRED_STORE_IF]]:
-; CHECK-NEXT: store i1 false, ptr [[A]], align 1, !alias.scope [[META10:![0-9]+]], !noalias [[META13:![0-9]+]]
-; CHECK-NEXT: br label %[[PRED_STORE_CONTINUE]]
-; CHECK: [[PRED_STORE_CONTINUE]]:
-; CHECK-NEXT: br i1 [[C]], label %[[PRED_STORE_IF5:.*]], label %[[PRED_STORE_CONTINUE6:.*]]
-; CHECK: [[PRED_STORE_IF5]]:
-; CHECK-NEXT: store i1 false, ptr [[A]], align 1, !alias.scope [[META10]], !noalias [[META13]]
-; CHECK-NEXT: br label %[[PRED_STORE_CONTINUE6]]
-; CHECK: [[PRED_STORE_CONTINUE6]]:
-; CHECK-NEXT: br i1 [[C]], label %[[PRED_STORE_IF7:.*]], label %[[PRED_STORE_CONTINUE8:.*]]
-; CHECK: [[PRED_STORE_IF7]]:
-; CHECK-NEXT: store i1 false, ptr [[A]], align 1, !alias.scope [[META10]], !noalias [[META13]]
-; CHECK-NEXT: br label %[[PRED_STORE_CONTINUE8]]
-; CHECK: [[PRED_STORE_CONTINUE8]]:
-; CHECK-NEXT: br i1 [[C]], label %[[PRED_STORE_IF9:.*]], label %[[PRED_STORE_CONTINUE10:.*]]
-; CHECK: [[PRED_STORE_IF9]]:
-; CHECK-NEXT: store i1 false, ptr [[A]], align 1, !alias.scope [[META10]], !noalias [[META13]]
-; CHECK-NEXT: br label %[[PRED_STORE_CONTINUE10]]
-; CHECK: [[PRED_STORE_CONTINUE10]]:
-; CHECK-NEXT: br i1 [[C]], label %[[PRED_STORE_IF11:.*]], label %[[PRED_STORE_CONTINUE12:.*]]
-; CHECK: [[PRED_STORE_IF11]]:
-; CHECK-NEXT: store i1 false, ptr [[A]], align 1, !alias.scope [[META10]], !noalias [[META13]]
-; CHECK-NEXT: br label %[[PRED_STORE_CONTINUE12]]
-; CHECK: [[PRED_STORE_CONTINUE12]]:
-; CHECK-NEXT: br i1 [[C]], label %[[PRED_STORE_IF13:.*]], label %[[PRED_STORE_CONTINUE14:.*]]
-; CHECK: [[PRED_STORE_IF13]]:
-; CHECK-NEXT: store i1 false, ptr [[A]], align 1, !alias.scope [[META10]], !noalias [[META13]]
-; CHECK-NEXT: br label %[[PRED_STORE_CONTINUE14]]
-; CHECK: [[PRED_STORE_CONTINUE14]]:
-; CHECK-NEXT: br i1 [[C]], label %[[PRED_STORE_IF15:.*]], label %[[PRED_STORE_CONTINUE16:.*]]
-; CHECK: [[PRED_STORE_IF15]]:
-; CHECK-NEXT: store i1 false, ptr [[A]], align 1, !alias.scope [[META10]], !noalias [[META13]]
-; CHECK-NEXT: br label %[[PRED_STORE_CONTINUE16]]
+; CHECK-NEXT: br label %[[PRED_STORE_CONTINUE16:.*]]
; CHECK: [[PRED_STORE_CONTINUE16]]:
-; CHECK-NEXT: br i1 [[C]], label %[[PRED_STORE_IF17:.*]], label %[[PRED_STORE_CONTINUE18:.*]]
+; CHECK-NEXT: [[INDEX:%.*]] = phi i32 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[PRED_STORE_CONTINUE18:.*]] ]
+; CHECK-NEXT: br i1 [[C]], label %[[PRED_STORE_IF17:.*]], label %[[PRED_STORE_CONTINUE18]]
; CHECK: [[PRED_STORE_IF17]]:
-; CHECK-NEXT: store i1 false, ptr [[A]], align 1, !alias.scope [[META10]], !noalias [[META13]]
+; CHECK-NEXT: [[TMP13:%.*]] = getelementptr inbounds i32, ptr [[B]], i32 [[INDEX]]
+; CHECK-NEXT: store i1 false, ptr [[A]], align 1, !alias.scope [[META10:![0-9]+]], !noalias [[META13:![0-9]+]]
+; CHECK-NEXT: store <8 x i32> zeroinitializer, ptr [[TMP13]], align 4, !alias.scope [[META13]]
; CHECK-NEXT: br label %[[PRED_STORE_CONTINUE18]]
; CHECK: [[PRED_STORE_CONTINUE18]]:
-; CHECK-NEXT: call void @llvm.masked.store.v8i32.p0(<8 x i32> zeroinitializer, ptr align 4 [[TMP11]], <8 x i1> [[BROADCAST_SPLAT]]), !alias.scope [[META13]]
-; CHECK-NEXT: br label %[[LOOP_EXITING18]]
-; CHECK: [[LOOP_EXITING18]]:
; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i32 [[INDEX]], 8
; CHECK-NEXT: [[TMP21:%.*]] = icmp eq i32 [[INDEX_NEXT]], [[N_VEC]]
-; CHECK-NEXT: br i1 [[TMP21]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP15:![0-9]+]]
+; CHECK-NEXT: br i1 [[TMP21]], label %[[MIDDLE_BLOCK:.*]], label %[[PRED_STORE_CONTINUE16]], !llvm.loop [[LOOP15:![0-9]+]]
; CHECK: [[MIDDLE_BLOCK]]:
; CHECK-NEXT: br label %[[SCALAR_PH]]
; CHECK: [[SCALAR_PH]]:
diff --git a/llvm/test/Transforms/LoopVectorize/RISCV/divrem.ll b/llvm/test/Transforms/LoopVectorize/RISCV/divrem.ll
index 450f5729393ec..2804b5729e8df 100644
--- a/llvm/test/Transforms/LoopVectorize/RISCV/divrem.ll
+++ b/llvm/test/Transforms/LoopVectorize/RISCV/divrem.ll
@@ -633,9 +633,6 @@ define void @udiv_sdiv_with_invariant_divisors(i8 %x, i16 %y, i1 %c, ptr %p) {
; CHECK-NEXT: entry:
; CHECK-NEXT: br label [[VECTOR_BODY:%.*]]
; CHECK: vector.ph:
-; CHECK-NEXT: [[BROADCAST_SPLATINSERT:%.*]] = insertelement <vscale x 4 x i1> poison, i1 [[C:%.*]], i64 0
-; CHECK-NEXT: [[BROADCAST_SPLAT:%.*]] = shufflevector <vscale x 4 x i1> [[BROADCAST_SPLATINSERT]], <vscale x 4 x i1> poison, <vscale x 4 x i32> zeroinitializer
-; CHECK-NEXT: [[TMP0:%.*]] = xor <vscale x 4 x i1> [[BROADCAST_SPLAT]], splat (i1 true)
; CHECK-NEXT: [[BROADCAST_SPLATINSERT1:%.*]] = insertelement <vscale x 4 x i8> poison, i8 [[X:%.*]], i64 0
; CHECK-NEXT: [[BROADCAST_SPLAT2:%.*]] = shufflevector <vscale x 4 x i8> [[BROADCAST_SPLATINSERT1]], <vscale x 4 x i8> poison, <vscale x 4 x i32> zeroinitializer
; CHECK-NEXT: [[BROADCAST_SPLATINSERT3:%.*]] = insertelement <vscale x 4 x i16> poison, i16 [[Y:%.*]], i64 0
@@ -652,16 +649,16 @@ define void @udiv_sdiv_with_invariant_divisors(i8 %x, i16 %y, i1 %c, ptr %p) {
; CHECK-NEXT: [[TMP3:%.*]] = trunc i32 [[TMP2]] to i8
; CHECK-NEXT: [[BROADCAST_SPLATINSERT7:%.*]] = insertelement <vscale x 4 x i8> poison, i8 [[TMP3]], i64 0
; CHECK-NEXT: [[BROADCAST_SPLAT8:%.*]] = shufflevector <vscale x 4 x i8> [[BROADCAST_SPLATINSERT7]], <vscale x 4 x i8> poison, <vscale x 4 x i32> zeroinitializer
-; CHECK-NEXT: br i1 [[C]], label [[LOOP_LATCH10]], label [[THEN9:%.*]]
-; CHECK: then9:
-; CHECK-NEXT: [[TMP5:%.*]] = call <vscale x 4 x i8> @llvm.vp.udiv.nxv4i8(<vscale x 4 x i8> [[VEC_IND]], <vscale x 4 x i8> [[BROADCAST_SPLAT2]], <vscale x 4 x i1> [[TMP0]], i32 [[TMP2]])
+; CHECK-NEXT: br i1 [[C:%.*]], label [[LOOP_LATCH10]], label [[THEN9:%.*]]
+; CHECK: then7:
+; CHECK-NEXT: [[TMP5:%.*]] = call <vscale x 4 x i8> @llvm.vp.udiv.nxv4i8(<vscale x 4 x i8> [[VEC_IND]], <vscale x 4 x i8> [[BROADCAST_SPLAT2]], <vscale x 4 x i1> splat (i1 true), i32 [[TMP2]])
; CHECK-NEXT: [[TMP6:%.*]] = zext <vscale x 4 x i8> [[TMP5]] to <vscale x 4 x i16>
-; CHECK-NEXT: [[TMP8:%.*]] = call <vscale x 4 x i16> @llvm.vp.sdiv.nxv4i16(<vscale x 4 x i16> [[TMP6]], <vscale x 4 x i16> [[BROADCAST_SPLAT4]], <vscale x 4 x i1> [[TMP0]], i32 [[TMP2]])
+; CHECK-NEXT: [[TMP8:%.*]] = call <vscale x 4 x i16> @llvm.vp.sdiv.nxv4i16(<vscale x 4 x i16> [[TMP6]], <vscale x 4 x i16> [[BROADCAST_SPLAT4]], <vscale x 4 x i1> splat (i1 true), i32 [[TMP2]])
; CHECK-NEXT: [[TMP9:%.*]] = sext <vscale x 4 x i16> [[TMP8]] to <vscale x 4 x i32>
; CHECK-NEXT: br label [[LOOP_LATCH10]]
-; CHECK: loop.latch10:
+; CHECK: loop.latch8:
; CHECK-NEXT: [[TMP10:%.*]] = phi <vscale x 4 x i32> [ poison, [[LOOP_LATCH]] ], [ [[TMP9]], [[THEN9]] ]
-; CHECK-NEXT: [[TMP11:%.*]] = phi <vscale x 4 x i1> [ zeroinitializer, [[LOOP_LATCH]] ], [ [[TMP0]], [[THEN9]] ]
+; CHECK-NEXT: [[TMP11:%.*]] = phi <vscale x 4 x i1> [ zeroinitializer, [[LOOP_LATCH]] ], [ splat (i1 true), [[THEN9]] ]
; CHECK-NEXT: [[PREDPHI:%.*]] = select <vscale x 4 x i1> [[TMP11]], <vscale x 4 x i32> [[TMP10]], <vscale x 4 x i32> zeroinitializer
; CHECK-NEXT: call void @llvm.vp.scatter.nxv4i32.nxv4p0(<vscale x 4 x i32> [[PREDPHI]], <vscale x 4 x ptr> align 4 [[BROADCAST_SPLAT6]], <vscale x 4 x i1> splat (i1 true), i32 [[TMP2]])
; CHECK-NEXT: [[AVL_NEXT]] = sub nuw i32 [[AVL]], [[TMP2]]
@@ -677,25 +674,22 @@ define void @udiv_sdiv_with_invariant_divisors(i8 %x, i16 %y, i1 %c, ptr %p) {
; FIXED-NEXT: entry:
; FIXED-NEXT: br label [[VECTOR_PH:%.*]]
; FIXED: vector.ph:
-; FIXED-NEXT: [[BROADCAST_SPLATINSERT:%.*]] = insertelement <8 x i1> poison, i1 [[C:%.*]], i64 0
-; FIXED-NEXT: [[BROADCAST_SPLAT:%.*]] = shufflevector <8 x i1> [[BROADCAST_SPLATINSERT]], <8 x i1> poison, <8 x i32> zeroinitializer
-; FIXED-NEXT: [[TMP0:%.*]] = xor <8 x i1> [[BROADCAST_SPLAT]], splat (i1 true)
; FIXED-NEXT: [[BROADCAST_SPLATINSERT1:%.*]] = insertelement <8 x i8> poison, i8 [[X:%.*]], i64 0
; FIXED-NEXT: [[BROADCAST_SPLAT2:%.*]] = shufflevector <8 x i8> [[BROADCAST_SPLATINSERT1]], <8 x i8> poison, <8 x i32> zeroinitializer
; FIXED-NEXT: [[BROADCAST_SPLATINSERT3:%.*]] = insertelement <8 x i16> poison, i16 [[Y:%.*]], i64 0
; FIXED-NEXT: [[BROADCAST_SPLAT4:%.*]] = shufflevector <8 x i16> [[BROADCAST_SPLATINSERT3]], <8 x i16> poison, <8 x i32> zeroinitializer
; FIXED-NEXT: br label [[VECTOR_BODY:%.*]]
; FIXED: vector.body:
-; FIXED-NEXT: br i1 [[C]], label [[LOOP_LATCH6:%.*]], label [[THEN5:%.*]]
-; FIXED: then5:
-; FIXED-NEXT: [[TMP1:%.*]] = call <8 x i8> @llvm.masked.udiv.v8i8(<8 x i8> <i8 -12, i8 -11, i8 -10, i8 -9, i8 -8, i8 -7, i8 -6, i8 -5>, <8 x i8> [[BROADCAST_SPLAT2]], <8 x i1> [[TMP0]])
+; FIXED-NEXT: br i1 [[C:%.*]], label [[LOOP_LATCH6:%.*]], label [[THEN5:%.*]]
+; FIXED: then3:
+; FIXED-NEXT: [[TMP1:%.*]] = udiv <8 x i8> <i8 -12, i8 -11, i8 -10, i8 -9, i8 -8, i8 -7, i8 -6, i8 -5>, [[BROADCAST_SPLAT2]]
; FIXED-NEXT: [[TMP2:%.*]] = zext <8 x i8> [[TMP1]] to <8 x i16>
-; FIXED-NEXT: [[TMP3:%.*]] = call <8 x i16> @llvm.masked.sdiv.v8i16(<8 x i16> [[TMP2]], <8 x i16> [[BROADCAST_SPLAT4]], <8 x i1> [[TMP0]])
+; FIXED-NEXT: [[TMP3:%.*]] = sdiv <8 x i16> [[TMP2]], [[BROADCAST_SPLAT4]]
; FIXED-NEXT: [[TMP4:%.*]] = sext <8 x i16> [[TMP3]] to <8 x i32>
; FIXED-NEXT: br label [[LOOP_LATCH6]]
-; FIXED: loop.latch6:
+; FIXED: loop.latch4:
; FIXED-NEXT: [[TMP5:%.*]] = phi <8 x i32> [ poison, [[VECTOR_BODY]] ], [ [[TMP4]], [[THEN5]] ]
-; FIXED-NEXT: [[TMP6:%.*]] = phi <8 x i1> [ zeroinitializer, [[VECTOR_BODY]] ], [ [[TMP0]], [[THEN5]] ]
+; FIXED-NEXT: [[TMP6:%.*]] = phi <8 x i1> [ zeroinitializer, [[VECTOR_BODY]] ], [ splat (i1 true), [[THEN5]] ]
; FIXED-NEXT: [[PREDPHI:%.*]] = select <8 x i1> [[TMP6]], <8 x i32> [[TMP5]], <8 x i32> zeroinitializer
; FIXED-NEXT: [[TMP7:%.*]] = extractelement <8 x i32> [[PREDPHI]], i64 7
; FIXED-NEXT: store i32 [[TMP7]], ptr [[P:%.*]], align 4
diff --git a/llvm/test/Transforms/LoopVectorize/RISCV/predicated-costs.ll b/llvm/test/Transforms/LoopVectorize/RISCV/predicated-costs.ll
index 48a60ffa0c30d..b27e5be10c71d 100644
--- a/llvm/test/Transforms/LoopVectorize/RISCV/predicated-costs.ll
+++ b/llvm/test/Transforms/LoopVectorize/RISCV/predicated-costs.ll
@@ -12,10 +12,7 @@ define void @nested(ptr noalias %p0, ptr noalias %p1, i1 %c0, i1 %c1) {
; CHECK-NEXT: br label %[[LOOP:.*]]
; CHECK: [[LOOP]]:
; CHECK-NEXT: [[BROADCAST_SPLATINSERT:%.*]] = insertelement <vscale x 4 x i1> poison, i1 [[C1]], i64 0
-; CHECK-NEXT: [[BROADCAST_SPLAT:%.*]] = shufflevector <vscale x 4 x i1> [[BROADCAST_SPLATINSERT]], <vscale x 4 x i1> poison, <vscale x 4 x i32> zeroinitializer
-; CHECK-NEXT: [[BROADCAST_SPLATINSERT1:%.*]] = insertelement <vscale x 4 x i1> poison, i1 [[C0]], i64 0
-; CHECK-NEXT: [[BROADCAST_SPLAT2:%.*]] = shufflevector <vscale x 4 x i1> [[BROADCAST_SPLATINSERT1]], <vscale x 4 x i1> poison, <vscale x 4 x i32> zeroinitializer
-; CHECK-NEXT: [[TMP0:%.*]] = select <vscale x 4 x i1> [[BROADCAST_SPLAT2]], <vscale x 4 x i1> [[BROADCAST_SPLAT]], <vscale x 4 x i1> zeroinitializer
+; CHECK-NEXT: [[TMP0:%.*]] = shufflevector <vscale x 4 x i1> [[BROADCAST_SPLATINSERT]], <vscale x 4 x i1> poison, <vscale x 4 x i32> zeroinitializer
; CHECK-NEXT: br label %[[THEN_1:.*]]
; CHECK: [[THEN_1]]:
; CHECK-NEXT: [[IV1:%.*]] = phi i32 [ 0, %[[LOOP]] ], [ [[INDEX_EVL_NEXT:%.*]], %[[LATCH4:.*]] ]
@@ -76,10 +73,7 @@ define void @always_taken(ptr noalias %p0, ptr noalias %p1, i1 %c0, i1 %c1) {
; CHECK-NEXT: br label %[[VECTOR_PH:.*]]
; CHECK: [[VECTOR_PH]]:
; CHECK-NEXT: [[BROADCAST_SPLATINSERT:%.*]] = insertelement <vscale x 4 x i1> poison, i1 [[C1]], i64 0
-; CHECK-NEXT: [[BROADCAST_SPLAT:%.*]] = shufflevector <vscale x 4 x i1> [[BROADCAST_SPLATINSERT]], <vscale x 4 x i1> poison, <vscale x 4 x i32> zeroinitializer
-; CHECK-NEXT: [[BROADCAST_SPLATINSERT1:%.*]] = insertelement <vscale x 4 x i1> poison, i1 [[C0]], i64 0
-; CHECK-NEXT: [[BROADCAST_SPLAT2:%.*]] = shufflevector <vscale x 4 x i1> [[BROADCAST_SPLATINSERT1]], <vscale x 4 x i1> poison, <vscale x 4 x i32> zeroinitializer
-; CHECK-NEXT: [[TMP0:%.*]] = select <vscale x 4 x i1> [[BROADCAST_SPLAT2]], <vscale x 4 x i1> [[BROADCAST_SPLAT]], <vscale x 4 x i1> zeroinitializer
+; CHECK-NEXT: [[TMP0:%.*]] = shufflevector <vscale x 4 x i1> [[BROADCAST_SPLATINSERT]], <vscale x 4 x i1> poison, <vscale x 4 x i32> zeroinitializer
; CHECK-NEXT: br label %[[VECTOR_BODY:.*]]
; CHECK: [[VECTOR_BODY]]:
; CHECK-NEXT: [[EVL_BASED_IV:%.*]] = phi i32 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_EVL_NEXT:%.*]], %[[LATCH4:.*]] ]
diff --git a/llvm/test/Transforms/LoopVectorize/RISCV/predicated-reverse-store.ll b/llvm/test/Transforms/LoopVectorize/RISCV/predicated-reverse-store.ll
index 29373388b3721..a5b3d96996c26 100644
--- a/llvm/test/Transforms/LoopVectorize/RISCV/predicated-reverse-store.ll
+++ b/llvm/test/Transforms/LoopVectorize/RISCV/predicated-reverse-store.ll
@@ -7,8 +7,6 @@ define void @reverse_predicated_store(i1 %c, ptr %dst, i64 %n) #0 {
; CHECK-NEXT: [[TMP0:%.*]] = add i64 [[N:%.*]], 1
; CHECK-NEXT: br label [[FOR_BODY:%.*]]
; CHECK: vector.ph:
-; CHECK-NEXT: [[BROADCAST_SPLATINSERT:%.*]] = insertelement <vscale x 4 x i1> poison, i1 [[C:%.*]], i64 0
-; CHECK-NEXT: [[BROADCAST_SPLAT:%.*]] = shufflevector <vscale x 4 x i1> [[BROADCAST_SPLATINSERT]], <vscale x 4 x i1> poison, <vscale x 4 x i32> zeroinitializer
; CHECK-NEXT: br label [[VECTOR_BODY:%.*]]
; CHECK: vector.body:
; CHECK-NEXT: [[EVL_BASED_IV:%.*]] = phi i64 [ 0, [[FOR_BODY]] ], [ [[CURRENT_ITERATION_NEXT:%.*]], [[LOOP_LATCH2:%.*]] ]
@@ -16,16 +14,15 @@ define void @reverse_predicated_store(i1 %c, ptr %dst, i64 %n) #0 {
; CHECK-NEXT: [[TMP1:%.*]] = call i32 @llvm.experimental.get.vector.length.i64(i64 [[AVL]], i32 4, i1 true)
; CHECK-NEXT: [[IV:%.*]] = sub i64 [[N]], [[EVL_BASED_IV]]
; CHECK-NEXT: [[IV_NEXT:%.*]] = add i64 [[IV]], -1
-; CHECK-NEXT: br i1 [[C]], label [[IF_THEN1:%.*]], label [[LOOP_LATCH2]]
+; CHECK-NEXT: br i1 [[C:%.*]], label [[IF_THEN1:%.*]], label [[LOOP_LATCH2]]
; CHECK: if.then1:
; CHECK-NEXT: [[ARRAYIDX:%.*]] = getelementptr float, ptr [[DST:%.*]], i64 [[IV_NEXT]]
-; CHECK-NEXT: [[VP_REVERSE_MASK:%.*]] = call <vscale x 4 x i1> @llvm.experimental.vp.reverse.nxv4i1(<vscale x 4 x i1> [[BROADCAST_SPLAT]], <vscale x 4 x i1> splat (i1 true), i32 [[TMP1]])
; CHECK-NEXT: [[TMP4:%.*]] = zext i32 [[TMP1]] to i64
; CHECK-NEXT: [[TMP6:%.*]] = sub nuw nsw i64 [[TMP4]], 1
; CHECK-NEXT: [[TMP7:%.*]] = sub i64 0, [[TMP6]]
; CHECK-NEXT: [[TMP9:%.*]] = getelementptr float, ptr [[ARRAYIDX]], i64 [[TMP7]]
; CHECK-NEXT: [[TMP12:%.*]] = call <vscale x 4 x float> @llvm.experimental.vp.reverse.nxv4f32(<vscale x 4 x float> zeroinitializer, <vscale x 4 x i1> splat (i1 true), i32 [[TMP1]])
-; CHECK-NEXT: call void @llvm.vp.store.nxv4f32.p0(<vscale x 4 x float> [[TMP12]], ptr align 4 [[TMP9]], <vscale x 4 x i1> [[VP_REVERSE_MASK]], i32 [[TMP1]])
+; CHECK-NEXT: call void @llvm.vp.store.nxv4f32.p0(<vscale x 4 x float> [[TMP12]], ptr align 4 [[TMP9]], <vscale x 4 x i1> splat (i1 true), i32 [[TMP1]])
; CHECK-NEXT: br label [[LOOP_LATCH2]]
; CHECK: loop.latch2:
; CHECK-NEXT: [[TMP13:%.*]] = zext i32 [[TMP1]] to i64
diff --git a/llvm/test/Transforms/LoopVectorize/RISCV/tail-folding-complex-mask.ll b/llvm/test/Transforms/LoopVectorize/RISCV/tail-folding-complex-mask.ll
index 0a115a9a4aa73..bf54e6219f211 100644
--- a/llvm/test/Transforms/LoopVectorize/RISCV/tail-folding-complex-mask.ll
+++ b/llvm/test/Transforms/LoopVectorize/RISCV/tail-folding-complex-mask.ll
@@ -19,8 +19,6 @@ define void @test(i64 %n, ptr noalias %src0, ptr noalias %src1, ptr noalias %src
; IF-EVL-NEXT: [[BROADCAST_SPLATINSERT1:%.*]] = insertelement <vscale x 4 x i1> poison, i1 [[TMP0]], i64 0
; IF-EVL-NEXT: [[BROADCAST_SPLAT2:%.*]] = shufflevector <vscale x 4 x i1> [[BROADCAST_SPLATINSERT1]], <vscale x 4 x i1> poison, <vscale x 4 x i32> zeroinitializer
; IF-EVL-NEXT: [[TMP1:%.*]] = or <vscale x 4 x i1> [[BROADCAST_SPLAT]], [[BROADCAST_SPLAT2]]
-; IF-EVL-NEXT: [[BROADCAST_SPLATINSERT3:%.*]] = insertelement <vscale x 4 x i1> poison, i1 [[C3]], i64 0
-; IF-EVL-NEXT: [[BROADCAST_SPLAT4:%.*]] = shufflevector <vscale x 4 x i1> [[BROADCAST_SPLATINSERT3]], <vscale x 4 x i1> poison, <vscale x 4 x i32> zeroinitializer
; IF-EVL-NEXT: br label %[[VECTOR_BODY:.*]]
; IF-EVL: [[VECTOR_BODY]]:
; IF-EVL-NEXT: [[EVL_BASED_IV:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_EVL_NEXT:%.*]], %[[LATCH12:.*]] ]
@@ -47,13 +45,13 @@ define void @test(i64 %n, ptr noalias %src0, ptr noalias %src1, ptr noalias %src
; IF-EVL-NEXT: [[PREDPHI8:%.*]] = select <vscale x 4 x i1> [[TMP4]], <vscale x 4 x i32> [[TMP12]], <vscale x 4 x i32> zeroinitializer
; IF-EVL-NEXT: br i1 [[C3]], label %[[LOAD_V210:.*]], label %[[LATCH12]]
; IF-EVL: [[LOAD_V210]]:
-; IF-EVL-NEXT: [[TMP18:%.*]] = getelementptr i32, ptr [[SRC2]], i64 [[EVL_BASED_IV]]
-; IF-EVL-NEXT: [[WIDE_MASKED_LOAD:%.*]] = call <vscale x 4 x i32> @llvm.vp.load.nxv4i32.p0(ptr align 4 [[TMP18]], <vscale x 4 x i1> [[BROADCAST_SPLAT4]], i32 [[TMP7]])
+; IF-EVL-NEXT: [[TMP14:%.*]] = getelementptr inbounds i32, ptr [[SRC2]], i64 [[EVL_BASED_IV]]
+; IF-EVL-NEXT: [[WIDE_MASKED_LOAD:%.*]] = call <vscale x 4 x i32> @llvm.vp.load.nxv4i32.p0(ptr align 4 [[TMP14]], <vscale x 4 x i1> splat (i1 true), i32 [[TMP7]])
; IF-EVL-NEXT: [[TMP19:%.*]] = add <vscale x 4 x i32> [[WIDE_MASKED_LOAD]], [[PREDPHI8]]
; IF-EVL-NEXT: br label %[[LATCH12]]
; IF-EVL: [[LATCH12]]:
; IF-EVL-NEXT: [[TMP15:%.*]] = phi <vscale x 4 x i32> [ poison, %[[LOAD_V17]] ], [ [[TMP19]], %[[LOAD_V210]] ]
-; IF-EVL-NEXT: [[TMP16:%.*]] = phi <vscale x 4 x i1> [ zeroinitializer, %[[LOAD_V17]] ], [ [[BROADCAST_SPLAT4]], %[[LOAD_V210]] ]
+; IF-EVL-NEXT: [[TMP16:%.*]] = phi <vscale x 4 x i1> [ zeroinitializer, %[[LOAD_V17]] ], [ splat (i1 true), %[[LOAD_V210]] ]
; IF-EVL-NEXT: [[PREDPHI9:%.*]] = select <vscale x 4 x i1> [[TMP16]], <vscale x 4 x i32> [[TMP15]], <vscale x 4 x i32> [[PREDPHI8]]
; IF-EVL-NEXT: [[TMP20:%.*]] = getelementptr inbounds i32, ptr [[DST]], i64 [[EVL_BASED_IV]]
; IF-EVL-NEXT: call void @llvm.vp.store.nxv4i32.p0(<vscale x 4 x i32> [[PREDPHI9]], ptr align 4 [[TMP20]], <vscale x 4 x i1> splat (i1 true), i32 [[TMP7]])
@@ -85,8 +83,6 @@ define void @test(i64 %n, ptr noalias %src0, ptr noalias %src1, ptr noalias %src
; NO-VP-NEXT: [[BROADCAST_SPLATINSERT3:%.*]] = insertelement <vscale x 4 x i1> poison, i1 [[TMP4]], i64 0
; NO-VP-NEXT: [[BROADCAST_SPLAT4:%.*]] = shufflevector <vscale x 4 x i1> [[BROADCAST_SPLATINSERT3]], <vscale x 4 x i1> poison, <vscale x 4 x i32> zeroinitializer
; NO-VP-NEXT: [[TMP9:%.*]] = or <vscale x 4 x i1> [[BROADCAST_SPLAT2]], [[BROADCAST_SPLAT4]]
-; NO-VP-NEXT: [[BROADCAST_SPLATINSERT4:%.*]] = insertelement <vscale x 4 x i1> poison, i1 [[C3]], i64 0
-; NO-VP-NEXT: [[TMP12:%.*]] = shufflevector <vscale x 4 x i1> [[BROADCAST_SPLATINSERT4]], <vscale x 4 x i1> poison, <vscale x 4 x i32> zeroinitializer
; NO-VP-NEXT: br label %[[VECTOR_BODY:.*]]
; NO-VP: [[VECTOR_BODY]]:
; NO-VP-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[LATCH12:.*]] ]
@@ -111,13 +107,13 @@ define void @test(i64 %n, ptr noalias %src0, ptr noalias %src1, ptr noalias %src
; NO-VP-NEXT: [[PREDPHI6:%.*]] = select <vscale x 4 x i1> [[TMP8]], <vscale x 4 x i32> [[TMP15]], <vscale x 4 x i32> zeroinitializer
; NO-VP-NEXT: br i1 [[C3]], label %[[LOAD_V210:.*]], label %[[LATCH12]]
; NO-VP: [[LOAD_V210]]:
-; NO-VP-NEXT: [[TMP16:%.*]] = getelementptr i32, ptr [[SRC2]], i64 [[INDEX]]
-; NO-VP-NEXT: [[WIDE_MASKED_LOAD7:%.*]] = call <vscale x 4 x i32> @llvm.masked.load.nxv4i32.p0(ptr align 4 [[TMP16]], <vscale x 4 x i1> [[TMP12]], <vscale x 4 x i32> poison)
+; NO-VP-NEXT: [[TMP16:%.*]] = getelementptr inbounds i32, ptr [[SRC2]], i64 [[INDEX]]
+; NO-VP-NEXT: [[WIDE_MASKED_LOAD7:%.*]] = load <vscale x 4 x i32>, ptr [[TMP16]], align 4
; NO-VP-NEXT: [[TMP17:%.*]] = add <vscale x 4 x i32> [[WIDE_MASKED_LOAD7]], [[PREDPHI6]]
; NO-VP-NEXT: br label %[[LATCH12]]
; NO-VP: [[LATCH12]]:
; NO-VP-NEXT: [[TMP22:%.*]] = phi <vscale x 4 x i32> [ poison, %[[LOAD_V17]] ], [ [[TMP17]], %[[LOAD_V210]] ]
-; NO-VP-NEXT: [[TMP23:%.*]] = phi <vscale x 4 x i1> [ zeroinitializer, %[[LOAD_V17]] ], [ [[TMP12]], %[[LOAD_V210]] ]
+; NO-VP-NEXT: [[TMP23:%.*]] = phi <vscale x 4 x i1> [ zeroinitializer, %[[LOAD_V17]] ], [ splat (i1 true), %[[LOAD_V210]] ]
; NO-VP-NEXT: [[PREDPHI8:%.*]] = select <vscale x 4 x i1> [[TMP23]], <vscale x 4 x i32> [[TMP22]], <vscale x 4 x i32> [[PREDPHI6]]
; NO-VP-NEXT: [[TMP18:%.*]] = getelementptr inbounds i32, ptr [[DST]], i64 [[INDEX]]
; NO-VP-NEXT: store <vscale x 4 x i32> [[PREDPHI8]], ptr [[TMP18]], align 4
diff --git a/llvm/test/Transforms/LoopVectorize/VPlan/dissolve-replicate-regions.ll b/llvm/test/Transforms/LoopVectorize/VPlan/dissolve-replicate-regions.ll
index dcbc4ed144e14..9c4edb07a8b91 100644
--- a/llvm/test/Transforms/LoopVectorize/VPlan/dissolve-replicate-regions.ll
+++ b/llvm/test/Transforms/LoopVectorize/VPlan/dissolve-replicate-regions.ll
@@ -31,36 +31,21 @@ define void @predicated_load(i1 %c, ptr %ptr, ptr %dst) {
; SCALAR-NEXT: vector.body:
; SCALAR-NEXT: vp<[[VP6:%[0-9]+]]> = SCALAR-STEPS vp<[[VP5]]>, ir<1>, vp<[[VP0]]>
; SCALAR-NEXT: vp<[[VP7:%[0-9]+]]> = SCALAR-STEPS vp<[[VP5]]>, ir<1>, vp<[[VP0]]>, vp<[[VP0]]>
+; SCALAR-NEXT: CLONE ir<%gep> = getelementptr ir<%ptr>, vp<[[VP6]]>
; SCALAR-NEXT: EMIT branch-on-cond ir<%c>
; SCALAR-NEXT: Successor(s): if.then, loop.latch
; SCALAR-EMPTY:
; SCALAR-NEXT: if.then:
-; SCALAR-NEXT: EMIT branch-on-cond ir<%c>
-; SCALAR-NEXT: Successor(s): pred.load.if, pred.load.continue
-; SCALAR-EMPTY:
-; SCALAR-NEXT: pred.load.if:
-; SCALAR-NEXT: CLONE ir<%gep> = getelementptr ir<%ptr>, vp<[[VP6]]>
; SCALAR-NEXT: CLONE ir<%lv> = load ir<%gep>
-; SCALAR-NEXT: Successor(s): pred.load.continue
-; SCALAR-EMPTY:
-; SCALAR-NEXT: pred.load.continue:
-; SCALAR-NEXT: EMIT-SCALAR vp<[[VP9:%[0-9]+]]> = phi [ ir<poison>, if.then ], [ ir<%lv>, pred.load.if ]
-; SCALAR-NEXT: EMIT branch-on-cond ir<%c>
-; SCALAR-NEXT: Successor(s): pred.load.if, pred.load.continue
-; SCALAR-EMPTY:
-; SCALAR-NEXT: pred.load.if:
-; SCALAR-NEXT: Successor(s): pred.load.continue
-; SCALAR-EMPTY:
-; SCALAR-NEXT: pred.load.continue:
; SCALAR-NEXT: Successor(s): loop.latch
; SCALAR-EMPTY:
; SCALAR-NEXT: loop.latch:
-; SCALAR-NEXT: WIDEN-PHI vp<[[VP11:%[0-9]+]]> = phi [ ir<poison>, vector.body ], [ vp<[[VP9]]>, pred.load.continue ]
-; SCALAR-NEXT: WIDEN-PHI vp<[[VP12:%[0-9]+]]> = phi [ ir<poison>, vector.body ], [ vp<[[VP9]]>, pred.load.continue ]
-; SCALAR-NEXT: WIDEN-PHI vp<[[VP13:%[0-9]+]]> = phi [ ir<false>, vector.body ], [ ir<%c>, pred.load.continue ]
-; SCALAR-NEXT: WIDEN-PHI vp<[[VP14:%[0-9]+]]> = phi [ ir<false>, vector.body ], [ ir<%c>, pred.load.continue ]
-; SCALAR-NEXT: BLEND ir<%pred.val> = ir<0> vp<%11>/vp<[[VP13]]>
-; SCALAR-NEXT: BLEND ir<%pred.val>.1 = ir<0> vp<%12>/vp<[[VP14]]>
+; SCALAR-NEXT: WIDEN-PHI vp<[[VP8:%[0-9]+]]> = phi [ ir<poison>, vector.body ], [ ir<%lv>, if.then ]
+; SCALAR-NEXT: WIDEN-PHI vp<[[VP9:%[0-9]+]]> = phi [ ir<poison>, vector.body ], [ ir<%lv>, if.then ]
+; SCALAR-NEXT: WIDEN-PHI vp<[[VP10:%[0-9]+]]> = phi [ ir<false>, vector.body ], [ ir<true>, if.then ]
+; SCALAR-NEXT: WIDEN-PHI vp<[[VP11:%[0-9]+]]> = phi [ ir<false>, vector.body ], [ ir<true>, if.then ]
+; SCALAR-NEXT: BLEND ir<%pred.val> = ir<0> vp<%8>/vp<[[VP10]]>
+; SCALAR-NEXT: BLEND ir<%pred.val>.1 = ir<0> vp<%9>/vp<[[VP11]]>
; SCALAR-NEXT: CLONE ir<%gep.dst> = getelementptr ir<%dst>, vp<[[VP6]]>
; SCALAR-NEXT: CLONE ir<%gep.dst>.1 = getelementptr ir<%dst>, vp<[[VP7]]>
; SCALAR-NEXT: CLONE store ir<%pred.val>, ir<%gep.dst>
@@ -108,72 +93,36 @@ define void @predicated_load(i1 %c, ptr %ptr, ptr %dst) {
; VECTOR-NEXT: Successor(s): scalar.ph, vector.ph
; VECTOR-EMPTY:
; VECTOR-NEXT: vector.ph:
-; VECTOR-NEXT: EMIT vp<[[VP5:%[0-9]+]]> = broadcast ir<%c>
; VECTOR-NEXT: Successor(s): vector loop
; VECTOR-EMPTY:
; VECTOR-NEXT: <x1> vector loop: {
-; VECTOR-NEXT: vp<[[VP6:%[0-9]+]]> = CANONICAL-IV
+; VECTOR-NEXT: vp<[[VP5:%[0-9]+]]> = CANONICAL-IV
; VECTOR-EMPTY:
; VECTOR-NEXT: vector.body:
-; VECTOR-NEXT: vp<[[VP7:%[0-9]+]]> = SCALAR-STEPS vp<[[VP6]]>, ir<1>, vp<[[VP0]]>
+; VECTOR-NEXT: vp<[[VP6:%[0-9]+]]> = SCALAR-STEPS vp<[[VP5]]>, ir<1>, vp<[[VP0]]>
+; VECTOR-NEXT: CLONE ir<%gep> = getelementptr ir<%ptr>, vp<[[VP6]]>
; VECTOR-NEXT: EMIT branch-on-cond ir<%c>
; VECTOR-NEXT: Successor(s): if.then, loop.latch
; VECTOR-EMPTY:
; VECTOR-NEXT: if.then:
-; VECTOR-NEXT: EMIT branch-on-cond ir<%c>
-; VECTOR-NEXT: Successor(s): pred.load.if, pred.load.continue
-; VECTOR-EMPTY:
-; VECTOR-NEXT: pred.load.if:
-; VECTOR-NEXT: vp<[[VP9:%[0-9]+]]> = SCALAR-STEPS vp<[[VP6]]>, ir<1>, vp<[[VP0]]>
-; VECTOR-NEXT: CLONE ir<%gep> = getelementptr ir<%ptr>, vp<[[VP9]]>
-; VECTOR-NEXT: CLONE ir<%lv> = load ir<%gep>
-; VECTOR-NEXT: EMIT vp<[[VP10:%[0-9]+]]> = insertelement ir<poison>, ir<%lv>, ir<0>
-; VECTOR-NEXT: Successor(s): pred.load.continue
-; VECTOR-EMPTY:
-; VECTOR-NEXT: pred.load.continue:
-; VECTOR-NEXT: WIDEN-PHI vp<[[VP11:%[0-9]+]]> = phi [ ir<poison>, if.then ], [ vp<[[VP10]]>, pred.load.if ]
-; VECTOR-NEXT: EMIT branch-on-cond ir<%c>
-; VECTOR-NEXT: Successor(s): pred.load.if, pred.load.continue
-; VECTOR-EMPTY:
-; VECTOR-NEXT: pred.load.if:
-; VECTOR-NEXT: vp<[[VP13:%[0-9]+]]> = SCALAR-STEPS vp<[[VP6]]>, ir<1>, vp<[[VP0]]>, ir<1>
-; VECTOR-NEXT: CLONE ir<%gep>.1 = getelementptr ir<%ptr>, vp<[[VP13]]>
-; VECTOR-NEXT: CLONE ir<%lv>.1 = load ir<%gep>.1
-; VECTOR-NEXT: EMIT vp<[[VP14:%[0-9]+]]> = insertelement vp<[[VP11]]>, ir<%lv>.1, ir<1>
-; VECTOR-NEXT: Successor(s): pred.load.continue
-; VECTOR-EMPTY:
-; VECTOR-NEXT: pred.load.continue:
-; VECTOR-NEXT: WIDEN-PHI vp<[[VP15:%[0-9]+]]> = phi [ vp<[[VP11]]>, pred.load.continue ], [ vp<[[VP14]]>, pred.load.if ]
-; VECTOR-NEXT: EMIT branch-on-cond ir<%c>
-; VECTOR-NEXT: Successor(s): pred.load.if, pred.load.continue
-; VECTOR-EMPTY:
-; VECTOR-NEXT: pred.load.if:
-; VECTOR-NEXT: Successor(s): pred.load.continue
-; VECTOR-EMPTY:
-; VECTOR-NEXT: pred.load.continue:
-; VECTOR-NEXT: EMIT branch-on-cond ir<%c>
-; VECTOR-NEXT: Successor(s): pred.load.if, pred.load.continue
-; VECTOR-EMPTY:
-; VECTOR-NEXT: pred.load.if:
-; VECTOR-NEXT: Successor(s): pred.load.continue
-; VECTOR-EMPTY:
-; VECTOR-NEXT: pred.load.continue:
+; VECTOR-NEXT: vp<[[VP7:%[0-9]+]]> = vector-pointer i8, ir<%gep>, ir<1>
+; VECTOR-NEXT: WIDEN ir<%lv> = load vp<[[VP7]]>
; VECTOR-NEXT: Successor(s): loop.latch
; VECTOR-EMPTY:
; VECTOR-NEXT: loop.latch:
-; VECTOR-NEXT: WIDEN-PHI vp<[[VP18:%[0-9]+]]> = phi [ ir<poison>, vector.body ], [ vp<[[VP15]]>, pred.load.continue ]
-; VECTOR-NEXT: WIDEN-PHI vp<[[VP19:%[0-9]+]]> = phi [ ir<poison>, vector.body ], [ vp<[[VP15]]>, pred.load.continue ]
-; VECTOR-NEXT: WIDEN-PHI vp<[[VP20:%[0-9]+]]> = phi [ ir<false>, vector.body ], [ vp<[[VP5]]>, pred.load.continue ]
-; VECTOR-NEXT: WIDEN-PHI vp<[[VP21:%[0-9]+]]> = phi [ ir<false>, vector.body ], [ vp<[[VP5]]>, pred.load.continue ]
-; VECTOR-NEXT: BLEND ir<%pred.val> = ir<0> vp<%18>/vp<[[VP20]]>
-; VECTOR-NEXT: BLEND ir<%pred.val>.1 = ir<0> vp<%19>/vp<[[VP21]]>
-; VECTOR-NEXT: CLONE ir<%gep.dst> = getelementptr ir<%dst>, vp<[[VP7]]>
-; VECTOR-NEXT: EMIT vp<[[VP22:%[0-9]+]]> = mul nuw nsw vp<[[VP0]]>, ir<1>
-; VECTOR-NEXT: vp<[[VP23:%[0-9]+]]> = vector-pointer i8, ir<%gep.dst>, ir<1>
-; VECTOR-NEXT: vp<[[VP24:%[0-9]+]]> = vector-pointer i8, ir<%gep.dst>, ir<1>, vp<[[VP22]]>
-; VECTOR-NEXT: WIDEN store vp<[[VP23]]>, ir<%pred.val>
-; VECTOR-NEXT: WIDEN store vp<[[VP24]]>, ir<%pred.val>.1
-; VECTOR-NEXT: EMIT vp<%index.next> = add nuw vp<[[VP6]]>, vp<[[VP1]]>
+; VECTOR-NEXT: WIDEN-PHI vp<[[VP8:%[0-9]+]]> = phi [ ir<poison>, vector.body ], [ ir<%lv>, if.then ]
+; VECTOR-NEXT: WIDEN-PHI vp<[[VP9:%[0-9]+]]> = phi [ ir<poison>, vector.body ], [ ir<%lv>, if.then ]
+; VECTOR-NEXT: WIDEN-PHI vp<[[VP10:%[0-9]+]]> = phi [ ir<false>, vector.body ], [ ir<true>, if.then ]
+; VECTOR-NEXT: WIDEN-PHI vp<[[VP11:%[0-9]+]]> = phi [ ir<false>, vector.body ], [ ir<true>, if.then ]
+; VECTOR-NEXT: BLEND ir<%pred.val> = ir<0> vp<%8>/vp<[[VP10]]>
+; VECTOR-NEXT: BLEND ir<%pred.val>.1 = ir<0> vp<%9>/vp<[[VP11]]>
+; VECTOR-NEXT: CLONE ir<%gep.dst> = getelementptr ir<%dst>, vp<[[VP6]]>
+; VECTOR-NEXT: EMIT vp<[[VP12:%[0-9]+]]> = mul nuw nsw vp<[[VP0]]>, ir<1>
+; VECTOR-NEXT: vp<[[VP13:%[0-9]+]]> = vector-pointer i8, ir<%gep.dst>, ir<1>
+; VECTOR-NEXT: vp<[[VP14:%[0-9]+]]> = vector-pointer i8, ir<%gep.dst>, ir<1>, vp<[[VP12]]>
+; VECTOR-NEXT: WIDEN store vp<[[VP13]]>, ir<%pred.val>
+; VECTOR-NEXT: WIDEN store vp<[[VP14]]>, ir<%pred.val>.1
+; VECTOR-NEXT: EMIT vp<%index.next> = add nuw vp<[[VP5]]>, vp<[[VP1]]>
; VECTOR-NEXT: EMIT branch-on-count vp<%index.next>, vp<[[VP2]]>
; VECTOR-NEXT: No successors
; VECTOR-NEXT: }
diff --git a/llvm/test/Transforms/LoopVectorize/VPlan/phi-with-fastflags-vplan.ll b/llvm/test/Transforms/LoopVectorize/VPlan/phi-with-fastflags-vplan.ll
index af349f102c2d2..190df273b72b5 100644
--- a/llvm/test/Transforms/LoopVectorize/VPlan/phi-with-fastflags-vplan.ll
+++ b/llvm/test/Transforms/LoopVectorize/VPlan/phi-with-fastflags-vplan.ll
@@ -30,7 +30,7 @@ define void @f(ptr noalias %p, i1 %c) {
; CHECK-NEXT: Successor(s): latch
; CHECK-EMPTY:
; CHECK-NEXT: latch:
-; CHECK-NEXT: WIDEN-PHI vp<[[VP6:%[0-9]+]]> = phi [ ir<false>, vector.body ], [ ir<%c>, then ]
+; CHECK-NEXT: WIDEN-PHI vp<[[VP6:%[0-9]+]]> = phi [ ir<false>, vector.body ], [ ir<true>, then ]
; CHECK-NEXT: BLEND ir<%phi> = fast ir<%x> ir<0.000000e+00>/vp<[[VP6]]>
; CHECK-NEXT: vp<[[VP7:%[0-9]+]]> = vector-pointer float, ir<%gep>, ir<1>
; CHECK-NEXT: WIDEN store vp<[[VP7]]>, ir<%phi>
diff --git a/llvm/test/Transforms/LoopVectorize/VPlan/predicator.ll b/llvm/test/Transforms/LoopVectorize/VPlan/predicator.ll
index 95cb8fa53d786..c08935e982f5e 100644
--- a/llvm/test/Transforms/LoopVectorize/VPlan/predicator.ll
+++ b/llvm/test/Transforms/LoopVectorize/VPlan/predicator.ll
@@ -774,13 +774,13 @@ define void @simplifiable_blend(i1 %c1, i1 %c2, i1 %c3, i32 %x, i32 %y, ptr %p)
; CHECK-NEXT: Successor(s): C, D
; CHECK-EMPTY:
; CHECK-NEXT: C:
-; CHECK-NEXT: EMIT vp<[[VP9:%[0-9]+]]> = logical-and ir<%c1>, ir<%c2>
+; CHECK-NEXT: EMIT vp<[[VP7:%[0-9]+]]> = logical-and ir<%c1>, ir<%c2>
; CHECK-NEXT: Successor(s): latch
; CHECK-EMPTY:
; CHECK-NEXT: latch:
-; CHECK-NEXT: WIDEN-PHI vp<[[VP10:%[0-9]+]]> = phi [ vp<[[VP4:%[0-9]+]]>, F ], [ ir<false>, D ], [ ir<false>, C ]
-; CHECK-NEXT: WIDEN-PHI vp<[[VP11:%[0-9]+]]> = phi [ ir<false>, F ], [ ir<%c1>, D ], [ ir<%c1>, C ]
-; CHECK-NEXT: BLEND ir<%phi> = ir<%y>/vp<[[VP10]]> ir<%x>/vp<[[VP11]]>
+; CHECK-NEXT: WIDEN-PHI vp<[[VP8:%[0-9]+]]> = phi [ vp<[[VP4:%[0-9]+]]>, F ], [ ir<false>, D ], [ ir<false>, C ]
+; CHECK-NEXT: WIDEN-PHI vp<[[VP9:%[0-9]+]]> = phi [ ir<false>, F ], [ ir<%c1>, D ], [ ir<%c1>, C ]
+; CHECK-NEXT: BLEND ir<%phi> = ir<%y>/vp<[[VP8]]> ir<%x>/vp<[[VP9]]>
; CHECK-NEXT: EMIT ir<%gep> = getelementptr ir<%p>, ir<%iv>
; CHECK-NEXT: EMIT store ir<%phi>, ir<%gep>
; CHECK-NEXT: EMIT ir<%iv.next> = add ir<%iv>, ir<1>
@@ -790,8 +790,8 @@ define void @simplifiable_blend(i1 %c1, i1 %c2, i1 %c3, i32 %x, i32 %y, ptr %p)
; CHECK-NEXT: No successors
; CHECK-EMPTY:
; CHECK-NEXT: D:
-; CHECK-NEXT: EMIT vp<[[VP7:%[0-9]+]]> = not ir<%c2>
-; CHECK-NEXT: EMIT vp<[[VP8:%[0-9]+]]> = logical-and ir<%c1>, vp<[[VP7]]>
+; CHECK-NEXT: EMIT vp<[[VP5:%[0-9]+]]> = not ir<%c2>
+; CHECK-NEXT: EMIT vp<[[VP6:%[0-9]+]]> = logical-and ir<%c1>, vp<[[VP5]]>
; CHECK-NEXT: Successor(s): latch
; CHECK-EMPTY:
; CHECK-NEXT: B:
@@ -803,8 +803,6 @@ define void @simplifiable_blend(i1 %c1, i1 %c2, i1 %c3, i32 %x, i32 %y, ptr %p)
; CHECK-NEXT: Successor(s): latch
; CHECK-EMPTY:
; CHECK-NEXT: E:
-; CHECK-NEXT: EMIT vp<[[VP5:%[0-9]+]]> = not ir<%c3>
-; CHECK-NEXT: EMIT vp<[[VP6:%[0-9]+]]> = logical-and vp<[[VP4]]>, vp<[[VP5]]>
; CHECK-NEXT: Successor(s): F
; CHECK-NEXT: }
; CHECK-NEXT: Successor(s): middle.block
@@ -990,24 +988,23 @@ define void @outermost_uniform_branch(ptr %a, i1 %u0) {
; CHECK-NEXT: Successor(s): bb1, bb4
; CHECK-EMPTY:
; CHECK-NEXT: bb1:
-; CHECK-NEXT: EMIT ir<%add1> = add ir<%iv>, ir<1>, ir<%u0>
-; CHECK-NEXT: EMIT ir<%v1> = icmp sle ir<%iv>, ir<1>, ir<%u0>
+; CHECK-NEXT: EMIT ir<%add1> = add ir<%iv>, ir<1>
+; CHECK-NEXT: EMIT ir<%v1> = icmp sle ir<%iv>, ir<1>
; CHECK-NEXT: Successor(s): bb2
; CHECK-EMPTY:
; CHECK-NEXT: bb2:
-; CHECK-NEXT: EMIT vp<[[VP4:%[0-9]+]]> = logical-and ir<%u0>, ir<%v1>
-; CHECK-NEXT: EMIT ir<%add2> = add ir<%iv>, ir<2>, vp<[[VP4]]>
+; CHECK-NEXT: EMIT ir<%add2> = add ir<%iv>, ir<2>, ir<%v1>
; CHECK-NEXT: Successor(s): bb3
; CHECK-EMPTY:
; CHECK-NEXT: bb3:
; CHECK-NEXT: BLEND ir<%phi3> = ir<%add1>/ir<true> ir<%add2>/ir<%v1>
-; CHECK-NEXT: EMIT ir<%add3> = add ir<%phi3>, ir<3>, ir<%u0>
+; CHECK-NEXT: EMIT ir<%add3> = add ir<%phi3>, ir<3>
; CHECK-NEXT: Successor(s): bb4
; CHECK-EMPTY:
; CHECK-NEXT: bb4:
-; CHECK-NEXT: WIDEN-PHI vp<[[VP5:%[0-9]+]]> = phi [ ir<poison>, vector.body ], [ ir<%add3>, bb3 ]
-; CHECK-NEXT: WIDEN-PHI vp<[[VP6:%[0-9]+]]> = phi [ ir<false>, vector.body ], [ ir<%u0>, bb3 ]
-; CHECK-NEXT: BLEND ir<%phi4> = ir<%iv>/ir<true> vp<%5>/vp<[[VP6]]>
+; CHECK-NEXT: WIDEN-PHI vp<[[VP4:%[0-9]+]]> = phi [ ir<poison>, vector.body ], [ ir<%add3>, bb3 ]
+; CHECK-NEXT: WIDEN-PHI vp<[[VP5:%[0-9]+]]> = phi [ ir<false>, vector.body ], [ ir<true>, bb3 ]
+; CHECK-NEXT: BLEND ir<%phi4> = ir<%iv>/ir<true> vp<%4>/vp<[[VP5]]>
; CHECK-NEXT: EMIT store ir<%phi4>, ir<%a>
; CHECK-NEXT: EMIT ir<%iv.next> = add nuw nsw ir<%iv>, ir<1>
; CHECK-NEXT: EMIT ir<%ec> = icmp eq ir<%iv.next>, ir<128>
@@ -1073,15 +1070,14 @@ define void @outermost_uniform_branch_no_phi(ptr %a, i1 %u0) {
; CHECK-NEXT: Successor(s): bb1, bb4
; CHECK-EMPTY:
; CHECK-NEXT: bb1:
-; CHECK-NEXT: EMIT ir<%add1> = add ir<%iv>, ir<1>, ir<%u0>
-; CHECK-NEXT: EMIT store ir<%add1>, ir<%gep>, ir<%u0>
-; CHECK-NEXT: EMIT ir<%v1> = icmp sle ir<%iv>, ir<1>, ir<%u0>
+; CHECK-NEXT: EMIT ir<%add1> = add ir<%iv>, ir<1>
+; CHECK-NEXT: EMIT store ir<%add1>, ir<%gep>
+; CHECK-NEXT: EMIT ir<%v1> = icmp sle ir<%iv>, ir<1>
; CHECK-NEXT: Successor(s): bb2
; CHECK-EMPTY:
; CHECK-NEXT: bb2:
-; CHECK-NEXT: EMIT vp<[[VP4:%[0-9]+]]> = logical-and ir<%u0>, ir<%v1>
-; CHECK-NEXT: EMIT ir<%add2> = add ir<%iv>, ir<2>, vp<[[VP4]]>
-; CHECK-NEXT: EMIT store ir<%add2>, ir<%gep>, vp<[[VP4]]>
+; CHECK-NEXT: EMIT ir<%add2> = add ir<%iv>, ir<2>, ir<%v1>
+; CHECK-NEXT: EMIT store ir<%add2>, ir<%gep>, ir<%v1>
; CHECK-NEXT: Successor(s): bb3
; CHECK-EMPTY:
; CHECK-NEXT: bb3:
@@ -1352,14 +1348,13 @@ define void @uniform_branch_after_varying_branch(ptr %a, i1 %u1) {
; CHECK-NEXT: Successor(s): bb2, bb3
; CHECK-EMPTY:
; CHECK-NEXT: bb2:
-; CHECK-NEXT: EMIT vp<[[VP5:%[0-9]+]]> = logical-and vp<[[VP4]]>, ir<%u1>
-; CHECK-NEXT: EMIT ir<%add2> = add ir<%iv>, ir<2>, vp<[[VP5]]>
+; CHECK-NEXT: EMIT ir<%add2> = add ir<%iv>, ir<2>, vp<[[VP4]]>
; CHECK-NEXT: Successor(s): bb3
; CHECK-EMPTY:
; CHECK-NEXT: bb3:
-; CHECK-NEXT: WIDEN-PHI vp<[[VP6:%[0-9]+]]> = phi [ ir<poison>, bb1 ], [ ir<%add2>, bb2 ]
-; CHECK-NEXT: WIDEN-PHI vp<[[VP7:%[0-9]+]]> = phi [ ir<false>, bb1 ], [ ir<%u1>, bb2 ]
-; CHECK-NEXT: BLEND ir<%phi3> = ir<%add1>/ir<true> vp<%6>/vp<[[VP7]]>
+; CHECK-NEXT: WIDEN-PHI vp<[[VP5:%[0-9]+]]> = phi [ ir<poison>, bb1 ], [ ir<%add2>, bb2 ]
+; CHECK-NEXT: WIDEN-PHI vp<[[VP6:%[0-9]+]]> = phi [ ir<false>, bb1 ], [ ir<true>, bb2 ]
+; CHECK-NEXT: BLEND ir<%phi3> = ir<%add1>/ir<true> vp<%5>/vp<[[VP6]]>
; CHECK-NEXT: EMIT ir<%add3> = add ir<%phi3>, ir<3>, vp<[[VP4]]>
; CHECK-NEXT: Successor(s): bb4
; CHECK-EMPTY:
@@ -1442,9 +1437,8 @@ define void @uniform_branch_after_varying_branch_no_phi(ptr %a, i1 %u1) {
; CHECK-NEXT: Successor(s): bb2, bb3
; CHECK-EMPTY:
; CHECK-NEXT: bb2:
-; CHECK-NEXT: EMIT vp<[[VP5:%[0-9]+]]> = logical-and vp<[[VP4]]>, ir<%u1>
-; CHECK-NEXT: EMIT ir<%add2> = add ir<%iv>, ir<2>, vp<[[VP5]]>
-; CHECK-NEXT: EMIT store ir<%add2>, ir<%gep>, vp<[[VP5]]>
+; CHECK-NEXT: EMIT ir<%add2> = add ir<%iv>, ir<2>, vp<[[VP4]]>
+; CHECK-NEXT: EMIT store ir<%add2>, ir<%gep>, vp<[[VP4]]>
; CHECK-NEXT: Successor(s): bb3
; CHECK-EMPTY:
; CHECK-NEXT: bb3:
diff --git a/llvm/test/Transforms/LoopVectorize/VPlan/vplan-sink-scalars-and-merge-vf1.ll b/llvm/test/Transforms/LoopVectorize/VPlan/vplan-sink-scalars-and-merge-vf1.ll
index c846b26096772..6595787aefd9d 100644
--- a/llvm/test/Transforms/LoopVectorize/VPlan/vplan-sink-scalars-and-merge-vf1.ll
+++ b/llvm/test/Transforms/LoopVectorize/VPlan/vplan-sink-scalars-and-merge-vf1.ll
@@ -32,23 +32,7 @@ define void @sink_with_sideeffects(i1 %c, ptr %ptr) {
; CHECK-NEXT: Successor(s): if.then, for.inc
; CHECK-EMPTY:
; CHECK-NEXT: if.then:
-; CHECK-NEXT: Successor(s): pred.store
-; CHECK-EMPTY:
-; CHECK-NEXT: <xVFxUF> pred.store: {
-; CHECK-NEXT: pred.store.entry:
-; CHECK-NEXT: BRANCH-ON-MASK ir<%c>
-; CHECK-NEXT: Successor(s): pred.store.if, pred.store.continue
-; CHECK-EMPTY:
-; CHECK-NEXT: pred.store.if:
-; CHECK-NEXT: CLONE store ir<%tmp3>, ir<%tmp2>
-; CHECK-NEXT: Successor(s): pred.store.continue
-; CHECK-EMPTY:
-; CHECK-NEXT: pred.store.continue:
-; CHECK-NEXT: No successors
-; CHECK-NEXT: }
-; CHECK-NEXT: Successor(s): if.then.0
-; CHECK-EMPTY:
-; CHECK-NEXT: if.then.0:
+; CHECK-NEXT: CLONE store ir<%tmp3>, ir<%tmp2>
; CHECK-NEXT: Successor(s): for.inc
; CHECK-EMPTY:
; CHECK-NEXT: for.inc:
diff --git a/llvm/test/Transforms/LoopVectorize/VPlan/vplan-sink-scalars-and-merge.ll b/llvm/test/Transforms/LoopVectorize/VPlan/vplan-sink-scalars-and-merge.ll
index c8503ebab5e45..43acb39578a7d 100644
--- a/llvm/test/Transforms/LoopVectorize/VPlan/vplan-sink-scalars-and-merge.ll
+++ b/llvm/test/Transforms/LoopVectorize/VPlan/vplan-sink-scalars-and-merge.ll
@@ -1164,27 +1164,13 @@ define void @update_multiple_users(ptr noalias %src, ptr noalias %dst, i1 %c) {
; CHECK-NEXT: Successor(s): loop.then, loop.latch
; CHECK-EMPTY:
; CHECK-NEXT: loop.then:
-; CHECK-NEXT: Successor(s): pred.store
-; CHECK-EMPTY:
-; CHECK-NEXT: <xVFxUF> pred.store: {
-; CHECK-NEXT: pred.store.entry:
-; CHECK-NEXT: BRANCH-ON-MASK ir<%c>
-; CHECK-NEXT: Successor(s): pred.store.if, pred.store.continue
-; CHECK-EMPTY:
-; CHECK-NEXT: pred.store.if:
-; CHECK-NEXT: REPLICATE ir<%l1> = load ir<%src>
-; CHECK-NEXT: REPLICATE ir<%l2> = trunc ir<%l1>
-; CHECK-NEXT: REPLICATE ir<%cmp> = icmp eq ir<%l1>, ir<0>
-; CHECK-NEXT: REPLICATE ir<%sel> = select ir<%cmp>, ir<5>, ir<%l2>
-; CHECK-NEXT: REPLICATE store ir<%sel>, ir<%dst>
-; CHECK-NEXT: Successor(s): pred.store.continue
-; CHECK-EMPTY:
-; CHECK-NEXT: pred.store.continue:
-; CHECK-NEXT: No successors
-; CHECK-NEXT: }
-; CHECK-NEXT: Successor(s): loop.then.1
-; CHECK-EMPTY:
-; CHECK-NEXT: loop.then.1:
+; CHECK-NEXT: REPLICATE ir<%l1> = load ir<%src>
+; CHECK-NEXT: REPLICATE ir<%l2> = trunc ir<%l1>
+; CHECK-NEXT: REPLICATE ir<%cmp> = icmp eq ir<%l1>, ir<0>
+; CHECK-NEXT: REPLICATE ir<%sel> = select ir<%cmp>, ir<5>, ir<%l2>
+; CHECK-NEXT: EMIT vp<[[VP3:%[0-9]+]]> = extract-last-part ir<%sel>
+; CHECK-NEXT: EMIT vp<[[VP4:%[0-9]+]]> = extract-last-lane vp<[[VP3]]>
+; CHECK-NEXT: CLONE store vp<[[VP4]]>, ir<%dst>
; CHECK-NEXT: Successor(s): loop.latch
; CHECK-EMPTY:
; CHECK-NEXT: loop.latch:
diff --git a/llvm/test/Transforms/LoopVectorize/X86/cost-conditional-branches.ll b/llvm/test/Transforms/LoopVectorize/X86/cost-conditional-branches.ll
index 207e203722f1c..78bb964a3bb50 100644
--- a/llvm/test/Transforms/LoopVectorize/X86/cost-conditional-branches.ll
+++ b/llvm/test/Transforms/LoopVectorize/X86/cost-conditional-branches.ll
@@ -868,442 +868,23 @@ exit:
; Test case for https://github.com/llvm/llvm-project/issues/158660.
define i64 @test_predicated_udiv(i32 %d, i1 %c) #2 {
; CHECK-LABEL: @test_predicated_udiv(
-; CHECK-NEXT: iter.check:
-; CHECK-NEXT: br i1 false, label [[VEC_EPILOG_SCALAR_PH:%.*]], label [[VECTOR_MAIN_LOOP_ITER_CHECK:%.*]]
-; CHECK: vector.main.loop.iter.check:
-; CHECK-NEXT: br i1 false, label [[VEC_EPILOG_PH:%.*]], label [[VECTOR_PH:%.*]]
-; CHECK: vector.ph:
-; CHECK-NEXT: [[BROADCAST_SPLATINSERT1:%.*]] = insertelement <32 x i1> poison, i1 [[C:%.*]], i64 0
-; CHECK-NEXT: [[BROADCAST_SPLAT1:%.*]] = shufflevector <32 x i1> [[BROADCAST_SPLATINSERT1]], <32 x i1> poison, <32 x i32> zeroinitializer
-; CHECK-NEXT: [[TMP131:%.*]] = xor <32 x i1> [[BROADCAST_SPLAT1]], splat (i1 true)
-; CHECK-NEXT: br label [[VECTOR_BODY1:%.*]]
-; CHECK: vector.body:
-; CHECK-NEXT: [[INDEX:%.*]] = phi i32 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[PRED_UDIV_CONTINUE62:%.*]] ]
-; CHECK-NEXT: [[VEC_IND:%.*]] = phi <32 x i32> [ <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15, i32 16, i32 17, i32 18, i32 19, i32 20, i32 21, i32 22, i32 23, i32 24, i32 25, i32 26, i32 27, i32 28, i32 29, i32 30, i32 31>, [[VECTOR_PH]] ], [ [[VEC_IND_NEXT:%.*]], [[PRED_UDIV_CONTINUE62]] ]
-; CHECK-NEXT: br i1 [[C]], label [[PRED_UDIV_CONTINUE62]], label [[VECTOR_BODY:%.*]]
-; CHECK: then1:
-; CHECK-NEXT: [[TMP1:%.*]] = call <32 x i32> @llvm.usub.sat.v32i32(<32 x i32> [[VEC_IND]], <32 x i32> splat (i32 1))
-; CHECK-NEXT: [[TMP0:%.*]] = extractelement <32 x i1> [[TMP131]], i64 0
-; CHECK-NEXT: br i1 [[TMP0]], label [[PRED_UDIV_IF:%.*]], label [[PRED_UDIV_CONTINUE:%.*]]
-; CHECK: pred.udiv.if:
-; CHECK-NEXT: [[TMP2:%.*]] = extractelement <32 x i32> [[TMP1]], i64 0
-; CHECK-NEXT: [[TMP3:%.*]] = udiv i32 [[TMP2]], [[D:%.*]]
-; CHECK-NEXT: [[TMP4:%.*]] = insertelement <32 x i32> poison, i32 [[TMP3]], i64 0
-; CHECK-NEXT: br label [[PRED_UDIV_CONTINUE]]
-; CHECK: pred.udiv.continue:
-; CHECK-NEXT: [[TMP5:%.*]] = phi <32 x i32> [ poison, [[VECTOR_BODY]] ], [ [[TMP4]], [[PRED_UDIV_IF]] ]
-; CHECK-NEXT: [[TMP168:%.*]] = extractelement <32 x i1> [[TMP131]], i64 0
-; CHECK-NEXT: br i1 [[TMP168]], label [[PRED_UDIV_IF1:%.*]], label [[PRED_UDIV_CONTINUE2:%.*]]
-; CHECK: pred.udiv.if2:
-; CHECK-NEXT: [[TMP6:%.*]] = extractelement <32 x i32> [[TMP1]], i64 1
-; CHECK-NEXT: [[TMP7:%.*]] = udiv i32 [[TMP6]], [[D]]
-; CHECK-NEXT: [[TMP8:%.*]] = insertelement <32 x i32> [[TMP5]], i32 [[TMP7]], i64 1
-; CHECK-NEXT: br label [[PRED_UDIV_CONTINUE2]]
-; CHECK: pred.udiv.continue3:
-; CHECK-NEXT: [[TMP9:%.*]] = phi <32 x i32> [ [[TMP5]], [[PRED_UDIV_CONTINUE]] ], [ [[TMP8]], [[PRED_UDIV_IF1]] ]
-; CHECK-NEXT: [[TMP170:%.*]] = extractelement <32 x i1> [[TMP131]], i64 0
-; CHECK-NEXT: br i1 [[TMP170]], label [[PRED_UDIV_IF3:%.*]], label [[PRED_UDIV_CONTINUE4:%.*]]
-; CHECK: pred.udiv.if4:
-; CHECK-NEXT: [[TMP10:%.*]] = extractelement <32 x i32> [[TMP1]], i64 2
-; CHECK-NEXT: [[TMP11:%.*]] = udiv i32 [[TMP10]], [[D]]
-; CHECK-NEXT: [[TMP12:%.*]] = insertelement <32 x i32> [[TMP9]], i32 [[TMP11]], i64 2
-; CHECK-NEXT: br label [[PRED_UDIV_CONTINUE4]]
-; CHECK: pred.udiv.continue5:
-; CHECK-NEXT: [[TMP13:%.*]] = phi <32 x i32> [ [[TMP9]], [[PRED_UDIV_CONTINUE2]] ], [ [[TMP12]], [[PRED_UDIV_IF3]] ]
-; CHECK-NEXT: [[TMP171:%.*]] = extractelement <32 x i1> [[TMP131]], i64 0
-; CHECK-NEXT: br i1 [[TMP171]], label [[PRED_UDIV_IF5:%.*]], label [[PRED_UDIV_CONTINUE6:%.*]]
-; CHECK: pred.udiv.if6:
-; CHECK-NEXT: [[TMP14:%.*]] = extractelement <32 x i32> [[TMP1]], i64 3
-; CHECK-NEXT: [[TMP15:%.*]] = udiv i32 [[TMP14]], [[D]]
-; CHECK-NEXT: [[TMP16:%.*]] = insertelement <32 x i32> [[TMP13]], i32 [[TMP15]], i64 3
-; CHECK-NEXT: br label [[PRED_UDIV_CONTINUE6]]
-; CHECK: pred.udiv.continue7:
-; CHECK-NEXT: [[TMP17:%.*]] = phi <32 x i32> [ [[TMP13]], [[PRED_UDIV_CONTINUE4]] ], [ [[TMP16]], [[PRED_UDIV_IF5]] ]
-; CHECK-NEXT: [[TMP172:%.*]] = extractelement <32 x i1> [[TMP131]], i64 0
-; CHECK-NEXT: br i1 [[TMP172]], label [[PRED_UDIV_IF7:%.*]], label [[PRED_UDIV_CONTINUE8:%.*]]
-; CHECK: pred.udiv.if8:
-; CHECK-NEXT: [[TMP18:%.*]] = extractelement <32 x i32> [[TMP1]], i64 4
-; CHECK-NEXT: [[TMP19:%.*]] = udiv i32 [[TMP18]], [[D]]
-; CHECK-NEXT: [[TMP20:%.*]] = insertelement <32 x i32> [[TMP17]], i32 [[TMP19]], i64 4
-; CHECK-NEXT: br label [[PRED_UDIV_CONTINUE8]]
-; CHECK: pred.udiv.continue9:
-; CHECK-NEXT: [[TMP21:%.*]] = phi <32 x i32> [ [[TMP17]], [[PRED_UDIV_CONTINUE6]] ], [ [[TMP20]], [[PRED_UDIV_IF7]] ]
-; CHECK-NEXT: [[TMP173:%.*]] = extractelement <32 x i1> [[TMP131]], i64 0
-; CHECK-NEXT: br i1 [[TMP173]], label [[PRED_UDIV_IF9:%.*]], label [[PRED_UDIV_CONTINUE10:%.*]]
-; CHECK: pred.udiv.if10:
-; CHECK-NEXT: [[TMP22:%.*]] = extractelement <32 x i32> [[TMP1]], i64 5
-; CHECK-NEXT: [[TMP23:%.*]] = udiv i32 [[TMP22]], [[D]]
-; CHECK-NEXT: [[TMP24:%.*]] = insertelement <32 x i32> [[TMP21]], i32 [[TMP23]], i64 5
-; CHECK-NEXT: br label [[PRED_UDIV_CONTINUE10]]
-; CHECK: pred.udiv.continue11:
-; CHECK-NEXT: [[TMP25:%.*]] = phi <32 x i32> [ [[TMP21]], [[PRED_UDIV_CONTINUE8]] ], [ [[TMP24]], [[PRED_UDIV_IF9]] ]
-; CHECK-NEXT: [[TMP175:%.*]] = extractelement <32 x i1> [[TMP131]], i64 0
-; CHECK-NEXT: br i1 [[TMP175]], label [[PRED_UDIV_IF11:%.*]], label [[PRED_UDIV_CONTINUE12:%.*]]
-; CHECK: pred.udiv.if12:
-; CHECK-NEXT: [[TMP26:%.*]] = extractelement <32 x i32> [[TMP1]], i64 6
-; CHECK-NEXT: [[TMP27:%.*]] = udiv i32 [[TMP26]], [[D]]
-; CHECK-NEXT: [[TMP28:%.*]] = insertelement <32 x i32> [[TMP25]], i32 [[TMP27]], i64 6
-; CHECK-NEXT: br label [[PRED_UDIV_CONTINUE12]]
-; CHECK: pred.udiv.continue13:
-; CHECK-NEXT: [[TMP29:%.*]] = phi <32 x i32> [ [[TMP25]], [[PRED_UDIV_CONTINUE10]] ], [ [[TMP28]], [[PRED_UDIV_IF11]] ]
-; CHECK-NEXT: [[TMP176:%.*]] = extractelement <32 x i1> [[TMP131]], i64 0
-; CHECK-NEXT: br i1 [[TMP176]], label [[PRED_UDIV_IF13:%.*]], label [[PRED_UDIV_CONTINUE14:%.*]]
-; CHECK: pred.udiv.if14:
-; CHECK-NEXT: [[TMP30:%.*]] = extractelement <32 x i32> [[TMP1]], i64 7
-; CHECK-NEXT: [[TMP31:%.*]] = udiv i32 [[TMP30]], [[D]]
-; CHECK-NEXT: [[TMP32:%.*]] = insertelement <32 x i32> [[TMP29]], i32 [[TMP31]], i64 7
-; CHECK-NEXT: br label [[PRED_UDIV_CONTINUE14]]
-; CHECK: pred.udiv.continue15:
-; CHECK-NEXT: [[TMP33:%.*]] = phi <32 x i32> [ [[TMP29]], [[PRED_UDIV_CONTINUE12]] ], [ [[TMP32]], [[PRED_UDIV_IF13]] ]
-; CHECK-NEXT: [[TMP177:%.*]] = extractelement <32 x i1> [[TMP131]], i64 0
-; CHECK-NEXT: br i1 [[TMP177]], label [[PRED_UDIV_IF15:%.*]], label [[PRED_UDIV_CONTINUE16:%.*]]
-; CHECK: pred.udiv.if16:
-; CHECK-NEXT: [[TMP34:%.*]] = extractelement <32 x i32> [[TMP1]], i64 8
-; CHECK-NEXT: [[TMP35:%.*]] = udiv i32 [[TMP34]], [[D]]
-; CHECK-NEXT: [[TMP36:%.*]] = insertelement <32 x i32> [[TMP33]], i32 [[TMP35]], i64 8
-; CHECK-NEXT: br label [[PRED_UDIV_CONTINUE16]]
-; CHECK: pred.udiv.continue17:
-; CHECK-NEXT: [[TMP37:%.*]] = phi <32 x i32> [ [[TMP33]], [[PRED_UDIV_CONTINUE14]] ], [ [[TMP36]], [[PRED_UDIV_IF15]] ]
-; CHECK-NEXT: [[TMP178:%.*]] = extractelement <32 x i1> [[TMP131]], i64 0
-; CHECK-NEXT: br i1 [[TMP178]], label [[PRED_UDIV_IF17:%.*]], label [[PRED_UDIV_CONTINUE18:%.*]]
-; CHECK: pred.udiv.if18:
-; CHECK-NEXT: [[TMP38:%.*]] = extractelement <32 x i32> [[TMP1]], i64 9
-; CHECK-NEXT: [[TMP39:%.*]] = udiv i32 [[TMP38]], [[D]]
-; CHECK-NEXT: [[TMP40:%.*]] = insertelement <32 x i32> [[TMP37]], i32 [[TMP39]], i64 9
-; CHECK-NEXT: br label [[PRED_UDIV_CONTINUE18]]
-; CHECK: pred.udiv.continue19:
-; CHECK-NEXT: [[TMP41:%.*]] = phi <32 x i32> [ [[TMP37]], [[PRED_UDIV_CONTINUE16]] ], [ [[TMP40]], [[PRED_UDIV_IF17]] ]
-; CHECK-NEXT: [[TMP180:%.*]] = extractelement <32 x i1> [[TMP131]], i64 0
-; CHECK-NEXT: br i1 [[TMP180]], label [[PRED_UDIV_IF19:%.*]], label [[PRED_UDIV_CONTINUE20:%.*]]
-; CHECK: pred.udiv.if20:
-; CHECK-NEXT: [[TMP42:%.*]] = extractelement <32 x i32> [[TMP1]], i64 10
-; CHECK-NEXT: [[TMP43:%.*]] = udiv i32 [[TMP42]], [[D]]
-; CHECK-NEXT: [[TMP44:%.*]] = insertelement <32 x i32> [[TMP41]], i32 [[TMP43]], i64 10
-; CHECK-NEXT: br label [[PRED_UDIV_CONTINUE20]]
-; CHECK: pred.udiv.continue21:
-; CHECK-NEXT: [[TMP45:%.*]] = phi <32 x i32> [ [[TMP41]], [[PRED_UDIV_CONTINUE18]] ], [ [[TMP44]], [[PRED_UDIV_IF19]] ]
-; CHECK-NEXT: [[TMP181:%.*]] = extractelement <32 x i1> [[TMP131]], i64 0
-; CHECK-NEXT: br i1 [[TMP181]], label [[PRED_UDIV_IF21:%.*]], label [[PRED_UDIV_CONTINUE22:%.*]]
-; CHECK: pred.udiv.if22:
-; CHECK-NEXT: [[TMP46:%.*]] = extractelement <32 x i32> [[TMP1]], i64 11
-; CHECK-NEXT: [[TMP47:%.*]] = udiv i32 [[TMP46]], [[D]]
-; CHECK-NEXT: [[TMP48:%.*]] = insertelement <32 x i32> [[TMP45]], i32 [[TMP47]], i64 11
-; CHECK-NEXT: br label [[PRED_UDIV_CONTINUE22]]
-; CHECK: pred.udiv.continue23:
-; CHECK-NEXT: [[TMP49:%.*]] = phi <32 x i32> [ [[TMP45]], [[PRED_UDIV_CONTINUE20]] ], [ [[TMP48]], [[PRED_UDIV_IF21]] ]
-; CHECK-NEXT: [[TMP182:%.*]] = extractelement <32 x i1> [[TMP131]], i64 0
-; CHECK-NEXT: br i1 [[TMP182]], label [[PRED_UDIV_IF23:%.*]], label [[PRED_UDIV_CONTINUE24:%.*]]
-; CHECK: pred.udiv.if24:
-; CHECK-NEXT: [[TMP50:%.*]] = extractelement <32 x i32> [[TMP1]], i64 12
-; CHECK-NEXT: [[TMP51:%.*]] = udiv i32 [[TMP50]], [[D]]
-; CHECK-NEXT: [[TMP52:%.*]] = insertelement <32 x i32> [[TMP49]], i32 [[TMP51]], i64 12
-; CHECK-NEXT: br label [[PRED_UDIV_CONTINUE24]]
-; CHECK: pred.udiv.continue25:
-; CHECK-NEXT: [[TMP53:%.*]] = phi <32 x i32> [ [[TMP49]], [[PRED_UDIV_CONTINUE22]] ], [ [[TMP52]], [[PRED_UDIV_IF23]] ]
-; CHECK-NEXT: [[TMP183:%.*]] = extractelement <32 x i1> [[TMP131]], i64 0
-; CHECK-NEXT: br i1 [[TMP183]], label [[PRED_UDIV_IF25:%.*]], label [[PRED_UDIV_CONTINUE26:%.*]]
-; CHECK: pred.udiv.if26:
-; CHECK-NEXT: [[TMP54:%.*]] = extractelement <32 x i32> [[TMP1]], i64 13
-; CHECK-NEXT: [[TMP55:%.*]] = udiv i32 [[TMP54]], [[D]]
-; CHECK-NEXT: [[TMP56:%.*]] = insertelement <32 x i32> [[TMP53]], i32 [[TMP55]], i64 13
-; CHECK-NEXT: br label [[PRED_UDIV_CONTINUE26]]
-; CHECK: pred.udiv.continue27:
-; CHECK-NEXT: [[TMP57:%.*]] = phi <32 x i32> [ [[TMP53]], [[PRED_UDIV_CONTINUE24]] ], [ [[TMP56]], [[PRED_UDIV_IF25]] ]
-; CHECK-NEXT: [[TMP185:%.*]] = extractelement <32 x i1> [[TMP131]], i64 0
-; CHECK-NEXT: br i1 [[TMP185]], label [[PRED_UDIV_IF27:%.*]], label [[PRED_UDIV_CONTINUE28:%.*]]
-; CHECK: pred.udiv.if28:
-; CHECK-NEXT: [[TMP58:%.*]] = extractelement <32 x i32> [[TMP1]], i64 14
-; CHECK-NEXT: [[TMP59:%.*]] = udiv i32 [[TMP58]], [[D]]
-; CHECK-NEXT: [[TMP60:%.*]] = insertelement <32 x i32> [[TMP57]], i32 [[TMP59]], i64 14
-; CHECK-NEXT: br label [[PRED_UDIV_CONTINUE28]]
-; CHECK: pred.udiv.continue29:
-; CHECK-NEXT: [[TMP61:%.*]] = phi <32 x i32> [ [[TMP57]], [[PRED_UDIV_CONTINUE26]] ], [ [[TMP60]], [[PRED_UDIV_IF27]] ]
-; CHECK-NEXT: [[TMP186:%.*]] = extractelement <32 x i1> [[TMP131]], i64 0
-; CHECK-NEXT: br i1 [[TMP186]], label [[PRED_UDIV_IF29:%.*]], label [[PRED_UDIV_CONTINUE30:%.*]]
-; CHECK: pred.udiv.if30:
-; CHECK-NEXT: [[TMP62:%.*]] = extractelement <32 x i32> [[TMP1]], i64 15
-; CHECK-NEXT: [[TMP63:%.*]] = udiv i32 [[TMP62]], [[D]]
-; CHECK-NEXT: [[TMP64:%.*]] = insertelement <32 x i32> [[TMP61]], i32 [[TMP63]], i64 15
-; CHECK-NEXT: br label [[PRED_UDIV_CONTINUE30]]
-; CHECK: pred.udiv.continue31:
-; CHECK-NEXT: [[TMP65:%.*]] = phi <32 x i32> [ [[TMP61]], [[PRED_UDIV_CONTINUE28]] ], [ [[TMP64]], [[PRED_UDIV_IF29]] ]
-; CHECK-NEXT: [[TMP187:%.*]] = extractelement <32 x i1> [[TMP131]], i64 0
-; CHECK-NEXT: br i1 [[TMP187]], label [[PRED_UDIV_IF31:%.*]], label [[PRED_UDIV_CONTINUE32:%.*]]
-; CHECK: pred.udiv.if32:
-; CHECK-NEXT: [[TMP66:%.*]] = extractelement <32 x i32> [[TMP1]], i64 16
-; CHECK-NEXT: [[TMP67:%.*]] = udiv i32 [[TMP66]], [[D]]
-; CHECK-NEXT: [[TMP68:%.*]] = insertelement <32 x i32> [[TMP65]], i32 [[TMP67]], i64 16
-; CHECK-NEXT: br label [[PRED_UDIV_CONTINUE32]]
-; CHECK: pred.udiv.continue33:
-; CHECK-NEXT: [[TMP69:%.*]] = phi <32 x i32> [ [[TMP65]], [[PRED_UDIV_CONTINUE30]] ], [ [[TMP68]], [[PRED_UDIV_IF31]] ]
-; CHECK-NEXT: [[TMP188:%.*]] = extractelement <32 x i1> [[TMP131]], i64 0
-; CHECK-NEXT: br i1 [[TMP188]], label [[PRED_UDIV_IF33:%.*]], label [[PRED_UDIV_CONTINUE34:%.*]]
-; CHECK: pred.udiv.if34:
-; CHECK-NEXT: [[TMP70:%.*]] = extractelement <32 x i32> [[TMP1]], i64 17
-; CHECK-NEXT: [[TMP71:%.*]] = udiv i32 [[TMP70]], [[D]]
-; CHECK-NEXT: [[TMP72:%.*]] = insertelement <32 x i32> [[TMP69]], i32 [[TMP71]], i64 17
-; CHECK-NEXT: br label [[PRED_UDIV_CONTINUE34]]
-; CHECK: pred.udiv.continue35:
-; CHECK-NEXT: [[TMP73:%.*]] = phi <32 x i32> [ [[TMP69]], [[PRED_UDIV_CONTINUE32]] ], [ [[TMP72]], [[PRED_UDIV_IF33]] ]
-; CHECK-NEXT: [[TMP190:%.*]] = extractelement <32 x i1> [[TMP131]], i64 0
-; CHECK-NEXT: br i1 [[TMP190]], label [[PRED_UDIV_IF35:%.*]], label [[PRED_UDIV_CONTINUE36:%.*]]
-; CHECK: pred.udiv.if36:
-; CHECK-NEXT: [[TMP74:%.*]] = extractelement <32 x i32> [[TMP1]], i64 18
-; CHECK-NEXT: [[TMP75:%.*]] = udiv i32 [[TMP74]], [[D]]
-; CHECK-NEXT: [[TMP76:%.*]] = insertelement <32 x i32> [[TMP73]], i32 [[TMP75]], i64 18
-; CHECK-NEXT: br label [[PRED_UDIV_CONTINUE36]]
-; CHECK: pred.udiv.continue37:
-; CHECK-NEXT: [[TMP77:%.*]] = phi <32 x i32> [ [[TMP73]], [[PRED_UDIV_CONTINUE34]] ], [ [[TMP76]], [[PRED_UDIV_IF35]] ]
-; CHECK-NEXT: [[TMP191:%.*]] = extractelement <32 x i1> [[TMP131]], i64 0
-; CHECK-NEXT: br i1 [[TMP191]], label [[PRED_UDIV_IF37:%.*]], label [[PRED_UDIV_CONTINUE38:%.*]]
-; CHECK: pred.udiv.if38:
-; CHECK-NEXT: [[TMP78:%.*]] = extractelement <32 x i32> [[TMP1]], i64 19
-; CHECK-NEXT: [[TMP79:%.*]] = udiv i32 [[TMP78]], [[D]]
-; CHECK-NEXT: [[TMP80:%.*]] = insertelement <32 x i32> [[TMP77]], i32 [[TMP79]], i64 19
-; CHECK-NEXT: br label [[PRED_UDIV_CONTINUE38]]
-; CHECK: pred.udiv.continue39:
-; CHECK-NEXT: [[TMP81:%.*]] = phi <32 x i32> [ [[TMP77]], [[PRED_UDIV_CONTINUE36]] ], [ [[TMP80]], [[PRED_UDIV_IF37]] ]
-; CHECK-NEXT: [[TMP192:%.*]] = extractelement <32 x i1> [[TMP131]], i64 0
-; CHECK-NEXT: br i1 [[TMP192]], label [[PRED_UDIV_IF39:%.*]], label [[PRED_UDIV_CONTINUE40:%.*]]
-; CHECK: pred.udiv.if40:
-; CHECK-NEXT: [[TMP82:%.*]] = extractelement <32 x i32> [[TMP1]], i64 20
-; CHECK-NEXT: [[TMP83:%.*]] = udiv i32 [[TMP82]], [[D]]
-; CHECK-NEXT: [[TMP84:%.*]] = insertelement <32 x i32> [[TMP81]], i32 [[TMP83]], i64 20
-; CHECK-NEXT: br label [[PRED_UDIV_CONTINUE40]]
-; CHECK: pred.udiv.continue41:
-; CHECK-NEXT: [[TMP85:%.*]] = phi <32 x i32> [ [[TMP81]], [[PRED_UDIV_CONTINUE38]] ], [ [[TMP84]], [[PRED_UDIV_IF39]] ]
-; CHECK-NEXT: [[TMP193:%.*]] = extractelement <32 x i1> [[TMP131]], i64 0
-; CHECK-NEXT: br i1 [[TMP193]], label [[PRED_UDIV_IF41:%.*]], label [[PRED_UDIV_CONTINUE42:%.*]]
-; CHECK: pred.udiv.if42:
-; CHECK-NEXT: [[TMP86:%.*]] = extractelement <32 x i32> [[TMP1]], i64 21
-; CHECK-NEXT: [[TMP87:%.*]] = udiv i32 [[TMP86]], [[D]]
-; CHECK-NEXT: [[TMP88:%.*]] = insertelement <32 x i32> [[TMP85]], i32 [[TMP87]], i64 21
-; CHECK-NEXT: br label [[PRED_UDIV_CONTINUE42]]
-; CHECK: pred.udiv.continue43:
-; CHECK-NEXT: [[TMP89:%.*]] = phi <32 x i32> [ [[TMP85]], [[PRED_UDIV_CONTINUE40]] ], [ [[TMP88]], [[PRED_UDIV_IF41]] ]
-; CHECK-NEXT: [[TMP195:%.*]] = extractelement <32 x i1> [[TMP131]], i64 0
-; CHECK-NEXT: br i1 [[TMP195]], label [[PRED_UDIV_IF43:%.*]], label [[PRED_UDIV_CONTINUE44:%.*]]
-; CHECK: pred.udiv.if44:
-; CHECK-NEXT: [[TMP90:%.*]] = extractelement <32 x i32> [[TMP1]], i64 22
-; CHECK-NEXT: [[TMP91:%.*]] = udiv i32 [[TMP90]], [[D]]
-; CHECK-NEXT: [[TMP92:%.*]] = insertelement <32 x i32> [[TMP89]], i32 [[TMP91]], i64 22
-; CHECK-NEXT: br label [[PRED_UDIV_CONTINUE44]]
-; CHECK: pred.udiv.continue45:
-; CHECK-NEXT: [[TMP93:%.*]] = phi <32 x i32> [ [[TMP89]], [[PRED_UDIV_CONTINUE42]] ], [ [[TMP92]], [[PRED_UDIV_IF43]] ]
-; CHECK-NEXT: [[TMP196:%.*]] = extractelement <32 x i1> [[TMP131]], i64 0
-; CHECK-NEXT: br i1 [[TMP196]], label [[PRED_UDIV_IF45:%.*]], label [[PRED_UDIV_CONTINUE46:%.*]]
-; CHECK: pred.udiv.if46:
-; CHECK-NEXT: [[TMP94:%.*]] = extractelement <32 x i32> [[TMP1]], i64 23
-; CHECK-NEXT: [[TMP95:%.*]] = udiv i32 [[TMP94]], [[D]]
-; CHECK-NEXT: [[TMP96:%.*]] = insertelement <32 x i32> [[TMP93]], i32 [[TMP95]], i64 23
-; CHECK-NEXT: br label [[PRED_UDIV_CONTINUE46]]
-; CHECK: pred.udiv.continue47:
-; CHECK-NEXT: [[TMP97:%.*]] = phi <32 x i32> [ [[TMP93]], [[PRED_UDIV_CONTINUE44]] ], [ [[TMP96]], [[PRED_UDIV_IF45]] ]
-; CHECK-NEXT: [[TMP197:%.*]] = extractelement <32 x i1> [[TMP131]], i64 0
-; CHECK-NEXT: br i1 [[TMP197]], label [[PRED_UDIV_IF47:%.*]], label [[PRED_UDIV_CONTINUE48:%.*]]
-; CHECK: pred.udiv.if48:
-; CHECK-NEXT: [[TMP98:%.*]] = extractelement <32 x i32> [[TMP1]], i64 24
-; CHECK-NEXT: [[TMP99:%.*]] = udiv i32 [[TMP98]], [[D]]
-; CHECK-NEXT: [[TMP100:%.*]] = insertelement <32 x i32> [[TMP97]], i32 [[TMP99]], i64 24
-; CHECK-NEXT: br label [[PRED_UDIV_CONTINUE48]]
-; CHECK: pred.udiv.continue49:
-; CHECK-NEXT: [[TMP101:%.*]] = phi <32 x i32> [ [[TMP97]], [[PRED_UDIV_CONTINUE46]] ], [ [[TMP100]], [[PRED_UDIV_IF47]] ]
-; CHECK-NEXT: [[TMP198:%.*]] = extractelement <32 x i1> [[TMP131]], i64 0
-; CHECK-NEXT: br i1 [[TMP198]], label [[PRED_UDIV_IF49:%.*]], label [[PRED_UDIV_CONTINUE50:%.*]]
-; CHECK: pred.udiv.if50:
-; CHECK-NEXT: [[TMP102:%.*]] = extractelement <32 x i32> [[TMP1]], i64 25
-; CHECK-NEXT: [[TMP103:%.*]] = udiv i32 [[TMP102]], [[D]]
-; CHECK-NEXT: [[TMP104:%.*]] = insertelement <32 x i32> [[TMP101]], i32 [[TMP103]], i64 25
-; CHECK-NEXT: br label [[PRED_UDIV_CONTINUE50]]
-; CHECK: pred.udiv.continue51:
-; CHECK-NEXT: [[TMP105:%.*]] = phi <32 x i32> [ [[TMP101]], [[PRED_UDIV_CONTINUE48]] ], [ [[TMP104]], [[PRED_UDIV_IF49]] ]
-; CHECK-NEXT: [[TMP200:%.*]] = extractelement <32 x i1> [[TMP131]], i64 0
-; CHECK-NEXT: br i1 [[TMP200]], label [[PRED_UDIV_IF51:%.*]], label [[PRED_UDIV_CONTINUE52:%.*]]
-; CHECK: pred.udiv.if52:
-; CHECK-NEXT: [[TMP106:%.*]] = extractelement <32 x i32> [[TMP1]], i64 26
-; CHECK-NEXT: [[TMP107:%.*]] = udiv i32 [[TMP106]], [[D]]
-; CHECK-NEXT: [[TMP108:%.*]] = insertelement <32 x i32> [[TMP105]], i32 [[TMP107]], i64 26
-; CHECK-NEXT: br label [[PRED_UDIV_CONTINUE52]]
-; CHECK: pred.udiv.continue53:
-; CHECK-NEXT: [[TMP109:%.*]] = phi <32 x i32> [ [[TMP105]], [[PRED_UDIV_CONTINUE50]] ], [ [[TMP108]], [[PRED_UDIV_IF51]] ]
-; CHECK-NEXT: [[TMP201:%.*]] = extractelement <32 x i1> [[TMP131]], i64 0
-; CHECK-NEXT: br i1 [[TMP201]], label [[PRED_UDIV_IF53:%.*]], label [[PRED_UDIV_CONTINUE54:%.*]]
-; CHECK: pred.udiv.if54:
-; CHECK-NEXT: [[TMP110:%.*]] = extractelement <32 x i32> [[TMP1]], i64 27
-; CHECK-NEXT: [[TMP111:%.*]] = udiv i32 [[TMP110]], [[D]]
-; CHECK-NEXT: [[TMP112:%.*]] = insertelement <32 x i32> [[TMP109]], i32 [[TMP111]], i64 27
-; CHECK-NEXT: br label [[PRED_UDIV_CONTINUE54]]
-; CHECK: pred.udiv.continue55:
-; CHECK-NEXT: [[TMP113:%.*]] = phi <32 x i32> [ [[TMP109]], [[PRED_UDIV_CONTINUE52]] ], [ [[TMP112]], [[PRED_UDIV_IF53]] ]
-; CHECK-NEXT: [[TMP202:%.*]] = extractelement <32 x i1> [[TMP131]], i64 0
-; CHECK-NEXT: br i1 [[TMP202]], label [[PRED_UDIV_IF55:%.*]], label [[PRED_UDIV_CONTINUE56:%.*]]
-; CHECK: pred.udiv.if56:
-; CHECK-NEXT: [[TMP114:%.*]] = extractelement <32 x i32> [[TMP1]], i64 28
-; CHECK-NEXT: [[TMP115:%.*]] = udiv i32 [[TMP114]], [[D]]
-; CHECK-NEXT: [[TMP116:%.*]] = insertelement <32 x i32> [[TMP113]], i32 [[TMP115]], i64 28
-; CHECK-NEXT: br label [[PRED_UDIV_CONTINUE56]]
-; CHECK: pred.udiv.continue57:
-; CHECK-NEXT: [[TMP117:%.*]] = phi <32 x i32> [ [[TMP113]], [[PRED_UDIV_CONTINUE54]] ], [ [[TMP116]], [[PRED_UDIV_IF55]] ]
-; CHECK-NEXT: [[TMP203:%.*]] = extractelement <32 x i1> [[TMP131]], i64 0
-; CHECK-NEXT: br i1 [[TMP203]], label [[PRED_UDIV_IF57:%.*]], label [[PRED_UDIV_CONTINUE58:%.*]]
-; CHECK: pred.udiv.if58:
-; CHECK-NEXT: [[TMP118:%.*]] = extractelement <32 x i32> [[TMP1]], i64 29
-; CHECK-NEXT: [[TMP119:%.*]] = udiv i32 [[TMP118]], [[D]]
-; CHECK-NEXT: [[TMP120:%.*]] = insertelement <32 x i32> [[TMP117]], i32 [[TMP119]], i64 29
-; CHECK-NEXT: br label [[PRED_UDIV_CONTINUE58]]
-; CHECK: pred.udiv.continue59:
-; CHECK-NEXT: [[TMP121:%.*]] = phi <32 x i32> [ [[TMP117]], [[PRED_UDIV_CONTINUE56]] ], [ [[TMP120]], [[PRED_UDIV_IF57]] ]
-; CHECK-NEXT: [[TMP205:%.*]] = extractelement <32 x i1> [[TMP131]], i64 0
-; CHECK-NEXT: br i1 [[TMP205]], label [[PRED_UDIV_IF59:%.*]], label [[PRED_UDIV_CONTINUE60:%.*]]
-; CHECK: pred.udiv.if60:
-; CHECK-NEXT: [[TMP122:%.*]] = extractelement <32 x i32> [[TMP1]], i64 30
-; CHECK-NEXT: [[TMP123:%.*]] = udiv i32 [[TMP122]], [[D]]
-; CHECK-NEXT: [[TMP124:%.*]] = insertelement <32 x i32> [[TMP121]], i32 [[TMP123]], i64 30
-; CHECK-NEXT: br label [[PRED_UDIV_CONTINUE60]]
-; CHECK: pred.udiv.continue61:
-; CHECK-NEXT: [[TMP125:%.*]] = phi <32 x i32> [ [[TMP121]], [[PRED_UDIV_CONTINUE58]] ], [ [[TMP124]], [[PRED_UDIV_IF59]] ]
-; CHECK-NEXT: [[TMP206:%.*]] = extractelement <32 x i1> [[TMP131]], i64 0
-; CHECK-NEXT: br i1 [[TMP206]], label [[PRED_UDIV_IF61:%.*]], label [[PRED_UDIV_CONTINUE63:%.*]]
-; CHECK: pred.udiv.if62:
-; CHECK-NEXT: [[TMP126:%.*]] = extractelement <32 x i32> [[TMP1]], i64 31
-; CHECK-NEXT: [[TMP127:%.*]] = udiv i32 [[TMP126]], [[D]]
-; CHECK-NEXT: [[TMP128:%.*]] = insertelement <32 x i32> [[TMP125]], i32 [[TMP127]], i64 31
-; CHECK-NEXT: br label [[PRED_UDIV_CONTINUE63]]
-; CHECK: pred.udiv.continue63:
-; CHECK-NEXT: [[TMP129:%.*]] = phi <32 x i32> [ [[TMP125]], [[PRED_UDIV_CONTINUE60]] ], [ [[TMP128]], [[PRED_UDIV_IF61]] ]
-; CHECK-NEXT: [[TMP207:%.*]] = zext <32 x i32> [[TMP129]] to <32 x i64>
-; CHECK-NEXT: br label [[PRED_UDIV_CONTINUE62]]
-; CHECK: loop.latch64:
-; CHECK-NEXT: [[TMP208:%.*]] = phi <32 x i64> [ poison, [[VECTOR_BODY1]] ], [ [[TMP207]], [[PRED_UDIV_CONTINUE63]] ]
-; CHECK-NEXT: [[TMP212:%.*]] = phi <32 x i1> [ zeroinitializer, [[VECTOR_BODY1]] ], [ [[TMP131]], [[PRED_UDIV_CONTINUE63]] ]
-; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i32 [[INDEX]], 32
-; CHECK-NEXT: [[VEC_IND_NEXT]] = add <32 x i32> [[VEC_IND]], splat (i32 32)
-; CHECK-NEXT: [[TMP130:%.*]] = icmp eq i32 [[INDEX_NEXT]], 992
-; CHECK-NEXT: br i1 [[TMP130]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY1]], !llvm.loop [[LOOP12:![0-9]+]]
-; CHECK: middle.block:
-; CHECK-NEXT: [[PREDPHI:%.*]] = select <32 x i1> [[TMP212]], <32 x i64> [[TMP208]], <32 x i64> zeroinitializer
-; CHECK-NEXT: [[TMP132:%.*]] = extractelement <32 x i64> [[PREDPHI]], i64 31
-; CHECK-NEXT: br i1 false, label [[EXIT:%.*]], label [[VEC_EPILOG_ITER_CHECK:%.*]]
-; CHECK: vec.epilog.iter.check:
-; CHECK-NEXT: br i1 false, label [[VEC_EPILOG_SCALAR_PH]], label [[VEC_EPILOG_PH]], !prof [[PROF13:![0-9]+]]
-; CHECK: vec.epilog.ph:
-; CHECK-NEXT: [[VEC_EPILOG_RESUME_VAL:%.*]] = phi i32 [ 992, [[VEC_EPILOG_ITER_CHECK]] ], [ 0, [[VECTOR_MAIN_LOOP_ITER_CHECK]] ]
-; CHECK-NEXT: [[BROADCAST_SPLATINSERT65:%.*]] = insertelement <8 x i1> poison, i1 [[C]], i64 0
-; CHECK-NEXT: [[BROADCAST_SPLAT66:%.*]] = shufflevector <8 x i1> [[BROADCAST_SPLATINSERT65]], <8 x i1> poison, <8 x i32> zeroinitializer
-; CHECK-NEXT: [[TMP213:%.*]] = xor <8 x i1> [[BROADCAST_SPLAT66]], splat (i1 true)
-; CHECK-NEXT: [[BROADCAST_SPLATINSERT:%.*]] = insertelement <8 x i32> poison, i32 [[VEC_EPILOG_RESUME_VAL]], i64 0
-; CHECK-NEXT: [[BROADCAST_SPLAT:%.*]] = shufflevector <8 x i32> [[BROADCAST_SPLATINSERT]], <8 x i32> poison, <8 x i32> zeroinitializer
-; CHECK-NEXT: [[INDUCTION:%.*]] = add <8 x i32> [[BROADCAST_SPLAT]], <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>
-; CHECK-NEXT: br label [[VEC_EPILOG_VECTOR_BODY1:%.*]]
-; CHECK: vec.epilog.vector.body:
-; CHECK-NEXT: [[INDEX63:%.*]] = phi i32 [ [[VEC_EPILOG_RESUME_VAL]], [[VEC_EPILOG_PH]] ], [ [[INDEX_NEXT81:%.*]], [[PRED_UDIV_CONTINUE80:%.*]] ]
-; CHECK-NEXT: [[VEC_IND64:%.*]] = phi <8 x i32> [ [[INDUCTION]], [[VEC_EPILOG_PH]] ], [ [[VEC_IND_NEXT82:%.*]], [[PRED_UDIV_CONTINUE80]] ]
-; CHECK-NEXT: br i1 [[C]], label [[PRED_UDIV_CONTINUE80]], label [[VEC_EPILOG_VECTOR_BODY:%.*]]
-; CHECK: then71:
-; CHECK-NEXT: [[TMP134:%.*]] = call <8 x i32> @llvm.usub.sat.v8i32(<8 x i32> [[VEC_IND64]], <8 x i32> splat (i32 1))
-; CHECK-NEXT: [[TMP133:%.*]] = extractelement <8 x i1> [[TMP213]], i64 0
-; CHECK-NEXT: br i1 [[TMP133]], label [[PRED_UDIV_IF65:%.*]], label [[PRED_UDIV_CONTINUE66:%.*]]
-; CHECK: pred.udiv.if72:
-; CHECK-NEXT: [[TMP135:%.*]] = extractelement <8 x i32> [[TMP134]], i64 0
-; CHECK-NEXT: [[TMP136:%.*]] = udiv i32 [[TMP135]], [[D]]
-; CHECK-NEXT: [[TMP137:%.*]] = insertelement <8 x i32> poison, i32 [[TMP136]], i64 0
-; CHECK-NEXT: br label [[PRED_UDIV_CONTINUE66]]
-; CHECK: pred.udiv.continue73:
-; CHECK-NEXT: [[TMP138:%.*]] = phi <8 x i32> [ poison, [[VEC_EPILOG_VECTOR_BODY]] ], [ [[TMP137]], [[PRED_UDIV_IF65]] ]
-; CHECK-NEXT: [[TMP174:%.*]] = extractelement <8 x i1> [[TMP213]], i64 0
-; CHECK-NEXT: br i1 [[TMP174]], label [[PRED_UDIV_IF67:%.*]], label [[PRED_UDIV_CONTINUE68:%.*]]
-; CHECK: pred.udiv.if74:
-; CHECK-NEXT: [[TMP139:%.*]] = extractelement <8 x i32> [[TMP134]], i64 1
-; CHECK-NEXT: [[TMP140:%.*]] = udiv i32 [[TMP139]], [[D]]
-; CHECK-NEXT: [[TMP141:%.*]] = insertelement <8 x i32> [[TMP138]], i32 [[TMP140]], i64 1
-; CHECK-NEXT: br label [[PRED_UDIV_CONTINUE68]]
-; CHECK: pred.udiv.continue75:
-; CHECK-NEXT: [[TMP142:%.*]] = phi <8 x i32> [ [[TMP138]], [[PRED_UDIV_CONTINUE66]] ], [ [[TMP141]], [[PRED_UDIV_IF67]] ]
-; CHECK-NEXT: [[TMP179:%.*]] = extractelement <8 x i1> [[TMP213]], i64 0
-; CHECK-NEXT: br i1 [[TMP179]], label [[PRED_UDIV_IF69:%.*]], label [[PRED_UDIV_CONTINUE70:%.*]]
-; CHECK: pred.udiv.if76:
-; CHECK-NEXT: [[TMP143:%.*]] = extractelement <8 x i32> [[TMP134]], i64 2
-; CHECK-NEXT: [[TMP144:%.*]] = udiv i32 [[TMP143]], [[D]]
-; CHECK-NEXT: [[TMP145:%.*]] = insertelement <8 x i32> [[TMP142]], i32 [[TMP144]], i64 2
-; CHECK-NEXT: br label [[PRED_UDIV_CONTINUE70]]
-; CHECK: pred.udiv.continue77:
-; CHECK-NEXT: [[TMP146:%.*]] = phi <8 x i32> [ [[TMP142]], [[PRED_UDIV_CONTINUE68]] ], [ [[TMP145]], [[PRED_UDIV_IF69]] ]
-; CHECK-NEXT: [[TMP184:%.*]] = extractelement <8 x i1> [[TMP213]], i64 0
-; CHECK-NEXT: br i1 [[TMP184]], label [[PRED_UDIV_IF71:%.*]], label [[PRED_UDIV_CONTINUE72:%.*]]
-; CHECK: pred.udiv.if78:
-; CHECK-NEXT: [[TMP147:%.*]] = extractelement <8 x i32> [[TMP134]], i64 3
-; CHECK-NEXT: [[TMP148:%.*]] = udiv i32 [[TMP147]], [[D]]
-; CHECK-NEXT: [[TMP149:%.*]] = insertelement <8 x i32> [[TMP146]], i32 [[TMP148]], i64 3
-; CHECK-NEXT: br label [[PRED_UDIV_CONTINUE72]]
-; CHECK: pred.udiv.continue79:
-; CHECK-NEXT: [[TMP150:%.*]] = phi <8 x i32> [ [[TMP146]], [[PRED_UDIV_CONTINUE70]] ], [ [[TMP149]], [[PRED_UDIV_IF71]] ]
-; CHECK-NEXT: [[TMP189:%.*]] = extractelement <8 x i1> [[TMP213]], i64 0
-; CHECK-NEXT: br i1 [[TMP189]], label [[PRED_UDIV_IF73:%.*]], label [[PRED_UDIV_CONTINUE74:%.*]]
-; CHECK: pred.udiv.if80:
-; CHECK-NEXT: [[TMP151:%.*]] = extractelement <8 x i32> [[TMP134]], i64 4
-; CHECK-NEXT: [[TMP152:%.*]] = udiv i32 [[TMP151]], [[D]]
-; CHECK-NEXT: [[TMP153:%.*]] = insertelement <8 x i32> [[TMP150]], i32 [[TMP152]], i64 4
-; CHECK-NEXT: br label [[PRED_UDIV_CONTINUE74]]
-; CHECK: pred.udiv.continue81:
-; CHECK-NEXT: [[TMP154:%.*]] = phi <8 x i32> [ [[TMP150]], [[PRED_UDIV_CONTINUE72]] ], [ [[TMP153]], [[PRED_UDIV_IF73]] ]
-; CHECK-NEXT: [[TMP194:%.*]] = extractelement <8 x i1> [[TMP213]], i64 0
-; CHECK-NEXT: br i1 [[TMP194]], label [[PRED_UDIV_IF75:%.*]], label [[PRED_UDIV_CONTINUE76:%.*]]
-; CHECK: pred.udiv.if82:
-; CHECK-NEXT: [[TMP155:%.*]] = extractelement <8 x i32> [[TMP134]], i64 5
-; CHECK-NEXT: [[TMP156:%.*]] = udiv i32 [[TMP155]], [[D]]
-; CHECK-NEXT: [[TMP157:%.*]] = insertelement <8 x i32> [[TMP154]], i32 [[TMP156]], i64 5
-; CHECK-NEXT: br label [[PRED_UDIV_CONTINUE76]]
-; CHECK: pred.udiv.continue83:
-; CHECK-NEXT: [[TMP158:%.*]] = phi <8 x i32> [ [[TMP154]], [[PRED_UDIV_CONTINUE74]] ], [ [[TMP157]], [[PRED_UDIV_IF75]] ]
-; CHECK-NEXT: [[TMP199:%.*]] = extractelement <8 x i1> [[TMP213]], i64 0
-; CHECK-NEXT: br i1 [[TMP199]], label [[PRED_UDIV_IF77:%.*]], label [[PRED_UDIV_CONTINUE78:%.*]]
-; CHECK: pred.udiv.if84:
-; CHECK-NEXT: [[TMP159:%.*]] = extractelement <8 x i32> [[TMP134]], i64 6
-; CHECK-NEXT: [[TMP160:%.*]] = udiv i32 [[TMP159]], [[D]]
-; CHECK-NEXT: [[TMP161:%.*]] = insertelement <8 x i32> [[TMP158]], i32 [[TMP160]], i64 6
-; CHECK-NEXT: br label [[PRED_UDIV_CONTINUE78]]
-; CHECK: pred.udiv.continue85:
-; CHECK-NEXT: [[TMP162:%.*]] = phi <8 x i32> [ [[TMP158]], [[PRED_UDIV_CONTINUE76]] ], [ [[TMP161]], [[PRED_UDIV_IF77]] ]
-; CHECK-NEXT: [[TMP204:%.*]] = extractelement <8 x i1> [[TMP213]], i64 0
-; CHECK-NEXT: br i1 [[TMP204]], label [[PRED_UDIV_IF79:%.*]], label [[PRED_UDIV_CONTINUE87:%.*]]
-; CHECK: pred.udiv.if86:
-; CHECK-NEXT: [[TMP163:%.*]] = extractelement <8 x i32> [[TMP134]], i64 7
-; CHECK-NEXT: [[TMP164:%.*]] = udiv i32 [[TMP163]], [[D]]
-; CHECK-NEXT: [[TMP165:%.*]] = insertelement <8 x i32> [[TMP162]], i32 [[TMP164]], i64 7
-; CHECK-NEXT: br label [[PRED_UDIV_CONTINUE87]]
-; CHECK: pred.udiv.continue87:
-; CHECK-NEXT: [[TMP166:%.*]] = phi <8 x i32> [ [[TMP162]], [[PRED_UDIV_CONTINUE78]] ], [ [[TMP165]], [[PRED_UDIV_IF79]] ]
-; CHECK-NEXT: [[TMP209:%.*]] = zext <8 x i32> [[TMP166]] to <8 x i64>
-; CHECK-NEXT: br label [[PRED_UDIV_CONTINUE80]]
-; CHECK: loop.latch88:
-; CHECK-NEXT: [[TMP210:%.*]] = phi <8 x i64> [ poison, [[VEC_EPILOG_VECTOR_BODY1]] ], [ [[TMP209]], [[PRED_UDIV_CONTINUE87]] ]
-; CHECK-NEXT: [[TMP211:%.*]] = phi <8 x i1> [ zeroinitializer, [[VEC_EPILOG_VECTOR_BODY1]] ], [ [[TMP213]], [[PRED_UDIV_CONTINUE87]] ]
-; CHECK-NEXT: [[INDEX_NEXT81]] = add nuw i32 [[INDEX63]], 8
-; CHECK-NEXT: [[VEC_IND_NEXT82]] = add <8 x i32> [[VEC_IND64]], splat (i32 8)
-; CHECK-NEXT: [[TMP167:%.*]] = icmp eq i32 [[INDEX_NEXT81]], 1000
-; CHECK-NEXT: br i1 [[TMP167]], label [[VEC_EPILOG_MIDDLE_BLOCK:%.*]], label [[VEC_EPILOG_VECTOR_BODY1]], !llvm.loop [[LOOP14:![0-9]+]]
-; CHECK: vec.epilog.middle.block:
-; CHECK-NEXT: [[PREDPHI83:%.*]] = select <8 x i1> [[TMP211]], <8 x i64> [[TMP210]], <8 x i64> zeroinitializer
-; CHECK-NEXT: [[TMP169:%.*]] = extractelement <8 x i64> [[PREDPHI83]], i64 7
-; CHECK-NEXT: br i1 false, label [[EXIT]], label [[VEC_EPILOG_SCALAR_PH]]
-; CHECK: vec.epilog.scalar.ph:
-; CHECK-NEXT: [[BC_RESUME_VAL:%.*]] = phi i32 [ 1000, [[VEC_EPILOG_MIDDLE_BLOCK]] ], [ 992, [[VEC_EPILOG_ITER_CHECK]] ], [ 0, [[ITER_CHECK:%.*]] ]
+; CHECK-NEXT: entry:
; CHECK-NEXT: br label [[LOOP_HEADER:%.*]]
; CHECK: loop.header:
-; CHECK-NEXT: [[IV:%.*]] = phi i32 [ [[BC_RESUME_VAL]], [[VEC_EPILOG_SCALAR_PH]] ], [ [[IV_NEXT:%.*]], [[LOOP_LATCH:%.*]] ]
-; CHECK-NEXT: br i1 [[C]], label [[LOOP_LATCH]], label [[THEN:%.*]]
+; CHECK-NEXT: [[IV:%.*]] = phi i32 [ 0, [[ENTRY:%.*]] ], [ [[IV_NEXT:%.*]], [[LOOP_LATCH:%.*]] ]
+; CHECK-NEXT: br i1 [[C:%.*]], label [[LOOP_LATCH]], label [[THEN:%.*]]
; CHECK: then:
; CHECK-NEXT: [[CALL:%.*]] = tail call i32 @llvm.usub.sat.i32(i32 [[IV]], i32 1)
-; CHECK-NEXT: [[UDIV:%.*]] = udiv i32 [[CALL]], [[D]]
+; CHECK-NEXT: [[UDIV:%.*]] = udiv i32 [[CALL]], [[D:%.*]]
; CHECK-NEXT: [[ZEXT:%.*]] = zext i32 [[UDIV]] to i64
; CHECK-NEXT: br label [[LOOP_LATCH]]
; CHECK: loop.latch:
; CHECK-NEXT: [[MERGE:%.*]] = phi i64 [ [[ZEXT]], [[THEN]] ], [ 0, [[LOOP_HEADER]] ]
; CHECK-NEXT: [[IV_NEXT]] = add i32 [[IV]], 1
; CHECK-NEXT: [[EC:%.*]] = icmp eq i32 [[IV]], 1000
-; CHECK-NEXT: br i1 [[EC]], label [[EXIT]], label [[LOOP_HEADER]], !llvm.loop [[LOOP15:![0-9]+]]
+; CHECK-NEXT: br i1 [[EC]], label [[EXIT:%.*]], label [[LOOP_HEADER]]
; CHECK: exit:
-; CHECK-NEXT: [[MERGE_LCSSA:%.*]] = phi i64 [ [[MERGE]], [[LOOP_LATCH]] ], [ [[TMP132]], [[MIDDLE_BLOCK]] ], [ [[TMP169]], [[VEC_EPILOG_MIDDLE_BLOCK]] ]
+; CHECK-NEXT: [[MERGE_LCSSA:%.*]] = phi i64 [ [[MERGE]], [[LOOP_LATCH]] ]
; CHECK-NEXT: ret i64 [[MERGE_LCSSA]]
;
entry:
diff --git a/llvm/test/Transforms/LoopVectorize/X86/masked-store-cost.ll b/llvm/test/Transforms/LoopVectorize/X86/masked-store-cost.ll
index 4faabbc7b8f35..ba84fbbd5eeda 100644
--- a/llvm/test/Transforms/LoopVectorize/X86/masked-store-cost.ll
+++ b/llvm/test/Transforms/LoopVectorize/X86/masked-store-cost.ll
@@ -125,8 +125,6 @@ define void @test_scalar_cost_single_store_loop_invariant_cond(ptr %dst, i1 %c)
; CHECK-NEXT: br label %[[VECTOR_PH:.*]]
; CHECK: [[VECTOR_PH]]:
; CHECK-NEXT: [[TMP0:%.*]] = getelementptr i8, ptr [[DST]], i64 96
-; CHECK-NEXT: [[BROADCAST_SPLATINSERT:%.*]] = insertelement <8 x i1> poison, i1 [[C]], i64 0
-; CHECK-NEXT: [[BROADCAST_SPLAT:%.*]] = shufflevector <8 x i1> [[BROADCAST_SPLATINSERT]], <8 x i1> poison, <8 x i32> zeroinitializer
; CHECK-NEXT: br label %[[VECTOR_BODY:.*]]
; CHECK: [[VECTOR_BODY]]:
; CHECK-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[LOOP_LATCH2:.*]] ]
@@ -134,7 +132,7 @@ define void @test_scalar_cost_single_store_loop_invariant_cond(ptr %dst, i1 %c)
; CHECK-NEXT: [[NEXT_GEP:%.*]] = getelementptr i8, ptr [[DST]], i64 [[TMP1]]
; CHECK-NEXT: br i1 [[C]], label %[[IF_THEN1:.*]], label %[[LOOP_LATCH2]]
; CHECK: [[IF_THEN1]]:
-; CHECK-NEXT: call void @llvm.masked.store.v8i32.p0(<8 x i32> zeroinitializer, ptr align 4 [[NEXT_GEP]], <8 x i1> [[BROADCAST_SPLAT]])
+; CHECK-NEXT: store <8 x i32> zeroinitializer, ptr [[NEXT_GEP]], align 4
; CHECK-NEXT: br label %[[LOOP_LATCH2]]
; CHECK: [[LOOP_LATCH2]]:
; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 8
diff --git a/llvm/test/Transforms/LoopVectorize/X86/pr141968-instsimplifyfolder.ll b/llvm/test/Transforms/LoopVectorize/X86/pr141968-instsimplifyfolder.ll
index 4db9db9598ae4..bbd18674a2664 100644
--- a/llvm/test/Transforms/LoopVectorize/X86/pr141968-instsimplifyfolder.ll
+++ b/llvm/test/Transforms/LoopVectorize/X86/pr141968-instsimplifyfolder.ll
@@ -11,9 +11,6 @@ define void @pr141968(i1 %cond, i8 %v, ptr %p) {
; CHECK-NEXT: [[SEXT:%.*]] = sext i8 [[V]] to i16
; CHECK-NEXT: br label %[[PRED_SDIV_CONTINUE28:.*]]
; CHECK: [[PRED_SDIV_CONTINUE28]]:
-; CHECK-NEXT: [[BROADCAST_SPLATINSERT:%.*]] = insertelement <16 x i1> poison, i1 [[COND]], i64 0
-; CHECK-NEXT: [[BROADCAST_SPLAT:%.*]] = shufflevector <16 x i1> [[BROADCAST_SPLATINSERT]], <16 x i1> poison, <16 x i32> zeroinitializer
-; CHECK-NEXT: [[TMP0:%.*]] = xor <16 x i1> [[BROADCAST_SPLAT]], splat (i1 true)
; CHECK-NEXT: [[BROADCAST_SPLATINSERT1:%.*]] = insertelement <16 x i8> poison, i8 [[V]], i64 0
; CHECK-NEXT: [[BROADCAST_SPLAT2:%.*]] = shufflevector <16 x i8> [[BROADCAST_SPLATINSERT1]], <16 x i8> poison, <16 x i32> zeroinitializer
; CHECK-NEXT: br label %[[VECTOR_BODY:.*]]
@@ -23,7 +20,7 @@ define void @pr141968(i1 %cond, i8 %v, ptr %p) {
; CHECK: [[PRED_SDIV_IF29]]:
; CHECK-NEXT: br label %[[PRED_SDIV_CONTINUE30]]
; CHECK: [[PRED_SDIV_CONTINUE30]]:
-; CHECK-NEXT: [[TMP1:%.*]] = phi <16 x i1> [ zeroinitializer, %[[VECTOR_BODY]] ], [ [[TMP0]], %[[PRED_SDIV_IF29]] ]
+; CHECK-NEXT: [[TMP1:%.*]] = phi <16 x i1> [ zeroinitializer, %[[VECTOR_BODY]] ], [ splat (i1 true), %[[PRED_SDIV_IF29]] ]
; CHECK-NEXT: [[PREDPHI1:%.*]] = select <16 x i1> [[TMP1]], <16 x i8> [[BROADCAST_SPLAT2]], <16 x i8> zeroinitializer
; CHECK-NEXT: [[PREDPHI:%.*]] = extractelement <16 x i8> [[PREDPHI1]], i64 15
; CHECK-NEXT: store i8 [[PREDPHI]], ptr [[P]], align 1
diff --git a/llvm/test/Transforms/LoopVectorize/X86/predicated-instruction-cost.ll b/llvm/test/Transforms/LoopVectorize/X86/predicated-instruction-cost.ll
index 5a479250e1490..aa4f9c83a68c2 100644
--- a/llvm/test/Transforms/LoopVectorize/X86/predicated-instruction-cost.ll
+++ b/llvm/test/Transforms/LoopVectorize/X86/predicated-instruction-cost.ll
@@ -57,23 +57,50 @@ exit:
define void @test_wide_shift_uses_predicated_invariant_instruction(i32 %d, i1 %c, ptr %dst) {
; CHECK-LABEL: define void @test_wide_shift_uses_predicated_invariant_instruction(
; CHECK-SAME: i32 [[D:%.*]], i1 [[C:%.*]], ptr [[DST:%.*]]) {
-; CHECK-NEXT: [[ENTRY:.*]]:
+; CHECK-NEXT: [[ENTRY:.*:]]
; CHECK-NEXT: br label %[[LOOP_HEADER:.*]]
; CHECK: [[LOOP_HEADER]]:
-; CHECK-NEXT: [[IV:%.*]] = phi i32 [ 0, %[[ENTRY]] ], [ [[IV_NEXT:%.*]], %[[LOOP_LATCH:.*]] ]
+; CHECK-NEXT: [[TMP0:%.*]] = urem i32 100, [[D]]
+; CHECK-NEXT: [[BROADCAST_SPLATINSERT:%.*]] = insertelement <4 x i32> poison, i32 [[TMP0]], i64 0
+; CHECK-NEXT: [[BROADCAST_SPLAT:%.*]] = shufflevector <4 x i32> [[BROADCAST_SPLATINSERT]], <4 x i32> poison, <4 x i32> zeroinitializer
+; CHECK-NEXT: [[TMP1:%.*]] = shl <4 x i32> [[BROADCAST_SPLAT]], splat (i32 12)
+; CHECK-NEXT: [[TMP2:%.*]] = shl <4 x i32> splat (i32 999), [[TMP1]]
+; CHECK-NEXT: br label %[[VECTOR_BODY:.*]]
+; CHECK: [[VECTOR_BODY]]:
+; CHECK-NEXT: [[INDEX:%.*]] = phi i32 [ 0, %[[LOOP_HEADER]] ], [ [[INDEX_NEXT:%.*]], %[[LOOP_LATCH:.*]] ]
; CHECK-NEXT: br i1 [[C]], label %[[LOOP_LATCH]], label %[[ELSE:.*]]
; CHECK: [[ELSE]]:
+; CHECK-NEXT: br label %[[LOOP_LATCH]]
+; CHECK: [[LOOP_LATCH]]:
+; CHECK-NEXT: [[TMP3:%.*]] = phi <4 x i32> [ poison, %[[VECTOR_BODY]] ], [ [[TMP2]], %[[ELSE]] ]
+; CHECK-NEXT: [[TMP4:%.*]] = phi <4 x i1> [ zeroinitializer, %[[VECTOR_BODY]] ], [ splat (i1 true), %[[ELSE]] ]
+; CHECK-NEXT: [[PREDPHI:%.*]] = select <4 x i1> [[TMP4]], <4 x i32> [[TMP3]], <4 x i32> zeroinitializer
+; CHECK-NEXT: [[TMP5:%.*]] = getelementptr inbounds i32, ptr [[DST]], i32 [[INDEX]]
+; CHECK-NEXT: [[TMP6:%.*]] = getelementptr inbounds i32, ptr [[TMP5]], i64 4
+; CHECK-NEXT: store <4 x i32> [[PREDPHI]], ptr [[TMP5]], align 4
+; CHECK-NEXT: store <4 x i32> [[PREDPHI]], ptr [[TMP6]], align 4
+; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i32 [[INDEX]], 8
+; CHECK-NEXT: [[TMP7:%.*]] = icmp eq i32 [[INDEX_NEXT]], 96
+; CHECK-NEXT: br i1 [[TMP7]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP0:![0-9]+]]
+; CHECK: [[MIDDLE_BLOCK]]:
+; CHECK-NEXT: br label %[[SCALAR_PH:.*]]
+; CHECK: [[SCALAR_PH]]:
+; CHECK-NEXT: br label %[[LOOP_HEADER1:.*]]
+; CHECK: [[LOOP_HEADER1]]:
+; CHECK-NEXT: [[IV:%.*]] = phi i32 [ 96, %[[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], %[[LOOP_LATCH1:.*]] ]
+; CHECK-NEXT: br i1 [[C]], label %[[LOOP_LATCH1]], label %[[ELSE1:.*]]
+; CHECK: [[ELSE1]]:
; CHECK-NEXT: [[REM:%.*]] = urem i32 100, [[D]]
; CHECK-NEXT: [[SEXT:%.*]] = shl i32 [[REM]], 12
; CHECK-NEXT: [[SHL_I:%.*]] = shl i32 999, [[SEXT]]
-; CHECK-NEXT: br label %[[LOOP_LATCH]]
-; CHECK: [[LOOP_LATCH]]:
-; CHECK-NEXT: [[P:%.*]] = phi i32 [ [[SHL_I]], %[[ELSE]] ], [ 0, %[[LOOP_HEADER]] ]
+; CHECK-NEXT: br label %[[LOOP_LATCH1]]
+; CHECK: [[LOOP_LATCH1]]:
+; CHECK-NEXT: [[P:%.*]] = phi i32 [ [[SHL_I]], %[[ELSE1]] ], [ 0, %[[LOOP_HEADER1]] ]
; CHECK-NEXT: [[GEP:%.*]] = getelementptr inbounds i32, ptr [[DST]], i32 [[IV]]
; CHECK-NEXT: store i32 [[P]], ptr [[GEP]], align 4
; CHECK-NEXT: [[IV_NEXT]] = add i32 [[IV]], 1
; CHECK-NEXT: [[EC:%.*]] = icmp eq i32 [[IV_NEXT]], 100
-; CHECK-NEXT: br i1 [[EC]], label %[[EXIT:.*]], label %[[LOOP_HEADER]]
+; CHECK-NEXT: br i1 [[EC]], label %[[EXIT:.*]], label %[[LOOP_HEADER1]], !llvm.loop [[LOOP3:![0-9]+]]
; CHECK: [[EXIT]]:
; CHECK-NEXT: ret void
;
@@ -238,7 +265,7 @@ define i32 @predicated_store_with_scatter_legal(ptr %dst, i64 %n) #0 {
; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 8
; CHECK-NEXT: [[VEC_IND_NEXT]] = add <8 x i64> [[VEC_IND]], splat (i64 8)
; CHECK-NEXT: [[TMP16:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
-; CHECK-NEXT: br i1 [[TMP16]], label %[[MIDDLE_BLOCK:.*]], label %[[LOOP]], !llvm.loop [[LOOP0:![0-9]+]]
+; CHECK-NEXT: br i1 [[TMP16]], label %[[MIDDLE_BLOCK:.*]], label %[[LOOP]], !llvm.loop [[LOOP4:![0-9]+]]
; CHECK: [[MIDDLE_BLOCK]]:
; CHECK-NEXT: [[CMP_N:%.*]] = icmp eq i64 [[TMP0]], [[N_VEC]]
; CHECK-NEXT: br i1 [[CMP_N]], label %[[EXIT:.*]], label %[[SCALAR_PH]]
@@ -264,7 +291,7 @@ define i32 @predicated_store_with_scatter_legal(ptr %dst, i64 %n) #0 {
; CHECK: [[LATCH]]:
; CHECK-NEXT: [[IV_NEXT]] = add i64 [[IV]], 1
; CHECK-NEXT: [[EC:%.*]] = icmp eq i64 [[IV]], [[N]]
-; CHECK-NEXT: br i1 [[EC]], label %[[EXIT]], label %[[LOOP1]], !llvm.loop [[LOOP3:![0-9]+]]
+; CHECK-NEXT: br i1 [[EC]], label %[[EXIT]], label %[[LOOP1]], !llvm.loop [[LOOP5:![0-9]+]]
; CHECK: [[EXIT]]:
; CHECK-NEXT: ret i32 0
;
diff --git a/llvm/test/Transforms/LoopVectorize/X86/predicated-replicate-feeding-cast.ll b/llvm/test/Transforms/LoopVectorize/X86/predicated-replicate-feeding-cast.ll
index b71828f574a80..8180d3088fafa 100644
--- a/llvm/test/Transforms/LoopVectorize/X86/predicated-replicate-feeding-cast.ll
+++ b/llvm/test/Transforms/LoopVectorize/X86/predicated-replicate-feeding-cast.ll
@@ -14,68 +14,37 @@ define i8 @predicated_replicate_feeding_cast(i16 %n, i1 %c1, i1 %c2, i16 %a, i8
; CHECK: [[VECTOR_PH]]:
; CHECK-NEXT: [[N_MOD_VF:%.*]] = and i32 [[TMP1]], 1
; CHECK-NEXT: [[N_VEC:%.*]] = sub i32 [[TMP1]], [[N_MOD_VF]]
-; CHECK-NEXT: [[BROADCAST_SPLATINSERT:%.*]] = insertelement <2 x i1> poison, i1 [[C2]], i64 0
-; CHECK-NEXT: [[BROADCAST_SPLAT:%.*]] = shufflevector <2 x i1> [[BROADCAST_SPLATINSERT]], <2 x i1> poison, <2 x i32> zeroinitializer
-; CHECK-NEXT: [[BROADCAST_SPLATINSERT1:%.*]] = insertelement <2 x i1> poison, i1 [[C1]], i64 0
-; CHECK-NEXT: [[BROADCAST_SPLAT2:%.*]] = shufflevector <2 x i1> [[BROADCAST_SPLATINSERT1]], <2 x i1> poison, <2 x i32> zeroinitializer
; CHECK-NEXT: [[TMP2:%.*]] = trunc i32 [[N_VEC]] to i16
-; CHECK-NEXT: [[TMP6:%.*]] = xor <2 x i1> [[BROADCAST_SPLAT]], splat (i1 true)
-; CHECK-NEXT: [[TMP7:%.*]] = xor <2 x i1> [[BROADCAST_SPLAT2]], splat (i1 true)
-; CHECK-NEXT: br label %[[VECTOR_BODY:.*]]
-; CHECK: [[VECTOR_BODY]]:
-; CHECK-NEXT: [[INDEX:%.*]] = phi i32 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[LATCH12:.*]] ]
-; CHECK-NEXT: br i1 [[C1]], label %[[MERGE6:.*]], label %[[IF13:.*]]
-; CHECK: [[IF13]]:
-; CHECK-NEXT: [[TMP3:%.*]] = extractelement <2 x i1> [[TMP7]], i64 0
-; CHECK-NEXT: br i1 [[TMP3]], label %[[PRED_SDIV_IF:.*]], label %[[PRED_SDIV_CONTINUE:.*]]
-; CHECK: [[PRED_SDIV_IF]]:
; CHECK-NEXT: [[TMP5:%.*]] = sdiv i16 1, [[A]]
; CHECK-NEXT: [[TMP8:%.*]] = insertelement <2 x i16> poison, i16 [[TMP5]], i64 0
-; CHECK-NEXT: br label %[[PRED_SDIV_CONTINUE]]
-; CHECK: [[PRED_SDIV_CONTINUE]]:
-; CHECK-NEXT: [[TMP9:%.*]] = phi <2 x i16> [ poison, %[[IF13]] ], [ [[TMP8]], %[[PRED_SDIV_IF]] ]
-; CHECK-NEXT: [[TMP10:%.*]] = extractelement <2 x i1> [[TMP7]], i64 0
-; CHECK-NEXT: br i1 [[TMP10]], label %[[PRED_SDIV_IF1:.*]], label %[[PRED_SDIV_CONTINUE2:.*]]
-; CHECK: [[PRED_SDIV_IF1]]:
-; CHECK-NEXT: [[TMP11:%.*]] = sdiv i16 1, [[A]]
-; CHECK-NEXT: [[TMP16:%.*]] = insertelement <2 x i16> [[TMP9]], i16 [[TMP11]], i64 1
-; CHECK-NEXT: br label %[[PRED_SDIV_CONTINUE2]]
+; CHECK-NEXT: [[BROADCAST_SPLAT:%.*]] = shufflevector <2 x i16> [[TMP8]], <2 x i16> poison, <2 x i32> zeroinitializer
+; CHECK-NEXT: br label %[[PRED_SDIV_CONTINUE2:.*]]
; CHECK: [[PRED_SDIV_CONTINUE2]]:
-; CHECK-NEXT: [[TMP20:%.*]] = phi <2 x i16> [ [[TMP9]], %[[PRED_SDIV_CONTINUE]] ], [ [[TMP16]], %[[PRED_SDIV_IF1]] ]
+; CHECK-NEXT: [[INDEX:%.*]] = phi i32 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[LATCH12:.*]] ]
+; CHECK-NEXT: br i1 [[C1]], label %[[MERGE6:.*]], label %[[IF11:.*]]
+; CHECK: [[IF11]]:
; CHECK-NEXT: br label %[[MERGE6]]
; CHECK: [[MERGE6]]:
-; CHECK-NEXT: [[TMP25:%.*]] = phi <2 x i16> [ poison, %[[VECTOR_BODY]] ], [ [[TMP20]], %[[PRED_SDIV_CONTINUE2]] ]
-; CHECK-NEXT: [[TMP26:%.*]] = phi <2 x i1> [ zeroinitializer, %[[VECTOR_BODY]] ], [ [[TMP7]], %[[PRED_SDIV_CONTINUE2]] ]
+; CHECK-NEXT: [[TMP25:%.*]] = phi <2 x i16> [ poison, %[[PRED_SDIV_CONTINUE2]] ], [ [[BROADCAST_SPLAT]], %[[IF11]] ]
+; CHECK-NEXT: [[TMP26:%.*]] = phi <2 x i1> [ zeroinitializer, %[[PRED_SDIV_CONTINUE2]] ], [ splat (i1 true), %[[IF11]] ]
; CHECK-NEXT: [[PREDPHI:%.*]] = select <2 x i1> [[TMP26]], <2 x i16> [[TMP25]], <2 x i16> zeroinitializer
; CHECK-NEXT: br i1 [[C2]], label %[[LATCH12]], label %[[IF27:.*]]
; CHECK: [[IF27]]:
-; CHECK-NEXT: [[TMP4:%.*]] = extractelement <2 x i1> [[TMP6]], i64 0
-; CHECK-NEXT: br i1 [[TMP4]], label %[[PRED_SDIV_IF3:.*]], label %[[PRED_SDIV_CONTINUE4:.*]]
-; CHECK: [[PRED_SDIV_IF3]]:
; CHECK-NEXT: [[TMP17:%.*]] = extractelement <2 x i16> [[PREDPHI]], i64 0
; CHECK-NEXT: [[TMP12:%.*]] = trunc i16 [[TMP17]] to i8
-; CHECK-NEXT: [[TMP13:%.*]] = sdiv i8 [[TMP12]], [[B]]
-; CHECK-NEXT: [[TMP14:%.*]] = insertelement <2 x i8> poison, i8 [[TMP13]], i64 0
-; CHECK-NEXT: br label %[[PRED_SDIV_CONTINUE4]]
-; CHECK: [[PRED_SDIV_CONTINUE4]]:
-; CHECK-NEXT: [[TMP15:%.*]] = phi <2 x i8> [ poison, %[[IF27]] ], [ [[TMP14]], %[[PRED_SDIV_IF3]] ]
-; CHECK-NEXT: [[TMP30:%.*]] = extractelement <2 x i1> [[TMP6]], i64 0
-; CHECK-NEXT: br i1 [[TMP30]], label %[[PRED_SDIV_IF5:.*]], label %[[PRED_SDIV_CONTINUE6:.*]]
-; CHECK: [[PRED_SDIV_IF5]]:
; CHECK-NEXT: [[TMP23:%.*]] = extractelement <2 x i16> [[PREDPHI]], i64 1
; CHECK-NEXT: [[TMP24:%.*]] = trunc i16 [[TMP23]] to i8
+; CHECK-NEXT: [[TMP11:%.*]] = sdiv i8 [[TMP12]], [[B]]
; CHECK-NEXT: [[TMP18:%.*]] = sdiv i8 [[TMP24]], [[B]]
+; CHECK-NEXT: [[TMP15:%.*]] = insertelement <2 x i8> poison, i8 [[TMP11]], i64 0
; CHECK-NEXT: [[TMP19:%.*]] = insertelement <2 x i8> [[TMP15]], i8 [[TMP18]], i64 1
-; CHECK-NEXT: br label %[[PRED_SDIV_CONTINUE6]]
-; CHECK: [[PRED_SDIV_CONTINUE6]]:
-; CHECK-NEXT: [[TMP27:%.*]] = phi <2 x i8> [ [[TMP15]], %[[PRED_SDIV_CONTINUE4]] ], [ [[TMP19]], %[[PRED_SDIV_IF5]] ]
; CHECK-NEXT: br label %[[LATCH12]]
; CHECK: [[LATCH12]]:
-; CHECK-NEXT: [[TMP28:%.*]] = phi <2 x i8> [ poison, %[[MERGE6]] ], [ [[TMP27]], %[[PRED_SDIV_CONTINUE6]] ]
-; CHECK-NEXT: [[TMP29:%.*]] = phi <2 x i1> [ zeroinitializer, %[[MERGE6]] ], [ [[TMP6]], %[[PRED_SDIV_CONTINUE6]] ]
+; CHECK-NEXT: [[TMP28:%.*]] = phi <2 x i8> [ poison, %[[MERGE6]] ], [ [[TMP19]], %[[IF27]] ]
+; CHECK-NEXT: [[TMP29:%.*]] = phi <2 x i1> [ zeroinitializer, %[[MERGE6]] ], [ splat (i1 true), %[[IF27]] ]
; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i32 [[INDEX]], 2
; CHECK-NEXT: [[TMP21:%.*]] = icmp eq i32 [[INDEX_NEXT]], [[N_VEC]]
-; CHECK-NEXT: br i1 [[TMP21]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP0:![0-9]+]]
+; CHECK-NEXT: br i1 [[TMP21]], label %[[MIDDLE_BLOCK:.*]], label %[[PRED_SDIV_CONTINUE2]], !llvm.loop [[LOOP0:![0-9]+]]
; CHECK: [[MIDDLE_BLOCK]]:
; CHECK-NEXT: [[PREDPHI7:%.*]] = select <2 x i1> [[TMP29]], <2 x i8> [[TMP28]], <2 x i8> zeroinitializer
; CHECK-NEXT: [[TMP22:%.*]] = extractelement <2 x i8> [[PREDPHI7]], i64 1
@@ -146,12 +115,6 @@ define i8 @predicated_replicate_feeding_cast_non_uniform(i64 %n, i1 %c1, i1 %c2,
; CHECK: [[VECTOR_PH]]:
; CHECK-NEXT: [[N_MOD_VF:%.*]] = and i64 [[TMP0]], 1
; CHECK-NEXT: [[N_VEC:%.*]] = sub i64 [[TMP0]], [[N_MOD_VF]]
-; CHECK-NEXT: [[BROADCAST_SPLATINSERT:%.*]] = insertelement <2 x i1> poison, i1 [[C2]], i64 0
-; CHECK-NEXT: [[BROADCAST_SPLAT:%.*]] = shufflevector <2 x i1> [[BROADCAST_SPLATINSERT]], <2 x i1> poison, <2 x i32> zeroinitializer
-; CHECK-NEXT: [[BROADCAST_SPLATINSERT1:%.*]] = insertelement <2 x i1> poison, i1 [[C1]], i64 0
-; CHECK-NEXT: [[BROADCAST_SPLAT2:%.*]] = shufflevector <2 x i1> [[BROADCAST_SPLATINSERT1]], <2 x i1> poison, <2 x i32> zeroinitializer
-; CHECK-NEXT: [[TMP11:%.*]] = xor <2 x i1> [[BROADCAST_SPLAT]], splat (i1 true)
-; CHECK-NEXT: [[TMP21:%.*]] = xor <2 x i1> [[BROADCAST_SPLAT2]], splat (i1 true)
; CHECK-NEXT: br label %[[VECTOR_BODY:.*]]
; CHECK: [[VECTOR_BODY]]:
; CHECK-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[LATCH12:.*]] ]
@@ -159,55 +122,31 @@ define i8 @predicated_replicate_feeding_cast_non_uniform(i64 %n, i1 %c1, i1 %c2,
; CHECK-NEXT: [[WIDE_LOAD:%.*]] = load <2 x i16>, ptr [[TMP3]], align 2
; CHECK-NEXT: br i1 [[C1]], label %[[MERGE6:.*]], label %[[IF13:.*]]
; CHECK: [[IF13]]:
-; CHECK-NEXT: [[TMP1:%.*]] = extractelement <2 x i1> [[TMP21]], i64 0
-; CHECK-NEXT: br i1 [[TMP1]], label %[[PRED_SDIV_IF:.*]], label %[[PRED_SDIV_CONTINUE:.*]]
-; CHECK: [[PRED_SDIV_IF]]:
; CHECK-NEXT: [[TMP4:%.*]] = extractelement <2 x i16> [[WIDE_LOAD]], i64 0
; CHECK-NEXT: [[TMP5:%.*]] = sdiv i16 1, [[TMP4]]
-; CHECK-NEXT: [[TMP6:%.*]] = insertelement <2 x i16> poison, i16 [[TMP5]], i64 0
-; CHECK-NEXT: br label %[[PRED_SDIV_CONTINUE]]
-; CHECK: [[PRED_SDIV_CONTINUE]]:
-; CHECK-NEXT: [[TMP7:%.*]] = phi <2 x i16> [ poison, %[[IF13]] ], [ [[TMP6]], %[[PRED_SDIV_IF]] ]
-; CHECK-NEXT: [[TMP24:%.*]] = extractelement <2 x i1> [[TMP21]], i64 0
-; CHECK-NEXT: br i1 [[TMP24]], label %[[PRED_SDIV_IF1:.*]], label %[[PRED_SDIV_CONTINUE2:.*]]
-; CHECK: [[PRED_SDIV_IF1]]:
; CHECK-NEXT: [[TMP8:%.*]] = extractelement <2 x i16> [[WIDE_LOAD]], i64 1
; CHECK-NEXT: [[TMP9:%.*]] = sdiv i16 1, [[TMP8]]
+; CHECK-NEXT: [[TMP7:%.*]] = insertelement <2 x i16> poison, i16 [[TMP5]], i64 0
; CHECK-NEXT: [[TMP10:%.*]] = insertelement <2 x i16> [[TMP7]], i16 [[TMP9]], i64 1
-; CHECK-NEXT: br label %[[PRED_SDIV_CONTINUE2]]
-; CHECK: [[PRED_SDIV_CONTINUE2]]:
-; CHECK-NEXT: [[TMP25:%.*]] = phi <2 x i16> [ [[TMP7]], %[[PRED_SDIV_CONTINUE]] ], [ [[TMP10]], %[[PRED_SDIV_IF1]] ]
; CHECK-NEXT: br label %[[MERGE6]]
; CHECK: [[MERGE6]]:
-; CHECK-NEXT: [[TMP26:%.*]] = phi <2 x i16> [ poison, %[[VECTOR_BODY]] ], [ [[TMP25]], %[[PRED_SDIV_CONTINUE2]] ]
-; CHECK-NEXT: [[TMP27:%.*]] = phi <2 x i1> [ zeroinitializer, %[[VECTOR_BODY]] ], [ [[TMP21]], %[[PRED_SDIV_CONTINUE2]] ]
+; CHECK-NEXT: [[TMP26:%.*]] = phi <2 x i16> [ poison, %[[VECTOR_BODY]] ], [ [[TMP10]], %[[IF13]] ]
+; CHECK-NEXT: [[TMP27:%.*]] = phi <2 x i1> [ zeroinitializer, %[[VECTOR_BODY]] ], [ splat (i1 true), %[[IF13]] ]
; CHECK-NEXT: [[PREDPHI:%.*]] = select <2 x i1> [[TMP27]], <2 x i16> [[TMP26]], <2 x i16> zeroinitializer
; CHECK-NEXT: br i1 [[C2]], label %[[LATCH12]], label %[[IF27:.*]]
; CHECK: [[IF27]]:
-; CHECK-NEXT: [[TMP2:%.*]] = extractelement <2 x i1> [[TMP11]], i64 0
-; CHECK-NEXT: br i1 [[TMP2]], label %[[PRED_SDIV_IF3:.*]], label %[[PRED_SDIV_CONTINUE4:.*]]
-; CHECK: [[PRED_SDIV_IF3]]:
; CHECK-NEXT: [[TMP12:%.*]] = extractelement <2 x i16> [[PREDPHI]], i64 0
; CHECK-NEXT: [[TMP13:%.*]] = trunc i16 [[TMP12]] to i8
-; CHECK-NEXT: [[TMP14:%.*]] = sdiv i8 [[TMP13]], [[B]]
-; CHECK-NEXT: [[TMP15:%.*]] = insertelement <2 x i8> poison, i8 [[TMP14]], i64 0
-; CHECK-NEXT: br label %[[PRED_SDIV_CONTINUE4]]
-; CHECK: [[PRED_SDIV_CONTINUE4]]:
-; CHECK-NEXT: [[TMP16:%.*]] = phi <2 x i8> [ poison, %[[IF27]] ], [ [[TMP15]], %[[PRED_SDIV_IF3]] ]
-; CHECK-NEXT: [[TMP31:%.*]] = extractelement <2 x i1> [[TMP11]], i64 0
-; CHECK-NEXT: br i1 [[TMP31]], label %[[PRED_SDIV_IF5:.*]], label %[[PRED_SDIV_CONTINUE6:.*]]
-; CHECK: [[PRED_SDIV_IF5]]:
; CHECK-NEXT: [[TMP17:%.*]] = extractelement <2 x i16> [[PREDPHI]], i64 1
; CHECK-NEXT: [[TMP18:%.*]] = trunc i16 [[TMP17]] to i8
+; CHECK-NEXT: [[TMP15:%.*]] = sdiv i8 [[TMP13]], [[B]]
; CHECK-NEXT: [[TMP19:%.*]] = sdiv i8 [[TMP18]], [[B]]
+; CHECK-NEXT: [[TMP16:%.*]] = insertelement <2 x i8> poison, i8 [[TMP15]], i64 0
; CHECK-NEXT: [[TMP20:%.*]] = insertelement <2 x i8> [[TMP16]], i8 [[TMP19]], i64 1
-; CHECK-NEXT: br label %[[PRED_SDIV_CONTINUE6]]
-; CHECK: [[PRED_SDIV_CONTINUE6]]:
-; CHECK-NEXT: [[TMP28:%.*]] = phi <2 x i8> [ [[TMP16]], %[[PRED_SDIV_CONTINUE4]] ], [ [[TMP20]], %[[PRED_SDIV_IF5]] ]
; CHECK-NEXT: br label %[[LATCH12]]
; CHECK: [[LATCH12]]:
-; CHECK-NEXT: [[TMP29:%.*]] = phi <2 x i8> [ poison, %[[MERGE6]] ], [ [[TMP28]], %[[PRED_SDIV_CONTINUE6]] ]
-; CHECK-NEXT: [[TMP30:%.*]] = phi <2 x i1> [ zeroinitializer, %[[MERGE6]] ], [ [[TMP11]], %[[PRED_SDIV_CONTINUE6]] ]
+; CHECK-NEXT: [[TMP29:%.*]] = phi <2 x i8> [ poison, %[[MERGE6]] ], [ [[TMP20]], %[[IF27]] ]
+; CHECK-NEXT: [[TMP30:%.*]] = phi <2 x i1> [ zeroinitializer, %[[MERGE6]] ], [ splat (i1 true), %[[IF27]] ]
; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 2
; CHECK-NEXT: [[TMP22:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
; CHECK-NEXT: br i1 [[TMP22]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP4:![0-9]+]]
diff --git a/llvm/test/Transforms/LoopVectorize/constant-fold-commutative-and.ll b/llvm/test/Transforms/LoopVectorize/constant-fold-commutative-and.ll
index cf11297d4564e..2e9a3760b5bcd 100644
--- a/llvm/test/Transforms/LoopVectorize/constant-fold-commutative-and.ll
+++ b/llvm/test/Transforms/LoopVectorize/constant-fold-commutative-and.ll
@@ -11,9 +11,6 @@ define void @constant_fold_commutative_and(ptr %ptr.n, ptr noalias %p, i1 %cond)
; CHECK-NEXT: [[ENTRY:.*:]]
; CHECK-NEXT: br label %[[VECTOR_PH:.*]]
; CHECK: [[VECTOR_PH]]:
-; CHECK-NEXT: [[BROADCAST_SPLATINSERT:%.*]] = insertelement <2 x i1> poison, i1 [[COND]], i64 0
-; CHECK-NEXT: [[BROADCAST_SPLAT:%.*]] = shufflevector <2 x i1> [[BROADCAST_SPLATINSERT]], <2 x i1> poison, <2 x i32> zeroinitializer
-; CHECK-NEXT: [[TMP0:%.*]] = xor <2 x i1> [[BROADCAST_SPLAT]], splat (i1 true)
; CHECK-NEXT: br label %[[VECTOR_BODY:.*]]
; CHECK: [[VECTOR_BODY]]:
; CHECK-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[PRED_STORE_CONTINUE5:.*]] ]
@@ -25,13 +22,12 @@ define void @constant_fold_commutative_and(ptr %ptr.n, ptr noalias %p, i1 %cond)
; CHECK-NEXT: br i1 [[COND]], label %[[LATCH4:.*]], label %[[PRED_11:.*]]
; CHECK: [[PRED_11]]:
; CHECK-NEXT: [[TMP3:%.*]] = icmp uge <2 x i64> [[BROADCAST_SPLAT2]], splat (i64 4)
-; CHECK-NEXT: [[TMP5:%.*]] = select <2 x i1> [[TMP0]], <2 x i1> [[TMP3]], <2 x i1> zeroinitializer
; CHECK-NEXT: [[TMP7:%.*]] = icmp ult <2 x i64> [[BROADCAST_SPLAT2]], splat (i64 7)
; CHECK-NEXT: br label %[[LATCH4]]
; CHECK: [[LATCH4]]:
-; CHECK-NEXT: [[TMP12:%.*]] = phi <2 x i1> [ zeroinitializer, %[[VECTOR_BODY]] ], [ [[TMP0]], %[[PRED_11]] ]
+; CHECK-NEXT: [[TMP12:%.*]] = phi <2 x i1> [ zeroinitializer, %[[VECTOR_BODY]] ], [ splat (i1 true), %[[PRED_11]] ]
; CHECK-NEXT: [[TMP13:%.*]] = phi <2 x i1> [ poison, %[[VECTOR_BODY]] ], [ [[TMP7]], %[[PRED_11]] ]
-; CHECK-NEXT: [[TMP14:%.*]] = phi <2 x i1> [ zeroinitializer, %[[VECTOR_BODY]] ], [ [[TMP5]], %[[PRED_11]] ]
+; CHECK-NEXT: [[TMP14:%.*]] = phi <2 x i1> [ zeroinitializer, %[[VECTOR_BODY]] ], [ [[TMP3]], %[[PRED_11]] ]
; CHECK-NEXT: [[PREDPHI3:%.*]] = select <2 x i1> [[TMP14]], <2 x i1> [[TMP13]], <2 x i1> [[TMP12]]
; CHECK-NEXT: [[TMP6:%.*]] = extractelement <2 x i1> [[TMP1]], i64 0
; CHECK-NEXT: br i1 [[TMP6]], label %[[PRED_STORE_IF:.*]], label %[[PRED_STORE_CONTINUE:.*]]
diff --git a/llvm/test/Transforms/LoopVectorize/div-exact.ll b/llvm/test/Transforms/LoopVectorize/div-exact.ll
index 7bec969285194..4cf09d41cc0e9 100644
--- a/llvm/test/Transforms/LoopVectorize/div-exact.ll
+++ b/llvm/test/Transforms/LoopVectorize/div-exact.ll
@@ -14,76 +14,18 @@ define void @unknown_divisor(ptr noalias %p, i32 %n, i1 %c) {
; CHECK: [[VECTOR_PH]]:
; CHECK-NEXT: [[N_MOD_VF:%.*]] = and i32 [[N]], 3
; CHECK-NEXT: [[N_VEC:%.*]] = sub i32 [[N]], [[N_MOD_VF]]
-; CHECK-NEXT: [[BROADCAST_SPLATINSERT:%.*]] = insertelement <4 x i1> poison, i1 [[C]], i64 0
-; CHECK-NEXT: [[BROADCAST_SPLAT:%.*]] = shufflevector <4 x i1> [[BROADCAST_SPLATINSERT]], <4 x i1> poison, <4 x i32> zeroinitializer
; CHECK-NEXT: br label %[[VECTOR_BODY:.*]]
; CHECK: [[VECTOR_BODY]]:
-; CHECK-NEXT: [[INDEX:%.*]] = phi i32 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[LATCH14:.*]] ]
-; CHECK-NEXT: [[TMP2:%.*]] = add i32 [[INDEX]], 1
-; CHECK-NEXT: [[TMP3:%.*]] = add i32 [[INDEX]], 2
-; CHECK-NEXT: [[TMP4:%.*]] = add i32 [[INDEX]], 3
+; CHECK-NEXT: [[TMP4:%.*]] = phi i32 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[LATCH14:.*]] ]
; CHECK-NEXT: br i1 [[C]], label %[[IF1:.*]], label %[[LATCH14]]
; CHECK: [[IF1]]:
-; CHECK-NEXT: [[TMP7:%.*]] = getelementptr i32, ptr [[P]], i32 [[INDEX]]
-; CHECK-NEXT: [[TMP8:%.*]] = getelementptr i32, ptr [[P]], i32 [[TMP2]]
-; CHECK-NEXT: [[TMP9:%.*]] = getelementptr i32, ptr [[P]], i32 [[TMP3]]
; CHECK-NEXT: [[TMP10:%.*]] = getelementptr i32, ptr [[P]], i32 [[TMP4]]
-; CHECK-NEXT: br i1 [[C]], label %[[PRED_LOAD_IF:.*]], label %[[PRED_LOAD_CONTINUE:.*]]
-; CHECK: [[PRED_LOAD_IF]]:
-; CHECK-NEXT: [[TMP12:%.*]] = load i32, ptr [[TMP7]], align 4
-; CHECK-NEXT: [[TMP13:%.*]] = insertelement <4 x i32> poison, i32 [[TMP12]], i64 0
-; CHECK-NEXT: br label %[[PRED_LOAD_CONTINUE]]
-; CHECK: [[PRED_LOAD_CONTINUE]]:
-; CHECK-NEXT: [[TMP14:%.*]] = phi <4 x i32> [ poison, %[[IF1]] ], [ [[TMP13]], %[[PRED_LOAD_IF]] ]
-; CHECK-NEXT: br i1 [[C]], label %[[PRED_LOAD_IF3:.*]], label %[[PRED_LOAD_CONTINUE4:.*]]
-; CHECK: [[PRED_LOAD_IF3]]:
-; CHECK-NEXT: [[TMP16:%.*]] = load i32, ptr [[TMP8]], align 4
-; CHECK-NEXT: [[TMP17:%.*]] = insertelement <4 x i32> [[TMP14]], i32 [[TMP16]], i64 1
-; CHECK-NEXT: br label %[[PRED_LOAD_CONTINUE4]]
-; CHECK: [[PRED_LOAD_CONTINUE4]]:
-; CHECK-NEXT: [[TMP18:%.*]] = phi <4 x i32> [ [[TMP14]], %[[PRED_LOAD_CONTINUE]] ], [ [[TMP17]], %[[PRED_LOAD_IF3]] ]
-; CHECK-NEXT: br i1 [[C]], label %[[PRED_LOAD_IF5:.*]], label %[[PRED_LOAD_CONTINUE6:.*]]
-; CHECK: [[PRED_LOAD_IF5]]:
-; CHECK-NEXT: [[TMP20:%.*]] = load i32, ptr [[TMP9]], align 4
-; CHECK-NEXT: [[TMP21:%.*]] = insertelement <4 x i32> [[TMP18]], i32 [[TMP20]], i64 2
-; CHECK-NEXT: br label %[[PRED_LOAD_CONTINUE6]]
-; CHECK: [[PRED_LOAD_CONTINUE6]]:
-; CHECK-NEXT: [[TMP22:%.*]] = phi <4 x i32> [ [[TMP18]], %[[PRED_LOAD_CONTINUE4]] ], [ [[TMP21]], %[[PRED_LOAD_IF5]] ]
-; CHECK-NEXT: br i1 [[C]], label %[[PRED_LOAD_IF7:.*]], label %[[PRED_LOAD_CONTINUE8:.*]]
-; CHECK: [[PRED_LOAD_IF7]]:
-; CHECK-NEXT: [[TMP24:%.*]] = load i32, ptr [[TMP10]], align 4
-; CHECK-NEXT: [[TMP25:%.*]] = insertelement <4 x i32> [[TMP22]], i32 [[TMP24]], i64 3
-; CHECK-NEXT: br label %[[PRED_LOAD_CONTINUE8]]
-; CHECK: [[PRED_LOAD_CONTINUE8]]:
-; CHECK-NEXT: [[TMP23:%.*]] = phi <4 x i32> [ [[TMP22]], %[[PRED_LOAD_CONTINUE6]] ], [ [[TMP25]], %[[PRED_LOAD_IF7]] ]
-; CHECK-NEXT: [[TMP28:%.*]] = call <4 x i32> @llvm.masked.udiv.v4i32(<4 x i32> splat (i32 100), <4 x i32> [[TMP23]], <4 x i1> [[BROADCAST_SPLAT]])
-; CHECK-NEXT: br i1 [[C]], label %[[PRED_STORE_IF:.*]], label %[[PRED_STORE_CONTINUE:.*]]
-; CHECK: [[PRED_STORE_IF]]:
-; CHECK-NEXT: [[TMP29:%.*]] = extractelement <4 x i32> [[TMP28]], i64 0
-; CHECK-NEXT: store i32 [[TMP29]], ptr [[TMP7]], align 4
-; CHECK-NEXT: br label %[[PRED_STORE_CONTINUE]]
-; CHECK: [[PRED_STORE_CONTINUE]]:
-; CHECK-NEXT: br i1 [[C]], label %[[PRED_STORE_IF9:.*]], label %[[PRED_STORE_CONTINUE10:.*]]
-; CHECK: [[PRED_STORE_IF9]]:
-; CHECK-NEXT: [[TMP30:%.*]] = extractelement <4 x i32> [[TMP28]], i64 1
-; CHECK-NEXT: store i32 [[TMP30]], ptr [[TMP8]], align 4
-; CHECK-NEXT: br label %[[PRED_STORE_CONTINUE10]]
-; CHECK: [[PRED_STORE_CONTINUE10]]:
-; CHECK-NEXT: br i1 [[C]], label %[[PRED_STORE_IF11:.*]], label %[[PRED_STORE_CONTINUE12:.*]]
-; CHECK: [[PRED_STORE_IF11]]:
-; CHECK-NEXT: [[TMP31:%.*]] = extractelement <4 x i32> [[TMP28]], i64 2
-; CHECK-NEXT: store i32 [[TMP31]], ptr [[TMP9]], align 4
-; CHECK-NEXT: br label %[[PRED_STORE_CONTINUE12]]
-; CHECK: [[PRED_STORE_CONTINUE12]]:
-; CHECK-NEXT: br i1 [[C]], label %[[PRED_STORE_IF13:.*]], label %[[PRED_STORE_CONTINUE14:.*]]
-; CHECK: [[PRED_STORE_IF13]]:
-; CHECK-NEXT: [[TMP32:%.*]] = extractelement <4 x i32> [[TMP28]], i64 3
-; CHECK-NEXT: store i32 [[TMP32]], ptr [[TMP10]], align 4
-; CHECK-NEXT: br label %[[PRED_STORE_CONTINUE14]]
-; CHECK: [[PRED_STORE_CONTINUE14]]:
+; CHECK-NEXT: [[WIDE_LOAD:%.*]] = load <4 x i32>, ptr [[TMP10]], align 4
+; CHECK-NEXT: [[TMP5:%.*]] = udiv exact <4 x i32> splat (i32 100), [[WIDE_LOAD]]
+; CHECK-NEXT: store <4 x i32> [[TMP5]], ptr [[TMP10]], align 4
; CHECK-NEXT: br label %[[LATCH14]]
; CHECK: [[LATCH14]]:
-; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i32 [[INDEX]], 4
+; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i32 [[TMP4]], 4
; CHECK-NEXT: [[TMP33:%.*]] = icmp eq i32 [[INDEX_NEXT]], [[N_VEC]]
; CHECK-NEXT: br i1 [[TMP33]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP0:![0-9]+]]
; CHECK: [[MIDDLE_BLOCK]]:
@@ -146,72 +88,16 @@ define void @unknown_dividend(ptr noalias %p, i32 %n, i1 %c) {
; CHECK-NEXT: [[N_VEC:%.*]] = sub i32 [[N]], [[N_MOD_VF]]
; CHECK-NEXT: br label %[[VECTOR_BODY:.*]]
; CHECK: [[VECTOR_BODY]]:
-; CHECK-NEXT: [[INDEX:%.*]] = phi i32 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[LATCH14:.*]] ]
-; CHECK-NEXT: [[TMP2:%.*]] = add i32 [[INDEX]], 1
-; CHECK-NEXT: [[TMP3:%.*]] = add i32 [[INDEX]], 2
-; CHECK-NEXT: [[TMP4:%.*]] = add i32 [[INDEX]], 3
+; CHECK-NEXT: [[TMP4:%.*]] = phi i32 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[LATCH14:.*]] ]
; CHECK-NEXT: br i1 [[C]], label %[[IF1:.*]], label %[[LATCH14]]
; CHECK: [[IF1]]:
-; CHECK-NEXT: [[TMP7:%.*]] = getelementptr i32, ptr [[P]], i32 [[INDEX]]
-; CHECK-NEXT: [[TMP8:%.*]] = getelementptr i32, ptr [[P]], i32 [[TMP2]]
-; CHECK-NEXT: [[TMP9:%.*]] = getelementptr i32, ptr [[P]], i32 [[TMP3]]
; CHECK-NEXT: [[TMP10:%.*]] = getelementptr i32, ptr [[P]], i32 [[TMP4]]
-; CHECK-NEXT: br i1 [[C]], label %[[PRED_LOAD_IF:.*]], label %[[PRED_LOAD_CONTINUE:.*]]
-; CHECK: [[PRED_LOAD_IF]]:
-; CHECK-NEXT: [[TMP12:%.*]] = load i32, ptr [[TMP7]], align 4
-; CHECK-NEXT: [[TMP13:%.*]] = insertelement <4 x i32> poison, i32 [[TMP12]], i64 0
-; CHECK-NEXT: br label %[[PRED_LOAD_CONTINUE]]
-; CHECK: [[PRED_LOAD_CONTINUE]]:
-; CHECK-NEXT: [[TMP14:%.*]] = phi <4 x i32> [ poison, %[[IF1]] ], [ [[TMP13]], %[[PRED_LOAD_IF]] ]
-; CHECK-NEXT: br i1 [[C]], label %[[PRED_LOAD_IF3:.*]], label %[[PRED_LOAD_CONTINUE4:.*]]
-; CHECK: [[PRED_LOAD_IF3]]:
-; CHECK-NEXT: [[TMP16:%.*]] = load i32, ptr [[TMP8]], align 4
-; CHECK-NEXT: [[TMP17:%.*]] = insertelement <4 x i32> [[TMP14]], i32 [[TMP16]], i64 1
-; CHECK-NEXT: br label %[[PRED_LOAD_CONTINUE4]]
-; CHECK: [[PRED_LOAD_CONTINUE4]]:
-; CHECK-NEXT: [[TMP18:%.*]] = phi <4 x i32> [ [[TMP14]], %[[PRED_LOAD_CONTINUE]] ], [ [[TMP17]], %[[PRED_LOAD_IF3]] ]
-; CHECK-NEXT: br i1 [[C]], label %[[PRED_LOAD_IF5:.*]], label %[[PRED_LOAD_CONTINUE6:.*]]
-; CHECK: [[PRED_LOAD_IF5]]:
-; CHECK-NEXT: [[TMP20:%.*]] = load i32, ptr [[TMP9]], align 4
-; CHECK-NEXT: [[TMP21:%.*]] = insertelement <4 x i32> [[TMP18]], i32 [[TMP20]], i64 2
-; CHECK-NEXT: br label %[[PRED_LOAD_CONTINUE6]]
-; CHECK: [[PRED_LOAD_CONTINUE6]]:
-; CHECK-NEXT: [[TMP22:%.*]] = phi <4 x i32> [ [[TMP18]], %[[PRED_LOAD_CONTINUE4]] ], [ [[TMP21]], %[[PRED_LOAD_IF5]] ]
-; CHECK-NEXT: br i1 [[C]], label %[[PRED_LOAD_IF7:.*]], label %[[PRED_LOAD_CONTINUE8:.*]]
-; CHECK: [[PRED_LOAD_IF7]]:
-; CHECK-NEXT: [[TMP24:%.*]] = load i32, ptr [[TMP10]], align 4
-; CHECK-NEXT: [[TMP25:%.*]] = insertelement <4 x i32> [[TMP22]], i32 [[TMP24]], i64 3
-; CHECK-NEXT: br label %[[PRED_LOAD_CONTINUE8]]
-; CHECK: [[PRED_LOAD_CONTINUE8]]:
-; CHECK-NEXT: [[TMP23:%.*]] = phi <4 x i32> [ [[TMP22]], %[[PRED_LOAD_CONTINUE6]] ], [ [[TMP25]], %[[PRED_LOAD_IF7]] ]
+; CHECK-NEXT: [[TMP23:%.*]] = load <4 x i32>, ptr [[TMP10]], align 4
; CHECK-NEXT: [[TMP27:%.*]] = udiv exact <4 x i32> [[TMP23]], splat (i32 100)
-; CHECK-NEXT: br i1 [[C]], label %[[PRED_STORE_IF:.*]], label %[[PRED_STORE_CONTINUE:.*]]
-; CHECK: [[PRED_STORE_IF]]:
-; CHECK-NEXT: [[TMP28:%.*]] = extractelement <4 x i32> [[TMP27]], i64 0
-; CHECK-NEXT: store i32 [[TMP28]], ptr [[TMP7]], align 4
-; CHECK-NEXT: br label %[[PRED_STORE_CONTINUE]]
-; CHECK: [[PRED_STORE_CONTINUE]]:
-; CHECK-NEXT: br i1 [[C]], label %[[PRED_STORE_IF9:.*]], label %[[PRED_STORE_CONTINUE10:.*]]
-; CHECK: [[PRED_STORE_IF9]]:
-; CHECK-NEXT: [[TMP29:%.*]] = extractelement <4 x i32> [[TMP27]], i64 1
-; CHECK-NEXT: store i32 [[TMP29]], ptr [[TMP8]], align 4
-; CHECK-NEXT: br label %[[PRED_STORE_CONTINUE10]]
-; CHECK: [[PRED_STORE_CONTINUE10]]:
-; CHECK-NEXT: br i1 [[C]], label %[[PRED_STORE_IF11:.*]], label %[[PRED_STORE_CONTINUE12:.*]]
-; CHECK: [[PRED_STORE_IF11]]:
-; CHECK-NEXT: [[TMP30:%.*]] = extractelement <4 x i32> [[TMP27]], i64 2
-; CHECK-NEXT: store i32 [[TMP30]], ptr [[TMP9]], align 4
-; CHECK-NEXT: br label %[[PRED_STORE_CONTINUE12]]
-; CHECK: [[PRED_STORE_CONTINUE12]]:
-; CHECK-NEXT: br i1 [[C]], label %[[PRED_STORE_IF13:.*]], label %[[PRED_STORE_CONTINUE14:.*]]
-; CHECK: [[PRED_STORE_IF13]]:
-; CHECK-NEXT: [[TMP31:%.*]] = extractelement <4 x i32> [[TMP27]], i64 3
-; CHECK-NEXT: store i32 [[TMP31]], ptr [[TMP10]], align 4
-; CHECK-NEXT: br label %[[PRED_STORE_CONTINUE14]]
-; CHECK: [[PRED_STORE_CONTINUE14]]:
+; CHECK-NEXT: store <4 x i32> [[TMP27]], ptr [[TMP10]], align 4
; CHECK-NEXT: br label %[[LATCH14]]
; CHECK: [[LATCH14]]:
-; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i32 [[INDEX]], 4
+; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i32 [[TMP4]], 4
; CHECK-NEXT: [[TMP32:%.*]] = icmp eq i32 [[INDEX_NEXT]], [[N_VEC]]
; CHECK-NEXT: br i1 [[TMP32]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP4:![0-9]+]]
; CHECK: [[MIDDLE_BLOCK]]:
diff --git a/llvm/test/Transforms/LoopVectorize/if-pred-stores.ll b/llvm/test/Transforms/LoopVectorize/if-pred-stores.ll
index b57899e31811a..381c84ec9782f 100644
--- a/llvm/test/Transforms/LoopVectorize/if-pred-stores.ll
+++ b/llvm/test/Transforms/LoopVectorize/if-pred-stores.ll
@@ -418,26 +418,18 @@ define void @minimal_bit_widths(ptr %p, i1 %c) {
; UNROLL: [[VECTOR_PH]]:
; UNROLL-NEXT: br label %[[VECTOR_BODY:.*]]
; UNROLL: [[VECTOR_BODY]]:
-; UNROLL-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[PRED_STORE_CONTINUE:.*]] ]
+; UNROLL-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[PRED_STORE_CONTINUE1:.*]] ]
; UNROLL-NEXT: [[TMP0:%.*]] = add i64 [[INDEX]], 1
; UNROLL-NEXT: [[TMP1:%.*]] = getelementptr i8, ptr [[P]], i64 [[INDEX]]
; UNROLL-NEXT: [[TMP2:%.*]] = getelementptr i8, ptr [[P]], i64 [[TMP0]]
; UNROLL-NEXT: [[TMP3:%.*]] = load i8, ptr [[TMP1]], align 1
; UNROLL-NEXT: [[TMP4:%.*]] = load i8, ptr [[TMP2]], align 1
-; UNROLL-NEXT: br i1 [[C]], label %[[PRED_STORE_IF:.*]], label %[[PRED_STORE_CONTINUE]]
-; UNROLL: [[PRED_STORE_IF]]:
-; UNROLL-NEXT: br i1 [[C]], label %[[PRED_STORE_IF2:.*]], label %[[PRED_STORE_CONTINUE1:.*]]
+; UNROLL-NEXT: br i1 [[C]], label %[[PRED_STORE_IF2:.*]], label %[[PRED_STORE_CONTINUE1]]
; UNROLL: [[PRED_STORE_IF2]]:
; UNROLL-NEXT: store i8 [[TMP3]], ptr [[TMP1]], align 1
+; UNROLL-NEXT: store i8 [[TMP4]], ptr [[TMP2]], align 1
; UNROLL-NEXT: br label %[[PRED_STORE_CONTINUE1]]
; UNROLL: [[PRED_STORE_CONTINUE1]]:
-; UNROLL-NEXT: br i1 [[C]], label %[[PRED_STORE_IF1:.*]], label %[[PRED_STORE_CONTINUE2:.*]]
-; UNROLL: [[PRED_STORE_IF1]]:
-; UNROLL-NEXT: store i8 [[TMP4]], ptr [[TMP2]], align 1
-; UNROLL-NEXT: br label %[[PRED_STORE_CONTINUE2]]
-; UNROLL: [[PRED_STORE_CONTINUE2]]:
-; UNROLL-NEXT: br label %[[PRED_STORE_CONTINUE]]
-; UNROLL: [[PRED_STORE_CONTINUE]]:
; UNROLL-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 2
; UNROLL-NEXT: [[TMP5:%.*]] = icmp eq i64 [[INDEX_NEXT]], 1000
; UNROLL-NEXT: br i1 [[TMP5]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP5:![0-9]+]]
@@ -453,26 +445,18 @@ define void @minimal_bit_widths(ptr %p, i1 %c) {
; UNROLL-NOSIMPLIFY: [[VECTOR_PH]]:
; UNROLL-NOSIMPLIFY-NEXT: br label %[[VECTOR_BODY:.*]]
; UNROLL-NOSIMPLIFY: [[VECTOR_BODY]]:
-; UNROLL-NOSIMPLIFY-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[PRED_STORE_CONTINUE:.*]] ]
+; UNROLL-NOSIMPLIFY-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[PRED_STORE_CONTINUE1:.*]] ]
; UNROLL-NOSIMPLIFY-NEXT: [[TMP0:%.*]] = add i64 [[INDEX]], 1
; UNROLL-NOSIMPLIFY-NEXT: [[TMP1:%.*]] = getelementptr i8, ptr [[P]], i64 [[INDEX]]
; UNROLL-NOSIMPLIFY-NEXT: [[TMP2:%.*]] = getelementptr i8, ptr [[P]], i64 [[TMP0]]
; UNROLL-NOSIMPLIFY-NEXT: [[TMP3:%.*]] = load i8, ptr [[TMP1]], align 1
; UNROLL-NOSIMPLIFY-NEXT: [[TMP4:%.*]] = load i8, ptr [[TMP2]], align 1
-; UNROLL-NOSIMPLIFY-NEXT: br i1 [[C]], label %[[PRED_STORE_IF:.*]], label %[[PRED_STORE_CONTINUE]]
-; UNROLL-NOSIMPLIFY: [[PRED_STORE_IF]]:
-; UNROLL-NOSIMPLIFY-NEXT: br i1 [[C]], label %[[PRED_STORE_IF2:.*]], label %[[PRED_STORE_CONTINUE1:.*]]
+; UNROLL-NOSIMPLIFY-NEXT: br i1 [[C]], label %[[PRED_STORE_IF2:.*]], label %[[PRED_STORE_CONTINUE1]]
; UNROLL-NOSIMPLIFY: [[PRED_STORE_IF2]]:
; UNROLL-NOSIMPLIFY-NEXT: store i8 [[TMP3]], ptr [[TMP1]], align 1
+; UNROLL-NOSIMPLIFY-NEXT: store i8 [[TMP4]], ptr [[TMP2]], align 1
; UNROLL-NOSIMPLIFY-NEXT: br label %[[PRED_STORE_CONTINUE1]]
; UNROLL-NOSIMPLIFY: [[PRED_STORE_CONTINUE1]]:
-; UNROLL-NOSIMPLIFY-NEXT: br i1 [[C]], label %[[PRED_STORE_IF1:.*]], label %[[PRED_STORE_CONTINUE2:.*]]
-; UNROLL-NOSIMPLIFY: [[PRED_STORE_IF1]]:
-; UNROLL-NOSIMPLIFY-NEXT: store i8 [[TMP4]], ptr [[TMP2]], align 1
-; UNROLL-NOSIMPLIFY-NEXT: br label %[[PRED_STORE_CONTINUE2]]
-; UNROLL-NOSIMPLIFY: [[PRED_STORE_CONTINUE2]]:
-; UNROLL-NOSIMPLIFY-NEXT: br label %[[PRED_STORE_CONTINUE]]
-; UNROLL-NOSIMPLIFY: [[PRED_STORE_CONTINUE]]:
; UNROLL-NOSIMPLIFY-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 2
; UNROLL-NOSIMPLIFY-NEXT: [[TMP5:%.*]] = icmp eq i64 [[INDEX_NEXT]], 1000
; UNROLL-NOSIMPLIFY-NEXT: br i1 [[TMP5]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP5:![0-9]+]]
@@ -488,27 +472,14 @@ define void @minimal_bit_widths(ptr %p, i1 %c) {
; VEC: [[VECTOR_PH]]:
; VEC-NEXT: br label %[[VECTOR_BODY:.*]]
; VEC: [[VECTOR_BODY]]:
-; VEC-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[PRED_STORE_CONTINUE:.*]] ]
+; VEC-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[PRED_STORE_CONTINUE2:.*]] ]
; VEC-NEXT: [[TMP0:%.*]] = getelementptr i8, ptr [[P]], i64 [[INDEX]]
; VEC-NEXT: [[WIDE_LOAD:%.*]] = load <2 x i8>, ptr [[TMP0]], align 1
-; VEC-NEXT: br i1 [[C]], label %[[PRED_STORE_IF:.*]], label %[[PRED_STORE_CONTINUE]]
-; VEC: [[PRED_STORE_IF]]:
-; VEC-NEXT: br i1 [[C]], label %[[PRED_STORE_IF2:.*]], label %[[PRED_STORE_CONTINUE1:.*]]
-; VEC: [[PRED_STORE_IF2]]:
-; VEC-NEXT: [[TMP1:%.*]] = extractelement <2 x i8> [[WIDE_LOAD]], i64 0
-; VEC-NEXT: store i8 [[TMP1]], ptr [[TMP0]], align 1
-; VEC-NEXT: br label %[[PRED_STORE_CONTINUE1]]
-; VEC: [[PRED_STORE_CONTINUE1]]:
-; VEC-NEXT: br i1 [[C]], label %[[PRED_STORE_IF1:.*]], label %[[PRED_STORE_CONTINUE2:.*]]
+; VEC-NEXT: br i1 [[C]], label %[[PRED_STORE_IF1:.*]], label %[[PRED_STORE_CONTINUE2]]
; VEC: [[PRED_STORE_IF1]]:
-; VEC-NEXT: [[TMP2:%.*]] = add i64 [[INDEX]], 1
-; VEC-NEXT: [[TMP3:%.*]] = getelementptr i8, ptr [[P]], i64 [[TMP2]]
-; VEC-NEXT: [[TMP4:%.*]] = extractelement <2 x i8> [[WIDE_LOAD]], i64 1
-; VEC-NEXT: store i8 [[TMP4]], ptr [[TMP3]], align 1
+; VEC-NEXT: store <2 x i8> [[WIDE_LOAD]], ptr [[TMP0]], align 1
; VEC-NEXT: br label %[[PRED_STORE_CONTINUE2]]
; VEC: [[PRED_STORE_CONTINUE2]]:
-; VEC-NEXT: br label %[[PRED_STORE_CONTINUE]]
-; VEC: [[PRED_STORE_CONTINUE]]:
; VEC-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 2
; VEC-NEXT: [[TMP5:%.*]] = icmp eq i64 [[INDEX_NEXT]], 1000
; VEC-NEXT: br i1 [[TMP5]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP5:![0-9]+]]
@@ -551,7 +522,7 @@ define void @minimal_bit_widths_with_aliasing_store(i1 %c, ptr %ptr) {
; UNROLL: [[VECTOR_PH]]:
; UNROLL-NEXT: br label %[[VECTOR_BODY:.*]]
; UNROLL: [[VECTOR_BODY]]:
-; UNROLL-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[PRED_STORE_CONTINUE:.*]] ]
+; UNROLL-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[PRED_STORE_CONTINUE1:.*]] ]
; UNROLL-NEXT: [[TMP0:%.*]] = add i64 [[INDEX]], 1
; UNROLL-NEXT: [[TMP1:%.*]] = getelementptr i8, ptr [[PTR]], i64 [[INDEX]]
; UNROLL-NEXT: [[TMP2:%.*]] = getelementptr i8, ptr [[PTR]], i64 [[TMP0]]
@@ -559,20 +530,12 @@ define void @minimal_bit_widths_with_aliasing_store(i1 %c, ptr %ptr) {
; UNROLL-NEXT: [[TMP4:%.*]] = load i8, ptr [[TMP2]], align 1
; UNROLL-NEXT: store i8 0, ptr [[TMP1]], align 1
; UNROLL-NEXT: store i8 0, ptr [[TMP2]], align 1
-; UNROLL-NEXT: br i1 [[C]], label %[[PRED_STORE_IF:.*]], label %[[PRED_STORE_CONTINUE]]
-; UNROLL: [[PRED_STORE_IF]]:
-; UNROLL-NEXT: br i1 [[C]], label %[[PRED_STORE_IF2:.*]], label %[[PRED_STORE_CONTINUE1:.*]]
+; UNROLL-NEXT: br i1 [[C]], label %[[PRED_STORE_IF2:.*]], label %[[PRED_STORE_CONTINUE1]]
; UNROLL: [[PRED_STORE_IF2]]:
; UNROLL-NEXT: store i8 [[TMP3]], ptr [[TMP1]], align 1
+; UNROLL-NEXT: store i8 [[TMP4]], ptr [[TMP2]], align 1
; UNROLL-NEXT: br label %[[PRED_STORE_CONTINUE1]]
; UNROLL: [[PRED_STORE_CONTINUE1]]:
-; UNROLL-NEXT: br i1 [[C]], label %[[PRED_STORE_IF1:.*]], label %[[PRED_STORE_CONTINUE2:.*]]
-; UNROLL: [[PRED_STORE_IF1]]:
-; UNROLL-NEXT: store i8 [[TMP4]], ptr [[TMP2]], align 1
-; UNROLL-NEXT: br label %[[PRED_STORE_CONTINUE2]]
-; UNROLL: [[PRED_STORE_CONTINUE2]]:
-; UNROLL-NEXT: br label %[[PRED_STORE_CONTINUE]]
-; UNROLL: [[PRED_STORE_CONTINUE]]:
; UNROLL-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 2
; UNROLL-NEXT: [[TMP5:%.*]] = icmp eq i64 [[INDEX_NEXT]], 1000
; UNROLL-NEXT: br i1 [[TMP5]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP6:![0-9]+]]
@@ -588,7 +551,7 @@ define void @minimal_bit_widths_with_aliasing_store(i1 %c, ptr %ptr) {
; UNROLL-NOSIMPLIFY: [[VECTOR_PH]]:
; UNROLL-NOSIMPLIFY-NEXT: br label %[[VECTOR_BODY:.*]]
; UNROLL-NOSIMPLIFY: [[VECTOR_BODY]]:
-; UNROLL-NOSIMPLIFY-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[PRED_STORE_CONTINUE:.*]] ]
+; UNROLL-NOSIMPLIFY-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[PRED_STORE_CONTINUE1:.*]] ]
; UNROLL-NOSIMPLIFY-NEXT: [[TMP0:%.*]] = add i64 [[INDEX]], 1
; UNROLL-NOSIMPLIFY-NEXT: [[TMP1:%.*]] = getelementptr i8, ptr [[PTR]], i64 [[INDEX]]
; UNROLL-NOSIMPLIFY-NEXT: [[TMP2:%.*]] = getelementptr i8, ptr [[PTR]], i64 [[TMP0]]
@@ -596,20 +559,12 @@ define void @minimal_bit_widths_with_aliasing_store(i1 %c, ptr %ptr) {
; UNROLL-NOSIMPLIFY-NEXT: [[TMP4:%.*]] = load i8, ptr [[TMP2]], align 1
; UNROLL-NOSIMPLIFY-NEXT: store i8 0, ptr [[TMP1]], align 1
; UNROLL-NOSIMPLIFY-NEXT: store i8 0, ptr [[TMP2]], align 1
-; UNROLL-NOSIMPLIFY-NEXT: br i1 [[C]], label %[[PRED_STORE_IF:.*]], label %[[PRED_STORE_CONTINUE]]
-; UNROLL-NOSIMPLIFY: [[PRED_STORE_IF]]:
-; UNROLL-NOSIMPLIFY-NEXT: br i1 [[C]], label %[[PRED_STORE_IF2:.*]], label %[[PRED_STORE_CONTINUE1:.*]]
+; UNROLL-NOSIMPLIFY-NEXT: br i1 [[C]], label %[[PRED_STORE_IF2:.*]], label %[[PRED_STORE_CONTINUE1]]
; UNROLL-NOSIMPLIFY: [[PRED_STORE_IF2]]:
; UNROLL-NOSIMPLIFY-NEXT: store i8 [[TMP3]], ptr [[TMP1]], align 1
+; UNROLL-NOSIMPLIFY-NEXT: store i8 [[TMP4]], ptr [[TMP2]], align 1
; UNROLL-NOSIMPLIFY-NEXT: br label %[[PRED_STORE_CONTINUE1]]
; UNROLL-NOSIMPLIFY: [[PRED_STORE_CONTINUE1]]:
-; UNROLL-NOSIMPLIFY-NEXT: br i1 [[C]], label %[[PRED_STORE_IF1:.*]], label %[[PRED_STORE_CONTINUE2:.*]]
-; UNROLL-NOSIMPLIFY: [[PRED_STORE_IF1]]:
-; UNROLL-NOSIMPLIFY-NEXT: store i8 [[TMP4]], ptr [[TMP2]], align 1
-; UNROLL-NOSIMPLIFY-NEXT: br label %[[PRED_STORE_CONTINUE2]]
-; UNROLL-NOSIMPLIFY: [[PRED_STORE_CONTINUE2]]:
-; UNROLL-NOSIMPLIFY-NEXT: br label %[[PRED_STORE_CONTINUE]]
-; UNROLL-NOSIMPLIFY: [[PRED_STORE_CONTINUE]]:
; UNROLL-NOSIMPLIFY-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 2
; UNROLL-NOSIMPLIFY-NEXT: [[TMP5:%.*]] = icmp eq i64 [[INDEX_NEXT]], 1000
; UNROLL-NOSIMPLIFY-NEXT: br i1 [[TMP5]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP6:![0-9]+]]
@@ -625,28 +580,15 @@ define void @minimal_bit_widths_with_aliasing_store(i1 %c, ptr %ptr) {
; VEC: [[VECTOR_PH]]:
; VEC-NEXT: br label %[[VECTOR_BODY:.*]]
; VEC: [[VECTOR_BODY]]:
-; VEC-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[PRED_STORE_CONTINUE:.*]] ]
+; VEC-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[PRED_STORE_CONTINUE2:.*]] ]
; VEC-NEXT: [[TMP0:%.*]] = getelementptr i8, ptr [[PTR]], i64 [[INDEX]]
; VEC-NEXT: [[WIDE_LOAD:%.*]] = load <2 x i8>, ptr [[TMP0]], align 1
; VEC-NEXT: store <2 x i8> zeroinitializer, ptr [[TMP0]], align 1
-; VEC-NEXT: br i1 [[C]], label %[[PRED_STORE_IF:.*]], label %[[PRED_STORE_CONTINUE]]
-; VEC: [[PRED_STORE_IF]]:
-; VEC-NEXT: br i1 [[C]], label %[[PRED_STORE_IF2:.*]], label %[[PRED_STORE_CONTINUE1:.*]]
-; VEC: [[PRED_STORE_IF2]]:
-; VEC-NEXT: [[TMP1:%.*]] = extractelement <2 x i8> [[WIDE_LOAD]], i64 0
-; VEC-NEXT: store i8 [[TMP1]], ptr [[TMP0]], align 1
-; VEC-NEXT: br label %[[PRED_STORE_CONTINUE1]]
-; VEC: [[PRED_STORE_CONTINUE1]]:
-; VEC-NEXT: br i1 [[C]], label %[[PRED_STORE_IF1:.*]], label %[[PRED_STORE_CONTINUE2:.*]]
+; VEC-NEXT: br i1 [[C]], label %[[PRED_STORE_IF1:.*]], label %[[PRED_STORE_CONTINUE2]]
; VEC: [[PRED_STORE_IF1]]:
-; VEC-NEXT: [[TMP2:%.*]] = add i64 [[INDEX]], 1
-; VEC-NEXT: [[TMP3:%.*]] = getelementptr i8, ptr [[PTR]], i64 [[TMP2]]
-; VEC-NEXT: [[TMP4:%.*]] = extractelement <2 x i8> [[WIDE_LOAD]], i64 1
-; VEC-NEXT: store i8 [[TMP4]], ptr [[TMP3]], align 1
+; VEC-NEXT: store <2 x i8> [[WIDE_LOAD]], ptr [[TMP0]], align 1
; VEC-NEXT: br label %[[PRED_STORE_CONTINUE2]]
; VEC: [[PRED_STORE_CONTINUE2]]:
-; VEC-NEXT: br label %[[PRED_STORE_CONTINUE]]
-; VEC: [[PRED_STORE_CONTINUE]]:
; VEC-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 2
; VEC-NEXT: [[TMP5:%.*]] = icmp eq i64 [[INDEX_NEXT]], 1000
; VEC-NEXT: br i1 [[TMP5]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP6:![0-9]+]]
@@ -685,31 +627,21 @@ for.end:
define void @sdiv_with_uniform_ops(i16 %0, i1 %c, ptr %dst) {
; UNROLL-LABEL: define void @sdiv_with_uniform_ops(
; UNROLL-SAME: i16 [[TMP0:%.*]], i1 [[C:%.*]], ptr [[DST:%.*]]) {
-; UNROLL-NEXT: [[ENTRY:.*:]]
-; UNROLL-NEXT: br label %[[VECTOR_PH:.*]]
-; UNROLL: [[VECTOR_PH]]:
+; UNROLL-NEXT: [[VECTOR_PH:.*:]]
; UNROLL-NEXT: br label %[[VECTOR_BODY:.*]]
; UNROLL: [[VECTOR_BODY]]:
-; UNROLL-NEXT: [[INDEX:%.*]] = phi i32 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[PRED_STORE_CONTINUE:.*]] ]
-; UNROLL-NEXT: br i1 [[C]], label %[[PRED_STORE_IF:.*]], label %[[PRED_STORE_CONTINUE]]
-; UNROLL: [[PRED_STORE_IF]]:
-; UNROLL-NEXT: br i1 [[C]], label %[[PRED_STORE_IF2:.*]], label %[[PRED_STORE_CONTINUE1:.*]]
-; UNROLL: [[PRED_STORE_IF2]]:
; UNROLL-NEXT: [[TMP1:%.*]] = sdiv i16 10, [[TMP0]]
-; UNROLL-NEXT: store i16 [[TMP1]], ptr [[DST]], align 1
-; UNROLL-NEXT: br label %[[PRED_STORE_CONTINUE1]]
+; UNROLL-NEXT: br label %[[PRED_STORE_CONTINUE1:.*]]
; UNROLL: [[PRED_STORE_CONTINUE1]]:
-; UNROLL-NEXT: br i1 [[C]], label %[[PRED_STORE_IF1:.*]], label %[[PRED_STORE_CONTINUE2:.*]]
+; UNROLL-NEXT: [[INDEX:%.*]] = phi i32 [ 0, %[[VECTOR_BODY]] ], [ [[INDEX_NEXT:%.*]], %[[PRED_STORE_CONTINUE2:.*]] ]
+; UNROLL-NEXT: br i1 [[C]], label %[[PRED_STORE_IF1:.*]], label %[[PRED_STORE_CONTINUE2]]
; UNROLL: [[PRED_STORE_IF1]]:
-; UNROLL-NEXT: [[TMP2:%.*]] = sdiv i16 10, [[TMP0]]
-; UNROLL-NEXT: store i16 [[TMP2]], ptr [[DST]], align 1
+; UNROLL-NEXT: store i16 [[TMP1]], ptr [[DST]], align 1
; UNROLL-NEXT: br label %[[PRED_STORE_CONTINUE2]]
; UNROLL: [[PRED_STORE_CONTINUE2]]:
-; UNROLL-NEXT: br label %[[PRED_STORE_CONTINUE]]
-; UNROLL: [[PRED_STORE_CONTINUE]]:
; UNROLL-NEXT: [[INDEX_NEXT]] = add nuw i32 [[INDEX]], 2
; UNROLL-NEXT: [[TMP3:%.*]] = icmp eq i32 [[INDEX_NEXT]], 98
-; UNROLL-NEXT: br i1 [[TMP3]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP7:![0-9]+]]
+; UNROLL-NEXT: br i1 [[TMP3]], label %[[MIDDLE_BLOCK:.*]], label %[[PRED_STORE_CONTINUE1]], !llvm.loop [[LOOP7:![0-9]+]]
; UNROLL: [[MIDDLE_BLOCK]]:
; UNROLL-NEXT: br label %[[SCALAR_PH:.*]]
; UNROLL: [[SCALAR_PH]]:
@@ -730,31 +662,21 @@ define void @sdiv_with_uniform_ops(i16 %0, i1 %c, ptr %dst) {
;
; UNROLL-NOSIMPLIFY-LABEL: define void @sdiv_with_uniform_ops(
; UNROLL-NOSIMPLIFY-SAME: i16 [[TMP0:%.*]], i1 [[C:%.*]], ptr [[DST:%.*]]) {
-; UNROLL-NOSIMPLIFY-NEXT: [[ENTRY:.*:]]
-; UNROLL-NOSIMPLIFY-NEXT: br label %[[VECTOR_PH:.*]]
-; UNROLL-NOSIMPLIFY: [[VECTOR_PH]]:
+; UNROLL-NOSIMPLIFY-NEXT: [[VECTOR_PH:.*:]]
; UNROLL-NOSIMPLIFY-NEXT: br label %[[VECTOR_BODY:.*]]
; UNROLL-NOSIMPLIFY: [[VECTOR_BODY]]:
-; UNROLL-NOSIMPLIFY-NEXT: [[INDEX:%.*]] = phi i32 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[PRED_STORE_CONTINUE:.*]] ]
-; UNROLL-NOSIMPLIFY-NEXT: br i1 [[C]], label %[[PRED_STORE_IF:.*]], label %[[PRED_STORE_CONTINUE]]
-; UNROLL-NOSIMPLIFY: [[PRED_STORE_IF]]:
-; UNROLL-NOSIMPLIFY-NEXT: br i1 [[C]], label %[[PRED_STORE_IF2:.*]], label %[[PRED_STORE_CONTINUE1:.*]]
-; UNROLL-NOSIMPLIFY: [[PRED_STORE_IF2]]:
; UNROLL-NOSIMPLIFY-NEXT: [[TMP1:%.*]] = sdiv i16 10, [[TMP0]]
-; UNROLL-NOSIMPLIFY-NEXT: store i16 [[TMP1]], ptr [[DST]], align 1
-; UNROLL-NOSIMPLIFY-NEXT: br label %[[PRED_STORE_CONTINUE1]]
+; UNROLL-NOSIMPLIFY-NEXT: br label %[[PRED_STORE_CONTINUE1:.*]]
; UNROLL-NOSIMPLIFY: [[PRED_STORE_CONTINUE1]]:
-; UNROLL-NOSIMPLIFY-NEXT: br i1 [[C]], label %[[PRED_STORE_IF1:.*]], label %[[PRED_STORE_CONTINUE2:.*]]
+; UNROLL-NOSIMPLIFY-NEXT: [[INDEX:%.*]] = phi i32 [ 0, %[[VECTOR_BODY]] ], [ [[INDEX_NEXT:%.*]], %[[PRED_STORE_CONTINUE2:.*]] ]
+; UNROLL-NOSIMPLIFY-NEXT: br i1 [[C]], label %[[PRED_STORE_IF1:.*]], label %[[PRED_STORE_CONTINUE2]]
; UNROLL-NOSIMPLIFY: [[PRED_STORE_IF1]]:
-; UNROLL-NOSIMPLIFY-NEXT: [[TMP2:%.*]] = sdiv i16 10, [[TMP0]]
-; UNROLL-NOSIMPLIFY-NEXT: store i16 [[TMP2]], ptr [[DST]], align 1
+; UNROLL-NOSIMPLIFY-NEXT: store i16 [[TMP1]], ptr [[DST]], align 1
; UNROLL-NOSIMPLIFY-NEXT: br label %[[PRED_STORE_CONTINUE2]]
; UNROLL-NOSIMPLIFY: [[PRED_STORE_CONTINUE2]]:
-; UNROLL-NOSIMPLIFY-NEXT: br label %[[PRED_STORE_CONTINUE]]
-; UNROLL-NOSIMPLIFY: [[PRED_STORE_CONTINUE]]:
; UNROLL-NOSIMPLIFY-NEXT: [[INDEX_NEXT]] = add nuw i32 [[INDEX]], 2
; UNROLL-NOSIMPLIFY-NEXT: [[TMP3:%.*]] = icmp eq i32 [[INDEX_NEXT]], 98
-; UNROLL-NOSIMPLIFY-NEXT: br i1 [[TMP3]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP7:![0-9]+]]
+; UNROLL-NOSIMPLIFY-NEXT: br i1 [[TMP3]], label %[[MIDDLE_BLOCK:.*]], label %[[PRED_STORE_CONTINUE1]], !llvm.loop [[LOOP7:![0-9]+]]
; UNROLL-NOSIMPLIFY: [[MIDDLE_BLOCK]]:
; UNROLL-NOSIMPLIFY-NEXT: br label %[[SCALAR_PH:.*]]
; UNROLL-NOSIMPLIFY: [[SCALAR_PH]]:
@@ -775,36 +697,21 @@ define void @sdiv_with_uniform_ops(i16 %0, i1 %c, ptr %dst) {
;
; VEC-LABEL: define void @sdiv_with_uniform_ops(
; VEC-SAME: i16 [[TMP0:%.*]], i1 [[C:%.*]], ptr [[DST:%.*]]) {
-; VEC-NEXT: [[ENTRY:.*:]]
-; VEC-NEXT: br label %[[VECTOR_PH:.*]]
-; VEC: [[VECTOR_PH]]:
-; VEC-NEXT: [[BROADCAST_SPLATINSERT:%.*]] = insertelement <2 x i16> poison, i16 [[TMP0]], i64 0
-; VEC-NEXT: [[BROADCAST_SPLAT:%.*]] = shufflevector <2 x i16> [[BROADCAST_SPLATINSERT]], <2 x i16> poison, <2 x i32> zeroinitializer
-; VEC-NEXT: [[BROADCAST_SPLATINSERT1:%.*]] = insertelement <2 x i1> poison, i1 [[C]], i64 0
-; VEC-NEXT: [[BROADCAST_SPLAT2:%.*]] = shufflevector <2 x i1> [[BROADCAST_SPLATINSERT1]], <2 x i1> poison, <2 x i32> zeroinitializer
-; VEC-NEXT: [[TMP1:%.*]] = call <2 x i16> @llvm.masked.sdiv.v2i16(<2 x i16> splat (i16 10), <2 x i16> [[BROADCAST_SPLAT]], <2 x i1> [[BROADCAST_SPLAT2]])
+; VEC-NEXT: [[VECTOR_PH:.*:]]
; VEC-NEXT: br label %[[VECTOR_BODY:.*]]
; VEC: [[VECTOR_BODY]]:
-; VEC-NEXT: [[INDEX:%.*]] = phi i32 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[PRED_STORE_CONTINUE:.*]] ]
-; VEC-NEXT: br i1 [[C]], label %[[PRED_STORE_IF:.*]], label %[[PRED_STORE_CONTINUE]]
-; VEC: [[PRED_STORE_IF]]:
-; VEC-NEXT: br i1 [[C]], label %[[PRED_STORE_IF1:.*]], label %[[PRED_STORE_CONTINUE1:.*]]
-; VEC: [[PRED_STORE_IF1]]:
-; VEC-NEXT: [[TMP2:%.*]] = extractelement <2 x i16> [[TMP1]], i64 0
-; VEC-NEXT: store i16 [[TMP2]], ptr [[DST]], align 1
-; VEC-NEXT: br label %[[PRED_STORE_CONTINUE1]]
+; VEC-NEXT: [[TMP3:%.*]] = sdiv i16 10, [[TMP0]]
+; VEC-NEXT: br label %[[PRED_STORE_CONTINUE1:.*]]
; VEC: [[PRED_STORE_CONTINUE1]]:
-; VEC-NEXT: br i1 [[C]], label %[[PRED_STORE_IF3:.*]], label %[[PRED_STORE_CONTINUE4:.*]]
+; VEC-NEXT: [[INDEX:%.*]] = phi i32 [ 0, %[[VECTOR_BODY]] ], [ [[INDEX_NEXT:%.*]], %[[PRED_STORE_CONTINUE4:.*]] ]
+; VEC-NEXT: br i1 [[C]], label %[[PRED_STORE_IF3:.*]], label %[[PRED_STORE_CONTINUE4]]
; VEC: [[PRED_STORE_IF3]]:
-; VEC-NEXT: [[TMP3:%.*]] = extractelement <2 x i16> [[TMP1]], i64 1
; VEC-NEXT: store i16 [[TMP3]], ptr [[DST]], align 1
; VEC-NEXT: br label %[[PRED_STORE_CONTINUE4]]
; VEC: [[PRED_STORE_CONTINUE4]]:
-; VEC-NEXT: br label %[[PRED_STORE_CONTINUE]]
-; VEC: [[PRED_STORE_CONTINUE]]:
; VEC-NEXT: [[INDEX_NEXT]] = add nuw i32 [[INDEX]], 2
; VEC-NEXT: [[TMP4:%.*]] = icmp eq i32 [[INDEX_NEXT]], 98
-; VEC-NEXT: br i1 [[TMP4]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP7:![0-9]+]]
+; VEC-NEXT: br i1 [[TMP4]], label %[[MIDDLE_BLOCK:.*]], label %[[PRED_STORE_CONTINUE1]], !llvm.loop [[LOOP7:![0-9]+]]
; VEC: [[MIDDLE_BLOCK]]:
; VEC-NEXT: br label %[[SCALAR_PH:.*]]
; VEC: [[SCALAR_PH]]:
diff --git a/llvm/test/Transforms/LoopVectorize/multiple-result-intrinsics.ll b/llvm/test/Transforms/LoopVectorize/multiple-result-intrinsics.ll
index ce9844e89e7b1..53a1e06be828a 100644
--- a/llvm/test/Transforms/LoopVectorize/multiple-result-intrinsics.ll
+++ b/llvm/test/Transforms/LoopVectorize/multiple-result-intrinsics.ll
@@ -827,18 +827,8 @@ define void @sincos_predicated_store(i1 %c, ptr noalias %src, ptr noalias %dst,
; CHECK: store <2 x double> zeroinitializer, ptr [[TMP3:%.*]], align 8
; CHECK: [[IF1:.*:]]
; CHECK: [[TMP2:%.*]] = extractvalue { <2 x double>, <2 x double> } [[TMP0]], 1
-; CHECK: br i1 [[TMP1:%.*]], label %[[PRED_STORE_IF:.*]], label %[[PRED_STORE_CONTINUE:.*]]
-; CHECK: [[PRED_STORE_IF]]:
-; CHECK: [[TMP4:%.*]] = extractelement <2 x double> [[TMP2]], i64 0
-; CHECK: store double [[TMP4]], ptr [[COS_DST]], align 8
-; CHECK: br label %[[PRED_STORE_CONTINUE]]
-; CHECK: [[PRED_STORE_CONTINUE]]:
-; CHECK: br i1 [[TMP1]], label %[[PRED_STORE_IF1:.*]], label %[[PRED_STORE_CONTINUE2:.*]]
-; CHECK: [[PRED_STORE_IF1]]:
; CHECK: [[TMP5:%.*]] = extractelement <2 x double> [[TMP2]], i64 1
; CHECK: store double [[TMP5]], ptr [[COS_DST]], align 8
-; CHECK: br label %[[PRED_STORE_CONTINUE2]]
-; CHECK: [[PRED_STORE_CONTINUE2]]:
; CHECK: [[MIDDLE_BLOCK:.*:]]
; CHECK: [[EXIT:.*:]]
; CHECK: [[EXIT1:.*:]]
diff --git a/llvm/test/Transforms/LoopVectorize/phi-with-fastflags.ll b/llvm/test/Transforms/LoopVectorize/phi-with-fastflags.ll
index 625c9e323ec6f..05e84042d5b24 100644
--- a/llvm/test/Transforms/LoopVectorize/phi-with-fastflags.ll
+++ b/llvm/test/Transforms/LoopVectorize/phi-with-fastflags.ll
@@ -7,8 +7,6 @@ define void @f(ptr noalias %p, i1 %c) {
; CHECK-NEXT: [[ENTRY:.*:]]
; CHECK-NEXT: br label %[[VECTOR_PH:.*]]
; CHECK: [[VECTOR_PH]]:
-; CHECK-NEXT: [[BROADCAST_SPLATINSERT:%.*]] = insertelement <4 x i1> poison, i1 [[C]], i64 0
-; CHECK-NEXT: [[BROADCAST_SPLAT:%.*]] = shufflevector <4 x i1> [[BROADCAST_SPLATINSERT]], <4 x i1> poison, <4 x i32> zeroinitializer
; CHECK-NEXT: br label %[[VECTOR_BODY:.*]]
; CHECK: [[VECTOR_BODY]]:
; CHECK-NEXT: [[INDEX:%.*]] = phi i32 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[LATCH2:.*]] ]
@@ -18,7 +16,7 @@ define void @f(ptr noalias %p, i1 %c) {
; CHECK: [[THEN1]]:
; CHECK-NEXT: br label %[[LATCH2]]
; CHECK: [[LATCH2]]:
-; CHECK-NEXT: [[TMP2:%.*]] = phi <4 x i1> [ zeroinitializer, %[[VECTOR_BODY]] ], [ [[BROADCAST_SPLAT]], %[[THEN1]] ]
+; CHECK-NEXT: [[TMP2:%.*]] = phi <4 x i1> [ zeroinitializer, %[[VECTOR_BODY]] ], [ splat (i1 true), %[[THEN1]] ]
; CHECK-NEXT: [[PREDPHI:%.*]] = select fast <4 x i1> [[TMP2]], <4 x float> zeroinitializer, <4 x float> [[WIDE_LOAD]]
; CHECK-NEXT: store <4 x float> [[PREDPHI]], ptr [[TMP0]], align 4
; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i32 [[INDEX]], 4
diff --git a/llvm/test/Transforms/LoopVectorize/pr37248.ll b/llvm/test/Transforms/LoopVectorize/pr37248.ll
index 54109d418f5ba..9d86032bd27bc 100644
--- a/llvm/test/Transforms/LoopVectorize/pr37248.ll
+++ b/llvm/test/Transforms/LoopVectorize/pr37248.ll
@@ -35,23 +35,13 @@ define void @f1(ptr noalias %b, i1 %c, i32 %start) {
; CHECK-NEXT: [[N_MOD_VF:%.*]] = and i32 [[TMP1]], 1
; CHECK-NEXT: [[N_VEC:%.*]] = sub i32 [[TMP1]], [[N_MOD_VF]]
; CHECK-NEXT: [[TMP10:%.*]] = sub i32 [[START]], [[N_VEC]]
-; CHECK-NEXT: [[TMP12:%.*]] = xor i1 [[C]], true
; CHECK-NEXT: br label %[[VECTOR_BODY:.*]]
; CHECK: [[VECTOR_BODY]]:
; CHECK-NEXT: [[INDEX:%.*]] = phi i32 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[LOOP_LATCH4:.*]] ]
; CHECK-NEXT: [[OFFSET_IDX:%.*]] = sub i32 [[START]], [[INDEX]]
; CHECK-NEXT: br i1 [[C]], label %[[LOOP_LATCH4]], label %[[ELSE1:.*]]
; CHECK: [[ELSE1]]:
-; CHECK-NEXT: br i1 [[TMP12]], label %[[PRED_STORE_IF:.*]], label %[[PRED_STORE_CONTINUE:.*]]
-; CHECK: [[PRED_STORE_IF]]:
; CHECK-NEXT: store i32 10, ptr [[B]], align 1
-; CHECK-NEXT: br label %[[PRED_STORE_CONTINUE]]
-; CHECK: [[PRED_STORE_CONTINUE]]:
-; CHECK-NEXT: br i1 [[TMP12]], label %[[PRED_STORE_IF2:.*]], label %[[PRED_STORE_CONTINUE3:.*]]
-; CHECK: [[PRED_STORE_IF2]]:
-; CHECK-NEXT: store i32 10, ptr [[B]], align 1
-; CHECK-NEXT: br label %[[PRED_STORE_CONTINUE3]]
-; CHECK: [[PRED_STORE_CONTINUE3]]:
; CHECK-NEXT: br label %[[LOOP_LATCH4]]
; CHECK: [[LOOP_LATCH4]]:
; CHECK-NEXT: [[TMP13:%.*]] = trunc i32 [[OFFSET_IDX]] to i16
diff --git a/llvm/test/Transforms/LoopVectorize/pr45525.ll b/llvm/test/Transforms/LoopVectorize/pr45525.ll
index 73f0c8cc9c4a5..11f56f3c90d02 100644
--- a/llvm/test/Transforms/LoopVectorize/pr45525.ll
+++ b/llvm/test/Transforms/LoopVectorize/pr45525.ll
@@ -9,9 +9,6 @@ define void @main(i1 %cond, ptr %arr) {
; CHECK-NEXT: [[BB_0:.*:]]
; CHECK-NEXT: br label %[[VECTOR_PH:.*]]
; CHECK: [[VECTOR_PH]]:
-; CHECK-NEXT: [[BROADCAST_SPLATINSERT:%.*]] = insertelement <4 x i1> poison, i1 [[COND]], i64 0
-; CHECK-NEXT: [[BROADCAST_SPLAT:%.*]] = shufflevector <4 x i1> [[BROADCAST_SPLATINSERT]], <4 x i1> poison, <4 x i32> zeroinitializer
-; CHECK-NEXT: [[TMP0:%.*]] = xor <4 x i1> [[BROADCAST_SPLAT]], splat (i1 true)
; CHECK-NEXT: br label %[[VECTOR_BODY:.*]]
; CHECK: [[VECTOR_BODY]]:
; CHECK-NEXT: [[INDEX:%.*]] = phi i32 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[BB_32:.*]] ]
@@ -22,7 +19,7 @@ define void @main(i1 %cond, ptr %arr) {
; CHECK-NEXT: br label %[[BB_32]]
; CHECK: [[BB_32]]:
; CHECK-NEXT: [[TMP4:%.*]] = phi <4 x i32> [ poison, %[[VECTOR_BODY]] ], [ [[TMP5]], %[[BB_21]] ]
-; CHECK-NEXT: [[TMP3:%.*]] = phi <4 x i1> [ zeroinitializer, %[[VECTOR_BODY]] ], [ [[TMP0]], %[[BB_21]] ]
+; CHECK-NEXT: [[TMP3:%.*]] = phi <4 x i1> [ zeroinitializer, %[[VECTOR_BODY]] ], [ splat (i1 true), %[[BB_21]] ]
; CHECK-NEXT: [[PREDPHI:%.*]] = select <4 x i1> [[TMP3]], <4 x i32> [[TMP4]], <4 x i32> splat (i32 7)
; CHECK-NEXT: [[TMP1:%.*]] = getelementptr inbounds i32, ptr [[ARR]], i32 [[INDEX]]
; CHECK-NEXT: store <4 x i32> [[PREDPHI]], ptr [[TMP1]], align 4
diff --git a/llvm/test/Transforms/LoopVectorize/predicator.ll b/llvm/test/Transforms/LoopVectorize/predicator.ll
index 1d8321f28b981..4c895a0452fd4 100644
--- a/llvm/test/Transforms/LoopVectorize/predicator.ll
+++ b/llvm/test/Transforms/LoopVectorize/predicator.ll
@@ -347,8 +347,6 @@ define void @outermost_uniform_branch(ptr %a, i1 %u0) {
; CHECK-NEXT: [[ENTRY:.*:]]
; CHECK-NEXT: br label %[[VECTOR_PH:.*]]
; CHECK: [[VECTOR_PH]]:
-; CHECK-NEXT: [[BROADCAST_SPLATINSERT:%.*]] = insertelement <4 x i1> poison, i1 [[U0]], i64 0
-; CHECK-NEXT: [[BROADCAST_SPLAT:%.*]] = shufflevector <4 x i1> [[BROADCAST_SPLATINSERT]], <4 x i1> poison, <4 x i32> zeroinitializer
; CHECK-NEXT: br label %[[VECTOR_BODY:.*]]
; CHECK: [[VECTOR_BODY]]:
; CHECK-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[BB42:.*]] ]
@@ -363,7 +361,7 @@ define void @outermost_uniform_branch(ptr %a, i1 %u0) {
; CHECK-NEXT: br label %[[BB42]]
; CHECK: [[BB42]]:
; CHECK-NEXT: [[TMP6:%.*]] = phi <4 x i64> [ poison, %[[VECTOR_BODY]] ], [ [[TMP4]], %[[MIDDLE_BLOCK]] ]
-; CHECK-NEXT: [[TMP8:%.*]] = phi <4 x i1> [ zeroinitializer, %[[VECTOR_BODY]] ], [ [[BROADCAST_SPLAT]], %[[MIDDLE_BLOCK]] ]
+; CHECK-NEXT: [[TMP8:%.*]] = phi <4 x i1> [ zeroinitializer, %[[VECTOR_BODY]] ], [ splat (i1 true), %[[MIDDLE_BLOCK]] ]
; CHECK-NEXT: [[PREDPHI1:%.*]] = select <4 x i1> [[TMP8]], <4 x i64> [[TMP6]], <4 x i64> [[VEC_IND]]
; CHECK-NEXT: [[TMP5:%.*]] = extractelement <4 x i64> [[PREDPHI1]], i64 3
; CHECK-NEXT: store i64 [[TMP5]], ptr [[A]], align 4
@@ -514,8 +512,6 @@ define void @uniform_branch_after_varying_branch(ptr %a, i1 %u1) {
; CHECK-NEXT: [[ENTRY:.*:]]
; CHECK-NEXT: br label %[[VECTOR_PH:.*]]
; CHECK: [[VECTOR_PH]]:
-; CHECK-NEXT: [[BROADCAST_SPLATINSERT:%.*]] = insertelement <4 x i1> poison, i1 [[U1]], i64 0
-; CHECK-NEXT: [[BROADCAST_SPLAT:%.*]] = shufflevector <4 x i1> [[BROADCAST_SPLATINSERT]], <4 x i1> poison, <4 x i32> zeroinitializer
; CHECK-NEXT: br label %[[VECTOR_BODY:.*]]
; CHECK: [[VECTOR_BODY]]:
; CHECK-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[BB32:.*]] ]
@@ -528,7 +524,7 @@ define void @uniform_branch_after_varying_branch(ptr %a, i1 %u1) {
; CHECK-NEXT: br label %[[BB32]]
; CHECK: [[BB32]]:
; CHECK-NEXT: [[TMP7:%.*]] = phi <4 x i64> [ poison, %[[VECTOR_BODY]] ], [ [[TMP3]], %[[BB21]] ]
-; CHECK-NEXT: [[TMP9:%.*]] = phi <4 x i1> [ zeroinitializer, %[[VECTOR_BODY]] ], [ [[BROADCAST_SPLAT]], %[[BB21]] ]
+; CHECK-NEXT: [[TMP9:%.*]] = phi <4 x i1> [ zeroinitializer, %[[VECTOR_BODY]] ], [ splat (i1 true), %[[BB21]] ]
; CHECK-NEXT: [[PREDPHI:%.*]] = select <4 x i1> [[TMP9]], <4 x i64> [[TMP7]], <4 x i64> [[TMP2]]
; CHECK-NEXT: [[TMP4:%.*]] = add <4 x i64> [[PREDPHI]], splat (i64 3)
; CHECK-NEXT: [[TMP5:%.*]] = add <4 x i64> [[VEC_IND]], splat (i64 4)
diff --git a/llvm/test/Transforms/LoopVectorize/scalarize-masked-call.ll b/llvm/test/Transforms/LoopVectorize/scalarize-masked-call.ll
index 2928a9f0cbf0b..56c3615636ec4 100644
--- a/llvm/test/Transforms/LoopVectorize/scalarize-masked-call.ll
+++ b/llvm/test/Transforms/LoopVectorize/scalarize-masked-call.ll
@@ -101,31 +101,19 @@ define void @masked_umax_used_by_load_address(ptr noalias %src, ptr noalias %dst
; CHECK: vector.ph:
; CHECK-NEXT: [[TMP0:%.*]] = call i64 @llvm.umax.i64(i64 [[N:%.*]], i64 1)
; CHECK-NEXT: [[TMP1:%.*]] = getelementptr i16, ptr [[SRC:%.*]], i64 [[TMP0]]
-; CHECK-NEXT: [[BROADCAST_SPLATINSERT:%.*]] = insertelement <2 x i1> poison, i1 [[C:%.*]], i64 0
-; CHECK-NEXT: [[BROADCAST_SPLAT:%.*]] = shufflevector <2 x i1> [[BROADCAST_SPLATINSERT]], <2 x i1> poison, <2 x i32> zeroinitializer
; CHECK-NEXT: br label [[VECTOR_BODY:%.*]]
; CHECK: vector.body:
; CHECK-NEXT: [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[PRED_LOAD_CONTINUE2:%.*]] ]
-; CHECK-NEXT: br i1 [[C]], label [[THEN1:%.*]], label [[PRED_LOAD_CONTINUE2]]
+; CHECK-NEXT: br i1 [[C:%.*]], label [[THEN1:%.*]], label [[PRED_LOAD_CONTINUE2]]
; CHECK: then1:
-; CHECK-NEXT: br i1 [[C]], label [[PRED_LOAD_IF:%.*]], label [[PRED_LOAD_CONTINUE:%.*]]
-; CHECK: pred.load.if:
; CHECK-NEXT: [[TMP2:%.*]] = load i16, ptr [[TMP1]], align 2
-; CHECK-NEXT: [[TMP3:%.*]] = insertelement <2 x i16> poison, i16 [[TMP2]], i64 0
-; CHECK-NEXT: br label [[PRED_LOAD_CONTINUE]]
-; CHECK: pred.load.continue:
-; CHECK-NEXT: [[TMP4:%.*]] = phi <2 x i16> [ poison, [[THEN1]] ], [ [[TMP3]], [[PRED_LOAD_IF]] ]
-; CHECK-NEXT: br i1 [[C]], label [[PRED_LOAD_IF1:%.*]], label [[PRED_LOAD_CONTINUE3:%.*]]
-; CHECK: pred.load.if2:
; CHECK-NEXT: [[TMP5:%.*]] = load i16, ptr [[TMP1]], align 2
+; CHECK-NEXT: [[TMP4:%.*]] = insertelement <2 x i16> poison, i16 [[TMP2]], i64 0
; CHECK-NEXT: [[TMP6:%.*]] = insertelement <2 x i16> [[TMP4]], i16 [[TMP5]], i64 1
-; CHECK-NEXT: br label [[PRED_LOAD_CONTINUE3]]
-; CHECK: pred.load.continue3:
-; CHECK-NEXT: [[TMP7:%.*]] = phi <2 x i16> [ [[TMP4]], [[PRED_LOAD_CONTINUE]] ], [ [[TMP6]], [[PRED_LOAD_IF1]] ]
; CHECK-NEXT: br label [[PRED_LOAD_CONTINUE2]]
-; CHECK: loop.latch4:
-; CHECK-NEXT: [[TMP10:%.*]] = phi <2 x i16> [ poison, [[VECTOR_BODY]] ], [ [[TMP7]], [[PRED_LOAD_CONTINUE3]] ]
-; CHECK-NEXT: [[TMP11:%.*]] = phi <2 x i1> [ zeroinitializer, [[VECTOR_BODY]] ], [ [[BROADCAST_SPLAT]], [[PRED_LOAD_CONTINUE3]] ]
+; CHECK: loop.latch2:
+; CHECK-NEXT: [[TMP10:%.*]] = phi <2 x i16> [ poison, [[VECTOR_BODY]] ], [ [[TMP6]], [[THEN1]] ]
+; CHECK-NEXT: [[TMP11:%.*]] = phi <2 x i1> [ zeroinitializer, [[VECTOR_BODY]] ], [ splat (i1 true), [[THEN1]] ]
; CHECK-NEXT: [[PREDPHI:%.*]] = select <2 x i1> [[TMP11]], <2 x i16> [[TMP10]], <2 x i16> zeroinitializer
; CHECK-NEXT: [[TMP8:%.*]] = getelementptr i16, ptr [[DST:%.*]], i64 [[INDEX]]
; CHECK-NEXT: store <2 x i16> [[PREDPHI]], ptr [[TMP8]], align 2
diff --git a/llvm/test/Transforms/LoopVectorize/struct-return-replicate.ll b/llvm/test/Transforms/LoopVectorize/struct-return-replicate.ll
index 4e2c20dd6f180..7c1f79a1761b7 100644
--- a/llvm/test/Transforms/LoopVectorize/struct-return-replicate.ll
+++ b/llvm/test/Transforms/LoopVectorize/struct-return-replicate.ll
@@ -662,7 +662,6 @@ define void @struct_return_predicated_extractvalue(i1 %c, ptr noalias %a, ptr no
; VF4-NEXT: [[ENTRY:.*:]]
; VF4-NEXT: br label %[[VECTOR_PH:.*]]
; VF4: [[VECTOR_PH]]:
-; VF4-NEXT: [[TMP0:%.*]] = xor i1 [[C]], true
; VF4-NEXT: br label %[[VECTOR_BODY:.*]]
; VF4: [[VECTOR_BODY]]:
; VF4-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[LATCH8:.*]] ]
@@ -711,30 +710,8 @@ define void @struct_return_predicated_extractvalue(i1 %c, ptr noalias %a, ptr no
; VF4-NEXT: br i1 [[C]], label %[[LATCH8]], label %[[IF1:.*]]
; VF4: [[IF1]]:
; VF4-NEXT: [[TMP36:%.*]] = extractvalue { <4 x float>, <4 x float> } [[TMP35]], 1
-; VF4-NEXT: br i1 [[TMP0]], label %[[PRED_STORE_IF:.*]], label %[[PRED_STORE_CONTINUE:.*]]
-; VF4: [[PRED_STORE_IF]]:
-; VF4-NEXT: [[TMP37:%.*]] = extractelement <4 x float> [[TMP36]], i64 0
-; VF4-NEXT: store float [[TMP37]], ptr [[Q]], align 4
-; VF4-NEXT: br label %[[PRED_STORE_CONTINUE]]
-; VF4: [[PRED_STORE_CONTINUE]]:
-; VF4-NEXT: br i1 [[TMP0]], label %[[PRED_STORE_IF1:.*]], label %[[PRED_STORE_CONTINUE2:.*]]
-; VF4: [[PRED_STORE_IF1]]:
-; VF4-NEXT: [[TMP38:%.*]] = extractelement <4 x float> [[TMP36]], i64 1
-; VF4-NEXT: store float [[TMP38]], ptr [[Q]], align 4
-; VF4-NEXT: br label %[[PRED_STORE_CONTINUE2]]
-; VF4: [[PRED_STORE_CONTINUE2]]:
-; VF4-NEXT: br i1 [[TMP0]], label %[[PRED_STORE_IF3:.*]], label %[[PRED_STORE_CONTINUE4:.*]]
-; VF4: [[PRED_STORE_IF3]]:
-; VF4-NEXT: [[TMP39:%.*]] = extractelement <4 x float> [[TMP36]], i64 2
-; VF4-NEXT: store float [[TMP39]], ptr [[Q]], align 4
-; VF4-NEXT: br label %[[PRED_STORE_CONTINUE4]]
-; VF4: [[PRED_STORE_CONTINUE4]]:
-; VF4-NEXT: br i1 [[TMP0]], label %[[PRED_STORE_IF5:.*]], label %[[PRED_STORE_CONTINUE6:.*]]
-; VF4: [[PRED_STORE_IF5]]:
; VF4-NEXT: [[TMP40:%.*]] = extractelement <4 x float> [[TMP36]], i64 3
; VF4-NEXT: store float [[TMP40]], ptr [[Q]], align 4
-; VF4-NEXT: br label %[[PRED_STORE_CONTINUE6]]
-; VF4: [[PRED_STORE_CONTINUE6]]:
; VF4-NEXT: br label %[[LATCH8]]
; VF4: [[LATCH8]]:
; VF4-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 4
@@ -747,7 +724,6 @@ define void @struct_return_predicated_extractvalue(i1 %c, ptr noalias %a, ptr no
; VF2IC2-NEXT: [[ENTRY:.*:]]
; VF2IC2-NEXT: br label %[[VECTOR_PH:.*]]
; VF2IC2: [[VECTOR_PH]]:
-; VF2IC2-NEXT: [[TMP0:%.*]] = xor i1 [[C]], true
; VF2IC2-NEXT: br label %[[VECTOR_BODY:.*]]
; VF2IC2: [[VECTOR_BODY]]:
; VF2IC2-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[LATCH9:.*]] ]
@@ -756,24 +732,9 @@ define void @struct_return_predicated_extractvalue(i1 %c, ptr noalias %a, ptr no
; VF2IC2-NEXT: [[WIDE_LOAD:%.*]] = load <2 x float>, ptr [[TMP42]], align 4
; VF2IC2-NEXT: [[WIDE_LOAD1:%.*]] = load <2 x float>, ptr [[TMP43]], align 4
; VF2IC2-NEXT: [[TMP44:%.*]] = extractelement <2 x float> [[WIDE_LOAD]], i64 0
-; VF2IC2-NEXT: [[TMP1:%.*]] = tail call { float, float } @fn2(float [[TMP44]]) #[[ATTR3]]
+; VF2IC2-NEXT: [[TMP3:%.*]] = tail call { float, float } @fn2(float [[TMP44]]) #[[ATTR3]]
; VF2IC2-NEXT: [[TMP45:%.*]] = extractelement <2 x float> [[WIDE_LOAD]], i64 1
-; VF2IC2-NEXT: [[TMP2:%.*]] = tail call { float, float } @fn2(float [[TMP45]]) #[[ATTR3]]
-; VF2IC2-NEXT: [[TMP3:%.*]] = extractvalue { float, float } [[TMP1]], 0
-; VF2IC2-NEXT: [[TMP4:%.*]] = insertelement <2 x float> poison, float [[TMP3]], i64 0
-; VF2IC2-NEXT: [[TMP5:%.*]] = insertvalue { <2 x float>, <2 x float> } poison, <2 x float> [[TMP4]], 0
-; VF2IC2-NEXT: [[TMP6:%.*]] = extractvalue { float, float } [[TMP1]], 1
-; VF2IC2-NEXT: [[TMP7:%.*]] = extractvalue { <2 x float>, <2 x float> } [[TMP5]], 1
-; VF2IC2-NEXT: [[TMP8:%.*]] = insertelement <2 x float> [[TMP7]], float [[TMP6]], i64 0
-; VF2IC2-NEXT: [[TMP9:%.*]] = insertvalue { <2 x float>, <2 x float> } [[TMP5]], <2 x float> [[TMP8]], 1
-; VF2IC2-NEXT: [[TMP10:%.*]] = extractvalue { float, float } [[TMP2]], 0
-; VF2IC2-NEXT: [[TMP11:%.*]] = extractvalue { <2 x float>, <2 x float> } [[TMP9]], 0
-; VF2IC2-NEXT: [[TMP12:%.*]] = insertelement <2 x float> [[TMP11]], float [[TMP10]], i64 1
-; VF2IC2-NEXT: [[TMP13:%.*]] = insertvalue { <2 x float>, <2 x float> } [[TMP9]], <2 x float> [[TMP12]], 0
-; VF2IC2-NEXT: [[TMP14:%.*]] = extractvalue { float, float } [[TMP2]], 1
-; VF2IC2-NEXT: [[TMP15:%.*]] = extractvalue { <2 x float>, <2 x float> } [[TMP13]], 1
-; VF2IC2-NEXT: [[TMP16:%.*]] = insertelement <2 x float> [[TMP15]], float [[TMP14]], i64 1
-; VF2IC2-NEXT: [[TMP17:%.*]] = insertvalue { <2 x float>, <2 x float> } [[TMP13]], <2 x float> [[TMP16]], 1
+; VF2IC2-NEXT: [[TMP5:%.*]] = tail call { float, float } @fn2(float [[TMP45]]) #[[ATTR3]]
; VF2IC2-NEXT: [[TMP46:%.*]] = extractelement <2 x float> [[WIDE_LOAD1]], i64 0
; VF2IC2-NEXT: [[TMP18:%.*]] = tail call { float, float } @fn2(float [[TMP46]]) #[[ATTR3]]
; VF2IC2-NEXT: [[TMP47:%.*]] = extractelement <2 x float> [[WIDE_LOAD1]], i64 1
@@ -796,32 +757,9 @@ define void @struct_return_predicated_extractvalue(i1 %c, ptr noalias %a, ptr no
; VF2IC2-NEXT: store float 0.000000e+00, ptr [[P]], align 4
; VF2IC2-NEXT: br i1 [[C]], label %[[LATCH9]], label %[[IF2:.*]]
; VF2IC2: [[IF2]]:
-; VF2IC2-NEXT: [[TMP35:%.*]] = extractvalue { <2 x float>, <2 x float> } [[TMP17]], 1
; VF2IC2-NEXT: [[TMP36:%.*]] = extractvalue { <2 x float>, <2 x float> } [[TMP34]], 1
-; VF2IC2-NEXT: br i1 [[TMP0]], label %[[PRED_STORE_IF:.*]], label %[[PRED_STORE_CONTINUE:.*]]
-; VF2IC2: [[PRED_STORE_IF]]:
-; VF2IC2-NEXT: [[TMP37:%.*]] = extractelement <2 x float> [[TMP35]], i64 0
-; VF2IC2-NEXT: store float [[TMP37]], ptr [[Q]], align 4
-; VF2IC2-NEXT: br label %[[PRED_STORE_CONTINUE]]
-; VF2IC2: [[PRED_STORE_CONTINUE]]:
-; VF2IC2-NEXT: br i1 [[TMP0]], label %[[PRED_STORE_IF1:.*]], label %[[PRED_STORE_CONTINUE2:.*]]
-; VF2IC2: [[PRED_STORE_IF1]]:
-; VF2IC2-NEXT: [[TMP38:%.*]] = extractelement <2 x float> [[TMP35]], i64 1
-; VF2IC2-NEXT: store float [[TMP38]], ptr [[Q]], align 4
-; VF2IC2-NEXT: br label %[[PRED_STORE_CONTINUE2]]
-; VF2IC2: [[PRED_STORE_CONTINUE2]]:
-; VF2IC2-NEXT: br i1 [[TMP0]], label %[[PRED_STORE_IF3:.*]], label %[[PRED_STORE_CONTINUE4:.*]]
-; VF2IC2: [[PRED_STORE_IF3]]:
-; VF2IC2-NEXT: [[TMP39:%.*]] = extractelement <2 x float> [[TMP36]], i64 0
-; VF2IC2-NEXT: store float [[TMP39]], ptr [[Q]], align 4
-; VF2IC2-NEXT: br label %[[PRED_STORE_CONTINUE4]]
-; VF2IC2: [[PRED_STORE_CONTINUE4]]:
-; VF2IC2-NEXT: br i1 [[TMP0]], label %[[PRED_STORE_IF5:.*]], label %[[PRED_STORE_CONTINUE6:.*]]
-; VF2IC2: [[PRED_STORE_IF5]]:
; VF2IC2-NEXT: [[TMP40:%.*]] = extractelement <2 x float> [[TMP36]], i64 1
; VF2IC2-NEXT: store float [[TMP40]], ptr [[Q]], align 4
-; VF2IC2-NEXT: br label %[[PRED_STORE_CONTINUE6]]
-; VF2IC2: [[PRED_STORE_CONTINUE6]]:
; VF2IC2-NEXT: br label %[[LATCH9]]
; VF2IC2: [[LATCH9]]:
; VF2IC2-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 4
diff --git a/llvm/test/Transforms/LoopVectorize/uniform-blend.ll b/llvm/test/Transforms/LoopVectorize/uniform-blend.ll
index 59033daefbae8..21a81fed25571 100644
--- a/llvm/test/Transforms/LoopVectorize/uniform-blend.ll
+++ b/llvm/test/Transforms/LoopVectorize/uniform-blend.ll
@@ -102,8 +102,6 @@ define void @blend_chain_iv(i1 %c) {
; CHECK-NEXT: [[ENTRY:.*:]]
; CHECK-NEXT: br label %[[VECTOR_PH:.*]]
; CHECK: [[VECTOR_PH]]:
-; CHECK-NEXT: [[BROADCAST_SPLATINSERT:%.*]] = insertelement <4 x i1> poison, i1 [[C]], i64 0
-; CHECK-NEXT: [[BROADCAST_SPLAT:%.*]] = shufflevector <4 x i1> [[BROADCAST_SPLATINSERT]], <4 x i1> poison, <4 x i32> zeroinitializer
; CHECK-NEXT: br label %[[VECTOR_BODY:.*]]
; CHECK: [[VECTOR_BODY]]:
; CHECK-NEXT: [[INDEX1:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[LOOP_LATCH4:.*]] ]
@@ -117,7 +115,7 @@ define void @blend_chain_iv(i1 %c) {
; CHECK-NEXT: br label %[[LOOP_LATCH4]]
; CHECK: [[LOOP_LATCH4]]:
; CHECK-NEXT: [[TMP0:%.*]] = phi <4 x i64> [ poison, %[[VECTOR_BODY]] ], [ [[VEC_IND]], %[[LOOP_NEXT_33]] ]
-; CHECK-NEXT: [[TMP1:%.*]] = phi <4 x i1> [ zeroinitializer, %[[VECTOR_BODY]] ], [ [[BROADCAST_SPLAT]], %[[LOOP_NEXT_33]] ]
+; CHECK-NEXT: [[TMP1:%.*]] = phi <4 x i1> [ zeroinitializer, %[[VECTOR_BODY]] ], [ splat (i1 true), %[[LOOP_NEXT_33]] ]
; CHECK-NEXT: [[PREDPHI:%.*]] = select <4 x i1> [[TMP1]], <4 x i64> [[TMP0]], <4 x i64> [[VEC_IND]]
; CHECK-NEXT: [[INDEX:%.*]] = extractelement <4 x i64> [[PREDPHI]], i64 0
; CHECK-NEXT: [[TMP2:%.*]] = getelementptr inbounds [32 x i16], ptr @dst, i16 0, i64 [[INDEX]]
diff --git a/llvm/test/Transforms/PhaseOrdering/loop-vectorize-bfi.ll b/llvm/test/Transforms/PhaseOrdering/loop-vectorize-bfi.ll
index 754058e6945fc..283942dab69a7 100644
--- a/llvm/test/Transforms/PhaseOrdering/loop-vectorize-bfi.ll
+++ b/llvm/test/Transforms/PhaseOrdering/loop-vectorize-bfi.ll
@@ -11,9 +11,6 @@ define void @f(i1 %x) !prof !0 {
; CHECK-LABEL: define void @f(
; CHECK-SAME: i1 [[X:%.*]]) local_unnamed_addr #[[ATTR0:[0-9]+]] {{.*}}{
; CHECK-NEXT: [[ENTRY:.*]]:
-; CHECK-NEXT: [[DOTSCALAR:%.*]] = xor i1 [[X]], true
-; CHECK-NEXT: [[TMP1:%.*]] = insertelement <vscale x 2 x i1> poison, i1 [[DOTSCALAR]], i64 0
-; CHECK-NEXT: [[TMP2:%.*]] = shufflevector <vscale x 2 x i1> [[TMP1]], <vscale x 2 x i1> poison, <vscale x 2 x i32> zeroinitializer
; CHECK-NEXT: br label %[[VECTOR_BODY:.*]]
; CHECK: [[VECTOR_BODY]]:
; CHECK-NEXT: [[EVL_BASED_IV:%.*]] = phi i64 [ 0, %[[ENTRY]] ], [ [[INDEX_EVL_NEXT:%.*]], %[[BAZ_14:.*]] ]
@@ -22,8 +19,8 @@ define void @f(i1 %x) !prof !0 {
; CHECK-NEXT: br i1 [[X]], label %[[BAZ_14]], label %[[BAR1:.*]]
; CHECK: [[BAR1]]:
; CHECK-NEXT: [[TMP4:%.*]] = getelementptr [8 x i8], ptr null, i64 [[EVL_BASED_IV]]
-; CHECK-NEXT: call void @llvm.vp.store.nxv2i64.p0(<vscale x 2 x i64> poison, ptr align 8 [[TMP4]], <vscale x 2 x i1> [[TMP2]], i32 [[TMP3]])
-; CHECK-NEXT: call void @llvm.vp.store.nxv2i64.p0(<vscale x 2 x i64> poison, ptr align 8 [[TMP4]], <vscale x 2 x i1> [[TMP2]], i32 [[TMP3]])
+; CHECK-NEXT: call void @llvm.vp.store.nxv2i64.p0(<vscale x 2 x i64> poison, ptr align 8 [[TMP4]], <vscale x 2 x i1> splat (i1 true), i32 [[TMP3]])
+; CHECK-NEXT: call void @llvm.vp.store.nxv2i64.p0(<vscale x 2 x i64> poison, ptr align 8 [[TMP4]], <vscale x 2 x i1> splat (i1 true), i32 [[TMP3]])
; CHECK-NEXT: br label %[[BAZ_14]]
; CHECK: [[BAZ_14]]:
; CHECK-NEXT: [[TMP5:%.*]] = zext nneg i32 [[TMP3]] to i64
More information about the llvm-branch-commits
mailing list