[llvm-branch-commits] [llvm] [VPlan] Skip branch term in masks for some preserved uniform edges (PR #218552)
Andrei Elovikov via llvm-branch-commits
llvm-branch-commits at lists.llvm.org
Wed Sep 9 17:07:24 PDT 2026
https://github.com/eas updated https://github.com/llvm/llvm-project/pull/218552
>From 3e45797211c09be12b67d8800f9ab94a5d4da473 Mon Sep 17 00:00:00 2001
From: Andrei Elovikov <andrei.elovikov at sifive.com>
Date: Tue, 1 Sep 2026 15:36:45 -0700
Subject: [PATCH] [VPlan] Skip branch term in masks for some preserved uniform
edges
---
.../Transforms/Vectorize/LoopVectorize.cpp | 6 +-
.../Vectorize/VPlanDominatorTree.cpp | 4 +
.../Transforms/Vectorize/VPlanDominatorTree.h | 4 +
.../Transforms/Vectorize/VPlanPredicator.cpp | 66 ++-
.../Transforms/Vectorize/VPlanTransforms.cpp | 3 +-
.../LoopVectorize/AArch64/blend-costs.ll | 27 +-
.../AArch64/conditional-branches-cost.ll | 25 +-
.../AArch64/force-target-instruction-cost.ll | 390 ++++------------
.../AArch64/invariant-replicate-region.ll | 49 +-
.../AArch64/predication_costs.ll | 38 +-
.../AArch64/sve-predicated-costs.ll | 40 +-
...-interleave-to-widen-memory-multi-block.ll | 36 +-
.../LoopVectorize/RISCV/dead-ops-cost.ll | 59 +--
.../Transforms/LoopVectorize/RISCV/divrem.ll | 48 +-
.../RISCV/gather-scatter-cost.ll | 15 +-
.../LoopVectorize/RISCV/predicated-costs.ll | 28 +-
.../RISCV/predicated-reverse-store.ll | 7 +-
.../RISCV/tail-folding-complex-mask.ll | 16 +-
.../AArch64/vplan-memory-op-decisions.ll | 54 ++-
.../VPlan/dissolve-replicate-regions.ll | 118 ++---
.../LoopVectorize/VPlan/memory-checks.ll | 10 +-
.../VPlan/phi-with-fastflags-vplan.ll | 4 +-
.../LoopVectorize/VPlan/predicator.ll | 200 ++++----
.../VPlan/vplan-sink-scalars-and-merge-vf1.ll | 20 +-
.../VPlan/vplan-sink-scalars-and-merge.ll | 36 +-
.../X86/cost-conditional-branches.ll | 431 +-----------------
.../X86/invariant-load-gather.ll | 22 +-
.../LoopVectorize/X86/masked-store-cost.ll | 4 +-
.../X86/narrow-to-single-scalar.ll | 20 +-
.../X86/pr141968-instsimplifyfolder.ll | 5 +-
.../X86/predicated-instruction-cost.ll | 43 +-
.../X86/predicated-replicate-feeding-cast.ll | 99 +---
...licate-recipe-with-only-first-lane-used.ll | 98 +---
.../X86/replicate-uniform-call.ll | 5 +-
.../LoopVectorize/X86/scatter_crash.ll | 34 +-
.../constant-fold-commutative-and.ll | 8 +-
.../Transforms/LoopVectorize/div-exact.ll | 132 +-----
...predicated-loads-with-predicated-stores.ll | 41 +-
.../LoopVectorize/hoist-predicated-loads.ll | 94 +---
.../LoopVectorize/if-pred-stores.ll | 165 ++-----
.../invariant-store-vectorization-2.ll | 10 +-
.../multiple-result-intrinsics.ll | 10 -
.../LoopVectorize/phi-with-fastflags.ll | 4 +-
llvm/test/Transforms/LoopVectorize/pr37248.ll | 10 -
.../LoopVectorize/pr44488-predication.ll | 24 +-
llvm/test/Transforms/LoopVectorize/pr45525.ll | 5 +-
.../Transforms/LoopVectorize/predicator.ll | 37 +-
.../LoopVectorize/scalarize-masked-call.ll | 22 +-
.../LoopVectorize/struct-return-replicate.ll | 66 +--
.../Transforms/LoopVectorize/uniform-blend.ll | 7 +-
.../PhaseOrdering/loop-vectorize-bfi.ll | 7 +-
51 files changed, 665 insertions(+), 2041 deletions(-)
diff --git a/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp b/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
index a2ee826c63f2d..e1defa4474558 100644
--- a/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
+++ b/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
@@ -6066,8 +6066,10 @@ VPRecipeWithIRFlags *VPRecipeBuilder::tryToWiden(VPInstruction *VPI) {
case Instruction::UDiv:
case Instruction::SRem:
case Instruction::URem:
- // If not provably safe, use a masked intrinsic.
- if (CM.isPredicatedInst(I))
+ // If not provably safe, use a masked intrinsic. Predicator might have
+ // preserved uniform control flow making this unmasked even if CM expected
+ // it to be masked.
+ if (CM.isPredicatedInst(I) && VPI->isMasked())
return new VPWidenIntrinsicRecipe(
getMaskedDivRemIntrinsic(VPI->getOpcode()), VPI->operands(),
I->getType(), {}, {}, VPI->getDebugLoc());
diff --git a/llvm/lib/Transforms/Vectorize/VPlanDominatorTree.cpp b/llvm/lib/Transforms/Vectorize/VPlanDominatorTree.cpp
index 2b32a3011aae3..95a5fda325a40 100644
--- a/llvm/lib/Transforms/Vectorize/VPlanDominatorTree.cpp
+++ b/llvm/lib/Transforms/Vectorize/VPlanDominatorTree.cpp
@@ -11,6 +11,10 @@
using namespace llvm;
+VPDominanceFrontier::VPDominanceFrontier(const DomTreeT &VPDT) {
+ analyze(VPDT);
+}
+
VPPostDominanceFrontier::VPPostDominanceFrontier(const DomTreeT &VPDT) {
analyze(VPDT);
}
diff --git a/llvm/lib/Transforms/Vectorize/VPlanDominatorTree.h b/llvm/lib/Transforms/Vectorize/VPlanDominatorTree.h
index e1e368eb72dfc..db87a075f1ff5 100644
--- a/llvm/lib/Transforms/Vectorize/VPlanDominatorTree.h
+++ b/llvm/lib/Transforms/Vectorize/VPlanDominatorTree.h
@@ -69,6 +69,10 @@ struct GraphTraits<const VPDomTreeNode *>
: public DomTreeGraphTraitsBase<const VPDomTreeNode,
VPDomTreeNode::const_iterator> {};
+struct VPDominanceFrontier : public DominanceFrontierBase<VPBlockBase, false> {
+ explicit VPDominanceFrontier(const DomTreeT &VPDT);
+};
+
struct VPPostDominanceFrontier
: public DominanceFrontierBase<VPBlockBase, true> {
explicit VPPostDominanceFrontier(const DomTreeT &VPDT);
diff --git a/llvm/lib/Transforms/Vectorize/VPlanPredicator.cpp b/llvm/lib/Transforms/Vectorize/VPlanPredicator.cpp
index c474f2a35834f..d535c73adbcb4 100644
--- a/llvm/lib/Transforms/Vectorize/VPlanPredicator.cpp
+++ b/llvm/lib/Transforms/Vectorize/VPlanPredicator.cpp
@@ -84,6 +84,9 @@ class VPPredicator {
/// Post-dominator tree for the VPlan.
VPPostDominatorTree VPPDT;
+ /// Dominance frontier for the VPlan.
+ VPDominanceFrontier VPDF;
+
DenseMap<VPValue *, DenseMap<VPBasicBlock *, VPValue *>>
SSAReconstructionDefsMap;
@@ -169,7 +172,7 @@ class VPPredicator {
public:
VPPredicator(VPlan &Plan)
- : Plan(Plan), VPDT(Plan), VPPDT(Plan),
+ : Plan(Plan), VPDT(Plan), VPPDT(Plan), VPDF(VPDT),
BlocksInCompactRPOTOrder(
Plan.getVectorLoopRegion()->getEntryBasicBlock(), VPDT),
BlendTerms(BlocksInCompactRPOTOrder.size()) {
@@ -371,9 +374,64 @@ void VPPredicator::createBlockInMask(VPBasicBlock *VPBB) {
// This is the block mask. We OR all unique incoming edges.
for (auto *Predecessor : SetVector<VPBlockBase *>(
VPBB->getPredecessors().begin(), VPBB->getPredecessors().end())) {
- VPValue *EdgeMask = createEdgeMask(cast<VPBasicBlock>(Predecessor), VPBB);
- if (!EdgeMask) { // Mask of predecessor is all-one so mask of block is
- // too.
+ auto *Pred = cast<VPBasicBlock>(Predecessor);
+ bool CanUseBlockMaskOnly = [&] {
+ if (!shouldPreserveTerminator(Pred))
+ return false;
+
+ LLVM_DEBUG(
+ dbgs()
+ << "Checking if can skip branch condition for a uniform branch at "
+ << Pred->getName() << "\n");
+
+ auto False = []([[maybe_unused]] StringRef Reason = "") {
+ LLVM_DEBUG(dbgs() << " can't skip"
+ << (Reason.empty() ? Twine() : (": " + Reason))
+ << "\n");
+ return false;
+ };
+
+ // Detect patterns like
+ //
+ // BB (uniform branch)
+ // / \
+ // +-------------------------+ +-------------------------+
+ // | no other incoming edges | | no other incoming edges |
+ // +-------------------------+ +-------------------------+
+ // \ / /
+ // \ / /
+ // PostDomBB <-- possible other edges originated
+ // before BB.
+ //
+ // where each box is either a single entry region with all exiting edges
+ // pointing to PostDomBB or just a direct edge BB->PostDomBB.
+ auto *IPostDomNode = VPPDT.getNode(Pred)->getIDom();
+ if (!IPostDomNode)
+ return False("No post-dom");
+ auto *IPostDom = cast<VPBasicBlock>(IPostDomNode->getBlock());
+ LLVM_DEBUG(dbgs() << " post-dom: " << IPostDom->getName() << "\n");
+
+ auto HasOnlyIPostDomInDF = [&](VPBlockBase *BB) {
+ auto It = VPDF.find(BB);
+ return It != VPDF.end() && It->second.size() == 1 &&
+ It->second.front() == IPostDom;
+ };
+ if (!VPDT.properlyDominates(Pred, IPostDom) && !HasOnlyIPostDomInDF(Pred))
+ return False("doesn't dominate its post-dom or have it as its only "
+ "dominance-frontier block");
+
+ if (!all_of(Pred->successors(), [&](VPBlockBase *Succ) {
+ return Succ == IPostDom || HasOnlyIPostDomInDF(Succ);
+ }))
+ return False("failed successor checks");
+
+ LLVM_DEBUG(dbgs() << " can skip\n");
+ return true;
+ }();
+ VPValue *EdgeMask =
+ CanUseBlockMaskOnly ? getBlockInMask(Pred) : createEdgeMask(Pred, VPBB);
+ if (!EdgeMask) {
+ // Mask of predecessor is all-one so mask of block is too.
setBlockInMask(VPBB, EdgeMask);
return;
}
diff --git a/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp b/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp
index 47dfa15687957..7e14f67695cf6 100644
--- a/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp
+++ b/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp
@@ -5566,7 +5566,8 @@ void VPlanTransforms::makeMemOpWideningDecisions(VPlan &Plan, VFRange &Range,
// A predicated access can only be widened (rather than scalarized) if
// the target supports a masked load/store for it.
// TODO: Determine if a load/store needs predication directly in VPlan.
- bool IsPredicated = RecipeBuilder.isPredicatedInst(I);
+ bool IsPredicated =
+ RecipeBuilder.isPredicatedInst(I) && VPI->isMasked();
if (IsPredicated && !CostCtx.Config.isLegalMaskedLoadOrStore(
IsLoad, ScalarTy, getLoadStoreAlignment(I),
getLoadStoreAddressSpace(I)))
diff --git a/llvm/test/Transforms/LoopVectorize/AArch64/blend-costs.ll b/llvm/test/Transforms/LoopVectorize/AArch64/blend-costs.ll
index 1a6d4e5ba7b0b..60023a8d4f8b9 100644
--- a/llvm/test/Transforms/LoopVectorize/AArch64/blend-costs.ll
+++ b/llvm/test/Transforms/LoopVectorize/AArch64/blend-costs.ll
@@ -490,8 +490,6 @@ define void @only_first_lane_used(i1 %c, ptr noalias %p1, ptr noalias %p2, ptr n
; CHECK-NEXT: [[ENTRY:.*:]]
; CHECK-NEXT: br label %[[VECTOR_PH:.*]]
; CHECK: [[VECTOR_PH]]:
-; CHECK-NEXT: [[BROADCAST_SPLATINSERT:%.*]] = insertelement <4 x i1> poison, i1 [[C]], i64 0
-; CHECK-NEXT: [[BROADCAST_SPLAT:%.*]] = shufflevector <4 x i1> [[BROADCAST_SPLATINSERT]], <4 x i1> poison, <4 x i32> zeroinitializer
; CHECK-NEXT: br label %[[VECTOR_BODY:.*]]
; CHECK: [[VECTOR_BODY]]:
; CHECK-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[LATCH9:.*]] ]
@@ -505,31 +503,12 @@ define void @only_first_lane_used(i1 %c, ptr noalias %p1, ptr noalias %p2, ptr n
; CHECK-NEXT: [[TMP8:%.*]] = add i64 [[INDEX]], -1
; CHECK-NEXT: [[BROADCAST_SPLATINSERT12:%.*]] = insertelement <4 x i64> poison, i64 [[TMP8]], i64 0
; CHECK-NEXT: [[BROADCAST_SPLAT13:%.*]] = shufflevector <4 x i64> [[BROADCAST_SPLATINSERT12]], <4 x i64> poison, <4 x i32> zeroinitializer
-; CHECK-NEXT: br i1 [[C]], label %[[PRED_STORE_IF:.*]], label %[[PRED_STORE_CONTINUE:.*]]
-; CHECK: [[PRED_STORE_IF]]:
-; CHECK-NEXT: store i32 0, ptr [[Q]], align 4
-; CHECK-NEXT: br label %[[PRED_STORE_CONTINUE]]
-; CHECK: [[PRED_STORE_CONTINUE]]:
-; CHECK-NEXT: br i1 [[C]], label %[[PRED_STORE_IF1:.*]], label %[[PRED_STORE_CONTINUE2:.*]]
-; CHECK: [[PRED_STORE_IF1]]:
; CHECK-NEXT: store i32 0, ptr [[Q]], align 4
-; CHECK-NEXT: br label %[[PRED_STORE_CONTINUE2]]
-; CHECK: [[PRED_STORE_CONTINUE2]]:
-; CHECK-NEXT: br i1 [[C]], label %[[PRED_STORE_IF3:.*]], label %[[PRED_STORE_CONTINUE4:.*]]
-; CHECK: [[PRED_STORE_IF3]]:
-; CHECK-NEXT: store i32 0, ptr [[Q]], align 4
-; CHECK-NEXT: br label %[[PRED_STORE_CONTINUE4]]
-; CHECK: [[PRED_STORE_CONTINUE4]]:
-; CHECK-NEXT: br i1 [[C]], label %[[PRED_STORE_IF5:.*]], label %[[PRED_STORE_CONTINUE6:.*]]
-; CHECK: [[PRED_STORE_IF5]]:
-; CHECK-NEXT: store i32 0, ptr [[Q]], align 4
-; CHECK-NEXT: br label %[[PRED_STORE_CONTINUE6]]
-; CHECK: [[PRED_STORE_CONTINUE6]]:
; CHECK-NEXT: br label %[[LATCH9]]
; CHECK: [[LATCH9]]:
-; CHECK-NEXT: [[TMP9:%.*]] = phi <4 x i64> [ [[BROADCAST_SPLAT11]], %[[FOO1]] ], [ poison, %[[PRED_STORE_CONTINUE6]] ]
-; CHECK-NEXT: [[TMP10:%.*]] = phi <4 x i64> [ poison, %[[FOO1]] ], [ [[BROADCAST_SPLAT13]], %[[PRED_STORE_CONTINUE6]] ]
-; CHECK-NEXT: [[TMP11:%.*]] = phi <4 x i1> [ zeroinitializer, %[[FOO1]] ], [ [[BROADCAST_SPLAT]], %[[PRED_STORE_CONTINUE6]] ]
+; CHECK-NEXT: [[TMP9:%.*]] = phi <4 x i64> [ [[BROADCAST_SPLAT11]], %[[FOO1]] ], [ poison, %[[BAR2]] ]
+; CHECK-NEXT: [[TMP10:%.*]] = phi <4 x i64> [ poison, %[[FOO1]] ], [ [[BROADCAST_SPLAT13]], %[[BAR2]] ]
+; CHECK-NEXT: [[TMP11:%.*]] = phi <4 x i1> [ zeroinitializer, %[[FOO1]] ], [ splat (i1 true), %[[BAR2]] ]
; CHECK-NEXT: [[TMP12:%.*]] = extractelement <4 x i1> [[TMP11]], i64 0
; CHECK-NEXT: [[TMP13:%.*]] = extractelement <4 x i64> [[TMP10]], i64 0
; CHECK-NEXT: [[TMP7:%.*]] = extractelement <4 x i64> [[TMP9]], i64 0
diff --git a/llvm/test/Transforms/LoopVectorize/AArch64/conditional-branches-cost.ll b/llvm/test/Transforms/LoopVectorize/AArch64/conditional-branches-cost.ll
index 69a3e7c519ac0..e980af90cd37f 100644
--- a/llvm/test/Transforms/LoopVectorize/AArch64/conditional-branches-cost.ll
+++ b/llvm/test/Transforms/LoopVectorize/AArch64/conditional-branches-cost.ll
@@ -384,7 +384,7 @@ define i32 @header_mask_and_invariant_compare(ptr %A, ptr %B, ptr %C, ptr %D, pt
; DEFAULT-SAME: ptr [[A:%.*]], ptr [[B:%.*]], ptr [[C:%.*]], ptr [[D:%.*]], ptr [[E:%.*]], i64 [[N:%.*]]) #[[ATTR0:[0-9]+]] {
; DEFAULT-NEXT: [[ENTRY:.*:]]
; DEFAULT-NEXT: [[TMP0:%.*]] = add i64 [[N]], 1
-; DEFAULT-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[TMP0]], 60
+; DEFAULT-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[TMP0]], 28
; DEFAULT-NEXT: br i1 [[MIN_ITERS_CHECK]], label %[[SCALAR_PH:.*]], label %[[VECTOR_MEMCHECK:.*]]
; DEFAULT: [[VECTOR_MEMCHECK]]:
; DEFAULT-NEXT: [[SCEVGEP:%.*]] = getelementptr i8, ptr [[E]], i64 4
@@ -433,32 +433,11 @@ define i32 @header_mask_and_invariant_compare(ptr %A, ptr %B, ptr %C, ptr %D, pt
; DEFAULT-NEXT: [[TMP5:%.*]] = or i32 [[TMP4]], [[TMP3]]
; DEFAULT-NEXT: [[TMP6:%.*]] = load i32, ptr [[C]], align 4, !alias.scope [[META13:![0-9]+]]
; DEFAULT-NEXT: [[TMP7:%.*]] = icmp ugt i32 [[TMP6]], [[TMP5]]
-; DEFAULT-NEXT: [[BROADCAST_SPLATINSERT:%.*]] = insertelement <4 x i1> poison, i1 [[TMP7]], i64 0
-; DEFAULT-NEXT: [[BROADCAST_SPLAT:%.*]] = shufflevector <4 x i1> [[BROADCAST_SPLATINSERT]], <4 x i1> poison, <4 x i32> zeroinitializer
; DEFAULT-NEXT: br i1 [[TMP7]], label %[[VECTOR_BODY:.*]], label %[[LOOP_LATCH35]]
; DEFAULT: [[VECTOR_BODY]]:
; DEFAULT-NEXT: [[TMP16:%.*]] = getelementptr i32, ptr [[D]], i64 [[INDEX]]
-; DEFAULT-NEXT: br i1 [[TMP7]], label %[[PRED_STORE_IF:.*]], label %[[PRED_STORE_CONTINUE:.*]]
-; DEFAULT: [[PRED_STORE_IF]]:
; DEFAULT-NEXT: store i32 [[TMP5]], ptr [[E]], align 4, !alias.scope [[META15:![0-9]+]], !noalias [[META17:![0-9]+]]
-; DEFAULT-NEXT: br label %[[PRED_STORE_CONTINUE]]
-; DEFAULT: [[PRED_STORE_CONTINUE]]:
-; DEFAULT-NEXT: br i1 [[TMP7]], label %[[PRED_STORE_IF28:.*]], label %[[PRED_STORE_CONTINUE29:.*]]
-; DEFAULT: [[PRED_STORE_IF28]]:
-; DEFAULT-NEXT: store i32 [[TMP5]], ptr [[E]], align 4, !alias.scope [[META15]], !noalias [[META17]]
-; DEFAULT-NEXT: br label %[[PRED_STORE_CONTINUE29]]
-; DEFAULT: [[PRED_STORE_CONTINUE29]]:
-; DEFAULT-NEXT: br i1 [[TMP7]], label %[[PRED_STORE_IF30:.*]], label %[[PRED_STORE_CONTINUE31:.*]]
-; DEFAULT: [[PRED_STORE_IF30]]:
-; DEFAULT-NEXT: store i32 [[TMP5]], ptr [[E]], align 4, !alias.scope [[META15]], !noalias [[META17]]
-; DEFAULT-NEXT: br label %[[PRED_STORE_CONTINUE31]]
-; DEFAULT: [[PRED_STORE_CONTINUE31]]:
-; DEFAULT-NEXT: br i1 [[TMP7]], label %[[PRED_STORE_IF32:.*]], label %[[PRED_STORE_CONTINUE33:.*]]
-; DEFAULT: [[PRED_STORE_IF32]]:
-; DEFAULT-NEXT: store i32 [[TMP5]], ptr [[E]], align 4, !alias.scope [[META15]], !noalias [[META17]]
-; DEFAULT-NEXT: br label %[[PRED_STORE_CONTINUE33]]
-; DEFAULT: [[PRED_STORE_CONTINUE33]]:
-; DEFAULT-NEXT: call void @llvm.masked.store.v4i32.p0(<4 x i32> zeroinitializer, ptr align 4 [[TMP16]], <4 x i1> [[BROADCAST_SPLAT]]), !alias.scope [[META19:![0-9]+]], !noalias [[META20:![0-9]+]]
+; DEFAULT-NEXT: store <4 x i32> zeroinitializer, ptr [[TMP16]], align 4, !alias.scope [[META19:![0-9]+]], !noalias [[META20:![0-9]+]]
; DEFAULT-NEXT: br label %[[LOOP_LATCH35]]
; DEFAULT: [[LOOP_LATCH35]]:
; DEFAULT-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 4
diff --git a/llvm/test/Transforms/LoopVectorize/AArch64/force-target-instruction-cost.ll b/llvm/test/Transforms/LoopVectorize/AArch64/force-target-instruction-cost.ll
index 8479b547b162b..073e97200d7cc 100644
--- a/llvm/test/Transforms/LoopVectorize/AArch64/force-target-instruction-cost.ll
+++ b/llvm/test/Transforms/LoopVectorize/AArch64/force-target-instruction-cost.ll
@@ -387,239 +387,29 @@ for.end:
}
define void @loop_with_freeze_and_conditional_srem(ptr %dst, ptr %keyinfo, ptr %invariant.ptr, i32 %divisor) {
-; COST1-LABEL: define void @loop_with_freeze_and_conditional_srem(
-; COST1-SAME: ptr [[DST:%.*]], ptr [[KEYINFO:%.*]], ptr [[INVARIANT_PTR:%.*]], i32 [[DIVISOR:%.*]]) {
-; COST1-NEXT: [[ENTRY:.*:]]
-; COST1-NEXT: br label %[[VECTOR_MEMCHECK:.*]]
-; COST1: [[VECTOR_MEMCHECK]]:
-; COST1-NEXT: [[SCEVGEP:%.*]] = getelementptr i8, ptr [[DST]], i64 4
-; COST1-NEXT: [[SCEVGEP1:%.*]] = getelementptr i8, ptr [[KEYINFO]], i64 4
-; COST1-NEXT: [[SCEVGEP2:%.*]] = getelementptr i8, ptr [[INVARIANT_PTR]], i64 4
-; COST1-NEXT: [[BOUND0:%.*]] = icmp ult ptr [[DST]], [[SCEVGEP1]]
-; COST1-NEXT: [[BOUND1:%.*]] = icmp ult ptr [[KEYINFO]], [[SCEVGEP]]
-; COST1-NEXT: [[FOUND_CONFLICT:%.*]] = and i1 [[BOUND0]], [[BOUND1]]
-; COST1-NEXT: [[BOUND03:%.*]] = icmp ult ptr [[DST]], [[SCEVGEP2]]
-; COST1-NEXT: [[BOUND14:%.*]] = icmp ult ptr [[INVARIANT_PTR]], [[SCEVGEP]]
-; COST1-NEXT: [[FOUND_CONFLICT5:%.*]] = and i1 [[BOUND03]], [[BOUND14]]
-; COST1-NEXT: [[CONFLICT_RDX:%.*]] = or i1 [[FOUND_CONFLICT]], [[FOUND_CONFLICT5]]
-; COST1-NEXT: [[BOUND06:%.*]] = icmp ult ptr [[KEYINFO]], [[SCEVGEP2]]
-; COST1-NEXT: [[BOUND17:%.*]] = icmp ult ptr [[INVARIANT_PTR]], [[SCEVGEP1]]
-; COST1-NEXT: [[FOUND_CONFLICT8:%.*]] = and i1 [[BOUND06]], [[BOUND17]]
-; COST1-NEXT: [[CONFLICT_RDX9:%.*]] = or i1 [[CONFLICT_RDX]], [[FOUND_CONFLICT8]]
-; COST1-NEXT: br i1 [[CONFLICT_RDX9]], label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]]
-; COST1: [[VECTOR_PH]]:
-; COST1-NEXT: br label %[[VECTOR_BODY:.*]]
-; COST1: [[VECTOR_BODY]]:
-; COST1-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[LOOP_LATCH42:.*]] ]
-; COST1-NEXT: [[TMP0:%.*]] = load i32, ptr [[INVARIANT_PTR]], align 4, !alias.scope [[META15:![0-9]+]]
-; COST1-NEXT: [[TMP1:%.*]] = freeze i32 [[TMP0]]
-; COST1-NEXT: [[BROADCAST_SPLATINSERT:%.*]] = insertelement <4 x i32> poison, i32 [[TMP1]], i64 0
-; COST1-NEXT: [[BROADCAST_SPLAT:%.*]] = shufflevector <4 x i32> [[BROADCAST_SPLATINSERT]], <4 x i32> poison, <4 x i32> zeroinitializer
-; COST1-NEXT: [[TMP2:%.*]] = icmp eq <4 x i32> [[BROADCAST_SPLAT]], zeroinitializer
-; COST1-NEXT: [[TMP3:%.*]] = extractelement <4 x i1> [[TMP2]], i64 0
-; COST1-NEXT: br i1 [[TMP3]], label %[[IF_ZERO25:.*]], label %[[IF_NONZERO10:.*]]
-; COST1: [[IF_NONZERO10]]:
-; COST1-NEXT: [[TMP4:%.*]] = extractelement <4 x i1> [[TMP2]], i64 0
-; COST1-NEXT: [[TMP5:%.*]] = xor i1 [[TMP4]], true
-; COST1-NEXT: br i1 [[TMP5]], label %[[PRED_STORE_IF:.*]], label %[[PRED_STORE_CONTINUE:.*]]
-; COST1: [[PRED_STORE_IF]]:
-; COST1-NEXT: [[TMP6:%.*]] = srem i32 1, [[DIVISOR]]
-; COST1-NEXT: store i32 [[TMP6]], ptr [[DST]], align 4, !alias.scope [[META18:![0-9]+]], !noalias [[META20:![0-9]+]]
-; COST1-NEXT: br label %[[PRED_STORE_CONTINUE]]
-; COST1: [[PRED_STORE_CONTINUE]]:
-; COST1-NEXT: br i1 [[TMP5]], label %[[PRED_STORE_IF11:.*]], label %[[PRED_STORE_CONTINUE12:.*]]
-; COST1: [[PRED_STORE_IF11]]:
-; COST1-NEXT: [[TMP7:%.*]] = srem i32 1, [[DIVISOR]]
-; COST1-NEXT: store i32 [[TMP7]], ptr [[DST]], align 4, !alias.scope [[META18]], !noalias [[META20]]
-; COST1-NEXT: br label %[[PRED_STORE_CONTINUE12]]
-; COST1: [[PRED_STORE_CONTINUE12]]:
-; COST1-NEXT: br i1 [[TMP5]], label %[[PRED_STORE_IF13:.*]], label %[[PRED_STORE_CONTINUE14:.*]]
-; COST1: [[PRED_STORE_IF13]]:
-; COST1-NEXT: [[TMP8:%.*]] = srem i32 1, [[DIVISOR]]
-; COST1-NEXT: store i32 [[TMP8]], ptr [[DST]], align 4, !alias.scope [[META18]], !noalias [[META20]]
-; COST1-NEXT: br label %[[PRED_STORE_CONTINUE14]]
-; COST1: [[PRED_STORE_CONTINUE14]]:
-; COST1-NEXT: br i1 [[TMP5]], label %[[PRED_STORE_IF15:.*]], label %[[PRED_STORE_CONTINUE16:.*]]
-; COST1: [[PRED_STORE_IF15]]:
-; COST1-NEXT: [[TMP9:%.*]] = srem i32 1, [[DIVISOR]]
-; COST1-NEXT: store i32 [[TMP9]], ptr [[DST]], align 4, !alias.scope [[META18]], !noalias [[META20]]
-; COST1-NEXT: br label %[[PRED_STORE_CONTINUE16]]
-; COST1: [[PRED_STORE_CONTINUE16]]:
-; COST1-NEXT: br i1 [[TMP5]], label %[[PRED_STORE_IF17:.*]], label %[[PRED_STORE_CONTINUE18:.*]]
-; COST1: [[PRED_STORE_IF17]]:
-; COST1-NEXT: [[TMP10:%.*]] = srem i32 1, [[DIVISOR]]
-; COST1-NEXT: store i32 [[TMP10]], ptr [[DST]], align 4, !alias.scope [[META18]], !noalias [[META20]]
-; COST1-NEXT: br label %[[PRED_STORE_CONTINUE18]]
-; COST1: [[PRED_STORE_CONTINUE18]]:
-; COST1-NEXT: br i1 [[TMP5]], label %[[PRED_STORE_IF19:.*]], label %[[PRED_STORE_CONTINUE20:.*]]
-; COST1: [[PRED_STORE_IF19]]:
-; COST1-NEXT: [[TMP11:%.*]] = srem i32 1, [[DIVISOR]]
-; COST1-NEXT: store i32 [[TMP11]], ptr [[DST]], align 4, !alias.scope [[META18]], !noalias [[META20]]
-; COST1-NEXT: br label %[[PRED_STORE_CONTINUE20]]
-; COST1: [[PRED_STORE_CONTINUE20]]:
-; COST1-NEXT: br i1 [[TMP5]], label %[[PRED_STORE_IF21:.*]], label %[[PRED_STORE_CONTINUE22:.*]]
-; COST1: [[PRED_STORE_IF21]]:
-; COST1-NEXT: [[TMP12:%.*]] = srem i32 1, [[DIVISOR]]
-; COST1-NEXT: store i32 [[TMP12]], ptr [[DST]], align 4, !alias.scope [[META18]], !noalias [[META20]]
-; COST1-NEXT: br label %[[PRED_STORE_CONTINUE22]]
-; COST1: [[PRED_STORE_CONTINUE22]]:
-; COST1-NEXT: br i1 [[TMP5]], label %[[PRED_STORE_IF23:.*]], label %[[PRED_STORE_CONTINUE24:.*]]
-; COST1: [[PRED_STORE_IF23]]:
-; COST1-NEXT: [[TMP13:%.*]] = srem i32 1, [[DIVISOR]]
-; COST1-NEXT: store i32 [[TMP13]], ptr [[DST]], align 4, !alias.scope [[META18]], !noalias [[META20]]
-; COST1-NEXT: br label %[[PRED_STORE_CONTINUE24]]
-; COST1: [[PRED_STORE_CONTINUE24]]:
-; COST1-NEXT: br label %[[LOOP_LATCH42]]
-; COST1: [[IF_ZERO25]]:
-; COST1-NEXT: [[TMP14:%.*]] = extractelement <4 x i1> [[TMP2]], i64 0
-; COST1-NEXT: br i1 [[TMP14]], label %[[PRED_STORE_IF26:.*]], label %[[PRED_STORE_CONTINUE27:.*]]
-; COST1: [[PRED_STORE_IF26]]:
-; COST1-NEXT: store i32 0, ptr [[KEYINFO]], align 4, !alias.scope [[META22:![0-9]+]], !noalias [[META15]]
-; COST1-NEXT: br label %[[PRED_STORE_CONTINUE27]]
-; COST1: [[PRED_STORE_CONTINUE27]]:
-; COST1-NEXT: [[TMP15:%.*]] = extractelement <4 x i1> [[TMP2]], i64 0
-; COST1-NEXT: br i1 [[TMP15]], label %[[PRED_STORE_IF28:.*]], label %[[PRED_STORE_CONTINUE29:.*]]
-; COST1: [[PRED_STORE_IF28]]:
-; COST1-NEXT: store i32 0, ptr [[KEYINFO]], align 4, !alias.scope [[META22]], !noalias [[META15]]
-; COST1-NEXT: br label %[[PRED_STORE_CONTINUE29]]
-; COST1: [[PRED_STORE_CONTINUE29]]:
-; COST1-NEXT: [[TMP16:%.*]] = extractelement <4 x i1> [[TMP2]], i64 0
-; COST1-NEXT: br i1 [[TMP16]], label %[[PRED_STORE_IF30:.*]], label %[[PRED_STORE_CONTINUE31:.*]]
-; COST1: [[PRED_STORE_IF30]]:
-; COST1-NEXT: store i32 0, ptr [[KEYINFO]], align 4, !alias.scope [[META22]], !noalias [[META15]]
-; COST1-NEXT: br label %[[PRED_STORE_CONTINUE31]]
-; COST1: [[PRED_STORE_CONTINUE31]]:
-; COST1-NEXT: [[TMP17:%.*]] = extractelement <4 x i1> [[TMP2]], i64 0
-; COST1-NEXT: br i1 [[TMP17]], label %[[PRED_STORE_IF32:.*]], label %[[PRED_STORE_CONTINUE33:.*]]
-; COST1: [[PRED_STORE_IF32]]:
-; COST1-NEXT: store i32 0, ptr [[KEYINFO]], align 4, !alias.scope [[META22]], !noalias [[META15]]
-; COST1-NEXT: br label %[[PRED_STORE_CONTINUE33]]
-; COST1: [[PRED_STORE_CONTINUE33]]:
-; COST1-NEXT: [[TMP18:%.*]] = extractelement <4 x i1> [[TMP2]], i64 0
-; COST1-NEXT: br i1 [[TMP18]], label %[[PRED_STORE_IF34:.*]], label %[[PRED_STORE_CONTINUE35:.*]]
-; COST1: [[PRED_STORE_IF34]]:
-; COST1-NEXT: store i32 0, ptr [[KEYINFO]], align 4, !alias.scope [[META22]], !noalias [[META15]]
-; COST1-NEXT: br label %[[PRED_STORE_CONTINUE35]]
-; COST1: [[PRED_STORE_CONTINUE35]]:
-; COST1-NEXT: [[TMP19:%.*]] = extractelement <4 x i1> [[TMP2]], i64 0
-; COST1-NEXT: br i1 [[TMP19]], label %[[PRED_STORE_IF36:.*]], label %[[PRED_STORE_CONTINUE37:.*]]
-; COST1: [[PRED_STORE_IF36]]:
-; COST1-NEXT: store i32 0, ptr [[KEYINFO]], align 4, !alias.scope [[META22]], !noalias [[META15]]
-; COST1-NEXT: br label %[[PRED_STORE_CONTINUE37]]
-; COST1: [[PRED_STORE_CONTINUE37]]:
-; COST1-NEXT: [[TMP20:%.*]] = extractelement <4 x i1> [[TMP2]], i64 0
-; COST1-NEXT: br i1 [[TMP20]], label %[[PRED_STORE_IF38:.*]], label %[[PRED_STORE_CONTINUE39:.*]]
-; COST1: [[PRED_STORE_IF38]]:
-; COST1-NEXT: store i32 0, ptr [[KEYINFO]], align 4, !alias.scope [[META22]], !noalias [[META15]]
-; COST1-NEXT: br label %[[PRED_STORE_CONTINUE39]]
-; COST1: [[PRED_STORE_CONTINUE39]]:
-; COST1-NEXT: [[TMP21:%.*]] = extractelement <4 x i1> [[TMP2]], i64 0
-; COST1-NEXT: br i1 [[TMP21]], label %[[PRED_STORE_IF40:.*]], label %[[PRED_STORE_CONTINUE41:.*]]
-; COST1: [[PRED_STORE_IF40]]:
-; COST1-NEXT: store i32 0, ptr [[KEYINFO]], align 4, !alias.scope [[META22]], !noalias [[META15]]
-; COST1-NEXT: br label %[[PRED_STORE_CONTINUE41]]
-; COST1: [[PRED_STORE_CONTINUE41]]:
-; COST1-NEXT: br label %[[LOOP_LATCH42]]
-; COST1: [[LOOP_LATCH42]]:
-; COST1-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 8
-; COST1-NEXT: [[TMP22:%.*]] = icmp eq i64 [[INDEX_NEXT]], 32
-; COST1-NEXT: br i1 [[TMP22]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP23:![0-9]+]]
-; COST1: [[MIDDLE_BLOCK]]:
-; COST1-NEXT: br label %[[SCALAR_PH]]
-; COST1: [[SCALAR_PH]]:
-;
-; COST10-LABEL: define void @loop_with_freeze_and_conditional_srem(
-; COST10-SAME: ptr [[DST:%.*]], ptr [[KEYINFO:%.*]], ptr [[INVARIANT_PTR:%.*]], i32 [[DIVISOR:%.*]]) {
-; COST10-NEXT: [[ENTRY:.*:]]
-; COST10-NEXT: br label %[[VECTOR_MEMCHECK:.*]]
-; COST10: [[VECTOR_MEMCHECK]]:
-; COST10-NEXT: [[SCEVGEP:%.*]] = getelementptr i8, ptr [[DST]], i64 4
-; COST10-NEXT: [[SCEVGEP1:%.*]] = getelementptr i8, ptr [[KEYINFO]], i64 4
-; COST10-NEXT: [[SCEVGEP2:%.*]] = getelementptr i8, ptr [[INVARIANT_PTR]], i64 4
-; COST10-NEXT: [[BOUND0:%.*]] = icmp ult ptr [[DST]], [[SCEVGEP1]]
-; COST10-NEXT: [[BOUND1:%.*]] = icmp ult ptr [[KEYINFO]], [[SCEVGEP]]
-; COST10-NEXT: [[FOUND_CONFLICT:%.*]] = and i1 [[BOUND0]], [[BOUND1]]
-; COST10-NEXT: [[BOUND03:%.*]] = icmp ult ptr [[DST]], [[SCEVGEP2]]
-; COST10-NEXT: [[BOUND14:%.*]] = icmp ult ptr [[INVARIANT_PTR]], [[SCEVGEP]]
-; COST10-NEXT: [[FOUND_CONFLICT5:%.*]] = and i1 [[BOUND03]], [[BOUND14]]
-; COST10-NEXT: [[CONFLICT_RDX:%.*]] = or i1 [[FOUND_CONFLICT]], [[FOUND_CONFLICT5]]
-; COST10-NEXT: [[BOUND06:%.*]] = icmp ult ptr [[KEYINFO]], [[SCEVGEP2]]
-; COST10-NEXT: [[BOUND17:%.*]] = icmp ult ptr [[INVARIANT_PTR]], [[SCEVGEP1]]
-; COST10-NEXT: [[FOUND_CONFLICT8:%.*]] = and i1 [[BOUND06]], [[BOUND17]]
-; COST10-NEXT: [[CONFLICT_RDX9:%.*]] = or i1 [[CONFLICT_RDX]], [[FOUND_CONFLICT8]]
-; COST10-NEXT: br i1 [[CONFLICT_RDX9]], label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]]
-; COST10: [[VECTOR_PH]]:
-; COST10-NEXT: br label %[[VECTOR_BODY:.*]]
-; COST10: [[VECTOR_BODY]]:
-; COST10-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[LOOP_LATCH26:.*]] ]
-; COST10-NEXT: [[TMP0:%.*]] = load i32, ptr [[INVARIANT_PTR]], align 4, !alias.scope [[META15:![0-9]+]]
-; COST10-NEXT: [[TMP1:%.*]] = freeze i32 [[TMP0]]
-; COST10-NEXT: [[BROADCAST_SPLATINSERT:%.*]] = insertelement <4 x i32> poison, i32 [[TMP1]], i64 0
-; COST10-NEXT: [[BROADCAST_SPLAT:%.*]] = shufflevector <4 x i32> [[BROADCAST_SPLATINSERT]], <4 x i32> poison, <4 x i32> zeroinitializer
-; COST10-NEXT: [[TMP2:%.*]] = icmp eq <4 x i32> [[BROADCAST_SPLAT]], zeroinitializer
-; COST10-NEXT: [[TMP3:%.*]] = extractelement <4 x i1> [[TMP2]], i64 0
-; COST10-NEXT: br i1 [[TMP3]], label %[[IF_ZERO17:.*]], label %[[IF_NONZERO10:.*]]
-; COST10: [[IF_NONZERO10]]:
-; COST10-NEXT: [[TMP4:%.*]] = extractelement <4 x i1> [[TMP2]], i64 0
-; COST10-NEXT: [[TMP5:%.*]] = xor i1 [[TMP4]], true
-; COST10-NEXT: br i1 [[TMP5]], label %[[PRED_STORE_IF:.*]], label %[[PRED_STORE_CONTINUE:.*]]
-; COST10: [[PRED_STORE_IF]]:
-; COST10-NEXT: [[TMP6:%.*]] = srem i32 1, [[DIVISOR]]
-; COST10-NEXT: store i32 [[TMP6]], ptr [[DST]], align 4, !alias.scope [[META18:![0-9]+]], !noalias [[META20:![0-9]+]]
-; COST10-NEXT: br label %[[PRED_STORE_CONTINUE]]
-; COST10: [[PRED_STORE_CONTINUE]]:
-; COST10-NEXT: br i1 [[TMP5]], label %[[PRED_STORE_IF11:.*]], label %[[PRED_STORE_CONTINUE12:.*]]
-; COST10: [[PRED_STORE_IF11]]:
-; COST10-NEXT: [[TMP7:%.*]] = srem i32 1, [[DIVISOR]]
-; COST10-NEXT: store i32 [[TMP7]], ptr [[DST]], align 4, !alias.scope [[META18]], !noalias [[META20]]
-; COST10-NEXT: br label %[[PRED_STORE_CONTINUE12]]
-; COST10: [[PRED_STORE_CONTINUE12]]:
-; COST10-NEXT: br i1 [[TMP5]], label %[[PRED_STORE_IF13:.*]], label %[[PRED_STORE_CONTINUE14:.*]]
-; COST10: [[PRED_STORE_IF13]]:
-; COST10-NEXT: [[TMP8:%.*]] = srem i32 1, [[DIVISOR]]
-; COST10-NEXT: store i32 [[TMP8]], ptr [[DST]], align 4, !alias.scope [[META18]], !noalias [[META20]]
-; COST10-NEXT: br label %[[PRED_STORE_CONTINUE14]]
-; COST10: [[PRED_STORE_CONTINUE14]]:
-; COST10-NEXT: br i1 [[TMP5]], label %[[PRED_STORE_IF15:.*]], label %[[PRED_STORE_CONTINUE16:.*]]
-; COST10: [[PRED_STORE_IF15]]:
-; COST10-NEXT: [[TMP9:%.*]] = srem i32 1, [[DIVISOR]]
-; COST10-NEXT: store i32 [[TMP9]], ptr [[DST]], align 4, !alias.scope [[META18]], !noalias [[META20]]
-; COST10-NEXT: br label %[[PRED_STORE_CONTINUE16]]
-; COST10: [[PRED_STORE_CONTINUE16]]:
-; COST10-NEXT: br label %[[LOOP_LATCH26]]
-; COST10: [[IF_ZERO17]]:
-; COST10-NEXT: [[TMP10:%.*]] = extractelement <4 x i1> [[TMP2]], i64 0
-; COST10-NEXT: br i1 [[TMP10]], label %[[PRED_STORE_IF18:.*]], label %[[PRED_STORE_CONTINUE19:.*]]
-; COST10: [[PRED_STORE_IF18]]:
-; COST10-NEXT: store i32 0, ptr [[KEYINFO]], align 4, !alias.scope [[META22:![0-9]+]], !noalias [[META15]]
-; COST10-NEXT: br label %[[PRED_STORE_CONTINUE19]]
-; COST10: [[PRED_STORE_CONTINUE19]]:
-; COST10-NEXT: [[TMP11:%.*]] = extractelement <4 x i1> [[TMP2]], i64 0
-; COST10-NEXT: br i1 [[TMP11]], label %[[PRED_STORE_IF20:.*]], label %[[PRED_STORE_CONTINUE21:.*]]
-; COST10: [[PRED_STORE_IF20]]:
-; COST10-NEXT: store i32 0, ptr [[KEYINFO]], align 4, !alias.scope [[META22]], !noalias [[META15]]
-; COST10-NEXT: br label %[[PRED_STORE_CONTINUE21]]
-; COST10: [[PRED_STORE_CONTINUE21]]:
-; COST10-NEXT: [[TMP12:%.*]] = extractelement <4 x i1> [[TMP2]], i64 0
-; COST10-NEXT: br i1 [[TMP12]], label %[[PRED_STORE_IF22:.*]], label %[[PRED_STORE_CONTINUE23:.*]]
-; COST10: [[PRED_STORE_IF22]]:
-; COST10-NEXT: store i32 0, ptr [[KEYINFO]], align 4, !alias.scope [[META22]], !noalias [[META15]]
-; COST10-NEXT: br label %[[PRED_STORE_CONTINUE23]]
-; COST10: [[PRED_STORE_CONTINUE23]]:
-; COST10-NEXT: [[TMP13:%.*]] = extractelement <4 x i1> [[TMP2]], i64 0
-; COST10-NEXT: br i1 [[TMP13]], label %[[PRED_STORE_IF24:.*]], label %[[PRED_STORE_CONTINUE25:.*]]
-; COST10: [[PRED_STORE_IF24]]:
-; COST10-NEXT: store i32 0, ptr [[KEYINFO]], align 4, !alias.scope [[META22]], !noalias [[META15]]
-; COST10-NEXT: br label %[[PRED_STORE_CONTINUE25]]
-; COST10: [[PRED_STORE_CONTINUE25]]:
-; COST10-NEXT: br label %[[LOOP_LATCH26]]
-; COST10: [[LOOP_LATCH26]]:
-; COST10-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 4
-; COST10-NEXT: [[TMP14:%.*]] = icmp eq i64 [[INDEX_NEXT]], 32
-; COST10-NEXT: br i1 [[TMP14]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP23:![0-9]+]]
-; COST10: [[MIDDLE_BLOCK]]:
-; COST10-NEXT: br label %[[SCALAR_PH]]
-; COST10: [[SCALAR_PH]]:
+; COMMON-LABEL: define void @loop_with_freeze_and_conditional_srem(
+; COMMON-SAME: ptr [[DST:%.*]], ptr [[KEYINFO:%.*]], ptr [[INVARIANT_PTR:%.*]], i32 [[DIVISOR:%.*]]) {
+; COMMON-NEXT: [[ENTRY:.*]]:
+; COMMON-NEXT: br label %[[LOOP:.*]]
+; COMMON: [[LOOP]]:
+; COMMON-NEXT: [[IV:%.*]] = phi i64 [ 0, %[[ENTRY]] ], [ [[IV_NEXT:%.*]], %[[LOOP_LATCH:.*]] ]
+; COMMON-NEXT: [[LOADED:%.*]] = load i32, ptr [[INVARIANT_PTR]], align 4
+; COMMON-NEXT: [[FROZEN:%.*]] = freeze i32 [[LOADED]]
+; COMMON-NEXT: [[CMP:%.*]] = icmp eq i32 [[FROZEN]], 0
+; COMMON-NEXT: br i1 [[CMP]], label %[[IF_ZERO:.*]], label %[[IF_NONZERO:.*]]
+; COMMON: [[IF_ZERO]]:
+; COMMON-NEXT: store i32 0, ptr [[KEYINFO]], align 4
+; COMMON-NEXT: br label %[[LOOP_LATCH]]
+; COMMON: [[IF_NONZERO]]:
+; COMMON-NEXT: [[REM:%.*]] = srem i32 1, [[DIVISOR]]
+; COMMON-NEXT: store i32 [[REM]], ptr [[DST]], align 4
+; COMMON-NEXT: br label %[[LOOP_LATCH]]
+; COMMON: [[LOOP_LATCH]]:
+; COMMON-NEXT: [[IV_NEXT]] = add i64 [[IV]], 1
+; COMMON-NEXT: [[EXITCOND:%.*]] = icmp eq i64 [[IV]], 32
+; COMMON-NEXT: br i1 [[EXITCOND]], label %[[EXIT:.*]], label %[[LOOP]]
+; COMMON: [[EXIT]]:
+; COMMON-NEXT: ret void
;
entry:
br label %loop
@@ -679,7 +469,7 @@ define void @interleave_group(ptr %dst) #1 {
; COST1-NEXT: store <vscale x 48 x i8> [[INTERLEAVED_VEC1]], ptr [[TMP4]], align 1
; COST1-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], [[TMP25]]
; COST1-NEXT: [[TMP29:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
-; COST1-NEXT: br i1 [[TMP29]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP25:![0-9]+]]
+; COST1-NEXT: br i1 [[TMP29]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP15:![0-9]+]]
; COST1: [[MIDDLE_BLOCK]]:
; COST1-NEXT: [[CMP_N:%.*]] = icmp eq i64 101, [[N_VEC]]
; COST1-NEXT: br i1 [[CMP_N]], [[EXIT:label %.*]], label %[[VEC_EPILOG_ITER_CHECK:.*]]
@@ -727,7 +517,7 @@ define void @interleave_group(ptr %dst) #1 {
; COST1-NEXT: [[INDEX_NEXT2]] = add nuw i64 [[INDEX1]], 4
; COST1-NEXT: [[VEC_IND_NEXT]] = add <4 x i64> [[VEC_IND]], splat (i64 4)
; COST1-NEXT: [[TMP23:%.*]] = icmp eq i64 [[INDEX_NEXT2]], 100
-; COST1-NEXT: br i1 [[TMP23]], label %[[VEC_EPILOG_MIDDLE_BLOCK:.*]], label %[[VEC_EPILOG_VECTOR_BODY]], !llvm.loop [[LOOP26:![0-9]+]]
+; COST1-NEXT: br i1 [[TMP23]], label %[[VEC_EPILOG_MIDDLE_BLOCK:.*]], label %[[VEC_EPILOG_VECTOR_BODY]], !llvm.loop [[LOOP16:![0-9]+]]
; COST1: [[VEC_EPILOG_MIDDLE_BLOCK]]:
; COST1-NEXT: br i1 false, [[EXIT]], label %[[VEC_EPILOG_SCALAR_PH]]
; COST1: [[VEC_EPILOG_SCALAR_PH]]:
@@ -753,7 +543,7 @@ define void @interleave_group(ptr %dst) #1 {
; COST10-NEXT: store <vscale x 48 x i8> [[INTERLEAVED_VEC]], ptr [[TMP1]], align 1
; COST10-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], [[TMP22]]
; COST10-NEXT: [[TMP23:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
-; COST10-NEXT: br i1 [[TMP23]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP25:![0-9]+]]
+; COST10-NEXT: br i1 [[TMP23]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP15:![0-9]+]]
; COST10: [[MIDDLE_BLOCK]]:
; COST10-NEXT: [[CMP_N:%.*]] = icmp eq i64 101, [[N_VEC]]
; COST10-NEXT: br i1 [[CMP_N]], [[EXIT:label %.*]], label %[[VEC_EPILOG_ITER_CHECK:.*]]
@@ -801,7 +591,7 @@ define void @interleave_group(ptr %dst) #1 {
; COST10-NEXT: [[INDEX_NEXT2]] = add nuw i64 [[INDEX1]], 4
; COST10-NEXT: [[VEC_IND_NEXT]] = add <4 x i64> [[VEC_IND]], splat (i64 4)
; COST10-NEXT: [[TMP20:%.*]] = icmp eq i64 [[INDEX_NEXT2]], 100
-; COST10-NEXT: br i1 [[TMP20]], label %[[VEC_EPILOG_MIDDLE_BLOCK:.*]], label %[[VEC_EPILOG_VECTOR_BODY]], !llvm.loop [[LOOP26:![0-9]+]]
+; COST10-NEXT: br i1 [[TMP20]], label %[[VEC_EPILOG_MIDDLE_BLOCK:.*]], label %[[VEC_EPILOG_VECTOR_BODY]], !llvm.loop [[LOOP16:![0-9]+]]
; COST10: [[VEC_EPILOG_MIDDLE_BLOCK]]:
; COST10-NEXT: br i1 false, [[EXIT]], label %[[VEC_EPILOG_SCALAR_PH]]
; COST10: [[VEC_EPILOG_SCALAR_PH]]:
@@ -879,7 +669,7 @@ define void @forced_scalar_instr(ptr %gep.dst) {
; COMMON-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 4
; COMMON-NEXT: [[VEC_IND_NEXT]] = add nuw <4 x i8> [[VEC_IND]], splat (i8 4)
; COMMON-NEXT: [[TMP22:%.*]] = icmp eq i64 [[INDEX_NEXT]], 8
-; COMMON-NEXT: br i1 [[TMP22]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP28:![0-9]+]]
+; COMMON-NEXT: br i1 [[TMP22]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP18:![0-9]+]]
; COMMON: [[MIDDLE_BLOCK]]:
; COMMON-NEXT: br label %[[EXIT:.*]]
; COMMON: [[EXIT]]:
@@ -950,18 +740,18 @@ define void @force_branch_cost(ptr readonly %src, ptr %dst) {
; COST1-NEXT: [[TMP22:%.*]] = getelementptr [4 x i8], ptr [[SRC]], i64 [[TMP6]]
; COST1-NEXT: [[TMP23:%.*]] = getelementptr [4 x i8], ptr [[SRC]], i64 [[TMP7]]
; COST1-NEXT: [[TMP24:%.*]] = getelementptr [4 x i8], ptr [[SRC]], i64 [[TMP8]]
-; COST1-NEXT: [[TMP25:%.*]] = load i8, ptr [[TMP17]], align 1, !alias.scope [[META29:![0-9]+]]
-; COST1-NEXT: [[TMP26:%.*]] = load i8, ptr [[TMP18]], align 1, !alias.scope [[META29]]
-; COST1-NEXT: [[TMP27:%.*]] = load i8, ptr [[TMP19]], align 1, !alias.scope [[META29]]
-; COST1-NEXT: [[TMP28:%.*]] = load i8, ptr [[TMP20]], align 1, !alias.scope [[META29]]
+; COST1-NEXT: [[TMP25:%.*]] = load i8, ptr [[TMP17]], align 1, !alias.scope [[META19:![0-9]+]]
+; COST1-NEXT: [[TMP26:%.*]] = load i8, ptr [[TMP18]], align 1, !alias.scope [[META19]]
+; COST1-NEXT: [[TMP27:%.*]] = load i8, ptr [[TMP19]], align 1, !alias.scope [[META19]]
+; COST1-NEXT: [[TMP28:%.*]] = load i8, ptr [[TMP20]], align 1, !alias.scope [[META19]]
; COST1-NEXT: [[TMP29:%.*]] = insertelement <4 x i8> poison, i8 [[TMP25]], i64 0
; COST1-NEXT: [[TMP30:%.*]] = insertelement <4 x i8> [[TMP29]], i8 [[TMP26]], i64 1
; COST1-NEXT: [[TMP31:%.*]] = insertelement <4 x i8> [[TMP30]], i8 [[TMP27]], i64 2
; COST1-NEXT: [[TMP32:%.*]] = insertelement <4 x i8> [[TMP31]], i8 [[TMP28]], i64 3
-; COST1-NEXT: [[TMP33:%.*]] = load i8, ptr [[TMP21]], align 1, !alias.scope [[META29]]
-; COST1-NEXT: [[TMP34:%.*]] = load i8, ptr [[TMP22]], align 1, !alias.scope [[META29]]
-; COST1-NEXT: [[TMP35:%.*]] = load i8, ptr [[TMP23]], align 1, !alias.scope [[META29]]
-; COST1-NEXT: [[TMP36:%.*]] = load i8, ptr [[TMP24]], align 1, !alias.scope [[META29]]
+; COST1-NEXT: [[TMP33:%.*]] = load i8, ptr [[TMP21]], align 1, !alias.scope [[META19]]
+; COST1-NEXT: [[TMP34:%.*]] = load i8, ptr [[TMP22]], align 1, !alias.scope [[META19]]
+; COST1-NEXT: [[TMP35:%.*]] = load i8, ptr [[TMP23]], align 1, !alias.scope [[META19]]
+; COST1-NEXT: [[TMP36:%.*]] = load i8, ptr [[TMP24]], align 1, !alias.scope [[META19]]
; COST1-NEXT: [[TMP37:%.*]] = insertelement <4 x i8> poison, i8 [[TMP33]], i64 0
; COST1-NEXT: [[TMP38:%.*]] = insertelement <4 x i8> [[TMP37]], i8 [[TMP34]], i64 1
; COST1-NEXT: [[TMP39:%.*]] = insertelement <4 x i8> [[TMP38]], i8 [[TMP35]], i64 2
@@ -969,21 +759,21 @@ define void @force_branch_cost(ptr readonly %src, ptr %dst) {
; COST1-NEXT: [[TMP41:%.*]] = zext <4 x i8> [[TMP32]] to <4 x i32>
; COST1-NEXT: [[TMP46:%.*]] = zext <4 x i8> [[TMP40]] to <4 x i32>
; COST1-NEXT: [[TMP44:%.*]] = extractelement <4 x i32> [[TMP41]], i64 0
-; COST1-NEXT: store i32 [[TMP44]], ptr [[NEXT_GEP]], align 4, !alias.scope [[META32:![0-9]+]], !noalias [[META29]]
+; COST1-NEXT: store i32 [[TMP44]], ptr [[NEXT_GEP]], align 4, !alias.scope [[META22:![0-9]+]], !noalias [[META19]]
; COST1-NEXT: [[TMP49:%.*]] = extractelement <4 x i32> [[TMP41]], i64 1
-; COST1-NEXT: store i32 [[TMP49]], ptr [[NEXT_GEP2]], align 4, !alias.scope [[META32]], !noalias [[META29]]
+; COST1-NEXT: store i32 [[TMP49]], ptr [[NEXT_GEP2]], align 4, !alias.scope [[META22]], !noalias [[META19]]
; COST1-NEXT: [[TMP42:%.*]] = extractelement <4 x i32> [[TMP41]], i64 2
-; COST1-NEXT: store i32 [[TMP42]], ptr [[NEXT_GEP3]], align 4, !alias.scope [[META32]], !noalias [[META29]]
+; COST1-NEXT: store i32 [[TMP42]], ptr [[NEXT_GEP3]], align 4, !alias.scope [[META22]], !noalias [[META19]]
; COST1-NEXT: [[TMP43:%.*]] = extractelement <4 x i32> [[TMP41]], i64 3
-; COST1-NEXT: store i32 [[TMP43]], ptr [[NEXT_GEP4]], align 4, !alias.scope [[META32]], !noalias [[META29]]
+; COST1-NEXT: store i32 [[TMP43]], ptr [[NEXT_GEP4]], align 4, !alias.scope [[META22]], !noalias [[META19]]
; COST1-NEXT: [[TMP45:%.*]] = extractelement <4 x i32> [[TMP46]], i64 0
-; COST1-NEXT: store i32 [[TMP45]], ptr [[NEXT_GEP5]], align 4, !alias.scope [[META32]], !noalias [[META29]]
+; COST1-NEXT: store i32 [[TMP45]], ptr [[NEXT_GEP5]], align 4, !alias.scope [[META22]], !noalias [[META19]]
; COST1-NEXT: [[TMP50:%.*]] = extractelement <4 x i32> [[TMP46]], i64 1
-; COST1-NEXT: store i32 [[TMP50]], ptr [[NEXT_GEP6]], align 4, !alias.scope [[META32]], !noalias [[META29]]
+; COST1-NEXT: store i32 [[TMP50]], ptr [[NEXT_GEP6]], align 4, !alias.scope [[META22]], !noalias [[META19]]
; COST1-NEXT: [[TMP47:%.*]] = extractelement <4 x i32> [[TMP46]], i64 2
-; COST1-NEXT: store i32 [[TMP47]], ptr [[NEXT_GEP7]], align 4, !alias.scope [[META32]], !noalias [[META29]]
+; COST1-NEXT: store i32 [[TMP47]], ptr [[NEXT_GEP7]], align 4, !alias.scope [[META22]], !noalias [[META19]]
; COST1-NEXT: [[TMP48:%.*]] = extractelement <4 x i32> [[TMP46]], i64 3
-; COST1-NEXT: store i32 [[TMP48]], ptr [[NEXT_GEP8]], align 4, !alias.scope [[META32]], !noalias [[META29]]
+; COST1-NEXT: store i32 [[TMP48]], ptr [[NEXT_GEP8]], align 4, !alias.scope [[META22]], !noalias [[META19]]
; COST1-NEXT: [[TMP51:%.*]] = getelementptr i8, ptr [[NEXT_GEP]], i64 24
; COST1-NEXT: [[TMP52:%.*]] = getelementptr i8, ptr [[NEXT_GEP2]], i64 24
; COST1-NEXT: [[TMP53:%.*]] = getelementptr i8, ptr [[NEXT_GEP3]], i64 24
@@ -992,14 +782,14 @@ define void @force_branch_cost(ptr readonly %src, ptr %dst) {
; COST1-NEXT: [[TMP56:%.*]] = getelementptr i8, ptr [[NEXT_GEP6]], i64 24
; COST1-NEXT: [[TMP57:%.*]] = getelementptr i8, ptr [[NEXT_GEP7]], i64 24
; COST1-NEXT: [[TMP58:%.*]] = getelementptr i8, ptr [[NEXT_GEP8]], i64 24
-; COST1-NEXT: store i32 [[TMP44]], ptr [[TMP51]], align 4, !alias.scope [[META32]], !noalias [[META29]]
-; COST1-NEXT: store i32 [[TMP49]], ptr [[TMP52]], align 4, !alias.scope [[META32]], !noalias [[META29]]
-; COST1-NEXT: store i32 [[TMP42]], ptr [[TMP53]], align 4, !alias.scope [[META32]], !noalias [[META29]]
-; COST1-NEXT: store i32 [[TMP43]], ptr [[TMP54]], align 4, !alias.scope [[META32]], !noalias [[META29]]
-; COST1-NEXT: store i32 [[TMP45]], ptr [[TMP55]], align 4, !alias.scope [[META32]], !noalias [[META29]]
-; COST1-NEXT: store i32 [[TMP50]], ptr [[TMP56]], align 4, !alias.scope [[META32]], !noalias [[META29]]
-; COST1-NEXT: store i32 [[TMP47]], ptr [[TMP57]], align 4, !alias.scope [[META32]], !noalias [[META29]]
-; COST1-NEXT: store i32 [[TMP48]], ptr [[TMP58]], align 4, !alias.scope [[META32]], !noalias [[META29]]
+; COST1-NEXT: store i32 [[TMP44]], ptr [[TMP51]], align 4, !alias.scope [[META22]], !noalias [[META19]]
+; COST1-NEXT: store i32 [[TMP49]], ptr [[TMP52]], align 4, !alias.scope [[META22]], !noalias [[META19]]
+; COST1-NEXT: store i32 [[TMP42]], ptr [[TMP53]], align 4, !alias.scope [[META22]], !noalias [[META19]]
+; COST1-NEXT: store i32 [[TMP43]], ptr [[TMP54]], align 4, !alias.scope [[META22]], !noalias [[META19]]
+; COST1-NEXT: store i32 [[TMP45]], ptr [[TMP55]], align 4, !alias.scope [[META22]], !noalias [[META19]]
+; COST1-NEXT: store i32 [[TMP50]], ptr [[TMP56]], align 4, !alias.scope [[META22]], !noalias [[META19]]
+; COST1-NEXT: store i32 [[TMP47]], ptr [[TMP57]], align 4, !alias.scope [[META22]], !noalias [[META19]]
+; COST1-NEXT: store i32 [[TMP48]], ptr [[TMP58]], align 4, !alias.scope [[META22]], !noalias [[META19]]
; COST1-NEXT: [[TMP59:%.*]] = getelementptr i8, ptr [[NEXT_GEP]], i64 48
; COST1-NEXT: [[TMP60:%.*]] = getelementptr i8, ptr [[NEXT_GEP2]], i64 48
; COST1-NEXT: [[TMP61:%.*]] = getelementptr i8, ptr [[NEXT_GEP3]], i64 48
@@ -1008,14 +798,14 @@ define void @force_branch_cost(ptr readonly %src, ptr %dst) {
; COST1-NEXT: [[TMP64:%.*]] = getelementptr i8, ptr [[NEXT_GEP6]], i64 48
; COST1-NEXT: [[TMP65:%.*]] = getelementptr i8, ptr [[NEXT_GEP7]], i64 48
; COST1-NEXT: [[TMP66:%.*]] = getelementptr i8, ptr [[NEXT_GEP8]], i64 48
-; COST1-NEXT: store i32 [[TMP44]], ptr [[TMP59]], align 4, !alias.scope [[META32]], !noalias [[META29]]
-; COST1-NEXT: store i32 [[TMP49]], ptr [[TMP60]], align 4, !alias.scope [[META32]], !noalias [[META29]]
-; COST1-NEXT: store i32 [[TMP42]], ptr [[TMP61]], align 4, !alias.scope [[META32]], !noalias [[META29]]
-; COST1-NEXT: store i32 [[TMP43]], ptr [[TMP62]], align 4, !alias.scope [[META32]], !noalias [[META29]]
-; COST1-NEXT: store i32 [[TMP45]], ptr [[TMP63]], align 4, !alias.scope [[META32]], !noalias [[META29]]
-; COST1-NEXT: store i32 [[TMP50]], ptr [[TMP64]], align 4, !alias.scope [[META32]], !noalias [[META29]]
-; COST1-NEXT: store i32 [[TMP47]], ptr [[TMP65]], align 4, !alias.scope [[META32]], !noalias [[META29]]
-; COST1-NEXT: store i32 [[TMP48]], ptr [[TMP66]], align 4, !alias.scope [[META32]], !noalias [[META29]]
+; COST1-NEXT: store i32 [[TMP44]], ptr [[TMP59]], align 4, !alias.scope [[META22]], !noalias [[META19]]
+; COST1-NEXT: store i32 [[TMP49]], ptr [[TMP60]], align 4, !alias.scope [[META22]], !noalias [[META19]]
+; COST1-NEXT: store i32 [[TMP42]], ptr [[TMP61]], align 4, !alias.scope [[META22]], !noalias [[META19]]
+; COST1-NEXT: store i32 [[TMP43]], ptr [[TMP62]], align 4, !alias.scope [[META22]], !noalias [[META19]]
+; COST1-NEXT: store i32 [[TMP45]], ptr [[TMP63]], align 4, !alias.scope [[META22]], !noalias [[META19]]
+; COST1-NEXT: store i32 [[TMP50]], ptr [[TMP64]], align 4, !alias.scope [[META22]], !noalias [[META19]]
+; COST1-NEXT: store i32 [[TMP47]], ptr [[TMP65]], align 4, !alias.scope [[META22]], !noalias [[META19]]
+; COST1-NEXT: store i32 [[TMP48]], ptr [[TMP66]], align 4, !alias.scope [[META22]], !noalias [[META19]]
; COST1-NEXT: [[TMP67:%.*]] = getelementptr i8, ptr [[NEXT_GEP]], i64 72
; COST1-NEXT: [[TMP68:%.*]] = getelementptr i8, ptr [[NEXT_GEP2]], i64 72
; COST1-NEXT: [[TMP69:%.*]] = getelementptr i8, ptr [[NEXT_GEP3]], i64 72
@@ -1024,17 +814,17 @@ define void @force_branch_cost(ptr readonly %src, ptr %dst) {
; COST1-NEXT: [[TMP72:%.*]] = getelementptr i8, ptr [[NEXT_GEP6]], i64 72
; COST1-NEXT: [[TMP73:%.*]] = getelementptr i8, ptr [[NEXT_GEP7]], i64 72
; COST1-NEXT: [[TMP74:%.*]] = getelementptr i8, ptr [[NEXT_GEP8]], i64 72
-; COST1-NEXT: store i32 [[TMP44]], ptr [[TMP67]], align 4, !alias.scope [[META32]], !noalias [[META29]]
-; COST1-NEXT: store i32 [[TMP49]], ptr [[TMP68]], align 4, !alias.scope [[META32]], !noalias [[META29]]
-; COST1-NEXT: store i32 [[TMP42]], ptr [[TMP69]], align 4, !alias.scope [[META32]], !noalias [[META29]]
-; COST1-NEXT: store i32 [[TMP43]], ptr [[TMP70]], align 4, !alias.scope [[META32]], !noalias [[META29]]
-; COST1-NEXT: store i32 [[TMP45]], ptr [[TMP71]], align 4, !alias.scope [[META32]], !noalias [[META29]]
-; COST1-NEXT: store i32 [[TMP50]], ptr [[TMP72]], align 4, !alias.scope [[META32]], !noalias [[META29]]
-; COST1-NEXT: store i32 [[TMP47]], ptr [[TMP73]], align 4, !alias.scope [[META32]], !noalias [[META29]]
-; COST1-NEXT: store i32 [[TMP48]], ptr [[TMP74]], align 4, !alias.scope [[META32]], !noalias [[META29]]
+; COST1-NEXT: store i32 [[TMP44]], ptr [[TMP67]], align 4, !alias.scope [[META22]], !noalias [[META19]]
+; COST1-NEXT: store i32 [[TMP49]], ptr [[TMP68]], align 4, !alias.scope [[META22]], !noalias [[META19]]
+; COST1-NEXT: store i32 [[TMP42]], ptr [[TMP69]], align 4, !alias.scope [[META22]], !noalias [[META19]]
+; COST1-NEXT: store i32 [[TMP43]], ptr [[TMP70]], align 4, !alias.scope [[META22]], !noalias [[META19]]
+; COST1-NEXT: store i32 [[TMP45]], ptr [[TMP71]], align 4, !alias.scope [[META22]], !noalias [[META19]]
+; COST1-NEXT: store i32 [[TMP50]], ptr [[TMP72]], align 4, !alias.scope [[META22]], !noalias [[META19]]
+; COST1-NEXT: store i32 [[TMP47]], ptr [[TMP73]], align 4, !alias.scope [[META22]], !noalias [[META19]]
+; COST1-NEXT: store i32 [[TMP48]], ptr [[TMP74]], align 4, !alias.scope [[META22]], !noalias [[META19]]
; COST1-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 8
; COST1-NEXT: [[TMP75:%.*]] = icmp eq i64 [[INDEX_NEXT]], 16
-; COST1-NEXT: br i1 [[TMP75]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP34:![0-9]+]]
+; COST1-NEXT: br i1 [[TMP75]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP24:![0-9]+]]
; COST1: [[MIDDLE_BLOCK]]:
; COST1-NEXT: br label %[[SCALAR_PH]]
; COST1: [[SCALAR_PH]]:
@@ -1070,50 +860,50 @@ define void @force_branch_cost(ptr readonly %src, ptr %dst) {
; COST10-NEXT: [[TMP10:%.*]] = getelementptr [4 x i8], ptr [[SRC]], i64 [[TMP2]]
; COST10-NEXT: [[TMP11:%.*]] = getelementptr [4 x i8], ptr [[SRC]], i64 [[TMP3]]
; COST10-NEXT: [[TMP12:%.*]] = getelementptr [4 x i8], ptr [[SRC]], i64 [[TMP4]]
-; COST10-NEXT: [[TMP13:%.*]] = load i8, ptr [[TMP9]], align 1, !alias.scope [[META29:![0-9]+]]
-; COST10-NEXT: [[TMP14:%.*]] = load i8, ptr [[TMP10]], align 1, !alias.scope [[META29]]
-; COST10-NEXT: [[TMP15:%.*]] = load i8, ptr [[TMP11]], align 1, !alias.scope [[META29]]
-; COST10-NEXT: [[TMP16:%.*]] = load i8, ptr [[TMP12]], align 1, !alias.scope [[META29]]
+; COST10-NEXT: [[TMP13:%.*]] = load i8, ptr [[TMP9]], align 1, !alias.scope [[META19:![0-9]+]]
+; COST10-NEXT: [[TMP14:%.*]] = load i8, ptr [[TMP10]], align 1, !alias.scope [[META19]]
+; COST10-NEXT: [[TMP15:%.*]] = load i8, ptr [[TMP11]], align 1, !alias.scope [[META19]]
+; COST10-NEXT: [[TMP16:%.*]] = load i8, ptr [[TMP12]], align 1, !alias.scope [[META19]]
; COST10-NEXT: [[TMP17:%.*]] = insertelement <4 x i8> poison, i8 [[TMP13]], i64 0
; COST10-NEXT: [[TMP18:%.*]] = insertelement <4 x i8> [[TMP17]], i8 [[TMP14]], i64 1
; COST10-NEXT: [[TMP19:%.*]] = insertelement <4 x i8> [[TMP18]], i8 [[TMP15]], i64 2
; COST10-NEXT: [[TMP20:%.*]] = insertelement <4 x i8> [[TMP19]], i8 [[TMP16]], i64 3
; COST10-NEXT: [[TMP21:%.*]] = zext <4 x i8> [[TMP20]] to <4 x i32>
; COST10-NEXT: [[TMP24:%.*]] = extractelement <4 x i32> [[TMP21]], i64 0
-; COST10-NEXT: store i32 [[TMP24]], ptr [[NEXT_GEP]], align 4, !alias.scope [[META32:![0-9]+]], !noalias [[META29]]
+; COST10-NEXT: store i32 [[TMP24]], ptr [[NEXT_GEP]], align 4, !alias.scope [[META22:![0-9]+]], !noalias [[META19]]
; COST10-NEXT: [[TMP25:%.*]] = extractelement <4 x i32> [[TMP21]], i64 1
-; COST10-NEXT: store i32 [[TMP25]], ptr [[NEXT_GEP2]], align 4, !alias.scope [[META32]], !noalias [[META29]]
+; COST10-NEXT: store i32 [[TMP25]], ptr [[NEXT_GEP2]], align 4, !alias.scope [[META22]], !noalias [[META19]]
; COST10-NEXT: [[TMP22:%.*]] = extractelement <4 x i32> [[TMP21]], i64 2
-; COST10-NEXT: store i32 [[TMP22]], ptr [[NEXT_GEP3]], align 4, !alias.scope [[META32]], !noalias [[META29]]
+; COST10-NEXT: store i32 [[TMP22]], ptr [[NEXT_GEP3]], align 4, !alias.scope [[META22]], !noalias [[META19]]
; COST10-NEXT: [[TMP23:%.*]] = extractelement <4 x i32> [[TMP21]], i64 3
-; COST10-NEXT: store i32 [[TMP23]], ptr [[NEXT_GEP4]], align 4, !alias.scope [[META32]], !noalias [[META29]]
+; COST10-NEXT: store i32 [[TMP23]], ptr [[NEXT_GEP4]], align 4, !alias.scope [[META22]], !noalias [[META19]]
; COST10-NEXT: [[TMP26:%.*]] = getelementptr i8, ptr [[NEXT_GEP]], i64 24
; COST10-NEXT: [[TMP27:%.*]] = getelementptr i8, ptr [[NEXT_GEP2]], i64 24
; COST10-NEXT: [[TMP28:%.*]] = getelementptr i8, ptr [[NEXT_GEP3]], i64 24
; COST10-NEXT: [[TMP29:%.*]] = getelementptr i8, ptr [[NEXT_GEP4]], i64 24
-; COST10-NEXT: store i32 [[TMP24]], ptr [[TMP26]], align 4, !alias.scope [[META32]], !noalias [[META29]]
-; COST10-NEXT: store i32 [[TMP25]], ptr [[TMP27]], align 4, !alias.scope [[META32]], !noalias [[META29]]
-; COST10-NEXT: store i32 [[TMP22]], ptr [[TMP28]], align 4, !alias.scope [[META32]], !noalias [[META29]]
-; COST10-NEXT: store i32 [[TMP23]], ptr [[TMP29]], align 4, !alias.scope [[META32]], !noalias [[META29]]
+; COST10-NEXT: store i32 [[TMP24]], ptr [[TMP26]], align 4, !alias.scope [[META22]], !noalias [[META19]]
+; COST10-NEXT: store i32 [[TMP25]], ptr [[TMP27]], align 4, !alias.scope [[META22]], !noalias [[META19]]
+; COST10-NEXT: store i32 [[TMP22]], ptr [[TMP28]], align 4, !alias.scope [[META22]], !noalias [[META19]]
+; COST10-NEXT: store i32 [[TMP23]], ptr [[TMP29]], align 4, !alias.scope [[META22]], !noalias [[META19]]
; COST10-NEXT: [[TMP30:%.*]] = getelementptr i8, ptr [[NEXT_GEP]], i64 48
; COST10-NEXT: [[TMP31:%.*]] = getelementptr i8, ptr [[NEXT_GEP2]], i64 48
; COST10-NEXT: [[TMP32:%.*]] = getelementptr i8, ptr [[NEXT_GEP3]], i64 48
; COST10-NEXT: [[TMP33:%.*]] = getelementptr i8, ptr [[NEXT_GEP4]], i64 48
-; COST10-NEXT: store i32 [[TMP24]], ptr [[TMP30]], align 4, !alias.scope [[META32]], !noalias [[META29]]
-; COST10-NEXT: store i32 [[TMP25]], ptr [[TMP31]], align 4, !alias.scope [[META32]], !noalias [[META29]]
-; COST10-NEXT: store i32 [[TMP22]], ptr [[TMP32]], align 4, !alias.scope [[META32]], !noalias [[META29]]
-; COST10-NEXT: store i32 [[TMP23]], ptr [[TMP33]], align 4, !alias.scope [[META32]], !noalias [[META29]]
+; COST10-NEXT: store i32 [[TMP24]], ptr [[TMP30]], align 4, !alias.scope [[META22]], !noalias [[META19]]
+; COST10-NEXT: store i32 [[TMP25]], ptr [[TMP31]], align 4, !alias.scope [[META22]], !noalias [[META19]]
+; COST10-NEXT: store i32 [[TMP22]], ptr [[TMP32]], align 4, !alias.scope [[META22]], !noalias [[META19]]
+; COST10-NEXT: store i32 [[TMP23]], ptr [[TMP33]], align 4, !alias.scope [[META22]], !noalias [[META19]]
; COST10-NEXT: [[TMP34:%.*]] = getelementptr i8, ptr [[NEXT_GEP]], i64 72
; COST10-NEXT: [[TMP35:%.*]] = getelementptr i8, ptr [[NEXT_GEP2]], i64 72
; COST10-NEXT: [[TMP36:%.*]] = getelementptr i8, ptr [[NEXT_GEP3]], i64 72
; COST10-NEXT: [[TMP37:%.*]] = getelementptr i8, ptr [[NEXT_GEP4]], i64 72
-; COST10-NEXT: store i32 [[TMP24]], ptr [[TMP34]], align 4, !alias.scope [[META32]], !noalias [[META29]]
-; COST10-NEXT: store i32 [[TMP25]], ptr [[TMP35]], align 4, !alias.scope [[META32]], !noalias [[META29]]
-; COST10-NEXT: store i32 [[TMP22]], ptr [[TMP36]], align 4, !alias.scope [[META32]], !noalias [[META29]]
-; COST10-NEXT: store i32 [[TMP23]], ptr [[TMP37]], align 4, !alias.scope [[META32]], !noalias [[META29]]
+; COST10-NEXT: store i32 [[TMP24]], ptr [[TMP34]], align 4, !alias.scope [[META22]], !noalias [[META19]]
+; COST10-NEXT: store i32 [[TMP25]], ptr [[TMP35]], align 4, !alias.scope [[META22]], !noalias [[META19]]
+; COST10-NEXT: store i32 [[TMP22]], ptr [[TMP36]], align 4, !alias.scope [[META22]], !noalias [[META19]]
+; COST10-NEXT: store i32 [[TMP23]], ptr [[TMP37]], align 4, !alias.scope [[META22]], !noalias [[META19]]
; COST10-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 4
; COST10-NEXT: [[TMP38:%.*]] = icmp eq i64 [[INDEX_NEXT]], 20
-; COST10-NEXT: br i1 [[TMP38]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP34:![0-9]+]]
+; COST10-NEXT: br i1 [[TMP38]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP24:![0-9]+]]
; COST10: [[MIDDLE_BLOCK]]:
; COST10-NEXT: br label %[[SCALAR_PH]]
; COST10: [[SCALAR_PH]]:
diff --git a/llvm/test/Transforms/LoopVectorize/AArch64/invariant-replicate-region.ll b/llvm/test/Transforms/LoopVectorize/AArch64/invariant-replicate-region.ll
index 78eaa586735d8..d05002391d357 100644
--- a/llvm/test/Transforms/LoopVectorize/AArch64/invariant-replicate-region.ll
+++ b/llvm/test/Transforms/LoopVectorize/AArch64/invariant-replicate-region.ll
@@ -11,53 +11,33 @@ define i32 @test_invariant_replicate_region(i32 %x, i1 %c) {
; CHECK-NEXT: [[ENTRY:.*:]]
; CHECK-NEXT: br label %[[VECTOR_PH:.*]]
; CHECK: [[VECTOR_PH]]:
-; CHECK-NEXT: [[BROADCAST_SPLATINSERT:%.*]] = insertelement <4 x i1> poison, i1 [[C]], i64 0
-; CHECK-NEXT: [[BROADCAST_SPLAT:%.*]] = shufflevector <4 x i1> [[BROADCAST_SPLATINSERT]], <4 x i1> poison, <4 x i32> zeroinitializer
; CHECK-NEXT: br label %[[VECTOR_BODY:.*]]
; CHECK: [[VECTOR_BODY]]:
-; CHECK-NEXT: [[INDEX:%.*]] = phi i32 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[PRED_UREM_CONTINUE:.*]] ]
-; CHECK-NEXT: br i1 [[C]], label %[[PRED_UREM_IF:.*]], label %[[PRED_UREM_CONTINUE]]
-; CHECK: [[PRED_UREM_IF]]:
-; CHECK-NEXT: br i1 [[C]], label %[[PRED_UREM_IF2:.*]], label %[[PRED_UREM_CONTINUE1:.*]]
+; CHECK-NEXT: [[INDEX:%.*]] = phi i32 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[PRED_UREM_CONTINUE1:.*]] ]
+; CHECK-NEXT: br i1 [[C]], label %[[PRED_UREM_IF2:.*]], label %[[PRED_UREM_CONTINUE1]]
; CHECK: [[PRED_UREM_IF2]]:
-; CHECK-NEXT: [[TMP1:%.*]] = urem i32 10, [[X]]
-; CHECK-NEXT: [[TMP3:%.*]] = insertelement <4 x i32> poison, i32 [[TMP1]], i64 0
; CHECK-NEXT: br label %[[PRED_UREM_CONTINUE1]]
; CHECK: [[PRED_UREM_CONTINUE1]]:
-; CHECK-NEXT: [[TMP2:%.*]] = phi <4 x i32> [ poison, %[[PRED_UREM_IF]] ], [ [[TMP3]], %[[PRED_UREM_IF2]] ]
-; CHECK-NEXT: br i1 [[C]], label %[[PRED_UREM_IF1:.*]], label %[[PRED_UREM_CONTINUE2:.*]]
+; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i32 [[INDEX]], 8
+; CHECK-NEXT: [[TMP0:%.*]] = icmp eq i32 [[INDEX_NEXT]], 96
+; CHECK-NEXT: br i1 [[TMP0]], label %[[PRED_UREM_IF1:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP0:![0-9]+]]
; CHECK: [[PRED_UREM_IF1]]:
-; CHECK-NEXT: [[TMP5:%.*]] = urem i32 10, [[X]]
-; CHECK-NEXT: [[TMP4:%.*]] = insertelement <4 x i32> [[TMP2]], i32 [[TMP5]], i64 1
-; CHECK-NEXT: br label %[[PRED_UREM_CONTINUE2]]
+; CHECK-NEXT: br label %[[PRED_UREM_CONTINUE2:.*]]
; CHECK: [[PRED_UREM_CONTINUE2]]:
-; CHECK-NEXT: [[TMP6:%.*]] = phi <4 x i32> [ [[TMP2]], %[[PRED_UREM_CONTINUE1]] ], [ [[TMP4]], %[[PRED_UREM_IF1]] ]
-; CHECK-NEXT: br i1 [[C]], label %[[PRED_UREM_IF3:.*]], label %[[PRED_UREM_CONTINUE4:.*]]
-; CHECK: [[PRED_UREM_IF3]]:
-; CHECK-NEXT: [[TMP9:%.*]] = urem i32 10, [[X]]
-; CHECK-NEXT: [[TMP7:%.*]] = insertelement <4 x i32> [[TMP6]], i32 [[TMP9]], i64 2
-; CHECK-NEXT: br label %[[PRED_UREM_CONTINUE4]]
+; CHECK-NEXT: br label %[[PRED_UREM_CONTINUE4:.*]]
; CHECK: [[PRED_UREM_CONTINUE4]]:
-; CHECK-NEXT: [[TMP8:%.*]] = phi <4 x i32> [ [[TMP6]], %[[PRED_UREM_CONTINUE2]] ], [ [[TMP7]], %[[PRED_UREM_IF3]] ]
-; CHECK-NEXT: br i1 [[C]], label %[[PRED_UREM_IF5:.*]], label %[[PRED_UREM_CONTINUE6:.*]]
+; CHECK-NEXT: [[IV:%.*]] = phi i32 [ 96, %[[PRED_UREM_CONTINUE2]] ], [ [[IV_NEXT:%.*]], %[[PRED_UREM_CONTINUE6:.*]] ]
+; CHECK-NEXT: br i1 [[C]], label %[[PRED_UREM_IF5:.*]], label %[[PRED_UREM_CONTINUE6]]
; CHECK: [[PRED_UREM_IF5]]:
; CHECK-NEXT: [[TMP13:%.*]] = urem i32 10, [[X]]
-; CHECK-NEXT: [[TMP10:%.*]] = insertelement <4 x i32> [[TMP8]], i32 [[TMP13]], i64 3
; CHECK-NEXT: br label %[[PRED_UREM_CONTINUE6]]
; CHECK: [[PRED_UREM_CONTINUE6]]:
-; CHECK-NEXT: [[TMP11:%.*]] = phi <4 x i32> [ [[TMP8]], %[[PRED_UREM_CONTINUE4]] ], [ [[TMP10]], %[[PRED_UREM_IF5]] ]
-; CHECK-NEXT: br label %[[PRED_UREM_CONTINUE]]
-; CHECK: [[PRED_UREM_CONTINUE]]:
-; CHECK-NEXT: [[TMP12:%.*]] = phi <4 x i32> [ poison, %[[VECTOR_BODY]] ], [ [[TMP11]], %[[PRED_UREM_CONTINUE6]] ]
-; CHECK-NEXT: [[TMP14:%.*]] = phi <4 x i1> [ zeroinitializer, %[[VECTOR_BODY]] ], [ [[BROADCAST_SPLAT]], %[[PRED_UREM_CONTINUE6]] ]
-; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i32 [[INDEX]], 4
-; CHECK-NEXT: [[TMP16:%.*]] = icmp eq i32 [[INDEX_NEXT]], 100
-; CHECK-NEXT: br i1 [[TMP16]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP0:![0-9]+]]
-; CHECK: [[MIDDLE_BLOCK]]:
-; CHECK-NEXT: [[PREDPHI:%.*]] = select <4 x i1> [[TMP14]], <4 x i32> [[TMP12]], <4 x i32> zeroinitializer
-; CHECK-NEXT: [[TMP17:%.*]] = extractelement <4 x i32> [[PREDPHI]], i64 3
-; CHECK-NEXT: br label %[[EXIT:.*]]
+; CHECK-NEXT: [[RES:%.*]] = phi i32 [ 0, %[[PRED_UREM_CONTINUE4]] ], [ [[TMP13]], %[[PRED_UREM_IF5]] ]
+; CHECK-NEXT: [[IV_NEXT]] = add i32 [[IV]], 1
+; CHECK-NEXT: [[EC:%.*]] = icmp eq i32 [[IV]], 99
+; CHECK-NEXT: br i1 [[EC]], label %[[EXIT:.*]], label %[[PRED_UREM_CONTINUE4]], !llvm.loop [[LOOP3:![0-9]+]]
; CHECK: [[EXIT]]:
+; CHECK-NEXT: [[TMP17:%.*]] = phi i32 [ [[RES]], %[[PRED_UREM_CONTINUE6]] ]
; CHECK-NEXT: ret i32 [[TMP17]]
;
entry:
@@ -84,4 +64,5 @@ exit:
; CHECK: [[LOOP0]] = distinct !{[[LOOP0]], [[META1:![0-9]+]], [[META2:![0-9]+]]}
; CHECK: [[META1]] = !{!"llvm.loop.isvectorized", i32 1}
; CHECK: [[META2]] = !{!"llvm.loop.unroll.runtime.disable"}
+; CHECK: [[LOOP3]] = distinct !{[[LOOP3]], [[META2]], [[META1]]}
;.
diff --git a/llvm/test/Transforms/LoopVectorize/AArch64/predication_costs.ll b/llvm/test/Transforms/LoopVectorize/AArch64/predication_costs.ll
index fdb22beb19695..e7b718dc0d483 100644
--- a/llvm/test/Transforms/LoopVectorize/AArch64/predication_costs.ll
+++ b/llvm/test/Transforms/LoopVectorize/AArch64/predication_costs.ll
@@ -62,7 +62,34 @@ for.end:
; CHECK: Scalarizing and predicating: store i32 %tmp2, ptr %tmp0, align 4
; CHECK: Cost of 4 for VF 2: profitable to scalarize store i32 %tmp2, ptr %tmp0, align 4
;
-define void @predicated_store(ptr %a, i1 %c, i32 %x, i64 %n) {
+define void @predicated_store(ptr %a, i32 %x, i64 %n) {
+entry:
+ br label %for.body
+
+for.body:
+ %i = phi i64 [ 0, %entry ], [ %i.next, %for.inc ]
+ %tmp0 = getelementptr inbounds i32, ptr %a, i64 %i
+ %tmp1 = load i32, ptr %tmp0, align 4
+ %tmp2 = add nsw i32 %tmp1, %x
+ %c = icmp eq i32 %tmp1, 42
+ br i1 %c, label %if.then, label %for.inc
+
+if.then:
+ store i32 %tmp2, ptr %tmp0, align 4
+ br label %for.inc
+
+for.inc:
+ %i.next = add nuw nsw i64 %i, 1
+ %cond = icmp slt i64 %i.next, %n
+ br i1 %cond, label %for.body, label %for.end
+
+for.end:
+ ret void
+}
+
+; CHECK-LABEL: store_in_preserved_uniform_control_flow
+; CHECK: Cost of 1 for VF 2: WIDEN store {{.*}}, ir<%tmp2>
+define void @store_in_preserved_uniform_control_flow(ptr %a, i1 %c, i32 %x, i64 %n) {
entry:
br label %for.body
@@ -96,7 +123,7 @@ for.end:
; CHECK: Cost of 0 for VF 2: induction instruction %addr = phi ptr [ %a, %entry ], [ %addr.next, %for.inc ]
; CHECK: Cost of 4 for VF 2: profitable to scalarize store i32 %tmp2, ptr %addr, align 4
;
-define void @predicated_store_phi(ptr %a, i1 %c, i32 %x, i64 %n) {
+define void @predicated_store_phi(ptr %a, i32 %x, i64 %n) {
entry:
br label %for.body
@@ -105,6 +132,7 @@ for.body:
%addr = phi ptr [ %a, %entry ], [ %addr.next, %for.inc ]
%tmp1 = load i32, ptr %addr, align 4
%tmp2 = add nsw i32 %tmp1, %x
+ %c = icmp eq i32 %tmp1, 42
br i1 %c, label %if.then, label %for.inc
if.then:
@@ -184,7 +212,7 @@ for.end:
; CHECK: Cost of 2 for VF 2: profitable to scalarize store i32 %tmp2, ptr %tmp0, align 4
; CHECK: Cost of 3 for VF 2: profitable to scalarize %tmp2 = add nsw i32 %tmp1, %x
;
-define void @predicated_store_scalarized_operand(ptr %a, i1 %c, i32 %x, i64 %n) {
+define void @predicated_store_scalarized_operand(ptr %a, i32 %x, i64 %n) {
entry:
br label %for.body
@@ -192,6 +220,7 @@ for.body:
%i = phi i64 [ 0, %entry ], [ %i.next, %for.inc ]
%tmp0 = getelementptr inbounds i32, ptr %a, i64 %i
%tmp1 = load i32, ptr %tmp0, align 4
+ %c = icmp eq i32 %tmp1, 42
br i1 %c, label %if.then, label %for.inc
if.then:
@@ -240,7 +269,7 @@ for.end:
; CHECK: Cost of 7 for VF 2: REPLICATE ir<%tmp3> = sdiv ir<%tmp1>, ir<%tmp2>
; CHECK: Cost of 5 for VF 2: REPLICATE ir<%tmp4> = udiv ir<%tmp3>, ir<%tmp2>
;
-define void @predication_multi_context(ptr %a, i1 %c, i32 %x, i64 %n) {
+define void @predication_multi_context(ptr %a, i32 %x, i64 %n) {
entry:
br label %for.body
@@ -248,6 +277,7 @@ for.body:
%i = phi i64 [ 0, %entry ], [ %i.next, %for.inc ]
%tmp0 = getelementptr inbounds i32, ptr %a, i64 %i
%tmp1 = load i32, ptr %tmp0, align 4
+ %c = icmp eq i32 %tmp1, 42
br i1 %c, label %if.then, label %for.inc
if.then:
diff --git a/llvm/test/Transforms/LoopVectorize/AArch64/sve-predicated-costs.ll b/llvm/test/Transforms/LoopVectorize/AArch64/sve-predicated-costs.ll
index 04d11a486d085..fe5b4ace854aa 100644
--- a/llvm/test/Transforms/LoopVectorize/AArch64/sve-predicated-costs.ll
+++ b/llvm/test/Transforms/LoopVectorize/AArch64/sve-predicated-costs.ll
@@ -78,43 +78,33 @@ define void @always_taken(ptr noalias %p0, ptr noalias %p1, i1 %c0, i1 %c1, i1 %
; CHECK-NEXT: [[TMP3:%.*]] = shl nuw i32 [[TMP0]], 1
; CHECK-NEXT: [[N_MOD_VF:%.*]] = urem i32 1024, [[TMP1]]
; CHECK-NEXT: [[N_VEC:%.*]] = sub i32 1024, [[N_MOD_VF]]
-; CHECK-NEXT: [[BROADCAST_SPLATINSERT2:%.*]] = insertelement <vscale x 2 x i1> poison, i1 [[C2]], i64 0
-; CHECK-NEXT: [[BROADCAST_SPLAT1:%.*]] = shufflevector <vscale x 2 x i1> [[BROADCAST_SPLATINSERT2]], <vscale x 2 x i1> poison, <vscale x 2 x i32> zeroinitializer
-; CHECK-NEXT: [[BROADCAST_SPLATINSERT:%.*]] = insertelement <vscale x 2 x i1> poison, i1 [[C1]], i64 0
-; CHECK-NEXT: [[BROADCAST_SPLAT:%.*]] = shufflevector <vscale x 2 x i1> [[BROADCAST_SPLATINSERT]], <vscale x 2 x i1> poison, <vscale x 2 x i32> zeroinitializer
-; CHECK-NEXT: [[BROADCAST_SPLATINSERT1:%.*]] = insertelement <vscale x 2 x i1> poison, i1 [[C0]], i64 0
-; CHECK-NEXT: [[BROADCAST_SPLAT2:%.*]] = shufflevector <vscale x 2 x i1> [[BROADCAST_SPLATINSERT1]], <vscale x 2 x i1> poison, <vscale x 2 x i32> zeroinitializer
-; CHECK-NEXT: [[TMP4:%.*]] = freeze i1 [[C1]]
-; CHECK-NEXT: [[TMP6:%.*]] = select <vscale x 2 x i1> [[BROADCAST_SPLAT2]], <vscale x 2 x i1> [[BROADCAST_SPLAT]], <vscale x 2 x i1> zeroinitializer
-; CHECK-NEXT: [[TMP5:%.*]] = freeze i1 [[C2]]
-; CHECK-NEXT: [[TMP8:%.*]] = select <vscale x 2 x i1> [[TMP6]], <vscale x 2 x i1> [[BROADCAST_SPLAT1]], <vscale x 2 x i1> zeroinitializer
; CHECK-NEXT: br label %[[VECTOR_BODY:.*]]
; CHECK: [[VECTOR_BODY]]:
; CHECK-NEXT: [[INDEX:%.*]] = phi i32 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[LATCH9:.*]] ]
-; CHECK-NEXT: br i1 [[C0]], label %[[THEN_05:.*]], label %[[LATCH9]]
+; CHECK-NEXT: br i1 [[C0]], label %[[THEN_05:.*]], label %[[LATCH9]], !prof [[PROF0:![0-9]+]]
; CHECK: [[THEN_05]]:
-; CHECK-NEXT: br i1 [[TMP4]], label %[[THEN_16:.*]], label %[[LATCH9]]
+; CHECK-NEXT: br i1 [[C1]], label %[[THEN_16:.*]], label %[[LATCH9]], !prof [[PROF0]]
; CHECK: [[THEN_16]]:
-; CHECK-NEXT: br i1 [[TMP5]], label %[[THEN_27:.*]], label %[[LATCH9]]
+; CHECK-NEXT: br i1 [[C2]], label %[[THEN_27:.*]], label %[[LATCH9]], !prof [[PROF0]]
; CHECK: [[THEN_27]]:
; CHECK-NEXT: [[TMP10:%.*]] = getelementptr i64, ptr [[P0]], i32 [[INDEX]]
; CHECK-NEXT: [[TMP7:%.*]] = zext i32 [[TMP3]] to i64
; CHECK-NEXT: [[TMP20:%.*]] = getelementptr i64, ptr [[TMP10]], i64 [[TMP7]]
-; CHECK-NEXT: [[WIDE_MASKED_LOAD:%.*]] = call <vscale x 2 x i64> @llvm.masked.load.nxv2i64.p0(ptr align 8 [[TMP10]], <vscale x 2 x i1> [[TMP8]], <vscale x 2 x i64> poison)
-; CHECK-NEXT: [[WIDE_MASKED_LOAD5:%.*]] = call <vscale x 2 x i64> @llvm.masked.load.nxv2i64.p0(ptr align 8 [[TMP20]], <vscale x 2 x i1> [[TMP8]], <vscale x 2 x i64> poison)
+; CHECK-NEXT: [[WIDE_LOAD:%.*]] = load <vscale x 2 x i64>, ptr [[TMP10]], align 8
+; CHECK-NEXT: [[WIDE_LOAD4:%.*]] = load <vscale x 2 x i64>, ptr [[TMP20]], align 8
; CHECK-NEXT: [[TMP9:%.*]] = getelementptr i64, ptr [[P1]], i32 [[INDEX]]
; CHECK-NEXT: [[TMP12:%.*]] = getelementptr i64, ptr [[TMP9]], i64 [[TMP7]]
-; CHECK-NEXT: [[WIDE_MASKED_LOAD6:%.*]] = call <vscale x 2 x i64> @llvm.masked.load.nxv2i64.p0(ptr align 8 [[TMP9]], <vscale x 2 x i1> [[TMP8]], <vscale x 2 x i64> poison)
-; CHECK-NEXT: [[WIDE_MASKED_LOAD7:%.*]] = call <vscale x 2 x i64> @llvm.masked.load.nxv2i64.p0(ptr align 8 [[TMP12]], <vscale x 2 x i1> [[TMP8]], <vscale x 2 x i64> poison)
-; CHECK-NEXT: [[TMP14:%.*]] = call <vscale x 2 x i64> @llvm.masked.udiv.nxv2i64(<vscale x 2 x i64> [[WIDE_MASKED_LOAD]], <vscale x 2 x i64> [[WIDE_MASKED_LOAD6]], <vscale x 2 x i1> [[TMP8]])
-; CHECK-NEXT: [[TMP13:%.*]] = call <vscale x 2 x i64> @llvm.masked.udiv.nxv2i64(<vscale x 2 x i64> [[WIDE_MASKED_LOAD5]], <vscale x 2 x i64> [[WIDE_MASKED_LOAD7]], <vscale x 2 x i1> [[TMP8]])
-; CHECK-NEXT: call void @llvm.masked.store.nxv2i64.p0(<vscale x 2 x i64> [[TMP14]], ptr align 8 [[TMP9]], <vscale x 2 x i1> [[TMP8]])
-; CHECK-NEXT: call void @llvm.masked.store.nxv2i64.p0(<vscale x 2 x i64> [[TMP13]], ptr align 8 [[TMP12]], <vscale x 2 x i1> [[TMP8]])
+; CHECK-NEXT: [[WIDE_LOAD5:%.*]] = load <vscale x 2 x i64>, ptr [[TMP9]], align 8
+; CHECK-NEXT: [[WIDE_LOAD6:%.*]] = load <vscale x 2 x i64>, ptr [[TMP12]], align 8
+; CHECK-NEXT: [[TMP8:%.*]] = udiv <vscale x 2 x i64> [[WIDE_LOAD]], [[WIDE_LOAD5]]
+; CHECK-NEXT: [[TMP11:%.*]] = udiv <vscale x 2 x i64> [[WIDE_LOAD4]], [[WIDE_LOAD6]]
+; CHECK-NEXT: store <vscale x 2 x i64> [[TMP8]], ptr [[TMP9]], align 8
+; CHECK-NEXT: store <vscale x 2 x i64> [[TMP11]], ptr [[TMP12]], align 8
; CHECK-NEXT: br label %[[LATCH9]]
; CHECK: [[LATCH9]]:
; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i32 [[INDEX]], [[TMP1]]
; CHECK-NEXT: [[TMP16:%.*]] = icmp eq i32 [[INDEX_NEXT]], [[N_VEC]]
-; CHECK-NEXT: br i1 [[TMP16]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP0:![0-9]+]]
+; CHECK-NEXT: br i1 [[TMP16]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP1:![0-9]+]]
; CHECK: [[MIDDLE_BLOCK]]:
; CHECK-NEXT: [[CMP_N:%.*]] = icmp eq i32 1024, [[N_VEC]]
; CHECK-NEXT: br i1 [[CMP_N]], label %[[EXIT:.*]], label %[[SCALAR_PH]]
@@ -123,11 +113,11 @@ define void @always_taken(ptr noalias %p0, ptr noalias %p1, i1 %c0, i1 %c1, i1 %
; CHECK-NEXT: br label %[[LOOP:.*]]
; CHECK: [[LOOP]]:
; CHECK-NEXT: [[IV1:%.*]] = phi i32 [ [[BC_RESUME_VAL]], %[[SCALAR_PH]] ], [ [[IV_NEXT1:%.*]], %[[LATCH:.*]] ]
-; CHECK-NEXT: br i1 [[C0]], label %[[THEN_0:.*]], label %[[LATCH]], !prof [[PROF3:![0-9]+]]
+; CHECK-NEXT: br i1 [[C0]], label %[[THEN_0:.*]], label %[[LATCH]], !prof [[PROF0]]
; CHECK: [[THEN_0]]:
-; CHECK-NEXT: br i1 [[C1]], label %[[THEN_1:.*]], label %[[LATCH]], !prof [[PROF3]]
+; CHECK-NEXT: br i1 [[C1]], label %[[THEN_1:.*]], label %[[LATCH]], !prof [[PROF0]]
; CHECK: [[THEN_1]]:
-; CHECK-NEXT: br i1 [[C2]], label %[[THEN_2:.*]], label %[[LATCH]], !prof [[PROF3]]
+; CHECK-NEXT: br i1 [[C2]], label %[[THEN_2:.*]], label %[[LATCH]], !prof [[PROF0]]
; CHECK: [[THEN_2]]:
; CHECK-NEXT: [[GEP0:%.*]] = getelementptr i64, ptr [[P0]], i32 [[IV1]]
; CHECK-NEXT: [[X:%.*]] = load i64, ptr [[GEP0]], align 8
diff --git a/llvm/test/Transforms/LoopVectorize/AArch64/transform-narrow-interleave-to-widen-memory-multi-block.ll b/llvm/test/Transforms/LoopVectorize/AArch64/transform-narrow-interleave-to-widen-memory-multi-block.ll
index 9efff93543685..3d69698888f16 100644
--- a/llvm/test/Transforms/LoopVectorize/AArch64/transform-narrow-interleave-to-widen-memory-multi-block.ll
+++ b/llvm/test/Transforms/LoopVectorize/AArch64/transform-narrow-interleave-to-widen-memory-multi-block.ll
@@ -13,7 +13,7 @@ define void @load_store_interleave_group_block_invar_cond(ptr noalias %data, ptr
; VF2IC1: [[VECTOR_PH]]:
; VF2IC1-NEXT: br label %[[VECTOR_BODY:.*]]
; VF2IC1: [[VECTOR_BODY]]:
-; VF2IC1-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[PRED_STORE_CONTINUE:.*]] ]
+; VF2IC1-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[PRED_STORE_CONTINUE3:.*]] ]
; VF2IC1-NEXT: [[TMP0:%.*]] = shl nsw i64 [[INDEX]], 1
; VF2IC1-NEXT: [[TMP1:%.*]] = getelementptr inbounds i64, ptr [[DATA]], i64 [[TMP0]]
; VF2IC1-NEXT: [[WIDE_VEC:%.*]] = load <4 x i64>, ptr [[TMP1]], align 8
@@ -22,20 +22,11 @@ define void @load_store_interleave_group_block_invar_cond(ptr noalias %data, ptr
; VF2IC1-NEXT: [[TMP4:%.*]] = shufflevector <2 x i64> [[STRIDED_VEC]], <2 x i64> [[STRIDED_VEC1]], <4 x i32> <i32 0, i32 1, i32 2, i32 3>
; VF2IC1-NEXT: [[INTERLEAVED_VEC:%.*]] = shufflevector <4 x i64> [[TMP4]], <4 x i64> poison, <4 x i32> <i32 0, i32 2, i32 1, i32 3>
; VF2IC1-NEXT: store <4 x i64> [[INTERLEAVED_VEC]], ptr [[TMP1]], align 8
-; VF2IC1-NEXT: br i1 [[C]], label %[[PRED_STORE_IF:.*]], label %[[PRED_STORE_CONTINUE]]
-; VF2IC1: [[PRED_STORE_IF]]:
-; VF2IC1-NEXT: br i1 [[C]], label %[[PRED_STORE_IF1:.*]], label %[[PRED_STORE_CONTINUE1:.*]]
-; VF2IC1: [[PRED_STORE_IF1]]:
-; VF2IC1-NEXT: store i8 1, ptr [[DST_0]], align 1
-; VF2IC1-NEXT: br label %[[PRED_STORE_CONTINUE1]]
-; VF2IC1: [[PRED_STORE_CONTINUE1]]:
-; VF2IC1-NEXT: br i1 [[C]], label %[[PRED_STORE_IF2:.*]], label %[[PRED_STORE_CONTINUE3:.*]]
+; VF2IC1-NEXT: br i1 [[C]], label %[[PRED_STORE_IF2:.*]], label %[[PRED_STORE_CONTINUE3]]
; VF2IC1: [[PRED_STORE_IF2]]:
; VF2IC1-NEXT: store i8 1, ptr [[DST_0]], align 1
; VF2IC1-NEXT: br label %[[PRED_STORE_CONTINUE3]]
; VF2IC1: [[PRED_STORE_CONTINUE3]]:
-; VF2IC1-NEXT: br label %[[PRED_STORE_CONTINUE]]
-; VF2IC1: [[PRED_STORE_CONTINUE]]:
; VF2IC1-NEXT: [[TMP2:%.*]] = getelementptr inbounds i8, ptr [[DST_1]], i64 [[INDEX]]
; VF2IC1-NEXT: store <2 x i8> zeroinitializer, ptr [[TMP2]], align 1
; VF2IC1-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 2
@@ -53,7 +44,7 @@ define void @load_store_interleave_group_block_invar_cond(ptr noalias %data, ptr
; VF2IC2: [[VECTOR_PH]]:
; VF2IC2-NEXT: br label %[[VECTOR_BODY:.*]]
; VF2IC2: [[VECTOR_BODY]]:
-; VF2IC2-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[PRED_STORE_CONTINUE:.*]] ]
+; VF2IC2-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[PRED_STORE_CONTINUE11:.*]] ]
; VF2IC2-NEXT: [[TMP0:%.*]] = add i64 [[INDEX]], 2
; VF2IC2-NEXT: [[TMP1:%.*]] = shl nsw i64 [[INDEX]], 1
; VF2IC2-NEXT: [[TMP2:%.*]] = shl nsw i64 [[TMP0]], 1
@@ -71,30 +62,11 @@ define void @load_store_interleave_group_block_invar_cond(ptr noalias %data, ptr
; VF2IC2-NEXT: [[TMP9:%.*]] = shufflevector <2 x i64> [[STRIDED_VEC3]], <2 x i64> [[STRIDED_VEC4]], <4 x i32> <i32 0, i32 1, i32 2, i32 3>
; VF2IC2-NEXT: [[INTERLEAVED_VEC5:%.*]] = shufflevector <4 x i64> [[TMP9]], <4 x i64> poison, <4 x i32> <i32 0, i32 2, i32 1, i32 3>
; VF2IC2-NEXT: store <4 x i64> [[INTERLEAVED_VEC5]], ptr [[TMP4]], align 8
-; VF2IC2-NEXT: br i1 [[C]], label %[[PRED_STORE_IF:.*]], label %[[PRED_STORE_CONTINUE]]
-; VF2IC2: [[PRED_STORE_IF]]:
-; VF2IC2-NEXT: br i1 [[C]], label %[[PRED_STORE_IF1:.*]], label %[[PRED_STORE_CONTINUE1:.*]]
-; VF2IC2: [[PRED_STORE_IF1]]:
-; VF2IC2-NEXT: store i8 1, ptr [[DST_0]], align 1
-; VF2IC2-NEXT: br label %[[PRED_STORE_CONTINUE1]]
-; VF2IC2: [[PRED_STORE_CONTINUE1]]:
-; VF2IC2-NEXT: br i1 [[C]], label %[[PRED_STORE_IF6:.*]], label %[[PRED_STORE_CONTINUE7:.*]]
-; VF2IC2: [[PRED_STORE_IF6]]:
-; VF2IC2-NEXT: store i8 1, ptr [[DST_0]], align 1
-; VF2IC2-NEXT: br label %[[PRED_STORE_CONTINUE7]]
-; VF2IC2: [[PRED_STORE_CONTINUE7]]:
-; VF2IC2-NEXT: br i1 [[C]], label %[[PRED_STORE_IF8:.*]], label %[[PRED_STORE_CONTINUE9:.*]]
-; VF2IC2: [[PRED_STORE_IF8]]:
-; VF2IC2-NEXT: store i8 1, ptr [[DST_0]], align 1
-; VF2IC2-NEXT: br label %[[PRED_STORE_CONTINUE9]]
-; VF2IC2: [[PRED_STORE_CONTINUE9]]:
-; VF2IC2-NEXT: br i1 [[C]], label %[[PRED_STORE_IF10:.*]], label %[[PRED_STORE_CONTINUE11:.*]]
+; VF2IC2-NEXT: br i1 [[C]], label %[[PRED_STORE_IF10:.*]], label %[[PRED_STORE_CONTINUE11]]
; VF2IC2: [[PRED_STORE_IF10]]:
; VF2IC2-NEXT: store i8 1, ptr [[DST_0]], align 1
; VF2IC2-NEXT: br label %[[PRED_STORE_CONTINUE11]]
; VF2IC2: [[PRED_STORE_CONTINUE11]]:
-; VF2IC2-NEXT: br label %[[PRED_STORE_CONTINUE]]
-; VF2IC2: [[PRED_STORE_CONTINUE]]:
; VF2IC2-NEXT: [[TMP5:%.*]] = getelementptr inbounds i8, ptr [[DST_1]], i64 [[INDEX]]
; VF2IC2-NEXT: [[TMP6:%.*]] = getelementptr inbounds i8, ptr [[TMP5]], i64 2
; VF2IC2-NEXT: store <2 x i8> zeroinitializer, ptr [[TMP5]], align 1
diff --git a/llvm/test/Transforms/LoopVectorize/RISCV/dead-ops-cost.ll b/llvm/test/Transforms/LoopVectorize/RISCV/dead-ops-cost.ll
index d0ee8e0075bfb..a112eed35ede3 100644
--- a/llvm/test/Transforms/LoopVectorize/RISCV/dead-ops-cost.ll
+++ b/llvm/test/Transforms/LoopVectorize/RISCV/dead-ops-cost.ll
@@ -130,7 +130,7 @@ exit:
; Test case for https://github.com/llvm/llvm-project/issues/106780.
define i32 @cost_of_exit_branch_and_cond_insts(ptr %a, ptr %b, i1 %c, i16 %x) vscale_range(2, 1024) {
; CHECK-LABEL: define i32 @cost_of_exit_branch_and_cond_insts(
-; CHECK-SAME: ptr [[A:%.*]], ptr [[B:%.*]], i1 [[C:%.*]], i16 [[X:%.*]]) #[[ATTR1:[0-9]+]] {
+; CHECK-SAME: ptr [[A:%.*]], ptr [[B:%.*]], i1 [[C:%.*]], i16 [[X:%.*]]) #[[ATTR0]] {
; CHECK-NEXT: [[ENTRY:.*]]:
; CHECK-NEXT: [[TMP0:%.*]] = zext i16 [[X]] to i32
; CHECK-NEXT: [[UMAX3:%.*]] = call i32 @llvm.umax.i32(i32 [[TMP0]], i32 111)
@@ -158,60 +158,19 @@ define i32 @cost_of_exit_branch_and_cond_insts(ptr %a, ptr %b, i1 %c, i16 %x) vs
; CHECK-NEXT: [[TMP8:%.*]] = icmp eq i32 [[N_MOD_VF]], 0
; CHECK-NEXT: [[TMP9:%.*]] = select i1 [[TMP8]], i32 8, i32 [[N_MOD_VF]]
; CHECK-NEXT: [[N_VEC:%.*]] = sub i32 [[TMP2]], [[TMP9]]
-; CHECK-NEXT: [[BROADCAST_SPLATINSERT:%.*]] = insertelement <8 x i1> poison, i1 [[C]], i64 0
-; CHECK-NEXT: [[BROADCAST_SPLAT:%.*]] = shufflevector <8 x i1> [[BROADCAST_SPLATINSERT]], <8 x i1> poison, <8 x i32> zeroinitializer
-; CHECK-NEXT: br label %[[VECTOR_BODY:.*]]
-; CHECK: [[VECTOR_BODY]]:
-; CHECK-NEXT: [[INDEX:%.*]] = phi i32 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[LOOP_EXITING18:.*]] ]
-; CHECK-NEXT: br i1 [[C]], label %[[THEN3:.*]], label %[[LOOP_EXITING18]]
-; CHECK: [[THEN3]]:
-; CHECK-NEXT: [[TMP11:%.*]] = getelementptr i32, ptr [[B]], i32 [[INDEX]]
-; CHECK-NEXT: br i1 [[C]], label %[[PRED_STORE_IF:.*]], label %[[PRED_STORE_CONTINUE:.*]]
-; CHECK: [[PRED_STORE_IF]]:
-; CHECK-NEXT: store i1 false, ptr [[A]], align 1, !alias.scope [[META10:![0-9]+]], !noalias [[META13:![0-9]+]]
-; CHECK-NEXT: br label %[[PRED_STORE_CONTINUE]]
-; CHECK: [[PRED_STORE_CONTINUE]]:
-; CHECK-NEXT: br i1 [[C]], label %[[PRED_STORE_IF5:.*]], label %[[PRED_STORE_CONTINUE6:.*]]
-; CHECK: [[PRED_STORE_IF5]]:
-; CHECK-NEXT: store i1 false, ptr [[A]], align 1, !alias.scope [[META10]], !noalias [[META13]]
-; CHECK-NEXT: br label %[[PRED_STORE_CONTINUE6]]
-; CHECK: [[PRED_STORE_CONTINUE6]]:
-; CHECK-NEXT: br i1 [[C]], label %[[PRED_STORE_IF7:.*]], label %[[PRED_STORE_CONTINUE8:.*]]
-; CHECK: [[PRED_STORE_IF7]]:
-; CHECK-NEXT: store i1 false, ptr [[A]], align 1, !alias.scope [[META10]], !noalias [[META13]]
-; CHECK-NEXT: br label %[[PRED_STORE_CONTINUE8]]
-; CHECK: [[PRED_STORE_CONTINUE8]]:
-; CHECK-NEXT: br i1 [[C]], label %[[PRED_STORE_IF9:.*]], label %[[PRED_STORE_CONTINUE10:.*]]
-; CHECK: [[PRED_STORE_IF9]]:
-; CHECK-NEXT: store i1 false, ptr [[A]], align 1, !alias.scope [[META10]], !noalias [[META13]]
-; CHECK-NEXT: br label %[[PRED_STORE_CONTINUE10]]
-; CHECK: [[PRED_STORE_CONTINUE10]]:
-; CHECK-NEXT: br i1 [[C]], label %[[PRED_STORE_IF11:.*]], label %[[PRED_STORE_CONTINUE12:.*]]
-; CHECK: [[PRED_STORE_IF11]]:
-; CHECK-NEXT: store i1 false, ptr [[A]], align 1, !alias.scope [[META10]], !noalias [[META13]]
-; CHECK-NEXT: br label %[[PRED_STORE_CONTINUE12]]
-; CHECK: [[PRED_STORE_CONTINUE12]]:
-; CHECK-NEXT: br i1 [[C]], label %[[PRED_STORE_IF13:.*]], label %[[PRED_STORE_CONTINUE14:.*]]
-; CHECK: [[PRED_STORE_IF13]]:
-; CHECK-NEXT: store i1 false, ptr [[A]], align 1, !alias.scope [[META10]], !noalias [[META13]]
-; CHECK-NEXT: br label %[[PRED_STORE_CONTINUE14]]
-; CHECK: [[PRED_STORE_CONTINUE14]]:
-; CHECK-NEXT: br i1 [[C]], label %[[PRED_STORE_IF15:.*]], label %[[PRED_STORE_CONTINUE16:.*]]
-; CHECK: [[PRED_STORE_IF15]]:
-; CHECK-NEXT: store i1 false, ptr [[A]], align 1, !alias.scope [[META10]], !noalias [[META13]]
-; CHECK-NEXT: br label %[[PRED_STORE_CONTINUE16]]
+; CHECK-NEXT: br label %[[PRED_STORE_CONTINUE16:.*]]
; CHECK: [[PRED_STORE_CONTINUE16]]:
-; CHECK-NEXT: br i1 [[C]], label %[[PRED_STORE_IF17:.*]], label %[[PRED_STORE_CONTINUE18:.*]]
+; CHECK-NEXT: [[INDEX:%.*]] = phi i32 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[PRED_STORE_CONTINUE18:.*]] ]
+; CHECK-NEXT: br i1 [[C]], label %[[PRED_STORE_IF17:.*]], label %[[PRED_STORE_CONTINUE18]]
; CHECK: [[PRED_STORE_IF17]]:
-; CHECK-NEXT: store i1 false, ptr [[A]], align 1, !alias.scope [[META10]], !noalias [[META13]]
+; CHECK-NEXT: [[TMP13:%.*]] = getelementptr inbounds i32, ptr [[B]], i32 [[INDEX]]
+; CHECK-NEXT: store i1 false, ptr [[A]], align 1, !alias.scope [[META10:![0-9]+]], !noalias [[META13:![0-9]+]]
+; CHECK-NEXT: store <8 x i32> zeroinitializer, ptr [[TMP13]], align 4, !alias.scope [[META13]]
; CHECK-NEXT: br label %[[PRED_STORE_CONTINUE18]]
; CHECK: [[PRED_STORE_CONTINUE18]]:
-; CHECK-NEXT: call void @llvm.masked.store.v8i32.p0(<8 x i32> zeroinitializer, ptr align 4 [[TMP11]], <8 x i1> [[BROADCAST_SPLAT]]), !alias.scope [[META13]]
-; CHECK-NEXT: br label %[[LOOP_EXITING18]]
-; CHECK: [[LOOP_EXITING18]]:
; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i32 [[INDEX]], 8
; CHECK-NEXT: [[TMP21:%.*]] = icmp eq i32 [[INDEX_NEXT]], [[N_VEC]]
-; CHECK-NEXT: br i1 [[TMP21]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP15:![0-9]+]]
+; CHECK-NEXT: br i1 [[TMP21]], label %[[MIDDLE_BLOCK:.*]], label %[[PRED_STORE_CONTINUE16]], !llvm.loop [[LOOP15:![0-9]+]]
; CHECK: [[MIDDLE_BLOCK]]:
; CHECK-NEXT: br label %[[SCALAR_PH]]
; CHECK: [[SCALAR_PH]]:
@@ -273,7 +232,7 @@ return:
; Test case for https://github.com/llvm/llvm-project/issues/107473.
define void @test_phi_in_latch_redundant(ptr %dst, i32 %a) vscale_range(2, 1024) {
; CHECK-LABEL: define void @test_phi_in_latch_redundant(
-; CHECK-SAME: ptr [[DST:%.*]], i32 [[A:%.*]]) #[[ATTR1]] {
+; CHECK-SAME: ptr [[DST:%.*]], i32 [[A:%.*]]) #[[ATTR0]] {
; CHECK-NEXT: [[ENTRY:.*:]]
; CHECK-NEXT: br label %[[VECTOR_PH:.*]]
; CHECK: [[VECTOR_PH]]:
diff --git a/llvm/test/Transforms/LoopVectorize/RISCV/divrem.ll b/llvm/test/Transforms/LoopVectorize/RISCV/divrem.ll
index f15d94c6fa276..287c4b6069910 100644
--- a/llvm/test/Transforms/LoopVectorize/RISCV/divrem.ll
+++ b/llvm/test/Transforms/LoopVectorize/RISCV/divrem.ll
@@ -296,22 +296,21 @@ define void @predicated_udiv(ptr noalias nocapture %a, i64 %v, i64 %n) vscale_ra
; FIXED-NEXT: entry:
; FIXED-NEXT: br label [[VECTOR_PH:%.*]]
; FIXED: vector.ph:
-; FIXED-NEXT: [[BROADCAST_SPLATINSERT:%.*]] = insertelement <4 x i64> poison, i64 [[V:%.*]], i64 0
+; FIXED-NEXT: [[TMP6:%.*]] = icmp ne i64 [[V:%.*]], 0
+; FIXED-NEXT: [[BROADCAST_SPLATINSERT:%.*]] = insertelement <4 x i64> poison, i64 [[V]], i64 0
; FIXED-NEXT: [[BROADCAST_SPLAT:%.*]] = shufflevector <4 x i64> [[BROADCAST_SPLATINSERT]], <4 x i64> poison, <4 x i32> zeroinitializer
-; FIXED-NEXT: [[TMP0:%.*]] = icmp ne <4 x i64> [[BROADCAST_SPLAT]], zeroinitializer
; FIXED-NEXT: br label [[VECTOR_BODY:%.*]]
; FIXED: vector.body:
; FIXED-NEXT: [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[LATCH2:%.*]] ]
; FIXED-NEXT: [[TMP2:%.*]] = getelementptr inbounds i64, ptr [[A:%.*]], i64 [[INDEX]]
; FIXED-NEXT: [[WIDE_LOAD1:%.*]] = load <4 x i64>, ptr [[TMP2]], align 8
-; FIXED-NEXT: [[TMP6:%.*]] = extractelement <4 x i1> [[TMP0]], i64 0
; FIXED-NEXT: br i1 [[TMP6]], label [[DO_OP1:%.*]], label [[LATCH2]]
; FIXED: do_op1:
-; FIXED-NEXT: [[TMP3:%.*]] = call <4 x i64> @llvm.masked.udiv.v4i64(<4 x i64> [[WIDE_LOAD1]], <4 x i64> [[BROADCAST_SPLAT]], <4 x i1> [[TMP0]])
+; FIXED-NEXT: [[TMP3:%.*]] = udiv <4 x i64> [[WIDE_LOAD1]], [[BROADCAST_SPLAT]]
; FIXED-NEXT: br label [[LATCH2]]
; FIXED: latch2:
; FIXED-NEXT: [[TMP4:%.*]] = phi <4 x i64> [ poison, [[VECTOR_BODY]] ], [ [[TMP3]], [[DO_OP1]] ]
-; FIXED-NEXT: [[TMP5:%.*]] = phi <4 x i1> [ zeroinitializer, [[VECTOR_BODY]] ], [ [[TMP0]], [[DO_OP1]] ]
+; FIXED-NEXT: [[TMP5:%.*]] = phi <4 x i1> [ zeroinitializer, [[VECTOR_BODY]] ], [ splat (i1 true), [[DO_OP1]] ]
; FIXED-NEXT: [[PREDPHI2:%.*]] = select <4 x i1> [[TMP5]], <4 x i64> [[TMP4]], <4 x i64> [[WIDE_LOAD1]]
; FIXED-NEXT: store <4 x i64> [[PREDPHI2]], ptr [[TMP2]], align 8
; FIXED-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 4
@@ -378,22 +377,21 @@ define void @predicated_sdiv(ptr noalias nocapture %a, i64 %v, i64 %n) vscale_ra
; FIXED-NEXT: entry:
; FIXED-NEXT: br label [[VECTOR_PH:%.*]]
; FIXED: vector.ph:
-; FIXED-NEXT: [[BROADCAST_SPLATINSERT:%.*]] = insertelement <4 x i64> poison, i64 [[V:%.*]], i64 0
+; FIXED-NEXT: [[TMP6:%.*]] = icmp ne i64 [[V:%.*]], 0
+; FIXED-NEXT: [[BROADCAST_SPLATINSERT:%.*]] = insertelement <4 x i64> poison, i64 [[V]], i64 0
; FIXED-NEXT: [[BROADCAST_SPLAT:%.*]] = shufflevector <4 x i64> [[BROADCAST_SPLATINSERT]], <4 x i64> poison, <4 x i32> zeroinitializer
-; FIXED-NEXT: [[TMP0:%.*]] = icmp ne <4 x i64> [[BROADCAST_SPLAT]], zeroinitializer
; FIXED-NEXT: br label [[VECTOR_BODY:%.*]]
; FIXED: vector.body:
; FIXED-NEXT: [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[LATCH2:%.*]] ]
; FIXED-NEXT: [[TMP2:%.*]] = getelementptr inbounds i64, ptr [[A:%.*]], i64 [[INDEX]]
; FIXED-NEXT: [[WIDE_LOAD1:%.*]] = load <4 x i64>, ptr [[TMP2]], align 8
-; FIXED-NEXT: [[TMP6:%.*]] = extractelement <4 x i1> [[TMP0]], i64 0
; FIXED-NEXT: br i1 [[TMP6]], label [[DO_OP1:%.*]], label [[LATCH2]]
; FIXED: do_op1:
-; FIXED-NEXT: [[TMP3:%.*]] = call <4 x i64> @llvm.masked.sdiv.v4i64(<4 x i64> [[WIDE_LOAD1]], <4 x i64> [[BROADCAST_SPLAT]], <4 x i1> [[TMP0]])
+; FIXED-NEXT: [[TMP3:%.*]] = sdiv <4 x i64> [[WIDE_LOAD1]], [[BROADCAST_SPLAT]]
; FIXED-NEXT: br label [[LATCH2]]
; FIXED: latch2:
; FIXED-NEXT: [[TMP4:%.*]] = phi <4 x i64> [ poison, [[VECTOR_BODY]] ], [ [[TMP3]], [[DO_OP1]] ]
-; FIXED-NEXT: [[TMP5:%.*]] = phi <4 x i1> [ zeroinitializer, [[VECTOR_BODY]] ], [ [[TMP0]], [[DO_OP1]] ]
+; FIXED-NEXT: [[TMP5:%.*]] = phi <4 x i1> [ zeroinitializer, [[VECTOR_BODY]] ], [ splat (i1 true), [[DO_OP1]] ]
; FIXED-NEXT: [[PREDPHI2:%.*]] = select <4 x i1> [[TMP5]], <4 x i64> [[TMP4]], <4 x i64> [[WIDE_LOAD1]]
; FIXED-NEXT: store <4 x i64> [[PREDPHI2]], ptr [[TMP2]], align 8
; FIXED-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 4
@@ -645,9 +643,6 @@ define void @udiv_sdiv_with_invariant_divisors(i8 %x, i16 %y, i1 %c, ptr %p) vsc
; CHECK-NEXT: entry:
; CHECK-NEXT: br label [[VECTOR_BODY:%.*]]
; CHECK: vector.ph:
-; CHECK-NEXT: [[BROADCAST_SPLATINSERT:%.*]] = insertelement <vscale x 4 x i1> poison, i1 [[C:%.*]], i64 0
-; CHECK-NEXT: [[BROADCAST_SPLAT:%.*]] = shufflevector <vscale x 4 x i1> [[BROADCAST_SPLATINSERT]], <vscale x 4 x i1> poison, <vscale x 4 x i32> zeroinitializer
-; CHECK-NEXT: [[TMP0:%.*]] = xor <vscale x 4 x i1> [[BROADCAST_SPLAT]], splat (i1 true)
; CHECK-NEXT: [[BROADCAST_SPLATINSERT1:%.*]] = insertelement <vscale x 4 x i8> poison, i8 [[X:%.*]], i64 0
; CHECK-NEXT: [[BROADCAST_SPLAT2:%.*]] = shufflevector <vscale x 4 x i8> [[BROADCAST_SPLATINSERT1]], <vscale x 4 x i8> poison, <vscale x 4 x i32> zeroinitializer
; CHECK-NEXT: [[BROADCAST_SPLATINSERT3:%.*]] = insertelement <vscale x 4 x i16> poison, i16 [[Y:%.*]], i64 0
@@ -664,16 +659,16 @@ define void @udiv_sdiv_with_invariant_divisors(i8 %x, i16 %y, i1 %c, ptr %p) vsc
; CHECK-NEXT: [[TMP3:%.*]] = trunc i32 [[TMP2]] to i8
; CHECK-NEXT: [[BROADCAST_SPLATINSERT7:%.*]] = insertelement <vscale x 4 x i8> poison, i8 [[TMP3]], i64 0
; CHECK-NEXT: [[BROADCAST_SPLAT8:%.*]] = shufflevector <vscale x 4 x i8> [[BROADCAST_SPLATINSERT7]], <vscale x 4 x i8> poison, <vscale x 4 x i32> zeroinitializer
-; CHECK-NEXT: br i1 [[C]], label [[LOOP_LATCH10]], label [[THEN9:%.*]]
-; CHECK: then9:
-; CHECK-NEXT: [[TMP5:%.*]] = call <vscale x 4 x i8> @llvm.vp.udiv.nxv4i8(<vscale x 4 x i8> [[VEC_IND]], <vscale x 4 x i8> [[BROADCAST_SPLAT2]], <vscale x 4 x i1> [[TMP0]], i32 [[TMP2]])
+; CHECK-NEXT: br i1 [[C:%.*]], label [[LOOP_LATCH10]], label [[THEN9:%.*]]
+; CHECK: then7:
+; CHECK-NEXT: [[TMP5:%.*]] = call <vscale x 4 x i8> @llvm.vp.udiv.nxv4i8(<vscale x 4 x i8> [[VEC_IND]], <vscale x 4 x i8> [[BROADCAST_SPLAT2]], <vscale x 4 x i1> splat (i1 true), i32 [[TMP2]])
; CHECK-NEXT: [[TMP6:%.*]] = zext <vscale x 4 x i8> [[TMP5]] to <vscale x 4 x i16>
-; CHECK-NEXT: [[TMP8:%.*]] = call <vscale x 4 x i16> @llvm.vp.sdiv.nxv4i16(<vscale x 4 x i16> [[TMP6]], <vscale x 4 x i16> [[BROADCAST_SPLAT4]], <vscale x 4 x i1> [[TMP0]], i32 [[TMP2]])
+; CHECK-NEXT: [[TMP8:%.*]] = call <vscale x 4 x i16> @llvm.vp.sdiv.nxv4i16(<vscale x 4 x i16> [[TMP6]], <vscale x 4 x i16> [[BROADCAST_SPLAT4]], <vscale x 4 x i1> splat (i1 true), i32 [[TMP2]])
; CHECK-NEXT: [[TMP9:%.*]] = sext <vscale x 4 x i16> [[TMP8]] to <vscale x 4 x i32>
; CHECK-NEXT: br label [[LOOP_LATCH10]]
-; CHECK: loop.latch10:
+; CHECK: loop.latch8:
; CHECK-NEXT: [[TMP10:%.*]] = phi <vscale x 4 x i32> [ poison, [[LOOP_LATCH]] ], [ [[TMP9]], [[THEN9]] ]
-; CHECK-NEXT: [[TMP11:%.*]] = phi <vscale x 4 x i1> [ zeroinitializer, [[LOOP_LATCH]] ], [ [[TMP0]], [[THEN9]] ]
+; CHECK-NEXT: [[TMP11:%.*]] = phi <vscale x 4 x i1> [ zeroinitializer, [[LOOP_LATCH]] ], [ splat (i1 true), [[THEN9]] ]
; CHECK-NEXT: [[PREDPHI:%.*]] = select <vscale x 4 x i1> [[TMP11]], <vscale x 4 x i32> [[TMP10]], <vscale x 4 x i32> zeroinitializer
; CHECK-NEXT: call void @llvm.vp.scatter.nxv4i32.nxv4p0(<vscale x 4 x i32> [[PREDPHI]], <vscale x 4 x ptr> align 4 [[BROADCAST_SPLAT6]], <vscale x 4 x i1> splat (i1 true), i32 [[TMP2]])
; CHECK-NEXT: [[AVL_NEXT]] = sub nuw i32 [[AVL]], [[TMP2]]
@@ -689,25 +684,22 @@ define void @udiv_sdiv_with_invariant_divisors(i8 %x, i16 %y, i1 %c, ptr %p) vsc
; FIXED-NEXT: entry:
; FIXED-NEXT: br label [[VECTOR_PH:%.*]]
; FIXED: vector.ph:
-; FIXED-NEXT: [[BROADCAST_SPLATINSERT:%.*]] = insertelement <8 x i1> poison, i1 [[C:%.*]], i64 0
-; FIXED-NEXT: [[BROADCAST_SPLAT:%.*]] = shufflevector <8 x i1> [[BROADCAST_SPLATINSERT]], <8 x i1> poison, <8 x i32> zeroinitializer
-; FIXED-NEXT: [[TMP0:%.*]] = xor <8 x i1> [[BROADCAST_SPLAT]], splat (i1 true)
; FIXED-NEXT: [[BROADCAST_SPLATINSERT1:%.*]] = insertelement <8 x i8> poison, i8 [[X:%.*]], i64 0
; FIXED-NEXT: [[BROADCAST_SPLAT2:%.*]] = shufflevector <8 x i8> [[BROADCAST_SPLATINSERT1]], <8 x i8> poison, <8 x i32> zeroinitializer
; FIXED-NEXT: [[BROADCAST_SPLATINSERT3:%.*]] = insertelement <8 x i16> poison, i16 [[Y:%.*]], i64 0
; FIXED-NEXT: [[BROADCAST_SPLAT4:%.*]] = shufflevector <8 x i16> [[BROADCAST_SPLATINSERT3]], <8 x i16> poison, <8 x i32> zeroinitializer
; FIXED-NEXT: br label [[VECTOR_BODY:%.*]]
; FIXED: vector.body:
-; FIXED-NEXT: br i1 [[C]], label [[LOOP_LATCH6:%.*]], label [[THEN5:%.*]]
-; FIXED: then5:
-; FIXED-NEXT: [[TMP1:%.*]] = call <8 x i8> @llvm.masked.udiv.v8i8(<8 x i8> <i8 -12, i8 -11, i8 -10, i8 -9, i8 -8, i8 -7, i8 -6, i8 -5>, <8 x i8> [[BROADCAST_SPLAT2]], <8 x i1> [[TMP0]])
+; FIXED-NEXT: br i1 [[C:%.*]], label [[LOOP_LATCH6:%.*]], label [[THEN5:%.*]]
+; FIXED: then3:
+; FIXED-NEXT: [[TMP1:%.*]] = udiv <8 x i8> <i8 -12, i8 -11, i8 -10, i8 -9, i8 -8, i8 -7, i8 -6, i8 -5>, [[BROADCAST_SPLAT2]]
; FIXED-NEXT: [[TMP2:%.*]] = zext <8 x i8> [[TMP1]] to <8 x i16>
-; FIXED-NEXT: [[TMP3:%.*]] = call <8 x i16> @llvm.masked.sdiv.v8i16(<8 x i16> [[TMP2]], <8 x i16> [[BROADCAST_SPLAT4]], <8 x i1> [[TMP0]])
+; FIXED-NEXT: [[TMP3:%.*]] = sdiv <8 x i16> [[TMP2]], [[BROADCAST_SPLAT4]]
; FIXED-NEXT: [[TMP4:%.*]] = sext <8 x i16> [[TMP3]] to <8 x i32>
; FIXED-NEXT: br label [[LOOP_LATCH6]]
-; FIXED: loop.latch6:
+; FIXED: loop.latch4:
; FIXED-NEXT: [[TMP5:%.*]] = phi <8 x i32> [ poison, [[VECTOR_BODY]] ], [ [[TMP4]], [[THEN5]] ]
-; FIXED-NEXT: [[TMP6:%.*]] = phi <8 x i1> [ zeroinitializer, [[VECTOR_BODY]] ], [ [[TMP0]], [[THEN5]] ]
+; FIXED-NEXT: [[TMP6:%.*]] = phi <8 x i1> [ zeroinitializer, [[VECTOR_BODY]] ], [ splat (i1 true), [[THEN5]] ]
; FIXED-NEXT: [[PREDPHI:%.*]] = select <8 x i1> [[TMP6]], <8 x i32> [[TMP5]], <8 x i32> zeroinitializer
; FIXED-NEXT: [[TMP7:%.*]] = extractelement <8 x i32> [[PREDPHI]], i64 7
; FIXED-NEXT: store i32 [[TMP7]], ptr [[P:%.*]], align 4
diff --git a/llvm/test/Transforms/LoopVectorize/RISCV/gather-scatter-cost.ll b/llvm/test/Transforms/LoopVectorize/RISCV/gather-scatter-cost.ll
index c446d8c645a06..93ce16c40ab54 100644
--- a/llvm/test/Transforms/LoopVectorize/RISCV/gather-scatter-cost.ll
+++ b/llvm/test/Transforms/LoopVectorize/RISCV/gather-scatter-cost.ll
@@ -28,9 +28,6 @@ define void @predicated_uniform_load(ptr %src, i32 %n, ptr %dst, i1 %cond) {
; CHECK-NEXT: [[FOUND_CONFLICT:%.*]] = and i1 [[BOUND0]], [[BOUND1]]
; CHECK-NEXT: br i1 [[FOUND_CONFLICT]], label [[SCALAR_PH]], label [[VECTOR_PH:%.*]]
; CHECK: vector.ph:
-; CHECK-NEXT: [[BROADCAST_SPLATINSERT1:%.*]] = insertelement <vscale x 4 x i1> poison, i1 [[COND:%.*]], i64 0
-; CHECK-NEXT: [[BROADCAST_SPLAT1:%.*]] = shufflevector <vscale x 4 x i1> [[BROADCAST_SPLATINSERT1]], <vscale x 4 x i1> poison, <vscale x 4 x i32> zeroinitializer
-; CHECK-NEXT: [[TMP13:%.*]] = xor <vscale x 4 x i1> [[BROADCAST_SPLAT1]], splat (i1 true)
; CHECK-NEXT: [[BROADCAST_SPLATINSERT:%.*]] = insertelement <vscale x 4 x ptr> poison, ptr [[BOXES]], i64 0
; CHECK-NEXT: [[BROADCAST_SPLAT:%.*]] = shufflevector <vscale x 4 x ptr> [[BROADCAST_SPLATINSERT]], <vscale x 4 x ptr> poison, <vscale x 4 x i32> zeroinitializer
; CHECK-NEXT: [[BROADCAST_SPLATINSERT3:%.*]] = insertelement <vscale x 4 x ptr> poison, ptr [[NBRBOXES]], i64 0
@@ -39,15 +36,15 @@ define void @predicated_uniform_load(ptr %src, i32 %n, ptr %dst, i1 %cond) {
; CHECK: vector.body:
; CHECK-NEXT: [[AVL:%.*]] = phi i32 [ [[TMP3]], [[VECTOR_PH]] ], [ [[AVL_NEXT:%.*]], [[LOOP_LATCH8:%.*]] ]
; CHECK-NEXT: [[TMP10:%.*]] = call i32 @llvm.experimental.get.vector.length.i32(i32 [[AVL]], i32 4, i1 true)
-; CHECK-NEXT: br i1 [[COND]], label [[LOOP_THEN7:%.*]], label [[LOOP_ELSE6:%.*]]
-; CHECK: loop.else6:
-; CHECK-NEXT: [[WIDE_MASKED_GATHER:%.*]] = call <vscale x 4 x i32> @llvm.vp.gather.nxv4i32.nxv4p0(<vscale x 4 x ptr> align 4 [[BROADCAST_SPLAT]], <vscale x 4 x i1> [[TMP13]], i32 [[TMP10]]), !alias.scope [[META0:![0-9]+]]
+; CHECK-NEXT: br i1 [[COND:%.*]], label [[LOOP_THEN7:%.*]], label [[LOOP_ELSE6:%.*]]
+; CHECK: loop.else4:
+; CHECK-NEXT: [[WIDE_MASKED_GATHER:%.*]] = call <vscale x 4 x i32> @llvm.vp.gather.nxv4i32.nxv4p0(<vscale x 4 x ptr> align 4 [[BROADCAST_SPLAT]], <vscale x 4 x i1> splat (i1 true), i32 [[TMP10]]), !alias.scope [[META0:![0-9]+]]
; CHECK-NEXT: br label [[LOOP_LATCH8]]
-; CHECK: loop.then7:
+; CHECK: loop.then5:
; CHECK-NEXT: br label [[LOOP_LATCH8]]
-; CHECK: loop.latch8:
+; CHECK: loop.latch6:
; CHECK-NEXT: [[TMP15:%.*]] = phi <vscale x 4 x i32> [ [[WIDE_MASKED_GATHER]], [[LOOP_ELSE6]] ], [ poison, [[LOOP_THEN7]] ]
-; CHECK-NEXT: [[TMP14:%.*]] = phi <vscale x 4 x i1> [ zeroinitializer, [[LOOP_ELSE6]] ], [ [[BROADCAST_SPLAT1]], [[LOOP_THEN7]] ]
+; CHECK-NEXT: [[TMP14:%.*]] = phi <vscale x 4 x i1> [ zeroinitializer, [[LOOP_ELSE6]] ], [ splat (i1 true), [[LOOP_THEN7]] ]
; CHECK-NEXT: [[PREDPHI:%.*]] = select <vscale x 4 x i1> [[TMP14]], <vscale x 4 x i32> zeroinitializer, <vscale x 4 x i32> [[TMP15]]
; CHECK-NEXT: call void @llvm.vp.scatter.nxv4i32.nxv4p0(<vscale x 4 x i32> [[PREDPHI]], <vscale x 4 x ptr> align 4 [[BROADCAST_SPLAT4]], <vscale x 4 x i1> splat (i1 true), i32 [[TMP10]]), !alias.scope [[META3:![0-9]+]], !noalias [[META0]]
; CHECK-NEXT: [[AVL_NEXT]] = sub nuw i32 [[AVL]], [[TMP10]]
diff --git a/llvm/test/Transforms/LoopVectorize/RISCV/predicated-costs.ll b/llvm/test/Transforms/LoopVectorize/RISCV/predicated-costs.ll
index 488ad483e8e86..f1b19e4a2110b 100644
--- a/llvm/test/Transforms/LoopVectorize/RISCV/predicated-costs.ll
+++ b/llvm/test/Transforms/LoopVectorize/RISCV/predicated-costs.ll
@@ -11,12 +11,6 @@ define void @nested(ptr noalias %p0, ptr noalias %p1, i1 %c0, i1 %c1) vscale_ran
; CHECK-NEXT: [[ENTRY:.*:]]
; CHECK-NEXT: br label %[[LOOP:.*]]
; CHECK: [[LOOP]]:
-; CHECK-NEXT: [[BROADCAST_SPLATINSERT:%.*]] = insertelement <vscale x 4 x i1> poison, i1 [[C1]], i64 0
-; CHECK-NEXT: [[BROADCAST_SPLAT:%.*]] = shufflevector <vscale x 4 x i1> [[BROADCAST_SPLATINSERT]], <vscale x 4 x i1> poison, <vscale x 4 x i32> zeroinitializer
-; CHECK-NEXT: [[BROADCAST_SPLATINSERT1:%.*]] = insertelement <vscale x 4 x i1> poison, i1 [[C0]], i64 0
-; CHECK-NEXT: [[BROADCAST_SPLAT2:%.*]] = shufflevector <vscale x 4 x i1> [[BROADCAST_SPLATINSERT1]], <vscale x 4 x i1> poison, <vscale x 4 x i32> zeroinitializer
-; CHECK-NEXT: [[TMP2:%.*]] = freeze i1 [[C1]]
-; CHECK-NEXT: [[TMP0:%.*]] = select <vscale x 4 x i1> [[BROADCAST_SPLAT2]], <vscale x 4 x i1> [[BROADCAST_SPLAT]], <vscale x 4 x i1> zeroinitializer
; CHECK-NEXT: br label %[[THEN_1:.*]]
; CHECK: [[THEN_1]]:
; CHECK-NEXT: [[IV1:%.*]] = phi i32 [ 0, %[[LOOP]] ], [ [[INDEX_EVL_NEXT:%.*]], %[[LATCH4:.*]] ]
@@ -24,12 +18,12 @@ define void @nested(ptr noalias %p0, ptr noalias %p1, i1 %c0, i1 %c1) vscale_ran
; CHECK-NEXT: [[TMP1:%.*]] = call i32 @llvm.experimental.get.vector.length.i32(i32 [[AVL]], i32 4, i1 true)
; CHECK-NEXT: br i1 [[C0]], label %[[THEN_03:.*]], label %[[LATCH4]]
; CHECK: [[THEN_03]]:
-; CHECK-NEXT: br i1 [[TMP2]], label %[[THEN_14:.*]], label %[[LATCH4]]
+; CHECK-NEXT: br i1 [[C1]], label %[[THEN_14:.*]], label %[[LATCH4]]
; CHECK: [[THEN_14]]:
; CHECK-NEXT: [[GEP2:%.*]] = getelementptr i32, ptr [[P0]], i32 [[IV1]]
-; CHECK-NEXT: [[VP_OP_LOAD:%.*]] = call <vscale x 4 x i32> @llvm.vp.load.nxv4i32.p0(ptr align 4 [[GEP2]], <vscale x 4 x i1> [[TMP0]], i32 [[TMP1]])
+; CHECK-NEXT: [[VP_OP_LOAD:%.*]] = call <vscale x 4 x i32> @llvm.vp.load.nxv4i32.p0(ptr align 4 [[GEP2]], <vscale x 4 x i1> splat (i1 true), i32 [[TMP1]])
; CHECK-NEXT: [[TMP3:%.*]] = getelementptr i32, ptr [[P1]], <vscale x 4 x i32> [[VP_OP_LOAD]]
-; CHECK-NEXT: call void @llvm.vp.scatter.nxv4i32.nxv4p0(<vscale x 4 x i32> zeroinitializer, <vscale x 4 x ptr> align 4 [[TMP3]], <vscale x 4 x i1> [[TMP0]], i32 [[TMP1]])
+; CHECK-NEXT: call void @llvm.vp.scatter.nxv4i32.nxv4p0(<vscale x 4 x i32> zeroinitializer, <vscale x 4 x ptr> align 4 [[TMP3]], <vscale x 4 x i1> splat (i1 true), i32 [[TMP1]])
; CHECK-NEXT: br label %[[LATCH4]]
; CHECK: [[LATCH4]]:
; CHECK-NEXT: [[INDEX_EVL_NEXT]] = add nuw i32 [[TMP1]], [[IV1]]
@@ -76,31 +70,25 @@ define void @always_taken(ptr noalias %p0, ptr noalias %p1, i1 %c0, i1 %c1) vsca
; CHECK-NEXT: [[ENTRY:.*:]]
; CHECK-NEXT: br label %[[VECTOR_PH:.*]]
; CHECK: [[VECTOR_PH]]:
-; CHECK-NEXT: [[BROADCAST_SPLATINSERT:%.*]] = insertelement <vscale x 4 x i1> poison, i1 [[C1]], i64 0
-; CHECK-NEXT: [[BROADCAST_SPLAT:%.*]] = shufflevector <vscale x 4 x i1> [[BROADCAST_SPLATINSERT]], <vscale x 4 x i1> poison, <vscale x 4 x i32> zeroinitializer
-; CHECK-NEXT: [[BROADCAST_SPLATINSERT1:%.*]] = insertelement <vscale x 4 x i1> poison, i1 [[C0]], i64 0
-; CHECK-NEXT: [[BROADCAST_SPLAT2:%.*]] = shufflevector <vscale x 4 x i1> [[BROADCAST_SPLATINSERT1]], <vscale x 4 x i1> poison, <vscale x 4 x i32> zeroinitializer
-; CHECK-NEXT: [[TMP5:%.*]] = freeze i1 [[C1]]
-; CHECK-NEXT: [[TMP0:%.*]] = select <vscale x 4 x i1> [[BROADCAST_SPLAT2]], <vscale x 4 x i1> [[BROADCAST_SPLAT]], <vscale x 4 x i1> zeroinitializer
; CHECK-NEXT: br label %[[VECTOR_BODY:.*]]
; CHECK: [[VECTOR_BODY]]:
; CHECK-NEXT: [[EVL_BASED_IV:%.*]] = phi i32 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_EVL_NEXT:%.*]], %[[LATCH4:.*]] ]
; CHECK-NEXT: [[AVL:%.*]] = phi i32 [ 1024, %[[VECTOR_PH]] ], [ [[AVL_NEXT:%.*]], %[[LATCH4]] ]
; CHECK-NEXT: [[TMP1:%.*]] = call i32 @llvm.experimental.get.vector.length.i32(i32 [[AVL]], i32 4, i1 true)
-; CHECK-NEXT: br i1 [[C0]], label %[[THEN_03:.*]], label %[[LATCH4]]
+; CHECK-NEXT: br i1 [[C0]], label %[[THEN_03:.*]], label %[[LATCH4]], !prof [[PROF3:![0-9]+]]
; CHECK: [[THEN_03]]:
-; CHECK-NEXT: br i1 [[TMP5]], label %[[THEN_14:.*]], label %[[LATCH4]]
+; CHECK-NEXT: br i1 [[C1]], label %[[THEN_14:.*]], label %[[LATCH4]], !prof [[PROF3]]
; CHECK: [[THEN_14]]:
; CHECK-NEXT: [[TMP2:%.*]] = getelementptr i32, ptr [[P0]], i32 [[EVL_BASED_IV]]
-; CHECK-NEXT: [[VP_OP_LOAD:%.*]] = call <vscale x 4 x i32> @llvm.vp.load.nxv4i32.p0(ptr align 4 [[TMP2]], <vscale x 4 x i1> [[TMP0]], i32 [[TMP1]])
+; CHECK-NEXT: [[VP_OP_LOAD:%.*]] = call <vscale x 4 x i32> @llvm.vp.load.nxv4i32.p0(ptr align 4 [[TMP2]], <vscale x 4 x i1> splat (i1 true), i32 [[TMP1]])
; CHECK-NEXT: [[TMP3:%.*]] = getelementptr i32, ptr [[P1]], <vscale x 4 x i32> [[VP_OP_LOAD]]
-; CHECK-NEXT: call void @llvm.vp.scatter.nxv4i32.nxv4p0(<vscale x 4 x i32> zeroinitializer, <vscale x 4 x ptr> align 4 [[TMP3]], <vscale x 4 x i1> [[TMP0]], i32 [[TMP1]])
+; CHECK-NEXT: call void @llvm.vp.scatter.nxv4i32.nxv4p0(<vscale x 4 x i32> zeroinitializer, <vscale x 4 x ptr> align 4 [[TMP3]], <vscale x 4 x i1> splat (i1 true), i32 [[TMP1]])
; CHECK-NEXT: br label %[[LATCH4]]
; CHECK: [[LATCH4]]:
; CHECK-NEXT: [[INDEX_EVL_NEXT]] = add nuw i32 [[TMP1]], [[EVL_BASED_IV]]
; CHECK-NEXT: [[AVL_NEXT]] = sub nuw i32 [[AVL]], [[TMP1]]
; CHECK-NEXT: [[TMP4:%.*]] = icmp eq i32 [[AVL_NEXT]], 0
-; CHECK-NEXT: br i1 [[TMP4]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP3:![0-9]+]]
+; CHECK-NEXT: br i1 [[TMP4]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP4:![0-9]+]]
; CHECK: [[MIDDLE_BLOCK]]:
; CHECK-NEXT: br label %[[EXIT:.*]]
; CHECK: [[EXIT]]:
diff --git a/llvm/test/Transforms/LoopVectorize/RISCV/predicated-reverse-store.ll b/llvm/test/Transforms/LoopVectorize/RISCV/predicated-reverse-store.ll
index 29373388b3721..a5b3d96996c26 100644
--- a/llvm/test/Transforms/LoopVectorize/RISCV/predicated-reverse-store.ll
+++ b/llvm/test/Transforms/LoopVectorize/RISCV/predicated-reverse-store.ll
@@ -7,8 +7,6 @@ define void @reverse_predicated_store(i1 %c, ptr %dst, i64 %n) #0 {
; CHECK-NEXT: [[TMP0:%.*]] = add i64 [[N:%.*]], 1
; CHECK-NEXT: br label [[FOR_BODY:%.*]]
; CHECK: vector.ph:
-; CHECK-NEXT: [[BROADCAST_SPLATINSERT:%.*]] = insertelement <vscale x 4 x i1> poison, i1 [[C:%.*]], i64 0
-; CHECK-NEXT: [[BROADCAST_SPLAT:%.*]] = shufflevector <vscale x 4 x i1> [[BROADCAST_SPLATINSERT]], <vscale x 4 x i1> poison, <vscale x 4 x i32> zeroinitializer
; CHECK-NEXT: br label [[VECTOR_BODY:%.*]]
; CHECK: vector.body:
; CHECK-NEXT: [[EVL_BASED_IV:%.*]] = phi i64 [ 0, [[FOR_BODY]] ], [ [[CURRENT_ITERATION_NEXT:%.*]], [[LOOP_LATCH2:%.*]] ]
@@ -16,16 +14,15 @@ define void @reverse_predicated_store(i1 %c, ptr %dst, i64 %n) #0 {
; CHECK-NEXT: [[TMP1:%.*]] = call i32 @llvm.experimental.get.vector.length.i64(i64 [[AVL]], i32 4, i1 true)
; CHECK-NEXT: [[IV:%.*]] = sub i64 [[N]], [[EVL_BASED_IV]]
; CHECK-NEXT: [[IV_NEXT:%.*]] = add i64 [[IV]], -1
-; CHECK-NEXT: br i1 [[C]], label [[IF_THEN1:%.*]], label [[LOOP_LATCH2]]
+; CHECK-NEXT: br i1 [[C:%.*]], label [[IF_THEN1:%.*]], label [[LOOP_LATCH2]]
; CHECK: if.then1:
; CHECK-NEXT: [[ARRAYIDX:%.*]] = getelementptr float, ptr [[DST:%.*]], i64 [[IV_NEXT]]
-; CHECK-NEXT: [[VP_REVERSE_MASK:%.*]] = call <vscale x 4 x i1> @llvm.experimental.vp.reverse.nxv4i1(<vscale x 4 x i1> [[BROADCAST_SPLAT]], <vscale x 4 x i1> splat (i1 true), i32 [[TMP1]])
; CHECK-NEXT: [[TMP4:%.*]] = zext i32 [[TMP1]] to i64
; CHECK-NEXT: [[TMP6:%.*]] = sub nuw nsw i64 [[TMP4]], 1
; CHECK-NEXT: [[TMP7:%.*]] = sub i64 0, [[TMP6]]
; CHECK-NEXT: [[TMP9:%.*]] = getelementptr float, ptr [[ARRAYIDX]], i64 [[TMP7]]
; CHECK-NEXT: [[TMP12:%.*]] = call <vscale x 4 x float> @llvm.experimental.vp.reverse.nxv4f32(<vscale x 4 x float> zeroinitializer, <vscale x 4 x i1> splat (i1 true), i32 [[TMP1]])
-; CHECK-NEXT: call void @llvm.vp.store.nxv4f32.p0(<vscale x 4 x float> [[TMP12]], ptr align 4 [[TMP9]], <vscale x 4 x i1> [[VP_REVERSE_MASK]], i32 [[TMP1]])
+; CHECK-NEXT: call void @llvm.vp.store.nxv4f32.p0(<vscale x 4 x float> [[TMP12]], ptr align 4 [[TMP9]], <vscale x 4 x i1> splat (i1 true), i32 [[TMP1]])
; CHECK-NEXT: br label [[LOOP_LATCH2]]
; CHECK: loop.latch2:
; CHECK-NEXT: [[TMP13:%.*]] = zext i32 [[TMP1]] to i64
diff --git a/llvm/test/Transforms/LoopVectorize/RISCV/tail-folding-complex-mask.ll b/llvm/test/Transforms/LoopVectorize/RISCV/tail-folding-complex-mask.ll
index 68d90a26075b4..289ad1847451c 100644
--- a/llvm/test/Transforms/LoopVectorize/RISCV/tail-folding-complex-mask.ll
+++ b/llvm/test/Transforms/LoopVectorize/RISCV/tail-folding-complex-mask.ll
@@ -19,8 +19,6 @@ define void @test(i64 %n, ptr noalias %src0, ptr noalias %src1, ptr noalias %src
; IF-EVL-NEXT: [[BROADCAST_SPLATINSERT1:%.*]] = insertelement <vscale x 4 x i1> poison, i1 [[TMP0]], i64 0
; IF-EVL-NEXT: [[BROADCAST_SPLAT2:%.*]] = shufflevector <vscale x 4 x i1> [[BROADCAST_SPLATINSERT1]], <vscale x 4 x i1> poison, <vscale x 4 x i32> zeroinitializer
; IF-EVL-NEXT: [[TMP1:%.*]] = or <vscale x 4 x i1> [[BROADCAST_SPLAT]], [[BROADCAST_SPLAT2]]
-; IF-EVL-NEXT: [[BROADCAST_SPLATINSERT3:%.*]] = insertelement <vscale x 4 x i1> poison, i1 [[C3]], i64 0
-; IF-EVL-NEXT: [[BROADCAST_SPLAT4:%.*]] = shufflevector <vscale x 4 x i1> [[BROADCAST_SPLATINSERT3]], <vscale x 4 x i1> poison, <vscale x 4 x i32> zeroinitializer
; IF-EVL-NEXT: br label %[[VECTOR_BODY:.*]]
; IF-EVL: [[VECTOR_BODY]]:
; IF-EVL-NEXT: [[EVL_BASED_IV:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_EVL_NEXT:%.*]], %[[LATCH12:.*]] ]
@@ -47,13 +45,13 @@ define void @test(i64 %n, ptr noalias %src0, ptr noalias %src1, ptr noalias %src
; IF-EVL-NEXT: [[PREDPHI8:%.*]] = select <vscale x 4 x i1> [[TMP4]], <vscale x 4 x i32> [[TMP12]], <vscale x 4 x i32> zeroinitializer
; IF-EVL-NEXT: br i1 [[C3]], label %[[LOAD_V210:.*]], label %[[LATCH12]]
; IF-EVL: [[LOAD_V210]]:
-; IF-EVL-NEXT: [[TMP18:%.*]] = getelementptr i32, ptr [[SRC2]], i64 [[EVL_BASED_IV]]
-; IF-EVL-NEXT: [[WIDE_MASKED_LOAD:%.*]] = call <vscale x 4 x i32> @llvm.vp.load.nxv4i32.p0(ptr align 4 [[TMP18]], <vscale x 4 x i1> [[BROADCAST_SPLAT4]], i32 [[TMP7]])
+; IF-EVL-NEXT: [[TMP14:%.*]] = getelementptr inbounds i32, ptr [[SRC2]], i64 [[EVL_BASED_IV]]
+; IF-EVL-NEXT: [[WIDE_MASKED_LOAD:%.*]] = call <vscale x 4 x i32> @llvm.vp.load.nxv4i32.p0(ptr align 4 [[TMP14]], <vscale x 4 x i1> splat (i1 true), i32 [[TMP7]])
; IF-EVL-NEXT: [[TMP19:%.*]] = add <vscale x 4 x i32> [[WIDE_MASKED_LOAD]], [[PREDPHI8]]
; IF-EVL-NEXT: br label %[[LATCH12]]
; IF-EVL: [[LATCH12]]:
; IF-EVL-NEXT: [[TMP15:%.*]] = phi <vscale x 4 x i32> [ poison, %[[LOAD_V17]] ], [ [[TMP19]], %[[LOAD_V210]] ]
-; IF-EVL-NEXT: [[TMP16:%.*]] = phi <vscale x 4 x i1> [ zeroinitializer, %[[LOAD_V17]] ], [ [[BROADCAST_SPLAT4]], %[[LOAD_V210]] ]
+; IF-EVL-NEXT: [[TMP16:%.*]] = phi <vscale x 4 x i1> [ zeroinitializer, %[[LOAD_V17]] ], [ splat (i1 true), %[[LOAD_V210]] ]
; IF-EVL-NEXT: [[PREDPHI9:%.*]] = select <vscale x 4 x i1> [[TMP16]], <vscale x 4 x i32> [[TMP15]], <vscale x 4 x i32> [[PREDPHI8]]
; IF-EVL-NEXT: [[TMP20:%.*]] = getelementptr inbounds i32, ptr [[DST]], i64 [[EVL_BASED_IV]]
; IF-EVL-NEXT: call void @llvm.vp.store.nxv4i32.p0(<vscale x 4 x i32> [[PREDPHI9]], ptr align 4 [[TMP20]], <vscale x 4 x i1> splat (i1 true), i32 [[TMP7]])
@@ -87,8 +85,6 @@ define void @test(i64 %n, ptr noalias %src0, ptr noalias %src1, ptr noalias %src
; NO-VP-NEXT: [[TMP9:%.*]] = or <vscale x 4 x i1> [[BROADCAST_SPLAT2]], [[BROADCAST_SPLAT4]]
; NO-VP-NEXT: [[TMP24:%.*]] = extractelement <vscale x 4 x i1> [[TMP9]], i64 0
; NO-VP-NEXT: [[TMP25:%.*]] = freeze i1 [[TMP24]]
-; NO-VP-NEXT: [[BROADCAST_SPLATINSERT4:%.*]] = insertelement <vscale x 4 x i1> poison, i1 [[C3]], i64 0
-; NO-VP-NEXT: [[TMP12:%.*]] = shufflevector <vscale x 4 x i1> [[BROADCAST_SPLATINSERT4]], <vscale x 4 x i1> poison, <vscale x 4 x i32> zeroinitializer
; NO-VP-NEXT: br label %[[VECTOR_BODY:.*]]
; NO-VP: [[VECTOR_BODY]]:
; NO-VP-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[LATCH12:.*]] ]
@@ -117,13 +113,13 @@ define void @test(i64 %n, ptr noalias %src0, ptr noalias %src1, ptr noalias %src
; NO-VP-NEXT: [[PREDPHI6:%.*]] = select <vscale x 4 x i1> [[TMP27]], <vscale x 4 x i32> [[TMP26]], <vscale x 4 x i32> zeroinitializer
; NO-VP-NEXT: br i1 [[C3]], label %[[LOAD_V210:.*]], label %[[LATCH12]]
; NO-VP: [[LOAD_V210]]:
-; NO-VP-NEXT: [[TMP16:%.*]] = getelementptr i32, ptr [[SRC2]], i64 [[INDEX]]
-; NO-VP-NEXT: [[WIDE_MASKED_LOAD7:%.*]] = call <vscale x 4 x i32> @llvm.masked.load.nxv4i32.p0(ptr align 4 [[TMP16]], <vscale x 4 x i1> [[TMP12]], <vscale x 4 x i32> poison)
+; NO-VP-NEXT: [[TMP28:%.*]] = getelementptr inbounds i32, ptr [[SRC2]], i64 [[INDEX]]
+; NO-VP-NEXT: [[WIDE_MASKED_LOAD7:%.*]] = load <vscale x 4 x i32>, ptr [[TMP28]], align 4
; NO-VP-NEXT: [[TMP17:%.*]] = add <vscale x 4 x i32> [[WIDE_MASKED_LOAD7]], [[PREDPHI6]]
; NO-VP-NEXT: br label %[[LATCH12]]
; NO-VP: [[LATCH12]]:
; NO-VP-NEXT: [[TMP22:%.*]] = phi <vscale x 4 x i32> [ poison, %[[LOAD_V2_CHECK9]] ], [ [[TMP17]], %[[LOAD_V210]] ]
-; NO-VP-NEXT: [[TMP23:%.*]] = phi <vscale x 4 x i1> [ zeroinitializer, %[[LOAD_V2_CHECK9]] ], [ [[TMP12]], %[[LOAD_V210]] ]
+; NO-VP-NEXT: [[TMP23:%.*]] = phi <vscale x 4 x i1> [ zeroinitializer, %[[LOAD_V2_CHECK9]] ], [ splat (i1 true), %[[LOAD_V210]] ]
; NO-VP-NEXT: [[PREDPHI8:%.*]] = select <vscale x 4 x i1> [[TMP23]], <vscale x 4 x i32> [[TMP22]], <vscale x 4 x i32> [[PREDPHI6]]
; NO-VP-NEXT: [[TMP18:%.*]] = getelementptr inbounds i32, ptr [[DST]], i64 [[INDEX]]
; NO-VP-NEXT: store <vscale x 4 x i32> [[PREDPHI8]], ptr [[TMP18]], align 4
diff --git a/llvm/test/Transforms/LoopVectorize/VPlan/AArch64/vplan-memory-op-decisions.ll b/llvm/test/Transforms/LoopVectorize/VPlan/AArch64/vplan-memory-op-decisions.ll
index 53a4267a063a4..8584e5c83a72b 100644
--- a/llvm/test/Transforms/LoopVectorize/VPlan/AArch64/vplan-memory-op-decisions.ll
+++ b/llvm/test/Transforms/LoopVectorize/VPlan/AArch64/vplan-memory-op-decisions.ll
@@ -83,8 +83,8 @@ define void @load_feeding_only_mask_not_scalarized(ptr noalias %A, ptr noalias %
; CHECK-NEXT: then:
; CHECK-NEXT: EMIT ir<%gep.B> = getelementptr ir<%B>, ir<%iv>
; CHECK-NEXT: vp<[[VP5:%[0-9]+]]> = vector-pointer ptr, ir<%gep.B>, ir<1>
-; CHECK-NEXT: WIDEN ir<%l.p> = load vp<[[VP5]]>, ir<%cmp>
-; CHECK-NEXT: EMIT store ir<42>, ir<%l.p>, ir<%cmp>
+; CHECK-NEXT: WIDEN ir<%l.p> = load vp<[[VP5]]>, ir<%cmp> (!vplan.execution.frequency 5764607523034234880 (62.5%, estimated))
+; CHECK-NEXT: EMIT store ir<42>, ir<%l.p>, ir<%cmp> (!vplan.execution.frequency 5764607523034234880 (62.5%, estimated))
; CHECK-NEXT: Successor(s): latch
; CHECK-EMPTY:
; CHECK-NEXT: latch:
@@ -379,11 +379,11 @@ define void @cond_load_store(ptr noalias %a, ptr noalias %b, ptr noalias %cond,
; CHECK-NEXT: then:
; CHECK-NEXT: EMIT ir<%gep.a> = getelementptr inbounds ir<%a>, ir<%iv>
; CHECK-NEXT: vp<[[VP5:%[0-9]+]]> = vector-pointer inbounds i32, ir<%gep.a>, ir<1>
-; CHECK-NEXT: WIDEN ir<%lv> = load vp<[[VP5]]>, ir<%cmp>
-; CHECK-NEXT: EMIT ir<%add> = add ir<%lv>, ir<1>, ir<%cmp>
+; CHECK-NEXT: WIDEN ir<%lv> = load vp<[[VP5]]>, ir<%cmp> (!vplan.execution.frequency 5764607523034234880 (62.5%, estimated))
+; CHECK-NEXT: EMIT ir<%add> = add ir<%lv>, ir<1>, ir<%cmp> (!vplan.execution.frequency 5764607523034234880 (62.5%, estimated))
; CHECK-NEXT: EMIT ir<%gep.b> = getelementptr inbounds ir<%b>, ir<%iv>
; CHECK-NEXT: vp<[[VP6:%[0-9]+]]> = vector-pointer inbounds i32, ir<%gep.b>, ir<1>
-; CHECK-NEXT: WIDEN store vp<[[VP6]]>, ir<%add>, ir<%cmp>
+; CHECK-NEXT: WIDEN store vp<[[VP6]]>, ir<%add>, ir<%cmp> (!vplan.execution.frequency 5764607523034234880 (62.5%, estimated))
; CHECK-NEXT: Successor(s): latch
; CHECK-EMPTY:
; CHECK-NEXT: latch:
@@ -456,14 +456,14 @@ define void @cond_reverse_load_store(ptr noalias %a, ptr noalias %b, ptr noalias
; CHECK-NEXT: EMIT ir<%gep.a> = getelementptr inbounds ir<%a>, ir<%iv>
; CHECK-NEXT: vp<[[VP6:%[0-9]+]]> = vector-end-pointer inbounds i32, ir<%gep.a>, vp<[[VP0]]>
; CHECK-NEXT: EMIT vp<[[VP7:%[0-9]+]]> = reverse ir<%cmp>
-; CHECK-NEXT: WIDEN ir<%lv> = load vp<[[VP6]]>, vp<[[VP7]]>
+; CHECK-NEXT: WIDEN ir<%lv> = load vp<[[VP6]]>, vp<[[VP7]]> (!vplan.execution.frequency 5764607523034234880 (62.5%, estimated))
; CHECK-NEXT: EMIT vp<[[VP8:%[0-9]+]]> = reverse ir<%lv>
-; CHECK-NEXT: EMIT ir<%add> = add vp<[[VP8]]>, ir<1>, ir<%cmp>
+; CHECK-NEXT: EMIT ir<%add> = add vp<[[VP8]]>, ir<1>, ir<%cmp> (!vplan.execution.frequency 5764607523034234880 (62.5%, estimated))
; CHECK-NEXT: EMIT ir<%gep.b> = getelementptr inbounds ir<%b>, ir<%iv>
; CHECK-NEXT: vp<[[VP9:%[0-9]+]]> = vector-end-pointer inbounds i32, ir<%gep.b>, vp<[[VP0]]>
; CHECK-NEXT: EMIT vp<[[VP10:%[0-9]+]]> = reverse ir<%cmp>
; CHECK-NEXT: EMIT vp<[[VP11:%[0-9]+]]> = reverse ir<%add>
-; CHECK-NEXT: WIDEN store vp<[[VP9]]>, vp<[[VP11]]>, vp<[[VP10]]>
+; CHECK-NEXT: WIDEN store vp<[[VP9]]>, vp<[[VP11]]>, vp<[[VP10]]> (!vplan.execution.frequency 5764607523034234880 (62.5%, estimated))
; CHECK-NEXT: Successor(s): latch
; CHECK-EMPTY:
; CHECK-NEXT: latch:
@@ -750,19 +750,19 @@ define void @blend_with_identical_incoming_values_address(ptr noalias %A, i1 %c)
; CHECK-EMPTY:
; CHECK-NEXT: vector.body:
; CHECK-NEXT: ir<%iv> = WIDEN-INDUCTION nuw nsw ir<0>, ir<1>, vp<[[VP0]]>
-; CHECK-NEXT: EMIT branch-on-cond ir<%c>
+; CHECK-NEXT: EMIT branch-on-cond ir<%c> (!vplan.prof.estimated estimated {1073741824, 1073741824})
; CHECK-NEXT: Successor(s): then, else
; CHECK-EMPTY:
; CHECK-NEXT: then:
-; CHECK-NEXT: EMIT ir<%idx.then> = add nsw ir<%iv>, ir<-1>, ir<%c>
+; CHECK-NEXT: EMIT ir<%idx.then> = add nsw ir<%iv>, ir<-1>
; CHECK-NEXT: Successor(s): latch
; CHECK-EMPTY:
; CHECK-NEXT: latch:
-; CHECK-NEXT: WIDEN-PHI vp<[[VP5:%[0-9]+]]> = phi [ ir<%idx.else>, else ], [ ir<poison>, then ]
-; CHECK-NEXT: WIDEN-PHI vp<[[VP6:%[0-9]+]]> = phi [ vp<[[VP4:%[0-9]+]]>, else ], [ ir<false>, then ]
-; CHECK-NEXT: WIDEN-PHI vp<[[VP7:%[0-9]+]]> = phi [ ir<poison>, else ], [ ir<%idx.then>, then ]
-; CHECK-NEXT: WIDEN-PHI vp<[[VP8:%[0-9]+]]> = phi [ ir<false>, else ], [ ir<%c>, then ]
-; CHECK-NEXT: BLEND ir<%idx> = vp<%5>/vp<[[VP6]]> vp<%7>/vp<[[VP8]]>
+; CHECK-NEXT: WIDEN-PHI vp<[[VP4:%[0-9]+]]> = phi [ ir<%idx.else>, else ], [ ir<poison>, then ]
+; CHECK-NEXT: WIDEN-PHI vp<[[VP5:%[0-9]+]]> = phi [ ir<true>, else ], [ ir<false>, then ]
+; CHECK-NEXT: WIDEN-PHI vp<[[VP6:%[0-9]+]]> = phi [ ir<poison>, else ], [ ir<%idx.then>, then ]
+; CHECK-NEXT: WIDEN-PHI vp<[[VP7:%[0-9]+]]> = phi [ ir<false>, else ], [ ir<true>, then ]
+; CHECK-NEXT: BLEND ir<%idx> = vp<%4>/vp<[[VP5]]> vp<%6>/vp<[[VP7]]>
; CHECK-NEXT: EMIT ir<%gep> = getelementptr inbounds ir<%A>, ir<%idx>
; CHECK-NEXT: EMIT-SCALAR ir<%lv> = load ir<%gep>
; CHECK-NEXT: EMIT ir<%add> = add ir<%lv>, ir<1>
@@ -774,8 +774,7 @@ define void @blend_with_identical_incoming_values_address(ptr noalias %A, i1 %c)
; CHECK-NEXT: No successors
; CHECK-EMPTY:
; CHECK-NEXT: else:
-; CHECK-NEXT: EMIT vp<[[VP4]]> = not ir<%c>
-; CHECK-NEXT: EMIT ir<%idx.else> = add nsw ir<%iv>, ir<-1>, vp<[[VP4]]>
+; CHECK-NEXT: EMIT ir<%idx.else> = add nsw ir<%iv>, ir<-1>
; CHECK-NEXT: Successor(s): latch
; CHECK-NEXT: }
; CHECK-NEXT: Successor(s): middle.block
@@ -998,24 +997,24 @@ define void @blend_with_different_incoming_values_address(ptr noalias %A, ptr no
; CHECK-EMPTY:
; CHECK-NEXT: vector.body:
; CHECK-NEXT: ir<%iv> = WIDEN-INDUCTION nuw nsw ir<0>, ir<1>, vp<[[VP0]]>
-; CHECK-NEXT: EMIT branch-on-cond ir<%c>
+; CHECK-NEXT: EMIT branch-on-cond ir<%c> (!vplan.prof.estimated estimated {1073741824, 1073741824})
; CHECK-NEXT: Successor(s): then, else
; CHECK-EMPTY:
; CHECK-NEXT: then:
-; CHECK-NEXT: EMIT ir<%idx.then> = add nsw ir<%iv>, ir<-1>, ir<%c>
+; CHECK-NEXT: EMIT ir<%idx.then> = add nsw ir<%iv>, ir<-1>
; CHECK-NEXT: Successor(s): latch
; CHECK-EMPTY:
; CHECK-NEXT: latch:
-; CHECK-NEXT: WIDEN-PHI vp<[[VP5:%[0-9]+]]> = phi [ ir<%idx.else>, else ], [ ir<poison>, then ]
-; CHECK-NEXT: WIDEN-PHI vp<[[VP6:%[0-9]+]]> = phi [ vp<[[VP4:%[0-9]+]]>, else ], [ ir<false>, then ]
-; CHECK-NEXT: WIDEN-PHI vp<[[VP7:%[0-9]+]]> = phi [ ir<poison>, else ], [ ir<%idx.then>, then ]
-; CHECK-NEXT: WIDEN-PHI vp<[[VP8:%[0-9]+]]> = phi [ ir<false>, else ], [ ir<%c>, then ]
-; CHECK-NEXT: BLEND ir<%idx> = vp<%5>/vp<[[VP6]]> vp<%7>/vp<[[VP8]]>
+; CHECK-NEXT: WIDEN-PHI vp<[[VP4:%[0-9]+]]> = phi [ ir<%idx.else>, else ], [ ir<poison>, then ]
+; CHECK-NEXT: WIDEN-PHI vp<[[VP5:%[0-9]+]]> = phi [ ir<true>, else ], [ ir<false>, then ]
+; CHECK-NEXT: WIDEN-PHI vp<[[VP6:%[0-9]+]]> = phi [ ir<poison>, else ], [ ir<%idx.then>, then ]
+; CHECK-NEXT: WIDEN-PHI vp<[[VP7:%[0-9]+]]> = phi [ ir<false>, else ], [ ir<true>, then ]
+; CHECK-NEXT: BLEND ir<%idx> = vp<%4>/vp<[[VP5]]> vp<%6>/vp<[[VP7]]>
; CHECK-NEXT: EMIT ir<%gep.B> = getelementptr inbounds ir<%B>, ir<%idx>
; CHECK-NEXT: EMIT-SCALAR ir<%lv> = load ir<%gep.B>
; CHECK-NEXT: EMIT ir<%gep.A> = getelementptr inbounds ir<%A>, ir<%iv>
-; CHECK-NEXT: vp<[[VP9:%[0-9]+]]> = vector-pointer inbounds i32, ir<%gep.A>, ir<1>
-; CHECK-NEXT: WIDEN store vp<[[VP9]]>, ir<%lv>
+; CHECK-NEXT: vp<[[VP8:%[0-9]+]]> = vector-pointer inbounds i32, ir<%gep.A>, ir<1>
+; CHECK-NEXT: WIDEN store vp<[[VP8]]>, ir<%lv>
; CHECK-NEXT: EMIT ir<%iv.next> = add nuw nsw ir<%iv>, ir<1>
; CHECK-NEXT: EMIT ir<%ec> = icmp eq ir<%iv>, ir<100>
; CHECK-NEXT: EMIT vp<%index.next> = add nuw vp<[[VP3]]>, vp<[[VP1]]>
@@ -1023,8 +1022,7 @@ define void @blend_with_different_incoming_values_address(ptr noalias %A, ptr no
; CHECK-NEXT: No successors
; CHECK-EMPTY:
; CHECK-NEXT: else:
-; CHECK-NEXT: EMIT vp<[[VP4]]> = not ir<%c>
-; CHECK-NEXT: EMIT ir<%idx.else> = add nsw ir<%iv>, ir<-2>, vp<[[VP4]]>
+; CHECK-NEXT: EMIT ir<%idx.else> = add nsw ir<%iv>, ir<-2>
; CHECK-NEXT: Successor(s): latch
; CHECK-NEXT: }
; CHECK-NEXT: Successor(s): middle.block
diff --git a/llvm/test/Transforms/LoopVectorize/VPlan/dissolve-replicate-regions.ll b/llvm/test/Transforms/LoopVectorize/VPlan/dissolve-replicate-regions.ll
index 0fdec49691c7c..732ba12c02793 100644
--- a/llvm/test/Transforms/LoopVectorize/VPlan/dissolve-replicate-regions.ll
+++ b/llvm/test/Transforms/LoopVectorize/VPlan/dissolve-replicate-regions.ll
@@ -31,39 +31,23 @@ define void @predicated_load(i1 %c, ptr %ptr, ptr %dst) {
; SCALAR-NEXT: vector.body:
; SCALAR-NEXT: vp<[[VP6:%[0-9]+]]> = SCALAR-STEPS vp<[[VP5]]>, ir<1>, vp<[[VP0]]>
; SCALAR-NEXT: vp<[[VP7:%[0-9]+]]> = SCALAR-STEPS vp<[[VP5]]>, ir<1>, vp<[[VP0]]>, vp<[[VP0]]>
+; SCALAR-NEXT: CLONE ir<%gep> = getelementptr ir<%ptr>, vp<[[VP6]]>
+; SCALAR-NEXT: CLONE ir<%gep>.1 = getelementptr ir<%ptr>, vp<[[VP7]]>
; SCALAR-NEXT: EMIT branch-on-cond ir<%c> (!vplan.prof.estimated estimated {1073741824, 1073741824})
; SCALAR-NEXT: Successor(s): if.then, loop.latch
; SCALAR-EMPTY:
; SCALAR-NEXT: if.then:
-; SCALAR-NEXT: EMIT branch-on-cond ir<%c>
-; SCALAR-NEXT: Successor(s): pred.load.if, pred.load.continue
-; SCALAR-EMPTY:
-; SCALAR-NEXT: pred.load.if:
-; SCALAR-NEXT: CLONE ir<%gep> = getelementptr ir<%ptr>, vp<[[VP6]]>
; SCALAR-NEXT: CLONE ir<%lv> = load ir<%gep>
-; SCALAR-NEXT: Successor(s): pred.load.continue
-; SCALAR-EMPTY:
-; SCALAR-NEXT: pred.load.continue:
-; SCALAR-NEXT: EMIT-SCALAR vp<[[VP9:%[0-9]+]]> = phi [ ir<poison>, if.then ], [ ir<%lv>, pred.load.if ]
-; SCALAR-NEXT: EMIT branch-on-cond ir<%c>
-; SCALAR-NEXT: Successor(s): pred.load.if, pred.load.continue
-; SCALAR-EMPTY:
-; SCALAR-NEXT: pred.load.if:
-; SCALAR-NEXT: CLONE ir<%gep>.1 = getelementptr ir<%ptr>, vp<[[VP7]]>
; SCALAR-NEXT: CLONE ir<%lv>.1 = load ir<%gep>.1
-; SCALAR-NEXT: Successor(s): pred.load.continue
-; SCALAR-EMPTY:
-; SCALAR-NEXT: pred.load.continue:
-; SCALAR-NEXT: EMIT-SCALAR vp<[[VP11:%[0-9]+]]> = phi [ ir<poison>, pred.load.continue ], [ ir<%lv>.1, pred.load.if ]
; SCALAR-NEXT: Successor(s): loop.latch
; SCALAR-EMPTY:
; SCALAR-NEXT: loop.latch:
-; SCALAR-NEXT: WIDEN-PHI vp<[[VP12:%[0-9]+]]> = phi [ ir<poison>, vector.body ], [ vp<[[VP9]]>, pred.load.continue ]
-; SCALAR-NEXT: WIDEN-PHI vp<[[VP13:%[0-9]+]]> = phi [ ir<poison>, vector.body ], [ vp<[[VP11]]>, pred.load.continue ]
-; SCALAR-NEXT: WIDEN-PHI vp<[[VP14:%[0-9]+]]> = phi [ ir<false>, vector.body ], [ ir<%c>, pred.load.continue ]
-; SCALAR-NEXT: WIDEN-PHI vp<[[VP15:%[0-9]+]]> = phi [ ir<false>, vector.body ], [ ir<%c>, pred.load.continue ]
-; SCALAR-NEXT: BLEND ir<%pred.val> = ir<0> vp<%12>/vp<[[VP14]]>
-; SCALAR-NEXT: BLEND ir<%pred.val>.1 = ir<0> vp<%13>/vp<[[VP15]]>
+; SCALAR-NEXT: WIDEN-PHI vp<[[VP8:%[0-9]+]]> = phi [ ir<poison>, vector.body ], [ ir<%lv>, if.then ]
+; SCALAR-NEXT: WIDEN-PHI vp<[[VP9:%[0-9]+]]> = phi [ ir<poison>, vector.body ], [ ir<%lv>.1, if.then ]
+; SCALAR-NEXT: WIDEN-PHI vp<[[VP10:%[0-9]+]]> = phi [ ir<false>, vector.body ], [ ir<true>, if.then ]
+; SCALAR-NEXT: WIDEN-PHI vp<[[VP11:%[0-9]+]]> = phi [ ir<false>, vector.body ], [ ir<true>, if.then ]
+; SCALAR-NEXT: BLEND ir<%pred.val> = ir<0> vp<%8>/vp<[[VP10]]>
+; SCALAR-NEXT: BLEND ir<%pred.val>.1 = ir<0> vp<%9>/vp<[[VP11]]>
; SCALAR-NEXT: CLONE ir<%gep.dst> = getelementptr ir<%dst>, vp<[[VP6]]>
; SCALAR-NEXT: CLONE ir<%gep.dst>.1 = getelementptr ir<%dst>, vp<[[VP7]]>
; SCALAR-NEXT: CLONE store ir<%pred.val>, ir<%gep.dst>
@@ -111,83 +95,39 @@ define void @predicated_load(i1 %c, ptr %ptr, ptr %dst) {
; VECTOR-NEXT: Successor(s): scalar.ph, vector.ph
; VECTOR-EMPTY:
; VECTOR-NEXT: vector.ph:
-; VECTOR-NEXT: EMIT vp<[[VP5:%[0-9]+]]> = broadcast ir<%c>
; VECTOR-NEXT: Successor(s): vector loop
; VECTOR-EMPTY:
; VECTOR-NEXT: <x1> vector loop: {
-; VECTOR-NEXT: vp<[[VP6:%[0-9]+]]> = CANONICAL-IV
+; VECTOR-NEXT: vp<[[VP5:%[0-9]+]]> = CANONICAL-IV
; VECTOR-EMPTY:
; VECTOR-NEXT: vector.body:
-; VECTOR-NEXT: vp<[[VP7:%[0-9]+]]> = SCALAR-STEPS vp<[[VP6]]>, ir<1>, vp<[[VP0]]>
+; VECTOR-NEXT: vp<[[VP6:%[0-9]+]]> = SCALAR-STEPS vp<[[VP5]]>, ir<1>, vp<[[VP0]]>
+; VECTOR-NEXT: CLONE ir<%gep> = getelementptr ir<%ptr>, vp<[[VP6]]>
; VECTOR-NEXT: EMIT branch-on-cond ir<%c> (!vplan.prof.estimated estimated {1073741824, 1073741824})
; VECTOR-NEXT: Successor(s): if.then, loop.latch
; VECTOR-EMPTY:
; VECTOR-NEXT: if.then:
-; VECTOR-NEXT: EMIT branch-on-cond ir<%c>
-; VECTOR-NEXT: Successor(s): pred.load.if, pred.load.continue
-; VECTOR-EMPTY:
-; VECTOR-NEXT: pred.load.if:
-; VECTOR-NEXT: vp<[[VP9:%[0-9]+]]> = SCALAR-STEPS vp<[[VP6]]>, ir<1>, vp<[[VP0]]>
-; VECTOR-NEXT: CLONE ir<%gep> = getelementptr ir<%ptr>, vp<[[VP9]]>
-; VECTOR-NEXT: CLONE ir<%lv> = load ir<%gep>
-; VECTOR-NEXT: EMIT vp<[[VP10:%[0-9]+]]> = insertelement ir<poison>, ir<%lv>, ir<0>
-; VECTOR-NEXT: Successor(s): pred.load.continue
-; VECTOR-EMPTY:
-; VECTOR-NEXT: pred.load.continue:
-; VECTOR-NEXT: WIDEN-PHI vp<[[VP11:%[0-9]+]]> = phi [ ir<poison>, if.then ], [ vp<[[VP10]]>, pred.load.if ]
-; VECTOR-NEXT: EMIT branch-on-cond ir<%c>
-; VECTOR-NEXT: Successor(s): pred.load.if, pred.load.continue
-; VECTOR-EMPTY:
-; VECTOR-NEXT: pred.load.if:
-; VECTOR-NEXT: vp<[[VP13:%[0-9]+]]> = SCALAR-STEPS vp<[[VP6]]>, ir<1>, vp<[[VP0]]>, ir<1>
-; VECTOR-NEXT: CLONE ir<%gep>.1 = getelementptr ir<%ptr>, vp<[[VP13]]>
-; VECTOR-NEXT: CLONE ir<%lv>.1 = load ir<%gep>.1
-; VECTOR-NEXT: EMIT vp<[[VP14:%[0-9]+]]> = insertelement vp<[[VP11]]>, ir<%lv>.1, ir<1>
-; VECTOR-NEXT: Successor(s): pred.load.continue
-; VECTOR-EMPTY:
-; VECTOR-NEXT: pred.load.continue:
-; VECTOR-NEXT: WIDEN-PHI vp<[[VP15:%[0-9]+]]> = phi [ vp<[[VP11]]>, pred.load.continue ], [ vp<[[VP14]]>, pred.load.if ]
-; VECTOR-NEXT: EMIT branch-on-cond ir<%c>
-; VECTOR-NEXT: Successor(s): pred.load.if, pred.load.continue
-; VECTOR-EMPTY:
-; VECTOR-NEXT: pred.load.if:
-; VECTOR-NEXT: vp<[[VP17:%[0-9]+]]> = SCALAR-STEPS vp<[[VP6]]>, ir<1>, vp<[[VP0]]>, vp<[[VP0]]>
-; VECTOR-NEXT: CLONE ir<%gep>.2 = getelementptr ir<%ptr>, vp<[[VP17]]>
-; VECTOR-NEXT: CLONE ir<%lv>.2 = load ir<%gep>.2
-; VECTOR-NEXT: EMIT vp<[[VP18:%[0-9]+]]> = insertelement ir<poison>, ir<%lv>.2, ir<0>
-; VECTOR-NEXT: Successor(s): pred.load.continue
-; VECTOR-EMPTY:
-; VECTOR-NEXT: pred.load.continue:
-; VECTOR-NEXT: WIDEN-PHI vp<[[VP19:%[0-9]+]]> = phi [ ir<poison>, pred.load.continue ], [ vp<[[VP18]]>, pred.load.if ]
-; VECTOR-NEXT: EMIT branch-on-cond ir<%c>
-; VECTOR-NEXT: Successor(s): pred.load.if, pred.load.continue
-; VECTOR-EMPTY:
-; VECTOR-NEXT: pred.load.if:
-; VECTOR-NEXT: EMIT vp<[[VP21:%[0-9]+]]> = add vp<[[VP0]]>, ir<1>
-; VECTOR-NEXT: vp<[[VP22:%[0-9]+]]> = SCALAR-STEPS vp<[[VP6]]>, ir<1>, vp<[[VP0]]>, vp<[[VP21]]>
-; VECTOR-NEXT: CLONE ir<%gep>.3 = getelementptr ir<%ptr>, vp<[[VP22]]>
-; VECTOR-NEXT: CLONE ir<%lv>.3 = load ir<%gep>.3
-; VECTOR-NEXT: EMIT vp<[[VP23:%[0-9]+]]> = insertelement vp<[[VP19]]>, ir<%lv>.3, ir<1>
-; VECTOR-NEXT: Successor(s): pred.load.continue
-; VECTOR-EMPTY:
-; VECTOR-NEXT: pred.load.continue:
-; VECTOR-NEXT: WIDEN-PHI vp<[[VP24:%[0-9]+]]> = phi [ vp<[[VP19]]>, pred.load.continue ], [ vp<[[VP23]]>, pred.load.if ]
+; VECTOR-NEXT: EMIT vp<[[VP7:%[0-9]+]]> = mul nuw nsw vp<[[VP0]]>, ir<1>
+; VECTOR-NEXT: vp<[[VP8:%[0-9]+]]> = vector-pointer i8, ir<%gep>, ir<1>
+; VECTOR-NEXT: vp<[[VP9:%[0-9]+]]> = vector-pointer i8, ir<%gep>, ir<1>, vp<[[VP7]]>
+; VECTOR-NEXT: WIDEN ir<%lv> = load vp<[[VP8]]>
+; VECTOR-NEXT: WIDEN ir<%lv>.1 = load vp<[[VP9]]>
; VECTOR-NEXT: Successor(s): loop.latch
; VECTOR-EMPTY:
; VECTOR-NEXT: loop.latch:
-; VECTOR-NEXT: WIDEN-PHI vp<[[VP25:%[0-9]+]]> = phi [ ir<poison>, vector.body ], [ vp<[[VP15]]>, pred.load.continue ]
-; VECTOR-NEXT: WIDEN-PHI vp<[[VP26:%[0-9]+]]> = phi [ ir<poison>, vector.body ], [ vp<[[VP24]]>, pred.load.continue ]
-; VECTOR-NEXT: WIDEN-PHI vp<[[VP27:%[0-9]+]]> = phi [ ir<false>, vector.body ], [ vp<[[VP5]]>, pred.load.continue ]
-; VECTOR-NEXT: WIDEN-PHI vp<[[VP28:%[0-9]+]]> = phi [ ir<false>, vector.body ], [ vp<[[VP5]]>, pred.load.continue ]
-; VECTOR-NEXT: BLEND ir<%pred.val> = ir<0> vp<%25>/vp<[[VP27]]>
-; VECTOR-NEXT: BLEND ir<%pred.val>.1 = ir<0> vp<%26>/vp<[[VP28]]>
-; VECTOR-NEXT: CLONE ir<%gep.dst> = getelementptr ir<%dst>, vp<[[VP7]]>
-; VECTOR-NEXT: EMIT vp<[[VP29:%[0-9]+]]> = mul nuw nsw vp<[[VP0]]>, ir<1>
-; VECTOR-NEXT: vp<[[VP30:%[0-9]+]]> = vector-pointer i8, ir<%gep.dst>, ir<1>
-; VECTOR-NEXT: vp<[[VP31:%[0-9]+]]> = vector-pointer i8, ir<%gep.dst>, ir<1>, vp<[[VP29]]>
-; VECTOR-NEXT: WIDEN store vp<[[VP30]]>, ir<%pred.val>
-; VECTOR-NEXT: WIDEN store vp<[[VP31]]>, ir<%pred.val>.1
-; VECTOR-NEXT: EMIT vp<%index.next> = add nuw vp<[[VP6]]>, vp<[[VP1]]>
+; VECTOR-NEXT: WIDEN-PHI vp<[[VP10:%[0-9]+]]> = phi [ ir<poison>, vector.body ], [ ir<%lv>, if.then ]
+; VECTOR-NEXT: WIDEN-PHI vp<[[VP11:%[0-9]+]]> = phi [ ir<poison>, vector.body ], [ ir<%lv>.1, if.then ]
+; VECTOR-NEXT: WIDEN-PHI vp<[[VP12:%[0-9]+]]> = phi [ ir<false>, vector.body ], [ ir<true>, if.then ]
+; VECTOR-NEXT: WIDEN-PHI vp<[[VP13:%[0-9]+]]> = phi [ ir<false>, vector.body ], [ ir<true>, if.then ]
+; VECTOR-NEXT: BLEND ir<%pred.val> = ir<0> vp<%10>/vp<[[VP12]]>
+; VECTOR-NEXT: BLEND ir<%pred.val>.1 = ir<0> vp<%11>/vp<[[VP13]]>
+; VECTOR-NEXT: CLONE ir<%gep.dst> = getelementptr ir<%dst>, vp<[[VP6]]>
+; VECTOR-NEXT: EMIT vp<[[VP14:%[0-9]+]]> = mul nuw nsw vp<[[VP0]]>, ir<1>
+; VECTOR-NEXT: vp<[[VP15:%[0-9]+]]> = vector-pointer i8, ir<%gep.dst>, ir<1>
+; VECTOR-NEXT: vp<[[VP16:%[0-9]+]]> = vector-pointer i8, ir<%gep.dst>, ir<1>, vp<[[VP14]]>
+; VECTOR-NEXT: WIDEN store vp<[[VP15]]>, ir<%pred.val>
+; VECTOR-NEXT: WIDEN store vp<[[VP16]]>, ir<%pred.val>.1
+; VECTOR-NEXT: EMIT vp<%index.next> = add nuw vp<[[VP5]]>, vp<[[VP1]]>
; VECTOR-NEXT: EMIT branch-on-count vp<%index.next>, vp<[[VP2]]>
; VECTOR-NEXT: No successors
; VECTOR-NEXT: }
diff --git a/llvm/test/Transforms/LoopVectorize/VPlan/memory-checks.ll b/llvm/test/Transforms/LoopVectorize/VPlan/memory-checks.ll
index 344ccf7d03685..d35e1fba3905a 100644
--- a/llvm/test/Transforms/LoopVectorize/VPlan/memory-checks.ll
+++ b/llvm/test/Transforms/LoopVectorize/VPlan/memory-checks.ll
@@ -215,13 +215,9 @@ define void @bound_is_addrec_of_sibling_loop(ptr %a, ptr %b, i64 %n, i64 %d, i1
; CHECK-NEXT: vector.ph:
; CHECK-NEXT: EMIT vp<[[VP5:%[0-9]+]]> = and ir<%n>, ir<3>
; CHECK-NEXT: EMIT vp<%n.vec> = sub ir<%n>, vp<[[VP5]]>
-; CHECK-NEXT: EMIT vp<[[VP6:%[0-9]+]]> = broadcast ir<%d>
-; CHECK-NEXT: EMIT vp<[[VP7:%[0-9]+]]> = broadcast ir<%n>
-; CHECK-NEXT: EMIT vp<[[VP8:%[0-9]+]]> = broadcast ir<%cond>
-; CHECK-NEXT: WIDEN-INTRINSIC vp<[[VP9:%[0-9]+]]> = call llvm.masked.udiv(vp<[[VP7]]>, vp<[[VP6]]>, vp<[[VP8]]>)
-; CHECK-NEXT: EMIT vp<[[VP10:%[0-9]+]]> = broadcast ir<%iv.1.lcssa>
-; CHECK-NEXT: EMIT vp<[[VP11:%[0-9]+]]> = step-vector i64
-; CHECK-NEXT: EMIT vp<[[VP12:%[0-9]+]]> = broadcast ir<4>
+; CHECK-NEXT: CLONE ir<%q> = udiv ir<%n>, ir<%d>
+; CHECK-NEXT: EMIT vp<[[VP6:%[0-9]+]]> = step-vector i64
+; CHECK-NEXT: EMIT vp<[[VP7:%[0-9]+]]> = broadcast ir<4>
; CHECK-NEXT: Successor(s): vector.body
; CHECK-EMPTY:
; CHECK-NEXT: vector.body:
diff --git a/llvm/test/Transforms/LoopVectorize/VPlan/phi-with-fastflags-vplan.ll b/llvm/test/Transforms/LoopVectorize/VPlan/phi-with-fastflags-vplan.ll
index af349f102c2d2..f496e7132ab8c 100644
--- a/llvm/test/Transforms/LoopVectorize/VPlan/phi-with-fastflags-vplan.ll
+++ b/llvm/test/Transforms/LoopVectorize/VPlan/phi-with-fastflags-vplan.ll
@@ -23,14 +23,14 @@ define void @f(ptr noalias %p, i1 %c) {
; CHECK-NEXT: CLONE ir<%gep> = getelementptr ir<%p>, vp<[[VP4]]>
; CHECK-NEXT: vp<[[VP5:%[0-9]+]]> = vector-pointer float, ir<%gep>, ir<1>
; CHECK-NEXT: WIDEN ir<%x> = load vp<[[VP5]]>
-; CHECK-NEXT: EMIT branch-on-cond ir<%c>
+; CHECK-NEXT: EMIT branch-on-cond ir<%c> (!vplan.prof.estimated estimated {1073741824, 1073741824})
; CHECK-NEXT: Successor(s): then, latch
; CHECK-EMPTY:
; CHECK-NEXT: then:
; CHECK-NEXT: Successor(s): latch
; CHECK-EMPTY:
; CHECK-NEXT: latch:
-; CHECK-NEXT: WIDEN-PHI vp<[[VP6:%[0-9]+]]> = phi [ ir<false>, vector.body ], [ ir<%c>, then ]
+; CHECK-NEXT: WIDEN-PHI vp<[[VP6:%[0-9]+]]> = phi [ ir<false>, vector.body ], [ ir<true>, then ]
; CHECK-NEXT: BLEND ir<%phi> = fast ir<%x> ir<0.000000e+00>/vp<[[VP6]]>
; CHECK-NEXT: vp<[[VP7:%[0-9]+]]> = vector-pointer float, ir<%gep>, ir<1>
; CHECK-NEXT: WIDEN store vp<[[VP7]]>, ir<%phi>
diff --git a/llvm/test/Transforms/LoopVectorize/VPlan/predicator.ll b/llvm/test/Transforms/LoopVectorize/VPlan/predicator.ll
index ccc7368cafd69..27a02568bd123 100644
--- a/llvm/test/Transforms/LoopVectorize/VPlan/predicator.ll
+++ b/llvm/test/Transforms/LoopVectorize/VPlan/predicator.ll
@@ -777,18 +777,16 @@ define void @simplifiable_blend(i1 %c1, i1 %c2, i1 %c3, i32 %x, i32 %y, ptr %p)
; CHECK-NEXT: Successor(s): A, B
; CHECK-EMPTY:
; CHECK-NEXT: A:
-; CHECK-NEXT: EMIT vp<[[VP8:%[0-9]+]]> = freeze ir<%c2>
-; CHECK-NEXT: EMIT branch-on-cond vp<[[VP8]]>
+; CHECK-NEXT: EMIT branch-on-cond ir<%c2>
; CHECK-NEXT: Successor(s): C, D
; CHECK-EMPTY:
; CHECK-NEXT: C:
-; CHECK-NEXT: EMIT vp<[[VP11:%[0-9]+]]> = logical-and ir<%c1>, ir<%c2>
; CHECK-NEXT: Successor(s): latch
; CHECK-EMPTY:
; CHECK-NEXT: latch:
-; CHECK-NEXT: WIDEN-PHI vp<[[VP12:%[0-9]+]]> = phi [ vp<[[VP4:%[0-9]+]]>, F ], [ ir<false>, D ], [ ir<false>, C ]
-; CHECK-NEXT: WIDEN-PHI vp<[[VP13:%[0-9]+]]> = phi [ ir<false>, F ], [ ir<%c1>, D ], [ ir<%c1>, C ]
-; CHECK-NEXT: BLEND ir<%phi> = ir<%y>/vp<[[VP12]]> ir<%x>/vp<[[VP13]]>
+; CHECK-NEXT: WIDEN-PHI vp<[[VP4:%[0-9]+]]> = phi [ ir<true>, F ], [ ir<false>, D ], [ ir<false>, C ]
+; CHECK-NEXT: WIDEN-PHI vp<[[VP5:%[0-9]+]]> = phi [ ir<false>, F ], [ ir<true>, D ], [ ir<true>, C ]
+; CHECK-NEXT: BLEND ir<%phi> = ir<%y>/vp<[[VP4]]> ir<%x>/vp<[[VP5]]>
; CHECK-NEXT: EMIT ir<%gep> = getelementptr ir<%p>, ir<%iv>
; CHECK-NEXT: EMIT store ir<%phi>, ir<%gep>
; CHECK-NEXT: EMIT ir<%iv.next> = add ir<%iv>, ir<1>
@@ -798,22 +796,16 @@ define void @simplifiable_blend(i1 %c1, i1 %c2, i1 %c3, i32 %x, i32 %y, ptr %p)
; CHECK-NEXT: No successors
; CHECK-EMPTY:
; CHECK-NEXT: D:
-; CHECK-NEXT: EMIT vp<[[VP9:%[0-9]+]]> = not ir<%c2>
-; CHECK-NEXT: EMIT vp<[[VP10:%[0-9]+]]> = logical-and ir<%c1>, vp<[[VP9]]>
; CHECK-NEXT: Successor(s): latch
; CHECK-EMPTY:
; CHECK-NEXT: B:
-; CHECK-NEXT: EMIT vp<[[VP4]]> = not ir<%c1>
-; CHECK-NEXT: EMIT vp<[[VP5:%[0-9]+]]> = freeze ir<%c3>
-; CHECK-NEXT: EMIT branch-on-cond vp<[[VP5]]>
+; CHECK-NEXT: EMIT branch-on-cond ir<%c3>
; CHECK-NEXT: Successor(s): F, E
; CHECK-EMPTY:
; CHECK-NEXT: F:
; CHECK-NEXT: Successor(s): latch
; CHECK-EMPTY:
; CHECK-NEXT: E:
-; CHECK-NEXT: EMIT vp<[[VP6:%[0-9]+]]> = not ir<%c3>
-; CHECK-NEXT: EMIT vp<[[VP7:%[0-9]+]]> = logical-and vp<[[VP4]]>, vp<[[VP6]]>
; CHECK-NEXT: Successor(s): F
; CHECK-NEXT: }
; CHECK-NEXT: Successor(s): middle.block
@@ -1000,24 +992,23 @@ define void @outermost_uniform_branch(ptr %a, i1 %u0) {
; CHECK-NEXT: Successor(s): bb1, bb4
; CHECK-EMPTY:
; CHECK-NEXT: bb1:
-; CHECK-NEXT: EMIT ir<%add1> = add ir<%iv>, ir<1>, ir<%u0>
-; CHECK-NEXT: EMIT ir<%v1> = icmp sle ir<%iv>, ir<1>, ir<%u0>
+; CHECK-NEXT: EMIT ir<%add1> = add ir<%iv>, ir<1>
+; CHECK-NEXT: EMIT ir<%v1> = icmp sle ir<%iv>, ir<1>
; CHECK-NEXT: Successor(s): bb2
; CHECK-EMPTY:
; CHECK-NEXT: bb2:
-; CHECK-NEXT: EMIT vp<[[VP4:%[0-9]+]]> = logical-and ir<%u0>, ir<%v1>
-; CHECK-NEXT: EMIT ir<%add2> = add ir<%iv>, ir<2>, vp<[[VP4]]>
+; CHECK-NEXT: EMIT ir<%add2> = add ir<%iv>, ir<2>, ir<%v1>
; CHECK-NEXT: Successor(s): bb3
; CHECK-EMPTY:
; CHECK-NEXT: bb3:
; CHECK-NEXT: BLEND ir<%phi3> = ir<%add1>/ir<true> ir<%add2>/ir<%v1>
-; CHECK-NEXT: EMIT ir<%add3> = add ir<%phi3>, ir<3>, ir<%u0>
+; CHECK-NEXT: EMIT ir<%add3> = add ir<%phi3>, ir<3>
; CHECK-NEXT: Successor(s): bb4
; CHECK-EMPTY:
; CHECK-NEXT: bb4:
-; CHECK-NEXT: WIDEN-PHI vp<[[VP5:%[0-9]+]]> = phi [ ir<poison>, vector.body ], [ ir<%add3>, bb3 ]
-; CHECK-NEXT: WIDEN-PHI vp<[[VP6:%[0-9]+]]> = phi [ ir<false>, vector.body ], [ ir<%u0>, bb3 ]
-; CHECK-NEXT: BLEND ir<%phi4> = ir<%iv>/ir<true> vp<%5>/vp<[[VP6]]>
+; CHECK-NEXT: WIDEN-PHI vp<[[VP4:%[0-9]+]]> = phi [ ir<poison>, vector.body ], [ ir<%add3>, bb3 ]
+; CHECK-NEXT: WIDEN-PHI vp<[[VP5:%[0-9]+]]> = phi [ ir<false>, vector.body ], [ ir<true>, bb3 ]
+; CHECK-NEXT: BLEND ir<%phi4> = ir<%iv>/ir<true> vp<%4>/vp<[[VP5]]>
; CHECK-NEXT: EMIT store ir<%phi4>, ir<%a>
; CHECK-NEXT: EMIT ir<%iv.next> = add nuw nsw ir<%iv>, ir<1>
; CHECK-NEXT: EMIT ir<%ec> = icmp eq ir<%iv.next>, ir<128>
@@ -1083,15 +1074,14 @@ define void @outermost_uniform_branch_no_phi(ptr %a, i1 %u0) {
; CHECK-NEXT: Successor(s): bb1, bb4
; CHECK-EMPTY:
; CHECK-NEXT: bb1:
-; CHECK-NEXT: EMIT ir<%add1> = add ir<%iv>, ir<1>, ir<%u0>
-; CHECK-NEXT: EMIT store ir<%add1>, ir<%gep>, ir<%u0>
-; CHECK-NEXT: EMIT ir<%v1> = icmp sle ir<%iv>, ir<1>, ir<%u0>
+; CHECK-NEXT: EMIT ir<%add1> = add ir<%iv>, ir<1>
+; CHECK-NEXT: EMIT store ir<%add1>, ir<%gep>
+; CHECK-NEXT: EMIT ir<%v1> = icmp sle ir<%iv>, ir<1>
; CHECK-NEXT: Successor(s): bb2
; CHECK-EMPTY:
; CHECK-NEXT: bb2:
-; CHECK-NEXT: EMIT vp<[[VP4:%[0-9]+]]> = logical-and ir<%u0>, ir<%v1>
-; CHECK-NEXT: EMIT ir<%add2> = add ir<%iv>, ir<2>, vp<[[VP4]]>
-; CHECK-NEXT: EMIT store ir<%add2>, ir<%gep>, vp<[[VP4]]>
+; CHECK-NEXT: EMIT ir<%add2> = add ir<%iv>, ir<2>, ir<%v1>
+; CHECK-NEXT: EMIT store ir<%add2>, ir<%gep>, ir<%v1>
; CHECK-NEXT: Successor(s): bb3
; CHECK-EMPTY:
; CHECK-NEXT: bb3:
@@ -1160,31 +1150,29 @@ define void @outermost_uniform_branch_more_blocks(ptr %a, i1 %u0) {
; CHECK-NEXT: Successor(s): bb2, bb1
; CHECK-EMPTY:
; CHECK-NEXT: bb2:
-; CHECK-NEXT: EMIT ir<%v2> = icmp sle ir<%iv>, ir<2>, ir<%u0>
+; CHECK-NEXT: EMIT ir<%v2> = icmp sle ir<%iv>, ir<2>
; CHECK-NEXT: Successor(s): bb3
; CHECK-EMPTY:
; CHECK-NEXT: bb3:
-; CHECK-NEXT: EMIT vp<[[VP5:%[0-9]+]]> = not ir<%v2>
-; CHECK-NEXT: EMIT vp<[[VP6:%[0-9]+]]> = logical-and ir<%u0>, vp<[[VP5]]>
-; CHECK-NEXT: EMIT ir<%add3> = add ir<%iv>, ir<3>, vp<[[VP6]]>
+; CHECK-NEXT: EMIT vp<[[VP4:%[0-9]+]]> = not ir<%v2>
+; CHECK-NEXT: EMIT ir<%add3> = add ir<%iv>, ir<3>, vp<[[VP4]]>
; CHECK-NEXT: Successor(s): bb4
; CHECK-EMPTY:
; CHECK-NEXT: bb4:
-; CHECK-NEXT: EMIT vp<[[VP7:%[0-9]+]]> = logical-and ir<%u0>, ir<%v2>
-; CHECK-NEXT: EMIT ir<%add4> = add ir<%iv>, ir<4>, vp<[[VP7]]>
+; CHECK-NEXT: EMIT ir<%add4> = add ir<%iv>, ir<4>, ir<%v2>
; CHECK-NEXT: Successor(s): bb5
; CHECK-EMPTY:
; CHECK-NEXT: bb5:
-; CHECK-NEXT: BLEND ir<%phi5> = ir<%add3>/vp<[[VP5]]> ir<%add4>/ir<%v2>
-; CHECK-NEXT: EMIT ir<%add5> = add ir<%phi5>, ir<5>, ir<%u0>
+; CHECK-NEXT: BLEND ir<%phi5> = ir<%add3>/vp<[[VP4]]> ir<%add4>/ir<%v2>
+; CHECK-NEXT: EMIT ir<%add5> = add ir<%phi5>, ir<5>
; CHECK-NEXT: Successor(s): bb6
; CHECK-EMPTY:
; CHECK-NEXT: bb6:
-; CHECK-NEXT: WIDEN-PHI vp<[[VP8:%[0-9]+]]> = phi [ ir<%add1>, bb1 ], [ ir<poison>, bb5 ]
-; CHECK-NEXT: WIDEN-PHI vp<[[VP9:%[0-9]+]]> = phi [ vp<[[VP4:%[0-9]+]]>, bb1 ], [ ir<false>, bb5 ]
-; CHECK-NEXT: WIDEN-PHI vp<[[VP10:%[0-9]+]]> = phi [ ir<poison>, bb1 ], [ ir<%add5>, bb5 ]
-; CHECK-NEXT: WIDEN-PHI vp<[[VP11:%[0-9]+]]> = phi [ ir<false>, bb1 ], [ ir<%u0>, bb5 ]
-; CHECK-NEXT: BLEND ir<%phi6> = vp<%8>/vp<[[VP9]]> vp<%10>/vp<[[VP11]]>
+; CHECK-NEXT: WIDEN-PHI vp<[[VP5:%[0-9]+]]> = phi [ ir<%add1>, bb1 ], [ ir<poison>, bb5 ]
+; CHECK-NEXT: WIDEN-PHI vp<[[VP6:%[0-9]+]]> = phi [ ir<true>, bb1 ], [ ir<false>, bb5 ]
+; CHECK-NEXT: WIDEN-PHI vp<[[VP7:%[0-9]+]]> = phi [ ir<poison>, bb1 ], [ ir<%add5>, bb5 ]
+; CHECK-NEXT: WIDEN-PHI vp<[[VP8:%[0-9]+]]> = phi [ ir<false>, bb1 ], [ ir<true>, bb5 ]
+; CHECK-NEXT: BLEND ir<%phi6> = vp<%5>/vp<[[VP6]]> vp<%7>/vp<[[VP8]]>
; CHECK-NEXT: EMIT store ir<%phi6>, ir<%a>
; CHECK-NEXT: EMIT ir<%iv.next> = add nuw nsw ir<%iv>, ir<1>
; CHECK-NEXT: EMIT ir<%ec> = icmp eq ir<%iv.next>, ir<128>
@@ -1193,8 +1181,7 @@ define void @outermost_uniform_branch_more_blocks(ptr %a, i1 %u0) {
; CHECK-NEXT: No successors
; CHECK-EMPTY:
; CHECK-NEXT: bb1:
-; CHECK-NEXT: EMIT vp<[[VP4]]> = not ir<%u0>
-; CHECK-NEXT: EMIT ir<%add1> = add ir<%iv>, ir<1>, vp<[[VP4]]>
+; CHECK-NEXT: EMIT ir<%add1> = add ir<%iv>, ir<1>
; CHECK-NEXT: Successor(s): bb6
; CHECK-NEXT: }
; CHECK-NEXT: Successor(s): middle.block
@@ -1261,20 +1248,18 @@ define void @outermost_uniform_branch_more_blocks_no_phi(ptr %a, i1 %u0) {
; CHECK-NEXT: Successor(s): bb2, bb1
; CHECK-EMPTY:
; CHECK-NEXT: bb2:
-; CHECK-NEXT: EMIT ir<%v2> = icmp sle ir<%iv>, ir<2>, ir<%u0>
+; CHECK-NEXT: EMIT ir<%v2> = icmp sle ir<%iv>, ir<2>
; CHECK-NEXT: Successor(s): bb3
; CHECK-EMPTY:
; CHECK-NEXT: bb3:
-; CHECK-NEXT: EMIT vp<[[VP5:%[0-9]+]]> = not ir<%v2>
-; CHECK-NEXT: EMIT vp<[[VP6:%[0-9]+]]> = logical-and ir<%u0>, vp<[[VP5]]>
-; CHECK-NEXT: EMIT ir<%add3> = add ir<%iv>, ir<3>, vp<[[VP6]]>
-; CHECK-NEXT: EMIT store ir<%add3>, ir<%gep>, vp<[[VP6]]>
+; CHECK-NEXT: EMIT vp<[[VP4:%[0-9]+]]> = not ir<%v2>
+; CHECK-NEXT: EMIT ir<%add3> = add ir<%iv>, ir<3>, vp<[[VP4]]>
+; CHECK-NEXT: EMIT store ir<%add3>, ir<%gep>, vp<[[VP4]]>
; CHECK-NEXT: Successor(s): bb4
; CHECK-EMPTY:
; CHECK-NEXT: bb4:
-; CHECK-NEXT: EMIT vp<[[VP7:%[0-9]+]]> = logical-and ir<%u0>, ir<%v2>
-; CHECK-NEXT: EMIT ir<%add4> = add ir<%iv>, ir<4>, vp<[[VP7]]>
-; CHECK-NEXT: EMIT store ir<%add4>, ir<%gep>, vp<[[VP7]]>
+; CHECK-NEXT: EMIT ir<%add4> = add ir<%iv>, ir<4>, ir<%v2>
+; CHECK-NEXT: EMIT store ir<%add4>, ir<%gep>, ir<%v2>
; CHECK-NEXT: Successor(s): bb5
; CHECK-EMPTY:
; CHECK-NEXT: bb5:
@@ -1288,9 +1273,8 @@ define void @outermost_uniform_branch_more_blocks_no_phi(ptr %a, i1 %u0) {
; CHECK-NEXT: No successors
; CHECK-EMPTY:
; CHECK-NEXT: bb1:
-; CHECK-NEXT: EMIT vp<[[VP4:%[0-9]+]]> = not ir<%u0>
-; CHECK-NEXT: EMIT ir<%add1> = add ir<%iv>, ir<1>, vp<[[VP4]]>
-; CHECK-NEXT: EMIT store ir<%add1>, ir<%gep>, vp<[[VP4]]>
+; CHECK-NEXT: EMIT ir<%add1> = add ir<%iv>, ir<1>
+; CHECK-NEXT: EMIT store ir<%add1>, ir<%gep>
; CHECK-NEXT: Successor(s): bb6
; CHECK-NEXT: }
; CHECK-NEXT: Successor(s): middle.block
@@ -1363,14 +1347,13 @@ define void @uniform_branch_after_varying_branch(ptr %a, i1 %u1) {
; CHECK-NEXT: Successor(s): bb2, bb3
; CHECK-EMPTY:
; CHECK-NEXT: bb2:
-; CHECK-NEXT: EMIT vp<[[VP6:%[0-9]+]]> = logical-and vp<[[VP4]]>, ir<%u1>
-; CHECK-NEXT: EMIT ir<%add2> = add ir<%iv>, ir<2>, vp<[[VP6]]>
+; CHECK-NEXT: EMIT ir<%add2> = add ir<%iv>, ir<2>, vp<[[VP4]]>
; CHECK-NEXT: Successor(s): bb3
; CHECK-EMPTY:
; CHECK-NEXT: bb3:
-; CHECK-NEXT: WIDEN-PHI vp<[[VP7:%[0-9]+]]> = phi [ ir<poison>, bb1 ], [ ir<%add2>, bb2 ]
-; CHECK-NEXT: WIDEN-PHI vp<[[VP8:%[0-9]+]]> = phi [ ir<false>, bb1 ], [ ir<%u1>, bb2 ]
-; CHECK-NEXT: BLEND ir<%phi3> = ir<%add1>/ir<true> vp<%7>/vp<[[VP8]]>
+; CHECK-NEXT: WIDEN-PHI vp<[[VP6:%[0-9]+]]> = phi [ ir<poison>, bb1 ], [ ir<%add2>, bb2 ]
+; CHECK-NEXT: WIDEN-PHI vp<[[VP7:%[0-9]+]]> = phi [ ir<false>, bb1 ], [ ir<true>, bb2 ]
+; CHECK-NEXT: BLEND ir<%phi3> = ir<%add1>/ir<true> vp<%6>/vp<[[VP7]]>
; CHECK-NEXT: EMIT ir<%add3> = add ir<%phi3>, ir<3>, vp<[[VP4]]>
; CHECK-NEXT: Successor(s): bb4
; CHECK-EMPTY:
@@ -1454,9 +1437,8 @@ define void @uniform_branch_after_varying_branch_no_phi(ptr %a, i1 %u1) {
; CHECK-NEXT: Successor(s): bb2, bb3
; CHECK-EMPTY:
; CHECK-NEXT: bb2:
-; CHECK-NEXT: EMIT vp<[[VP6:%[0-9]+]]> = logical-and vp<[[VP4]]>, ir<%u1>
-; CHECK-NEXT: EMIT ir<%add2> = add ir<%iv>, ir<2>, vp<[[VP6]]>
-; CHECK-NEXT: EMIT store ir<%add2>, ir<%gep>, vp<[[VP6]]>
+; CHECK-NEXT: EMIT ir<%add2> = add ir<%iv>, ir<2>, vp<[[VP4]]>
+; CHECK-NEXT: EMIT store ir<%add2>, ir<%gep>, vp<[[VP4]]>
; CHECK-NEXT: Successor(s): bb3
; CHECK-EMPTY:
; CHECK-NEXT: bb3:
@@ -1538,16 +1520,15 @@ define void @uniform_branch_after_varying_branch_more_blocks(ptr %a, i1 %u1) {
; CHECK-NEXT: Successor(s): bb3, bb2
; CHECK-EMPTY:
; CHECK-NEXT: bb3:
-; CHECK-NEXT: EMIT vp<[[VP8:%[0-9]+]]> = logical-and vp<[[VP4]]>, ir<%u1>
-; CHECK-NEXT: EMIT ir<%add3> = add ir<%iv>, ir<3>, vp<[[VP8]]>
+; CHECK-NEXT: EMIT ir<%add3> = add ir<%iv>, ir<3>, vp<[[VP4]]>
; CHECK-NEXT: Successor(s): bb4
; CHECK-EMPTY:
; CHECK-NEXT: bb4:
-; CHECK-NEXT: WIDEN-PHI vp<[[VP9:%[0-9]+]]> = phi [ ir<%add2>, bb2 ], [ ir<poison>, bb3 ]
-; CHECK-NEXT: WIDEN-PHI vp<[[VP10:%[0-9]+]]> = phi [ vp<[[VP6:%[0-9]+]]>, bb2 ], [ ir<false>, bb3 ]
-; CHECK-NEXT: WIDEN-PHI vp<[[VP11:%[0-9]+]]> = phi [ ir<poison>, bb2 ], [ ir<%add3>, bb3 ]
-; CHECK-NEXT: WIDEN-PHI vp<[[VP12:%[0-9]+]]> = phi [ ir<false>, bb2 ], [ ir<%u1>, bb3 ]
-; CHECK-NEXT: BLEND ir<%phi4> = vp<%9>/vp<[[VP10]]> vp<%11>/vp<[[VP12]]>
+; CHECK-NEXT: WIDEN-PHI vp<[[VP6:%[0-9]+]]> = phi [ ir<%add2>, bb2 ], [ ir<poison>, bb3 ]
+; CHECK-NEXT: WIDEN-PHI vp<[[VP7:%[0-9]+]]> = phi [ ir<true>, bb2 ], [ ir<false>, bb3 ]
+; CHECK-NEXT: WIDEN-PHI vp<[[VP8:%[0-9]+]]> = phi [ ir<poison>, bb2 ], [ ir<%add3>, bb3 ]
+; CHECK-NEXT: WIDEN-PHI vp<[[VP9:%[0-9]+]]> = phi [ ir<false>, bb2 ], [ ir<true>, bb3 ]
+; CHECK-NEXT: BLEND ir<%phi4> = vp<%6>/vp<[[VP7]]> vp<%8>/vp<[[VP9]]>
; CHECK-NEXT: EMIT ir<%add4> = add ir<%phi4>, ir<4>, vp<[[VP4]]>
; CHECK-NEXT: Successor(s): bb5
; CHECK-EMPTY:
@@ -1565,9 +1546,7 @@ define void @uniform_branch_after_varying_branch_more_blocks(ptr %a, i1 %u1) {
; CHECK-NEXT: No successors
; CHECK-EMPTY:
; CHECK-NEXT: bb2:
-; CHECK-NEXT: EMIT vp<[[VP6]]> = not ir<%u1>
-; CHECK-NEXT: EMIT vp<[[VP7:%[0-9]+]]> = logical-and vp<[[VP4]]>, vp<[[VP6]]>
-; CHECK-NEXT: EMIT ir<%add2> = add ir<%iv>, ir<2>, vp<[[VP7]]>
+; CHECK-NEXT: EMIT ir<%add2> = add ir<%iv>, ir<2>, vp<[[VP4]]>
; CHECK-NEXT: Successor(s): bb4
; CHECK-NEXT: }
; CHECK-NEXT: Successor(s): middle.block
@@ -1640,9 +1619,8 @@ define void @uniform_branch_after_varying_branch_more_blocks_no_phi(ptr %a, i1 %
; CHECK-NEXT: Successor(s): bb3, bb2
; CHECK-EMPTY:
; CHECK-NEXT: bb3:
-; CHECK-NEXT: EMIT vp<[[VP8:%[0-9]+]]> = logical-and vp<[[VP4]]>, ir<%u1>
-; CHECK-NEXT: EMIT ir<%add3> = add ir<%iv>, ir<3>, vp<[[VP8]]>
-; CHECK-NEXT: EMIT store ir<%add3>, ir<%gep>, vp<[[VP8]]>
+; CHECK-NEXT: EMIT ir<%add3> = add ir<%iv>, ir<3>, vp<[[VP4]]>
+; CHECK-NEXT: EMIT store ir<%add3>, ir<%gep>, vp<[[VP4]]>
; CHECK-NEXT: Successor(s): bb4
; CHECK-EMPTY:
; CHECK-NEXT: bb4:
@@ -1661,10 +1639,8 @@ define void @uniform_branch_after_varying_branch_more_blocks_no_phi(ptr %a, i1 %
; CHECK-NEXT: No successors
; CHECK-EMPTY:
; CHECK-NEXT: bb2:
-; CHECK-NEXT: EMIT vp<[[VP6:%[0-9]+]]> = not ir<%u1>
-; CHECK-NEXT: EMIT vp<[[VP7:%[0-9]+]]> = logical-and vp<[[VP4]]>, vp<[[VP6]]>
-; CHECK-NEXT: EMIT ir<%add2> = add ir<%iv>, ir<2>, vp<[[VP7]]>
-; CHECK-NEXT: EMIT store ir<%add2>, ir<%gep>, vp<[[VP7]]>
+; CHECK-NEXT: EMIT ir<%add2> = add ir<%iv>, ir<2>, vp<[[VP4]]>
+; CHECK-NEXT: EMIT store ir<%add2>, ir<%gep>, vp<[[VP4]]>
; CHECK-NEXT: Successor(s): bb4
; CHECK-NEXT: }
; CHECK-NEXT: Successor(s): middle.block
@@ -1740,16 +1716,15 @@ define void @uniform_branch_after_varying_branch_more_blocks_mirrored(ptr %a, i1
; CHECK-NEXT: Successor(s): bb4, bb3
; CHECK-EMPTY:
; CHECK-NEXT: bb4:
-; CHECK-NEXT: EMIT vp<[[VP8:%[0-9]+]]> = logical-and ir<%v0>, ir<%u2>
-; CHECK-NEXT: EMIT ir<%add4> = add ir<%iv>, ir<4>, vp<[[VP8]]>
+; CHECK-NEXT: EMIT ir<%add4> = add ir<%iv>, ir<4>, ir<%v0>
; CHECK-NEXT: Successor(s): bb5
; CHECK-EMPTY:
; CHECK-NEXT: bb5:
-; CHECK-NEXT: WIDEN-PHI vp<[[VP9:%[0-9]+]]> = phi [ ir<%add3>, bb3 ], [ ir<poison>, bb4 ]
-; CHECK-NEXT: WIDEN-PHI vp<[[VP10:%[0-9]+]]> = phi [ vp<[[VP6:%[0-9]+]]>, bb3 ], [ ir<false>, bb4 ]
-; CHECK-NEXT: WIDEN-PHI vp<[[VP11:%[0-9]+]]> = phi [ ir<poison>, bb3 ], [ ir<%add4>, bb4 ]
-; CHECK-NEXT: WIDEN-PHI vp<[[VP12:%[0-9]+]]> = phi [ ir<false>, bb3 ], [ ir<%u2>, bb4 ]
-; CHECK-NEXT: BLEND ir<%phi5> = vp<%9>/vp<[[VP10]]> vp<%11>/vp<[[VP12]]>
+; CHECK-NEXT: WIDEN-PHI vp<[[VP6:%[0-9]+]]> = phi [ ir<%add3>, bb3 ], [ ir<poison>, bb4 ]
+; CHECK-NEXT: WIDEN-PHI vp<[[VP7:%[0-9]+]]> = phi [ ir<true>, bb3 ], [ ir<false>, bb4 ]
+; CHECK-NEXT: WIDEN-PHI vp<[[VP8:%[0-9]+]]> = phi [ ir<poison>, bb3 ], [ ir<%add4>, bb4 ]
+; CHECK-NEXT: WIDEN-PHI vp<[[VP9:%[0-9]+]]> = phi [ ir<false>, bb3 ], [ ir<true>, bb4 ]
+; CHECK-NEXT: BLEND ir<%phi5> = vp<%6>/vp<[[VP7]]> vp<%8>/vp<[[VP9]]>
; CHECK-NEXT: EMIT ir<%add5> = add ir<%phi5>, ir<5>, ir<%v0>
; CHECK-NEXT: Successor(s): bb6
; CHECK-EMPTY:
@@ -1763,9 +1738,7 @@ define void @uniform_branch_after_varying_branch_more_blocks_mirrored(ptr %a, i1
; CHECK-NEXT: No successors
; CHECK-EMPTY:
; CHECK-NEXT: bb3:
-; CHECK-NEXT: EMIT vp<[[VP6]]> = not ir<%u2>
-; CHECK-NEXT: EMIT vp<[[VP7:%[0-9]+]]> = logical-and ir<%v0>, vp<[[VP6]]>
-; CHECK-NEXT: EMIT ir<%add3> = add ir<%iv>, ir<3>, vp<[[VP7]]>
+; CHECK-NEXT: EMIT ir<%add3> = add ir<%iv>, ir<3>, ir<%v0>
; CHECK-NEXT: Successor(s): bb5
; CHECK-NEXT: }
; CHECK-NEXT: Successor(s): middle.block
@@ -1843,9 +1816,8 @@ define void @uniform_branch_after_varying_branch_more_blocks_mirrored_no_phi(ptr
; CHECK-NEXT: Successor(s): bb4, bb3
; CHECK-EMPTY:
; CHECK-NEXT: bb4:
-; CHECK-NEXT: EMIT vp<[[VP8:%[0-9]+]]> = logical-and ir<%v0>, ir<%u2>
-; CHECK-NEXT: EMIT ir<%add4> = add ir<%iv>, ir<4>, vp<[[VP8]]>
-; CHECK-NEXT: EMIT store ir<%add4>, ir<%gep>, vp<[[VP8]]>
+; CHECK-NEXT: EMIT ir<%add4> = add ir<%iv>, ir<4>, ir<%v0>
+; CHECK-NEXT: EMIT store ir<%add4>, ir<%gep>, ir<%v0>
; CHECK-NEXT: Successor(s): bb5
; CHECK-EMPTY:
; CHECK-NEXT: bb5:
@@ -1859,10 +1831,8 @@ define void @uniform_branch_after_varying_branch_more_blocks_mirrored_no_phi(ptr
; CHECK-NEXT: No successors
; CHECK-EMPTY:
; CHECK-NEXT: bb3:
-; CHECK-NEXT: EMIT vp<[[VP6:%[0-9]+]]> = not ir<%u2>
-; CHECK-NEXT: EMIT vp<[[VP7:%[0-9]+]]> = logical-and ir<%v0>, vp<[[VP6]]>
-; CHECK-NEXT: EMIT ir<%add3> = add ir<%iv>, ir<3>, vp<[[VP7]]>
-; CHECK-NEXT: EMIT store ir<%add3>, ir<%gep>, vp<[[VP7]]>
+; CHECK-NEXT: EMIT ir<%add3> = add ir<%iv>, ir<3>, ir<%v0>
+; CHECK-NEXT: EMIT store ir<%add3>, ir<%gep>, ir<%v0>
; CHECK-NEXT: Successor(s): bb5
; CHECK-NEXT: }
; CHECK-NEXT: Successor(s): middle.block
@@ -2860,23 +2830,22 @@ define void @uniform_branch_shared_join_with_varying(ptr %a, i1 %u0) {
; CHECK-NEXT: Successor(s): bb2, bb1
; CHECK-EMPTY:
; CHECK-NEXT: bb2:
-; CHECK-NEXT: EMIT ir<%add2> = add ir<%iv>, ir<2>, ir<%u0>
-; CHECK-NEXT: EMIT ir<%v2> = icmp sle ir<%iv>, ir<2>, ir<%u0>
+; CHECK-NEXT: EMIT ir<%add2> = add ir<%iv>, ir<2>
+; CHECK-NEXT: EMIT ir<%v2> = icmp sle ir<%iv>, ir<2>
; CHECK-NEXT: Successor(s): bb3
; CHECK-EMPTY:
; CHECK-NEXT: bb3:
-; CHECK-NEXT: EMIT vp<[[VP5:%[0-9]+]]> = logical-and ir<%u0>, ir<%v2>
-; CHECK-NEXT: EMIT ir<%add3> = add ir<%iv>, ir<3>, vp<[[VP5]]>
+; CHECK-NEXT: EMIT ir<%add3> = add ir<%iv>, ir<3>, ir<%v2>
; CHECK-NEXT: Successor(s): bb4
; CHECK-EMPTY:
; CHECK-NEXT: bb4:
-; CHECK-NEXT: WIDEN-PHI vp<[[VP6:%[0-9]+]]> = phi [ ir<%add1>, bb1 ], [ ir<poison>, bb3 ]
-; CHECK-NEXT: WIDEN-PHI vp<[[VP7:%[0-9]+]]> = phi [ vp<[[VP4:%[0-9]+]]>, bb1 ], [ ir<false>, bb3 ]
-; CHECK-NEXT: WIDEN-PHI vp<[[VP8:%[0-9]+]]> = phi [ ir<poison>, bb1 ], [ ir<%add2>, bb3 ]
-; CHECK-NEXT: WIDEN-PHI vp<[[VP9:%[0-9]+]]> = phi [ ir<false>, bb1 ], [ ir<%u0>, bb3 ]
-; CHECK-NEXT: WIDEN-PHI vp<[[VP10:%[0-9]+]]> = phi [ ir<poison>, bb1 ], [ ir<%add3>, bb3 ]
-; CHECK-NEXT: WIDEN-PHI vp<[[VP11:%[0-9]+]]> = phi [ ir<false>, bb1 ], [ vp<[[VP5]]>, bb3 ]
-; CHECK-NEXT: BLEND ir<%phi4> = vp<%6>/vp<[[VP7]]> vp<%8>/vp<[[VP9]]> vp<%10>/vp<[[VP11]]>
+; CHECK-NEXT: WIDEN-PHI vp<[[VP4:%[0-9]+]]> = phi [ ir<%add1>, bb1 ], [ ir<poison>, bb3 ]
+; CHECK-NEXT: WIDEN-PHI vp<[[VP5:%[0-9]+]]> = phi [ ir<true>, bb1 ], [ ir<false>, bb3 ]
+; CHECK-NEXT: WIDEN-PHI vp<[[VP6:%[0-9]+]]> = phi [ ir<poison>, bb1 ], [ ir<%add2>, bb3 ]
+; CHECK-NEXT: WIDEN-PHI vp<[[VP7:%[0-9]+]]> = phi [ ir<false>, bb1 ], [ ir<true>, bb3 ]
+; CHECK-NEXT: WIDEN-PHI vp<[[VP8:%[0-9]+]]> = phi [ ir<poison>, bb1 ], [ ir<%add3>, bb3 ]
+; CHECK-NEXT: WIDEN-PHI vp<[[VP9:%[0-9]+]]> = phi [ ir<false>, bb1 ], [ ir<%v2>, bb3 ]
+; CHECK-NEXT: BLEND ir<%phi4> = vp<%4>/vp<[[VP5]]> vp<%6>/vp<[[VP7]]> vp<%8>/vp<[[VP9]]>
; CHECK-NEXT: EMIT store ir<%phi4>, ir<%a>
; CHECK-NEXT: EMIT ir<%iv.next> = add nuw nsw ir<%iv>, ir<1>
; CHECK-NEXT: EMIT ir<%ec> = icmp eq ir<%iv.next>, ir<128>
@@ -2885,8 +2854,7 @@ define void @uniform_branch_shared_join_with_varying(ptr %a, i1 %u0) {
; CHECK-NEXT: No successors
; CHECK-EMPTY:
; CHECK-NEXT: bb1:
-; CHECK-NEXT: EMIT vp<[[VP4]]> = not ir<%u0>
-; CHECK-NEXT: EMIT ir<%add1> = add ir<%iv>, ir<1>, vp<[[VP4]]>
+; CHECK-NEXT: EMIT ir<%add1> = add ir<%iv>, ir<1>
; CHECK-NEXT: Successor(s): bb4
; CHECK-NEXT: }
; CHECK-NEXT: Successor(s): middle.block
@@ -2942,15 +2910,14 @@ define void @uniform_branch_shared_join_with_varying_no_phi(ptr %a, i1 %u0) {
; CHECK-NEXT: Successor(s): bb2, bb1
; CHECK-EMPTY:
; CHECK-NEXT: bb2:
-; CHECK-NEXT: EMIT ir<%add2> = add ir<%iv>, ir<2>, ir<%u0>
-; CHECK-NEXT: EMIT store ir<%add2>, ir<%gep>, ir<%u0>
-; CHECK-NEXT: EMIT ir<%v2> = icmp sle ir<%iv>, ir<2>, ir<%u0>
+; CHECK-NEXT: EMIT ir<%add2> = add ir<%iv>, ir<2>
+; CHECK-NEXT: EMIT store ir<%add2>, ir<%gep>
+; CHECK-NEXT: EMIT ir<%v2> = icmp sle ir<%iv>, ir<2>
; CHECK-NEXT: Successor(s): bb3
; CHECK-EMPTY:
; CHECK-NEXT: bb3:
-; CHECK-NEXT: EMIT vp<[[VP5:%[0-9]+]]> = logical-and ir<%u0>, ir<%v2>
-; CHECK-NEXT: EMIT ir<%add3> = add ir<%iv>, ir<3>, vp<[[VP5]]>
-; CHECK-NEXT: EMIT store ir<%add3>, ir<%gep>, vp<[[VP5]]>
+; CHECK-NEXT: EMIT ir<%add3> = add ir<%iv>, ir<3>, ir<%v2>
+; CHECK-NEXT: EMIT store ir<%add3>, ir<%gep>, ir<%v2>
; CHECK-NEXT: Successor(s): bb4
; CHECK-EMPTY:
; CHECK-NEXT: bb4:
@@ -2961,9 +2928,8 @@ define void @uniform_branch_shared_join_with_varying_no_phi(ptr %a, i1 %u0) {
; CHECK-NEXT: No successors
; CHECK-EMPTY:
; CHECK-NEXT: bb1:
-; CHECK-NEXT: EMIT vp<[[VP4:%[0-9]+]]> = not ir<%u0>
-; CHECK-NEXT: EMIT ir<%add1> = add ir<%iv>, ir<1>, vp<[[VP4]]>
-; CHECK-NEXT: EMIT store ir<%add1>, ir<%gep>, vp<[[VP4]]>
+; CHECK-NEXT: EMIT ir<%add1> = add ir<%iv>, ir<1>
+; CHECK-NEXT: EMIT store ir<%add1>, ir<%gep>
; CHECK-NEXT: Successor(s): bb4
; CHECK-NEXT: }
; CHECK-NEXT: Successor(s): middle.block
diff --git a/llvm/test/Transforms/LoopVectorize/VPlan/vplan-sink-scalars-and-merge-vf1.ll b/llvm/test/Transforms/LoopVectorize/VPlan/vplan-sink-scalars-and-merge-vf1.ll
index c846b26096772..5ec9276f8a5c5 100644
--- a/llvm/test/Transforms/LoopVectorize/VPlan/vplan-sink-scalars-and-merge-vf1.ll
+++ b/llvm/test/Transforms/LoopVectorize/VPlan/vplan-sink-scalars-and-merge-vf1.ll
@@ -28,27 +28,11 @@ define void @sink_with_sideeffects(i1 %c, ptr %ptr) {
; CHECK-NEXT: CLONE ir<%tmp2> = getelementptr ir<%ptr>, vp<[[VP5]]>
; CHECK-NEXT: CLONE ir<%tmp3> = load ir<%tmp2>
; CHECK-NEXT: CLONE store ir<0>, ir<%tmp2>
-; CHECK-NEXT: EMIT branch-on-cond ir<%c>
+; CHECK-NEXT: EMIT branch-on-cond ir<%c> (!vplan.prof.estimated estimated {1073741824, 1073741824})
; CHECK-NEXT: Successor(s): if.then, for.inc
; CHECK-EMPTY:
; CHECK-NEXT: if.then:
-; CHECK-NEXT: Successor(s): pred.store
-; CHECK-EMPTY:
-; CHECK-NEXT: <xVFxUF> pred.store: {
-; CHECK-NEXT: pred.store.entry:
-; CHECK-NEXT: BRANCH-ON-MASK ir<%c>
-; CHECK-NEXT: Successor(s): pred.store.if, pred.store.continue
-; CHECK-EMPTY:
-; CHECK-NEXT: pred.store.if:
-; CHECK-NEXT: CLONE store ir<%tmp3>, ir<%tmp2>
-; CHECK-NEXT: Successor(s): pred.store.continue
-; CHECK-EMPTY:
-; CHECK-NEXT: pred.store.continue:
-; CHECK-NEXT: No successors
-; CHECK-NEXT: }
-; CHECK-NEXT: Successor(s): if.then.0
-; CHECK-EMPTY:
-; CHECK-NEXT: if.then.0:
+; CHECK-NEXT: CLONE store ir<%tmp3>, ir<%tmp2>
; CHECK-NEXT: Successor(s): for.inc
; CHECK-EMPTY:
; CHECK-NEXT: for.inc:
diff --git a/llvm/test/Transforms/LoopVectorize/VPlan/vplan-sink-scalars-and-merge.ll b/llvm/test/Transforms/LoopVectorize/VPlan/vplan-sink-scalars-and-merge.ll
index c8503ebab5e45..6c296097a5f93 100644
--- a/llvm/test/Transforms/LoopVectorize/VPlan/vplan-sink-scalars-and-merge.ll
+++ b/llvm/test/Transforms/LoopVectorize/VPlan/vplan-sink-scalars-and-merge.ll
@@ -1160,31 +1160,17 @@ define void @update_multiple_users(ptr noalias %src, ptr noalias %dst, i1 %c) {
; CHECK-NEXT: vp<[[VP2:%[0-9]+]]> = CANONICAL-IV
; CHECK-EMPTY:
; CHECK-NEXT: vector.body:
-; CHECK-NEXT: EMIT branch-on-cond ir<%c>
+; CHECK-NEXT: EMIT branch-on-cond ir<%c> (!vplan.prof.estimated estimated {1073741824, 1073741824})
; CHECK-NEXT: Successor(s): loop.then, loop.latch
; CHECK-EMPTY:
; CHECK-NEXT: loop.then:
-; CHECK-NEXT: Successor(s): pred.store
-; CHECK-EMPTY:
-; CHECK-NEXT: <xVFxUF> pred.store: {
-; CHECK-NEXT: pred.store.entry:
-; CHECK-NEXT: BRANCH-ON-MASK ir<%c>
-; CHECK-NEXT: Successor(s): pred.store.if, pred.store.continue
-; CHECK-EMPTY:
-; CHECK-NEXT: pred.store.if:
-; CHECK-NEXT: REPLICATE ir<%l1> = load ir<%src>
-; CHECK-NEXT: REPLICATE ir<%l2> = trunc ir<%l1>
-; CHECK-NEXT: REPLICATE ir<%cmp> = icmp eq ir<%l1>, ir<0>
-; CHECK-NEXT: REPLICATE ir<%sel> = select ir<%cmp>, ir<5>, ir<%l2>
-; CHECK-NEXT: REPLICATE store ir<%sel>, ir<%dst>
-; CHECK-NEXT: Successor(s): pred.store.continue
-; CHECK-EMPTY:
-; CHECK-NEXT: pred.store.continue:
-; CHECK-NEXT: No successors
-; CHECK-NEXT: }
-; CHECK-NEXT: Successor(s): loop.then.1
-; CHECK-EMPTY:
-; CHECK-NEXT: loop.then.1:
+; CHECK-NEXT: REPLICATE ir<%l1> = load ir<%src>
+; CHECK-NEXT: REPLICATE ir<%l2> = trunc ir<%l1>
+; CHECK-NEXT: REPLICATE ir<%cmp> = icmp eq ir<%l1>, ir<0>
+; CHECK-NEXT: REPLICATE ir<%sel> = select ir<%cmp>, ir<5>, ir<%l2>
+; CHECK-NEXT: EMIT vp<[[VP3:%[0-9]+]]> = extract-last-part ir<%sel>
+; CHECK-NEXT: EMIT vp<[[VP4:%[0-9]+]]> = extract-last-lane vp<[[VP3]]>
+; CHECK-NEXT: CLONE store vp<[[VP4]]>, ir<%dst>
; CHECK-NEXT: Successor(s): loop.latch
; CHECK-EMPTY:
; CHECK-NEXT: loop.latch:
@@ -1262,7 +1248,7 @@ define void @sinking_requires_duplication(ptr %addr) {
; CHECK-EMPTY:
; CHECK-NEXT: <xVFxUF> pred.store: {
; CHECK-NEXT: pred.store.entry:
-; CHECK-NEXT: BRANCH-ON-MASK ir<%pred>
+; CHECK-NEXT: BRANCH-ON-MASK ir<%pred> (!vplan.execution.frequency 3458764513820540928 (37.5%, estimated))
; CHECK-NEXT: Successor(s): pred.store.if, pred.store.continue
; CHECK-EMPTY:
; CHECK-NEXT: pred.store.if:
@@ -1351,7 +1337,7 @@ define void @merge_with_dead_gep_between_regions(i32 %n, i32 %k, ptr noalias %sr
; CHECK-EMPTY:
; CHECK-NEXT: <xVFxUF> pred.store: {
; CHECK-NEXT: pred.store.entry:
-; CHECK-NEXT: BRANCH-ON-MASK ir<%cond>
+; CHECK-NEXT: BRANCH-ON-MASK ir<%cond> (!vplan.execution.frequency 4611686018427387904 (50%, estimated))
; CHECK-NEXT: Successor(s): pred.store.if, pred.store.continue
; CHECK-EMPTY:
; CHECK-NEXT: pred.store.if:
@@ -1450,7 +1436,7 @@ define void @ptr_induction_remove_dead_recipe(ptr %start, ptr %end) {
; CHECK-EMPTY:
; CHECK-NEXT: <xVFxUF> pred.store: {
; CHECK-NEXT: pred.store.entry:
-; CHECK-NEXT: BRANCH-ON-MASK ir<%c.1>
+; CHECK-NEXT: BRANCH-ON-MASK ir<%c.1> (!vplan.execution.frequency 5764607523034234880 (62.5%, estimated))
; CHECK-NEXT: Successor(s): pred.store.if, pred.store.continue
; CHECK-EMPTY:
; CHECK-NEXT: pred.store.if:
diff --git a/llvm/test/Transforms/LoopVectorize/X86/cost-conditional-branches.ll b/llvm/test/Transforms/LoopVectorize/X86/cost-conditional-branches.ll
index 207e203722f1c..78bb964a3bb50 100644
--- a/llvm/test/Transforms/LoopVectorize/X86/cost-conditional-branches.ll
+++ b/llvm/test/Transforms/LoopVectorize/X86/cost-conditional-branches.ll
@@ -868,442 +868,23 @@ exit:
; Test case for https://github.com/llvm/llvm-project/issues/158660.
define i64 @test_predicated_udiv(i32 %d, i1 %c) #2 {
; CHECK-LABEL: @test_predicated_udiv(
-; CHECK-NEXT: iter.check:
-; CHECK-NEXT: br i1 false, label [[VEC_EPILOG_SCALAR_PH:%.*]], label [[VECTOR_MAIN_LOOP_ITER_CHECK:%.*]]
-; CHECK: vector.main.loop.iter.check:
-; CHECK-NEXT: br i1 false, label [[VEC_EPILOG_PH:%.*]], label [[VECTOR_PH:%.*]]
-; CHECK: vector.ph:
-; CHECK-NEXT: [[BROADCAST_SPLATINSERT1:%.*]] = insertelement <32 x i1> poison, i1 [[C:%.*]], i64 0
-; CHECK-NEXT: [[BROADCAST_SPLAT1:%.*]] = shufflevector <32 x i1> [[BROADCAST_SPLATINSERT1]], <32 x i1> poison, <32 x i32> zeroinitializer
-; CHECK-NEXT: [[TMP131:%.*]] = xor <32 x i1> [[BROADCAST_SPLAT1]], splat (i1 true)
-; CHECK-NEXT: br label [[VECTOR_BODY1:%.*]]
-; CHECK: vector.body:
-; CHECK-NEXT: [[INDEX:%.*]] = phi i32 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[PRED_UDIV_CONTINUE62:%.*]] ]
-; CHECK-NEXT: [[VEC_IND:%.*]] = phi <32 x i32> [ <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15, i32 16, i32 17, i32 18, i32 19, i32 20, i32 21, i32 22, i32 23, i32 24, i32 25, i32 26, i32 27, i32 28, i32 29, i32 30, i32 31>, [[VECTOR_PH]] ], [ [[VEC_IND_NEXT:%.*]], [[PRED_UDIV_CONTINUE62]] ]
-; CHECK-NEXT: br i1 [[C]], label [[PRED_UDIV_CONTINUE62]], label [[VECTOR_BODY:%.*]]
-; CHECK: then1:
-; CHECK-NEXT: [[TMP1:%.*]] = call <32 x i32> @llvm.usub.sat.v32i32(<32 x i32> [[VEC_IND]], <32 x i32> splat (i32 1))
-; CHECK-NEXT: [[TMP0:%.*]] = extractelement <32 x i1> [[TMP131]], i64 0
-; CHECK-NEXT: br i1 [[TMP0]], label [[PRED_UDIV_IF:%.*]], label [[PRED_UDIV_CONTINUE:%.*]]
-; CHECK: pred.udiv.if:
-; CHECK-NEXT: [[TMP2:%.*]] = extractelement <32 x i32> [[TMP1]], i64 0
-; CHECK-NEXT: [[TMP3:%.*]] = udiv i32 [[TMP2]], [[D:%.*]]
-; CHECK-NEXT: [[TMP4:%.*]] = insertelement <32 x i32> poison, i32 [[TMP3]], i64 0
-; CHECK-NEXT: br label [[PRED_UDIV_CONTINUE]]
-; CHECK: pred.udiv.continue:
-; CHECK-NEXT: [[TMP5:%.*]] = phi <32 x i32> [ poison, [[VECTOR_BODY]] ], [ [[TMP4]], [[PRED_UDIV_IF]] ]
-; CHECK-NEXT: [[TMP168:%.*]] = extractelement <32 x i1> [[TMP131]], i64 0
-; CHECK-NEXT: br i1 [[TMP168]], label [[PRED_UDIV_IF1:%.*]], label [[PRED_UDIV_CONTINUE2:%.*]]
-; CHECK: pred.udiv.if2:
-; CHECK-NEXT: [[TMP6:%.*]] = extractelement <32 x i32> [[TMP1]], i64 1
-; CHECK-NEXT: [[TMP7:%.*]] = udiv i32 [[TMP6]], [[D]]
-; CHECK-NEXT: [[TMP8:%.*]] = insertelement <32 x i32> [[TMP5]], i32 [[TMP7]], i64 1
-; CHECK-NEXT: br label [[PRED_UDIV_CONTINUE2]]
-; CHECK: pred.udiv.continue3:
-; CHECK-NEXT: [[TMP9:%.*]] = phi <32 x i32> [ [[TMP5]], [[PRED_UDIV_CONTINUE]] ], [ [[TMP8]], [[PRED_UDIV_IF1]] ]
-; CHECK-NEXT: [[TMP170:%.*]] = extractelement <32 x i1> [[TMP131]], i64 0
-; CHECK-NEXT: br i1 [[TMP170]], label [[PRED_UDIV_IF3:%.*]], label [[PRED_UDIV_CONTINUE4:%.*]]
-; CHECK: pred.udiv.if4:
-; CHECK-NEXT: [[TMP10:%.*]] = extractelement <32 x i32> [[TMP1]], i64 2
-; CHECK-NEXT: [[TMP11:%.*]] = udiv i32 [[TMP10]], [[D]]
-; CHECK-NEXT: [[TMP12:%.*]] = insertelement <32 x i32> [[TMP9]], i32 [[TMP11]], i64 2
-; CHECK-NEXT: br label [[PRED_UDIV_CONTINUE4]]
-; CHECK: pred.udiv.continue5:
-; CHECK-NEXT: [[TMP13:%.*]] = phi <32 x i32> [ [[TMP9]], [[PRED_UDIV_CONTINUE2]] ], [ [[TMP12]], [[PRED_UDIV_IF3]] ]
-; CHECK-NEXT: [[TMP171:%.*]] = extractelement <32 x i1> [[TMP131]], i64 0
-; CHECK-NEXT: br i1 [[TMP171]], label [[PRED_UDIV_IF5:%.*]], label [[PRED_UDIV_CONTINUE6:%.*]]
-; CHECK: pred.udiv.if6:
-; CHECK-NEXT: [[TMP14:%.*]] = extractelement <32 x i32> [[TMP1]], i64 3
-; CHECK-NEXT: [[TMP15:%.*]] = udiv i32 [[TMP14]], [[D]]
-; CHECK-NEXT: [[TMP16:%.*]] = insertelement <32 x i32> [[TMP13]], i32 [[TMP15]], i64 3
-; CHECK-NEXT: br label [[PRED_UDIV_CONTINUE6]]
-; CHECK: pred.udiv.continue7:
-; CHECK-NEXT: [[TMP17:%.*]] = phi <32 x i32> [ [[TMP13]], [[PRED_UDIV_CONTINUE4]] ], [ [[TMP16]], [[PRED_UDIV_IF5]] ]
-; CHECK-NEXT: [[TMP172:%.*]] = extractelement <32 x i1> [[TMP131]], i64 0
-; CHECK-NEXT: br i1 [[TMP172]], label [[PRED_UDIV_IF7:%.*]], label [[PRED_UDIV_CONTINUE8:%.*]]
-; CHECK: pred.udiv.if8:
-; CHECK-NEXT: [[TMP18:%.*]] = extractelement <32 x i32> [[TMP1]], i64 4
-; CHECK-NEXT: [[TMP19:%.*]] = udiv i32 [[TMP18]], [[D]]
-; CHECK-NEXT: [[TMP20:%.*]] = insertelement <32 x i32> [[TMP17]], i32 [[TMP19]], i64 4
-; CHECK-NEXT: br label [[PRED_UDIV_CONTINUE8]]
-; CHECK: pred.udiv.continue9:
-; CHECK-NEXT: [[TMP21:%.*]] = phi <32 x i32> [ [[TMP17]], [[PRED_UDIV_CONTINUE6]] ], [ [[TMP20]], [[PRED_UDIV_IF7]] ]
-; CHECK-NEXT: [[TMP173:%.*]] = extractelement <32 x i1> [[TMP131]], i64 0
-; CHECK-NEXT: br i1 [[TMP173]], label [[PRED_UDIV_IF9:%.*]], label [[PRED_UDIV_CONTINUE10:%.*]]
-; CHECK: pred.udiv.if10:
-; CHECK-NEXT: [[TMP22:%.*]] = extractelement <32 x i32> [[TMP1]], i64 5
-; CHECK-NEXT: [[TMP23:%.*]] = udiv i32 [[TMP22]], [[D]]
-; CHECK-NEXT: [[TMP24:%.*]] = insertelement <32 x i32> [[TMP21]], i32 [[TMP23]], i64 5
-; CHECK-NEXT: br label [[PRED_UDIV_CONTINUE10]]
-; CHECK: pred.udiv.continue11:
-; CHECK-NEXT: [[TMP25:%.*]] = phi <32 x i32> [ [[TMP21]], [[PRED_UDIV_CONTINUE8]] ], [ [[TMP24]], [[PRED_UDIV_IF9]] ]
-; CHECK-NEXT: [[TMP175:%.*]] = extractelement <32 x i1> [[TMP131]], i64 0
-; CHECK-NEXT: br i1 [[TMP175]], label [[PRED_UDIV_IF11:%.*]], label [[PRED_UDIV_CONTINUE12:%.*]]
-; CHECK: pred.udiv.if12:
-; CHECK-NEXT: [[TMP26:%.*]] = extractelement <32 x i32> [[TMP1]], i64 6
-; CHECK-NEXT: [[TMP27:%.*]] = udiv i32 [[TMP26]], [[D]]
-; CHECK-NEXT: [[TMP28:%.*]] = insertelement <32 x i32> [[TMP25]], i32 [[TMP27]], i64 6
-; CHECK-NEXT: br label [[PRED_UDIV_CONTINUE12]]
-; CHECK: pred.udiv.continue13:
-; CHECK-NEXT: [[TMP29:%.*]] = phi <32 x i32> [ [[TMP25]], [[PRED_UDIV_CONTINUE10]] ], [ [[TMP28]], [[PRED_UDIV_IF11]] ]
-; CHECK-NEXT: [[TMP176:%.*]] = extractelement <32 x i1> [[TMP131]], i64 0
-; CHECK-NEXT: br i1 [[TMP176]], label [[PRED_UDIV_IF13:%.*]], label [[PRED_UDIV_CONTINUE14:%.*]]
-; CHECK: pred.udiv.if14:
-; CHECK-NEXT: [[TMP30:%.*]] = extractelement <32 x i32> [[TMP1]], i64 7
-; CHECK-NEXT: [[TMP31:%.*]] = udiv i32 [[TMP30]], [[D]]
-; CHECK-NEXT: [[TMP32:%.*]] = insertelement <32 x i32> [[TMP29]], i32 [[TMP31]], i64 7
-; CHECK-NEXT: br label [[PRED_UDIV_CONTINUE14]]
-; CHECK: pred.udiv.continue15:
-; CHECK-NEXT: [[TMP33:%.*]] = phi <32 x i32> [ [[TMP29]], [[PRED_UDIV_CONTINUE12]] ], [ [[TMP32]], [[PRED_UDIV_IF13]] ]
-; CHECK-NEXT: [[TMP177:%.*]] = extractelement <32 x i1> [[TMP131]], i64 0
-; CHECK-NEXT: br i1 [[TMP177]], label [[PRED_UDIV_IF15:%.*]], label [[PRED_UDIV_CONTINUE16:%.*]]
-; CHECK: pred.udiv.if16:
-; CHECK-NEXT: [[TMP34:%.*]] = extractelement <32 x i32> [[TMP1]], i64 8
-; CHECK-NEXT: [[TMP35:%.*]] = udiv i32 [[TMP34]], [[D]]
-; CHECK-NEXT: [[TMP36:%.*]] = insertelement <32 x i32> [[TMP33]], i32 [[TMP35]], i64 8
-; CHECK-NEXT: br label [[PRED_UDIV_CONTINUE16]]
-; CHECK: pred.udiv.continue17:
-; CHECK-NEXT: [[TMP37:%.*]] = phi <32 x i32> [ [[TMP33]], [[PRED_UDIV_CONTINUE14]] ], [ [[TMP36]], [[PRED_UDIV_IF15]] ]
-; CHECK-NEXT: [[TMP178:%.*]] = extractelement <32 x i1> [[TMP131]], i64 0
-; CHECK-NEXT: br i1 [[TMP178]], label [[PRED_UDIV_IF17:%.*]], label [[PRED_UDIV_CONTINUE18:%.*]]
-; CHECK: pred.udiv.if18:
-; CHECK-NEXT: [[TMP38:%.*]] = extractelement <32 x i32> [[TMP1]], i64 9
-; CHECK-NEXT: [[TMP39:%.*]] = udiv i32 [[TMP38]], [[D]]
-; CHECK-NEXT: [[TMP40:%.*]] = insertelement <32 x i32> [[TMP37]], i32 [[TMP39]], i64 9
-; CHECK-NEXT: br label [[PRED_UDIV_CONTINUE18]]
-; CHECK: pred.udiv.continue19:
-; CHECK-NEXT: [[TMP41:%.*]] = phi <32 x i32> [ [[TMP37]], [[PRED_UDIV_CONTINUE16]] ], [ [[TMP40]], [[PRED_UDIV_IF17]] ]
-; CHECK-NEXT: [[TMP180:%.*]] = extractelement <32 x i1> [[TMP131]], i64 0
-; CHECK-NEXT: br i1 [[TMP180]], label [[PRED_UDIV_IF19:%.*]], label [[PRED_UDIV_CONTINUE20:%.*]]
-; CHECK: pred.udiv.if20:
-; CHECK-NEXT: [[TMP42:%.*]] = extractelement <32 x i32> [[TMP1]], i64 10
-; CHECK-NEXT: [[TMP43:%.*]] = udiv i32 [[TMP42]], [[D]]
-; CHECK-NEXT: [[TMP44:%.*]] = insertelement <32 x i32> [[TMP41]], i32 [[TMP43]], i64 10
-; CHECK-NEXT: br label [[PRED_UDIV_CONTINUE20]]
-; CHECK: pred.udiv.continue21:
-; CHECK-NEXT: [[TMP45:%.*]] = phi <32 x i32> [ [[TMP41]], [[PRED_UDIV_CONTINUE18]] ], [ [[TMP44]], [[PRED_UDIV_IF19]] ]
-; CHECK-NEXT: [[TMP181:%.*]] = extractelement <32 x i1> [[TMP131]], i64 0
-; CHECK-NEXT: br i1 [[TMP181]], label [[PRED_UDIV_IF21:%.*]], label [[PRED_UDIV_CONTINUE22:%.*]]
-; CHECK: pred.udiv.if22:
-; CHECK-NEXT: [[TMP46:%.*]] = extractelement <32 x i32> [[TMP1]], i64 11
-; CHECK-NEXT: [[TMP47:%.*]] = udiv i32 [[TMP46]], [[D]]
-; CHECK-NEXT: [[TMP48:%.*]] = insertelement <32 x i32> [[TMP45]], i32 [[TMP47]], i64 11
-; CHECK-NEXT: br label [[PRED_UDIV_CONTINUE22]]
-; CHECK: pred.udiv.continue23:
-; CHECK-NEXT: [[TMP49:%.*]] = phi <32 x i32> [ [[TMP45]], [[PRED_UDIV_CONTINUE20]] ], [ [[TMP48]], [[PRED_UDIV_IF21]] ]
-; CHECK-NEXT: [[TMP182:%.*]] = extractelement <32 x i1> [[TMP131]], i64 0
-; CHECK-NEXT: br i1 [[TMP182]], label [[PRED_UDIV_IF23:%.*]], label [[PRED_UDIV_CONTINUE24:%.*]]
-; CHECK: pred.udiv.if24:
-; CHECK-NEXT: [[TMP50:%.*]] = extractelement <32 x i32> [[TMP1]], i64 12
-; CHECK-NEXT: [[TMP51:%.*]] = udiv i32 [[TMP50]], [[D]]
-; CHECK-NEXT: [[TMP52:%.*]] = insertelement <32 x i32> [[TMP49]], i32 [[TMP51]], i64 12
-; CHECK-NEXT: br label [[PRED_UDIV_CONTINUE24]]
-; CHECK: pred.udiv.continue25:
-; CHECK-NEXT: [[TMP53:%.*]] = phi <32 x i32> [ [[TMP49]], [[PRED_UDIV_CONTINUE22]] ], [ [[TMP52]], [[PRED_UDIV_IF23]] ]
-; CHECK-NEXT: [[TMP183:%.*]] = extractelement <32 x i1> [[TMP131]], i64 0
-; CHECK-NEXT: br i1 [[TMP183]], label [[PRED_UDIV_IF25:%.*]], label [[PRED_UDIV_CONTINUE26:%.*]]
-; CHECK: pred.udiv.if26:
-; CHECK-NEXT: [[TMP54:%.*]] = extractelement <32 x i32> [[TMP1]], i64 13
-; CHECK-NEXT: [[TMP55:%.*]] = udiv i32 [[TMP54]], [[D]]
-; CHECK-NEXT: [[TMP56:%.*]] = insertelement <32 x i32> [[TMP53]], i32 [[TMP55]], i64 13
-; CHECK-NEXT: br label [[PRED_UDIV_CONTINUE26]]
-; CHECK: pred.udiv.continue27:
-; CHECK-NEXT: [[TMP57:%.*]] = phi <32 x i32> [ [[TMP53]], [[PRED_UDIV_CONTINUE24]] ], [ [[TMP56]], [[PRED_UDIV_IF25]] ]
-; CHECK-NEXT: [[TMP185:%.*]] = extractelement <32 x i1> [[TMP131]], i64 0
-; CHECK-NEXT: br i1 [[TMP185]], label [[PRED_UDIV_IF27:%.*]], label [[PRED_UDIV_CONTINUE28:%.*]]
-; CHECK: pred.udiv.if28:
-; CHECK-NEXT: [[TMP58:%.*]] = extractelement <32 x i32> [[TMP1]], i64 14
-; CHECK-NEXT: [[TMP59:%.*]] = udiv i32 [[TMP58]], [[D]]
-; CHECK-NEXT: [[TMP60:%.*]] = insertelement <32 x i32> [[TMP57]], i32 [[TMP59]], i64 14
-; CHECK-NEXT: br label [[PRED_UDIV_CONTINUE28]]
-; CHECK: pred.udiv.continue29:
-; CHECK-NEXT: [[TMP61:%.*]] = phi <32 x i32> [ [[TMP57]], [[PRED_UDIV_CONTINUE26]] ], [ [[TMP60]], [[PRED_UDIV_IF27]] ]
-; CHECK-NEXT: [[TMP186:%.*]] = extractelement <32 x i1> [[TMP131]], i64 0
-; CHECK-NEXT: br i1 [[TMP186]], label [[PRED_UDIV_IF29:%.*]], label [[PRED_UDIV_CONTINUE30:%.*]]
-; CHECK: pred.udiv.if30:
-; CHECK-NEXT: [[TMP62:%.*]] = extractelement <32 x i32> [[TMP1]], i64 15
-; CHECK-NEXT: [[TMP63:%.*]] = udiv i32 [[TMP62]], [[D]]
-; CHECK-NEXT: [[TMP64:%.*]] = insertelement <32 x i32> [[TMP61]], i32 [[TMP63]], i64 15
-; CHECK-NEXT: br label [[PRED_UDIV_CONTINUE30]]
-; CHECK: pred.udiv.continue31:
-; CHECK-NEXT: [[TMP65:%.*]] = phi <32 x i32> [ [[TMP61]], [[PRED_UDIV_CONTINUE28]] ], [ [[TMP64]], [[PRED_UDIV_IF29]] ]
-; CHECK-NEXT: [[TMP187:%.*]] = extractelement <32 x i1> [[TMP131]], i64 0
-; CHECK-NEXT: br i1 [[TMP187]], label [[PRED_UDIV_IF31:%.*]], label [[PRED_UDIV_CONTINUE32:%.*]]
-; CHECK: pred.udiv.if32:
-; CHECK-NEXT: [[TMP66:%.*]] = extractelement <32 x i32> [[TMP1]], i64 16
-; CHECK-NEXT: [[TMP67:%.*]] = udiv i32 [[TMP66]], [[D]]
-; CHECK-NEXT: [[TMP68:%.*]] = insertelement <32 x i32> [[TMP65]], i32 [[TMP67]], i64 16
-; CHECK-NEXT: br label [[PRED_UDIV_CONTINUE32]]
-; CHECK: pred.udiv.continue33:
-; CHECK-NEXT: [[TMP69:%.*]] = phi <32 x i32> [ [[TMP65]], [[PRED_UDIV_CONTINUE30]] ], [ [[TMP68]], [[PRED_UDIV_IF31]] ]
-; CHECK-NEXT: [[TMP188:%.*]] = extractelement <32 x i1> [[TMP131]], i64 0
-; CHECK-NEXT: br i1 [[TMP188]], label [[PRED_UDIV_IF33:%.*]], label [[PRED_UDIV_CONTINUE34:%.*]]
-; CHECK: pred.udiv.if34:
-; CHECK-NEXT: [[TMP70:%.*]] = extractelement <32 x i32> [[TMP1]], i64 17
-; CHECK-NEXT: [[TMP71:%.*]] = udiv i32 [[TMP70]], [[D]]
-; CHECK-NEXT: [[TMP72:%.*]] = insertelement <32 x i32> [[TMP69]], i32 [[TMP71]], i64 17
-; CHECK-NEXT: br label [[PRED_UDIV_CONTINUE34]]
-; CHECK: pred.udiv.continue35:
-; CHECK-NEXT: [[TMP73:%.*]] = phi <32 x i32> [ [[TMP69]], [[PRED_UDIV_CONTINUE32]] ], [ [[TMP72]], [[PRED_UDIV_IF33]] ]
-; CHECK-NEXT: [[TMP190:%.*]] = extractelement <32 x i1> [[TMP131]], i64 0
-; CHECK-NEXT: br i1 [[TMP190]], label [[PRED_UDIV_IF35:%.*]], label [[PRED_UDIV_CONTINUE36:%.*]]
-; CHECK: pred.udiv.if36:
-; CHECK-NEXT: [[TMP74:%.*]] = extractelement <32 x i32> [[TMP1]], i64 18
-; CHECK-NEXT: [[TMP75:%.*]] = udiv i32 [[TMP74]], [[D]]
-; CHECK-NEXT: [[TMP76:%.*]] = insertelement <32 x i32> [[TMP73]], i32 [[TMP75]], i64 18
-; CHECK-NEXT: br label [[PRED_UDIV_CONTINUE36]]
-; CHECK: pred.udiv.continue37:
-; CHECK-NEXT: [[TMP77:%.*]] = phi <32 x i32> [ [[TMP73]], [[PRED_UDIV_CONTINUE34]] ], [ [[TMP76]], [[PRED_UDIV_IF35]] ]
-; CHECK-NEXT: [[TMP191:%.*]] = extractelement <32 x i1> [[TMP131]], i64 0
-; CHECK-NEXT: br i1 [[TMP191]], label [[PRED_UDIV_IF37:%.*]], label [[PRED_UDIV_CONTINUE38:%.*]]
-; CHECK: pred.udiv.if38:
-; CHECK-NEXT: [[TMP78:%.*]] = extractelement <32 x i32> [[TMP1]], i64 19
-; CHECK-NEXT: [[TMP79:%.*]] = udiv i32 [[TMP78]], [[D]]
-; CHECK-NEXT: [[TMP80:%.*]] = insertelement <32 x i32> [[TMP77]], i32 [[TMP79]], i64 19
-; CHECK-NEXT: br label [[PRED_UDIV_CONTINUE38]]
-; CHECK: pred.udiv.continue39:
-; CHECK-NEXT: [[TMP81:%.*]] = phi <32 x i32> [ [[TMP77]], [[PRED_UDIV_CONTINUE36]] ], [ [[TMP80]], [[PRED_UDIV_IF37]] ]
-; CHECK-NEXT: [[TMP192:%.*]] = extractelement <32 x i1> [[TMP131]], i64 0
-; CHECK-NEXT: br i1 [[TMP192]], label [[PRED_UDIV_IF39:%.*]], label [[PRED_UDIV_CONTINUE40:%.*]]
-; CHECK: pred.udiv.if40:
-; CHECK-NEXT: [[TMP82:%.*]] = extractelement <32 x i32> [[TMP1]], i64 20
-; CHECK-NEXT: [[TMP83:%.*]] = udiv i32 [[TMP82]], [[D]]
-; CHECK-NEXT: [[TMP84:%.*]] = insertelement <32 x i32> [[TMP81]], i32 [[TMP83]], i64 20
-; CHECK-NEXT: br label [[PRED_UDIV_CONTINUE40]]
-; CHECK: pred.udiv.continue41:
-; CHECK-NEXT: [[TMP85:%.*]] = phi <32 x i32> [ [[TMP81]], [[PRED_UDIV_CONTINUE38]] ], [ [[TMP84]], [[PRED_UDIV_IF39]] ]
-; CHECK-NEXT: [[TMP193:%.*]] = extractelement <32 x i1> [[TMP131]], i64 0
-; CHECK-NEXT: br i1 [[TMP193]], label [[PRED_UDIV_IF41:%.*]], label [[PRED_UDIV_CONTINUE42:%.*]]
-; CHECK: pred.udiv.if42:
-; CHECK-NEXT: [[TMP86:%.*]] = extractelement <32 x i32> [[TMP1]], i64 21
-; CHECK-NEXT: [[TMP87:%.*]] = udiv i32 [[TMP86]], [[D]]
-; CHECK-NEXT: [[TMP88:%.*]] = insertelement <32 x i32> [[TMP85]], i32 [[TMP87]], i64 21
-; CHECK-NEXT: br label [[PRED_UDIV_CONTINUE42]]
-; CHECK: pred.udiv.continue43:
-; CHECK-NEXT: [[TMP89:%.*]] = phi <32 x i32> [ [[TMP85]], [[PRED_UDIV_CONTINUE40]] ], [ [[TMP88]], [[PRED_UDIV_IF41]] ]
-; CHECK-NEXT: [[TMP195:%.*]] = extractelement <32 x i1> [[TMP131]], i64 0
-; CHECK-NEXT: br i1 [[TMP195]], label [[PRED_UDIV_IF43:%.*]], label [[PRED_UDIV_CONTINUE44:%.*]]
-; CHECK: pred.udiv.if44:
-; CHECK-NEXT: [[TMP90:%.*]] = extractelement <32 x i32> [[TMP1]], i64 22
-; CHECK-NEXT: [[TMP91:%.*]] = udiv i32 [[TMP90]], [[D]]
-; CHECK-NEXT: [[TMP92:%.*]] = insertelement <32 x i32> [[TMP89]], i32 [[TMP91]], i64 22
-; CHECK-NEXT: br label [[PRED_UDIV_CONTINUE44]]
-; CHECK: pred.udiv.continue45:
-; CHECK-NEXT: [[TMP93:%.*]] = phi <32 x i32> [ [[TMP89]], [[PRED_UDIV_CONTINUE42]] ], [ [[TMP92]], [[PRED_UDIV_IF43]] ]
-; CHECK-NEXT: [[TMP196:%.*]] = extractelement <32 x i1> [[TMP131]], i64 0
-; CHECK-NEXT: br i1 [[TMP196]], label [[PRED_UDIV_IF45:%.*]], label [[PRED_UDIV_CONTINUE46:%.*]]
-; CHECK: pred.udiv.if46:
-; CHECK-NEXT: [[TMP94:%.*]] = extractelement <32 x i32> [[TMP1]], i64 23
-; CHECK-NEXT: [[TMP95:%.*]] = udiv i32 [[TMP94]], [[D]]
-; CHECK-NEXT: [[TMP96:%.*]] = insertelement <32 x i32> [[TMP93]], i32 [[TMP95]], i64 23
-; CHECK-NEXT: br label [[PRED_UDIV_CONTINUE46]]
-; CHECK: pred.udiv.continue47:
-; CHECK-NEXT: [[TMP97:%.*]] = phi <32 x i32> [ [[TMP93]], [[PRED_UDIV_CONTINUE44]] ], [ [[TMP96]], [[PRED_UDIV_IF45]] ]
-; CHECK-NEXT: [[TMP197:%.*]] = extractelement <32 x i1> [[TMP131]], i64 0
-; CHECK-NEXT: br i1 [[TMP197]], label [[PRED_UDIV_IF47:%.*]], label [[PRED_UDIV_CONTINUE48:%.*]]
-; CHECK: pred.udiv.if48:
-; CHECK-NEXT: [[TMP98:%.*]] = extractelement <32 x i32> [[TMP1]], i64 24
-; CHECK-NEXT: [[TMP99:%.*]] = udiv i32 [[TMP98]], [[D]]
-; CHECK-NEXT: [[TMP100:%.*]] = insertelement <32 x i32> [[TMP97]], i32 [[TMP99]], i64 24
-; CHECK-NEXT: br label [[PRED_UDIV_CONTINUE48]]
-; CHECK: pred.udiv.continue49:
-; CHECK-NEXT: [[TMP101:%.*]] = phi <32 x i32> [ [[TMP97]], [[PRED_UDIV_CONTINUE46]] ], [ [[TMP100]], [[PRED_UDIV_IF47]] ]
-; CHECK-NEXT: [[TMP198:%.*]] = extractelement <32 x i1> [[TMP131]], i64 0
-; CHECK-NEXT: br i1 [[TMP198]], label [[PRED_UDIV_IF49:%.*]], label [[PRED_UDIV_CONTINUE50:%.*]]
-; CHECK: pred.udiv.if50:
-; CHECK-NEXT: [[TMP102:%.*]] = extractelement <32 x i32> [[TMP1]], i64 25
-; CHECK-NEXT: [[TMP103:%.*]] = udiv i32 [[TMP102]], [[D]]
-; CHECK-NEXT: [[TMP104:%.*]] = insertelement <32 x i32> [[TMP101]], i32 [[TMP103]], i64 25
-; CHECK-NEXT: br label [[PRED_UDIV_CONTINUE50]]
-; CHECK: pred.udiv.continue51:
-; CHECK-NEXT: [[TMP105:%.*]] = phi <32 x i32> [ [[TMP101]], [[PRED_UDIV_CONTINUE48]] ], [ [[TMP104]], [[PRED_UDIV_IF49]] ]
-; CHECK-NEXT: [[TMP200:%.*]] = extractelement <32 x i1> [[TMP131]], i64 0
-; CHECK-NEXT: br i1 [[TMP200]], label [[PRED_UDIV_IF51:%.*]], label [[PRED_UDIV_CONTINUE52:%.*]]
-; CHECK: pred.udiv.if52:
-; CHECK-NEXT: [[TMP106:%.*]] = extractelement <32 x i32> [[TMP1]], i64 26
-; CHECK-NEXT: [[TMP107:%.*]] = udiv i32 [[TMP106]], [[D]]
-; CHECK-NEXT: [[TMP108:%.*]] = insertelement <32 x i32> [[TMP105]], i32 [[TMP107]], i64 26
-; CHECK-NEXT: br label [[PRED_UDIV_CONTINUE52]]
-; CHECK: pred.udiv.continue53:
-; CHECK-NEXT: [[TMP109:%.*]] = phi <32 x i32> [ [[TMP105]], [[PRED_UDIV_CONTINUE50]] ], [ [[TMP108]], [[PRED_UDIV_IF51]] ]
-; CHECK-NEXT: [[TMP201:%.*]] = extractelement <32 x i1> [[TMP131]], i64 0
-; CHECK-NEXT: br i1 [[TMP201]], label [[PRED_UDIV_IF53:%.*]], label [[PRED_UDIV_CONTINUE54:%.*]]
-; CHECK: pred.udiv.if54:
-; CHECK-NEXT: [[TMP110:%.*]] = extractelement <32 x i32> [[TMP1]], i64 27
-; CHECK-NEXT: [[TMP111:%.*]] = udiv i32 [[TMP110]], [[D]]
-; CHECK-NEXT: [[TMP112:%.*]] = insertelement <32 x i32> [[TMP109]], i32 [[TMP111]], i64 27
-; CHECK-NEXT: br label [[PRED_UDIV_CONTINUE54]]
-; CHECK: pred.udiv.continue55:
-; CHECK-NEXT: [[TMP113:%.*]] = phi <32 x i32> [ [[TMP109]], [[PRED_UDIV_CONTINUE52]] ], [ [[TMP112]], [[PRED_UDIV_IF53]] ]
-; CHECK-NEXT: [[TMP202:%.*]] = extractelement <32 x i1> [[TMP131]], i64 0
-; CHECK-NEXT: br i1 [[TMP202]], label [[PRED_UDIV_IF55:%.*]], label [[PRED_UDIV_CONTINUE56:%.*]]
-; CHECK: pred.udiv.if56:
-; CHECK-NEXT: [[TMP114:%.*]] = extractelement <32 x i32> [[TMP1]], i64 28
-; CHECK-NEXT: [[TMP115:%.*]] = udiv i32 [[TMP114]], [[D]]
-; CHECK-NEXT: [[TMP116:%.*]] = insertelement <32 x i32> [[TMP113]], i32 [[TMP115]], i64 28
-; CHECK-NEXT: br label [[PRED_UDIV_CONTINUE56]]
-; CHECK: pred.udiv.continue57:
-; CHECK-NEXT: [[TMP117:%.*]] = phi <32 x i32> [ [[TMP113]], [[PRED_UDIV_CONTINUE54]] ], [ [[TMP116]], [[PRED_UDIV_IF55]] ]
-; CHECK-NEXT: [[TMP203:%.*]] = extractelement <32 x i1> [[TMP131]], i64 0
-; CHECK-NEXT: br i1 [[TMP203]], label [[PRED_UDIV_IF57:%.*]], label [[PRED_UDIV_CONTINUE58:%.*]]
-; CHECK: pred.udiv.if58:
-; CHECK-NEXT: [[TMP118:%.*]] = extractelement <32 x i32> [[TMP1]], i64 29
-; CHECK-NEXT: [[TMP119:%.*]] = udiv i32 [[TMP118]], [[D]]
-; CHECK-NEXT: [[TMP120:%.*]] = insertelement <32 x i32> [[TMP117]], i32 [[TMP119]], i64 29
-; CHECK-NEXT: br label [[PRED_UDIV_CONTINUE58]]
-; CHECK: pred.udiv.continue59:
-; CHECK-NEXT: [[TMP121:%.*]] = phi <32 x i32> [ [[TMP117]], [[PRED_UDIV_CONTINUE56]] ], [ [[TMP120]], [[PRED_UDIV_IF57]] ]
-; CHECK-NEXT: [[TMP205:%.*]] = extractelement <32 x i1> [[TMP131]], i64 0
-; CHECK-NEXT: br i1 [[TMP205]], label [[PRED_UDIV_IF59:%.*]], label [[PRED_UDIV_CONTINUE60:%.*]]
-; CHECK: pred.udiv.if60:
-; CHECK-NEXT: [[TMP122:%.*]] = extractelement <32 x i32> [[TMP1]], i64 30
-; CHECK-NEXT: [[TMP123:%.*]] = udiv i32 [[TMP122]], [[D]]
-; CHECK-NEXT: [[TMP124:%.*]] = insertelement <32 x i32> [[TMP121]], i32 [[TMP123]], i64 30
-; CHECK-NEXT: br label [[PRED_UDIV_CONTINUE60]]
-; CHECK: pred.udiv.continue61:
-; CHECK-NEXT: [[TMP125:%.*]] = phi <32 x i32> [ [[TMP121]], [[PRED_UDIV_CONTINUE58]] ], [ [[TMP124]], [[PRED_UDIV_IF59]] ]
-; CHECK-NEXT: [[TMP206:%.*]] = extractelement <32 x i1> [[TMP131]], i64 0
-; CHECK-NEXT: br i1 [[TMP206]], label [[PRED_UDIV_IF61:%.*]], label [[PRED_UDIV_CONTINUE63:%.*]]
-; CHECK: pred.udiv.if62:
-; CHECK-NEXT: [[TMP126:%.*]] = extractelement <32 x i32> [[TMP1]], i64 31
-; CHECK-NEXT: [[TMP127:%.*]] = udiv i32 [[TMP126]], [[D]]
-; CHECK-NEXT: [[TMP128:%.*]] = insertelement <32 x i32> [[TMP125]], i32 [[TMP127]], i64 31
-; CHECK-NEXT: br label [[PRED_UDIV_CONTINUE63]]
-; CHECK: pred.udiv.continue63:
-; CHECK-NEXT: [[TMP129:%.*]] = phi <32 x i32> [ [[TMP125]], [[PRED_UDIV_CONTINUE60]] ], [ [[TMP128]], [[PRED_UDIV_IF61]] ]
-; CHECK-NEXT: [[TMP207:%.*]] = zext <32 x i32> [[TMP129]] to <32 x i64>
-; CHECK-NEXT: br label [[PRED_UDIV_CONTINUE62]]
-; CHECK: loop.latch64:
-; CHECK-NEXT: [[TMP208:%.*]] = phi <32 x i64> [ poison, [[VECTOR_BODY1]] ], [ [[TMP207]], [[PRED_UDIV_CONTINUE63]] ]
-; CHECK-NEXT: [[TMP212:%.*]] = phi <32 x i1> [ zeroinitializer, [[VECTOR_BODY1]] ], [ [[TMP131]], [[PRED_UDIV_CONTINUE63]] ]
-; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i32 [[INDEX]], 32
-; CHECK-NEXT: [[VEC_IND_NEXT]] = add <32 x i32> [[VEC_IND]], splat (i32 32)
-; CHECK-NEXT: [[TMP130:%.*]] = icmp eq i32 [[INDEX_NEXT]], 992
-; CHECK-NEXT: br i1 [[TMP130]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY1]], !llvm.loop [[LOOP12:![0-9]+]]
-; CHECK: middle.block:
-; CHECK-NEXT: [[PREDPHI:%.*]] = select <32 x i1> [[TMP212]], <32 x i64> [[TMP208]], <32 x i64> zeroinitializer
-; CHECK-NEXT: [[TMP132:%.*]] = extractelement <32 x i64> [[PREDPHI]], i64 31
-; CHECK-NEXT: br i1 false, label [[EXIT:%.*]], label [[VEC_EPILOG_ITER_CHECK:%.*]]
-; CHECK: vec.epilog.iter.check:
-; CHECK-NEXT: br i1 false, label [[VEC_EPILOG_SCALAR_PH]], label [[VEC_EPILOG_PH]], !prof [[PROF13:![0-9]+]]
-; CHECK: vec.epilog.ph:
-; CHECK-NEXT: [[VEC_EPILOG_RESUME_VAL:%.*]] = phi i32 [ 992, [[VEC_EPILOG_ITER_CHECK]] ], [ 0, [[VECTOR_MAIN_LOOP_ITER_CHECK]] ]
-; CHECK-NEXT: [[BROADCAST_SPLATINSERT65:%.*]] = insertelement <8 x i1> poison, i1 [[C]], i64 0
-; CHECK-NEXT: [[BROADCAST_SPLAT66:%.*]] = shufflevector <8 x i1> [[BROADCAST_SPLATINSERT65]], <8 x i1> poison, <8 x i32> zeroinitializer
-; CHECK-NEXT: [[TMP213:%.*]] = xor <8 x i1> [[BROADCAST_SPLAT66]], splat (i1 true)
-; CHECK-NEXT: [[BROADCAST_SPLATINSERT:%.*]] = insertelement <8 x i32> poison, i32 [[VEC_EPILOG_RESUME_VAL]], i64 0
-; CHECK-NEXT: [[BROADCAST_SPLAT:%.*]] = shufflevector <8 x i32> [[BROADCAST_SPLATINSERT]], <8 x i32> poison, <8 x i32> zeroinitializer
-; CHECK-NEXT: [[INDUCTION:%.*]] = add <8 x i32> [[BROADCAST_SPLAT]], <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>
-; CHECK-NEXT: br label [[VEC_EPILOG_VECTOR_BODY1:%.*]]
-; CHECK: vec.epilog.vector.body:
-; CHECK-NEXT: [[INDEX63:%.*]] = phi i32 [ [[VEC_EPILOG_RESUME_VAL]], [[VEC_EPILOG_PH]] ], [ [[INDEX_NEXT81:%.*]], [[PRED_UDIV_CONTINUE80:%.*]] ]
-; CHECK-NEXT: [[VEC_IND64:%.*]] = phi <8 x i32> [ [[INDUCTION]], [[VEC_EPILOG_PH]] ], [ [[VEC_IND_NEXT82:%.*]], [[PRED_UDIV_CONTINUE80]] ]
-; CHECK-NEXT: br i1 [[C]], label [[PRED_UDIV_CONTINUE80]], label [[VEC_EPILOG_VECTOR_BODY:%.*]]
-; CHECK: then71:
-; CHECK-NEXT: [[TMP134:%.*]] = call <8 x i32> @llvm.usub.sat.v8i32(<8 x i32> [[VEC_IND64]], <8 x i32> splat (i32 1))
-; CHECK-NEXT: [[TMP133:%.*]] = extractelement <8 x i1> [[TMP213]], i64 0
-; CHECK-NEXT: br i1 [[TMP133]], label [[PRED_UDIV_IF65:%.*]], label [[PRED_UDIV_CONTINUE66:%.*]]
-; CHECK: pred.udiv.if72:
-; CHECK-NEXT: [[TMP135:%.*]] = extractelement <8 x i32> [[TMP134]], i64 0
-; CHECK-NEXT: [[TMP136:%.*]] = udiv i32 [[TMP135]], [[D]]
-; CHECK-NEXT: [[TMP137:%.*]] = insertelement <8 x i32> poison, i32 [[TMP136]], i64 0
-; CHECK-NEXT: br label [[PRED_UDIV_CONTINUE66]]
-; CHECK: pred.udiv.continue73:
-; CHECK-NEXT: [[TMP138:%.*]] = phi <8 x i32> [ poison, [[VEC_EPILOG_VECTOR_BODY]] ], [ [[TMP137]], [[PRED_UDIV_IF65]] ]
-; CHECK-NEXT: [[TMP174:%.*]] = extractelement <8 x i1> [[TMP213]], i64 0
-; CHECK-NEXT: br i1 [[TMP174]], label [[PRED_UDIV_IF67:%.*]], label [[PRED_UDIV_CONTINUE68:%.*]]
-; CHECK: pred.udiv.if74:
-; CHECK-NEXT: [[TMP139:%.*]] = extractelement <8 x i32> [[TMP134]], i64 1
-; CHECK-NEXT: [[TMP140:%.*]] = udiv i32 [[TMP139]], [[D]]
-; CHECK-NEXT: [[TMP141:%.*]] = insertelement <8 x i32> [[TMP138]], i32 [[TMP140]], i64 1
-; CHECK-NEXT: br label [[PRED_UDIV_CONTINUE68]]
-; CHECK: pred.udiv.continue75:
-; CHECK-NEXT: [[TMP142:%.*]] = phi <8 x i32> [ [[TMP138]], [[PRED_UDIV_CONTINUE66]] ], [ [[TMP141]], [[PRED_UDIV_IF67]] ]
-; CHECK-NEXT: [[TMP179:%.*]] = extractelement <8 x i1> [[TMP213]], i64 0
-; CHECK-NEXT: br i1 [[TMP179]], label [[PRED_UDIV_IF69:%.*]], label [[PRED_UDIV_CONTINUE70:%.*]]
-; CHECK: pred.udiv.if76:
-; CHECK-NEXT: [[TMP143:%.*]] = extractelement <8 x i32> [[TMP134]], i64 2
-; CHECK-NEXT: [[TMP144:%.*]] = udiv i32 [[TMP143]], [[D]]
-; CHECK-NEXT: [[TMP145:%.*]] = insertelement <8 x i32> [[TMP142]], i32 [[TMP144]], i64 2
-; CHECK-NEXT: br label [[PRED_UDIV_CONTINUE70]]
-; CHECK: pred.udiv.continue77:
-; CHECK-NEXT: [[TMP146:%.*]] = phi <8 x i32> [ [[TMP142]], [[PRED_UDIV_CONTINUE68]] ], [ [[TMP145]], [[PRED_UDIV_IF69]] ]
-; CHECK-NEXT: [[TMP184:%.*]] = extractelement <8 x i1> [[TMP213]], i64 0
-; CHECK-NEXT: br i1 [[TMP184]], label [[PRED_UDIV_IF71:%.*]], label [[PRED_UDIV_CONTINUE72:%.*]]
-; CHECK: pred.udiv.if78:
-; CHECK-NEXT: [[TMP147:%.*]] = extractelement <8 x i32> [[TMP134]], i64 3
-; CHECK-NEXT: [[TMP148:%.*]] = udiv i32 [[TMP147]], [[D]]
-; CHECK-NEXT: [[TMP149:%.*]] = insertelement <8 x i32> [[TMP146]], i32 [[TMP148]], i64 3
-; CHECK-NEXT: br label [[PRED_UDIV_CONTINUE72]]
-; CHECK: pred.udiv.continue79:
-; CHECK-NEXT: [[TMP150:%.*]] = phi <8 x i32> [ [[TMP146]], [[PRED_UDIV_CONTINUE70]] ], [ [[TMP149]], [[PRED_UDIV_IF71]] ]
-; CHECK-NEXT: [[TMP189:%.*]] = extractelement <8 x i1> [[TMP213]], i64 0
-; CHECK-NEXT: br i1 [[TMP189]], label [[PRED_UDIV_IF73:%.*]], label [[PRED_UDIV_CONTINUE74:%.*]]
-; CHECK: pred.udiv.if80:
-; CHECK-NEXT: [[TMP151:%.*]] = extractelement <8 x i32> [[TMP134]], i64 4
-; CHECK-NEXT: [[TMP152:%.*]] = udiv i32 [[TMP151]], [[D]]
-; CHECK-NEXT: [[TMP153:%.*]] = insertelement <8 x i32> [[TMP150]], i32 [[TMP152]], i64 4
-; CHECK-NEXT: br label [[PRED_UDIV_CONTINUE74]]
-; CHECK: pred.udiv.continue81:
-; CHECK-NEXT: [[TMP154:%.*]] = phi <8 x i32> [ [[TMP150]], [[PRED_UDIV_CONTINUE72]] ], [ [[TMP153]], [[PRED_UDIV_IF73]] ]
-; CHECK-NEXT: [[TMP194:%.*]] = extractelement <8 x i1> [[TMP213]], i64 0
-; CHECK-NEXT: br i1 [[TMP194]], label [[PRED_UDIV_IF75:%.*]], label [[PRED_UDIV_CONTINUE76:%.*]]
-; CHECK: pred.udiv.if82:
-; CHECK-NEXT: [[TMP155:%.*]] = extractelement <8 x i32> [[TMP134]], i64 5
-; CHECK-NEXT: [[TMP156:%.*]] = udiv i32 [[TMP155]], [[D]]
-; CHECK-NEXT: [[TMP157:%.*]] = insertelement <8 x i32> [[TMP154]], i32 [[TMP156]], i64 5
-; CHECK-NEXT: br label [[PRED_UDIV_CONTINUE76]]
-; CHECK: pred.udiv.continue83:
-; CHECK-NEXT: [[TMP158:%.*]] = phi <8 x i32> [ [[TMP154]], [[PRED_UDIV_CONTINUE74]] ], [ [[TMP157]], [[PRED_UDIV_IF75]] ]
-; CHECK-NEXT: [[TMP199:%.*]] = extractelement <8 x i1> [[TMP213]], i64 0
-; CHECK-NEXT: br i1 [[TMP199]], label [[PRED_UDIV_IF77:%.*]], label [[PRED_UDIV_CONTINUE78:%.*]]
-; CHECK: pred.udiv.if84:
-; CHECK-NEXT: [[TMP159:%.*]] = extractelement <8 x i32> [[TMP134]], i64 6
-; CHECK-NEXT: [[TMP160:%.*]] = udiv i32 [[TMP159]], [[D]]
-; CHECK-NEXT: [[TMP161:%.*]] = insertelement <8 x i32> [[TMP158]], i32 [[TMP160]], i64 6
-; CHECK-NEXT: br label [[PRED_UDIV_CONTINUE78]]
-; CHECK: pred.udiv.continue85:
-; CHECK-NEXT: [[TMP162:%.*]] = phi <8 x i32> [ [[TMP158]], [[PRED_UDIV_CONTINUE76]] ], [ [[TMP161]], [[PRED_UDIV_IF77]] ]
-; CHECK-NEXT: [[TMP204:%.*]] = extractelement <8 x i1> [[TMP213]], i64 0
-; CHECK-NEXT: br i1 [[TMP204]], label [[PRED_UDIV_IF79:%.*]], label [[PRED_UDIV_CONTINUE87:%.*]]
-; CHECK: pred.udiv.if86:
-; CHECK-NEXT: [[TMP163:%.*]] = extractelement <8 x i32> [[TMP134]], i64 7
-; CHECK-NEXT: [[TMP164:%.*]] = udiv i32 [[TMP163]], [[D]]
-; CHECK-NEXT: [[TMP165:%.*]] = insertelement <8 x i32> [[TMP162]], i32 [[TMP164]], i64 7
-; CHECK-NEXT: br label [[PRED_UDIV_CONTINUE87]]
-; CHECK: pred.udiv.continue87:
-; CHECK-NEXT: [[TMP166:%.*]] = phi <8 x i32> [ [[TMP162]], [[PRED_UDIV_CONTINUE78]] ], [ [[TMP165]], [[PRED_UDIV_IF79]] ]
-; CHECK-NEXT: [[TMP209:%.*]] = zext <8 x i32> [[TMP166]] to <8 x i64>
-; CHECK-NEXT: br label [[PRED_UDIV_CONTINUE80]]
-; CHECK: loop.latch88:
-; CHECK-NEXT: [[TMP210:%.*]] = phi <8 x i64> [ poison, [[VEC_EPILOG_VECTOR_BODY1]] ], [ [[TMP209]], [[PRED_UDIV_CONTINUE87]] ]
-; CHECK-NEXT: [[TMP211:%.*]] = phi <8 x i1> [ zeroinitializer, [[VEC_EPILOG_VECTOR_BODY1]] ], [ [[TMP213]], [[PRED_UDIV_CONTINUE87]] ]
-; CHECK-NEXT: [[INDEX_NEXT81]] = add nuw i32 [[INDEX63]], 8
-; CHECK-NEXT: [[VEC_IND_NEXT82]] = add <8 x i32> [[VEC_IND64]], splat (i32 8)
-; CHECK-NEXT: [[TMP167:%.*]] = icmp eq i32 [[INDEX_NEXT81]], 1000
-; CHECK-NEXT: br i1 [[TMP167]], label [[VEC_EPILOG_MIDDLE_BLOCK:%.*]], label [[VEC_EPILOG_VECTOR_BODY1]], !llvm.loop [[LOOP14:![0-9]+]]
-; CHECK: vec.epilog.middle.block:
-; CHECK-NEXT: [[PREDPHI83:%.*]] = select <8 x i1> [[TMP211]], <8 x i64> [[TMP210]], <8 x i64> zeroinitializer
-; CHECK-NEXT: [[TMP169:%.*]] = extractelement <8 x i64> [[PREDPHI83]], i64 7
-; CHECK-NEXT: br i1 false, label [[EXIT]], label [[VEC_EPILOG_SCALAR_PH]]
-; CHECK: vec.epilog.scalar.ph:
-; CHECK-NEXT: [[BC_RESUME_VAL:%.*]] = phi i32 [ 1000, [[VEC_EPILOG_MIDDLE_BLOCK]] ], [ 992, [[VEC_EPILOG_ITER_CHECK]] ], [ 0, [[ITER_CHECK:%.*]] ]
+; CHECK-NEXT: entry:
; CHECK-NEXT: br label [[LOOP_HEADER:%.*]]
; CHECK: loop.header:
-; CHECK-NEXT: [[IV:%.*]] = phi i32 [ [[BC_RESUME_VAL]], [[VEC_EPILOG_SCALAR_PH]] ], [ [[IV_NEXT:%.*]], [[LOOP_LATCH:%.*]] ]
-; CHECK-NEXT: br i1 [[C]], label [[LOOP_LATCH]], label [[THEN:%.*]]
+; CHECK-NEXT: [[IV:%.*]] = phi i32 [ 0, [[ENTRY:%.*]] ], [ [[IV_NEXT:%.*]], [[LOOP_LATCH:%.*]] ]
+; CHECK-NEXT: br i1 [[C:%.*]], label [[LOOP_LATCH]], label [[THEN:%.*]]
; CHECK: then:
; CHECK-NEXT: [[CALL:%.*]] = tail call i32 @llvm.usub.sat.i32(i32 [[IV]], i32 1)
-; CHECK-NEXT: [[UDIV:%.*]] = udiv i32 [[CALL]], [[D]]
+; CHECK-NEXT: [[UDIV:%.*]] = udiv i32 [[CALL]], [[D:%.*]]
; CHECK-NEXT: [[ZEXT:%.*]] = zext i32 [[UDIV]] to i64
; CHECK-NEXT: br label [[LOOP_LATCH]]
; CHECK: loop.latch:
; CHECK-NEXT: [[MERGE:%.*]] = phi i64 [ [[ZEXT]], [[THEN]] ], [ 0, [[LOOP_HEADER]] ]
; CHECK-NEXT: [[IV_NEXT]] = add i32 [[IV]], 1
; CHECK-NEXT: [[EC:%.*]] = icmp eq i32 [[IV]], 1000
-; CHECK-NEXT: br i1 [[EC]], label [[EXIT]], label [[LOOP_HEADER]], !llvm.loop [[LOOP15:![0-9]+]]
+; CHECK-NEXT: br i1 [[EC]], label [[EXIT:%.*]], label [[LOOP_HEADER]]
; CHECK: exit:
-; CHECK-NEXT: [[MERGE_LCSSA:%.*]] = phi i64 [ [[MERGE]], [[LOOP_LATCH]] ], [ [[TMP132]], [[MIDDLE_BLOCK]] ], [ [[TMP169]], [[VEC_EPILOG_MIDDLE_BLOCK]] ]
+; CHECK-NEXT: [[MERGE_LCSSA:%.*]] = phi i64 [ [[MERGE]], [[LOOP_LATCH]] ]
; CHECK-NEXT: ret i64 [[MERGE_LCSSA]]
;
entry:
diff --git a/llvm/test/Transforms/LoopVectorize/X86/invariant-load-gather.ll b/llvm/test/Transforms/LoopVectorize/X86/invariant-load-gather.ll
index 141061587c78d..9661c9505f7da 100644
--- a/llvm/test/Transforms/LoopVectorize/X86/invariant-load-gather.ll
+++ b/llvm/test/Transforms/LoopVectorize/X86/invariant-load-gather.ll
@@ -26,24 +26,23 @@ define i32 @inv_load_conditional(ptr %a, i64 %n, ptr %b, i32 %k) {
; CHECK: vector.ph:
; CHECK-NEXT: [[N_MOD_VF:%.*]] = and i64 [[SMAX2]], 15
; CHECK-NEXT: [[N_VEC:%.*]] = sub i64 [[SMAX2]], [[N_MOD_VF]]
-; CHECK-NEXT: [[BROADCAST_SPLATINSERT:%.*]] = insertelement <16 x ptr> poison, ptr [[A]], i64 0
-; CHECK-NEXT: [[BROADCAST_SPLAT:%.*]] = shufflevector <16 x ptr> [[BROADCAST_SPLATINSERT]], <16 x ptr> poison, <16 x i32> zeroinitializer
-; CHECK-NEXT: [[TMP1:%.*]] = icmp ne <16 x ptr> [[BROADCAST_SPLAT]], splat (ptr null)
+; CHECK-NEXT: [[TMP9:%.*]] = icmp ne ptr [[A]], null
; CHECK-NEXT: [[BROADCAST_SPLATINSERT4:%.*]] = insertelement <16 x i32> poison, i32 [[NTRUNC]], i64 0
; CHECK-NEXT: [[BROADCAST_SPLAT5:%.*]] = shufflevector <16 x i32> [[BROADCAST_SPLATINSERT4]], <16 x i32> poison, <16 x i32> zeroinitializer
+; CHECK-NEXT: [[BROADCAST_SPLATINSERT5:%.*]] = insertelement <16 x ptr> poison, ptr [[A]], i64 0
+; CHECK-NEXT: [[BROADCAST_SPLAT6:%.*]] = shufflevector <16 x ptr> [[BROADCAST_SPLATINSERT5]], <16 x ptr> poison, <16 x i32> zeroinitializer
; CHECK-NEXT: br label [[VECTOR_BODY:%.*]]
; CHECK: vector.body:
; CHECK-NEXT: [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[LATCH7:%.*]] ]
; CHECK-NEXT: [[TMP2:%.*]] = getelementptr inbounds i32, ptr [[B]], i64 [[INDEX]]
; CHECK-NEXT: store <16 x i32> [[BROADCAST_SPLAT5]], ptr [[TMP2]], align 4, !alias.scope [[META0:![0-9]+]], !noalias [[META3:![0-9]+]]
-; CHECK-NEXT: [[TMP9:%.*]] = extractelement <16 x i1> [[TMP1]], i64 0
; CHECK-NEXT: br i1 [[TMP9]], label [[COND_LOAD6:%.*]], label [[LATCH7]]
; CHECK: cond_load6:
-; CHECK-NEXT: [[WIDE_MASKED_GATHER:%.*]] = call <16 x i32> @llvm.masked.gather.v16i32.v16p0(<16 x ptr> align 4 [[BROADCAST_SPLAT]], <16 x i1> [[TMP1]], <16 x i32> poison), !alias.scope [[META3]]
+; CHECK-NEXT: [[WIDE_MASKED_GATHER:%.*]] = call <16 x i32> @llvm.masked.gather.v16i32.v16p0(<16 x ptr> align 4 [[BROADCAST_SPLAT6]], <16 x i1> splat (i1 true), <16 x i32> poison), !alias.scope [[META3]]
; CHECK-NEXT: br label [[LATCH7]]
; CHECK: latch7:
; CHECK-NEXT: [[TMP10:%.*]] = phi <16 x i32> [ poison, [[VECTOR_BODY]] ], [ [[WIDE_MASKED_GATHER]], [[COND_LOAD6]] ]
-; CHECK-NEXT: [[TMP11:%.*]] = phi <16 x i1> [ zeroinitializer, [[VECTOR_BODY]] ], [ [[TMP1]], [[COND_LOAD6]] ]
+; CHECK-NEXT: [[TMP11:%.*]] = phi <16 x i1> [ zeroinitializer, [[VECTOR_BODY]] ], [ splat (i1 true), [[COND_LOAD6]] ]
; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 16
; CHECK-NEXT: [[TMP3:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
; CHECK-NEXT: br i1 [[TMP3]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP5:![0-9]+]]
@@ -59,24 +58,23 @@ define i32 @inv_load_conditional(ptr %a, i64 %n, ptr %b, i32 %k) {
; CHECK-NEXT: [[VEC_EPILOG_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC]], [[VEC_EPILOG_ITER_CHECK]] ], [ 0, [[VECTOR_MAIN_LOOP_ITER_CHECK]] ]
; CHECK-NEXT: [[N_MOD_VF6:%.*]] = and i64 [[SMAX2]], 7
; CHECK-NEXT: [[N_VEC7:%.*]] = sub i64 [[SMAX2]], [[N_MOD_VF6]]
-; CHECK-NEXT: [[BROADCAST_SPLATINSERT8:%.*]] = insertelement <8 x ptr> poison, ptr [[A]], i64 0
-; CHECK-NEXT: [[BROADCAST_SPLAT9:%.*]] = shufflevector <8 x ptr> [[BROADCAST_SPLATINSERT8]], <8 x ptr> poison, <8 x i32> zeroinitializer
-; CHECK-NEXT: [[TMP5:%.*]] = icmp ne <8 x ptr> [[BROADCAST_SPLAT9]], splat (ptr null)
+; CHECK-NEXT: [[TMP12:%.*]] = icmp ne ptr [[A]], null
; CHECK-NEXT: [[BROADCAST_SPLATINSERT11:%.*]] = insertelement <8 x i32> poison, i32 [[NTRUNC]], i64 0
; CHECK-NEXT: [[BROADCAST_SPLAT12:%.*]] = shufflevector <8 x i32> [[BROADCAST_SPLATINSERT11]], <8 x i32> poison, <8 x i32> zeroinitializer
+; CHECK-NEXT: [[BROADCAST_SPLATINSERT12:%.*]] = insertelement <8 x ptr> poison, ptr [[A]], i64 0
+; CHECK-NEXT: [[BROADCAST_SPLAT13:%.*]] = shufflevector <8 x ptr> [[BROADCAST_SPLATINSERT12]], <8 x ptr> poison, <8 x i32> zeroinitializer
; CHECK-NEXT: br label [[VEC_EPILOG_VECTOR_BODY:%.*]]
; CHECK: vec.epilog.vector.body:
; CHECK-NEXT: [[INDEX10:%.*]] = phi i64 [ [[VEC_EPILOG_RESUME_VAL]], [[VEC_EPILOG_PH]] ], [ [[INDEX_NEXT15:%.*]], [[LATCH16:%.*]] ]
; CHECK-NEXT: [[TMP6:%.*]] = getelementptr inbounds i32, ptr [[B]], i64 [[INDEX10]]
; CHECK-NEXT: store <8 x i32> [[BROADCAST_SPLAT12]], ptr [[TMP6]], align 4, !alias.scope [[META0]], !noalias [[META3]]
-; CHECK-NEXT: [[TMP12:%.*]] = extractelement <8 x i1> [[TMP5]], i64 0
; CHECK-NEXT: br i1 [[TMP12]], label [[COND_LOAD14:%.*]], label [[LATCH16]]
; CHECK: cond_load14:
-; CHECK-NEXT: [[WIDE_MASKED_GATHER15:%.*]] = call <8 x i32> @llvm.masked.gather.v8i32.v8p0(<8 x ptr> align 4 [[BROADCAST_SPLAT9]], <8 x i1> [[TMP5]], <8 x i32> poison), !alias.scope [[META3]]
+; CHECK-NEXT: [[WIDE_MASKED_GATHER15:%.*]] = call <8 x i32> @llvm.masked.gather.v8i32.v8p0(<8 x ptr> align 4 [[BROADCAST_SPLAT13]], <8 x i1> splat (i1 true), <8 x i32> poison), !alias.scope [[META3]]
; CHECK-NEXT: br label [[LATCH16]]
; CHECK: latch16:
; CHECK-NEXT: [[TMP13:%.*]] = phi <8 x i32> [ poison, [[VEC_EPILOG_VECTOR_BODY]] ], [ [[WIDE_MASKED_GATHER15]], [[COND_LOAD14]] ]
-; CHECK-NEXT: [[TMP14:%.*]] = phi <8 x i1> [ zeroinitializer, [[VEC_EPILOG_VECTOR_BODY]] ], [ [[TMP5]], [[COND_LOAD14]] ]
+; CHECK-NEXT: [[TMP14:%.*]] = phi <8 x i1> [ zeroinitializer, [[VEC_EPILOG_VECTOR_BODY]] ], [ splat (i1 true), [[COND_LOAD14]] ]
; CHECK-NEXT: [[INDEX_NEXT15]] = add nuw i64 [[INDEX10]], 8
; CHECK-NEXT: [[TMP7:%.*]] = icmp eq i64 [[INDEX_NEXT15]], [[N_VEC7]]
; CHECK-NEXT: br i1 [[TMP7]], label [[VEC_EPILOG_MIDDLE_BLOCK:%.*]], label [[VEC_EPILOG_VECTOR_BODY]], !llvm.loop [[LOOP9:![0-9]+]]
diff --git a/llvm/test/Transforms/LoopVectorize/X86/masked-store-cost.ll b/llvm/test/Transforms/LoopVectorize/X86/masked-store-cost.ll
index 4faabbc7b8f35..ba84fbbd5eeda 100644
--- a/llvm/test/Transforms/LoopVectorize/X86/masked-store-cost.ll
+++ b/llvm/test/Transforms/LoopVectorize/X86/masked-store-cost.ll
@@ -125,8 +125,6 @@ define void @test_scalar_cost_single_store_loop_invariant_cond(ptr %dst, i1 %c)
; CHECK-NEXT: br label %[[VECTOR_PH:.*]]
; CHECK: [[VECTOR_PH]]:
; CHECK-NEXT: [[TMP0:%.*]] = getelementptr i8, ptr [[DST]], i64 96
-; CHECK-NEXT: [[BROADCAST_SPLATINSERT:%.*]] = insertelement <8 x i1> poison, i1 [[C]], i64 0
-; CHECK-NEXT: [[BROADCAST_SPLAT:%.*]] = shufflevector <8 x i1> [[BROADCAST_SPLATINSERT]], <8 x i1> poison, <8 x i32> zeroinitializer
; CHECK-NEXT: br label %[[VECTOR_BODY:.*]]
; CHECK: [[VECTOR_BODY]]:
; CHECK-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[LOOP_LATCH2:.*]] ]
@@ -134,7 +132,7 @@ define void @test_scalar_cost_single_store_loop_invariant_cond(ptr %dst, i1 %c)
; CHECK-NEXT: [[NEXT_GEP:%.*]] = getelementptr i8, ptr [[DST]], i64 [[TMP1]]
; CHECK-NEXT: br i1 [[C]], label %[[IF_THEN1:.*]], label %[[LOOP_LATCH2]]
; CHECK: [[IF_THEN1]]:
-; CHECK-NEXT: call void @llvm.masked.store.v8i32.p0(<8 x i32> zeroinitializer, ptr align 4 [[NEXT_GEP]], <8 x i1> [[BROADCAST_SPLAT]])
+; CHECK-NEXT: store <8 x i32> zeroinitializer, ptr [[NEXT_GEP]], align 4
; CHECK-NEXT: br label %[[LOOP_LATCH2]]
; CHECK: [[LOOP_LATCH2]]:
; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 8
diff --git a/llvm/test/Transforms/LoopVectorize/X86/narrow-to-single-scalar.ll b/llvm/test/Transforms/LoopVectorize/X86/narrow-to-single-scalar.ll
index 94a05a67a0bdc..82215b39d91fb 100644
--- a/llvm/test/Transforms/LoopVectorize/X86/narrow-to-single-scalar.ll
+++ b/llvm/test/Transforms/LoopVectorize/X86/narrow-to-single-scalar.ll
@@ -9,23 +9,25 @@ target triple = "x86_64-unknown-linux-gnu"
define void @narrow_store_user_mask_operand(i32 %x) {
; CHECK-LABEL: define void @narrow_store_user_mask_operand(
; CHECK-SAME: i32 [[X:%.*]]) #[[ATTR0:[0-9]+]] {
-; CHECK-NEXT: [[ENTRY:.*]]:
+; CHECK-NEXT: [[ENTRY:.*:]]
; CHECK-NEXT: br label %[[LOOP_PH:.*]]
; CHECK: [[LOOP_PH]]:
-; CHECK-NEXT: [[IV:%.*]] = phi i64 [ 0, %[[ENTRY]] ], [ [[IV_NEXT:%.*]], %[[LOOP_TAIL:.*]] ]
; CHECK-NEXT: [[X_POS:%.*]] = icmp sgt i32 [[X]], 0
-; CHECK-NEXT: br i1 [[X_POS]], label %[[LOOP_BODY:.*]], label %[[LOOP_TAIL]]
+; CHECK-NEXT: br label %[[VECTOR_BODY:.*]]
+; CHECK: [[VECTOR_BODY]]:
+; CHECK-NEXT: br i1 [[X_POS]], label %[[LOOP_BODY:.*]], label %[[LOOP_TAIL:.*]]
; CHECK: [[LOOP_BODY]]:
; CHECK-NEXT: [[LD_P:%.*]] = load double, ptr @p, align 8
-; CHECK-NEXT: [[GEP_Q_IV:%.*]] = getelementptr double, ptr @q, i64 [[IV]]
-; CHECK-NEXT: [[GEP_Q_IV_8:%.*]] = getelementptr i8, ptr [[GEP_Q_IV]], i64 -8
-; CHECK-NEXT: store double [[LD_P]], ptr [[GEP_Q_IV_8]], align 8
+; CHECK-NEXT: [[TMP2:%.*]] = load double, ptr @p, align 8
+; CHECK-NEXT: [[TMP3:%.*]] = insertelement <2 x double> poison, double [[LD_P]], i64 0
+; CHECK-NEXT: [[TMP4:%.*]] = insertelement <2 x double> [[TMP3]], double [[TMP2]], i64 1
+; CHECK-NEXT: store <2 x double> [[TMP4]], ptr getelementptr (i8, ptr @q, i64 -8), align 8
; CHECK-NEXT: br label %[[LOOP_TAIL]]
; CHECK: [[LOOP_TAIL]]:
-; CHECK-NEXT: [[IV_NEXT]] = add i64 [[IV]], 1
-; CHECK-NEXT: [[EC:%.*]] = icmp eq i64 [[IV]], 1
-; CHECK-NEXT: br i1 [[EC]], label %[[EXIT:.*]], label %[[LOOP_PH]]
+; CHECK-NEXT: br label %[[EXIT:.*]]
; CHECK: [[EXIT]]:
+; CHECK-NEXT: br label %[[EXIT1:.*]]
+; CHECK: [[EXIT1]]:
; CHECK-NEXT: ret void
;
entry:
diff --git a/llvm/test/Transforms/LoopVectorize/X86/pr141968-instsimplifyfolder.ll b/llvm/test/Transforms/LoopVectorize/X86/pr141968-instsimplifyfolder.ll
index 4db9db9598ae4..bbd18674a2664 100644
--- a/llvm/test/Transforms/LoopVectorize/X86/pr141968-instsimplifyfolder.ll
+++ b/llvm/test/Transforms/LoopVectorize/X86/pr141968-instsimplifyfolder.ll
@@ -11,9 +11,6 @@ define void @pr141968(i1 %cond, i8 %v, ptr %p) {
; CHECK-NEXT: [[SEXT:%.*]] = sext i8 [[V]] to i16
; CHECK-NEXT: br label %[[PRED_SDIV_CONTINUE28:.*]]
; CHECK: [[PRED_SDIV_CONTINUE28]]:
-; CHECK-NEXT: [[BROADCAST_SPLATINSERT:%.*]] = insertelement <16 x i1> poison, i1 [[COND]], i64 0
-; CHECK-NEXT: [[BROADCAST_SPLAT:%.*]] = shufflevector <16 x i1> [[BROADCAST_SPLATINSERT]], <16 x i1> poison, <16 x i32> zeroinitializer
-; CHECK-NEXT: [[TMP0:%.*]] = xor <16 x i1> [[BROADCAST_SPLAT]], splat (i1 true)
; CHECK-NEXT: [[BROADCAST_SPLATINSERT1:%.*]] = insertelement <16 x i8> poison, i8 [[V]], i64 0
; CHECK-NEXT: [[BROADCAST_SPLAT2:%.*]] = shufflevector <16 x i8> [[BROADCAST_SPLATINSERT1]], <16 x i8> poison, <16 x i32> zeroinitializer
; CHECK-NEXT: br label %[[VECTOR_BODY:.*]]
@@ -23,7 +20,7 @@ define void @pr141968(i1 %cond, i8 %v, ptr %p) {
; CHECK: [[PRED_SDIV_IF29]]:
; CHECK-NEXT: br label %[[PRED_SDIV_CONTINUE30]]
; CHECK: [[PRED_SDIV_CONTINUE30]]:
-; CHECK-NEXT: [[TMP1:%.*]] = phi <16 x i1> [ zeroinitializer, %[[VECTOR_BODY]] ], [ [[TMP0]], %[[PRED_SDIV_IF29]] ]
+; CHECK-NEXT: [[TMP1:%.*]] = phi <16 x i1> [ zeroinitializer, %[[VECTOR_BODY]] ], [ splat (i1 true), %[[PRED_SDIV_IF29]] ]
; CHECK-NEXT: [[PREDPHI1:%.*]] = select <16 x i1> [[TMP1]], <16 x i8> [[BROADCAST_SPLAT2]], <16 x i8> zeroinitializer
; CHECK-NEXT: [[PREDPHI:%.*]] = extractelement <16 x i8> [[PREDPHI1]], i64 15
; CHECK-NEXT: store i8 [[PREDPHI]], ptr [[P]], align 1
diff --git a/llvm/test/Transforms/LoopVectorize/X86/predicated-instruction-cost.ll b/llvm/test/Transforms/LoopVectorize/X86/predicated-instruction-cost.ll
index 5a479250e1490..aa4f9c83a68c2 100644
--- a/llvm/test/Transforms/LoopVectorize/X86/predicated-instruction-cost.ll
+++ b/llvm/test/Transforms/LoopVectorize/X86/predicated-instruction-cost.ll
@@ -57,23 +57,50 @@ exit:
define void @test_wide_shift_uses_predicated_invariant_instruction(i32 %d, i1 %c, ptr %dst) {
; CHECK-LABEL: define void @test_wide_shift_uses_predicated_invariant_instruction(
; CHECK-SAME: i32 [[D:%.*]], i1 [[C:%.*]], ptr [[DST:%.*]]) {
-; CHECK-NEXT: [[ENTRY:.*]]:
+; CHECK-NEXT: [[ENTRY:.*:]]
; CHECK-NEXT: br label %[[LOOP_HEADER:.*]]
; CHECK: [[LOOP_HEADER]]:
-; CHECK-NEXT: [[IV:%.*]] = phi i32 [ 0, %[[ENTRY]] ], [ [[IV_NEXT:%.*]], %[[LOOP_LATCH:.*]] ]
+; CHECK-NEXT: [[TMP0:%.*]] = urem i32 100, [[D]]
+; CHECK-NEXT: [[BROADCAST_SPLATINSERT:%.*]] = insertelement <4 x i32> poison, i32 [[TMP0]], i64 0
+; CHECK-NEXT: [[BROADCAST_SPLAT:%.*]] = shufflevector <4 x i32> [[BROADCAST_SPLATINSERT]], <4 x i32> poison, <4 x i32> zeroinitializer
+; CHECK-NEXT: [[TMP1:%.*]] = shl <4 x i32> [[BROADCAST_SPLAT]], splat (i32 12)
+; CHECK-NEXT: [[TMP2:%.*]] = shl <4 x i32> splat (i32 999), [[TMP1]]
+; CHECK-NEXT: br label %[[VECTOR_BODY:.*]]
+; CHECK: [[VECTOR_BODY]]:
+; CHECK-NEXT: [[INDEX:%.*]] = phi i32 [ 0, %[[LOOP_HEADER]] ], [ [[INDEX_NEXT:%.*]], %[[LOOP_LATCH:.*]] ]
; CHECK-NEXT: br i1 [[C]], label %[[LOOP_LATCH]], label %[[ELSE:.*]]
; CHECK: [[ELSE]]:
+; CHECK-NEXT: br label %[[LOOP_LATCH]]
+; CHECK: [[LOOP_LATCH]]:
+; CHECK-NEXT: [[TMP3:%.*]] = phi <4 x i32> [ poison, %[[VECTOR_BODY]] ], [ [[TMP2]], %[[ELSE]] ]
+; CHECK-NEXT: [[TMP4:%.*]] = phi <4 x i1> [ zeroinitializer, %[[VECTOR_BODY]] ], [ splat (i1 true), %[[ELSE]] ]
+; CHECK-NEXT: [[PREDPHI:%.*]] = select <4 x i1> [[TMP4]], <4 x i32> [[TMP3]], <4 x i32> zeroinitializer
+; CHECK-NEXT: [[TMP5:%.*]] = getelementptr inbounds i32, ptr [[DST]], i32 [[INDEX]]
+; CHECK-NEXT: [[TMP6:%.*]] = getelementptr inbounds i32, ptr [[TMP5]], i64 4
+; CHECK-NEXT: store <4 x i32> [[PREDPHI]], ptr [[TMP5]], align 4
+; CHECK-NEXT: store <4 x i32> [[PREDPHI]], ptr [[TMP6]], align 4
+; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i32 [[INDEX]], 8
+; CHECK-NEXT: [[TMP7:%.*]] = icmp eq i32 [[INDEX_NEXT]], 96
+; CHECK-NEXT: br i1 [[TMP7]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP0:![0-9]+]]
+; CHECK: [[MIDDLE_BLOCK]]:
+; CHECK-NEXT: br label %[[SCALAR_PH:.*]]
+; CHECK: [[SCALAR_PH]]:
+; CHECK-NEXT: br label %[[LOOP_HEADER1:.*]]
+; CHECK: [[LOOP_HEADER1]]:
+; CHECK-NEXT: [[IV:%.*]] = phi i32 [ 96, %[[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], %[[LOOP_LATCH1:.*]] ]
+; CHECK-NEXT: br i1 [[C]], label %[[LOOP_LATCH1]], label %[[ELSE1:.*]]
+; CHECK: [[ELSE1]]:
; CHECK-NEXT: [[REM:%.*]] = urem i32 100, [[D]]
; CHECK-NEXT: [[SEXT:%.*]] = shl i32 [[REM]], 12
; CHECK-NEXT: [[SHL_I:%.*]] = shl i32 999, [[SEXT]]
-; CHECK-NEXT: br label %[[LOOP_LATCH]]
-; CHECK: [[LOOP_LATCH]]:
-; CHECK-NEXT: [[P:%.*]] = phi i32 [ [[SHL_I]], %[[ELSE]] ], [ 0, %[[LOOP_HEADER]] ]
+; CHECK-NEXT: br label %[[LOOP_LATCH1]]
+; CHECK: [[LOOP_LATCH1]]:
+; CHECK-NEXT: [[P:%.*]] = phi i32 [ [[SHL_I]], %[[ELSE1]] ], [ 0, %[[LOOP_HEADER1]] ]
; CHECK-NEXT: [[GEP:%.*]] = getelementptr inbounds i32, ptr [[DST]], i32 [[IV]]
; CHECK-NEXT: store i32 [[P]], ptr [[GEP]], align 4
; CHECK-NEXT: [[IV_NEXT]] = add i32 [[IV]], 1
; CHECK-NEXT: [[EC:%.*]] = icmp eq i32 [[IV_NEXT]], 100
-; CHECK-NEXT: br i1 [[EC]], label %[[EXIT:.*]], label %[[LOOP_HEADER]]
+; CHECK-NEXT: br i1 [[EC]], label %[[EXIT:.*]], label %[[LOOP_HEADER1]], !llvm.loop [[LOOP3:![0-9]+]]
; CHECK: [[EXIT]]:
; CHECK-NEXT: ret void
;
@@ -238,7 +265,7 @@ define i32 @predicated_store_with_scatter_legal(ptr %dst, i64 %n) #0 {
; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 8
; CHECK-NEXT: [[VEC_IND_NEXT]] = add <8 x i64> [[VEC_IND]], splat (i64 8)
; CHECK-NEXT: [[TMP16:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
-; CHECK-NEXT: br i1 [[TMP16]], label %[[MIDDLE_BLOCK:.*]], label %[[LOOP]], !llvm.loop [[LOOP0:![0-9]+]]
+; CHECK-NEXT: br i1 [[TMP16]], label %[[MIDDLE_BLOCK:.*]], label %[[LOOP]], !llvm.loop [[LOOP4:![0-9]+]]
; CHECK: [[MIDDLE_BLOCK]]:
; CHECK-NEXT: [[CMP_N:%.*]] = icmp eq i64 [[TMP0]], [[N_VEC]]
; CHECK-NEXT: br i1 [[CMP_N]], label %[[EXIT:.*]], label %[[SCALAR_PH]]
@@ -264,7 +291,7 @@ define i32 @predicated_store_with_scatter_legal(ptr %dst, i64 %n) #0 {
; CHECK: [[LATCH]]:
; CHECK-NEXT: [[IV_NEXT]] = add i64 [[IV]], 1
; CHECK-NEXT: [[EC:%.*]] = icmp eq i64 [[IV]], [[N]]
-; CHECK-NEXT: br i1 [[EC]], label %[[EXIT]], label %[[LOOP1]], !llvm.loop [[LOOP3:![0-9]+]]
+; CHECK-NEXT: br i1 [[EC]], label %[[EXIT]], label %[[LOOP1]], !llvm.loop [[LOOP5:![0-9]+]]
; CHECK: [[EXIT]]:
; CHECK-NEXT: ret i32 0
;
diff --git a/llvm/test/Transforms/LoopVectorize/X86/predicated-replicate-feeding-cast.ll b/llvm/test/Transforms/LoopVectorize/X86/predicated-replicate-feeding-cast.ll
index b71828f574a80..8180d3088fafa 100644
--- a/llvm/test/Transforms/LoopVectorize/X86/predicated-replicate-feeding-cast.ll
+++ b/llvm/test/Transforms/LoopVectorize/X86/predicated-replicate-feeding-cast.ll
@@ -14,68 +14,37 @@ define i8 @predicated_replicate_feeding_cast(i16 %n, i1 %c1, i1 %c2, i16 %a, i8
; CHECK: [[VECTOR_PH]]:
; CHECK-NEXT: [[N_MOD_VF:%.*]] = and i32 [[TMP1]], 1
; CHECK-NEXT: [[N_VEC:%.*]] = sub i32 [[TMP1]], [[N_MOD_VF]]
-; CHECK-NEXT: [[BROADCAST_SPLATINSERT:%.*]] = insertelement <2 x i1> poison, i1 [[C2]], i64 0
-; CHECK-NEXT: [[BROADCAST_SPLAT:%.*]] = shufflevector <2 x i1> [[BROADCAST_SPLATINSERT]], <2 x i1> poison, <2 x i32> zeroinitializer
-; CHECK-NEXT: [[BROADCAST_SPLATINSERT1:%.*]] = insertelement <2 x i1> poison, i1 [[C1]], i64 0
-; CHECK-NEXT: [[BROADCAST_SPLAT2:%.*]] = shufflevector <2 x i1> [[BROADCAST_SPLATINSERT1]], <2 x i1> poison, <2 x i32> zeroinitializer
; CHECK-NEXT: [[TMP2:%.*]] = trunc i32 [[N_VEC]] to i16
-; CHECK-NEXT: [[TMP6:%.*]] = xor <2 x i1> [[BROADCAST_SPLAT]], splat (i1 true)
-; CHECK-NEXT: [[TMP7:%.*]] = xor <2 x i1> [[BROADCAST_SPLAT2]], splat (i1 true)
-; CHECK-NEXT: br label %[[VECTOR_BODY:.*]]
-; CHECK: [[VECTOR_BODY]]:
-; CHECK-NEXT: [[INDEX:%.*]] = phi i32 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[LATCH12:.*]] ]
-; CHECK-NEXT: br i1 [[C1]], label %[[MERGE6:.*]], label %[[IF13:.*]]
-; CHECK: [[IF13]]:
-; CHECK-NEXT: [[TMP3:%.*]] = extractelement <2 x i1> [[TMP7]], i64 0
-; CHECK-NEXT: br i1 [[TMP3]], label %[[PRED_SDIV_IF:.*]], label %[[PRED_SDIV_CONTINUE:.*]]
-; CHECK: [[PRED_SDIV_IF]]:
; CHECK-NEXT: [[TMP5:%.*]] = sdiv i16 1, [[A]]
; CHECK-NEXT: [[TMP8:%.*]] = insertelement <2 x i16> poison, i16 [[TMP5]], i64 0
-; CHECK-NEXT: br label %[[PRED_SDIV_CONTINUE]]
-; CHECK: [[PRED_SDIV_CONTINUE]]:
-; CHECK-NEXT: [[TMP9:%.*]] = phi <2 x i16> [ poison, %[[IF13]] ], [ [[TMP8]], %[[PRED_SDIV_IF]] ]
-; CHECK-NEXT: [[TMP10:%.*]] = extractelement <2 x i1> [[TMP7]], i64 0
-; CHECK-NEXT: br i1 [[TMP10]], label %[[PRED_SDIV_IF1:.*]], label %[[PRED_SDIV_CONTINUE2:.*]]
-; CHECK: [[PRED_SDIV_IF1]]:
-; CHECK-NEXT: [[TMP11:%.*]] = sdiv i16 1, [[A]]
-; CHECK-NEXT: [[TMP16:%.*]] = insertelement <2 x i16> [[TMP9]], i16 [[TMP11]], i64 1
-; CHECK-NEXT: br label %[[PRED_SDIV_CONTINUE2]]
+; CHECK-NEXT: [[BROADCAST_SPLAT:%.*]] = shufflevector <2 x i16> [[TMP8]], <2 x i16> poison, <2 x i32> zeroinitializer
+; CHECK-NEXT: br label %[[PRED_SDIV_CONTINUE2:.*]]
; CHECK: [[PRED_SDIV_CONTINUE2]]:
-; CHECK-NEXT: [[TMP20:%.*]] = phi <2 x i16> [ [[TMP9]], %[[PRED_SDIV_CONTINUE]] ], [ [[TMP16]], %[[PRED_SDIV_IF1]] ]
+; CHECK-NEXT: [[INDEX:%.*]] = phi i32 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[LATCH12:.*]] ]
+; CHECK-NEXT: br i1 [[C1]], label %[[MERGE6:.*]], label %[[IF11:.*]]
+; CHECK: [[IF11]]:
; CHECK-NEXT: br label %[[MERGE6]]
; CHECK: [[MERGE6]]:
-; CHECK-NEXT: [[TMP25:%.*]] = phi <2 x i16> [ poison, %[[VECTOR_BODY]] ], [ [[TMP20]], %[[PRED_SDIV_CONTINUE2]] ]
-; CHECK-NEXT: [[TMP26:%.*]] = phi <2 x i1> [ zeroinitializer, %[[VECTOR_BODY]] ], [ [[TMP7]], %[[PRED_SDIV_CONTINUE2]] ]
+; CHECK-NEXT: [[TMP25:%.*]] = phi <2 x i16> [ poison, %[[PRED_SDIV_CONTINUE2]] ], [ [[BROADCAST_SPLAT]], %[[IF11]] ]
+; CHECK-NEXT: [[TMP26:%.*]] = phi <2 x i1> [ zeroinitializer, %[[PRED_SDIV_CONTINUE2]] ], [ splat (i1 true), %[[IF11]] ]
; CHECK-NEXT: [[PREDPHI:%.*]] = select <2 x i1> [[TMP26]], <2 x i16> [[TMP25]], <2 x i16> zeroinitializer
; CHECK-NEXT: br i1 [[C2]], label %[[LATCH12]], label %[[IF27:.*]]
; CHECK: [[IF27]]:
-; CHECK-NEXT: [[TMP4:%.*]] = extractelement <2 x i1> [[TMP6]], i64 0
-; CHECK-NEXT: br i1 [[TMP4]], label %[[PRED_SDIV_IF3:.*]], label %[[PRED_SDIV_CONTINUE4:.*]]
-; CHECK: [[PRED_SDIV_IF3]]:
; CHECK-NEXT: [[TMP17:%.*]] = extractelement <2 x i16> [[PREDPHI]], i64 0
; CHECK-NEXT: [[TMP12:%.*]] = trunc i16 [[TMP17]] to i8
-; CHECK-NEXT: [[TMP13:%.*]] = sdiv i8 [[TMP12]], [[B]]
-; CHECK-NEXT: [[TMP14:%.*]] = insertelement <2 x i8> poison, i8 [[TMP13]], i64 0
-; CHECK-NEXT: br label %[[PRED_SDIV_CONTINUE4]]
-; CHECK: [[PRED_SDIV_CONTINUE4]]:
-; CHECK-NEXT: [[TMP15:%.*]] = phi <2 x i8> [ poison, %[[IF27]] ], [ [[TMP14]], %[[PRED_SDIV_IF3]] ]
-; CHECK-NEXT: [[TMP30:%.*]] = extractelement <2 x i1> [[TMP6]], i64 0
-; CHECK-NEXT: br i1 [[TMP30]], label %[[PRED_SDIV_IF5:.*]], label %[[PRED_SDIV_CONTINUE6:.*]]
-; CHECK: [[PRED_SDIV_IF5]]:
; CHECK-NEXT: [[TMP23:%.*]] = extractelement <2 x i16> [[PREDPHI]], i64 1
; CHECK-NEXT: [[TMP24:%.*]] = trunc i16 [[TMP23]] to i8
+; CHECK-NEXT: [[TMP11:%.*]] = sdiv i8 [[TMP12]], [[B]]
; CHECK-NEXT: [[TMP18:%.*]] = sdiv i8 [[TMP24]], [[B]]
+; CHECK-NEXT: [[TMP15:%.*]] = insertelement <2 x i8> poison, i8 [[TMP11]], i64 0
; CHECK-NEXT: [[TMP19:%.*]] = insertelement <2 x i8> [[TMP15]], i8 [[TMP18]], i64 1
-; CHECK-NEXT: br label %[[PRED_SDIV_CONTINUE6]]
-; CHECK: [[PRED_SDIV_CONTINUE6]]:
-; CHECK-NEXT: [[TMP27:%.*]] = phi <2 x i8> [ [[TMP15]], %[[PRED_SDIV_CONTINUE4]] ], [ [[TMP19]], %[[PRED_SDIV_IF5]] ]
; CHECK-NEXT: br label %[[LATCH12]]
; CHECK: [[LATCH12]]:
-; CHECK-NEXT: [[TMP28:%.*]] = phi <2 x i8> [ poison, %[[MERGE6]] ], [ [[TMP27]], %[[PRED_SDIV_CONTINUE6]] ]
-; CHECK-NEXT: [[TMP29:%.*]] = phi <2 x i1> [ zeroinitializer, %[[MERGE6]] ], [ [[TMP6]], %[[PRED_SDIV_CONTINUE6]] ]
+; CHECK-NEXT: [[TMP28:%.*]] = phi <2 x i8> [ poison, %[[MERGE6]] ], [ [[TMP19]], %[[IF27]] ]
+; CHECK-NEXT: [[TMP29:%.*]] = phi <2 x i1> [ zeroinitializer, %[[MERGE6]] ], [ splat (i1 true), %[[IF27]] ]
; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i32 [[INDEX]], 2
; CHECK-NEXT: [[TMP21:%.*]] = icmp eq i32 [[INDEX_NEXT]], [[N_VEC]]
-; CHECK-NEXT: br i1 [[TMP21]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP0:![0-9]+]]
+; CHECK-NEXT: br i1 [[TMP21]], label %[[MIDDLE_BLOCK:.*]], label %[[PRED_SDIV_CONTINUE2]], !llvm.loop [[LOOP0:![0-9]+]]
; CHECK: [[MIDDLE_BLOCK]]:
; CHECK-NEXT: [[PREDPHI7:%.*]] = select <2 x i1> [[TMP29]], <2 x i8> [[TMP28]], <2 x i8> zeroinitializer
; CHECK-NEXT: [[TMP22:%.*]] = extractelement <2 x i8> [[PREDPHI7]], i64 1
@@ -146,12 +115,6 @@ define i8 @predicated_replicate_feeding_cast_non_uniform(i64 %n, i1 %c1, i1 %c2,
; CHECK: [[VECTOR_PH]]:
; CHECK-NEXT: [[N_MOD_VF:%.*]] = and i64 [[TMP0]], 1
; CHECK-NEXT: [[N_VEC:%.*]] = sub i64 [[TMP0]], [[N_MOD_VF]]
-; CHECK-NEXT: [[BROADCAST_SPLATINSERT:%.*]] = insertelement <2 x i1> poison, i1 [[C2]], i64 0
-; CHECK-NEXT: [[BROADCAST_SPLAT:%.*]] = shufflevector <2 x i1> [[BROADCAST_SPLATINSERT]], <2 x i1> poison, <2 x i32> zeroinitializer
-; CHECK-NEXT: [[BROADCAST_SPLATINSERT1:%.*]] = insertelement <2 x i1> poison, i1 [[C1]], i64 0
-; CHECK-NEXT: [[BROADCAST_SPLAT2:%.*]] = shufflevector <2 x i1> [[BROADCAST_SPLATINSERT1]], <2 x i1> poison, <2 x i32> zeroinitializer
-; CHECK-NEXT: [[TMP11:%.*]] = xor <2 x i1> [[BROADCAST_SPLAT]], splat (i1 true)
-; CHECK-NEXT: [[TMP21:%.*]] = xor <2 x i1> [[BROADCAST_SPLAT2]], splat (i1 true)
; CHECK-NEXT: br label %[[VECTOR_BODY:.*]]
; CHECK: [[VECTOR_BODY]]:
; CHECK-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[LATCH12:.*]] ]
@@ -159,55 +122,31 @@ define i8 @predicated_replicate_feeding_cast_non_uniform(i64 %n, i1 %c1, i1 %c2,
; CHECK-NEXT: [[WIDE_LOAD:%.*]] = load <2 x i16>, ptr [[TMP3]], align 2
; CHECK-NEXT: br i1 [[C1]], label %[[MERGE6:.*]], label %[[IF13:.*]]
; CHECK: [[IF13]]:
-; CHECK-NEXT: [[TMP1:%.*]] = extractelement <2 x i1> [[TMP21]], i64 0
-; CHECK-NEXT: br i1 [[TMP1]], label %[[PRED_SDIV_IF:.*]], label %[[PRED_SDIV_CONTINUE:.*]]
-; CHECK: [[PRED_SDIV_IF]]:
; CHECK-NEXT: [[TMP4:%.*]] = extractelement <2 x i16> [[WIDE_LOAD]], i64 0
; CHECK-NEXT: [[TMP5:%.*]] = sdiv i16 1, [[TMP4]]
-; CHECK-NEXT: [[TMP6:%.*]] = insertelement <2 x i16> poison, i16 [[TMP5]], i64 0
-; CHECK-NEXT: br label %[[PRED_SDIV_CONTINUE]]
-; CHECK: [[PRED_SDIV_CONTINUE]]:
-; CHECK-NEXT: [[TMP7:%.*]] = phi <2 x i16> [ poison, %[[IF13]] ], [ [[TMP6]], %[[PRED_SDIV_IF]] ]
-; CHECK-NEXT: [[TMP24:%.*]] = extractelement <2 x i1> [[TMP21]], i64 0
-; CHECK-NEXT: br i1 [[TMP24]], label %[[PRED_SDIV_IF1:.*]], label %[[PRED_SDIV_CONTINUE2:.*]]
-; CHECK: [[PRED_SDIV_IF1]]:
; CHECK-NEXT: [[TMP8:%.*]] = extractelement <2 x i16> [[WIDE_LOAD]], i64 1
; CHECK-NEXT: [[TMP9:%.*]] = sdiv i16 1, [[TMP8]]
+; CHECK-NEXT: [[TMP7:%.*]] = insertelement <2 x i16> poison, i16 [[TMP5]], i64 0
; CHECK-NEXT: [[TMP10:%.*]] = insertelement <2 x i16> [[TMP7]], i16 [[TMP9]], i64 1
-; CHECK-NEXT: br label %[[PRED_SDIV_CONTINUE2]]
-; CHECK: [[PRED_SDIV_CONTINUE2]]:
-; CHECK-NEXT: [[TMP25:%.*]] = phi <2 x i16> [ [[TMP7]], %[[PRED_SDIV_CONTINUE]] ], [ [[TMP10]], %[[PRED_SDIV_IF1]] ]
; CHECK-NEXT: br label %[[MERGE6]]
; CHECK: [[MERGE6]]:
-; CHECK-NEXT: [[TMP26:%.*]] = phi <2 x i16> [ poison, %[[VECTOR_BODY]] ], [ [[TMP25]], %[[PRED_SDIV_CONTINUE2]] ]
-; CHECK-NEXT: [[TMP27:%.*]] = phi <2 x i1> [ zeroinitializer, %[[VECTOR_BODY]] ], [ [[TMP21]], %[[PRED_SDIV_CONTINUE2]] ]
+; CHECK-NEXT: [[TMP26:%.*]] = phi <2 x i16> [ poison, %[[VECTOR_BODY]] ], [ [[TMP10]], %[[IF13]] ]
+; CHECK-NEXT: [[TMP27:%.*]] = phi <2 x i1> [ zeroinitializer, %[[VECTOR_BODY]] ], [ splat (i1 true), %[[IF13]] ]
; CHECK-NEXT: [[PREDPHI:%.*]] = select <2 x i1> [[TMP27]], <2 x i16> [[TMP26]], <2 x i16> zeroinitializer
; CHECK-NEXT: br i1 [[C2]], label %[[LATCH12]], label %[[IF27:.*]]
; CHECK: [[IF27]]:
-; CHECK-NEXT: [[TMP2:%.*]] = extractelement <2 x i1> [[TMP11]], i64 0
-; CHECK-NEXT: br i1 [[TMP2]], label %[[PRED_SDIV_IF3:.*]], label %[[PRED_SDIV_CONTINUE4:.*]]
-; CHECK: [[PRED_SDIV_IF3]]:
; CHECK-NEXT: [[TMP12:%.*]] = extractelement <2 x i16> [[PREDPHI]], i64 0
; CHECK-NEXT: [[TMP13:%.*]] = trunc i16 [[TMP12]] to i8
-; CHECK-NEXT: [[TMP14:%.*]] = sdiv i8 [[TMP13]], [[B]]
-; CHECK-NEXT: [[TMP15:%.*]] = insertelement <2 x i8> poison, i8 [[TMP14]], i64 0
-; CHECK-NEXT: br label %[[PRED_SDIV_CONTINUE4]]
-; CHECK: [[PRED_SDIV_CONTINUE4]]:
-; CHECK-NEXT: [[TMP16:%.*]] = phi <2 x i8> [ poison, %[[IF27]] ], [ [[TMP15]], %[[PRED_SDIV_IF3]] ]
-; CHECK-NEXT: [[TMP31:%.*]] = extractelement <2 x i1> [[TMP11]], i64 0
-; CHECK-NEXT: br i1 [[TMP31]], label %[[PRED_SDIV_IF5:.*]], label %[[PRED_SDIV_CONTINUE6:.*]]
-; CHECK: [[PRED_SDIV_IF5]]:
; CHECK-NEXT: [[TMP17:%.*]] = extractelement <2 x i16> [[PREDPHI]], i64 1
; CHECK-NEXT: [[TMP18:%.*]] = trunc i16 [[TMP17]] to i8
+; CHECK-NEXT: [[TMP15:%.*]] = sdiv i8 [[TMP13]], [[B]]
; CHECK-NEXT: [[TMP19:%.*]] = sdiv i8 [[TMP18]], [[B]]
+; CHECK-NEXT: [[TMP16:%.*]] = insertelement <2 x i8> poison, i8 [[TMP15]], i64 0
; CHECK-NEXT: [[TMP20:%.*]] = insertelement <2 x i8> [[TMP16]], i8 [[TMP19]], i64 1
-; CHECK-NEXT: br label %[[PRED_SDIV_CONTINUE6]]
-; CHECK: [[PRED_SDIV_CONTINUE6]]:
-; CHECK-NEXT: [[TMP28:%.*]] = phi <2 x i8> [ [[TMP16]], %[[PRED_SDIV_CONTINUE4]] ], [ [[TMP20]], %[[PRED_SDIV_IF5]] ]
; CHECK-NEXT: br label %[[LATCH12]]
; CHECK: [[LATCH12]]:
-; CHECK-NEXT: [[TMP29:%.*]] = phi <2 x i8> [ poison, %[[MERGE6]] ], [ [[TMP28]], %[[PRED_SDIV_CONTINUE6]] ]
-; CHECK-NEXT: [[TMP30:%.*]] = phi <2 x i1> [ zeroinitializer, %[[MERGE6]] ], [ [[TMP11]], %[[PRED_SDIV_CONTINUE6]] ]
+; CHECK-NEXT: [[TMP29:%.*]] = phi <2 x i8> [ poison, %[[MERGE6]] ], [ [[TMP20]], %[[IF27]] ]
+; CHECK-NEXT: [[TMP30:%.*]] = phi <2 x i1> [ zeroinitializer, %[[MERGE6]] ], [ splat (i1 true), %[[IF27]] ]
; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 2
; CHECK-NEXT: [[TMP22:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
; CHECK-NEXT: br i1 [[TMP22]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP4:![0-9]+]]
diff --git a/llvm/test/Transforms/LoopVectorize/X86/replicate-recipe-with-only-first-lane-used.ll b/llvm/test/Transforms/LoopVectorize/X86/replicate-recipe-with-only-first-lane-used.ll
index 4bdbf6bcce5a3..460ce34fe949c 100644
--- a/llvm/test/Transforms/LoopVectorize/X86/replicate-recipe-with-only-first-lane-used.ll
+++ b/llvm/test/Transforms/LoopVectorize/X86/replicate-recipe-with-only-first-lane-used.ll
@@ -69,95 +69,21 @@ define void @replicate_udiv_with_only_first_lane_used2(i32 %x, ptr %dst, i64 %d)
; CHECK-SAME: i32 [[X:%.*]], ptr [[DST:%.*]], i64 [[D:%.*]]) {
; CHECK-NEXT: [[ENTRY:.*:]]
; CHECK-NEXT: [[C:%.*]] = icmp eq i32 [[X]], 10
-; CHECK-NEXT: br label %[[VECTOR_PH:.*]]
-; CHECK: [[VECTOR_PH]]:
-; CHECK-NEXT: [[BROADCAST_SPLATINSERT:%.*]] = insertelement <4 x i1> poison, i1 [[C]], i64 0
-; CHECK-NEXT: [[BROADCAST_SPLAT:%.*]] = shufflevector <4 x i1> [[BROADCAST_SPLATINSERT]], <4 x i1> poison, <4 x i32> zeroinitializer
-; CHECK-NEXT: [[TMP0:%.*]] = xor <4 x i1> [[BROADCAST_SPLAT]], splat (i1 true)
-; CHECK-NEXT: br label %[[VECTOR_BODY:.*]]
-; CHECK: [[VECTOR_BODY]]:
-; CHECK-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[LOOP_LATCH16:.*]] ]
-; CHECK-NEXT: br i1 [[C]], label %[[LOOP_LATCH16]], label %[[PRED_UDIV_IF:.*]]
-; CHECK: [[PRED_UDIV_IF]]:
-; CHECK-NEXT: [[TMP5:%.*]] = extractelement <4 x i1> [[TMP0]], i64 0
-; CHECK-NEXT: br i1 [[TMP5]], label %[[PRED_UDIV_IF2:.*]], label %[[PRED_UDIV_CONTINUE:.*]]
-; CHECK: [[PRED_UDIV_IF2]]:
-; CHECK-NEXT: [[TMP2:%.*]] = udiv i64 99, [[D]]
-; CHECK-NEXT: [[TMP3:%.*]] = insertelement <4 x i64> poison, i64 [[TMP2]], i64 0
-; CHECK-NEXT: br label %[[PRED_UDIV_CONTINUE]]
-; CHECK: [[PRED_UDIV_CONTINUE]]:
-; CHECK-NEXT: [[TMP4:%.*]] = phi <4 x i64> [ poison, %[[PRED_UDIV_IF]] ], [ [[TMP3]], %[[PRED_UDIV_IF2]] ]
-; CHECK-NEXT: [[TMP1:%.*]] = extractelement <4 x i1> [[TMP0]], i64 0
-; CHECK-NEXT: br i1 [[TMP1]], label %[[PRED_UDIV_IF1:.*]], label %[[PRED_UDIV_CONTINUE2:.*]]
-; CHECK: [[PRED_UDIV_IF1]]:
-; CHECK-NEXT: [[TMP6:%.*]] = udiv i64 99, [[D]]
-; CHECK-NEXT: [[TMP7:%.*]] = insertelement <4 x i64> [[TMP4]], i64 [[TMP6]], i64 1
-; CHECK-NEXT: br label %[[PRED_UDIV_CONTINUE2]]
-; CHECK: [[PRED_UDIV_CONTINUE2]]:
-; CHECK-NEXT: [[TMP8:%.*]] = phi <4 x i64> [ [[TMP4]], %[[PRED_UDIV_CONTINUE]] ], [ [[TMP7]], %[[PRED_UDIV_IF1]] ]
-; CHECK-NEXT: [[TMP9:%.*]] = extractelement <4 x i1> [[TMP0]], i64 0
-; CHECK-NEXT: br i1 [[TMP9]], label %[[PRED_UDIV_IF3:.*]], label %[[PRED_UDIV_CONTINUE4:.*]]
-; CHECK: [[PRED_UDIV_IF3]]:
-; CHECK-NEXT: [[TMP10:%.*]] = udiv i64 99, [[D]]
-; CHECK-NEXT: [[TMP11:%.*]] = insertelement <4 x i64> [[TMP8]], i64 [[TMP10]], i64 2
-; CHECK-NEXT: br label %[[PRED_UDIV_CONTINUE4]]
-; CHECK: [[PRED_UDIV_CONTINUE4]]:
-; CHECK-NEXT: [[TMP23:%.*]] = phi <4 x i64> [ [[TMP8]], %[[PRED_UDIV_CONTINUE2]] ], [ [[TMP11]], %[[PRED_UDIV_IF3]] ]
-; CHECK-NEXT: [[TMP13:%.*]] = extractelement <4 x i1> [[TMP0]], i64 0
-; CHECK-NEXT: br i1 [[TMP13]], label %[[PRED_UDIV_IF5:.*]], label %[[PRED_UDIV_CONTINUE6:.*]]
-; CHECK: [[PRED_UDIV_IF5]]:
-; CHECK-NEXT: [[TMP14:%.*]] = udiv i64 99, [[D]]
-; CHECK-NEXT: [[TMP15:%.*]] = insertelement <4 x i64> [[TMP23]], i64 [[TMP14]], i64 3
-; CHECK-NEXT: br label %[[PRED_UDIV_CONTINUE6]]
+; CHECK-NEXT: br label %[[PRED_UDIV_CONTINUE6:.*]]
; CHECK: [[PRED_UDIV_CONTINUE6]]:
-; CHECK-NEXT: [[TMP16:%.*]] = phi <4 x i64> [ [[TMP23]], %[[PRED_UDIV_CONTINUE4]] ], [ [[TMP15]], %[[PRED_UDIV_IF5]] ]
-; CHECK-NEXT: [[TMP17:%.*]] = extractelement <4 x i1> [[TMP0]], i64 0
-; CHECK-NEXT: br i1 [[TMP17]], label %[[PRED_UDIV_IF7:.*]], label %[[PRED_UDIV_CONTINUE8:.*]]
-; CHECK: [[PRED_UDIV_IF7]]:
; CHECK-NEXT: [[TMP25:%.*]] = udiv i64 99, [[D]]
; CHECK-NEXT: [[TMP33:%.*]] = insertelement <4 x i64> poison, i64 [[TMP25]], i64 0
-; CHECK-NEXT: br label %[[PRED_UDIV_CONTINUE8]]
-; CHECK: [[PRED_UDIV_CONTINUE8]]:
-; CHECK-NEXT: [[TMP35:%.*]] = phi <4 x i64> [ poison, %[[PRED_UDIV_CONTINUE6]] ], [ [[TMP33]], %[[PRED_UDIV_IF7]] ]
-; CHECK-NEXT: [[TMP18:%.*]] = extractelement <4 x i1> [[TMP0]], i64 0
-; CHECK-NEXT: br i1 [[TMP18]], label %[[PRED_UDIV_IF9:.*]], label %[[PRED_UDIV_CONTINUE10:.*]]
-; CHECK: [[PRED_UDIV_IF9]]:
-; CHECK-NEXT: [[TMP36:%.*]] = udiv i64 99, [[D]]
-; CHECK-NEXT: [[TMP37:%.*]] = insertelement <4 x i64> [[TMP35]], i64 [[TMP36]], i64 1
-; CHECK-NEXT: br label %[[PRED_UDIV_CONTINUE10]]
-; CHECK: [[PRED_UDIV_CONTINUE10]]:
-; CHECK-NEXT: [[TMP24:%.*]] = phi <4 x i64> [ [[TMP35]], %[[PRED_UDIV_CONTINUE8]] ], [ [[TMP37]], %[[PRED_UDIV_IF9]] ]
-; CHECK-NEXT: [[TMP19:%.*]] = extractelement <4 x i1> [[TMP0]], i64 0
-; CHECK-NEXT: br i1 [[TMP19]], label %[[PRED_UDIV_IF11:.*]], label %[[PRED_UDIV_CONTINUE12:.*]]
-; CHECK: [[PRED_UDIV_IF11]]:
-; CHECK-NEXT: [[TMP26:%.*]] = udiv i64 99, [[D]]
-; CHECK-NEXT: [[TMP38:%.*]] = insertelement <4 x i64> [[TMP24]], i64 [[TMP26]], i64 2
-; CHECK-NEXT: br label %[[PRED_UDIV_CONTINUE12]]
+; CHECK-NEXT: [[BROADCAST_SPLAT:%.*]] = shufflevector <4 x i64> [[TMP33]], <4 x i64> poison, <4 x i32> zeroinitializer
+; CHECK-NEXT: br label %[[PRED_UDIV_CONTINUE12:.*]]
; CHECK: [[PRED_UDIV_CONTINUE12]]:
-; CHECK-NEXT: [[TMP39:%.*]] = phi <4 x i64> [ [[TMP24]], %[[PRED_UDIV_CONTINUE10]] ], [ [[TMP38]], %[[PRED_UDIV_IF11]] ]
-; CHECK-NEXT: [[TMP20:%.*]] = extractelement <4 x i1> [[TMP0]], i64 0
-; CHECK-NEXT: br i1 [[TMP20]], label %[[PRED_UDIV_IF13:.*]], label %[[PRED_UDIV_CONTINUE14:.*]]
-; CHECK: [[PRED_UDIV_IF13]]:
-; CHECK-NEXT: [[TMP40:%.*]] = udiv i64 99, [[D]]
-; CHECK-NEXT: [[TMP31:%.*]] = insertelement <4 x i64> [[TMP39]], i64 [[TMP40]], i64 3
-; CHECK-NEXT: br label %[[PRED_UDIV_CONTINUE14]]
+; CHECK-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[PRED_UDIV_CONTINUE6]] ], [ [[INDEX_NEXT:%.*]], %[[LOOP_LATCH16:.*]] ]
+; CHECK-NEXT: br i1 [[C]], label %[[LOOP_LATCH16]], label %[[PRED_UDIV_CONTINUE14:.*]]
; CHECK: [[PRED_UDIV_CONTINUE14]]:
-; CHECK-NEXT: [[TMP32:%.*]] = phi <4 x i64> [ [[TMP39]], %[[PRED_UDIV_CONTINUE12]] ], [ [[TMP31]], %[[PRED_UDIV_IF13]] ]
; CHECK-NEXT: br label %[[LOOP_LATCH16]]
; CHECK: [[LOOP_LATCH16]]:
-; CHECK-NEXT: [[TMP21:%.*]] = phi <4 x i64> [ poison, %[[VECTOR_BODY]] ], [ [[TMP16]], %[[PRED_UDIV_CONTINUE14]] ]
-; CHECK-NEXT: [[TMP34:%.*]] = phi <4 x i64> [ poison, %[[VECTOR_BODY]] ], [ [[TMP32]], %[[PRED_UDIV_CONTINUE14]] ]
-; CHECK-NEXT: [[TMP22:%.*]] = phi <4 x i1> [ zeroinitializer, %[[VECTOR_BODY]] ], [ [[TMP0]], %[[PRED_UDIV_CONTINUE14]] ]
-; CHECK-NEXT: [[PREDPHI:%.*]] = select <4 x i1> [[TMP22]], <4 x i64> [[TMP21]], <4 x i64> zeroinitializer
+; CHECK-NEXT: [[TMP34:%.*]] = phi <4 x i64> [ poison, %[[PRED_UDIV_CONTINUE12]] ], [ [[BROADCAST_SPLAT]], %[[PRED_UDIV_CONTINUE14]] ]
+; CHECK-NEXT: [[TMP22:%.*]] = phi <4 x i1> [ zeroinitializer, %[[PRED_UDIV_CONTINUE12]] ], [ splat (i1 true), %[[PRED_UDIV_CONTINUE14]] ]
; CHECK-NEXT: [[PREDPHI17:%.*]] = select <4 x i1> [[TMP22]], <4 x i64> [[TMP34]], <4 x i64> zeroinitializer
-; CHECK-NEXT: [[TMP45:%.*]] = extractelement <4 x i64> [[PREDPHI]], i64 0
-; CHECK-NEXT: [[TMP46:%.*]] = getelementptr i16, ptr [[DST]], i64 [[TMP45]]
-; CHECK-NEXT: [[TMP47:%.*]] = extractelement <4 x i64> [[PREDPHI]], i64 1
-; CHECK-NEXT: [[TMP48:%.*]] = getelementptr i16, ptr [[DST]], i64 [[TMP47]]
-; CHECK-NEXT: [[TMP27:%.*]] = extractelement <4 x i64> [[PREDPHI]], i64 2
-; CHECK-NEXT: [[TMP28:%.*]] = getelementptr i16, ptr [[DST]], i64 [[TMP27]]
-; CHECK-NEXT: [[TMP29:%.*]] = extractelement <4 x i64> [[PREDPHI]], i64 3
-; CHECK-NEXT: [[TMP30:%.*]] = getelementptr i16, ptr [[DST]], i64 [[TMP29]]
; CHECK-NEXT: [[TMP44:%.*]] = extractelement <4 x i64> [[PREDPHI17]], i64 0
; CHECK-NEXT: [[TMP52:%.*]] = getelementptr i16, ptr [[DST]], i64 [[TMP44]]
; CHECK-NEXT: [[TMP53:%.*]] = extractelement <4 x i64> [[PREDPHI17]], i64 1
@@ -166,17 +92,17 @@ define void @replicate_udiv_with_only_first_lane_used2(i32 %x, ptr %dst, i64 %d)
; CHECK-NEXT: [[TMP49:%.*]] = getelementptr i16, ptr [[DST]], i64 [[TMP55]]
; CHECK-NEXT: [[TMP50:%.*]] = extractelement <4 x i64> [[PREDPHI17]], i64 3
; CHECK-NEXT: [[TMP51:%.*]] = getelementptr i16, ptr [[DST]], i64 [[TMP50]]
-; CHECK-NEXT: store i16 0, ptr [[TMP46]], align 2
-; CHECK-NEXT: store i16 0, ptr [[TMP48]], align 2
-; CHECK-NEXT: store i16 0, ptr [[TMP28]], align 2
-; CHECK-NEXT: store i16 0, ptr [[TMP30]], align 2
+; CHECK-NEXT: store i16 0, ptr [[TMP52]], align 2
+; CHECK-NEXT: store i16 0, ptr [[TMP54]], align 2
+; CHECK-NEXT: store i16 0, ptr [[TMP49]], align 2
+; CHECK-NEXT: store i16 0, ptr [[TMP51]], align 2
; CHECK-NEXT: store i16 0, ptr [[TMP52]], align 2
; CHECK-NEXT: store i16 0, ptr [[TMP54]], align 2
; CHECK-NEXT: store i16 0, ptr [[TMP49]], align 2
; CHECK-NEXT: store i16 0, ptr [[TMP51]], align 2
; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 8
; CHECK-NEXT: [[TMP12:%.*]] = icmp eq i64 [[INDEX_NEXT]], 96
-; CHECK-NEXT: br i1 [[TMP12]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP4:![0-9]+]]
+; CHECK-NEXT: br i1 [[TMP12]], label %[[MIDDLE_BLOCK:.*]], label %[[PRED_UDIV_CONTINUE12]], !llvm.loop [[LOOP4:![0-9]+]]
; CHECK: [[MIDDLE_BLOCK]]:
; CHECK-NEXT: br label %[[SCALAR_PH:.*]]
; CHECK: [[SCALAR_PH]]:
diff --git a/llvm/test/Transforms/LoopVectorize/X86/replicate-uniform-call.ll b/llvm/test/Transforms/LoopVectorize/X86/replicate-uniform-call.ll
index e742f82b18787..15a0372365607 100644
--- a/llvm/test/Transforms/LoopVectorize/X86/replicate-uniform-call.ll
+++ b/llvm/test/Transforms/LoopVectorize/X86/replicate-uniform-call.ll
@@ -13,9 +13,6 @@ define void @smax_call_uniform(ptr %dst, i64 %x) {
; CHECK-NEXT: [[MUL:%.*]] = mul nuw nsw i64 [[X]], 0
; CHECK-NEXT: br label %[[LOOP_HEADER:.*]]
; CHECK: [[LOOP_HEADER]]:
-; CHECK-NEXT: [[BROADCAST_SPLATINSERT:%.*]] = insertelement <2 x i1> poison, i1 [[C]], i64 0
-; CHECK-NEXT: [[BROADCAST_SPLAT:%.*]] = shufflevector <2 x i1> [[BROADCAST_SPLATINSERT]], <2 x i1> poison, <2 x i32> zeroinitializer
-; CHECK-NEXT: [[TMP0:%.*]] = xor <2 x i1> [[BROADCAST_SPLAT]], splat (i1 true)
; CHECK-NEXT: br label %[[VECTOR_BODY:.*]]
; CHECK: [[VECTOR_BODY]]:
; CHECK-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[LOOP_HEADER]] ], [ [[INDEX_NEXT:%.*]], %[[LOOP_LATCH:.*]] ]
@@ -23,7 +20,7 @@ define void @smax_call_uniform(ptr %dst, i64 %x) {
; CHECK: [[ELSE]]:
; CHECK-NEXT: br label %[[LOOP_LATCH]]
; CHECK: [[LOOP_LATCH]]:
-; CHECK-NEXT: [[TMP1:%.*]] = phi <2 x i1> [ zeroinitializer, %[[VECTOR_BODY]] ], [ [[TMP0]], %[[ELSE]] ]
+; CHECK-NEXT: [[TMP1:%.*]] = phi <2 x i1> [ zeroinitializer, %[[VECTOR_BODY]] ], [ splat (i1 true), %[[ELSE]] ]
; CHECK-NEXT: [[TMP2:%.*]] = extractelement <2 x i1> [[TMP1]], i64 0
; CHECK-NEXT: [[PREDPHI7:%.*]] = select i1 [[TMP2]], i64 0, i64 1
; CHECK-NEXT: [[TMP17:%.*]] = add i64 [[PREDPHI7]], 1
diff --git a/llvm/test/Transforms/LoopVectorize/X86/scatter_crash.ll b/llvm/test/Transforms/LoopVectorize/X86/scatter_crash.ll
index ed23958346f9b..c8e289796894e 100644
--- a/llvm/test/Transforms/LoopVectorize/X86/scatter_crash.ll
+++ b/llvm/test/Transforms/LoopVectorize/X86/scatter_crash.ll
@@ -119,11 +119,8 @@ define void @_Z3fn1v() #0 {
; CHECK: [[VECTOR_PH25]]:
; CHECK-NEXT: [[N_MOD_VF31:%.*]] = and i64 [[TMP28]], 15
; CHECK-NEXT: [[N_VEC32:%.*]] = sub i64 [[TMP28]], [[N_MOD_VF31]]
-; CHECK-NEXT: [[BROADCAST_SPLATINSERT:%.*]] = insertelement <16 x i1> poison, i1 [[TOBOOL6]], i64 0
-; CHECK-NEXT: [[BROADCAST_SPLAT:%.*]] = shufflevector <16 x i1> [[BROADCAST_SPLATINSERT]], <16 x i1> poison, <16 x i32> zeroinitializer
; CHECK-NEXT: [[TMP29:%.*]] = shl i64 [[N_VEC32]], 1
; CHECK-NEXT: [[IND_END41:%.*]] = add i64 8, [[TMP29]]
-; CHECK-NEXT: [[TMP34:%.*]] = xor <16 x i1> [[BROADCAST_SPLAT]], splat (i1 true)
; CHECK-NEXT: br label %[[VECTOR_BODY30:.*]]
; CHECK: [[VECTOR_BODY30]]:
; CHECK-NEXT: [[INDEX34:%.*]] = phi i64 [ 0, %[[VECTOR_PH25]] ], [ [[INDEX_NEXT39:%.*]], %[[FOR_COND_CLEANUP4_US_LCSSA_US_US40:.*]] ]
@@ -135,18 +132,18 @@ define void @_Z3fn1v() #0 {
; CHECK-NEXT: [[TMP33:%.*]] = getelementptr inbounds [10 x i32], <16 x ptr> [[TMP31]], <16 x i64> [[TMP32]], i64 0
; CHECK-NEXT: br i1 [[TOBOOL6]], label %[[FOR_BODY5_US_US_US_PREHEADER38:.*]], label %[[FOR_BODY5_US_US48_PREHEADER36:.*]]
; CHECK: [[FOR_BODY5_US_US48_PREHEADER36]]:
-; CHECK-NEXT: call void @llvm.masked.scatter.v16i32.v16p0(<16 x i32> splat (i32 8), <16 x ptr> align 16 [[TMP33]], <16 x i1> [[TMP34]])
+; CHECK-NEXT: call void @llvm.masked.scatter.v16i32.v16p0(<16 x i32> splat (i32 8), <16 x ptr> align 16 [[TMP33]], <16 x i1> splat (i1 true))
; CHECK-NEXT: [[TMP49:%.*]] = or disjoint <16 x i64> [[VEC_IND37]], splat (i64 1)
; CHECK-NEXT: [[TMP36:%.*]] = add nsw <16 x i64> [[TMP30]], [[TMP49]]
; CHECK-NEXT: [[TMP37:%.*]] = getelementptr inbounds [10 x i32], <16 x ptr> [[TMP31]], <16 x i64> [[TMP36]], i64 0
-; CHECK-NEXT: call void @llvm.masked.scatter.v16i32.v16p0(<16 x i32> splat (i32 8), <16 x ptr> align 8 [[TMP37]], <16 x i1> [[TMP34]])
+; CHECK-NEXT: call void @llvm.masked.scatter.v16i32.v16p0(<16 x i32> splat (i32 8), <16 x ptr> align 8 [[TMP37]], <16 x i1> splat (i1 true))
; CHECK-NEXT: br label %[[FOR_COND_CLEANUP4_US_LCSSA_US_US40]]
; CHECK: [[FOR_BODY5_US_US_US_PREHEADER38]]:
-; CHECK-NEXT: call void @llvm.masked.scatter.v16i32.v16p0(<16 x i32> splat (i32 7), <16 x ptr> align 16 [[TMP33]], <16 x i1> [[BROADCAST_SPLAT]])
+; CHECK-NEXT: call void @llvm.masked.scatter.v16i32.v16p0(<16 x i32> splat (i32 7), <16 x ptr> align 16 [[TMP33]], <16 x i1> splat (i1 true))
; CHECK-NEXT: [[TMP38:%.*]] = or disjoint <16 x i64> [[VEC_IND37]], splat (i64 1)
; CHECK-NEXT: [[TMP35:%.*]] = add nsw <16 x i64> [[TMP30]], [[TMP38]]
; CHECK-NEXT: [[WIDE_GEP39:%.*]] = getelementptr inbounds [10 x i32], <16 x ptr> [[TMP31]], <16 x i64> [[TMP35]], i64 0
-; CHECK-NEXT: call void @llvm.masked.scatter.v16i32.v16p0(<16 x i32> splat (i32 7), <16 x ptr> align 8 [[WIDE_GEP39]], <16 x i1> [[BROADCAST_SPLAT]])
+; CHECK-NEXT: call void @llvm.masked.scatter.v16i32.v16p0(<16 x i32> splat (i32 7), <16 x ptr> align 8 [[WIDE_GEP39]], <16 x i1> splat (i1 true))
; CHECK-NEXT: br label %[[FOR_COND_CLEANUP4_US_LCSSA_US_US40]]
; CHECK: [[FOR_COND_CLEANUP4_US_LCSSA_US_US40]]:
; CHECK-NEXT: [[INDEX_NEXT39]] = add nuw i64 [[INDEX34]], 16
@@ -161,16 +158,13 @@ define void @_Z3fn1v() #0 {
; CHECK-NEXT: [[MIN_EPILOG_ITERS_CHECK50:%.*]] = icmp ult i64 [[N_MOD_VF31]], 8
; CHECK-NEXT: br i1 [[MIN_EPILOG_ITERS_CHECK50]], label %[[VEC_EPILOG_SCALAR_PH42]], label %[[VEC_EPILOG_PH45]], !prof [[PROF3]]
; CHECK: [[VEC_EPILOG_PH45]]:
-; CHECK-NEXT: [[VEC_EPILOG_RESUME_VAL46:%.*]] = phi i64 [ [[N_VEC32]], %[[VEC_EPILOG_ITER_CHECK43]] ], [ 0, %[[VECTOR_MAIN_LOOP_ITER_CHECK24]] ]
+; CHECK-NEXT: [[VEC_EPILOG_RESUME_VAL44:%.*]] = phi i64 [ [[N_VEC32]], %[[VEC_EPILOG_ITER_CHECK43]] ], [ 0, %[[VECTOR_MAIN_LOOP_ITER_CHECK24]] ]
; CHECK-NEXT: [[BC_RESUME_VAL42:%.*]] = phi i64 [ [[IND_END41]], %[[VEC_EPILOG_ITER_CHECK43]] ], [ 8, %[[VECTOR_MAIN_LOOP_ITER_CHECK24]] ]
; CHECK-NEXT: [[BC_RESUME_VAL44:%.*]] = phi i64 [ [[TMP29]], %[[VEC_EPILOG_ITER_CHECK43]] ], [ 0, %[[VECTOR_MAIN_LOOP_ITER_CHECK24]] ]
; CHECK-NEXT: [[N_MOD_VF52:%.*]] = and i64 [[TMP28]], 7
; CHECK-NEXT: [[N_VEC53:%.*]] = sub i64 [[TMP28]], [[N_MOD_VF52]]
-; CHECK-NEXT: [[BROADCAST_SPLATINSERT49:%.*]] = insertelement <8 x i1> poison, i1 [[TOBOOL6]], i64 0
-; CHECK-NEXT: [[BROADCAST_SPLAT73:%.*]] = shufflevector <8 x i1> [[BROADCAST_SPLATINSERT49]], <8 x i1> poison, <8 x i32> zeroinitializer
; CHECK-NEXT: [[TMP43:%.*]] = shl i64 [[N_VEC53]], 1
; CHECK-NEXT: [[IND_END54:%.*]] = add i64 8, [[TMP43]]
-; CHECK-NEXT: [[TMP48:%.*]] = xor <8 x i1> [[BROADCAST_SPLAT73]], splat (i1 true)
; CHECK-NEXT: [[DOTSPLATINSERT62:%.*]] = insertelement <8 x i64> poison, i64 [[BC_RESUME_VAL42]], i64 0
; CHECK-NEXT: [[DOTSPLAT63:%.*]] = shufflevector <8 x i64> [[DOTSPLATINSERT62]], <8 x i64> poison, <8 x i32> zeroinitializer
; CHECK-NEXT: [[INDUCTION52:%.*]] = add nuw nsw <8 x i64> [[DOTSPLAT63]], <i64 0, i64 2, i64 4, i64 6, i64 8, i64 10, i64 12, i64 14>
@@ -179,7 +173,7 @@ define void @_Z3fn1v() #0 {
; CHECK-NEXT: [[INDUCTION55:%.*]] = add nuw nsw <8 x i64> [[DOTSPLAT68]], <i64 0, i64 2, i64 4, i64 6, i64 8, i64 10, i64 12, i64 14>
; CHECK-NEXT: br label %[[VEC_EPILOG_VECTOR_BODY56:.*]]
; CHECK: [[VEC_EPILOG_VECTOR_BODY56]]:
-; CHECK-NEXT: [[INDEX61:%.*]] = phi i64 [ [[VEC_EPILOG_RESUME_VAL46]], %[[VEC_EPILOG_PH45]] ], [ [[INDEX_NEXT74:%.*]], %[[FOR_COND_CLEANUP4_US_LCSSA_US_US73:.*]] ]
+; CHECK-NEXT: [[INDEX61:%.*]] = phi i64 [ [[VEC_EPILOG_RESUME_VAL44]], %[[VEC_EPILOG_PH45]] ], [ [[INDEX_NEXT74:%.*]], %[[FOR_COND_CLEANUP4_US_LCSSA_US_US73:.*]] ]
; CHECK-NEXT: [[VEC_IND65:%.*]] = phi <8 x i64> [ [[INDUCTION52]], %[[VEC_EPILOG_PH45]] ], [ [[VEC_IND_NEXT61:%.*]], %[[FOR_COND_CLEANUP4_US_LCSSA_US_US73]] ]
; CHECK-NEXT: [[VEC_IND70:%.*]] = phi <8 x i64> [ [[INDUCTION55]], %[[VEC_EPILOG_PH45]] ], [ [[VEC_IND_NEXT62:%.*]], %[[FOR_COND_CLEANUP4_US_LCSSA_US_US73]] ]
; CHECK-NEXT: [[TMP44:%.*]] = sub nsw <8 x i64> splat (i64 8), [[VEC_IND65]]
@@ -188,18 +182,18 @@ define void @_Z3fn1v() #0 {
; CHECK-NEXT: [[TMP47:%.*]] = getelementptr inbounds [10 x i32], <8 x ptr> [[TMP45]], <8 x i64> [[TMP46]], i64 0
; CHECK-NEXT: br i1 [[TOBOOL6]], label %[[FOR_BODY5_US_US_US_PREHEADER71:.*]], label %[[FOR_BODY5_US_US48_PREHEADER69:.*]]
; CHECK: [[FOR_BODY5_US_US48_PREHEADER69]]:
-; CHECK-NEXT: call void @llvm.masked.scatter.v8i32.v8p0(<8 x i32> splat (i32 8), <8 x ptr> align 16 [[TMP47]], <8 x i1> [[TMP48]])
+; CHECK-NEXT: call void @llvm.masked.scatter.v8i32.v8p0(<8 x i32> splat (i32 8), <8 x ptr> align 16 [[TMP47]], <8 x i1> splat (i1 true))
; CHECK-NEXT: [[TMP54:%.*]] = or disjoint <8 x i64> [[VEC_IND70]], splat (i64 1)
; CHECK-NEXT: [[TMP50:%.*]] = add nsw <8 x i64> [[TMP44]], [[TMP54]]
; CHECK-NEXT: [[TMP51:%.*]] = getelementptr inbounds [10 x i32], <8 x ptr> [[TMP45]], <8 x i64> [[TMP50]], i64 0
-; CHECK-NEXT: call void @llvm.masked.scatter.v8i32.v8p0(<8 x i32> splat (i32 8), <8 x ptr> align 8 [[TMP51]], <8 x i1> [[TMP48]])
+; CHECK-NEXT: call void @llvm.masked.scatter.v8i32.v8p0(<8 x i32> splat (i32 8), <8 x ptr> align 8 [[TMP51]], <8 x i1> splat (i1 true))
; CHECK-NEXT: br label %[[FOR_COND_CLEANUP4_US_LCSSA_US_US73]]
; CHECK: [[FOR_BODY5_US_US_US_PREHEADER71]]:
-; CHECK-NEXT: call void @llvm.masked.scatter.v8i32.v8p0(<8 x i32> splat (i32 7), <8 x ptr> align 16 [[TMP47]], <8 x i1> [[BROADCAST_SPLAT73]])
+; CHECK-NEXT: call void @llvm.masked.scatter.v8i32.v8p0(<8 x i32> splat (i32 7), <8 x ptr> align 16 [[TMP47]], <8 x i1> splat (i1 true))
; CHECK-NEXT: [[TMP52:%.*]] = or disjoint <8 x i64> [[VEC_IND70]], splat (i64 1)
; CHECK-NEXT: [[TMP53:%.*]] = add nsw <8 x i64> [[TMP44]], [[TMP52]]
; CHECK-NEXT: [[WIDE_GEP72:%.*]] = getelementptr inbounds [10 x i32], <8 x ptr> [[TMP45]], <8 x i64> [[TMP53]], i64 0
-; CHECK-NEXT: call void @llvm.masked.scatter.v8i32.v8p0(<8 x i32> splat (i32 7), <8 x ptr> align 8 [[WIDE_GEP72]], <8 x i1> [[BROADCAST_SPLAT73]])
+; CHECK-NEXT: call void @llvm.masked.scatter.v8i32.v8p0(<8 x i32> splat (i32 7), <8 x ptr> align 8 [[WIDE_GEP72]], <8 x i1> splat (i1 true))
; CHECK-NEXT: br label %[[FOR_COND_CLEANUP4_US_LCSSA_US_US73]]
; CHECK: [[FOR_COND_CLEANUP4_US_LCSSA_US_US73]]:
; CHECK-NEXT: [[INDEX_NEXT74]] = add nuw i64 [[INDEX61]], 8
@@ -211,12 +205,12 @@ define void @_Z3fn1v() #0 {
; CHECK-NEXT: [[CMP_N65:%.*]] = icmp eq i64 [[TMP28]], [[N_VEC53]]
; CHECK-NEXT: br i1 [[CMP_N65]], label %[[FOR_COND_CLEANUP_LOOPEXIT]], label %[[VEC_EPILOG_SCALAR_PH42]]
; CHECK: [[VEC_EPILOG_SCALAR_PH42]]:
-; CHECK-NEXT: [[BC_RESUME_VAL79:%.*]] = phi i64 [ [[IND_END54]], %[[VEC_EPILOG_MIDDLE_BLOCK63]] ], [ [[IND_END41]], %[[VEC_EPILOG_ITER_CHECK43]] ], [ 8, %[[ITER_CHECK22]] ]
-; CHECK-NEXT: [[BC_RESUME_VAL80:%.*]] = phi i64 [ [[TMP43]], %[[VEC_EPILOG_MIDDLE_BLOCK63]] ], [ [[TMP29]], %[[VEC_EPILOG_ITER_CHECK43]] ], [ 0, %[[ITER_CHECK22]] ]
+; CHECK-NEXT: [[BC_RESUME_VAL75:%.*]] = phi i64 [ [[IND_END54]], %[[VEC_EPILOG_MIDDLE_BLOCK63]] ], [ [[IND_END41]], %[[VEC_EPILOG_ITER_CHECK43]] ], [ 8, %[[ITER_CHECK22]] ]
+; CHECK-NEXT: [[BC_RESUME_VAL76:%.*]] = phi i64 [ [[TMP43]], %[[VEC_EPILOG_MIDDLE_BLOCK63]] ], [ [[TMP29]], %[[VEC_EPILOG_ITER_CHECK43]] ], [ 0, %[[ITER_CHECK22]] ]
; CHECK-NEXT: br label %[[FOR_BODY_US:.*]]
; CHECK: [[FOR_BODY_US]]:
-; CHECK-NEXT: [[INDVARS_IV78:%.*]] = phi i64 [ [[INDVARS_IV_NEXT79:%.*]], %[[FOR_COND_CLEANUP4_US_LCSSA_US_US:.*]] ], [ [[BC_RESUME_VAL79]], %[[VEC_EPILOG_SCALAR_PH42]] ]
-; CHECK-NEXT: [[INDVARS_IV70:%.*]] = phi i64 [ [[INDVARS_IV_NEXT71:%.*]], %[[FOR_COND_CLEANUP4_US_LCSSA_US_US]] ], [ [[BC_RESUME_VAL80]], %[[VEC_EPILOG_SCALAR_PH42]] ]
+; CHECK-NEXT: [[INDVARS_IV78:%.*]] = phi i64 [ [[INDVARS_IV_NEXT79:%.*]], %[[FOR_COND_CLEANUP4_US_LCSSA_US_US:.*]] ], [ [[BC_RESUME_VAL75]], %[[VEC_EPILOG_SCALAR_PH42]] ]
+; CHECK-NEXT: [[INDVARS_IV70:%.*]] = phi i64 [ [[INDVARS_IV_NEXT71:%.*]], %[[FOR_COND_CLEANUP4_US_LCSSA_US_US]] ], [ [[BC_RESUME_VAL76]], %[[VEC_EPILOG_SCALAR_PH42]] ]
; CHECK-NEXT: [[TMP56:%.*]] = sub nsw i64 8, [[INDVARS_IV78]]
; CHECK-NEXT: [[ADD_PTR_US:%.*]] = getelementptr inbounds [10 x [10 x i32]], ptr @d, i64 0, i64 [[INDVARS_IV78]]
; CHECK-NEXT: [[TMP57:%.*]] = add nsw i64 [[TMP56]], [[INDVARS_IV70]]
diff --git a/llvm/test/Transforms/LoopVectorize/constant-fold-commutative-and.ll b/llvm/test/Transforms/LoopVectorize/constant-fold-commutative-and.ll
index cf11297d4564e..2e9a3760b5bcd 100644
--- a/llvm/test/Transforms/LoopVectorize/constant-fold-commutative-and.ll
+++ b/llvm/test/Transforms/LoopVectorize/constant-fold-commutative-and.ll
@@ -11,9 +11,6 @@ define void @constant_fold_commutative_and(ptr %ptr.n, ptr noalias %p, i1 %cond)
; CHECK-NEXT: [[ENTRY:.*:]]
; CHECK-NEXT: br label %[[VECTOR_PH:.*]]
; CHECK: [[VECTOR_PH]]:
-; CHECK-NEXT: [[BROADCAST_SPLATINSERT:%.*]] = insertelement <2 x i1> poison, i1 [[COND]], i64 0
-; CHECK-NEXT: [[BROADCAST_SPLAT:%.*]] = shufflevector <2 x i1> [[BROADCAST_SPLATINSERT]], <2 x i1> poison, <2 x i32> zeroinitializer
-; CHECK-NEXT: [[TMP0:%.*]] = xor <2 x i1> [[BROADCAST_SPLAT]], splat (i1 true)
; CHECK-NEXT: br label %[[VECTOR_BODY:.*]]
; CHECK: [[VECTOR_BODY]]:
; CHECK-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[PRED_STORE_CONTINUE5:.*]] ]
@@ -25,13 +22,12 @@ define void @constant_fold_commutative_and(ptr %ptr.n, ptr noalias %p, i1 %cond)
; CHECK-NEXT: br i1 [[COND]], label %[[LATCH4:.*]], label %[[PRED_11:.*]]
; CHECK: [[PRED_11]]:
; CHECK-NEXT: [[TMP3:%.*]] = icmp uge <2 x i64> [[BROADCAST_SPLAT2]], splat (i64 4)
-; CHECK-NEXT: [[TMP5:%.*]] = select <2 x i1> [[TMP0]], <2 x i1> [[TMP3]], <2 x i1> zeroinitializer
; CHECK-NEXT: [[TMP7:%.*]] = icmp ult <2 x i64> [[BROADCAST_SPLAT2]], splat (i64 7)
; CHECK-NEXT: br label %[[LATCH4]]
; CHECK: [[LATCH4]]:
-; CHECK-NEXT: [[TMP12:%.*]] = phi <2 x i1> [ zeroinitializer, %[[VECTOR_BODY]] ], [ [[TMP0]], %[[PRED_11]] ]
+; CHECK-NEXT: [[TMP12:%.*]] = phi <2 x i1> [ zeroinitializer, %[[VECTOR_BODY]] ], [ splat (i1 true), %[[PRED_11]] ]
; CHECK-NEXT: [[TMP13:%.*]] = phi <2 x i1> [ poison, %[[VECTOR_BODY]] ], [ [[TMP7]], %[[PRED_11]] ]
-; CHECK-NEXT: [[TMP14:%.*]] = phi <2 x i1> [ zeroinitializer, %[[VECTOR_BODY]] ], [ [[TMP5]], %[[PRED_11]] ]
+; CHECK-NEXT: [[TMP14:%.*]] = phi <2 x i1> [ zeroinitializer, %[[VECTOR_BODY]] ], [ [[TMP3]], %[[PRED_11]] ]
; CHECK-NEXT: [[PREDPHI3:%.*]] = select <2 x i1> [[TMP14]], <2 x i1> [[TMP13]], <2 x i1> [[TMP12]]
; CHECK-NEXT: [[TMP6:%.*]] = extractelement <2 x i1> [[TMP1]], i64 0
; CHECK-NEXT: br i1 [[TMP6]], label %[[PRED_STORE_IF:.*]], label %[[PRED_STORE_CONTINUE:.*]]
diff --git a/llvm/test/Transforms/LoopVectorize/div-exact.ll b/llvm/test/Transforms/LoopVectorize/div-exact.ll
index 7bec969285194..4cf09d41cc0e9 100644
--- a/llvm/test/Transforms/LoopVectorize/div-exact.ll
+++ b/llvm/test/Transforms/LoopVectorize/div-exact.ll
@@ -14,76 +14,18 @@ define void @unknown_divisor(ptr noalias %p, i32 %n, i1 %c) {
; CHECK: [[VECTOR_PH]]:
; CHECK-NEXT: [[N_MOD_VF:%.*]] = and i32 [[N]], 3
; CHECK-NEXT: [[N_VEC:%.*]] = sub i32 [[N]], [[N_MOD_VF]]
-; CHECK-NEXT: [[BROADCAST_SPLATINSERT:%.*]] = insertelement <4 x i1> poison, i1 [[C]], i64 0
-; CHECK-NEXT: [[BROADCAST_SPLAT:%.*]] = shufflevector <4 x i1> [[BROADCAST_SPLATINSERT]], <4 x i1> poison, <4 x i32> zeroinitializer
; CHECK-NEXT: br label %[[VECTOR_BODY:.*]]
; CHECK: [[VECTOR_BODY]]:
-; CHECK-NEXT: [[INDEX:%.*]] = phi i32 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[LATCH14:.*]] ]
-; CHECK-NEXT: [[TMP2:%.*]] = add i32 [[INDEX]], 1
-; CHECK-NEXT: [[TMP3:%.*]] = add i32 [[INDEX]], 2
-; CHECK-NEXT: [[TMP4:%.*]] = add i32 [[INDEX]], 3
+; CHECK-NEXT: [[TMP4:%.*]] = phi i32 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[LATCH14:.*]] ]
; CHECK-NEXT: br i1 [[C]], label %[[IF1:.*]], label %[[LATCH14]]
; CHECK: [[IF1]]:
-; CHECK-NEXT: [[TMP7:%.*]] = getelementptr i32, ptr [[P]], i32 [[INDEX]]
-; CHECK-NEXT: [[TMP8:%.*]] = getelementptr i32, ptr [[P]], i32 [[TMP2]]
-; CHECK-NEXT: [[TMP9:%.*]] = getelementptr i32, ptr [[P]], i32 [[TMP3]]
; CHECK-NEXT: [[TMP10:%.*]] = getelementptr i32, ptr [[P]], i32 [[TMP4]]
-; CHECK-NEXT: br i1 [[C]], label %[[PRED_LOAD_IF:.*]], label %[[PRED_LOAD_CONTINUE:.*]]
-; CHECK: [[PRED_LOAD_IF]]:
-; CHECK-NEXT: [[TMP12:%.*]] = load i32, ptr [[TMP7]], align 4
-; CHECK-NEXT: [[TMP13:%.*]] = insertelement <4 x i32> poison, i32 [[TMP12]], i64 0
-; CHECK-NEXT: br label %[[PRED_LOAD_CONTINUE]]
-; CHECK: [[PRED_LOAD_CONTINUE]]:
-; CHECK-NEXT: [[TMP14:%.*]] = phi <4 x i32> [ poison, %[[IF1]] ], [ [[TMP13]], %[[PRED_LOAD_IF]] ]
-; CHECK-NEXT: br i1 [[C]], label %[[PRED_LOAD_IF3:.*]], label %[[PRED_LOAD_CONTINUE4:.*]]
-; CHECK: [[PRED_LOAD_IF3]]:
-; CHECK-NEXT: [[TMP16:%.*]] = load i32, ptr [[TMP8]], align 4
-; CHECK-NEXT: [[TMP17:%.*]] = insertelement <4 x i32> [[TMP14]], i32 [[TMP16]], i64 1
-; CHECK-NEXT: br label %[[PRED_LOAD_CONTINUE4]]
-; CHECK: [[PRED_LOAD_CONTINUE4]]:
-; CHECK-NEXT: [[TMP18:%.*]] = phi <4 x i32> [ [[TMP14]], %[[PRED_LOAD_CONTINUE]] ], [ [[TMP17]], %[[PRED_LOAD_IF3]] ]
-; CHECK-NEXT: br i1 [[C]], label %[[PRED_LOAD_IF5:.*]], label %[[PRED_LOAD_CONTINUE6:.*]]
-; CHECK: [[PRED_LOAD_IF5]]:
-; CHECK-NEXT: [[TMP20:%.*]] = load i32, ptr [[TMP9]], align 4
-; CHECK-NEXT: [[TMP21:%.*]] = insertelement <4 x i32> [[TMP18]], i32 [[TMP20]], i64 2
-; CHECK-NEXT: br label %[[PRED_LOAD_CONTINUE6]]
-; CHECK: [[PRED_LOAD_CONTINUE6]]:
-; CHECK-NEXT: [[TMP22:%.*]] = phi <4 x i32> [ [[TMP18]], %[[PRED_LOAD_CONTINUE4]] ], [ [[TMP21]], %[[PRED_LOAD_IF5]] ]
-; CHECK-NEXT: br i1 [[C]], label %[[PRED_LOAD_IF7:.*]], label %[[PRED_LOAD_CONTINUE8:.*]]
-; CHECK: [[PRED_LOAD_IF7]]:
-; CHECK-NEXT: [[TMP24:%.*]] = load i32, ptr [[TMP10]], align 4
-; CHECK-NEXT: [[TMP25:%.*]] = insertelement <4 x i32> [[TMP22]], i32 [[TMP24]], i64 3
-; CHECK-NEXT: br label %[[PRED_LOAD_CONTINUE8]]
-; CHECK: [[PRED_LOAD_CONTINUE8]]:
-; CHECK-NEXT: [[TMP23:%.*]] = phi <4 x i32> [ [[TMP22]], %[[PRED_LOAD_CONTINUE6]] ], [ [[TMP25]], %[[PRED_LOAD_IF7]] ]
-; CHECK-NEXT: [[TMP28:%.*]] = call <4 x i32> @llvm.masked.udiv.v4i32(<4 x i32> splat (i32 100), <4 x i32> [[TMP23]], <4 x i1> [[BROADCAST_SPLAT]])
-; CHECK-NEXT: br i1 [[C]], label %[[PRED_STORE_IF:.*]], label %[[PRED_STORE_CONTINUE:.*]]
-; CHECK: [[PRED_STORE_IF]]:
-; CHECK-NEXT: [[TMP29:%.*]] = extractelement <4 x i32> [[TMP28]], i64 0
-; CHECK-NEXT: store i32 [[TMP29]], ptr [[TMP7]], align 4
-; CHECK-NEXT: br label %[[PRED_STORE_CONTINUE]]
-; CHECK: [[PRED_STORE_CONTINUE]]:
-; CHECK-NEXT: br i1 [[C]], label %[[PRED_STORE_IF9:.*]], label %[[PRED_STORE_CONTINUE10:.*]]
-; CHECK: [[PRED_STORE_IF9]]:
-; CHECK-NEXT: [[TMP30:%.*]] = extractelement <4 x i32> [[TMP28]], i64 1
-; CHECK-NEXT: store i32 [[TMP30]], ptr [[TMP8]], align 4
-; CHECK-NEXT: br label %[[PRED_STORE_CONTINUE10]]
-; CHECK: [[PRED_STORE_CONTINUE10]]:
-; CHECK-NEXT: br i1 [[C]], label %[[PRED_STORE_IF11:.*]], label %[[PRED_STORE_CONTINUE12:.*]]
-; CHECK: [[PRED_STORE_IF11]]:
-; CHECK-NEXT: [[TMP31:%.*]] = extractelement <4 x i32> [[TMP28]], i64 2
-; CHECK-NEXT: store i32 [[TMP31]], ptr [[TMP9]], align 4
-; CHECK-NEXT: br label %[[PRED_STORE_CONTINUE12]]
-; CHECK: [[PRED_STORE_CONTINUE12]]:
-; CHECK-NEXT: br i1 [[C]], label %[[PRED_STORE_IF13:.*]], label %[[PRED_STORE_CONTINUE14:.*]]
-; CHECK: [[PRED_STORE_IF13]]:
-; CHECK-NEXT: [[TMP32:%.*]] = extractelement <4 x i32> [[TMP28]], i64 3
-; CHECK-NEXT: store i32 [[TMP32]], ptr [[TMP10]], align 4
-; CHECK-NEXT: br label %[[PRED_STORE_CONTINUE14]]
-; CHECK: [[PRED_STORE_CONTINUE14]]:
+; CHECK-NEXT: [[WIDE_LOAD:%.*]] = load <4 x i32>, ptr [[TMP10]], align 4
+; CHECK-NEXT: [[TMP5:%.*]] = udiv exact <4 x i32> splat (i32 100), [[WIDE_LOAD]]
+; CHECK-NEXT: store <4 x i32> [[TMP5]], ptr [[TMP10]], align 4
; CHECK-NEXT: br label %[[LATCH14]]
; CHECK: [[LATCH14]]:
-; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i32 [[INDEX]], 4
+; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i32 [[TMP4]], 4
; CHECK-NEXT: [[TMP33:%.*]] = icmp eq i32 [[INDEX_NEXT]], [[N_VEC]]
; CHECK-NEXT: br i1 [[TMP33]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP0:![0-9]+]]
; CHECK: [[MIDDLE_BLOCK]]:
@@ -146,72 +88,16 @@ define void @unknown_dividend(ptr noalias %p, i32 %n, i1 %c) {
; CHECK-NEXT: [[N_VEC:%.*]] = sub i32 [[N]], [[N_MOD_VF]]
; CHECK-NEXT: br label %[[VECTOR_BODY:.*]]
; CHECK: [[VECTOR_BODY]]:
-; CHECK-NEXT: [[INDEX:%.*]] = phi i32 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[LATCH14:.*]] ]
-; CHECK-NEXT: [[TMP2:%.*]] = add i32 [[INDEX]], 1
-; CHECK-NEXT: [[TMP3:%.*]] = add i32 [[INDEX]], 2
-; CHECK-NEXT: [[TMP4:%.*]] = add i32 [[INDEX]], 3
+; CHECK-NEXT: [[TMP4:%.*]] = phi i32 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[LATCH14:.*]] ]
; CHECK-NEXT: br i1 [[C]], label %[[IF1:.*]], label %[[LATCH14]]
; CHECK: [[IF1]]:
-; CHECK-NEXT: [[TMP7:%.*]] = getelementptr i32, ptr [[P]], i32 [[INDEX]]
-; CHECK-NEXT: [[TMP8:%.*]] = getelementptr i32, ptr [[P]], i32 [[TMP2]]
-; CHECK-NEXT: [[TMP9:%.*]] = getelementptr i32, ptr [[P]], i32 [[TMP3]]
; CHECK-NEXT: [[TMP10:%.*]] = getelementptr i32, ptr [[P]], i32 [[TMP4]]
-; CHECK-NEXT: br i1 [[C]], label %[[PRED_LOAD_IF:.*]], label %[[PRED_LOAD_CONTINUE:.*]]
-; CHECK: [[PRED_LOAD_IF]]:
-; CHECK-NEXT: [[TMP12:%.*]] = load i32, ptr [[TMP7]], align 4
-; CHECK-NEXT: [[TMP13:%.*]] = insertelement <4 x i32> poison, i32 [[TMP12]], i64 0
-; CHECK-NEXT: br label %[[PRED_LOAD_CONTINUE]]
-; CHECK: [[PRED_LOAD_CONTINUE]]:
-; CHECK-NEXT: [[TMP14:%.*]] = phi <4 x i32> [ poison, %[[IF1]] ], [ [[TMP13]], %[[PRED_LOAD_IF]] ]
-; CHECK-NEXT: br i1 [[C]], label %[[PRED_LOAD_IF3:.*]], label %[[PRED_LOAD_CONTINUE4:.*]]
-; CHECK: [[PRED_LOAD_IF3]]:
-; CHECK-NEXT: [[TMP16:%.*]] = load i32, ptr [[TMP8]], align 4
-; CHECK-NEXT: [[TMP17:%.*]] = insertelement <4 x i32> [[TMP14]], i32 [[TMP16]], i64 1
-; CHECK-NEXT: br label %[[PRED_LOAD_CONTINUE4]]
-; CHECK: [[PRED_LOAD_CONTINUE4]]:
-; CHECK-NEXT: [[TMP18:%.*]] = phi <4 x i32> [ [[TMP14]], %[[PRED_LOAD_CONTINUE]] ], [ [[TMP17]], %[[PRED_LOAD_IF3]] ]
-; CHECK-NEXT: br i1 [[C]], label %[[PRED_LOAD_IF5:.*]], label %[[PRED_LOAD_CONTINUE6:.*]]
-; CHECK: [[PRED_LOAD_IF5]]:
-; CHECK-NEXT: [[TMP20:%.*]] = load i32, ptr [[TMP9]], align 4
-; CHECK-NEXT: [[TMP21:%.*]] = insertelement <4 x i32> [[TMP18]], i32 [[TMP20]], i64 2
-; CHECK-NEXT: br label %[[PRED_LOAD_CONTINUE6]]
-; CHECK: [[PRED_LOAD_CONTINUE6]]:
-; CHECK-NEXT: [[TMP22:%.*]] = phi <4 x i32> [ [[TMP18]], %[[PRED_LOAD_CONTINUE4]] ], [ [[TMP21]], %[[PRED_LOAD_IF5]] ]
-; CHECK-NEXT: br i1 [[C]], label %[[PRED_LOAD_IF7:.*]], label %[[PRED_LOAD_CONTINUE8:.*]]
-; CHECK: [[PRED_LOAD_IF7]]:
-; CHECK-NEXT: [[TMP24:%.*]] = load i32, ptr [[TMP10]], align 4
-; CHECK-NEXT: [[TMP25:%.*]] = insertelement <4 x i32> [[TMP22]], i32 [[TMP24]], i64 3
-; CHECK-NEXT: br label %[[PRED_LOAD_CONTINUE8]]
-; CHECK: [[PRED_LOAD_CONTINUE8]]:
-; CHECK-NEXT: [[TMP23:%.*]] = phi <4 x i32> [ [[TMP22]], %[[PRED_LOAD_CONTINUE6]] ], [ [[TMP25]], %[[PRED_LOAD_IF7]] ]
+; CHECK-NEXT: [[TMP23:%.*]] = load <4 x i32>, ptr [[TMP10]], align 4
; CHECK-NEXT: [[TMP27:%.*]] = udiv exact <4 x i32> [[TMP23]], splat (i32 100)
-; CHECK-NEXT: br i1 [[C]], label %[[PRED_STORE_IF:.*]], label %[[PRED_STORE_CONTINUE:.*]]
-; CHECK: [[PRED_STORE_IF]]:
-; CHECK-NEXT: [[TMP28:%.*]] = extractelement <4 x i32> [[TMP27]], i64 0
-; CHECK-NEXT: store i32 [[TMP28]], ptr [[TMP7]], align 4
-; CHECK-NEXT: br label %[[PRED_STORE_CONTINUE]]
-; CHECK: [[PRED_STORE_CONTINUE]]:
-; CHECK-NEXT: br i1 [[C]], label %[[PRED_STORE_IF9:.*]], label %[[PRED_STORE_CONTINUE10:.*]]
-; CHECK: [[PRED_STORE_IF9]]:
-; CHECK-NEXT: [[TMP29:%.*]] = extractelement <4 x i32> [[TMP27]], i64 1
-; CHECK-NEXT: store i32 [[TMP29]], ptr [[TMP8]], align 4
-; CHECK-NEXT: br label %[[PRED_STORE_CONTINUE10]]
-; CHECK: [[PRED_STORE_CONTINUE10]]:
-; CHECK-NEXT: br i1 [[C]], label %[[PRED_STORE_IF11:.*]], label %[[PRED_STORE_CONTINUE12:.*]]
-; CHECK: [[PRED_STORE_IF11]]:
-; CHECK-NEXT: [[TMP30:%.*]] = extractelement <4 x i32> [[TMP27]], i64 2
-; CHECK-NEXT: store i32 [[TMP30]], ptr [[TMP9]], align 4
-; CHECK-NEXT: br label %[[PRED_STORE_CONTINUE12]]
-; CHECK: [[PRED_STORE_CONTINUE12]]:
-; CHECK-NEXT: br i1 [[C]], label %[[PRED_STORE_IF13:.*]], label %[[PRED_STORE_CONTINUE14:.*]]
-; CHECK: [[PRED_STORE_IF13]]:
-; CHECK-NEXT: [[TMP31:%.*]] = extractelement <4 x i32> [[TMP27]], i64 3
-; CHECK-NEXT: store i32 [[TMP31]], ptr [[TMP10]], align 4
-; CHECK-NEXT: br label %[[PRED_STORE_CONTINUE14]]
-; CHECK: [[PRED_STORE_CONTINUE14]]:
+; CHECK-NEXT: store <4 x i32> [[TMP27]], ptr [[TMP10]], align 4
; CHECK-NEXT: br label %[[LATCH14]]
; CHECK: [[LATCH14]]:
-; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i32 [[INDEX]], 4
+; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i32 [[TMP4]], 4
; CHECK-NEXT: [[TMP32:%.*]] = icmp eq i32 [[INDEX_NEXT]], [[N_VEC]]
; CHECK-NEXT: br i1 [[TMP32]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP4:![0-9]+]]
; CHECK: [[MIDDLE_BLOCK]]:
diff --git a/llvm/test/Transforms/LoopVectorize/hoist-predicated-loads-with-predicated-stores.ll b/llvm/test/Transforms/LoopVectorize/hoist-predicated-loads-with-predicated-stores.ll
index 386c695205be8..51ff513c4716d 100644
--- a/llvm/test/Transforms/LoopVectorize/hoist-predicated-loads-with-predicated-stores.ll
+++ b/llvm/test/Transforms/LoopVectorize/hoist-predicated-loads-with-predicated-stores.ll
@@ -1616,58 +1616,31 @@ define void @sink_stores_cse_select_dropping_fmf(ptr %dst, ptr %src, ptr %invar.
; CHECK: [[VECTOR_PH]]:
; CHECK-NEXT: [[N_MOD_VF:%.*]] = and i64 [[N]], 1
; CHECK-NEXT: [[N_VEC:%.*]] = sub i64 [[N]], [[N_MOD_VF]]
-; CHECK-NEXT: [[BROADCAST_SPLATINSERT1:%.*]] = insertelement <2 x i1> poison, i1 [[C]], i64 0
-; CHECK-NEXT: [[BROADCAST_SPLAT1:%.*]] = shufflevector <2 x i1> [[BROADCAST_SPLATINSERT1]], <2 x i1> poison, <2 x i32> zeroinitializer
-; CHECK-NEXT: [[TMP7:%.*]] = xor i1 [[C]], true
; CHECK-NEXT: br label %[[VECTOR_BODY1:.*]]
; CHECK: [[VECTOR_BODY1]]:
-; CHECK-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[LOOP_LATCH20:.*]] ]
-; CHECK-NEXT: [[TMP10:%.*]] = add i64 [[INDEX]], 1
+; CHECK-NEXT: [[TMP10:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[LOOP_LATCH20:.*]] ]
; CHECK-NEXT: [[TMP1:%.*]] = load float, ptr [[SRC]], align 4, !alias.scope [[META119:![0-9]+]]
; CHECK-NEXT: [[BROADCAST_SPLATINSERT:%.*]] = insertelement <2 x float> poison, float [[TMP1]], i64 0
; CHECK-NEXT: [[BROADCAST_SPLAT:%.*]] = shufflevector <2 x float> [[BROADCAST_SPLATINSERT]], <2 x float> poison, <2 x i32> zeroinitializer
-; CHECK-NEXT: [[TMP5:%.*]] = getelementptr inbounds float, ptr [[DST]], i64 [[INDEX]]
; CHECK-NEXT: [[TMP6:%.*]] = getelementptr inbounds float, ptr [[DST]], i64 [[TMP10]]
; CHECK-NEXT: br i1 [[C]], label %[[THEN15:.*]], label %[[ELSE10:.*]]
; CHECK: [[ELSE10]]:
; CHECK-NEXT: [[TMP2:%.*]] = fmul <2 x float> [[BROADCAST_SPLAT]], splat (float 3.000000e+00)
-; CHECK-NEXT: br i1 [[TMP7]], label %[[PRED_STORE_IF:.*]], label %[[PRED_STORE_CONTINUE:.*]]
-; CHECK: [[PRED_STORE_IF]]:
-; CHECK-NEXT: [[TMP8:%.*]] = extractelement <2 x float> [[TMP2]], i64 0
-; CHECK-NEXT: store float [[TMP8]], ptr [[TMP5]], align 4, !alias.scope [[META122:![0-9]+]], !noalias [[META124:![0-9]+]]
-; CHECK-NEXT: br label %[[PRED_STORE_CONTINUE]]
-; CHECK: [[PRED_STORE_CONTINUE]]:
-; CHECK-NEXT: br i1 [[TMP7]], label %[[PRED_STORE_IF13:.*]], label %[[PRED_STORE_CONTINUE14:.*]]
-; CHECK: [[PRED_STORE_IF13]]:
-; CHECK-NEXT: [[TMP9:%.*]] = extractelement <2 x float> [[TMP2]], i64 0
-; CHECK-NEXT: store float [[TMP9]], ptr [[TMP6]], align 4, !alias.scope [[META122]], !noalias [[META124]]
-; CHECK-NEXT: br label %[[PRED_STORE_CONTINUE14]]
-; CHECK: [[PRED_STORE_CONTINUE14]]:
+; CHECK-NEXT: store <2 x float> [[TMP2]], ptr [[TMP6]], align 4, !alias.scope [[META122:![0-9]+]], !noalias [[META124:![0-9]+]]
; CHECK-NEXT: br label %[[LOOP_LATCH20]]
; CHECK: [[THEN15]]:
; CHECK-NEXT: [[TMP3:%.*]] = fmul nnan <2 x float> [[BROADCAST_SPLAT]], splat (float 2.000000e+00)
-; CHECK-NEXT: br i1 [[C]], label %[[PRED_STORE_IF16:.*]], label %[[PRED_STORE_CONTINUE17:.*]]
-; CHECK: [[PRED_STORE_IF16]]:
-; CHECK-NEXT: [[TMP4:%.*]] = extractelement <2 x float> [[TMP3]], i64 0
-; CHECK-NEXT: store float [[TMP4]], ptr [[TMP5]], align 4, !alias.scope [[META122]], !noalias [[META124]]
-; CHECK-NEXT: br label %[[PRED_STORE_CONTINUE17]]
-; CHECK: [[PRED_STORE_CONTINUE17]]:
-; CHECK-NEXT: br i1 [[C]], label %[[PRED_STORE_IF18:.*]], label %[[VECTOR_BODY:.*]]
-; CHECK: [[PRED_STORE_IF18]]:
-; CHECK-NEXT: [[TMP12:%.*]] = extractelement <2 x float> [[TMP3]], i64 0
-; CHECK-NEXT: store float [[TMP12]], ptr [[TMP6]], align 4, !alias.scope [[META122]], !noalias [[META124]]
-; CHECK-NEXT: br label %[[VECTOR_BODY]]
-; CHECK: [[VECTOR_BODY]]:
+; CHECK-NEXT: store <2 x float> [[TMP3]], ptr [[TMP6]], align 4, !alias.scope [[META122]], !noalias [[META124]]
; CHECK-NEXT: br label %[[LOOP_LATCH20]]
; CHECK: [[LOOP_LATCH20]]:
-; CHECK-NEXT: [[TMP13:%.*]] = phi <2 x float> [ [[TMP2]], %[[PRED_STORE_CONTINUE14]] ], [ poison, %[[VECTOR_BODY]] ]
-; CHECK-NEXT: [[TMP14:%.*]] = phi <2 x float> [ poison, %[[PRED_STORE_CONTINUE14]] ], [ [[TMP3]], %[[VECTOR_BODY]] ]
-; CHECK-NEXT: [[TMP15:%.*]] = phi <2 x i1> [ zeroinitializer, %[[PRED_STORE_CONTINUE14]] ], [ [[BROADCAST_SPLAT1]], %[[VECTOR_BODY]] ]
+; CHECK-NEXT: [[TMP13:%.*]] = phi <2 x float> [ [[TMP2]], %[[ELSE10]] ], [ poison, %[[THEN15]] ]
+; CHECK-NEXT: [[TMP14:%.*]] = phi <2 x float> [ poison, %[[ELSE10]] ], [ [[TMP3]], %[[THEN15]] ]
+; CHECK-NEXT: [[TMP15:%.*]] = phi <2 x i1> [ zeroinitializer, %[[ELSE10]] ], [ splat (i1 true), %[[THEN15]] ]
; CHECK-NEXT: [[PREDPHI:%.*]] = select <2 x i1> [[TMP15]], <2 x float> [[TMP14]], <2 x float> [[TMP13]]
; CHECK-NEXT: [[TMP16:%.*]] = fmul nnan <2 x float> [[PREDPHI]], splat (float 5.000000e+00)
; CHECK-NEXT: [[TMP17:%.*]] = extractelement <2 x float> [[TMP16]], i64 1
; CHECK-NEXT: store float [[TMP17]], ptr [[INVAR_DST]], align 4, !alias.scope [[META126:![0-9]+]], !noalias [[META119]]
-; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 2
+; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[TMP10]], 2
; CHECK-NEXT: [[TMP11:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
; CHECK-NEXT: br i1 [[TMP11]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY1]], !llvm.loop [[LOOP127:![0-9]+]]
; CHECK: [[MIDDLE_BLOCK]]:
diff --git a/llvm/test/Transforms/LoopVectorize/hoist-predicated-loads.ll b/llvm/test/Transforms/LoopVectorize/hoist-predicated-loads.ll
index 5dbff3c406c86..56c0c1d95e7cf 100644
--- a/llvm/test/Transforms/LoopVectorize/hoist-predicated-loads.ll
+++ b/llvm/test/Transforms/LoopVectorize/hoist-predicated-loads.ll
@@ -912,58 +912,35 @@ define void @hoist_predicated_load_with_chained_geps1(ptr %dst, ptr %src, i1 %co
; CHECK-NEXT: [[FOUND_CONFLICT:%.*]] = and i1 [[BOUND0]], [[BOUND1]]
; CHECK-NEXT: br i1 [[FOUND_CONFLICT]], label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]]
; CHECK: [[VECTOR_PH]]:
-; CHECK-NEXT: [[BROADCAST_SPLATINSERT:%.*]] = insertelement <2 x i1> poison, i1 [[COND]], i64 0
-; CHECK-NEXT: [[BROADCAST_SPLAT:%.*]] = shufflevector <2 x i1> [[BROADCAST_SPLATINSERT]], <2 x i1> poison, <2 x i32> zeroinitializer
-; CHECK-NEXT: [[TMP0:%.*]] = xor i1 [[COND]], true
; CHECK-NEXT: br label %[[VECTOR_BODY:.*]]
; CHECK: [[VECTOR_BODY]]:
-; CHECK-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[LOOP_LATCH11:.*]] ]
+; CHECK-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[PRED_LOAD_CONTINUE8:.*]] ]
; CHECK-NEXT: [[TMP2:%.*]] = add i64 [[INDEX]], 1
; CHECK-NEXT: br i1 [[COND]], label %[[THEN6:.*]], label %[[ELSE3:.*]]
; CHECK: [[ELSE3]]:
-; CHECK-NEXT: br i1 [[TMP0]], label %[[PRED_LOAD_IF:.*]], label %[[PRED_LOAD_CONTINUE:.*]]
-; CHECK: [[PRED_LOAD_IF]]:
; CHECK-NEXT: [[TMP20:%.*]] = getelementptr [11 x i16], ptr [[SRC]], i64 [[INDEX]]
-; CHECK-NEXT: [[TMP21:%.*]] = getelementptr i8, ptr [[TMP20]], i64 8
-; CHECK-NEXT: [[TMP4:%.*]] = load i16, ptr [[TMP21]], align 2, !alias.scope [[META68:![0-9]+]]
-; CHECK-NEXT: [[TMP5:%.*]] = insertelement <2 x i16> poison, i16 [[TMP4]], i64 0
-; CHECK-NEXT: br label %[[PRED_LOAD_CONTINUE]]
-; CHECK: [[PRED_LOAD_CONTINUE]]:
-; CHECK-NEXT: [[TMP6:%.*]] = phi <2 x i16> [ poison, %[[ELSE3]] ], [ [[TMP5]], %[[PRED_LOAD_IF]] ]
-; CHECK-NEXT: br i1 [[TMP0]], label %[[PRED_LOAD_IF4:.*]], label %[[PRED_LOAD_CONTINUE5:.*]]
-; CHECK: [[PRED_LOAD_IF4]]:
; CHECK-NEXT: [[TMP7:%.*]] = getelementptr [11 x i16], ptr [[SRC]], i64 [[TMP2]]
+; CHECK-NEXT: [[TMP3:%.*]] = getelementptr i8, ptr [[TMP20]], i64 8
; CHECK-NEXT: [[TMP8:%.*]] = getelementptr i8, ptr [[TMP7]], i64 8
-; CHECK-NEXT: [[TMP9:%.*]] = load i16, ptr [[TMP8]], align 2, !alias.scope [[META68]]
-; CHECK-NEXT: [[TMP10:%.*]] = insertelement <2 x i16> [[TMP6]], i16 [[TMP9]], i64 1
-; CHECK-NEXT: br label %[[PRED_LOAD_CONTINUE5]]
-; CHECK: [[PRED_LOAD_CONTINUE5]]:
-; CHECK-NEXT: [[TMP11:%.*]] = phi <2 x i16> [ [[TMP6]], %[[PRED_LOAD_CONTINUE]] ], [ [[TMP10]], %[[PRED_LOAD_IF4]] ]
-; CHECK-NEXT: br label %[[LOOP_LATCH11]]
-; CHECK: [[THEN6]]:
-; CHECK-NEXT: br i1 [[COND]], label %[[PRED_LOAD_IF7:.*]], label %[[PRED_LOAD_CONTINUE8:.*]]
-; CHECK: [[PRED_LOAD_IF7]]:
-; CHECK-NEXT: [[TMP12:%.*]] = getelementptr [11 x i16], ptr [[SRC]], i64 [[INDEX]]
-; CHECK-NEXT: [[TMP13:%.*]] = getelementptr i8, ptr [[TMP12]], i64 8
-; CHECK-NEXT: [[TMP14:%.*]] = load i16, ptr [[TMP13]], align 2, !alias.scope [[META68]]
+; CHECK-NEXT: [[TMP14:%.*]] = load i16, ptr [[TMP3]], align 2, !alias.scope [[META68:![0-9]+]]
+; CHECK-NEXT: [[TMP6:%.*]] = load i16, ptr [[TMP8]], align 2, !alias.scope [[META68]]
; CHECK-NEXT: [[TMP15:%.*]] = insertelement <2 x i16> poison, i16 [[TMP14]], i64 0
+; CHECK-NEXT: [[TMP9:%.*]] = insertelement <2 x i16> [[TMP15]], i16 [[TMP6]], i64 1
; CHECK-NEXT: br label %[[PRED_LOAD_CONTINUE8]]
-; CHECK: [[PRED_LOAD_CONTINUE8]]:
-; CHECK-NEXT: [[TMP16:%.*]] = phi <2 x i16> [ poison, %[[THEN6]] ], [ [[TMP15]], %[[PRED_LOAD_IF7]] ]
-; CHECK-NEXT: br i1 [[COND]], label %[[PRED_LOAD_IF9:.*]], label %[[PRED_LOAD_CONTINUE10:.*]]
-; CHECK: [[PRED_LOAD_IF9]]:
-; CHECK-NEXT: [[TMP17:%.*]] = getelementptr [11 x i16], ptr [[SRC]], i64 [[TMP2]]
+; CHECK: [[THEN6]]:
+; CHECK-NEXT: [[TMP17:%.*]] = getelementptr [11 x i16], ptr [[SRC]], i64 [[INDEX]]
+; CHECK-NEXT: [[TMP10:%.*]] = getelementptr [11 x i16], ptr [[SRC]], i64 [[TMP2]]
; CHECK-NEXT: [[TMP18:%.*]] = getelementptr i8, ptr [[TMP17]], i64 8
-; CHECK-NEXT: [[TMP19:%.*]] = load i16, ptr [[TMP18]], align 2, !alias.scope [[META68]]
+; CHECK-NEXT: [[TMP12:%.*]] = getelementptr i8, ptr [[TMP10]], i64 8
+; CHECK-NEXT: [[TMP13:%.*]] = load i16, ptr [[TMP18]], align 2, !alias.scope [[META68]]
+; CHECK-NEXT: [[TMP19:%.*]] = load i16, ptr [[TMP12]], align 2, !alias.scope [[META68]]
+; CHECK-NEXT: [[TMP16:%.*]] = insertelement <2 x i16> poison, i16 [[TMP13]], i64 0
; CHECK-NEXT: [[TMP27:%.*]] = insertelement <2 x i16> [[TMP16]], i16 [[TMP19]], i64 1
-; CHECK-NEXT: br label %[[PRED_LOAD_CONTINUE10]]
-; CHECK: [[PRED_LOAD_CONTINUE10]]:
-; CHECK-NEXT: [[TMP28:%.*]] = phi <2 x i16> [ [[TMP16]], %[[PRED_LOAD_CONTINUE8]] ], [ [[TMP27]], %[[PRED_LOAD_IF9]] ]
-; CHECK-NEXT: br label %[[LOOP_LATCH11]]
-; CHECK: [[LOOP_LATCH11]]:
-; CHECK-NEXT: [[TMP22:%.*]] = phi <2 x i16> [ [[TMP11]], %[[PRED_LOAD_CONTINUE5]] ], [ poison, %[[PRED_LOAD_CONTINUE10]] ]
-; CHECK-NEXT: [[TMP23:%.*]] = phi <2 x i16> [ poison, %[[PRED_LOAD_CONTINUE5]] ], [ [[TMP28]], %[[PRED_LOAD_CONTINUE10]] ]
-; CHECK-NEXT: [[TMP24:%.*]] = phi <2 x i1> [ zeroinitializer, %[[PRED_LOAD_CONTINUE5]] ], [ [[BROADCAST_SPLAT]], %[[PRED_LOAD_CONTINUE10]] ]
+; CHECK-NEXT: br label %[[PRED_LOAD_CONTINUE8]]
+; CHECK: [[PRED_LOAD_CONTINUE8]]:
+; CHECK-NEXT: [[TMP22:%.*]] = phi <2 x i16> [ [[TMP9]], %[[ELSE3]] ], [ poison, %[[THEN6]] ]
+; CHECK-NEXT: [[TMP23:%.*]] = phi <2 x i16> [ poison, %[[ELSE3]] ], [ [[TMP27]], %[[THEN6]] ]
+; CHECK-NEXT: [[TMP24:%.*]] = phi <2 x i1> [ zeroinitializer, %[[ELSE3]] ], [ splat (i1 true), %[[THEN6]] ]
; CHECK-NEXT: [[PREDPHI:%.*]] = select <2 x i1> [[TMP24]], <2 x i16> [[TMP23]], <2 x i16> [[TMP22]]
; CHECK-NEXT: [[TMP25:%.*]] = extractelement <2 x i16> [[PREDPHI]], i64 1
; CHECK-NEXT: store i16 [[TMP25]], ptr [[DST]], align 2, !alias.scope [[META71:![0-9]+]], !noalias [[META68]]
@@ -1018,9 +995,6 @@ define void @hoist_predicated_load_with_chained_geps2(ptr %dst, ptr %src, i1 %co
; CHECK-NEXT: [[FOUND_CONFLICT:%.*]] = and i1 [[BOUND0]], [[BOUND1]]
; CHECK-NEXT: br i1 [[FOUND_CONFLICT]], label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]]
; CHECK: [[VECTOR_PH]]:
-; CHECK-NEXT: [[BROADCAST_SPLATINSERT:%.*]] = insertelement <2 x i1> poison, i1 [[COND]], i64 0
-; CHECK-NEXT: [[BROADCAST_SPLAT:%.*]] = shufflevector <2 x i1> [[BROADCAST_SPLATINSERT]], <2 x i1> poison, <2 x i32> zeroinitializer
-; CHECK-NEXT: [[TMP0:%.*]] = xor i1 [[COND]], true
; CHECK-NEXT: br label %[[VECTOR_BODY:.*]]
; CHECK: [[VECTOR_BODY]]:
; CHECK-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[LOOP_LATCH11:.*]] ]
@@ -1029,45 +1003,25 @@ define void @hoist_predicated_load_with_chained_geps2(ptr %dst, ptr %src, i1 %co
; CHECK-NEXT: [[TMP3:%.*]] = getelementptr [11 x i16], ptr [[SRC]], i64 [[TMP2]]
; CHECK-NEXT: br i1 [[COND]], label %[[THEN6:.*]], label %[[ELSE3:.*]]
; CHECK: [[ELSE3]]:
-; CHECK-NEXT: br i1 [[TMP0]], label %[[PRED_LOAD_IF:.*]], label %[[PRED_LOAD_CONTINUE:.*]]
-; CHECK: [[PRED_LOAD_IF]]:
; CHECK-NEXT: [[TMP21:%.*]] = getelementptr i8, ptr [[TMP4]], i64 8
-; CHECK-NEXT: [[TMP5:%.*]] = load i16, ptr [[TMP21]], align 2, !alias.scope [[META75:![0-9]+]]
-; CHECK-NEXT: [[TMP6:%.*]] = insertelement <2 x i16> poison, i16 [[TMP5]], i64 0
-; CHECK-NEXT: br label %[[PRED_LOAD_CONTINUE]]
-; CHECK: [[PRED_LOAD_CONTINUE]]:
-; CHECK-NEXT: [[TMP7:%.*]] = phi <2 x i16> [ poison, %[[ELSE3]] ], [ [[TMP6]], %[[PRED_LOAD_IF]] ]
-; CHECK-NEXT: br i1 [[TMP0]], label %[[PRED_LOAD_IF4:.*]], label %[[PRED_LOAD_CONTINUE5:.*]]
-; CHECK: [[PRED_LOAD_IF4]]:
; CHECK-NEXT: [[TMP8:%.*]] = getelementptr i8, ptr [[TMP3]], i64 8
+; CHECK-NEXT: [[TMP5:%.*]] = load i16, ptr [[TMP21]], align 2, !alias.scope [[META75:![0-9]+]]
; CHECK-NEXT: [[TMP9:%.*]] = load i16, ptr [[TMP8]], align 2, !alias.scope [[META75]]
+; CHECK-NEXT: [[TMP7:%.*]] = insertelement <2 x i16> poison, i16 [[TMP5]], i64 0
; CHECK-NEXT: [[TMP10:%.*]] = insertelement <2 x i16> [[TMP7]], i16 [[TMP9]], i64 1
-; CHECK-NEXT: br label %[[PRED_LOAD_CONTINUE5]]
-; CHECK: [[PRED_LOAD_CONTINUE5]]:
-; CHECK-NEXT: [[TMP11:%.*]] = phi <2 x i16> [ [[TMP7]], %[[PRED_LOAD_CONTINUE]] ], [ [[TMP10]], %[[PRED_LOAD_IF4]] ]
; CHECK-NEXT: br label %[[LOOP_LATCH11]]
; CHECK: [[THEN6]]:
-; CHECK-NEXT: br i1 [[COND]], label %[[PRED_LOAD_IF7:.*]], label %[[PRED_LOAD_CONTINUE8:.*]]
-; CHECK: [[PRED_LOAD_IF7]]:
; CHECK-NEXT: [[TMP12:%.*]] = getelementptr i8, ptr [[TMP4]], i64 8
-; CHECK-NEXT: [[TMP13:%.*]] = load i16, ptr [[TMP12]], align 2, !alias.scope [[META75]]
-; CHECK-NEXT: [[TMP14:%.*]] = insertelement <2 x i16> poison, i16 [[TMP13]], i64 0
-; CHECK-NEXT: br label %[[PRED_LOAD_CONTINUE8]]
-; CHECK: [[PRED_LOAD_CONTINUE8]]:
-; CHECK-NEXT: [[TMP15:%.*]] = phi <2 x i16> [ poison, %[[THEN6]] ], [ [[TMP14]], %[[PRED_LOAD_IF7]] ]
-; CHECK-NEXT: br i1 [[COND]], label %[[PRED_LOAD_IF9:.*]], label %[[PRED_LOAD_CONTINUE10:.*]]
-; CHECK: [[PRED_LOAD_IF9]]:
; CHECK-NEXT: [[TMP16:%.*]] = getelementptr i8, ptr [[TMP3]], i64 8
+; CHECK-NEXT: [[TMP11:%.*]] = load i16, ptr [[TMP12]], align 2, !alias.scope [[META75]]
; CHECK-NEXT: [[TMP17:%.*]] = load i16, ptr [[TMP16]], align 2, !alias.scope [[META75]]
+; CHECK-NEXT: [[TMP15:%.*]] = insertelement <2 x i16> poison, i16 [[TMP11]], i64 0
; CHECK-NEXT: [[TMP18:%.*]] = insertelement <2 x i16> [[TMP15]], i16 [[TMP17]], i64 1
-; CHECK-NEXT: br label %[[PRED_LOAD_CONTINUE10]]
-; CHECK: [[PRED_LOAD_CONTINUE10]]:
-; CHECK-NEXT: [[TMP19:%.*]] = phi <2 x i16> [ [[TMP15]], %[[PRED_LOAD_CONTINUE8]] ], [ [[TMP18]], %[[PRED_LOAD_IF9]] ]
; CHECK-NEXT: br label %[[LOOP_LATCH11]]
; CHECK: [[LOOP_LATCH11]]:
-; CHECK-NEXT: [[TMP20:%.*]] = phi <2 x i16> [ [[TMP11]], %[[PRED_LOAD_CONTINUE5]] ], [ poison, %[[PRED_LOAD_CONTINUE10]] ]
-; CHECK-NEXT: [[TMP24:%.*]] = phi <2 x i16> [ poison, %[[PRED_LOAD_CONTINUE5]] ], [ [[TMP19]], %[[PRED_LOAD_CONTINUE10]] ]
-; CHECK-NEXT: [[TMP22:%.*]] = phi <2 x i1> [ zeroinitializer, %[[PRED_LOAD_CONTINUE5]] ], [ [[BROADCAST_SPLAT]], %[[PRED_LOAD_CONTINUE10]] ]
+; CHECK-NEXT: [[TMP20:%.*]] = phi <2 x i16> [ [[TMP10]], %[[ELSE3]] ], [ poison, %[[THEN6]] ]
+; CHECK-NEXT: [[TMP24:%.*]] = phi <2 x i16> [ poison, %[[ELSE3]] ], [ [[TMP18]], %[[THEN6]] ]
+; CHECK-NEXT: [[TMP22:%.*]] = phi <2 x i1> [ zeroinitializer, %[[ELSE3]] ], [ splat (i1 true), %[[THEN6]] ]
; CHECK-NEXT: [[PREDPHI:%.*]] = select <2 x i1> [[TMP22]], <2 x i16> [[TMP24]], <2 x i16> [[TMP20]]
; CHECK-NEXT: [[TMP23:%.*]] = extractelement <2 x i16> [[PREDPHI]], i64 1
; CHECK-NEXT: store i16 [[TMP23]], ptr [[DST]], align 2, !alias.scope [[META78:![0-9]+]], !noalias [[META75]]
diff --git a/llvm/test/Transforms/LoopVectorize/if-pred-stores.ll b/llvm/test/Transforms/LoopVectorize/if-pred-stores.ll
index b57899e31811a..381c84ec9782f 100644
--- a/llvm/test/Transforms/LoopVectorize/if-pred-stores.ll
+++ b/llvm/test/Transforms/LoopVectorize/if-pred-stores.ll
@@ -418,26 +418,18 @@ define void @minimal_bit_widths(ptr %p, i1 %c) {
; UNROLL: [[VECTOR_PH]]:
; UNROLL-NEXT: br label %[[VECTOR_BODY:.*]]
; UNROLL: [[VECTOR_BODY]]:
-; UNROLL-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[PRED_STORE_CONTINUE:.*]] ]
+; UNROLL-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[PRED_STORE_CONTINUE1:.*]] ]
; UNROLL-NEXT: [[TMP0:%.*]] = add i64 [[INDEX]], 1
; UNROLL-NEXT: [[TMP1:%.*]] = getelementptr i8, ptr [[P]], i64 [[INDEX]]
; UNROLL-NEXT: [[TMP2:%.*]] = getelementptr i8, ptr [[P]], i64 [[TMP0]]
; UNROLL-NEXT: [[TMP3:%.*]] = load i8, ptr [[TMP1]], align 1
; UNROLL-NEXT: [[TMP4:%.*]] = load i8, ptr [[TMP2]], align 1
-; UNROLL-NEXT: br i1 [[C]], label %[[PRED_STORE_IF:.*]], label %[[PRED_STORE_CONTINUE]]
-; UNROLL: [[PRED_STORE_IF]]:
-; UNROLL-NEXT: br i1 [[C]], label %[[PRED_STORE_IF2:.*]], label %[[PRED_STORE_CONTINUE1:.*]]
+; UNROLL-NEXT: br i1 [[C]], label %[[PRED_STORE_IF2:.*]], label %[[PRED_STORE_CONTINUE1]]
; UNROLL: [[PRED_STORE_IF2]]:
; UNROLL-NEXT: store i8 [[TMP3]], ptr [[TMP1]], align 1
+; UNROLL-NEXT: store i8 [[TMP4]], ptr [[TMP2]], align 1
; UNROLL-NEXT: br label %[[PRED_STORE_CONTINUE1]]
; UNROLL: [[PRED_STORE_CONTINUE1]]:
-; UNROLL-NEXT: br i1 [[C]], label %[[PRED_STORE_IF1:.*]], label %[[PRED_STORE_CONTINUE2:.*]]
-; UNROLL: [[PRED_STORE_IF1]]:
-; UNROLL-NEXT: store i8 [[TMP4]], ptr [[TMP2]], align 1
-; UNROLL-NEXT: br label %[[PRED_STORE_CONTINUE2]]
-; UNROLL: [[PRED_STORE_CONTINUE2]]:
-; UNROLL-NEXT: br label %[[PRED_STORE_CONTINUE]]
-; UNROLL: [[PRED_STORE_CONTINUE]]:
; UNROLL-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 2
; UNROLL-NEXT: [[TMP5:%.*]] = icmp eq i64 [[INDEX_NEXT]], 1000
; UNROLL-NEXT: br i1 [[TMP5]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP5:![0-9]+]]
@@ -453,26 +445,18 @@ define void @minimal_bit_widths(ptr %p, i1 %c) {
; UNROLL-NOSIMPLIFY: [[VECTOR_PH]]:
; UNROLL-NOSIMPLIFY-NEXT: br label %[[VECTOR_BODY:.*]]
; UNROLL-NOSIMPLIFY: [[VECTOR_BODY]]:
-; UNROLL-NOSIMPLIFY-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[PRED_STORE_CONTINUE:.*]] ]
+; UNROLL-NOSIMPLIFY-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[PRED_STORE_CONTINUE1:.*]] ]
; UNROLL-NOSIMPLIFY-NEXT: [[TMP0:%.*]] = add i64 [[INDEX]], 1
; UNROLL-NOSIMPLIFY-NEXT: [[TMP1:%.*]] = getelementptr i8, ptr [[P]], i64 [[INDEX]]
; UNROLL-NOSIMPLIFY-NEXT: [[TMP2:%.*]] = getelementptr i8, ptr [[P]], i64 [[TMP0]]
; UNROLL-NOSIMPLIFY-NEXT: [[TMP3:%.*]] = load i8, ptr [[TMP1]], align 1
; UNROLL-NOSIMPLIFY-NEXT: [[TMP4:%.*]] = load i8, ptr [[TMP2]], align 1
-; UNROLL-NOSIMPLIFY-NEXT: br i1 [[C]], label %[[PRED_STORE_IF:.*]], label %[[PRED_STORE_CONTINUE]]
-; UNROLL-NOSIMPLIFY: [[PRED_STORE_IF]]:
-; UNROLL-NOSIMPLIFY-NEXT: br i1 [[C]], label %[[PRED_STORE_IF2:.*]], label %[[PRED_STORE_CONTINUE1:.*]]
+; UNROLL-NOSIMPLIFY-NEXT: br i1 [[C]], label %[[PRED_STORE_IF2:.*]], label %[[PRED_STORE_CONTINUE1]]
; UNROLL-NOSIMPLIFY: [[PRED_STORE_IF2]]:
; UNROLL-NOSIMPLIFY-NEXT: store i8 [[TMP3]], ptr [[TMP1]], align 1
+; UNROLL-NOSIMPLIFY-NEXT: store i8 [[TMP4]], ptr [[TMP2]], align 1
; UNROLL-NOSIMPLIFY-NEXT: br label %[[PRED_STORE_CONTINUE1]]
; UNROLL-NOSIMPLIFY: [[PRED_STORE_CONTINUE1]]:
-; UNROLL-NOSIMPLIFY-NEXT: br i1 [[C]], label %[[PRED_STORE_IF1:.*]], label %[[PRED_STORE_CONTINUE2:.*]]
-; UNROLL-NOSIMPLIFY: [[PRED_STORE_IF1]]:
-; UNROLL-NOSIMPLIFY-NEXT: store i8 [[TMP4]], ptr [[TMP2]], align 1
-; UNROLL-NOSIMPLIFY-NEXT: br label %[[PRED_STORE_CONTINUE2]]
-; UNROLL-NOSIMPLIFY: [[PRED_STORE_CONTINUE2]]:
-; UNROLL-NOSIMPLIFY-NEXT: br label %[[PRED_STORE_CONTINUE]]
-; UNROLL-NOSIMPLIFY: [[PRED_STORE_CONTINUE]]:
; UNROLL-NOSIMPLIFY-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 2
; UNROLL-NOSIMPLIFY-NEXT: [[TMP5:%.*]] = icmp eq i64 [[INDEX_NEXT]], 1000
; UNROLL-NOSIMPLIFY-NEXT: br i1 [[TMP5]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP5:![0-9]+]]
@@ -488,27 +472,14 @@ define void @minimal_bit_widths(ptr %p, i1 %c) {
; VEC: [[VECTOR_PH]]:
; VEC-NEXT: br label %[[VECTOR_BODY:.*]]
; VEC: [[VECTOR_BODY]]:
-; VEC-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[PRED_STORE_CONTINUE:.*]] ]
+; VEC-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[PRED_STORE_CONTINUE2:.*]] ]
; VEC-NEXT: [[TMP0:%.*]] = getelementptr i8, ptr [[P]], i64 [[INDEX]]
; VEC-NEXT: [[WIDE_LOAD:%.*]] = load <2 x i8>, ptr [[TMP0]], align 1
-; VEC-NEXT: br i1 [[C]], label %[[PRED_STORE_IF:.*]], label %[[PRED_STORE_CONTINUE]]
-; VEC: [[PRED_STORE_IF]]:
-; VEC-NEXT: br i1 [[C]], label %[[PRED_STORE_IF2:.*]], label %[[PRED_STORE_CONTINUE1:.*]]
-; VEC: [[PRED_STORE_IF2]]:
-; VEC-NEXT: [[TMP1:%.*]] = extractelement <2 x i8> [[WIDE_LOAD]], i64 0
-; VEC-NEXT: store i8 [[TMP1]], ptr [[TMP0]], align 1
-; VEC-NEXT: br label %[[PRED_STORE_CONTINUE1]]
-; VEC: [[PRED_STORE_CONTINUE1]]:
-; VEC-NEXT: br i1 [[C]], label %[[PRED_STORE_IF1:.*]], label %[[PRED_STORE_CONTINUE2:.*]]
+; VEC-NEXT: br i1 [[C]], label %[[PRED_STORE_IF1:.*]], label %[[PRED_STORE_CONTINUE2]]
; VEC: [[PRED_STORE_IF1]]:
-; VEC-NEXT: [[TMP2:%.*]] = add i64 [[INDEX]], 1
-; VEC-NEXT: [[TMP3:%.*]] = getelementptr i8, ptr [[P]], i64 [[TMP2]]
-; VEC-NEXT: [[TMP4:%.*]] = extractelement <2 x i8> [[WIDE_LOAD]], i64 1
-; VEC-NEXT: store i8 [[TMP4]], ptr [[TMP3]], align 1
+; VEC-NEXT: store <2 x i8> [[WIDE_LOAD]], ptr [[TMP0]], align 1
; VEC-NEXT: br label %[[PRED_STORE_CONTINUE2]]
; VEC: [[PRED_STORE_CONTINUE2]]:
-; VEC-NEXT: br label %[[PRED_STORE_CONTINUE]]
-; VEC: [[PRED_STORE_CONTINUE]]:
; VEC-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 2
; VEC-NEXT: [[TMP5:%.*]] = icmp eq i64 [[INDEX_NEXT]], 1000
; VEC-NEXT: br i1 [[TMP5]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP5:![0-9]+]]
@@ -551,7 +522,7 @@ define void @minimal_bit_widths_with_aliasing_store(i1 %c, ptr %ptr) {
; UNROLL: [[VECTOR_PH]]:
; UNROLL-NEXT: br label %[[VECTOR_BODY:.*]]
; UNROLL: [[VECTOR_BODY]]:
-; UNROLL-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[PRED_STORE_CONTINUE:.*]] ]
+; UNROLL-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[PRED_STORE_CONTINUE1:.*]] ]
; UNROLL-NEXT: [[TMP0:%.*]] = add i64 [[INDEX]], 1
; UNROLL-NEXT: [[TMP1:%.*]] = getelementptr i8, ptr [[PTR]], i64 [[INDEX]]
; UNROLL-NEXT: [[TMP2:%.*]] = getelementptr i8, ptr [[PTR]], i64 [[TMP0]]
@@ -559,20 +530,12 @@ define void @minimal_bit_widths_with_aliasing_store(i1 %c, ptr %ptr) {
; UNROLL-NEXT: [[TMP4:%.*]] = load i8, ptr [[TMP2]], align 1
; UNROLL-NEXT: store i8 0, ptr [[TMP1]], align 1
; UNROLL-NEXT: store i8 0, ptr [[TMP2]], align 1
-; UNROLL-NEXT: br i1 [[C]], label %[[PRED_STORE_IF:.*]], label %[[PRED_STORE_CONTINUE]]
-; UNROLL: [[PRED_STORE_IF]]:
-; UNROLL-NEXT: br i1 [[C]], label %[[PRED_STORE_IF2:.*]], label %[[PRED_STORE_CONTINUE1:.*]]
+; UNROLL-NEXT: br i1 [[C]], label %[[PRED_STORE_IF2:.*]], label %[[PRED_STORE_CONTINUE1]]
; UNROLL: [[PRED_STORE_IF2]]:
; UNROLL-NEXT: store i8 [[TMP3]], ptr [[TMP1]], align 1
+; UNROLL-NEXT: store i8 [[TMP4]], ptr [[TMP2]], align 1
; UNROLL-NEXT: br label %[[PRED_STORE_CONTINUE1]]
; UNROLL: [[PRED_STORE_CONTINUE1]]:
-; UNROLL-NEXT: br i1 [[C]], label %[[PRED_STORE_IF1:.*]], label %[[PRED_STORE_CONTINUE2:.*]]
-; UNROLL: [[PRED_STORE_IF1]]:
-; UNROLL-NEXT: store i8 [[TMP4]], ptr [[TMP2]], align 1
-; UNROLL-NEXT: br label %[[PRED_STORE_CONTINUE2]]
-; UNROLL: [[PRED_STORE_CONTINUE2]]:
-; UNROLL-NEXT: br label %[[PRED_STORE_CONTINUE]]
-; UNROLL: [[PRED_STORE_CONTINUE]]:
; UNROLL-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 2
; UNROLL-NEXT: [[TMP5:%.*]] = icmp eq i64 [[INDEX_NEXT]], 1000
; UNROLL-NEXT: br i1 [[TMP5]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP6:![0-9]+]]
@@ -588,7 +551,7 @@ define void @minimal_bit_widths_with_aliasing_store(i1 %c, ptr %ptr) {
; UNROLL-NOSIMPLIFY: [[VECTOR_PH]]:
; UNROLL-NOSIMPLIFY-NEXT: br label %[[VECTOR_BODY:.*]]
; UNROLL-NOSIMPLIFY: [[VECTOR_BODY]]:
-; UNROLL-NOSIMPLIFY-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[PRED_STORE_CONTINUE:.*]] ]
+; UNROLL-NOSIMPLIFY-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[PRED_STORE_CONTINUE1:.*]] ]
; UNROLL-NOSIMPLIFY-NEXT: [[TMP0:%.*]] = add i64 [[INDEX]], 1
; UNROLL-NOSIMPLIFY-NEXT: [[TMP1:%.*]] = getelementptr i8, ptr [[PTR]], i64 [[INDEX]]
; UNROLL-NOSIMPLIFY-NEXT: [[TMP2:%.*]] = getelementptr i8, ptr [[PTR]], i64 [[TMP0]]
@@ -596,20 +559,12 @@ define void @minimal_bit_widths_with_aliasing_store(i1 %c, ptr %ptr) {
; UNROLL-NOSIMPLIFY-NEXT: [[TMP4:%.*]] = load i8, ptr [[TMP2]], align 1
; UNROLL-NOSIMPLIFY-NEXT: store i8 0, ptr [[TMP1]], align 1
; UNROLL-NOSIMPLIFY-NEXT: store i8 0, ptr [[TMP2]], align 1
-; UNROLL-NOSIMPLIFY-NEXT: br i1 [[C]], label %[[PRED_STORE_IF:.*]], label %[[PRED_STORE_CONTINUE]]
-; UNROLL-NOSIMPLIFY: [[PRED_STORE_IF]]:
-; UNROLL-NOSIMPLIFY-NEXT: br i1 [[C]], label %[[PRED_STORE_IF2:.*]], label %[[PRED_STORE_CONTINUE1:.*]]
+; UNROLL-NOSIMPLIFY-NEXT: br i1 [[C]], label %[[PRED_STORE_IF2:.*]], label %[[PRED_STORE_CONTINUE1]]
; UNROLL-NOSIMPLIFY: [[PRED_STORE_IF2]]:
; UNROLL-NOSIMPLIFY-NEXT: store i8 [[TMP3]], ptr [[TMP1]], align 1
+; UNROLL-NOSIMPLIFY-NEXT: store i8 [[TMP4]], ptr [[TMP2]], align 1
; UNROLL-NOSIMPLIFY-NEXT: br label %[[PRED_STORE_CONTINUE1]]
; UNROLL-NOSIMPLIFY: [[PRED_STORE_CONTINUE1]]:
-; UNROLL-NOSIMPLIFY-NEXT: br i1 [[C]], label %[[PRED_STORE_IF1:.*]], label %[[PRED_STORE_CONTINUE2:.*]]
-; UNROLL-NOSIMPLIFY: [[PRED_STORE_IF1]]:
-; UNROLL-NOSIMPLIFY-NEXT: store i8 [[TMP4]], ptr [[TMP2]], align 1
-; UNROLL-NOSIMPLIFY-NEXT: br label %[[PRED_STORE_CONTINUE2]]
-; UNROLL-NOSIMPLIFY: [[PRED_STORE_CONTINUE2]]:
-; UNROLL-NOSIMPLIFY-NEXT: br label %[[PRED_STORE_CONTINUE]]
-; UNROLL-NOSIMPLIFY: [[PRED_STORE_CONTINUE]]:
; UNROLL-NOSIMPLIFY-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 2
; UNROLL-NOSIMPLIFY-NEXT: [[TMP5:%.*]] = icmp eq i64 [[INDEX_NEXT]], 1000
; UNROLL-NOSIMPLIFY-NEXT: br i1 [[TMP5]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP6:![0-9]+]]
@@ -625,28 +580,15 @@ define void @minimal_bit_widths_with_aliasing_store(i1 %c, ptr %ptr) {
; VEC: [[VECTOR_PH]]:
; VEC-NEXT: br label %[[VECTOR_BODY:.*]]
; VEC: [[VECTOR_BODY]]:
-; VEC-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[PRED_STORE_CONTINUE:.*]] ]
+; VEC-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[PRED_STORE_CONTINUE2:.*]] ]
; VEC-NEXT: [[TMP0:%.*]] = getelementptr i8, ptr [[PTR]], i64 [[INDEX]]
; VEC-NEXT: [[WIDE_LOAD:%.*]] = load <2 x i8>, ptr [[TMP0]], align 1
; VEC-NEXT: store <2 x i8> zeroinitializer, ptr [[TMP0]], align 1
-; VEC-NEXT: br i1 [[C]], label %[[PRED_STORE_IF:.*]], label %[[PRED_STORE_CONTINUE]]
-; VEC: [[PRED_STORE_IF]]:
-; VEC-NEXT: br i1 [[C]], label %[[PRED_STORE_IF2:.*]], label %[[PRED_STORE_CONTINUE1:.*]]
-; VEC: [[PRED_STORE_IF2]]:
-; VEC-NEXT: [[TMP1:%.*]] = extractelement <2 x i8> [[WIDE_LOAD]], i64 0
-; VEC-NEXT: store i8 [[TMP1]], ptr [[TMP0]], align 1
-; VEC-NEXT: br label %[[PRED_STORE_CONTINUE1]]
-; VEC: [[PRED_STORE_CONTINUE1]]:
-; VEC-NEXT: br i1 [[C]], label %[[PRED_STORE_IF1:.*]], label %[[PRED_STORE_CONTINUE2:.*]]
+; VEC-NEXT: br i1 [[C]], label %[[PRED_STORE_IF1:.*]], label %[[PRED_STORE_CONTINUE2]]
; VEC: [[PRED_STORE_IF1]]:
-; VEC-NEXT: [[TMP2:%.*]] = add i64 [[INDEX]], 1
-; VEC-NEXT: [[TMP3:%.*]] = getelementptr i8, ptr [[PTR]], i64 [[TMP2]]
-; VEC-NEXT: [[TMP4:%.*]] = extractelement <2 x i8> [[WIDE_LOAD]], i64 1
-; VEC-NEXT: store i8 [[TMP4]], ptr [[TMP3]], align 1
+; VEC-NEXT: store <2 x i8> [[WIDE_LOAD]], ptr [[TMP0]], align 1
; VEC-NEXT: br label %[[PRED_STORE_CONTINUE2]]
; VEC: [[PRED_STORE_CONTINUE2]]:
-; VEC-NEXT: br label %[[PRED_STORE_CONTINUE]]
-; VEC: [[PRED_STORE_CONTINUE]]:
; VEC-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 2
; VEC-NEXT: [[TMP5:%.*]] = icmp eq i64 [[INDEX_NEXT]], 1000
; VEC-NEXT: br i1 [[TMP5]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP6:![0-9]+]]
@@ -685,31 +627,21 @@ for.end:
define void @sdiv_with_uniform_ops(i16 %0, i1 %c, ptr %dst) {
; UNROLL-LABEL: define void @sdiv_with_uniform_ops(
; UNROLL-SAME: i16 [[TMP0:%.*]], i1 [[C:%.*]], ptr [[DST:%.*]]) {
-; UNROLL-NEXT: [[ENTRY:.*:]]
-; UNROLL-NEXT: br label %[[VECTOR_PH:.*]]
-; UNROLL: [[VECTOR_PH]]:
+; UNROLL-NEXT: [[VECTOR_PH:.*:]]
; UNROLL-NEXT: br label %[[VECTOR_BODY:.*]]
; UNROLL: [[VECTOR_BODY]]:
-; UNROLL-NEXT: [[INDEX:%.*]] = phi i32 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[PRED_STORE_CONTINUE:.*]] ]
-; UNROLL-NEXT: br i1 [[C]], label %[[PRED_STORE_IF:.*]], label %[[PRED_STORE_CONTINUE]]
-; UNROLL: [[PRED_STORE_IF]]:
-; UNROLL-NEXT: br i1 [[C]], label %[[PRED_STORE_IF2:.*]], label %[[PRED_STORE_CONTINUE1:.*]]
-; UNROLL: [[PRED_STORE_IF2]]:
; UNROLL-NEXT: [[TMP1:%.*]] = sdiv i16 10, [[TMP0]]
-; UNROLL-NEXT: store i16 [[TMP1]], ptr [[DST]], align 1
-; UNROLL-NEXT: br label %[[PRED_STORE_CONTINUE1]]
+; UNROLL-NEXT: br label %[[PRED_STORE_CONTINUE1:.*]]
; UNROLL: [[PRED_STORE_CONTINUE1]]:
-; UNROLL-NEXT: br i1 [[C]], label %[[PRED_STORE_IF1:.*]], label %[[PRED_STORE_CONTINUE2:.*]]
+; UNROLL-NEXT: [[INDEX:%.*]] = phi i32 [ 0, %[[VECTOR_BODY]] ], [ [[INDEX_NEXT:%.*]], %[[PRED_STORE_CONTINUE2:.*]] ]
+; UNROLL-NEXT: br i1 [[C]], label %[[PRED_STORE_IF1:.*]], label %[[PRED_STORE_CONTINUE2]]
; UNROLL: [[PRED_STORE_IF1]]:
-; UNROLL-NEXT: [[TMP2:%.*]] = sdiv i16 10, [[TMP0]]
-; UNROLL-NEXT: store i16 [[TMP2]], ptr [[DST]], align 1
+; UNROLL-NEXT: store i16 [[TMP1]], ptr [[DST]], align 1
; UNROLL-NEXT: br label %[[PRED_STORE_CONTINUE2]]
; UNROLL: [[PRED_STORE_CONTINUE2]]:
-; UNROLL-NEXT: br label %[[PRED_STORE_CONTINUE]]
-; UNROLL: [[PRED_STORE_CONTINUE]]:
; UNROLL-NEXT: [[INDEX_NEXT]] = add nuw i32 [[INDEX]], 2
; UNROLL-NEXT: [[TMP3:%.*]] = icmp eq i32 [[INDEX_NEXT]], 98
-; UNROLL-NEXT: br i1 [[TMP3]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP7:![0-9]+]]
+; UNROLL-NEXT: br i1 [[TMP3]], label %[[MIDDLE_BLOCK:.*]], label %[[PRED_STORE_CONTINUE1]], !llvm.loop [[LOOP7:![0-9]+]]
; UNROLL: [[MIDDLE_BLOCK]]:
; UNROLL-NEXT: br label %[[SCALAR_PH:.*]]
; UNROLL: [[SCALAR_PH]]:
@@ -730,31 +662,21 @@ define void @sdiv_with_uniform_ops(i16 %0, i1 %c, ptr %dst) {
;
; UNROLL-NOSIMPLIFY-LABEL: define void @sdiv_with_uniform_ops(
; UNROLL-NOSIMPLIFY-SAME: i16 [[TMP0:%.*]], i1 [[C:%.*]], ptr [[DST:%.*]]) {
-; UNROLL-NOSIMPLIFY-NEXT: [[ENTRY:.*:]]
-; UNROLL-NOSIMPLIFY-NEXT: br label %[[VECTOR_PH:.*]]
-; UNROLL-NOSIMPLIFY: [[VECTOR_PH]]:
+; UNROLL-NOSIMPLIFY-NEXT: [[VECTOR_PH:.*:]]
; UNROLL-NOSIMPLIFY-NEXT: br label %[[VECTOR_BODY:.*]]
; UNROLL-NOSIMPLIFY: [[VECTOR_BODY]]:
-; UNROLL-NOSIMPLIFY-NEXT: [[INDEX:%.*]] = phi i32 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[PRED_STORE_CONTINUE:.*]] ]
-; UNROLL-NOSIMPLIFY-NEXT: br i1 [[C]], label %[[PRED_STORE_IF:.*]], label %[[PRED_STORE_CONTINUE]]
-; UNROLL-NOSIMPLIFY: [[PRED_STORE_IF]]:
-; UNROLL-NOSIMPLIFY-NEXT: br i1 [[C]], label %[[PRED_STORE_IF2:.*]], label %[[PRED_STORE_CONTINUE1:.*]]
-; UNROLL-NOSIMPLIFY: [[PRED_STORE_IF2]]:
; UNROLL-NOSIMPLIFY-NEXT: [[TMP1:%.*]] = sdiv i16 10, [[TMP0]]
-; UNROLL-NOSIMPLIFY-NEXT: store i16 [[TMP1]], ptr [[DST]], align 1
-; UNROLL-NOSIMPLIFY-NEXT: br label %[[PRED_STORE_CONTINUE1]]
+; UNROLL-NOSIMPLIFY-NEXT: br label %[[PRED_STORE_CONTINUE1:.*]]
; UNROLL-NOSIMPLIFY: [[PRED_STORE_CONTINUE1]]:
-; UNROLL-NOSIMPLIFY-NEXT: br i1 [[C]], label %[[PRED_STORE_IF1:.*]], label %[[PRED_STORE_CONTINUE2:.*]]
+; UNROLL-NOSIMPLIFY-NEXT: [[INDEX:%.*]] = phi i32 [ 0, %[[VECTOR_BODY]] ], [ [[INDEX_NEXT:%.*]], %[[PRED_STORE_CONTINUE2:.*]] ]
+; UNROLL-NOSIMPLIFY-NEXT: br i1 [[C]], label %[[PRED_STORE_IF1:.*]], label %[[PRED_STORE_CONTINUE2]]
; UNROLL-NOSIMPLIFY: [[PRED_STORE_IF1]]:
-; UNROLL-NOSIMPLIFY-NEXT: [[TMP2:%.*]] = sdiv i16 10, [[TMP0]]
-; UNROLL-NOSIMPLIFY-NEXT: store i16 [[TMP2]], ptr [[DST]], align 1
+; UNROLL-NOSIMPLIFY-NEXT: store i16 [[TMP1]], ptr [[DST]], align 1
; UNROLL-NOSIMPLIFY-NEXT: br label %[[PRED_STORE_CONTINUE2]]
; UNROLL-NOSIMPLIFY: [[PRED_STORE_CONTINUE2]]:
-; UNROLL-NOSIMPLIFY-NEXT: br label %[[PRED_STORE_CONTINUE]]
-; UNROLL-NOSIMPLIFY: [[PRED_STORE_CONTINUE]]:
; UNROLL-NOSIMPLIFY-NEXT: [[INDEX_NEXT]] = add nuw i32 [[INDEX]], 2
; UNROLL-NOSIMPLIFY-NEXT: [[TMP3:%.*]] = icmp eq i32 [[INDEX_NEXT]], 98
-; UNROLL-NOSIMPLIFY-NEXT: br i1 [[TMP3]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP7:![0-9]+]]
+; UNROLL-NOSIMPLIFY-NEXT: br i1 [[TMP3]], label %[[MIDDLE_BLOCK:.*]], label %[[PRED_STORE_CONTINUE1]], !llvm.loop [[LOOP7:![0-9]+]]
; UNROLL-NOSIMPLIFY: [[MIDDLE_BLOCK]]:
; UNROLL-NOSIMPLIFY-NEXT: br label %[[SCALAR_PH:.*]]
; UNROLL-NOSIMPLIFY: [[SCALAR_PH]]:
@@ -775,36 +697,21 @@ define void @sdiv_with_uniform_ops(i16 %0, i1 %c, ptr %dst) {
;
; VEC-LABEL: define void @sdiv_with_uniform_ops(
; VEC-SAME: i16 [[TMP0:%.*]], i1 [[C:%.*]], ptr [[DST:%.*]]) {
-; VEC-NEXT: [[ENTRY:.*:]]
-; VEC-NEXT: br label %[[VECTOR_PH:.*]]
-; VEC: [[VECTOR_PH]]:
-; VEC-NEXT: [[BROADCAST_SPLATINSERT:%.*]] = insertelement <2 x i16> poison, i16 [[TMP0]], i64 0
-; VEC-NEXT: [[BROADCAST_SPLAT:%.*]] = shufflevector <2 x i16> [[BROADCAST_SPLATINSERT]], <2 x i16> poison, <2 x i32> zeroinitializer
-; VEC-NEXT: [[BROADCAST_SPLATINSERT1:%.*]] = insertelement <2 x i1> poison, i1 [[C]], i64 0
-; VEC-NEXT: [[BROADCAST_SPLAT2:%.*]] = shufflevector <2 x i1> [[BROADCAST_SPLATINSERT1]], <2 x i1> poison, <2 x i32> zeroinitializer
-; VEC-NEXT: [[TMP1:%.*]] = call <2 x i16> @llvm.masked.sdiv.v2i16(<2 x i16> splat (i16 10), <2 x i16> [[BROADCAST_SPLAT]], <2 x i1> [[BROADCAST_SPLAT2]])
+; VEC-NEXT: [[VECTOR_PH:.*:]]
; VEC-NEXT: br label %[[VECTOR_BODY:.*]]
; VEC: [[VECTOR_BODY]]:
-; VEC-NEXT: [[INDEX:%.*]] = phi i32 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[PRED_STORE_CONTINUE:.*]] ]
-; VEC-NEXT: br i1 [[C]], label %[[PRED_STORE_IF:.*]], label %[[PRED_STORE_CONTINUE]]
-; VEC: [[PRED_STORE_IF]]:
-; VEC-NEXT: br i1 [[C]], label %[[PRED_STORE_IF1:.*]], label %[[PRED_STORE_CONTINUE1:.*]]
-; VEC: [[PRED_STORE_IF1]]:
-; VEC-NEXT: [[TMP2:%.*]] = extractelement <2 x i16> [[TMP1]], i64 0
-; VEC-NEXT: store i16 [[TMP2]], ptr [[DST]], align 1
-; VEC-NEXT: br label %[[PRED_STORE_CONTINUE1]]
+; VEC-NEXT: [[TMP3:%.*]] = sdiv i16 10, [[TMP0]]
+; VEC-NEXT: br label %[[PRED_STORE_CONTINUE1:.*]]
; VEC: [[PRED_STORE_CONTINUE1]]:
-; VEC-NEXT: br i1 [[C]], label %[[PRED_STORE_IF3:.*]], label %[[PRED_STORE_CONTINUE4:.*]]
+; VEC-NEXT: [[INDEX:%.*]] = phi i32 [ 0, %[[VECTOR_BODY]] ], [ [[INDEX_NEXT:%.*]], %[[PRED_STORE_CONTINUE4:.*]] ]
+; VEC-NEXT: br i1 [[C]], label %[[PRED_STORE_IF3:.*]], label %[[PRED_STORE_CONTINUE4]]
; VEC: [[PRED_STORE_IF3]]:
-; VEC-NEXT: [[TMP3:%.*]] = extractelement <2 x i16> [[TMP1]], i64 1
; VEC-NEXT: store i16 [[TMP3]], ptr [[DST]], align 1
; VEC-NEXT: br label %[[PRED_STORE_CONTINUE4]]
; VEC: [[PRED_STORE_CONTINUE4]]:
-; VEC-NEXT: br label %[[PRED_STORE_CONTINUE]]
-; VEC: [[PRED_STORE_CONTINUE]]:
; VEC-NEXT: [[INDEX_NEXT]] = add nuw i32 [[INDEX]], 2
; VEC-NEXT: [[TMP4:%.*]] = icmp eq i32 [[INDEX_NEXT]], 98
-; VEC-NEXT: br i1 [[TMP4]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP7:![0-9]+]]
+; VEC-NEXT: br i1 [[TMP4]], label %[[MIDDLE_BLOCK:.*]], label %[[PRED_STORE_CONTINUE1]], !llvm.loop [[LOOP7:![0-9]+]]
; VEC: [[MIDDLE_BLOCK]]:
; VEC-NEXT: br label %[[SCALAR_PH:.*]]
; VEC: [[SCALAR_PH]]:
diff --git a/llvm/test/Transforms/LoopVectorize/invariant-store-vectorization-2.ll b/llvm/test/Transforms/LoopVectorize/invariant-store-vectorization-2.ll
index 9aac5503cf92b..f713713cff44f 100644
--- a/llvm/test/Transforms/LoopVectorize/invariant-store-vectorization-2.ll
+++ b/llvm/test/Transforms/LoopVectorize/invariant-store-vectorization-2.ll
@@ -141,8 +141,6 @@ define void @inv_val_store_to_inv_address_conditional_inv(ptr %a, i64 %n, ptr %b
; CHECK-NEXT: [[N_VEC:%.*]] = sub i64 [[SMAX2]], [[N_MOD_VF]]
; CHECK-NEXT: [[BROADCAST_SPLATINSERT:%.*]] = insertelement <4 x i32> poison, i32 [[NTRUNC]], i64 0
; CHECK-NEXT: [[BROADCAST_SPLAT:%.*]] = shufflevector <4 x i32> [[BROADCAST_SPLATINSERT]], <4 x i32> poison, <4 x i32> zeroinitializer
-; CHECK-NEXT: [[BROADCAST_SPLATINSERT2:%.*]] = insertelement <4 x i1> poison, i1 [[CMP]], i64 0
-; CHECK-NEXT: [[BROADCAST_SPLAT3:%.*]] = shufflevector <4 x i1> [[BROADCAST_SPLATINSERT2]], <4 x i1> poison, <4 x i32> zeroinitializer
; CHECK-NEXT: [[BROADCAST_SPLATINSERT4:%.*]] = insertelement <4 x i32> poison, i32 [[K]], i64 0
; CHECK-NEXT: [[BROADCAST_SPLAT5:%.*]] = shufflevector <4 x i32> [[BROADCAST_SPLATINSERT4]], <4 x i32> poison, <4 x i32> zeroinitializer
; CHECK-NEXT: br label [[VECTOR_BODY:%.*]]
@@ -151,12 +149,12 @@ define void @inv_val_store_to_inv_address_conditional_inv(ptr %a, i64 %n, ptr %b
; CHECK-NEXT: [[TMP2:%.*]] = getelementptr inbounds i32, ptr [[B]], i64 [[INDEX]]
; CHECK-NEXT: store <4 x i32> [[BROADCAST_SPLAT]], ptr [[TMP2]], align 4, !alias.scope [[META9:![0-9]+]], !noalias [[META12:![0-9]+]]
; CHECK-NEXT: br i1 [[CMP]], label [[COND_STORE7:%.*]], label [[COND_STORE_K6:%.*]]
-; CHECK: cond_store_k6:
+; CHECK: cond_store_k4:
; CHECK-NEXT: br label [[LATCH8]]
-; CHECK: cond_store7:
+; CHECK: cond_store5:
; CHECK-NEXT: br label [[LATCH8]]
-; CHECK: latch8:
-; CHECK-NEXT: [[TMP6:%.*]] = phi <4 x i1> [ zeroinitializer, [[COND_STORE_K6]] ], [ [[BROADCAST_SPLAT3]], [[COND_STORE7]] ]
+; CHECK: latch6:
+; CHECK-NEXT: [[TMP6:%.*]] = phi <4 x i1> [ zeroinitializer, [[COND_STORE_K6]] ], [ splat (i1 true), [[COND_STORE7]] ]
; CHECK-NEXT: [[PREDPHI:%.*]] = select <4 x i1> [[TMP6]], <4 x i32> [[BROADCAST_SPLAT]], <4 x i32> [[BROADCAST_SPLAT5]]
; CHECK-NEXT: [[TMP5:%.*]] = extractelement <4 x i32> [[PREDPHI]], i64 3
; CHECK-NEXT: store i32 [[TMP5]], ptr [[A]], align 4, !alias.scope [[META12]]
diff --git a/llvm/test/Transforms/LoopVectorize/multiple-result-intrinsics.ll b/llvm/test/Transforms/LoopVectorize/multiple-result-intrinsics.ll
index 6a57f1ad89ae9..97a61fdbc15d8 100644
--- a/llvm/test/Transforms/LoopVectorize/multiple-result-intrinsics.ll
+++ b/llvm/test/Transforms/LoopVectorize/multiple-result-intrinsics.ll
@@ -827,18 +827,8 @@ define void @sincos_predicated_store(i1 %c, ptr noalias %src, ptr noalias %dst,
; CHECK: store <2 x double> zeroinitializer, ptr [[TMP3:%.*]], align 8
; CHECK: [[IF1:.*:]]
; CHECK: [[TMP2:%.*]] = extractvalue { <2 x double>, <2 x double> } [[TMP0]], 1
-; CHECK: br i1 [[TMP1:%.*]], label %[[PRED_STORE_IF:.*]], label %[[PRED_STORE_CONTINUE:.*]]
-; CHECK: [[PRED_STORE_IF]]:
-; CHECK: [[TMP4:%.*]] = extractelement <2 x double> [[TMP2]], i64 0
-; CHECK: store double [[TMP4]], ptr [[COS_DST]], align 8
-; CHECK: br label %[[PRED_STORE_CONTINUE]]
-; CHECK: [[PRED_STORE_CONTINUE]]:
-; CHECK: br i1 [[TMP1]], label %[[PRED_STORE_IF1:.*]], label %[[PRED_STORE_CONTINUE2:.*]]
-; CHECK: [[PRED_STORE_IF1]]:
; CHECK: [[TMP5:%.*]] = extractelement <2 x double> [[TMP2]], i64 1
; CHECK: store double [[TMP5]], ptr [[COS_DST]], align 8
-; CHECK: br label %[[PRED_STORE_CONTINUE2]]
-; CHECK: [[PRED_STORE_CONTINUE2]]:
; CHECK: [[MIDDLE_BLOCK:.*:]]
; CHECK: [[EXIT:.*:]]
; CHECK: [[EXIT1:.*:]]
diff --git a/llvm/test/Transforms/LoopVectorize/phi-with-fastflags.ll b/llvm/test/Transforms/LoopVectorize/phi-with-fastflags.ll
index 625c9e323ec6f..05e84042d5b24 100644
--- a/llvm/test/Transforms/LoopVectorize/phi-with-fastflags.ll
+++ b/llvm/test/Transforms/LoopVectorize/phi-with-fastflags.ll
@@ -7,8 +7,6 @@ define void @f(ptr noalias %p, i1 %c) {
; CHECK-NEXT: [[ENTRY:.*:]]
; CHECK-NEXT: br label %[[VECTOR_PH:.*]]
; CHECK: [[VECTOR_PH]]:
-; CHECK-NEXT: [[BROADCAST_SPLATINSERT:%.*]] = insertelement <4 x i1> poison, i1 [[C]], i64 0
-; CHECK-NEXT: [[BROADCAST_SPLAT:%.*]] = shufflevector <4 x i1> [[BROADCAST_SPLATINSERT]], <4 x i1> poison, <4 x i32> zeroinitializer
; CHECK-NEXT: br label %[[VECTOR_BODY:.*]]
; CHECK: [[VECTOR_BODY]]:
; CHECK-NEXT: [[INDEX:%.*]] = phi i32 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[LATCH2:.*]] ]
@@ -18,7 +16,7 @@ define void @f(ptr noalias %p, i1 %c) {
; CHECK: [[THEN1]]:
; CHECK-NEXT: br label %[[LATCH2]]
; CHECK: [[LATCH2]]:
-; CHECK-NEXT: [[TMP2:%.*]] = phi <4 x i1> [ zeroinitializer, %[[VECTOR_BODY]] ], [ [[BROADCAST_SPLAT]], %[[THEN1]] ]
+; CHECK-NEXT: [[TMP2:%.*]] = phi <4 x i1> [ zeroinitializer, %[[VECTOR_BODY]] ], [ splat (i1 true), %[[THEN1]] ]
; CHECK-NEXT: [[PREDPHI:%.*]] = select fast <4 x i1> [[TMP2]], <4 x float> zeroinitializer, <4 x float> [[WIDE_LOAD]]
; CHECK-NEXT: store <4 x float> [[PREDPHI]], ptr [[TMP0]], align 4
; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i32 [[INDEX]], 4
diff --git a/llvm/test/Transforms/LoopVectorize/pr37248.ll b/llvm/test/Transforms/LoopVectorize/pr37248.ll
index 54109d418f5ba..9d86032bd27bc 100644
--- a/llvm/test/Transforms/LoopVectorize/pr37248.ll
+++ b/llvm/test/Transforms/LoopVectorize/pr37248.ll
@@ -35,23 +35,13 @@ define void @f1(ptr noalias %b, i1 %c, i32 %start) {
; CHECK-NEXT: [[N_MOD_VF:%.*]] = and i32 [[TMP1]], 1
; CHECK-NEXT: [[N_VEC:%.*]] = sub i32 [[TMP1]], [[N_MOD_VF]]
; CHECK-NEXT: [[TMP10:%.*]] = sub i32 [[START]], [[N_VEC]]
-; CHECK-NEXT: [[TMP12:%.*]] = xor i1 [[C]], true
; CHECK-NEXT: br label %[[VECTOR_BODY:.*]]
; CHECK: [[VECTOR_BODY]]:
; CHECK-NEXT: [[INDEX:%.*]] = phi i32 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[LOOP_LATCH4:.*]] ]
; CHECK-NEXT: [[OFFSET_IDX:%.*]] = sub i32 [[START]], [[INDEX]]
; CHECK-NEXT: br i1 [[C]], label %[[LOOP_LATCH4]], label %[[ELSE1:.*]]
; CHECK: [[ELSE1]]:
-; CHECK-NEXT: br i1 [[TMP12]], label %[[PRED_STORE_IF:.*]], label %[[PRED_STORE_CONTINUE:.*]]
-; CHECK: [[PRED_STORE_IF]]:
; CHECK-NEXT: store i32 10, ptr [[B]], align 1
-; CHECK-NEXT: br label %[[PRED_STORE_CONTINUE]]
-; CHECK: [[PRED_STORE_CONTINUE]]:
-; CHECK-NEXT: br i1 [[TMP12]], label %[[PRED_STORE_IF2:.*]], label %[[PRED_STORE_CONTINUE3:.*]]
-; CHECK: [[PRED_STORE_IF2]]:
-; CHECK-NEXT: store i32 10, ptr [[B]], align 1
-; CHECK-NEXT: br label %[[PRED_STORE_CONTINUE3]]
-; CHECK: [[PRED_STORE_CONTINUE3]]:
; CHECK-NEXT: br label %[[LOOP_LATCH4]]
; CHECK: [[LOOP_LATCH4]]:
; CHECK-NEXT: [[TMP13:%.*]] = trunc i32 [[OFFSET_IDX]] to i16
diff --git a/llvm/test/Transforms/LoopVectorize/pr44488-predication.ll b/llvm/test/Transforms/LoopVectorize/pr44488-predication.ll
index 4a10f8a380e93..6969eb5ddb3b0 100644
--- a/llvm/test/Transforms/LoopVectorize/pr44488-predication.ll
+++ b/llvm/test/Transforms/LoopVectorize/pr44488-predication.ll
@@ -18,31 +18,15 @@ define i16 @test_true_and_false_branch_equal() {
; CHECK-NEXT: [[INDEX:%.*]] = phi i32 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[PRED_SREM_CONTINUE2:%.*]] ]
; CHECK-NEXT: [[TMP0:%.*]] = load i16, ptr @v_38, align 1
; CHECK-NEXT: [[TMP1:%.*]] = icmp eq i16 [[TMP0]], 0
-; CHECK-NEXT: [[BROADCAST_SPLATINSERT:%.*]] = insertelement <2 x i1> poison, i1 [[TMP1]], i64 0
-; CHECK-NEXT: [[BROADCAST_SPLAT:%.*]] = shufflevector <2 x i1> [[BROADCAST_SPLATINSERT]], <2 x i1> poison, <2 x i32> zeroinitializer
; CHECK-NEXT: br i1 [[TMP1]], label [[PRED_SREM_CONTINUE2]], label [[COND_FALSE41:%.*]]
; CHECK: cond.false41:
-; CHECK-NEXT: [[TMP2:%.*]] = xor <2 x i1> [[BROADCAST_SPLAT]], splat (i1 true)
-; CHECK-NEXT: [[TMP3:%.*]] = extractelement <2 x i1> [[TMP2]], i64 0
-; CHECK-NEXT: br i1 [[TMP3]], label [[PRED_SREM_IF:%.*]], label [[PRED_SREM_CONTINUE:%.*]]
-; CHECK: pred.srem.if:
; CHECK-NEXT: [[TMP4:%.*]] = srem i16 5786, [[TMP0]]
; CHECK-NEXT: [[TMP5:%.*]] = insertelement <2 x i16> poison, i16 [[TMP4]], i64 0
-; CHECK-NEXT: br label [[PRED_SREM_CONTINUE]]
-; CHECK: pred.srem.continue:
-; CHECK-NEXT: [[TMP6:%.*]] = phi <2 x i16> [ poison, [[COND_FALSE41]] ], [ [[TMP5]], [[PRED_SREM_IF]] ]
-; CHECK-NEXT: [[TMP7:%.*]] = extractelement <2 x i1> [[TMP2]], i64 0
-; CHECK-NEXT: br i1 [[TMP7]], label [[PRED_SREM_IF1:%.*]], label [[PRED_SREM_CONTINUE3:%.*]]
-; CHECK: pred.srem.if2:
-; CHECK-NEXT: [[TMP8:%.*]] = srem i16 5786, [[TMP0]]
-; CHECK-NEXT: [[TMP9:%.*]] = insertelement <2 x i16> [[TMP6]], i16 [[TMP8]], i64 1
-; CHECK-NEXT: br label [[PRED_SREM_CONTINUE3]]
-; CHECK: pred.srem.continue3:
-; CHECK-NEXT: [[TMP10:%.*]] = phi <2 x i16> [ [[TMP6]], [[PRED_SREM_CONTINUE]] ], [ [[TMP9]], [[PRED_SREM_IF1]] ]
+; CHECK-NEXT: [[BROADCAST_SPLAT:%.*]] = shufflevector <2 x i16> [[TMP5]], <2 x i16> poison, <2 x i32> zeroinitializer
; CHECK-NEXT: br label [[PRED_SREM_CONTINUE2]]
-; CHECK: for.latch4:
-; CHECK-NEXT: [[TMP13:%.*]] = phi <2 x i16> [ poison, [[VECTOR_BODY]] ], [ [[TMP10]], [[PRED_SREM_CONTINUE3]] ]
-; CHECK-NEXT: [[TMP14:%.*]] = phi <2 x i1> [ zeroinitializer, [[VECTOR_BODY]] ], [ [[TMP2]], [[PRED_SREM_CONTINUE3]] ]
+; CHECK: for.latch2:
+; CHECK-NEXT: [[TMP13:%.*]] = phi <2 x i16> [ poison, [[VECTOR_BODY]] ], [ [[BROADCAST_SPLAT]], [[COND_FALSE41]] ]
+; CHECK-NEXT: [[TMP14:%.*]] = phi <2 x i1> [ zeroinitializer, [[VECTOR_BODY]] ], [ splat (i1 true), [[COND_FALSE41]] ]
; CHECK-NEXT: [[PREDPHI:%.*]] = select <2 x i1> [[TMP14]], <2 x i16> [[TMP13]], <2 x i16> splat (i16 5786)
; CHECK-NEXT: [[TMP11:%.*]] = extractelement <2 x i16> [[PREDPHI]], i64 1
; CHECK-NEXT: store i16 [[TMP11]], ptr @v_39, align 1
diff --git a/llvm/test/Transforms/LoopVectorize/pr45525.ll b/llvm/test/Transforms/LoopVectorize/pr45525.ll
index 73f0c8cc9c4a5..11f56f3c90d02 100644
--- a/llvm/test/Transforms/LoopVectorize/pr45525.ll
+++ b/llvm/test/Transforms/LoopVectorize/pr45525.ll
@@ -9,9 +9,6 @@ define void @main(i1 %cond, ptr %arr) {
; CHECK-NEXT: [[BB_0:.*:]]
; CHECK-NEXT: br label %[[VECTOR_PH:.*]]
; CHECK: [[VECTOR_PH]]:
-; CHECK-NEXT: [[BROADCAST_SPLATINSERT:%.*]] = insertelement <4 x i1> poison, i1 [[COND]], i64 0
-; CHECK-NEXT: [[BROADCAST_SPLAT:%.*]] = shufflevector <4 x i1> [[BROADCAST_SPLATINSERT]], <4 x i1> poison, <4 x i32> zeroinitializer
-; CHECK-NEXT: [[TMP0:%.*]] = xor <4 x i1> [[BROADCAST_SPLAT]], splat (i1 true)
; CHECK-NEXT: br label %[[VECTOR_BODY:.*]]
; CHECK: [[VECTOR_BODY]]:
; CHECK-NEXT: [[INDEX:%.*]] = phi i32 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[BB_32:.*]] ]
@@ -22,7 +19,7 @@ define void @main(i1 %cond, ptr %arr) {
; CHECK-NEXT: br label %[[BB_32]]
; CHECK: [[BB_32]]:
; CHECK-NEXT: [[TMP4:%.*]] = phi <4 x i32> [ poison, %[[VECTOR_BODY]] ], [ [[TMP5]], %[[BB_21]] ]
-; CHECK-NEXT: [[TMP3:%.*]] = phi <4 x i1> [ zeroinitializer, %[[VECTOR_BODY]] ], [ [[TMP0]], %[[BB_21]] ]
+; CHECK-NEXT: [[TMP3:%.*]] = phi <4 x i1> [ zeroinitializer, %[[VECTOR_BODY]] ], [ splat (i1 true), %[[BB_21]] ]
; CHECK-NEXT: [[PREDPHI:%.*]] = select <4 x i1> [[TMP3]], <4 x i32> [[TMP4]], <4 x i32> splat (i32 7)
; CHECK-NEXT: [[TMP1:%.*]] = getelementptr inbounds i32, ptr [[ARR]], i32 [[INDEX]]
; CHECK-NEXT: store <4 x i32> [[PREDPHI]], ptr [[TMP1]], align 4
diff --git a/llvm/test/Transforms/LoopVectorize/predicator.ll b/llvm/test/Transforms/LoopVectorize/predicator.ll
index 48b428458a1e6..3b6da5e6091ce 100644
--- a/llvm/test/Transforms/LoopVectorize/predicator.ll
+++ b/llvm/test/Transforms/LoopVectorize/predicator.ll
@@ -274,10 +274,6 @@ define void @simplifiable_blend(i1 %c1, i1 %c2, i1 %c3, i32 %x, i32 %y, ptr %p)
; CHECK-NEXT: [[ENTRY:.*:]]
; CHECK-NEXT: br label %[[VECTOR_PH:.*]]
; CHECK: [[VECTOR_PH]]:
-; CHECK-NEXT: [[TMP2:%.*]] = freeze i1 [[C2]]
-; CHECK-NEXT: [[TMP1:%.*]] = freeze i1 [[C3]]
-; CHECK-NEXT: [[BROADCAST_SPLATINSERT:%.*]] = insertelement <4 x i1> poison, i1 [[C1]], i64 0
-; CHECK-NEXT: [[BROADCAST_SPLAT:%.*]] = shufflevector <4 x i1> [[BROADCAST_SPLATINSERT]], <4 x i1> poison, <4 x i32> zeroinitializer
; CHECK-NEXT: [[BROADCAST_SPLATINSERT1:%.*]] = insertelement <4 x i32> poison, i32 [[X]], i64 0
; CHECK-NEXT: [[BROADCAST_SPLAT2:%.*]] = shufflevector <4 x i32> [[BROADCAST_SPLATINSERT1]], <4 x i32> poison, <4 x i32> zeroinitializer
; CHECK-NEXT: [[BROADCAST_SPLATINSERT3:%.*]] = insertelement <4 x i32> poison, i32 [[Y]], i64 0
@@ -287,19 +283,19 @@ define void @simplifiable_blend(i1 %c1, i1 %c2, i1 %c3, i32 %x, i32 %y, ptr %p)
; CHECK-NEXT: [[INDEX:%.*]] = phi i32 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[LATCH11:.*]] ]
; CHECK-NEXT: br i1 [[C1]], label %[[A8:.*]], label %[[B5:.*]]
; CHECK: [[B5]]:
-; CHECK-NEXT: br i1 [[TMP1]], label %[[F7:.*]], label %[[E6:.*]]
+; CHECK-NEXT: br i1 [[C3]], label %[[F7:.*]], label %[[E6:.*]]
; CHECK: [[E6]]:
; CHECK-NEXT: br label %[[F7]]
; CHECK: [[F7]]:
; CHECK-NEXT: br label %[[LATCH11]]
; CHECK: [[A8]]:
-; CHECK-NEXT: br i1 [[TMP2]], label %[[C10:.*]], label %[[D9:.*]]
+; CHECK-NEXT: br i1 [[C2]], label %[[C10:.*]], label %[[D9:.*]]
; CHECK: [[D9]]:
; CHECK-NEXT: br label %[[LATCH11]]
; CHECK: [[C10]]:
; CHECK-NEXT: br label %[[LATCH11]]
; CHECK: [[LATCH11]]:
-; CHECK-NEXT: [[TMP0:%.*]] = phi <4 x i1> [ zeroinitializer, %[[F7]] ], [ [[BROADCAST_SPLAT]], %[[D9]] ], [ [[BROADCAST_SPLAT]], %[[C10]] ]
+; CHECK-NEXT: [[TMP0:%.*]] = phi <4 x i1> [ zeroinitializer, %[[F7]] ], [ splat (i1 true), %[[D9]] ], [ splat (i1 true), %[[C10]] ]
; CHECK-NEXT: [[PREDPHI7:%.*]] = select <4 x i1> [[TMP0]], <4 x i32> [[BROADCAST_SPLAT2]], <4 x i32> [[BROADCAST_SPLAT4]]
; CHECK-NEXT: [[TMP3:%.*]] = getelementptr i32, ptr [[P]], i32 [[INDEX]]
; CHECK-NEXT: store <4 x i32> [[PREDPHI7]], ptr [[TMP3]], align 4
@@ -354,8 +350,6 @@ define void @outermost_uniform_branch(ptr %a, i1 %u0) {
; CHECK-NEXT: [[ENTRY:.*:]]
; CHECK-NEXT: br label %[[VECTOR_PH:.*]]
; CHECK: [[VECTOR_PH]]:
-; CHECK-NEXT: [[BROADCAST_SPLATINSERT:%.*]] = insertelement <4 x i1> poison, i1 [[U0]], i64 0
-; CHECK-NEXT: [[BROADCAST_SPLAT:%.*]] = shufflevector <4 x i1> [[BROADCAST_SPLATINSERT]], <4 x i1> poison, <4 x i32> zeroinitializer
; CHECK-NEXT: br label %[[VECTOR_BODY:.*]]
; CHECK: [[VECTOR_BODY]]:
; CHECK-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[BB42:.*]] ]
@@ -370,7 +364,7 @@ define void @outermost_uniform_branch(ptr %a, i1 %u0) {
; CHECK-NEXT: br label %[[BB42]]
; CHECK: [[BB42]]:
; CHECK-NEXT: [[TMP6:%.*]] = phi <4 x i64> [ poison, %[[VECTOR_BODY]] ], [ [[TMP4]], %[[MIDDLE_BLOCK]] ]
-; CHECK-NEXT: [[TMP8:%.*]] = phi <4 x i1> [ zeroinitializer, %[[VECTOR_BODY]] ], [ [[BROADCAST_SPLAT]], %[[MIDDLE_BLOCK]] ]
+; CHECK-NEXT: [[TMP8:%.*]] = phi <4 x i1> [ zeroinitializer, %[[VECTOR_BODY]] ], [ splat (i1 true), %[[MIDDLE_BLOCK]] ]
; CHECK-NEXT: [[PREDPHI1:%.*]] = select <4 x i1> [[TMP8]], <4 x i64> [[TMP6]], <4 x i64> [[VEC_IND]]
; CHECK-NEXT: [[TMP5:%.*]] = extractelement <4 x i64> [[PREDPHI1]], i64 3
; CHECK-NEXT: store i64 [[TMP5]], ptr [[A]], align 4
@@ -432,8 +426,6 @@ define void @outermost_uniform_branch_more_blocks(ptr %a, i1 %u0) {
; CHECK-NEXT: [[ENTRY:.*:]]
; CHECK-NEXT: br label %[[VECTOR_PH:.*]]
; CHECK: [[VECTOR_PH]]:
-; CHECK-NEXT: [[BROADCAST_SPLATINSERT:%.*]] = insertelement <4 x i1> poison, i1 [[U0]], i64 0
-; CHECK-NEXT: [[BROADCAST_SPLAT:%.*]] = shufflevector <4 x i1> [[BROADCAST_SPLATINSERT]], <4 x i1> poison, <4 x i32> zeroinitializer
; CHECK-NEXT: br label %[[VECTOR_BODY:.*]]
; CHECK: [[VECTOR_BODY]]:
; CHECK-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[BB63:.*]] ]
@@ -452,7 +444,7 @@ define void @outermost_uniform_branch_more_blocks(ptr %a, i1 %u0) {
; CHECK: [[BB63]]:
; CHECK-NEXT: [[TMP8:%.*]] = phi <4 x i64> [ [[TMP1]], %[[MIDDLE_BLOCK]] ], [ poison, %[[BB22]] ]
; CHECK-NEXT: [[TMP10:%.*]] = phi <4 x i64> [ poison, %[[MIDDLE_BLOCK]] ], [ [[TMP5]], %[[BB22]] ]
-; CHECK-NEXT: [[TMP7:%.*]] = phi <4 x i1> [ zeroinitializer, %[[MIDDLE_BLOCK]] ], [ [[BROADCAST_SPLAT]], %[[BB22]] ]
+; CHECK-NEXT: [[TMP7:%.*]] = phi <4 x i1> [ zeroinitializer, %[[MIDDLE_BLOCK]] ], [ splat (i1 true), %[[BB22]] ]
; CHECK-NEXT: [[PREDPHI1:%.*]] = select <4 x i1> [[TMP7]], <4 x i64> [[TMP10]], <4 x i64> [[TMP8]]
; CHECK-NEXT: [[TMP6:%.*]] = extractelement <4 x i64> [[PREDPHI1]], i64 3
; CHECK-NEXT: store i64 [[TMP6]], ptr [[A]], align 4
@@ -522,8 +514,6 @@ define void @uniform_branch_after_varying_branch(ptr %a, i1 %u1) {
; CHECK-NEXT: br label %[[VECTOR_PH:.*]]
; CHECK: [[VECTOR_PH]]:
; CHECK-NEXT: [[TMP0:%.*]] = freeze i1 [[U1]]
-; CHECK-NEXT: [[BROADCAST_SPLATINSERT:%.*]] = insertelement <4 x i1> poison, i1 [[U1]], i64 0
-; CHECK-NEXT: [[BROADCAST_SPLAT:%.*]] = shufflevector <4 x i1> [[BROADCAST_SPLATINSERT]], <4 x i1> poison, <4 x i32> zeroinitializer
; CHECK-NEXT: br label %[[VECTOR_BODY:.*]]
; CHECK: [[VECTOR_BODY]]:
; CHECK-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[BB32:.*]] ]
@@ -536,7 +526,7 @@ define void @uniform_branch_after_varying_branch(ptr %a, i1 %u1) {
; CHECK-NEXT: br label %[[BB32]]
; CHECK: [[BB32]]:
; CHECK-NEXT: [[TMP7:%.*]] = phi <4 x i64> [ poison, %[[VECTOR_BODY]] ], [ [[TMP3]], %[[BB21]] ]
-; CHECK-NEXT: [[TMP9:%.*]] = phi <4 x i1> [ zeroinitializer, %[[VECTOR_BODY]] ], [ [[BROADCAST_SPLAT]], %[[BB21]] ]
+; CHECK-NEXT: [[TMP9:%.*]] = phi <4 x i1> [ zeroinitializer, %[[VECTOR_BODY]] ], [ splat (i1 true), %[[BB21]] ]
; CHECK-NEXT: [[PREDPHI:%.*]] = select <4 x i1> [[TMP9]], <4 x i64> [[TMP7]], <4 x i64> [[TMP2]]
; CHECK-NEXT: [[TMP4:%.*]] = add <4 x i64> [[PREDPHI]], splat (i64 3)
; CHECK-NEXT: [[TMP5:%.*]] = add <4 x i64> [[VEC_IND]], splat (i64 4)
@@ -604,8 +594,6 @@ define void @uniform_branch_after_varying_branch_more_blocks(ptr %a, i1 %u1) {
; CHECK-NEXT: br label %[[VECTOR_PH:.*]]
; CHECK: [[VECTOR_PH]]:
; CHECK-NEXT: [[TMP0:%.*]] = freeze i1 [[U1]]
-; CHECK-NEXT: [[BROADCAST_SPLATINSERT:%.*]] = insertelement <4 x i1> poison, i1 [[U1]], i64 0
-; CHECK-NEXT: [[BROADCAST_SPLAT:%.*]] = shufflevector <4 x i1> [[BROADCAST_SPLATINSERT]], <4 x i1> poison, <4 x i32> zeroinitializer
; CHECK-NEXT: br label %[[VECTOR_BODY:.*]]
; CHECK: [[VECTOR_BODY]]:
; CHECK-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[BB43:.*]] ]
@@ -621,7 +609,7 @@ define void @uniform_branch_after_varying_branch_more_blocks(ptr %a, i1 %u1) {
; CHECK: [[BB43]]:
; CHECK-NEXT: [[TMP7:%.*]] = phi <4 x i64> [ [[TMP2]], %[[BB21]] ], [ poison, %[[BB32]] ]
; CHECK-NEXT: [[TMP8:%.*]] = phi <4 x i64> [ poison, %[[BB21]] ], [ [[TMP3]], %[[BB32]] ]
-; CHECK-NEXT: [[TMP10:%.*]] = phi <4 x i1> [ zeroinitializer, %[[BB21]] ], [ [[BROADCAST_SPLAT]], %[[BB32]] ]
+; CHECK-NEXT: [[TMP10:%.*]] = phi <4 x i1> [ zeroinitializer, %[[BB21]] ], [ splat (i1 true), %[[BB32]] ]
; CHECK-NEXT: [[PREDPHI:%.*]] = select <4 x i1> [[TMP10]], <4 x i64> [[TMP8]], <4 x i64> [[TMP7]]
; CHECK-NEXT: [[TMP4:%.*]] = add <4 x i64> [[PREDPHI]], splat (i64 4)
; CHECK-NEXT: [[TMP5:%.*]] = add <4 x i64> [[VEC_IND]], splat (i64 5)
@@ -694,8 +682,6 @@ define void @uniform_branch_after_varying_branch_more_blocks_mirrored(ptr %a, i1
; CHECK-NEXT: br label %[[VECTOR_PH:.*]]
; CHECK: [[VECTOR_PH]]:
; CHECK-NEXT: [[TMP0:%.*]] = freeze i1 [[U2]]
-; CHECK-NEXT: [[BROADCAST_SPLATINSERT:%.*]] = insertelement <4 x i1> poison, i1 [[U2]], i64 0
-; CHECK-NEXT: [[BROADCAST_SPLAT:%.*]] = shufflevector <4 x i1> [[BROADCAST_SPLATINSERT]], <4 x i1> poison, <4 x i32> zeroinitializer
; CHECK-NEXT: br label %[[VECTOR_BODY:.*]]
; CHECK: [[VECTOR_BODY]]:
; CHECK-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[BB53:.*]] ]
@@ -712,7 +698,7 @@ define void @uniform_branch_after_varying_branch_more_blocks_mirrored(ptr %a, i1
; CHECK: [[BB53]]:
; CHECK-NEXT: [[TMP7:%.*]] = phi <4 x i64> [ [[TMP3]], %[[BB31]] ], [ poison, %[[BB42]] ]
; CHECK-NEXT: [[TMP8:%.*]] = phi <4 x i64> [ poison, %[[BB31]] ], [ [[TMP4]], %[[BB42]] ]
-; CHECK-NEXT: [[TMP10:%.*]] = phi <4 x i1> [ zeroinitializer, %[[BB31]] ], [ [[BROADCAST_SPLAT]], %[[BB42]] ]
+; CHECK-NEXT: [[TMP10:%.*]] = phi <4 x i1> [ zeroinitializer, %[[BB31]] ], [ splat (i1 true), %[[BB42]] ]
; CHECK-NEXT: [[PREDPHI:%.*]] = select <4 x i1> [[TMP10]], <4 x i64> [[TMP8]], <4 x i64> [[TMP7]]
; CHECK-NEXT: [[TMP5:%.*]] = add <4 x i64> [[PREDPHI]], splat (i64 5)
; CHECK-NEXT: [[PREDPHI1:%.*]] = select <4 x i1> [[TMP1]], <4 x i64> [[TMP5]], <4 x i64> [[TMP2]]
@@ -1285,8 +1271,6 @@ define void @uniform_branch_shared_join_with_varying(ptr %a, i1 %u0) {
; CHECK-NEXT: [[ENTRY:.*:]]
; CHECK-NEXT: br label %[[VECTOR_PH:.*]]
; CHECK: [[VECTOR_PH]]:
-; CHECK-NEXT: [[BROADCAST_SPLATINSERT:%.*]] = insertelement <4 x i1> poison, i1 [[U0]], i64 0
-; CHECK-NEXT: [[BROADCAST_SPLAT:%.*]] = shufflevector <4 x i1> [[BROADCAST_SPLATINSERT]], <4 x i1> poison, <4 x i32> zeroinitializer
; CHECK-NEXT: br label %[[VECTOR_BODY:.*]]
; CHECK: [[VECTOR_BODY]]:
; CHECK-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[BB43:.*]] ]
@@ -1298,15 +1282,14 @@ define void @uniform_branch_shared_join_with_varying(ptr %a, i1 %u0) {
; CHECK: [[BB22]]:
; CHECK-NEXT: [[TMP2:%.*]] = add <4 x i64> [[VEC_IND]], splat (i64 2)
; CHECK-NEXT: [[TMP3:%.*]] = icmp sle <4 x i64> [[VEC_IND]], splat (i64 2)
-; CHECK-NEXT: [[TMP4:%.*]] = select <4 x i1> [[BROADCAST_SPLAT]], <4 x i1> [[TMP3]], <4 x i1> zeroinitializer
; CHECK-NEXT: [[TMP5:%.*]] = add <4 x i64> [[VEC_IND]], splat (i64 3)
; CHECK-NEXT: br label %[[BB43]]
; CHECK: [[BB43]]:
; CHECK-NEXT: [[TMP10:%.*]] = phi <4 x i64> [ [[TMP1]], %[[MIDDLE_BLOCK]] ], [ poison, %[[BB22]] ]
; CHECK-NEXT: [[TMP12:%.*]] = phi <4 x i64> [ poison, %[[MIDDLE_BLOCK]] ], [ [[TMP2]], %[[BB22]] ]
-; CHECK-NEXT: [[TMP7:%.*]] = phi <4 x i1> [ zeroinitializer, %[[MIDDLE_BLOCK]] ], [ [[BROADCAST_SPLAT]], %[[BB22]] ]
+; CHECK-NEXT: [[TMP7:%.*]] = phi <4 x i1> [ zeroinitializer, %[[MIDDLE_BLOCK]] ], [ splat (i1 true), %[[BB22]] ]
; CHECK-NEXT: [[TMP8:%.*]] = phi <4 x i64> [ poison, %[[MIDDLE_BLOCK]] ], [ [[TMP5]], %[[BB22]] ]
-; CHECK-NEXT: [[TMP9:%.*]] = phi <4 x i1> [ zeroinitializer, %[[MIDDLE_BLOCK]] ], [ [[TMP4]], %[[BB22]] ]
+; CHECK-NEXT: [[TMP9:%.*]] = phi <4 x i1> [ zeroinitializer, %[[MIDDLE_BLOCK]] ], [ [[TMP3]], %[[BB22]] ]
; CHECK-NEXT: [[PREDPHI:%.*]] = select <4 x i1> [[TMP7]], <4 x i64> [[TMP12]], <4 x i64> [[TMP10]]
; CHECK-NEXT: [[PREDPHI1:%.*]] = select <4 x i1> [[TMP9]], <4 x i64> [[TMP8]], <4 x i64> [[PREDPHI]]
; CHECK-NEXT: [[TMP6:%.*]] = extractelement <4 x i64> [[PREDPHI1]], i64 3
diff --git a/llvm/test/Transforms/LoopVectorize/scalarize-masked-call.ll b/llvm/test/Transforms/LoopVectorize/scalarize-masked-call.ll
index 2928a9f0cbf0b..56c3615636ec4 100644
--- a/llvm/test/Transforms/LoopVectorize/scalarize-masked-call.ll
+++ b/llvm/test/Transforms/LoopVectorize/scalarize-masked-call.ll
@@ -101,31 +101,19 @@ define void @masked_umax_used_by_load_address(ptr noalias %src, ptr noalias %dst
; CHECK: vector.ph:
; CHECK-NEXT: [[TMP0:%.*]] = call i64 @llvm.umax.i64(i64 [[N:%.*]], i64 1)
; CHECK-NEXT: [[TMP1:%.*]] = getelementptr i16, ptr [[SRC:%.*]], i64 [[TMP0]]
-; CHECK-NEXT: [[BROADCAST_SPLATINSERT:%.*]] = insertelement <2 x i1> poison, i1 [[C:%.*]], i64 0
-; CHECK-NEXT: [[BROADCAST_SPLAT:%.*]] = shufflevector <2 x i1> [[BROADCAST_SPLATINSERT]], <2 x i1> poison, <2 x i32> zeroinitializer
; CHECK-NEXT: br label [[VECTOR_BODY:%.*]]
; CHECK: vector.body:
; CHECK-NEXT: [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[PRED_LOAD_CONTINUE2:%.*]] ]
-; CHECK-NEXT: br i1 [[C]], label [[THEN1:%.*]], label [[PRED_LOAD_CONTINUE2]]
+; CHECK-NEXT: br i1 [[C:%.*]], label [[THEN1:%.*]], label [[PRED_LOAD_CONTINUE2]]
; CHECK: then1:
-; CHECK-NEXT: br i1 [[C]], label [[PRED_LOAD_IF:%.*]], label [[PRED_LOAD_CONTINUE:%.*]]
-; CHECK: pred.load.if:
; CHECK-NEXT: [[TMP2:%.*]] = load i16, ptr [[TMP1]], align 2
-; CHECK-NEXT: [[TMP3:%.*]] = insertelement <2 x i16> poison, i16 [[TMP2]], i64 0
-; CHECK-NEXT: br label [[PRED_LOAD_CONTINUE]]
-; CHECK: pred.load.continue:
-; CHECK-NEXT: [[TMP4:%.*]] = phi <2 x i16> [ poison, [[THEN1]] ], [ [[TMP3]], [[PRED_LOAD_IF]] ]
-; CHECK-NEXT: br i1 [[C]], label [[PRED_LOAD_IF1:%.*]], label [[PRED_LOAD_CONTINUE3:%.*]]
-; CHECK: pred.load.if2:
; CHECK-NEXT: [[TMP5:%.*]] = load i16, ptr [[TMP1]], align 2
+; CHECK-NEXT: [[TMP4:%.*]] = insertelement <2 x i16> poison, i16 [[TMP2]], i64 0
; CHECK-NEXT: [[TMP6:%.*]] = insertelement <2 x i16> [[TMP4]], i16 [[TMP5]], i64 1
-; CHECK-NEXT: br label [[PRED_LOAD_CONTINUE3]]
-; CHECK: pred.load.continue3:
-; CHECK-NEXT: [[TMP7:%.*]] = phi <2 x i16> [ [[TMP4]], [[PRED_LOAD_CONTINUE]] ], [ [[TMP6]], [[PRED_LOAD_IF1]] ]
; CHECK-NEXT: br label [[PRED_LOAD_CONTINUE2]]
-; CHECK: loop.latch4:
-; CHECK-NEXT: [[TMP10:%.*]] = phi <2 x i16> [ poison, [[VECTOR_BODY]] ], [ [[TMP7]], [[PRED_LOAD_CONTINUE3]] ]
-; CHECK-NEXT: [[TMP11:%.*]] = phi <2 x i1> [ zeroinitializer, [[VECTOR_BODY]] ], [ [[BROADCAST_SPLAT]], [[PRED_LOAD_CONTINUE3]] ]
+; CHECK: loop.latch2:
+; CHECK-NEXT: [[TMP10:%.*]] = phi <2 x i16> [ poison, [[VECTOR_BODY]] ], [ [[TMP6]], [[THEN1]] ]
+; CHECK-NEXT: [[TMP11:%.*]] = phi <2 x i1> [ zeroinitializer, [[VECTOR_BODY]] ], [ splat (i1 true), [[THEN1]] ]
; CHECK-NEXT: [[PREDPHI:%.*]] = select <2 x i1> [[TMP11]], <2 x i16> [[TMP10]], <2 x i16> zeroinitializer
; CHECK-NEXT: [[TMP8:%.*]] = getelementptr i16, ptr [[DST:%.*]], i64 [[INDEX]]
; CHECK-NEXT: store <2 x i16> [[PREDPHI]], ptr [[TMP8]], align 2
diff --git a/llvm/test/Transforms/LoopVectorize/struct-return-replicate.ll b/llvm/test/Transforms/LoopVectorize/struct-return-replicate.ll
index 4e2c20dd6f180..7c1f79a1761b7 100644
--- a/llvm/test/Transforms/LoopVectorize/struct-return-replicate.ll
+++ b/llvm/test/Transforms/LoopVectorize/struct-return-replicate.ll
@@ -662,7 +662,6 @@ define void @struct_return_predicated_extractvalue(i1 %c, ptr noalias %a, ptr no
; VF4-NEXT: [[ENTRY:.*:]]
; VF4-NEXT: br label %[[VECTOR_PH:.*]]
; VF4: [[VECTOR_PH]]:
-; VF4-NEXT: [[TMP0:%.*]] = xor i1 [[C]], true
; VF4-NEXT: br label %[[VECTOR_BODY:.*]]
; VF4: [[VECTOR_BODY]]:
; VF4-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[LATCH8:.*]] ]
@@ -711,30 +710,8 @@ define void @struct_return_predicated_extractvalue(i1 %c, ptr noalias %a, ptr no
; VF4-NEXT: br i1 [[C]], label %[[LATCH8]], label %[[IF1:.*]]
; VF4: [[IF1]]:
; VF4-NEXT: [[TMP36:%.*]] = extractvalue { <4 x float>, <4 x float> } [[TMP35]], 1
-; VF4-NEXT: br i1 [[TMP0]], label %[[PRED_STORE_IF:.*]], label %[[PRED_STORE_CONTINUE:.*]]
-; VF4: [[PRED_STORE_IF]]:
-; VF4-NEXT: [[TMP37:%.*]] = extractelement <4 x float> [[TMP36]], i64 0
-; VF4-NEXT: store float [[TMP37]], ptr [[Q]], align 4
-; VF4-NEXT: br label %[[PRED_STORE_CONTINUE]]
-; VF4: [[PRED_STORE_CONTINUE]]:
-; VF4-NEXT: br i1 [[TMP0]], label %[[PRED_STORE_IF1:.*]], label %[[PRED_STORE_CONTINUE2:.*]]
-; VF4: [[PRED_STORE_IF1]]:
-; VF4-NEXT: [[TMP38:%.*]] = extractelement <4 x float> [[TMP36]], i64 1
-; VF4-NEXT: store float [[TMP38]], ptr [[Q]], align 4
-; VF4-NEXT: br label %[[PRED_STORE_CONTINUE2]]
-; VF4: [[PRED_STORE_CONTINUE2]]:
-; VF4-NEXT: br i1 [[TMP0]], label %[[PRED_STORE_IF3:.*]], label %[[PRED_STORE_CONTINUE4:.*]]
-; VF4: [[PRED_STORE_IF3]]:
-; VF4-NEXT: [[TMP39:%.*]] = extractelement <4 x float> [[TMP36]], i64 2
-; VF4-NEXT: store float [[TMP39]], ptr [[Q]], align 4
-; VF4-NEXT: br label %[[PRED_STORE_CONTINUE4]]
-; VF4: [[PRED_STORE_CONTINUE4]]:
-; VF4-NEXT: br i1 [[TMP0]], label %[[PRED_STORE_IF5:.*]], label %[[PRED_STORE_CONTINUE6:.*]]
-; VF4: [[PRED_STORE_IF5]]:
; VF4-NEXT: [[TMP40:%.*]] = extractelement <4 x float> [[TMP36]], i64 3
; VF4-NEXT: store float [[TMP40]], ptr [[Q]], align 4
-; VF4-NEXT: br label %[[PRED_STORE_CONTINUE6]]
-; VF4: [[PRED_STORE_CONTINUE6]]:
; VF4-NEXT: br label %[[LATCH8]]
; VF4: [[LATCH8]]:
; VF4-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 4
@@ -747,7 +724,6 @@ define void @struct_return_predicated_extractvalue(i1 %c, ptr noalias %a, ptr no
; VF2IC2-NEXT: [[ENTRY:.*:]]
; VF2IC2-NEXT: br label %[[VECTOR_PH:.*]]
; VF2IC2: [[VECTOR_PH]]:
-; VF2IC2-NEXT: [[TMP0:%.*]] = xor i1 [[C]], true
; VF2IC2-NEXT: br label %[[VECTOR_BODY:.*]]
; VF2IC2: [[VECTOR_BODY]]:
; VF2IC2-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[LATCH9:.*]] ]
@@ -756,24 +732,9 @@ define void @struct_return_predicated_extractvalue(i1 %c, ptr noalias %a, ptr no
; VF2IC2-NEXT: [[WIDE_LOAD:%.*]] = load <2 x float>, ptr [[TMP42]], align 4
; VF2IC2-NEXT: [[WIDE_LOAD1:%.*]] = load <2 x float>, ptr [[TMP43]], align 4
; VF2IC2-NEXT: [[TMP44:%.*]] = extractelement <2 x float> [[WIDE_LOAD]], i64 0
-; VF2IC2-NEXT: [[TMP1:%.*]] = tail call { float, float } @fn2(float [[TMP44]]) #[[ATTR3]]
+; VF2IC2-NEXT: [[TMP3:%.*]] = tail call { float, float } @fn2(float [[TMP44]]) #[[ATTR3]]
; VF2IC2-NEXT: [[TMP45:%.*]] = extractelement <2 x float> [[WIDE_LOAD]], i64 1
-; VF2IC2-NEXT: [[TMP2:%.*]] = tail call { float, float } @fn2(float [[TMP45]]) #[[ATTR3]]
-; VF2IC2-NEXT: [[TMP3:%.*]] = extractvalue { float, float } [[TMP1]], 0
-; VF2IC2-NEXT: [[TMP4:%.*]] = insertelement <2 x float> poison, float [[TMP3]], i64 0
-; VF2IC2-NEXT: [[TMP5:%.*]] = insertvalue { <2 x float>, <2 x float> } poison, <2 x float> [[TMP4]], 0
-; VF2IC2-NEXT: [[TMP6:%.*]] = extractvalue { float, float } [[TMP1]], 1
-; VF2IC2-NEXT: [[TMP7:%.*]] = extractvalue { <2 x float>, <2 x float> } [[TMP5]], 1
-; VF2IC2-NEXT: [[TMP8:%.*]] = insertelement <2 x float> [[TMP7]], float [[TMP6]], i64 0
-; VF2IC2-NEXT: [[TMP9:%.*]] = insertvalue { <2 x float>, <2 x float> } [[TMP5]], <2 x float> [[TMP8]], 1
-; VF2IC2-NEXT: [[TMP10:%.*]] = extractvalue { float, float } [[TMP2]], 0
-; VF2IC2-NEXT: [[TMP11:%.*]] = extractvalue { <2 x float>, <2 x float> } [[TMP9]], 0
-; VF2IC2-NEXT: [[TMP12:%.*]] = insertelement <2 x float> [[TMP11]], float [[TMP10]], i64 1
-; VF2IC2-NEXT: [[TMP13:%.*]] = insertvalue { <2 x float>, <2 x float> } [[TMP9]], <2 x float> [[TMP12]], 0
-; VF2IC2-NEXT: [[TMP14:%.*]] = extractvalue { float, float } [[TMP2]], 1
-; VF2IC2-NEXT: [[TMP15:%.*]] = extractvalue { <2 x float>, <2 x float> } [[TMP13]], 1
-; VF2IC2-NEXT: [[TMP16:%.*]] = insertelement <2 x float> [[TMP15]], float [[TMP14]], i64 1
-; VF2IC2-NEXT: [[TMP17:%.*]] = insertvalue { <2 x float>, <2 x float> } [[TMP13]], <2 x float> [[TMP16]], 1
+; VF2IC2-NEXT: [[TMP5:%.*]] = tail call { float, float } @fn2(float [[TMP45]]) #[[ATTR3]]
; VF2IC2-NEXT: [[TMP46:%.*]] = extractelement <2 x float> [[WIDE_LOAD1]], i64 0
; VF2IC2-NEXT: [[TMP18:%.*]] = tail call { float, float } @fn2(float [[TMP46]]) #[[ATTR3]]
; VF2IC2-NEXT: [[TMP47:%.*]] = extractelement <2 x float> [[WIDE_LOAD1]], i64 1
@@ -796,32 +757,9 @@ define void @struct_return_predicated_extractvalue(i1 %c, ptr noalias %a, ptr no
; VF2IC2-NEXT: store float 0.000000e+00, ptr [[P]], align 4
; VF2IC2-NEXT: br i1 [[C]], label %[[LATCH9]], label %[[IF2:.*]]
; VF2IC2: [[IF2]]:
-; VF2IC2-NEXT: [[TMP35:%.*]] = extractvalue { <2 x float>, <2 x float> } [[TMP17]], 1
; VF2IC2-NEXT: [[TMP36:%.*]] = extractvalue { <2 x float>, <2 x float> } [[TMP34]], 1
-; VF2IC2-NEXT: br i1 [[TMP0]], label %[[PRED_STORE_IF:.*]], label %[[PRED_STORE_CONTINUE:.*]]
-; VF2IC2: [[PRED_STORE_IF]]:
-; VF2IC2-NEXT: [[TMP37:%.*]] = extractelement <2 x float> [[TMP35]], i64 0
-; VF2IC2-NEXT: store float [[TMP37]], ptr [[Q]], align 4
-; VF2IC2-NEXT: br label %[[PRED_STORE_CONTINUE]]
-; VF2IC2: [[PRED_STORE_CONTINUE]]:
-; VF2IC2-NEXT: br i1 [[TMP0]], label %[[PRED_STORE_IF1:.*]], label %[[PRED_STORE_CONTINUE2:.*]]
-; VF2IC2: [[PRED_STORE_IF1]]:
-; VF2IC2-NEXT: [[TMP38:%.*]] = extractelement <2 x float> [[TMP35]], i64 1
-; VF2IC2-NEXT: store float [[TMP38]], ptr [[Q]], align 4
-; VF2IC2-NEXT: br label %[[PRED_STORE_CONTINUE2]]
-; VF2IC2: [[PRED_STORE_CONTINUE2]]:
-; VF2IC2-NEXT: br i1 [[TMP0]], label %[[PRED_STORE_IF3:.*]], label %[[PRED_STORE_CONTINUE4:.*]]
-; VF2IC2: [[PRED_STORE_IF3]]:
-; VF2IC2-NEXT: [[TMP39:%.*]] = extractelement <2 x float> [[TMP36]], i64 0
-; VF2IC2-NEXT: store float [[TMP39]], ptr [[Q]], align 4
-; VF2IC2-NEXT: br label %[[PRED_STORE_CONTINUE4]]
-; VF2IC2: [[PRED_STORE_CONTINUE4]]:
-; VF2IC2-NEXT: br i1 [[TMP0]], label %[[PRED_STORE_IF5:.*]], label %[[PRED_STORE_CONTINUE6:.*]]
-; VF2IC2: [[PRED_STORE_IF5]]:
; VF2IC2-NEXT: [[TMP40:%.*]] = extractelement <2 x float> [[TMP36]], i64 1
; VF2IC2-NEXT: store float [[TMP40]], ptr [[Q]], align 4
-; VF2IC2-NEXT: br label %[[PRED_STORE_CONTINUE6]]
-; VF2IC2: [[PRED_STORE_CONTINUE6]]:
; VF2IC2-NEXT: br label %[[LATCH9]]
; VF2IC2: [[LATCH9]]:
; VF2IC2-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 4
diff --git a/llvm/test/Transforms/LoopVectorize/uniform-blend.ll b/llvm/test/Transforms/LoopVectorize/uniform-blend.ll
index 8cb37a98a6de0..21a81fed25571 100644
--- a/llvm/test/Transforms/LoopVectorize/uniform-blend.ll
+++ b/llvm/test/Transforms/LoopVectorize/uniform-blend.ll
@@ -102,23 +102,20 @@ define void @blend_chain_iv(i1 %c) {
; CHECK-NEXT: [[ENTRY:.*:]]
; CHECK-NEXT: br label %[[VECTOR_PH:.*]]
; CHECK: [[VECTOR_PH]]:
-; CHECK-NEXT: [[TMP10:%.*]] = freeze i1 [[C]]
-; CHECK-NEXT: [[BROADCAST_SPLATINSERT:%.*]] = insertelement <4 x i1> poison, i1 [[C]], i64 0
-; CHECK-NEXT: [[BROADCAST_SPLAT:%.*]] = shufflevector <4 x i1> [[BROADCAST_SPLATINSERT]], <4 x i1> poison, <4 x i32> zeroinitializer
; CHECK-NEXT: br label %[[VECTOR_BODY:.*]]
; CHECK: [[VECTOR_BODY]]:
; CHECK-NEXT: [[INDEX1:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[LOOP_LATCH4:.*]] ]
; CHECK-NEXT: [[VEC_IND:%.*]] = phi <4 x i64> [ <i64 0, i64 1, i64 2, i64 3>, %[[VECTOR_PH]] ], [ [[VEC_IND_NEXT:%.*]], %[[LOOP_LATCH4]] ]
; CHECK-NEXT: br i1 [[C]], label %[[LOOP_NEXT1:.*]], label %[[LOOP_LATCH4]]
; CHECK: [[LOOP_NEXT1]]:
-; CHECK-NEXT: br i1 [[TMP10]], label %[[LOOP_NEXT_22:.*]], label %[[LOOP_NEXT_33:.*]]
+; CHECK-NEXT: br i1 [[C]], label %[[LOOP_NEXT_22:.*]], label %[[LOOP_NEXT_33:.*]]
; CHECK: [[LOOP_NEXT_22]]:
; CHECK-NEXT: br label %[[LOOP_NEXT_33]]
; CHECK: [[LOOP_NEXT_33]]:
; CHECK-NEXT: br label %[[LOOP_LATCH4]]
; CHECK: [[LOOP_LATCH4]]:
; CHECK-NEXT: [[TMP0:%.*]] = phi <4 x i64> [ poison, %[[VECTOR_BODY]] ], [ [[VEC_IND]], %[[LOOP_NEXT_33]] ]
-; CHECK-NEXT: [[TMP1:%.*]] = phi <4 x i1> [ zeroinitializer, %[[VECTOR_BODY]] ], [ [[BROADCAST_SPLAT]], %[[LOOP_NEXT_33]] ]
+; CHECK-NEXT: [[TMP1:%.*]] = phi <4 x i1> [ zeroinitializer, %[[VECTOR_BODY]] ], [ splat (i1 true), %[[LOOP_NEXT_33]] ]
; CHECK-NEXT: [[PREDPHI:%.*]] = select <4 x i1> [[TMP1]], <4 x i64> [[TMP0]], <4 x i64> [[VEC_IND]]
; CHECK-NEXT: [[INDEX:%.*]] = extractelement <4 x i64> [[PREDPHI]], i64 0
; CHECK-NEXT: [[TMP2:%.*]] = getelementptr inbounds [32 x i16], ptr @dst, i16 0, i64 [[INDEX]]
diff --git a/llvm/test/Transforms/PhaseOrdering/loop-vectorize-bfi.ll b/llvm/test/Transforms/PhaseOrdering/loop-vectorize-bfi.ll
index 3d9af892e869f..22ee8c142c2a8 100644
--- a/llvm/test/Transforms/PhaseOrdering/loop-vectorize-bfi.ll
+++ b/llvm/test/Transforms/PhaseOrdering/loop-vectorize-bfi.ll
@@ -11,9 +11,6 @@ define void @f(i1 %x) vscale_range(2, 1024) !prof !0 {
; CHECK-LABEL: define void @f(
; CHECK-SAME: i1 [[X:%.*]]) local_unnamed_addr #[[ATTR0:[0-9]+]] {{.*}}{
; CHECK-NEXT: [[ENTRY:.*]]:
-; CHECK-NEXT: [[DOTSCALAR:%.*]] = xor i1 [[X]], true
-; CHECK-NEXT: [[TMP1:%.*]] = insertelement <vscale x 2 x i1> poison, i1 [[DOTSCALAR]], i64 0
-; CHECK-NEXT: [[TMP2:%.*]] = shufflevector <vscale x 2 x i1> [[TMP1]], <vscale x 2 x i1> poison, <vscale x 2 x i32> zeroinitializer
; CHECK-NEXT: br label %[[VECTOR_BODY:.*]]
; CHECK: [[VECTOR_BODY]]:
; CHECK-NEXT: [[EVL_BASED_IV:%.*]] = phi i64 [ 0, %[[ENTRY]] ], [ [[INDEX_EVL_NEXT:%.*]], %[[BAZ_14:.*]] ]
@@ -22,8 +19,8 @@ define void @f(i1 %x) vscale_range(2, 1024) !prof !0 {
; CHECK-NEXT: br i1 [[X]], label %[[BAZ_14]], label %[[BAR1:.*]]
; CHECK: [[BAR1]]:
; CHECK-NEXT: [[TMP4:%.*]] = getelementptr [8 x i8], ptr null, i64 [[EVL_BASED_IV]]
-; CHECK-NEXT: call void @llvm.vp.store.nxv2i64.p0(<vscale x 2 x i64> poison, ptr align 8 [[TMP4]], <vscale x 2 x i1> [[TMP2]], i32 [[TMP3]])
-; CHECK-NEXT: call void @llvm.vp.store.nxv2i64.p0(<vscale x 2 x i64> poison, ptr align 8 [[TMP4]], <vscale x 2 x i1> [[TMP2]], i32 [[TMP3]])
+; CHECK-NEXT: call void @llvm.vp.store.nxv2i64.p0(<vscale x 2 x i64> poison, ptr align 8 [[TMP4]], <vscale x 2 x i1> splat (i1 true), i32 [[TMP3]])
+; CHECK-NEXT: call void @llvm.vp.store.nxv2i64.p0(<vscale x 2 x i64> poison, ptr align 8 [[TMP4]], <vscale x 2 x i1> splat (i1 true), i32 [[TMP3]])
; CHECK-NEXT: br label %[[BAZ_14]]
; CHECK: [[BAZ_14]]:
; CHECK-NEXT: [[TMP5:%.*]] = zext nneg i32 [[TMP3]] to i64
More information about the llvm-branch-commits
mailing list