[llvm-branch-commits] [llvm] [WIP][VPlan] Exclude preserved uniform conditions from masks (PR #218552)
Andrei Elovikov via llvm-branch-commits
llvm-branch-commits at lists.llvm.org
Fri Aug 28 10:49:42 PDT 2026
https://github.com/eas updated https://github.com/llvm/llvm-project/pull/218552
>From 15f4d3ebc6a0a9ebbf5dfda1c60ee6d0e2b517db Mon Sep 17 00:00:00 2001
From: Andrei Elovikov <andrei.elovikov at sifive.com>
Date: Wed, 26 Aug 2026 12:34:31 -0700
Subject: [PATCH 1/2] Outline `shouldPreserveTerminator`
---
.../Transforms/Vectorize/VPlanPredicator.cpp | 112 +++++++++---------
1 file changed, 56 insertions(+), 56 deletions(-)
diff --git a/llvm/lib/Transforms/Vectorize/VPlanPredicator.cpp b/llvm/lib/Transforms/Vectorize/VPlanPredicator.cpp
index 7f37bb2bcc562..f84a4a11f3283 100644
--- a/llvm/lib/Transforms/Vectorize/VPlanPredicator.cpp
+++ b/llvm/lib/Transforms/Vectorize/VPlanPredicator.cpp
@@ -110,6 +110,8 @@ class VPPredicator {
/// Create a logical-or, factoring out a common header mask if present.
VPValue *createMaskOr(VPValue *LHS, VPValue *RHS, DebugLoc DL);
+ bool shouldPreserveTerminator(VPBasicBlock *VPBB);
+
/// Record \p Mask as the *entry* mask of \p VPBB, which is expected to not
/// already have a mask.
void setBlockInMask(const VPBasicBlock *VPBB, VPValue *Mask) {
@@ -207,6 +209,59 @@ VPValue *VPPredicator::createMaskOr(VPValue *LHS, VPValue *RHS, DebugLoc DL) {
HeaderMask, Builder.createOr(LHSRemainder, RHSRemainder, DL), DL);
}
+bool VPPredicator::shouldPreserveTerminator(VPBasicBlock *VPBB) {
+ LLVM_DEBUG(dbgs() << "Checking if can preserve the branch at the end of "
+ << VPBB->getName() << "\n");
+ auto False = []([[maybe_unused]] StringRef Reason = "") {
+ LLVM_DEBUG(dbgs() << " can't be preserved"
+ << (Reason.empty() ? Twine() : (": " + Reason)) << "\n");
+ return false;
+ };
+ if (VPBB->getNumSuccessors() != 2)
+ return False("#Successors != 2");
+ auto *Term = dyn_cast<VPInstruction>(VPBB->getTerminator());
+ if (!Term || Term->getOpcode() != VPInstruction::BranchOnCond)
+ return False("Not a branch");
+
+ bool IsUniformAndAvailable = [&](VPValue *V) {
+ auto *IRV = dyn_cast<VPIRValue>(V);
+ return IRV && isa<Argument, Constant>(IRV->getValue());
+ }(Term->getOperand(0));
+
+ if (!IsUniformAndAvailable)
+ return False("non-uniform");
+
+ // Should not happen in the end-to-end pass pipeline (simplifycfg would
+ // have handled it), but possible when running `loop-vectorize` pass
+ // alone. Just don't preserve so that we won't have to handle that when
+ // executing VPlan.
+ if (all_equal(VPBB->successors()))
+ return False("All successors are the same");
+
+ auto *IPostDomNode = VPPDT.getNode(VPBB)->getIDom();
+ if (!IPostDomNode)
+ return False("no post-dom");
+ auto *IPostDom = dyn_cast<VPBasicBlock>(IPostDomNode->getBlock());
+ if (!IPostDom || !VPDT.properlyDominates(VPBB, IPostDom))
+ return False("doesn't dominate its post-dom");
+
+ LLVM_DEBUG(dbgs() << "IPostDom: " << IPostDom->getName() << "\n";);
+
+ for (VPBlockBase *Pred : IPostDom->getPredecessors()) {
+ if (Pred == VPBB)
+ continue;
+ if (count_if(VPBB->successors(),
+ [&](auto *Succ) { return VPDT.dominates(Succ, Pred); }) != 1)
+ return False("Bailing out due to successors not being independent");
+ }
+
+ if (any_of(*IPostDom, IsaPred<VPBlendRecipe>))
+ return False("Blends at reconvergence");
+
+ LLVM_DEBUG(dbgs() << "...yes, can be preserved.\n");
+ return true;
+}
+
VPValue *VPPredicator::createEdgeMask(const VPBasicBlock *Src,
const VPBasicBlock *Dst) {
assert(is_contained(Dst->getPredecessors(), Src) && "Invalid edge");
@@ -519,62 +574,7 @@ void VPPredicator::run() {
LLVM_DEBUG(dbgs() << "Setting successors for " << VPBB->getName() << "\n");
auto Successors = to_vector(VPBB->getSuccessors());
- auto PreserveUniform = [&]() -> bool {
- LLVM_DEBUG(dbgs() << "Checking if can preserve the branch at the end of "
- << VPBB->getName() << "\n");
- auto False = []([[maybe_unused]] StringRef Reason = "") {
- LLVM_DEBUG(dbgs() << " can't be preserved"
- << (Reason.empty() ? Twine() : (": " + Reason))
- << "\n");
- return false;
- };
- if (VPBB->getNumSuccessors() != 2)
- return False("#Successors != 2");
- auto *Term = dyn_cast<VPInstruction>(VPBB->getTerminator());
- if (!Term || Term->getOpcode() != VPInstruction::BranchOnCond)
- return False("Not a branch");
-
- bool IsUniformAndAvailable = [&](VPValue *V) {
- auto *IRV = dyn_cast<VPIRValue>(V);
- return IRV && isa<Argument, Constant>(IRV->getValue());
- }(Term->getOperand(0));
-
- if (!IsUniformAndAvailable)
- return False("non-uniform");
-
- // Should not happen in the end-to-end pass pipeline (simplifycfg would
- // have handled it), but possible when running `loop-vectorize` pass
- // alone. Just don't preserve so that we won't have to handle that when
- // executing VPlan.
- if (all_equal(VPBB->successors()))
- return False("All successors are the same");
-
- auto *IPostDomNode = VPPDT.getNode(VPBB)->getIDom();
- if (!IPostDomNode)
- return False("no post-dom");
- auto *IPostDom = dyn_cast<VPBasicBlock>(IPostDomNode->getBlock());
- if (!IPostDom || !VPDT.properlyDominates(VPBB, IPostDom))
- return False("doesn't dominate its post-dom");
-
- LLVM_DEBUG(dbgs() << "IPostDom: " << IPostDom->getName() << "\n";);
-
- for (VPBlockBase *Pred : IPostDom->getPredecessors()) {
- if (Pred == VPBB)
- continue;
- if (count_if(VPBB->successors(), [&](auto *Succ) {
- return VPDT.dominates(Succ, Pred);
- }) != 1)
- return False("Bailing out due to successors not being independent");
- }
-
- if (any_of(*IPostDom, IsaPred<VPBlendRecipe>))
- return False("Blends at reconvergence");
-
- LLVM_DEBUG(dbgs() << "...yes, can be preserved.\n");
- return true;
- }();
-
- if (PreserveUniform) {
+ if (shouldPreserveTerminator(VPBB)) {
for (auto *Succ : Successors)
VPBlockUtils::disconnectBlocks(VPBB, Succ);
>From b3475d42676f12dc4b76343231def4adf492a33d Mon Sep 17 00:00:00 2001
From: Andrei Elovikov <andrei.elovikov at sifive.com>
Date: Fri, 21 Aug 2026 11:08:53 -0700
Subject: [PATCH 2/2] WIP Don't adjust mask for preserved uniform edges
---
.../Transforms/Vectorize/LoopVectorize.cpp | 6 +-
.../Transforms/Vectorize/VPlanPredicator.cpp | 14 +-
.../Transforms/Vectorize/VPlanTransforms.cpp | 3 +-
.../AArch64/predication_costs.ll | 38 +++-
.../AArch64/sve-predicated-costs.ll | 5 +-
...-interleave-to-widen-memory-multi-block.ll | 36 +---
.../LoopVectorize/RISCV/dead-ops-cost.ll | 55 +-----
.../LoopVectorize/RISCV/predicated-costs.ll | 10 +-
.../RISCV/predicated-reverse-store.ll | 7 +-
.../LoopVectorize/VPlan/predicator.ll | 81 ++++----
.../VPlan/vplan-sink-scalars-and-merge-vf1.ll | 18 +-
.../VPlan/vplan-sink-scalars-and-merge.ll | 28 +--
.../LoopVectorize/X86/masked-store-cost.ll | 4 +-
.../Transforms/LoopVectorize/div-exact.ll | 134 +------------
.../LoopVectorize/if-pred-stores.ll | 179 +++++-------------
llvm/test/Transforms/LoopVectorize/pr37248.ll | 10 -
.../Transforms/LoopVectorize/predicator.ll | 51 ++---
17 files changed, 177 insertions(+), 502 deletions(-)
diff --git a/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp b/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
index 28895d1d2b418..0eb7e59080780 100644
--- a/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
+++ b/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
@@ -6240,8 +6240,10 @@ VPRecipeWithIRFlags *VPRecipeBuilder::tryToWiden(VPInstruction *VPI) {
case Instruction::UDiv:
case Instruction::SRem:
case Instruction::URem:
- // If not provably safe, use a masked intrinsic.
- if (CM.isPredicatedInst(I))
+ // If not provably safe, use a masked intrinsic. Predicator might have
+ // preserved uniform control flow making this unmasked even if CM expected
+ // it to be masked.
+ if (CM.isPredicatedInst(I) && VPI->isMasked())
return new VPWidenIntrinsicRecipe(
getMaskedDivRemIntrinsic(VPI->getOpcode()), VPI->operands(),
I->getType(), {}, {}, VPI->getDebugLoc());
diff --git a/llvm/lib/Transforms/Vectorize/VPlanPredicator.cpp b/llvm/lib/Transforms/Vectorize/VPlanPredicator.cpp
index f84a4a11f3283..58b1fd2ccc0ce 100644
--- a/llvm/lib/Transforms/Vectorize/VPlanPredicator.cpp
+++ b/llvm/lib/Transforms/Vectorize/VPlanPredicator.cpp
@@ -255,6 +255,10 @@ bool VPPredicator::shouldPreserveTerminator(VPBasicBlock *VPBB) {
return False("Bailing out due to successors not being independent");
}
+ // FIXME: Only one of the two checks below should remain in the end.
+ if (!IPostDom->phis().empty())
+ return False("Phis at reconvergence");
+
if (any_of(*IPostDom, IsaPred<VPBlendRecipe>))
return False("Blends at reconvergence");
@@ -323,7 +327,10 @@ void VPPredicator::createBlockInMask(VPBasicBlock *VPBB) {
// This is the block mask. We OR all unique incoming edges.
for (auto *Predecessor : SetVector<VPBlockBase *>(
VPBB->getPredecessors().begin(), VPBB->getPredecessors().end())) {
- VPValue *EdgeMask = createEdgeMask(cast<VPBasicBlock>(Predecessor), VPBB);
+ auto *Pred = cast<VPBasicBlock>(Predecessor);
+ VPValue *EdgeMask = shouldPreserveTerminator(Pred)
+ ? getBlockInMask(Pred)
+ : createEdgeMask(Pred, VPBB);
if (!EdgeMask) { // Mask of predecessor is all-one so mask of block is
// too.
setBlockInMask(VPBB, EdgeMask);
@@ -481,6 +488,7 @@ void VPPredicator::convertPhisToBlends(VPBasicBlock *VPBB) {
for (VPRecipeBase &R : VPBB->phis())
Phis.push_back(cast<VPPhi>(&R));
for (VPPhi *PhiR : Phis) {
+ LLVM_DEBUG(dbgs() << "Converting " << *PhiR << " to blend\n");
// The non-header Phi is converted into a Blend recipe below,
// so we don't have to worry about the insertion order and we can just use
// the builder. At this point we generate the predication tree. There may
@@ -491,6 +499,9 @@ void VPPredicator::convertPhisToBlends(VPBasicBlock *VPBB) {
return !match(V, m_Poison());
});
if (all_equal(NotPoison)) {
+ LLVM_DEBUG(dbgs() << " all incoming values are the same or poison, "
+ "replacing with single value "
+ << PhiR->getIncomingValue(0) << "\n");
PhiR->replaceAllUsesWith(NotPoison.empty() ? PhiR->getIncomingValue(0)
: *NotPoison.begin());
PhiR->eraseFromParent();
@@ -525,6 +536,7 @@ void VPPredicator::convertPhisToBlends(VPBasicBlock *VPBB) {
auto *Blend =
new VPBlendRecipe(IRPhi, OperandsWithMask, *PhiR, PhiR->getDebugLoc());
Builder.insert(Blend);
+ LLVM_DEBUG(dbgs() << " blend: " << *Blend << "\n");
PhiR->replaceAllUsesWith(Blend);
PhiR->eraseFromParent();
}
diff --git a/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp b/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp
index a0775ca6e4345..049814c054713 100644
--- a/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp
+++ b/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp
@@ -5401,7 +5401,8 @@ void VPlanTransforms::makeMemOpWideningDecisions(VPlan &Plan, VFRange &Range,
// A predicated access can only be widened (rather than scalarized) if
// the target supports a masked load/store for it.
// TODO: Determine if a load/store needs predication directly in VPlan.
- bool IsPredicated = RecipeBuilder.isPredicatedInst(I);
+ bool IsPredicated =
+ RecipeBuilder.isPredicatedInst(I) && VPI->isMasked();
if (IsPredicated && !CostCtx.Config.isLegalMaskedLoadOrStore(
IsLoad, ScalarTy, getLoadStoreAlignment(I),
getLoadStoreAddressSpace(I)))
diff --git a/llvm/test/Transforms/LoopVectorize/AArch64/predication_costs.ll b/llvm/test/Transforms/LoopVectorize/AArch64/predication_costs.ll
index fdb22beb19695..e7b718dc0d483 100644
--- a/llvm/test/Transforms/LoopVectorize/AArch64/predication_costs.ll
+++ b/llvm/test/Transforms/LoopVectorize/AArch64/predication_costs.ll
@@ -62,7 +62,34 @@ for.end:
; CHECK: Scalarizing and predicating: store i32 %tmp2, ptr %tmp0, align 4
; CHECK: Cost of 4 for VF 2: profitable to scalarize store i32 %tmp2, ptr %tmp0, align 4
;
-define void @predicated_store(ptr %a, i1 %c, i32 %x, i64 %n) {
+define void @predicated_store(ptr %a, i32 %x, i64 %n) {
+entry:
+ br label %for.body
+
+for.body:
+ %i = phi i64 [ 0, %entry ], [ %i.next, %for.inc ]
+ %tmp0 = getelementptr inbounds i32, ptr %a, i64 %i
+ %tmp1 = load i32, ptr %tmp0, align 4
+ %tmp2 = add nsw i32 %tmp1, %x
+ %c = icmp eq i32 %tmp1, 42
+ br i1 %c, label %if.then, label %for.inc
+
+if.then:
+ store i32 %tmp2, ptr %tmp0, align 4
+ br label %for.inc
+
+for.inc:
+ %i.next = add nuw nsw i64 %i, 1
+ %cond = icmp slt i64 %i.next, %n
+ br i1 %cond, label %for.body, label %for.end
+
+for.end:
+ ret void
+}
+
+; CHECK-LABEL: store_in_preserved_uniform_control_flow
+; CHECK: Cost of 1 for VF 2: WIDEN store {{.*}}, ir<%tmp2>
+define void @store_in_preserved_uniform_control_flow(ptr %a, i1 %c, i32 %x, i64 %n) {
entry:
br label %for.body
@@ -96,7 +123,7 @@ for.end:
; CHECK: Cost of 0 for VF 2: induction instruction %addr = phi ptr [ %a, %entry ], [ %addr.next, %for.inc ]
; CHECK: Cost of 4 for VF 2: profitable to scalarize store i32 %tmp2, ptr %addr, align 4
;
-define void @predicated_store_phi(ptr %a, i1 %c, i32 %x, i64 %n) {
+define void @predicated_store_phi(ptr %a, i32 %x, i64 %n) {
entry:
br label %for.body
@@ -105,6 +132,7 @@ for.body:
%addr = phi ptr [ %a, %entry ], [ %addr.next, %for.inc ]
%tmp1 = load i32, ptr %addr, align 4
%tmp2 = add nsw i32 %tmp1, %x
+ %c = icmp eq i32 %tmp1, 42
br i1 %c, label %if.then, label %for.inc
if.then:
@@ -184,7 +212,7 @@ for.end:
; CHECK: Cost of 2 for VF 2: profitable to scalarize store i32 %tmp2, ptr %tmp0, align 4
; CHECK: Cost of 3 for VF 2: profitable to scalarize %tmp2 = add nsw i32 %tmp1, %x
;
-define void @predicated_store_scalarized_operand(ptr %a, i1 %c, i32 %x, i64 %n) {
+define void @predicated_store_scalarized_operand(ptr %a, i32 %x, i64 %n) {
entry:
br label %for.body
@@ -192,6 +220,7 @@ for.body:
%i = phi i64 [ 0, %entry ], [ %i.next, %for.inc ]
%tmp0 = getelementptr inbounds i32, ptr %a, i64 %i
%tmp1 = load i32, ptr %tmp0, align 4
+ %c = icmp eq i32 %tmp1, 42
br i1 %c, label %if.then, label %for.inc
if.then:
@@ -240,7 +269,7 @@ for.end:
; CHECK: Cost of 7 for VF 2: REPLICATE ir<%tmp3> = sdiv ir<%tmp1>, ir<%tmp2>
; CHECK: Cost of 5 for VF 2: REPLICATE ir<%tmp4> = udiv ir<%tmp3>, ir<%tmp2>
;
-define void @predication_multi_context(ptr %a, i1 %c, i32 %x, i64 %n) {
+define void @predication_multi_context(ptr %a, i32 %x, i64 %n) {
entry:
br label %for.body
@@ -248,6 +277,7 @@ for.body:
%i = phi i64 [ 0, %entry ], [ %i.next, %for.inc ]
%tmp0 = getelementptr inbounds i32, ptr %a, i64 %i
%tmp1 = load i32, ptr %tmp0, align 4
+ %c = icmp eq i32 %tmp1, 42
br i1 %c, label %if.then, label %for.inc
if.then:
diff --git a/llvm/test/Transforms/LoopVectorize/AArch64/sve-predicated-costs.ll b/llvm/test/Transforms/LoopVectorize/AArch64/sve-predicated-costs.ll
index 9360ce4aea08f..fd30156408e6a 100644
--- a/llvm/test/Transforms/LoopVectorize/AArch64/sve-predicated-costs.ll
+++ b/llvm/test/Transforms/LoopVectorize/AArch64/sve-predicated-costs.ll
@@ -81,10 +81,7 @@ define void @always_taken(ptr noalias %p0, ptr noalias %p1, i1 %c0, i1 %c1, i1 %
; CHECK-NEXT: [[BROADCAST_SPLATINSERT2:%.*]] = insertelement <vscale x 2 x i1> poison, i1 [[C2]], i64 0
; CHECK-NEXT: [[BROADCAST_SPLAT1:%.*]] = shufflevector <vscale x 2 x i1> [[BROADCAST_SPLATINSERT2]], <vscale x 2 x i1> poison, <vscale x 2 x i32> zeroinitializer
; CHECK-NEXT: [[BROADCAST_SPLATINSERT:%.*]] = insertelement <vscale x 2 x i1> poison, i1 [[C1]], i64 0
-; CHECK-NEXT: [[BROADCAST_SPLAT:%.*]] = shufflevector <vscale x 2 x i1> [[BROADCAST_SPLATINSERT]], <vscale x 2 x i1> poison, <vscale x 2 x i32> zeroinitializer
-; CHECK-NEXT: [[BROADCAST_SPLATINSERT1:%.*]] = insertelement <vscale x 2 x i1> poison, i1 [[C0]], i64 0
-; CHECK-NEXT: [[BROADCAST_SPLAT2:%.*]] = shufflevector <vscale x 2 x i1> [[BROADCAST_SPLATINSERT1]], <vscale x 2 x i1> poison, <vscale x 2 x i32> zeroinitializer
-; CHECK-NEXT: [[TMP6:%.*]] = select <vscale x 2 x i1> [[BROADCAST_SPLAT2]], <vscale x 2 x i1> [[BROADCAST_SPLAT]], <vscale x 2 x i1> zeroinitializer
+; CHECK-NEXT: [[TMP6:%.*]] = shufflevector <vscale x 2 x i1> [[BROADCAST_SPLATINSERT]], <vscale x 2 x i1> poison, <vscale x 2 x i32> zeroinitializer
; CHECK-NEXT: [[TMP8:%.*]] = select <vscale x 2 x i1> [[TMP6]], <vscale x 2 x i1> [[BROADCAST_SPLAT1]], <vscale x 2 x i1> zeroinitializer
; CHECK-NEXT: br label %[[VECTOR_BODY:.*]]
; CHECK: [[VECTOR_BODY]]:
diff --git a/llvm/test/Transforms/LoopVectorize/AArch64/transform-narrow-interleave-to-widen-memory-multi-block.ll b/llvm/test/Transforms/LoopVectorize/AArch64/transform-narrow-interleave-to-widen-memory-multi-block.ll
index 9efff93543685..3d69698888f16 100644
--- a/llvm/test/Transforms/LoopVectorize/AArch64/transform-narrow-interleave-to-widen-memory-multi-block.ll
+++ b/llvm/test/Transforms/LoopVectorize/AArch64/transform-narrow-interleave-to-widen-memory-multi-block.ll
@@ -13,7 +13,7 @@ define void @load_store_interleave_group_block_invar_cond(ptr noalias %data, ptr
; VF2IC1: [[VECTOR_PH]]:
; VF2IC1-NEXT: br label %[[VECTOR_BODY:.*]]
; VF2IC1: [[VECTOR_BODY]]:
-; VF2IC1-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[PRED_STORE_CONTINUE:.*]] ]
+; VF2IC1-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[PRED_STORE_CONTINUE3:.*]] ]
; VF2IC1-NEXT: [[TMP0:%.*]] = shl nsw i64 [[INDEX]], 1
; VF2IC1-NEXT: [[TMP1:%.*]] = getelementptr inbounds i64, ptr [[DATA]], i64 [[TMP0]]
; VF2IC1-NEXT: [[WIDE_VEC:%.*]] = load <4 x i64>, ptr [[TMP1]], align 8
@@ -22,20 +22,11 @@ define void @load_store_interleave_group_block_invar_cond(ptr noalias %data, ptr
; VF2IC1-NEXT: [[TMP4:%.*]] = shufflevector <2 x i64> [[STRIDED_VEC]], <2 x i64> [[STRIDED_VEC1]], <4 x i32> <i32 0, i32 1, i32 2, i32 3>
; VF2IC1-NEXT: [[INTERLEAVED_VEC:%.*]] = shufflevector <4 x i64> [[TMP4]], <4 x i64> poison, <4 x i32> <i32 0, i32 2, i32 1, i32 3>
; VF2IC1-NEXT: store <4 x i64> [[INTERLEAVED_VEC]], ptr [[TMP1]], align 8
-; VF2IC1-NEXT: br i1 [[C]], label %[[PRED_STORE_IF:.*]], label %[[PRED_STORE_CONTINUE]]
-; VF2IC1: [[PRED_STORE_IF]]:
-; VF2IC1-NEXT: br i1 [[C]], label %[[PRED_STORE_IF1:.*]], label %[[PRED_STORE_CONTINUE1:.*]]
-; VF2IC1: [[PRED_STORE_IF1]]:
-; VF2IC1-NEXT: store i8 1, ptr [[DST_0]], align 1
-; VF2IC1-NEXT: br label %[[PRED_STORE_CONTINUE1]]
-; VF2IC1: [[PRED_STORE_CONTINUE1]]:
-; VF2IC1-NEXT: br i1 [[C]], label %[[PRED_STORE_IF2:.*]], label %[[PRED_STORE_CONTINUE3:.*]]
+; VF2IC1-NEXT: br i1 [[C]], label %[[PRED_STORE_IF2:.*]], label %[[PRED_STORE_CONTINUE3]]
; VF2IC1: [[PRED_STORE_IF2]]:
; VF2IC1-NEXT: store i8 1, ptr [[DST_0]], align 1
; VF2IC1-NEXT: br label %[[PRED_STORE_CONTINUE3]]
; VF2IC1: [[PRED_STORE_CONTINUE3]]:
-; VF2IC1-NEXT: br label %[[PRED_STORE_CONTINUE]]
-; VF2IC1: [[PRED_STORE_CONTINUE]]:
; VF2IC1-NEXT: [[TMP2:%.*]] = getelementptr inbounds i8, ptr [[DST_1]], i64 [[INDEX]]
; VF2IC1-NEXT: store <2 x i8> zeroinitializer, ptr [[TMP2]], align 1
; VF2IC1-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 2
@@ -53,7 +44,7 @@ define void @load_store_interleave_group_block_invar_cond(ptr noalias %data, ptr
; VF2IC2: [[VECTOR_PH]]:
; VF2IC2-NEXT: br label %[[VECTOR_BODY:.*]]
; VF2IC2: [[VECTOR_BODY]]:
-; VF2IC2-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[PRED_STORE_CONTINUE:.*]] ]
+; VF2IC2-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[PRED_STORE_CONTINUE11:.*]] ]
; VF2IC2-NEXT: [[TMP0:%.*]] = add i64 [[INDEX]], 2
; VF2IC2-NEXT: [[TMP1:%.*]] = shl nsw i64 [[INDEX]], 1
; VF2IC2-NEXT: [[TMP2:%.*]] = shl nsw i64 [[TMP0]], 1
@@ -71,30 +62,11 @@ define void @load_store_interleave_group_block_invar_cond(ptr noalias %data, ptr
; VF2IC2-NEXT: [[TMP9:%.*]] = shufflevector <2 x i64> [[STRIDED_VEC3]], <2 x i64> [[STRIDED_VEC4]], <4 x i32> <i32 0, i32 1, i32 2, i32 3>
; VF2IC2-NEXT: [[INTERLEAVED_VEC5:%.*]] = shufflevector <4 x i64> [[TMP9]], <4 x i64> poison, <4 x i32> <i32 0, i32 2, i32 1, i32 3>
; VF2IC2-NEXT: store <4 x i64> [[INTERLEAVED_VEC5]], ptr [[TMP4]], align 8
-; VF2IC2-NEXT: br i1 [[C]], label %[[PRED_STORE_IF:.*]], label %[[PRED_STORE_CONTINUE]]
-; VF2IC2: [[PRED_STORE_IF]]:
-; VF2IC2-NEXT: br i1 [[C]], label %[[PRED_STORE_IF1:.*]], label %[[PRED_STORE_CONTINUE1:.*]]
-; VF2IC2: [[PRED_STORE_IF1]]:
-; VF2IC2-NEXT: store i8 1, ptr [[DST_0]], align 1
-; VF2IC2-NEXT: br label %[[PRED_STORE_CONTINUE1]]
-; VF2IC2: [[PRED_STORE_CONTINUE1]]:
-; VF2IC2-NEXT: br i1 [[C]], label %[[PRED_STORE_IF6:.*]], label %[[PRED_STORE_CONTINUE7:.*]]
-; VF2IC2: [[PRED_STORE_IF6]]:
-; VF2IC2-NEXT: store i8 1, ptr [[DST_0]], align 1
-; VF2IC2-NEXT: br label %[[PRED_STORE_CONTINUE7]]
-; VF2IC2: [[PRED_STORE_CONTINUE7]]:
-; VF2IC2-NEXT: br i1 [[C]], label %[[PRED_STORE_IF8:.*]], label %[[PRED_STORE_CONTINUE9:.*]]
-; VF2IC2: [[PRED_STORE_IF8]]:
-; VF2IC2-NEXT: store i8 1, ptr [[DST_0]], align 1
-; VF2IC2-NEXT: br label %[[PRED_STORE_CONTINUE9]]
-; VF2IC2: [[PRED_STORE_CONTINUE9]]:
-; VF2IC2-NEXT: br i1 [[C]], label %[[PRED_STORE_IF10:.*]], label %[[PRED_STORE_CONTINUE11:.*]]
+; VF2IC2-NEXT: br i1 [[C]], label %[[PRED_STORE_IF10:.*]], label %[[PRED_STORE_CONTINUE11]]
; VF2IC2: [[PRED_STORE_IF10]]:
; VF2IC2-NEXT: store i8 1, ptr [[DST_0]], align 1
; VF2IC2-NEXT: br label %[[PRED_STORE_CONTINUE11]]
; VF2IC2: [[PRED_STORE_CONTINUE11]]:
-; VF2IC2-NEXT: br label %[[PRED_STORE_CONTINUE]]
-; VF2IC2: [[PRED_STORE_CONTINUE]]:
; VF2IC2-NEXT: [[TMP5:%.*]] = getelementptr inbounds i8, ptr [[DST_1]], i64 [[INDEX]]
; VF2IC2-NEXT: [[TMP6:%.*]] = getelementptr inbounds i8, ptr [[TMP5]], i64 2
; VF2IC2-NEXT: store <2 x i8> zeroinitializer, ptr [[TMP5]], align 1
diff --git a/llvm/test/Transforms/LoopVectorize/RISCV/dead-ops-cost.ll b/llvm/test/Transforms/LoopVectorize/RISCV/dead-ops-cost.ll
index c4ca7d1d47673..913eba75abf17 100644
--- a/llvm/test/Transforms/LoopVectorize/RISCV/dead-ops-cost.ll
+++ b/llvm/test/Transforms/LoopVectorize/RISCV/dead-ops-cost.ll
@@ -158,60 +158,19 @@ define i32 @cost_of_exit_branch_and_cond_insts(ptr %a, ptr %b, i1 %c, i16 %x) #0
; CHECK-NEXT: [[TMP8:%.*]] = icmp eq i32 [[N_MOD_VF]], 0
; CHECK-NEXT: [[TMP9:%.*]] = select i1 [[TMP8]], i32 8, i32 [[N_MOD_VF]]
; CHECK-NEXT: [[N_VEC:%.*]] = sub i32 [[TMP2]], [[TMP9]]
-; CHECK-NEXT: [[BROADCAST_SPLATINSERT:%.*]] = insertelement <8 x i1> poison, i1 [[C]], i64 0
-; CHECK-NEXT: [[BROADCAST_SPLAT:%.*]] = shufflevector <8 x i1> [[BROADCAST_SPLATINSERT]], <8 x i1> poison, <8 x i32> zeroinitializer
-; CHECK-NEXT: br label %[[VECTOR_BODY:.*]]
-; CHECK: [[VECTOR_BODY]]:
-; CHECK-NEXT: [[INDEX:%.*]] = phi i32 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[LOOP_EXITING18:.*]] ]
-; CHECK-NEXT: br i1 [[C]], label %[[THEN3:.*]], label %[[LOOP_EXITING18]]
-; CHECK: [[THEN3]]:
-; CHECK-NEXT: [[TMP11:%.*]] = getelementptr i32, ptr [[B]], i32 [[INDEX]]
-; CHECK-NEXT: br i1 [[C]], label %[[PRED_STORE_IF:.*]], label %[[PRED_STORE_CONTINUE:.*]]
-; CHECK: [[PRED_STORE_IF]]:
-; CHECK-NEXT: store i1 false, ptr [[A]], align 1, !alias.scope [[META10:![0-9]+]], !noalias [[META13:![0-9]+]]
-; CHECK-NEXT: br label %[[PRED_STORE_CONTINUE]]
-; CHECK: [[PRED_STORE_CONTINUE]]:
-; CHECK-NEXT: br i1 [[C]], label %[[PRED_STORE_IF5:.*]], label %[[PRED_STORE_CONTINUE6:.*]]
-; CHECK: [[PRED_STORE_IF5]]:
-; CHECK-NEXT: store i1 false, ptr [[A]], align 1, !alias.scope [[META10]], !noalias [[META13]]
-; CHECK-NEXT: br label %[[PRED_STORE_CONTINUE6]]
-; CHECK: [[PRED_STORE_CONTINUE6]]:
-; CHECK-NEXT: br i1 [[C]], label %[[PRED_STORE_IF7:.*]], label %[[PRED_STORE_CONTINUE8:.*]]
-; CHECK: [[PRED_STORE_IF7]]:
-; CHECK-NEXT: store i1 false, ptr [[A]], align 1, !alias.scope [[META10]], !noalias [[META13]]
-; CHECK-NEXT: br label %[[PRED_STORE_CONTINUE8]]
-; CHECK: [[PRED_STORE_CONTINUE8]]:
-; CHECK-NEXT: br i1 [[C]], label %[[PRED_STORE_IF9:.*]], label %[[PRED_STORE_CONTINUE10:.*]]
-; CHECK: [[PRED_STORE_IF9]]:
-; CHECK-NEXT: store i1 false, ptr [[A]], align 1, !alias.scope [[META10]], !noalias [[META13]]
-; CHECK-NEXT: br label %[[PRED_STORE_CONTINUE10]]
-; CHECK: [[PRED_STORE_CONTINUE10]]:
-; CHECK-NEXT: br i1 [[C]], label %[[PRED_STORE_IF11:.*]], label %[[PRED_STORE_CONTINUE12:.*]]
-; CHECK: [[PRED_STORE_IF11]]:
-; CHECK-NEXT: store i1 false, ptr [[A]], align 1, !alias.scope [[META10]], !noalias [[META13]]
-; CHECK-NEXT: br label %[[PRED_STORE_CONTINUE12]]
-; CHECK: [[PRED_STORE_CONTINUE12]]:
-; CHECK-NEXT: br i1 [[C]], label %[[PRED_STORE_IF13:.*]], label %[[PRED_STORE_CONTINUE14:.*]]
-; CHECK: [[PRED_STORE_IF13]]:
-; CHECK-NEXT: store i1 false, ptr [[A]], align 1, !alias.scope [[META10]], !noalias [[META13]]
-; CHECK-NEXT: br label %[[PRED_STORE_CONTINUE14]]
-; CHECK: [[PRED_STORE_CONTINUE14]]:
-; CHECK-NEXT: br i1 [[C]], label %[[PRED_STORE_IF15:.*]], label %[[PRED_STORE_CONTINUE16:.*]]
-; CHECK: [[PRED_STORE_IF15]]:
-; CHECK-NEXT: store i1 false, ptr [[A]], align 1, !alias.scope [[META10]], !noalias [[META13]]
-; CHECK-NEXT: br label %[[PRED_STORE_CONTINUE16]]
+; CHECK-NEXT: br label %[[PRED_STORE_CONTINUE16:.*]]
; CHECK: [[PRED_STORE_CONTINUE16]]:
-; CHECK-NEXT: br i1 [[C]], label %[[PRED_STORE_IF17:.*]], label %[[PRED_STORE_CONTINUE18:.*]]
+; CHECK-NEXT: [[INDEX:%.*]] = phi i32 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[PRED_STORE_CONTINUE18:.*]] ]
+; CHECK-NEXT: br i1 [[C]], label %[[PRED_STORE_IF17:.*]], label %[[PRED_STORE_CONTINUE18]]
; CHECK: [[PRED_STORE_IF17]]:
-; CHECK-NEXT: store i1 false, ptr [[A]], align 1, !alias.scope [[META10]], !noalias [[META13]]
+; CHECK-NEXT: [[TMP13:%.*]] = getelementptr inbounds i32, ptr [[B]], i32 [[INDEX]]
+; CHECK-NEXT: store i1 false, ptr [[A]], align 1, !alias.scope [[META10:![0-9]+]], !noalias [[META13:![0-9]+]]
+; CHECK-NEXT: store <8 x i32> zeroinitializer, ptr [[TMP13]], align 4, !alias.scope [[META13]]
; CHECK-NEXT: br label %[[PRED_STORE_CONTINUE18]]
; CHECK: [[PRED_STORE_CONTINUE18]]:
-; CHECK-NEXT: call void @llvm.masked.store.v8i32.p0(<8 x i32> zeroinitializer, ptr align 4 [[TMP11]], <8 x i1> [[BROADCAST_SPLAT]]), !alias.scope [[META13]]
-; CHECK-NEXT: br label %[[LOOP_EXITING18]]
-; CHECK: [[LOOP_EXITING18]]:
; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i32 [[INDEX]], 8
; CHECK-NEXT: [[TMP21:%.*]] = icmp eq i32 [[INDEX_NEXT]], [[N_VEC]]
-; CHECK-NEXT: br i1 [[TMP21]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP15:![0-9]+]]
+; CHECK-NEXT: br i1 [[TMP21]], label %[[MIDDLE_BLOCK:.*]], label %[[PRED_STORE_CONTINUE16]], !llvm.loop [[LOOP15:![0-9]+]]
; CHECK: [[MIDDLE_BLOCK]]:
; CHECK-NEXT: br label %[[SCALAR_PH]]
; CHECK: [[SCALAR_PH]]:
diff --git a/llvm/test/Transforms/LoopVectorize/RISCV/predicated-costs.ll b/llvm/test/Transforms/LoopVectorize/RISCV/predicated-costs.ll
index e24e78fcf0602..532d9bc35e854 100644
--- a/llvm/test/Transforms/LoopVectorize/RISCV/predicated-costs.ll
+++ b/llvm/test/Transforms/LoopVectorize/RISCV/predicated-costs.ll
@@ -12,10 +12,7 @@ define void @nested(ptr noalias %p0, ptr noalias %p1, i1 %c0, i1 %c1) {
; CHECK-NEXT: br label %[[LOOP:.*]]
; CHECK: [[LOOP]]:
; CHECK-NEXT: [[BROADCAST_SPLATINSERT:%.*]] = insertelement <vscale x 4 x i1> poison, i1 [[C1]], i64 0
-; CHECK-NEXT: [[BROADCAST_SPLAT:%.*]] = shufflevector <vscale x 4 x i1> [[BROADCAST_SPLATINSERT]], <vscale x 4 x i1> poison, <vscale x 4 x i32> zeroinitializer
-; CHECK-NEXT: [[BROADCAST_SPLATINSERT1:%.*]] = insertelement <vscale x 4 x i1> poison, i1 [[C0]], i64 0
-; CHECK-NEXT: [[BROADCAST_SPLAT2:%.*]] = shufflevector <vscale x 4 x i1> [[BROADCAST_SPLATINSERT1]], <vscale x 4 x i1> poison, <vscale x 4 x i32> zeroinitializer
-; CHECK-NEXT: [[TMP0:%.*]] = select <vscale x 4 x i1> [[BROADCAST_SPLAT2]], <vscale x 4 x i1> [[BROADCAST_SPLAT]], <vscale x 4 x i1> zeroinitializer
+; CHECK-NEXT: [[TMP0:%.*]] = shufflevector <vscale x 4 x i1> [[BROADCAST_SPLATINSERT]], <vscale x 4 x i1> poison, <vscale x 4 x i32> zeroinitializer
; CHECK-NEXT: br label %[[THEN_1:.*]]
; CHECK: [[THEN_1]]:
; CHECK-NEXT: [[IV1:%.*]] = phi i32 [ 0, %[[LOOP]] ], [ [[INDEX_EVL_NEXT:%.*]], %[[LATCH4:.*]] ]
@@ -74,10 +71,7 @@ define void @always_taken(ptr noalias %p0, ptr noalias %p1, i1 %c0, i1 %c1) {
; CHECK-NEXT: br label %[[VECTOR_PH:.*]]
; CHECK: [[VECTOR_PH]]:
; CHECK-NEXT: [[BROADCAST_SPLATINSERT:%.*]] = insertelement <vscale x 4 x i1> poison, i1 [[C1]], i64 0
-; CHECK-NEXT: [[BROADCAST_SPLAT:%.*]] = shufflevector <vscale x 4 x i1> [[BROADCAST_SPLATINSERT]], <vscale x 4 x i1> poison, <vscale x 4 x i32> zeroinitializer
-; CHECK-NEXT: [[BROADCAST_SPLATINSERT1:%.*]] = insertelement <vscale x 4 x i1> poison, i1 [[C0]], i64 0
-; CHECK-NEXT: [[BROADCAST_SPLAT2:%.*]] = shufflevector <vscale x 4 x i1> [[BROADCAST_SPLATINSERT1]], <vscale x 4 x i1> poison, <vscale x 4 x i32> zeroinitializer
-; CHECK-NEXT: [[TMP0:%.*]] = select <vscale x 4 x i1> [[BROADCAST_SPLAT2]], <vscale x 4 x i1> [[BROADCAST_SPLAT]], <vscale x 4 x i1> zeroinitializer
+; CHECK-NEXT: [[TMP0:%.*]] = shufflevector <vscale x 4 x i1> [[BROADCAST_SPLATINSERT]], <vscale x 4 x i1> poison, <vscale x 4 x i32> zeroinitializer
; CHECK-NEXT: br label %[[VECTOR_BODY:.*]]
; CHECK: [[VECTOR_BODY]]:
; CHECK-NEXT: [[EVL_BASED_IV:%.*]] = phi i32 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_EVL_NEXT:%.*]], %[[LATCH4:.*]] ]
diff --git a/llvm/test/Transforms/LoopVectorize/RISCV/predicated-reverse-store.ll b/llvm/test/Transforms/LoopVectorize/RISCV/predicated-reverse-store.ll
index 29373388b3721..a5b3d96996c26 100644
--- a/llvm/test/Transforms/LoopVectorize/RISCV/predicated-reverse-store.ll
+++ b/llvm/test/Transforms/LoopVectorize/RISCV/predicated-reverse-store.ll
@@ -7,8 +7,6 @@ define void @reverse_predicated_store(i1 %c, ptr %dst, i64 %n) #0 {
; CHECK-NEXT: [[TMP0:%.*]] = add i64 [[N:%.*]], 1
; CHECK-NEXT: br label [[FOR_BODY:%.*]]
; CHECK: vector.ph:
-; CHECK-NEXT: [[BROADCAST_SPLATINSERT:%.*]] = insertelement <vscale x 4 x i1> poison, i1 [[C:%.*]], i64 0
-; CHECK-NEXT: [[BROADCAST_SPLAT:%.*]] = shufflevector <vscale x 4 x i1> [[BROADCAST_SPLATINSERT]], <vscale x 4 x i1> poison, <vscale x 4 x i32> zeroinitializer
; CHECK-NEXT: br label [[VECTOR_BODY:%.*]]
; CHECK: vector.body:
; CHECK-NEXT: [[EVL_BASED_IV:%.*]] = phi i64 [ 0, [[FOR_BODY]] ], [ [[CURRENT_ITERATION_NEXT:%.*]], [[LOOP_LATCH2:%.*]] ]
@@ -16,16 +14,15 @@ define void @reverse_predicated_store(i1 %c, ptr %dst, i64 %n) #0 {
; CHECK-NEXT: [[TMP1:%.*]] = call i32 @llvm.experimental.get.vector.length.i64(i64 [[AVL]], i32 4, i1 true)
; CHECK-NEXT: [[IV:%.*]] = sub i64 [[N]], [[EVL_BASED_IV]]
; CHECK-NEXT: [[IV_NEXT:%.*]] = add i64 [[IV]], -1
-; CHECK-NEXT: br i1 [[C]], label [[IF_THEN1:%.*]], label [[LOOP_LATCH2]]
+; CHECK-NEXT: br i1 [[C:%.*]], label [[IF_THEN1:%.*]], label [[LOOP_LATCH2]]
; CHECK: if.then1:
; CHECK-NEXT: [[ARRAYIDX:%.*]] = getelementptr float, ptr [[DST:%.*]], i64 [[IV_NEXT]]
-; CHECK-NEXT: [[VP_REVERSE_MASK:%.*]] = call <vscale x 4 x i1> @llvm.experimental.vp.reverse.nxv4i1(<vscale x 4 x i1> [[BROADCAST_SPLAT]], <vscale x 4 x i1> splat (i1 true), i32 [[TMP1]])
; CHECK-NEXT: [[TMP4:%.*]] = zext i32 [[TMP1]] to i64
; CHECK-NEXT: [[TMP6:%.*]] = sub nuw nsw i64 [[TMP4]], 1
; CHECK-NEXT: [[TMP7:%.*]] = sub i64 0, [[TMP6]]
; CHECK-NEXT: [[TMP9:%.*]] = getelementptr float, ptr [[ARRAYIDX]], i64 [[TMP7]]
; CHECK-NEXT: [[TMP12:%.*]] = call <vscale x 4 x float> @llvm.experimental.vp.reverse.nxv4f32(<vscale x 4 x float> zeroinitializer, <vscale x 4 x i1> splat (i1 true), i32 [[TMP1]])
-; CHECK-NEXT: call void @llvm.vp.store.nxv4f32.p0(<vscale x 4 x float> [[TMP12]], ptr align 4 [[TMP9]], <vscale x 4 x i1> [[VP_REVERSE_MASK]], i32 [[TMP1]])
+; CHECK-NEXT: call void @llvm.vp.store.nxv4f32.p0(<vscale x 4 x float> [[TMP12]], ptr align 4 [[TMP9]], <vscale x 4 x i1> splat (i1 true), i32 [[TMP1]])
; CHECK-NEXT: br label [[LOOP_LATCH2]]
; CHECK: loop.latch2:
; CHECK-NEXT: [[TMP13:%.*]] = zext i32 [[TMP1]] to i64
diff --git a/llvm/test/Transforms/LoopVectorize/VPlan/predicator.ll b/llvm/test/Transforms/LoopVectorize/VPlan/predicator.ll
index 239bd1b324257..a8e69d1ab4f4d 100644
--- a/llvm/test/Transforms/LoopVectorize/VPlan/predicator.ll
+++ b/llvm/test/Transforms/LoopVectorize/VPlan/predicator.ll
@@ -762,12 +762,12 @@ define void @simplifiable_blend(i1 %c1, i1 %c2, i1 %c3, i32 %x, i32 %y, ptr %p)
; CHECK-NEXT: Successor(s): D
; CHECK-EMPTY:
; CHECK-NEXT: D:
-; CHECK-NEXT: EMIT vp<[[VP7:%[0-9]+]]> = not ir<%c2>
-; CHECK-NEXT: EMIT vp<[[VP8:%[0-9]+]]> = logical-and ir<%c1>, vp<[[VP7]]>
+; CHECK-NEXT: EMIT vp<[[VP5:%[0-9]+]]> = not ir<%c2>
+; CHECK-NEXT: EMIT vp<[[VP6:%[0-9]+]]> = logical-and ir<%c1>, vp<[[VP5]]>
; CHECK-NEXT: Successor(s): C
; CHECK-EMPTY:
; CHECK-NEXT: C:
-; CHECK-NEXT: EMIT vp<[[VP9:%[0-9]+]]> = logical-and ir<%c1>, ir<%c2>
+; CHECK-NEXT: EMIT vp<[[VP7:%[0-9]+]]> = logical-and ir<%c1>, ir<%c2>
; CHECK-NEXT: Successor(s): latch
; CHECK-EMPTY:
; CHECK-NEXT: latch:
@@ -781,8 +781,6 @@ define void @simplifiable_blend(i1 %c1, i1 %c2, i1 %c3, i32 %x, i32 %y, ptr %p)
; CHECK-NEXT: No successors
; CHECK-EMPTY:
; CHECK-NEXT: E:
-; CHECK-NEXT: EMIT vp<[[VP5:%[0-9]+]]> = not ir<%c3>
-; CHECK-NEXT: EMIT vp<[[VP6:%[0-9]+]]> = logical-and vp<[[VP4]]>, vp<[[VP5]]>
; CHECK-NEXT: Successor(s): F
; CHECK-NEXT: }
; CHECK-NEXT: Successor(s): middle.block
@@ -1043,15 +1041,14 @@ define void @outermost_uniform_branch_no_phi(ptr %a, i1 %u0) {
; CHECK-NEXT: Successor(s): bb1, bb4
; CHECK-EMPTY:
; CHECK-NEXT: bb1:
-; CHECK-NEXT: EMIT ir<%add1> = add ir<%iv>, ir<1>, ir<%u0>
-; CHECK-NEXT: EMIT store ir<%add1>, ir<%gep>, ir<%u0>
-; CHECK-NEXT: EMIT ir<%v1> = icmp sle ir<%iv>, ir<1>, ir<%u0>
+; CHECK-NEXT: EMIT ir<%add1> = add ir<%iv>, ir<1>
+; CHECK-NEXT: EMIT store ir<%add1>, ir<%gep>
+; CHECK-NEXT: EMIT ir<%v1> = icmp sle ir<%iv>, ir<1>
; CHECK-NEXT: Successor(s): bb2
; CHECK-EMPTY:
; CHECK-NEXT: bb2:
-; CHECK-NEXT: EMIT vp<[[VP4:%[0-9]+]]> = logical-and ir<%u0>, ir<%v1>
-; CHECK-NEXT: EMIT ir<%add2> = add ir<%iv>, ir<2>, vp<[[VP4]]>
-; CHECK-NEXT: EMIT store ir<%add2>, ir<%gep>, vp<[[VP4]]>
+; CHECK-NEXT: EMIT ir<%add2> = add ir<%iv>, ir<2>, ir<%v1>
+; CHECK-NEXT: EMIT store ir<%add2>, ir<%gep>, ir<%v1>
; CHECK-NEXT: Successor(s): bb3
; CHECK-EMPTY:
; CHECK-NEXT: bb3:
@@ -1216,20 +1213,18 @@ define void @outermost_uniform_branch_more_blocks_no_phi(ptr %a, i1 %u0) {
; CHECK-NEXT: Successor(s): bb2, bb1
; CHECK-EMPTY:
; CHECK-NEXT: bb2:
-; CHECK-NEXT: EMIT ir<%v2> = icmp sle ir<%iv>, ir<2>, ir<%u0>
+; CHECK-NEXT: EMIT ir<%v2> = icmp sle ir<%iv>, ir<2>
; CHECK-NEXT: Successor(s): bb3
; CHECK-EMPTY:
; CHECK-NEXT: bb3:
-; CHECK-NEXT: EMIT vp<[[VP5:%[0-9]+]]> = not ir<%v2>
-; CHECK-NEXT: EMIT vp<[[VP6:%[0-9]+]]> = logical-and ir<%u0>, vp<[[VP5]]>
-; CHECK-NEXT: EMIT ir<%add3> = add ir<%iv>, ir<3>, vp<[[VP6]]>
-; CHECK-NEXT: EMIT store ir<%add3>, ir<%gep>, vp<[[VP6]]>
+; CHECK-NEXT: EMIT vp<[[VP4:%[0-9]+]]> = not ir<%v2>
+; CHECK-NEXT: EMIT ir<%add3> = add ir<%iv>, ir<3>, vp<[[VP4]]>
+; CHECK-NEXT: EMIT store ir<%add3>, ir<%gep>, vp<[[VP4]]>
; CHECK-NEXT: Successor(s): bb4
; CHECK-EMPTY:
; CHECK-NEXT: bb4:
-; CHECK-NEXT: EMIT vp<[[VP7:%[0-9]+]]> = logical-and ir<%u0>, ir<%v2>
-; CHECK-NEXT: EMIT ir<%add4> = add ir<%iv>, ir<4>, vp<[[VP7]]>
-; CHECK-NEXT: EMIT store ir<%add4>, ir<%gep>, vp<[[VP7]]>
+; CHECK-NEXT: EMIT ir<%add4> = add ir<%iv>, ir<4>, ir<%v2>
+; CHECK-NEXT: EMIT store ir<%add4>, ir<%gep>, ir<%v2>
; CHECK-NEXT: Successor(s): bb5
; CHECK-EMPTY:
; CHECK-NEXT: bb5:
@@ -1243,9 +1238,8 @@ define void @outermost_uniform_branch_more_blocks_no_phi(ptr %a, i1 %u0) {
; CHECK-NEXT: No successors
; CHECK-EMPTY:
; CHECK-NEXT: bb1:
-; CHECK-NEXT: EMIT vp<[[VP4:%[0-9]+]]> = not ir<%u0>
-; CHECK-NEXT: EMIT ir<%add1> = add ir<%iv>, ir<1>, vp<[[VP4]]>
-; CHECK-NEXT: EMIT store ir<%add1>, ir<%gep>, vp<[[VP4]]>
+; CHECK-NEXT: EMIT ir<%add1> = add ir<%iv>, ir<1>
+; CHECK-NEXT: EMIT store ir<%add1>, ir<%gep>
; CHECK-NEXT: Successor(s): bb6
; CHECK-NEXT: }
; CHECK-NEXT: Successor(s): middle.block
@@ -1404,9 +1398,8 @@ define void @uniform_branch_after_varying_branch_no_phi(ptr %a, i1 %u1) {
; CHECK-NEXT: Successor(s): bb2, bb3
; CHECK-EMPTY:
; CHECK-NEXT: bb2:
-; CHECK-NEXT: EMIT vp<[[VP5:%[0-9]+]]> = logical-and vp<[[VP4]]>, ir<%u1>
-; CHECK-NEXT: EMIT ir<%add2> = add ir<%iv>, ir<2>, vp<[[VP5]]>
-; CHECK-NEXT: EMIT store ir<%add2>, ir<%gep>, vp<[[VP5]]>
+; CHECK-NEXT: EMIT ir<%add2> = add ir<%iv>, ir<2>, vp<[[VP4]]>
+; CHECK-NEXT: EMIT store ir<%add2>, ir<%gep>, vp<[[VP4]]>
; CHECK-NEXT: Successor(s): bb3
; CHECK-EMPTY:
; CHECK-NEXT: bb3:
@@ -1583,9 +1576,8 @@ define void @uniform_branch_after_varying_branch_more_blocks_no_phi(ptr %a, i1 %
; CHECK-NEXT: Successor(s): bb3, bb2
; CHECK-EMPTY:
; CHECK-NEXT: bb3:
-; CHECK-NEXT: EMIT vp<[[VP7:%[0-9]+]]> = logical-and vp<[[VP4]]>, ir<%u1>
-; CHECK-NEXT: EMIT ir<%add3> = add ir<%iv>, ir<3>, vp<[[VP7]]>
-; CHECK-NEXT: EMIT store ir<%add3>, ir<%gep>, vp<[[VP7]]>
+; CHECK-NEXT: EMIT ir<%add3> = add ir<%iv>, ir<3>, vp<[[VP4]]>
+; CHECK-NEXT: EMIT store ir<%add3>, ir<%gep>, vp<[[VP4]]>
; CHECK-NEXT: Successor(s): bb4
; CHECK-EMPTY:
; CHECK-NEXT: bb4:
@@ -1604,10 +1596,8 @@ define void @uniform_branch_after_varying_branch_more_blocks_no_phi(ptr %a, i1 %
; CHECK-NEXT: No successors
; CHECK-EMPTY:
; CHECK-NEXT: bb2:
-; CHECK-NEXT: EMIT vp<[[VP5:%[0-9]+]]> = not ir<%u1>
-; CHECK-NEXT: EMIT vp<[[VP6:%[0-9]+]]> = logical-and vp<[[VP4]]>, vp<[[VP5]]>
-; CHECK-NEXT: EMIT ir<%add2> = add ir<%iv>, ir<2>, vp<[[VP6]]>
-; CHECK-NEXT: EMIT store ir<%add2>, ir<%gep>, vp<[[VP6]]>
+; CHECK-NEXT: EMIT ir<%add2> = add ir<%iv>, ir<2>, vp<[[VP4]]>
+; CHECK-NEXT: EMIT store ir<%add2>, ir<%gep>, vp<[[VP4]]>
; CHECK-NEXT: Successor(s): bb4
; CHECK-NEXT: }
; CHECK-NEXT: Successor(s): middle.block
@@ -1779,9 +1769,8 @@ define void @uniform_branch_after_varying_branch_more_blocks_mirrored_no_phi(ptr
; CHECK-NEXT: Successor(s): bb4, bb3
; CHECK-EMPTY:
; CHECK-NEXT: bb4:
-; CHECK-NEXT: EMIT vp<[[VP7:%[0-9]+]]> = logical-and ir<%v0>, ir<%u2>
-; CHECK-NEXT: EMIT ir<%add4> = add ir<%iv>, ir<4>, vp<[[VP7]]>
-; CHECK-NEXT: EMIT store ir<%add4>, ir<%gep>, vp<[[VP7]]>
+; CHECK-NEXT: EMIT ir<%add4> = add ir<%iv>, ir<4>, ir<%v0>
+; CHECK-NEXT: EMIT store ir<%add4>, ir<%gep>, ir<%v0>
; CHECK-NEXT: Successor(s): bb5
; CHECK-EMPTY:
; CHECK-NEXT: bb5:
@@ -1795,10 +1784,8 @@ define void @uniform_branch_after_varying_branch_more_blocks_mirrored_no_phi(ptr
; CHECK-NEXT: No successors
; CHECK-EMPTY:
; CHECK-NEXT: bb3:
-; CHECK-NEXT: EMIT vp<[[VP5:%[0-9]+]]> = not ir<%u2>
-; CHECK-NEXT: EMIT vp<[[VP6:%[0-9]+]]> = logical-and ir<%v0>, vp<[[VP5]]>
-; CHECK-NEXT: EMIT ir<%add3> = add ir<%iv>, ir<3>, vp<[[VP6]]>
-; CHECK-NEXT: EMIT store ir<%add3>, ir<%gep>, vp<[[VP6]]>
+; CHECK-NEXT: EMIT ir<%add3> = add ir<%iv>, ir<3>, ir<%v0>
+; CHECK-NEXT: EMIT store ir<%add3>, ir<%gep>, ir<%v0>
; CHECK-NEXT: Successor(s): bb5
; CHECK-NEXT: }
; CHECK-NEXT: Successor(s): middle.block
@@ -2823,15 +2810,14 @@ define void @uniform_branch_shared_join_with_varying_no_phi(ptr %a, i1 %u0) {
; CHECK-NEXT: Successor(s): bb2, bb1
; CHECK-EMPTY:
; CHECK-NEXT: bb2:
-; CHECK-NEXT: EMIT ir<%add2> = add ir<%iv>, ir<2>, ir<%u0>
-; CHECK-NEXT: EMIT store ir<%add2>, ir<%gep>, ir<%u0>
-; CHECK-NEXT: EMIT ir<%v2> = icmp sle ir<%iv>, ir<2>, ir<%u0>
+; CHECK-NEXT: EMIT ir<%add2> = add ir<%iv>, ir<2>
+; CHECK-NEXT: EMIT store ir<%add2>, ir<%gep>
+; CHECK-NEXT: EMIT ir<%v2> = icmp sle ir<%iv>, ir<2>
; CHECK-NEXT: Successor(s): bb3
; CHECK-EMPTY:
; CHECK-NEXT: bb3:
-; CHECK-NEXT: EMIT vp<[[VP5:%[0-9]+]]> = logical-and ir<%u0>, ir<%v2>
-; CHECK-NEXT: EMIT ir<%add3> = add ir<%iv>, ir<3>, vp<[[VP5]]>
-; CHECK-NEXT: EMIT store ir<%add3>, ir<%gep>, vp<[[VP5]]>
+; CHECK-NEXT: EMIT ir<%add3> = add ir<%iv>, ir<3>, ir<%v2>
+; CHECK-NEXT: EMIT store ir<%add3>, ir<%gep>, ir<%v2>
; CHECK-NEXT: Successor(s): bb4
; CHECK-EMPTY:
; CHECK-NEXT: bb4:
@@ -2842,9 +2828,8 @@ define void @uniform_branch_shared_join_with_varying_no_phi(ptr %a, i1 %u0) {
; CHECK-NEXT: No successors
; CHECK-EMPTY:
; CHECK-NEXT: bb1:
-; CHECK-NEXT: EMIT vp<[[VP4:%[0-9]+]]> = not ir<%u0>
-; CHECK-NEXT: EMIT ir<%add1> = add ir<%iv>, ir<1>, vp<[[VP4]]>
-; CHECK-NEXT: EMIT store ir<%add1>, ir<%gep>, vp<[[VP4]]>
+; CHECK-NEXT: EMIT ir<%add1> = add ir<%iv>, ir<1>
+; CHECK-NEXT: EMIT store ir<%add1>, ir<%gep>
; CHECK-NEXT: Successor(s): bb4
; CHECK-NEXT: }
; CHECK-NEXT: Successor(s): middle.block
diff --git a/llvm/test/Transforms/LoopVectorize/VPlan/vplan-sink-scalars-and-merge-vf1.ll b/llvm/test/Transforms/LoopVectorize/VPlan/vplan-sink-scalars-and-merge-vf1.ll
index c846b26096772..6595787aefd9d 100644
--- a/llvm/test/Transforms/LoopVectorize/VPlan/vplan-sink-scalars-and-merge-vf1.ll
+++ b/llvm/test/Transforms/LoopVectorize/VPlan/vplan-sink-scalars-and-merge-vf1.ll
@@ -32,23 +32,7 @@ define void @sink_with_sideeffects(i1 %c, ptr %ptr) {
; CHECK-NEXT: Successor(s): if.then, for.inc
; CHECK-EMPTY:
; CHECK-NEXT: if.then:
-; CHECK-NEXT: Successor(s): pred.store
-; CHECK-EMPTY:
-; CHECK-NEXT: <xVFxUF> pred.store: {
-; CHECK-NEXT: pred.store.entry:
-; CHECK-NEXT: BRANCH-ON-MASK ir<%c>
-; CHECK-NEXT: Successor(s): pred.store.if, pred.store.continue
-; CHECK-EMPTY:
-; CHECK-NEXT: pred.store.if:
-; CHECK-NEXT: CLONE store ir<%tmp3>, ir<%tmp2>
-; CHECK-NEXT: Successor(s): pred.store.continue
-; CHECK-EMPTY:
-; CHECK-NEXT: pred.store.continue:
-; CHECK-NEXT: No successors
-; CHECK-NEXT: }
-; CHECK-NEXT: Successor(s): if.then.0
-; CHECK-EMPTY:
-; CHECK-NEXT: if.then.0:
+; CHECK-NEXT: CLONE store ir<%tmp3>, ir<%tmp2>
; CHECK-NEXT: Successor(s): for.inc
; CHECK-EMPTY:
; CHECK-NEXT: for.inc:
diff --git a/llvm/test/Transforms/LoopVectorize/VPlan/vplan-sink-scalars-and-merge.ll b/llvm/test/Transforms/LoopVectorize/VPlan/vplan-sink-scalars-and-merge.ll
index c8503ebab5e45..43acb39578a7d 100644
--- a/llvm/test/Transforms/LoopVectorize/VPlan/vplan-sink-scalars-and-merge.ll
+++ b/llvm/test/Transforms/LoopVectorize/VPlan/vplan-sink-scalars-and-merge.ll
@@ -1164,27 +1164,13 @@ define void @update_multiple_users(ptr noalias %src, ptr noalias %dst, i1 %c) {
; CHECK-NEXT: Successor(s): loop.then, loop.latch
; CHECK-EMPTY:
; CHECK-NEXT: loop.then:
-; CHECK-NEXT: Successor(s): pred.store
-; CHECK-EMPTY:
-; CHECK-NEXT: <xVFxUF> pred.store: {
-; CHECK-NEXT: pred.store.entry:
-; CHECK-NEXT: BRANCH-ON-MASK ir<%c>
-; CHECK-NEXT: Successor(s): pred.store.if, pred.store.continue
-; CHECK-EMPTY:
-; CHECK-NEXT: pred.store.if:
-; CHECK-NEXT: REPLICATE ir<%l1> = load ir<%src>
-; CHECK-NEXT: REPLICATE ir<%l2> = trunc ir<%l1>
-; CHECK-NEXT: REPLICATE ir<%cmp> = icmp eq ir<%l1>, ir<0>
-; CHECK-NEXT: REPLICATE ir<%sel> = select ir<%cmp>, ir<5>, ir<%l2>
-; CHECK-NEXT: REPLICATE store ir<%sel>, ir<%dst>
-; CHECK-NEXT: Successor(s): pred.store.continue
-; CHECK-EMPTY:
-; CHECK-NEXT: pred.store.continue:
-; CHECK-NEXT: No successors
-; CHECK-NEXT: }
-; CHECK-NEXT: Successor(s): loop.then.1
-; CHECK-EMPTY:
-; CHECK-NEXT: loop.then.1:
+; CHECK-NEXT: REPLICATE ir<%l1> = load ir<%src>
+; CHECK-NEXT: REPLICATE ir<%l2> = trunc ir<%l1>
+; CHECK-NEXT: REPLICATE ir<%cmp> = icmp eq ir<%l1>, ir<0>
+; CHECK-NEXT: REPLICATE ir<%sel> = select ir<%cmp>, ir<5>, ir<%l2>
+; CHECK-NEXT: EMIT vp<[[VP3:%[0-9]+]]> = extract-last-part ir<%sel>
+; CHECK-NEXT: EMIT vp<[[VP4:%[0-9]+]]> = extract-last-lane vp<[[VP3]]>
+; CHECK-NEXT: CLONE store vp<[[VP4]]>, ir<%dst>
; CHECK-NEXT: Successor(s): loop.latch
; CHECK-EMPTY:
; CHECK-NEXT: loop.latch:
diff --git a/llvm/test/Transforms/LoopVectorize/X86/masked-store-cost.ll b/llvm/test/Transforms/LoopVectorize/X86/masked-store-cost.ll
index 4faabbc7b8f35..ba84fbbd5eeda 100644
--- a/llvm/test/Transforms/LoopVectorize/X86/masked-store-cost.ll
+++ b/llvm/test/Transforms/LoopVectorize/X86/masked-store-cost.ll
@@ -125,8 +125,6 @@ define void @test_scalar_cost_single_store_loop_invariant_cond(ptr %dst, i1 %c)
; CHECK-NEXT: br label %[[VECTOR_PH:.*]]
; CHECK: [[VECTOR_PH]]:
; CHECK-NEXT: [[TMP0:%.*]] = getelementptr i8, ptr [[DST]], i64 96
-; CHECK-NEXT: [[BROADCAST_SPLATINSERT:%.*]] = insertelement <8 x i1> poison, i1 [[C]], i64 0
-; CHECK-NEXT: [[BROADCAST_SPLAT:%.*]] = shufflevector <8 x i1> [[BROADCAST_SPLATINSERT]], <8 x i1> poison, <8 x i32> zeroinitializer
; CHECK-NEXT: br label %[[VECTOR_BODY:.*]]
; CHECK: [[VECTOR_BODY]]:
; CHECK-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[LOOP_LATCH2:.*]] ]
@@ -134,7 +132,7 @@ define void @test_scalar_cost_single_store_loop_invariant_cond(ptr %dst, i1 %c)
; CHECK-NEXT: [[NEXT_GEP:%.*]] = getelementptr i8, ptr [[DST]], i64 [[TMP1]]
; CHECK-NEXT: br i1 [[C]], label %[[IF_THEN1:.*]], label %[[LOOP_LATCH2]]
; CHECK: [[IF_THEN1]]:
-; CHECK-NEXT: call void @llvm.masked.store.v8i32.p0(<8 x i32> zeroinitializer, ptr align 4 [[NEXT_GEP]], <8 x i1> [[BROADCAST_SPLAT]])
+; CHECK-NEXT: store <8 x i32> zeroinitializer, ptr [[NEXT_GEP]], align 4
; CHECK-NEXT: br label %[[LOOP_LATCH2]]
; CHECK: [[LOOP_LATCH2]]:
; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 8
diff --git a/llvm/test/Transforms/LoopVectorize/div-exact.ll b/llvm/test/Transforms/LoopVectorize/div-exact.ll
index 7bec969285194..b302e88a99656 100644
--- a/llvm/test/Transforms/LoopVectorize/div-exact.ll
+++ b/llvm/test/Transforms/LoopVectorize/div-exact.ll
@@ -14,76 +14,18 @@ define void @unknown_divisor(ptr noalias %p, i32 %n, i1 %c) {
; CHECK: [[VECTOR_PH]]:
; CHECK-NEXT: [[N_MOD_VF:%.*]] = and i32 [[N]], 3
; CHECK-NEXT: [[N_VEC:%.*]] = sub i32 [[N]], [[N_MOD_VF]]
-; CHECK-NEXT: [[BROADCAST_SPLATINSERT:%.*]] = insertelement <4 x i1> poison, i1 [[C]], i64 0
-; CHECK-NEXT: [[BROADCAST_SPLAT:%.*]] = shufflevector <4 x i1> [[BROADCAST_SPLATINSERT]], <4 x i1> poison, <4 x i32> zeroinitializer
; CHECK-NEXT: br label %[[VECTOR_BODY:.*]]
; CHECK: [[VECTOR_BODY]]:
-; CHECK-NEXT: [[INDEX:%.*]] = phi i32 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[LATCH14:.*]] ]
-; CHECK-NEXT: [[TMP2:%.*]] = add i32 [[INDEX]], 1
-; CHECK-NEXT: [[TMP3:%.*]] = add i32 [[INDEX]], 2
-; CHECK-NEXT: [[TMP4:%.*]] = add i32 [[INDEX]], 3
+; CHECK-NEXT: [[TMP4:%.*]] = phi i32 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[LATCH14:.*]] ]
; CHECK-NEXT: br i1 [[C]], label %[[IF1:.*]], label %[[LATCH14]]
; CHECK: [[IF1]]:
-; CHECK-NEXT: [[TMP7:%.*]] = getelementptr i32, ptr [[P]], i32 [[INDEX]]
-; CHECK-NEXT: [[TMP8:%.*]] = getelementptr i32, ptr [[P]], i32 [[TMP2]]
-; CHECK-NEXT: [[TMP9:%.*]] = getelementptr i32, ptr [[P]], i32 [[TMP3]]
; CHECK-NEXT: [[TMP10:%.*]] = getelementptr i32, ptr [[P]], i32 [[TMP4]]
-; CHECK-NEXT: br i1 [[C]], label %[[PRED_LOAD_IF:.*]], label %[[PRED_LOAD_CONTINUE:.*]]
-; CHECK: [[PRED_LOAD_IF]]:
-; CHECK-NEXT: [[TMP12:%.*]] = load i32, ptr [[TMP7]], align 4
-; CHECK-NEXT: [[TMP13:%.*]] = insertelement <4 x i32> poison, i32 [[TMP12]], i64 0
-; CHECK-NEXT: br label %[[PRED_LOAD_CONTINUE]]
-; CHECK: [[PRED_LOAD_CONTINUE]]:
-; CHECK-NEXT: [[TMP14:%.*]] = phi <4 x i32> [ poison, %[[IF1]] ], [ [[TMP13]], %[[PRED_LOAD_IF]] ]
-; CHECK-NEXT: br i1 [[C]], label %[[PRED_LOAD_IF3:.*]], label %[[PRED_LOAD_CONTINUE4:.*]]
-; CHECK: [[PRED_LOAD_IF3]]:
-; CHECK-NEXT: [[TMP16:%.*]] = load i32, ptr [[TMP8]], align 4
-; CHECK-NEXT: [[TMP17:%.*]] = insertelement <4 x i32> [[TMP14]], i32 [[TMP16]], i64 1
-; CHECK-NEXT: br label %[[PRED_LOAD_CONTINUE4]]
-; CHECK: [[PRED_LOAD_CONTINUE4]]:
-; CHECK-NEXT: [[TMP18:%.*]] = phi <4 x i32> [ [[TMP14]], %[[PRED_LOAD_CONTINUE]] ], [ [[TMP17]], %[[PRED_LOAD_IF3]] ]
-; CHECK-NEXT: br i1 [[C]], label %[[PRED_LOAD_IF5:.*]], label %[[PRED_LOAD_CONTINUE6:.*]]
-; CHECK: [[PRED_LOAD_IF5]]:
-; CHECK-NEXT: [[TMP20:%.*]] = load i32, ptr [[TMP9]], align 4
-; CHECK-NEXT: [[TMP21:%.*]] = insertelement <4 x i32> [[TMP18]], i32 [[TMP20]], i64 2
-; CHECK-NEXT: br label %[[PRED_LOAD_CONTINUE6]]
-; CHECK: [[PRED_LOAD_CONTINUE6]]:
-; CHECK-NEXT: [[TMP22:%.*]] = phi <4 x i32> [ [[TMP18]], %[[PRED_LOAD_CONTINUE4]] ], [ [[TMP21]], %[[PRED_LOAD_IF5]] ]
-; CHECK-NEXT: br i1 [[C]], label %[[PRED_LOAD_IF7:.*]], label %[[PRED_LOAD_CONTINUE8:.*]]
-; CHECK: [[PRED_LOAD_IF7]]:
-; CHECK-NEXT: [[TMP24:%.*]] = load i32, ptr [[TMP10]], align 4
-; CHECK-NEXT: [[TMP25:%.*]] = insertelement <4 x i32> [[TMP22]], i32 [[TMP24]], i64 3
-; CHECK-NEXT: br label %[[PRED_LOAD_CONTINUE8]]
-; CHECK: [[PRED_LOAD_CONTINUE8]]:
-; CHECK-NEXT: [[TMP23:%.*]] = phi <4 x i32> [ [[TMP22]], %[[PRED_LOAD_CONTINUE6]] ], [ [[TMP25]], %[[PRED_LOAD_IF7]] ]
-; CHECK-NEXT: [[TMP28:%.*]] = call <4 x i32> @llvm.masked.udiv.v4i32(<4 x i32> splat (i32 100), <4 x i32> [[TMP23]], <4 x i1> [[BROADCAST_SPLAT]])
-; CHECK-NEXT: br i1 [[C]], label %[[PRED_STORE_IF:.*]], label %[[PRED_STORE_CONTINUE:.*]]
-; CHECK: [[PRED_STORE_IF]]:
-; CHECK-NEXT: [[TMP29:%.*]] = extractelement <4 x i32> [[TMP28]], i64 0
-; CHECK-NEXT: store i32 [[TMP29]], ptr [[TMP7]], align 4
-; CHECK-NEXT: br label %[[PRED_STORE_CONTINUE]]
-; CHECK: [[PRED_STORE_CONTINUE]]:
-; CHECK-NEXT: br i1 [[C]], label %[[PRED_STORE_IF9:.*]], label %[[PRED_STORE_CONTINUE10:.*]]
-; CHECK: [[PRED_STORE_IF9]]:
-; CHECK-NEXT: [[TMP30:%.*]] = extractelement <4 x i32> [[TMP28]], i64 1
-; CHECK-NEXT: store i32 [[TMP30]], ptr [[TMP8]], align 4
-; CHECK-NEXT: br label %[[PRED_STORE_CONTINUE10]]
-; CHECK: [[PRED_STORE_CONTINUE10]]:
-; CHECK-NEXT: br i1 [[C]], label %[[PRED_STORE_IF11:.*]], label %[[PRED_STORE_CONTINUE12:.*]]
-; CHECK: [[PRED_STORE_IF11]]:
-; CHECK-NEXT: [[TMP31:%.*]] = extractelement <4 x i32> [[TMP28]], i64 2
-; CHECK-NEXT: store i32 [[TMP31]], ptr [[TMP9]], align 4
-; CHECK-NEXT: br label %[[PRED_STORE_CONTINUE12]]
-; CHECK: [[PRED_STORE_CONTINUE12]]:
-; CHECK-NEXT: br i1 [[C]], label %[[PRED_STORE_IF13:.*]], label %[[PRED_STORE_CONTINUE14:.*]]
-; CHECK: [[PRED_STORE_IF13]]:
-; CHECK-NEXT: [[TMP32:%.*]] = extractelement <4 x i32> [[TMP28]], i64 3
-; CHECK-NEXT: store i32 [[TMP32]], ptr [[TMP10]], align 4
-; CHECK-NEXT: br label %[[PRED_STORE_CONTINUE14]]
-; CHECK: [[PRED_STORE_CONTINUE14]]:
+; CHECK-NEXT: [[TMP25:%.*]] = load <4 x i32>, ptr [[TMP10]], align 4
+; CHECK-NEXT: [[TMP28:%.*]] = udiv exact <4 x i32> splat (i32 100), [[TMP25]]
+; CHECK-NEXT: store <4 x i32> [[TMP28]], ptr [[TMP10]], align 4
; CHECK-NEXT: br label %[[LATCH14]]
; CHECK: [[LATCH14]]:
-; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i32 [[INDEX]], 4
+; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i32 [[TMP4]], 4
; CHECK-NEXT: [[TMP33:%.*]] = icmp eq i32 [[INDEX_NEXT]], [[N_VEC]]
; CHECK-NEXT: br i1 [[TMP33]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP0:![0-9]+]]
; CHECK: [[MIDDLE_BLOCK]]:
@@ -146,72 +88,16 @@ define void @unknown_dividend(ptr noalias %p, i32 %n, i1 %c) {
; CHECK-NEXT: [[N_VEC:%.*]] = sub i32 [[N]], [[N_MOD_VF]]
; CHECK-NEXT: br label %[[VECTOR_BODY:.*]]
; CHECK: [[VECTOR_BODY]]:
-; CHECK-NEXT: [[INDEX:%.*]] = phi i32 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[LATCH14:.*]] ]
-; CHECK-NEXT: [[TMP2:%.*]] = add i32 [[INDEX]], 1
-; CHECK-NEXT: [[TMP3:%.*]] = add i32 [[INDEX]], 2
-; CHECK-NEXT: [[TMP4:%.*]] = add i32 [[INDEX]], 3
+; CHECK-NEXT: [[TMP4:%.*]] = phi i32 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[LATCH14:.*]] ]
; CHECK-NEXT: br i1 [[C]], label %[[IF1:.*]], label %[[LATCH14]]
; CHECK: [[IF1]]:
-; CHECK-NEXT: [[TMP7:%.*]] = getelementptr i32, ptr [[P]], i32 [[INDEX]]
-; CHECK-NEXT: [[TMP8:%.*]] = getelementptr i32, ptr [[P]], i32 [[TMP2]]
-; CHECK-NEXT: [[TMP9:%.*]] = getelementptr i32, ptr [[P]], i32 [[TMP3]]
; CHECK-NEXT: [[TMP10:%.*]] = getelementptr i32, ptr [[P]], i32 [[TMP4]]
-; CHECK-NEXT: br i1 [[C]], label %[[PRED_LOAD_IF:.*]], label %[[PRED_LOAD_CONTINUE:.*]]
-; CHECK: [[PRED_LOAD_IF]]:
-; CHECK-NEXT: [[TMP12:%.*]] = load i32, ptr [[TMP7]], align 4
-; CHECK-NEXT: [[TMP13:%.*]] = insertelement <4 x i32> poison, i32 [[TMP12]], i64 0
-; CHECK-NEXT: br label %[[PRED_LOAD_CONTINUE]]
-; CHECK: [[PRED_LOAD_CONTINUE]]:
-; CHECK-NEXT: [[TMP14:%.*]] = phi <4 x i32> [ poison, %[[IF1]] ], [ [[TMP13]], %[[PRED_LOAD_IF]] ]
-; CHECK-NEXT: br i1 [[C]], label %[[PRED_LOAD_IF3:.*]], label %[[PRED_LOAD_CONTINUE4:.*]]
-; CHECK: [[PRED_LOAD_IF3]]:
-; CHECK-NEXT: [[TMP16:%.*]] = load i32, ptr [[TMP8]], align 4
-; CHECK-NEXT: [[TMP17:%.*]] = insertelement <4 x i32> [[TMP14]], i32 [[TMP16]], i64 1
-; CHECK-NEXT: br label %[[PRED_LOAD_CONTINUE4]]
-; CHECK: [[PRED_LOAD_CONTINUE4]]:
-; CHECK-NEXT: [[TMP18:%.*]] = phi <4 x i32> [ [[TMP14]], %[[PRED_LOAD_CONTINUE]] ], [ [[TMP17]], %[[PRED_LOAD_IF3]] ]
-; CHECK-NEXT: br i1 [[C]], label %[[PRED_LOAD_IF5:.*]], label %[[PRED_LOAD_CONTINUE6:.*]]
-; CHECK: [[PRED_LOAD_IF5]]:
-; CHECK-NEXT: [[TMP20:%.*]] = load i32, ptr [[TMP9]], align 4
-; CHECK-NEXT: [[TMP21:%.*]] = insertelement <4 x i32> [[TMP18]], i32 [[TMP20]], i64 2
-; CHECK-NEXT: br label %[[PRED_LOAD_CONTINUE6]]
-; CHECK: [[PRED_LOAD_CONTINUE6]]:
-; CHECK-NEXT: [[TMP22:%.*]] = phi <4 x i32> [ [[TMP18]], %[[PRED_LOAD_CONTINUE4]] ], [ [[TMP21]], %[[PRED_LOAD_IF5]] ]
-; CHECK-NEXT: br i1 [[C]], label %[[PRED_LOAD_IF7:.*]], label %[[PRED_LOAD_CONTINUE8:.*]]
-; CHECK: [[PRED_LOAD_IF7]]:
-; CHECK-NEXT: [[TMP24:%.*]] = load i32, ptr [[TMP10]], align 4
-; CHECK-NEXT: [[TMP25:%.*]] = insertelement <4 x i32> [[TMP22]], i32 [[TMP24]], i64 3
-; CHECK-NEXT: br label %[[PRED_LOAD_CONTINUE8]]
-; CHECK: [[PRED_LOAD_CONTINUE8]]:
-; CHECK-NEXT: [[TMP23:%.*]] = phi <4 x i32> [ [[TMP22]], %[[PRED_LOAD_CONTINUE6]] ], [ [[TMP25]], %[[PRED_LOAD_IF7]] ]
-; CHECK-NEXT: [[TMP27:%.*]] = udiv exact <4 x i32> [[TMP23]], splat (i32 100)
-; CHECK-NEXT: br i1 [[C]], label %[[PRED_STORE_IF:.*]], label %[[PRED_STORE_CONTINUE:.*]]
-; CHECK: [[PRED_STORE_IF]]:
-; CHECK-NEXT: [[TMP28:%.*]] = extractelement <4 x i32> [[TMP27]], i64 0
-; CHECK-NEXT: store i32 [[TMP28]], ptr [[TMP7]], align 4
-; CHECK-NEXT: br label %[[PRED_STORE_CONTINUE]]
-; CHECK: [[PRED_STORE_CONTINUE]]:
-; CHECK-NEXT: br i1 [[C]], label %[[PRED_STORE_IF9:.*]], label %[[PRED_STORE_CONTINUE10:.*]]
-; CHECK: [[PRED_STORE_IF9]]:
-; CHECK-NEXT: [[TMP29:%.*]] = extractelement <4 x i32> [[TMP27]], i64 1
-; CHECK-NEXT: store i32 [[TMP29]], ptr [[TMP8]], align 4
-; CHECK-NEXT: br label %[[PRED_STORE_CONTINUE10]]
-; CHECK: [[PRED_STORE_CONTINUE10]]:
-; CHECK-NEXT: br i1 [[C]], label %[[PRED_STORE_IF11:.*]], label %[[PRED_STORE_CONTINUE12:.*]]
-; CHECK: [[PRED_STORE_IF11]]:
-; CHECK-NEXT: [[TMP30:%.*]] = extractelement <4 x i32> [[TMP27]], i64 2
-; CHECK-NEXT: store i32 [[TMP30]], ptr [[TMP9]], align 4
-; CHECK-NEXT: br label %[[PRED_STORE_CONTINUE12]]
-; CHECK: [[PRED_STORE_CONTINUE12]]:
-; CHECK-NEXT: br i1 [[C]], label %[[PRED_STORE_IF13:.*]], label %[[PRED_STORE_CONTINUE14:.*]]
-; CHECK: [[PRED_STORE_IF13]]:
-; CHECK-NEXT: [[TMP31:%.*]] = extractelement <4 x i32> [[TMP27]], i64 3
-; CHECK-NEXT: store i32 [[TMP31]], ptr [[TMP10]], align 4
-; CHECK-NEXT: br label %[[PRED_STORE_CONTINUE14]]
-; CHECK: [[PRED_STORE_CONTINUE14]]:
+; CHECK-NEXT: [[TMP25:%.*]] = load <4 x i32>, ptr [[TMP10]], align 4
+; CHECK-NEXT: [[TMP27:%.*]] = udiv exact <4 x i32> [[TMP25]], splat (i32 100)
+; CHECK-NEXT: store <4 x i32> [[TMP27]], ptr [[TMP10]], align 4
; CHECK-NEXT: br label %[[LATCH14]]
; CHECK: [[LATCH14]]:
-; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i32 [[INDEX]], 4
+; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i32 [[TMP4]], 4
; CHECK-NEXT: [[TMP32:%.*]] = icmp eq i32 [[INDEX_NEXT]], [[N_VEC]]
; CHECK-NEXT: br i1 [[TMP32]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP4:![0-9]+]]
; CHECK: [[MIDDLE_BLOCK]]:
diff --git a/llvm/test/Transforms/LoopVectorize/if-pred-stores.ll b/llvm/test/Transforms/LoopVectorize/if-pred-stores.ll
index b57899e31811a..d89ad9ed28ea9 100644
--- a/llvm/test/Transforms/LoopVectorize/if-pred-stores.ll
+++ b/llvm/test/Transforms/LoopVectorize/if-pred-stores.ll
@@ -418,26 +418,18 @@ define void @minimal_bit_widths(ptr %p, i1 %c) {
; UNROLL: [[VECTOR_PH]]:
; UNROLL-NEXT: br label %[[VECTOR_BODY:.*]]
; UNROLL: [[VECTOR_BODY]]:
-; UNROLL-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[PRED_STORE_CONTINUE:.*]] ]
+; UNROLL-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[PRED_STORE_CONTINUE1:.*]] ]
; UNROLL-NEXT: [[TMP0:%.*]] = add i64 [[INDEX]], 1
; UNROLL-NEXT: [[TMP1:%.*]] = getelementptr i8, ptr [[P]], i64 [[INDEX]]
; UNROLL-NEXT: [[TMP2:%.*]] = getelementptr i8, ptr [[P]], i64 [[TMP0]]
; UNROLL-NEXT: [[TMP3:%.*]] = load i8, ptr [[TMP1]], align 1
; UNROLL-NEXT: [[TMP4:%.*]] = load i8, ptr [[TMP2]], align 1
-; UNROLL-NEXT: br i1 [[C]], label %[[PRED_STORE_IF:.*]], label %[[PRED_STORE_CONTINUE]]
-; UNROLL: [[PRED_STORE_IF]]:
-; UNROLL-NEXT: br i1 [[C]], label %[[PRED_STORE_IF2:.*]], label %[[PRED_STORE_CONTINUE1:.*]]
+; UNROLL-NEXT: br i1 [[C]], label %[[PRED_STORE_IF2:.*]], label %[[PRED_STORE_CONTINUE1]]
; UNROLL: [[PRED_STORE_IF2]]:
; UNROLL-NEXT: store i8 [[TMP3]], ptr [[TMP1]], align 1
+; UNROLL-NEXT: store i8 [[TMP4]], ptr [[TMP2]], align 1
; UNROLL-NEXT: br label %[[PRED_STORE_CONTINUE1]]
; UNROLL: [[PRED_STORE_CONTINUE1]]:
-; UNROLL-NEXT: br i1 [[C]], label %[[PRED_STORE_IF1:.*]], label %[[PRED_STORE_CONTINUE2:.*]]
-; UNROLL: [[PRED_STORE_IF1]]:
-; UNROLL-NEXT: store i8 [[TMP4]], ptr [[TMP2]], align 1
-; UNROLL-NEXT: br label %[[PRED_STORE_CONTINUE2]]
-; UNROLL: [[PRED_STORE_CONTINUE2]]:
-; UNROLL-NEXT: br label %[[PRED_STORE_CONTINUE]]
-; UNROLL: [[PRED_STORE_CONTINUE]]:
; UNROLL-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 2
; UNROLL-NEXT: [[TMP5:%.*]] = icmp eq i64 [[INDEX_NEXT]], 1000
; UNROLL-NEXT: br i1 [[TMP5]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP5:![0-9]+]]
@@ -453,26 +445,18 @@ define void @minimal_bit_widths(ptr %p, i1 %c) {
; UNROLL-NOSIMPLIFY: [[VECTOR_PH]]:
; UNROLL-NOSIMPLIFY-NEXT: br label %[[VECTOR_BODY:.*]]
; UNROLL-NOSIMPLIFY: [[VECTOR_BODY]]:
-; UNROLL-NOSIMPLIFY-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[PRED_STORE_CONTINUE:.*]] ]
+; UNROLL-NOSIMPLIFY-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[PRED_STORE_CONTINUE1:.*]] ]
; UNROLL-NOSIMPLIFY-NEXT: [[TMP0:%.*]] = add i64 [[INDEX]], 1
; UNROLL-NOSIMPLIFY-NEXT: [[TMP1:%.*]] = getelementptr i8, ptr [[P]], i64 [[INDEX]]
; UNROLL-NOSIMPLIFY-NEXT: [[TMP2:%.*]] = getelementptr i8, ptr [[P]], i64 [[TMP0]]
; UNROLL-NOSIMPLIFY-NEXT: [[TMP3:%.*]] = load i8, ptr [[TMP1]], align 1
; UNROLL-NOSIMPLIFY-NEXT: [[TMP4:%.*]] = load i8, ptr [[TMP2]], align 1
-; UNROLL-NOSIMPLIFY-NEXT: br i1 [[C]], label %[[PRED_STORE_IF:.*]], label %[[PRED_STORE_CONTINUE]]
-; UNROLL-NOSIMPLIFY: [[PRED_STORE_IF]]:
-; UNROLL-NOSIMPLIFY-NEXT: br i1 [[C]], label %[[PRED_STORE_IF2:.*]], label %[[PRED_STORE_CONTINUE1:.*]]
+; UNROLL-NOSIMPLIFY-NEXT: br i1 [[C]], label %[[PRED_STORE_IF2:.*]], label %[[PRED_STORE_CONTINUE1]]
; UNROLL-NOSIMPLIFY: [[PRED_STORE_IF2]]:
; UNROLL-NOSIMPLIFY-NEXT: store i8 [[TMP3]], ptr [[TMP1]], align 1
+; UNROLL-NOSIMPLIFY-NEXT: store i8 [[TMP4]], ptr [[TMP2]], align 1
; UNROLL-NOSIMPLIFY-NEXT: br label %[[PRED_STORE_CONTINUE1]]
; UNROLL-NOSIMPLIFY: [[PRED_STORE_CONTINUE1]]:
-; UNROLL-NOSIMPLIFY-NEXT: br i1 [[C]], label %[[PRED_STORE_IF1:.*]], label %[[PRED_STORE_CONTINUE2:.*]]
-; UNROLL-NOSIMPLIFY: [[PRED_STORE_IF1]]:
-; UNROLL-NOSIMPLIFY-NEXT: store i8 [[TMP4]], ptr [[TMP2]], align 1
-; UNROLL-NOSIMPLIFY-NEXT: br label %[[PRED_STORE_CONTINUE2]]
-; UNROLL-NOSIMPLIFY: [[PRED_STORE_CONTINUE2]]:
-; UNROLL-NOSIMPLIFY-NEXT: br label %[[PRED_STORE_CONTINUE]]
-; UNROLL-NOSIMPLIFY: [[PRED_STORE_CONTINUE]]:
; UNROLL-NOSIMPLIFY-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 2
; UNROLL-NOSIMPLIFY-NEXT: [[TMP5:%.*]] = icmp eq i64 [[INDEX_NEXT]], 1000
; UNROLL-NOSIMPLIFY-NEXT: br i1 [[TMP5]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP5:![0-9]+]]
@@ -488,28 +472,15 @@ define void @minimal_bit_widths(ptr %p, i1 %c) {
; VEC: [[VECTOR_PH]]:
; VEC-NEXT: br label %[[VECTOR_BODY:.*]]
; VEC: [[VECTOR_BODY]]:
-; VEC-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[PRED_STORE_CONTINUE:.*]] ]
-; VEC-NEXT: [[TMP0:%.*]] = getelementptr i8, ptr [[P]], i64 [[INDEX]]
-; VEC-NEXT: [[WIDE_LOAD:%.*]] = load <2 x i8>, ptr [[TMP0]], align 1
-; VEC-NEXT: br i1 [[C]], label %[[PRED_STORE_IF:.*]], label %[[PRED_STORE_CONTINUE]]
-; VEC: [[PRED_STORE_IF]]:
-; VEC-NEXT: br i1 [[C]], label %[[PRED_STORE_IF2:.*]], label %[[PRED_STORE_CONTINUE1:.*]]
+; VEC-NEXT: [[TMP2:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[PRED_STORE_CONTINUE1:.*]] ]
+; VEC-NEXT: [[TMP3:%.*]] = getelementptr i8, ptr [[P]], i64 [[TMP2]]
+; VEC-NEXT: [[WIDE_LOAD:%.*]] = load <2 x i8>, ptr [[TMP3]], align 1
+; VEC-NEXT: br i1 [[C]], label %[[PRED_STORE_IF2:.*]], label %[[PRED_STORE_CONTINUE1]]
; VEC: [[PRED_STORE_IF2]]:
-; VEC-NEXT: [[TMP1:%.*]] = extractelement <2 x i8> [[WIDE_LOAD]], i64 0
-; VEC-NEXT: store i8 [[TMP1]], ptr [[TMP0]], align 1
+; VEC-NEXT: store <2 x i8> [[WIDE_LOAD]], ptr [[TMP3]], align 1
; VEC-NEXT: br label %[[PRED_STORE_CONTINUE1]]
; VEC: [[PRED_STORE_CONTINUE1]]:
-; VEC-NEXT: br i1 [[C]], label %[[PRED_STORE_IF1:.*]], label %[[PRED_STORE_CONTINUE2:.*]]
-; VEC: [[PRED_STORE_IF1]]:
-; VEC-NEXT: [[TMP2:%.*]] = add i64 [[INDEX]], 1
-; VEC-NEXT: [[TMP3:%.*]] = getelementptr i8, ptr [[P]], i64 [[TMP2]]
-; VEC-NEXT: [[TMP4:%.*]] = extractelement <2 x i8> [[WIDE_LOAD]], i64 1
-; VEC-NEXT: store i8 [[TMP4]], ptr [[TMP3]], align 1
-; VEC-NEXT: br label %[[PRED_STORE_CONTINUE2]]
-; VEC: [[PRED_STORE_CONTINUE2]]:
-; VEC-NEXT: br label %[[PRED_STORE_CONTINUE]]
-; VEC: [[PRED_STORE_CONTINUE]]:
-; VEC-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 2
+; VEC-NEXT: [[INDEX_NEXT]] = add nuw i64 [[TMP2]], 2
; VEC-NEXT: [[TMP5:%.*]] = icmp eq i64 [[INDEX_NEXT]], 1000
; VEC-NEXT: br i1 [[TMP5]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP5:![0-9]+]]
; VEC: [[MIDDLE_BLOCK]]:
@@ -551,7 +522,7 @@ define void @minimal_bit_widths_with_aliasing_store(i1 %c, ptr %ptr) {
; UNROLL: [[VECTOR_PH]]:
; UNROLL-NEXT: br label %[[VECTOR_BODY:.*]]
; UNROLL: [[VECTOR_BODY]]:
-; UNROLL-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[PRED_STORE_CONTINUE:.*]] ]
+; UNROLL-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[PRED_STORE_CONTINUE1:.*]] ]
; UNROLL-NEXT: [[TMP0:%.*]] = add i64 [[INDEX]], 1
; UNROLL-NEXT: [[TMP1:%.*]] = getelementptr i8, ptr [[PTR]], i64 [[INDEX]]
; UNROLL-NEXT: [[TMP2:%.*]] = getelementptr i8, ptr [[PTR]], i64 [[TMP0]]
@@ -559,20 +530,12 @@ define void @minimal_bit_widths_with_aliasing_store(i1 %c, ptr %ptr) {
; UNROLL-NEXT: [[TMP4:%.*]] = load i8, ptr [[TMP2]], align 1
; UNROLL-NEXT: store i8 0, ptr [[TMP1]], align 1
; UNROLL-NEXT: store i8 0, ptr [[TMP2]], align 1
-; UNROLL-NEXT: br i1 [[C]], label %[[PRED_STORE_IF:.*]], label %[[PRED_STORE_CONTINUE]]
-; UNROLL: [[PRED_STORE_IF]]:
-; UNROLL-NEXT: br i1 [[C]], label %[[PRED_STORE_IF2:.*]], label %[[PRED_STORE_CONTINUE1:.*]]
+; UNROLL-NEXT: br i1 [[C]], label %[[PRED_STORE_IF2:.*]], label %[[PRED_STORE_CONTINUE1]]
; UNROLL: [[PRED_STORE_IF2]]:
; UNROLL-NEXT: store i8 [[TMP3]], ptr [[TMP1]], align 1
+; UNROLL-NEXT: store i8 [[TMP4]], ptr [[TMP2]], align 1
; UNROLL-NEXT: br label %[[PRED_STORE_CONTINUE1]]
; UNROLL: [[PRED_STORE_CONTINUE1]]:
-; UNROLL-NEXT: br i1 [[C]], label %[[PRED_STORE_IF1:.*]], label %[[PRED_STORE_CONTINUE2:.*]]
-; UNROLL: [[PRED_STORE_IF1]]:
-; UNROLL-NEXT: store i8 [[TMP4]], ptr [[TMP2]], align 1
-; UNROLL-NEXT: br label %[[PRED_STORE_CONTINUE2]]
-; UNROLL: [[PRED_STORE_CONTINUE2]]:
-; UNROLL-NEXT: br label %[[PRED_STORE_CONTINUE]]
-; UNROLL: [[PRED_STORE_CONTINUE]]:
; UNROLL-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 2
; UNROLL-NEXT: [[TMP5:%.*]] = icmp eq i64 [[INDEX_NEXT]], 1000
; UNROLL-NEXT: br i1 [[TMP5]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP6:![0-9]+]]
@@ -588,7 +551,7 @@ define void @minimal_bit_widths_with_aliasing_store(i1 %c, ptr %ptr) {
; UNROLL-NOSIMPLIFY: [[VECTOR_PH]]:
; UNROLL-NOSIMPLIFY-NEXT: br label %[[VECTOR_BODY:.*]]
; UNROLL-NOSIMPLIFY: [[VECTOR_BODY]]:
-; UNROLL-NOSIMPLIFY-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[PRED_STORE_CONTINUE:.*]] ]
+; UNROLL-NOSIMPLIFY-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[PRED_STORE_CONTINUE1:.*]] ]
; UNROLL-NOSIMPLIFY-NEXT: [[TMP0:%.*]] = add i64 [[INDEX]], 1
; UNROLL-NOSIMPLIFY-NEXT: [[TMP1:%.*]] = getelementptr i8, ptr [[PTR]], i64 [[INDEX]]
; UNROLL-NOSIMPLIFY-NEXT: [[TMP2:%.*]] = getelementptr i8, ptr [[PTR]], i64 [[TMP0]]
@@ -596,20 +559,12 @@ define void @minimal_bit_widths_with_aliasing_store(i1 %c, ptr %ptr) {
; UNROLL-NOSIMPLIFY-NEXT: [[TMP4:%.*]] = load i8, ptr [[TMP2]], align 1
; UNROLL-NOSIMPLIFY-NEXT: store i8 0, ptr [[TMP1]], align 1
; UNROLL-NOSIMPLIFY-NEXT: store i8 0, ptr [[TMP2]], align 1
-; UNROLL-NOSIMPLIFY-NEXT: br i1 [[C]], label %[[PRED_STORE_IF:.*]], label %[[PRED_STORE_CONTINUE]]
-; UNROLL-NOSIMPLIFY: [[PRED_STORE_IF]]:
-; UNROLL-NOSIMPLIFY-NEXT: br i1 [[C]], label %[[PRED_STORE_IF2:.*]], label %[[PRED_STORE_CONTINUE1:.*]]
+; UNROLL-NOSIMPLIFY-NEXT: br i1 [[C]], label %[[PRED_STORE_IF2:.*]], label %[[PRED_STORE_CONTINUE1]]
; UNROLL-NOSIMPLIFY: [[PRED_STORE_IF2]]:
; UNROLL-NOSIMPLIFY-NEXT: store i8 [[TMP3]], ptr [[TMP1]], align 1
+; UNROLL-NOSIMPLIFY-NEXT: store i8 [[TMP4]], ptr [[TMP2]], align 1
; UNROLL-NOSIMPLIFY-NEXT: br label %[[PRED_STORE_CONTINUE1]]
; UNROLL-NOSIMPLIFY: [[PRED_STORE_CONTINUE1]]:
-; UNROLL-NOSIMPLIFY-NEXT: br i1 [[C]], label %[[PRED_STORE_IF1:.*]], label %[[PRED_STORE_CONTINUE2:.*]]
-; UNROLL-NOSIMPLIFY: [[PRED_STORE_IF1]]:
-; UNROLL-NOSIMPLIFY-NEXT: store i8 [[TMP4]], ptr [[TMP2]], align 1
-; UNROLL-NOSIMPLIFY-NEXT: br label %[[PRED_STORE_CONTINUE2]]
-; UNROLL-NOSIMPLIFY: [[PRED_STORE_CONTINUE2]]:
-; UNROLL-NOSIMPLIFY-NEXT: br label %[[PRED_STORE_CONTINUE]]
-; UNROLL-NOSIMPLIFY: [[PRED_STORE_CONTINUE]]:
; UNROLL-NOSIMPLIFY-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 2
; UNROLL-NOSIMPLIFY-NEXT: [[TMP5:%.*]] = icmp eq i64 [[INDEX_NEXT]], 1000
; UNROLL-NOSIMPLIFY-NEXT: br i1 [[TMP5]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP6:![0-9]+]]
@@ -625,29 +580,16 @@ define void @minimal_bit_widths_with_aliasing_store(i1 %c, ptr %ptr) {
; VEC: [[VECTOR_PH]]:
; VEC-NEXT: br label %[[VECTOR_BODY:.*]]
; VEC: [[VECTOR_BODY]]:
-; VEC-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[PRED_STORE_CONTINUE:.*]] ]
-; VEC-NEXT: [[TMP0:%.*]] = getelementptr i8, ptr [[PTR]], i64 [[INDEX]]
-; VEC-NEXT: [[WIDE_LOAD:%.*]] = load <2 x i8>, ptr [[TMP0]], align 1
-; VEC-NEXT: store <2 x i8> zeroinitializer, ptr [[TMP0]], align 1
-; VEC-NEXT: br i1 [[C]], label %[[PRED_STORE_IF:.*]], label %[[PRED_STORE_CONTINUE]]
-; VEC: [[PRED_STORE_IF]]:
-; VEC-NEXT: br i1 [[C]], label %[[PRED_STORE_IF2:.*]], label %[[PRED_STORE_CONTINUE1:.*]]
+; VEC-NEXT: [[TMP2:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[PRED_STORE_CONTINUE1:.*]] ]
+; VEC-NEXT: [[TMP3:%.*]] = getelementptr i8, ptr [[PTR]], i64 [[TMP2]]
+; VEC-NEXT: [[WIDE_LOAD:%.*]] = load <2 x i8>, ptr [[TMP3]], align 1
+; VEC-NEXT: store <2 x i8> zeroinitializer, ptr [[TMP3]], align 1
+; VEC-NEXT: br i1 [[C]], label %[[PRED_STORE_IF2:.*]], label %[[PRED_STORE_CONTINUE1]]
; VEC: [[PRED_STORE_IF2]]:
-; VEC-NEXT: [[TMP1:%.*]] = extractelement <2 x i8> [[WIDE_LOAD]], i64 0
-; VEC-NEXT: store i8 [[TMP1]], ptr [[TMP0]], align 1
+; VEC-NEXT: store <2 x i8> [[WIDE_LOAD]], ptr [[TMP3]], align 1
; VEC-NEXT: br label %[[PRED_STORE_CONTINUE1]]
; VEC: [[PRED_STORE_CONTINUE1]]:
-; VEC-NEXT: br i1 [[C]], label %[[PRED_STORE_IF1:.*]], label %[[PRED_STORE_CONTINUE2:.*]]
-; VEC: [[PRED_STORE_IF1]]:
-; VEC-NEXT: [[TMP2:%.*]] = add i64 [[INDEX]], 1
-; VEC-NEXT: [[TMP3:%.*]] = getelementptr i8, ptr [[PTR]], i64 [[TMP2]]
-; VEC-NEXT: [[TMP4:%.*]] = extractelement <2 x i8> [[WIDE_LOAD]], i64 1
-; VEC-NEXT: store i8 [[TMP4]], ptr [[TMP3]], align 1
-; VEC-NEXT: br label %[[PRED_STORE_CONTINUE2]]
-; VEC: [[PRED_STORE_CONTINUE2]]:
-; VEC-NEXT: br label %[[PRED_STORE_CONTINUE]]
-; VEC: [[PRED_STORE_CONTINUE]]:
-; VEC-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 2
+; VEC-NEXT: [[INDEX_NEXT]] = add nuw i64 [[TMP2]], 2
; VEC-NEXT: [[TMP5:%.*]] = icmp eq i64 [[INDEX_NEXT]], 1000
; VEC-NEXT: br i1 [[TMP5]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP6:![0-9]+]]
; VEC: [[MIDDLE_BLOCK]]:
@@ -685,31 +627,21 @@ for.end:
define void @sdiv_with_uniform_ops(i16 %0, i1 %c, ptr %dst) {
; UNROLL-LABEL: define void @sdiv_with_uniform_ops(
; UNROLL-SAME: i16 [[TMP0:%.*]], i1 [[C:%.*]], ptr [[DST:%.*]]) {
-; UNROLL-NEXT: [[ENTRY:.*:]]
-; UNROLL-NEXT: br label %[[VECTOR_PH:.*]]
-; UNROLL: [[VECTOR_PH]]:
+; UNROLL-NEXT: [[VECTOR_PH:.*:]]
; UNROLL-NEXT: br label %[[VECTOR_BODY:.*]]
; UNROLL: [[VECTOR_BODY]]:
-; UNROLL-NEXT: [[INDEX:%.*]] = phi i32 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[PRED_STORE_CONTINUE:.*]] ]
-; UNROLL-NEXT: br i1 [[C]], label %[[PRED_STORE_IF:.*]], label %[[PRED_STORE_CONTINUE]]
-; UNROLL: [[PRED_STORE_IF]]:
-; UNROLL-NEXT: br i1 [[C]], label %[[PRED_STORE_IF2:.*]], label %[[PRED_STORE_CONTINUE1:.*]]
-; UNROLL: [[PRED_STORE_IF2]]:
; UNROLL-NEXT: [[TMP1:%.*]] = sdiv i16 10, [[TMP0]]
-; UNROLL-NEXT: store i16 [[TMP1]], ptr [[DST]], align 1
-; UNROLL-NEXT: br label %[[PRED_STORE_CONTINUE1]]
+; UNROLL-NEXT: br label %[[PRED_STORE_CONTINUE1:.*]]
; UNROLL: [[PRED_STORE_CONTINUE1]]:
-; UNROLL-NEXT: br i1 [[C]], label %[[PRED_STORE_IF1:.*]], label %[[PRED_STORE_CONTINUE2:.*]]
+; UNROLL-NEXT: [[INDEX:%.*]] = phi i32 [ 0, %[[VECTOR_BODY]] ], [ [[INDEX_NEXT:%.*]], %[[PRED_STORE_CONTINUE2:.*]] ]
+; UNROLL-NEXT: br i1 [[C]], label %[[PRED_STORE_IF1:.*]], label %[[PRED_STORE_CONTINUE2]]
; UNROLL: [[PRED_STORE_IF1]]:
-; UNROLL-NEXT: [[TMP2:%.*]] = sdiv i16 10, [[TMP0]]
-; UNROLL-NEXT: store i16 [[TMP2]], ptr [[DST]], align 1
+; UNROLL-NEXT: store i16 [[TMP1]], ptr [[DST]], align 1
; UNROLL-NEXT: br label %[[PRED_STORE_CONTINUE2]]
; UNROLL: [[PRED_STORE_CONTINUE2]]:
-; UNROLL-NEXT: br label %[[PRED_STORE_CONTINUE]]
-; UNROLL: [[PRED_STORE_CONTINUE]]:
; UNROLL-NEXT: [[INDEX_NEXT]] = add nuw i32 [[INDEX]], 2
; UNROLL-NEXT: [[TMP3:%.*]] = icmp eq i32 [[INDEX_NEXT]], 98
-; UNROLL-NEXT: br i1 [[TMP3]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP7:![0-9]+]]
+; UNROLL-NEXT: br i1 [[TMP3]], label %[[MIDDLE_BLOCK:.*]], label %[[PRED_STORE_CONTINUE1]], !llvm.loop [[LOOP7:![0-9]+]]
; UNROLL: [[MIDDLE_BLOCK]]:
; UNROLL-NEXT: br label %[[SCALAR_PH:.*]]
; UNROLL: [[SCALAR_PH]]:
@@ -730,31 +662,21 @@ define void @sdiv_with_uniform_ops(i16 %0, i1 %c, ptr %dst) {
;
; UNROLL-NOSIMPLIFY-LABEL: define void @sdiv_with_uniform_ops(
; UNROLL-NOSIMPLIFY-SAME: i16 [[TMP0:%.*]], i1 [[C:%.*]], ptr [[DST:%.*]]) {
-; UNROLL-NOSIMPLIFY-NEXT: [[ENTRY:.*:]]
-; UNROLL-NOSIMPLIFY-NEXT: br label %[[VECTOR_PH:.*]]
-; UNROLL-NOSIMPLIFY: [[VECTOR_PH]]:
+; UNROLL-NOSIMPLIFY-NEXT: [[VECTOR_PH:.*:]]
; UNROLL-NOSIMPLIFY-NEXT: br label %[[VECTOR_BODY:.*]]
; UNROLL-NOSIMPLIFY: [[VECTOR_BODY]]:
-; UNROLL-NOSIMPLIFY-NEXT: [[INDEX:%.*]] = phi i32 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[PRED_STORE_CONTINUE:.*]] ]
-; UNROLL-NOSIMPLIFY-NEXT: br i1 [[C]], label %[[PRED_STORE_IF:.*]], label %[[PRED_STORE_CONTINUE]]
-; UNROLL-NOSIMPLIFY: [[PRED_STORE_IF]]:
-; UNROLL-NOSIMPLIFY-NEXT: br i1 [[C]], label %[[PRED_STORE_IF2:.*]], label %[[PRED_STORE_CONTINUE1:.*]]
-; UNROLL-NOSIMPLIFY: [[PRED_STORE_IF2]]:
; UNROLL-NOSIMPLIFY-NEXT: [[TMP1:%.*]] = sdiv i16 10, [[TMP0]]
-; UNROLL-NOSIMPLIFY-NEXT: store i16 [[TMP1]], ptr [[DST]], align 1
-; UNROLL-NOSIMPLIFY-NEXT: br label %[[PRED_STORE_CONTINUE1]]
+; UNROLL-NOSIMPLIFY-NEXT: br label %[[PRED_STORE_CONTINUE1:.*]]
; UNROLL-NOSIMPLIFY: [[PRED_STORE_CONTINUE1]]:
-; UNROLL-NOSIMPLIFY-NEXT: br i1 [[C]], label %[[PRED_STORE_IF1:.*]], label %[[PRED_STORE_CONTINUE2:.*]]
+; UNROLL-NOSIMPLIFY-NEXT: [[INDEX:%.*]] = phi i32 [ 0, %[[VECTOR_BODY]] ], [ [[INDEX_NEXT:%.*]], %[[PRED_STORE_CONTINUE2:.*]] ]
+; UNROLL-NOSIMPLIFY-NEXT: br i1 [[C]], label %[[PRED_STORE_IF1:.*]], label %[[PRED_STORE_CONTINUE2]]
; UNROLL-NOSIMPLIFY: [[PRED_STORE_IF1]]:
-; UNROLL-NOSIMPLIFY-NEXT: [[TMP2:%.*]] = sdiv i16 10, [[TMP0]]
-; UNROLL-NOSIMPLIFY-NEXT: store i16 [[TMP2]], ptr [[DST]], align 1
+; UNROLL-NOSIMPLIFY-NEXT: store i16 [[TMP1]], ptr [[DST]], align 1
; UNROLL-NOSIMPLIFY-NEXT: br label %[[PRED_STORE_CONTINUE2]]
; UNROLL-NOSIMPLIFY: [[PRED_STORE_CONTINUE2]]:
-; UNROLL-NOSIMPLIFY-NEXT: br label %[[PRED_STORE_CONTINUE]]
-; UNROLL-NOSIMPLIFY: [[PRED_STORE_CONTINUE]]:
; UNROLL-NOSIMPLIFY-NEXT: [[INDEX_NEXT]] = add nuw i32 [[INDEX]], 2
; UNROLL-NOSIMPLIFY-NEXT: [[TMP3:%.*]] = icmp eq i32 [[INDEX_NEXT]], 98
-; UNROLL-NOSIMPLIFY-NEXT: br i1 [[TMP3]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP7:![0-9]+]]
+; UNROLL-NOSIMPLIFY-NEXT: br i1 [[TMP3]], label %[[MIDDLE_BLOCK:.*]], label %[[PRED_STORE_CONTINUE1]], !llvm.loop [[LOOP7:![0-9]+]]
; UNROLL-NOSIMPLIFY: [[MIDDLE_BLOCK]]:
; UNROLL-NOSIMPLIFY-NEXT: br label %[[SCALAR_PH:.*]]
; UNROLL-NOSIMPLIFY: [[SCALAR_PH]]:
@@ -775,36 +697,21 @@ define void @sdiv_with_uniform_ops(i16 %0, i1 %c, ptr %dst) {
;
; VEC-LABEL: define void @sdiv_with_uniform_ops(
; VEC-SAME: i16 [[TMP0:%.*]], i1 [[C:%.*]], ptr [[DST:%.*]]) {
-; VEC-NEXT: [[ENTRY:.*:]]
-; VEC-NEXT: br label %[[VECTOR_PH:.*]]
-; VEC: [[VECTOR_PH]]:
-; VEC-NEXT: [[BROADCAST_SPLATINSERT:%.*]] = insertelement <2 x i16> poison, i16 [[TMP0]], i64 0
-; VEC-NEXT: [[BROADCAST_SPLAT:%.*]] = shufflevector <2 x i16> [[BROADCAST_SPLATINSERT]], <2 x i16> poison, <2 x i32> zeroinitializer
-; VEC-NEXT: [[BROADCAST_SPLATINSERT1:%.*]] = insertelement <2 x i1> poison, i1 [[C]], i64 0
-; VEC-NEXT: [[BROADCAST_SPLAT2:%.*]] = shufflevector <2 x i1> [[BROADCAST_SPLATINSERT1]], <2 x i1> poison, <2 x i32> zeroinitializer
-; VEC-NEXT: [[TMP1:%.*]] = call <2 x i16> @llvm.masked.sdiv.v2i16(<2 x i16> splat (i16 10), <2 x i16> [[BROADCAST_SPLAT]], <2 x i1> [[BROADCAST_SPLAT2]])
+; VEC-NEXT: [[VECTOR_PH:.*:]]
; VEC-NEXT: br label %[[VECTOR_BODY:.*]]
; VEC: [[VECTOR_BODY]]:
-; VEC-NEXT: [[INDEX:%.*]] = phi i32 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[PRED_STORE_CONTINUE:.*]] ]
-; VEC-NEXT: br i1 [[C]], label %[[PRED_STORE_IF:.*]], label %[[PRED_STORE_CONTINUE]]
-; VEC: [[PRED_STORE_IF]]:
-; VEC-NEXT: br i1 [[C]], label %[[PRED_STORE_IF1:.*]], label %[[PRED_STORE_CONTINUE1:.*]]
-; VEC: [[PRED_STORE_IF1]]:
-; VEC-NEXT: [[TMP2:%.*]] = extractelement <2 x i16> [[TMP1]], i64 0
-; VEC-NEXT: store i16 [[TMP2]], ptr [[DST]], align 1
-; VEC-NEXT: br label %[[PRED_STORE_CONTINUE1]]
+; VEC-NEXT: [[TMP3:%.*]] = sdiv i16 10, [[TMP0]]
+; VEC-NEXT: br label %[[PRED_STORE_CONTINUE1:.*]]
; VEC: [[PRED_STORE_CONTINUE1]]:
-; VEC-NEXT: br i1 [[C]], label %[[PRED_STORE_IF3:.*]], label %[[PRED_STORE_CONTINUE4:.*]]
+; VEC-NEXT: [[INDEX:%.*]] = phi i32 [ 0, %[[VECTOR_BODY]] ], [ [[INDEX_NEXT:%.*]], %[[PRED_STORE_CONTINUE4:.*]] ]
+; VEC-NEXT: br i1 [[C]], label %[[PRED_STORE_IF3:.*]], label %[[PRED_STORE_CONTINUE4]]
; VEC: [[PRED_STORE_IF3]]:
-; VEC-NEXT: [[TMP3:%.*]] = extractelement <2 x i16> [[TMP1]], i64 1
; VEC-NEXT: store i16 [[TMP3]], ptr [[DST]], align 1
; VEC-NEXT: br label %[[PRED_STORE_CONTINUE4]]
; VEC: [[PRED_STORE_CONTINUE4]]:
-; VEC-NEXT: br label %[[PRED_STORE_CONTINUE]]
-; VEC: [[PRED_STORE_CONTINUE]]:
; VEC-NEXT: [[INDEX_NEXT]] = add nuw i32 [[INDEX]], 2
; VEC-NEXT: [[TMP4:%.*]] = icmp eq i32 [[INDEX_NEXT]], 98
-; VEC-NEXT: br i1 [[TMP4]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP7:![0-9]+]]
+; VEC-NEXT: br i1 [[TMP4]], label %[[MIDDLE_BLOCK:.*]], label %[[PRED_STORE_CONTINUE1]], !llvm.loop [[LOOP7:![0-9]+]]
; VEC: [[MIDDLE_BLOCK]]:
; VEC-NEXT: br label %[[SCALAR_PH:.*]]
; VEC: [[SCALAR_PH]]:
diff --git a/llvm/test/Transforms/LoopVectorize/pr37248.ll b/llvm/test/Transforms/LoopVectorize/pr37248.ll
index 54109d418f5ba..9d86032bd27bc 100644
--- a/llvm/test/Transforms/LoopVectorize/pr37248.ll
+++ b/llvm/test/Transforms/LoopVectorize/pr37248.ll
@@ -35,23 +35,13 @@ define void @f1(ptr noalias %b, i1 %c, i32 %start) {
; CHECK-NEXT: [[N_MOD_VF:%.*]] = and i32 [[TMP1]], 1
; CHECK-NEXT: [[N_VEC:%.*]] = sub i32 [[TMP1]], [[N_MOD_VF]]
; CHECK-NEXT: [[TMP10:%.*]] = sub i32 [[START]], [[N_VEC]]
-; CHECK-NEXT: [[TMP12:%.*]] = xor i1 [[C]], true
; CHECK-NEXT: br label %[[VECTOR_BODY:.*]]
; CHECK: [[VECTOR_BODY]]:
; CHECK-NEXT: [[INDEX:%.*]] = phi i32 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[LOOP_LATCH4:.*]] ]
; CHECK-NEXT: [[OFFSET_IDX:%.*]] = sub i32 [[START]], [[INDEX]]
; CHECK-NEXT: br i1 [[C]], label %[[LOOP_LATCH4]], label %[[ELSE1:.*]]
; CHECK: [[ELSE1]]:
-; CHECK-NEXT: br i1 [[TMP12]], label %[[PRED_STORE_IF:.*]], label %[[PRED_STORE_CONTINUE:.*]]
-; CHECK: [[PRED_STORE_IF]]:
; CHECK-NEXT: store i32 10, ptr [[B]], align 1
-; CHECK-NEXT: br label %[[PRED_STORE_CONTINUE]]
-; CHECK: [[PRED_STORE_CONTINUE]]:
-; CHECK-NEXT: br i1 [[TMP12]], label %[[PRED_STORE_IF2:.*]], label %[[PRED_STORE_CONTINUE3:.*]]
-; CHECK: [[PRED_STORE_IF2]]:
-; CHECK-NEXT: store i32 10, ptr [[B]], align 1
-; CHECK-NEXT: br label %[[PRED_STORE_CONTINUE3]]
-; CHECK: [[PRED_STORE_CONTINUE3]]:
; CHECK-NEXT: br label %[[LOOP_LATCH4]]
; CHECK: [[LOOP_LATCH4]]:
; CHECK-NEXT: [[TMP13:%.*]] = trunc i32 [[OFFSET_IDX]] to i16
diff --git a/llvm/test/Transforms/LoopVectorize/predicator.ll b/llvm/test/Transforms/LoopVectorize/predicator.ll
index 6623c01a01ab5..3e9b0dfbe2caa 100644
--- a/llvm/test/Transforms/LoopVectorize/predicator.ll
+++ b/llvm/test/Transforms/LoopVectorize/predicator.ll
@@ -377,8 +377,6 @@ define void @outermost_uniform_branch_no_phi(ptr %a, i1 %u0) {
; CHECK-NEXT: [[ENTRY:.*:]]
; CHECK-NEXT: br label %[[VECTOR_PH:.*]]
; CHECK: [[VECTOR_PH]]:
-; CHECK-NEXT: [[BROADCAST_SPLATINSERT:%.*]] = insertelement <4 x i1> poison, i1 [[U0]], i64 0
-; CHECK-NEXT: [[BROADCAST_SPLAT:%.*]] = shufflevector <4 x i1> [[BROADCAST_SPLATINSERT]], <4 x i1> poison, <4 x i32> zeroinitializer
; CHECK-NEXT: br label %[[VECTOR_BODY:.*]]
; CHECK: [[VECTOR_BODY]]:
; CHECK-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[BB42:.*]] ]
@@ -388,11 +386,10 @@ define void @outermost_uniform_branch_no_phi(ptr %a, i1 %u0) {
; CHECK-NEXT: br i1 [[U0]], label %[[BB11:.*]], label %[[BB42]]
; CHECK: [[BB11]]:
; CHECK-NEXT: [[TMP1:%.*]] = add <4 x i64> [[VEC_IND]], splat (i64 1)
-; CHECK-NEXT: call void @llvm.masked.store.v4i64.p0(<4 x i64> [[TMP1]], ptr align 4 [[TMP0]], <4 x i1> [[BROADCAST_SPLAT]])
+; CHECK-NEXT: store <4 x i64> [[TMP1]], ptr [[TMP0]], align 4
; CHECK-NEXT: [[TMP2:%.*]] = icmp sle <4 x i64> [[VEC_IND]], splat (i64 1)
-; CHECK-NEXT: [[TMP3:%.*]] = select <4 x i1> [[BROADCAST_SPLAT]], <4 x i1> [[TMP2]], <4 x i1> zeroinitializer
; CHECK-NEXT: [[TMP4:%.*]] = add <4 x i64> [[VEC_IND]], splat (i64 2)
-; CHECK-NEXT: call void @llvm.masked.store.v4i64.p0(<4 x i64> [[TMP4]], ptr align 4 [[TMP0]], <4 x i1> [[TMP3]])
+; CHECK-NEXT: call void @llvm.masked.store.v4i64.p0(<4 x i64> [[TMP4]], ptr align 4 [[TMP0]], <4 x i1> [[TMP2]])
; CHECK-NEXT: br label %[[BB42]]
; CHECK: [[BB42]]:
; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 4
@@ -531,9 +528,6 @@ define void @outermost_uniform_branch_more_blocks_no_phi(ptr %a, i1 %u0) {
; CHECK-NEXT: [[ENTRY:.*:]]
; CHECK-NEXT: br label %[[VECTOR_PH:.*]]
; CHECK: [[VECTOR_PH]]:
-; CHECK-NEXT: [[BROADCAST_SPLATINSERT:%.*]] = insertelement <4 x i1> poison, i1 [[U0]], i64 0
-; CHECK-NEXT: [[BROADCAST_SPLAT:%.*]] = shufflevector <4 x i1> [[BROADCAST_SPLATINSERT]], <4 x i1> poison, <4 x i32> zeroinitializer
-; CHECK-NEXT: [[TMP0:%.*]] = xor <4 x i1> [[BROADCAST_SPLAT]], splat (i1 true)
; CHECK-NEXT: br label %[[VECTOR_BODY:.*]]
; CHECK: [[VECTOR_BODY]]:
; CHECK-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[BB63:.*]] ]
@@ -542,17 +536,15 @@ define void @outermost_uniform_branch_more_blocks_no_phi(ptr %a, i1 %u0) {
; CHECK-NEXT: br i1 [[U0]], label %[[BB22:.*]], label %[[BB11:.*]]
; CHECK: [[BB11]]:
; CHECK-NEXT: [[TMP2:%.*]] = add <4 x i64> [[VEC_IND]], splat (i64 1)
-; CHECK-NEXT: call void @llvm.masked.store.v4i64.p0(<4 x i64> [[TMP2]], ptr align 4 [[TMP1]], <4 x i1> [[TMP0]])
+; CHECK-NEXT: store <4 x i64> [[TMP2]], ptr [[TMP1]], align 4
; CHECK-NEXT: br label %[[BB63]]
; CHECK: [[BB22]]:
; CHECK-NEXT: [[TMP3:%.*]] = icmp sle <4 x i64> [[VEC_IND]], splat (i64 2)
; CHECK-NEXT: [[TMP4:%.*]] = xor <4 x i1> [[TMP3]], splat (i1 true)
-; CHECK-NEXT: [[TMP5:%.*]] = select <4 x i1> [[BROADCAST_SPLAT]], <4 x i1> [[TMP4]], <4 x i1> zeroinitializer
; CHECK-NEXT: [[TMP6:%.*]] = add <4 x i64> [[VEC_IND]], splat (i64 3)
-; CHECK-NEXT: call void @llvm.masked.store.v4i64.p0(<4 x i64> [[TMP6]], ptr align 4 [[TMP1]], <4 x i1> [[TMP5]])
-; CHECK-NEXT: [[TMP7:%.*]] = select <4 x i1> [[BROADCAST_SPLAT]], <4 x i1> [[TMP3]], <4 x i1> zeroinitializer
+; CHECK-NEXT: call void @llvm.masked.store.v4i64.p0(<4 x i64> [[TMP6]], ptr align 4 [[TMP1]], <4 x i1> [[TMP4]])
; CHECK-NEXT: [[TMP8:%.*]] = add <4 x i64> [[VEC_IND]], splat (i64 4)
-; CHECK-NEXT: call void @llvm.masked.store.v4i64.p0(<4 x i64> [[TMP8]], ptr align 4 [[TMP1]], <4 x i1> [[TMP7]])
+; CHECK-NEXT: call void @llvm.masked.store.v4i64.p0(<4 x i64> [[TMP8]], ptr align 4 [[TMP1]], <4 x i1> [[TMP3]])
; CHECK-NEXT: br label %[[BB63]]
; CHECK: [[BB63]]:
; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 4
@@ -693,8 +685,6 @@ define void @uniform_branch_after_varying_branch_no_phi(ptr %a, i1 %u1) {
; CHECK-NEXT: [[ENTRY:.*:]]
; CHECK-NEXT: br label %[[VECTOR_PH:.*]]
; CHECK: [[VECTOR_PH]]:
-; CHECK-NEXT: [[BROADCAST_SPLATINSERT:%.*]] = insertelement <4 x i1> poison, i1 [[U1]], i64 0
-; CHECK-NEXT: [[BROADCAST_SPLAT:%.*]] = shufflevector <4 x i1> [[BROADCAST_SPLATINSERT]], <4 x i1> poison, <4 x i32> zeroinitializer
; CHECK-NEXT: br label %[[VECTOR_BODY:.*]]
; CHECK: [[VECTOR_BODY]]:
; CHECK-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[BB32:.*]] ]
@@ -706,9 +696,8 @@ define void @uniform_branch_after_varying_branch_no_phi(ptr %a, i1 %u1) {
; CHECK-NEXT: call void @llvm.masked.store.v4i64.p0(<4 x i64> [[TMP3]], ptr align 4 [[TMP0]], <4 x i1> [[TMP2]])
; CHECK-NEXT: br i1 [[U1]], label %[[BB21:.*]], label %[[BB32]]
; CHECK: [[BB21]]:
-; CHECK-NEXT: [[TMP4:%.*]] = select <4 x i1> [[TMP2]], <4 x i1> [[BROADCAST_SPLAT]], <4 x i1> zeroinitializer
; CHECK-NEXT: [[TMP5:%.*]] = add <4 x i64> [[VEC_IND]], splat (i64 2)
-; CHECK-NEXT: call void @llvm.masked.store.v4i64.p0(<4 x i64> [[TMP5]], ptr align 4 [[TMP0]], <4 x i1> [[TMP4]])
+; CHECK-NEXT: call void @llvm.masked.store.v4i64.p0(<4 x i64> [[TMP5]], ptr align 4 [[TMP0]], <4 x i1> [[TMP2]])
; CHECK-NEXT: br label %[[BB32]]
; CHECK: [[BB32]]:
; CHECK-NEXT: [[TMP6:%.*]] = add <4 x i64> [[VEC_IND]], splat (i64 4)
@@ -851,9 +840,6 @@ define void @uniform_branch_after_varying_branch_more_blocks_no_phi(ptr %a, i1 %
; CHECK-NEXT: [[ENTRY:.*:]]
; CHECK-NEXT: br label %[[VECTOR_PH:.*]]
; CHECK: [[VECTOR_PH]]:
-; CHECK-NEXT: [[BROADCAST_SPLATINSERT:%.*]] = insertelement <4 x i1> poison, i1 [[U1]], i64 0
-; CHECK-NEXT: [[BROADCAST_SPLAT:%.*]] = shufflevector <4 x i1> [[BROADCAST_SPLATINSERT]], <4 x i1> poison, <4 x i32> zeroinitializer
-; CHECK-NEXT: [[TMP0:%.*]] = xor <4 x i1> [[BROADCAST_SPLAT]], splat (i1 true)
; CHECK-NEXT: br label %[[VECTOR_BODY:.*]]
; CHECK: [[VECTOR_BODY]]:
; CHECK-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[BB43:.*]] ]
@@ -863,14 +849,12 @@ define void @uniform_branch_after_varying_branch_more_blocks_no_phi(ptr %a, i1 %
; CHECK-NEXT: [[TMP3:%.*]] = xor <4 x i1> [[TMP2]], splat (i1 true)
; CHECK-NEXT: br i1 [[U1]], label %[[BB32:.*]], label %[[BB21:.*]]
; CHECK: [[BB21]]:
-; CHECK-NEXT: [[TMP4:%.*]] = select <4 x i1> [[TMP3]], <4 x i1> [[TMP0]], <4 x i1> zeroinitializer
; CHECK-NEXT: [[TMP5:%.*]] = add <4 x i64> [[VEC_IND]], splat (i64 2)
-; CHECK-NEXT: call void @llvm.masked.store.v4i64.p0(<4 x i64> [[TMP5]], ptr align 4 [[TMP1]], <4 x i1> [[TMP4]])
+; CHECK-NEXT: call void @llvm.masked.store.v4i64.p0(<4 x i64> [[TMP5]], ptr align 4 [[TMP1]], <4 x i1> [[TMP3]])
; CHECK-NEXT: br label %[[BB43]]
; CHECK: [[BB32]]:
-; CHECK-NEXT: [[TMP6:%.*]] = select <4 x i1> [[TMP3]], <4 x i1> [[BROADCAST_SPLAT]], <4 x i1> zeroinitializer
; CHECK-NEXT: [[TMP7:%.*]] = add <4 x i64> [[VEC_IND]], splat (i64 3)
-; CHECK-NEXT: call void @llvm.masked.store.v4i64.p0(<4 x i64> [[TMP7]], ptr align 4 [[TMP1]], <4 x i1> [[TMP6]])
+; CHECK-NEXT: call void @llvm.masked.store.v4i64.p0(<4 x i64> [[TMP7]], ptr align 4 [[TMP1]], <4 x i1> [[TMP3]])
; CHECK-NEXT: br label %[[BB43]]
; CHECK: [[BB43]]:
; CHECK-NEXT: [[TMP8:%.*]] = add <4 x i64> [[VEC_IND]], splat (i64 5)
@@ -1018,9 +1002,6 @@ define void @uniform_branch_after_varying_branch_more_blocks_mirrored_no_phi(ptr
; CHECK-NEXT: [[ENTRY:.*:]]
; CHECK-NEXT: br label %[[VECTOR_PH:.*]]
; CHECK: [[VECTOR_PH]]:
-; CHECK-NEXT: [[BROADCAST_SPLATINSERT:%.*]] = insertelement <4 x i1> poison, i1 [[U2]], i64 0
-; CHECK-NEXT: [[BROADCAST_SPLAT:%.*]] = shufflevector <4 x i1> [[BROADCAST_SPLATINSERT]], <4 x i1> poison, <4 x i32> zeroinitializer
-; CHECK-NEXT: [[TMP0:%.*]] = xor <4 x i1> [[BROADCAST_SPLAT]], splat (i1 true)
; CHECK-NEXT: br label %[[VECTOR_BODY:.*]]
; CHECK: [[VECTOR_BODY]]:
; CHECK-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[BB53:.*]] ]
@@ -1032,14 +1013,12 @@ define void @uniform_branch_after_varying_branch_more_blocks_mirrored_no_phi(ptr
; CHECK-NEXT: call void @llvm.masked.store.v4i64.p0(<4 x i64> [[TMP4]], ptr align 4 [[TMP1]], <4 x i1> [[TMP3]])
; CHECK-NEXT: br i1 [[U2]], label %[[BB42:.*]], label %[[BB31:.*]]
; CHECK: [[BB31]]:
-; CHECK-NEXT: [[TMP5:%.*]] = select <4 x i1> [[TMP2]], <4 x i1> [[TMP0]], <4 x i1> zeroinitializer
; CHECK-NEXT: [[TMP6:%.*]] = add <4 x i64> [[VEC_IND]], splat (i64 3)
-; CHECK-NEXT: call void @llvm.masked.store.v4i64.p0(<4 x i64> [[TMP6]], ptr align 4 [[TMP1]], <4 x i1> [[TMP5]])
+; CHECK-NEXT: call void @llvm.masked.store.v4i64.p0(<4 x i64> [[TMP6]], ptr align 4 [[TMP1]], <4 x i1> [[TMP2]])
; CHECK-NEXT: br label %[[BB53]]
; CHECK: [[BB42]]:
-; CHECK-NEXT: [[TMP7:%.*]] = select <4 x i1> [[TMP2]], <4 x i1> [[BROADCAST_SPLAT]], <4 x i1> zeroinitializer
; CHECK-NEXT: [[TMP8:%.*]] = add <4 x i64> [[VEC_IND]], splat (i64 4)
-; CHECK-NEXT: call void @llvm.masked.store.v4i64.p0(<4 x i64> [[TMP8]], ptr align 4 [[TMP1]], <4 x i1> [[TMP7]])
+; CHECK-NEXT: call void @llvm.masked.store.v4i64.p0(<4 x i64> [[TMP8]], ptr align 4 [[TMP1]], <4 x i1> [[TMP2]])
; CHECK-NEXT: br label %[[BB53]]
; CHECK: [[BB53]]:
; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 4
@@ -2074,9 +2053,6 @@ define void @uniform_branch_shared_join_with_varying_no_phi(ptr %a, i1 %u0) {
; CHECK-NEXT: [[ENTRY:.*:]]
; CHECK-NEXT: br label %[[VECTOR_PH:.*]]
; CHECK: [[VECTOR_PH]]:
-; CHECK-NEXT: [[BROADCAST_SPLATINSERT:%.*]] = insertelement <4 x i1> poison, i1 [[U0]], i64 0
-; CHECK-NEXT: [[BROADCAST_SPLAT:%.*]] = shufflevector <4 x i1> [[BROADCAST_SPLATINSERT]], <4 x i1> poison, <4 x i32> zeroinitializer
-; CHECK-NEXT: [[TMP0:%.*]] = xor <4 x i1> [[BROADCAST_SPLAT]], splat (i1 true)
; CHECK-NEXT: br label %[[VECTOR_BODY:.*]]
; CHECK: [[VECTOR_BODY]]:
; CHECK-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[BB43:.*]] ]
@@ -2085,15 +2061,14 @@ define void @uniform_branch_shared_join_with_varying_no_phi(ptr %a, i1 %u0) {
; CHECK-NEXT: br i1 [[U0]], label %[[BB22:.*]], label %[[BB11:.*]]
; CHECK: [[BB11]]:
; CHECK-NEXT: [[TMP2:%.*]] = add <4 x i64> [[VEC_IND]], splat (i64 1)
-; CHECK-NEXT: call void @llvm.masked.store.v4i64.p0(<4 x i64> [[TMP2]], ptr align 4 [[TMP1]], <4 x i1> [[TMP0]])
+; CHECK-NEXT: store <4 x i64> [[TMP2]], ptr [[TMP1]], align 4
; CHECK-NEXT: br label %[[BB43]]
; CHECK: [[BB22]]:
; CHECK-NEXT: [[TMP3:%.*]] = add <4 x i64> [[VEC_IND]], splat (i64 2)
-; CHECK-NEXT: call void @llvm.masked.store.v4i64.p0(<4 x i64> [[TMP3]], ptr align 4 [[TMP1]], <4 x i1> [[BROADCAST_SPLAT]])
+; CHECK-NEXT: store <4 x i64> [[TMP3]], ptr [[TMP1]], align 4
; CHECK-NEXT: [[TMP4:%.*]] = icmp sle <4 x i64> [[VEC_IND]], splat (i64 2)
-; CHECK-NEXT: [[TMP5:%.*]] = select <4 x i1> [[BROADCAST_SPLAT]], <4 x i1> [[TMP4]], <4 x i1> zeroinitializer
; CHECK-NEXT: [[TMP6:%.*]] = add <4 x i64> [[VEC_IND]], splat (i64 3)
-; CHECK-NEXT: call void @llvm.masked.store.v4i64.p0(<4 x i64> [[TMP6]], ptr align 4 [[TMP1]], <4 x i1> [[TMP5]])
+; CHECK-NEXT: call void @llvm.masked.store.v4i64.p0(<4 x i64> [[TMP6]], ptr align 4 [[TMP1]], <4 x i1> [[TMP4]])
; CHECK-NEXT: br label %[[BB43]]
; CHECK: [[BB43]]:
; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 4
More information about the llvm-branch-commits
mailing list