[llvm-branch-commits] [llvm] [VPlan][Predicator] Preserve some uniform control flow (PR #217485)
Andrei Elovikov via llvm-branch-commits
llvm-branch-commits at lists.llvm.org
Fri Aug 28 16:52:13 PDT 2026
https://github.com/eas updated https://github.com/llvm/llvm-project/pull/217485
>From 7409e181d5d0db80dc9c7e5045a4c6d7a7257e85 Mon Sep 17 00:00:00 2001
From: Andrei Elovikov <andrei.elovikov at sifive.com>
Date: Tue, 18 Aug 2026 17:24:40 -0700
Subject: [PATCH 1/2] Implement non-uniform part of partial linearization
algorithm
---
.../Transforms/Vectorize/VPlanPredicator.cpp | 70 +++++++++++++++----
1 file changed, 56 insertions(+), 14 deletions(-)
diff --git a/llvm/lib/Transforms/Vectorize/VPlanPredicator.cpp b/llvm/lib/Transforms/Vectorize/VPlanPredicator.cpp
index 1c9ba77fe3587..85aa306b3a250 100644
--- a/llvm/lib/Transforms/Vectorize/VPlanPredicator.cpp
+++ b/llvm/lib/Transforms/Vectorize/VPlanPredicator.cpp
@@ -492,22 +492,64 @@ void VPPredicator::run() {
if (VPBB != Header)
convertPhisToBlends(cast<VPBasicBlock>(VPBB));
- // Linearize the blocks of the loop into one serial chain.
- VPBlockBase *PrevVPBB = nullptr;
+ using DeferredSuccessorsTy = SmallPtrSet<VPBlockBase *, 4>;
+ DenseMap<VPBlockBase *, DeferredSuccessorsTy> DeferredMap;
+ auto PopDeferred = [&](VPBlockBase *VPBB) -> DeferredSuccessorsTy {
+ auto It = DeferredMap.find(VPBB);
+ if (It != DeferredMap.end()) {
+ DeferredSuccessorsTy Res = std::move(It->second);
+ DeferredMap.erase(It);
+ return Res;
+ }
+ return {};
+ };
+
for (VPBasicBlock *VPBB :
VPBlockUtils::blocksOnly<VPBasicBlock>(BlocksInCompactRPOTOrder)) {
- auto Successors = to_vector(VPBB->getSuccessors());
- if (Successors.size() > 1)
- VPBB->getTerminator()->eraseFromParent();
-
- // Flatten the CFG in the loop. To do so, first disconnect VPBB from its
- // successors. Then connect VPBB to the previously visited VPBB.
- for (auto *Succ : Successors)
- VPBlockUtils::disconnectBlocks(VPBB, Succ);
- if (PrevVPBB)
- VPBlockUtils::connectBlocks(PrevVPBB, VPBB);
-
- PrevVPBB = VPBB;
+ LLVM_DEBUG(dbgs() << "Setting successors for " << VPBB->getName() << "\n");
+ bool PreserveUniform = false;
+ if (PreserveUniform) {
+ /* ... */
+ } else {
+ auto Successors = to_vector(VPBB->getSuccessors());
+ if (Successors.size() > 1)
+ VPBB->getTerminator()->eraseFromParent();
+
+ for (auto *Succ : Successors)
+ VPBlockUtils::disconnectBlocks(VPBB, Succ);
+
+ auto CombinedSuccessors = PopDeferred(VPBB);
+ CombinedSuccessors.insert_range(Successors);
+
+ if (CombinedSuccessors.size() == 0)
+ continue;
+
+ LLVM_DEBUG({
+ dbgs() << "Combined successors: ";
+ for (auto *B : CombinedSuccessors) {
+ dbgs() << " " << B->getName();
+ }
+ dbgs() << "\n";
+ });
+
+ VPBlockBase *Next = *std::min_element(
+ CombinedSuccessors.begin(), CombinedSuccessors.end(),
+ [&](VPBlockBase *A, VPBlockBase *B) {
+ return BlocksInCompactRPOTOrder.getIndex(A) <
+ BlocksInCompactRPOTOrder.getIndex(B);
+ });
+
+ LLVM_DEBUG(dbgs() << "Connecting to: " << Next->getName() << "\n");
+ VPBlockUtils::connectBlocks(VPBB, Next);
+
+ CombinedSuccessors.erase(Next);
+ auto &Entry = DeferredMap[Next];
+ if (Entry.empty()) {
+ Entry = std::move(CombinedSuccessors);
+ } else {
+ Entry.insert_range(CombinedSuccessors);
+ }
+ }
}
}
>From b9e4ef1caed0154f48a9c321bbc98585c83aad96 Mon Sep 17 00:00:00 2001
From: Andrei Elovikov <andrei.elovikov at sifive.com>
Date: Wed, 19 Aug 2026 14:45:40 -0700
Subject: [PATCH 2/2] [VPlan][Predicator] Preserve some uniform control flow
Implements "Partial Control-Flow Linearization" by Simon Moll and
Sebastian Hack. Does **NOT** improve predication/masking yet, so
applicability is artificially narrowed, only some uniform branches are
preserved. In particular, the following is left for future PRs:
* Block masks still contains now-unnecessary term for the preserved
uniform branches.
* Mixed blends/phis aren't supported yet. Detecting where they would be
necessary is as complex as implementing proper support (which would
need either Luke's `reconstructSSA` or Iterated Dominance Frontier),
so we also limit it to a trivial/structured CFG where there's only
single block where those would need to be inserted.
I think even the current version might be enough to start implementing
an alternative to https://github.com/llvm/llvm-project/pull/141900 (see
BOSCC in the paper).
---
.../Transforms/Vectorize/LoopVectorize.cpp | 4 +
.../Transforms/Vectorize/VPlanPredicator.cpp | 108 ++++++++++++++-
.../Transforms/Vectorize/VPlanTransforms.cpp | 9 ++
.../AArch64/conditional-branches-cost.ll | 16 ++-
.../AArch64/sve-predicated-costs.ll | 6 +-
...-interleave-to-widen-memory-multi-block.ll | 28 ++--
.../LoopVectorize/RISCV/dead-ops-cost.ll | 8 +-
.../LoopVectorize/RISCV/predicated-costs.ll | 16 ++-
.../RISCV/predicated-reverse-store.ll | 13 +-
.../LoopVectorize/VPlan/predicator.ll | 83 ++++++------
.../VPlan/vplan-sink-scalars-and-merge-vf1.ll | 7 +
.../VPlan/vplan-sink-scalars-and-merge.ll | 7 +
.../LoopVectorize/X86/masked-store-cost.ll | 6 +-
.../Transforms/LoopVectorize/div-exact.ll | 20 ++-
.../LoopVectorize/if-pred-stores.ll | 126 +++++++++++-------
...r154045-dont-fold-extractelement-livein.ll | 6 +-
llvm/test/Transforms/LoopVectorize/pr37248.ll | 14 +-
.../Transforms/LoopVectorize/predicator.ll | 62 +++++++--
.../Transforms/LoopVectorize/uniform-blend.ll | 40 ++++--
.../unused-blend-mask-for-first-operand.ll | 4 +-
.../PhaseOrdering/loop-vectorize-bfi.ll | 8 +-
21 files changed, 442 insertions(+), 149 deletions(-)
diff --git a/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp b/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
index 9c0b8e88f62c5..dce5ded69332f 100644
--- a/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
+++ b/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
@@ -6722,6 +6722,10 @@ VPlanPtr LoopVectorizationPlanner::tryToBuildVPlan(VPlanPtr Plan,
vputils::onlyFirstLaneUsed(R.getVPSingleValue())))
continue;
auto *VPI = cast<VPInstruction>(&R);
+ // No processing needed for the uniform branch preserved by the
+ // predicator.
+ if (VPI->getOpcode() == VPInstruction::BranchOnCond)
+ continue;
if (!VPI->getUnderlyingValue())
continue;
diff --git a/llvm/lib/Transforms/Vectorize/VPlanPredicator.cpp b/llvm/lib/Transforms/Vectorize/VPlanPredicator.cpp
index 85aa306b3a250..8907acd9faaad 100644
--- a/llvm/lib/Transforms/Vectorize/VPlanPredicator.cpp
+++ b/llvm/lib/Transforms/Vectorize/VPlanPredicator.cpp
@@ -19,6 +19,7 @@
#include "VPlanTransforms.h"
#include "VPlanUtils.h"
#include "llvm/ADT/PostOrderIterator.h"
+#include "llvm/ADT/STLExtras.h"
#define DEBUG_TYPE "vplan-predicator"
@@ -67,6 +68,7 @@ class CompactRPOT {
auto begin() const { return Blocks.begin(); }
auto end() const { return Blocks.end(); }
unsigned getIndex(const VPBlockBase *BB) const { return BlockIndex.at(BB); }
+ unsigned size() const { return Blocks.size(); }
};
class VPPredicator {
@@ -111,6 +113,8 @@ class VPPredicator {
/// Create a logical-or, factoring out a common header mask if present.
VPValue *createMaskOr(VPValue *LHS, VPValue *RHS, DebugLoc DL);
+ bool shouldPreserveTerminator(VPBasicBlock *VPBB);
+
/// Record \p Mask as the *entry* mask of \p VPBB, which is expected to not
/// already have a mask.
void setBlockInMask(const VPBasicBlock *VPBB, VPValue *Mask) {
@@ -210,6 +214,59 @@ VPValue *VPPredicator::createMaskOr(VPValue *LHS, VPValue *RHS, DebugLoc DL) {
HeaderMask, Builder.createOr(LHSRemainder, RHSRemainder, DL), DL);
}
+bool VPPredicator::shouldPreserveTerminator(VPBasicBlock *VPBB) {
+ LLVM_DEBUG(dbgs() << "Checking if can preserve the branch at the end of "
+ << VPBB->getName() << "\n");
+ auto False = []([[maybe_unused]] StringRef Reason = "") {
+ LLVM_DEBUG(dbgs() << " can't be preserved"
+ << (Reason.empty() ? Twine() : (": " + Reason)) << "\n");
+ return false;
+ };
+ if (VPBB->getNumSuccessors() != 2)
+ return False("#Successors != 2");
+ auto *Term = dyn_cast<VPInstruction>(VPBB->getTerminator());
+ if (!Term || Term->getOpcode() != VPInstruction::BranchOnCond)
+ return False("Not a branch");
+
+ bool IsUniformAndAvailable = [&](VPValue *V) {
+ auto *IRV = dyn_cast<VPIRValue>(V);
+ return IRV && isa<Argument, Constant>(IRV->getValue());
+ }(Term->getOperand(0));
+
+ if (!IsUniformAndAvailable)
+ return False("non-uniform");
+
+ // Should not happen in the end-to-end pass pipeline (simplifycfg would
+ // have handled it), but possible when running `loop-vectorize` pass
+ // alone. Just don't preserve so that we won't have to handle that when
+ // executing VPlan.
+ if (all_equal(VPBB->successors()))
+ return False("All successors are the same");
+
+ auto *IPostDomNode = VPPDT.getNode(VPBB)->getIDom();
+ if (!IPostDomNode)
+ return False("no post-dom");
+ auto *IPostDom = dyn_cast<VPBasicBlock>(IPostDomNode->getBlock());
+ if (!IPostDom || !VPDT.properlyDominates(VPBB, IPostDom))
+ return False("doesn't dominate its post-dom");
+
+ LLVM_DEBUG(dbgs() << "IPostDom: " << IPostDom->getName() << "\n";);
+
+ for (VPBlockBase *Pred : IPostDom->getPredecessors()) {
+ if (Pred == VPBB)
+ continue;
+ if (count_if(VPBB->successors(),
+ [&](auto *Succ) { return VPDT.dominates(Succ, Pred); }) != 1)
+ return False("Bailing out due to successors not being independent");
+ }
+
+ if (any_of(*IPostDom, IsaPred<VPBlendRecipe>))
+ return False("Blends at reconvergence");
+
+ LLVM_DEBUG(dbgs() << "...yes, can be preserved.\n");
+ return true;
+}
+
VPValue *VPPredicator::createEdgeMask(const VPBasicBlock *Src,
const VPBasicBlock *Dst) {
assert(is_contained(Dst->getPredecessors(), Src) && "Invalid edge");
@@ -484,7 +541,8 @@ void VPPredicator::run() {
// Mask all VPInstructions in the block.
for (VPRecipeBase &R : *VPBB) {
if (auto *VPI = dyn_cast<VPInstruction>(&R))
- VPI->addMask(BlockMask);
+ if (VPI->getOpcode() != VPInstruction::BranchOnCond)
+ VPI->addMask(BlockMask);
}
}
@@ -507,11 +565,53 @@ void VPPredicator::run() {
for (VPBasicBlock *VPBB :
VPBlockUtils::blocksOnly<VPBasicBlock>(BlocksInCompactRPOTOrder)) {
LLVM_DEBUG(dbgs() << "Setting successors for " << VPBB->getName() << "\n");
- bool PreserveUniform = false;
- if (PreserveUniform) {
+ auto Successors = to_vector(VPBB->getSuccessors());
+
+ if (shouldPreserveTerminator(VPBB)) {
+ for (auto *Succ : Successors)
+ VPBlockUtils::disconnectBlocks(VPBB, Succ);
+
+ auto Deferred = PopDeferred(VPBB);
+ LLVM_DEBUG({
+ dbgs() << "Deferred successors: ";
+ for (auto *B : Deferred) {
+ dbgs() << " " << B->getName();
+ }
+ dbgs() << "\n";
+ });
+
+ VPBlockBase *MinDeferred = nullptr;
+ unsigned MinDeferredIdx = BlocksInCompactRPOTOrder.size() + 1;
+
+ if (!Deferred.empty()) {
+ MinDeferred =
+ *std::min_element(Deferred.begin(), Deferred.end(),
+ [&](VPBlockBase *A, VPBlockBase *B) {
+ return BlocksInCompactRPOTOrder.getIndex(A) <
+ BlocksInCompactRPOTOrder.getIndex(B);
+ });
+ MinDeferredIdx = BlocksInCompactRPOTOrder.getIndex(MinDeferred);
+ LLVM_DEBUG(dbgs() << "Min deferred: " << MinDeferred->getName() << "("
+ << MinDeferredIdx << ")\n");
+ }
+ for (auto *Succ : Successors) {
+ VPBlockBase *Next =
+ BlocksInCompactRPOTOrder.getIndex(Succ) < MinDeferredIdx
+ ? Succ
+ : MinDeferred;
+ LLVM_DEBUG(dbgs() << "Original Succ: " << Succ->getName()
+ << ", connecting to " << Next->getName() << "\n");
+ VPBlockUtils::connectBlocks(VPBB, Next);
+ auto &Entry = DeferredMap[Next];
+ assert(Entry.count(Next) == 0 &&
+ "If that fails, then erase below must be done on Deferred/Succ "
+ "before insertion.");
+ Entry.insert_range(Deferred);
+ Entry.insert(Succ);
+ Entry.erase(Next);
+ }
/* ... */
} else {
- auto Successors = to_vector(VPBB->getSuccessors());
if (Successors.size() > 1)
VPBB->getTerminator()->eraseFromParent();
diff --git a/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp b/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp
index 2216b17499096..a0775ca6e4345 100644
--- a/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp
+++ b/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp
@@ -256,6 +256,15 @@ static bool
canHoistOrSinkWithNoAliasCheck(const MemoryLocation &MemLoc,
VPBasicBlock *FirstBB, VPBasicBlock *LastBB,
std::optional<SinkStoreInfo> SinkInfo = {}) {
+ // Predicated memory motion currently only handles a linear VPlan CFG. A
+ // preserved uniform branch may put FirstBB and LastBB on different paths;
+ // conservatively decline the transformation in that case.
+ VPBlockBase *BB = FirstBB;
+ while (BB && BB != LastBB)
+ BB = BB->getSingleSuccessor();
+ if (BB != LastBB)
+ return false;
+
bool CheckReads = SinkInfo.has_value();
for (VPBasicBlock *VPBB :
VPBlockUtils::blocksInSingleSuccessorChainBetween(FirstBB, LastBB)) {
diff --git a/llvm/test/Transforms/LoopVectorize/AArch64/conditional-branches-cost.ll b/llvm/test/Transforms/LoopVectorize/AArch64/conditional-branches-cost.ll
index 6be0aec66125f..86a3938a24a41 100644
--- a/llvm/test/Transforms/LoopVectorize/AArch64/conditional-branches-cost.ll
+++ b/llvm/test/Transforms/LoopVectorize/AArch64/conditional-branches-cost.ll
@@ -1008,18 +1008,22 @@ define void @redundant_branch_and_tail_folding(ptr %dst, i1 %c) {
; DEFAULT: [[VECTOR_PH]]:
; DEFAULT-NEXT: br label %[[VECTOR_BODY:.*]]
; DEFAULT: [[VECTOR_BODY]]:
-; DEFAULT-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
-; DEFAULT-NEXT: [[VEC_IND:%.*]] = phi <4 x i64> [ <i64 0, i64 1, i64 2, i64 3>, %[[VECTOR_PH]] ], [ [[VEC_IND_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; DEFAULT-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[LOOP_LATCH2:.*]] ]
+; DEFAULT-NEXT: [[VEC_IND:%.*]] = phi <4 x i64> [ <i64 0, i64 1, i64 2, i64 3>, %[[VECTOR_PH]] ], [ [[VEC_IND_NEXT:%.*]], %[[LOOP_LATCH2]] ]
; DEFAULT-NEXT: [[STEP_ADD:%.*]] = add nuw <4 x i64> [[VEC_IND]], splat (i64 4)
-; DEFAULT-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 8
-; DEFAULT-NEXT: [[VEC_IND_NEXT]] = add nuw nsw <4 x i64> [[STEP_ADD]], splat (i64 4)
-; DEFAULT-NEXT: [[TMP3:%.*]] = icmp eq i64 [[INDEX_NEXT]], 16
-; DEFAULT-NEXT: br i1 [[TMP3]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP27:![0-9]+]]
+; DEFAULT-NEXT: br i1 [[C]], label %[[LOOP_LATCH2]], label %[[MIDDLE_BLOCK:.*]]
; DEFAULT: [[MIDDLE_BLOCK]]:
+; DEFAULT-NEXT: br label %[[LOOP_LATCH2]]
+; DEFAULT: [[LOOP_LATCH2]]:
; DEFAULT-NEXT: [[TMP1:%.*]] = add nuw nsw <4 x i64> [[STEP_ADD]], splat (i64 1)
; DEFAULT-NEXT: [[TMP2:%.*]] = trunc nuw nsw <4 x i64> [[TMP1]] to <4 x i32>
; DEFAULT-NEXT: [[TMP4:%.*]] = extractelement <4 x i32> [[TMP2]], i64 3
; DEFAULT-NEXT: store i32 [[TMP4]], ptr [[DST]], align 4
+; DEFAULT-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 8
+; DEFAULT-NEXT: [[VEC_IND_NEXT]] = add nuw nsw <4 x i64> [[STEP_ADD]], splat (i64 4)
+; DEFAULT-NEXT: [[TMP3:%.*]] = icmp eq i64 [[INDEX_NEXT]], 16
+; DEFAULT-NEXT: br i1 [[TMP3]], label %[[MIDDLE_BLOCK1:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP27:![0-9]+]]
+; DEFAULT: [[MIDDLE_BLOCK1]]:
; DEFAULT-NEXT: br label %[[SCALAR_PH:.*]]
; DEFAULT: [[SCALAR_PH]]:
;
diff --git a/llvm/test/Transforms/LoopVectorize/AArch64/sve-predicated-costs.ll b/llvm/test/Transforms/LoopVectorize/AArch64/sve-predicated-costs.ll
index 7bd7eea1a11d0..9360ce4aea08f 100644
--- a/llvm/test/Transforms/LoopVectorize/AArch64/sve-predicated-costs.ll
+++ b/llvm/test/Transforms/LoopVectorize/AArch64/sve-predicated-costs.ll
@@ -88,7 +88,9 @@ define void @always_taken(ptr noalias %p0, ptr noalias %p1, i1 %c0, i1 %c1, i1 %
; CHECK-NEXT: [[TMP8:%.*]] = select <vscale x 2 x i1> [[TMP6]], <vscale x 2 x i1> [[BROADCAST_SPLAT1]], <vscale x 2 x i1> zeroinitializer
; CHECK-NEXT: br label %[[VECTOR_BODY:.*]]
; CHECK: [[VECTOR_BODY]]:
-; CHECK-NEXT: [[INDEX:%.*]] = phi i32 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-NEXT: [[INDEX:%.*]] = phi i32 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[LATCH9:.*]] ]
+; CHECK-NEXT: br i1 [[C0]], label %[[THEN_05:.*]], label %[[LATCH9]]
+; CHECK: [[THEN_05]]:
; CHECK-NEXT: [[TMP10:%.*]] = getelementptr i64, ptr [[P0]], i32 [[INDEX]]
; CHECK-NEXT: [[TMP7:%.*]] = zext i32 [[TMP3]] to i64
; CHECK-NEXT: [[TMP20:%.*]] = getelementptr i64, ptr [[TMP10]], i64 [[TMP7]]
@@ -102,6 +104,8 @@ define void @always_taken(ptr noalias %p0, ptr noalias %p1, i1 %c0, i1 %c1, i1 %
; CHECK-NEXT: [[TMP13:%.*]] = call <vscale x 2 x i64> @llvm.masked.udiv.nxv2i64(<vscale x 2 x i64> [[WIDE_MASKED_LOAD5]], <vscale x 2 x i64> [[WIDE_MASKED_LOAD7]], <vscale x 2 x i1> [[TMP8]])
; CHECK-NEXT: call void @llvm.masked.store.nxv2i64.p0(<vscale x 2 x i64> [[TMP14]], ptr align 8 [[TMP9]], <vscale x 2 x i1> [[TMP8]])
; CHECK-NEXT: call void @llvm.masked.store.nxv2i64.p0(<vscale x 2 x i64> [[TMP13]], ptr align 8 [[TMP12]], <vscale x 2 x i1> [[TMP8]])
+; CHECK-NEXT: br label %[[LATCH9]]
+; CHECK: [[LATCH9]]:
; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i32 [[INDEX]], [[TMP1]]
; CHECK-NEXT: [[TMP16:%.*]] = icmp eq i32 [[INDEX_NEXT]], [[N_VEC]]
; CHECK-NEXT: br i1 [[TMP16]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP0:![0-9]+]]
diff --git a/llvm/test/Transforms/LoopVectorize/AArch64/transform-narrow-interleave-to-widen-memory-multi-block.ll b/llvm/test/Transforms/LoopVectorize/AArch64/transform-narrow-interleave-to-widen-memory-multi-block.ll
index 478a724f49c21..9efff93543685 100644
--- a/llvm/test/Transforms/LoopVectorize/AArch64/transform-narrow-interleave-to-widen-memory-multi-block.ll
+++ b/llvm/test/Transforms/LoopVectorize/AArch64/transform-narrow-interleave-to-widen-memory-multi-block.ll
@@ -13,7 +13,7 @@ define void @load_store_interleave_group_block_invar_cond(ptr noalias %data, ptr
; VF2IC1: [[VECTOR_PH]]:
; VF2IC1-NEXT: br label %[[VECTOR_BODY:.*]]
; VF2IC1: [[VECTOR_BODY]]:
-; VF2IC1-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[PRED_STORE_CONTINUE3:.*]] ]
+; VF2IC1-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[PRED_STORE_CONTINUE:.*]] ]
; VF2IC1-NEXT: [[TMP0:%.*]] = shl nsw i64 [[INDEX]], 1
; VF2IC1-NEXT: [[TMP1:%.*]] = getelementptr inbounds i64, ptr [[DATA]], i64 [[TMP0]]
; VF2IC1-NEXT: [[WIDE_VEC:%.*]] = load <4 x i64>, ptr [[TMP1]], align 8
@@ -22,16 +22,20 @@ define void @load_store_interleave_group_block_invar_cond(ptr noalias %data, ptr
; VF2IC1-NEXT: [[TMP4:%.*]] = shufflevector <2 x i64> [[STRIDED_VEC]], <2 x i64> [[STRIDED_VEC1]], <4 x i32> <i32 0, i32 1, i32 2, i32 3>
; VF2IC1-NEXT: [[INTERLEAVED_VEC:%.*]] = shufflevector <4 x i64> [[TMP4]], <4 x i64> poison, <4 x i32> <i32 0, i32 2, i32 1, i32 3>
; VF2IC1-NEXT: store <4 x i64> [[INTERLEAVED_VEC]], ptr [[TMP1]], align 8
-; VF2IC1-NEXT: br i1 [[C]], label %[[PRED_STORE_IF:.*]], label %[[PRED_STORE_CONTINUE:.*]]
+; VF2IC1-NEXT: br i1 [[C]], label %[[PRED_STORE_IF:.*]], label %[[PRED_STORE_CONTINUE]]
; VF2IC1: [[PRED_STORE_IF]]:
+; VF2IC1-NEXT: br i1 [[C]], label %[[PRED_STORE_IF1:.*]], label %[[PRED_STORE_CONTINUE1:.*]]
+; VF2IC1: [[PRED_STORE_IF1]]:
; VF2IC1-NEXT: store i8 1, ptr [[DST_0]], align 1
-; VF2IC1-NEXT: br label %[[PRED_STORE_CONTINUE]]
-; VF2IC1: [[PRED_STORE_CONTINUE]]:
-; VF2IC1-NEXT: br i1 [[C]], label %[[PRED_STORE_IF2:.*]], label %[[PRED_STORE_CONTINUE3]]
+; VF2IC1-NEXT: br label %[[PRED_STORE_CONTINUE1]]
+; VF2IC1: [[PRED_STORE_CONTINUE1]]:
+; VF2IC1-NEXT: br i1 [[C]], label %[[PRED_STORE_IF2:.*]], label %[[PRED_STORE_CONTINUE3:.*]]
; VF2IC1: [[PRED_STORE_IF2]]:
; VF2IC1-NEXT: store i8 1, ptr [[DST_0]], align 1
; VF2IC1-NEXT: br label %[[PRED_STORE_CONTINUE3]]
; VF2IC1: [[PRED_STORE_CONTINUE3]]:
+; VF2IC1-NEXT: br label %[[PRED_STORE_CONTINUE]]
+; VF2IC1: [[PRED_STORE_CONTINUE]]:
; VF2IC1-NEXT: [[TMP2:%.*]] = getelementptr inbounds i8, ptr [[DST_1]], i64 [[INDEX]]
; VF2IC1-NEXT: store <2 x i8> zeroinitializer, ptr [[TMP2]], align 1
; VF2IC1-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 2
@@ -49,7 +53,7 @@ define void @load_store_interleave_group_block_invar_cond(ptr noalias %data, ptr
; VF2IC2: [[VECTOR_PH]]:
; VF2IC2-NEXT: br label %[[VECTOR_BODY:.*]]
; VF2IC2: [[VECTOR_BODY]]:
-; VF2IC2-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[PRED_STORE_CONTINUE11:.*]] ]
+; VF2IC2-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[PRED_STORE_CONTINUE:.*]] ]
; VF2IC2-NEXT: [[TMP0:%.*]] = add i64 [[INDEX]], 2
; VF2IC2-NEXT: [[TMP1:%.*]] = shl nsw i64 [[INDEX]], 1
; VF2IC2-NEXT: [[TMP2:%.*]] = shl nsw i64 [[TMP0]], 1
@@ -67,11 +71,13 @@ define void @load_store_interleave_group_block_invar_cond(ptr noalias %data, ptr
; VF2IC2-NEXT: [[TMP9:%.*]] = shufflevector <2 x i64> [[STRIDED_VEC3]], <2 x i64> [[STRIDED_VEC4]], <4 x i32> <i32 0, i32 1, i32 2, i32 3>
; VF2IC2-NEXT: [[INTERLEAVED_VEC5:%.*]] = shufflevector <4 x i64> [[TMP9]], <4 x i64> poison, <4 x i32> <i32 0, i32 2, i32 1, i32 3>
; VF2IC2-NEXT: store <4 x i64> [[INTERLEAVED_VEC5]], ptr [[TMP4]], align 8
-; VF2IC2-NEXT: br i1 [[C]], label %[[PRED_STORE_IF:.*]], label %[[PRED_STORE_CONTINUE:.*]]
+; VF2IC2-NEXT: br i1 [[C]], label %[[PRED_STORE_IF:.*]], label %[[PRED_STORE_CONTINUE]]
; VF2IC2: [[PRED_STORE_IF]]:
+; VF2IC2-NEXT: br i1 [[C]], label %[[PRED_STORE_IF1:.*]], label %[[PRED_STORE_CONTINUE1:.*]]
+; VF2IC2: [[PRED_STORE_IF1]]:
; VF2IC2-NEXT: store i8 1, ptr [[DST_0]], align 1
-; VF2IC2-NEXT: br label %[[PRED_STORE_CONTINUE]]
-; VF2IC2: [[PRED_STORE_CONTINUE]]:
+; VF2IC2-NEXT: br label %[[PRED_STORE_CONTINUE1]]
+; VF2IC2: [[PRED_STORE_CONTINUE1]]:
; VF2IC2-NEXT: br i1 [[C]], label %[[PRED_STORE_IF6:.*]], label %[[PRED_STORE_CONTINUE7:.*]]
; VF2IC2: [[PRED_STORE_IF6]]:
; VF2IC2-NEXT: store i8 1, ptr [[DST_0]], align 1
@@ -82,11 +88,13 @@ define void @load_store_interleave_group_block_invar_cond(ptr noalias %data, ptr
; VF2IC2-NEXT: store i8 1, ptr [[DST_0]], align 1
; VF2IC2-NEXT: br label %[[PRED_STORE_CONTINUE9]]
; VF2IC2: [[PRED_STORE_CONTINUE9]]:
-; VF2IC2-NEXT: br i1 [[C]], label %[[PRED_STORE_IF10:.*]], label %[[PRED_STORE_CONTINUE11]]
+; VF2IC2-NEXT: br i1 [[C]], label %[[PRED_STORE_IF10:.*]], label %[[PRED_STORE_CONTINUE11:.*]]
; VF2IC2: [[PRED_STORE_IF10]]:
; VF2IC2-NEXT: store i8 1, ptr [[DST_0]], align 1
; VF2IC2-NEXT: br label %[[PRED_STORE_CONTINUE11]]
; VF2IC2: [[PRED_STORE_CONTINUE11]]:
+; VF2IC2-NEXT: br label %[[PRED_STORE_CONTINUE]]
+; VF2IC2: [[PRED_STORE_CONTINUE]]:
; VF2IC2-NEXT: [[TMP5:%.*]] = getelementptr inbounds i8, ptr [[DST_1]], i64 [[INDEX]]
; VF2IC2-NEXT: [[TMP6:%.*]] = getelementptr inbounds i8, ptr [[TMP5]], i64 2
; VF2IC2-NEXT: store <2 x i8> zeroinitializer, ptr [[TMP5]], align 1
diff --git a/llvm/test/Transforms/LoopVectorize/RISCV/dead-ops-cost.ll b/llvm/test/Transforms/LoopVectorize/RISCV/dead-ops-cost.ll
index 662e256b09c09..c4ca7d1d47673 100644
--- a/llvm/test/Transforms/LoopVectorize/RISCV/dead-ops-cost.ll
+++ b/llvm/test/Transforms/LoopVectorize/RISCV/dead-ops-cost.ll
@@ -162,7 +162,9 @@ define i32 @cost_of_exit_branch_and_cond_insts(ptr %a, ptr %b, i1 %c, i16 %x) #0
; CHECK-NEXT: [[BROADCAST_SPLAT:%.*]] = shufflevector <8 x i1> [[BROADCAST_SPLATINSERT]], <8 x i1> poison, <8 x i32> zeroinitializer
; CHECK-NEXT: br label %[[VECTOR_BODY:.*]]
; CHECK: [[VECTOR_BODY]]:
-; CHECK-NEXT: [[INDEX:%.*]] = phi i32 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[PRED_STORE_CONTINUE18:.*]] ]
+; CHECK-NEXT: [[INDEX:%.*]] = phi i32 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[LOOP_EXITING18:.*]] ]
+; CHECK-NEXT: br i1 [[C]], label %[[THEN3:.*]], label %[[LOOP_EXITING18]]
+; CHECK: [[THEN3]]:
; CHECK-NEXT: [[TMP11:%.*]] = getelementptr i32, ptr [[B]], i32 [[INDEX]]
; CHECK-NEXT: br i1 [[C]], label %[[PRED_STORE_IF:.*]], label %[[PRED_STORE_CONTINUE:.*]]
; CHECK: [[PRED_STORE_IF]]:
@@ -199,12 +201,14 @@ define i32 @cost_of_exit_branch_and_cond_insts(ptr %a, ptr %b, i1 %c, i16 %x) #0
; CHECK-NEXT: store i1 false, ptr [[A]], align 1, !alias.scope [[META10]], !noalias [[META13]]
; CHECK-NEXT: br label %[[PRED_STORE_CONTINUE16]]
; CHECK: [[PRED_STORE_CONTINUE16]]:
-; CHECK-NEXT: br i1 [[C]], label %[[PRED_STORE_IF17:.*]], label %[[PRED_STORE_CONTINUE18]]
+; CHECK-NEXT: br i1 [[C]], label %[[PRED_STORE_IF17:.*]], label %[[PRED_STORE_CONTINUE18:.*]]
; CHECK: [[PRED_STORE_IF17]]:
; CHECK-NEXT: store i1 false, ptr [[A]], align 1, !alias.scope [[META10]], !noalias [[META13]]
; CHECK-NEXT: br label %[[PRED_STORE_CONTINUE18]]
; CHECK: [[PRED_STORE_CONTINUE18]]:
; CHECK-NEXT: call void @llvm.masked.store.v8i32.p0(<8 x i32> zeroinitializer, ptr align 4 [[TMP11]], <8 x i1> [[BROADCAST_SPLAT]]), !alias.scope [[META13]]
+; CHECK-NEXT: br label %[[LOOP_EXITING18]]
+; CHECK: [[LOOP_EXITING18]]:
; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i32 [[INDEX]], 8
; CHECK-NEXT: [[TMP21:%.*]] = icmp eq i32 [[INDEX_NEXT]], [[N_VEC]]
; CHECK-NEXT: br i1 [[TMP21]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP15:![0-9]+]]
diff --git a/llvm/test/Transforms/LoopVectorize/RISCV/predicated-costs.ll b/llvm/test/Transforms/LoopVectorize/RISCV/predicated-costs.ll
index 2316a478becc5..e24e78fcf0602 100644
--- a/llvm/test/Transforms/LoopVectorize/RISCV/predicated-costs.ll
+++ b/llvm/test/Transforms/LoopVectorize/RISCV/predicated-costs.ll
@@ -18,13 +18,17 @@ define void @nested(ptr noalias %p0, ptr noalias %p1, i1 %c0, i1 %c1) {
; CHECK-NEXT: [[TMP0:%.*]] = select <vscale x 4 x i1> [[BROADCAST_SPLAT2]], <vscale x 4 x i1> [[BROADCAST_SPLAT]], <vscale x 4 x i1> zeroinitializer
; CHECK-NEXT: br label %[[THEN_1:.*]]
; CHECK: [[THEN_1]]:
-; CHECK-NEXT: [[IV1:%.*]] = phi i32 [ 0, %[[LOOP]] ], [ [[INDEX_EVL_NEXT:%.*]], %[[THEN_1]] ]
-; CHECK-NEXT: [[AVL:%.*]] = phi i32 [ 1024, %[[LOOP]] ], [ [[AVL_NEXT:%.*]], %[[THEN_1]] ]
+; CHECK-NEXT: [[IV1:%.*]] = phi i32 [ 0, %[[LOOP]] ], [ [[INDEX_EVL_NEXT:%.*]], %[[LATCH4:.*]] ]
+; CHECK-NEXT: [[AVL:%.*]] = phi i32 [ 1024, %[[LOOP]] ], [ [[AVL_NEXT:%.*]], %[[LATCH4]] ]
; CHECK-NEXT: [[TMP1:%.*]] = call i32 @llvm.experimental.get.vector.length.i32(i32 [[AVL]], i32 4, i1 true)
+; CHECK-NEXT: br i1 [[C0]], label %[[THEN_03:.*]], label %[[LATCH4]]
+; CHECK: [[THEN_03]]:
; CHECK-NEXT: [[GEP2:%.*]] = getelementptr i32, ptr [[P0]], i32 [[IV1]]
; CHECK-NEXT: [[VP_OP_LOAD:%.*]] = call <vscale x 4 x i32> @llvm.vp.load.nxv4i32.p0(ptr align 4 [[GEP2]], <vscale x 4 x i1> [[TMP0]], i32 [[TMP1]])
; CHECK-NEXT: [[TMP3:%.*]] = getelementptr i32, ptr [[P1]], <vscale x 4 x i32> [[VP_OP_LOAD]]
; CHECK-NEXT: call void @llvm.vp.scatter.nxv4i32.nxv4p0(<vscale x 4 x i32> zeroinitializer, <vscale x 4 x ptr> align 4 [[TMP3]], <vscale x 4 x i1> [[TMP0]], i32 [[TMP1]])
+; CHECK-NEXT: br label %[[LATCH4]]
+; CHECK: [[LATCH4]]:
; CHECK-NEXT: [[INDEX_EVL_NEXT]] = add nuw i32 [[TMP1]], [[IV1]]
; CHECK-NEXT: [[AVL_NEXT]] = sub nuw i32 [[AVL]], [[TMP1]]
; CHECK-NEXT: [[TMP4:%.*]] = icmp eq i32 [[AVL_NEXT]], 0
@@ -76,13 +80,17 @@ define void @always_taken(ptr noalias %p0, ptr noalias %p1, i1 %c0, i1 %c1) {
; CHECK-NEXT: [[TMP0:%.*]] = select <vscale x 4 x i1> [[BROADCAST_SPLAT2]], <vscale x 4 x i1> [[BROADCAST_SPLAT]], <vscale x 4 x i1> zeroinitializer
; CHECK-NEXT: br label %[[VECTOR_BODY:.*]]
; CHECK: [[VECTOR_BODY]]:
-; CHECK-NEXT: [[EVL_BASED_IV:%.*]] = phi i32 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_EVL_NEXT:%.*]], %[[VECTOR_BODY]] ]
-; CHECK-NEXT: [[AVL:%.*]] = phi i32 [ 1024, %[[VECTOR_PH]] ], [ [[AVL_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-NEXT: [[EVL_BASED_IV:%.*]] = phi i32 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_EVL_NEXT:%.*]], %[[LATCH4:.*]] ]
+; CHECK-NEXT: [[AVL:%.*]] = phi i32 [ 1024, %[[VECTOR_PH]] ], [ [[AVL_NEXT:%.*]], %[[LATCH4]] ]
; CHECK-NEXT: [[TMP1:%.*]] = call i32 @llvm.experimental.get.vector.length.i32(i32 [[AVL]], i32 4, i1 true)
+; CHECK-NEXT: br i1 [[C0]], label %[[THEN_03:.*]], label %[[LATCH4]]
+; CHECK: [[THEN_03]]:
; CHECK-NEXT: [[TMP2:%.*]] = getelementptr i32, ptr [[P0]], i32 [[EVL_BASED_IV]]
; CHECK-NEXT: [[VP_OP_LOAD:%.*]] = call <vscale x 4 x i32> @llvm.vp.load.nxv4i32.p0(ptr align 4 [[TMP2]], <vscale x 4 x i1> [[TMP0]], i32 [[TMP1]])
; CHECK-NEXT: [[TMP3:%.*]] = getelementptr i32, ptr [[P1]], <vscale x 4 x i32> [[VP_OP_LOAD]]
; CHECK-NEXT: call void @llvm.vp.scatter.nxv4i32.nxv4p0(<vscale x 4 x i32> zeroinitializer, <vscale x 4 x ptr> align 4 [[TMP3]], <vscale x 4 x i1> [[TMP0]], i32 [[TMP1]])
+; CHECK-NEXT: br label %[[LATCH4]]
+; CHECK: [[LATCH4]]:
; CHECK-NEXT: [[INDEX_EVL_NEXT]] = add nuw i32 [[TMP1]], [[EVL_BASED_IV]]
; CHECK-NEXT: [[AVL_NEXT]] = sub nuw i32 [[AVL]], [[TMP1]]
; CHECK-NEXT: [[TMP4:%.*]] = icmp eq i32 [[AVL_NEXT]], 0
diff --git a/llvm/test/Transforms/LoopVectorize/RISCV/predicated-reverse-store.ll b/llvm/test/Transforms/LoopVectorize/RISCV/predicated-reverse-store.ll
index 9c96f44a9d19b..29373388b3721 100644
--- a/llvm/test/Transforms/LoopVectorize/RISCV/predicated-reverse-store.ll
+++ b/llvm/test/Transforms/LoopVectorize/RISCV/predicated-reverse-store.ll
@@ -11,11 +11,13 @@ define void @reverse_predicated_store(i1 %c, ptr %dst, i64 %n) #0 {
; CHECK-NEXT: [[BROADCAST_SPLAT:%.*]] = shufflevector <vscale x 4 x i1> [[BROADCAST_SPLATINSERT]], <vscale x 4 x i1> poison, <vscale x 4 x i32> zeroinitializer
; CHECK-NEXT: br label [[VECTOR_BODY:%.*]]
; CHECK: vector.body:
-; CHECK-NEXT: [[EVL_BASED_IV:%.*]] = phi i64 [ 0, [[FOR_BODY]] ], [ [[INDEX_EVL_NEXT:%.*]], [[VECTOR_BODY]] ]
-; CHECK-NEXT: [[AVL:%.*]] = phi i64 [ [[TMP0]], [[FOR_BODY]] ], [ [[AVL_NEXT:%.*]], [[VECTOR_BODY]] ]
+; CHECK-NEXT: [[EVL_BASED_IV:%.*]] = phi i64 [ 0, [[FOR_BODY]] ], [ [[CURRENT_ITERATION_NEXT:%.*]], [[LOOP_LATCH2:%.*]] ]
+; CHECK-NEXT: [[AVL:%.*]] = phi i64 [ [[TMP0]], [[FOR_BODY]] ], [ [[AVL_NEXT:%.*]], [[LOOP_LATCH2]] ]
; CHECK-NEXT: [[TMP1:%.*]] = call i32 @llvm.experimental.get.vector.length.i64(i64 [[AVL]], i32 4, i1 true)
; CHECK-NEXT: [[IV:%.*]] = sub i64 [[N]], [[EVL_BASED_IV]]
; CHECK-NEXT: [[IV_NEXT:%.*]] = add i64 [[IV]], -1
+; CHECK-NEXT: br i1 [[C]], label [[IF_THEN1:%.*]], label [[LOOP_LATCH2]]
+; CHECK: if.then1:
; CHECK-NEXT: [[ARRAYIDX:%.*]] = getelementptr float, ptr [[DST:%.*]], i64 [[IV_NEXT]]
; CHECK-NEXT: [[VP_REVERSE_MASK:%.*]] = call <vscale x 4 x i1> @llvm.experimental.vp.reverse.nxv4i1(<vscale x 4 x i1> [[BROADCAST_SPLAT]], <vscale x 4 x i1> splat (i1 true), i32 [[TMP1]])
; CHECK-NEXT: [[TMP4:%.*]] = zext i32 [[TMP1]] to i64
@@ -24,8 +26,11 @@ define void @reverse_predicated_store(i1 %c, ptr %dst, i64 %n) #0 {
; CHECK-NEXT: [[TMP9:%.*]] = getelementptr float, ptr [[ARRAYIDX]], i64 [[TMP7]]
; CHECK-NEXT: [[TMP12:%.*]] = call <vscale x 4 x float> @llvm.experimental.vp.reverse.nxv4f32(<vscale x 4 x float> zeroinitializer, <vscale x 4 x i1> splat (i1 true), i32 [[TMP1]])
; CHECK-NEXT: call void @llvm.vp.store.nxv4f32.p0(<vscale x 4 x float> [[TMP12]], ptr align 4 [[TMP9]], <vscale x 4 x i1> [[VP_REVERSE_MASK]], i32 [[TMP1]])
-; CHECK-NEXT: [[INDEX_EVL_NEXT]] = add i64 [[TMP4]], [[EVL_BASED_IV]]
-; CHECK-NEXT: [[AVL_NEXT]] = sub nuw i64 [[AVL]], [[TMP4]]
+; CHECK-NEXT: br label [[LOOP_LATCH2]]
+; CHECK: loop.latch2:
+; CHECK-NEXT: [[TMP13:%.*]] = zext i32 [[TMP1]] to i64
+; CHECK-NEXT: [[CURRENT_ITERATION_NEXT]] = add i64 [[TMP13]], [[EVL_BASED_IV]]
+; CHECK-NEXT: [[AVL_NEXT]] = sub nuw i64 [[AVL]], [[TMP13]]
; CHECK-NEXT: [[TMP11:%.*]] = icmp eq i64 [[AVL_NEXT]], 0
; CHECK-NEXT: br i1 [[TMP11]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP0:![0-9]+]]
; CHECK: middle.block:
diff --git a/llvm/test/Transforms/LoopVectorize/VPlan/predicator.ll b/llvm/test/Transforms/LoopVectorize/VPlan/predicator.ll
index 4b4f8c4b035f0..239bd1b324257 100644
--- a/llvm/test/Transforms/LoopVectorize/VPlan/predicator.ll
+++ b/llvm/test/Transforms/LoopVectorize/VPlan/predicator.ll
@@ -752,12 +752,8 @@ define void @simplifiable_blend(i1 %c1, i1 %c2, i1 %c3, i32 %x, i32 %y, ptr %p)
; CHECK-EMPTY:
; CHECK-NEXT: B:
; CHECK-NEXT: EMIT vp<[[VP4:%[0-9]+]]> = not ir<%c1>
-; CHECK-NEXT: Successor(s): E
-; CHECK-EMPTY:
-; CHECK-NEXT: E:
-; CHECK-NEXT: EMIT vp<[[VP5:%[0-9]+]]> = not ir<%c3>
-; CHECK-NEXT: EMIT vp<[[VP6:%[0-9]+]]> = logical-and vp<[[VP4]]>, vp<[[VP5]]>
-; CHECK-NEXT: Successor(s): F
+; CHECK-NEXT: EMIT branch-on-cond ir<%c3>
+; CHECK-NEXT: Successor(s): F, E
; CHECK-EMPTY:
; CHECK-NEXT: F:
; CHECK-NEXT: Successor(s): A
@@ -783,6 +779,11 @@ define void @simplifiable_blend(i1 %c1, i1 %c2, i1 %c3, i32 %x, i32 %y, ptr %p)
; CHECK-NEXT: EMIT vp<%index.next> = add nuw vp<[[VP3]]>, vp<[[VP1:%[0-9]+]]>
; CHECK-NEXT: EMIT branch-on-count vp<%index.next>, vp<[[VP2:%[0-9]+]]>
; CHECK-NEXT: No successors
+; CHECK-EMPTY:
+; CHECK-NEXT: E:
+; CHECK-NEXT: EMIT vp<[[VP5:%[0-9]+]]> = not ir<%c3>
+; CHECK-NEXT: EMIT vp<[[VP6:%[0-9]+]]> = logical-and vp<[[VP4]]>, vp<[[VP5]]>
+; CHECK-NEXT: Successor(s): F
; CHECK-NEXT: }
; CHECK-NEXT: Successor(s): middle.block
;
@@ -1038,7 +1039,8 @@ define void @outermost_uniform_branch_no_phi(ptr %a, i1 %u0) {
; CHECK-NEXT: ir<%iv> = WIDEN-INDUCTION nuw nsw ir<0>, ir<1>, vp<[[VP0:%[0-9]+]]>
; CHECK-NEXT: EMIT ir<%gep> = getelementptr ir<%a>, ir<%iv>
; CHECK-NEXT: EMIT store ir<%iv>, ir<%gep>
-; CHECK-NEXT: Successor(s): bb1
+; CHECK-NEXT: EMIT branch-on-cond ir<%u0>
+; CHECK-NEXT: Successor(s): bb1, bb4
; CHECK-EMPTY:
; CHECK-NEXT: bb1:
; CHECK-NEXT: EMIT ir<%add1> = add ir<%iv>, ir<1>, ir<%u0>
@@ -1210,13 +1212,8 @@ define void @outermost_uniform_branch_more_blocks_no_phi(ptr %a, i1 %u0) {
; CHECK-NEXT: vector.body:
; CHECK-NEXT: ir<%iv> = WIDEN-INDUCTION nuw nsw ir<0>, ir<1>, vp<[[VP0:%[0-9]+]]>
; CHECK-NEXT: EMIT ir<%gep> = getelementptr ir<%a>, ir<%iv>
-; CHECK-NEXT: Successor(s): bb1
-; CHECK-EMPTY:
-; CHECK-NEXT: bb1:
-; CHECK-NEXT: EMIT vp<[[VP4:%[0-9]+]]> = not ir<%u0>
-; CHECK-NEXT: EMIT ir<%add1> = add ir<%iv>, ir<1>, vp<[[VP4]]>
-; CHECK-NEXT: EMIT store ir<%add1>, ir<%gep>, vp<[[VP4]]>
-; CHECK-NEXT: Successor(s): bb2
+; CHECK-NEXT: EMIT branch-on-cond ir<%u0>
+; CHECK-NEXT: Successor(s): bb2, bb1
; CHECK-EMPTY:
; CHECK-NEXT: bb2:
; CHECK-NEXT: EMIT ir<%v2> = icmp sle ir<%iv>, ir<2>, ir<%u0>
@@ -1244,6 +1241,12 @@ define void @outermost_uniform_branch_more_blocks_no_phi(ptr %a, i1 %u0) {
; CHECK-NEXT: EMIT vp<%index.next> = add nuw vp<[[VP3]]>, vp<[[VP1:%[0-9]+]]>
; CHECK-NEXT: EMIT branch-on-count vp<%index.next>, vp<[[VP2:%[0-9]+]]>
; CHECK-NEXT: No successors
+; CHECK-EMPTY:
+; CHECK-NEXT: bb1:
+; CHECK-NEXT: EMIT vp<[[VP4:%[0-9]+]]> = not ir<%u0>
+; CHECK-NEXT: EMIT ir<%add1> = add ir<%iv>, ir<1>, vp<[[VP4]]>
+; CHECK-NEXT: EMIT store ir<%add1>, ir<%gep>, vp<[[VP4]]>
+; CHECK-NEXT: Successor(s): bb6
; CHECK-NEXT: }
; CHECK-NEXT: Successor(s): middle.block
;
@@ -1397,7 +1400,8 @@ define void @uniform_branch_after_varying_branch_no_phi(ptr %a, i1 %u1) {
; CHECK-NEXT: EMIT vp<[[VP4:%[0-9]+]]> = not ir<%v0>
; CHECK-NEXT: EMIT ir<%add1> = add ir<%iv>, ir<1>, vp<[[VP4]]>
; CHECK-NEXT: EMIT store ir<%add1>, ir<%gep>, vp<[[VP4]]>
-; CHECK-NEXT: Successor(s): bb2
+; CHECK-NEXT: EMIT branch-on-cond ir<%u1>
+; CHECK-NEXT: Successor(s): bb2, bb3
; CHECK-EMPTY:
; CHECK-NEXT: bb2:
; CHECK-NEXT: EMIT vp<[[VP5:%[0-9]+]]> = logical-and vp<[[VP4]]>, ir<%u1>
@@ -1575,14 +1579,8 @@ define void @uniform_branch_after_varying_branch_more_blocks_no_phi(ptr %a, i1 %
; CHECK-EMPTY:
; CHECK-NEXT: bb1:
; CHECK-NEXT: EMIT vp<[[VP4:%[0-9]+]]> = not ir<%v0>
-; CHECK-NEXT: Successor(s): bb2
-; CHECK-EMPTY:
-; CHECK-NEXT: bb2:
-; CHECK-NEXT: EMIT vp<[[VP5:%[0-9]+]]> = not ir<%u1>
-; CHECK-NEXT: EMIT vp<[[VP6:%[0-9]+]]> = logical-and vp<[[VP4]]>, vp<[[VP5]]>
-; CHECK-NEXT: EMIT ir<%add2> = add ir<%iv>, ir<2>, vp<[[VP6]]>
-; CHECK-NEXT: EMIT store ir<%add2>, ir<%gep>, vp<[[VP6]]>
-; CHECK-NEXT: Successor(s): bb3
+; CHECK-NEXT: EMIT branch-on-cond ir<%u1>
+; CHECK-NEXT: Successor(s): bb3, bb2
; CHECK-EMPTY:
; CHECK-NEXT: bb3:
; CHECK-NEXT: EMIT vp<[[VP7:%[0-9]+]]> = logical-and vp<[[VP4]]>, ir<%u1>
@@ -1604,6 +1602,13 @@ define void @uniform_branch_after_varying_branch_more_blocks_no_phi(ptr %a, i1 %
; CHECK-NEXT: EMIT vp<%index.next> = add nuw vp<[[VP3]]>, vp<[[VP1:%[0-9]+]]>
; CHECK-NEXT: EMIT branch-on-count vp<%index.next>, vp<[[VP2:%[0-9]+]]>
; CHECK-NEXT: No successors
+; CHECK-EMPTY:
+; CHECK-NEXT: bb2:
+; CHECK-NEXT: EMIT vp<[[VP5:%[0-9]+]]> = not ir<%u1>
+; CHECK-NEXT: EMIT vp<[[VP6:%[0-9]+]]> = logical-and vp<[[VP4]]>, vp<[[VP5]]>
+; CHECK-NEXT: EMIT ir<%add2> = add ir<%iv>, ir<2>, vp<[[VP6]]>
+; CHECK-NEXT: EMIT store ir<%add2>, ir<%gep>, vp<[[VP6]]>
+; CHECK-NEXT: Successor(s): bb4
; CHECK-NEXT: }
; CHECK-NEXT: Successor(s): middle.block
;
@@ -1770,14 +1775,8 @@ define void @uniform_branch_after_varying_branch_more_blocks_mirrored_no_phi(ptr
; CHECK-NEXT: Successor(s): bb2
; CHECK-EMPTY:
; CHECK-NEXT: bb2:
-; CHECK-NEXT: Successor(s): bb3
-; CHECK-EMPTY:
-; CHECK-NEXT: bb3:
-; CHECK-NEXT: EMIT vp<[[VP5:%[0-9]+]]> = not ir<%u2>
-; CHECK-NEXT: EMIT vp<[[VP6:%[0-9]+]]> = logical-and ir<%v0>, vp<[[VP5]]>
-; CHECK-NEXT: EMIT ir<%add3> = add ir<%iv>, ir<3>, vp<[[VP6]]>
-; CHECK-NEXT: EMIT store ir<%add3>, ir<%gep>, vp<[[VP6]]>
-; CHECK-NEXT: Successor(s): bb4
+; CHECK-NEXT: EMIT branch-on-cond ir<%u2>
+; CHECK-NEXT: Successor(s): bb4, bb3
; CHECK-EMPTY:
; CHECK-NEXT: bb4:
; CHECK-NEXT: EMIT vp<[[VP7:%[0-9]+]]> = logical-and ir<%v0>, ir<%u2>
@@ -1794,6 +1793,13 @@ define void @uniform_branch_after_varying_branch_more_blocks_mirrored_no_phi(ptr
; CHECK-NEXT: EMIT vp<%index.next> = add nuw vp<[[VP3]]>, vp<[[VP1:%[0-9]+]]>
; CHECK-NEXT: EMIT branch-on-count vp<%index.next>, vp<[[VP2:%[0-9]+]]>
; CHECK-NEXT: No successors
+; CHECK-EMPTY:
+; CHECK-NEXT: bb3:
+; CHECK-NEXT: EMIT vp<[[VP5:%[0-9]+]]> = not ir<%u2>
+; CHECK-NEXT: EMIT vp<[[VP6:%[0-9]+]]> = logical-and ir<%v0>, vp<[[VP5]]>
+; CHECK-NEXT: EMIT ir<%add3> = add ir<%iv>, ir<3>, vp<[[VP6]]>
+; CHECK-NEXT: EMIT store ir<%add3>, ir<%gep>, vp<[[VP6]]>
+; CHECK-NEXT: Successor(s): bb5
; CHECK-NEXT: }
; CHECK-NEXT: Successor(s): middle.block
;
@@ -2813,13 +2819,8 @@ define void @uniform_branch_shared_join_with_varying_no_phi(ptr %a, i1 %u0) {
; CHECK-NEXT: vector.body:
; CHECK-NEXT: ir<%iv> = WIDEN-INDUCTION nuw nsw ir<0>, ir<1>, vp<[[VP0:%[0-9]+]]>
; CHECK-NEXT: EMIT ir<%gep> = getelementptr ir<%a>, ir<%iv>
-; CHECK-NEXT: Successor(s): bb1
-; CHECK-EMPTY:
-; CHECK-NEXT: bb1:
-; CHECK-NEXT: EMIT vp<[[VP4:%[0-9]+]]> = not ir<%u0>
-; CHECK-NEXT: EMIT ir<%add1> = add ir<%iv>, ir<1>, vp<[[VP4]]>
-; CHECK-NEXT: EMIT store ir<%add1>, ir<%gep>, vp<[[VP4]]>
-; CHECK-NEXT: Successor(s): bb2
+; CHECK-NEXT: EMIT branch-on-cond ir<%u0>
+; CHECK-NEXT: Successor(s): bb2, bb1
; CHECK-EMPTY:
; CHECK-NEXT: bb2:
; CHECK-NEXT: EMIT ir<%add2> = add ir<%iv>, ir<2>, ir<%u0>
@@ -2839,6 +2840,12 @@ define void @uniform_branch_shared_join_with_varying_no_phi(ptr %a, i1 %u0) {
; CHECK-NEXT: EMIT vp<%index.next> = add nuw vp<[[VP3]]>, vp<[[VP1:%[0-9]+]]>
; CHECK-NEXT: EMIT branch-on-count vp<%index.next>, vp<[[VP2:%[0-9]+]]>
; CHECK-NEXT: No successors
+; CHECK-EMPTY:
+; CHECK-NEXT: bb1:
+; CHECK-NEXT: EMIT vp<[[VP4:%[0-9]+]]> = not ir<%u0>
+; CHECK-NEXT: EMIT ir<%add1> = add ir<%iv>, ir<1>, vp<[[VP4]]>
+; CHECK-NEXT: EMIT store ir<%add1>, ir<%gep>, vp<[[VP4]]>
+; CHECK-NEXT: Successor(s): bb4
; CHECK-NEXT: }
; CHECK-NEXT: Successor(s): middle.block
;
diff --git a/llvm/test/Transforms/LoopVectorize/VPlan/vplan-sink-scalars-and-merge-vf1.ll b/llvm/test/Transforms/LoopVectorize/VPlan/vplan-sink-scalars-and-merge-vf1.ll
index 1bdf23d5de144..c846b26096772 100644
--- a/llvm/test/Transforms/LoopVectorize/VPlan/vplan-sink-scalars-and-merge-vf1.ll
+++ b/llvm/test/Transforms/LoopVectorize/VPlan/vplan-sink-scalars-and-merge-vf1.ll
@@ -28,6 +28,10 @@ define void @sink_with_sideeffects(i1 %c, ptr %ptr) {
; CHECK-NEXT: CLONE ir<%tmp2> = getelementptr ir<%ptr>, vp<[[VP5]]>
; CHECK-NEXT: CLONE ir<%tmp3> = load ir<%tmp2>
; CHECK-NEXT: CLONE store ir<0>, ir<%tmp2>
+; CHECK-NEXT: EMIT branch-on-cond ir<%c>
+; CHECK-NEXT: Successor(s): if.then, for.inc
+; CHECK-EMPTY:
+; CHECK-NEXT: if.then:
; CHECK-NEXT: Successor(s): pred.store
; CHECK-EMPTY:
; CHECK-NEXT: <xVFxUF> pred.store: {
@@ -45,6 +49,9 @@ define void @sink_with_sideeffects(i1 %c, ptr %ptr) {
; CHECK-NEXT: Successor(s): if.then.0
; CHECK-EMPTY:
; CHECK-NEXT: if.then.0:
+; CHECK-NEXT: Successor(s): for.inc
+; CHECK-EMPTY:
+; CHECK-NEXT: for.inc:
; CHECK-NEXT: EMIT vp<%index.next> = add nuw vp<[[VP4]]>, vp<[[VP1]]>
; CHECK-NEXT: EMIT branch-on-count vp<%index.next>, vp<[[VP2]]>
; CHECK-NEXT: No successors
diff --git a/llvm/test/Transforms/LoopVectorize/VPlan/vplan-sink-scalars-and-merge.ll b/llvm/test/Transforms/LoopVectorize/VPlan/vplan-sink-scalars-and-merge.ll
index bd9da43107f38..c8503ebab5e45 100644
--- a/llvm/test/Transforms/LoopVectorize/VPlan/vplan-sink-scalars-and-merge.ll
+++ b/llvm/test/Transforms/LoopVectorize/VPlan/vplan-sink-scalars-and-merge.ll
@@ -1160,6 +1160,10 @@ define void @update_multiple_users(ptr noalias %src, ptr noalias %dst, i1 %c) {
; CHECK-NEXT: vp<[[VP2:%[0-9]+]]> = CANONICAL-IV
; CHECK-EMPTY:
; CHECK-NEXT: vector.body:
+; CHECK-NEXT: EMIT branch-on-cond ir<%c>
+; CHECK-NEXT: Successor(s): loop.then, loop.latch
+; CHECK-EMPTY:
+; CHECK-NEXT: loop.then:
; CHECK-NEXT: Successor(s): pred.store
; CHECK-EMPTY:
; CHECK-NEXT: <xVFxUF> pred.store: {
@@ -1181,6 +1185,9 @@ define void @update_multiple_users(ptr noalias %src, ptr noalias %dst, i1 %c) {
; CHECK-NEXT: Successor(s): loop.then.1
; CHECK-EMPTY:
; CHECK-NEXT: loop.then.1:
+; CHECK-NEXT: Successor(s): loop.latch
+; CHECK-EMPTY:
+; CHECK-NEXT: loop.latch:
; CHECK-NEXT: EMIT vp<%index.next> = add nuw vp<[[VP2]]>, vp<[[VP0]]>
; CHECK-NEXT: EMIT branch-on-count vp<%index.next>, vp<[[VP1]]>
; CHECK-NEXT: No successors
diff --git a/llvm/test/Transforms/LoopVectorize/X86/masked-store-cost.ll b/llvm/test/Transforms/LoopVectorize/X86/masked-store-cost.ll
index f1b49668f2b10..4faabbc7b8f35 100644
--- a/llvm/test/Transforms/LoopVectorize/X86/masked-store-cost.ll
+++ b/llvm/test/Transforms/LoopVectorize/X86/masked-store-cost.ll
@@ -129,10 +129,14 @@ define void @test_scalar_cost_single_store_loop_invariant_cond(ptr %dst, i1 %c)
; CHECK-NEXT: [[BROADCAST_SPLAT:%.*]] = shufflevector <8 x i1> [[BROADCAST_SPLATINSERT]], <8 x i1> poison, <8 x i32> zeroinitializer
; CHECK-NEXT: br label %[[VECTOR_BODY:.*]]
; CHECK: [[VECTOR_BODY]]:
-; CHECK-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[LOOP_LATCH2:.*]] ]
; CHECK-NEXT: [[TMP1:%.*]] = shl i64 [[INDEX]], 2
; CHECK-NEXT: [[NEXT_GEP:%.*]] = getelementptr i8, ptr [[DST]], i64 [[TMP1]]
+; CHECK-NEXT: br i1 [[C]], label %[[IF_THEN1:.*]], label %[[LOOP_LATCH2]]
+; CHECK: [[IF_THEN1]]:
; CHECK-NEXT: call void @llvm.masked.store.v8i32.p0(<8 x i32> zeroinitializer, ptr align 4 [[NEXT_GEP]], <8 x i1> [[BROADCAST_SPLAT]])
+; CHECK-NEXT: br label %[[LOOP_LATCH2]]
+; CHECK: [[LOOP_LATCH2]]:
; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 8
; CHECK-NEXT: [[TMP2:%.*]] = icmp eq i64 [[INDEX_NEXT]], 24
; CHECK-NEXT: br i1 [[TMP2]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP6:![0-9]+]]
diff --git a/llvm/test/Transforms/LoopVectorize/div-exact.ll b/llvm/test/Transforms/LoopVectorize/div-exact.ll
index 11ea08f7da4e9..7bec969285194 100644
--- a/llvm/test/Transforms/LoopVectorize/div-exact.ll
+++ b/llvm/test/Transforms/LoopVectorize/div-exact.ll
@@ -18,10 +18,12 @@ define void @unknown_divisor(ptr noalias %p, i32 %n, i1 %c) {
; CHECK-NEXT: [[BROADCAST_SPLAT:%.*]] = shufflevector <4 x i1> [[BROADCAST_SPLATINSERT]], <4 x i1> poison, <4 x i32> zeroinitializer
; CHECK-NEXT: br label %[[VECTOR_BODY:.*]]
; CHECK: [[VECTOR_BODY]]:
-; CHECK-NEXT: [[INDEX:%.*]] = phi i32 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[PRED_STORE_CONTINUE14:.*]] ]
+; CHECK-NEXT: [[INDEX:%.*]] = phi i32 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[LATCH14:.*]] ]
; CHECK-NEXT: [[TMP2:%.*]] = add i32 [[INDEX]], 1
; CHECK-NEXT: [[TMP3:%.*]] = add i32 [[INDEX]], 2
; CHECK-NEXT: [[TMP4:%.*]] = add i32 [[INDEX]], 3
+; CHECK-NEXT: br i1 [[C]], label %[[IF1:.*]], label %[[LATCH14]]
+; CHECK: [[IF1]]:
; CHECK-NEXT: [[TMP7:%.*]] = getelementptr i32, ptr [[P]], i32 [[INDEX]]
; CHECK-NEXT: [[TMP8:%.*]] = getelementptr i32, ptr [[P]], i32 [[TMP2]]
; CHECK-NEXT: [[TMP9:%.*]] = getelementptr i32, ptr [[P]], i32 [[TMP3]]
@@ -32,7 +34,7 @@ define void @unknown_divisor(ptr noalias %p, i32 %n, i1 %c) {
; CHECK-NEXT: [[TMP13:%.*]] = insertelement <4 x i32> poison, i32 [[TMP12]], i64 0
; CHECK-NEXT: br label %[[PRED_LOAD_CONTINUE]]
; CHECK: [[PRED_LOAD_CONTINUE]]:
-; CHECK-NEXT: [[TMP14:%.*]] = phi <4 x i32> [ poison, %[[VECTOR_BODY]] ], [ [[TMP13]], %[[PRED_LOAD_IF]] ]
+; CHECK-NEXT: [[TMP14:%.*]] = phi <4 x i32> [ poison, %[[IF1]] ], [ [[TMP13]], %[[PRED_LOAD_IF]] ]
; CHECK-NEXT: br i1 [[C]], label %[[PRED_LOAD_IF3:.*]], label %[[PRED_LOAD_CONTINUE4:.*]]
; CHECK: [[PRED_LOAD_IF3]]:
; CHECK-NEXT: [[TMP16:%.*]] = load i32, ptr [[TMP8]], align 4
@@ -73,12 +75,14 @@ define void @unknown_divisor(ptr noalias %p, i32 %n, i1 %c) {
; CHECK-NEXT: store i32 [[TMP31]], ptr [[TMP9]], align 4
; CHECK-NEXT: br label %[[PRED_STORE_CONTINUE12]]
; CHECK: [[PRED_STORE_CONTINUE12]]:
-; CHECK-NEXT: br i1 [[C]], label %[[PRED_STORE_IF13:.*]], label %[[PRED_STORE_CONTINUE14]]
+; CHECK-NEXT: br i1 [[C]], label %[[PRED_STORE_IF13:.*]], label %[[PRED_STORE_CONTINUE14:.*]]
; CHECK: [[PRED_STORE_IF13]]:
; CHECK-NEXT: [[TMP32:%.*]] = extractelement <4 x i32> [[TMP28]], i64 3
; CHECK-NEXT: store i32 [[TMP32]], ptr [[TMP10]], align 4
; CHECK-NEXT: br label %[[PRED_STORE_CONTINUE14]]
; CHECK: [[PRED_STORE_CONTINUE14]]:
+; CHECK-NEXT: br label %[[LATCH14]]
+; CHECK: [[LATCH14]]:
; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i32 [[INDEX]], 4
; CHECK-NEXT: [[TMP33:%.*]] = icmp eq i32 [[INDEX_NEXT]], [[N_VEC]]
; CHECK-NEXT: br i1 [[TMP33]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP0:![0-9]+]]
@@ -142,10 +146,12 @@ define void @unknown_dividend(ptr noalias %p, i32 %n, i1 %c) {
; CHECK-NEXT: [[N_VEC:%.*]] = sub i32 [[N]], [[N_MOD_VF]]
; CHECK-NEXT: br label %[[VECTOR_BODY:.*]]
; CHECK: [[VECTOR_BODY]]:
-; CHECK-NEXT: [[INDEX:%.*]] = phi i32 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[PRED_STORE_CONTINUE14:.*]] ]
+; CHECK-NEXT: [[INDEX:%.*]] = phi i32 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[LATCH14:.*]] ]
; CHECK-NEXT: [[TMP2:%.*]] = add i32 [[INDEX]], 1
; CHECK-NEXT: [[TMP3:%.*]] = add i32 [[INDEX]], 2
; CHECK-NEXT: [[TMP4:%.*]] = add i32 [[INDEX]], 3
+; CHECK-NEXT: br i1 [[C]], label %[[IF1:.*]], label %[[LATCH14]]
+; CHECK: [[IF1]]:
; CHECK-NEXT: [[TMP7:%.*]] = getelementptr i32, ptr [[P]], i32 [[INDEX]]
; CHECK-NEXT: [[TMP8:%.*]] = getelementptr i32, ptr [[P]], i32 [[TMP2]]
; CHECK-NEXT: [[TMP9:%.*]] = getelementptr i32, ptr [[P]], i32 [[TMP3]]
@@ -156,7 +162,7 @@ define void @unknown_dividend(ptr noalias %p, i32 %n, i1 %c) {
; CHECK-NEXT: [[TMP13:%.*]] = insertelement <4 x i32> poison, i32 [[TMP12]], i64 0
; CHECK-NEXT: br label %[[PRED_LOAD_CONTINUE]]
; CHECK: [[PRED_LOAD_CONTINUE]]:
-; CHECK-NEXT: [[TMP14:%.*]] = phi <4 x i32> [ poison, %[[VECTOR_BODY]] ], [ [[TMP13]], %[[PRED_LOAD_IF]] ]
+; CHECK-NEXT: [[TMP14:%.*]] = phi <4 x i32> [ poison, %[[IF1]] ], [ [[TMP13]], %[[PRED_LOAD_IF]] ]
; CHECK-NEXT: br i1 [[C]], label %[[PRED_LOAD_IF3:.*]], label %[[PRED_LOAD_CONTINUE4:.*]]
; CHECK: [[PRED_LOAD_IF3]]:
; CHECK-NEXT: [[TMP16:%.*]] = load i32, ptr [[TMP8]], align 4
@@ -197,12 +203,14 @@ define void @unknown_dividend(ptr noalias %p, i32 %n, i1 %c) {
; CHECK-NEXT: store i32 [[TMP30]], ptr [[TMP9]], align 4
; CHECK-NEXT: br label %[[PRED_STORE_CONTINUE12]]
; CHECK: [[PRED_STORE_CONTINUE12]]:
-; CHECK-NEXT: br i1 [[C]], label %[[PRED_STORE_IF13:.*]], label %[[PRED_STORE_CONTINUE14]]
+; CHECK-NEXT: br i1 [[C]], label %[[PRED_STORE_IF13:.*]], label %[[PRED_STORE_CONTINUE14:.*]]
; CHECK: [[PRED_STORE_IF13]]:
; CHECK-NEXT: [[TMP31:%.*]] = extractelement <4 x i32> [[TMP27]], i64 3
; CHECK-NEXT: store i32 [[TMP31]], ptr [[TMP10]], align 4
; CHECK-NEXT: br label %[[PRED_STORE_CONTINUE14]]
; CHECK: [[PRED_STORE_CONTINUE14]]:
+; CHECK-NEXT: br label %[[LATCH14]]
+; CHECK: [[LATCH14]]:
; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i32 [[INDEX]], 4
; CHECK-NEXT: [[TMP32:%.*]] = icmp eq i32 [[INDEX_NEXT]], [[N_VEC]]
; CHECK-NEXT: br i1 [[TMP32]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP4:![0-9]+]]
diff --git a/llvm/test/Transforms/LoopVectorize/if-pred-stores.ll b/llvm/test/Transforms/LoopVectorize/if-pred-stores.ll
index 4615e95e0cd91..b57899e31811a 100644
--- a/llvm/test/Transforms/LoopVectorize/if-pred-stores.ll
+++ b/llvm/test/Transforms/LoopVectorize/if-pred-stores.ll
@@ -418,22 +418,26 @@ define void @minimal_bit_widths(ptr %p, i1 %c) {
; UNROLL: [[VECTOR_PH]]:
; UNROLL-NEXT: br label %[[VECTOR_BODY:.*]]
; UNROLL: [[VECTOR_BODY]]:
-; UNROLL-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[PRED_STORE_CONTINUE2:.*]] ]
+; UNROLL-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[PRED_STORE_CONTINUE:.*]] ]
; UNROLL-NEXT: [[TMP0:%.*]] = add i64 [[INDEX]], 1
; UNROLL-NEXT: [[TMP1:%.*]] = getelementptr i8, ptr [[P]], i64 [[INDEX]]
; UNROLL-NEXT: [[TMP2:%.*]] = getelementptr i8, ptr [[P]], i64 [[TMP0]]
; UNROLL-NEXT: [[TMP3:%.*]] = load i8, ptr [[TMP1]], align 1
; UNROLL-NEXT: [[TMP4:%.*]] = load i8, ptr [[TMP2]], align 1
-; UNROLL-NEXT: br i1 [[C]], label %[[PRED_STORE_IF:.*]], label %[[PRED_STORE_CONTINUE:.*]]
+; UNROLL-NEXT: br i1 [[C]], label %[[PRED_STORE_IF:.*]], label %[[PRED_STORE_CONTINUE]]
; UNROLL: [[PRED_STORE_IF]]:
+; UNROLL-NEXT: br i1 [[C]], label %[[PRED_STORE_IF2:.*]], label %[[PRED_STORE_CONTINUE1:.*]]
+; UNROLL: [[PRED_STORE_IF2]]:
; UNROLL-NEXT: store i8 [[TMP3]], ptr [[TMP1]], align 1
-; UNROLL-NEXT: br label %[[PRED_STORE_CONTINUE]]
-; UNROLL: [[PRED_STORE_CONTINUE]]:
-; UNROLL-NEXT: br i1 [[C]], label %[[PRED_STORE_IF1:.*]], label %[[PRED_STORE_CONTINUE2]]
+; UNROLL-NEXT: br label %[[PRED_STORE_CONTINUE1]]
+; UNROLL: [[PRED_STORE_CONTINUE1]]:
+; UNROLL-NEXT: br i1 [[C]], label %[[PRED_STORE_IF1:.*]], label %[[PRED_STORE_CONTINUE2:.*]]
; UNROLL: [[PRED_STORE_IF1]]:
; UNROLL-NEXT: store i8 [[TMP4]], ptr [[TMP2]], align 1
; UNROLL-NEXT: br label %[[PRED_STORE_CONTINUE2]]
; UNROLL: [[PRED_STORE_CONTINUE2]]:
+; UNROLL-NEXT: br label %[[PRED_STORE_CONTINUE]]
+; UNROLL: [[PRED_STORE_CONTINUE]]:
; UNROLL-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 2
; UNROLL-NEXT: [[TMP5:%.*]] = icmp eq i64 [[INDEX_NEXT]], 1000
; UNROLL-NEXT: br i1 [[TMP5]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP5:![0-9]+]]
@@ -449,22 +453,26 @@ define void @minimal_bit_widths(ptr %p, i1 %c) {
; UNROLL-NOSIMPLIFY: [[VECTOR_PH]]:
; UNROLL-NOSIMPLIFY-NEXT: br label %[[VECTOR_BODY:.*]]
; UNROLL-NOSIMPLIFY: [[VECTOR_BODY]]:
-; UNROLL-NOSIMPLIFY-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[PRED_STORE_CONTINUE2:.*]] ]
+; UNROLL-NOSIMPLIFY-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[PRED_STORE_CONTINUE:.*]] ]
; UNROLL-NOSIMPLIFY-NEXT: [[TMP0:%.*]] = add i64 [[INDEX]], 1
; UNROLL-NOSIMPLIFY-NEXT: [[TMP1:%.*]] = getelementptr i8, ptr [[P]], i64 [[INDEX]]
; UNROLL-NOSIMPLIFY-NEXT: [[TMP2:%.*]] = getelementptr i8, ptr [[P]], i64 [[TMP0]]
; UNROLL-NOSIMPLIFY-NEXT: [[TMP3:%.*]] = load i8, ptr [[TMP1]], align 1
; UNROLL-NOSIMPLIFY-NEXT: [[TMP4:%.*]] = load i8, ptr [[TMP2]], align 1
-; UNROLL-NOSIMPLIFY-NEXT: br i1 [[C]], label %[[PRED_STORE_IF:.*]], label %[[PRED_STORE_CONTINUE:.*]]
+; UNROLL-NOSIMPLIFY-NEXT: br i1 [[C]], label %[[PRED_STORE_IF:.*]], label %[[PRED_STORE_CONTINUE]]
; UNROLL-NOSIMPLIFY: [[PRED_STORE_IF]]:
+; UNROLL-NOSIMPLIFY-NEXT: br i1 [[C]], label %[[PRED_STORE_IF2:.*]], label %[[PRED_STORE_CONTINUE1:.*]]
+; UNROLL-NOSIMPLIFY: [[PRED_STORE_IF2]]:
; UNROLL-NOSIMPLIFY-NEXT: store i8 [[TMP3]], ptr [[TMP1]], align 1
-; UNROLL-NOSIMPLIFY-NEXT: br label %[[PRED_STORE_CONTINUE]]
-; UNROLL-NOSIMPLIFY: [[PRED_STORE_CONTINUE]]:
-; UNROLL-NOSIMPLIFY-NEXT: br i1 [[C]], label %[[PRED_STORE_IF1:.*]], label %[[PRED_STORE_CONTINUE2]]
+; UNROLL-NOSIMPLIFY-NEXT: br label %[[PRED_STORE_CONTINUE1]]
+; UNROLL-NOSIMPLIFY: [[PRED_STORE_CONTINUE1]]:
+; UNROLL-NOSIMPLIFY-NEXT: br i1 [[C]], label %[[PRED_STORE_IF1:.*]], label %[[PRED_STORE_CONTINUE2:.*]]
; UNROLL-NOSIMPLIFY: [[PRED_STORE_IF1]]:
; UNROLL-NOSIMPLIFY-NEXT: store i8 [[TMP4]], ptr [[TMP2]], align 1
; UNROLL-NOSIMPLIFY-NEXT: br label %[[PRED_STORE_CONTINUE2]]
; UNROLL-NOSIMPLIFY: [[PRED_STORE_CONTINUE2]]:
+; UNROLL-NOSIMPLIFY-NEXT: br label %[[PRED_STORE_CONTINUE]]
+; UNROLL-NOSIMPLIFY: [[PRED_STORE_CONTINUE]]:
; UNROLL-NOSIMPLIFY-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 2
; UNROLL-NOSIMPLIFY-NEXT: [[TMP5:%.*]] = icmp eq i64 [[INDEX_NEXT]], 1000
; UNROLL-NOSIMPLIFY-NEXT: br i1 [[TMP5]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP5:![0-9]+]]
@@ -480,16 +488,18 @@ define void @minimal_bit_widths(ptr %p, i1 %c) {
; VEC: [[VECTOR_PH]]:
; VEC-NEXT: br label %[[VECTOR_BODY:.*]]
; VEC: [[VECTOR_BODY]]:
-; VEC-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[PRED_STORE_CONTINUE2:.*]] ]
+; VEC-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[PRED_STORE_CONTINUE:.*]] ]
; VEC-NEXT: [[TMP0:%.*]] = getelementptr i8, ptr [[P]], i64 [[INDEX]]
; VEC-NEXT: [[WIDE_LOAD:%.*]] = load <2 x i8>, ptr [[TMP0]], align 1
-; VEC-NEXT: br i1 [[C]], label %[[PRED_STORE_IF:.*]], label %[[PRED_STORE_CONTINUE:.*]]
+; VEC-NEXT: br i1 [[C]], label %[[PRED_STORE_IF:.*]], label %[[PRED_STORE_CONTINUE]]
; VEC: [[PRED_STORE_IF]]:
+; VEC-NEXT: br i1 [[C]], label %[[PRED_STORE_IF2:.*]], label %[[PRED_STORE_CONTINUE1:.*]]
+; VEC: [[PRED_STORE_IF2]]:
; VEC-NEXT: [[TMP1:%.*]] = extractelement <2 x i8> [[WIDE_LOAD]], i64 0
; VEC-NEXT: store i8 [[TMP1]], ptr [[TMP0]], align 1
-; VEC-NEXT: br label %[[PRED_STORE_CONTINUE]]
-; VEC: [[PRED_STORE_CONTINUE]]:
-; VEC-NEXT: br i1 [[C]], label %[[PRED_STORE_IF1:.*]], label %[[PRED_STORE_CONTINUE2]]
+; VEC-NEXT: br label %[[PRED_STORE_CONTINUE1]]
+; VEC: [[PRED_STORE_CONTINUE1]]:
+; VEC-NEXT: br i1 [[C]], label %[[PRED_STORE_IF1:.*]], label %[[PRED_STORE_CONTINUE2:.*]]
; VEC: [[PRED_STORE_IF1]]:
; VEC-NEXT: [[TMP2:%.*]] = add i64 [[INDEX]], 1
; VEC-NEXT: [[TMP3:%.*]] = getelementptr i8, ptr [[P]], i64 [[TMP2]]
@@ -497,6 +507,8 @@ define void @minimal_bit_widths(ptr %p, i1 %c) {
; VEC-NEXT: store i8 [[TMP4]], ptr [[TMP3]], align 1
; VEC-NEXT: br label %[[PRED_STORE_CONTINUE2]]
; VEC: [[PRED_STORE_CONTINUE2]]:
+; VEC-NEXT: br label %[[PRED_STORE_CONTINUE]]
+; VEC: [[PRED_STORE_CONTINUE]]:
; VEC-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 2
; VEC-NEXT: [[TMP5:%.*]] = icmp eq i64 [[INDEX_NEXT]], 1000
; VEC-NEXT: br i1 [[TMP5]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP5:![0-9]+]]
@@ -539,7 +551,7 @@ define void @minimal_bit_widths_with_aliasing_store(i1 %c, ptr %ptr) {
; UNROLL: [[VECTOR_PH]]:
; UNROLL-NEXT: br label %[[VECTOR_BODY:.*]]
; UNROLL: [[VECTOR_BODY]]:
-; UNROLL-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[PRED_STORE_CONTINUE2:.*]] ]
+; UNROLL-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[PRED_STORE_CONTINUE:.*]] ]
; UNROLL-NEXT: [[TMP0:%.*]] = add i64 [[INDEX]], 1
; UNROLL-NEXT: [[TMP1:%.*]] = getelementptr i8, ptr [[PTR]], i64 [[INDEX]]
; UNROLL-NEXT: [[TMP2:%.*]] = getelementptr i8, ptr [[PTR]], i64 [[TMP0]]
@@ -547,16 +559,20 @@ define void @minimal_bit_widths_with_aliasing_store(i1 %c, ptr %ptr) {
; UNROLL-NEXT: [[TMP4:%.*]] = load i8, ptr [[TMP2]], align 1
; UNROLL-NEXT: store i8 0, ptr [[TMP1]], align 1
; UNROLL-NEXT: store i8 0, ptr [[TMP2]], align 1
-; UNROLL-NEXT: br i1 [[C]], label %[[PRED_STORE_IF:.*]], label %[[PRED_STORE_CONTINUE:.*]]
+; UNROLL-NEXT: br i1 [[C]], label %[[PRED_STORE_IF:.*]], label %[[PRED_STORE_CONTINUE]]
; UNROLL: [[PRED_STORE_IF]]:
+; UNROLL-NEXT: br i1 [[C]], label %[[PRED_STORE_IF2:.*]], label %[[PRED_STORE_CONTINUE1:.*]]
+; UNROLL: [[PRED_STORE_IF2]]:
; UNROLL-NEXT: store i8 [[TMP3]], ptr [[TMP1]], align 1
-; UNROLL-NEXT: br label %[[PRED_STORE_CONTINUE]]
-; UNROLL: [[PRED_STORE_CONTINUE]]:
-; UNROLL-NEXT: br i1 [[C]], label %[[PRED_STORE_IF1:.*]], label %[[PRED_STORE_CONTINUE2]]
+; UNROLL-NEXT: br label %[[PRED_STORE_CONTINUE1]]
+; UNROLL: [[PRED_STORE_CONTINUE1]]:
+; UNROLL-NEXT: br i1 [[C]], label %[[PRED_STORE_IF1:.*]], label %[[PRED_STORE_CONTINUE2:.*]]
; UNROLL: [[PRED_STORE_IF1]]:
; UNROLL-NEXT: store i8 [[TMP4]], ptr [[TMP2]], align 1
; UNROLL-NEXT: br label %[[PRED_STORE_CONTINUE2]]
; UNROLL: [[PRED_STORE_CONTINUE2]]:
+; UNROLL-NEXT: br label %[[PRED_STORE_CONTINUE]]
+; UNROLL: [[PRED_STORE_CONTINUE]]:
; UNROLL-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 2
; UNROLL-NEXT: [[TMP5:%.*]] = icmp eq i64 [[INDEX_NEXT]], 1000
; UNROLL-NEXT: br i1 [[TMP5]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP6:![0-9]+]]
@@ -572,7 +588,7 @@ define void @minimal_bit_widths_with_aliasing_store(i1 %c, ptr %ptr) {
; UNROLL-NOSIMPLIFY: [[VECTOR_PH]]:
; UNROLL-NOSIMPLIFY-NEXT: br label %[[VECTOR_BODY:.*]]
; UNROLL-NOSIMPLIFY: [[VECTOR_BODY]]:
-; UNROLL-NOSIMPLIFY-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[PRED_STORE_CONTINUE2:.*]] ]
+; UNROLL-NOSIMPLIFY-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[PRED_STORE_CONTINUE:.*]] ]
; UNROLL-NOSIMPLIFY-NEXT: [[TMP0:%.*]] = add i64 [[INDEX]], 1
; UNROLL-NOSIMPLIFY-NEXT: [[TMP1:%.*]] = getelementptr i8, ptr [[PTR]], i64 [[INDEX]]
; UNROLL-NOSIMPLIFY-NEXT: [[TMP2:%.*]] = getelementptr i8, ptr [[PTR]], i64 [[TMP0]]
@@ -580,16 +596,20 @@ define void @minimal_bit_widths_with_aliasing_store(i1 %c, ptr %ptr) {
; UNROLL-NOSIMPLIFY-NEXT: [[TMP4:%.*]] = load i8, ptr [[TMP2]], align 1
; UNROLL-NOSIMPLIFY-NEXT: store i8 0, ptr [[TMP1]], align 1
; UNROLL-NOSIMPLIFY-NEXT: store i8 0, ptr [[TMP2]], align 1
-; UNROLL-NOSIMPLIFY-NEXT: br i1 [[C]], label %[[PRED_STORE_IF:.*]], label %[[PRED_STORE_CONTINUE:.*]]
+; UNROLL-NOSIMPLIFY-NEXT: br i1 [[C]], label %[[PRED_STORE_IF:.*]], label %[[PRED_STORE_CONTINUE]]
; UNROLL-NOSIMPLIFY: [[PRED_STORE_IF]]:
+; UNROLL-NOSIMPLIFY-NEXT: br i1 [[C]], label %[[PRED_STORE_IF2:.*]], label %[[PRED_STORE_CONTINUE1:.*]]
+; UNROLL-NOSIMPLIFY: [[PRED_STORE_IF2]]:
; UNROLL-NOSIMPLIFY-NEXT: store i8 [[TMP3]], ptr [[TMP1]], align 1
-; UNROLL-NOSIMPLIFY-NEXT: br label %[[PRED_STORE_CONTINUE]]
-; UNROLL-NOSIMPLIFY: [[PRED_STORE_CONTINUE]]:
-; UNROLL-NOSIMPLIFY-NEXT: br i1 [[C]], label %[[PRED_STORE_IF1:.*]], label %[[PRED_STORE_CONTINUE2]]
+; UNROLL-NOSIMPLIFY-NEXT: br label %[[PRED_STORE_CONTINUE1]]
+; UNROLL-NOSIMPLIFY: [[PRED_STORE_CONTINUE1]]:
+; UNROLL-NOSIMPLIFY-NEXT: br i1 [[C]], label %[[PRED_STORE_IF1:.*]], label %[[PRED_STORE_CONTINUE2:.*]]
; UNROLL-NOSIMPLIFY: [[PRED_STORE_IF1]]:
; UNROLL-NOSIMPLIFY-NEXT: store i8 [[TMP4]], ptr [[TMP2]], align 1
; UNROLL-NOSIMPLIFY-NEXT: br label %[[PRED_STORE_CONTINUE2]]
; UNROLL-NOSIMPLIFY: [[PRED_STORE_CONTINUE2]]:
+; UNROLL-NOSIMPLIFY-NEXT: br label %[[PRED_STORE_CONTINUE]]
+; UNROLL-NOSIMPLIFY: [[PRED_STORE_CONTINUE]]:
; UNROLL-NOSIMPLIFY-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 2
; UNROLL-NOSIMPLIFY-NEXT: [[TMP5:%.*]] = icmp eq i64 [[INDEX_NEXT]], 1000
; UNROLL-NOSIMPLIFY-NEXT: br i1 [[TMP5]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP6:![0-9]+]]
@@ -605,17 +625,19 @@ define void @minimal_bit_widths_with_aliasing_store(i1 %c, ptr %ptr) {
; VEC: [[VECTOR_PH]]:
; VEC-NEXT: br label %[[VECTOR_BODY:.*]]
; VEC: [[VECTOR_BODY]]:
-; VEC-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[PRED_STORE_CONTINUE2:.*]] ]
+; VEC-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[PRED_STORE_CONTINUE:.*]] ]
; VEC-NEXT: [[TMP0:%.*]] = getelementptr i8, ptr [[PTR]], i64 [[INDEX]]
; VEC-NEXT: [[WIDE_LOAD:%.*]] = load <2 x i8>, ptr [[TMP0]], align 1
; VEC-NEXT: store <2 x i8> zeroinitializer, ptr [[TMP0]], align 1
-; VEC-NEXT: br i1 [[C]], label %[[PRED_STORE_IF:.*]], label %[[PRED_STORE_CONTINUE:.*]]
+; VEC-NEXT: br i1 [[C]], label %[[PRED_STORE_IF:.*]], label %[[PRED_STORE_CONTINUE]]
; VEC: [[PRED_STORE_IF]]:
+; VEC-NEXT: br i1 [[C]], label %[[PRED_STORE_IF2:.*]], label %[[PRED_STORE_CONTINUE1:.*]]
+; VEC: [[PRED_STORE_IF2]]:
; VEC-NEXT: [[TMP1:%.*]] = extractelement <2 x i8> [[WIDE_LOAD]], i64 0
; VEC-NEXT: store i8 [[TMP1]], ptr [[TMP0]], align 1
-; VEC-NEXT: br label %[[PRED_STORE_CONTINUE]]
-; VEC: [[PRED_STORE_CONTINUE]]:
-; VEC-NEXT: br i1 [[C]], label %[[PRED_STORE_IF1:.*]], label %[[PRED_STORE_CONTINUE2]]
+; VEC-NEXT: br label %[[PRED_STORE_CONTINUE1]]
+; VEC: [[PRED_STORE_CONTINUE1]]:
+; VEC-NEXT: br i1 [[C]], label %[[PRED_STORE_IF1:.*]], label %[[PRED_STORE_CONTINUE2:.*]]
; VEC: [[PRED_STORE_IF1]]:
; VEC-NEXT: [[TMP2:%.*]] = add i64 [[INDEX]], 1
; VEC-NEXT: [[TMP3:%.*]] = getelementptr i8, ptr [[PTR]], i64 [[TMP2]]
@@ -623,6 +645,8 @@ define void @minimal_bit_widths_with_aliasing_store(i1 %c, ptr %ptr) {
; VEC-NEXT: store i8 [[TMP4]], ptr [[TMP3]], align 1
; VEC-NEXT: br label %[[PRED_STORE_CONTINUE2]]
; VEC: [[PRED_STORE_CONTINUE2]]:
+; VEC-NEXT: br label %[[PRED_STORE_CONTINUE]]
+; VEC: [[PRED_STORE_CONTINUE]]:
; VEC-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 2
; VEC-NEXT: [[TMP5:%.*]] = icmp eq i64 [[INDEX_NEXT]], 1000
; VEC-NEXT: br i1 [[TMP5]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP6:![0-9]+]]
@@ -666,19 +690,23 @@ define void @sdiv_with_uniform_ops(i16 %0, i1 %c, ptr %dst) {
; UNROLL: [[VECTOR_PH]]:
; UNROLL-NEXT: br label %[[VECTOR_BODY:.*]]
; UNROLL: [[VECTOR_BODY]]:
-; UNROLL-NEXT: [[INDEX:%.*]] = phi i32 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[PRED_STORE_CONTINUE2:.*]] ]
-; UNROLL-NEXT: br i1 [[C]], label %[[PRED_STORE_IF:.*]], label %[[PRED_STORE_CONTINUE:.*]]
+; UNROLL-NEXT: [[INDEX:%.*]] = phi i32 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[PRED_STORE_CONTINUE:.*]] ]
+; UNROLL-NEXT: br i1 [[C]], label %[[PRED_STORE_IF:.*]], label %[[PRED_STORE_CONTINUE]]
; UNROLL: [[PRED_STORE_IF]]:
+; UNROLL-NEXT: br i1 [[C]], label %[[PRED_STORE_IF2:.*]], label %[[PRED_STORE_CONTINUE1:.*]]
+; UNROLL: [[PRED_STORE_IF2]]:
; UNROLL-NEXT: [[TMP1:%.*]] = sdiv i16 10, [[TMP0]]
; UNROLL-NEXT: store i16 [[TMP1]], ptr [[DST]], align 1
-; UNROLL-NEXT: br label %[[PRED_STORE_CONTINUE]]
-; UNROLL: [[PRED_STORE_CONTINUE]]:
-; UNROLL-NEXT: br i1 [[C]], label %[[PRED_STORE_IF1:.*]], label %[[PRED_STORE_CONTINUE2]]
+; UNROLL-NEXT: br label %[[PRED_STORE_CONTINUE1]]
+; UNROLL: [[PRED_STORE_CONTINUE1]]:
+; UNROLL-NEXT: br i1 [[C]], label %[[PRED_STORE_IF1:.*]], label %[[PRED_STORE_CONTINUE2:.*]]
; UNROLL: [[PRED_STORE_IF1]]:
; UNROLL-NEXT: [[TMP2:%.*]] = sdiv i16 10, [[TMP0]]
; UNROLL-NEXT: store i16 [[TMP2]], ptr [[DST]], align 1
; UNROLL-NEXT: br label %[[PRED_STORE_CONTINUE2]]
; UNROLL: [[PRED_STORE_CONTINUE2]]:
+; UNROLL-NEXT: br label %[[PRED_STORE_CONTINUE]]
+; UNROLL: [[PRED_STORE_CONTINUE]]:
; UNROLL-NEXT: [[INDEX_NEXT]] = add nuw i32 [[INDEX]], 2
; UNROLL-NEXT: [[TMP3:%.*]] = icmp eq i32 [[INDEX_NEXT]], 98
; UNROLL-NEXT: br i1 [[TMP3]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP7:![0-9]+]]
@@ -707,19 +735,23 @@ define void @sdiv_with_uniform_ops(i16 %0, i1 %c, ptr %dst) {
; UNROLL-NOSIMPLIFY: [[VECTOR_PH]]:
; UNROLL-NOSIMPLIFY-NEXT: br label %[[VECTOR_BODY:.*]]
; UNROLL-NOSIMPLIFY: [[VECTOR_BODY]]:
-; UNROLL-NOSIMPLIFY-NEXT: [[INDEX:%.*]] = phi i32 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[PRED_STORE_CONTINUE2:.*]] ]
-; UNROLL-NOSIMPLIFY-NEXT: br i1 [[C]], label %[[PRED_STORE_IF:.*]], label %[[PRED_STORE_CONTINUE:.*]]
+; UNROLL-NOSIMPLIFY-NEXT: [[INDEX:%.*]] = phi i32 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[PRED_STORE_CONTINUE:.*]] ]
+; UNROLL-NOSIMPLIFY-NEXT: br i1 [[C]], label %[[PRED_STORE_IF:.*]], label %[[PRED_STORE_CONTINUE]]
; UNROLL-NOSIMPLIFY: [[PRED_STORE_IF]]:
+; UNROLL-NOSIMPLIFY-NEXT: br i1 [[C]], label %[[PRED_STORE_IF2:.*]], label %[[PRED_STORE_CONTINUE1:.*]]
+; UNROLL-NOSIMPLIFY: [[PRED_STORE_IF2]]:
; UNROLL-NOSIMPLIFY-NEXT: [[TMP1:%.*]] = sdiv i16 10, [[TMP0]]
; UNROLL-NOSIMPLIFY-NEXT: store i16 [[TMP1]], ptr [[DST]], align 1
-; UNROLL-NOSIMPLIFY-NEXT: br label %[[PRED_STORE_CONTINUE]]
-; UNROLL-NOSIMPLIFY: [[PRED_STORE_CONTINUE]]:
-; UNROLL-NOSIMPLIFY-NEXT: br i1 [[C]], label %[[PRED_STORE_IF1:.*]], label %[[PRED_STORE_CONTINUE2]]
+; UNROLL-NOSIMPLIFY-NEXT: br label %[[PRED_STORE_CONTINUE1]]
+; UNROLL-NOSIMPLIFY: [[PRED_STORE_CONTINUE1]]:
+; UNROLL-NOSIMPLIFY-NEXT: br i1 [[C]], label %[[PRED_STORE_IF1:.*]], label %[[PRED_STORE_CONTINUE2:.*]]
; UNROLL-NOSIMPLIFY: [[PRED_STORE_IF1]]:
; UNROLL-NOSIMPLIFY-NEXT: [[TMP2:%.*]] = sdiv i16 10, [[TMP0]]
; UNROLL-NOSIMPLIFY-NEXT: store i16 [[TMP2]], ptr [[DST]], align 1
; UNROLL-NOSIMPLIFY-NEXT: br label %[[PRED_STORE_CONTINUE2]]
; UNROLL-NOSIMPLIFY: [[PRED_STORE_CONTINUE2]]:
+; UNROLL-NOSIMPLIFY-NEXT: br label %[[PRED_STORE_CONTINUE]]
+; UNROLL-NOSIMPLIFY: [[PRED_STORE_CONTINUE]]:
; UNROLL-NOSIMPLIFY-NEXT: [[INDEX_NEXT]] = add nuw i32 [[INDEX]], 2
; UNROLL-NOSIMPLIFY-NEXT: [[TMP3:%.*]] = icmp eq i32 [[INDEX_NEXT]], 98
; UNROLL-NOSIMPLIFY-NEXT: br i1 [[TMP3]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP7:![0-9]+]]
@@ -753,19 +785,23 @@ define void @sdiv_with_uniform_ops(i16 %0, i1 %c, ptr %dst) {
; VEC-NEXT: [[TMP1:%.*]] = call <2 x i16> @llvm.masked.sdiv.v2i16(<2 x i16> splat (i16 10), <2 x i16> [[BROADCAST_SPLAT]], <2 x i1> [[BROADCAST_SPLAT2]])
; VEC-NEXT: br label %[[VECTOR_BODY:.*]]
; VEC: [[VECTOR_BODY]]:
-; VEC-NEXT: [[INDEX:%.*]] = phi i32 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[PRED_STORE_CONTINUE4:.*]] ]
-; VEC-NEXT: br i1 [[C]], label %[[PRED_STORE_IF:.*]], label %[[PRED_STORE_CONTINUE:.*]]
+; VEC-NEXT: [[INDEX:%.*]] = phi i32 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[PRED_STORE_CONTINUE:.*]] ]
+; VEC-NEXT: br i1 [[C]], label %[[PRED_STORE_IF:.*]], label %[[PRED_STORE_CONTINUE]]
; VEC: [[PRED_STORE_IF]]:
+; VEC-NEXT: br i1 [[C]], label %[[PRED_STORE_IF1:.*]], label %[[PRED_STORE_CONTINUE1:.*]]
+; VEC: [[PRED_STORE_IF1]]:
; VEC-NEXT: [[TMP2:%.*]] = extractelement <2 x i16> [[TMP1]], i64 0
; VEC-NEXT: store i16 [[TMP2]], ptr [[DST]], align 1
-; VEC-NEXT: br label %[[PRED_STORE_CONTINUE]]
-; VEC: [[PRED_STORE_CONTINUE]]:
-; VEC-NEXT: br i1 [[C]], label %[[PRED_STORE_IF3:.*]], label %[[PRED_STORE_CONTINUE4]]
+; VEC-NEXT: br label %[[PRED_STORE_CONTINUE1]]
+; VEC: [[PRED_STORE_CONTINUE1]]:
+; VEC-NEXT: br i1 [[C]], label %[[PRED_STORE_IF3:.*]], label %[[PRED_STORE_CONTINUE4:.*]]
; VEC: [[PRED_STORE_IF3]]:
; VEC-NEXT: [[TMP3:%.*]] = extractelement <2 x i16> [[TMP1]], i64 1
; VEC-NEXT: store i16 [[TMP3]], ptr [[DST]], align 1
; VEC-NEXT: br label %[[PRED_STORE_CONTINUE4]]
; VEC: [[PRED_STORE_CONTINUE4]]:
+; VEC-NEXT: br label %[[PRED_STORE_CONTINUE]]
+; VEC: [[PRED_STORE_CONTINUE]]:
; VEC-NEXT: [[INDEX_NEXT]] = add nuw i32 [[INDEX]], 2
; VEC-NEXT: [[TMP4:%.*]] = icmp eq i32 [[INDEX_NEXT]], 98
; VEC-NEXT: br i1 [[TMP4]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP7:![0-9]+]]
diff --git a/llvm/test/Transforms/LoopVectorize/pr154045-dont-fold-extractelement-livein.ll b/llvm/test/Transforms/LoopVectorize/pr154045-dont-fold-extractelement-livein.ll
index 2b63506440074..c2708cd3bf608 100644
--- a/llvm/test/Transforms/LoopVectorize/pr154045-dont-fold-extractelement-livein.ll
+++ b/llvm/test/Transforms/LoopVectorize/pr154045-dont-fold-extractelement-livein.ll
@@ -12,11 +12,15 @@ define void @pr154045(ptr %p, i1 %c, i64 %x) {
; CHECK: [[VECTOR_PH]]:
; CHECK-NEXT: br label %[[VECTOR_BODY:.*]]
; CHECK: [[VECTOR_BODY]]:
-; CHECK-NEXT: br label %[[MIDDLE_BLOCK:.*]]
+; CHECK-NEXT: br i1 [[C]], label %[[MIDDLE_BLOCK:.*]], label %[[ELSE1:.*]]
+; CHECK: [[ELSE1]]:
+; CHECK-NEXT: br label %[[MIDDLE_BLOCK]]
; CHECK: [[MIDDLE_BLOCK]]:
; CHECK-NEXT: store i32 0, ptr [[P]], align 4
; CHECK-NEXT: br label %[[EXIT:.*]]
; CHECK: [[EXIT]]:
+; CHECK-NEXT: br label %[[EXIT1:.*]]
+; CHECK: [[EXIT1]]:
; CHECK-NEXT: ret void
;
entry:
diff --git a/llvm/test/Transforms/LoopVectorize/pr37248.ll b/llvm/test/Transforms/LoopVectorize/pr37248.ll
index 45540d86f54a2..54109d418f5ba 100644
--- a/llvm/test/Transforms/LoopVectorize/pr37248.ll
+++ b/llvm/test/Transforms/LoopVectorize/pr37248.ll
@@ -38,18 +38,22 @@ define void @f1(ptr noalias %b, i1 %c, i32 %start) {
; CHECK-NEXT: [[TMP12:%.*]] = xor i1 [[C]], true
; CHECK-NEXT: br label %[[VECTOR_BODY:.*]]
; CHECK: [[VECTOR_BODY]]:
-; CHECK-NEXT: [[INDEX:%.*]] = phi i32 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[PRED_STORE_CONTINUE3:.*]] ]
+; CHECK-NEXT: [[INDEX:%.*]] = phi i32 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[LOOP_LATCH4:.*]] ]
; CHECK-NEXT: [[OFFSET_IDX:%.*]] = sub i32 [[START]], [[INDEX]]
+; CHECK-NEXT: br i1 [[C]], label %[[LOOP_LATCH4]], label %[[ELSE1:.*]]
+; CHECK: [[ELSE1]]:
; CHECK-NEXT: br i1 [[TMP12]], label %[[PRED_STORE_IF:.*]], label %[[PRED_STORE_CONTINUE:.*]]
; CHECK: [[PRED_STORE_IF]]:
; CHECK-NEXT: store i32 10, ptr [[B]], align 1
; CHECK-NEXT: br label %[[PRED_STORE_CONTINUE]]
; CHECK: [[PRED_STORE_CONTINUE]]:
-; CHECK-NEXT: br i1 [[TMP12]], label %[[PRED_STORE_IF2:.*]], label %[[PRED_STORE_CONTINUE3]]
+; CHECK-NEXT: br i1 [[TMP12]], label %[[PRED_STORE_IF2:.*]], label %[[PRED_STORE_CONTINUE3:.*]]
; CHECK: [[PRED_STORE_IF2]]:
; CHECK-NEXT: store i32 10, ptr [[B]], align 1
; CHECK-NEXT: br label %[[PRED_STORE_CONTINUE3]]
; CHECK: [[PRED_STORE_CONTINUE3]]:
+; CHECK-NEXT: br label %[[LOOP_LATCH4]]
+; CHECK: [[LOOP_LATCH4]]:
; CHECK-NEXT: [[TMP13:%.*]] = trunc i32 [[OFFSET_IDX]] to i16
; CHECK-NEXT: [[TMP15:%.*]] = getelementptr inbounds [2 x i16], ptr @a, i16 0, i16 [[TMP13]]
; CHECK-NEXT: [[TMP17:%.*]] = getelementptr inbounds i16, ptr [[TMP15]], i64 -1
@@ -110,8 +114,12 @@ define void @f2(ptr noalias %b, i1 %c, i32 %start) {
; CHECK-NEXT: [[TMP10:%.*]] = sub i32 [[START]], [[N_VEC]]
; CHECK-NEXT: br label %[[VECTOR_BODY:.*]]
; CHECK: [[VECTOR_BODY]]:
-; CHECK-NEXT: [[INDEX:%.*]] = phi i32 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-NEXT: [[INDEX:%.*]] = phi i32 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[LOOP_LATCH2:.*]] ]
; CHECK-NEXT: [[OFFSET_IDX:%.*]] = sub i32 [[START]], [[INDEX]]
+; CHECK-NEXT: br i1 [[C]], label %[[LOOP_LATCH2]], label %[[ELSE1:.*]]
+; CHECK: [[ELSE1]]:
+; CHECK-NEXT: br label %[[LOOP_LATCH2]]
+; CHECK: [[LOOP_LATCH2]]:
; CHECK-NEXT: [[TMP11:%.*]] = trunc i32 [[OFFSET_IDX]] to i16
; CHECK-NEXT: [[TMP12:%.*]] = getelementptr inbounds [2 x i16], ptr @a, i16 0, i16 [[TMP11]]
; CHECK-NEXT: [[TMP14:%.*]] = getelementptr inbounds i16, ptr [[TMP12]], i64 -1
diff --git a/llvm/test/Transforms/LoopVectorize/predicator.ll b/llvm/test/Transforms/LoopVectorize/predicator.ll
index 8a1afd76d60cc..6623c01a01ab5 100644
--- a/llvm/test/Transforms/LoopVectorize/predicator.ll
+++ b/llvm/test/Transforms/LoopVectorize/predicator.ll
@@ -248,7 +248,11 @@ define void @simplifiable_blend(i1 %c1, i1 %c2, i1 %c3, i32 %x, i32 %y, ptr %p)
; CHECK-NEXT: [[PREDPHI7:%.*]] = select i1 [[C1]], <4 x i32> [[BROADCAST_SPLAT2]], <4 x i32> [[BROADCAST_SPLAT]]
; CHECK-NEXT: br label %[[VECTOR_BODY:.*]]
; CHECK: [[VECTOR_BODY]]:
-; CHECK-NEXT: [[INDEX:%.*]] = phi i32 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-NEXT: [[INDEX:%.*]] = phi i32 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[F4:.*]] ]
+; CHECK-NEXT: br i1 [[C3]], label %[[F4]], label %[[E3:.*]]
+; CHECK: [[E3]]:
+; CHECK-NEXT: br label %[[F4]]
+; CHECK: [[F4]]:
; CHECK-NEXT: [[TMP3:%.*]] = getelementptr i32, ptr [[P]], i32 [[INDEX]]
; CHECK-NEXT: store <4 x i32> [[PREDPHI7]], ptr [[TMP3]], align 4
; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i32 [[INDEX]], 4
@@ -377,16 +381,20 @@ define void @outermost_uniform_branch_no_phi(ptr %a, i1 %u0) {
; CHECK-NEXT: [[BROADCAST_SPLAT:%.*]] = shufflevector <4 x i1> [[BROADCAST_SPLATINSERT]], <4 x i1> poison, <4 x i32> zeroinitializer
; CHECK-NEXT: br label %[[VECTOR_BODY:.*]]
; CHECK: [[VECTOR_BODY]]:
-; CHECK-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
-; CHECK-NEXT: [[VEC_IND:%.*]] = phi <4 x i64> [ <i64 0, i64 1, i64 2, i64 3>, %[[VECTOR_PH]] ], [ [[VEC_IND_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[BB42:.*]] ]
+; CHECK-NEXT: [[VEC_IND:%.*]] = phi <4 x i64> [ <i64 0, i64 1, i64 2, i64 3>, %[[VECTOR_PH]] ], [ [[VEC_IND_NEXT:%.*]], %[[BB42]] ]
; CHECK-NEXT: [[TMP0:%.*]] = getelementptr i64, ptr [[A]], i64 [[INDEX]]
; CHECK-NEXT: store <4 x i64> [[VEC_IND]], ptr [[TMP0]], align 4
+; CHECK-NEXT: br i1 [[U0]], label %[[BB11:.*]], label %[[BB42]]
+; CHECK: [[BB11]]:
; CHECK-NEXT: [[TMP1:%.*]] = add <4 x i64> [[VEC_IND]], splat (i64 1)
; CHECK-NEXT: call void @llvm.masked.store.v4i64.p0(<4 x i64> [[TMP1]], ptr align 4 [[TMP0]], <4 x i1> [[BROADCAST_SPLAT]])
; CHECK-NEXT: [[TMP2:%.*]] = icmp sle <4 x i64> [[VEC_IND]], splat (i64 1)
; CHECK-NEXT: [[TMP3:%.*]] = select <4 x i1> [[BROADCAST_SPLAT]], <4 x i1> [[TMP2]], <4 x i1> zeroinitializer
; CHECK-NEXT: [[TMP4:%.*]] = add <4 x i64> [[VEC_IND]], splat (i64 2)
; CHECK-NEXT: call void @llvm.masked.store.v4i64.p0(<4 x i64> [[TMP4]], ptr align 4 [[TMP0]], <4 x i1> [[TMP3]])
+; CHECK-NEXT: br label %[[BB42]]
+; CHECK: [[BB42]]:
; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 4
; CHECK-NEXT: [[VEC_IND_NEXT]] = add nuw nsw <4 x i64> [[VEC_IND]], splat (i64 4)
; CHECK-NEXT: [[TMP5:%.*]] = icmp eq i64 [[INDEX_NEXT]], 128
@@ -528,11 +536,15 @@ define void @outermost_uniform_branch_more_blocks_no_phi(ptr %a, i1 %u0) {
; CHECK-NEXT: [[TMP0:%.*]] = xor <4 x i1> [[BROADCAST_SPLAT]], splat (i1 true)
; CHECK-NEXT: br label %[[VECTOR_BODY:.*]]
; CHECK: [[VECTOR_BODY]]:
-; CHECK-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
-; CHECK-NEXT: [[VEC_IND:%.*]] = phi <4 x i64> [ <i64 0, i64 1, i64 2, i64 3>, %[[VECTOR_PH]] ], [ [[VEC_IND_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[BB63:.*]] ]
+; CHECK-NEXT: [[VEC_IND:%.*]] = phi <4 x i64> [ <i64 0, i64 1, i64 2, i64 3>, %[[VECTOR_PH]] ], [ [[VEC_IND_NEXT:%.*]], %[[BB63]] ]
; CHECK-NEXT: [[TMP1:%.*]] = getelementptr i64, ptr [[A]], i64 [[INDEX]]
+; CHECK-NEXT: br i1 [[U0]], label %[[BB22:.*]], label %[[BB11:.*]]
+; CHECK: [[BB11]]:
; CHECK-NEXT: [[TMP2:%.*]] = add <4 x i64> [[VEC_IND]], splat (i64 1)
; CHECK-NEXT: call void @llvm.masked.store.v4i64.p0(<4 x i64> [[TMP2]], ptr align 4 [[TMP1]], <4 x i1> [[TMP0]])
+; CHECK-NEXT: br label %[[BB63]]
+; CHECK: [[BB22]]:
; CHECK-NEXT: [[TMP3:%.*]] = icmp sle <4 x i64> [[VEC_IND]], splat (i64 2)
; CHECK-NEXT: [[TMP4:%.*]] = xor <4 x i1> [[TMP3]], splat (i1 true)
; CHECK-NEXT: [[TMP5:%.*]] = select <4 x i1> [[BROADCAST_SPLAT]], <4 x i1> [[TMP4]], <4 x i1> zeroinitializer
@@ -541,6 +553,8 @@ define void @outermost_uniform_branch_more_blocks_no_phi(ptr %a, i1 %u0) {
; CHECK-NEXT: [[TMP7:%.*]] = select <4 x i1> [[BROADCAST_SPLAT]], <4 x i1> [[TMP3]], <4 x i1> zeroinitializer
; CHECK-NEXT: [[TMP8:%.*]] = add <4 x i64> [[VEC_IND]], splat (i64 4)
; CHECK-NEXT: call void @llvm.masked.store.v4i64.p0(<4 x i64> [[TMP8]], ptr align 4 [[TMP1]], <4 x i1> [[TMP7]])
+; CHECK-NEXT: br label %[[BB63]]
+; CHECK: [[BB63]]:
; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 4
; CHECK-NEXT: [[VEC_IND_NEXT]] = add nuw nsw <4 x i64> [[VEC_IND]], splat (i64 4)
; CHECK-NEXT: [[TMP9:%.*]] = icmp eq i64 [[INDEX_NEXT]], 128
@@ -683,16 +697,20 @@ define void @uniform_branch_after_varying_branch_no_phi(ptr %a, i1 %u1) {
; CHECK-NEXT: [[BROADCAST_SPLAT:%.*]] = shufflevector <4 x i1> [[BROADCAST_SPLATINSERT]], <4 x i1> poison, <4 x i32> zeroinitializer
; CHECK-NEXT: br label %[[VECTOR_BODY:.*]]
; CHECK: [[VECTOR_BODY]]:
-; CHECK-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
-; CHECK-NEXT: [[VEC_IND:%.*]] = phi <4 x i64> [ <i64 0, i64 1, i64 2, i64 3>, %[[VECTOR_PH]] ], [ [[VEC_IND_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[BB32:.*]] ]
+; CHECK-NEXT: [[VEC_IND:%.*]] = phi <4 x i64> [ <i64 0, i64 1, i64 2, i64 3>, %[[VECTOR_PH]] ], [ [[VEC_IND_NEXT:%.*]], %[[BB32]] ]
; CHECK-NEXT: [[TMP0:%.*]] = getelementptr i64, ptr [[A]], i64 [[INDEX]]
; CHECK-NEXT: [[TMP1:%.*]] = icmp sle <4 x i64> [[VEC_IND]], zeroinitializer
; CHECK-NEXT: [[TMP2:%.*]] = xor <4 x i1> [[TMP1]], splat (i1 true)
; CHECK-NEXT: [[TMP3:%.*]] = add <4 x i64> [[VEC_IND]], splat (i64 1)
; CHECK-NEXT: call void @llvm.masked.store.v4i64.p0(<4 x i64> [[TMP3]], ptr align 4 [[TMP0]], <4 x i1> [[TMP2]])
+; CHECK-NEXT: br i1 [[U1]], label %[[BB21:.*]], label %[[BB32]]
+; CHECK: [[BB21]]:
; CHECK-NEXT: [[TMP4:%.*]] = select <4 x i1> [[TMP2]], <4 x i1> [[BROADCAST_SPLAT]], <4 x i1> zeroinitializer
; CHECK-NEXT: [[TMP5:%.*]] = add <4 x i64> [[VEC_IND]], splat (i64 2)
; CHECK-NEXT: call void @llvm.masked.store.v4i64.p0(<4 x i64> [[TMP5]], ptr align 4 [[TMP0]], <4 x i1> [[TMP4]])
+; CHECK-NEXT: br label %[[BB32]]
+; CHECK: [[BB32]]:
; CHECK-NEXT: [[TMP6:%.*]] = add <4 x i64> [[VEC_IND]], splat (i64 4)
; CHECK-NEXT: call void @llvm.masked.store.v4i64.p0(<4 x i64> [[TMP6]], ptr align 4 [[TMP0]], <4 x i1> [[TMP1]])
; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 4
@@ -838,17 +856,23 @@ define void @uniform_branch_after_varying_branch_more_blocks_no_phi(ptr %a, i1 %
; CHECK-NEXT: [[TMP0:%.*]] = xor <4 x i1> [[BROADCAST_SPLAT]], splat (i1 true)
; CHECK-NEXT: br label %[[VECTOR_BODY:.*]]
; CHECK: [[VECTOR_BODY]]:
-; CHECK-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
-; CHECK-NEXT: [[VEC_IND:%.*]] = phi <4 x i64> [ <i64 0, i64 1, i64 2, i64 3>, %[[VECTOR_PH]] ], [ [[VEC_IND_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[BB43:.*]] ]
+; CHECK-NEXT: [[VEC_IND:%.*]] = phi <4 x i64> [ <i64 0, i64 1, i64 2, i64 3>, %[[VECTOR_PH]] ], [ [[VEC_IND_NEXT:%.*]], %[[BB43]] ]
; CHECK-NEXT: [[TMP1:%.*]] = getelementptr i64, ptr [[A]], i64 [[INDEX]]
; CHECK-NEXT: [[TMP2:%.*]] = icmp sle <4 x i64> [[VEC_IND]], zeroinitializer
; CHECK-NEXT: [[TMP3:%.*]] = xor <4 x i1> [[TMP2]], splat (i1 true)
+; CHECK-NEXT: br i1 [[U1]], label %[[BB32:.*]], label %[[BB21:.*]]
+; CHECK: [[BB21]]:
; CHECK-NEXT: [[TMP4:%.*]] = select <4 x i1> [[TMP3]], <4 x i1> [[TMP0]], <4 x i1> zeroinitializer
; CHECK-NEXT: [[TMP5:%.*]] = add <4 x i64> [[VEC_IND]], splat (i64 2)
; CHECK-NEXT: call void @llvm.masked.store.v4i64.p0(<4 x i64> [[TMP5]], ptr align 4 [[TMP1]], <4 x i1> [[TMP4]])
+; CHECK-NEXT: br label %[[BB43]]
+; CHECK: [[BB32]]:
; CHECK-NEXT: [[TMP6:%.*]] = select <4 x i1> [[TMP3]], <4 x i1> [[BROADCAST_SPLAT]], <4 x i1> zeroinitializer
; CHECK-NEXT: [[TMP7:%.*]] = add <4 x i64> [[VEC_IND]], splat (i64 3)
; CHECK-NEXT: call void @llvm.masked.store.v4i64.p0(<4 x i64> [[TMP7]], ptr align 4 [[TMP1]], <4 x i1> [[TMP6]])
+; CHECK-NEXT: br label %[[BB43]]
+; CHECK: [[BB43]]:
; CHECK-NEXT: [[TMP8:%.*]] = add <4 x i64> [[VEC_IND]], splat (i64 5)
; CHECK-NEXT: call void @llvm.masked.store.v4i64.p0(<4 x i64> [[TMP8]], ptr align 4 [[TMP1]], <4 x i1> [[TMP2]])
; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 4
@@ -999,19 +1023,25 @@ define void @uniform_branch_after_varying_branch_more_blocks_mirrored_no_phi(ptr
; CHECK-NEXT: [[TMP0:%.*]] = xor <4 x i1> [[BROADCAST_SPLAT]], splat (i1 true)
; CHECK-NEXT: br label %[[VECTOR_BODY:.*]]
; CHECK: [[VECTOR_BODY]]:
-; CHECK-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
-; CHECK-NEXT: [[VEC_IND:%.*]] = phi <4 x i64> [ <i64 0, i64 1, i64 2, i64 3>, %[[VECTOR_PH]] ], [ [[VEC_IND_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[BB53:.*]] ]
+; CHECK-NEXT: [[VEC_IND:%.*]] = phi <4 x i64> [ <i64 0, i64 1, i64 2, i64 3>, %[[VECTOR_PH]] ], [ [[VEC_IND_NEXT:%.*]], %[[BB53]] ]
; CHECK-NEXT: [[TMP1:%.*]] = getelementptr i64, ptr [[A]], i64 [[INDEX]]
; CHECK-NEXT: [[TMP2:%.*]] = icmp sle <4 x i64> [[VEC_IND]], zeroinitializer
; CHECK-NEXT: [[TMP3:%.*]] = xor <4 x i1> [[TMP2]], splat (i1 true)
; CHECK-NEXT: [[TMP4:%.*]] = add <4 x i64> [[VEC_IND]], splat (i64 1)
; CHECK-NEXT: call void @llvm.masked.store.v4i64.p0(<4 x i64> [[TMP4]], ptr align 4 [[TMP1]], <4 x i1> [[TMP3]])
+; CHECK-NEXT: br i1 [[U2]], label %[[BB42:.*]], label %[[BB31:.*]]
+; CHECK: [[BB31]]:
; CHECK-NEXT: [[TMP5:%.*]] = select <4 x i1> [[TMP2]], <4 x i1> [[TMP0]], <4 x i1> zeroinitializer
; CHECK-NEXT: [[TMP6:%.*]] = add <4 x i64> [[VEC_IND]], splat (i64 3)
; CHECK-NEXT: call void @llvm.masked.store.v4i64.p0(<4 x i64> [[TMP6]], ptr align 4 [[TMP1]], <4 x i1> [[TMP5]])
+; CHECK-NEXT: br label %[[BB53]]
+; CHECK: [[BB42]]:
; CHECK-NEXT: [[TMP7:%.*]] = select <4 x i1> [[TMP2]], <4 x i1> [[BROADCAST_SPLAT]], <4 x i1> zeroinitializer
; CHECK-NEXT: [[TMP8:%.*]] = add <4 x i64> [[VEC_IND]], splat (i64 4)
; CHECK-NEXT: call void @llvm.masked.store.v4i64.p0(<4 x i64> [[TMP8]], ptr align 4 [[TMP1]], <4 x i1> [[TMP7]])
+; CHECK-NEXT: br label %[[BB53]]
+; CHECK: [[BB53]]:
; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 4
; CHECK-NEXT: [[VEC_IND_NEXT]] = add nuw nsw <4 x i64> [[VEC_IND]], splat (i64 4)
; CHECK-NEXT: [[TMP9:%.*]] = icmp eq i64 [[INDEX_NEXT]], 128
@@ -2049,17 +2079,23 @@ define void @uniform_branch_shared_join_with_varying_no_phi(ptr %a, i1 %u0) {
; CHECK-NEXT: [[TMP0:%.*]] = xor <4 x i1> [[BROADCAST_SPLAT]], splat (i1 true)
; CHECK-NEXT: br label %[[VECTOR_BODY:.*]]
; CHECK: [[VECTOR_BODY]]:
-; CHECK-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
-; CHECK-NEXT: [[VEC_IND:%.*]] = phi <4 x i64> [ <i64 0, i64 1, i64 2, i64 3>, %[[VECTOR_PH]] ], [ [[VEC_IND_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[BB43:.*]] ]
+; CHECK-NEXT: [[VEC_IND:%.*]] = phi <4 x i64> [ <i64 0, i64 1, i64 2, i64 3>, %[[VECTOR_PH]] ], [ [[VEC_IND_NEXT:%.*]], %[[BB43]] ]
; CHECK-NEXT: [[TMP1:%.*]] = getelementptr i64, ptr [[A]], i64 [[INDEX]]
+; CHECK-NEXT: br i1 [[U0]], label %[[BB22:.*]], label %[[BB11:.*]]
+; CHECK: [[BB11]]:
; CHECK-NEXT: [[TMP2:%.*]] = add <4 x i64> [[VEC_IND]], splat (i64 1)
; CHECK-NEXT: call void @llvm.masked.store.v4i64.p0(<4 x i64> [[TMP2]], ptr align 4 [[TMP1]], <4 x i1> [[TMP0]])
+; CHECK-NEXT: br label %[[BB43]]
+; CHECK: [[BB22]]:
; CHECK-NEXT: [[TMP3:%.*]] = add <4 x i64> [[VEC_IND]], splat (i64 2)
; CHECK-NEXT: call void @llvm.masked.store.v4i64.p0(<4 x i64> [[TMP3]], ptr align 4 [[TMP1]], <4 x i1> [[BROADCAST_SPLAT]])
; CHECK-NEXT: [[TMP4:%.*]] = icmp sle <4 x i64> [[VEC_IND]], splat (i64 2)
; CHECK-NEXT: [[TMP5:%.*]] = select <4 x i1> [[BROADCAST_SPLAT]], <4 x i1> [[TMP4]], <4 x i1> zeroinitializer
; CHECK-NEXT: [[TMP6:%.*]] = add <4 x i64> [[VEC_IND]], splat (i64 3)
; CHECK-NEXT: call void @llvm.masked.store.v4i64.p0(<4 x i64> [[TMP6]], ptr align 4 [[TMP1]], <4 x i1> [[TMP5]])
+; CHECK-NEXT: br label %[[BB43]]
+; CHECK: [[BB43]]:
; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 4
; CHECK-NEXT: [[VEC_IND_NEXT]] = add nuw nsw <4 x i64> [[VEC_IND]], splat (i64 4)
; CHECK-NEXT: [[TMP7:%.*]] = icmp eq i64 [[INDEX_NEXT]], 128
diff --git a/llvm/test/Transforms/LoopVectorize/uniform-blend.ll b/llvm/test/Transforms/LoopVectorize/uniform-blend.ll
index 8d86c8ca2bd9d..92f714dcf3a20 100644
--- a/llvm/test/Transforms/LoopVectorize/uniform-blend.ll
+++ b/llvm/test/Transforms/LoopVectorize/uniform-blend.ll
@@ -11,8 +11,12 @@ define void @blend_uniform_iv_trunc(i1 %c) {
; CHECK: [[VECTOR_PH]]:
; CHECK-NEXT: br label %[[VECTOR_BODY:.*]]
; CHECK: [[VECTOR_BODY]]:
-; CHECK-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[LOOP_LATCH2:.*]] ]
; CHECK-NEXT: [[TMP0:%.*]] = trunc i64 [[INDEX]] to i16
+; CHECK-NEXT: br i1 [[C]], label %[[LOOP_NEXT1:.*]], label %[[LOOP_LATCH2]]
+; CHECK: [[LOOP_NEXT1]]:
+; CHECK-NEXT: br label %[[LOOP_LATCH2]]
+; CHECK: [[LOOP_LATCH2]]:
; CHECK-NEXT: [[TMP7:%.*]] = getelementptr inbounds [32 x i16], ptr @dst, i16 0, i16 [[TMP0]]
; CHECK-NEXT: store <4 x i16> zeroinitializer, ptr [[TMP7]], align 2
; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 4
@@ -54,7 +58,11 @@ define void @blend_uniform_iv(i1 %c) {
; CHECK: [[VECTOR_PH]]:
; CHECK-NEXT: br label %[[VECTOR_BODY:.*]]
; CHECK: [[VECTOR_BODY]]:
-; CHECK-NEXT: [[TMP6:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-NEXT: [[TMP6:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[LOOP_LATCH2:.*]] ]
+; CHECK-NEXT: br i1 [[C]], label %[[LOOP_NEXT1:.*]], label %[[LOOP_LATCH2]]
+; CHECK: [[LOOP_NEXT1]]:
+; CHECK-NEXT: br label %[[LOOP_LATCH2]]
+; CHECK: [[LOOP_LATCH2]]:
; CHECK-NEXT: [[TMP7:%.*]] = getelementptr inbounds [32 x i16], ptr @dst, i16 0, i64 [[TMP6]]
; CHECK-NEXT: store <4 x i16> zeroinitializer, ptr [[TMP7]], align 2
; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[TMP6]], 4
@@ -96,10 +104,14 @@ define void @blend_chain_iv(i1 %c) {
; CHECK: [[VECTOR_PH]]:
; CHECK-NEXT: br label %[[VECTOR_BODY:.*]]
; CHECK: [[VECTOR_BODY]]:
-; CHECK-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[LOOP_NEXT_32:.*]] ]
; CHECK-NEXT: [[TMP3:%.*]] = add i64 [[INDEX]], 1
; CHECK-NEXT: [[TMP5:%.*]] = add i64 [[INDEX]], 2
; CHECK-NEXT: [[TMP7:%.*]] = add i64 [[INDEX]], 3
+; CHECK-NEXT: br i1 [[C]], label %[[LOOP_NEXT_21:.*]], label %[[LOOP_NEXT_32]]
+; CHECK: [[LOOP_NEXT_21]]:
+; CHECK-NEXT: br label %[[LOOP_NEXT_32]]
+; CHECK: [[LOOP_NEXT_32]]:
; CHECK-NEXT: [[TMP2:%.*]] = getelementptr inbounds [32 x i16], ptr @dst, i16 0, i64 [[INDEX]]
; CHECK-NEXT: [[TMP4:%.*]] = getelementptr inbounds [32 x i16], ptr @dst, i16 0, i64 [[TMP3]]
; CHECK-NEXT: [[TMP6:%.*]] = getelementptr inbounds [32 x i16], ptr @dst, i16 0, i64 [[TMP5]]
@@ -250,8 +262,12 @@ define void @blend_poison(ptr %p, i1 %c) {
; CHECK: [[VECTOR_PH]]:
; CHECK-NEXT: br label %[[VECTOR_BODY:.*]]
; CHECK: [[VECTOR_BODY]]:
-; CHECK-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
-; CHECK-NEXT: [[VEC_IND:%.*]] = phi <4 x i64> [ <i64 0, i64 1, i64 2, i64 3>, %[[VECTOR_PH]] ], [ [[VEC_IND_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[LATCH2:.*]] ]
+; CHECK-NEXT: [[VEC_IND:%.*]] = phi <4 x i64> [ <i64 0, i64 1, i64 2, i64 3>, %[[VECTOR_PH]] ], [ [[VEC_IND_NEXT:%.*]], %[[LATCH2]] ]
+; CHECK-NEXT: br i1 [[C]], label %[[IF1:.*]], label %[[LATCH2]]
+; CHECK: [[IF1]]:
+; CHECK-NEXT: br label %[[LATCH2]]
+; CHECK: [[LATCH2]]:
; CHECK-NEXT: [[TMP0:%.*]] = getelementptr i64, ptr [[P]], i64 [[INDEX]]
; CHECK-NEXT: store <4 x i64> [[VEC_IND]], ptr [[TMP0]], align 4
; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 4
@@ -293,8 +309,12 @@ define void @blend_poison_first_argument(ptr %p, i1 %c) {
; CHECK: [[VECTOR_PH]]:
; CHECK-NEXT: br label %[[VECTOR_BODY:.*]]
; CHECK: [[VECTOR_BODY]]:
-; CHECK-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
-; CHECK-NEXT: [[VEC_IND:%.*]] = phi <4 x i64> [ <i64 0, i64 1, i64 2, i64 3>, %[[VECTOR_PH]] ], [ [[VEC_IND_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[LATCH2:.*]] ]
+; CHECK-NEXT: [[VEC_IND:%.*]] = phi <4 x i64> [ <i64 0, i64 1, i64 2, i64 3>, %[[VECTOR_PH]] ], [ [[VEC_IND_NEXT:%.*]], %[[LATCH2]] ]
+; CHECK-NEXT: br i1 [[C]], label %[[IF1:.*]], label %[[LATCH2]]
+; CHECK: [[IF1]]:
+; CHECK-NEXT: br label %[[LATCH2]]
+; CHECK: [[LATCH2]]:
; CHECK-NEXT: [[TMP0:%.*]] = getelementptr i64, ptr [[P]], i64 [[INDEX]]
; CHECK-NEXT: store <4 x i64> [[VEC_IND]], ptr [[TMP0]], align 4
; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 4
@@ -336,7 +356,11 @@ define void @blend_all_poison(ptr %p, i1 %c) {
; CHECK: [[VECTOR_PH]]:
; CHECK-NEXT: br label %[[VECTOR_BODY:.*]]
; CHECK: [[VECTOR_BODY]]:
-; CHECK-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[LATCH2:.*]] ]
+; CHECK-NEXT: br i1 [[C]], label %[[IF1:.*]], label %[[LATCH2]]
+; CHECK: [[IF1]]:
+; CHECK-NEXT: br label %[[LATCH2]]
+; CHECK: [[LATCH2]]:
; CHECK-NEXT: [[TMP0:%.*]] = getelementptr i64, ptr [[P]], i64 [[INDEX]]
; CHECK-NEXT: store <4 x i64> poison, ptr [[TMP0]], align 4
; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 4
diff --git a/llvm/test/Transforms/LoopVectorize/unused-blend-mask-for-first-operand.ll b/llvm/test/Transforms/LoopVectorize/unused-blend-mask-for-first-operand.ll
index feab77583a594..c4916058759c2 100644
--- a/llvm/test/Transforms/LoopVectorize/unused-blend-mask-for-first-operand.ll
+++ b/llvm/test/Transforms/LoopVectorize/unused-blend-mask-for-first-operand.ll
@@ -12,9 +12,11 @@ define void @test_not_first_lane_only_constant(ptr %A, ptr noalias %B, ptr noali
; CHECK: vector.ph:
; CHECK-NEXT: br label [[VECTOR_BODY:%.*]]
; CHECK: vector.body:
-; CHECK-NEXT: [[INDEX:%.*]] = phi i32 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ]
+; CHECK-NEXT: [[INDEX:%.*]] = phi i32 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[ELSE_21:%.*]] ]
; CHECK-NEXT: [[OFFSET_IDX:%.*]] = trunc i32 [[INDEX]] to i16
; CHECK-NEXT: [[TMP1:%.*]] = getelementptr inbounds i16, ptr [[A]], i16 [[OFFSET_IDX]]
+; CHECK-NEXT: br label [[ELSE_21]]
+; CHECK: else.21:
; CHECK-NEXT: [[TMP13:%.*]] = load i16, ptr [[B]], align 2
; CHECK-NEXT: [[BROADCAST_SPLATINSERT5:%.*]] = insertelement <4 x i16> poison, i16 [[TMP13]], i64 0
; CHECK-NEXT: [[BROADCAST_SPLAT6:%.*]] = shufflevector <4 x i16> [[BROADCAST_SPLATINSERT5]], <4 x i16> poison, <4 x i32> zeroinitializer
diff --git a/llvm/test/Transforms/PhaseOrdering/loop-vectorize-bfi.ll b/llvm/test/Transforms/PhaseOrdering/loop-vectorize-bfi.ll
index baf7bf38f3351..754058e6945fc 100644
--- a/llvm/test/Transforms/PhaseOrdering/loop-vectorize-bfi.ll
+++ b/llvm/test/Transforms/PhaseOrdering/loop-vectorize-bfi.ll
@@ -16,12 +16,16 @@ define void @f(i1 %x) !prof !0 {
; CHECK-NEXT: [[TMP2:%.*]] = shufflevector <vscale x 2 x i1> [[TMP1]], <vscale x 2 x i1> poison, <vscale x 2 x i32> zeroinitializer
; CHECK-NEXT: br label %[[VECTOR_BODY:.*]]
; CHECK: [[VECTOR_BODY]]:
-; CHECK-NEXT: [[EVL_BASED_IV:%.*]] = phi i64 [ 0, %[[ENTRY]] ], [ [[INDEX_EVL_NEXT:%.*]], %[[VECTOR_BODY]] ]
-; CHECK-NEXT: [[AVL:%.*]] = phi i64 [ 65, %[[ENTRY]] ], [ [[AVL_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-NEXT: [[EVL_BASED_IV:%.*]] = phi i64 [ 0, %[[ENTRY]] ], [ [[INDEX_EVL_NEXT:%.*]], %[[BAZ_14:.*]] ]
+; CHECK-NEXT: [[AVL:%.*]] = phi i64 [ 65, %[[ENTRY]] ], [ [[AVL_NEXT:%.*]], %[[BAZ_14]] ]
; CHECK-NEXT: [[TMP3:%.*]] = call i32 @llvm.experimental.get.vector.length.i64(i64 [[AVL]], i32 2, i1 true)
+; CHECK-NEXT: br i1 [[X]], label %[[BAZ_14]], label %[[BAR1:.*]]
+; CHECK: [[BAR1]]:
; CHECK-NEXT: [[TMP4:%.*]] = getelementptr [8 x i8], ptr null, i64 [[EVL_BASED_IV]]
; CHECK-NEXT: call void @llvm.vp.store.nxv2i64.p0(<vscale x 2 x i64> poison, ptr align 8 [[TMP4]], <vscale x 2 x i1> [[TMP2]], i32 [[TMP3]])
; CHECK-NEXT: call void @llvm.vp.store.nxv2i64.p0(<vscale x 2 x i64> poison, ptr align 8 [[TMP4]], <vscale x 2 x i1> [[TMP2]], i32 [[TMP3]])
+; CHECK-NEXT: br label %[[BAZ_14]]
+; CHECK: [[BAZ_14]]:
; CHECK-NEXT: [[TMP5:%.*]] = zext nneg i32 [[TMP3]] to i64
; CHECK-NEXT: [[INDEX_EVL_NEXT]] = add nuw i64 [[EVL_BASED_IV]], [[TMP5]]
; CHECK-NEXT: [[AVL_NEXT]] = sub nuw nsw i64 [[AVL]], [[TMP5]]
More information about the llvm-branch-commits
mailing list