[llvm-branch-commits] [llvm] [VPlan] Fix SSA after partial linearization, enable it for unstructured branches (PR #219316)
Andrei Elovikov via llvm-branch-commits
llvm-branch-commits at lists.llvm.org
Fri Aug 28 16:52:20 PDT 2026
https://github.com/eas updated https://github.com/llvm/llvm-project/pull/219316
>From 614af89119915abdca86e3dbb7c1346684c67356 Mon Sep 17 00:00:00 2001
From: Andrei Elovikov <andrei.elovikov at sifive.com>
Date: Wed, 26 Aug 2026 12:44:36 -0700
Subject: [PATCH 01/11] Luke's reconstructSSA (#212209)
---
llvm/lib/Transforms/Vectorize/VPlanUtils.cpp | 37 +++
llvm/lib/Transforms/Vectorize/VPlanUtils.h | 9 +
.../Transforms/Vectorize/VPlanTest.cpp | 268 ++++++++++++++++++
3 files changed, 314 insertions(+)
diff --git a/llvm/lib/Transforms/Vectorize/VPlanUtils.cpp b/llvm/lib/Transforms/Vectorize/VPlanUtils.cpp
index b2a91b701fcaa..b99940b0a15fe 100644
--- a/llvm/lib/Transforms/Vectorize/VPlanUtils.cpp
+++ b/llvm/lib/Transforms/Vectorize/VPlanUtils.cpp
@@ -1217,3 +1217,40 @@ void vputils::detail::pullOutPermutationsImpl(
}
}
}
+
+// Implements the algorithm described in "Simple and Efficient Construction of
+// Static Single Assignment Form" by Braun et al.
+VPValue *vputils::reconstructSSA(VPBasicBlock *VPBB,
+ DenseMap<VPBasicBlock *, VPValue *> &Defs) {
+ assert(!Defs.empty() && "Defs shouldn't be empty");
+ assert(VPBB->getPlan() && "VPBB isn't reachable from entry");
+ if (VPValue *Def = Defs.lookup(VPBB))
+ return Def;
+ // If the entry block is reached and there's still no def, then Defs is
+ // missing a definition that covers this path.
+ assert(VPBB->getNumPredecessors() && "Not all paths have def");
+
+ if (VPBlockBase *Pred = VPBB->getSinglePredecessor())
+ return reconstructSSA(cast<VPBasicBlock>(Pred), Defs);
+
+ // Multiple predecessors, create a join.
+ Type *Ty = Defs.begin()->second->getScalarType();
+ auto *Phi = new VPPhi({}, VPIRFlags::getDefaultFlags(Instruction::PHI, Ty),
+ DebugLoc::getUnknown(), "", Ty);
+ VPBB->insert(Phi, VPBB->getFirstNonPhi());
+ Defs[VPBB] = Phi;
+ for (auto *Pred : VPBB->predecessors())
+ Phi->addIncoming(reconstructSSA(cast<VPBasicBlock>(Pred), Defs));
+
+ // Fold away trivial phis.
+ // TODO: Remove phi users which have become trivial too.
+ if (all_equal(Phi->incoming_values())) {
+ VPValue *Common = Phi->getIncomingValue(0);
+ Phi->replaceAllUsesWith(Common);
+ Phi->eraseFromParent();
+ Defs[VPBB] = Common;
+ return Common;
+ }
+
+ return Phi;
+}
diff --git a/llvm/lib/Transforms/Vectorize/VPlanUtils.h b/llvm/lib/Transforms/Vectorize/VPlanUtils.h
index f53fd2cb18a84..c71bef8a40ef8 100644
--- a/llvm/lib/Transforms/Vectorize/VPlanUtils.h
+++ b/llvm/lib/Transforms/Vectorize/VPlanUtils.h
@@ -221,6 +221,15 @@ SmallVector<VPUser *> collectUsersRecursively(VPValue *V);
VPIRValue *tryToFoldLiveIns(VPSingleDefRecipe &R, ArrayRef<VPValue *> Operands,
const DataLayout &DL);
+/// Insert phis to reconstruct SSA for a single value starting from \p VPBB. \p
+/// Defs is a map of definitions at specific blocks. Returns the
+/// reconstructed value at VPBB. Use if the CFG has been modified such that a
+/// def no longer dominates all its uses. Every block leading to VPBB must be
+/// reachable from the entry and the plan must be plain-CFG (not contain any
+/// regions).
+LLVM_ABI_FOR_TEST VPValue *
+reconstructSSA(VPBasicBlock *VPBB, DenseMap<VPBasicBlock *, VPValue *> &Defs);
+
namespace detail {
/// Template-independent implementation for pullOutPermutations.
diff --git a/llvm/unittests/Transforms/Vectorize/VPlanTest.cpp b/llvm/unittests/Transforms/Vectorize/VPlanTest.cpp
index 1f93adabb6f47..f046b4e9cc842 100644
--- a/llvm/unittests/Transforms/Vectorize/VPlanTest.cpp
+++ b/llvm/unittests/Transforms/Vectorize/VPlanTest.cpp
@@ -1900,6 +1900,274 @@ TEST_F(VPUtilsTest, IsUniformAcrossVFsAndUFsForSingleScalarOpcodes) {
EXPECT_FALSE(vputils::isUniformAcrossVFsAndUFs(FirstActiveLaneNonUniform));
}
+TEST_F(VPUtilsTest, ReconstructSSA) {
+ VPlan &Plan = getPlan();
+ VPBasicBlock *VPBB1 = Plan.getEntry();
+ VPBasicBlock *VPBB2 = Plan.createVPBasicBlock("");
+ VPBasicBlock *VPBB3 = Plan.createVPBasicBlock("");
+ VPBasicBlock *VPBB4 = Plan.createVPBasicBlock("");
+
+ // VPBB1
+ // / \
+ // VPBB2 VPBB3
+ // \ /
+ // VPBB4
+ VPBlockUtils::connectBlocks(VPBB1, VPBB2);
+ VPBlockUtils::connectBlocks(VPBB1, VPBB3);
+ VPBlockUtils::connectBlocks(VPBB2, VPBB4);
+ VPBlockUtils::connectBlocks(VPBB3, VPBB4);
+
+ VPValue *C = Plan.getConstantInt(32, 1);
+ VPIRFlags AddFlags = VPIRFlags::getDefaultFlags(Instruction::Add);
+ auto *Def1 = new VPInstruction(Instruction::Add, {C, C}, AddFlags);
+ VPBB1->appendRecipe(Def1);
+ auto *Def2 = new VPInstruction(Instruction::Add, {C, C}, AddFlags);
+ VPBB2->appendRecipe(Def2);
+
+ DenseMap<VPBasicBlock *, VPValue *> Defs = {{VPBB1, Def1}, {VPBB2, Def2}};
+ auto *Res = cast<VPPhi>(vputils::reconstructSSA(VPBB4, Defs));
+ EXPECT_EQ(Res->getIncomingValueForBlock(VPBB2), Def2);
+ EXPECT_EQ(Res->getIncomingValueForBlock(VPBB3), Def1);
+}
+
+TEST_F(VPUtilsTest, ReconstructSSAPoisonExample) {
+ // Test that the resulting phi isn't affected by the parent block of any
+ // definition. The resulting value in VPBB4 should be Def1 on any path that
+ // goes through VPPB2, and poison otherwise.
+ VPlan &Plan = getPlan();
+ VPBasicBlock *VPBB1 = Plan.getEntry();
+ VPBasicBlock *VPBB2 = Plan.createVPBasicBlock("");
+ VPBasicBlock *VPBB3 = Plan.createVPBasicBlock("");
+ VPBasicBlock *VPBB4 = Plan.createVPBasicBlock("");
+
+ // VPBB1
+ // / \
+ // VPBB2 VPBB3
+ // \ /
+ // VPBB4
+ VPBlockUtils::connectBlocks(VPBB1, VPBB2);
+ VPBlockUtils::connectBlocks(VPBB1, VPBB3);
+ VPBlockUtils::connectBlocks(VPBB2, VPBB4);
+ VPBlockUtils::connectBlocks(VPBB3, VPBB4);
+
+ VPValue *C = Plan.getConstantInt(32, 1);
+ VPValue *Poison = Plan.getPoison(C->getScalarType());
+ VPIRFlags AddFlags = VPIRFlags::getDefaultFlags(Instruction::Add);
+ auto *Def1 = new VPInstruction(Instruction::Add, {C, C}, AddFlags);
+ VPBB1->appendRecipe(Def1);
+
+ DenseMap<VPBasicBlock *, VPValue *> Defs = {{VPBB1, Poison}, {VPBB2, Def1}};
+ auto *Res = cast<VPPhi>(vputils::reconstructSSA(VPBB4, Defs));
+ EXPECT_EQ(Res->getIncomingValueForBlock(VPBB2), Def1);
+ EXPECT_EQ(Res->getIncomingValueForBlock(VPBB3), Poison);
+}
+
+TEST_F(VPUtilsTest, ReconstructSSAMultiplePhis) {
+ VPlan &Plan = getPlan();
+ VPBasicBlock *VPBB1 = Plan.getEntry();
+ VPBasicBlock *VPBB2 = Plan.createVPBasicBlock("");
+ VPBasicBlock *VPBB3 = Plan.createVPBasicBlock("");
+ VPBasicBlock *VPBB4 = Plan.createVPBasicBlock("");
+ VPBasicBlock *VPBB5 = Plan.createVPBasicBlock("");
+ VPBasicBlock *VPBB6 = Plan.createVPBasicBlock("");
+
+ // VPBB1
+ // / \
+ // VPBB2 VPBB3
+ // \ / \
+ // VPBB4 VPBB5
+ // \ /
+ // VPBB6
+ VPBlockUtils::connectBlocks(VPBB1, VPBB2);
+ VPBlockUtils::connectBlocks(VPBB1, VPBB3);
+ VPBlockUtils::connectBlocks(VPBB2, VPBB4);
+ VPBlockUtils::connectBlocks(VPBB3, VPBB4);
+ VPBlockUtils::connectBlocks(VPBB3, VPBB5);
+ VPBlockUtils::connectBlocks(VPBB4, VPBB6);
+ VPBlockUtils::connectBlocks(VPBB5, VPBB6);
+
+ VPValue *C = Plan.getConstantInt(32, 1);
+ VPIRFlags AddFlags = VPIRFlags::getDefaultFlags(Instruction::Add);
+ auto *Def2 = new VPInstruction(Instruction::Add, {C, C}, AddFlags);
+ VPBB2->appendRecipe(Def2);
+ auto *Def3 = new VPInstruction(Instruction::Add, {C, C}, AddFlags);
+ VPBB3->appendRecipe(Def3);
+
+ DenseMap<VPBasicBlock *, VPValue *> Defs = {{VPBB2, Def2}, {VPBB3, Def3}};
+ auto *Phi6 = cast<VPPhi>(vputils::reconstructSSA(VPBB6, Defs));
+ EXPECT_EQ(Phi6->getIncomingValueForBlock(VPBB5), Def3);
+ EXPECT_TRUE(isa<VPPhi>(Phi6->getIncomingValueForBlock(VPBB4)));
+
+ auto *Phi4 = cast<VPPhi>(Phi6->getIncomingValueForBlock(VPBB4));
+ EXPECT_EQ(Phi4->getIncomingValueForBlock(VPBB2), Def2);
+ EXPECT_EQ(Phi4->getIncomingValueForBlock(VPBB3), Def3);
+}
+
+TEST_F(VPUtilsTest, ReconstructSSAFold) {
+ VPlan &Plan = getPlan();
+ VPBasicBlock *VPBB1 = Plan.getEntry();
+ VPBasicBlock *VPBB2 = Plan.createVPBasicBlock("");
+ VPBasicBlock *VPBB3 = Plan.createVPBasicBlock("");
+ VPBasicBlock *VPBB4 = Plan.createVPBasicBlock("");
+
+ // VPBB1
+ // / \
+ // VPBB2 VPBB3
+ // \ /
+ // VPBB4
+ VPBlockUtils::connectBlocks(VPBB1, VPBB2);
+ VPBlockUtils::connectBlocks(VPBB1, VPBB3);
+ VPBlockUtils::connectBlocks(VPBB2, VPBB4);
+ VPBlockUtils::connectBlocks(VPBB3, VPBB4);
+
+ VPValue *C = Plan.getConstantInt(32, 1);
+ VPIRFlags AddFlags = VPIRFlags::getDefaultFlags(Instruction::Add);
+ auto *Def = new VPInstruction(Instruction::Add, {C, C}, AddFlags);
+ VPBB1->appendRecipe(Def);
+
+ // Check that phis with all equal incoming values are folded away.
+ DenseMap<VPBasicBlock *, VPValue *> Defs = {{VPBB2, Def}, {VPBB3, Def}};
+ EXPECT_EQ(vputils::reconstructSSA(VPBB4, Defs), Def);
+}
+
+TEST_F(VPUtilsTest, ReconstructSSACycle) {
+ VPlan &Plan = getPlan();
+ VPBasicBlock *VPBB1 = Plan.getEntry();
+ VPBasicBlock *VPBB2 = Plan.createVPBasicBlock("");
+ VPBasicBlock *VPBB3 = Plan.createVPBasicBlock("");
+ VPBasicBlock *VPBB4 = Plan.createVPBasicBlock("");
+
+ // VPBB1
+ // |
+ // VPBB2
+ // / | ^
+ // VPBB3 | |
+ // \ | /
+ // VPBB4
+ VPBlockUtils::connectBlocks(VPBB1, VPBB2);
+ VPBlockUtils::connectBlocks(VPBB2, VPBB3);
+ VPBlockUtils::connectBlocks(VPBB2, VPBB4);
+ VPBlockUtils::connectBlocks(VPBB3, VPBB4);
+ VPBlockUtils::connectBlocks(VPBB4, VPBB2);
+
+ VPValue *C = Plan.getConstantInt(32, 1);
+ VPIRFlags AddFlags = VPIRFlags::getDefaultFlags(Instruction::Add);
+ auto *Def1 = new VPInstruction(Instruction::Add, {C, C}, AddFlags);
+ VPBB1->appendRecipe(Def1);
+ auto *Def2 = new VPInstruction(Instruction::Add, {C, C}, AddFlags);
+ VPBB3->appendRecipe(Def2);
+
+ DenseMap<VPBasicBlock *, VPValue *> Defs = {{VPBB1, Def1}, {VPBB3, Def2}};
+ auto *Phi1 = cast<VPPhi>(vputils::reconstructSSA(VPBB4, Defs));
+ EXPECT_EQ(Phi1->getIncomingValueForBlock(VPBB3), Def2);
+ EXPECT_TRUE(isa<VPPhi>(Phi1->getIncomingValueForBlock(VPBB2)));
+
+ auto *Phi2 = cast<VPPhi>(Phi1->getIncomingValueForBlock(VPBB2));
+ EXPECT_EQ(Phi2->getIncomingValueForBlock(VPBB4), Phi1);
+ EXPECT_EQ(Phi2->getIncomingValueForBlock(VPBB1), Def1);
+}
+
+#if defined(GTEST_HAS_DEATH_TEST) && !defined(NDEBUG)
+TEST_F(VPUtilsTest, ReconstructSSAUnreachableCycle) {
+ VPlan &Plan = getPlan();
+ VPBasicBlock *VPBB1 = Plan.getEntry();
+ VPBasicBlock *VPBB2 = Plan.createVPBasicBlock("");
+
+ // VPBB1 VPBB2 <-+
+ // | |
+ // +-----+
+ VPBlockUtils::connectBlocks(VPBB2, VPBB2);
+
+ VPValue *C = Plan.getConstantInt(32, 1);
+ VPIRFlags AddFlags = VPIRFlags::getDefaultFlags(Instruction::Add);
+ auto *Def1 = new VPInstruction(Instruction::Add, {C, C}, AddFlags);
+ VPBB1->appendRecipe(Def1);
+
+ DenseMap<VPBasicBlock *, VPValue *> Defs = {{VPBB1, Def1}};
+ EXPECT_DEATH(vputils::reconstructSSA(VPBB2, Defs),
+ "VPlan without any entry node without predecessors");
+}
+
+TEST_F(VPUtilsTest, ReconstructSSAUnreachableCyclePredecessor) {
+ VPlan &Plan = getPlan();
+ VPBasicBlock *VPBB1 = Plan.getEntry();
+ VPBasicBlock *VPBB2 = Plan.createVPBasicBlock("");
+ VPBasicBlock *VPBB3 = Plan.createVPBasicBlock("");
+
+ // VPBB1 VPBB2 <-+
+ // \ / | |
+ // \ / +-----+
+ // VPBB3
+ VPBlockUtils::connectBlocks(VPBB2, VPBB2);
+ VPBlockUtils::connectBlocks(VPBB1, VPBB3);
+ VPBlockUtils::connectBlocks(VPBB2, VPBB3);
+
+ VPValue *C = Plan.getConstantInt(32, 1);
+ VPIRFlags AddFlags = VPIRFlags::getDefaultFlags(Instruction::Add);
+ auto *Def1 = new VPInstruction(Instruction::Add, {C, C}, AddFlags);
+ VPBB1->appendRecipe(Def1);
+
+ DenseMap<VPBasicBlock *, VPValue *> Defs = {{VPBB1, Def1}};
+ EXPECT_DEATH(vputils::reconstructSSA(VPBB3, Defs),
+ "VPlan without any entry node without predecessors");
+}
+#endif
+
+TEST_F(VPUtilsTest, ReconstructSSADuplicatePredecessor) {
+ VPlan &Plan = getPlan();
+ VPBasicBlock *VPBB1 = Plan.getEntry();
+ VPBasicBlock *VPBB2 = Plan.createVPBasicBlock("");
+ VPBasicBlock *VPBB3 = Plan.createVPBasicBlock("");
+
+ // VPBB1
+ // / \ \
+ // | | VPBB2
+ // \ / /
+ // VPBB3
+ VPBlockUtils::connectBlocks(VPBB1, VPBB3);
+ VPBlockUtils::connectBlocks(VPBB1, VPBB3);
+ VPBlockUtils::connectBlocks(VPBB1, VPBB2);
+ VPBlockUtils::connectBlocks(VPBB2, VPBB3);
+
+ VPValue *C = Plan.getConstantInt(32, 1);
+ VPIRFlags AddFlags = VPIRFlags::getDefaultFlags(Instruction::Add);
+ auto *Def1 = new VPInstruction(Instruction::Add, {C, C}, AddFlags);
+ VPBB1->appendRecipe(Def1);
+ auto *Def2 = new VPInstruction(Instruction::Add, {C, C}, AddFlags);
+ VPBB2->appendRecipe(Def2);
+
+ DenseMap<VPBasicBlock *, VPValue *> Defs = {{VPBB1, Def1}, {VPBB2, Def2}};
+ auto *Phi = cast<VPPhi>(vputils::reconstructSSA(VPBB3, Defs));
+ EXPECT_EQ(Phi->getIncomingValue(0), Def1);
+ EXPECT_EQ(Phi->getIncomingValue(1), Def1);
+ EXPECT_EQ(Phi->getIncomingValue(2), Def2);
+}
+
+TEST_F(VPUtilsTest, ReconstructSSADuplicatePredecessorAllEqual) {
+ VPlan &Plan = getPlan();
+ VPBasicBlock *VPBB1 = Plan.getEntry();
+ VPBasicBlock *VPBB2 = Plan.createVPBasicBlock("");
+ VPBasicBlock *VPBB3 = Plan.createVPBasicBlock("");
+
+ // VPBB1
+ // / \ \
+ // / \ \
+ // VPBB2 VPBB3
+ VPBlockUtils::connectBlocks(VPBB1, VPBB2);
+ VPBlockUtils::connectBlocks(VPBB1, VPBB3);
+ VPBlockUtils::connectBlocks(VPBB1, VPBB3);
+
+ VPValue *C = Plan.getConstantInt(32, 1);
+ VPIRFlags AddFlags = VPIRFlags::getDefaultFlags(Instruction::Add);
+ auto *Def1 = new VPInstruction(Instruction::Add, {C, C}, AddFlags);
+ VPBB1->appendRecipe(Def1);
+
+ DenseMap<VPBasicBlock *, VPValue *> Defs = {{VPBB1, Def1}};
+ EXPECT_EQ(vputils::reconstructSSA(VPBB3, Defs), Def1);
+ Defs = {{VPBB1, Def1}};
+ EXPECT_EQ(vputils::reconstructSSA(VPBB2, Defs), Def1);
+}
+
TEST_F(VPBasicBlockTest, VPRegionValueClonePropagatesMaterialized) {
VPlan &Plan = getPlan();
VPBasicBlock *Preheader = Plan.getEntry();
>From d94c9a06b57a62a3f58b01a887d63d45bdd1706e Mon Sep 17 00:00:00 2001
From: Andrei Elovikov <andrei.elovikov at sifive.com>
Date: Wed, 26 Aug 2026 16:16:32 -0700
Subject: [PATCH 02/11] [AI] Move convertPhisToBlends to post-linearization
---
.../Transforms/Vectorize/VPlanPredicator.cpp | 52 ++++++++++++++-----
1 file changed, 40 insertions(+), 12 deletions(-)
diff --git a/llvm/lib/Transforms/Vectorize/VPlanPredicator.cpp b/llvm/lib/Transforms/Vectorize/VPlanPredicator.cpp
index 3f1e42e52dc88..6f4a908b1c9f3 100644
--- a/llvm/lib/Transforms/Vectorize/VPlanPredicator.cpp
+++ b/llvm/lib/Transforms/Vectorize/VPlanPredicator.cpp
@@ -93,10 +93,16 @@ class VPPredicator {
DenseMap<std::pair<const VPBasicBlock *, const VPBasicBlock *>,
VPValue *>;
using BlockMaskCacheTy = DenseMap<const VPBasicBlock *, VPValue *>;
+ using BlendTermTy = std::pair<VPValue *, VPBasicBlock *>;
EdgeMaskCacheTy EdgeMaskCache;
BlockMaskCacheTy BlockMaskCache;
+ /// Pre-linearization blend terms, indexed by block and phi.
+ SmallVector</* BlockIdx -> */ DenseMap</* Phi -> Terms */ VPPhi *,
+ SmallVector<BlendTermTy>>>
+ BlendTerms;
+
/// Create an edge mask for every destination of cases and/or default.
void createSwitchEdgeMasks(const VPInstruction *SI);
@@ -142,8 +148,6 @@ class VPPredicator {
return VPBB->getFirstNonPhi();
}
- using BlendTermTy = std::pair<VPValue *, VPBasicBlock *>;
-
/// Return true if every path starting at \p Root reaches one of \p Blocks.
/// All blocks in \p Blocks are expected to be dominated by \p Root.
bool isJointlyPostDominated(
@@ -159,7 +163,8 @@ class VPPredicator {
VPPredicator(VPlan &Plan)
: Plan(Plan), VPDT(Plan), VPPDT(Plan),
BlocksInCompactRPOTOrder(
- Plan.getVectorLoopRegion()->getEntryBasicBlock(), VPDT) {}
+ Plan.getVectorLoopRegion()->getEntryBasicBlock(), VPDT),
+ BlendTerms(BlocksInCompactRPOTOrder.size()) {}
/// Returns the *entry* mask for \p VPBB.
VPValue *getBlockInMask(const VPBasicBlock *VPBB) const {
@@ -260,9 +265,16 @@ bool VPPredicator::shouldPreserveTerminator(VPBasicBlock *VPBB) {
return False("Bailing out due to successors not being independent");
}
- // FIXME: Only one of the two checks below should remain in the end.
- if (!IPostDom->phis().empty())
- return False("Phis at reconvergence");
+ // This is equivalent to checking for VPBlendRecipes after phi conversion:
+ // only non-trivial phis are converted to blends.
+ if (any_of(IPostDom->phis(), [](VPRecipeBase &R) {
+ auto *Phi = cast<VPPhi>(&R);
+ auto NotPoison =
+ make_filter_range(Phi->incoming_values(),
+ [](VPValue *V) { return !match(V, m_Poison()); });
+ return !all_equal(NotPoison);
+ }))
+ return False("Non-trivial phi at reconvergence");
if (any_of(*IPostDom, IsaPred<VPBlendRecipe>))
return False("Blends at reconvergence");
@@ -501,9 +513,11 @@ void VPPredicator::convertPhisToBlends(VPBasicBlock *VPBB) {
continue;
}
- auto Terms = computeBlendTerms(PhiR);
SmallVector<VPBasicBlock *> MaskBlocks;
- for (auto [_, ConstMaskBlock] : Terms)
+ auto &BlockBlendTerms = BlendTerms[BlocksInCompactRPOTOrder.getIndex(VPBB)];
+ auto Terms = BlockBlendTerms.find(PhiR);
+ assert(Terms != BlockBlendTerms.end() && "Missing cached blend terms");
+ for (auto [_, ConstMaskBlock] : Terms->second)
MaskBlocks.push_back(const_cast<VPBasicBlock *>(ConstMaskBlock));
// The in-mask of the common dominator is true on all paths from an
@@ -514,7 +528,7 @@ void VPPredicator::convertPhisToBlends(VPBasicBlock *VPBB) {
VPValue *CommonIncomingMask = getBlockInMask(CommonIncomingDom);
SmallVector<VPValue *, 2> OperandsWithMask;
- for (auto [V, ConstMaskBlock] : Terms) {
+ for (auto [V, ConstMaskBlock] : Terms->second) {
auto *MaskBlock = const_cast<VPBasicBlock *>(ConstMaskBlock);
VPValue *Mask = getBlockInMask(MaskBlock);
VPValue *RemainingMask = nullptr;
@@ -558,9 +572,19 @@ void VPPredicator::run() {
}
}
- for (VPBlockBase *VPBB : reverse(BlocksInCompactRPOTOrder))
- if (VPBB != Header)
- convertPhisToBlends(cast<VPBasicBlock>(VPBB));
+ // Cache blend terms before linearization. Computing them requires the
+ // original phi predecessor mappings and CFG successor relation, both of
+ // which are rewritten below.
+ for (VPBlockBase *VPB : reverse(BlocksInCompactRPOTOrder)) {
+ if (VPB == Header)
+ continue;
+ auto *VPBB = cast<VPBasicBlock>(VPB);
+ auto &Terms = BlendTerms[BlocksInCompactRPOTOrder.getIndex(VPBB)];
+ for (VPRecipeBase &R : VPBB->phis()) {
+ auto *Phi = cast<VPPhi>(&R);
+ Terms[Phi] = computeBlendTerms(Phi);
+ }
+ }
using DeferredSuccessorsTy = SmallPtrSet<VPBlockBase *, 4>;
DenseMap<VPBlockBase *, DeferredSuccessorsTy> DeferredMap;
@@ -663,6 +687,10 @@ void VPPredicator::run() {
}
}
}
+
+ for (VPBlockBase *VPBB : reverse(BlocksInCompactRPOTOrder))
+ if (VPBB != Header)
+ convertPhisToBlends(cast<VPBasicBlock>(VPBB));
}
void VPlanTransforms::introduceMasksAndLinearize(VPlan &Plan) {
>From 4af269ea688d95fb14936e2a859608f398c04e02 Mon Sep 17 00:00:00 2001
From: Andrei Elovikov <andrei.elovikov at sifive.com>
Date: Wed, 26 Aug 2026 16:21:05 -0700
Subject: [PATCH 03/11] WIP preserve move uniform cfg, BROKEN
Masks for blends originated from preserved CFG are `true` which is wrong
with the current algorithm, see blend_masks_triangle_phi in
predicator.ll
---
.../Transforms/Vectorize/LoopVectorize.cpp | 27 +-
llvm/lib/Transforms/Vectorize/VPlan.h | 24 +-
.../Transforms/Vectorize/VPlanPredicator.cpp | 99 ++++-
.../LoopVectorize/AArch64/blend-costs.ll | 33 +-
.../AArch64/invariant-replicate-region.ll | 39 +-
.../LoopVectorize/RISCV/dead-ops-cost.ll | 10 +-
.../Transforms/LoopVectorize/RISCV/divrem.ll | 80 ++--
.../RISCV/tail-folding-complex-mask.ll | 44 +-
.../VPlan/phi-with-fastflags-vplan.ll | 9 +-
.../LoopVectorize/VPlan/predicator.ll | 151 +++----
.../X86/cost-conditional-branches.ll | 375 +-----------------
.../X86/pr109581-unused-blend.ll | 46 +--
.../X86/predicated-instruction-cost.ll | 42 +-
.../X86/predicated-replicate-feeding-cast.ll | 74 +---
...licate-recipe-with-only-first-lane-used.ll | 10 +-
.../LoopVectorize/X86/x86-predication.ll | 200 +++++-----
.../LoopVectorize/blend-in-header.ll | 24 +-
.../constant-fold-commutative-and.ll | 22 +-
.../LoopVectorize/constantfolder.ll | 62 ++-
.../LoopVectorize/hoist-predicated-loads.ll | 45 ++-
.../LoopVectorize/if-pred-non-void.ll | 28 +-
.../LoopVectorize/if-pred-stores.ll | 30 +-
.../Transforms/LoopVectorize/induction.ll | 194 ++++-----
.../LoopVectorize/phi-with-fastflags.ll | 8 +-
llvm/test/Transforms/LoopVectorize/pr45525.ll | 15 +-
.../pr55167-fold-tail-live-out.ll | 28 +-
.../predicatedinst-loop-invariant.ll | 26 +-
.../Transforms/LoopVectorize/predicator.ll | 211 ++++++----
.../LoopVectorize/reduction-small-size.ll | 22 +-
.../LoopVectorize/scalarize-masked-call.ll | 20 +-
.../Transforms/LoopVectorize/uniform-blend.ll | 4 +
.../unused-blend-mask-for-first-operand.ll | 6 +-
32 files changed, 955 insertions(+), 1053 deletions(-)
diff --git a/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp b/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
index 496ee491c7059..ea57dc37c2a57 100644
--- a/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
+++ b/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
@@ -6944,10 +6944,33 @@ void LoopVectorizationPlanner::addReductionResultComputation(
CloneChain(cast<VPSingleDefRecipe>(Op));
NewOps.push_back(Substitutions.lookup_or(Op, Op));
}
+
+ DEBUG_WITH_TYPE("my", {
+ dbgs() << "Cloning " << *Old
+ << ", type: " << *Old->getScalarType() << "\n";
+ for (auto V : Old->operands()) {
+ dbgs() << " op " << *V->getScalarType() << " ";
+ V->dump();
+ }
+ dbgs() << "New ops:\n";
+ for (auto V : NewOps) {
+ dbgs() << " op " << *V->getScalarType() << " ";
+ V->dump();
+ }
+ });
VPSingleDefRecipe *New;
- if (auto *B = dyn_cast<VPBlendRecipe>(Old))
+ if (auto *B = dyn_cast<VPBlendRecipe>(Old)) {
+
+ DEBUG_WITH_TYPE("my", {
+ dbgs() << "It is a blend:\n";
+ for (unsigned i = 0; i < B->getNumIncomingValues(); ++i) {
+ auto V = B->getIncomingValue(i);
+ dbgs() << " op " << *V->getScalarType() << " ";
+ V->dump();
+ }
+ });
New = B->cloneWithOperands(NewOps);
- else if (auto *W = dyn_cast<VPWidenRecipe>(Old))
+ } else if (auto *W = dyn_cast<VPWidenRecipe>(Old))
New = W->cloneWithOperands(NewOps);
else if (auto *Rep = dyn_cast<VPReplicateRecipe>(Old))
New = Rep->cloneWithOperands(NewOps);
diff --git a/llvm/lib/Transforms/Vectorize/VPlan.h b/llvm/lib/Transforms/Vectorize/VPlan.h
index 7d2c2fa1bdd23..fb5281f656d70 100644
--- a/llvm/lib/Transforms/Vectorize/VPlan.h
+++ b/llvm/lib/Transforms/Vectorize/VPlan.h
@@ -2971,12 +2971,24 @@ class LLVM_ABI_FOR_TEST VPBlendRecipe : public VPRecipeWithIRFlags {
: VPRecipeWithIRFlags(VPRecipeBase::VPBlendSC, Operands,
Operands[0]->getScalarType(), Flags, DL) {
assert(Operands.size() >= 2 && "Expected at least two operands!");
- assert(all_of(seq<unsigned>(0, getNumIncomingValues()),
- [this](unsigned I) {
- return getIncomingValue(I)->getScalarType() ==
- getScalarType();
- }) &&
- "all incoming values must have the same type");
+ assert(
+ all_of(seq<unsigned>(0, getNumIncomingValues()),
+ [&, this](unsigned I) {
+ if (getIncomingValue(I)->getScalarType() != getScalarType()) {
+ auto V = getIncomingValue(I);
+ dbgs() << "MISMATCH: idx " << I << " " << *V->getScalarType()
+ << " != " << *getScalarType() << " for ";
+ V->dump();
+ dbgs() << "\nOperands:\n";
+ for (auto *V : Operands) {
+ dbgs() << " " << *V->getScalarType() << " ";
+ V->dump();
+ }
+ dbgs() << "\n";
+ }
+ return getIncomingValue(I)->getScalarType() == getScalarType();
+ }) &&
+ "all incoming values must have the same type");
assert(all_of(seq<unsigned>(isNormalized(), getNumIncomingValues()),
[this](unsigned I) {
return getMask(I)->getScalarType()->isIntegerTy(1);
diff --git a/llvm/lib/Transforms/Vectorize/VPlanPredicator.cpp b/llvm/lib/Transforms/Vectorize/VPlanPredicator.cpp
index 6f4a908b1c9f3..6667da573edb0 100644
--- a/llvm/lib/Transforms/Vectorize/VPlanPredicator.cpp
+++ b/llvm/lib/Transforms/Vectorize/VPlanPredicator.cpp
@@ -83,10 +83,15 @@ class VPPredicator {
/// Post-dominator tree for the VPlan.
VPPostDominatorTree VPPDT;
+ DenseMap<VPValue *, DenseMap<VPBasicBlock *, VPValue *>>
+ SSAReconstructionDefsMap;
+
// Scan the body of the loop in a topological order to visit each basic
// block after having visited its predecessor basic blocks.
CompactRPOT BlocksInCompactRPOTOrder;
+ bool DisablePartialLinearization = false;
+
/// When we if-convert we need to create edge masks. We have to cache values
/// so that we don't end up with exponential recursion/IR.
using EdgeMaskCacheTy =
@@ -119,6 +124,7 @@ class VPPredicator {
/// Create a logical-or, factoring out a common header mask if present.
VPValue *createMaskOr(VPValue *LHS, VPValue *RHS, DebugLoc DL);
+ void fixSSA(VPRecipeBase *U, VPValue *V, bool IsMask);
bool shouldPreserveTerminator(VPBasicBlock *VPBB);
/// Record \p Mask as the *entry* mask of \p VPBB, which is expected to not
@@ -164,7 +170,17 @@ class VPPredicator {
: Plan(Plan), VPDT(Plan), VPPDT(Plan),
BlocksInCompactRPOTOrder(
Plan.getVectorLoopRegion()->getEntryBasicBlock(), VPDT),
- BlendTerms(BlocksInCompactRPOTOrder.size()) {}
+ BlendTerms(BlocksInCompactRPOTOrder.size()) {
+ if (any_of(Plan.getVectorLoopRegion()->getEntryBasicBlock()->phis(),
+ IsaPred<VPReductionPHIRecipe>)) {
+ // TODO: `LoopVectorizationPlanner::addReductionResultComputation` needs
+ // fixes.
+ LLVM_DEBUG(
+ dbgs()
+ << "Partial linearization disabled due to reductions present\n");
+ DisablePartialLinearization = true;
+ }
+ }
/// Returns the *entry* mask for \p VPBB.
VPValue *getBlockInMask(const VPBasicBlock *VPBB) const {
@@ -219,6 +235,33 @@ VPValue *VPPredicator::createMaskOr(VPValue *LHS, VPValue *RHS, DebugLoc DL) {
HeaderMask, Builder.createOr(LHSRemainder, RHSRemainder, DL), DL);
}
+void VPPredicator::fixSSA(VPRecipeBase *U, VPValue *V, bool IsMask) {
+ auto *RecipeValue = dyn_cast<VPRecipeValue>(V);
+ if (!RecipeValue)
+ return;
+
+ auto &SSADefs = SSAReconstructionDefsMap[RecipeValue];
+ VPBasicBlock *DefBB = RecipeValue->getDefiningRecipe()->getParent();
+ SSADefs[DefBB] = RecipeValue;
+ VPBasicBlock *Header = Plan.getVectorLoopRegion()->getEntryBasicBlock();
+ if (DefBB != Header)
+ SSADefs[Header] =
+ IsMask ? Plan.getFalse() : Plan.getPoison(RecipeValue->getScalarType());
+ auto *Fixed = vputils::reconstructSSA(U->getParent(), SSADefs);
+ if (Fixed == RecipeValue)
+ return;
+
+ LLVM_DEBUG({
+ dbgs() << "SSA Fixup in ";
+ U->dump();
+ dbgs() << "\n replacing ";
+ V->dump();
+ dbgs() << " with ";
+ Fixed->dump();
+ });
+ U->replaceUsesOfWith(RecipeValue, Fixed);
+}
+
bool VPPredicator::shouldPreserveTerminator(VPBasicBlock *VPBB) {
LLVM_DEBUG(dbgs() << "Checking if can preserve the branch at the end of "
<< VPBB->getName() << "\n");
@@ -227,6 +270,9 @@ bool VPPredicator::shouldPreserveTerminator(VPBasicBlock *VPBB) {
<< (Reason.empty() ? Twine() : (": " + Reason)) << "\n");
return false;
};
+ if (DisablePartialLinearization)
+ return False("Disabled");
+
if (VPBB->getNumSuccessors() != 2)
return False("#Successors != 2");
auto *Term = dyn_cast<VPInstruction>(VPBB->getTerminator());
@@ -265,20 +311,6 @@ bool VPPredicator::shouldPreserveTerminator(VPBasicBlock *VPBB) {
return False("Bailing out due to successors not being independent");
}
- // This is equivalent to checking for VPBlendRecipes after phi conversion:
- // only non-trivial phis are converted to blends.
- if (any_of(IPostDom->phis(), [](VPRecipeBase &R) {
- auto *Phi = cast<VPPhi>(&R);
- auto NotPoison =
- make_filter_range(Phi->incoming_values(),
- [](VPValue *V) { return !match(V, m_Poison()); });
- return !all_equal(NotPoison);
- }))
- return False("Non-trivial phi at reconvergence");
-
- if (any_of(*IPostDom, IsaPred<VPBlendRecipe>))
- return False("Blends at reconvergence");
-
LLVM_DEBUG(dbgs() << "...yes, can be preserved.\n");
return true;
}
@@ -688,9 +720,46 @@ void VPPredicator::run() {
}
}
+ LLVM_DEBUG({
+ dbgs() << "VPlan before predicating phis:\n";
+ Plan.dump();
+ });
for (VPBlockBase *VPBB : reverse(BlocksInCompactRPOTOrder))
if (VPBB != Header)
convertPhisToBlends(cast<VPBasicBlock>(VPBB));
+
+ LLVM_DEBUG({
+ dbgs() << "VPlan before SSA-fixup:\n";
+ Plan.dump();
+ });
+
+ for (VPBasicBlock *VPBB :
+ VPBlockUtils::blocksOnly<VPBasicBlock>(BlocksInCompactRPOTOrder)) {
+ for (VPRecipeBase &R : *VPBB) {
+ if (auto *Blend = dyn_cast<VPBlendRecipe>(&R)) {
+ // TODO: We can probably do better than this by performing partial
+ // blends along the path and have at most one phi per blend in a given
+ // basic block.
+ for (unsigned I = 0; I < Blend->getNumIncomingValues(); ++I) {
+ fixSSA(Blend, Blend->getIncomingValue(I), /*IsMask*/ false);
+ fixSSA(Blend, Blend->getMask(I), /*IsMask*/ true);
+ }
+ }
+ auto *I = dyn_cast<VPInstruction>(&R);
+ if (!I)
+ continue;
+
+ switch (I->getOpcode()) {
+ case VPInstruction::Not:
+ case Instruction::And:
+ case Instruction::Or: {
+ LLVM_DEBUG(dbgs() << "visiting " << *I << " for SSA fixup\n");
+ for (auto *V : I->operands())
+ fixSSA(I, V, /*IsMask*/ true);
+ }
+ }
+ }
+ }
}
void VPlanTransforms::introduceMasksAndLinearize(VPlan &Plan) {
diff --git a/llvm/test/Transforms/LoopVectorize/AArch64/blend-costs.ll b/llvm/test/Transforms/LoopVectorize/AArch64/blend-costs.ll
index c41e9d78de209..559b53dc87336 100644
--- a/llvm/test/Transforms/LoopVectorize/AArch64/blend-costs.ll
+++ b/llvm/test/Transforms/LoopVectorize/AArch64/blend-costs.ll
@@ -489,33 +489,24 @@ exit:
define void @only_first_lane_used(i1 %c, ptr noalias %p1, ptr noalias %p2, ptr noalias %q) #0 {
; CHECK-LABEL: define void @only_first_lane_used(
; CHECK-SAME: i1 [[C:%.*]], ptr noalias [[P1:%.*]], ptr noalias [[P2:%.*]], ptr noalias [[Q:%.*]]) #[[ATTR0:[0-9]+]] {
-; CHECK-NEXT: [[ENTRY:.*:]]
-; CHECK-NEXT: br label %[[VECTOR_PH:.*]]
-; CHECK: [[VECTOR_PH]]:
-; CHECK-NEXT: br label %[[VECTOR_BODY:.*]]
-; CHECK: [[VECTOR_BODY]]:
-; CHECK-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[PRED_STORE_CONTINUE6:.*]] ]
-; CHECK-NEXT: [[TMP4:%.*]] = add i64 [[INDEX]], -1
-; CHECK-NEXT: br i1 [[C]], label %[[PRED_STORE_IF:.*]], label %[[PRED_STORE_CONTINUE:.*]]
-; CHECK: [[PRED_STORE_IF]]:
-; CHECK-NEXT: store i32 0, ptr [[Q]], align 4
-; CHECK-NEXT: br label %[[PRED_STORE_CONTINUE]]
+; CHECK-NEXT: [[PRED_STORE_IF:.*:]]
+; CHECK-NEXT: br label %[[PRED_STORE_CONTINUE:.*]]
; CHECK: [[PRED_STORE_CONTINUE]]:
-; CHECK-NEXT: br i1 [[C]], label %[[PRED_STORE_IF1:.*]], label %[[PRED_STORE_CONTINUE2:.*]]
-; CHECK: [[PRED_STORE_IF1]]:
-; CHECK-NEXT: store i32 0, ptr [[Q]], align 4
-; CHECK-NEXT: br label %[[PRED_STORE_CONTINUE2]]
+; CHECK-NEXT: br label %[[PRED_STORE_CONTINUE2:.*]]
; CHECK: [[PRED_STORE_CONTINUE2]]:
+; CHECK-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[PRED_STORE_CONTINUE]] ], [ [[INDEX_NEXT:%.*]], %[[PRED_STORE_CONTINUE6:.*]] ]
; CHECK-NEXT: br i1 [[C]], label %[[PRED_STORE_IF3:.*]], label %[[PRED_STORE_CONTINUE4:.*]]
-; CHECK: [[PRED_STORE_IF3]]:
-; CHECK-NEXT: store i32 0, ptr [[Q]], align 4
-; CHECK-NEXT: br label %[[PRED_STORE_CONTINUE4]]
; CHECK: [[PRED_STORE_CONTINUE4]]:
-; CHECK-NEXT: br i1 [[C]], label %[[PRED_STORE_IF5:.*]], label %[[PRED_STORE_CONTINUE6]]
-; CHECK: [[PRED_STORE_IF5]]:
+; CHECK-NEXT: [[TMP0:%.*]] = add i64 [[INDEX]], -1
+; CHECK-NEXT: br label %[[PRED_STORE_CONTINUE6]]
+; CHECK: [[PRED_STORE_IF3]]:
+; CHECK-NEXT: [[TMP7:%.*]] = add i64 [[INDEX]], -1
; CHECK-NEXT: store i32 0, ptr [[Q]], align 4
; CHECK-NEXT: br label %[[PRED_STORE_CONTINUE6]]
; CHECK: [[PRED_STORE_CONTINUE6]]:
+; CHECK-NEXT: [[TMP8:%.*]] = phi i64 [ [[TMP0]], %[[PRED_STORE_CONTINUE4]] ], [ poison, %[[PRED_STORE_IF3]] ]
+; CHECK-NEXT: [[TMP9:%.*]] = phi i64 [ poison, %[[PRED_STORE_CONTINUE4]] ], [ [[TMP7]], %[[PRED_STORE_IF3]] ]
+; CHECK-NEXT: [[TMP4:%.*]] = select i1 true, i64 [[TMP9]], i64 [[TMP8]]
; CHECK-NEXT: [[TMP1:%.*]] = getelementptr double, ptr [[P1]], i64 [[TMP4]]
; CHECK-NEXT: [[WIDE_LOAD:%.*]] = load <4 x double>, ptr [[TMP1]], align 8
; CHECK-NEXT: [[TMP2:%.*]] = fadd <4 x double> [[WIDE_LOAD]], splat (double 1.000000e+00)
@@ -526,7 +517,7 @@ define void @only_first_lane_used(i1 %c, ptr noalias %p1, ptr noalias %p2, ptr n
; CHECK-NEXT: store <4 x double> [[TMP6]], ptr [[TMP3]], align 8
; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 4
; CHECK-NEXT: [[TMP5:%.*]] = icmp eq i64 [[INDEX_NEXT]], 1024
-; CHECK-NEXT: br i1 [[TMP5]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP6:![0-9]+]]
+; CHECK-NEXT: br i1 [[TMP5]], label %[[MIDDLE_BLOCK:.*]], label %[[PRED_STORE_CONTINUE2]], !llvm.loop [[LOOP6:![0-9]+]]
; CHECK: [[MIDDLE_BLOCK]]:
; CHECK-NEXT: br label %[[EXIT:.*]]
; CHECK: [[EXIT]]:
diff --git a/llvm/test/Transforms/LoopVectorize/AArch64/invariant-replicate-region.ll b/llvm/test/Transforms/LoopVectorize/AArch64/invariant-replicate-region.ll
index fce7da1572989..48d13eaa28ca6 100644
--- a/llvm/test/Transforms/LoopVectorize/AArch64/invariant-replicate-region.ll
+++ b/llvm/test/Transforms/LoopVectorize/AArch64/invariant-replicate-region.ll
@@ -13,43 +13,31 @@ define i32 @test_invariant_replicate_region(i32 %x, i1 %c) {
; CHECK: [[VECTOR_PH]]:
; CHECK-NEXT: br label %[[VECTOR_BODY:.*]]
; CHECK: [[VECTOR_BODY]]:
-; CHECK-NEXT: [[INDEX:%.*]] = phi i32 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[PRED_UREM_CONTINUE6:.*]] ]
-; CHECK-NEXT: br i1 [[C]], label %[[PRED_UREM_IF:.*]], label %[[PRED_UREM_CONTINUE:.*]]
+; CHECK-NEXT: [[INDEX:%.*]] = phi i32 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[PRED_UREM_CONTINUE:.*]] ]
+; CHECK-NEXT: br i1 [[C]], label %[[PRED_UREM_IF:.*]], label %[[PRED_UREM_CONTINUE]]
; CHECK: [[PRED_UREM_IF]]:
-; CHECK-NEXT: [[TMP1:%.*]] = urem i32 10, [[X]]
-; CHECK-NEXT: [[TMP3:%.*]] = insertelement <4 x i32> poison, i32 [[TMP1]], i64 0
; CHECK-NEXT: br label %[[PRED_UREM_CONTINUE]]
; CHECK: [[PRED_UREM_CONTINUE]]:
-; CHECK-NEXT: [[TMP2:%.*]] = phi <4 x i32> [ poison, %[[VECTOR_BODY]] ], [ [[TMP3]], %[[PRED_UREM_IF]] ]
-; CHECK-NEXT: br i1 [[C]], label %[[PRED_UREM_IF1:.*]], label %[[PRED_UREM_CONTINUE2:.*]]
+; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i32 [[INDEX]], 8
+; CHECK-NEXT: [[TMP0:%.*]] = icmp eq i32 [[INDEX_NEXT]], 96
+; CHECK-NEXT: br i1 [[TMP0]], label %[[PRED_UREM_IF1:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP0:![0-9]+]]
; CHECK: [[PRED_UREM_IF1]]:
-; CHECK-NEXT: [[TMP5:%.*]] = urem i32 10, [[X]]
-; CHECK-NEXT: [[TMP4:%.*]] = insertelement <4 x i32> [[TMP2]], i32 [[TMP5]], i64 1
-; CHECK-NEXT: br label %[[PRED_UREM_CONTINUE2]]
+; CHECK-NEXT: br label %[[PRED_UREM_CONTINUE2:.*]]
; CHECK: [[PRED_UREM_CONTINUE2]]:
-; CHECK-NEXT: [[TMP6:%.*]] = phi <4 x i32> [ [[TMP2]], %[[PRED_UREM_CONTINUE]] ], [ [[TMP4]], %[[PRED_UREM_IF1]] ]
-; CHECK-NEXT: br i1 [[C]], label %[[PRED_UREM_IF3:.*]], label %[[PRED_UREM_CONTINUE4:.*]]
-; CHECK: [[PRED_UREM_IF3]]:
-; CHECK-NEXT: [[TMP9:%.*]] = urem i32 10, [[X]]
-; CHECK-NEXT: [[TMP7:%.*]] = insertelement <4 x i32> [[TMP6]], i32 [[TMP9]], i64 2
-; CHECK-NEXT: br label %[[PRED_UREM_CONTINUE4]]
+; CHECK-NEXT: br label %[[PRED_UREM_CONTINUE4:.*]]
; CHECK: [[PRED_UREM_CONTINUE4]]:
-; CHECK-NEXT: [[TMP8:%.*]] = phi <4 x i32> [ [[TMP6]], %[[PRED_UREM_CONTINUE2]] ], [ [[TMP7]], %[[PRED_UREM_IF3]] ]
+; CHECK-NEXT: [[IV:%.*]] = phi i32 [ 96, %[[PRED_UREM_CONTINUE2]] ], [ [[IV_NEXT:%.*]], %[[PRED_UREM_CONTINUE6:.*]] ]
; CHECK-NEXT: br i1 [[C]], label %[[PRED_UREM_IF5:.*]], label %[[PRED_UREM_CONTINUE6]]
; CHECK: [[PRED_UREM_IF5]]:
; CHECK-NEXT: [[TMP13:%.*]] = urem i32 10, [[X]]
-; CHECK-NEXT: [[TMP10:%.*]] = insertelement <4 x i32> [[TMP8]], i32 [[TMP13]], i64 3
; CHECK-NEXT: br label %[[PRED_UREM_CONTINUE6]]
; CHECK: [[PRED_UREM_CONTINUE6]]:
-; CHECK-NEXT: [[TMP12:%.*]] = phi <4 x i32> [ [[TMP8]], %[[PRED_UREM_CONTINUE4]] ], [ [[TMP10]], %[[PRED_UREM_IF5]] ]
-; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i32 [[INDEX]], 4
-; CHECK-NEXT: [[TMP16:%.*]] = icmp eq i32 [[INDEX_NEXT]], 100
-; CHECK-NEXT: br i1 [[TMP16]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP0:![0-9]+]]
-; CHECK: [[MIDDLE_BLOCK]]:
-; CHECK-NEXT: [[PREDPHI:%.*]] = select i1 [[C]], <4 x i32> [[TMP12]], <4 x i32> zeroinitializer
-; CHECK-NEXT: [[TMP17:%.*]] = extractelement <4 x i32> [[PREDPHI]], i64 3
-; CHECK-NEXT: br label %[[EXIT:.*]]
+; CHECK-NEXT: [[RES:%.*]] = phi i32 [ 0, %[[PRED_UREM_CONTINUE4]] ], [ [[TMP13]], %[[PRED_UREM_IF5]] ]
+; CHECK-NEXT: [[IV_NEXT]] = add i32 [[IV]], 1
+; CHECK-NEXT: [[EC:%.*]] = icmp eq i32 [[IV]], 99
+; CHECK-NEXT: br i1 [[EC]], label %[[EXIT:.*]], label %[[PRED_UREM_CONTINUE4]], !llvm.loop [[LOOP3:![0-9]+]]
; CHECK: [[EXIT]]:
+; CHECK-NEXT: [[TMP17:%.*]] = phi i32 [ [[RES]], %[[PRED_UREM_CONTINUE6]] ]
; CHECK-NEXT: ret i32 [[TMP17]]
;
entry:
@@ -76,4 +64,5 @@ exit:
; CHECK: [[LOOP0]] = distinct !{[[LOOP0]], [[META1:![0-9]+]], [[META2:![0-9]+]]}
; CHECK: [[META1]] = !{!"llvm.loop.isvectorized", i32 1}
; CHECK: [[META2]] = !{!"llvm.loop.unroll.runtime.disable"}
+; CHECK: [[LOOP3]] = distinct !{[[LOOP3]], [[META2]], [[META1]]}
;.
diff --git a/llvm/test/Transforms/LoopVectorize/RISCV/dead-ops-cost.ll b/llvm/test/Transforms/LoopVectorize/RISCV/dead-ops-cost.ll
index 913eba75abf17..88edf52074be6 100644
--- a/llvm/test/Transforms/LoopVectorize/RISCV/dead-ops-cost.ll
+++ b/llvm/test/Transforms/LoopVectorize/RISCV/dead-ops-cost.ll
@@ -239,12 +239,16 @@ define void @test_phi_in_latch_redundant(ptr %dst, i32 %a) {
; CHECK-NEXT: [[TMP0:%.*]] = xor i32 [[A]], -1
; CHECK-NEXT: [[BROADCAST_SPLATINSERT:%.*]] = insertelement <vscale x 4 x i32> poison, i32 [[TMP0]], i64 0
; CHECK-NEXT: [[BROADCAST_SPLAT:%.*]] = shufflevector <vscale x 4 x i32> [[BROADCAST_SPLATINSERT]], <vscale x 4 x i32> poison, <vscale x 4 x i32> zeroinitializer
-; CHECK-NEXT: [[PREDPHI:%.*]] = select i1 true, <vscale x 4 x i32> [[BROADCAST_SPLAT]], <vscale x 4 x i32> zeroinitializer
; CHECK-NEXT: br label %[[VECTOR_BODY:.*]]
; CHECK: [[VECTOR_BODY]]:
-; CHECK-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[CURRENT_ITERATION_NEXT:%.*]], %[[VECTOR_BODY]] ]
-; CHECK-NEXT: [[AVL:%.*]] = phi i64 [ 37, %[[VECTOR_PH]] ], [ [[AVL_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[CURRENT_ITERATION_NEXT:%.*]], %[[LOOP_LATCH2:.*]] ]
+; CHECK-NEXT: [[AVL:%.*]] = phi i64 [ 37, %[[VECTOR_PH]] ], [ [[AVL_NEXT:%.*]], %[[LOOP_LATCH2]] ]
; CHECK-NEXT: [[TMP8:%.*]] = call i32 @llvm.experimental.get.vector.length.i64(i64 [[AVL]], i32 4, i1 true)
+; CHECK-NEXT: br label %[[THEN1:.*]]
+; CHECK: [[THEN1]]:
+; CHECK-NEXT: br label %[[LOOP_LATCH2]]
+; CHECK: [[LOOP_LATCH2]]:
+; CHECK-NEXT: [[PREDPHI:%.*]] = call <vscale x 4 x i32> @llvm.vp.merge.nxv4i32(<vscale x 4 x i1> splat (i1 true), <vscale x 4 x i32> [[BROADCAST_SPLAT]], <vscale x 4 x i32> zeroinitializer, i32 [[TMP8]])
; CHECK-NEXT: [[TMP2:%.*]] = mul i64 [[INDEX]], 36
; CHECK-NEXT: [[TMP3:%.*]] = getelementptr i8, ptr [[DST]], i64 [[TMP2]]
; CHECK-NEXT: call void @llvm.experimental.vp.strided.store.nxv4i32.p0.i64(<vscale x 4 x i32> [[PREDPHI]], ptr align 4 [[TMP3]], i64 36, <vscale x 4 x i1> splat (i1 true), i32 [[TMP8]])
diff --git a/llvm/test/Transforms/LoopVectorize/RISCV/divrem.ll b/llvm/test/Transforms/LoopVectorize/RISCV/divrem.ll
index d02dc36f7a6ae..bc0b7d532584e 100644
--- a/llvm/test/Transforms/LoopVectorize/RISCV/divrem.ll
+++ b/llvm/test/Transforms/LoopVectorize/RISCV/divrem.ll
@@ -633,9 +633,6 @@ define void @udiv_sdiv_with_invariant_divisors(i8 %x, i16 %y, i1 %c, ptr %p) {
; CHECK-NEXT: entry:
; CHECK-NEXT: br label [[VECTOR_BODY:%.*]]
; CHECK: vector.ph:
-; CHECK-NEXT: [[BROADCAST_SPLATINSERT:%.*]] = insertelement <vscale x 4 x i1> poison, i1 [[C:%.*]], i64 0
-; CHECK-NEXT: [[BROADCAST_SPLAT:%.*]] = shufflevector <vscale x 4 x i1> [[BROADCAST_SPLATINSERT]], <vscale x 4 x i1> poison, <vscale x 4 x i32> zeroinitializer
-; CHECK-NEXT: [[TMP0:%.*]] = xor <vscale x 4 x i1> [[BROADCAST_SPLAT]], splat (i1 true)
; CHECK-NEXT: [[BROADCAST_SPLATINSERT1:%.*]] = insertelement <vscale x 4 x i8> poison, i8 [[X:%.*]], i64 0
; CHECK-NEXT: [[BROADCAST_SPLAT2:%.*]] = shufflevector <vscale x 4 x i8> [[BROADCAST_SPLATINSERT1]], <vscale x 4 x i8> poison, <vscale x 4 x i32> zeroinitializer
; CHECK-NEXT: [[BROADCAST_SPLATINSERT3:%.*]] = insertelement <vscale x 4 x i16> poison, i16 [[Y:%.*]], i64 0
@@ -646,17 +643,29 @@ define void @udiv_sdiv_with_invariant_divisors(i8 %x, i16 %y, i1 %c, ptr %p) {
; CHECK-NEXT: [[INDUCTION:%.*]] = add <vscale x 4 x i8> splat (i8 -12), [[TMP1]]
; CHECK-NEXT: br label [[LOOP_LATCH:%.*]]
; CHECK: vector.body:
-; CHECK-NEXT: [[VEC_IND:%.*]] = phi <vscale x 4 x i8> [ [[INDUCTION]], [[VECTOR_BODY]] ], [ [[VEC_IND_NEXT:%.*]], [[LOOP_LATCH]] ]
-; CHECK-NEXT: [[AVL:%.*]] = phi i32 [ 12, [[VECTOR_BODY]] ], [ [[AVL_NEXT:%.*]], [[LOOP_LATCH]] ]
+; CHECK-NEXT: [[VEC_IND:%.*]] = phi <vscale x 4 x i8> [ [[INDUCTION]], [[VECTOR_BODY]] ], [ [[VEC_IND_NEXT:%.*]], [[LOOP_LATCH8:%.*]] ]
+; CHECK-NEXT: [[AVL:%.*]] = phi i32 [ 12, [[VECTOR_BODY]] ], [ [[AVL_NEXT:%.*]], [[LOOP_LATCH8]] ]
; CHECK-NEXT: [[TMP2:%.*]] = call i32 @llvm.experimental.get.vector.length.i32(i32 [[AVL]], i32 4, i1 true)
; CHECK-NEXT: [[TMP3:%.*]] = trunc i32 [[TMP2]] to i8
; CHECK-NEXT: [[BROADCAST_SPLATINSERT7:%.*]] = insertelement <vscale x 4 x i8> poison, i8 [[TMP3]], i64 0
; CHECK-NEXT: [[BROADCAST_SPLAT8:%.*]] = shufflevector <vscale x 4 x i8> [[BROADCAST_SPLATINSERT7]], <vscale x 4 x i8> poison, <vscale x 4 x i32> zeroinitializer
-; CHECK-NEXT: [[TMP5:%.*]] = call <vscale x 4 x i8> @llvm.vp.udiv.nxv4i8(<vscale x 4 x i8> [[VEC_IND]], <vscale x 4 x i8> [[BROADCAST_SPLAT2]], <vscale x 4 x i1> [[TMP0]], i32 [[TMP2]])
-; CHECK-NEXT: [[TMP6:%.*]] = zext <vscale x 4 x i8> [[TMP5]] to <vscale x 4 x i16>
-; CHECK-NEXT: [[TMP8:%.*]] = call <vscale x 4 x i16> @llvm.vp.sdiv.nxv4i16(<vscale x 4 x i16> [[TMP6]], <vscale x 4 x i16> [[BROADCAST_SPLAT4]], <vscale x 4 x i1> [[TMP0]], i32 [[TMP2]])
-; CHECK-NEXT: [[TMP9:%.*]] = sext <vscale x 4 x i16> [[TMP8]] to <vscale x 4 x i32>
-; CHECK-NEXT: [[PREDPHI:%.*]] = select i1 [[C]], <vscale x 4 x i32> zeroinitializer, <vscale x 4 x i32> [[TMP9]]
+; CHECK-NEXT: br i1 [[C:%.*]], label [[LOOP_LATCH8]], label [[THEN7:%.*]]
+; CHECK: then7:
+; CHECK-NEXT: [[TMP13:%.*]] = call <vscale x 4 x i8> @llvm.vp.merge.nxv4i8(<vscale x 4 x i1> splat (i1 true), <vscale x 4 x i8> [[BROADCAST_SPLAT2]], <vscale x 4 x i8> splat (i8 1), i32 [[TMP2]])
+; CHECK-NEXT: [[TMP14:%.*]] = extractelement <vscale x 4 x i8> [[VEC_IND]], i64 0
+; CHECK-NEXT: [[TMP5:%.*]] = extractelement <vscale x 4 x i8> [[TMP13]], i64 0
+; CHECK-NEXT: [[TMP6:%.*]] = udiv i8 [[TMP14]], [[TMP5]]
+; CHECK-NEXT: [[TMP7:%.*]] = zext i8 [[TMP6]] to i16
+; CHECK-NEXT: [[TMP8:%.*]] = call <vscale x 4 x i16> @llvm.vp.merge.nxv4i16(<vscale x 4 x i1> splat (i1 true), <vscale x 4 x i16> [[BROADCAST_SPLAT4]], <vscale x 4 x i16> splat (i16 1), i32 [[TMP2]])
+; CHECK-NEXT: [[TMP9:%.*]] = extractelement <vscale x 4 x i16> [[TMP8]], i64 0
+; CHECK-NEXT: [[TMP10:%.*]] = sdiv i16 [[TMP7]], [[TMP9]]
+; CHECK-NEXT: [[TMP11:%.*]] = sext i16 [[TMP10]] to i32
+; CHECK-NEXT: br label [[LOOP_LATCH8]]
+; CHECK: loop.latch8:
+; CHECK-NEXT: [[TMP12:%.*]] = phi i32 [ poison, [[LOOP_LATCH]] ], [ [[TMP11]], [[THEN7]] ]
+; CHECK-NEXT: [[BROADCAST_SPLATINSERT9:%.*]] = insertelement <vscale x 4 x i32> poison, i32 [[TMP12]], i64 0
+; CHECK-NEXT: [[BROADCAST_SPLAT10:%.*]] = shufflevector <vscale x 4 x i32> [[BROADCAST_SPLATINSERT9]], <vscale x 4 x i32> poison, <vscale x 4 x i32> zeroinitializer
+; CHECK-NEXT: [[PREDPHI:%.*]] = call <vscale x 4 x i32> @llvm.vp.merge.nxv4i32(<vscale x 4 x i1> splat (i1 true), <vscale x 4 x i32> [[BROADCAST_SPLAT10]], <vscale x 4 x i32> zeroinitializer, i32 [[TMP2]])
; CHECK-NEXT: call void @llvm.vp.scatter.nxv4i32.nxv4p0(<vscale x 4 x i32> [[PREDPHI]], <vscale x 4 x ptr> align 4 [[BROADCAST_SPLAT6]], <vscale x 4 x i1> splat (i1 true), i32 [[TMP2]])
; CHECK-NEXT: [[AVL_NEXT]] = sub nuw i32 [[AVL]], [[TMP2]]
; CHECK-NEXT: [[VEC_IND_NEXT]] = add <vscale x 4 x i8> [[VEC_IND]], [[BROADCAST_SPLAT8]]
@@ -671,30 +680,41 @@ define void @udiv_sdiv_with_invariant_divisors(i8 %x, i16 %y, i1 %c, ptr %p) {
; FIXED-NEXT: entry:
; FIXED-NEXT: br label [[VECTOR_PH:%.*]]
; FIXED: vector.ph:
-; FIXED-NEXT: [[BROADCAST_SPLATINSERT:%.*]] = insertelement <4 x i1> poison, i1 [[C:%.*]], i64 0
-; FIXED-NEXT: [[BROADCAST_SPLAT:%.*]] = shufflevector <4 x i1> [[BROADCAST_SPLATINSERT]], <4 x i1> poison, <4 x i32> zeroinitializer
-; FIXED-NEXT: [[TMP0:%.*]] = xor <4 x i1> [[BROADCAST_SPLAT]], splat (i1 true)
-; FIXED-NEXT: [[BROADCAST_SPLATINSERT1:%.*]] = insertelement <4 x i8> poison, i8 [[X:%.*]], i64 0
-; FIXED-NEXT: [[BROADCAST_SPLAT2:%.*]] = shufflevector <4 x i8> [[BROADCAST_SPLATINSERT1]], <4 x i8> poison, <4 x i32> zeroinitializer
-; FIXED-NEXT: [[BROADCAST_SPLATINSERT3:%.*]] = insertelement <4 x i16> poison, i16 [[Y:%.*]], i64 0
-; FIXED-NEXT: [[BROADCAST_SPLAT4:%.*]] = shufflevector <4 x i16> [[BROADCAST_SPLATINSERT3]], <4 x i16> poison, <4 x i32> zeroinitializer
; FIXED-NEXT: br label [[VECTOR_BODY:%.*]]
; FIXED: vector.body:
-; FIXED-NEXT: [[INDEX:%.*]] = phi i32 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ]
-; FIXED-NEXT: [[VEC_IND:%.*]] = phi <4 x i8> [ <i8 -12, i8 -11, i8 -10, i8 -9>, [[VECTOR_PH]] ], [ [[VEC_IND_NEXT:%.*]], [[VECTOR_BODY]] ]
-; FIXED-NEXT: [[INDEX_NEXT]] = add nuw i32 [[INDEX]], 4
-; FIXED-NEXT: [[VEC_IND_NEXT]] = add <4 x i8> [[VEC_IND]], splat (i8 4)
-; FIXED-NEXT: [[TMP5:%.*]] = icmp eq i32 [[INDEX_NEXT]], 12
-; FIXED-NEXT: br i1 [[TMP5]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP11:![0-9]+]]
-; FIXED: middle.block:
-; FIXED-NEXT: [[TMP1:%.*]] = call <4 x i8> @llvm.masked.udiv.v4i8(<4 x i8> [[VEC_IND]], <4 x i8> [[BROADCAST_SPLAT2]], <4 x i1> [[TMP0]])
-; FIXED-NEXT: [[TMP2:%.*]] = zext <4 x i8> [[TMP1]] to <4 x i16>
-; FIXED-NEXT: [[TMP3:%.*]] = call <4 x i16> @llvm.masked.sdiv.v4i16(<4 x i16> [[TMP2]], <4 x i16> [[BROADCAST_SPLAT4]], <4 x i1> [[TMP0]])
-; FIXED-NEXT: [[TMP4:%.*]] = sext <4 x i16> [[TMP3]] to <4 x i32>
-; FIXED-NEXT: [[PREDPHI:%.*]] = select i1 [[C]], <4 x i32> zeroinitializer, <4 x i32> [[TMP4]]
-; FIXED-NEXT: [[TMP7:%.*]] = extractelement <4 x i32> [[PREDPHI]], i64 3
+; FIXED-NEXT: br i1 [[C:%.*]], label [[LOOP_LATCH2:%.*]], label [[THEN1:%.*]]
+; FIXED: then1:
+; FIXED-NEXT: [[TMP0:%.*]] = udiv i8 -12, [[X:%.*]]
+; FIXED-NEXT: [[TMP1:%.*]] = zext i8 [[TMP0]] to i16
+; FIXED-NEXT: [[TMP2:%.*]] = sdiv i16 [[TMP1]], [[Y:%.*]]
+; FIXED-NEXT: [[TMP3:%.*]] = sext i16 [[TMP2]] to i32
+; FIXED-NEXT: br label [[LOOP_LATCH2]]
+; FIXED: loop.latch2:
+; FIXED-NEXT: [[TMP4:%.*]] = phi i32 [ poison, [[VECTOR_BODY]] ], [ [[TMP3]], [[THEN1]] ]
+; FIXED-NEXT: [[TMP7:%.*]] = select i1 true, i32 [[TMP4]], i32 0
; FIXED-NEXT: store i32 [[TMP7]], ptr [[P:%.*]], align 4
; FIXED-NEXT: br label [[LOOP_LATCH:%.*]]
+; FIXED: middle.block:
+; FIXED-NEXT: br label [[SCALAR_PH:%.*]]
+; FIXED: scalar.ph:
+; FIXED-NEXT: br label [[LOOP_HEADER:%.*]]
+; FIXED: loop.header:
+; FIXED-NEXT: [[IV:%.*]] = phi i16 [ -4, [[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], [[LOOP_LATCH1:%.*]] ]
+; FIXED-NEXT: [[NARROW_IV:%.*]] = phi i8 [ -4, [[SCALAR_PH]] ], [ [[IV_NEXT_TRUNC:%.*]], [[LOOP_LATCH1]] ]
+; FIXED-NEXT: br i1 [[C]], label [[LOOP_LATCH1]], label [[THEN:%.*]]
+; FIXED: then:
+; FIXED-NEXT: [[UD:%.*]] = udiv i8 [[NARROW_IV]], [[X]]
+; FIXED-NEXT: [[UD_EXT:%.*]] = zext i8 [[UD]] to i16
+; FIXED-NEXT: [[SD:%.*]] = sdiv i16 [[UD_EXT]], [[Y]]
+; FIXED-NEXT: [[SD_EXT:%.*]] = sext i16 [[SD]] to i32
+; FIXED-NEXT: br label [[LOOP_LATCH1]]
+; FIXED: loop.latch:
+; FIXED-NEXT: [[MERGE:%.*]] = phi i32 [ 0, [[LOOP_HEADER]] ], [ [[SD_EXT]], [[THEN]] ]
+; FIXED-NEXT: store i32 [[MERGE]], ptr [[P]], align 4
+; FIXED-NEXT: [[IV_NEXT]] = add nsw i16 [[IV]], 1
+; FIXED-NEXT: [[EC:%.*]] = icmp eq i16 [[IV_NEXT]], 0
+; FIXED-NEXT: [[IV_NEXT_TRUNC]] = trunc i16 [[IV_NEXT]] to i8
+; FIXED-NEXT: br i1 [[EC]], label [[EXIT:%.*]], label [[LOOP_HEADER]], !llvm.loop [[LOOP11:![0-9]+]]
; FIXED: exit:
; FIXED-NEXT: ret void
;
diff --git a/llvm/test/Transforms/LoopVectorize/RISCV/tail-folding-complex-mask.ll b/llvm/test/Transforms/LoopVectorize/RISCV/tail-folding-complex-mask.ll
index 9b3df00b602ef..303193a29d99f 100644
--- a/llvm/test/Transforms/LoopVectorize/RISCV/tail-folding-complex-mask.ll
+++ b/llvm/test/Transforms/LoopVectorize/RISCV/tail-folding-complex-mask.ll
@@ -21,12 +21,10 @@ define void @test(i64 %n, ptr noalias %src0, ptr noalias %src1, ptr noalias %src
; IF-EVL-NEXT: [[TMP1:%.*]] = or <vscale x 4 x i1> [[BROADCAST_SPLAT]], [[BROADCAST_SPLAT2]]
; IF-EVL-NEXT: [[TMP3:%.*]] = select <vscale x 4 x i1> [[TMP2]], <vscale x 4 x i1> [[TMP1]], <vscale x 4 x i1> zeroinitializer
; IF-EVL-NEXT: [[TMP4:%.*]] = or <vscale x 4 x i1> [[BROADCAST_SPLAT]], [[TMP3]]
-; IF-EVL-NEXT: [[BROADCAST_SPLATINSERT3:%.*]] = insertelement <vscale x 4 x i1> poison, i1 [[C3]], i64 0
-; IF-EVL-NEXT: [[BROADCAST_SPLAT4:%.*]] = shufflevector <vscale x 4 x i1> [[BROADCAST_SPLATINSERT3]], <vscale x 4 x i1> poison, <vscale x 4 x i32> zeroinitializer
; IF-EVL-NEXT: br label %[[VECTOR_BODY:.*]]
; IF-EVL: [[VECTOR_BODY]]:
-; IF-EVL-NEXT: [[EVL_BASED_IV:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_EVL_NEXT:%.*]], %[[VECTOR_BODY]] ]
-; IF-EVL-NEXT: [[AVL:%.*]] = phi i64 [ [[N]], %[[VECTOR_PH]] ], [ [[AVL_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; IF-EVL-NEXT: [[EVL_BASED_IV:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_EVL_NEXT:%.*]], %[[LATCH7:.*]] ]
+; IF-EVL-NEXT: [[AVL:%.*]] = phi i64 [ [[N]], %[[VECTOR_PH]] ], [ [[AVL_NEXT:%.*]], %[[LATCH7]] ]
; IF-EVL-NEXT: [[TMP7:%.*]] = call i32 @llvm.experimental.get.vector.length.i64(i64 [[AVL]], i32 4, i1 true)
; IF-EVL-NEXT: [[TMP10:%.*]] = getelementptr i32, ptr [[SRC0]], i64 [[EVL_BASED_IV]]
; IF-EVL-NEXT: [[VP_OP_LOAD:%.*]] = call <vscale x 4 x i32> @llvm.vp.load.nxv4i32.p0(ptr align 4 [[TMP10]], <vscale x 4 x i1> [[BROADCAST_SPLAT]], i32 [[TMP7]])
@@ -35,10 +33,19 @@ define void @test(i64 %n, ptr noalias %src0, ptr noalias %src1, ptr noalias %src
; IF-EVL-NEXT: [[VP_OP_LOAD7:%.*]] = call <vscale x 4 x i32> @llvm.vp.load.nxv4i32.p0(ptr align 4 [[TMP11]], <vscale x 4 x i1> [[TMP4]], i32 [[TMP7]])
; IF-EVL-NEXT: [[TMP12:%.*]] = add <vscale x 4 x i32> [[VP_OP_LOAD7]], [[PREDPHI]]
; IF-EVL-NEXT: [[PREDPHI8:%.*]] = select <vscale x 4 x i1> [[TMP4]], <vscale x 4 x i32> [[TMP12]], <vscale x 4 x i32> zeroinitializer
-; IF-EVL-NEXT: [[TMP18:%.*]] = getelementptr i32, ptr [[SRC2]], i64 [[EVL_BASED_IV]]
-; IF-EVL-NEXT: [[WIDE_MASKED_LOAD:%.*]] = call <vscale x 4 x i32> @llvm.vp.load.nxv4i32.p0(ptr align 4 [[TMP18]], <vscale x 4 x i1> [[BROADCAST_SPLAT4]], i32 [[TMP7]])
-; IF-EVL-NEXT: [[TMP19:%.*]] = add <vscale x 4 x i32> [[WIDE_MASKED_LOAD]], [[PREDPHI8]]
-; IF-EVL-NEXT: [[PREDPHI9:%.*]] = select i1 [[C3]], <vscale x 4 x i32> [[TMP19]], <vscale x 4 x i32> [[PREDPHI8]]
+; IF-EVL-NEXT: [[TMP9:%.*]] = extractelement <vscale x 4 x i32> [[PREDPHI8]], i64 0
+; IF-EVL-NEXT: br i1 [[C3]], label %[[LOAD_V25:.*]], label %[[LATCH7]]
+; IF-EVL: [[LOAD_V25]]:
+; IF-EVL-NEXT: [[TMP14:%.*]] = getelementptr inbounds i32, ptr [[SRC2]], i64 [[EVL_BASED_IV]]
+; IF-EVL-NEXT: [[VP_OP_LOAD6:%.*]] = call <vscale x 4 x i32> @llvm.vp.load.nxv4i32.p0(ptr align 4 [[TMP14]], <vscale x 4 x i1> splat (i1 true), i32 [[TMP7]])
+; IF-EVL-NEXT: [[TMP15:%.*]] = extractelement <vscale x 4 x i32> [[VP_OP_LOAD6]], i64 0
+; IF-EVL-NEXT: [[TMP16:%.*]] = add i32 [[TMP15]], [[TMP9]]
+; IF-EVL-NEXT: br label %[[LATCH7]]
+; IF-EVL: [[LATCH7]]:
+; IF-EVL-NEXT: [[TMP13:%.*]] = phi i32 [ poison, %[[VECTOR_BODY]] ], [ [[TMP16]], %[[LOAD_V25]] ]
+; IF-EVL-NEXT: [[BROADCAST_SPLATINSERT8:%.*]] = insertelement <vscale x 4 x i32> poison, i32 [[TMP13]], i64 0
+; IF-EVL-NEXT: [[BROADCAST_SPLAT9:%.*]] = shufflevector <vscale x 4 x i32> [[BROADCAST_SPLATINSERT8]], <vscale x 4 x i32> poison, <vscale x 4 x i32> zeroinitializer
+; IF-EVL-NEXT: [[PREDPHI9:%.*]] = call <vscale x 4 x i32> @llvm.vp.merge.nxv4i32(<vscale x 4 x i1> splat (i1 true), <vscale x 4 x i32> [[BROADCAST_SPLAT9]], <vscale x 4 x i32> [[PREDPHI8]], i32 [[TMP7]])
; IF-EVL-NEXT: [[TMP20:%.*]] = getelementptr inbounds i32, ptr [[DST]], i64 [[EVL_BASED_IV]]
; IF-EVL-NEXT: call void @llvm.vp.store.nxv4i32.p0(<vscale x 4 x i32> [[PREDPHI9]], ptr align 4 [[TMP20]], <vscale x 4 x i1> splat (i1 true), i32 [[TMP7]])
; IF-EVL-NEXT: [[TMP21:%.*]] = zext i32 [[TMP7]] to i64
@@ -71,11 +78,9 @@ define void @test(i64 %n, ptr noalias %src0, ptr noalias %src1, ptr noalias %src
; NO-VP-NEXT: [[TMP9:%.*]] = or <vscale x 4 x i1> [[BROADCAST_SPLAT2]], [[BROADCAST_SPLAT4]]
; NO-VP-NEXT: [[TMP10:%.*]] = select <vscale x 4 x i1> [[TMP6]], <vscale x 4 x i1> [[TMP9]], <vscale x 4 x i1> zeroinitializer
; NO-VP-NEXT: [[TMP8:%.*]] = or <vscale x 4 x i1> [[BROADCAST_SPLAT2]], [[TMP10]]
-; NO-VP-NEXT: [[BROADCAST_SPLATINSERT4:%.*]] = insertelement <vscale x 4 x i1> poison, i1 [[C3]], i64 0
-; NO-VP-NEXT: [[TMP12:%.*]] = shufflevector <vscale x 4 x i1> [[BROADCAST_SPLATINSERT4]], <vscale x 4 x i1> poison, <vscale x 4 x i32> zeroinitializer
; NO-VP-NEXT: br label %[[VECTOR_BODY:.*]]
; NO-VP: [[VECTOR_BODY]]:
-; NO-VP-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; NO-VP-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[LATCH6:.*]] ]
; NO-VP-NEXT: [[TMP13:%.*]] = getelementptr i32, ptr [[SRC0]], i64 [[INDEX]]
; NO-VP-NEXT: [[WIDE_MASKED_LOAD:%.*]] = call <vscale x 4 x i32> @llvm.masked.load.nxv4i32.p0(ptr align 4 [[TMP13]], <vscale x 4 x i1> [[BROADCAST_SPLAT2]], <vscale x 4 x i32> poison)
; NO-VP-NEXT: [[PREDPHI:%.*]] = select i1 [[C1]], <vscale x 4 x i32> [[WIDE_MASKED_LOAD]], <vscale x 4 x i32> zeroinitializer
@@ -83,10 +88,19 @@ define void @test(i64 %n, ptr noalias %src0, ptr noalias %src1, ptr noalias %src
; NO-VP-NEXT: [[WIDE_MASKED_LOAD5:%.*]] = call <vscale x 4 x i32> @llvm.masked.load.nxv4i32.p0(ptr align 4 [[TMP14]], <vscale x 4 x i1> [[TMP8]], <vscale x 4 x i32> poison)
; NO-VP-NEXT: [[TMP15:%.*]] = add <vscale x 4 x i32> [[WIDE_MASKED_LOAD5]], [[PREDPHI]]
; NO-VP-NEXT: [[PREDPHI6:%.*]] = select <vscale x 4 x i1> [[TMP8]], <vscale x 4 x i32> [[TMP15]], <vscale x 4 x i32> zeroinitializer
-; NO-VP-NEXT: [[TMP16:%.*]] = getelementptr i32, ptr [[SRC2]], i64 [[INDEX]]
-; NO-VP-NEXT: [[WIDE_MASKED_LOAD7:%.*]] = call <vscale x 4 x i32> @llvm.masked.load.nxv4i32.p0(ptr align 4 [[TMP16]], <vscale x 4 x i1> [[TMP12]], <vscale x 4 x i32> poison)
-; NO-VP-NEXT: [[TMP17:%.*]] = add <vscale x 4 x i32> [[WIDE_MASKED_LOAD7]], [[PREDPHI6]]
-; NO-VP-NEXT: [[PREDPHI8:%.*]] = select i1 [[C3]], <vscale x 4 x i32> [[TMP17]], <vscale x 4 x i32> [[PREDPHI6]]
+; NO-VP-NEXT: [[TMP11:%.*]] = extractelement <vscale x 4 x i32> [[PREDPHI6]], i64 0
+; NO-VP-NEXT: br i1 [[C3]], label %[[LOAD_V25:.*]], label %[[LATCH6]]
+; NO-VP: [[LOAD_V25]]:
+; NO-VP-NEXT: [[TMP12:%.*]] = getelementptr inbounds i32, ptr [[SRC2]], i64 [[INDEX]]
+; NO-VP-NEXT: [[WIDE_LOAD:%.*]] = load <vscale x 4 x i32>, ptr [[TMP12]], align 4
+; NO-VP-NEXT: [[TMP16:%.*]] = extractelement <vscale x 4 x i32> [[WIDE_LOAD]], i64 0
+; NO-VP-NEXT: [[TMP17:%.*]] = add i32 [[TMP16]], [[TMP11]]
+; NO-VP-NEXT: br label %[[LATCH6]]
+; NO-VP: [[LATCH6]]:
+; NO-VP-NEXT: [[TMP20:%.*]] = phi i32 [ poison, %[[VECTOR_BODY]] ], [ [[TMP17]], %[[LOAD_V25]] ]
+; NO-VP-NEXT: [[BROADCAST_SPLATINSERT7:%.*]] = insertelement <vscale x 4 x i32> poison, i32 [[TMP20]], i64 0
+; NO-VP-NEXT: [[BROADCAST_SPLAT8:%.*]] = shufflevector <vscale x 4 x i32> [[BROADCAST_SPLATINSERT7]], <vscale x 4 x i32> poison, <vscale x 4 x i32> zeroinitializer
+; NO-VP-NEXT: [[PREDPHI8:%.*]] = select i1 true, <vscale x 4 x i32> [[BROADCAST_SPLAT8]], <vscale x 4 x i32> [[PREDPHI6]]
; NO-VP-NEXT: [[TMP18:%.*]] = getelementptr inbounds i32, ptr [[DST]], i64 [[INDEX]]
; NO-VP-NEXT: store <vscale x 4 x i32> [[PREDPHI8]], ptr [[TMP18]], align 4
; NO-VP-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], [[TMP1]]
diff --git a/llvm/test/Transforms/LoopVectorize/VPlan/phi-with-fastflags-vplan.ll b/llvm/test/Transforms/LoopVectorize/VPlan/phi-with-fastflags-vplan.ll
index 35ba001b2fb50..d618de9f05680 100644
--- a/llvm/test/Transforms/LoopVectorize/VPlan/phi-with-fastflags-vplan.ll
+++ b/llvm/test/Transforms/LoopVectorize/VPlan/phi-with-fastflags-vplan.ll
@@ -23,7 +23,14 @@ define void @f(ptr noalias %p, i1 %c) {
; CHECK-NEXT: CLONE ir<%gep> = getelementptr ir<%p>, vp<[[VP4]]>
; CHECK-NEXT: vp<[[VP5:%[0-9]+]]> = vector-pointer float, ir<%gep>, ir<1>
; CHECK-NEXT: WIDEN ir<%x> = load vp<[[VP5]]>
-; CHECK-NEXT: BLEND ir<%phi> = fast ir<%x> ir<0.000000e+00>/ir<%c>
+; CHECK-NEXT: EMIT branch-on-cond ir<%c>
+; CHECK-NEXT: Successor(s): then, latch
+; CHECK-EMPTY:
+; CHECK-NEXT: then:
+; CHECK-NEXT: Successor(s): latch
+; CHECK-EMPTY:
+; CHECK-NEXT: latch:
+; CHECK-NEXT: BLEND ir<%phi> = fast ir<%x> ir<0.000000e+00>/ir<true>
; CHECK-NEXT: vp<[[VP6:%[0-9]+]]> = vector-pointer float, ir<%gep>, ir<1>
; CHECK-NEXT: WIDEN store vp<[[VP6]]>, ir<%phi>
; CHECK-NEXT: EMIT vp<%index.next> = add nuw vp<[[VP3]]>, vp<[[VP1]]>
diff --git a/llvm/test/Transforms/LoopVectorize/VPlan/predicator.ll b/llvm/test/Transforms/LoopVectorize/VPlan/predicator.ll
index a8e69d1ab4f4d..e6e10623f6822 100644
--- a/llvm/test/Transforms/LoopVectorize/VPlan/predicator.ll
+++ b/llvm/test/Transforms/LoopVectorize/VPlan/predicator.ll
@@ -582,20 +582,18 @@ define void @blend_masks_triangle_phi(ptr noalias %p, i1 %c0, i1 %c1) {
; CHECK-EMPTY:
; CHECK-NEXT: vector.body:
; CHECK-NEXT: ir<%iv> = WIDEN-INDUCTION ir<0>, ir<1>, vp<[[VP0:%[0-9]+]]>
-; CHECK-NEXT: Successor(s): bb1
+; CHECK-NEXT: EMIT branch-on-cond ir<%c0>
+; CHECK-NEXT: Successor(s): bb1, bb2
; CHECK-EMPTY:
; CHECK-NEXT: bb1:
; CHECK-NEXT: Successor(s): bb2
; CHECK-EMPTY:
; CHECK-NEXT: bb2:
; CHECK-NEXT: EMIT vp<[[VP4:%[0-9]+]]> = not ir<%c1>
-; CHECK-NEXT: EMIT vp<[[VP5:%[0-9]+]]> = logical-and ir<%c0>, vp<[[VP4]]>
-; CHECK-NEXT: EMIT vp<[[VP6:%[0-9]+]]> = not ir<%c0>
-; CHECK-NEXT: EMIT vp<[[VP7:%[0-9]+]]> = or vp<[[VP5]]>, vp<[[VP6]]>
; CHECK-NEXT: Successor(s): bb3
; CHECK-EMPTY:
; CHECK-NEXT: bb3:
-; CHECK-NEXT: BLEND ir<%phi> = ir<0>/ir<%c0> ir<1>/vp<[[VP7]]>
+; CHECK-NEXT: BLEND ir<%phi> = ir<0>/ir<true> ir<1>/ir<true>
; CHECK-NEXT: EMIT ir<%gep> = getelementptr ir<%p>, ir<%iv>
; CHECK-NEXT: EMIT store ir<%phi>, ir<%gep>
; CHECK-NEXT: EMIT ir<%iv.next> = add ir<%iv>, ir<1>
@@ -748,30 +746,21 @@ define void @simplifiable_blend(i1 %c1, i1 %c2, i1 %c3, i32 %x, i32 %y, ptr %p)
; CHECK-EMPTY:
; CHECK-NEXT: vector.body:
; CHECK-NEXT: ir<%iv> = WIDEN-INDUCTION ir<0>, ir<1>, vp<[[VP0:%[0-9]+]]>
-; CHECK-NEXT: Successor(s): B
-; CHECK-EMPTY:
-; CHECK-NEXT: B:
-; CHECK-NEXT: EMIT vp<[[VP4:%[0-9]+]]> = not ir<%c1>
-; CHECK-NEXT: EMIT branch-on-cond ir<%c3>
-; CHECK-NEXT: Successor(s): F, E
-; CHECK-EMPTY:
-; CHECK-NEXT: F:
-; CHECK-NEXT: Successor(s): A
+; CHECK-NEXT: EMIT branch-on-cond ir<%c1>
+; CHECK-NEXT: Successor(s): A, B
; CHECK-EMPTY:
; CHECK-NEXT: A:
; CHECK-NEXT: Successor(s): D
; CHECK-EMPTY:
; CHECK-NEXT: D:
-; CHECK-NEXT: EMIT vp<[[VP5:%[0-9]+]]> = not ir<%c2>
-; CHECK-NEXT: EMIT vp<[[VP6:%[0-9]+]]> = logical-and ir<%c1>, vp<[[VP5]]>
+; CHECK-NEXT: EMIT vp<[[VP4:%[0-9]+]]> = not ir<%c2>
; CHECK-NEXT: Successor(s): C
; CHECK-EMPTY:
; CHECK-NEXT: C:
-; CHECK-NEXT: EMIT vp<[[VP7:%[0-9]+]]> = logical-and ir<%c1>, ir<%c2>
; CHECK-NEXT: Successor(s): latch
; CHECK-EMPTY:
; CHECK-NEXT: latch:
-; CHECK-NEXT: BLEND ir<%phi> = ir<%y>/vp<[[VP4]]> ir<%x>/ir<%c1>
+; CHECK-NEXT: BLEND ir<%phi> = ir<%y>/ir<true> ir<%x>/ir<true>
; CHECK-NEXT: EMIT ir<%gep> = getelementptr ir<%p>, ir<%iv>
; CHECK-NEXT: EMIT store ir<%phi>, ir<%gep>
; CHECK-NEXT: EMIT ir<%iv.next> = add ir<%iv>, ir<1>
@@ -780,6 +769,13 @@ define void @simplifiable_blend(i1 %c1, i1 %c2, i1 %c3, i32 %x, i32 %y, ptr %p)
; CHECK-NEXT: EMIT branch-on-count vp<%index.next>, vp<[[VP2:%[0-9]+]]>
; CHECK-NEXT: No successors
; CHECK-EMPTY:
+; CHECK-NEXT: B:
+; CHECK-NEXT: EMIT branch-on-cond ir<%c3>
+; CHECK-NEXT: Successor(s): F, E
+; CHECK-EMPTY:
+; CHECK-NEXT: F:
+; CHECK-NEXT: Successor(s): latch
+; CHECK-EMPTY:
; CHECK-NEXT: E:
; CHECK-NEXT: Successor(s): F
; CHECK-NEXT: }
@@ -957,25 +953,26 @@ define void @outermost_uniform_branch(ptr %a, i1 %u0) {
; CHECK-EMPTY:
; CHECK-NEXT: vector.body:
; CHECK-NEXT: ir<%iv> = WIDEN-INDUCTION nuw nsw ir<0>, ir<1>, vp<[[VP0:%[0-9]+]]>
-; CHECK-NEXT: Successor(s): bb1
+; CHECK-NEXT: EMIT branch-on-cond ir<%u0>
+; CHECK-NEXT: Successor(s): bb1, bb4
; CHECK-EMPTY:
; CHECK-NEXT: bb1:
-; CHECK-NEXT: EMIT ir<%add1> = add ir<%iv>, ir<1>, ir<%u0>
-; CHECK-NEXT: EMIT ir<%v1> = icmp sle ir<%iv>, ir<1>, ir<%u0>
+; CHECK-NEXT: EMIT ir<%add1> = add ir<%iv>, ir<1>
+; CHECK-NEXT: EMIT ir<%v1> = icmp sle ir<%iv>, ir<1>
; CHECK-NEXT: Successor(s): bb2
; CHECK-EMPTY:
; CHECK-NEXT: bb2:
-; CHECK-NEXT: EMIT vp<[[VP4:%[0-9]+]]> = logical-and ir<%u0>, ir<%v1>
-; CHECK-NEXT: EMIT ir<%add2> = add ir<%iv>, ir<2>, vp<[[VP4]]>
+; CHECK-NEXT: EMIT ir<%add2> = add ir<%iv>, ir<2>, ir<%v1>
; CHECK-NEXT: Successor(s): bb3
; CHECK-EMPTY:
; CHECK-NEXT: bb3:
; CHECK-NEXT: BLEND ir<%phi3> = ir<%add1>/ir<true> ir<%add2>/ir<%v1>
-; CHECK-NEXT: EMIT ir<%add3> = add ir<%phi3>, ir<3>, ir<%u0>
+; CHECK-NEXT: EMIT ir<%add3> = add ir<%phi3>, ir<3>
; CHECK-NEXT: Successor(s): bb4
; CHECK-EMPTY:
; CHECK-NEXT: bb4:
-; CHECK-NEXT: BLEND ir<%phi4> = ir<%iv>/ir<true> ir<%add3>/ir<%u0>
+; CHECK-NEXT: EMIT-SCALAR vp<[[VP4:%[0-9]+]]> = phi [ ir<poison>, vector.body ], [ ir<%add3>, bb3 ]
+; CHECK-NEXT: BLEND ir<%phi4> = ir<%iv>/ir<true> vp<%4>/ir<true>
; CHECK-NEXT: EMIT store ir<%phi4>, ir<%a>
; CHECK-NEXT: EMIT ir<%iv.next> = add nuw nsw ir<%iv>, ir<1>
; CHECK-NEXT: EMIT ir<%ec> = icmp eq ir<%iv.next>, ir<128>
@@ -1113,41 +1110,41 @@ define void @outermost_uniform_branch_more_blocks(ptr %a, i1 %u0) {
; CHECK-EMPTY:
; CHECK-NEXT: vector.body:
; CHECK-NEXT: ir<%iv> = WIDEN-INDUCTION nuw nsw ir<0>, ir<1>, vp<[[VP0:%[0-9]+]]>
-; CHECK-NEXT: Successor(s): bb1
-; CHECK-EMPTY:
-; CHECK-NEXT: bb1:
-; CHECK-NEXT: EMIT vp<[[VP4:%[0-9]+]]> = not ir<%u0>
-; CHECK-NEXT: EMIT ir<%add1> = add ir<%iv>, ir<1>, vp<[[VP4]]>
-; CHECK-NEXT: Successor(s): bb2
+; CHECK-NEXT: EMIT branch-on-cond ir<%u0>
+; CHECK-NEXT: Successor(s): bb2, bb1
; CHECK-EMPTY:
; CHECK-NEXT: bb2:
-; CHECK-NEXT: EMIT ir<%v2> = icmp sle ir<%iv>, ir<2>, ir<%u0>
+; CHECK-NEXT: EMIT ir<%v2> = icmp sle ir<%iv>, ir<2>
; CHECK-NEXT: Successor(s): bb3
; CHECK-EMPTY:
; CHECK-NEXT: bb3:
-; CHECK-NEXT: EMIT vp<[[VP5:%[0-9]+]]> = not ir<%v2>
-; CHECK-NEXT: EMIT vp<[[VP6:%[0-9]+]]> = logical-and ir<%u0>, vp<[[VP5]]>
-; CHECK-NEXT: EMIT ir<%add3> = add ir<%iv>, ir<3>, vp<[[VP6]]>
+; CHECK-NEXT: EMIT vp<[[VP4:%[0-9]+]]> = not ir<%v2>
+; CHECK-NEXT: EMIT ir<%add3> = add ir<%iv>, ir<3>, vp<[[VP4]]>
; CHECK-NEXT: Successor(s): bb4
; CHECK-EMPTY:
; CHECK-NEXT: bb4:
-; CHECK-NEXT: EMIT vp<[[VP7:%[0-9]+]]> = logical-and ir<%u0>, ir<%v2>
-; CHECK-NEXT: EMIT ir<%add4> = add ir<%iv>, ir<4>, vp<[[VP7]]>
+; CHECK-NEXT: EMIT ir<%add4> = add ir<%iv>, ir<4>, ir<%v2>
; CHECK-NEXT: Successor(s): bb5
; CHECK-EMPTY:
; CHECK-NEXT: bb5:
-; CHECK-NEXT: BLEND ir<%phi5> = ir<%add3>/vp<[[VP5]]> ir<%add4>/ir<%v2>
-; CHECK-NEXT: EMIT ir<%add5> = add ir<%phi5>, ir<5>, ir<%u0>
+; CHECK-NEXT: BLEND ir<%phi5> = ir<%add3>/vp<[[VP4]]> ir<%add4>/ir<%v2>
+; CHECK-NEXT: EMIT ir<%add5> = add ir<%phi5>, ir<5>
; CHECK-NEXT: Successor(s): bb6
; CHECK-EMPTY:
; CHECK-NEXT: bb6:
-; CHECK-NEXT: BLEND ir<%phi6> = ir<%add1>/vp<[[VP4]]> ir<%add5>/ir<%u0>
+; CHECK-NEXT: EMIT-SCALAR vp<[[VP5:%[0-9]+]]> = phi [ ir<%add1>, bb1 ], [ ir<poison>, bb5 ]
+; CHECK-NEXT: EMIT-SCALAR vp<[[VP6:%[0-9]+]]> = phi [ ir<poison>, bb1 ], [ ir<%add5>, bb5 ]
+; CHECK-NEXT: BLEND ir<%phi6> = vp<%5>/ir<true> vp<%6>/ir<true>
; CHECK-NEXT: EMIT store ir<%phi6>, ir<%a>
; CHECK-NEXT: EMIT ir<%iv.next> = add nuw nsw ir<%iv>, ir<1>
; CHECK-NEXT: EMIT ir<%ec> = icmp eq ir<%iv.next>, ir<128>
; CHECK-NEXT: EMIT vp<%index.next> = add nuw vp<[[VP3]]>, vp<[[VP1:%[0-9]+]]>
; CHECK-NEXT: EMIT branch-on-count vp<%index.next>, vp<[[VP2:%[0-9]+]]>
; CHECK-NEXT: No successors
+; CHECK-EMPTY:
+; CHECK-NEXT: bb1:
+; CHECK-NEXT: EMIT ir<%add1> = add ir<%iv>, ir<1>
+; CHECK-NEXT: Successor(s): bb6
; CHECK-NEXT: }
; CHECK-NEXT: Successor(s): middle.block
;
@@ -1307,15 +1304,16 @@ define void @uniform_branch_after_varying_branch(ptr %a, i1 %u1) {
; CHECK-NEXT: bb1:
; CHECK-NEXT: EMIT vp<[[VP4:%[0-9]+]]> = not ir<%v0>
; CHECK-NEXT: EMIT ir<%add1> = add ir<%iv>, ir<1>, vp<[[VP4]]>
-; CHECK-NEXT: Successor(s): bb2
+; CHECK-NEXT: EMIT branch-on-cond ir<%u1>
+; CHECK-NEXT: Successor(s): bb2, bb3
; CHECK-EMPTY:
; CHECK-NEXT: bb2:
-; CHECK-NEXT: EMIT vp<[[VP5:%[0-9]+]]> = logical-and vp<[[VP4]]>, ir<%u1>
-; CHECK-NEXT: EMIT ir<%add2> = add ir<%iv>, ir<2>, vp<[[VP5]]>
+; CHECK-NEXT: EMIT ir<%add2> = add ir<%iv>, ir<2>, vp<[[VP4]]>
; CHECK-NEXT: Successor(s): bb3
; CHECK-EMPTY:
; CHECK-NEXT: bb3:
-; CHECK-NEXT: BLEND ir<%phi3> = ir<%add1>/ir<true> ir<%add2>/ir<%u1>
+; CHECK-NEXT: EMIT-SCALAR vp<[[VP5:%[0-9]+]]> = phi [ ir<poison>, bb1 ], [ ir<%add2>, bb2 ]
+; CHECK-NEXT: BLEND ir<%phi3> = ir<%add1>/vp<[[VP4]]> vp<%5>/vp<[[VP4]]>
; CHECK-NEXT: EMIT ir<%add3> = add ir<%phi3>, ir<3>, vp<[[VP4]]>
; CHECK-NEXT: Successor(s): bb4
; CHECK-EMPTY:
@@ -1476,21 +1474,17 @@ define void @uniform_branch_after_varying_branch_more_blocks(ptr %a, i1 %u1) {
; CHECK-EMPTY:
; CHECK-NEXT: bb1:
; CHECK-NEXT: EMIT vp<[[VP4:%[0-9]+]]> = not ir<%v0>
-; CHECK-NEXT: Successor(s): bb2
-; CHECK-EMPTY:
-; CHECK-NEXT: bb2:
-; CHECK-NEXT: EMIT vp<[[VP5:%[0-9]+]]> = not ir<%u1>
-; CHECK-NEXT: EMIT vp<[[VP6:%[0-9]+]]> = logical-and vp<[[VP4]]>, vp<[[VP5]]>
-; CHECK-NEXT: EMIT ir<%add2> = add ir<%iv>, ir<2>, vp<[[VP6]]>
-; CHECK-NEXT: Successor(s): bb3
+; CHECK-NEXT: EMIT branch-on-cond ir<%u1>
+; CHECK-NEXT: Successor(s): bb3, bb2
; CHECK-EMPTY:
; CHECK-NEXT: bb3:
-; CHECK-NEXT: EMIT vp<[[VP7:%[0-9]+]]> = logical-and vp<[[VP4]]>, ir<%u1>
-; CHECK-NEXT: EMIT ir<%add3> = add ir<%iv>, ir<3>, vp<[[VP7]]>
+; CHECK-NEXT: EMIT ir<%add3> = add ir<%iv>, ir<3>, vp<[[VP4]]>
; CHECK-NEXT: Successor(s): bb4
; CHECK-EMPTY:
; CHECK-NEXT: bb4:
-; CHECK-NEXT: BLEND ir<%phi4> = ir<%add2>/vp<[[VP5]]> ir<%add3>/ir<%u1>
+; CHECK-NEXT: EMIT-SCALAR vp<[[VP5:%[0-9]+]]> = phi [ ir<%add2>, bb2 ], [ ir<poison>, bb3 ]
+; CHECK-NEXT: EMIT-SCALAR vp<[[VP6:%[0-9]+]]> = phi [ ir<poison>, bb2 ], [ ir<%add3>, bb3 ]
+; CHECK-NEXT: BLEND ir<%phi4> = vp<%5>/vp<[[VP4]]> vp<%6>/vp<[[VP4]]>
; CHECK-NEXT: EMIT ir<%add4> = add ir<%phi4>, ir<4>, vp<[[VP4]]>
; CHECK-NEXT: Successor(s): bb5
; CHECK-EMPTY:
@@ -1506,6 +1500,10 @@ define void @uniform_branch_after_varying_branch_more_blocks(ptr %a, i1 %u1) {
; CHECK-NEXT: EMIT vp<%index.next> = add nuw vp<[[VP3]]>, vp<[[VP1:%[0-9]+]]>
; CHECK-NEXT: EMIT branch-on-count vp<%index.next>, vp<[[VP2:%[0-9]+]]>
; CHECK-NEXT: No successors
+; CHECK-EMPTY:
+; CHECK-NEXT: bb2:
+; CHECK-NEXT: EMIT ir<%add2> = add ir<%iv>, ir<2>, vp<[[VP4]]>
+; CHECK-NEXT: Successor(s): bb4
; CHECK-NEXT: }
; CHECK-NEXT: Successor(s): middle.block
;
@@ -1668,21 +1666,17 @@ define void @uniform_branch_after_varying_branch_more_blocks_mirrored(ptr %a, i1
; CHECK-NEXT: Successor(s): bb2
; CHECK-EMPTY:
; CHECK-NEXT: bb2:
-; CHECK-NEXT: Successor(s): bb3
-; CHECK-EMPTY:
-; CHECK-NEXT: bb3:
-; CHECK-NEXT: EMIT vp<[[VP5:%[0-9]+]]> = not ir<%u2>
-; CHECK-NEXT: EMIT vp<[[VP6:%[0-9]+]]> = logical-and ir<%v0>, vp<[[VP5]]>
-; CHECK-NEXT: EMIT ir<%add3> = add ir<%iv>, ir<3>, vp<[[VP6]]>
-; CHECK-NEXT: Successor(s): bb4
+; CHECK-NEXT: EMIT branch-on-cond ir<%u2>
+; CHECK-NEXT: Successor(s): bb4, bb3
; CHECK-EMPTY:
; CHECK-NEXT: bb4:
-; CHECK-NEXT: EMIT vp<[[VP7:%[0-9]+]]> = logical-and ir<%v0>, ir<%u2>
-; CHECK-NEXT: EMIT ir<%add4> = add ir<%iv>, ir<4>, vp<[[VP7]]>
+; CHECK-NEXT: EMIT ir<%add4> = add ir<%iv>, ir<4>, ir<%v0>
; CHECK-NEXT: Successor(s): bb5
; CHECK-EMPTY:
; CHECK-NEXT: bb5:
-; CHECK-NEXT: BLEND ir<%phi5> = ir<%add3>/vp<[[VP5]]> ir<%add4>/ir<%u2>
+; CHECK-NEXT: EMIT-SCALAR vp<[[VP5:%[0-9]+]]> = phi [ ir<%add3>, bb3 ], [ ir<poison>, bb4 ]
+; CHECK-NEXT: EMIT-SCALAR vp<[[VP6:%[0-9]+]]> = phi [ ir<poison>, bb3 ], [ ir<%add4>, bb4 ]
+; CHECK-NEXT: BLEND ir<%phi5> = vp<%5>/ir<%v0> vp<%6>/ir<%v0>
; CHECK-NEXT: EMIT ir<%add5> = add ir<%phi5>, ir<5>, ir<%v0>
; CHECK-NEXT: Successor(s): bb6
; CHECK-EMPTY:
@@ -1694,6 +1688,10 @@ define void @uniform_branch_after_varying_branch_more_blocks_mirrored(ptr %a, i1
; CHECK-NEXT: EMIT vp<%index.next> = add nuw vp<[[VP3]]>, vp<[[VP1:%[0-9]+]]>
; CHECK-NEXT: EMIT branch-on-count vp<%index.next>, vp<[[VP2:%[0-9]+]]>
; CHECK-NEXT: No successors
+; CHECK-EMPTY:
+; CHECK-NEXT: bb3:
+; CHECK-NEXT: EMIT ir<%add3> = add ir<%iv>, ir<3>, ir<%v0>
+; CHECK-NEXT: Successor(s): bb5
; CHECK-NEXT: }
; CHECK-NEXT: Successor(s): middle.block
;
@@ -2731,31 +2729,34 @@ define void @uniform_branch_shared_join_with_varying(ptr %a, i1 %u0) {
; CHECK-EMPTY:
; CHECK-NEXT: vector.body:
; CHECK-NEXT: ir<%iv> = WIDEN-INDUCTION nuw nsw ir<0>, ir<1>, vp<[[VP0:%[0-9]+]]>
-; CHECK-NEXT: Successor(s): bb1
-; CHECK-EMPTY:
-; CHECK-NEXT: bb1:
-; CHECK-NEXT: EMIT vp<[[VP4:%[0-9]+]]> = not ir<%u0>
-; CHECK-NEXT: EMIT ir<%add1> = add ir<%iv>, ir<1>, vp<[[VP4]]>
-; CHECK-NEXT: Successor(s): bb2
+; CHECK-NEXT: EMIT branch-on-cond ir<%u0>
+; CHECK-NEXT: Successor(s): bb2, bb1
; CHECK-EMPTY:
; CHECK-NEXT: bb2:
-; CHECK-NEXT: EMIT ir<%add2> = add ir<%iv>, ir<2>, ir<%u0>
-; CHECK-NEXT: EMIT ir<%v2> = icmp sle ir<%iv>, ir<2>, ir<%u0>
+; CHECK-NEXT: EMIT ir<%add2> = add ir<%iv>, ir<2>
+; CHECK-NEXT: EMIT ir<%v2> = icmp sle ir<%iv>, ir<2>
; CHECK-NEXT: Successor(s): bb3
; CHECK-EMPTY:
; CHECK-NEXT: bb3:
-; CHECK-NEXT: EMIT vp<[[VP5:%[0-9]+]]> = logical-and ir<%u0>, ir<%v2>
-; CHECK-NEXT: EMIT ir<%add3> = add ir<%iv>, ir<3>, vp<[[VP5]]>
+; CHECK-NEXT: EMIT ir<%add3> = add ir<%iv>, ir<3>, ir<%v2>
; CHECK-NEXT: Successor(s): bb4
; CHECK-EMPTY:
; CHECK-NEXT: bb4:
-; CHECK-NEXT: BLEND ir<%phi4> = ir<%add1>/vp<[[VP4]]> ir<%add2>/ir<%u0> ir<%add3>/vp<[[VP5]]>
+; CHECK-NEXT: EMIT-SCALAR vp<[[VP4:%[0-9]+]]> = phi [ ir<%add1>, bb1 ], [ ir<poison>, bb3 ]
+; CHECK-NEXT: EMIT-SCALAR vp<[[VP5:%[0-9]+]]> = phi [ ir<poison>, bb1 ], [ ir<%add2>, bb3 ]
+; CHECK-NEXT: EMIT-SCALAR vp<[[VP6:%[0-9]+]]> = phi [ ir<poison>, bb1 ], [ ir<%add3>, bb3 ]
+; CHECK-NEXT: EMIT-SCALAR vp<[[VP7:%[0-9]+]]> = phi [ ir<false>, bb1 ], [ ir<%v2>, bb3 ]
+; CHECK-NEXT: BLEND ir<%phi4> = vp<%4>/ir<true> vp<%5>/ir<true> vp<%6>/vp<[[VP7]]>
; CHECK-NEXT: EMIT store ir<%phi4>, ir<%a>
; CHECK-NEXT: EMIT ir<%iv.next> = add nuw nsw ir<%iv>, ir<1>
; CHECK-NEXT: EMIT ir<%ec> = icmp eq ir<%iv.next>, ir<128>
; CHECK-NEXT: EMIT vp<%index.next> = add nuw vp<[[VP3]]>, vp<[[VP1:%[0-9]+]]>
; CHECK-NEXT: EMIT branch-on-count vp<%index.next>, vp<[[VP2:%[0-9]+]]>
; CHECK-NEXT: No successors
+; CHECK-EMPTY:
+; CHECK-NEXT: bb1:
+; CHECK-NEXT: EMIT ir<%add1> = add ir<%iv>, ir<1>
+; CHECK-NEXT: Successor(s): bb4
; CHECK-NEXT: }
; CHECK-NEXT: Successor(s): middle.block
;
diff --git a/llvm/test/Transforms/LoopVectorize/X86/cost-conditional-branches.ll b/llvm/test/Transforms/LoopVectorize/X86/cost-conditional-branches.ll
index f0c94e0a78629..78bb964a3bb50 100644
--- a/llvm/test/Transforms/LoopVectorize/X86/cost-conditional-branches.ll
+++ b/llvm/test/Transforms/LoopVectorize/X86/cost-conditional-branches.ll
@@ -868,386 +868,23 @@ exit:
; Test case for https://github.com/llvm/llvm-project/issues/158660.
define i64 @test_predicated_udiv(i32 %d, i1 %c) #2 {
; CHECK-LABEL: @test_predicated_udiv(
-; CHECK-NEXT: iter.check:
-; CHECK-NEXT: br i1 false, label [[VEC_EPILOG_SCALAR_PH:%.*]], label [[VECTOR_MAIN_LOOP_ITER_CHECK:%.*]]
-; CHECK: vector.main.loop.iter.check:
-; CHECK-NEXT: br i1 false, label [[VEC_EPILOG_PH:%.*]], label [[VECTOR_PH:%.*]]
-; CHECK: vector.ph:
-; CHECK-NEXT: [[TMP0:%.*]] = xor i1 [[C:%.*]], true
-; CHECK-NEXT: br label [[VECTOR_BODY:%.*]]
-; CHECK: vector.body:
-; CHECK-NEXT: [[INDEX:%.*]] = phi i32 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[PRED_UDIV_CONTINUE62:%.*]] ]
-; CHECK-NEXT: [[VEC_IND:%.*]] = phi <32 x i32> [ <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15, i32 16, i32 17, i32 18, i32 19, i32 20, i32 21, i32 22, i32 23, i32 24, i32 25, i32 26, i32 27, i32 28, i32 29, i32 30, i32 31>, [[VECTOR_PH]] ], [ [[VEC_IND_NEXT:%.*]], [[PRED_UDIV_CONTINUE62]] ]
-; CHECK-NEXT: [[TMP1:%.*]] = call <32 x i32> @llvm.usub.sat.v32i32(<32 x i32> [[VEC_IND]], <32 x i32> splat (i32 1))
-; CHECK-NEXT: br i1 [[TMP0]], label [[PRED_UDIV_IF:%.*]], label [[PRED_UDIV_CONTINUE:%.*]]
-; CHECK: pred.udiv.if:
-; CHECK-NEXT: [[TMP2:%.*]] = extractelement <32 x i32> [[TMP1]], i64 0
-; CHECK-NEXT: [[TMP3:%.*]] = udiv i32 [[TMP2]], [[D:%.*]]
-; CHECK-NEXT: [[TMP4:%.*]] = insertelement <32 x i32> poison, i32 [[TMP3]], i64 0
-; CHECK-NEXT: br label [[PRED_UDIV_CONTINUE]]
-; CHECK: pred.udiv.continue:
-; CHECK-NEXT: [[TMP5:%.*]] = phi <32 x i32> [ poison, [[VECTOR_BODY]] ], [ [[TMP4]], [[PRED_UDIV_IF]] ]
-; CHECK-NEXT: br i1 [[TMP0]], label [[PRED_UDIV_IF1:%.*]], label [[PRED_UDIV_CONTINUE2:%.*]]
-; CHECK: pred.udiv.if1:
-; CHECK-NEXT: [[TMP6:%.*]] = extractelement <32 x i32> [[TMP1]], i64 1
-; CHECK-NEXT: [[TMP7:%.*]] = udiv i32 [[TMP6]], [[D]]
-; CHECK-NEXT: [[TMP8:%.*]] = insertelement <32 x i32> [[TMP5]], i32 [[TMP7]], i64 1
-; CHECK-NEXT: br label [[PRED_UDIV_CONTINUE2]]
-; CHECK: pred.udiv.continue2:
-; CHECK-NEXT: [[TMP9:%.*]] = phi <32 x i32> [ [[TMP5]], [[PRED_UDIV_CONTINUE]] ], [ [[TMP8]], [[PRED_UDIV_IF1]] ]
-; CHECK-NEXT: br i1 [[TMP0]], label [[PRED_UDIV_IF3:%.*]], label [[PRED_UDIV_CONTINUE4:%.*]]
-; CHECK: pred.udiv.if3:
-; CHECK-NEXT: [[TMP10:%.*]] = extractelement <32 x i32> [[TMP1]], i64 2
-; CHECK-NEXT: [[TMP11:%.*]] = udiv i32 [[TMP10]], [[D]]
-; CHECK-NEXT: [[TMP12:%.*]] = insertelement <32 x i32> [[TMP9]], i32 [[TMP11]], i64 2
-; CHECK-NEXT: br label [[PRED_UDIV_CONTINUE4]]
-; CHECK: pred.udiv.continue4:
-; CHECK-NEXT: [[TMP13:%.*]] = phi <32 x i32> [ [[TMP9]], [[PRED_UDIV_CONTINUE2]] ], [ [[TMP12]], [[PRED_UDIV_IF3]] ]
-; CHECK-NEXT: br i1 [[TMP0]], label [[PRED_UDIV_IF5:%.*]], label [[PRED_UDIV_CONTINUE6:%.*]]
-; CHECK: pred.udiv.if5:
-; CHECK-NEXT: [[TMP14:%.*]] = extractelement <32 x i32> [[TMP1]], i64 3
-; CHECK-NEXT: [[TMP15:%.*]] = udiv i32 [[TMP14]], [[D]]
-; CHECK-NEXT: [[TMP16:%.*]] = insertelement <32 x i32> [[TMP13]], i32 [[TMP15]], i64 3
-; CHECK-NEXT: br label [[PRED_UDIV_CONTINUE6]]
-; CHECK: pred.udiv.continue6:
-; CHECK-NEXT: [[TMP17:%.*]] = phi <32 x i32> [ [[TMP13]], [[PRED_UDIV_CONTINUE4]] ], [ [[TMP16]], [[PRED_UDIV_IF5]] ]
-; CHECK-NEXT: br i1 [[TMP0]], label [[PRED_UDIV_IF7:%.*]], label [[PRED_UDIV_CONTINUE8:%.*]]
-; CHECK: pred.udiv.if7:
-; CHECK-NEXT: [[TMP18:%.*]] = extractelement <32 x i32> [[TMP1]], i64 4
-; CHECK-NEXT: [[TMP19:%.*]] = udiv i32 [[TMP18]], [[D]]
-; CHECK-NEXT: [[TMP20:%.*]] = insertelement <32 x i32> [[TMP17]], i32 [[TMP19]], i64 4
-; CHECK-NEXT: br label [[PRED_UDIV_CONTINUE8]]
-; CHECK: pred.udiv.continue8:
-; CHECK-NEXT: [[TMP21:%.*]] = phi <32 x i32> [ [[TMP17]], [[PRED_UDIV_CONTINUE6]] ], [ [[TMP20]], [[PRED_UDIV_IF7]] ]
-; CHECK-NEXT: br i1 [[TMP0]], label [[PRED_UDIV_IF9:%.*]], label [[PRED_UDIV_CONTINUE10:%.*]]
-; CHECK: pred.udiv.if9:
-; CHECK-NEXT: [[TMP22:%.*]] = extractelement <32 x i32> [[TMP1]], i64 5
-; CHECK-NEXT: [[TMP23:%.*]] = udiv i32 [[TMP22]], [[D]]
-; CHECK-NEXT: [[TMP24:%.*]] = insertelement <32 x i32> [[TMP21]], i32 [[TMP23]], i64 5
-; CHECK-NEXT: br label [[PRED_UDIV_CONTINUE10]]
-; CHECK: pred.udiv.continue10:
-; CHECK-NEXT: [[TMP25:%.*]] = phi <32 x i32> [ [[TMP21]], [[PRED_UDIV_CONTINUE8]] ], [ [[TMP24]], [[PRED_UDIV_IF9]] ]
-; CHECK-NEXT: br i1 [[TMP0]], label [[PRED_UDIV_IF11:%.*]], label [[PRED_UDIV_CONTINUE12:%.*]]
-; CHECK: pred.udiv.if11:
-; CHECK-NEXT: [[TMP26:%.*]] = extractelement <32 x i32> [[TMP1]], i64 6
-; CHECK-NEXT: [[TMP27:%.*]] = udiv i32 [[TMP26]], [[D]]
-; CHECK-NEXT: [[TMP28:%.*]] = insertelement <32 x i32> [[TMP25]], i32 [[TMP27]], i64 6
-; CHECK-NEXT: br label [[PRED_UDIV_CONTINUE12]]
-; CHECK: pred.udiv.continue12:
-; CHECK-NEXT: [[TMP29:%.*]] = phi <32 x i32> [ [[TMP25]], [[PRED_UDIV_CONTINUE10]] ], [ [[TMP28]], [[PRED_UDIV_IF11]] ]
-; CHECK-NEXT: br i1 [[TMP0]], label [[PRED_UDIV_IF13:%.*]], label [[PRED_UDIV_CONTINUE14:%.*]]
-; CHECK: pred.udiv.if13:
-; CHECK-NEXT: [[TMP30:%.*]] = extractelement <32 x i32> [[TMP1]], i64 7
-; CHECK-NEXT: [[TMP31:%.*]] = udiv i32 [[TMP30]], [[D]]
-; CHECK-NEXT: [[TMP32:%.*]] = insertelement <32 x i32> [[TMP29]], i32 [[TMP31]], i64 7
-; CHECK-NEXT: br label [[PRED_UDIV_CONTINUE14]]
-; CHECK: pred.udiv.continue14:
-; CHECK-NEXT: [[TMP33:%.*]] = phi <32 x i32> [ [[TMP29]], [[PRED_UDIV_CONTINUE12]] ], [ [[TMP32]], [[PRED_UDIV_IF13]] ]
-; CHECK-NEXT: br i1 [[TMP0]], label [[PRED_UDIV_IF15:%.*]], label [[PRED_UDIV_CONTINUE16:%.*]]
-; CHECK: pred.udiv.if15:
-; CHECK-NEXT: [[TMP34:%.*]] = extractelement <32 x i32> [[TMP1]], i64 8
-; CHECK-NEXT: [[TMP35:%.*]] = udiv i32 [[TMP34]], [[D]]
-; CHECK-NEXT: [[TMP36:%.*]] = insertelement <32 x i32> [[TMP33]], i32 [[TMP35]], i64 8
-; CHECK-NEXT: br label [[PRED_UDIV_CONTINUE16]]
-; CHECK: pred.udiv.continue16:
-; CHECK-NEXT: [[TMP37:%.*]] = phi <32 x i32> [ [[TMP33]], [[PRED_UDIV_CONTINUE14]] ], [ [[TMP36]], [[PRED_UDIV_IF15]] ]
-; CHECK-NEXT: br i1 [[TMP0]], label [[PRED_UDIV_IF17:%.*]], label [[PRED_UDIV_CONTINUE18:%.*]]
-; CHECK: pred.udiv.if17:
-; CHECK-NEXT: [[TMP38:%.*]] = extractelement <32 x i32> [[TMP1]], i64 9
-; CHECK-NEXT: [[TMP39:%.*]] = udiv i32 [[TMP38]], [[D]]
-; CHECK-NEXT: [[TMP40:%.*]] = insertelement <32 x i32> [[TMP37]], i32 [[TMP39]], i64 9
-; CHECK-NEXT: br label [[PRED_UDIV_CONTINUE18]]
-; CHECK: pred.udiv.continue18:
-; CHECK-NEXT: [[TMP41:%.*]] = phi <32 x i32> [ [[TMP37]], [[PRED_UDIV_CONTINUE16]] ], [ [[TMP40]], [[PRED_UDIV_IF17]] ]
-; CHECK-NEXT: br i1 [[TMP0]], label [[PRED_UDIV_IF19:%.*]], label [[PRED_UDIV_CONTINUE20:%.*]]
-; CHECK: pred.udiv.if19:
-; CHECK-NEXT: [[TMP42:%.*]] = extractelement <32 x i32> [[TMP1]], i64 10
-; CHECK-NEXT: [[TMP43:%.*]] = udiv i32 [[TMP42]], [[D]]
-; CHECK-NEXT: [[TMP44:%.*]] = insertelement <32 x i32> [[TMP41]], i32 [[TMP43]], i64 10
-; CHECK-NEXT: br label [[PRED_UDIV_CONTINUE20]]
-; CHECK: pred.udiv.continue20:
-; CHECK-NEXT: [[TMP45:%.*]] = phi <32 x i32> [ [[TMP41]], [[PRED_UDIV_CONTINUE18]] ], [ [[TMP44]], [[PRED_UDIV_IF19]] ]
-; CHECK-NEXT: br i1 [[TMP0]], label [[PRED_UDIV_IF21:%.*]], label [[PRED_UDIV_CONTINUE22:%.*]]
-; CHECK: pred.udiv.if21:
-; CHECK-NEXT: [[TMP46:%.*]] = extractelement <32 x i32> [[TMP1]], i64 11
-; CHECK-NEXT: [[TMP47:%.*]] = udiv i32 [[TMP46]], [[D]]
-; CHECK-NEXT: [[TMP48:%.*]] = insertelement <32 x i32> [[TMP45]], i32 [[TMP47]], i64 11
-; CHECK-NEXT: br label [[PRED_UDIV_CONTINUE22]]
-; CHECK: pred.udiv.continue22:
-; CHECK-NEXT: [[TMP49:%.*]] = phi <32 x i32> [ [[TMP45]], [[PRED_UDIV_CONTINUE20]] ], [ [[TMP48]], [[PRED_UDIV_IF21]] ]
-; CHECK-NEXT: br i1 [[TMP0]], label [[PRED_UDIV_IF23:%.*]], label [[PRED_UDIV_CONTINUE24:%.*]]
-; CHECK: pred.udiv.if23:
-; CHECK-NEXT: [[TMP50:%.*]] = extractelement <32 x i32> [[TMP1]], i64 12
-; CHECK-NEXT: [[TMP51:%.*]] = udiv i32 [[TMP50]], [[D]]
-; CHECK-NEXT: [[TMP52:%.*]] = insertelement <32 x i32> [[TMP49]], i32 [[TMP51]], i64 12
-; CHECK-NEXT: br label [[PRED_UDIV_CONTINUE24]]
-; CHECK: pred.udiv.continue24:
-; CHECK-NEXT: [[TMP53:%.*]] = phi <32 x i32> [ [[TMP49]], [[PRED_UDIV_CONTINUE22]] ], [ [[TMP52]], [[PRED_UDIV_IF23]] ]
-; CHECK-NEXT: br i1 [[TMP0]], label [[PRED_UDIV_IF25:%.*]], label [[PRED_UDIV_CONTINUE26:%.*]]
-; CHECK: pred.udiv.if25:
-; CHECK-NEXT: [[TMP54:%.*]] = extractelement <32 x i32> [[TMP1]], i64 13
-; CHECK-NEXT: [[TMP55:%.*]] = udiv i32 [[TMP54]], [[D]]
-; CHECK-NEXT: [[TMP56:%.*]] = insertelement <32 x i32> [[TMP53]], i32 [[TMP55]], i64 13
-; CHECK-NEXT: br label [[PRED_UDIV_CONTINUE26]]
-; CHECK: pred.udiv.continue26:
-; CHECK-NEXT: [[TMP57:%.*]] = phi <32 x i32> [ [[TMP53]], [[PRED_UDIV_CONTINUE24]] ], [ [[TMP56]], [[PRED_UDIV_IF25]] ]
-; CHECK-NEXT: br i1 [[TMP0]], label [[PRED_UDIV_IF27:%.*]], label [[PRED_UDIV_CONTINUE28:%.*]]
-; CHECK: pred.udiv.if27:
-; CHECK-NEXT: [[TMP58:%.*]] = extractelement <32 x i32> [[TMP1]], i64 14
-; CHECK-NEXT: [[TMP59:%.*]] = udiv i32 [[TMP58]], [[D]]
-; CHECK-NEXT: [[TMP60:%.*]] = insertelement <32 x i32> [[TMP57]], i32 [[TMP59]], i64 14
-; CHECK-NEXT: br label [[PRED_UDIV_CONTINUE28]]
-; CHECK: pred.udiv.continue28:
-; CHECK-NEXT: [[TMP61:%.*]] = phi <32 x i32> [ [[TMP57]], [[PRED_UDIV_CONTINUE26]] ], [ [[TMP60]], [[PRED_UDIV_IF27]] ]
-; CHECK-NEXT: br i1 [[TMP0]], label [[PRED_UDIV_IF29:%.*]], label [[PRED_UDIV_CONTINUE30:%.*]]
-; CHECK: pred.udiv.if29:
-; CHECK-NEXT: [[TMP62:%.*]] = extractelement <32 x i32> [[TMP1]], i64 15
-; CHECK-NEXT: [[TMP63:%.*]] = udiv i32 [[TMP62]], [[D]]
-; CHECK-NEXT: [[TMP64:%.*]] = insertelement <32 x i32> [[TMP61]], i32 [[TMP63]], i64 15
-; CHECK-NEXT: br label [[PRED_UDIV_CONTINUE30]]
-; CHECK: pred.udiv.continue30:
-; CHECK-NEXT: [[TMP65:%.*]] = phi <32 x i32> [ [[TMP61]], [[PRED_UDIV_CONTINUE28]] ], [ [[TMP64]], [[PRED_UDIV_IF29]] ]
-; CHECK-NEXT: br i1 [[TMP0]], label [[PRED_UDIV_IF31:%.*]], label [[PRED_UDIV_CONTINUE32:%.*]]
-; CHECK: pred.udiv.if31:
-; CHECK-NEXT: [[TMP66:%.*]] = extractelement <32 x i32> [[TMP1]], i64 16
-; CHECK-NEXT: [[TMP67:%.*]] = udiv i32 [[TMP66]], [[D]]
-; CHECK-NEXT: [[TMP68:%.*]] = insertelement <32 x i32> [[TMP65]], i32 [[TMP67]], i64 16
-; CHECK-NEXT: br label [[PRED_UDIV_CONTINUE32]]
-; CHECK: pred.udiv.continue32:
-; CHECK-NEXT: [[TMP69:%.*]] = phi <32 x i32> [ [[TMP65]], [[PRED_UDIV_CONTINUE30]] ], [ [[TMP68]], [[PRED_UDIV_IF31]] ]
-; CHECK-NEXT: br i1 [[TMP0]], label [[PRED_UDIV_IF33:%.*]], label [[PRED_UDIV_CONTINUE34:%.*]]
-; CHECK: pred.udiv.if33:
-; CHECK-NEXT: [[TMP70:%.*]] = extractelement <32 x i32> [[TMP1]], i64 17
-; CHECK-NEXT: [[TMP71:%.*]] = udiv i32 [[TMP70]], [[D]]
-; CHECK-NEXT: [[TMP72:%.*]] = insertelement <32 x i32> [[TMP69]], i32 [[TMP71]], i64 17
-; CHECK-NEXT: br label [[PRED_UDIV_CONTINUE34]]
-; CHECK: pred.udiv.continue34:
-; CHECK-NEXT: [[TMP73:%.*]] = phi <32 x i32> [ [[TMP69]], [[PRED_UDIV_CONTINUE32]] ], [ [[TMP72]], [[PRED_UDIV_IF33]] ]
-; CHECK-NEXT: br i1 [[TMP0]], label [[PRED_UDIV_IF35:%.*]], label [[PRED_UDIV_CONTINUE36:%.*]]
-; CHECK: pred.udiv.if35:
-; CHECK-NEXT: [[TMP74:%.*]] = extractelement <32 x i32> [[TMP1]], i64 18
-; CHECK-NEXT: [[TMP75:%.*]] = udiv i32 [[TMP74]], [[D]]
-; CHECK-NEXT: [[TMP76:%.*]] = insertelement <32 x i32> [[TMP73]], i32 [[TMP75]], i64 18
-; CHECK-NEXT: br label [[PRED_UDIV_CONTINUE36]]
-; CHECK: pred.udiv.continue36:
-; CHECK-NEXT: [[TMP77:%.*]] = phi <32 x i32> [ [[TMP73]], [[PRED_UDIV_CONTINUE34]] ], [ [[TMP76]], [[PRED_UDIV_IF35]] ]
-; CHECK-NEXT: br i1 [[TMP0]], label [[PRED_UDIV_IF37:%.*]], label [[PRED_UDIV_CONTINUE38:%.*]]
-; CHECK: pred.udiv.if37:
-; CHECK-NEXT: [[TMP78:%.*]] = extractelement <32 x i32> [[TMP1]], i64 19
-; CHECK-NEXT: [[TMP79:%.*]] = udiv i32 [[TMP78]], [[D]]
-; CHECK-NEXT: [[TMP80:%.*]] = insertelement <32 x i32> [[TMP77]], i32 [[TMP79]], i64 19
-; CHECK-NEXT: br label [[PRED_UDIV_CONTINUE38]]
-; CHECK: pred.udiv.continue38:
-; CHECK-NEXT: [[TMP81:%.*]] = phi <32 x i32> [ [[TMP77]], [[PRED_UDIV_CONTINUE36]] ], [ [[TMP80]], [[PRED_UDIV_IF37]] ]
-; CHECK-NEXT: br i1 [[TMP0]], label [[PRED_UDIV_IF39:%.*]], label [[PRED_UDIV_CONTINUE40:%.*]]
-; CHECK: pred.udiv.if39:
-; CHECK-NEXT: [[TMP82:%.*]] = extractelement <32 x i32> [[TMP1]], i64 20
-; CHECK-NEXT: [[TMP83:%.*]] = udiv i32 [[TMP82]], [[D]]
-; CHECK-NEXT: [[TMP84:%.*]] = insertelement <32 x i32> [[TMP81]], i32 [[TMP83]], i64 20
-; CHECK-NEXT: br label [[PRED_UDIV_CONTINUE40]]
-; CHECK: pred.udiv.continue40:
-; CHECK-NEXT: [[TMP85:%.*]] = phi <32 x i32> [ [[TMP81]], [[PRED_UDIV_CONTINUE38]] ], [ [[TMP84]], [[PRED_UDIV_IF39]] ]
-; CHECK-NEXT: br i1 [[TMP0]], label [[PRED_UDIV_IF41:%.*]], label [[PRED_UDIV_CONTINUE42:%.*]]
-; CHECK: pred.udiv.if41:
-; CHECK-NEXT: [[TMP86:%.*]] = extractelement <32 x i32> [[TMP1]], i64 21
-; CHECK-NEXT: [[TMP87:%.*]] = udiv i32 [[TMP86]], [[D]]
-; CHECK-NEXT: [[TMP88:%.*]] = insertelement <32 x i32> [[TMP85]], i32 [[TMP87]], i64 21
-; CHECK-NEXT: br label [[PRED_UDIV_CONTINUE42]]
-; CHECK: pred.udiv.continue42:
-; CHECK-NEXT: [[TMP89:%.*]] = phi <32 x i32> [ [[TMP85]], [[PRED_UDIV_CONTINUE40]] ], [ [[TMP88]], [[PRED_UDIV_IF41]] ]
-; CHECK-NEXT: br i1 [[TMP0]], label [[PRED_UDIV_IF43:%.*]], label [[PRED_UDIV_CONTINUE44:%.*]]
-; CHECK: pred.udiv.if43:
-; CHECK-NEXT: [[TMP90:%.*]] = extractelement <32 x i32> [[TMP1]], i64 22
-; CHECK-NEXT: [[TMP91:%.*]] = udiv i32 [[TMP90]], [[D]]
-; CHECK-NEXT: [[TMP92:%.*]] = insertelement <32 x i32> [[TMP89]], i32 [[TMP91]], i64 22
-; CHECK-NEXT: br label [[PRED_UDIV_CONTINUE44]]
-; CHECK: pred.udiv.continue44:
-; CHECK-NEXT: [[TMP93:%.*]] = phi <32 x i32> [ [[TMP89]], [[PRED_UDIV_CONTINUE42]] ], [ [[TMP92]], [[PRED_UDIV_IF43]] ]
-; CHECK-NEXT: br i1 [[TMP0]], label [[PRED_UDIV_IF45:%.*]], label [[PRED_UDIV_CONTINUE46:%.*]]
-; CHECK: pred.udiv.if45:
-; CHECK-NEXT: [[TMP94:%.*]] = extractelement <32 x i32> [[TMP1]], i64 23
-; CHECK-NEXT: [[TMP95:%.*]] = udiv i32 [[TMP94]], [[D]]
-; CHECK-NEXT: [[TMP96:%.*]] = insertelement <32 x i32> [[TMP93]], i32 [[TMP95]], i64 23
-; CHECK-NEXT: br label [[PRED_UDIV_CONTINUE46]]
-; CHECK: pred.udiv.continue46:
-; CHECK-NEXT: [[TMP97:%.*]] = phi <32 x i32> [ [[TMP93]], [[PRED_UDIV_CONTINUE44]] ], [ [[TMP96]], [[PRED_UDIV_IF45]] ]
-; CHECK-NEXT: br i1 [[TMP0]], label [[PRED_UDIV_IF47:%.*]], label [[PRED_UDIV_CONTINUE48:%.*]]
-; CHECK: pred.udiv.if47:
-; CHECK-NEXT: [[TMP98:%.*]] = extractelement <32 x i32> [[TMP1]], i64 24
-; CHECK-NEXT: [[TMP99:%.*]] = udiv i32 [[TMP98]], [[D]]
-; CHECK-NEXT: [[TMP100:%.*]] = insertelement <32 x i32> [[TMP97]], i32 [[TMP99]], i64 24
-; CHECK-NEXT: br label [[PRED_UDIV_CONTINUE48]]
-; CHECK: pred.udiv.continue48:
-; CHECK-NEXT: [[TMP101:%.*]] = phi <32 x i32> [ [[TMP97]], [[PRED_UDIV_CONTINUE46]] ], [ [[TMP100]], [[PRED_UDIV_IF47]] ]
-; CHECK-NEXT: br i1 [[TMP0]], label [[PRED_UDIV_IF49:%.*]], label [[PRED_UDIV_CONTINUE50:%.*]]
-; CHECK: pred.udiv.if49:
-; CHECK-NEXT: [[TMP102:%.*]] = extractelement <32 x i32> [[TMP1]], i64 25
-; CHECK-NEXT: [[TMP103:%.*]] = udiv i32 [[TMP102]], [[D]]
-; CHECK-NEXT: [[TMP104:%.*]] = insertelement <32 x i32> [[TMP101]], i32 [[TMP103]], i64 25
-; CHECK-NEXT: br label [[PRED_UDIV_CONTINUE50]]
-; CHECK: pred.udiv.continue50:
-; CHECK-NEXT: [[TMP105:%.*]] = phi <32 x i32> [ [[TMP101]], [[PRED_UDIV_CONTINUE48]] ], [ [[TMP104]], [[PRED_UDIV_IF49]] ]
-; CHECK-NEXT: br i1 [[TMP0]], label [[PRED_UDIV_IF51:%.*]], label [[PRED_UDIV_CONTINUE52:%.*]]
-; CHECK: pred.udiv.if51:
-; CHECK-NEXT: [[TMP106:%.*]] = extractelement <32 x i32> [[TMP1]], i64 26
-; CHECK-NEXT: [[TMP107:%.*]] = udiv i32 [[TMP106]], [[D]]
-; CHECK-NEXT: [[TMP108:%.*]] = insertelement <32 x i32> [[TMP105]], i32 [[TMP107]], i64 26
-; CHECK-NEXT: br label [[PRED_UDIV_CONTINUE52]]
-; CHECK: pred.udiv.continue52:
-; CHECK-NEXT: [[TMP109:%.*]] = phi <32 x i32> [ [[TMP105]], [[PRED_UDIV_CONTINUE50]] ], [ [[TMP108]], [[PRED_UDIV_IF51]] ]
-; CHECK-NEXT: br i1 [[TMP0]], label [[PRED_UDIV_IF53:%.*]], label [[PRED_UDIV_CONTINUE54:%.*]]
-; CHECK: pred.udiv.if53:
-; CHECK-NEXT: [[TMP110:%.*]] = extractelement <32 x i32> [[TMP1]], i64 27
-; CHECK-NEXT: [[TMP111:%.*]] = udiv i32 [[TMP110]], [[D]]
-; CHECK-NEXT: [[TMP112:%.*]] = insertelement <32 x i32> [[TMP109]], i32 [[TMP111]], i64 27
-; CHECK-NEXT: br label [[PRED_UDIV_CONTINUE54]]
-; CHECK: pred.udiv.continue54:
-; CHECK-NEXT: [[TMP113:%.*]] = phi <32 x i32> [ [[TMP109]], [[PRED_UDIV_CONTINUE52]] ], [ [[TMP112]], [[PRED_UDIV_IF53]] ]
-; CHECK-NEXT: br i1 [[TMP0]], label [[PRED_UDIV_IF55:%.*]], label [[PRED_UDIV_CONTINUE56:%.*]]
-; CHECK: pred.udiv.if55:
-; CHECK-NEXT: [[TMP114:%.*]] = extractelement <32 x i32> [[TMP1]], i64 28
-; CHECK-NEXT: [[TMP115:%.*]] = udiv i32 [[TMP114]], [[D]]
-; CHECK-NEXT: [[TMP116:%.*]] = insertelement <32 x i32> [[TMP113]], i32 [[TMP115]], i64 28
-; CHECK-NEXT: br label [[PRED_UDIV_CONTINUE56]]
-; CHECK: pred.udiv.continue56:
-; CHECK-NEXT: [[TMP117:%.*]] = phi <32 x i32> [ [[TMP113]], [[PRED_UDIV_CONTINUE54]] ], [ [[TMP116]], [[PRED_UDIV_IF55]] ]
-; CHECK-NEXT: br i1 [[TMP0]], label [[PRED_UDIV_IF57:%.*]], label [[PRED_UDIV_CONTINUE58:%.*]]
-; CHECK: pred.udiv.if57:
-; CHECK-NEXT: [[TMP118:%.*]] = extractelement <32 x i32> [[TMP1]], i64 29
-; CHECK-NEXT: [[TMP119:%.*]] = udiv i32 [[TMP118]], [[D]]
-; CHECK-NEXT: [[TMP120:%.*]] = insertelement <32 x i32> [[TMP117]], i32 [[TMP119]], i64 29
-; CHECK-NEXT: br label [[PRED_UDIV_CONTINUE58]]
-; CHECK: pred.udiv.continue58:
-; CHECK-NEXT: [[TMP121:%.*]] = phi <32 x i32> [ [[TMP117]], [[PRED_UDIV_CONTINUE56]] ], [ [[TMP120]], [[PRED_UDIV_IF57]] ]
-; CHECK-NEXT: br i1 [[TMP0]], label [[PRED_UDIV_IF59:%.*]], label [[PRED_UDIV_CONTINUE60:%.*]]
-; CHECK: pred.udiv.if59:
-; CHECK-NEXT: [[TMP122:%.*]] = extractelement <32 x i32> [[TMP1]], i64 30
-; CHECK-NEXT: [[TMP123:%.*]] = udiv i32 [[TMP122]], [[D]]
-; CHECK-NEXT: [[TMP124:%.*]] = insertelement <32 x i32> [[TMP121]], i32 [[TMP123]], i64 30
-; CHECK-NEXT: br label [[PRED_UDIV_CONTINUE60]]
-; CHECK: pred.udiv.continue60:
-; CHECK-NEXT: [[TMP125:%.*]] = phi <32 x i32> [ [[TMP121]], [[PRED_UDIV_CONTINUE58]] ], [ [[TMP124]], [[PRED_UDIV_IF59]] ]
-; CHECK-NEXT: br i1 [[TMP0]], label [[PRED_UDIV_IF61:%.*]], label [[PRED_UDIV_CONTINUE62]]
-; CHECK: pred.udiv.if61:
-; CHECK-NEXT: [[TMP126:%.*]] = extractelement <32 x i32> [[TMP1]], i64 31
-; CHECK-NEXT: [[TMP127:%.*]] = udiv i32 [[TMP126]], [[D]]
-; CHECK-NEXT: [[TMP128:%.*]] = insertelement <32 x i32> [[TMP125]], i32 [[TMP127]], i64 31
-; CHECK-NEXT: br label [[PRED_UDIV_CONTINUE62]]
-; CHECK: pred.udiv.continue62:
-; CHECK-NEXT: [[TMP129:%.*]] = phi <32 x i32> [ [[TMP125]], [[PRED_UDIV_CONTINUE60]] ], [ [[TMP128]], [[PRED_UDIV_IF61]] ]
-; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i32 [[INDEX]], 32
-; CHECK-NEXT: [[VEC_IND_NEXT]] = add <32 x i32> [[VEC_IND]], splat (i32 32)
-; CHECK-NEXT: [[TMP130:%.*]] = icmp eq i32 [[INDEX_NEXT]], 992
-; CHECK-NEXT: br i1 [[TMP130]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP12:![0-9]+]]
-; CHECK: middle.block:
-; CHECK-NEXT: [[TMP131:%.*]] = zext <32 x i32> [[TMP129]] to <32 x i64>
-; CHECK-NEXT: [[PREDPHI:%.*]] = select i1 [[C]], <32 x i64> zeroinitializer, <32 x i64> [[TMP131]]
-; CHECK-NEXT: [[TMP132:%.*]] = extractelement <32 x i64> [[PREDPHI]], i64 31
-; CHECK-NEXT: br i1 false, label [[EXIT:%.*]], label [[VEC_EPILOG_ITER_CHECK:%.*]]
-; CHECK: vec.epilog.iter.check:
-; CHECK-NEXT: br i1 false, label [[VEC_EPILOG_SCALAR_PH]], label [[VEC_EPILOG_PH]], !prof [[PROF13:![0-9]+]]
-; CHECK: vec.epilog.ph:
-; CHECK-NEXT: [[VEC_EPILOG_RESUME_VAL:%.*]] = phi i32 [ 992, [[VEC_EPILOG_ITER_CHECK]] ], [ 0, [[VECTOR_MAIN_LOOP_ITER_CHECK]] ]
-; CHECK-NEXT: [[TMP133:%.*]] = xor i1 [[C]], true
-; CHECK-NEXT: [[BROADCAST_SPLATINSERT:%.*]] = insertelement <8 x i32> poison, i32 [[VEC_EPILOG_RESUME_VAL]], i64 0
-; CHECK-NEXT: [[BROADCAST_SPLAT:%.*]] = shufflevector <8 x i32> [[BROADCAST_SPLATINSERT]], <8 x i32> poison, <8 x i32> zeroinitializer
-; CHECK-NEXT: [[INDUCTION:%.*]] = add <8 x i32> [[BROADCAST_SPLAT]], <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>
-; CHECK-NEXT: br label [[VEC_EPILOG_VECTOR_BODY:%.*]]
-; CHECK: vec.epilog.vector.body:
-; CHECK-NEXT: [[INDEX63:%.*]] = phi i32 [ [[VEC_EPILOG_RESUME_VAL]], [[VEC_EPILOG_PH]] ], [ [[INDEX_NEXT81:%.*]], [[PRED_UDIV_CONTINUE80:%.*]] ]
-; CHECK-NEXT: [[VEC_IND64:%.*]] = phi <8 x i32> [ [[INDUCTION]], [[VEC_EPILOG_PH]] ], [ [[VEC_IND_NEXT82:%.*]], [[PRED_UDIV_CONTINUE80]] ]
-; CHECK-NEXT: [[TMP134:%.*]] = call <8 x i32> @llvm.usub.sat.v8i32(<8 x i32> [[VEC_IND64]], <8 x i32> splat (i32 1))
-; CHECK-NEXT: br i1 [[TMP133]], label [[PRED_UDIV_IF65:%.*]], label [[PRED_UDIV_CONTINUE66:%.*]]
-; CHECK: pred.udiv.if65:
-; CHECK-NEXT: [[TMP135:%.*]] = extractelement <8 x i32> [[TMP134]], i64 0
-; CHECK-NEXT: [[TMP136:%.*]] = udiv i32 [[TMP135]], [[D]]
-; CHECK-NEXT: [[TMP137:%.*]] = insertelement <8 x i32> poison, i32 [[TMP136]], i64 0
-; CHECK-NEXT: br label [[PRED_UDIV_CONTINUE66]]
-; CHECK: pred.udiv.continue66:
-; CHECK-NEXT: [[TMP138:%.*]] = phi <8 x i32> [ poison, [[VEC_EPILOG_VECTOR_BODY]] ], [ [[TMP137]], [[PRED_UDIV_IF65]] ]
-; CHECK-NEXT: br i1 [[TMP133]], label [[PRED_UDIV_IF67:%.*]], label [[PRED_UDIV_CONTINUE68:%.*]]
-; CHECK: pred.udiv.if67:
-; CHECK-NEXT: [[TMP139:%.*]] = extractelement <8 x i32> [[TMP134]], i64 1
-; CHECK-NEXT: [[TMP140:%.*]] = udiv i32 [[TMP139]], [[D]]
-; CHECK-NEXT: [[TMP141:%.*]] = insertelement <8 x i32> [[TMP138]], i32 [[TMP140]], i64 1
-; CHECK-NEXT: br label [[PRED_UDIV_CONTINUE68]]
-; CHECK: pred.udiv.continue68:
-; CHECK-NEXT: [[TMP142:%.*]] = phi <8 x i32> [ [[TMP138]], [[PRED_UDIV_CONTINUE66]] ], [ [[TMP141]], [[PRED_UDIV_IF67]] ]
-; CHECK-NEXT: br i1 [[TMP133]], label [[PRED_UDIV_IF69:%.*]], label [[PRED_UDIV_CONTINUE70:%.*]]
-; CHECK: pred.udiv.if69:
-; CHECK-NEXT: [[TMP143:%.*]] = extractelement <8 x i32> [[TMP134]], i64 2
-; CHECK-NEXT: [[TMP144:%.*]] = udiv i32 [[TMP143]], [[D]]
-; CHECK-NEXT: [[TMP145:%.*]] = insertelement <8 x i32> [[TMP142]], i32 [[TMP144]], i64 2
-; CHECK-NEXT: br label [[PRED_UDIV_CONTINUE70]]
-; CHECK: pred.udiv.continue70:
-; CHECK-NEXT: [[TMP146:%.*]] = phi <8 x i32> [ [[TMP142]], [[PRED_UDIV_CONTINUE68]] ], [ [[TMP145]], [[PRED_UDIV_IF69]] ]
-; CHECK-NEXT: br i1 [[TMP133]], label [[PRED_UDIV_IF71:%.*]], label [[PRED_UDIV_CONTINUE72:%.*]]
-; CHECK: pred.udiv.if71:
-; CHECK-NEXT: [[TMP147:%.*]] = extractelement <8 x i32> [[TMP134]], i64 3
-; CHECK-NEXT: [[TMP148:%.*]] = udiv i32 [[TMP147]], [[D]]
-; CHECK-NEXT: [[TMP149:%.*]] = insertelement <8 x i32> [[TMP146]], i32 [[TMP148]], i64 3
-; CHECK-NEXT: br label [[PRED_UDIV_CONTINUE72]]
-; CHECK: pred.udiv.continue72:
-; CHECK-NEXT: [[TMP150:%.*]] = phi <8 x i32> [ [[TMP146]], [[PRED_UDIV_CONTINUE70]] ], [ [[TMP149]], [[PRED_UDIV_IF71]] ]
-; CHECK-NEXT: br i1 [[TMP133]], label [[PRED_UDIV_IF73:%.*]], label [[PRED_UDIV_CONTINUE74:%.*]]
-; CHECK: pred.udiv.if73:
-; CHECK-NEXT: [[TMP151:%.*]] = extractelement <8 x i32> [[TMP134]], i64 4
-; CHECK-NEXT: [[TMP152:%.*]] = udiv i32 [[TMP151]], [[D]]
-; CHECK-NEXT: [[TMP153:%.*]] = insertelement <8 x i32> [[TMP150]], i32 [[TMP152]], i64 4
-; CHECK-NEXT: br label [[PRED_UDIV_CONTINUE74]]
-; CHECK: pred.udiv.continue74:
-; CHECK-NEXT: [[TMP154:%.*]] = phi <8 x i32> [ [[TMP150]], [[PRED_UDIV_CONTINUE72]] ], [ [[TMP153]], [[PRED_UDIV_IF73]] ]
-; CHECK-NEXT: br i1 [[TMP133]], label [[PRED_UDIV_IF75:%.*]], label [[PRED_UDIV_CONTINUE76:%.*]]
-; CHECK: pred.udiv.if75:
-; CHECK-NEXT: [[TMP155:%.*]] = extractelement <8 x i32> [[TMP134]], i64 5
-; CHECK-NEXT: [[TMP156:%.*]] = udiv i32 [[TMP155]], [[D]]
-; CHECK-NEXT: [[TMP157:%.*]] = insertelement <8 x i32> [[TMP154]], i32 [[TMP156]], i64 5
-; CHECK-NEXT: br label [[PRED_UDIV_CONTINUE76]]
-; CHECK: pred.udiv.continue76:
-; CHECK-NEXT: [[TMP158:%.*]] = phi <8 x i32> [ [[TMP154]], [[PRED_UDIV_CONTINUE74]] ], [ [[TMP157]], [[PRED_UDIV_IF75]] ]
-; CHECK-NEXT: br i1 [[TMP133]], label [[PRED_UDIV_IF77:%.*]], label [[PRED_UDIV_CONTINUE78:%.*]]
-; CHECK: pred.udiv.if77:
-; CHECK-NEXT: [[TMP159:%.*]] = extractelement <8 x i32> [[TMP134]], i64 6
-; CHECK-NEXT: [[TMP160:%.*]] = udiv i32 [[TMP159]], [[D]]
-; CHECK-NEXT: [[TMP161:%.*]] = insertelement <8 x i32> [[TMP158]], i32 [[TMP160]], i64 6
-; CHECK-NEXT: br label [[PRED_UDIV_CONTINUE78]]
-; CHECK: pred.udiv.continue78:
-; CHECK-NEXT: [[TMP162:%.*]] = phi <8 x i32> [ [[TMP158]], [[PRED_UDIV_CONTINUE76]] ], [ [[TMP161]], [[PRED_UDIV_IF77]] ]
-; CHECK-NEXT: br i1 [[TMP133]], label [[PRED_UDIV_IF79:%.*]], label [[PRED_UDIV_CONTINUE80]]
-; CHECK: pred.udiv.if79:
-; CHECK-NEXT: [[TMP163:%.*]] = extractelement <8 x i32> [[TMP134]], i64 7
-; CHECK-NEXT: [[TMP164:%.*]] = udiv i32 [[TMP163]], [[D]]
-; CHECK-NEXT: [[TMP165:%.*]] = insertelement <8 x i32> [[TMP162]], i32 [[TMP164]], i64 7
-; CHECK-NEXT: br label [[PRED_UDIV_CONTINUE80]]
-; CHECK: pred.udiv.continue80:
-; CHECK-NEXT: [[TMP166:%.*]] = phi <8 x i32> [ [[TMP162]], [[PRED_UDIV_CONTINUE78]] ], [ [[TMP165]], [[PRED_UDIV_IF79]] ]
-; CHECK-NEXT: [[INDEX_NEXT81]] = add nuw i32 [[INDEX63]], 8
-; CHECK-NEXT: [[VEC_IND_NEXT82]] = add <8 x i32> [[VEC_IND64]], splat (i32 8)
-; CHECK-NEXT: [[TMP167:%.*]] = icmp eq i32 [[INDEX_NEXT81]], 1000
-; CHECK-NEXT: br i1 [[TMP167]], label [[VEC_EPILOG_MIDDLE_BLOCK:%.*]], label [[VEC_EPILOG_VECTOR_BODY]], !llvm.loop [[LOOP14:![0-9]+]]
-; CHECK: vec.epilog.middle.block:
-; CHECK-NEXT: [[TMP168:%.*]] = zext <8 x i32> [[TMP166]] to <8 x i64>
-; CHECK-NEXT: [[PREDPHI83:%.*]] = select i1 [[C]], <8 x i64> zeroinitializer, <8 x i64> [[TMP168]]
-; CHECK-NEXT: [[TMP169:%.*]] = extractelement <8 x i64> [[PREDPHI83]], i64 7
-; CHECK-NEXT: br i1 false, label [[EXIT]], label [[VEC_EPILOG_SCALAR_PH]]
-; CHECK: vec.epilog.scalar.ph:
-; CHECK-NEXT: [[BC_RESUME_VAL:%.*]] = phi i32 [ 1000, [[VEC_EPILOG_MIDDLE_BLOCK]] ], [ 992, [[VEC_EPILOG_ITER_CHECK]] ], [ 0, [[ITER_CHECK:%.*]] ]
+; CHECK-NEXT: entry:
; CHECK-NEXT: br label [[LOOP_HEADER:%.*]]
; CHECK: loop.header:
-; CHECK-NEXT: [[IV:%.*]] = phi i32 [ [[BC_RESUME_VAL]], [[VEC_EPILOG_SCALAR_PH]] ], [ [[IV_NEXT:%.*]], [[LOOP_LATCH:%.*]] ]
-; CHECK-NEXT: br i1 [[C]], label [[LOOP_LATCH]], label [[THEN:%.*]]
+; CHECK-NEXT: [[IV:%.*]] = phi i32 [ 0, [[ENTRY:%.*]] ], [ [[IV_NEXT:%.*]], [[LOOP_LATCH:%.*]] ]
+; CHECK-NEXT: br i1 [[C:%.*]], label [[LOOP_LATCH]], label [[THEN:%.*]]
; CHECK: then:
; CHECK-NEXT: [[CALL:%.*]] = tail call i32 @llvm.usub.sat.i32(i32 [[IV]], i32 1)
-; CHECK-NEXT: [[UDIV:%.*]] = udiv i32 [[CALL]], [[D]]
+; CHECK-NEXT: [[UDIV:%.*]] = udiv i32 [[CALL]], [[D:%.*]]
; CHECK-NEXT: [[ZEXT:%.*]] = zext i32 [[UDIV]] to i64
; CHECK-NEXT: br label [[LOOP_LATCH]]
; CHECK: loop.latch:
; CHECK-NEXT: [[MERGE:%.*]] = phi i64 [ [[ZEXT]], [[THEN]] ], [ 0, [[LOOP_HEADER]] ]
; CHECK-NEXT: [[IV_NEXT]] = add i32 [[IV]], 1
; CHECK-NEXT: [[EC:%.*]] = icmp eq i32 [[IV]], 1000
-; CHECK-NEXT: br i1 [[EC]], label [[EXIT]], label [[LOOP_HEADER]], !llvm.loop [[LOOP15:![0-9]+]]
+; CHECK-NEXT: br i1 [[EC]], label [[EXIT:%.*]], label [[LOOP_HEADER]]
; CHECK: exit:
-; CHECK-NEXT: [[MERGE_LCSSA:%.*]] = phi i64 [ [[MERGE]], [[LOOP_LATCH]] ], [ [[TMP132]], [[MIDDLE_BLOCK]] ], [ [[TMP169]], [[VEC_EPILOG_MIDDLE_BLOCK]] ]
+; CHECK-NEXT: [[MERGE_LCSSA:%.*]] = phi i64 [ [[MERGE]], [[LOOP_LATCH]] ]
; CHECK-NEXT: ret i64 [[MERGE_LCSSA]]
;
entry:
diff --git a/llvm/test/Transforms/LoopVectorize/X86/pr109581-unused-blend.ll b/llvm/test/Transforms/LoopVectorize/X86/pr109581-unused-blend.ll
index 0323a6771949d..89096bb9126ed 100644
--- a/llvm/test/Transforms/LoopVectorize/X86/pr109581-unused-blend.ll
+++ b/llvm/test/Transforms/LoopVectorize/X86/pr109581-unused-blend.ll
@@ -14,50 +14,50 @@ define i32 @unused_blend_after_unrolling(ptr %p, i32 %a, i1 %c.1, i16 %x, i16 %y
; CHECK: [[VECTOR_PH]]:
; CHECK-NEXT: [[BROADCAST_SPLATINSERT16:%.*]] = insertelement <4 x i1> poison, i1 [[C]], i64 0
; CHECK-NEXT: [[BROADCAST_SPLAT17:%.*]] = shufflevector <4 x i1> [[BROADCAST_SPLATINSERT16]], <4 x i1> poison, <4 x i32> zeroinitializer
-; CHECK-NEXT: [[TMP2:%.*]] = xor i1 [[C_1]], true
+; CHECK-NEXT: [[TMP0:%.*]] = xor i1 [[C_1]], true
; CHECK-NEXT: [[TMP22:%.*]] = xor <4 x i1> [[BROADCAST_SPLAT17]], splat (i1 true)
-; CHECK-NEXT: br label %[[VECTOR_BODY:.*]]
-; CHECK: [[VECTOR_BODY]]:
-; CHECK-NEXT: [[INDEX:%.*]] = phi i32 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[PRED_SDIV_CONTINUE17:.*]] ]
-; CHECK-NEXT: [[VEC_PHI:%.*]] = phi <4 x i1> [ zeroinitializer, %[[VECTOR_PH]] ], [ [[TMP24:%.*]], %[[PRED_SDIV_CONTINUE17]] ]
-; CHECK-NEXT: [[VEC_PHI3:%.*]] = phi <4 x i1> [ zeroinitializer, %[[VECTOR_PH]] ], [ [[TMP25:%.*]], %[[PRED_SDIV_CONTINUE17]] ]
-; CHECK-NEXT: br i1 [[TMP2]], label %[[PRED_SDIV_IF:.*]], label %[[PRED_SDIV_CONTINUE:.*]]
-; CHECK: [[PRED_SDIV_IF]]:
-; CHECK-NEXT: br label %[[PRED_SDIV_CONTINUE]]
-; CHECK: [[PRED_SDIV_CONTINUE]]:
-; CHECK-NEXT: br i1 [[TMP2]], label %[[PRED_SDIV_IF4:.*]], label %[[PRED_SDIV_CONTINUE5:.*]]
+; CHECK-NEXT: br label %[[PRED_SDIV_CONTINUE15:.*]]
+; CHECK: [[PRED_SDIV_CONTINUE15]]:
+; CHECK-NEXT: [[INDEX:%.*]] = phi i32 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[PRED_SDIV_CONTINUE16:.*]] ]
+; CHECK-NEXT: [[VEC_PHI:%.*]] = phi <4 x i1> [ zeroinitializer, %[[VECTOR_PH]] ], [ [[TMP24:%.*]], %[[PRED_SDIV_CONTINUE16]] ]
+; CHECK-NEXT: [[VEC_PHI3:%.*]] = phi <4 x i1> [ zeroinitializer, %[[VECTOR_PH]] ], [ [[TMP25:%.*]], %[[PRED_SDIV_CONTINUE16]] ]
+; CHECK-NEXT: br i1 [[TMP0]], label %[[PRED_SDIV_IF16:.*]], label %[[PRED_SDIV_CONTINUE17:.*]]
+; CHECK: [[PRED_SDIV_IF16]]:
+; CHECK-NEXT: br label %[[PRED_SDIV_CONTINUE17]]
+; CHECK: [[PRED_SDIV_CONTINUE17]]:
+; CHECK-NEXT: br i1 [[TMP0]], label %[[PRED_SDIV_IF2:.*]], label %[[PRED_SDIV_CONTINUE3:.*]]
+; CHECK: [[PRED_SDIV_IF2]]:
+; CHECK-NEXT: br label %[[PRED_SDIV_CONTINUE3]]
+; CHECK: [[PRED_SDIV_CONTINUE3]]:
+; CHECK-NEXT: br i1 [[TMP0]], label %[[PRED_SDIV_IF4:.*]], label %[[PRED_SDIV_CONTINUE5:.*]]
; CHECK: [[PRED_SDIV_IF4]]:
; CHECK-NEXT: br label %[[PRED_SDIV_CONTINUE5]]
; CHECK: [[PRED_SDIV_CONTINUE5]]:
-; CHECK-NEXT: br i1 [[TMP2]], label %[[PRED_SDIV_IF6:.*]], label %[[PRED_SDIV_CONTINUE7:.*]]
+; CHECK-NEXT: br i1 [[TMP0]], label %[[PRED_SDIV_IF6:.*]], label %[[PRED_SDIV_CONTINUE7:.*]]
; CHECK: [[PRED_SDIV_IF6]]:
; CHECK-NEXT: br label %[[PRED_SDIV_CONTINUE7]]
; CHECK: [[PRED_SDIV_CONTINUE7]]:
-; CHECK-NEXT: br i1 [[TMP2]], label %[[PRED_SDIV_IF8:.*]], label %[[PRED_SDIV_CONTINUE9:.*]]
+; CHECK-NEXT: br i1 [[TMP0]], label %[[PRED_SDIV_IF8:.*]], label %[[PRED_SDIV_CONTINUE9:.*]]
; CHECK: [[PRED_SDIV_IF8]]:
; CHECK-NEXT: br label %[[PRED_SDIV_CONTINUE9]]
; CHECK: [[PRED_SDIV_CONTINUE9]]:
-; CHECK-NEXT: br i1 [[TMP2]], label %[[PRED_SDIV_IF10:.*]], label %[[PRED_SDIV_CONTINUE11:.*]]
+; CHECK-NEXT: br i1 [[TMP0]], label %[[PRED_SDIV_IF10:.*]], label %[[PRED_SDIV_CONTINUE11:.*]]
; CHECK: [[PRED_SDIV_IF10]]:
; CHECK-NEXT: br label %[[PRED_SDIV_CONTINUE11]]
; CHECK: [[PRED_SDIV_CONTINUE11]]:
-; CHECK-NEXT: br i1 [[TMP2]], label %[[PRED_SDIV_IF12:.*]], label %[[PRED_SDIV_CONTINUE13:.*]]
+; CHECK-NEXT: br i1 [[TMP0]], label %[[PRED_SDIV_IF12:.*]], label %[[PRED_SDIV_CONTINUE13:.*]]
; CHECK: [[PRED_SDIV_IF12]]:
; CHECK-NEXT: br label %[[PRED_SDIV_CONTINUE13]]
; CHECK: [[PRED_SDIV_CONTINUE13]]:
-; CHECK-NEXT: br i1 [[TMP2]], label %[[PRED_SDIV_IF14:.*]], label %[[PRED_SDIV_CONTINUE15:.*]]
+; CHECK-NEXT: br i1 [[TMP0]], label %[[PRED_SDIV_IF14:.*]], label %[[PRED_SDIV_CONTINUE16]]
; CHECK: [[PRED_SDIV_IF14]]:
-; CHECK-NEXT: br label %[[PRED_SDIV_CONTINUE15]]
-; CHECK: [[PRED_SDIV_CONTINUE15]]:
-; CHECK-NEXT: br i1 [[TMP2]], label %[[PRED_SDIV_IF16:.*]], label %[[PRED_SDIV_CONTINUE17]]
-; CHECK: [[PRED_SDIV_IF16]]:
-; CHECK-NEXT: br label %[[PRED_SDIV_CONTINUE17]]
-; CHECK: [[PRED_SDIV_CONTINUE17]]:
+; CHECK-NEXT: br label %[[PRED_SDIV_CONTINUE16]]
+; CHECK: [[PRED_SDIV_CONTINUE16]]:
; CHECK-NEXT: [[TMP24]] = or <4 x i1> [[VEC_PHI]], [[TMP22]]
; CHECK-NEXT: [[TMP25]] = or <4 x i1> [[VEC_PHI3]], [[TMP22]]
; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i32 [[INDEX]], 8
; CHECK-NEXT: [[TMP26:%.*]] = icmp eq i32 [[INDEX_NEXT]], 96
-; CHECK-NEXT: br i1 [[TMP26]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP0:![0-9]+]]
+; CHECK-NEXT: br i1 [[TMP26]], label %[[MIDDLE_BLOCK:.*]], label %[[PRED_SDIV_CONTINUE15]], !llvm.loop [[LOOP0:![0-9]+]]
; CHECK: [[MIDDLE_BLOCK]]:
; CHECK-NEXT: [[BIN_RDX:%.*]] = or <4 x i1> [[TMP25]], [[TMP24]]
; CHECK-NEXT: [[TMP27:%.*]] = call i1 @llvm.vector.reduce.or.v4i1(<4 x i1> [[BIN_RDX]])
diff --git a/llvm/test/Transforms/LoopVectorize/X86/predicated-instruction-cost.ll b/llvm/test/Transforms/LoopVectorize/X86/predicated-instruction-cost.ll
index 5a479250e1490..f871a5f7379b6 100644
--- a/llvm/test/Transforms/LoopVectorize/X86/predicated-instruction-cost.ll
+++ b/llvm/test/Transforms/LoopVectorize/X86/predicated-instruction-cost.ll
@@ -57,23 +57,49 @@ exit:
define void @test_wide_shift_uses_predicated_invariant_instruction(i32 %d, i1 %c, ptr %dst) {
; CHECK-LABEL: define void @test_wide_shift_uses_predicated_invariant_instruction(
; CHECK-SAME: i32 [[D:%.*]], i1 [[C:%.*]], ptr [[DST:%.*]]) {
-; CHECK-NEXT: [[ENTRY:.*]]:
+; CHECK-NEXT: [[ENTRY:.*:]]
; CHECK-NEXT: br label %[[LOOP_HEADER:.*]]
; CHECK: [[LOOP_HEADER]]:
-; CHECK-NEXT: [[IV:%.*]] = phi i32 [ 0, %[[ENTRY]] ], [ [[IV_NEXT:%.*]], %[[LOOP_LATCH:.*]] ]
+; CHECK-NEXT: [[TMP0:%.*]] = urem i32 100, [[D]]
+; CHECK-NEXT: [[TMP1:%.*]] = shl i32 [[TMP0]], 12
+; CHECK-NEXT: [[TMP2:%.*]] = shl i32 999, [[TMP1]]
+; CHECK-NEXT: br label %[[VECTOR_BODY:.*]]
+; CHECK: [[VECTOR_BODY]]:
+; CHECK-NEXT: [[INDEX:%.*]] = phi i32 [ 0, %[[LOOP_HEADER]] ], [ [[INDEX_NEXT:%.*]], %[[LOOP_LATCH:.*]] ]
; CHECK-NEXT: br i1 [[C]], label %[[LOOP_LATCH]], label %[[ELSE:.*]]
; CHECK: [[ELSE]]:
+; CHECK-NEXT: br label %[[LOOP_LATCH]]
+; CHECK: [[LOOP_LATCH]]:
+; CHECK-NEXT: [[TMP3:%.*]] = phi i32 [ poison, %[[VECTOR_BODY]] ], [ [[TMP2]], %[[ELSE]] ]
+; CHECK-NEXT: [[BROADCAST_SPLATINSERT:%.*]] = insertelement <4 x i32> poison, i32 [[TMP3]], i64 0
+; CHECK-NEXT: [[BROADCAST_SPLAT:%.*]] = shufflevector <4 x i32> [[BROADCAST_SPLATINSERT]], <4 x i32> poison, <4 x i32> zeroinitializer
+; CHECK-NEXT: [[PREDPHI:%.*]] = select i1 true, <4 x i32> [[BROADCAST_SPLAT]], <4 x i32> zeroinitializer
+; CHECK-NEXT: [[TMP4:%.*]] = getelementptr inbounds i32, ptr [[DST]], i32 [[INDEX]]
+; CHECK-NEXT: [[TMP5:%.*]] = getelementptr inbounds i32, ptr [[TMP4]], i64 4
+; CHECK-NEXT: store <4 x i32> [[PREDPHI]], ptr [[TMP4]], align 4
+; CHECK-NEXT: store <4 x i32> [[PREDPHI]], ptr [[TMP5]], align 4
+; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i32 [[INDEX]], 8
+; CHECK-NEXT: [[TMP6:%.*]] = icmp eq i32 [[INDEX_NEXT]], 96
+; CHECK-NEXT: br i1 [[TMP6]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP0:![0-9]+]]
+; CHECK: [[MIDDLE_BLOCK]]:
+; CHECK-NEXT: br label %[[SCALAR_PH:.*]]
+; CHECK: [[SCALAR_PH]]:
+; CHECK-NEXT: br label %[[LOOP_HEADER1:.*]]
+; CHECK: [[LOOP_HEADER1]]:
+; CHECK-NEXT: [[IV:%.*]] = phi i32 [ 96, %[[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], %[[LOOP_LATCH1:.*]] ]
+; CHECK-NEXT: br i1 [[C]], label %[[LOOP_LATCH1]], label %[[ELSE1:.*]]
+; CHECK: [[ELSE1]]:
; CHECK-NEXT: [[REM:%.*]] = urem i32 100, [[D]]
; CHECK-NEXT: [[SEXT:%.*]] = shl i32 [[REM]], 12
; CHECK-NEXT: [[SHL_I:%.*]] = shl i32 999, [[SEXT]]
-; CHECK-NEXT: br label %[[LOOP_LATCH]]
-; CHECK: [[LOOP_LATCH]]:
-; CHECK-NEXT: [[P:%.*]] = phi i32 [ [[SHL_I]], %[[ELSE]] ], [ 0, %[[LOOP_HEADER]] ]
+; CHECK-NEXT: br label %[[LOOP_LATCH1]]
+; CHECK: [[LOOP_LATCH1]]:
+; CHECK-NEXT: [[P:%.*]] = phi i32 [ [[SHL_I]], %[[ELSE1]] ], [ 0, %[[LOOP_HEADER1]] ]
; CHECK-NEXT: [[GEP:%.*]] = getelementptr inbounds i32, ptr [[DST]], i32 [[IV]]
; CHECK-NEXT: store i32 [[P]], ptr [[GEP]], align 4
; CHECK-NEXT: [[IV_NEXT]] = add i32 [[IV]], 1
; CHECK-NEXT: [[EC:%.*]] = icmp eq i32 [[IV_NEXT]], 100
-; CHECK-NEXT: br i1 [[EC]], label %[[EXIT:.*]], label %[[LOOP_HEADER]]
+; CHECK-NEXT: br i1 [[EC]], label %[[EXIT:.*]], label %[[LOOP_HEADER1]], !llvm.loop [[LOOP3:![0-9]+]]
; CHECK: [[EXIT]]:
; CHECK-NEXT: ret void
;
@@ -238,7 +264,7 @@ define i32 @predicated_store_with_scatter_legal(ptr %dst, i64 %n) #0 {
; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 8
; CHECK-NEXT: [[VEC_IND_NEXT]] = add <8 x i64> [[VEC_IND]], splat (i64 8)
; CHECK-NEXT: [[TMP16:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
-; CHECK-NEXT: br i1 [[TMP16]], label %[[MIDDLE_BLOCK:.*]], label %[[LOOP]], !llvm.loop [[LOOP0:![0-9]+]]
+; CHECK-NEXT: br i1 [[TMP16]], label %[[MIDDLE_BLOCK:.*]], label %[[LOOP]], !llvm.loop [[LOOP4:![0-9]+]]
; CHECK: [[MIDDLE_BLOCK]]:
; CHECK-NEXT: [[CMP_N:%.*]] = icmp eq i64 [[TMP0]], [[N_VEC]]
; CHECK-NEXT: br i1 [[CMP_N]], label %[[EXIT:.*]], label %[[SCALAR_PH]]
@@ -264,7 +290,7 @@ define i32 @predicated_store_with_scatter_legal(ptr %dst, i64 %n) #0 {
; CHECK: [[LATCH]]:
; CHECK-NEXT: [[IV_NEXT]] = add i64 [[IV]], 1
; CHECK-NEXT: [[EC:%.*]] = icmp eq i64 [[IV]], [[N]]
-; CHECK-NEXT: br i1 [[EC]], label %[[EXIT]], label %[[LOOP1]], !llvm.loop [[LOOP3:![0-9]+]]
+; CHECK-NEXT: br i1 [[EC]], label %[[EXIT]], label %[[LOOP1]], !llvm.loop [[LOOP5:![0-9]+]]
; CHECK: [[EXIT]]:
; CHECK-NEXT: ret i32 0
;
diff --git a/llvm/test/Transforms/LoopVectorize/X86/predicated-replicate-feeding-cast.ll b/llvm/test/Transforms/LoopVectorize/X86/predicated-replicate-feeding-cast.ll
index 5ea8625a17aff..05d69eb87b0bd 100644
--- a/llvm/test/Transforms/LoopVectorize/X86/predicated-replicate-feeding-cast.ll
+++ b/llvm/test/Transforms/LoopVectorize/X86/predicated-replicate-feeding-cast.ll
@@ -15,43 +15,28 @@ define i8 @predicated_replicate_feeding_cast(i16 %n, i1 %c1, i1 %c2, i16 %a, i8
; CHECK-NEXT: [[N_MOD_VF:%.*]] = and i32 [[TMP1]], 1
; CHECK-NEXT: [[N_VEC:%.*]] = sub i32 [[TMP1]], [[N_MOD_VF]]
; CHECK-NEXT: [[TMP2:%.*]] = trunc i32 [[N_VEC]] to i16
-; CHECK-NEXT: [[TMP3:%.*]] = xor i1 [[C1]], true
-; CHECK-NEXT: [[TMP4:%.*]] = xor i1 [[C2]], true
-; CHECK-NEXT: br label %[[VECTOR_BODY:.*]]
-; CHECK: [[VECTOR_BODY]]:
-; CHECK-NEXT: [[INDEX:%.*]] = phi i32 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[PRED_SDIV_CONTINUE6:.*]] ]
-; CHECK-NEXT: br i1 [[TMP3]], label %[[PRED_SDIV_IF:.*]], label %[[PRED_SDIV_CONTINUE:.*]]
-; CHECK: [[PRED_SDIV_IF]]:
; CHECK-NEXT: [[TMP5:%.*]] = sdiv i16 1, [[A]]
-; CHECK-NEXT: br label %[[PRED_SDIV_CONTINUE]]
+; CHECK-NEXT: br label %[[PRED_SDIV_CONTINUE:.*]]
; CHECK: [[PRED_SDIV_CONTINUE]]:
-; CHECK-NEXT: [[TMP6:%.*]] = phi i16 [ poison, %[[VECTOR_BODY]] ], [ [[TMP5]], %[[PRED_SDIV_IF]] ]
-; CHECK-NEXT: br i1 [[TMP3]], label %[[PRED_SDIV_IF1:.*]], label %[[PRED_SDIV_CONTINUE2:.*]]
+; CHECK-NEXT: [[INDEX:%.*]] = phi i32 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[PRED_SDIV_CONTINUE6:.*]] ]
+; CHECK-NEXT: br i1 [[C1]], label %[[PRED_SDIV_CONTINUE2:.*]], label %[[PRED_SDIV_IF1:.*]]
; CHECK: [[PRED_SDIV_IF1]]:
; CHECK-NEXT: br label %[[PRED_SDIV_CONTINUE2]]
; CHECK: [[PRED_SDIV_CONTINUE2]]:
-; CHECK-NEXT: [[TMP11:%.*]] = select i1 [[C1]], i16 0, i16 [[TMP6]]
+; CHECK-NEXT: [[TMP6:%.*]] = phi i16 [ poison, %[[PRED_SDIV_CONTINUE]] ], [ [[TMP5]], %[[PRED_SDIV_IF1]] ]
+; CHECK-NEXT: [[TMP11:%.*]] = select i1 true, i16 [[TMP6]], i16 0
+; CHECK-NEXT: br i1 [[C2]], label %[[PRED_SDIV_CONTINUE6]], label %[[IF23:.*]]
+; CHECK: [[IF23]]:
; CHECK-NEXT: [[TMP12:%.*]] = trunc i16 [[TMP11]] to i8
-; CHECK-NEXT: br i1 [[TMP4]], label %[[PRED_SDIV_IF3:.*]], label %[[PRED_SDIV_CONTINUE4:.*]]
-; CHECK: [[PRED_SDIV_IF3]]:
-; CHECK-NEXT: [[TMP13:%.*]] = sdiv i8 [[TMP12]], [[B]]
-; CHECK-NEXT: [[TMP14:%.*]] = insertelement <2 x i8> poison, i8 [[TMP13]], i64 0
-; CHECK-NEXT: br label %[[PRED_SDIV_CONTINUE4]]
-; CHECK: [[PRED_SDIV_CONTINUE4]]:
-; CHECK-NEXT: [[TMP15:%.*]] = phi <2 x i8> [ poison, %[[PRED_SDIV_CONTINUE2]] ], [ [[TMP14]], %[[PRED_SDIV_IF3]] ]
-; CHECK-NEXT: br i1 [[TMP4]], label %[[PRED_SDIV_IF5:.*]], label %[[PRED_SDIV_CONTINUE6]]
-; CHECK: [[PRED_SDIV_IF5]]:
; CHECK-NEXT: [[TMP18:%.*]] = sdiv i8 [[TMP12]], [[B]]
-; CHECK-NEXT: [[TMP19:%.*]] = insertelement <2 x i8> [[TMP15]], i8 [[TMP18]], i64 1
; CHECK-NEXT: br label %[[PRED_SDIV_CONTINUE6]]
; CHECK: [[PRED_SDIV_CONTINUE6]]:
-; CHECK-NEXT: [[TMP20:%.*]] = phi <2 x i8> [ [[TMP15]], %[[PRED_SDIV_CONTINUE4]] ], [ [[TMP19]], %[[PRED_SDIV_IF5]] ]
+; CHECK-NEXT: [[TMP8:%.*]] = phi i8 [ poison, %[[PRED_SDIV_CONTINUE2]] ], [ [[TMP18]], %[[IF23]] ]
; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i32 [[INDEX]], 2
; CHECK-NEXT: [[TMP21:%.*]] = icmp eq i32 [[INDEX_NEXT]], [[N_VEC]]
-; CHECK-NEXT: br i1 [[TMP21]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP0:![0-9]+]]
+; CHECK-NEXT: br i1 [[TMP21]], label %[[MIDDLE_BLOCK:.*]], label %[[PRED_SDIV_CONTINUE]], !llvm.loop [[LOOP0:![0-9]+]]
; CHECK: [[MIDDLE_BLOCK]]:
-; CHECK-NEXT: [[PREDPHI7:%.*]] = select i1 [[C2]], <2 x i8> zeroinitializer, <2 x i8> [[TMP20]]
-; CHECK-NEXT: [[TMP22:%.*]] = extractelement <2 x i8> [[PREDPHI7]], i64 1
+; CHECK-NEXT: [[PREDPHI5:%.*]] = select i1 true, i8 [[TMP8]], i8 0
; CHECK-NEXT: [[CMP_N:%.*]] = icmp eq i32 [[TMP1]], [[N_VEC]]
; CHECK-NEXT: br i1 [[CMP_N]], label %[[EXIT:.*]], label %[[SCALAR_PH]]
; CHECK: [[SCALAR_PH]]:
@@ -76,7 +61,7 @@ define i8 @predicated_replicate_feeding_cast(i16 %n, i1 %c1, i1 %c2, i16 %a, i8
; CHECK-NEXT: [[EC:%.*]] = icmp eq i16 [[IV]], [[N]]
; CHECK-NEXT: br i1 [[EC]], label %[[EXIT]], label %[[LOOP]], !llvm.loop [[LOOP3:![0-9]+]]
; CHECK: [[EXIT]]:
-; CHECK-NEXT: [[RES_LCSSA:%.*]] = phi i8 [ [[RES]], %[[LATCH]] ], [ [[TMP22]], %[[MIDDLE_BLOCK]] ]
+; CHECK-NEXT: [[RES_LCSSA:%.*]] = phi i8 [ [[RES]], %[[LATCH]] ], [ [[PREDPHI5]], %[[MIDDLE_BLOCK]] ]
; CHECK-NEXT: ret i8 [[RES_LCSSA]]
;
entry:
@@ -119,54 +104,31 @@ define i8 @predicated_replicate_feeding_cast_non_uniform(i64 %n, i1 %c1, i1 %c2,
; CHECK: [[VECTOR_PH]]:
; CHECK-NEXT: [[N_MOD_VF:%.*]] = and i64 [[TMP0]], 1
; CHECK-NEXT: [[N_VEC:%.*]] = sub i64 [[TMP0]], [[N_MOD_VF]]
-; CHECK-NEXT: [[TMP1:%.*]] = xor i1 [[C1]], true
-; CHECK-NEXT: [[TMP2:%.*]] = xor i1 [[C2]], true
; CHECK-NEXT: br label %[[VECTOR_BODY:.*]]
; CHECK: [[VECTOR_BODY]]:
; CHECK-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[PRED_SDIV_CONTINUE6:.*]] ]
; CHECK-NEXT: [[TMP3:%.*]] = getelementptr i16, ptr [[A_PTR]], i64 [[INDEX]]
; CHECK-NEXT: [[WIDE_LOAD:%.*]] = load <2 x i16>, ptr [[TMP3]], align 2
-; CHECK-NEXT: br i1 [[TMP1]], label %[[PRED_SDIV_IF:.*]], label %[[PRED_SDIV_CONTINUE:.*]]
+; CHECK-NEXT: br i1 [[C1]], label %[[PRED_SDIV_CONTINUE4:.*]], label %[[PRED_SDIV_IF:.*]]
; CHECK: [[PRED_SDIV_IF]]:
; CHECK-NEXT: [[TMP4:%.*]] = extractelement <2 x i16> [[WIDE_LOAD]], i64 0
; CHECK-NEXT: [[TMP5:%.*]] = sdiv i16 1, [[TMP4]]
-; CHECK-NEXT: [[TMP6:%.*]] = insertelement <2 x i16> poison, i16 [[TMP5]], i64 0
-; CHECK-NEXT: br label %[[PRED_SDIV_CONTINUE]]
-; CHECK: [[PRED_SDIV_CONTINUE]]:
-; CHECK-NEXT: [[TMP7:%.*]] = phi <2 x i16> [ poison, %[[VECTOR_BODY]] ], [ [[TMP6]], %[[PRED_SDIV_IF]] ]
-; CHECK-NEXT: br i1 [[TMP1]], label %[[PRED_SDIV_IF1:.*]], label %[[PRED_SDIV_CONTINUE2:.*]]
-; CHECK: [[PRED_SDIV_IF1]]:
-; CHECK-NEXT: [[TMP8:%.*]] = extractelement <2 x i16> [[WIDE_LOAD]], i64 1
-; CHECK-NEXT: [[TMP9:%.*]] = sdiv i16 1, [[TMP8]]
-; CHECK-NEXT: [[TMP10:%.*]] = insertelement <2 x i16> [[TMP7]], i16 [[TMP9]], i64 1
-; CHECK-NEXT: br label %[[PRED_SDIV_CONTINUE2]]
-; CHECK: [[PRED_SDIV_CONTINUE2]]:
-; CHECK-NEXT: [[TMP11:%.*]] = phi <2 x i16> [ [[TMP7]], %[[PRED_SDIV_CONTINUE]] ], [ [[TMP10]], %[[PRED_SDIV_IF1]] ]
-; CHECK-NEXT: [[PREDPHI:%.*]] = select i1 [[C1]], <2 x i16> zeroinitializer, <2 x i16> [[TMP11]]
-; CHECK-NEXT: br i1 [[TMP2]], label %[[PRED_SDIV_IF3:.*]], label %[[PRED_SDIV_CONTINUE4:.*]]
-; CHECK: [[PRED_SDIV_IF3]]:
-; CHECK-NEXT: [[TMP12:%.*]] = extractelement <2 x i16> [[PREDPHI]], i64 0
-; CHECK-NEXT: [[TMP13:%.*]] = trunc i16 [[TMP12]] to i8
-; CHECK-NEXT: [[TMP14:%.*]] = sdiv i8 [[TMP13]], [[B]]
-; CHECK-NEXT: [[TMP15:%.*]] = insertelement <2 x i8> poison, i8 [[TMP14]], i64 0
; CHECK-NEXT: br label %[[PRED_SDIV_CONTINUE4]]
; CHECK: [[PRED_SDIV_CONTINUE4]]:
-; CHECK-NEXT: [[TMP16:%.*]] = phi <2 x i8> [ poison, %[[PRED_SDIV_CONTINUE2]] ], [ [[TMP15]], %[[PRED_SDIV_IF3]] ]
-; CHECK-NEXT: br i1 [[TMP2]], label %[[PRED_SDIV_IF5:.*]], label %[[PRED_SDIV_CONTINUE6]]
+; CHECK-NEXT: [[TMP6:%.*]] = phi i16 [ poison, %[[VECTOR_BODY]] ], [ [[TMP5]], %[[PRED_SDIV_IF]] ]
+; CHECK-NEXT: [[TMP17:%.*]] = select i1 true, i16 [[TMP6]], i16 0
+; CHECK-NEXT: br i1 [[C2]], label %[[PRED_SDIV_CONTINUE6]], label %[[PRED_SDIV_IF5:.*]]
; CHECK: [[PRED_SDIV_IF5]]:
-; CHECK-NEXT: [[TMP17:%.*]] = extractelement <2 x i16> [[PREDPHI]], i64 1
; CHECK-NEXT: [[TMP18:%.*]] = trunc i16 [[TMP17]] to i8
; CHECK-NEXT: [[TMP19:%.*]] = sdiv i8 [[TMP18]], [[B]]
-; CHECK-NEXT: [[TMP20:%.*]] = insertelement <2 x i8> [[TMP16]], i8 [[TMP19]], i64 1
; CHECK-NEXT: br label %[[PRED_SDIV_CONTINUE6]]
; CHECK: [[PRED_SDIV_CONTINUE6]]:
-; CHECK-NEXT: [[TMP21:%.*]] = phi <2 x i8> [ [[TMP16]], %[[PRED_SDIV_CONTINUE4]] ], [ [[TMP20]], %[[PRED_SDIV_IF5]] ]
+; CHECK-NEXT: [[TMP8:%.*]] = phi i8 [ poison, %[[PRED_SDIV_CONTINUE4]] ], [ [[TMP19]], %[[PRED_SDIV_IF5]] ]
; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 2
; CHECK-NEXT: [[TMP22:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
; CHECK-NEXT: br i1 [[TMP22]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP4:![0-9]+]]
; CHECK: [[MIDDLE_BLOCK]]:
-; CHECK-NEXT: [[PREDPHI7:%.*]] = select i1 [[C2]], <2 x i8> zeroinitializer, <2 x i8> [[TMP21]]
-; CHECK-NEXT: [[TMP23:%.*]] = extractelement <2 x i8> [[PREDPHI7]], i64 1
+; CHECK-NEXT: [[PREDPHI5:%.*]] = select i1 true, i8 [[TMP8]], i8 0
; CHECK-NEXT: [[CMP_N:%.*]] = icmp eq i64 [[TMP0]], [[N_VEC]]
; CHECK-NEXT: br i1 [[CMP_N]], label %[[EXIT:.*]], label %[[SCALAR_PH]]
; CHECK: [[SCALAR_PH]]:
@@ -193,7 +155,7 @@ define i8 @predicated_replicate_feeding_cast_non_uniform(i64 %n, i1 %c1, i1 %c2,
; CHECK-NEXT: [[EC:%.*]] = icmp eq i64 [[IV]], [[N]]
; CHECK-NEXT: br i1 [[EC]], label %[[EXIT]], label %[[LOOP]], !llvm.loop [[LOOP5:![0-9]+]]
; CHECK: [[EXIT]]:
-; CHECK-NEXT: [[RES_LCSSA:%.*]] = phi i8 [ [[RES]], %[[LATCH]] ], [ [[TMP23]], %[[MIDDLE_BLOCK]] ]
+; CHECK-NEXT: [[RES_LCSSA:%.*]] = phi i8 [ [[RES]], %[[LATCH]] ], [ [[PREDPHI5]], %[[MIDDLE_BLOCK]] ]
; CHECK-NEXT: ret i8 [[RES_LCSSA]]
;
entry:
diff --git a/llvm/test/Transforms/LoopVectorize/X86/replicate-recipe-with-only-first-lane-used.ll b/llvm/test/Transforms/LoopVectorize/X86/replicate-recipe-with-only-first-lane-used.ll
index 1e0a27b306fa3..bbcf52b05733e 100644
--- a/llvm/test/Transforms/LoopVectorize/X86/replicate-recipe-with-only-first-lane-used.ll
+++ b/llvm/test/Transforms/LoopVectorize/X86/replicate-recipe-with-only-first-lane-used.ll
@@ -14,12 +14,18 @@ define void @replicate_udiv_with_only_first_lane_used(i32 %x, ptr %dst, i64 %d)
; CHECK: [[VECTOR_PH]]:
; CHECK-NEXT: br label %[[VECTOR_BODY:.*]]
; CHECK: [[VECTOR_BODY]]:
-; CHECK-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[LOOP_LATCH1:.*]] ]
+; CHECK-NEXT: br label %[[LOOP_LATCH1]]
+; CHECK: [[LOOP_LATCH1]]:
+; CHECK-NEXT: [[TMP0:%.*]] = phi i64 [ poison, %[[VECTOR_BODY]] ]
+; CHECK-NEXT: [[PREDPHI:%.*]] = select i1 true, i64 [[TMP0]], i64 0
+; CHECK-NEXT: [[TMP1:%.*]] = getelementptr i16, ptr [[DST]], i64 [[PREDPHI]]
+; CHECK-NEXT: store i16 0, ptr [[TMP1]], align 2
+; CHECK-NEXT: store i16 0, ptr [[TMP1]], align 2
; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 8
; CHECK-NEXT: [[TMP12:%.*]] = icmp eq i64 [[INDEX_NEXT]], 96
; CHECK-NEXT: br i1 [[TMP12]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP0:![0-9]+]]
; CHECK: [[MIDDLE_BLOCK]]:
-; CHECK-NEXT: store i16 0, ptr [[DST]], align 2
; CHECK-NEXT: br label %[[SCALAR_PH:.*]]
; CHECK: [[SCALAR_PH]]:
; CHECK-NEXT: br label %[[LOOP_HEADER:.*]]
diff --git a/llvm/test/Transforms/LoopVectorize/X86/x86-predication.ll b/llvm/test/Transforms/LoopVectorize/X86/x86-predication.ll
index 8947813a4d670..dd8a4ceb20072 100644
--- a/llvm/test/Transforms/LoopVectorize/X86/x86-predication.ll
+++ b/llvm/test/Transforms/LoopVectorize/X86/x86-predication.ll
@@ -19,28 +19,28 @@ define i32 @predicated_sdiv_masked_load(ptr %a, ptr %b, i32 %x, i1 %c) {
; CHECK-NEXT: [[BROADCAST_SPLAT:%.*]] = shufflevector <2 x i1> [[BROADCAST_SPLATINSERT]], <2 x i1> poison, <2 x i32> zeroinitializer
; CHECK-NEXT: br label %[[VECTOR_BODY:.*]]
; CHECK: [[VECTOR_BODY]]:
-; CHECK-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[PRED_SDIV_CONTINUE2:.*]] ]
-; CHECK-NEXT: [[VEC_PHI:%.*]] = phi <2 x i32> [ zeroinitializer, %[[VECTOR_PH]] ], [ [[TMP11:%.*]], %[[PRED_SDIV_CONTINUE2]] ]
+; CHECK-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[PRED_SDIV_CONTINUE3:.*]] ]
+; CHECK-NEXT: [[VEC_PHI:%.*]] = phi <2 x i32> [ zeroinitializer, %[[VECTOR_PH]] ], [ [[TMP11:%.*]], %[[PRED_SDIV_CONTINUE3]] ]
; CHECK-NEXT: [[TMP0:%.*]] = getelementptr inbounds i32, ptr [[A]], i64 [[INDEX]]
; CHECK-NEXT: [[WIDE_LOAD:%.*]] = load <2 x i32>, ptr [[TMP0]], align 4
; CHECK-NEXT: [[TMP1:%.*]] = getelementptr i32, ptr [[B]], i64 [[INDEX]]
-; CHECK-NEXT: [[WIDE_MASKED_LOAD:%.*]] = call <2 x i32> @llvm.masked.load.v2i32.p0(ptr align 4 [[TMP1]], <2 x i1> [[BROADCAST_SPLAT]], <2 x i32> poison)
-; CHECK-NEXT: br i1 [[C]], label %[[PRED_SDIV_IF:.*]], label %[[PRED_SDIV_CONTINUE:.*]]
-; CHECK: [[PRED_SDIV_IF]]:
-; CHECK-NEXT: [[TMP2:%.*]] = extractelement <2 x i32> [[WIDE_MASKED_LOAD]], i64 0
-; CHECK-NEXT: [[TMP3:%.*]] = sdiv i32 [[TMP2]], [[X]]
-; CHECK-NEXT: [[TMP4:%.*]] = insertelement <2 x i32> poison, i32 [[TMP3]], i64 0
-; CHECK-NEXT: br label %[[PRED_SDIV_CONTINUE]]
-; CHECK: [[PRED_SDIV_CONTINUE]]:
-; CHECK-NEXT: [[TMP5:%.*]] = phi <2 x i32> [ poison, %[[VECTOR_BODY]] ], [ [[TMP4]], %[[PRED_SDIV_IF]] ]
-; CHECK-NEXT: br i1 [[C]], label %[[PRED_SDIV_IF1:.*]], label %[[PRED_SDIV_CONTINUE2]]
+; CHECK-NEXT: [[WIDE_LOAD2:%.*]] = call <2 x i32> @llvm.masked.load.v2i32.p0(ptr align 4 [[TMP1]], <2 x i1> [[BROADCAST_SPLAT]], <2 x i32> poison)
+; CHECK-NEXT: br i1 [[C]], label %[[PRED_SDIV_IF1:.*]], label %[[PRED_SDIV_CONTINUE2:.*]]
; CHECK: [[PRED_SDIV_IF1]]:
-; CHECK-NEXT: [[TMP6:%.*]] = extractelement <2 x i32> [[WIDE_MASKED_LOAD]], i64 1
+; CHECK-NEXT: [[TMP6:%.*]] = extractelement <2 x i32> [[WIDE_LOAD2]], i64 0
; CHECK-NEXT: [[TMP7:%.*]] = sdiv i32 [[TMP6]], [[X]]
-; CHECK-NEXT: [[TMP8:%.*]] = insertelement <2 x i32> [[TMP5]], i32 [[TMP7]], i64 1
+; CHECK-NEXT: [[TMP4:%.*]] = insertelement <2 x i32> poison, i32 [[TMP7]], i64 0
; CHECK-NEXT: br label %[[PRED_SDIV_CONTINUE2]]
; CHECK: [[PRED_SDIV_CONTINUE2]]:
-; CHECK-NEXT: [[TMP9:%.*]] = phi <2 x i32> [ [[TMP5]], %[[PRED_SDIV_CONTINUE]] ], [ [[TMP8]], %[[PRED_SDIV_IF1]] ]
+; CHECK-NEXT: [[TMP5:%.*]] = phi <2 x i32> [ poison, %[[VECTOR_BODY]] ], [ [[TMP4]], %[[PRED_SDIV_IF1]] ]
+; CHECK-NEXT: br i1 [[C]], label %[[PRED_SDIV_IF2:.*]], label %[[PRED_SDIV_CONTINUE3]]
+; CHECK: [[PRED_SDIV_IF2]]:
+; CHECK-NEXT: [[TMP14:%.*]] = extractelement <2 x i32> [[WIDE_LOAD2]], i64 1
+; CHECK-NEXT: [[TMP15:%.*]] = sdiv i32 [[TMP14]], [[X]]
+; CHECK-NEXT: [[TMP8:%.*]] = insertelement <2 x i32> [[TMP5]], i32 [[TMP15]], i64 1
+; CHECK-NEXT: br label %[[PRED_SDIV_CONTINUE3]]
+; CHECK: [[PRED_SDIV_CONTINUE3]]:
+; CHECK-NEXT: [[TMP9:%.*]] = phi <2 x i32> [ [[TMP5]], %[[PRED_SDIV_CONTINUE2]] ], [ [[TMP8]], %[[PRED_SDIV_IF2]] ]
; CHECK-NEXT: [[TMP10:%.*]] = add nsw <2 x i32> [[TMP9]], [[WIDE_LOAD]]
; CHECK-NEXT: [[PREDPHI:%.*]] = select i1 [[C]], <2 x i32> [[TMP10]], <2 x i32> [[WIDE_LOAD]]
; CHECK-NEXT: [[TMP11]] = add <2 x i32> [[VEC_PHI]], [[PREDPHI]]
@@ -62,31 +62,31 @@ define i32 @predicated_sdiv_masked_load(ptr %a, ptr %b, i32 %x, i1 %c) {
; SINK-GATHER-NEXT: [[BROADCAST_SPLAT:%.*]] = shufflevector <8 x i1> [[BROADCAST_SPLATINSERT]], <8 x i1> poison, <8 x i32> zeroinitializer
; SINK-GATHER-NEXT: br label %[[VECTOR_BODY:.*]]
; SINK-GATHER: [[VECTOR_BODY]]:
-; SINK-GATHER-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[PRED_SDIV_CONTINUE14:.*]] ]
-; SINK-GATHER-NEXT: [[VEC_PHI:%.*]] = phi <8 x i32> [ zeroinitializer, %[[VECTOR_PH]] ], [ [[TMP35:%.*]], %[[PRED_SDIV_CONTINUE14]] ]
+; SINK-GATHER-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[PRED_SDIV_CONTINUE15:.*]] ]
+; SINK-GATHER-NEXT: [[VEC_PHI:%.*]] = phi <8 x i32> [ zeroinitializer, %[[VECTOR_PH]] ], [ [[TMP35:%.*]], %[[PRED_SDIV_CONTINUE15]] ]
; SINK-GATHER-NEXT: [[TMP0:%.*]] = getelementptr inbounds i32, ptr [[A]], i64 [[INDEX]]
; SINK-GATHER-NEXT: [[WIDE_LOAD:%.*]] = load <8 x i32>, ptr [[TMP0]], align 4
; SINK-GATHER-NEXT: [[TMP1:%.*]] = getelementptr i32, ptr [[B]], i64 [[INDEX]]
-; SINK-GATHER-NEXT: [[WIDE_MASKED_LOAD:%.*]] = call <8 x i32> @llvm.masked.load.v8i32.p0(ptr align 4 [[TMP1]], <8 x i1> [[BROADCAST_SPLAT]], <8 x i32> poison)
-; SINK-GATHER-NEXT: br i1 [[C]], label %[[PRED_SDIV_IF:.*]], label %[[PRED_SDIV_CONTINUE:.*]]
-; SINK-GATHER: [[PRED_SDIV_IF]]:
-; SINK-GATHER-NEXT: [[TMP2:%.*]] = extractelement <8 x i32> [[WIDE_MASKED_LOAD]], i64 0
-; SINK-GATHER-NEXT: [[TMP3:%.*]] = sdiv i32 [[TMP2]], [[X]]
-; SINK-GATHER-NEXT: [[TMP4:%.*]] = insertelement <8 x i32> poison, i32 [[TMP3]], i64 0
-; SINK-GATHER-NEXT: br label %[[PRED_SDIV_CONTINUE]]
-; SINK-GATHER: [[PRED_SDIV_CONTINUE]]:
-; SINK-GATHER-NEXT: [[TMP5:%.*]] = phi <8 x i32> [ poison, %[[VECTOR_BODY]] ], [ [[TMP4]], %[[PRED_SDIV_IF]] ]
+; SINK-GATHER-NEXT: [[WIDE_LOAD2:%.*]] = call <8 x i32> @llvm.masked.load.v8i32.p0(ptr align 4 [[TMP1]], <8 x i1> [[BROADCAST_SPLAT]], <8 x i32> poison)
+; SINK-GATHER-NEXT: br i1 [[C]], label %[[PRED_SDIV_IF13:.*]], label %[[PRED_SDIV_CONTINUE14:.*]]
+; SINK-GATHER: [[PRED_SDIV_IF13]]:
+; SINK-GATHER-NEXT: [[TMP30:%.*]] = extractelement <8 x i32> [[WIDE_LOAD2]], i64 0
+; SINK-GATHER-NEXT: [[TMP31:%.*]] = sdiv i32 [[TMP30]], [[X]]
+; SINK-GATHER-NEXT: [[TMP4:%.*]] = insertelement <8 x i32> poison, i32 [[TMP31]], i64 0
+; SINK-GATHER-NEXT: br label %[[PRED_SDIV_CONTINUE14]]
+; SINK-GATHER: [[PRED_SDIV_CONTINUE14]]:
+; SINK-GATHER-NEXT: [[TMP5:%.*]] = phi <8 x i32> [ poison, %[[VECTOR_BODY]] ], [ [[TMP4]], %[[PRED_SDIV_IF13]] ]
; SINK-GATHER-NEXT: br i1 [[C]], label %[[PRED_SDIV_IF1:.*]], label %[[PRED_SDIV_CONTINUE2:.*]]
; SINK-GATHER: [[PRED_SDIV_IF1]]:
-; SINK-GATHER-NEXT: [[TMP6:%.*]] = extractelement <8 x i32> [[WIDE_MASKED_LOAD]], i64 1
+; SINK-GATHER-NEXT: [[TMP6:%.*]] = extractelement <8 x i32> [[WIDE_LOAD2]], i64 1
; SINK-GATHER-NEXT: [[TMP7:%.*]] = sdiv i32 [[TMP6]], [[X]]
; SINK-GATHER-NEXT: [[TMP8:%.*]] = insertelement <8 x i32> [[TMP5]], i32 [[TMP7]], i64 1
; SINK-GATHER-NEXT: br label %[[PRED_SDIV_CONTINUE2]]
; SINK-GATHER: [[PRED_SDIV_CONTINUE2]]:
-; SINK-GATHER-NEXT: [[TMP9:%.*]] = phi <8 x i32> [ [[TMP5]], %[[PRED_SDIV_CONTINUE]] ], [ [[TMP8]], %[[PRED_SDIV_IF1]] ]
+; SINK-GATHER-NEXT: [[TMP9:%.*]] = phi <8 x i32> [ [[TMP5]], %[[PRED_SDIV_CONTINUE14]] ], [ [[TMP8]], %[[PRED_SDIV_IF1]] ]
; SINK-GATHER-NEXT: br i1 [[C]], label %[[PRED_SDIV_IF3:.*]], label %[[PRED_SDIV_CONTINUE4:.*]]
; SINK-GATHER: [[PRED_SDIV_IF3]]:
-; SINK-GATHER-NEXT: [[TMP10:%.*]] = extractelement <8 x i32> [[WIDE_MASKED_LOAD]], i64 2
+; SINK-GATHER-NEXT: [[TMP10:%.*]] = extractelement <8 x i32> [[WIDE_LOAD2]], i64 2
; SINK-GATHER-NEXT: [[TMP11:%.*]] = sdiv i32 [[TMP10]], [[X]]
; SINK-GATHER-NEXT: [[TMP12:%.*]] = insertelement <8 x i32> [[TMP9]], i32 [[TMP11]], i64 2
; SINK-GATHER-NEXT: br label %[[PRED_SDIV_CONTINUE4]]
@@ -94,7 +94,7 @@ define i32 @predicated_sdiv_masked_load(ptr %a, ptr %b, i32 %x, i1 %c) {
; SINK-GATHER-NEXT: [[TMP13:%.*]] = phi <8 x i32> [ [[TMP9]], %[[PRED_SDIV_CONTINUE2]] ], [ [[TMP12]], %[[PRED_SDIV_IF3]] ]
; SINK-GATHER-NEXT: br i1 [[C]], label %[[PRED_SDIV_IF5:.*]], label %[[PRED_SDIV_CONTINUE6:.*]]
; SINK-GATHER: [[PRED_SDIV_IF5]]:
-; SINK-GATHER-NEXT: [[TMP14:%.*]] = extractelement <8 x i32> [[WIDE_MASKED_LOAD]], i64 3
+; SINK-GATHER-NEXT: [[TMP14:%.*]] = extractelement <8 x i32> [[WIDE_LOAD2]], i64 3
; SINK-GATHER-NEXT: [[TMP15:%.*]] = sdiv i32 [[TMP14]], [[X]]
; SINK-GATHER-NEXT: [[TMP16:%.*]] = insertelement <8 x i32> [[TMP13]], i32 [[TMP15]], i64 3
; SINK-GATHER-NEXT: br label %[[PRED_SDIV_CONTINUE6]]
@@ -102,7 +102,7 @@ define i32 @predicated_sdiv_masked_load(ptr %a, ptr %b, i32 %x, i1 %c) {
; SINK-GATHER-NEXT: [[TMP17:%.*]] = phi <8 x i32> [ [[TMP13]], %[[PRED_SDIV_CONTINUE4]] ], [ [[TMP16]], %[[PRED_SDIV_IF5]] ]
; SINK-GATHER-NEXT: br i1 [[C]], label %[[PRED_SDIV_IF7:.*]], label %[[PRED_SDIV_CONTINUE8:.*]]
; SINK-GATHER: [[PRED_SDIV_IF7]]:
-; SINK-GATHER-NEXT: [[TMP18:%.*]] = extractelement <8 x i32> [[WIDE_MASKED_LOAD]], i64 4
+; SINK-GATHER-NEXT: [[TMP18:%.*]] = extractelement <8 x i32> [[WIDE_LOAD2]], i64 4
; SINK-GATHER-NEXT: [[TMP19:%.*]] = sdiv i32 [[TMP18]], [[X]]
; SINK-GATHER-NEXT: [[TMP20:%.*]] = insertelement <8 x i32> [[TMP17]], i32 [[TMP19]], i64 4
; SINK-GATHER-NEXT: br label %[[PRED_SDIV_CONTINUE8]]
@@ -110,7 +110,7 @@ define i32 @predicated_sdiv_masked_load(ptr %a, ptr %b, i32 %x, i1 %c) {
; SINK-GATHER-NEXT: [[TMP21:%.*]] = phi <8 x i32> [ [[TMP17]], %[[PRED_SDIV_CONTINUE6]] ], [ [[TMP20]], %[[PRED_SDIV_IF7]] ]
; SINK-GATHER-NEXT: br i1 [[C]], label %[[PRED_SDIV_IF9:.*]], label %[[PRED_SDIV_CONTINUE10:.*]]
; SINK-GATHER: [[PRED_SDIV_IF9]]:
-; SINK-GATHER-NEXT: [[TMP22:%.*]] = extractelement <8 x i32> [[WIDE_MASKED_LOAD]], i64 5
+; SINK-GATHER-NEXT: [[TMP22:%.*]] = extractelement <8 x i32> [[WIDE_LOAD2]], i64 5
; SINK-GATHER-NEXT: [[TMP23:%.*]] = sdiv i32 [[TMP22]], [[X]]
; SINK-GATHER-NEXT: [[TMP24:%.*]] = insertelement <8 x i32> [[TMP21]], i32 [[TMP23]], i64 5
; SINK-GATHER-NEXT: br label %[[PRED_SDIV_CONTINUE10]]
@@ -118,20 +118,20 @@ define i32 @predicated_sdiv_masked_load(ptr %a, ptr %b, i32 %x, i1 %c) {
; SINK-GATHER-NEXT: [[TMP25:%.*]] = phi <8 x i32> [ [[TMP21]], %[[PRED_SDIV_CONTINUE8]] ], [ [[TMP24]], %[[PRED_SDIV_IF9]] ]
; SINK-GATHER-NEXT: br i1 [[C]], label %[[PRED_SDIV_IF11:.*]], label %[[PRED_SDIV_CONTINUE12:.*]]
; SINK-GATHER: [[PRED_SDIV_IF11]]:
-; SINK-GATHER-NEXT: [[TMP26:%.*]] = extractelement <8 x i32> [[WIDE_MASKED_LOAD]], i64 6
+; SINK-GATHER-NEXT: [[TMP26:%.*]] = extractelement <8 x i32> [[WIDE_LOAD2]], i64 6
; SINK-GATHER-NEXT: [[TMP27:%.*]] = sdiv i32 [[TMP26]], [[X]]
; SINK-GATHER-NEXT: [[TMP28:%.*]] = insertelement <8 x i32> [[TMP25]], i32 [[TMP27]], i64 6
; SINK-GATHER-NEXT: br label %[[PRED_SDIV_CONTINUE12]]
; SINK-GATHER: [[PRED_SDIV_CONTINUE12]]:
; SINK-GATHER-NEXT: [[TMP29:%.*]] = phi <8 x i32> [ [[TMP25]], %[[PRED_SDIV_CONTINUE10]] ], [ [[TMP28]], %[[PRED_SDIV_IF11]] ]
-; SINK-GATHER-NEXT: br i1 [[C]], label %[[PRED_SDIV_IF13:.*]], label %[[PRED_SDIV_CONTINUE14]]
-; SINK-GATHER: [[PRED_SDIV_IF13]]:
-; SINK-GATHER-NEXT: [[TMP30:%.*]] = extractelement <8 x i32> [[WIDE_MASKED_LOAD]], i64 7
-; SINK-GATHER-NEXT: [[TMP31:%.*]] = sdiv i32 [[TMP30]], [[X]]
-; SINK-GATHER-NEXT: [[TMP32:%.*]] = insertelement <8 x i32> [[TMP29]], i32 [[TMP31]], i64 7
-; SINK-GATHER-NEXT: br label %[[PRED_SDIV_CONTINUE14]]
-; SINK-GATHER: [[PRED_SDIV_CONTINUE14]]:
-; SINK-GATHER-NEXT: [[TMP33:%.*]] = phi <8 x i32> [ [[TMP29]], %[[PRED_SDIV_CONTINUE12]] ], [ [[TMP32]], %[[PRED_SDIV_IF13]] ]
+; SINK-GATHER-NEXT: br i1 [[C]], label %[[PRED_SDIV_IF14:.*]], label %[[PRED_SDIV_CONTINUE15]]
+; SINK-GATHER: [[PRED_SDIV_IF14]]:
+; SINK-GATHER-NEXT: [[TMP38:%.*]] = extractelement <8 x i32> [[WIDE_LOAD2]], i64 7
+; SINK-GATHER-NEXT: [[TMP39:%.*]] = sdiv i32 [[TMP38]], [[X]]
+; SINK-GATHER-NEXT: [[TMP32:%.*]] = insertelement <8 x i32> [[TMP29]], i32 [[TMP39]], i64 7
+; SINK-GATHER-NEXT: br label %[[PRED_SDIV_CONTINUE15]]
+; SINK-GATHER: [[PRED_SDIV_CONTINUE15]]:
+; SINK-GATHER-NEXT: [[TMP33:%.*]] = phi <8 x i32> [ [[TMP29]], %[[PRED_SDIV_CONTINUE12]] ], [ [[TMP32]], %[[PRED_SDIV_IF14]] ]
; SINK-GATHER-NEXT: [[TMP34:%.*]] = add nsw <8 x i32> [[TMP33]], [[WIDE_LOAD]]
; SINK-GATHER-NEXT: [[PREDPHI:%.*]] = select i1 [[C]], <8 x i32> [[TMP34]], <8 x i32> [[WIDE_LOAD]]
; SINK-GATHER-NEXT: [[TMP35]] = add <8 x i32> [[VEC_PHI]], [[PREDPHI]]
@@ -195,25 +195,25 @@ define i32 @scalarize_and_sink_gather(ptr %a, i1 %c, i32 %x, i64 %n) {
; CHECK-NEXT: br i1 [[C]], label %[[PRED_UDIV_IF:.*]], label %[[PRED_UDIV_CONTINUE:.*]]
; CHECK: [[PRED_UDIV_IF]]:
; CHECK-NEXT: [[TMP1:%.*]] = mul i64 [[INDEX]], 777
-; CHECK-NEXT: [[TMP2:%.*]] = getelementptr inbounds i32, ptr [[A]], i64 [[TMP1]]
-; CHECK-NEXT: [[TMP3:%.*]] = load i32, ptr [[TMP2]], align 4
-; CHECK-NEXT: [[TMP4:%.*]] = udiv i32 [[TMP3]], [[X]]
-; CHECK-NEXT: [[TMP5:%.*]] = insertelement <2 x i32> poison, i32 [[TMP4]], i64 0
+; CHECK-NEXT: [[TMP3:%.*]] = getelementptr inbounds i32, ptr [[A]], i64 [[TMP1]]
+; CHECK-NEXT: [[TMP4:%.*]] = load i32, ptr [[TMP3]], align 4
+; CHECK-NEXT: [[TMP5:%.*]] = udiv i32 [[TMP4]], [[X]]
+; CHECK-NEXT: [[TMP6:%.*]] = insertelement <2 x i32> poison, i32 [[TMP5]], i64 0
; CHECK-NEXT: br label %[[PRED_UDIV_CONTINUE]]
; CHECK: [[PRED_UDIV_CONTINUE]]:
-; CHECK-NEXT: [[TMP6:%.*]] = phi <2 x i32> [ poison, %[[VECTOR_BODY]] ], [ [[TMP5]], %[[PRED_UDIV_IF]] ]
+; CHECK-NEXT: [[TMP7:%.*]] = phi <2 x i32> [ poison, %[[VECTOR_BODY]] ], [ [[TMP6]], %[[PRED_UDIV_IF]] ]
; CHECK-NEXT: br i1 [[C]], label %[[PRED_UDIV_IF1:.*]], label %[[PRED_UDIV_CONTINUE2]]
; CHECK: [[PRED_UDIV_IF1]]:
-; CHECK-NEXT: [[TMP7:%.*]] = add i64 [[INDEX]], 1
-; CHECK-NEXT: [[TMP8:%.*]] = mul i64 [[TMP7]], 777
-; CHECK-NEXT: [[TMP9:%.*]] = getelementptr inbounds i32, ptr [[A]], i64 [[TMP8]]
+; CHECK-NEXT: [[TMP8:%.*]] = add i64 [[INDEX]], 1
+; CHECK-NEXT: [[TMP12:%.*]] = mul i64 [[TMP8]], 777
+; CHECK-NEXT: [[TMP9:%.*]] = getelementptr inbounds i32, ptr [[A]], i64 [[TMP12]]
; CHECK-NEXT: [[TMP10:%.*]] = load i32, ptr [[TMP9]], align 4
; CHECK-NEXT: [[TMP11:%.*]] = udiv i32 [[TMP10]], [[X]]
-; CHECK-NEXT: [[TMP12:%.*]] = insertelement <2 x i32> [[TMP6]], i32 [[TMP11]], i64 1
+; CHECK-NEXT: [[TMP13:%.*]] = insertelement <2 x i32> [[TMP7]], i32 [[TMP11]], i64 1
; CHECK-NEXT: br label %[[PRED_UDIV_CONTINUE2]]
; CHECK: [[PRED_UDIV_CONTINUE2]]:
-; CHECK-NEXT: [[TMP13:%.*]] = phi <2 x i32> [ [[TMP6]], %[[PRED_UDIV_CONTINUE]] ], [ [[TMP12]], %[[PRED_UDIV_IF1]] ]
-; CHECK-NEXT: [[PREDPHI:%.*]] = select i1 [[C]], <2 x i32> [[TMP13]], <2 x i32> [[BROADCAST_SPLAT]]
+; CHECK-NEXT: [[TMP17:%.*]] = phi <2 x i32> [ [[TMP7]], %[[PRED_UDIV_CONTINUE]] ], [ [[TMP13]], %[[PRED_UDIV_IF1]] ]
+; CHECK-NEXT: [[PREDPHI:%.*]] = select i1 [[C]], <2 x i32> [[TMP17]], <2 x i32> [[BROADCAST_SPLAT]]
; CHECK-NEXT: [[TMP14]] = add <2 x i32> [[VEC_PHI]], [[PREDPHI]]
; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 2
; CHECK-NEXT: [[TMP15:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
@@ -266,84 +266,84 @@ define i32 @scalarize_and_sink_gather(ptr %a, i1 %c, i32 %x, i64 %n) {
; SINK-GATHER-NEXT: br i1 [[C]], label %[[PRED_UDIV_IF:.*]], label %[[PRED_UDIV_CONTINUE:.*]]
; SINK-GATHER: [[PRED_UDIV_IF]]:
; SINK-GATHER-NEXT: [[TMP1:%.*]] = extractelement <8 x i64> [[TMP0]], i64 0
-; SINK-GATHER-NEXT: [[TMP2:%.*]] = getelementptr inbounds i32, ptr [[A]], i64 [[TMP1]]
-; SINK-GATHER-NEXT: [[TMP3:%.*]] = load i32, ptr [[TMP2]], align 4
-; SINK-GATHER-NEXT: [[TMP4:%.*]] = udiv i32 [[TMP3]], [[X]]
-; SINK-GATHER-NEXT: [[TMP5:%.*]] = insertelement <8 x i32> poison, i32 [[TMP4]], i64 0
+; SINK-GATHER-NEXT: [[TMP44:%.*]] = getelementptr inbounds i32, ptr [[A]], i64 [[TMP1]]
+; SINK-GATHER-NEXT: [[TMP45:%.*]] = load i32, ptr [[TMP44]], align 4
+; SINK-GATHER-NEXT: [[TMP46:%.*]] = udiv i32 [[TMP45]], [[X]]
+; SINK-GATHER-NEXT: [[TMP7:%.*]] = insertelement <8 x i32> poison, i32 [[TMP46]], i64 0
; SINK-GATHER-NEXT: br label %[[PRED_UDIV_CONTINUE]]
; SINK-GATHER: [[PRED_UDIV_CONTINUE]]:
-; SINK-GATHER-NEXT: [[TMP6:%.*]] = phi <8 x i32> [ poison, %[[VECTOR_BODY]] ], [ [[TMP5]], %[[PRED_UDIV_IF]] ]
+; SINK-GATHER-NEXT: [[TMP8:%.*]] = phi <8 x i32> [ poison, %[[VECTOR_BODY]] ], [ [[TMP7]], %[[PRED_UDIV_IF]] ]
; SINK-GATHER-NEXT: br i1 [[C]], label %[[PRED_UDIV_IF1:.*]], label %[[PRED_UDIV_CONTINUE2:.*]]
; SINK-GATHER: [[PRED_UDIV_IF1]]:
-; SINK-GATHER-NEXT: [[TMP7:%.*]] = extractelement <8 x i64> [[TMP0]], i64 1
-; SINK-GATHER-NEXT: [[TMP8:%.*]] = getelementptr inbounds i32, ptr [[A]], i64 [[TMP7]]
-; SINK-GATHER-NEXT: [[TMP9:%.*]] = load i32, ptr [[TMP8]], align 4
-; SINK-GATHER-NEXT: [[TMP10:%.*]] = udiv i32 [[TMP9]], [[X]]
-; SINK-GATHER-NEXT: [[TMP11:%.*]] = insertelement <8 x i32> [[TMP6]], i32 [[TMP10]], i64 1
+; SINK-GATHER-NEXT: [[TMP9:%.*]] = extractelement <8 x i64> [[TMP0]], i64 1
+; SINK-GATHER-NEXT: [[TMP10:%.*]] = getelementptr inbounds i32, ptr [[A]], i64 [[TMP9]]
+; SINK-GATHER-NEXT: [[TMP11:%.*]] = load i32, ptr [[TMP10]], align 4
+; SINK-GATHER-NEXT: [[TMP12:%.*]] = udiv i32 [[TMP11]], [[X]]
+; SINK-GATHER-NEXT: [[TMP13:%.*]] = insertelement <8 x i32> [[TMP8]], i32 [[TMP12]], i64 1
; SINK-GATHER-NEXT: br label %[[PRED_UDIV_CONTINUE2]]
; SINK-GATHER: [[PRED_UDIV_CONTINUE2]]:
-; SINK-GATHER-NEXT: [[TMP12:%.*]] = phi <8 x i32> [ [[TMP6]], %[[PRED_UDIV_CONTINUE]] ], [ [[TMP11]], %[[PRED_UDIV_IF1]] ]
+; SINK-GATHER-NEXT: [[TMP14:%.*]] = phi <8 x i32> [ [[TMP8]], %[[PRED_UDIV_CONTINUE]] ], [ [[TMP13]], %[[PRED_UDIV_IF1]] ]
; SINK-GATHER-NEXT: br i1 [[C]], label %[[PRED_UDIV_IF3:.*]], label %[[PRED_UDIV_CONTINUE4:.*]]
; SINK-GATHER: [[PRED_UDIV_IF3]]:
-; SINK-GATHER-NEXT: [[TMP13:%.*]] = extractelement <8 x i64> [[TMP0]], i64 2
-; SINK-GATHER-NEXT: [[TMP14:%.*]] = getelementptr inbounds i32, ptr [[A]], i64 [[TMP13]]
-; SINK-GATHER-NEXT: [[TMP15:%.*]] = load i32, ptr [[TMP14]], align 4
-; SINK-GATHER-NEXT: [[TMP16:%.*]] = udiv i32 [[TMP15]], [[X]]
-; SINK-GATHER-NEXT: [[TMP17:%.*]] = insertelement <8 x i32> [[TMP12]], i32 [[TMP16]], i64 2
+; SINK-GATHER-NEXT: [[TMP15:%.*]] = extractelement <8 x i64> [[TMP0]], i64 2
+; SINK-GATHER-NEXT: [[TMP16:%.*]] = getelementptr inbounds i32, ptr [[A]], i64 [[TMP15]]
+; SINK-GATHER-NEXT: [[TMP17:%.*]] = load i32, ptr [[TMP16]], align 4
+; SINK-GATHER-NEXT: [[TMP18:%.*]] = udiv i32 [[TMP17]], [[X]]
+; SINK-GATHER-NEXT: [[TMP19:%.*]] = insertelement <8 x i32> [[TMP14]], i32 [[TMP18]], i64 2
; SINK-GATHER-NEXT: br label %[[PRED_UDIV_CONTINUE4]]
; SINK-GATHER: [[PRED_UDIV_CONTINUE4]]:
-; SINK-GATHER-NEXT: [[TMP18:%.*]] = phi <8 x i32> [ [[TMP12]], %[[PRED_UDIV_CONTINUE2]] ], [ [[TMP17]], %[[PRED_UDIV_IF3]] ]
+; SINK-GATHER-NEXT: [[TMP20:%.*]] = phi <8 x i32> [ [[TMP14]], %[[PRED_UDIV_CONTINUE2]] ], [ [[TMP19]], %[[PRED_UDIV_IF3]] ]
; SINK-GATHER-NEXT: br i1 [[C]], label %[[PRED_UDIV_IF5:.*]], label %[[PRED_UDIV_CONTINUE6:.*]]
; SINK-GATHER: [[PRED_UDIV_IF5]]:
-; SINK-GATHER-NEXT: [[TMP19:%.*]] = extractelement <8 x i64> [[TMP0]], i64 3
-; SINK-GATHER-NEXT: [[TMP20:%.*]] = getelementptr inbounds i32, ptr [[A]], i64 [[TMP19]]
-; SINK-GATHER-NEXT: [[TMP21:%.*]] = load i32, ptr [[TMP20]], align 4
-; SINK-GATHER-NEXT: [[TMP22:%.*]] = udiv i32 [[TMP21]], [[X]]
-; SINK-GATHER-NEXT: [[TMP23:%.*]] = insertelement <8 x i32> [[TMP18]], i32 [[TMP22]], i64 3
+; SINK-GATHER-NEXT: [[TMP21:%.*]] = extractelement <8 x i64> [[TMP0]], i64 3
+; SINK-GATHER-NEXT: [[TMP22:%.*]] = getelementptr inbounds i32, ptr [[A]], i64 [[TMP21]]
+; SINK-GATHER-NEXT: [[TMP23:%.*]] = load i32, ptr [[TMP22]], align 4
+; SINK-GATHER-NEXT: [[TMP24:%.*]] = udiv i32 [[TMP23]], [[X]]
+; SINK-GATHER-NEXT: [[TMP25:%.*]] = insertelement <8 x i32> [[TMP20]], i32 [[TMP24]], i64 3
; SINK-GATHER-NEXT: br label %[[PRED_UDIV_CONTINUE6]]
; SINK-GATHER: [[PRED_UDIV_CONTINUE6]]:
-; SINK-GATHER-NEXT: [[TMP24:%.*]] = phi <8 x i32> [ [[TMP18]], %[[PRED_UDIV_CONTINUE4]] ], [ [[TMP23]], %[[PRED_UDIV_IF5]] ]
+; SINK-GATHER-NEXT: [[TMP26:%.*]] = phi <8 x i32> [ [[TMP20]], %[[PRED_UDIV_CONTINUE4]] ], [ [[TMP25]], %[[PRED_UDIV_IF5]] ]
; SINK-GATHER-NEXT: br i1 [[C]], label %[[PRED_UDIV_IF7:.*]], label %[[PRED_UDIV_CONTINUE8:.*]]
; SINK-GATHER: [[PRED_UDIV_IF7]]:
-; SINK-GATHER-NEXT: [[TMP25:%.*]] = extractelement <8 x i64> [[TMP0]], i64 4
-; SINK-GATHER-NEXT: [[TMP26:%.*]] = getelementptr inbounds i32, ptr [[A]], i64 [[TMP25]]
-; SINK-GATHER-NEXT: [[TMP27:%.*]] = load i32, ptr [[TMP26]], align 4
-; SINK-GATHER-NEXT: [[TMP28:%.*]] = udiv i32 [[TMP27]], [[X]]
-; SINK-GATHER-NEXT: [[TMP29:%.*]] = insertelement <8 x i32> [[TMP24]], i32 [[TMP28]], i64 4
+; SINK-GATHER-NEXT: [[TMP27:%.*]] = extractelement <8 x i64> [[TMP0]], i64 4
+; SINK-GATHER-NEXT: [[TMP28:%.*]] = getelementptr inbounds i32, ptr [[A]], i64 [[TMP27]]
+; SINK-GATHER-NEXT: [[TMP29:%.*]] = load i32, ptr [[TMP28]], align 4
+; SINK-GATHER-NEXT: [[TMP30:%.*]] = udiv i32 [[TMP29]], [[X]]
+; SINK-GATHER-NEXT: [[TMP31:%.*]] = insertelement <8 x i32> [[TMP26]], i32 [[TMP30]], i64 4
; SINK-GATHER-NEXT: br label %[[PRED_UDIV_CONTINUE8]]
; SINK-GATHER: [[PRED_UDIV_CONTINUE8]]:
-; SINK-GATHER-NEXT: [[TMP30:%.*]] = phi <8 x i32> [ [[TMP24]], %[[PRED_UDIV_CONTINUE6]] ], [ [[TMP29]], %[[PRED_UDIV_IF7]] ]
+; SINK-GATHER-NEXT: [[TMP32:%.*]] = phi <8 x i32> [ [[TMP26]], %[[PRED_UDIV_CONTINUE6]] ], [ [[TMP31]], %[[PRED_UDIV_IF7]] ]
; SINK-GATHER-NEXT: br i1 [[C]], label %[[PRED_UDIV_IF9:.*]], label %[[PRED_UDIV_CONTINUE10:.*]]
; SINK-GATHER: [[PRED_UDIV_IF9]]:
-; SINK-GATHER-NEXT: [[TMP31:%.*]] = extractelement <8 x i64> [[TMP0]], i64 5
-; SINK-GATHER-NEXT: [[TMP32:%.*]] = getelementptr inbounds i32, ptr [[A]], i64 [[TMP31]]
-; SINK-GATHER-NEXT: [[TMP33:%.*]] = load i32, ptr [[TMP32]], align 4
-; SINK-GATHER-NEXT: [[TMP34:%.*]] = udiv i32 [[TMP33]], [[X]]
-; SINK-GATHER-NEXT: [[TMP35:%.*]] = insertelement <8 x i32> [[TMP30]], i32 [[TMP34]], i64 5
+; SINK-GATHER-NEXT: [[TMP33:%.*]] = extractelement <8 x i64> [[TMP0]], i64 5
+; SINK-GATHER-NEXT: [[TMP34:%.*]] = getelementptr inbounds i32, ptr [[A]], i64 [[TMP33]]
+; SINK-GATHER-NEXT: [[TMP35:%.*]] = load i32, ptr [[TMP34]], align 4
+; SINK-GATHER-NEXT: [[TMP36:%.*]] = udiv i32 [[TMP35]], [[X]]
+; SINK-GATHER-NEXT: [[TMP37:%.*]] = insertelement <8 x i32> [[TMP32]], i32 [[TMP36]], i64 5
; SINK-GATHER-NEXT: br label %[[PRED_UDIV_CONTINUE10]]
; SINK-GATHER: [[PRED_UDIV_CONTINUE10]]:
-; SINK-GATHER-NEXT: [[TMP36:%.*]] = phi <8 x i32> [ [[TMP30]], %[[PRED_UDIV_CONTINUE8]] ], [ [[TMP35]], %[[PRED_UDIV_IF9]] ]
+; SINK-GATHER-NEXT: [[TMP38:%.*]] = phi <8 x i32> [ [[TMP32]], %[[PRED_UDIV_CONTINUE8]] ], [ [[TMP37]], %[[PRED_UDIV_IF9]] ]
; SINK-GATHER-NEXT: br i1 [[C]], label %[[PRED_UDIV_IF11:.*]], label %[[PRED_UDIV_CONTINUE12:.*]]
; SINK-GATHER: [[PRED_UDIV_IF11]]:
-; SINK-GATHER-NEXT: [[TMP37:%.*]] = extractelement <8 x i64> [[TMP0]], i64 6
-; SINK-GATHER-NEXT: [[TMP38:%.*]] = getelementptr inbounds i32, ptr [[A]], i64 [[TMP37]]
-; SINK-GATHER-NEXT: [[TMP39:%.*]] = load i32, ptr [[TMP38]], align 4
-; SINK-GATHER-NEXT: [[TMP40:%.*]] = udiv i32 [[TMP39]], [[X]]
-; SINK-GATHER-NEXT: [[TMP41:%.*]] = insertelement <8 x i32> [[TMP36]], i32 [[TMP40]], i64 6
+; SINK-GATHER-NEXT: [[TMP39:%.*]] = extractelement <8 x i64> [[TMP0]], i64 6
+; SINK-GATHER-NEXT: [[TMP40:%.*]] = getelementptr inbounds i32, ptr [[A]], i64 [[TMP39]]
+; SINK-GATHER-NEXT: [[TMP41:%.*]] = load i32, ptr [[TMP40]], align 4
+; SINK-GATHER-NEXT: [[TMP42:%.*]] = udiv i32 [[TMP41]], [[X]]
+; SINK-GATHER-NEXT: [[TMP43:%.*]] = insertelement <8 x i32> [[TMP38]], i32 [[TMP42]], i64 6
; SINK-GATHER-NEXT: br label %[[PRED_UDIV_CONTINUE12]]
; SINK-GATHER: [[PRED_UDIV_CONTINUE12]]:
-; SINK-GATHER-NEXT: [[TMP42:%.*]] = phi <8 x i32> [ [[TMP36]], %[[PRED_UDIV_CONTINUE10]] ], [ [[TMP41]], %[[PRED_UDIV_IF11]] ]
+; SINK-GATHER-NEXT: [[TMP52:%.*]] = phi <8 x i32> [ [[TMP38]], %[[PRED_UDIV_CONTINUE10]] ], [ [[TMP43]], %[[PRED_UDIV_IF11]] ]
; SINK-GATHER-NEXT: br i1 [[C]], label %[[PRED_UDIV_IF13:.*]], label %[[PRED_UDIV_CONTINUE14]]
; SINK-GATHER: [[PRED_UDIV_IF13]]:
-; SINK-GATHER-NEXT: [[TMP43:%.*]] = extractelement <8 x i64> [[TMP0]], i64 7
-; SINK-GATHER-NEXT: [[TMP44:%.*]] = getelementptr inbounds i32, ptr [[A]], i64 [[TMP43]]
-; SINK-GATHER-NEXT: [[TMP45:%.*]] = load i32, ptr [[TMP44]], align 4
-; SINK-GATHER-NEXT: [[TMP46:%.*]] = udiv i32 [[TMP45]], [[X]]
-; SINK-GATHER-NEXT: [[TMP47:%.*]] = insertelement <8 x i32> [[TMP42]], i32 [[TMP46]], i64 7
+; SINK-GATHER-NEXT: [[TMP53:%.*]] = extractelement <8 x i64> [[TMP0]], i64 7
+; SINK-GATHER-NEXT: [[TMP54:%.*]] = getelementptr inbounds i32, ptr [[A]], i64 [[TMP53]]
+; SINK-GATHER-NEXT: [[TMP47:%.*]] = load i32, ptr [[TMP54]], align 4
+; SINK-GATHER-NEXT: [[TMP48:%.*]] = udiv i32 [[TMP47]], [[X]]
+; SINK-GATHER-NEXT: [[TMP55:%.*]] = insertelement <8 x i32> [[TMP52]], i32 [[TMP48]], i64 7
; SINK-GATHER-NEXT: br label %[[PRED_UDIV_CONTINUE14]]
; SINK-GATHER: [[PRED_UDIV_CONTINUE14]]:
-; SINK-GATHER-NEXT: [[TMP48:%.*]] = phi <8 x i32> [ [[TMP42]], %[[PRED_UDIV_CONTINUE12]] ], [ [[TMP47]], %[[PRED_UDIV_IF13]] ]
-; SINK-GATHER-NEXT: [[PREDPHI:%.*]] = select i1 [[C]], <8 x i32> [[TMP48]], <8 x i32> [[BROADCAST_SPLAT]]
+; SINK-GATHER-NEXT: [[TMP56:%.*]] = phi <8 x i32> [ [[TMP52]], %[[PRED_UDIV_CONTINUE12]] ], [ [[TMP55]], %[[PRED_UDIV_IF13]] ]
+; SINK-GATHER-NEXT: [[PREDPHI:%.*]] = select i1 [[C]], <8 x i32> [[TMP56]], <8 x i32> [[BROADCAST_SPLAT]]
; SINK-GATHER-NEXT: [[TMP49]] = add <8 x i32> [[VEC_PHI]], [[PREDPHI]]
; SINK-GATHER-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 8
; SINK-GATHER-NEXT: [[VEC_IND_NEXT]] = add nuw nsw <8 x i64> [[VEC_IND]], splat (i64 8)
diff --git a/llvm/test/Transforms/LoopVectorize/blend-in-header.ll b/llvm/test/Transforms/LoopVectorize/blend-in-header.ll
index 6f262109f95be..153011444af8c 100644
--- a/llvm/test/Transforms/LoopVectorize/blend-in-header.ll
+++ b/llvm/test/Transforms/LoopVectorize/blend-in-header.ll
@@ -12,7 +12,11 @@ define i64 @pr88297() {
; CHECK: vector.ph:
; CHECK-NEXT: br label [[VECTOR_BODY:%.*]]
; CHECK: vector.body:
-; CHECK-NEXT: [[INDEX:%.*]] = phi i32 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ]
+; CHECK-NEXT: [[INDEX:%.*]] = phi i32 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[LOOP_LATCH2:%.*]] ]
+; CHECK-NEXT: br label [[THEN1:%.*]]
+; CHECK: then1:
+; CHECK-NEXT: br label [[LOOP_LATCH2]]
+; CHECK: loop.latch2:
; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i32 [[INDEX]], 4
; CHECK-NEXT: [[TMP0:%.*]] = icmp eq i32 [[INDEX_NEXT]], 1000
; CHECK-NEXT: br i1 [[TMP0]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP0:![0-9]+]]
@@ -62,7 +66,11 @@ define i64 @pr88297_incoming_ops_reordered() {
; CHECK: vector.ph:
; CHECK-NEXT: br label [[VECTOR_BODY:%.*]]
; CHECK: vector.body:
-; CHECK-NEXT: [[INDEX:%.*]] = phi i32 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ]
+; CHECK-NEXT: [[INDEX:%.*]] = phi i32 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[LOOP_LATCH2:%.*]] ]
+; CHECK-NEXT: br label [[THEN1:%.*]]
+; CHECK: then1:
+; CHECK-NEXT: br label [[LOOP_LATCH2]]
+; CHECK: loop.latch2:
; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i32 [[INDEX]], 4
; CHECK-NEXT: [[TMP0:%.*]] = icmp eq i32 [[INDEX_NEXT]], 1000
; CHECK-NEXT: br i1 [[TMP0]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP4:![0-9]+]]
@@ -113,7 +121,11 @@ define i64 @invar_cond(i1 %c) {
; CHECK: vector.ph:
; CHECK-NEXT: br label [[VECTOR_BODY:%.*]]
; CHECK: vector.body:
-; CHECK-NEXT: [[INDEX:%.*]] = phi i32 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ]
+; CHECK-NEXT: [[INDEX:%.*]] = phi i32 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[LOOP_LATCH2:%.*]] ]
+; CHECK-NEXT: br i1 [[C]], label [[LOOP_LATCH2]], label [[THEN1:%.*]]
+; CHECK: then1:
+; CHECK-NEXT: br label [[LOOP_LATCH2]]
+; CHECK: loop.latch2:
; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i32 [[INDEX]], 4
; CHECK-NEXT: [[TMP0:%.*]] = icmp eq i32 [[INDEX_NEXT]], 1000
; CHECK-NEXT: br i1 [[TMP0]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP6:![0-9]+]]
@@ -164,7 +176,11 @@ define i64 @invar_cond_incoming_ops_reordered(i1 %c) {
; CHECK: vector.ph:
; CHECK-NEXT: br label [[VECTOR_BODY:%.*]]
; CHECK: vector.body:
-; CHECK-NEXT: [[INDEX:%.*]] = phi i32 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ]
+; CHECK-NEXT: [[INDEX:%.*]] = phi i32 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[LOOP_LATCH2:%.*]] ]
+; CHECK-NEXT: br i1 [[C]], label [[LOOP_LATCH2]], label [[THEN1:%.*]]
+; CHECK: then1:
+; CHECK-NEXT: br label [[LOOP_LATCH2]]
+; CHECK: loop.latch2:
; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i32 [[INDEX]], 4
; CHECK-NEXT: [[TMP0:%.*]] = icmp eq i32 [[INDEX_NEXT]], 1000
; CHECK-NEXT: br i1 [[TMP0]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP8:![0-9]+]]
diff --git a/llvm/test/Transforms/LoopVectorize/constant-fold-commutative-and.ll b/llvm/test/Transforms/LoopVectorize/constant-fold-commutative-and.ll
index ebb59122b6458..cd7592765603c 100644
--- a/llvm/test/Transforms/LoopVectorize/constant-fold-commutative-and.ll
+++ b/llvm/test/Transforms/LoopVectorize/constant-fold-commutative-and.ll
@@ -11,26 +11,25 @@ define void @constant_fold_commutative_and(ptr %ptr.n, ptr noalias %p, i1 %cond)
; CHECK-NEXT: [[ENTRY:.*:]]
; CHECK-NEXT: br label %[[VECTOR_PH:.*]]
; CHECK: [[VECTOR_PH]]:
-; CHECK-NEXT: [[BROADCAST_SPLATINSERT:%.*]] = insertelement <2 x i1> poison, i1 [[COND]], i64 0
-; CHECK-NEXT: [[BROADCAST_SPLAT:%.*]] = shufflevector <2 x i1> [[BROADCAST_SPLATINSERT]], <2 x i1> poison, <2 x i32> zeroinitializer
-; CHECK-NEXT: [[TMP0:%.*]] = xor <2 x i1> [[BROADCAST_SPLAT]], splat (i1 true)
; CHECK-NEXT: br label %[[VECTOR_BODY:.*]]
; CHECK: [[VECTOR_BODY]]:
; CHECK-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[PRED_STORE_CONTINUE5:.*]] ]
; CHECK-NEXT: [[VEC_IND:%.*]] = phi <2 x i8> [ <i8 0, i8 1>, %[[VECTOR_PH]] ], [ [[VEC_IND_NEXT:%.*]], %[[PRED_STORE_CONTINUE5]] ]
; CHECK-NEXT: [[TMP1:%.*]] = icmp ule <2 x i8> [[VEC_IND]], splat (i8 16)
; CHECK-NEXT: [[TMP2:%.*]] = load i64, ptr [[PTR_N]], align 4
-; CHECK-NEXT: [[BROADCAST_SPLATINSERT1:%.*]] = insertelement <2 x i64> poison, i64 [[TMP2]], i64 0
-; CHECK-NEXT: [[BROADCAST_SPLAT2:%.*]] = shufflevector <2 x i64> [[BROADCAST_SPLATINSERT1]], <2 x i64> poison, <2 x i32> zeroinitializer
-; CHECK-NEXT: [[TMP3:%.*]] = icmp uge <2 x i64> [[BROADCAST_SPLAT2]], splat (i64 4)
-; CHECK-NEXT: [[TMP5:%.*]] = select <2 x i1> [[TMP0]], <2 x i1> [[TMP3]], <2 x i1> zeroinitializer
-; CHECK-NEXT: [[TMP7:%.*]] = icmp ult <2 x i64> [[BROADCAST_SPLAT2]], splat (i64 7)
-; CHECK-NEXT: [[PREDPHI3:%.*]] = select <2 x i1> [[TMP5]], <2 x i1> [[TMP7]], <2 x i1> [[TMP0]]
+; CHECK-NEXT: br i1 [[COND]], label %[[LATCH2:.*]], label %[[PRED_11:.*]]
+; CHECK: [[PRED_11]]:
+; CHECK-NEXT: [[TMP7:%.*]] = icmp uge i64 [[TMP2]], 4
+; CHECK-NEXT: [[TMP3:%.*]] = icmp ult i64 [[TMP2]], 7
+; CHECK-NEXT: br label %[[LATCH2]]
+; CHECK: [[LATCH2]]:
+; CHECK-NEXT: [[TMP4:%.*]] = phi i1 [ poison, %[[VECTOR_BODY]] ], [ [[TMP3]], %[[PRED_11]] ]
+; CHECK-NEXT: [[TMP5:%.*]] = phi i1 [ false, %[[VECTOR_BODY]] ], [ [[TMP7]], %[[PRED_11]] ]
+; CHECK-NEXT: [[TMP9:%.*]] = select i1 [[TMP5]], i1 [[TMP4]], i1 true
; CHECK-NEXT: [[TMP6:%.*]] = extractelement <2 x i1> [[TMP1]], i64 0
; CHECK-NEXT: br i1 [[TMP6]], label %[[PRED_STORE_IF:.*]], label %[[PRED_STORE_CONTINUE:.*]]
; CHECK: [[PRED_STORE_IF]]:
; CHECK-NEXT: [[TMP8:%.*]] = getelementptr i8, ptr [[P]], i64 [[INDEX]]
-; CHECK-NEXT: [[TMP9:%.*]] = extractelement <2 x i1> [[PREDPHI3]], i64 0
; CHECK-NEXT: store i1 [[TMP9]], ptr [[TMP8]], align 1
; CHECK-NEXT: br label %[[PRED_STORE_CONTINUE]]
; CHECK: [[PRED_STORE_CONTINUE]]:
@@ -39,8 +38,7 @@ define void @constant_fold_commutative_and(ptr %ptr.n, ptr noalias %p, i1 %cond)
; CHECK: [[PRED_STORE_IF4]]:
; CHECK-NEXT: [[TMP11:%.*]] = add i64 [[INDEX]], 1
; CHECK-NEXT: [[TMP20:%.*]] = getelementptr i8, ptr [[P]], i64 [[TMP11]]
-; CHECK-NEXT: [[TMP21:%.*]] = extractelement <2 x i1> [[PREDPHI3]], i64 1
-; CHECK-NEXT: store i1 [[TMP21]], ptr [[TMP20]], align 1
+; CHECK-NEXT: store i1 [[TMP9]], ptr [[TMP20]], align 1
; CHECK-NEXT: br label %[[PRED_STORE_CONTINUE5]]
; CHECK: [[PRED_STORE_CONTINUE5]]:
; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 2
diff --git a/llvm/test/Transforms/LoopVectorize/constantfolder.ll b/llvm/test/Transforms/LoopVectorize/constantfolder.ll
index 861515e4a01b4..918075586224c 100644
--- a/llvm/test/Transforms/LoopVectorize/constantfolder.ll
+++ b/llvm/test/Transforms/LoopVectorize/constantfolder.ll
@@ -7,14 +7,17 @@ define void @const_fold_ptradd(ptr %dst, i64 %d) {
; CHECK-NEXT: [[ENTRY:.*:]]
; CHECK-NEXT: br label %[[VECTOR_PH:.*]]
; CHECK: [[VECTOR_PH]]:
+; CHECK-NEXT: [[TMP0:%.*]] = getelementptr i16, ptr [[DST]], i64 [[D]]
; CHECK-NEXT: br label %[[VECTOR_BODY:.*]]
; CHECK: [[VECTOR_BODY]]:
-; CHECK-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[LOOP_LATCH1:.*]] ]
+; CHECK-NEXT: br label %[[LOOP_LATCH1]]
+; CHECK: [[LOOP_LATCH1]]:
+; CHECK-NEXT: store i16 0, ptr [[TMP0]], align 2
; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 4
; CHECK-NEXT: [[TMP1:%.*]] = icmp eq i64 [[INDEX_NEXT]], 100
; CHECK-NEXT: br i1 [[TMP1]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP0:![0-9]+]]
; CHECK: [[MIDDLE_BLOCK]]:
-; CHECK-NEXT: store i16 0, ptr [[DST]], align 2
; CHECK-NEXT: br label %[[EXIT:.*]]
; CHECK: [[EXIT]]:
; CHECK-NEXT: ret void
@@ -47,14 +50,17 @@ define void @const_fold_inbounds_ptradd(ptr %dst, i64 %d) {
; CHECK-NEXT: [[ENTRY:.*:]]
; CHECK-NEXT: br label %[[VECTOR_PH:.*]]
; CHECK: [[VECTOR_PH]]:
+; CHECK-NEXT: [[TMP0:%.*]] = getelementptr inbounds i16, ptr [[DST]], i64 [[D]]
; CHECK-NEXT: br label %[[VECTOR_BODY:.*]]
; CHECK: [[VECTOR_BODY]]:
-; CHECK-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[LOOP_LATCH1:.*]] ]
+; CHECK-NEXT: br label %[[LOOP_LATCH1]]
+; CHECK: [[LOOP_LATCH1]]:
+; CHECK-NEXT: store i16 0, ptr [[TMP0]], align 2
; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 4
; CHECK-NEXT: [[TMP1:%.*]] = icmp eq i64 [[INDEX_NEXT]], 100
; CHECK-NEXT: br i1 [[TMP1]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP3:![0-9]+]]
; CHECK: [[MIDDLE_BLOCK]]:
-; CHECK-NEXT: store i16 0, ptr [[DST]], align 2
; CHECK-NEXT: br label %[[EXIT:.*]]
; CHECK: [[EXIT]]:
; CHECK-NEXT: ret void
@@ -87,15 +93,16 @@ define void @const_fold_select(ptr %dst, i64 %d) {
; CHECK-NEXT: [[ENTRY:.*:]]
; CHECK-NEXT: br label %[[VECTOR_PH:.*]]
; CHECK: [[VECTOR_PH]]:
-; CHECK-NEXT: [[TMP3:%.*]] = or i64 [[D]], 1
; CHECK-NEXT: br label %[[VECTOR_BODY:.*]]
; CHECK: [[VECTOR_BODY]]:
-; CHECK-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[LOOP_LATCH1:.*]] ]
+; CHECK-NEXT: br label %[[LOOP_LATCH1]]
+; CHECK: [[LOOP_LATCH1]]:
+; CHECK-NEXT: store i64 [[D]], ptr [[DST]], align 8
; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 4
; CHECK-NEXT: [[TMP4:%.*]] = icmp eq i64 [[INDEX_NEXT]], 100
; CHECK-NEXT: br i1 [[TMP4]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP4:![0-9]+]]
; CHECK: [[MIDDLE_BLOCK]]:
-; CHECK-NEXT: store i64 [[TMP3]], ptr [[DST]], align 8
; CHECK-NEXT: br label %[[EXIT:.*]]
; CHECK: [[EXIT]]:
; CHECK-NEXT: ret void
@@ -128,14 +135,21 @@ define void @const_fold_add_sub_mul_ashr_lshr(ptr %dst, i64 %d) {
; CHECK-NEXT: [[ENTRY:.*:]]
; CHECK-NEXT: br label %[[VECTOR_PH:.*]]
; CHECK: [[VECTOR_PH]]:
+; CHECK-NEXT: [[TMP0:%.*]] = add i64 2, [[D]]
+; CHECK-NEXT: [[TMP1:%.*]] = sub i64 [[TMP0]], [[D]]
+; CHECK-NEXT: [[TMP2:%.*]] = ashr i64 [[TMP1]], [[D]]
+; CHECK-NEXT: [[TMP3:%.*]] = mul i64 [[TMP2]], 3
+; CHECK-NEXT: [[TMP4:%.*]] = lshr i64 [[TMP3]], [[D]]
; CHECK-NEXT: br label %[[VECTOR_BODY:.*]]
; CHECK: [[VECTOR_BODY]]:
-; CHECK-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[LOOP_LATCH1:.*]] ]
+; CHECK-NEXT: br label %[[LOOP_LATCH1]]
+; CHECK: [[LOOP_LATCH1]]:
+; CHECK-NEXT: store i64 [[TMP4]], ptr [[DST]], align 8
; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 4
; CHECK-NEXT: [[TMP6:%.*]] = icmp eq i64 [[INDEX_NEXT]], 100
; CHECK-NEXT: br i1 [[TMP6]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP5:![0-9]+]]
; CHECK: [[MIDDLE_BLOCK]]:
-; CHECK-NEXT: store i64 1, ptr [[DST]], align 8
; CHECK-NEXT: br label %[[EXIT:.*]]
; CHECK: [[EXIT]]:
; CHECK-NEXT: ret void
@@ -172,14 +186,19 @@ define void @const_fold_and_or_xor(ptr %dst, i64 %d) {
; CHECK-NEXT: [[ENTRY:.*:]]
; CHECK-NEXT: br label %[[VECTOR_PH:.*]]
; CHECK: [[VECTOR_PH]]:
+; CHECK-NEXT: [[TMP0:%.*]] = or i64 2, [[D]]
+; CHECK-NEXT: [[TMP1:%.*]] = and i64 [[TMP0]], [[D]]
+; CHECK-NEXT: [[TMP2:%.*]] = and i64 [[TMP1]], [[D]]
; CHECK-NEXT: br label %[[VECTOR_BODY:.*]]
; CHECK: [[VECTOR_BODY]]:
-; CHECK-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[LOOP_LATCH1:.*]] ]
+; CHECK-NEXT: br label %[[LOOP_LATCH1]]
+; CHECK: [[LOOP_LATCH1]]:
+; CHECK-NEXT: store i64 [[TMP2]], ptr [[DST]], align 8
; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 4
; CHECK-NEXT: [[TMP4:%.*]] = icmp eq i64 [[INDEX_NEXT]], 100
; CHECK-NEXT: br i1 [[TMP4]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP6:![0-9]+]]
; CHECK: [[MIDDLE_BLOCK]]:
-; CHECK-NEXT: store i64 1, ptr [[DST]], align 8
; CHECK-NEXT: br label %[[EXIT:.*]]
; CHECK: [[EXIT]]:
; CHECK-NEXT: ret void
@@ -214,14 +233,21 @@ define void @const_fold_cmp_zext(ptr %dst, i64 %d) {
; CHECK-NEXT: [[ENTRY:.*:]]
; CHECK-NEXT: br label %[[VECTOR_PH:.*]]
; CHECK: [[VECTOR_PH]]:
+; CHECK-NEXT: [[BROADCAST_SPLATINSERT:%.*]] = insertelement <4 x i64> poison, i64 [[D]], i64 0
+; CHECK-NEXT: [[BROADCAST_SPLAT:%.*]] = shufflevector <4 x i64> [[BROADCAST_SPLATINSERT]], <4 x i64> poison, <4 x i32> zeroinitializer
+; CHECK-NEXT: [[TMP0:%.*]] = icmp ugt <4 x i64> splat (i64 2), [[BROADCAST_SPLAT]]
+; CHECK-NEXT: [[TMP1:%.*]] = zext <4 x i1> [[TMP0]] to <4 x i8>
+; CHECK-NEXT: [[TMP2:%.*]] = extractelement <4 x i8> [[TMP1]], i64 3
; CHECK-NEXT: br label %[[VECTOR_BODY:.*]]
; CHECK: [[VECTOR_BODY]]:
-; CHECK-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[LOOP_LATCH1:.*]] ]
+; CHECK-NEXT: br label %[[LOOP_LATCH1]]
+; CHECK: [[LOOP_LATCH1]]:
+; CHECK-NEXT: store i8 [[TMP2]], ptr [[DST]], align 1
; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 4
; CHECK-NEXT: [[TMP3:%.*]] = icmp eq i64 [[INDEX_NEXT]], 100
; CHECK-NEXT: br i1 [[TMP3]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP7:![0-9]+]]
; CHECK: [[MIDDLE_BLOCK]]:
-; CHECK-NEXT: store i8 1, ptr [[DST]], align 1
; CHECK-NEXT: br label %[[EXIT:.*]]
; CHECK: [[EXIT]]:
; CHECK-NEXT: ret void
@@ -255,14 +281,20 @@ define void @const_fold_trunc(ptr %dst, i64 %d) {
; CHECK-NEXT: [[ENTRY:.*:]]
; CHECK-NEXT: br label %[[VECTOR_PH:.*]]
; CHECK: [[VECTOR_PH]]:
+; CHECK-NEXT: [[BROADCAST_SPLATINSERT:%.*]] = insertelement <4 x i64> poison, i64 [[D]], i64 0
+; CHECK-NEXT: [[BROADCAST_SPLAT:%.*]] = shufflevector <4 x i64> [[BROADCAST_SPLATINSERT]], <4 x i64> poison, <4 x i32> zeroinitializer
+; CHECK-NEXT: [[TMP0:%.*]] = trunc <4 x i64> [[BROADCAST_SPLAT]] to <4 x i16>
+; CHECK-NEXT: [[TMP1:%.*]] = extractelement <4 x i16> [[TMP0]], i64 3
; CHECK-NEXT: br label %[[VECTOR_BODY:.*]]
; CHECK: [[VECTOR_BODY]]:
-; CHECK-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[LOOP_LATCH1:.*]] ]
+; CHECK-NEXT: br label %[[LOOP_LATCH1]]
+; CHECK: [[LOOP_LATCH1]]:
+; CHECK-NEXT: store i16 [[TMP1]], ptr [[DST]], align 2
; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 4
; CHECK-NEXT: [[TMP2:%.*]] = icmp eq i64 [[INDEX_NEXT]], 100
; CHECK-NEXT: br i1 [[TMP2]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP8:![0-9]+]]
; CHECK: [[MIDDLE_BLOCK]]:
-; CHECK-NEXT: store i16 0, ptr [[DST]], align 2
; CHECK-NEXT: br label %[[EXIT:.*]]
; CHECK: [[EXIT]]:
; CHECK-NEXT: ret void
diff --git a/llvm/test/Transforms/LoopVectorize/hoist-predicated-loads.ll b/llvm/test/Transforms/LoopVectorize/hoist-predicated-loads.ll
index 843b71a507f3a..53c7dd5188e0d 100644
--- a/llvm/test/Transforms/LoopVectorize/hoist-predicated-loads.ll
+++ b/llvm/test/Transforms/LoopVectorize/hoist-predicated-loads.ll
@@ -914,16 +914,27 @@ define void @hoist_predicated_load_with_chained_geps1(ptr %dst, ptr %src, i1 %co
; CHECK: [[VECTOR_PH]]:
; CHECK-NEXT: br label %[[VECTOR_BODY:.*]]
; CHECK: [[VECTOR_BODY]]:
-; CHECK-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
-; CHECK-NEXT: [[TMP2:%.*]] = add i64 [[INDEX]], 1
+; CHECK-NEXT: [[TMP2:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[LOOP_LATCH5:.*]] ]
+; CHECK-NEXT: br i1 [[COND]], label %[[THEN4:.*]], label %[[ELSE3:.*]]
+; CHECK: [[ELSE3]]:
; CHECK-NEXT: [[TMP20:%.*]] = getelementptr [11 x i16], ptr [[SRC]], i64 [[TMP2]]
; CHECK-NEXT: [[TMP21:%.*]] = getelementptr i8, ptr [[TMP20]], i64 8
-; CHECK-NEXT: [[TMP3:%.*]] = load i16, ptr [[TMP21]], align 2, !alias.scope [[META68:![0-9]+]]
-; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 2
+; CHECK-NEXT: [[TMP8:%.*]] = load i16, ptr [[TMP21]], align 2
+; CHECK-NEXT: br label %[[LOOP_LATCH5]]
+; CHECK: [[THEN4]]:
+; CHECK-NEXT: [[TMP3:%.*]] = getelementptr [11 x i16], ptr [[SRC]], i64 [[TMP2]]
+; CHECK-NEXT: [[TMP4:%.*]] = getelementptr i8, ptr [[TMP3]], i64 8
+; CHECK-NEXT: [[TMP5:%.*]] = load i16, ptr [[TMP4]], align 2
+; CHECK-NEXT: br label %[[LOOP_LATCH5]]
+; CHECK: [[LOOP_LATCH5]]:
+; CHECK-NEXT: [[TMP6:%.*]] = phi i16 [ [[TMP8]], %[[ELSE3]] ], [ poison, %[[THEN4]] ]
+; CHECK-NEXT: [[TMP7:%.*]] = phi i16 [ poison, %[[ELSE3]] ], [ [[TMP5]], %[[THEN4]] ]
+; CHECK-NEXT: [[PREDPHI:%.*]] = select i1 true, i16 [[TMP7]], i16 [[TMP6]]
+; CHECK-NEXT: store i16 [[PREDPHI]], ptr [[DST]], align 2, !alias.scope [[META68:![0-9]+]], !noalias [[META71:![0-9]+]]
+; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[TMP2]], 2
; CHECK-NEXT: [[TMP26:%.*]] = icmp eq i64 [[INDEX_NEXT]], 100
-; CHECK-NEXT: br i1 [[TMP26]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP71:![0-9]+]]
+; CHECK-NEXT: br i1 [[TMP26]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP73:![0-9]+]]
; CHECK: [[MIDDLE_BLOCK]]:
-; CHECK-NEXT: store i16 [[TMP3]], ptr [[DST]], align 2, !alias.scope [[META72:![0-9]+]], !noalias [[META68]]
; CHECK-NEXT: br label %[[SCALAR_PH]]
; CHECK: [[SCALAR_PH]]:
;
@@ -973,16 +984,26 @@ define void @hoist_predicated_load_with_chained_geps2(ptr %dst, ptr %src, i1 %co
; CHECK: [[VECTOR_PH]]:
; CHECK-NEXT: br label %[[VECTOR_BODY:.*]]
; CHECK: [[VECTOR_BODY]]:
-; CHECK-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
-; CHECK-NEXT: [[TMP2:%.*]] = add i64 [[INDEX]], 1
+; CHECK-NEXT: [[TMP2:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[LOOP_LATCH5:.*]] ]
; CHECK-NEXT: [[TMP4:%.*]] = getelementptr [11 x i16], ptr [[SRC]], i64 [[TMP2]]
+; CHECK-NEXT: br i1 [[COND]], label %[[THEN4:.*]], label %[[ELSE3:.*]]
+; CHECK: [[ELSE3]]:
; CHECK-NEXT: [[TMP21:%.*]] = getelementptr i8, ptr [[TMP4]], i64 8
-; CHECK-NEXT: [[TMP3:%.*]] = load i16, ptr [[TMP21]], align 2, !alias.scope [[META75:![0-9]+]]
-; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 2
+; CHECK-NEXT: [[TMP7:%.*]] = load i16, ptr [[TMP21]], align 2
+; CHECK-NEXT: br label %[[LOOP_LATCH5]]
+; CHECK: [[THEN4]]:
+; CHECK-NEXT: [[TMP3:%.*]] = getelementptr i8, ptr [[TMP4]], i64 8
+; CHECK-NEXT: [[TMP8:%.*]] = load i16, ptr [[TMP3]], align 2
+; CHECK-NEXT: br label %[[LOOP_LATCH5]]
+; CHECK: [[LOOP_LATCH5]]:
+; CHECK-NEXT: [[TMP5:%.*]] = phi i16 [ [[TMP7]], %[[ELSE3]] ], [ poison, %[[THEN4]] ]
+; CHECK-NEXT: [[TMP6:%.*]] = phi i16 [ poison, %[[ELSE3]] ], [ [[TMP8]], %[[THEN4]] ]
+; CHECK-NEXT: [[PREDPHI:%.*]] = select i1 true, i16 [[TMP6]], i16 [[TMP5]]
+; CHECK-NEXT: store i16 [[PREDPHI]], ptr [[DST]], align 2, !alias.scope [[META75:![0-9]+]], !noalias [[META78:![0-9]+]]
+; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[TMP2]], 2
; CHECK-NEXT: [[TMP26:%.*]] = icmp eq i64 [[INDEX_NEXT]], 100
-; CHECK-NEXT: br i1 [[TMP26]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP78:![0-9]+]]
+; CHECK-NEXT: br i1 [[TMP26]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP80:![0-9]+]]
; CHECK: [[MIDDLE_BLOCK]]:
-; CHECK-NEXT: store i16 [[TMP3]], ptr [[DST]], align 2, !alias.scope [[META79:![0-9]+]], !noalias [[META75]]
; CHECK-NEXT: br label %[[SCALAR_PH]]
; CHECK: [[SCALAR_PH]]:
;
diff --git a/llvm/test/Transforms/LoopVectorize/if-pred-non-void.ll b/llvm/test/Transforms/LoopVectorize/if-pred-non-void.ll
index 873bda4d4d6a3..e2685ecbc32d0 100644
--- a/llvm/test/Transforms/LoopVectorize/if-pred-non-void.ll
+++ b/llvm/test/Transforms/LoopVectorize/if-pred-non-void.ll
@@ -757,21 +757,21 @@ define i32 @predicated_udiv_scalarized_operand(ptr %a, i1 %c, i32 %x, i64 %n) {
; CHECK-NEXT: br i1 [[C]], label %[[PRED_UDIV_IF:.*]], label %[[PRED_UDIV_CONTINUE:.*]]
; CHECK: [[PRED_UDIV_IF]]:
; CHECK-NEXT: [[TMP1:%.*]] = extractelement <2 x i32> [[WIDE_LOAD]], i64 0
-; CHECK-NEXT: [[TMP2:%.*]] = add nsw i32 [[TMP1]], [[X]]
-; CHECK-NEXT: [[TMP4:%.*]] = udiv i32 [[TMP1]], [[TMP2]]
-; CHECK-NEXT: [[TMP5:%.*]] = insertelement <2 x i32> poison, i32 [[TMP4]], i64 0
+; CHECK-NEXT: [[TMP4:%.*]] = add nsw i32 [[TMP1]], [[X]]
+; CHECK-NEXT: [[TMP5:%.*]] = udiv i32 [[TMP1]], [[TMP4]]
+; CHECK-NEXT: [[TMP6:%.*]] = insertelement <2 x i32> poison, i32 [[TMP5]], i64 0
; CHECK-NEXT: br label %[[PRED_UDIV_CONTINUE]]
; CHECK: [[PRED_UDIV_CONTINUE]]:
-; CHECK-NEXT: [[TMP6:%.*]] = phi <2 x i32> [ poison, %[[VECTOR_BODY]] ], [ [[TMP5]], %[[PRED_UDIV_IF]] ]
+; CHECK-NEXT: [[TMP7:%.*]] = phi <2 x i32> [ poison, %[[VECTOR_BODY]] ], [ [[TMP6]], %[[PRED_UDIV_IF]] ]
; CHECK-NEXT: br i1 [[C]], label %[[PRED_UDIV_IF1:.*]], label %[[PRED_UDIV_CONTINUE2]]
; CHECK: [[PRED_UDIV_IF1]]:
-; CHECK-NEXT: [[TMP7:%.*]] = extractelement <2 x i32> [[WIDE_LOAD]], i64 1
-; CHECK-NEXT: [[TMP8:%.*]] = add nsw i32 [[TMP7]], [[X]]
-; CHECK-NEXT: [[TMP10:%.*]] = udiv i32 [[TMP7]], [[TMP8]]
-; CHECK-NEXT: [[TMP11:%.*]] = insertelement <2 x i32> [[TMP6]], i32 [[TMP10]], i64 1
+; CHECK-NEXT: [[TMP8:%.*]] = extractelement <2 x i32> [[WIDE_LOAD]], i64 1
+; CHECK-NEXT: [[TMP9:%.*]] = add nsw i32 [[TMP8]], [[X]]
+; CHECK-NEXT: [[TMP10:%.*]] = udiv i32 [[TMP8]], [[TMP9]]
+; CHECK-NEXT: [[TMP11:%.*]] = insertelement <2 x i32> [[TMP7]], i32 [[TMP10]], i64 1
; CHECK-NEXT: br label %[[PRED_UDIV_CONTINUE2]]
; CHECK: [[PRED_UDIV_CONTINUE2]]:
-; CHECK-NEXT: [[TMP12:%.*]] = phi <2 x i32> [ [[TMP6]], %[[PRED_UDIV_CONTINUE]] ], [ [[TMP11]], %[[PRED_UDIV_IF1]] ]
+; CHECK-NEXT: [[TMP12:%.*]] = phi <2 x i32> [ [[TMP7]], %[[PRED_UDIV_CONTINUE]] ], [ [[TMP11]], %[[PRED_UDIV_IF1]] ]
; CHECK-NEXT: [[PREDPHI:%.*]] = select i1 [[C]], <2 x i32> [[TMP12]], <2 x i32> [[WIDE_LOAD]]
; CHECK-NEXT: [[TMP13]] = add <2 x i32> [[VEC_PHI]], [[PREDPHI]]
; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 2
@@ -827,19 +827,19 @@ define i32 @predicated_udiv_scalarized_operand(ptr %a, i1 %c, i32 %x, i64 %n) {
; UNROLL-NO-VF-NEXT: br i1 [[C]], label %[[PRED_UDIV_IF:.*]], label %[[PRED_UDIV_CONTINUE:.*]]
; UNROLL-NO-VF: [[PRED_UDIV_IF]]:
; UNROLL-NO-VF-NEXT: [[TMP5:%.*]] = add nsw i32 [[TMP3]], [[X]]
-; UNROLL-NO-VF-NEXT: [[TMP6:%.*]] = udiv i32 [[TMP3]], [[TMP5]]
+; UNROLL-NO-VF-NEXT: [[TMP10:%.*]] = udiv i32 [[TMP3]], [[TMP5]]
; UNROLL-NO-VF-NEXT: br label %[[PRED_UDIV_CONTINUE]]
; UNROLL-NO-VF: [[PRED_UDIV_CONTINUE]]:
-; UNROLL-NO-VF-NEXT: [[TMP7:%.*]] = phi i32 [ poison, %[[VECTOR_BODY]] ], [ [[TMP6]], %[[PRED_UDIV_IF]] ]
+; UNROLL-NO-VF-NEXT: [[TMP14:%.*]] = phi i32 [ poison, %[[VECTOR_BODY]] ], [ [[TMP10]], %[[PRED_UDIV_IF]] ]
; UNROLL-NO-VF-NEXT: br i1 [[C]], label %[[PRED_UDIV_IF2:.*]], label %[[PRED_UDIV_CONTINUE3]]
; UNROLL-NO-VF: [[PRED_UDIV_IF2]]:
; UNROLL-NO-VF-NEXT: [[TMP8:%.*]] = add nsw i32 [[TMP4]], [[X]]
; UNROLL-NO-VF-NEXT: [[TMP9:%.*]] = udiv i32 [[TMP4]], [[TMP8]]
; UNROLL-NO-VF-NEXT: br label %[[PRED_UDIV_CONTINUE3]]
; UNROLL-NO-VF: [[PRED_UDIV_CONTINUE3]]:
-; UNROLL-NO-VF-NEXT: [[TMP10:%.*]] = phi i32 [ poison, %[[PRED_UDIV_CONTINUE]] ], [ [[TMP9]], %[[PRED_UDIV_IF2]] ]
-; UNROLL-NO-VF-NEXT: [[PREDPHI:%.*]] = select i1 [[C]], i32 [[TMP7]], i32 [[TMP3]]
-; UNROLL-NO-VF-NEXT: [[PREDPHI4:%.*]] = select i1 [[C]], i32 [[TMP10]], i32 [[TMP4]]
+; UNROLL-NO-VF-NEXT: [[TMP15:%.*]] = phi i32 [ poison, %[[PRED_UDIV_CONTINUE]] ], [ [[TMP9]], %[[PRED_UDIV_IF2]] ]
+; UNROLL-NO-VF-NEXT: [[PREDPHI:%.*]] = select i1 [[C]], i32 [[TMP14]], i32 [[TMP3]]
+; UNROLL-NO-VF-NEXT: [[PREDPHI4:%.*]] = select i1 [[C]], i32 [[TMP15]], i32 [[TMP4]]
; UNROLL-NO-VF-NEXT: [[TMP11]] = add i32 [[VEC_PHI]], [[PREDPHI]]
; UNROLL-NO-VF-NEXT: [[TMP12]] = add i32 [[VEC_PHI1]], [[PREDPHI4]]
; UNROLL-NO-VF-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 2
diff --git a/llvm/test/Transforms/LoopVectorize/if-pred-stores.ll b/llvm/test/Transforms/LoopVectorize/if-pred-stores.ll
index d89ad9ed28ea9..f3ed03987bcbd 100644
--- a/llvm/test/Transforms/LoopVectorize/if-pred-stores.ll
+++ b/llvm/test/Transforms/LoopVectorize/if-pred-stores.ll
@@ -321,27 +321,27 @@ define void @bug18724(i1 %cond, ptr %ptr, i1 %cond.2, i64 %v.1, i32 %v.2) {
; VEC-NEXT: [[TMP6:%.*]] = insertelement <2 x i32> zeroinitializer, i32 [[V_2]], i64 0
; VEC-NEXT: br label %[[VECTOR_BODY:.*]]
; VEC: [[VECTOR_BODY]]:
-; VEC-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[PRED_STORE_CONTINUE2:.*]] ]
-; VEC-NEXT: [[VEC_PHI:%.*]] = phi <2 x i32> [ [[TMP6]], %[[VECTOR_PH]] ], [ [[PREDPHI:%.*]], %[[PRED_STORE_CONTINUE2]] ]
+; VEC-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[PRED_STORE_CONTINUE3:.*]] ]
+; VEC-NEXT: [[VEC_PHI:%.*]] = phi <2 x i32> [ [[TMP6]], %[[VECTOR_PH]] ], [ [[PREDPHI:%.*]], %[[PRED_STORE_CONTINUE3]] ]
; VEC-NEXT: [[TMP7:%.*]] = add i64 [[V_1]], [[INDEX]]
; VEC-NEXT: [[TMP8:%.*]] = getelementptr inbounds [768 x i32], ptr [[PTR]], i64 0, i64 [[TMP7]]
; VEC-NEXT: [[WIDE_LOAD:%.*]] = load <2 x i32>, ptr [[TMP8]], align 4
-; VEC-NEXT: br i1 [[COND_2]], label %[[PRED_STORE_IF:.*]], label %[[PRED_STORE_CONTINUE:.*]]
-; VEC: [[PRED_STORE_IF]]:
-; VEC-NEXT: [[TMP9:%.*]] = extractelement <2 x i32> [[WIDE_LOAD]], i64 0
-; VEC-NEXT: store i32 [[TMP9]], ptr [[TMP8]], align 4
-; VEC-NEXT: br label %[[PRED_STORE_CONTINUE]]
-; VEC: [[PRED_STORE_CONTINUE]]:
-; VEC-NEXT: br i1 [[COND_2]], label %[[PRED_STORE_IF1:.*]], label %[[PRED_STORE_CONTINUE2]]
+; VEC-NEXT: br i1 [[COND_2]], label %[[PRED_STORE_IF1:.*]], label %[[PRED_STORE_CONTINUE2:.*]]
; VEC: [[PRED_STORE_IF1]]:
-; VEC-NEXT: [[TMP10:%.*]] = add i64 [[TMP7]], 1
-; VEC-NEXT: [[TMP11:%.*]] = getelementptr inbounds [768 x i32], ptr [[PTR]], i64 0, i64 [[TMP10]]
-; VEC-NEXT: [[TMP12:%.*]] = extractelement <2 x i32> [[WIDE_LOAD]], i64 1
-; VEC-NEXT: store i32 [[TMP12]], ptr [[TMP11]], align 4
+; VEC-NEXT: [[TMP10:%.*]] = extractelement <2 x i32> [[WIDE_LOAD]], i64 0
+; VEC-NEXT: store i32 [[TMP10]], ptr [[TMP8]], align 4
; VEC-NEXT: br label %[[PRED_STORE_CONTINUE2]]
; VEC: [[PRED_STORE_CONTINUE2]]:
-; VEC-NEXT: [[TMP13:%.*]] = add <2 x i32> [[VEC_PHI]], splat (i32 1)
-; VEC-NEXT: [[PREDPHI]] = select i1 [[COND_2]], <2 x i32> [[TMP13]], <2 x i32> [[VEC_PHI]]
+; VEC-NEXT: br i1 [[COND_2]], label %[[PRED_STORE_IF2:.*]], label %[[PRED_STORE_CONTINUE3]]
+; VEC: [[PRED_STORE_IF2]]:
+; VEC-NEXT: [[TMP11:%.*]] = add i64 [[TMP7]], 1
+; VEC-NEXT: [[TMP12:%.*]] = getelementptr inbounds [768 x i32], ptr [[PTR]], i64 0, i64 [[TMP11]]
+; VEC-NEXT: [[TMP13:%.*]] = extractelement <2 x i32> [[WIDE_LOAD]], i64 1
+; VEC-NEXT: store i32 [[TMP13]], ptr [[TMP12]], align 4
+; VEC-NEXT: br label %[[PRED_STORE_CONTINUE3]]
+; VEC: [[PRED_STORE_CONTINUE3]]:
+; VEC-NEXT: [[TMP16:%.*]] = add <2 x i32> [[VEC_PHI]], splat (i32 1)
+; VEC-NEXT: [[PREDPHI]] = select i1 [[COND_2]], <2 x i32> [[TMP16]], <2 x i32> [[VEC_PHI]]
; VEC-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 2
; VEC-NEXT: [[TMP14:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
; VEC-NEXT: br i1 [[TMP14]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP3:![0-9]+]]
diff --git a/llvm/test/Transforms/LoopVectorize/induction.ll b/llvm/test/Transforms/LoopVectorize/induction.ll
index 1683f784daf75..66766a9c03cd6 100644
--- a/llvm/test/Transforms/LoopVectorize/induction.ll
+++ b/llvm/test/Transforms/LoopVectorize/induction.ll
@@ -1969,20 +1969,20 @@ define i32 @scalarize_induction_variable_05(ptr %a, i32 %x, i1 %c, i32 %n) {
; CHECK-NEXT: br i1 [[C:%.*]], label [[PRED_UDIV_IF:%.*]], label [[PRED_UDIV_CONTINUE:%.*]]
; CHECK: pred.udiv.if:
; CHECK-NEXT: [[TMP5:%.*]] = extractelement <2 x i32> [[WIDE_LOAD]], i64 0
-; CHECK-NEXT: [[TMP6:%.*]] = udiv i32 [[TMP5]], [[INDEX]]
-; CHECK-NEXT: [[TMP7:%.*]] = insertelement <2 x i32> poison, i32 [[TMP6]], i64 0
+; CHECK-NEXT: [[TMP4:%.*]] = udiv i32 [[TMP5]], [[INDEX]]
+; CHECK-NEXT: [[TMP12:%.*]] = insertelement <2 x i32> poison, i32 [[TMP4]], i64 0
; CHECK-NEXT: br label [[PRED_UDIV_CONTINUE]]
; CHECK: pred.udiv.continue:
-; CHECK-NEXT: [[TMP12:%.*]] = phi <2 x i32> [ poison, [[VECTOR_BODY]] ], [ [[TMP7]], [[PRED_UDIV_IF]] ]
+; CHECK-NEXT: [[TMP6:%.*]] = phi <2 x i32> [ poison, [[VECTOR_BODY]] ], [ [[TMP12]], [[PRED_UDIV_IF]] ]
; CHECK-NEXT: br i1 [[C]], label [[PRED_UDIV_IF1:%.*]], label [[PRED_UDIV_CONTINUE2]]
; CHECK: pred.udiv.if1:
-; CHECK-NEXT: [[TMP14:%.*]] = add i32 [[INDEX]], 1
+; CHECK-NEXT: [[TMP7:%.*]] = add i32 [[INDEX]], 1
; CHECK-NEXT: [[TMP8:%.*]] = extractelement <2 x i32> [[WIDE_LOAD]], i64 1
-; CHECK-NEXT: [[TMP9:%.*]] = udiv i32 [[TMP8]], [[TMP14]]
-; CHECK-NEXT: [[TMP10:%.*]] = insertelement <2 x i32> [[TMP12]], i32 [[TMP9]], i64 1
+; CHECK-NEXT: [[TMP9:%.*]] = udiv i32 [[TMP8]], [[TMP7]]
+; CHECK-NEXT: [[TMP10:%.*]] = insertelement <2 x i32> [[TMP6]], i32 [[TMP9]], i64 1
; CHECK-NEXT: br label [[PRED_UDIV_CONTINUE2]]
; CHECK: pred.udiv.continue2:
-; CHECK-NEXT: [[TMP11:%.*]] = phi <2 x i32> [ [[TMP12]], [[PRED_UDIV_CONTINUE]] ], [ [[TMP10]], [[PRED_UDIV_IF1]] ]
+; CHECK-NEXT: [[TMP11:%.*]] = phi <2 x i32> [ [[TMP6]], [[PRED_UDIV_CONTINUE]] ], [ [[TMP10]], [[PRED_UDIV_IF1]] ]
; CHECK-NEXT: [[PREDPHI:%.*]] = select i1 [[C]], <2 x i32> [[TMP11]], <2 x i32> [[WIDE_LOAD]]
; CHECK-NEXT: [[TMP16]] = add <2 x i32> [[PREDPHI]], [[VEC_PHI]]
; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i32 [[INDEX]], 2
@@ -2032,21 +2032,21 @@ define i32 @scalarize_induction_variable_05(ptr %a, i32 %x, i1 %c, i32 %n) {
; IND-NEXT: br i1 [[C:%.*]], label [[PRED_UDIV_IF:%.*]], label [[PRED_UDIV_CONTINUE:%.*]]
; IND: pred.udiv.if:
; IND-NEXT: [[TMP2:%.*]] = extractelement <2 x i32> [[WIDE_LOAD]], i64 0
-; IND-NEXT: [[TMP3:%.*]] = udiv i32 [[TMP2]], [[INDEX]]
-; IND-NEXT: [[TMP4:%.*]] = insertelement <2 x i32> poison, i32 [[TMP3]], i64 0
+; IND-NEXT: [[TMP4:%.*]] = udiv i32 [[TMP2]], [[INDEX]]
+; IND-NEXT: [[TMP5:%.*]] = insertelement <2 x i32> poison, i32 [[TMP4]], i64 0
; IND-NEXT: br label [[PRED_UDIV_CONTINUE]]
; IND: pred.udiv.continue:
-; IND-NEXT: [[TMP5:%.*]] = phi <2 x i32> [ poison, [[VECTOR_BODY]] ], [ [[TMP4]], [[PRED_UDIV_IF]] ]
+; IND-NEXT: [[TMP6:%.*]] = phi <2 x i32> [ poison, [[VECTOR_BODY]] ], [ [[TMP5]], [[PRED_UDIV_IF]] ]
; IND-NEXT: br i1 [[C]], label [[PRED_UDIV_IF1:%.*]], label [[PRED_UDIV_CONTINUE2]]
; IND: pred.udiv.if1:
-; IND-NEXT: [[TMP6:%.*]] = add i32 [[INDEX]], 1
-; IND-NEXT: [[TMP7:%.*]] = extractelement <2 x i32> [[WIDE_LOAD]], i64 1
-; IND-NEXT: [[TMP8:%.*]] = udiv i32 [[TMP7]], [[TMP6]]
-; IND-NEXT: [[TMP9:%.*]] = insertelement <2 x i32> [[TMP5]], i32 [[TMP8]], i64 1
+; IND-NEXT: [[TMP7:%.*]] = add i32 [[INDEX]], 1
+; IND-NEXT: [[TMP8:%.*]] = extractelement <2 x i32> [[WIDE_LOAD]], i64 1
+; IND-NEXT: [[TMP9:%.*]] = udiv i32 [[TMP8]], [[TMP7]]
+; IND-NEXT: [[TMP10:%.*]] = insertelement <2 x i32> [[TMP6]], i32 [[TMP9]], i64 1
; IND-NEXT: br label [[PRED_UDIV_CONTINUE2]]
; IND: pred.udiv.continue2:
-; IND-NEXT: [[TMP10:%.*]] = phi <2 x i32> [ [[TMP5]], [[PRED_UDIV_CONTINUE]] ], [ [[TMP9]], [[PRED_UDIV_IF1]] ]
-; IND-NEXT: [[PREDPHI:%.*]] = select i1 [[C]], <2 x i32> [[TMP10]], <2 x i32> [[WIDE_LOAD]]
+; IND-NEXT: [[TMP11:%.*]] = phi <2 x i32> [ [[TMP6]], [[PRED_UDIV_CONTINUE]] ], [ [[TMP10]], [[PRED_UDIV_IF1]] ]
+; IND-NEXT: [[PREDPHI:%.*]] = select i1 [[C]], <2 x i32> [[TMP11]], <2 x i32> [[WIDE_LOAD]]
; IND-NEXT: [[TMP13]] = add <2 x i32> [[PREDPHI]], [[VEC_PHI]]
; IND-NEXT: [[INDEX_NEXT]] = add nuw i32 [[INDEX]], 2
; IND-NEXT: [[TMP14:%.*]] = icmp eq i32 [[INDEX_NEXT]], [[N_VEC]]
@@ -2097,41 +2097,41 @@ define i32 @scalarize_induction_variable_05(ptr %a, i32 %x, i1 %c, i32 %n) {
; UNROLL-NEXT: [[WIDE_LOAD2:%.*]] = load <2 x i32>, ptr [[TMP2]], align 4
; UNROLL-NEXT: br i1 [[C:%.*]], label [[PRED_UDIV_IF:%.*]], label [[PRED_UDIV_CONTINUE:%.*]]
; UNROLL: pred.udiv.if:
-; UNROLL-NEXT: [[TMP3:%.*]] = extractelement <2 x i32> [[WIDE_LOAD]], i64 0
-; UNROLL-NEXT: [[TMP4:%.*]] = udiv i32 [[TMP3]], [[INDEX]]
-; UNROLL-NEXT: [[TMP5:%.*]] = insertelement <2 x i32> poison, i32 [[TMP4]], i64 0
+; UNROLL-NEXT: [[TMP5:%.*]] = extractelement <2 x i32> [[WIDE_LOAD]], i64 0
+; UNROLL-NEXT: [[TMP7:%.*]] = udiv i32 [[TMP5]], [[INDEX]]
+; UNROLL-NEXT: [[TMP6:%.*]] = insertelement <2 x i32> poison, i32 [[TMP7]], i64 0
; UNROLL-NEXT: br label [[PRED_UDIV_CONTINUE]]
; UNROLL: pred.udiv.continue:
-; UNROLL-NEXT: [[TMP6:%.*]] = phi <2 x i32> [ poison, [[VECTOR_BODY]] ], [ [[TMP5]], [[PRED_UDIV_IF]] ]
+; UNROLL-NEXT: [[TMP8:%.*]] = phi <2 x i32> [ poison, [[VECTOR_BODY]] ], [ [[TMP6]], [[PRED_UDIV_IF]] ]
; UNROLL-NEXT: br i1 [[C]], label [[PRED_UDIV_IF3:%.*]], label [[PRED_UDIV_CONTINUE4:%.*]]
; UNROLL: pred.udiv.if3:
-; UNROLL-NEXT: [[TMP7:%.*]] = add i32 [[INDEX]], 1
-; UNROLL-NEXT: [[TMP8:%.*]] = extractelement <2 x i32> [[WIDE_LOAD]], i64 1
-; UNROLL-NEXT: [[TMP9:%.*]] = udiv i32 [[TMP8]], [[TMP7]]
-; UNROLL-NEXT: [[TMP10:%.*]] = insertelement <2 x i32> [[TMP6]], i32 [[TMP9]], i64 1
+; UNROLL-NEXT: [[TMP17:%.*]] = add i32 [[INDEX]], 1
+; UNROLL-NEXT: [[TMP18:%.*]] = extractelement <2 x i32> [[WIDE_LOAD]], i64 1
+; UNROLL-NEXT: [[TMP19:%.*]] = udiv i32 [[TMP18]], [[TMP17]]
+; UNROLL-NEXT: [[TMP11:%.*]] = insertelement <2 x i32> [[TMP8]], i32 [[TMP19]], i64 1
; UNROLL-NEXT: br label [[PRED_UDIV_CONTINUE4]]
; UNROLL: pred.udiv.continue4:
-; UNROLL-NEXT: [[TMP11:%.*]] = phi <2 x i32> [ [[TMP6]], [[PRED_UDIV_CONTINUE]] ], [ [[TMP10]], [[PRED_UDIV_IF3]] ]
+; UNROLL-NEXT: [[TMP12:%.*]] = phi <2 x i32> [ [[TMP8]], [[PRED_UDIV_CONTINUE]] ], [ [[TMP11]], [[PRED_UDIV_IF3]] ]
; UNROLL-NEXT: br i1 [[C]], label [[PRED_UDIV_IF5:%.*]], label [[PRED_UDIV_CONTINUE6:%.*]]
; UNROLL: pred.udiv.if5:
-; UNROLL-NEXT: [[TMP12:%.*]] = add i32 [[INDEX]], 2
-; UNROLL-NEXT: [[TMP13:%.*]] = extractelement <2 x i32> [[WIDE_LOAD2]], i64 0
-; UNROLL-NEXT: [[TMP14:%.*]] = udiv i32 [[TMP13]], [[TMP12]]
-; UNROLL-NEXT: [[TMP15:%.*]] = insertelement <2 x i32> poison, i32 [[TMP14]], i64 0
+; UNROLL-NEXT: [[TMP13:%.*]] = add i32 [[INDEX]], 2
+; UNROLL-NEXT: [[TMP14:%.*]] = extractelement <2 x i32> [[WIDE_LOAD2]], i64 0
+; UNROLL-NEXT: [[TMP9:%.*]] = udiv i32 [[TMP14]], [[TMP13]]
+; UNROLL-NEXT: [[BROADCAST_SPLATINSERT:%.*]] = insertelement <2 x i32> poison, i32 [[TMP9]], i64 0
; UNROLL-NEXT: br label [[PRED_UDIV_CONTINUE6]]
; UNROLL: pred.udiv.continue6:
-; UNROLL-NEXT: [[TMP16:%.*]] = phi <2 x i32> [ poison, [[PRED_UDIV_CONTINUE4]] ], [ [[TMP15]], [[PRED_UDIV_IF5]] ]
+; UNROLL-NEXT: [[TMP26:%.*]] = phi <2 x i32> [ poison, [[PRED_UDIV_CONTINUE4]] ], [ [[BROADCAST_SPLATINSERT]], [[PRED_UDIV_IF5]] ]
; UNROLL-NEXT: br i1 [[C]], label [[PRED_UDIV_IF7:%.*]], label [[PRED_UDIV_CONTINUE8]]
; UNROLL: pred.udiv.if7:
-; UNROLL-NEXT: [[TMP17:%.*]] = add i32 [[INDEX]], 3
-; UNROLL-NEXT: [[TMP18:%.*]] = extractelement <2 x i32> [[WIDE_LOAD2]], i64 1
-; UNROLL-NEXT: [[TMP19:%.*]] = udiv i32 [[TMP18]], [[TMP17]]
-; UNROLL-NEXT: [[TMP20:%.*]] = insertelement <2 x i32> [[TMP16]], i32 [[TMP19]], i64 1
+; UNROLL-NEXT: [[TMP27:%.*]] = add i32 [[INDEX]], 3
+; UNROLL-NEXT: [[TMP28:%.*]] = extractelement <2 x i32> [[WIDE_LOAD2]], i64 1
+; UNROLL-NEXT: [[TMP20:%.*]] = udiv i32 [[TMP28]], [[TMP27]]
+; UNROLL-NEXT: [[TMP21:%.*]] = insertelement <2 x i32> [[TMP26]], i32 [[TMP20]], i64 1
; UNROLL-NEXT: br label [[PRED_UDIV_CONTINUE8]]
; UNROLL: pred.udiv.continue8:
-; UNROLL-NEXT: [[TMP21:%.*]] = phi <2 x i32> [ [[TMP16]], [[PRED_UDIV_CONTINUE6]] ], [ [[TMP20]], [[PRED_UDIV_IF7]] ]
-; UNROLL-NEXT: [[PREDPHI:%.*]] = select i1 [[C]], <2 x i32> [[TMP11]], <2 x i32> [[WIDE_LOAD]]
-; UNROLL-NEXT: [[PREDPHI9:%.*]] = select i1 [[C]], <2 x i32> [[TMP21]], <2 x i32> [[WIDE_LOAD2]]
+; UNROLL-NEXT: [[TMP29:%.*]] = phi <2 x i32> [ [[TMP26]], [[PRED_UDIV_CONTINUE6]] ], [ [[TMP21]], [[PRED_UDIV_IF7]] ]
+; UNROLL-NEXT: [[PREDPHI:%.*]] = select i1 [[C]], <2 x i32> [[TMP12]], <2 x i32> [[WIDE_LOAD]]
+; UNROLL-NEXT: [[PREDPHI9:%.*]] = select i1 [[C]], <2 x i32> [[TMP29]], <2 x i32> [[WIDE_LOAD2]]
; UNROLL-NEXT: [[TMP22]] = add <2 x i32> [[PREDPHI]], [[VEC_PHI]]
; UNROLL-NEXT: [[TMP23]] = add <2 x i32> [[PREDPHI9]], [[VEC_PHI1]]
; UNROLL-NEXT: [[INDEX_NEXT]] = add nuw i32 [[INDEX]], 4
@@ -2184,41 +2184,41 @@ define i32 @scalarize_induction_variable_05(ptr %a, i32 %x, i1 %c, i32 %n) {
; UNROLL-NO-IC-NEXT: [[WIDE_LOAD2:%.*]] = load <2 x i32>, ptr [[TMP2]], align 4
; UNROLL-NO-IC-NEXT: br i1 [[C:%.*]], label [[PRED_UDIV_IF:%.*]], label [[PRED_UDIV_CONTINUE:%.*]]
; UNROLL-NO-IC: pred.udiv.if:
-; UNROLL-NO-IC-NEXT: [[TMP8:%.*]] = extractelement <2 x i32> [[WIDE_LOAD]], i64 0
-; UNROLL-NO-IC-NEXT: [[TMP9:%.*]] = udiv i32 [[TMP8]], [[INDEX]]
-; UNROLL-NO-IC-NEXT: [[TMP10:%.*]] = insertelement <2 x i32> poison, i32 [[TMP9]], i64 0
+; UNROLL-NO-IC-NEXT: [[TMP5:%.*]] = extractelement <2 x i32> [[WIDE_LOAD]], i64 0
+; UNROLL-NO-IC-NEXT: [[TMP7:%.*]] = udiv i32 [[TMP5]], [[INDEX]]
+; UNROLL-NO-IC-NEXT: [[TMP6:%.*]] = insertelement <2 x i32> poison, i32 [[TMP7]], i64 0
; UNROLL-NO-IC-NEXT: br label [[PRED_UDIV_CONTINUE]]
; UNROLL-NO-IC: pred.udiv.continue:
-; UNROLL-NO-IC-NEXT: [[TMP11:%.*]] = phi <2 x i32> [ poison, [[VECTOR_BODY]] ], [ [[TMP10]], [[PRED_UDIV_IF]] ]
+; UNROLL-NO-IC-NEXT: [[TMP8:%.*]] = phi <2 x i32> [ poison, [[VECTOR_BODY]] ], [ [[TMP6]], [[PRED_UDIV_IF]] ]
; UNROLL-NO-IC-NEXT: br i1 [[C]], label [[PRED_UDIV_IF3:%.*]], label [[PRED_UDIV_CONTINUE4:%.*]]
; UNROLL-NO-IC: pred.udiv.if3:
-; UNROLL-NO-IC-NEXT: [[TMP13:%.*]] = add i32 [[INDEX]], 1
-; UNROLL-NO-IC-NEXT: [[TMP14:%.*]] = extractelement <2 x i32> [[WIDE_LOAD]], i64 1
-; UNROLL-NO-IC-NEXT: [[TMP15:%.*]] = udiv i32 [[TMP14]], [[TMP13]]
-; UNROLL-NO-IC-NEXT: [[TMP16:%.*]] = insertelement <2 x i32> [[TMP11]], i32 [[TMP15]], i64 1
+; UNROLL-NO-IC-NEXT: [[TMP25:%.*]] = add i32 [[INDEX]], 1
+; UNROLL-NO-IC-NEXT: [[TMP26:%.*]] = extractelement <2 x i32> [[WIDE_LOAD]], i64 1
+; UNROLL-NO-IC-NEXT: [[TMP27:%.*]] = udiv i32 [[TMP26]], [[TMP25]]
+; UNROLL-NO-IC-NEXT: [[TMP11:%.*]] = insertelement <2 x i32> [[TMP8]], i32 [[TMP27]], i64 1
; UNROLL-NO-IC-NEXT: br label [[PRED_UDIV_CONTINUE4]]
; UNROLL-NO-IC: pred.udiv.continue4:
-; UNROLL-NO-IC-NEXT: [[TMP17:%.*]] = phi <2 x i32> [ [[TMP11]], [[PRED_UDIV_CONTINUE]] ], [ [[TMP16]], [[PRED_UDIV_IF3]] ]
+; UNROLL-NO-IC-NEXT: [[TMP12:%.*]] = phi <2 x i32> [ [[TMP8]], [[PRED_UDIV_CONTINUE]] ], [ [[TMP11]], [[PRED_UDIV_IF3]] ]
; UNROLL-NO-IC-NEXT: br i1 [[C]], label [[PRED_UDIV_IF5:%.*]], label [[PRED_UDIV_CONTINUE6:%.*]]
; UNROLL-NO-IC: pred.udiv.if5:
-; UNROLL-NO-IC-NEXT: [[TMP19:%.*]] = add i32 [[INDEX]], 2
-; UNROLL-NO-IC-NEXT: [[TMP20:%.*]] = extractelement <2 x i32> [[WIDE_LOAD2]], i64 0
-; UNROLL-NO-IC-NEXT: [[TMP21:%.*]] = udiv i32 [[TMP20]], [[TMP19]]
-; UNROLL-NO-IC-NEXT: [[TMP22:%.*]] = insertelement <2 x i32> poison, i32 [[TMP21]], i64 0
+; UNROLL-NO-IC-NEXT: [[TMP13:%.*]] = add i32 [[INDEX]], 2
+; UNROLL-NO-IC-NEXT: [[TMP14:%.*]] = extractelement <2 x i32> [[WIDE_LOAD2]], i64 0
+; UNROLL-NO-IC-NEXT: [[TMP9:%.*]] = udiv i32 [[TMP14]], [[TMP13]]
+; UNROLL-NO-IC-NEXT: [[BROADCAST_SPLATINSERT:%.*]] = insertelement <2 x i32> poison, i32 [[TMP9]], i64 0
; UNROLL-NO-IC-NEXT: br label [[PRED_UDIV_CONTINUE6]]
; UNROLL-NO-IC: pred.udiv.continue6:
-; UNROLL-NO-IC-NEXT: [[TMP23:%.*]] = phi <2 x i32> [ poison, [[PRED_UDIV_CONTINUE4]] ], [ [[TMP22]], [[PRED_UDIV_IF5]] ]
+; UNROLL-NO-IC-NEXT: [[TMP17:%.*]] = phi <2 x i32> [ poison, [[PRED_UDIV_CONTINUE4]] ], [ [[BROADCAST_SPLATINSERT]], [[PRED_UDIV_IF5]] ]
; UNROLL-NO-IC-NEXT: br i1 [[C]], label [[PRED_UDIV_IF7:%.*]], label [[PRED_UDIV_CONTINUE8]]
; UNROLL-NO-IC: pred.udiv.if7:
-; UNROLL-NO-IC-NEXT: [[TMP25:%.*]] = add i32 [[INDEX]], 3
-; UNROLL-NO-IC-NEXT: [[TMP26:%.*]] = extractelement <2 x i32> [[WIDE_LOAD2]], i64 1
-; UNROLL-NO-IC-NEXT: [[TMP27:%.*]] = udiv i32 [[TMP26]], [[TMP25]]
-; UNROLL-NO-IC-NEXT: [[TMP28:%.*]] = insertelement <2 x i32> [[TMP23]], i32 [[TMP27]], i64 1
+; UNROLL-NO-IC-NEXT: [[TMP18:%.*]] = add i32 [[INDEX]], 3
+; UNROLL-NO-IC-NEXT: [[TMP19:%.*]] = extractelement <2 x i32> [[WIDE_LOAD2]], i64 1
+; UNROLL-NO-IC-NEXT: [[TMP20:%.*]] = udiv i32 [[TMP19]], [[TMP18]]
+; UNROLL-NO-IC-NEXT: [[TMP21:%.*]] = insertelement <2 x i32> [[TMP17]], i32 [[TMP20]], i64 1
; UNROLL-NO-IC-NEXT: br label [[PRED_UDIV_CONTINUE8]]
; UNROLL-NO-IC: pred.udiv.continue8:
-; UNROLL-NO-IC-NEXT: [[TMP29:%.*]] = phi <2 x i32> [ [[TMP23]], [[PRED_UDIV_CONTINUE6]] ], [ [[TMP28]], [[PRED_UDIV_IF7]] ]
-; UNROLL-NO-IC-NEXT: [[PREDPHI:%.*]] = select i1 [[C]], <2 x i32> [[TMP17]], <2 x i32> [[WIDE_LOAD]]
-; UNROLL-NO-IC-NEXT: [[PREDPHI9:%.*]] = select i1 [[C]], <2 x i32> [[TMP29]], <2 x i32> [[WIDE_LOAD2]]
+; UNROLL-NO-IC-NEXT: [[TMP22:%.*]] = phi <2 x i32> [ [[TMP17]], [[PRED_UDIV_CONTINUE6]] ], [ [[TMP21]], [[PRED_UDIV_IF7]] ]
+; UNROLL-NO-IC-NEXT: [[PREDPHI:%.*]] = select i1 [[C]], <2 x i32> [[TMP12]], <2 x i32> [[WIDE_LOAD]]
+; UNROLL-NO-IC-NEXT: [[PREDPHI9:%.*]] = select i1 [[C]], <2 x i32> [[TMP22]], <2 x i32> [[WIDE_LOAD2]]
; UNROLL-NO-IC-NEXT: [[TMP32]] = add <2 x i32> [[PREDPHI]], [[VEC_PHI]]
; UNROLL-NO-IC-NEXT: [[TMP33]] = add <2 x i32> [[PREDPHI9]], [[VEC_PHI1]]
; UNROLL-NO-IC-NEXT: [[INDEX_NEXT]] = add nuw i32 [[INDEX]], 4
@@ -2273,75 +2273,75 @@ define i32 @scalarize_induction_variable_05(ptr %a, i32 %x, i1 %c, i32 %n) {
; INTERLEAVE: pred.udiv.if:
; INTERLEAVE-NEXT: [[TMP3:%.*]] = extractelement <4 x i32> [[WIDE_LOAD]], i64 0
; INTERLEAVE-NEXT: [[TMP4:%.*]] = udiv i32 [[TMP3]], [[INDEX]]
-; INTERLEAVE-NEXT: [[TMP5:%.*]] = insertelement <4 x i32> poison, i32 [[TMP4]], i64 0
+; INTERLEAVE-NEXT: [[TMP6:%.*]] = insertelement <4 x i32> poison, i32 [[TMP4]], i64 0
; INTERLEAVE-NEXT: br label [[PRED_UDIV_CONTINUE]]
; INTERLEAVE: pred.udiv.continue:
-; INTERLEAVE-NEXT: [[TMP6:%.*]] = phi <4 x i32> [ poison, [[VECTOR_BODY]] ], [ [[TMP5]], [[PRED_UDIV_IF]] ]
+; INTERLEAVE-NEXT: [[TMP7:%.*]] = phi <4 x i32> [ poison, [[VECTOR_BODY]] ], [ [[TMP6]], [[PRED_UDIV_IF]] ]
; INTERLEAVE-NEXT: br i1 [[C]], label [[PRED_UDIV_IF3:%.*]], label [[PRED_UDIV_CONTINUE4:%.*]]
; INTERLEAVE: pred.udiv.if3:
-; INTERLEAVE-NEXT: [[TMP7:%.*]] = add i32 [[INDEX]], 1
-; INTERLEAVE-NEXT: [[TMP8:%.*]] = extractelement <4 x i32> [[WIDE_LOAD]], i64 1
-; INTERLEAVE-NEXT: [[TMP9:%.*]] = udiv i32 [[TMP8]], [[TMP7]]
-; INTERLEAVE-NEXT: [[TMP10:%.*]] = insertelement <4 x i32> [[TMP6]], i32 [[TMP9]], i64 1
+; INTERLEAVE-NEXT: [[TMP8:%.*]] = add i32 [[INDEX]], 1
+; INTERLEAVE-NEXT: [[TMP13:%.*]] = extractelement <4 x i32> [[WIDE_LOAD]], i64 1
+; INTERLEAVE-NEXT: [[TMP10:%.*]] = udiv i32 [[TMP13]], [[TMP8]]
+; INTERLEAVE-NEXT: [[TMP11:%.*]] = insertelement <4 x i32> [[TMP7]], i32 [[TMP10]], i64 1
; INTERLEAVE-NEXT: br label [[PRED_UDIV_CONTINUE4]]
; INTERLEAVE: pred.udiv.continue4:
-; INTERLEAVE-NEXT: [[TMP11:%.*]] = phi <4 x i32> [ [[TMP6]], [[PRED_UDIV_CONTINUE]] ], [ [[TMP10]], [[PRED_UDIV_IF3]] ]
+; INTERLEAVE-NEXT: [[TMP12:%.*]] = phi <4 x i32> [ [[TMP7]], [[PRED_UDIV_CONTINUE]] ], [ [[TMP11]], [[PRED_UDIV_IF3]] ]
; INTERLEAVE-NEXT: br i1 [[C]], label [[PRED_UDIV_IF5:%.*]], label [[PRED_UDIV_CONTINUE6:%.*]]
; INTERLEAVE: pred.udiv.if5:
-; INTERLEAVE-NEXT: [[TMP12:%.*]] = add i32 [[INDEX]], 2
-; INTERLEAVE-NEXT: [[TMP13:%.*]] = extractelement <4 x i32> [[WIDE_LOAD]], i64 2
-; INTERLEAVE-NEXT: [[TMP14:%.*]] = udiv i32 [[TMP13]], [[TMP12]]
-; INTERLEAVE-NEXT: [[TMP15:%.*]] = insertelement <4 x i32> [[TMP11]], i32 [[TMP14]], i64 2
+; INTERLEAVE-NEXT: [[TMP17:%.*]] = add i32 [[INDEX]], 2
+; INTERLEAVE-NEXT: [[TMP18:%.*]] = extractelement <4 x i32> [[WIDE_LOAD]], i64 2
+; INTERLEAVE-NEXT: [[TMP19:%.*]] = udiv i32 [[TMP18]], [[TMP17]]
+; INTERLEAVE-NEXT: [[TMP16:%.*]] = insertelement <4 x i32> [[TMP12]], i32 [[TMP19]], i64 2
; INTERLEAVE-NEXT: br label [[PRED_UDIV_CONTINUE6]]
; INTERLEAVE: pred.udiv.continue6:
-; INTERLEAVE-NEXT: [[TMP16:%.*]] = phi <4 x i32> [ [[TMP11]], [[PRED_UDIV_CONTINUE4]] ], [ [[TMP15]], [[PRED_UDIV_IF5]] ]
+; INTERLEAVE-NEXT: [[TMP25:%.*]] = phi <4 x i32> [ [[TMP12]], [[PRED_UDIV_CONTINUE4]] ], [ [[TMP16]], [[PRED_UDIV_IF5]] ]
; INTERLEAVE-NEXT: br i1 [[C]], label [[PRED_UDIV_IF7:%.*]], label [[PRED_UDIV_CONTINUE8:%.*]]
; INTERLEAVE: pred.udiv.if7:
-; INTERLEAVE-NEXT: [[TMP17:%.*]] = add i32 [[INDEX]], 3
-; INTERLEAVE-NEXT: [[TMP18:%.*]] = extractelement <4 x i32> [[WIDE_LOAD]], i64 3
-; INTERLEAVE-NEXT: [[TMP19:%.*]] = udiv i32 [[TMP18]], [[TMP17]]
-; INTERLEAVE-NEXT: [[TMP20:%.*]] = insertelement <4 x i32> [[TMP16]], i32 [[TMP19]], i64 3
+; INTERLEAVE-NEXT: [[TMP26:%.*]] = add i32 [[INDEX]], 3
+; INTERLEAVE-NEXT: [[TMP46:%.*]] = extractelement <4 x i32> [[WIDE_LOAD]], i64 3
+; INTERLEAVE-NEXT: [[TMP20:%.*]] = udiv i32 [[TMP46]], [[TMP26]]
+; INTERLEAVE-NEXT: [[TMP21:%.*]] = insertelement <4 x i32> [[TMP25]], i32 [[TMP20]], i64 3
; INTERLEAVE-NEXT: br label [[PRED_UDIV_CONTINUE8]]
; INTERLEAVE: pred.udiv.continue8:
-; INTERLEAVE-NEXT: [[TMP21:%.*]] = phi <4 x i32> [ [[TMP16]], [[PRED_UDIV_CONTINUE6]] ], [ [[TMP20]], [[PRED_UDIV_IF7]] ]
+; INTERLEAVE-NEXT: [[TMP22:%.*]] = phi <4 x i32> [ [[TMP25]], [[PRED_UDIV_CONTINUE6]] ], [ [[TMP21]], [[PRED_UDIV_IF7]] ]
; INTERLEAVE-NEXT: br i1 [[C]], label [[PRED_UDIV_IF9:%.*]], label [[PRED_UDIV_CONTINUE10:%.*]]
; INTERLEAVE: pred.udiv.if9:
-; INTERLEAVE-NEXT: [[TMP22:%.*]] = add i32 [[INDEX]], 4
-; INTERLEAVE-NEXT: [[TMP23:%.*]] = extractelement <4 x i32> [[WIDE_LOAD2]], i64 0
-; INTERLEAVE-NEXT: [[TMP24:%.*]] = udiv i32 [[TMP23]], [[TMP22]]
-; INTERLEAVE-NEXT: [[TMP25:%.*]] = insertelement <4 x i32> poison, i32 [[TMP24]], i64 0
+; INTERLEAVE-NEXT: [[TMP23:%.*]] = add i32 [[INDEX]], 4
+; INTERLEAVE-NEXT: [[TMP24:%.*]] = extractelement <4 x i32> [[WIDE_LOAD2]], i64 0
+; INTERLEAVE-NEXT: [[TMP9:%.*]] = udiv i32 [[TMP24]], [[TMP23]]
+; INTERLEAVE-NEXT: [[BROADCAST_SPLATINSERT:%.*]] = insertelement <4 x i32> poison, i32 [[TMP9]], i64 0
; INTERLEAVE-NEXT: br label [[PRED_UDIV_CONTINUE10]]
; INTERLEAVE: pred.udiv.continue10:
-; INTERLEAVE-NEXT: [[TMP26:%.*]] = phi <4 x i32> [ poison, [[PRED_UDIV_CONTINUE8]] ], [ [[TMP25]], [[PRED_UDIV_IF9]] ]
+; INTERLEAVE-NEXT: [[TMP27:%.*]] = phi <4 x i32> [ poison, [[PRED_UDIV_CONTINUE8]] ], [ [[BROADCAST_SPLATINSERT]], [[PRED_UDIV_IF9]] ]
; INTERLEAVE-NEXT: br i1 [[C]], label [[PRED_UDIV_IF11:%.*]], label [[PRED_UDIV_CONTINUE12:%.*]]
; INTERLEAVE: pred.udiv.if11:
-; INTERLEAVE-NEXT: [[TMP27:%.*]] = add i32 [[INDEX]], 5
-; INTERLEAVE-NEXT: [[TMP28:%.*]] = extractelement <4 x i32> [[WIDE_LOAD2]], i64 1
-; INTERLEAVE-NEXT: [[TMP29:%.*]] = udiv i32 [[TMP28]], [[TMP27]]
-; INTERLEAVE-NEXT: [[TMP30:%.*]] = insertelement <4 x i32> [[TMP26]], i32 [[TMP29]], i64 1
+; INTERLEAVE-NEXT: [[TMP28:%.*]] = add i32 [[INDEX]], 5
+; INTERLEAVE-NEXT: [[TMP29:%.*]] = extractelement <4 x i32> [[WIDE_LOAD2]], i64 1
+; INTERLEAVE-NEXT: [[TMP30:%.*]] = udiv i32 [[TMP29]], [[TMP28]]
+; INTERLEAVE-NEXT: [[TMP31:%.*]] = insertelement <4 x i32> [[TMP27]], i32 [[TMP30]], i64 1
; INTERLEAVE-NEXT: br label [[PRED_UDIV_CONTINUE12]]
; INTERLEAVE: pred.udiv.continue12:
-; INTERLEAVE-NEXT: [[TMP31:%.*]] = phi <4 x i32> [ [[TMP26]], [[PRED_UDIV_CONTINUE10]] ], [ [[TMP30]], [[PRED_UDIV_IF11]] ]
+; INTERLEAVE-NEXT: [[TMP32:%.*]] = phi <4 x i32> [ [[TMP27]], [[PRED_UDIV_CONTINUE10]] ], [ [[TMP31]], [[PRED_UDIV_IF11]] ]
; INTERLEAVE-NEXT: br i1 [[C]], label [[PRED_UDIV_IF13:%.*]], label [[PRED_UDIV_CONTINUE14:%.*]]
; INTERLEAVE: pred.udiv.if13:
-; INTERLEAVE-NEXT: [[TMP32:%.*]] = add i32 [[INDEX]], 6
-; INTERLEAVE-NEXT: [[TMP33:%.*]] = extractelement <4 x i32> [[WIDE_LOAD2]], i64 2
-; INTERLEAVE-NEXT: [[TMP34:%.*]] = udiv i32 [[TMP33]], [[TMP32]]
-; INTERLEAVE-NEXT: [[TMP35:%.*]] = insertelement <4 x i32> [[TMP31]], i32 [[TMP34]], i64 2
+; INTERLEAVE-NEXT: [[TMP33:%.*]] = add i32 [[INDEX]], 6
+; INTERLEAVE-NEXT: [[TMP34:%.*]] = extractelement <4 x i32> [[WIDE_LOAD2]], i64 2
+; INTERLEAVE-NEXT: [[TMP35:%.*]] = udiv i32 [[TMP34]], [[TMP33]]
+; INTERLEAVE-NEXT: [[TMP36:%.*]] = insertelement <4 x i32> [[TMP32]], i32 [[TMP35]], i64 2
; INTERLEAVE-NEXT: br label [[PRED_UDIV_CONTINUE14]]
; INTERLEAVE: pred.udiv.continue14:
-; INTERLEAVE-NEXT: [[TMP36:%.*]] = phi <4 x i32> [ [[TMP31]], [[PRED_UDIV_CONTINUE12]] ], [ [[TMP35]], [[PRED_UDIV_IF13]] ]
+; INTERLEAVE-NEXT: [[TMP37:%.*]] = phi <4 x i32> [ [[TMP32]], [[PRED_UDIV_CONTINUE12]] ], [ [[TMP36]], [[PRED_UDIV_IF13]] ]
; INTERLEAVE-NEXT: br i1 [[C]], label [[PRED_UDIV_IF15:%.*]], label [[PRED_UDIV_CONTINUE16]]
; INTERLEAVE: pred.udiv.if15:
-; INTERLEAVE-NEXT: [[TMP37:%.*]] = add i32 [[INDEX]], 7
-; INTERLEAVE-NEXT: [[TMP38:%.*]] = extractelement <4 x i32> [[WIDE_LOAD2]], i64 3
-; INTERLEAVE-NEXT: [[TMP39:%.*]] = udiv i32 [[TMP38]], [[TMP37]]
-; INTERLEAVE-NEXT: [[TMP40:%.*]] = insertelement <4 x i32> [[TMP36]], i32 [[TMP39]], i64 3
+; INTERLEAVE-NEXT: [[TMP38:%.*]] = add i32 [[INDEX]], 7
+; INTERLEAVE-NEXT: [[TMP39:%.*]] = extractelement <4 x i32> [[WIDE_LOAD2]], i64 3
+; INTERLEAVE-NEXT: [[TMP40:%.*]] = udiv i32 [[TMP39]], [[TMP38]]
+; INTERLEAVE-NEXT: [[TMP41:%.*]] = insertelement <4 x i32> [[TMP37]], i32 [[TMP40]], i64 3
; INTERLEAVE-NEXT: br label [[PRED_UDIV_CONTINUE16]]
; INTERLEAVE: pred.udiv.continue16:
-; INTERLEAVE-NEXT: [[TMP41:%.*]] = phi <4 x i32> [ [[TMP36]], [[PRED_UDIV_CONTINUE14]] ], [ [[TMP40]], [[PRED_UDIV_IF15]] ]
-; INTERLEAVE-NEXT: [[PREDPHI:%.*]] = select i1 [[C]], <4 x i32> [[TMP21]], <4 x i32> [[WIDE_LOAD]]
-; INTERLEAVE-NEXT: [[PREDPHI17:%.*]] = select i1 [[C]], <4 x i32> [[TMP41]], <4 x i32> [[WIDE_LOAD2]]
+; INTERLEAVE-NEXT: [[TMP47:%.*]] = phi <4 x i32> [ [[TMP37]], [[PRED_UDIV_CONTINUE14]] ], [ [[TMP41]], [[PRED_UDIV_IF15]] ]
+; INTERLEAVE-NEXT: [[PREDPHI:%.*]] = select i1 [[C]], <4 x i32> [[TMP22]], <4 x i32> [[WIDE_LOAD]]
+; INTERLEAVE-NEXT: [[PREDPHI17:%.*]] = select i1 [[C]], <4 x i32> [[TMP47]], <4 x i32> [[WIDE_LOAD2]]
; INTERLEAVE-NEXT: [[TMP42]] = add <4 x i32> [[PREDPHI]], [[VEC_PHI]]
; INTERLEAVE-NEXT: [[TMP43]] = add <4 x i32> [[PREDPHI17]], [[VEC_PHI1]]
; INTERLEAVE-NEXT: [[INDEX_NEXT]] = add nuw i32 [[INDEX]], 8
diff --git a/llvm/test/Transforms/LoopVectorize/phi-with-fastflags.ll b/llvm/test/Transforms/LoopVectorize/phi-with-fastflags.ll
index 2ce2c932de2cb..488b72f2fe85e 100644
--- a/llvm/test/Transforms/LoopVectorize/phi-with-fastflags.ll
+++ b/llvm/test/Transforms/LoopVectorize/phi-with-fastflags.ll
@@ -9,10 +9,14 @@ define void @f(ptr noalias %p, i1 %c) {
; CHECK: [[VECTOR_PH]]:
; CHECK-NEXT: br label %[[VECTOR_BODY:.*]]
; CHECK: [[VECTOR_BODY]]:
-; CHECK-NEXT: [[INDEX:%.*]] = phi i32 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-NEXT: [[INDEX:%.*]] = phi i32 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[LATCH2:.*]] ]
; CHECK-NEXT: [[TMP0:%.*]] = getelementptr float, ptr [[P]], i32 [[INDEX]]
; CHECK-NEXT: [[WIDE_LOAD:%.*]] = load <4 x float>, ptr [[TMP0]], align 4
-; CHECK-NEXT: [[PREDPHI:%.*]] = select fast i1 [[C]], <4 x float> zeroinitializer, <4 x float> [[WIDE_LOAD]]
+; CHECK-NEXT: br i1 [[C]], label %[[THEN1:.*]], label %[[LATCH2]]
+; CHECK: [[THEN1]]:
+; CHECK-NEXT: br label %[[LATCH2]]
+; CHECK: [[LATCH2]]:
+; CHECK-NEXT: [[PREDPHI:%.*]] = select fast i1 true, <4 x float> zeroinitializer, <4 x float> [[WIDE_LOAD]]
; CHECK-NEXT: store <4 x float> [[PREDPHI]], ptr [[TMP0]], align 4
; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i32 [[INDEX]], 4
; CHECK-NEXT: [[TMP1:%.*]] = icmp eq i32 [[INDEX_NEXT]], 1024
diff --git a/llvm/test/Transforms/LoopVectorize/pr45525.ll b/llvm/test/Transforms/LoopVectorize/pr45525.ll
index ccba5001966c2..bbe279a6fae54 100644
--- a/llvm/test/Transforms/LoopVectorize/pr45525.ll
+++ b/llvm/test/Transforms/LoopVectorize/pr45525.ll
@@ -11,14 +11,19 @@ define void @main(i1 %cond, ptr %arr) {
; CHECK: [[VECTOR_PH]]:
; CHECK-NEXT: br label %[[VECTOR_BODY:.*]]
; CHECK: [[VECTOR_BODY]]:
-; CHECK-NEXT: [[INDEX:%.*]] = phi i32 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
-; CHECK-NEXT: [[VEC_IND:%.*]] = phi <4 x i32> [ <i32 0, i32 1, i32 2, i32 3>, %[[VECTOR_PH]] ], [ [[VEC_IND_NEXT:%.*]], %[[VECTOR_BODY]] ]
-; CHECK-NEXT: [[TMP5:%.*]] = mul <4 x i32> [[VEC_IND]], splat (i32 3)
-; CHECK-NEXT: [[PREDPHI:%.*]] = select i1 [[COND]], <4 x i32> splat (i32 7), <4 x i32> [[TMP5]]
+; CHECK-NEXT: [[INDEX:%.*]] = phi i32 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[BB_32:.*]] ]
+; CHECK-NEXT: br i1 [[COND]], label %[[BB_32]], label %[[BB_21:.*]]
+; CHECK: [[BB_21]]:
+; CHECK-NEXT: [[TMP0:%.*]] = mul i32 [[INDEX]], 3
+; CHECK-NEXT: br label %[[BB_32]]
+; CHECK: [[BB_32]]:
+; CHECK-NEXT: [[TMP3:%.*]] = phi i32 [ poison, %[[VECTOR_BODY]] ], [ [[TMP0]], %[[BB_21]] ]
+; CHECK-NEXT: [[BROADCAST_SPLATINSERT:%.*]] = insertelement <4 x i32> poison, i32 [[TMP3]], i64 0
+; CHECK-NEXT: [[BROADCAST_SPLAT:%.*]] = shufflevector <4 x i32> [[BROADCAST_SPLATINSERT]], <4 x i32> poison, <4 x i32> zeroinitializer
+; CHECK-NEXT: [[PREDPHI:%.*]] = select i1 true, <4 x i32> [[BROADCAST_SPLAT]], <4 x i32> splat (i32 7)
; CHECK-NEXT: [[TMP1:%.*]] = getelementptr inbounds i32, ptr [[ARR]], i32 [[INDEX]]
; CHECK-NEXT: store <4 x i32> [[PREDPHI]], ptr [[TMP1]], align 4
; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i32 [[INDEX]], 4
-; CHECK-NEXT: [[VEC_IND_NEXT]] = add <4 x i32> [[VEC_IND]], splat (i32 4)
; CHECK-NEXT: [[TMP2:%.*]] = icmp eq i32 [[INDEX_NEXT]], 32
; CHECK-NEXT: br i1 [[TMP2]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP0:![0-9]+]]
; CHECK: [[MIDDLE_BLOCK]]:
diff --git a/llvm/test/Transforms/LoopVectorize/pr55167-fold-tail-live-out.ll b/llvm/test/Transforms/LoopVectorize/pr55167-fold-tail-live-out.ll
index 88513fee4657c..2410514c735a2 100644
--- a/llvm/test/Transforms/LoopVectorize/pr55167-fold-tail-live-out.ll
+++ b/llvm/test/Transforms/LoopVectorize/pr55167-fold-tail-live-out.ll
@@ -6,34 +6,34 @@ define i32 @test(i32 %a, i1 %c.1, i1 %c.2 ) #0 {
; CHECK-NEXT: bb:
; CHECK-NEXT: br label [[VECTOR_PH:%.*]]
; CHECK: vector.ph:
+; CHECK-NEXT: [[BROADCAST_SPLATINSERT:%.*]] = insertelement <2 x i1> poison, i1 [[C_2:%.*]], i64 0
+; CHECK-NEXT: [[BROADCAST_SPLAT:%.*]] = shufflevector <2 x i1> [[BROADCAST_SPLATINSERT]], <2 x i1> poison, <2 x i32> zeroinitializer
; CHECK-NEXT: [[BROADCAST_SPLATINSERT1:%.*]] = insertelement <2 x i1> poison, i1 [[C_1:%.*]], i64 0
; CHECK-NEXT: [[BROADCAST_SPLAT2:%.*]] = shufflevector <2 x i1> [[BROADCAST_SPLATINSERT1]], <2 x i1> poison, <2 x i32> zeroinitializer
-; CHECK-NEXT: [[BROADCAST_SPLATINSERT3:%.*]] = insertelement <2 x i1> poison, i1 [[C_2:%.*]], i64 0
-; CHECK-NEXT: [[BROADCAST_SPLAT4:%.*]] = shufflevector <2 x i1> [[BROADCAST_SPLATINSERT3]], <2 x i1> poison, <2 x i32> zeroinitializer
-; CHECK-NEXT: [[TMP6:%.*]] = xor <2 x i1> [[BROADCAST_SPLAT4]], splat (i1 true)
-; CHECK-NEXT: [[TMP5:%.*]] = xor <2 x i1> [[BROADCAST_SPLAT2]], splat (i1 true)
-; CHECK-NEXT: [[TMP11:%.*]] = select <2 x i1> [[TMP6]], <2 x i1> [[TMP5]], <2 x i1> zeroinitializer
+; CHECK-NEXT: [[TMP3:%.*]] = xor <2 x i1> [[BROADCAST_SPLAT2]], splat (i1 true)
+; CHECK-NEXT: [[TMP1:%.*]] = xor <2 x i1> [[BROADCAST_SPLAT]], splat (i1 true)
+; CHECK-NEXT: [[TMP2:%.*]] = select <2 x i1> [[TMP3]], <2 x i1> [[TMP1]], <2 x i1> zeroinitializer
; CHECK-NEXT: [[TMP4:%.*]] = xor i32 [[A:%.*]], 1
-; CHECK-NEXT: [[BROADCAST_SPLATINSERT4:%.*]] = insertelement <2 x i32> poison, i32 [[TMP4]], i64 0
-; CHECK-NEXT: [[TMP2:%.*]] = shufflevector <2 x i32> [[BROADCAST_SPLATINSERT4]], <2 x i32> poison, <2 x i32> zeroinitializer
+; CHECK-NEXT: [[BROADCAST_SPLATINSERT3:%.*]] = insertelement <2 x i32> poison, i32 [[TMP4]], i64 0
+; CHECK-NEXT: [[BROADCAST_SPLAT4:%.*]] = shufflevector <2 x i32> [[BROADCAST_SPLATINSERT3]], <2 x i32> poison, <2 x i32> zeroinitializer
; CHECK-NEXT: br label [[VECTOR_BODY:%.*]]
; CHECK: vector.body:
; CHECK-NEXT: [[INDEX:%.*]] = phi i32 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ]
; CHECK-NEXT: [[VEC_IND:%.*]] = phi <2 x i32> [ <i32 6, i32 7>, [[VECTOR_PH]] ], [ [[VEC_IND_NEXT:%.*]], [[VECTOR_BODY]] ]
-; CHECK-NEXT: [[VEC_PHI:%.*]] = phi <2 x i32> [ <i32 35902, i32 0>, [[VECTOR_PH]] ], [ [[PREDPHI7:%.*]], [[VECTOR_BODY]] ]
+; CHECK-NEXT: [[VEC_PHI:%.*]] = phi <2 x i32> [ <i32 35902, i32 0>, [[VECTOR_PH]] ], [ [[PREDPHI6:%.*]], [[VECTOR_BODY]] ]
; CHECK-NEXT: [[TMP0:%.*]] = add <2 x i32> [[VEC_PHI]], splat (i32 10)
-; CHECK-NEXT: [[TMP1:%.*]] = add <2 x i32> [[TMP0]], splat (i32 20)
-; CHECK-NEXT: [[TMP3:%.*]] = add <2 x i32> [[TMP1]], [[TMP2]]
-; CHECK-NEXT: [[TMP7:%.*]] = extractelement <2 x i1> [[BROADCAST_SPLAT4]], i64 0
+; CHECK-NEXT: [[TMP5:%.*]] = add <2 x i32> [[TMP0]], splat (i32 20)
+; CHECK-NEXT: [[TMP6:%.*]] = add <2 x i32> [[TMP5]], [[BROADCAST_SPLAT4]]
+; CHECK-NEXT: [[TMP7:%.*]] = extractelement <2 x i1> [[BROADCAST_SPLAT2]], i64 0
; CHECK-NEXT: [[PREDPHI:%.*]] = select i1 [[TMP7]], <2 x i32> [[VEC_PHI]], <2 x i32> [[TMP0]]
-; CHECK-NEXT: [[PREDPHI7]] = select <2 x i1> [[TMP11]], <2 x i32> [[TMP3]], <2 x i32> [[PREDPHI]]
+; CHECK-NEXT: [[PREDPHI6]] = select <2 x i1> [[TMP2]], <2 x i32> [[TMP6]], <2 x i32> [[PREDPHI]]
; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i32 [[INDEX]], 2
; CHECK-NEXT: [[VEC_IND_NEXT]] = add nuw nsw <2 x i32> [[VEC_IND]], splat (i32 2)
; CHECK-NEXT: [[TMP8:%.*]] = icmp eq i32 [[INDEX_NEXT]], 176
; CHECK-NEXT: br i1 [[TMP8]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP0:![0-9]+]]
; CHECK: middle.block:
-; CHECK-NEXT: [[PREDPHI5:%.*]] = select i1 [[C_2]], <2 x i32> [[VEC_IND]], <2 x i32> splat (i32 9)
-; CHECK-NEXT: [[TMP10:%.*]] = call i32 @llvm.vector.reduce.add.v2i32(<2 x i32> [[PREDPHI7]])
+; CHECK-NEXT: [[PREDPHI5:%.*]] = select i1 [[C_1]], <2 x i32> [[VEC_IND]], <2 x i32> splat (i32 9)
+; CHECK-NEXT: [[TMP10:%.*]] = call i32 @llvm.vector.reduce.add.v2i32(<2 x i32> [[PREDPHI6]])
; CHECK-NEXT: [[TMP9:%.*]] = extractelement <2 x i32> [[PREDPHI5]], i64 1
; CHECK-NEXT: br label [[LOOP_LATCH:%.*]]
; CHECK: exit:
diff --git a/llvm/test/Transforms/LoopVectorize/predicatedinst-loop-invariant.ll b/llvm/test/Transforms/LoopVectorize/predicatedinst-loop-invariant.ll
index f26d0ddf66ddf..f02eb35547250 100644
--- a/llvm/test/Transforms/LoopVectorize/predicatedinst-loop-invariant.ll
+++ b/llvm/test/Transforms/LoopVectorize/predicatedinst-loop-invariant.ll
@@ -236,14 +236,32 @@ define void @test_store_to_invariant_address_needs_mask_due_to_low_trip_count(pt
; CHECK: [[VECTOR_PH]]:
; CHECK-NEXT: br label %[[VECTOR_BODY:.*]]
; CHECK: [[VECTOR_BODY]]:
+; CHECK-NEXT: br label %[[LOOP_LATCH1:.*]]
+; CHECK: [[LOOP_LATCH1]]:
+; CHECK-NEXT: br i1 true, label %[[PRED_STORE_IF:.*]], label %[[PRED_STORE_CONTINUE:.*]]
+; CHECK: [[PRED_STORE_IF]]:
+; CHECK-NEXT: store i32 0, ptr [[DST]], align 4
+; CHECK-NEXT: br label %[[PRED_STORE_CONTINUE]]
+; CHECK: [[PRED_STORE_CONTINUE]]:
+; CHECK-NEXT: br i1 true, label %[[PRED_STORE_IF2:.*]], label %[[PRED_STORE_CONTINUE3:.*]]
+; CHECK: [[PRED_STORE_IF2]]:
+; CHECK-NEXT: store i32 0, ptr [[DST]], align 4
+; CHECK-NEXT: br label %[[PRED_STORE_CONTINUE3]]
+; CHECK: [[PRED_STORE_CONTINUE3]]:
+; CHECK-NEXT: br i1 true, label %[[PRED_STORE_IF4:.*]], label %[[PRED_STORE_CONTINUE5:.*]]
+; CHECK: [[PRED_STORE_IF4]]:
+; CHECK-NEXT: store i32 0, ptr [[DST]], align 4
+; CHECK-NEXT: br label %[[PRED_STORE_CONTINUE5]]
+; CHECK: [[PRED_STORE_CONTINUE5]]:
+; CHECK-NEXT: br i1 false, label %[[PRED_STORE_IF6:.*]], label %[[MIDDLE_BLOCK:.*]]
+; CHECK: [[PRED_STORE_IF6]]:
; CHECK-NEXT: store i32 1, ptr [[DST]], align 4
-; CHECK-NEXT: store i32 1, ptr [[DST]], align 4
-; CHECK-NEXT: store i32 1, ptr [[DST]], align 4
-; CHECK-NEXT: store i32 1, ptr [[DST]], align 4
-; CHECK-NEXT: br label %[[MIDDLE_BLOCK:.*]]
+; CHECK-NEXT: br label %[[MIDDLE_BLOCK]]
; CHECK: [[MIDDLE_BLOCK]]:
; CHECK-NEXT: br label %[[EXIT:.*]]
; CHECK: [[EXIT]]:
+; CHECK-NEXT: br label %[[EXIT1:.*]]
+; CHECK: [[EXIT1]]:
; CHECK-NEXT: ret void
;
entry:
diff --git a/llvm/test/Transforms/LoopVectorize/predicator.ll b/llvm/test/Transforms/LoopVectorize/predicator.ll
index 3e9b0dfbe2caa..1822a2f59fc5d 100644
--- a/llvm/test/Transforms/LoopVectorize/predicator.ll
+++ b/llvm/test/Transforms/LoopVectorize/predicator.ll
@@ -178,20 +178,15 @@ define void @blend_masks_triangle_phi(ptr noalias %p, i1 %c0, i1 %c1) {
; CHECK-NEXT: [[ENTRY:.*:]]
; CHECK-NEXT: br label %[[VECTOR_PH:.*]]
; CHECK: [[VECTOR_PH]]:
-; CHECK-NEXT: [[BROADCAST_SPLATINSERT:%.*]] = insertelement <4 x i1> poison, i1 [[C1]], i64 0
-; CHECK-NEXT: [[BROADCAST_SPLAT:%.*]] = shufflevector <4 x i1> [[BROADCAST_SPLATINSERT]], <4 x i1> poison, <4 x i32> zeroinitializer
-; CHECK-NEXT: [[BROADCAST_SPLATINSERT1:%.*]] = insertelement <4 x i1> poison, i1 [[C0]], i64 0
-; CHECK-NEXT: [[BROADCAST_SPLAT2:%.*]] = shufflevector <4 x i1> [[BROADCAST_SPLATINSERT1]], <4 x i1> poison, <4 x i32> zeroinitializer
-; CHECK-NEXT: [[TMP0:%.*]] = xor <4 x i1> [[BROADCAST_SPLAT]], splat (i1 true)
-; CHECK-NEXT: [[TMP4:%.*]] = select <4 x i1> [[BROADCAST_SPLAT2]], <4 x i1> [[TMP0]], <4 x i1> zeroinitializer
-; CHECK-NEXT: [[TMP5:%.*]] = xor <4 x i1> [[BROADCAST_SPLAT2]], splat (i1 true)
-; CHECK-NEXT: [[TMP3:%.*]] = or <4 x i1> [[TMP4]], [[TMP5]]
-; CHECK-NEXT: [[PREDPHI:%.*]] = select <4 x i1> [[TMP3]], <4 x i32> splat (i32 1), <4 x i32> zeroinitializer
; CHECK-NEXT: br label %[[VECTOR_BODY:.*]]
; CHECK: [[VECTOR_BODY]]:
-; CHECK-NEXT: [[INDEX:%.*]] = phi i32 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-NEXT: [[INDEX:%.*]] = phi i32 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[BB22:.*]] ]
+; CHECK-NEXT: br i1 [[C0]], label %[[BB11:.*]], label %[[BB22]]
+; CHECK: [[BB11]]:
+; CHECK-NEXT: br label %[[BB22]]
+; CHECK: [[BB22]]:
; CHECK-NEXT: [[TMP1:%.*]] = getelementptr i32, ptr [[P]], i32 [[INDEX]]
-; CHECK-NEXT: store <4 x i32> [[PREDPHI]], ptr [[TMP1]], align 4
+; CHECK-NEXT: store <4 x i32> splat (i32 1), ptr [[TMP1]], align 4
; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i32 [[INDEX]], 4
; CHECK-NEXT: [[TMP2:%.*]] = icmp eq i32 [[INDEX_NEXT]], 128
; CHECK-NEXT: br i1 [[TMP2]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP4:![0-9]+]]
@@ -241,19 +236,22 @@ define void @simplifiable_blend(i1 %c1, i1 %c2, i1 %c3, i32 %x, i32 %y, ptr %p)
; CHECK-NEXT: [[ENTRY:.*:]]
; CHECK-NEXT: br label %[[VECTOR_PH:.*]]
; CHECK: [[VECTOR_PH]]:
-; CHECK-NEXT: [[BROADCAST_SPLATINSERT:%.*]] = insertelement <4 x i32> poison, i32 [[Y]], i64 0
-; CHECK-NEXT: [[BROADCAST_SPLAT:%.*]] = shufflevector <4 x i32> [[BROADCAST_SPLATINSERT]], <4 x i32> poison, <4 x i32> zeroinitializer
-; CHECK-NEXT: [[BROADCAST_SPLATINSERT1:%.*]] = insertelement <4 x i32> poison, i32 [[X]], i64 0
-; CHECK-NEXT: [[BROADCAST_SPLAT2:%.*]] = shufflevector <4 x i32> [[BROADCAST_SPLATINSERT1]], <4 x i32> poison, <4 x i32> zeroinitializer
-; CHECK-NEXT: [[PREDPHI7:%.*]] = select i1 [[C1]], <4 x i32> [[BROADCAST_SPLAT2]], <4 x i32> [[BROADCAST_SPLAT]]
; CHECK-NEXT: br label %[[VECTOR_BODY:.*]]
; CHECK: [[VECTOR_BODY]]:
-; CHECK-NEXT: [[INDEX:%.*]] = phi i32 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[F4:.*]] ]
-; CHECK-NEXT: br i1 [[C3]], label %[[F4]], label %[[E3:.*]]
+; CHECK-NEXT: [[INDEX:%.*]] = phi i32 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[LATCH5:.*]] ]
+; CHECK-NEXT: br i1 [[C1]], label %[[A4:.*]], label %[[B1:.*]]
+; CHECK: [[B1]]:
+; CHECK-NEXT: br i1 [[C3]], label %[[F4:.*]], label %[[E3:.*]]
; CHECK: [[E3]]:
; CHECK-NEXT: br label %[[F4]]
; CHECK: [[F4]]:
+; CHECK-NEXT: br label %[[LATCH5]]
+; CHECK: [[A4]]:
+; CHECK-NEXT: br label %[[LATCH5]]
+; CHECK: [[LATCH5]]:
; CHECK-NEXT: [[TMP3:%.*]] = getelementptr i32, ptr [[P]], i32 [[INDEX]]
+; CHECK-NEXT: [[BROADCAST_SPLATINSERT:%.*]] = insertelement <4 x i32> poison, i32 [[X]], i64 0
+; CHECK-NEXT: [[PREDPHI7:%.*]] = shufflevector <4 x i32> [[BROADCAST_SPLATINSERT]], <4 x i32> poison, <4 x i32> zeroinitializer
; CHECK-NEXT: store <4 x i32> [[PREDPHI7]], ptr [[TMP3]], align 4
; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i32 [[INDEX]], 4
; CHECK-NEXT: [[TMP4:%.*]] = icmp eq i32 [[INDEX_NEXT]], 128
@@ -309,21 +307,28 @@ define void @outermost_uniform_branch(ptr %a, i1 %u0) {
; CHECK: [[VECTOR_PH]]:
; CHECK-NEXT: br label %[[VECTOR_BODY:.*]]
; CHECK: [[VECTOR_BODY]]:
-; CHECK-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
-; CHECK-NEXT: [[VEC_IND:%.*]] = phi <4 x i64> [ <i64 0, i64 1, i64 2, i64 3>, %[[VECTOR_PH]] ], [ [[VEC_IND_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[BB42:.*]] ]
+; CHECK-NEXT: [[VEC_IND:%.*]] = phi <4 x i64> [ <i64 0, i64 1, i64 2, i64 3>, %[[VECTOR_PH]] ], [ [[VEC_IND_NEXT:%.*]], %[[BB42]] ]
+; CHECK-NEXT: br i1 [[U0]], label %[[BB11:.*]], label %[[BB42]]
+; CHECK: [[BB11]]:
+; CHECK-NEXT: [[TMP6:%.*]] = add i64 [[INDEX]], 1
+; CHECK-NEXT: [[TMP1:%.*]] = icmp sle i64 [[INDEX]], 1
+; CHECK-NEXT: [[TMP2:%.*]] = add i64 [[INDEX]], 2
+; CHECK-NEXT: [[PREDPHI:%.*]] = select i1 [[TMP1]], i64 [[TMP2]], i64 [[TMP6]]
+; CHECK-NEXT: [[TMP3:%.*]] = add i64 [[PREDPHI]], 3
+; CHECK-NEXT: br label %[[BB42]]
+; CHECK: [[BB42]]:
+; CHECK-NEXT: [[TMP4:%.*]] = phi i64 [ poison, %[[VECTOR_BODY]] ], [ [[TMP3]], %[[BB11]] ]
+; CHECK-NEXT: [[BROADCAST_SPLATINSERT:%.*]] = insertelement <4 x i64> poison, i64 [[TMP4]], i64 0
+; CHECK-NEXT: [[BROADCAST_SPLAT:%.*]] = shufflevector <4 x i64> [[BROADCAST_SPLATINSERT]], <4 x i64> poison, <4 x i32> zeroinitializer
+; CHECK-NEXT: [[PREDPHI3:%.*]] = select i1 true, <4 x i64> [[BROADCAST_SPLAT]], <4 x i64> [[VEC_IND]]
+; CHECK-NEXT: [[TMP5:%.*]] = extractelement <4 x i64> [[PREDPHI3]], i64 3
+; CHECK-NEXT: store i64 [[TMP5]], ptr [[A]], align 4
; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 4
; CHECK-NEXT: [[VEC_IND_NEXT]] = add nuw nsw <4 x i64> [[VEC_IND]], splat (i64 4)
; CHECK-NEXT: [[TMP0:%.*]] = icmp eq i64 [[INDEX_NEXT]], 128
; CHECK-NEXT: br i1 [[TMP0]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP6:![0-9]+]]
; CHECK: [[MIDDLE_BLOCK]]:
-; CHECK-NEXT: [[TMP1:%.*]] = add <4 x i64> [[VEC_IND]], splat (i64 1)
-; CHECK-NEXT: [[TMP2:%.*]] = icmp sle <4 x i64> [[VEC_IND]], splat (i64 1)
-; CHECK-NEXT: [[TMP3:%.*]] = add <4 x i64> [[VEC_IND]], splat (i64 2)
-; CHECK-NEXT: [[PREDPHI:%.*]] = select <4 x i1> [[TMP2]], <4 x i64> [[TMP3]], <4 x i64> [[TMP1]]
-; CHECK-NEXT: [[TMP4:%.*]] = add <4 x i64> [[PREDPHI]], splat (i64 3)
-; CHECK-NEXT: [[PREDPHI1:%.*]] = select i1 [[U0]], <4 x i64> [[TMP4]], <4 x i64> [[VEC_IND]]
-; CHECK-NEXT: [[TMP5:%.*]] = extractelement <4 x i64> [[PREDPHI1]], i64 3
-; CHECK-NEXT: store i64 [[TMP5]], ptr [[A]], align 4
; CHECK-NEXT: br label %[[EXIT:.*]]
; CHECK: [[EXIT]]:
; CHECK-NEXT: ret void
@@ -452,22 +457,27 @@ define void @outermost_uniform_branch_more_blocks(ptr %a, i1 %u0) {
; CHECK: [[VECTOR_PH]]:
; CHECK-NEXT: br label %[[VECTOR_BODY:.*]]
; CHECK: [[VECTOR_BODY]]:
-; CHECK-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
-; CHECK-NEXT: [[VEC_IND:%.*]] = phi <4 x i64> [ <i64 0, i64 1, i64 2, i64 3>, %[[VECTOR_PH]] ], [ [[VEC_IND_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[BB63:.*]] ]
+; CHECK-NEXT: br i1 [[U0]], label %[[BB22:.*]], label %[[BB11:.*]]
+; CHECK: [[BB11]]:
+; CHECK-NEXT: [[TMP7:%.*]] = add i64 [[INDEX]], 1
+; CHECK-NEXT: br label %[[BB63]]
+; CHECK: [[BB22]]:
+; CHECK-NEXT: [[TMP1:%.*]] = icmp sle i64 [[INDEX]], 2
+; CHECK-NEXT: [[TMP2:%.*]] = add i64 [[INDEX]], 3
+; CHECK-NEXT: [[TMP3:%.*]] = add i64 [[INDEX]], 4
+; CHECK-NEXT: [[PREDPHI:%.*]] = select i1 [[TMP1]], i64 [[TMP3]], i64 [[TMP2]]
+; CHECK-NEXT: [[TMP4:%.*]] = add i64 [[PREDPHI]], 5
+; CHECK-NEXT: br label %[[BB63]]
+; CHECK: [[BB63]]:
+; CHECK-NEXT: [[TMP5:%.*]] = phi i64 [ [[TMP7]], %[[BB11]] ], [ poison, %[[BB22]] ]
+; CHECK-NEXT: [[TMP6:%.*]] = phi i64 [ poison, %[[BB11]] ], [ [[TMP4]], %[[BB22]] ]
+; CHECK-NEXT: [[PREDPHI4:%.*]] = select i1 true, i64 [[TMP6]], i64 [[TMP5]]
+; CHECK-NEXT: store i64 [[PREDPHI4]], ptr [[A]], align 4
; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 4
-; CHECK-NEXT: [[VEC_IND_NEXT]] = add nuw nsw <4 x i64> [[VEC_IND]], splat (i64 4)
; CHECK-NEXT: [[TMP0:%.*]] = icmp eq i64 [[INDEX_NEXT]], 128
; CHECK-NEXT: br i1 [[TMP0]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP8:![0-9]+]]
; CHECK: [[MIDDLE_BLOCK]]:
-; CHECK-NEXT: [[TMP1:%.*]] = add <4 x i64> [[VEC_IND]], splat (i64 1)
-; CHECK-NEXT: [[TMP2:%.*]] = icmp sle <4 x i64> [[VEC_IND]], splat (i64 2)
-; CHECK-NEXT: [[TMP3:%.*]] = add <4 x i64> [[VEC_IND]], splat (i64 3)
-; CHECK-NEXT: [[TMP4:%.*]] = add <4 x i64> [[VEC_IND]], splat (i64 4)
-; CHECK-NEXT: [[PREDPHI:%.*]] = select <4 x i1> [[TMP2]], <4 x i64> [[TMP4]], <4 x i64> [[TMP3]]
-; CHECK-NEXT: [[TMP5:%.*]] = add <4 x i64> [[PREDPHI]], splat (i64 5)
-; CHECK-NEXT: [[PREDPHI1:%.*]] = select i1 [[U0]], <4 x i64> [[TMP5]], <4 x i64> [[TMP1]]
-; CHECK-NEXT: [[TMP6:%.*]] = extractelement <4 x i64> [[PREDPHI1]], i64 3
-; CHECK-NEXT: store i64 [[TMP6]], ptr [[A]], align 4
; CHECK-NEXT: br label %[[EXIT:.*]]
; CHECK: [[EXIT]]:
; CHECK-NEXT: ret void
@@ -614,22 +624,29 @@ define void @uniform_branch_after_varying_branch(ptr %a, i1 %u1) {
; CHECK: [[VECTOR_PH]]:
; CHECK-NEXT: br label %[[VECTOR_BODY:.*]]
; CHECK: [[VECTOR_BODY]]:
-; CHECK-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
-; CHECK-NEXT: [[VEC_IND:%.*]] = phi <4 x i64> [ <i64 0, i64 1, i64 2, i64 3>, %[[VECTOR_PH]] ], [ [[VEC_IND_NEXT:%.*]], %[[VECTOR_BODY]] ]
-; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 4
-; CHECK-NEXT: [[VEC_IND_NEXT]] = add nuw nsw <4 x i64> [[VEC_IND]], splat (i64 4)
-; CHECK-NEXT: [[TMP0:%.*]] = icmp eq i64 [[INDEX_NEXT]], 128
-; CHECK-NEXT: br i1 [[TMP0]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP10:![0-9]+]]
-; CHECK: [[MIDDLE_BLOCK]]:
+; CHECK-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[BB32:.*]] ]
+; CHECK-NEXT: [[VEC_IND:%.*]] = phi <4 x i64> [ <i64 0, i64 1, i64 2, i64 3>, %[[VECTOR_PH]] ], [ [[VEC_IND_NEXT:%.*]], %[[BB32]] ]
; CHECK-NEXT: [[TMP1:%.*]] = icmp sle <4 x i64> [[VEC_IND]], zeroinitializer
; CHECK-NEXT: [[TMP2:%.*]] = add <4 x i64> [[VEC_IND]], splat (i64 1)
-; CHECK-NEXT: [[TMP3:%.*]] = add <4 x i64> [[VEC_IND]], splat (i64 2)
-; CHECK-NEXT: [[PREDPHI:%.*]] = select i1 [[U1]], <4 x i64> [[TMP3]], <4 x i64> [[TMP2]]
+; CHECK-NEXT: br i1 [[U1]], label %[[BB21:.*]], label %[[BB32]]
+; CHECK: [[BB21]]:
+; CHECK-NEXT: [[TMP8:%.*]] = add i64 [[INDEX]], 2
+; CHECK-NEXT: br label %[[BB32]]
+; CHECK: [[BB32]]:
+; CHECK-NEXT: [[TMP3:%.*]] = phi i64 [ poison, %[[VECTOR_BODY]] ], [ [[TMP8]], %[[BB21]] ]
+; CHECK-NEXT: [[BROADCAST_SPLATINSERT:%.*]] = insertelement <4 x i64> poison, i64 [[TMP3]], i64 0
+; CHECK-NEXT: [[BROADCAST_SPLAT:%.*]] = shufflevector <4 x i64> [[BROADCAST_SPLATINSERT]], <4 x i64> poison, <4 x i32> zeroinitializer
+; CHECK-NEXT: [[PREDPHI:%.*]] = select <4 x i1> [[TMP1]], <4 x i64> [[TMP2]], <4 x i64> [[BROADCAST_SPLAT]]
; CHECK-NEXT: [[TMP4:%.*]] = add <4 x i64> [[PREDPHI]], splat (i64 3)
; CHECK-NEXT: [[TMP5:%.*]] = add <4 x i64> [[VEC_IND]], splat (i64 4)
; CHECK-NEXT: [[PREDPHI1:%.*]] = select <4 x i1> [[TMP1]], <4 x i64> [[TMP5]], <4 x i64> [[TMP4]]
; CHECK-NEXT: [[TMP6:%.*]] = extractelement <4 x i64> [[PREDPHI1]], i64 3
; CHECK-NEXT: store i64 [[TMP6]], ptr [[A]], align 4
+; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 4
+; CHECK-NEXT: [[VEC_IND_NEXT]] = add nuw nsw <4 x i64> [[VEC_IND]], splat (i64 4)
+; CHECK-NEXT: [[TMP7:%.*]] = icmp eq i64 [[INDEX_NEXT]], 128
+; CHECK-NEXT: br i1 [[TMP7]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP10:![0-9]+]]
+; CHECK: [[MIDDLE_BLOCK]]:
; CHECK-NEXT: br label %[[EXIT:.*]]
; CHECK: [[EXIT]]:
; CHECK-NEXT: ret void
@@ -764,22 +781,34 @@ define void @uniform_branch_after_varying_branch_more_blocks(ptr %a, i1 %u1) {
; CHECK: [[VECTOR_PH]]:
; CHECK-NEXT: br label %[[VECTOR_BODY:.*]]
; CHECK: [[VECTOR_BODY]]:
-; CHECK-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
-; CHECK-NEXT: [[VEC_IND:%.*]] = phi <4 x i64> [ <i64 0, i64 1, i64 2, i64 3>, %[[VECTOR_PH]] ], [ [[VEC_IND_NEXT:%.*]], %[[VECTOR_BODY]] ]
-; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 4
-; CHECK-NEXT: [[VEC_IND_NEXT]] = add nuw nsw <4 x i64> [[VEC_IND]], splat (i64 4)
-; CHECK-NEXT: [[TMP0:%.*]] = icmp eq i64 [[INDEX_NEXT]], 128
-; CHECK-NEXT: br i1 [[TMP0]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP12:![0-9]+]]
-; CHECK: [[MIDDLE_BLOCK]]:
+; CHECK-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[BB43:.*]] ]
+; CHECK-NEXT: [[VEC_IND:%.*]] = phi <4 x i64> [ <i64 0, i64 1, i64 2, i64 3>, %[[VECTOR_PH]] ], [ [[VEC_IND_NEXT:%.*]], %[[BB43]] ]
; CHECK-NEXT: [[TMP1:%.*]] = icmp sle <4 x i64> [[VEC_IND]], zeroinitializer
-; CHECK-NEXT: [[TMP2:%.*]] = add <4 x i64> [[VEC_IND]], splat (i64 2)
-; CHECK-NEXT: [[TMP3:%.*]] = add <4 x i64> [[VEC_IND]], splat (i64 3)
-; CHECK-NEXT: [[PREDPHI:%.*]] = select i1 [[U1]], <4 x i64> [[TMP3]], <4 x i64> [[TMP2]]
+; CHECK-NEXT: br i1 [[U1]], label %[[BB32:.*]], label %[[BB21:.*]]
+; CHECK: [[BB21]]:
+; CHECK-NEXT: [[TMP7:%.*]] = add i64 [[INDEX]], 2
+; CHECK-NEXT: br label %[[BB43]]
+; CHECK: [[BB32]]:
+; CHECK-NEXT: [[TMP2:%.*]] = add i64 [[INDEX]], 3
+; CHECK-NEXT: br label %[[BB43]]
+; CHECK: [[BB43]]:
+; CHECK-NEXT: [[TMP3:%.*]] = phi i64 [ [[TMP7]], %[[BB21]] ], [ poison, %[[BB32]] ]
+; CHECK-NEXT: [[TMP9:%.*]] = phi i64 [ poison, %[[BB21]] ], [ [[TMP2]], %[[BB32]] ]
+; CHECK-NEXT: [[BROADCAST_SPLATINSERT4:%.*]] = insertelement <4 x i64> poison, i64 [[TMP9]], i64 0
+; CHECK-NEXT: [[BROADCAST_SPLAT5:%.*]] = shufflevector <4 x i64> [[BROADCAST_SPLATINSERT4]], <4 x i64> poison, <4 x i32> zeroinitializer
+; CHECK-NEXT: [[BROADCAST_SPLATINSERT:%.*]] = insertelement <4 x i64> poison, i64 [[TMP3]], i64 0
+; CHECK-NEXT: [[BROADCAST_SPLAT:%.*]] = shufflevector <4 x i64> [[BROADCAST_SPLATINSERT]], <4 x i64> poison, <4 x i32> zeroinitializer
+; CHECK-NEXT: [[PREDPHI:%.*]] = select <4 x i1> [[TMP1]], <4 x i64> [[BROADCAST_SPLAT]], <4 x i64> [[BROADCAST_SPLAT5]]
; CHECK-NEXT: [[TMP4:%.*]] = add <4 x i64> [[PREDPHI]], splat (i64 4)
; CHECK-NEXT: [[TMP5:%.*]] = add <4 x i64> [[VEC_IND]], splat (i64 5)
; CHECK-NEXT: [[PREDPHI1:%.*]] = select <4 x i1> [[TMP1]], <4 x i64> [[TMP5]], <4 x i64> [[TMP4]]
; CHECK-NEXT: [[TMP6:%.*]] = extractelement <4 x i64> [[PREDPHI1]], i64 3
; CHECK-NEXT: store i64 [[TMP6]], ptr [[A]], align 4
+; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 4
+; CHECK-NEXT: [[VEC_IND_NEXT]] = add nuw nsw <4 x i64> [[VEC_IND]], splat (i64 4)
+; CHECK-NEXT: [[TMP8:%.*]] = icmp eq i64 [[INDEX_NEXT]], 128
+; CHECK-NEXT: br i1 [[TMP8]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP12:![0-9]+]]
+; CHECK: [[MIDDLE_BLOCK]]:
; CHECK-NEXT: br label %[[EXIT:.*]]
; CHECK: [[EXIT]]:
; CHECK-NEXT: ret void
@@ -926,22 +955,34 @@ define void @uniform_branch_after_varying_branch_more_blocks_mirrored(ptr %a, i1
; CHECK: [[VECTOR_PH]]:
; CHECK-NEXT: br label %[[VECTOR_BODY:.*]]
; CHECK: [[VECTOR_BODY]]:
-; CHECK-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
-; CHECK-NEXT: [[VEC_IND:%.*]] = phi <4 x i64> [ <i64 0, i64 1, i64 2, i64 3>, %[[VECTOR_PH]] ], [ [[VEC_IND_NEXT:%.*]], %[[VECTOR_BODY]] ]
-; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 4
-; CHECK-NEXT: [[VEC_IND_NEXT]] = add nuw nsw <4 x i64> [[VEC_IND]], splat (i64 4)
-; CHECK-NEXT: [[TMP0:%.*]] = icmp eq i64 [[INDEX_NEXT]], 128
-; CHECK-NEXT: br i1 [[TMP0]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP14:![0-9]+]]
-; CHECK: [[MIDDLE_BLOCK]]:
+; CHECK-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[BB53:.*]] ]
+; CHECK-NEXT: [[VEC_IND:%.*]] = phi <4 x i64> [ <i64 0, i64 1, i64 2, i64 3>, %[[VECTOR_PH]] ], [ [[VEC_IND_NEXT:%.*]], %[[BB53]] ]
; CHECK-NEXT: [[TMP1:%.*]] = icmp sle <4 x i64> [[VEC_IND]], zeroinitializer
; CHECK-NEXT: [[TMP2:%.*]] = add <4 x i64> [[VEC_IND]], splat (i64 1)
-; CHECK-NEXT: [[TMP3:%.*]] = add <4 x i64> [[VEC_IND]], splat (i64 3)
-; CHECK-NEXT: [[TMP4:%.*]] = add <4 x i64> [[VEC_IND]], splat (i64 4)
-; CHECK-NEXT: [[PREDPHI:%.*]] = select i1 [[U2]], <4 x i64> [[TMP4]], <4 x i64> [[TMP3]]
+; CHECK-NEXT: br i1 [[U2]], label %[[BB42:.*]], label %[[BB31:.*]]
+; CHECK: [[BB31]]:
+; CHECK-NEXT: [[TMP7:%.*]] = add i64 [[INDEX]], 3
+; CHECK-NEXT: br label %[[BB53]]
+; CHECK: [[BB42]]:
+; CHECK-NEXT: [[TMP3:%.*]] = add i64 [[INDEX]], 4
+; CHECK-NEXT: br label %[[BB53]]
+; CHECK: [[BB53]]:
+; CHECK-NEXT: [[TMP4:%.*]] = phi i64 [ [[TMP7]], %[[BB31]] ], [ poison, %[[BB42]] ]
+; CHECK-NEXT: [[TMP9:%.*]] = phi i64 [ poison, %[[BB31]] ], [ [[TMP3]], %[[BB42]] ]
+; CHECK-NEXT: [[BROADCAST_SPLATINSERT4:%.*]] = insertelement <4 x i64> poison, i64 [[TMP4]], i64 0
+; CHECK-NEXT: [[BROADCAST_SPLAT5:%.*]] = shufflevector <4 x i64> [[BROADCAST_SPLATINSERT4]], <4 x i64> poison, <4 x i32> zeroinitializer
+; CHECK-NEXT: [[BROADCAST_SPLATINSERT:%.*]] = insertelement <4 x i64> poison, i64 [[TMP9]], i64 0
+; CHECK-NEXT: [[BROADCAST_SPLAT:%.*]] = shufflevector <4 x i64> [[BROADCAST_SPLATINSERT]], <4 x i64> poison, <4 x i32> zeroinitializer
+; CHECK-NEXT: [[PREDPHI:%.*]] = select <4 x i1> [[TMP1]], <4 x i64> [[BROADCAST_SPLAT]], <4 x i64> [[BROADCAST_SPLAT5]]
; CHECK-NEXT: [[TMP5:%.*]] = add <4 x i64> [[PREDPHI]], splat (i64 5)
; CHECK-NEXT: [[PREDPHI1:%.*]] = select <4 x i1> [[TMP1]], <4 x i64> [[TMP5]], <4 x i64> [[TMP2]]
; CHECK-NEXT: [[TMP6:%.*]] = extractelement <4 x i64> [[PREDPHI1]], i64 3
; CHECK-NEXT: store i64 [[TMP6]], ptr [[A]], align 4
+; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 4
+; CHECK-NEXT: [[VEC_IND_NEXT]] = add nuw nsw <4 x i64> [[VEC_IND]], splat (i64 4)
+; CHECK-NEXT: [[TMP8:%.*]] = icmp eq i64 [[INDEX_NEXT]], 128
+; CHECK-NEXT: br i1 [[TMP8]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP14:![0-9]+]]
+; CHECK: [[MIDDLE_BLOCK]]:
; CHECK-NEXT: br label %[[EXIT:.*]]
; CHECK: [[EXIT]]:
; CHECK-NEXT: ret void
@@ -1984,26 +2025,30 @@ define void @uniform_branch_shared_join_with_varying(ptr %a, i1 %u0) {
; CHECK-NEXT: [[ENTRY:.*:]]
; CHECK-NEXT: br label %[[VECTOR_PH:.*]]
; CHECK: [[VECTOR_PH]]:
-; CHECK-NEXT: [[BROADCAST_SPLATINSERT:%.*]] = insertelement <4 x i1> poison, i1 [[U0]], i64 0
-; CHECK-NEXT: [[BROADCAST_SPLAT:%.*]] = shufflevector <4 x i1> [[BROADCAST_SPLATINSERT]], <4 x i1> poison, <4 x i32> zeroinitializer
; CHECK-NEXT: br label %[[VECTOR_BODY:.*]]
; CHECK: [[VECTOR_BODY]]:
-; CHECK-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
-; CHECK-NEXT: [[VEC_IND:%.*]] = phi <4 x i64> [ <i64 0, i64 1, i64 2, i64 3>, %[[VECTOR_PH]] ], [ [[VEC_IND_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[BB43:.*]] ]
+; CHECK-NEXT: br i1 [[U0]], label %[[BB22:.*]], label %[[BB11:.*]]
+; CHECK: [[BB11]]:
+; CHECK-NEXT: [[TMP8:%.*]] = add i64 [[INDEX]], 1
+; CHECK-NEXT: br label %[[BB43]]
+; CHECK: [[BB22]]:
+; CHECK-NEXT: [[TMP1:%.*]] = add i64 [[INDEX]], 2
+; CHECK-NEXT: [[TMP2:%.*]] = icmp sle i64 [[INDEX]], 2
+; CHECK-NEXT: [[TMP3:%.*]] = add i64 [[INDEX]], 3
+; CHECK-NEXT: br label %[[BB43]]
+; CHECK: [[BB43]]:
+; CHECK-NEXT: [[TMP4:%.*]] = phi i64 [ [[TMP8]], %[[BB11]] ], [ poison, %[[BB22]] ]
+; CHECK-NEXT: [[TMP5:%.*]] = phi i64 [ poison, %[[BB11]] ], [ [[TMP1]], %[[BB22]] ]
+; CHECK-NEXT: [[TMP6:%.*]] = phi i64 [ poison, %[[BB11]] ], [ [[TMP3]], %[[BB22]] ]
+; CHECK-NEXT: [[TMP7:%.*]] = phi i1 [ false, %[[BB11]] ], [ [[TMP2]], %[[BB22]] ]
+; CHECK-NEXT: [[PREDPHI:%.*]] = select i1 true, i64 [[TMP5]], i64 [[TMP4]]
+; CHECK-NEXT: [[PREDPHI4:%.*]] = select i1 [[TMP7]], i64 [[TMP6]], i64 [[PREDPHI]]
+; CHECK-NEXT: store i64 [[PREDPHI4]], ptr [[A]], align 4
; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 4
-; CHECK-NEXT: [[VEC_IND_NEXT]] = add nuw nsw <4 x i64> [[VEC_IND]], splat (i64 4)
; CHECK-NEXT: [[TMP0:%.*]] = icmp eq i64 [[INDEX_NEXT]], 128
; CHECK-NEXT: br i1 [[TMP0]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP38:![0-9]+]]
; CHECK: [[MIDDLE_BLOCK]]:
-; CHECK-NEXT: [[TMP1:%.*]] = add <4 x i64> [[VEC_IND]], splat (i64 1)
-; CHECK-NEXT: [[TMP2:%.*]] = add <4 x i64> [[VEC_IND]], splat (i64 2)
-; CHECK-NEXT: [[TMP3:%.*]] = icmp sle <4 x i64> [[VEC_IND]], splat (i64 2)
-; CHECK-NEXT: [[TMP4:%.*]] = select <4 x i1> [[BROADCAST_SPLAT]], <4 x i1> [[TMP3]], <4 x i1> zeroinitializer
-; CHECK-NEXT: [[TMP5:%.*]] = add <4 x i64> [[VEC_IND]], splat (i64 3)
-; CHECK-NEXT: [[PREDPHI:%.*]] = select i1 [[U0]], <4 x i64> [[TMP2]], <4 x i64> [[TMP1]]
-; CHECK-NEXT: [[PREDPHI1:%.*]] = select <4 x i1> [[TMP4]], <4 x i64> [[TMP5]], <4 x i64> [[PREDPHI]]
-; CHECK-NEXT: [[TMP6:%.*]] = extractelement <4 x i64> [[PREDPHI1]], i64 3
-; CHECK-NEXT: store i64 [[TMP6]], ptr [[A]], align 4
; CHECK-NEXT: br label %[[EXIT:.*]]
; CHECK: [[EXIT]]:
; CHECK-NEXT: ret void
diff --git a/llvm/test/Transforms/LoopVectorize/reduction-small-size.ll b/llvm/test/Transforms/LoopVectorize/reduction-small-size.ll
index 592c60a40bf6f..1131272009f4f 100644
--- a/llvm/test/Transforms/LoopVectorize/reduction-small-size.ll
+++ b/llvm/test/Transforms/LoopVectorize/reduction-small-size.ll
@@ -13,17 +13,17 @@ define i8 @PR34687(i1 %c, i32 %x, i32 %n, i32 %divisor) {
; CHECK-NEXT: [[N_VEC:%.*]] = sub i32 [[N]], [[N_MOD_VF]]
; CHECK-NEXT: [[BROADCAST_SPLATINSERT:%.*]] = insertelement <4 x i1> poison, i1 [[C:%.*]], i64 0
; CHECK-NEXT: [[BROADCAST_SPLAT:%.*]] = shufflevector <4 x i1> [[BROADCAST_SPLATINSERT]], <4 x i1> poison, <4 x i32> zeroinitializer
-; CHECK-NEXT: [[BROADCAST_SPLATINSERT1:%.*]] = insertelement <4 x i32> poison, i32 [[X:%.*]], i64 0
-; CHECK-NEXT: [[BROADCAST_SPLAT2:%.*]] = shufflevector <4 x i32> [[BROADCAST_SPLATINSERT1]], <4 x i32> poison, <4 x i32> zeroinitializer
; CHECK-NEXT: [[BROADCAST_SPLATINSERT3:%.*]] = insertelement <4 x i32> poison, i32 [[X1:%.*]], i64 0
; CHECK-NEXT: [[BROADCAST_SPLAT4:%.*]] = shufflevector <4 x i32> [[BROADCAST_SPLATINSERT3]], <4 x i32> poison, <4 x i32> zeroinitializer
+; CHECK-NEXT: [[BROADCAST_SPLATINSERT4:%.*]] = insertelement <4 x i32> poison, i32 [[X:%.*]], i64 0
+; CHECK-NEXT: [[BROADCAST_SPLAT5:%.*]] = shufflevector <4 x i32> [[BROADCAST_SPLATINSERT4]], <4 x i32> poison, <4 x i32> zeroinitializer
; CHECK-NEXT: br label [[VECTOR_BODY:%.*]]
; CHECK: vector.body:
; CHECK-NEXT: [[INDEX:%.*]] = phi i32 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ]
; CHECK-NEXT: [[VEC_IND:%.*]] = phi <4 x i32> [ <i32 0, i32 1, i32 2, i32 3>, [[VECTOR_PH]] ], [ [[VEC_IND_NEXT:%.*]], [[VECTOR_BODY]] ]
; CHECK-NEXT: [[VEC_PHI:%.*]] = phi <4 x i32> [ zeroinitializer, [[VECTOR_PH]] ], [ [[TMP4:%.*]], [[VECTOR_BODY]] ]
; CHECK-NEXT: [[TMP1:%.*]] = and <4 x i32> [[VEC_PHI]], splat (i32 255)
-; CHECK-NEXT: [[TMP2:%.*]] = add <4 x i32> [[TMP1]], [[BROADCAST_SPLAT4]]
+; CHECK-NEXT: [[TMP2:%.*]] = add <4 x i32> [[TMP1]], [[BROADCAST_SPLAT5]]
; CHECK-NEXT: [[TMP3:%.*]] = trunc <4 x i32> [[TMP2]] to <4 x i8>
; CHECK-NEXT: [[TMP4]] = zext <4 x i8> [[TMP3]] to <4 x i32>
; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i32 [[INDEX]], 4
@@ -31,7 +31,7 @@ define i8 @PR34687(i1 %c, i32 %x, i32 %n, i32 %divisor) {
; CHECK-NEXT: [[TMP5:%.*]] = icmp eq i32 [[INDEX_NEXT]], [[N_VEC]]
; CHECK-NEXT: br i1 [[TMP5]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP0:![0-9]+]]
; CHECK: middle.block:
-; CHECK-NEXT: [[TMP6:%.*]] = call <4 x i32> @llvm.masked.sdiv.v4i32(<4 x i32> [[VEC_IND]], <4 x i32> [[BROADCAST_SPLAT2]], <4 x i1> [[BROADCAST_SPLAT]])
+; CHECK-NEXT: [[TMP6:%.*]] = call <4 x i32> @llvm.masked.sdiv.v4i32(<4 x i32> [[VEC_IND]], <4 x i32> [[BROADCAST_SPLAT4]], <4 x i1> [[BROADCAST_SPLAT]])
; CHECK-NEXT: [[PREDPHI1:%.*]] = select i1 [[C]], <4 x i32> [[TMP6]], <4 x i32> zeroinitializer
; CHECK-NEXT: [[TMP7:%.*]] = call i8 @llvm.vector.reduce.add.v4i8(<4 x i8> [[TMP3]])
; CHECK-NEXT: [[TMP8:%.*]] = zext i8 [[TMP7]] to i32
@@ -47,13 +47,13 @@ define i8 @PR34687(i1 %c, i32 %x, i32 %n, i32 %divisor) {
; CHECK-NEXT: [[R:%.*]] = phi i32 [ [[BC_MERGE_RDX]], [[SCALAR_PH]] ], [ [[R_NEXT:%.*]], [[IF_END]] ]
; CHECK-NEXT: br i1 [[C]], label [[IF_THEN:%.*]], label [[IF_END]]
; CHECK: if.then:
-; CHECK-NEXT: [[T0:%.*]] = sdiv i32 [[I]], [[X]]
+; CHECK-NEXT: [[T0:%.*]] = sdiv i32 [[I]], [[X1]]
; CHECK-NEXT: br label [[IF_END]]
; CHECK: if.end:
; CHECK-NEXT: [[DIV_PHI:%.*]] = phi i32 [ 0, [[FOR_BODY]] ], [ [[T0]], [[IF_THEN]] ]
; CHECK-NEXT: [[T1:%.*]] = and i32 [[R]], 255
; CHECK-NEXT: [[I_NEXT]] = add nsw i32 [[I]], 1
-; CHECK-NEXT: [[R_NEXT]] = add nuw nsw i32 [[T1]], [[X1]]
+; CHECK-NEXT: [[R_NEXT]] = add nuw nsw i32 [[T1]], [[X]]
; CHECK-NEXT: [[COND:%.*]] = icmp eq i32 [[I_NEXT]], [[N]]
; CHECK-NEXT: br i1 [[COND]], label [[FOR_END]], label [[FOR_BODY]], !llvm.loop [[LOOP3:![0-9]+]]
; CHECK: for.end:
@@ -97,11 +97,11 @@ define i8 @PR34687_no_undef(i1 %c, i32 %x, i32 %n) {
; CHECK: vector.ph:
; CHECK-NEXT: [[N_MOD_VF:%.*]] = and i32 [[N]], 3
; CHECK-NEXT: [[N_VEC:%.*]] = sub i32 [[N]], [[N_MOD_VF]]
-; CHECK-NEXT: [[BROADCAST_SPLATINSERT1:%.*]] = insertelement <4 x i32> poison, i32 [[X:%.*]], i64 0
-; CHECK-NEXT: [[BROADCAST_SPLAT2:%.*]] = shufflevector <4 x i32> [[BROADCAST_SPLATINSERT1]], <4 x i32> poison, <4 x i32> zeroinitializer
-; CHECK-NEXT: [[BROADCAST_SPLATINSERT2:%.*]] = insertelement <4 x i1> poison, i1 [[C:%.*]], i64 0
-; CHECK-NEXT: [[BROADCAST_SPLAT3:%.*]] = shufflevector <4 x i1> [[BROADCAST_SPLATINSERT2]], <4 x i1> poison, <4 x i32> zeroinitializer
-; CHECK-NEXT: [[TMP1:%.*]] = call <4 x i32> @llvm.masked.sdiv.v4i32(<4 x i32> splat (i32 99), <4 x i32> [[BROADCAST_SPLAT2]], <4 x i1> [[BROADCAST_SPLAT3]])
+; CHECK-NEXT: [[BROADCAST_SPLATINSERT:%.*]] = insertelement <4 x i32> poison, i32 [[X:%.*]], i64 0
+; CHECK-NEXT: [[BROADCAST_SPLAT:%.*]] = shufflevector <4 x i32> [[BROADCAST_SPLATINSERT]], <4 x i32> poison, <4 x i32> zeroinitializer
+; CHECK-NEXT: [[BROADCAST_SPLATINSERT1:%.*]] = insertelement <4 x i1> poison, i1 [[C:%.*]], i64 0
+; CHECK-NEXT: [[BROADCAST_SPLAT2:%.*]] = shufflevector <4 x i1> [[BROADCAST_SPLATINSERT1]], <4 x i1> poison, <4 x i32> zeroinitializer
+; CHECK-NEXT: [[TMP1:%.*]] = call <4 x i32> @llvm.masked.sdiv.v4i32(<4 x i32> splat (i32 99), <4 x i32> [[BROADCAST_SPLAT]], <4 x i1> [[BROADCAST_SPLAT2]])
; CHECK-NEXT: [[PREDPHI:%.*]] = select i1 [[C]], <4 x i32> [[TMP1]], <4 x i32> zeroinitializer
; CHECK-NEXT: br label [[VECTOR_BODY:%.*]]
; CHECK: vector.body:
diff --git a/llvm/test/Transforms/LoopVectorize/scalarize-masked-call.ll b/llvm/test/Transforms/LoopVectorize/scalarize-masked-call.ll
index 396aba95beb1c..1158686c08c19 100644
--- a/llvm/test/Transforms/LoopVectorize/scalarize-masked-call.ll
+++ b/llvm/test/Transforms/LoopVectorize/scalarize-masked-call.ll
@@ -104,21 +104,15 @@ define void @masked_umax_used_by_load_address(ptr noalias %src, ptr noalias %dst
; CHECK-NEXT: br label [[VECTOR_BODY:%.*]]
; CHECK: vector.body:
; CHECK-NEXT: [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[PRED_LOAD_CONTINUE2:%.*]] ]
-; CHECK-NEXT: br i1 [[C:%.*]], label [[PRED_LOAD_IF:%.*]], label [[PRED_LOAD_CONTINUE:%.*]]
-; CHECK: pred.load.if:
-; CHECK-NEXT: [[TMP2:%.*]] = load i16, ptr [[TMP1]], align 2
-; CHECK-NEXT: [[TMP3:%.*]] = insertelement <2 x i16> poison, i16 [[TMP2]], i64 0
-; CHECK-NEXT: br label [[PRED_LOAD_CONTINUE]]
-; CHECK: pred.load.continue:
-; CHECK-NEXT: [[TMP4:%.*]] = phi <2 x i16> [ poison, [[VECTOR_BODY]] ], [ [[TMP3]], [[PRED_LOAD_IF]] ]
-; CHECK-NEXT: br i1 [[C]], label [[PRED_LOAD_IF1:%.*]], label [[PRED_LOAD_CONTINUE2]]
-; CHECK: pred.load.if1:
+; CHECK-NEXT: br i1 [[C:%.*]], label [[PRED_LOAD_IF1:%.*]], label [[PRED_LOAD_CONTINUE2]]
+; CHECK: then1:
; CHECK-NEXT: [[TMP5:%.*]] = load i16, ptr [[TMP1]], align 2
-; CHECK-NEXT: [[TMP6:%.*]] = insertelement <2 x i16> [[TMP4]], i16 [[TMP5]], i64 1
; CHECK-NEXT: br label [[PRED_LOAD_CONTINUE2]]
-; CHECK: pred.load.continue2:
-; CHECK-NEXT: [[TMP7:%.*]] = phi <2 x i16> [ [[TMP4]], [[PRED_LOAD_CONTINUE]] ], [ [[TMP6]], [[PRED_LOAD_IF1]] ]
-; CHECK-NEXT: [[PREDPHI:%.*]] = select i1 [[C]], <2 x i16> [[TMP7]], <2 x i16> zeroinitializer
+; CHECK: loop.latch2:
+; CHECK-NEXT: [[TMP3:%.*]] = phi i16 [ poison, [[VECTOR_BODY]] ], [ [[TMP5]], [[PRED_LOAD_IF1]] ]
+; CHECK-NEXT: [[BROADCAST_SPLATINSERT:%.*]] = insertelement <2 x i16> poison, i16 [[TMP3]], i64 0
+; CHECK-NEXT: [[BROADCAST_SPLAT:%.*]] = shufflevector <2 x i16> [[BROADCAST_SPLATINSERT]], <2 x i16> poison, <2 x i32> zeroinitializer
+; CHECK-NEXT: [[PREDPHI:%.*]] = select i1 true, <2 x i16> [[BROADCAST_SPLAT]], <2 x i16> zeroinitializer
; CHECK-NEXT: [[TMP8:%.*]] = getelementptr i16, ptr [[DST:%.*]], i64 [[INDEX]]
; CHECK-NEXT: store <2 x i16> [[PREDPHI]], ptr [[TMP8]], align 2
; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 2
diff --git a/llvm/test/Transforms/LoopVectorize/uniform-blend.ll b/llvm/test/Transforms/LoopVectorize/uniform-blend.ll
index 92f714dcf3a20..e8c70a23f0ebc 100644
--- a/llvm/test/Transforms/LoopVectorize/uniform-blend.ll
+++ b/llvm/test/Transforms/LoopVectorize/uniform-blend.ll
@@ -110,6 +110,10 @@ define void @blend_chain_iv(i1 %c) {
; CHECK-NEXT: [[TMP7:%.*]] = add i64 [[INDEX]], 3
; CHECK-NEXT: br i1 [[C]], label %[[LOOP_NEXT_21:.*]], label %[[LOOP_NEXT_32]]
; CHECK: [[LOOP_NEXT_21]]:
+; CHECK-NEXT: br i1 [[C]], label %[[LOOP_NEXT_22:.*]], label %[[LOOP_NEXT_33:.*]]
+; CHECK: [[LOOP_NEXT_22]]:
+; CHECK-NEXT: br label %[[LOOP_NEXT_33]]
+; CHECK: [[LOOP_NEXT_33]]:
; CHECK-NEXT: br label %[[LOOP_NEXT_32]]
; CHECK: [[LOOP_NEXT_32]]:
; CHECK-NEXT: [[TMP2:%.*]] = getelementptr inbounds [32 x i16], ptr @dst, i16 0, i64 [[INDEX]]
diff --git a/llvm/test/Transforms/LoopVectorize/unused-blend-mask-for-first-operand.ll b/llvm/test/Transforms/LoopVectorize/unused-blend-mask-for-first-operand.ll
index c4916058759c2..3d4e353c7b4fe 100644
--- a/llvm/test/Transforms/LoopVectorize/unused-blend-mask-for-first-operand.ll
+++ b/llvm/test/Transforms/LoopVectorize/unused-blend-mask-for-first-operand.ll
@@ -15,8 +15,12 @@ define void @test_not_first_lane_only_constant(ptr %A, ptr noalias %B, ptr noali
; CHECK-NEXT: [[INDEX:%.*]] = phi i32 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[ELSE_21:%.*]] ]
; CHECK-NEXT: [[OFFSET_IDX:%.*]] = trunc i32 [[INDEX]] to i16
; CHECK-NEXT: [[TMP1:%.*]] = getelementptr inbounds i16, ptr [[A]], i16 [[OFFSET_IDX]]
+; CHECK-NEXT: br label [[ELSE_11:%.*]]
+; CHECK: else.11:
+; CHECK-NEXT: br label [[ELSE_22:%.*]]
+; CHECK: else.22:
; CHECK-NEXT: br label [[ELSE_21]]
-; CHECK: else.21:
+; CHECK: loop.latch3:
; CHECK-NEXT: [[TMP13:%.*]] = load i16, ptr [[B]], align 2
; CHECK-NEXT: [[BROADCAST_SPLATINSERT5:%.*]] = insertelement <4 x i16> poison, i16 [[TMP13]], i64 0
; CHECK-NEXT: [[BROADCAST_SPLAT6:%.*]] = shufflevector <4 x i16> [[BROADCAST_SPLATINSERT5]], <4 x i16> poison, <4 x i32> zeroinitializer
>From fe0ee8e25ea1175d0df38deae299f519d45e9274 Mon Sep 17 00:00:00 2001
From: Andrei Elovikov <andrei.elovikov at sifive.com>
Date: Wed, 26 Aug 2026 14:45:53 -0700
Subject: [PATCH 04/11] Awful AI-generated hack to deal with
bcast/extractelement breaking SSA
---
llvm/lib/Transforms/Vectorize/VPlan.cpp | 34 +++++-
llvm/lib/Transforms/Vectorize/VPlan.h | 6 +
llvm/lib/Transforms/Vectorize/VPlanHelpers.h | 12 ++
llvm/lib/Transforms/Vectorize/VPlanUtils.cpp | 1 +
.../RISCV/gather-scatter-cost.ll | 24 ++--
.../VPlan/dissolve-replicate-regions.ll | 112 ++++++------------
.../constant-fold-commutative-and.ll | 2 +-
...predicated-loads-with-predicated-stores.ll | 30 +++--
8 files changed, 120 insertions(+), 101 deletions(-)
diff --git a/llvm/lib/Transforms/Vectorize/VPlan.cpp b/llvm/lib/Transforms/Vectorize/VPlan.cpp
index 87d24ed9d8d4b..4139a40a24115 100644
--- a/llvm/lib/Transforms/Vectorize/VPlan.cpp
+++ b/llvm/lib/Transforms/Vectorize/VPlan.cpp
@@ -306,6 +306,11 @@ Value *VPTransformState::get(const VPValue *Def, const VPLane &Lane) {
return VecPart;
}
// TODO: Cache created scalar values.
+ std::optional<IRBuilderBase::InsertPointGuard> Guard;
+ if (requiresLocalImplicitConversions()) {
+ Guard.emplace(Builder);
+ setInsertPointForImplicitConversion(Def, VecPart);
+ }
Value *LaneV = Lane.getAsRuntimeExpr(Builder, VF);
auto *Extract = Builder.CreateExtractElement(VecPart, LaneV);
// set(Def, Extract, Instance);
@@ -340,17 +345,43 @@ Value *VPTransformState::get(const VPValue *Def, bool NeedsScalar) {
Value *ScalarValue = get(Def, VPLane(0));
VPLane LastLane = VPLane::getLastLaneForVF(VF);
IRBuilderBase::InsertPointGuard Guard(Builder);
- if (auto *LastInst = dyn_cast<Instruction>(get(Def, LastLane)))
+ if (requiresLocalImplicitConversions()) {
+ setInsertPointForImplicitConversion(Def, get(Def, LastLane));
+ } else if (auto *LastInst = dyn_cast<Instruction>(get(Def, LastLane))) {
// Set the insert point after the last scalarized instruction. This
// ensures the insertelement sequence will directly follow the scalar
// definitions.
if (auto InsertPt = LastInst->getInsertionPointAfterDef())
Builder.SetInsertPoint(*InsertPt);
+ }
Value *VectorValue = GetBroadcastInstrs(ScalarValue);
set(Def, VectorValue);
return VectorValue;
}
+bool VPTransformState::requiresLocalImplicitConversions() const {
+ VPBasicBlock *Current = CFG.PrevVPBB;
+ return Current && any_of(Current->phis(), [](VPRecipeBase &R) {
+ auto *Phi = dyn_cast<VPPhi>(&R);
+ return Phi && Phi->isSSAReconstructionPhi();
+ });
+}
+
+void VPTransformState::setInsertPointForImplicitConversion(const VPValue *Def,
+ Value *V) {
+ auto *DefR = Def->getDefiningRecipe();
+ if (DefR && !isa<VPIRInstruction>(DefR)) {
+ if (auto *I = dyn_cast<Instruction>(V)) {
+ if (auto InsertPt = I->getInsertionPointAfterDef()) {
+ Builder.SetInsertPoint(*InsertPt);
+ return;
+ }
+ }
+ }
+
+ Builder.SetInsertPoint(CFG.VectorPreHeader->getTerminator());
+}
+
void VPTransformState::setDebugLocFrom(DebugLoc DL) {
const DILocation *DIL = DL;
// When a FSDiscriminator is enabled, we don't need to add the multiply
@@ -966,6 +997,7 @@ void VPlan::execute(VPTransformState *State) {
// Disconnect VectorPreHeader from ExitBB in both the CFG and DT.
BasicBlock *VectorPreHeader = State->CFG.PrevBB;
+ State->CFG.VectorPreHeader = VectorPreHeader;
cast<UncondBrInst>(VectorPreHeader->getTerminator())->setSuccessor(nullptr);
State->CFG.DTU.applyUpdates(
{{DominatorTree::Delete, VectorPreHeader, State->CFG.ExitBB}});
diff --git a/llvm/lib/Transforms/Vectorize/VPlan.h b/llvm/lib/Transforms/Vectorize/VPlan.h
index fb5281f656d70..f6f7267d611ba 100644
--- a/llvm/lib/Transforms/Vectorize/VPlan.h
+++ b/llvm/lib/Transforms/Vectorize/VPlan.h
@@ -1688,6 +1688,8 @@ class VPPhiAccessors {
};
struct LLVM_ABI_FOR_TEST VPPhi : public VPInstruction, public VPPhiAccessors {
+ bool IsSSAReconstructionPhi = false;
+
VPPhi(ArrayRef<VPValue *> Operands, const VPIRFlags &Flags, DebugLoc DL,
const Twine &Name = "", Type *ResultTy = nullptr)
: VPInstruction(Instruction::PHI, Operands, Flags, {}, DL, Name,
@@ -1711,9 +1713,13 @@ struct LLVM_ABI_FOR_TEST VPPhi : public VPInstruction, public VPPhiAccessors {
VPPhi *clone() override {
auto *PhiR = new VPPhi(operands(), *this, getDebugLoc(), getName());
PhiR->setUnderlyingValue(getUnderlyingValue());
+ PhiR->IsSSAReconstructionPhi = IsSSAReconstructionPhi;
return PhiR;
}
+ void markAsSSAReconstructionPhi() { IsSSAReconstructionPhi = true; }
+ bool isSSAReconstructionPhi() const { return IsSSAReconstructionPhi; }
+
void execute(VPTransformState &State) override;
protected:
diff --git a/llvm/lib/Transforms/Vectorize/VPlanHelpers.h b/llvm/lib/Transforms/Vectorize/VPlanHelpers.h
index 1c987abc649c8..e03fb5afa4d5d 100644
--- a/llvm/lib/Transforms/Vectorize/VPlanHelpers.h
+++ b/llvm/lib/Transforms/Vectorize/VPlanHelpers.h
@@ -275,6 +275,15 @@ struct VPTransformState {
Value *packScalarIntoVectorizedValue(const VPValue *Def, Value *WideValue,
const VPLane &Lane);
+ /// Return whether the current VPBB needs local placement of implicit
+ /// scalar/vector conversions.
+ bool requiresLocalImplicitConversions() const;
+
+ /// Set the insertion point for an implicit scalar/vector conversion of
+ /// \p Def. Insert after \p V when it is a generated definition, otherwise
+ /// insert in the vector preheader.
+ void setInsertPointForImplicitConversion(const VPValue *Def, Value *V);
+
/// Add the backedge (latch) incoming value to the canonical, reduction and
/// first-order recurrence phis in all loop headers state's plan, after
/// the loop body has been generated.
@@ -294,6 +303,9 @@ struct VPTransformState {
/// vector loop.
BasicBlock *ExitBB = nullptr;
+ /// The IR preheader of the vector loop.
+ BasicBlock *VectorPreHeader = nullptr;
+
/// A mapping of each VPBasicBlock to the corresponding BasicBlock. In case
/// of replication, maps the BasicBlock of the last replica created.
SmallDenseMap<const VPBasicBlock *, BasicBlock *> VPBB2IRBB;
diff --git a/llvm/lib/Transforms/Vectorize/VPlanUtils.cpp b/llvm/lib/Transforms/Vectorize/VPlanUtils.cpp
index b99940b0a15fe..fc234f9eac387 100644
--- a/llvm/lib/Transforms/Vectorize/VPlanUtils.cpp
+++ b/llvm/lib/Transforms/Vectorize/VPlanUtils.cpp
@@ -1237,6 +1237,7 @@ VPValue *vputils::reconstructSSA(VPBasicBlock *VPBB,
Type *Ty = Defs.begin()->second->getScalarType();
auto *Phi = new VPPhi({}, VPIRFlags::getDefaultFlags(Instruction::PHI, Ty),
DebugLoc::getUnknown(), "", Ty);
+ Phi->markAsSSAReconstructionPhi();
VPBB->insert(Phi, VPBB->getFirstNonPhi());
Defs[VPBB] = Phi;
for (auto *Pred : VPBB->predecessors())
diff --git a/llvm/test/Transforms/LoopVectorize/RISCV/gather-scatter-cost.ll b/llvm/test/Transforms/LoopVectorize/RISCV/gather-scatter-cost.ll
index c1ed5b1327073..fc9fef1d1d11f 100644
--- a/llvm/test/Transforms/LoopVectorize/RISCV/gather-scatter-cost.ll
+++ b/llvm/test/Transforms/LoopVectorize/RISCV/gather-scatter-cost.ll
@@ -28,20 +28,27 @@ define void @predicated_uniform_load(ptr %src, i32 %n, ptr %dst, i1 %cond) {
; CHECK-NEXT: [[FOUND_CONFLICT:%.*]] = and i1 [[BOUND0]], [[BOUND1]]
; CHECK-NEXT: br i1 [[FOUND_CONFLICT]], label [[SCALAR_PH]], label [[VECTOR_PH:%.*]]
; CHECK: vector.ph:
-; CHECK-NEXT: [[BROADCAST_SPLATINSERT1:%.*]] = insertelement <vscale x 4 x i1> poison, i1 [[COND:%.*]], i64 0
-; CHECK-NEXT: [[BROADCAST_SPLAT1:%.*]] = shufflevector <vscale x 4 x i1> [[BROADCAST_SPLATINSERT1]], <vscale x 4 x i1> poison, <vscale x 4 x i32> zeroinitializer
-; CHECK-NEXT: [[TMP13:%.*]] = xor <vscale x 4 x i1> [[BROADCAST_SPLAT1]], splat (i1 true)
; CHECK-NEXT: [[BROADCAST_SPLATINSERT:%.*]] = insertelement <vscale x 4 x ptr> poison, ptr [[BOXES]], i64 0
; CHECK-NEXT: [[BROADCAST_SPLAT:%.*]] = shufflevector <vscale x 4 x ptr> [[BROADCAST_SPLATINSERT]], <vscale x 4 x ptr> poison, <vscale x 4 x i32> zeroinitializer
; CHECK-NEXT: [[BROADCAST_SPLATINSERT3:%.*]] = insertelement <vscale x 4 x ptr> poison, ptr [[NBRBOXES]], i64 0
; CHECK-NEXT: [[BROADCAST_SPLAT4:%.*]] = shufflevector <vscale x 4 x ptr> [[BROADCAST_SPLATINSERT3]], <vscale x 4 x ptr> poison, <vscale x 4 x i32> zeroinitializer
; CHECK-NEXT: br label [[VECTOR_BODY:%.*]]
; CHECK: vector.body:
-; CHECK-NEXT: [[AVL:%.*]] = phi i32 [ [[TMP3]], [[VECTOR_PH]] ], [ [[AVL_NEXT:%.*]], [[VECTOR_BODY]] ]
+; CHECK-NEXT: [[AVL:%.*]] = phi i32 [ [[TMP3]], [[VECTOR_PH]] ], [ [[AVL_NEXT:%.*]], [[LOOP_LATCH6:%.*]] ]
; CHECK-NEXT: [[TMP10:%.*]] = call i32 @llvm.experimental.get.vector.length.i32(i32 [[AVL]], i32 4, i1 true)
-; CHECK-NEXT: [[WIDE_MASKED_GATHER:%.*]] = call <vscale x 4 x i32> @llvm.vp.gather.nxv4i32.nxv4p0(<vscale x 4 x ptr> align 4 [[BROADCAST_SPLAT]], <vscale x 4 x i1> [[TMP13]], i32 [[TMP10]]), !alias.scope [[META0:![0-9]+]]
-; CHECK-NEXT: [[PREDPHI:%.*]] = select i1 [[COND]], <vscale x 4 x i32> zeroinitializer, <vscale x 4 x i32> [[WIDE_MASKED_GATHER]]
-; CHECK-NEXT: call void @llvm.vp.scatter.nxv4i32.nxv4p0(<vscale x 4 x i32> [[PREDPHI]], <vscale x 4 x ptr> align 4 [[BROADCAST_SPLAT4]], <vscale x 4 x i1> splat (i1 true), i32 [[TMP10]]), !alias.scope [[META3:![0-9]+]], !noalias [[META0]]
+; CHECK-NEXT: br i1 [[COND:%.*]], label [[LOOP_THEN5:%.*]], label [[LOOP_ELSE4:%.*]]
+; CHECK: loop.else4:
+; CHECK-NEXT: [[WIDE_MASKED_GATHER:%.*]] = call <vscale x 4 x i32> @llvm.vp.gather.nxv4i32.nxv4p0(<vscale x 4 x ptr> align 4 [[BROADCAST_SPLAT]], <vscale x 4 x i1> splat (i1 true), i32 [[TMP10]]), !alias.scope [[META0:![0-9]+]]
+; CHECK-NEXT: [[TMP12:%.*]] = extractelement <vscale x 4 x i32> [[WIDE_MASKED_GATHER]], i64 0
+; CHECK-NEXT: br label [[LOOP_LATCH6]]
+; CHECK: loop.then5:
+; CHECK-NEXT: br label [[LOOP_LATCH6]]
+; CHECK: loop.latch6:
+; CHECK-NEXT: [[TMP13:%.*]] = phi i32 [ [[TMP12]], [[LOOP_ELSE4]] ], [ poison, [[LOOP_THEN5]] ]
+; CHECK-NEXT: [[BROADCAST_SPLATINSERT7:%.*]] = insertelement <vscale x 4 x i32> poison, i32 [[TMP13]], i64 0
+; CHECK-NEXT: [[BROADCAST_SPLAT8:%.*]] = shufflevector <vscale x 4 x i32> [[BROADCAST_SPLATINSERT7]], <vscale x 4 x i32> poison, <vscale x 4 x i32> zeroinitializer
+; CHECK-NEXT: [[TMP14:%.*]] = call <vscale x 4 x i32> @llvm.vp.merge.nxv4i32(<vscale x 4 x i1> splat (i1 true), <vscale x 4 x i32> zeroinitializer, <vscale x 4 x i32> [[BROADCAST_SPLAT8]], i32 [[TMP10]])
+; CHECK-NEXT: call void @llvm.vp.scatter.nxv4i32.nxv4p0(<vscale x 4 x i32> [[TMP14]], <vscale x 4 x ptr> align 4 [[BROADCAST_SPLAT4]], <vscale x 4 x i1> splat (i1 true), i32 [[TMP10]]), !alias.scope [[META3:![0-9]+]], !noalias [[META0]]
; CHECK-NEXT: [[AVL_NEXT]] = sub nuw i32 [[AVL]], [[TMP10]]
; CHECK-NEXT: [[TMP16:%.*]] = icmp eq i32 [[AVL_NEXT]], 0
; CHECK-NEXT: br i1 [[TMP16]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP5:![0-9]+]]
@@ -279,6 +286,3 @@ exit:
}
attributes #0 = { "target-features"="+zve64x,+zvl256b" }
-;; NOTE: These prefixes are unused and the list is autogenerated. Do not add tests below this line:
-; RVA23: {{.*}}
-; RVA23ZVL1024B: {{.*}}
diff --git a/llvm/test/Transforms/LoopVectorize/VPlan/dissolve-replicate-regions.ll b/llvm/test/Transforms/LoopVectorize/VPlan/dissolve-replicate-regions.ll
index de1f217a4a25b..f7a53e57a127a 100644
--- a/llvm/test/Transforms/LoopVectorize/VPlan/dissolve-replicate-regions.ll
+++ b/llvm/test/Transforms/LoopVectorize/VPlan/dissolve-replicate-regions.ll
@@ -31,28 +31,21 @@ define void @predicated_load(i1 %c, ptr %ptr, ptr %dst) {
; SCALAR-NEXT: vector.body:
; SCALAR-NEXT: vp<[[VP6:%[0-9]+]]> = SCALAR-STEPS vp<[[VP5]]>, ir<1>, vp<[[VP0]]>
; SCALAR-NEXT: vp<[[VP7:%[0-9]+]]> = SCALAR-STEPS vp<[[VP5]]>, ir<1>, vp<[[VP0]]>, vp<[[VP0]]>
-; SCALAR-NEXT: EMIT branch-on-cond ir<%c>
-; SCALAR-NEXT: Successor(s): pred.load.if, pred.load.continue
-; SCALAR-EMPTY:
-; SCALAR-NEXT: pred.load.if:
; SCALAR-NEXT: CLONE ir<%gep> = getelementptr ir<%ptr>, vp<[[VP6]]>
-; SCALAR-NEXT: CLONE ir<%lv> = load ir<%gep>
-; SCALAR-NEXT: Successor(s): pred.load.continue
-; SCALAR-EMPTY:
-; SCALAR-NEXT: pred.load.continue:
-; SCALAR-NEXT: EMIT-SCALAR vp<[[VP9:%[0-9]+]]> = phi [ ir<poison>, vector.body ], [ ir<%lv>, pred.load.if ]
+; SCALAR-NEXT: CLONE ir<%gep>.1 = getelementptr ir<%ptr>, vp<[[VP7]]>
; SCALAR-NEXT: EMIT branch-on-cond ir<%c>
-; SCALAR-NEXT: Successor(s): pred.load.if, pred.load.continue
+; SCALAR-NEXT: Successor(s): if.then, loop.latch
; SCALAR-EMPTY:
-; SCALAR-NEXT: pred.load.if:
-; SCALAR-NEXT: CLONE ir<%gep>.1 = getelementptr ir<%ptr>, vp<[[VP7]]>
+; SCALAR-NEXT: if.then:
+; SCALAR-NEXT: CLONE ir<%lv> = load ir<%gep>
; SCALAR-NEXT: CLONE ir<%lv>.1 = load ir<%gep>.1
-; SCALAR-NEXT: Successor(s): pred.load.continue
+; SCALAR-NEXT: Successor(s): loop.latch
; SCALAR-EMPTY:
-; SCALAR-NEXT: pred.load.continue:
-; SCALAR-NEXT: EMIT-SCALAR vp<[[VP11:%[0-9]+]]> = phi [ ir<poison>, pred.load.continue ], [ ir<%lv>.1, pred.load.if ]
-; SCALAR-NEXT: BLEND ir<%pred.val> = ir<0> vp<%9>/ir<%c>
-; SCALAR-NEXT: BLEND ir<%pred.val>.1 = ir<0> vp<%11>/ir<%c>
+; SCALAR-NEXT: loop.latch:
+; SCALAR-NEXT: EMIT-SCALAR vp<[[VP8:%[0-9]+]]> = phi [ ir<poison>, vector.body ], [ ir<%lv>, if.then ]
+; SCALAR-NEXT: EMIT-SCALAR vp<[[VP9:%[0-9]+]]> = phi [ ir<poison>, vector.body ], [ ir<%lv>.1, if.then ]
+; SCALAR-NEXT: BLEND ir<%pred.val> = ir<0> vp<%8>/ir<true>
+; SCALAR-NEXT: BLEND ir<%pred.val>.1 = ir<0> vp<%9>/ir<true>
; SCALAR-NEXT: CLONE ir<%gep.dst> = getelementptr ir<%dst>, vp<[[VP6]]>
; SCALAR-NEXT: CLONE ir<%gep.dst>.1 = getelementptr ir<%dst>, vp<[[VP7]]>
; SCALAR-NEXT: CLONE store ir<%pred.val>, ir<%gep.dst>
@@ -100,72 +93,37 @@ define void @predicated_load(i1 %c, ptr %ptr, ptr %dst) {
; VECTOR-NEXT: Successor(s): scalar.ph, vector.ph
; VECTOR-EMPTY:
; VECTOR-NEXT: vector.ph:
-; VECTOR-NEXT: EMIT vp<[[VP5:%[0-9]+]]> = broadcast ir<%c>
; VECTOR-NEXT: Successor(s): vector loop
; VECTOR-EMPTY:
; VECTOR-NEXT: <x1> vector loop: {
-; VECTOR-NEXT: vp<[[VP6:%[0-9]+]]> = CANONICAL-IV
+; VECTOR-NEXT: vp<[[VP5:%[0-9]+]]> = CANONICAL-IV
; VECTOR-EMPTY:
; VECTOR-NEXT: vector.body:
-; VECTOR-NEXT: vp<[[VP7:%[0-9]+]]> = SCALAR-STEPS vp<[[VP6]]>, ir<1>, vp<[[VP0]]>
-; VECTOR-NEXT: EMIT branch-on-cond ir<%c>
-; VECTOR-NEXT: Successor(s): pred.load.if, pred.load.continue
-; VECTOR-EMPTY:
-; VECTOR-NEXT: pred.load.if:
-; VECTOR-NEXT: vp<[[VP9:%[0-9]+]]> = SCALAR-STEPS vp<[[VP6]]>, ir<1>, vp<[[VP0]]>
-; VECTOR-NEXT: CLONE ir<%gep> = getelementptr ir<%ptr>, vp<[[VP9]]>
-; VECTOR-NEXT: CLONE ir<%lv> = load ir<%gep>
-; VECTOR-NEXT: EMIT vp<[[VP10:%[0-9]+]]> = insertelement ir<poison>, ir<%lv>, ir<0>
-; VECTOR-NEXT: Successor(s): pred.load.continue
-; VECTOR-EMPTY:
-; VECTOR-NEXT: pred.load.continue:
-; VECTOR-NEXT: WIDEN-PHI vp<[[VP11:%[0-9]+]]> = phi [ ir<poison>, vector.body ], [ vp<[[VP10]]>, pred.load.if ]
+; VECTOR-NEXT: vp<[[VP6:%[0-9]+]]> = SCALAR-STEPS vp<[[VP5]]>, ir<1>, vp<[[VP0]]>
+; VECTOR-NEXT: CLONE ir<%gep> = getelementptr ir<%ptr>, vp<[[VP6]]>
; VECTOR-NEXT: EMIT branch-on-cond ir<%c>
-; VECTOR-NEXT: Successor(s): pred.load.if, pred.load.continue
-; VECTOR-EMPTY:
-; VECTOR-NEXT: pred.load.if:
-; VECTOR-NEXT: vp<[[VP13:%[0-9]+]]> = SCALAR-STEPS vp<[[VP6]]>, ir<1>, vp<[[VP0]]>, ir<1>
-; VECTOR-NEXT: CLONE ir<%gep>.1 = getelementptr ir<%ptr>, vp<[[VP13]]>
-; VECTOR-NEXT: CLONE ir<%lv>.1 = load ir<%gep>.1
-; VECTOR-NEXT: EMIT vp<[[VP14:%[0-9]+]]> = insertelement vp<[[VP11]]>, ir<%lv>.1, ir<1>
-; VECTOR-NEXT: Successor(s): pred.load.continue
-; VECTOR-EMPTY:
-; VECTOR-NEXT: pred.load.continue:
-; VECTOR-NEXT: WIDEN-PHI vp<[[VP15:%[0-9]+]]> = phi [ vp<[[VP11]]>, pred.load.continue ], [ vp<[[VP14]]>, pred.load.if ]
-; VECTOR-NEXT: EMIT branch-on-cond ir<%c>
-; VECTOR-NEXT: Successor(s): pred.load.if, pred.load.continue
-; VECTOR-EMPTY:
-; VECTOR-NEXT: pred.load.if:
-; VECTOR-NEXT: vp<[[VP17:%[0-9]+]]> = SCALAR-STEPS vp<[[VP6]]>, ir<1>, vp<[[VP0]]>, vp<[[VP0]]>
-; VECTOR-NEXT: CLONE ir<%gep>.2 = getelementptr ir<%ptr>, vp<[[VP17]]>
-; VECTOR-NEXT: CLONE ir<%lv>.2 = load ir<%gep>.2
-; VECTOR-NEXT: EMIT vp<[[VP18:%[0-9]+]]> = insertelement ir<poison>, ir<%lv>.2, ir<0>
-; VECTOR-NEXT: Successor(s): pred.load.continue
-; VECTOR-EMPTY:
-; VECTOR-NEXT: pred.load.continue:
-; VECTOR-NEXT: WIDEN-PHI vp<[[VP19:%[0-9]+]]> = phi [ ir<poison>, pred.load.continue ], [ vp<[[VP18]]>, pred.load.if ]
-; VECTOR-NEXT: EMIT branch-on-cond ir<%c>
-; VECTOR-NEXT: Successor(s): pred.load.if, pred.load.continue
-; VECTOR-EMPTY:
-; VECTOR-NEXT: pred.load.if:
-; VECTOR-NEXT: EMIT vp<[[VP21:%[0-9]+]]> = add vp<[[VP0]]>, ir<1>
-; VECTOR-NEXT: vp<[[VP22:%[0-9]+]]> = SCALAR-STEPS vp<[[VP6]]>, ir<1>, vp<[[VP0]]>, vp<[[VP21]]>
-; VECTOR-NEXT: CLONE ir<%gep>.3 = getelementptr ir<%ptr>, vp<[[VP22]]>
-; VECTOR-NEXT: CLONE ir<%lv>.3 = load ir<%gep>.3
-; VECTOR-NEXT: EMIT vp<[[VP23:%[0-9]+]]> = insertelement vp<[[VP19]]>, ir<%lv>.3, ir<1>
-; VECTOR-NEXT: Successor(s): pred.load.continue
-; VECTOR-EMPTY:
-; VECTOR-NEXT: pred.load.continue:
-; VECTOR-NEXT: WIDEN-PHI vp<[[VP24:%[0-9]+]]> = phi [ vp<[[VP19]]>, pred.load.continue ], [ vp<[[VP23]]>, pred.load.if ]
-; VECTOR-NEXT: BLEND ir<%pred.val> = ir<0> vp<%15>/vp<[[VP5]]>
-; VECTOR-NEXT: BLEND ir<%pred.val>.1 = ir<0> vp<%24>/vp<[[VP5]]>
-; VECTOR-NEXT: CLONE ir<%gep.dst> = getelementptr ir<%dst>, vp<[[VP7]]>
-; VECTOR-NEXT: EMIT vp<[[VP25:%[0-9]+]]> = mul nuw nsw vp<[[VP0]]>, ir<1>
-; VECTOR-NEXT: vp<[[VP26:%[0-9]+]]> = vector-pointer i8, ir<%gep.dst>, ir<1>
-; VECTOR-NEXT: vp<[[VP27:%[0-9]+]]> = vector-pointer i8, ir<%gep.dst>, ir<1>, vp<[[VP25]]>
-; VECTOR-NEXT: WIDEN store vp<[[VP26]]>, ir<%pred.val>
-; VECTOR-NEXT: WIDEN store vp<[[VP27]]>, ir<%pred.val>.1
-; VECTOR-NEXT: EMIT vp<%index.next> = add nuw vp<[[VP6]]>, vp<[[VP1]]>
+; VECTOR-NEXT: Successor(s): if.then, loop.latch
+; VECTOR-EMPTY:
+; VECTOR-NEXT: if.then:
+; VECTOR-NEXT: EMIT vp<[[VP7:%[0-9]+]]> = mul nuw nsw vp<[[VP0]]>, ir<1>
+; VECTOR-NEXT: vp<[[VP8:%[0-9]+]]> = vector-pointer i8, ir<%gep>, ir<1>
+; VECTOR-NEXT: vp<[[VP9:%[0-9]+]]> = vector-pointer i8, ir<%gep>, ir<1>, vp<[[VP7]]>
+; VECTOR-NEXT: WIDEN ir<%lv> = load vp<[[VP8]]>
+; VECTOR-NEXT: WIDEN ir<%lv>.1 = load vp<[[VP9]]>
+; VECTOR-NEXT: Successor(s): loop.latch
+; VECTOR-EMPTY:
+; VECTOR-NEXT: loop.latch:
+; VECTOR-NEXT: EMIT-SCALAR vp<[[VP10:%[0-9]+]]> = phi [ ir<poison>, vector.body ], [ ir<%lv>, if.then ]
+; VECTOR-NEXT: EMIT-SCALAR vp<[[VP11:%[0-9]+]]> = phi [ ir<poison>, vector.body ], [ ir<%lv>.1, if.then ]
+; VECTOR-NEXT: BLEND ir<%pred.val> = ir<0> vp<%10>/ir<true>
+; VECTOR-NEXT: BLEND ir<%pred.val>.1 = ir<0> vp<%11>/ir<true>
+; VECTOR-NEXT: CLONE ir<%gep.dst> = getelementptr ir<%dst>, vp<[[VP6]]>
+; VECTOR-NEXT: EMIT vp<[[VP12:%[0-9]+]]> = mul nuw nsw vp<[[VP0]]>, ir<1>
+; VECTOR-NEXT: vp<[[VP13:%[0-9]+]]> = vector-pointer i8, ir<%gep.dst>, ir<1>
+; VECTOR-NEXT: vp<[[VP14:%[0-9]+]]> = vector-pointer i8, ir<%gep.dst>, ir<1>, vp<[[VP12]]>
+; VECTOR-NEXT: WIDEN store vp<[[VP13]]>, ir<%pred.val>
+; VECTOR-NEXT: WIDEN store vp<[[VP14]]>, ir<%pred.val>.1
+; VECTOR-NEXT: EMIT vp<%index.next> = add nuw vp<[[VP5]]>, vp<[[VP1]]>
; VECTOR-NEXT: EMIT branch-on-count vp<%index.next>, vp<[[VP2]]>
; VECTOR-NEXT: No successors
; VECTOR-NEXT: }
diff --git a/llvm/test/Transforms/LoopVectorize/constant-fold-commutative-and.ll b/llvm/test/Transforms/LoopVectorize/constant-fold-commutative-and.ll
index cd7592765603c..ab2fcc8682a35 100644
--- a/llvm/test/Transforms/LoopVectorize/constant-fold-commutative-and.ll
+++ b/llvm/test/Transforms/LoopVectorize/constant-fold-commutative-and.ll
@@ -16,6 +16,7 @@ define void @constant_fold_commutative_and(ptr %ptr.n, ptr noalias %p, i1 %cond)
; CHECK-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[PRED_STORE_CONTINUE5:.*]] ]
; CHECK-NEXT: [[VEC_IND:%.*]] = phi <2 x i8> [ <i8 0, i8 1>, %[[VECTOR_PH]] ], [ [[VEC_IND_NEXT:%.*]], %[[PRED_STORE_CONTINUE5]] ]
; CHECK-NEXT: [[TMP1:%.*]] = icmp ule <2 x i8> [[VEC_IND]], splat (i8 16)
+; CHECK-NEXT: [[TMP6:%.*]] = extractelement <2 x i1> [[TMP1]], i64 0
; CHECK-NEXT: [[TMP2:%.*]] = load i64, ptr [[PTR_N]], align 4
; CHECK-NEXT: br i1 [[COND]], label %[[LATCH2:.*]], label %[[PRED_11:.*]]
; CHECK: [[PRED_11]]:
@@ -26,7 +27,6 @@ define void @constant_fold_commutative_and(ptr %ptr.n, ptr noalias %p, i1 %cond)
; CHECK-NEXT: [[TMP4:%.*]] = phi i1 [ poison, %[[VECTOR_BODY]] ], [ [[TMP3]], %[[PRED_11]] ]
; CHECK-NEXT: [[TMP5:%.*]] = phi i1 [ false, %[[VECTOR_BODY]] ], [ [[TMP7]], %[[PRED_11]] ]
; CHECK-NEXT: [[TMP9:%.*]] = select i1 [[TMP5]], i1 [[TMP4]], i1 true
-; CHECK-NEXT: [[TMP6:%.*]] = extractelement <2 x i1> [[TMP1]], i64 0
; CHECK-NEXT: br i1 [[TMP6]], label %[[PRED_STORE_IF:.*]], label %[[PRED_STORE_CONTINUE:.*]]
; CHECK: [[PRED_STORE_IF]]:
; CHECK-NEXT: [[TMP8:%.*]] = getelementptr i8, ptr [[P]], i64 [[INDEX]]
diff --git a/llvm/test/Transforms/LoopVectorize/hoist-predicated-loads-with-predicated-stores.ll b/llvm/test/Transforms/LoopVectorize/hoist-predicated-loads-with-predicated-stores.ll
index b3a7b495f1205..3b7a06e517a0e 100644
--- a/llvm/test/Transforms/LoopVectorize/hoist-predicated-loads-with-predicated-stores.ll
+++ b/llvm/test/Transforms/LoopVectorize/hoist-predicated-loads-with-predicated-stores.ll
@@ -1616,28 +1616,34 @@ define void @sink_stores_cse_select_dropping_fmf(ptr %dst, ptr %src, ptr %invar.
; CHECK: [[VECTOR_PH]]:
; CHECK-NEXT: [[N_MOD_VF:%.*]] = and i64 [[N]], 1
; CHECK-NEXT: [[N_VEC:%.*]] = sub i64 [[N]], [[N_MOD_VF]]
+; CHECK-NEXT: br label %[[VECTOR_BODY1:.*]]
+; CHECK: [[VECTOR_BODY1]]:
+; CHECK-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY:.*]] ]
; CHECK-NEXT: [[TMP1:%.*]] = load float, ptr [[SRC]], align 4, !alias.scope [[META119:![0-9]+]]
; CHECK-NEXT: [[BROADCAST_SPLATINSERT:%.*]] = insertelement <2 x float> poison, float [[TMP1]], i64 0
; CHECK-NEXT: [[BROADCAST_SPLAT:%.*]] = shufflevector <2 x float> [[BROADCAST_SPLATINSERT]], <2 x float> poison, <2 x i32> zeroinitializer
+; CHECK-NEXT: [[TMP6:%.*]] = getelementptr inbounds float, ptr [[DST]], i64 [[INDEX]]
+; CHECK-NEXT: br i1 [[C]], label %[[THEN11:.*]], label %[[ELSE10:.*]]
+; CHECK: [[ELSE10]]:
; CHECK-NEXT: [[TMP2:%.*]] = fmul <2 x float> [[BROADCAST_SPLAT]], splat (float 3.000000e+00)
+; CHECK-NEXT: [[TMP5:%.*]] = extractelement <2 x float> [[TMP2]], i64 0
+; CHECK-NEXT: store <2 x float> [[TMP2]], ptr [[TMP6]], align 4, !alias.scope [[META122:![0-9]+]], !noalias [[META124:![0-9]+]]
+; CHECK-NEXT: br label %[[VECTOR_BODY]]
+; CHECK: [[THEN11]]:
; CHECK-NEXT: [[TMP3:%.*]] = fmul nnan <2 x float> [[BROADCAST_SPLAT]], splat (float 2.000000e+00)
; CHECK-NEXT: [[TMP4:%.*]] = extractelement <2 x float> [[TMP3]], i64 0
-; CHECK-NEXT: [[TMP5:%.*]] = extractelement <2 x float> [[TMP2]], i64 0
-; CHECK-NEXT: [[TMP6:%.*]] = select i1 [[C]], float [[TMP4]], float [[TMP5]]
-; CHECK-NEXT: [[TMP7:%.*]] = fmul nnan float [[TMP6]], 5.000000e+00
-; CHECK-NEXT: br label %[[VECTOR_BODY:.*]]
+; CHECK-NEXT: store <2 x float> [[TMP3]], ptr [[TMP6]], align 4, !alias.scope [[META122]], !noalias [[META124]]
+; CHECK-NEXT: br label %[[VECTOR_BODY]]
; CHECK: [[VECTOR_BODY]]:
-; CHECK-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
-; CHECK-NEXT: [[TMP8:%.*]] = add i64 [[INDEX]], 1
-; CHECK-NEXT: [[TMP9:%.*]] = getelementptr inbounds float, ptr [[DST]], i64 [[INDEX]]
-; CHECK-NEXT: [[TMP10:%.*]] = getelementptr inbounds float, ptr [[DST]], i64 [[TMP8]]
-; CHECK-NEXT: store float [[TMP6]], ptr [[TMP9]], align 4, !alias.scope [[META122:![0-9]+]], !noalias [[META124:![0-9]+]]
-; CHECK-NEXT: store float [[TMP6]], ptr [[TMP10]], align 4, !alias.scope [[META122]], !noalias [[META124]]
+; CHECK-NEXT: [[TMP8:%.*]] = phi float [ [[TMP5]], %[[ELSE10]] ], [ poison, %[[THEN11]] ]
+; CHECK-NEXT: [[TMP9:%.*]] = phi float [ poison, %[[ELSE10]] ], [ [[TMP4]], %[[THEN11]] ]
+; CHECK-NEXT: [[PREDPHI:%.*]] = select i1 true, float [[TMP9]], float [[TMP8]]
+; CHECK-NEXT: [[TMP10:%.*]] = fmul nnan float [[PREDPHI]], 5.000000e+00
+; CHECK-NEXT: store float [[TMP10]], ptr [[INVAR_DST]], align 4, !alias.scope [[META126:![0-9]+]], !noalias [[META119]]
; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 2
; CHECK-NEXT: [[TMP11:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
-; CHECK-NEXT: br i1 [[TMP11]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP126:![0-9]+]]
+; CHECK-NEXT: br i1 [[TMP11]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY1]], !llvm.loop [[LOOP127:![0-9]+]]
; CHECK: [[MIDDLE_BLOCK]]:
-; CHECK-NEXT: store float [[TMP7]], ptr [[INVAR_DST]], align 4, !alias.scope [[META127:![0-9]+]], !noalias [[META119]]
; CHECK-NEXT: [[CMP_N:%.*]] = icmp eq i64 [[N]], [[N_VEC]]
; CHECK-NEXT: br i1 [[CMP_N]], [[EXIT:label %.*]], label %[[SCALAR_PH]]
; CHECK: [[SCALAR_PH]]:
>From 3e09aced66c113cc616f1d949c450ca058562962 Mon Sep 17 00:00:00 2001
From: Andrei Elovikov <andrei.elovikov at sifive.com>
Date: Wed, 26 Aug 2026 16:40:12 -0700
Subject: [PATCH 05/11] WIP
---
.../Transforms/Vectorize/VPlanPredicator.cpp | 53 +-
.../LoopVectorize/AArch64/blend-costs.ll | 3 +-
.../Transforms/LoopVectorize/RISCV/divrem.ll | 624 ++++++++++--------
.../RISCV/tail-folding-complex-mask.ll | 3 +-
.../VPlan/dissolve-replicate-regions.ll | 22 +-
.../VPlan/phi-with-fastflags-vplan.ll | 7 +-
.../LoopVectorize/VPlan/predicator.ll | 28 +-
.../X86/pr141968-instsimplifyfolder.ll | 22 +-
.../X86/predicated-instruction-cost.ll | 3 +-
.../X86/predicated-replicate-feeding-cast.ll | 18 +-
...licate-recipe-with-only-first-lane-used.ll | 3 +-
.../constant-fold-commutative-and.ll | 18 +-
.../LoopVectorize/constantfolder.ll | 47 +-
...predicated-loads-with-predicated-stores.ll | 3 +-
.../LoopVectorize/hoist-predicated-loads.ll | 6 +-
.../LoopVectorize/phi-with-fastflags.ll | 3 +-
llvm/test/Transforms/LoopVectorize/pr45525.ll | 3 +-
.../Transforms/LoopVectorize/predicator.ll | 27 +-
.../LoopVectorize/scalarize-masked-call.ll | 122 ++--
.../Transforms/LoopVectorize/uniform-blend.ll | 18 +-
20 files changed, 579 insertions(+), 454 deletions(-)
diff --git a/llvm/lib/Transforms/Vectorize/VPlanPredicator.cpp b/llvm/lib/Transforms/Vectorize/VPlanPredicator.cpp
index 6667da573edb0..ac4cdf607eff3 100644
--- a/llvm/lib/Transforms/Vectorize/VPlanPredicator.cpp
+++ b/llvm/lib/Transforms/Vectorize/VPlanPredicator.cpp
@@ -124,6 +124,7 @@ class VPPredicator {
/// Create a logical-or, factoring out a common header mask if present.
VPValue *createMaskOr(VPValue *LHS, VPValue *RHS, DebugLoc DL);
+ VPValue *reconstructSSA(VPBasicBlock *UseBB, VPValue *V, bool IsMask);
void fixSSA(VPRecipeBase *U, VPValue *V, bool IsMask);
bool shouldPreserveTerminator(VPBasicBlock *VPBB);
@@ -235,10 +236,11 @@ VPValue *VPPredicator::createMaskOr(VPValue *LHS, VPValue *RHS, DebugLoc DL) {
HeaderMask, Builder.createOr(LHSRemainder, RHSRemainder, DL), DL);
}
-void VPPredicator::fixSSA(VPRecipeBase *U, VPValue *V, bool IsMask) {
+VPValue *VPPredicator::reconstructSSA(VPBasicBlock *UseBB, VPValue *V,
+ bool IsMask) {
auto *RecipeValue = dyn_cast<VPRecipeValue>(V);
if (!RecipeValue)
- return;
+ return V;
auto &SSADefs = SSAReconstructionDefsMap[RecipeValue];
VPBasicBlock *DefBB = RecipeValue->getDefiningRecipe()->getParent();
@@ -247,8 +249,12 @@ void VPPredicator::fixSSA(VPRecipeBase *U, VPValue *V, bool IsMask) {
if (DefBB != Header)
SSADefs[Header] =
IsMask ? Plan.getFalse() : Plan.getPoison(RecipeValue->getScalarType());
- auto *Fixed = vputils::reconstructSSA(U->getParent(), SSADefs);
- if (Fixed == RecipeValue)
+ return vputils::reconstructSSA(UseBB, SSADefs);
+}
+
+void VPPredicator::fixSSA(VPRecipeBase *U, VPValue *V, bool IsMask) {
+ VPValue *Fixed = reconstructSSA(U->getParent(), V, IsMask);
+ if (Fixed == V)
return;
LLVM_DEBUG({
@@ -259,7 +265,7 @@ void VPPredicator::fixSSA(VPRecipeBase *U, VPValue *V, bool IsMask) {
dbgs() << " with ";
Fixed->dump();
});
- U->replaceUsesOfWith(RecipeValue, Fixed);
+ U->replaceUsesOfWith(V, Fixed);
}
bool VPPredicator::shouldPreserveTerminator(VPBasicBlock *VPBB) {
@@ -522,8 +528,11 @@ void VPPredicator::convertPhisToBlends(VPBasicBlock *VPBB) {
Builder.setInsertPoint(VPBB, getMaskInsertPoint(VPBB));
SmallVector<VPPhi *> Phis;
- for (VPRecipeBase &R : VPBB->phis())
- Phis.push_back(cast<VPPhi>(&R));
+ for (VPRecipeBase &R : VPBB->phis()) {
+ auto *Phi = cast<VPPhi>(&R);
+ if (!Phi->isSSAReconstructionPhi())
+ Phis.push_back(Phi);
+ }
for (VPPhi *PhiR : Phis) {
LLVM_DEBUG(dbgs() << "Converting " << *PhiR << " to blend\n");
// The non-header Phi is converted into a Blend recipe below,
@@ -546,6 +555,7 @@ void VPPredicator::convertPhisToBlends(VPBasicBlock *VPBB) {
}
SmallVector<VPBasicBlock *> MaskBlocks;
+ SmallVector<VPValue *, 2> OperandsWithMask;
auto &BlockBlendTerms = BlendTerms[BlocksInCompactRPOTOrder.getIndex(VPBB)];
auto Terms = BlockBlendTerms.find(PhiR);
assert(Terms != BlockBlendTerms.end() && "Missing cached blend terms");
@@ -559,9 +569,13 @@ void VPPredicator::convertPhisToBlends(VPBasicBlock *VPBB) {
make_range(MaskBlocks.begin(), MaskBlocks.end())));
VPValue *CommonIncomingMask = getBlockInMask(CommonIncomingDom);
- SmallVector<VPValue *, 2> OperandsWithMask;
for (auto [V, ConstMaskBlock] : Terms->second) {
auto *MaskBlock = const_cast<VPBasicBlock *>(ConstMaskBlock);
+
+ // Reconstruct values at the blend location rather than fixing up the
+ // blend after it has been created.
+ V = reconstructSSA(VPBB, V, /*IsMask=*/false);
+
VPValue *Mask = getBlockInMask(MaskBlock);
VPValue *RemainingMask = nullptr;
bool RemovedCommonMask =
@@ -721,30 +735,15 @@ void VPPredicator::run() {
}
LLVM_DEBUG({
- dbgs() << "VPlan before predicating phis:\n";
+ dbgs() << "VPlan before predicating phis and ssa fixup:\n";
Plan.dump();
});
- for (VPBlockBase *VPBB : reverse(BlocksInCompactRPOTOrder))
+ for (VPBlockBase *VPBB : reverse(BlocksInCompactRPOTOrder)) {
+
if (VPBB != Header)
convertPhisToBlends(cast<VPBasicBlock>(VPBB));
- LLVM_DEBUG({
- dbgs() << "VPlan before SSA-fixup:\n";
- Plan.dump();
- });
-
- for (VPBasicBlock *VPBB :
- VPBlockUtils::blocksOnly<VPBasicBlock>(BlocksInCompactRPOTOrder)) {
- for (VPRecipeBase &R : *VPBB) {
- if (auto *Blend = dyn_cast<VPBlendRecipe>(&R)) {
- // TODO: We can probably do better than this by performing partial
- // blends along the path and have at most one phi per blend in a given
- // basic block.
- for (unsigned I = 0; I < Blend->getNumIncomingValues(); ++I) {
- fixSSA(Blend, Blend->getIncomingValue(I), /*IsMask*/ false);
- fixSSA(Blend, Blend->getMask(I), /*IsMask*/ true);
- }
- }
+ for (VPRecipeBase &R : *cast<VPBasicBlock>(VPBB)) {
auto *I = dyn_cast<VPInstruction>(&R);
if (!I)
continue;
diff --git a/llvm/test/Transforms/LoopVectorize/AArch64/blend-costs.ll b/llvm/test/Transforms/LoopVectorize/AArch64/blend-costs.ll
index 559b53dc87336..13e21316dbe49 100644
--- a/llvm/test/Transforms/LoopVectorize/AArch64/blend-costs.ll
+++ b/llvm/test/Transforms/LoopVectorize/AArch64/blend-costs.ll
@@ -506,7 +506,8 @@ define void @only_first_lane_used(i1 %c, ptr noalias %p1, ptr noalias %p2, ptr n
; CHECK: [[PRED_STORE_CONTINUE6]]:
; CHECK-NEXT: [[TMP8:%.*]] = phi i64 [ [[TMP0]], %[[PRED_STORE_CONTINUE4]] ], [ poison, %[[PRED_STORE_IF3]] ]
; CHECK-NEXT: [[TMP9:%.*]] = phi i64 [ poison, %[[PRED_STORE_CONTINUE4]] ], [ [[TMP7]], %[[PRED_STORE_IF3]] ]
-; CHECK-NEXT: [[TMP4:%.*]] = select i1 true, i64 [[TMP9]], i64 [[TMP8]]
+; CHECK-NEXT: [[TMP10:%.*]] = phi i1 [ false, %[[PRED_STORE_CONTINUE4]] ], [ true, %[[PRED_STORE_IF3]] ]
+; CHECK-NEXT: [[TMP4:%.*]] = select i1 [[TMP10]], i64 [[TMP9]], i64 [[TMP8]]
; CHECK-NEXT: [[TMP1:%.*]] = getelementptr double, ptr [[P1]], i64 [[TMP4]]
; CHECK-NEXT: [[WIDE_LOAD:%.*]] = load <4 x double>, ptr [[TMP1]], align 8
; CHECK-NEXT: [[TMP2:%.*]] = fadd <4 x double> [[WIDE_LOAD]], splat (double 1.000000e+00)
diff --git a/llvm/test/Transforms/LoopVectorize/RISCV/divrem.ll b/llvm/test/Transforms/LoopVectorize/RISCV/divrem.ll
index bc0b7d532584e..596b67bc2f73e 100644
--- a/llvm/test/Transforms/LoopVectorize/RISCV/divrem.ll
+++ b/llvm/test/Transforms/LoopVectorize/RISCV/divrem.ll
@@ -1,4 +1,4 @@
-; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --version 6
; RUN: opt < %s -passes=loop-vectorize -scalable-vectorization=on -mtriple riscv64-linux-gnu -mattr=+v,+f,+m -S 2>%t | FileCheck %s
; RUN: opt < %s -passes=loop-vectorize -scalable-vectorization=off -mtriple riscv64-linux-gnu -mattr=+v,+f,+m -S 2>%t | FileCheck --check-prefix=FIXED %s
@@ -8,18 +8,19 @@ target datalayout = "e-m:e-p:64:64-i64:64-i128:128-n64-S128"
target triple = "riscv64"
define void @vector_udiv(ptr noalias nocapture %a, i64 %v, i64 %n) {
-; CHECK-LABEL: @vector_udiv(
-; CHECK-NEXT: entry:
-; CHECK-NEXT: br label [[VECTOR_PH:%.*]]
-; CHECK: vector.ph:
-; CHECK-NEXT: [[BROADCAST_SPLATINSERT:%.*]] = insertelement <vscale x 2 x i64> poison, i64 [[V:%.*]], i64 0
+; CHECK-LABEL: define void @vector_udiv(
+; CHECK-SAME: ptr noalias captures(none) [[A:%.*]], i64 [[V:%.*]], i64 [[N:%.*]]) #[[ATTR0:[0-9]+]] {
+; CHECK-NEXT: [[ENTRY:.*:]]
+; CHECK-NEXT: br label %[[VECTOR_PH:.*]]
+; CHECK: [[VECTOR_PH]]:
+; CHECK-NEXT: [[BROADCAST_SPLATINSERT:%.*]] = insertelement <vscale x 2 x i64> poison, i64 [[V]], i64 0
; CHECK-NEXT: [[BROADCAST_SPLAT:%.*]] = shufflevector <vscale x 2 x i64> [[BROADCAST_SPLATINSERT]], <vscale x 2 x i64> poison, <vscale x 2 x i32> zeroinitializer
-; CHECK-NEXT: br label [[VECTOR_BODY:%.*]]
-; CHECK: vector.body:
-; CHECK-NEXT: [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ]
-; CHECK-NEXT: [[AVL:%.*]] = phi i64 [ 1024, [[VECTOR_PH]] ], [ [[AVL_NEXT:%.*]], [[VECTOR_BODY]] ]
+; CHECK-NEXT: br label %[[VECTOR_BODY:.*]]
+; CHECK: [[VECTOR_BODY]]:
+; CHECK-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-NEXT: [[AVL:%.*]] = phi i64 [ 1024, %[[VECTOR_PH]] ], [ [[AVL_NEXT:%.*]], %[[VECTOR_BODY]] ]
; CHECK-NEXT: [[TMP10:%.*]] = call i32 @llvm.experimental.get.vector.length.i64(i64 [[AVL]], i32 2, i1 true)
-; CHECK-NEXT: [[TMP7:%.*]] = getelementptr inbounds i64, ptr [[A:%.*]], i64 [[INDEX]]
+; CHECK-NEXT: [[TMP7:%.*]] = getelementptr inbounds i64, ptr [[A]], i64 [[INDEX]]
; CHECK-NEXT: [[WIDE_LOAD:%.*]] = call <vscale x 2 x i64> @llvm.vp.load.nxv2i64.p0(ptr align 8 [[TMP7]], <vscale x 2 x i1> splat (i1 true), i32 [[TMP10]])
; CHECK-NEXT: [[TMP9:%.*]] = udiv <vscale x 2 x i64> [[WIDE_LOAD]], [[BROADCAST_SPLAT]]
; CHECK-NEXT: call void @llvm.vp.store.nxv2i64.p0(<vscale x 2 x i64> [[TMP9]], ptr align 8 [[TMP7]], <vscale x 2 x i1> splat (i1 true), i32 [[TMP10]])
@@ -27,31 +28,32 @@ define void @vector_udiv(ptr noalias nocapture %a, i64 %v, i64 %n) {
; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[TMP8]], [[INDEX]]
; CHECK-NEXT: [[AVL_NEXT]] = sub nuw i64 [[AVL]], [[TMP8]]
; CHECK-NEXT: [[TMP6:%.*]] = icmp eq i64 [[AVL_NEXT]], 0
-; CHECK-NEXT: br i1 [[TMP6]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP0:![0-9]+]]
-; CHECK: middle.block:
-; CHECK-NEXT: br label [[FOR_BODY:%.*]]
-; CHECK: for.end:
+; CHECK-NEXT: br i1 [[TMP6]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP0:![0-9]+]]
+; CHECK: [[MIDDLE_BLOCK]]:
+; CHECK-NEXT: br label %[[FOR_END:.*]]
+; CHECK: [[FOR_END]]:
; CHECK-NEXT: ret void
;
-; FIXED-LABEL: @vector_udiv(
-; FIXED-NEXT: entry:
-; FIXED-NEXT: br label [[VECTOR_PH:%.*]]
-; FIXED: vector.ph:
-; FIXED-NEXT: [[BROADCAST_SPLATINSERT:%.*]] = insertelement <4 x i64> poison, i64 [[V:%.*]], i64 0
+; FIXED-LABEL: define void @vector_udiv(
+; FIXED-SAME: ptr noalias captures(none) [[A:%.*]], i64 [[V:%.*]], i64 [[N:%.*]]) #[[ATTR0:[0-9]+]] {
+; FIXED-NEXT: [[ENTRY:.*:]]
+; FIXED-NEXT: br label %[[VECTOR_PH:.*]]
+; FIXED: [[VECTOR_PH]]:
+; FIXED-NEXT: [[BROADCAST_SPLATINSERT:%.*]] = insertelement <4 x i64> poison, i64 [[V]], i64 0
; FIXED-NEXT: [[BROADCAST_SPLAT:%.*]] = shufflevector <4 x i64> [[BROADCAST_SPLATINSERT]], <4 x i64> poison, <4 x i32> zeroinitializer
-; FIXED-NEXT: br label [[VECTOR_BODY:%.*]]
-; FIXED: vector.body:
-; FIXED-NEXT: [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ]
-; FIXED-NEXT: [[TMP1:%.*]] = getelementptr inbounds i64, ptr [[A:%.*]], i64 [[INDEX]]
+; FIXED-NEXT: br label %[[VECTOR_BODY:.*]]
+; FIXED: [[VECTOR_BODY]]:
+; FIXED-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; FIXED-NEXT: [[TMP1:%.*]] = getelementptr inbounds i64, ptr [[A]], i64 [[INDEX]]
; FIXED-NEXT: [[WIDE_LOAD1:%.*]] = load <4 x i64>, ptr [[TMP1]], align 8
; FIXED-NEXT: [[TMP5:%.*]] = udiv <4 x i64> [[WIDE_LOAD1]], [[BROADCAST_SPLAT]]
; FIXED-NEXT: store <4 x i64> [[TMP5]], ptr [[TMP1]], align 8
; FIXED-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 4
; FIXED-NEXT: [[TMP6:%.*]] = icmp eq i64 [[INDEX_NEXT]], 1024
-; FIXED-NEXT: br i1 [[TMP6]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP0:![0-9]+]]
-; FIXED: middle.block:
-; FIXED-NEXT: br label [[FOR_BODY:%.*]]
-; FIXED: for.end:
+; FIXED-NEXT: br i1 [[TMP6]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP0:![0-9]+]]
+; FIXED: [[MIDDLE_BLOCK]]:
+; FIXED-NEXT: br label %[[FOR_END:.*]]
+; FIXED: [[FOR_END]]:
; FIXED-NEXT: ret void
;
entry:
@@ -72,18 +74,19 @@ for.end:
}
define void @vector_sdiv(ptr noalias nocapture %a, i64 %v, i64 %n) {
-; CHECK-LABEL: @vector_sdiv(
-; CHECK-NEXT: entry:
-; CHECK-NEXT: br label [[VECTOR_PH:%.*]]
-; CHECK: vector.ph:
-; CHECK-NEXT: [[BROADCAST_SPLATINSERT:%.*]] = insertelement <vscale x 2 x i64> poison, i64 [[V:%.*]], i64 0
+; CHECK-LABEL: define void @vector_sdiv(
+; CHECK-SAME: ptr noalias captures(none) [[A:%.*]], i64 [[V:%.*]], i64 [[N:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT: [[ENTRY:.*:]]
+; CHECK-NEXT: br label %[[VECTOR_PH:.*]]
+; CHECK: [[VECTOR_PH]]:
+; CHECK-NEXT: [[BROADCAST_SPLATINSERT:%.*]] = insertelement <vscale x 2 x i64> poison, i64 [[V]], i64 0
; CHECK-NEXT: [[BROADCAST_SPLAT:%.*]] = shufflevector <vscale x 2 x i64> [[BROADCAST_SPLATINSERT]], <vscale x 2 x i64> poison, <vscale x 2 x i32> zeroinitializer
-; CHECK-NEXT: br label [[VECTOR_BODY:%.*]]
-; CHECK: vector.body:
-; CHECK-NEXT: [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ]
-; CHECK-NEXT: [[AVL:%.*]] = phi i64 [ 1024, [[VECTOR_PH]] ], [ [[AVL_NEXT:%.*]], [[VECTOR_BODY]] ]
+; CHECK-NEXT: br label %[[VECTOR_BODY:.*]]
+; CHECK: [[VECTOR_BODY]]:
+; CHECK-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-NEXT: [[AVL:%.*]] = phi i64 [ 1024, %[[VECTOR_PH]] ], [ [[AVL_NEXT:%.*]], %[[VECTOR_BODY]] ]
; CHECK-NEXT: [[TMP10:%.*]] = call i32 @llvm.experimental.get.vector.length.i64(i64 [[AVL]], i32 2, i1 true)
-; CHECK-NEXT: [[TMP7:%.*]] = getelementptr inbounds i64, ptr [[A:%.*]], i64 [[INDEX]]
+; CHECK-NEXT: [[TMP7:%.*]] = getelementptr inbounds i64, ptr [[A]], i64 [[INDEX]]
; CHECK-NEXT: [[WIDE_LOAD:%.*]] = call <vscale x 2 x i64> @llvm.vp.load.nxv2i64.p0(ptr align 8 [[TMP7]], <vscale x 2 x i1> splat (i1 true), i32 [[TMP10]])
; CHECK-NEXT: [[TMP9:%.*]] = call <vscale x 2 x i64> @llvm.vp.sdiv.nxv2i64(<vscale x 2 x i64> [[WIDE_LOAD]], <vscale x 2 x i64> [[BROADCAST_SPLAT]], <vscale x 2 x i1> splat (i1 true), i32 [[TMP10]])
; CHECK-NEXT: call void @llvm.vp.store.nxv2i64.p0(<vscale x 2 x i64> [[TMP9]], ptr align 8 [[TMP7]], <vscale x 2 x i1> splat (i1 true), i32 [[TMP10]])
@@ -91,31 +94,32 @@ define void @vector_sdiv(ptr noalias nocapture %a, i64 %v, i64 %n) {
; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[TMP8]], [[INDEX]]
; CHECK-NEXT: [[AVL_NEXT]] = sub nuw i64 [[AVL]], [[TMP8]]
; CHECK-NEXT: [[TMP6:%.*]] = icmp eq i64 [[AVL_NEXT]], 0
-; CHECK-NEXT: br i1 [[TMP6]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP3:![0-9]+]]
-; CHECK: middle.block:
-; CHECK-NEXT: br label [[FOR_BODY:%.*]]
-; CHECK: for.end:
+; CHECK-NEXT: br i1 [[TMP6]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP3:![0-9]+]]
+; CHECK: [[MIDDLE_BLOCK]]:
+; CHECK-NEXT: br label %[[FOR_END:.*]]
+; CHECK: [[FOR_END]]:
; CHECK-NEXT: ret void
;
-; FIXED-LABEL: @vector_sdiv(
-; FIXED-NEXT: entry:
-; FIXED-NEXT: br label [[VECTOR_PH:%.*]]
-; FIXED: vector.ph:
-; FIXED-NEXT: [[BROADCAST_SPLATINSERT:%.*]] = insertelement <4 x i64> poison, i64 [[V:%.*]], i64 0
+; FIXED-LABEL: define void @vector_sdiv(
+; FIXED-SAME: ptr noalias captures(none) [[A:%.*]], i64 [[V:%.*]], i64 [[N:%.*]]) #[[ATTR0]] {
+; FIXED-NEXT: [[ENTRY:.*:]]
+; FIXED-NEXT: br label %[[VECTOR_PH:.*]]
+; FIXED: [[VECTOR_PH]]:
+; FIXED-NEXT: [[BROADCAST_SPLATINSERT:%.*]] = insertelement <4 x i64> poison, i64 [[V]], i64 0
; FIXED-NEXT: [[BROADCAST_SPLAT:%.*]] = shufflevector <4 x i64> [[BROADCAST_SPLATINSERT]], <4 x i64> poison, <4 x i32> zeroinitializer
-; FIXED-NEXT: br label [[VECTOR_BODY:%.*]]
-; FIXED: vector.body:
-; FIXED-NEXT: [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ]
-; FIXED-NEXT: [[TMP1:%.*]] = getelementptr inbounds i64, ptr [[A:%.*]], i64 [[INDEX]]
+; FIXED-NEXT: br label %[[VECTOR_BODY:.*]]
+; FIXED: [[VECTOR_BODY]]:
+; FIXED-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; FIXED-NEXT: [[TMP1:%.*]] = getelementptr inbounds i64, ptr [[A]], i64 [[INDEX]]
; FIXED-NEXT: [[WIDE_LOAD1:%.*]] = load <4 x i64>, ptr [[TMP1]], align 8
; FIXED-NEXT: [[TMP5:%.*]] = sdiv <4 x i64> [[WIDE_LOAD1]], [[BROADCAST_SPLAT]]
; FIXED-NEXT: store <4 x i64> [[TMP5]], ptr [[TMP1]], align 8
; FIXED-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 4
; FIXED-NEXT: [[TMP6:%.*]] = icmp eq i64 [[INDEX_NEXT]], 1024
-; FIXED-NEXT: br i1 [[TMP6]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP3:![0-9]+]]
-; FIXED: middle.block:
-; FIXED-NEXT: br label [[FOR_BODY:%.*]]
-; FIXED: for.end:
+; FIXED-NEXT: br i1 [[TMP6]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP3:![0-9]+]]
+; FIXED: [[MIDDLE_BLOCK]]:
+; FIXED-NEXT: br label %[[FOR_END:.*]]
+; FIXED: [[FOR_END]]:
; FIXED-NEXT: ret void
;
entry:
@@ -136,18 +140,19 @@ for.end:
}
define void @vector_urem(ptr noalias nocapture %a, i64 %v, i64 %n) {
-; CHECK-LABEL: @vector_urem(
-; CHECK-NEXT: entry:
-; CHECK-NEXT: br label [[VECTOR_PH:%.*]]
-; CHECK: vector.ph:
-; CHECK-NEXT: [[BROADCAST_SPLATINSERT:%.*]] = insertelement <vscale x 2 x i64> poison, i64 [[V:%.*]], i64 0
+; CHECK-LABEL: define void @vector_urem(
+; CHECK-SAME: ptr noalias captures(none) [[A:%.*]], i64 [[V:%.*]], i64 [[N:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT: [[ENTRY:.*:]]
+; CHECK-NEXT: br label %[[VECTOR_PH:.*]]
+; CHECK: [[VECTOR_PH]]:
+; CHECK-NEXT: [[BROADCAST_SPLATINSERT:%.*]] = insertelement <vscale x 2 x i64> poison, i64 [[V]], i64 0
; CHECK-NEXT: [[BROADCAST_SPLAT:%.*]] = shufflevector <vscale x 2 x i64> [[BROADCAST_SPLATINSERT]], <vscale x 2 x i64> poison, <vscale x 2 x i32> zeroinitializer
-; CHECK-NEXT: br label [[VECTOR_BODY:%.*]]
-; CHECK: vector.body:
-; CHECK-NEXT: [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ]
-; CHECK-NEXT: [[AVL:%.*]] = phi i64 [ 1024, [[VECTOR_PH]] ], [ [[AVL_NEXT:%.*]], [[VECTOR_BODY]] ]
+; CHECK-NEXT: br label %[[VECTOR_BODY:.*]]
+; CHECK: [[VECTOR_BODY]]:
+; CHECK-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-NEXT: [[AVL:%.*]] = phi i64 [ 1024, %[[VECTOR_PH]] ], [ [[AVL_NEXT:%.*]], %[[VECTOR_BODY]] ]
; CHECK-NEXT: [[TMP10:%.*]] = call i32 @llvm.experimental.get.vector.length.i64(i64 [[AVL]], i32 2, i1 true)
-; CHECK-NEXT: [[TMP7:%.*]] = getelementptr inbounds i64, ptr [[A:%.*]], i64 [[INDEX]]
+; CHECK-NEXT: [[TMP7:%.*]] = getelementptr inbounds i64, ptr [[A]], i64 [[INDEX]]
; CHECK-NEXT: [[WIDE_LOAD:%.*]] = call <vscale x 2 x i64> @llvm.vp.load.nxv2i64.p0(ptr align 8 [[TMP7]], <vscale x 2 x i1> splat (i1 true), i32 [[TMP10]])
; CHECK-NEXT: [[TMP9:%.*]] = urem <vscale x 2 x i64> [[WIDE_LOAD]], [[BROADCAST_SPLAT]]
; CHECK-NEXT: call void @llvm.vp.store.nxv2i64.p0(<vscale x 2 x i64> [[TMP9]], ptr align 8 [[TMP7]], <vscale x 2 x i1> splat (i1 true), i32 [[TMP10]])
@@ -155,31 +160,32 @@ define void @vector_urem(ptr noalias nocapture %a, i64 %v, i64 %n) {
; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[TMP8]], [[INDEX]]
; CHECK-NEXT: [[AVL_NEXT]] = sub nuw i64 [[AVL]], [[TMP8]]
; CHECK-NEXT: [[TMP6:%.*]] = icmp eq i64 [[AVL_NEXT]], 0
-; CHECK-NEXT: br i1 [[TMP6]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP4:![0-9]+]]
-; CHECK: middle.block:
-; CHECK-NEXT: br label [[FOR_BODY:%.*]]
-; CHECK: for.end:
+; CHECK-NEXT: br i1 [[TMP6]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP4:![0-9]+]]
+; CHECK: [[MIDDLE_BLOCK]]:
+; CHECK-NEXT: br label %[[FOR_END:.*]]
+; CHECK: [[FOR_END]]:
; CHECK-NEXT: ret void
;
-; FIXED-LABEL: @vector_urem(
-; FIXED-NEXT: entry:
-; FIXED-NEXT: br label [[VECTOR_PH:%.*]]
-; FIXED: vector.ph:
-; FIXED-NEXT: [[BROADCAST_SPLATINSERT:%.*]] = insertelement <4 x i64> poison, i64 [[V:%.*]], i64 0
+; FIXED-LABEL: define void @vector_urem(
+; FIXED-SAME: ptr noalias captures(none) [[A:%.*]], i64 [[V:%.*]], i64 [[N:%.*]]) #[[ATTR0]] {
+; FIXED-NEXT: [[ENTRY:.*:]]
+; FIXED-NEXT: br label %[[VECTOR_PH:.*]]
+; FIXED: [[VECTOR_PH]]:
+; FIXED-NEXT: [[BROADCAST_SPLATINSERT:%.*]] = insertelement <4 x i64> poison, i64 [[V]], i64 0
; FIXED-NEXT: [[BROADCAST_SPLAT:%.*]] = shufflevector <4 x i64> [[BROADCAST_SPLATINSERT]], <4 x i64> poison, <4 x i32> zeroinitializer
-; FIXED-NEXT: br label [[VECTOR_BODY:%.*]]
-; FIXED: vector.body:
-; FIXED-NEXT: [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ]
-; FIXED-NEXT: [[TMP1:%.*]] = getelementptr inbounds i64, ptr [[A:%.*]], i64 [[INDEX]]
+; FIXED-NEXT: br label %[[VECTOR_BODY:.*]]
+; FIXED: [[VECTOR_BODY]]:
+; FIXED-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; FIXED-NEXT: [[TMP1:%.*]] = getelementptr inbounds i64, ptr [[A]], i64 [[INDEX]]
; FIXED-NEXT: [[WIDE_LOAD1:%.*]] = load <4 x i64>, ptr [[TMP1]], align 8
; FIXED-NEXT: [[TMP5:%.*]] = urem <4 x i64> [[WIDE_LOAD1]], [[BROADCAST_SPLAT]]
; FIXED-NEXT: store <4 x i64> [[TMP5]], ptr [[TMP1]], align 8
; FIXED-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 4
; FIXED-NEXT: [[TMP6:%.*]] = icmp eq i64 [[INDEX_NEXT]], 1024
-; FIXED-NEXT: br i1 [[TMP6]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP4:![0-9]+]]
-; FIXED: middle.block:
-; FIXED-NEXT: br label [[FOR_BODY:%.*]]
-; FIXED: for.end:
+; FIXED-NEXT: br i1 [[TMP6]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP4:![0-9]+]]
+; FIXED: [[MIDDLE_BLOCK]]:
+; FIXED-NEXT: br label %[[FOR_END:.*]]
+; FIXED: [[FOR_END]]:
; FIXED-NEXT: ret void
;
entry:
@@ -200,18 +206,19 @@ for.end:
}
define void @vector_srem(ptr noalias nocapture %a, i64 %v, i64 %n) {
-; CHECK-LABEL: @vector_srem(
-; CHECK-NEXT: entry:
-; CHECK-NEXT: br label [[VECTOR_PH:%.*]]
-; CHECK: vector.ph:
-; CHECK-NEXT: [[BROADCAST_SPLATINSERT:%.*]] = insertelement <vscale x 2 x i64> poison, i64 [[V:%.*]], i64 0
+; CHECK-LABEL: define void @vector_srem(
+; CHECK-SAME: ptr noalias captures(none) [[A:%.*]], i64 [[V:%.*]], i64 [[N:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT: [[ENTRY:.*:]]
+; CHECK-NEXT: br label %[[VECTOR_PH:.*]]
+; CHECK: [[VECTOR_PH]]:
+; CHECK-NEXT: [[BROADCAST_SPLATINSERT:%.*]] = insertelement <vscale x 2 x i64> poison, i64 [[V]], i64 0
; CHECK-NEXT: [[BROADCAST_SPLAT:%.*]] = shufflevector <vscale x 2 x i64> [[BROADCAST_SPLATINSERT]], <vscale x 2 x i64> poison, <vscale x 2 x i32> zeroinitializer
-; CHECK-NEXT: br label [[VECTOR_BODY:%.*]]
-; CHECK: vector.body:
-; CHECK-NEXT: [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ]
-; CHECK-NEXT: [[AVL:%.*]] = phi i64 [ 1024, [[VECTOR_PH]] ], [ [[AVL_NEXT:%.*]], [[VECTOR_BODY]] ]
+; CHECK-NEXT: br label %[[VECTOR_BODY:.*]]
+; CHECK: [[VECTOR_BODY]]:
+; CHECK-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-NEXT: [[AVL:%.*]] = phi i64 [ 1024, %[[VECTOR_PH]] ], [ [[AVL_NEXT:%.*]], %[[VECTOR_BODY]] ]
; CHECK-NEXT: [[TMP10:%.*]] = call i32 @llvm.experimental.get.vector.length.i64(i64 [[AVL]], i32 2, i1 true)
-; CHECK-NEXT: [[TMP7:%.*]] = getelementptr inbounds i64, ptr [[A:%.*]], i64 [[INDEX]]
+; CHECK-NEXT: [[TMP7:%.*]] = getelementptr inbounds i64, ptr [[A]], i64 [[INDEX]]
; CHECK-NEXT: [[WIDE_LOAD:%.*]] = call <vscale x 2 x i64> @llvm.vp.load.nxv2i64.p0(ptr align 8 [[TMP7]], <vscale x 2 x i1> splat (i1 true), i32 [[TMP10]])
; CHECK-NEXT: [[TMP9:%.*]] = call <vscale x 2 x i64> @llvm.vp.srem.nxv2i64(<vscale x 2 x i64> [[WIDE_LOAD]], <vscale x 2 x i64> [[BROADCAST_SPLAT]], <vscale x 2 x i1> splat (i1 true), i32 [[TMP10]])
; CHECK-NEXT: call void @llvm.vp.store.nxv2i64.p0(<vscale x 2 x i64> [[TMP9]], ptr align 8 [[TMP7]], <vscale x 2 x i1> splat (i1 true), i32 [[TMP10]])
@@ -219,31 +226,32 @@ define void @vector_srem(ptr noalias nocapture %a, i64 %v, i64 %n) {
; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[TMP8]], [[INDEX]]
; CHECK-NEXT: [[AVL_NEXT]] = sub nuw i64 [[AVL]], [[TMP8]]
; CHECK-NEXT: [[TMP6:%.*]] = icmp eq i64 [[AVL_NEXT]], 0
-; CHECK-NEXT: br i1 [[TMP6]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP5:![0-9]+]]
-; CHECK: middle.block:
-; CHECK-NEXT: br label [[FOR_BODY:%.*]]
-; CHECK: for.end:
+; CHECK-NEXT: br i1 [[TMP6]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP5:![0-9]+]]
+; CHECK: [[MIDDLE_BLOCK]]:
+; CHECK-NEXT: br label %[[FOR_END:.*]]
+; CHECK: [[FOR_END]]:
; CHECK-NEXT: ret void
;
-; FIXED-LABEL: @vector_srem(
-; FIXED-NEXT: entry:
-; FIXED-NEXT: br label [[VECTOR_PH:%.*]]
-; FIXED: vector.ph:
-; FIXED-NEXT: [[BROADCAST_SPLATINSERT:%.*]] = insertelement <4 x i64> poison, i64 [[V:%.*]], i64 0
+; FIXED-LABEL: define void @vector_srem(
+; FIXED-SAME: ptr noalias captures(none) [[A:%.*]], i64 [[V:%.*]], i64 [[N:%.*]]) #[[ATTR0]] {
+; FIXED-NEXT: [[ENTRY:.*:]]
+; FIXED-NEXT: br label %[[VECTOR_PH:.*]]
+; FIXED: [[VECTOR_PH]]:
+; FIXED-NEXT: [[BROADCAST_SPLATINSERT:%.*]] = insertelement <4 x i64> poison, i64 [[V]], i64 0
; FIXED-NEXT: [[BROADCAST_SPLAT:%.*]] = shufflevector <4 x i64> [[BROADCAST_SPLATINSERT]], <4 x i64> poison, <4 x i32> zeroinitializer
-; FIXED-NEXT: br label [[VECTOR_BODY:%.*]]
-; FIXED: vector.body:
-; FIXED-NEXT: [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ]
-; FIXED-NEXT: [[TMP1:%.*]] = getelementptr inbounds i64, ptr [[A:%.*]], i64 [[INDEX]]
+; FIXED-NEXT: br label %[[VECTOR_BODY:.*]]
+; FIXED: [[VECTOR_BODY]]:
+; FIXED-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; FIXED-NEXT: [[TMP1:%.*]] = getelementptr inbounds i64, ptr [[A]], i64 [[INDEX]]
; FIXED-NEXT: [[WIDE_LOAD1:%.*]] = load <4 x i64>, ptr [[TMP1]], align 8
; FIXED-NEXT: [[TMP5:%.*]] = srem <4 x i64> [[WIDE_LOAD1]], [[BROADCAST_SPLAT]]
; FIXED-NEXT: store <4 x i64> [[TMP5]], ptr [[TMP1]], align 8
; FIXED-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 4
; FIXED-NEXT: [[TMP6:%.*]] = icmp eq i64 [[INDEX_NEXT]], 1024
-; FIXED-NEXT: br i1 [[TMP6]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP5:![0-9]+]]
-; FIXED: middle.block:
-; FIXED-NEXT: br label [[FOR_BODY:%.*]]
-; FIXED: for.end:
+; FIXED-NEXT: br i1 [[TMP6]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP5:![0-9]+]]
+; FIXED: [[MIDDLE_BLOCK]]:
+; FIXED-NEXT: br label %[[FOR_END:.*]]
+; FIXED: [[FOR_END]]:
; FIXED-NEXT: ret void
;
entry:
@@ -264,19 +272,20 @@ for.end:
}
define void @predicated_udiv(ptr noalias nocapture %a, i64 %v, i64 %n) {
-; CHECK-LABEL: @predicated_udiv(
-; CHECK-NEXT: entry:
-; CHECK-NEXT: br label [[VECTOR_PH:%.*]]
-; CHECK: vector.ph:
-; CHECK-NEXT: [[BROADCAST_SPLATINSERT:%.*]] = insertelement <vscale x 2 x i64> poison, i64 [[V:%.*]], i64 0
+; CHECK-LABEL: define void @predicated_udiv(
+; CHECK-SAME: ptr noalias captures(none) [[A:%.*]], i64 [[V:%.*]], i64 [[N:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT: [[ENTRY:.*:]]
+; CHECK-NEXT: br label %[[VECTOR_PH:.*]]
+; CHECK: [[VECTOR_PH]]:
+; CHECK-NEXT: [[BROADCAST_SPLATINSERT:%.*]] = insertelement <vscale x 2 x i64> poison, i64 [[V]], i64 0
; CHECK-NEXT: [[BROADCAST_SPLAT:%.*]] = shufflevector <vscale x 2 x i64> [[BROADCAST_SPLATINSERT]], <vscale x 2 x i64> poison, <vscale x 2 x i32> zeroinitializer
; CHECK-NEXT: [[TMP6:%.*]] = icmp ne <vscale x 2 x i64> [[BROADCAST_SPLAT]], zeroinitializer
-; CHECK-NEXT: br label [[VECTOR_BODY:%.*]]
-; CHECK: vector.body:
-; CHECK-NEXT: [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ]
-; CHECK-NEXT: [[AVL:%.*]] = phi i64 [ 1024, [[VECTOR_PH]] ], [ [[AVL_NEXT:%.*]], [[VECTOR_BODY]] ]
+; CHECK-NEXT: br label %[[VECTOR_BODY:.*]]
+; CHECK: [[VECTOR_BODY]]:
+; CHECK-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-NEXT: [[AVL:%.*]] = phi i64 [ 1024, %[[VECTOR_PH]] ], [ [[AVL_NEXT:%.*]], %[[VECTOR_BODY]] ]
; CHECK-NEXT: [[TMP12:%.*]] = call i32 @llvm.experimental.get.vector.length.i64(i64 [[AVL]], i32 2, i1 true)
-; CHECK-NEXT: [[TMP8:%.*]] = getelementptr inbounds i64, ptr [[A:%.*]], i64 [[INDEX]]
+; CHECK-NEXT: [[TMP8:%.*]] = getelementptr inbounds i64, ptr [[A]], i64 [[INDEX]]
; CHECK-NEXT: [[WIDE_LOAD:%.*]] = call <vscale x 2 x i64> @llvm.vp.load.nxv2i64.p0(ptr align 8 [[TMP8]], <vscale x 2 x i1> splat (i1 true), i32 [[TMP12]])
; CHECK-NEXT: [[TMP11:%.*]] = call <vscale x 2 x i64> @llvm.vp.udiv.nxv2i64(<vscale x 2 x i64> [[WIDE_LOAD]], <vscale x 2 x i64> [[BROADCAST_SPLAT]], <vscale x 2 x i1> [[TMP6]], i32 [[TMP12]])
; CHECK-NEXT: [[TMP9:%.*]] = extractelement <vscale x 2 x i1> [[TMP6]], i64 0
@@ -286,23 +295,24 @@ define void @predicated_udiv(ptr noalias nocapture %a, i64 %v, i64 %n) {
; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[TMP13]], [[INDEX]]
; CHECK-NEXT: [[AVL_NEXT]] = sub nuw i64 [[AVL]], [[TMP13]]
; CHECK-NEXT: [[TMP14:%.*]] = icmp eq i64 [[AVL_NEXT]], 0
-; CHECK-NEXT: br i1 [[TMP14]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP6:![0-9]+]]
-; CHECK: middle.block:
-; CHECK-NEXT: br label [[LATCH:%.*]]
-; CHECK: for.end:
+; CHECK-NEXT: br i1 [[TMP14]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP6:![0-9]+]]
+; CHECK: [[MIDDLE_BLOCK]]:
+; CHECK-NEXT: br label %[[FOR_END:.*]]
+; CHECK: [[FOR_END]]:
; CHECK-NEXT: ret void
;
-; FIXED-LABEL: @predicated_udiv(
-; FIXED-NEXT: entry:
-; FIXED-NEXT: br label [[VECTOR_PH:%.*]]
-; FIXED: vector.ph:
-; FIXED-NEXT: [[BROADCAST_SPLATINSERT:%.*]] = insertelement <4 x i64> poison, i64 [[V:%.*]], i64 0
+; FIXED-LABEL: define void @predicated_udiv(
+; FIXED-SAME: ptr noalias captures(none) [[A:%.*]], i64 [[V:%.*]], i64 [[N:%.*]]) #[[ATTR0]] {
+; FIXED-NEXT: [[ENTRY:.*:]]
+; FIXED-NEXT: br label %[[VECTOR_PH:.*]]
+; FIXED: [[VECTOR_PH]]:
+; FIXED-NEXT: [[BROADCAST_SPLATINSERT:%.*]] = insertelement <4 x i64> poison, i64 [[V]], i64 0
; FIXED-NEXT: [[BROADCAST_SPLAT:%.*]] = shufflevector <4 x i64> [[BROADCAST_SPLATINSERT]], <4 x i64> poison, <4 x i32> zeroinitializer
; FIXED-NEXT: [[TMP0:%.*]] = icmp ne <4 x i64> [[BROADCAST_SPLAT]], zeroinitializer
-; FIXED-NEXT: br label [[VECTOR_BODY:%.*]]
-; FIXED: vector.body:
-; FIXED-NEXT: [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ]
-; FIXED-NEXT: [[TMP2:%.*]] = getelementptr inbounds i64, ptr [[A:%.*]], i64 [[INDEX]]
+; FIXED-NEXT: br label %[[VECTOR_BODY:.*]]
+; FIXED: [[VECTOR_BODY]]:
+; FIXED-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; FIXED-NEXT: [[TMP2:%.*]] = getelementptr inbounds i64, ptr [[A]], i64 [[INDEX]]
; FIXED-NEXT: [[WIDE_LOAD1:%.*]] = load <4 x i64>, ptr [[TMP2]], align 8
; FIXED-NEXT: [[TMP8:%.*]] = call <4 x i64> @llvm.masked.udiv.v4i64(<4 x i64> [[WIDE_LOAD1]], <4 x i64> [[BROADCAST_SPLAT]], <4 x i1> [[TMP0]])
; FIXED-NEXT: [[TMP6:%.*]] = extractelement <4 x i1> [[TMP0]], i64 0
@@ -310,10 +320,10 @@ define void @predicated_udiv(ptr noalias nocapture %a, i64 %v, i64 %n) {
; FIXED-NEXT: store <4 x i64> [[PREDPHI2]], ptr [[TMP2]], align 8
; FIXED-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 4
; FIXED-NEXT: [[TMP9:%.*]] = icmp eq i64 [[INDEX_NEXT]], 1024
-; FIXED-NEXT: br i1 [[TMP9]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP6:![0-9]+]]
-; FIXED: middle.block:
-; FIXED-NEXT: br label [[LATCH:%.*]]
-; FIXED: for.end:
+; FIXED-NEXT: br i1 [[TMP9]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP6:![0-9]+]]
+; FIXED: [[MIDDLE_BLOCK]]:
+; FIXED-NEXT: br label %[[FOR_END:.*]]
+; FIXED: [[FOR_END]]:
; FIXED-NEXT: ret void
;
entry:
@@ -340,19 +350,20 @@ for.end:
}
define void @predicated_sdiv(ptr noalias nocapture %a, i64 %v, i64 %n) {
-; CHECK-LABEL: @predicated_sdiv(
-; CHECK-NEXT: entry:
-; CHECK-NEXT: br label [[VECTOR_PH:%.*]]
-; CHECK: vector.ph:
-; CHECK-NEXT: [[BROADCAST_SPLATINSERT:%.*]] = insertelement <vscale x 2 x i64> poison, i64 [[V:%.*]], i64 0
+; CHECK-LABEL: define void @predicated_sdiv(
+; CHECK-SAME: ptr noalias captures(none) [[A:%.*]], i64 [[V:%.*]], i64 [[N:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT: [[ENTRY:.*:]]
+; CHECK-NEXT: br label %[[VECTOR_PH:.*]]
+; CHECK: [[VECTOR_PH]]:
+; CHECK-NEXT: [[BROADCAST_SPLATINSERT:%.*]] = insertelement <vscale x 2 x i64> poison, i64 [[V]], i64 0
; CHECK-NEXT: [[BROADCAST_SPLAT:%.*]] = shufflevector <vscale x 2 x i64> [[BROADCAST_SPLATINSERT]], <vscale x 2 x i64> poison, <vscale x 2 x i32> zeroinitializer
; CHECK-NEXT: [[TMP6:%.*]] = icmp ne <vscale x 2 x i64> [[BROADCAST_SPLAT]], zeroinitializer
-; CHECK-NEXT: br label [[VECTOR_BODY:%.*]]
-; CHECK: vector.body:
-; CHECK-NEXT: [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ]
-; CHECK-NEXT: [[AVL:%.*]] = phi i64 [ 1024, [[VECTOR_PH]] ], [ [[AVL_NEXT:%.*]], [[VECTOR_BODY]] ]
+; CHECK-NEXT: br label %[[VECTOR_BODY:.*]]
+; CHECK: [[VECTOR_BODY]]:
+; CHECK-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-NEXT: [[AVL:%.*]] = phi i64 [ 1024, %[[VECTOR_PH]] ], [ [[AVL_NEXT:%.*]], %[[VECTOR_BODY]] ]
; CHECK-NEXT: [[TMP12:%.*]] = call i32 @llvm.experimental.get.vector.length.i64(i64 [[AVL]], i32 2, i1 true)
-; CHECK-NEXT: [[TMP8:%.*]] = getelementptr inbounds i64, ptr [[A:%.*]], i64 [[INDEX]]
+; CHECK-NEXT: [[TMP8:%.*]] = getelementptr inbounds i64, ptr [[A]], i64 [[INDEX]]
; CHECK-NEXT: [[WIDE_LOAD:%.*]] = call <vscale x 2 x i64> @llvm.vp.load.nxv2i64.p0(ptr align 8 [[TMP8]], <vscale x 2 x i1> splat (i1 true), i32 [[TMP12]])
; CHECK-NEXT: [[TMP11:%.*]] = call <vscale x 2 x i64> @llvm.vp.sdiv.nxv2i64(<vscale x 2 x i64> [[WIDE_LOAD]], <vscale x 2 x i64> [[BROADCAST_SPLAT]], <vscale x 2 x i1> [[TMP6]], i32 [[TMP12]])
; CHECK-NEXT: [[TMP9:%.*]] = extractelement <vscale x 2 x i1> [[TMP6]], i64 0
@@ -362,23 +373,24 @@ define void @predicated_sdiv(ptr noalias nocapture %a, i64 %v, i64 %n) {
; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[TMP13]], [[INDEX]]
; CHECK-NEXT: [[AVL_NEXT]] = sub nuw i64 [[AVL]], [[TMP13]]
; CHECK-NEXT: [[TMP14:%.*]] = icmp eq i64 [[AVL_NEXT]], 0
-; CHECK-NEXT: br i1 [[TMP14]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP7:![0-9]+]]
-; CHECK: middle.block:
-; CHECK-NEXT: br label [[LATCH:%.*]]
-; CHECK: for.end:
+; CHECK-NEXT: br i1 [[TMP14]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP7:![0-9]+]]
+; CHECK: [[MIDDLE_BLOCK]]:
+; CHECK-NEXT: br label %[[FOR_END:.*]]
+; CHECK: [[FOR_END]]:
; CHECK-NEXT: ret void
;
-; FIXED-LABEL: @predicated_sdiv(
-; FIXED-NEXT: entry:
-; FIXED-NEXT: br label [[VECTOR_PH:%.*]]
-; FIXED: vector.ph:
-; FIXED-NEXT: [[BROADCAST_SPLATINSERT:%.*]] = insertelement <4 x i64> poison, i64 [[V:%.*]], i64 0
+; FIXED-LABEL: define void @predicated_sdiv(
+; FIXED-SAME: ptr noalias captures(none) [[A:%.*]], i64 [[V:%.*]], i64 [[N:%.*]]) #[[ATTR0]] {
+; FIXED-NEXT: [[ENTRY:.*:]]
+; FIXED-NEXT: br label %[[VECTOR_PH:.*]]
+; FIXED: [[VECTOR_PH]]:
+; FIXED-NEXT: [[BROADCAST_SPLATINSERT:%.*]] = insertelement <4 x i64> poison, i64 [[V]], i64 0
; FIXED-NEXT: [[BROADCAST_SPLAT:%.*]] = shufflevector <4 x i64> [[BROADCAST_SPLATINSERT]], <4 x i64> poison, <4 x i32> zeroinitializer
; FIXED-NEXT: [[TMP0:%.*]] = icmp ne <4 x i64> [[BROADCAST_SPLAT]], zeroinitializer
-; FIXED-NEXT: br label [[VECTOR_BODY:%.*]]
-; FIXED: vector.body:
-; FIXED-NEXT: [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ]
-; FIXED-NEXT: [[TMP2:%.*]] = getelementptr inbounds i64, ptr [[A:%.*]], i64 [[INDEX]]
+; FIXED-NEXT: br label %[[VECTOR_BODY:.*]]
+; FIXED: [[VECTOR_BODY]]:
+; FIXED-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; FIXED-NEXT: [[TMP2:%.*]] = getelementptr inbounds i64, ptr [[A]], i64 [[INDEX]]
; FIXED-NEXT: [[WIDE_LOAD1:%.*]] = load <4 x i64>, ptr [[TMP2]], align 8
; FIXED-NEXT: [[TMP8:%.*]] = call <4 x i64> @llvm.masked.sdiv.v4i64(<4 x i64> [[WIDE_LOAD1]], <4 x i64> [[BROADCAST_SPLAT]], <4 x i1> [[TMP0]])
; FIXED-NEXT: [[TMP6:%.*]] = extractelement <4 x i1> [[TMP0]], i64 0
@@ -386,10 +398,10 @@ define void @predicated_sdiv(ptr noalias nocapture %a, i64 %v, i64 %n) {
; FIXED-NEXT: store <4 x i64> [[PREDPHI2]], ptr [[TMP2]], align 8
; FIXED-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 4
; FIXED-NEXT: [[TMP9:%.*]] = icmp eq i64 [[INDEX_NEXT]], 1024
-; FIXED-NEXT: br i1 [[TMP9]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP7:![0-9]+]]
-; FIXED: middle.block:
-; FIXED-NEXT: br label [[LATCH:%.*]]
-; FIXED: for.end:
+; FIXED-NEXT: br i1 [[TMP9]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP7:![0-9]+]]
+; FIXED: [[MIDDLE_BLOCK]]:
+; FIXED-NEXT: br label %[[FOR_END:.*]]
+; FIXED: [[FOR_END]]:
; FIXED-NEXT: ret void
;
entry:
@@ -416,16 +428,17 @@ for.end:
}
define void @predicated_udiv_by_constant(ptr noalias nocapture %a, i64 %n) {
-; CHECK-LABEL: @predicated_udiv_by_constant(
-; CHECK-NEXT: entry:
-; CHECK-NEXT: br label [[VECTOR_PH:%.*]]
-; CHECK: vector.ph:
-; CHECK-NEXT: br label [[VECTOR_BODY:%.*]]
-; CHECK: vector.body:
-; CHECK-NEXT: [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ]
-; CHECK-NEXT: [[AVL:%.*]] = phi i64 [ 1024, [[VECTOR_PH]] ], [ [[AVL_NEXT:%.*]], [[VECTOR_BODY]] ]
+; CHECK-LABEL: define void @predicated_udiv_by_constant(
+; CHECK-SAME: ptr noalias captures(none) [[A:%.*]], i64 [[N:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT: [[ENTRY:.*:]]
+; CHECK-NEXT: br label %[[VECTOR_PH:.*]]
+; CHECK: [[VECTOR_PH]]:
+; CHECK-NEXT: br label %[[VECTOR_BODY:.*]]
+; CHECK: [[VECTOR_BODY]]:
+; CHECK-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-NEXT: [[AVL:%.*]] = phi i64 [ 1024, %[[VECTOR_PH]] ], [ [[AVL_NEXT:%.*]], %[[VECTOR_BODY]] ]
; CHECK-NEXT: [[TMP14:%.*]] = call i32 @llvm.experimental.get.vector.length.i64(i64 [[AVL]], i32 2, i1 true)
-; CHECK-NEXT: [[TMP7:%.*]] = getelementptr inbounds i64, ptr [[A:%.*]], i64 [[INDEX]]
+; CHECK-NEXT: [[TMP7:%.*]] = getelementptr inbounds i64, ptr [[A]], i64 [[INDEX]]
; CHECK-NEXT: [[WIDE_LOAD:%.*]] = call <vscale x 2 x i64> @llvm.vp.load.nxv2i64.p0(ptr align 8 [[TMP7]], <vscale x 2 x i1> splat (i1 true), i32 [[TMP14]])
; CHECK-NEXT: [[TMP2:%.*]] = icmp ne <vscale x 2 x i64> [[WIDE_LOAD]], splat (i64 42)
; CHECK-NEXT: [[TMP10:%.*]] = udiv <vscale x 2 x i64> [[WIDE_LOAD]], splat (i64 27)
@@ -435,20 +448,21 @@ define void @predicated_udiv_by_constant(ptr noalias nocapture %a, i64 %n) {
; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[TMP12]], [[INDEX]]
; CHECK-NEXT: [[AVL_NEXT]] = sub nuw i64 [[AVL]], [[TMP12]]
; CHECK-NEXT: [[TMP13:%.*]] = icmp eq i64 [[AVL_NEXT]], 0
-; CHECK-NEXT: br i1 [[TMP13]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP8:![0-9]+]]
-; CHECK: middle.block:
-; CHECK-NEXT: br label [[LATCH:%.*]]
-; CHECK: for.end:
+; CHECK-NEXT: br i1 [[TMP13]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP8:![0-9]+]]
+; CHECK: [[MIDDLE_BLOCK]]:
+; CHECK-NEXT: br label %[[FOR_END:.*]]
+; CHECK: [[FOR_END]]:
; CHECK-NEXT: ret void
;
-; FIXED-LABEL: @predicated_udiv_by_constant(
-; FIXED-NEXT: entry:
-; FIXED-NEXT: br label [[VECTOR_PH:%.*]]
-; FIXED: vector.ph:
-; FIXED-NEXT: br label [[VECTOR_BODY:%.*]]
-; FIXED: vector.body:
-; FIXED-NEXT: [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ]
-; FIXED-NEXT: [[TMP1:%.*]] = getelementptr inbounds i64, ptr [[A:%.*]], i64 [[INDEX]]
+; FIXED-LABEL: define void @predicated_udiv_by_constant(
+; FIXED-SAME: ptr noalias captures(none) [[A:%.*]], i64 [[N:%.*]]) #[[ATTR0]] {
+; FIXED-NEXT: [[ENTRY:.*:]]
+; FIXED-NEXT: br label %[[VECTOR_PH:.*]]
+; FIXED: [[VECTOR_PH]]:
+; FIXED-NEXT: br label %[[VECTOR_BODY:.*]]
+; FIXED: [[VECTOR_BODY]]:
+; FIXED-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; FIXED-NEXT: [[TMP1:%.*]] = getelementptr inbounds i64, ptr [[A]], i64 [[INDEX]]
; FIXED-NEXT: [[WIDE_LOAD1:%.*]] = load <4 x i64>, ptr [[TMP1]], align 8
; FIXED-NEXT: [[TMP5:%.*]] = icmp ne <4 x i64> [[WIDE_LOAD1]], splat (i64 42)
; FIXED-NEXT: [[TMP7:%.*]] = udiv <4 x i64> [[WIDE_LOAD1]], splat (i64 27)
@@ -456,10 +470,10 @@ define void @predicated_udiv_by_constant(ptr noalias nocapture %a, i64 %n) {
; FIXED-NEXT: store <4 x i64> [[PREDPHI2]], ptr [[TMP1]], align 8
; FIXED-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 4
; FIXED-NEXT: [[TMP8:%.*]] = icmp eq i64 [[INDEX_NEXT]], 1024
-; FIXED-NEXT: br i1 [[TMP8]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP8:![0-9]+]]
-; FIXED: middle.block:
-; FIXED-NEXT: br label [[LATCH:%.*]]
-; FIXED: for.end:
+; FIXED-NEXT: br i1 [[TMP8]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP8:![0-9]+]]
+; FIXED: [[MIDDLE_BLOCK]]:
+; FIXED-NEXT: br label %[[FOR_END:.*]]
+; FIXED: [[FOR_END]]:
; FIXED-NEXT: ret void
;
entry:
@@ -486,16 +500,17 @@ for.end:
}
define void @predicated_sdiv_by_constant(ptr noalias nocapture %a, i64 %n) {
-; CHECK-LABEL: @predicated_sdiv_by_constant(
-; CHECK-NEXT: entry:
-; CHECK-NEXT: br label [[VECTOR_PH:%.*]]
-; CHECK: vector.ph:
-; CHECK-NEXT: br label [[VECTOR_BODY:%.*]]
-; CHECK: vector.body:
-; CHECK-NEXT: [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ]
-; CHECK-NEXT: [[AVL:%.*]] = phi i64 [ 1024, [[VECTOR_PH]] ], [ [[AVL_NEXT:%.*]], [[VECTOR_BODY]] ]
+; CHECK-LABEL: define void @predicated_sdiv_by_constant(
+; CHECK-SAME: ptr noalias captures(none) [[A:%.*]], i64 [[N:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT: [[ENTRY:.*:]]
+; CHECK-NEXT: br label %[[VECTOR_PH:.*]]
+; CHECK: [[VECTOR_PH]]:
+; CHECK-NEXT: br label %[[VECTOR_BODY:.*]]
+; CHECK: [[VECTOR_BODY]]:
+; CHECK-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-NEXT: [[AVL:%.*]] = phi i64 [ 1024, %[[VECTOR_PH]] ], [ [[AVL_NEXT:%.*]], %[[VECTOR_BODY]] ]
; CHECK-NEXT: [[TMP14:%.*]] = call i32 @llvm.experimental.get.vector.length.i64(i64 [[AVL]], i32 2, i1 true)
-; CHECK-NEXT: [[TMP7:%.*]] = getelementptr inbounds i64, ptr [[A:%.*]], i64 [[INDEX]]
+; CHECK-NEXT: [[TMP7:%.*]] = getelementptr inbounds i64, ptr [[A]], i64 [[INDEX]]
; CHECK-NEXT: [[WIDE_LOAD:%.*]] = call <vscale x 2 x i64> @llvm.vp.load.nxv2i64.p0(ptr align 8 [[TMP7]], <vscale x 2 x i1> splat (i1 true), i32 [[TMP14]])
; CHECK-NEXT: [[TMP2:%.*]] = icmp ne <vscale x 2 x i64> [[WIDE_LOAD]], splat (i64 42)
; CHECK-NEXT: [[TMP10:%.*]] = sdiv <vscale x 2 x i64> [[WIDE_LOAD]], splat (i64 27)
@@ -505,20 +520,21 @@ define void @predicated_sdiv_by_constant(ptr noalias nocapture %a, i64 %n) {
; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[TMP12]], [[INDEX]]
; CHECK-NEXT: [[AVL_NEXT]] = sub nuw i64 [[AVL]], [[TMP12]]
; CHECK-NEXT: [[TMP13:%.*]] = icmp eq i64 [[AVL_NEXT]], 0
-; CHECK-NEXT: br i1 [[TMP13]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP9:![0-9]+]]
-; CHECK: middle.block:
-; CHECK-NEXT: br label [[LATCH:%.*]]
-; CHECK: for.end:
+; CHECK-NEXT: br i1 [[TMP13]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP9:![0-9]+]]
+; CHECK: [[MIDDLE_BLOCK]]:
+; CHECK-NEXT: br label %[[FOR_END:.*]]
+; CHECK: [[FOR_END]]:
; CHECK-NEXT: ret void
;
-; FIXED-LABEL: @predicated_sdiv_by_constant(
-; FIXED-NEXT: entry:
-; FIXED-NEXT: br label [[VECTOR_PH:%.*]]
-; FIXED: vector.ph:
-; FIXED-NEXT: br label [[VECTOR_BODY:%.*]]
-; FIXED: vector.body:
-; FIXED-NEXT: [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ]
-; FIXED-NEXT: [[TMP1:%.*]] = getelementptr inbounds i64, ptr [[A:%.*]], i64 [[INDEX]]
+; FIXED-LABEL: define void @predicated_sdiv_by_constant(
+; FIXED-SAME: ptr noalias captures(none) [[A:%.*]], i64 [[N:%.*]]) #[[ATTR0]] {
+; FIXED-NEXT: [[ENTRY:.*:]]
+; FIXED-NEXT: br label %[[VECTOR_PH:.*]]
+; FIXED: [[VECTOR_PH]]:
+; FIXED-NEXT: br label %[[VECTOR_BODY:.*]]
+; FIXED: [[VECTOR_BODY]]:
+; FIXED-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; FIXED-NEXT: [[TMP1:%.*]] = getelementptr inbounds i64, ptr [[A]], i64 [[INDEX]]
; FIXED-NEXT: [[WIDE_LOAD1:%.*]] = load <4 x i64>, ptr [[TMP1]], align 8
; FIXED-NEXT: [[TMP5:%.*]] = icmp ne <4 x i64> [[WIDE_LOAD1]], splat (i64 42)
; FIXED-NEXT: [[TMP7:%.*]] = sdiv <4 x i64> [[WIDE_LOAD1]], splat (i64 27)
@@ -526,10 +542,10 @@ define void @predicated_sdiv_by_constant(ptr noalias nocapture %a, i64 %n) {
; FIXED-NEXT: store <4 x i64> [[PREDPHI2]], ptr [[TMP1]], align 8
; FIXED-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 4
; FIXED-NEXT: [[TMP8:%.*]] = icmp eq i64 [[INDEX_NEXT]], 1024
-; FIXED-NEXT: br i1 [[TMP8]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP9:![0-9]+]]
-; FIXED: middle.block:
-; FIXED-NEXT: br label [[LATCH:%.*]]
-; FIXED: for.end:
+; FIXED-NEXT: br i1 [[TMP8]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP9:![0-9]+]]
+; FIXED: [[MIDDLE_BLOCK]]:
+; FIXED-NEXT: br label %[[FOR_END:.*]]
+; FIXED: [[FOR_END]]:
; FIXED-NEXT: ret void
;
entry:
@@ -556,16 +572,17 @@ for.end:
}
define void @predicated_sdiv_by_minus_one(ptr noalias nocapture %a, i64 %n) {
-; CHECK-LABEL: @predicated_sdiv_by_minus_one(
-; CHECK-NEXT: entry:
-; CHECK-NEXT: br label [[VECTOR_PH:%.*]]
-; CHECK: vector.ph:
-; CHECK-NEXT: br label [[VECTOR_BODY:%.*]]
-; CHECK: vector.body:
-; CHECK-NEXT: [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ]
-; CHECK-NEXT: [[AVL:%.*]] = phi i64 [ 1024, [[VECTOR_PH]] ], [ [[AVL_NEXT:%.*]], [[VECTOR_BODY]] ]
+; CHECK-LABEL: define void @predicated_sdiv_by_minus_one(
+; CHECK-SAME: ptr noalias captures(none) [[A:%.*]], i64 [[N:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT: [[ENTRY:.*:]]
+; CHECK-NEXT: br label %[[VECTOR_PH:.*]]
+; CHECK: [[VECTOR_PH]]:
+; CHECK-NEXT: br label %[[VECTOR_BODY:.*]]
+; CHECK: [[VECTOR_BODY]]:
+; CHECK-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-NEXT: [[AVL:%.*]] = phi i64 [ 1024, %[[VECTOR_PH]] ], [ [[AVL_NEXT:%.*]], %[[VECTOR_BODY]] ]
; CHECK-NEXT: [[TMP12:%.*]] = call i32 @llvm.experimental.get.vector.length.i64(i64 [[AVL]], i32 16, i1 true)
-; CHECK-NEXT: [[TMP7:%.*]] = getelementptr inbounds i8, ptr [[A:%.*]], i64 [[INDEX]]
+; CHECK-NEXT: [[TMP7:%.*]] = getelementptr inbounds i8, ptr [[A]], i64 [[INDEX]]
; CHECK-NEXT: [[WIDE_LOAD:%.*]] = call <vscale x 16 x i8> @llvm.vp.load.nxv16i8.p0(ptr align 1 [[TMP7]], <vscale x 16 x i1> splat (i1 true), i32 [[TMP12]])
; CHECK-NEXT: [[TMP9:%.*]] = icmp ne <vscale x 16 x i8> [[WIDE_LOAD]], splat (i8 -128)
; CHECK-NEXT: [[TMP11:%.*]] = call <vscale x 16 x i8> @llvm.vp.sdiv.nxv16i8(<vscale x 16 x i8> [[WIDE_LOAD]], <vscale x 16 x i8> splat (i8 -1), <vscale x 16 x i1> [[TMP9]], i32 [[TMP12]])
@@ -575,20 +592,21 @@ define void @predicated_sdiv_by_minus_one(ptr noalias nocapture %a, i64 %n) {
; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[TMP13]], [[INDEX]]
; CHECK-NEXT: [[AVL_NEXT]] = sub nuw i64 [[AVL]], [[TMP13]]
; CHECK-NEXT: [[TMP14:%.*]] = icmp eq i64 [[AVL_NEXT]], 0
-; CHECK-NEXT: br i1 [[TMP14]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP10:![0-9]+]]
-; CHECK: middle.block:
-; CHECK-NEXT: br label [[LATCH:%.*]]
-; CHECK: for.end:
+; CHECK-NEXT: br i1 [[TMP14]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP10:![0-9]+]]
+; CHECK: [[MIDDLE_BLOCK]]:
+; CHECK-NEXT: br label %[[FOR_END:.*]]
+; CHECK: [[FOR_END]]:
; CHECK-NEXT: ret void
;
-; FIXED-LABEL: @predicated_sdiv_by_minus_one(
-; FIXED-NEXT: entry:
-; FIXED-NEXT: br label [[VECTOR_PH:%.*]]
-; FIXED: vector.ph:
-; FIXED-NEXT: br label [[VECTOR_BODY:%.*]]
-; FIXED: vector.body:
-; FIXED-NEXT: [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ]
-; FIXED-NEXT: [[TMP1:%.*]] = getelementptr inbounds i8, ptr [[A:%.*]], i64 [[INDEX]]
+; FIXED-LABEL: define void @predicated_sdiv_by_minus_one(
+; FIXED-SAME: ptr noalias captures(none) [[A:%.*]], i64 [[N:%.*]]) #[[ATTR0]] {
+; FIXED-NEXT: [[ENTRY:.*:]]
+; FIXED-NEXT: br label %[[VECTOR_PH:.*]]
+; FIXED: [[VECTOR_PH]]:
+; FIXED-NEXT: br label %[[VECTOR_BODY:.*]]
+; FIXED: [[VECTOR_BODY]]:
+; FIXED-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; FIXED-NEXT: [[TMP1:%.*]] = getelementptr inbounds i8, ptr [[A]], i64 [[INDEX]]
; FIXED-NEXT: [[WIDE_LOAD1:%.*]] = load <32 x i8>, ptr [[TMP1]], align 1
; FIXED-NEXT: [[TMP5:%.*]] = icmp ne <32 x i8> [[WIDE_LOAD1]], splat (i8 -128)
; FIXED-NEXT: [[TMP9:%.*]] = call <32 x i8> @llvm.masked.sdiv.v32i8(<32 x i8> [[WIDE_LOAD1]], <32 x i8> splat (i8 -1), <32 x i1> [[TMP5]])
@@ -596,10 +614,10 @@ define void @predicated_sdiv_by_minus_one(ptr noalias nocapture %a, i64 %n) {
; FIXED-NEXT: store <32 x i8> [[PREDPHI2]], ptr [[TMP1]], align 1
; FIXED-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 32
; FIXED-NEXT: [[TMP10:%.*]] = icmp eq i64 [[INDEX_NEXT]], 1024
-; FIXED-NEXT: br i1 [[TMP10]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP10:![0-9]+]]
-; FIXED: middle.block:
-; FIXED-NEXT: br label [[LATCH:%.*]]
-; FIXED: for.end:
+; FIXED-NEXT: br i1 [[TMP10]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP10:![0-9]+]]
+; FIXED: [[MIDDLE_BLOCK]]:
+; FIXED-NEXT: br label %[[FOR_END:.*]]
+; FIXED: [[FOR_END]]:
; FIXED-NEXT: ret void
;
entry:
@@ -629,28 +647,29 @@ for.end:
; selects can be introduced outside the vector loop and their cost should not be
; considered for each loop iteration.
define void @udiv_sdiv_with_invariant_divisors(i8 %x, i16 %y, i1 %c, ptr %p) {
-; CHECK-LABEL: @udiv_sdiv_with_invariant_divisors(
-; CHECK-NEXT: entry:
-; CHECK-NEXT: br label [[VECTOR_BODY:%.*]]
-; CHECK: vector.ph:
-; CHECK-NEXT: [[BROADCAST_SPLATINSERT1:%.*]] = insertelement <vscale x 4 x i8> poison, i8 [[X:%.*]], i64 0
+; CHECK-LABEL: define void @udiv_sdiv_with_invariant_divisors(
+; CHECK-SAME: i8 [[X:%.*]], i16 [[Y:%.*]], i1 [[C:%.*]], ptr [[P:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT: [[ENTRY:.*:]]
+; CHECK-NEXT: br label %[[VECTOR_BODY:.*]]
+; CHECK: [[VECTOR_BODY]]:
+; CHECK-NEXT: [[BROADCAST_SPLATINSERT1:%.*]] = insertelement <vscale x 4 x i8> poison, i8 [[X]], i64 0
; CHECK-NEXT: [[BROADCAST_SPLAT2:%.*]] = shufflevector <vscale x 4 x i8> [[BROADCAST_SPLATINSERT1]], <vscale x 4 x i8> poison, <vscale x 4 x i32> zeroinitializer
-; CHECK-NEXT: [[BROADCAST_SPLATINSERT3:%.*]] = insertelement <vscale x 4 x i16> poison, i16 [[Y:%.*]], i64 0
+; CHECK-NEXT: [[BROADCAST_SPLATINSERT3:%.*]] = insertelement <vscale x 4 x i16> poison, i16 [[Y]], i64 0
; CHECK-NEXT: [[BROADCAST_SPLAT4:%.*]] = shufflevector <vscale x 4 x i16> [[BROADCAST_SPLATINSERT3]], <vscale x 4 x i16> poison, <vscale x 4 x i32> zeroinitializer
-; CHECK-NEXT: [[BROADCAST_SPLATINSERT5:%.*]] = insertelement <vscale x 4 x ptr> poison, ptr [[P:%.*]], i64 0
+; CHECK-NEXT: [[BROADCAST_SPLATINSERT5:%.*]] = insertelement <vscale x 4 x ptr> poison, ptr [[P]], i64 0
; CHECK-NEXT: [[BROADCAST_SPLAT6:%.*]] = shufflevector <vscale x 4 x ptr> [[BROADCAST_SPLATINSERT5]], <vscale x 4 x ptr> poison, <vscale x 4 x i32> zeroinitializer
; CHECK-NEXT: [[TMP1:%.*]] = call <vscale x 4 x i8> @llvm.stepvector.nxv4i8()
; CHECK-NEXT: [[INDUCTION:%.*]] = add <vscale x 4 x i8> splat (i8 -12), [[TMP1]]
-; CHECK-NEXT: br label [[LOOP_LATCH:%.*]]
-; CHECK: vector.body:
-; CHECK-NEXT: [[VEC_IND:%.*]] = phi <vscale x 4 x i8> [ [[INDUCTION]], [[VECTOR_BODY]] ], [ [[VEC_IND_NEXT:%.*]], [[LOOP_LATCH8:%.*]] ]
-; CHECK-NEXT: [[AVL:%.*]] = phi i32 [ 12, [[VECTOR_BODY]] ], [ [[AVL_NEXT:%.*]], [[LOOP_LATCH8]] ]
+; CHECK-NEXT: br label %[[LOOP_LATCH:.*]]
+; CHECK: [[LOOP_LATCH]]:
+; CHECK-NEXT: [[VEC_IND:%.*]] = phi <vscale x 4 x i8> [ [[INDUCTION]], %[[VECTOR_BODY]] ], [ [[VEC_IND_NEXT:%.*]], %[[LOOP_LATCH8:.*]] ]
+; CHECK-NEXT: [[AVL:%.*]] = phi i32 [ 12, %[[VECTOR_BODY]] ], [ [[AVL_NEXT:%.*]], %[[LOOP_LATCH8]] ]
; CHECK-NEXT: [[TMP2:%.*]] = call i32 @llvm.experimental.get.vector.length.i32(i32 [[AVL]], i32 4, i1 true)
; CHECK-NEXT: [[TMP3:%.*]] = trunc i32 [[TMP2]] to i8
; CHECK-NEXT: [[BROADCAST_SPLATINSERT7:%.*]] = insertelement <vscale x 4 x i8> poison, i8 [[TMP3]], i64 0
; CHECK-NEXT: [[BROADCAST_SPLAT8:%.*]] = shufflevector <vscale x 4 x i8> [[BROADCAST_SPLATINSERT7]], <vscale x 4 x i8> poison, <vscale x 4 x i32> zeroinitializer
-; CHECK-NEXT: br i1 [[C:%.*]], label [[LOOP_LATCH8]], label [[THEN7:%.*]]
-; CHECK: then7:
+; CHECK-NEXT: br i1 [[C]], label %[[LOOP_LATCH8]], label %[[THEN7:.*]]
+; CHECK: [[THEN7]]:
; CHECK-NEXT: [[TMP13:%.*]] = call <vscale x 4 x i8> @llvm.vp.merge.nxv4i8(<vscale x 4 x i1> splat (i1 true), <vscale x 4 x i8> [[BROADCAST_SPLAT2]], <vscale x 4 x i8> splat (i8 1), i32 [[TMP2]])
; CHECK-NEXT: [[TMP14:%.*]] = extractelement <vscale x 4 x i8> [[VEC_IND]], i64 0
; CHECK-NEXT: [[TMP5:%.*]] = extractelement <vscale x 4 x i8> [[TMP13]], i64 0
@@ -660,9 +679,9 @@ define void @udiv_sdiv_with_invariant_divisors(i8 %x, i16 %y, i1 %c, ptr %p) {
; CHECK-NEXT: [[TMP9:%.*]] = extractelement <vscale x 4 x i16> [[TMP8]], i64 0
; CHECK-NEXT: [[TMP10:%.*]] = sdiv i16 [[TMP7]], [[TMP9]]
; CHECK-NEXT: [[TMP11:%.*]] = sext i16 [[TMP10]] to i32
-; CHECK-NEXT: br label [[LOOP_LATCH8]]
-; CHECK: loop.latch8:
-; CHECK-NEXT: [[TMP12:%.*]] = phi i32 [ poison, [[LOOP_LATCH]] ], [ [[TMP11]], [[THEN7]] ]
+; CHECK-NEXT: br label %[[LOOP_LATCH8]]
+; CHECK: [[LOOP_LATCH8]]:
+; CHECK-NEXT: [[TMP12:%.*]] = phi i32 [ poison, %[[LOOP_LATCH]] ], [ [[TMP11]], %[[THEN7]] ]
; CHECK-NEXT: [[BROADCAST_SPLATINSERT9:%.*]] = insertelement <vscale x 4 x i32> poison, i32 [[TMP12]], i64 0
; CHECK-NEXT: [[BROADCAST_SPLAT10:%.*]] = shufflevector <vscale x 4 x i32> [[BROADCAST_SPLATINSERT9]], <vscale x 4 x i32> poison, <vscale x 4 x i32> zeroinitializer
; CHECK-NEXT: [[PREDPHI:%.*]] = call <vscale x 4 x i32> @llvm.vp.merge.nxv4i32(<vscale x 4 x i1> splat (i1 true), <vscale x 4 x i32> [[BROADCAST_SPLAT10]], <vscale x 4 x i32> zeroinitializer, i32 [[TMP2]])
@@ -670,52 +689,54 @@ define void @udiv_sdiv_with_invariant_divisors(i8 %x, i16 %y, i1 %c, ptr %p) {
; CHECK-NEXT: [[AVL_NEXT]] = sub nuw i32 [[AVL]], [[TMP2]]
; CHECK-NEXT: [[VEC_IND_NEXT]] = add <vscale x 4 x i8> [[VEC_IND]], [[BROADCAST_SPLAT8]]
; CHECK-NEXT: [[TMP4:%.*]] = icmp eq i32 [[AVL_NEXT]], 0
-; CHECK-NEXT: br i1 [[TMP4]], label [[MIDDLE_BLOCK:%.*]], label [[LOOP_LATCH]], !llvm.loop [[LOOP11:![0-9]+]]
-; CHECK: middle.block:
-; CHECK-NEXT: br label [[EXIT:%.*]]
-; CHECK: exit:
+; CHECK-NEXT: br i1 [[TMP4]], label %[[MIDDLE_BLOCK:.*]], label %[[LOOP_LATCH]], !llvm.loop [[LOOP11:![0-9]+]]
+; CHECK: [[MIDDLE_BLOCK]]:
+; CHECK-NEXT: br label %[[EXIT:.*]]
+; CHECK: [[EXIT]]:
; CHECK-NEXT: ret void
;
-; FIXED-LABEL: @udiv_sdiv_with_invariant_divisors(
-; FIXED-NEXT: entry:
-; FIXED-NEXT: br label [[VECTOR_PH:%.*]]
-; FIXED: vector.ph:
-; FIXED-NEXT: br label [[VECTOR_BODY:%.*]]
-; FIXED: vector.body:
-; FIXED-NEXT: br i1 [[C:%.*]], label [[LOOP_LATCH2:%.*]], label [[THEN1:%.*]]
-; FIXED: then1:
-; FIXED-NEXT: [[TMP0:%.*]] = udiv i8 -12, [[X:%.*]]
+; FIXED-LABEL: define void @udiv_sdiv_with_invariant_divisors(
+; FIXED-SAME: i8 [[X:%.*]], i16 [[Y:%.*]], i1 [[C:%.*]], ptr [[P:%.*]]) #[[ATTR0]] {
+; FIXED-NEXT: [[ENTRY:.*:]]
+; FIXED-NEXT: br label %[[VECTOR_PH:.*]]
+; FIXED: [[VECTOR_PH]]:
+; FIXED-NEXT: br label %[[VECTOR_BODY:.*]]
+; FIXED: [[VECTOR_BODY]]:
+; FIXED-NEXT: br i1 [[C]], label %[[LOOP_LATCH2:.*]], label %[[THEN1:.*]]
+; FIXED: [[THEN1]]:
+; FIXED-NEXT: [[TMP0:%.*]] = udiv i8 -12, [[X]]
; FIXED-NEXT: [[TMP1:%.*]] = zext i8 [[TMP0]] to i16
-; FIXED-NEXT: [[TMP2:%.*]] = sdiv i16 [[TMP1]], [[Y:%.*]]
+; FIXED-NEXT: [[TMP2:%.*]] = sdiv i16 [[TMP1]], [[Y]]
; FIXED-NEXT: [[TMP3:%.*]] = sext i16 [[TMP2]] to i32
-; FIXED-NEXT: br label [[LOOP_LATCH2]]
-; FIXED: loop.latch2:
-; FIXED-NEXT: [[TMP4:%.*]] = phi i32 [ poison, [[VECTOR_BODY]] ], [ [[TMP3]], [[THEN1]] ]
-; FIXED-NEXT: [[TMP7:%.*]] = select i1 true, i32 [[TMP4]], i32 0
-; FIXED-NEXT: store i32 [[TMP7]], ptr [[P:%.*]], align 4
-; FIXED-NEXT: br label [[LOOP_LATCH:%.*]]
-; FIXED: middle.block:
-; FIXED-NEXT: br label [[SCALAR_PH:%.*]]
-; FIXED: scalar.ph:
-; FIXED-NEXT: br label [[LOOP_HEADER:%.*]]
-; FIXED: loop.header:
-; FIXED-NEXT: [[IV:%.*]] = phi i16 [ -4, [[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], [[LOOP_LATCH1:%.*]] ]
-; FIXED-NEXT: [[NARROW_IV:%.*]] = phi i8 [ -4, [[SCALAR_PH]] ], [ [[IV_NEXT_TRUNC:%.*]], [[LOOP_LATCH1]] ]
-; FIXED-NEXT: br i1 [[C]], label [[LOOP_LATCH1]], label [[THEN:%.*]]
-; FIXED: then:
+; FIXED-NEXT: br label %[[LOOP_LATCH2]]
+; FIXED: [[LOOP_LATCH2]]:
+; FIXED-NEXT: [[TMP4:%.*]] = phi i32 [ poison, %[[VECTOR_BODY]] ], [ [[TMP3]], %[[THEN1]] ]
+; FIXED-NEXT: [[TMP5:%.*]] = phi i1 [ false, %[[VECTOR_BODY]] ], [ true, %[[THEN1]] ]
+; FIXED-NEXT: [[TMP7:%.*]] = select i1 [[TMP5]], i32 [[TMP4]], i32 0
+; FIXED-NEXT: store i32 [[TMP7]], ptr [[P]], align 4
+; FIXED-NEXT: br label %[[MIDDLE_BLOCK:.*]]
+; FIXED: [[MIDDLE_BLOCK]]:
+; FIXED-NEXT: br label %[[SCALAR_PH:.*]]
+; FIXED: [[SCALAR_PH]]:
+; FIXED-NEXT: br label %[[LOOP_HEADER:.*]]
+; FIXED: [[LOOP_HEADER]]:
+; FIXED-NEXT: [[IV:%.*]] = phi i16 [ -4, %[[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], %[[LOOP_LATCH1:.*]] ]
+; FIXED-NEXT: [[NARROW_IV:%.*]] = phi i8 [ -4, %[[SCALAR_PH]] ], [ [[IV_NEXT_TRUNC:%.*]], %[[LOOP_LATCH1]] ]
+; FIXED-NEXT: br i1 [[C]], label %[[LOOP_LATCH1]], label %[[THEN:.*]]
+; FIXED: [[THEN]]:
; FIXED-NEXT: [[UD:%.*]] = udiv i8 [[NARROW_IV]], [[X]]
; FIXED-NEXT: [[UD_EXT:%.*]] = zext i8 [[UD]] to i16
; FIXED-NEXT: [[SD:%.*]] = sdiv i16 [[UD_EXT]], [[Y]]
; FIXED-NEXT: [[SD_EXT:%.*]] = sext i16 [[SD]] to i32
-; FIXED-NEXT: br label [[LOOP_LATCH1]]
-; FIXED: loop.latch:
-; FIXED-NEXT: [[MERGE:%.*]] = phi i32 [ 0, [[LOOP_HEADER]] ], [ [[SD_EXT]], [[THEN]] ]
+; FIXED-NEXT: br label %[[LOOP_LATCH1]]
+; FIXED: [[LOOP_LATCH1]]:
+; FIXED-NEXT: [[MERGE:%.*]] = phi i32 [ 0, %[[LOOP_HEADER]] ], [ [[SD_EXT]], %[[THEN]] ]
; FIXED-NEXT: store i32 [[MERGE]], ptr [[P]], align 4
; FIXED-NEXT: [[IV_NEXT]] = add nsw i16 [[IV]], 1
; FIXED-NEXT: [[EC:%.*]] = icmp eq i16 [[IV_NEXT]], 0
; FIXED-NEXT: [[IV_NEXT_TRUNC]] = trunc i16 [[IV_NEXT]] to i8
-; FIXED-NEXT: br i1 [[EC]], label [[EXIT:%.*]], label [[LOOP_HEADER]], !llvm.loop [[LOOP11:![0-9]+]]
-; FIXED: exit:
+; FIXED-NEXT: br i1 [[EC]], label %[[EXIT:.*]], label %[[LOOP_HEADER]], !llvm.loop [[LOOP11:![0-9]+]]
+; FIXED: [[EXIT]]:
; FIXED-NEXT: ret void
;
entry:
@@ -744,3 +765,30 @@ loop.latch:
exit:
ret void
}
+;.
+; CHECK: [[LOOP0]] = distinct !{[[LOOP0]], [[META1:![0-9]+]], [[META2:![0-9]+]]}
+; CHECK: [[META1]] = !{!"llvm.loop.isvectorized", i32 1}
+; CHECK: [[META2]] = !{!"llvm.loop.unroll.runtime.disable"}
+; CHECK: [[LOOP3]] = distinct !{[[LOOP3]], [[META1]], [[META2]]}
+; CHECK: [[LOOP4]] = distinct !{[[LOOP4]], [[META1]], [[META2]]}
+; CHECK: [[LOOP5]] = distinct !{[[LOOP5]], [[META1]], [[META2]]}
+; CHECK: [[LOOP6]] = distinct !{[[LOOP6]], [[META1]], [[META2]]}
+; CHECK: [[LOOP7]] = distinct !{[[LOOP7]], [[META1]], [[META2]]}
+; CHECK: [[LOOP8]] = distinct !{[[LOOP8]], [[META1]], [[META2]]}
+; CHECK: [[LOOP9]] = distinct !{[[LOOP9]], [[META1]], [[META2]]}
+; CHECK: [[LOOP10]] = distinct !{[[LOOP10]], [[META1]], [[META2]]}
+; CHECK: [[LOOP11]] = distinct !{[[LOOP11]], [[META1]], [[META2]]}
+;.
+; FIXED: [[LOOP0]] = distinct !{[[LOOP0]], [[META1:![0-9]+]], [[META2:![0-9]+]]}
+; FIXED: [[META1]] = !{!"llvm.loop.isvectorized", i32 1}
+; FIXED: [[META2]] = !{!"llvm.loop.unroll.runtime.disable"}
+; FIXED: [[LOOP3]] = distinct !{[[LOOP3]], [[META1]], [[META2]]}
+; FIXED: [[LOOP4]] = distinct !{[[LOOP4]], [[META1]], [[META2]]}
+; FIXED: [[LOOP5]] = distinct !{[[LOOP5]], [[META1]], [[META2]]}
+; FIXED: [[LOOP6]] = distinct !{[[LOOP6]], [[META1]], [[META2]]}
+; FIXED: [[LOOP7]] = distinct !{[[LOOP7]], [[META1]], [[META2]]}
+; FIXED: [[LOOP8]] = distinct !{[[LOOP8]], [[META1]], [[META2]]}
+; FIXED: [[LOOP9]] = distinct !{[[LOOP9]], [[META1]], [[META2]]}
+; FIXED: [[LOOP10]] = distinct !{[[LOOP10]], [[META1]], [[META2]]}
+; FIXED: [[LOOP11]] = distinct !{[[LOOP11]], [[META2]], [[META1]]}
+;.
diff --git a/llvm/test/Transforms/LoopVectorize/RISCV/tail-folding-complex-mask.ll b/llvm/test/Transforms/LoopVectorize/RISCV/tail-folding-complex-mask.ll
index 303193a29d99f..4065943ab42fb 100644
--- a/llvm/test/Transforms/LoopVectorize/RISCV/tail-folding-complex-mask.ll
+++ b/llvm/test/Transforms/LoopVectorize/RISCV/tail-folding-complex-mask.ll
@@ -98,9 +98,10 @@ define void @test(i64 %n, ptr noalias %src0, ptr noalias %src1, ptr noalias %src
; NO-VP-NEXT: br label %[[LATCH6]]
; NO-VP: [[LATCH6]]:
; NO-VP-NEXT: [[TMP20:%.*]] = phi i32 [ poison, %[[VECTOR_BODY]] ], [ [[TMP17]], %[[LOAD_V25]] ]
+; NO-VP-NEXT: [[TMP21:%.*]] = phi i1 [ false, %[[VECTOR_BODY]] ], [ true, %[[LOAD_V25]] ]
; NO-VP-NEXT: [[BROADCAST_SPLATINSERT7:%.*]] = insertelement <vscale x 4 x i32> poison, i32 [[TMP20]], i64 0
; NO-VP-NEXT: [[BROADCAST_SPLAT8:%.*]] = shufflevector <vscale x 4 x i32> [[BROADCAST_SPLATINSERT7]], <vscale x 4 x i32> poison, <vscale x 4 x i32> zeroinitializer
-; NO-VP-NEXT: [[PREDPHI8:%.*]] = select i1 true, <vscale x 4 x i32> [[BROADCAST_SPLAT8]], <vscale x 4 x i32> [[PREDPHI6]]
+; NO-VP-NEXT: [[PREDPHI8:%.*]] = select i1 [[TMP21]], <vscale x 4 x i32> [[BROADCAST_SPLAT8]], <vscale x 4 x i32> [[PREDPHI6]]
; NO-VP-NEXT: [[TMP18:%.*]] = getelementptr inbounds i32, ptr [[DST]], i64 [[INDEX]]
; NO-VP-NEXT: store <vscale x 4 x i32> [[PREDPHI8]], ptr [[TMP18]], align 4
; NO-VP-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], [[TMP1]]
diff --git a/llvm/test/Transforms/LoopVectorize/VPlan/dissolve-replicate-regions.ll b/llvm/test/Transforms/LoopVectorize/VPlan/dissolve-replicate-regions.ll
index f7a53e57a127a..05a25883f2f98 100644
--- a/llvm/test/Transforms/LoopVectorize/VPlan/dissolve-replicate-regions.ll
+++ b/llvm/test/Transforms/LoopVectorize/VPlan/dissolve-replicate-regions.ll
@@ -44,8 +44,10 @@ define void @predicated_load(i1 %c, ptr %ptr, ptr %dst) {
; SCALAR-NEXT: loop.latch:
; SCALAR-NEXT: EMIT-SCALAR vp<[[VP8:%[0-9]+]]> = phi [ ir<poison>, vector.body ], [ ir<%lv>, if.then ]
; SCALAR-NEXT: EMIT-SCALAR vp<[[VP9:%[0-9]+]]> = phi [ ir<poison>, vector.body ], [ ir<%lv>.1, if.then ]
-; SCALAR-NEXT: BLEND ir<%pred.val> = ir<0> vp<%8>/ir<true>
-; SCALAR-NEXT: BLEND ir<%pred.val>.1 = ir<0> vp<%9>/ir<true>
+; SCALAR-NEXT: EMIT-SCALAR vp<[[VP10:%[0-9]+]]> = phi [ ir<false>, vector.body ], [ ir<true>, if.then ]
+; SCALAR-NEXT: EMIT-SCALAR vp<[[VP11:%[0-9]+]]> = phi [ ir<false>, vector.body ], [ ir<true>, if.then ]
+; SCALAR-NEXT: BLEND ir<%pred.val> = ir<0> vp<%8>/vp<[[VP10]]>
+; SCALAR-NEXT: BLEND ir<%pred.val>.1 = ir<0> vp<%9>/vp<[[VP11]]>
; SCALAR-NEXT: CLONE ir<%gep.dst> = getelementptr ir<%dst>, vp<[[VP6]]>
; SCALAR-NEXT: CLONE ir<%gep.dst>.1 = getelementptr ir<%dst>, vp<[[VP7]]>
; SCALAR-NEXT: CLONE store ir<%pred.val>, ir<%gep.dst>
@@ -115,14 +117,16 @@ define void @predicated_load(i1 %c, ptr %ptr, ptr %dst) {
; VECTOR-NEXT: loop.latch:
; VECTOR-NEXT: EMIT-SCALAR vp<[[VP10:%[0-9]+]]> = phi [ ir<poison>, vector.body ], [ ir<%lv>, if.then ]
; VECTOR-NEXT: EMIT-SCALAR vp<[[VP11:%[0-9]+]]> = phi [ ir<poison>, vector.body ], [ ir<%lv>.1, if.then ]
-; VECTOR-NEXT: BLEND ir<%pred.val> = ir<0> vp<%10>/ir<true>
-; VECTOR-NEXT: BLEND ir<%pred.val>.1 = ir<0> vp<%11>/ir<true>
+; VECTOR-NEXT: EMIT-SCALAR vp<[[VP12:%[0-9]+]]> = phi [ ir<false>, vector.body ], [ ir<true>, if.then ]
+; VECTOR-NEXT: EMIT-SCALAR vp<[[VP13:%[0-9]+]]> = phi [ ir<false>, vector.body ], [ ir<true>, if.then ]
+; VECTOR-NEXT: BLEND ir<%pred.val> = ir<0> vp<%10>/vp<[[VP12]]>
+; VECTOR-NEXT: BLEND ir<%pred.val>.1 = ir<0> vp<%11>/vp<[[VP13]]>
; VECTOR-NEXT: CLONE ir<%gep.dst> = getelementptr ir<%dst>, vp<[[VP6]]>
-; VECTOR-NEXT: EMIT vp<[[VP12:%[0-9]+]]> = mul nuw nsw vp<[[VP0]]>, ir<1>
-; VECTOR-NEXT: vp<[[VP13:%[0-9]+]]> = vector-pointer i8, ir<%gep.dst>, ir<1>
-; VECTOR-NEXT: vp<[[VP14:%[0-9]+]]> = vector-pointer i8, ir<%gep.dst>, ir<1>, vp<[[VP12]]>
-; VECTOR-NEXT: WIDEN store vp<[[VP13]]>, ir<%pred.val>
-; VECTOR-NEXT: WIDEN store vp<[[VP14]]>, ir<%pred.val>.1
+; VECTOR-NEXT: EMIT vp<[[VP14:%[0-9]+]]> = mul nuw nsw vp<[[VP0]]>, ir<1>
+; VECTOR-NEXT: vp<[[VP15:%[0-9]+]]> = vector-pointer i8, ir<%gep.dst>, ir<1>
+; VECTOR-NEXT: vp<[[VP16:%[0-9]+]]> = vector-pointer i8, ir<%gep.dst>, ir<1>, vp<[[VP14]]>
+; VECTOR-NEXT: WIDEN store vp<[[VP15]]>, ir<%pred.val>
+; VECTOR-NEXT: WIDEN store vp<[[VP16]]>, ir<%pred.val>.1
; VECTOR-NEXT: EMIT vp<%index.next> = add nuw vp<[[VP5]]>, vp<[[VP1]]>
; VECTOR-NEXT: EMIT branch-on-count vp<%index.next>, vp<[[VP2]]>
; VECTOR-NEXT: No successors
diff --git a/llvm/test/Transforms/LoopVectorize/VPlan/phi-with-fastflags-vplan.ll b/llvm/test/Transforms/LoopVectorize/VPlan/phi-with-fastflags-vplan.ll
index d618de9f05680..b127c8376df00 100644
--- a/llvm/test/Transforms/LoopVectorize/VPlan/phi-with-fastflags-vplan.ll
+++ b/llvm/test/Transforms/LoopVectorize/VPlan/phi-with-fastflags-vplan.ll
@@ -30,9 +30,10 @@ define void @f(ptr noalias %p, i1 %c) {
; CHECK-NEXT: Successor(s): latch
; CHECK-EMPTY:
; CHECK-NEXT: latch:
-; CHECK-NEXT: BLEND ir<%phi> = fast ir<%x> ir<0.000000e+00>/ir<true>
-; CHECK-NEXT: vp<[[VP6:%[0-9]+]]> = vector-pointer float, ir<%gep>, ir<1>
-; CHECK-NEXT: WIDEN store vp<[[VP6]]>, ir<%phi>
+; CHECK-NEXT: EMIT-SCALAR vp<[[VP6:%[0-9]+]]> = phi [ ir<false>, vector.body ], [ ir<true>, then ]
+; CHECK-NEXT: BLEND ir<%phi> = fast ir<%x> ir<0.000000e+00>/vp<[[VP6]]>
+; CHECK-NEXT: vp<[[VP7:%[0-9]+]]> = vector-pointer float, ir<%gep>, ir<1>
+; CHECK-NEXT: WIDEN store vp<[[VP7]]>, ir<%phi>
; CHECK-NEXT: EMIT vp<%index.next> = add nuw vp<[[VP3]]>, vp<[[VP1]]>
; CHECK-NEXT: EMIT branch-on-count vp<%index.next>, vp<[[VP2]]>
; CHECK-NEXT: No successors
diff --git a/llvm/test/Transforms/LoopVectorize/VPlan/predicator.ll b/llvm/test/Transforms/LoopVectorize/VPlan/predicator.ll
index e6e10623f6822..b594c018c0800 100644
--- a/llvm/test/Transforms/LoopVectorize/VPlan/predicator.ll
+++ b/llvm/test/Transforms/LoopVectorize/VPlan/predicator.ll
@@ -589,11 +589,12 @@ define void @blend_masks_triangle_phi(ptr noalias %p, i1 %c0, i1 %c1) {
; CHECK-NEXT: Successor(s): bb2
; CHECK-EMPTY:
; CHECK-NEXT: bb2:
-; CHECK-NEXT: EMIT vp<[[VP4:%[0-9]+]]> = not ir<%c1>
+; CHECK-NEXT: EMIT-SCALAR vp<[[VP4:%[0-9]+]]> = phi [ ir<false>, vector.body ], [ ir<true>, bb1 ]
+; CHECK-NEXT: EMIT vp<[[VP5:%[0-9]+]]> = not ir<%c1>
; CHECK-NEXT: Successor(s): bb3
; CHECK-EMPTY:
; CHECK-NEXT: bb3:
-; CHECK-NEXT: BLEND ir<%phi> = ir<0>/ir<true> ir<1>/ir<true>
+; CHECK-NEXT: BLEND ir<%phi> = ir<0>/vp<[[VP4]]> ir<1>/ir<true>
; CHECK-NEXT: EMIT ir<%gep> = getelementptr ir<%p>, ir<%iv>
; CHECK-NEXT: EMIT store ir<%phi>, ir<%gep>
; CHECK-NEXT: EMIT ir<%iv.next> = add ir<%iv>, ir<1>
@@ -760,7 +761,9 @@ define void @simplifiable_blend(i1 %c1, i1 %c2, i1 %c3, i32 %x, i32 %y, ptr %p)
; CHECK-NEXT: Successor(s): latch
; CHECK-EMPTY:
; CHECK-NEXT: latch:
-; CHECK-NEXT: BLEND ir<%phi> = ir<%y>/ir<true> ir<%x>/ir<true>
+; CHECK-NEXT: EMIT-SCALAR vp<[[VP5:%[0-9]+]]> = phi [ ir<true>, F ], [ ir<false>, C ]
+; CHECK-NEXT: EMIT-SCALAR vp<[[VP6:%[0-9]+]]> = phi [ ir<false>, F ], [ ir<true>, C ]
+; CHECK-NEXT: BLEND ir<%phi> = ir<%y>/vp<[[VP5]]> ir<%x>/vp<[[VP6]]>
; CHECK-NEXT: EMIT ir<%gep> = getelementptr ir<%p>, ir<%iv>
; CHECK-NEXT: EMIT store ir<%phi>, ir<%gep>
; CHECK-NEXT: EMIT ir<%iv.next> = add ir<%iv>, ir<1>
@@ -972,7 +975,8 @@ define void @outermost_uniform_branch(ptr %a, i1 %u0) {
; CHECK-EMPTY:
; CHECK-NEXT: bb4:
; CHECK-NEXT: EMIT-SCALAR vp<[[VP4:%[0-9]+]]> = phi [ ir<poison>, vector.body ], [ ir<%add3>, bb3 ]
-; CHECK-NEXT: BLEND ir<%phi4> = ir<%iv>/ir<true> vp<%4>/ir<true>
+; CHECK-NEXT: EMIT-SCALAR vp<[[VP5:%[0-9]+]]> = phi [ ir<false>, vector.body ], [ ir<true>, bb3 ]
+; CHECK-NEXT: BLEND ir<%phi4> = ir<%iv>/ir<true> vp<%4>/vp<[[VP5]]>
; CHECK-NEXT: EMIT store ir<%phi4>, ir<%a>
; CHECK-NEXT: EMIT ir<%iv.next> = add nuw nsw ir<%iv>, ir<1>
; CHECK-NEXT: EMIT ir<%ec> = icmp eq ir<%iv.next>, ir<128>
@@ -1133,8 +1137,10 @@ define void @outermost_uniform_branch_more_blocks(ptr %a, i1 %u0) {
; CHECK-EMPTY:
; CHECK-NEXT: bb6:
; CHECK-NEXT: EMIT-SCALAR vp<[[VP5:%[0-9]+]]> = phi [ ir<%add1>, bb1 ], [ ir<poison>, bb5 ]
-; CHECK-NEXT: EMIT-SCALAR vp<[[VP6:%[0-9]+]]> = phi [ ir<poison>, bb1 ], [ ir<%add5>, bb5 ]
-; CHECK-NEXT: BLEND ir<%phi6> = vp<%5>/ir<true> vp<%6>/ir<true>
+; CHECK-NEXT: EMIT-SCALAR vp<[[VP6:%[0-9]+]]> = phi [ ir<true>, bb1 ], [ ir<false>, bb5 ]
+; CHECK-NEXT: EMIT-SCALAR vp<[[VP7:%[0-9]+]]> = phi [ ir<poison>, bb1 ], [ ir<%add5>, bb5 ]
+; CHECK-NEXT: EMIT-SCALAR vp<[[VP8:%[0-9]+]]> = phi [ ir<false>, bb1 ], [ ir<true>, bb5 ]
+; CHECK-NEXT: BLEND ir<%phi6> = vp<%5>/vp<[[VP6]]> vp<%7>/vp<[[VP8]]>
; CHECK-NEXT: EMIT store ir<%phi6>, ir<%a>
; CHECK-NEXT: EMIT ir<%iv.next> = add nuw nsw ir<%iv>, ir<1>
; CHECK-NEXT: EMIT ir<%ec> = icmp eq ir<%iv.next>, ir<128>
@@ -2743,10 +2749,12 @@ define void @uniform_branch_shared_join_with_varying(ptr %a, i1 %u0) {
; CHECK-EMPTY:
; CHECK-NEXT: bb4:
; CHECK-NEXT: EMIT-SCALAR vp<[[VP4:%[0-9]+]]> = phi [ ir<%add1>, bb1 ], [ ir<poison>, bb3 ]
-; CHECK-NEXT: EMIT-SCALAR vp<[[VP5:%[0-9]+]]> = phi [ ir<poison>, bb1 ], [ ir<%add2>, bb3 ]
-; CHECK-NEXT: EMIT-SCALAR vp<[[VP6:%[0-9]+]]> = phi [ ir<poison>, bb1 ], [ ir<%add3>, bb3 ]
-; CHECK-NEXT: EMIT-SCALAR vp<[[VP7:%[0-9]+]]> = phi [ ir<false>, bb1 ], [ ir<%v2>, bb3 ]
-; CHECK-NEXT: BLEND ir<%phi4> = vp<%4>/ir<true> vp<%5>/ir<true> vp<%6>/vp<[[VP7]]>
+; CHECK-NEXT: EMIT-SCALAR vp<[[VP5:%[0-9]+]]> = phi [ ir<true>, bb1 ], [ ir<false>, bb3 ]
+; CHECK-NEXT: EMIT-SCALAR vp<[[VP6:%[0-9]+]]> = phi [ ir<poison>, bb1 ], [ ir<%add2>, bb3 ]
+; CHECK-NEXT: EMIT-SCALAR vp<[[VP7:%[0-9]+]]> = phi [ ir<false>, bb1 ], [ ir<true>, bb3 ]
+; CHECK-NEXT: EMIT-SCALAR vp<[[VP8:%[0-9]+]]> = phi [ ir<poison>, bb1 ], [ ir<%add3>, bb3 ]
+; CHECK-NEXT: EMIT-SCALAR vp<[[VP9:%[0-9]+]]> = phi [ ir<false>, bb1 ], [ ir<%v2>, bb3 ]
+; CHECK-NEXT: BLEND ir<%phi4> = vp<%4>/vp<[[VP5]]> vp<%6>/vp<[[VP7]]> vp<%8>/vp<[[VP9]]>
; CHECK-NEXT: EMIT store ir<%phi4>, ir<%a>
; CHECK-NEXT: EMIT ir<%iv.next> = add nuw nsw ir<%iv>, ir<1>
; CHECK-NEXT: EMIT ir<%ec> = icmp eq ir<%iv.next>, ir<128>
diff --git a/llvm/test/Transforms/LoopVectorize/X86/pr141968-instsimplifyfolder.ll b/llvm/test/Transforms/LoopVectorize/X86/pr141968-instsimplifyfolder.ll
index d3784afeb1f47..e41f8aa420f66 100644
--- a/llvm/test/Transforms/LoopVectorize/X86/pr141968-instsimplifyfolder.ll
+++ b/llvm/test/Transforms/LoopVectorize/X86/pr141968-instsimplifyfolder.ll
@@ -6,24 +6,30 @@ target triple = "x86_64"
define void @pr141968(i1 %cond, i8 %v, ptr %p) {
; CHECK-LABEL: define void @pr141968(
; CHECK-SAME: i1 [[COND:%.*]], i8 [[V:%.*]], ptr [[P:%.*]]) {
-; CHECK-NEXT: [[ENTRY:.*]]:
+; CHECK-NEXT: [[ENTRY:.*:]]
; CHECK-NEXT: [[ZEXT_TRUE:%.*]] = zext i1 true to i16
; CHECK-NEXT: [[SEXT:%.*]] = sext i8 [[V]] to i16
; CHECK-NEXT: br label %[[PRED_SDIV_CONTINUE28:.*]]
; CHECK: [[PRED_SDIV_CONTINUE28]]:
-; CHECK-NEXT: [[IV:%.*]] = phi i8 [ [[IV_NEXT:%.*]], %[[PRED_SDIV_CONTINUE30:.*]] ], [ 0, %[[ENTRY]] ]
+; CHECK-NEXT: [[BROADCAST_SPLATINSERT:%.*]] = insertelement <16 x i8> poison, i8 [[V]], i64 0
+; CHECK-NEXT: [[BROADCAST_SPLAT:%.*]] = shufflevector <16 x i8> [[BROADCAST_SPLATINSERT]], <16 x i8> poison, <16 x i32> zeroinitializer
+; CHECK-NEXT: [[TMP0:%.*]] = extractelement <16 x i8> [[BROADCAST_SPLAT]], i64 0
+; CHECK-NEXT: br label %[[VECTOR_BODY:.*]]
+; CHECK: [[VECTOR_BODY]]:
+; CHECK-NEXT: [[INDEX:%.*]] = phi i32 [ 0, %[[PRED_SDIV_CONTINUE28]] ], [ [[INDEX_NEXT:%.*]], %[[PRED_SDIV_CONTINUE30:.*]] ]
; CHECK-NEXT: br i1 [[COND]], label %[[PRED_SDIV_CONTINUE30]], label %[[PRED_SDIV_IF29:.*]]
; CHECK: [[PRED_SDIV_IF29]]:
-; CHECK-NEXT: [[SDIV:%.*]] = sdiv i16 [[SEXT]], [[ZEXT_TRUE]]
-; CHECK-NEXT: [[SDIV_TRUNC:%.*]] = trunc i16 [[SDIV]] to i8
; CHECK-NEXT: br label %[[PRED_SDIV_CONTINUE30]]
; CHECK: [[PRED_SDIV_CONTINUE30]]:
-; CHECK-NEXT: [[PREDPHI:%.*]] = phi i8 [ [[SDIV_TRUNC]], %[[PRED_SDIV_IF29]] ], [ 0, %[[PRED_SDIV_CONTINUE28]] ]
+; CHECK-NEXT: [[TMP1:%.*]] = phi i1 [ false, %[[VECTOR_BODY]] ], [ true, %[[PRED_SDIV_IF29]] ]
+; CHECK-NEXT: [[PREDPHI:%.*]] = select i1 [[TMP1]], i8 [[TMP0]], i8 0
; CHECK-NEXT: store i8 [[PREDPHI]], ptr [[P]], align 1
-; CHECK-NEXT: [[IV_NEXT]] = add i8 [[IV]], 1
-; CHECK-NEXT: [[EXITCOND:%.*]] = icmp eq i8 [[IV_NEXT]], 0
-; CHECK-NEXT: br i1 [[EXITCOND]], label %[[EXIT1:.*]], label %[[PRED_SDIV_CONTINUE28]]
+; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i32 [[INDEX]], 32
+; CHECK-NEXT: [[TMP2:%.*]] = icmp eq i32 [[INDEX_NEXT]], 256
+; CHECK-NEXT: br i1 [[TMP2]], label %[[EXIT1:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP0:![0-9]+]]
; CHECK: [[EXIT1]]:
+; CHECK-NEXT: br label %[[EXIT:.*]]
+; CHECK: [[EXIT]]:
; CHECK-NEXT: ret void
;
entry:
diff --git a/llvm/test/Transforms/LoopVectorize/X86/predicated-instruction-cost.ll b/llvm/test/Transforms/LoopVectorize/X86/predicated-instruction-cost.ll
index f871a5f7379b6..201168b20c859 100644
--- a/llvm/test/Transforms/LoopVectorize/X86/predicated-instruction-cost.ll
+++ b/llvm/test/Transforms/LoopVectorize/X86/predicated-instruction-cost.ll
@@ -71,9 +71,10 @@ define void @test_wide_shift_uses_predicated_invariant_instruction(i32 %d, i1 %c
; CHECK-NEXT: br label %[[LOOP_LATCH]]
; CHECK: [[LOOP_LATCH]]:
; CHECK-NEXT: [[TMP3:%.*]] = phi i32 [ poison, %[[VECTOR_BODY]] ], [ [[TMP2]], %[[ELSE]] ]
+; CHECK-NEXT: [[TMP7:%.*]] = phi i1 [ false, %[[VECTOR_BODY]] ], [ true, %[[ELSE]] ]
; CHECK-NEXT: [[BROADCAST_SPLATINSERT:%.*]] = insertelement <4 x i32> poison, i32 [[TMP3]], i64 0
; CHECK-NEXT: [[BROADCAST_SPLAT:%.*]] = shufflevector <4 x i32> [[BROADCAST_SPLATINSERT]], <4 x i32> poison, <4 x i32> zeroinitializer
-; CHECK-NEXT: [[PREDPHI:%.*]] = select i1 true, <4 x i32> [[BROADCAST_SPLAT]], <4 x i32> zeroinitializer
+; CHECK-NEXT: [[PREDPHI:%.*]] = select i1 [[TMP7]], <4 x i32> [[BROADCAST_SPLAT]], <4 x i32> zeroinitializer
; CHECK-NEXT: [[TMP4:%.*]] = getelementptr inbounds i32, ptr [[DST]], i32 [[INDEX]]
; CHECK-NEXT: [[TMP5:%.*]] = getelementptr inbounds i32, ptr [[TMP4]], i64 4
; CHECK-NEXT: store <4 x i32> [[PREDPHI]], ptr [[TMP4]], align 4
diff --git a/llvm/test/Transforms/LoopVectorize/X86/predicated-replicate-feeding-cast.ll b/llvm/test/Transforms/LoopVectorize/X86/predicated-replicate-feeding-cast.ll
index 05d69eb87b0bd..4b387917b1cf6 100644
--- a/llvm/test/Transforms/LoopVectorize/X86/predicated-replicate-feeding-cast.ll
+++ b/llvm/test/Transforms/LoopVectorize/X86/predicated-replicate-feeding-cast.ll
@@ -23,20 +23,21 @@ define i8 @predicated_replicate_feeding_cast(i16 %n, i1 %c1, i1 %c2, i16 %a, i8
; CHECK: [[PRED_SDIV_IF1]]:
; CHECK-NEXT: br label %[[PRED_SDIV_CONTINUE2]]
; CHECK: [[PRED_SDIV_CONTINUE2]]:
-; CHECK-NEXT: [[TMP6:%.*]] = phi i16 [ poison, %[[PRED_SDIV_CONTINUE]] ], [ [[TMP5]], %[[PRED_SDIV_IF1]] ]
-; CHECK-NEXT: [[TMP11:%.*]] = select i1 true, i16 [[TMP6]], i16 0
+; CHECK-NEXT: [[TMP7:%.*]] = phi i16 [ poison, %[[PRED_SDIV_CONTINUE]] ], [ [[TMP5]], %[[PRED_SDIV_IF1]] ]
+; CHECK-NEXT: [[TMP6:%.*]] = phi i1 [ false, %[[PRED_SDIV_CONTINUE]] ], [ true, %[[PRED_SDIV_IF1]] ]
+; CHECK-NEXT: [[TMP11:%.*]] = select i1 [[TMP6]], i16 [[TMP7]], i16 0
; CHECK-NEXT: br i1 [[C2]], label %[[PRED_SDIV_CONTINUE6]], label %[[IF23:.*]]
; CHECK: [[IF23]]:
; CHECK-NEXT: [[TMP12:%.*]] = trunc i16 [[TMP11]] to i8
; CHECK-NEXT: [[TMP18:%.*]] = sdiv i8 [[TMP12]], [[B]]
; CHECK-NEXT: br label %[[PRED_SDIV_CONTINUE6]]
; CHECK: [[PRED_SDIV_CONTINUE6]]:
-; CHECK-NEXT: [[TMP8:%.*]] = phi i8 [ poison, %[[PRED_SDIV_CONTINUE2]] ], [ [[TMP18]], %[[IF23]] ]
+; CHECK-NEXT: [[TMP9:%.*]] = phi i8 [ poison, %[[PRED_SDIV_CONTINUE2]] ], [ [[TMP18]], %[[IF23]] ]
; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i32 [[INDEX]], 2
; CHECK-NEXT: [[TMP21:%.*]] = icmp eq i32 [[INDEX_NEXT]], [[N_VEC]]
; CHECK-NEXT: br i1 [[TMP21]], label %[[MIDDLE_BLOCK:.*]], label %[[PRED_SDIV_CONTINUE]], !llvm.loop [[LOOP0:![0-9]+]]
; CHECK: [[MIDDLE_BLOCK]]:
-; CHECK-NEXT: [[PREDPHI5:%.*]] = select i1 true, i8 [[TMP8]], i8 0
+; CHECK-NEXT: [[PREDPHI5:%.*]] = select i1 [[TMP6]], i8 [[TMP9]], i8 0
; CHECK-NEXT: [[CMP_N:%.*]] = icmp eq i32 [[TMP1]], [[N_VEC]]
; CHECK-NEXT: br i1 [[CMP_N]], label %[[EXIT:.*]], label %[[SCALAR_PH]]
; CHECK: [[SCALAR_PH]]:
@@ -115,20 +116,21 @@ define i8 @predicated_replicate_feeding_cast_non_uniform(i64 %n, i1 %c1, i1 %c2,
; CHECK-NEXT: [[TMP5:%.*]] = sdiv i16 1, [[TMP4]]
; CHECK-NEXT: br label %[[PRED_SDIV_CONTINUE4]]
; CHECK: [[PRED_SDIV_CONTINUE4]]:
-; CHECK-NEXT: [[TMP6:%.*]] = phi i16 [ poison, %[[VECTOR_BODY]] ], [ [[TMP5]], %[[PRED_SDIV_IF]] ]
-; CHECK-NEXT: [[TMP17:%.*]] = select i1 true, i16 [[TMP6]], i16 0
+; CHECK-NEXT: [[TMP7:%.*]] = phi i16 [ poison, %[[VECTOR_BODY]] ], [ [[TMP5]], %[[PRED_SDIV_IF]] ]
+; CHECK-NEXT: [[TMP6:%.*]] = phi i1 [ false, %[[VECTOR_BODY]] ], [ true, %[[PRED_SDIV_IF]] ]
+; CHECK-NEXT: [[TMP17:%.*]] = select i1 [[TMP6]], i16 [[TMP7]], i16 0
; CHECK-NEXT: br i1 [[C2]], label %[[PRED_SDIV_CONTINUE6]], label %[[PRED_SDIV_IF5:.*]]
; CHECK: [[PRED_SDIV_IF5]]:
; CHECK-NEXT: [[TMP18:%.*]] = trunc i16 [[TMP17]] to i8
; CHECK-NEXT: [[TMP19:%.*]] = sdiv i8 [[TMP18]], [[B]]
; CHECK-NEXT: br label %[[PRED_SDIV_CONTINUE6]]
; CHECK: [[PRED_SDIV_CONTINUE6]]:
-; CHECK-NEXT: [[TMP8:%.*]] = phi i8 [ poison, %[[PRED_SDIV_CONTINUE4]] ], [ [[TMP19]], %[[PRED_SDIV_IF5]] ]
+; CHECK-NEXT: [[TMP9:%.*]] = phi i8 [ poison, %[[PRED_SDIV_CONTINUE4]] ], [ [[TMP19]], %[[PRED_SDIV_IF5]] ]
; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 2
; CHECK-NEXT: [[TMP22:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
; CHECK-NEXT: br i1 [[TMP22]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP4:![0-9]+]]
; CHECK: [[MIDDLE_BLOCK]]:
-; CHECK-NEXT: [[PREDPHI5:%.*]] = select i1 true, i8 [[TMP8]], i8 0
+; CHECK-NEXT: [[PREDPHI5:%.*]] = select i1 [[TMP6]], i8 [[TMP9]], i8 0
; CHECK-NEXT: [[CMP_N:%.*]] = icmp eq i64 [[TMP0]], [[N_VEC]]
; CHECK-NEXT: br i1 [[CMP_N]], label %[[EXIT:.*]], label %[[SCALAR_PH]]
; CHECK: [[SCALAR_PH]]:
diff --git a/llvm/test/Transforms/LoopVectorize/X86/replicate-recipe-with-only-first-lane-used.ll b/llvm/test/Transforms/LoopVectorize/X86/replicate-recipe-with-only-first-lane-used.ll
index bbcf52b05733e..864cc9fa56394 100644
--- a/llvm/test/Transforms/LoopVectorize/X86/replicate-recipe-with-only-first-lane-used.ll
+++ b/llvm/test/Transforms/LoopVectorize/X86/replicate-recipe-with-only-first-lane-used.ll
@@ -18,7 +18,8 @@ define void @replicate_udiv_with_only_first_lane_used(i32 %x, ptr %dst, i64 %d)
; CHECK-NEXT: br label %[[LOOP_LATCH1]]
; CHECK: [[LOOP_LATCH1]]:
; CHECK-NEXT: [[TMP0:%.*]] = phi i64 [ poison, %[[VECTOR_BODY]] ]
-; CHECK-NEXT: [[PREDPHI:%.*]] = select i1 true, i64 [[TMP0]], i64 0
+; CHECK-NEXT: [[TMP2:%.*]] = phi i1 [ false, %[[VECTOR_BODY]] ]
+; CHECK-NEXT: [[PREDPHI:%.*]] = select i1 [[TMP2]], i64 [[TMP0]], i64 0
; CHECK-NEXT: [[TMP1:%.*]] = getelementptr i16, ptr [[DST]], i64 [[PREDPHI]]
; CHECK-NEXT: store i16 0, ptr [[TMP1]], align 2
; CHECK-NEXT: store i16 0, ptr [[TMP1]], align 2
diff --git a/llvm/test/Transforms/LoopVectorize/constant-fold-commutative-and.ll b/llvm/test/Transforms/LoopVectorize/constant-fold-commutative-and.ll
index ab2fcc8682a35..f4c85ca8ae3f2 100644
--- a/llvm/test/Transforms/LoopVectorize/constant-fold-commutative-and.ll
+++ b/llvm/test/Transforms/LoopVectorize/constant-fold-commutative-and.ll
@@ -18,18 +18,25 @@ define void @constant_fold_commutative_and(ptr %ptr.n, ptr noalias %p, i1 %cond)
; CHECK-NEXT: [[TMP1:%.*]] = icmp ule <2 x i8> [[VEC_IND]], splat (i8 16)
; CHECK-NEXT: [[TMP6:%.*]] = extractelement <2 x i1> [[TMP1]], i64 0
; CHECK-NEXT: [[TMP2:%.*]] = load i64, ptr [[PTR_N]], align 4
+; CHECK-NEXT: [[BROADCAST_SPLATINSERT:%.*]] = insertelement <2 x i64> poison, i64 [[TMP2]], i64 0
+; CHECK-NEXT: [[BROADCAST_SPLAT:%.*]] = shufflevector <2 x i64> [[BROADCAST_SPLATINSERT]], <2 x i64> poison, <2 x i32> zeroinitializer
; CHECK-NEXT: br i1 [[COND]], label %[[LATCH2:.*]], label %[[PRED_11:.*]]
; CHECK: [[PRED_11]]:
-; CHECK-NEXT: [[TMP7:%.*]] = icmp uge i64 [[TMP2]], 4
+; CHECK-NEXT: [[TMP12:%.*]] = icmp uge <2 x i64> [[BROADCAST_SPLAT]], splat (i64 4)
+; CHECK-NEXT: [[TMP4:%.*]] = select <2 x i1> [[TMP1]], <2 x i1> [[TMP12]], <2 x i1> zeroinitializer
+; CHECK-NEXT: [[TMP5:%.*]] = extractelement <2 x i1> [[TMP4]], i64 0
; CHECK-NEXT: [[TMP3:%.*]] = icmp ult i64 [[TMP2]], 7
; CHECK-NEXT: br label %[[LATCH2]]
; CHECK: [[LATCH2]]:
-; CHECK-NEXT: [[TMP4:%.*]] = phi i1 [ poison, %[[VECTOR_BODY]] ], [ [[TMP3]], %[[PRED_11]] ]
-; CHECK-NEXT: [[TMP5:%.*]] = phi i1 [ false, %[[VECTOR_BODY]] ], [ [[TMP7]], %[[PRED_11]] ]
-; CHECK-NEXT: [[TMP9:%.*]] = select i1 [[TMP5]], i1 [[TMP4]], i1 true
+; CHECK-NEXT: [[TMP7:%.*]] = phi i1 [ poison, %[[VECTOR_BODY]] ], [ [[TMP3]], %[[PRED_11]] ]
+; CHECK-NEXT: [[TMP13:%.*]] = phi i1 [ false, %[[VECTOR_BODY]] ], [ [[TMP5]], %[[PRED_11]] ]
+; CHECK-NEXT: [[BROADCAST_SPLATINSERT3:%.*]] = insertelement <2 x i1> poison, i1 [[TMP7]], i64 0
+; CHECK-NEXT: [[BROADCAST_SPLAT4:%.*]] = shufflevector <2 x i1> [[BROADCAST_SPLATINSERT3]], <2 x i1> poison, <2 x i32> zeroinitializer
+; CHECK-NEXT: [[PREDPHI:%.*]] = select i1 [[TMP13]], <2 x i1> [[BROADCAST_SPLAT4]], <2 x i1> [[TMP1]]
; CHECK-NEXT: br i1 [[TMP6]], label %[[PRED_STORE_IF:.*]], label %[[PRED_STORE_CONTINUE:.*]]
; CHECK: [[PRED_STORE_IF]]:
; CHECK-NEXT: [[TMP8:%.*]] = getelementptr i8, ptr [[P]], i64 [[INDEX]]
+; CHECK-NEXT: [[TMP9:%.*]] = extractelement <2 x i1> [[PREDPHI]], i64 0
; CHECK-NEXT: store i1 [[TMP9]], ptr [[TMP8]], align 1
; CHECK-NEXT: br label %[[PRED_STORE_CONTINUE]]
; CHECK: [[PRED_STORE_CONTINUE]]:
@@ -38,7 +45,8 @@ define void @constant_fold_commutative_and(ptr %ptr.n, ptr noalias %p, i1 %cond)
; CHECK: [[PRED_STORE_IF4]]:
; CHECK-NEXT: [[TMP11:%.*]] = add i64 [[INDEX]], 1
; CHECK-NEXT: [[TMP20:%.*]] = getelementptr i8, ptr [[P]], i64 [[TMP11]]
-; CHECK-NEXT: store i1 [[TMP9]], ptr [[TMP20]], align 1
+; CHECK-NEXT: [[TMP14:%.*]] = extractelement <2 x i1> [[PREDPHI]], i64 1
+; CHECK-NEXT: store i1 [[TMP14]], ptr [[TMP20]], align 1
; CHECK-NEXT: br label %[[PRED_STORE_CONTINUE5]]
; CHECK: [[PRED_STORE_CONTINUE5]]:
; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 2
diff --git a/llvm/test/Transforms/LoopVectorize/constantfolder.ll b/llvm/test/Transforms/LoopVectorize/constantfolder.ll
index 918075586224c..bc5b2d4f8e307 100644
--- a/llvm/test/Transforms/LoopVectorize/constantfolder.ll
+++ b/llvm/test/Transforms/LoopVectorize/constantfolder.ll
@@ -7,12 +7,14 @@ define void @const_fold_ptradd(ptr %dst, i64 %d) {
; CHECK-NEXT: [[ENTRY:.*:]]
; CHECK-NEXT: br label %[[VECTOR_PH:.*]]
; CHECK: [[VECTOR_PH]]:
-; CHECK-NEXT: [[TMP0:%.*]] = getelementptr i16, ptr [[DST]], i64 [[D]]
; CHECK-NEXT: br label %[[VECTOR_BODY:.*]]
; CHECK: [[VECTOR_BODY]]:
; CHECK-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[LOOP_LATCH1:.*]] ]
; CHECK-NEXT: br label %[[LOOP_LATCH1]]
; CHECK: [[LOOP_LATCH1]]:
+; CHECK-NEXT: [[TMP2:%.*]] = phi i1 [ false, %[[VECTOR_BODY]] ]
+; CHECK-NEXT: [[PREDPHI:%.*]] = select i1 [[TMP2]], i64 [[D]], i64 0
+; CHECK-NEXT: [[TMP0:%.*]] = getelementptr i16, ptr [[DST]], i64 [[PREDPHI]]
; CHECK-NEXT: store i16 0, ptr [[TMP0]], align 2
; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 4
; CHECK-NEXT: [[TMP1:%.*]] = icmp eq i64 [[INDEX_NEXT]], 100
@@ -50,12 +52,14 @@ define void @const_fold_inbounds_ptradd(ptr %dst, i64 %d) {
; CHECK-NEXT: [[ENTRY:.*:]]
; CHECK-NEXT: br label %[[VECTOR_PH:.*]]
; CHECK: [[VECTOR_PH]]:
-; CHECK-NEXT: [[TMP0:%.*]] = getelementptr inbounds i16, ptr [[DST]], i64 [[D]]
; CHECK-NEXT: br label %[[VECTOR_BODY:.*]]
; CHECK: [[VECTOR_BODY]]:
; CHECK-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[LOOP_LATCH1:.*]] ]
; CHECK-NEXT: br label %[[LOOP_LATCH1]]
; CHECK: [[LOOP_LATCH1]]:
+; CHECK-NEXT: [[TMP2:%.*]] = phi i1 [ false, %[[VECTOR_BODY]] ]
+; CHECK-NEXT: [[PREDPHI:%.*]] = select i1 [[TMP2]], i64 [[D]], i64 0
+; CHECK-NEXT: [[TMP0:%.*]] = getelementptr inbounds i16, ptr [[DST]], i64 [[PREDPHI]]
; CHECK-NEXT: store i16 0, ptr [[TMP0]], align 2
; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 4
; CHECK-NEXT: [[TMP1:%.*]] = icmp eq i64 [[INDEX_NEXT]], 100
@@ -98,7 +102,10 @@ define void @const_fold_select(ptr %dst, i64 %d) {
; CHECK-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[LOOP_LATCH1:.*]] ]
; CHECK-NEXT: br label %[[LOOP_LATCH1]]
; CHECK: [[LOOP_LATCH1]]:
-; CHECK-NEXT: store i64 [[D]], ptr [[DST]], align 8
+; CHECK-NEXT: [[TMP0:%.*]] = phi i1 [ false, %[[VECTOR_BODY]] ]
+; CHECK-NEXT: [[PREDPHI:%.*]] = select i1 [[TMP0]], i64 [[D]], i64 1
+; CHECK-NEXT: [[TMP1:%.*]] = or i64 [[D]], [[PREDPHI]]
+; CHECK-NEXT: store i64 [[TMP1]], ptr [[DST]], align 8
; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 4
; CHECK-NEXT: [[TMP4:%.*]] = icmp eq i64 [[INDEX_NEXT]], 100
; CHECK-NEXT: br i1 [[TMP4]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP4:![0-9]+]]
@@ -135,16 +142,18 @@ define void @const_fold_add_sub_mul_ashr_lshr(ptr %dst, i64 %d) {
; CHECK-NEXT: [[ENTRY:.*:]]
; CHECK-NEXT: br label %[[VECTOR_PH:.*]]
; CHECK: [[VECTOR_PH]]:
-; CHECK-NEXT: [[TMP0:%.*]] = add i64 2, [[D]]
-; CHECK-NEXT: [[TMP1:%.*]] = sub i64 [[TMP0]], [[D]]
-; CHECK-NEXT: [[TMP2:%.*]] = ashr i64 [[TMP1]], [[D]]
-; CHECK-NEXT: [[TMP3:%.*]] = mul i64 [[TMP2]], 3
-; CHECK-NEXT: [[TMP4:%.*]] = lshr i64 [[TMP3]], [[D]]
; CHECK-NEXT: br label %[[VECTOR_BODY:.*]]
; CHECK: [[VECTOR_BODY]]:
; CHECK-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[LOOP_LATCH1:.*]] ]
; CHECK-NEXT: br label %[[LOOP_LATCH1]]
; CHECK: [[LOOP_LATCH1]]:
+; CHECK-NEXT: [[TMP0:%.*]] = phi i1 [ false, %[[VECTOR_BODY]] ]
+; CHECK-NEXT: [[PREDPHI:%.*]] = select i1 [[TMP0]], i64 [[D]], i64 1
+; CHECK-NEXT: [[TMP1:%.*]] = add i64 2, [[PREDPHI]]
+; CHECK-NEXT: [[TMP2:%.*]] = sub i64 [[TMP1]], [[PREDPHI]]
+; CHECK-NEXT: [[TMP3:%.*]] = ashr i64 [[TMP2]], [[PREDPHI]]
+; CHECK-NEXT: [[TMP5:%.*]] = mul i64 [[TMP3]], 3
+; CHECK-NEXT: [[TMP4:%.*]] = lshr i64 [[TMP5]], [[PREDPHI]]
; CHECK-NEXT: store i64 [[TMP4]], ptr [[DST]], align 8
; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 4
; CHECK-NEXT: [[TMP6:%.*]] = icmp eq i64 [[INDEX_NEXT]], 100
@@ -186,14 +195,16 @@ define void @const_fold_and_or_xor(ptr %dst, i64 %d) {
; CHECK-NEXT: [[ENTRY:.*:]]
; CHECK-NEXT: br label %[[VECTOR_PH:.*]]
; CHECK: [[VECTOR_PH]]:
-; CHECK-NEXT: [[TMP0:%.*]] = or i64 2, [[D]]
-; CHECK-NEXT: [[TMP1:%.*]] = and i64 [[TMP0]], [[D]]
-; CHECK-NEXT: [[TMP2:%.*]] = and i64 [[TMP1]], [[D]]
; CHECK-NEXT: br label %[[VECTOR_BODY:.*]]
; CHECK: [[VECTOR_BODY]]:
; CHECK-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[LOOP_LATCH1:.*]] ]
; CHECK-NEXT: br label %[[LOOP_LATCH1]]
; CHECK: [[LOOP_LATCH1]]:
+; CHECK-NEXT: [[TMP0:%.*]] = phi i1 [ false, %[[VECTOR_BODY]] ]
+; CHECK-NEXT: [[PREDPHI:%.*]] = select i1 [[TMP0]], i64 [[D]], i64 1
+; CHECK-NEXT: [[TMP1:%.*]] = or i64 2, [[PREDPHI]]
+; CHECK-NEXT: [[TMP3:%.*]] = and i64 [[TMP1]], [[PREDPHI]]
+; CHECK-NEXT: [[TMP2:%.*]] = and i64 [[TMP3]], [[PREDPHI]]
; CHECK-NEXT: store i64 [[TMP2]], ptr [[DST]], align 8
; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 4
; CHECK-NEXT: [[TMP4:%.*]] = icmp eq i64 [[INDEX_NEXT]], 100
@@ -235,14 +246,16 @@ define void @const_fold_cmp_zext(ptr %dst, i64 %d) {
; CHECK: [[VECTOR_PH]]:
; CHECK-NEXT: [[BROADCAST_SPLATINSERT:%.*]] = insertelement <4 x i64> poison, i64 [[D]], i64 0
; CHECK-NEXT: [[BROADCAST_SPLAT:%.*]] = shufflevector <4 x i64> [[BROADCAST_SPLATINSERT]], <4 x i64> poison, <4 x i32> zeroinitializer
-; CHECK-NEXT: [[TMP0:%.*]] = icmp ugt <4 x i64> splat (i64 2), [[BROADCAST_SPLAT]]
-; CHECK-NEXT: [[TMP1:%.*]] = zext <4 x i1> [[TMP0]] to <4 x i8>
-; CHECK-NEXT: [[TMP2:%.*]] = extractelement <4 x i8> [[TMP1]], i64 3
; CHECK-NEXT: br label %[[VECTOR_BODY:.*]]
; CHECK: [[VECTOR_BODY]]:
; CHECK-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[LOOP_LATCH1:.*]] ]
; CHECK-NEXT: br label %[[LOOP_LATCH1]]
; CHECK: [[LOOP_LATCH1]]:
+; CHECK-NEXT: [[TMP0:%.*]] = phi i1 [ false, %[[VECTOR_BODY]] ]
+; CHECK-NEXT: [[PREDPHI:%.*]] = select i1 [[TMP0]], <4 x i64> [[BROADCAST_SPLAT]], <4 x i64> splat (i64 1)
+; CHECK-NEXT: [[TMP1:%.*]] = icmp ugt <4 x i64> splat (i64 2), [[PREDPHI]]
+; CHECK-NEXT: [[TMP4:%.*]] = zext <4 x i1> [[TMP1]] to <4 x i8>
+; CHECK-NEXT: [[TMP2:%.*]] = extractelement <4 x i8> [[TMP4]], i64 3
; CHECK-NEXT: store i8 [[TMP2]], ptr [[DST]], align 1
; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 4
; CHECK-NEXT: [[TMP3:%.*]] = icmp eq i64 [[INDEX_NEXT]], 100
@@ -283,13 +296,15 @@ define void @const_fold_trunc(ptr %dst, i64 %d) {
; CHECK: [[VECTOR_PH]]:
; CHECK-NEXT: [[BROADCAST_SPLATINSERT:%.*]] = insertelement <4 x i64> poison, i64 [[D]], i64 0
; CHECK-NEXT: [[BROADCAST_SPLAT:%.*]] = shufflevector <4 x i64> [[BROADCAST_SPLATINSERT]], <4 x i64> poison, <4 x i32> zeroinitializer
-; CHECK-NEXT: [[TMP0:%.*]] = trunc <4 x i64> [[BROADCAST_SPLAT]] to <4 x i16>
-; CHECK-NEXT: [[TMP1:%.*]] = extractelement <4 x i16> [[TMP0]], i64 3
; CHECK-NEXT: br label %[[VECTOR_BODY:.*]]
; CHECK: [[VECTOR_BODY]]:
; CHECK-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[LOOP_LATCH1:.*]] ]
; CHECK-NEXT: br label %[[LOOP_LATCH1]]
; CHECK: [[LOOP_LATCH1]]:
+; CHECK-NEXT: [[TMP0:%.*]] = phi i1 [ false, %[[VECTOR_BODY]] ]
+; CHECK-NEXT: [[PREDPHI:%.*]] = select i1 [[TMP0]], <4 x i64> [[BROADCAST_SPLAT]], <4 x i64> zeroinitializer
+; CHECK-NEXT: [[TMP3:%.*]] = trunc <4 x i64> [[PREDPHI]] to <4 x i16>
+; CHECK-NEXT: [[TMP1:%.*]] = extractelement <4 x i16> [[TMP3]], i64 3
; CHECK-NEXT: store i16 [[TMP1]], ptr [[DST]], align 2
; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 4
; CHECK-NEXT: [[TMP2:%.*]] = icmp eq i64 [[INDEX_NEXT]], 100
diff --git a/llvm/test/Transforms/LoopVectorize/hoist-predicated-loads-with-predicated-stores.ll b/llvm/test/Transforms/LoopVectorize/hoist-predicated-loads-with-predicated-stores.ll
index 3b7a06e517a0e..06aff8ed57d2e 100644
--- a/llvm/test/Transforms/LoopVectorize/hoist-predicated-loads-with-predicated-stores.ll
+++ b/llvm/test/Transforms/LoopVectorize/hoist-predicated-loads-with-predicated-stores.ll
@@ -1637,7 +1637,8 @@ define void @sink_stores_cse_select_dropping_fmf(ptr %dst, ptr %src, ptr %invar.
; CHECK: [[VECTOR_BODY]]:
; CHECK-NEXT: [[TMP8:%.*]] = phi float [ [[TMP5]], %[[ELSE10]] ], [ poison, %[[THEN11]] ]
; CHECK-NEXT: [[TMP9:%.*]] = phi float [ poison, %[[ELSE10]] ], [ [[TMP4]], %[[THEN11]] ]
-; CHECK-NEXT: [[PREDPHI:%.*]] = select i1 true, float [[TMP9]], float [[TMP8]]
+; CHECK-NEXT: [[TMP12:%.*]] = phi i1 [ false, %[[ELSE10]] ], [ true, %[[THEN11]] ]
+; CHECK-NEXT: [[PREDPHI:%.*]] = select i1 [[TMP12]], float [[TMP9]], float [[TMP8]]
; CHECK-NEXT: [[TMP10:%.*]] = fmul nnan float [[PREDPHI]], 5.000000e+00
; CHECK-NEXT: store float [[TMP10]], ptr [[INVAR_DST]], align 4, !alias.scope [[META126:![0-9]+]], !noalias [[META119]]
; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 2
diff --git a/llvm/test/Transforms/LoopVectorize/hoist-predicated-loads.ll b/llvm/test/Transforms/LoopVectorize/hoist-predicated-loads.ll
index 53c7dd5188e0d..7c2470f7d590e 100644
--- a/llvm/test/Transforms/LoopVectorize/hoist-predicated-loads.ll
+++ b/llvm/test/Transforms/LoopVectorize/hoist-predicated-loads.ll
@@ -929,7 +929,8 @@ define void @hoist_predicated_load_with_chained_geps1(ptr %dst, ptr %src, i1 %co
; CHECK: [[LOOP_LATCH5]]:
; CHECK-NEXT: [[TMP6:%.*]] = phi i16 [ [[TMP8]], %[[ELSE3]] ], [ poison, %[[THEN4]] ]
; CHECK-NEXT: [[TMP7:%.*]] = phi i16 [ poison, %[[ELSE3]] ], [ [[TMP5]], %[[THEN4]] ]
-; CHECK-NEXT: [[PREDPHI:%.*]] = select i1 true, i16 [[TMP7]], i16 [[TMP6]]
+; CHECK-NEXT: [[TMP9:%.*]] = phi i1 [ false, %[[ELSE3]] ], [ true, %[[THEN4]] ]
+; CHECK-NEXT: [[PREDPHI:%.*]] = select i1 [[TMP9]], i16 [[TMP7]], i16 [[TMP6]]
; CHECK-NEXT: store i16 [[PREDPHI]], ptr [[DST]], align 2, !alias.scope [[META68:![0-9]+]], !noalias [[META71:![0-9]+]]
; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[TMP2]], 2
; CHECK-NEXT: [[TMP26:%.*]] = icmp eq i64 [[INDEX_NEXT]], 100
@@ -998,7 +999,8 @@ define void @hoist_predicated_load_with_chained_geps2(ptr %dst, ptr %src, i1 %co
; CHECK: [[LOOP_LATCH5]]:
; CHECK-NEXT: [[TMP5:%.*]] = phi i16 [ [[TMP7]], %[[ELSE3]] ], [ poison, %[[THEN4]] ]
; CHECK-NEXT: [[TMP6:%.*]] = phi i16 [ poison, %[[ELSE3]] ], [ [[TMP8]], %[[THEN4]] ]
-; CHECK-NEXT: [[PREDPHI:%.*]] = select i1 true, i16 [[TMP6]], i16 [[TMP5]]
+; CHECK-NEXT: [[TMP9:%.*]] = phi i1 [ false, %[[ELSE3]] ], [ true, %[[THEN4]] ]
+; CHECK-NEXT: [[PREDPHI:%.*]] = select i1 [[TMP9]], i16 [[TMP6]], i16 [[TMP5]]
; CHECK-NEXT: store i16 [[PREDPHI]], ptr [[DST]], align 2, !alias.scope [[META75:![0-9]+]], !noalias [[META78:![0-9]+]]
; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[TMP2]], 2
; CHECK-NEXT: [[TMP26:%.*]] = icmp eq i64 [[INDEX_NEXT]], 100
diff --git a/llvm/test/Transforms/LoopVectorize/phi-with-fastflags.ll b/llvm/test/Transforms/LoopVectorize/phi-with-fastflags.ll
index 488b72f2fe85e..99b036a99d5e8 100644
--- a/llvm/test/Transforms/LoopVectorize/phi-with-fastflags.ll
+++ b/llvm/test/Transforms/LoopVectorize/phi-with-fastflags.ll
@@ -16,7 +16,8 @@ define void @f(ptr noalias %p, i1 %c) {
; CHECK: [[THEN1]]:
; CHECK-NEXT: br label %[[LATCH2]]
; CHECK: [[LATCH2]]:
-; CHECK-NEXT: [[PREDPHI:%.*]] = select fast i1 true, <4 x float> zeroinitializer, <4 x float> [[WIDE_LOAD]]
+; CHECK-NEXT: [[TMP2:%.*]] = phi i1 [ false, %[[VECTOR_BODY]] ], [ true, %[[THEN1]] ]
+; CHECK-NEXT: [[PREDPHI:%.*]] = select fast i1 [[TMP2]], <4 x float> zeroinitializer, <4 x float> [[WIDE_LOAD]]
; CHECK-NEXT: store <4 x float> [[PREDPHI]], ptr [[TMP0]], align 4
; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i32 [[INDEX]], 4
; CHECK-NEXT: [[TMP1:%.*]] = icmp eq i32 [[INDEX_NEXT]], 1024
diff --git a/llvm/test/Transforms/LoopVectorize/pr45525.ll b/llvm/test/Transforms/LoopVectorize/pr45525.ll
index bbe279a6fae54..10e5dbc546c4c 100644
--- a/llvm/test/Transforms/LoopVectorize/pr45525.ll
+++ b/llvm/test/Transforms/LoopVectorize/pr45525.ll
@@ -18,9 +18,10 @@ define void @main(i1 %cond, ptr %arr) {
; CHECK-NEXT: br label %[[BB_32]]
; CHECK: [[BB_32]]:
; CHECK-NEXT: [[TMP3:%.*]] = phi i32 [ poison, %[[VECTOR_BODY]] ], [ [[TMP0]], %[[BB_21]] ]
+; CHECK-NEXT: [[TMP4:%.*]] = phi i1 [ false, %[[VECTOR_BODY]] ], [ true, %[[BB_21]] ]
; CHECK-NEXT: [[BROADCAST_SPLATINSERT:%.*]] = insertelement <4 x i32> poison, i32 [[TMP3]], i64 0
; CHECK-NEXT: [[BROADCAST_SPLAT:%.*]] = shufflevector <4 x i32> [[BROADCAST_SPLATINSERT]], <4 x i32> poison, <4 x i32> zeroinitializer
-; CHECK-NEXT: [[PREDPHI:%.*]] = select i1 true, <4 x i32> [[BROADCAST_SPLAT]], <4 x i32> splat (i32 7)
+; CHECK-NEXT: [[PREDPHI:%.*]] = select i1 [[TMP4]], <4 x i32> [[BROADCAST_SPLAT]], <4 x i32> splat (i32 7)
; CHECK-NEXT: [[TMP1:%.*]] = getelementptr inbounds i32, ptr [[ARR]], i32 [[INDEX]]
; CHECK-NEXT: store <4 x i32> [[PREDPHI]], ptr [[TMP1]], align 4
; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i32 [[INDEX]], 4
diff --git a/llvm/test/Transforms/LoopVectorize/predicator.ll b/llvm/test/Transforms/LoopVectorize/predicator.ll
index 1822a2f59fc5d..9fe0178921ed4 100644
--- a/llvm/test/Transforms/LoopVectorize/predicator.ll
+++ b/llvm/test/Transforms/LoopVectorize/predicator.ll
@@ -236,6 +236,10 @@ define void @simplifiable_blend(i1 %c1, i1 %c2, i1 %c3, i32 %x, i32 %y, ptr %p)
; CHECK-NEXT: [[ENTRY:.*:]]
; CHECK-NEXT: br label %[[VECTOR_PH:.*]]
; CHECK: [[VECTOR_PH]]:
+; CHECK-NEXT: [[BROADCAST_SPLATINSERT:%.*]] = insertelement <4 x i32> poison, i32 [[X]], i64 0
+; CHECK-NEXT: [[BROADCAST_SPLAT:%.*]] = shufflevector <4 x i32> [[BROADCAST_SPLATINSERT]], <4 x i32> poison, <4 x i32> zeroinitializer
+; CHECK-NEXT: [[BROADCAST_SPLATINSERT1:%.*]] = insertelement <4 x i32> poison, i32 [[Y]], i64 0
+; CHECK-NEXT: [[BROADCAST_SPLAT2:%.*]] = shufflevector <4 x i32> [[BROADCAST_SPLATINSERT1]], <4 x i32> poison, <4 x i32> zeroinitializer
; CHECK-NEXT: br label %[[VECTOR_BODY:.*]]
; CHECK: [[VECTOR_BODY]]:
; CHECK-NEXT: [[INDEX:%.*]] = phi i32 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[LATCH5:.*]] ]
@@ -249,9 +253,9 @@ define void @simplifiable_blend(i1 %c1, i1 %c2, i1 %c3, i32 %x, i32 %y, ptr %p)
; CHECK: [[A4]]:
; CHECK-NEXT: br label %[[LATCH5]]
; CHECK: [[LATCH5]]:
+; CHECK-NEXT: [[TMP0:%.*]] = phi i1 [ false, %[[F4]] ], [ true, %[[A4]] ]
+; CHECK-NEXT: [[PREDPHI7:%.*]] = select i1 [[TMP0]], <4 x i32> [[BROADCAST_SPLAT]], <4 x i32> [[BROADCAST_SPLAT2]]
; CHECK-NEXT: [[TMP3:%.*]] = getelementptr i32, ptr [[P]], i32 [[INDEX]]
-; CHECK-NEXT: [[BROADCAST_SPLATINSERT:%.*]] = insertelement <4 x i32> poison, i32 [[X]], i64 0
-; CHECK-NEXT: [[PREDPHI7:%.*]] = shufflevector <4 x i32> [[BROADCAST_SPLATINSERT]], <4 x i32> poison, <4 x i32> zeroinitializer
; CHECK-NEXT: store <4 x i32> [[PREDPHI7]], ptr [[TMP3]], align 4
; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i32 [[INDEX]], 4
; CHECK-NEXT: [[TMP4:%.*]] = icmp eq i32 [[INDEX_NEXT]], 128
@@ -319,9 +323,10 @@ define void @outermost_uniform_branch(ptr %a, i1 %u0) {
; CHECK-NEXT: br label %[[BB42]]
; CHECK: [[BB42]]:
; CHECK-NEXT: [[TMP4:%.*]] = phi i64 [ poison, %[[VECTOR_BODY]] ], [ [[TMP3]], %[[BB11]] ]
+; CHECK-NEXT: [[TMP7:%.*]] = phi i1 [ false, %[[VECTOR_BODY]] ], [ true, %[[BB11]] ]
; CHECK-NEXT: [[BROADCAST_SPLATINSERT:%.*]] = insertelement <4 x i64> poison, i64 [[TMP4]], i64 0
; CHECK-NEXT: [[BROADCAST_SPLAT:%.*]] = shufflevector <4 x i64> [[BROADCAST_SPLATINSERT]], <4 x i64> poison, <4 x i32> zeroinitializer
-; CHECK-NEXT: [[PREDPHI3:%.*]] = select i1 true, <4 x i64> [[BROADCAST_SPLAT]], <4 x i64> [[VEC_IND]]
+; CHECK-NEXT: [[PREDPHI3:%.*]] = select i1 [[TMP7]], <4 x i64> [[BROADCAST_SPLAT]], <4 x i64> [[VEC_IND]]
; CHECK-NEXT: [[TMP5:%.*]] = extractelement <4 x i64> [[PREDPHI3]], i64 3
; CHECK-NEXT: store i64 [[TMP5]], ptr [[A]], align 4
; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 4
@@ -472,7 +477,8 @@ define void @outermost_uniform_branch_more_blocks(ptr %a, i1 %u0) {
; CHECK: [[BB63]]:
; CHECK-NEXT: [[TMP5:%.*]] = phi i64 [ [[TMP7]], %[[BB11]] ], [ poison, %[[BB22]] ]
; CHECK-NEXT: [[TMP6:%.*]] = phi i64 [ poison, %[[BB11]] ], [ [[TMP4]], %[[BB22]] ]
-; CHECK-NEXT: [[PREDPHI4:%.*]] = select i1 true, i64 [[TMP6]], i64 [[TMP5]]
+; CHECK-NEXT: [[TMP8:%.*]] = phi i1 [ false, %[[BB11]] ], [ true, %[[BB22]] ]
+; CHECK-NEXT: [[PREDPHI4:%.*]] = select i1 [[TMP8]], i64 [[TMP6]], i64 [[TMP5]]
; CHECK-NEXT: store i64 [[PREDPHI4]], ptr [[A]], align 4
; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 4
; CHECK-NEXT: [[TMP0:%.*]] = icmp eq i64 [[INDEX_NEXT]], 128
@@ -2038,13 +2044,14 @@ define void @uniform_branch_shared_join_with_varying(ptr %a, i1 %u0) {
; CHECK-NEXT: [[TMP3:%.*]] = add i64 [[INDEX]], 3
; CHECK-NEXT: br label %[[BB43]]
; CHECK: [[BB43]]:
-; CHECK-NEXT: [[TMP4:%.*]] = phi i64 [ [[TMP8]], %[[BB11]] ], [ poison, %[[BB22]] ]
-; CHECK-NEXT: [[TMP5:%.*]] = phi i64 [ poison, %[[BB11]] ], [ [[TMP1]], %[[BB22]] ]
-; CHECK-NEXT: [[TMP6:%.*]] = phi i64 [ poison, %[[BB11]] ], [ [[TMP3]], %[[BB22]] ]
-; CHECK-NEXT: [[TMP7:%.*]] = phi i1 [ false, %[[BB11]] ], [ [[TMP2]], %[[BB22]] ]
-; CHECK-NEXT: [[PREDPHI:%.*]] = select i1 true, i64 [[TMP5]], i64 [[TMP4]]
+; CHECK-NEXT: [[PREDPHI:%.*]] = phi i64 [ [[TMP8]], %[[BB11]] ], [ poison, %[[BB22]] ]
+; CHECK-NEXT: [[TMP6:%.*]] = phi i64 [ poison, %[[BB11]] ], [ [[TMP1]], %[[BB22]] ]
+; CHECK-NEXT: [[TMP7:%.*]] = phi i1 [ false, %[[BB11]] ], [ true, %[[BB22]] ]
+; CHECK-NEXT: [[TMP9:%.*]] = phi i64 [ poison, %[[BB11]] ], [ [[TMP3]], %[[BB22]] ]
+; CHECK-NEXT: [[TMP10:%.*]] = phi i1 [ false, %[[BB11]] ], [ [[TMP2]], %[[BB22]] ]
; CHECK-NEXT: [[PREDPHI4:%.*]] = select i1 [[TMP7]], i64 [[TMP6]], i64 [[PREDPHI]]
-; CHECK-NEXT: store i64 [[PREDPHI4]], ptr [[A]], align 4
+; CHECK-NEXT: [[PREDPHI5:%.*]] = select i1 [[TMP10]], i64 [[TMP9]], i64 [[PREDPHI4]]
+; CHECK-NEXT: store i64 [[PREDPHI5]], ptr [[A]], align 4
; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 4
; CHECK-NEXT: [[TMP0:%.*]] = icmp eq i64 [[INDEX_NEXT]], 128
; CHECK-NEXT: br i1 [[TMP0]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP38:![0-9]+]]
diff --git a/llvm/test/Transforms/LoopVectorize/scalarize-masked-call.ll b/llvm/test/Transforms/LoopVectorize/scalarize-masked-call.ll
index 1158686c08c19..920e982a52af7 100644
--- a/llvm/test/Transforms/LoopVectorize/scalarize-masked-call.ll
+++ b/llvm/test/Transforms/LoopVectorize/scalarize-masked-call.ll
@@ -1,69 +1,70 @@
-; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --version 6
; RUN: opt < %s -passes=loop-vectorize -force-vector-interleave=1 -force-vector-width=2 -S 2>&1 | FileCheck %s
target datalayout = "e-m:e-i64:64-f80:128-n8:16:32:64-S128"
define void @cond_call(ptr readonly %src, ptr noalias %dest, i64 %N) {
-; CHECK-LABEL: @cond_call(
-; CHECK-NEXT: entry:
-; CHECK-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[N:%.*]], 2
-; CHECK-NEXT: br i1 [[MIN_ITERS_CHECK]], label [[SCALAR_PH:%.*]], label [[VECTOR_PH:%.*]]
-; CHECK: vector.ph:
+; CHECK-LABEL: define void @cond_call(
+; CHECK-SAME: ptr readonly [[SRC:%.*]], ptr noalias [[DEST:%.*]], i64 [[N:%.*]]) {
+; CHECK-NEXT: [[ENTRY:.*]]:
+; CHECK-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[N]], 2
+; CHECK-NEXT: br i1 [[MIN_ITERS_CHECK]], label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]]
+; CHECK: [[VECTOR_PH]]:
; CHECK-NEXT: [[N_MOD_VF:%.*]] = and i64 [[N]], 1
; CHECK-NEXT: [[N_VEC:%.*]] = sub i64 [[N]], [[N_MOD_VF]]
-; CHECK-NEXT: br label [[VECTOR_BODY:%.*]]
-; CHECK: vector.body:
-; CHECK-NEXT: [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[PRED_CALL_CONTINUE2:%.*]] ]
-; CHECK-NEXT: [[TMP0:%.*]] = getelementptr inbounds i64, ptr [[SRC:%.*]], i64 [[INDEX]]
+; CHECK-NEXT: br label %[[VECTOR_BODY:.*]]
+; CHECK: [[VECTOR_BODY]]:
+; CHECK-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[PRED_CALL_CONTINUE2:.*]] ]
+; CHECK-NEXT: [[TMP0:%.*]] = getelementptr inbounds i64, ptr [[SRC]], i64 [[INDEX]]
; CHECK-NEXT: [[WIDE_LOAD:%.*]] = load <2 x i64>, ptr [[TMP0]], align 8
; CHECK-NEXT: [[TMP1:%.*]] = icmp ult <2 x i64> [[WIDE_LOAD]], splat (i64 5)
; CHECK-NEXT: [[TMP2:%.*]] = extractelement <2 x i1> [[TMP1]], i64 0
-; CHECK-NEXT: br i1 [[TMP2]], label [[PRED_CALL_IF:%.*]], label [[PRED_CALL_CONTINUE:%.*]]
-; CHECK: pred.call.if:
+; CHECK-NEXT: br i1 [[TMP2]], label %[[PRED_CALL_IF:.*]], label %[[PRED_CALL_CONTINUE:.*]]
+; CHECK: [[PRED_CALL_IF]]:
; CHECK-NEXT: [[TMP3:%.*]] = extractelement <2 x i64> [[WIDE_LOAD]], i64 0
; CHECK-NEXT: [[TMP4:%.*]] = call i64 @foo(i64 [[TMP3]]) #[[ATTR0:[0-9]+]]
; CHECK-NEXT: [[TMP5:%.*]] = insertelement <2 x i64> poison, i64 [[TMP4]], i64 0
-; CHECK-NEXT: br label [[PRED_CALL_CONTINUE]]
-; CHECK: pred.call.continue:
-; CHECK-NEXT: [[TMP6:%.*]] = phi <2 x i64> [ poison, [[VECTOR_BODY]] ], [ [[TMP5]], [[PRED_CALL_IF]] ]
+; CHECK-NEXT: br label %[[PRED_CALL_CONTINUE]]
+; CHECK: [[PRED_CALL_CONTINUE]]:
+; CHECK-NEXT: [[TMP6:%.*]] = phi <2 x i64> [ poison, %[[VECTOR_BODY]] ], [ [[TMP5]], %[[PRED_CALL_IF]] ]
; CHECK-NEXT: [[TMP7:%.*]] = extractelement <2 x i1> [[TMP1]], i64 1
-; CHECK-NEXT: br i1 [[TMP7]], label [[PRED_CALL_IF1:%.*]], label [[PRED_CALL_CONTINUE2]]
-; CHECK: pred.call.if1:
+; CHECK-NEXT: br i1 [[TMP7]], label %[[PRED_CALL_IF1:.*]], label %[[PRED_CALL_CONTINUE2]]
+; CHECK: [[PRED_CALL_IF1]]:
; CHECK-NEXT: [[TMP8:%.*]] = extractelement <2 x i64> [[WIDE_LOAD]], i64 1
; CHECK-NEXT: [[TMP9:%.*]] = call i64 @foo(i64 [[TMP8]]) #[[ATTR0]]
; CHECK-NEXT: [[TMP10:%.*]] = insertelement <2 x i64> [[TMP6]], i64 [[TMP9]], i64 1
-; CHECK-NEXT: br label [[PRED_CALL_CONTINUE2]]
-; CHECK: pred.call.continue2:
-; CHECK-NEXT: [[TMP11:%.*]] = phi <2 x i64> [ [[TMP6]], [[PRED_CALL_CONTINUE]] ], [ [[TMP10]], [[PRED_CALL_IF1]] ]
+; CHECK-NEXT: br label %[[PRED_CALL_CONTINUE2]]
+; CHECK: [[PRED_CALL_CONTINUE2]]:
+; CHECK-NEXT: [[TMP11:%.*]] = phi <2 x i64> [ [[TMP6]], %[[PRED_CALL_CONTINUE]] ], [ [[TMP10]], %[[PRED_CALL_IF1]] ]
; CHECK-NEXT: [[PREDPHI:%.*]] = select <2 x i1> [[TMP1]], <2 x i64> [[TMP11]], <2 x i64> [[WIDE_LOAD]]
-; CHECK-NEXT: [[TMP12:%.*]] = getelementptr inbounds i64, ptr [[DEST:%.*]], i64 [[INDEX]]
+; CHECK-NEXT: [[TMP12:%.*]] = getelementptr inbounds i64, ptr [[DEST]], i64 [[INDEX]]
; CHECK-NEXT: store <2 x i64> [[PREDPHI]], ptr [[TMP12]], align 8
; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 2
; CHECK-NEXT: [[TMP13:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
-; CHECK-NEXT: br i1 [[TMP13]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP0:![0-9]+]]
-; CHECK: middle.block:
+; CHECK-NEXT: br i1 [[TMP13]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP0:![0-9]+]]
+; CHECK: [[MIDDLE_BLOCK]]:
; CHECK-NEXT: [[CMP_N:%.*]] = icmp eq i64 [[N]], [[N_VEC]]
-; CHECK-NEXT: br i1 [[CMP_N]], label [[END:%.*]], label [[SCALAR_PH]]
-; CHECK: scalar.ph:
-; CHECK-NEXT: [[BC_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC]], [[MIDDLE_BLOCK]] ], [ 0, [[ENTRY:%.*]] ]
-; CHECK-NEXT: br label [[FOR_BODY:%.*]]
-; CHECK: for.body:
-; CHECK-NEXT: [[IV:%.*]] = phi i64 [ [[BC_RESUME_VAL]], [[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], [[FOR_LOOP:%.*]] ]
+; CHECK-NEXT: br i1 [[CMP_N]], label %[[END:.*]], label %[[SCALAR_PH]]
+; CHECK: [[SCALAR_PH]]:
+; CHECK-NEXT: [[BC_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC]], %[[MIDDLE_BLOCK]] ], [ 0, %[[ENTRY]] ]
+; CHECK-NEXT: br label %[[FOR_BODY:.*]]
+; CHECK: [[FOR_BODY]]:
+; CHECK-NEXT: [[IV:%.*]] = phi i64 [ [[BC_RESUME_VAL]], %[[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], %[[FOR_LOOP:.*]] ]
; CHECK-NEXT: [[LD_ADDR:%.*]] = getelementptr inbounds i64, ptr [[SRC]], i64 [[IV]]
; CHECK-NEXT: [[LD_VALUE:%.*]] = load i64, ptr [[LD_ADDR]], align 8
; CHECK-NEXT: [[IFCOND:%.*]] = icmp ult i64 [[LD_VALUE]], 5
-; CHECK-NEXT: br i1 [[IFCOND]], label [[IF_THEN:%.*]], label [[FOR_LOOP]]
-; CHECK: if.then:
+; CHECK-NEXT: br i1 [[IFCOND]], label %[[IF_THEN:.*]], label %[[FOR_LOOP]]
+; CHECK: [[IF_THEN]]:
; CHECK-NEXT: [[FOO_RET:%.*]] = call i64 @foo(i64 [[LD_VALUE]]) #[[ATTR0]]
-; CHECK-NEXT: br label [[FOR_LOOP]]
-; CHECK: for.loop:
-; CHECK-NEXT: [[ST_VALUE:%.*]] = phi i64 [ [[LD_VALUE]], [[FOR_BODY]] ], [ [[FOO_RET]], [[IF_THEN]] ]
+; CHECK-NEXT: br label %[[FOR_LOOP]]
+; CHECK: [[FOR_LOOP]]:
+; CHECK-NEXT: [[ST_VALUE:%.*]] = phi i64 [ [[LD_VALUE]], %[[FOR_BODY]] ], [ [[FOO_RET]], %[[IF_THEN]] ]
; CHECK-NEXT: [[ST_ADDR:%.*]] = getelementptr inbounds i64, ptr [[DEST]], i64 [[IV]]
; CHECK-NEXT: store i64 [[ST_VALUE]], ptr [[ST_ADDR]], align 8
; CHECK-NEXT: [[IV_NEXT]] = add nuw nsw i64 [[IV]], 1
; CHECK-NEXT: [[LOOPCOND:%.*]] = icmp eq i64 [[IV_NEXT]], [[N]]
-; CHECK-NEXT: br i1 [[LOOPCOND]], label [[END]], label [[FOR_BODY]], !llvm.loop [[LOOP3:![0-9]+]]
-; CHECK: end:
+; CHECK-NEXT: br i1 [[LOOPCOND]], label %[[END]], label %[[FOR_BODY]], !llvm.loop [[LOOP3:![0-9]+]]
+; CHECK: [[END]]:
; CHECK-NEXT: ret void
;
entry:
@@ -95,32 +96,34 @@ end:
; Test for a masked scalar call to an intrinsic whose result is used as part of
; a load address.
define void @masked_umax_used_by_load_address(ptr noalias %src, ptr noalias %dst, i64 %n, i1 %c) {
-; CHECK-LABEL: @masked_umax_used_by_load_address(
-; CHECK-NEXT: entry:
-; CHECK-NEXT: br label [[VECTOR_PH:%.*]]
-; CHECK: vector.ph:
-; CHECK-NEXT: [[TMP0:%.*]] = call i64 @llvm.umax.i64(i64 [[N:%.*]], i64 1)
-; CHECK-NEXT: [[TMP1:%.*]] = getelementptr i16, ptr [[SRC:%.*]], i64 [[TMP0]]
-; CHECK-NEXT: br label [[VECTOR_BODY:%.*]]
-; CHECK: vector.body:
-; CHECK-NEXT: [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[PRED_LOAD_CONTINUE2:%.*]] ]
-; CHECK-NEXT: br i1 [[C:%.*]], label [[PRED_LOAD_IF1:%.*]], label [[PRED_LOAD_CONTINUE2]]
-; CHECK: then1:
+; CHECK-LABEL: define void @masked_umax_used_by_load_address(
+; CHECK-SAME: ptr noalias [[SRC:%.*]], ptr noalias [[DST:%.*]], i64 [[N:%.*]], i1 [[C:%.*]]) {
+; CHECK-NEXT: [[ENTRY:.*:]]
+; CHECK-NEXT: br label %[[VECTOR_PH:.*]]
+; CHECK: [[VECTOR_PH]]:
+; CHECK-NEXT: [[TMP0:%.*]] = call i64 @llvm.umax.i64(i64 [[N]], i64 1)
+; CHECK-NEXT: [[TMP1:%.*]] = getelementptr i16, ptr [[SRC]], i64 [[TMP0]]
+; CHECK-NEXT: br label %[[VECTOR_BODY:.*]]
+; CHECK: [[VECTOR_BODY]]:
+; CHECK-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[PRED_LOAD_CONTINUE2:.*]] ]
+; CHECK-NEXT: br i1 [[C]], label %[[PRED_LOAD_IF1:.*]], label %[[PRED_LOAD_CONTINUE2]]
+; CHECK: [[PRED_LOAD_IF1]]:
; CHECK-NEXT: [[TMP5:%.*]] = load i16, ptr [[TMP1]], align 2
-; CHECK-NEXT: br label [[PRED_LOAD_CONTINUE2]]
-; CHECK: loop.latch2:
-; CHECK-NEXT: [[TMP3:%.*]] = phi i16 [ poison, [[VECTOR_BODY]] ], [ [[TMP5]], [[PRED_LOAD_IF1]] ]
+; CHECK-NEXT: br label %[[PRED_LOAD_CONTINUE2]]
+; CHECK: [[PRED_LOAD_CONTINUE2]]:
+; CHECK-NEXT: [[TMP3:%.*]] = phi i16 [ poison, %[[VECTOR_BODY]] ], [ [[TMP5]], %[[PRED_LOAD_IF1]] ]
+; CHECK-NEXT: [[TMP4:%.*]] = phi i1 [ false, %[[VECTOR_BODY]] ], [ true, %[[PRED_LOAD_IF1]] ]
; CHECK-NEXT: [[BROADCAST_SPLATINSERT:%.*]] = insertelement <2 x i16> poison, i16 [[TMP3]], i64 0
; CHECK-NEXT: [[BROADCAST_SPLAT:%.*]] = shufflevector <2 x i16> [[BROADCAST_SPLATINSERT]], <2 x i16> poison, <2 x i32> zeroinitializer
-; CHECK-NEXT: [[PREDPHI:%.*]] = select i1 true, <2 x i16> [[BROADCAST_SPLAT]], <2 x i16> zeroinitializer
-; CHECK-NEXT: [[TMP8:%.*]] = getelementptr i16, ptr [[DST:%.*]], i64 [[INDEX]]
+; CHECK-NEXT: [[PREDPHI:%.*]] = select i1 [[TMP4]], <2 x i16> [[BROADCAST_SPLAT]], <2 x i16> zeroinitializer
+; CHECK-NEXT: [[TMP8:%.*]] = getelementptr i16, ptr [[DST]], i64 [[INDEX]]
; CHECK-NEXT: store <2 x i16> [[PREDPHI]], ptr [[TMP8]], align 2
; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 2
; CHECK-NEXT: [[TMP9:%.*]] = icmp eq i64 [[INDEX_NEXT]], 100
-; CHECK-NEXT: br i1 [[TMP9]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP4:![0-9]+]]
-; CHECK: middle.block:
-; CHECK-NEXT: br label [[EXIT:%.*]]
-; CHECK: exit:
+; CHECK-NEXT: br i1 [[TMP9]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP4:![0-9]+]]
+; CHECK: [[MIDDLE_BLOCK]]:
+; CHECK-NEXT: br label %[[EXIT:.*]]
+; CHECK: [[EXIT]]:
; CHECK-NEXT: ret void
;
entry:
@@ -157,3 +160,10 @@ declare <4 x i64> @vector_foo(<4 x i64>, <4 x i1>)
; "-force-vector-width=2", then it should pass the legality checks but
; scalarize. TODO: Remove the requirement to have a variant.
attributes #0 = { readonly nounwind "vector-function-abi-variant"="_ZGV_LLVM_M4v_foo(vector_foo)" }
+;.
+; CHECK: [[LOOP0]] = distinct !{[[LOOP0]], [[META1:![0-9]+]], [[META2:![0-9]+]]}
+; CHECK: [[META1]] = !{!"llvm.loop.isvectorized", i32 1}
+; CHECK: [[META2]] = !{!"llvm.loop.unroll.runtime.disable"}
+; CHECK: [[LOOP3]] = distinct !{[[LOOP3]], [[META2]], [[META1]]}
+; CHECK: [[LOOP4]] = distinct !{[[LOOP4]], [[META1]], [[META2]]}
+;.
diff --git a/llvm/test/Transforms/LoopVectorize/uniform-blend.ll b/llvm/test/Transforms/LoopVectorize/uniform-blend.ll
index e8c70a23f0ebc..9d9def7d7750e 100644
--- a/llvm/test/Transforms/LoopVectorize/uniform-blend.ll
+++ b/llvm/test/Transforms/LoopVectorize/uniform-blend.ll
@@ -104,10 +104,8 @@ define void @blend_chain_iv(i1 %c) {
; CHECK: [[VECTOR_PH]]:
; CHECK-NEXT: br label %[[VECTOR_BODY:.*]]
; CHECK: [[VECTOR_BODY]]:
-; CHECK-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[LOOP_NEXT_32:.*]] ]
-; CHECK-NEXT: [[TMP3:%.*]] = add i64 [[INDEX]], 1
-; CHECK-NEXT: [[TMP5:%.*]] = add i64 [[INDEX]], 2
-; CHECK-NEXT: [[TMP7:%.*]] = add i64 [[INDEX]], 3
+; CHECK-NEXT: [[INDEX1:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[LOOP_NEXT_32:.*]] ]
+; CHECK-NEXT: [[VEC_IND:%.*]] = phi <4 x i64> [ <i64 0, i64 1, i64 2, i64 3>, %[[VECTOR_PH]] ], [ [[VEC_IND_NEXT:%.*]], %[[LOOP_NEXT_32]] ]
; CHECK-NEXT: br i1 [[C]], label %[[LOOP_NEXT_21:.*]], label %[[LOOP_NEXT_32]]
; CHECK: [[LOOP_NEXT_21]]:
; CHECK-NEXT: br i1 [[C]], label %[[LOOP_NEXT_22:.*]], label %[[LOOP_NEXT_33:.*]]
@@ -116,6 +114,15 @@ define void @blend_chain_iv(i1 %c) {
; CHECK: [[LOOP_NEXT_33]]:
; CHECK-NEXT: br label %[[LOOP_NEXT_32]]
; CHECK: [[LOOP_NEXT_32]]:
+; CHECK-NEXT: [[TMP0:%.*]] = phi i64 [ poison, %[[VECTOR_BODY]] ], [ [[INDEX1]], %[[LOOP_NEXT_33]] ]
+; CHECK-NEXT: [[TMP1:%.*]] = phi i1 [ false, %[[VECTOR_BODY]] ], [ true, %[[LOOP_NEXT_33]] ]
+; CHECK-NEXT: [[BROADCAST_SPLATINSERT:%.*]] = insertelement <4 x i64> poison, i64 [[TMP0]], i64 0
+; CHECK-NEXT: [[BROADCAST_SPLAT:%.*]] = shufflevector <4 x i64> [[BROADCAST_SPLATINSERT]], <4 x i64> poison, <4 x i32> zeroinitializer
+; CHECK-NEXT: [[PREDPHI:%.*]] = select i1 [[TMP1]], <4 x i64> [[BROADCAST_SPLAT]], <4 x i64> [[VEC_IND]]
+; CHECK-NEXT: [[TMP7:%.*]] = extractelement <4 x i64> [[PREDPHI]], i64 3
+; CHECK-NEXT: [[TMP5:%.*]] = extractelement <4 x i64> [[PREDPHI]], i64 2
+; CHECK-NEXT: [[TMP3:%.*]] = extractelement <4 x i64> [[PREDPHI]], i64 1
+; CHECK-NEXT: [[INDEX:%.*]] = extractelement <4 x i64> [[PREDPHI]], i64 0
; CHECK-NEXT: [[TMP2:%.*]] = getelementptr inbounds [32 x i16], ptr @dst, i16 0, i64 [[INDEX]]
; CHECK-NEXT: [[TMP4:%.*]] = getelementptr inbounds [32 x i16], ptr @dst, i16 0, i64 [[TMP3]]
; CHECK-NEXT: [[TMP6:%.*]] = getelementptr inbounds [32 x i16], ptr @dst, i16 0, i64 [[TMP5]]
@@ -124,7 +131,8 @@ define void @blend_chain_iv(i1 %c) {
; CHECK-NEXT: store i16 0, ptr [[TMP4]], align 2
; CHECK-NEXT: store i16 0, ptr [[TMP6]], align 2
; CHECK-NEXT: store i16 0, ptr [[TMP8]], align 2
-; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 4
+; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX1]], 4
+; CHECK-NEXT: [[VEC_IND_NEXT]] = add nuw nsw <4 x i64> [[VEC_IND]], splat (i64 4)
; CHECK-NEXT: [[TMP9:%.*]] = icmp eq i64 [[INDEX_NEXT]], 32
; CHECK-NEXT: br i1 [[TMP9]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP4:![0-9]+]]
; CHECK: [[MIDDLE_BLOCK]]:
>From b8ec3340502bd93bbbb4aaddd2b680af114d99c9 Mon Sep 17 00:00:00 2001
From: Andrei Elovikov <andrei.elovikov at sifive.com>
Date: Wed, 26 Aug 2026 17:13:24 -0700
Subject: [PATCH 06/11] WIP2
---
.../Transforms/Vectorize/VPlanPredicator.cpp | 22 +-
.../LoopVectorize/RISCV/dead-ops-cost.ll | 6 +-
.../Transforms/LoopVectorize/RISCV/divrem.ll | 649 ++++++++----------
.../RISCV/gather-scatter-cost.ll | 22 +-
.../RISCV/tail-folding-complex-mask.ll | 32 +-
.../LoopVectorize/VPlan/predicator.ll | 15 +-
.../X86/pr109581-unused-blend.ll | 46 +-
.../X86/predicated-instruction-cost.ll | 14 +-
...licate-recipe-with-only-first-lane-used.ll | 10 +-
.../LoopVectorize/X86/x86-predication.ll | 200 +++---
.../constant-fold-commutative-and.ll | 23 +-
.../LoopVectorize/constantfolder.ll | 40 +-
...predicated-loads-with-predicated-stores.ll | 8 +-
.../LoopVectorize/hoist-predicated-loads.ll | 20 +-
.../LoopVectorize/if-pred-non-void.ll | 28 +-
.../LoopVectorize/if-pred-stores.ll | 30 +-
.../Transforms/LoopVectorize/induction.ll | 194 +++---
.../pr55167-fold-tail-live-out.ll | 28 +-
.../predicatedinst-loop-invariant.ll | 30 +-
.../Transforms/LoopVectorize/predicator.ll | 124 ++--
.../LoopVectorize/reduction-small-size.ll | 22 +-
.../LoopVectorize/scalarize-masked-call.ll | 121 ++--
22 files changed, 835 insertions(+), 849 deletions(-)
diff --git a/llvm/lib/Transforms/Vectorize/VPlanPredicator.cpp b/llvm/lib/Transforms/Vectorize/VPlanPredicator.cpp
index ac4cdf607eff3..9fe829538e6d3 100644
--- a/llvm/lib/Transforms/Vectorize/VPlanPredicator.cpp
+++ b/llvm/lib/Transforms/Vectorize/VPlanPredicator.cpp
@@ -125,6 +125,8 @@ class VPPredicator {
VPValue *createMaskOr(VPValue *LHS, VPValue *RHS, DebugLoc DL);
VPValue *reconstructSSA(VPBasicBlock *UseBB, VPValue *V, bool IsMask);
+ VPValue *reconstructBlendMask(VPBasicBlock *UseBB, VPBasicBlock *MaskBlock,
+ VPValue *Mask);
void fixSSA(VPRecipeBase *U, VPValue *V, bool IsMask);
bool shouldPreserveTerminator(VPBasicBlock *VPBB);
@@ -252,6 +254,21 @@ VPValue *VPPredicator::reconstructSSA(VPBasicBlock *UseBB, VPValue *V,
return vputils::reconstructSSA(UseBB, SSADefs);
}
+VPValue *VPPredicator::reconstructBlendMask(VPBasicBlock *UseBB,
+ VPBasicBlock *MaskBlock,
+ VPValue *Mask) {
+ if (!Mask)
+ Mask = Plan.getTrue();
+
+ // The mask is only valid along paths from MaskBlock. Make that explicit
+ // before reconstructing it at the blend location.
+ DenseMap<VPBasicBlock *, VPValue *> BlendMaskDefs;
+ BlendMaskDefs[Plan.getVectorLoopRegion()->getEntryBasicBlock()] =
+ Plan.getFalse();
+ BlendMaskDefs[MaskBlock] = Mask;
+ return vputils::reconstructSSA(UseBB, BlendMaskDefs);
+}
+
void VPPredicator::fixSSA(VPRecipeBase *U, VPValue *V, bool IsMask) {
VPValue *Fixed = reconstructSSA(U->getParent(), V, IsMask);
if (Fixed == V)
@@ -581,8 +598,9 @@ void VPPredicator::convertPhisToBlends(VPBasicBlock *VPBB) {
bool RemovedCommonMask =
CommonIncomingMask && Mask &&
match(Mask, m_RemoveMask(CommonIncomingMask, RemainingMask));
- VPValue *BlendMask = RemovedCommonMask ? RemainingMask : Mask;
- OperandsWithMask.append({V, BlendMask ? BlendMask : Plan.getTrue()});
+ if (RemovedCommonMask)
+ Mask = RemainingMask ? RemainingMask : Plan.getTrue();
+ OperandsWithMask.append({V, reconstructBlendMask(VPBB, MaskBlock, Mask)});
}
PHINode *IRPhi = cast_or_null<PHINode>(PhiR->getUnderlyingValue());
diff --git a/llvm/test/Transforms/LoopVectorize/RISCV/dead-ops-cost.ll b/llvm/test/Transforms/LoopVectorize/RISCV/dead-ops-cost.ll
index 88edf52074be6..80863da53f38d 100644
--- a/llvm/test/Transforms/LoopVectorize/RISCV/dead-ops-cost.ll
+++ b/llvm/test/Transforms/LoopVectorize/RISCV/dead-ops-cost.ll
@@ -244,11 +244,15 @@ define void @test_phi_in_latch_redundant(ptr %dst, i32 %a) {
; CHECK-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[CURRENT_ITERATION_NEXT:%.*]], %[[LOOP_LATCH2:.*]] ]
; CHECK-NEXT: [[AVL:%.*]] = phi i64 [ 37, %[[VECTOR_PH]] ], [ [[AVL_NEXT:%.*]], %[[LOOP_LATCH2]] ]
; CHECK-NEXT: [[TMP8:%.*]] = call i32 @llvm.experimental.get.vector.length.i64(i64 [[AVL]], i32 4, i1 true)
+; CHECK-NEXT: [[BROADCAST_SPLATINSERT1:%.*]] = insertelement <vscale x 4 x i32> poison, i32 [[TMP8]], i64 0
+; CHECK-NEXT: [[BROADCAST_SPLAT1:%.*]] = shufflevector <vscale x 4 x i32> [[BROADCAST_SPLATINSERT1]], <vscale x 4 x i32> poison, <vscale x 4 x i32> zeroinitializer
+; CHECK-NEXT: [[TMP4:%.*]] = call <vscale x 4 x i32> @llvm.stepvector.nxv4i32()
+; CHECK-NEXT: [[TMP6:%.*]] = icmp ult <vscale x 4 x i32> [[TMP4]], [[BROADCAST_SPLAT1]]
; CHECK-NEXT: br label %[[THEN1:.*]]
; CHECK: [[THEN1]]:
; CHECK-NEXT: br label %[[LOOP_LATCH2]]
; CHECK: [[LOOP_LATCH2]]:
-; CHECK-NEXT: [[PREDPHI:%.*]] = call <vscale x 4 x i32> @llvm.vp.merge.nxv4i32(<vscale x 4 x i1> splat (i1 true), <vscale x 4 x i32> [[BROADCAST_SPLAT]], <vscale x 4 x i32> zeroinitializer, i32 [[TMP8]])
+; CHECK-NEXT: [[PREDPHI:%.*]] = select <vscale x 4 x i1> [[TMP6]], <vscale x 4 x i32> [[BROADCAST_SPLAT]], <vscale x 4 x i32> zeroinitializer
; CHECK-NEXT: [[TMP2:%.*]] = mul i64 [[INDEX]], 36
; CHECK-NEXT: [[TMP3:%.*]] = getelementptr i8, ptr [[DST]], i64 [[TMP2]]
; CHECK-NEXT: call void @llvm.experimental.vp.strided.store.nxv4i32.p0.i64(<vscale x 4 x i32> [[PREDPHI]], ptr align 4 [[TMP3]], i64 36, <vscale x 4 x i1> splat (i1 true), i32 [[TMP8]])
diff --git a/llvm/test/Transforms/LoopVectorize/RISCV/divrem.ll b/llvm/test/Transforms/LoopVectorize/RISCV/divrem.ll
index 596b67bc2f73e..f04134f54b15f 100644
--- a/llvm/test/Transforms/LoopVectorize/RISCV/divrem.ll
+++ b/llvm/test/Transforms/LoopVectorize/RISCV/divrem.ll
@@ -1,4 +1,4 @@
-; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --version 6
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
; RUN: opt < %s -passes=loop-vectorize -scalable-vectorization=on -mtriple riscv64-linux-gnu -mattr=+v,+f,+m -S 2>%t | FileCheck %s
; RUN: opt < %s -passes=loop-vectorize -scalable-vectorization=off -mtriple riscv64-linux-gnu -mattr=+v,+f,+m -S 2>%t | FileCheck --check-prefix=FIXED %s
@@ -8,19 +8,18 @@ target datalayout = "e-m:e-p:64:64-i64:64-i128:128-n64-S128"
target triple = "riscv64"
define void @vector_udiv(ptr noalias nocapture %a, i64 %v, i64 %n) {
-; CHECK-LABEL: define void @vector_udiv(
-; CHECK-SAME: ptr noalias captures(none) [[A:%.*]], i64 [[V:%.*]], i64 [[N:%.*]]) #[[ATTR0:[0-9]+]] {
-; CHECK-NEXT: [[ENTRY:.*:]]
-; CHECK-NEXT: br label %[[VECTOR_PH:.*]]
-; CHECK: [[VECTOR_PH]]:
-; CHECK-NEXT: [[BROADCAST_SPLATINSERT:%.*]] = insertelement <vscale x 2 x i64> poison, i64 [[V]], i64 0
+; CHECK-LABEL: @vector_udiv(
+; CHECK-NEXT: entry:
+; CHECK-NEXT: br label [[VECTOR_PH:%.*]]
+; CHECK: vector.ph:
+; CHECK-NEXT: [[BROADCAST_SPLATINSERT:%.*]] = insertelement <vscale x 2 x i64> poison, i64 [[V:%.*]], i64 0
; CHECK-NEXT: [[BROADCAST_SPLAT:%.*]] = shufflevector <vscale x 2 x i64> [[BROADCAST_SPLATINSERT]], <vscale x 2 x i64> poison, <vscale x 2 x i32> zeroinitializer
-; CHECK-NEXT: br label %[[VECTOR_BODY:.*]]
-; CHECK: [[VECTOR_BODY]]:
-; CHECK-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
-; CHECK-NEXT: [[AVL:%.*]] = phi i64 [ 1024, %[[VECTOR_PH]] ], [ [[AVL_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-NEXT: br label [[VECTOR_BODY:%.*]]
+; CHECK: vector.body:
+; CHECK-NEXT: [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ]
+; CHECK-NEXT: [[AVL:%.*]] = phi i64 [ 1024, [[VECTOR_PH]] ], [ [[AVL_NEXT:%.*]], [[VECTOR_BODY]] ]
; CHECK-NEXT: [[TMP10:%.*]] = call i32 @llvm.experimental.get.vector.length.i64(i64 [[AVL]], i32 2, i1 true)
-; CHECK-NEXT: [[TMP7:%.*]] = getelementptr inbounds i64, ptr [[A]], i64 [[INDEX]]
+; CHECK-NEXT: [[TMP7:%.*]] = getelementptr inbounds i64, ptr [[A:%.*]], i64 [[INDEX]]
; CHECK-NEXT: [[WIDE_LOAD:%.*]] = call <vscale x 2 x i64> @llvm.vp.load.nxv2i64.p0(ptr align 8 [[TMP7]], <vscale x 2 x i1> splat (i1 true), i32 [[TMP10]])
; CHECK-NEXT: [[TMP9:%.*]] = udiv <vscale x 2 x i64> [[WIDE_LOAD]], [[BROADCAST_SPLAT]]
; CHECK-NEXT: call void @llvm.vp.store.nxv2i64.p0(<vscale x 2 x i64> [[TMP9]], ptr align 8 [[TMP7]], <vscale x 2 x i1> splat (i1 true), i32 [[TMP10]])
@@ -28,32 +27,31 @@ define void @vector_udiv(ptr noalias nocapture %a, i64 %v, i64 %n) {
; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[TMP8]], [[INDEX]]
; CHECK-NEXT: [[AVL_NEXT]] = sub nuw i64 [[AVL]], [[TMP8]]
; CHECK-NEXT: [[TMP6:%.*]] = icmp eq i64 [[AVL_NEXT]], 0
-; CHECK-NEXT: br i1 [[TMP6]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP0:![0-9]+]]
-; CHECK: [[MIDDLE_BLOCK]]:
-; CHECK-NEXT: br label %[[FOR_END:.*]]
-; CHECK: [[FOR_END]]:
+; CHECK-NEXT: br i1 [[TMP6]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP0:![0-9]+]]
+; CHECK: middle.block:
+; CHECK-NEXT: br label [[FOR_BODY:%.*]]
+; CHECK: for.end:
; CHECK-NEXT: ret void
;
-; FIXED-LABEL: define void @vector_udiv(
-; FIXED-SAME: ptr noalias captures(none) [[A:%.*]], i64 [[V:%.*]], i64 [[N:%.*]]) #[[ATTR0:[0-9]+]] {
-; FIXED-NEXT: [[ENTRY:.*:]]
-; FIXED-NEXT: br label %[[VECTOR_PH:.*]]
-; FIXED: [[VECTOR_PH]]:
-; FIXED-NEXT: [[BROADCAST_SPLATINSERT:%.*]] = insertelement <4 x i64> poison, i64 [[V]], i64 0
+; FIXED-LABEL: @vector_udiv(
+; FIXED-NEXT: entry:
+; FIXED-NEXT: br label [[VECTOR_PH:%.*]]
+; FIXED: vector.ph:
+; FIXED-NEXT: [[BROADCAST_SPLATINSERT:%.*]] = insertelement <4 x i64> poison, i64 [[V:%.*]], i64 0
; FIXED-NEXT: [[BROADCAST_SPLAT:%.*]] = shufflevector <4 x i64> [[BROADCAST_SPLATINSERT]], <4 x i64> poison, <4 x i32> zeroinitializer
-; FIXED-NEXT: br label %[[VECTOR_BODY:.*]]
-; FIXED: [[VECTOR_BODY]]:
-; FIXED-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
-; FIXED-NEXT: [[TMP1:%.*]] = getelementptr inbounds i64, ptr [[A]], i64 [[INDEX]]
+; FIXED-NEXT: br label [[VECTOR_BODY:%.*]]
+; FIXED: vector.body:
+; FIXED-NEXT: [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ]
+; FIXED-NEXT: [[TMP1:%.*]] = getelementptr inbounds i64, ptr [[A:%.*]], i64 [[INDEX]]
; FIXED-NEXT: [[WIDE_LOAD1:%.*]] = load <4 x i64>, ptr [[TMP1]], align 8
; FIXED-NEXT: [[TMP5:%.*]] = udiv <4 x i64> [[WIDE_LOAD1]], [[BROADCAST_SPLAT]]
; FIXED-NEXT: store <4 x i64> [[TMP5]], ptr [[TMP1]], align 8
; FIXED-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 4
; FIXED-NEXT: [[TMP6:%.*]] = icmp eq i64 [[INDEX_NEXT]], 1024
-; FIXED-NEXT: br i1 [[TMP6]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP0:![0-9]+]]
-; FIXED: [[MIDDLE_BLOCK]]:
-; FIXED-NEXT: br label %[[FOR_END:.*]]
-; FIXED: [[FOR_END]]:
+; FIXED-NEXT: br i1 [[TMP6]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP0:![0-9]+]]
+; FIXED: middle.block:
+; FIXED-NEXT: br label [[FOR_BODY:%.*]]
+; FIXED: for.end:
; FIXED-NEXT: ret void
;
entry:
@@ -74,19 +72,18 @@ for.end:
}
define void @vector_sdiv(ptr noalias nocapture %a, i64 %v, i64 %n) {
-; CHECK-LABEL: define void @vector_sdiv(
-; CHECK-SAME: ptr noalias captures(none) [[A:%.*]], i64 [[V:%.*]], i64 [[N:%.*]]) #[[ATTR0]] {
-; CHECK-NEXT: [[ENTRY:.*:]]
-; CHECK-NEXT: br label %[[VECTOR_PH:.*]]
-; CHECK: [[VECTOR_PH]]:
-; CHECK-NEXT: [[BROADCAST_SPLATINSERT:%.*]] = insertelement <vscale x 2 x i64> poison, i64 [[V]], i64 0
+; CHECK-LABEL: @vector_sdiv(
+; CHECK-NEXT: entry:
+; CHECK-NEXT: br label [[VECTOR_PH:%.*]]
+; CHECK: vector.ph:
+; CHECK-NEXT: [[BROADCAST_SPLATINSERT:%.*]] = insertelement <vscale x 2 x i64> poison, i64 [[V:%.*]], i64 0
; CHECK-NEXT: [[BROADCAST_SPLAT:%.*]] = shufflevector <vscale x 2 x i64> [[BROADCAST_SPLATINSERT]], <vscale x 2 x i64> poison, <vscale x 2 x i32> zeroinitializer
-; CHECK-NEXT: br label %[[VECTOR_BODY:.*]]
-; CHECK: [[VECTOR_BODY]]:
-; CHECK-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
-; CHECK-NEXT: [[AVL:%.*]] = phi i64 [ 1024, %[[VECTOR_PH]] ], [ [[AVL_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-NEXT: br label [[VECTOR_BODY:%.*]]
+; CHECK: vector.body:
+; CHECK-NEXT: [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ]
+; CHECK-NEXT: [[AVL:%.*]] = phi i64 [ 1024, [[VECTOR_PH]] ], [ [[AVL_NEXT:%.*]], [[VECTOR_BODY]] ]
; CHECK-NEXT: [[TMP10:%.*]] = call i32 @llvm.experimental.get.vector.length.i64(i64 [[AVL]], i32 2, i1 true)
-; CHECK-NEXT: [[TMP7:%.*]] = getelementptr inbounds i64, ptr [[A]], i64 [[INDEX]]
+; CHECK-NEXT: [[TMP7:%.*]] = getelementptr inbounds i64, ptr [[A:%.*]], i64 [[INDEX]]
; CHECK-NEXT: [[WIDE_LOAD:%.*]] = call <vscale x 2 x i64> @llvm.vp.load.nxv2i64.p0(ptr align 8 [[TMP7]], <vscale x 2 x i1> splat (i1 true), i32 [[TMP10]])
; CHECK-NEXT: [[TMP9:%.*]] = call <vscale x 2 x i64> @llvm.vp.sdiv.nxv2i64(<vscale x 2 x i64> [[WIDE_LOAD]], <vscale x 2 x i64> [[BROADCAST_SPLAT]], <vscale x 2 x i1> splat (i1 true), i32 [[TMP10]])
; CHECK-NEXT: call void @llvm.vp.store.nxv2i64.p0(<vscale x 2 x i64> [[TMP9]], ptr align 8 [[TMP7]], <vscale x 2 x i1> splat (i1 true), i32 [[TMP10]])
@@ -94,32 +91,31 @@ define void @vector_sdiv(ptr noalias nocapture %a, i64 %v, i64 %n) {
; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[TMP8]], [[INDEX]]
; CHECK-NEXT: [[AVL_NEXT]] = sub nuw i64 [[AVL]], [[TMP8]]
; CHECK-NEXT: [[TMP6:%.*]] = icmp eq i64 [[AVL_NEXT]], 0
-; CHECK-NEXT: br i1 [[TMP6]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP3:![0-9]+]]
-; CHECK: [[MIDDLE_BLOCK]]:
-; CHECK-NEXT: br label %[[FOR_END:.*]]
-; CHECK: [[FOR_END]]:
+; CHECK-NEXT: br i1 [[TMP6]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP3:![0-9]+]]
+; CHECK: middle.block:
+; CHECK-NEXT: br label [[FOR_BODY:%.*]]
+; CHECK: for.end:
; CHECK-NEXT: ret void
;
-; FIXED-LABEL: define void @vector_sdiv(
-; FIXED-SAME: ptr noalias captures(none) [[A:%.*]], i64 [[V:%.*]], i64 [[N:%.*]]) #[[ATTR0]] {
-; FIXED-NEXT: [[ENTRY:.*:]]
-; FIXED-NEXT: br label %[[VECTOR_PH:.*]]
-; FIXED: [[VECTOR_PH]]:
-; FIXED-NEXT: [[BROADCAST_SPLATINSERT:%.*]] = insertelement <4 x i64> poison, i64 [[V]], i64 0
+; FIXED-LABEL: @vector_sdiv(
+; FIXED-NEXT: entry:
+; FIXED-NEXT: br label [[VECTOR_PH:%.*]]
+; FIXED: vector.ph:
+; FIXED-NEXT: [[BROADCAST_SPLATINSERT:%.*]] = insertelement <4 x i64> poison, i64 [[V:%.*]], i64 0
; FIXED-NEXT: [[BROADCAST_SPLAT:%.*]] = shufflevector <4 x i64> [[BROADCAST_SPLATINSERT]], <4 x i64> poison, <4 x i32> zeroinitializer
-; FIXED-NEXT: br label %[[VECTOR_BODY:.*]]
-; FIXED: [[VECTOR_BODY]]:
-; FIXED-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
-; FIXED-NEXT: [[TMP1:%.*]] = getelementptr inbounds i64, ptr [[A]], i64 [[INDEX]]
+; FIXED-NEXT: br label [[VECTOR_BODY:%.*]]
+; FIXED: vector.body:
+; FIXED-NEXT: [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ]
+; FIXED-NEXT: [[TMP1:%.*]] = getelementptr inbounds i64, ptr [[A:%.*]], i64 [[INDEX]]
; FIXED-NEXT: [[WIDE_LOAD1:%.*]] = load <4 x i64>, ptr [[TMP1]], align 8
; FIXED-NEXT: [[TMP5:%.*]] = sdiv <4 x i64> [[WIDE_LOAD1]], [[BROADCAST_SPLAT]]
; FIXED-NEXT: store <4 x i64> [[TMP5]], ptr [[TMP1]], align 8
; FIXED-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 4
; FIXED-NEXT: [[TMP6:%.*]] = icmp eq i64 [[INDEX_NEXT]], 1024
-; FIXED-NEXT: br i1 [[TMP6]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP3:![0-9]+]]
-; FIXED: [[MIDDLE_BLOCK]]:
-; FIXED-NEXT: br label %[[FOR_END:.*]]
-; FIXED: [[FOR_END]]:
+; FIXED-NEXT: br i1 [[TMP6]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP3:![0-9]+]]
+; FIXED: middle.block:
+; FIXED-NEXT: br label [[FOR_BODY:%.*]]
+; FIXED: for.end:
; FIXED-NEXT: ret void
;
entry:
@@ -140,19 +136,18 @@ for.end:
}
define void @vector_urem(ptr noalias nocapture %a, i64 %v, i64 %n) {
-; CHECK-LABEL: define void @vector_urem(
-; CHECK-SAME: ptr noalias captures(none) [[A:%.*]], i64 [[V:%.*]], i64 [[N:%.*]]) #[[ATTR0]] {
-; CHECK-NEXT: [[ENTRY:.*:]]
-; CHECK-NEXT: br label %[[VECTOR_PH:.*]]
-; CHECK: [[VECTOR_PH]]:
-; CHECK-NEXT: [[BROADCAST_SPLATINSERT:%.*]] = insertelement <vscale x 2 x i64> poison, i64 [[V]], i64 0
+; CHECK-LABEL: @vector_urem(
+; CHECK-NEXT: entry:
+; CHECK-NEXT: br label [[VECTOR_PH:%.*]]
+; CHECK: vector.ph:
+; CHECK-NEXT: [[BROADCAST_SPLATINSERT:%.*]] = insertelement <vscale x 2 x i64> poison, i64 [[V:%.*]], i64 0
; CHECK-NEXT: [[BROADCAST_SPLAT:%.*]] = shufflevector <vscale x 2 x i64> [[BROADCAST_SPLATINSERT]], <vscale x 2 x i64> poison, <vscale x 2 x i32> zeroinitializer
-; CHECK-NEXT: br label %[[VECTOR_BODY:.*]]
-; CHECK: [[VECTOR_BODY]]:
-; CHECK-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
-; CHECK-NEXT: [[AVL:%.*]] = phi i64 [ 1024, %[[VECTOR_PH]] ], [ [[AVL_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-NEXT: br label [[VECTOR_BODY:%.*]]
+; CHECK: vector.body:
+; CHECK-NEXT: [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ]
+; CHECK-NEXT: [[AVL:%.*]] = phi i64 [ 1024, [[VECTOR_PH]] ], [ [[AVL_NEXT:%.*]], [[VECTOR_BODY]] ]
; CHECK-NEXT: [[TMP10:%.*]] = call i32 @llvm.experimental.get.vector.length.i64(i64 [[AVL]], i32 2, i1 true)
-; CHECK-NEXT: [[TMP7:%.*]] = getelementptr inbounds i64, ptr [[A]], i64 [[INDEX]]
+; CHECK-NEXT: [[TMP7:%.*]] = getelementptr inbounds i64, ptr [[A:%.*]], i64 [[INDEX]]
; CHECK-NEXT: [[WIDE_LOAD:%.*]] = call <vscale x 2 x i64> @llvm.vp.load.nxv2i64.p0(ptr align 8 [[TMP7]], <vscale x 2 x i1> splat (i1 true), i32 [[TMP10]])
; CHECK-NEXT: [[TMP9:%.*]] = urem <vscale x 2 x i64> [[WIDE_LOAD]], [[BROADCAST_SPLAT]]
; CHECK-NEXT: call void @llvm.vp.store.nxv2i64.p0(<vscale x 2 x i64> [[TMP9]], ptr align 8 [[TMP7]], <vscale x 2 x i1> splat (i1 true), i32 [[TMP10]])
@@ -160,32 +155,31 @@ define void @vector_urem(ptr noalias nocapture %a, i64 %v, i64 %n) {
; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[TMP8]], [[INDEX]]
; CHECK-NEXT: [[AVL_NEXT]] = sub nuw i64 [[AVL]], [[TMP8]]
; CHECK-NEXT: [[TMP6:%.*]] = icmp eq i64 [[AVL_NEXT]], 0
-; CHECK-NEXT: br i1 [[TMP6]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP4:![0-9]+]]
-; CHECK: [[MIDDLE_BLOCK]]:
-; CHECK-NEXT: br label %[[FOR_END:.*]]
-; CHECK: [[FOR_END]]:
+; CHECK-NEXT: br i1 [[TMP6]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP4:![0-9]+]]
+; CHECK: middle.block:
+; CHECK-NEXT: br label [[FOR_BODY:%.*]]
+; CHECK: for.end:
; CHECK-NEXT: ret void
;
-; FIXED-LABEL: define void @vector_urem(
-; FIXED-SAME: ptr noalias captures(none) [[A:%.*]], i64 [[V:%.*]], i64 [[N:%.*]]) #[[ATTR0]] {
-; FIXED-NEXT: [[ENTRY:.*:]]
-; FIXED-NEXT: br label %[[VECTOR_PH:.*]]
-; FIXED: [[VECTOR_PH]]:
-; FIXED-NEXT: [[BROADCAST_SPLATINSERT:%.*]] = insertelement <4 x i64> poison, i64 [[V]], i64 0
+; FIXED-LABEL: @vector_urem(
+; FIXED-NEXT: entry:
+; FIXED-NEXT: br label [[VECTOR_PH:%.*]]
+; FIXED: vector.ph:
+; FIXED-NEXT: [[BROADCAST_SPLATINSERT:%.*]] = insertelement <4 x i64> poison, i64 [[V:%.*]], i64 0
; FIXED-NEXT: [[BROADCAST_SPLAT:%.*]] = shufflevector <4 x i64> [[BROADCAST_SPLATINSERT]], <4 x i64> poison, <4 x i32> zeroinitializer
-; FIXED-NEXT: br label %[[VECTOR_BODY:.*]]
-; FIXED: [[VECTOR_BODY]]:
-; FIXED-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
-; FIXED-NEXT: [[TMP1:%.*]] = getelementptr inbounds i64, ptr [[A]], i64 [[INDEX]]
+; FIXED-NEXT: br label [[VECTOR_BODY:%.*]]
+; FIXED: vector.body:
+; FIXED-NEXT: [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ]
+; FIXED-NEXT: [[TMP1:%.*]] = getelementptr inbounds i64, ptr [[A:%.*]], i64 [[INDEX]]
; FIXED-NEXT: [[WIDE_LOAD1:%.*]] = load <4 x i64>, ptr [[TMP1]], align 8
; FIXED-NEXT: [[TMP5:%.*]] = urem <4 x i64> [[WIDE_LOAD1]], [[BROADCAST_SPLAT]]
; FIXED-NEXT: store <4 x i64> [[TMP5]], ptr [[TMP1]], align 8
; FIXED-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 4
; FIXED-NEXT: [[TMP6:%.*]] = icmp eq i64 [[INDEX_NEXT]], 1024
-; FIXED-NEXT: br i1 [[TMP6]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP4:![0-9]+]]
-; FIXED: [[MIDDLE_BLOCK]]:
-; FIXED-NEXT: br label %[[FOR_END:.*]]
-; FIXED: [[FOR_END]]:
+; FIXED-NEXT: br i1 [[TMP6]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP4:![0-9]+]]
+; FIXED: middle.block:
+; FIXED-NEXT: br label [[FOR_BODY:%.*]]
+; FIXED: for.end:
; FIXED-NEXT: ret void
;
entry:
@@ -206,19 +200,18 @@ for.end:
}
define void @vector_srem(ptr noalias nocapture %a, i64 %v, i64 %n) {
-; CHECK-LABEL: define void @vector_srem(
-; CHECK-SAME: ptr noalias captures(none) [[A:%.*]], i64 [[V:%.*]], i64 [[N:%.*]]) #[[ATTR0]] {
-; CHECK-NEXT: [[ENTRY:.*:]]
-; CHECK-NEXT: br label %[[VECTOR_PH:.*]]
-; CHECK: [[VECTOR_PH]]:
-; CHECK-NEXT: [[BROADCAST_SPLATINSERT:%.*]] = insertelement <vscale x 2 x i64> poison, i64 [[V]], i64 0
+; CHECK-LABEL: @vector_srem(
+; CHECK-NEXT: entry:
+; CHECK-NEXT: br label [[VECTOR_PH:%.*]]
+; CHECK: vector.ph:
+; CHECK-NEXT: [[BROADCAST_SPLATINSERT:%.*]] = insertelement <vscale x 2 x i64> poison, i64 [[V:%.*]], i64 0
; CHECK-NEXT: [[BROADCAST_SPLAT:%.*]] = shufflevector <vscale x 2 x i64> [[BROADCAST_SPLATINSERT]], <vscale x 2 x i64> poison, <vscale x 2 x i32> zeroinitializer
-; CHECK-NEXT: br label %[[VECTOR_BODY:.*]]
-; CHECK: [[VECTOR_BODY]]:
-; CHECK-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
-; CHECK-NEXT: [[AVL:%.*]] = phi i64 [ 1024, %[[VECTOR_PH]] ], [ [[AVL_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-NEXT: br label [[VECTOR_BODY:%.*]]
+; CHECK: vector.body:
+; CHECK-NEXT: [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ]
+; CHECK-NEXT: [[AVL:%.*]] = phi i64 [ 1024, [[VECTOR_PH]] ], [ [[AVL_NEXT:%.*]], [[VECTOR_BODY]] ]
; CHECK-NEXT: [[TMP10:%.*]] = call i32 @llvm.experimental.get.vector.length.i64(i64 [[AVL]], i32 2, i1 true)
-; CHECK-NEXT: [[TMP7:%.*]] = getelementptr inbounds i64, ptr [[A]], i64 [[INDEX]]
+; CHECK-NEXT: [[TMP7:%.*]] = getelementptr inbounds i64, ptr [[A:%.*]], i64 [[INDEX]]
; CHECK-NEXT: [[WIDE_LOAD:%.*]] = call <vscale x 2 x i64> @llvm.vp.load.nxv2i64.p0(ptr align 8 [[TMP7]], <vscale x 2 x i1> splat (i1 true), i32 [[TMP10]])
; CHECK-NEXT: [[TMP9:%.*]] = call <vscale x 2 x i64> @llvm.vp.srem.nxv2i64(<vscale x 2 x i64> [[WIDE_LOAD]], <vscale x 2 x i64> [[BROADCAST_SPLAT]], <vscale x 2 x i1> splat (i1 true), i32 [[TMP10]])
; CHECK-NEXT: call void @llvm.vp.store.nxv2i64.p0(<vscale x 2 x i64> [[TMP9]], ptr align 8 [[TMP7]], <vscale x 2 x i1> splat (i1 true), i32 [[TMP10]])
@@ -226,32 +219,31 @@ define void @vector_srem(ptr noalias nocapture %a, i64 %v, i64 %n) {
; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[TMP8]], [[INDEX]]
; CHECK-NEXT: [[AVL_NEXT]] = sub nuw i64 [[AVL]], [[TMP8]]
; CHECK-NEXT: [[TMP6:%.*]] = icmp eq i64 [[AVL_NEXT]], 0
-; CHECK-NEXT: br i1 [[TMP6]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP5:![0-9]+]]
-; CHECK: [[MIDDLE_BLOCK]]:
-; CHECK-NEXT: br label %[[FOR_END:.*]]
-; CHECK: [[FOR_END]]:
+; CHECK-NEXT: br i1 [[TMP6]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP5:![0-9]+]]
+; CHECK: middle.block:
+; CHECK-NEXT: br label [[FOR_BODY:%.*]]
+; CHECK: for.end:
; CHECK-NEXT: ret void
;
-; FIXED-LABEL: define void @vector_srem(
-; FIXED-SAME: ptr noalias captures(none) [[A:%.*]], i64 [[V:%.*]], i64 [[N:%.*]]) #[[ATTR0]] {
-; FIXED-NEXT: [[ENTRY:.*:]]
-; FIXED-NEXT: br label %[[VECTOR_PH:.*]]
-; FIXED: [[VECTOR_PH]]:
-; FIXED-NEXT: [[BROADCAST_SPLATINSERT:%.*]] = insertelement <4 x i64> poison, i64 [[V]], i64 0
+; FIXED-LABEL: @vector_srem(
+; FIXED-NEXT: entry:
+; FIXED-NEXT: br label [[VECTOR_PH:%.*]]
+; FIXED: vector.ph:
+; FIXED-NEXT: [[BROADCAST_SPLATINSERT:%.*]] = insertelement <4 x i64> poison, i64 [[V:%.*]], i64 0
; FIXED-NEXT: [[BROADCAST_SPLAT:%.*]] = shufflevector <4 x i64> [[BROADCAST_SPLATINSERT]], <4 x i64> poison, <4 x i32> zeroinitializer
-; FIXED-NEXT: br label %[[VECTOR_BODY:.*]]
-; FIXED: [[VECTOR_BODY]]:
-; FIXED-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
-; FIXED-NEXT: [[TMP1:%.*]] = getelementptr inbounds i64, ptr [[A]], i64 [[INDEX]]
+; FIXED-NEXT: br label [[VECTOR_BODY:%.*]]
+; FIXED: vector.body:
+; FIXED-NEXT: [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ]
+; FIXED-NEXT: [[TMP1:%.*]] = getelementptr inbounds i64, ptr [[A:%.*]], i64 [[INDEX]]
; FIXED-NEXT: [[WIDE_LOAD1:%.*]] = load <4 x i64>, ptr [[TMP1]], align 8
; FIXED-NEXT: [[TMP5:%.*]] = srem <4 x i64> [[WIDE_LOAD1]], [[BROADCAST_SPLAT]]
; FIXED-NEXT: store <4 x i64> [[TMP5]], ptr [[TMP1]], align 8
; FIXED-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 4
; FIXED-NEXT: [[TMP6:%.*]] = icmp eq i64 [[INDEX_NEXT]], 1024
-; FIXED-NEXT: br i1 [[TMP6]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP5:![0-9]+]]
-; FIXED: [[MIDDLE_BLOCK]]:
-; FIXED-NEXT: br label %[[FOR_END:.*]]
-; FIXED: [[FOR_END]]:
+; FIXED-NEXT: br i1 [[TMP6]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP5:![0-9]+]]
+; FIXED: middle.block:
+; FIXED-NEXT: br label [[FOR_BODY:%.*]]
+; FIXED: for.end:
; FIXED-NEXT: ret void
;
entry:
@@ -272,20 +264,19 @@ for.end:
}
define void @predicated_udiv(ptr noalias nocapture %a, i64 %v, i64 %n) {
-; CHECK-LABEL: define void @predicated_udiv(
-; CHECK-SAME: ptr noalias captures(none) [[A:%.*]], i64 [[V:%.*]], i64 [[N:%.*]]) #[[ATTR0]] {
-; CHECK-NEXT: [[ENTRY:.*:]]
-; CHECK-NEXT: br label %[[VECTOR_PH:.*]]
-; CHECK: [[VECTOR_PH]]:
-; CHECK-NEXT: [[BROADCAST_SPLATINSERT:%.*]] = insertelement <vscale x 2 x i64> poison, i64 [[V]], i64 0
+; CHECK-LABEL: @predicated_udiv(
+; CHECK-NEXT: entry:
+; CHECK-NEXT: br label [[VECTOR_PH:%.*]]
+; CHECK: vector.ph:
+; CHECK-NEXT: [[BROADCAST_SPLATINSERT:%.*]] = insertelement <vscale x 2 x i64> poison, i64 [[V:%.*]], i64 0
; CHECK-NEXT: [[BROADCAST_SPLAT:%.*]] = shufflevector <vscale x 2 x i64> [[BROADCAST_SPLATINSERT]], <vscale x 2 x i64> poison, <vscale x 2 x i32> zeroinitializer
; CHECK-NEXT: [[TMP6:%.*]] = icmp ne <vscale x 2 x i64> [[BROADCAST_SPLAT]], zeroinitializer
-; CHECK-NEXT: br label %[[VECTOR_BODY:.*]]
-; CHECK: [[VECTOR_BODY]]:
-; CHECK-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
-; CHECK-NEXT: [[AVL:%.*]] = phi i64 [ 1024, %[[VECTOR_PH]] ], [ [[AVL_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-NEXT: br label [[VECTOR_BODY:%.*]]
+; CHECK: vector.body:
+; CHECK-NEXT: [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ]
+; CHECK-NEXT: [[AVL:%.*]] = phi i64 [ 1024, [[VECTOR_PH]] ], [ [[AVL_NEXT:%.*]], [[VECTOR_BODY]] ]
; CHECK-NEXT: [[TMP12:%.*]] = call i32 @llvm.experimental.get.vector.length.i64(i64 [[AVL]], i32 2, i1 true)
-; CHECK-NEXT: [[TMP8:%.*]] = getelementptr inbounds i64, ptr [[A]], i64 [[INDEX]]
+; CHECK-NEXT: [[TMP8:%.*]] = getelementptr inbounds i64, ptr [[A:%.*]], i64 [[INDEX]]
; CHECK-NEXT: [[WIDE_LOAD:%.*]] = call <vscale x 2 x i64> @llvm.vp.load.nxv2i64.p0(ptr align 8 [[TMP8]], <vscale x 2 x i1> splat (i1 true), i32 [[TMP12]])
; CHECK-NEXT: [[TMP11:%.*]] = call <vscale x 2 x i64> @llvm.vp.udiv.nxv2i64(<vscale x 2 x i64> [[WIDE_LOAD]], <vscale x 2 x i64> [[BROADCAST_SPLAT]], <vscale x 2 x i1> [[TMP6]], i32 [[TMP12]])
; CHECK-NEXT: [[TMP9:%.*]] = extractelement <vscale x 2 x i1> [[TMP6]], i64 0
@@ -295,24 +286,23 @@ define void @predicated_udiv(ptr noalias nocapture %a, i64 %v, i64 %n) {
; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[TMP13]], [[INDEX]]
; CHECK-NEXT: [[AVL_NEXT]] = sub nuw i64 [[AVL]], [[TMP13]]
; CHECK-NEXT: [[TMP14:%.*]] = icmp eq i64 [[AVL_NEXT]], 0
-; CHECK-NEXT: br i1 [[TMP14]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP6:![0-9]+]]
-; CHECK: [[MIDDLE_BLOCK]]:
-; CHECK-NEXT: br label %[[FOR_END:.*]]
-; CHECK: [[FOR_END]]:
+; CHECK-NEXT: br i1 [[TMP14]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP6:![0-9]+]]
+; CHECK: middle.block:
+; CHECK-NEXT: br label [[LATCH:%.*]]
+; CHECK: for.end:
; CHECK-NEXT: ret void
;
-; FIXED-LABEL: define void @predicated_udiv(
-; FIXED-SAME: ptr noalias captures(none) [[A:%.*]], i64 [[V:%.*]], i64 [[N:%.*]]) #[[ATTR0]] {
-; FIXED-NEXT: [[ENTRY:.*:]]
-; FIXED-NEXT: br label %[[VECTOR_PH:.*]]
-; FIXED: [[VECTOR_PH]]:
-; FIXED-NEXT: [[BROADCAST_SPLATINSERT:%.*]] = insertelement <4 x i64> poison, i64 [[V]], i64 0
+; FIXED-LABEL: @predicated_udiv(
+; FIXED-NEXT: entry:
+; FIXED-NEXT: br label [[VECTOR_PH:%.*]]
+; FIXED: vector.ph:
+; FIXED-NEXT: [[BROADCAST_SPLATINSERT:%.*]] = insertelement <4 x i64> poison, i64 [[V:%.*]], i64 0
; FIXED-NEXT: [[BROADCAST_SPLAT:%.*]] = shufflevector <4 x i64> [[BROADCAST_SPLATINSERT]], <4 x i64> poison, <4 x i32> zeroinitializer
; FIXED-NEXT: [[TMP0:%.*]] = icmp ne <4 x i64> [[BROADCAST_SPLAT]], zeroinitializer
-; FIXED-NEXT: br label %[[VECTOR_BODY:.*]]
-; FIXED: [[VECTOR_BODY]]:
-; FIXED-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
-; FIXED-NEXT: [[TMP2:%.*]] = getelementptr inbounds i64, ptr [[A]], i64 [[INDEX]]
+; FIXED-NEXT: br label [[VECTOR_BODY:%.*]]
+; FIXED: vector.body:
+; FIXED-NEXT: [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ]
+; FIXED-NEXT: [[TMP2:%.*]] = getelementptr inbounds i64, ptr [[A:%.*]], i64 [[INDEX]]
; FIXED-NEXT: [[WIDE_LOAD1:%.*]] = load <4 x i64>, ptr [[TMP2]], align 8
; FIXED-NEXT: [[TMP8:%.*]] = call <4 x i64> @llvm.masked.udiv.v4i64(<4 x i64> [[WIDE_LOAD1]], <4 x i64> [[BROADCAST_SPLAT]], <4 x i1> [[TMP0]])
; FIXED-NEXT: [[TMP6:%.*]] = extractelement <4 x i1> [[TMP0]], i64 0
@@ -320,10 +310,10 @@ define void @predicated_udiv(ptr noalias nocapture %a, i64 %v, i64 %n) {
; FIXED-NEXT: store <4 x i64> [[PREDPHI2]], ptr [[TMP2]], align 8
; FIXED-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 4
; FIXED-NEXT: [[TMP9:%.*]] = icmp eq i64 [[INDEX_NEXT]], 1024
-; FIXED-NEXT: br i1 [[TMP9]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP6:![0-9]+]]
-; FIXED: [[MIDDLE_BLOCK]]:
-; FIXED-NEXT: br label %[[FOR_END:.*]]
-; FIXED: [[FOR_END]]:
+; FIXED-NEXT: br i1 [[TMP9]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP6:![0-9]+]]
+; FIXED: middle.block:
+; FIXED-NEXT: br label [[LATCH:%.*]]
+; FIXED: for.end:
; FIXED-NEXT: ret void
;
entry:
@@ -350,20 +340,19 @@ for.end:
}
define void @predicated_sdiv(ptr noalias nocapture %a, i64 %v, i64 %n) {
-; CHECK-LABEL: define void @predicated_sdiv(
-; CHECK-SAME: ptr noalias captures(none) [[A:%.*]], i64 [[V:%.*]], i64 [[N:%.*]]) #[[ATTR0]] {
-; CHECK-NEXT: [[ENTRY:.*:]]
-; CHECK-NEXT: br label %[[VECTOR_PH:.*]]
-; CHECK: [[VECTOR_PH]]:
-; CHECK-NEXT: [[BROADCAST_SPLATINSERT:%.*]] = insertelement <vscale x 2 x i64> poison, i64 [[V]], i64 0
+; CHECK-LABEL: @predicated_sdiv(
+; CHECK-NEXT: entry:
+; CHECK-NEXT: br label [[VECTOR_PH:%.*]]
+; CHECK: vector.ph:
+; CHECK-NEXT: [[BROADCAST_SPLATINSERT:%.*]] = insertelement <vscale x 2 x i64> poison, i64 [[V:%.*]], i64 0
; CHECK-NEXT: [[BROADCAST_SPLAT:%.*]] = shufflevector <vscale x 2 x i64> [[BROADCAST_SPLATINSERT]], <vscale x 2 x i64> poison, <vscale x 2 x i32> zeroinitializer
; CHECK-NEXT: [[TMP6:%.*]] = icmp ne <vscale x 2 x i64> [[BROADCAST_SPLAT]], zeroinitializer
-; CHECK-NEXT: br label %[[VECTOR_BODY:.*]]
-; CHECK: [[VECTOR_BODY]]:
-; CHECK-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
-; CHECK-NEXT: [[AVL:%.*]] = phi i64 [ 1024, %[[VECTOR_PH]] ], [ [[AVL_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-NEXT: br label [[VECTOR_BODY:%.*]]
+; CHECK: vector.body:
+; CHECK-NEXT: [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ]
+; CHECK-NEXT: [[AVL:%.*]] = phi i64 [ 1024, [[VECTOR_PH]] ], [ [[AVL_NEXT:%.*]], [[VECTOR_BODY]] ]
; CHECK-NEXT: [[TMP12:%.*]] = call i32 @llvm.experimental.get.vector.length.i64(i64 [[AVL]], i32 2, i1 true)
-; CHECK-NEXT: [[TMP8:%.*]] = getelementptr inbounds i64, ptr [[A]], i64 [[INDEX]]
+; CHECK-NEXT: [[TMP8:%.*]] = getelementptr inbounds i64, ptr [[A:%.*]], i64 [[INDEX]]
; CHECK-NEXT: [[WIDE_LOAD:%.*]] = call <vscale x 2 x i64> @llvm.vp.load.nxv2i64.p0(ptr align 8 [[TMP8]], <vscale x 2 x i1> splat (i1 true), i32 [[TMP12]])
; CHECK-NEXT: [[TMP11:%.*]] = call <vscale x 2 x i64> @llvm.vp.sdiv.nxv2i64(<vscale x 2 x i64> [[WIDE_LOAD]], <vscale x 2 x i64> [[BROADCAST_SPLAT]], <vscale x 2 x i1> [[TMP6]], i32 [[TMP12]])
; CHECK-NEXT: [[TMP9:%.*]] = extractelement <vscale x 2 x i1> [[TMP6]], i64 0
@@ -373,24 +362,23 @@ define void @predicated_sdiv(ptr noalias nocapture %a, i64 %v, i64 %n) {
; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[TMP13]], [[INDEX]]
; CHECK-NEXT: [[AVL_NEXT]] = sub nuw i64 [[AVL]], [[TMP13]]
; CHECK-NEXT: [[TMP14:%.*]] = icmp eq i64 [[AVL_NEXT]], 0
-; CHECK-NEXT: br i1 [[TMP14]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP7:![0-9]+]]
-; CHECK: [[MIDDLE_BLOCK]]:
-; CHECK-NEXT: br label %[[FOR_END:.*]]
-; CHECK: [[FOR_END]]:
+; CHECK-NEXT: br i1 [[TMP14]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP7:![0-9]+]]
+; CHECK: middle.block:
+; CHECK-NEXT: br label [[LATCH:%.*]]
+; CHECK: for.end:
; CHECK-NEXT: ret void
;
-; FIXED-LABEL: define void @predicated_sdiv(
-; FIXED-SAME: ptr noalias captures(none) [[A:%.*]], i64 [[V:%.*]], i64 [[N:%.*]]) #[[ATTR0]] {
-; FIXED-NEXT: [[ENTRY:.*:]]
-; FIXED-NEXT: br label %[[VECTOR_PH:.*]]
-; FIXED: [[VECTOR_PH]]:
-; FIXED-NEXT: [[BROADCAST_SPLATINSERT:%.*]] = insertelement <4 x i64> poison, i64 [[V]], i64 0
+; FIXED-LABEL: @predicated_sdiv(
+; FIXED-NEXT: entry:
+; FIXED-NEXT: br label [[VECTOR_PH:%.*]]
+; FIXED: vector.ph:
+; FIXED-NEXT: [[BROADCAST_SPLATINSERT:%.*]] = insertelement <4 x i64> poison, i64 [[V:%.*]], i64 0
; FIXED-NEXT: [[BROADCAST_SPLAT:%.*]] = shufflevector <4 x i64> [[BROADCAST_SPLATINSERT]], <4 x i64> poison, <4 x i32> zeroinitializer
; FIXED-NEXT: [[TMP0:%.*]] = icmp ne <4 x i64> [[BROADCAST_SPLAT]], zeroinitializer
-; FIXED-NEXT: br label %[[VECTOR_BODY:.*]]
-; FIXED: [[VECTOR_BODY]]:
-; FIXED-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
-; FIXED-NEXT: [[TMP2:%.*]] = getelementptr inbounds i64, ptr [[A]], i64 [[INDEX]]
+; FIXED-NEXT: br label [[VECTOR_BODY:%.*]]
+; FIXED: vector.body:
+; FIXED-NEXT: [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ]
+; FIXED-NEXT: [[TMP2:%.*]] = getelementptr inbounds i64, ptr [[A:%.*]], i64 [[INDEX]]
; FIXED-NEXT: [[WIDE_LOAD1:%.*]] = load <4 x i64>, ptr [[TMP2]], align 8
; FIXED-NEXT: [[TMP8:%.*]] = call <4 x i64> @llvm.masked.sdiv.v4i64(<4 x i64> [[WIDE_LOAD1]], <4 x i64> [[BROADCAST_SPLAT]], <4 x i1> [[TMP0]])
; FIXED-NEXT: [[TMP6:%.*]] = extractelement <4 x i1> [[TMP0]], i64 0
@@ -398,10 +386,10 @@ define void @predicated_sdiv(ptr noalias nocapture %a, i64 %v, i64 %n) {
; FIXED-NEXT: store <4 x i64> [[PREDPHI2]], ptr [[TMP2]], align 8
; FIXED-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 4
; FIXED-NEXT: [[TMP9:%.*]] = icmp eq i64 [[INDEX_NEXT]], 1024
-; FIXED-NEXT: br i1 [[TMP9]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP7:![0-9]+]]
-; FIXED: [[MIDDLE_BLOCK]]:
-; FIXED-NEXT: br label %[[FOR_END:.*]]
-; FIXED: [[FOR_END]]:
+; FIXED-NEXT: br i1 [[TMP9]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP7:![0-9]+]]
+; FIXED: middle.block:
+; FIXED-NEXT: br label [[LATCH:%.*]]
+; FIXED: for.end:
; FIXED-NEXT: ret void
;
entry:
@@ -428,17 +416,16 @@ for.end:
}
define void @predicated_udiv_by_constant(ptr noalias nocapture %a, i64 %n) {
-; CHECK-LABEL: define void @predicated_udiv_by_constant(
-; CHECK-SAME: ptr noalias captures(none) [[A:%.*]], i64 [[N:%.*]]) #[[ATTR0]] {
-; CHECK-NEXT: [[ENTRY:.*:]]
-; CHECK-NEXT: br label %[[VECTOR_PH:.*]]
-; CHECK: [[VECTOR_PH]]:
-; CHECK-NEXT: br label %[[VECTOR_BODY:.*]]
-; CHECK: [[VECTOR_BODY]]:
-; CHECK-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
-; CHECK-NEXT: [[AVL:%.*]] = phi i64 [ 1024, %[[VECTOR_PH]] ], [ [[AVL_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-LABEL: @predicated_udiv_by_constant(
+; CHECK-NEXT: entry:
+; CHECK-NEXT: br label [[VECTOR_PH:%.*]]
+; CHECK: vector.ph:
+; CHECK-NEXT: br label [[VECTOR_BODY:%.*]]
+; CHECK: vector.body:
+; CHECK-NEXT: [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ]
+; CHECK-NEXT: [[AVL:%.*]] = phi i64 [ 1024, [[VECTOR_PH]] ], [ [[AVL_NEXT:%.*]], [[VECTOR_BODY]] ]
; CHECK-NEXT: [[TMP14:%.*]] = call i32 @llvm.experimental.get.vector.length.i64(i64 [[AVL]], i32 2, i1 true)
-; CHECK-NEXT: [[TMP7:%.*]] = getelementptr inbounds i64, ptr [[A]], i64 [[INDEX]]
+; CHECK-NEXT: [[TMP7:%.*]] = getelementptr inbounds i64, ptr [[A:%.*]], i64 [[INDEX]]
; CHECK-NEXT: [[WIDE_LOAD:%.*]] = call <vscale x 2 x i64> @llvm.vp.load.nxv2i64.p0(ptr align 8 [[TMP7]], <vscale x 2 x i1> splat (i1 true), i32 [[TMP14]])
; CHECK-NEXT: [[TMP2:%.*]] = icmp ne <vscale x 2 x i64> [[WIDE_LOAD]], splat (i64 42)
; CHECK-NEXT: [[TMP10:%.*]] = udiv <vscale x 2 x i64> [[WIDE_LOAD]], splat (i64 27)
@@ -448,21 +435,20 @@ define void @predicated_udiv_by_constant(ptr noalias nocapture %a, i64 %n) {
; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[TMP12]], [[INDEX]]
; CHECK-NEXT: [[AVL_NEXT]] = sub nuw i64 [[AVL]], [[TMP12]]
; CHECK-NEXT: [[TMP13:%.*]] = icmp eq i64 [[AVL_NEXT]], 0
-; CHECK-NEXT: br i1 [[TMP13]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP8:![0-9]+]]
-; CHECK: [[MIDDLE_BLOCK]]:
-; CHECK-NEXT: br label %[[FOR_END:.*]]
-; CHECK: [[FOR_END]]:
+; CHECK-NEXT: br i1 [[TMP13]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP8:![0-9]+]]
+; CHECK: middle.block:
+; CHECK-NEXT: br label [[LATCH:%.*]]
+; CHECK: for.end:
; CHECK-NEXT: ret void
;
-; FIXED-LABEL: define void @predicated_udiv_by_constant(
-; FIXED-SAME: ptr noalias captures(none) [[A:%.*]], i64 [[N:%.*]]) #[[ATTR0]] {
-; FIXED-NEXT: [[ENTRY:.*:]]
-; FIXED-NEXT: br label %[[VECTOR_PH:.*]]
-; FIXED: [[VECTOR_PH]]:
-; FIXED-NEXT: br label %[[VECTOR_BODY:.*]]
-; FIXED: [[VECTOR_BODY]]:
-; FIXED-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
-; FIXED-NEXT: [[TMP1:%.*]] = getelementptr inbounds i64, ptr [[A]], i64 [[INDEX]]
+; FIXED-LABEL: @predicated_udiv_by_constant(
+; FIXED-NEXT: entry:
+; FIXED-NEXT: br label [[VECTOR_PH:%.*]]
+; FIXED: vector.ph:
+; FIXED-NEXT: br label [[VECTOR_BODY:%.*]]
+; FIXED: vector.body:
+; FIXED-NEXT: [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ]
+; FIXED-NEXT: [[TMP1:%.*]] = getelementptr inbounds i64, ptr [[A:%.*]], i64 [[INDEX]]
; FIXED-NEXT: [[WIDE_LOAD1:%.*]] = load <4 x i64>, ptr [[TMP1]], align 8
; FIXED-NEXT: [[TMP5:%.*]] = icmp ne <4 x i64> [[WIDE_LOAD1]], splat (i64 42)
; FIXED-NEXT: [[TMP7:%.*]] = udiv <4 x i64> [[WIDE_LOAD1]], splat (i64 27)
@@ -470,10 +456,10 @@ define void @predicated_udiv_by_constant(ptr noalias nocapture %a, i64 %n) {
; FIXED-NEXT: store <4 x i64> [[PREDPHI2]], ptr [[TMP1]], align 8
; FIXED-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 4
; FIXED-NEXT: [[TMP8:%.*]] = icmp eq i64 [[INDEX_NEXT]], 1024
-; FIXED-NEXT: br i1 [[TMP8]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP8:![0-9]+]]
-; FIXED: [[MIDDLE_BLOCK]]:
-; FIXED-NEXT: br label %[[FOR_END:.*]]
-; FIXED: [[FOR_END]]:
+; FIXED-NEXT: br i1 [[TMP8]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP8:![0-9]+]]
+; FIXED: middle.block:
+; FIXED-NEXT: br label [[LATCH:%.*]]
+; FIXED: for.end:
; FIXED-NEXT: ret void
;
entry:
@@ -500,17 +486,16 @@ for.end:
}
define void @predicated_sdiv_by_constant(ptr noalias nocapture %a, i64 %n) {
-; CHECK-LABEL: define void @predicated_sdiv_by_constant(
-; CHECK-SAME: ptr noalias captures(none) [[A:%.*]], i64 [[N:%.*]]) #[[ATTR0]] {
-; CHECK-NEXT: [[ENTRY:.*:]]
-; CHECK-NEXT: br label %[[VECTOR_PH:.*]]
-; CHECK: [[VECTOR_PH]]:
-; CHECK-NEXT: br label %[[VECTOR_BODY:.*]]
-; CHECK: [[VECTOR_BODY]]:
-; CHECK-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
-; CHECK-NEXT: [[AVL:%.*]] = phi i64 [ 1024, %[[VECTOR_PH]] ], [ [[AVL_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-LABEL: @predicated_sdiv_by_constant(
+; CHECK-NEXT: entry:
+; CHECK-NEXT: br label [[VECTOR_PH:%.*]]
+; CHECK: vector.ph:
+; CHECK-NEXT: br label [[VECTOR_BODY:%.*]]
+; CHECK: vector.body:
+; CHECK-NEXT: [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ]
+; CHECK-NEXT: [[AVL:%.*]] = phi i64 [ 1024, [[VECTOR_PH]] ], [ [[AVL_NEXT:%.*]], [[VECTOR_BODY]] ]
; CHECK-NEXT: [[TMP14:%.*]] = call i32 @llvm.experimental.get.vector.length.i64(i64 [[AVL]], i32 2, i1 true)
-; CHECK-NEXT: [[TMP7:%.*]] = getelementptr inbounds i64, ptr [[A]], i64 [[INDEX]]
+; CHECK-NEXT: [[TMP7:%.*]] = getelementptr inbounds i64, ptr [[A:%.*]], i64 [[INDEX]]
; CHECK-NEXT: [[WIDE_LOAD:%.*]] = call <vscale x 2 x i64> @llvm.vp.load.nxv2i64.p0(ptr align 8 [[TMP7]], <vscale x 2 x i1> splat (i1 true), i32 [[TMP14]])
; CHECK-NEXT: [[TMP2:%.*]] = icmp ne <vscale x 2 x i64> [[WIDE_LOAD]], splat (i64 42)
; CHECK-NEXT: [[TMP10:%.*]] = sdiv <vscale x 2 x i64> [[WIDE_LOAD]], splat (i64 27)
@@ -520,21 +505,20 @@ define void @predicated_sdiv_by_constant(ptr noalias nocapture %a, i64 %n) {
; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[TMP12]], [[INDEX]]
; CHECK-NEXT: [[AVL_NEXT]] = sub nuw i64 [[AVL]], [[TMP12]]
; CHECK-NEXT: [[TMP13:%.*]] = icmp eq i64 [[AVL_NEXT]], 0
-; CHECK-NEXT: br i1 [[TMP13]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP9:![0-9]+]]
-; CHECK: [[MIDDLE_BLOCK]]:
-; CHECK-NEXT: br label %[[FOR_END:.*]]
-; CHECK: [[FOR_END]]:
+; CHECK-NEXT: br i1 [[TMP13]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP9:![0-9]+]]
+; CHECK: middle.block:
+; CHECK-NEXT: br label [[LATCH:%.*]]
+; CHECK: for.end:
; CHECK-NEXT: ret void
;
-; FIXED-LABEL: define void @predicated_sdiv_by_constant(
-; FIXED-SAME: ptr noalias captures(none) [[A:%.*]], i64 [[N:%.*]]) #[[ATTR0]] {
-; FIXED-NEXT: [[ENTRY:.*:]]
-; FIXED-NEXT: br label %[[VECTOR_PH:.*]]
-; FIXED: [[VECTOR_PH]]:
-; FIXED-NEXT: br label %[[VECTOR_BODY:.*]]
-; FIXED: [[VECTOR_BODY]]:
-; FIXED-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
-; FIXED-NEXT: [[TMP1:%.*]] = getelementptr inbounds i64, ptr [[A]], i64 [[INDEX]]
+; FIXED-LABEL: @predicated_sdiv_by_constant(
+; FIXED-NEXT: entry:
+; FIXED-NEXT: br label [[VECTOR_PH:%.*]]
+; FIXED: vector.ph:
+; FIXED-NEXT: br label [[VECTOR_BODY:%.*]]
+; FIXED: vector.body:
+; FIXED-NEXT: [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ]
+; FIXED-NEXT: [[TMP1:%.*]] = getelementptr inbounds i64, ptr [[A:%.*]], i64 [[INDEX]]
; FIXED-NEXT: [[WIDE_LOAD1:%.*]] = load <4 x i64>, ptr [[TMP1]], align 8
; FIXED-NEXT: [[TMP5:%.*]] = icmp ne <4 x i64> [[WIDE_LOAD1]], splat (i64 42)
; FIXED-NEXT: [[TMP7:%.*]] = sdiv <4 x i64> [[WIDE_LOAD1]], splat (i64 27)
@@ -542,10 +526,10 @@ define void @predicated_sdiv_by_constant(ptr noalias nocapture %a, i64 %n) {
; FIXED-NEXT: store <4 x i64> [[PREDPHI2]], ptr [[TMP1]], align 8
; FIXED-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 4
; FIXED-NEXT: [[TMP8:%.*]] = icmp eq i64 [[INDEX_NEXT]], 1024
-; FIXED-NEXT: br i1 [[TMP8]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP9:![0-9]+]]
-; FIXED: [[MIDDLE_BLOCK]]:
-; FIXED-NEXT: br label %[[FOR_END:.*]]
-; FIXED: [[FOR_END]]:
+; FIXED-NEXT: br i1 [[TMP8]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP9:![0-9]+]]
+; FIXED: middle.block:
+; FIXED-NEXT: br label [[LATCH:%.*]]
+; FIXED: for.end:
; FIXED-NEXT: ret void
;
entry:
@@ -572,17 +556,16 @@ for.end:
}
define void @predicated_sdiv_by_minus_one(ptr noalias nocapture %a, i64 %n) {
-; CHECK-LABEL: define void @predicated_sdiv_by_minus_one(
-; CHECK-SAME: ptr noalias captures(none) [[A:%.*]], i64 [[N:%.*]]) #[[ATTR0]] {
-; CHECK-NEXT: [[ENTRY:.*:]]
-; CHECK-NEXT: br label %[[VECTOR_PH:.*]]
-; CHECK: [[VECTOR_PH]]:
-; CHECK-NEXT: br label %[[VECTOR_BODY:.*]]
-; CHECK: [[VECTOR_BODY]]:
-; CHECK-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
-; CHECK-NEXT: [[AVL:%.*]] = phi i64 [ 1024, %[[VECTOR_PH]] ], [ [[AVL_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-LABEL: @predicated_sdiv_by_minus_one(
+; CHECK-NEXT: entry:
+; CHECK-NEXT: br label [[VECTOR_PH:%.*]]
+; CHECK: vector.ph:
+; CHECK-NEXT: br label [[VECTOR_BODY:%.*]]
+; CHECK: vector.body:
+; CHECK-NEXT: [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ]
+; CHECK-NEXT: [[AVL:%.*]] = phi i64 [ 1024, [[VECTOR_PH]] ], [ [[AVL_NEXT:%.*]], [[VECTOR_BODY]] ]
; CHECK-NEXT: [[TMP12:%.*]] = call i32 @llvm.experimental.get.vector.length.i64(i64 [[AVL]], i32 16, i1 true)
-; CHECK-NEXT: [[TMP7:%.*]] = getelementptr inbounds i8, ptr [[A]], i64 [[INDEX]]
+; CHECK-NEXT: [[TMP7:%.*]] = getelementptr inbounds i8, ptr [[A:%.*]], i64 [[INDEX]]
; CHECK-NEXT: [[WIDE_LOAD:%.*]] = call <vscale x 16 x i8> @llvm.vp.load.nxv16i8.p0(ptr align 1 [[TMP7]], <vscale x 16 x i1> splat (i1 true), i32 [[TMP12]])
; CHECK-NEXT: [[TMP9:%.*]] = icmp ne <vscale x 16 x i8> [[WIDE_LOAD]], splat (i8 -128)
; CHECK-NEXT: [[TMP11:%.*]] = call <vscale x 16 x i8> @llvm.vp.sdiv.nxv16i8(<vscale x 16 x i8> [[WIDE_LOAD]], <vscale x 16 x i8> splat (i8 -1), <vscale x 16 x i1> [[TMP9]], i32 [[TMP12]])
@@ -592,21 +575,20 @@ define void @predicated_sdiv_by_minus_one(ptr noalias nocapture %a, i64 %n) {
; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[TMP13]], [[INDEX]]
; CHECK-NEXT: [[AVL_NEXT]] = sub nuw i64 [[AVL]], [[TMP13]]
; CHECK-NEXT: [[TMP14:%.*]] = icmp eq i64 [[AVL_NEXT]], 0
-; CHECK-NEXT: br i1 [[TMP14]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP10:![0-9]+]]
-; CHECK: [[MIDDLE_BLOCK]]:
-; CHECK-NEXT: br label %[[FOR_END:.*]]
-; CHECK: [[FOR_END]]:
+; CHECK-NEXT: br i1 [[TMP14]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP10:![0-9]+]]
+; CHECK: middle.block:
+; CHECK-NEXT: br label [[LATCH:%.*]]
+; CHECK: for.end:
; CHECK-NEXT: ret void
;
-; FIXED-LABEL: define void @predicated_sdiv_by_minus_one(
-; FIXED-SAME: ptr noalias captures(none) [[A:%.*]], i64 [[N:%.*]]) #[[ATTR0]] {
-; FIXED-NEXT: [[ENTRY:.*:]]
-; FIXED-NEXT: br label %[[VECTOR_PH:.*]]
-; FIXED: [[VECTOR_PH]]:
-; FIXED-NEXT: br label %[[VECTOR_BODY:.*]]
-; FIXED: [[VECTOR_BODY]]:
-; FIXED-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
-; FIXED-NEXT: [[TMP1:%.*]] = getelementptr inbounds i8, ptr [[A]], i64 [[INDEX]]
+; FIXED-LABEL: @predicated_sdiv_by_minus_one(
+; FIXED-NEXT: entry:
+; FIXED-NEXT: br label [[VECTOR_PH:%.*]]
+; FIXED: vector.ph:
+; FIXED-NEXT: br label [[VECTOR_BODY:%.*]]
+; FIXED: vector.body:
+; FIXED-NEXT: [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ]
+; FIXED-NEXT: [[TMP1:%.*]] = getelementptr inbounds i8, ptr [[A:%.*]], i64 [[INDEX]]
; FIXED-NEXT: [[WIDE_LOAD1:%.*]] = load <32 x i8>, ptr [[TMP1]], align 1
; FIXED-NEXT: [[TMP5:%.*]] = icmp ne <32 x i8> [[WIDE_LOAD1]], splat (i8 -128)
; FIXED-NEXT: [[TMP9:%.*]] = call <32 x i8> @llvm.masked.sdiv.v32i8(<32 x i8> [[WIDE_LOAD1]], <32 x i8> splat (i8 -1), <32 x i1> [[TMP5]])
@@ -614,10 +596,10 @@ define void @predicated_sdiv_by_minus_one(ptr noalias nocapture %a, i64 %n) {
; FIXED-NEXT: store <32 x i8> [[PREDPHI2]], ptr [[TMP1]], align 1
; FIXED-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 32
; FIXED-NEXT: [[TMP10:%.*]] = icmp eq i64 [[INDEX_NEXT]], 1024
-; FIXED-NEXT: br i1 [[TMP10]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP10:![0-9]+]]
-; FIXED: [[MIDDLE_BLOCK]]:
-; FIXED-NEXT: br label %[[FOR_END:.*]]
-; FIXED: [[FOR_END]]:
+; FIXED-NEXT: br i1 [[TMP10]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP10:![0-9]+]]
+; FIXED: middle.block:
+; FIXED-NEXT: br label [[LATCH:%.*]]
+; FIXED: for.end:
; FIXED-NEXT: ret void
;
entry:
@@ -647,96 +629,98 @@ for.end:
; selects can be introduced outside the vector loop and their cost should not be
; considered for each loop iteration.
define void @udiv_sdiv_with_invariant_divisors(i8 %x, i16 %y, i1 %c, ptr %p) {
-; CHECK-LABEL: define void @udiv_sdiv_with_invariant_divisors(
-; CHECK-SAME: i8 [[X:%.*]], i16 [[Y:%.*]], i1 [[C:%.*]], ptr [[P:%.*]]) #[[ATTR0]] {
-; CHECK-NEXT: [[ENTRY:.*:]]
-; CHECK-NEXT: br label %[[VECTOR_BODY:.*]]
-; CHECK: [[VECTOR_BODY]]:
-; CHECK-NEXT: [[BROADCAST_SPLATINSERT1:%.*]] = insertelement <vscale x 4 x i8> poison, i8 [[X]], i64 0
+; CHECK-LABEL: @udiv_sdiv_with_invariant_divisors(
+; CHECK-NEXT: entry:
+; CHECK-NEXT: br label [[VECTOR_BODY:%.*]]
+; CHECK: vector.ph:
+; CHECK-NEXT: [[BROADCAST_SPLATINSERT1:%.*]] = insertelement <vscale x 4 x i8> poison, i8 [[X:%.*]], i64 0
; CHECK-NEXT: [[BROADCAST_SPLAT2:%.*]] = shufflevector <vscale x 4 x i8> [[BROADCAST_SPLATINSERT1]], <vscale x 4 x i8> poison, <vscale x 4 x i32> zeroinitializer
-; CHECK-NEXT: [[BROADCAST_SPLATINSERT3:%.*]] = insertelement <vscale x 4 x i16> poison, i16 [[Y]], i64 0
+; CHECK-NEXT: [[BROADCAST_SPLATINSERT3:%.*]] = insertelement <vscale x 4 x i16> poison, i16 [[Y:%.*]], i64 0
; CHECK-NEXT: [[BROADCAST_SPLAT4:%.*]] = shufflevector <vscale x 4 x i16> [[BROADCAST_SPLATINSERT3]], <vscale x 4 x i16> poison, <vscale x 4 x i32> zeroinitializer
-; CHECK-NEXT: [[BROADCAST_SPLATINSERT5:%.*]] = insertelement <vscale x 4 x ptr> poison, ptr [[P]], i64 0
+; CHECK-NEXT: [[BROADCAST_SPLATINSERT5:%.*]] = insertelement <vscale x 4 x ptr> poison, ptr [[P:%.*]], i64 0
; CHECK-NEXT: [[BROADCAST_SPLAT6:%.*]] = shufflevector <vscale x 4 x ptr> [[BROADCAST_SPLATINSERT5]], <vscale x 4 x ptr> poison, <vscale x 4 x i32> zeroinitializer
; CHECK-NEXT: [[TMP1:%.*]] = call <vscale x 4 x i8> @llvm.stepvector.nxv4i8()
; CHECK-NEXT: [[INDUCTION:%.*]] = add <vscale x 4 x i8> splat (i8 -12), [[TMP1]]
-; CHECK-NEXT: br label %[[LOOP_LATCH:.*]]
-; CHECK: [[LOOP_LATCH]]:
-; CHECK-NEXT: [[VEC_IND:%.*]] = phi <vscale x 4 x i8> [ [[INDUCTION]], %[[VECTOR_BODY]] ], [ [[VEC_IND_NEXT:%.*]], %[[LOOP_LATCH8:.*]] ]
-; CHECK-NEXT: [[AVL:%.*]] = phi i32 [ 12, %[[VECTOR_BODY]] ], [ [[AVL_NEXT:%.*]], %[[LOOP_LATCH8]] ]
+; CHECK-NEXT: br label [[LOOP_LATCH:%.*]]
+; CHECK: vector.body:
+; CHECK-NEXT: [[VEC_IND:%.*]] = phi <vscale x 4 x i8> [ [[INDUCTION]], [[VECTOR_BODY]] ], [ [[VEC_IND_NEXT:%.*]], [[LOOP_LATCH8:%.*]] ]
+; CHECK-NEXT: [[AVL:%.*]] = phi i32 [ 12, [[VECTOR_BODY]] ], [ [[AVL_NEXT:%.*]], [[LOOP_LATCH8]] ]
; CHECK-NEXT: [[TMP2:%.*]] = call i32 @llvm.experimental.get.vector.length.i32(i32 [[AVL]], i32 4, i1 true)
; CHECK-NEXT: [[TMP3:%.*]] = trunc i32 [[TMP2]] to i8
; CHECK-NEXT: [[BROADCAST_SPLATINSERT7:%.*]] = insertelement <vscale x 4 x i8> poison, i8 [[TMP3]], i64 0
; CHECK-NEXT: [[BROADCAST_SPLAT8:%.*]] = shufflevector <vscale x 4 x i8> [[BROADCAST_SPLATINSERT7]], <vscale x 4 x i8> poison, <vscale x 4 x i32> zeroinitializer
-; CHECK-NEXT: br i1 [[C]], label %[[LOOP_LATCH8]], label %[[THEN7:.*]]
-; CHECK: [[THEN7]]:
-; CHECK-NEXT: [[TMP13:%.*]] = call <vscale x 4 x i8> @llvm.vp.merge.nxv4i8(<vscale x 4 x i1> splat (i1 true), <vscale x 4 x i8> [[BROADCAST_SPLAT2]], <vscale x 4 x i8> splat (i8 1), i32 [[TMP2]])
-; CHECK-NEXT: [[TMP14:%.*]] = extractelement <vscale x 4 x i8> [[VEC_IND]], i64 0
-; CHECK-NEXT: [[TMP5:%.*]] = extractelement <vscale x 4 x i8> [[TMP13]], i64 0
-; CHECK-NEXT: [[TMP6:%.*]] = udiv i8 [[TMP14]], [[TMP5]]
-; CHECK-NEXT: [[TMP7:%.*]] = zext i8 [[TMP6]] to i16
-; CHECK-NEXT: [[TMP8:%.*]] = call <vscale x 4 x i16> @llvm.vp.merge.nxv4i16(<vscale x 4 x i1> splat (i1 true), <vscale x 4 x i16> [[BROADCAST_SPLAT4]], <vscale x 4 x i16> splat (i16 1), i32 [[TMP2]])
-; CHECK-NEXT: [[TMP9:%.*]] = extractelement <vscale x 4 x i16> [[TMP8]], i64 0
-; CHECK-NEXT: [[TMP10:%.*]] = sdiv i16 [[TMP7]], [[TMP9]]
-; CHECK-NEXT: [[TMP11:%.*]] = sext i16 [[TMP10]] to i32
-; CHECK-NEXT: br label %[[LOOP_LATCH8]]
-; CHECK: [[LOOP_LATCH8]]:
-; CHECK-NEXT: [[TMP12:%.*]] = phi i32 [ poison, %[[LOOP_LATCH]] ], [ [[TMP11]], %[[THEN7]] ]
-; CHECK-NEXT: [[BROADCAST_SPLATINSERT9:%.*]] = insertelement <vscale x 4 x i32> poison, i32 [[TMP12]], i64 0
+; CHECK-NEXT: [[TMP17:%.*]] = call <vscale x 4 x i32> @llvm.stepvector.nxv4i32()
+; CHECK-NEXT: [[TMP18:%.*]] = extractelement <vscale x 4 x i32> [[TMP17]], i64 0
+; CHECK-NEXT: [[TMP5:%.*]] = icmp ult i32 [[TMP18]], [[TMP2]]
+; CHECK-NEXT: br i1 [[C:%.*]], label [[LOOP_LATCH8]], label [[THEN7:%.*]]
+; CHECK: then7:
+; CHECK-NEXT: [[TMP6:%.*]] = call <vscale x 4 x i8> @llvm.vp.merge.nxv4i8(<vscale x 4 x i1> splat (i1 true), <vscale x 4 x i8> [[BROADCAST_SPLAT2]], <vscale x 4 x i8> splat (i8 1), i32 [[TMP2]])
+; CHECK-NEXT: [[TMP7:%.*]] = extractelement <vscale x 4 x i8> [[VEC_IND]], i64 0
+; CHECK-NEXT: [[TMP8:%.*]] = extractelement <vscale x 4 x i8> [[TMP6]], i64 0
+; CHECK-NEXT: [[TMP9:%.*]] = udiv i8 [[TMP7]], [[TMP8]]
+; CHECK-NEXT: [[TMP10:%.*]] = zext i8 [[TMP9]] to i16
+; CHECK-NEXT: [[TMP11:%.*]] = call <vscale x 4 x i16> @llvm.vp.merge.nxv4i16(<vscale x 4 x i1> splat (i1 true), <vscale x 4 x i16> [[BROADCAST_SPLAT4]], <vscale x 4 x i16> splat (i16 1), i32 [[TMP2]])
+; CHECK-NEXT: [[TMP12:%.*]] = extractelement <vscale x 4 x i16> [[TMP11]], i64 0
+; CHECK-NEXT: [[TMP13:%.*]] = sdiv i16 [[TMP10]], [[TMP12]]
+; CHECK-NEXT: [[TMP14:%.*]] = sext i16 [[TMP13]] to i32
+; CHECK-NEXT: br label [[LOOP_LATCH8]]
+; CHECK: loop.latch8:
+; CHECK-NEXT: [[TMP15:%.*]] = phi i32 [ poison, [[LOOP_LATCH]] ], [ [[TMP14]], [[THEN7]] ]
+; CHECK-NEXT: [[TMP16:%.*]] = phi i1 [ false, [[LOOP_LATCH]] ], [ [[TMP5]], [[THEN7]] ]
+; CHECK-NEXT: [[BROADCAST_SPLATINSERT9:%.*]] = insertelement <vscale x 4 x i32> poison, i32 [[TMP15]], i64 0
; CHECK-NEXT: [[BROADCAST_SPLAT10:%.*]] = shufflevector <vscale x 4 x i32> [[BROADCAST_SPLATINSERT9]], <vscale x 4 x i32> poison, <vscale x 4 x i32> zeroinitializer
-; CHECK-NEXT: [[PREDPHI:%.*]] = call <vscale x 4 x i32> @llvm.vp.merge.nxv4i32(<vscale x 4 x i1> splat (i1 true), <vscale x 4 x i32> [[BROADCAST_SPLAT10]], <vscale x 4 x i32> zeroinitializer, i32 [[TMP2]])
+; CHECK-NEXT: [[PREDPHI:%.*]] = select i1 [[TMP16]], <vscale x 4 x i32> [[BROADCAST_SPLAT10]], <vscale x 4 x i32> zeroinitializer
; CHECK-NEXT: call void @llvm.vp.scatter.nxv4i32.nxv4p0(<vscale x 4 x i32> [[PREDPHI]], <vscale x 4 x ptr> align 4 [[BROADCAST_SPLAT6]], <vscale x 4 x i1> splat (i1 true), i32 [[TMP2]])
; CHECK-NEXT: [[AVL_NEXT]] = sub nuw i32 [[AVL]], [[TMP2]]
; CHECK-NEXT: [[VEC_IND_NEXT]] = add <vscale x 4 x i8> [[VEC_IND]], [[BROADCAST_SPLAT8]]
; CHECK-NEXT: [[TMP4:%.*]] = icmp eq i32 [[AVL_NEXT]], 0
-; CHECK-NEXT: br i1 [[TMP4]], label %[[MIDDLE_BLOCK:.*]], label %[[LOOP_LATCH]], !llvm.loop [[LOOP11:![0-9]+]]
-; CHECK: [[MIDDLE_BLOCK]]:
-; CHECK-NEXT: br label %[[EXIT:.*]]
-; CHECK: [[EXIT]]:
+; CHECK-NEXT: br i1 [[TMP4]], label [[MIDDLE_BLOCK:%.*]], label [[LOOP_LATCH]], !llvm.loop [[LOOP11:![0-9]+]]
+; CHECK: middle.block:
+; CHECK-NEXT: br label [[EXIT:%.*]]
+; CHECK: exit:
; CHECK-NEXT: ret void
;
-; FIXED-LABEL: define void @udiv_sdiv_with_invariant_divisors(
-; FIXED-SAME: i8 [[X:%.*]], i16 [[Y:%.*]], i1 [[C:%.*]], ptr [[P:%.*]]) #[[ATTR0]] {
-; FIXED-NEXT: [[ENTRY:.*:]]
-; FIXED-NEXT: br label %[[VECTOR_PH:.*]]
-; FIXED: [[VECTOR_PH]]:
-; FIXED-NEXT: br label %[[VECTOR_BODY:.*]]
-; FIXED: [[VECTOR_BODY]]:
-; FIXED-NEXT: br i1 [[C]], label %[[LOOP_LATCH2:.*]], label %[[THEN1:.*]]
-; FIXED: [[THEN1]]:
-; FIXED-NEXT: [[TMP0:%.*]] = udiv i8 -12, [[X]]
+; FIXED-LABEL: @udiv_sdiv_with_invariant_divisors(
+; FIXED-NEXT: entry:
+; FIXED-NEXT: br label [[VECTOR_PH:%.*]]
+; FIXED: vector.ph:
+; FIXED-NEXT: br label [[VECTOR_BODY:%.*]]
+; FIXED: vector.body:
+; FIXED-NEXT: br i1 [[C:%.*]], label [[LOOP_LATCH2:%.*]], label [[THEN1:%.*]]
+; FIXED: then1:
+; FIXED-NEXT: [[TMP0:%.*]] = udiv i8 -12, [[X:%.*]]
; FIXED-NEXT: [[TMP1:%.*]] = zext i8 [[TMP0]] to i16
-; FIXED-NEXT: [[TMP2:%.*]] = sdiv i16 [[TMP1]], [[Y]]
+; FIXED-NEXT: [[TMP2:%.*]] = sdiv i16 [[TMP1]], [[Y:%.*]]
; FIXED-NEXT: [[TMP3:%.*]] = sext i16 [[TMP2]] to i32
-; FIXED-NEXT: br label %[[LOOP_LATCH2]]
-; FIXED: [[LOOP_LATCH2]]:
-; FIXED-NEXT: [[TMP4:%.*]] = phi i32 [ poison, %[[VECTOR_BODY]] ], [ [[TMP3]], %[[THEN1]] ]
-; FIXED-NEXT: [[TMP5:%.*]] = phi i1 [ false, %[[VECTOR_BODY]] ], [ true, %[[THEN1]] ]
+; FIXED-NEXT: br label [[LOOP_LATCH2]]
+; FIXED: loop.latch2:
+; FIXED-NEXT: [[TMP4:%.*]] = phi i32 [ poison, [[VECTOR_BODY]] ], [ [[TMP3]], [[THEN1]] ]
+; FIXED-NEXT: [[TMP5:%.*]] = phi i1 [ false, [[VECTOR_BODY]] ], [ true, [[THEN1]] ]
; FIXED-NEXT: [[TMP7:%.*]] = select i1 [[TMP5]], i32 [[TMP4]], i32 0
-; FIXED-NEXT: store i32 [[TMP7]], ptr [[P]], align 4
-; FIXED-NEXT: br label %[[MIDDLE_BLOCK:.*]]
-; FIXED: [[MIDDLE_BLOCK]]:
-; FIXED-NEXT: br label %[[SCALAR_PH:.*]]
-; FIXED: [[SCALAR_PH]]:
-; FIXED-NEXT: br label %[[LOOP_HEADER:.*]]
-; FIXED: [[LOOP_HEADER]]:
-; FIXED-NEXT: [[IV:%.*]] = phi i16 [ -4, %[[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], %[[LOOP_LATCH1:.*]] ]
-; FIXED-NEXT: [[NARROW_IV:%.*]] = phi i8 [ -4, %[[SCALAR_PH]] ], [ [[IV_NEXT_TRUNC:%.*]], %[[LOOP_LATCH1]] ]
-; FIXED-NEXT: br i1 [[C]], label %[[LOOP_LATCH1]], label %[[THEN:.*]]
-; FIXED: [[THEN]]:
+; FIXED-NEXT: store i32 [[TMP7]], ptr [[P:%.*]], align 4
+; FIXED-NEXT: br label [[LOOP_LATCH:%.*]]
+; FIXED: middle.block:
+; FIXED-NEXT: br label [[SCALAR_PH:%.*]]
+; FIXED: scalar.ph:
+; FIXED-NEXT: br label [[LOOP_HEADER:%.*]]
+; FIXED: loop.header:
+; FIXED-NEXT: [[IV:%.*]] = phi i16 [ -4, [[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], [[LOOP_LATCH1:%.*]] ]
+; FIXED-NEXT: [[NARROW_IV:%.*]] = phi i8 [ -4, [[SCALAR_PH]] ], [ [[IV_NEXT_TRUNC:%.*]], [[LOOP_LATCH1]] ]
+; FIXED-NEXT: br i1 [[C]], label [[LOOP_LATCH1]], label [[THEN:%.*]]
+; FIXED: then:
; FIXED-NEXT: [[UD:%.*]] = udiv i8 [[NARROW_IV]], [[X]]
; FIXED-NEXT: [[UD_EXT:%.*]] = zext i8 [[UD]] to i16
; FIXED-NEXT: [[SD:%.*]] = sdiv i16 [[UD_EXT]], [[Y]]
; FIXED-NEXT: [[SD_EXT:%.*]] = sext i16 [[SD]] to i32
-; FIXED-NEXT: br label %[[LOOP_LATCH1]]
-; FIXED: [[LOOP_LATCH1]]:
-; FIXED-NEXT: [[MERGE:%.*]] = phi i32 [ 0, %[[LOOP_HEADER]] ], [ [[SD_EXT]], %[[THEN]] ]
+; FIXED-NEXT: br label [[LOOP_LATCH1]]
+; FIXED: loop.latch:
+; FIXED-NEXT: [[MERGE:%.*]] = phi i32 [ 0, [[LOOP_HEADER]] ], [ [[SD_EXT]], [[THEN]] ]
; FIXED-NEXT: store i32 [[MERGE]], ptr [[P]], align 4
; FIXED-NEXT: [[IV_NEXT]] = add nsw i16 [[IV]], 1
; FIXED-NEXT: [[EC:%.*]] = icmp eq i16 [[IV_NEXT]], 0
; FIXED-NEXT: [[IV_NEXT_TRUNC]] = trunc i16 [[IV_NEXT]] to i8
-; FIXED-NEXT: br i1 [[EC]], label %[[EXIT:.*]], label %[[LOOP_HEADER]], !llvm.loop [[LOOP11:![0-9]+]]
-; FIXED: [[EXIT]]:
+; FIXED-NEXT: br i1 [[EC]], label [[EXIT:%.*]], label [[LOOP_HEADER]], !llvm.loop [[LOOP11:![0-9]+]]
+; FIXED: exit:
; FIXED-NEXT: ret void
;
entry:
@@ -765,30 +749,3 @@ loop.latch:
exit:
ret void
}
-;.
-; CHECK: [[LOOP0]] = distinct !{[[LOOP0]], [[META1:![0-9]+]], [[META2:![0-9]+]]}
-; CHECK: [[META1]] = !{!"llvm.loop.isvectorized", i32 1}
-; CHECK: [[META2]] = !{!"llvm.loop.unroll.runtime.disable"}
-; CHECK: [[LOOP3]] = distinct !{[[LOOP3]], [[META1]], [[META2]]}
-; CHECK: [[LOOP4]] = distinct !{[[LOOP4]], [[META1]], [[META2]]}
-; CHECK: [[LOOP5]] = distinct !{[[LOOP5]], [[META1]], [[META2]]}
-; CHECK: [[LOOP6]] = distinct !{[[LOOP6]], [[META1]], [[META2]]}
-; CHECK: [[LOOP7]] = distinct !{[[LOOP7]], [[META1]], [[META2]]}
-; CHECK: [[LOOP8]] = distinct !{[[LOOP8]], [[META1]], [[META2]]}
-; CHECK: [[LOOP9]] = distinct !{[[LOOP9]], [[META1]], [[META2]]}
-; CHECK: [[LOOP10]] = distinct !{[[LOOP10]], [[META1]], [[META2]]}
-; CHECK: [[LOOP11]] = distinct !{[[LOOP11]], [[META1]], [[META2]]}
-;.
-; FIXED: [[LOOP0]] = distinct !{[[LOOP0]], [[META1:![0-9]+]], [[META2:![0-9]+]]}
-; FIXED: [[META1]] = !{!"llvm.loop.isvectorized", i32 1}
-; FIXED: [[META2]] = !{!"llvm.loop.unroll.runtime.disable"}
-; FIXED: [[LOOP3]] = distinct !{[[LOOP3]], [[META1]], [[META2]]}
-; FIXED: [[LOOP4]] = distinct !{[[LOOP4]], [[META1]], [[META2]]}
-; FIXED: [[LOOP5]] = distinct !{[[LOOP5]], [[META1]], [[META2]]}
-; FIXED: [[LOOP6]] = distinct !{[[LOOP6]], [[META1]], [[META2]]}
-; FIXED: [[LOOP7]] = distinct !{[[LOOP7]], [[META1]], [[META2]]}
-; FIXED: [[LOOP8]] = distinct !{[[LOOP8]], [[META1]], [[META2]]}
-; FIXED: [[LOOP9]] = distinct !{[[LOOP9]], [[META1]], [[META2]]}
-; FIXED: [[LOOP10]] = distinct !{[[LOOP10]], [[META1]], [[META2]]}
-; FIXED: [[LOOP11]] = distinct !{[[LOOP11]], [[META2]], [[META1]]}
-;.
diff --git a/llvm/test/Transforms/LoopVectorize/RISCV/gather-scatter-cost.ll b/llvm/test/Transforms/LoopVectorize/RISCV/gather-scatter-cost.ll
index fc9fef1d1d11f..b0b9e6c3723c0 100644
--- a/llvm/test/Transforms/LoopVectorize/RISCV/gather-scatter-cost.ll
+++ b/llvm/test/Transforms/LoopVectorize/RISCV/gather-scatter-cost.ll
@@ -36,19 +36,23 @@ define void @predicated_uniform_load(ptr %src, i32 %n, ptr %dst, i1 %cond) {
; CHECK: vector.body:
; CHECK-NEXT: [[AVL:%.*]] = phi i32 [ [[TMP3]], [[VECTOR_PH]] ], [ [[AVL_NEXT:%.*]], [[LOOP_LATCH6:%.*]] ]
; CHECK-NEXT: [[TMP10:%.*]] = call i32 @llvm.experimental.get.vector.length.i32(i32 [[AVL]], i32 4, i1 true)
-; CHECK-NEXT: br i1 [[COND:%.*]], label [[LOOP_THEN5:%.*]], label [[LOOP_ELSE4:%.*]]
+; CHECK-NEXT: [[TMP12:%.*]] = call <vscale x 4 x i32> @llvm.stepvector.nxv4i32()
+; CHECK-NEXT: [[TMP13:%.*]] = extractelement <vscale x 4 x i32> [[TMP12]], i64 0
+; CHECK-NEXT: [[TMP14:%.*]] = icmp ult i32 [[TMP13]], [[TMP10]]
+; CHECK-NEXT: br i1 [[COND1:%.*]], label [[LOOP_THEN5:%.*]], label [[LOOP_ELSE4:%.*]]
; CHECK: loop.else4:
-; CHECK-NEXT: [[WIDE_MASKED_GATHER:%.*]] = call <vscale x 4 x i32> @llvm.vp.gather.nxv4i32.nxv4p0(<vscale x 4 x ptr> align 4 [[BROADCAST_SPLAT]], <vscale x 4 x i1> splat (i1 true), i32 [[TMP10]]), !alias.scope [[META0:![0-9]+]]
-; CHECK-NEXT: [[TMP12:%.*]] = extractelement <vscale x 4 x i32> [[WIDE_MASKED_GATHER]], i64 0
+; CHECK-NEXT: [[WIDE_MASKED_GATHER1:%.*]] = call <vscale x 4 x i32> @llvm.vp.gather.nxv4i32.nxv4p0(<vscale x 4 x ptr> align 4 [[BROADCAST_SPLAT]], <vscale x 4 x i1> splat (i1 true), i32 [[TMP10]]), !alias.scope [[META0:![0-9]+]]
+; CHECK-NEXT: [[TMP15:%.*]] = extractelement <vscale x 4 x i32> [[WIDE_MASKED_GATHER1]], i64 0
; CHECK-NEXT: br label [[LOOP_LATCH6]]
; CHECK: loop.then5:
; CHECK-NEXT: br label [[LOOP_LATCH6]]
; CHECK: loop.latch6:
-; CHECK-NEXT: [[TMP13:%.*]] = phi i32 [ [[TMP12]], [[LOOP_ELSE4]] ], [ poison, [[LOOP_THEN5]] ]
-; CHECK-NEXT: [[BROADCAST_SPLATINSERT7:%.*]] = insertelement <vscale x 4 x i32> poison, i32 [[TMP13]], i64 0
-; CHECK-NEXT: [[BROADCAST_SPLAT8:%.*]] = shufflevector <vscale x 4 x i32> [[BROADCAST_SPLATINSERT7]], <vscale x 4 x i32> poison, <vscale x 4 x i32> zeroinitializer
-; CHECK-NEXT: [[TMP14:%.*]] = call <vscale x 4 x i32> @llvm.vp.merge.nxv4i32(<vscale x 4 x i1> splat (i1 true), <vscale x 4 x i32> zeroinitializer, <vscale x 4 x i32> [[BROADCAST_SPLAT8]], i32 [[TMP10]])
-; CHECK-NEXT: call void @llvm.vp.scatter.nxv4i32.nxv4p0(<vscale x 4 x i32> [[TMP14]], <vscale x 4 x ptr> align 4 [[BROADCAST_SPLAT4]], <vscale x 4 x i1> splat (i1 true), i32 [[TMP10]]), !alias.scope [[META3:![0-9]+]], !noalias [[META0]]
+; CHECK-NEXT: [[TMP18:%.*]] = phi i32 [ [[TMP15]], [[LOOP_ELSE4]] ], [ poison, [[LOOP_THEN5]] ]
+; CHECK-NEXT: [[COND:%.*]] = phi i1 [ false, [[LOOP_ELSE4]] ], [ [[TMP14]], [[LOOP_THEN5]] ]
+; CHECK-NEXT: [[BROADCAST_SPLATINSERT7:%.*]] = insertelement <vscale x 4 x i32> poison, i32 [[TMP18]], i64 0
+; CHECK-NEXT: [[WIDE_MASKED_GATHER:%.*]] = shufflevector <vscale x 4 x i32> [[BROADCAST_SPLATINSERT7]], <vscale x 4 x i32> poison, <vscale x 4 x i32> zeroinitializer
+; CHECK-NEXT: [[PREDPHI:%.*]] = select i1 [[COND]], <vscale x 4 x i32> zeroinitializer, <vscale x 4 x i32> [[WIDE_MASKED_GATHER]]
+; CHECK-NEXT: call void @llvm.vp.scatter.nxv4i32.nxv4p0(<vscale x 4 x i32> [[PREDPHI]], <vscale x 4 x ptr> align 4 [[BROADCAST_SPLAT4]], <vscale x 4 x i1> splat (i1 true), i32 [[TMP10]]), !alias.scope [[META3:![0-9]+]], !noalias [[META0]]
; CHECK-NEXT: [[AVL_NEXT]] = sub nuw i32 [[AVL]], [[TMP10]]
; CHECK-NEXT: [[TMP16:%.*]] = icmp eq i32 [[AVL_NEXT]], 0
; CHECK-NEXT: br i1 [[TMP16]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP5:![0-9]+]]
@@ -58,7 +62,7 @@ define void @predicated_uniform_load(ptr %src, i32 %n, ptr %dst, i1 %cond) {
; CHECK-NEXT: br label [[LOOP:%.*]]
; CHECK: loop:
; CHECK-NEXT: [[IV:%.*]] = phi i32 [ 0, [[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], [[LOOP_LATCH:%.*]] ]
-; CHECK-NEXT: br i1 [[COND]], label [[LOOP_THEN:%.*]], label [[LOOP_ELSE:%.*]]
+; CHECK-NEXT: br i1 [[COND1]], label [[LOOP_THEN:%.*]], label [[LOOP_ELSE:%.*]]
; CHECK: loop.then:
; CHECK-NEXT: br label [[LOOP_LATCH]]
; CHECK: loop.else:
diff --git a/llvm/test/Transforms/LoopVectorize/RISCV/tail-folding-complex-mask.ll b/llvm/test/Transforms/LoopVectorize/RISCV/tail-folding-complex-mask.ll
index 4065943ab42fb..311e2f4d8932a 100644
--- a/llvm/test/Transforms/LoopVectorize/RISCV/tail-folding-complex-mask.ll
+++ b/llvm/test/Transforms/LoopVectorize/RISCV/tail-folding-complex-mask.ll
@@ -26,6 +26,9 @@ define void @test(i64 %n, ptr noalias %src0, ptr noalias %src1, ptr noalias %src
; IF-EVL-NEXT: [[EVL_BASED_IV:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_EVL_NEXT:%.*]], %[[LATCH7:.*]] ]
; IF-EVL-NEXT: [[AVL:%.*]] = phi i64 [ [[N]], %[[VECTOR_PH]] ], [ [[AVL_NEXT:%.*]], %[[LATCH7]] ]
; IF-EVL-NEXT: [[TMP7:%.*]] = call i32 @llvm.experimental.get.vector.length.i64(i64 [[AVL]], i32 4, i1 true)
+; IF-EVL-NEXT: [[TMP6:%.*]] = call <vscale x 4 x i32> @llvm.stepvector.nxv4i32()
+; IF-EVL-NEXT: [[TMP9:%.*]] = extractelement <vscale x 4 x i32> [[TMP6]], i64 0
+; IF-EVL-NEXT: [[TMP8:%.*]] = icmp ult i32 [[TMP9]], [[TMP7]]
; IF-EVL-NEXT: [[TMP10:%.*]] = getelementptr i32, ptr [[SRC0]], i64 [[EVL_BASED_IV]]
; IF-EVL-NEXT: [[VP_OP_LOAD:%.*]] = call <vscale x 4 x i32> @llvm.vp.load.nxv4i32.p0(ptr align 4 [[TMP10]], <vscale x 4 x i1> [[BROADCAST_SPLAT]], i32 [[TMP7]])
; IF-EVL-NEXT: [[PREDPHI:%.*]] = select i1 [[C1]], <vscale x 4 x i32> [[VP_OP_LOAD]], <vscale x 4 x i32> zeroinitializer
@@ -33,19 +36,20 @@ define void @test(i64 %n, ptr noalias %src0, ptr noalias %src1, ptr noalias %src
; IF-EVL-NEXT: [[VP_OP_LOAD7:%.*]] = call <vscale x 4 x i32> @llvm.vp.load.nxv4i32.p0(ptr align 4 [[TMP11]], <vscale x 4 x i1> [[TMP4]], i32 [[TMP7]])
; IF-EVL-NEXT: [[TMP12:%.*]] = add <vscale x 4 x i32> [[VP_OP_LOAD7]], [[PREDPHI]]
; IF-EVL-NEXT: [[PREDPHI8:%.*]] = select <vscale x 4 x i1> [[TMP4]], <vscale x 4 x i32> [[TMP12]], <vscale x 4 x i32> zeroinitializer
-; IF-EVL-NEXT: [[TMP9:%.*]] = extractelement <vscale x 4 x i32> [[PREDPHI8]], i64 0
+; IF-EVL-NEXT: [[TMP18:%.*]] = extractelement <vscale x 4 x i32> [[PREDPHI8]], i64 0
; IF-EVL-NEXT: br i1 [[C3]], label %[[LOAD_V25:.*]], label %[[LATCH7]]
; IF-EVL: [[LOAD_V25]]:
-; IF-EVL-NEXT: [[TMP14:%.*]] = getelementptr inbounds i32, ptr [[SRC2]], i64 [[EVL_BASED_IV]]
-; IF-EVL-NEXT: [[VP_OP_LOAD6:%.*]] = call <vscale x 4 x i32> @llvm.vp.load.nxv4i32.p0(ptr align 4 [[TMP14]], <vscale x 4 x i1> splat (i1 true), i32 [[TMP7]])
-; IF-EVL-NEXT: [[TMP15:%.*]] = extractelement <vscale x 4 x i32> [[VP_OP_LOAD6]], i64 0
-; IF-EVL-NEXT: [[TMP16:%.*]] = add i32 [[TMP15]], [[TMP9]]
+; IF-EVL-NEXT: [[TMP13:%.*]] = getelementptr inbounds i32, ptr [[SRC2]], i64 [[EVL_BASED_IV]]
+; IF-EVL-NEXT: [[VP_OP_LOAD6:%.*]] = call <vscale x 4 x i32> @llvm.vp.load.nxv4i32.p0(ptr align 4 [[TMP13]], <vscale x 4 x i1> splat (i1 true), i32 [[TMP7]])
+; IF-EVL-NEXT: [[TMP14:%.*]] = extractelement <vscale x 4 x i32> [[VP_OP_LOAD6]], i64 0
+; IF-EVL-NEXT: [[TMP15:%.*]] = add i32 [[TMP14]], [[TMP18]]
; IF-EVL-NEXT: br label %[[LATCH7]]
; IF-EVL: [[LATCH7]]:
-; IF-EVL-NEXT: [[TMP13:%.*]] = phi i32 [ poison, %[[VECTOR_BODY]] ], [ [[TMP16]], %[[LOAD_V25]] ]
-; IF-EVL-NEXT: [[BROADCAST_SPLATINSERT8:%.*]] = insertelement <vscale x 4 x i32> poison, i32 [[TMP13]], i64 0
+; IF-EVL-NEXT: [[TMP16:%.*]] = phi i32 [ poison, %[[VECTOR_BODY]] ], [ [[TMP15]], %[[LOAD_V25]] ]
+; IF-EVL-NEXT: [[TMP17:%.*]] = phi i1 [ false, %[[VECTOR_BODY]] ], [ [[TMP8]], %[[LOAD_V25]] ]
+; IF-EVL-NEXT: [[BROADCAST_SPLATINSERT8:%.*]] = insertelement <vscale x 4 x i32> poison, i32 [[TMP16]], i64 0
; IF-EVL-NEXT: [[BROADCAST_SPLAT9:%.*]] = shufflevector <vscale x 4 x i32> [[BROADCAST_SPLATINSERT8]], <vscale x 4 x i32> poison, <vscale x 4 x i32> zeroinitializer
-; IF-EVL-NEXT: [[PREDPHI9:%.*]] = call <vscale x 4 x i32> @llvm.vp.merge.nxv4i32(<vscale x 4 x i1> splat (i1 true), <vscale x 4 x i32> [[BROADCAST_SPLAT9]], <vscale x 4 x i32> [[PREDPHI8]], i32 [[TMP7]])
+; IF-EVL-NEXT: [[PREDPHI9:%.*]] = select i1 [[TMP17]], <vscale x 4 x i32> [[BROADCAST_SPLAT9]], <vscale x 4 x i32> [[PREDPHI8]]
; IF-EVL-NEXT: [[TMP20:%.*]] = getelementptr inbounds i32, ptr [[DST]], i64 [[EVL_BASED_IV]]
; IF-EVL-NEXT: call void @llvm.vp.store.nxv4i32.p0(<vscale x 4 x i32> [[PREDPHI9]], ptr align 4 [[TMP20]], <vscale x 4 x i1> splat (i1 true), i32 [[TMP7]])
; IF-EVL-NEXT: [[TMP21:%.*]] = zext i32 [[TMP7]] to i64
@@ -93,15 +97,15 @@ define void @test(i64 %n, ptr noalias %src0, ptr noalias %src1, ptr noalias %src
; NO-VP: [[LOAD_V25]]:
; NO-VP-NEXT: [[TMP12:%.*]] = getelementptr inbounds i32, ptr [[SRC2]], i64 [[INDEX]]
; NO-VP-NEXT: [[WIDE_LOAD:%.*]] = load <vscale x 4 x i32>, ptr [[TMP12]], align 4
-; NO-VP-NEXT: [[TMP16:%.*]] = extractelement <vscale x 4 x i32> [[WIDE_LOAD]], i64 0
-; NO-VP-NEXT: [[TMP17:%.*]] = add i32 [[TMP16]], [[TMP11]]
+; NO-VP-NEXT: [[TMP17:%.*]] = extractelement <vscale x 4 x i32> [[WIDE_LOAD]], i64 0
+; NO-VP-NEXT: [[TMP20:%.*]] = add i32 [[TMP17]], [[TMP11]]
; NO-VP-NEXT: br label %[[LATCH6]]
; NO-VP: [[LATCH6]]:
-; NO-VP-NEXT: [[TMP20:%.*]] = phi i32 [ poison, %[[VECTOR_BODY]] ], [ [[TMP17]], %[[LOAD_V25]] ]
-; NO-VP-NEXT: [[TMP21:%.*]] = phi i1 [ false, %[[VECTOR_BODY]] ], [ true, %[[LOAD_V25]] ]
-; NO-VP-NEXT: [[BROADCAST_SPLATINSERT7:%.*]] = insertelement <vscale x 4 x i32> poison, i32 [[TMP20]], i64 0
+; NO-VP-NEXT: [[TMP21:%.*]] = phi i32 [ poison, %[[VECTOR_BODY]] ], [ [[TMP20]], %[[LOAD_V25]] ]
+; NO-VP-NEXT: [[TMP16:%.*]] = phi i1 [ false, %[[VECTOR_BODY]] ], [ true, %[[LOAD_V25]] ]
+; NO-VP-NEXT: [[BROADCAST_SPLATINSERT7:%.*]] = insertelement <vscale x 4 x i32> poison, i32 [[TMP21]], i64 0
; NO-VP-NEXT: [[BROADCAST_SPLAT8:%.*]] = shufflevector <vscale x 4 x i32> [[BROADCAST_SPLATINSERT7]], <vscale x 4 x i32> poison, <vscale x 4 x i32> zeroinitializer
-; NO-VP-NEXT: [[PREDPHI8:%.*]] = select i1 [[TMP21]], <vscale x 4 x i32> [[BROADCAST_SPLAT8]], <vscale x 4 x i32> [[PREDPHI6]]
+; NO-VP-NEXT: [[PREDPHI8:%.*]] = select i1 [[TMP16]], <vscale x 4 x i32> [[BROADCAST_SPLAT8]], <vscale x 4 x i32> [[PREDPHI6]]
; NO-VP-NEXT: [[TMP18:%.*]] = getelementptr inbounds i32, ptr [[DST]], i64 [[INDEX]]
; NO-VP-NEXT: store <vscale x 4 x i32> [[PREDPHI8]], ptr [[TMP18]], align 4
; NO-VP-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], [[TMP1]]
diff --git a/llvm/test/Transforms/LoopVectorize/VPlan/predicator.ll b/llvm/test/Transforms/LoopVectorize/VPlan/predicator.ll
index b594c018c0800..cd3f9b23e5e7f 100644
--- a/llvm/test/Transforms/LoopVectorize/VPlan/predicator.ll
+++ b/llvm/test/Transforms/LoopVectorize/VPlan/predicator.ll
@@ -1319,7 +1319,8 @@ define void @uniform_branch_after_varying_branch(ptr %a, i1 %u1) {
; CHECK-EMPTY:
; CHECK-NEXT: bb3:
; CHECK-NEXT: EMIT-SCALAR vp<[[VP5:%[0-9]+]]> = phi [ ir<poison>, bb1 ], [ ir<%add2>, bb2 ]
-; CHECK-NEXT: BLEND ir<%phi3> = ir<%add1>/vp<[[VP4]]> vp<%5>/vp<[[VP4]]>
+; CHECK-NEXT: EMIT-SCALAR vp<[[VP6:%[0-9]+]]> = phi [ ir<false>, bb1 ], [ vp<[[VP4]]>, bb2 ]
+; CHECK-NEXT: BLEND ir<%phi3> = ir<%add1>/vp<[[VP4]]> vp<%5>/vp<[[VP6]]>
; CHECK-NEXT: EMIT ir<%add3> = add ir<%phi3>, ir<3>, vp<[[VP4]]>
; CHECK-NEXT: Successor(s): bb4
; CHECK-EMPTY:
@@ -1489,8 +1490,10 @@ define void @uniform_branch_after_varying_branch_more_blocks(ptr %a, i1 %u1) {
; CHECK-EMPTY:
; CHECK-NEXT: bb4:
; CHECK-NEXT: EMIT-SCALAR vp<[[VP5:%[0-9]+]]> = phi [ ir<%add2>, bb2 ], [ ir<poison>, bb3 ]
-; CHECK-NEXT: EMIT-SCALAR vp<[[VP6:%[0-9]+]]> = phi [ ir<poison>, bb2 ], [ ir<%add3>, bb3 ]
-; CHECK-NEXT: BLEND ir<%phi4> = vp<%5>/vp<[[VP4]]> vp<%6>/vp<[[VP4]]>
+; CHECK-NEXT: EMIT-SCALAR vp<[[VP6:%[0-9]+]]> = phi [ vp<[[VP4]]>, bb2 ], [ ir<false>, bb3 ]
+; CHECK-NEXT: EMIT-SCALAR vp<[[VP7:%[0-9]+]]> = phi [ ir<poison>, bb2 ], [ ir<%add3>, bb3 ]
+; CHECK-NEXT: EMIT-SCALAR vp<[[VP8:%[0-9]+]]> = phi [ ir<false>, bb2 ], [ vp<[[VP4]]>, bb3 ]
+; CHECK-NEXT: BLEND ir<%phi4> = vp<%5>/vp<[[VP6]]> vp<%7>/vp<[[VP8]]>
; CHECK-NEXT: EMIT ir<%add4> = add ir<%phi4>, ir<4>, vp<[[VP4]]>
; CHECK-NEXT: Successor(s): bb5
; CHECK-EMPTY:
@@ -1681,8 +1684,10 @@ define void @uniform_branch_after_varying_branch_more_blocks_mirrored(ptr %a, i1
; CHECK-EMPTY:
; CHECK-NEXT: bb5:
; CHECK-NEXT: EMIT-SCALAR vp<[[VP5:%[0-9]+]]> = phi [ ir<%add3>, bb3 ], [ ir<poison>, bb4 ]
-; CHECK-NEXT: EMIT-SCALAR vp<[[VP6:%[0-9]+]]> = phi [ ir<poison>, bb3 ], [ ir<%add4>, bb4 ]
-; CHECK-NEXT: BLEND ir<%phi5> = vp<%5>/ir<%v0> vp<%6>/ir<%v0>
+; CHECK-NEXT: EMIT-SCALAR vp<[[VP6:%[0-9]+]]> = phi [ ir<%v0>, bb3 ], [ ir<false>, bb4 ]
+; CHECK-NEXT: EMIT-SCALAR vp<[[VP7:%[0-9]+]]> = phi [ ir<poison>, bb3 ], [ ir<%add4>, bb4 ]
+; CHECK-NEXT: EMIT-SCALAR vp<[[VP8:%[0-9]+]]> = phi [ ir<false>, bb3 ], [ ir<%v0>, bb4 ]
+; CHECK-NEXT: BLEND ir<%phi5> = vp<%5>/vp<[[VP6]]> vp<%7>/vp<[[VP8]]>
; CHECK-NEXT: EMIT ir<%add5> = add ir<%phi5>, ir<5>, ir<%v0>
; CHECK-NEXT: Successor(s): bb6
; CHECK-EMPTY:
diff --git a/llvm/test/Transforms/LoopVectorize/X86/pr109581-unused-blend.ll b/llvm/test/Transforms/LoopVectorize/X86/pr109581-unused-blend.ll
index 89096bb9126ed..0323a6771949d 100644
--- a/llvm/test/Transforms/LoopVectorize/X86/pr109581-unused-blend.ll
+++ b/llvm/test/Transforms/LoopVectorize/X86/pr109581-unused-blend.ll
@@ -14,50 +14,50 @@ define i32 @unused_blend_after_unrolling(ptr %p, i32 %a, i1 %c.1, i16 %x, i16 %y
; CHECK: [[VECTOR_PH]]:
; CHECK-NEXT: [[BROADCAST_SPLATINSERT16:%.*]] = insertelement <4 x i1> poison, i1 [[C]], i64 0
; CHECK-NEXT: [[BROADCAST_SPLAT17:%.*]] = shufflevector <4 x i1> [[BROADCAST_SPLATINSERT16]], <4 x i1> poison, <4 x i32> zeroinitializer
-; CHECK-NEXT: [[TMP0:%.*]] = xor i1 [[C_1]], true
+; CHECK-NEXT: [[TMP2:%.*]] = xor i1 [[C_1]], true
; CHECK-NEXT: [[TMP22:%.*]] = xor <4 x i1> [[BROADCAST_SPLAT17]], splat (i1 true)
-; CHECK-NEXT: br label %[[PRED_SDIV_CONTINUE15:.*]]
-; CHECK: [[PRED_SDIV_CONTINUE15]]:
-; CHECK-NEXT: [[INDEX:%.*]] = phi i32 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[PRED_SDIV_CONTINUE16:.*]] ]
-; CHECK-NEXT: [[VEC_PHI:%.*]] = phi <4 x i1> [ zeroinitializer, %[[VECTOR_PH]] ], [ [[TMP24:%.*]], %[[PRED_SDIV_CONTINUE16]] ]
-; CHECK-NEXT: [[VEC_PHI3:%.*]] = phi <4 x i1> [ zeroinitializer, %[[VECTOR_PH]] ], [ [[TMP25:%.*]], %[[PRED_SDIV_CONTINUE16]] ]
-; CHECK-NEXT: br i1 [[TMP0]], label %[[PRED_SDIV_IF16:.*]], label %[[PRED_SDIV_CONTINUE17:.*]]
-; CHECK: [[PRED_SDIV_IF16]]:
-; CHECK-NEXT: br label %[[PRED_SDIV_CONTINUE17]]
-; CHECK: [[PRED_SDIV_CONTINUE17]]:
-; CHECK-NEXT: br i1 [[TMP0]], label %[[PRED_SDIV_IF2:.*]], label %[[PRED_SDIV_CONTINUE3:.*]]
-; CHECK: [[PRED_SDIV_IF2]]:
-; CHECK-NEXT: br label %[[PRED_SDIV_CONTINUE3]]
-; CHECK: [[PRED_SDIV_CONTINUE3]]:
-; CHECK-NEXT: br i1 [[TMP0]], label %[[PRED_SDIV_IF4:.*]], label %[[PRED_SDIV_CONTINUE5:.*]]
+; CHECK-NEXT: br label %[[VECTOR_BODY:.*]]
+; CHECK: [[VECTOR_BODY]]:
+; CHECK-NEXT: [[INDEX:%.*]] = phi i32 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[PRED_SDIV_CONTINUE17:.*]] ]
+; CHECK-NEXT: [[VEC_PHI:%.*]] = phi <4 x i1> [ zeroinitializer, %[[VECTOR_PH]] ], [ [[TMP24:%.*]], %[[PRED_SDIV_CONTINUE17]] ]
+; CHECK-NEXT: [[VEC_PHI3:%.*]] = phi <4 x i1> [ zeroinitializer, %[[VECTOR_PH]] ], [ [[TMP25:%.*]], %[[PRED_SDIV_CONTINUE17]] ]
+; CHECK-NEXT: br i1 [[TMP2]], label %[[PRED_SDIV_IF:.*]], label %[[PRED_SDIV_CONTINUE:.*]]
+; CHECK: [[PRED_SDIV_IF]]:
+; CHECK-NEXT: br label %[[PRED_SDIV_CONTINUE]]
+; CHECK: [[PRED_SDIV_CONTINUE]]:
+; CHECK-NEXT: br i1 [[TMP2]], label %[[PRED_SDIV_IF4:.*]], label %[[PRED_SDIV_CONTINUE5:.*]]
; CHECK: [[PRED_SDIV_IF4]]:
; CHECK-NEXT: br label %[[PRED_SDIV_CONTINUE5]]
; CHECK: [[PRED_SDIV_CONTINUE5]]:
-; CHECK-NEXT: br i1 [[TMP0]], label %[[PRED_SDIV_IF6:.*]], label %[[PRED_SDIV_CONTINUE7:.*]]
+; CHECK-NEXT: br i1 [[TMP2]], label %[[PRED_SDIV_IF6:.*]], label %[[PRED_SDIV_CONTINUE7:.*]]
; CHECK: [[PRED_SDIV_IF6]]:
; CHECK-NEXT: br label %[[PRED_SDIV_CONTINUE7]]
; CHECK: [[PRED_SDIV_CONTINUE7]]:
-; CHECK-NEXT: br i1 [[TMP0]], label %[[PRED_SDIV_IF8:.*]], label %[[PRED_SDIV_CONTINUE9:.*]]
+; CHECK-NEXT: br i1 [[TMP2]], label %[[PRED_SDIV_IF8:.*]], label %[[PRED_SDIV_CONTINUE9:.*]]
; CHECK: [[PRED_SDIV_IF8]]:
; CHECK-NEXT: br label %[[PRED_SDIV_CONTINUE9]]
; CHECK: [[PRED_SDIV_CONTINUE9]]:
-; CHECK-NEXT: br i1 [[TMP0]], label %[[PRED_SDIV_IF10:.*]], label %[[PRED_SDIV_CONTINUE11:.*]]
+; CHECK-NEXT: br i1 [[TMP2]], label %[[PRED_SDIV_IF10:.*]], label %[[PRED_SDIV_CONTINUE11:.*]]
; CHECK: [[PRED_SDIV_IF10]]:
; CHECK-NEXT: br label %[[PRED_SDIV_CONTINUE11]]
; CHECK: [[PRED_SDIV_CONTINUE11]]:
-; CHECK-NEXT: br i1 [[TMP0]], label %[[PRED_SDIV_IF12:.*]], label %[[PRED_SDIV_CONTINUE13:.*]]
+; CHECK-NEXT: br i1 [[TMP2]], label %[[PRED_SDIV_IF12:.*]], label %[[PRED_SDIV_CONTINUE13:.*]]
; CHECK: [[PRED_SDIV_IF12]]:
; CHECK-NEXT: br label %[[PRED_SDIV_CONTINUE13]]
; CHECK: [[PRED_SDIV_CONTINUE13]]:
-; CHECK-NEXT: br i1 [[TMP0]], label %[[PRED_SDIV_IF14:.*]], label %[[PRED_SDIV_CONTINUE16]]
+; CHECK-NEXT: br i1 [[TMP2]], label %[[PRED_SDIV_IF14:.*]], label %[[PRED_SDIV_CONTINUE15:.*]]
; CHECK: [[PRED_SDIV_IF14]]:
-; CHECK-NEXT: br label %[[PRED_SDIV_CONTINUE16]]
-; CHECK: [[PRED_SDIV_CONTINUE16]]:
+; CHECK-NEXT: br label %[[PRED_SDIV_CONTINUE15]]
+; CHECK: [[PRED_SDIV_CONTINUE15]]:
+; CHECK-NEXT: br i1 [[TMP2]], label %[[PRED_SDIV_IF16:.*]], label %[[PRED_SDIV_CONTINUE17]]
+; CHECK: [[PRED_SDIV_IF16]]:
+; CHECK-NEXT: br label %[[PRED_SDIV_CONTINUE17]]
+; CHECK: [[PRED_SDIV_CONTINUE17]]:
; CHECK-NEXT: [[TMP24]] = or <4 x i1> [[VEC_PHI]], [[TMP22]]
; CHECK-NEXT: [[TMP25]] = or <4 x i1> [[VEC_PHI3]], [[TMP22]]
; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i32 [[INDEX]], 8
; CHECK-NEXT: [[TMP26:%.*]] = icmp eq i32 [[INDEX_NEXT]], 96
-; CHECK-NEXT: br i1 [[TMP26]], label %[[MIDDLE_BLOCK:.*]], label %[[PRED_SDIV_CONTINUE15]], !llvm.loop [[LOOP0:![0-9]+]]
+; CHECK-NEXT: br i1 [[TMP26]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP0:![0-9]+]]
; CHECK: [[MIDDLE_BLOCK]]:
; CHECK-NEXT: [[BIN_RDX:%.*]] = or <4 x i1> [[TMP25]], [[TMP24]]
; CHECK-NEXT: [[TMP27:%.*]] = call i1 @llvm.vector.reduce.or.v4i1(<4 x i1> [[BIN_RDX]])
diff --git a/llvm/test/Transforms/LoopVectorize/X86/predicated-instruction-cost.ll b/llvm/test/Transforms/LoopVectorize/X86/predicated-instruction-cost.ll
index 201168b20c859..61d59ce014c43 100644
--- a/llvm/test/Transforms/LoopVectorize/X86/predicated-instruction-cost.ll
+++ b/llvm/test/Transforms/LoopVectorize/X86/predicated-instruction-cost.ll
@@ -71,17 +71,17 @@ define void @test_wide_shift_uses_predicated_invariant_instruction(i32 %d, i1 %c
; CHECK-NEXT: br label %[[LOOP_LATCH]]
; CHECK: [[LOOP_LATCH]]:
; CHECK-NEXT: [[TMP3:%.*]] = phi i32 [ poison, %[[VECTOR_BODY]] ], [ [[TMP2]], %[[ELSE]] ]
-; CHECK-NEXT: [[TMP7:%.*]] = phi i1 [ false, %[[VECTOR_BODY]] ], [ true, %[[ELSE]] ]
+; CHECK-NEXT: [[TMP4:%.*]] = phi i1 [ false, %[[VECTOR_BODY]] ], [ true, %[[ELSE]] ]
; CHECK-NEXT: [[BROADCAST_SPLATINSERT:%.*]] = insertelement <4 x i32> poison, i32 [[TMP3]], i64 0
; CHECK-NEXT: [[BROADCAST_SPLAT:%.*]] = shufflevector <4 x i32> [[BROADCAST_SPLATINSERT]], <4 x i32> poison, <4 x i32> zeroinitializer
-; CHECK-NEXT: [[PREDPHI:%.*]] = select i1 [[TMP7]], <4 x i32> [[BROADCAST_SPLAT]], <4 x i32> zeroinitializer
-; CHECK-NEXT: [[TMP4:%.*]] = getelementptr inbounds i32, ptr [[DST]], i32 [[INDEX]]
-; CHECK-NEXT: [[TMP5:%.*]] = getelementptr inbounds i32, ptr [[TMP4]], i64 4
-; CHECK-NEXT: store <4 x i32> [[PREDPHI]], ptr [[TMP4]], align 4
+; CHECK-NEXT: [[PREDPHI:%.*]] = select i1 [[TMP4]], <4 x i32> [[BROADCAST_SPLAT]], <4 x i32> zeroinitializer
+; CHECK-NEXT: [[TMP5:%.*]] = getelementptr inbounds i32, ptr [[DST]], i32 [[INDEX]]
+; CHECK-NEXT: [[TMP6:%.*]] = getelementptr inbounds i32, ptr [[TMP5]], i64 4
; CHECK-NEXT: store <4 x i32> [[PREDPHI]], ptr [[TMP5]], align 4
+; CHECK-NEXT: store <4 x i32> [[PREDPHI]], ptr [[TMP6]], align 4
; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i32 [[INDEX]], 8
-; CHECK-NEXT: [[TMP6:%.*]] = icmp eq i32 [[INDEX_NEXT]], 96
-; CHECK-NEXT: br i1 [[TMP6]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP0:![0-9]+]]
+; CHECK-NEXT: [[TMP7:%.*]] = icmp eq i32 [[INDEX_NEXT]], 96
+; CHECK-NEXT: br i1 [[TMP7]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP0:![0-9]+]]
; CHECK: [[MIDDLE_BLOCK]]:
; CHECK-NEXT: br label %[[SCALAR_PH:.*]]
; CHECK: [[SCALAR_PH]]:
diff --git a/llvm/test/Transforms/LoopVectorize/X86/replicate-recipe-with-only-first-lane-used.ll b/llvm/test/Transforms/LoopVectorize/X86/replicate-recipe-with-only-first-lane-used.ll
index 864cc9fa56394..6068a036557b3 100644
--- a/llvm/test/Transforms/LoopVectorize/X86/replicate-recipe-with-only-first-lane-used.ll
+++ b/llvm/test/Transforms/LoopVectorize/X86/replicate-recipe-with-only-first-lane-used.ll
@@ -18,11 +18,11 @@ define void @replicate_udiv_with_only_first_lane_used(i32 %x, ptr %dst, i64 %d)
; CHECK-NEXT: br label %[[LOOP_LATCH1]]
; CHECK: [[LOOP_LATCH1]]:
; CHECK-NEXT: [[TMP0:%.*]] = phi i64 [ poison, %[[VECTOR_BODY]] ]
-; CHECK-NEXT: [[TMP2:%.*]] = phi i1 [ false, %[[VECTOR_BODY]] ]
-; CHECK-NEXT: [[PREDPHI:%.*]] = select i1 [[TMP2]], i64 [[TMP0]], i64 0
-; CHECK-NEXT: [[TMP1:%.*]] = getelementptr i16, ptr [[DST]], i64 [[PREDPHI]]
-; CHECK-NEXT: store i16 0, ptr [[TMP1]], align 2
-; CHECK-NEXT: store i16 0, ptr [[TMP1]], align 2
+; CHECK-NEXT: [[TMP1:%.*]] = phi i1 [ false, %[[VECTOR_BODY]] ]
+; CHECK-NEXT: [[PREDPHI:%.*]] = select i1 [[TMP1]], i64 [[TMP0]], i64 0
+; CHECK-NEXT: [[TMP2:%.*]] = getelementptr i16, ptr [[DST]], i64 [[PREDPHI]]
+; CHECK-NEXT: store i16 0, ptr [[TMP2]], align 2
+; CHECK-NEXT: store i16 0, ptr [[TMP2]], align 2
; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 8
; CHECK-NEXT: [[TMP12:%.*]] = icmp eq i64 [[INDEX_NEXT]], 96
; CHECK-NEXT: br i1 [[TMP12]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP0:![0-9]+]]
diff --git a/llvm/test/Transforms/LoopVectorize/X86/x86-predication.ll b/llvm/test/Transforms/LoopVectorize/X86/x86-predication.ll
index dd8a4ceb20072..8947813a4d670 100644
--- a/llvm/test/Transforms/LoopVectorize/X86/x86-predication.ll
+++ b/llvm/test/Transforms/LoopVectorize/X86/x86-predication.ll
@@ -19,28 +19,28 @@ define i32 @predicated_sdiv_masked_load(ptr %a, ptr %b, i32 %x, i1 %c) {
; CHECK-NEXT: [[BROADCAST_SPLAT:%.*]] = shufflevector <2 x i1> [[BROADCAST_SPLATINSERT]], <2 x i1> poison, <2 x i32> zeroinitializer
; CHECK-NEXT: br label %[[VECTOR_BODY:.*]]
; CHECK: [[VECTOR_BODY]]:
-; CHECK-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[PRED_SDIV_CONTINUE3:.*]] ]
-; CHECK-NEXT: [[VEC_PHI:%.*]] = phi <2 x i32> [ zeroinitializer, %[[VECTOR_PH]] ], [ [[TMP11:%.*]], %[[PRED_SDIV_CONTINUE3]] ]
+; CHECK-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[PRED_SDIV_CONTINUE2:.*]] ]
+; CHECK-NEXT: [[VEC_PHI:%.*]] = phi <2 x i32> [ zeroinitializer, %[[VECTOR_PH]] ], [ [[TMP11:%.*]], %[[PRED_SDIV_CONTINUE2]] ]
; CHECK-NEXT: [[TMP0:%.*]] = getelementptr inbounds i32, ptr [[A]], i64 [[INDEX]]
; CHECK-NEXT: [[WIDE_LOAD:%.*]] = load <2 x i32>, ptr [[TMP0]], align 4
; CHECK-NEXT: [[TMP1:%.*]] = getelementptr i32, ptr [[B]], i64 [[INDEX]]
-; CHECK-NEXT: [[WIDE_LOAD2:%.*]] = call <2 x i32> @llvm.masked.load.v2i32.p0(ptr align 4 [[TMP1]], <2 x i1> [[BROADCAST_SPLAT]], <2 x i32> poison)
-; CHECK-NEXT: br i1 [[C]], label %[[PRED_SDIV_IF1:.*]], label %[[PRED_SDIV_CONTINUE2:.*]]
+; CHECK-NEXT: [[WIDE_MASKED_LOAD:%.*]] = call <2 x i32> @llvm.masked.load.v2i32.p0(ptr align 4 [[TMP1]], <2 x i1> [[BROADCAST_SPLAT]], <2 x i32> poison)
+; CHECK-NEXT: br i1 [[C]], label %[[PRED_SDIV_IF:.*]], label %[[PRED_SDIV_CONTINUE:.*]]
+; CHECK: [[PRED_SDIV_IF]]:
+; CHECK-NEXT: [[TMP2:%.*]] = extractelement <2 x i32> [[WIDE_MASKED_LOAD]], i64 0
+; CHECK-NEXT: [[TMP3:%.*]] = sdiv i32 [[TMP2]], [[X]]
+; CHECK-NEXT: [[TMP4:%.*]] = insertelement <2 x i32> poison, i32 [[TMP3]], i64 0
+; CHECK-NEXT: br label %[[PRED_SDIV_CONTINUE]]
+; CHECK: [[PRED_SDIV_CONTINUE]]:
+; CHECK-NEXT: [[TMP5:%.*]] = phi <2 x i32> [ poison, %[[VECTOR_BODY]] ], [ [[TMP4]], %[[PRED_SDIV_IF]] ]
+; CHECK-NEXT: br i1 [[C]], label %[[PRED_SDIV_IF1:.*]], label %[[PRED_SDIV_CONTINUE2]]
; CHECK: [[PRED_SDIV_IF1]]:
-; CHECK-NEXT: [[TMP6:%.*]] = extractelement <2 x i32> [[WIDE_LOAD2]], i64 0
+; CHECK-NEXT: [[TMP6:%.*]] = extractelement <2 x i32> [[WIDE_MASKED_LOAD]], i64 1
; CHECK-NEXT: [[TMP7:%.*]] = sdiv i32 [[TMP6]], [[X]]
-; CHECK-NEXT: [[TMP4:%.*]] = insertelement <2 x i32> poison, i32 [[TMP7]], i64 0
+; CHECK-NEXT: [[TMP8:%.*]] = insertelement <2 x i32> [[TMP5]], i32 [[TMP7]], i64 1
; CHECK-NEXT: br label %[[PRED_SDIV_CONTINUE2]]
; CHECK: [[PRED_SDIV_CONTINUE2]]:
-; CHECK-NEXT: [[TMP5:%.*]] = phi <2 x i32> [ poison, %[[VECTOR_BODY]] ], [ [[TMP4]], %[[PRED_SDIV_IF1]] ]
-; CHECK-NEXT: br i1 [[C]], label %[[PRED_SDIV_IF2:.*]], label %[[PRED_SDIV_CONTINUE3]]
-; CHECK: [[PRED_SDIV_IF2]]:
-; CHECK-NEXT: [[TMP14:%.*]] = extractelement <2 x i32> [[WIDE_LOAD2]], i64 1
-; CHECK-NEXT: [[TMP15:%.*]] = sdiv i32 [[TMP14]], [[X]]
-; CHECK-NEXT: [[TMP8:%.*]] = insertelement <2 x i32> [[TMP5]], i32 [[TMP15]], i64 1
-; CHECK-NEXT: br label %[[PRED_SDIV_CONTINUE3]]
-; CHECK: [[PRED_SDIV_CONTINUE3]]:
-; CHECK-NEXT: [[TMP9:%.*]] = phi <2 x i32> [ [[TMP5]], %[[PRED_SDIV_CONTINUE2]] ], [ [[TMP8]], %[[PRED_SDIV_IF2]] ]
+; CHECK-NEXT: [[TMP9:%.*]] = phi <2 x i32> [ [[TMP5]], %[[PRED_SDIV_CONTINUE]] ], [ [[TMP8]], %[[PRED_SDIV_IF1]] ]
; CHECK-NEXT: [[TMP10:%.*]] = add nsw <2 x i32> [[TMP9]], [[WIDE_LOAD]]
; CHECK-NEXT: [[PREDPHI:%.*]] = select i1 [[C]], <2 x i32> [[TMP10]], <2 x i32> [[WIDE_LOAD]]
; CHECK-NEXT: [[TMP11]] = add <2 x i32> [[VEC_PHI]], [[PREDPHI]]
@@ -62,31 +62,31 @@ define i32 @predicated_sdiv_masked_load(ptr %a, ptr %b, i32 %x, i1 %c) {
; SINK-GATHER-NEXT: [[BROADCAST_SPLAT:%.*]] = shufflevector <8 x i1> [[BROADCAST_SPLATINSERT]], <8 x i1> poison, <8 x i32> zeroinitializer
; SINK-GATHER-NEXT: br label %[[VECTOR_BODY:.*]]
; SINK-GATHER: [[VECTOR_BODY]]:
-; SINK-GATHER-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[PRED_SDIV_CONTINUE15:.*]] ]
-; SINK-GATHER-NEXT: [[VEC_PHI:%.*]] = phi <8 x i32> [ zeroinitializer, %[[VECTOR_PH]] ], [ [[TMP35:%.*]], %[[PRED_SDIV_CONTINUE15]] ]
+; SINK-GATHER-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[PRED_SDIV_CONTINUE14:.*]] ]
+; SINK-GATHER-NEXT: [[VEC_PHI:%.*]] = phi <8 x i32> [ zeroinitializer, %[[VECTOR_PH]] ], [ [[TMP35:%.*]], %[[PRED_SDIV_CONTINUE14]] ]
; SINK-GATHER-NEXT: [[TMP0:%.*]] = getelementptr inbounds i32, ptr [[A]], i64 [[INDEX]]
; SINK-GATHER-NEXT: [[WIDE_LOAD:%.*]] = load <8 x i32>, ptr [[TMP0]], align 4
; SINK-GATHER-NEXT: [[TMP1:%.*]] = getelementptr i32, ptr [[B]], i64 [[INDEX]]
-; SINK-GATHER-NEXT: [[WIDE_LOAD2:%.*]] = call <8 x i32> @llvm.masked.load.v8i32.p0(ptr align 4 [[TMP1]], <8 x i1> [[BROADCAST_SPLAT]], <8 x i32> poison)
-; SINK-GATHER-NEXT: br i1 [[C]], label %[[PRED_SDIV_IF13:.*]], label %[[PRED_SDIV_CONTINUE14:.*]]
-; SINK-GATHER: [[PRED_SDIV_IF13]]:
-; SINK-GATHER-NEXT: [[TMP30:%.*]] = extractelement <8 x i32> [[WIDE_LOAD2]], i64 0
-; SINK-GATHER-NEXT: [[TMP31:%.*]] = sdiv i32 [[TMP30]], [[X]]
-; SINK-GATHER-NEXT: [[TMP4:%.*]] = insertelement <8 x i32> poison, i32 [[TMP31]], i64 0
-; SINK-GATHER-NEXT: br label %[[PRED_SDIV_CONTINUE14]]
-; SINK-GATHER: [[PRED_SDIV_CONTINUE14]]:
-; SINK-GATHER-NEXT: [[TMP5:%.*]] = phi <8 x i32> [ poison, %[[VECTOR_BODY]] ], [ [[TMP4]], %[[PRED_SDIV_IF13]] ]
+; SINK-GATHER-NEXT: [[WIDE_MASKED_LOAD:%.*]] = call <8 x i32> @llvm.masked.load.v8i32.p0(ptr align 4 [[TMP1]], <8 x i1> [[BROADCAST_SPLAT]], <8 x i32> poison)
+; SINK-GATHER-NEXT: br i1 [[C]], label %[[PRED_SDIV_IF:.*]], label %[[PRED_SDIV_CONTINUE:.*]]
+; SINK-GATHER: [[PRED_SDIV_IF]]:
+; SINK-GATHER-NEXT: [[TMP2:%.*]] = extractelement <8 x i32> [[WIDE_MASKED_LOAD]], i64 0
+; SINK-GATHER-NEXT: [[TMP3:%.*]] = sdiv i32 [[TMP2]], [[X]]
+; SINK-GATHER-NEXT: [[TMP4:%.*]] = insertelement <8 x i32> poison, i32 [[TMP3]], i64 0
+; SINK-GATHER-NEXT: br label %[[PRED_SDIV_CONTINUE]]
+; SINK-GATHER: [[PRED_SDIV_CONTINUE]]:
+; SINK-GATHER-NEXT: [[TMP5:%.*]] = phi <8 x i32> [ poison, %[[VECTOR_BODY]] ], [ [[TMP4]], %[[PRED_SDIV_IF]] ]
; SINK-GATHER-NEXT: br i1 [[C]], label %[[PRED_SDIV_IF1:.*]], label %[[PRED_SDIV_CONTINUE2:.*]]
; SINK-GATHER: [[PRED_SDIV_IF1]]:
-; SINK-GATHER-NEXT: [[TMP6:%.*]] = extractelement <8 x i32> [[WIDE_LOAD2]], i64 1
+; SINK-GATHER-NEXT: [[TMP6:%.*]] = extractelement <8 x i32> [[WIDE_MASKED_LOAD]], i64 1
; SINK-GATHER-NEXT: [[TMP7:%.*]] = sdiv i32 [[TMP6]], [[X]]
; SINK-GATHER-NEXT: [[TMP8:%.*]] = insertelement <8 x i32> [[TMP5]], i32 [[TMP7]], i64 1
; SINK-GATHER-NEXT: br label %[[PRED_SDIV_CONTINUE2]]
; SINK-GATHER: [[PRED_SDIV_CONTINUE2]]:
-; SINK-GATHER-NEXT: [[TMP9:%.*]] = phi <8 x i32> [ [[TMP5]], %[[PRED_SDIV_CONTINUE14]] ], [ [[TMP8]], %[[PRED_SDIV_IF1]] ]
+; SINK-GATHER-NEXT: [[TMP9:%.*]] = phi <8 x i32> [ [[TMP5]], %[[PRED_SDIV_CONTINUE]] ], [ [[TMP8]], %[[PRED_SDIV_IF1]] ]
; SINK-GATHER-NEXT: br i1 [[C]], label %[[PRED_SDIV_IF3:.*]], label %[[PRED_SDIV_CONTINUE4:.*]]
; SINK-GATHER: [[PRED_SDIV_IF3]]:
-; SINK-GATHER-NEXT: [[TMP10:%.*]] = extractelement <8 x i32> [[WIDE_LOAD2]], i64 2
+; SINK-GATHER-NEXT: [[TMP10:%.*]] = extractelement <8 x i32> [[WIDE_MASKED_LOAD]], i64 2
; SINK-GATHER-NEXT: [[TMP11:%.*]] = sdiv i32 [[TMP10]], [[X]]
; SINK-GATHER-NEXT: [[TMP12:%.*]] = insertelement <8 x i32> [[TMP9]], i32 [[TMP11]], i64 2
; SINK-GATHER-NEXT: br label %[[PRED_SDIV_CONTINUE4]]
@@ -94,7 +94,7 @@ define i32 @predicated_sdiv_masked_load(ptr %a, ptr %b, i32 %x, i1 %c) {
; SINK-GATHER-NEXT: [[TMP13:%.*]] = phi <8 x i32> [ [[TMP9]], %[[PRED_SDIV_CONTINUE2]] ], [ [[TMP12]], %[[PRED_SDIV_IF3]] ]
; SINK-GATHER-NEXT: br i1 [[C]], label %[[PRED_SDIV_IF5:.*]], label %[[PRED_SDIV_CONTINUE6:.*]]
; SINK-GATHER: [[PRED_SDIV_IF5]]:
-; SINK-GATHER-NEXT: [[TMP14:%.*]] = extractelement <8 x i32> [[WIDE_LOAD2]], i64 3
+; SINK-GATHER-NEXT: [[TMP14:%.*]] = extractelement <8 x i32> [[WIDE_MASKED_LOAD]], i64 3
; SINK-GATHER-NEXT: [[TMP15:%.*]] = sdiv i32 [[TMP14]], [[X]]
; SINK-GATHER-NEXT: [[TMP16:%.*]] = insertelement <8 x i32> [[TMP13]], i32 [[TMP15]], i64 3
; SINK-GATHER-NEXT: br label %[[PRED_SDIV_CONTINUE6]]
@@ -102,7 +102,7 @@ define i32 @predicated_sdiv_masked_load(ptr %a, ptr %b, i32 %x, i1 %c) {
; SINK-GATHER-NEXT: [[TMP17:%.*]] = phi <8 x i32> [ [[TMP13]], %[[PRED_SDIV_CONTINUE4]] ], [ [[TMP16]], %[[PRED_SDIV_IF5]] ]
; SINK-GATHER-NEXT: br i1 [[C]], label %[[PRED_SDIV_IF7:.*]], label %[[PRED_SDIV_CONTINUE8:.*]]
; SINK-GATHER: [[PRED_SDIV_IF7]]:
-; SINK-GATHER-NEXT: [[TMP18:%.*]] = extractelement <8 x i32> [[WIDE_LOAD2]], i64 4
+; SINK-GATHER-NEXT: [[TMP18:%.*]] = extractelement <8 x i32> [[WIDE_MASKED_LOAD]], i64 4
; SINK-GATHER-NEXT: [[TMP19:%.*]] = sdiv i32 [[TMP18]], [[X]]
; SINK-GATHER-NEXT: [[TMP20:%.*]] = insertelement <8 x i32> [[TMP17]], i32 [[TMP19]], i64 4
; SINK-GATHER-NEXT: br label %[[PRED_SDIV_CONTINUE8]]
@@ -110,7 +110,7 @@ define i32 @predicated_sdiv_masked_load(ptr %a, ptr %b, i32 %x, i1 %c) {
; SINK-GATHER-NEXT: [[TMP21:%.*]] = phi <8 x i32> [ [[TMP17]], %[[PRED_SDIV_CONTINUE6]] ], [ [[TMP20]], %[[PRED_SDIV_IF7]] ]
; SINK-GATHER-NEXT: br i1 [[C]], label %[[PRED_SDIV_IF9:.*]], label %[[PRED_SDIV_CONTINUE10:.*]]
; SINK-GATHER: [[PRED_SDIV_IF9]]:
-; SINK-GATHER-NEXT: [[TMP22:%.*]] = extractelement <8 x i32> [[WIDE_LOAD2]], i64 5
+; SINK-GATHER-NEXT: [[TMP22:%.*]] = extractelement <8 x i32> [[WIDE_MASKED_LOAD]], i64 5
; SINK-GATHER-NEXT: [[TMP23:%.*]] = sdiv i32 [[TMP22]], [[X]]
; SINK-GATHER-NEXT: [[TMP24:%.*]] = insertelement <8 x i32> [[TMP21]], i32 [[TMP23]], i64 5
; SINK-GATHER-NEXT: br label %[[PRED_SDIV_CONTINUE10]]
@@ -118,20 +118,20 @@ define i32 @predicated_sdiv_masked_load(ptr %a, ptr %b, i32 %x, i1 %c) {
; SINK-GATHER-NEXT: [[TMP25:%.*]] = phi <8 x i32> [ [[TMP21]], %[[PRED_SDIV_CONTINUE8]] ], [ [[TMP24]], %[[PRED_SDIV_IF9]] ]
; SINK-GATHER-NEXT: br i1 [[C]], label %[[PRED_SDIV_IF11:.*]], label %[[PRED_SDIV_CONTINUE12:.*]]
; SINK-GATHER: [[PRED_SDIV_IF11]]:
-; SINK-GATHER-NEXT: [[TMP26:%.*]] = extractelement <8 x i32> [[WIDE_LOAD2]], i64 6
+; SINK-GATHER-NEXT: [[TMP26:%.*]] = extractelement <8 x i32> [[WIDE_MASKED_LOAD]], i64 6
; SINK-GATHER-NEXT: [[TMP27:%.*]] = sdiv i32 [[TMP26]], [[X]]
; SINK-GATHER-NEXT: [[TMP28:%.*]] = insertelement <8 x i32> [[TMP25]], i32 [[TMP27]], i64 6
; SINK-GATHER-NEXT: br label %[[PRED_SDIV_CONTINUE12]]
; SINK-GATHER: [[PRED_SDIV_CONTINUE12]]:
; SINK-GATHER-NEXT: [[TMP29:%.*]] = phi <8 x i32> [ [[TMP25]], %[[PRED_SDIV_CONTINUE10]] ], [ [[TMP28]], %[[PRED_SDIV_IF11]] ]
-; SINK-GATHER-NEXT: br i1 [[C]], label %[[PRED_SDIV_IF14:.*]], label %[[PRED_SDIV_CONTINUE15]]
-; SINK-GATHER: [[PRED_SDIV_IF14]]:
-; SINK-GATHER-NEXT: [[TMP38:%.*]] = extractelement <8 x i32> [[WIDE_LOAD2]], i64 7
-; SINK-GATHER-NEXT: [[TMP39:%.*]] = sdiv i32 [[TMP38]], [[X]]
-; SINK-GATHER-NEXT: [[TMP32:%.*]] = insertelement <8 x i32> [[TMP29]], i32 [[TMP39]], i64 7
-; SINK-GATHER-NEXT: br label %[[PRED_SDIV_CONTINUE15]]
-; SINK-GATHER: [[PRED_SDIV_CONTINUE15]]:
-; SINK-GATHER-NEXT: [[TMP33:%.*]] = phi <8 x i32> [ [[TMP29]], %[[PRED_SDIV_CONTINUE12]] ], [ [[TMP32]], %[[PRED_SDIV_IF14]] ]
+; SINK-GATHER-NEXT: br i1 [[C]], label %[[PRED_SDIV_IF13:.*]], label %[[PRED_SDIV_CONTINUE14]]
+; SINK-GATHER: [[PRED_SDIV_IF13]]:
+; SINK-GATHER-NEXT: [[TMP30:%.*]] = extractelement <8 x i32> [[WIDE_MASKED_LOAD]], i64 7
+; SINK-GATHER-NEXT: [[TMP31:%.*]] = sdiv i32 [[TMP30]], [[X]]
+; SINK-GATHER-NEXT: [[TMP32:%.*]] = insertelement <8 x i32> [[TMP29]], i32 [[TMP31]], i64 7
+; SINK-GATHER-NEXT: br label %[[PRED_SDIV_CONTINUE14]]
+; SINK-GATHER: [[PRED_SDIV_CONTINUE14]]:
+; SINK-GATHER-NEXT: [[TMP33:%.*]] = phi <8 x i32> [ [[TMP29]], %[[PRED_SDIV_CONTINUE12]] ], [ [[TMP32]], %[[PRED_SDIV_IF13]] ]
; SINK-GATHER-NEXT: [[TMP34:%.*]] = add nsw <8 x i32> [[TMP33]], [[WIDE_LOAD]]
; SINK-GATHER-NEXT: [[PREDPHI:%.*]] = select i1 [[C]], <8 x i32> [[TMP34]], <8 x i32> [[WIDE_LOAD]]
; SINK-GATHER-NEXT: [[TMP35]] = add <8 x i32> [[VEC_PHI]], [[PREDPHI]]
@@ -195,25 +195,25 @@ define i32 @scalarize_and_sink_gather(ptr %a, i1 %c, i32 %x, i64 %n) {
; CHECK-NEXT: br i1 [[C]], label %[[PRED_UDIV_IF:.*]], label %[[PRED_UDIV_CONTINUE:.*]]
; CHECK: [[PRED_UDIV_IF]]:
; CHECK-NEXT: [[TMP1:%.*]] = mul i64 [[INDEX]], 777
-; CHECK-NEXT: [[TMP3:%.*]] = getelementptr inbounds i32, ptr [[A]], i64 [[TMP1]]
-; CHECK-NEXT: [[TMP4:%.*]] = load i32, ptr [[TMP3]], align 4
-; CHECK-NEXT: [[TMP5:%.*]] = udiv i32 [[TMP4]], [[X]]
-; CHECK-NEXT: [[TMP6:%.*]] = insertelement <2 x i32> poison, i32 [[TMP5]], i64 0
+; CHECK-NEXT: [[TMP2:%.*]] = getelementptr inbounds i32, ptr [[A]], i64 [[TMP1]]
+; CHECK-NEXT: [[TMP3:%.*]] = load i32, ptr [[TMP2]], align 4
+; CHECK-NEXT: [[TMP4:%.*]] = udiv i32 [[TMP3]], [[X]]
+; CHECK-NEXT: [[TMP5:%.*]] = insertelement <2 x i32> poison, i32 [[TMP4]], i64 0
; CHECK-NEXT: br label %[[PRED_UDIV_CONTINUE]]
; CHECK: [[PRED_UDIV_CONTINUE]]:
-; CHECK-NEXT: [[TMP7:%.*]] = phi <2 x i32> [ poison, %[[VECTOR_BODY]] ], [ [[TMP6]], %[[PRED_UDIV_IF]] ]
+; CHECK-NEXT: [[TMP6:%.*]] = phi <2 x i32> [ poison, %[[VECTOR_BODY]] ], [ [[TMP5]], %[[PRED_UDIV_IF]] ]
; CHECK-NEXT: br i1 [[C]], label %[[PRED_UDIV_IF1:.*]], label %[[PRED_UDIV_CONTINUE2]]
; CHECK: [[PRED_UDIV_IF1]]:
-; CHECK-NEXT: [[TMP8:%.*]] = add i64 [[INDEX]], 1
-; CHECK-NEXT: [[TMP12:%.*]] = mul i64 [[TMP8]], 777
-; CHECK-NEXT: [[TMP9:%.*]] = getelementptr inbounds i32, ptr [[A]], i64 [[TMP12]]
+; CHECK-NEXT: [[TMP7:%.*]] = add i64 [[INDEX]], 1
+; CHECK-NEXT: [[TMP8:%.*]] = mul i64 [[TMP7]], 777
+; CHECK-NEXT: [[TMP9:%.*]] = getelementptr inbounds i32, ptr [[A]], i64 [[TMP8]]
; CHECK-NEXT: [[TMP10:%.*]] = load i32, ptr [[TMP9]], align 4
; CHECK-NEXT: [[TMP11:%.*]] = udiv i32 [[TMP10]], [[X]]
-; CHECK-NEXT: [[TMP13:%.*]] = insertelement <2 x i32> [[TMP7]], i32 [[TMP11]], i64 1
+; CHECK-NEXT: [[TMP12:%.*]] = insertelement <2 x i32> [[TMP6]], i32 [[TMP11]], i64 1
; CHECK-NEXT: br label %[[PRED_UDIV_CONTINUE2]]
; CHECK: [[PRED_UDIV_CONTINUE2]]:
-; CHECK-NEXT: [[TMP17:%.*]] = phi <2 x i32> [ [[TMP7]], %[[PRED_UDIV_CONTINUE]] ], [ [[TMP13]], %[[PRED_UDIV_IF1]] ]
-; CHECK-NEXT: [[PREDPHI:%.*]] = select i1 [[C]], <2 x i32> [[TMP17]], <2 x i32> [[BROADCAST_SPLAT]]
+; CHECK-NEXT: [[TMP13:%.*]] = phi <2 x i32> [ [[TMP6]], %[[PRED_UDIV_CONTINUE]] ], [ [[TMP12]], %[[PRED_UDIV_IF1]] ]
+; CHECK-NEXT: [[PREDPHI:%.*]] = select i1 [[C]], <2 x i32> [[TMP13]], <2 x i32> [[BROADCAST_SPLAT]]
; CHECK-NEXT: [[TMP14]] = add <2 x i32> [[VEC_PHI]], [[PREDPHI]]
; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 2
; CHECK-NEXT: [[TMP15:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
@@ -266,84 +266,84 @@ define i32 @scalarize_and_sink_gather(ptr %a, i1 %c, i32 %x, i64 %n) {
; SINK-GATHER-NEXT: br i1 [[C]], label %[[PRED_UDIV_IF:.*]], label %[[PRED_UDIV_CONTINUE:.*]]
; SINK-GATHER: [[PRED_UDIV_IF]]:
; SINK-GATHER-NEXT: [[TMP1:%.*]] = extractelement <8 x i64> [[TMP0]], i64 0
-; SINK-GATHER-NEXT: [[TMP44:%.*]] = getelementptr inbounds i32, ptr [[A]], i64 [[TMP1]]
-; SINK-GATHER-NEXT: [[TMP45:%.*]] = load i32, ptr [[TMP44]], align 4
-; SINK-GATHER-NEXT: [[TMP46:%.*]] = udiv i32 [[TMP45]], [[X]]
-; SINK-GATHER-NEXT: [[TMP7:%.*]] = insertelement <8 x i32> poison, i32 [[TMP46]], i64 0
+; SINK-GATHER-NEXT: [[TMP2:%.*]] = getelementptr inbounds i32, ptr [[A]], i64 [[TMP1]]
+; SINK-GATHER-NEXT: [[TMP3:%.*]] = load i32, ptr [[TMP2]], align 4
+; SINK-GATHER-NEXT: [[TMP4:%.*]] = udiv i32 [[TMP3]], [[X]]
+; SINK-GATHER-NEXT: [[TMP5:%.*]] = insertelement <8 x i32> poison, i32 [[TMP4]], i64 0
; SINK-GATHER-NEXT: br label %[[PRED_UDIV_CONTINUE]]
; SINK-GATHER: [[PRED_UDIV_CONTINUE]]:
-; SINK-GATHER-NEXT: [[TMP8:%.*]] = phi <8 x i32> [ poison, %[[VECTOR_BODY]] ], [ [[TMP7]], %[[PRED_UDIV_IF]] ]
+; SINK-GATHER-NEXT: [[TMP6:%.*]] = phi <8 x i32> [ poison, %[[VECTOR_BODY]] ], [ [[TMP5]], %[[PRED_UDIV_IF]] ]
; SINK-GATHER-NEXT: br i1 [[C]], label %[[PRED_UDIV_IF1:.*]], label %[[PRED_UDIV_CONTINUE2:.*]]
; SINK-GATHER: [[PRED_UDIV_IF1]]:
-; SINK-GATHER-NEXT: [[TMP9:%.*]] = extractelement <8 x i64> [[TMP0]], i64 1
-; SINK-GATHER-NEXT: [[TMP10:%.*]] = getelementptr inbounds i32, ptr [[A]], i64 [[TMP9]]
-; SINK-GATHER-NEXT: [[TMP11:%.*]] = load i32, ptr [[TMP10]], align 4
-; SINK-GATHER-NEXT: [[TMP12:%.*]] = udiv i32 [[TMP11]], [[X]]
-; SINK-GATHER-NEXT: [[TMP13:%.*]] = insertelement <8 x i32> [[TMP8]], i32 [[TMP12]], i64 1
+; SINK-GATHER-NEXT: [[TMP7:%.*]] = extractelement <8 x i64> [[TMP0]], i64 1
+; SINK-GATHER-NEXT: [[TMP8:%.*]] = getelementptr inbounds i32, ptr [[A]], i64 [[TMP7]]
+; SINK-GATHER-NEXT: [[TMP9:%.*]] = load i32, ptr [[TMP8]], align 4
+; SINK-GATHER-NEXT: [[TMP10:%.*]] = udiv i32 [[TMP9]], [[X]]
+; SINK-GATHER-NEXT: [[TMP11:%.*]] = insertelement <8 x i32> [[TMP6]], i32 [[TMP10]], i64 1
; SINK-GATHER-NEXT: br label %[[PRED_UDIV_CONTINUE2]]
; SINK-GATHER: [[PRED_UDIV_CONTINUE2]]:
-; SINK-GATHER-NEXT: [[TMP14:%.*]] = phi <8 x i32> [ [[TMP8]], %[[PRED_UDIV_CONTINUE]] ], [ [[TMP13]], %[[PRED_UDIV_IF1]] ]
+; SINK-GATHER-NEXT: [[TMP12:%.*]] = phi <8 x i32> [ [[TMP6]], %[[PRED_UDIV_CONTINUE]] ], [ [[TMP11]], %[[PRED_UDIV_IF1]] ]
; SINK-GATHER-NEXT: br i1 [[C]], label %[[PRED_UDIV_IF3:.*]], label %[[PRED_UDIV_CONTINUE4:.*]]
; SINK-GATHER: [[PRED_UDIV_IF3]]:
-; SINK-GATHER-NEXT: [[TMP15:%.*]] = extractelement <8 x i64> [[TMP0]], i64 2
-; SINK-GATHER-NEXT: [[TMP16:%.*]] = getelementptr inbounds i32, ptr [[A]], i64 [[TMP15]]
-; SINK-GATHER-NEXT: [[TMP17:%.*]] = load i32, ptr [[TMP16]], align 4
-; SINK-GATHER-NEXT: [[TMP18:%.*]] = udiv i32 [[TMP17]], [[X]]
-; SINK-GATHER-NEXT: [[TMP19:%.*]] = insertelement <8 x i32> [[TMP14]], i32 [[TMP18]], i64 2
+; SINK-GATHER-NEXT: [[TMP13:%.*]] = extractelement <8 x i64> [[TMP0]], i64 2
+; SINK-GATHER-NEXT: [[TMP14:%.*]] = getelementptr inbounds i32, ptr [[A]], i64 [[TMP13]]
+; SINK-GATHER-NEXT: [[TMP15:%.*]] = load i32, ptr [[TMP14]], align 4
+; SINK-GATHER-NEXT: [[TMP16:%.*]] = udiv i32 [[TMP15]], [[X]]
+; SINK-GATHER-NEXT: [[TMP17:%.*]] = insertelement <8 x i32> [[TMP12]], i32 [[TMP16]], i64 2
; SINK-GATHER-NEXT: br label %[[PRED_UDIV_CONTINUE4]]
; SINK-GATHER: [[PRED_UDIV_CONTINUE4]]:
-; SINK-GATHER-NEXT: [[TMP20:%.*]] = phi <8 x i32> [ [[TMP14]], %[[PRED_UDIV_CONTINUE2]] ], [ [[TMP19]], %[[PRED_UDIV_IF3]] ]
+; SINK-GATHER-NEXT: [[TMP18:%.*]] = phi <8 x i32> [ [[TMP12]], %[[PRED_UDIV_CONTINUE2]] ], [ [[TMP17]], %[[PRED_UDIV_IF3]] ]
; SINK-GATHER-NEXT: br i1 [[C]], label %[[PRED_UDIV_IF5:.*]], label %[[PRED_UDIV_CONTINUE6:.*]]
; SINK-GATHER: [[PRED_UDIV_IF5]]:
-; SINK-GATHER-NEXT: [[TMP21:%.*]] = extractelement <8 x i64> [[TMP0]], i64 3
-; SINK-GATHER-NEXT: [[TMP22:%.*]] = getelementptr inbounds i32, ptr [[A]], i64 [[TMP21]]
-; SINK-GATHER-NEXT: [[TMP23:%.*]] = load i32, ptr [[TMP22]], align 4
-; SINK-GATHER-NEXT: [[TMP24:%.*]] = udiv i32 [[TMP23]], [[X]]
-; SINK-GATHER-NEXT: [[TMP25:%.*]] = insertelement <8 x i32> [[TMP20]], i32 [[TMP24]], i64 3
+; SINK-GATHER-NEXT: [[TMP19:%.*]] = extractelement <8 x i64> [[TMP0]], i64 3
+; SINK-GATHER-NEXT: [[TMP20:%.*]] = getelementptr inbounds i32, ptr [[A]], i64 [[TMP19]]
+; SINK-GATHER-NEXT: [[TMP21:%.*]] = load i32, ptr [[TMP20]], align 4
+; SINK-GATHER-NEXT: [[TMP22:%.*]] = udiv i32 [[TMP21]], [[X]]
+; SINK-GATHER-NEXT: [[TMP23:%.*]] = insertelement <8 x i32> [[TMP18]], i32 [[TMP22]], i64 3
; SINK-GATHER-NEXT: br label %[[PRED_UDIV_CONTINUE6]]
; SINK-GATHER: [[PRED_UDIV_CONTINUE6]]:
-; SINK-GATHER-NEXT: [[TMP26:%.*]] = phi <8 x i32> [ [[TMP20]], %[[PRED_UDIV_CONTINUE4]] ], [ [[TMP25]], %[[PRED_UDIV_IF5]] ]
+; SINK-GATHER-NEXT: [[TMP24:%.*]] = phi <8 x i32> [ [[TMP18]], %[[PRED_UDIV_CONTINUE4]] ], [ [[TMP23]], %[[PRED_UDIV_IF5]] ]
; SINK-GATHER-NEXT: br i1 [[C]], label %[[PRED_UDIV_IF7:.*]], label %[[PRED_UDIV_CONTINUE8:.*]]
; SINK-GATHER: [[PRED_UDIV_IF7]]:
-; SINK-GATHER-NEXT: [[TMP27:%.*]] = extractelement <8 x i64> [[TMP0]], i64 4
-; SINK-GATHER-NEXT: [[TMP28:%.*]] = getelementptr inbounds i32, ptr [[A]], i64 [[TMP27]]
-; SINK-GATHER-NEXT: [[TMP29:%.*]] = load i32, ptr [[TMP28]], align 4
-; SINK-GATHER-NEXT: [[TMP30:%.*]] = udiv i32 [[TMP29]], [[X]]
-; SINK-GATHER-NEXT: [[TMP31:%.*]] = insertelement <8 x i32> [[TMP26]], i32 [[TMP30]], i64 4
+; SINK-GATHER-NEXT: [[TMP25:%.*]] = extractelement <8 x i64> [[TMP0]], i64 4
+; SINK-GATHER-NEXT: [[TMP26:%.*]] = getelementptr inbounds i32, ptr [[A]], i64 [[TMP25]]
+; SINK-GATHER-NEXT: [[TMP27:%.*]] = load i32, ptr [[TMP26]], align 4
+; SINK-GATHER-NEXT: [[TMP28:%.*]] = udiv i32 [[TMP27]], [[X]]
+; SINK-GATHER-NEXT: [[TMP29:%.*]] = insertelement <8 x i32> [[TMP24]], i32 [[TMP28]], i64 4
; SINK-GATHER-NEXT: br label %[[PRED_UDIV_CONTINUE8]]
; SINK-GATHER: [[PRED_UDIV_CONTINUE8]]:
-; SINK-GATHER-NEXT: [[TMP32:%.*]] = phi <8 x i32> [ [[TMP26]], %[[PRED_UDIV_CONTINUE6]] ], [ [[TMP31]], %[[PRED_UDIV_IF7]] ]
+; SINK-GATHER-NEXT: [[TMP30:%.*]] = phi <8 x i32> [ [[TMP24]], %[[PRED_UDIV_CONTINUE6]] ], [ [[TMP29]], %[[PRED_UDIV_IF7]] ]
; SINK-GATHER-NEXT: br i1 [[C]], label %[[PRED_UDIV_IF9:.*]], label %[[PRED_UDIV_CONTINUE10:.*]]
; SINK-GATHER: [[PRED_UDIV_IF9]]:
-; SINK-GATHER-NEXT: [[TMP33:%.*]] = extractelement <8 x i64> [[TMP0]], i64 5
-; SINK-GATHER-NEXT: [[TMP34:%.*]] = getelementptr inbounds i32, ptr [[A]], i64 [[TMP33]]
-; SINK-GATHER-NEXT: [[TMP35:%.*]] = load i32, ptr [[TMP34]], align 4
-; SINK-GATHER-NEXT: [[TMP36:%.*]] = udiv i32 [[TMP35]], [[X]]
-; SINK-GATHER-NEXT: [[TMP37:%.*]] = insertelement <8 x i32> [[TMP32]], i32 [[TMP36]], i64 5
+; SINK-GATHER-NEXT: [[TMP31:%.*]] = extractelement <8 x i64> [[TMP0]], i64 5
+; SINK-GATHER-NEXT: [[TMP32:%.*]] = getelementptr inbounds i32, ptr [[A]], i64 [[TMP31]]
+; SINK-GATHER-NEXT: [[TMP33:%.*]] = load i32, ptr [[TMP32]], align 4
+; SINK-GATHER-NEXT: [[TMP34:%.*]] = udiv i32 [[TMP33]], [[X]]
+; SINK-GATHER-NEXT: [[TMP35:%.*]] = insertelement <8 x i32> [[TMP30]], i32 [[TMP34]], i64 5
; SINK-GATHER-NEXT: br label %[[PRED_UDIV_CONTINUE10]]
; SINK-GATHER: [[PRED_UDIV_CONTINUE10]]:
-; SINK-GATHER-NEXT: [[TMP38:%.*]] = phi <8 x i32> [ [[TMP32]], %[[PRED_UDIV_CONTINUE8]] ], [ [[TMP37]], %[[PRED_UDIV_IF9]] ]
+; SINK-GATHER-NEXT: [[TMP36:%.*]] = phi <8 x i32> [ [[TMP30]], %[[PRED_UDIV_CONTINUE8]] ], [ [[TMP35]], %[[PRED_UDIV_IF9]] ]
; SINK-GATHER-NEXT: br i1 [[C]], label %[[PRED_UDIV_IF11:.*]], label %[[PRED_UDIV_CONTINUE12:.*]]
; SINK-GATHER: [[PRED_UDIV_IF11]]:
-; SINK-GATHER-NEXT: [[TMP39:%.*]] = extractelement <8 x i64> [[TMP0]], i64 6
-; SINK-GATHER-NEXT: [[TMP40:%.*]] = getelementptr inbounds i32, ptr [[A]], i64 [[TMP39]]
-; SINK-GATHER-NEXT: [[TMP41:%.*]] = load i32, ptr [[TMP40]], align 4
-; SINK-GATHER-NEXT: [[TMP42:%.*]] = udiv i32 [[TMP41]], [[X]]
-; SINK-GATHER-NEXT: [[TMP43:%.*]] = insertelement <8 x i32> [[TMP38]], i32 [[TMP42]], i64 6
+; SINK-GATHER-NEXT: [[TMP37:%.*]] = extractelement <8 x i64> [[TMP0]], i64 6
+; SINK-GATHER-NEXT: [[TMP38:%.*]] = getelementptr inbounds i32, ptr [[A]], i64 [[TMP37]]
+; SINK-GATHER-NEXT: [[TMP39:%.*]] = load i32, ptr [[TMP38]], align 4
+; SINK-GATHER-NEXT: [[TMP40:%.*]] = udiv i32 [[TMP39]], [[X]]
+; SINK-GATHER-NEXT: [[TMP41:%.*]] = insertelement <8 x i32> [[TMP36]], i32 [[TMP40]], i64 6
; SINK-GATHER-NEXT: br label %[[PRED_UDIV_CONTINUE12]]
; SINK-GATHER: [[PRED_UDIV_CONTINUE12]]:
-; SINK-GATHER-NEXT: [[TMP52:%.*]] = phi <8 x i32> [ [[TMP38]], %[[PRED_UDIV_CONTINUE10]] ], [ [[TMP43]], %[[PRED_UDIV_IF11]] ]
+; SINK-GATHER-NEXT: [[TMP42:%.*]] = phi <8 x i32> [ [[TMP36]], %[[PRED_UDIV_CONTINUE10]] ], [ [[TMP41]], %[[PRED_UDIV_IF11]] ]
; SINK-GATHER-NEXT: br i1 [[C]], label %[[PRED_UDIV_IF13:.*]], label %[[PRED_UDIV_CONTINUE14]]
; SINK-GATHER: [[PRED_UDIV_IF13]]:
-; SINK-GATHER-NEXT: [[TMP53:%.*]] = extractelement <8 x i64> [[TMP0]], i64 7
-; SINK-GATHER-NEXT: [[TMP54:%.*]] = getelementptr inbounds i32, ptr [[A]], i64 [[TMP53]]
-; SINK-GATHER-NEXT: [[TMP47:%.*]] = load i32, ptr [[TMP54]], align 4
-; SINK-GATHER-NEXT: [[TMP48:%.*]] = udiv i32 [[TMP47]], [[X]]
-; SINK-GATHER-NEXT: [[TMP55:%.*]] = insertelement <8 x i32> [[TMP52]], i32 [[TMP48]], i64 7
+; SINK-GATHER-NEXT: [[TMP43:%.*]] = extractelement <8 x i64> [[TMP0]], i64 7
+; SINK-GATHER-NEXT: [[TMP44:%.*]] = getelementptr inbounds i32, ptr [[A]], i64 [[TMP43]]
+; SINK-GATHER-NEXT: [[TMP45:%.*]] = load i32, ptr [[TMP44]], align 4
+; SINK-GATHER-NEXT: [[TMP46:%.*]] = udiv i32 [[TMP45]], [[X]]
+; SINK-GATHER-NEXT: [[TMP47:%.*]] = insertelement <8 x i32> [[TMP42]], i32 [[TMP46]], i64 7
; SINK-GATHER-NEXT: br label %[[PRED_UDIV_CONTINUE14]]
; SINK-GATHER: [[PRED_UDIV_CONTINUE14]]:
-; SINK-GATHER-NEXT: [[TMP56:%.*]] = phi <8 x i32> [ [[TMP52]], %[[PRED_UDIV_CONTINUE12]] ], [ [[TMP55]], %[[PRED_UDIV_IF13]] ]
-; SINK-GATHER-NEXT: [[PREDPHI:%.*]] = select i1 [[C]], <8 x i32> [[TMP56]], <8 x i32> [[BROADCAST_SPLAT]]
+; SINK-GATHER-NEXT: [[TMP48:%.*]] = phi <8 x i32> [ [[TMP42]], %[[PRED_UDIV_CONTINUE12]] ], [ [[TMP47]], %[[PRED_UDIV_IF13]] ]
+; SINK-GATHER-NEXT: [[PREDPHI:%.*]] = select i1 [[C]], <8 x i32> [[TMP48]], <8 x i32> [[BROADCAST_SPLAT]]
; SINK-GATHER-NEXT: [[TMP49]] = add <8 x i32> [[VEC_PHI]], [[PREDPHI]]
; SINK-GATHER-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 8
; SINK-GATHER-NEXT: [[VEC_IND_NEXT]] = add nuw nsw <8 x i64> [[VEC_IND]], splat (i64 8)
diff --git a/llvm/test/Transforms/LoopVectorize/constant-fold-commutative-and.ll b/llvm/test/Transforms/LoopVectorize/constant-fold-commutative-and.ll
index f4c85ca8ae3f2..94230d4061977 100644
--- a/llvm/test/Transforms/LoopVectorize/constant-fold-commutative-and.ll
+++ b/llvm/test/Transforms/LoopVectorize/constant-fold-commutative-and.ll
@@ -18,25 +18,23 @@ define void @constant_fold_commutative_and(ptr %ptr.n, ptr noalias %p, i1 %cond)
; CHECK-NEXT: [[TMP1:%.*]] = icmp ule <2 x i8> [[VEC_IND]], splat (i8 16)
; CHECK-NEXT: [[TMP6:%.*]] = extractelement <2 x i1> [[TMP1]], i64 0
; CHECK-NEXT: [[TMP2:%.*]] = load i64, ptr [[PTR_N]], align 4
-; CHECK-NEXT: [[BROADCAST_SPLATINSERT:%.*]] = insertelement <2 x i64> poison, i64 [[TMP2]], i64 0
-; CHECK-NEXT: [[BROADCAST_SPLAT:%.*]] = shufflevector <2 x i64> [[BROADCAST_SPLATINSERT]], <2 x i64> poison, <2 x i32> zeroinitializer
+; CHECK-NEXT: [[BROADCAST_SPLATINSERT1:%.*]] = insertelement <2 x i64> poison, i64 [[TMP2]], i64 0
+; CHECK-NEXT: [[BROADCAST_SPLAT2:%.*]] = shufflevector <2 x i64> [[BROADCAST_SPLATINSERT1]], <2 x i64> poison, <2 x i32> zeroinitializer
; CHECK-NEXT: br i1 [[COND]], label %[[LATCH2:.*]], label %[[PRED_11:.*]]
; CHECK: [[PRED_11]]:
-; CHECK-NEXT: [[TMP12:%.*]] = icmp uge <2 x i64> [[BROADCAST_SPLAT]], splat (i64 4)
-; CHECK-NEXT: [[TMP4:%.*]] = select <2 x i1> [[TMP1]], <2 x i1> [[TMP12]], <2 x i1> zeroinitializer
+; CHECK-NEXT: [[TMP3:%.*]] = icmp uge <2 x i64> [[BROADCAST_SPLAT2]], splat (i64 4)
+; CHECK-NEXT: [[TMP4:%.*]] = select <2 x i1> [[TMP1]], <2 x i1> [[TMP3]], <2 x i1> zeroinitializer
; CHECK-NEXT: [[TMP5:%.*]] = extractelement <2 x i1> [[TMP4]], i64 0
-; CHECK-NEXT: [[TMP3:%.*]] = icmp ult i64 [[TMP2]], 7
+; CHECK-NEXT: [[TMP12:%.*]] = icmp ult i64 [[TMP2]], 7
; CHECK-NEXT: br label %[[LATCH2]]
; CHECK: [[LATCH2]]:
-; CHECK-NEXT: [[TMP7:%.*]] = phi i1 [ poison, %[[VECTOR_BODY]] ], [ [[TMP3]], %[[PRED_11]] ]
-; CHECK-NEXT: [[TMP13:%.*]] = phi i1 [ false, %[[VECTOR_BODY]] ], [ [[TMP5]], %[[PRED_11]] ]
-; CHECK-NEXT: [[BROADCAST_SPLATINSERT3:%.*]] = insertelement <2 x i1> poison, i1 [[TMP7]], i64 0
-; CHECK-NEXT: [[BROADCAST_SPLAT4:%.*]] = shufflevector <2 x i1> [[BROADCAST_SPLATINSERT3]], <2 x i1> poison, <2 x i32> zeroinitializer
-; CHECK-NEXT: [[PREDPHI:%.*]] = select i1 [[TMP13]], <2 x i1> [[BROADCAST_SPLAT4]], <2 x i1> [[TMP1]]
+; CHECK-NEXT: [[TMP7:%.*]] = phi i1 [ false, %[[VECTOR_BODY]] ], [ [[TMP6]], %[[PRED_11]] ]
+; CHECK-NEXT: [[TMP13:%.*]] = phi i1 [ poison, %[[VECTOR_BODY]] ], [ [[TMP12]], %[[PRED_11]] ]
+; CHECK-NEXT: [[TMP14:%.*]] = phi i1 [ false, %[[VECTOR_BODY]] ], [ [[TMP5]], %[[PRED_11]] ]
+; CHECK-NEXT: [[TMP9:%.*]] = select i1 [[TMP14]], i1 [[TMP13]], i1 [[TMP7]]
; CHECK-NEXT: br i1 [[TMP6]], label %[[PRED_STORE_IF:.*]], label %[[PRED_STORE_CONTINUE:.*]]
; CHECK: [[PRED_STORE_IF]]:
; CHECK-NEXT: [[TMP8:%.*]] = getelementptr i8, ptr [[P]], i64 [[INDEX]]
-; CHECK-NEXT: [[TMP9:%.*]] = extractelement <2 x i1> [[PREDPHI]], i64 0
; CHECK-NEXT: store i1 [[TMP9]], ptr [[TMP8]], align 1
; CHECK-NEXT: br label %[[PRED_STORE_CONTINUE]]
; CHECK: [[PRED_STORE_CONTINUE]]:
@@ -45,8 +43,7 @@ define void @constant_fold_commutative_and(ptr %ptr.n, ptr noalias %p, i1 %cond)
; CHECK: [[PRED_STORE_IF4]]:
; CHECK-NEXT: [[TMP11:%.*]] = add i64 [[INDEX]], 1
; CHECK-NEXT: [[TMP20:%.*]] = getelementptr i8, ptr [[P]], i64 [[TMP11]]
-; CHECK-NEXT: [[TMP14:%.*]] = extractelement <2 x i1> [[PREDPHI]], i64 1
-; CHECK-NEXT: store i1 [[TMP14]], ptr [[TMP20]], align 1
+; CHECK-NEXT: store i1 [[TMP9]], ptr [[TMP20]], align 1
; CHECK-NEXT: br label %[[PRED_STORE_CONTINUE5]]
; CHECK: [[PRED_STORE_CONTINUE5]]:
; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 2
diff --git a/llvm/test/Transforms/LoopVectorize/constantfolder.ll b/llvm/test/Transforms/LoopVectorize/constantfolder.ll
index bc5b2d4f8e307..d16203adfc1f4 100644
--- a/llvm/test/Transforms/LoopVectorize/constantfolder.ll
+++ b/llvm/test/Transforms/LoopVectorize/constantfolder.ll
@@ -12,10 +12,10 @@ define void @const_fold_ptradd(ptr %dst, i64 %d) {
; CHECK-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[LOOP_LATCH1:.*]] ]
; CHECK-NEXT: br label %[[LOOP_LATCH1]]
; CHECK: [[LOOP_LATCH1]]:
-; CHECK-NEXT: [[TMP2:%.*]] = phi i1 [ false, %[[VECTOR_BODY]] ]
-; CHECK-NEXT: [[PREDPHI:%.*]] = select i1 [[TMP2]], i64 [[D]], i64 0
-; CHECK-NEXT: [[TMP0:%.*]] = getelementptr i16, ptr [[DST]], i64 [[PREDPHI]]
-; CHECK-NEXT: store i16 0, ptr [[TMP0]], align 2
+; CHECK-NEXT: [[TMP0:%.*]] = phi i1 [ false, %[[VECTOR_BODY]] ]
+; CHECK-NEXT: [[PREDPHI:%.*]] = select i1 [[TMP0]], i64 [[D]], i64 0
+; CHECK-NEXT: [[TMP2:%.*]] = getelementptr i16, ptr [[DST]], i64 [[PREDPHI]]
+; CHECK-NEXT: store i16 0, ptr [[TMP2]], align 2
; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 4
; CHECK-NEXT: [[TMP1:%.*]] = icmp eq i64 [[INDEX_NEXT]], 100
; CHECK-NEXT: br i1 [[TMP1]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP0:![0-9]+]]
@@ -57,10 +57,10 @@ define void @const_fold_inbounds_ptradd(ptr %dst, i64 %d) {
; CHECK-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[LOOP_LATCH1:.*]] ]
; CHECK-NEXT: br label %[[LOOP_LATCH1]]
; CHECK: [[LOOP_LATCH1]]:
-; CHECK-NEXT: [[TMP2:%.*]] = phi i1 [ false, %[[VECTOR_BODY]] ]
-; CHECK-NEXT: [[PREDPHI:%.*]] = select i1 [[TMP2]], i64 [[D]], i64 0
-; CHECK-NEXT: [[TMP0:%.*]] = getelementptr inbounds i16, ptr [[DST]], i64 [[PREDPHI]]
-; CHECK-NEXT: store i16 0, ptr [[TMP0]], align 2
+; CHECK-NEXT: [[TMP0:%.*]] = phi i1 [ false, %[[VECTOR_BODY]] ]
+; CHECK-NEXT: [[PREDPHI:%.*]] = select i1 [[TMP0]], i64 [[D]], i64 0
+; CHECK-NEXT: [[TMP2:%.*]] = getelementptr inbounds i16, ptr [[DST]], i64 [[PREDPHI]]
+; CHECK-NEXT: store i16 0, ptr [[TMP2]], align 2
; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 4
; CHECK-NEXT: [[TMP1:%.*]] = icmp eq i64 [[INDEX_NEXT]], 100
; CHECK-NEXT: br i1 [[TMP1]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP3:![0-9]+]]
@@ -152,9 +152,9 @@ define void @const_fold_add_sub_mul_ashr_lshr(ptr %dst, i64 %d) {
; CHECK-NEXT: [[TMP1:%.*]] = add i64 2, [[PREDPHI]]
; CHECK-NEXT: [[TMP2:%.*]] = sub i64 [[TMP1]], [[PREDPHI]]
; CHECK-NEXT: [[TMP3:%.*]] = ashr i64 [[TMP2]], [[PREDPHI]]
-; CHECK-NEXT: [[TMP5:%.*]] = mul i64 [[TMP3]], 3
-; CHECK-NEXT: [[TMP4:%.*]] = lshr i64 [[TMP5]], [[PREDPHI]]
-; CHECK-NEXT: store i64 [[TMP4]], ptr [[DST]], align 8
+; CHECK-NEXT: [[TMP4:%.*]] = mul i64 [[TMP3]], 3
+; CHECK-NEXT: [[TMP5:%.*]] = lshr i64 [[TMP4]], [[PREDPHI]]
+; CHECK-NEXT: store i64 [[TMP5]], ptr [[DST]], align 8
; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 4
; CHECK-NEXT: [[TMP6:%.*]] = icmp eq i64 [[INDEX_NEXT]], 100
; CHECK-NEXT: br i1 [[TMP6]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP5:![0-9]+]]
@@ -203,9 +203,9 @@ define void @const_fold_and_or_xor(ptr %dst, i64 %d) {
; CHECK-NEXT: [[TMP0:%.*]] = phi i1 [ false, %[[VECTOR_BODY]] ]
; CHECK-NEXT: [[PREDPHI:%.*]] = select i1 [[TMP0]], i64 [[D]], i64 1
; CHECK-NEXT: [[TMP1:%.*]] = or i64 2, [[PREDPHI]]
-; CHECK-NEXT: [[TMP3:%.*]] = and i64 [[TMP1]], [[PREDPHI]]
-; CHECK-NEXT: [[TMP2:%.*]] = and i64 [[TMP3]], [[PREDPHI]]
-; CHECK-NEXT: store i64 [[TMP2]], ptr [[DST]], align 8
+; CHECK-NEXT: [[TMP2:%.*]] = and i64 [[TMP1]], [[PREDPHI]]
+; CHECK-NEXT: [[TMP3:%.*]] = and i64 [[TMP2]], [[PREDPHI]]
+; CHECK-NEXT: store i64 [[TMP3]], ptr [[DST]], align 8
; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 4
; CHECK-NEXT: [[TMP4:%.*]] = icmp eq i64 [[INDEX_NEXT]], 100
; CHECK-NEXT: br i1 [[TMP4]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP6:![0-9]+]]
@@ -254,9 +254,9 @@ define void @const_fold_cmp_zext(ptr %dst, i64 %d) {
; CHECK-NEXT: [[TMP0:%.*]] = phi i1 [ false, %[[VECTOR_BODY]] ]
; CHECK-NEXT: [[PREDPHI:%.*]] = select i1 [[TMP0]], <4 x i64> [[BROADCAST_SPLAT]], <4 x i64> splat (i64 1)
; CHECK-NEXT: [[TMP1:%.*]] = icmp ugt <4 x i64> splat (i64 2), [[PREDPHI]]
-; CHECK-NEXT: [[TMP4:%.*]] = zext <4 x i1> [[TMP1]] to <4 x i8>
-; CHECK-NEXT: [[TMP2:%.*]] = extractelement <4 x i8> [[TMP4]], i64 3
-; CHECK-NEXT: store i8 [[TMP2]], ptr [[DST]], align 1
+; CHECK-NEXT: [[TMP2:%.*]] = zext <4 x i1> [[TMP1]] to <4 x i8>
+; CHECK-NEXT: [[TMP4:%.*]] = extractelement <4 x i8> [[TMP2]], i64 3
+; CHECK-NEXT: store i8 [[TMP4]], ptr [[DST]], align 1
; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 4
; CHECK-NEXT: [[TMP3:%.*]] = icmp eq i64 [[INDEX_NEXT]], 100
; CHECK-NEXT: br i1 [[TMP3]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP7:![0-9]+]]
@@ -303,9 +303,9 @@ define void @const_fold_trunc(ptr %dst, i64 %d) {
; CHECK: [[LOOP_LATCH1]]:
; CHECK-NEXT: [[TMP0:%.*]] = phi i1 [ false, %[[VECTOR_BODY]] ]
; CHECK-NEXT: [[PREDPHI:%.*]] = select i1 [[TMP0]], <4 x i64> [[BROADCAST_SPLAT]], <4 x i64> zeroinitializer
-; CHECK-NEXT: [[TMP3:%.*]] = trunc <4 x i64> [[PREDPHI]] to <4 x i16>
-; CHECK-NEXT: [[TMP1:%.*]] = extractelement <4 x i16> [[TMP3]], i64 3
-; CHECK-NEXT: store i16 [[TMP1]], ptr [[DST]], align 2
+; CHECK-NEXT: [[TMP1:%.*]] = trunc <4 x i64> [[PREDPHI]] to <4 x i16>
+; CHECK-NEXT: [[TMP3:%.*]] = extractelement <4 x i16> [[TMP1]], i64 3
+; CHECK-NEXT: store i16 [[TMP3]], ptr [[DST]], align 2
; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 4
; CHECK-NEXT: [[TMP2:%.*]] = icmp eq i64 [[INDEX_NEXT]], 100
; CHECK-NEXT: br i1 [[TMP2]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP8:![0-9]+]]
diff --git a/llvm/test/Transforms/LoopVectorize/hoist-predicated-loads-with-predicated-stores.ll b/llvm/test/Transforms/LoopVectorize/hoist-predicated-loads-with-predicated-stores.ll
index 06aff8ed57d2e..19bc19a506677 100644
--- a/llvm/test/Transforms/LoopVectorize/hoist-predicated-loads-with-predicated-stores.ll
+++ b/llvm/test/Transforms/LoopVectorize/hoist-predicated-loads-with-predicated-stores.ll
@@ -1637,10 +1637,10 @@ define void @sink_stores_cse_select_dropping_fmf(ptr %dst, ptr %src, ptr %invar.
; CHECK: [[VECTOR_BODY]]:
; CHECK-NEXT: [[TMP8:%.*]] = phi float [ [[TMP5]], %[[ELSE10]] ], [ poison, %[[THEN11]] ]
; CHECK-NEXT: [[TMP9:%.*]] = phi float [ poison, %[[ELSE10]] ], [ [[TMP4]], %[[THEN11]] ]
-; CHECK-NEXT: [[TMP12:%.*]] = phi i1 [ false, %[[ELSE10]] ], [ true, %[[THEN11]] ]
-; CHECK-NEXT: [[PREDPHI:%.*]] = select i1 [[TMP12]], float [[TMP9]], float [[TMP8]]
-; CHECK-NEXT: [[TMP10:%.*]] = fmul nnan float [[PREDPHI]], 5.000000e+00
-; CHECK-NEXT: store float [[TMP10]], ptr [[INVAR_DST]], align 4, !alias.scope [[META126:![0-9]+]], !noalias [[META119]]
+; CHECK-NEXT: [[TMP10:%.*]] = phi i1 [ false, %[[ELSE10]] ], [ true, %[[THEN11]] ]
+; CHECK-NEXT: [[PREDPHI:%.*]] = select i1 [[TMP10]], float [[TMP9]], float [[TMP8]]
+; CHECK-NEXT: [[TMP12:%.*]] = fmul nnan float [[PREDPHI]], 5.000000e+00
+; CHECK-NEXT: store float [[TMP12]], ptr [[INVAR_DST]], align 4, !alias.scope [[META126:![0-9]+]], !noalias [[META119]]
; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 2
; CHECK-NEXT: [[TMP11:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
; CHECK-NEXT: br i1 [[TMP11]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY1]], !llvm.loop [[LOOP127:![0-9]+]]
diff --git a/llvm/test/Transforms/LoopVectorize/hoist-predicated-loads.ll b/llvm/test/Transforms/LoopVectorize/hoist-predicated-loads.ll
index 7c2470f7d590e..3dae330587dc9 100644
--- a/llvm/test/Transforms/LoopVectorize/hoist-predicated-loads.ll
+++ b/llvm/test/Transforms/LoopVectorize/hoist-predicated-loads.ll
@@ -919,7 +919,7 @@ define void @hoist_predicated_load_with_chained_geps1(ptr %dst, ptr %src, i1 %co
; CHECK: [[ELSE3]]:
; CHECK-NEXT: [[TMP20:%.*]] = getelementptr [11 x i16], ptr [[SRC]], i64 [[TMP2]]
; CHECK-NEXT: [[TMP21:%.*]] = getelementptr i8, ptr [[TMP20]], i64 8
-; CHECK-NEXT: [[TMP8:%.*]] = load i16, ptr [[TMP21]], align 2
+; CHECK-NEXT: [[TMP9:%.*]] = load i16, ptr [[TMP21]], align 2
; CHECK-NEXT: br label %[[LOOP_LATCH5]]
; CHECK: [[THEN4]]:
; CHECK-NEXT: [[TMP3:%.*]] = getelementptr [11 x i16], ptr [[SRC]], i64 [[TMP2]]
@@ -927,10 +927,10 @@ define void @hoist_predicated_load_with_chained_geps1(ptr %dst, ptr %src, i1 %co
; CHECK-NEXT: [[TMP5:%.*]] = load i16, ptr [[TMP4]], align 2
; CHECK-NEXT: br label %[[LOOP_LATCH5]]
; CHECK: [[LOOP_LATCH5]]:
-; CHECK-NEXT: [[TMP6:%.*]] = phi i16 [ [[TMP8]], %[[ELSE3]] ], [ poison, %[[THEN4]] ]
+; CHECK-NEXT: [[TMP6:%.*]] = phi i16 [ [[TMP9]], %[[ELSE3]] ], [ poison, %[[THEN4]] ]
; CHECK-NEXT: [[TMP7:%.*]] = phi i16 [ poison, %[[ELSE3]] ], [ [[TMP5]], %[[THEN4]] ]
-; CHECK-NEXT: [[TMP9:%.*]] = phi i1 [ false, %[[ELSE3]] ], [ true, %[[THEN4]] ]
-; CHECK-NEXT: [[PREDPHI:%.*]] = select i1 [[TMP9]], i16 [[TMP7]], i16 [[TMP6]]
+; CHECK-NEXT: [[TMP8:%.*]] = phi i1 [ false, %[[ELSE3]] ], [ true, %[[THEN4]] ]
+; CHECK-NEXT: [[PREDPHI:%.*]] = select i1 [[TMP8]], i16 [[TMP7]], i16 [[TMP6]]
; CHECK-NEXT: store i16 [[PREDPHI]], ptr [[DST]], align 2, !alias.scope [[META68:![0-9]+]], !noalias [[META71:![0-9]+]]
; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[TMP2]], 2
; CHECK-NEXT: [[TMP26:%.*]] = icmp eq i64 [[INDEX_NEXT]], 100
@@ -990,17 +990,17 @@ define void @hoist_predicated_load_with_chained_geps2(ptr %dst, ptr %src, i1 %co
; CHECK-NEXT: br i1 [[COND]], label %[[THEN4:.*]], label %[[ELSE3:.*]]
; CHECK: [[ELSE3]]:
; CHECK-NEXT: [[TMP21:%.*]] = getelementptr i8, ptr [[TMP4]], i64 8
-; CHECK-NEXT: [[TMP7:%.*]] = load i16, ptr [[TMP21]], align 2
+; CHECK-NEXT: [[TMP8:%.*]] = load i16, ptr [[TMP21]], align 2
; CHECK-NEXT: br label %[[LOOP_LATCH5]]
; CHECK: [[THEN4]]:
; CHECK-NEXT: [[TMP3:%.*]] = getelementptr i8, ptr [[TMP4]], i64 8
-; CHECK-NEXT: [[TMP8:%.*]] = load i16, ptr [[TMP3]], align 2
+; CHECK-NEXT: [[TMP9:%.*]] = load i16, ptr [[TMP3]], align 2
; CHECK-NEXT: br label %[[LOOP_LATCH5]]
; CHECK: [[LOOP_LATCH5]]:
-; CHECK-NEXT: [[TMP5:%.*]] = phi i16 [ [[TMP7]], %[[ELSE3]] ], [ poison, %[[THEN4]] ]
-; CHECK-NEXT: [[TMP6:%.*]] = phi i16 [ poison, %[[ELSE3]] ], [ [[TMP8]], %[[THEN4]] ]
-; CHECK-NEXT: [[TMP9:%.*]] = phi i1 [ false, %[[ELSE3]] ], [ true, %[[THEN4]] ]
-; CHECK-NEXT: [[PREDPHI:%.*]] = select i1 [[TMP9]], i16 [[TMP6]], i16 [[TMP5]]
+; CHECK-NEXT: [[TMP5:%.*]] = phi i16 [ [[TMP8]], %[[ELSE3]] ], [ poison, %[[THEN4]] ]
+; CHECK-NEXT: [[TMP6:%.*]] = phi i16 [ poison, %[[ELSE3]] ], [ [[TMP9]], %[[THEN4]] ]
+; CHECK-NEXT: [[TMP7:%.*]] = phi i1 [ false, %[[ELSE3]] ], [ true, %[[THEN4]] ]
+; CHECK-NEXT: [[PREDPHI:%.*]] = select i1 [[TMP7]], i16 [[TMP6]], i16 [[TMP5]]
; CHECK-NEXT: store i16 [[PREDPHI]], ptr [[DST]], align 2, !alias.scope [[META75:![0-9]+]], !noalias [[META78:![0-9]+]]
; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[TMP2]], 2
; CHECK-NEXT: [[TMP26:%.*]] = icmp eq i64 [[INDEX_NEXT]], 100
diff --git a/llvm/test/Transforms/LoopVectorize/if-pred-non-void.ll b/llvm/test/Transforms/LoopVectorize/if-pred-non-void.ll
index e2685ecbc32d0..873bda4d4d6a3 100644
--- a/llvm/test/Transforms/LoopVectorize/if-pred-non-void.ll
+++ b/llvm/test/Transforms/LoopVectorize/if-pred-non-void.ll
@@ -757,21 +757,21 @@ define i32 @predicated_udiv_scalarized_operand(ptr %a, i1 %c, i32 %x, i64 %n) {
; CHECK-NEXT: br i1 [[C]], label %[[PRED_UDIV_IF:.*]], label %[[PRED_UDIV_CONTINUE:.*]]
; CHECK: [[PRED_UDIV_IF]]:
; CHECK-NEXT: [[TMP1:%.*]] = extractelement <2 x i32> [[WIDE_LOAD]], i64 0
-; CHECK-NEXT: [[TMP4:%.*]] = add nsw i32 [[TMP1]], [[X]]
-; CHECK-NEXT: [[TMP5:%.*]] = udiv i32 [[TMP1]], [[TMP4]]
-; CHECK-NEXT: [[TMP6:%.*]] = insertelement <2 x i32> poison, i32 [[TMP5]], i64 0
+; CHECK-NEXT: [[TMP2:%.*]] = add nsw i32 [[TMP1]], [[X]]
+; CHECK-NEXT: [[TMP4:%.*]] = udiv i32 [[TMP1]], [[TMP2]]
+; CHECK-NEXT: [[TMP5:%.*]] = insertelement <2 x i32> poison, i32 [[TMP4]], i64 0
; CHECK-NEXT: br label %[[PRED_UDIV_CONTINUE]]
; CHECK: [[PRED_UDIV_CONTINUE]]:
-; CHECK-NEXT: [[TMP7:%.*]] = phi <2 x i32> [ poison, %[[VECTOR_BODY]] ], [ [[TMP6]], %[[PRED_UDIV_IF]] ]
+; CHECK-NEXT: [[TMP6:%.*]] = phi <2 x i32> [ poison, %[[VECTOR_BODY]] ], [ [[TMP5]], %[[PRED_UDIV_IF]] ]
; CHECK-NEXT: br i1 [[C]], label %[[PRED_UDIV_IF1:.*]], label %[[PRED_UDIV_CONTINUE2]]
; CHECK: [[PRED_UDIV_IF1]]:
-; CHECK-NEXT: [[TMP8:%.*]] = extractelement <2 x i32> [[WIDE_LOAD]], i64 1
-; CHECK-NEXT: [[TMP9:%.*]] = add nsw i32 [[TMP8]], [[X]]
-; CHECK-NEXT: [[TMP10:%.*]] = udiv i32 [[TMP8]], [[TMP9]]
-; CHECK-NEXT: [[TMP11:%.*]] = insertelement <2 x i32> [[TMP7]], i32 [[TMP10]], i64 1
+; CHECK-NEXT: [[TMP7:%.*]] = extractelement <2 x i32> [[WIDE_LOAD]], i64 1
+; CHECK-NEXT: [[TMP8:%.*]] = add nsw i32 [[TMP7]], [[X]]
+; CHECK-NEXT: [[TMP10:%.*]] = udiv i32 [[TMP7]], [[TMP8]]
+; CHECK-NEXT: [[TMP11:%.*]] = insertelement <2 x i32> [[TMP6]], i32 [[TMP10]], i64 1
; CHECK-NEXT: br label %[[PRED_UDIV_CONTINUE2]]
; CHECK: [[PRED_UDIV_CONTINUE2]]:
-; CHECK-NEXT: [[TMP12:%.*]] = phi <2 x i32> [ [[TMP7]], %[[PRED_UDIV_CONTINUE]] ], [ [[TMP11]], %[[PRED_UDIV_IF1]] ]
+; CHECK-NEXT: [[TMP12:%.*]] = phi <2 x i32> [ [[TMP6]], %[[PRED_UDIV_CONTINUE]] ], [ [[TMP11]], %[[PRED_UDIV_IF1]] ]
; CHECK-NEXT: [[PREDPHI:%.*]] = select i1 [[C]], <2 x i32> [[TMP12]], <2 x i32> [[WIDE_LOAD]]
; CHECK-NEXT: [[TMP13]] = add <2 x i32> [[VEC_PHI]], [[PREDPHI]]
; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 2
@@ -827,19 +827,19 @@ define i32 @predicated_udiv_scalarized_operand(ptr %a, i1 %c, i32 %x, i64 %n) {
; UNROLL-NO-VF-NEXT: br i1 [[C]], label %[[PRED_UDIV_IF:.*]], label %[[PRED_UDIV_CONTINUE:.*]]
; UNROLL-NO-VF: [[PRED_UDIV_IF]]:
; UNROLL-NO-VF-NEXT: [[TMP5:%.*]] = add nsw i32 [[TMP3]], [[X]]
-; UNROLL-NO-VF-NEXT: [[TMP10:%.*]] = udiv i32 [[TMP3]], [[TMP5]]
+; UNROLL-NO-VF-NEXT: [[TMP6:%.*]] = udiv i32 [[TMP3]], [[TMP5]]
; UNROLL-NO-VF-NEXT: br label %[[PRED_UDIV_CONTINUE]]
; UNROLL-NO-VF: [[PRED_UDIV_CONTINUE]]:
-; UNROLL-NO-VF-NEXT: [[TMP14:%.*]] = phi i32 [ poison, %[[VECTOR_BODY]] ], [ [[TMP10]], %[[PRED_UDIV_IF]] ]
+; UNROLL-NO-VF-NEXT: [[TMP7:%.*]] = phi i32 [ poison, %[[VECTOR_BODY]] ], [ [[TMP6]], %[[PRED_UDIV_IF]] ]
; UNROLL-NO-VF-NEXT: br i1 [[C]], label %[[PRED_UDIV_IF2:.*]], label %[[PRED_UDIV_CONTINUE3]]
; UNROLL-NO-VF: [[PRED_UDIV_IF2]]:
; UNROLL-NO-VF-NEXT: [[TMP8:%.*]] = add nsw i32 [[TMP4]], [[X]]
; UNROLL-NO-VF-NEXT: [[TMP9:%.*]] = udiv i32 [[TMP4]], [[TMP8]]
; UNROLL-NO-VF-NEXT: br label %[[PRED_UDIV_CONTINUE3]]
; UNROLL-NO-VF: [[PRED_UDIV_CONTINUE3]]:
-; UNROLL-NO-VF-NEXT: [[TMP15:%.*]] = phi i32 [ poison, %[[PRED_UDIV_CONTINUE]] ], [ [[TMP9]], %[[PRED_UDIV_IF2]] ]
-; UNROLL-NO-VF-NEXT: [[PREDPHI:%.*]] = select i1 [[C]], i32 [[TMP14]], i32 [[TMP3]]
-; UNROLL-NO-VF-NEXT: [[PREDPHI4:%.*]] = select i1 [[C]], i32 [[TMP15]], i32 [[TMP4]]
+; UNROLL-NO-VF-NEXT: [[TMP10:%.*]] = phi i32 [ poison, %[[PRED_UDIV_CONTINUE]] ], [ [[TMP9]], %[[PRED_UDIV_IF2]] ]
+; UNROLL-NO-VF-NEXT: [[PREDPHI:%.*]] = select i1 [[C]], i32 [[TMP7]], i32 [[TMP3]]
+; UNROLL-NO-VF-NEXT: [[PREDPHI4:%.*]] = select i1 [[C]], i32 [[TMP10]], i32 [[TMP4]]
; UNROLL-NO-VF-NEXT: [[TMP11]] = add i32 [[VEC_PHI]], [[PREDPHI]]
; UNROLL-NO-VF-NEXT: [[TMP12]] = add i32 [[VEC_PHI1]], [[PREDPHI4]]
; UNROLL-NO-VF-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 2
diff --git a/llvm/test/Transforms/LoopVectorize/if-pred-stores.ll b/llvm/test/Transforms/LoopVectorize/if-pred-stores.ll
index f3ed03987bcbd..d89ad9ed28ea9 100644
--- a/llvm/test/Transforms/LoopVectorize/if-pred-stores.ll
+++ b/llvm/test/Transforms/LoopVectorize/if-pred-stores.ll
@@ -321,27 +321,27 @@ define void @bug18724(i1 %cond, ptr %ptr, i1 %cond.2, i64 %v.1, i32 %v.2) {
; VEC-NEXT: [[TMP6:%.*]] = insertelement <2 x i32> zeroinitializer, i32 [[V_2]], i64 0
; VEC-NEXT: br label %[[VECTOR_BODY:.*]]
; VEC: [[VECTOR_BODY]]:
-; VEC-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[PRED_STORE_CONTINUE3:.*]] ]
-; VEC-NEXT: [[VEC_PHI:%.*]] = phi <2 x i32> [ [[TMP6]], %[[VECTOR_PH]] ], [ [[PREDPHI:%.*]], %[[PRED_STORE_CONTINUE3]] ]
+; VEC-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[PRED_STORE_CONTINUE2:.*]] ]
+; VEC-NEXT: [[VEC_PHI:%.*]] = phi <2 x i32> [ [[TMP6]], %[[VECTOR_PH]] ], [ [[PREDPHI:%.*]], %[[PRED_STORE_CONTINUE2]] ]
; VEC-NEXT: [[TMP7:%.*]] = add i64 [[V_1]], [[INDEX]]
; VEC-NEXT: [[TMP8:%.*]] = getelementptr inbounds [768 x i32], ptr [[PTR]], i64 0, i64 [[TMP7]]
; VEC-NEXT: [[WIDE_LOAD:%.*]] = load <2 x i32>, ptr [[TMP8]], align 4
-; VEC-NEXT: br i1 [[COND_2]], label %[[PRED_STORE_IF1:.*]], label %[[PRED_STORE_CONTINUE2:.*]]
+; VEC-NEXT: br i1 [[COND_2]], label %[[PRED_STORE_IF:.*]], label %[[PRED_STORE_CONTINUE:.*]]
+; VEC: [[PRED_STORE_IF]]:
+; VEC-NEXT: [[TMP9:%.*]] = extractelement <2 x i32> [[WIDE_LOAD]], i64 0
+; VEC-NEXT: store i32 [[TMP9]], ptr [[TMP8]], align 4
+; VEC-NEXT: br label %[[PRED_STORE_CONTINUE]]
+; VEC: [[PRED_STORE_CONTINUE]]:
+; VEC-NEXT: br i1 [[COND_2]], label %[[PRED_STORE_IF1:.*]], label %[[PRED_STORE_CONTINUE2]]
; VEC: [[PRED_STORE_IF1]]:
-; VEC-NEXT: [[TMP10:%.*]] = extractelement <2 x i32> [[WIDE_LOAD]], i64 0
-; VEC-NEXT: store i32 [[TMP10]], ptr [[TMP8]], align 4
+; VEC-NEXT: [[TMP10:%.*]] = add i64 [[TMP7]], 1
+; VEC-NEXT: [[TMP11:%.*]] = getelementptr inbounds [768 x i32], ptr [[PTR]], i64 0, i64 [[TMP10]]
+; VEC-NEXT: [[TMP12:%.*]] = extractelement <2 x i32> [[WIDE_LOAD]], i64 1
+; VEC-NEXT: store i32 [[TMP12]], ptr [[TMP11]], align 4
; VEC-NEXT: br label %[[PRED_STORE_CONTINUE2]]
; VEC: [[PRED_STORE_CONTINUE2]]:
-; VEC-NEXT: br i1 [[COND_2]], label %[[PRED_STORE_IF2:.*]], label %[[PRED_STORE_CONTINUE3]]
-; VEC: [[PRED_STORE_IF2]]:
-; VEC-NEXT: [[TMP11:%.*]] = add i64 [[TMP7]], 1
-; VEC-NEXT: [[TMP12:%.*]] = getelementptr inbounds [768 x i32], ptr [[PTR]], i64 0, i64 [[TMP11]]
-; VEC-NEXT: [[TMP13:%.*]] = extractelement <2 x i32> [[WIDE_LOAD]], i64 1
-; VEC-NEXT: store i32 [[TMP13]], ptr [[TMP12]], align 4
-; VEC-NEXT: br label %[[PRED_STORE_CONTINUE3]]
-; VEC: [[PRED_STORE_CONTINUE3]]:
-; VEC-NEXT: [[TMP16:%.*]] = add <2 x i32> [[VEC_PHI]], splat (i32 1)
-; VEC-NEXT: [[PREDPHI]] = select i1 [[COND_2]], <2 x i32> [[TMP16]], <2 x i32> [[VEC_PHI]]
+; VEC-NEXT: [[TMP13:%.*]] = add <2 x i32> [[VEC_PHI]], splat (i32 1)
+; VEC-NEXT: [[PREDPHI]] = select i1 [[COND_2]], <2 x i32> [[TMP13]], <2 x i32> [[VEC_PHI]]
; VEC-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 2
; VEC-NEXT: [[TMP14:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
; VEC-NEXT: br i1 [[TMP14]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP3:![0-9]+]]
diff --git a/llvm/test/Transforms/LoopVectorize/induction.ll b/llvm/test/Transforms/LoopVectorize/induction.ll
index 66766a9c03cd6..1683f784daf75 100644
--- a/llvm/test/Transforms/LoopVectorize/induction.ll
+++ b/llvm/test/Transforms/LoopVectorize/induction.ll
@@ -1969,20 +1969,20 @@ define i32 @scalarize_induction_variable_05(ptr %a, i32 %x, i1 %c, i32 %n) {
; CHECK-NEXT: br i1 [[C:%.*]], label [[PRED_UDIV_IF:%.*]], label [[PRED_UDIV_CONTINUE:%.*]]
; CHECK: pred.udiv.if:
; CHECK-NEXT: [[TMP5:%.*]] = extractelement <2 x i32> [[WIDE_LOAD]], i64 0
-; CHECK-NEXT: [[TMP4:%.*]] = udiv i32 [[TMP5]], [[INDEX]]
-; CHECK-NEXT: [[TMP12:%.*]] = insertelement <2 x i32> poison, i32 [[TMP4]], i64 0
+; CHECK-NEXT: [[TMP6:%.*]] = udiv i32 [[TMP5]], [[INDEX]]
+; CHECK-NEXT: [[TMP7:%.*]] = insertelement <2 x i32> poison, i32 [[TMP6]], i64 0
; CHECK-NEXT: br label [[PRED_UDIV_CONTINUE]]
; CHECK: pred.udiv.continue:
-; CHECK-NEXT: [[TMP6:%.*]] = phi <2 x i32> [ poison, [[VECTOR_BODY]] ], [ [[TMP12]], [[PRED_UDIV_IF]] ]
+; CHECK-NEXT: [[TMP12:%.*]] = phi <2 x i32> [ poison, [[VECTOR_BODY]] ], [ [[TMP7]], [[PRED_UDIV_IF]] ]
; CHECK-NEXT: br i1 [[C]], label [[PRED_UDIV_IF1:%.*]], label [[PRED_UDIV_CONTINUE2]]
; CHECK: pred.udiv.if1:
-; CHECK-NEXT: [[TMP7:%.*]] = add i32 [[INDEX]], 1
+; CHECK-NEXT: [[TMP14:%.*]] = add i32 [[INDEX]], 1
; CHECK-NEXT: [[TMP8:%.*]] = extractelement <2 x i32> [[WIDE_LOAD]], i64 1
-; CHECK-NEXT: [[TMP9:%.*]] = udiv i32 [[TMP8]], [[TMP7]]
-; CHECK-NEXT: [[TMP10:%.*]] = insertelement <2 x i32> [[TMP6]], i32 [[TMP9]], i64 1
+; CHECK-NEXT: [[TMP9:%.*]] = udiv i32 [[TMP8]], [[TMP14]]
+; CHECK-NEXT: [[TMP10:%.*]] = insertelement <2 x i32> [[TMP12]], i32 [[TMP9]], i64 1
; CHECK-NEXT: br label [[PRED_UDIV_CONTINUE2]]
; CHECK: pred.udiv.continue2:
-; CHECK-NEXT: [[TMP11:%.*]] = phi <2 x i32> [ [[TMP6]], [[PRED_UDIV_CONTINUE]] ], [ [[TMP10]], [[PRED_UDIV_IF1]] ]
+; CHECK-NEXT: [[TMP11:%.*]] = phi <2 x i32> [ [[TMP12]], [[PRED_UDIV_CONTINUE]] ], [ [[TMP10]], [[PRED_UDIV_IF1]] ]
; CHECK-NEXT: [[PREDPHI:%.*]] = select i1 [[C]], <2 x i32> [[TMP11]], <2 x i32> [[WIDE_LOAD]]
; CHECK-NEXT: [[TMP16]] = add <2 x i32> [[PREDPHI]], [[VEC_PHI]]
; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i32 [[INDEX]], 2
@@ -2032,21 +2032,21 @@ define i32 @scalarize_induction_variable_05(ptr %a, i32 %x, i1 %c, i32 %n) {
; IND-NEXT: br i1 [[C:%.*]], label [[PRED_UDIV_IF:%.*]], label [[PRED_UDIV_CONTINUE:%.*]]
; IND: pred.udiv.if:
; IND-NEXT: [[TMP2:%.*]] = extractelement <2 x i32> [[WIDE_LOAD]], i64 0
-; IND-NEXT: [[TMP4:%.*]] = udiv i32 [[TMP2]], [[INDEX]]
-; IND-NEXT: [[TMP5:%.*]] = insertelement <2 x i32> poison, i32 [[TMP4]], i64 0
+; IND-NEXT: [[TMP3:%.*]] = udiv i32 [[TMP2]], [[INDEX]]
+; IND-NEXT: [[TMP4:%.*]] = insertelement <2 x i32> poison, i32 [[TMP3]], i64 0
; IND-NEXT: br label [[PRED_UDIV_CONTINUE]]
; IND: pred.udiv.continue:
-; IND-NEXT: [[TMP6:%.*]] = phi <2 x i32> [ poison, [[VECTOR_BODY]] ], [ [[TMP5]], [[PRED_UDIV_IF]] ]
+; IND-NEXT: [[TMP5:%.*]] = phi <2 x i32> [ poison, [[VECTOR_BODY]] ], [ [[TMP4]], [[PRED_UDIV_IF]] ]
; IND-NEXT: br i1 [[C]], label [[PRED_UDIV_IF1:%.*]], label [[PRED_UDIV_CONTINUE2]]
; IND: pred.udiv.if1:
-; IND-NEXT: [[TMP7:%.*]] = add i32 [[INDEX]], 1
-; IND-NEXT: [[TMP8:%.*]] = extractelement <2 x i32> [[WIDE_LOAD]], i64 1
-; IND-NEXT: [[TMP9:%.*]] = udiv i32 [[TMP8]], [[TMP7]]
-; IND-NEXT: [[TMP10:%.*]] = insertelement <2 x i32> [[TMP6]], i32 [[TMP9]], i64 1
+; IND-NEXT: [[TMP6:%.*]] = add i32 [[INDEX]], 1
+; IND-NEXT: [[TMP7:%.*]] = extractelement <2 x i32> [[WIDE_LOAD]], i64 1
+; IND-NEXT: [[TMP8:%.*]] = udiv i32 [[TMP7]], [[TMP6]]
+; IND-NEXT: [[TMP9:%.*]] = insertelement <2 x i32> [[TMP5]], i32 [[TMP8]], i64 1
; IND-NEXT: br label [[PRED_UDIV_CONTINUE2]]
; IND: pred.udiv.continue2:
-; IND-NEXT: [[TMP11:%.*]] = phi <2 x i32> [ [[TMP6]], [[PRED_UDIV_CONTINUE]] ], [ [[TMP10]], [[PRED_UDIV_IF1]] ]
-; IND-NEXT: [[PREDPHI:%.*]] = select i1 [[C]], <2 x i32> [[TMP11]], <2 x i32> [[WIDE_LOAD]]
+; IND-NEXT: [[TMP10:%.*]] = phi <2 x i32> [ [[TMP5]], [[PRED_UDIV_CONTINUE]] ], [ [[TMP9]], [[PRED_UDIV_IF1]] ]
+; IND-NEXT: [[PREDPHI:%.*]] = select i1 [[C]], <2 x i32> [[TMP10]], <2 x i32> [[WIDE_LOAD]]
; IND-NEXT: [[TMP13]] = add <2 x i32> [[PREDPHI]], [[VEC_PHI]]
; IND-NEXT: [[INDEX_NEXT]] = add nuw i32 [[INDEX]], 2
; IND-NEXT: [[TMP14:%.*]] = icmp eq i32 [[INDEX_NEXT]], [[N_VEC]]
@@ -2097,41 +2097,41 @@ define i32 @scalarize_induction_variable_05(ptr %a, i32 %x, i1 %c, i32 %n) {
; UNROLL-NEXT: [[WIDE_LOAD2:%.*]] = load <2 x i32>, ptr [[TMP2]], align 4
; UNROLL-NEXT: br i1 [[C:%.*]], label [[PRED_UDIV_IF:%.*]], label [[PRED_UDIV_CONTINUE:%.*]]
; UNROLL: pred.udiv.if:
-; UNROLL-NEXT: [[TMP5:%.*]] = extractelement <2 x i32> [[WIDE_LOAD]], i64 0
-; UNROLL-NEXT: [[TMP7:%.*]] = udiv i32 [[TMP5]], [[INDEX]]
-; UNROLL-NEXT: [[TMP6:%.*]] = insertelement <2 x i32> poison, i32 [[TMP7]], i64 0
+; UNROLL-NEXT: [[TMP3:%.*]] = extractelement <2 x i32> [[WIDE_LOAD]], i64 0
+; UNROLL-NEXT: [[TMP4:%.*]] = udiv i32 [[TMP3]], [[INDEX]]
+; UNROLL-NEXT: [[TMP5:%.*]] = insertelement <2 x i32> poison, i32 [[TMP4]], i64 0
; UNROLL-NEXT: br label [[PRED_UDIV_CONTINUE]]
; UNROLL: pred.udiv.continue:
-; UNROLL-NEXT: [[TMP8:%.*]] = phi <2 x i32> [ poison, [[VECTOR_BODY]] ], [ [[TMP6]], [[PRED_UDIV_IF]] ]
+; UNROLL-NEXT: [[TMP6:%.*]] = phi <2 x i32> [ poison, [[VECTOR_BODY]] ], [ [[TMP5]], [[PRED_UDIV_IF]] ]
; UNROLL-NEXT: br i1 [[C]], label [[PRED_UDIV_IF3:%.*]], label [[PRED_UDIV_CONTINUE4:%.*]]
; UNROLL: pred.udiv.if3:
-; UNROLL-NEXT: [[TMP17:%.*]] = add i32 [[INDEX]], 1
-; UNROLL-NEXT: [[TMP18:%.*]] = extractelement <2 x i32> [[WIDE_LOAD]], i64 1
-; UNROLL-NEXT: [[TMP19:%.*]] = udiv i32 [[TMP18]], [[TMP17]]
-; UNROLL-NEXT: [[TMP11:%.*]] = insertelement <2 x i32> [[TMP8]], i32 [[TMP19]], i64 1
+; UNROLL-NEXT: [[TMP7:%.*]] = add i32 [[INDEX]], 1
+; UNROLL-NEXT: [[TMP8:%.*]] = extractelement <2 x i32> [[WIDE_LOAD]], i64 1
+; UNROLL-NEXT: [[TMP9:%.*]] = udiv i32 [[TMP8]], [[TMP7]]
+; UNROLL-NEXT: [[TMP10:%.*]] = insertelement <2 x i32> [[TMP6]], i32 [[TMP9]], i64 1
; UNROLL-NEXT: br label [[PRED_UDIV_CONTINUE4]]
; UNROLL: pred.udiv.continue4:
-; UNROLL-NEXT: [[TMP12:%.*]] = phi <2 x i32> [ [[TMP8]], [[PRED_UDIV_CONTINUE]] ], [ [[TMP11]], [[PRED_UDIV_IF3]] ]
+; UNROLL-NEXT: [[TMP11:%.*]] = phi <2 x i32> [ [[TMP6]], [[PRED_UDIV_CONTINUE]] ], [ [[TMP10]], [[PRED_UDIV_IF3]] ]
; UNROLL-NEXT: br i1 [[C]], label [[PRED_UDIV_IF5:%.*]], label [[PRED_UDIV_CONTINUE6:%.*]]
; UNROLL: pred.udiv.if5:
-; UNROLL-NEXT: [[TMP13:%.*]] = add i32 [[INDEX]], 2
-; UNROLL-NEXT: [[TMP14:%.*]] = extractelement <2 x i32> [[WIDE_LOAD2]], i64 0
-; UNROLL-NEXT: [[TMP9:%.*]] = udiv i32 [[TMP14]], [[TMP13]]
-; UNROLL-NEXT: [[BROADCAST_SPLATINSERT:%.*]] = insertelement <2 x i32> poison, i32 [[TMP9]], i64 0
+; UNROLL-NEXT: [[TMP12:%.*]] = add i32 [[INDEX]], 2
+; UNROLL-NEXT: [[TMP13:%.*]] = extractelement <2 x i32> [[WIDE_LOAD2]], i64 0
+; UNROLL-NEXT: [[TMP14:%.*]] = udiv i32 [[TMP13]], [[TMP12]]
+; UNROLL-NEXT: [[TMP15:%.*]] = insertelement <2 x i32> poison, i32 [[TMP14]], i64 0
; UNROLL-NEXT: br label [[PRED_UDIV_CONTINUE6]]
; UNROLL: pred.udiv.continue6:
-; UNROLL-NEXT: [[TMP26:%.*]] = phi <2 x i32> [ poison, [[PRED_UDIV_CONTINUE4]] ], [ [[BROADCAST_SPLATINSERT]], [[PRED_UDIV_IF5]] ]
+; UNROLL-NEXT: [[TMP16:%.*]] = phi <2 x i32> [ poison, [[PRED_UDIV_CONTINUE4]] ], [ [[TMP15]], [[PRED_UDIV_IF5]] ]
; UNROLL-NEXT: br i1 [[C]], label [[PRED_UDIV_IF7:%.*]], label [[PRED_UDIV_CONTINUE8]]
; UNROLL: pred.udiv.if7:
-; UNROLL-NEXT: [[TMP27:%.*]] = add i32 [[INDEX]], 3
-; UNROLL-NEXT: [[TMP28:%.*]] = extractelement <2 x i32> [[WIDE_LOAD2]], i64 1
-; UNROLL-NEXT: [[TMP20:%.*]] = udiv i32 [[TMP28]], [[TMP27]]
-; UNROLL-NEXT: [[TMP21:%.*]] = insertelement <2 x i32> [[TMP26]], i32 [[TMP20]], i64 1
+; UNROLL-NEXT: [[TMP17:%.*]] = add i32 [[INDEX]], 3
+; UNROLL-NEXT: [[TMP18:%.*]] = extractelement <2 x i32> [[WIDE_LOAD2]], i64 1
+; UNROLL-NEXT: [[TMP19:%.*]] = udiv i32 [[TMP18]], [[TMP17]]
+; UNROLL-NEXT: [[TMP20:%.*]] = insertelement <2 x i32> [[TMP16]], i32 [[TMP19]], i64 1
; UNROLL-NEXT: br label [[PRED_UDIV_CONTINUE8]]
; UNROLL: pred.udiv.continue8:
-; UNROLL-NEXT: [[TMP29:%.*]] = phi <2 x i32> [ [[TMP26]], [[PRED_UDIV_CONTINUE6]] ], [ [[TMP21]], [[PRED_UDIV_IF7]] ]
-; UNROLL-NEXT: [[PREDPHI:%.*]] = select i1 [[C]], <2 x i32> [[TMP12]], <2 x i32> [[WIDE_LOAD]]
-; UNROLL-NEXT: [[PREDPHI9:%.*]] = select i1 [[C]], <2 x i32> [[TMP29]], <2 x i32> [[WIDE_LOAD2]]
+; UNROLL-NEXT: [[TMP21:%.*]] = phi <2 x i32> [ [[TMP16]], [[PRED_UDIV_CONTINUE6]] ], [ [[TMP20]], [[PRED_UDIV_IF7]] ]
+; UNROLL-NEXT: [[PREDPHI:%.*]] = select i1 [[C]], <2 x i32> [[TMP11]], <2 x i32> [[WIDE_LOAD]]
+; UNROLL-NEXT: [[PREDPHI9:%.*]] = select i1 [[C]], <2 x i32> [[TMP21]], <2 x i32> [[WIDE_LOAD2]]
; UNROLL-NEXT: [[TMP22]] = add <2 x i32> [[PREDPHI]], [[VEC_PHI]]
; UNROLL-NEXT: [[TMP23]] = add <2 x i32> [[PREDPHI9]], [[VEC_PHI1]]
; UNROLL-NEXT: [[INDEX_NEXT]] = add nuw i32 [[INDEX]], 4
@@ -2184,41 +2184,41 @@ define i32 @scalarize_induction_variable_05(ptr %a, i32 %x, i1 %c, i32 %n) {
; UNROLL-NO-IC-NEXT: [[WIDE_LOAD2:%.*]] = load <2 x i32>, ptr [[TMP2]], align 4
; UNROLL-NO-IC-NEXT: br i1 [[C:%.*]], label [[PRED_UDIV_IF:%.*]], label [[PRED_UDIV_CONTINUE:%.*]]
; UNROLL-NO-IC: pred.udiv.if:
-; UNROLL-NO-IC-NEXT: [[TMP5:%.*]] = extractelement <2 x i32> [[WIDE_LOAD]], i64 0
-; UNROLL-NO-IC-NEXT: [[TMP7:%.*]] = udiv i32 [[TMP5]], [[INDEX]]
-; UNROLL-NO-IC-NEXT: [[TMP6:%.*]] = insertelement <2 x i32> poison, i32 [[TMP7]], i64 0
+; UNROLL-NO-IC-NEXT: [[TMP8:%.*]] = extractelement <2 x i32> [[WIDE_LOAD]], i64 0
+; UNROLL-NO-IC-NEXT: [[TMP9:%.*]] = udiv i32 [[TMP8]], [[INDEX]]
+; UNROLL-NO-IC-NEXT: [[TMP10:%.*]] = insertelement <2 x i32> poison, i32 [[TMP9]], i64 0
; UNROLL-NO-IC-NEXT: br label [[PRED_UDIV_CONTINUE]]
; UNROLL-NO-IC: pred.udiv.continue:
-; UNROLL-NO-IC-NEXT: [[TMP8:%.*]] = phi <2 x i32> [ poison, [[VECTOR_BODY]] ], [ [[TMP6]], [[PRED_UDIV_IF]] ]
+; UNROLL-NO-IC-NEXT: [[TMP11:%.*]] = phi <2 x i32> [ poison, [[VECTOR_BODY]] ], [ [[TMP10]], [[PRED_UDIV_IF]] ]
; UNROLL-NO-IC-NEXT: br i1 [[C]], label [[PRED_UDIV_IF3:%.*]], label [[PRED_UDIV_CONTINUE4:%.*]]
; UNROLL-NO-IC: pred.udiv.if3:
-; UNROLL-NO-IC-NEXT: [[TMP25:%.*]] = add i32 [[INDEX]], 1
-; UNROLL-NO-IC-NEXT: [[TMP26:%.*]] = extractelement <2 x i32> [[WIDE_LOAD]], i64 1
-; UNROLL-NO-IC-NEXT: [[TMP27:%.*]] = udiv i32 [[TMP26]], [[TMP25]]
-; UNROLL-NO-IC-NEXT: [[TMP11:%.*]] = insertelement <2 x i32> [[TMP8]], i32 [[TMP27]], i64 1
+; UNROLL-NO-IC-NEXT: [[TMP13:%.*]] = add i32 [[INDEX]], 1
+; UNROLL-NO-IC-NEXT: [[TMP14:%.*]] = extractelement <2 x i32> [[WIDE_LOAD]], i64 1
+; UNROLL-NO-IC-NEXT: [[TMP15:%.*]] = udiv i32 [[TMP14]], [[TMP13]]
+; UNROLL-NO-IC-NEXT: [[TMP16:%.*]] = insertelement <2 x i32> [[TMP11]], i32 [[TMP15]], i64 1
; UNROLL-NO-IC-NEXT: br label [[PRED_UDIV_CONTINUE4]]
; UNROLL-NO-IC: pred.udiv.continue4:
-; UNROLL-NO-IC-NEXT: [[TMP12:%.*]] = phi <2 x i32> [ [[TMP8]], [[PRED_UDIV_CONTINUE]] ], [ [[TMP11]], [[PRED_UDIV_IF3]] ]
+; UNROLL-NO-IC-NEXT: [[TMP17:%.*]] = phi <2 x i32> [ [[TMP11]], [[PRED_UDIV_CONTINUE]] ], [ [[TMP16]], [[PRED_UDIV_IF3]] ]
; UNROLL-NO-IC-NEXT: br i1 [[C]], label [[PRED_UDIV_IF5:%.*]], label [[PRED_UDIV_CONTINUE6:%.*]]
; UNROLL-NO-IC: pred.udiv.if5:
-; UNROLL-NO-IC-NEXT: [[TMP13:%.*]] = add i32 [[INDEX]], 2
-; UNROLL-NO-IC-NEXT: [[TMP14:%.*]] = extractelement <2 x i32> [[WIDE_LOAD2]], i64 0
-; UNROLL-NO-IC-NEXT: [[TMP9:%.*]] = udiv i32 [[TMP14]], [[TMP13]]
-; UNROLL-NO-IC-NEXT: [[BROADCAST_SPLATINSERT:%.*]] = insertelement <2 x i32> poison, i32 [[TMP9]], i64 0
+; UNROLL-NO-IC-NEXT: [[TMP19:%.*]] = add i32 [[INDEX]], 2
+; UNROLL-NO-IC-NEXT: [[TMP20:%.*]] = extractelement <2 x i32> [[WIDE_LOAD2]], i64 0
+; UNROLL-NO-IC-NEXT: [[TMP21:%.*]] = udiv i32 [[TMP20]], [[TMP19]]
+; UNROLL-NO-IC-NEXT: [[TMP22:%.*]] = insertelement <2 x i32> poison, i32 [[TMP21]], i64 0
; UNROLL-NO-IC-NEXT: br label [[PRED_UDIV_CONTINUE6]]
; UNROLL-NO-IC: pred.udiv.continue6:
-; UNROLL-NO-IC-NEXT: [[TMP17:%.*]] = phi <2 x i32> [ poison, [[PRED_UDIV_CONTINUE4]] ], [ [[BROADCAST_SPLATINSERT]], [[PRED_UDIV_IF5]] ]
+; UNROLL-NO-IC-NEXT: [[TMP23:%.*]] = phi <2 x i32> [ poison, [[PRED_UDIV_CONTINUE4]] ], [ [[TMP22]], [[PRED_UDIV_IF5]] ]
; UNROLL-NO-IC-NEXT: br i1 [[C]], label [[PRED_UDIV_IF7:%.*]], label [[PRED_UDIV_CONTINUE8]]
; UNROLL-NO-IC: pred.udiv.if7:
-; UNROLL-NO-IC-NEXT: [[TMP18:%.*]] = add i32 [[INDEX]], 3
-; UNROLL-NO-IC-NEXT: [[TMP19:%.*]] = extractelement <2 x i32> [[WIDE_LOAD2]], i64 1
-; UNROLL-NO-IC-NEXT: [[TMP20:%.*]] = udiv i32 [[TMP19]], [[TMP18]]
-; UNROLL-NO-IC-NEXT: [[TMP21:%.*]] = insertelement <2 x i32> [[TMP17]], i32 [[TMP20]], i64 1
+; UNROLL-NO-IC-NEXT: [[TMP25:%.*]] = add i32 [[INDEX]], 3
+; UNROLL-NO-IC-NEXT: [[TMP26:%.*]] = extractelement <2 x i32> [[WIDE_LOAD2]], i64 1
+; UNROLL-NO-IC-NEXT: [[TMP27:%.*]] = udiv i32 [[TMP26]], [[TMP25]]
+; UNROLL-NO-IC-NEXT: [[TMP28:%.*]] = insertelement <2 x i32> [[TMP23]], i32 [[TMP27]], i64 1
; UNROLL-NO-IC-NEXT: br label [[PRED_UDIV_CONTINUE8]]
; UNROLL-NO-IC: pred.udiv.continue8:
-; UNROLL-NO-IC-NEXT: [[TMP22:%.*]] = phi <2 x i32> [ [[TMP17]], [[PRED_UDIV_CONTINUE6]] ], [ [[TMP21]], [[PRED_UDIV_IF7]] ]
-; UNROLL-NO-IC-NEXT: [[PREDPHI:%.*]] = select i1 [[C]], <2 x i32> [[TMP12]], <2 x i32> [[WIDE_LOAD]]
-; UNROLL-NO-IC-NEXT: [[PREDPHI9:%.*]] = select i1 [[C]], <2 x i32> [[TMP22]], <2 x i32> [[WIDE_LOAD2]]
+; UNROLL-NO-IC-NEXT: [[TMP29:%.*]] = phi <2 x i32> [ [[TMP23]], [[PRED_UDIV_CONTINUE6]] ], [ [[TMP28]], [[PRED_UDIV_IF7]] ]
+; UNROLL-NO-IC-NEXT: [[PREDPHI:%.*]] = select i1 [[C]], <2 x i32> [[TMP17]], <2 x i32> [[WIDE_LOAD]]
+; UNROLL-NO-IC-NEXT: [[PREDPHI9:%.*]] = select i1 [[C]], <2 x i32> [[TMP29]], <2 x i32> [[WIDE_LOAD2]]
; UNROLL-NO-IC-NEXT: [[TMP32]] = add <2 x i32> [[PREDPHI]], [[VEC_PHI]]
; UNROLL-NO-IC-NEXT: [[TMP33]] = add <2 x i32> [[PREDPHI9]], [[VEC_PHI1]]
; UNROLL-NO-IC-NEXT: [[INDEX_NEXT]] = add nuw i32 [[INDEX]], 4
@@ -2273,75 +2273,75 @@ define i32 @scalarize_induction_variable_05(ptr %a, i32 %x, i1 %c, i32 %n) {
; INTERLEAVE: pred.udiv.if:
; INTERLEAVE-NEXT: [[TMP3:%.*]] = extractelement <4 x i32> [[WIDE_LOAD]], i64 0
; INTERLEAVE-NEXT: [[TMP4:%.*]] = udiv i32 [[TMP3]], [[INDEX]]
-; INTERLEAVE-NEXT: [[TMP6:%.*]] = insertelement <4 x i32> poison, i32 [[TMP4]], i64 0
+; INTERLEAVE-NEXT: [[TMP5:%.*]] = insertelement <4 x i32> poison, i32 [[TMP4]], i64 0
; INTERLEAVE-NEXT: br label [[PRED_UDIV_CONTINUE]]
; INTERLEAVE: pred.udiv.continue:
-; INTERLEAVE-NEXT: [[TMP7:%.*]] = phi <4 x i32> [ poison, [[VECTOR_BODY]] ], [ [[TMP6]], [[PRED_UDIV_IF]] ]
+; INTERLEAVE-NEXT: [[TMP6:%.*]] = phi <4 x i32> [ poison, [[VECTOR_BODY]] ], [ [[TMP5]], [[PRED_UDIV_IF]] ]
; INTERLEAVE-NEXT: br i1 [[C]], label [[PRED_UDIV_IF3:%.*]], label [[PRED_UDIV_CONTINUE4:%.*]]
; INTERLEAVE: pred.udiv.if3:
-; INTERLEAVE-NEXT: [[TMP8:%.*]] = add i32 [[INDEX]], 1
-; INTERLEAVE-NEXT: [[TMP13:%.*]] = extractelement <4 x i32> [[WIDE_LOAD]], i64 1
-; INTERLEAVE-NEXT: [[TMP10:%.*]] = udiv i32 [[TMP13]], [[TMP8]]
-; INTERLEAVE-NEXT: [[TMP11:%.*]] = insertelement <4 x i32> [[TMP7]], i32 [[TMP10]], i64 1
+; INTERLEAVE-NEXT: [[TMP7:%.*]] = add i32 [[INDEX]], 1
+; INTERLEAVE-NEXT: [[TMP8:%.*]] = extractelement <4 x i32> [[WIDE_LOAD]], i64 1
+; INTERLEAVE-NEXT: [[TMP9:%.*]] = udiv i32 [[TMP8]], [[TMP7]]
+; INTERLEAVE-NEXT: [[TMP10:%.*]] = insertelement <4 x i32> [[TMP6]], i32 [[TMP9]], i64 1
; INTERLEAVE-NEXT: br label [[PRED_UDIV_CONTINUE4]]
; INTERLEAVE: pred.udiv.continue4:
-; INTERLEAVE-NEXT: [[TMP12:%.*]] = phi <4 x i32> [ [[TMP7]], [[PRED_UDIV_CONTINUE]] ], [ [[TMP11]], [[PRED_UDIV_IF3]] ]
+; INTERLEAVE-NEXT: [[TMP11:%.*]] = phi <4 x i32> [ [[TMP6]], [[PRED_UDIV_CONTINUE]] ], [ [[TMP10]], [[PRED_UDIV_IF3]] ]
; INTERLEAVE-NEXT: br i1 [[C]], label [[PRED_UDIV_IF5:%.*]], label [[PRED_UDIV_CONTINUE6:%.*]]
; INTERLEAVE: pred.udiv.if5:
-; INTERLEAVE-NEXT: [[TMP17:%.*]] = add i32 [[INDEX]], 2
-; INTERLEAVE-NEXT: [[TMP18:%.*]] = extractelement <4 x i32> [[WIDE_LOAD]], i64 2
-; INTERLEAVE-NEXT: [[TMP19:%.*]] = udiv i32 [[TMP18]], [[TMP17]]
-; INTERLEAVE-NEXT: [[TMP16:%.*]] = insertelement <4 x i32> [[TMP12]], i32 [[TMP19]], i64 2
+; INTERLEAVE-NEXT: [[TMP12:%.*]] = add i32 [[INDEX]], 2
+; INTERLEAVE-NEXT: [[TMP13:%.*]] = extractelement <4 x i32> [[WIDE_LOAD]], i64 2
+; INTERLEAVE-NEXT: [[TMP14:%.*]] = udiv i32 [[TMP13]], [[TMP12]]
+; INTERLEAVE-NEXT: [[TMP15:%.*]] = insertelement <4 x i32> [[TMP11]], i32 [[TMP14]], i64 2
; INTERLEAVE-NEXT: br label [[PRED_UDIV_CONTINUE6]]
; INTERLEAVE: pred.udiv.continue6:
-; INTERLEAVE-NEXT: [[TMP25:%.*]] = phi <4 x i32> [ [[TMP12]], [[PRED_UDIV_CONTINUE4]] ], [ [[TMP16]], [[PRED_UDIV_IF5]] ]
+; INTERLEAVE-NEXT: [[TMP16:%.*]] = phi <4 x i32> [ [[TMP11]], [[PRED_UDIV_CONTINUE4]] ], [ [[TMP15]], [[PRED_UDIV_IF5]] ]
; INTERLEAVE-NEXT: br i1 [[C]], label [[PRED_UDIV_IF7:%.*]], label [[PRED_UDIV_CONTINUE8:%.*]]
; INTERLEAVE: pred.udiv.if7:
-; INTERLEAVE-NEXT: [[TMP26:%.*]] = add i32 [[INDEX]], 3
-; INTERLEAVE-NEXT: [[TMP46:%.*]] = extractelement <4 x i32> [[WIDE_LOAD]], i64 3
-; INTERLEAVE-NEXT: [[TMP20:%.*]] = udiv i32 [[TMP46]], [[TMP26]]
-; INTERLEAVE-NEXT: [[TMP21:%.*]] = insertelement <4 x i32> [[TMP25]], i32 [[TMP20]], i64 3
+; INTERLEAVE-NEXT: [[TMP17:%.*]] = add i32 [[INDEX]], 3
+; INTERLEAVE-NEXT: [[TMP18:%.*]] = extractelement <4 x i32> [[WIDE_LOAD]], i64 3
+; INTERLEAVE-NEXT: [[TMP19:%.*]] = udiv i32 [[TMP18]], [[TMP17]]
+; INTERLEAVE-NEXT: [[TMP20:%.*]] = insertelement <4 x i32> [[TMP16]], i32 [[TMP19]], i64 3
; INTERLEAVE-NEXT: br label [[PRED_UDIV_CONTINUE8]]
; INTERLEAVE: pred.udiv.continue8:
-; INTERLEAVE-NEXT: [[TMP22:%.*]] = phi <4 x i32> [ [[TMP25]], [[PRED_UDIV_CONTINUE6]] ], [ [[TMP21]], [[PRED_UDIV_IF7]] ]
+; INTERLEAVE-NEXT: [[TMP21:%.*]] = phi <4 x i32> [ [[TMP16]], [[PRED_UDIV_CONTINUE6]] ], [ [[TMP20]], [[PRED_UDIV_IF7]] ]
; INTERLEAVE-NEXT: br i1 [[C]], label [[PRED_UDIV_IF9:%.*]], label [[PRED_UDIV_CONTINUE10:%.*]]
; INTERLEAVE: pred.udiv.if9:
-; INTERLEAVE-NEXT: [[TMP23:%.*]] = add i32 [[INDEX]], 4
-; INTERLEAVE-NEXT: [[TMP24:%.*]] = extractelement <4 x i32> [[WIDE_LOAD2]], i64 0
-; INTERLEAVE-NEXT: [[TMP9:%.*]] = udiv i32 [[TMP24]], [[TMP23]]
-; INTERLEAVE-NEXT: [[BROADCAST_SPLATINSERT:%.*]] = insertelement <4 x i32> poison, i32 [[TMP9]], i64 0
+; INTERLEAVE-NEXT: [[TMP22:%.*]] = add i32 [[INDEX]], 4
+; INTERLEAVE-NEXT: [[TMP23:%.*]] = extractelement <4 x i32> [[WIDE_LOAD2]], i64 0
+; INTERLEAVE-NEXT: [[TMP24:%.*]] = udiv i32 [[TMP23]], [[TMP22]]
+; INTERLEAVE-NEXT: [[TMP25:%.*]] = insertelement <4 x i32> poison, i32 [[TMP24]], i64 0
; INTERLEAVE-NEXT: br label [[PRED_UDIV_CONTINUE10]]
; INTERLEAVE: pred.udiv.continue10:
-; INTERLEAVE-NEXT: [[TMP27:%.*]] = phi <4 x i32> [ poison, [[PRED_UDIV_CONTINUE8]] ], [ [[BROADCAST_SPLATINSERT]], [[PRED_UDIV_IF9]] ]
+; INTERLEAVE-NEXT: [[TMP26:%.*]] = phi <4 x i32> [ poison, [[PRED_UDIV_CONTINUE8]] ], [ [[TMP25]], [[PRED_UDIV_IF9]] ]
; INTERLEAVE-NEXT: br i1 [[C]], label [[PRED_UDIV_IF11:%.*]], label [[PRED_UDIV_CONTINUE12:%.*]]
; INTERLEAVE: pred.udiv.if11:
-; INTERLEAVE-NEXT: [[TMP28:%.*]] = add i32 [[INDEX]], 5
-; INTERLEAVE-NEXT: [[TMP29:%.*]] = extractelement <4 x i32> [[WIDE_LOAD2]], i64 1
-; INTERLEAVE-NEXT: [[TMP30:%.*]] = udiv i32 [[TMP29]], [[TMP28]]
-; INTERLEAVE-NEXT: [[TMP31:%.*]] = insertelement <4 x i32> [[TMP27]], i32 [[TMP30]], i64 1
+; INTERLEAVE-NEXT: [[TMP27:%.*]] = add i32 [[INDEX]], 5
+; INTERLEAVE-NEXT: [[TMP28:%.*]] = extractelement <4 x i32> [[WIDE_LOAD2]], i64 1
+; INTERLEAVE-NEXT: [[TMP29:%.*]] = udiv i32 [[TMP28]], [[TMP27]]
+; INTERLEAVE-NEXT: [[TMP30:%.*]] = insertelement <4 x i32> [[TMP26]], i32 [[TMP29]], i64 1
; INTERLEAVE-NEXT: br label [[PRED_UDIV_CONTINUE12]]
; INTERLEAVE: pred.udiv.continue12:
-; INTERLEAVE-NEXT: [[TMP32:%.*]] = phi <4 x i32> [ [[TMP27]], [[PRED_UDIV_CONTINUE10]] ], [ [[TMP31]], [[PRED_UDIV_IF11]] ]
+; INTERLEAVE-NEXT: [[TMP31:%.*]] = phi <4 x i32> [ [[TMP26]], [[PRED_UDIV_CONTINUE10]] ], [ [[TMP30]], [[PRED_UDIV_IF11]] ]
; INTERLEAVE-NEXT: br i1 [[C]], label [[PRED_UDIV_IF13:%.*]], label [[PRED_UDIV_CONTINUE14:%.*]]
; INTERLEAVE: pred.udiv.if13:
-; INTERLEAVE-NEXT: [[TMP33:%.*]] = add i32 [[INDEX]], 6
-; INTERLEAVE-NEXT: [[TMP34:%.*]] = extractelement <4 x i32> [[WIDE_LOAD2]], i64 2
-; INTERLEAVE-NEXT: [[TMP35:%.*]] = udiv i32 [[TMP34]], [[TMP33]]
-; INTERLEAVE-NEXT: [[TMP36:%.*]] = insertelement <4 x i32> [[TMP32]], i32 [[TMP35]], i64 2
+; INTERLEAVE-NEXT: [[TMP32:%.*]] = add i32 [[INDEX]], 6
+; INTERLEAVE-NEXT: [[TMP33:%.*]] = extractelement <4 x i32> [[WIDE_LOAD2]], i64 2
+; INTERLEAVE-NEXT: [[TMP34:%.*]] = udiv i32 [[TMP33]], [[TMP32]]
+; INTERLEAVE-NEXT: [[TMP35:%.*]] = insertelement <4 x i32> [[TMP31]], i32 [[TMP34]], i64 2
; INTERLEAVE-NEXT: br label [[PRED_UDIV_CONTINUE14]]
; INTERLEAVE: pred.udiv.continue14:
-; INTERLEAVE-NEXT: [[TMP37:%.*]] = phi <4 x i32> [ [[TMP32]], [[PRED_UDIV_CONTINUE12]] ], [ [[TMP36]], [[PRED_UDIV_IF13]] ]
+; INTERLEAVE-NEXT: [[TMP36:%.*]] = phi <4 x i32> [ [[TMP31]], [[PRED_UDIV_CONTINUE12]] ], [ [[TMP35]], [[PRED_UDIV_IF13]] ]
; INTERLEAVE-NEXT: br i1 [[C]], label [[PRED_UDIV_IF15:%.*]], label [[PRED_UDIV_CONTINUE16]]
; INTERLEAVE: pred.udiv.if15:
-; INTERLEAVE-NEXT: [[TMP38:%.*]] = add i32 [[INDEX]], 7
-; INTERLEAVE-NEXT: [[TMP39:%.*]] = extractelement <4 x i32> [[WIDE_LOAD2]], i64 3
-; INTERLEAVE-NEXT: [[TMP40:%.*]] = udiv i32 [[TMP39]], [[TMP38]]
-; INTERLEAVE-NEXT: [[TMP41:%.*]] = insertelement <4 x i32> [[TMP37]], i32 [[TMP40]], i64 3
+; INTERLEAVE-NEXT: [[TMP37:%.*]] = add i32 [[INDEX]], 7
+; INTERLEAVE-NEXT: [[TMP38:%.*]] = extractelement <4 x i32> [[WIDE_LOAD2]], i64 3
+; INTERLEAVE-NEXT: [[TMP39:%.*]] = udiv i32 [[TMP38]], [[TMP37]]
+; INTERLEAVE-NEXT: [[TMP40:%.*]] = insertelement <4 x i32> [[TMP36]], i32 [[TMP39]], i64 3
; INTERLEAVE-NEXT: br label [[PRED_UDIV_CONTINUE16]]
; INTERLEAVE: pred.udiv.continue16:
-; INTERLEAVE-NEXT: [[TMP47:%.*]] = phi <4 x i32> [ [[TMP37]], [[PRED_UDIV_CONTINUE14]] ], [ [[TMP41]], [[PRED_UDIV_IF15]] ]
-; INTERLEAVE-NEXT: [[PREDPHI:%.*]] = select i1 [[C]], <4 x i32> [[TMP22]], <4 x i32> [[WIDE_LOAD]]
-; INTERLEAVE-NEXT: [[PREDPHI17:%.*]] = select i1 [[C]], <4 x i32> [[TMP47]], <4 x i32> [[WIDE_LOAD2]]
+; INTERLEAVE-NEXT: [[TMP41:%.*]] = phi <4 x i32> [ [[TMP36]], [[PRED_UDIV_CONTINUE14]] ], [ [[TMP40]], [[PRED_UDIV_IF15]] ]
+; INTERLEAVE-NEXT: [[PREDPHI:%.*]] = select i1 [[C]], <4 x i32> [[TMP21]], <4 x i32> [[WIDE_LOAD]]
+; INTERLEAVE-NEXT: [[PREDPHI17:%.*]] = select i1 [[C]], <4 x i32> [[TMP41]], <4 x i32> [[WIDE_LOAD2]]
; INTERLEAVE-NEXT: [[TMP42]] = add <4 x i32> [[PREDPHI]], [[VEC_PHI]]
; INTERLEAVE-NEXT: [[TMP43]] = add <4 x i32> [[PREDPHI17]], [[VEC_PHI1]]
; INTERLEAVE-NEXT: [[INDEX_NEXT]] = add nuw i32 [[INDEX]], 8
diff --git a/llvm/test/Transforms/LoopVectorize/pr55167-fold-tail-live-out.ll b/llvm/test/Transforms/LoopVectorize/pr55167-fold-tail-live-out.ll
index 2410514c735a2..88513fee4657c 100644
--- a/llvm/test/Transforms/LoopVectorize/pr55167-fold-tail-live-out.ll
+++ b/llvm/test/Transforms/LoopVectorize/pr55167-fold-tail-live-out.ll
@@ -6,34 +6,34 @@ define i32 @test(i32 %a, i1 %c.1, i1 %c.2 ) #0 {
; CHECK-NEXT: bb:
; CHECK-NEXT: br label [[VECTOR_PH:%.*]]
; CHECK: vector.ph:
-; CHECK-NEXT: [[BROADCAST_SPLATINSERT:%.*]] = insertelement <2 x i1> poison, i1 [[C_2:%.*]], i64 0
-; CHECK-NEXT: [[BROADCAST_SPLAT:%.*]] = shufflevector <2 x i1> [[BROADCAST_SPLATINSERT]], <2 x i1> poison, <2 x i32> zeroinitializer
; CHECK-NEXT: [[BROADCAST_SPLATINSERT1:%.*]] = insertelement <2 x i1> poison, i1 [[C_1:%.*]], i64 0
; CHECK-NEXT: [[BROADCAST_SPLAT2:%.*]] = shufflevector <2 x i1> [[BROADCAST_SPLATINSERT1]], <2 x i1> poison, <2 x i32> zeroinitializer
-; CHECK-NEXT: [[TMP3:%.*]] = xor <2 x i1> [[BROADCAST_SPLAT2]], splat (i1 true)
-; CHECK-NEXT: [[TMP1:%.*]] = xor <2 x i1> [[BROADCAST_SPLAT]], splat (i1 true)
-; CHECK-NEXT: [[TMP2:%.*]] = select <2 x i1> [[TMP3]], <2 x i1> [[TMP1]], <2 x i1> zeroinitializer
+; CHECK-NEXT: [[BROADCAST_SPLATINSERT3:%.*]] = insertelement <2 x i1> poison, i1 [[C_2:%.*]], i64 0
+; CHECK-NEXT: [[BROADCAST_SPLAT4:%.*]] = shufflevector <2 x i1> [[BROADCAST_SPLATINSERT3]], <2 x i1> poison, <2 x i32> zeroinitializer
+; CHECK-NEXT: [[TMP6:%.*]] = xor <2 x i1> [[BROADCAST_SPLAT4]], splat (i1 true)
+; CHECK-NEXT: [[TMP5:%.*]] = xor <2 x i1> [[BROADCAST_SPLAT2]], splat (i1 true)
+; CHECK-NEXT: [[TMP11:%.*]] = select <2 x i1> [[TMP6]], <2 x i1> [[TMP5]], <2 x i1> zeroinitializer
; CHECK-NEXT: [[TMP4:%.*]] = xor i32 [[A:%.*]], 1
-; CHECK-NEXT: [[BROADCAST_SPLATINSERT3:%.*]] = insertelement <2 x i32> poison, i32 [[TMP4]], i64 0
-; CHECK-NEXT: [[BROADCAST_SPLAT4:%.*]] = shufflevector <2 x i32> [[BROADCAST_SPLATINSERT3]], <2 x i32> poison, <2 x i32> zeroinitializer
+; CHECK-NEXT: [[BROADCAST_SPLATINSERT4:%.*]] = insertelement <2 x i32> poison, i32 [[TMP4]], i64 0
+; CHECK-NEXT: [[TMP2:%.*]] = shufflevector <2 x i32> [[BROADCAST_SPLATINSERT4]], <2 x i32> poison, <2 x i32> zeroinitializer
; CHECK-NEXT: br label [[VECTOR_BODY:%.*]]
; CHECK: vector.body:
; CHECK-NEXT: [[INDEX:%.*]] = phi i32 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ]
; CHECK-NEXT: [[VEC_IND:%.*]] = phi <2 x i32> [ <i32 6, i32 7>, [[VECTOR_PH]] ], [ [[VEC_IND_NEXT:%.*]], [[VECTOR_BODY]] ]
-; CHECK-NEXT: [[VEC_PHI:%.*]] = phi <2 x i32> [ <i32 35902, i32 0>, [[VECTOR_PH]] ], [ [[PREDPHI6:%.*]], [[VECTOR_BODY]] ]
+; CHECK-NEXT: [[VEC_PHI:%.*]] = phi <2 x i32> [ <i32 35902, i32 0>, [[VECTOR_PH]] ], [ [[PREDPHI7:%.*]], [[VECTOR_BODY]] ]
; CHECK-NEXT: [[TMP0:%.*]] = add <2 x i32> [[VEC_PHI]], splat (i32 10)
-; CHECK-NEXT: [[TMP5:%.*]] = add <2 x i32> [[TMP0]], splat (i32 20)
-; CHECK-NEXT: [[TMP6:%.*]] = add <2 x i32> [[TMP5]], [[BROADCAST_SPLAT4]]
-; CHECK-NEXT: [[TMP7:%.*]] = extractelement <2 x i1> [[BROADCAST_SPLAT2]], i64 0
+; CHECK-NEXT: [[TMP1:%.*]] = add <2 x i32> [[TMP0]], splat (i32 20)
+; CHECK-NEXT: [[TMP3:%.*]] = add <2 x i32> [[TMP1]], [[TMP2]]
+; CHECK-NEXT: [[TMP7:%.*]] = extractelement <2 x i1> [[BROADCAST_SPLAT4]], i64 0
; CHECK-NEXT: [[PREDPHI:%.*]] = select i1 [[TMP7]], <2 x i32> [[VEC_PHI]], <2 x i32> [[TMP0]]
-; CHECK-NEXT: [[PREDPHI6]] = select <2 x i1> [[TMP2]], <2 x i32> [[TMP6]], <2 x i32> [[PREDPHI]]
+; CHECK-NEXT: [[PREDPHI7]] = select <2 x i1> [[TMP11]], <2 x i32> [[TMP3]], <2 x i32> [[PREDPHI]]
; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i32 [[INDEX]], 2
; CHECK-NEXT: [[VEC_IND_NEXT]] = add nuw nsw <2 x i32> [[VEC_IND]], splat (i32 2)
; CHECK-NEXT: [[TMP8:%.*]] = icmp eq i32 [[INDEX_NEXT]], 176
; CHECK-NEXT: br i1 [[TMP8]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP0:![0-9]+]]
; CHECK: middle.block:
-; CHECK-NEXT: [[PREDPHI5:%.*]] = select i1 [[C_1]], <2 x i32> [[VEC_IND]], <2 x i32> splat (i32 9)
-; CHECK-NEXT: [[TMP10:%.*]] = call i32 @llvm.vector.reduce.add.v2i32(<2 x i32> [[PREDPHI6]])
+; CHECK-NEXT: [[PREDPHI5:%.*]] = select i1 [[C_2]], <2 x i32> [[VEC_IND]], <2 x i32> splat (i32 9)
+; CHECK-NEXT: [[TMP10:%.*]] = call i32 @llvm.vector.reduce.add.v2i32(<2 x i32> [[PREDPHI7]])
; CHECK-NEXT: [[TMP9:%.*]] = extractelement <2 x i32> [[PREDPHI5]], i64 1
; CHECK-NEXT: br label [[LOOP_LATCH:%.*]]
; CHECK: exit:
diff --git a/llvm/test/Transforms/LoopVectorize/predicatedinst-loop-invariant.ll b/llvm/test/Transforms/LoopVectorize/predicatedinst-loop-invariant.ll
index f02eb35547250..3292b2776ab02 100644
--- a/llvm/test/Transforms/LoopVectorize/predicatedinst-loop-invariant.ll
+++ b/llvm/test/Transforms/LoopVectorize/predicatedinst-loop-invariant.ll
@@ -236,30 +236,32 @@ define void @test_store_to_invariant_address_needs_mask_due_to_low_trip_count(pt
; CHECK: [[VECTOR_PH]]:
; CHECK-NEXT: br label %[[VECTOR_BODY:.*]]
; CHECK: [[VECTOR_BODY]]:
-; CHECK-NEXT: br label %[[LOOP_LATCH1:.*]]
-; CHECK: [[LOOP_LATCH1]]:
-; CHECK-NEXT: br i1 true, label %[[PRED_STORE_IF:.*]], label %[[PRED_STORE_CONTINUE:.*]]
+; CHECK-NEXT: br label %[[MIDDLE_BLOCK:.*]]
+; CHECK: [[MIDDLE_BLOCK]]:
+; CHECK-NEXT: [[TMP0:%.*]] = phi i1 [ false, %[[VECTOR_BODY]] ]
+; CHECK-NEXT: [[PREDPHI:%.*]] = select i1 [[TMP0]], i32 0, i32 1
+; CHECK-NEXT: br i1 true, label %[[PRED_STORE_IF:.*]], label %[[EXIT:.*]]
; CHECK: [[PRED_STORE_IF]]:
-; CHECK-NEXT: store i32 0, ptr [[DST]], align 4
-; CHECK-NEXT: br label %[[PRED_STORE_CONTINUE]]
-; CHECK: [[PRED_STORE_CONTINUE]]:
+; CHECK-NEXT: store i32 [[PREDPHI]], ptr [[DST]], align 4
+; CHECK-NEXT: br label %[[EXIT]]
+; CHECK: [[EXIT]]:
; CHECK-NEXT: br i1 true, label %[[PRED_STORE_IF2:.*]], label %[[PRED_STORE_CONTINUE3:.*]]
; CHECK: [[PRED_STORE_IF2]]:
-; CHECK-NEXT: store i32 0, ptr [[DST]], align 4
+; CHECK-NEXT: store i32 [[PREDPHI]], ptr [[DST]], align 4
; CHECK-NEXT: br label %[[PRED_STORE_CONTINUE3]]
; CHECK: [[PRED_STORE_CONTINUE3]]:
; CHECK-NEXT: br i1 true, label %[[PRED_STORE_IF4:.*]], label %[[PRED_STORE_CONTINUE5:.*]]
; CHECK: [[PRED_STORE_IF4]]:
-; CHECK-NEXT: store i32 0, ptr [[DST]], align 4
+; CHECK-NEXT: store i32 [[PREDPHI]], ptr [[DST]], align 4
; CHECK-NEXT: br label %[[PRED_STORE_CONTINUE5]]
; CHECK: [[PRED_STORE_CONTINUE5]]:
-; CHECK-NEXT: br i1 false, label %[[PRED_STORE_IF6:.*]], label %[[MIDDLE_BLOCK:.*]]
+; CHECK-NEXT: br i1 false, label %[[PRED_STORE_IF6:.*]], label %[[PRED_STORE_CONTINUE7:.*]]
; CHECK: [[PRED_STORE_IF6]]:
-; CHECK-NEXT: store i32 1, ptr [[DST]], align 4
-; CHECK-NEXT: br label %[[MIDDLE_BLOCK]]
-; CHECK: [[MIDDLE_BLOCK]]:
-; CHECK-NEXT: br label %[[EXIT:.*]]
-; CHECK: [[EXIT]]:
+; CHECK-NEXT: store i32 [[PREDPHI]], ptr [[DST]], align 4
+; CHECK-NEXT: br label %[[PRED_STORE_CONTINUE7]]
+; CHECK: [[PRED_STORE_CONTINUE7]]:
+; CHECK-NEXT: br label %[[MIDDLE_BLOCK1:.*]]
+; CHECK: [[MIDDLE_BLOCK1]]:
; CHECK-NEXT: br label %[[EXIT1:.*]]
; CHECK: [[EXIT1]]:
; CHECK-NEXT: ret void
diff --git a/llvm/test/Transforms/LoopVectorize/predicator.ll b/llvm/test/Transforms/LoopVectorize/predicator.ll
index 9fe0178921ed4..0f5761e2c46f1 100644
--- a/llvm/test/Transforms/LoopVectorize/predicator.ll
+++ b/llvm/test/Transforms/LoopVectorize/predicator.ll
@@ -236,25 +236,25 @@ define void @simplifiable_blend(i1 %c1, i1 %c2, i1 %c3, i32 %x, i32 %y, ptr %p)
; CHECK-NEXT: [[ENTRY:.*:]]
; CHECK-NEXT: br label %[[VECTOR_PH:.*]]
; CHECK: [[VECTOR_PH]]:
-; CHECK-NEXT: [[BROADCAST_SPLATINSERT:%.*]] = insertelement <4 x i32> poison, i32 [[X]], i64 0
-; CHECK-NEXT: [[BROADCAST_SPLAT:%.*]] = shufflevector <4 x i32> [[BROADCAST_SPLATINSERT]], <4 x i32> poison, <4 x i32> zeroinitializer
-; CHECK-NEXT: [[BROADCAST_SPLATINSERT1:%.*]] = insertelement <4 x i32> poison, i32 [[Y]], i64 0
+; CHECK-NEXT: [[BROADCAST_SPLATINSERT1:%.*]] = insertelement <4 x i32> poison, i32 [[X]], i64 0
; CHECK-NEXT: [[BROADCAST_SPLAT2:%.*]] = shufflevector <4 x i32> [[BROADCAST_SPLATINSERT1]], <4 x i32> poison, <4 x i32> zeroinitializer
+; CHECK-NEXT: [[BROADCAST_SPLATINSERT2:%.*]] = insertelement <4 x i32> poison, i32 [[Y]], i64 0
+; CHECK-NEXT: [[BROADCAST_SPLAT3:%.*]] = shufflevector <4 x i32> [[BROADCAST_SPLATINSERT2]], <4 x i32> poison, <4 x i32> zeroinitializer
; CHECK-NEXT: br label %[[VECTOR_BODY:.*]]
; CHECK: [[VECTOR_BODY]]:
-; CHECK-NEXT: [[INDEX:%.*]] = phi i32 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[LATCH5:.*]] ]
-; CHECK-NEXT: br i1 [[C1]], label %[[A4:.*]], label %[[B1:.*]]
-; CHECK: [[B1]]:
+; CHECK-NEXT: [[INDEX:%.*]] = phi i32 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[LATCH7:.*]] ]
+; CHECK-NEXT: br i1 [[C1]], label %[[A6:.*]], label %[[B3:.*]]
+; CHECK: [[B3]]:
; CHECK-NEXT: br i1 [[C3]], label %[[F4:.*]], label %[[E3:.*]]
; CHECK: [[E3]]:
; CHECK-NEXT: br label %[[F4]]
; CHECK: [[F4]]:
-; CHECK-NEXT: br label %[[LATCH5]]
-; CHECK: [[A4]]:
-; CHECK-NEXT: br label %[[LATCH5]]
-; CHECK: [[LATCH5]]:
-; CHECK-NEXT: [[TMP0:%.*]] = phi i1 [ false, %[[F4]] ], [ true, %[[A4]] ]
-; CHECK-NEXT: [[PREDPHI7:%.*]] = select i1 [[TMP0]], <4 x i32> [[BROADCAST_SPLAT]], <4 x i32> [[BROADCAST_SPLAT2]]
+; CHECK-NEXT: br label %[[LATCH7]]
+; CHECK: [[A6]]:
+; CHECK-NEXT: br label %[[LATCH7]]
+; CHECK: [[LATCH7]]:
+; CHECK-NEXT: [[TMP0:%.*]] = phi i1 [ false, %[[F4]] ], [ true, %[[A6]] ]
+; CHECK-NEXT: [[PREDPHI7:%.*]] = select i1 [[TMP0]], <4 x i32> [[BROADCAST_SPLAT2]], <4 x i32> [[BROADCAST_SPLAT3]]
; CHECK-NEXT: [[TMP3:%.*]] = getelementptr i32, ptr [[P]], i32 [[INDEX]]
; CHECK-NEXT: store <4 x i32> [[PREDPHI7]], ptr [[TMP3]], align 4
; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i32 [[INDEX]], 4
@@ -315,20 +315,20 @@ define void @outermost_uniform_branch(ptr %a, i1 %u0) {
; CHECK-NEXT: [[VEC_IND:%.*]] = phi <4 x i64> [ <i64 0, i64 1, i64 2, i64 3>, %[[VECTOR_PH]] ], [ [[VEC_IND_NEXT:%.*]], %[[BB42]] ]
; CHECK-NEXT: br i1 [[U0]], label %[[BB11:.*]], label %[[BB42]]
; CHECK: [[BB11]]:
-; CHECK-NEXT: [[TMP6:%.*]] = add i64 [[INDEX]], 1
+; CHECK-NEXT: [[TMP7:%.*]] = add i64 [[INDEX]], 1
; CHECK-NEXT: [[TMP1:%.*]] = icmp sle i64 [[INDEX]], 1
; CHECK-NEXT: [[TMP2:%.*]] = add i64 [[INDEX]], 2
-; CHECK-NEXT: [[PREDPHI:%.*]] = select i1 [[TMP1]], i64 [[TMP2]], i64 [[TMP6]]
+; CHECK-NEXT: [[PREDPHI:%.*]] = select i1 [[TMP1]], i64 [[TMP2]], i64 [[TMP7]]
; CHECK-NEXT: [[TMP3:%.*]] = add i64 [[PREDPHI]], 3
; CHECK-NEXT: br label %[[BB42]]
; CHECK: [[BB42]]:
; CHECK-NEXT: [[TMP4:%.*]] = phi i64 [ poison, %[[VECTOR_BODY]] ], [ [[TMP3]], %[[BB11]] ]
-; CHECK-NEXT: [[TMP7:%.*]] = phi i1 [ false, %[[VECTOR_BODY]] ], [ true, %[[BB11]] ]
+; CHECK-NEXT: [[TMP5:%.*]] = phi i1 [ false, %[[VECTOR_BODY]] ], [ true, %[[BB11]] ]
; CHECK-NEXT: [[BROADCAST_SPLATINSERT:%.*]] = insertelement <4 x i64> poison, i64 [[TMP4]], i64 0
; CHECK-NEXT: [[BROADCAST_SPLAT:%.*]] = shufflevector <4 x i64> [[BROADCAST_SPLATINSERT]], <4 x i64> poison, <4 x i32> zeroinitializer
-; CHECK-NEXT: [[PREDPHI3:%.*]] = select i1 [[TMP7]], <4 x i64> [[BROADCAST_SPLAT]], <4 x i64> [[VEC_IND]]
-; CHECK-NEXT: [[TMP5:%.*]] = extractelement <4 x i64> [[PREDPHI3]], i64 3
-; CHECK-NEXT: store i64 [[TMP5]], ptr [[A]], align 4
+; CHECK-NEXT: [[PREDPHI3:%.*]] = select i1 [[TMP5]], <4 x i64> [[BROADCAST_SPLAT]], <4 x i64> [[VEC_IND]]
+; CHECK-NEXT: [[TMP6:%.*]] = extractelement <4 x i64> [[PREDPHI3]], i64 3
+; CHECK-NEXT: store i64 [[TMP6]], ptr [[A]], align 4
; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 4
; CHECK-NEXT: [[VEC_IND_NEXT]] = add nuw nsw <4 x i64> [[VEC_IND]], splat (i64 4)
; CHECK-NEXT: [[TMP0:%.*]] = icmp eq i64 [[INDEX_NEXT]], 128
@@ -465,7 +465,7 @@ define void @outermost_uniform_branch_more_blocks(ptr %a, i1 %u0) {
; CHECK-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[BB63:.*]] ]
; CHECK-NEXT: br i1 [[U0]], label %[[BB22:.*]], label %[[BB11:.*]]
; CHECK: [[BB11]]:
-; CHECK-NEXT: [[TMP7:%.*]] = add i64 [[INDEX]], 1
+; CHECK-NEXT: [[TMP8:%.*]] = add i64 [[INDEX]], 1
; CHECK-NEXT: br label %[[BB63]]
; CHECK: [[BB22]]:
; CHECK-NEXT: [[TMP1:%.*]] = icmp sle i64 [[INDEX]], 2
@@ -475,10 +475,10 @@ define void @outermost_uniform_branch_more_blocks(ptr %a, i1 %u0) {
; CHECK-NEXT: [[TMP4:%.*]] = add i64 [[PREDPHI]], 5
; CHECK-NEXT: br label %[[BB63]]
; CHECK: [[BB63]]:
-; CHECK-NEXT: [[TMP5:%.*]] = phi i64 [ [[TMP7]], %[[BB11]] ], [ poison, %[[BB22]] ]
+; CHECK-NEXT: [[TMP5:%.*]] = phi i64 [ [[TMP8]], %[[BB11]] ], [ poison, %[[BB22]] ]
; CHECK-NEXT: [[TMP6:%.*]] = phi i64 [ poison, %[[BB11]] ], [ [[TMP4]], %[[BB22]] ]
-; CHECK-NEXT: [[TMP8:%.*]] = phi i1 [ false, %[[BB11]] ], [ true, %[[BB22]] ]
-; CHECK-NEXT: [[PREDPHI4:%.*]] = select i1 [[TMP8]], i64 [[TMP6]], i64 [[TMP5]]
+; CHECK-NEXT: [[TMP7:%.*]] = phi i1 [ false, %[[BB11]] ], [ true, %[[BB22]] ]
+; CHECK-NEXT: [[PREDPHI4:%.*]] = select i1 [[TMP7]], i64 [[TMP6]], i64 [[TMP5]]
; CHECK-NEXT: store i64 [[PREDPHI4]], ptr [[A]], align 4
; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 4
; CHECK-NEXT: [[TMP0:%.*]] = icmp eq i64 [[INDEX_NEXT]], 128
@@ -633,16 +633,19 @@ define void @uniform_branch_after_varying_branch(ptr %a, i1 %u1) {
; CHECK-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[BB32:.*]] ]
; CHECK-NEXT: [[VEC_IND:%.*]] = phi <4 x i64> [ <i64 0, i64 1, i64 2, i64 3>, %[[VECTOR_PH]] ], [ [[VEC_IND_NEXT:%.*]], %[[BB32]] ]
; CHECK-NEXT: [[TMP1:%.*]] = icmp sle <4 x i64> [[VEC_IND]], zeroinitializer
+; CHECK-NEXT: [[TMP3:%.*]] = extractelement <4 x i1> [[TMP1]], i64 0
+; CHECK-NEXT: [[TMP7:%.*]] = xor i1 [[TMP3]], true
; CHECK-NEXT: [[TMP2:%.*]] = add <4 x i64> [[VEC_IND]], splat (i64 1)
; CHECK-NEXT: br i1 [[U1]], label %[[BB21:.*]], label %[[BB32]]
; CHECK: [[BB21]]:
; CHECK-NEXT: [[TMP8:%.*]] = add i64 [[INDEX]], 2
; CHECK-NEXT: br label %[[BB32]]
; CHECK: [[BB32]]:
-; CHECK-NEXT: [[TMP3:%.*]] = phi i64 [ poison, %[[VECTOR_BODY]] ], [ [[TMP8]], %[[BB21]] ]
-; CHECK-NEXT: [[BROADCAST_SPLATINSERT:%.*]] = insertelement <4 x i64> poison, i64 [[TMP3]], i64 0
+; CHECK-NEXT: [[TMP9:%.*]] = phi i64 [ poison, %[[VECTOR_BODY]] ], [ [[TMP8]], %[[BB21]] ]
+; CHECK-NEXT: [[TMP11:%.*]] = phi i1 [ false, %[[VECTOR_BODY]] ], [ [[TMP7]], %[[BB21]] ]
+; CHECK-NEXT: [[BROADCAST_SPLATINSERT:%.*]] = insertelement <4 x i64> poison, i64 [[TMP9]], i64 0
; CHECK-NEXT: [[BROADCAST_SPLAT:%.*]] = shufflevector <4 x i64> [[BROADCAST_SPLATINSERT]], <4 x i64> poison, <4 x i32> zeroinitializer
-; CHECK-NEXT: [[PREDPHI:%.*]] = select <4 x i1> [[TMP1]], <4 x i64> [[TMP2]], <4 x i64> [[BROADCAST_SPLAT]]
+; CHECK-NEXT: [[PREDPHI:%.*]] = select i1 [[TMP11]], <4 x i64> [[BROADCAST_SPLAT]], <4 x i64> [[TMP2]]
; CHECK-NEXT: [[TMP4:%.*]] = add <4 x i64> [[PREDPHI]], splat (i64 3)
; CHECK-NEXT: [[TMP5:%.*]] = add <4 x i64> [[VEC_IND]], splat (i64 4)
; CHECK-NEXT: [[PREDPHI1:%.*]] = select <4 x i1> [[TMP1]], <4 x i64> [[TMP5]], <4 x i64> [[TMP4]]
@@ -650,8 +653,8 @@ define void @uniform_branch_after_varying_branch(ptr %a, i1 %u1) {
; CHECK-NEXT: store i64 [[TMP6]], ptr [[A]], align 4
; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 4
; CHECK-NEXT: [[VEC_IND_NEXT]] = add nuw nsw <4 x i64> [[VEC_IND]], splat (i64 4)
-; CHECK-NEXT: [[TMP7:%.*]] = icmp eq i64 [[INDEX_NEXT]], 128
-; CHECK-NEXT: br i1 [[TMP7]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP10:![0-9]+]]
+; CHECK-NEXT: [[TMP10:%.*]] = icmp eq i64 [[INDEX_NEXT]], 128
+; CHECK-NEXT: br i1 [[TMP10]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP10:![0-9]+]]
; CHECK: [[MIDDLE_BLOCK]]:
; CHECK-NEXT: br label %[[EXIT:.*]]
; CHECK: [[EXIT]]:
@@ -790,30 +793,31 @@ define void @uniform_branch_after_varying_branch_more_blocks(ptr %a, i1 %u1) {
; CHECK-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[BB43:.*]] ]
; CHECK-NEXT: [[VEC_IND:%.*]] = phi <4 x i64> [ <i64 0, i64 1, i64 2, i64 3>, %[[VECTOR_PH]] ], [ [[VEC_IND_NEXT:%.*]], %[[BB43]] ]
; CHECK-NEXT: [[TMP1:%.*]] = icmp sle <4 x i64> [[VEC_IND]], zeroinitializer
+; CHECK-NEXT: [[TMP9:%.*]] = extractelement <4 x i1> [[TMP1]], i64 0
+; CHECK-NEXT: [[TMP2:%.*]] = xor i1 [[TMP9]], true
; CHECK-NEXT: br i1 [[U1]], label %[[BB32:.*]], label %[[BB21:.*]]
; CHECK: [[BB21]]:
-; CHECK-NEXT: [[TMP7:%.*]] = add i64 [[INDEX]], 2
+; CHECK-NEXT: [[TMP3:%.*]] = add i64 [[INDEX]], 2
; CHECK-NEXT: br label %[[BB43]]
; CHECK: [[BB32]]:
-; CHECK-NEXT: [[TMP2:%.*]] = add i64 [[INDEX]], 3
+; CHECK-NEXT: [[TMP10:%.*]] = add i64 [[INDEX]], 3
; CHECK-NEXT: br label %[[BB43]]
; CHECK: [[BB43]]:
-; CHECK-NEXT: [[TMP3:%.*]] = phi i64 [ [[TMP7]], %[[BB21]] ], [ poison, %[[BB32]] ]
-; CHECK-NEXT: [[TMP9:%.*]] = phi i64 [ poison, %[[BB21]] ], [ [[TMP2]], %[[BB32]] ]
-; CHECK-NEXT: [[BROADCAST_SPLATINSERT4:%.*]] = insertelement <4 x i64> poison, i64 [[TMP9]], i64 0
-; CHECK-NEXT: [[BROADCAST_SPLAT5:%.*]] = shufflevector <4 x i64> [[BROADCAST_SPLATINSERT4]], <4 x i64> poison, <4 x i32> zeroinitializer
-; CHECK-NEXT: [[BROADCAST_SPLATINSERT:%.*]] = insertelement <4 x i64> poison, i64 [[TMP3]], i64 0
-; CHECK-NEXT: [[BROADCAST_SPLAT:%.*]] = shufflevector <4 x i64> [[BROADCAST_SPLATINSERT]], <4 x i64> poison, <4 x i32> zeroinitializer
-; CHECK-NEXT: [[PREDPHI:%.*]] = select <4 x i1> [[TMP1]], <4 x i64> [[BROADCAST_SPLAT]], <4 x i64> [[BROADCAST_SPLAT5]]
-; CHECK-NEXT: [[TMP4:%.*]] = add <4 x i64> [[PREDPHI]], splat (i64 4)
+; CHECK-NEXT: [[TMP12:%.*]] = phi i64 [ [[TMP3]], %[[BB21]] ], [ poison, %[[BB32]] ]
+; CHECK-NEXT: [[TMP13:%.*]] = phi i64 [ poison, %[[BB21]] ], [ [[TMP10]], %[[BB32]] ]
+; CHECK-NEXT: [[TMP7:%.*]] = phi i1 [ false, %[[BB21]] ], [ [[TMP2]], %[[BB32]] ]
+; CHECK-NEXT: [[PREDPHI:%.*]] = select i1 [[TMP7]], i64 [[TMP13]], i64 [[TMP12]]
+; CHECK-NEXT: [[TMP8:%.*]] = add i64 [[PREDPHI]], 4
+; CHECK-NEXT: [[BROADCAST_SPLATINSERT:%.*]] = insertelement <4 x i64> poison, i64 [[TMP8]], i64 0
+; CHECK-NEXT: [[TMP4:%.*]] = shufflevector <4 x i64> [[BROADCAST_SPLATINSERT]], <4 x i64> poison, <4 x i32> zeroinitializer
; CHECK-NEXT: [[TMP5:%.*]] = add <4 x i64> [[VEC_IND]], splat (i64 5)
; CHECK-NEXT: [[PREDPHI1:%.*]] = select <4 x i1> [[TMP1]], <4 x i64> [[TMP5]], <4 x i64> [[TMP4]]
; CHECK-NEXT: [[TMP6:%.*]] = extractelement <4 x i64> [[PREDPHI1]], i64 3
; CHECK-NEXT: store i64 [[TMP6]], ptr [[A]], align 4
; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 4
; CHECK-NEXT: [[VEC_IND_NEXT]] = add nuw nsw <4 x i64> [[VEC_IND]], splat (i64 4)
-; CHECK-NEXT: [[TMP8:%.*]] = icmp eq i64 [[INDEX_NEXT]], 128
-; CHECK-NEXT: br i1 [[TMP8]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP12:![0-9]+]]
+; CHECK-NEXT: [[TMP11:%.*]] = icmp eq i64 [[INDEX_NEXT]], 128
+; CHECK-NEXT: br i1 [[TMP11]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP12:![0-9]+]]
; CHECK: [[MIDDLE_BLOCK]]:
; CHECK-NEXT: br label %[[EXIT:.*]]
; CHECK: [[EXIT]]:
@@ -964,30 +968,30 @@ define void @uniform_branch_after_varying_branch_more_blocks_mirrored(ptr %a, i1
; CHECK-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[BB53:.*]] ]
; CHECK-NEXT: [[VEC_IND:%.*]] = phi <4 x i64> [ <i64 0, i64 1, i64 2, i64 3>, %[[VECTOR_PH]] ], [ [[VEC_IND_NEXT:%.*]], %[[BB53]] ]
; CHECK-NEXT: [[TMP1:%.*]] = icmp sle <4 x i64> [[VEC_IND]], zeroinitializer
+; CHECK-NEXT: [[TMP9:%.*]] = extractelement <4 x i1> [[TMP1]], i64 0
; CHECK-NEXT: [[TMP2:%.*]] = add <4 x i64> [[VEC_IND]], splat (i64 1)
; CHECK-NEXT: br i1 [[U2]], label %[[BB42:.*]], label %[[BB31:.*]]
; CHECK: [[BB31]]:
-; CHECK-NEXT: [[TMP7:%.*]] = add i64 [[INDEX]], 3
+; CHECK-NEXT: [[TMP3:%.*]] = add i64 [[INDEX]], 3
; CHECK-NEXT: br label %[[BB53]]
; CHECK: [[BB42]]:
-; CHECK-NEXT: [[TMP3:%.*]] = add i64 [[INDEX]], 4
+; CHECK-NEXT: [[TMP4:%.*]] = add i64 [[INDEX]], 4
; CHECK-NEXT: br label %[[BB53]]
; CHECK: [[BB53]]:
-; CHECK-NEXT: [[TMP4:%.*]] = phi i64 [ [[TMP7]], %[[BB31]] ], [ poison, %[[BB42]] ]
-; CHECK-NEXT: [[TMP9:%.*]] = phi i64 [ poison, %[[BB31]] ], [ [[TMP3]], %[[BB42]] ]
-; CHECK-NEXT: [[BROADCAST_SPLATINSERT4:%.*]] = insertelement <4 x i64> poison, i64 [[TMP4]], i64 0
-; CHECK-NEXT: [[BROADCAST_SPLAT5:%.*]] = shufflevector <4 x i64> [[BROADCAST_SPLATINSERT4]], <4 x i64> poison, <4 x i32> zeroinitializer
-; CHECK-NEXT: [[BROADCAST_SPLATINSERT:%.*]] = insertelement <4 x i64> poison, i64 [[TMP9]], i64 0
-; CHECK-NEXT: [[BROADCAST_SPLAT:%.*]] = shufflevector <4 x i64> [[BROADCAST_SPLATINSERT]], <4 x i64> poison, <4 x i32> zeroinitializer
-; CHECK-NEXT: [[PREDPHI:%.*]] = select <4 x i1> [[TMP1]], <4 x i64> [[BROADCAST_SPLAT]], <4 x i64> [[BROADCAST_SPLAT5]]
-; CHECK-NEXT: [[TMP5:%.*]] = add <4 x i64> [[PREDPHI]], splat (i64 5)
+; CHECK-NEXT: [[TMP11:%.*]] = phi i64 [ [[TMP3]], %[[BB31]] ], [ poison, %[[BB42]] ]
+; CHECK-NEXT: [[TMP12:%.*]] = phi i64 [ poison, %[[BB31]] ], [ [[TMP4]], %[[BB42]] ]
+; CHECK-NEXT: [[TMP7:%.*]] = phi i1 [ false, %[[BB31]] ], [ [[TMP9]], %[[BB42]] ]
+; CHECK-NEXT: [[PREDPHI:%.*]] = select i1 [[TMP7]], i64 [[TMP12]], i64 [[TMP11]]
+; CHECK-NEXT: [[TMP8:%.*]] = add i64 [[PREDPHI]], 5
+; CHECK-NEXT: [[BROADCAST_SPLATINSERT:%.*]] = insertelement <4 x i64> poison, i64 [[TMP8]], i64 0
+; CHECK-NEXT: [[TMP5:%.*]] = shufflevector <4 x i64> [[BROADCAST_SPLATINSERT]], <4 x i64> poison, <4 x i32> zeroinitializer
; CHECK-NEXT: [[PREDPHI1:%.*]] = select <4 x i1> [[TMP1]], <4 x i64> [[TMP5]], <4 x i64> [[TMP2]]
; CHECK-NEXT: [[TMP6:%.*]] = extractelement <4 x i64> [[PREDPHI1]], i64 3
; CHECK-NEXT: store i64 [[TMP6]], ptr [[A]], align 4
; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 4
; CHECK-NEXT: [[VEC_IND_NEXT]] = add nuw nsw <4 x i64> [[VEC_IND]], splat (i64 4)
-; CHECK-NEXT: [[TMP8:%.*]] = icmp eq i64 [[INDEX_NEXT]], 128
-; CHECK-NEXT: br i1 [[TMP8]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP14:![0-9]+]]
+; CHECK-NEXT: [[TMP10:%.*]] = icmp eq i64 [[INDEX_NEXT]], 128
+; CHECK-NEXT: br i1 [[TMP10]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP14:![0-9]+]]
; CHECK: [[MIDDLE_BLOCK]]:
; CHECK-NEXT: br label %[[EXIT:.*]]
; CHECK: [[EXIT]]:
@@ -2036,7 +2040,7 @@ define void @uniform_branch_shared_join_with_varying(ptr %a, i1 %u0) {
; CHECK-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[BB43:.*]] ]
; CHECK-NEXT: br i1 [[U0]], label %[[BB22:.*]], label %[[BB11:.*]]
; CHECK: [[BB11]]:
-; CHECK-NEXT: [[TMP8:%.*]] = add i64 [[INDEX]], 1
+; CHECK-NEXT: [[TMP9:%.*]] = add i64 [[INDEX]], 1
; CHECK-NEXT: br label %[[BB43]]
; CHECK: [[BB22]]:
; CHECK-NEXT: [[TMP1:%.*]] = add i64 [[INDEX]], 2
@@ -2044,14 +2048,14 @@ define void @uniform_branch_shared_join_with_varying(ptr %a, i1 %u0) {
; CHECK-NEXT: [[TMP3:%.*]] = add i64 [[INDEX]], 3
; CHECK-NEXT: br label %[[BB43]]
; CHECK: [[BB43]]:
-; CHECK-NEXT: [[PREDPHI:%.*]] = phi i64 [ [[TMP8]], %[[BB11]] ], [ poison, %[[BB22]] ]
-; CHECK-NEXT: [[TMP6:%.*]] = phi i64 [ poison, %[[BB11]] ], [ [[TMP1]], %[[BB22]] ]
-; CHECK-NEXT: [[TMP7:%.*]] = phi i1 [ false, %[[BB11]] ], [ true, %[[BB22]] ]
-; CHECK-NEXT: [[TMP9:%.*]] = phi i64 [ poison, %[[BB11]] ], [ [[TMP3]], %[[BB22]] ]
-; CHECK-NEXT: [[TMP10:%.*]] = phi i1 [ false, %[[BB11]] ], [ [[TMP2]], %[[BB22]] ]
-; CHECK-NEXT: [[PREDPHI4:%.*]] = select i1 [[TMP7]], i64 [[TMP6]], i64 [[PREDPHI]]
-; CHECK-NEXT: [[PREDPHI5:%.*]] = select i1 [[TMP10]], i64 [[TMP9]], i64 [[PREDPHI4]]
-; CHECK-NEXT: store i64 [[PREDPHI5]], ptr [[A]], align 4
+; CHECK-NEXT: [[TMP4:%.*]] = phi i64 [ [[TMP9]], %[[BB11]] ], [ poison, %[[BB22]] ]
+; CHECK-NEXT: [[TMP5:%.*]] = phi i64 [ poison, %[[BB11]] ], [ [[TMP1]], %[[BB22]] ]
+; CHECK-NEXT: [[TMP6:%.*]] = phi i1 [ false, %[[BB11]] ], [ true, %[[BB22]] ]
+; CHECK-NEXT: [[TMP7:%.*]] = phi i64 [ poison, %[[BB11]] ], [ [[TMP3]], %[[BB22]] ]
+; CHECK-NEXT: [[TMP8:%.*]] = phi i1 [ false, %[[BB11]] ], [ [[TMP2]], %[[BB22]] ]
+; CHECK-NEXT: [[PREDPHI:%.*]] = select i1 [[TMP6]], i64 [[TMP5]], i64 [[TMP4]]
+; CHECK-NEXT: [[PREDPHI4:%.*]] = select i1 [[TMP8]], i64 [[TMP7]], i64 [[PREDPHI]]
+; CHECK-NEXT: store i64 [[PREDPHI4]], ptr [[A]], align 4
; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 4
; CHECK-NEXT: [[TMP0:%.*]] = icmp eq i64 [[INDEX_NEXT]], 128
; CHECK-NEXT: br i1 [[TMP0]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP38:![0-9]+]]
diff --git a/llvm/test/Transforms/LoopVectorize/reduction-small-size.ll b/llvm/test/Transforms/LoopVectorize/reduction-small-size.ll
index 1131272009f4f..592c60a40bf6f 100644
--- a/llvm/test/Transforms/LoopVectorize/reduction-small-size.ll
+++ b/llvm/test/Transforms/LoopVectorize/reduction-small-size.ll
@@ -13,17 +13,17 @@ define i8 @PR34687(i1 %c, i32 %x, i32 %n, i32 %divisor) {
; CHECK-NEXT: [[N_VEC:%.*]] = sub i32 [[N]], [[N_MOD_VF]]
; CHECK-NEXT: [[BROADCAST_SPLATINSERT:%.*]] = insertelement <4 x i1> poison, i1 [[C:%.*]], i64 0
; CHECK-NEXT: [[BROADCAST_SPLAT:%.*]] = shufflevector <4 x i1> [[BROADCAST_SPLATINSERT]], <4 x i1> poison, <4 x i32> zeroinitializer
+; CHECK-NEXT: [[BROADCAST_SPLATINSERT1:%.*]] = insertelement <4 x i32> poison, i32 [[X:%.*]], i64 0
+; CHECK-NEXT: [[BROADCAST_SPLAT2:%.*]] = shufflevector <4 x i32> [[BROADCAST_SPLATINSERT1]], <4 x i32> poison, <4 x i32> zeroinitializer
; CHECK-NEXT: [[BROADCAST_SPLATINSERT3:%.*]] = insertelement <4 x i32> poison, i32 [[X1:%.*]], i64 0
; CHECK-NEXT: [[BROADCAST_SPLAT4:%.*]] = shufflevector <4 x i32> [[BROADCAST_SPLATINSERT3]], <4 x i32> poison, <4 x i32> zeroinitializer
-; CHECK-NEXT: [[BROADCAST_SPLATINSERT4:%.*]] = insertelement <4 x i32> poison, i32 [[X:%.*]], i64 0
-; CHECK-NEXT: [[BROADCAST_SPLAT5:%.*]] = shufflevector <4 x i32> [[BROADCAST_SPLATINSERT4]], <4 x i32> poison, <4 x i32> zeroinitializer
; CHECK-NEXT: br label [[VECTOR_BODY:%.*]]
; CHECK: vector.body:
; CHECK-NEXT: [[INDEX:%.*]] = phi i32 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ]
; CHECK-NEXT: [[VEC_IND:%.*]] = phi <4 x i32> [ <i32 0, i32 1, i32 2, i32 3>, [[VECTOR_PH]] ], [ [[VEC_IND_NEXT:%.*]], [[VECTOR_BODY]] ]
; CHECK-NEXT: [[VEC_PHI:%.*]] = phi <4 x i32> [ zeroinitializer, [[VECTOR_PH]] ], [ [[TMP4:%.*]], [[VECTOR_BODY]] ]
; CHECK-NEXT: [[TMP1:%.*]] = and <4 x i32> [[VEC_PHI]], splat (i32 255)
-; CHECK-NEXT: [[TMP2:%.*]] = add <4 x i32> [[TMP1]], [[BROADCAST_SPLAT5]]
+; CHECK-NEXT: [[TMP2:%.*]] = add <4 x i32> [[TMP1]], [[BROADCAST_SPLAT4]]
; CHECK-NEXT: [[TMP3:%.*]] = trunc <4 x i32> [[TMP2]] to <4 x i8>
; CHECK-NEXT: [[TMP4]] = zext <4 x i8> [[TMP3]] to <4 x i32>
; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i32 [[INDEX]], 4
@@ -31,7 +31,7 @@ define i8 @PR34687(i1 %c, i32 %x, i32 %n, i32 %divisor) {
; CHECK-NEXT: [[TMP5:%.*]] = icmp eq i32 [[INDEX_NEXT]], [[N_VEC]]
; CHECK-NEXT: br i1 [[TMP5]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP0:![0-9]+]]
; CHECK: middle.block:
-; CHECK-NEXT: [[TMP6:%.*]] = call <4 x i32> @llvm.masked.sdiv.v4i32(<4 x i32> [[VEC_IND]], <4 x i32> [[BROADCAST_SPLAT4]], <4 x i1> [[BROADCAST_SPLAT]])
+; CHECK-NEXT: [[TMP6:%.*]] = call <4 x i32> @llvm.masked.sdiv.v4i32(<4 x i32> [[VEC_IND]], <4 x i32> [[BROADCAST_SPLAT2]], <4 x i1> [[BROADCAST_SPLAT]])
; CHECK-NEXT: [[PREDPHI1:%.*]] = select i1 [[C]], <4 x i32> [[TMP6]], <4 x i32> zeroinitializer
; CHECK-NEXT: [[TMP7:%.*]] = call i8 @llvm.vector.reduce.add.v4i8(<4 x i8> [[TMP3]])
; CHECK-NEXT: [[TMP8:%.*]] = zext i8 [[TMP7]] to i32
@@ -47,13 +47,13 @@ define i8 @PR34687(i1 %c, i32 %x, i32 %n, i32 %divisor) {
; CHECK-NEXT: [[R:%.*]] = phi i32 [ [[BC_MERGE_RDX]], [[SCALAR_PH]] ], [ [[R_NEXT:%.*]], [[IF_END]] ]
; CHECK-NEXT: br i1 [[C]], label [[IF_THEN:%.*]], label [[IF_END]]
; CHECK: if.then:
-; CHECK-NEXT: [[T0:%.*]] = sdiv i32 [[I]], [[X1]]
+; CHECK-NEXT: [[T0:%.*]] = sdiv i32 [[I]], [[X]]
; CHECK-NEXT: br label [[IF_END]]
; CHECK: if.end:
; CHECK-NEXT: [[DIV_PHI:%.*]] = phi i32 [ 0, [[FOR_BODY]] ], [ [[T0]], [[IF_THEN]] ]
; CHECK-NEXT: [[T1:%.*]] = and i32 [[R]], 255
; CHECK-NEXT: [[I_NEXT]] = add nsw i32 [[I]], 1
-; CHECK-NEXT: [[R_NEXT]] = add nuw nsw i32 [[T1]], [[X]]
+; CHECK-NEXT: [[R_NEXT]] = add nuw nsw i32 [[T1]], [[X1]]
; CHECK-NEXT: [[COND:%.*]] = icmp eq i32 [[I_NEXT]], [[N]]
; CHECK-NEXT: br i1 [[COND]], label [[FOR_END]], label [[FOR_BODY]], !llvm.loop [[LOOP3:![0-9]+]]
; CHECK: for.end:
@@ -97,11 +97,11 @@ define i8 @PR34687_no_undef(i1 %c, i32 %x, i32 %n) {
; CHECK: vector.ph:
; CHECK-NEXT: [[N_MOD_VF:%.*]] = and i32 [[N]], 3
; CHECK-NEXT: [[N_VEC:%.*]] = sub i32 [[N]], [[N_MOD_VF]]
-; CHECK-NEXT: [[BROADCAST_SPLATINSERT:%.*]] = insertelement <4 x i32> poison, i32 [[X:%.*]], i64 0
-; CHECK-NEXT: [[BROADCAST_SPLAT:%.*]] = shufflevector <4 x i32> [[BROADCAST_SPLATINSERT]], <4 x i32> poison, <4 x i32> zeroinitializer
-; CHECK-NEXT: [[BROADCAST_SPLATINSERT1:%.*]] = insertelement <4 x i1> poison, i1 [[C:%.*]], i64 0
-; CHECK-NEXT: [[BROADCAST_SPLAT2:%.*]] = shufflevector <4 x i1> [[BROADCAST_SPLATINSERT1]], <4 x i1> poison, <4 x i32> zeroinitializer
-; CHECK-NEXT: [[TMP1:%.*]] = call <4 x i32> @llvm.masked.sdiv.v4i32(<4 x i32> splat (i32 99), <4 x i32> [[BROADCAST_SPLAT]], <4 x i1> [[BROADCAST_SPLAT2]])
+; CHECK-NEXT: [[BROADCAST_SPLATINSERT1:%.*]] = insertelement <4 x i32> poison, i32 [[X:%.*]], i64 0
+; CHECK-NEXT: [[BROADCAST_SPLAT2:%.*]] = shufflevector <4 x i32> [[BROADCAST_SPLATINSERT1]], <4 x i32> poison, <4 x i32> zeroinitializer
+; CHECK-NEXT: [[BROADCAST_SPLATINSERT2:%.*]] = insertelement <4 x i1> poison, i1 [[C:%.*]], i64 0
+; CHECK-NEXT: [[BROADCAST_SPLAT3:%.*]] = shufflevector <4 x i1> [[BROADCAST_SPLATINSERT2]], <4 x i1> poison, <4 x i32> zeroinitializer
+; CHECK-NEXT: [[TMP1:%.*]] = call <4 x i32> @llvm.masked.sdiv.v4i32(<4 x i32> splat (i32 99), <4 x i32> [[BROADCAST_SPLAT2]], <4 x i1> [[BROADCAST_SPLAT3]])
; CHECK-NEXT: [[PREDPHI:%.*]] = select i1 [[C]], <4 x i32> [[TMP1]], <4 x i32> zeroinitializer
; CHECK-NEXT: br label [[VECTOR_BODY:%.*]]
; CHECK: vector.body:
diff --git a/llvm/test/Transforms/LoopVectorize/scalarize-masked-call.ll b/llvm/test/Transforms/LoopVectorize/scalarize-masked-call.ll
index 920e982a52af7..32841dd259618 100644
--- a/llvm/test/Transforms/LoopVectorize/scalarize-masked-call.ll
+++ b/llvm/test/Transforms/LoopVectorize/scalarize-masked-call.ll
@@ -1,70 +1,69 @@
-; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --version 6
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
; RUN: opt < %s -passes=loop-vectorize -force-vector-interleave=1 -force-vector-width=2 -S 2>&1 | FileCheck %s
target datalayout = "e-m:e-i64:64-f80:128-n8:16:32:64-S128"
define void @cond_call(ptr readonly %src, ptr noalias %dest, i64 %N) {
-; CHECK-LABEL: define void @cond_call(
-; CHECK-SAME: ptr readonly [[SRC:%.*]], ptr noalias [[DEST:%.*]], i64 [[N:%.*]]) {
-; CHECK-NEXT: [[ENTRY:.*]]:
-; CHECK-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[N]], 2
-; CHECK-NEXT: br i1 [[MIN_ITERS_CHECK]], label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]]
-; CHECK: [[VECTOR_PH]]:
+; CHECK-LABEL: @cond_call(
+; CHECK-NEXT: entry:
+; CHECK-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[N:%.*]], 2
+; CHECK-NEXT: br i1 [[MIN_ITERS_CHECK]], label [[SCALAR_PH:%.*]], label [[VECTOR_PH:%.*]]
+; CHECK: vector.ph:
; CHECK-NEXT: [[N_MOD_VF:%.*]] = and i64 [[N]], 1
; CHECK-NEXT: [[N_VEC:%.*]] = sub i64 [[N]], [[N_MOD_VF]]
-; CHECK-NEXT: br label %[[VECTOR_BODY:.*]]
-; CHECK: [[VECTOR_BODY]]:
-; CHECK-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[PRED_CALL_CONTINUE2:.*]] ]
-; CHECK-NEXT: [[TMP0:%.*]] = getelementptr inbounds i64, ptr [[SRC]], i64 [[INDEX]]
+; CHECK-NEXT: br label [[VECTOR_BODY:%.*]]
+; CHECK: vector.body:
+; CHECK-NEXT: [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[PRED_CALL_CONTINUE2:%.*]] ]
+; CHECK-NEXT: [[TMP0:%.*]] = getelementptr inbounds i64, ptr [[SRC:%.*]], i64 [[INDEX]]
; CHECK-NEXT: [[WIDE_LOAD:%.*]] = load <2 x i64>, ptr [[TMP0]], align 8
; CHECK-NEXT: [[TMP1:%.*]] = icmp ult <2 x i64> [[WIDE_LOAD]], splat (i64 5)
; CHECK-NEXT: [[TMP2:%.*]] = extractelement <2 x i1> [[TMP1]], i64 0
-; CHECK-NEXT: br i1 [[TMP2]], label %[[PRED_CALL_IF:.*]], label %[[PRED_CALL_CONTINUE:.*]]
-; CHECK: [[PRED_CALL_IF]]:
+; CHECK-NEXT: br i1 [[TMP2]], label [[PRED_CALL_IF:%.*]], label [[PRED_CALL_CONTINUE:%.*]]
+; CHECK: pred.call.if:
; CHECK-NEXT: [[TMP3:%.*]] = extractelement <2 x i64> [[WIDE_LOAD]], i64 0
; CHECK-NEXT: [[TMP4:%.*]] = call i64 @foo(i64 [[TMP3]]) #[[ATTR0:[0-9]+]]
; CHECK-NEXT: [[TMP5:%.*]] = insertelement <2 x i64> poison, i64 [[TMP4]], i64 0
-; CHECK-NEXT: br label %[[PRED_CALL_CONTINUE]]
-; CHECK: [[PRED_CALL_CONTINUE]]:
-; CHECK-NEXT: [[TMP6:%.*]] = phi <2 x i64> [ poison, %[[VECTOR_BODY]] ], [ [[TMP5]], %[[PRED_CALL_IF]] ]
+; CHECK-NEXT: br label [[PRED_CALL_CONTINUE]]
+; CHECK: pred.call.continue:
+; CHECK-NEXT: [[TMP6:%.*]] = phi <2 x i64> [ poison, [[VECTOR_BODY]] ], [ [[TMP5]], [[PRED_CALL_IF]] ]
; CHECK-NEXT: [[TMP7:%.*]] = extractelement <2 x i1> [[TMP1]], i64 1
-; CHECK-NEXT: br i1 [[TMP7]], label %[[PRED_CALL_IF1:.*]], label %[[PRED_CALL_CONTINUE2]]
-; CHECK: [[PRED_CALL_IF1]]:
+; CHECK-NEXT: br i1 [[TMP7]], label [[PRED_CALL_IF1:%.*]], label [[PRED_CALL_CONTINUE2]]
+; CHECK: pred.call.if1:
; CHECK-NEXT: [[TMP8:%.*]] = extractelement <2 x i64> [[WIDE_LOAD]], i64 1
; CHECK-NEXT: [[TMP9:%.*]] = call i64 @foo(i64 [[TMP8]]) #[[ATTR0]]
; CHECK-NEXT: [[TMP10:%.*]] = insertelement <2 x i64> [[TMP6]], i64 [[TMP9]], i64 1
-; CHECK-NEXT: br label %[[PRED_CALL_CONTINUE2]]
-; CHECK: [[PRED_CALL_CONTINUE2]]:
-; CHECK-NEXT: [[TMP11:%.*]] = phi <2 x i64> [ [[TMP6]], %[[PRED_CALL_CONTINUE]] ], [ [[TMP10]], %[[PRED_CALL_IF1]] ]
+; CHECK-NEXT: br label [[PRED_CALL_CONTINUE2]]
+; CHECK: pred.call.continue2:
+; CHECK-NEXT: [[TMP11:%.*]] = phi <2 x i64> [ [[TMP6]], [[PRED_CALL_CONTINUE]] ], [ [[TMP10]], [[PRED_CALL_IF1]] ]
; CHECK-NEXT: [[PREDPHI:%.*]] = select <2 x i1> [[TMP1]], <2 x i64> [[TMP11]], <2 x i64> [[WIDE_LOAD]]
-; CHECK-NEXT: [[TMP12:%.*]] = getelementptr inbounds i64, ptr [[DEST]], i64 [[INDEX]]
+; CHECK-NEXT: [[TMP12:%.*]] = getelementptr inbounds i64, ptr [[DEST:%.*]], i64 [[INDEX]]
; CHECK-NEXT: store <2 x i64> [[PREDPHI]], ptr [[TMP12]], align 8
; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 2
; CHECK-NEXT: [[TMP13:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
-; CHECK-NEXT: br i1 [[TMP13]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP0:![0-9]+]]
-; CHECK: [[MIDDLE_BLOCK]]:
+; CHECK-NEXT: br i1 [[TMP13]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP0:![0-9]+]]
+; CHECK: middle.block:
; CHECK-NEXT: [[CMP_N:%.*]] = icmp eq i64 [[N]], [[N_VEC]]
-; CHECK-NEXT: br i1 [[CMP_N]], label %[[END:.*]], label %[[SCALAR_PH]]
-; CHECK: [[SCALAR_PH]]:
-; CHECK-NEXT: [[BC_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC]], %[[MIDDLE_BLOCK]] ], [ 0, %[[ENTRY]] ]
-; CHECK-NEXT: br label %[[FOR_BODY:.*]]
-; CHECK: [[FOR_BODY]]:
-; CHECK-NEXT: [[IV:%.*]] = phi i64 [ [[BC_RESUME_VAL]], %[[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], %[[FOR_LOOP:.*]] ]
+; CHECK-NEXT: br i1 [[CMP_N]], label [[END:%.*]], label [[SCALAR_PH]]
+; CHECK: scalar.ph:
+; CHECK-NEXT: [[BC_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC]], [[MIDDLE_BLOCK]] ], [ 0, [[ENTRY:%.*]] ]
+; CHECK-NEXT: br label [[FOR_BODY:%.*]]
+; CHECK: for.body:
+; CHECK-NEXT: [[IV:%.*]] = phi i64 [ [[BC_RESUME_VAL]], [[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], [[FOR_LOOP:%.*]] ]
; CHECK-NEXT: [[LD_ADDR:%.*]] = getelementptr inbounds i64, ptr [[SRC]], i64 [[IV]]
; CHECK-NEXT: [[LD_VALUE:%.*]] = load i64, ptr [[LD_ADDR]], align 8
; CHECK-NEXT: [[IFCOND:%.*]] = icmp ult i64 [[LD_VALUE]], 5
-; CHECK-NEXT: br i1 [[IFCOND]], label %[[IF_THEN:.*]], label %[[FOR_LOOP]]
-; CHECK: [[IF_THEN]]:
+; CHECK-NEXT: br i1 [[IFCOND]], label [[IF_THEN:%.*]], label [[FOR_LOOP]]
+; CHECK: if.then:
; CHECK-NEXT: [[FOO_RET:%.*]] = call i64 @foo(i64 [[LD_VALUE]]) #[[ATTR0]]
-; CHECK-NEXT: br label %[[FOR_LOOP]]
-; CHECK: [[FOR_LOOP]]:
-; CHECK-NEXT: [[ST_VALUE:%.*]] = phi i64 [ [[LD_VALUE]], %[[FOR_BODY]] ], [ [[FOO_RET]], %[[IF_THEN]] ]
+; CHECK-NEXT: br label [[FOR_LOOP]]
+; CHECK: for.loop:
+; CHECK-NEXT: [[ST_VALUE:%.*]] = phi i64 [ [[LD_VALUE]], [[FOR_BODY]] ], [ [[FOO_RET]], [[IF_THEN]] ]
; CHECK-NEXT: [[ST_ADDR:%.*]] = getelementptr inbounds i64, ptr [[DEST]], i64 [[IV]]
; CHECK-NEXT: store i64 [[ST_VALUE]], ptr [[ST_ADDR]], align 8
; CHECK-NEXT: [[IV_NEXT]] = add nuw nsw i64 [[IV]], 1
; CHECK-NEXT: [[LOOPCOND:%.*]] = icmp eq i64 [[IV_NEXT]], [[N]]
-; CHECK-NEXT: br i1 [[LOOPCOND]], label %[[END]], label %[[FOR_BODY]], !llvm.loop [[LOOP3:![0-9]+]]
-; CHECK: [[END]]:
+; CHECK-NEXT: br i1 [[LOOPCOND]], label [[END]], label [[FOR_BODY]], !llvm.loop [[LOOP3:![0-9]+]]
+; CHECK: end:
; CHECK-NEXT: ret void
;
entry:
@@ -96,34 +95,33 @@ end:
; Test for a masked scalar call to an intrinsic whose result is used as part of
; a load address.
define void @masked_umax_used_by_load_address(ptr noalias %src, ptr noalias %dst, i64 %n, i1 %c) {
-; CHECK-LABEL: define void @masked_umax_used_by_load_address(
-; CHECK-SAME: ptr noalias [[SRC:%.*]], ptr noalias [[DST:%.*]], i64 [[N:%.*]], i1 [[C:%.*]]) {
-; CHECK-NEXT: [[ENTRY:.*:]]
-; CHECK-NEXT: br label %[[VECTOR_PH:.*]]
-; CHECK: [[VECTOR_PH]]:
-; CHECK-NEXT: [[TMP0:%.*]] = call i64 @llvm.umax.i64(i64 [[N]], i64 1)
-; CHECK-NEXT: [[TMP1:%.*]] = getelementptr i16, ptr [[SRC]], i64 [[TMP0]]
-; CHECK-NEXT: br label %[[VECTOR_BODY:.*]]
-; CHECK: [[VECTOR_BODY]]:
-; CHECK-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[PRED_LOAD_CONTINUE2:.*]] ]
-; CHECK-NEXT: br i1 [[C]], label %[[PRED_LOAD_IF1:.*]], label %[[PRED_LOAD_CONTINUE2]]
-; CHECK: [[PRED_LOAD_IF1]]:
+; CHECK-LABEL: @masked_umax_used_by_load_address(
+; CHECK-NEXT: entry:
+; CHECK-NEXT: br label [[VECTOR_PH:%.*]]
+; CHECK: vector.ph:
+; CHECK-NEXT: [[TMP0:%.*]] = call i64 @llvm.umax.i64(i64 [[N:%.*]], i64 1)
+; CHECK-NEXT: [[TMP1:%.*]] = getelementptr i16, ptr [[SRC:%.*]], i64 [[TMP0]]
+; CHECK-NEXT: br label [[VECTOR_BODY:%.*]]
+; CHECK: vector.body:
+; CHECK-NEXT: [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[PRED_LOAD_CONTINUE2:%.*]] ]
+; CHECK-NEXT: br i1 [[C:%.*]], label [[PRED_LOAD_IF1:%.*]], label [[PRED_LOAD_CONTINUE2]]
+; CHECK: then1:
; CHECK-NEXT: [[TMP5:%.*]] = load i16, ptr [[TMP1]], align 2
-; CHECK-NEXT: br label %[[PRED_LOAD_CONTINUE2]]
-; CHECK: [[PRED_LOAD_CONTINUE2]]:
-; CHECK-NEXT: [[TMP3:%.*]] = phi i16 [ poison, %[[VECTOR_BODY]] ], [ [[TMP5]], %[[PRED_LOAD_IF1]] ]
-; CHECK-NEXT: [[TMP4:%.*]] = phi i1 [ false, %[[VECTOR_BODY]] ], [ true, %[[PRED_LOAD_IF1]] ]
+; CHECK-NEXT: br label [[PRED_LOAD_CONTINUE2]]
+; CHECK: loop.latch2:
+; CHECK-NEXT: [[TMP3:%.*]] = phi i16 [ poison, [[VECTOR_BODY]] ], [ [[TMP5]], [[PRED_LOAD_IF1]] ]
+; CHECK-NEXT: [[TMP4:%.*]] = phi i1 [ false, [[VECTOR_BODY]] ], [ true, [[PRED_LOAD_IF1]] ]
; CHECK-NEXT: [[BROADCAST_SPLATINSERT:%.*]] = insertelement <2 x i16> poison, i16 [[TMP3]], i64 0
; CHECK-NEXT: [[BROADCAST_SPLAT:%.*]] = shufflevector <2 x i16> [[BROADCAST_SPLATINSERT]], <2 x i16> poison, <2 x i32> zeroinitializer
; CHECK-NEXT: [[PREDPHI:%.*]] = select i1 [[TMP4]], <2 x i16> [[BROADCAST_SPLAT]], <2 x i16> zeroinitializer
-; CHECK-NEXT: [[TMP8:%.*]] = getelementptr i16, ptr [[DST]], i64 [[INDEX]]
+; CHECK-NEXT: [[TMP8:%.*]] = getelementptr i16, ptr [[DST:%.*]], i64 [[INDEX]]
; CHECK-NEXT: store <2 x i16> [[PREDPHI]], ptr [[TMP8]], align 2
; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 2
; CHECK-NEXT: [[TMP9:%.*]] = icmp eq i64 [[INDEX_NEXT]], 100
-; CHECK-NEXT: br i1 [[TMP9]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP4:![0-9]+]]
-; CHECK: [[MIDDLE_BLOCK]]:
-; CHECK-NEXT: br label %[[EXIT:.*]]
-; CHECK: [[EXIT]]:
+; CHECK-NEXT: br i1 [[TMP9]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP4:![0-9]+]]
+; CHECK: middle.block:
+; CHECK-NEXT: br label [[EXIT:%.*]]
+; CHECK: exit:
; CHECK-NEXT: ret void
;
entry:
@@ -160,10 +158,3 @@ declare <4 x i64> @vector_foo(<4 x i64>, <4 x i1>)
; "-force-vector-width=2", then it should pass the legality checks but
; scalarize. TODO: Remove the requirement to have a variant.
attributes #0 = { readonly nounwind "vector-function-abi-variant"="_ZGV_LLVM_M4v_foo(vector_foo)" }
-;.
-; CHECK: [[LOOP0]] = distinct !{[[LOOP0]], [[META1:![0-9]+]], [[META2:![0-9]+]]}
-; CHECK: [[META1]] = !{!"llvm.loop.isvectorized", i32 1}
-; CHECK: [[META2]] = !{!"llvm.loop.unroll.runtime.disable"}
-; CHECK: [[LOOP3]] = distinct !{[[LOOP3]], [[META2]], [[META1]]}
-; CHECK: [[LOOP4]] = distinct !{[[LOOP4]], [[META1]], [[META2]]}
-;.
>From 0d96669b80f2a0bb8455882683b107b83b45975e Mon Sep 17 00:00:00 2001
From: Andrei Elovikov <andrei.elovikov at sifive.com>
Date: Thu, 27 Aug 2026 08:37:48 -0700
Subject: [PATCH 07/11] Do CommonDom mask removal before reconstructSSA run for
blend masks
---
.../Transforms/Vectorize/VPlanPredicator.cpp | 12 +++--
.../LoopVectorize/RISCV/dead-ops-cost.ll | 6 +--
.../Transforms/LoopVectorize/RISCV/divrem.ll | 28 +++++-----
.../RISCV/gather-scatter-cost.ll | 20 +++----
.../RISCV/tail-folding-complex-mask.ll | 20 +++----
.../LoopVectorize/VPlan/predicator.ll | 15 ++----
.../constant-fold-commutative-and.ll | 15 ++----
.../predicatedinst-loop-invariant.ll | 10 ++--
.../Transforms/LoopVectorize/predicator.ll | 54 ++++++++-----------
9 files changed, 74 insertions(+), 106 deletions(-)
diff --git a/llvm/lib/Transforms/Vectorize/VPlanPredicator.cpp b/llvm/lib/Transforms/Vectorize/VPlanPredicator.cpp
index 9fe829538e6d3..13ac3cce79c46 100644
--- a/llvm/lib/Transforms/Vectorize/VPlanPredicator.cpp
+++ b/llvm/lib/Transforms/Vectorize/VPlanPredicator.cpp
@@ -580,7 +580,8 @@ void VPPredicator::convertPhisToBlends(VPBasicBlock *VPBB) {
MaskBlocks.push_back(const_cast<VPBasicBlock *>(ConstMaskBlock));
// The in-mask of the common dominator is true on all paths from an
- // incoming block to the phi. Remove it from the blend masks.
+ // incoming block to the phi. Remove it before reconstructing the masks,
+ // which may otherwise obscure the common subexpression with SSA phis.
VPBasicBlock *CommonIncomingDom =
cast<VPBasicBlock>(VPDT.findNearestCommonDominator(
make_range(MaskBlocks.begin(), MaskBlocks.end())));
@@ -598,9 +599,12 @@ void VPPredicator::convertPhisToBlends(VPBasicBlock *VPBB) {
bool RemovedCommonMask =
CommonIncomingMask && Mask &&
match(Mask, m_RemoveMask(CommonIncomingMask, RemainingMask));
- if (RemovedCommonMask)
- Mask = RemainingMask ? RemainingMask : Plan.getTrue();
- OperandsWithMask.append({V, reconstructBlendMask(VPBB, MaskBlock, Mask)});
+ OperandsWithMask.append(
+ {V, RemovedCommonMask && !RemainingMask
+ ? Plan.getTrue()
+ : reconstructBlendMask(VPBB, MaskBlock,
+ RemovedCommonMask ? RemainingMask
+ : Mask)});
}
PHINode *IRPhi = cast_or_null<PHINode>(PhiR->getUnderlyingValue());
diff --git a/llvm/test/Transforms/LoopVectorize/RISCV/dead-ops-cost.ll b/llvm/test/Transforms/LoopVectorize/RISCV/dead-ops-cost.ll
index 80863da53f38d..541b2d724efae 100644
--- a/llvm/test/Transforms/LoopVectorize/RISCV/dead-ops-cost.ll
+++ b/llvm/test/Transforms/LoopVectorize/RISCV/dead-ops-cost.ll
@@ -244,15 +244,11 @@ define void @test_phi_in_latch_redundant(ptr %dst, i32 %a) {
; CHECK-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[CURRENT_ITERATION_NEXT:%.*]], %[[LOOP_LATCH2:.*]] ]
; CHECK-NEXT: [[AVL:%.*]] = phi i64 [ 37, %[[VECTOR_PH]] ], [ [[AVL_NEXT:%.*]], %[[LOOP_LATCH2]] ]
; CHECK-NEXT: [[TMP8:%.*]] = call i32 @llvm.experimental.get.vector.length.i64(i64 [[AVL]], i32 4, i1 true)
-; CHECK-NEXT: [[BROADCAST_SPLATINSERT1:%.*]] = insertelement <vscale x 4 x i32> poison, i32 [[TMP8]], i64 0
-; CHECK-NEXT: [[BROADCAST_SPLAT1:%.*]] = shufflevector <vscale x 4 x i32> [[BROADCAST_SPLATINSERT1]], <vscale x 4 x i32> poison, <vscale x 4 x i32> zeroinitializer
-; CHECK-NEXT: [[TMP4:%.*]] = call <vscale x 4 x i32> @llvm.stepvector.nxv4i32()
-; CHECK-NEXT: [[TMP6:%.*]] = icmp ult <vscale x 4 x i32> [[TMP4]], [[BROADCAST_SPLAT1]]
; CHECK-NEXT: br label %[[THEN1:.*]]
; CHECK: [[THEN1]]:
; CHECK-NEXT: br label %[[LOOP_LATCH2]]
; CHECK: [[LOOP_LATCH2]]:
-; CHECK-NEXT: [[PREDPHI:%.*]] = select <vscale x 4 x i1> [[TMP6]], <vscale x 4 x i32> [[BROADCAST_SPLAT]], <vscale x 4 x i32> zeroinitializer
+; CHECK-NEXT: [[PREDPHI:%.*]] = select i1 true, <vscale x 4 x i32> [[BROADCAST_SPLAT]], <vscale x 4 x i32> zeroinitializer
; CHECK-NEXT: [[TMP2:%.*]] = mul i64 [[INDEX]], 36
; CHECK-NEXT: [[TMP3:%.*]] = getelementptr i8, ptr [[DST]], i64 [[TMP2]]
; CHECK-NEXT: call void @llvm.experimental.vp.strided.store.nxv4i32.p0.i64(<vscale x 4 x i32> [[PREDPHI]], ptr align 4 [[TMP3]], i64 36, <vscale x 4 x i1> splat (i1 true), i32 [[TMP8]])
diff --git a/llvm/test/Transforms/LoopVectorize/RISCV/divrem.ll b/llvm/test/Transforms/LoopVectorize/RISCV/divrem.ll
index f04134f54b15f..19bab9853c48f 100644
--- a/llvm/test/Transforms/LoopVectorize/RISCV/divrem.ll
+++ b/llvm/test/Transforms/LoopVectorize/RISCV/divrem.ll
@@ -649,27 +649,23 @@ define void @udiv_sdiv_with_invariant_divisors(i8 %x, i16 %y, i1 %c, ptr %p) {
; CHECK-NEXT: [[TMP3:%.*]] = trunc i32 [[TMP2]] to i8
; CHECK-NEXT: [[BROADCAST_SPLATINSERT7:%.*]] = insertelement <vscale x 4 x i8> poison, i8 [[TMP3]], i64 0
; CHECK-NEXT: [[BROADCAST_SPLAT8:%.*]] = shufflevector <vscale x 4 x i8> [[BROADCAST_SPLATINSERT7]], <vscale x 4 x i8> poison, <vscale x 4 x i32> zeroinitializer
-; CHECK-NEXT: [[TMP17:%.*]] = call <vscale x 4 x i32> @llvm.stepvector.nxv4i32()
-; CHECK-NEXT: [[TMP18:%.*]] = extractelement <vscale x 4 x i32> [[TMP17]], i64 0
-; CHECK-NEXT: [[TMP5:%.*]] = icmp ult i32 [[TMP18]], [[TMP2]]
; CHECK-NEXT: br i1 [[C:%.*]], label [[LOOP_LATCH8]], label [[THEN7:%.*]]
; CHECK: then7:
-; CHECK-NEXT: [[TMP6:%.*]] = call <vscale x 4 x i8> @llvm.vp.merge.nxv4i8(<vscale x 4 x i1> splat (i1 true), <vscale x 4 x i8> [[BROADCAST_SPLAT2]], <vscale x 4 x i8> splat (i8 1), i32 [[TMP2]])
-; CHECK-NEXT: [[TMP7:%.*]] = extractelement <vscale x 4 x i8> [[VEC_IND]], i64 0
-; CHECK-NEXT: [[TMP8:%.*]] = extractelement <vscale x 4 x i8> [[TMP6]], i64 0
-; CHECK-NEXT: [[TMP9:%.*]] = udiv i8 [[TMP7]], [[TMP8]]
-; CHECK-NEXT: [[TMP10:%.*]] = zext i8 [[TMP9]] to i16
-; CHECK-NEXT: [[TMP11:%.*]] = call <vscale x 4 x i16> @llvm.vp.merge.nxv4i16(<vscale x 4 x i1> splat (i1 true), <vscale x 4 x i16> [[BROADCAST_SPLAT4]], <vscale x 4 x i16> splat (i16 1), i32 [[TMP2]])
-; CHECK-NEXT: [[TMP12:%.*]] = extractelement <vscale x 4 x i16> [[TMP11]], i64 0
-; CHECK-NEXT: [[TMP13:%.*]] = sdiv i16 [[TMP10]], [[TMP12]]
-; CHECK-NEXT: [[TMP14:%.*]] = sext i16 [[TMP13]] to i32
+; CHECK-NEXT: [[TMP13:%.*]] = call <vscale x 4 x i8> @llvm.vp.merge.nxv4i8(<vscale x 4 x i1> splat (i1 true), <vscale x 4 x i8> [[BROADCAST_SPLAT2]], <vscale x 4 x i8> splat (i8 1), i32 [[TMP2]])
+; CHECK-NEXT: [[TMP14:%.*]] = extractelement <vscale x 4 x i8> [[VEC_IND]], i64 0
+; CHECK-NEXT: [[TMP5:%.*]] = extractelement <vscale x 4 x i8> [[TMP13]], i64 0
+; CHECK-NEXT: [[TMP6:%.*]] = udiv i8 [[TMP14]], [[TMP5]]
+; CHECK-NEXT: [[TMP7:%.*]] = zext i8 [[TMP6]] to i16
+; CHECK-NEXT: [[TMP8:%.*]] = call <vscale x 4 x i16> @llvm.vp.merge.nxv4i16(<vscale x 4 x i1> splat (i1 true), <vscale x 4 x i16> [[BROADCAST_SPLAT4]], <vscale x 4 x i16> splat (i16 1), i32 [[TMP2]])
+; CHECK-NEXT: [[TMP9:%.*]] = extractelement <vscale x 4 x i16> [[TMP8]], i64 0
+; CHECK-NEXT: [[TMP10:%.*]] = sdiv i16 [[TMP7]], [[TMP9]]
+; CHECK-NEXT: [[TMP11:%.*]] = sext i16 [[TMP10]] to i32
; CHECK-NEXT: br label [[LOOP_LATCH8]]
; CHECK: loop.latch8:
-; CHECK-NEXT: [[TMP15:%.*]] = phi i32 [ poison, [[LOOP_LATCH]] ], [ [[TMP14]], [[THEN7]] ]
-; CHECK-NEXT: [[TMP16:%.*]] = phi i1 [ false, [[LOOP_LATCH]] ], [ [[TMP5]], [[THEN7]] ]
-; CHECK-NEXT: [[BROADCAST_SPLATINSERT9:%.*]] = insertelement <vscale x 4 x i32> poison, i32 [[TMP15]], i64 0
+; CHECK-NEXT: [[TMP12:%.*]] = phi i32 [ poison, [[LOOP_LATCH]] ], [ [[TMP11]], [[THEN7]] ]
+; CHECK-NEXT: [[BROADCAST_SPLATINSERT9:%.*]] = insertelement <vscale x 4 x i32> poison, i32 [[TMP12]], i64 0
; CHECK-NEXT: [[BROADCAST_SPLAT10:%.*]] = shufflevector <vscale x 4 x i32> [[BROADCAST_SPLATINSERT9]], <vscale x 4 x i32> poison, <vscale x 4 x i32> zeroinitializer
-; CHECK-NEXT: [[PREDPHI:%.*]] = select i1 [[TMP16]], <vscale x 4 x i32> [[BROADCAST_SPLAT10]], <vscale x 4 x i32> zeroinitializer
+; CHECK-NEXT: [[PREDPHI:%.*]] = select i1 true, <vscale x 4 x i32> [[BROADCAST_SPLAT10]], <vscale x 4 x i32> zeroinitializer
; CHECK-NEXT: call void @llvm.vp.scatter.nxv4i32.nxv4p0(<vscale x 4 x i32> [[PREDPHI]], <vscale x 4 x ptr> align 4 [[BROADCAST_SPLAT6]], <vscale x 4 x i1> splat (i1 true), i32 [[TMP2]])
; CHECK-NEXT: [[AVL_NEXT]] = sub nuw i32 [[AVL]], [[TMP2]]
; CHECK-NEXT: [[VEC_IND_NEXT]] = add <vscale x 4 x i8> [[VEC_IND]], [[BROADCAST_SPLAT8]]
diff --git a/llvm/test/Transforms/LoopVectorize/RISCV/gather-scatter-cost.ll b/llvm/test/Transforms/LoopVectorize/RISCV/gather-scatter-cost.ll
index b0b9e6c3723c0..dbbc08638eda4 100644
--- a/llvm/test/Transforms/LoopVectorize/RISCV/gather-scatter-cost.ll
+++ b/llvm/test/Transforms/LoopVectorize/RISCV/gather-scatter-cost.ll
@@ -36,22 +36,18 @@ define void @predicated_uniform_load(ptr %src, i32 %n, ptr %dst, i1 %cond) {
; CHECK: vector.body:
; CHECK-NEXT: [[AVL:%.*]] = phi i32 [ [[TMP3]], [[VECTOR_PH]] ], [ [[AVL_NEXT:%.*]], [[LOOP_LATCH6:%.*]] ]
; CHECK-NEXT: [[TMP10:%.*]] = call i32 @llvm.experimental.get.vector.length.i32(i32 [[AVL]], i32 4, i1 true)
-; CHECK-NEXT: [[TMP12:%.*]] = call <vscale x 4 x i32> @llvm.stepvector.nxv4i32()
-; CHECK-NEXT: [[TMP13:%.*]] = extractelement <vscale x 4 x i32> [[TMP12]], i64 0
-; CHECK-NEXT: [[TMP14:%.*]] = icmp ult i32 [[TMP13]], [[TMP10]]
-; CHECK-NEXT: br i1 [[COND1:%.*]], label [[LOOP_THEN5:%.*]], label [[LOOP_ELSE4:%.*]]
+; CHECK-NEXT: br i1 [[COND:%.*]], label [[LOOP_THEN5:%.*]], label [[LOOP_ELSE4:%.*]]
; CHECK: loop.else4:
-; CHECK-NEXT: [[WIDE_MASKED_GATHER1:%.*]] = call <vscale x 4 x i32> @llvm.vp.gather.nxv4i32.nxv4p0(<vscale x 4 x ptr> align 4 [[BROADCAST_SPLAT]], <vscale x 4 x i1> splat (i1 true), i32 [[TMP10]]), !alias.scope [[META0:![0-9]+]]
-; CHECK-NEXT: [[TMP15:%.*]] = extractelement <vscale x 4 x i32> [[WIDE_MASKED_GATHER1]], i64 0
+; CHECK-NEXT: [[WIDE_MASKED_GATHER:%.*]] = call <vscale x 4 x i32> @llvm.vp.gather.nxv4i32.nxv4p0(<vscale x 4 x ptr> align 4 [[BROADCAST_SPLAT]], <vscale x 4 x i1> splat (i1 true), i32 [[TMP10]]), !alias.scope [[META0:![0-9]+]]
+; CHECK-NEXT: [[TMP12:%.*]] = extractelement <vscale x 4 x i32> [[WIDE_MASKED_GATHER]], i64 0
; CHECK-NEXT: br label [[LOOP_LATCH6]]
; CHECK: loop.then5:
; CHECK-NEXT: br label [[LOOP_LATCH6]]
; CHECK: loop.latch6:
-; CHECK-NEXT: [[TMP18:%.*]] = phi i32 [ [[TMP15]], [[LOOP_ELSE4]] ], [ poison, [[LOOP_THEN5]] ]
-; CHECK-NEXT: [[COND:%.*]] = phi i1 [ false, [[LOOP_ELSE4]] ], [ [[TMP14]], [[LOOP_THEN5]] ]
-; CHECK-NEXT: [[BROADCAST_SPLATINSERT7:%.*]] = insertelement <vscale x 4 x i32> poison, i32 [[TMP18]], i64 0
-; CHECK-NEXT: [[WIDE_MASKED_GATHER:%.*]] = shufflevector <vscale x 4 x i32> [[BROADCAST_SPLATINSERT7]], <vscale x 4 x i32> poison, <vscale x 4 x i32> zeroinitializer
-; CHECK-NEXT: [[PREDPHI:%.*]] = select i1 [[COND]], <vscale x 4 x i32> zeroinitializer, <vscale x 4 x i32> [[WIDE_MASKED_GATHER]]
+; CHECK-NEXT: [[TMP13:%.*]] = phi i32 [ [[TMP12]], [[LOOP_ELSE4]] ], [ poison, [[LOOP_THEN5]] ]
+; CHECK-NEXT: [[BROADCAST_SPLATINSERT7:%.*]] = insertelement <vscale x 4 x i32> poison, i32 [[TMP13]], i64 0
+; CHECK-NEXT: [[BROADCAST_SPLAT8:%.*]] = shufflevector <vscale x 4 x i32> [[BROADCAST_SPLATINSERT7]], <vscale x 4 x i32> poison, <vscale x 4 x i32> zeroinitializer
+; CHECK-NEXT: [[PREDPHI:%.*]] = select i1 true, <vscale x 4 x i32> zeroinitializer, <vscale x 4 x i32> [[BROADCAST_SPLAT8]]
; CHECK-NEXT: call void @llvm.vp.scatter.nxv4i32.nxv4p0(<vscale x 4 x i32> [[PREDPHI]], <vscale x 4 x ptr> align 4 [[BROADCAST_SPLAT4]], <vscale x 4 x i1> splat (i1 true), i32 [[TMP10]]), !alias.scope [[META3:![0-9]+]], !noalias [[META0]]
; CHECK-NEXT: [[AVL_NEXT]] = sub nuw i32 [[AVL]], [[TMP10]]
; CHECK-NEXT: [[TMP16:%.*]] = icmp eq i32 [[AVL_NEXT]], 0
@@ -62,7 +58,7 @@ define void @predicated_uniform_load(ptr %src, i32 %n, ptr %dst, i1 %cond) {
; CHECK-NEXT: br label [[LOOP:%.*]]
; CHECK: loop:
; CHECK-NEXT: [[IV:%.*]] = phi i32 [ 0, [[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], [[LOOP_LATCH:%.*]] ]
-; CHECK-NEXT: br i1 [[COND1]], label [[LOOP_THEN:%.*]], label [[LOOP_ELSE:%.*]]
+; CHECK-NEXT: br i1 [[COND]], label [[LOOP_THEN:%.*]], label [[LOOP_ELSE:%.*]]
; CHECK: loop.then:
; CHECK-NEXT: br label [[LOOP_LATCH]]
; CHECK: loop.else:
diff --git a/llvm/test/Transforms/LoopVectorize/RISCV/tail-folding-complex-mask.ll b/llvm/test/Transforms/LoopVectorize/RISCV/tail-folding-complex-mask.ll
index 311e2f4d8932a..fd2eccb482255 100644
--- a/llvm/test/Transforms/LoopVectorize/RISCV/tail-folding-complex-mask.ll
+++ b/llvm/test/Transforms/LoopVectorize/RISCV/tail-folding-complex-mask.ll
@@ -26,9 +26,6 @@ define void @test(i64 %n, ptr noalias %src0, ptr noalias %src1, ptr noalias %src
; IF-EVL-NEXT: [[EVL_BASED_IV:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_EVL_NEXT:%.*]], %[[LATCH7:.*]] ]
; IF-EVL-NEXT: [[AVL:%.*]] = phi i64 [ [[N]], %[[VECTOR_PH]] ], [ [[AVL_NEXT:%.*]], %[[LATCH7]] ]
; IF-EVL-NEXT: [[TMP7:%.*]] = call i32 @llvm.experimental.get.vector.length.i64(i64 [[AVL]], i32 4, i1 true)
-; IF-EVL-NEXT: [[TMP6:%.*]] = call <vscale x 4 x i32> @llvm.stepvector.nxv4i32()
-; IF-EVL-NEXT: [[TMP9:%.*]] = extractelement <vscale x 4 x i32> [[TMP6]], i64 0
-; IF-EVL-NEXT: [[TMP8:%.*]] = icmp ult i32 [[TMP9]], [[TMP7]]
; IF-EVL-NEXT: [[TMP10:%.*]] = getelementptr i32, ptr [[SRC0]], i64 [[EVL_BASED_IV]]
; IF-EVL-NEXT: [[VP_OP_LOAD:%.*]] = call <vscale x 4 x i32> @llvm.vp.load.nxv4i32.p0(ptr align 4 [[TMP10]], <vscale x 4 x i1> [[BROADCAST_SPLAT]], i32 [[TMP7]])
; IF-EVL-NEXT: [[PREDPHI:%.*]] = select i1 [[C1]], <vscale x 4 x i32> [[VP_OP_LOAD]], <vscale x 4 x i32> zeroinitializer
@@ -36,20 +33,19 @@ define void @test(i64 %n, ptr noalias %src0, ptr noalias %src1, ptr noalias %src
; IF-EVL-NEXT: [[VP_OP_LOAD7:%.*]] = call <vscale x 4 x i32> @llvm.vp.load.nxv4i32.p0(ptr align 4 [[TMP11]], <vscale x 4 x i1> [[TMP4]], i32 [[TMP7]])
; IF-EVL-NEXT: [[TMP12:%.*]] = add <vscale x 4 x i32> [[VP_OP_LOAD7]], [[PREDPHI]]
; IF-EVL-NEXT: [[PREDPHI8:%.*]] = select <vscale x 4 x i1> [[TMP4]], <vscale x 4 x i32> [[TMP12]], <vscale x 4 x i32> zeroinitializer
-; IF-EVL-NEXT: [[TMP18:%.*]] = extractelement <vscale x 4 x i32> [[PREDPHI8]], i64 0
+; IF-EVL-NEXT: [[TMP9:%.*]] = extractelement <vscale x 4 x i32> [[PREDPHI8]], i64 0
; IF-EVL-NEXT: br i1 [[C3]], label %[[LOAD_V25:.*]], label %[[LATCH7]]
; IF-EVL: [[LOAD_V25]]:
-; IF-EVL-NEXT: [[TMP13:%.*]] = getelementptr inbounds i32, ptr [[SRC2]], i64 [[EVL_BASED_IV]]
-; IF-EVL-NEXT: [[VP_OP_LOAD6:%.*]] = call <vscale x 4 x i32> @llvm.vp.load.nxv4i32.p0(ptr align 4 [[TMP13]], <vscale x 4 x i1> splat (i1 true), i32 [[TMP7]])
-; IF-EVL-NEXT: [[TMP14:%.*]] = extractelement <vscale x 4 x i32> [[VP_OP_LOAD6]], i64 0
-; IF-EVL-NEXT: [[TMP15:%.*]] = add i32 [[TMP14]], [[TMP18]]
+; IF-EVL-NEXT: [[TMP14:%.*]] = getelementptr inbounds i32, ptr [[SRC2]], i64 [[EVL_BASED_IV]]
+; IF-EVL-NEXT: [[VP_OP_LOAD6:%.*]] = call <vscale x 4 x i32> @llvm.vp.load.nxv4i32.p0(ptr align 4 [[TMP14]], <vscale x 4 x i1> splat (i1 true), i32 [[TMP7]])
+; IF-EVL-NEXT: [[TMP15:%.*]] = extractelement <vscale x 4 x i32> [[VP_OP_LOAD6]], i64 0
+; IF-EVL-NEXT: [[TMP16:%.*]] = add i32 [[TMP15]], [[TMP9]]
; IF-EVL-NEXT: br label %[[LATCH7]]
; IF-EVL: [[LATCH7]]:
-; IF-EVL-NEXT: [[TMP16:%.*]] = phi i32 [ poison, %[[VECTOR_BODY]] ], [ [[TMP15]], %[[LOAD_V25]] ]
-; IF-EVL-NEXT: [[TMP17:%.*]] = phi i1 [ false, %[[VECTOR_BODY]] ], [ [[TMP8]], %[[LOAD_V25]] ]
-; IF-EVL-NEXT: [[BROADCAST_SPLATINSERT8:%.*]] = insertelement <vscale x 4 x i32> poison, i32 [[TMP16]], i64 0
+; IF-EVL-NEXT: [[TMP13:%.*]] = phi i32 [ poison, %[[VECTOR_BODY]] ], [ [[TMP16]], %[[LOAD_V25]] ]
+; IF-EVL-NEXT: [[BROADCAST_SPLATINSERT8:%.*]] = insertelement <vscale x 4 x i32> poison, i32 [[TMP13]], i64 0
; IF-EVL-NEXT: [[BROADCAST_SPLAT9:%.*]] = shufflevector <vscale x 4 x i32> [[BROADCAST_SPLATINSERT8]], <vscale x 4 x i32> poison, <vscale x 4 x i32> zeroinitializer
-; IF-EVL-NEXT: [[PREDPHI9:%.*]] = select i1 [[TMP17]], <vscale x 4 x i32> [[BROADCAST_SPLAT9]], <vscale x 4 x i32> [[PREDPHI8]]
+; IF-EVL-NEXT: [[PREDPHI9:%.*]] = select i1 true, <vscale x 4 x i32> [[BROADCAST_SPLAT9]], <vscale x 4 x i32> [[PREDPHI8]]
; IF-EVL-NEXT: [[TMP20:%.*]] = getelementptr inbounds i32, ptr [[DST]], i64 [[EVL_BASED_IV]]
; IF-EVL-NEXT: call void @llvm.vp.store.nxv4i32.p0(<vscale x 4 x i32> [[PREDPHI9]], ptr align 4 [[TMP20]], <vscale x 4 x i1> splat (i1 true), i32 [[TMP7]])
; IF-EVL-NEXT: [[TMP21:%.*]] = zext i32 [[TMP7]] to i64
diff --git a/llvm/test/Transforms/LoopVectorize/VPlan/predicator.ll b/llvm/test/Transforms/LoopVectorize/VPlan/predicator.ll
index cd3f9b23e5e7f..063744fe2d5e8 100644
--- a/llvm/test/Transforms/LoopVectorize/VPlan/predicator.ll
+++ b/llvm/test/Transforms/LoopVectorize/VPlan/predicator.ll
@@ -1319,8 +1319,7 @@ define void @uniform_branch_after_varying_branch(ptr %a, i1 %u1) {
; CHECK-EMPTY:
; CHECK-NEXT: bb3:
; CHECK-NEXT: EMIT-SCALAR vp<[[VP5:%[0-9]+]]> = phi [ ir<poison>, bb1 ], [ ir<%add2>, bb2 ]
-; CHECK-NEXT: EMIT-SCALAR vp<[[VP6:%[0-9]+]]> = phi [ ir<false>, bb1 ], [ vp<[[VP4]]>, bb2 ]
-; CHECK-NEXT: BLEND ir<%phi3> = ir<%add1>/vp<[[VP4]]> vp<%5>/vp<[[VP6]]>
+; CHECK-NEXT: BLEND ir<%phi3> = ir<%add1>/ir<true> vp<%5>/ir<true>
; CHECK-NEXT: EMIT ir<%add3> = add ir<%phi3>, ir<3>, vp<[[VP4]]>
; CHECK-NEXT: Successor(s): bb4
; CHECK-EMPTY:
@@ -1490,10 +1489,8 @@ define void @uniform_branch_after_varying_branch_more_blocks(ptr %a, i1 %u1) {
; CHECK-EMPTY:
; CHECK-NEXT: bb4:
; CHECK-NEXT: EMIT-SCALAR vp<[[VP5:%[0-9]+]]> = phi [ ir<%add2>, bb2 ], [ ir<poison>, bb3 ]
-; CHECK-NEXT: EMIT-SCALAR vp<[[VP6:%[0-9]+]]> = phi [ vp<[[VP4]]>, bb2 ], [ ir<false>, bb3 ]
-; CHECK-NEXT: EMIT-SCALAR vp<[[VP7:%[0-9]+]]> = phi [ ir<poison>, bb2 ], [ ir<%add3>, bb3 ]
-; CHECK-NEXT: EMIT-SCALAR vp<[[VP8:%[0-9]+]]> = phi [ ir<false>, bb2 ], [ vp<[[VP4]]>, bb3 ]
-; CHECK-NEXT: BLEND ir<%phi4> = vp<%5>/vp<[[VP6]]> vp<%7>/vp<[[VP8]]>
+; CHECK-NEXT: EMIT-SCALAR vp<[[VP6:%[0-9]+]]> = phi [ ir<poison>, bb2 ], [ ir<%add3>, bb3 ]
+; CHECK-NEXT: BLEND ir<%phi4> = vp<%5>/ir<true> vp<%6>/ir<true>
; CHECK-NEXT: EMIT ir<%add4> = add ir<%phi4>, ir<4>, vp<[[VP4]]>
; CHECK-NEXT: Successor(s): bb5
; CHECK-EMPTY:
@@ -1684,10 +1681,8 @@ define void @uniform_branch_after_varying_branch_more_blocks_mirrored(ptr %a, i1
; CHECK-EMPTY:
; CHECK-NEXT: bb5:
; CHECK-NEXT: EMIT-SCALAR vp<[[VP5:%[0-9]+]]> = phi [ ir<%add3>, bb3 ], [ ir<poison>, bb4 ]
-; CHECK-NEXT: EMIT-SCALAR vp<[[VP6:%[0-9]+]]> = phi [ ir<%v0>, bb3 ], [ ir<false>, bb4 ]
-; CHECK-NEXT: EMIT-SCALAR vp<[[VP7:%[0-9]+]]> = phi [ ir<poison>, bb3 ], [ ir<%add4>, bb4 ]
-; CHECK-NEXT: EMIT-SCALAR vp<[[VP8:%[0-9]+]]> = phi [ ir<false>, bb3 ], [ ir<%v0>, bb4 ]
-; CHECK-NEXT: BLEND ir<%phi5> = vp<%5>/vp<[[VP6]]> vp<%7>/vp<[[VP8]]>
+; CHECK-NEXT: EMIT-SCALAR vp<[[VP6:%[0-9]+]]> = phi [ ir<poison>, bb3 ], [ ir<%add4>, bb4 ]
+; CHECK-NEXT: BLEND ir<%phi5> = vp<%5>/ir<true> vp<%6>/ir<true>
; CHECK-NEXT: EMIT ir<%add5> = add ir<%phi5>, ir<5>, ir<%v0>
; CHECK-NEXT: Successor(s): bb6
; CHECK-EMPTY:
diff --git a/llvm/test/Transforms/LoopVectorize/constant-fold-commutative-and.ll b/llvm/test/Transforms/LoopVectorize/constant-fold-commutative-and.ll
index 94230d4061977..2f079172d5ba6 100644
--- a/llvm/test/Transforms/LoopVectorize/constant-fold-commutative-and.ll
+++ b/llvm/test/Transforms/LoopVectorize/constant-fold-commutative-and.ll
@@ -18,20 +18,15 @@ define void @constant_fold_commutative_and(ptr %ptr.n, ptr noalias %p, i1 %cond)
; CHECK-NEXT: [[TMP1:%.*]] = icmp ule <2 x i8> [[VEC_IND]], splat (i8 16)
; CHECK-NEXT: [[TMP6:%.*]] = extractelement <2 x i1> [[TMP1]], i64 0
; CHECK-NEXT: [[TMP2:%.*]] = load i64, ptr [[PTR_N]], align 4
-; CHECK-NEXT: [[BROADCAST_SPLATINSERT1:%.*]] = insertelement <2 x i64> poison, i64 [[TMP2]], i64 0
-; CHECK-NEXT: [[BROADCAST_SPLAT2:%.*]] = shufflevector <2 x i64> [[BROADCAST_SPLATINSERT1]], <2 x i64> poison, <2 x i32> zeroinitializer
; CHECK-NEXT: br i1 [[COND]], label %[[LATCH2:.*]], label %[[PRED_11:.*]]
; CHECK: [[PRED_11]]:
-; CHECK-NEXT: [[TMP3:%.*]] = icmp uge <2 x i64> [[BROADCAST_SPLAT2]], splat (i64 4)
-; CHECK-NEXT: [[TMP4:%.*]] = select <2 x i1> [[TMP1]], <2 x i1> [[TMP3]], <2 x i1> zeroinitializer
-; CHECK-NEXT: [[TMP5:%.*]] = extractelement <2 x i1> [[TMP4]], i64 0
-; CHECK-NEXT: [[TMP12:%.*]] = icmp ult i64 [[TMP2]], 7
+; CHECK-NEXT: [[TMP3:%.*]] = icmp uge i64 [[TMP2]], 4
+; CHECK-NEXT: [[TMP4:%.*]] = icmp ult i64 [[TMP2]], 7
; CHECK-NEXT: br label %[[LATCH2]]
; CHECK: [[LATCH2]]:
-; CHECK-NEXT: [[TMP7:%.*]] = phi i1 [ false, %[[VECTOR_BODY]] ], [ [[TMP6]], %[[PRED_11]] ]
-; CHECK-NEXT: [[TMP13:%.*]] = phi i1 [ poison, %[[VECTOR_BODY]] ], [ [[TMP12]], %[[PRED_11]] ]
-; CHECK-NEXT: [[TMP14:%.*]] = phi i1 [ false, %[[VECTOR_BODY]] ], [ [[TMP5]], %[[PRED_11]] ]
-; CHECK-NEXT: [[TMP9:%.*]] = select i1 [[TMP14]], i1 [[TMP13]], i1 [[TMP7]]
+; CHECK-NEXT: [[TMP5:%.*]] = phi i1 [ poison, %[[VECTOR_BODY]] ], [ [[TMP4]], %[[PRED_11]] ]
+; CHECK-NEXT: [[TMP7:%.*]] = phi i1 [ false, %[[VECTOR_BODY]] ], [ [[TMP3]], %[[PRED_11]] ]
+; CHECK-NEXT: [[TMP9:%.*]] = select i1 [[TMP7]], i1 [[TMP5]], i1 true
; CHECK-NEXT: br i1 [[TMP6]], label %[[PRED_STORE_IF:.*]], label %[[PRED_STORE_CONTINUE:.*]]
; CHECK: [[PRED_STORE_IF]]:
; CHECK-NEXT: [[TMP8:%.*]] = getelementptr i8, ptr [[P]], i64 [[INDEX]]
diff --git a/llvm/test/Transforms/LoopVectorize/predicatedinst-loop-invariant.ll b/llvm/test/Transforms/LoopVectorize/predicatedinst-loop-invariant.ll
index 3292b2776ab02..e6904dabbcb6a 100644
--- a/llvm/test/Transforms/LoopVectorize/predicatedinst-loop-invariant.ll
+++ b/llvm/test/Transforms/LoopVectorize/predicatedinst-loop-invariant.ll
@@ -238,26 +238,24 @@ define void @test_store_to_invariant_address_needs_mask_due_to_low_trip_count(pt
; CHECK: [[VECTOR_BODY]]:
; CHECK-NEXT: br label %[[MIDDLE_BLOCK:.*]]
; CHECK: [[MIDDLE_BLOCK]]:
-; CHECK-NEXT: [[TMP0:%.*]] = phi i1 [ false, %[[VECTOR_BODY]] ]
-; CHECK-NEXT: [[PREDPHI:%.*]] = select i1 [[TMP0]], i32 0, i32 1
; CHECK-NEXT: br i1 true, label %[[PRED_STORE_IF:.*]], label %[[EXIT:.*]]
; CHECK: [[PRED_STORE_IF]]:
-; CHECK-NEXT: store i32 [[PREDPHI]], ptr [[DST]], align 4
+; CHECK-NEXT: store i32 0, ptr [[DST]], align 4
; CHECK-NEXT: br label %[[EXIT]]
; CHECK: [[EXIT]]:
; CHECK-NEXT: br i1 true, label %[[PRED_STORE_IF2:.*]], label %[[PRED_STORE_CONTINUE3:.*]]
; CHECK: [[PRED_STORE_IF2]]:
-; CHECK-NEXT: store i32 [[PREDPHI]], ptr [[DST]], align 4
+; CHECK-NEXT: store i32 0, ptr [[DST]], align 4
; CHECK-NEXT: br label %[[PRED_STORE_CONTINUE3]]
; CHECK: [[PRED_STORE_CONTINUE3]]:
; CHECK-NEXT: br i1 true, label %[[PRED_STORE_IF4:.*]], label %[[PRED_STORE_CONTINUE5:.*]]
; CHECK: [[PRED_STORE_IF4]]:
-; CHECK-NEXT: store i32 [[PREDPHI]], ptr [[DST]], align 4
+; CHECK-NEXT: store i32 0, ptr [[DST]], align 4
; CHECK-NEXT: br label %[[PRED_STORE_CONTINUE5]]
; CHECK: [[PRED_STORE_CONTINUE5]]:
; CHECK-NEXT: br i1 false, label %[[PRED_STORE_IF6:.*]], label %[[PRED_STORE_CONTINUE7:.*]]
; CHECK: [[PRED_STORE_IF6]]:
-; CHECK-NEXT: store i32 [[PREDPHI]], ptr [[DST]], align 4
+; CHECK-NEXT: store i32 0, ptr [[DST]], align 4
; CHECK-NEXT: br label %[[PRED_STORE_CONTINUE7]]
; CHECK: [[PRED_STORE_CONTINUE7]]:
; CHECK-NEXT: br label %[[MIDDLE_BLOCK1:.*]]
diff --git a/llvm/test/Transforms/LoopVectorize/predicator.ll b/llvm/test/Transforms/LoopVectorize/predicator.ll
index 0f5761e2c46f1..9dd3b4e47796c 100644
--- a/llvm/test/Transforms/LoopVectorize/predicator.ll
+++ b/llvm/test/Transforms/LoopVectorize/predicator.ll
@@ -633,19 +633,16 @@ define void @uniform_branch_after_varying_branch(ptr %a, i1 %u1) {
; CHECK-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[BB32:.*]] ]
; CHECK-NEXT: [[VEC_IND:%.*]] = phi <4 x i64> [ <i64 0, i64 1, i64 2, i64 3>, %[[VECTOR_PH]] ], [ [[VEC_IND_NEXT:%.*]], %[[BB32]] ]
; CHECK-NEXT: [[TMP1:%.*]] = icmp sle <4 x i64> [[VEC_IND]], zeroinitializer
-; CHECK-NEXT: [[TMP3:%.*]] = extractelement <4 x i1> [[TMP1]], i64 0
-; CHECK-NEXT: [[TMP7:%.*]] = xor i1 [[TMP3]], true
; CHECK-NEXT: [[TMP2:%.*]] = add <4 x i64> [[VEC_IND]], splat (i64 1)
; CHECK-NEXT: br i1 [[U1]], label %[[BB21:.*]], label %[[BB32]]
; CHECK: [[BB21]]:
; CHECK-NEXT: [[TMP8:%.*]] = add i64 [[INDEX]], 2
; CHECK-NEXT: br label %[[BB32]]
; CHECK: [[BB32]]:
-; CHECK-NEXT: [[TMP9:%.*]] = phi i64 [ poison, %[[VECTOR_BODY]] ], [ [[TMP8]], %[[BB21]] ]
-; CHECK-NEXT: [[TMP11:%.*]] = phi i1 [ false, %[[VECTOR_BODY]] ], [ [[TMP7]], %[[BB21]] ]
-; CHECK-NEXT: [[BROADCAST_SPLATINSERT:%.*]] = insertelement <4 x i64> poison, i64 [[TMP9]], i64 0
+; CHECK-NEXT: [[TMP3:%.*]] = phi i64 [ poison, %[[VECTOR_BODY]] ], [ [[TMP8]], %[[BB21]] ]
+; CHECK-NEXT: [[BROADCAST_SPLATINSERT:%.*]] = insertelement <4 x i64> poison, i64 [[TMP3]], i64 0
; CHECK-NEXT: [[BROADCAST_SPLAT:%.*]] = shufflevector <4 x i64> [[BROADCAST_SPLATINSERT]], <4 x i64> poison, <4 x i32> zeroinitializer
-; CHECK-NEXT: [[PREDPHI:%.*]] = select i1 [[TMP11]], <4 x i64> [[BROADCAST_SPLAT]], <4 x i64> [[TMP2]]
+; CHECK-NEXT: [[PREDPHI:%.*]] = select i1 true, <4 x i64> [[BROADCAST_SPLAT]], <4 x i64> [[TMP2]]
; CHECK-NEXT: [[TMP4:%.*]] = add <4 x i64> [[PREDPHI]], splat (i64 3)
; CHECK-NEXT: [[TMP5:%.*]] = add <4 x i64> [[VEC_IND]], splat (i64 4)
; CHECK-NEXT: [[PREDPHI1:%.*]] = select <4 x i1> [[TMP1]], <4 x i64> [[TMP5]], <4 x i64> [[TMP4]]
@@ -653,8 +650,8 @@ define void @uniform_branch_after_varying_branch(ptr %a, i1 %u1) {
; CHECK-NEXT: store i64 [[TMP6]], ptr [[A]], align 4
; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 4
; CHECK-NEXT: [[VEC_IND_NEXT]] = add nuw nsw <4 x i64> [[VEC_IND]], splat (i64 4)
-; CHECK-NEXT: [[TMP10:%.*]] = icmp eq i64 [[INDEX_NEXT]], 128
-; CHECK-NEXT: br i1 [[TMP10]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP10:![0-9]+]]
+; CHECK-NEXT: [[TMP7:%.*]] = icmp eq i64 [[INDEX_NEXT]], 128
+; CHECK-NEXT: br i1 [[TMP7]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP10:![0-9]+]]
; CHECK: [[MIDDLE_BLOCK]]:
; CHECK-NEXT: br label %[[EXIT:.*]]
; CHECK: [[EXIT]]:
@@ -793,22 +790,19 @@ define void @uniform_branch_after_varying_branch_more_blocks(ptr %a, i1 %u1) {
; CHECK-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[BB43:.*]] ]
; CHECK-NEXT: [[VEC_IND:%.*]] = phi <4 x i64> [ <i64 0, i64 1, i64 2, i64 3>, %[[VECTOR_PH]] ], [ [[VEC_IND_NEXT:%.*]], %[[BB43]] ]
; CHECK-NEXT: [[TMP1:%.*]] = icmp sle <4 x i64> [[VEC_IND]], zeroinitializer
-; CHECK-NEXT: [[TMP9:%.*]] = extractelement <4 x i1> [[TMP1]], i64 0
-; CHECK-NEXT: [[TMP2:%.*]] = xor i1 [[TMP9]], true
; CHECK-NEXT: br i1 [[U1]], label %[[BB32:.*]], label %[[BB21:.*]]
; CHECK: [[BB21]]:
-; CHECK-NEXT: [[TMP3:%.*]] = add i64 [[INDEX]], 2
+; CHECK-NEXT: [[TMP7:%.*]] = add i64 [[INDEX]], 2
; CHECK-NEXT: br label %[[BB43]]
; CHECK: [[BB32]]:
-; CHECK-NEXT: [[TMP10:%.*]] = add i64 [[INDEX]], 3
+; CHECK-NEXT: [[TMP2:%.*]] = add i64 [[INDEX]], 3
; CHECK-NEXT: br label %[[BB43]]
; CHECK: [[BB43]]:
-; CHECK-NEXT: [[TMP12:%.*]] = phi i64 [ [[TMP3]], %[[BB21]] ], [ poison, %[[BB32]] ]
-; CHECK-NEXT: [[TMP13:%.*]] = phi i64 [ poison, %[[BB21]] ], [ [[TMP10]], %[[BB32]] ]
-; CHECK-NEXT: [[TMP7:%.*]] = phi i1 [ false, %[[BB21]] ], [ [[TMP2]], %[[BB32]] ]
-; CHECK-NEXT: [[PREDPHI:%.*]] = select i1 [[TMP7]], i64 [[TMP13]], i64 [[TMP12]]
-; CHECK-NEXT: [[TMP8:%.*]] = add i64 [[PREDPHI]], 4
-; CHECK-NEXT: [[BROADCAST_SPLATINSERT:%.*]] = insertelement <4 x i64> poison, i64 [[TMP8]], i64 0
+; CHECK-NEXT: [[TMP3:%.*]] = phi i64 [ [[TMP7]], %[[BB21]] ], [ poison, %[[BB32]] ]
+; CHECK-NEXT: [[TMP9:%.*]] = phi i64 [ poison, %[[BB21]] ], [ [[TMP2]], %[[BB32]] ]
+; CHECK-NEXT: [[PREDPHI:%.*]] = select i1 true, i64 [[TMP9]], i64 [[TMP3]]
+; CHECK-NEXT: [[TMP10:%.*]] = add i64 [[PREDPHI]], 4
+; CHECK-NEXT: [[BROADCAST_SPLATINSERT:%.*]] = insertelement <4 x i64> poison, i64 [[TMP10]], i64 0
; CHECK-NEXT: [[TMP4:%.*]] = shufflevector <4 x i64> [[BROADCAST_SPLATINSERT]], <4 x i64> poison, <4 x i32> zeroinitializer
; CHECK-NEXT: [[TMP5:%.*]] = add <4 x i64> [[VEC_IND]], splat (i64 5)
; CHECK-NEXT: [[PREDPHI1:%.*]] = select <4 x i1> [[TMP1]], <4 x i64> [[TMP5]], <4 x i64> [[TMP4]]
@@ -816,8 +810,8 @@ define void @uniform_branch_after_varying_branch_more_blocks(ptr %a, i1 %u1) {
; CHECK-NEXT: store i64 [[TMP6]], ptr [[A]], align 4
; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 4
; CHECK-NEXT: [[VEC_IND_NEXT]] = add nuw nsw <4 x i64> [[VEC_IND]], splat (i64 4)
-; CHECK-NEXT: [[TMP11:%.*]] = icmp eq i64 [[INDEX_NEXT]], 128
-; CHECK-NEXT: br i1 [[TMP11]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP12:![0-9]+]]
+; CHECK-NEXT: [[TMP8:%.*]] = icmp eq i64 [[INDEX_NEXT]], 128
+; CHECK-NEXT: br i1 [[TMP8]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP12:![0-9]+]]
; CHECK: [[MIDDLE_BLOCK]]:
; CHECK-NEXT: br label %[[EXIT:.*]]
; CHECK: [[EXIT]]:
@@ -968,30 +962,28 @@ define void @uniform_branch_after_varying_branch_more_blocks_mirrored(ptr %a, i1
; CHECK-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[BB53:.*]] ]
; CHECK-NEXT: [[VEC_IND:%.*]] = phi <4 x i64> [ <i64 0, i64 1, i64 2, i64 3>, %[[VECTOR_PH]] ], [ [[VEC_IND_NEXT:%.*]], %[[BB53]] ]
; CHECK-NEXT: [[TMP1:%.*]] = icmp sle <4 x i64> [[VEC_IND]], zeroinitializer
-; CHECK-NEXT: [[TMP9:%.*]] = extractelement <4 x i1> [[TMP1]], i64 0
; CHECK-NEXT: [[TMP2:%.*]] = add <4 x i64> [[VEC_IND]], splat (i64 1)
; CHECK-NEXT: br i1 [[U2]], label %[[BB42:.*]], label %[[BB31:.*]]
; CHECK: [[BB31]]:
-; CHECK-NEXT: [[TMP3:%.*]] = add i64 [[INDEX]], 3
+; CHECK-NEXT: [[TMP7:%.*]] = add i64 [[INDEX]], 3
; CHECK-NEXT: br label %[[BB53]]
; CHECK: [[BB42]]:
-; CHECK-NEXT: [[TMP4:%.*]] = add i64 [[INDEX]], 4
+; CHECK-NEXT: [[TMP3:%.*]] = add i64 [[INDEX]], 4
; CHECK-NEXT: br label %[[BB53]]
; CHECK: [[BB53]]:
-; CHECK-NEXT: [[TMP11:%.*]] = phi i64 [ [[TMP3]], %[[BB31]] ], [ poison, %[[BB42]] ]
-; CHECK-NEXT: [[TMP12:%.*]] = phi i64 [ poison, %[[BB31]] ], [ [[TMP4]], %[[BB42]] ]
-; CHECK-NEXT: [[TMP7:%.*]] = phi i1 [ false, %[[BB31]] ], [ [[TMP9]], %[[BB42]] ]
-; CHECK-NEXT: [[PREDPHI:%.*]] = select i1 [[TMP7]], i64 [[TMP12]], i64 [[TMP11]]
-; CHECK-NEXT: [[TMP8:%.*]] = add i64 [[PREDPHI]], 5
-; CHECK-NEXT: [[BROADCAST_SPLATINSERT:%.*]] = insertelement <4 x i64> poison, i64 [[TMP8]], i64 0
+; CHECK-NEXT: [[TMP4:%.*]] = phi i64 [ [[TMP7]], %[[BB31]] ], [ poison, %[[BB42]] ]
+; CHECK-NEXT: [[TMP9:%.*]] = phi i64 [ poison, %[[BB31]] ], [ [[TMP3]], %[[BB42]] ]
+; CHECK-NEXT: [[PREDPHI:%.*]] = select i1 true, i64 [[TMP9]], i64 [[TMP4]]
+; CHECK-NEXT: [[TMP10:%.*]] = add i64 [[PREDPHI]], 5
+; CHECK-NEXT: [[BROADCAST_SPLATINSERT:%.*]] = insertelement <4 x i64> poison, i64 [[TMP10]], i64 0
; CHECK-NEXT: [[TMP5:%.*]] = shufflevector <4 x i64> [[BROADCAST_SPLATINSERT]], <4 x i64> poison, <4 x i32> zeroinitializer
; CHECK-NEXT: [[PREDPHI1:%.*]] = select <4 x i1> [[TMP1]], <4 x i64> [[TMP5]], <4 x i64> [[TMP2]]
; CHECK-NEXT: [[TMP6:%.*]] = extractelement <4 x i64> [[PREDPHI1]], i64 3
; CHECK-NEXT: store i64 [[TMP6]], ptr [[A]], align 4
; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 4
; CHECK-NEXT: [[VEC_IND_NEXT]] = add nuw nsw <4 x i64> [[VEC_IND]], splat (i64 4)
-; CHECK-NEXT: [[TMP10:%.*]] = icmp eq i64 [[INDEX_NEXT]], 128
-; CHECK-NEXT: br i1 [[TMP10]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP14:![0-9]+]]
+; CHECK-NEXT: [[TMP8:%.*]] = icmp eq i64 [[INDEX_NEXT]], 128
+; CHECK-NEXT: br i1 [[TMP8]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP14:![0-9]+]]
; CHECK: [[MIDDLE_BLOCK]]:
; CHECK-NEXT: br label %[[EXIT:.*]]
; CHECK: [[EXIT]]:
>From 47adb4f1268b6137be1a7030d6300d79ddee620b Mon Sep 17 00:00:00 2001
From: Andrei Elovikov <andrei.elovikov at sifive.com>
Date: Fri, 28 Aug 2026 14:49:59 -0700
Subject: [PATCH 08/11] Fix bug in previous commit
---
.../Transforms/Vectorize/VPlanPredicator.cpp | 92 ++++++++++++++++---
.../Transforms/LoopVectorize/RISCV/divrem.ll | 11 ++-
.../RISCV/gather-scatter-cost.ll | 13 +--
.../RISCV/tail-folding-complex-mask.ll | 13 +--
.../AArch64/vplan-memory-op-decisions.ll | 44 +++++----
.../LoopVectorize/VPlan/predicator.ll | 15 ++-
.../constant-fold-commutative-and.ll | 5 +-
.../predicatedinst-loop-invariant.ll | 10 +-
.../Transforms/LoopVectorize/predicator.ll | 41 +++++----
9 files changed, 167 insertions(+), 77 deletions(-)
diff --git a/llvm/lib/Transforms/Vectorize/VPlanPredicator.cpp b/llvm/lib/Transforms/Vectorize/VPlanPredicator.cpp
index 13ac3cce79c46..ca935f4694cf8 100644
--- a/llvm/lib/Transforms/Vectorize/VPlanPredicator.cpp
+++ b/llvm/lib/Transforms/Vectorize/VPlanPredicator.cpp
@@ -15,6 +15,7 @@
#include "VPlan.h"
#include "VPlanCFG.h"
#include "VPlanDominatorTree.h"
+#include "VPlanHelpers.h"
#include "VPlanPatternMatch.h"
#include "VPlanTransforms.h"
#include "VPlanUtils.h"
@@ -576,8 +577,35 @@ void VPPredicator::convertPhisToBlends(VPBasicBlock *VPBB) {
auto &BlockBlendTerms = BlendTerms[BlocksInCompactRPOTOrder.getIndex(VPBB)];
auto Terms = BlockBlendTerms.find(PhiR);
assert(Terms != BlockBlendTerms.end() && "Missing cached blend terms");
- for (auto [_, ConstMaskBlock] : Terms->second)
- MaskBlocks.push_back(const_cast<VPBasicBlock *>(ConstMaskBlock));
+ for (auto [_, MaskBlock] : Terms->second)
+ MaskBlocks.push_back(MaskBlock);
+
+ LLVM_DEBUG({
+ VPSlotTracker SlotTracker(&Plan);
+ auto PrintValue = [&](VPValue *V) {
+ if (V)
+ V->printAsOperand(dbgs(), SlotTracker);
+ else
+ dbgs() << "<none>";
+ };
+ dbgs() << " blend input terms:\n";
+ for (auto [V, IncomingBlock] : PhiR->incoming_values_and_blocks()) {
+ auto *IncomingBB = cast<VPBasicBlock>(IncomingBlock);
+ dbgs() << " " << IncomingBB->getName() << ": value ";
+ PrintValue(V);
+ dbgs() << ", in-mask ";
+ PrintValue(getBlockInMask(IncomingBB));
+ dbgs() << "\n";
+ }
+ dbgs() << " selected blend terms:\n";
+ for (auto [V, MaskBlock] : Terms->second) {
+ dbgs() << " " << MaskBlock->getName() << ": value ";
+ PrintValue(V);
+ dbgs() << ", in-mask ";
+ PrintValue(getBlockInMask(MaskBlock));
+ dbgs() << "\n";
+ }
+ });
// The in-mask of the common dominator is true on all paths from an
// incoming block to the phi. Remove it before reconstructing the masks,
@@ -586,28 +614,68 @@ void VPPredicator::convertPhisToBlends(VPBasicBlock *VPBB) {
cast<VPBasicBlock>(VPDT.findNearestCommonDominator(
make_range(MaskBlocks.begin(), MaskBlocks.end())));
VPValue *CommonIncomingMask = getBlockInMask(CommonIncomingDom);
+ LLVM_DEBUG({
+ VPSlotTracker SlotTracker(&Plan);
+ dbgs() << " common incoming dominator " << CommonIncomingDom->getName()
+ << ", in-mask ";
+ if (CommonIncomingMask)
+ CommonIncomingMask->printAsOperand(dbgs(), SlotTracker);
+ else
+ dbgs() << "<none>";
+ dbgs() << "\n";
+ });
- for (auto [V, ConstMaskBlock] : Terms->second) {
- auto *MaskBlock = const_cast<VPBasicBlock *>(ConstMaskBlock);
+ for (auto [V, MaskBlock] : Terms->second) {
+ VPValue *OriginalV = V;
+ VPValue *Mask = getBlockInMask(MaskBlock);
// Reconstruct values at the blend location rather than fixing up the
// blend after it has been created.
- V = reconstructSSA(VPBB, V, /*IsMask=*/false);
+ V = reconstructSSA(VPBB, OriginalV, /*IsMask=*/false);
- VPValue *Mask = getBlockInMask(MaskBlock);
VPValue *RemainingMask = nullptr;
bool RemovedCommonMask =
CommonIncomingMask && Mask &&
match(Mask, m_RemoveMask(CommonIncomingMask, RemainingMask));
- OperandsWithMask.append(
- {V, RemovedCommonMask && !RemainingMask
- ? Plan.getTrue()
- : reconstructBlendMask(VPBB, MaskBlock,
- RemovedCommonMask ? RemainingMask
- : Mask)});
+ if (RemovedCommonMask) {
+ Mask = RemainingMask ? RemainingMask : Plan.getTrue();
+ }
+
+ VPValue *BlendMask = reconstructBlendMask(VPBB, MaskBlock, Mask);
+ LLVM_DEBUG({
+ VPSlotTracker SlotTracker(&Plan);
+ auto PrintValue = [&](VPValue *VPV) {
+ if (VPV)
+ VPV->printAsOperand(dbgs(), SlotTracker);
+ else
+ dbgs() << "<none>";
+ };
+ dbgs() << " blend term from " << MaskBlock->getName() << ": value ";
+ PrintValue(OriginalV);
+ dbgs() << " -> ";
+ PrintValue(V);
+ dbgs() << ", mask ";
+ PrintValue(Mask);
+ if (RemovedCommonMask) {
+ dbgs() << " -> ";
+ PrintValue(RemainingMask ? RemainingMask : Plan.getTrue());
+ dbgs() << " after removing the common in-mask";
+ }
+ dbgs() << " -> ";
+ PrintValue(BlendMask);
+ dbgs() << " at " << VPBB->getName() << "\n";
+ });
+ OperandsWithMask.append({V, BlendMask});
}
PHINode *IRPhi = cast_or_null<PHINode>(PhiR->getUnderlyingValue());
+ LLVM_DEBUG({
+ dbgs() << " creating blend with " << Terms->second.size() << " term(s)";
+ if (Terms->second.size() != PhiR->getNumIncoming())
+ dbgs() << " after coalescing " << PhiR->getNumIncoming()
+ << " incoming value(s)";
+ dbgs() << "\n";
+ });
auto *Blend =
new VPBlendRecipe(IRPhi, OperandsWithMask, *PhiR, PhiR->getDebugLoc());
Builder.insert(Blend);
diff --git a/llvm/test/Transforms/LoopVectorize/RISCV/divrem.ll b/llvm/test/Transforms/LoopVectorize/RISCV/divrem.ll
index 19bab9853c48f..c948ecc51f36c 100644
--- a/llvm/test/Transforms/LoopVectorize/RISCV/divrem.ll
+++ b/llvm/test/Transforms/LoopVectorize/RISCV/divrem.ll
@@ -651,10 +651,10 @@ define void @udiv_sdiv_with_invariant_divisors(i8 %x, i16 %y, i1 %c, ptr %p) {
; CHECK-NEXT: [[BROADCAST_SPLAT8:%.*]] = shufflevector <vscale x 4 x i8> [[BROADCAST_SPLATINSERT7]], <vscale x 4 x i8> poison, <vscale x 4 x i32> zeroinitializer
; CHECK-NEXT: br i1 [[C:%.*]], label [[LOOP_LATCH8]], label [[THEN7:%.*]]
; CHECK: then7:
-; CHECK-NEXT: [[TMP13:%.*]] = call <vscale x 4 x i8> @llvm.vp.merge.nxv4i8(<vscale x 4 x i1> splat (i1 true), <vscale x 4 x i8> [[BROADCAST_SPLAT2]], <vscale x 4 x i8> splat (i8 1), i32 [[TMP2]])
-; CHECK-NEXT: [[TMP14:%.*]] = extractelement <vscale x 4 x i8> [[VEC_IND]], i64 0
-; CHECK-NEXT: [[TMP5:%.*]] = extractelement <vscale x 4 x i8> [[TMP13]], i64 0
-; CHECK-NEXT: [[TMP6:%.*]] = udiv i8 [[TMP14]], [[TMP5]]
+; CHECK-NEXT: [[TMP14:%.*]] = call <vscale x 4 x i8> @llvm.vp.merge.nxv4i8(<vscale x 4 x i1> splat (i1 true), <vscale x 4 x i8> [[BROADCAST_SPLAT2]], <vscale x 4 x i8> splat (i8 1), i32 [[TMP2]])
+; CHECK-NEXT: [[TMP15:%.*]] = extractelement <vscale x 4 x i8> [[VEC_IND]], i64 0
+; CHECK-NEXT: [[TMP5:%.*]] = extractelement <vscale x 4 x i8> [[TMP14]], i64 0
+; CHECK-NEXT: [[TMP6:%.*]] = udiv i8 [[TMP15]], [[TMP5]]
; CHECK-NEXT: [[TMP7:%.*]] = zext i8 [[TMP6]] to i16
; CHECK-NEXT: [[TMP8:%.*]] = call <vscale x 4 x i16> @llvm.vp.merge.nxv4i16(<vscale x 4 x i1> splat (i1 true), <vscale x 4 x i16> [[BROADCAST_SPLAT4]], <vscale x 4 x i16> splat (i16 1), i32 [[TMP2]])
; CHECK-NEXT: [[TMP9:%.*]] = extractelement <vscale x 4 x i16> [[TMP8]], i64 0
@@ -663,9 +663,10 @@ define void @udiv_sdiv_with_invariant_divisors(i8 %x, i16 %y, i1 %c, ptr %p) {
; CHECK-NEXT: br label [[LOOP_LATCH8]]
; CHECK: loop.latch8:
; CHECK-NEXT: [[TMP12:%.*]] = phi i32 [ poison, [[LOOP_LATCH]] ], [ [[TMP11]], [[THEN7]] ]
+; CHECK-NEXT: [[TMP13:%.*]] = phi i1 [ false, [[LOOP_LATCH]] ], [ true, [[THEN7]] ]
; CHECK-NEXT: [[BROADCAST_SPLATINSERT9:%.*]] = insertelement <vscale x 4 x i32> poison, i32 [[TMP12]], i64 0
; CHECK-NEXT: [[BROADCAST_SPLAT10:%.*]] = shufflevector <vscale x 4 x i32> [[BROADCAST_SPLATINSERT9]], <vscale x 4 x i32> poison, <vscale x 4 x i32> zeroinitializer
-; CHECK-NEXT: [[PREDPHI:%.*]] = select i1 true, <vscale x 4 x i32> [[BROADCAST_SPLAT10]], <vscale x 4 x i32> zeroinitializer
+; CHECK-NEXT: [[PREDPHI:%.*]] = select i1 [[TMP13]], <vscale x 4 x i32> [[BROADCAST_SPLAT10]], <vscale x 4 x i32> zeroinitializer
; CHECK-NEXT: call void @llvm.vp.scatter.nxv4i32.nxv4p0(<vscale x 4 x i32> [[PREDPHI]], <vscale x 4 x ptr> align 4 [[BROADCAST_SPLAT6]], <vscale x 4 x i1> splat (i1 true), i32 [[TMP2]])
; CHECK-NEXT: [[AVL_NEXT]] = sub nuw i32 [[AVL]], [[TMP2]]
; CHECK-NEXT: [[VEC_IND_NEXT]] = add <vscale x 4 x i8> [[VEC_IND]], [[BROADCAST_SPLAT8]]
diff --git a/llvm/test/Transforms/LoopVectorize/RISCV/gather-scatter-cost.ll b/llvm/test/Transforms/LoopVectorize/RISCV/gather-scatter-cost.ll
index dbbc08638eda4..6230e6eb9d22d 100644
--- a/llvm/test/Transforms/LoopVectorize/RISCV/gather-scatter-cost.ll
+++ b/llvm/test/Transforms/LoopVectorize/RISCV/gather-scatter-cost.ll
@@ -36,18 +36,19 @@ define void @predicated_uniform_load(ptr %src, i32 %n, ptr %dst, i1 %cond) {
; CHECK: vector.body:
; CHECK-NEXT: [[AVL:%.*]] = phi i32 [ [[TMP3]], [[VECTOR_PH]] ], [ [[AVL_NEXT:%.*]], [[LOOP_LATCH6:%.*]] ]
; CHECK-NEXT: [[TMP10:%.*]] = call i32 @llvm.experimental.get.vector.length.i32(i32 [[AVL]], i32 4, i1 true)
-; CHECK-NEXT: br i1 [[COND:%.*]], label [[LOOP_THEN5:%.*]], label [[LOOP_ELSE4:%.*]]
+; CHECK-NEXT: br i1 [[COND1:%.*]], label [[LOOP_THEN5:%.*]], label [[LOOP_ELSE4:%.*]]
; CHECK: loop.else4:
-; CHECK-NEXT: [[WIDE_MASKED_GATHER:%.*]] = call <vscale x 4 x i32> @llvm.vp.gather.nxv4i32.nxv4p0(<vscale x 4 x ptr> align 4 [[BROADCAST_SPLAT]], <vscale x 4 x i1> splat (i1 true), i32 [[TMP10]]), !alias.scope [[META0:![0-9]+]]
-; CHECK-NEXT: [[TMP12:%.*]] = extractelement <vscale x 4 x i32> [[WIDE_MASKED_GATHER]], i64 0
+; CHECK-NEXT: [[WIDE_MASKED_GATHER1:%.*]] = call <vscale x 4 x i32> @llvm.vp.gather.nxv4i32.nxv4p0(<vscale x 4 x ptr> align 4 [[BROADCAST_SPLAT]], <vscale x 4 x i1> splat (i1 true), i32 [[TMP10]]), !alias.scope [[META0:![0-9]+]]
+; CHECK-NEXT: [[TMP12:%.*]] = extractelement <vscale x 4 x i32> [[WIDE_MASKED_GATHER1]], i64 0
; CHECK-NEXT: br label [[LOOP_LATCH6]]
; CHECK: loop.then5:
; CHECK-NEXT: br label [[LOOP_LATCH6]]
; CHECK: loop.latch6:
; CHECK-NEXT: [[TMP13:%.*]] = phi i32 [ [[TMP12]], [[LOOP_ELSE4]] ], [ poison, [[LOOP_THEN5]] ]
+; CHECK-NEXT: [[COND:%.*]] = phi i1 [ false, [[LOOP_ELSE4]] ], [ true, [[LOOP_THEN5]] ]
; CHECK-NEXT: [[BROADCAST_SPLATINSERT7:%.*]] = insertelement <vscale x 4 x i32> poison, i32 [[TMP13]], i64 0
-; CHECK-NEXT: [[BROADCAST_SPLAT8:%.*]] = shufflevector <vscale x 4 x i32> [[BROADCAST_SPLATINSERT7]], <vscale x 4 x i32> poison, <vscale x 4 x i32> zeroinitializer
-; CHECK-NEXT: [[PREDPHI:%.*]] = select i1 true, <vscale x 4 x i32> zeroinitializer, <vscale x 4 x i32> [[BROADCAST_SPLAT8]]
+; CHECK-NEXT: [[WIDE_MASKED_GATHER:%.*]] = shufflevector <vscale x 4 x i32> [[BROADCAST_SPLATINSERT7]], <vscale x 4 x i32> poison, <vscale x 4 x i32> zeroinitializer
+; CHECK-NEXT: [[PREDPHI:%.*]] = select i1 [[COND]], <vscale x 4 x i32> zeroinitializer, <vscale x 4 x i32> [[WIDE_MASKED_GATHER]]
; CHECK-NEXT: call void @llvm.vp.scatter.nxv4i32.nxv4p0(<vscale x 4 x i32> [[PREDPHI]], <vscale x 4 x ptr> align 4 [[BROADCAST_SPLAT4]], <vscale x 4 x i1> splat (i1 true), i32 [[TMP10]]), !alias.scope [[META3:![0-9]+]], !noalias [[META0]]
; CHECK-NEXT: [[AVL_NEXT]] = sub nuw i32 [[AVL]], [[TMP10]]
; CHECK-NEXT: [[TMP16:%.*]] = icmp eq i32 [[AVL_NEXT]], 0
@@ -58,7 +59,7 @@ define void @predicated_uniform_load(ptr %src, i32 %n, ptr %dst, i1 %cond) {
; CHECK-NEXT: br label [[LOOP:%.*]]
; CHECK: loop:
; CHECK-NEXT: [[IV:%.*]] = phi i32 [ 0, [[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], [[LOOP_LATCH:%.*]] ]
-; CHECK-NEXT: br i1 [[COND]], label [[LOOP_THEN:%.*]], label [[LOOP_ELSE:%.*]]
+; CHECK-NEXT: br i1 [[COND1]], label [[LOOP_THEN:%.*]], label [[LOOP_ELSE:%.*]]
; CHECK: loop.then:
; CHECK-NEXT: br label [[LOOP_LATCH]]
; CHECK: loop.else:
diff --git a/llvm/test/Transforms/LoopVectorize/RISCV/tail-folding-complex-mask.ll b/llvm/test/Transforms/LoopVectorize/RISCV/tail-folding-complex-mask.ll
index fd2eccb482255..e8506999d070d 100644
--- a/llvm/test/Transforms/LoopVectorize/RISCV/tail-folding-complex-mask.ll
+++ b/llvm/test/Transforms/LoopVectorize/RISCV/tail-folding-complex-mask.ll
@@ -36,16 +36,17 @@ define void @test(i64 %n, ptr noalias %src0, ptr noalias %src1, ptr noalias %src
; IF-EVL-NEXT: [[TMP9:%.*]] = extractelement <vscale x 4 x i32> [[PREDPHI8]], i64 0
; IF-EVL-NEXT: br i1 [[C3]], label %[[LOAD_V25:.*]], label %[[LATCH7]]
; IF-EVL: [[LOAD_V25]]:
-; IF-EVL-NEXT: [[TMP14:%.*]] = getelementptr inbounds i32, ptr [[SRC2]], i64 [[EVL_BASED_IV]]
-; IF-EVL-NEXT: [[VP_OP_LOAD6:%.*]] = call <vscale x 4 x i32> @llvm.vp.load.nxv4i32.p0(ptr align 4 [[TMP14]], <vscale x 4 x i1> splat (i1 true), i32 [[TMP7]])
-; IF-EVL-NEXT: [[TMP15:%.*]] = extractelement <vscale x 4 x i32> [[VP_OP_LOAD6]], i64 0
-; IF-EVL-NEXT: [[TMP16:%.*]] = add i32 [[TMP15]], [[TMP9]]
+; IF-EVL-NEXT: [[TMP15:%.*]] = getelementptr inbounds i32, ptr [[SRC2]], i64 [[EVL_BASED_IV]]
+; IF-EVL-NEXT: [[VP_OP_LOAD6:%.*]] = call <vscale x 4 x i32> @llvm.vp.load.nxv4i32.p0(ptr align 4 [[TMP15]], <vscale x 4 x i1> splat (i1 true), i32 [[TMP7]])
+; IF-EVL-NEXT: [[TMP16:%.*]] = extractelement <vscale x 4 x i32> [[VP_OP_LOAD6]], i64 0
+; IF-EVL-NEXT: [[TMP17:%.*]] = add i32 [[TMP16]], [[TMP9]]
; IF-EVL-NEXT: br label %[[LATCH7]]
; IF-EVL: [[LATCH7]]:
-; IF-EVL-NEXT: [[TMP13:%.*]] = phi i32 [ poison, %[[VECTOR_BODY]] ], [ [[TMP16]], %[[LOAD_V25]] ]
+; IF-EVL-NEXT: [[TMP13:%.*]] = phi i32 [ poison, %[[VECTOR_BODY]] ], [ [[TMP17]], %[[LOAD_V25]] ]
+; IF-EVL-NEXT: [[TMP14:%.*]] = phi i1 [ false, %[[VECTOR_BODY]] ], [ true, %[[LOAD_V25]] ]
; IF-EVL-NEXT: [[BROADCAST_SPLATINSERT8:%.*]] = insertelement <vscale x 4 x i32> poison, i32 [[TMP13]], i64 0
; IF-EVL-NEXT: [[BROADCAST_SPLAT9:%.*]] = shufflevector <vscale x 4 x i32> [[BROADCAST_SPLATINSERT8]], <vscale x 4 x i32> poison, <vscale x 4 x i32> zeroinitializer
-; IF-EVL-NEXT: [[PREDPHI9:%.*]] = select i1 true, <vscale x 4 x i32> [[BROADCAST_SPLAT9]], <vscale x 4 x i32> [[PREDPHI8]]
+; IF-EVL-NEXT: [[PREDPHI9:%.*]] = select i1 [[TMP14]], <vscale x 4 x i32> [[BROADCAST_SPLAT9]], <vscale x 4 x i32> [[PREDPHI8]]
; IF-EVL-NEXT: [[TMP20:%.*]] = getelementptr inbounds i32, ptr [[DST]], i64 [[EVL_BASED_IV]]
; IF-EVL-NEXT: call void @llvm.vp.store.nxv4i32.p0(<vscale x 4 x i32> [[PREDPHI9]], ptr align 4 [[TMP20]], <vscale x 4 x i1> splat (i1 true), i32 [[TMP7]])
; IF-EVL-NEXT: [[TMP21:%.*]] = zext i32 [[TMP7]] to i64
diff --git a/llvm/test/Transforms/LoopVectorize/VPlan/AArch64/vplan-memory-op-decisions.ll b/llvm/test/Transforms/LoopVectorize/VPlan/AArch64/vplan-memory-op-decisions.ll
index 2e9950791642c..dd326743d29af 100644
--- a/llvm/test/Transforms/LoopVectorize/VPlan/AArch64/vplan-memory-op-decisions.ll
+++ b/llvm/test/Transforms/LoopVectorize/VPlan/AArch64/vplan-memory-op-decisions.ll
@@ -748,19 +748,19 @@ define void @blend_with_identical_incoming_values_address(ptr noalias %A, i1 %c)
; CHECK-EMPTY:
; CHECK-NEXT: vector.body:
; CHECK-NEXT: ir<%iv> = WIDEN-INDUCTION nuw nsw ir<0>, ir<1>, vp<[[VP0]]>
-; CHECK-NEXT: Successor(s): else
-; CHECK-EMPTY:
-; CHECK-NEXT: else:
-; CHECK-NEXT: EMIT vp<[[VP4:%[0-9]+]]> = not ir<%c>
-; CHECK-NEXT: EMIT ir<%idx.else> = add nsw ir<%iv>, ir<-1>, vp<[[VP4]]>
-; CHECK-NEXT: Successor(s): then
+; CHECK-NEXT: EMIT branch-on-cond ir<%c>
+; CHECK-NEXT: Successor(s): then, else
; CHECK-EMPTY:
; CHECK-NEXT: then:
-; CHECK-NEXT: EMIT ir<%idx.then> = add nsw ir<%iv>, ir<-1>, ir<%c>
+; CHECK-NEXT: EMIT ir<%idx.then> = add nsw ir<%iv>, ir<-1>
; CHECK-NEXT: Successor(s): latch
; CHECK-EMPTY:
; CHECK-NEXT: latch:
-; CHECK-NEXT: BLEND ir<%idx> = ir<%idx.else>/vp<[[VP4]]> ir<%idx.then>/ir<%c>
+; CHECK-NEXT: EMIT-SCALAR vp<[[VP4:%[0-9]+]]> = phi [ ir<%idx.else>, else ], [ ir<poison>, then ]
+; CHECK-NEXT: EMIT-SCALAR vp<[[VP5:%[0-9]+]]> = phi [ ir<true>, else ], [ ir<false>, then ]
+; CHECK-NEXT: EMIT-SCALAR vp<[[VP6:%[0-9]+]]> = phi [ ir<poison>, else ], [ ir<%idx.then>, then ]
+; CHECK-NEXT: EMIT-SCALAR vp<[[VP7:%[0-9]+]]> = phi [ ir<false>, else ], [ ir<true>, then ]
+; CHECK-NEXT: BLEND ir<%idx> = vp<%4>/vp<[[VP5]]> vp<%6>/vp<[[VP7]]>
; CHECK-NEXT: EMIT ir<%gep> = getelementptr inbounds ir<%A>, ir<%idx>
; CHECK-NEXT: EMIT-SCALAR ir<%lv> = load ir<%gep>
; CHECK-NEXT: EMIT ir<%add> = add ir<%lv>, ir<1>
@@ -770,6 +770,10 @@ define void @blend_with_identical_incoming_values_address(ptr noalias %A, i1 %c)
; CHECK-NEXT: EMIT vp<%index.next> = add nuw vp<[[VP3]]>, vp<[[VP1]]>
; CHECK-NEXT: EMIT branch-on-count vp<%index.next>, vp<[[VP2]]>
; CHECK-NEXT: No successors
+; CHECK-EMPTY:
+; CHECK-NEXT: else:
+; CHECK-NEXT: EMIT ir<%idx.else> = add nsw ir<%iv>, ir<-1>
+; CHECK-NEXT: Successor(s): latch
; CHECK-NEXT: }
; CHECK-NEXT: Successor(s): middle.block
; CHECK-EMPTY:
@@ -991,29 +995,33 @@ define void @blend_with_different_incoming_values_address(ptr noalias %A, ptr no
; CHECK-EMPTY:
; CHECK-NEXT: vector.body:
; CHECK-NEXT: ir<%iv> = WIDEN-INDUCTION nuw nsw ir<0>, ir<1>, vp<[[VP0]]>
-; CHECK-NEXT: Successor(s): else
-; CHECK-EMPTY:
-; CHECK-NEXT: else:
-; CHECK-NEXT: EMIT vp<[[VP4:%[0-9]+]]> = not ir<%c>
-; CHECK-NEXT: EMIT ir<%idx.else> = add nsw ir<%iv>, ir<-2>, vp<[[VP4]]>
-; CHECK-NEXT: Successor(s): then
+; CHECK-NEXT: EMIT branch-on-cond ir<%c>
+; CHECK-NEXT: Successor(s): then, else
; CHECK-EMPTY:
; CHECK-NEXT: then:
-; CHECK-NEXT: EMIT ir<%idx.then> = add nsw ir<%iv>, ir<-1>, ir<%c>
+; CHECK-NEXT: EMIT ir<%idx.then> = add nsw ir<%iv>, ir<-1>
; CHECK-NEXT: Successor(s): latch
; CHECK-EMPTY:
; CHECK-NEXT: latch:
-; CHECK-NEXT: BLEND ir<%idx> = ir<%idx.else>/vp<[[VP4]]> ir<%idx.then>/ir<%c>
+; CHECK-NEXT: EMIT-SCALAR vp<[[VP4:%[0-9]+]]> = phi [ ir<%idx.else>, else ], [ ir<poison>, then ]
+; CHECK-NEXT: EMIT-SCALAR vp<[[VP5:%[0-9]+]]> = phi [ ir<true>, else ], [ ir<false>, then ]
+; CHECK-NEXT: EMIT-SCALAR vp<[[VP6:%[0-9]+]]> = phi [ ir<poison>, else ], [ ir<%idx.then>, then ]
+; CHECK-NEXT: EMIT-SCALAR vp<[[VP7:%[0-9]+]]> = phi [ ir<false>, else ], [ ir<true>, then ]
+; CHECK-NEXT: BLEND ir<%idx> = vp<%4>/vp<[[VP5]]> vp<%6>/vp<[[VP7]]>
; CHECK-NEXT: EMIT ir<%gep.B> = getelementptr inbounds ir<%B>, ir<%idx>
; CHECK-NEXT: EMIT-SCALAR ir<%lv> = load ir<%gep.B>
; CHECK-NEXT: EMIT ir<%gep.A> = getelementptr inbounds ir<%A>, ir<%iv>
-; CHECK-NEXT: vp<[[VP5:%[0-9]+]]> = vector-pointer inbounds i32, ir<%gep.A>, ir<1>
-; CHECK-NEXT: WIDEN store vp<[[VP5]]>, ir<%lv>
+; CHECK-NEXT: vp<[[VP8:%[0-9]+]]> = vector-pointer inbounds i32, ir<%gep.A>, ir<1>
+; CHECK-NEXT: WIDEN store vp<[[VP8]]>, ir<%lv>
; CHECK-NEXT: EMIT ir<%iv.next> = add nuw nsw ir<%iv>, ir<1>
; CHECK-NEXT: EMIT ir<%ec> = icmp eq ir<%iv>, ir<100>
; CHECK-NEXT: EMIT vp<%index.next> = add nuw vp<[[VP3]]>, vp<[[VP1]]>
; CHECK-NEXT: EMIT branch-on-count vp<%index.next>, vp<[[VP2]]>
; CHECK-NEXT: No successors
+; CHECK-EMPTY:
+; CHECK-NEXT: else:
+; CHECK-NEXT: EMIT ir<%idx.else> = add nsw ir<%iv>, ir<-2>
+; CHECK-NEXT: Successor(s): latch
; CHECK-NEXT: }
; CHECK-NEXT: Successor(s): middle.block
; CHECK-EMPTY:
diff --git a/llvm/test/Transforms/LoopVectorize/VPlan/predicator.ll b/llvm/test/Transforms/LoopVectorize/VPlan/predicator.ll
index 063744fe2d5e8..5d137beea26b8 100644
--- a/llvm/test/Transforms/LoopVectorize/VPlan/predicator.ll
+++ b/llvm/test/Transforms/LoopVectorize/VPlan/predicator.ll
@@ -1319,7 +1319,8 @@ define void @uniform_branch_after_varying_branch(ptr %a, i1 %u1) {
; CHECK-EMPTY:
; CHECK-NEXT: bb3:
; CHECK-NEXT: EMIT-SCALAR vp<[[VP5:%[0-9]+]]> = phi [ ir<poison>, bb1 ], [ ir<%add2>, bb2 ]
-; CHECK-NEXT: BLEND ir<%phi3> = ir<%add1>/ir<true> vp<%5>/ir<true>
+; CHECK-NEXT: EMIT-SCALAR vp<[[VP6:%[0-9]+]]> = phi [ ir<false>, bb1 ], [ ir<true>, bb2 ]
+; CHECK-NEXT: BLEND ir<%phi3> = ir<%add1>/ir<true> vp<%5>/vp<[[VP6]]>
; CHECK-NEXT: EMIT ir<%add3> = add ir<%phi3>, ir<3>, vp<[[VP4]]>
; CHECK-NEXT: Successor(s): bb4
; CHECK-EMPTY:
@@ -1489,8 +1490,10 @@ define void @uniform_branch_after_varying_branch_more_blocks(ptr %a, i1 %u1) {
; CHECK-EMPTY:
; CHECK-NEXT: bb4:
; CHECK-NEXT: EMIT-SCALAR vp<[[VP5:%[0-9]+]]> = phi [ ir<%add2>, bb2 ], [ ir<poison>, bb3 ]
-; CHECK-NEXT: EMIT-SCALAR vp<[[VP6:%[0-9]+]]> = phi [ ir<poison>, bb2 ], [ ir<%add3>, bb3 ]
-; CHECK-NEXT: BLEND ir<%phi4> = vp<%5>/ir<true> vp<%6>/ir<true>
+; CHECK-NEXT: EMIT-SCALAR vp<[[VP6:%[0-9]+]]> = phi [ ir<true>, bb2 ], [ ir<false>, bb3 ]
+; CHECK-NEXT: EMIT-SCALAR vp<[[VP7:%[0-9]+]]> = phi [ ir<poison>, bb2 ], [ ir<%add3>, bb3 ]
+; CHECK-NEXT: EMIT-SCALAR vp<[[VP8:%[0-9]+]]> = phi [ ir<false>, bb2 ], [ ir<true>, bb3 ]
+; CHECK-NEXT: BLEND ir<%phi4> = vp<%5>/vp<[[VP6]]> vp<%7>/vp<[[VP8]]>
; CHECK-NEXT: EMIT ir<%add4> = add ir<%phi4>, ir<4>, vp<[[VP4]]>
; CHECK-NEXT: Successor(s): bb5
; CHECK-EMPTY:
@@ -1681,8 +1684,10 @@ define void @uniform_branch_after_varying_branch_more_blocks_mirrored(ptr %a, i1
; CHECK-EMPTY:
; CHECK-NEXT: bb5:
; CHECK-NEXT: EMIT-SCALAR vp<[[VP5:%[0-9]+]]> = phi [ ir<%add3>, bb3 ], [ ir<poison>, bb4 ]
-; CHECK-NEXT: EMIT-SCALAR vp<[[VP6:%[0-9]+]]> = phi [ ir<poison>, bb3 ], [ ir<%add4>, bb4 ]
-; CHECK-NEXT: BLEND ir<%phi5> = vp<%5>/ir<true> vp<%6>/ir<true>
+; CHECK-NEXT: EMIT-SCALAR vp<[[VP6:%[0-9]+]]> = phi [ ir<true>, bb3 ], [ ir<false>, bb4 ]
+; CHECK-NEXT: EMIT-SCALAR vp<[[VP7:%[0-9]+]]> = phi [ ir<poison>, bb3 ], [ ir<%add4>, bb4 ]
+; CHECK-NEXT: EMIT-SCALAR vp<[[VP8:%[0-9]+]]> = phi [ ir<false>, bb3 ], [ ir<true>, bb4 ]
+; CHECK-NEXT: BLEND ir<%phi5> = vp<%5>/vp<[[VP6]]> vp<%7>/vp<[[VP8]]>
; CHECK-NEXT: EMIT ir<%add5> = add ir<%phi5>, ir<5>, ir<%v0>
; CHECK-NEXT: Successor(s): bb6
; CHECK-EMPTY:
diff --git a/llvm/test/Transforms/LoopVectorize/constant-fold-commutative-and.ll b/llvm/test/Transforms/LoopVectorize/constant-fold-commutative-and.ll
index 2f079172d5ba6..165be4a0921fc 100644
--- a/llvm/test/Transforms/LoopVectorize/constant-fold-commutative-and.ll
+++ b/llvm/test/Transforms/LoopVectorize/constant-fold-commutative-and.ll
@@ -24,9 +24,10 @@ define void @constant_fold_commutative_and(ptr %ptr.n, ptr noalias %p, i1 %cond)
; CHECK-NEXT: [[TMP4:%.*]] = icmp ult i64 [[TMP2]], 7
; CHECK-NEXT: br label %[[LATCH2]]
; CHECK: [[LATCH2]]:
-; CHECK-NEXT: [[TMP5:%.*]] = phi i1 [ poison, %[[VECTOR_BODY]] ], [ [[TMP4]], %[[PRED_11]] ]
+; CHECK-NEXT: [[TMP5:%.*]] = phi i1 [ false, %[[VECTOR_BODY]] ], [ true, %[[PRED_11]] ]
+; CHECK-NEXT: [[TMP12:%.*]] = phi i1 [ poison, %[[VECTOR_BODY]] ], [ [[TMP4]], %[[PRED_11]] ]
; CHECK-NEXT: [[TMP7:%.*]] = phi i1 [ false, %[[VECTOR_BODY]] ], [ [[TMP3]], %[[PRED_11]] ]
-; CHECK-NEXT: [[TMP9:%.*]] = select i1 [[TMP7]], i1 [[TMP5]], i1 true
+; CHECK-NEXT: [[TMP9:%.*]] = select i1 [[TMP7]], i1 [[TMP12]], i1 [[TMP5]]
; CHECK-NEXT: br i1 [[TMP6]], label %[[PRED_STORE_IF:.*]], label %[[PRED_STORE_CONTINUE:.*]]
; CHECK: [[PRED_STORE_IF]]:
; CHECK-NEXT: [[TMP8:%.*]] = getelementptr i8, ptr [[P]], i64 [[INDEX]]
diff --git a/llvm/test/Transforms/LoopVectorize/predicatedinst-loop-invariant.ll b/llvm/test/Transforms/LoopVectorize/predicatedinst-loop-invariant.ll
index e6904dabbcb6a..3292b2776ab02 100644
--- a/llvm/test/Transforms/LoopVectorize/predicatedinst-loop-invariant.ll
+++ b/llvm/test/Transforms/LoopVectorize/predicatedinst-loop-invariant.ll
@@ -238,24 +238,26 @@ define void @test_store_to_invariant_address_needs_mask_due_to_low_trip_count(pt
; CHECK: [[VECTOR_BODY]]:
; CHECK-NEXT: br label %[[MIDDLE_BLOCK:.*]]
; CHECK: [[MIDDLE_BLOCK]]:
+; CHECK-NEXT: [[TMP0:%.*]] = phi i1 [ false, %[[VECTOR_BODY]] ]
+; CHECK-NEXT: [[PREDPHI:%.*]] = select i1 [[TMP0]], i32 0, i32 1
; CHECK-NEXT: br i1 true, label %[[PRED_STORE_IF:.*]], label %[[EXIT:.*]]
; CHECK: [[PRED_STORE_IF]]:
-; CHECK-NEXT: store i32 0, ptr [[DST]], align 4
+; CHECK-NEXT: store i32 [[PREDPHI]], ptr [[DST]], align 4
; CHECK-NEXT: br label %[[EXIT]]
; CHECK: [[EXIT]]:
; CHECK-NEXT: br i1 true, label %[[PRED_STORE_IF2:.*]], label %[[PRED_STORE_CONTINUE3:.*]]
; CHECK: [[PRED_STORE_IF2]]:
-; CHECK-NEXT: store i32 0, ptr [[DST]], align 4
+; CHECK-NEXT: store i32 [[PREDPHI]], ptr [[DST]], align 4
; CHECK-NEXT: br label %[[PRED_STORE_CONTINUE3]]
; CHECK: [[PRED_STORE_CONTINUE3]]:
; CHECK-NEXT: br i1 true, label %[[PRED_STORE_IF4:.*]], label %[[PRED_STORE_CONTINUE5:.*]]
; CHECK: [[PRED_STORE_IF4]]:
-; CHECK-NEXT: store i32 0, ptr [[DST]], align 4
+; CHECK-NEXT: store i32 [[PREDPHI]], ptr [[DST]], align 4
; CHECK-NEXT: br label %[[PRED_STORE_CONTINUE5]]
; CHECK: [[PRED_STORE_CONTINUE5]]:
; CHECK-NEXT: br i1 false, label %[[PRED_STORE_IF6:.*]], label %[[PRED_STORE_CONTINUE7:.*]]
; CHECK: [[PRED_STORE_IF6]]:
-; CHECK-NEXT: store i32 0, ptr [[DST]], align 4
+; CHECK-NEXT: store i32 [[PREDPHI]], ptr [[DST]], align 4
; CHECK-NEXT: br label %[[PRED_STORE_CONTINUE7]]
; CHECK: [[PRED_STORE_CONTINUE7]]:
; CHECK-NEXT: br label %[[MIDDLE_BLOCK1:.*]]
diff --git a/llvm/test/Transforms/LoopVectorize/predicator.ll b/llvm/test/Transforms/LoopVectorize/predicator.ll
index 9dd3b4e47796c..9d45adbe36dac 100644
--- a/llvm/test/Transforms/LoopVectorize/predicator.ll
+++ b/llvm/test/Transforms/LoopVectorize/predicator.ll
@@ -636,13 +636,14 @@ define void @uniform_branch_after_varying_branch(ptr %a, i1 %u1) {
; CHECK-NEXT: [[TMP2:%.*]] = add <4 x i64> [[VEC_IND]], splat (i64 1)
; CHECK-NEXT: br i1 [[U1]], label %[[BB21:.*]], label %[[BB32]]
; CHECK: [[BB21]]:
-; CHECK-NEXT: [[TMP8:%.*]] = add i64 [[INDEX]], 2
+; CHECK-NEXT: [[TMP7:%.*]] = add i64 [[INDEX]], 2
; CHECK-NEXT: br label %[[BB32]]
; CHECK: [[BB32]]:
-; CHECK-NEXT: [[TMP3:%.*]] = phi i64 [ poison, %[[VECTOR_BODY]] ], [ [[TMP8]], %[[BB21]] ]
+; CHECK-NEXT: [[TMP3:%.*]] = phi i64 [ poison, %[[VECTOR_BODY]] ], [ [[TMP7]], %[[BB21]] ]
+; CHECK-NEXT: [[TMP9:%.*]] = phi i1 [ false, %[[VECTOR_BODY]] ], [ true, %[[BB21]] ]
; CHECK-NEXT: [[BROADCAST_SPLATINSERT:%.*]] = insertelement <4 x i64> poison, i64 [[TMP3]], i64 0
; CHECK-NEXT: [[BROADCAST_SPLAT:%.*]] = shufflevector <4 x i64> [[BROADCAST_SPLATINSERT]], <4 x i64> poison, <4 x i32> zeroinitializer
-; CHECK-NEXT: [[PREDPHI:%.*]] = select i1 true, <4 x i64> [[BROADCAST_SPLAT]], <4 x i64> [[TMP2]]
+; CHECK-NEXT: [[PREDPHI:%.*]] = select i1 [[TMP9]], <4 x i64> [[BROADCAST_SPLAT]], <4 x i64> [[TMP2]]
; CHECK-NEXT: [[TMP4:%.*]] = add <4 x i64> [[PREDPHI]], splat (i64 3)
; CHECK-NEXT: [[TMP5:%.*]] = add <4 x i64> [[VEC_IND]], splat (i64 4)
; CHECK-NEXT: [[PREDPHI1:%.*]] = select <4 x i1> [[TMP1]], <4 x i64> [[TMP5]], <4 x i64> [[TMP4]]
@@ -650,8 +651,8 @@ define void @uniform_branch_after_varying_branch(ptr %a, i1 %u1) {
; CHECK-NEXT: store i64 [[TMP6]], ptr [[A]], align 4
; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 4
; CHECK-NEXT: [[VEC_IND_NEXT]] = add nuw nsw <4 x i64> [[VEC_IND]], splat (i64 4)
-; CHECK-NEXT: [[TMP7:%.*]] = icmp eq i64 [[INDEX_NEXT]], 128
-; CHECK-NEXT: br i1 [[TMP7]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP10:![0-9]+]]
+; CHECK-NEXT: [[TMP8:%.*]] = icmp eq i64 [[INDEX_NEXT]], 128
+; CHECK-NEXT: br i1 [[TMP8]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP10:![0-9]+]]
; CHECK: [[MIDDLE_BLOCK]]:
; CHECK-NEXT: br label %[[EXIT:.*]]
; CHECK: [[EXIT]]:
@@ -799,10 +800,11 @@ define void @uniform_branch_after_varying_branch_more_blocks(ptr %a, i1 %u1) {
; CHECK-NEXT: br label %[[BB43]]
; CHECK: [[BB43]]:
; CHECK-NEXT: [[TMP3:%.*]] = phi i64 [ [[TMP7]], %[[BB21]] ], [ poison, %[[BB32]] ]
-; CHECK-NEXT: [[TMP9:%.*]] = phi i64 [ poison, %[[BB21]] ], [ [[TMP2]], %[[BB32]] ]
-; CHECK-NEXT: [[PREDPHI:%.*]] = select i1 true, i64 [[TMP9]], i64 [[TMP3]]
-; CHECK-NEXT: [[TMP10:%.*]] = add i64 [[PREDPHI]], 4
-; CHECK-NEXT: [[BROADCAST_SPLATINSERT:%.*]] = insertelement <4 x i64> poison, i64 [[TMP10]], i64 0
+; CHECK-NEXT: [[TMP8:%.*]] = phi i64 [ poison, %[[BB21]] ], [ [[TMP2]], %[[BB32]] ]
+; CHECK-NEXT: [[TMP10:%.*]] = phi i1 [ false, %[[BB21]] ], [ true, %[[BB32]] ]
+; CHECK-NEXT: [[PREDPHI:%.*]] = select i1 [[TMP10]], i64 [[TMP8]], i64 [[TMP3]]
+; CHECK-NEXT: [[TMP11:%.*]] = add i64 [[PREDPHI]], 4
+; CHECK-NEXT: [[BROADCAST_SPLATINSERT:%.*]] = insertelement <4 x i64> poison, i64 [[TMP11]], i64 0
; CHECK-NEXT: [[TMP4:%.*]] = shufflevector <4 x i64> [[BROADCAST_SPLATINSERT]], <4 x i64> poison, <4 x i32> zeroinitializer
; CHECK-NEXT: [[TMP5:%.*]] = add <4 x i64> [[VEC_IND]], splat (i64 5)
; CHECK-NEXT: [[PREDPHI1:%.*]] = select <4 x i1> [[TMP1]], <4 x i64> [[TMP5]], <4 x i64> [[TMP4]]
@@ -810,8 +812,8 @@ define void @uniform_branch_after_varying_branch_more_blocks(ptr %a, i1 %u1) {
; CHECK-NEXT: store i64 [[TMP6]], ptr [[A]], align 4
; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 4
; CHECK-NEXT: [[VEC_IND_NEXT]] = add nuw nsw <4 x i64> [[VEC_IND]], splat (i64 4)
-; CHECK-NEXT: [[TMP8:%.*]] = icmp eq i64 [[INDEX_NEXT]], 128
-; CHECK-NEXT: br i1 [[TMP8]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP12:![0-9]+]]
+; CHECK-NEXT: [[TMP9:%.*]] = icmp eq i64 [[INDEX_NEXT]], 128
+; CHECK-NEXT: br i1 [[TMP9]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP12:![0-9]+]]
; CHECK: [[MIDDLE_BLOCK]]:
; CHECK-NEXT: br label %[[EXIT:.*]]
; CHECK: [[EXIT]]:
@@ -965,25 +967,26 @@ define void @uniform_branch_after_varying_branch_more_blocks_mirrored(ptr %a, i1
; CHECK-NEXT: [[TMP2:%.*]] = add <4 x i64> [[VEC_IND]], splat (i64 1)
; CHECK-NEXT: br i1 [[U2]], label %[[BB42:.*]], label %[[BB31:.*]]
; CHECK: [[BB31]]:
-; CHECK-NEXT: [[TMP7:%.*]] = add i64 [[INDEX]], 3
+; CHECK-NEXT: [[TMP8:%.*]] = add i64 [[INDEX]], 3
; CHECK-NEXT: br label %[[BB53]]
; CHECK: [[BB42]]:
; CHECK-NEXT: [[TMP3:%.*]] = add i64 [[INDEX]], 4
; CHECK-NEXT: br label %[[BB53]]
; CHECK: [[BB53]]:
-; CHECK-NEXT: [[TMP4:%.*]] = phi i64 [ [[TMP7]], %[[BB31]] ], [ poison, %[[BB42]] ]
-; CHECK-NEXT: [[TMP9:%.*]] = phi i64 [ poison, %[[BB31]] ], [ [[TMP3]], %[[BB42]] ]
-; CHECK-NEXT: [[PREDPHI:%.*]] = select i1 true, i64 [[TMP9]], i64 [[TMP4]]
-; CHECK-NEXT: [[TMP10:%.*]] = add i64 [[PREDPHI]], 5
-; CHECK-NEXT: [[BROADCAST_SPLATINSERT:%.*]] = insertelement <4 x i64> poison, i64 [[TMP10]], i64 0
+; CHECK-NEXT: [[TMP4:%.*]] = phi i64 [ [[TMP8]], %[[BB31]] ], [ poison, %[[BB42]] ]
+; CHECK-NEXT: [[TMP10:%.*]] = phi i64 [ poison, %[[BB31]] ], [ [[TMP3]], %[[BB42]] ]
+; CHECK-NEXT: [[TMP11:%.*]] = phi i1 [ false, %[[BB31]] ], [ true, %[[BB42]] ]
+; CHECK-NEXT: [[PREDPHI:%.*]] = select i1 [[TMP11]], i64 [[TMP10]], i64 [[TMP4]]
+; CHECK-NEXT: [[TMP7:%.*]] = add i64 [[PREDPHI]], 5
+; CHECK-NEXT: [[BROADCAST_SPLATINSERT:%.*]] = insertelement <4 x i64> poison, i64 [[TMP7]], i64 0
; CHECK-NEXT: [[TMP5:%.*]] = shufflevector <4 x i64> [[BROADCAST_SPLATINSERT]], <4 x i64> poison, <4 x i32> zeroinitializer
; CHECK-NEXT: [[PREDPHI1:%.*]] = select <4 x i1> [[TMP1]], <4 x i64> [[TMP5]], <4 x i64> [[TMP2]]
; CHECK-NEXT: [[TMP6:%.*]] = extractelement <4 x i64> [[PREDPHI1]], i64 3
; CHECK-NEXT: store i64 [[TMP6]], ptr [[A]], align 4
; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 4
; CHECK-NEXT: [[VEC_IND_NEXT]] = add nuw nsw <4 x i64> [[VEC_IND]], splat (i64 4)
-; CHECK-NEXT: [[TMP8:%.*]] = icmp eq i64 [[INDEX_NEXT]], 128
-; CHECK-NEXT: br i1 [[TMP8]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP14:![0-9]+]]
+; CHECK-NEXT: [[TMP9:%.*]] = icmp eq i64 [[INDEX_NEXT]], 128
+; CHECK-NEXT: br i1 [[TMP9]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP14:![0-9]+]]
; CHECK: [[MIDDLE_BLOCK]]:
; CHECK-NEXT: br label %[[EXIT:.*]]
; CHECK: [[EXIT]]:
>From 728da227143a44f42b32f711f907ef8dc457b9c7 Mon Sep 17 00:00:00 2001
From: Andrei Elovikov <andrei.elovikov at sifive.com>
Date: Thu, 27 Aug 2026 12:12:45 -0700
Subject: [PATCH 09/11] Extra tests for mask/phi re-use
---
.../LoopVectorize/VPlan/predicator.ll | 149 ++++++++++++++++++
1 file changed, 149 insertions(+)
diff --git a/llvm/test/Transforms/LoopVectorize/VPlan/predicator.ll b/llvm/test/Transforms/LoopVectorize/VPlan/predicator.ll
index 5d137beea26b8..d5caf71e146d3 100644
--- a/llvm/test/Transforms/LoopVectorize/VPlan/predicator.ll
+++ b/llvm/test/Transforms/LoopVectorize/VPlan/predicator.ll
@@ -630,6 +630,79 @@ exit:
ret void
}
+; Check that multile blends in the same block re-use as much computation as possible.
+define void @blend_masks_triangle_phi_multiple_blends(ptr noalias %p, i1 %c0, i1 %c1) {
+; CHECK-LABEL: VPlan for loop in 'blend_masks_triangle_phi_multiple_blends'
+; CHECK-NEXT: <x1> vector loop: {
+; CHECK-NEXT: vp<[[VP3:%[0-9]+]]> = CANONICAL-IV
+; CHECK-EMPTY:
+; CHECK-NEXT: vector.body:
+; CHECK-NEXT: ir<%iv> = WIDEN-INDUCTION ir<0>, ir<1>, vp<[[VP0:%[0-9]+]]>
+; CHECK-NEXT: EMIT ir<%add0> = add ir<%iv>, ir<42>
+; CHECK-NEXT: EMIT branch-on-cond ir<%c0>
+; CHECK-NEXT: Successor(s): bb1, bb2
+; CHECK-EMPTY:
+; CHECK-NEXT: bb1:
+; CHECK-NEXT: EMIT ir<%add1> = add ir<%iv>, ir<1>
+; CHECK-NEXT: Successor(s): bb2
+; CHECK-EMPTY:
+; CHECK-NEXT: bb2:
+; CHECK-NEXT: EMIT-SCALAR vp<[[VP4:%[0-9]+]]> = phi [ ir<poison>, vector.body ], [ ir<%add1>, bb1 ]
+; CHECK-NEXT: EMIT-SCALAR vp<[[VP5:%[0-9]+]]> = phi [ ir<false>, vector.body ], [ ir<true>, bb1 ]
+; CHECK-NEXT: EMIT-SCALAR vp<[[VP6:%[0-9]+]]> = phi [ ir<false>, vector.body ], [ ir<true>, bb1 ]
+; CHECK-NEXT: EMIT-SCALAR vp<[[VP7:%[0-9]+]]> = phi [ ir<false>, vector.body ], [ ir<true>, bb1 ]
+; CHECK-NEXT: EMIT vp<[[VP8:%[0-9]+]]> = not ir<%c1>
+; CHECK-NEXT: EMIT ir<%add2> = add ir<%iv>, ir<1>
+; CHECK-NEXT: Successor(s): bb3
+; CHECK-EMPTY:
+; CHECK-NEXT: bb3:
+; CHECK-NEXT: BLEND ir<%phi1> = vp<%4>/vp<[[VP5]]> ir<%add0>/ir<true>
+; CHECK-NEXT: BLEND ir<%phi2> = ir<%add0>/vp<[[VP6]]> ir<%add2>/ir<true>
+; CHECK-NEXT: BLEND ir<%phi3> = vp<%4>/vp<[[VP7]]> ir<%add2>/ir<true>
+; CHECK-NEXT: EMIT ir<%sum12> = add ir<%phi1>, ir<%phi2>
+; CHECK-NEXT: EMIT ir<%sum123> = add ir<%sum12>, ir<%phi3>
+; CHECK-NEXT: EMIT ir<%gep> = getelementptr ir<%p>, ir<%iv>
+; CHECK-NEXT: EMIT store ir<%sum123>, ir<%gep>
+; CHECK-NEXT: EMIT ir<%iv.next> = add ir<%iv>, ir<1>
+; CHECK-NEXT: EMIT ir<%ec> = icmp eq ir<%iv.next>, ir<128>
+; CHECK-NEXT: EMIT vp<%index.next> = add nuw vp<[[VP3]]>, vp<[[VP1:%[0-9]+]]>
+; CHECK-NEXT: EMIT branch-on-count vp<%index.next>, vp<[[VP2:%[0-9]+]]>
+; CHECK-NEXT: No successors
+; CHECK-NEXT: }
+; CHECK-NEXT: Successor(s): middle.block
+;
+entry:
+ br label %bb0
+
+bb0:
+ %iv = phi i32 [0, %entry], [%iv.next, %bb3]
+ %add0 = add i32 %iv, 42
+ br i1 %c0, label %bb1, label %bb2
+
+bb1:
+ %add1 = add i32 %iv, 1
+ br i1 %c1, label %bb3, label %bb2
+
+bb2:
+ %add2 = add i32 %iv, 1
+ br label %bb3
+
+bb3:
+ %phi1= phi i32 [%add1, %bb1], [%add0, %bb2]
+ %phi2 = phi i32 [%add0, %bb1], [%add2, %bb2]
+ %phi3 = phi i32 [%add1, %bb1], [%add2, %bb2]
+ %sum12 = add i32 %phi1, %phi2
+ %sum123 = add i32 %sum12, %phi3
+ %gep = getelementptr i32, ptr %p, i32 %iv
+ store i32 %sum123, ptr %gep
+ %iv.next = add i32 %iv, 1
+ %ec = icmp eq i32 %iv.next, 128
+ br i1 %ec, label %exit, label %bb0
+
+exit:
+ ret void
+}
+
define void @blend_chain_non_trivial(ptr noalias %a, ptr noalias %b) {
; CHECK-LABEL: VPlan for loop in 'blend_chain_non_trivial'
; CHECK-NEXT: <x1> vector loop: {
@@ -1029,6 +1102,82 @@ exit:
ret void
}
+; Check that multile blends in the same block re-use as much computation as possible.
+define void @outermost_uniform_branch_multiple_blends(ptr %a, i1 %u0) {
+; CHECK-LABEL: VPlan for loop in 'outermost_uniform_branch_multiple_blends'
+; CHECK-NEXT: <x1> vector loop: {
+; CHECK-NEXT: vp<[[VP3:%[0-9]+]]> = CANONICAL-IV
+; CHECK-EMPTY:
+; CHECK-NEXT: vector.body:
+; CHECK-NEXT: ir<%iv> = WIDEN-INDUCTION nuw nsw ir<0>, ir<1>, vp<[[VP0:%[0-9]+]]>
+; CHECK-NEXT: EMIT branch-on-cond ir<%u0>
+; CHECK-NEXT: Successor(s): bb1, bb4
+; CHECK-EMPTY:
+; CHECK-NEXT: bb1:
+; CHECK-NEXT: EMIT ir<%add1> = add ir<%iv>, ir<1>
+; CHECK-NEXT: EMIT ir<%v1> = icmp sle ir<%iv>, ir<1>
+; CHECK-NEXT: Successor(s): bb2
+; CHECK-EMPTY:
+; CHECK-NEXT: bb2:
+; CHECK-NEXT: EMIT ir<%add2> = add ir<%iv>, ir<2>, ir<%v1>
+; CHECK-NEXT: Successor(s): bb3
+; CHECK-EMPTY:
+; CHECK-NEXT: bb3:
+; CHECK-NEXT: BLEND ir<%phi3> = ir<%add1>/ir<true> ir<%add2>/ir<%v1>
+; CHECK-NEXT: EMIT ir<%add3.1> = add ir<%phi3>, ir<3>
+; CHECK-NEXT: Successor(s): bb4
+; CHECK-EMPTY:
+; CHECK-NEXT: bb4:
+; CHECK-NEXT: EMIT-SCALAR vp<[[VP4:%[0-9]+]]> = phi [ ir<poison>, vector.body ], [ ir<%add3.1>, bb3 ]
+; CHECK-NEXT: EMIT-SCALAR vp<[[VP5:%[0-9]+]]> = phi [ ir<false>, vector.body ], [ ir<true>, bb3 ]
+; CHECK-NEXT: EMIT-SCALAR vp<[[VP6:%[0-9]+]]> = phi [ ir<false>, vector.body ], [ ir<true>, bb3 ]
+; CHECK-NEXT: BLEND ir<%phi4.1> = ir<0>/ir<true> vp<%4>/vp<[[VP5]]>
+; CHECK-NEXT: BLEND ir<%phi4.2> = ir<-1>/ir<true> vp<%4>/vp<[[VP6]]>
+; CHECK-NEXT: EMIT ir<%sum> = add ir<%phi4.1>, ir<%phi4.2>
+; CHECK-NEXT: EMIT store ir<%sum>, ir<%a>
+; CHECK-NEXT: EMIT ir<%iv.next> = add nuw nsw ir<%iv>, ir<1>
+; CHECK-NEXT: EMIT ir<%ec> = icmp eq ir<%iv.next>, ir<128>
+; CHECK-NEXT: EMIT vp<%index.next> = add nuw vp<[[VP3]]>, vp<[[VP1:%[0-9]+]]>
+; CHECK-NEXT: EMIT branch-on-count vp<%index.next>, vp<[[VP2:%[0-9]+]]>
+; CHECK-NEXT: No successors
+; CHECK-NEXT: }
+; CHECK-NEXT: Successor(s): middle.block
+;
+entry:
+ br label %bb0
+
+bb0:
+ %iv = phi i64 [ 0, %entry ], [ %iv.next, %bb4 ]
+ br i1 %u0, label %bb1, label %bb4
+
+bb1:
+ %add1 = add i64 %iv, 1
+ %v1 = icmp sle i64 %iv, 1
+ br i1 %v1, label %bb2, label %bb3
+
+bb2:
+ %add2 = add i64 %iv, 2
+ br label %bb3
+
+bb3:
+ %phi3 = phi i64 [ %add1, %bb1 ], [ %add2, %bb2 ]
+ %add3.1 = add i64 %phi3, 3
+ %add3.2 = add i64 %phi3, 30
+ br label %bb4
+
+bb4:
+ %phi4.1 = phi i64 [ %add3.1, %bb3 ], [ 0, %bb0 ]
+ %phi4.2 = phi i64 [ %add3.1, %bb3 ], [ -1, %bb0 ]
+ %sum = add i64 %phi4.1, %phi4.2
+ store i64 %sum, ptr %a
+ %iv.next = add nuw nsw i64 %iv, 1
+ %ec = icmp eq i64 %iv.next, 128
+ br i1 %ec, label %exit, label %bb0
+
+exit:
+ ret void
+}
+
define void @outermost_uniform_branch_no_phi(ptr %a, i1 %u0) {
; CHECK-LABEL: VPlan for loop in 'outermost_uniform_branch_no_phi'
; CHECK-NEXT: <x1> vector loop: {
>From a26117a4951bfa8e2275e0522fb11e26816149dc Mon Sep 17 00:00:00 2001
From: Andrei Elovikov <andrei.elovikov at sifive.com>
Date: Thu, 27 Aug 2026 12:14:41 -0700
Subject: [PATCH 10/11] Share block masks in reconstructSSA for blend masks
---
.../Transforms/Vectorize/VPlanPredicator.cpp | 28 +++++++------------
.../LoopVectorize/VPlan/predicator.ll | 11 +++-----
2 files changed, 14 insertions(+), 25 deletions(-)
diff --git a/llvm/lib/Transforms/Vectorize/VPlanPredicator.cpp b/llvm/lib/Transforms/Vectorize/VPlanPredicator.cpp
index ca935f4694cf8..ba234e6426e9b 100644
--- a/llvm/lib/Transforms/Vectorize/VPlanPredicator.cpp
+++ b/llvm/lib/Transforms/Vectorize/VPlanPredicator.cpp
@@ -126,8 +126,6 @@ class VPPredicator {
VPValue *createMaskOr(VPValue *LHS, VPValue *RHS, DebugLoc DL);
VPValue *reconstructSSA(VPBasicBlock *UseBB, VPValue *V, bool IsMask);
- VPValue *reconstructBlendMask(VPBasicBlock *UseBB, VPBasicBlock *MaskBlock,
- VPValue *Mask);
void fixSSA(VPRecipeBase *U, VPValue *V, bool IsMask);
bool shouldPreserveTerminator(VPBasicBlock *VPBB);
@@ -255,21 +253,6 @@ VPValue *VPPredicator::reconstructSSA(VPBasicBlock *UseBB, VPValue *V,
return vputils::reconstructSSA(UseBB, SSADefs);
}
-VPValue *VPPredicator::reconstructBlendMask(VPBasicBlock *UseBB,
- VPBasicBlock *MaskBlock,
- VPValue *Mask) {
- if (!Mask)
- Mask = Plan.getTrue();
-
- // The mask is only valid along paths from MaskBlock. Make that explicit
- // before reconstructing it at the blend location.
- DenseMap<VPBasicBlock *, VPValue *> BlendMaskDefs;
- BlendMaskDefs[Plan.getVectorLoopRegion()->getEntryBasicBlock()] =
- Plan.getFalse();
- BlendMaskDefs[MaskBlock] = Mask;
- return vputils::reconstructSSA(UseBB, BlendMaskDefs);
-}
-
void VPPredicator::fixSSA(VPRecipeBase *U, VPValue *V, bool IsMask) {
VPValue *Fixed = reconstructSSA(U->getParent(), V, IsMask);
if (Fixed == V)
@@ -551,6 +534,10 @@ void VPPredicator::convertPhisToBlends(VPBasicBlock *VPBB) {
if (!Phi->isSSAReconstructionPhi())
Phis.push_back(Phi);
}
+ // All PHIs in VPBB use the same blend masks. Keep the SSA reconstruction
+ // definitions for each mask block so subsequent PHIs reuse reconstructed
+ // phis.
+ DenseMap<VPBasicBlock *, DenseMap<VPBasicBlock *, VPValue *>> BlendMaskDefs;
for (VPPhi *PhiR : Phis) {
LLVM_DEBUG(dbgs() << "Converting " << *PhiR << " to blend\n");
// The non-header Phi is converted into a Blend recipe below,
@@ -640,8 +627,13 @@ void VPPredicator::convertPhisToBlends(VPBasicBlock *VPBB) {
if (RemovedCommonMask) {
Mask = RemainingMask ? RemainingMask : Plan.getTrue();
}
+ if (!Mask)
+ Mask = Plan.getTrue();
- VPValue *BlendMask = reconstructBlendMask(VPBB, MaskBlock, Mask);
+ auto &Defs = BlendMaskDefs[MaskBlock];
+ Defs[Plan.getVectorLoopRegion()->getEntryBasicBlock()] = Plan.getFalse();
+ Defs[MaskBlock] = Mask;
+ VPValue *BlendMask = vputils::reconstructSSA(VPBB, Defs);
LLVM_DEBUG({
VPSlotTracker SlotTracker(&Plan);
auto PrintValue = [&](VPValue *VPV) {
diff --git a/llvm/test/Transforms/LoopVectorize/VPlan/predicator.ll b/llvm/test/Transforms/LoopVectorize/VPlan/predicator.ll
index d5caf71e146d3..6662859a434fb 100644
--- a/llvm/test/Transforms/LoopVectorize/VPlan/predicator.ll
+++ b/llvm/test/Transforms/LoopVectorize/VPlan/predicator.ll
@@ -649,16 +649,14 @@ define void @blend_masks_triangle_phi_multiple_blends(ptr noalias %p, i1 %c0, i1
; CHECK-NEXT: bb2:
; CHECK-NEXT: EMIT-SCALAR vp<[[VP4:%[0-9]+]]> = phi [ ir<poison>, vector.body ], [ ir<%add1>, bb1 ]
; CHECK-NEXT: EMIT-SCALAR vp<[[VP5:%[0-9]+]]> = phi [ ir<false>, vector.body ], [ ir<true>, bb1 ]
-; CHECK-NEXT: EMIT-SCALAR vp<[[VP6:%[0-9]+]]> = phi [ ir<false>, vector.body ], [ ir<true>, bb1 ]
-; CHECK-NEXT: EMIT-SCALAR vp<[[VP7:%[0-9]+]]> = phi [ ir<false>, vector.body ], [ ir<true>, bb1 ]
-; CHECK-NEXT: EMIT vp<[[VP8:%[0-9]+]]> = not ir<%c1>
+; CHECK-NEXT: EMIT vp<[[VP6:%[0-9]+]]> = not ir<%c1>
; CHECK-NEXT: EMIT ir<%add2> = add ir<%iv>, ir<1>
; CHECK-NEXT: Successor(s): bb3
; CHECK-EMPTY:
; CHECK-NEXT: bb3:
; CHECK-NEXT: BLEND ir<%phi1> = vp<%4>/vp<[[VP5]]> ir<%add0>/ir<true>
-; CHECK-NEXT: BLEND ir<%phi2> = ir<%add0>/vp<[[VP6]]> ir<%add2>/ir<true>
-; CHECK-NEXT: BLEND ir<%phi3> = vp<%4>/vp<[[VP7]]> ir<%add2>/ir<true>
+; CHECK-NEXT: BLEND ir<%phi2> = ir<%add0>/vp<[[VP5]]> ir<%add2>/ir<true>
+; CHECK-NEXT: BLEND ir<%phi3> = vp<%4>/vp<[[VP5]]> ir<%add2>/ir<true>
; CHECK-NEXT: EMIT ir<%sum12> = add ir<%phi1>, ir<%phi2>
; CHECK-NEXT: EMIT ir<%sum123> = add ir<%sum12>, ir<%phi3>
; CHECK-NEXT: EMIT ir<%gep> = getelementptr ir<%p>, ir<%iv>
@@ -1130,9 +1128,8 @@ define void @outermost_uniform_branch_multiple_blends(ptr %a, i1 %u0) {
; CHECK-NEXT: bb4:
; CHECK-NEXT: EMIT-SCALAR vp<[[VP4:%[0-9]+]]> = phi [ ir<poison>, vector.body ], [ ir<%add3.1>, bb3 ]
; CHECK-NEXT: EMIT-SCALAR vp<[[VP5:%[0-9]+]]> = phi [ ir<false>, vector.body ], [ ir<true>, bb3 ]
-; CHECK-NEXT: EMIT-SCALAR vp<[[VP6:%[0-9]+]]> = phi [ ir<false>, vector.body ], [ ir<true>, bb3 ]
; CHECK-NEXT: BLEND ir<%phi4.1> = ir<0>/ir<true> vp<%4>/vp<[[VP5]]>
-; CHECK-NEXT: BLEND ir<%phi4.2> = ir<-1>/ir<true> vp<%4>/vp<[[VP6]]>
+; CHECK-NEXT: BLEND ir<%phi4.2> = ir<-1>/ir<true> vp<%4>/vp<[[VP5]]>
; CHECK-NEXT: EMIT ir<%sum> = add ir<%phi4.1>, ir<%phi4.2>
; CHECK-NEXT: EMIT store ir<%sum>, ir<%a>
; CHECK-NEXT: EMIT ir<%iv.next> = add nuw nsw ir<%iv>, ir<1>
>From f470fc1b5f1dec51be3001aab591bb805c33ca1a Mon Sep 17 00:00:00 2001
From: Andrei Elovikov <andrei.elovikov at sifive.com>
Date: Thu, 27 Aug 2026 14:20:00 -0700
Subject: [PATCH 11/11] Don't preserve branch-on-constant
---
.../Transforms/Vectorize/VPlanPredicator.cpp | 12 ++-
.../LoopVectorize/RISCV/dead-ops-cost.ll | 10 +--
...licate-recipe-with-only-first-lane-used.ll | 11 +--
.../LoopVectorize/blend-in-header.ll | 12 +--
.../LoopVectorize/constantfolder.ll | 77 ++++---------------
.../predicatedinst-loop-invariant.ll | 30 ++------
.../unused-blend-mask-for-first-operand.ll | 8 +-
7 files changed, 39 insertions(+), 121 deletions(-)
diff --git a/llvm/lib/Transforms/Vectorize/VPlanPredicator.cpp b/llvm/lib/Transforms/Vectorize/VPlanPredicator.cpp
index ba234e6426e9b..fc35c2abc97a0 100644
--- a/llvm/lib/Transforms/Vectorize/VPlanPredicator.cpp
+++ b/llvm/lib/Transforms/Vectorize/VPlanPredicator.cpp
@@ -286,9 +286,19 @@ bool VPPredicator::shouldPreserveTerminator(VPBasicBlock *VPBB) {
if (!Term || Term->getOpcode() != VPInstruction::BranchOnCond)
return False("Not a branch");
+ if (auto *IRV = dyn_cast<VPIRValue>(Term->getOperand(0))) {
+ if (isa<Constant>(IRV->getValue())) {
+ // `removeBranchOnConst` only does shallow traversal so can't optimize
+ // this away. Changing it to deep requires extra work as some tests
+ // crash. As such, treat as non-uniform and optimize away post
+ // linearization/predication.
+ return False("branch on constant");
+ }
+ }
+
bool IsUniformAndAvailable = [&](VPValue *V) {
auto *IRV = dyn_cast<VPIRValue>(V);
- return IRV && isa<Argument, Constant>(IRV->getValue());
+ return IRV && isa<Argument>(IRV->getValue());
}(Term->getOperand(0));
if (!IsUniformAndAvailable)
diff --git a/llvm/test/Transforms/LoopVectorize/RISCV/dead-ops-cost.ll b/llvm/test/Transforms/LoopVectorize/RISCV/dead-ops-cost.ll
index 541b2d724efae..913eba75abf17 100644
--- a/llvm/test/Transforms/LoopVectorize/RISCV/dead-ops-cost.ll
+++ b/llvm/test/Transforms/LoopVectorize/RISCV/dead-ops-cost.ll
@@ -239,16 +239,12 @@ define void @test_phi_in_latch_redundant(ptr %dst, i32 %a) {
; CHECK-NEXT: [[TMP0:%.*]] = xor i32 [[A]], -1
; CHECK-NEXT: [[BROADCAST_SPLATINSERT:%.*]] = insertelement <vscale x 4 x i32> poison, i32 [[TMP0]], i64 0
; CHECK-NEXT: [[BROADCAST_SPLAT:%.*]] = shufflevector <vscale x 4 x i32> [[BROADCAST_SPLATINSERT]], <vscale x 4 x i32> poison, <vscale x 4 x i32> zeroinitializer
+; CHECK-NEXT: [[PREDPHI:%.*]] = select i1 true, <vscale x 4 x i32> [[BROADCAST_SPLAT]], <vscale x 4 x i32> zeroinitializer
; CHECK-NEXT: br label %[[VECTOR_BODY:.*]]
; CHECK: [[VECTOR_BODY]]:
-; CHECK-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[CURRENT_ITERATION_NEXT:%.*]], %[[LOOP_LATCH2:.*]] ]
-; CHECK-NEXT: [[AVL:%.*]] = phi i64 [ 37, %[[VECTOR_PH]] ], [ [[AVL_NEXT:%.*]], %[[LOOP_LATCH2]] ]
+; CHECK-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[CURRENT_ITERATION_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-NEXT: [[AVL:%.*]] = phi i64 [ 37, %[[VECTOR_PH]] ], [ [[AVL_NEXT:%.*]], %[[VECTOR_BODY]] ]
; CHECK-NEXT: [[TMP8:%.*]] = call i32 @llvm.experimental.get.vector.length.i64(i64 [[AVL]], i32 4, i1 true)
-; CHECK-NEXT: br label %[[THEN1:.*]]
-; CHECK: [[THEN1]]:
-; CHECK-NEXT: br label %[[LOOP_LATCH2]]
-; CHECK: [[LOOP_LATCH2]]:
-; CHECK-NEXT: [[PREDPHI:%.*]] = select i1 true, <vscale x 4 x i32> [[BROADCAST_SPLAT]], <vscale x 4 x i32> zeroinitializer
; CHECK-NEXT: [[TMP2:%.*]] = mul i64 [[INDEX]], 36
; CHECK-NEXT: [[TMP3:%.*]] = getelementptr i8, ptr [[DST]], i64 [[TMP2]]
; CHECK-NEXT: call void @llvm.experimental.vp.strided.store.nxv4i32.p0.i64(<vscale x 4 x i32> [[PREDPHI]], ptr align 4 [[TMP3]], i64 36, <vscale x 4 x i1> splat (i1 true), i32 [[TMP8]])
diff --git a/llvm/test/Transforms/LoopVectorize/X86/replicate-recipe-with-only-first-lane-used.ll b/llvm/test/Transforms/LoopVectorize/X86/replicate-recipe-with-only-first-lane-used.ll
index 6068a036557b3..1e0a27b306fa3 100644
--- a/llvm/test/Transforms/LoopVectorize/X86/replicate-recipe-with-only-first-lane-used.ll
+++ b/llvm/test/Transforms/LoopVectorize/X86/replicate-recipe-with-only-first-lane-used.ll
@@ -14,19 +14,12 @@ define void @replicate_udiv_with_only_first_lane_used(i32 %x, ptr %dst, i64 %d)
; CHECK: [[VECTOR_PH]]:
; CHECK-NEXT: br label %[[VECTOR_BODY:.*]]
; CHECK: [[VECTOR_BODY]]:
-; CHECK-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[LOOP_LATCH1:.*]] ]
-; CHECK-NEXT: br label %[[LOOP_LATCH1]]
-; CHECK: [[LOOP_LATCH1]]:
-; CHECK-NEXT: [[TMP0:%.*]] = phi i64 [ poison, %[[VECTOR_BODY]] ]
-; CHECK-NEXT: [[TMP1:%.*]] = phi i1 [ false, %[[VECTOR_BODY]] ]
-; CHECK-NEXT: [[PREDPHI:%.*]] = select i1 [[TMP1]], i64 [[TMP0]], i64 0
-; CHECK-NEXT: [[TMP2:%.*]] = getelementptr i16, ptr [[DST]], i64 [[PREDPHI]]
-; CHECK-NEXT: store i16 0, ptr [[TMP2]], align 2
-; CHECK-NEXT: store i16 0, ptr [[TMP2]], align 2
+; CHECK-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 8
; CHECK-NEXT: [[TMP12:%.*]] = icmp eq i64 [[INDEX_NEXT]], 96
; CHECK-NEXT: br i1 [[TMP12]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP0:![0-9]+]]
; CHECK: [[MIDDLE_BLOCK]]:
+; CHECK-NEXT: store i16 0, ptr [[DST]], align 2
; CHECK-NEXT: br label %[[SCALAR_PH:.*]]
; CHECK: [[SCALAR_PH]]:
; CHECK-NEXT: br label %[[LOOP_HEADER:.*]]
diff --git a/llvm/test/Transforms/LoopVectorize/blend-in-header.ll b/llvm/test/Transforms/LoopVectorize/blend-in-header.ll
index 153011444af8c..d9772d357c77b 100644
--- a/llvm/test/Transforms/LoopVectorize/blend-in-header.ll
+++ b/llvm/test/Transforms/LoopVectorize/blend-in-header.ll
@@ -12,11 +12,7 @@ define i64 @pr88297() {
; CHECK: vector.ph:
; CHECK-NEXT: br label [[VECTOR_BODY:%.*]]
; CHECK: vector.body:
-; CHECK-NEXT: [[INDEX:%.*]] = phi i32 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[LOOP_LATCH2:%.*]] ]
-; CHECK-NEXT: br label [[THEN1:%.*]]
-; CHECK: then1:
-; CHECK-NEXT: br label [[LOOP_LATCH2]]
-; CHECK: loop.latch2:
+; CHECK-NEXT: [[INDEX:%.*]] = phi i32 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ]
; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i32 [[INDEX]], 4
; CHECK-NEXT: [[TMP0:%.*]] = icmp eq i32 [[INDEX_NEXT]], 1000
; CHECK-NEXT: br i1 [[TMP0]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP0:![0-9]+]]
@@ -66,11 +62,7 @@ define i64 @pr88297_incoming_ops_reordered() {
; CHECK: vector.ph:
; CHECK-NEXT: br label [[VECTOR_BODY:%.*]]
; CHECK: vector.body:
-; CHECK-NEXT: [[INDEX:%.*]] = phi i32 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[LOOP_LATCH2:%.*]] ]
-; CHECK-NEXT: br label [[THEN1:%.*]]
-; CHECK: then1:
-; CHECK-NEXT: br label [[LOOP_LATCH2]]
-; CHECK: loop.latch2:
+; CHECK-NEXT: [[INDEX:%.*]] = phi i32 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ]
; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i32 [[INDEX]], 4
; CHECK-NEXT: [[TMP0:%.*]] = icmp eq i32 [[INDEX_NEXT]], 1000
; CHECK-NEXT: br i1 [[TMP0]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP4:![0-9]+]]
diff --git a/llvm/test/Transforms/LoopVectorize/constantfolder.ll b/llvm/test/Transforms/LoopVectorize/constantfolder.ll
index d16203adfc1f4..861515e4a01b4 100644
--- a/llvm/test/Transforms/LoopVectorize/constantfolder.ll
+++ b/llvm/test/Transforms/LoopVectorize/constantfolder.ll
@@ -9,17 +9,12 @@ define void @const_fold_ptradd(ptr %dst, i64 %d) {
; CHECK: [[VECTOR_PH]]:
; CHECK-NEXT: br label %[[VECTOR_BODY:.*]]
; CHECK: [[VECTOR_BODY]]:
-; CHECK-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[LOOP_LATCH1:.*]] ]
-; CHECK-NEXT: br label %[[LOOP_LATCH1]]
-; CHECK: [[LOOP_LATCH1]]:
-; CHECK-NEXT: [[TMP0:%.*]] = phi i1 [ false, %[[VECTOR_BODY]] ]
-; CHECK-NEXT: [[PREDPHI:%.*]] = select i1 [[TMP0]], i64 [[D]], i64 0
-; CHECK-NEXT: [[TMP2:%.*]] = getelementptr i16, ptr [[DST]], i64 [[PREDPHI]]
-; CHECK-NEXT: store i16 0, ptr [[TMP2]], align 2
+; CHECK-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 4
; CHECK-NEXT: [[TMP1:%.*]] = icmp eq i64 [[INDEX_NEXT]], 100
; CHECK-NEXT: br i1 [[TMP1]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP0:![0-9]+]]
; CHECK: [[MIDDLE_BLOCK]]:
+; CHECK-NEXT: store i16 0, ptr [[DST]], align 2
; CHECK-NEXT: br label %[[EXIT:.*]]
; CHECK: [[EXIT]]:
; CHECK-NEXT: ret void
@@ -54,17 +49,12 @@ define void @const_fold_inbounds_ptradd(ptr %dst, i64 %d) {
; CHECK: [[VECTOR_PH]]:
; CHECK-NEXT: br label %[[VECTOR_BODY:.*]]
; CHECK: [[VECTOR_BODY]]:
-; CHECK-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[LOOP_LATCH1:.*]] ]
-; CHECK-NEXT: br label %[[LOOP_LATCH1]]
-; CHECK: [[LOOP_LATCH1]]:
-; CHECK-NEXT: [[TMP0:%.*]] = phi i1 [ false, %[[VECTOR_BODY]] ]
-; CHECK-NEXT: [[PREDPHI:%.*]] = select i1 [[TMP0]], i64 [[D]], i64 0
-; CHECK-NEXT: [[TMP2:%.*]] = getelementptr inbounds i16, ptr [[DST]], i64 [[PREDPHI]]
-; CHECK-NEXT: store i16 0, ptr [[TMP2]], align 2
+; CHECK-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 4
; CHECK-NEXT: [[TMP1:%.*]] = icmp eq i64 [[INDEX_NEXT]], 100
; CHECK-NEXT: br i1 [[TMP1]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP3:![0-9]+]]
; CHECK: [[MIDDLE_BLOCK]]:
+; CHECK-NEXT: store i16 0, ptr [[DST]], align 2
; CHECK-NEXT: br label %[[EXIT:.*]]
; CHECK: [[EXIT]]:
; CHECK-NEXT: ret void
@@ -97,19 +87,15 @@ define void @const_fold_select(ptr %dst, i64 %d) {
; CHECK-NEXT: [[ENTRY:.*:]]
; CHECK-NEXT: br label %[[VECTOR_PH:.*]]
; CHECK: [[VECTOR_PH]]:
+; CHECK-NEXT: [[TMP3:%.*]] = or i64 [[D]], 1
; CHECK-NEXT: br label %[[VECTOR_BODY:.*]]
; CHECK: [[VECTOR_BODY]]:
-; CHECK-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[LOOP_LATCH1:.*]] ]
-; CHECK-NEXT: br label %[[LOOP_LATCH1]]
-; CHECK: [[LOOP_LATCH1]]:
-; CHECK-NEXT: [[TMP0:%.*]] = phi i1 [ false, %[[VECTOR_BODY]] ]
-; CHECK-NEXT: [[PREDPHI:%.*]] = select i1 [[TMP0]], i64 [[D]], i64 1
-; CHECK-NEXT: [[TMP1:%.*]] = or i64 [[D]], [[PREDPHI]]
-; CHECK-NEXT: store i64 [[TMP1]], ptr [[DST]], align 8
+; CHECK-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 4
; CHECK-NEXT: [[TMP4:%.*]] = icmp eq i64 [[INDEX_NEXT]], 100
; CHECK-NEXT: br i1 [[TMP4]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP4:![0-9]+]]
; CHECK: [[MIDDLE_BLOCK]]:
+; CHECK-NEXT: store i64 [[TMP3]], ptr [[DST]], align 8
; CHECK-NEXT: br label %[[EXIT:.*]]
; CHECK: [[EXIT]]:
; CHECK-NEXT: ret void
@@ -144,21 +130,12 @@ define void @const_fold_add_sub_mul_ashr_lshr(ptr %dst, i64 %d) {
; CHECK: [[VECTOR_PH]]:
; CHECK-NEXT: br label %[[VECTOR_BODY:.*]]
; CHECK: [[VECTOR_BODY]]:
-; CHECK-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[LOOP_LATCH1:.*]] ]
-; CHECK-NEXT: br label %[[LOOP_LATCH1]]
-; CHECK: [[LOOP_LATCH1]]:
-; CHECK-NEXT: [[TMP0:%.*]] = phi i1 [ false, %[[VECTOR_BODY]] ]
-; CHECK-NEXT: [[PREDPHI:%.*]] = select i1 [[TMP0]], i64 [[D]], i64 1
-; CHECK-NEXT: [[TMP1:%.*]] = add i64 2, [[PREDPHI]]
-; CHECK-NEXT: [[TMP2:%.*]] = sub i64 [[TMP1]], [[PREDPHI]]
-; CHECK-NEXT: [[TMP3:%.*]] = ashr i64 [[TMP2]], [[PREDPHI]]
-; CHECK-NEXT: [[TMP4:%.*]] = mul i64 [[TMP3]], 3
-; CHECK-NEXT: [[TMP5:%.*]] = lshr i64 [[TMP4]], [[PREDPHI]]
-; CHECK-NEXT: store i64 [[TMP5]], ptr [[DST]], align 8
+; CHECK-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 4
; CHECK-NEXT: [[TMP6:%.*]] = icmp eq i64 [[INDEX_NEXT]], 100
; CHECK-NEXT: br i1 [[TMP6]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP5:![0-9]+]]
; CHECK: [[MIDDLE_BLOCK]]:
+; CHECK-NEXT: store i64 1, ptr [[DST]], align 8
; CHECK-NEXT: br label %[[EXIT:.*]]
; CHECK: [[EXIT]]:
; CHECK-NEXT: ret void
@@ -197,19 +174,12 @@ define void @const_fold_and_or_xor(ptr %dst, i64 %d) {
; CHECK: [[VECTOR_PH]]:
; CHECK-NEXT: br label %[[VECTOR_BODY:.*]]
; CHECK: [[VECTOR_BODY]]:
-; CHECK-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[LOOP_LATCH1:.*]] ]
-; CHECK-NEXT: br label %[[LOOP_LATCH1]]
-; CHECK: [[LOOP_LATCH1]]:
-; CHECK-NEXT: [[TMP0:%.*]] = phi i1 [ false, %[[VECTOR_BODY]] ]
-; CHECK-NEXT: [[PREDPHI:%.*]] = select i1 [[TMP0]], i64 [[D]], i64 1
-; CHECK-NEXT: [[TMP1:%.*]] = or i64 2, [[PREDPHI]]
-; CHECK-NEXT: [[TMP2:%.*]] = and i64 [[TMP1]], [[PREDPHI]]
-; CHECK-NEXT: [[TMP3:%.*]] = and i64 [[TMP2]], [[PREDPHI]]
-; CHECK-NEXT: store i64 [[TMP3]], ptr [[DST]], align 8
+; CHECK-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 4
; CHECK-NEXT: [[TMP4:%.*]] = icmp eq i64 [[INDEX_NEXT]], 100
; CHECK-NEXT: br i1 [[TMP4]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP6:![0-9]+]]
; CHECK: [[MIDDLE_BLOCK]]:
+; CHECK-NEXT: store i64 1, ptr [[DST]], align 8
; CHECK-NEXT: br label %[[EXIT:.*]]
; CHECK: [[EXIT]]:
; CHECK-NEXT: ret void
@@ -244,23 +214,14 @@ define void @const_fold_cmp_zext(ptr %dst, i64 %d) {
; CHECK-NEXT: [[ENTRY:.*:]]
; CHECK-NEXT: br label %[[VECTOR_PH:.*]]
; CHECK: [[VECTOR_PH]]:
-; CHECK-NEXT: [[BROADCAST_SPLATINSERT:%.*]] = insertelement <4 x i64> poison, i64 [[D]], i64 0
-; CHECK-NEXT: [[BROADCAST_SPLAT:%.*]] = shufflevector <4 x i64> [[BROADCAST_SPLATINSERT]], <4 x i64> poison, <4 x i32> zeroinitializer
; CHECK-NEXT: br label %[[VECTOR_BODY:.*]]
; CHECK: [[VECTOR_BODY]]:
-; CHECK-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[LOOP_LATCH1:.*]] ]
-; CHECK-NEXT: br label %[[LOOP_LATCH1]]
-; CHECK: [[LOOP_LATCH1]]:
-; CHECK-NEXT: [[TMP0:%.*]] = phi i1 [ false, %[[VECTOR_BODY]] ]
-; CHECK-NEXT: [[PREDPHI:%.*]] = select i1 [[TMP0]], <4 x i64> [[BROADCAST_SPLAT]], <4 x i64> splat (i64 1)
-; CHECK-NEXT: [[TMP1:%.*]] = icmp ugt <4 x i64> splat (i64 2), [[PREDPHI]]
-; CHECK-NEXT: [[TMP2:%.*]] = zext <4 x i1> [[TMP1]] to <4 x i8>
-; CHECK-NEXT: [[TMP4:%.*]] = extractelement <4 x i8> [[TMP2]], i64 3
-; CHECK-NEXT: store i8 [[TMP4]], ptr [[DST]], align 1
+; CHECK-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 4
; CHECK-NEXT: [[TMP3:%.*]] = icmp eq i64 [[INDEX_NEXT]], 100
; CHECK-NEXT: br i1 [[TMP3]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP7:![0-9]+]]
; CHECK: [[MIDDLE_BLOCK]]:
+; CHECK-NEXT: store i8 1, ptr [[DST]], align 1
; CHECK-NEXT: br label %[[EXIT:.*]]
; CHECK: [[EXIT]]:
; CHECK-NEXT: ret void
@@ -294,22 +255,14 @@ define void @const_fold_trunc(ptr %dst, i64 %d) {
; CHECK-NEXT: [[ENTRY:.*:]]
; CHECK-NEXT: br label %[[VECTOR_PH:.*]]
; CHECK: [[VECTOR_PH]]:
-; CHECK-NEXT: [[BROADCAST_SPLATINSERT:%.*]] = insertelement <4 x i64> poison, i64 [[D]], i64 0
-; CHECK-NEXT: [[BROADCAST_SPLAT:%.*]] = shufflevector <4 x i64> [[BROADCAST_SPLATINSERT]], <4 x i64> poison, <4 x i32> zeroinitializer
; CHECK-NEXT: br label %[[VECTOR_BODY:.*]]
; CHECK: [[VECTOR_BODY]]:
-; CHECK-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[LOOP_LATCH1:.*]] ]
-; CHECK-NEXT: br label %[[LOOP_LATCH1]]
-; CHECK: [[LOOP_LATCH1]]:
-; CHECK-NEXT: [[TMP0:%.*]] = phi i1 [ false, %[[VECTOR_BODY]] ]
-; CHECK-NEXT: [[PREDPHI:%.*]] = select i1 [[TMP0]], <4 x i64> [[BROADCAST_SPLAT]], <4 x i64> zeroinitializer
-; CHECK-NEXT: [[TMP1:%.*]] = trunc <4 x i64> [[PREDPHI]] to <4 x i16>
-; CHECK-NEXT: [[TMP3:%.*]] = extractelement <4 x i16> [[TMP1]], i64 3
-; CHECK-NEXT: store i16 [[TMP3]], ptr [[DST]], align 2
+; CHECK-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 4
; CHECK-NEXT: [[TMP2:%.*]] = icmp eq i64 [[INDEX_NEXT]], 100
; CHECK-NEXT: br i1 [[TMP2]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP8:![0-9]+]]
; CHECK: [[MIDDLE_BLOCK]]:
+; CHECK-NEXT: store i16 0, ptr [[DST]], align 2
; CHECK-NEXT: br label %[[EXIT:.*]]
; CHECK: [[EXIT]]:
; CHECK-NEXT: ret void
diff --git a/llvm/test/Transforms/LoopVectorize/predicatedinst-loop-invariant.ll b/llvm/test/Transforms/LoopVectorize/predicatedinst-loop-invariant.ll
index 3292b2776ab02..f26d0ddf66ddf 100644
--- a/llvm/test/Transforms/LoopVectorize/predicatedinst-loop-invariant.ll
+++ b/llvm/test/Transforms/LoopVectorize/predicatedinst-loop-invariant.ll
@@ -236,34 +236,14 @@ define void @test_store_to_invariant_address_needs_mask_due_to_low_trip_count(pt
; CHECK: [[VECTOR_PH]]:
; CHECK-NEXT: br label %[[VECTOR_BODY:.*]]
; CHECK: [[VECTOR_BODY]]:
+; CHECK-NEXT: store i32 1, ptr [[DST]], align 4
+; CHECK-NEXT: store i32 1, ptr [[DST]], align 4
+; CHECK-NEXT: store i32 1, ptr [[DST]], align 4
+; CHECK-NEXT: store i32 1, ptr [[DST]], align 4
; CHECK-NEXT: br label %[[MIDDLE_BLOCK:.*]]
; CHECK: [[MIDDLE_BLOCK]]:
-; CHECK-NEXT: [[TMP0:%.*]] = phi i1 [ false, %[[VECTOR_BODY]] ]
-; CHECK-NEXT: [[PREDPHI:%.*]] = select i1 [[TMP0]], i32 0, i32 1
-; CHECK-NEXT: br i1 true, label %[[PRED_STORE_IF:.*]], label %[[EXIT:.*]]
-; CHECK: [[PRED_STORE_IF]]:
-; CHECK-NEXT: store i32 [[PREDPHI]], ptr [[DST]], align 4
-; CHECK-NEXT: br label %[[EXIT]]
+; CHECK-NEXT: br label %[[EXIT:.*]]
; CHECK: [[EXIT]]:
-; CHECK-NEXT: br i1 true, label %[[PRED_STORE_IF2:.*]], label %[[PRED_STORE_CONTINUE3:.*]]
-; CHECK: [[PRED_STORE_IF2]]:
-; CHECK-NEXT: store i32 [[PREDPHI]], ptr [[DST]], align 4
-; CHECK-NEXT: br label %[[PRED_STORE_CONTINUE3]]
-; CHECK: [[PRED_STORE_CONTINUE3]]:
-; CHECK-NEXT: br i1 true, label %[[PRED_STORE_IF4:.*]], label %[[PRED_STORE_CONTINUE5:.*]]
-; CHECK: [[PRED_STORE_IF4]]:
-; CHECK-NEXT: store i32 [[PREDPHI]], ptr [[DST]], align 4
-; CHECK-NEXT: br label %[[PRED_STORE_CONTINUE5]]
-; CHECK: [[PRED_STORE_CONTINUE5]]:
-; CHECK-NEXT: br i1 false, label %[[PRED_STORE_IF6:.*]], label %[[PRED_STORE_CONTINUE7:.*]]
-; CHECK: [[PRED_STORE_IF6]]:
-; CHECK-NEXT: store i32 [[PREDPHI]], ptr [[DST]], align 4
-; CHECK-NEXT: br label %[[PRED_STORE_CONTINUE7]]
-; CHECK: [[PRED_STORE_CONTINUE7]]:
-; CHECK-NEXT: br label %[[MIDDLE_BLOCK1:.*]]
-; CHECK: [[MIDDLE_BLOCK1]]:
-; CHECK-NEXT: br label %[[EXIT1:.*]]
-; CHECK: [[EXIT1]]:
; CHECK-NEXT: ret void
;
entry:
diff --git a/llvm/test/Transforms/LoopVectorize/unused-blend-mask-for-first-operand.ll b/llvm/test/Transforms/LoopVectorize/unused-blend-mask-for-first-operand.ll
index 3d4e353c7b4fe..feab77583a594 100644
--- a/llvm/test/Transforms/LoopVectorize/unused-blend-mask-for-first-operand.ll
+++ b/llvm/test/Transforms/LoopVectorize/unused-blend-mask-for-first-operand.ll
@@ -12,15 +12,9 @@ define void @test_not_first_lane_only_constant(ptr %A, ptr noalias %B, ptr noali
; CHECK: vector.ph:
; CHECK-NEXT: br label [[VECTOR_BODY:%.*]]
; CHECK: vector.body:
-; CHECK-NEXT: [[INDEX:%.*]] = phi i32 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[ELSE_21:%.*]] ]
+; CHECK-NEXT: [[INDEX:%.*]] = phi i32 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ]
; CHECK-NEXT: [[OFFSET_IDX:%.*]] = trunc i32 [[INDEX]] to i16
; CHECK-NEXT: [[TMP1:%.*]] = getelementptr inbounds i16, ptr [[A]], i16 [[OFFSET_IDX]]
-; CHECK-NEXT: br label [[ELSE_11:%.*]]
-; CHECK: else.11:
-; CHECK-NEXT: br label [[ELSE_22:%.*]]
-; CHECK: else.22:
-; CHECK-NEXT: br label [[ELSE_21]]
-; CHECK: loop.latch3:
; CHECK-NEXT: [[TMP13:%.*]] = load i16, ptr [[B]], align 2
; CHECK-NEXT: [[BROADCAST_SPLATINSERT5:%.*]] = insertelement <4 x i16> poison, i16 [[TMP13]], i64 0
; CHECK-NEXT: [[BROADCAST_SPLAT6:%.*]] = shufflevector <4 x i16> [[BROADCAST_SPLATINSERT5]], <4 x i16> poison, <4 x i32> zeroinitializer
More information about the llvm-branch-commits
mailing list