[llvm] [VPlan] Extend licm to sink replicate stores (PR #191026)
Ramkumar Ramachandra via llvm-commits
llvm-commits at lists.llvm.org
Wed Apr 8 15:07:24 PDT 2026
https://github.com/artagnon updated https://github.com/llvm/llvm-project/pull/191026
>From 8634503a636625ef82a1099103894ba72895d81a Mon Sep 17 00:00:00 2001
From: Ramkumar Ramachandra <artagnon at tenstorrent.com>
Date: Tue, 3 Feb 2026 12:31:50 +0000
Subject: [PATCH 1/4] [VPlan] Extend licm to hoist replicate loads
This patch eliminates the need for a separate hoistInvariantLoads
transform. A planned follow-up is to get licm to sink replicate stores.
---
.../Transforms/Vectorize/LoopVectorize.cpp | 1 +
.../Transforms/Vectorize/VPlanTransforms.cpp | 96 ++++++++-----------
.../Transforms/Vectorize/VPlanTransforms.h | 7 +-
.../AArch64/conditional-branches-cost.ll | 10 +-
.../LoopVectorize/AArch64/predicated-costs.ll | 2 +-
.../LoopVectorize/AArch64/store-costs-sve.ll | 2 +-
.../first-order-recurrence-chains-vplan.ll | 16 ++--
...-order-recurrence-sink-replicate-region.ll | 10 +-
.../VPlan/vplan-print-after-all.ll | 3 +-
.../VPlan/vplan-printing-reductions.ll | 8 +-
.../LoopVectorize/VPlan/vplan-printing.ll | 2 +-
.../LoopVectorize/X86/cost-model.ll | 8 +-
.../LoopVectorize/if-pred-stores.ll | 12 ++-
.../interleaved-accesses-metadata.ll | 6 +-
.../pointer-select-runtime-checks.ll | 6 +-
llvm/test/Transforms/LoopVectorize/pr50686.ll | 6 +-
...ive-path-inner-loop-with-runtime-checks.ll | 2 +-
17 files changed, 89 insertions(+), 108 deletions(-)
diff --git a/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp b/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
index e907b5621b817..d78f5b351b213 100644
--- a/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
+++ b/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
@@ -7989,6 +7989,7 @@ void LoopVectorizationPlanner::buildVPlansWithVPRecipes(ElementCount MinVF,
RUN_VPLAN_PASS(VPlanTransforms::truncateToMinimalBitwidths, *Plan,
CM.getMinimalBitwidths());
RUN_VPLAN_PASS(VPlanTransforms::optimize, *Plan);
+ RUN_VPLAN_PASS(VPlanTransforms::licm, *Plan, PSE, *OrigLoop);
// TODO: try to put addExplicitVectorLength close to addActiveLaneMask
if (CM.foldTailWithEVL()) {
RUN_VPLAN_PASS(VPlanTransforms::addExplicitVectorLength, *Plan,
diff --git a/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp b/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp
index 5fbdb2aa98d9f..5292209af44f7 100644
--- a/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp
+++ b/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp
@@ -258,7 +258,9 @@ template <unsigned Opcode>
static SmallVector<SmallVector<VPReplicateRecipe *, 4>>
collectGroupedReplicateMemOps(
VPlan &Plan, PredicatedScalarEvolution &PSE, const Loop *L,
- function_ref<bool(VPReplicateRecipe *)> FilterFn) {
+ function_ref<bool(VPReplicateRecipe *)> FilterFn = [](VPReplicateRecipe *) {
+ return true;
+ }) {
static_assert(Opcode == Instruction::Load || Opcode == Instruction::Store,
"Only Load and Store opcodes supported");
constexpr bool IsLoad = (Opcode == Instruction::Load);
@@ -289,16 +291,14 @@ collectGroupedReplicateMemOps(
return Groups;
}
-/// Return true if we do not know how to (mechanically) hoist or sink \p R out
-/// of a loop region.
+/// Return true if we do not know how to (mechanically) hoist or sink a
+/// non-memory recipe \p R out of a loop region.
static bool cannotHoistOrSinkRecipe(const VPRecipeBase &R) {
// Assumes don't alias anything or throw; as long as they're guaranteed to
// execute, they're safe to hoist.
if (match(&R, m_Intrinsic<Intrinsic::assume>()))
return false;
- // TODO: Relax checks in the future, e.g. we could also hoist reads, if their
- // memory location is not modified in the vector loop.
if (R.mayHaveSideEffects() || R.mayReadFromMemory() || R.isPhi())
return true;
@@ -307,6 +307,30 @@ static bool cannotHoistOrSinkRecipe(const VPRecipeBase &R) {
return RepR && RepR->getOpcode() == Instruction::Alloca;
}
+/// Return true if we do not know how to (mechanically) hoist or sink a
+/// non-memory or memory recipe \p R out of a loop region.
+static bool
+cannotHoistOrSinkRecipe(VPRecipeBase &R,
+ ArrayRef<SmallVector<VPReplicateRecipe *, 4>> MemGroups,
+ VPBasicBlock *FirstBB, VPBasicBlock *LastBB) {
+ if (!isa<VPReplicateRecipe>(R) || !R.mayReadFromMemory())
+ return cannotHoistOrSinkRecipe(R);
+
+ auto FilteredMemGroup =
+ make_filter_range(MemGroups, [&R](ArrayRef<VPReplicateRecipe *> Group) {
+ return is_contained(Group, &R);
+ });
+ if (FilteredMemGroup.empty())
+ return true;
+ assert(std::distance(FilteredMemGroup.begin(), FilteredMemGroup.end()) == 1 &&
+ "Recipe expected to be found in exactly one MemGroup");
+ auto MemGroup = *FilteredMemGroup.begin();
+
+ // Check that the load doesn't alias with stores between FirstBB and LastBB.
+ auto MemLoc = vputils::getMemoryLocation(*MemGroup.front());
+ return !MemLoc || !canHoistOrSinkWithNoAliasCheck(*MemLoc, FirstBB, LastBB);
+}
+
static bool sinkScalarOperands(VPlan &Plan) {
auto Iter = vp_depth_first_deep(Plan.getEntry());
bool ScalarVFOnly = Plan.hasScalarVFOnly();
@@ -2680,10 +2704,14 @@ void VPlanTransforms::cse(VPlan &Plan) {
}
}
-/// Move loop-invariant recipes out of the vector loop region in \p Plan.
-static void licm(VPlan &Plan) {
+void VPlanTransforms::licm(VPlan &Plan, PredicatedScalarEvolution &PSE,
+ const Loop &L) {
VPBasicBlock *Preheader = Plan.getVectorPreheader();
+ // Collect loads, as hoist candidates.
+ auto LoadGroups =
+ collectGroupedReplicateMemOps<Instruction::Load>(Plan, PSE, &L);
+
// Hoist any loop invariant recipes from the vector loop region to the
// preheader. Preform a shallow traversal of the vector loop region, to
// exclude recipes in replicate regions. Since the top-level blocks in the
@@ -2695,7 +2723,9 @@ static void licm(VPlan &Plan) {
for (VPBasicBlock *VPBB : VPBlockUtils::blocksOnly<VPBasicBlock>(
vp_depth_first_shallow(LoopRegion->getEntry()))) {
for (VPRecipeBase &R : make_early_inc_range(*VPBB)) {
- if (cannotHoistOrSinkRecipe(R))
+ if (cannotHoistOrSinkRecipe(R, LoadGroups,
+ LoopRegion->getEntryBasicBlock(),
+ LoopRegion->getExitingBasicBlock()))
continue;
if (any_of(R.operands(), [](VPValue *Op) {
return !Op->isDefinedOutsideLoopRegions();
@@ -2925,9 +2955,7 @@ void VPlanTransforms::optimize(VPlan &Plan) {
RUN_VPLAN_PASS(removeDeadRecipes, Plan);
RUN_VPLAN_PASS(createAndOptimizeReplicateRegions, Plan);
- RUN_VPLAN_PASS(hoistInvariantLoads, Plan);
RUN_VPLAN_PASS(mergeBlocksIntoPredecessors, Plan);
- RUN_VPLAN_PASS(licm, Plan);
}
// Add a VPActiveLaneMaskPHIRecipe and related recipes to \p Plan and replace
@@ -4709,54 +4737,6 @@ void VPlanTransforms::materializeBroadcasts(VPlan &Plan) {
}
}
-void VPlanTransforms::hoistInvariantLoads(VPlan &Plan) {
- VPRegionBlock *LoopRegion = Plan.getVectorLoopRegion();
-
- // Collect candidate loads with invariant addresses and noalias scopes
- // metadata and memory-writing recipes with noalias metadata.
- SmallVector<std::pair<VPRecipeBase *, MemoryLocation>> CandidateLoads;
- SmallVector<MemoryLocation> Stores;
- for (VPBasicBlock *VPBB : VPBlockUtils::blocksOnly<VPBasicBlock>(
- vp_depth_first_shallow(LoopRegion->getEntry()))) {
- for (VPRecipeBase &R : *VPBB) {
- // Only handle single-scalar replicated loads with invariant addresses.
- if (auto *RepR = dyn_cast<VPReplicateRecipe>(&R)) {
- if (RepR->isPredicated() || !RepR->isSingleScalar() ||
- RepR->getOpcode() != Instruction::Load)
- continue;
-
- VPValue *Addr = RepR->getOperand(0);
- if (Addr->isDefinedOutsideLoopRegions()) {
- MemoryLocation Loc = *vputils::getMemoryLocation(*RepR);
- if (!Loc.AATags.Scope)
- continue;
- CandidateLoads.push_back({RepR, Loc});
- }
- }
- if (R.mayWriteToMemory()) {
- auto Loc = vputils::getMemoryLocation(R);
- if (!Loc || !Loc->AATags.Scope || !Loc->AATags.NoAlias)
- return;
- Stores.push_back(*Loc);
- }
- }
- }
-
- VPBasicBlock *Preheader = Plan.getVectorPreheader();
- for (auto &[LoadRecipe, LoadLoc] : CandidateLoads) {
- // Hoist the load to the preheader if it doesn't alias with any stores
- // according to the noalias metadata. Other loads should have been hoisted
- // by other passes
- const AAMDNodes &LoadAA = LoadLoc.AATags;
- if (all_of(Stores, [&](const MemoryLocation &StoreLoc) {
- return !ScopedNoAliasAAResult::mayAliasInScopes(
- LoadAA.Scope, StoreLoc.AATags.NoAlias);
- })) {
- LoadRecipe->moveBefore(*Preheader, Preheader->getFirstNonPhi());
- }
- }
-}
-
// Collect common metadata from a group of replicate recipes by intersecting
// metadata from all recipes in the group.
static VPIRMetadata getCommonMetadata(ArrayRef<VPReplicateRecipe *> Recipes) {
diff --git a/llvm/lib/Transforms/Vectorize/VPlanTransforms.h b/llvm/lib/Transforms/Vectorize/VPlanTransforms.h
index 6312b823e5d33..20d46f9b88797 100644
--- a/llvm/lib/Transforms/Vectorize/VPlanTransforms.h
+++ b/llvm/lib/Transforms/Vectorize/VPlanTransforms.h
@@ -401,10 +401,9 @@ struct VPlanTransforms {
/// Add explicit broadcasts for live-ins and VPValues defined in \p Plan's entry block if they are used as vectors.
static void materializeBroadcasts(VPlan &Plan);
- /// Hoist single-scalar loads with invariant addresses out of the vector loop
- /// to the preheader, if they are proven not to alias with any stores in the
- /// plan using noalias metadata.
- static void hoistInvariantLoads(VPlan &Plan);
+ /// Hoist or sink loop-invariant non-memory recipes, as well as unpredicated
+ /// memory recipes, out of the vector loop in \p Plan.
+ static void licm(VPlan &Plan, PredicatedScalarEvolution &PSE, const Loop &L);
/// Hoist predicated loads from the same address to the loop entry block, if
/// they are guaranteed to execute on both paths (i.e., in replicate regions
diff --git a/llvm/test/Transforms/LoopVectorize/AArch64/conditional-branches-cost.ll b/llvm/test/Transforms/LoopVectorize/AArch64/conditional-branches-cost.ll
index cec944ee831da..6ce416728d84c 100644
--- a/llvm/test/Transforms/LoopVectorize/AArch64/conditional-branches-cost.ll
+++ b/llvm/test/Transforms/LoopVectorize/AArch64/conditional-branches-cost.ll
@@ -426,10 +426,10 @@ define i32 @header_mask_and_invariant_compare(ptr %A, ptr %B, ptr %C, ptr %D, pt
; DEFAULT: [[VECTOR_PH]]:
; DEFAULT-NEXT: [[N_MOD_VF:%.*]] = urem i64 [[TMP0]], 4
; DEFAULT-NEXT: [[N_VEC:%.*]] = sub i64 [[TMP0]], [[N_MOD_VF]]
-; DEFAULT-NEXT: [[TMP6:%.*]] = load i32, ptr [[C]], align 4, !alias.scope [[META8:![0-9]+]]
+; DEFAULT-NEXT: [[TMP3:%.*]] = load i32, ptr [[A]], align 4, !alias.scope [[META8:![0-9]+]]
; DEFAULT-NEXT: [[TMP4:%.*]] = load i32, ptr [[B]], align 4, !alias.scope [[META11:![0-9]+]]
-; DEFAULT-NEXT: [[TMP3:%.*]] = load i32, ptr [[A]], align 4, !alias.scope [[META13:![0-9]+]]
; DEFAULT-NEXT: [[TMP5:%.*]] = or i32 [[TMP4]], [[TMP3]]
+; DEFAULT-NEXT: [[TMP6:%.*]] = load i32, ptr [[C]], align 4, !alias.scope [[META13:![0-9]+]]
; DEFAULT-NEXT: [[TMP7:%.*]] = icmp ugt i32 [[TMP6]], [[TMP5]]
; DEFAULT-NEXT: [[BROADCAST_SPLATINSERT:%.*]] = insertelement <4 x i1> poison, i1 [[TMP7]], i64 0
; DEFAULT-NEXT: [[BROADCAST_SPLAT:%.*]] = shufflevector <4 x i1> [[BROADCAST_SPLATINSERT]], <4 x i1> poison, <4 x i32> zeroinitializer
@@ -510,13 +510,13 @@ define i32 @header_mask_and_invariant_compare(ptr %A, ptr %B, ptr %C, ptr %D, pt
; PRED: [[VECTOR_PH]]:
; PRED-NEXT: [[TMP8:%.*]] = call i64 @llvm.vscale.i64()
; PRED-NEXT: [[TMP9:%.*]] = shl nuw i64 [[TMP8]], 2
-; PRED-NEXT: [[L_C:%.*]] = load i32, ptr [[C]], align 4, !alias.scope [[META3:![0-9]+]]
-; PRED-NEXT: [[L_B:%.*]] = load i32, ptr [[B]], align 4, !alias.scope [[META6:![0-9]+]]
-; PRED-NEXT: [[L_A:%.*]] = load i32, ptr [[A]], align 4, !alias.scope [[META8:![0-9]+]]
; PRED-NEXT: [[ACTIVE_LANE_MASK_ENTRY:%.*]] = call <vscale x 4 x i1> @llvm.get.active.lane.mask.nxv4i1.i64(i64 0, i64 [[TMP0]])
+; PRED-NEXT: [[L_A:%.*]] = load i32, ptr [[A]], align 4, !alias.scope [[META3:![0-9]+]]
+; PRED-NEXT: [[L_B:%.*]] = load i32, ptr [[B]], align 4, !alias.scope [[META6:![0-9]+]]
; PRED-NEXT: [[OR:%.*]] = or i32 [[L_B]], [[L_A]]
; PRED-NEXT: [[BROADCAST_SPLATINSERT30:%.*]] = insertelement <vscale x 4 x i32> poison, i32 [[OR]], i64 0
; PRED-NEXT: [[BROADCAST_SPLAT31:%.*]] = shufflevector <vscale x 4 x i32> [[BROADCAST_SPLATINSERT30]], <vscale x 4 x i32> poison, <vscale x 4 x i32> zeroinitializer
+; PRED-NEXT: [[L_C:%.*]] = load i32, ptr [[C]], align 4, !alias.scope [[META8:![0-9]+]]
; PRED-NEXT: [[C_0:%.*]] = icmp ugt i32 [[L_C]], [[OR]]
; PRED-NEXT: [[BROADCAST_SPLATINSERT28:%.*]] = insertelement <vscale x 4 x i1> poison, i1 [[C_0]], i64 0
; PRED-NEXT: [[BROADCAST_SPLAT29:%.*]] = shufflevector <vscale x 4 x i1> [[BROADCAST_SPLATINSERT28]], <vscale x 4 x i1> poison, <vscale x 4 x i32> zeroinitializer
diff --git a/llvm/test/Transforms/LoopVectorize/AArch64/predicated-costs.ll b/llvm/test/Transforms/LoopVectorize/AArch64/predicated-costs.ll
index d96a96836bd3a..0f8991bae574e 100644
--- a/llvm/test/Transforms/LoopVectorize/AArch64/predicated-costs.ll
+++ b/llvm/test/Transforms/LoopVectorize/AArch64/predicated-costs.ll
@@ -66,10 +66,10 @@ define void @test_predicated_load_cast_hint(ptr %dst.1, ptr %dst.2, ptr %src, i8
; CHECK-NEXT: [[CONFLICT_RDX15:%.*]] = or i1 [[CONFLICT_RDX]], [[FOUND_CONFLICT14]]
; CHECK-NEXT: br i1 [[CONFLICT_RDX15]], label %[[SCALAR_PH]], label %[[VECTOR_PH:.*]]
; CHECK: [[VECTOR_PH]]:
+; CHECK-NEXT: [[ACTIVE_LANE_MASK_ENTRY:%.*]] = call <4 x i1> @llvm.get.active.lane.mask.v4i1.i32(i32 0, i32 [[TMP2]])
; CHECK-NEXT: [[TMP28:%.*]] = load i8, ptr [[SRC]], align 1, !alias.scope [[META0:![0-9]+]], !noalias [[META3:![0-9]+]]
; CHECK-NEXT: [[BROADCAST_SPLATINSERT:%.*]] = insertelement <4 x i8> poison, i8 [[TMP28]], i64 0
; CHECK-NEXT: [[BROADCAST_SPLAT:%.*]] = shufflevector <4 x i8> [[BROADCAST_SPLATINSERT]], <4 x i8> poison, <4 x i32> zeroinitializer
-; CHECK-NEXT: [[ACTIVE_LANE_MASK_ENTRY:%.*]] = call <4 x i1> @llvm.get.active.lane.mask.v4i1.i32(i32 0, i32 [[TMP2]])
; CHECK-NEXT: [[TMP25:%.*]] = zext <4 x i8> [[BROADCAST_SPLAT]] to <4 x i64>
; CHECK-NEXT: br label %[[VECTOR_BODY:.*]]
; CHECK: [[VECTOR_BODY]]:
diff --git a/llvm/test/Transforms/LoopVectorize/AArch64/store-costs-sve.ll b/llvm/test/Transforms/LoopVectorize/AArch64/store-costs-sve.ll
index 9ee7e82a48c4e..b94dda6d82d61 100644
--- a/llvm/test/Transforms/LoopVectorize/AArch64/store-costs-sve.ll
+++ b/llvm/test/Transforms/LoopVectorize/AArch64/store-costs-sve.ll
@@ -207,10 +207,10 @@ define void @trunc_store(ptr %dst, ptr %src, i16 %x) #1 {
; PRED-NEXT: [[TMP1:%.*]] = shl nuw i64 [[TMP0]], 4
; PRED-NEXT: [[BROADCAST_SPLATINSERT:%.*]] = insertelement <vscale x 16 x i16> poison, i16 [[X]], i64 0
; PRED-NEXT: [[BROADCAST_SPLAT:%.*]] = shufflevector <vscale x 16 x i16> [[BROADCAST_SPLATINSERT]], <vscale x 16 x i16> poison, <vscale x 16 x i32> zeroinitializer
+; PRED-NEXT: [[ACTIVE_LANE_MASK_ENTRY:%.*]] = call <vscale x 16 x i1> @llvm.get.active.lane.mask.nxv16i1.i64(i64 0, i64 1000)
; PRED-NEXT: [[TMP4:%.*]] = load i64, ptr [[SRC]], align 8, !alias.scope [[META3:![0-9]+]]
; PRED-NEXT: [[BROADCAST_SPLATINSERT2:%.*]] = insertelement <vscale x 16 x i64> poison, i64 [[TMP4]], i64 0
; PRED-NEXT: [[BROADCAST_SPLAT3:%.*]] = shufflevector <vscale x 16 x i64> [[BROADCAST_SPLATINSERT2]], <vscale x 16 x i64> poison, <vscale x 16 x i32> zeroinitializer
-; PRED-NEXT: [[ACTIVE_LANE_MASK_ENTRY:%.*]] = call <vscale x 16 x i1> @llvm.get.active.lane.mask.nxv16i1.i64(i64 0, i64 1000)
; PRED-NEXT: [[TMP3:%.*]] = trunc <vscale x 16 x i64> [[BROADCAST_SPLAT3]] to <vscale x 16 x i8>
; PRED-NEXT: [[TMP2:%.*]] = trunc <vscale x 16 x i16> [[BROADCAST_SPLAT]] to <vscale x 16 x i8>
; PRED-NEXT: [[TMP5:%.*]] = and <vscale x 16 x i8> [[TMP3]], [[TMP2]]
diff --git a/llvm/test/Transforms/LoopVectorize/VPlan/first-order-recurrence-chains-vplan.ll b/llvm/test/Transforms/LoopVectorize/VPlan/first-order-recurrence-chains-vplan.ll
index a0206b2e18eb2..a502a8962b503 100644
--- a/llvm/test/Transforms/LoopVectorize/VPlan/first-order-recurrence-chains-vplan.ll
+++ b/llvm/test/Transforms/LoopVectorize/VPlan/first-order-recurrence-chains-vplan.ll
@@ -196,30 +196,30 @@ define i32 @test_chained_first_order_recurrences_4(ptr %base, i64 %x) {
; CHECK-NEXT: Successor(s): scalar.ph, vector.ph
; CHECK-EMPTY:
; CHECK-NEXT: vector.ph:
-; CHECK-NEXT: EMIT vp<[[VP3:%[0-9]+]]> = shl ir<%x>, ir<1>
; CHECK-NEXT: Successor(s): vector loop
; CHECK-EMPTY:
; CHECK-NEXT: <x1> vector loop: {
; CHECK-NEXT: vector.body:
-; CHECK-NEXT: EMIT vp<[[VP4:%[0-9]+]]> = CANONICAL-INDUCTION ir<0>, vp<%index.next>
-; CHECK-NEXT: FIRST-ORDER-RECURRENCE-PHI ir<%for.x> = phi ir<0>, vp<[[VP3]]>
+; CHECK-NEXT: EMIT vp<[[VP3:%[0-9]+]]> = CANONICAL-INDUCTION ir<0>, vp<%index.next>
+; CHECK-NEXT: FIRST-ORDER-RECURRENCE-PHI ir<%for.x> = phi ir<0>, vp<[[VP5:%[0-9]+]]>
; CHECK-NEXT: FIRST-ORDER-RECURRENCE-PHI ir<%for.y> = phi ir<0>, ir<%for.x.prev>
-; CHECK-NEXT: vp<[[VP5:%[0-9]+]]> = SCALAR-STEPS vp<[[VP4]]>, ir<1>, vp<[[VP0]]>
-; CHECK-NEXT: CLONE ir<%gep> = getelementptr ir<%base>, vp<[[VP5]]>
-; CHECK-NEXT: EMIT vp<[[VP6:%[0-9]+]]> = first-order splice ir<%for.x>, vp<[[VP3]]>
+; CHECK-NEXT: vp<[[VP4:%[0-9]+]]> = SCALAR-STEPS vp<[[VP3]]>, ir<1>, vp<[[VP0]]>
+; CHECK-NEXT: CLONE ir<%gep> = getelementptr ir<%base>, vp<[[VP4]]>
+; CHECK-NEXT: EMIT vp<[[VP5]]> = shl ir<%x>, ir<1>
+; CHECK-NEXT: EMIT vp<[[VP6:%[0-9]+]]> = first-order splice ir<%for.x>, vp<[[VP5]]>
; CHECK-NEXT: WIDEN-CAST ir<%for.x.prev> = trunc vp<[[VP6]]> to i32
; CHECK-NEXT: EMIT vp<[[VP7:%[0-9]+]]> = first-order splice ir<%for.y>, ir<%for.x.prev>
; CHECK-NEXT: WIDEN-CAST ir<%for.y.i64> = sext vp<[[VP7]]> to i64
; CHECK-NEXT: vp<[[VP8:%[0-9]+]]> = vector-pointer ir<%gep>
; CHECK-NEXT: WIDEN store vp<[[VP8]]>, ir<%for.y.i64>
-; CHECK-NEXT: EMIT vp<%index.next> = add nuw vp<[[VP4]]>, vp<[[VP1]]>
+; CHECK-NEXT: EMIT vp<%index.next> = add nuw vp<[[VP3]]>, vp<[[VP1]]>
; CHECK-NEXT: EMIT branch-on-count vp<%index.next>, vp<[[VP2]]>
; CHECK-NEXT: No successors
; CHECK-NEXT: }
; CHECK-NEXT: Successor(s): middle.block
; CHECK-EMPTY:
; CHECK-NEXT: middle.block:
-; CHECK-NEXT: EMIT vp<[[VP10:%[0-9]+]]> = extract-last-part vp<[[VP3]]>
+; CHECK-NEXT: EMIT vp<[[VP10:%[0-9]+]]> = extract-last-part vp<[[VP5]]>
; CHECK-NEXT: EMIT vp<%vector.recur.extract> = extract-last-lane vp<[[VP10]]>
; CHECK-NEXT: EMIT vp<[[VP11:%[0-9]+]]> = extract-last-part ir<%for.x.prev>
; CHECK-NEXT: EMIT vp<%vector.recur.extract>.1 = extract-last-lane vp<[[VP11]]>
diff --git a/llvm/test/Transforms/LoopVectorize/VPlan/first-order-recurrence-sink-replicate-region.ll b/llvm/test/Transforms/LoopVectorize/VPlan/first-order-recurrence-sink-replicate-region.ll
index 08720f73ef0ef..9d12607ba03f9 100644
--- a/llvm/test/Transforms/LoopVectorize/VPlan/first-order-recurrence-sink-replicate-region.ll
+++ b/llvm/test/Transforms/LoopVectorize/VPlan/first-order-recurrence-sink-replicate-region.ll
@@ -135,7 +135,6 @@ define void @sink_replicate_region_2(i32 %x, i8 %y, ptr %ptr, i32 %z) optsize {
; CHECK-NEXT: Successor(s): scalar.ph, vector.ph
; CHECK-EMPTY:
; CHECK-NEXT: vector.ph:
-; CHECK-NEXT: WIDEN-CAST ir<%recur.next> = sext ir<%y> to i32
; CHECK-NEXT: Successor(s): vector loop
; CHECK-EMPTY:
; CHECK-NEXT: <x1> vector loop: {
@@ -144,6 +143,7 @@ define void @sink_replicate_region_2(i32 %x, i8 %y, ptr %ptr, i32 %z) optsize {
; CHECK-NEXT: FIRST-ORDER-RECURRENCE-PHI ir<%recur> = phi ir<0>, ir<%recur.next>
; CHECK-NEXT: ir<%iv> = WIDEN-INDUCTION ir<0>, ir<1>, vp<[[VP0]]>
; CHECK-NEXT: EMIT vp<[[VP5:%[0-9]+]]> = icmp ule ir<%iv>, vp<[[VP3]]>
+; CHECK-NEXT: WIDEN-CAST ir<%recur.next> = sext ir<%y> to i32
; CHECK-NEXT: EMIT vp<[[VP6:%[0-9]+]]> = first-order splice ir<%recur>, ir<%recur.next>
; CHECK-NEXT: WIDEN ir<%cond> = icmp eq ir<%iv>, ir<%z>
; CHECK-NEXT: EMIT vp<[[VP7:%[0-9]+]]> = logical-and vp<[[VP5]]>, ir<%cond>
@@ -232,7 +232,6 @@ define i32 @sink_replicate_region_3_reduction(i32 %x, i8 %y, ptr %ptr) optsize {
; CHECK-EMPTY:
; CHECK-NEXT: vector.ph:
; CHECK-NEXT: EMIT vp<[[VP3:%[0-9]+]]> = reduction-start-vector ir<1234>, ir<-1>, ir<1>
-; CHECK-NEXT: WIDEN-CAST ir<%recur.next> = sext ir<%y> to i32
; CHECK-NEXT: Successor(s): vector loop
; CHECK-EMPTY:
; CHECK-NEXT: <x1> vector loop: {
@@ -242,6 +241,7 @@ define i32 @sink_replicate_region_3_reduction(i32 %x, i8 %y, ptr %ptr) optsize {
; CHECK-NEXT: WIDEN-REDUCTION-PHI ir<%and.red> = phi vp<[[VP3]]>, ir<%and.red.next>
; CHECK-NEXT: EMIT vp<[[VP5:%[0-9]+]]> = WIDEN-CANONICAL-INDUCTION vp<[[VP4]]>
; CHECK-NEXT: EMIT vp<[[VP6:%[0-9]+]]> = icmp ule vp<[[VP5]]>, vp<[[VP2]]>
+; CHECK-NEXT: WIDEN-CAST ir<%recur.next> = sext ir<%y> to i32
; CHECK-NEXT: EMIT vp<[[VP7:%[0-9]+]]> = first-order splice ir<%recur>, ir<%recur.next>
; CHECK-NEXT: Successor(s): pred.srem
; CHECK-EMPTY:
@@ -263,6 +263,7 @@ define i32 @sink_replicate_region_3_reduction(i32 %x, i8 %y, ptr %ptr) optsize {
; CHECK-NEXT: loop.0:
; CHECK-NEXT: WIDEN ir<%add> = add vp<[[VP8]]>, ir<%recur.next>
; CHECK-NEXT: WIDEN ir<%and.red.next> = and ir<%and.red>, ir<%add>
+; CHECK-NEXT: EMIT vp<[[VP9:%[0-9]+]]> = select vp<[[VP6]]>, ir<%and.red.next>, ir<%and.red>
; CHECK-NEXT: EMIT vp<%index.next> = add nuw vp<[[VP4]]>, vp<[[VP0]]>
; CHECK-NEXT: EMIT branch-on-count vp<%index.next>, vp<[[VP1]]>
; CHECK-NEXT: No successors
@@ -270,8 +271,7 @@ define i32 @sink_replicate_region_3_reduction(i32 %x, i8 %y, ptr %ptr) optsize {
; CHECK-NEXT: Successor(s): middle.block
; CHECK-EMPTY:
; CHECK-NEXT: middle.block:
-; CHECK-NEXT: EMIT vp<[[VP10:%[0-9]+]]> = select vp<[[VP6]]>, ir<%and.red.next>, ir<%and.red>
-; CHECK-NEXT: EMIT vp<[[VP11:%[0-9]+]]> = compute-reduction-result (and) vp<[[VP10]]>
+; CHECK-NEXT: EMIT vp<[[VP11:%[0-9]+]]> = compute-reduction-result (and) vp<[[VP9]]>
; CHECK-NEXT: Successor(s): ir-bb<exit>
; CHECK-EMPTY:
; CHECK-NEXT: ir-bb<exit>:
@@ -477,7 +477,6 @@ define void @sink_replicate_region_after_replicate_region(ptr %ptr, ptr noalias
; CHECK-NEXT: Successor(s): scalar.ph, vector.ph
; CHECK-EMPTY:
; CHECK-NEXT: vector.ph:
-; CHECK-NEXT: WIDEN-CAST ir<%recur.next> = sext ir<%y> to i32
; CHECK-NEXT: Successor(s): vector loop
; CHECK-EMPTY:
; CHECK-NEXT: <x1> vector loop: {
@@ -486,6 +485,7 @@ define void @sink_replicate_region_after_replicate_region(ptr %ptr, ptr noalias
; CHECK-NEXT: FIRST-ORDER-RECURRENCE-PHI ir<%recur> = phi ir<0>, ir<%recur.next>
; CHECK-NEXT: ir<%iv> = WIDEN-INDUCTION ir<0>, ir<1>, vp<[[VP0]]>
; CHECK-NEXT: EMIT vp<[[VP6:%[0-9]+]]> = icmp ule ir<%iv>, vp<[[VP3]]>
+; CHECK-NEXT: WIDEN-CAST ir<%recur.next> = sext ir<%y> to i32
; CHECK-NEXT: EMIT vp<[[VP7:%[0-9]+]]> = first-order splice ir<%recur>, ir<%recur.next>
; CHECK-NEXT: Successor(s): pred.store
; CHECK-EMPTY:
diff --git a/llvm/test/Transforms/LoopVectorize/VPlan/vplan-print-after-all.ll b/llvm/test/Transforms/LoopVectorize/VPlan/vplan-print-after-all.ll
index cba0948478955..bb20727450df3 100644
--- a/llvm/test/Transforms/LoopVectorize/VPlan/vplan-print-after-all.ll
+++ b/llvm/test/Transforms/LoopVectorize/VPlan/vplan-print-after-all.ll
@@ -32,10 +32,9 @@
; CHECK: VPlan for loop in 'foo' after removeBranchOnConst
; CHECK: VPlan for loop in 'foo' after removeDeadRecipes
; CHECK: VPlan for loop in 'foo' after createAndOptimizeReplicateRegions
-; CHECK: VPlan for loop in 'foo' after hoistInvariantLoads
; CHECK: VPlan for loop in 'foo' after mergeBlocksIntoPredecessors
-; CHECK: VPlan for loop in 'foo' after licm
; CHECK: VPlan for loop in 'foo' after VPlanTransforms::optimize
+; CHECK: VPlan for loop in 'foo' after VPlanTransforms::licm
; CHECK: VPlan for loop in 'foo' after VPlanTransforms::materializeConstantVectorTripCount
; CHECK: VPlan for loop in 'foo' after VPlanTransforms::unrollByUF
; CHECK: VPlan for loop in 'foo' after VPlanTransforms::materializePacksAndUnpacks
diff --git a/llvm/test/Transforms/LoopVectorize/VPlan/vplan-printing-reductions.ll b/llvm/test/Transforms/LoopVectorize/VPlan/vplan-printing-reductions.ll
index cd95f5cb99507..6f46c292c786a 100644
--- a/llvm/test/Transforms/LoopVectorize/VPlan/vplan-printing-reductions.ll
+++ b/llvm/test/Transforms/LoopVectorize/VPlan/vplan-printing-reductions.ll
@@ -1393,8 +1393,6 @@ define i64 @print_ext_mul_two_uses(i64 %n, ptr %a, i16 %b, i32 %c) {
; CHECK-EMPTY:
; CHECK-NEXT: vector.ph:
; CHECK-NEXT: EMIT vp<[[VP3:%[0-9]+]]> = reduction-start-vector ir<0>, ir<0>, ir<1>
-; CHECK-NEXT: WIDEN-CAST ir<%conv> = sext ir<%b> to i32
-; CHECK-NEXT: WIDEN ir<%mul> = mul ir<%conv>, ir<%conv>
; CHECK-NEXT: Successor(s): vector loop
; CHECK-EMPTY:
; CHECK-NEXT: <x1> vector loop: {
@@ -1402,6 +1400,10 @@ define i64 @print_ext_mul_two_uses(i64 %n, ptr %a, i16 %b, i32 %c) {
; CHECK-NEXT: EMIT vp<[[VP4:%[0-9]+]]> = CANONICAL-INDUCTION ir<0>, vp<%index.next>
; CHECK-NEXT: WIDEN-REDUCTION-PHI ir<%res2> = phi vp<[[VP3]]>, vp<[[VP5:%[0-9]+]]>
; CHECK-NEXT: CLONE ir<%load> = load ir<%a>
+; CHECK-NEXT: WIDEN-CAST ir<%conv> = sext ir<%b> to i32
+; CHECK-NEXT: WIDEN ir<%mul> = mul ir<%conv>, ir<%conv>
+; CHECK-NEXT: WIDEN-CAST ir<%load.ext> = sext ir<%load> to i32
+; CHECK-NEXT: WIDEN-CAST ir<%load.ext.ext> = sext ir<%load.ext> to i64
; CHECK-NEXT: EXPRESSION vp<[[VP5]]> = ir<%res2> + reduce.add (ir<%mul> zext to i64)
; CHECK-NEXT: EMIT vp<%index.next> = add nuw vp<[[VP4]]>, vp<[[VP0]]>
; CHECK-NEXT: EMIT branch-on-count vp<%index.next>, vp<[[VP1]]>
@@ -1410,8 +1412,6 @@ define i64 @print_ext_mul_two_uses(i64 %n, ptr %a, i16 %b, i32 %c) {
; CHECK-NEXT: Successor(s): middle.block
; CHECK-EMPTY:
; CHECK-NEXT: middle.block:
-; CHECK-NEXT: WIDEN-CAST ir<%load.ext> = sext ir<%load> to i32
-; CHECK-NEXT: WIDEN-CAST ir<%load.ext.ext> = sext ir<%load.ext> to i64
; CHECK-NEXT: EMIT vp<[[VP7:%[0-9]+]]> = compute-reduction-result (add, in-loop) vp<[[VP5]]>
; CHECK-NEXT: EMIT vp<[[VP8:%[0-9]+]]> = extract-last-part ir<%load.ext.ext>
; CHECK-NEXT: EMIT vp<%vector.recur.extract> = extract-last-lane vp<[[VP8]]>
diff --git a/llvm/test/Transforms/LoopVectorize/VPlan/vplan-printing.ll b/llvm/test/Transforms/LoopVectorize/VPlan/vplan-printing.ll
index 3d97a6e7215fe..047c791b2790f 100644
--- a/llvm/test/Transforms/LoopVectorize/VPlan/vplan-printing.ll
+++ b/llvm/test/Transforms/LoopVectorize/VPlan/vplan-printing.ll
@@ -567,6 +567,7 @@ define i32 @print_exit_value(ptr %ptr, i32 %off) {
; CHECK-NEXT: ir<%iv> = WIDEN-INDUCTION nsw ir<0>, ir<1>, vp<[[VP0]]>
; CHECK-NEXT: vp<[[VP4:%[0-9]+]]> = SCALAR-STEPS vp<[[VP3]]>, ir<1>, vp<[[VP0]]>
; CHECK-NEXT: CLONE ir<%gep> = getelementptr inbounds ir<%ptr>, vp<[[VP4]]>
+; CHECK-NEXT: WIDEN ir<%add> = add ir<%iv>, ir<%off>
; CHECK-NEXT: vp<[[VP5:%[0-9]+]]> = vector-pointer inbounds ir<%gep>
; CHECK-NEXT: WIDEN store vp<[[VP5]]>, ir<0>
; CHECK-NEXT: EMIT vp<%index.next> = add nuw vp<[[VP3]]>, vp<[[VP1]]>
@@ -576,7 +577,6 @@ define i32 @print_exit_value(ptr %ptr, i32 %off) {
; CHECK-NEXT: Successor(s): middle.block
; CHECK-EMPTY:
; CHECK-NEXT: middle.block:
-; CHECK-NEXT: WIDEN ir<%add> = add ir<%iv>, ir<%off>
; CHECK-NEXT: EMIT vp<[[VP7:%[0-9]+]]> = extract-last-part ir<%add>
; CHECK-NEXT: EMIT vp<[[VP8:%[0-9]+]]> = extract-last-lane vp<[[VP7]]>
; CHECK-NEXT: EMIT vp<%cmp.n> = icmp eq ir<1000>, vp<[[VP2]]>
diff --git a/llvm/test/Transforms/LoopVectorize/X86/cost-model.ll b/llvm/test/Transforms/LoopVectorize/X86/cost-model.ll
index 77f305b7115de..b9e38a2b0baf5 100644
--- a/llvm/test/Transforms/LoopVectorize/X86/cost-model.ll
+++ b/llvm/test/Transforms/LoopVectorize/X86/cost-model.ll
@@ -355,8 +355,8 @@ define void @multi_exit(ptr %dst, ptr %src.1, ptr %src.2, i64 %A, i64 %B) #0 {
; CHECK-NEXT: [[TMP19:%.*]] = icmp eq i64 [[N_MOD_VF]], 0
; CHECK-NEXT: [[TMP20:%.*]] = select i1 [[TMP19]], i64 4, i64 [[N_MOD_VF]]
; CHECK-NEXT: [[N_VEC:%.*]] = sub i64 [[TMP2]], [[TMP20]]
-; CHECK-NEXT: [[TMP21:%.*]] = load i64, ptr [[SRC_2]], align 8, !alias.scope [[META6:![0-9]+]]
; CHECK-NEXT: [[TMP22:%.*]] = trunc i64 [[N_VEC]] to i32
+; CHECK-NEXT: [[TMP21:%.*]] = load i64, ptr [[SRC_2]], align 8, !alias.scope [[META6:![0-9]+]]
; CHECK-NEXT: [[TMP23:%.*]] = icmp ne i64 [[TMP21]], 0
; CHECK-NEXT: [[BROADCAST_SPLATINSERT:%.*]] = insertelement <2 x i1> poison, i1 [[TMP23]], i64 0
; CHECK-NEXT: [[BROADCAST_SPLAT:%.*]] = shufflevector <2 x i1> [[BROADCAST_SPLATINSERT]], <2 x i1> poison, <2 x i32> zeroinitializer
@@ -774,7 +774,7 @@ define i32 @g(i64 %n) {
; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i32 [[INDEX]], 16
; CHECK-NEXT: [[VEC_IND_NEXT]] = add <4 x i32> [[STEP_ADD_3]], splat (i32 4)
; CHECK-NEXT: [[TMP19:%.*]] = icmp eq i32 [[INDEX_NEXT]], [[N_VEC]]
-; CHECK-NEXT: br i1 [[TMP19]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP25:![0-9]+]]
+; CHECK-NEXT: br i1 [[TMP19]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP23:![0-9]+]]
; CHECK: [[MIDDLE_BLOCK]]:
; CHECK-NEXT: [[BIN_RDX:%.*]] = or <4 x i32> [[TMP16]], [[TMP15]]
; CHECK-NEXT: [[BIN_RDX5:%.*]] = or <4 x i32> [[TMP17]], [[BIN_RDX]]
@@ -808,7 +808,7 @@ define i32 @g(i64 %n) {
; CHECK-NEXT: [[INDEX_NEXT15]] = add nuw i32 [[INDEX9]], 4
; CHECK-NEXT: [[VEC_IND_NEXT11]] = add <4 x i32> [[VEC_IND10]], splat (i32 4)
; CHECK-NEXT: [[TMP26:%.*]] = icmp eq i32 [[INDEX_NEXT15]], [[N_VEC8]]
-; CHECK-NEXT: br i1 [[TMP26]], label %[[VEC_EPILOG_MIDDLE_BLOCK:.*]], label %[[VEC_EPILOG_VECTOR_BODY]], !llvm.loop [[LOOP26:![0-9]+]]
+; CHECK-NEXT: br i1 [[TMP26]], label %[[VEC_EPILOG_MIDDLE_BLOCK:.*]], label %[[VEC_EPILOG_VECTOR_BODY]], !llvm.loop [[LOOP24:![0-9]+]]
; CHECK: [[VEC_EPILOG_MIDDLE_BLOCK]]:
; CHECK-NEXT: [[TMP27:%.*]] = call i32 @llvm.vector.reduce.or.v4i32(<4 x i32> [[TMP25]])
; CHECK-NEXT: [[CMP_N18:%.*]] = icmp eq i32 [[TMP1]], [[N_VEC8]]
@@ -933,7 +933,7 @@ define void @known_deref_load_tail_folding() #4 {
; CHECK: [[PRED_STORE_CONTINUE6]]:
; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 4
; CHECK-NEXT: [[TMP18:%.*]] = icmp eq i64 [[INDEX_NEXT]], 12
-; CHECK-NEXT: br i1 [[TMP18]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP28:![0-9]+]]
+; CHECK-NEXT: br i1 [[TMP18]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP26:![0-9]+]]
; CHECK: [[MIDDLE_BLOCK]]:
; CHECK-NEXT: br label %[[EXIT:.*]]
; CHECK: [[EXIT]]:
diff --git a/llvm/test/Transforms/LoopVectorize/if-pred-stores.ll b/llvm/test/Transforms/LoopVectorize/if-pred-stores.ll
index f216024303a05..06d8e37c2ea01 100644
--- a/llvm/test/Transforms/LoopVectorize/if-pred-stores.ll
+++ b/llvm/test/Transforms/LoopVectorize/if-pred-stores.ll
@@ -1032,16 +1032,18 @@ define void @hoistable_predicated_store(ptr %A, ptr %B, ptr %C, ptr %D) {
; VEC-NEXT: [[FOUND_CONFLICT17:%.*]] = and i1 [[BOUND015]], [[BOUND116]]
; VEC-NEXT: [[CONFLICT_RDX18:%.*]] = or i1 [[CONFLICT_RDX14]], [[FOUND_CONFLICT17]]
; VEC-NEXT: br i1 [[CONFLICT_RDX18]], label [[SCALAR_PH:%.*]], label [[VECTOR_BODY:%.*]]
+; VEC: vector.ph:
+; VEC-NEXT: [[TMP0:%.*]] = load i32, ptr [[A]], align 8, !alias.scope [[META15:![0-9]+]]
+; VEC-NEXT: br label [[VECTOR_BODY1:%.*]]
; VEC: vector.body:
-; VEC-NEXT: [[INDEX:%.*]] = phi i64 [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ], [ 0, [[ENTRY:%.*]] ]
-; VEC-NEXT: store i32 0, ptr [[C]], align 4, !alias.scope [[META15:![0-9]+]], !noalias [[META18:![0-9]+]]
-; VEC-NEXT: [[TMP0:%.*]] = load i32, ptr [[A]], align 8, !alias.scope [[META22:![0-9]+]]
+; VEC-NEXT: [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_BODY]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY1]] ]
+; VEC-NEXT: store i32 0, ptr [[C]], align 4, !alias.scope [[META18:![0-9]+]], !noalias [[META20:![0-9]+]]
; VEC-NEXT: store i32 [[TMP0]], ptr [[B]], align 4, !alias.scope [[META23:![0-9]+]], !noalias [[META24:![0-9]+]]
; VEC-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 2
; VEC-NEXT: [[TMP1:%.*]] = icmp eq i64 [[INDEX_NEXT]], 100
-; VEC-NEXT: br i1 [[TMP1]], label [[SCALAR_PH]], label [[VECTOR_BODY]], !llvm.loop [[LOOP25:![0-9]+]]
+; VEC-NEXT: br i1 [[TMP1]], label [[SCALAR_PH]], label [[VECTOR_BODY1]], !llvm.loop [[LOOP25:![0-9]+]]
; VEC: scalar.ph:
-; VEC-NEXT: [[BC_RESUME_VAL:%.*]] = phi i64 [ 0, [[ENTRY]] ], [ 100, [[VECTOR_BODY]] ]
+; VEC-NEXT: [[BC_RESUME_VAL:%.*]] = phi i64 [ 0, [[ENTRY:%.*]] ], [ 100, [[VECTOR_BODY1]] ]
; VEC-NEXT: br label [[LOOP:%.*]]
; VEC: loop:
; VEC-NEXT: [[IV:%.*]] = phi i64 [ [[BC_RESUME_VAL]], [[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], [[LOOP]] ]
diff --git a/llvm/test/Transforms/LoopVectorize/interleaved-accesses-metadata.ll b/llvm/test/Transforms/LoopVectorize/interleaved-accesses-metadata.ll
index 74d912352f513..ac2b3eb545dee 100644
--- a/llvm/test/Transforms/LoopVectorize/interleaved-accesses-metadata.ll
+++ b/llvm/test/Transforms/LoopVectorize/interleaved-accesses-metadata.ll
@@ -28,7 +28,7 @@ define void @merge_tbaa_interleave_group(ptr nocapture readonly %p, ptr noalias
; CHECK-NEXT: [[TMP6:%.*]] = insertelement <2 x double> poison, double [[TMP4]], i32 0
; CHECK-NEXT: [[TMP7:%.*]] = insertelement <2 x double> [[TMP6]], double [[TMP5]], i32 1
; CHECK-NEXT: [[TMP8:%.*]] = fmul <2 x double> [[TMP7]], splat (double 2.000000e+00)
-; CHECK-NEXT: [[TMP9:%.*]] = getelementptr inbounds [20 x %struct.Vec2r], ptr [[CP]], i64 0, i64 [[INDEX]], i32 0
+; CHECK-NEXT: [[TMP9:%.*]] = getelementptr inbounds [20 x [[STRUCT_VEC2R:%.*]]], ptr [[CP]], i64 0, i64 [[INDEX]], i32 0
; CHECK-NEXT: [[TMP10:%.*]] = getelementptr inbounds [[STRUCT_VEC4R]], ptr [[P]], i64 [[INDEX]], i32 1
; CHECK-NEXT: [[TMP11:%.*]] = getelementptr inbounds [[STRUCT_VEC4R]], ptr [[P]], i64 [[TMP1]], i32 1
; CHECK-NEXT: [[TMP12:%.*]] = load double, ptr [[TMP10]], align 8, !tbaa [[TBAA5:![0-9]+]]
@@ -101,11 +101,11 @@ define void @ir_tbaa_different(ptr %base, ptr %end, ptr %src) {
; CHECK: [[VECTOR_PH]]:
; CHECK-NEXT: [[N_MOD_VF:%.*]] = urem i64 [[TMP3]], 2
; CHECK-NEXT: [[N_VEC:%.*]] = sub i64 [[TMP3]], [[N_MOD_VF]]
+; CHECK-NEXT: [[TMP4:%.*]] = mul i64 [[N_VEC]], 8
+; CHECK-NEXT: [[TMP5:%.*]] = getelementptr i8, ptr [[BASE]], i64 [[TMP4]]
; CHECK-NEXT: [[TMP11:%.*]] = load float, ptr [[SRC]], align 4, !alias.scope [[META10:![0-9]+]]
; CHECK-NEXT: [[BROADCAST_SPLATINSERT:%.*]] = insertelement <2 x float> poison, float [[TMP11]], i64 0
; CHECK-NEXT: [[BROADCAST_SPLAT:%.*]] = shufflevector <2 x float> [[BROADCAST_SPLATINSERT]], <2 x float> poison, <2 x i32> zeroinitializer
-; CHECK-NEXT: [[TMP4:%.*]] = mul i64 [[N_VEC]], 8
-; CHECK-NEXT: [[TMP5:%.*]] = getelementptr i8, ptr [[BASE]], i64 [[TMP4]]
; CHECK-NEXT: br label %[[VECTOR_BODY:.*]]
; CHECK: [[VECTOR_BODY]]:
; CHECK-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
diff --git a/llvm/test/Transforms/LoopVectorize/pointer-select-runtime-checks.ll b/llvm/test/Transforms/LoopVectorize/pointer-select-runtime-checks.ll
index 66ab7939c3cac..9b383bcb88a69 100644
--- a/llvm/test/Transforms/LoopVectorize/pointer-select-runtime-checks.ll
+++ b/llvm/test/Transforms/LoopVectorize/pointer-select-runtime-checks.ll
@@ -23,8 +23,8 @@ define void @test1_select_invariant(ptr %src.1, ptr %src.2, ptr %dst, i1 %c, i8
; CHECK: vector.ph:
; CHECK-NEXT: [[N_MOD_VF:%.*]] = urem i32 [[TMP2]], 2
; CHECK-NEXT: [[N_VEC:%.*]] = sub i32 [[TMP2]], [[N_MOD_VF]]
-; CHECK-NEXT: [[TMP6:%.*]] = load i8, ptr [[PTR_SEL]], align 8, !alias.scope [[META0:![0-9]+]]
; CHECK-NEXT: [[IND_END:%.*]] = trunc i32 [[N_VEC]] to i8
+; CHECK-NEXT: [[TMP7:%.*]] = load i8, ptr [[PTR_SEL]], align 8, !alias.scope [[META0:![0-9]+]]
; CHECK-NEXT: br label [[VECTOR_BODY:%.*]]
; CHECK: vector.body:
; CHECK-NEXT: [[INDEX:%.*]] = phi i32 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ]
@@ -32,8 +32,8 @@ define void @test1_select_invariant(ptr %src.1, ptr %src.2, ptr %dst, i1 %c, i8
; CHECK-NEXT: [[INDUCTION2:%.*]] = add i8 [[OFFSET_IDX]], 1
; CHECK-NEXT: [[TMP11:%.*]] = getelementptr i8, ptr [[DST]], i8 [[OFFSET_IDX]]
; CHECK-NEXT: [[TMP8:%.*]] = getelementptr i8, ptr [[DST]], i8 [[INDUCTION2]]
-; CHECK-NEXT: store i8 [[TMP6]], ptr [[TMP11]], align 2, !alias.scope [[META3:![0-9]+]], !noalias [[META0]]
-; CHECK-NEXT: store i8 [[TMP6]], ptr [[TMP8]], align 2, !alias.scope [[META3]], !noalias [[META0]]
+; CHECK-NEXT: store i8 [[TMP7]], ptr [[TMP11]], align 2, !alias.scope [[META3:![0-9]+]], !noalias [[META0]]
+; CHECK-NEXT: store i8 [[TMP7]], ptr [[TMP8]], align 2, !alias.scope [[META3]], !noalias [[META0]]
; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i32 [[INDEX]], 2
; CHECK-NEXT: [[TMP9:%.*]] = icmp eq i32 [[INDEX_NEXT]], [[N_VEC]]
; CHECK-NEXT: br i1 [[TMP9]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP5:![0-9]+]]
diff --git a/llvm/test/Transforms/LoopVectorize/pr50686.ll b/llvm/test/Transforms/LoopVectorize/pr50686.ll
index 5a56cdfefbb8f..e6db9d5e66bb8 100644
--- a/llvm/test/Transforms/LoopVectorize/pr50686.ll
+++ b/llvm/test/Transforms/LoopVectorize/pr50686.ll
@@ -15,11 +15,11 @@ define void @m(ptr nocapture %p, ptr nocapture %p2, i32 %q) {
; CHECK-NEXT: [[FOUND_CONFLICT:%.*]] = and i1 [[BOUND0]], [[BOUND1]]
; CHECK-NEXT: br i1 [[FOUND_CONFLICT]], label [[SCALAR_PH:%.*]], label [[VECTOR_PH:%.*]]
; CHECK: vector.ph:
-; CHECK-NEXT: [[TMP4:%.*]] = load i32, ptr [[ARRAYIDX9_2]], align 4, !alias.scope [[META0:![0-9]+]]
-; CHECK-NEXT: [[TMP2:%.*]] = load i32, ptr [[ARRAYIDX9_1]], align 4, !alias.scope [[META0]]
-; CHECK-NEXT: [[TMP0:%.*]] = load i32, ptr [[P2]], align 4, !alias.scope [[META0]]
+; CHECK-NEXT: [[TMP0:%.*]] = load i32, ptr [[P2]], align 4, !alias.scope [[META0:![0-9]+]]
; CHECK-NEXT: [[TMP1:%.*]] = sub nsw i32 0, [[TMP0]]
+; CHECK-NEXT: [[TMP2:%.*]] = load i32, ptr [[ARRAYIDX9_1]], align 4, !alias.scope [[META0]]
; CHECK-NEXT: [[TMP3:%.*]] = sub nsw i32 [[TMP1]], [[TMP2]]
+; CHECK-NEXT: [[TMP4:%.*]] = load i32, ptr [[ARRAYIDX9_2]], align 4, !alias.scope [[META0]]
; CHECK-NEXT: [[TMP5:%.*]] = sub nsw i32 [[TMP3]], [[TMP4]]
; CHECK-NEXT: [[BROADCAST_SPLATINSERT4:%.*]] = insertelement <4 x i32> poison, i32 [[TMP5]], i64 0
; CHECK-NEXT: [[BROADCAST_SPLAT5:%.*]] = shufflevector <4 x i32> [[BROADCAST_SPLATINSERT4]], <4 x i32> poison, <4 x i32> zeroinitializer
diff --git a/llvm/test/Transforms/LoopVectorize/vplan-native-path-inner-loop-with-runtime-checks.ll b/llvm/test/Transforms/LoopVectorize/vplan-native-path-inner-loop-with-runtime-checks.ll
index 22cf860c8b58c..f8bcad3872efb 100644
--- a/llvm/test/Transforms/LoopVectorize/vplan-native-path-inner-loop-with-runtime-checks.ll
+++ b/llvm/test/Transforms/LoopVectorize/vplan-native-path-inner-loop-with-runtime-checks.ll
@@ -50,8 +50,8 @@ define void @expand(ptr %src, ptr %dst, i64 %0) {
; CHECK: [[VECTOR_PH]]:
; CHECK-NEXT: [[N_MOD_VF:%.*]] = urem i64 [[TMP8]], 4
; CHECK-NEXT: [[N_VEC:%.*]] = sub i64 [[TMP8]], [[N_MOD_VF]]
-; CHECK-NEXT: [[TMP19:%.*]] = load double, ptr [[SRC]], align 8, !alias.scope [[META0:![0-9]+]], !noalias [[META3:![0-9]+]]
; CHECK-NEXT: [[TMP18:%.*]] = add i64 [[TMP0]], [[N_VEC]]
+; CHECK-NEXT: [[TMP19:%.*]] = load double, ptr [[SRC]], align 8, !alias.scope [[META0:![0-9]+]], !noalias [[META3:![0-9]+]]
; CHECK-NEXT: [[DOTSPLATINSERT:%.*]] = insertelement <4 x i64> poison, i64 [[TMP0]], i64 0
; CHECK-NEXT: [[DOTSPLAT:%.*]] = shufflevector <4 x i64> [[DOTSPLATINSERT]], <4 x i64> poison, <4 x i32> zeroinitializer
; CHECK-NEXT: [[INDUCTION:%.*]] = add <4 x i64> [[DOTSPLAT]], <i64 0, i64 1, i64 2, i64 3>
>From 381bde2be06e1f996da9cd66958a908b7903fd58 Mon Sep 17 00:00:00 2001
From: Ramkumar Ramachandra <artagnon at tenstorrent.com>
Date: Wed, 1 Apr 2026 13:36:54 +0100
Subject: [PATCH 2/4] [VPlan] Permit licm-sinking recipes with no users
The patch is a preparatory step for sinking invariant stores in licm. We
have extended cannotHoistOrSink to accept a flag indicating whether we
are sinking, in order to prevent sinking assumes: without this change,
there is a bad test update in tree. Unfortunately, as the only possible
instructions without users that will not be trivially dce'd, and on
which the vectorizer will not bail out are, are stores and assumes:
since we forbid assume-sinking, and store sinking will only be done in a
follow-up, we have no test changes to show. The patch has not been
marked as an NFC, because the additonal flag in cannotHoistOrSink is
used in more sinking transforms, which would prevent assumes from being
sunk.
---
.../Transforms/Vectorize/VPlanTransforms.cpp | 56 +++++++++----------
1 file changed, 26 insertions(+), 30 deletions(-)
diff --git a/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp b/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp
index 5292209af44f7..ebd2376492d3b 100644
--- a/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp
+++ b/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp
@@ -293,11 +293,13 @@ collectGroupedReplicateMemOps(
/// Return true if we do not know how to (mechanically) hoist or sink a
/// non-memory recipe \p R out of a loop region.
-static bool cannotHoistOrSinkRecipe(const VPRecipeBase &R) {
+static bool cannotHoistOrSinkRecipe(const VPRecipeBase &R,
+ bool Sinking = false) {
// Assumes don't alias anything or throw; as long as they're guaranteed to
- // execute, they're safe to hoist.
+ // execute, they're safe to hoist. They should however not be sunk, as it
+ // would destroy information.
if (match(&R, m_Intrinsic<Intrinsic::assume>()))
- return false;
+ return Sinking;
if (R.mayHaveSideEffects() || R.mayReadFromMemory() || R.isPhi())
return true;
@@ -349,7 +351,8 @@ static bool sinkScalarOperands(VPlan &Plan) {
if (!isa<VPReplicateRecipe, VPScalarIVStepsRecipe>(Candidate))
return;
- if (Candidate->getParent() == SinkTo || cannotHoistOrSinkRecipe(*Candidate))
+ if (Candidate->getParent() == SinkTo ||
+ cannotHoistOrSinkRecipe(*Candidate, /*Sinking=*/true))
return;
if (auto *RepR = dyn_cast<VPReplicateRecipe>(Candidate))
@@ -2361,7 +2364,7 @@ sinkRecurrenceUsersAfterPrevious(VPFirstOrderRecurrencePHIRecipe *FOR,
VPDT.properlyDominates(Previous, SinkCandidate))
return true;
- if (cannotHoistOrSinkRecipe(*SinkCandidate))
+ if (cannotHoistOrSinkRecipe(*SinkCandidate, /*Sinking=*/true))
return false;
WorkList.push_back(SinkCandidate);
@@ -2738,13 +2741,12 @@ void VPlanTransforms::licm(VPlan &Plan, PredicatedScalarEvolution &PSE,
#ifndef NDEBUG
VPDominatorTree VPDT(Plan);
#endif
- // Sink recipes with no users inside the vector loop region if all users are
- // in the same exit block of the region.
- // TODO: Extend to sink recipes from inner loops.
+ // Sink recipes with in the vector loop region to a unique successor of the
+ // loop region.
for (VPBasicBlock *VPBB : VPBlockUtils::blocksOnly<VPBasicBlock>(
vp_post_order_shallow(LoopRegion->getEntry()))) {
for (VPRecipeBase &R : make_early_inc_range(reverse(*VPBB))) {
- if (cannotHoistOrSinkRecipe(R))
+ if (cannotHoistOrSinkRecipe(R, /*Sinking=*/true))
continue;
if (auto *RepR = dyn_cast<VPReplicateRecipe>(&R)) {
@@ -2762,42 +2764,36 @@ void VPlanTransforms::licm(VPlan &Plan, PredicatedScalarEvolution &PSE,
// TODO: Use R.definedValues() instead of casting to VPSingleDefRecipe to
// support recipes with multiple defined values (e.g., interleaved loads).
auto *Def = cast<VPSingleDefRecipe>(&R);
- // Skip recipes without users as we cannot determine a sink block.
- // TODO: Clone sinkable recipes without users to all exit blocks to reduce
- // their execution frequency.
- if (Def->getNumUsers() == 0)
- continue;
+ // Cannot sink the recipe if the user is defined in a loop region or a
+ // non-successor of the vector loop region. Cannot sink if user is a phi
+ // either.
VPBasicBlock *SinkBB = nullptr;
- // Cannot sink the recipe if any user
- // * is defined in any loop region, or
- // * is a phi, or
- // * multiple users in different blocks.
- if (any_of(Def->users(), [&SinkBB](VPUser *U) {
+ if (any_of(Def->users(), [&SinkBB, &LoopRegion](VPUser *U) {
auto *UserR = cast<VPRecipeBase>(U);
VPBasicBlock *Parent = UserR->getParent();
- // TODO: If the user is a PHI node, we should check the block of
- // incoming value. Support PHI node users if needed.
- if (UserR->isPhi() || Parent->getEnclosingLoopRegion())
- return true;
// TODO: Support sinking when users are in multiple blocks.
if (SinkBB && SinkBB != Parent)
return true;
SinkBB = Parent;
- return false;
+ // TODO: If the user is a PHI node, we should check the block of
+ // incoming value.
+ return UserR->isPhi() || Parent->getEnclosingLoopRegion() ||
+ Parent->getSinglePredecessor() != LoopRegion;
}))
continue;
- // Only sink to dedicated exit blocks of the loop region.
- if (SinkBB->getSinglePredecessor() != LoopRegion)
+ // Attempt to set SinkBB to the LoopRegion's single successor, if one
+ // wasn't found.
+ if (!SinkBB && !(SinkBB = cast_or_null<VPBasicBlock>(
+ LoopRegion->getSingleSuccessor())))
continue;
- // TODO: This will need to be a check instead of a assert after
- // conditional branches in vectorized loops are supported.
+ // This will need to be a check instead of a assert after conditional
+ // branches in vectorized loops are supported.
assert(VPDT.properlyDominates(VPBB, SinkBB) &&
"Defining block must dominate sink block");
- // TODO: Clone the recipe if users are on multiple exit paths, instead of
- // just moving.
+ // TODO: Clone the recipe if users are on multiple exit paths.
Def->moveBefore(*SinkBB, SinkBB->getFirstNonPhi());
}
}
>From 08dd2a4ee8b085fd4cd916bdc2e0d146ec134ead Mon Sep 17 00:00:00 2001
From: Ramkumar Ramachandra <artagnon at tenstorrent.com>
Date: Wed, 8 Apr 2026 22:15:39 +0100
Subject: [PATCH 3/4] [VPlan] Improve SinkStoreInfo API (NFC)
Improve the API of SinkStoreInfo, allowing us to inline
canSinkStoreWithNoAliasCheck.
---
.../Transforms/Vectorize/VPlanTransforms.cpp | 40 ++++++++-----------
1 file changed, 16 insertions(+), 24 deletions(-)
diff --git a/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp b/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp
index ebd2376492d3b..badd94c0e5dda 100644
--- a/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp
+++ b/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp
@@ -154,7 +154,7 @@ bool VPlanTransforms::tryToConvertVPInstructionsToVPRecipes(
/// Helper for extra no-alias checks via known-safe recipe and SCEV.
class SinkStoreInfo {
- const SmallPtrSetImpl<VPRecipeBase *> &ExcludeRecipes;
+ SmallPtrSet<VPRecipeBase *, 4> ExcludeRecipes;
VPReplicateRecipe &GroupLeader;
PredicatedScalarEvolution &PSE;
const Loop &L;
@@ -199,11 +199,11 @@ class SinkStoreInfo {
}
public:
- SinkStoreInfo(const SmallPtrSetImpl<VPRecipeBase *> &ExcludeRecipes,
+ SinkStoreInfo(ArrayRef<VPReplicateRecipe *> ExcludeRecipes,
VPReplicateRecipe &GroupLeader, PredicatedScalarEvolution &PSE,
const Loop &L, VPTypeAnalysis &TypeInfo)
- : ExcludeRecipes(ExcludeRecipes), GroupLeader(GroupLeader), PSE(PSE),
- L(L), TypeInfo(TypeInfo) {}
+ : ExcludeRecipes(ExcludeRecipes.begin(), ExcludeRecipes.end()),
+ GroupLeader(GroupLeader), PSE(PSE), L(L), TypeInfo(TypeInfo) {}
/// Return true if \p R should be skipped during alias checking, either
/// because it's in the exclude set or because no-alias can be proven via
@@ -4862,25 +4862,6 @@ void VPlanTransforms::hoistPredicatedLoads(VPlan &Plan,
}
}
-static bool
-canSinkStoreWithNoAliasCheck(ArrayRef<VPReplicateRecipe *> StoresToSink,
- PredicatedScalarEvolution &PSE, const Loop &L,
- VPTypeAnalysis &TypeInfo) {
- auto StoreLoc = vputils::getMemoryLocation(*StoresToSink.front());
- if (!StoreLoc || !StoreLoc->AATags.Scope)
- return false;
-
- // When sinking a group of stores, all members of the group alias each other.
- // Skip them during the alias checks.
- SmallPtrSet<VPRecipeBase *, 4> StoresToSinkSet(StoresToSink.begin(),
- StoresToSink.end());
-
- VPBasicBlock *FirstBB = StoresToSink.front()->getParent();
- VPBasicBlock *LastBB = StoresToSink.back()->getParent();
- SinkStoreInfo SinkInfo(StoresToSinkSet, *StoresToSink[0], PSE, L, TypeInfo);
- return canHoistOrSinkWithNoAliasCheck(*StoreLoc, FirstBB, LastBB, SinkInfo);
-}
-
void VPlanTransforms::sinkPredicatedStores(VPlan &Plan,
PredicatedScalarEvolution &PSE,
const Loop *L) {
@@ -4892,7 +4873,18 @@ void VPlanTransforms::sinkPredicatedStores(VPlan &Plan,
VPTypeAnalysis TypeInfo(Plan);
for (auto &Group : Groups) {
- if (!canSinkStoreWithNoAliasCheck(Group, PSE, *L, TypeInfo))
+ // Try to use the earliest (most dominating) store to replace all others.
+ VPReplicateRecipe *EarliestStore = Group.front();
+ VPBasicBlock *FirstBB = Group.front()->getParent();
+ VPBasicBlock *LastBB = Group.back()->getParent();
+ auto StoreLoc = vputils::getMemoryLocation(*EarliestStore);
+
+ // When sinking a group of stores, all members of the group alias each
+ // other. Skip them during the alias checks.
+ if (!StoreLoc ||
+ !canHoistOrSinkWithNoAliasCheck(
+ *StoreLoc, FirstBB, LastBB,
+ SinkStoreInfo(Group, *EarliestStore, PSE, *L, TypeInfo)))
continue;
// Use the last (most dominated) store's location for the unconditional
>From e4cdf3ad0db0ac0232becace265d790f23ebf2ac Mon Sep 17 00:00:00 2001
From: Ramkumar Ramachandra <artagnon at tenstorrent.com>
Date: Wed, 8 Apr 2026 17:23:15 +0100
Subject: [PATCH 4/4] [VPlan] Extend licm to sink replicate stores
---
.../Transforms/Vectorize/VPlanTransforms.cpp | 57 ++++++++++++----
.../LoopVectorize/AArch64/reg-usage.ll | 7 +-
.../LoopVectorize/PowerPC/reg-usage.ll | 8 +--
.../VPlan/interleave-and-scalarize-only.ll | 7 +-
.../LoopVectorize/X86/cost-model.ll | 20 ++++--
.../X86/invariant-store-vectorization.ll | 6 +-
.../Transforms/LoopVectorize/X86/pr36524.ll | 12 ++--
.../LoopVectorize/X86/uniform_mem_op.ll | 4 +-
llvm/test/Transforms/LoopVectorize/assume.ll | 4 +-
.../LoopVectorize/hoist-predicated-loads.ll | 12 ++--
.../LoopVectorize/if-pred-stores.ll | 66 +++++++++----------
.../invariant-store-vectorization-2.ll | 18 ++---
.../invariant-store-vectorization.ll | 10 +--
.../test/Transforms/LoopVectorize/metadata.ll | 40 +++++------
.../LoopVectorize/operand-bundles.ll | 2 +-
.../pointer-select-runtime-checks.ll | 57 ++++++----------
...pr47343-expander-lcssa-after-cfg-update.ll | 6 +-
.../pr59319-loop-access-info-invalidation.ll | 8 +--
.../reduction-with-invariant-store.ll | 4 +-
.../Transforms/LoopVectorize/runtime-check.ll | 2 +-
.../LoopVectorize/scalable-assume.ll | 4 +-
.../LoopVectorize/single-scalar-cast-minbw.ll | 4 +-
.../LoopVectorize/skeleton-lcssa-crash.ll | 6 +-
23 files changed, 188 insertions(+), 176 deletions(-)
diff --git a/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp b/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp
index badd94c0e5dda..c9e09a849e3f0 100644
--- a/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp
+++ b/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp
@@ -152,13 +152,21 @@ bool VPlanTransforms::tryToConvertVPInstructionsToVPRecipes(
return true;
}
+/// Helper to hold data required by SinkStoreInfo.
+struct SinkStoreData {
+ PredicatedScalarEvolution &PSE;
+ const Loop &L;
+ VPTypeAnalysis &TypeInfo;
+ SinkStoreData(PredicatedScalarEvolution &PSE, const Loop &L,
+ VPTypeAnalysis &TypeInfo)
+ : PSE(PSE), L(L), TypeInfo(TypeInfo) {};
+};
+
/// Helper for extra no-alias checks via known-safe recipe and SCEV.
class SinkStoreInfo {
SmallPtrSet<VPRecipeBase *, 4> ExcludeRecipes;
VPReplicateRecipe &GroupLeader;
- PredicatedScalarEvolution &PSE;
- const Loop &L;
- VPTypeAnalysis &TypeInfo;
+ SinkStoreData Data;
// Return true if \p A and \p B are known to not alias for all VFs in the
// plan, checked via the distance between the accesses
@@ -168,21 +176,23 @@ class SinkStoreInfo {
return false;
VPValue *AddrA = A->getOperand(1);
- const SCEV *SCEVA = vputils::getSCEVExprForVPValue(AddrA, PSE, &L);
+ const SCEV *SCEVA =
+ vputils::getSCEVExprForVPValue(AddrA, Data.PSE, &Data.L);
VPValue *AddrB = B->getOperand(1);
- const SCEV *SCEVB = vputils::getSCEVExprForVPValue(AddrB, PSE, &L);
+ const SCEV *SCEVB =
+ vputils::getSCEVExprForVPValue(AddrB, Data.PSE, &Data.L);
if (isa<SCEVCouldNotCompute>(SCEVA) || isa<SCEVCouldNotCompute>(SCEVB))
return false;
const APInt *Distance;
- ScalarEvolution &SE = *PSE.getSE();
+ ScalarEvolution &SE = *Data.PSE.getSE();
if (!match(SE.getMinusSCEV(SCEVA, SCEVB), m_scev_APInt(Distance)))
return false;
const DataLayout &DL = SE.getDataLayout();
- Type *TyA = TypeInfo.inferScalarType(A->getOperand(0));
+ Type *TyA = Data.TypeInfo.inferScalarType(A->getOperand(0));
uint64_t SizeA = DL.getTypeStoreSize(TyA);
- Type *TyB = TypeInfo.inferScalarType(B->getOperand(0));
+ Type *TyB = Data.TypeInfo.inferScalarType(B->getOperand(0));
uint64_t SizeB = DL.getTypeStoreSize(TyB);
// Use the maximum store size to ensure no overlap from either direction.
@@ -203,7 +213,11 @@ class SinkStoreInfo {
VPReplicateRecipe &GroupLeader, PredicatedScalarEvolution &PSE,
const Loop &L, VPTypeAnalysis &TypeInfo)
: ExcludeRecipes(ExcludeRecipes.begin(), ExcludeRecipes.end()),
- GroupLeader(GroupLeader), PSE(PSE), L(L), TypeInfo(TypeInfo) {}
+ GroupLeader(GroupLeader), Data(PSE, L, TypeInfo) {}
+ SinkStoreInfo(ArrayRef<VPReplicateRecipe *> ExcludeRecipes,
+ VPReplicateRecipe &GroupLeader, const SinkStoreData &Data)
+ : ExcludeRecipes(ExcludeRecipes.begin(), ExcludeRecipes.end()),
+ GroupLeader(GroupLeader), Data(Data) {}
/// Return true if \p R should be skipped during alias checking, either
/// because it's in the exclude set or because no-alias can be proven via
@@ -314,9 +328,10 @@ static bool cannotHoistOrSinkRecipe(const VPRecipeBase &R,
static bool
cannotHoistOrSinkRecipe(VPRecipeBase &R,
ArrayRef<SmallVector<VPReplicateRecipe *, 4>> MemGroups,
- VPBasicBlock *FirstBB, VPBasicBlock *LastBB) {
- if (!isa<VPReplicateRecipe>(R) || !R.mayReadFromMemory())
- return cannotHoistOrSinkRecipe(R);
+ VPBasicBlock *FirstBB, VPBasicBlock *LastBB,
+ std::optional<SinkStoreData> StoreData = {}) {
+ if (!isa<VPReplicateRecipe>(R) || !R.mayReadOrWriteMemory())
+ return cannotHoistOrSinkRecipe(R, StoreData.has_value());
auto FilteredMemGroup =
make_filter_range(MemGroups, [&R](ArrayRef<VPReplicateRecipe *> Group) {
@@ -328,9 +343,15 @@ cannotHoistOrSinkRecipe(VPRecipeBase &R,
"Recipe expected to be found in exactly one MemGroup");
auto MemGroup = *FilteredMemGroup.begin();
+ std::optional<SinkStoreInfo> SinkInfo =
+ StoreData.has_value() ? std::make_optional(SinkStoreInfo(
+ MemGroup, *MemGroup.front(), *StoreData))
+ : std::nullopt;
+
// Check that the load doesn't alias with stores between FirstBB and LastBB.
auto MemLoc = vputils::getMemoryLocation(*MemGroup.front());
- return !MemLoc || !canHoistOrSinkWithNoAliasCheck(*MemLoc, FirstBB, LastBB);
+ return !MemLoc ||
+ !canHoistOrSinkWithNoAliasCheck(*MemLoc, FirstBB, LastBB, SinkInfo);
}
static bool sinkScalarOperands(VPlan &Plan) {
@@ -2741,12 +2762,20 @@ void VPlanTransforms::licm(VPlan &Plan, PredicatedScalarEvolution &PSE,
#ifndef NDEBUG
VPDominatorTree VPDT(Plan);
#endif
+ // Collect stores, as sink candidates.
+ auto StoreGroups =
+ collectGroupedReplicateMemOps<Instruction::Store>(Plan, PSE, &L);
+
// Sink recipes with in the vector loop region to a unique successor of the
// loop region.
+ VPTypeAnalysis TypeInfo(Plan);
for (VPBasicBlock *VPBB : VPBlockUtils::blocksOnly<VPBasicBlock>(
vp_post_order_shallow(LoopRegion->getEntry()))) {
for (VPRecipeBase &R : make_early_inc_range(reverse(*VPBB))) {
- if (cannotHoistOrSinkRecipe(R, /*Sinking=*/true))
+ if (cannotHoistOrSinkRecipe(R, StoreGroups,
+ LoopRegion->getEntryBasicBlock(),
+ LoopRegion->getExitingBasicBlock(),
+ SinkStoreData(PSE, L, TypeInfo)))
continue;
if (auto *RepR = dyn_cast<VPReplicateRecipe>(&R)) {
diff --git a/llvm/test/Transforms/LoopVectorize/AArch64/reg-usage.ll b/llvm/test/Transforms/LoopVectorize/AArch64/reg-usage.ll
index 19090d9525d69..243bb13674fb6 100644
--- a/llvm/test/Transforms/LoopVectorize/AArch64/reg-usage.ll
+++ b/llvm/test/Transforms/LoopVectorize/AArch64/reg-usage.ll
@@ -14,12 +14,11 @@
define void @get_invariant_reg_usage(ptr %z) {
; CHECK-LABEL: LV: Checking a loop in 'get_invariant_reg_usage'
-; CHECK: LV(REG): VF = 16
-; CHECK-NEXT: LV(REG): Found max usage: 2 item
+; CHECK: LV(REG): VF = vscale x 16
+; CHECK-NEXT: LV(REG): Found max usage: 1 item
; CHECK-NEXT: LV(REG): RegisterClass: Generic::ScalarRC, 2 registers
-; CHECK-NEXT: LV(REG): RegisterClass: Generic::VectorRC, 1 registers
; CHECK-NEXT: LV(REG): Found invariant usage: 1 item
-; CHECK-NEXT: LV(REG): RegisterClass: Generic::ScalarRC, 3 registers
+; CHECK-NEXT: LV(REG): RegisterClass: Generic::ScalarRC, 2 registers
L.entry:
%0 = load i128, ptr %z, align 16
diff --git a/llvm/test/Transforms/LoopVectorize/PowerPC/reg-usage.ll b/llvm/test/Transforms/LoopVectorize/PowerPC/reg-usage.ll
index d69d900b49732..ffe1ac54d9ace 100644
--- a/llvm/test/Transforms/LoopVectorize/PowerPC/reg-usage.ll
+++ b/llvm/test/Transforms/LoopVectorize/PowerPC/reg-usage.ll
@@ -247,12 +247,10 @@ for.end: ; preds = %for.body
define void @fp16_(ptr nocapture readonly %pIn, ptr nocapture %pOut, i32 %numRows, i32 %numCols, i32 %scale.coerce) #0 {
;CHECK-LABEL: fp16_
;CHECK: LV(REG): VF = 1
-;CHECK: LV(REG): Found max usage: 2 item
-;CHECK: LV(REG): RegisterClass: PPC::GPRRC, 3 registers
-;CHECK: LV(REG): RegisterClass: PPC::VSXRC, 2 registers
-;CHECK: LV(REG): Found invariant usage: 2 item
+;CHECK: LV(REG): Found max usage: 1 item
+;CHECK: LV(REG): RegisterClass: PPC::GPRRC, 2 registers
+;CHECK: LV(REG): Found invariant usage: 1 item
;CHECK: LV(REG): RegisterClass: PPC::GPRRC, 1 registers
-;CHECK: LV(REG): RegisterClass: PPC::VSXRC, 1 registers
entry:
%tmp.0.extract.trunc = trunc i32 %scale.coerce to i16
diff --git a/llvm/test/Transforms/LoopVectorize/VPlan/interleave-and-scalarize-only.ll b/llvm/test/Transforms/LoopVectorize/VPlan/interleave-and-scalarize-only.ll
index 0a17319c04d35..15fcfc1a3df97 100644
--- a/llvm/test/Transforms/LoopVectorize/VPlan/interleave-and-scalarize-only.ll
+++ b/llvm/test/Transforms/LoopVectorize/VPlan/interleave-and-scalarize-only.ll
@@ -307,13 +307,14 @@ define void @scalarize_ptrtoint(ptr %src, ptr %dst) {
; CHECK-NEXT: [[TMP1:%.*]] = add i64 [[INDEX]], 1
; CHECK-NEXT: [[TMP3:%.*]] = getelementptr ptr, ptr %src, i64 [[TMP1]]
; CHECK-NEXT: [[TMP5:%.*]] = load ptr, ptr [[TMP3]], align 8
+; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 2
+; CHECK-NEXT: [[TMP12:%.*]] = icmp eq i64 [[INDEX_NEXT]], 1024
+; CHECK-NEXT: br i1 [[TMP12]], label %middle.block, label %vector.body
+; CHECK: middle.block:
; CHECK-NEXT: [[TMP7:%.*]] = ptrtoint ptr [[TMP5]] to i64
; CHECK-NEXT: [[TMP9:%.*]] = add i64 [[TMP7]], 10
; CHECK-NEXT: [[TMP11:%.*]] = inttoptr i64 [[TMP9]] to ptr
; CHECK-NEXT: store ptr [[TMP11]], ptr %dst, align 8
-; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 2
-; CHECK-NEXT: [[TMP12:%.*]] = icmp eq i64 [[INDEX_NEXT]], 1024
-; CHECK-NEXT: br i1 [[TMP12]], label %middle.block, label %vector.body
entry:
br label %loop
diff --git a/llvm/test/Transforms/LoopVectorize/X86/cost-model.ll b/llvm/test/Transforms/LoopVectorize/X86/cost-model.ll
index b9e38a2b0baf5..cd632b3d80c54 100644
--- a/llvm/test/Transforms/LoopVectorize/X86/cost-model.ll
+++ b/llvm/test/Transforms/LoopVectorize/X86/cost-model.ll
@@ -367,15 +367,15 @@ define void @multi_exit(ptr %dst, ptr %src.1, ptr %src.2, i64 %A, i64 %B) #0 {
; CHECK-NEXT: [[TMP24:%.*]] = getelementptr inbounds i64, ptr [[SRC_1]], i32 [[OFFSET_IDX]]
; CHECK-NEXT: [[TMP25:%.*]] = getelementptr inbounds i64, ptr [[TMP24]], i64 2
; CHECK-NEXT: [[WIDE_LOAD:%.*]] = load <2 x i64>, ptr [[TMP25]], align 8, !alias.scope [[META9:![0-9]+]]
+; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 4
+; CHECK-NEXT: [[TMP30:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
+; CHECK-NEXT: br i1 [[TMP30]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP11:![0-9]+]]
+; CHECK: [[MIDDLE_BLOCK]]:
; CHECK-NEXT: [[TMP26:%.*]] = icmp eq <2 x i64> [[WIDE_LOAD]], zeroinitializer
; CHECK-NEXT: [[TMP27:%.*]] = and <2 x i1> [[BROADCAST_SPLAT]], [[TMP26]]
; CHECK-NEXT: [[TMP28:%.*]] = zext <2 x i1> [[TMP27]] to <2 x i8>
; CHECK-NEXT: [[TMP29:%.*]] = extractelement <2 x i8> [[TMP28]], i32 1
-; CHECK-NEXT: store i8 [[TMP29]], ptr [[DST]], align 1, !alias.scope [[META11:![0-9]+]], !noalias [[META13:![0-9]+]]
-; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 4
-; CHECK-NEXT: [[TMP30:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
-; CHECK-NEXT: br i1 [[TMP30]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP14:![0-9]+]]
-; CHECK: [[MIDDLE_BLOCK]]:
+; CHECK-NEXT: store i8 [[TMP29]], ptr [[DST]], align 1, !alias.scope [[META12:![0-9]+]], !noalias [[META14:![0-9]+]]
; CHECK-NEXT: br label %[[SCALAR_PH]]
; CHECK: [[SCALAR_PH]]:
;
@@ -496,8 +496,6 @@ define i64 @cost_assume(ptr %end, i64 %N) {
; CHECK-NEXT: [[N_MOD_VF:%.*]] = urem i64 [[TMP2]], 8
; CHECK-NEXT: [[N_VEC:%.*]] = sub i64 [[TMP2]], [[N_MOD_VF]]
; CHECK-NEXT: [[TMP3:%.*]] = icmp ne i64 [[N]], 0
-; CHECK-NEXT: tail call void @llvm.assume(i1 [[TMP3]])
-; CHECK-NEXT: tail call void @llvm.assume(i1 [[TMP3]])
; CHECK-NEXT: br label %[[VECTOR_BODY:.*]]
; CHECK: [[VECTOR_BODY]]:
; CHECK-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
@@ -509,6 +507,14 @@ define i64 @cost_assume(ptr %end, i64 %N) {
; CHECK-NEXT: [[TMP8]] = add <2 x i64> [[VEC_PHI2]], splat (i64 1)
; CHECK-NEXT: [[TMP9]] = add <2 x i64> [[VEC_PHI3]], splat (i64 1)
; CHECK-NEXT: [[TMP10]] = add <2 x i64> [[VEC_PHI4]], splat (i64 1)
+; CHECK-NEXT: tail call void @llvm.assume(i1 [[TMP3]])
+; CHECK-NEXT: tail call void @llvm.assume(i1 [[TMP3]])
+; CHECK-NEXT: tail call void @llvm.assume(i1 [[TMP3]])
+; CHECK-NEXT: tail call void @llvm.assume(i1 [[TMP3]])
+; CHECK-NEXT: tail call void @llvm.assume(i1 [[TMP3]])
+; CHECK-NEXT: tail call void @llvm.assume(i1 [[TMP3]])
+; CHECK-NEXT: tail call void @llvm.assume(i1 [[TMP3]])
+; CHECK-NEXT: tail call void @llvm.assume(i1 [[TMP3]])
; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 8
; CHECK-NEXT: [[TMP13:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
; CHECK-NEXT: br i1 [[TMP13]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP18:![0-9]+]]
diff --git a/llvm/test/Transforms/LoopVectorize/X86/invariant-store-vectorization.ll b/llvm/test/Transforms/LoopVectorize/X86/invariant-store-vectorization.ll
index 751e885733f17..6d302189c7a1f 100644
--- a/llvm/test/Transforms/LoopVectorize/X86/invariant-store-vectorization.ll
+++ b/llvm/test/Transforms/LoopVectorize/X86/invariant-store-vectorization.ll
@@ -49,11 +49,11 @@ define i32 @inv_val_store_to_inv_address_with_reduction(ptr %a, i64 %n, ptr %b)
; CHECK-NEXT: [[TMP6]] = add <16 x i32> [[VEC_PHI4]], [[WIDE_LOAD7]]
; CHECK-NEXT: [[TMP7]] = add <16 x i32> [[VEC_PHI5]], [[WIDE_LOAD8]]
; CHECK-NEXT: [[TMP8]] = add <16 x i32> [[VEC_PHI6]], [[WIDE_LOAD9]]
-; CHECK-NEXT: store i32 [[NTRUNC]], ptr [[A]], align 4, !alias.scope [[META3:![0-9]+]], !noalias [[META0]]
; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 64
; CHECK-NEXT: [[TMP9:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
-; CHECK-NEXT: br i1 [[TMP9]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP5:![0-9]+]]
+; CHECK-NEXT: br i1 [[TMP9]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP3:![0-9]+]]
; CHECK: [[MIDDLE_BLOCK]]:
+; CHECK-NEXT: store i32 [[NTRUNC]], ptr [[A]], align 4, !alias.scope [[META6:![0-9]+]], !noalias [[META0]]
; CHECK-NEXT: [[BIN_RDX:%.*]] = add <16 x i32> [[TMP6]], [[TMP5]]
; CHECK-NEXT: [[BIN_RDX10:%.*]] = add <16 x i32> [[TMP7]], [[BIN_RDX]]
; CHECK-NEXT: [[BIN_RDX11:%.*]] = add <16 x i32> [[TMP8]], [[BIN_RDX10]]
@@ -76,11 +76,11 @@ define i32 @inv_val_store_to_inv_address_with_reduction(ptr %a, i64 %n, ptr %b)
; CHECK-NEXT: [[TMP12:%.*]] = getelementptr inbounds i32, ptr [[B]], i64 [[INDEX14]]
; CHECK-NEXT: [[WIDE_LOAD16:%.*]] = load <8 x i32>, ptr [[TMP12]], align 8, !alias.scope [[META0]]
; CHECK-NEXT: [[TMP13]] = add <8 x i32> [[VEC_PHI15]], [[WIDE_LOAD16]]
-; CHECK-NEXT: store i32 [[NTRUNC]], ptr [[A]], align 4, !alias.scope [[META3]], !noalias [[META0]]
; CHECK-NEXT: [[INDEX_NEXT17]] = add nuw i64 [[INDEX14]], 8
; CHECK-NEXT: [[TMP14:%.*]] = icmp eq i64 [[INDEX_NEXT17]], [[N_VEC13]]
; CHECK-NEXT: br i1 [[TMP14]], label %[[VEC_EPILOG_MIDDLE_BLOCK:.*]], label %[[VEC_EPILOG_VECTOR_BODY]], !llvm.loop [[LOOP9:![0-9]+]]
; CHECK: [[VEC_EPILOG_MIDDLE_BLOCK]]:
+; CHECK-NEXT: store i32 [[NTRUNC]], ptr [[A]], align 4, !alias.scope [[META6]], !noalias [[META0]]
; CHECK-NEXT: [[TMP15:%.*]] = call i32 @llvm.vector.reduce.add.v8i32(<8 x i32> [[TMP13]])
; CHECK-NEXT: [[CMP_N18:%.*]] = icmp eq i64 [[SMAX2]], [[N_VEC13]]
; CHECK-NEXT: br i1 [[CMP_N18]], label %[[FOR_END]], label %[[VEC_EPILOG_SCALAR_PH]]
diff --git a/llvm/test/Transforms/LoopVectorize/X86/pr36524.ll b/llvm/test/Transforms/LoopVectorize/X86/pr36524.ll
index 1396029b20c9f..0682371da4317 100644
--- a/llvm/test/Transforms/LoopVectorize/X86/pr36524.ll
+++ b/llvm/test/Transforms/LoopVectorize/X86/pr36524.ll
@@ -19,17 +19,17 @@ define void @foo(ptr %ptr, ptr %ptr.2) {
; CHECK: vector.body:
; CHECK-NEXT: [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ]
; CHECK-NEXT: [[VEC_IND:%.*]] = phi <4 x i64> [ <i64 2, i64 3, i64 4, i64 5>, [[VECTOR_PH]] ], [ [[VEC_IND_NEXT:%.*]], [[VECTOR_BODY]] ]
-; CHECK-NEXT: [[OFFSET_IDX:%.*]] = add i64 2, [[INDEX]]
-; CHECK-NEXT: [[TMP0:%.*]] = trunc i64 [[OFFSET_IDX]] to i32
-; CHECK-NEXT: [[TMP4:%.*]] = add i32 [[TMP0]], 3
-; CHECK-NEXT: store i32 [[TMP4]], ptr [[PTR_2]], align 4, !alias.scope [[META0:![0-9]+]], !noalias [[META3:![0-9]+]]
; CHECK-NEXT: [[TMP6:%.*]] = getelementptr inbounds i64, ptr [[PTR]], i64 [[INDEX]]
-; CHECK-NEXT: store <4 x i64> [[VEC_IND]], ptr [[TMP6]], align 8, !alias.scope [[META3]]
+; CHECK-NEXT: store <4 x i64> [[VEC_IND]], ptr [[TMP6]], align 8, !alias.scope [[META0:![0-9]+]]
; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 4
; CHECK-NEXT: [[VEC_IND_NEXT]] = add nuw nsw <4 x i64> [[VEC_IND]], splat (i64 4)
; CHECK-NEXT: [[TMP8:%.*]] = icmp eq i64 [[INDEX_NEXT]], 80
-; CHECK-NEXT: br i1 [[TMP8]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP5:![0-9]+]]
+; CHECK-NEXT: br i1 [[TMP8]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP3:![0-9]+]]
; CHECK: middle.block:
+; CHECK-NEXT: [[OFFSET_IDX:%.*]] = add i64 2, [[INDEX]]
+; CHECK-NEXT: [[TMP2:%.*]] = trunc i64 [[OFFSET_IDX]] to i32
+; CHECK-NEXT: [[TMP3:%.*]] = add i32 [[TMP2]], 3
+; CHECK-NEXT: store i32 [[TMP3]], ptr [[PTR_2]], align 4, !alias.scope [[META6:![0-9]+]], !noalias [[META0]]
; CHECK-NEXT: br label [[EXIT:%.*]]
; CHECK: scalar.ph:
; CHECK-NEXT: br label [[LOOP:%.*]]
diff --git a/llvm/test/Transforms/LoopVectorize/X86/uniform_mem_op.ll b/llvm/test/Transforms/LoopVectorize/X86/uniform_mem_op.ll
index 90469ebc1cc79..76f8a523b7a4d 100644
--- a/llvm/test/Transforms/LoopVectorize/X86/uniform_mem_op.ll
+++ b/llvm/test/Transforms/LoopVectorize/X86/uniform_mem_op.ll
@@ -279,11 +279,11 @@ define void @uniform_copy(ptr %A, ptr %B) {
; CHECK-NEXT: br label [[VECTOR_BODY:%.*]]
; CHECK: vector.body:
; CHECK-NEXT: [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ]
-; CHECK-NEXT: store i32 [[TMP0]], ptr [[B]], align 4, !alias.scope [[META15:![0-9]+]], !noalias [[META12]]
; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 16
; CHECK-NEXT: [[TMP1:%.*]] = icmp eq i64 [[INDEX_NEXT]], 4096
-; CHECK-NEXT: br i1 [[TMP1]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP17:![0-9]+]]
+; CHECK-NEXT: br i1 [[TMP1]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP15:![0-9]+]]
; CHECK: middle.block:
+; CHECK-NEXT: store i32 [[TMP0]], ptr [[B]], align 4, !alias.scope [[META16:![0-9]+]], !noalias [[META12]]
; CHECK-NEXT: br label [[SCALAR_PH]]
; CHECK: scalar.ph:
; CHECK-NEXT: [[BC_RESUME_VAL:%.*]] = phi i64 [ 4096, [[MIDDLE_BLOCK]] ], [ 0, [[VECTOR_MEMCHECK]] ]
diff --git a/llvm/test/Transforms/LoopVectorize/assume.ll b/llvm/test/Transforms/LoopVectorize/assume.ll
index f43d94687de06..db9f1efe7e123 100644
--- a/llvm/test/Transforms/LoopVectorize/assume.ll
+++ b/llvm/test/Transforms/LoopVectorize/assume.ll
@@ -72,17 +72,17 @@ define void @test2(ptr noalias %a, ptr noalias %b) {
; CHECK-NEXT: [[MASKCOND4:%.*]] = icmp eq i64 [[MASKEDPTR3]], 0
; CHECK-NEXT: br label %[[VECTOR_PH:.*]]
; CHECK: [[VECTOR_PH]]:
-; CHECK-NEXT: tail call void @llvm.assume(i1 [[MASKCOND]])
-; CHECK-NEXT: tail call void @llvm.assume(i1 [[MASKCOND4]])
; CHECK-NEXT: br label %[[VECTOR_BODY:.*]]
; CHECK: [[VECTOR_BODY]]:
; CHECK-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-NEXT: tail call void @llvm.assume(i1 [[MASKCOND]])
; CHECK-NEXT: [[TMP3:%.*]] = getelementptr inbounds float, ptr [[A]], i64 [[INDEX]]
; CHECK-NEXT: [[TMP4:%.*]] = getelementptr inbounds float, ptr [[TMP3]], i64 2
; CHECK-NEXT: [[WIDE_LOAD:%.*]] = load <2 x float>, ptr [[TMP3]], align 4
; CHECK-NEXT: [[WIDE_LOAD1:%.*]] = load <2 x float>, ptr [[TMP4]], align 4
; CHECK-NEXT: [[TMP5:%.*]] = fadd <2 x float> [[WIDE_LOAD]], splat (float 1.000000e+00)
; CHECK-NEXT: [[TMP6:%.*]] = fadd <2 x float> [[WIDE_LOAD1]], splat (float 1.000000e+00)
+; CHECK-NEXT: tail call void @llvm.assume(i1 [[MASKCOND4]])
; CHECK-NEXT: [[TMP7:%.*]] = getelementptr inbounds float, ptr [[B]], i64 [[INDEX]]
; CHECK-NEXT: [[TMP8:%.*]] = getelementptr inbounds float, ptr [[TMP7]], i64 2
; CHECK-NEXT: store <2 x float> [[TMP5]], ptr [[TMP7]], align 4
diff --git a/llvm/test/Transforms/LoopVectorize/hoist-predicated-loads.ll b/llvm/test/Transforms/LoopVectorize/hoist-predicated-loads.ll
index 5ecd7e5feb7aa..5c3efd4f48fe0 100644
--- a/llvm/test/Transforms/LoopVectorize/hoist-predicated-loads.ll
+++ b/llvm/test/Transforms/LoopVectorize/hoist-predicated-loads.ll
@@ -923,12 +923,12 @@ define void @hoist_predicated_load_with_chained_geps1(ptr %dst, ptr %src, i1 %co
; CHECK-NEXT: [[TMP2:%.*]] = add i64 [[INDEX]], 1
; CHECK-NEXT: [[TMP20:%.*]] = getelementptr [11 x i16], ptr [[SRC]], i64 [[TMP2]]
; CHECK-NEXT: [[TMP21:%.*]] = getelementptr i8, ptr [[TMP20]], i64 8
-; CHECK-NEXT: [[TMP4:%.*]] = load i16, ptr [[TMP21]], align 2, !alias.scope [[META68:![0-9]+]]
-; CHECK-NEXT: store i16 [[TMP4]], ptr [[DST]], align 2, !alias.scope [[META71:![0-9]+]], !noalias [[META68]]
+; CHECK-NEXT: [[TMP3:%.*]] = load i16, ptr [[TMP21]], align 2, !alias.scope [[META68:![0-9]+]]
; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 2
; CHECK-NEXT: [[TMP26:%.*]] = icmp eq i64 [[INDEX_NEXT]], 100
-; CHECK-NEXT: br i1 [[TMP26]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP73:![0-9]+]]
+; CHECK-NEXT: br i1 [[TMP26]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP71:![0-9]+]]
; CHECK: [[MIDDLE_BLOCK]]:
+; CHECK-NEXT: store i16 [[TMP3]], ptr [[DST]], align 2, !alias.scope [[META72:![0-9]+]], !noalias [[META68]]
; CHECK-NEXT: br label %[[SCALAR_PH]]
; CHECK: [[SCALAR_PH]]:
;
@@ -982,12 +982,12 @@ define void @hoist_predicated_load_with_chained_geps2(ptr %dst, ptr %src, i1 %co
; CHECK-NEXT: [[TMP2:%.*]] = add i64 [[INDEX]], 1
; CHECK-NEXT: [[TMP4:%.*]] = getelementptr [11 x i16], ptr [[SRC]], i64 [[TMP2]]
; CHECK-NEXT: [[TMP21:%.*]] = getelementptr i8, ptr [[TMP4]], i64 8
-; CHECK-NEXT: [[TMP5:%.*]] = load i16, ptr [[TMP21]], align 2, !alias.scope [[META75:![0-9]+]]
-; CHECK-NEXT: store i16 [[TMP5]], ptr [[DST]], align 2, !alias.scope [[META78:![0-9]+]], !noalias [[META75]]
+; CHECK-NEXT: [[TMP3:%.*]] = load i16, ptr [[TMP21]], align 2, !alias.scope [[META75:![0-9]+]]
; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 2
; CHECK-NEXT: [[TMP26:%.*]] = icmp eq i64 [[INDEX_NEXT]], 100
-; CHECK-NEXT: br i1 [[TMP26]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP80:![0-9]+]]
+; CHECK-NEXT: br i1 [[TMP26]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP78:![0-9]+]]
; CHECK: [[MIDDLE_BLOCK]]:
+; CHECK-NEXT: store i16 [[TMP3]], ptr [[DST]], align 2, !alias.scope [[META79:![0-9]+]], !noalias [[META75]]
; CHECK-NEXT: br label %[[SCALAR_PH]]
; CHECK: [[SCALAR_PH]]:
;
diff --git a/llvm/test/Transforms/LoopVectorize/if-pred-stores.ll b/llvm/test/Transforms/LoopVectorize/if-pred-stores.ll
index 06d8e37c2ea01..6f408c5481cc4 100644
--- a/llvm/test/Transforms/LoopVectorize/if-pred-stores.ll
+++ b/llvm/test/Transforms/LoopVectorize/if-pred-stores.ll
@@ -722,23 +722,19 @@ define void @sinkable_predicated_store(ptr %A, ptr %B) {
; UNROLL: vector.body:
; UNROLL-NEXT: [[INDEX:%.*]] = phi i64 [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ], [ 0, [[ENTRY:%.*]] ]
; UNROLL-NEXT: [[TMP0:%.*]] = add i64 [[INDEX]], 1
-; UNROLL-NEXT: [[TMP1:%.*]] = getelementptr i32, ptr [[A]], i64 [[INDEX]]
-; UNROLL-NEXT: [[TMP2:%.*]] = getelementptr i32, ptr [[A]], i64 [[TMP0]]
-; UNROLL-NEXT: [[TMP3:%.*]] = getelementptr i32, ptr [[B]], i64 [[INDEX]]
; UNROLL-NEXT: [[TMP4:%.*]] = getelementptr i32, ptr [[B]], i64 [[TMP0]]
-; UNROLL-NEXT: [[TMP5:%.*]] = load i32, ptr [[TMP3]], align 4, !alias.scope [[META8:![0-9]+]]
-; UNROLL-NEXT: [[TMP6:%.*]] = load i32, ptr [[TMP4]], align 4, !alias.scope [[META8]]
-; UNROLL-NEXT: [[TMP7:%.*]] = icmp ne i32 [[TMP5]], 0
+; UNROLL-NEXT: [[TMP6:%.*]] = load i32, ptr [[TMP4]], align 4, !alias.scope [[META8:![0-9]+]]
+; UNROLL-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 2
+; UNROLL-NEXT: [[TMP3:%.*]] = icmp eq i64 [[INDEX_NEXT]], 100
+; UNROLL-NEXT: br i1 [[TMP3]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP11:![0-9]+]]
+; UNROLL: middle.block:
+; UNROLL-NEXT: [[TMP5:%.*]] = getelementptr i32, ptr [[A]], i64 [[TMP0]]
; UNROLL-NEXT: [[TMP8:%.*]] = icmp ne i32 [[TMP6]], 0
-; UNROLL-NEXT: [[TMP9:%.*]] = select i1 [[TMP7]], i32 1, i32 0
; UNROLL-NEXT: [[TMP10:%.*]] = select i1 [[TMP8]], i32 1, i32 0
-; UNROLL-NEXT: store i32 [[TMP9]], ptr [[TMP1]], align 4, !alias.scope [[META11:![0-9]+]], !noalias [[META8]]
-; UNROLL-NEXT: store i32 [[TMP10]], ptr [[TMP2]], align 4, !alias.scope [[META11]], !noalias [[META8]]
-; UNROLL-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 2
-; UNROLL-NEXT: [[TMP11:%.*]] = icmp eq i64 [[INDEX_NEXT]], 100
-; UNROLL-NEXT: br i1 [[TMP11]], label [[SCALAR_PH]], label [[VECTOR_BODY]], !llvm.loop [[LOOP13:![0-9]+]]
+; UNROLL-NEXT: store i32 [[TMP10]], ptr [[TMP5]], align 4, !alias.scope [[META12:![0-9]+]], !noalias [[META8]]
+; UNROLL-NEXT: br label [[SCALAR_PH]]
; UNROLL: scalar.ph:
-; UNROLL-NEXT: [[BC_RESUME_VAL:%.*]] = phi i64 [ 0, [[ENTRY]] ], [ 100, [[VECTOR_BODY]] ]
+; UNROLL-NEXT: [[BC_RESUME_VAL:%.*]] = phi i64 [ 100, [[MIDDLE_BLOCK]] ], [ 0, [[ENTRY]] ]
; UNROLL-NEXT: br label [[LOOP:%.*]]
; UNROLL: loop:
; UNROLL-NEXT: [[IV:%.*]] = phi i64 [ [[BC_RESUME_VAL]], [[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], [[LOOP_LATCH:%.*]] ]
@@ -775,22 +771,16 @@ define void @sinkable_predicated_store(ptr %A, ptr %B) {
; UNROLL-NOSIMPLIFY: vector.body:
; UNROLL-NOSIMPLIFY-NEXT: [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ]
; UNROLL-NOSIMPLIFY-NEXT: [[TMP0:%.*]] = add i64 [[INDEX]], 1
-; UNROLL-NOSIMPLIFY-NEXT: [[TMP1:%.*]] = getelementptr i32, ptr [[A]], i64 [[INDEX]]
-; UNROLL-NOSIMPLIFY-NEXT: [[TMP2:%.*]] = getelementptr i32, ptr [[A]], i64 [[TMP0]]
-; UNROLL-NOSIMPLIFY-NEXT: [[TMP3:%.*]] = getelementptr i32, ptr [[B]], i64 [[INDEX]]
; UNROLL-NOSIMPLIFY-NEXT: [[TMP4:%.*]] = getelementptr i32, ptr [[B]], i64 [[TMP0]]
-; UNROLL-NOSIMPLIFY-NEXT: [[TMP5:%.*]] = load i32, ptr [[TMP3]], align 4, !alias.scope [[META9:![0-9]+]]
-; UNROLL-NOSIMPLIFY-NEXT: [[TMP6:%.*]] = load i32, ptr [[TMP4]], align 4, !alias.scope [[META9]]
-; UNROLL-NOSIMPLIFY-NEXT: [[TMP7:%.*]] = icmp ne i32 [[TMP5]], 0
-; UNROLL-NOSIMPLIFY-NEXT: [[TMP8:%.*]] = icmp ne i32 [[TMP6]], 0
-; UNROLL-NOSIMPLIFY-NEXT: [[TMP9:%.*]] = select i1 [[TMP7]], i32 1, i32 0
-; UNROLL-NOSIMPLIFY-NEXT: [[TMP10:%.*]] = select i1 [[TMP8]], i32 1, i32 0
-; UNROLL-NOSIMPLIFY-NEXT: store i32 [[TMP9]], ptr [[TMP1]], align 4, !alias.scope [[META12:![0-9]+]], !noalias [[META9]]
-; UNROLL-NOSIMPLIFY-NEXT: store i32 [[TMP10]], ptr [[TMP2]], align 4, !alias.scope [[META12]], !noalias [[META9]]
+; UNROLL-NOSIMPLIFY-NEXT: [[TMP2:%.*]] = load i32, ptr [[TMP4]], align 4, !alias.scope [[META9:![0-9]+]]
; UNROLL-NOSIMPLIFY-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 2
; UNROLL-NOSIMPLIFY-NEXT: [[TMP11:%.*]] = icmp eq i64 [[INDEX_NEXT]], 100
-; UNROLL-NOSIMPLIFY-NEXT: br i1 [[TMP11]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP14:![0-9]+]]
+; UNROLL-NOSIMPLIFY-NEXT: br i1 [[TMP11]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP12:![0-9]+]]
; UNROLL-NOSIMPLIFY: middle.block:
+; UNROLL-NOSIMPLIFY-NEXT: [[TMP7:%.*]] = getelementptr i32, ptr [[A]], i64 [[TMP0]]
+; UNROLL-NOSIMPLIFY-NEXT: [[TMP5:%.*]] = icmp ne i32 [[TMP2]], 0
+; UNROLL-NOSIMPLIFY-NEXT: [[TMP6:%.*]] = select i1 [[TMP5]], i32 1, i32 0
+; UNROLL-NOSIMPLIFY-NEXT: store i32 [[TMP6]], ptr [[TMP7]], align 4, !alias.scope [[META13:![0-9]+]], !noalias [[META9]]
; UNROLL-NOSIMPLIFY-NEXT: br label [[SCALAR_PH]]
; UNROLL-NOSIMPLIFY: scalar.ph:
; UNROLL-NOSIMPLIFY-NEXT: [[BC_RESUME_VAL:%.*]] = phi i64 [ 100, [[MIDDLE_BLOCK]] ], [ 0, [[VECTOR_MEMCHECK]] ]
@@ -922,13 +912,15 @@ define void @hoistable_predicated_store(ptr %A, ptr %B, ptr %C, ptr %D) {
; UNROLL-NEXT: br label [[VECTOR_BODY:%.*]]
; UNROLL: vector.body:
; UNROLL-NEXT: [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ]
-; UNROLL-NEXT: store i32 0, ptr [[C]], align 4, !alias.scope [[META18:![0-9]+]], !noalias [[META20:![0-9]+]]
-; UNROLL-NEXT: store i32 [[TMP0]], ptr [[B]], align 4, !alias.scope [[META23:![0-9]+]], !noalias [[META24:![0-9]+]]
; UNROLL-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 2
; UNROLL-NEXT: [[TMP1:%.*]] = icmp eq i64 [[INDEX_NEXT]], 100
-; UNROLL-NEXT: br i1 [[TMP1]], label [[SCALAR_PH]], label [[VECTOR_BODY]], !llvm.loop [[LOOP25:![0-9]+]]
+; UNROLL-NEXT: br i1 [[TMP1]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP18:![0-9]+]]
+; UNROLL: middle.block:
+; UNROLL-NEXT: store i32 0, ptr [[C]], align 4, !alias.scope [[META19:![0-9]+]], !noalias [[META21:![0-9]+]]
+; UNROLL-NEXT: store i32 [[TMP0]], ptr [[B]], align 4, !alias.scope [[META24:![0-9]+]], !noalias [[META25:![0-9]+]]
+; UNROLL-NEXT: br label [[SCALAR_PH]]
; UNROLL: scalar.ph:
-; UNROLL-NEXT: [[BC_RESUME_VAL:%.*]] = phi i64 [ 0, [[ENTRY:%.*]] ], [ 100, [[VECTOR_BODY]] ]
+; UNROLL-NEXT: [[BC_RESUME_VAL:%.*]] = phi i64 [ 100, [[MIDDLE_BLOCK]] ], [ 0, [[ENTRY:%.*]] ]
; UNROLL-NEXT: br label [[LOOP:%.*]]
; UNROLL: loop:
; UNROLL-NEXT: [[IV:%.*]] = phi i64 [ [[BC_RESUME_VAL]], [[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], [[LOOP]] ]
@@ -974,12 +966,12 @@ define void @hoistable_predicated_store(ptr %A, ptr %B, ptr %C, ptr %D) {
; UNROLL-NOSIMPLIFY-NEXT: br label [[VECTOR_BODY:%.*]]
; UNROLL-NOSIMPLIFY: vector.body:
; UNROLL-NOSIMPLIFY-NEXT: [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ]
-; UNROLL-NOSIMPLIFY-NEXT: store i32 0, ptr [[C]], align 4, !alias.scope [[META19:![0-9]+]], !noalias [[META21:![0-9]+]]
-; UNROLL-NOSIMPLIFY-NEXT: store i32 [[TMP0]], ptr [[B]], align 4, !alias.scope [[META24:![0-9]+]], !noalias [[META25:![0-9]+]]
; UNROLL-NOSIMPLIFY-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 2
; UNROLL-NOSIMPLIFY-NEXT: [[TMP1:%.*]] = icmp eq i64 [[INDEX_NEXT]], 100
-; UNROLL-NOSIMPLIFY-NEXT: br i1 [[TMP1]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP26:![0-9]+]]
+; UNROLL-NOSIMPLIFY-NEXT: br i1 [[TMP1]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP19:![0-9]+]]
; UNROLL-NOSIMPLIFY: middle.block:
+; UNROLL-NOSIMPLIFY-NEXT: store i32 0, ptr [[C]], align 4, !alias.scope [[META20:![0-9]+]], !noalias [[META22:![0-9]+]]
+; UNROLL-NOSIMPLIFY-NEXT: store i32 [[TMP0]], ptr [[B]], align 4, !alias.scope [[META25:![0-9]+]], !noalias [[META26:![0-9]+]]
; UNROLL-NOSIMPLIFY-NEXT: br label [[SCALAR_PH]]
; UNROLL-NOSIMPLIFY: scalar.ph:
; UNROLL-NOSIMPLIFY-NEXT: [[BC_RESUME_VAL:%.*]] = phi i64 [ 100, [[MIDDLE_BLOCK]] ], [ 0, [[VECTOR_MEMCHECK]] ]
@@ -1037,13 +1029,15 @@ define void @hoistable_predicated_store(ptr %A, ptr %B, ptr %C, ptr %D) {
; VEC-NEXT: br label [[VECTOR_BODY1:%.*]]
; VEC: vector.body:
; VEC-NEXT: [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_BODY]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY1]] ]
-; VEC-NEXT: store i32 0, ptr [[C]], align 4, !alias.scope [[META18:![0-9]+]], !noalias [[META20:![0-9]+]]
-; VEC-NEXT: store i32 [[TMP0]], ptr [[B]], align 4, !alias.scope [[META23:![0-9]+]], !noalias [[META24:![0-9]+]]
; VEC-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 2
; VEC-NEXT: [[TMP1:%.*]] = icmp eq i64 [[INDEX_NEXT]], 100
-; VEC-NEXT: br i1 [[TMP1]], label [[SCALAR_PH]], label [[VECTOR_BODY1]], !llvm.loop [[LOOP25:![0-9]+]]
+; VEC-NEXT: br i1 [[TMP1]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY1]], !llvm.loop [[LOOP18:![0-9]+]]
+; VEC: middle.block:
+; VEC-NEXT: store i32 0, ptr [[C]], align 4, !alias.scope [[META19:![0-9]+]], !noalias [[META21:![0-9]+]]
+; VEC-NEXT: store i32 [[TMP0]], ptr [[B]], align 4, !alias.scope [[META24:![0-9]+]], !noalias [[META25:![0-9]+]]
+; VEC-NEXT: br label [[SCALAR_PH]]
; VEC: scalar.ph:
-; VEC-NEXT: [[BC_RESUME_VAL:%.*]] = phi i64 [ 0, [[ENTRY:%.*]] ], [ 100, [[VECTOR_BODY1]] ]
+; VEC-NEXT: [[BC_RESUME_VAL:%.*]] = phi i64 [ 100, [[MIDDLE_BLOCK]] ], [ 0, [[ENTRY:%.*]] ]
; VEC-NEXT: br label [[LOOP:%.*]]
; VEC: loop:
; VEC-NEXT: [[IV:%.*]] = phi i64 [ [[BC_RESUME_VAL]], [[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], [[LOOP]] ]
diff --git a/llvm/test/Transforms/LoopVectorize/invariant-store-vectorization-2.ll b/llvm/test/Transforms/LoopVectorize/invariant-store-vectorization-2.ll
index cea9a145c1b61..83a2eb9e9e7d2 100644
--- a/llvm/test/Transforms/LoopVectorize/invariant-store-vectorization-2.ll
+++ b/llvm/test/Transforms/LoopVectorize/invariant-store-vectorization-2.ll
@@ -43,15 +43,16 @@ define void @inv_val_store_to_inv_address_conditional_diff_values_ic(ptr %a, i64
; CHECK-NEXT: [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ]
; CHECK-NEXT: [[TMP1:%.*]] = getelementptr inbounds [4 x i8], ptr [[B]], i64 [[INDEX]]
; CHECK-NEXT: [[WIDE_LOAD:%.*]] = load <4 x i32>, ptr [[TMP1]], align 8, !alias.scope [[META0:![0-9]+]], !noalias [[META3:![0-9]+]]
-; CHECK-NEXT: [[DOTNOT:%.*]] = icmp eq <4 x i32> [[WIDE_LOAD]], [[BROADCAST_SPLAT4]]
; CHECK-NEXT: store <4 x i32> [[BROADCAST_SPLAT]], ptr [[TMP1]], align 4, !alias.scope [[META0]], !noalias [[META3]]
-; CHECK-NEXT: [[PREDPHI:%.*]] = select <4 x i1> [[DOTNOT]], <4 x i32> [[BROADCAST_SPLAT]], <4 x i32> [[BROADCAST_SPLAT4]]
-; CHECK-NEXT: [[TMP2:%.*]] = extractelement <4 x i32> [[PREDPHI]], i64 3
-; CHECK-NEXT: store i32 [[TMP2]], ptr [[A]], align 4, !alias.scope [[META3]]
; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 4
; CHECK-NEXT: [[TMP3:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
; CHECK-NEXT: br i1 [[TMP3]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP5:![0-9]+]]
; CHECK: middle.block:
+; CHECK-NEXT: [[WIDE_LOAD_LCSSA:%.*]] = phi <4 x i32> [ [[WIDE_LOAD]], [[VECTOR_BODY]] ]
+; CHECK-NEXT: [[TMP5:%.*]] = icmp eq <4 x i32> [[WIDE_LOAD_LCSSA]], [[BROADCAST_SPLAT4]]
+; CHECK-NEXT: [[PREDPHI:%.*]] = select <4 x i1> [[TMP5]], <4 x i32> [[BROADCAST_SPLAT]], <4 x i32> [[BROADCAST_SPLAT4]]
+; CHECK-NEXT: [[TMP4:%.*]] = extractelement <4 x i32> [[PREDPHI]], i64 3
+; CHECK-NEXT: store i32 [[TMP4]], ptr [[A]], align 4, !alias.scope [[META3]]
; CHECK-NEXT: [[CMP_N:%.*]] = icmp eq i64 [[SMAX2]], [[N_VEC]]
; CHECK-NEXT: br i1 [[CMP_N]], label [[FOR_END:%.*]], label [[SCALAR_PH]]
; CHECK: scalar.ph:
@@ -141,11 +142,11 @@ define void @inv_val_store_to_inv_address_conditional_inv(ptr %a, i64 %n, ptr %b
; CHECK-NEXT: [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ]
; CHECK-NEXT: [[TMP1:%.*]] = getelementptr inbounds [4 x i8], ptr [[B]], i64 [[INDEX]]
; CHECK-NEXT: store <4 x i32> [[BROADCAST_SPLAT]], ptr [[TMP1]], align 4, !alias.scope [[META9:![0-9]+]], !noalias [[META12:![0-9]+]]
-; CHECK-NEXT: store i32 [[K]], ptr [[A]], align 4, !alias.scope [[META12]]
; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 4
; CHECK-NEXT: [[TMP4:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
; CHECK-NEXT: br i1 [[TMP4]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP14:![0-9]+]]
; CHECK: middle.block:
+; CHECK-NEXT: store i32 [[K]], ptr [[A]], align 4, !alias.scope [[META12]]
; CHECK-NEXT: [[CMP_N:%.*]] = icmp eq i64 [[SMAX2]], [[N_VEC]]
; CHECK-NEXT: br i1 [[CMP_N]], label [[FOR_END:%.*]], label [[SCALAR_PH]]
; CHECK: scalar.ph:
@@ -225,14 +226,15 @@ define i32 @variant_val_store_to_inv_address(ptr %a, i64 %n, ptr %b, i32 %k) {
; CHECK-NEXT: [[VEC_PHI:%.*]] = phi <4 x i32> [ zeroinitializer, [[VECTOR_PH]] ], [ [[TMP3:%.*]], [[VECTOR_BODY]] ]
; CHECK-NEXT: [[TMP1:%.*]] = getelementptr inbounds [4 x i8], ptr [[B]], i64 [[INDEX]]
; CHECK-NEXT: [[WIDE_LOAD:%.*]] = load <4 x i32>, ptr [[TMP1]], align 8, !alias.scope [[META16:![0-9]+]]
-; CHECK-NEXT: [[TMP2:%.*]] = extractelement <4 x i32> [[WIDE_LOAD]], i64 3
-; CHECK-NEXT: store i32 [[TMP2]], ptr [[A]], align 4, !alias.scope [[META19:![0-9]+]], !noalias [[META16]]
; CHECK-NEXT: [[TMP3]] = add <4 x i32> [[VEC_PHI]], [[WIDE_LOAD]]
; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 4
; CHECK-NEXT: [[TMP4:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
-; CHECK-NEXT: br i1 [[TMP4]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP21:![0-9]+]]
+; CHECK-NEXT: br i1 [[TMP4]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP19:![0-9]+]]
; CHECK: middle.block:
+; CHECK-NEXT: [[WIDE_LOAD_LCSSA:%.*]] = phi <4 x i32> [ [[WIDE_LOAD]], [[VECTOR_BODY]] ]
; CHECK-NEXT: [[DOTLCSSA:%.*]] = phi <4 x i32> [ [[TMP3]], [[VECTOR_BODY]] ]
+; CHECK-NEXT: [[TMP6:%.*]] = extractelement <4 x i32> [[WIDE_LOAD_LCSSA]], i64 3
+; CHECK-NEXT: store i32 [[TMP6]], ptr [[A]], align 4, !alias.scope [[META20:![0-9]+]], !noalias [[META16]]
; CHECK-NEXT: [[TMP5:%.*]] = call i32 @llvm.vector.reduce.add.v4i32(<4 x i32> [[DOTLCSSA]])
; CHECK-NEXT: [[CMP_N:%.*]] = icmp eq i64 [[SMAX2]], [[N_VEC]]
; CHECK-NEXT: br i1 [[CMP_N]], label [[FOR_END:%.*]], label [[SCALAR_PH]]
diff --git a/llvm/test/Transforms/LoopVectorize/invariant-store-vectorization.ll b/llvm/test/Transforms/LoopVectorize/invariant-store-vectorization.ll
index bf26813afc7be..f29f8d98c08a8 100644
--- a/llvm/test/Transforms/LoopVectorize/invariant-store-vectorization.ll
+++ b/llvm/test/Transforms/LoopVectorize/invariant-store-vectorization.ll
@@ -40,12 +40,12 @@ define i32 @inv_val_store_to_inv_address_with_reduction(ptr %a, i64 %n, ptr %b)
; CHECK-NEXT: [[TMP1:%.*]] = getelementptr inbounds [4 x i8], ptr [[B]], i64 [[INDEX]]
; CHECK-NEXT: [[WIDE_LOAD:%.*]] = load <4 x i32>, ptr [[TMP1]], align 8, !alias.scope [[META0:![0-9]+]]
; CHECK-NEXT: [[TMP2]] = add <4 x i32> [[VEC_PHI]], [[WIDE_LOAD]]
-; CHECK-NEXT: store i32 [[NTRUNC]], ptr [[A]], align 4, !alias.scope [[META3:![0-9]+]], !noalias [[META0]]
; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 4
; CHECK-NEXT: [[TMP3:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
-; CHECK-NEXT: br i1 [[TMP3]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP5:![0-9]+]]
+; CHECK-NEXT: br i1 [[TMP3]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP3:![0-9]+]]
; CHECK: middle.block:
; CHECK-NEXT: [[DOTLCSSA:%.*]] = phi <4 x i32> [ [[TMP2]], [[VECTOR_BODY]] ]
+; CHECK-NEXT: store i32 [[NTRUNC]], ptr [[A]], align 4, !alias.scope [[META6:![0-9]+]], !noalias [[META0]]
; CHECK-NEXT: [[TMP4:%.*]] = call i32 @llvm.vector.reduce.add.v4i32(<4 x i32> [[DOTLCSSA]])
; CHECK-NEXT: [[CMP_N:%.*]] = icmp eq i64 [[SMAX2]], [[N_VEC]]
; CHECK-NEXT: br i1 [[CMP_N]], label [[FOR_END:%.*]], label [[SCALAR_PH]]
@@ -113,12 +113,12 @@ define void @inv_val_store_to_inv_address(ptr %a, i64 %n, ptr %b) {
; CHECK: vector.body:
; CHECK-NEXT: [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ]
; CHECK-NEXT: [[TMP1:%.*]] = getelementptr inbounds [4 x i8], ptr [[B]], i64 [[INDEX]]
-; CHECK-NEXT: store i32 [[NTRUNC]], ptr [[A]], align 4, !alias.scope [[META9:![0-9]+]], !noalias [[META12:![0-9]+]]
-; CHECK-NEXT: store <4 x i32> [[BROADCAST_SPLAT]], ptr [[TMP1]], align 4, !alias.scope [[META12]]
+; CHECK-NEXT: store <4 x i32> [[BROADCAST_SPLAT]], ptr [[TMP1]], align 4, !alias.scope [[META9:![0-9]+]]
; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 4
; CHECK-NEXT: [[TMP2:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
-; CHECK-NEXT: br i1 [[TMP2]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP14:![0-9]+]]
+; CHECK-NEXT: br i1 [[TMP2]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP12:![0-9]+]]
; CHECK: middle.block:
+; CHECK-NEXT: store i32 [[NTRUNC]], ptr [[A]], align 4, !alias.scope [[META13:![0-9]+]], !noalias [[META9]]
; CHECK-NEXT: [[CMP_N:%.*]] = icmp eq i64 [[SMAX2]], [[N_VEC]]
; CHECK-NEXT: br i1 [[CMP_N]], label [[FOR_END:%.*]], label [[SCALAR_PH]]
; CHECK: scalar.ph:
diff --git a/llvm/test/Transforms/LoopVectorize/metadata.ll b/llvm/test/Transforms/LoopVectorize/metadata.ll
index 0a22c4b1e70a6..90360e7a2af98 100644
--- a/llvm/test/Transforms/LoopVectorize/metadata.ll
+++ b/llvm/test/Transforms/LoopVectorize/metadata.ll
@@ -530,12 +530,12 @@ define void @noalias_metadata(ptr align 8 %dst, ptr align 8 %src) {
; CHECK-NEXT: [[TMP26:%.*]] = mul i64 [[INDEX]], 8
; CHECK-NEXT: [[NEXT_GEP:%.*]] = getelementptr i8, ptr [[SRC]], i64 [[TMP26]]
; CHECK-NEXT: [[WIDE_LOAD:%.*]] = load <2 x ptr>, ptr [[NEXT_GEP]], align 8, !alias.scope [[META14:![0-9]+]]
-; CHECK-NEXT: [[TMP7:%.*]] = extractelement <2 x ptr> [[WIDE_LOAD]], i32 1
-; CHECK-NEXT: store ptr [[TMP7]], ptr [[DST]], align 8, !alias.scope [[META17:![0-9]+]], !noalias [[META19:![0-9]+]]
; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 2
; CHECK-NEXT: [[TMP28:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
-; CHECK-NEXT: br i1 [[TMP28]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP22:![0-9]+]]
+; CHECK-NEXT: br i1 [[TMP28]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP17:![0-9]+]]
; CHECK: [[MIDDLE_BLOCK]]:
+; CHECK-NEXT: [[TMP8:%.*]] = extractelement <2 x ptr> [[WIDE_LOAD]], i32 1
+; CHECK-NEXT: store ptr [[TMP8]], ptr [[DST]], align 8, !alias.scope [[META18:![0-9]+]], !noalias [[META20:![0-9]+]]
; CHECK-NEXT: [[CMP_N:%.*]] = icmp eq i64 [[TMP4]], [[N_VEC]]
; CHECK-NEXT: br i1 [[CMP_N]], label %[[EXIT:.*]], label %[[SCALAR_PH]]
; CHECK: [[SCALAR_PH]]:
@@ -584,12 +584,12 @@ define void @noalias_metadata(ptr align 8 %dst, ptr align 8 %src) {
; INTERLEAVE-NEXT: [[NEXT_GEP:%.*]] = getelementptr i8, ptr [[SRC]], i64 [[TMP26]]
; INTERLEAVE-NEXT: [[TMP7:%.*]] = getelementptr ptr, ptr [[NEXT_GEP]], i64 2
; INTERLEAVE-NEXT: [[WIDE_LOAD:%.*]] = load <2 x ptr>, ptr [[TMP7]], align 8, !alias.scope [[META14:![0-9]+]]
-; INTERLEAVE-NEXT: [[TMP8:%.*]] = extractelement <2 x ptr> [[WIDE_LOAD]], i32 1
-; INTERLEAVE-NEXT: store ptr [[TMP8]], ptr [[DST]], align 8, !alias.scope [[META17:![0-9]+]], !noalias [[META19:![0-9]+]]
; INTERLEAVE-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 4
; INTERLEAVE-NEXT: [[TMP28:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
-; INTERLEAVE-NEXT: br i1 [[TMP28]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP22:![0-9]+]]
+; INTERLEAVE-NEXT: br i1 [[TMP28]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP17:![0-9]+]]
; INTERLEAVE: [[MIDDLE_BLOCK]]:
+; INTERLEAVE-NEXT: [[TMP9:%.*]] = extractelement <2 x ptr> [[WIDE_LOAD]], i32 1
+; INTERLEAVE-NEXT: store ptr [[TMP9]], ptr [[DST]], align 8, !alias.scope [[META18:![0-9]+]], !noalias [[META20:![0-9]+]]
; INTERLEAVE-NEXT: [[CMP_N:%.*]] = icmp eq i64 [[TMP4]], [[N_VEC]]
; INTERLEAVE-NEXT: br i1 [[CMP_N]], label %[[EXIT:.*]], label %[[SCALAR_PH]]
; INTERLEAVE: [[SCALAR_PH]]:
@@ -655,13 +655,13 @@ attributes #1 = { nounwind "vector-function-abi-variant"="_ZGV_LLVM_N2v_bar(bar_
; CHECK: [[META14]] = !{[[META15:![0-9]+]]}
; CHECK: [[META15]] = distinct !{[[META15]], [[META16:![0-9]+]]}
; CHECK: [[META16]] = distinct !{[[META16]], !"LVerDomain"}
-; CHECK: [[META17]] = !{[[META18:![0-9]+]]}
-; CHECK: [[META18]] = distinct !{[[META18]], [[META16]]}
-; CHECK: [[META19]] = !{[[META20:![0-9]+]], [[META15]]}
-; CHECK: [[META20]] = distinct !{[[META20]], [[META21:![0-9]+]], !"g1"}
-; CHECK: [[META21]] = distinct !{[[META21]], !"t2"}
-; CHECK: [[LOOP22]] = distinct !{[[LOOP22]], [[META5]], [[META6]]}
-; CHECK: [[META23]] = !{[[META20]]}
+; CHECK: [[LOOP17]] = distinct !{[[LOOP17]], [[META5]], [[META6]]}
+; CHECK: [[META18]] = !{[[META19:![0-9]+]]}
+; CHECK: [[META19]] = distinct !{[[META19]], [[META16]]}
+; CHECK: [[META20]] = !{[[META21:![0-9]+]], [[META15]]}
+; CHECK: [[META21]] = distinct !{[[META21]], [[META22:![0-9]+]], !"g1"}
+; CHECK: [[META22]] = distinct !{[[META22]], !"t2"}
+; CHECK: [[META23]] = !{[[META21]]}
; CHECK: [[LOOP24]] = distinct !{[[LOOP24]], [[META5]]}
;.
; INTERLEAVE: [[CHAR_TBAA0]] = !{[[META1:![0-9]+]], [[META1]], i64 0, i64 0}
@@ -681,12 +681,12 @@ attributes #1 = { nounwind "vector-function-abi-variant"="_ZGV_LLVM_N2v_bar(bar_
; INTERLEAVE: [[META14]] = !{[[META15:![0-9]+]]}
; INTERLEAVE: [[META15]] = distinct !{[[META15]], [[META16:![0-9]+]]}
; INTERLEAVE: [[META16]] = distinct !{[[META16]], !"LVerDomain"}
-; INTERLEAVE: [[META17]] = !{[[META18:![0-9]+]]}
-; INTERLEAVE: [[META18]] = distinct !{[[META18]], [[META16]]}
-; INTERLEAVE: [[META19]] = !{[[META20:![0-9]+]], [[META15]]}
-; INTERLEAVE: [[META20]] = distinct !{[[META20]], [[META21:![0-9]+]], !"g1"}
-; INTERLEAVE: [[META21]] = distinct !{[[META21]], !"t2"}
-; INTERLEAVE: [[LOOP22]] = distinct !{[[LOOP22]], [[META5]], [[META6]]}
-; INTERLEAVE: [[META23]] = !{[[META20]]}
+; INTERLEAVE: [[LOOP17]] = distinct !{[[LOOP17]], [[META5]], [[META6]]}
+; INTERLEAVE: [[META18]] = !{[[META19:![0-9]+]]}
+; INTERLEAVE: [[META19]] = distinct !{[[META19]], [[META16]]}
+; INTERLEAVE: [[META20]] = !{[[META21:![0-9]+]], [[META15]]}
+; INTERLEAVE: [[META21]] = distinct !{[[META21]], [[META22:![0-9]+]], !"g1"}
+; INTERLEAVE: [[META22]] = distinct !{[[META22]], !"t2"}
+; INTERLEAVE: [[META23]] = !{[[META21]]}
; INTERLEAVE: [[LOOP24]] = distinct !{[[LOOP24]], [[META5]]}
;.
diff --git a/llvm/test/Transforms/LoopVectorize/operand-bundles.ll b/llvm/test/Transforms/LoopVectorize/operand-bundles.ll
index 18bd16873c1d5..d3af26873a390 100644
--- a/llvm/test/Transforms/LoopVectorize/operand-bundles.ll
+++ b/llvm/test/Transforms/LoopVectorize/operand-bundles.ll
@@ -188,12 +188,12 @@ define void @assume_cold_operand_bundle(ptr noalias %a, ptr noalias %b) {
; CHECK-NEXT: [[ENTRY:.*:]]
; CHECK-NEXT: br label %[[VECTOR_PH:.*]]
; CHECK: [[VECTOR_PH]]:
-; CHECK-NEXT: tail call void @llvm.assume(i1 true) [ "cold"() ]
; CHECK-NEXT: br label %[[VECTOR_BODY:.*]]
; CHECK: [[VECTOR_BODY]]:
; CHECK-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
; CHECK-NEXT: [[TMP0:%.*]] = getelementptr inbounds float, ptr [[B]], i64 [[INDEX]]
; CHECK-NEXT: [[WIDE_LOAD:%.*]] = load <4 x float>, ptr [[TMP0]], align 4
+; CHECK-NEXT: tail call void @llvm.assume(i1 true) [ "cold"() ]
; CHECK-NEXT: [[TMP1:%.*]] = fadd <4 x float> [[WIDE_LOAD]], splat (float 1.000000e+00)
; CHECK-NEXT: [[TMP2:%.*]] = getelementptr inbounds float, ptr [[A]], i64 [[INDEX]]
; CHECK-NEXT: store <4 x float> [[TMP1]], ptr [[TMP2]], align 4
diff --git a/llvm/test/Transforms/LoopVectorize/pointer-select-runtime-checks.ll b/llvm/test/Transforms/LoopVectorize/pointer-select-runtime-checks.ll
index 9b383bcb88a69..18b46b52a2d74 100644
--- a/llvm/test/Transforms/LoopVectorize/pointer-select-runtime-checks.ll
+++ b/llvm/test/Transforms/LoopVectorize/pointer-select-runtime-checks.ll
@@ -28,16 +28,14 @@ define void @test1_select_invariant(ptr %src.1, ptr %src.2, ptr %dst, i1 %c, i8
; CHECK-NEXT: br label [[VECTOR_BODY:%.*]]
; CHECK: vector.body:
; CHECK-NEXT: [[INDEX:%.*]] = phi i32 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ]
-; CHECK-NEXT: [[OFFSET_IDX:%.*]] = trunc i32 [[INDEX]] to i8
-; CHECK-NEXT: [[INDUCTION2:%.*]] = add i8 [[OFFSET_IDX]], 1
-; CHECK-NEXT: [[TMP11:%.*]] = getelementptr i8, ptr [[DST]], i8 [[OFFSET_IDX]]
-; CHECK-NEXT: [[TMP8:%.*]] = getelementptr i8, ptr [[DST]], i8 [[INDUCTION2]]
-; CHECK-NEXT: store i8 [[TMP7]], ptr [[TMP11]], align 2, !alias.scope [[META3:![0-9]+]], !noalias [[META0]]
-; CHECK-NEXT: store i8 [[TMP7]], ptr [[TMP8]], align 2, !alias.scope [[META3]], !noalias [[META0]]
; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i32 [[INDEX]], 2
; CHECK-NEXT: [[TMP9:%.*]] = icmp eq i32 [[INDEX_NEXT]], [[N_VEC]]
-; CHECK-NEXT: br i1 [[TMP9]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP5:![0-9]+]]
+; CHECK-NEXT: br i1 [[TMP9]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP3:![0-9]+]]
; CHECK: middle.block:
+; CHECK-NEXT: [[OFFSET_IDX:%.*]] = trunc i32 [[INDEX]] to i8
+; CHECK-NEXT: [[TMP11:%.*]] = add i8 [[OFFSET_IDX]], 1
+; CHECK-NEXT: [[TMP10:%.*]] = getelementptr i8, ptr [[DST]], i8 [[TMP11]]
+; CHECK-NEXT: store i8 [[TMP7]], ptr [[TMP10]], align 2, !alias.scope [[META6:![0-9]+]], !noalias [[META0]]
; CHECK-NEXT: [[CMP_N:%.*]] = icmp eq i32 [[TMP2]], [[N_VEC]]
; CHECK-NEXT: br i1 [[CMP_N]], label [[EXIT:%.*]], label [[SCALAR_PH]]
; CHECK: scalar.ph:
@@ -191,20 +189,15 @@ define void @test_loop_dependent_select2(ptr %src.1, ptr %src.2, ptr %dst, i8 %n
; CHECK-NEXT: [[INDEX:%.*]] = phi i32 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ]
; CHECK-NEXT: [[OFFSET_IDX:%.*]] = trunc i32 [[INDEX]] to i8
; CHECK-NEXT: [[INDUCTION6:%.*]] = add i8 [[OFFSET_IDX]], 1
-; CHECK-NEXT: [[TMP6:%.*]] = icmp ult i8 [[OFFSET_IDX]], [[X:%.*]]
-; CHECK-NEXT: [[TMP7:%.*]] = icmp ult i8 [[INDUCTION6]], [[X]]
-; CHECK-NEXT: [[TMP8:%.*]] = select i1 [[TMP6]], ptr [[SRC_1]], ptr [[SRC_2]]
+; CHECK-NEXT: [[TMP7:%.*]] = icmp ult i8 [[INDUCTION6]], [[X:%.*]]
; CHECK-NEXT: [[TMP9:%.*]] = select i1 [[TMP7]], ptr [[SRC_1]], ptr [[SRC_2]]
-; CHECK-NEXT: [[TMP12:%.*]] = load i8, ptr [[TMP8]], align 8, !alias.scope [[META11:![0-9]+]]
-; CHECK-NEXT: [[TMP15:%.*]] = load i8, ptr [[TMP9]], align 8, !alias.scope [[META11]]
-; CHECK-NEXT: [[TMP16:%.*]] = getelementptr i8, ptr [[DST]], i8 [[OFFSET_IDX]]
-; CHECK-NEXT: [[TMP13:%.*]] = getelementptr i8, ptr [[DST]], i8 [[INDUCTION6]]
-; CHECK-NEXT: store i8 [[TMP12]], ptr [[TMP16]], align 2, !alias.scope [[META14:![0-9]+]], !noalias [[META16:![0-9]+]]
-; CHECK-NEXT: store i8 [[TMP15]], ptr [[TMP13]], align 2, !alias.scope [[META14]], !noalias [[META16]]
+; CHECK-NEXT: [[TMP10:%.*]] = load i8, ptr [[TMP9]], align 8, !alias.scope [[META11:![0-9]+]]
; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i32 [[INDEX]], 2
; CHECK-NEXT: [[TMP14:%.*]] = icmp eq i32 [[INDEX_NEXT]], [[N_VEC]]
-; CHECK-NEXT: br i1 [[TMP14]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP18:![0-9]+]]
+; CHECK-NEXT: br i1 [[TMP14]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP14:![0-9]+]]
; CHECK: middle.block:
+; CHECK-NEXT: [[TMP12:%.*]] = getelementptr i8, ptr [[DST]], i8 [[INDUCTION6]]
+; CHECK-NEXT: store i8 [[TMP10]], ptr [[TMP12]], align 2, !alias.scope [[META15:![0-9]+]], !noalias [[META17:![0-9]+]]
; CHECK-NEXT: [[CMP_N:%.*]] = icmp eq i32 [[TMP2]], [[N_VEC]]
; CHECK-NEXT: br i1 [[CMP_N]], label [[EXIT:%.*]], label [[SCALAR_PH]]
; CHECK: scalar.ph:
@@ -275,20 +268,15 @@ define void @test_loop_dependent_select_first_ptr_noundef(ptr noundef %src.1, pt
; CHECK-NEXT: [[INDEX:%.*]] = phi i32 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ]
; CHECK-NEXT: [[OFFSET_IDX:%.*]] = trunc i32 [[INDEX]] to i8
; CHECK-NEXT: [[INDUCTION6:%.*]] = add i8 [[OFFSET_IDX]], 1
-; CHECK-NEXT: [[TMP6:%.*]] = icmp ult i8 [[OFFSET_IDX]], [[X:%.*]]
-; CHECK-NEXT: [[TMP7:%.*]] = icmp ult i8 [[INDUCTION6]], [[X]]
-; CHECK-NEXT: [[TMP8:%.*]] = select i1 [[TMP6]], ptr [[SRC_1]], ptr [[SRC_2]]
+; CHECK-NEXT: [[TMP7:%.*]] = icmp ult i8 [[INDUCTION6]], [[X:%.*]]
; CHECK-NEXT: [[TMP9:%.*]] = select i1 [[TMP7]], ptr [[SRC_1]], ptr [[SRC_2]]
-; CHECK-NEXT: [[TMP12:%.*]] = load i8, ptr [[TMP8]], align 8, !alias.scope [[META20:![0-9]+]]
-; CHECK-NEXT: [[TMP15:%.*]] = load i8, ptr [[TMP9]], align 8, !alias.scope [[META20]]
-; CHECK-NEXT: [[TMP16:%.*]] = getelementptr i8, ptr [[DST]], i8 [[OFFSET_IDX]]
-; CHECK-NEXT: [[TMP13:%.*]] = getelementptr i8, ptr [[DST]], i8 [[INDUCTION6]]
-; CHECK-NEXT: store i8 [[TMP12]], ptr [[TMP16]], align 2, !alias.scope [[META23:![0-9]+]], !noalias [[META25:![0-9]+]]
-; CHECK-NEXT: store i8 [[TMP15]], ptr [[TMP13]], align 2, !alias.scope [[META23]], !noalias [[META25]]
+; CHECK-NEXT: [[TMP10:%.*]] = load i8, ptr [[TMP9]], align 8, !alias.scope [[META20:![0-9]+]]
; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i32 [[INDEX]], 2
; CHECK-NEXT: [[TMP14:%.*]] = icmp eq i32 [[INDEX_NEXT]], [[N_VEC]]
-; CHECK-NEXT: br i1 [[TMP14]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP27:![0-9]+]]
+; CHECK-NEXT: br i1 [[TMP14]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP23:![0-9]+]]
; CHECK: middle.block:
+; CHECK-NEXT: [[TMP12:%.*]] = getelementptr i8, ptr [[DST]], i8 [[INDUCTION6]]
+; CHECK-NEXT: store i8 [[TMP10]], ptr [[TMP12]], align 2, !alias.scope [[META24:![0-9]+]], !noalias [[META26:![0-9]+]]
; CHECK-NEXT: [[CMP_N:%.*]] = icmp eq i32 [[TMP2]], [[N_VEC]]
; CHECK-NEXT: br i1 [[CMP_N]], label [[EXIT:%.*]], label [[SCALAR_PH]]
; CHECK: scalar.ph:
@@ -359,20 +347,15 @@ define void @test_loop_dependent_select_second_ptr_noundef(ptr %src.1, ptr nound
; CHECK-NEXT: [[INDEX:%.*]] = phi i32 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ]
; CHECK-NEXT: [[OFFSET_IDX:%.*]] = trunc i32 [[INDEX]] to i8
; CHECK-NEXT: [[INDUCTION6:%.*]] = add i8 [[OFFSET_IDX]], 1
-; CHECK-NEXT: [[TMP6:%.*]] = icmp ult i8 [[OFFSET_IDX]], [[X:%.*]]
-; CHECK-NEXT: [[TMP7:%.*]] = icmp ult i8 [[INDUCTION6]], [[X]]
-; CHECK-NEXT: [[TMP8:%.*]] = select i1 [[TMP6]], ptr [[SRC_1]], ptr [[SRC_2]]
+; CHECK-NEXT: [[TMP7:%.*]] = icmp ult i8 [[INDUCTION6]], [[X:%.*]]
; CHECK-NEXT: [[TMP9:%.*]] = select i1 [[TMP7]], ptr [[SRC_1]], ptr [[SRC_2]]
-; CHECK-NEXT: [[TMP12:%.*]] = load i8, ptr [[TMP8]], align 8, !alias.scope [[META29:![0-9]+]]
-; CHECK-NEXT: [[TMP15:%.*]] = load i8, ptr [[TMP9]], align 8, !alias.scope [[META29]]
-; CHECK-NEXT: [[TMP16:%.*]] = getelementptr i8, ptr [[DST]], i8 [[OFFSET_IDX]]
-; CHECK-NEXT: [[TMP13:%.*]] = getelementptr i8, ptr [[DST]], i8 [[INDUCTION6]]
-; CHECK-NEXT: store i8 [[TMP12]], ptr [[TMP16]], align 2, !alias.scope [[META32:![0-9]+]], !noalias [[META34:![0-9]+]]
-; CHECK-NEXT: store i8 [[TMP15]], ptr [[TMP13]], align 2, !alias.scope [[META32]], !noalias [[META34]]
+; CHECK-NEXT: [[TMP10:%.*]] = load i8, ptr [[TMP9]], align 8, !alias.scope [[META29:![0-9]+]]
; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i32 [[INDEX]], 2
; CHECK-NEXT: [[TMP14:%.*]] = icmp eq i32 [[INDEX_NEXT]], [[N_VEC]]
-; CHECK-NEXT: br i1 [[TMP14]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP36:![0-9]+]]
+; CHECK-NEXT: br i1 [[TMP14]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP32:![0-9]+]]
; CHECK: middle.block:
+; CHECK-NEXT: [[TMP12:%.*]] = getelementptr i8, ptr [[DST]], i8 [[INDUCTION6]]
+; CHECK-NEXT: store i8 [[TMP10]], ptr [[TMP12]], align 2, !alias.scope [[META33:![0-9]+]], !noalias [[META35:![0-9]+]]
; CHECK-NEXT: [[CMP_N:%.*]] = icmp eq i32 [[TMP2]], [[N_VEC]]
; CHECK-NEXT: br i1 [[CMP_N]], label [[EXIT:%.*]], label [[SCALAR_PH]]
; CHECK: scalar.ph:
diff --git a/llvm/test/Transforms/LoopVectorize/pr47343-expander-lcssa-after-cfg-update.ll b/llvm/test/Transforms/LoopVectorize/pr47343-expander-lcssa-after-cfg-update.ll
index b9cb1cb5abae8..fa9c8b19e1171 100644
--- a/llvm/test/Transforms/LoopVectorize/pr47343-expander-lcssa-after-cfg-update.ll
+++ b/llvm/test/Transforms/LoopVectorize/pr47343-expander-lcssa-after-cfg-update.ll
@@ -43,12 +43,12 @@ define void @f() {
; CHECK-NEXT: br label [[VECTOR_BODY:%.*]]
; CHECK: vector.body:
; CHECK-NEXT: [[INDEX:%.*]] = phi i32 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ]
-; CHECK-NEXT: store i32 0, ptr @f.e, align 1, !alias.scope [[META0:![0-9]+]], !noalias [[META3:![0-9]+]]
-; CHECK-NEXT: store i8 10, ptr [[TMP0]], align 1, !alias.scope [[META3]]
; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i32 [[INDEX]], 2
; CHECK-NEXT: [[TMP2:%.*]] = icmp eq i32 [[INDEX_NEXT]], 500
-; CHECK-NEXT: br i1 [[TMP2]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP5:![0-9]+]]
+; CHECK-NEXT: br i1 [[TMP2]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP0:![0-9]+]]
; CHECK: middle.block:
+; CHECK-NEXT: store i32 0, ptr @f.e, align 1, !alias.scope [[META3:![0-9]+]], !noalias [[META6:![0-9]+]]
+; CHECK-NEXT: store i8 10, ptr [[TMP0]], align 1, !alias.scope [[META6]]
; CHECK-NEXT: br label [[EXIT:%.*]]
; CHECK: scalar.ph:
; CHECK-NEXT: br label [[LOOP:%.*]]
diff --git a/llvm/test/Transforms/LoopVectorize/pr59319-loop-access-info-invalidation.ll b/llvm/test/Transforms/LoopVectorize/pr59319-loop-access-info-invalidation.ll
index 611e6c184625c..07e4c7dea50f1 100644
--- a/llvm/test/Transforms/LoopVectorize/pr59319-loop-access-info-invalidation.ll
+++ b/llvm/test/Transforms/LoopVectorize/pr59319-loop-access-info-invalidation.ll
@@ -54,11 +54,11 @@ define void @reduced(ptr %0, ptr %1, i64 %iv, ptr %2, i64 %iv76, i64 %iv93) {
; CHECK-NEXT: br label [[VECTOR_BODY10:%.*]]
; CHECK: vector.body8:
; CHECK-NEXT: [[INDEX12:%.*]] = phi i64 [ 0, [[VECTOR_PH6]] ], [ [[INDEX_NEXT13:%.*]], [[VECTOR_BODY10]] ]
-; CHECK-NEXT: store i32 0, ptr [[TMP1]], align 4, !alias.scope [[META4:![0-9]+]], !noalias [[META7:![0-9]+]]
; CHECK-NEXT: [[INDEX_NEXT13]] = add nuw i64 [[INDEX12]], 4
; CHECK-NEXT: [[TMP11:%.*]] = icmp eq i64 [[INDEX_NEXT13]], [[N_VEC]]
-; CHECK-NEXT: br i1 [[TMP11]], label [[MIDDLE_BLOCK12:%.*]], label [[VECTOR_BODY10]], !llvm.loop [[LOOP9:![0-9]+]]
+; CHECK-NEXT: br i1 [[TMP11]], label [[MIDDLE_BLOCK12:%.*]], label [[VECTOR_BODY10]], !llvm.loop [[LOOP4:![0-9]+]]
; CHECK: middle.block11:
+; CHECK-NEXT: store i32 0, ptr [[TMP1]], align 4, !alias.scope [[META5:![0-9]+]], !noalias [[META8:![0-9]+]]
; CHECK-NEXT: [[CMP_N10:%.*]] = icmp eq i64 [[TMP3]], [[N_VEC]]
; CHECK-NEXT: br i1 [[CMP_N10]], label [[LOOP_3_LR_PH:%.*]], label [[SCALAR_PH5]]
; CHECK: scalar.ph3:
@@ -84,11 +84,11 @@ define void @reduced(ptr %0, ptr %1, i64 %iv, ptr %2, i64 %iv76, i64 %iv93) {
; CHECK-NEXT: br label [[VECTOR_BODY27:%.*]]
; CHECK: vector.body25:
; CHECK-NEXT: [[INDEX29:%.*]] = phi i64 [ 0, [[VECTOR_PH24]] ], [ [[INDEX_NEXT29:%.*]], [[VECTOR_BODY27]] ]
-; CHECK-NEXT: store i32 0, ptr [[TMP1]], align 4, !alias.scope [[META10:![0-9]+]], !noalias [[META13:![0-9]+]]
; CHECK-NEXT: [[INDEX_NEXT29]] = add nuw i64 [[INDEX29]], 4
; CHECK-NEXT: [[TMP14:%.*]] = icmp eq i64 [[INDEX_NEXT29]], [[N_VEC25]]
-; CHECK-NEXT: br i1 [[TMP14]], label [[MIDDLE_BLOCK29:%.*]], label [[VECTOR_BODY27]], !llvm.loop [[LOOP15:![0-9]+]]
+; CHECK-NEXT: br i1 [[TMP14]], label [[MIDDLE_BLOCK29:%.*]], label [[VECTOR_BODY27]], !llvm.loop [[LOOP10:![0-9]+]]
; CHECK: middle.block28:
+; CHECK-NEXT: store i32 0, ptr [[TMP1]], align 4, !alias.scope [[META11:![0-9]+]], !noalias [[META14:![0-9]+]]
; CHECK-NEXT: [[CMP_N27:%.*]] = icmp eq i64 [[TMP3]], [[N_VEC25]]
; CHECK-NEXT: br i1 [[CMP_N27]], label [[LOOP_CLEANUP:%.*]], label [[SCALAR_PH21]]
; CHECK: scalar.ph20:
diff --git a/llvm/test/Transforms/LoopVectorize/reduction-with-invariant-store.ll b/llvm/test/Transforms/LoopVectorize/reduction-with-invariant-store.ll
index df50e6f12d360..c3a30ae0f63ce 100644
--- a/llvm/test/Transforms/LoopVectorize/reduction-with-invariant-store.ll
+++ b/llvm/test/Transforms/LoopVectorize/reduction-with-invariant-store.ll
@@ -885,11 +885,11 @@ define i32 @non_reduc_store_invariant_addr_not_hoisted(ptr %dst, ptr readonly %s
; CHECK-NEXT: [[TMP1:%.*]] = getelementptr inbounds i32, ptr [[SRC]], i64 [[INDEX]]
; CHECK-NEXT: [[WIDE_LOAD:%.*]] = load <4 x i32>, ptr [[TMP1]], align 4, !alias.scope [[META39:![0-9]+]]
; CHECK-NEXT: [[TMP2]] = add <4 x i32> [[VEC_PHI]], [[WIDE_LOAD]]
-; CHECK-NEXT: store i32 0, ptr [[TMP0]], align 4, !alias.scope [[META42:![0-9]+]], !noalias [[META39]]
; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 4
; CHECK-NEXT: [[TMP3:%.*]] = icmp eq i64 [[INDEX_NEXT]], 1000
-; CHECK-NEXT: br i1 [[TMP3]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP44:![0-9]+]]
+; CHECK-NEXT: br i1 [[TMP3]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP42:![0-9]+]]
; CHECK: [[MIDDLE_BLOCK]]:
+; CHECK-NEXT: store i32 0, ptr [[TMP0]], align 4, !alias.scope [[META43:![0-9]+]], !noalias [[META39]]
; CHECK-NEXT: [[TMP4:%.*]] = call i32 @llvm.vector.reduce.add.v4i32(<4 x i32> [[TMP2]])
; CHECK-NEXT: br [[EXIT:label %.*]]
; CHECK: [[SCALAR_PH]]:
diff --git a/llvm/test/Transforms/LoopVectorize/runtime-check.ll b/llvm/test/Transforms/LoopVectorize/runtime-check.ll
index 2bfa63fe41e11..b13ff424bd5d5 100644
--- a/llvm/test/Transforms/LoopVectorize/runtime-check.ll
+++ b/llvm/test/Transforms/LoopVectorize/runtime-check.ll
@@ -506,11 +506,11 @@ define void @test_scev_check_mul_add_expansion(ptr %out, ptr %in, i32 %len, i32
; CHECK-NEXT: [[TMP6:%.*]] = sext i32 [[OFFSET_IDX]] to i64
; CHECK-NEXT: [[TMP7:%.*]] = getelementptr [2 x i8], ptr [[OUT]], i64 [[TMP6]]
; CHECK-NEXT: store <4 x i16> zeroinitializer, ptr [[TMP7]], align 2, !alias.scope [[META36:![0-9]+]], !noalias [[META39:![0-9]+]]
-; CHECK-NEXT: store i32 0, ptr [[IN]], align 4, !alias.scope [[META39]]
; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i32 [[INDEX]], 4
; CHECK-NEXT: [[TMP8:%.*]] = icmp eq i32 [[INDEX_NEXT]], [[N_VEC]]
; CHECK-NEXT: br i1 [[TMP8]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP41:![0-9]+]]
; CHECK: middle.block:
+; CHECK-NEXT: store i32 0, ptr [[IN]], align 4, !alias.scope [[META39]]
; CHECK-NEXT: [[CMP_N:%.*]] = icmp eq i32 [[TMP0]], [[N_VEC]]
; CHECK-NEXT: br i1 [[CMP_N]], label [[EXIT:%.*]], label [[SCALAR_PH]]
; CHECK: scalar.ph:
diff --git a/llvm/test/Transforms/LoopVectorize/scalable-assume.ll b/llvm/test/Transforms/LoopVectorize/scalable-assume.ll
index db482211dc162..ad9416e995cd5 100644
--- a/llvm/test/Transforms/LoopVectorize/scalable-assume.ll
+++ b/llvm/test/Transforms/LoopVectorize/scalable-assume.ll
@@ -84,17 +84,17 @@ define void @test2(ptr %a, ptr noalias %b) {
; CHECK-NEXT: [[TMP7:%.*]] = shl nuw i64 [[TMP3]], 1
; CHECK-NEXT: [[N_MOD_VF:%.*]] = urem i64 1600, [[TMP7]]
; CHECK-NEXT: [[N_VEC:%.*]] = sub i64 1600, [[N_MOD_VF]]
-; CHECK-NEXT: tail call void @llvm.assume(i1 [[MASKCOND]])
-; CHECK-NEXT: tail call void @llvm.assume(i1 [[MASKCOND4]])
; CHECK-NEXT: br label %[[VECTOR_BODY:.*]]
; CHECK: [[VECTOR_BODY]]:
; CHECK-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-NEXT: tail call void @llvm.assume(i1 [[MASKCOND]])
; CHECK-NEXT: [[TMP10:%.*]] = getelementptr inbounds float, ptr [[A]], i64 [[INDEX]]
; CHECK-NEXT: [[TMP13:%.*]] = getelementptr inbounds float, ptr [[TMP10]], i64 [[TMP3]]
; CHECK-NEXT: [[WIDE_LOAD:%.*]] = load <vscale x 2 x float>, ptr [[TMP10]], align 4
; CHECK-NEXT: [[WIDE_LOAD3:%.*]] = load <vscale x 2 x float>, ptr [[TMP13]], align 4
; CHECK-NEXT: [[TMP14:%.*]] = fadd <vscale x 2 x float> [[WIDE_LOAD]], splat (float 1.000000e+00)
; CHECK-NEXT: [[TMP15:%.*]] = fadd <vscale x 2 x float> [[WIDE_LOAD3]], splat (float 1.000000e+00)
+; CHECK-NEXT: tail call void @llvm.assume(i1 [[MASKCOND4]])
; CHECK-NEXT: [[TMP16:%.*]] = getelementptr inbounds float, ptr [[B]], i64 [[INDEX]]
; CHECK-NEXT: [[TMP19:%.*]] = getelementptr inbounds float, ptr [[TMP16]], i64 [[TMP3]]
; CHECK-NEXT: store <vscale x 2 x float> [[TMP14]], ptr [[TMP16]], align 4
diff --git a/llvm/test/Transforms/LoopVectorize/single-scalar-cast-minbw.ll b/llvm/test/Transforms/LoopVectorize/single-scalar-cast-minbw.ll
index fb25b2bc7b906..90ac94a7c6e7b 100644
--- a/llvm/test/Transforms/LoopVectorize/single-scalar-cast-minbw.ll
+++ b/llvm/test/Transforms/LoopVectorize/single-scalar-cast-minbw.ll
@@ -87,11 +87,11 @@ define void @single_scalar_cast_stored(ptr %src, ptr %dst, i32 %n) {
; CHECK-NEXT: br label %[[VECTOR_BODY:.*]]
; CHECK: [[VECTOR_BODY]]:
; CHECK-NEXT: [[INDEX:%.*]] = phi i32 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
-; CHECK-NEXT: store i16 [[TMP5]], ptr [[DST]], align 2, !alias.scope [[META7:![0-9]+]], !noalias [[META4]]
; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i32 [[INDEX]], 4
; CHECK-NEXT: [[TMP6:%.*]] = icmp eq i32 [[INDEX_NEXT]], [[N_VEC]]
-; CHECK-NEXT: br i1 [[TMP6]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP9:![0-9]+]]
+; CHECK-NEXT: br i1 [[TMP6]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP7:![0-9]+]]
; CHECK: [[MIDDLE_BLOCK]]:
+; CHECK-NEXT: store i16 [[TMP5]], ptr [[DST]], align 2, !alias.scope [[META8:![0-9]+]], !noalias [[META4]]
; CHECK-NEXT: [[CMP_N:%.*]] = icmp eq i32 [[N]], [[N_VEC]]
; CHECK-NEXT: br i1 [[CMP_N]], label %[[EXIT:.*]], label %[[SCALAR_PH]]
; CHECK: [[SCALAR_PH]]:
diff --git a/llvm/test/Transforms/LoopVectorize/skeleton-lcssa-crash.ll b/llvm/test/Transforms/LoopVectorize/skeleton-lcssa-crash.ll
index 4a4675f71af9b..f7209d7f57f0a 100644
--- a/llvm/test/Transforms/LoopVectorize/skeleton-lcssa-crash.ll
+++ b/llvm/test/Transforms/LoopVectorize/skeleton-lcssa-crash.ll
@@ -45,12 +45,12 @@ define i16 @test(ptr %arg, i64 %N) {
; CHECK-NEXT: [[TMP4:%.*]] = add nuw nsw i64 [[INDEX]], 1
; CHECK-NEXT: [[TMP5:%.*]] = getelementptr inbounds i16, ptr [[L_1]], i64 [[TMP4]]
; CHECK-NEXT: [[WIDE_LOAD:%.*]] = load <2 x i16>, ptr [[TMP5]], align 2, !alias.scope [[META0:![0-9]+]]
-; CHECK-NEXT: [[TMP8:%.*]] = extractelement <2 x i16> [[WIDE_LOAD]], i32 1
-; CHECK-NEXT: store i16 [[TMP8]], ptr [[L_2]], align 2, !alias.scope [[META3:![0-9]+]], !noalias [[META0]]
; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 2
; CHECK-NEXT: [[TMP9:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
-; CHECK-NEXT: br i1 [[TMP9]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP5:![0-9]+]]
+; CHECK-NEXT: br i1 [[TMP9]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP3:![0-9]+]]
; CHECK: middle.block:
+; CHECK-NEXT: [[TMP6:%.*]] = extractelement <2 x i16> [[WIDE_LOAD]], i32 1
+; CHECK-NEXT: store i16 [[TMP6]], ptr [[L_2]], align 2, !alias.scope [[META6:![0-9]+]], !noalias [[META0]]
; CHECK-NEXT: [[CMP_N:%.*]] = icmp eq i64 [[TMP0]], [[N_VEC]]
; CHECK-NEXT: br i1 [[CMP_N]], label [[EXIT_LOOPEXIT:%.*]], label [[SCALAR_PH]]
; CHECK: scalar.ph:
More information about the llvm-commits
mailing list