[llvm] [VPlan] Extend licm to hoist replicate loads (PR #179506)
Ramkumar Ramachandra via llvm-commits
llvm-commits at lists.llvm.org
Wed May 6 02:09:25 PDT 2026
https://github.com/artagnon updated https://github.com/llvm/llvm-project/pull/179506
>From 5610f451650e4c163688b6abf6a75e15fe3a3394 Mon Sep 17 00:00:00 2001
From: Ramkumar Ramachandra <artagnon at tenstorrent.com>
Date: Fri, 1 May 2026 08:41:32 +0100
Subject: [PATCH 1/4] [LV] Pre-commit hoist-invariant-load test
---
.../LoopVectorize/hoist-invariant-loads.ll | 116 ++++++++++++++++++
1 file changed, 116 insertions(+)
create mode 100644 llvm/test/Transforms/LoopVectorize/hoist-invariant-loads.ll
diff --git a/llvm/test/Transforms/LoopVectorize/hoist-invariant-loads.ll b/llvm/test/Transforms/LoopVectorize/hoist-invariant-loads.ll
new file mode 100644
index 0000000000000..1f7e58a9141b4
--- /dev/null
+++ b/llvm/test/Transforms/LoopVectorize/hoist-invariant-loads.ll
@@ -0,0 +1,116 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --check-globals none --filter-out-after "^scalar.ph" --version 6
+; RUN: opt -p loop-vectorize -force-vector-width=4 -S %s | FileCheck %s
+
+define void @load_not_hoisted(ptr %p, ptr %q, ptr %n) {
+; CHECK-LABEL: define void @load_not_hoisted(
+; CHECK-SAME: ptr [[P:%.*]], ptr [[Q:%.*]], ptr [[N:%.*]]) {
+; CHECK-NEXT: [[ENTRY:.*:]]
+; CHECK-NEXT: [[P6:%.*]] = ptrtoint ptr [[P]] to i64
+; CHECK-NEXT: [[N5:%.*]] = ptrtoint ptr [[N]] to i64
+; CHECK-NEXT: [[P2:%.*]] = ptrtoint ptr [[P]] to i64
+; CHECK-NEXT: [[N1:%.*]] = ptrtoint ptr [[N]] to i64
+; CHECK-NEXT: [[TMP0:%.*]] = mul i64 [[N5]], 3074457345618258603
+; CHECK-NEXT: [[TMP1:%.*]] = mul i64 [[P6]], 3074457345618258603
+; CHECK-NEXT: [[TMP2:%.*]] = sub i64 [[TMP0]], [[TMP1]]
+; CHECK-NEXT: [[TMP3:%.*]] = lshr i64 [[TMP2]], 2
+; CHECK-NEXT: [[TMP4:%.*]] = add nuw nsw i64 [[TMP3]], 1
+; CHECK-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[TMP4]], 4
+; CHECK-NEXT: br i1 [[MIN_ITERS_CHECK]], label %[[SCALAR_PH:.*]], label %[[VECTOR_SCEVCHECK:.*]]
+; CHECK: [[VECTOR_SCEVCHECK]]:
+; CHECK-NEXT: [[TMP5:%.*]] = trunc i64 [[N1]] to i2
+; CHECK-NEXT: [[TMP6:%.*]] = trunc i64 [[P2]] to i2
+; CHECK-NEXT: [[TMP7:%.*]] = sub i2 [[TMP5]], [[TMP6]]
+; CHECK-NEXT: [[TMP8:%.*]] = zext i2 [[TMP7]] to i64
+; CHECK-NEXT: [[IDENT_CHECK:%.*]] = icmp ne i64 [[TMP8]], 0
+; CHECK-NEXT: [[TMP9:%.*]] = mul i64 [[N1]], 3074457345618258603
+; CHECK-NEXT: [[TMP10:%.*]] = mul i64 [[P2]], 3074457345618258603
+; CHECK-NEXT: [[TMP11:%.*]] = sub i64 [[TMP9]], [[TMP10]]
+; CHECK-NEXT: [[TMP12:%.*]] = lshr i64 [[TMP11]], 2
+; CHECK-NEXT: [[SCEVGEP:%.*]] = getelementptr i8, ptr [[P]], i64 4
+; CHECK-NEXT: [[MUL:%.*]] = call { i64, i1 } @llvm.umul.with.overflow.i64(i64 12, i64 [[TMP12]])
+; CHECK-NEXT: [[MUL_RESULT:%.*]] = extractvalue { i64, i1 } [[MUL]], 0
+; CHECK-NEXT: [[MUL_OVERFLOW:%.*]] = extractvalue { i64, i1 } [[MUL]], 1
+; CHECK-NEXT: [[TMP13:%.*]] = getelementptr i8, ptr [[SCEVGEP]], i64 [[MUL_RESULT]]
+; CHECK-NEXT: [[TMP14:%.*]] = icmp ult ptr [[TMP13]], [[SCEVGEP]]
+; CHECK-NEXT: [[TMP15:%.*]] = or i1 [[TMP14]], [[MUL_OVERFLOW]]
+; CHECK-NEXT: [[TMP16:%.*]] = or i1 [[IDENT_CHECK]], [[TMP15]]
+; CHECK-NEXT: br i1 [[TMP16]], label %[[SCALAR_PH]], label %[[VECTOR_MEMCHECK:.*]]
+; CHECK: [[VECTOR_MEMCHECK]]:
+; CHECK-NEXT: [[SCEVGEP3:%.*]] = getelementptr i8, ptr [[P]], i64 4
+; CHECK-NEXT: [[SCEVGEP4:%.*]] = getelementptr i8, ptr [[Q]], i64 4
+; CHECK-NEXT: [[BOUND0:%.*]] = icmp ult ptr [[P]], [[SCEVGEP4]]
+; CHECK-NEXT: [[BOUND1:%.*]] = icmp ult ptr [[Q]], [[SCEVGEP3]]
+; CHECK-NEXT: [[FOUND_CONFLICT:%.*]] = and i1 [[BOUND0]], [[BOUND1]]
+; CHECK-NEXT: br i1 [[FOUND_CONFLICT]], label %[[SCALAR_PH]], label %[[VECTOR_PH:.*]]
+; CHECK: [[VECTOR_PH]]:
+; CHECK-NEXT: [[N_MOD_VF:%.*]] = urem i64 [[TMP4]], 4
+; CHECK-NEXT: [[N_VEC:%.*]] = sub i64 [[TMP4]], [[N_MOD_VF]]
+; CHECK-NEXT: [[TMP17:%.*]] = load float, ptr [[Q]], align 4, !tbaa [[FLOAT_TBAA0:![0-9]+]], !alias.scope [[META6:![0-9]+]]
+; CHECK-NEXT: [[BROADCAST_SPLATINSERT:%.*]] = insertelement <4 x float> poison, float [[TMP17]], i64 0
+; CHECK-NEXT: [[BROADCAST_SPLAT:%.*]] = shufflevector <4 x float> [[BROADCAST_SPLATINSERT]], <4 x float> poison, <4 x i32> zeroinitializer
+; CHECK-NEXT: [[TMP18:%.*]] = mul i64 [[N_VEC]], 12
+; CHECK-NEXT: [[TMP19:%.*]] = getelementptr i8, ptr [[P]], i64 [[TMP18]]
+; CHECK-NEXT: br label %[[VECTOR_BODY:.*]]
+; CHECK: [[VECTOR_BODY]]:
+; CHECK-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-NEXT: [[TMP20:%.*]] = mul i64 [[INDEX]], 12
+; CHECK-NEXT: [[TMP21:%.*]] = add i64 [[TMP20]], 12
+; CHECK-NEXT: [[TMP22:%.*]] = add i64 [[TMP20]], 24
+; CHECK-NEXT: [[TMP23:%.*]] = add i64 [[TMP20]], 36
+; CHECK-NEXT: [[NEXT_GEP:%.*]] = getelementptr i8, ptr [[P]], i64 [[TMP20]]
+; CHECK-NEXT: [[NEXT_GEP7:%.*]] = getelementptr i8, ptr [[P]], i64 [[TMP21]]
+; CHECK-NEXT: [[NEXT_GEP8:%.*]] = getelementptr i8, ptr [[P]], i64 [[TMP22]]
+; CHECK-NEXT: [[NEXT_GEP9:%.*]] = getelementptr i8, ptr [[P]], i64 [[TMP23]]
+; CHECK-NEXT: store float 0.000000e+00, ptr [[NEXT_GEP]], align 4, !tbaa [[FLOAT_TBAA9:![0-9]+]]
+; CHECK-NEXT: store float 0.000000e+00, ptr [[NEXT_GEP7]], align 4, !tbaa [[FLOAT_TBAA9]]
+; CHECK-NEXT: store float 0.000000e+00, ptr [[NEXT_GEP8]], align 4, !tbaa [[FLOAT_TBAA9]]
+; CHECK-NEXT: store float 0.000000e+00, ptr [[NEXT_GEP9]], align 4, !tbaa [[FLOAT_TBAA9]]
+; CHECK-NEXT: [[TMP24:%.*]] = getelementptr i8, ptr [[NEXT_GEP]], i64 4
+; CHECK-NEXT: [[TMP25:%.*]] = getelementptr i8, ptr [[NEXT_GEP7]], i64 4
+; CHECK-NEXT: [[TMP26:%.*]] = getelementptr i8, ptr [[NEXT_GEP8]], i64 4
+; CHECK-NEXT: [[TMP27:%.*]] = getelementptr i8, ptr [[NEXT_GEP9]], i64 4
+; CHECK-NEXT: [[TMP28:%.*]] = load float, ptr [[TMP24]], align 4, !tbaa [[FLOAT_TBAA0]], !alias.scope [[META10:![0-9]+]]
+; CHECK-NEXT: [[TMP29:%.*]] = load float, ptr [[TMP25]], align 4, !tbaa [[FLOAT_TBAA0]], !alias.scope [[META10]]
+; CHECK-NEXT: [[TMP30:%.*]] = load float, ptr [[TMP26]], align 4, !tbaa [[FLOAT_TBAA0]], !alias.scope [[META10]]
+; CHECK-NEXT: [[TMP31:%.*]] = load float, ptr [[TMP27]], align 4, !tbaa [[FLOAT_TBAA0]], !alias.scope [[META10]]
+; CHECK-NEXT: [[TMP32:%.*]] = insertelement <4 x float> poison, float [[TMP28]], i32 0
+; CHECK-NEXT: [[TMP33:%.*]] = insertelement <4 x float> [[TMP32]], float [[TMP29]], i32 1
+; CHECK-NEXT: [[TMP34:%.*]] = insertelement <4 x float> [[TMP33]], float [[TMP30]], i32 2
+; CHECK-NEXT: [[TMP35:%.*]] = insertelement <4 x float> [[TMP34]], float [[TMP31]], i32 3
+; CHECK-NEXT: [[TMP36:%.*]] = fadd <4 x float> [[BROADCAST_SPLAT]], [[TMP35]]
+; CHECK-NEXT: [[TMP37:%.*]] = extractelement <4 x float> [[TMP36]], i64 3
+; CHECK-NEXT: store float [[TMP37]], ptr [[P]], align 4, !tbaa [[FLOAT_TBAA0]], !alias.scope [[META12:![0-9]+]], !noalias [[META6]]
+; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 4
+; CHECK-NEXT: [[TMP38:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
+; CHECK-NEXT: br i1 [[TMP38]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP14:![0-9]+]]
+; CHECK: [[MIDDLE_BLOCK]]:
+; CHECK-NEXT: [[CMP_N:%.*]] = icmp eq i64 [[TMP4]], [[N_VEC]]
+; CHECK-NEXT: br i1 [[CMP_N]], [[EXIT:label %.*]], label %[[SCALAR_PH]]
+; CHECK: [[SCALAR_PH]]:
+;
+entry:
+ br label %loop
+
+loop:
+ %ptr.iv = phi ptr [ %p, %entry ], [ %ptr.iv.next, %loop ]
+ store float 0.000000e+00, ptr %ptr.iv, align 4, !tbaa !0
+ %ld.q = load float, ptr %q, align 4, !tbaa !6
+ %gep.iv.4 = getelementptr i8, ptr %ptr.iv, i64 4
+ %ld.iv = load float, ptr %gep.iv.4, align 4, !tbaa !6
+ %res = fadd float %ld.q, %ld.iv
+ store float %res, ptr %p, align 4, !tbaa !6
+ %ptr.iv.next = getelementptr i8, ptr %ptr.iv, i64 12
+ %ec = icmp eq ptr %ptr.iv, %n
+ br i1 %ec, label %exit, label %loop
+
+exit:
+ ret void
+}
+
+!0 = !{!1, !2, i64 0}
+!1 = !{!"foo", !2, i64 0, !2, i64 4, !3, i64 8, !5, i64 9}
+!2 = !{!"float", !3, i64 0}
+!3 = !{!"omnipotent char", !4, i64 0}
+!4 = !{!"Simple C++ TBAA"}
+!5 = !{!"bool", !3, i64 0}
+!6 = !{!1, !2, i64 4}
>From 70eca57655235ff58c2623463e920f06fcd48239 Mon Sep 17 00:00:00 2001
From: Ramkumar Ramachandra <artagnon at tenstorrent.com>
Date: Tue, 3 Feb 2026 12:31:50 +0000
Subject: [PATCH 2/4] [VPlan] Extend licm to hoist replicate loads
This patch eliminates the need for a separate hoistInvariantLoads
transform. A planned follow-up is to get licm to sink replicate stores.
---
.../Transforms/Vectorize/VPlanTransforms.cpp | 64 ++++---------------
.../Transforms/Vectorize/VPlanTransforms.h | 5 --
llvm/lib/Transforms/Vectorize/VPlanUtils.h | 1 +
.../AArch64/conditional-branches-cost.ll | 10 +--
.../LoopVectorize/AArch64/predicated-costs.ll | 2 +-
.../LoopVectorize/AArch64/store-costs-sve.ll | 2 +-
.../VPlan/vplan-print-after-all.ll | 1 -
.../LoopVectorize/X86/cost-model.ll | 2 +-
.../LoopVectorize/hoist-invariant-loads.ll | 24 +++----
.../LoopVectorize/if-pred-stores.ll | 12 ++--
.../interleaved-accesses-metadata.ll | 4 +-
.../pointer-select-runtime-checks.ll | 6 +-
llvm/test/Transforms/LoopVectorize/pr50686.ll | 6 +-
...ive-path-inner-loop-with-runtime-checks.ll | 2 +-
14 files changed, 51 insertions(+), 90 deletions(-)
diff --git a/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp b/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp
index bcdb91a54e305..c4eb94b251008 100644
--- a/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp
+++ b/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp
@@ -2473,6 +2473,18 @@ void VPlanTransforms::cse(VPlan &Plan) {
}
}
+/// Return true if we do not know how to (mechanically) hoist or sink a
+/// non-memory or memory recipe \p R out of a loop region.
+static bool cannotHoistOrSinkRecipe(VPRecipeBase &R, VPBasicBlock *FirstBB,
+ VPBasicBlock *LastBB) {
+ if (!isa<VPReplicateRecipe>(R) || !R.mayReadFromMemory())
+ return vputils::cannotHoistOrSinkRecipe(R);
+
+ // Check that the load doesn't alias with stores between FirstBB and LastBB.
+ auto MemLoc = vputils::getMemoryLocation(R);
+ return !MemLoc || !canHoistOrSinkWithNoAliasCheck(*MemLoc, FirstBB, LastBB);
+}
+
/// Move loop-invariant recipes out of the vector loop region in \p Plan.
static void licm(VPlan &Plan) {
VPBasicBlock *Preheader = Plan.getVectorPreheader();
@@ -2488,7 +2500,8 @@ static void licm(VPlan &Plan) {
for (VPBasicBlock *VPBB : VPBlockUtils::blocksOnly<VPBasicBlock>(
vp_depth_first_shallow(LoopRegion->getEntry()))) {
for (VPRecipeBase &R : make_early_inc_range(*VPBB)) {
- if (vputils::cannotHoistOrSinkRecipe(R))
+ if (cannotHoistOrSinkRecipe(R, LoopRegion->getEntryBasicBlock(),
+ LoopRegion->getExitingBasicBlock()))
continue;
if (any_of(R.operands(), [](VPValue *Op) {
return !Op->isDefinedOutsideLoopRegions();
@@ -2740,7 +2753,6 @@ void VPlanTransforms::optimize(VPlan &Plan) {
RUN_VPLAN_PASS(removeDeadRecipes, Plan);
RUN_VPLAN_PASS(createAndOptimizeReplicateRegions, Plan);
- RUN_VPLAN_PASS(hoistInvariantLoads, Plan);
RUN_VPLAN_PASS(mergeBlocksIntoPredecessors, Plan);
RUN_VPLAN_PASS(licm, Plan);
}
@@ -4584,54 +4596,6 @@ void VPlanTransforms::materializeBroadcasts(VPlan &Plan) {
}
}
-void VPlanTransforms::hoistInvariantLoads(VPlan &Plan) {
- VPRegionBlock *LoopRegion = Plan.getVectorLoopRegion();
-
- // Collect candidate loads with invariant addresses and noalias scopes
- // metadata and memory-writing recipes with noalias metadata.
- SmallVector<std::pair<VPRecipeBase *, MemoryLocation>> CandidateLoads;
- SmallVector<MemoryLocation> Stores;
- for (VPBasicBlock *VPBB : VPBlockUtils::blocksOnly<VPBasicBlock>(
- vp_depth_first_shallow(LoopRegion->getEntry()))) {
- for (VPRecipeBase &R : *VPBB) {
- // Only handle single-scalar replicated loads with invariant addresses.
- if (auto *RepR = dyn_cast<VPReplicateRecipe>(&R)) {
- if (RepR->isPredicated() || !RepR->isSingleScalar() ||
- RepR->getOpcode() != Instruction::Load)
- continue;
-
- VPValue *Addr = RepR->getOperand(0);
- if (Addr->isDefinedOutsideLoopRegions()) {
- MemoryLocation Loc = *vputils::getMemoryLocation(*RepR);
- if (!Loc.AATags.Scope)
- continue;
- CandidateLoads.push_back({RepR, Loc});
- }
- }
- if (R.mayWriteToMemory()) {
- auto Loc = vputils::getMemoryLocation(R);
- if (!Loc || !Loc->AATags.Scope || !Loc->AATags.NoAlias)
- return;
- Stores.push_back(*Loc);
- }
- }
- }
-
- VPBasicBlock *Preheader = Plan.getVectorPreheader();
- for (auto &[LoadRecipe, LoadLoc] : CandidateLoads) {
- // Hoist the load to the preheader if it doesn't alias with any stores
- // according to the noalias metadata. Other loads should have been hoisted
- // by other passes
- const AAMDNodes &LoadAA = LoadLoc.AATags;
- if (all_of(Stores, [&](const MemoryLocation &StoreLoc) {
- return !ScopedNoAliasAAResult::mayAliasInScopes(
- LoadAA.Scope, StoreLoc.AATags.NoAlias);
- })) {
- LoadRecipe->moveBefore(*Preheader, Preheader->getFirstNonPhi());
- }
- }
-}
-
// Collect common metadata from a group of replicate recipes by intersecting
// metadata from all recipes in the group.
static VPIRMetadata getCommonMetadata(ArrayRef<VPReplicateRecipe *> Recipes) {
diff --git a/llvm/lib/Transforms/Vectorize/VPlanTransforms.h b/llvm/lib/Transforms/Vectorize/VPlanTransforms.h
index d0ce6ce71d80c..ce16ad5f3d82b 100644
--- a/llvm/lib/Transforms/Vectorize/VPlanTransforms.h
+++ b/llvm/lib/Transforms/Vectorize/VPlanTransforms.h
@@ -392,11 +392,6 @@ struct VPlanTransforms {
/// Add explicit broadcasts for live-ins and VPValues defined in \p Plan's entry block if they are used as vectors.
static void materializeBroadcasts(VPlan &Plan);
- /// Hoist single-scalar loads with invariant addresses out of the vector loop
- /// to the preheader, if they are proven not to alias with any stores in the
- /// plan using noalias metadata.
- static void hoistInvariantLoads(VPlan &Plan);
-
/// Hoist predicated loads from the same address to the loop entry block, if
/// they are guaranteed to execute on both paths (i.e., in replicate regions
/// with complementary masks P and NOT P).
diff --git a/llvm/lib/Transforms/Vectorize/VPlanUtils.h b/llvm/lib/Transforms/Vectorize/VPlanUtils.h
index ac3a1005c8f24..203ace4572535 100644
--- a/llvm/lib/Transforms/Vectorize/VPlanUtils.h
+++ b/llvm/lib/Transforms/Vectorize/VPlanUtils.h
@@ -74,6 +74,7 @@ unsigned getVFScaleFactor(VPRecipeBase *R);
/// Return true if we do not know how to (mechanically) hoist or sink \p R.
/// When sinking, passing \p Sinking = true ensures that assumes aren't sunk.
+/// Returns true for recipes that access memory.
bool cannotHoistOrSinkRecipe(const VPRecipeBase &R, bool Sinking = false);
/// Returns the VPValue representing the uncountable exit comparison used by
diff --git a/llvm/test/Transforms/LoopVectorize/AArch64/conditional-branches-cost.ll b/llvm/test/Transforms/LoopVectorize/AArch64/conditional-branches-cost.ll
index abee7f8a8d9c2..690a61e3e05c2 100644
--- a/llvm/test/Transforms/LoopVectorize/AArch64/conditional-branches-cost.ll
+++ b/llvm/test/Transforms/LoopVectorize/AArch64/conditional-branches-cost.ll
@@ -393,10 +393,10 @@ define i32 @header_mask_and_invariant_compare(ptr %A, ptr %B, ptr %C, ptr %D, pt
; DEFAULT: [[VECTOR_PH]]:
; DEFAULT-NEXT: [[N_MOD_VF:%.*]] = urem i64 [[TMP0]], 4
; DEFAULT-NEXT: [[N_VEC:%.*]] = sub i64 [[TMP0]], [[N_MOD_VF]]
-; DEFAULT-NEXT: [[TMP6:%.*]] = load i32, ptr [[C]], align 4, !alias.scope [[META8:![0-9]+]]
+; DEFAULT-NEXT: [[TMP3:%.*]] = load i32, ptr [[A]], align 4, !alias.scope [[META8:![0-9]+]]
; DEFAULT-NEXT: [[TMP4:%.*]] = load i32, ptr [[B]], align 4, !alias.scope [[META11:![0-9]+]]
-; DEFAULT-NEXT: [[TMP3:%.*]] = load i32, ptr [[A]], align 4, !alias.scope [[META13:![0-9]+]]
; DEFAULT-NEXT: [[TMP5:%.*]] = or i32 [[TMP4]], [[TMP3]]
+; DEFAULT-NEXT: [[TMP6:%.*]] = load i32, ptr [[C]], align 4, !alias.scope [[META13:![0-9]+]]
; DEFAULT-NEXT: [[TMP7:%.*]] = icmp ugt i32 [[TMP6]], [[TMP5]]
; DEFAULT-NEXT: [[BROADCAST_SPLATINSERT:%.*]] = insertelement <4 x i1> poison, i1 [[TMP7]], i64 0
; DEFAULT-NEXT: [[BROADCAST_SPLAT:%.*]] = shufflevector <4 x i1> [[BROADCAST_SPLATINSERT]], <4 x i1> poison, <4 x i32> zeroinitializer
@@ -477,13 +477,13 @@ define i32 @header_mask_and_invariant_compare(ptr %A, ptr %B, ptr %C, ptr %D, pt
; PRED: [[VECTOR_PH]]:
; PRED-NEXT: [[TMP8:%.*]] = call i64 @llvm.vscale.i64()
; PRED-NEXT: [[TMP9:%.*]] = shl nuw i64 [[TMP8]], 2
-; PRED-NEXT: [[L_C:%.*]] = load i32, ptr [[C]], align 4, !alias.scope [[META3:![0-9]+]]
-; PRED-NEXT: [[L_B:%.*]] = load i32, ptr [[B]], align 4, !alias.scope [[META6:![0-9]+]]
-; PRED-NEXT: [[L_A:%.*]] = load i32, ptr [[A]], align 4, !alias.scope [[META8:![0-9]+]]
; PRED-NEXT: [[ACTIVE_LANE_MASK_ENTRY:%.*]] = call <vscale x 4 x i1> @llvm.get.active.lane.mask.nxv4i1.i64(i64 0, i64 [[TMP0]])
+; PRED-NEXT: [[L_A:%.*]] = load i32, ptr [[A]], align 4, !alias.scope [[META3:![0-9]+]]
+; PRED-NEXT: [[L_B:%.*]] = load i32, ptr [[B]], align 4, !alias.scope [[META6:![0-9]+]]
; PRED-NEXT: [[OR:%.*]] = or i32 [[L_B]], [[L_A]]
; PRED-NEXT: [[BROADCAST_SPLATINSERT30:%.*]] = insertelement <vscale x 4 x i32> poison, i32 [[OR]], i64 0
; PRED-NEXT: [[BROADCAST_SPLAT31:%.*]] = shufflevector <vscale x 4 x i32> [[BROADCAST_SPLATINSERT30]], <vscale x 4 x i32> poison, <vscale x 4 x i32> zeroinitializer
+; PRED-NEXT: [[L_C:%.*]] = load i32, ptr [[C]], align 4, !alias.scope [[META8:![0-9]+]]
; PRED-NEXT: [[C_0:%.*]] = icmp ugt i32 [[L_C]], [[OR]]
; PRED-NEXT: [[BROADCAST_SPLATINSERT28:%.*]] = insertelement <vscale x 4 x i1> poison, i1 [[C_0]], i64 0
; PRED-NEXT: [[BROADCAST_SPLAT29:%.*]] = shufflevector <vscale x 4 x i1> [[BROADCAST_SPLATINSERT28]], <vscale x 4 x i1> poison, <vscale x 4 x i32> zeroinitializer
diff --git a/llvm/test/Transforms/LoopVectorize/AArch64/predicated-costs.ll b/llvm/test/Transforms/LoopVectorize/AArch64/predicated-costs.ll
index a57cdee91d9c2..6808eddd5a1c6 100644
--- a/llvm/test/Transforms/LoopVectorize/AArch64/predicated-costs.ll
+++ b/llvm/test/Transforms/LoopVectorize/AArch64/predicated-costs.ll
@@ -66,10 +66,10 @@ define void @test_predicated_load_cast_hint(ptr %dst.1, ptr %dst.2, ptr %src, i8
; CHECK-NEXT: [[CONFLICT_RDX15:%.*]] = or i1 [[CONFLICT_RDX]], [[FOUND_CONFLICT14]]
; CHECK-NEXT: br i1 [[CONFLICT_RDX15]], label %[[SCALAR_PH]], label %[[VECTOR_PH:.*]]
; CHECK: [[VECTOR_PH]]:
+; CHECK-NEXT: [[ACTIVE_LANE_MASK_ENTRY:%.*]] = call <4 x i1> @llvm.get.active.lane.mask.v4i1.i32(i32 0, i32 [[TMP2]])
; CHECK-NEXT: [[TMP28:%.*]] = load i8, ptr [[SRC]], align 1, !alias.scope [[META0:![0-9]+]], !noalias [[META3:![0-9]+]]
; CHECK-NEXT: [[BROADCAST_SPLATINSERT:%.*]] = insertelement <4 x i8> poison, i8 [[TMP28]], i64 0
; CHECK-NEXT: [[BROADCAST_SPLAT:%.*]] = shufflevector <4 x i8> [[BROADCAST_SPLATINSERT]], <4 x i8> poison, <4 x i32> zeroinitializer
-; CHECK-NEXT: [[ACTIVE_LANE_MASK_ENTRY:%.*]] = call <4 x i1> @llvm.get.active.lane.mask.v4i1.i32(i32 0, i32 [[TMP2]])
; CHECK-NEXT: [[TMP25:%.*]] = zext <4 x i8> [[BROADCAST_SPLAT]] to <4 x i64>
; CHECK-NEXT: br label %[[VECTOR_BODY:.*]]
; CHECK: [[VECTOR_BODY]]:
diff --git a/llvm/test/Transforms/LoopVectorize/AArch64/store-costs-sve.ll b/llvm/test/Transforms/LoopVectorize/AArch64/store-costs-sve.ll
index 1dc82491f944d..fdd23d30025a2 100644
--- a/llvm/test/Transforms/LoopVectorize/AArch64/store-costs-sve.ll
+++ b/llvm/test/Transforms/LoopVectorize/AArch64/store-costs-sve.ll
@@ -207,10 +207,10 @@ define void @trunc_store(ptr %dst, ptr %src, i16 %x) #1 {
; PRED-NEXT: [[TMP1:%.*]] = shl nuw i64 [[TMP0]], 4
; PRED-NEXT: [[BROADCAST_SPLATINSERT:%.*]] = insertelement <vscale x 16 x i16> poison, i16 [[X]], i64 0
; PRED-NEXT: [[BROADCAST_SPLAT:%.*]] = shufflevector <vscale x 16 x i16> [[BROADCAST_SPLATINSERT]], <vscale x 16 x i16> poison, <vscale x 16 x i32> zeroinitializer
+; PRED-NEXT: [[ACTIVE_LANE_MASK_ENTRY:%.*]] = call <vscale x 16 x i1> @llvm.get.active.lane.mask.nxv16i1.i64(i64 0, i64 1000)
; PRED-NEXT: [[TMP4:%.*]] = load i64, ptr [[SRC]], align 8, !alias.scope [[META3:![0-9]+]]
; PRED-NEXT: [[BROADCAST_SPLATINSERT2:%.*]] = insertelement <vscale x 16 x i64> poison, i64 [[TMP4]], i64 0
; PRED-NEXT: [[BROADCAST_SPLAT3:%.*]] = shufflevector <vscale x 16 x i64> [[BROADCAST_SPLATINSERT2]], <vscale x 16 x i64> poison, <vscale x 16 x i32> zeroinitializer
-; PRED-NEXT: [[ACTIVE_LANE_MASK_ENTRY:%.*]] = call <vscale x 16 x i1> @llvm.get.active.lane.mask.nxv16i1.i64(i64 0, i64 1000)
; PRED-NEXT: [[TMP3:%.*]] = trunc <vscale x 16 x i64> [[BROADCAST_SPLAT3]] to <vscale x 16 x i8>
; PRED-NEXT: [[TMP2:%.*]] = trunc <vscale x 16 x i16> [[BROADCAST_SPLAT]] to <vscale x 16 x i8>
; PRED-NEXT: [[TMP5:%.*]] = and <vscale x 16 x i8> [[TMP3]], [[TMP2]]
diff --git a/llvm/test/Transforms/LoopVectorize/VPlan/vplan-print-after-all.ll b/llvm/test/Transforms/LoopVectorize/VPlan/vplan-print-after-all.ll
index 1fb23a06401d4..a876bd5b6e2cd 100644
--- a/llvm/test/Transforms/LoopVectorize/VPlan/vplan-print-after-all.ll
+++ b/llvm/test/Transforms/LoopVectorize/VPlan/vplan-print-after-all.ll
@@ -44,7 +44,6 @@
; CHECK: VPlan for loop in 'foo' after removeBranchOnConst
; CHECK: VPlan for loop in 'foo' after removeDeadRecipes
; CHECK: VPlan for loop in 'foo' after createAndOptimizeReplicateRegions
-; CHECK: VPlan for loop in 'foo' after hoistInvariantLoads
; CHECK: VPlan for loop in 'foo' after mergeBlocksIntoPredecessors
; CHECK: VPlan for loop in 'foo' after licm
; CHECK: VPlan for loop in 'foo' after VPlanTransforms::optimize
diff --git a/llvm/test/Transforms/LoopVectorize/X86/cost-model.ll b/llvm/test/Transforms/LoopVectorize/X86/cost-model.ll
index 476bbe8fba51e..71a3c5bccb30a 100644
--- a/llvm/test/Transforms/LoopVectorize/X86/cost-model.ll
+++ b/llvm/test/Transforms/LoopVectorize/X86/cost-model.ll
@@ -355,8 +355,8 @@ define void @multi_exit(ptr %dst, ptr %src.1, ptr %src.2, i64 %A, i64 %B) #0 {
; CHECK-NEXT: [[TMP19:%.*]] = icmp eq i64 [[N_MOD_VF]], 0
; CHECK-NEXT: [[TMP20:%.*]] = select i1 [[TMP19]], i64 4, i64 [[N_MOD_VF]]
; CHECK-NEXT: [[N_VEC:%.*]] = sub i64 [[TMP2]], [[TMP20]]
-; CHECK-NEXT: [[TMP21:%.*]] = load i64, ptr [[SRC_2]], align 8, !alias.scope [[META6:![0-9]+]]
; CHECK-NEXT: [[TMP22:%.*]] = trunc i64 [[N_VEC]] to i32
+; CHECK-NEXT: [[TMP21:%.*]] = load i64, ptr [[SRC_2]], align 8, !alias.scope [[META6:![0-9]+]]
; CHECK-NEXT: [[TMP23:%.*]] = icmp ne i64 [[TMP21]], 0
; CHECK-NEXT: [[BROADCAST_SPLATINSERT:%.*]] = insertelement <2 x i1> poison, i1 [[TMP23]], i64 0
; CHECK-NEXT: [[BROADCAST_SPLAT:%.*]] = shufflevector <2 x i1> [[BROADCAST_SPLATINSERT]], <2 x i1> poison, <2 x i32> zeroinitializer
diff --git a/llvm/test/Transforms/LoopVectorize/hoist-invariant-loads.ll b/llvm/test/Transforms/LoopVectorize/hoist-invariant-loads.ll
index 1f7e58a9141b4..c4dd71bc12077 100644
--- a/llvm/test/Transforms/LoopVectorize/hoist-invariant-loads.ll
+++ b/llvm/test/Transforms/LoopVectorize/hoist-invariant-loads.ll
@@ -45,9 +45,6 @@ define void @load_not_hoisted(ptr %p, ptr %q, ptr %n) {
; CHECK: [[VECTOR_PH]]:
; CHECK-NEXT: [[N_MOD_VF:%.*]] = urem i64 [[TMP4]], 4
; CHECK-NEXT: [[N_VEC:%.*]] = sub i64 [[TMP4]], [[N_MOD_VF]]
-; CHECK-NEXT: [[TMP17:%.*]] = load float, ptr [[Q]], align 4, !tbaa [[FLOAT_TBAA0:![0-9]+]], !alias.scope [[META6:![0-9]+]]
-; CHECK-NEXT: [[BROADCAST_SPLATINSERT:%.*]] = insertelement <4 x float> poison, float [[TMP17]], i64 0
-; CHECK-NEXT: [[BROADCAST_SPLAT:%.*]] = shufflevector <4 x float> [[BROADCAST_SPLATINSERT]], <4 x float> poison, <4 x i32> zeroinitializer
; CHECK-NEXT: [[TMP18:%.*]] = mul i64 [[N_VEC]], 12
; CHECK-NEXT: [[TMP19:%.*]] = getelementptr i8, ptr [[P]], i64 [[TMP18]]
; CHECK-NEXT: br label %[[VECTOR_BODY:.*]]
@@ -61,25 +58,28 @@ define void @load_not_hoisted(ptr %p, ptr %q, ptr %n) {
; CHECK-NEXT: [[NEXT_GEP7:%.*]] = getelementptr i8, ptr [[P]], i64 [[TMP21]]
; CHECK-NEXT: [[NEXT_GEP8:%.*]] = getelementptr i8, ptr [[P]], i64 [[TMP22]]
; CHECK-NEXT: [[NEXT_GEP9:%.*]] = getelementptr i8, ptr [[P]], i64 [[TMP23]]
-; CHECK-NEXT: store float 0.000000e+00, ptr [[NEXT_GEP]], align 4, !tbaa [[FLOAT_TBAA9:![0-9]+]]
-; CHECK-NEXT: store float 0.000000e+00, ptr [[NEXT_GEP7]], align 4, !tbaa [[FLOAT_TBAA9]]
-; CHECK-NEXT: store float 0.000000e+00, ptr [[NEXT_GEP8]], align 4, !tbaa [[FLOAT_TBAA9]]
-; CHECK-NEXT: store float 0.000000e+00, ptr [[NEXT_GEP9]], align 4, !tbaa [[FLOAT_TBAA9]]
+; CHECK-NEXT: store float 0.000000e+00, ptr [[NEXT_GEP]], align 4, !tbaa [[FLOAT_TBAA0:![0-9]+]]
+; CHECK-NEXT: store float 0.000000e+00, ptr [[NEXT_GEP7]], align 4, !tbaa [[FLOAT_TBAA0]]
+; CHECK-NEXT: store float 0.000000e+00, ptr [[NEXT_GEP8]], align 4, !tbaa [[FLOAT_TBAA0]]
+; CHECK-NEXT: store float 0.000000e+00, ptr [[NEXT_GEP9]], align 4, !tbaa [[FLOAT_TBAA0]]
+; CHECK-NEXT: [[TMP39:%.*]] = load float, ptr [[Q]], align 4, !tbaa [[FLOAT_TBAA6:![0-9]+]], !alias.scope [[META7:![0-9]+]]
+; CHECK-NEXT: [[BROADCAST_SPLATINSERT:%.*]] = insertelement <4 x float> poison, float [[TMP39]], i64 0
+; CHECK-NEXT: [[BROADCAST_SPLAT:%.*]] = shufflevector <4 x float> [[BROADCAST_SPLATINSERT]], <4 x float> poison, <4 x i32> zeroinitializer
; CHECK-NEXT: [[TMP24:%.*]] = getelementptr i8, ptr [[NEXT_GEP]], i64 4
; CHECK-NEXT: [[TMP25:%.*]] = getelementptr i8, ptr [[NEXT_GEP7]], i64 4
; CHECK-NEXT: [[TMP26:%.*]] = getelementptr i8, ptr [[NEXT_GEP8]], i64 4
; CHECK-NEXT: [[TMP27:%.*]] = getelementptr i8, ptr [[NEXT_GEP9]], i64 4
-; CHECK-NEXT: [[TMP28:%.*]] = load float, ptr [[TMP24]], align 4, !tbaa [[FLOAT_TBAA0]], !alias.scope [[META10:![0-9]+]]
-; CHECK-NEXT: [[TMP29:%.*]] = load float, ptr [[TMP25]], align 4, !tbaa [[FLOAT_TBAA0]], !alias.scope [[META10]]
-; CHECK-NEXT: [[TMP30:%.*]] = load float, ptr [[TMP26]], align 4, !tbaa [[FLOAT_TBAA0]], !alias.scope [[META10]]
-; CHECK-NEXT: [[TMP31:%.*]] = load float, ptr [[TMP27]], align 4, !tbaa [[FLOAT_TBAA0]], !alias.scope [[META10]]
+; CHECK-NEXT: [[TMP28:%.*]] = load float, ptr [[TMP24]], align 4, !tbaa [[FLOAT_TBAA6]], !alias.scope [[META10:![0-9]+]]
+; CHECK-NEXT: [[TMP29:%.*]] = load float, ptr [[TMP25]], align 4, !tbaa [[FLOAT_TBAA6]], !alias.scope [[META10]]
+; CHECK-NEXT: [[TMP30:%.*]] = load float, ptr [[TMP26]], align 4, !tbaa [[FLOAT_TBAA6]], !alias.scope [[META10]]
+; CHECK-NEXT: [[TMP31:%.*]] = load float, ptr [[TMP27]], align 4, !tbaa [[FLOAT_TBAA6]], !alias.scope [[META10]]
; CHECK-NEXT: [[TMP32:%.*]] = insertelement <4 x float> poison, float [[TMP28]], i32 0
; CHECK-NEXT: [[TMP33:%.*]] = insertelement <4 x float> [[TMP32]], float [[TMP29]], i32 1
; CHECK-NEXT: [[TMP34:%.*]] = insertelement <4 x float> [[TMP33]], float [[TMP30]], i32 2
; CHECK-NEXT: [[TMP35:%.*]] = insertelement <4 x float> [[TMP34]], float [[TMP31]], i32 3
; CHECK-NEXT: [[TMP36:%.*]] = fadd <4 x float> [[BROADCAST_SPLAT]], [[TMP35]]
; CHECK-NEXT: [[TMP37:%.*]] = extractelement <4 x float> [[TMP36]], i64 3
-; CHECK-NEXT: store float [[TMP37]], ptr [[P]], align 4, !tbaa [[FLOAT_TBAA0]], !alias.scope [[META12:![0-9]+]], !noalias [[META6]]
+; CHECK-NEXT: store float [[TMP37]], ptr [[P]], align 4, !tbaa [[FLOAT_TBAA6]], !alias.scope [[META12:![0-9]+]], !noalias [[META7]]
; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 4
; CHECK-NEXT: [[TMP38:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
; CHECK-NEXT: br i1 [[TMP38]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP14:![0-9]+]]
diff --git a/llvm/test/Transforms/LoopVectorize/if-pred-stores.ll b/llvm/test/Transforms/LoopVectorize/if-pred-stores.ll
index db56646fe34b5..97fb7ed7dcb91 100644
--- a/llvm/test/Transforms/LoopVectorize/if-pred-stores.ll
+++ b/llvm/test/Transforms/LoopVectorize/if-pred-stores.ll
@@ -1054,15 +1054,17 @@ define void @hoistable_predicated_store(ptr %A, ptr %B, ptr %C, ptr %D) {
; VEC-NEXT: [[CONFLICT_RDX18:%.*]] = or i1 [[CONFLICT_RDX14]], [[FOUND_CONFLICT17]]
; VEC-NEXT: br i1 [[CONFLICT_RDX18]], label %[[SCALAR_PH:.*]], label %[[VECTOR_BODY:.*]]
; VEC: [[VECTOR_BODY]]:
-; VEC-NEXT: [[INDEX:%.*]] = phi i64 [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ], [ 0, %[[ENTRY]] ]
-; VEC-NEXT: store i32 0, ptr [[C]], align 4, !alias.scope [[META15:![0-9]+]], !noalias [[META18:![0-9]+]]
-; VEC-NEXT: [[TMP0:%.*]] = load i32, ptr [[A]], align 8, !alias.scope [[META22:![0-9]+]]
+; VEC-NEXT: [[TMP0:%.*]] = load i32, ptr [[A]], align 8, !alias.scope [[META15:![0-9]+]]
+; VEC-NEXT: br label %[[VECTOR_BODY1:.*]]
+; VEC: [[VECTOR_BODY1]]:
+; VEC-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_BODY]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY1]] ]
+; VEC-NEXT: store i32 0, ptr [[C]], align 4, !alias.scope [[META18:![0-9]+]], !noalias [[META20:![0-9]+]]
; VEC-NEXT: store i32 [[TMP0]], ptr [[B]], align 4, !alias.scope [[META23:![0-9]+]], !noalias [[META24:![0-9]+]]
; VEC-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 2
; VEC-NEXT: [[TMP1:%.*]] = icmp eq i64 [[INDEX_NEXT]], 100
-; VEC-NEXT: br i1 [[TMP1]], label %[[SCALAR_PH]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP25:![0-9]+]]
+; VEC-NEXT: br i1 [[TMP1]], label %[[SCALAR_PH]], label %[[VECTOR_BODY1]], !llvm.loop [[LOOP25:![0-9]+]]
; VEC: [[SCALAR_PH]]:
-; VEC-NEXT: [[BC_RESUME_VAL:%.*]] = phi i64 [ 0, %[[ENTRY]] ], [ 100, %[[VECTOR_BODY]] ]
+; VEC-NEXT: [[BC_RESUME_VAL:%.*]] = phi i64 [ 0, %[[ENTRY]] ], [ 100, %[[VECTOR_BODY1]] ]
; VEC-NEXT: br label %[[LOOP:.*]]
; VEC: [[LOOP]]:
; VEC-NEXT: [[IV:%.*]] = phi i64 [ [[BC_RESUME_VAL]], %[[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], %[[LOOP]] ]
diff --git a/llvm/test/Transforms/LoopVectorize/interleaved-accesses-metadata.ll b/llvm/test/Transforms/LoopVectorize/interleaved-accesses-metadata.ll
index a8ad9121475a1..e9dfebff0d809 100644
--- a/llvm/test/Transforms/LoopVectorize/interleaved-accesses-metadata.ll
+++ b/llvm/test/Transforms/LoopVectorize/interleaved-accesses-metadata.ll
@@ -101,11 +101,11 @@ define void @ir_tbaa_different(ptr %base, ptr %end, ptr %src) {
; CHECK: [[VECTOR_PH]]:
; CHECK-NEXT: [[N_MOD_VF:%.*]] = urem i64 [[TMP3]], 2
; CHECK-NEXT: [[N_VEC:%.*]] = sub i64 [[TMP3]], [[N_MOD_VF]]
+; CHECK-NEXT: [[TMP4:%.*]] = shl i64 [[N_VEC]], 3
+; CHECK-NEXT: [[TMP5:%.*]] = getelementptr i8, ptr [[BASE]], i64 [[TMP4]]
; CHECK-NEXT: [[TMP11:%.*]] = load float, ptr [[SRC]], align 4, !alias.scope [[META10:![0-9]+]]
; CHECK-NEXT: [[BROADCAST_SPLATINSERT:%.*]] = insertelement <2 x float> poison, float [[TMP11]], i64 0
; CHECK-NEXT: [[BROADCAST_SPLAT:%.*]] = shufflevector <2 x float> [[BROADCAST_SPLATINSERT]], <2 x float> poison, <2 x i32> zeroinitializer
-; CHECK-NEXT: [[TMP4:%.*]] = shl i64 [[N_VEC]], 3
-; CHECK-NEXT: [[TMP5:%.*]] = getelementptr i8, ptr [[BASE]], i64 [[TMP4]]
; CHECK-NEXT: br label %[[VECTOR_BODY:.*]]
; CHECK: [[VECTOR_BODY]]:
; CHECK-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
diff --git a/llvm/test/Transforms/LoopVectorize/pointer-select-runtime-checks.ll b/llvm/test/Transforms/LoopVectorize/pointer-select-runtime-checks.ll
index 66ab7939c3cac..9b383bcb88a69 100644
--- a/llvm/test/Transforms/LoopVectorize/pointer-select-runtime-checks.ll
+++ b/llvm/test/Transforms/LoopVectorize/pointer-select-runtime-checks.ll
@@ -23,8 +23,8 @@ define void @test1_select_invariant(ptr %src.1, ptr %src.2, ptr %dst, i1 %c, i8
; CHECK: vector.ph:
; CHECK-NEXT: [[N_MOD_VF:%.*]] = urem i32 [[TMP2]], 2
; CHECK-NEXT: [[N_VEC:%.*]] = sub i32 [[TMP2]], [[N_MOD_VF]]
-; CHECK-NEXT: [[TMP6:%.*]] = load i8, ptr [[PTR_SEL]], align 8, !alias.scope [[META0:![0-9]+]]
; CHECK-NEXT: [[IND_END:%.*]] = trunc i32 [[N_VEC]] to i8
+; CHECK-NEXT: [[TMP7:%.*]] = load i8, ptr [[PTR_SEL]], align 8, !alias.scope [[META0:![0-9]+]]
; CHECK-NEXT: br label [[VECTOR_BODY:%.*]]
; CHECK: vector.body:
; CHECK-NEXT: [[INDEX:%.*]] = phi i32 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ]
@@ -32,8 +32,8 @@ define void @test1_select_invariant(ptr %src.1, ptr %src.2, ptr %dst, i1 %c, i8
; CHECK-NEXT: [[INDUCTION2:%.*]] = add i8 [[OFFSET_IDX]], 1
; CHECK-NEXT: [[TMP11:%.*]] = getelementptr i8, ptr [[DST]], i8 [[OFFSET_IDX]]
; CHECK-NEXT: [[TMP8:%.*]] = getelementptr i8, ptr [[DST]], i8 [[INDUCTION2]]
-; CHECK-NEXT: store i8 [[TMP6]], ptr [[TMP11]], align 2, !alias.scope [[META3:![0-9]+]], !noalias [[META0]]
-; CHECK-NEXT: store i8 [[TMP6]], ptr [[TMP8]], align 2, !alias.scope [[META3]], !noalias [[META0]]
+; CHECK-NEXT: store i8 [[TMP7]], ptr [[TMP11]], align 2, !alias.scope [[META3:![0-9]+]], !noalias [[META0]]
+; CHECK-NEXT: store i8 [[TMP7]], ptr [[TMP8]], align 2, !alias.scope [[META3]], !noalias [[META0]]
; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i32 [[INDEX]], 2
; CHECK-NEXT: [[TMP9:%.*]] = icmp eq i32 [[INDEX_NEXT]], [[N_VEC]]
; CHECK-NEXT: br i1 [[TMP9]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP5:![0-9]+]]
diff --git a/llvm/test/Transforms/LoopVectorize/pr50686.ll b/llvm/test/Transforms/LoopVectorize/pr50686.ll
index 5e1f1b3cbed7e..2d13256c9757c 100644
--- a/llvm/test/Transforms/LoopVectorize/pr50686.ll
+++ b/llvm/test/Transforms/LoopVectorize/pr50686.ll
@@ -15,11 +15,11 @@ define void @m(ptr nocapture %p, ptr nocapture %p2, i32 %q) {
; CHECK-NEXT: [[FOUND_CONFLICT:%.*]] = and i1 [[BOUND0]], [[BOUND1]]
; CHECK-NEXT: br i1 [[FOUND_CONFLICT]], label [[SCALAR_PH:%.*]], label [[VECTOR_PH:%.*]]
; CHECK: vector.ph:
-; CHECK-NEXT: [[TMP4:%.*]] = load i32, ptr [[ARRAYIDX9_2]], align 4, !alias.scope [[META0:![0-9]+]]
-; CHECK-NEXT: [[TMP2:%.*]] = load i32, ptr [[ARRAYIDX9_1]], align 4, !alias.scope [[META0]]
-; CHECK-NEXT: [[TMP0:%.*]] = load i32, ptr [[P2]], align 4, !alias.scope [[META0]]
+; CHECK-NEXT: [[TMP0:%.*]] = load i32, ptr [[P2]], align 4, !alias.scope [[META0:![0-9]+]]
; CHECK-NEXT: [[TMP1:%.*]] = sub nsw i32 0, [[TMP0]]
+; CHECK-NEXT: [[TMP2:%.*]] = load i32, ptr [[ARRAYIDX9_1]], align 4, !alias.scope [[META0]]
; CHECK-NEXT: [[TMP3:%.*]] = sub nsw i32 [[TMP1]], [[TMP2]]
+; CHECK-NEXT: [[TMP4:%.*]] = load i32, ptr [[ARRAYIDX9_2]], align 4, !alias.scope [[META0]]
; CHECK-NEXT: [[TMP5:%.*]] = sub nsw i32 [[TMP3]], [[TMP4]]
; CHECK-NEXT: [[BROADCAST_SPLATINSERT4:%.*]] = insertelement <4 x i32> poison, i32 [[TMP5]], i64 0
; CHECK-NEXT: [[BROADCAST_SPLAT5:%.*]] = shufflevector <4 x i32> [[BROADCAST_SPLATINSERT4]], <4 x i32> poison, <4 x i32> zeroinitializer
diff --git a/llvm/test/Transforms/LoopVectorize/vplan-native-path-inner-loop-with-runtime-checks.ll b/llvm/test/Transforms/LoopVectorize/vplan-native-path-inner-loop-with-runtime-checks.ll
index 14d82834bdfda..9b62a6391a663 100644
--- a/llvm/test/Transforms/LoopVectorize/vplan-native-path-inner-loop-with-runtime-checks.ll
+++ b/llvm/test/Transforms/LoopVectorize/vplan-native-path-inner-loop-with-runtime-checks.ll
@@ -50,8 +50,8 @@ define void @expand(ptr %src, ptr %dst, i64 %0) {
; CHECK: [[VECTOR_PH]]:
; CHECK-NEXT: [[N_MOD_VF:%.*]] = urem i64 [[TMP8]], 4
; CHECK-NEXT: [[N_VEC:%.*]] = sub i64 [[TMP8]], [[N_MOD_VF]]
-; CHECK-NEXT: [[TMP19:%.*]] = load double, ptr [[SRC]], align 8, !alias.scope [[META0:![0-9]+]], !noalias [[META3:![0-9]+]]
; CHECK-NEXT: [[TMP18:%.*]] = add i64 [[TMP0]], [[N_VEC]]
+; CHECK-NEXT: [[TMP19:%.*]] = load double, ptr [[SRC]], align 8, !alias.scope [[META0:![0-9]+]], !noalias [[META3:![0-9]+]]
; CHECK-NEXT: [[DOTSPLATINSERT:%.*]] = insertelement <4 x i64> poison, i64 [[TMP0]], i64 0
; CHECK-NEXT: [[DOTSPLAT:%.*]] = shufflevector <4 x i64> [[DOTSPLATINSERT]], <4 x i64> poison, <4 x i32> zeroinitializer
; CHECK-NEXT: [[INDUCTION:%.*]] = add <4 x i64> [[DOTSPLAT]], <i64 0, i64 1, i64 2, i64 3>
>From ef280819e3cc7f7ccb748adad4476d75c54b7096 Mon Sep 17 00:00:00 2001
From: Ramkumar Ramachandra <artagnon at tenstorrent.com>
Date: Sat, 2 May 2026 21:15:59 +0100
Subject: [PATCH 3/4] [LV/test] Move test, add TODO
---
...nd-sink-mem-ops-with-invariant-pointers.ll | 175 +++++++++++++-----
.../LoopVectorize/hoist-invariant-loads.ll | 116 ------------
2 files changed, 124 insertions(+), 167 deletions(-)
delete mode 100644 llvm/test/Transforms/LoopVectorize/hoist-invariant-loads.ll
diff --git a/llvm/test/Transforms/LoopVectorize/hoist-and-sink-mem-ops-with-invariant-pointers.ll b/llvm/test/Transforms/LoopVectorize/hoist-and-sink-mem-ops-with-invariant-pointers.ll
index 4079d8f364cbe..c1815226a3883 100644
--- a/llvm/test/Transforms/LoopVectorize/hoist-and-sink-mem-ops-with-invariant-pointers.ll
+++ b/llvm/test/Transforms/LoopVectorize/hoist-and-sink-mem-ops-with-invariant-pointers.ll
@@ -1,10 +1,10 @@
-; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --check-globals none --version 6
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --check-globals none --filter-out-after "^scalar.ph" --version 6
; RUN: opt -passes=loop-vectorize -force-vector-width=4 -force-vector-interleave=1 -S %s | FileCheck %s
define void @hoist_invariant_load_noalias_due_to_memchecks(ptr %dst, ptr %invariant_ptr, i32 %n) {
; CHECK-LABEL: define void @hoist_invariant_load_noalias_due_to_memchecks(
; CHECK-SAME: ptr [[DST:%.*]], ptr [[INVARIANT_PTR:%.*]], i32 [[N:%.*]]) {
-; CHECK-NEXT: [[ENTRY:.*]]:
+; CHECK-NEXT: [[ENTRY:.*:]]
; CHECK-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ult i32 [[N]], 4
; CHECK-NEXT: br i1 [[MIN_ITERS_CHECK]], label %[[SCALAR_PH:.*]], label %[[VECTOR_MEMCHECK:.*]]
; CHECK: [[VECTOR_MEMCHECK]]:
@@ -34,20 +34,8 @@ define void @hoist_invariant_load_noalias_due_to_memchecks(ptr %dst, ptr %invari
; CHECK-NEXT: br i1 [[TMP6]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP5:![0-9]+]]
; CHECK: [[MIDDLE_BLOCK]]:
; CHECK-NEXT: [[CMP_N:%.*]] = icmp eq i32 [[N]], [[N_VEC]]
-; CHECK-NEXT: br i1 [[CMP_N]], label %[[EXIT:.*]], label %[[SCALAR_PH]]
+; CHECK-NEXT: br i1 [[CMP_N]], [[EXIT:label %.*]], label %[[SCALAR_PH]]
; CHECK: [[SCALAR_PH]]:
-; CHECK-NEXT: [[BC_RESUME_VAL:%.*]] = phi i32 [ [[N_VEC]], %[[MIDDLE_BLOCK]] ], [ 0, %[[ENTRY]] ], [ 0, %[[VECTOR_MEMCHECK]] ]
-; CHECK-NEXT: br label %[[LOOP:.*]]
-; CHECK: [[LOOP]]:
-; CHECK-NEXT: [[IV:%.*]] = phi i32 [ [[BC_RESUME_VAL]], %[[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], %[[LOOP]] ]
-; CHECK-NEXT: [[INV_VAL:%.*]] = load i32, ptr [[INVARIANT_PTR]], align 4
-; CHECK-NEXT: [[GEP:%.*]] = getelementptr inbounds i32, ptr [[DST]], i32 [[IV]]
-; CHECK-NEXT: store i32 [[INV_VAL]], ptr [[GEP]], align 4
-; CHECK-NEXT: [[IV_NEXT]] = add nuw nsw i32 [[IV]], 1
-; CHECK-NEXT: [[EC:%.*]] = icmp eq i32 [[IV_NEXT]], [[N]]
-; CHECK-NEXT: br i1 [[EC]], label %[[EXIT]], label %[[LOOP]], !llvm.loop [[LOOP8:![0-9]+]]
-; CHECK: [[EXIT]]:
-; CHECK-NEXT: ret void
;
entry:
br label %loop
@@ -69,7 +57,7 @@ exit:
define void @dont_hoist_variant_address(ptr %dst, ptr %src, i32 %n) {
; CHECK-LABEL: define void @dont_hoist_variant_address(
; CHECK-SAME: ptr [[DST:%.*]], ptr [[SRC:%.*]], i32 [[N:%.*]]) {
-; CHECK-NEXT: [[ENTRY:.*]]:
+; CHECK-NEXT: [[ENTRY:.*:]]
; CHECK-NEXT: [[SRC2:%.*]] = ptrtoaddr ptr [[SRC]] to i64
; CHECK-NEXT: [[A1:%.*]] = ptrtoaddr ptr [[DST]] to i64
; CHECK-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ult i32 [[N]], 4
@@ -93,21 +81,8 @@ define void @dont_hoist_variant_address(ptr %dst, ptr %src, i32 %n) {
; CHECK-NEXT: br i1 [[TMP3]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP9:![0-9]+]]
; CHECK: [[MIDDLE_BLOCK]]:
; CHECK-NEXT: [[CMP_N:%.*]] = icmp eq i32 [[N]], [[N_VEC]]
-; CHECK-NEXT: br i1 [[CMP_N]], label %[[EXIT:.*]], label %[[SCALAR_PH]]
+; CHECK-NEXT: br i1 [[CMP_N]], [[EXIT:label %.*]], label %[[SCALAR_PH]]
; CHECK: [[SCALAR_PH]]:
-; CHECK-NEXT: [[BC_RESUME_VAL:%.*]] = phi i32 [ [[N_VEC]], %[[MIDDLE_BLOCK]] ], [ 0, %[[ENTRY]] ], [ 0, %[[VECTOR_MEMCHECK]] ]
-; CHECK-NEXT: br label %[[LOOP:.*]]
-; CHECK: [[LOOP]]:
-; CHECK-NEXT: [[IV:%.*]] = phi i32 [ [[BC_RESUME_VAL]], %[[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], %[[LOOP]] ]
-; CHECK-NEXT: [[GEP_SRC:%.*]] = getelementptr inbounds i32, ptr [[SRC]], i32 [[IV]]
-; CHECK-NEXT: [[VAL:%.*]] = load i32, ptr [[GEP_SRC]], align 4
-; CHECK-NEXT: [[GEP_DST:%.*]] = getelementptr inbounds i32, ptr [[DST]], i32 [[IV]]
-; CHECK-NEXT: store i32 [[VAL]], ptr [[GEP_DST]], align 4
-; CHECK-NEXT: [[IV_NEXT]] = add nuw nsw i32 [[IV]], 1
-; CHECK-NEXT: [[EC:%.*]] = icmp eq i32 [[IV_NEXT]], [[N]]
-; CHECK-NEXT: br i1 [[EC]], label %[[EXIT]], label %[[LOOP]], !llvm.loop [[LOOP10:![0-9]+]]
-; CHECK: [[EXIT]]:
-; CHECK-NEXT: ret void
;
entry:
br label %loop
@@ -130,7 +105,7 @@ exit:
define void @dont_hoist_predicated_load(ptr %dst, ptr %invariant_ptr, ptr %cond_ptr, i32 %n) {
; CHECK-LABEL: define void @dont_hoist_predicated_load(
; CHECK-SAME: ptr [[DST:%.*]], ptr [[INVARIANT_PTR:%.*]], ptr [[COND_PTR:%.*]], i32 [[N:%.*]]) {
-; CHECK-NEXT: [[ENTRY:.*]]:
+; CHECK-NEXT: [[ENTRY:.*:]]
; CHECK-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ult i32 [[N]], 4
; CHECK-NEXT: br i1 [[MIN_ITERS_CHECK]], label %[[SCALAR_PH:.*]], label %[[VECTOR_MEMCHECK:.*]]
; CHECK: [[VECTOR_MEMCHECK]]:
@@ -198,27 +173,8 @@ define void @dont_hoist_predicated_load(ptr %dst, ptr %invariant_ptr, ptr %cond_
; CHECK-NEXT: br i1 [[TMP18]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP19:![0-9]+]]
; CHECK: [[MIDDLE_BLOCK]]:
; CHECK-NEXT: [[CMP_N:%.*]] = icmp eq i32 [[N]], [[N_VEC]]
-; CHECK-NEXT: br i1 [[CMP_N]], label %[[EXIT:.*]], label %[[SCALAR_PH]]
+; CHECK-NEXT: br i1 [[CMP_N]], [[EXIT:label %.*]], label %[[SCALAR_PH]]
; CHECK: [[SCALAR_PH]]:
-; CHECK-NEXT: [[BC_RESUME_VAL:%.*]] = phi i32 [ [[N_VEC]], %[[MIDDLE_BLOCK]] ], [ 0, %[[ENTRY]] ], [ 0, %[[VECTOR_MEMCHECK]] ]
-; CHECK-NEXT: br label %[[LOOP:.*]]
-; CHECK: [[LOOP]]:
-; CHECK-NEXT: [[IV:%.*]] = phi i32 [ [[BC_RESUME_VAL]], %[[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], %[[LOOP_LATCH:.*]] ]
-; CHECK-NEXT: [[GEP_COND:%.*]] = getelementptr inbounds i32, ptr [[COND_PTR]], i32 [[IV]]
-; CHECK-NEXT: [[COND:%.*]] = load i32, ptr [[GEP_COND]], align 4
-; CHECK-NEXT: [[CMP:%.*]] = icmp sgt i32 [[COND]], 0
-; CHECK-NEXT: br i1 [[CMP]], label %[[IF_THEN:.*]], label %[[LOOP_LATCH]]
-; CHECK: [[IF_THEN]]:
-; CHECK-NEXT: [[INV_VAL:%.*]] = load i32, ptr [[INVARIANT_PTR]], align 4
-; CHECK-NEXT: [[GEP:%.*]] = getelementptr inbounds i32, ptr [[DST]], i32 [[IV]]
-; CHECK-NEXT: store i32 [[INV_VAL]], ptr [[GEP]], align 4
-; CHECK-NEXT: br label %[[LOOP_LATCH]]
-; CHECK: [[LOOP_LATCH]]:
-; CHECK-NEXT: [[IV_NEXT]] = add nuw nsw i32 [[IV]], 1
-; CHECK-NEXT: [[EC:%.*]] = icmp eq i32 [[IV_NEXT]], [[N]]
-; CHECK-NEXT: br i1 [[EC]], label %[[EXIT]], label %[[LOOP]], !llvm.loop [[LOOP20:![0-9]+]]
-; CHECK: [[EXIT]]:
-; CHECK-NEXT: ret void
;
entry:
br label %loop
@@ -244,3 +200,120 @@ loop.latch:
exit:
ret void
}
+
+; TODO: Here, the load of q can be hoisted past the preceding store,
+; as it doesn't alias via TBAA. However, we do not hoist it at the
+; moment because there no scoped noalias metadata.
+define void @load_store_noalias_via_tbaa(ptr %p, ptr %q, ptr %n) {
+; CHECK-LABEL: define void @load_store_noalias_via_tbaa(
+; CHECK-SAME: ptr [[P:%.*]], ptr [[Q:%.*]], ptr [[N:%.*]]) {
+; CHECK-NEXT: [[ENTRY:.*:]]
+; CHECK-NEXT: [[P6:%.*]] = ptrtoint ptr [[P]] to i64
+; CHECK-NEXT: [[N5:%.*]] = ptrtoint ptr [[N]] to i64
+; CHECK-NEXT: [[P2:%.*]] = ptrtoint ptr [[P]] to i64
+; CHECK-NEXT: [[N1:%.*]] = ptrtoint ptr [[N]] to i64
+; CHECK-NEXT: [[TMP0:%.*]] = mul i64 [[N5]], 3074457345618258603
+; CHECK-NEXT: [[TMP1:%.*]] = mul i64 [[P6]], 3074457345618258603
+; CHECK-NEXT: [[TMP2:%.*]] = sub i64 [[TMP0]], [[TMP1]]
+; CHECK-NEXT: [[TMP3:%.*]] = lshr i64 [[TMP2]], 2
+; CHECK-NEXT: [[TMP4:%.*]] = add nuw nsw i64 [[TMP3]], 1
+; CHECK-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[TMP4]], 4
+; CHECK-NEXT: br i1 [[MIN_ITERS_CHECK]], label %[[SCALAR_PH:.*]], label %[[VECTOR_SCEVCHECK:.*]]
+; CHECK: [[VECTOR_SCEVCHECK]]:
+; CHECK-NEXT: [[TMP5:%.*]] = trunc i64 [[N1]] to i2
+; CHECK-NEXT: [[TMP6:%.*]] = trunc i64 [[P2]] to i2
+; CHECK-NEXT: [[TMP7:%.*]] = sub i2 [[TMP5]], [[TMP6]]
+; CHECK-NEXT: [[TMP8:%.*]] = zext i2 [[TMP7]] to i64
+; CHECK-NEXT: [[IDENT_CHECK:%.*]] = icmp ne i64 [[TMP8]], 0
+; CHECK-NEXT: [[TMP9:%.*]] = mul i64 [[N1]], 3074457345618258603
+; CHECK-NEXT: [[TMP10:%.*]] = mul i64 [[P2]], 3074457345618258603
+; CHECK-NEXT: [[TMP11:%.*]] = sub i64 [[TMP9]], [[TMP10]]
+; CHECK-NEXT: [[TMP12:%.*]] = lshr i64 [[TMP11]], 2
+; CHECK-NEXT: [[SCEVGEP:%.*]] = getelementptr i8, ptr [[P]], i64 4
+; CHECK-NEXT: [[MUL:%.*]] = call { i64, i1 } @llvm.umul.with.overflow.i64(i64 12, i64 [[TMP12]])
+; CHECK-NEXT: [[MUL_RESULT:%.*]] = extractvalue { i64, i1 } [[MUL]], 0
+; CHECK-NEXT: [[MUL_OVERFLOW:%.*]] = extractvalue { i64, i1 } [[MUL]], 1
+; CHECK-NEXT: [[TMP13:%.*]] = getelementptr i8, ptr [[SCEVGEP]], i64 [[MUL_RESULT]]
+; CHECK-NEXT: [[TMP14:%.*]] = icmp ult ptr [[TMP13]], [[SCEVGEP]]
+; CHECK-NEXT: [[TMP15:%.*]] = or i1 [[TMP14]], [[MUL_OVERFLOW]]
+; CHECK-NEXT: [[TMP16:%.*]] = or i1 [[IDENT_CHECK]], [[TMP15]]
+; CHECK-NEXT: br i1 [[TMP16]], label %[[SCALAR_PH]], label %[[VECTOR_MEMCHECK:.*]]
+; CHECK: [[VECTOR_MEMCHECK]]:
+; CHECK-NEXT: [[SCEVGEP3:%.*]] = getelementptr i8, ptr [[P]], i64 4
+; CHECK-NEXT: [[SCEVGEP4:%.*]] = getelementptr i8, ptr [[Q]], i64 4
+; CHECK-NEXT: [[BOUND0:%.*]] = icmp ult ptr [[P]], [[SCEVGEP4]]
+; CHECK-NEXT: [[BOUND1:%.*]] = icmp ult ptr [[Q]], [[SCEVGEP3]]
+; CHECK-NEXT: [[FOUND_CONFLICT:%.*]] = and i1 [[BOUND0]], [[BOUND1]]
+; CHECK-NEXT: br i1 [[FOUND_CONFLICT]], label %[[SCALAR_PH]], label %[[VECTOR_PH:.*]]
+; CHECK: [[VECTOR_PH]]:
+; CHECK-NEXT: [[N_MOD_VF:%.*]] = urem i64 [[TMP4]], 4
+; CHECK-NEXT: [[N_VEC:%.*]] = sub i64 [[TMP4]], [[N_MOD_VF]]
+; CHECK-NEXT: [[TMP18:%.*]] = mul i64 [[N_VEC]], 12
+; CHECK-NEXT: [[TMP19:%.*]] = getelementptr i8, ptr [[P]], i64 [[TMP18]]
+; CHECK-NEXT: br label %[[VECTOR_BODY:.*]]
+; CHECK: [[VECTOR_BODY]]:
+; CHECK-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-NEXT: [[TMP20:%.*]] = mul i64 [[INDEX]], 12
+; CHECK-NEXT: [[TMP21:%.*]] = add i64 [[TMP20]], 12
+; CHECK-NEXT: [[TMP22:%.*]] = add i64 [[TMP20]], 24
+; CHECK-NEXT: [[TMP23:%.*]] = add i64 [[TMP20]], 36
+; CHECK-NEXT: [[NEXT_GEP:%.*]] = getelementptr i8, ptr [[P]], i64 [[TMP20]]
+; CHECK-NEXT: [[NEXT_GEP7:%.*]] = getelementptr i8, ptr [[P]], i64 [[TMP21]]
+; CHECK-NEXT: [[NEXT_GEP8:%.*]] = getelementptr i8, ptr [[P]], i64 [[TMP22]]
+; CHECK-NEXT: [[NEXT_GEP9:%.*]] = getelementptr i8, ptr [[P]], i64 [[TMP23]]
+; CHECK-NEXT: store float 0.000000e+00, ptr [[NEXT_GEP]], align 4, !tbaa [[FLOAT_TBAA21:![0-9]+]]
+; CHECK-NEXT: store float 0.000000e+00, ptr [[NEXT_GEP7]], align 4, !tbaa [[FLOAT_TBAA21]]
+; CHECK-NEXT: store float 0.000000e+00, ptr [[NEXT_GEP8]], align 4, !tbaa [[FLOAT_TBAA21]]
+; CHECK-NEXT: store float 0.000000e+00, ptr [[NEXT_GEP9]], align 4, !tbaa [[FLOAT_TBAA21]]
+; CHECK-NEXT: [[TMP39:%.*]] = load float, ptr [[Q]], align 4, !tbaa [[FLOAT_TBAA27:![0-9]+]], !alias.scope [[META28:![0-9]+]]
+; CHECK-NEXT: [[BROADCAST_SPLATINSERT:%.*]] = insertelement <4 x float> poison, float [[TMP39]], i64 0
+; CHECK-NEXT: [[BROADCAST_SPLAT:%.*]] = shufflevector <4 x float> [[BROADCAST_SPLATINSERT]], <4 x float> poison, <4 x i32> zeroinitializer
+; CHECK-NEXT: [[TMP24:%.*]] = getelementptr i8, ptr [[NEXT_GEP]], i64 4
+; CHECK-NEXT: [[TMP25:%.*]] = getelementptr i8, ptr [[NEXT_GEP7]], i64 4
+; CHECK-NEXT: [[TMP26:%.*]] = getelementptr i8, ptr [[NEXT_GEP8]], i64 4
+; CHECK-NEXT: [[TMP27:%.*]] = getelementptr i8, ptr [[NEXT_GEP9]], i64 4
+; CHECK-NEXT: [[TMP28:%.*]] = load float, ptr [[TMP24]], align 4, !tbaa [[FLOAT_TBAA27]], !alias.scope [[META31:![0-9]+]]
+; CHECK-NEXT: [[TMP29:%.*]] = load float, ptr [[TMP25]], align 4, !tbaa [[FLOAT_TBAA27]], !alias.scope [[META31]]
+; CHECK-NEXT: [[TMP30:%.*]] = load float, ptr [[TMP26]], align 4, !tbaa [[FLOAT_TBAA27]], !alias.scope [[META31]]
+; CHECK-NEXT: [[TMP31:%.*]] = load float, ptr [[TMP27]], align 4, !tbaa [[FLOAT_TBAA27]], !alias.scope [[META31]]
+; CHECK-NEXT: [[TMP32:%.*]] = insertelement <4 x float> poison, float [[TMP28]], i32 0
+; CHECK-NEXT: [[TMP33:%.*]] = insertelement <4 x float> [[TMP32]], float [[TMP29]], i32 1
+; CHECK-NEXT: [[TMP34:%.*]] = insertelement <4 x float> [[TMP33]], float [[TMP30]], i32 2
+; CHECK-NEXT: [[TMP35:%.*]] = insertelement <4 x float> [[TMP34]], float [[TMP31]], i32 3
+; CHECK-NEXT: [[TMP36:%.*]] = fadd <4 x float> [[BROADCAST_SPLAT]], [[TMP35]]
+; CHECK-NEXT: [[TMP37:%.*]] = extractelement <4 x float> [[TMP36]], i64 3
+; CHECK-NEXT: store float [[TMP37]], ptr [[P]], align 4, !tbaa [[FLOAT_TBAA27]], !alias.scope [[META33:![0-9]+]], !noalias [[META28]]
+; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 4
+; CHECK-NEXT: [[TMP38:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
+; CHECK-NEXT: br i1 [[TMP38]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP35:![0-9]+]]
+; CHECK: [[MIDDLE_BLOCK]]:
+; CHECK-NEXT: [[CMP_N:%.*]] = icmp eq i64 [[TMP4]], [[N_VEC]]
+; CHECK-NEXT: br i1 [[CMP_N]], [[EXIT:label %.*]], label %[[SCALAR_PH]]
+; CHECK: [[SCALAR_PH]]:
+;
+entry:
+ br label %loop
+
+loop:
+ %ptr.iv = phi ptr [ %p, %entry ], [ %ptr.iv.next, %loop ]
+ store float 0.000000e+00, ptr %ptr.iv, align 4, !tbaa !0
+ %ld.q = load float, ptr %q, align 4, !tbaa !6
+ %gep.iv.4 = getelementptr i8, ptr %ptr.iv, i64 4
+ %ld.iv = load float, ptr %gep.iv.4, align 4, !tbaa !6
+ %res = fadd float %ld.q, %ld.iv
+ store float %res, ptr %p, align 4, !tbaa !6
+ %ptr.iv.next = getelementptr i8, ptr %ptr.iv, i64 12
+ %ec = icmp eq ptr %ptr.iv, %n
+ br i1 %ec, label %exit, label %loop
+
+exit:
+ ret void
+}
+
+!0 = !{!1, !2, i64 0}
+!1 = !{!"foo", !2, i64 0, !2, i64 4, !3, i64 8, !5, i64 9}
+!2 = !{!"float", !3, i64 0}
+!3 = !{!"omnipotent char", !4, i64 0}
+!4 = !{!"Simple C++ TBAA"}
+!5 = !{!"bool", !3, i64 0}
+!6 = !{!1, !2, i64 4}
diff --git a/llvm/test/Transforms/LoopVectorize/hoist-invariant-loads.ll b/llvm/test/Transforms/LoopVectorize/hoist-invariant-loads.ll
deleted file mode 100644
index c4dd71bc12077..0000000000000
--- a/llvm/test/Transforms/LoopVectorize/hoist-invariant-loads.ll
+++ /dev/null
@@ -1,116 +0,0 @@
-; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --check-globals none --filter-out-after "^scalar.ph" --version 6
-; RUN: opt -p loop-vectorize -force-vector-width=4 -S %s | FileCheck %s
-
-define void @load_not_hoisted(ptr %p, ptr %q, ptr %n) {
-; CHECK-LABEL: define void @load_not_hoisted(
-; CHECK-SAME: ptr [[P:%.*]], ptr [[Q:%.*]], ptr [[N:%.*]]) {
-; CHECK-NEXT: [[ENTRY:.*:]]
-; CHECK-NEXT: [[P6:%.*]] = ptrtoint ptr [[P]] to i64
-; CHECK-NEXT: [[N5:%.*]] = ptrtoint ptr [[N]] to i64
-; CHECK-NEXT: [[P2:%.*]] = ptrtoint ptr [[P]] to i64
-; CHECK-NEXT: [[N1:%.*]] = ptrtoint ptr [[N]] to i64
-; CHECK-NEXT: [[TMP0:%.*]] = mul i64 [[N5]], 3074457345618258603
-; CHECK-NEXT: [[TMP1:%.*]] = mul i64 [[P6]], 3074457345618258603
-; CHECK-NEXT: [[TMP2:%.*]] = sub i64 [[TMP0]], [[TMP1]]
-; CHECK-NEXT: [[TMP3:%.*]] = lshr i64 [[TMP2]], 2
-; CHECK-NEXT: [[TMP4:%.*]] = add nuw nsw i64 [[TMP3]], 1
-; CHECK-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[TMP4]], 4
-; CHECK-NEXT: br i1 [[MIN_ITERS_CHECK]], label %[[SCALAR_PH:.*]], label %[[VECTOR_SCEVCHECK:.*]]
-; CHECK: [[VECTOR_SCEVCHECK]]:
-; CHECK-NEXT: [[TMP5:%.*]] = trunc i64 [[N1]] to i2
-; CHECK-NEXT: [[TMP6:%.*]] = trunc i64 [[P2]] to i2
-; CHECK-NEXT: [[TMP7:%.*]] = sub i2 [[TMP5]], [[TMP6]]
-; CHECK-NEXT: [[TMP8:%.*]] = zext i2 [[TMP7]] to i64
-; CHECK-NEXT: [[IDENT_CHECK:%.*]] = icmp ne i64 [[TMP8]], 0
-; CHECK-NEXT: [[TMP9:%.*]] = mul i64 [[N1]], 3074457345618258603
-; CHECK-NEXT: [[TMP10:%.*]] = mul i64 [[P2]], 3074457345618258603
-; CHECK-NEXT: [[TMP11:%.*]] = sub i64 [[TMP9]], [[TMP10]]
-; CHECK-NEXT: [[TMP12:%.*]] = lshr i64 [[TMP11]], 2
-; CHECK-NEXT: [[SCEVGEP:%.*]] = getelementptr i8, ptr [[P]], i64 4
-; CHECK-NEXT: [[MUL:%.*]] = call { i64, i1 } @llvm.umul.with.overflow.i64(i64 12, i64 [[TMP12]])
-; CHECK-NEXT: [[MUL_RESULT:%.*]] = extractvalue { i64, i1 } [[MUL]], 0
-; CHECK-NEXT: [[MUL_OVERFLOW:%.*]] = extractvalue { i64, i1 } [[MUL]], 1
-; CHECK-NEXT: [[TMP13:%.*]] = getelementptr i8, ptr [[SCEVGEP]], i64 [[MUL_RESULT]]
-; CHECK-NEXT: [[TMP14:%.*]] = icmp ult ptr [[TMP13]], [[SCEVGEP]]
-; CHECK-NEXT: [[TMP15:%.*]] = or i1 [[TMP14]], [[MUL_OVERFLOW]]
-; CHECK-NEXT: [[TMP16:%.*]] = or i1 [[IDENT_CHECK]], [[TMP15]]
-; CHECK-NEXT: br i1 [[TMP16]], label %[[SCALAR_PH]], label %[[VECTOR_MEMCHECK:.*]]
-; CHECK: [[VECTOR_MEMCHECK]]:
-; CHECK-NEXT: [[SCEVGEP3:%.*]] = getelementptr i8, ptr [[P]], i64 4
-; CHECK-NEXT: [[SCEVGEP4:%.*]] = getelementptr i8, ptr [[Q]], i64 4
-; CHECK-NEXT: [[BOUND0:%.*]] = icmp ult ptr [[P]], [[SCEVGEP4]]
-; CHECK-NEXT: [[BOUND1:%.*]] = icmp ult ptr [[Q]], [[SCEVGEP3]]
-; CHECK-NEXT: [[FOUND_CONFLICT:%.*]] = and i1 [[BOUND0]], [[BOUND1]]
-; CHECK-NEXT: br i1 [[FOUND_CONFLICT]], label %[[SCALAR_PH]], label %[[VECTOR_PH:.*]]
-; CHECK: [[VECTOR_PH]]:
-; CHECK-NEXT: [[N_MOD_VF:%.*]] = urem i64 [[TMP4]], 4
-; CHECK-NEXT: [[N_VEC:%.*]] = sub i64 [[TMP4]], [[N_MOD_VF]]
-; CHECK-NEXT: [[TMP18:%.*]] = mul i64 [[N_VEC]], 12
-; CHECK-NEXT: [[TMP19:%.*]] = getelementptr i8, ptr [[P]], i64 [[TMP18]]
-; CHECK-NEXT: br label %[[VECTOR_BODY:.*]]
-; CHECK: [[VECTOR_BODY]]:
-; CHECK-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
-; CHECK-NEXT: [[TMP20:%.*]] = mul i64 [[INDEX]], 12
-; CHECK-NEXT: [[TMP21:%.*]] = add i64 [[TMP20]], 12
-; CHECK-NEXT: [[TMP22:%.*]] = add i64 [[TMP20]], 24
-; CHECK-NEXT: [[TMP23:%.*]] = add i64 [[TMP20]], 36
-; CHECK-NEXT: [[NEXT_GEP:%.*]] = getelementptr i8, ptr [[P]], i64 [[TMP20]]
-; CHECK-NEXT: [[NEXT_GEP7:%.*]] = getelementptr i8, ptr [[P]], i64 [[TMP21]]
-; CHECK-NEXT: [[NEXT_GEP8:%.*]] = getelementptr i8, ptr [[P]], i64 [[TMP22]]
-; CHECK-NEXT: [[NEXT_GEP9:%.*]] = getelementptr i8, ptr [[P]], i64 [[TMP23]]
-; CHECK-NEXT: store float 0.000000e+00, ptr [[NEXT_GEP]], align 4, !tbaa [[FLOAT_TBAA0:![0-9]+]]
-; CHECK-NEXT: store float 0.000000e+00, ptr [[NEXT_GEP7]], align 4, !tbaa [[FLOAT_TBAA0]]
-; CHECK-NEXT: store float 0.000000e+00, ptr [[NEXT_GEP8]], align 4, !tbaa [[FLOAT_TBAA0]]
-; CHECK-NEXT: store float 0.000000e+00, ptr [[NEXT_GEP9]], align 4, !tbaa [[FLOAT_TBAA0]]
-; CHECK-NEXT: [[TMP39:%.*]] = load float, ptr [[Q]], align 4, !tbaa [[FLOAT_TBAA6:![0-9]+]], !alias.scope [[META7:![0-9]+]]
-; CHECK-NEXT: [[BROADCAST_SPLATINSERT:%.*]] = insertelement <4 x float> poison, float [[TMP39]], i64 0
-; CHECK-NEXT: [[BROADCAST_SPLAT:%.*]] = shufflevector <4 x float> [[BROADCAST_SPLATINSERT]], <4 x float> poison, <4 x i32> zeroinitializer
-; CHECK-NEXT: [[TMP24:%.*]] = getelementptr i8, ptr [[NEXT_GEP]], i64 4
-; CHECK-NEXT: [[TMP25:%.*]] = getelementptr i8, ptr [[NEXT_GEP7]], i64 4
-; CHECK-NEXT: [[TMP26:%.*]] = getelementptr i8, ptr [[NEXT_GEP8]], i64 4
-; CHECK-NEXT: [[TMP27:%.*]] = getelementptr i8, ptr [[NEXT_GEP9]], i64 4
-; CHECK-NEXT: [[TMP28:%.*]] = load float, ptr [[TMP24]], align 4, !tbaa [[FLOAT_TBAA6]], !alias.scope [[META10:![0-9]+]]
-; CHECK-NEXT: [[TMP29:%.*]] = load float, ptr [[TMP25]], align 4, !tbaa [[FLOAT_TBAA6]], !alias.scope [[META10]]
-; CHECK-NEXT: [[TMP30:%.*]] = load float, ptr [[TMP26]], align 4, !tbaa [[FLOAT_TBAA6]], !alias.scope [[META10]]
-; CHECK-NEXT: [[TMP31:%.*]] = load float, ptr [[TMP27]], align 4, !tbaa [[FLOAT_TBAA6]], !alias.scope [[META10]]
-; CHECK-NEXT: [[TMP32:%.*]] = insertelement <4 x float> poison, float [[TMP28]], i32 0
-; CHECK-NEXT: [[TMP33:%.*]] = insertelement <4 x float> [[TMP32]], float [[TMP29]], i32 1
-; CHECK-NEXT: [[TMP34:%.*]] = insertelement <4 x float> [[TMP33]], float [[TMP30]], i32 2
-; CHECK-NEXT: [[TMP35:%.*]] = insertelement <4 x float> [[TMP34]], float [[TMP31]], i32 3
-; CHECK-NEXT: [[TMP36:%.*]] = fadd <4 x float> [[BROADCAST_SPLAT]], [[TMP35]]
-; CHECK-NEXT: [[TMP37:%.*]] = extractelement <4 x float> [[TMP36]], i64 3
-; CHECK-NEXT: store float [[TMP37]], ptr [[P]], align 4, !tbaa [[FLOAT_TBAA6]], !alias.scope [[META12:![0-9]+]], !noalias [[META7]]
-; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 4
-; CHECK-NEXT: [[TMP38:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
-; CHECK-NEXT: br i1 [[TMP38]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP14:![0-9]+]]
-; CHECK: [[MIDDLE_BLOCK]]:
-; CHECK-NEXT: [[CMP_N:%.*]] = icmp eq i64 [[TMP4]], [[N_VEC]]
-; CHECK-NEXT: br i1 [[CMP_N]], [[EXIT:label %.*]], label %[[SCALAR_PH]]
-; CHECK: [[SCALAR_PH]]:
-;
-entry:
- br label %loop
-
-loop:
- %ptr.iv = phi ptr [ %p, %entry ], [ %ptr.iv.next, %loop ]
- store float 0.000000e+00, ptr %ptr.iv, align 4, !tbaa !0
- %ld.q = load float, ptr %q, align 4, !tbaa !6
- %gep.iv.4 = getelementptr i8, ptr %ptr.iv, i64 4
- %ld.iv = load float, ptr %gep.iv.4, align 4, !tbaa !6
- %res = fadd float %ld.q, %ld.iv
- store float %res, ptr %p, align 4, !tbaa !6
- %ptr.iv.next = getelementptr i8, ptr %ptr.iv, i64 12
- %ec = icmp eq ptr %ptr.iv, %n
- br i1 %ec, label %exit, label %loop
-
-exit:
- ret void
-}
-
-!0 = !{!1, !2, i64 0}
-!1 = !{!"foo", !2, i64 0, !2, i64 4, !3, i64 8, !5, i64 9}
-!2 = !{!"float", !3, i64 0}
-!3 = !{!"omnipotent char", !4, i64 0}
-!4 = !{!"Simple C++ TBAA"}
-!5 = !{!"bool", !3, i64 0}
-!6 = !{!1, !2, i64 4}
>From 546647d2e024fed910edcfbfb228396f59573ce9 Mon Sep 17 00:00:00 2001
From: Ramkumar Ramachandra <artagnon at tenstorrent.com>
Date: Wed, 6 May 2026 10:08:12 +0100
Subject: [PATCH 4/4] [LV] Test update after rebase
---
.../LoopVectorize/X86/induction-step.ll | 25 ++++++++++++-------
1 file changed, 16 insertions(+), 9 deletions(-)
diff --git a/llvm/test/Transforms/LoopVectorize/X86/induction-step.ll b/llvm/test/Transforms/LoopVectorize/X86/induction-step.ll
index 124431fa5d311..7f3b22ed5df09 100644
--- a/llvm/test/Transforms/LoopVectorize/X86/induction-step.ll
+++ b/llvm/test/Transforms/LoopVectorize/X86/induction-step.ll
@@ -179,23 +179,30 @@ define void @canonical_iv_matches_user_iv(i64 %n, ptr %A, ptr %B) {
; CHECK-NEXT: [[FOUND_CONFLICT:%.*]] = and i1 [[BOUND0]], [[BOUND1]]
; CHECK-NEXT: br i1 [[FOUND_CONFLICT]], label %[[SCALAR_PH]], label %[[VECTOR_PH:.*]]
; CHECK: [[VECTOR_PH]]:
-; CHECK-NEXT: br label %[[VECTOR_BODY:.*]]
-; CHECK: [[VECTOR_BODY]]:
-; CHECK-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
-; CHECK-NEXT: [[TMP10:%.*]] = add i64 [[INDEX]], 1
+; CHECK-NEXT: [[N_MOD_VF:%.*]] = urem i64 [[N]], 2
+; CHECK-NEXT: [[N_VEC:%.*]] = sub i64 [[N]], [[N_MOD_VF]]
; CHECK-NEXT: [[TMP11:%.*]] = load double, ptr [[A]], align 8, !alias.scope [[META6:![0-9]+]], !noalias [[META9:![0-9]+]]
; CHECK-NEXT: [[TMP12:%.*]] = fdiv double 1.000000e+00, [[TMP11]]
; CHECK-NEXT: [[BROADCAST_SPLATINSERT:%.*]] = insertelement <2 x double> poison, double [[TMP12]], i64 0
; CHECK-NEXT: [[BROADCAST_SPLAT:%.*]] = shufflevector <2 x double> [[BROADCAST_SPLATINSERT]], <2 x double> poison, <2 x i32> zeroinitializer
+; CHECK-NEXT: br label %[[VECTOR_BODY:.*]]
+; CHECK: [[VECTOR_BODY]]:
+; CHECK-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-NEXT: [[TMP15:%.*]] = add i64 [[INDEX]], 1
+; CHECK-NEXT: [[TMP10:%.*]] = add i64 [[INDEX]], 1
+; CHECK-NEXT: [[TMP14:%.*]] = add i64 [[TMP15]], 1
; CHECK-NEXT: [[TMP13:%.*]] = getelementptr [16 x i8], ptr [[B]], i64 [[TMP10]]
+; CHECK-NEXT: [[TMP16:%.*]] = getelementptr [16 x i8], ptr [[B]], i64 [[TMP14]]
; CHECK-NEXT: store <2 x double> [[BROADCAST_SPLAT]], ptr [[TMP13]], align 8
-; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 1
-; CHECK-NEXT: [[TMP14:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N]]
-; CHECK-NEXT: br i1 [[TMP14]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP11:![0-9]+]]
+; CHECK-NEXT: store <2 x double> [[BROADCAST_SPLAT]], ptr [[TMP16]], align 8
+; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 2
+; CHECK-NEXT: [[TMP17:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
+; CHECK-NEXT: br i1 [[TMP17]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP11:![0-9]+]]
; CHECK: [[MIDDLE_BLOCK]]:
-; CHECK-NEXT: br label %[[EXIT:.*]]
+; CHECK-NEXT: [[CMP_N:%.*]] = icmp eq i64 [[N]], [[N_VEC]]
+; CHECK-NEXT: br i1 [[CMP_N]], label %[[EXIT:.*]], label %[[SCALAR_PH]]
; CHECK: [[SCALAR_PH]]:
-; CHECK-NEXT: [[BC_RESUME_VAL:%.*]] = phi i64 [ 0, %[[ENTRY]] ], [ 0, %[[VECTOR_SCEVCHECK]] ], [ 0, %[[VECTOR_MEMCHECK]] ]
+; CHECK-NEXT: [[BC_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC]], %[[MIDDLE_BLOCK]] ], [ 0, %[[ENTRY]] ], [ 0, %[[VECTOR_SCEVCHECK]] ], [ 0, %[[VECTOR_MEMCHECK]] ]
; CHECK-NEXT: br label %[[LOOP:.*]]
; CHECK: [[LOOP]]:
; CHECK-NEXT: [[IV:%.*]] = phi i64 [ [[BC_RESUME_VAL]], %[[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], %[[LOOP]] ]
More information about the llvm-commits
mailing list