[llvm] [VPlan] Enhance VPlan-based LICM using argument noalias attributes (PR #217238)
Mel Chen via llvm-commits
llvm-commits at lists.llvm.org
Wed Aug 19 01:59:29 PDT 2026
https://github.com/Mel-Chen created https://github.com/llvm/llvm-project/pull/217238
Currently, VPlan-based LICM only uses Scoped AA. This patch extends it to retrieve underlying objects and check if they have the noalias attribute.
>From 0fd2acbcf6741cd1c924c9de3ce155e63f8a94aa Mon Sep 17 00:00:00 2001
From: Mel Chen <mel.chen at sifive.com>
Date: Mon, 17 Aug 2026 07:29:39 -0700
Subject: [PATCH] fix by getUnderlyObject
---
.../Transforms/Vectorize/VPlanTransforms.cpp | 41 +++-
llvm/lib/Transforms/Vectorize/VPlanUtils.cpp | 17 ++
llvm/lib/Transforms/Vectorize/VPlanUtils.h | 5 +
.../AArch64/masked-call-scalarize.ll | 38 ++--
.../LoopVectorize/AArch64/masked-call.ll | 187 ++++++++++++------
.../partial-reduce-incomplete-chains.ll | 14 +-
.../replicating-load-store-costs-apple.ll | 22 +--
.../AArch64/replicating-load-store-costs.ll | 22 +--
.../AArch64/type-shrinkage-insertelt.ll | 26 +--
.../LoopVectorize/RISCV/strided-accesses.ll | 6 +-
.../LoopVectorize/SystemZ/pr47665.ll | 2 +-
...-order-recurrence-sink-replicate-region.ll | 2 +-
.../VPlan/vplan-sink-scalars-and-merge.ll | 2 +-
.../constant-fold-commutative-and.ll | 12 +-
.../Transforms/LoopVectorize/cse-casts.ll | 14 +-
.../first-order-recurrence-chains.ll | 28 +--
.../Transforms/LoopVectorize/lcssa-crashes.ll | 6 +-
.../LoopVectorize/narrow-to-single-scalar.ll | 47 ++---
.../vector-loop-backedge-elimination.ll | 22 +--
.../widen-gep-all-indices-invariant.ll | 10 +-
20 files changed, 319 insertions(+), 204 deletions(-)
diff --git a/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp b/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp
index 3abfbf5cfb444..324d9e858fd82 100644
--- a/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp
+++ b/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp
@@ -246,16 +246,41 @@ class SinkStoreInfo {
}
};
+static VPValue *getAddressOperand(VPRecipeBase &R) {
+ auto *Rep = dyn_cast<VPReplicateRecipe>(&R);
+ // TODO: support more memory operation.
+ if (!Rep)
+ return nullptr;
+ if (Rep->getOpcode() == Instruction::Load)
+ return Rep->getOperand(0);
+ if (Rep->getOpcode() == Instruction::Store)
+ return Rep->getOperand(1);
+ return nullptr;
+}
+
+static bool isNoAliasViaUnderlyingObject(VPValue *PtrA, VPValue *PtrB,
+ VPlan *Plan) {
+ auto *ObjA = dyn_cast_if_present<VPIRValue>(
+ vputils::getUnderlyingObject(PtrA, Plan));
+ auto *ObjB = dyn_cast_if_present<VPIRValue>(
+ vputils::getUnderlyingObject(PtrB, Plan));
+ if (!ObjA || !ObjB || ObjA == ObjB)
+ return false;
+ Value *ValA = ObjA->getValue();
+ Value *ValB = ObjB->getValue();
+ return (isa<Argument>(ValA) && isIdentifiedFunctionLocal(ValB)) ||
+ (isa<Argument>(ValB) && isIdentifiedFunctionLocal(ValA));
+}
+
/// Check if a memory operation doesn't alias with memory operations using
/// scoped noalias metadata, in blocks in the single-successor chain between \p
/// FirstBB and \p LastBB. If \p SinkInfo is std::nullopt, only recipes that may
/// write to memory are checked (for load hoisting). Otherwise recipes that both
/// read and write memory are checked, and SCEV is used to prove no-alias
/// between the group leader and other replicate recipes (for store sinking).
-static bool
-canHoistOrSinkWithNoAliasCheck(const MemoryLocation &MemLoc,
- VPBasicBlock *FirstBB, VPBasicBlock *LastBB,
- std::optional<SinkStoreInfo> SinkInfo = {}) {
+static bool canHoistOrSinkWithNoAliasCheck(
+ const MemoryLocation &MemLoc, VPBasicBlock *FirstBB, VPBasicBlock *LastBB,
+ std::optional<SinkStoreInfo> SinkInfo = {}, VPValue *Ptr = nullptr) {
bool CheckReads = SinkInfo.has_value();
for (VPBasicBlock *VPBB :
VPBlockUtils::blocksInSingleSuccessorChainBetween(FirstBB, LastBB)) {
@@ -267,6 +292,11 @@ canHoistOrSinkWithNoAliasCheck(const MemoryLocation &MemLoc,
if (!R.mayWriteToMemory() && !(CheckReads && R.mayReadFromMemory()))
continue;
+ auto *Addr = getAddressOperand(R);
+ if (Ptr && Addr &&
+ isNoAliasViaUnderlyingObject(Ptr, Addr, FirstBB->getPlan()))
+ continue;
+
auto Loc = vputils::getMemoryLocation(R);
if (!Loc)
// Conservatively assume aliasing for memory operations without
@@ -2308,7 +2338,8 @@ static bool cannotHoistOrSinkRecipe(VPRecipeBase &R, VPBasicBlock *FirstBB,
: std::nullopt;
return !MemLoc ||
- !canHoistOrSinkWithNoAliasCheck(*MemLoc, FirstBB, LastBB, SinkInfo);
+ !canHoistOrSinkWithNoAliasCheck(*MemLoc, FirstBB, LastBB, SinkInfo,
+ getAddressOperand(R));
}
/// Move loop-invariant recipes out of the vector loop region in \p Plan.
diff --git a/llvm/lib/Transforms/Vectorize/VPlanUtils.cpp b/llvm/lib/Transforms/Vectorize/VPlanUtils.cpp
index daf8114d482d0..c253a4ffc63f5 100644
--- a/llvm/lib/Transforms/Vectorize/VPlanUtils.cpp
+++ b/llvm/lib/Transforms/Vectorize/VPlanUtils.cpp
@@ -18,6 +18,7 @@
#include "llvm/Analysis/MemoryLocation.h"
#include "llvm/Analysis/ScalarEvolutionExpressions.h"
#include "llvm/Analysis/ScalarEvolutionPatternMatch.h"
+#include "llvm/Analysis/ValueTracking.h"
#include "llvm/IR/Dominators.h"
#include "llvm/Transforms/Utils/ScalarEvolutionExpander.h"
@@ -125,6 +126,22 @@ GEPNoWrapFlags vputils::getGEPFlagsForPtr(VPValue *Ptr) {
return GEPNoWrapFlags::none();
}
+VPValue *vputils::getUnderlyingObject(VPValue *V, VPlan *Plan) {
+ if (auto *IRV = dyn_cast<VPIRValue>(V))
+ return Plan->getOrAddLiveIn(llvm::getUnderlyingObject(IRV->getValue()));
+
+ auto *R = V->getDefiningRecipe();
+ if (!R)
+ return V;
+
+ if (auto *Rep = dyn_cast<VPReplicateRecipe>(R)) {
+ if (Rep->getOpcode() != Instruction::GetElementPtr)
+ return V;
+ return getUnderlyingObject(Rep->getOperand(0), Plan);
+ }
+ return V;
+}
+
const SCEV *vputils::getSCEVExprForVPValue(const VPValue *V,
PredicatedScalarEvolution &PSE,
const Loop *L) {
diff --git a/llvm/lib/Transforms/Vectorize/VPlanUtils.h b/llvm/lib/Transforms/Vectorize/VPlanUtils.h
index 0c556dbab1eab..049d981347cb1 100644
--- a/llvm/lib/Transforms/Vectorize/VPlanUtils.h
+++ b/llvm/lib/Transforms/Vectorize/VPlanUtils.h
@@ -172,6 +172,11 @@ VPInstruction *findCanonicalIVIncrement(VPlan &Plan);
/// mirroring Value::stripPointerCasts.
GEPNoWrapFlags getGEPFlagsForPtr(VPValue *Ptr);
+/// Return the underlying object live-in for \p V, looking through
+/// pointer-preserving VPlan recipes. Returns \p V itself if no underlying
+/// object can be determined.
+VPValue *getUnderlyingObject(VPValue *V, VPlan *Plan);
+
/// Returns true if \p V is used as part of the address of another load or
/// store.
bool isUsedByLoadStoreAddress(const VPValue *V);
diff --git a/llvm/test/Transforms/LoopVectorize/AArch64/masked-call-scalarize.ll b/llvm/test/Transforms/LoopVectorize/AArch64/masked-call-scalarize.ll
index 8aa223c0f927c..376e6124d5648 100644
--- a/llvm/test/Transforms/LoopVectorize/AArch64/masked-call-scalarize.ll
+++ b/llvm/test/Transforms/LoopVectorize/AArch64/masked-call-scalarize.ll
@@ -16,9 +16,6 @@ define void @test_widen_exp_v2(ptr noalias %p2, ptr noalias %p, i64 %n) #5 {
; TFNONE: [[VECTOR_PH]]:
; TFNONE-NEXT: [[N_MOD_VF:%.*]] = and i64 [[TMP0]], 1
; TFNONE-NEXT: [[N_VEC:%.*]] = sub i64 [[TMP0]], [[N_MOD_VF]]
-; TFNONE-NEXT: br label %[[VECTOR_BODY:.*]]
-; TFNONE: [[VECTOR_BODY]]:
-; TFNONE-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
; TFNONE-NEXT: [[TMP1:%.*]] = load double, ptr [[P2]], align 8
; TFNONE-NEXT: [[BROADCAST_SPLATINSERT:%.*]] = insertelement <2 x double> poison, double [[TMP1]], i64 0
; TFNONE-NEXT: [[BROADCAST_SPLAT:%.*]] = shufflevector <2 x double> [[BROADCAST_SPLATINSERT]], <2 x double> poison, <2 x i32> zeroinitializer
@@ -26,11 +23,14 @@ define void @test_widen_exp_v2(ptr noalias %p2, ptr noalias %p, i64 %n) #5 {
; TFNONE-NEXT: [[TMP3:%.*]] = fcmp ogt <2 x double> [[TMP2]], zeroinitializer
; TFNONE-NEXT: [[PREDPHI:%.*]] = select <2 x i1> [[TMP3]], <2 x double> zeroinitializer, <2 x double> splat (double 1.000000e+00)
; TFNONE-NEXT: [[TMP4:%.*]] = extractelement <2 x double> [[PREDPHI]], i64 1
-; TFNONE-NEXT: store double [[TMP4]], ptr [[P]], align 8
+; TFNONE-NEXT: br label %[[VECTOR_BODY:.*]]
+; TFNONE: [[VECTOR_BODY]]:
+; TFNONE-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
; TFNONE-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 2
; TFNONE-NEXT: [[TMP5:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
; TFNONE-NEXT: br i1 [[TMP5]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP0:![0-9]+]]
; TFNONE: [[MIDDLE_BLOCK]]:
+; TFNONE-NEXT: store double [[TMP4]], ptr [[P]], align 8
; TFNONE-NEXT: [[CMP_N:%.*]] = icmp eq i64 [[TMP0]], [[N_VEC]]
; TFNONE-NEXT: br i1 [[CMP_N]], label %[[END:.*]], label %[[SCALAR_PH]]
; TFNONE: [[SCALAR_PH]]:
@@ -57,13 +57,8 @@ define void @test_widen_exp_v2(ptr noalias %p2, ptr noalias %p, i64 %n) #5 {
; TFCOMMON-SAME: ptr noalias [[P2:%.*]], ptr noalias [[P:%.*]], i64 [[N:%.*]]) #[[ATTR0:[0-9]+]] {
; TFCOMMON-NEXT: [[ENTRY:.*:]]
; TFCOMMON-NEXT: [[TMP0:%.*]] = add i64 [[N]], 1
-; TFCOMMON-NEXT: br label %[[VECTOR_PH:.*]]
-; TFCOMMON: [[VECTOR_PH]]:
-; TFCOMMON-NEXT: [[ACTIVE_LANE_MASK_ENTRY:%.*]] = call <2 x i1> @llvm.get.active.lane.mask.v2i1.i64(i64 0, i64 [[TMP0]])
; TFCOMMON-NEXT: br label %[[VECTOR_BODY:.*]]
; TFCOMMON: [[VECTOR_BODY]]:
-; TFCOMMON-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[PRED_STORE_CONTINUE2:.*]] ]
-; TFCOMMON-NEXT: [[ACTIVE_LANE_MASK:%.*]] = phi <2 x i1> [ [[ACTIVE_LANE_MASK_ENTRY]], %[[VECTOR_PH]] ], [ [[ACTIVE_LANE_MASK_NEXT:%.*]], %[[PRED_STORE_CONTINUE2]] ]
; TFCOMMON-NEXT: [[TMP1:%.*]] = load double, ptr [[P2]], align 8
; TFCOMMON-NEXT: [[TMP2:%.*]] = tail call double @llvm.exp.f64(double [[TMP1]]) #[[ATTR3:[0-9]+]]
; TFCOMMON-NEXT: [[BROADCAST_SPLATINSERT:%.*]] = insertelement <2 x double> poison, double [[TMP2]], i64 0
@@ -71,6 +66,11 @@ define void @test_widen_exp_v2(ptr noalias %p2, ptr noalias %p, i64 %n) #5 {
; TFCOMMON-NEXT: [[TMP3:%.*]] = fcmp ogt <2 x double> [[BROADCAST_SPLAT]], zeroinitializer
; TFCOMMON-NEXT: [[TMP4:%.*]] = extractelement <2 x i1> [[TMP3]], i64 0
; TFCOMMON-NEXT: [[PREDPHI:%.*]] = select i1 [[TMP4]], <2 x double> zeroinitializer, <2 x double> splat (double 1.000000e+00)
+; TFCOMMON-NEXT: [[ACTIVE_LANE_MASK_ENTRY:%.*]] = call <2 x i1> @llvm.get.active.lane.mask.v2i1.i64(i64 0, i64 [[TMP0]])
+; TFCOMMON-NEXT: br label %[[VECTOR_BODY1:.*]]
+; TFCOMMON: [[VECTOR_BODY1]]:
+; TFCOMMON-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_BODY]] ], [ [[INDEX_NEXT:%.*]], %[[PRED_STORE_CONTINUE2:.*]] ]
+; TFCOMMON-NEXT: [[ACTIVE_LANE_MASK:%.*]] = phi <2 x i1> [ [[ACTIVE_LANE_MASK_ENTRY]], %[[VECTOR_BODY]] ], [ [[ACTIVE_LANE_MASK_NEXT:%.*]], %[[PRED_STORE_CONTINUE2]] ]
; TFCOMMON-NEXT: [[TMP5:%.*]] = extractelement <2 x i1> [[ACTIVE_LANE_MASK]], i64 0
; TFCOMMON-NEXT: br i1 [[TMP5]], label %[[PRED_STORE_IF:.*]], label %[[PRED_STORE_CONTINUE:.*]]
; TFCOMMON: [[PRED_STORE_IF]]:
@@ -89,7 +89,7 @@ define void @test_widen_exp_v2(ptr noalias %p2, ptr noalias %p, i64 %n) #5 {
; TFCOMMON-NEXT: [[ACTIVE_LANE_MASK_NEXT]] = call <2 x i1> @llvm.get.active.lane.mask.v2i1.i64(i64 [[INDEX_NEXT]], i64 [[TMP0]])
; TFCOMMON-NEXT: [[TMP9:%.*]] = extractelement <2 x i1> [[ACTIVE_LANE_MASK_NEXT]], i64 0
; TFCOMMON-NEXT: [[TMP10:%.*]] = xor i1 [[TMP9]], true
-; TFCOMMON-NEXT: br i1 [[TMP10]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP0:![0-9]+]]
+; TFCOMMON-NEXT: br i1 [[TMP10]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY1]], !llvm.loop [[LOOP0:![0-9]+]]
; TFCOMMON: [[MIDDLE_BLOCK]]:
; TFCOMMON-NEXT: br label %[[END:.*]]
; TFCOMMON: [[END]]:
@@ -99,16 +99,8 @@ define void @test_widen_exp_v2(ptr noalias %p2, ptr noalias %p, i64 %n) #5 {
; TFA_INTERLEAVE-SAME: ptr noalias [[P2:%.*]], ptr noalias [[P:%.*]], i64 [[N:%.*]]) #[[ATTR0:[0-9]+]] {
; TFA_INTERLEAVE-NEXT: [[ENTRY:.*:]]
; TFA_INTERLEAVE-NEXT: [[TMP0:%.*]] = add i64 [[N]], 1
-; TFA_INTERLEAVE-NEXT: br label %[[VECTOR_PH:.*]]
-; TFA_INTERLEAVE: [[VECTOR_PH]]:
-; TFA_INTERLEAVE-NEXT: [[ACTIVE_LANE_MASK_ENTRY:%.*]] = call <4 x i1> @llvm.get.active.lane.mask.v4i1.i64(i64 0, i64 [[TMP0]])
-; TFA_INTERLEAVE-NEXT: [[TMP13:%.*]] = call <2 x i1> @llvm.vector.extract.v2i1.v4i1(<4 x i1> [[ACTIVE_LANE_MASK_ENTRY]], i64 0)
-; TFA_INTERLEAVE-NEXT: [[TMP17:%.*]] = call <2 x i1> @llvm.vector.extract.v2i1.v4i1(<4 x i1> [[ACTIVE_LANE_MASK_ENTRY]], i64 2)
; TFA_INTERLEAVE-NEXT: br label %[[VECTOR_BODY:.*]]
; TFA_INTERLEAVE: [[VECTOR_BODY]]:
-; TFA_INTERLEAVE-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[PRED_STORE_CONTINUE8:.*]] ]
-; TFA_INTERLEAVE-NEXT: [[ACTIVE_LANE_MASK:%.*]] = phi <2 x i1> [ [[TMP13]], %[[VECTOR_PH]] ], [ [[ACTIVE_LANE_MASK_NEXT:%.*]], %[[PRED_STORE_CONTINUE8]] ]
-; TFA_INTERLEAVE-NEXT: [[ACTIVE_LANE_MASK2:%.*]] = phi <2 x i1> [ [[TMP17]], %[[VECTOR_PH]] ], [ [[TMP16:%.*]], %[[PRED_STORE_CONTINUE8]] ]
; TFA_INTERLEAVE-NEXT: [[TMP1:%.*]] = load double, ptr [[P2]], align 8
; TFA_INTERLEAVE-NEXT: [[TMP2:%.*]] = tail call double @llvm.exp.f64(double [[TMP1]]) #[[ATTR3:[0-9]+]]
; TFA_INTERLEAVE-NEXT: [[BROADCAST_SPLATINSERT:%.*]] = insertelement <2 x double> poison, double [[TMP2]], i64 0
@@ -116,6 +108,14 @@ define void @test_widen_exp_v2(ptr noalias %p2, ptr noalias %p, i64 %n) #5 {
; TFA_INTERLEAVE-NEXT: [[TMP3:%.*]] = fcmp ogt <2 x double> [[BROADCAST_SPLAT]], zeroinitializer
; TFA_INTERLEAVE-NEXT: [[TMP4:%.*]] = extractelement <2 x i1> [[TMP3]], i64 0
; TFA_INTERLEAVE-NEXT: [[PREDPHI:%.*]] = select i1 [[TMP4]], <2 x double> zeroinitializer, <2 x double> splat (double 1.000000e+00)
+; TFA_INTERLEAVE-NEXT: [[ACTIVE_LANE_MASK_ENTRY:%.*]] = call <4 x i1> @llvm.get.active.lane.mask.v4i1.i64(i64 0, i64 [[TMP0]])
+; TFA_INTERLEAVE-NEXT: [[EXTRACT_ENTRY_ALM_PART:%.*]] = call <2 x i1> @llvm.vector.extract.v2i1.v4i1(<4 x i1> [[ACTIVE_LANE_MASK_ENTRY]], i64 0)
+; TFA_INTERLEAVE-NEXT: [[EXTRACT_ENTRY_ALM_PART1:%.*]] = call <2 x i1> @llvm.vector.extract.v2i1.v4i1(<4 x i1> [[ACTIVE_LANE_MASK_ENTRY]], i64 2)
+; TFA_INTERLEAVE-NEXT: br label %[[VECTOR_BODY1:.*]]
+; TFA_INTERLEAVE: [[VECTOR_BODY1]]:
+; TFA_INTERLEAVE-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_BODY]] ], [ [[INDEX_NEXT:%.*]], %[[PRED_STORE_CONTINUE8:.*]] ]
+; TFA_INTERLEAVE-NEXT: [[ACTIVE_LANE_MASK:%.*]] = phi <2 x i1> [ [[EXTRACT_ENTRY_ALM_PART]], %[[VECTOR_BODY]] ], [ [[ACTIVE_LANE_MASK_NEXT:%.*]], %[[PRED_STORE_CONTINUE8]] ]
+; TFA_INTERLEAVE-NEXT: [[ACTIVE_LANE_MASK2:%.*]] = phi <2 x i1> [ [[EXTRACT_ENTRY_ALM_PART1]], %[[VECTOR_BODY]] ], [ [[TMP16:%.*]], %[[PRED_STORE_CONTINUE8]] ]
; TFA_INTERLEAVE-NEXT: [[TMP5:%.*]] = extractelement <2 x i1> [[ACTIVE_LANE_MASK]], i64 0
; TFA_INTERLEAVE-NEXT: br i1 [[TMP5]], label %[[PRED_STORE_IF:.*]], label %[[PRED_STORE_CONTINUE:.*]]
; TFA_INTERLEAVE: [[PRED_STORE_IF]]:
@@ -150,7 +150,7 @@ define void @test_widen_exp_v2(ptr noalias %p2, ptr noalias %p, i64 %n) #5 {
; TFA_INTERLEAVE-NEXT: [[TMP16]] = call <2 x i1> @llvm.vector.extract.v2i1.v4i1(<4 x i1> [[ACTIVE_LANE_MASK_NEXT1]], i64 2)
; TFA_INTERLEAVE-NEXT: [[TMP14:%.*]] = extractelement <2 x i1> [[ACTIVE_LANE_MASK_NEXT]], i64 0
; TFA_INTERLEAVE-NEXT: [[TMP15:%.*]] = xor i1 [[TMP14]], true
-; TFA_INTERLEAVE-NEXT: br i1 [[TMP15]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP0:![0-9]+]]
+; TFA_INTERLEAVE-NEXT: br i1 [[TMP15]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY1]], !llvm.loop [[LOOP0:![0-9]+]]
; TFA_INTERLEAVE: [[MIDDLE_BLOCK]]:
; TFA_INTERLEAVE-NEXT: br label %[[END:.*]]
; TFA_INTERLEAVE: [[END]]:
diff --git a/llvm/test/Transforms/LoopVectorize/AArch64/masked-call.ll b/llvm/test/Transforms/LoopVectorize/AArch64/masked-call.ll
index 2bfbe87b307d2..2df6eeab489bb 100644
--- a/llvm/test/Transforms/LoopVectorize/AArch64/masked-call.ll
+++ b/llvm/test/Transforms/LoopVectorize/AArch64/masked-call.ll
@@ -925,35 +925,10 @@ for.cond.cleanup:
define void @test_widen_exp_v2(ptr noalias %p2, ptr noalias %p, i64 %n) #5 {
; TFNONE-LABEL: define void @test_widen_exp_v2(
; TFNONE-SAME: ptr noalias [[P2:%.*]], ptr noalias [[P:%.*]], i64 [[N:%.*]]) #[[ATTR1:[0-9]+]] {
-; TFNONE-NEXT: [[ENTRY:.*]]:
-; TFNONE-NEXT: [[TMP0:%.*]] = add i64 [[N]], 1
-; TFNONE-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[TMP0]], 2
-; TFNONE-NEXT: br i1 [[MIN_ITERS_CHECK]], label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]]
-; TFNONE: [[VECTOR_PH]]:
-; TFNONE-NEXT: [[N_MOD_VF:%.*]] = and i64 [[TMP0]], 1
-; TFNONE-NEXT: [[N_VEC:%.*]] = sub i64 [[TMP0]], [[N_MOD_VF]]
-; TFNONE-NEXT: br label %[[VECTOR_BODY:.*]]
-; TFNONE: [[VECTOR_BODY]]:
-; TFNONE-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
-; TFNONE-NEXT: [[TMP1:%.*]] = load double, ptr [[P2]], align 8
-; TFNONE-NEXT: [[BROADCAST_SPLATINSERT:%.*]] = insertelement <2 x double> poison, double [[TMP1]], i64 0
-; TFNONE-NEXT: [[BROADCAST_SPLAT:%.*]] = shufflevector <2 x double> [[BROADCAST_SPLATINSERT]], <2 x double> poison, <2 x i32> zeroinitializer
-; TFNONE-NEXT: [[TMP2:%.*]] = call <2 x double> @exp_fixed(<2 x double> [[BROADCAST_SPLAT]])
-; TFNONE-NEXT: [[TMP3:%.*]] = fcmp ogt <2 x double> [[TMP2]], zeroinitializer
-; TFNONE-NEXT: [[PREDPHI:%.*]] = select <2 x i1> [[TMP3]], <2 x double> zeroinitializer, <2 x double> splat (double 1.000000e+00)
-; TFNONE-NEXT: [[TMP4:%.*]] = extractelement <2 x double> [[PREDPHI]], i64 1
-; TFNONE-NEXT: store double [[TMP4]], ptr [[P]], align 8
-; TFNONE-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 2
-; TFNONE-NEXT: [[TMP5:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
-; TFNONE-NEXT: br i1 [[TMP5]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP14:![0-9]+]]
-; TFNONE: [[MIDDLE_BLOCK]]:
-; TFNONE-NEXT: [[CMP_N:%.*]] = icmp eq i64 [[TMP0]], [[N_VEC]]
-; TFNONE-NEXT: br i1 [[CMP_N]], label %[[END:.*]], label %[[SCALAR_PH]]
-; TFNONE: [[SCALAR_PH]]:
-; TFNONE-NEXT: [[BC_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC]], %[[MIDDLE_BLOCK]] ], [ 0, %[[ENTRY]] ]
+; TFNONE-NEXT: [[SCALAR_PH:.*]]:
; TFNONE-NEXT: br label %[[LOOP:.*]]
; TFNONE: [[LOOP]]:
-; TFNONE-NEXT: [[IV:%.*]] = phi i64 [ [[BC_RESUME_VAL]], %[[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], %[[LOOP_END:.*]] ]
+; TFNONE-NEXT: [[IV:%.*]] = phi i64 [ 0, %[[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], %[[LOOP_END:.*]] ]
; TFNONE-NEXT: [[LD:%.*]] = load double, ptr [[P2]], align 8
; TFNONE-NEXT: [[EXP:%.*]] = tail call double @llvm.exp.f64(double [[LD]]) #[[ATTR8:[0-9]+]]
; TFNONE-NEXT: [[COND1:%.*]] = fcmp ogt double [[EXP]], 0.000000e+00
@@ -965,30 +940,93 @@ define void @test_widen_exp_v2(ptr noalias %p2, ptr noalias %p, i64 %n) #5 {
; TFNONE-NEXT: store double [[SINK]], ptr [[P]], align 8
; TFNONE-NEXT: [[IV_NEXT]] = add i64 [[IV]], 1
; TFNONE-NEXT: [[COND2:%.*]] = icmp eq i64 [[IV]], [[N]]
-; TFNONE-NEXT: br i1 [[COND2]], label %[[END]], label %[[LOOP]], !llvm.loop [[LOOP15:![0-9]+]]
+; TFNONE-NEXT: br i1 [[COND2]], label %[[END:.*]], label %[[LOOP]]
; TFNONE: [[END]]:
; TFNONE-NEXT: ret void
;
-; TFCOMMON-LABEL: define void @test_widen_exp_v2(
-; TFCOMMON-SAME: ptr noalias [[P2:%.*]], ptr noalias [[P:%.*]], i64 [[N:%.*]]) #[[ATTR1:[0-9]+]] {
-; TFCOMMON-NEXT: [[ENTRY:.*]]:
-; TFCOMMON-NEXT: br label %[[LOOP:.*]]
-; TFCOMMON: [[LOOP]]:
-; TFCOMMON-NEXT: [[IV:%.*]] = phi i64 [ 0, %[[ENTRY]] ], [ [[IV_NEXT:%.*]], %[[LOOP_END:.*]] ]
-; TFCOMMON-NEXT: [[LD:%.*]] = load double, ptr [[P2]], align 8
-; TFCOMMON-NEXT: [[EXP:%.*]] = tail call double @llvm.exp.f64(double [[LD]]) #[[ATTR7:[0-9]+]]
-; TFCOMMON-NEXT: [[COND1:%.*]] = fcmp ogt double [[EXP]], 0.000000e+00
-; TFCOMMON-NEXT: br i1 [[COND1]], label %[[LOOP_MIDDLE:.*]], label %[[LOOP_END]]
-; TFCOMMON: [[LOOP_MIDDLE]]:
-; TFCOMMON-NEXT: br label %[[LOOP_END]]
-; TFCOMMON: [[LOOP_END]]:
-; TFCOMMON-NEXT: [[SINK:%.*]] = phi double [ 0.000000e+00, %[[LOOP_MIDDLE]] ], [ 1.000000e+00, %[[LOOP]] ]
-; TFCOMMON-NEXT: store double [[SINK]], ptr [[P]], align 8
-; TFCOMMON-NEXT: [[IV_NEXT]] = add i64 [[IV]], 1
-; TFCOMMON-NEXT: [[COND2:%.*]] = icmp eq i64 [[IV]], [[N]]
-; TFCOMMON-NEXT: br i1 [[COND2]], label %[[END:.*]], label %[[LOOP]]
-; TFCOMMON: [[END]]:
-; TFCOMMON-NEXT: ret void
+; TFALWAYS-LABEL: define void @test_widen_exp_v2(
+; TFALWAYS-SAME: ptr noalias [[P2:%.*]], ptr noalias [[P:%.*]], i64 [[N:%.*]]) #[[ATTR1:[0-9]+]] {
+; TFALWAYS-NEXT: [[ENTRY:.*:]]
+; TFALWAYS-NEXT: [[TMP0:%.*]] = add i64 [[N]], 1
+; TFALWAYS-NEXT: br label %[[VECTOR_PH:.*]]
+; TFALWAYS: [[VECTOR_PH]]:
+; TFALWAYS-NEXT: [[TMP1:%.*]] = load double, ptr [[P2]], align 8
+; TFALWAYS-NEXT: [[TMP2:%.*]] = tail call double @llvm.exp.f64(double [[TMP1]]) #[[ATTR7:[0-9]+]]
+; TFALWAYS-NEXT: [[BROADCAST_SPLATINSERT:%.*]] = insertelement <2 x double> poison, double [[TMP2]], i64 0
+; TFALWAYS-NEXT: [[BROADCAST_SPLAT:%.*]] = shufflevector <2 x double> [[BROADCAST_SPLATINSERT]], <2 x double> poison, <2 x i32> zeroinitializer
+; TFALWAYS-NEXT: [[TMP3:%.*]] = fcmp ogt <2 x double> [[BROADCAST_SPLAT]], zeroinitializer
+; TFALWAYS-NEXT: [[TMP4:%.*]] = extractelement <2 x i1> [[TMP3]], i64 0
+; TFALWAYS-NEXT: [[PREDPHI:%.*]] = select i1 [[TMP4]], <2 x double> zeroinitializer, <2 x double> splat (double 1.000000e+00)
+; TFALWAYS-NEXT: [[ACTIVE_LANE_MASK_ENTRY:%.*]] = call <2 x i1> @llvm.get.active.lane.mask.v2i1.i64(i64 0, i64 [[TMP0]])
+; TFALWAYS-NEXT: br label %[[VECTOR_BODY:.*]]
+; TFALWAYS: [[VECTOR_BODY]]:
+; TFALWAYS-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[PRED_STORE_CONTINUE2:.*]] ]
+; TFALWAYS-NEXT: [[ACTIVE_LANE_MASK:%.*]] = phi <2 x i1> [ [[ACTIVE_LANE_MASK_ENTRY]], %[[VECTOR_PH]] ], [ [[ACTIVE_LANE_MASK_NEXT:%.*]], %[[PRED_STORE_CONTINUE2]] ]
+; TFALWAYS-NEXT: [[TMP5:%.*]] = extractelement <2 x i1> [[ACTIVE_LANE_MASK]], i64 0
+; TFALWAYS-NEXT: br i1 [[TMP5]], label %[[PRED_STORE_IF:.*]], label %[[PRED_STORE_CONTINUE:.*]]
+; TFALWAYS: [[PRED_STORE_IF]]:
+; TFALWAYS-NEXT: [[TMP6:%.*]] = extractelement <2 x double> [[PREDPHI]], i64 0
+; TFALWAYS-NEXT: store double [[TMP6]], ptr [[P]], align 8
+; TFALWAYS-NEXT: br label %[[PRED_STORE_CONTINUE]]
+; TFALWAYS: [[PRED_STORE_CONTINUE]]:
+; TFALWAYS-NEXT: [[TMP7:%.*]] = extractelement <2 x i1> [[ACTIVE_LANE_MASK]], i64 1
+; TFALWAYS-NEXT: br i1 [[TMP7]], label %[[PRED_STORE_IF1:.*]], label %[[PRED_STORE_CONTINUE2]]
+; TFALWAYS: [[PRED_STORE_IF1]]:
+; TFALWAYS-NEXT: [[TMP8:%.*]] = extractelement <2 x double> [[PREDPHI]], i64 1
+; TFALWAYS-NEXT: store double [[TMP8]], ptr [[P]], align 8
+; TFALWAYS-NEXT: br label %[[PRED_STORE_CONTINUE2]]
+; TFALWAYS: [[PRED_STORE_CONTINUE2]]:
+; TFALWAYS-NEXT: [[INDEX_NEXT]] = add i64 [[INDEX]], 2
+; TFALWAYS-NEXT: [[ACTIVE_LANE_MASK_NEXT]] = call <2 x i1> @llvm.get.active.lane.mask.v2i1.i64(i64 [[INDEX_NEXT]], i64 [[TMP0]])
+; TFALWAYS-NEXT: [[TMP9:%.*]] = extractelement <2 x i1> [[ACTIVE_LANE_MASK_NEXT]], i64 0
+; TFALWAYS-NEXT: [[TMP10:%.*]] = xor i1 [[TMP9]], true
+; TFALWAYS-NEXT: br i1 [[TMP10]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP7:![0-9]+]]
+; TFALWAYS: [[MIDDLE_BLOCK]]:
+; TFALWAYS-NEXT: br label %[[END:.*]]
+; TFALWAYS: [[END]]:
+; TFALWAYS-NEXT: ret void
+;
+; TFFALLBACK-LABEL: define void @test_widen_exp_v2(
+; TFFALLBACK-SAME: ptr noalias [[P2:%.*]], ptr noalias [[P:%.*]], i64 [[N:%.*]]) #[[ATTR1:[0-9]+]] {
+; TFFALLBACK-NEXT: [[ENTRY:.*:]]
+; TFFALLBACK-NEXT: [[TMP0:%.*]] = add i64 [[N]], 1
+; TFFALLBACK-NEXT: br label %[[VECTOR_PH:.*]]
+; TFFALLBACK: [[VECTOR_PH]]:
+; TFFALLBACK-NEXT: [[TMP1:%.*]] = load double, ptr [[P2]], align 8
+; TFFALLBACK-NEXT: [[TMP2:%.*]] = tail call double @llvm.exp.f64(double [[TMP1]]) #[[ATTR7:[0-9]+]]
+; TFFALLBACK-NEXT: [[BROADCAST_SPLATINSERT:%.*]] = insertelement <2 x double> poison, double [[TMP2]], i64 0
+; TFFALLBACK-NEXT: [[BROADCAST_SPLAT:%.*]] = shufflevector <2 x double> [[BROADCAST_SPLATINSERT]], <2 x double> poison, <2 x i32> zeroinitializer
+; TFFALLBACK-NEXT: [[TMP3:%.*]] = fcmp ogt <2 x double> [[BROADCAST_SPLAT]], zeroinitializer
+; TFFALLBACK-NEXT: [[TMP4:%.*]] = extractelement <2 x i1> [[TMP3]], i64 0
+; TFFALLBACK-NEXT: [[PREDPHI:%.*]] = select i1 [[TMP4]], <2 x double> zeroinitializer, <2 x double> splat (double 1.000000e+00)
+; TFFALLBACK-NEXT: [[ACTIVE_LANE_MASK_ENTRY:%.*]] = call <2 x i1> @llvm.get.active.lane.mask.v2i1.i64(i64 0, i64 [[TMP0]])
+; TFFALLBACK-NEXT: br label %[[VECTOR_BODY:.*]]
+; TFFALLBACK: [[VECTOR_BODY]]:
+; TFFALLBACK-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[PRED_STORE_CONTINUE2:.*]] ]
+; TFFALLBACK-NEXT: [[ACTIVE_LANE_MASK:%.*]] = phi <2 x i1> [ [[ACTIVE_LANE_MASK_ENTRY]], %[[VECTOR_PH]] ], [ [[ACTIVE_LANE_MASK_NEXT:%.*]], %[[PRED_STORE_CONTINUE2]] ]
+; TFFALLBACK-NEXT: [[TMP5:%.*]] = extractelement <2 x i1> [[ACTIVE_LANE_MASK]], i64 0
+; TFFALLBACK-NEXT: br i1 [[TMP5]], label %[[PRED_STORE_IF:.*]], label %[[PRED_STORE_CONTINUE:.*]]
+; TFFALLBACK: [[PRED_STORE_IF]]:
+; TFFALLBACK-NEXT: [[TMP6:%.*]] = extractelement <2 x double> [[PREDPHI]], i64 0
+; TFFALLBACK-NEXT: store double [[TMP6]], ptr [[P]], align 8
+; TFFALLBACK-NEXT: br label %[[PRED_STORE_CONTINUE]]
+; TFFALLBACK: [[PRED_STORE_CONTINUE]]:
+; TFFALLBACK-NEXT: [[TMP7:%.*]] = extractelement <2 x i1> [[ACTIVE_LANE_MASK]], i64 1
+; TFFALLBACK-NEXT: br i1 [[TMP7]], label %[[PRED_STORE_IF1:.*]], label %[[PRED_STORE_CONTINUE2]]
+; TFFALLBACK: [[PRED_STORE_IF1]]:
+; TFFALLBACK-NEXT: [[TMP8:%.*]] = extractelement <2 x double> [[PREDPHI]], i64 1
+; TFFALLBACK-NEXT: store double [[TMP8]], ptr [[P]], align 8
+; TFFALLBACK-NEXT: br label %[[PRED_STORE_CONTINUE2]]
+; TFFALLBACK: [[PRED_STORE_CONTINUE2]]:
+; TFFALLBACK-NEXT: [[INDEX_NEXT]] = add i64 [[INDEX]], 2
+; TFFALLBACK-NEXT: [[ACTIVE_LANE_MASK_NEXT]] = call <2 x i1> @llvm.get.active.lane.mask.v2i1.i64(i64 [[INDEX_NEXT]], i64 [[TMP0]])
+; TFFALLBACK-NEXT: [[TMP9:%.*]] = extractelement <2 x i1> [[ACTIVE_LANE_MASK_NEXT]], i64 0
+; TFFALLBACK-NEXT: [[TMP10:%.*]] = xor i1 [[TMP9]], true
+; TFFALLBACK-NEXT: br i1 [[TMP10]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP9:![0-9]+]]
+; TFFALLBACK: [[MIDDLE_BLOCK]]:
+; TFFALLBACK-NEXT: br label %[[END:.*]]
+; TFFALLBACK: [[END]]:
+; TFFALLBACK-NEXT: ret void
;
; TFA_INTERLEAVE-LABEL: define void @test_widen_exp_v2(
; TFA_INTERLEAVE-SAME: ptr noalias [[P2:%.*]], ptr noalias [[P:%.*]], i64 [[N:%.*]]) #[[ATTR1:[0-9]+]] {
@@ -996,31 +1034,54 @@ define void @test_widen_exp_v2(ptr noalias %p2, ptr noalias %p, i64 %n) #5 {
; TFA_INTERLEAVE-NEXT: [[TMP0:%.*]] = add i64 [[N]], 1
; TFA_INTERLEAVE-NEXT: br label %[[VECTOR_PH:.*]]
; TFA_INTERLEAVE: [[VECTOR_PH]]:
-; TFA_INTERLEAVE-NEXT: [[ACTIVE_LANE_MASK_ENTRY:%.*]] = icmp ult i64 0, [[TMP0]]
-; TFA_INTERLEAVE-NEXT: [[ACTIVE_LANE_MASK_ENTRY1:%.*]] = icmp ult i64 1, [[TMP0]]
-; TFA_INTERLEAVE-NEXT: br label %[[VECTOR_BODY:.*]]
-; TFA_INTERLEAVE: [[VECTOR_BODY]]:
-; TFA_INTERLEAVE-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[PRED_STORE_CONTINUE4:.*]] ]
-; TFA_INTERLEAVE-NEXT: [[ACTIVE_LANE_MASK:%.*]] = phi i1 [ [[ACTIVE_LANE_MASK_ENTRY]], %[[VECTOR_PH]] ], [ [[ACTIVE_LANE_MASK_NEXT:%.*]], %[[PRED_STORE_CONTINUE4]] ]
-; TFA_INTERLEAVE-NEXT: [[ACTIVE_LANE_MASK2:%.*]] = phi i1 [ [[ACTIVE_LANE_MASK_ENTRY1]], %[[VECTOR_PH]] ], [ [[ACTIVE_LANE_MASK_NEXT5:%.*]], %[[PRED_STORE_CONTINUE4]] ]
; TFA_INTERLEAVE-NEXT: [[TMP1:%.*]] = load double, ptr [[P2]], align 8
; TFA_INTERLEAVE-NEXT: [[TMP2:%.*]] = tail call double @llvm.exp.f64(double [[TMP1]]) #[[ATTR7:[0-9]+]]
-; TFA_INTERLEAVE-NEXT: [[TMP3:%.*]] = fcmp ogt double [[TMP2]], 0.000000e+00
-; TFA_INTERLEAVE-NEXT: [[PREDPHI:%.*]] = select i1 [[TMP3]], double 0.000000e+00, double 1.000000e+00
+; TFA_INTERLEAVE-NEXT: [[BROADCAST_SPLATINSERT:%.*]] = insertelement <2 x double> poison, double [[TMP2]], i64 0
+; TFA_INTERLEAVE-NEXT: [[BROADCAST_SPLAT:%.*]] = shufflevector <2 x double> [[BROADCAST_SPLATINSERT]], <2 x double> poison, <2 x i32> zeroinitializer
+; TFA_INTERLEAVE-NEXT: [[TMP3:%.*]] = fcmp ogt <2 x double> [[BROADCAST_SPLAT]], zeroinitializer
+; TFA_INTERLEAVE-NEXT: [[TMP4:%.*]] = extractelement <2 x i1> [[TMP3]], i64 0
+; TFA_INTERLEAVE-NEXT: [[PREDPHI1:%.*]] = select i1 [[TMP4]], <2 x double> zeroinitializer, <2 x double> splat (double 1.000000e+00)
+; TFA_INTERLEAVE-NEXT: [[ACTIVE_LANE_MASK_ENTRY:%.*]] = call <4 x i1> @llvm.get.active.lane.mask.v4i1.i64(i64 0, i64 [[TMP0]])
+; TFA_INTERLEAVE-NEXT: [[EXTRACT_ENTRY_ALM_PART:%.*]] = call <2 x i1> @llvm.vector.extract.v2i1.v4i1(<4 x i1> [[ACTIVE_LANE_MASK_ENTRY]], i64 0)
+; TFA_INTERLEAVE-NEXT: [[EXTRACT_ENTRY_ALM_PART1:%.*]] = call <2 x i1> @llvm.vector.extract.v2i1.v4i1(<4 x i1> [[ACTIVE_LANE_MASK_ENTRY]], i64 2)
+; TFA_INTERLEAVE-NEXT: br label %[[VECTOR_BODY:.*]]
+; TFA_INTERLEAVE: [[VECTOR_BODY]]:
+; TFA_INTERLEAVE-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[PRED_STORE_CONTINUE8:.*]] ]
+; TFA_INTERLEAVE-NEXT: [[ACTIVE_LANE_MASK1:%.*]] = phi <2 x i1> [ [[EXTRACT_ENTRY_ALM_PART]], %[[VECTOR_PH]] ], [ [[EXTRACT_NEXT_ALM_PART:%.*]], %[[PRED_STORE_CONTINUE8]] ]
+; TFA_INTERLEAVE-NEXT: [[ACTIVE_LANE_MASK3:%.*]] = phi <2 x i1> [ [[EXTRACT_ENTRY_ALM_PART1]], %[[VECTOR_PH]] ], [ [[EXTRACT_NEXT_ALM_PART9:%.*]], %[[PRED_STORE_CONTINUE8]] ]
+; TFA_INTERLEAVE-NEXT: [[ACTIVE_LANE_MASK:%.*]] = extractelement <2 x i1> [[ACTIVE_LANE_MASK1]], i64 0
; TFA_INTERLEAVE-NEXT: br i1 [[ACTIVE_LANE_MASK]], label %[[PRED_STORE_IF:.*]], label %[[PRED_STORE_CONTINUE:.*]]
; TFA_INTERLEAVE: [[PRED_STORE_IF]]:
+; TFA_INTERLEAVE-NEXT: [[PREDPHI:%.*]] = extractelement <2 x double> [[PREDPHI1]], i64 0
; TFA_INTERLEAVE-NEXT: store double [[PREDPHI]], ptr [[P]], align 8
; TFA_INTERLEAVE-NEXT: br label %[[PRED_STORE_CONTINUE]]
; TFA_INTERLEAVE: [[PRED_STORE_CONTINUE]]:
-; TFA_INTERLEAVE-NEXT: br i1 [[ACTIVE_LANE_MASK2]], label %[[PRED_STORE_IF3:.*]], label %[[PRED_STORE_CONTINUE4]]
+; TFA_INTERLEAVE-NEXT: [[ACTIVE_LANE_MASK2:%.*]] = extractelement <2 x i1> [[ACTIVE_LANE_MASK1]], i64 1
+; TFA_INTERLEAVE-NEXT: br i1 [[ACTIVE_LANE_MASK2]], label %[[PRED_STORE_IF3:.*]], label %[[PRED_STORE_CONTINUE4:.*]]
; TFA_INTERLEAVE: [[PRED_STORE_IF3]]:
-; TFA_INTERLEAVE-NEXT: store double [[PREDPHI]], ptr [[P]], align 8
+; TFA_INTERLEAVE-NEXT: [[TMP8:%.*]] = extractelement <2 x double> [[PREDPHI1]], i64 1
+; TFA_INTERLEAVE-NEXT: store double [[TMP8]], ptr [[P]], align 8
; TFA_INTERLEAVE-NEXT: br label %[[PRED_STORE_CONTINUE4]]
; TFA_INTERLEAVE: [[PRED_STORE_CONTINUE4]]:
-; TFA_INTERLEAVE-NEXT: [[INDEX_NEXT]] = add i64 [[INDEX]], 2
-; TFA_INTERLEAVE-NEXT: [[ACTIVE_LANE_MASK_NEXT]] = icmp ult i64 [[INDEX_NEXT]], [[TMP0]]
-; TFA_INTERLEAVE-NEXT: [[TMP4:%.*]] = add i64 [[INDEX_NEXT]], 1
-; TFA_INTERLEAVE-NEXT: [[ACTIVE_LANE_MASK_NEXT5]] = icmp ult i64 [[TMP4]], [[TMP0]]
+; TFA_INTERLEAVE-NEXT: [[TMP9:%.*]] = extractelement <2 x i1> [[ACTIVE_LANE_MASK3]], i64 0
+; TFA_INTERLEAVE-NEXT: br i1 [[TMP9]], label %[[PRED_STORE_IF5:.*]], label %[[PRED_STORE_CONTINUE6:.*]]
+; TFA_INTERLEAVE: [[PRED_STORE_IF5]]:
+; TFA_INTERLEAVE-NEXT: [[TMP10:%.*]] = extractelement <2 x double> [[PREDPHI1]], i64 0
+; TFA_INTERLEAVE-NEXT: store double [[TMP10]], ptr [[P]], align 8
+; TFA_INTERLEAVE-NEXT: br label %[[PRED_STORE_CONTINUE6]]
+; TFA_INTERLEAVE: [[PRED_STORE_CONTINUE6]]:
+; TFA_INTERLEAVE-NEXT: [[TMP11:%.*]] = extractelement <2 x i1> [[ACTIVE_LANE_MASK3]], i64 1
+; TFA_INTERLEAVE-NEXT: br i1 [[TMP11]], label %[[PRED_STORE_IF7:.*]], label %[[PRED_STORE_CONTINUE8]]
+; TFA_INTERLEAVE: [[PRED_STORE_IF7]]:
+; TFA_INTERLEAVE-NEXT: [[TMP12:%.*]] = extractelement <2 x double> [[PREDPHI1]], i64 1
+; TFA_INTERLEAVE-NEXT: store double [[TMP12]], ptr [[P]], align 8
+; TFA_INTERLEAVE-NEXT: br label %[[PRED_STORE_CONTINUE8]]
+; TFA_INTERLEAVE: [[PRED_STORE_CONTINUE8]]:
+; TFA_INTERLEAVE-NEXT: [[INDEX_NEXT]] = add i64 [[INDEX]], 4
+; TFA_INTERLEAVE-NEXT: [[ACTIVE_LANE_MASK_NEXT1:%.*]] = call <4 x i1> @llvm.get.active.lane.mask.v4i1.i64(i64 [[INDEX_NEXT]], i64 [[TMP0]])
+; TFA_INTERLEAVE-NEXT: [[EXTRACT_NEXT_ALM_PART]] = call <2 x i1> @llvm.vector.extract.v2i1.v4i1(<4 x i1> [[ACTIVE_LANE_MASK_NEXT1]], i64 0)
+; TFA_INTERLEAVE-NEXT: [[EXTRACT_NEXT_ALM_PART9]] = call <2 x i1> @llvm.vector.extract.v2i1.v4i1(<4 x i1> [[ACTIVE_LANE_MASK_NEXT1]], i64 2)
+; TFA_INTERLEAVE-NEXT: [[ACTIVE_LANE_MASK_NEXT:%.*]] = extractelement <2 x i1> [[EXTRACT_NEXT_ALM_PART]], i64 0
; TFA_INTERLEAVE-NEXT: [[TMP5:%.*]] = xor i1 [[ACTIVE_LANE_MASK_NEXT]], true
; TFA_INTERLEAVE-NEXT: br i1 [[TMP5]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP7:![0-9]+]]
; TFA_INTERLEAVE: [[MIDDLE_BLOCK]]:
diff --git a/llvm/test/Transforms/LoopVectorize/AArch64/partial-reduce-incomplete-chains.ll b/llvm/test/Transforms/LoopVectorize/AArch64/partial-reduce-incomplete-chains.ll
index 48d709840082c..03baf2830c5c7 100644
--- a/llvm/test/Transforms/LoopVectorize/AArch64/partial-reduce-incomplete-chains.ll
+++ b/llvm/test/Transforms/LoopVectorize/AArch64/partial-reduce-incomplete-chains.ll
@@ -74,30 +74,30 @@ exit:
define i16 @test_incomplete_chain_without_mul(ptr noalias %dst, ptr %A, ptr %B) #0 {
; CHECK-LABEL: define i16 @test_incomplete_chain_without_mul(
; CHECK-SAME: ptr noalias [[DST:%.*]], ptr [[A:%.*]], ptr [[B:%.*]]) #[[ATTR1:[0-9]+]] {
-; CHECK-NEXT: [[ENTRY:.*:]]
-; CHECK-NEXT: br label %[[VECTOR_PH:.*]]
-; CHECK: [[VECTOR_PH]]:
+; CHECK-NEXT: [[VECTOR_PH:.*:]]
; CHECK-NEXT: br label %[[VECTOR_BODY:.*]]
; CHECK: [[VECTOR_BODY]]:
-; CHECK-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
-; CHECK-NEXT: [[VEC_PHI:%.*]] = phi <16 x i16> [ zeroinitializer, %[[VECTOR_PH]] ], [ [[TMP7:%.*]], %[[VECTOR_BODY]] ]
; CHECK-NEXT: [[TMP0:%.*]] = load i8, ptr [[A]], align 1
; CHECK-NEXT: [[BROADCAST_SPLATINSERT:%.*]] = insertelement <16 x i8> poison, i8 [[TMP0]], i64 0
; CHECK-NEXT: [[BROADCAST_SPLAT:%.*]] = shufflevector <16 x i8> [[BROADCAST_SPLATINSERT]], <16 x i8> poison, <16 x i32> zeroinitializer
; CHECK-NEXT: [[TMP1:%.*]] = zext <16 x i8> [[BROADCAST_SPLAT]] to <16 x i16>
; CHECK-NEXT: [[TMP2:%.*]] = extractelement <16 x i16> [[TMP1]], i64 15
-; CHECK-NEXT: store i16 [[TMP2]], ptr [[DST]], align 2
; CHECK-NEXT: [[TMP3:%.*]] = load i8, ptr [[B]], align 1
; CHECK-NEXT: [[BROADCAST_SPLATINSERT1:%.*]] = insertelement <16 x i8> poison, i8 [[TMP3]], i64 0
; CHECK-NEXT: [[BROADCAST_SPLAT2:%.*]] = shufflevector <16 x i8> [[BROADCAST_SPLATINSERT1]], <16 x i8> poison, <16 x i32> zeroinitializer
; CHECK-NEXT: [[TMP4:%.*]] = zext <16 x i8> [[BROADCAST_SPLAT2]] to <16 x i16>
+; CHECK-NEXT: br label %[[VECTOR_BODY1:.*]]
+; CHECK: [[VECTOR_BODY1]]:
+; CHECK-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_BODY]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY1]] ]
+; CHECK-NEXT: [[VEC_PHI:%.*]] = phi <16 x i16> [ zeroinitializer, %[[VECTOR_BODY]] ], [ [[TMP7:%.*]], %[[VECTOR_BODY1]] ]
; CHECK-NEXT: [[TMP5:%.*]] = add <16 x i16> [[VEC_PHI]], [[TMP4]]
; CHECK-NEXT: [[TMP6:%.*]] = add <16 x i16> [[TMP5]], [[TMP1]]
; CHECK-NEXT: [[TMP7]] = add <16 x i16> [[TMP6]], [[TMP4]]
; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 16
; CHECK-NEXT: [[TMP8:%.*]] = icmp eq i64 [[INDEX_NEXT]], 1024
-; CHECK-NEXT: br i1 [[TMP8]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP4:![0-9]+]]
+; CHECK-NEXT: br i1 [[TMP8]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY1]], !llvm.loop [[LOOP4:![0-9]+]]
; CHECK: [[MIDDLE_BLOCK]]:
+; CHECK-NEXT: store i16 [[TMP2]], ptr [[DST]], align 2
; CHECK-NEXT: [[TMP9:%.*]] = call i16 @llvm.vector.reduce.add.v16i16(<16 x i16> [[TMP7]])
; CHECK-NEXT: br label %[[EXIT:.*]]
; CHECK: [[EXIT]]:
diff --git a/llvm/test/Transforms/LoopVectorize/AArch64/replicating-load-store-costs-apple.ll b/llvm/test/Transforms/LoopVectorize/AArch64/replicating-load-store-costs-apple.ll
index c16f93c902239..67d18d58ab3e9 100644
--- a/llvm/test/Transforms/LoopVectorize/AArch64/replicating-load-store-costs-apple.ll
+++ b/llvm/test/Transforms/LoopVectorize/AArch64/replicating-load-store-costs-apple.ll
@@ -62,22 +62,22 @@ exit:
define void @replicating_load_used_as_store_addr_2(ptr noalias %invar.dst, ptr noalias %invar.src, ptr noalias %src) {
; CHECK-LABEL: define void @replicating_load_used_as_store_addr_2(
; CHECK-SAME: ptr noalias [[INVAR_DST:%.*]], ptr noalias [[INVAR_SRC:%.*]], ptr noalias [[SRC:%.*]]) #[[ATTR0]] {
-; CHECK-NEXT: [[ENTRY:.*:]]
-; CHECK-NEXT: br label %[[VECTOR_PH:.*]]
-; CHECK: [[VECTOR_PH]]:
+; CHECK-NEXT: [[VECTOR_PH:.*:]]
; CHECK-NEXT: br label %[[VECTOR_BODY:.*]]
; CHECK: [[VECTOR_BODY]]:
-; CHECK-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
; CHECK-NEXT: [[TMP0:%.*]] = load i32, ptr [[INVAR_SRC]], align 4
; CHECK-NEXT: [[TMP1:%.*]] = sext i32 [[TMP0]] to i64
; CHECK-NEXT: [[TMP2:%.*]] = getelementptr i128, ptr [[SRC]], i64 [[TMP1]]
; CHECK-NEXT: [[TMP3:%.*]] = load i32, ptr [[TMP2]], align 4
; CHECK-NEXT: [[TMP4:%.*]] = add i32 [[TMP3]], 123
-; CHECK-NEXT: store i32 [[TMP4]], ptr [[INVAR_DST]], align 8
+; CHECK-NEXT: br label %[[VECTOR_BODY1:.*]]
+; CHECK: [[VECTOR_BODY1]]:
+; CHECK-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_BODY]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY1]] ]
; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 4
; CHECK-NEXT: [[TMP5:%.*]] = icmp eq i64 [[INDEX_NEXT]], 100
-; CHECK-NEXT: br i1 [[TMP5]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP4:![0-9]+]]
+; CHECK-NEXT: br i1 [[TMP5]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY1]], !llvm.loop [[LOOP4:![0-9]+]]
; CHECK: [[MIDDLE_BLOCK]]:
+; CHECK-NEXT: store i32 [[TMP4]], ptr [[INVAR_DST]], align 8
; CHECK-NEXT: br label %[[EXIT:.*]]
; CHECK: [[EXIT]]:
; CHECK-NEXT: ret void
@@ -111,21 +111,21 @@ define void @replicating_load_used_as_store_addr_3(ptr noalias %src, ptr noalias
; CHECK-NEXT: [[TMP0:%.*]] = xor i8 [[X]], 10
; CHECK-NEXT: [[TMP1:%.*]] = zext i8 [[TMP0]] to i64
; CHECK-NEXT: [[TMP2:%.*]] = getelementptr i8, ptr [[SRC]], i64 [[TMP1]]
-; CHECK-NEXT: br label %[[VECTOR_BODY:.*]]
-; CHECK: [[VECTOR_BODY]]:
-; CHECK-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
; CHECK-NEXT: [[TMP3:%.*]] = load i8, ptr [[TMP2]], align 1
; CHECK-NEXT: [[TMP4:%.*]] = zext i8 [[TMP3]] to i32
; CHECK-NEXT: [[TMP5:%.*]] = xor i32 [[TMP4]], 111
; CHECK-NEXT: [[TMP6:%.*]] = zext i32 [[TMP4]] to i64
; CHECK-NEXT: [[TMP7:%.*]] = getelementptr i8, ptr [[DST]], i64 [[TMP6]]
-; CHECK-NEXT: store i8 0, ptr [[TMP7]], align 1
; CHECK-NEXT: [[TMP8:%.*]] = trunc i32 [[TMP5]] to i8
-; CHECK-NEXT: store i8 [[TMP8]], ptr [[INVAR_DST]], align 1
+; CHECK-NEXT: br label %[[VECTOR_BODY:.*]]
+; CHECK: [[VECTOR_BODY]]:
+; CHECK-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 4
; CHECK-NEXT: [[TMP9:%.*]] = icmp eq i64 [[INDEX_NEXT]], 100
; CHECK-NEXT: br i1 [[TMP9]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP5:![0-9]+]]
; CHECK: [[MIDDLE_BLOCK]]:
+; CHECK-NEXT: store i8 0, ptr [[TMP7]], align 1
+; CHECK-NEXT: store i8 [[TMP8]], ptr [[INVAR_DST]], align 1
; CHECK-NEXT: br label %[[EXIT:.*]]
; CHECK: [[EXIT]]:
; CHECK-NEXT: ret void
diff --git a/llvm/test/Transforms/LoopVectorize/AArch64/replicating-load-store-costs.ll b/llvm/test/Transforms/LoopVectorize/AArch64/replicating-load-store-costs.ll
index ee038890245ae..ad4db68b6b6dc 100644
--- a/llvm/test/Transforms/LoopVectorize/AArch64/replicating-load-store-costs.ll
+++ b/llvm/test/Transforms/LoopVectorize/AArch64/replicating-load-store-costs.ll
@@ -62,22 +62,22 @@ exit:
define void @replicating_load_used_as_store_addr_2(ptr noalias %invar.dst, ptr noalias %invar.src, ptr noalias %src) {
; CHECK-LABEL: define void @replicating_load_used_as_store_addr_2(
; CHECK-SAME: ptr noalias [[INVAR_DST:%.*]], ptr noalias [[INVAR_SRC:%.*]], ptr noalias [[SRC:%.*]]) {
-; CHECK-NEXT: [[ENTRY:.*:]]
-; CHECK-NEXT: br label %[[VECTOR_PH:.*]]
-; CHECK: [[VECTOR_PH]]:
+; CHECK-NEXT: [[VECTOR_PH:.*:]]
; CHECK-NEXT: br label %[[VECTOR_BODY:.*]]
; CHECK: [[VECTOR_BODY]]:
-; CHECK-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
; CHECK-NEXT: [[TMP0:%.*]] = load i32, ptr [[INVAR_SRC]], align 4
; CHECK-NEXT: [[TMP1:%.*]] = sext i32 [[TMP0]] to i64
; CHECK-NEXT: [[TMP2:%.*]] = getelementptr i128, ptr [[SRC]], i64 [[TMP1]]
; CHECK-NEXT: [[TMP3:%.*]] = load i32, ptr [[TMP2]], align 4
; CHECK-NEXT: [[TMP4:%.*]] = add i32 [[TMP3]], 123
-; CHECK-NEXT: store i32 [[TMP4]], ptr [[INVAR_DST]], align 8
+; CHECK-NEXT: br label %[[VECTOR_BODY1:.*]]
+; CHECK: [[VECTOR_BODY1]]:
+; CHECK-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_BODY]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY1]] ]
; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 4
; CHECK-NEXT: [[TMP5:%.*]] = icmp eq i64 [[INDEX_NEXT]], 100
-; CHECK-NEXT: br i1 [[TMP5]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP4:![0-9]+]]
+; CHECK-NEXT: br i1 [[TMP5]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY1]], !llvm.loop [[LOOP4:![0-9]+]]
; CHECK: [[MIDDLE_BLOCK]]:
+; CHECK-NEXT: store i32 [[TMP4]], ptr [[INVAR_DST]], align 8
; CHECK-NEXT: br label %[[EXIT:.*]]
; CHECK: [[EXIT]]:
; CHECK-NEXT: ret void
@@ -111,21 +111,21 @@ define void @replicating_load_used_as_store_addr_3(ptr noalias %src, ptr noalias
; CHECK-NEXT: [[TMP0:%.*]] = xor i8 [[X]], 10
; CHECK-NEXT: [[TMP1:%.*]] = zext i8 [[TMP0]] to i64
; CHECK-NEXT: [[TMP2:%.*]] = getelementptr i8, ptr [[SRC]], i64 [[TMP1]]
-; CHECK-NEXT: br label %[[VECTOR_BODY:.*]]
-; CHECK: [[VECTOR_BODY]]:
-; CHECK-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
; CHECK-NEXT: [[TMP3:%.*]] = load i8, ptr [[TMP2]], align 1
; CHECK-NEXT: [[TMP4:%.*]] = zext i8 [[TMP3]] to i32
; CHECK-NEXT: [[TMP5:%.*]] = xor i32 [[TMP4]], 111
; CHECK-NEXT: [[TMP6:%.*]] = zext i32 [[TMP4]] to i64
; CHECK-NEXT: [[TMP7:%.*]] = getelementptr i8, ptr [[DST]], i64 [[TMP6]]
-; CHECK-NEXT: store i8 0, ptr [[TMP7]], align 1
; CHECK-NEXT: [[TMP8:%.*]] = trunc i32 [[TMP5]] to i8
-; CHECK-NEXT: store i8 [[TMP8]], ptr [[INVAR_DST]], align 1
+; CHECK-NEXT: br label %[[VECTOR_BODY:.*]]
+; CHECK: [[VECTOR_BODY]]:
+; CHECK-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 4
; CHECK-NEXT: [[TMP9:%.*]] = icmp eq i64 [[INDEX_NEXT]], 100
; CHECK-NEXT: br i1 [[TMP9]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP5:![0-9]+]]
; CHECK: [[MIDDLE_BLOCK]]:
+; CHECK-NEXT: store i8 0, ptr [[TMP7]], align 1
+; CHECK-NEXT: store i8 [[TMP8]], ptr [[INVAR_DST]], align 1
; CHECK-NEXT: br label %[[EXIT:.*]]
; CHECK: [[EXIT]]:
; CHECK-NEXT: ret void
diff --git a/llvm/test/Transforms/LoopVectorize/AArch64/type-shrinkage-insertelt.ll b/llvm/test/Transforms/LoopVectorize/AArch64/type-shrinkage-insertelt.ll
index 434a427197e0a..3b8f132f3e756 100644
--- a/llvm/test/Transforms/LoopVectorize/AArch64/type-shrinkage-insertelt.ll
+++ b/llvm/test/Transforms/LoopVectorize/AArch64/type-shrinkage-insertelt.ll
@@ -80,23 +80,23 @@ define void @test1(ptr noalias %M3, ptr noalias %A, ptr noalias %B, ptr noalias
; CHECK-NEXT: entry:
; CHECK-NEXT: br label [[VECTOR_PH:%.*]]
; CHECK: vector.ph:
+; CHECK-NEXT: [[TMP0:%.*]] = load i32, ptr [[C]], align 4
+; CHECK-NEXT: [[BROADCAST_SPLATINSERT:%.*]] = insertelement <4 x i32> poison, i32 [[TMP0]], i64 0
+; CHECK-NEXT: [[BROADCAST_SPLAT:%.*]] = shufflevector <4 x i32> [[BROADCAST_SPLATINSERT]], <4 x i32> poison, <4 x i32> zeroinitializer
+; CHECK-NEXT: [[TMP1:%.*]] = trunc <4 x i32> [[BROADCAST_SPLAT]] to <4 x i16>
; CHECK-NEXT: br label [[VECTOR_BODY:%.*]]
; CHECK: vector.body:
; CHECK-NEXT: [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ]
-; CHECK-NEXT: [[TMP0:%.*]] = add i64 [[INDEX]], 1
-; CHECK-NEXT: [[TMP1:%.*]] = add i64 [[INDEX]], 2
-; CHECK-NEXT: [[TMP2:%.*]] = add i64 [[INDEX]], 3
-; CHECK-NEXT: [[TMP3:%.*]] = load i32, ptr [[C]], align 4
-; CHECK-NEXT: [[BROADCAST_SPLATINSERT:%.*]] = insertelement <4 x i32> poison, i32 [[TMP3]], i64 0
-; CHECK-NEXT: [[BROADCAST_SPLAT:%.*]] = shufflevector <4 x i32> [[BROADCAST_SPLATINSERT]], <4 x i32> poison, <4 x i32> zeroinitializer
-; CHECK-NEXT: [[TMP4:%.*]] = getelementptr inbounds i16, ptr [[A]], i64 [[INDEX]]
-; CHECK-NEXT: [[WIDE_LOAD:%.*]] = load <4 x i16>, ptr [[TMP4]], align 2
-; CHECK-NEXT: [[TMP5:%.*]] = trunc <4 x i32> [[BROADCAST_SPLAT]] to <4 x i16>
-; CHECK-NEXT: [[TMP6:%.*]] = add <4 x i16> [[WIDE_LOAD]], [[TMP5]]
+; CHECK-NEXT: [[TMP2:%.*]] = add i64 [[INDEX]], 1
+; CHECK-NEXT: [[TMP3:%.*]] = add i64 [[INDEX]], 2
+; CHECK-NEXT: [[TMP4:%.*]] = add i64 [[INDEX]], 3
+; CHECK-NEXT: [[TMP5:%.*]] = getelementptr inbounds i16, ptr [[A]], i64 [[INDEX]]
+; CHECK-NEXT: [[WIDE_LOAD:%.*]] = load <4 x i16>, ptr [[TMP5]], align 2
+; CHECK-NEXT: [[TMP6:%.*]] = add <4 x i16> [[WIDE_LOAD]], [[TMP1]]
; CHECK-NEXT: [[TMP7:%.*]] = getelementptr inbounds i64, ptr [[B]], i64 [[INDEX]]
-; CHECK-NEXT: [[TMP8:%.*]] = getelementptr inbounds i64, ptr [[B]], i64 [[TMP0]]
-; CHECK-NEXT: [[TMP9:%.*]] = getelementptr inbounds i64, ptr [[B]], i64 [[TMP1]]
-; CHECK-NEXT: [[TMP10:%.*]] = getelementptr inbounds i64, ptr [[B]], i64 [[TMP2]]
+; CHECK-NEXT: [[TMP8:%.*]] = getelementptr inbounds i64, ptr [[B]], i64 [[TMP2]]
+; CHECK-NEXT: [[TMP9:%.*]] = getelementptr inbounds i64, ptr [[B]], i64 [[TMP3]]
+; CHECK-NEXT: [[TMP10:%.*]] = getelementptr inbounds i64, ptr [[B]], i64 [[TMP4]]
; CHECK-NEXT: [[TMP11:%.*]] = load i64, ptr [[TMP7]], align 8
; CHECK-NEXT: [[TMP12:%.*]] = load i64, ptr [[TMP8]], align 8
; CHECK-NEXT: [[TMP13:%.*]] = load i64, ptr [[TMP9]], align 8
diff --git a/llvm/test/Transforms/LoopVectorize/RISCV/strided-accesses.ll b/llvm/test/Transforms/LoopVectorize/RISCV/strided-accesses.ll
index 805ec6f1b4004..23dd7710e2737 100644
--- a/llvm/test/Transforms/LoopVectorize/RISCV/strided-accesses.ll
+++ b/llvm/test/Transforms/LoopVectorize/RISCV/strided-accesses.ll
@@ -1573,15 +1573,15 @@ define void @type_cache_crash(ptr noalias %a, ptr noalias %b, ptr noalias %c, i3
; CHECK-UF2-NEXT: [[BROADCAST_SPLAT6:%.*]] = shufflevector <vscale x 2 x double> [[BROADCAST_SPLATINSERT5]], <vscale x 2 x double> poison, <vscale x 2 x i32> zeroinitializer
; CHECK-UF2-NEXT: [[TMP12:%.*]] = getelementptr [8 x i8], ptr [[B]], i64 [[C3]]
; CHECK-UF2-NEXT: [[TMP13:%.*]] = getelementptr i8, ptr [[TMP12]], i64 48
+; CHECK-UF2-NEXT: [[TMP14:%.*]] = load double, ptr [[TMP13]], align 8
+; CHECK-UF2-NEXT: [[BROADCAST_SPLATINSERT3:%.*]] = insertelement <vscale x 2 x double> poison, double [[TMP14]], i64 0
+; CHECK-UF2-NEXT: [[BROADCAST_SPLAT4:%.*]] = shufflevector <vscale x 2 x double> [[BROADCAST_SPLATINSERT3]], <vscale x 2 x double> poison, <vscale x 2 x i32> zeroinitializer
; CHECK-UF2-NEXT: [[BROADCAST_SPLATINSERT1:%.*]] = insertelement <vscale x 2 x double> poison, double [[C2]], i64 0
; CHECK-UF2-NEXT: [[BROADCAST_SPLAT2:%.*]] = shufflevector <vscale x 2 x double> [[BROADCAST_SPLATINSERT1]], <vscale x 2 x double> poison, <vscale x 2 x i32> zeroinitializer
; CHECK-UF2-NEXT: br label %[[VECTOR_BODY:.*]]
; CHECK-UF2: [[VECTOR_BODY]]:
; CHECK-UF2-NEXT: [[INDEX:%.*]] = phi i32 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
; CHECK-UF2-NEXT: store double 0.000000e+00, ptr [[A]], align 8
-; CHECK-UF2-NEXT: [[TMP14:%.*]] = load double, ptr [[TMP13]], align 8
-; CHECK-UF2-NEXT: [[BROADCAST_SPLATINSERT3:%.*]] = insertelement <vscale x 2 x double> poison, double [[TMP14]], i64 0
-; CHECK-UF2-NEXT: [[BROADCAST_SPLAT4:%.*]] = shufflevector <vscale x 2 x double> [[BROADCAST_SPLATINSERT3]], <vscale x 2 x double> poison, <vscale x 2 x i32> zeroinitializer
; CHECK-UF2-NEXT: [[TMP15:%.*]] = zext i32 [[INDEX]] to i64
; CHECK-UF2-NEXT: [[TMP29:%.*]] = shl i64 [[TMP15]], 5
; CHECK-UF2-NEXT: [[TMP16:%.*]] = getelementptr i8, ptr [[B]], i64 [[TMP29]]
diff --git a/llvm/test/Transforms/LoopVectorize/SystemZ/pr47665.ll b/llvm/test/Transforms/LoopVectorize/SystemZ/pr47665.ll
index 20a2604b2a9ba..5c8d4c44c631e 100644
--- a/llvm/test/Transforms/LoopVectorize/SystemZ/pr47665.ll
+++ b/llvm/test/Transforms/LoopVectorize/SystemZ/pr47665.ll
@@ -12,7 +12,6 @@ define void @test(ptr noalias %p, ptr noalias %q, i40 %a) {
; CHECK-NEXT: [[IV:%.*]] = phi i32 [ 0, %[[FOR_BODY]] ], [ [[INDEX_NEXT:%.*]], %[[PRED_STORE_CONTINUE6:.*]] ]
; CHECK-NEXT: [[VEC_IND:%.*]] = phi <4 x i8> [ <i8 0, i8 1, i8 2, i8 3>, %[[FOR_BODY]] ], [ [[VEC_IND_NEXT:%.*]], %[[PRED_STORE_CONTINUE6]] ]
; CHECK-NEXT: [[TMP0:%.*]] = icmp ule <4 x i8> [[VEC_IND]], splat (i8 9)
-; CHECK-NEXT: store i1 false, ptr [[P]], align 1
; CHECK-NEXT: [[TMP1:%.*]] = extractelement <4 x i1> [[TMP0]], i64 0
; CHECK-NEXT: br i1 [[TMP1]], label %[[PRED_STORE_IF:.*]], label %[[PRED_STORE_CONTINUE:.*]]
; CHECK: [[PRED_STORE_IF]]:
@@ -49,6 +48,7 @@ define void @test(ptr noalias %p, ptr noalias %q, i40 %a) {
; CHECK-NEXT: [[TMP12:%.*]] = icmp eq i32 [[INDEX_NEXT]], 12
; CHECK-NEXT: br i1 [[TMP12]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP0:![0-9]+]]
; CHECK: [[MIDDLE_BLOCK]]:
+; CHECK-NEXT: store i1 false, ptr [[P]], align 1
; CHECK-NEXT: br label %[[EXIT1:.*]]
; CHECK: [[EXIT1]]:
; CHECK-NEXT: ret void
diff --git a/llvm/test/Transforms/LoopVectorize/VPlan/first-order-recurrence-sink-replicate-region.ll b/llvm/test/Transforms/LoopVectorize/VPlan/first-order-recurrence-sink-replicate-region.ll
index 313dda16a48bc..ad2e05498e1f3 100644
--- a/llvm/test/Transforms/LoopVectorize/VPlan/first-order-recurrence-sink-replicate-region.ll
+++ b/llvm/test/Transforms/LoopVectorize/VPlan/first-order-recurrence-sink-replicate-region.ll
@@ -565,6 +565,7 @@ define void @need_new_block_after_sinking_pr56146(i32 %x, ptr %src, ptr noalias
; CHECK-NEXT: Successor(s): scalar.ph, vector.ph
; CHECK-EMPTY:
; CHECK-NEXT: vector.ph:
+; CHECK-NEXT: CLONE ir<%l> = load ir<%src>
; CHECK-NEXT: Successor(s): vector loop
; CHECK-EMPTY:
; CHECK-NEXT: <x1> vector loop: {
@@ -575,7 +576,6 @@ define void @need_new_block_after_sinking_pr56146(i32 %x, ptr %src, ptr noalias
; CHECK-NEXT: EMIT vp<[[VP6:%[0-9]+]]> = WIDEN-CANONICAL-INDUCTION nuw vp<[[VP4]]>
; CHECK-NEXT: EMIT vp<[[VP7:%[0-9]+]]> = icmp ule vp<[[VP6]]>, vp<[[VP3]]>
; CHECK-NEXT: vp<[[VP8:%[0-9]+]]> = DERIVED-IV nuw ir<2> + vp<[[VP4]]> * ir<1>
-; CHECK-NEXT: CLONE ir<%l> = load ir<%src>
; CHECK-NEXT: EMIT vp<[[VP9:%[0-9]+]]> = first-order splice ir<%.pn>, ir<%l>
; CHECK-NEXT: Successor(s): pred.store
; CHECK-EMPTY:
diff --git a/llvm/test/Transforms/LoopVectorize/VPlan/vplan-sink-scalars-and-merge.ll b/llvm/test/Transforms/LoopVectorize/VPlan/vplan-sink-scalars-and-merge.ll
index bd9da43107f38..b0f57a318e49e 100644
--- a/llvm/test/Transforms/LoopVectorize/VPlan/vplan-sink-scalars-and-merge.ll
+++ b/llvm/test/Transforms/LoopVectorize/VPlan/vplan-sink-scalars-and-merge.ll
@@ -360,6 +360,7 @@ define void @uniform_gep(i64 %k, ptr noalias %A, ptr noalias %B) {
; CHECK-NEXT: Successor(s): scalar.ph, vector.ph
; CHECK-EMPTY:
; CHECK-NEXT: vector.ph:
+; CHECK-NEXT: CLONE ir<%lv> = load ir<%A>
; CHECK-NEXT: Successor(s): vector loop
; CHECK-EMPTY:
; CHECK-NEXT: <x1> vector loop: {
@@ -370,7 +371,6 @@ define void @uniform_gep(i64 %k, ptr noalias %A, ptr noalias %B) {
; CHECK-NEXT: EMIT vp<[[VP6:%[0-9]+]]> = WIDEN-CANONICAL-INDUCTION nuw vp<[[VP4]]>
; CHECK-NEXT: EMIT vp<[[VP7:%[0-9]+]]> = icmp ule vp<[[VP6]]>, vp<[[VP3]]>
; CHECK-NEXT: vp<[[VP8:%[0-9]+]]> = DERIVED-IV ir<21> + vp<[[VP4]]> * ir<1>
-; CHECK-NEXT: CLONE ir<%lv> = load ir<%A>
; CHECK-NEXT: WIDEN ir<%cmp> = icmp uge ir<%iv>, ir<%k>
; CHECK-NEXT: EMIT vp<[[VP9:%[0-9]+]]> = logical-and vp<[[VP7]]>, ir<%cmp>
; CHECK-NEXT: Successor(s): pred.store
diff --git a/llvm/test/Transforms/LoopVectorize/constant-fold-commutative-and.ll b/llvm/test/Transforms/LoopVectorize/constant-fold-commutative-and.ll
index 7385c4114c8ad..1c9b5791ee6a5 100644
--- a/llvm/test/Transforms/LoopVectorize/constant-fold-commutative-and.ll
+++ b/llvm/test/Transforms/LoopVectorize/constant-fold-commutative-and.ll
@@ -13,20 +13,20 @@ define void @constant_fold_commutative_and(ptr %ptr.n, ptr noalias %p, i1 %cond)
; CHECK: [[VECTOR_PH]]:
; CHECK-NEXT: [[BROADCAST_SPLATINSERT:%.*]] = insertelement <2 x i1> poison, i1 [[COND]], i64 0
; CHECK-NEXT: [[BROADCAST_SPLAT:%.*]] = shufflevector <2 x i1> [[BROADCAST_SPLATINSERT]], <2 x i1> poison, <2 x i32> zeroinitializer
-; CHECK-NEXT: [[TMP0:%.*]] = xor <2 x i1> [[BROADCAST_SPLAT]], splat (i1 true)
-; CHECK-NEXT: br label %[[VECTOR_BODY:.*]]
-; CHECK: [[VECTOR_BODY]]:
-; CHECK-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[PRED_STORE_CONTINUE5:.*]] ]
-; CHECK-NEXT: [[VEC_IND:%.*]] = phi <2 x i8> [ <i8 0, i8 1>, %[[VECTOR_PH]] ], [ [[VEC_IND_NEXT:%.*]], %[[PRED_STORE_CONTINUE5]] ]
-; CHECK-NEXT: [[TMP1:%.*]] = icmp ule <2 x i8> [[VEC_IND]], splat (i8 16)
; CHECK-NEXT: [[TMP2:%.*]] = load i64, ptr [[PTR_N]], align 4
; CHECK-NEXT: [[BROADCAST_SPLATINSERT1:%.*]] = insertelement <2 x i64> poison, i64 [[TMP2]], i64 0
; CHECK-NEXT: [[BROADCAST_SPLAT2:%.*]] = shufflevector <2 x i64> [[BROADCAST_SPLATINSERT1]], <2 x i64> poison, <2 x i32> zeroinitializer
+; CHECK-NEXT: [[TMP0:%.*]] = xor <2 x i1> [[BROADCAST_SPLAT]], splat (i1 true)
; CHECK-NEXT: [[TMP3:%.*]] = icmp ult <2 x i64> [[BROADCAST_SPLAT2]], splat (i64 4)
; CHECK-NEXT: [[TMP4:%.*]] = icmp ult <2 x i64> [[BROADCAST_SPLAT2]], splat (i64 7)
; CHECK-NEXT: [[TMP5:%.*]] = select <2 x i1> [[TMP0]], <2 x i1> [[TMP3]], <2 x i1> zeroinitializer
; CHECK-NEXT: [[PREDPHI:%.*]] = select <2 x i1> [[TMP5]], <2 x i1> splat (i1 true), <2 x i1> [[TMP4]]
; CHECK-NEXT: [[PREDPHI3:%.*]] = select i1 [[COND]], <2 x i1> zeroinitializer, <2 x i1> [[PREDPHI]]
+; CHECK-NEXT: br label %[[VECTOR_BODY:.*]]
+; CHECK: [[VECTOR_BODY]]:
+; CHECK-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[PRED_STORE_CONTINUE5:.*]] ]
+; CHECK-NEXT: [[VEC_IND:%.*]] = phi <2 x i8> [ <i8 0, i8 1>, %[[VECTOR_PH]] ], [ [[VEC_IND_NEXT:%.*]], %[[PRED_STORE_CONTINUE5]] ]
+; CHECK-NEXT: [[TMP1:%.*]] = icmp ule <2 x i8> [[VEC_IND]], splat (i8 16)
; CHECK-NEXT: [[TMP6:%.*]] = extractelement <2 x i1> [[TMP1]], i64 0
; CHECK-NEXT: br i1 [[TMP6]], label %[[PRED_STORE_IF:.*]], label %[[PRED_STORE_CONTINUE:.*]]
; CHECK: [[PRED_STORE_IF]]:
diff --git a/llvm/test/Transforms/LoopVectorize/cse-casts.ll b/llvm/test/Transforms/LoopVectorize/cse-casts.ll
index df97c0d129654..c713aeeca6a17 100644
--- a/llvm/test/Transforms/LoopVectorize/cse-casts.ll
+++ b/llvm/test/Transforms/LoopVectorize/cse-casts.ll
@@ -341,23 +341,23 @@ exit:
define void @simplified_cast_preserves_irflag_type(ptr noalias %p, ptr noalias %q, ptr noalias %r) {
; CHECK-LABEL: define void @simplified_cast_preserves_irflag_type(
; CHECK-SAME: ptr noalias [[P:%.*]], ptr noalias [[Q:%.*]], ptr noalias [[R:%.*]]) {
-; CHECK-NEXT: [[ENTRY:.*:]]
-; CHECK-NEXT: br label %[[VECTOR_PH:.*]]
-; CHECK: [[VECTOR_PH]]:
+; CHECK-NEXT: [[VECTOR_PH:.*:]]
; CHECK-NEXT: br label %[[VECTOR_BODY:.*]]
; CHECK: [[VECTOR_BODY]]:
-; CHECK-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
; CHECK-NEXT: [[TMP0:%.*]] = load i8, ptr [[P]], align 1
; CHECK-NEXT: [[BROADCAST_SPLATINSERT:%.*]] = insertelement <4 x i8> poison, i8 [[TMP0]], i64 0
; CHECK-NEXT: [[BROADCAST_SPLAT:%.*]] = shufflevector <4 x i8> [[BROADCAST_SPLATINSERT]], <4 x i8> poison, <4 x i32> zeroinitializer
; CHECK-NEXT: [[TMP1:%.*]] = zext <4 x i8> [[BROADCAST_SPLAT]] to <4 x i16>
; CHECK-NEXT: [[TMP2:%.*]] = extractelement <4 x i16> [[TMP1]], i64 3
-; CHECK-NEXT: store i16 [[TMP2]], ptr [[Q]], align 2
-; CHECK-NEXT: store i16 [[TMP2]], ptr [[R]], align 2
+; CHECK-NEXT: br label %[[VECTOR_BODY1:.*]]
+; CHECK: [[VECTOR_BODY1]]:
+; CHECK-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_BODY]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY1]] ]
; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 8
; CHECK-NEXT: [[TMP3:%.*]] = icmp eq i64 [[INDEX_NEXT]], 48
-; CHECK-NEXT: br i1 [[TMP3]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP4:![0-9]+]]
+; CHECK-NEXT: br i1 [[TMP3]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY1]], !llvm.loop [[LOOP4:![0-9]+]]
; CHECK: [[MIDDLE_BLOCK]]:
+; CHECK-NEXT: store i16 [[TMP2]], ptr [[Q]], align 2
+; CHECK-NEXT: store i16 [[TMP2]], ptr [[R]], align 2
; CHECK-NEXT: br label %[[SCALAR_PH:.*]]
; CHECK: [[SCALAR_PH]]:
;
diff --git a/llvm/test/Transforms/LoopVectorize/first-order-recurrence-chains.ll b/llvm/test/Transforms/LoopVectorize/first-order-recurrence-chains.ll
index cb3d7b5369251..b0c1006b8021d 100644
--- a/llvm/test/Transforms/LoopVectorize/first-order-recurrence-chains.ll
+++ b/llvm/test/Transforms/LoopVectorize/first-order-recurrence-chains.ll
@@ -1069,29 +1069,29 @@ define double @test_resinking_required(ptr %p, ptr noalias %a, ptr noalias %b, i
; CHECK: [[VECTOR_PH]]:
; CHECK-NEXT: [[N_MOD_VF:%.*]] = and i64 [[N]], 3
; CHECK-NEXT: [[N_VEC:%.*]] = sub i64 [[N]], [[N_MOD_VF]]
-; CHECK-NEXT: br label %[[VECTOR_BODY:.*]]
-; CHECK: [[VECTOR_BODY]]:
-; CHECK-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
-; CHECK-NEXT: [[VECTOR_RECUR:%.*]] = phi <4 x double> [ <double poison, double poison, double poison, double 0.000000e+00>, %[[VECTOR_PH]] ], [ [[BROADCAST_SPLAT:%.*]], %[[VECTOR_BODY]] ]
-; CHECK-NEXT: [[VECTOR_RECUR1:%.*]] = phi <4 x double> [ <double poison, double poison, double poison, double 0.000000e+00>, %[[VECTOR_PH]] ], [ [[BROADCAST_SPLAT4:%.*]], %[[VECTOR_BODY]] ]
-; CHECK-NEXT: [[VECTOR_RECUR2:%.*]] = phi <4 x double> [ <double poison, double poison, double poison, double 0.000000e+00>, %[[VECTOR_PH]] ], [ [[TMP4:%.*]], %[[VECTOR_BODY]] ]
; CHECK-NEXT: [[TMP0:%.*]] = load double, ptr [[A]], align 8
; CHECK-NEXT: [[BROADCAST_SPLATINSERT:%.*]] = insertelement <4 x double> poison, double [[TMP0]], i64 0
-; CHECK-NEXT: [[BROADCAST_SPLAT]] = shufflevector <4 x double> [[BROADCAST_SPLATINSERT]], <4 x double> poison, <4 x i32> zeroinitializer
-; CHECK-NEXT: [[TMP1:%.*]] = shufflevector <4 x double> [[VECTOR_RECUR]], <4 x double> [[BROADCAST_SPLAT]], <4 x i32> <i32 3, i32 4, i32 5, i32 6>
-; CHECK-NEXT: [[TMP2:%.*]] = fdiv <4 x double> zeroinitializer, [[TMP1]]
+; CHECK-NEXT: [[BROADCAST_SPLAT:%.*]] = shufflevector <4 x double> [[BROADCAST_SPLATINSERT]], <4 x double> poison, <4 x i32> zeroinitializer
; CHECK-NEXT: [[TMP3:%.*]] = load double, ptr [[B]], align 8
; CHECK-NEXT: [[BROADCAST_SPLATINSERT3:%.*]] = insertelement <4 x double> poison, double [[TMP3]], i64 0
-; CHECK-NEXT: [[BROADCAST_SPLAT4]] = shufflevector <4 x double> [[BROADCAST_SPLATINSERT3]], <4 x double> poison, <4 x i32> zeroinitializer
+; CHECK-NEXT: [[BROADCAST_SPLAT4:%.*]] = shufflevector <4 x double> [[BROADCAST_SPLATINSERT3]], <4 x double> poison, <4 x i32> zeroinitializer
+; CHECK-NEXT: br label %[[VECTOR_BODY:.*]]
+; CHECK: [[VECTOR_BODY]]:
+; CHECK-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-NEXT: [[VECTOR_RECUR:%.*]] = phi <4 x double> [ <double poison, double poison, double poison, double 0.000000e+00>, %[[VECTOR_PH]] ], [ [[BROADCAST_SPLAT]], %[[VECTOR_BODY]] ]
+; CHECK-NEXT: [[VECTOR_RECUR1:%.*]] = phi <4 x double> [ <double poison, double poison, double poison, double 0.000000e+00>, %[[VECTOR_PH]] ], [ [[BROADCAST_SPLAT4]], %[[VECTOR_BODY]] ]
+; CHECK-NEXT: [[VECTOR_RECUR2:%.*]] = phi <4 x double> [ <double poison, double poison, double poison, double 0.000000e+00>, %[[VECTOR_PH]] ], [ [[TMP4:%.*]], %[[VECTOR_BODY]] ]
; CHECK-NEXT: [[TMP4]] = shufflevector <4 x double> [[VECTOR_RECUR1]], <4 x double> [[BROADCAST_SPLAT4]], <4 x i32> <i32 3, i32 4, i32 5, i32 6>
+; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 4
+; CHECK-NEXT: [[TMP7:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
+; CHECK-NEXT: br i1 [[TMP7]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP28:![0-9]+]]
+; CHECK: [[MIDDLE_BLOCK]]:
+; CHECK-NEXT: [[TMP9:%.*]] = shufflevector <4 x double> [[VECTOR_RECUR]], <4 x double> [[BROADCAST_SPLAT]], <4 x i32> <i32 3, i32 4, i32 5, i32 6>
+; CHECK-NEXT: [[TMP2:%.*]] = fdiv <4 x double> zeroinitializer, [[TMP9]]
; CHECK-NEXT: [[TMP5:%.*]] = shufflevector <4 x double> [[VECTOR_RECUR2]], <4 x double> [[TMP4]], <4 x i32> <i32 3, i32 4, i32 5, i32 6>
; CHECK-NEXT: [[TMP8:%.*]] = fadd <4 x double> [[TMP2]], [[TMP5]]
; CHECK-NEXT: [[TMP6:%.*]] = extractelement <4 x double> [[TMP8]], i64 3
; CHECK-NEXT: store double [[TMP6]], ptr [[P]], align 8
-; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 4
-; CHECK-NEXT: [[TMP9:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
-; CHECK-NEXT: br i1 [[TMP9]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP28:![0-9]+]]
-; CHECK: [[MIDDLE_BLOCK]]:
; CHECK-NEXT: [[VECTOR_RECUR_EXTRACT_FOR_PHI:%.*]] = extractelement <4 x double> [[TMP4]], i64 2
; CHECK-NEXT: [[VECTOR_RECUR_EXTRACT:%.*]] = extractelement <4 x double> [[TMP4]], i64 3
; CHECK-NEXT: [[CMP_N:%.*]] = icmp eq i64 [[N]], [[N_VEC]]
diff --git a/llvm/test/Transforms/LoopVectorize/lcssa-crashes.ll b/llvm/test/Transforms/LoopVectorize/lcssa-crashes.ll
index 3261ba937fefa..4cd926144d5c3 100644
--- a/llvm/test/Transforms/LoopVectorize/lcssa-crashes.ll
+++ b/llvm/test/Transforms/LoopVectorize/lcssa-crashes.ll
@@ -197,9 +197,11 @@ define i32 @exit_phi_sunk_def(ptr noalias %src, ptr noalias %dst) {
; CHECK-NEXT: entry:
; CHECK-NEXT: br label [[VECTOR_PH:%.*]]
; CHECK: vector.ph:
+; CHECK-NEXT: [[TMP0:%.*]] = load i32, ptr [[SRC:%.*]], align 4
+; CHECK-NEXT: [[TMP1:%.*]] = icmp eq i32 [[TMP0]], 0
+; CHECK-NEXT: [[SEL:%.*]] = select i1 [[TMP1]], <4 x i32> zeroinitializer, <4 x i32> splat (i32 2)
; CHECK-NEXT: br label [[VECTOR_BODY:%.*]]
; CHECK: vector.body:
-; CHECK-NEXT: [[TMP0:%.*]] = load i32, ptr [[SRC:%.*]], align 4
; CHECK-NEXT: br i1 true, label [[PRED_STORE_IF:%.*]], label [[PRED_STORE_CONTINUE:%.*]]
; CHECK: pred.store.if:
; CHECK-NEXT: store i32 1, ptr [[DST:%.*]], align 4
@@ -222,8 +224,6 @@ define i32 @exit_phi_sunk_def(ptr noalias %src, ptr noalias %dst) {
; CHECK: pred.store.continue6:
; CHECK-NEXT: br label [[MIDDLE_BLOCK:%.*]]
; CHECK: middle.block:
-; CHECK-NEXT: [[TMP1:%.*]] = icmp eq i32 [[TMP0]], 0
-; CHECK-NEXT: [[SEL:%.*]] = select i1 [[TMP1]], <4 x i32> zeroinitializer, <4 x i32> splat (i32 2)
; CHECK-NEXT: [[EXT:%.*]] = extractelement <4 x i32> [[SEL]], i64 0
; CHECK-NEXT: br label [[EXIT:%.*]]
; CHECK: exit:
diff --git a/llvm/test/Transforms/LoopVectorize/narrow-to-single-scalar.ll b/llvm/test/Transforms/LoopVectorize/narrow-to-single-scalar.ll
index adf02a9a7f1fd..5f5bcd4b1bedc 100644
--- a/llvm/test/Transforms/LoopVectorize/narrow-to-single-scalar.ll
+++ b/llvm/test/Transforms/LoopVectorize/narrow-to-single-scalar.ll
@@ -17,12 +17,12 @@ define void @narrow_select_to_single_scalar(i1 %invar.cond, ptr noalias %A, ptr
; VF4IC1-NEXT: [[TMP5:%.*]] = add i16 [[OFFSET_IDX]], 3
; VF4IC1-NEXT: [[TMP6:%.*]] = getelementptr i8, ptr [[A]], i16 [[TMP5]]
; VF4IC1-NEXT: [[TMP7:%.*]] = load i16, ptr [[TMP6]], align 1
-; VF4IC1-NEXT: store i16 [[TMP7]], ptr [[B]], align 1
-; VF4IC1-NEXT: store i16 0, ptr [[TMP1]], align 1
; VF4IC1-NEXT: [[INDEX_NEXT]] = add nuw i32 [[INDEX]], 4
; VF4IC1-NEXT: [[TMP8:%.*]] = icmp eq i32 [[INDEX_NEXT]], 1024
; VF4IC1-NEXT: br i1 [[TMP8]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP0:![0-9]+]]
; VF4IC1: [[MIDDLE_BLOCK]]:
+; VF4IC1-NEXT: store i16 [[TMP7]], ptr [[B]], align 1
+; VF4IC1-NEXT: store i16 0, ptr [[TMP1]], align 1
; VF4IC1-NEXT: br label %[[EXIT:.*]]
; VF4IC1: [[EXIT]]:
; VF4IC1-NEXT: ret void
@@ -41,12 +41,12 @@ define void @narrow_select_to_single_scalar(i1 %invar.cond, ptr noalias %A, ptr
; VF2IC2-NEXT: [[TMP3:%.*]] = add i16 [[OFFSET_IDX]], 3
; VF2IC2-NEXT: [[TMP4:%.*]] = getelementptr i8, ptr [[A]], i16 [[TMP3]]
; VF2IC2-NEXT: [[TMP5:%.*]] = load i16, ptr [[TMP4]], align 1
-; VF2IC2-NEXT: store i16 [[TMP5]], ptr [[B]], align 1
-; VF2IC2-NEXT: store i16 0, ptr [[TMP1]], align 1
; VF2IC2-NEXT: [[INDEX_NEXT]] = add nuw i32 [[INDEX]], 4
; VF2IC2-NEXT: [[TMP6:%.*]] = icmp eq i32 [[INDEX_NEXT]], 1024
; VF2IC2-NEXT: br i1 [[TMP6]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP0:![0-9]+]]
; VF2IC2: [[MIDDLE_BLOCK]]:
+; VF2IC2-NEXT: store i16 [[TMP5]], ptr [[B]], align 1
+; VF2IC2-NEXT: store i16 0, ptr [[TMP1]], align 1
; VF2IC2-NEXT: br label %[[EXIT:.*]]
; VF2IC2: [[EXIT]]:
; VF2IC2-NEXT: ret void
@@ -223,46 +223,46 @@ exit:
define void @narrow_scatter_with_uniform_addr_to_scalar(ptr noalias %src, ptr noalias %dst, ptr %dst2) {
; VF4IC1-LABEL: define void @narrow_scatter_with_uniform_addr_to_scalar(
; VF4IC1-SAME: ptr noalias [[SRC:%.*]], ptr noalias [[DST:%.*]], ptr [[DST2:%.*]]) {
-; VF4IC1-NEXT: [[ENTRY:.*:]]
-; VF4IC1-NEXT: br label %[[VECTOR_PH:.*]]
-; VF4IC1: [[VECTOR_PH]]:
+; VF4IC1-NEXT: [[VECTOR_PH:.*:]]
; VF4IC1-NEXT: br label %[[VECTOR_BODY:.*]]
; VF4IC1: [[VECTOR_BODY]]:
-; VF4IC1-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
; VF4IC1-NEXT: [[TMP0:%.*]] = load i16, ptr [[SRC]], align 2
; VF4IC1-NEXT: [[BROADCAST_SPLATINSERT:%.*]] = insertelement <4 x i16> poison, i16 [[TMP0]], i64 0
; VF4IC1-NEXT: [[BROADCAST_SPLAT:%.*]] = shufflevector <4 x i16> [[BROADCAST_SPLATINSERT]], <4 x i16> poison, <4 x i32> zeroinitializer
; VF4IC1-NEXT: [[TMP1:%.*]] = trunc <4 x i16> [[BROADCAST_SPLAT]] to <4 x i8>
; VF4IC1-NEXT: [[TMP2:%.*]] = extractelement <4 x i8> [[TMP1]], i64 3
-; VF4IC1-NEXT: store i8 [[TMP2]], ptr [[DST2]], align 1
-; VF4IC1-NEXT: store i8 [[TMP2]], ptr [[DST]], align 1
+; VF4IC1-NEXT: br label %[[VECTOR_BODY1:.*]]
+; VF4IC1: [[VECTOR_BODY1]]:
+; VF4IC1-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_BODY]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY1]] ]
; VF4IC1-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 4
; VF4IC1-NEXT: [[TMP3:%.*]] = icmp eq i64 [[INDEX_NEXT]], 256
-; VF4IC1-NEXT: br i1 [[TMP3]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP5:![0-9]+]]
+; VF4IC1-NEXT: br i1 [[TMP3]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY1]], !llvm.loop [[LOOP5:![0-9]+]]
; VF4IC1: [[MIDDLE_BLOCK]]:
+; VF4IC1-NEXT: store i8 [[TMP2]], ptr [[DST2]], align 1
+; VF4IC1-NEXT: store i8 [[TMP2]], ptr [[DST]], align 1
; VF4IC1-NEXT: br label %[[EXIT:.*]]
; VF4IC1: [[EXIT]]:
; VF4IC1-NEXT: ret void
;
; VF2IC2-LABEL: define void @narrow_scatter_with_uniform_addr_to_scalar(
; VF2IC2-SAME: ptr noalias [[SRC:%.*]], ptr noalias [[DST:%.*]], ptr [[DST2:%.*]]) {
-; VF2IC2-NEXT: [[ENTRY:.*:]]
-; VF2IC2-NEXT: br label %[[VECTOR_PH:.*]]
-; VF2IC2: [[VECTOR_PH]]:
+; VF2IC2-NEXT: [[VECTOR_PH:.*:]]
; VF2IC2-NEXT: br label %[[VECTOR_BODY:.*]]
; VF2IC2: [[VECTOR_BODY]]:
-; VF2IC2-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
; VF2IC2-NEXT: [[TMP0:%.*]] = load i16, ptr [[SRC]], align 2
; VF2IC2-NEXT: [[BROADCAST_SPLATINSERT:%.*]] = insertelement <2 x i16> poison, i16 [[TMP0]], i64 0
; VF2IC2-NEXT: [[BROADCAST_SPLAT:%.*]] = shufflevector <2 x i16> [[BROADCAST_SPLATINSERT]], <2 x i16> poison, <2 x i32> zeroinitializer
; VF2IC2-NEXT: [[TMP1:%.*]] = trunc <2 x i16> [[BROADCAST_SPLAT]] to <2 x i8>
; VF2IC2-NEXT: [[TMP2:%.*]] = extractelement <2 x i8> [[TMP1]], i64 1
-; VF2IC2-NEXT: store i8 [[TMP2]], ptr [[DST2]], align 1
-; VF2IC2-NEXT: store i8 [[TMP2]], ptr [[DST]], align 1
+; VF2IC2-NEXT: br label %[[VECTOR_BODY1:.*]]
+; VF2IC2: [[VECTOR_BODY1]]:
+; VF2IC2-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_BODY]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY1]] ]
; VF2IC2-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 4
; VF2IC2-NEXT: [[TMP3:%.*]] = icmp eq i64 [[INDEX_NEXT]], 256
-; VF2IC2-NEXT: br i1 [[TMP3]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP5:![0-9]+]]
+; VF2IC2-NEXT: br i1 [[TMP3]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY1]], !llvm.loop [[LOOP5:![0-9]+]]
; VF2IC2: [[MIDDLE_BLOCK]]:
+; VF2IC2-NEXT: store i8 [[TMP2]], ptr [[DST2]], align 1
+; VF2IC2-NEXT: store i8 [[TMP2]], ptr [[DST]], align 1
; VF2IC2-NEXT: br label %[[EXIT:.*]]
; VF2IC2: [[EXIT]]:
; VF2IC2-NEXT: ret void
@@ -318,20 +318,21 @@ define void @narrow_scatter_with_uniform_addr_to_scalar_unroll(ptr noalias %src,
; VF4IC1-NEXT: [[TMP19:%.*]] = insertelement <4 x i16> [[TMP18]], i16 [[TMP25]], i64 2
; VF4IC1-NEXT: [[BROADCAST_SPLAT:%.*]] = insertelement <4 x i16> [[TMP19]], i16 [[TMP16]], i64 3
; VF4IC1-NEXT: [[TMP10:%.*]] = trunc <4 x i16> [[BROADCAST_SPLAT]] to <4 x i8>
-; VF4IC1-NEXT: [[TMP11:%.*]] = extractelement <4 x i8> [[TMP10]], i64 3
-; VF4IC1-NEXT: store i8 [[TMP11]], ptr [[DST2]], align 4
; VF4IC1-NEXT: [[TMP12:%.*]] = extractelement <4 x i8> [[TMP10]], i64 0
; VF4IC1-NEXT: store i8 [[TMP12]], ptr [[TMP5]], align 4
; VF4IC1-NEXT: [[TMP13:%.*]] = extractelement <4 x i8> [[TMP10]], i64 1
; VF4IC1-NEXT: store i8 [[TMP13]], ptr [[TMP6]], align 4
; VF4IC1-NEXT: [[TMP14:%.*]] = extractelement <4 x i8> [[TMP10]], i64 2
; VF4IC1-NEXT: store i8 [[TMP14]], ptr [[TMP7]], align 4
+; VF4IC1-NEXT: [[TMP11:%.*]] = extractelement <4 x i8> [[TMP10]], i64 3
; VF4IC1-NEXT: store i8 [[TMP11]], ptr [[TMP8]], align 4
; VF4IC1-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 4
; VF4IC1-NEXT: [[VEC_IND_NEXT]] = add <4 x i64> [[VEC_IND]], splat (i64 4)
; VF4IC1-NEXT: [[TMP15:%.*]] = icmp eq i64 [[INDEX_NEXT]], 256
; VF4IC1-NEXT: br i1 [[TMP15]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP6:![0-9]+]]
; VF4IC1: [[MIDDLE_BLOCK]]:
+; VF4IC1-NEXT: [[TMP27:%.*]] = extractelement <4 x i8> [[TMP10]], i64 3
+; VF4IC1-NEXT: store i8 [[TMP27]], ptr [[DST2]], align 4
; VF4IC1-NEXT: br label %[[EXIT:.*]]
; VF4IC1: [[EXIT]]:
; VF4IC1-NEXT: ret void
@@ -357,17 +358,17 @@ define void @narrow_scatter_with_uniform_addr_to_scalar_unroll(ptr noalias %src,
; VF2IC2-NEXT: [[TMP13:%.*]] = load i16, ptr [[TMP10]], align 4
; VF2IC2-NEXT: [[BROADCAST_SPLATINSERT1:%.*]] = insertelement <2 x i16> poison, i16 [[TMP13]], i64 0
; VF2IC2-NEXT: [[BROADCAST_SPLAT1:%.*]] = shufflevector <2 x i16> [[BROADCAST_SPLATINSERT1]], <2 x i16> poison, <2 x i32> zeroinitializer
-; VF2IC2-NEXT: [[TMP6:%.*]] = trunc <2 x i16> [[BROADCAST_SPLAT1]] to <2 x i8>
-; VF2IC2-NEXT: [[TMP7:%.*]] = extractelement <2 x i8> [[TMP6]], i64 1
-; VF2IC2-NEXT: store i8 [[TMP7]], ptr [[DST2]], align 4
; VF2IC2-NEXT: [[TMP11:%.*]] = trunc <2 x i16> [[BROADCAST_SPLAT]] to <2 x i8>
+; VF2IC2-NEXT: [[TMP14:%.*]] = trunc <2 x i16> [[BROADCAST_SPLAT1]] to <2 x i8>
; VF2IC2-NEXT: [[TMP12:%.*]] = extractelement <2 x i8> [[TMP11]], i64 1
+; VF2IC2-NEXT: [[TMP7:%.*]] = extractelement <2 x i8> [[TMP14]], i64 1
; VF2IC2-NEXT: store i8 [[TMP12]], ptr [[TMP3]], align 4
; VF2IC2-NEXT: store i8 [[TMP7]], ptr [[TMP4]], align 4
; VF2IC2-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 4
; VF2IC2-NEXT: [[TMP8:%.*]] = icmp eq i64 [[INDEX_NEXT]], 256
; VF2IC2-NEXT: br i1 [[TMP8]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP6:![0-9]+]]
; VF2IC2: [[MIDDLE_BLOCK]]:
+; VF2IC2-NEXT: store i8 [[TMP7]], ptr [[DST2]], align 4
; VF2IC2-NEXT: br label %[[EXIT:.*]]
; VF2IC2: [[EXIT]]:
; VF2IC2-NEXT: ret void
diff --git a/llvm/test/Transforms/LoopVectorize/vector-loop-backedge-elimination.ll b/llvm/test/Transforms/LoopVectorize/vector-loop-backedge-elimination.ll
index 17cca5100e9ec..b415b72279a12 100644
--- a/llvm/test/Transforms/LoopVectorize/vector-loop-backedge-elimination.ll
+++ b/llvm/test/Transforms/LoopVectorize/vector-loop-backedge-elimination.ll
@@ -1207,33 +1207,33 @@ exit:
define void @first_order_recurrence_single_vector_iteration(ptr noalias %pkt, ptr noalias %dst) {
; VF8UF1-LABEL: define void @first_order_recurrence_single_vector_iteration(
; VF8UF1-SAME: ptr noalias [[PKT:%.*]], ptr noalias [[DST:%.*]]) {
-; VF8UF1-NEXT: [[ENTRY:.*:]]
-; VF8UF1-NEXT: br label %[[VECTOR_PH:.*]]
-; VF8UF1: [[VECTOR_PH]]:
+; VF8UF1-NEXT: [[VECTOR_PH:.*:]]
; VF8UF1-NEXT: br label %[[VECTOR_BODY:.*]]
; VF8UF1: [[VECTOR_BODY]]:
; VF8UF1-NEXT: [[TMP0:%.*]] = load i8, ptr [[PKT]], align 1
; VF8UF1-NEXT: [[BROADCAST_SPLATINSERT:%.*]] = insertelement <8 x i8> poison, i8 [[TMP0]], i64 0
; VF8UF1-NEXT: [[BROADCAST_SPLAT:%.*]] = shufflevector <8 x i8> [[BROADCAST_SPLATINSERT]], <8 x i8> poison, <8 x i32> zeroinitializer
+; VF8UF1-NEXT: br label %[[VECTOR_BODY1:.*]]
+; VF8UF1: [[VECTOR_BODY1]]:
+; VF8UF1-NEXT: br label %[[MIDDLE_BLOCK:.*]]
+; VF8UF1: [[MIDDLE_BLOCK]]:
; VF8UF1-NEXT: [[TMP1:%.*]] = shufflevector <8 x i8> zeroinitializer, <8 x i8> [[BROADCAST_SPLAT]], <8 x i32> <i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14>
; VF8UF1-NEXT: [[TMP2:%.*]] = extractelement <8 x i8> [[TMP1]], i64 7
; VF8UF1-NEXT: store i8 [[TMP2]], ptr [[DST]], align 1
-; VF8UF1-NEXT: br label %[[MIDDLE_BLOCK:.*]]
-; VF8UF1: [[MIDDLE_BLOCK]]:
; VF8UF1-NEXT: br label %[[EXIT:.*]]
; VF8UF1: [[EXIT]]:
; VF8UF1-NEXT: ret void
;
; VF8UF2-LABEL: define void @first_order_recurrence_single_vector_iteration(
; VF8UF2-SAME: ptr noalias [[PKT:%.*]], ptr noalias [[DST:%.*]]) {
-; VF8UF2-NEXT: [[ENTRY:.*:]]
-; VF8UF2-NEXT: br label %[[VECTOR_PH:.*]]
-; VF8UF2: [[VECTOR_PH]]:
+; VF8UF2-NEXT: [[VECTOR_PH:.*:]]
; VF8UF2-NEXT: br label %[[VECTOR_BODY:.*]]
; VF8UF2: [[VECTOR_BODY]]:
; VF8UF2-NEXT: [[TMP0:%.*]] = load i8, ptr [[PKT]], align 1
; VF8UF2-NEXT: [[BROADCAST_SPLATINSERT:%.*]] = insertelement <8 x i8> poison, i8 [[TMP0]], i64 0
; VF8UF2-NEXT: [[BROADCAST_SPLAT:%.*]] = shufflevector <8 x i8> [[BROADCAST_SPLATINSERT]], <8 x i8> poison, <8 x i32> zeroinitializer
+; VF8UF2-NEXT: br label %[[VECTOR_BODY1:.*]]
+; VF8UF2: [[VECTOR_BODY1]]:
; VF8UF2-NEXT: [[TMP1:%.*]] = shufflevector <8 x i8> zeroinitializer, <8 x i8> [[BROADCAST_SPLAT]], <8 x i32> <i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14>
; VF8UF2-NEXT: [[TMP2:%.*]] = shufflevector <8 x i8> [[BROADCAST_SPLAT]], <8 x i8> [[BROADCAST_SPLAT]], <8 x i32> <i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14>
; VF8UF2-NEXT: br i1 true, label %[[PRED_STORE_IF:.*]], label %[[PRED_STORE_CONTINUE:.*]]
@@ -1340,14 +1340,14 @@ define void @first_order_recurrence_single_vector_iteration(ptr noalias %pkt, pt
;
; VF16UF1-LABEL: define void @first_order_recurrence_single_vector_iteration(
; VF16UF1-SAME: ptr noalias [[PKT:%.*]], ptr noalias [[DST:%.*]]) {
-; VF16UF1-NEXT: [[ENTRY:.*:]]
-; VF16UF1-NEXT: br label %[[VECTOR_PH:.*]]
-; VF16UF1: [[VECTOR_PH]]:
+; VF16UF1-NEXT: [[VECTOR_PH:.*:]]
; VF16UF1-NEXT: br label %[[VECTOR_BODY:.*]]
; VF16UF1: [[VECTOR_BODY]]:
; VF16UF1-NEXT: [[TMP0:%.*]] = load i8, ptr [[PKT]], align 1
; VF16UF1-NEXT: [[BROADCAST_SPLATINSERT:%.*]] = insertelement <16 x i8> poison, i8 [[TMP0]], i64 0
; VF16UF1-NEXT: [[BROADCAST_SPLAT:%.*]] = shufflevector <16 x i8> [[BROADCAST_SPLATINSERT]], <16 x i8> poison, <16 x i32> zeroinitializer
+; VF16UF1-NEXT: br label %[[VECTOR_BODY1:.*]]
+; VF16UF1: [[VECTOR_BODY1]]:
; VF16UF1-NEXT: [[TMP1:%.*]] = shufflevector <16 x i8> zeroinitializer, <16 x i8> [[BROADCAST_SPLAT]], <16 x i32> <i32 15, i32 16, i32 17, i32 18, i32 19, i32 20, i32 21, i32 22, i32 23, i32 24, i32 25, i32 26, i32 27, i32 28, i32 29, i32 30>
; VF16UF1-NEXT: br i1 true, label %[[PRED_STORE_IF:.*]], label %[[PRED_STORE_CONTINUE:.*]]
; VF16UF1: [[PRED_STORE_IF]]:
diff --git a/llvm/test/Transforms/LoopVectorize/widen-gep-all-indices-invariant.ll b/llvm/test/Transforms/LoopVectorize/widen-gep-all-indices-invariant.ll
index 6ce37889bb5c2..7d0320952942d 100644
--- a/llvm/test/Transforms/LoopVectorize/widen-gep-all-indices-invariant.ll
+++ b/llvm/test/Transforms/LoopVectorize/widen-gep-all-indices-invariant.ll
@@ -135,17 +135,17 @@ define void @pr173761(i8 %c, ptr %p, ptr noalias %q, ptr noalias %r) {
; CHECK-NEXT: [[BROADCAST_SPLAT:%.*]] = shufflevector <4 x i8> [[BROADCAST_SPLATINSERT]], <4 x i8> poison, <4 x i32> zeroinitializer
; CHECK-NEXT: [[BROADCAST_SPLATINSERT1:%.*]] = insertelement <4 x ptr> poison, ptr [[P]], i64 0
; CHECK-NEXT: [[BROADCAST_SPLAT2:%.*]] = shufflevector <4 x ptr> [[BROADCAST_SPLATINSERT1]], <4 x ptr> poison, <4 x i32> zeroinitializer
+; CHECK-NEXT: [[TMP0:%.*]] = load i8, ptr [[P]], align 1
; CHECK-NEXT: [[WIDE_GEP:%.*]] = getelementptr inbounds nuw i8, ptr [[P]], i64 16
; CHECK-NEXT: [[BROADCAST_SPLATINSERT3:%.*]] = insertelement <4 x ptr> poison, ptr [[WIDE_GEP]], i64 0
; CHECK-NEXT: [[BROADCAST_SPLAT4:%.*]] = shufflevector <4 x ptr> [[BROADCAST_SPLATINSERT3]], <4 x ptr> poison, <4 x i32> zeroinitializer
-; CHECK-NEXT: [[TMP0:%.*]] = trunc <4 x i8> [[BROADCAST_SPLAT]] to <4 x i1>
-; CHECK-NEXT: [[TMP1:%.*]] = select <4 x i1> [[TMP0]], <4 x ptr> [[BROADCAST_SPLAT4]], <4 x ptr> [[BROADCAST_SPLAT2]]
+; CHECK-NEXT: [[TMP1:%.*]] = trunc <4 x i8> [[BROADCAST_SPLAT]] to <4 x i1>
+; CHECK-NEXT: [[TMP2:%.*]] = select <4 x i1> [[TMP1]], <4 x ptr> [[BROADCAST_SPLAT4]], <4 x ptr> [[BROADCAST_SPLAT2]]
; CHECK-NEXT: br label [[VECTOR_BODY:%.*]]
; CHECK: vector.body:
; CHECK-NEXT: [[INDEX:%.*]] = phi i32 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ]
-; CHECK-NEXT: [[TMP2:%.*]] = load i8, ptr [[P]], align 1
-; CHECK-NEXT: store i8 [[TMP2]], ptr [[R]], align 1
-; CHECK-NEXT: [[TMP3:%.*]] = extractelement <4 x ptr> [[TMP1]], i64 3
+; CHECK-NEXT: store i8 [[TMP0]], ptr [[R]], align 1
+; CHECK-NEXT: [[TMP3:%.*]] = extractelement <4 x ptr> [[TMP2]], i64 3
; CHECK-NEXT: [[TMP4:%.*]] = load i8, ptr [[TMP3]], align 1
; CHECK-NEXT: store i8 [[TMP4]], ptr [[Q]], align 1
; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i32 [[INDEX]], 4
More information about the llvm-commits
mailing list