[llvm] [VPlan] Use VPlan-based costs for pointer inductions. (PR #223394)
Florian Hahn via llvm-commits
llvm-commits at lists.llvm.org
Mon Sep 14 12:57:17 PDT 2026
https://github.com/fhahn updated https://github.com/llvm/llvm-project/pull/223394
>From e9871d1e205725bd357a85a8df64d4a95673e5ae Mon Sep 17 00:00:00 2001
From: Florian Hahn <flo at fhahn.com>
Date: Thu, 27 Aug 2026 12:12:01 +0100
Subject: [PATCH] [VPlan] Use VPlan-based costs for pointer inductions.
Drop the last legacy induction fallback in precomputeCosts, completing the
migration of induction costs to be VPlan-based.
The fallback charged the cost of the IV phi. The cost of the GEP
incrementing the IV was considered free.
VPWidenPointerInduction::computeCost now charges the cost of the phi
plus a TODO to also consider cost of the increment and wide pointer add.
For scalarized pointer inductions, we do not charge any cost for the
scalar GEPs, like before, assuming they can be folded into the
addressing mode.
Test changes are due to the fact that scalarized pointer inductions do
not require an extra phi (on most platforms, phis are considered free,
so not many tests are impacted).
---
.../Transforms/Vectorize/LoopVectorize.cpp | 54 ---
llvm/lib/Transforms/Vectorize/VPlan.h | 4 +
.../lib/Transforms/Vectorize/VPlanRecipes.cpp | 9 +
.../AArch64/force-target-instruction-cost.ll | 445 +++---------------
.../AArch64/predication_costs.ll | 2 +-
.../LoopVectorize/ARM/mve-icmpcost.ll | 32 --
.../WebAssembly/memory-interleave.ll | 22 +-
.../X86/CostModel/vpinstruction-cost.ll | 4 -
.../LoopVectorize/induction-cost.ll | 58 +--
.../test/Transforms/LoopVectorize/phi-cost.ll | 4 +-
10 files changed, 117 insertions(+), 517 deletions(-)
diff --git a/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp b/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
index 04abdac5ee680..0f48f6a6f3c2e 100644
--- a/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
+++ b/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
@@ -5443,21 +5443,6 @@ InstructionCost
LoopVectorizationPlanner::precomputeCosts(VPlan &Plan, ElementCount VF,
VPCostContext &CostCtx) const {
InstructionCost Cost;
- // Cost modeling for inductions is inaccurate in the legacy cost model
- // compared to the recipes that are generated. To match here initially during
- // VPlan cost model bring up directly use the induction costs from the legacy
- // cost model. Note that we do this as pre-processing; the VPlan may not have
- // any recipes associated with the original induction increment instruction
- // and may replace truncates with VPWidenIntOrFpInductionRecipe. We precompute
- // the cost of induction phis and increments (both that are represented by
- // recipes and those that are not), to avoid distinguishing between them here,
- // and skip all recipes that represent induction phis and increments (the
- // former case) later on, if they exist, to avoid counting them twice.
- // Similarly we pre-compute the cost of any optimized truncates.
- // Inductions that are represented by a VPWidenIntOrFpInductionRecipe are an
- // exception: their cost is computed by the recipe's computeCost (see below),
- // so they are not precomputed here.
- // TODO: Switch to more accurate costing based on VPlan.
// If the vector loop gets executed exactly once with the given VF, ignore the
// costs of comparison and induction instructions, as they'll get simplified
@@ -5469,45 +5454,6 @@ LoopVectorizationPlanner::precomputeCosts(VPlan &Plan, ElementCount VF,
addFullyUnrolledInstructionsToIgnore(OrigLoop, Legal->getInductionVars(),
CostCtx.SkipCostComputation);
- for (const auto &[IV, IndDesc] : Legal->getInductionVars()) {
- // Integer and FP inductions are always costed via the VPlan-based cost
- // model.
- // TODO: Also migrate pointer inductions.
- if (IndDesc.getKind() == InductionDescriptor::IK_IntInduction ||
- IndDesc.getKind() == InductionDescriptor::IK_FpInduction)
- continue;
- Instruction *IVInc = cast<Instruction>(
- IV->getIncomingValueForBlock(OrigLoop->getLoopLatch()));
- SmallVector<Instruction *> IVInsts = {IVInc};
- for (unsigned I = 0; I != IVInsts.size(); I++) {
- for (Value *Op : IVInsts[I]->operands()) {
- auto *OpI = dyn_cast<Instruction>(Op);
- if (Op == IV || !OpI || !OrigLoop->contains(OpI) || !Op->hasOneUse())
- continue;
- IVInsts.push_back(OpI);
- }
- }
- IVInsts.push_back(IV);
- for (User *U : IV->users()) {
- auto *CI = cast<Instruction>(U);
- if (!CostCtx.CM.isOptimizableIVTruncate(CI, VF))
- continue;
- IVInsts.push_back(CI);
- }
-
- for (Instruction *IVInst : IVInsts) {
- if (CostCtx.skipCostComputation(IVInst, VF.isVector()))
- continue;
- InstructionCost InductionCost = CostCtx.getLegacyCost(IVInst, VF);
- LLVM_DEBUG({
- dbgs() << "Cost of " << InductionCost << " for VF " << VF
- << ": induction instruction " << *IVInst << "\n";
- });
- Cost += InductionCost;
- CostCtx.SkipCostComputation.insert(IVInst);
- }
- }
-
// Pre-compute the costs for branches except for the backedge, as the number
// of replicate regions in a VPlan may not directly match the number of
// branches, which would lead to different decisions.
diff --git a/llvm/lib/Transforms/Vectorize/VPlan.h b/llvm/lib/Transforms/Vectorize/VPlan.h
index a4b2596a025d0..daabca902724b 100644
--- a/llvm/lib/Transforms/Vectorize/VPlan.h
+++ b/llvm/lib/Transforms/Vectorize/VPlan.h
@@ -2730,6 +2730,10 @@ class VPWidenPointerInductionRecipe : public VPWidenInductionRecipe {
/// Returns true if only scalar values will be generated.
bool onlyScalarsGenerated(bool IsScalable);
+ /// Return the cost of this VPWidenPointerInductionRecipe.
+ InstructionCost computeCost(ElementCount VF,
+ VPCostContext &Ctx) const override;
+
protected:
#if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP)
/// Print the recipe.
diff --git a/llvm/lib/Transforms/Vectorize/VPlanRecipes.cpp b/llvm/lib/Transforms/Vectorize/VPlanRecipes.cpp
index 02c570d0437bb..5997b6e0fef76 100644
--- a/llvm/lib/Transforms/Vectorize/VPlanRecipes.cpp
+++ b/llvm/lib/Transforms/Vectorize/VPlanRecipes.cpp
@@ -4989,6 +4989,15 @@ InstructionCost VPInterleaveBase::computeCost(ElementCount VF,
0);
}
+InstructionCost
+VPWidenPointerInductionRecipe::computeCost(ElementCount VF,
+ VPCostContext &Ctx) const {
+ // The recipe creates a scalar phi, a GEP to increment the induction and
+ // vector add to compute the vector of pointers.
+ // TODO: Charge costs for induction increment and vector add as well.
+ return Ctx.TTI.getCFInstrCost(Instruction::PHI, Ctx.CostKind);
+}
+
bool VPWidenPointerInductionRecipe::onlyScalarsGenerated(bool IsScalable) {
return vputils::onlyScalarValuesUsed(this) &&
(!IsScalable || vputils::onlyFirstLaneUsed(this));
diff --git a/llvm/test/Transforms/LoopVectorize/AArch64/force-target-instruction-cost.ll b/llvm/test/Transforms/LoopVectorize/AArch64/force-target-instruction-cost.ll
index 618f349ed798f..081e2a18df04f 100644
--- a/llvm/test/Transforms/LoopVectorize/AArch64/force-target-instruction-cost.ll
+++ b/llvm/test/Transforms/LoopVectorize/AArch64/force-target-instruction-cost.ll
@@ -690,21 +690,26 @@ define void @force_branch_cost(ptr readonly %src, ptr %dst) {
; COST1-LABEL: define void @force_branch_cost(
; COST1-SAME: ptr readonly [[SRC:%.*]], ptr [[DST:%.*]]) {
; COST1-NEXT: [[ENTRY:.*:]]
-; COST1-NEXT: br i1 false, label %[[SCALAR_PH:.*]], label %[[VECTOR_MEMCHECK:.*]]
+; COST1-NEXT: br label %[[VECTOR_MEMCHECK:.*]]
; COST1: [[VECTOR_MEMCHECK]]:
; COST1-NEXT: [[SCEVGEP:%.*]] = getelementptr i8, ptr [[DST]], i64 2092
; COST1-NEXT: [[SCEVGEP1:%.*]] = getelementptr i8, ptr [[SRC]], i64 85
; COST1-NEXT: [[BOUND0:%.*]] = icmp ult ptr [[DST]], [[SCEVGEP1]]
; COST1-NEXT: [[BOUND1:%.*]] = icmp ult ptr [[SRC]], [[SCEVGEP]]
; COST1-NEXT: [[FOUND_CONFLICT:%.*]] = and i1 [[BOUND0]], [[BOUND1]]
-; COST1-NEXT: br i1 [[FOUND_CONFLICT]], label %[[SCALAR_PH]], label %[[VECTOR_PH:.*]]
+; COST1-NEXT: br i1 [[FOUND_CONFLICT]], label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]]
; COST1: [[VECTOR_PH]]:
-; COST1-NEXT: br i1 false, label %[[VEC_EPILOG_PH:.*]], label %[[VECTOR_PH1:.*]]
-; COST1: [[VECTOR_PH1]]:
; COST1-NEXT: [[TMP0:%.*]] = getelementptr i8, ptr [[DST]], i64 1536
; COST1-NEXT: br label %[[VECTOR_BODY:.*]]
; COST1: [[VECTOR_BODY]]:
-; COST1-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH1]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; COST1-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; COST1-NEXT: [[TMP5:%.*]] = add i64 [[INDEX]], 1
+; COST1-NEXT: [[TMP6:%.*]] = add i64 [[INDEX]], 2
+; COST1-NEXT: [[TMP7:%.*]] = add i64 [[INDEX]], 3
+; COST1-NEXT: [[TMP8:%.*]] = add i64 [[INDEX]], 4
+; COST1-NEXT: [[TMP9:%.*]] = add i64 [[INDEX]], 5
+; COST1-NEXT: [[TMP18:%.*]] = add i64 [[INDEX]], 6
+; COST1-NEXT: [[TMP19:%.*]] = add i64 [[INDEX]], 7
; COST1-NEXT: [[OFFSET_IDX:%.*]] = mul i64 [[INDEX]], 96
; COST1-NEXT: [[TMP10:%.*]] = add i64 [[OFFSET_IDX]], 96
; COST1-NEXT: [[TMP11:%.*]] = add i64 [[OFFSET_IDX]], 192
@@ -713,234 +718,107 @@ define void @force_branch_cost(ptr readonly %src, ptr %dst) {
; COST1-NEXT: [[TMP14:%.*]] = add i64 [[OFFSET_IDX]], 480
; COST1-NEXT: [[TMP15:%.*]] = add i64 [[OFFSET_IDX]], 576
; COST1-NEXT: [[TMP16:%.*]] = add i64 [[OFFSET_IDX]], 672
-; COST1-NEXT: [[TMP9:%.*]] = add i64 [[OFFSET_IDX]], 768
-; COST1-NEXT: [[TMP85:%.*]] = add i64 [[OFFSET_IDX]], 864
-; COST1-NEXT: [[TMP86:%.*]] = add i64 [[OFFSET_IDX]], 960
-; COST1-NEXT: [[TMP87:%.*]] = add i64 [[OFFSET_IDX]], 1056
-; COST1-NEXT: [[TMP92:%.*]] = add i64 [[OFFSET_IDX]], 1152
-; COST1-NEXT: [[TMP93:%.*]] = add i64 [[OFFSET_IDX]], 1248
-; COST1-NEXT: [[TMP94:%.*]] = add i64 [[OFFSET_IDX]], 1344
-; COST1-NEXT: [[TMP95:%.*]] = add i64 [[OFFSET_IDX]], 1440
-; COST1-NEXT: [[NEXT_GEP:%.*]] = getelementptr i8, ptr [[DST]], i64 [[OFFSET_IDX]]
-; COST1-NEXT: [[NEXT_GEP2:%.*]] = getelementptr i8, ptr [[DST]], i64 [[TMP10]]
-; COST1-NEXT: [[NEXT_GEP3:%.*]] = getelementptr i8, ptr [[DST]], i64 [[TMP11]]
-; COST1-NEXT: [[NEXT_GEP4:%.*]] = getelementptr i8, ptr [[DST]], i64 [[TMP12]]
+; COST1-NEXT: [[NEXT_GEP18:%.*]] = getelementptr i8, ptr [[DST]], i64 [[OFFSET_IDX]]
+; COST1-NEXT: [[NEXT_GEP19:%.*]] = getelementptr i8, ptr [[DST]], i64 [[TMP10]]
+; COST1-NEXT: [[NEXT_GEP20:%.*]] = getelementptr i8, ptr [[DST]], i64 [[TMP11]]
+; COST1-NEXT: [[NEXT_GEP21:%.*]] = getelementptr i8, ptr [[DST]], i64 [[TMP12]]
; COST1-NEXT: [[NEXT_GEP5:%.*]] = getelementptr i8, ptr [[DST]], i64 [[TMP13]]
; COST1-NEXT: [[NEXT_GEP6:%.*]] = getelementptr i8, ptr [[DST]], i64 [[TMP14]]
; COST1-NEXT: [[NEXT_GEP7:%.*]] = getelementptr i8, ptr [[DST]], i64 [[TMP15]]
; COST1-NEXT: [[NEXT_GEP8:%.*]] = getelementptr i8, ptr [[DST]], i64 [[TMP16]]
-; COST1-NEXT: [[NEXT_GEP9:%.*]] = getelementptr i8, ptr [[DST]], i64 [[TMP9]]
-; COST1-NEXT: [[NEXT_GEP10:%.*]] = getelementptr i8, ptr [[DST]], i64 [[TMP85]]
-; COST1-NEXT: [[NEXT_GEP11:%.*]] = getelementptr i8, ptr [[DST]], i64 [[TMP86]]
-; COST1-NEXT: [[NEXT_GEP12:%.*]] = getelementptr i8, ptr [[DST]], i64 [[TMP87]]
-; COST1-NEXT: [[NEXT_GEP13:%.*]] = getelementptr i8, ptr [[DST]], i64 [[TMP92]]
-; COST1-NEXT: [[NEXT_GEP14:%.*]] = getelementptr i8, ptr [[DST]], i64 [[TMP93]]
-; COST1-NEXT: [[NEXT_GEP15:%.*]] = getelementptr i8, ptr [[DST]], i64 [[TMP94]]
-; COST1-NEXT: [[NEXT_GEP16:%.*]] = getelementptr i8, ptr [[DST]], i64 [[TMP95]]
; COST1-NEXT: [[TMP17:%.*]] = getelementptr [4 x i8], ptr [[SRC]], i64 [[INDEX]]
-; COST1-NEXT: [[WIDE_VEC:%.*]] = load <64 x i8>, ptr [[TMP17]], align 1, !alias.scope [[META19:![0-9]+]]
-; COST1-NEXT: [[STRIDED_VEC:%.*]] = shufflevector <64 x i8> [[WIDE_VEC]], <64 x i8> poison, <16 x i32> <i32 0, i32 4, i32 8, i32 12, i32 16, i32 20, i32 24, i32 28, i32 32, i32 36, i32 40, i32 44, i32 48, i32 52, i32 56, i32 60>
-; COST1-NEXT: [[TMP18:%.*]] = zext <16 x i8> [[STRIDED_VEC]] to <16 x i32>
-; COST1-NEXT: [[TMP19:%.*]] = extractelement <16 x i32> [[TMP18]], i64 0
-; COST1-NEXT: store i32 [[TMP19]], ptr [[NEXT_GEP]], align 4, !alias.scope [[META22:![0-9]+]], !noalias [[META19]]
-; COST1-NEXT: [[TMP20:%.*]] = extractelement <16 x i32> [[TMP18]], i64 1
-; COST1-NEXT: store i32 [[TMP20]], ptr [[NEXT_GEP2]], align 4, !alias.scope [[META22]], !noalias [[META19]]
-; COST1-NEXT: [[TMP96:%.*]] = extractelement <16 x i32> [[TMP18]], i64 2
-; COST1-NEXT: store i32 [[TMP96]], ptr [[NEXT_GEP3]], align 4, !alias.scope [[META22]], !noalias [[META19]]
-; COST1-NEXT: [[TMP97:%.*]] = extractelement <16 x i32> [[TMP18]], i64 3
-; COST1-NEXT: store i32 [[TMP97]], ptr [[NEXT_GEP4]], align 4, !alias.scope [[META22]], !noalias [[META19]]
-; COST1-NEXT: [[TMP98:%.*]] = extractelement <16 x i32> [[TMP18]], i64 4
-; COST1-NEXT: store i32 [[TMP98]], ptr [[NEXT_GEP5]], align 4, !alias.scope [[META22]], !noalias [[META19]]
-; COST1-NEXT: [[TMP99:%.*]] = extractelement <16 x i32> [[TMP18]], i64 5
-; COST1-NEXT: store i32 [[TMP99]], ptr [[NEXT_GEP6]], align 4, !alias.scope [[META22]], !noalias [[META19]]
-; COST1-NEXT: [[TMP25:%.*]] = extractelement <16 x i32> [[TMP18]], i64 6
-; COST1-NEXT: store i32 [[TMP25]], ptr [[NEXT_GEP7]], align 4, !alias.scope [[META22]], !noalias [[META19]]
-; COST1-NEXT: [[TMP26:%.*]] = extractelement <16 x i32> [[TMP18]], i64 7
-; COST1-NEXT: store i32 [[TMP26]], ptr [[NEXT_GEP8]], align 4, !alias.scope [[META22]], !noalias [[META19]]
-; COST1-NEXT: [[TMP27:%.*]] = extractelement <16 x i32> [[TMP18]], i64 8
-; COST1-NEXT: store i32 [[TMP27]], ptr [[NEXT_GEP9]], align 4, !alias.scope [[META22]], !noalias [[META19]]
-; COST1-NEXT: [[TMP28:%.*]] = extractelement <16 x i32> [[TMP18]], i64 9
-; COST1-NEXT: store i32 [[TMP28]], ptr [[NEXT_GEP10]], align 4, !alias.scope [[META22]], !noalias [[META19]]
-; COST1-NEXT: [[TMP29:%.*]] = extractelement <16 x i32> [[TMP18]], i64 10
-; COST1-NEXT: store i32 [[TMP29]], ptr [[NEXT_GEP11]], align 4, !alias.scope [[META22]], !noalias [[META19]]
-; COST1-NEXT: [[TMP30:%.*]] = extractelement <16 x i32> [[TMP18]], i64 11
-; COST1-NEXT: store i32 [[TMP30]], ptr [[NEXT_GEP12]], align 4, !alias.scope [[META22]], !noalias [[META19]]
-; COST1-NEXT: [[TMP31:%.*]] = extractelement <16 x i32> [[TMP18]], i64 12
-; COST1-NEXT: store i32 [[TMP31]], ptr [[NEXT_GEP13]], align 4, !alias.scope [[META22]], !noalias [[META19]]
-; COST1-NEXT: [[TMP32:%.*]] = extractelement <16 x i32> [[TMP18]], i64 13
-; COST1-NEXT: store i32 [[TMP32]], ptr [[NEXT_GEP14]], align 4, !alias.scope [[META22]], !noalias [[META19]]
-; COST1-NEXT: [[TMP100:%.*]] = extractelement <16 x i32> [[TMP18]], i64 14
-; COST1-NEXT: store i32 [[TMP100]], ptr [[NEXT_GEP15]], align 4, !alias.scope [[META22]], !noalias [[META19]]
-; COST1-NEXT: [[TMP101:%.*]] = extractelement <16 x i32> [[TMP18]], i64 15
-; COST1-NEXT: store i32 [[TMP101]], ptr [[NEXT_GEP16]], align 4, !alias.scope [[META22]], !noalias [[META19]]
-; COST1-NEXT: [[TMP102:%.*]] = getelementptr i8, ptr [[NEXT_GEP]], i64 24
-; COST1-NEXT: [[TMP103:%.*]] = getelementptr i8, ptr [[NEXT_GEP2]], i64 24
-; COST1-NEXT: [[TMP104:%.*]] = getelementptr i8, ptr [[NEXT_GEP3]], i64 24
-; COST1-NEXT: [[TMP105:%.*]] = getelementptr i8, ptr [[NEXT_GEP4]], i64 24
-; COST1-NEXT: [[TMP106:%.*]] = getelementptr i8, ptr [[NEXT_GEP5]], i64 24
-; COST1-NEXT: [[TMP107:%.*]] = getelementptr i8, ptr [[NEXT_GEP6]], i64 24
-; COST1-NEXT: [[TMP41:%.*]] = getelementptr i8, ptr [[NEXT_GEP7]], i64 24
-; COST1-NEXT: [[TMP42:%.*]] = getelementptr i8, ptr [[NEXT_GEP8]], i64 24
-; COST1-NEXT: [[TMP43:%.*]] = getelementptr i8, ptr [[NEXT_GEP9]], i64 24
-; COST1-NEXT: [[TMP44:%.*]] = getelementptr i8, ptr [[NEXT_GEP10]], i64 24
-; COST1-NEXT: [[TMP108:%.*]] = getelementptr i8, ptr [[NEXT_GEP11]], i64 24
-; COST1-NEXT: [[TMP122:%.*]] = getelementptr i8, ptr [[NEXT_GEP12]], i64 24
-; COST1-NEXT: [[TMP123:%.*]] = getelementptr i8, ptr [[NEXT_GEP13]], i64 24
-; COST1-NEXT: [[TMP124:%.*]] = getelementptr i8, ptr [[NEXT_GEP14]], i64 24
-; COST1-NEXT: [[TMP49:%.*]] = getelementptr i8, ptr [[NEXT_GEP15]], i64 24
-; COST1-NEXT: [[TMP125:%.*]] = getelementptr i8, ptr [[NEXT_GEP16]], i64 24
-; COST1-NEXT: store i32 [[TMP19]], ptr [[TMP102]], align 4, !alias.scope [[META22]], !noalias [[META19]]
-; COST1-NEXT: store i32 [[TMP20]], ptr [[TMP103]], align 4, !alias.scope [[META22]], !noalias [[META19]]
-; COST1-NEXT: store i32 [[TMP96]], ptr [[TMP104]], align 4, !alias.scope [[META22]], !noalias [[META19]]
-; COST1-NEXT: store i32 [[TMP97]], ptr [[TMP105]], align 4, !alias.scope [[META22]], !noalias [[META19]]
-; COST1-NEXT: store i32 [[TMP98]], ptr [[TMP106]], align 4, !alias.scope [[META22]], !noalias [[META19]]
-; COST1-NEXT: store i32 [[TMP99]], ptr [[TMP107]], align 4, !alias.scope [[META22]], !noalias [[META19]]
-; COST1-NEXT: store i32 [[TMP25]], ptr [[TMP41]], align 4, !alias.scope [[META22]], !noalias [[META19]]
-; COST1-NEXT: store i32 [[TMP26]], ptr [[TMP42]], align 4, !alias.scope [[META22]], !noalias [[META19]]
-; COST1-NEXT: store i32 [[TMP27]], ptr [[TMP43]], align 4, !alias.scope [[META22]], !noalias [[META19]]
-; COST1-NEXT: store i32 [[TMP28]], ptr [[TMP44]], align 4, !alias.scope [[META22]], !noalias [[META19]]
-; COST1-NEXT: store i32 [[TMP29]], ptr [[TMP108]], align 4, !alias.scope [[META22]], !noalias [[META19]]
-; COST1-NEXT: store i32 [[TMP30]], ptr [[TMP122]], align 4, !alias.scope [[META22]], !noalias [[META19]]
-; COST1-NEXT: store i32 [[TMP31]], ptr [[TMP123]], align 4, !alias.scope [[META22]], !noalias [[META19]]
-; COST1-NEXT: store i32 [[TMP32]], ptr [[TMP124]], align 4, !alias.scope [[META22]], !noalias [[META19]]
-; COST1-NEXT: store i32 [[TMP100]], ptr [[TMP49]], align 4, !alias.scope [[META22]], !noalias [[META19]]
-; COST1-NEXT: store i32 [[TMP101]], ptr [[TMP125]], align 4, !alias.scope [[META22]], !noalias [[META19]]
-; COST1-NEXT: [[TMP51:%.*]] = getelementptr i8, ptr [[NEXT_GEP]], i64 48
-; COST1-NEXT: [[TMP52:%.*]] = getelementptr i8, ptr [[NEXT_GEP2]], i64 48
-; COST1-NEXT: [[TMP53:%.*]] = getelementptr i8, ptr [[NEXT_GEP3]], i64 48
-; COST1-NEXT: [[TMP54:%.*]] = getelementptr i8, ptr [[NEXT_GEP4]], i64 48
-; COST1-NEXT: [[TMP55:%.*]] = getelementptr i8, ptr [[NEXT_GEP5]], i64 48
-; COST1-NEXT: [[TMP56:%.*]] = getelementptr i8, ptr [[NEXT_GEP6]], i64 48
-; COST1-NEXT: [[TMP57:%.*]] = getelementptr i8, ptr [[NEXT_GEP7]], i64 48
-; COST1-NEXT: [[TMP58:%.*]] = getelementptr i8, ptr [[NEXT_GEP8]], i64 48
-; COST1-NEXT: [[TMP59:%.*]] = getelementptr i8, ptr [[NEXT_GEP9]], i64 48
-; COST1-NEXT: [[TMP60:%.*]] = getelementptr i8, ptr [[NEXT_GEP10]], i64 48
-; COST1-NEXT: [[TMP61:%.*]] = getelementptr i8, ptr [[NEXT_GEP11]], i64 48
-; COST1-NEXT: [[TMP62:%.*]] = getelementptr i8, ptr [[NEXT_GEP12]], i64 48
-; COST1-NEXT: [[TMP63:%.*]] = getelementptr i8, ptr [[NEXT_GEP13]], i64 48
-; COST1-NEXT: [[TMP64:%.*]] = getelementptr i8, ptr [[NEXT_GEP14]], i64 48
-; COST1-NEXT: [[TMP65:%.*]] = getelementptr i8, ptr [[NEXT_GEP15]], i64 48
-; COST1-NEXT: [[TMP66:%.*]] = getelementptr i8, ptr [[NEXT_GEP16]], i64 48
-; COST1-NEXT: store i32 [[TMP19]], ptr [[TMP51]], align 4, !alias.scope [[META22]], !noalias [[META19]]
-; COST1-NEXT: store i32 [[TMP20]], ptr [[TMP52]], align 4, !alias.scope [[META22]], !noalias [[META19]]
-; COST1-NEXT: store i32 [[TMP96]], ptr [[TMP53]], align 4, !alias.scope [[META22]], !noalias [[META19]]
-; COST1-NEXT: store i32 [[TMP97]], ptr [[TMP54]], align 4, !alias.scope [[META22]], !noalias [[META19]]
-; COST1-NEXT: store i32 [[TMP98]], ptr [[TMP55]], align 4, !alias.scope [[META22]], !noalias [[META19]]
-; COST1-NEXT: store i32 [[TMP99]], ptr [[TMP56]], align 4, !alias.scope [[META22]], !noalias [[META19]]
-; COST1-NEXT: store i32 [[TMP25]], ptr [[TMP57]], align 4, !alias.scope [[META22]], !noalias [[META19]]
-; COST1-NEXT: store i32 [[TMP26]], ptr [[TMP58]], align 4, !alias.scope [[META22]], !noalias [[META19]]
-; COST1-NEXT: store i32 [[TMP27]], ptr [[TMP59]], align 4, !alias.scope [[META22]], !noalias [[META19]]
-; COST1-NEXT: store i32 [[TMP28]], ptr [[TMP60]], align 4, !alias.scope [[META22]], !noalias [[META19]]
-; COST1-NEXT: store i32 [[TMP29]], ptr [[TMP61]], align 4, !alias.scope [[META22]], !noalias [[META19]]
-; COST1-NEXT: store i32 [[TMP30]], ptr [[TMP62]], align 4, !alias.scope [[META22]], !noalias [[META19]]
-; COST1-NEXT: store i32 [[TMP31]], ptr [[TMP63]], align 4, !alias.scope [[META22]], !noalias [[META19]]
-; COST1-NEXT: store i32 [[TMP32]], ptr [[TMP64]], align 4, !alias.scope [[META22]], !noalias [[META19]]
-; COST1-NEXT: store i32 [[TMP100]], ptr [[TMP65]], align 4, !alias.scope [[META22]], !noalias [[META19]]
-; COST1-NEXT: store i32 [[TMP101]], ptr [[TMP66]], align 4, !alias.scope [[META22]], !noalias [[META19]]
-; COST1-NEXT: [[TMP67:%.*]] = getelementptr i8, ptr [[NEXT_GEP]], i64 72
-; COST1-NEXT: [[TMP68:%.*]] = getelementptr i8, ptr [[NEXT_GEP2]], i64 72
-; COST1-NEXT: [[TMP69:%.*]] = getelementptr i8, ptr [[NEXT_GEP3]], i64 72
-; COST1-NEXT: [[TMP70:%.*]] = getelementptr i8, ptr [[NEXT_GEP4]], i64 72
-; COST1-NEXT: [[TMP71:%.*]] = getelementptr i8, ptr [[NEXT_GEP5]], i64 72
-; COST1-NEXT: [[TMP72:%.*]] = getelementptr i8, ptr [[NEXT_GEP6]], i64 72
-; COST1-NEXT: [[TMP73:%.*]] = getelementptr i8, ptr [[NEXT_GEP7]], i64 72
-; COST1-NEXT: [[TMP74:%.*]] = getelementptr i8, ptr [[NEXT_GEP8]], i64 72
-; COST1-NEXT: [[TMP75:%.*]] = getelementptr i8, ptr [[NEXT_GEP9]], i64 72
-; COST1-NEXT: [[TMP76:%.*]] = getelementptr i8, ptr [[NEXT_GEP10]], i64 72
-; COST1-NEXT: [[TMP77:%.*]] = getelementptr i8, ptr [[NEXT_GEP11]], i64 72
-; COST1-NEXT: [[TMP78:%.*]] = getelementptr i8, ptr [[NEXT_GEP12]], i64 72
-; COST1-NEXT: [[TMP79:%.*]] = getelementptr i8, ptr [[NEXT_GEP13]], i64 72
-; COST1-NEXT: [[TMP80:%.*]] = getelementptr i8, ptr [[NEXT_GEP14]], i64 72
-; COST1-NEXT: [[TMP81:%.*]] = getelementptr i8, ptr [[NEXT_GEP15]], i64 72
-; COST1-NEXT: [[TMP82:%.*]] = getelementptr i8, ptr [[NEXT_GEP16]], i64 72
-; COST1-NEXT: store i32 [[TMP19]], ptr [[TMP67]], align 4, !alias.scope [[META22]], !noalias [[META19]]
-; COST1-NEXT: store i32 [[TMP20]], ptr [[TMP68]], align 4, !alias.scope [[META22]], !noalias [[META19]]
-; COST1-NEXT: store i32 [[TMP96]], ptr [[TMP69]], align 4, !alias.scope [[META22]], !noalias [[META19]]
-; COST1-NEXT: store i32 [[TMP97]], ptr [[TMP70]], align 4, !alias.scope [[META22]], !noalias [[META19]]
-; COST1-NEXT: store i32 [[TMP98]], ptr [[TMP71]], align 4, !alias.scope [[META22]], !noalias [[META19]]
-; COST1-NEXT: store i32 [[TMP99]], ptr [[TMP72]], align 4, !alias.scope [[META22]], !noalias [[META19]]
-; COST1-NEXT: store i32 [[TMP25]], ptr [[TMP73]], align 4, !alias.scope [[META22]], !noalias [[META19]]
-; COST1-NEXT: store i32 [[TMP26]], ptr [[TMP74]], align 4, !alias.scope [[META22]], !noalias [[META19]]
-; COST1-NEXT: store i32 [[TMP27]], ptr [[TMP75]], align 4, !alias.scope [[META22]], !noalias [[META19]]
-; COST1-NEXT: store i32 [[TMP28]], ptr [[TMP76]], align 4, !alias.scope [[META22]], !noalias [[META19]]
-; COST1-NEXT: store i32 [[TMP29]], ptr [[TMP77]], align 4, !alias.scope [[META22]], !noalias [[META19]]
-; COST1-NEXT: store i32 [[TMP30]], ptr [[TMP78]], align 4, !alias.scope [[META22]], !noalias [[META19]]
-; COST1-NEXT: store i32 [[TMP31]], ptr [[TMP79]], align 4, !alias.scope [[META22]], !noalias [[META19]]
-; COST1-NEXT: store i32 [[TMP32]], ptr [[TMP80]], align 4, !alias.scope [[META22]], !noalias [[META19]]
-; COST1-NEXT: store i32 [[TMP100]], ptr [[TMP81]], align 4, !alias.scope [[META22]], !noalias [[META19]]
-; COST1-NEXT: store i32 [[TMP101]], ptr [[TMP82]], align 4, !alias.scope [[META22]], !noalias [[META19]]
-; COST1-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 16
-; COST1-NEXT: [[TMP83:%.*]] = icmp eq i64 [[INDEX_NEXT]], 16
-; COST1-NEXT: br i1 [[TMP83]], label %[[MIDDLE_BLOCK1:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP24:![0-9]+]]
-; COST1: [[MIDDLE_BLOCK1]]:
-; COST1-NEXT: br label %[[VEC_EPILOG_ITER_CHECK:.*]]
-; COST1: [[VEC_EPILOG_ITER_CHECK]]:
-; COST1-NEXT: br i1 false, label %[[SCALAR_PH]], label %[[VEC_EPILOG_PH]], !prof [[PROF25:![0-9]+]]
-; COST1: [[VEC_EPILOG_PH]]:
-; COST1-NEXT: [[VEC_EPILOG_RESUME_VAL:%.*]] = phi i64 [ 16, %[[VEC_EPILOG_ITER_CHECK]] ], [ 0, %[[VECTOR_PH]] ]
-; COST1-NEXT: [[TMP84:%.*]] = getelementptr i8, ptr [[DST]], i64 1920
-; COST1-NEXT: br label %[[VEC_EPILOG_VECTOR_BODY:.*]]
-; COST1: [[VEC_EPILOG_VECTOR_BODY]]:
-; COST1-NEXT: [[TMP5:%.*]] = phi i64 [ [[VEC_EPILOG_RESUME_VAL]], %[[VEC_EPILOG_PH]] ], [ [[INDEX_NEXT22:%.*]], %[[VEC_EPILOG_VECTOR_BODY]] ]
-; COST1-NEXT: [[TMP6:%.*]] = add i64 [[TMP5]], 1
-; COST1-NEXT: [[TMP7:%.*]] = add i64 [[TMP5]], 2
-; COST1-NEXT: [[TMP8:%.*]] = add i64 [[TMP5]], 3
-; COST1-NEXT: [[TMP88:%.*]] = mul i64 [[TMP5]], 96
-; COST1-NEXT: [[TMP89:%.*]] = add i64 [[TMP88]], 96
-; COST1-NEXT: [[TMP90:%.*]] = add i64 [[TMP88]], 192
-; COST1-NEXT: [[TMP91:%.*]] = add i64 [[TMP88]], 288
-; COST1-NEXT: [[NEXT_GEP18:%.*]] = getelementptr i8, ptr [[DST]], i64 [[TMP88]]
-; COST1-NEXT: [[NEXT_GEP19:%.*]] = getelementptr i8, ptr [[DST]], i64 [[TMP89]]
-; COST1-NEXT: [[NEXT_GEP20:%.*]] = getelementptr i8, ptr [[DST]], i64 [[TMP90]]
-; COST1-NEXT: [[NEXT_GEP21:%.*]] = getelementptr i8, ptr [[DST]], i64 [[TMP91]]
; COST1-NEXT: [[TMP21:%.*]] = getelementptr [4 x i8], ptr [[SRC]], i64 [[TMP5]]
; COST1-NEXT: [[TMP22:%.*]] = getelementptr [4 x i8], ptr [[SRC]], i64 [[TMP6]]
; COST1-NEXT: [[TMP23:%.*]] = getelementptr [4 x i8], ptr [[SRC]], i64 [[TMP7]]
; COST1-NEXT: [[TMP24:%.*]] = getelementptr [4 x i8], ptr [[SRC]], i64 [[TMP8]]
-; COST1-NEXT: [[TMP33:%.*]] = load i8, ptr [[TMP21]], align 1, !alias.scope [[META19]]
-; COST1-NEXT: [[TMP34:%.*]] = load i8, ptr [[TMP22]], align 1, !alias.scope [[META19]]
-; COST1-NEXT: [[TMP35:%.*]] = load i8, ptr [[TMP23]], align 1, !alias.scope [[META19]]
-; COST1-NEXT: [[TMP36:%.*]] = load i8, ptr [[TMP24]], align 1, !alias.scope [[META19]]
+; COST1-NEXT: [[TMP25:%.*]] = getelementptr [4 x i8], ptr [[SRC]], i64 [[TMP9]]
+; COST1-NEXT: [[TMP26:%.*]] = getelementptr [4 x i8], ptr [[SRC]], i64 [[TMP18]]
+; COST1-NEXT: [[TMP27:%.*]] = getelementptr [4 x i8], ptr [[SRC]], i64 [[TMP19]]
+; COST1-NEXT: [[TMP33:%.*]] = load i8, ptr [[TMP17]], align 1, !alias.scope [[META19:![0-9]+]]
+; COST1-NEXT: [[TMP34:%.*]] = load i8, ptr [[TMP21]], align 1, !alias.scope [[META19]]
+; COST1-NEXT: [[TMP35:%.*]] = load i8, ptr [[TMP22]], align 1, !alias.scope [[META19]]
+; COST1-NEXT: [[TMP36:%.*]] = load i8, ptr [[TMP23]], align 1, !alias.scope [[META19]]
; COST1-NEXT: [[TMP37:%.*]] = insertelement <4 x i8> poison, i8 [[TMP33]], i64 0
; COST1-NEXT: [[TMP38:%.*]] = insertelement <4 x i8> [[TMP37]], i8 [[TMP34]], i64 1
; COST1-NEXT: [[TMP39:%.*]] = insertelement <4 x i8> [[TMP38]], i8 [[TMP35]], i64 2
; COST1-NEXT: [[TMP40:%.*]] = insertelement <4 x i8> [[TMP39]], i8 [[TMP36]], i64 3
+; COST1-NEXT: [[TMP32:%.*]] = load i8, ptr [[TMP24]], align 1, !alias.scope [[META19]]
+; COST1-NEXT: [[TMP42:%.*]] = load i8, ptr [[TMP25]], align 1, !alias.scope [[META19]]
+; COST1-NEXT: [[TMP43:%.*]] = load i8, ptr [[TMP26]], align 1, !alias.scope [[META19]]
+; COST1-NEXT: [[TMP44:%.*]] = load i8, ptr [[TMP27]], align 1, !alias.scope [[META19]]
+; COST1-NEXT: [[TMP51:%.*]] = insertelement <4 x i8> poison, i8 [[TMP32]], i64 0
+; COST1-NEXT: [[TMP52:%.*]] = insertelement <4 x i8> [[TMP51]], i8 [[TMP42]], i64 1
+; COST1-NEXT: [[TMP53:%.*]] = insertelement <4 x i8> [[TMP52]], i8 [[TMP43]], i64 2
+; COST1-NEXT: [[TMP58:%.*]] = insertelement <4 x i8> [[TMP53]], i8 [[TMP44]], i64 3
; COST1-NEXT: [[TMP46:%.*]] = zext <4 x i8> [[TMP40]] to <4 x i32>
+; COST1-NEXT: [[TMP41:%.*]] = zext <4 x i8> [[TMP58]] to <4 x i32>
; COST1-NEXT: [[TMP45:%.*]] = extractelement <4 x i32> [[TMP46]], i64 0
-; COST1-NEXT: store i32 [[TMP45]], ptr [[NEXT_GEP18]], align 4, !alias.scope [[META22]], !noalias [[META19]]
+; COST1-NEXT: store i32 [[TMP45]], ptr [[NEXT_GEP18]], align 4, !alias.scope [[META22:![0-9]+]], !noalias [[META19]]
; COST1-NEXT: [[TMP50:%.*]] = extractelement <4 x i32> [[TMP46]], i64 1
; COST1-NEXT: store i32 [[TMP50]], ptr [[NEXT_GEP19]], align 4, !alias.scope [[META22]], !noalias [[META19]]
; COST1-NEXT: [[TMP47:%.*]] = extractelement <4 x i32> [[TMP46]], i64 2
; COST1-NEXT: store i32 [[TMP47]], ptr [[NEXT_GEP20]], align 4, !alias.scope [[META22]], !noalias [[META19]]
; COST1-NEXT: [[TMP48:%.*]] = extractelement <4 x i32> [[TMP46]], i64 3
; COST1-NEXT: store i32 [[TMP48]], ptr [[NEXT_GEP21]], align 4, !alias.scope [[META22]], !noalias [[META19]]
+; COST1-NEXT: [[TMP59:%.*]] = extractelement <4 x i32> [[TMP41]], i64 0
+; COST1-NEXT: store i32 [[TMP59]], ptr [[NEXT_GEP5]], align 4, !alias.scope [[META22]], !noalias [[META19]]
+; COST1-NEXT: [[TMP60:%.*]] = extractelement <4 x i32> [[TMP41]], i64 1
+; COST1-NEXT: store i32 [[TMP60]], ptr [[NEXT_GEP6]], align 4, !alias.scope [[META22]], !noalias [[META19]]
+; COST1-NEXT: [[TMP61:%.*]] = extractelement <4 x i32> [[TMP41]], i64 2
+; COST1-NEXT: store i32 [[TMP61]], ptr [[NEXT_GEP7]], align 4, !alias.scope [[META22]], !noalias [[META19]]
+; COST1-NEXT: [[TMP49:%.*]] = extractelement <4 x i32> [[TMP41]], i64 3
+; COST1-NEXT: store i32 [[TMP49]], ptr [[NEXT_GEP8]], align 4, !alias.scope [[META22]], !noalias [[META19]]
; COST1-NEXT: [[TMP109:%.*]] = getelementptr i8, ptr [[NEXT_GEP18]], i64 24
; COST1-NEXT: [[TMP110:%.*]] = getelementptr i8, ptr [[NEXT_GEP19]], i64 24
; COST1-NEXT: [[TMP111:%.*]] = getelementptr i8, ptr [[NEXT_GEP20]], i64 24
; COST1-NEXT: [[TMP112:%.*]] = getelementptr i8, ptr [[NEXT_GEP21]], i64 24
+; COST1-NEXT: [[TMP54:%.*]] = getelementptr i8, ptr [[NEXT_GEP5]], i64 24
+; COST1-NEXT: [[TMP55:%.*]] = getelementptr i8, ptr [[NEXT_GEP6]], i64 24
+; COST1-NEXT: [[TMP56:%.*]] = getelementptr i8, ptr [[NEXT_GEP7]], i64 24
+; COST1-NEXT: [[TMP57:%.*]] = getelementptr i8, ptr [[NEXT_GEP8]], i64 24
; COST1-NEXT: store i32 [[TMP45]], ptr [[TMP109]], align 4, !alias.scope [[META22]], !noalias [[META19]]
; COST1-NEXT: store i32 [[TMP50]], ptr [[TMP110]], align 4, !alias.scope [[META22]], !noalias [[META19]]
; COST1-NEXT: store i32 [[TMP47]], ptr [[TMP111]], align 4, !alias.scope [[META22]], !noalias [[META19]]
; COST1-NEXT: store i32 [[TMP48]], ptr [[TMP112]], align 4, !alias.scope [[META22]], !noalias [[META19]]
+; COST1-NEXT: store i32 [[TMP59]], ptr [[TMP54]], align 4, !alias.scope [[META22]], !noalias [[META19]]
+; COST1-NEXT: store i32 [[TMP60]], ptr [[TMP55]], align 4, !alias.scope [[META22]], !noalias [[META19]]
+; COST1-NEXT: store i32 [[TMP61]], ptr [[TMP56]], align 4, !alias.scope [[META22]], !noalias [[META19]]
+; COST1-NEXT: store i32 [[TMP49]], ptr [[TMP57]], align 4, !alias.scope [[META22]], !noalias [[META19]]
; COST1-NEXT: [[TMP113:%.*]] = getelementptr i8, ptr [[NEXT_GEP18]], i64 48
; COST1-NEXT: [[TMP114:%.*]] = getelementptr i8, ptr [[NEXT_GEP19]], i64 48
; COST1-NEXT: [[TMP115:%.*]] = getelementptr i8, ptr [[NEXT_GEP20]], i64 48
; COST1-NEXT: [[TMP116:%.*]] = getelementptr i8, ptr [[NEXT_GEP21]], i64 48
+; COST1-NEXT: [[TMP62:%.*]] = getelementptr i8, ptr [[NEXT_GEP5]], i64 48
+; COST1-NEXT: [[TMP63:%.*]] = getelementptr i8, ptr [[NEXT_GEP6]], i64 48
+; COST1-NEXT: [[TMP64:%.*]] = getelementptr i8, ptr [[NEXT_GEP7]], i64 48
+; COST1-NEXT: [[TMP65:%.*]] = getelementptr i8, ptr [[NEXT_GEP8]], i64 48
; COST1-NEXT: store i32 [[TMP45]], ptr [[TMP113]], align 4, !alias.scope [[META22]], !noalias [[META19]]
; COST1-NEXT: store i32 [[TMP50]], ptr [[TMP114]], align 4, !alias.scope [[META22]], !noalias [[META19]]
; COST1-NEXT: store i32 [[TMP47]], ptr [[TMP115]], align 4, !alias.scope [[META22]], !noalias [[META19]]
; COST1-NEXT: store i32 [[TMP48]], ptr [[TMP116]], align 4, !alias.scope [[META22]], !noalias [[META19]]
+; COST1-NEXT: store i32 [[TMP59]], ptr [[TMP62]], align 4, !alias.scope [[META22]], !noalias [[META19]]
+; COST1-NEXT: store i32 [[TMP60]], ptr [[TMP63]], align 4, !alias.scope [[META22]], !noalias [[META19]]
+; COST1-NEXT: store i32 [[TMP61]], ptr [[TMP64]], align 4, !alias.scope [[META22]], !noalias [[META19]]
+; COST1-NEXT: store i32 [[TMP49]], ptr [[TMP65]], align 4, !alias.scope [[META22]], !noalias [[META19]]
; COST1-NEXT: [[TMP117:%.*]] = getelementptr i8, ptr [[NEXT_GEP18]], i64 72
; COST1-NEXT: [[TMP118:%.*]] = getelementptr i8, ptr [[NEXT_GEP19]], i64 72
; COST1-NEXT: [[TMP119:%.*]] = getelementptr i8, ptr [[NEXT_GEP20]], i64 72
; COST1-NEXT: [[TMP120:%.*]] = getelementptr i8, ptr [[NEXT_GEP21]], i64 72
+; COST1-NEXT: [[TMP70:%.*]] = getelementptr i8, ptr [[NEXT_GEP5]], i64 72
+; COST1-NEXT: [[TMP71:%.*]] = getelementptr i8, ptr [[NEXT_GEP6]], i64 72
+; COST1-NEXT: [[TMP72:%.*]] = getelementptr i8, ptr [[NEXT_GEP7]], i64 72
+; COST1-NEXT: [[TMP73:%.*]] = getelementptr i8, ptr [[NEXT_GEP8]], i64 72
; COST1-NEXT: store i32 [[TMP45]], ptr [[TMP117]], align 4, !alias.scope [[META22]], !noalias [[META19]]
; COST1-NEXT: store i32 [[TMP50]], ptr [[TMP118]], align 4, !alias.scope [[META22]], !noalias [[META19]]
; COST1-NEXT: store i32 [[TMP47]], ptr [[TMP119]], align 4, !alias.scope [[META22]], !noalias [[META19]]
; COST1-NEXT: store i32 [[TMP48]], ptr [[TMP120]], align 4, !alias.scope [[META22]], !noalias [[META19]]
-; COST1-NEXT: [[INDEX_NEXT22]] = add nuw i64 [[TMP5]], 4
-; COST1-NEXT: [[TMP121:%.*]] = icmp eq i64 [[INDEX_NEXT22]], 20
-; COST1-NEXT: br i1 [[TMP121]], label %[[MIDDLE_BLOCK:.*]], label %[[VEC_EPILOG_VECTOR_BODY]], !llvm.loop [[LOOP26:![0-9]+]]
+; COST1-NEXT: store i32 [[TMP59]], ptr [[TMP70]], align 4, !alias.scope [[META22]], !noalias [[META19]]
+; COST1-NEXT: store i32 [[TMP60]], ptr [[TMP71]], align 4, !alias.scope [[META22]], !noalias [[META19]]
+; COST1-NEXT: store i32 [[TMP61]], ptr [[TMP72]], align 4, !alias.scope [[META22]], !noalias [[META19]]
+; COST1-NEXT: store i32 [[TMP49]], ptr [[TMP73]], align 4, !alias.scope [[META22]], !noalias [[META19]]
+; COST1-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 8
+; COST1-NEXT: [[TMP74:%.*]] = icmp eq i64 [[INDEX_NEXT]], 16
+; COST1-NEXT: br i1 [[TMP74]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP24:![0-9]+]]
; COST1: [[MIDDLE_BLOCK]]:
; COST1-NEXT: br label %[[SCALAR_PH]]
; COST1: [[SCALAR_PH]]:
@@ -948,198 +826,19 @@ define void @force_branch_cost(ptr readonly %src, ptr %dst) {
; COST10-LABEL: define void @force_branch_cost(
; COST10-SAME: ptr readonly [[SRC:%.*]], ptr [[DST:%.*]]) {
; COST10-NEXT: [[ENTRY:.*:]]
-; COST10-NEXT: br i1 false, label %[[SCALAR_PH:.*]], label %[[VECTOR_MEMCHECK:.*]]
+; COST10-NEXT: br label %[[VECTOR_MEMCHECK:.*]]
; COST10: [[VECTOR_MEMCHECK]]:
; COST10-NEXT: [[SCEVGEP:%.*]] = getelementptr i8, ptr [[DST]], i64 2092
; COST10-NEXT: [[SCEVGEP1:%.*]] = getelementptr i8, ptr [[SRC]], i64 85
; COST10-NEXT: [[BOUND0:%.*]] = icmp ult ptr [[DST]], [[SCEVGEP1]]
; COST10-NEXT: [[BOUND1:%.*]] = icmp ult ptr [[SRC]], [[SCEVGEP]]
; COST10-NEXT: [[FOUND_CONFLICT:%.*]] = and i1 [[BOUND0]], [[BOUND1]]
-; COST10-NEXT: br i1 [[FOUND_CONFLICT]], label %[[SCALAR_PH]], label %[[VECTOR_PH:.*]]
+; COST10-NEXT: br i1 [[FOUND_CONFLICT]], label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]]
; COST10: [[VECTOR_PH]]:
-; COST10-NEXT: br i1 false, label %[[VEC_EPILOG_PH:.*]], label %[[VECTOR_PH1:.*]]
-; COST10: [[VECTOR_PH1]]:
-; COST10-NEXT: [[TMP84:%.*]] = getelementptr i8, ptr [[DST]], i64 1536
-; COST10-NEXT: br label %[[VECTOR_BODY1:.*]]
-; COST10: [[VECTOR_BODY1]]:
-; COST10-NEXT: [[INDEX1:%.*]] = phi i64 [ 0, %[[VECTOR_PH1]] ], [ [[INDEX_NEXT1:%.*]], %[[VECTOR_BODY1]] ]
-; COST10-NEXT: [[TMP1:%.*]] = mul i64 [[INDEX1]], 96
-; COST10-NEXT: [[TMP85:%.*]] = add i64 [[TMP1]], 96
-; COST10-NEXT: [[TMP86:%.*]] = add i64 [[TMP1]], 192
-; COST10-NEXT: [[TMP87:%.*]] = add i64 [[TMP1]], 288
-; COST10-NEXT: [[TMP5:%.*]] = add i64 [[TMP1]], 384
-; COST10-NEXT: [[TMP88:%.*]] = add i64 [[TMP1]], 480
-; COST10-NEXT: [[TMP89:%.*]] = add i64 [[TMP1]], 576
-; COST10-NEXT: [[TMP90:%.*]] = add i64 [[TMP1]], 672
-; COST10-NEXT: [[TMP91:%.*]] = add i64 [[TMP1]], 768
-; COST10-NEXT: [[TMP92:%.*]] = add i64 [[TMP1]], 864
-; COST10-NEXT: [[TMP93:%.*]] = add i64 [[TMP1]], 960
-; COST10-NEXT: [[TMP94:%.*]] = add i64 [[TMP1]], 1056
-; COST10-NEXT: [[TMP95:%.*]] = add i64 [[TMP1]], 1152
-; COST10-NEXT: [[TMP96:%.*]] = add i64 [[TMP1]], 1248
-; COST10-NEXT: [[TMP97:%.*]] = add i64 [[TMP1]], 1344
-; COST10-NEXT: [[TMP98:%.*]] = add i64 [[TMP1]], 1440
-; COST10-NEXT: [[NEXT_GEP1:%.*]] = getelementptr i8, ptr [[DST]], i64 [[TMP1]]
-; COST10-NEXT: [[NEXT_GEP17:%.*]] = getelementptr i8, ptr [[DST]], i64 [[TMP85]]
-; COST10-NEXT: [[NEXT_GEP18:%.*]] = getelementptr i8, ptr [[DST]], i64 [[TMP86]]
-; COST10-NEXT: [[NEXT_GEP19:%.*]] = getelementptr i8, ptr [[DST]], i64 [[TMP87]]
-; COST10-NEXT: [[NEXT_GEP5:%.*]] = getelementptr i8, ptr [[DST]], i64 [[TMP5]]
-; COST10-NEXT: [[NEXT_GEP6:%.*]] = getelementptr i8, ptr [[DST]], i64 [[TMP88]]
-; COST10-NEXT: [[NEXT_GEP7:%.*]] = getelementptr i8, ptr [[DST]], i64 [[TMP89]]
-; COST10-NEXT: [[NEXT_GEP8:%.*]] = getelementptr i8, ptr [[DST]], i64 [[TMP90]]
-; COST10-NEXT: [[NEXT_GEP9:%.*]] = getelementptr i8, ptr [[DST]], i64 [[TMP91]]
-; COST10-NEXT: [[NEXT_GEP10:%.*]] = getelementptr i8, ptr [[DST]], i64 [[TMP92]]
-; COST10-NEXT: [[NEXT_GEP11:%.*]] = getelementptr i8, ptr [[DST]], i64 [[TMP93]]
-; COST10-NEXT: [[NEXT_GEP12:%.*]] = getelementptr i8, ptr [[DST]], i64 [[TMP94]]
-; COST10-NEXT: [[NEXT_GEP13:%.*]] = getelementptr i8, ptr [[DST]], i64 [[TMP95]]
-; COST10-NEXT: [[NEXT_GEP14:%.*]] = getelementptr i8, ptr [[DST]], i64 [[TMP96]]
-; COST10-NEXT: [[NEXT_GEP15:%.*]] = getelementptr i8, ptr [[DST]], i64 [[TMP97]]
-; COST10-NEXT: [[NEXT_GEP16:%.*]] = getelementptr i8, ptr [[DST]], i64 [[TMP98]]
-; COST10-NEXT: [[TMP99:%.*]] = getelementptr [4 x i8], ptr [[SRC]], i64 [[INDEX1]]
-; COST10-NEXT: [[WIDE_VEC:%.*]] = load <64 x i8>, ptr [[TMP99]], align 1, !alias.scope [[META19:![0-9]+]]
-; COST10-NEXT: [[STRIDED_VEC:%.*]] = shufflevector <64 x i8> [[WIDE_VEC]], <64 x i8> poison, <16 x i32> <i32 0, i32 4, i32 8, i32 12, i32 16, i32 20, i32 24, i32 28, i32 32, i32 36, i32 40, i32 44, i32 48, i32 52, i32 56, i32 60>
-; COST10-NEXT: [[TMP100:%.*]] = zext <16 x i8> [[STRIDED_VEC]] to <16 x i32>
-; COST10-NEXT: [[TMP101:%.*]] = extractelement <16 x i32> [[TMP100]], i64 0
-; COST10-NEXT: store i32 [[TMP101]], ptr [[NEXT_GEP1]], align 4, !alias.scope [[META22:![0-9]+]], !noalias [[META19]]
-; COST10-NEXT: [[TMP102:%.*]] = extractelement <16 x i32> [[TMP100]], i64 1
-; COST10-NEXT: store i32 [[TMP102]], ptr [[NEXT_GEP17]], align 4, !alias.scope [[META22]], !noalias [[META19]]
-; COST10-NEXT: [[TMP103:%.*]] = extractelement <16 x i32> [[TMP100]], i64 2
-; COST10-NEXT: store i32 [[TMP103]], ptr [[NEXT_GEP18]], align 4, !alias.scope [[META22]], !noalias [[META19]]
-; COST10-NEXT: [[TMP104:%.*]] = extractelement <16 x i32> [[TMP100]], i64 3
-; COST10-NEXT: store i32 [[TMP104]], ptr [[NEXT_GEP19]], align 4, !alias.scope [[META22]], !noalias [[META19]]
-; COST10-NEXT: [[TMP105:%.*]] = extractelement <16 x i32> [[TMP100]], i64 4
-; COST10-NEXT: store i32 [[TMP105]], ptr [[NEXT_GEP5]], align 4, !alias.scope [[META22]], !noalias [[META19]]
-; COST10-NEXT: [[TMP106:%.*]] = extractelement <16 x i32> [[TMP100]], i64 5
-; COST10-NEXT: store i32 [[TMP106]], ptr [[NEXT_GEP6]], align 4, !alias.scope [[META22]], !noalias [[META19]]
-; COST10-NEXT: [[TMP107:%.*]] = extractelement <16 x i32> [[TMP100]], i64 6
-; COST10-NEXT: store i32 [[TMP107]], ptr [[NEXT_GEP7]], align 4, !alias.scope [[META22]], !noalias [[META19]]
-; COST10-NEXT: [[TMP108:%.*]] = extractelement <16 x i32> [[TMP100]], i64 7
-; COST10-NEXT: store i32 [[TMP108]], ptr [[NEXT_GEP8]], align 4, !alias.scope [[META22]], !noalias [[META19]]
-; COST10-NEXT: [[TMP109:%.*]] = extractelement <16 x i32> [[TMP100]], i64 8
-; COST10-NEXT: store i32 [[TMP109]], ptr [[NEXT_GEP9]], align 4, !alias.scope [[META22]], !noalias [[META19]]
-; COST10-NEXT: [[TMP110:%.*]] = extractelement <16 x i32> [[TMP100]], i64 9
-; COST10-NEXT: store i32 [[TMP110]], ptr [[NEXT_GEP10]], align 4, !alias.scope [[META22]], !noalias [[META19]]
-; COST10-NEXT: [[TMP111:%.*]] = extractelement <16 x i32> [[TMP100]], i64 10
-; COST10-NEXT: store i32 [[TMP111]], ptr [[NEXT_GEP11]], align 4, !alias.scope [[META22]], !noalias [[META19]]
-; COST10-NEXT: [[TMP112:%.*]] = extractelement <16 x i32> [[TMP100]], i64 11
-; COST10-NEXT: store i32 [[TMP112]], ptr [[NEXT_GEP12]], align 4, !alias.scope [[META22]], !noalias [[META19]]
-; COST10-NEXT: [[TMP113:%.*]] = extractelement <16 x i32> [[TMP100]], i64 12
-; COST10-NEXT: store i32 [[TMP113]], ptr [[NEXT_GEP13]], align 4, !alias.scope [[META22]], !noalias [[META19]]
-; COST10-NEXT: [[TMP114:%.*]] = extractelement <16 x i32> [[TMP100]], i64 13
-; COST10-NEXT: store i32 [[TMP114]], ptr [[NEXT_GEP14]], align 4, !alias.scope [[META22]], !noalias [[META19]]
-; COST10-NEXT: [[TMP115:%.*]] = extractelement <16 x i32> [[TMP100]], i64 14
-; COST10-NEXT: store i32 [[TMP115]], ptr [[NEXT_GEP15]], align 4, !alias.scope [[META22]], !noalias [[META19]]
-; COST10-NEXT: [[TMP116:%.*]] = extractelement <16 x i32> [[TMP100]], i64 15
-; COST10-NEXT: store i32 [[TMP116]], ptr [[NEXT_GEP16]], align 4, !alias.scope [[META22]], !noalias [[META19]]
-; COST10-NEXT: [[TMP117:%.*]] = getelementptr i8, ptr [[NEXT_GEP1]], i64 24
-; COST10-NEXT: [[TMP118:%.*]] = getelementptr i8, ptr [[NEXT_GEP17]], i64 24
-; COST10-NEXT: [[TMP119:%.*]] = getelementptr i8, ptr [[NEXT_GEP18]], i64 24
-; COST10-NEXT: [[TMP120:%.*]] = getelementptr i8, ptr [[NEXT_GEP19]], i64 24
-; COST10-NEXT: [[TMP39:%.*]] = getelementptr i8, ptr [[NEXT_GEP5]], i64 24
-; COST10-NEXT: [[TMP40:%.*]] = getelementptr i8, ptr [[NEXT_GEP6]], i64 24
-; COST10-NEXT: [[TMP41:%.*]] = getelementptr i8, ptr [[NEXT_GEP7]], i64 24
-; COST10-NEXT: [[TMP42:%.*]] = getelementptr i8, ptr [[NEXT_GEP8]], i64 24
-; COST10-NEXT: [[TMP43:%.*]] = getelementptr i8, ptr [[NEXT_GEP9]], i64 24
-; COST10-NEXT: [[TMP44:%.*]] = getelementptr i8, ptr [[NEXT_GEP10]], i64 24
-; COST10-NEXT: [[TMP45:%.*]] = getelementptr i8, ptr [[NEXT_GEP11]], i64 24
-; COST10-NEXT: [[TMP46:%.*]] = getelementptr i8, ptr [[NEXT_GEP12]], i64 24
-; COST10-NEXT: [[TMP47:%.*]] = getelementptr i8, ptr [[NEXT_GEP13]], i64 24
-; COST10-NEXT: [[TMP48:%.*]] = getelementptr i8, ptr [[NEXT_GEP14]], i64 24
-; COST10-NEXT: [[TMP49:%.*]] = getelementptr i8, ptr [[NEXT_GEP15]], i64 24
-; COST10-NEXT: [[TMP50:%.*]] = getelementptr i8, ptr [[NEXT_GEP16]], i64 24
-; COST10-NEXT: store i32 [[TMP101]], ptr [[TMP117]], align 4, !alias.scope [[META22]], !noalias [[META19]]
-; COST10-NEXT: store i32 [[TMP102]], ptr [[TMP118]], align 4, !alias.scope [[META22]], !noalias [[META19]]
-; COST10-NEXT: store i32 [[TMP103]], ptr [[TMP119]], align 4, !alias.scope [[META22]], !noalias [[META19]]
-; COST10-NEXT: store i32 [[TMP104]], ptr [[TMP120]], align 4, !alias.scope [[META22]], !noalias [[META19]]
-; COST10-NEXT: store i32 [[TMP105]], ptr [[TMP39]], align 4, !alias.scope [[META22]], !noalias [[META19]]
-; COST10-NEXT: store i32 [[TMP106]], ptr [[TMP40]], align 4, !alias.scope [[META22]], !noalias [[META19]]
-; COST10-NEXT: store i32 [[TMP107]], ptr [[TMP41]], align 4, !alias.scope [[META22]], !noalias [[META19]]
-; COST10-NEXT: store i32 [[TMP108]], ptr [[TMP42]], align 4, !alias.scope [[META22]], !noalias [[META19]]
-; COST10-NEXT: store i32 [[TMP109]], ptr [[TMP43]], align 4, !alias.scope [[META22]], !noalias [[META19]]
-; COST10-NEXT: store i32 [[TMP110]], ptr [[TMP44]], align 4, !alias.scope [[META22]], !noalias [[META19]]
-; COST10-NEXT: store i32 [[TMP111]], ptr [[TMP45]], align 4, !alias.scope [[META22]], !noalias [[META19]]
-; COST10-NEXT: store i32 [[TMP112]], ptr [[TMP46]], align 4, !alias.scope [[META22]], !noalias [[META19]]
-; COST10-NEXT: store i32 [[TMP113]], ptr [[TMP47]], align 4, !alias.scope [[META22]], !noalias [[META19]]
-; COST10-NEXT: store i32 [[TMP114]], ptr [[TMP48]], align 4, !alias.scope [[META22]], !noalias [[META19]]
-; COST10-NEXT: store i32 [[TMP115]], ptr [[TMP49]], align 4, !alias.scope [[META22]], !noalias [[META19]]
-; COST10-NEXT: store i32 [[TMP116]], ptr [[TMP50]], align 4, !alias.scope [[META22]], !noalias [[META19]]
-; COST10-NEXT: [[TMP51:%.*]] = getelementptr i8, ptr [[NEXT_GEP1]], i64 48
-; COST10-NEXT: [[TMP52:%.*]] = getelementptr i8, ptr [[NEXT_GEP17]], i64 48
-; COST10-NEXT: [[TMP53:%.*]] = getelementptr i8, ptr [[NEXT_GEP18]], i64 48
-; COST10-NEXT: [[TMP54:%.*]] = getelementptr i8, ptr [[NEXT_GEP19]], i64 48
-; COST10-NEXT: [[TMP55:%.*]] = getelementptr i8, ptr [[NEXT_GEP5]], i64 48
-; COST10-NEXT: [[TMP56:%.*]] = getelementptr i8, ptr [[NEXT_GEP6]], i64 48
-; COST10-NEXT: [[TMP57:%.*]] = getelementptr i8, ptr [[NEXT_GEP7]], i64 48
-; COST10-NEXT: [[TMP58:%.*]] = getelementptr i8, ptr [[NEXT_GEP8]], i64 48
-; COST10-NEXT: [[TMP59:%.*]] = getelementptr i8, ptr [[NEXT_GEP9]], i64 48
-; COST10-NEXT: [[TMP60:%.*]] = getelementptr i8, ptr [[NEXT_GEP10]], i64 48
-; COST10-NEXT: [[TMP61:%.*]] = getelementptr i8, ptr [[NEXT_GEP11]], i64 48
-; COST10-NEXT: [[TMP62:%.*]] = getelementptr i8, ptr [[NEXT_GEP12]], i64 48
-; COST10-NEXT: [[TMP63:%.*]] = getelementptr i8, ptr [[NEXT_GEP13]], i64 48
-; COST10-NEXT: [[TMP64:%.*]] = getelementptr i8, ptr [[NEXT_GEP14]], i64 48
-; COST10-NEXT: [[TMP65:%.*]] = getelementptr i8, ptr [[NEXT_GEP15]], i64 48
-; COST10-NEXT: [[TMP66:%.*]] = getelementptr i8, ptr [[NEXT_GEP16]], i64 48
-; COST10-NEXT: store i32 [[TMP101]], ptr [[TMP51]], align 4, !alias.scope [[META22]], !noalias [[META19]]
-; COST10-NEXT: store i32 [[TMP102]], ptr [[TMP52]], align 4, !alias.scope [[META22]], !noalias [[META19]]
-; COST10-NEXT: store i32 [[TMP103]], ptr [[TMP53]], align 4, !alias.scope [[META22]], !noalias [[META19]]
-; COST10-NEXT: store i32 [[TMP104]], ptr [[TMP54]], align 4, !alias.scope [[META22]], !noalias [[META19]]
-; COST10-NEXT: store i32 [[TMP105]], ptr [[TMP55]], align 4, !alias.scope [[META22]], !noalias [[META19]]
-; COST10-NEXT: store i32 [[TMP106]], ptr [[TMP56]], align 4, !alias.scope [[META22]], !noalias [[META19]]
-; COST10-NEXT: store i32 [[TMP107]], ptr [[TMP57]], align 4, !alias.scope [[META22]], !noalias [[META19]]
-; COST10-NEXT: store i32 [[TMP108]], ptr [[TMP58]], align 4, !alias.scope [[META22]], !noalias [[META19]]
-; COST10-NEXT: store i32 [[TMP109]], ptr [[TMP59]], align 4, !alias.scope [[META22]], !noalias [[META19]]
-; COST10-NEXT: store i32 [[TMP110]], ptr [[TMP60]], align 4, !alias.scope [[META22]], !noalias [[META19]]
-; COST10-NEXT: store i32 [[TMP111]], ptr [[TMP61]], align 4, !alias.scope [[META22]], !noalias [[META19]]
-; COST10-NEXT: store i32 [[TMP112]], ptr [[TMP62]], align 4, !alias.scope [[META22]], !noalias [[META19]]
-; COST10-NEXT: store i32 [[TMP113]], ptr [[TMP63]], align 4, !alias.scope [[META22]], !noalias [[META19]]
-; COST10-NEXT: store i32 [[TMP114]], ptr [[TMP64]], align 4, !alias.scope [[META22]], !noalias [[META19]]
-; COST10-NEXT: store i32 [[TMP115]], ptr [[TMP65]], align 4, !alias.scope [[META22]], !noalias [[META19]]
-; COST10-NEXT: store i32 [[TMP116]], ptr [[TMP66]], align 4, !alias.scope [[META22]], !noalias [[META19]]
-; COST10-NEXT: [[TMP67:%.*]] = getelementptr i8, ptr [[NEXT_GEP1]], i64 72
-; COST10-NEXT: [[TMP68:%.*]] = getelementptr i8, ptr [[NEXT_GEP17]], i64 72
-; COST10-NEXT: [[TMP69:%.*]] = getelementptr i8, ptr [[NEXT_GEP18]], i64 72
-; COST10-NEXT: [[TMP70:%.*]] = getelementptr i8, ptr [[NEXT_GEP19]], i64 72
-; COST10-NEXT: [[TMP71:%.*]] = getelementptr i8, ptr [[NEXT_GEP5]], i64 72
-; COST10-NEXT: [[TMP72:%.*]] = getelementptr i8, ptr [[NEXT_GEP6]], i64 72
-; COST10-NEXT: [[TMP73:%.*]] = getelementptr i8, ptr [[NEXT_GEP7]], i64 72
-; COST10-NEXT: [[TMP74:%.*]] = getelementptr i8, ptr [[NEXT_GEP8]], i64 72
-; COST10-NEXT: [[TMP75:%.*]] = getelementptr i8, ptr [[NEXT_GEP9]], i64 72
-; COST10-NEXT: [[TMP76:%.*]] = getelementptr i8, ptr [[NEXT_GEP10]], i64 72
-; COST10-NEXT: [[TMP77:%.*]] = getelementptr i8, ptr [[NEXT_GEP11]], i64 72
-; COST10-NEXT: [[TMP78:%.*]] = getelementptr i8, ptr [[NEXT_GEP12]], i64 72
-; COST10-NEXT: [[TMP79:%.*]] = getelementptr i8, ptr [[NEXT_GEP13]], i64 72
-; COST10-NEXT: [[TMP80:%.*]] = getelementptr i8, ptr [[NEXT_GEP14]], i64 72
-; COST10-NEXT: [[TMP81:%.*]] = getelementptr i8, ptr [[NEXT_GEP15]], i64 72
-; COST10-NEXT: [[TMP82:%.*]] = getelementptr i8, ptr [[NEXT_GEP16]], i64 72
-; COST10-NEXT: store i32 [[TMP101]], ptr [[TMP67]], align 4, !alias.scope [[META22]], !noalias [[META19]]
-; COST10-NEXT: store i32 [[TMP102]], ptr [[TMP68]], align 4, !alias.scope [[META22]], !noalias [[META19]]
-; COST10-NEXT: store i32 [[TMP103]], ptr [[TMP69]], align 4, !alias.scope [[META22]], !noalias [[META19]]
-; COST10-NEXT: store i32 [[TMP104]], ptr [[TMP70]], align 4, !alias.scope [[META22]], !noalias [[META19]]
-; COST10-NEXT: store i32 [[TMP105]], ptr [[TMP71]], align 4, !alias.scope [[META22]], !noalias [[META19]]
-; COST10-NEXT: store i32 [[TMP106]], ptr [[TMP72]], align 4, !alias.scope [[META22]], !noalias [[META19]]
-; COST10-NEXT: store i32 [[TMP107]], ptr [[TMP73]], align 4, !alias.scope [[META22]], !noalias [[META19]]
-; COST10-NEXT: store i32 [[TMP108]], ptr [[TMP74]], align 4, !alias.scope [[META22]], !noalias [[META19]]
-; COST10-NEXT: store i32 [[TMP109]], ptr [[TMP75]], align 4, !alias.scope [[META22]], !noalias [[META19]]
-; COST10-NEXT: store i32 [[TMP110]], ptr [[TMP76]], align 4, !alias.scope [[META22]], !noalias [[META19]]
-; COST10-NEXT: store i32 [[TMP111]], ptr [[TMP77]], align 4, !alias.scope [[META22]], !noalias [[META19]]
-; COST10-NEXT: store i32 [[TMP112]], ptr [[TMP78]], align 4, !alias.scope [[META22]], !noalias [[META19]]
-; COST10-NEXT: store i32 [[TMP113]], ptr [[TMP79]], align 4, !alias.scope [[META22]], !noalias [[META19]]
-; COST10-NEXT: store i32 [[TMP114]], ptr [[TMP80]], align 4, !alias.scope [[META22]], !noalias [[META19]]
-; COST10-NEXT: store i32 [[TMP115]], ptr [[TMP81]], align 4, !alias.scope [[META22]], !noalias [[META19]]
-; COST10-NEXT: store i32 [[TMP116]], ptr [[TMP82]], align 4, !alias.scope [[META22]], !noalias [[META19]]
-; COST10-NEXT: [[INDEX_NEXT1]] = add nuw i64 [[INDEX1]], 16
-; COST10-NEXT: [[TMP83:%.*]] = icmp eq i64 [[INDEX_NEXT1]], 16
-; COST10-NEXT: br i1 [[TMP83]], label %[[MIDDLE_BLOCK1:.*]], label %[[VECTOR_BODY1]], !llvm.loop [[LOOP24:![0-9]+]]
-; COST10: [[MIDDLE_BLOCK1]]:
-; COST10-NEXT: br label %[[VEC_EPILOG_ITER_CHECK:.*]]
-; COST10: [[VEC_EPILOG_ITER_CHECK]]:
-; COST10-NEXT: br i1 false, label %[[SCALAR_PH]], label %[[VEC_EPILOG_PH]], !prof [[PROF3]]
-; COST10: [[VEC_EPILOG_PH]]:
-; COST10-NEXT: [[VEC_EPILOG_RESUME_VAL:%.*]] = phi i64 [ 16, %[[VEC_EPILOG_ITER_CHECK]] ], [ 0, %[[VECTOR_PH]] ]
; COST10-NEXT: [[TMP0:%.*]] = getelementptr i8, ptr [[DST]], i64 1920
; COST10-NEXT: br label %[[VECTOR_BODY:.*]]
; COST10: [[VECTOR_BODY]]:
-; COST10-NEXT: [[INDEX:%.*]] = phi i64 [ [[VEC_EPILOG_RESUME_VAL]], %[[VEC_EPILOG_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; COST10-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
; COST10-NEXT: [[TMP2:%.*]] = add i64 [[INDEX]], 1
; COST10-NEXT: [[TMP3:%.*]] = add i64 [[INDEX]], 2
; COST10-NEXT: [[TMP4:%.*]] = add i64 [[INDEX]], 3
@@ -1155,7 +854,7 @@ define void @force_branch_cost(ptr readonly %src, ptr %dst) {
; COST10-NEXT: [[TMP10:%.*]] = getelementptr [4 x i8], ptr [[SRC]], i64 [[TMP2]]
; COST10-NEXT: [[TMP11:%.*]] = getelementptr [4 x i8], ptr [[SRC]], i64 [[TMP3]]
; COST10-NEXT: [[TMP12:%.*]] = getelementptr [4 x i8], ptr [[SRC]], i64 [[TMP4]]
-; COST10-NEXT: [[TMP13:%.*]] = load i8, ptr [[TMP9]], align 1, !alias.scope [[META19]]
+; COST10-NEXT: [[TMP13:%.*]] = load i8, ptr [[TMP9]], align 1, !alias.scope [[META19:![0-9]+]]
; COST10-NEXT: [[TMP14:%.*]] = load i8, ptr [[TMP10]], align 1, !alias.scope [[META19]]
; COST10-NEXT: [[TMP15:%.*]] = load i8, ptr [[TMP11]], align 1, !alias.scope [[META19]]
; COST10-NEXT: [[TMP16:%.*]] = load i8, ptr [[TMP12]], align 1, !alias.scope [[META19]]
@@ -1165,7 +864,7 @@ define void @force_branch_cost(ptr readonly %src, ptr %dst) {
; COST10-NEXT: [[TMP20:%.*]] = insertelement <4 x i8> [[TMP19]], i8 [[TMP16]], i64 3
; COST10-NEXT: [[TMP21:%.*]] = zext <4 x i8> [[TMP20]] to <4 x i32>
; COST10-NEXT: [[TMP24:%.*]] = extractelement <4 x i32> [[TMP21]], i64 0
-; COST10-NEXT: store i32 [[TMP24]], ptr [[NEXT_GEP]], align 4, !alias.scope [[META22]], !noalias [[META19]]
+; COST10-NEXT: store i32 [[TMP24]], ptr [[NEXT_GEP]], align 4, !alias.scope [[META22:![0-9]+]], !noalias [[META19]]
; COST10-NEXT: [[TMP25:%.*]] = extractelement <4 x i32> [[TMP21]], i64 1
; COST10-NEXT: store i32 [[TMP25]], ptr [[NEXT_GEP2]], align 4, !alias.scope [[META22]], !noalias [[META19]]
; COST10-NEXT: [[TMP22:%.*]] = extractelement <4 x i32> [[TMP21]], i64 2
@@ -1198,7 +897,7 @@ define void @force_branch_cost(ptr readonly %src, ptr %dst) {
; COST10-NEXT: store i32 [[TMP23]], ptr [[TMP37]], align 4, !alias.scope [[META22]], !noalias [[META19]]
; COST10-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 4
; COST10-NEXT: [[TMP38:%.*]] = icmp eq i64 [[INDEX_NEXT]], 20
-; COST10-NEXT: br i1 [[TMP38]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP25:![0-9]+]]
+; COST10-NEXT: br i1 [[TMP38]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP24:![0-9]+]]
; COST10: [[MIDDLE_BLOCK]]:
; COST10-NEXT: br label %[[SCALAR_PH]]
; COST10: [[SCALAR_PH]]:
diff --git a/llvm/test/Transforms/LoopVectorize/AArch64/predication_costs.ll b/llvm/test/Transforms/LoopVectorize/AArch64/predication_costs.ll
index fdb22beb19695..940416c04324d 100644
--- a/llvm/test/Transforms/LoopVectorize/AArch64/predication_costs.ll
+++ b/llvm/test/Transforms/LoopVectorize/AArch64/predication_costs.ll
@@ -93,7 +93,7 @@ for.end:
; CHECK: Found scalar instruction: %addr = phi ptr [ %a, %entry ], [ %addr.next, %for.inc ]
; CHECK: Found scalar instruction: %addr.next = getelementptr inbounds i32, ptr %addr, i64 1
; CHECK: Scalarizing and predicating: store i32 %tmp2, ptr %addr, align 4
-; CHECK: Cost of 0 for VF 2: induction instruction %addr = phi ptr [ %a, %entry ], [ %addr.next, %for.inc ]
+; CHECK: Cost of 0 for VF 2: forced scalar %addr = phi ptr [ %a, %entry ], [ %addr.next, %for.inc ]
; CHECK: Cost of 4 for VF 2: profitable to scalarize store i32 %tmp2, ptr %addr, align 4
;
define void @predicated_store_phi(ptr %a, i1 %c, i32 %x, i64 %n) {
diff --git a/llvm/test/Transforms/LoopVectorize/ARM/mve-icmpcost.ll b/llvm/test/Transforms/LoopVectorize/ARM/mve-icmpcost.ll
index ebf62272564ef..7369ef688f5e2 100644
--- a/llvm/test/Transforms/LoopVectorize/ARM/mve-icmpcost.ll
+++ b/llvm/test/Transforms/LoopVectorize/ARM/mve-icmpcost.ll
@@ -153,12 +153,6 @@ define void @cheap_icmp(ptr nocapture readonly %pSrcA, ptr nocapture readonly %p
; CHECK: LV: Found an estimated cost of 1 for VF 1 For instruction: %dec = add i32 %blkCnt.012, -1
; CHECK: LV: Found an estimated cost of 1 for VF 1 For instruction: %cmp.not = icmp eq i32 %dec, 0
; CHECK: LV: Found an estimated cost of 0 for VF 1 For instruction: br i1 %cmp.not, label %while.end.loopexit, label %while.body
-; CHECK: Cost of 0 for VF 2: induction instruction %incdec.ptr = getelementptr inbounds i8, ptr %pSrcA.addr.011, i32 1
-; CHECK: Cost of 0 for VF 2: induction instruction %pSrcA.addr.011 = phi ptr [ %incdec.ptr, %while.body ], [ %pSrcA, %while.body.preheader ]
-; CHECK: Cost of 0 for VF 2: induction instruction %incdec.ptr5 = getelementptr inbounds i8, ptr %pDst.addr.010, i32 1
-; CHECK: Cost of 0 for VF 2: induction instruction %pDst.addr.010 = phi ptr [ %incdec.ptr5, %while.body ], [ %pDst, %while.body.preheader ]
-; CHECK: Cost of 0 for VF 2: induction instruction %incdec.ptr2 = getelementptr inbounds i8, ptr %pSrcB.addr.09, i32 1
-; CHECK: Cost of 0 for VF 2: induction instruction %pSrcB.addr.09 = phi ptr [ %incdec.ptr2, %while.body ], [ %pSrcB, %while.body.preheader ]
; CHECK: Cost of 0 for VF 2: vp<[[VP8:%[0-9]+]]> = SCALAR-STEPS vp<[[VP7:%[0-9]+]]>, ir<1>, vp<[[VP0:%[0-9]+]]>
; CHECK: Cost of 0 for VF 2: EMIT vp<%next.gep> = ptradd ir<%pSrcA>, vp<[[VP8]]>
; CHECK: Cost of 0 for VF 2: vp<[[VP9:%[0-9]+]]> = SCALAR-STEPS vp<[[VP7]]>, ir<1>, vp<[[VP0]]>
@@ -212,12 +206,6 @@ define void @cheap_icmp(ptr nocapture readonly %pSrcA, ptr nocapture readonly %p
; CHECK: Cost of 1 for VF 2: EMIT vp<%cmp.n> = icmp eq ir<%blockSize>, vp<[[VP2]]>
; CHECK: Cost of 0 for VF 2: EMIT branch-on-cond vp<%cmp.n>
; CHECK: Cost for VF 2: 130 (Estimated cost per lane: 65)
-; CHECK: Cost of 0 for VF 4: induction instruction %incdec.ptr = getelementptr inbounds i8, ptr %pSrcA.addr.011, i32 1
-; CHECK: Cost of 0 for VF 4: induction instruction %pSrcA.addr.011 = phi ptr [ %incdec.ptr, %while.body ], [ %pSrcA, %while.body.preheader ]
-; CHECK: Cost of 0 for VF 4: induction instruction %incdec.ptr5 = getelementptr inbounds i8, ptr %pDst.addr.010, i32 1
-; CHECK: Cost of 0 for VF 4: induction instruction %pDst.addr.010 = phi ptr [ %incdec.ptr5, %while.body ], [ %pDst, %while.body.preheader ]
-; CHECK: Cost of 0 for VF 4: induction instruction %incdec.ptr2 = getelementptr inbounds i8, ptr %pSrcB.addr.09, i32 1
-; CHECK: Cost of 0 for VF 4: induction instruction %pSrcB.addr.09 = phi ptr [ %incdec.ptr2, %while.body ], [ %pSrcB, %while.body.preheader ]
; CHECK: Cost of 0 for VF 4: vp<[[VP8]]> = SCALAR-STEPS vp<[[VP7]]>, ir<1>, vp<[[VP0]]>
; CHECK: Cost of 0 for VF 4: EMIT vp<%next.gep> = ptradd ir<%pSrcA>, vp<[[VP8]]>
; CHECK: Cost of 0 for VF 4: vp<[[VP9]]> = SCALAR-STEPS vp<[[VP7]]>, ir<1>, vp<[[VP0]]>
@@ -271,12 +259,6 @@ define void @cheap_icmp(ptr nocapture readonly %pSrcA, ptr nocapture readonly %p
; CHECK: Cost of 1 for VF 4: EMIT vp<%cmp.n> = icmp eq ir<%blockSize>, vp<[[VP2]]>
; CHECK: Cost of 0 for VF 4: EMIT branch-on-cond vp<%cmp.n>
; CHECK: Cost for VF 4: 14 (Estimated cost per lane: 3.5)
-; CHECK: Cost of 0 for VF 8: induction instruction %incdec.ptr = getelementptr inbounds i8, ptr %pSrcA.addr.011, i32 1
-; CHECK: Cost of 0 for VF 8: induction instruction %pSrcA.addr.011 = phi ptr [ %incdec.ptr, %while.body ], [ %pSrcA, %while.body.preheader ]
-; CHECK: Cost of 0 for VF 8: induction instruction %incdec.ptr5 = getelementptr inbounds i8, ptr %pDst.addr.010, i32 1
-; CHECK: Cost of 0 for VF 8: induction instruction %pDst.addr.010 = phi ptr [ %incdec.ptr5, %while.body ], [ %pDst, %while.body.preheader ]
-; CHECK: Cost of 0 for VF 8: induction instruction %incdec.ptr2 = getelementptr inbounds i8, ptr %pSrcB.addr.09, i32 1
-; CHECK: Cost of 0 for VF 8: induction instruction %pSrcB.addr.09 = phi ptr [ %incdec.ptr2, %while.body ], [ %pSrcB, %while.body.preheader ]
; CHECK: Cost of 0 for VF 8: vp<[[VP8]]> = SCALAR-STEPS vp<[[VP7]]>, ir<1>, vp<[[VP0]]>
; CHECK: Cost of 0 for VF 8: EMIT vp<%next.gep> = ptradd ir<%pSrcA>, vp<[[VP8]]>
; CHECK: Cost of 0 for VF 8: vp<[[VP9]]> = SCALAR-STEPS vp<[[VP7]]>, ir<1>, vp<[[VP0]]>
@@ -330,12 +312,6 @@ define void @cheap_icmp(ptr nocapture readonly %pSrcA, ptr nocapture readonly %p
; CHECK: Cost of 1 for VF 8: EMIT vp<%cmp.n> = icmp eq ir<%blockSize>, vp<[[VP2]]>
; CHECK: Cost of 0 for VF 8: EMIT branch-on-cond vp<%cmp.n>
; CHECK: Cost for VF 8: 26 (Estimated cost per lane: 3.25)
-; CHECK: Cost of 0 for VF 16: induction instruction %incdec.ptr = getelementptr inbounds i8, ptr %pSrcA.addr.011, i32 1
-; CHECK: Cost of 0 for VF 16: induction instruction %pSrcA.addr.011 = phi ptr [ %incdec.ptr, %while.body ], [ %pSrcA, %while.body.preheader ]
-; CHECK: Cost of 0 for VF 16: induction instruction %incdec.ptr5 = getelementptr inbounds i8, ptr %pDst.addr.010, i32 1
-; CHECK: Cost of 0 for VF 16: induction instruction %pDst.addr.010 = phi ptr [ %incdec.ptr5, %while.body ], [ %pDst, %while.body.preheader ]
-; CHECK: Cost of 0 for VF 16: induction instruction %incdec.ptr2 = getelementptr inbounds i8, ptr %pSrcB.addr.09, i32 1
-; CHECK: Cost of 0 for VF 16: induction instruction %pSrcB.addr.09 = phi ptr [ %incdec.ptr2, %while.body ], [ %pSrcB, %while.body.preheader ]
; CHECK: Cost of 0 for VF 16: vp<[[VP8]]> = SCALAR-STEPS vp<[[VP7]]>, ir<1>, vp<[[VP0]]>
; CHECK: Cost of 0 for VF 16: EMIT vp<%next.gep> = ptradd ir<%pSrcA>, vp<[[VP8]]>
; CHECK: Cost of 0 for VF 16: vp<[[VP9]]> = SCALAR-STEPS vp<[[VP7]]>, ir<1>, vp<[[VP0]]>
@@ -444,10 +420,6 @@ define void @floatcmp(ptr nocapture readonly %pSrc, ptr nocapture %pDst, i32 %bl
; CHECK: LV: Found an estimated cost of 1 for VF 1 For instruction: %dec = add i32 %blockSize.addr.09, -1
; CHECK: LV: Found an estimated cost of 1 for VF 1 For instruction: %cmp.not = icmp eq i32 %dec, 0
; CHECK: LV: Found an estimated cost of 0 for VF 1 For instruction: br i1 %cmp.not, label %while.end.loopexit, label %while.body
-; CHECK: Cost of 0 for VF 2: induction instruction %incdec.ptr2 = getelementptr inbounds float, ptr %pSrc.addr.010, i32 1
-; CHECK: Cost of 0 for VF 2: induction instruction %pSrc.addr.010 = phi ptr [ %incdec.ptr2, %while.body ], [ %pSrc, %while.body.preheader ]
-; CHECK: Cost of 0 for VF 2: induction instruction %incdec.ptr = getelementptr inbounds i32, ptr %pDst.addr.08, i32 1
-; CHECK: Cost of 0 for VF 2: induction instruction %pDst.addr.08 = phi ptr [ %incdec.ptr, %while.body ], [ %pDst, %while.body.preheader ]
; CHECK: Cost of 1 for VF 2: vp<[[VP7:%[0-9]+]]> = DERIVED-IV ir<0> + vp<[[VP6:%[0-9]+]]> * ir<4>
; CHECK: Cost of 0 for VF 2: vp<[[VP8:%[0-9]+]]> = SCALAR-STEPS vp<[[VP7]]>, ir<4>, vp<[[VP0:%[0-9]+]]>
; CHECK: Cost of 0 for VF 2: EMIT vp<%next.gep> = ptradd ir<%pSrc>, vp<[[VP8]]>
@@ -486,10 +458,6 @@ define void @floatcmp(ptr nocapture readonly %pSrc, ptr nocapture %pDst, i32 %bl
; CHECK: Cost of 1 for VF 2: EMIT vp<%cmp.n> = icmp eq ir<%blockSize>, vp<[[VP2]]>
; CHECK: Cost of 0 for VF 2: EMIT branch-on-cond vp<%cmp.n>
; CHECK: Cost for VF 2: 84 (Estimated cost per lane: 42)
-; CHECK: Cost of 0 for VF 4: induction instruction %incdec.ptr2 = getelementptr inbounds float, ptr %pSrc.addr.010, i32 1
-; CHECK: Cost of 0 for VF 4: induction instruction %pSrc.addr.010 = phi ptr [ %incdec.ptr2, %while.body ], [ %pSrc, %while.body.preheader ]
-; CHECK: Cost of 0 for VF 4: induction instruction %incdec.ptr = getelementptr inbounds i32, ptr %pDst.addr.08, i32 1
-; CHECK: Cost of 0 for VF 4: induction instruction %pDst.addr.08 = phi ptr [ %incdec.ptr, %while.body ], [ %pDst, %while.body.preheader ]
; CHECK: Cost of 1 for VF 4: vp<[[VP7]]> = DERIVED-IV ir<0> + vp<[[VP6]]> * ir<4>
; CHECK: Cost of 0 for VF 4: vp<[[VP8]]> = SCALAR-STEPS vp<[[VP7]]>, ir<4>, vp<[[VP0]]>
; CHECK: Cost of 0 for VF 4: EMIT vp<%next.gep> = ptradd ir<%pSrc>, vp<[[VP8]]>
diff --git a/llvm/test/Transforms/LoopVectorize/WebAssembly/memory-interleave.ll b/llvm/test/Transforms/LoopVectorize/WebAssembly/memory-interleave.ll
index 14e3967f0a387..146e25c2f7f00 100644
--- a/llvm/test/Transforms/LoopVectorize/WebAssembly/memory-interleave.ll
+++ b/llvm/test/Transforms/LoopVectorize/WebAssembly/memory-interleave.ll
@@ -2030,21 +2030,21 @@ define hidden void @scale_uv_row_down2(ptr nocapture noundef readonly %0, i32 no
; CHECK: Cost of 11 for VF 4: INTERLEAVE-GROUP with factor 2, vp<%next.gep>.1
; CHECK: store ir<%11> to index 0
; CHECK: store ir<%13> to index 1
-; CHECK: Cost for VF 4: 36 (Estimated cost per lane: 9)
+; CHECK: Cost for VF 4: 34 (Estimated cost per lane: 8.5)
; CHECK: Cost of 26 for VF 8: INTERLEAVE-GROUP with factor 4, ir<%10>
; CHECK: ir<%11> = load from index 0
; CHECK: ir<%13> = load from index 1
; CHECK: Cost of 7 for VF 8: INTERLEAVE-GROUP with factor 2, vp<%next.gep>.1
; CHECK: store ir<%11> to index 0
; CHECK: store ir<%13> to index 1
-; CHECK: Cost for VF 8: 40 (Estimated cost per lane: 5)
+; CHECK: Cost for VF 8: 38 (Estimated cost per lane: 4.75)
; CHECK: Cost of 68 for VF 16: INTERLEAVE-GROUP with factor 4, ir<%10>
; CHECK: ir<%11> = load from index 0
; CHECK: ir<%13> = load from index 1
; CHECK: Cost of 6 for VF 16: INTERLEAVE-GROUP with factor 2, vp<%next.gep>.1
; CHECK: store ir<%11> to index 0
; CHECK: store ir<%13> to index 1
-; CHECK: Cost for VF 16: 81 (Estimated cost per lane: 5.06)
+; CHECK: Cost for VF 16: 79 (Estimated cost per lane: 4.94)
; CHECK: LV: Selecting VF: 8.
;
%5 = icmp sgt i32 %3, 0
@@ -2084,7 +2084,7 @@ define hidden void @scale_uv_row_down2_box(ptr nocapture noundef readonly %0, i3
; CHECK: Cost of 6 for VF 2: REPLICATE ir<%38> = load ir<%37>
; CHECK: Cost of 6 for VF 2: REPLICATE ir<%41> = load ir<%40>
; CHECK: Cost of 6 for VF 2: REPLICATE store ir<%48>, ir<%49>
-; CHECK: Cost for VF 2: 81 (Estimated cost per lane: 40.5)
+; CHECK: Cost for VF 2: 79 (Estimated cost per lane: 39.5)
; CHECK: Cost of 18 for VF 4: INTERLEAVE-GROUP with factor 4, vp<%next.gep>
; CHECK: ir<%14> = load from index 0
; CHECK: ir<%32> = load from index 1
@@ -2098,7 +2098,7 @@ define hidden void @scale_uv_row_down2_box(ptr nocapture noundef readonly %0, i3
; CHECK: Cost of 11 for VF 4: INTERLEAVE-GROUP with factor 2, vp<%next.gep>.1
; CHECK: store ir<%30> to index 0
; CHECK: store ir<%48> to index 1
-; CHECK: Cost for VF 4: 74 (Estimated cost per lane: 18.5)
+; CHECK: Cost for VF 4: 72 (Estimated cost per lane: 18)
; CHECK: Cost of 26 for VF 8: INTERLEAVE-GROUP with factor 4, vp<%next.gep>
; CHECK: ir<%14> = load from index 0
; CHECK: ir<%32> = load from index 1
@@ -2112,7 +2112,7 @@ define hidden void @scale_uv_row_down2_box(ptr nocapture noundef readonly %0, i3
; CHECK: Cost of 7 for VF 8: INTERLEAVE-GROUP with factor 2, vp<%next.gep>.1
; CHECK: store ir<%30> to index 0
; CHECK: store ir<%48> to index 1
-; CHECK: Cost for VF 8: 90 (Estimated cost per lane: 11.3)
+; CHECK: Cost for VF 8: 88 (Estimated cost per lane: 11)
; CHECK: Cost of 132 for VF 16: INTERLEAVE-GROUP with factor 4, vp<%next.gep>
; CHECK: ir<%14> = load from index 0
; CHECK: ir<%32> = load from index 1
@@ -2126,7 +2126,7 @@ define hidden void @scale_uv_row_down2_box(ptr nocapture noundef readonly %0, i3
; CHECK: Cost of 6 for VF 16: INTERLEAVE-GROUP with factor 2, vp<%next.gep>.1
; CHECK: store ir<%30> to index 0
; CHECK: store ir<%48> to index 1
-; CHECK: Cost for VF 16: 323 (Estimated cost per lane: 20.2)
+; CHECK: Cost for VF 16: 321 (Estimated cost per lane: 20.1)
; CHECK: LV: Selecting VF: 8.
;
%5 = icmp sgt i32 %3, 0
@@ -2199,7 +2199,7 @@ define hidden void @scale_uv_row_down2_linear(ptr nocapture noundef readonly %0,
; CHECK: Cost of 6 for VF 2: REPLICATE ir<%20> = load ir<%19>
; CHECK: Cost of 6 for VF 2: REPLICATE ir<%23> = load ir<%22>
; CHECK: Cost of 6 for VF 2: REPLICATE store ir<%28>, ir<%29>
-; CHECK: Cost for VF 2: 53 (Estimated cost per lane: 26.5)
+; CHECK: Cost for VF 2: 51 (Estimated cost per lane: 25.5)
; CHECK: Cost of 18 for VF 4: INTERLEAVE-GROUP with factor 4, vp<%next.gep>
; CHECK: ir<%10> = load from index 0
; CHECK: ir<%20> = load from index 1
@@ -2208,7 +2208,7 @@ define hidden void @scale_uv_row_down2_linear(ptr nocapture noundef readonly %0,
; CHECK: Cost of 11 for VF 4: INTERLEAVE-GROUP with factor 2, vp<%next.gep>.1
; CHECK: store ir<%18> to index 0
; CHECK: store ir<%28> to index 1
-; CHECK: Cost for VF 4: 48 (Estimated cost per lane: 12)
+; CHECK: Cost for VF 4: 46 (Estimated cost per lane: 11.5)
; CHECK: Cost of 26 for VF 8: INTERLEAVE-GROUP with factor 4, vp<%next.gep>
; CHECK: ir<%10> = load from index 0
; CHECK: ir<%20> = load from index 1
@@ -2217,7 +2217,7 @@ define hidden void @scale_uv_row_down2_linear(ptr nocapture noundef readonly %0,
; CHECK: Cost of 7 for VF 8: INTERLEAVE-GROUP with factor 2, vp<%next.gep>.1
; CHECK: store ir<%18> to index 0
; CHECK: store ir<%28> to index 1
-; CHECK: Cost for VF 8: 56 (Estimated cost per lane: 7)
+; CHECK: Cost for VF 8: 54 (Estimated cost per lane: 6.75)
; CHECK: Cost of 132 for VF 16: INTERLEAVE-GROUP with factor 4, vp<%next.gep>
; CHECK: ir<%10> = load from index 0
; CHECK: ir<%20> = load from index 1
@@ -2226,7 +2226,7 @@ define hidden void @scale_uv_row_down2_linear(ptr nocapture noundef readonly %0,
; CHECK: Cost of 6 for VF 16: INTERLEAVE-GROUP with factor 2, vp<%next.gep>.1
; CHECK: store ir<%18> to index 0
; CHECK: store ir<%28> to index 1
-; CHECK: Cost for VF 16: 175 (Estimated cost per lane: 10.9)
+; CHECK: Cost for VF 16: 173 (Estimated cost per lane: 10.8)
; CHECK: LV: Selecting VF: 8.
;
%5 = icmp sgt i32 %3, 0
diff --git a/llvm/test/Transforms/LoopVectorize/X86/CostModel/vpinstruction-cost.ll b/llvm/test/Transforms/LoopVectorize/X86/CostModel/vpinstruction-cost.ll
index 84cc4631202cd..be1b6b42496b1 100644
--- a/llvm/test/Transforms/LoopVectorize/X86/CostModel/vpinstruction-cost.ll
+++ b/llvm/test/Transforms/LoopVectorize/X86/CostModel/vpinstruction-cost.ll
@@ -189,8 +189,6 @@ define void @test_vpinstruction_switch_cost(ptr %start, ptr %end) {
; CHECK: LV: Found an estimated cost of 0 for VF 1 For instruction: %ptr.iv.next = getelementptr inbounds i64, ptr %ptr.iv, i64 1
; CHECK: LV: Found an estimated cost of 1 for VF 1 For instruction: %ec = icmp eq ptr %ptr.iv.next, %end
; CHECK: LV: Found an estimated cost of 0 for VF 1 For instruction: br i1 %ec, label %exit, label %loop.header
-; CHECK: Cost of 0 for VF 2: induction instruction %ptr.iv.next = getelementptr inbounds i64, ptr %ptr.iv, i64 1
-; CHECK: Cost of 0 for VF 2: induction instruction %ptr.iv = phi ptr [ %start, %entry ], [ %ptr.iv.next, %loop.latch ]
; CHECK: Cost of 1 for VF 2: vp<[[VP6:%[0-9]+]]> = DERIVED-IV ir<0> + vp<[[VP5:%[0-9]+]]> * ir<8>
; CHECK: Cost of 0 for VF 2: vp<[[VP7:%[0-9]+]]> = SCALAR-STEPS vp<[[VP6]]>, ir<8>, vp<[[VP0:%[0-9]+]]>
; CHECK: Cost of 0 for VF 2: EMIT vp<%next.gep> = ptradd ir<%start>, vp<[[VP7]]>
@@ -221,8 +219,6 @@ define void @test_vpinstruction_switch_cost(ptr %start, ptr %end) {
; CHECK: Cost of 0 for VF 2: vp<[[VP4]]> = DERIVED-IV ir<%start> + vp<[[VP2]]> * ir<8>
; CHECK: Cost of 1 for VF 2: EMIT vp<%cmp.n> = icmp eq vp<[[VP3]]>, vp<[[VP2]]>
; CHECK: Cost of 0 for VF 2: EMIT branch-on-cond vp<%cmp.n>
-; CHECK: Cost of 0 for VF 4: induction instruction %ptr.iv.next = getelementptr inbounds i64, ptr %ptr.iv, i64 1
-; CHECK: Cost of 0 for VF 4: induction instruction %ptr.iv = phi ptr [ %start, %entry ], [ %ptr.iv.next, %loop.latch ]
; CHECK: Cost of 1 for VF 4: vp<[[VP6]]> = DERIVED-IV ir<0> + vp<[[VP5]]> * ir<8>
; CHECK: Cost of 0 for VF 4: vp<[[VP7]]> = SCALAR-STEPS vp<[[VP6]]>, ir<8>, vp<[[VP0]]>
; CHECK: Cost of 0 for VF 4: EMIT vp<%next.gep> = ptradd ir<%start>, vp<[[VP7]]>
diff --git a/llvm/test/Transforms/LoopVectorize/induction-cost.ll b/llvm/test/Transforms/LoopVectorize/induction-cost.ll
index c5cd7ad3ce055..64fe4252b89c3 100644
--- a/llvm/test/Transforms/LoopVectorize/induction-cost.ll
+++ b/llvm/test/Transforms/LoopVectorize/induction-cost.ll
@@ -1,4 +1,4 @@
-; NOTE: Assertions have been autogenerated by utils/update_analyze_test_checks.py UTC_ARGS: --filter "(Cost of .*(WIDEN-INDUCTION|SCALAR-STEPS|DERIVED-IV|WIDEN-POINTER-INDUCTION))|(canonical IV increment)|(induction instruction)|(Cost for VF)" --filter-out-after "LV: (Selecting|Using user) VF" --version 6
+; NOTE: Assertions have been autogenerated by utils/update_analyze_test_checks.py UTC_ARGS: --filter "(Cost of .*(WIDEN-INDUCTION|SCALAR-STEPS|DERIVED-IV|WIDEN-POINTER-INDUCTION))|(canonical IV increment)|(Cost for VF)" --filter-out-after "LV: (Selecting|Using user) VF" --version 6
; REQUIRES: asserts
; RUN: opt -passes=loop-vectorize -force-vector-width=2 -force-vector-interleave=1 -debug-only=loop-vectorize -disable-output %s 2>&1 | FileCheck --check-prefix=VF2 %s
; RUN: opt -passes=loop-vectorize -force-vector-width=4 -force-vector-interleave=1 -debug-only=loop-vectorize -disable-output %s 2>&1 | FileCheck --check-prefix=VF4 %s
@@ -655,31 +655,27 @@ exit:
ret void
}
-; The loop is advanced by an FP induction, so the canonical IV increment is
-; accounted for, even though the pointer induction's increment is not part of
-; the scalar cost we compare against.
+; The loop is advanced by both an FP and a pointer induction. Neither induction
+; is costed by the legacy cost model any more; the scalarized pointer induction
+; is covered by its DERIVED-IV and SCALAR-STEPS recipes.
define void @fp_and_ptr_induction(ptr noalias %dst, ptr %end) {
; VF2-LABEL: 'fp_and_ptr_induction'
-; VF2: Cost of 0 for VF 2: induction instruction %ptr.iv.next = getelementptr inbounds float, ptr %ptr.iv, i64 1
-; VF2: Cost of 1 for VF 2: induction instruction %ptr.iv = phi ptr [ %dst, %entry ], [ %ptr.iv.next, %loop ]
; VF2: Cost of 2 for VF 2: ir<%fp.iv> = WIDEN-INDUCTION fast ir<0.000000e+00>, ir<1.000000e+00>, vp<[[VP0:%[0-9]+]]>
; VF2: Cost of 1 for VF 2: vp<[[VP7:%[0-9]+]]> = DERIVED-IV ir<0> + vp<[[VP6:%[0-9]+]]> * ir<4>
; VF2: Cost of 0 for VF 2: vp<[[VP8:%[0-9]+]]> = SCALAR-STEPS vp<[[VP7]]>, ir<4>, vp<[[VP0]]>
; VF2: Cost of 1 for VF 2: canonical IV increment
; VF2: Cost of 0 for VF 2: vp<[[VP4:%[0-9]+]]> = DERIVED-IV ir<%dst> + vp<[[VP2:%[0-9]+]]> * ir<4>
; VF2: Cost of 1 for VF 2: vp<[[VP5:%[0-9]+]]> = DERIVED-IV ir<0.000000e+00> + vp<[[VP2]]> * ir<1.000000e+00>
-; VF2: Cost for VF 2: 8 (Estimated cost per lane: 4)
+; VF2: Cost for VF 2: 7 (Estimated cost per lane: 3.5)
;
; VF4-LABEL: 'fp_and_ptr_induction'
-; VF4: Cost of 0 for VF 4: induction instruction %ptr.iv.next = getelementptr inbounds float, ptr %ptr.iv, i64 1
-; VF4: Cost of 1 for VF 4: induction instruction %ptr.iv = phi ptr [ %dst, %entry ], [ %ptr.iv.next, %loop ]
; VF4: Cost of 2 for VF 4: ir<%fp.iv> = WIDEN-INDUCTION fast ir<0.000000e+00>, ir<1.000000e+00>, vp<[[VP0:%[0-9]+]]>
; VF4: Cost of 1 for VF 4: vp<[[VP7:%[0-9]+]]> = DERIVED-IV ir<0> + vp<[[VP6:%[0-9]+]]> * ir<4>
; VF4: Cost of 0 for VF 4: vp<[[VP8:%[0-9]+]]> = SCALAR-STEPS vp<[[VP7]]>, ir<4>, vp<[[VP0]]>
; VF4: Cost of 1 for VF 4: canonical IV increment
; VF4: Cost of 0 for VF 4: vp<[[VP4:%[0-9]+]]> = DERIVED-IV ir<%dst> + vp<[[VP2:%[0-9]+]]> * ir<4>
; VF4: Cost of 1 for VF 4: vp<[[VP5:%[0-9]+]]> = DERIVED-IV ir<0.000000e+00> + vp<[[VP2]]> * ir<1.000000e+00>
-; VF4: Cost for VF 4: 8 (Estimated cost per lane: 2)
+; VF4: Cost for VF 4: 7 (Estimated cost per lane: 1.75)
;
entry:
br label %loop
@@ -697,25 +693,21 @@ exit:
ret void
}
-; Negative test for @fp_and_ptr_induction: the loop is only advanced by a
-; pointer induction, whose increment is a GEP that the scalar cost charges
-; nothing for, so the canonical IV increment must not be charged either.
+; Same as @fp_and_ptr_induction, but the loop is only advanced by a pointer
+; induction. The canonical IV increment is still charged, even though the GEP
+; incrementing the pointer induction is free in the scalar cost.
define void @ptr_induction_only(ptr %start, ptr %end) {
; VF2-LABEL: 'ptr_induction_only'
-; VF2: Cost of 0 for VF 2: induction instruction %ptr.iv.next = getelementptr inbounds i8, ptr %ptr.iv, i64 1
-; VF2: Cost of 1 for VF 2: induction instruction %ptr.iv = phi ptr [ %start, %entry ], [ %ptr.iv.next, %loop ]
; VF2: Cost of 0 for VF 2: vp<[[VP6:%[0-9]+]]> = SCALAR-STEPS vp<[[VP5:%[0-9]+]]>, ir<1>, vp<[[VP0:%[0-9]+]]>
; VF2: Cost of 1 for VF 2: canonical IV increment
; VF2: Cost of 0 for VF 2: vp<[[VP4:%[0-9]+]]> = DERIVED-IV ir<%start> + vp<[[VP2:%[0-9]+]]> * ir<1>
-; VF2: Cost for VF 2: 7 (Estimated cost per lane: 3.5)
+; VF2: Cost for VF 2: 6 (Estimated cost per lane: 3)
;
; VF4-LABEL: 'ptr_induction_only'
-; VF4: Cost of 0 for VF 4: induction instruction %ptr.iv.next = getelementptr inbounds i8, ptr %ptr.iv, i64 1
-; VF4: Cost of 1 for VF 4: induction instruction %ptr.iv = phi ptr [ %start, %entry ], [ %ptr.iv.next, %loop ]
; VF4: Cost of 0 for VF 4: vp<[[VP6:%[0-9]+]]> = SCALAR-STEPS vp<[[VP5:%[0-9]+]]>, ir<1>, vp<[[VP0:%[0-9]+]]>
; VF4: Cost of 1 for VF 4: canonical IV increment
; VF4: Cost of 0 for VF 4: vp<[[VP4:%[0-9]+]]> = DERIVED-IV ir<%start> + vp<[[VP2:%[0-9]+]]> * ir<1>
-; VF4: Cost for VF 4: 7 (Estimated cost per lane: 1.75)
+; VF4: Cost for VF 4: 6 (Estimated cost per lane: 1.5)
;
entry:
br label %loop
@@ -735,22 +727,18 @@ exit:
define void @ptr_induction(ptr noalias %dst, ptr %end) {
; VF2-LABEL: 'ptr_induction'
-; VF2: Cost of 0 for VF 2: induction instruction %ptr.iv.next = getelementptr inbounds i64, ptr %ptr.iv, i64 1
-; VF2: Cost of 1 for VF 2: induction instruction %ptr.iv = phi ptr [ %dst, %entry ], [ %ptr.iv.next, %loop ]
; VF2: Cost of 1 for VF 2: vp<[[VP6:%[0-9]+]]> = DERIVED-IV ir<0> + vp<[[VP5:%[0-9]+]]> * ir<8>
; VF2: Cost of 0 for VF 2: vp<[[VP7:%[0-9]+]]> = SCALAR-STEPS vp<[[VP6]]>, ir<8>, vp<[[VP0:%[0-9]+]]>
; VF2: Cost of 1 for VF 2: canonical IV increment
; VF2: Cost of 0 for VF 2: vp<[[VP4:%[0-9]+]]> = DERIVED-IV ir<%dst> + vp<[[VP2:%[0-9]+]]> * ir<8>
-; VF2: Cost for VF 2: 6 (Estimated cost per lane: 3)
+; VF2: Cost for VF 2: 5 (Estimated cost per lane: 2.5)
;
; VF4-LABEL: 'ptr_induction'
-; VF4: Cost of 0 for VF 4: induction instruction %ptr.iv.next = getelementptr inbounds i64, ptr %ptr.iv, i64 1
-; VF4: Cost of 1 for VF 4: induction instruction %ptr.iv = phi ptr [ %dst, %entry ], [ %ptr.iv.next, %loop ]
; VF4: Cost of 1 for VF 4: vp<[[VP6:%[0-9]+]]> = DERIVED-IV ir<0> + vp<[[VP5:%[0-9]+]]> * ir<8>
; VF4: Cost of 0 for VF 4: vp<[[VP7:%[0-9]+]]> = SCALAR-STEPS vp<[[VP6]]>, ir<8>, vp<[[VP0:%[0-9]+]]>
; VF4: Cost of 1 for VF 4: canonical IV increment
; VF4: Cost of 0 for VF 4: vp<[[VP4:%[0-9]+]]> = DERIVED-IV ir<%dst> + vp<[[VP2:%[0-9]+]]> * ir<8>
-; VF4: Cost for VF 4: 6 (Estimated cost per lane: 1.5)
+; VF4: Cost for VF 4: 5 (Estimated cost per lane: 1.25)
;
entry:
br label %loop
@@ -771,13 +759,11 @@ exit:
; costed.
define void @ptr_induction_tc_eq_vf(ptr noalias %dst) {
; VF2-LABEL: 'ptr_induction_tc_eq_vf'
-; VF2: Cost of 0 for VF 2: induction instruction %ptr.iv.next = getelementptr inbounds i64, ptr %ptr.iv, i64 1
-; VF2: Cost of 1 for VF 2: induction instruction %ptr.iv = phi ptr [ %dst, %entry ], [ %ptr.iv.next, %loop ]
; VF2: Cost of 1 for VF 2: vp<[[VP5:%[0-9]+]]> = DERIVED-IV ir<0> + vp<[[VP4:%[0-9]+]]> * ir<8>
; VF2: Cost of 0 for VF 2: vp<[[VP6:%[0-9]+]]> = SCALAR-STEPS vp<[[VP5]]>, ir<8>, vp<[[VP0:%[0-9]+]]>
; VF2: Cost of 1 for VF 2: canonical IV increment
; VF2: Cost of 0 for VF 2: vp<[[VP3:%[0-9]+]]> = DERIVED-IV ir<%dst> + vp<[[VP2:%[0-9]+]]> * ir<8>
-; VF2: Cost for VF 2: 6 (Estimated cost per lane: 3)
+; VF2: Cost for VF 2: 5 (Estimated cost per lane: 2.5)
;
; VF4-LABEL: 'ptr_induction_tc_eq_vf'
; VF4: Cost of 1 for VF 4: vp<[[VP5:%[0-9]+]]> = DERIVED-IV ir<0> + vp<[[VP4:%[0-9]+]]> * ir<8>
@@ -804,30 +790,22 @@ exit:
; Test case where %ptr.iv remains a wide pointer induction, while %arr.iv is scalarized.
define void @ptr_induction_widened(ptr noalias %dst, ptr noalias %arr, ptr %end) {
; VF2-LABEL: 'ptr_induction_widened'
-; VF2: Cost of 0 for VF 2: induction instruction %ptr.iv.next = getelementptr inbounds i64, ptr %ptr.iv, i64 1
-; VF2: Cost of 1 for VF 2: induction instruction %ptr.iv = phi ptr [ %dst, %entry ], [ %ptr.iv.next, %loop ]
-; VF2: Cost of 0 for VF 2: induction instruction %arr.iv.next = getelementptr inbounds ptr, ptr %arr.iv, i64 1
-; VF2: Cost of 1 for VF 2: induction instruction %arr.iv = phi ptr [ %arr, %entry ], [ %arr.iv.next, %loop ]
-; VF2: Cost of 0 for VF 2: EMIT ir<%ptr.iv> = WIDEN-POINTER-INDUCTION ir<%dst>, ir<8>, vp<[[VP1:%[0-9]+]]>
+; VF2: Cost of 1 for VF 2: EMIT ir<%ptr.iv> = WIDEN-POINTER-INDUCTION ir<%dst>, ir<8>, vp<[[VP1:%[0-9]+]]>
; VF2: Cost of 1 for VF 2: vp<[[VP7:%[0-9]+]]> = DERIVED-IV ir<0> + vp<[[VP6:%[0-9]+]]> * ir<8>
; VF2: Cost of 0 for VF 2: vp<[[VP8:%[0-9]+]]> = SCALAR-STEPS vp<[[VP7]]>, ir<8>, vp<[[VP0:%[0-9]+]]>
; VF2: Cost of 1 for VF 2: canonical IV increment
; VF2: Cost of 0 for VF 2: vp<[[VP4:%[0-9]+]]> = DERIVED-IV ir<%dst> + vp<[[VP2:%[0-9]+]]> * ir<8>
; VF2: Cost of 0 for VF 2: vp<[[VP5:%[0-9]+]]> = DERIVED-IV ir<%arr> + vp<[[VP2]]> * ir<8>
-; VF2: Cost for VF 2: 8 (Estimated cost per lane: 4)
+; VF2: Cost for VF 2: 7 (Estimated cost per lane: 3.5)
;
; VF4-LABEL: 'ptr_induction_widened'
-; VF4: Cost of 0 for VF 4: induction instruction %ptr.iv.next = getelementptr inbounds i64, ptr %ptr.iv, i64 1
-; VF4: Cost of 1 for VF 4: induction instruction %ptr.iv = phi ptr [ %dst, %entry ], [ %ptr.iv.next, %loop ]
-; VF4: Cost of 0 for VF 4: induction instruction %arr.iv.next = getelementptr inbounds ptr, ptr %arr.iv, i64 1
-; VF4: Cost of 1 for VF 4: induction instruction %arr.iv = phi ptr [ %arr, %entry ], [ %arr.iv.next, %loop ]
-; VF4: Cost of 0 for VF 4: EMIT ir<%ptr.iv> = WIDEN-POINTER-INDUCTION ir<%dst>, ir<8>, vp<[[VP1:%[0-9]+]]>
+; VF4: Cost of 1 for VF 4: EMIT ir<%ptr.iv> = WIDEN-POINTER-INDUCTION ir<%dst>, ir<8>, vp<[[VP1:%[0-9]+]]>
; VF4: Cost of 1 for VF 4: vp<[[VP7:%[0-9]+]]> = DERIVED-IV ir<0> + vp<[[VP6:%[0-9]+]]> * ir<8>
; VF4: Cost of 0 for VF 4: vp<[[VP8:%[0-9]+]]> = SCALAR-STEPS vp<[[VP7]]>, ir<8>, vp<[[VP0:%[0-9]+]]>
; VF4: Cost of 1 for VF 4: canonical IV increment
; VF4: Cost of 0 for VF 4: vp<[[VP4:%[0-9]+]]> = DERIVED-IV ir<%dst> + vp<[[VP2:%[0-9]+]]> * ir<8>
; VF4: Cost of 0 for VF 4: vp<[[VP5:%[0-9]+]]> = DERIVED-IV ir<%arr> + vp<[[VP2]]> * ir<8>
-; VF4: Cost for VF 4: 8 (Estimated cost per lane: 2)
+; VF4: Cost for VF 4: 7 (Estimated cost per lane: 1.75)
;
entry:
br label %loop
diff --git a/llvm/test/Transforms/LoopVectorize/phi-cost.ll b/llvm/test/Transforms/LoopVectorize/phi-cost.ll
index 929467d5ecc28..a450830881f31 100644
--- a/llvm/test/Transforms/LoopVectorize/phi-cost.ll
+++ b/llvm/test/Transforms/LoopVectorize/phi-cost.ll
@@ -6,7 +6,7 @@
target datalayout = "e-m:e-i64:64-i128:128-n32:64-S128"
; DEBUG-LABEL: phi_two_incoming_values
-; DEBUG: Cost of 1 for VF 2: induction instruction %i = phi i64 [ %i.next, %if.end ], [ 0, %entry ]
+; DEBUG: Cost of 0 for VF 2: vp<{{.+}}> = SCALAR-STEPS vp<{{.+}}>, ir<1>, vp<{{.+}}>
; DEBUG: Cost of 1 for VF 2: BLEND ir<%tmp5> = ir<%tmp1> ir<%tmp4>/ir<%tmp3>
;
define void @phi_two_incoming_values(ptr noalias %a, ptr noalias %b, i64 %n) {
@@ -84,7 +84,7 @@ for.end:
}
; DEBUG-LABEL: phi_three_incoming_values
-; DEBUG: Cost of 1 for VF 2: induction instruction %i = phi i64 [ %i.next, %if.end ], [ 0, %entry ]
+; DEBUG: Cost of 0 for VF 2: vp<{{.+}}> = SCALAR-STEPS vp<{{.+}}>, ir<1>, vp<{{.+}}>
; DEBUG: Cost of 2 for VF 2: BLEND ir<%tmp8> = ir<%tmp7> ir<3>/vp<{{.*}}> ir<9>/vp<{{.*}}>
;
define void @phi_three_incoming_values(ptr noalias %a, ptr noalias %b, i64 %n) {
More information about the llvm-commits
mailing list