[llvm] [VPlan] Consider VPDerivedIV & VPScalarIVSteps for forced costs. (PR #222918)
via llvm-commits
llvm-commits at lists.llvm.org
Fri Sep 11 03:58:10 PDT 2026
llvmorg-github-actions[bot] wrote:
<!--LLVM PR SUMMARY COMMENT-->
@llvm/pr-subscribers-vectorizers
Author: Florian Hahn (fhahn)
<details>
<summary>Changes</summary>
Now that we compute VPDerivedIV and VPScalarIVSteps costs directly in VPlan, also account for them with forced costs. They never have underlying instructions, so always add the forced cost for them.
Previously they would have been accounted for in precomputeCosts.
---
Patch is 55.98 KiB, truncated to 20.00 KiB below, full version: https://github.com/llvm/llvm-project/pull/222918.diff
4 Files Affected:
- (modified) llvm/lib/Transforms/Vectorize/VPlanRecipes.cpp (+3-1)
- (modified) llvm/test/Transforms/LoopVectorize/AArch64/force-target-instruction-cost.ll (+392-91)
- (modified) llvm/test/Transforms/LoopVectorize/AArch64/scalable-vectorization-cost-tuning.ll (+12-12)
- (modified) llvm/test/Transforms/LoopVectorize/force-instruction-cost.ll (+6-6)
``````````diff
diff --git a/llvm/lib/Transforms/Vectorize/VPlanRecipes.cpp b/llvm/lib/Transforms/Vectorize/VPlanRecipes.cpp
index 049e8b70c70d8..02c570d0437bb 100644
--- a/llvm/lib/Transforms/Vectorize/VPlanRecipes.cpp
+++ b/llvm/lib/Transforms/Vectorize/VPlanRecipes.cpp
@@ -329,7 +329,9 @@ InstructionCost VPRecipeBase::cost(ElementCount VF, VPCostContext &Ctx) {
RecipeCost = computeCost(VF, Ctx);
if (ForceTargetInstructionCost.getNumOccurrences() > 0 &&
RecipeCost.isValid()) {
- if (UI)
+ // VPDerivedIVRecipe and VPScalarIVStepsRecipe never have underlying
+ // instructions.
+ if (UI || isa<VPDerivedIVRecipe, VPScalarIVStepsRecipe>(this))
RecipeCost = InstructionCost(ForceTargetInstructionCost);
else
RecipeCost = InstructionCost(0);
diff --git a/llvm/test/Transforms/LoopVectorize/AArch64/force-target-instruction-cost.ll b/llvm/test/Transforms/LoopVectorize/AArch64/force-target-instruction-cost.ll
index 37c86b6dc0607..618f349ed798f 100644
--- a/llvm/test/Transforms/LoopVectorize/AArch64/force-target-instruction-cost.ll
+++ b/llvm/test/Transforms/LoopVectorize/AArch64/force-target-instruction-cost.ll
@@ -690,26 +690,21 @@ define void @force_branch_cost(ptr readonly %src, ptr %dst) {
; COST1-LABEL: define void @force_branch_cost(
; COST1-SAME: ptr readonly [[SRC:%.*]], ptr [[DST:%.*]]) {
; COST1-NEXT: [[ENTRY:.*:]]
-; COST1-NEXT: br label %[[VECTOR_MEMCHECK:.*]]
+; COST1-NEXT: br i1 false, label %[[SCALAR_PH:.*]], label %[[VECTOR_MEMCHECK:.*]]
; COST1: [[VECTOR_MEMCHECK]]:
; COST1-NEXT: [[SCEVGEP:%.*]] = getelementptr i8, ptr [[DST]], i64 2092
; COST1-NEXT: [[SCEVGEP1:%.*]] = getelementptr i8, ptr [[SRC]], i64 85
; COST1-NEXT: [[BOUND0:%.*]] = icmp ult ptr [[DST]], [[SCEVGEP1]]
; COST1-NEXT: [[BOUND1:%.*]] = icmp ult ptr [[SRC]], [[SCEVGEP]]
; COST1-NEXT: [[FOUND_CONFLICT:%.*]] = and i1 [[BOUND0]], [[BOUND1]]
-; COST1-NEXT: br i1 [[FOUND_CONFLICT]], label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]]
+; COST1-NEXT: br i1 [[FOUND_CONFLICT]], label %[[SCALAR_PH]], label %[[VECTOR_PH:.*]]
; COST1: [[VECTOR_PH]]:
+; COST1-NEXT: br i1 false, label %[[VEC_EPILOG_PH:.*]], label %[[VECTOR_PH1:.*]]
+; COST1: [[VECTOR_PH1]]:
; COST1-NEXT: [[TMP0:%.*]] = getelementptr i8, ptr [[DST]], i64 1536
; COST1-NEXT: br label %[[VECTOR_BODY:.*]]
; COST1: [[VECTOR_BODY]]:
-; COST1-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
-; COST1-NEXT: [[TMP2:%.*]] = add i64 [[INDEX]], 1
-; COST1-NEXT: [[TMP3:%.*]] = add i64 [[INDEX]], 2
-; COST1-NEXT: [[TMP4:%.*]] = add i64 [[INDEX]], 3
-; COST1-NEXT: [[TMP5:%.*]] = add i64 [[INDEX]], 4
-; COST1-NEXT: [[TMP6:%.*]] = add i64 [[INDEX]], 5
-; COST1-NEXT: [[TMP7:%.*]] = add i64 [[INDEX]], 6
-; COST1-NEXT: [[TMP8:%.*]] = add i64 [[INDEX]], 7
+; COST1-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH1]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
; COST1-NEXT: [[OFFSET_IDX:%.*]] = mul i64 [[INDEX]], 96
; COST1-NEXT: [[TMP10:%.*]] = add i64 [[OFFSET_IDX]], 96
; COST1-NEXT: [[TMP11:%.*]] = add i64 [[OFFSET_IDX]], 192
@@ -718,6 +713,14 @@ define void @force_branch_cost(ptr readonly %src, ptr %dst) {
; COST1-NEXT: [[TMP14:%.*]] = add i64 [[OFFSET_IDX]], 480
; COST1-NEXT: [[TMP15:%.*]] = add i64 [[OFFSET_IDX]], 576
; COST1-NEXT: [[TMP16:%.*]] = add i64 [[OFFSET_IDX]], 672
+; COST1-NEXT: [[TMP9:%.*]] = add i64 [[OFFSET_IDX]], 768
+; COST1-NEXT: [[TMP85:%.*]] = add i64 [[OFFSET_IDX]], 864
+; COST1-NEXT: [[TMP86:%.*]] = add i64 [[OFFSET_IDX]], 960
+; COST1-NEXT: [[TMP87:%.*]] = add i64 [[OFFSET_IDX]], 1056
+; COST1-NEXT: [[TMP92:%.*]] = add i64 [[OFFSET_IDX]], 1152
+; COST1-NEXT: [[TMP93:%.*]] = add i64 [[OFFSET_IDX]], 1248
+; COST1-NEXT: [[TMP94:%.*]] = add i64 [[OFFSET_IDX]], 1344
+; COST1-NEXT: [[TMP95:%.*]] = add i64 [[OFFSET_IDX]], 1440
; COST1-NEXT: [[NEXT_GEP:%.*]] = getelementptr i8, ptr [[DST]], i64 [[OFFSET_IDX]]
; COST1-NEXT: [[NEXT_GEP2:%.*]] = getelementptr i8, ptr [[DST]], i64 [[TMP10]]
; COST1-NEXT: [[NEXT_GEP3:%.*]] = getelementptr i8, ptr [[DST]], i64 [[TMP11]]
@@ -726,22 +729,174 @@ define void @force_branch_cost(ptr readonly %src, ptr %dst) {
; COST1-NEXT: [[NEXT_GEP6:%.*]] = getelementptr i8, ptr [[DST]], i64 [[TMP14]]
; COST1-NEXT: [[NEXT_GEP7:%.*]] = getelementptr i8, ptr [[DST]], i64 [[TMP15]]
; COST1-NEXT: [[NEXT_GEP8:%.*]] = getelementptr i8, ptr [[DST]], i64 [[TMP16]]
+; COST1-NEXT: [[NEXT_GEP9:%.*]] = getelementptr i8, ptr [[DST]], i64 [[TMP9]]
+; COST1-NEXT: [[NEXT_GEP10:%.*]] = getelementptr i8, ptr [[DST]], i64 [[TMP85]]
+; COST1-NEXT: [[NEXT_GEP11:%.*]] = getelementptr i8, ptr [[DST]], i64 [[TMP86]]
+; COST1-NEXT: [[NEXT_GEP12:%.*]] = getelementptr i8, ptr [[DST]], i64 [[TMP87]]
+; COST1-NEXT: [[NEXT_GEP13:%.*]] = getelementptr i8, ptr [[DST]], i64 [[TMP92]]
+; COST1-NEXT: [[NEXT_GEP14:%.*]] = getelementptr i8, ptr [[DST]], i64 [[TMP93]]
+; COST1-NEXT: [[NEXT_GEP15:%.*]] = getelementptr i8, ptr [[DST]], i64 [[TMP94]]
+; COST1-NEXT: [[NEXT_GEP16:%.*]] = getelementptr i8, ptr [[DST]], i64 [[TMP95]]
; COST1-NEXT: [[TMP17:%.*]] = getelementptr [4 x i8], ptr [[SRC]], i64 [[INDEX]]
-; COST1-NEXT: [[TMP18:%.*]] = getelementptr [4 x i8], ptr [[SRC]], i64 [[TMP2]]
-; COST1-NEXT: [[TMP19:%.*]] = getelementptr [4 x i8], ptr [[SRC]], i64 [[TMP3]]
-; COST1-NEXT: [[TMP20:%.*]] = getelementptr [4 x i8], ptr [[SRC]], i64 [[TMP4]]
+; COST1-NEXT: [[WIDE_VEC:%.*]] = load <64 x i8>, ptr [[TMP17]], align 1, !alias.scope [[META19:![0-9]+]]
+; COST1-NEXT: [[STRIDED_VEC:%.*]] = shufflevector <64 x i8> [[WIDE_VEC]], <64 x i8> poison, <16 x i32> <i32 0, i32 4, i32 8, i32 12, i32 16, i32 20, i32 24, i32 28, i32 32, i32 36, i32 40, i32 44, i32 48, i32 52, i32 56, i32 60>
+; COST1-NEXT: [[TMP18:%.*]] = zext <16 x i8> [[STRIDED_VEC]] to <16 x i32>
+; COST1-NEXT: [[TMP19:%.*]] = extractelement <16 x i32> [[TMP18]], i64 0
+; COST1-NEXT: store i32 [[TMP19]], ptr [[NEXT_GEP]], align 4, !alias.scope [[META22:![0-9]+]], !noalias [[META19]]
+; COST1-NEXT: [[TMP20:%.*]] = extractelement <16 x i32> [[TMP18]], i64 1
+; COST1-NEXT: store i32 [[TMP20]], ptr [[NEXT_GEP2]], align 4, !alias.scope [[META22]], !noalias [[META19]]
+; COST1-NEXT: [[TMP96:%.*]] = extractelement <16 x i32> [[TMP18]], i64 2
+; COST1-NEXT: store i32 [[TMP96]], ptr [[NEXT_GEP3]], align 4, !alias.scope [[META22]], !noalias [[META19]]
+; COST1-NEXT: [[TMP97:%.*]] = extractelement <16 x i32> [[TMP18]], i64 3
+; COST1-NEXT: store i32 [[TMP97]], ptr [[NEXT_GEP4]], align 4, !alias.scope [[META22]], !noalias [[META19]]
+; COST1-NEXT: [[TMP98:%.*]] = extractelement <16 x i32> [[TMP18]], i64 4
+; COST1-NEXT: store i32 [[TMP98]], ptr [[NEXT_GEP5]], align 4, !alias.scope [[META22]], !noalias [[META19]]
+; COST1-NEXT: [[TMP99:%.*]] = extractelement <16 x i32> [[TMP18]], i64 5
+; COST1-NEXT: store i32 [[TMP99]], ptr [[NEXT_GEP6]], align 4, !alias.scope [[META22]], !noalias [[META19]]
+; COST1-NEXT: [[TMP25:%.*]] = extractelement <16 x i32> [[TMP18]], i64 6
+; COST1-NEXT: store i32 [[TMP25]], ptr [[NEXT_GEP7]], align 4, !alias.scope [[META22]], !noalias [[META19]]
+; COST1-NEXT: [[TMP26:%.*]] = extractelement <16 x i32> [[TMP18]], i64 7
+; COST1-NEXT: store i32 [[TMP26]], ptr [[NEXT_GEP8]], align 4, !alias.scope [[META22]], !noalias [[META19]]
+; COST1-NEXT: [[TMP27:%.*]] = extractelement <16 x i32> [[TMP18]], i64 8
+; COST1-NEXT: store i32 [[TMP27]], ptr [[NEXT_GEP9]], align 4, !alias.scope [[META22]], !noalias [[META19]]
+; COST1-NEXT: [[TMP28:%.*]] = extractelement <16 x i32> [[TMP18]], i64 9
+; COST1-NEXT: store i32 [[TMP28]], ptr [[NEXT_GEP10]], align 4, !alias.scope [[META22]], !noalias [[META19]]
+; COST1-NEXT: [[TMP29:%.*]] = extractelement <16 x i32> [[TMP18]], i64 10
+; COST1-NEXT: store i32 [[TMP29]], ptr [[NEXT_GEP11]], align 4, !alias.scope [[META22]], !noalias [[META19]]
+; COST1-NEXT: [[TMP30:%.*]] = extractelement <16 x i32> [[TMP18]], i64 11
+; COST1-NEXT: store i32 [[TMP30]], ptr [[NEXT_GEP12]], align 4, !alias.scope [[META22]], !noalias [[META19]]
+; COST1-NEXT: [[TMP31:%.*]] = extractelement <16 x i32> [[TMP18]], i64 12
+; COST1-NEXT: store i32 [[TMP31]], ptr [[NEXT_GEP13]], align 4, !alias.scope [[META22]], !noalias [[META19]]
+; COST1-NEXT: [[TMP32:%.*]] = extractelement <16 x i32> [[TMP18]], i64 13
+; COST1-NEXT: store i32 [[TMP32]], ptr [[NEXT_GEP14]], align 4, !alias.scope [[META22]], !noalias [[META19]]
+; COST1-NEXT: [[TMP100:%.*]] = extractelement <16 x i32> [[TMP18]], i64 14
+; COST1-NEXT: store i32 [[TMP100]], ptr [[NEXT_GEP15]], align 4, !alias.scope [[META22]], !noalias [[META19]]
+; COST1-NEXT: [[TMP101:%.*]] = extractelement <16 x i32> [[TMP18]], i64 15
+; COST1-NEXT: store i32 [[TMP101]], ptr [[NEXT_GEP16]], align 4, !alias.scope [[META22]], !noalias [[META19]]
+; COST1-NEXT: [[TMP102:%.*]] = getelementptr i8, ptr [[NEXT_GEP]], i64 24
+; COST1-NEXT: [[TMP103:%.*]] = getelementptr i8, ptr [[NEXT_GEP2]], i64 24
+; COST1-NEXT: [[TMP104:%.*]] = getelementptr i8, ptr [[NEXT_GEP3]], i64 24
+; COST1-NEXT: [[TMP105:%.*]] = getelementptr i8, ptr [[NEXT_GEP4]], i64 24
+; COST1-NEXT: [[TMP106:%.*]] = getelementptr i8, ptr [[NEXT_GEP5]], i64 24
+; COST1-NEXT: [[TMP107:%.*]] = getelementptr i8, ptr [[NEXT_GEP6]], i64 24
+; COST1-NEXT: [[TMP41:%.*]] = getelementptr i8, ptr [[NEXT_GEP7]], i64 24
+; COST1-NEXT: [[TMP42:%.*]] = getelementptr i8, ptr [[NEXT_GEP8]], i64 24
+; COST1-NEXT: [[TMP43:%.*]] = getelementptr i8, ptr [[NEXT_GEP9]], i64 24
+; COST1-NEXT: [[TMP44:%.*]] = getelementptr i8, ptr [[NEXT_GEP10]], i64 24
+; COST1-NEXT: [[TMP108:%.*]] = getelementptr i8, ptr [[NEXT_GEP11]], i64 24
+; COST1-NEXT: [[TMP122:%.*]] = getelementptr i8, ptr [[NEXT_GEP12]], i64 24
+; COST1-NEXT: [[TMP123:%.*]] = getelementptr i8, ptr [[NEXT_GEP13]], i64 24
+; COST1-NEXT: [[TMP124:%.*]] = getelementptr i8, ptr [[NEXT_GEP14]], i64 24
+; COST1-NEXT: [[TMP49:%.*]] = getelementptr i8, ptr [[NEXT_GEP15]], i64 24
+; COST1-NEXT: [[TMP125:%.*]] = getelementptr i8, ptr [[NEXT_GEP16]], i64 24
+; COST1-NEXT: store i32 [[TMP19]], ptr [[TMP102]], align 4, !alias.scope [[META22]], !noalias [[META19]]
+; COST1-NEXT: store i32 [[TMP20]], ptr [[TMP103]], align 4, !alias.scope [[META22]], !noalias [[META19]]
+; COST1-NEXT: store i32 [[TMP96]], ptr [[TMP104]], align 4, !alias.scope [[META22]], !noalias [[META19]]
+; COST1-NEXT: store i32 [[TMP97]], ptr [[TMP105]], align 4, !alias.scope [[META22]], !noalias [[META19]]
+; COST1-NEXT: store i32 [[TMP98]], ptr [[TMP106]], align 4, !alias.scope [[META22]], !noalias [[META19]]
+; COST1-NEXT: store i32 [[TMP99]], ptr [[TMP107]], align 4, !alias.scope [[META22]], !noalias [[META19]]
+; COST1-NEXT: store i32 [[TMP25]], ptr [[TMP41]], align 4, !alias.scope [[META22]], !noalias [[META19]]
+; COST1-NEXT: store i32 [[TMP26]], ptr [[TMP42]], align 4, !alias.scope [[META22]], !noalias [[META19]]
+; COST1-NEXT: store i32 [[TMP27]], ptr [[TMP43]], align 4, !alias.scope [[META22]], !noalias [[META19]]
+; COST1-NEXT: store i32 [[TMP28]], ptr [[TMP44]], align 4, !alias.scope [[META22]], !noalias [[META19]]
+; COST1-NEXT: store i32 [[TMP29]], ptr [[TMP108]], align 4, !alias.scope [[META22]], !noalias [[META19]]
+; COST1-NEXT: store i32 [[TMP30]], ptr [[TMP122]], align 4, !alias.scope [[META22]], !noalias [[META19]]
+; COST1-NEXT: store i32 [[TMP31]], ptr [[TMP123]], align 4, !alias.scope [[META22]], !noalias [[META19]]
+; COST1-NEXT: store i32 [[TMP32]], ptr [[TMP124]], align 4, !alias.scope [[META22]], !noalias [[META19]]
+; COST1-NEXT: store i32 [[TMP100]], ptr [[TMP49]], align 4, !alias.scope [[META22]], !noalias [[META19]]
+; COST1-NEXT: store i32 [[TMP101]], ptr [[TMP125]], align 4, !alias.scope [[META22]], !noalias [[META19]]
+; COST1-NEXT: [[TMP51:%.*]] = getelementptr i8, ptr [[NEXT_GEP]], i64 48
+; COST1-NEXT: [[TMP52:%.*]] = getelementptr i8, ptr [[NEXT_GEP2]], i64 48
+; COST1-NEXT: [[TMP53:%.*]] = getelementptr i8, ptr [[NEXT_GEP3]], i64 48
+; COST1-NEXT: [[TMP54:%.*]] = getelementptr i8, ptr [[NEXT_GEP4]], i64 48
+; COST1-NEXT: [[TMP55:%.*]] = getelementptr i8, ptr [[NEXT_GEP5]], i64 48
+; COST1-NEXT: [[TMP56:%.*]] = getelementptr i8, ptr [[NEXT_GEP6]], i64 48
+; COST1-NEXT: [[TMP57:%.*]] = getelementptr i8, ptr [[NEXT_GEP7]], i64 48
+; COST1-NEXT: [[TMP58:%.*]] = getelementptr i8, ptr [[NEXT_GEP8]], i64 48
+; COST1-NEXT: [[TMP59:%.*]] = getelementptr i8, ptr [[NEXT_GEP9]], i64 48
+; COST1-NEXT: [[TMP60:%.*]] = getelementptr i8, ptr [[NEXT_GEP10]], i64 48
+; COST1-NEXT: [[TMP61:%.*]] = getelementptr i8, ptr [[NEXT_GEP11]], i64 48
+; COST1-NEXT: [[TMP62:%.*]] = getelementptr i8, ptr [[NEXT_GEP12]], i64 48
+; COST1-NEXT: [[TMP63:%.*]] = getelementptr i8, ptr [[NEXT_GEP13]], i64 48
+; COST1-NEXT: [[TMP64:%.*]] = getelementptr i8, ptr [[NEXT_GEP14]], i64 48
+; COST1-NEXT: [[TMP65:%.*]] = getelementptr i8, ptr [[NEXT_GEP15]], i64 48
+; COST1-NEXT: [[TMP66:%.*]] = getelementptr i8, ptr [[NEXT_GEP16]], i64 48
+; COST1-NEXT: store i32 [[TMP19]], ptr [[TMP51]], align 4, !alias.scope [[META22]], !noalias [[META19]]
+; COST1-NEXT: store i32 [[TMP20]], ptr [[TMP52]], align 4, !alias.scope [[META22]], !noalias [[META19]]
+; COST1-NEXT: store i32 [[TMP96]], ptr [[TMP53]], align 4, !alias.scope [[META22]], !noalias [[META19]]
+; COST1-NEXT: store i32 [[TMP97]], ptr [[TMP54]], align 4, !alias.scope [[META22]], !noalias [[META19]]
+; COST1-NEXT: store i32 [[TMP98]], ptr [[TMP55]], align 4, !alias.scope [[META22]], !noalias [[META19]]
+; COST1-NEXT: store i32 [[TMP99]], ptr [[TMP56]], align 4, !alias.scope [[META22]], !noalias [[META19]]
+; COST1-NEXT: store i32 [[TMP25]], ptr [[TMP57]], align 4, !alias.scope [[META22]], !noalias [[META19]]
+; COST1-NEXT: store i32 [[TMP26]], ptr [[TMP58]], align 4, !alias.scope [[META22]], !noalias [[META19]]
+; COST1-NEXT: store i32 [[TMP27]], ptr [[TMP59]], align 4, !alias.scope [[META22]], !noalias [[META19]]
+; COST1-NEXT: store i32 [[TMP28]], ptr [[TMP60]], align 4, !alias.scope [[META22]], !noalias [[META19]]
+; COST1-NEXT: store i32 [[TMP29]], ptr [[TMP61]], align 4, !alias.scope [[META22]], !noalias [[META19]]
+; COST1-NEXT: store i32 [[TMP30]], ptr [[TMP62]], align 4, !alias.scope [[META22]], !noalias [[META19]]
+; COST1-NEXT: store i32 [[TMP31]], ptr [[TMP63]], align 4, !alias.scope [[META22]], !noalias [[META19]]
+; COST1-NEXT: store i32 [[TMP32]], ptr [[TMP64]], align 4, !alias.scope [[META22]], !noalias [[META19]]
+; COST1-NEXT: store i32 [[TMP100]], ptr [[TMP65]], align 4, !alias.scope [[META22]], !noalias [[META19]]
+; COST1-NEXT: store i32 [[TMP101]], ptr [[TMP66]], align 4, !alias.scope [[META22]], !noalias [[META19]]
+; COST1-NEXT: [[TMP67:%.*]] = getelementptr i8, ptr [[NEXT_GEP]], i64 72
+; COST1-NEXT: [[TMP68:%.*]] = getelementptr i8, ptr [[NEXT_GEP2]], i64 72
+; COST1-NEXT: [[TMP69:%.*]] = getelementptr i8, ptr [[NEXT_GEP3]], i64 72
+; COST1-NEXT: [[TMP70:%.*]] = getelementptr i8, ptr [[NEXT_GEP4]], i64 72
+; COST1-NEXT: [[TMP71:%.*]] = getelementptr i8, ptr [[NEXT_GEP5]], i64 72
+; COST1-NEXT: [[TMP72:%.*]] = getelementptr i8, ptr [[NEXT_GEP6]], i64 72
+; COST1-NEXT: [[TMP73:%.*]] = getelementptr i8, ptr [[NEXT_GEP7]], i64 72
+; COST1-NEXT: [[TMP74:%.*]] = getelementptr i8, ptr [[NEXT_GEP8]], i64 72
+; COST1-NEXT: [[TMP75:%.*]] = getelementptr i8, ptr [[NEXT_GEP9]], i64 72
+; COST1-NEXT: [[TMP76:%.*]] = getelementptr i8, ptr [[NEXT_GEP10]], i64 72
+; COST1-NEXT: [[TMP77:%.*]] = getelementptr i8, ptr [[NEXT_GEP11]], i64 72
+; COST1-NEXT: [[TMP78:%.*]] = getelementptr i8, ptr [[NEXT_GEP12]], i64 72
+; COST1-NEXT: [[TMP79:%.*]] = getelementptr i8, ptr [[NEXT_GEP13]], i64 72
+; COST1-NEXT: [[TMP80:%.*]] = getelementptr i8, ptr [[NEXT_GEP14]], i64 72
+; COST1-NEXT: [[TMP81:%.*]] = getelementptr i8, ptr [[NEXT_GEP15]], i64 72
+; COST1-NEXT: [[TMP82:%.*]] = getelementptr i8, ptr [[NEXT_GEP16]], i64 72
+; COST1-NEXT: store i32 [[TMP19]], ptr [[TMP67]], align 4, !alias.scope [[META22]], !noalias [[META19]]
+; COST1-NEXT: store i32 [[TMP20]], ptr [[TMP68]], align 4, !alias.scope [[META22]], !noalias [[META19]]
+; COST1-NEXT: store i32 [[TMP96]], ptr [[TMP69]], align 4, !alias.scope [[META22]], !noalias [[META19]]
+; COST1-NEXT: store i32 [[TMP97]], ptr [[TMP70]], align 4, !alias.scope [[META22]], !noalias [[META19]]
+; COST1-NEXT: store i32 [[TMP98]], ptr [[TMP71]], align 4, !alias.scope [[META22]], !noalias [[META19]]
+; COST1-NEXT: store i32 [[TMP99]], ptr [[TMP72]], align 4, !alias.scope [[META22]], !noalias [[META19]]
+; COST1-NEXT: store i32 [[TMP25]], ptr [[TMP73]], align 4, !alias.scope [[META22]], !noalias [[META19]]
+; COST1-NEXT: store i32 [[TMP26]], ptr [[TMP74]], align 4, !alias.scope [[META22]], !noalias [[META19]]
+; COST1-NEXT: store i32 [[TMP27]], ptr [[TMP75]], align 4, !alias.scope [[META22]], !noalias [[META19]]
+; COST1-NEXT: store i32 [[TMP28]], ptr [[TMP76]], align 4, !alias.scope [[META22]], !noalias [[META19]]
+; COST1-NEXT: store i32 [[TMP29]], ptr [[TMP77]], align 4, !alias.scope [[META22]], !noalias [[META19]]
+; COST1-NEXT: store i32 [[TMP30]], ptr [[TMP78]], align 4, !alias.scope [[META22]], !noalias [[META19]]
+; COST1-NEXT: store i32 [[TMP31]], ptr [[TMP79]], align 4, !alias.scope [[META22]], !noalias [[META19]]
+; COST1-NEXT: store i32 [[TMP32]], ptr [[TMP80]], align 4, !alias.scope [[META22]], !noalias [[META19]]
+; COST1-NEXT: store i32 [[TMP100]], ptr [[TMP81]], align 4, !alias.scope [[META22]], !noalias [[META19]]
+; COST1-NEXT: store i32 [[TMP101]], ptr [[TMP82]], align 4, !alias.scope [[META22]], !noalias [[META19]]
+; COST1-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 16
+; COST1-NEXT: [[TMP83:%.*]] = icmp eq i64 [[INDEX_NEXT]], 16
+; COST1-NEXT: br i1 [[TMP83]], label %[[MIDDLE_BLOCK1:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP24:![0-9]+]]
+; COST1: [[MIDDLE_BLOCK1]]:
+; COST1-NEXT: br label %[[VEC_EPILOG_ITER_CHECK:.*]]
+; COST1: [[VEC_EPILOG_ITER_CHECK]]:
+; COST1-NEXT: br i1 false, label %[[SCALAR_PH]], label %[[VEC_EPILOG_PH]], !prof [[PROF25:![0-9]+]]
+; COST1: [[VEC_EPILOG_PH]]:
+; COST1-NEXT: [[VEC_EPILOG_RESUME_VAL:%.*]] = phi i64 [ 16, %[[VEC_EPILOG_ITER_CHECK]] ], [ 0, %[[VECTOR_PH]] ]
+; COST1-NEXT: [[TMP84:%.*]] = getelementptr i8, ptr [[DST]], i64 1920
+; COST1-NEXT: br label %[[VEC_EPILOG_VECTOR_BODY:.*]]
+; COST1: [[VEC_EPILOG_VECTOR_BODY]]:
+; COST1-NEXT: [[TMP5:%.*]] = phi i64 [ [[VEC_EPILOG_RESUME_VAL]], %[[VEC_EPILOG_PH]] ], [ [[INDEX_NEXT22:%.*]], %[[VEC_EPILOG_VECTOR_BODY]] ]
+; COST1-NEXT: [[TMP6:%.*]] = add i64 [[TMP5]], 1
+; COST1-NEXT: [[TMP7:%.*]] = add i64 [[TMP5]], 2
+; COST1-NEXT: [[TMP8:%.*]] = add i64 [[TMP5]], 3
+; COST1-NEXT: [[TMP88:%.*]] = mul i64 [[TMP5]], 96
+; COST1-NEXT: [[TMP89:%.*]] = add i64 [[TMP88]], 96
+; COST1-NEXT: [[TMP90:%.*]] = add i64 [[TMP88]], 192
+; COST1-NEXT: [[TMP91:%.*]] = add i64 [[TMP88]], 288
+; COST1-NEXT: [[NEXT_GEP18:%.*]] = getelementptr i8, ptr [[DST]], i64 [[TMP88]]
+; COST1-NEXT: [[NEXT_GEP19:%.*]] = getelementptr i8, ptr [[DST]], i64 [[TMP89]]
+; COST1-NEXT: [[NEXT_GEP20:%.*]] = getelementptr i8, ptr [[DST]], i64 [[TMP90]]
+; COST1-NEXT: [[NEXT_GEP21:%.*]] = getelementptr i8, ptr [[DST]], i64 [[TMP91]]
; COST1-NEXT: [[TMP21:%.*]] = getelementptr [4 x i8], ptr [[SRC]], i64 [[TMP5]]
; COST1-NEXT: [[TMP22:%.*]] = getelementptr [4 x i8], ptr [[SRC]], i64 [[TMP6]]
; COST1-NEXT: [[TMP23:%.*]] = getelementptr [4 x i8], ptr [[SRC]], i64 [[TMP7]]
; COST1-NEXT: [[TMP24:%.*]] = getelementptr [4 x i8], ptr [[SRC]], i64 [[TMP8]]
-; COST1-NEXT: [[TMP25:%.*]] = load i8, ptr [[TMP17]], align 1, !alias.scope [[META19:![0-9]+]]
-; COST1-NEXT: [[TMP26:%.*]] = load i8, ptr [[TMP18]], align 1, !alias.scope [[META19]]
-; COST1-NEXT: [[TMP27:%.*]] = load i8, ptr [[TMP19]], align 1, !alias.scope [[META19]]
-; COST1-NEXT: [[TMP28:%.*]] = load i8, ptr [[TMP20]], align 1, !alias.scope [[META19]]
-; COST1-NEXT: [[TMP29:%.*]] = insertelement <4 x i8> poison, i8 [[TMP25]], i64 0
-; COST1-NEXT: [[T...
[truncated]
``````````
</details>
https://github.com/llvm/llvm-project/pull/222918
More information about the llvm-commits
mailing list