[llvm] [LV] Improve cost model for some replicating recipes (PR #192056)
via llvm-commits
llvm-commits at lists.llvm.org
Tue Apr 14 06:21:50 PDT 2026
llvmbot wrote:
<!--LLVM PR SUMMARY COMMENT-->
@llvm/pr-subscribers-vectorizers
Author: David Sherwood (david-arm)
<details>
<summary>Changes</summary>
In VPCostContext::getScalarizationOverhead we pessimistically assume that the result always needs scalarizing. By this we mean that the scalar result for each lane will need inserting into the corresponding lane of a vector. However, if the only users of the recipe are VPReplicateRecipes in the same block then the result will remain scalar and no inserts are needed. This has the effect of lowering the cost of some replicating instructions, as seen by the affected tests in this PR.
---
Patch is 22.68 KiB, truncated to 20.00 KiB below, full version: https://github.com/llvm/llvm-project/pull/192056.diff
6 Files Affected:
- (modified) llvm/lib/Transforms/Vectorize/VPlan.cpp (+13-2)
- (modified) llvm/lib/Transforms/Vectorize/VPlanHelpers.h (+1)
- (modified) llvm/lib/Transforms/Vectorize/VPlanRecipes.cpp (+3-2)
- (modified) llvm/test/Transforms/LoopVectorize/AArch64/predicated-costs.ll (+58-9)
- (modified) llvm/test/Transforms/LoopVectorize/AArch64/predication_costs.ll (+4-4)
- (modified) llvm/test/Transforms/LoopVectorize/X86/replicating-load-store-costs.ll (+121-7)
``````````diff
diff --git a/llvm/lib/Transforms/Vectorize/VPlan.cpp b/llvm/lib/Transforms/Vectorize/VPlan.cpp
index 8048c3227a9a5..41a17742d14b8 100644
--- a/llvm/lib/Transforms/Vectorize/VPlan.cpp
+++ b/llvm/lib/Transforms/Vectorize/VPlan.cpp
@@ -1828,7 +1828,8 @@ VPCostContext::getOperandInfo(VPValue *V) const {
InstructionCost VPCostContext::getScalarizationOverhead(
Type *ResultTy, ArrayRef<const VPValue *> Operands, ElementCount VF,
- TTI::VectorInstrContext VIC, bool AlwaysIncludeReplicatingR) {
+ const VPSingleDefRecipe *R, TTI::VectorInstrContext VIC,
+ bool AlwaysIncludeReplicatingR) {
if (VF.isScalar())
return 0;
@@ -1837,7 +1838,17 @@ InstructionCost VPCostContext::getScalarizationOverhead(
InstructionCost ScalarizationCost = 0;
// Compute the cost of scalarizing the result if needed.
- if (!ResultTy->isVoidTy()) {
+ bool ScalarizeResult = !ResultTy->isVoidTy();
+ if (ScalarizeResult && R) {
+ // Is this recipe only used by other replicate recipes in the same block?
+ // If so, the result does not need scalarizing.
+ ScalarizeResult = llvm::any_of(R->users(), [R](VPUser *U) -> bool {
+ auto *URR = dyn_cast<VPReplicateRecipe>(U);
+ return !URR || URR->getParent() != R->getParent();
+ });
+ }
+
+ if (ScalarizeResult) {
for (Type *VectorTy :
to_vector(getContainedTypes(toVectorizedTy(ResultTy, VF)))) {
ScalarizationCost += TTI.getScalarizationOverhead(
diff --git a/llvm/lib/Transforms/Vectorize/VPlanHelpers.h b/llvm/lib/Transforms/Vectorize/VPlanHelpers.h
index b54b80b496668..80cf0c431b67b 100644
--- a/llvm/lib/Transforms/Vectorize/VPlanHelpers.h
+++ b/llvm/lib/Transforms/Vectorize/VPlanHelpers.h
@@ -375,6 +375,7 @@ struct VPCostContext {
/// replicating operands.
InstructionCost getScalarizationOverhead(
Type *ResultTy, ArrayRef<const VPValue *> Operands, ElementCount VF,
+ const VPSingleDefRecipe *R = nullptr,
TTI::VectorInstrContext VIC = TTI::VectorInstrContext::None,
bool AlwaysIncludeReplicatingR = false);
diff --git a/llvm/lib/Transforms/Vectorize/VPlanRecipes.cpp b/llvm/lib/Transforms/Vectorize/VPlanRecipes.cpp
index cbb44198fe362..7ffc9a227d5c7 100644
--- a/llvm/lib/Transforms/Vectorize/VPlanRecipes.cpp
+++ b/llvm/lib/Transforms/Vectorize/VPlanRecipes.cpp
@@ -3534,7 +3534,7 @@ InstructionCost VPReplicateRecipe::computeCost(ElementCount VF,
ScalarCost = ScalarCost * VF.getFixedValue() +
Ctx.getScalarizationOverhead(Ctx.Types.inferScalarType(this),
- to_vector(operands()), VF);
+ to_vector(operands()), VF, this);
// If the recipe is not predicated (i.e. not in a replicate region), return
// the scalar cost. Otherwise handle predicated cost.
if (!getRegion()->isReplicator())
@@ -3620,7 +3620,8 @@ InstructionCost VPReplicateRecipe::computeCost(ElementCount VF,
IsLoad ? TTI::VectorInstrContext::Load : TTI::VectorInstrContext::Store;
InstructionCost Cost =
(ScalarCost * VF.getFixedValue()) +
- Ctx.getScalarizationOverhead(ResultTy, OpsToScalarize, VF, VIC, true);
+ Ctx.getScalarizationOverhead(ResultTy, OpsToScalarize, VF, this, VIC,
+ true);
const VPRegionBlock *ParentRegion = getRegion();
if (ParentRegion && ParentRegion->isReplicator()) {
diff --git a/llvm/test/Transforms/LoopVectorize/AArch64/predicated-costs.ll b/llvm/test/Transforms/LoopVectorize/AArch64/predicated-costs.ll
index d96a96836bd3a..7de338a3a6b5f 100644
--- a/llvm/test/Transforms/LoopVectorize/AArch64/predicated-costs.ll
+++ b/llvm/test/Transforms/LoopVectorize/AArch64/predicated-costs.ll
@@ -380,22 +380,70 @@ exit:
define void @sdiv_power_of_2_divisor_in_replicate_region(i32 %x, ptr %dst, i64 %n) {
; CHECK-LABEL: define void @sdiv_power_of_2_divisor_in_replicate_region(
; CHECK-SAME: i32 [[X:%.*]], ptr [[DST:%.*]], i64 [[N:%.*]]) {
-; CHECK-NEXT: [[ENTRY:.*]]:
+; CHECK-NEXT: [[ENTRY:.*:]]
; CHECK-NEXT: [[C_1:%.*]] = icmp sgt i64 [[N]], 0
; CHECK-NEXT: [[C_2:%.*]] = icmp slt i64 [[N]], 9
; CHECK-NEXT: call void @llvm.assume(i1 [[C_1]])
; CHECK-NEXT: call void @llvm.assume(i1 [[C_2]])
; CHECK-NEXT: br label %[[LOOP:.*]]
; CHECK: [[LOOP]]:
-; CHECK-NEXT: [[IV:%.*]] = phi i64 [ 0, %[[ENTRY]] ], [ [[IV_NEXT:%.*]], %[[LOOP]] ]
+; CHECK-NEXT: [[N_RND_UP:%.*]] = add i64 [[N]], 3
+; CHECK-NEXT: [[N_MOD_VF:%.*]] = urem i64 [[N_RND_UP]], 4
+; CHECK-NEXT: [[N_VEC:%.*]] = sub i64 [[N_RND_UP]], [[N_MOD_VF]]
+; CHECK-NEXT: [[TRIP_COUNT_MINUS_1:%.*]] = sub i64 [[N]], 1
+; CHECK-NEXT: [[BROADCAST_SPLATINSERT:%.*]] = insertelement <4 x i64> poison, i64 [[TRIP_COUNT_MINUS_1]], i64 0
+; CHECK-NEXT: [[BROADCAST_SPLAT:%.*]] = shufflevector <4 x i64> [[BROADCAST_SPLATINSERT]], <4 x i64> poison, <4 x i32> zeroinitializer
+; CHECK-NEXT: br label %[[VECTOR_BODY:.*]]
+; CHECK: [[VECTOR_BODY]]:
+; CHECK-NEXT: [[IV:%.*]] = phi i64 [ 0, %[[LOOP]] ], [ [[INDEX_NEXT:%.*]], %[[PRED_STORE_CONTINUE6:.*]] ]
+; CHECK-NEXT: [[VEC_IND:%.*]] = phi <4 x i64> [ <i64 0, i64 1, i64 2, i64 3>, %[[LOOP]] ], [ [[VEC_IND_NEXT:%.*]], %[[PRED_STORE_CONTINUE6]] ]
+; CHECK-NEXT: [[TMP0:%.*]] = icmp ule <4 x i64> [[VEC_IND]], [[BROADCAST_SPLAT]]
+; CHECK-NEXT: [[TMP1:%.*]] = extractelement <4 x i1> [[TMP0]], i32 0
+; CHECK-NEXT: br i1 [[TMP1]], label %[[PRED_STORE_IF:.*]], label %[[PRED_STORE_CONTINUE:.*]]
+; CHECK: [[PRED_STORE_IF]]:
; CHECK-NEXT: [[DIV:%.*]] = sdiv i32 99, [[X]]
-; CHECK-NEXT: [[DIV2:%.*]] = sdiv i32 [[DIV]], 2
; CHECK-NEXT: [[GEP:%.*]] = getelementptr i32, ptr [[DST]], i64 [[IV]]
+; CHECK-NEXT: [[DIV2:%.*]] = sdiv i32 [[DIV]], 2
; CHECK-NEXT: store i32 [[DIV2]], ptr [[GEP]], align 4
-; CHECK-NEXT: [[IV_NEXT]] = add i64 [[IV]], 1
-; CHECK-NEXT: [[DONE:%.*]] = icmp eq i64 [[IV_NEXT]], [[N]]
-; CHECK-NEXT: br i1 [[DONE]], label %[[EXIT:.*]], label %[[LOOP]]
+; CHECK-NEXT: br label %[[PRED_STORE_CONTINUE]]
+; CHECK: [[PRED_STORE_CONTINUE]]:
+; CHECK-NEXT: [[TMP5:%.*]] = extractelement <4 x i1> [[TMP0]], i32 1
+; CHECK-NEXT: br i1 [[TMP5]], label %[[PRED_STORE_IF1:.*]], label %[[PRED_STORE_CONTINUE2:.*]]
+; CHECK: [[PRED_STORE_IF1]]:
+; CHECK-NEXT: [[TMP6:%.*]] = sdiv i32 99, [[X]]
+; CHECK-NEXT: [[IV_NEXT:%.*]] = add i64 [[IV]], 1
+; CHECK-NEXT: [[TMP8:%.*]] = getelementptr i32, ptr [[DST]], i64 [[IV_NEXT]]
+; CHECK-NEXT: [[TMP9:%.*]] = sdiv i32 [[TMP6]], 2
+; CHECK-NEXT: store i32 [[TMP9]], ptr [[TMP8]], align 4
+; CHECK-NEXT: br label %[[PRED_STORE_CONTINUE2]]
+; CHECK: [[PRED_STORE_CONTINUE2]]:
+; CHECK-NEXT: [[TMP10:%.*]] = extractelement <4 x i1> [[TMP0]], i32 2
+; CHECK-NEXT: br i1 [[TMP10]], label %[[EXIT:.*]], label %[[PRED_STORE_CONTINUE4:.*]]
; CHECK: [[EXIT]]:
+; CHECK-NEXT: [[TMP11:%.*]] = sdiv i32 99, [[X]]
+; CHECK-NEXT: [[TMP12:%.*]] = add i64 [[IV]], 2
+; CHECK-NEXT: [[TMP13:%.*]] = getelementptr i32, ptr [[DST]], i64 [[TMP12]]
+; CHECK-NEXT: [[TMP14:%.*]] = sdiv i32 [[TMP11]], 2
+; CHECK-NEXT: store i32 [[TMP14]], ptr [[TMP13]], align 4
+; CHECK-NEXT: br label %[[PRED_STORE_CONTINUE4]]
+; CHECK: [[PRED_STORE_CONTINUE4]]:
+; CHECK-NEXT: [[TMP15:%.*]] = extractelement <4 x i1> [[TMP0]], i32 3
+; CHECK-NEXT: br i1 [[TMP15]], label %[[PRED_STORE_IF5:.*]], label %[[PRED_STORE_CONTINUE6]]
+; CHECK: [[PRED_STORE_IF5]]:
+; CHECK-NEXT: [[TMP16:%.*]] = sdiv i32 99, [[X]]
+; CHECK-NEXT: [[TMP17:%.*]] = add i64 [[IV]], 3
+; CHECK-NEXT: [[TMP18:%.*]] = getelementptr i32, ptr [[DST]], i64 [[TMP17]]
+; CHECK-NEXT: [[TMP19:%.*]] = sdiv i32 [[TMP16]], 2
+; CHECK-NEXT: store i32 [[TMP19]], ptr [[TMP18]], align 4
+; CHECK-NEXT: br label %[[PRED_STORE_CONTINUE6]]
+; CHECK: [[PRED_STORE_CONTINUE6]]:
+; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[IV]], 4
+; CHECK-NEXT: [[VEC_IND_NEXT]] = add <4 x i64> [[VEC_IND]], splat (i64 4)
+; CHECK-NEXT: [[TMP20:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
+; CHECK-NEXT: br i1 [[TMP20]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP15:![0-9]+]]
+; CHECK: [[MIDDLE_BLOCK]]:
+; CHECK-NEXT: br label %[[EXIT1:.*]]
+; CHECK: [[EXIT1]]:
; CHECK-NEXT: ret void
;
entry:
@@ -490,9 +538,9 @@ define void @getPredBlockCostDivisor_truncate(i32 %0, i1 %c1, i1 %c2, ptr %p) {
; CHECK-NEXT: br label %[[LOOP:.*]]
; CHECK: [[LOOP]]:
; CHECK-NEXT: [[IV:%.*]] = phi i32 [ [[TMP0]], %[[ENTRY]] ], [ [[IV_NEXT:%.*]], %[[LATCH:.*]] ]
-; CHECK-NEXT: br i1 [[C1]], label %[[IF_1:.*]], label %[[LATCH]], !prof [[PROF15:![0-9]+]]
+; CHECK-NEXT: br i1 [[C1]], label %[[IF_1:.*]], label %[[LATCH]], !prof [[PROF16:![0-9]+]]
; CHECK: [[IF_1]]:
-; CHECK-NEXT: br i1 [[C2]], label %[[IF_2:.*]], label %[[LATCH]], !prof [[PROF15]]
+; CHECK-NEXT: br i1 [[C2]], label %[[IF_2:.*]], label %[[LATCH]], !prof [[PROF16]]
; CHECK: [[IF_2]]:
; CHECK-NEXT: [[GEP:%.*]] = getelementptr i32, ptr [[P]], i32 [[IV]]
; CHECK-NEXT: store i32 0, ptr [[GEP]], align 4
@@ -546,5 +594,6 @@ exit:
; CHECK: [[LOOP12]] = distinct !{[[LOOP12]], [[META9]], [[META10]]}
; CHECK: [[LOOP13]] = distinct !{[[LOOP13]], [[META10]], [[META11]]}
; CHECK: [[LOOP14]] = distinct !{[[LOOP14]], [[META11]], [[META10]]}
-; CHECK: [[PROF15]] = !{!"branch_weights", i32 0, i32 1}
+; CHECK: [[LOOP15]] = distinct !{[[LOOP15]], [[META10]], [[META11]]}
+; CHECK: [[PROF16]] = !{!"branch_weights", i32 0, i32 1}
;.
diff --git a/llvm/test/Transforms/LoopVectorize/AArch64/predication_costs.ll b/llvm/test/Transforms/LoopVectorize/AArch64/predication_costs.ll
index b9b91be9b7a65..b1568599f8242 100644
--- a/llvm/test/Transforms/LoopVectorize/AArch64/predication_costs.ll
+++ b/llvm/test/Transforms/LoopVectorize/AArch64/predication_costs.ll
@@ -220,9 +220,9 @@ for.end:
; Cost of add:
; add(1) = 1
; Cost of sdiv:
-; (sdiv(2) + extractelement(8) + insertelement(4)) / 2 = 7
+; (sdiv(2) + extractelement(8)) / 2 = 5
; Cost of udiv:
-; (udiv(2) + extractelement(4) + insertelement(4)) / 2 = 5
+; (udiv(2) + extractelement(4)) / 2 = 3
; Cost of sub:
; (sub(2) + extractelement(4)) / 2 = 3
; Cost of store:
@@ -236,8 +236,8 @@ for.end:
; CHECK: Cost of 2 for VF 2: profitable to scalarize store i32 %tmp5, ptr %tmp0, align 4
; CHECK: Cost of 3 for VF 2: profitable to scalarize %tmp5 = sub i32 %tmp4, %x
; CHECK: Cost of 1 for VF 2: WIDEN ir<%tmp2> = add ir<%tmp1>, ir<%x>
-; CHECK: Cost of 7 for VF 2: REPLICATE ir<%tmp3> = sdiv ir<%tmp1>, ir<%tmp2>
-; CHECK: Cost of 5 for VF 2: REPLICATE ir<%tmp4> = udiv ir<%tmp3>, ir<%tmp2>
+; CHECK: Cost of 5 for VF 2: REPLICATE ir<%tmp3> = sdiv ir<%tmp1>, ir<%tmp2>
+; CHECK: Cost of 3 for VF 2: REPLICATE ir<%tmp4> = udiv ir<%tmp3>, ir<%tmp2>
;
define void @predication_multi_context(ptr %a, i1 %c, i32 %x, i64 %n) {
entry:
diff --git a/llvm/test/Transforms/LoopVectorize/X86/replicating-load-store-costs.ll b/llvm/test/Transforms/LoopVectorize/X86/replicating-load-store-costs.ll
index a5a4191995591..785f705949bdd 100644
--- a/llvm/test/Transforms/LoopVectorize/X86/replicating-load-store-costs.ll
+++ b/llvm/test/Transforms/LoopVectorize/X86/replicating-load-store-costs.ll
@@ -343,11 +343,125 @@ define void @test_store_loaded_value(ptr noalias %src, ptr noalias %dst, i32 %n)
; I64-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[N_EXT]], 4
; I64-NEXT: br i1 [[MIN_ITERS_CHECK]], label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]]
; I64: [[VECTOR_PH]]:
+; I64-NEXT: [[MIN_ITERS_CHECK1:%.*]] = icmp ult i64 [[N_EXT]], 16
+; I64-NEXT: br i1 [[MIN_ITERS_CHECK1]], label %[[VEC_EPILOG_PH:.*]], label %[[VECTOR_PH1:.*]]
+; I64: [[VECTOR_PH1]]:
+; I64-NEXT: [[N_MOD_VF1:%.*]] = urem i64 [[N_EXT]], 16
+; I64-NEXT: [[N_VEC1:%.*]] = sub i64 [[N_EXT]], [[N_MOD_VF1]]
+; I64-NEXT: br label %[[VECTOR_BODY1:.*]]
+; I64: [[VECTOR_BODY1]]:
+; I64-NEXT: [[INDEX1:%.*]] = phi i64 [ 0, %[[VECTOR_PH1]] ], [ [[INDEX_NEXT1:%.*]], %[[VECTOR_BODY1]] ]
+; I64-NEXT: [[TMP0:%.*]] = add i64 [[INDEX1]], 1
+; I64-NEXT: [[TMP80:%.*]] = add i64 [[INDEX1]], 2
+; I64-NEXT: [[TMP81:%.*]] = add i64 [[INDEX1]], 3
+; I64-NEXT: [[TMP82:%.*]] = add i64 [[INDEX1]], 4
+; I64-NEXT: [[TMP83:%.*]] = add i64 [[INDEX1]], 5
+; I64-NEXT: [[TMP84:%.*]] = add i64 [[INDEX1]], 6
+; I64-NEXT: [[TMP85:%.*]] = add i64 [[INDEX1]], 7
+; I64-NEXT: [[TMP86:%.*]] = add i64 [[INDEX1]], 8
+; I64-NEXT: [[TMP87:%.*]] = add i64 [[INDEX1]], 9
+; I64-NEXT: [[TMP88:%.*]] = add i64 [[INDEX1]], 10
+; I64-NEXT: [[TMP89:%.*]] = add i64 [[INDEX1]], 11
+; I64-NEXT: [[TMP90:%.*]] = add i64 [[INDEX1]], 12
+; I64-NEXT: [[TMP91:%.*]] = add i64 [[INDEX1]], 13
+; I64-NEXT: [[TMP92:%.*]] = add i64 [[INDEX1]], 14
+; I64-NEXT: [[TMP93:%.*]] = add i64 [[INDEX1]], 15
+; I64-NEXT: [[TMP94:%.*]] = getelementptr i8, ptr [[SRC]], i64 [[INDEX1]]
+; I64-NEXT: [[TMP95:%.*]] = getelementptr i8, ptr [[SRC]], i64 [[TMP0]]
+; I64-NEXT: [[TMP96:%.*]] = getelementptr i8, ptr [[SRC]], i64 [[TMP80]]
+; I64-NEXT: [[TMP97:%.*]] = getelementptr i8, ptr [[SRC]], i64 [[TMP81]]
+; I64-NEXT: [[TMP98:%.*]] = getelementptr i8, ptr [[SRC]], i64 [[TMP82]]
+; I64-NEXT: [[TMP99:%.*]] = getelementptr i8, ptr [[SRC]], i64 [[TMP83]]
+; I64-NEXT: [[TMP21:%.*]] = getelementptr i8, ptr [[SRC]], i64 [[TMP84]]
+; I64-NEXT: [[TMP22:%.*]] = getelementptr i8, ptr [[SRC]], i64 [[TMP85]]
+; I64-NEXT: [[TMP23:%.*]] = getelementptr i8, ptr [[SRC]], i64 [[TMP86]]
+; I64-NEXT: [[TMP24:%.*]] = getelementptr i8, ptr [[SRC]], i64 [[TMP87]]
+; I64-NEXT: [[TMP25:%.*]] = getelementptr i8, ptr [[SRC]], i64 [[TMP88]]
+; I64-NEXT: [[TMP26:%.*]] = getelementptr i8, ptr [[SRC]], i64 [[TMP89]]
+; I64-NEXT: [[TMP27:%.*]] = getelementptr i8, ptr [[SRC]], i64 [[TMP90]]
+; I64-NEXT: [[TMP28:%.*]] = getelementptr i8, ptr [[SRC]], i64 [[TMP91]]
+; I64-NEXT: [[TMP29:%.*]] = getelementptr i8, ptr [[SRC]], i64 [[TMP92]]
+; I64-NEXT: [[TMP30:%.*]] = getelementptr i8, ptr [[SRC]], i64 [[TMP93]]
+; I64-NEXT: [[TMP31:%.*]] = load double, ptr [[TMP94]], align 8
+; I64-NEXT: [[TMP32:%.*]] = load double, ptr [[TMP95]], align 8
+; I64-NEXT: [[TMP33:%.*]] = load double, ptr [[TMP96]], align 8
+; I64-NEXT: [[TMP34:%.*]] = load double, ptr [[TMP97]], align 8
+; I64-NEXT: [[TMP35:%.*]] = load double, ptr [[TMP98]], align 8
+; I64-NEXT: [[TMP36:%.*]] = load double, ptr [[TMP99]], align 8
+; I64-NEXT: [[TMP37:%.*]] = load double, ptr [[TMP21]], align 8
+; I64-NEXT: [[TMP38:%.*]] = load double, ptr [[TMP22]], align 8
+; I64-NEXT: [[TMP39:%.*]] = load double, ptr [[TMP23]], align 8
+; I64-NEXT: [[TMP40:%.*]] = load double, ptr [[TMP24]], align 8
+; I64-NEXT: [[TMP41:%.*]] = load double, ptr [[TMP25]], align 8
+; I64-NEXT: [[TMP42:%.*]] = load double, ptr [[TMP26]], align 8
+; I64-NEXT: [[TMP43:%.*]] = load double, ptr [[TMP27]], align 8
+; I64-NEXT: [[TMP44:%.*]] = load double, ptr [[TMP28]], align 8
+; I64-NEXT: [[TMP45:%.*]] = load double, ptr [[TMP29]], align 8
+; I64-NEXT: [[TMP46:%.*]] = load double, ptr [[TMP30]], align 8
+; I64-NEXT: [[TMP47:%.*]] = shl i64 [[INDEX1]], 1
+; I64-NEXT: [[TMP48:%.*]] = shl i64 [[TMP0]], 1
+; I64-NEXT: [[TMP49:%.*]] = shl i64 [[TMP80]], 1
+; I64-NEXT: [[TMP50:%.*]] = shl i64 [[TMP81]], 1
+; I64-NEXT: [[TMP51:%.*]] = shl i64 [[TMP82]], 1
+; I64-NEXT: [[TMP52:%.*]] = shl i64 [[TMP83]], 1
+; I64-NEXT: [[TMP53:%.*]] = shl i64 [[TMP84]], 1
+; I64-NEXT: [[TMP54:%.*]] = shl i64 [[TMP85]], 1
+; I64-NEXT: [[TMP55:%.*]] = shl i64 [[TMP86]], 1
+; I64-NEXT: [[TMP56:%.*]] = shl i64 [[TMP87]], 1
+; I64-NEXT: [[TMP57:%.*]] = shl i64 [[TMP88]], 1
+; I64-NEXT: [[TMP58:%.*]] = shl i64 [[TMP89]], 1
+; I64-NEXT: [[TMP59:%.*]] = shl i64 [[TMP90]], 1
+; I64-NEXT: [[TMP60:%.*]] = shl i64 [[TMP91]], 1
+; I64-NEXT: [[TMP61:%.*]] = shl i64 [[TMP92]], 1
+; I64-NEXT: [[TMP62:%.*]] = shl i64 [[TMP93]], 1
+; I64-NEXT: [[TMP63:%.*]] = getelementptr i8, ptr [[DST]], i64 [[TMP47]]
+; I64-NEXT: [[TMP64:%.*]] = getelementptr i8, ptr [[DST]], i64 [[TMP48]]
+; I64-NEXT: [[TMP65:%.*]] = getelementptr i8, ptr [[DST]], i64 [[TMP49]]
+; I64-NEXT: [[TMP66:%.*]] = getelementptr i8, ptr [[DST]], i64 [[TMP50]]
+; I64-NEXT: [[TMP67:%.*]] = getelementptr i8, ptr [[DST]], i64 [[TMP51]]
+; I64-NEXT: [[TMP68:%.*]] = getelementptr i8, ptr [[DST]], i64 [[TMP52]]
+; I64-NEXT: [[TMP69:%.*]] = getelementptr i8, ptr [[DST]], i64 [[TMP53]]
+; I64-NEXT: [[TMP70:%.*]] = getelementptr i8, ptr [[DST]], i64 [[TMP54]]
+; I64-NEXT: [[TMP71:%.*]] = getelementptr i8, ptr [[DST]], i64 [[TMP55]]
+; I64-NEXT: [[TMP72:%.*]] = getelementptr i8, ptr [[DST]], i64 [[TMP56]]
+; I64-NEXT: [[TMP73:%.*]] = getelementptr i8, ptr [[DST]], i64 [[TMP57]]
+; I64-NEXT: [[TMP74:%.*]] = getelementptr i8, ptr [[DST]], i64 [[TMP58]]
+; I64-NEXT: [[TMP75:%.*]] = getelementptr i8, ptr [[DST]], i64 [[TMP59]]
+; I64-NEXT: [[TMP76:%.*]] = getelementptr i8, ptr [[DST]], i64 [[TMP60]]
+; I64-NEXT: [[TMP77:%.*]] = getelementptr i8, ptr [[DST]], i64 [[TMP61]]
+; I64-NEXT: [[TMP78:%.*]] = getelementptr i8, ptr [[DST]], i64 [[TMP62]]
+; I64-NEXT: store double [[TMP31]], ptr [[TMP63]], align 8
+; I64-NEXT: store double [[TMP32]], ptr [[TMP64]], align 8
+; I64-NEXT: store double [[TMP33]], ptr [[TMP65]], align 8
+; I64-NEXT: store double [[TMP34]], ptr [[TMP66]], align 8
+; I64-NEXT: store double [[TMP35]], ptr [[TMP67]], align 8
+; I64-NEXT: store double [[TMP36]], ptr [[TMP68]], align 8
+; I64-NEXT: store double [[TMP37]], ptr [[TMP69]], align 8
+; I64-NEXT: store double [[TMP38]], ptr [[TMP70]], align 8
+; I64-NEXT: store double [[TMP39]], ptr [[TMP71]], align 8
+; I64-NEXT: store double [[TMP40]], ptr [[TMP72]], align 8
+; I64-NEXT: store double [[TMP41]], ptr [[TMP73]], align 8
+; I64-NEXT: store double [[TMP42]], ptr [[TMP74]], align 8
+; I64-NEXT: store double [[TMP43]], ptr [[TMP75]], align 8
+; I64-NEXT: store double [[TMP44]], ptr [[TMP76]], align 8
+; I64-NEXT: store double [[TMP45]], ptr [[TMP77]], align 8
+; I64-NEXT: store double [[TMP46]], ptr [[TMP78]], align 8
+; I64-NEXT: [[INDEX_NEXT1]] = add nuw i64 [[INDEX1]], 16
+; I64-NEXT: [[TMP79:%.*]] = icmp eq i64 [[INDEX_NEXT1]], [[N_VEC1]]
+; I64-NEXT: br i1 [[TMP79]], label %[[MIDDLE_BLOCK1:.*]], label %[[VECTOR_BODY1]], !llvm.loop [[LOOP6:![0-9]+]]
+; I64: [[MIDDLE_BLOCK1]]:
+; I64-NEXT: [[CMP_N1:%.*]] = icmp eq i64 [[N_EXT]], [[N_VEC1]]
+; I64-NEXT: br i1 [[CMP_N1]], [[EXIT_LOOPEXIT:label %.*]], label %[[VEC_EPILOG_ITER_CHECK:.*]]
+; I64: [[VEC_EPILOG_ITER_CHECK]]:
+; I64-NEXT: [[MIN_EPILOG_ITERS_CHECK:%.*]] = icmp ult i64 [[N_MOD_VF1]], 4
+; I64-NEXT: br i1 [[MIN_EPILOG_ITERS_CHECK]], label %[[SCALAR_PH]], label %[[VEC_EPILOG_PH]], !prof [[PROF3]]
+; I64: [[VEC_EPILOG_PH]]:
+; I64-NEXT: [[VEC_EPILOG_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC1]], %[[VEC_EPILOG_ITER_CHECK]] ], [ 0, %[[VECTOR_PH]] ]
; I64-NEXT: [[N_MOD_VF:%.*]] = urem i64 [[N_EXT]], 4
; I64-NEXT: [[N_VEC:%.*]] = sub i64 [[N_EXT]], [[N_MOD_VF]]
; I64-NEXT: br label %[[VECTOR_BODY:.*]]
; I64: [[VECTOR_BODY]]:
-; I64-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; I64-NEXT: [[INDEX:%.*]] = phi i64 [ [[VEC_EPILOG_RESUME_VAL]], %[[VEC_EPILOG_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
; I64-NEXT: [[TMP1:%.*]] = add i64 [[INDEX]], 1
; I64-NEXT: [[TMP2:%.*]] = add i64 [[INDEX]], 2
; I64-NEXT: [[TMP3:%.*]] = add i64 [[INDEX]], 3
@@ -373,10 +487,10 @@ define void @test_store_loaded_value(ptr noalias %src, ptr noalias %dst, i32 %n)
; I64-NEXT: store double [[TMP11]], ptr [[TMP19]], align 8
; I64-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 4
; I64-NEXT: [[TMP20:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
-; I64-NEXT: br i1 [[TMP20]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP6:![0-9]+]]
+; I64-NEXT: br i1 [[TMP20]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP7:![0-9]+]]
; I64: [[MIDDLE_BLOCK]]:
; I64-NEXT: [[CMP_N:%.*]] = icmp eq i64 [[N_EXT]], [[N_VEC]]
-; I64-NEXT: br i1 [[CMP_N]], [[EXIT_LOOPEXIT:label %.*]],...
[truncated]
``````````
</details>
https://github.com/llvm/llvm-project/pull/192056
More information about the llvm-commits
mailing list