[llvm] [LV] Improve cost model for some replicating recipes (PR #192056)
David Sherwood via llvm-commits
llvm-commits at lists.llvm.org
Tue Apr 14 06:26:36 PDT 2026
https://github.com/david-arm updated https://github.com/llvm/llvm-project/pull/192056
>From eab43cb64edb72232df8e10dde53d06963b88ac4 Mon Sep 17 00:00:00 2001
From: David Sherwood <david.sherwood at arm.com>
Date: Tue, 14 Apr 2026 13:18:27 +0000
Subject: [PATCH 1/2] [LV] Improve cost model for some replicating recipes
In VPCostContext::getScalarizationOverhead we pessimistically
assume that the result always needs scalarizing. By this we
mean that the scalar result for each lane will need inserting
into the corresponding lane of a vector. However, if the only
users of the recipe are VPReplicateRecipes in the same block
then the result will remain scalar and no inserts are needed.
This has the effect of lowering the cost of some replicating
instructions, as seen by the affected tests in this PR.
---
llvm/lib/Transforms/Vectorize/VPlan.cpp | 15 +-
llvm/lib/Transforms/Vectorize/VPlanHelpers.h | 1 +
.../lib/Transforms/Vectorize/VPlanRecipes.cpp | 5 +-
.../LoopVectorize/AArch64/predicated-costs.ll | 67 +++++++--
.../AArch64/predication_costs.ll | 8 +-
.../X86/replicating-load-store-costs.ll | 128 +++++++++++++++++-
6 files changed, 200 insertions(+), 24 deletions(-)
diff --git a/llvm/lib/Transforms/Vectorize/VPlan.cpp b/llvm/lib/Transforms/Vectorize/VPlan.cpp
index 8048c3227a9a5..41a17742d14b8 100644
--- a/llvm/lib/Transforms/Vectorize/VPlan.cpp
+++ b/llvm/lib/Transforms/Vectorize/VPlan.cpp
@@ -1828,7 +1828,8 @@ VPCostContext::getOperandInfo(VPValue *V) const {
InstructionCost VPCostContext::getScalarizationOverhead(
Type *ResultTy, ArrayRef<const VPValue *> Operands, ElementCount VF,
- TTI::VectorInstrContext VIC, bool AlwaysIncludeReplicatingR) {
+ const VPSingleDefRecipe *R, TTI::VectorInstrContext VIC,
+ bool AlwaysIncludeReplicatingR) {
if (VF.isScalar())
return 0;
@@ -1837,7 +1838,17 @@ InstructionCost VPCostContext::getScalarizationOverhead(
InstructionCost ScalarizationCost = 0;
// Compute the cost of scalarizing the result if needed.
- if (!ResultTy->isVoidTy()) {
+ bool ScalarizeResult = !ResultTy->isVoidTy();
+ if (ScalarizeResult && R) {
+ // Is this recipe only used by other replicate recipes in the same block?
+ // If so, the result does not need scalarizing.
+ ScalarizeResult = llvm::any_of(R->users(), [R](VPUser *U) -> bool {
+ auto *URR = dyn_cast<VPReplicateRecipe>(U);
+ return !URR || URR->getParent() != R->getParent();
+ });
+ }
+
+ if (ScalarizeResult) {
for (Type *VectorTy :
to_vector(getContainedTypes(toVectorizedTy(ResultTy, VF)))) {
ScalarizationCost += TTI.getScalarizationOverhead(
diff --git a/llvm/lib/Transforms/Vectorize/VPlanHelpers.h b/llvm/lib/Transforms/Vectorize/VPlanHelpers.h
index b54b80b496668..80cf0c431b67b 100644
--- a/llvm/lib/Transforms/Vectorize/VPlanHelpers.h
+++ b/llvm/lib/Transforms/Vectorize/VPlanHelpers.h
@@ -375,6 +375,7 @@ struct VPCostContext {
/// replicating operands.
InstructionCost getScalarizationOverhead(
Type *ResultTy, ArrayRef<const VPValue *> Operands, ElementCount VF,
+ const VPSingleDefRecipe *R = nullptr,
TTI::VectorInstrContext VIC = TTI::VectorInstrContext::None,
bool AlwaysIncludeReplicatingR = false);
diff --git a/llvm/lib/Transforms/Vectorize/VPlanRecipes.cpp b/llvm/lib/Transforms/Vectorize/VPlanRecipes.cpp
index cbb44198fe362..7ffc9a227d5c7 100644
--- a/llvm/lib/Transforms/Vectorize/VPlanRecipes.cpp
+++ b/llvm/lib/Transforms/Vectorize/VPlanRecipes.cpp
@@ -3534,7 +3534,7 @@ InstructionCost VPReplicateRecipe::computeCost(ElementCount VF,
ScalarCost = ScalarCost * VF.getFixedValue() +
Ctx.getScalarizationOverhead(Ctx.Types.inferScalarType(this),
- to_vector(operands()), VF);
+ to_vector(operands()), VF, this);
// If the recipe is not predicated (i.e. not in a replicate region), return
// the scalar cost. Otherwise handle predicated cost.
if (!getRegion()->isReplicator())
@@ -3620,7 +3620,8 @@ InstructionCost VPReplicateRecipe::computeCost(ElementCount VF,
IsLoad ? TTI::VectorInstrContext::Load : TTI::VectorInstrContext::Store;
InstructionCost Cost =
(ScalarCost * VF.getFixedValue()) +
- Ctx.getScalarizationOverhead(ResultTy, OpsToScalarize, VF, VIC, true);
+ Ctx.getScalarizationOverhead(ResultTy, OpsToScalarize, VF, this, VIC,
+ true);
const VPRegionBlock *ParentRegion = getRegion();
if (ParentRegion && ParentRegion->isReplicator()) {
diff --git a/llvm/test/Transforms/LoopVectorize/AArch64/predicated-costs.ll b/llvm/test/Transforms/LoopVectorize/AArch64/predicated-costs.ll
index d96a96836bd3a..7de338a3a6b5f 100644
--- a/llvm/test/Transforms/LoopVectorize/AArch64/predicated-costs.ll
+++ b/llvm/test/Transforms/LoopVectorize/AArch64/predicated-costs.ll
@@ -380,22 +380,70 @@ exit:
define void @sdiv_power_of_2_divisor_in_replicate_region(i32 %x, ptr %dst, i64 %n) {
; CHECK-LABEL: define void @sdiv_power_of_2_divisor_in_replicate_region(
; CHECK-SAME: i32 [[X:%.*]], ptr [[DST:%.*]], i64 [[N:%.*]]) {
-; CHECK-NEXT: [[ENTRY:.*]]:
+; CHECK-NEXT: [[ENTRY:.*:]]
; CHECK-NEXT: [[C_1:%.*]] = icmp sgt i64 [[N]], 0
; CHECK-NEXT: [[C_2:%.*]] = icmp slt i64 [[N]], 9
; CHECK-NEXT: call void @llvm.assume(i1 [[C_1]])
; CHECK-NEXT: call void @llvm.assume(i1 [[C_2]])
; CHECK-NEXT: br label %[[LOOP:.*]]
; CHECK: [[LOOP]]:
-; CHECK-NEXT: [[IV:%.*]] = phi i64 [ 0, %[[ENTRY]] ], [ [[IV_NEXT:%.*]], %[[LOOP]] ]
+; CHECK-NEXT: [[N_RND_UP:%.*]] = add i64 [[N]], 3
+; CHECK-NEXT: [[N_MOD_VF:%.*]] = urem i64 [[N_RND_UP]], 4
+; CHECK-NEXT: [[N_VEC:%.*]] = sub i64 [[N_RND_UP]], [[N_MOD_VF]]
+; CHECK-NEXT: [[TRIP_COUNT_MINUS_1:%.*]] = sub i64 [[N]], 1
+; CHECK-NEXT: [[BROADCAST_SPLATINSERT:%.*]] = insertelement <4 x i64> poison, i64 [[TRIP_COUNT_MINUS_1]], i64 0
+; CHECK-NEXT: [[BROADCAST_SPLAT:%.*]] = shufflevector <4 x i64> [[BROADCAST_SPLATINSERT]], <4 x i64> poison, <4 x i32> zeroinitializer
+; CHECK-NEXT: br label %[[VECTOR_BODY:.*]]
+; CHECK: [[VECTOR_BODY]]:
+; CHECK-NEXT: [[IV:%.*]] = phi i64 [ 0, %[[LOOP]] ], [ [[INDEX_NEXT:%.*]], %[[PRED_STORE_CONTINUE6:.*]] ]
+; CHECK-NEXT: [[VEC_IND:%.*]] = phi <4 x i64> [ <i64 0, i64 1, i64 2, i64 3>, %[[LOOP]] ], [ [[VEC_IND_NEXT:%.*]], %[[PRED_STORE_CONTINUE6]] ]
+; CHECK-NEXT: [[TMP0:%.*]] = icmp ule <4 x i64> [[VEC_IND]], [[BROADCAST_SPLAT]]
+; CHECK-NEXT: [[TMP1:%.*]] = extractelement <4 x i1> [[TMP0]], i32 0
+; CHECK-NEXT: br i1 [[TMP1]], label %[[PRED_STORE_IF:.*]], label %[[PRED_STORE_CONTINUE:.*]]
+; CHECK: [[PRED_STORE_IF]]:
; CHECK-NEXT: [[DIV:%.*]] = sdiv i32 99, [[X]]
-; CHECK-NEXT: [[DIV2:%.*]] = sdiv i32 [[DIV]], 2
; CHECK-NEXT: [[GEP:%.*]] = getelementptr i32, ptr [[DST]], i64 [[IV]]
+; CHECK-NEXT: [[DIV2:%.*]] = sdiv i32 [[DIV]], 2
; CHECK-NEXT: store i32 [[DIV2]], ptr [[GEP]], align 4
-; CHECK-NEXT: [[IV_NEXT]] = add i64 [[IV]], 1
-; CHECK-NEXT: [[DONE:%.*]] = icmp eq i64 [[IV_NEXT]], [[N]]
-; CHECK-NEXT: br i1 [[DONE]], label %[[EXIT:.*]], label %[[LOOP]]
+; CHECK-NEXT: br label %[[PRED_STORE_CONTINUE]]
+; CHECK: [[PRED_STORE_CONTINUE]]:
+; CHECK-NEXT: [[TMP5:%.*]] = extractelement <4 x i1> [[TMP0]], i32 1
+; CHECK-NEXT: br i1 [[TMP5]], label %[[PRED_STORE_IF1:.*]], label %[[PRED_STORE_CONTINUE2:.*]]
+; CHECK: [[PRED_STORE_IF1]]:
+; CHECK-NEXT: [[TMP6:%.*]] = sdiv i32 99, [[X]]
+; CHECK-NEXT: [[IV_NEXT:%.*]] = add i64 [[IV]], 1
+; CHECK-NEXT: [[TMP8:%.*]] = getelementptr i32, ptr [[DST]], i64 [[IV_NEXT]]
+; CHECK-NEXT: [[TMP9:%.*]] = sdiv i32 [[TMP6]], 2
+; CHECK-NEXT: store i32 [[TMP9]], ptr [[TMP8]], align 4
+; CHECK-NEXT: br label %[[PRED_STORE_CONTINUE2]]
+; CHECK: [[PRED_STORE_CONTINUE2]]:
+; CHECK-NEXT: [[TMP10:%.*]] = extractelement <4 x i1> [[TMP0]], i32 2
+; CHECK-NEXT: br i1 [[TMP10]], label %[[EXIT:.*]], label %[[PRED_STORE_CONTINUE4:.*]]
; CHECK: [[EXIT]]:
+; CHECK-NEXT: [[TMP11:%.*]] = sdiv i32 99, [[X]]
+; CHECK-NEXT: [[TMP12:%.*]] = add i64 [[IV]], 2
+; CHECK-NEXT: [[TMP13:%.*]] = getelementptr i32, ptr [[DST]], i64 [[TMP12]]
+; CHECK-NEXT: [[TMP14:%.*]] = sdiv i32 [[TMP11]], 2
+; CHECK-NEXT: store i32 [[TMP14]], ptr [[TMP13]], align 4
+; CHECK-NEXT: br label %[[PRED_STORE_CONTINUE4]]
+; CHECK: [[PRED_STORE_CONTINUE4]]:
+; CHECK-NEXT: [[TMP15:%.*]] = extractelement <4 x i1> [[TMP0]], i32 3
+; CHECK-NEXT: br i1 [[TMP15]], label %[[PRED_STORE_IF5:.*]], label %[[PRED_STORE_CONTINUE6]]
+; CHECK: [[PRED_STORE_IF5]]:
+; CHECK-NEXT: [[TMP16:%.*]] = sdiv i32 99, [[X]]
+; CHECK-NEXT: [[TMP17:%.*]] = add i64 [[IV]], 3
+; CHECK-NEXT: [[TMP18:%.*]] = getelementptr i32, ptr [[DST]], i64 [[TMP17]]
+; CHECK-NEXT: [[TMP19:%.*]] = sdiv i32 [[TMP16]], 2
+; CHECK-NEXT: store i32 [[TMP19]], ptr [[TMP18]], align 4
+; CHECK-NEXT: br label %[[PRED_STORE_CONTINUE6]]
+; CHECK: [[PRED_STORE_CONTINUE6]]:
+; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[IV]], 4
+; CHECK-NEXT: [[VEC_IND_NEXT]] = add <4 x i64> [[VEC_IND]], splat (i64 4)
+; CHECK-NEXT: [[TMP20:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
+; CHECK-NEXT: br i1 [[TMP20]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP15:![0-9]+]]
+; CHECK: [[MIDDLE_BLOCK]]:
+; CHECK-NEXT: br label %[[EXIT1:.*]]
+; CHECK: [[EXIT1]]:
; CHECK-NEXT: ret void
;
entry:
@@ -490,9 +538,9 @@ define void @getPredBlockCostDivisor_truncate(i32 %0, i1 %c1, i1 %c2, ptr %p) {
; CHECK-NEXT: br label %[[LOOP:.*]]
; CHECK: [[LOOP]]:
; CHECK-NEXT: [[IV:%.*]] = phi i32 [ [[TMP0]], %[[ENTRY]] ], [ [[IV_NEXT:%.*]], %[[LATCH:.*]] ]
-; CHECK-NEXT: br i1 [[C1]], label %[[IF_1:.*]], label %[[LATCH]], !prof [[PROF15:![0-9]+]]
+; CHECK-NEXT: br i1 [[C1]], label %[[IF_1:.*]], label %[[LATCH]], !prof [[PROF16:![0-9]+]]
; CHECK: [[IF_1]]:
-; CHECK-NEXT: br i1 [[C2]], label %[[IF_2:.*]], label %[[LATCH]], !prof [[PROF15]]
+; CHECK-NEXT: br i1 [[C2]], label %[[IF_2:.*]], label %[[LATCH]], !prof [[PROF16]]
; CHECK: [[IF_2]]:
; CHECK-NEXT: [[GEP:%.*]] = getelementptr i32, ptr [[P]], i32 [[IV]]
; CHECK-NEXT: store i32 0, ptr [[GEP]], align 4
@@ -546,5 +594,6 @@ exit:
; CHECK: [[LOOP12]] = distinct !{[[LOOP12]], [[META9]], [[META10]]}
; CHECK: [[LOOP13]] = distinct !{[[LOOP13]], [[META10]], [[META11]]}
; CHECK: [[LOOP14]] = distinct !{[[LOOP14]], [[META11]], [[META10]]}
-; CHECK: [[PROF15]] = !{!"branch_weights", i32 0, i32 1}
+; CHECK: [[LOOP15]] = distinct !{[[LOOP15]], [[META10]], [[META11]]}
+; CHECK: [[PROF16]] = !{!"branch_weights", i32 0, i32 1}
;.
diff --git a/llvm/test/Transforms/LoopVectorize/AArch64/predication_costs.ll b/llvm/test/Transforms/LoopVectorize/AArch64/predication_costs.ll
index b9b91be9b7a65..b1568599f8242 100644
--- a/llvm/test/Transforms/LoopVectorize/AArch64/predication_costs.ll
+++ b/llvm/test/Transforms/LoopVectorize/AArch64/predication_costs.ll
@@ -220,9 +220,9 @@ for.end:
; Cost of add:
; add(1) = 1
; Cost of sdiv:
-; (sdiv(2) + extractelement(8) + insertelement(4)) / 2 = 7
+; (sdiv(2) + extractelement(8)) / 2 = 5
; Cost of udiv:
-; (udiv(2) + extractelement(4) + insertelement(4)) / 2 = 5
+; (udiv(2) + extractelement(4)) / 2 = 3
; Cost of sub:
; (sub(2) + extractelement(4)) / 2 = 3
; Cost of store:
@@ -236,8 +236,8 @@ for.end:
; CHECK: Cost of 2 for VF 2: profitable to scalarize store i32 %tmp5, ptr %tmp0, align 4
; CHECK: Cost of 3 for VF 2: profitable to scalarize %tmp5 = sub i32 %tmp4, %x
; CHECK: Cost of 1 for VF 2: WIDEN ir<%tmp2> = add ir<%tmp1>, ir<%x>
-; CHECK: Cost of 7 for VF 2: REPLICATE ir<%tmp3> = sdiv ir<%tmp1>, ir<%tmp2>
-; CHECK: Cost of 5 for VF 2: REPLICATE ir<%tmp4> = udiv ir<%tmp3>, ir<%tmp2>
+; CHECK: Cost of 5 for VF 2: REPLICATE ir<%tmp3> = sdiv ir<%tmp1>, ir<%tmp2>
+; CHECK: Cost of 3 for VF 2: REPLICATE ir<%tmp4> = udiv ir<%tmp3>, ir<%tmp2>
;
define void @predication_multi_context(ptr %a, i1 %c, i32 %x, i64 %n) {
entry:
diff --git a/llvm/test/Transforms/LoopVectorize/X86/replicating-load-store-costs.ll b/llvm/test/Transforms/LoopVectorize/X86/replicating-load-store-costs.ll
index a5a4191995591..785f705949bdd 100644
--- a/llvm/test/Transforms/LoopVectorize/X86/replicating-load-store-costs.ll
+++ b/llvm/test/Transforms/LoopVectorize/X86/replicating-load-store-costs.ll
@@ -343,11 +343,125 @@ define void @test_store_loaded_value(ptr noalias %src, ptr noalias %dst, i32 %n)
; I64-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[N_EXT]], 4
; I64-NEXT: br i1 [[MIN_ITERS_CHECK]], label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]]
; I64: [[VECTOR_PH]]:
+; I64-NEXT: [[MIN_ITERS_CHECK1:%.*]] = icmp ult i64 [[N_EXT]], 16
+; I64-NEXT: br i1 [[MIN_ITERS_CHECK1]], label %[[VEC_EPILOG_PH:.*]], label %[[VECTOR_PH1:.*]]
+; I64: [[VECTOR_PH1]]:
+; I64-NEXT: [[N_MOD_VF1:%.*]] = urem i64 [[N_EXT]], 16
+; I64-NEXT: [[N_VEC1:%.*]] = sub i64 [[N_EXT]], [[N_MOD_VF1]]
+; I64-NEXT: br label %[[VECTOR_BODY1:.*]]
+; I64: [[VECTOR_BODY1]]:
+; I64-NEXT: [[INDEX1:%.*]] = phi i64 [ 0, %[[VECTOR_PH1]] ], [ [[INDEX_NEXT1:%.*]], %[[VECTOR_BODY1]] ]
+; I64-NEXT: [[TMP0:%.*]] = add i64 [[INDEX1]], 1
+; I64-NEXT: [[TMP80:%.*]] = add i64 [[INDEX1]], 2
+; I64-NEXT: [[TMP81:%.*]] = add i64 [[INDEX1]], 3
+; I64-NEXT: [[TMP82:%.*]] = add i64 [[INDEX1]], 4
+; I64-NEXT: [[TMP83:%.*]] = add i64 [[INDEX1]], 5
+; I64-NEXT: [[TMP84:%.*]] = add i64 [[INDEX1]], 6
+; I64-NEXT: [[TMP85:%.*]] = add i64 [[INDEX1]], 7
+; I64-NEXT: [[TMP86:%.*]] = add i64 [[INDEX1]], 8
+; I64-NEXT: [[TMP87:%.*]] = add i64 [[INDEX1]], 9
+; I64-NEXT: [[TMP88:%.*]] = add i64 [[INDEX1]], 10
+; I64-NEXT: [[TMP89:%.*]] = add i64 [[INDEX1]], 11
+; I64-NEXT: [[TMP90:%.*]] = add i64 [[INDEX1]], 12
+; I64-NEXT: [[TMP91:%.*]] = add i64 [[INDEX1]], 13
+; I64-NEXT: [[TMP92:%.*]] = add i64 [[INDEX1]], 14
+; I64-NEXT: [[TMP93:%.*]] = add i64 [[INDEX1]], 15
+; I64-NEXT: [[TMP94:%.*]] = getelementptr i8, ptr [[SRC]], i64 [[INDEX1]]
+; I64-NEXT: [[TMP95:%.*]] = getelementptr i8, ptr [[SRC]], i64 [[TMP0]]
+; I64-NEXT: [[TMP96:%.*]] = getelementptr i8, ptr [[SRC]], i64 [[TMP80]]
+; I64-NEXT: [[TMP97:%.*]] = getelementptr i8, ptr [[SRC]], i64 [[TMP81]]
+; I64-NEXT: [[TMP98:%.*]] = getelementptr i8, ptr [[SRC]], i64 [[TMP82]]
+; I64-NEXT: [[TMP99:%.*]] = getelementptr i8, ptr [[SRC]], i64 [[TMP83]]
+; I64-NEXT: [[TMP21:%.*]] = getelementptr i8, ptr [[SRC]], i64 [[TMP84]]
+; I64-NEXT: [[TMP22:%.*]] = getelementptr i8, ptr [[SRC]], i64 [[TMP85]]
+; I64-NEXT: [[TMP23:%.*]] = getelementptr i8, ptr [[SRC]], i64 [[TMP86]]
+; I64-NEXT: [[TMP24:%.*]] = getelementptr i8, ptr [[SRC]], i64 [[TMP87]]
+; I64-NEXT: [[TMP25:%.*]] = getelementptr i8, ptr [[SRC]], i64 [[TMP88]]
+; I64-NEXT: [[TMP26:%.*]] = getelementptr i8, ptr [[SRC]], i64 [[TMP89]]
+; I64-NEXT: [[TMP27:%.*]] = getelementptr i8, ptr [[SRC]], i64 [[TMP90]]
+; I64-NEXT: [[TMP28:%.*]] = getelementptr i8, ptr [[SRC]], i64 [[TMP91]]
+; I64-NEXT: [[TMP29:%.*]] = getelementptr i8, ptr [[SRC]], i64 [[TMP92]]
+; I64-NEXT: [[TMP30:%.*]] = getelementptr i8, ptr [[SRC]], i64 [[TMP93]]
+; I64-NEXT: [[TMP31:%.*]] = load double, ptr [[TMP94]], align 8
+; I64-NEXT: [[TMP32:%.*]] = load double, ptr [[TMP95]], align 8
+; I64-NEXT: [[TMP33:%.*]] = load double, ptr [[TMP96]], align 8
+; I64-NEXT: [[TMP34:%.*]] = load double, ptr [[TMP97]], align 8
+; I64-NEXT: [[TMP35:%.*]] = load double, ptr [[TMP98]], align 8
+; I64-NEXT: [[TMP36:%.*]] = load double, ptr [[TMP99]], align 8
+; I64-NEXT: [[TMP37:%.*]] = load double, ptr [[TMP21]], align 8
+; I64-NEXT: [[TMP38:%.*]] = load double, ptr [[TMP22]], align 8
+; I64-NEXT: [[TMP39:%.*]] = load double, ptr [[TMP23]], align 8
+; I64-NEXT: [[TMP40:%.*]] = load double, ptr [[TMP24]], align 8
+; I64-NEXT: [[TMP41:%.*]] = load double, ptr [[TMP25]], align 8
+; I64-NEXT: [[TMP42:%.*]] = load double, ptr [[TMP26]], align 8
+; I64-NEXT: [[TMP43:%.*]] = load double, ptr [[TMP27]], align 8
+; I64-NEXT: [[TMP44:%.*]] = load double, ptr [[TMP28]], align 8
+; I64-NEXT: [[TMP45:%.*]] = load double, ptr [[TMP29]], align 8
+; I64-NEXT: [[TMP46:%.*]] = load double, ptr [[TMP30]], align 8
+; I64-NEXT: [[TMP47:%.*]] = shl i64 [[INDEX1]], 1
+; I64-NEXT: [[TMP48:%.*]] = shl i64 [[TMP0]], 1
+; I64-NEXT: [[TMP49:%.*]] = shl i64 [[TMP80]], 1
+; I64-NEXT: [[TMP50:%.*]] = shl i64 [[TMP81]], 1
+; I64-NEXT: [[TMP51:%.*]] = shl i64 [[TMP82]], 1
+; I64-NEXT: [[TMP52:%.*]] = shl i64 [[TMP83]], 1
+; I64-NEXT: [[TMP53:%.*]] = shl i64 [[TMP84]], 1
+; I64-NEXT: [[TMP54:%.*]] = shl i64 [[TMP85]], 1
+; I64-NEXT: [[TMP55:%.*]] = shl i64 [[TMP86]], 1
+; I64-NEXT: [[TMP56:%.*]] = shl i64 [[TMP87]], 1
+; I64-NEXT: [[TMP57:%.*]] = shl i64 [[TMP88]], 1
+; I64-NEXT: [[TMP58:%.*]] = shl i64 [[TMP89]], 1
+; I64-NEXT: [[TMP59:%.*]] = shl i64 [[TMP90]], 1
+; I64-NEXT: [[TMP60:%.*]] = shl i64 [[TMP91]], 1
+; I64-NEXT: [[TMP61:%.*]] = shl i64 [[TMP92]], 1
+; I64-NEXT: [[TMP62:%.*]] = shl i64 [[TMP93]], 1
+; I64-NEXT: [[TMP63:%.*]] = getelementptr i8, ptr [[DST]], i64 [[TMP47]]
+; I64-NEXT: [[TMP64:%.*]] = getelementptr i8, ptr [[DST]], i64 [[TMP48]]
+; I64-NEXT: [[TMP65:%.*]] = getelementptr i8, ptr [[DST]], i64 [[TMP49]]
+; I64-NEXT: [[TMP66:%.*]] = getelementptr i8, ptr [[DST]], i64 [[TMP50]]
+; I64-NEXT: [[TMP67:%.*]] = getelementptr i8, ptr [[DST]], i64 [[TMP51]]
+; I64-NEXT: [[TMP68:%.*]] = getelementptr i8, ptr [[DST]], i64 [[TMP52]]
+; I64-NEXT: [[TMP69:%.*]] = getelementptr i8, ptr [[DST]], i64 [[TMP53]]
+; I64-NEXT: [[TMP70:%.*]] = getelementptr i8, ptr [[DST]], i64 [[TMP54]]
+; I64-NEXT: [[TMP71:%.*]] = getelementptr i8, ptr [[DST]], i64 [[TMP55]]
+; I64-NEXT: [[TMP72:%.*]] = getelementptr i8, ptr [[DST]], i64 [[TMP56]]
+; I64-NEXT: [[TMP73:%.*]] = getelementptr i8, ptr [[DST]], i64 [[TMP57]]
+; I64-NEXT: [[TMP74:%.*]] = getelementptr i8, ptr [[DST]], i64 [[TMP58]]
+; I64-NEXT: [[TMP75:%.*]] = getelementptr i8, ptr [[DST]], i64 [[TMP59]]
+; I64-NEXT: [[TMP76:%.*]] = getelementptr i8, ptr [[DST]], i64 [[TMP60]]
+; I64-NEXT: [[TMP77:%.*]] = getelementptr i8, ptr [[DST]], i64 [[TMP61]]
+; I64-NEXT: [[TMP78:%.*]] = getelementptr i8, ptr [[DST]], i64 [[TMP62]]
+; I64-NEXT: store double [[TMP31]], ptr [[TMP63]], align 8
+; I64-NEXT: store double [[TMP32]], ptr [[TMP64]], align 8
+; I64-NEXT: store double [[TMP33]], ptr [[TMP65]], align 8
+; I64-NEXT: store double [[TMP34]], ptr [[TMP66]], align 8
+; I64-NEXT: store double [[TMP35]], ptr [[TMP67]], align 8
+; I64-NEXT: store double [[TMP36]], ptr [[TMP68]], align 8
+; I64-NEXT: store double [[TMP37]], ptr [[TMP69]], align 8
+; I64-NEXT: store double [[TMP38]], ptr [[TMP70]], align 8
+; I64-NEXT: store double [[TMP39]], ptr [[TMP71]], align 8
+; I64-NEXT: store double [[TMP40]], ptr [[TMP72]], align 8
+; I64-NEXT: store double [[TMP41]], ptr [[TMP73]], align 8
+; I64-NEXT: store double [[TMP42]], ptr [[TMP74]], align 8
+; I64-NEXT: store double [[TMP43]], ptr [[TMP75]], align 8
+; I64-NEXT: store double [[TMP44]], ptr [[TMP76]], align 8
+; I64-NEXT: store double [[TMP45]], ptr [[TMP77]], align 8
+; I64-NEXT: store double [[TMP46]], ptr [[TMP78]], align 8
+; I64-NEXT: [[INDEX_NEXT1]] = add nuw i64 [[INDEX1]], 16
+; I64-NEXT: [[TMP79:%.*]] = icmp eq i64 [[INDEX_NEXT1]], [[N_VEC1]]
+; I64-NEXT: br i1 [[TMP79]], label %[[MIDDLE_BLOCK1:.*]], label %[[VECTOR_BODY1]], !llvm.loop [[LOOP6:![0-9]+]]
+; I64: [[MIDDLE_BLOCK1]]:
+; I64-NEXT: [[CMP_N1:%.*]] = icmp eq i64 [[N_EXT]], [[N_VEC1]]
+; I64-NEXT: br i1 [[CMP_N1]], [[EXIT_LOOPEXIT:label %.*]], label %[[VEC_EPILOG_ITER_CHECK:.*]]
+; I64: [[VEC_EPILOG_ITER_CHECK]]:
+; I64-NEXT: [[MIN_EPILOG_ITERS_CHECK:%.*]] = icmp ult i64 [[N_MOD_VF1]], 4
+; I64-NEXT: br i1 [[MIN_EPILOG_ITERS_CHECK]], label %[[SCALAR_PH]], label %[[VEC_EPILOG_PH]], !prof [[PROF3]]
+; I64: [[VEC_EPILOG_PH]]:
+; I64-NEXT: [[VEC_EPILOG_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC1]], %[[VEC_EPILOG_ITER_CHECK]] ], [ 0, %[[VECTOR_PH]] ]
; I64-NEXT: [[N_MOD_VF:%.*]] = urem i64 [[N_EXT]], 4
; I64-NEXT: [[N_VEC:%.*]] = sub i64 [[N_EXT]], [[N_MOD_VF]]
; I64-NEXT: br label %[[VECTOR_BODY:.*]]
; I64: [[VECTOR_BODY]]:
-; I64-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; I64-NEXT: [[INDEX:%.*]] = phi i64 [ [[VEC_EPILOG_RESUME_VAL]], %[[VEC_EPILOG_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
; I64-NEXT: [[TMP1:%.*]] = add i64 [[INDEX]], 1
; I64-NEXT: [[TMP2:%.*]] = add i64 [[INDEX]], 2
; I64-NEXT: [[TMP3:%.*]] = add i64 [[INDEX]], 3
@@ -373,10 +487,10 @@ define void @test_store_loaded_value(ptr noalias %src, ptr noalias %dst, i32 %n)
; I64-NEXT: store double [[TMP11]], ptr [[TMP19]], align 8
; I64-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 4
; I64-NEXT: [[TMP20:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
-; I64-NEXT: br i1 [[TMP20]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP6:![0-9]+]]
+; I64-NEXT: br i1 [[TMP20]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP7:![0-9]+]]
; I64: [[MIDDLE_BLOCK]]:
; I64-NEXT: [[CMP_N:%.*]] = icmp eq i64 [[N_EXT]], [[N_VEC]]
-; I64-NEXT: br i1 [[CMP_N]], [[EXIT_LOOPEXIT:label %.*]], label %[[SCALAR_PH]]
+; I64-NEXT: br i1 [[CMP_N]], [[EXIT_LOOPEXIT]], label %[[SCALAR_PH]]
; I64: [[SCALAR_PH]]:
;
; I32-LABEL: define void @test_store_loaded_value(
@@ -475,7 +589,7 @@ define double @test_load_used_by_other_load_scev(ptr %ptr.a, ptr %ptr.b, ptr %pt
; I64-NEXT: [[TMP2:%.*]] = fmul <2 x double> [[BROADCAST_SPLAT]], zeroinitializer
; I64-NEXT: [[IV]] = add nuw i64 [[INDEX]], 2
; I64-NEXT: [[EXITCOND:%.*]] = icmp eq i64 [[IV]], 100
-; I64-NEXT: br i1 [[EXITCOND]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP8:![0-9]+]]
+; I64-NEXT: br i1 [[EXITCOND]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP9:![0-9]+]]
; I64: [[MIDDLE_BLOCK]]:
; I64-NEXT: [[VECTOR_RECUR_EXTRACT:%.*]] = extractelement <2 x double> [[TMP2]], i32 1
; I64-NEXT: br label %[[SCALAR_PH:.*]]
@@ -950,7 +1064,7 @@ define void @address_use_in_different_block(ptr noalias %dst, ptr %src.0, ptr %s
; I64-NEXT: store double [[TMP91]], ptr [[TMP83]], align 8
; I64-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 8
; I64-NEXT: [[TMP92:%.*]] = icmp eq i64 [[INDEX_NEXT]], 96
-; I64-NEXT: br i1 [[TMP92]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP10:![0-9]+]]
+; I64-NEXT: br i1 [[TMP92]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP11:![0-9]+]]
; I64: [[MIDDLE_BLOCK]]:
; I64-NEXT: br label %[[SCALAR_PH:.*]]
; I64: [[SCALAR_PH]]:
@@ -1139,7 +1253,7 @@ define void @replicated_load_wide_store_derived_iv_zext_and(ptr noalias %src, pt
; I64-NEXT: store <4 x float> [[TMP62]], ptr [[TMP64]], align 4
; I64-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 8
; I64-NEXT: [[TMP65:%.*]] = icmp eq i64 [[INDEX_NEXT]], 128
-; I64-NEXT: br i1 [[TMP65]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP12:![0-9]+]]
+; I64-NEXT: br i1 [[TMP65]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP13:![0-9]+]]
; I64: [[MIDDLE_BLOCK]]:
; I64-NEXT: br label %[[SCALAR_PH]]
; I64: [[SCALAR_PH]]:
@@ -1261,7 +1375,7 @@ define void @replicated_load_wide_store_derived_iv_zext_and2(ptr noalias %dst, p
; I64-NEXT: store <4 x float> [[TMP54]], ptr [[TMP56]], align 4
; I64-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 8
; I64-NEXT: [[TMP57:%.*]] = icmp eq i64 [[INDEX_NEXT]], 128
-; I64-NEXT: br i1 [[TMP57]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP14:![0-9]+]]
+; I64-NEXT: br i1 [[TMP57]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP15:![0-9]+]]
; I64: [[MIDDLE_BLOCK]]:
; I64-NEXT: br label %[[SCALAR_PH]]
; I64: [[SCALAR_PH]]:
>From 1ced5834ce0d363a30976a8580453098c9a315a1 Mon Sep 17 00:00:00 2001
From: David Sherwood <david.sherwood at arm.com>
Date: Tue, 14 Apr 2026 13:25:58 +0000
Subject: [PATCH 2/2] Fix formatting
---
llvm/lib/Transforms/Vectorize/VPlanRecipes.cpp | 7 +++----
1 file changed, 3 insertions(+), 4 deletions(-)
diff --git a/llvm/lib/Transforms/Vectorize/VPlanRecipes.cpp b/llvm/lib/Transforms/Vectorize/VPlanRecipes.cpp
index 7ffc9a227d5c7..b6387113de404 100644
--- a/llvm/lib/Transforms/Vectorize/VPlanRecipes.cpp
+++ b/llvm/lib/Transforms/Vectorize/VPlanRecipes.cpp
@@ -3618,10 +3618,9 @@ InstructionCost VPReplicateRecipe::computeCost(ElementCount VF,
TTI::VectorInstrContext VIC =
IsLoad ? TTI::VectorInstrContext::Load : TTI::VectorInstrContext::Store;
- InstructionCost Cost =
- (ScalarCost * VF.getFixedValue()) +
- Ctx.getScalarizationOverhead(ResultTy, OpsToScalarize, VF, this, VIC,
- true);
+ InstructionCost Cost = (ScalarCost * VF.getFixedValue()) +
+ Ctx.getScalarizationOverhead(
+ ResultTy, OpsToScalarize, VF, this, VIC, true);
const VPRegionBlock *ParentRegion = getRegion();
if (ParentRegion && ParentRegion->isReplicator()) {
More information about the llvm-commits
mailing list