[llvm] [LV] Only use legacy scalarization costs with replicate regions. (PR #212738)
Florian Hahn via llvm-commits
llvm-commits at lists.llvm.org
Wed Jul 29 04:06:54 PDT 2026
https://github.com/fhahn created https://github.com/llvm/llvm-project/pull/212738
The scalarization costs in InstsToScalarize are based on the assumption that the instructions are scalarized and predicated. There are a number of VPlan transformations that can simplify/remove replicate regions. If there are no replicate regions in a plan, nothing is predicated and scalarized, so the costs in InstsToScalarize will be inaccurate.
Skip the fallback in those cases, using the more accurate VPlan-based cost info.
>From f863e06ab31bb57cec373ae5b003d6e186a34899 Mon Sep 17 00:00:00 2001
From: Florian Hahn <flo at fhahn.com>
Date: Tue, 28 Jul 2026 12:58:30 +0100
Subject: [PATCH] [LV] Only use legacy scalarization costs with replicate
regions.
The scalarization costs in InstsToScalarize are based on the assumption
that the instructions are scalarized and predicated. There are a number
of VPlan transformations that can simplify/remove replicate regions. If
there are no replicate regions in a plan, nothing is predicated and
scalarized, so the costs in InstsToScalarize will be inaccurate.
Skip the fallback in those cases, using the more accurate VPlan-based
cost info.
---
.../Transforms/Vectorize/LoopVectorize.cpp | 5 +++
.../CostModel/masked-interleaved-store-i16.ll | 4 +--
.../X86/CostModel/store-scalarization-cost.ll | 6 ++--
.../X86/replicating-load-store-costs.ll | 33 +++++++++++--------
4 files changed, 29 insertions(+), 19 deletions(-)
diff --git a/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp b/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
index aa099b84ed61b..ec674f12a7c1a 100644
--- a/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
+++ b/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
@@ -5743,6 +5743,11 @@ LoopVectorizationPlanner::precomputeCosts(VPlan &Plan, ElementCount VF,
Cost += ForcedCost;
}
+ // Don't apply legacy scalarization costs if nothing remains scalar &
+ // predicated.
+ if (!hasReplicatorRegion(Plan))
+ return Cost;
+
auto UseVPlanCostModel = [](Instruction *I) -> bool {
switch (I->getOpcode()) {
case Instruction::SDiv:
diff --git a/llvm/test/Transforms/LoopVectorize/X86/CostModel/masked-interleaved-store-i16.ll b/llvm/test/Transforms/LoopVectorize/X86/CostModel/masked-interleaved-store-i16.ll
index c2e4cebc9aae6..cd59ce8482f60 100644
--- a/llvm/test/Transforms/LoopVectorize/X86/CostModel/masked-interleaved-store-i16.ll
+++ b/llvm/test/Transforms/LoopVectorize/X86/CostModel/masked-interleaved-store-i16.ll
@@ -147,8 +147,8 @@ define void @test(ptr noalias nocapture %points, ptr noalias nocapture readonly
; ENABLED_MASKED_STRIDED: LV: Found an estimated cost of 1 for VF 1 For instruction: store i16 %0, ptr %arrayidx6, align 2
; ENABLED_MASKED_STRIDED: Cost of 2 for VF 2: profitable to scalarize store i16 %0, ptr %arrayidx6, align 2
; ENABLED_MASKED_STRIDED: Cost of 4 for VF 4: profitable to scalarize store i16 %0, ptr %arrayidx6, align 2
-; ENABLED_MASKED_STRIDED: Cost of 8 for VF 8: profitable to scalarize store i16 %0, ptr %arrayidx6, align 2
-; ENABLED_MASKED_STRIDED: Cost of 16.5 for VF 16: profitable to scalarize store i16 %0, ptr %arrayidx6, align 2
+; ENABLED_MASKED_STRIDED: Cost of 12 for VF 8: INTERLEAVE-GROUP with factor 3, ir<%arrayidx6>, ir<%cmp1>
+; ENABLED_MASKED_STRIDED: Cost of 22 for VF 16: INTERLEAVE-GROUP with factor 3, ir<%arrayidx6>, ir<%cmp1>
;
entry:
br label %for.body
diff --git a/llvm/test/Transforms/LoopVectorize/X86/CostModel/store-scalarization-cost.ll b/llvm/test/Transforms/LoopVectorize/X86/CostModel/store-scalarization-cost.ll
index 18831c3317b45..fc604b185d6d4 100644
--- a/llvm/test/Transforms/LoopVectorize/X86/CostModel/store-scalarization-cost.ll
+++ b/llvm/test/Transforms/LoopVectorize/X86/CostModel/store-scalarization-cost.ll
@@ -12,10 +12,10 @@ target triple = "x86_64-unknown-linux-gnu"
; The predicated store to the invariant address is sunk out of the loop.
define void @invariant_pred_store_sunk_out_of_loop(ptr noalias %dst, ptr noalias readonly %src) {
; CHECK-LABEL: 'invariant_pred_store_sunk_out_of_loop'
-; CHECK: Cost of 5 for VF 2: profitable to scalarize store i64 %sum.1, ptr %gep.dst, align 8
; CHECK: Cost of 1 for VF 2: CLONE store vp<[[VP8:%[0-9]+]]>, ir<%gep.dst>
-; CHECK: Cost for VF 2: 17 (Estimated cost per lane: 8.5)
-; CHECK: LV: Selecting VF: 1.
+; CHECK: Cost for VF 2: 12 (Estimated cost per lane: 6)
+; CHECK: LV: Selecting VF: 2.
+; CHECK: Cost of 1 for VF 2: CLONE store vp<[[VP8]]>, ir<%gep.dst>
;
entry:
%gep.dst = getelementptr inbounds i64, ptr %dst, i64 42
diff --git a/llvm/test/Transforms/LoopVectorize/X86/replicating-load-store-costs.ll b/llvm/test/Transforms/LoopVectorize/X86/replicating-load-store-costs.ll
index 39e80134f4fa9..8ad4876b367bc 100644
--- a/llvm/test/Transforms/LoopVectorize/X86/replicating-load-store-costs.ll
+++ b/llvm/test/Transforms/LoopVectorize/X86/replicating-load-store-costs.ll
@@ -1264,27 +1264,32 @@ exit:
define void @invariant_pred_store_sunk_out_of_loop(ptr noalias %dst, ptr noalias readonly %src) #1 {
; I64-LABEL: define void @invariant_pred_store_sunk_out_of_loop(
; I64-SAME: ptr noalias [[DST:%.*]], ptr noalias readonly [[SRC:%.*]]) #[[ATTR1:[0-9]+]] {
-; I64-NEXT: [[ENTRY:.*]]:
+; I64-NEXT: [[ENTRY:.*:]]
; I64-NEXT: [[GEP_DST:%.*]] = getelementptr inbounds i64, ptr [[DST]], i64 42
; I64-NEXT: br label %[[LOOP:.*]]
; I64: [[LOOP]]:
-; I64-NEXT: [[IV:%.*]] = phi i64 [ 0, %[[ENTRY]] ], [ [[IV_NEXT:%.*]], %[[LATCH:.*]] ]
-; I64-NEXT: [[SUM:%.*]] = phi i64 [ 0, %[[ENTRY]] ], [ [[SUM_2:%.*]], %[[LATCH]] ]
+; I64-NEXT: br label %[[VECTOR_BODY:.*]]
+; I64: [[VECTOR_BODY]]:
+; I64-NEXT: [[IV:%.*]] = phi i64 [ 0, %[[LOOP]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; I64-NEXT: [[VEC_PHI:%.*]] = phi <2 x i64> [ zeroinitializer, %[[LOOP]] ], [ [[TMP4:%.*]], %[[VECTOR_BODY]] ]
+; I64-NEXT: [[VEC_PHI1:%.*]] = phi <2 x i64> [ zeroinitializer, %[[LOOP]] ], [ [[TMP5:%.*]], %[[VECTOR_BODY]] ]
; I64-NEXT: [[GEP_SRC:%.*]] = getelementptr inbounds i64, ptr [[SRC]], i64 [[IV]]
-; I64-NEXT: [[L:%.*]] = load i64, ptr [[GEP_SRC]], align 8
-; I64-NEXT: [[SUM_1:%.*]] = add nsw i64 [[L]], [[SUM]]
-; I64-NEXT: [[C:%.*]] = icmp sgt i64 [[L]], 0
-; I64-NEXT: br i1 [[C]], label %[[IF_THEN:.*]], label %[[LATCH]]
+; I64-NEXT: [[TMP1:%.*]] = getelementptr inbounds i64, ptr [[GEP_SRC]], i64 2
+; I64-NEXT: [[WIDE_LOAD:%.*]] = load <2 x i64>, ptr [[GEP_SRC]], align 8
+; I64-NEXT: [[WIDE_LOAD2:%.*]] = load <2 x i64>, ptr [[TMP1]], align 8
+; I64-NEXT: [[TMP2:%.*]] = add <2 x i64> [[WIDE_LOAD]], [[VEC_PHI]]
+; I64-NEXT: [[TMP3:%.*]] = add <2 x i64> [[WIDE_LOAD2]], [[VEC_PHI1]]
+; I64-NEXT: [[TMP4]] = add <2 x i64> [[TMP2]], splat (i64 1)
+; I64-NEXT: [[TMP5]] = add <2 x i64> [[TMP3]], splat (i64 1)
+; I64-NEXT: [[INDEX_NEXT]] = add nuw i64 [[IV]], 4
+; I64-NEXT: [[TMP6:%.*]] = icmp eq i64 [[INDEX_NEXT]], 1000
+; I64-NEXT: br i1 [[TMP6]], label %[[IF_THEN:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP16:![0-9]+]]
; I64: [[IF_THEN]]:
+; I64-NEXT: [[BIN_RDX:%.*]] = add <2 x i64> [[TMP5]], [[TMP4]]
+; I64-NEXT: [[SUM_1:%.*]] = call i64 @llvm.vector.reduce.add.v2i64(<2 x i64> [[BIN_RDX]])
; I64-NEXT: store i64 [[SUM_1]], ptr [[GEP_DST]], align 8
-; I64-NEXT: br label %[[LATCH]]
+; I64-NEXT: br label %[[LATCH:.*]]
; I64: [[LATCH]]:
-; I64-NEXT: [[SUM_2]] = add nsw i64 [[SUM_1]], 1
-; I64-NEXT: store i64 [[SUM_2]], ptr [[GEP_DST]], align 8
-; I64-NEXT: [[IV_NEXT]] = add nuw nsw i64 [[IV]], 1
-; I64-NEXT: [[EC:%.*]] = icmp eq i64 [[IV_NEXT]], 1000
-; I64-NEXT: br i1 [[EC]], label %[[EXIT:.*]], label %[[LOOP]]
-; I64: [[EXIT]]:
; I64-NEXT: ret void
;
; I32-LABEL: define void @invariant_pred_store_sunk_out_of_loop(
More information about the llvm-commits
mailing list