[llvm] [AArch64] [CostModel] Improve costs for scalar inserts into fixed-length SVE constant vector (PR #223638)
Utpal Bora via llvm-commits
llvm-commits at lists.llvm.org
Tue Sep 15 02:42:09 PDT 2026
https://github.com/utpalbora created https://github.com/llvm/llvm-project/pull/223638
SLP does not account for constant materialization when modelling the cost of
gathering operands that insert scalars into non-uniform constant vectors. This
can underestimate the total cost of constructing a fixed-length vector on AArch64.
For fixed-length floating-point vectors legalized to more than 128 bits, this
change conservatively models:
- scalar insertion across the legalized 128-bit subparts;
- the splices required to combine those subparts;
- one predicate shared by the splice operations; and
- constant-pool address materialization.
>From aeb4b149e420b50f0ccef9533a2d26879962c9d9 Mon Sep 17 00:00:00 2001
From: Utpal Bora <utpal.bora2 at arm.com>
Date: Mon, 14 Sep 2026 14:20:08 +0000
Subject: [PATCH 1/2] pre-commit test
---
.../AArch64/sve-constant-vector-inserts.ll | 40 +++++++++++++++++++
1 file changed, 40 insertions(+)
create mode 100644 llvm/test/Transforms/SLPVectorizer/AArch64/sve-constant-vector-inserts.ll
diff --git a/llvm/test/Transforms/SLPVectorizer/AArch64/sve-constant-vector-inserts.ll b/llvm/test/Transforms/SLPVectorizer/AArch64/sve-constant-vector-inserts.ll
new file mode 100644
index 0000000000000..69c0c5dd0c1c2
--- /dev/null
+++ b/llvm/test/Transforms/SLPVectorizer/AArch64/sve-constant-vector-inserts.ll
@@ -0,0 +1,40 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
+; RUN: opt -passes=slp-vectorizer -mtriple=aarch64-none-linux-gnu \
+; RUN: -mattr=+sve -S %s | FileCheck %s
+
+; Check that SLP leaves the reduction scalar when constructing its fixed-length
+; vector operands would require stack materialization
+
+define double @insert_into_constant_vector(double %x, double %y) vscale_range(2, 2) {
+; CHECK-LABEL: @insert_into_constant_vector(
+; CHECK-NEXT: [[MUL0:%.*]] = fmul fast double [[X:%.*]], 0.000000e+00
+; CHECK-NEXT: [[TMP1:%.*]] = insertelement <4 x double> <double poison, double poison, double 0.000000e+00, double 1.000000e+00>, double [[Y:%.*]], i64 0
+; CHECK-NEXT: [[TMP2:%.*]] = insertelement <4 x double> [[TMP1]], double [[X]], i64 1
+; CHECK-NEXT: [[TMP3:%.*]] = fmul fast <4 x double> <double 0.000000e+00, double 1.000000e+00, double 0.000000e+00, double 0.000000e+00>, [[TMP2]]
+; CHECK-NEXT: [[TMP4:%.*]] = insertelement <4 x double> <double 1.000000e+00, double poison, double 0.000000e+00, double 0.000000e+00>, double [[X]], i64 1
+; CHECK-NEXT: [[TMP5:%.*]] = insertelement <4 x double> <double 1.000000e+00, double poison, double 0.000000e+00, double 0.000000e+00>, double [[Y]], i64 1
+; CHECK-NEXT: [[TMP6:%.*]] = fmul fast <4 x double> [[TMP4]], [[TMP5]]
+; CHECK-NEXT: [[MUL6:%.*]] = call reassoc nsz double @llvm.vector.reduce.fadd.v4f64(double 0.000000e+00, <4 x double> [[TMP6]])
+; CHECK-NEXT: [[TMP8:%.*]] = call reassoc nsz double @llvm.vector.reduce.fadd.v4f64(double 0.000000e+00, <4 x double> [[TMP3]])
+; CHECK-NEXT: [[ADD8:%.*]] = fadd reassoc nsz double [[TMP8]], [[MUL0]]
+; CHECK-NEXT: [[ADD9:%.*]] = fadd reassoc nsz double [[ADD8]], [[MUL6]]
+; CHECK-NEXT: ret double [[ADD9]]
+;
+ %mul0 = fmul fast double %x, 0.0
+ %add0 = fadd reassoc nsz double 0.0, %mul0
+ %mul1 = fmul fast double 0.0, %y
+ %add1 = fadd reassoc nsz double %add0, %mul1
+ %mul2 = fmul fast double 1.0, %x
+ %add2 = fadd reassoc nsz double %add1, %mul2
+ %mul3 = fmul fast double 0.0, 0.0
+ %add3 = fadd reassoc nsz double %add2, %mul3
+ %mul4 = fmul fast double 1.0, 1.0
+ %add4 = fadd reassoc nsz double %add3, %mul4
+ %mul5 = fmul fast double %x, %y
+ %add5 = fadd reassoc nsz double %add4, %mul5
+ %mul6 = fmul fast double 0.0, 0.0
+ %add6 = fadd reassoc nsz double %add5, %mul6
+ %mul7 = fmul fast double 0.0, 0.0
+ %add7 = fadd reassoc nsz double %add6, %mul7
+ ret double %add7
+}
>From fbb40f8af5894e8ee58388346433bb2e9052a9d1 Mon Sep 17 00:00:00 2001
From: Utpal Bora <utpal.bora2 at arm.com>
Date: Mon, 14 Sep 2026 15:15:53 +0000
Subject: [PATCH 2/2] Improved scalarization overhead for constant vector
inserts
---
.../AArch64/AArch64TargetTransformInfo.cpp | 54 +++++++++++++++++--
.../AArch64/sve-constant-vector-inserts.ll | 21 ++++----
2 files changed, 62 insertions(+), 13 deletions(-)
diff --git a/llvm/lib/Target/AArch64/AArch64TargetTransformInfo.cpp b/llvm/lib/Target/AArch64/AArch64TargetTransformInfo.cpp
index e98be7e5a6371..65fc48f86f5bd 100644
--- a/llvm/lib/Target/AArch64/AArch64TargetTransformInfo.cpp
+++ b/llvm/lib/Target/AArch64/AArch64TargetTransformInfo.cpp
@@ -4866,9 +4866,57 @@ InstructionCost AArch64TTIImpl::getScalarizationOverhead(
TTI::VectorInstrContext VIC) const {
if (isa<ScalableVectorType>(Ty))
return InstructionCost::getInvalid();
- if (Ty->getElementType()->isFloatingPointTy())
- return BaseT::getScalarizationOverhead(Ty, DemandedElts, Insert, Extract,
- CostKind);
+ if (Ty->getElementType()->isFloatingPointTy()) {
+ InstructionCost Cost = BaseT::getScalarizationOverhead(
+ Ty, DemandedElts, Insert, Extract, CostKind);
+
+ if (!Insert || VL.empty())
+ return Cost;
+
+ auto LT = getTypeLegalizationCost(Ty);
+ if (!ST->isNeonAvailable() || !LT.second.isFixedLengthVector() ||
+ LT.second.getFixedSizeInBits() <= 128)
+ return Cost;
+
+ auto HasNonUniformConstants = [&VL]() -> bool {
+ Value *FirstConst = nullptr;
+ for (Value *V : VL) {
+ if (isa<UndefValue>(V) || !isa<Constant>(V) ||
+ isa<ConstantExpr, GlobalValue>(V))
+ continue;
+ if (!FirstConst)
+ FirstConst = V;
+ else if (V != FirstConst)
+ return true;
+ }
+ return false;
+ };
+
+ if (!HasNonUniformConstants())
+ return Cost;
+
+ // Conservatively assume each legalized vector part is assembled from
+ // 128-bit subvectors, requiring NumSubParts - 1 splices and one shared
+ // predicate.
+ unsigned Num128BitSubParts =
+ LT.second.getFixedSizeInBits() / AArch64::SVEBitsPerBlock;
+ InstructionCost InsertExtractCost = LT.first * Num128BitSubParts *
+ DemandedElts.popcount() *
+ (Insert + Extract);
+
+ auto *ContainerTy = ScalableVectorType::get(Ty->getElementType(),
+ AArch64::SVEBitsPerBlock /
+ Ty->getScalarSizeInBits());
+ InstructionCost SpliceCost =
+ LT.first * (Num128BitSubParts - 1) *
+ getSpliceCost(ContainerTy, /*Index=*/0, CostKind);
+ Cost = InsertExtractCost + SpliceCost;
+ Cost += 1; // shared predicate cost
+ Cost += 2; // constant-pool address materialization.
+
+ return Cost;
+ }
+
unsigned VecInstCost =
CostKind == TTI::TCK_CodeSize ? 1 : ST->getVectorInsertExtractBaseCost();
return DemandedElts.popcount() * (Insert + Extract) * VecInstCost;
diff --git a/llvm/test/Transforms/SLPVectorizer/AArch64/sve-constant-vector-inserts.ll b/llvm/test/Transforms/SLPVectorizer/AArch64/sve-constant-vector-inserts.ll
index 69c0c5dd0c1c2..7f2f331e37730 100644
--- a/llvm/test/Transforms/SLPVectorizer/AArch64/sve-constant-vector-inserts.ll
+++ b/llvm/test/Transforms/SLPVectorizer/AArch64/sve-constant-vector-inserts.ll
@@ -8,17 +8,18 @@
define double @insert_into_constant_vector(double %x, double %y) vscale_range(2, 2) {
; CHECK-LABEL: @insert_into_constant_vector(
; CHECK-NEXT: [[MUL0:%.*]] = fmul fast double [[X:%.*]], 0.000000e+00
-; CHECK-NEXT: [[TMP1:%.*]] = insertelement <4 x double> <double poison, double poison, double 0.000000e+00, double 1.000000e+00>, double [[Y:%.*]], i64 0
-; CHECK-NEXT: [[TMP2:%.*]] = insertelement <4 x double> [[TMP1]], double [[X]], i64 1
-; CHECK-NEXT: [[TMP3:%.*]] = fmul fast <4 x double> <double 0.000000e+00, double 1.000000e+00, double 0.000000e+00, double 0.000000e+00>, [[TMP2]]
-; CHECK-NEXT: [[TMP4:%.*]] = insertelement <4 x double> <double 1.000000e+00, double poison, double 0.000000e+00, double 0.000000e+00>, double [[X]], i64 1
-; CHECK-NEXT: [[TMP5:%.*]] = insertelement <4 x double> <double 1.000000e+00, double poison, double 0.000000e+00, double 0.000000e+00>, double [[Y]], i64 1
-; CHECK-NEXT: [[TMP6:%.*]] = fmul fast <4 x double> [[TMP4]], [[TMP5]]
-; CHECK-NEXT: [[MUL6:%.*]] = call reassoc nsz double @llvm.vector.reduce.fadd.v4f64(double 0.000000e+00, <4 x double> [[TMP6]])
-; CHECK-NEXT: [[TMP8:%.*]] = call reassoc nsz double @llvm.vector.reduce.fadd.v4f64(double 0.000000e+00, <4 x double> [[TMP3]])
-; CHECK-NEXT: [[ADD8:%.*]] = fadd reassoc nsz double [[TMP8]], [[MUL0]]
+; CHECK-NEXT: [[ADD0:%.*]] = fadd reassoc nsz double 0.000000e+00, [[MUL0]]
+; CHECK-NEXT: [[MUL1:%.*]] = fmul fast double 0.000000e+00, [[Y:%.*]]
+; CHECK-NEXT: [[ADD8:%.*]] = fadd reassoc nsz double [[ADD0]], [[MUL1]]
+; CHECK-NEXT: [[MUL6:%.*]] = fmul fast double 1.000000e+00, [[X]]
; CHECK-NEXT: [[ADD9:%.*]] = fadd reassoc nsz double [[ADD8]], [[MUL6]]
-; CHECK-NEXT: ret double [[ADD9]]
+; CHECK-NEXT: [[ADD3:%.*]] = fadd reassoc nsz double [[ADD9]], 0.000000e+00
+; CHECK-NEXT: [[ADD4:%.*]] = fadd reassoc nsz double [[ADD3]], 1.000000e+00
+; CHECK-NEXT: [[MUL5:%.*]] = fmul fast double [[X]], [[Y]]
+; CHECK-NEXT: [[ADD5:%.*]] = fadd reassoc nsz double [[ADD4]], [[MUL5]]
+; CHECK-NEXT: [[ADD6:%.*]] = fadd reassoc nsz double [[ADD5]], 0.000000e+00
+; CHECK-NEXT: [[ADD7:%.*]] = fadd reassoc nsz double [[ADD6]], 0.000000e+00
+; CHECK-NEXT: ret double [[ADD7]]
;
%mul0 = fmul fast double %x, 0.0
%add0 = fadd reassoc nsz double 0.0, %mul0
More information about the llvm-commits
mailing list