[llvm] [AArch64] Increase the relative cost of vector i64 multiply on Neoverse V3ae. (PR #207723)
David Green via llvm-commits
llvm-commits at lists.llvm.org
Mon Jul 6 05:48:04 PDT 2026
https://github.com/davemgreen created https://github.com/llvm/llvm-project/pull/207723
The throughput of vector nxv2i64 multiplies on neoverse v3ae is 1/2, compared
to the throughput of 2 for integer multiplies. This large difference can mean
it is more profitable than normal to use scalar loops as oppposed to
vectorization.
This adds a subtarget feature that increases the cost multiple by 4 for 64bit
vector multiplies for specific CPUs. The cost model of llvm does not mean that
we can model throughputs correctly, but this should help. The same feature is
added to N2 as it has a similar difference between vector and scalar multiply
cost throughputs.
>From ed51cd5b45b136cee28fff55e39f77ceec5d7272 Mon Sep 17 00:00:00 2001
From: David Green <david.green at arm.com>
Date: Mon, 6 Jul 2026 10:32:20 +0100
Subject: [PATCH 1/2] [AArch64] Add a basic i128 mul cost.
This adds a basic cost for i128 multiply instructions, which has a basic cost
of 4 and scales n^2 with bitsize.
---
.../Target/AArch64/AArch64TargetTransformInfo.cpp | 11 +++++++++++
.../Analysis/CostModel/AArch64/arith-overflow.ll | 12 ++++++------
llvm/test/Analysis/CostModel/AArch64/arith.ll | 6 +++---
llvm/test/Analysis/CostModel/AArch64/div.ll | 8 ++++----
llvm/test/Analysis/CostModel/AArch64/mul.ll | 2 +-
llvm/test/Analysis/CostModel/AArch64/rem.ll | 8 ++++----
6 files changed, 29 insertions(+), 18 deletions(-)
diff --git a/llvm/lib/Target/AArch64/AArch64TargetTransformInfo.cpp b/llvm/lib/Target/AArch64/AArch64TargetTransformInfo.cpp
index edd95625e13ea..dfd0895b5c748 100644
--- a/llvm/lib/Target/AArch64/AArch64TargetTransformInfo.cpp
+++ b/llvm/lib/Target/AArch64/AArch64TargetTransformInfo.cpp
@@ -4535,6 +4535,17 @@ InstructionCost AArch64TTIImpl::getArithmeticInstrCost(
case ISD::SUB:
return LT.first; // Also works for i128
case ISD::MUL:
+ // i128 multiply is umulh + 2*madd + mul and grows ~O(Bitwidth^2). For
+ // scalable vectors the cost of LT.first will be invalid, leading to an
+ // invalid cost overall.
+ if (Ty->getScalarSizeInBits() > 64) {
+ unsigned NumLanes = isa<FixedVectorType>(Ty)
+ ? cast<FixedVectorType>(Ty)->getNumElements()
+ : 1;
+ InstructionCost CostPerLane = LT.first / NumLanes;
+ return CostPerLane * CostPerLane * NumLanes;
+ }
+
if (LT.second == MVT::v2i64) {
// When SVE is available, then we can lower the v2i64 operation using
// the SVE mul instruction, which has a lower cost.
diff --git a/llvm/test/Analysis/CostModel/AArch64/arith-overflow.ll b/llvm/test/Analysis/CostModel/AArch64/arith-overflow.ll
index 5bbb80f693655..60cd03e57f885 100644
--- a/llvm/test/Analysis/CostModel/AArch64/arith-overflow.ll
+++ b/llvm/test/Analysis/CostModel/AArch64/arith-overflow.ll
@@ -278,9 +278,9 @@ declare {<64 x i8>, <64 x i1>} @llvm.smul.with.overflow.v64i8(<64 x i8>, <64 x
define i32 @smul(i32 %arg) {
; CHECK-LABEL: 'smul'
; CHECK-NEXT: Cost Model: Found costs of 3 for: %I64 = call { i64, i1 } @llvm.smul.with.overflow.i64(i64 undef, i64 undef)
-; CHECK-NEXT: Cost Model: Found costs of RThru:34 CodeSize:20 Lat:28 SizeLat:28 for: %V2I64 = call { <2 x i64>, <2 x i1> } @llvm.smul.with.overflow.v2i64(<2 x i64> undef, <2 x i64> undef)
-; CHECK-NEXT: Cost Model: Found costs of RThru:68 CodeSize:37 Lat:53 SizeLat:53 for: %V4I64 = call { <4 x i64>, <4 x i1> } @llvm.smul.with.overflow.v4i64(<4 x i64> undef, <4 x i64> undef)
-; CHECK-NEXT: Cost Model: Found costs of RThru:136 CodeSize:71 Lat:103 SizeLat:103 for: %V8I64 = call { <8 x i64>, <8 x i1> } @llvm.smul.with.overflow.v8i64(<8 x i64> undef, <8 x i64> undef)
+; CHECK-NEXT: Cost Model: Found costs of RThru:38 CodeSize:20 Lat:28 SizeLat:28 for: %V2I64 = call { <2 x i64>, <2 x i1> } @llvm.smul.with.overflow.v2i64(<2 x i64> undef, <2 x i64> undef)
+; CHECK-NEXT: Cost Model: Found costs of RThru:76 CodeSize:37 Lat:53 SizeLat:53 for: %V4I64 = call { <4 x i64>, <4 x i1> } @llvm.smul.with.overflow.v4i64(<4 x i64> undef, <4 x i64> undef)
+; CHECK-NEXT: Cost Model: Found costs of RThru:152 CodeSize:71 Lat:103 SizeLat:103 for: %V8I64 = call { <8 x i64>, <8 x i1> } @llvm.smul.with.overflow.v8i64(<8 x i64> undef, <8 x i64> undef)
; CHECK-NEXT: Cost Model: Found costs of 2 for: %I32 = call { i32, i1 } @llvm.smul.with.overflow.i32(i32 undef, i32 undef)
; CHECK-NEXT: Cost Model: Found costs of RThru:38 CodeSize:10 Lat:10 SizeLat:10 for: %V4I32 = call { <4 x i32>, <4 x i1> } @llvm.smul.with.overflow.v4i32(<4 x i32> undef, <4 x i32> undef)
; CHECK-NEXT: Cost Model: Found costs of RThru:76 CodeSize:17 Lat:17 SizeLat:17 for: %V8I32 = call { <8 x i32>, <8 x i1> } @llvm.smul.with.overflow.v8i32(<8 x i32> undef, <8 x i32> undef)
@@ -341,9 +341,9 @@ declare {<64 x i8>, <64 x i1>} @llvm.umul.with.overflow.v64i8(<64 x i8>, <64 x
define i32 @umul(i32 %arg) {
; CHECK-LABEL: 'umul'
; CHECK-NEXT: Cost Model: Found costs of 3 for: %I64 = call { i64, i1 } @llvm.umul.with.overflow.i64(i64 undef, i64 undef)
-; CHECK-NEXT: Cost Model: Found costs of RThru:33 CodeSize:19 Lat:27 SizeLat:27 for: %V2I64 = call { <2 x i64>, <2 x i1> } @llvm.umul.with.overflow.v2i64(<2 x i64> undef, <2 x i64> undef)
-; CHECK-NEXT: Cost Model: Found costs of RThru:66 CodeSize:36 Lat:52 SizeLat:52 for: %V4I64 = call { <4 x i64>, <4 x i1> } @llvm.umul.with.overflow.v4i64(<4 x i64> undef, <4 x i64> undef)
-; CHECK-NEXT: Cost Model: Found costs of RThru:132 CodeSize:70 Lat:102 SizeLat:102 for: %V8I64 = call { <8 x i64>, <8 x i1> } @llvm.umul.with.overflow.v8i64(<8 x i64> undef, <8 x i64> undef)
+; CHECK-NEXT: Cost Model: Found costs of RThru:37 CodeSize:19 Lat:27 SizeLat:27 for: %V2I64 = call { <2 x i64>, <2 x i1> } @llvm.umul.with.overflow.v2i64(<2 x i64> undef, <2 x i64> undef)
+; CHECK-NEXT: Cost Model: Found costs of RThru:74 CodeSize:36 Lat:52 SizeLat:52 for: %V4I64 = call { <4 x i64>, <4 x i1> } @llvm.umul.with.overflow.v4i64(<4 x i64> undef, <4 x i64> undef)
+; CHECK-NEXT: Cost Model: Found costs of RThru:148 CodeSize:70 Lat:102 SizeLat:102 for: %V8I64 = call { <8 x i64>, <8 x i1> } @llvm.umul.with.overflow.v8i64(<8 x i64> undef, <8 x i64> undef)
; CHECK-NEXT: Cost Model: Found costs of 2 for: %I32 = call { i32, i1 } @llvm.umul.with.overflow.i32(i32 undef, i32 undef)
; CHECK-NEXT: Cost Model: Found costs of RThru:37 CodeSize:9 Lat:9 SizeLat:9 for: %V4I32 = call { <4 x i32>, <4 x i1> } @llvm.umul.with.overflow.v4i32(<4 x i32> undef, <4 x i32> undef)
; CHECK-NEXT: Cost Model: Found costs of RThru:74 CodeSize:16 Lat:16 SizeLat:16 for: %V8I32 = call { <8 x i32>, <8 x i1> } @llvm.umul.with.overflow.v8i32(<8 x i32> undef, <8 x i32> undef)
diff --git a/llvm/test/Analysis/CostModel/AArch64/arith.ll b/llvm/test/Analysis/CostModel/AArch64/arith.ll
index dd30dc63ddc85..f390fa8c163b9 100644
--- a/llvm/test/Analysis/CostModel/AArch64/arith.ll
+++ b/llvm/test/Analysis/CostModel/AArch64/arith.ll
@@ -132,7 +132,7 @@ define void @i128() {
; CHECK-LABEL: 'i128'
; CHECK-NEXT: Cost Model: Found costs of RThru:2 CodeSize:1 Lat:1 SizeLat:1 for: %c = add i128 undef, undef
; CHECK-NEXT: Cost Model: Found costs of RThru:2 CodeSize:1 Lat:1 SizeLat:1 for: %d = sub i128 undef, undef
-; CHECK-NEXT: Cost Model: Found costs of RThru:2 CodeSize:1 Lat:1 SizeLat:1 for: %e = mul i128 undef, undef
+; CHECK-NEXT: Cost Model: Found costs of RThru:4 CodeSize:1 Lat:1 SizeLat:1 for: %e = mul i128 undef, undef
; CHECK-NEXT: Cost Model: Found costs of RThru:2 CodeSize:1 Lat:1 SizeLat:1 for: %f = ashr i128 undef, undef
; CHECK-NEXT: Cost Model: Found costs of RThru:2 CodeSize:1 Lat:1 SizeLat:1 for: %g = lshr i128 undef, undef
; CHECK-NEXT: Cost Model: Found costs of RThru:2 CodeSize:1 Lat:1 SizeLat:1 for: %h = shl i128 undef, undef
@@ -157,7 +157,7 @@ define void @i256() {
; CHECK-LABEL: 'i256'
; CHECK-NEXT: Cost Model: Found costs of RThru:4 CodeSize:1 Lat:1 SizeLat:1 for: %c = add i256 undef, undef
; CHECK-NEXT: Cost Model: Found costs of RThru:4 CodeSize:1 Lat:1 SizeLat:1 for: %d = sub i256 undef, undef
-; CHECK-NEXT: Cost Model: Found costs of RThru:4 CodeSize:1 Lat:1 SizeLat:1 for: %e = mul i256 undef, undef
+; CHECK-NEXT: Cost Model: Found costs of RThru:16 CodeSize:1 Lat:1 SizeLat:1 for: %e = mul i256 undef, undef
; CHECK-NEXT: Cost Model: Found costs of RThru:4 CodeSize:1 Lat:1 SizeLat:1 for: %f = ashr i256 undef, undef
; CHECK-NEXT: Cost Model: Found costs of RThru:4 CodeSize:1 Lat:1 SizeLat:1 for: %g = lshr i256 undef, undef
; CHECK-NEXT: Cost Model: Found costs of RThru:4 CodeSize:1 Lat:1 SizeLat:1 for: %h = shl i256 undef, undef
@@ -499,7 +499,7 @@ define void @vi128() {
; CHECK-LABEL: 'vi128'
; CHECK-NEXT: Cost Model: Found costs of RThru:4 CodeSize:1 Lat:1 SizeLat:1 for: %c2 = add <2 x i128> undef, undef
; CHECK-NEXT: Cost Model: Found costs of RThru:4 CodeSize:1 Lat:1 SizeLat:1 for: %d2 = sub <2 x i128> undef, undef
-; CHECK-NEXT: Cost Model: Found costs of RThru:4 CodeSize:1 Lat:1 SizeLat:1 for: %e2 = mul <2 x i128> undef, undef
+; CHECK-NEXT: Cost Model: Found costs of RThru:8 CodeSize:1 Lat:1 SizeLat:1 for: %e2 = mul <2 x i128> undef, undef
; CHECK-NEXT: Cost Model: Found costs of RThru:4 CodeSize:1 Lat:1 SizeLat:1 for: %f2 = ashr <2 x i128> undef, undef
; CHECK-NEXT: Cost Model: Found costs of RThru:4 CodeSize:1 Lat:1 SizeLat:1 for: %g2 = lshr <2 x i128> undef, undef
; CHECK-NEXT: Cost Model: Found costs of RThru:4 CodeSize:1 Lat:1 SizeLat:1 for: %h2 = shl <2 x i128> undef, undef
diff --git a/llvm/test/Analysis/CostModel/AArch64/div.ll b/llvm/test/Analysis/CostModel/AArch64/div.ll
index 3a2358dba51b2..2c5be1acbbd7b 100644
--- a/llvm/test/Analysis/CostModel/AArch64/div.ll
+++ b/llvm/test/Analysis/CostModel/AArch64/div.ll
@@ -345,7 +345,7 @@ define void @sdiv_const() {
define void @udiv_const() {
; CHECK-LABEL: 'udiv_const'
-; CHECK-NEXT: Cost Model: Found costs of RThru:10 CodeSize:4 Lat:4 SizeLat:4 for: %I128 = udiv i128 undef, 7
+; CHECK-NEXT: Cost Model: Found costs of RThru:12 CodeSize:4 Lat:4 SizeLat:4 for: %I128 = udiv i128 undef, 7
; CHECK-NEXT: Cost Model: Found costs of 4 for: %I64 = udiv i64 undef, 7
; CHECK-NEXT: Cost Model: Found costs of RThru:32 CodeSize:4 Lat:4 SizeLat:4 for: %V2i64 = udiv <2 x i64> undef, <i64 6, i64 7>
; CHECK-NEXT: Cost Model: Found costs of RThru:64 CodeSize:4 Lat:4 SizeLat:4 for: %V4i64 = udiv <4 x i64> undef, <i64 4, i64 5, i64 6, i64 7>
@@ -462,7 +462,7 @@ define void @sdiv_uniformconst() {
define void @udiv_uniformconst() {
; CHECK-LABEL: 'udiv_uniformconst'
-; CHECK-NEXT: Cost Model: Found costs of RThru:10 CodeSize:4 Lat:4 SizeLat:4 for: %I128 = udiv i128 undef, 7
+; CHECK-NEXT: Cost Model: Found costs of RThru:12 CodeSize:4 Lat:4 SizeLat:4 for: %I128 = udiv i128 undef, 7
; CHECK-NEXT: Cost Model: Found costs of 4 for: %I64 = udiv i64 undef, 7
; CHECK-NEXT: Cost Model: Found costs of RThru:32 CodeSize:4 Lat:4 SizeLat:4 for: %V2i64 = udiv <2 x i64> undef, splat (i64 7)
; CHECK-NEXT: Cost Model: Found costs of RThru:64 CodeSize:4 Lat:4 SizeLat:4 for: %V4i64 = udiv <4 x i64> undef, splat (i64 7)
@@ -810,7 +810,7 @@ define void @sdiv_constnegpow2() {
define void @udiv_constnegpow2() {
; CHECK-LABEL: 'udiv_constnegpow2'
-; CHECK-NEXT: Cost Model: Found costs of RThru:10 CodeSize:4 Lat:4 SizeLat:4 for: %I128 = udiv i128 undef, -16
+; CHECK-NEXT: Cost Model: Found costs of RThru:12 CodeSize:4 Lat:4 SizeLat:4 for: %I128 = udiv i128 undef, -16
; CHECK-NEXT: Cost Model: Found costs of 4 for: %I64 = udiv i64 undef, -16
; CHECK-NEXT: Cost Model: Found costs of RThru:32 CodeSize:4 Lat:4 SizeLat:4 for: %V2i64 = udiv <2 x i64> undef, <i64 -8, i64 -16>
; CHECK-NEXT: Cost Model: Found costs of RThru:64 CodeSize:4 Lat:4 SizeLat:4 for: %V4i64 = udiv <4 x i64> undef, <i64 -2, i64 -4, i64 -8, i64 -16>
@@ -926,7 +926,7 @@ define void @sdiv_uniformconstnegpow2() {
define void @udiv_uniformconstnegpow2() {
; CHECK-LABEL: 'udiv_uniformconstnegpow2'
-; CHECK-NEXT: Cost Model: Found costs of RThru:10 CodeSize:4 Lat:4 SizeLat:4 for: %I128 = udiv i128 undef, -16
+; CHECK-NEXT: Cost Model: Found costs of RThru:12 CodeSize:4 Lat:4 SizeLat:4 for: %I128 = udiv i128 undef, -16
; CHECK-NEXT: Cost Model: Found costs of 4 for: %I64 = udiv i64 undef, -16
; CHECK-NEXT: Cost Model: Found costs of RThru:32 CodeSize:4 Lat:4 SizeLat:4 for: %V2i64 = udiv <2 x i64> undef, splat (i64 -16)
; CHECK-NEXT: Cost Model: Found costs of RThru:64 CodeSize:4 Lat:4 SizeLat:4 for: %V4i64 = udiv <4 x i64> undef, splat (i64 -16)
diff --git a/llvm/test/Analysis/CostModel/AArch64/mul.ll b/llvm/test/Analysis/CostModel/AArch64/mul.ll
index 9b3f1ff898dd3..dc4eca200fe4e 100644
--- a/llvm/test/Analysis/CostModel/AArch64/mul.ll
+++ b/llvm/test/Analysis/CostModel/AArch64/mul.ll
@@ -75,7 +75,7 @@ define void @vi64() {
define void @vi128() {
; CHECK-LABEL: 'vi128'
-; CHECK-NEXT: Cost Model: Found costs of RThru:4 CodeSize:1 Lat:1 SizeLat:1 for: %e2 = mul <2 x i128> undef, undef
+; CHECK-NEXT: Cost Model: Found costs of RThru:8 CodeSize:1 Lat:1 SizeLat:1 for: %e2 = mul <2 x i128> undef, undef
; CHECK-NEXT: Cost Model: Found costs of RThru:0 CodeSize:1 Lat:1 SizeLat:1 for: ret void
;
%e2 = mul <2 x i128> undef, undef
diff --git a/llvm/test/Analysis/CostModel/AArch64/rem.ll b/llvm/test/Analysis/CostModel/AArch64/rem.ll
index 2fa62f1705911..22b9e5cd545af 100644
--- a/llvm/test/Analysis/CostModel/AArch64/rem.ll
+++ b/llvm/test/Analysis/CostModel/AArch64/rem.ll
@@ -345,7 +345,7 @@ define void @srem_const() {
define void @urem_const() {
; CHECK-LABEL: 'urem_const'
-; CHECK-NEXT: Cost Model: Found costs of RThru:14 CodeSize:4 Lat:4 SizeLat:4 for: %I128 = urem i128 undef, 7
+; CHECK-NEXT: Cost Model: Found costs of RThru:18 CodeSize:4 Lat:4 SizeLat:4 for: %I128 = urem i128 undef, 7
; CHECK-NEXT: Cost Model: Found costs of RThru:6 CodeSize:4 Lat:4 SizeLat:4 for: %I64 = urem i64 undef, 7
; CHECK-NEXT: Cost Model: Found costs of RThru:47 CodeSize:4 Lat:4 SizeLat:4 for: %V2i64 = urem <2 x i64> undef, <i64 6, i64 7>
; CHECK-NEXT: Cost Model: Found costs of RThru:94 CodeSize:4 Lat:4 SizeLat:4 for: %V4i64 = urem <4 x i64> undef, <i64 4, i64 5, i64 6, i64 7>
@@ -462,7 +462,7 @@ define void @srem_uniformconst() {
define void @urem_uniformconst() {
; CHECK-LABEL: 'urem_uniformconst'
-; CHECK-NEXT: Cost Model: Found costs of RThru:14 CodeSize:4 Lat:4 SizeLat:4 for: %I128 = urem i128 undef, 7
+; CHECK-NEXT: Cost Model: Found costs of RThru:18 CodeSize:4 Lat:4 SizeLat:4 for: %I128 = urem i128 undef, 7
; CHECK-NEXT: Cost Model: Found costs of RThru:6 CodeSize:4 Lat:4 SizeLat:4 for: %I64 = urem i64 undef, 7
; CHECK-NEXT: Cost Model: Found costs of RThru:47 CodeSize:4 Lat:4 SizeLat:4 for: %V2i64 = urem <2 x i64> undef, splat (i64 7)
; CHECK-NEXT: Cost Model: Found costs of RThru:94 CodeSize:4 Lat:4 SizeLat:4 for: %V4i64 = urem <4 x i64> undef, splat (i64 7)
@@ -810,7 +810,7 @@ define void @srem_constnegpow2() {
define void @urem_constnegpow2() {
; CHECK-LABEL: 'urem_constnegpow2'
-; CHECK-NEXT: Cost Model: Found costs of RThru:14 CodeSize:4 Lat:4 SizeLat:4 for: %I128 = urem i128 undef, -16
+; CHECK-NEXT: Cost Model: Found costs of RThru:18 CodeSize:4 Lat:4 SizeLat:4 for: %I128 = urem i128 undef, -16
; CHECK-NEXT: Cost Model: Found costs of RThru:6 CodeSize:4 Lat:4 SizeLat:4 for: %I64 = urem i64 undef, -16
; CHECK-NEXT: Cost Model: Found costs of RThru:47 CodeSize:4 Lat:4 SizeLat:4 for: %V2i64 = urem <2 x i64> undef, <i64 -8, i64 -16>
; CHECK-NEXT: Cost Model: Found costs of RThru:94 CodeSize:4 Lat:4 SizeLat:4 for: %V4i64 = urem <4 x i64> undef, <i64 -2, i64 -4, i64 -8, i64 -16>
@@ -926,7 +926,7 @@ define void @srem_uniformconstnegpow2() {
define void @urem_uniformconstnegpow2() {
; CHECK-LABEL: 'urem_uniformconstnegpow2'
-; CHECK-NEXT: Cost Model: Found costs of RThru:14 CodeSize:4 Lat:4 SizeLat:4 for: %I128 = urem i128 undef, -16
+; CHECK-NEXT: Cost Model: Found costs of RThru:18 CodeSize:4 Lat:4 SizeLat:4 for: %I128 = urem i128 undef, -16
; CHECK-NEXT: Cost Model: Found costs of RThru:6 CodeSize:4 Lat:4 SizeLat:4 for: %I64 = urem i64 undef, -16
; CHECK-NEXT: Cost Model: Found costs of RThru:47 CodeSize:4 Lat:4 SizeLat:4 for: %V2i64 = urem <2 x i64> undef, splat (i64 -16)
; CHECK-NEXT: Cost Model: Found costs of RThru:94 CodeSize:4 Lat:4 SizeLat:4 for: %V4i64 = urem <4 x i64> undef, splat (i64 -16)
>From 0e7081e6829ad145ef87a8a89e9d8b9943258c80 Mon Sep 17 00:00:00 2001
From: David Green <david.green at arm.com>
Date: Mon, 6 Jul 2026 13:45:28 +0100
Subject: [PATCH 2/2] [AArch64] Increase the relative cost of vector i64
multiply on Neoverse V3ae.
The throughput of vector nxv2i64 multiplies on neoverse v3ae is 1/2, compared
to the throughput of 2 for integer multiplies. This large difference can mean
it is more profitable than normal to use scalar loops as oppposed to
vectorization.
This adds a subtarget feature that increases the cost multiple by 4 for 64bit
vector multiplies for specific CPUs. The cost model of llvm does not mean that
we can model throughputs correctly, but this should help. The same feature is
added to N2 as it has a similar difference between vector and scalar multiply
cost throughputs.
---
llvm/lib/Target/AArch64/AArch64Features.td | 5 +
llvm/lib/Target/AArch64/AArch64Processors.td | 6 +-
.../AArch64/AArch64TargetTransformInfo.cpp | 15 ++-
llvm/test/Analysis/CostModel/AArch64/mul.ll | 115 ++++++++++++------
4 files changed, 102 insertions(+), 39 deletions(-)
diff --git a/llvm/lib/Target/AArch64/AArch64Features.td b/llvm/lib/Target/AArch64/AArch64Features.td
index fc23318c0582d..7cb7cd32f673b 100644
--- a/llvm/lib/Target/AArch64/AArch64Features.td
+++ b/llvm/lib/Target/AArch64/AArch64Features.td
@@ -1010,6 +1010,11 @@ def FeatureFixedLoadLatency4 : SubtargetFeature<
"fixed-load-latency-4", "FixedLoadLatency", "4",
"Use a fixed latency of 4 for loads instead of querying the scheduling model">;
+def FeatureLimited64bitVectorMulBandwidth : SubtargetFeature<
+ "has-limited-64bit-vector-mul-bandwidth", "HasLimited64bitVectorMulBandwidth", "true",
+ "Has limited 64bit vector multiply bandwidth compared to scalar multiply",
+ [], InlineIgnore>;
+
//===----------------------------------------------------------------------===//
// Architectures.
//
diff --git a/llvm/lib/Target/AArch64/AArch64Processors.td b/llvm/lib/Target/AArch64/AArch64Processors.td
index 1c8d5ec34e71e..a8e7e897265e4 100644
--- a/llvm/lib/Target/AArch64/AArch64Processors.td
+++ b/llvm/lib/Target/AArch64/AArch64Processors.td
@@ -535,7 +535,8 @@ def TuneNeoverseN2 : SubtargetFeature<"neoversen2", "ARMProcFamily", "NeoverseN2
FeaturePostRAScheduler,
FeatureEnableSelectOptimize,
FeatureUseFixedOverScalableIfEqualCost,
- FeaturePredictableSelectIsExpensive]>;
+ FeaturePredictableSelectIsExpensive,
+ FeatureLimited64bitVectorMulBandwidth]>;
def TuneNeoverseN3 : SubtargetFeature<"neoversen3", "ARMProcFamily", "NeoverseN3",
"Neoverse N3 ARM processors", [
@@ -613,7 +614,8 @@ def TuneNeoverseV3AE : SubtargetFeature<"neoversev3AE", "ARMProcFamily", "Neover
FeatureEnableSelectOptimize,
FeatureUseFixedOverScalableIfEqualCost,
FeatureAvoidLDAPUR,
- FeaturePredictableSelectIsExpensive]>;
+ FeaturePredictableSelectIsExpensive,
+ FeatureLimited64bitVectorMulBandwidth]>;
def TuneSaphira : SubtargetFeature<"saphira", "ARMProcFamily", "Saphira",
"Qualcomm Saphira processors", [
diff --git a/llvm/lib/Target/AArch64/AArch64TargetTransformInfo.cpp b/llvm/lib/Target/AArch64/AArch64TargetTransformInfo.cpp
index dfd0895b5c748..e59b6e0d4485b 100644
--- a/llvm/lib/Target/AArch64/AArch64TargetTransformInfo.cpp
+++ b/llvm/lib/Target/AArch64/AArch64TargetTransformInfo.cpp
@@ -4534,23 +4534,27 @@ InstructionCost AArch64TTIImpl::getArithmeticInstrCost(
case ISD::ADD:
case ISD::SUB:
return LT.first; // Also works for i128
- case ISD::MUL:
+ case ISD::MUL: {
// i128 multiply is umulh + 2*madd + mul and grows ~O(Bitwidth^2). For
// scalable vectors the cost of LT.first will be invalid, leading to an
// invalid cost overall.
+ unsigned Mul64CostFactor = (CostKind == TTI::TCK_RecipThroughput &&
+ ST->hasLimited64bitVectorMulBandwidth())
+ ? 4
+ : 1;
if (Ty->getScalarSizeInBits() > 64) {
unsigned NumLanes = isa<FixedVectorType>(Ty)
? cast<FixedVectorType>(Ty)->getNumElements()
: 1;
InstructionCost CostPerLane = LT.first / NumLanes;
- return CostPerLane * CostPerLane * NumLanes;
+ return CostPerLane * CostPerLane * NumLanes * Mul64CostFactor;
}
if (LT.second == MVT::v2i64) {
// When SVE is available, then we can lower the v2i64 operation using
// the SVE mul instruction, which has a lower cost.
if (ST->hasSVE())
- return LT.first;
+ return LT.first * Mul64CostFactor;
// When SVE is not available, there is no MUL.2d instruction,
// which means mul <2 x i64> is expensive as elements are extracted
@@ -4570,7 +4574,12 @@ InstructionCost AArch64TTIImpl::getArithmeticInstrCost(
getVectorInstrCost(Instruction::InsertElement, Ty, CostKind, -1,
nullptr, nullptr));
}
+
+ if (LT.second == MVT::nxv2i64)
+ return LT.first * Mul64CostFactor;
+
return LT.first;
+ }
case ISD::SREM:
case ISD::SDIV:
/*
diff --git a/llvm/test/Analysis/CostModel/AArch64/mul.ll b/llvm/test/Analysis/CostModel/AArch64/mul.ll
index dc4eca200fe4e..713c072fae927 100644
--- a/llvm/test/Analysis/CostModel/AArch64/mul.ll
+++ b/llvm/test/Analysis/CostModel/AArch64/mul.ll
@@ -1,6 +1,8 @@
; NOTE: Assertions have been autogenerated by utils/update_analyze_test_checks.py
; RUN: opt -passes="print<cost-model>" 2>&1 -disable-output -cost-kind=all -mtriple=aarch64 < %s | FileCheck %s --check-prefixes=CHECK,CHECK-NOSVE
-; RUN: opt -passes="print<cost-model>" 2>&1 -disable-output -cost-kind=all -mtriple=aarch64 -mattr=+sve2 < %s | FileCheck %s --check-prefixes=CHECK,CHECK-SVE
+; RUN: opt -passes="print<cost-model>" 2>&1 -disable-output -cost-kind=all -mtriple=aarch64 -mattr=+sve2 < %s | FileCheck %s --check-prefixes=CHECK,CHECK-SVE,CHECK-STD
+; RUN: opt -passes="print<cost-model>" 2>&1 -disable-output -cost-kind=all -mtriple=aarch64 -mcpu=neoverse-n2 < %s | FileCheck %s --check-prefixes=CHECK,CHECK-SVE,CHECK-HIGH
+; RUN: opt -passes="print<cost-model>" 2>&1 -disable-output -cost-kind=all -mtriple=aarch64 -mcpu=neoverse-v3ae < %s | FileCheck %s --check-prefixes=CHECK,CHECK-SVE,CHECK-HIGH
target datalayout = "e-m:e-i8:8:32-i16:16:32-i64:64-i128:128-n32:64-S128"
@@ -59,12 +61,19 @@ define void @vi64() {
; CHECK-NOSVE-NEXT: Cost Model: Found costs of RThru:112 CodeSize:1 Lat:1 SizeLat:1 for: %e16 = mul <16 x i64> undef, undef
; CHECK-NOSVE-NEXT: Cost Model: Found costs of RThru:0 CodeSize:1 Lat:1 SizeLat:1 for: ret void
;
-; CHECK-SVE-LABEL: 'vi64'
-; CHECK-SVE-NEXT: Cost Model: Found costs of 1 for: %e2 = mul <2 x i64> undef, undef
-; CHECK-SVE-NEXT: Cost Model: Found costs of RThru:2 CodeSize:1 Lat:1 SizeLat:1 for: %e4 = mul <4 x i64> undef, undef
-; CHECK-SVE-NEXT: Cost Model: Found costs of RThru:4 CodeSize:1 Lat:1 SizeLat:1 for: %e8 = mul <8 x i64> undef, undef
-; CHECK-SVE-NEXT: Cost Model: Found costs of RThru:8 CodeSize:1 Lat:1 SizeLat:1 for: %e16 = mul <16 x i64> undef, undef
-; CHECK-SVE-NEXT: Cost Model: Found costs of RThru:0 CodeSize:1 Lat:1 SizeLat:1 for: ret void
+; CHECK-STD-LABEL: 'vi64'
+; CHECK-STD-NEXT: Cost Model: Found costs of 1 for: %e2 = mul <2 x i64> undef, undef
+; CHECK-STD-NEXT: Cost Model: Found costs of RThru:2 CodeSize:1 Lat:1 SizeLat:1 for: %e4 = mul <4 x i64> undef, undef
+; CHECK-STD-NEXT: Cost Model: Found costs of RThru:4 CodeSize:1 Lat:1 SizeLat:1 for: %e8 = mul <8 x i64> undef, undef
+; CHECK-STD-NEXT: Cost Model: Found costs of RThru:8 CodeSize:1 Lat:1 SizeLat:1 for: %e16 = mul <16 x i64> undef, undef
+; CHECK-STD-NEXT: Cost Model: Found costs of RThru:0 CodeSize:1 Lat:1 SizeLat:1 for: ret void
+;
+; CHECK-HIGH-LABEL: 'vi64'
+; CHECK-HIGH-NEXT: Cost Model: Found costs of RThru:4 CodeSize:1 Lat:1 SizeLat:1 for: %e2 = mul <2 x i64> undef, undef
+; CHECK-HIGH-NEXT: Cost Model: Found costs of RThru:8 CodeSize:1 Lat:1 SizeLat:1 for: %e4 = mul <4 x i64> undef, undef
+; CHECK-HIGH-NEXT: Cost Model: Found costs of RThru:16 CodeSize:1 Lat:1 SizeLat:1 for: %e8 = mul <8 x i64> undef, undef
+; CHECK-HIGH-NEXT: Cost Model: Found costs of RThru:32 CodeSize:1 Lat:1 SizeLat:1 for: %e16 = mul <16 x i64> undef, undef
+; CHECK-HIGH-NEXT: Cost Model: Found costs of RThru:0 CodeSize:1 Lat:1 SizeLat:1 for: ret void
;
%e2 = mul <2 x i64> undef, undef
%e4 = mul <4 x i64> undef, undef
@@ -74,9 +83,17 @@ define void @vi64() {
}
define void @vi128() {
-; CHECK-LABEL: 'vi128'
-; CHECK-NEXT: Cost Model: Found costs of RThru:8 CodeSize:1 Lat:1 SizeLat:1 for: %e2 = mul <2 x i128> undef, undef
-; CHECK-NEXT: Cost Model: Found costs of RThru:0 CodeSize:1 Lat:1 SizeLat:1 for: ret void
+; CHECK-NOSVE-LABEL: 'vi128'
+; CHECK-NOSVE-NEXT: Cost Model: Found costs of RThru:8 CodeSize:1 Lat:1 SizeLat:1 for: %e2 = mul <2 x i128> undef, undef
+; CHECK-NOSVE-NEXT: Cost Model: Found costs of RThru:0 CodeSize:1 Lat:1 SizeLat:1 for: ret void
+;
+; CHECK-STD-LABEL: 'vi128'
+; CHECK-STD-NEXT: Cost Model: Found costs of RThru:8 CodeSize:1 Lat:1 SizeLat:1 for: %e2 = mul <2 x i128> undef, undef
+; CHECK-STD-NEXT: Cost Model: Found costs of RThru:0 CodeSize:1 Lat:1 SizeLat:1 for: ret void
+;
+; CHECK-HIGH-LABEL: 'vi128'
+; CHECK-HIGH-NEXT: Cost Model: Found costs of RThru:32 CodeSize:1 Lat:1 SizeLat:1 for: %e2 = mul <2 x i128> undef, undef
+; CHECK-HIGH-NEXT: Cost Model: Found costs of RThru:0 CodeSize:1 Lat:1 SizeLat:1 for: ret void
;
%e2 = mul <2 x i128> undef, undef
ret void
@@ -90,12 +107,19 @@ define void @xvi8() {
; CHECK-NOSVE-NEXT: Cost Model: Found costs of RThru:Invalid CodeSize:1 Lat:1 SizeLat:1 for: %e16 = mul <vscale x 16 x i8> undef, undef
; CHECK-NOSVE-NEXT: Cost Model: Found costs of RThru:0 CodeSize:1 Lat:1 SizeLat:1 for: ret void
;
-; CHECK-SVE-LABEL: 'xvi8'
-; CHECK-SVE-NEXT: Cost Model: Found costs of 1 for: %e2 = mul <vscale x 2 x i8> undef, undef
-; CHECK-SVE-NEXT: Cost Model: Found costs of 1 for: %e4 = mul <vscale x 4 x i8> undef, undef
-; CHECK-SVE-NEXT: Cost Model: Found costs of 1 for: %e8 = mul <vscale x 8 x i8> undef, undef
-; CHECK-SVE-NEXT: Cost Model: Found costs of 1 for: %e16 = mul <vscale x 16 x i8> undef, undef
-; CHECK-SVE-NEXT: Cost Model: Found costs of RThru:0 CodeSize:1 Lat:1 SizeLat:1 for: ret void
+; CHECK-STD-LABEL: 'xvi8'
+; CHECK-STD-NEXT: Cost Model: Found costs of 1 for: %e2 = mul <vscale x 2 x i8> undef, undef
+; CHECK-STD-NEXT: Cost Model: Found costs of 1 for: %e4 = mul <vscale x 4 x i8> undef, undef
+; CHECK-STD-NEXT: Cost Model: Found costs of 1 for: %e8 = mul <vscale x 8 x i8> undef, undef
+; CHECK-STD-NEXT: Cost Model: Found costs of 1 for: %e16 = mul <vscale x 16 x i8> undef, undef
+; CHECK-STD-NEXT: Cost Model: Found costs of RThru:0 CodeSize:1 Lat:1 SizeLat:1 for: ret void
+;
+; CHECK-HIGH-LABEL: 'xvi8'
+; CHECK-HIGH-NEXT: Cost Model: Found costs of RThru:4 CodeSize:1 Lat:1 SizeLat:1 for: %e2 = mul <vscale x 2 x i8> undef, undef
+; CHECK-HIGH-NEXT: Cost Model: Found costs of 1 for: %e4 = mul <vscale x 4 x i8> undef, undef
+; CHECK-HIGH-NEXT: Cost Model: Found costs of 1 for: %e8 = mul <vscale x 8 x i8> undef, undef
+; CHECK-HIGH-NEXT: Cost Model: Found costs of 1 for: %e16 = mul <vscale x 16 x i8> undef, undef
+; CHECK-HIGH-NEXT: Cost Model: Found costs of RThru:0 CodeSize:1 Lat:1 SizeLat:1 for: ret void
;
%e2 = mul <vscale x 2 x i8> undef, undef
%e4 = mul <vscale x 4 x i8> undef, undef
@@ -112,12 +136,19 @@ define void @nxvi16() {
; CHECK-NOSVE-NEXT: Cost Model: Found costs of RThru:Invalid CodeSize:1 Lat:1 SizeLat:1 for: %e16 = mul <vscale x 16 x i16> undef, undef
; CHECK-NOSVE-NEXT: Cost Model: Found costs of RThru:0 CodeSize:1 Lat:1 SizeLat:1 for: ret void
;
-; CHECK-SVE-LABEL: 'nxvi16'
-; CHECK-SVE-NEXT: Cost Model: Found costs of 1 for: %e2 = mul <vscale x 2 x i16> undef, undef
-; CHECK-SVE-NEXT: Cost Model: Found costs of 1 for: %e4 = mul <vscale x 4 x i16> undef, undef
-; CHECK-SVE-NEXT: Cost Model: Found costs of 1 for: %e8 = mul <vscale x 8 x i16> undef, undef
-; CHECK-SVE-NEXT: Cost Model: Found costs of RThru:2 CodeSize:1 Lat:1 SizeLat:1 for: %e16 = mul <vscale x 16 x i16> undef, undef
-; CHECK-SVE-NEXT: Cost Model: Found costs of RThru:0 CodeSize:1 Lat:1 SizeLat:1 for: ret void
+; CHECK-STD-LABEL: 'nxvi16'
+; CHECK-STD-NEXT: Cost Model: Found costs of 1 for: %e2 = mul <vscale x 2 x i16> undef, undef
+; CHECK-STD-NEXT: Cost Model: Found costs of 1 for: %e4 = mul <vscale x 4 x i16> undef, undef
+; CHECK-STD-NEXT: Cost Model: Found costs of 1 for: %e8 = mul <vscale x 8 x i16> undef, undef
+; CHECK-STD-NEXT: Cost Model: Found costs of RThru:2 CodeSize:1 Lat:1 SizeLat:1 for: %e16 = mul <vscale x 16 x i16> undef, undef
+; CHECK-STD-NEXT: Cost Model: Found costs of RThru:0 CodeSize:1 Lat:1 SizeLat:1 for: ret void
+;
+; CHECK-HIGH-LABEL: 'nxvi16'
+; CHECK-HIGH-NEXT: Cost Model: Found costs of RThru:4 CodeSize:1 Lat:1 SizeLat:1 for: %e2 = mul <vscale x 2 x i16> undef, undef
+; CHECK-HIGH-NEXT: Cost Model: Found costs of 1 for: %e4 = mul <vscale x 4 x i16> undef, undef
+; CHECK-HIGH-NEXT: Cost Model: Found costs of 1 for: %e8 = mul <vscale x 8 x i16> undef, undef
+; CHECK-HIGH-NEXT: Cost Model: Found costs of RThru:2 CodeSize:1 Lat:1 SizeLat:1 for: %e16 = mul <vscale x 16 x i16> undef, undef
+; CHECK-HIGH-NEXT: Cost Model: Found costs of RThru:0 CodeSize:1 Lat:1 SizeLat:1 for: ret void
;
%e2 = mul <vscale x 2 x i16> undef, undef
%e4 = mul <vscale x 4 x i16> undef, undef
@@ -134,12 +165,19 @@ define void @nxvi32() {
; CHECK-NOSVE-NEXT: Cost Model: Found costs of RThru:Invalid CodeSize:1 Lat:1 SizeLat:1 for: %e16 = mul <vscale x 16 x i32> undef, undef
; CHECK-NOSVE-NEXT: Cost Model: Found costs of RThru:0 CodeSize:1 Lat:1 SizeLat:1 for: ret void
;
-; CHECK-SVE-LABEL: 'nxvi32'
-; CHECK-SVE-NEXT: Cost Model: Found costs of 1 for: %e2 = mul <vscale x 2 x i32> undef, undef
-; CHECK-SVE-NEXT: Cost Model: Found costs of 1 for: %e4 = mul <vscale x 4 x i32> undef, undef
-; CHECK-SVE-NEXT: Cost Model: Found costs of RThru:2 CodeSize:1 Lat:1 SizeLat:1 for: %e8 = mul <vscale x 8 x i32> undef, undef
-; CHECK-SVE-NEXT: Cost Model: Found costs of RThru:4 CodeSize:1 Lat:1 SizeLat:1 for: %e16 = mul <vscale x 16 x i32> undef, undef
-; CHECK-SVE-NEXT: Cost Model: Found costs of RThru:0 CodeSize:1 Lat:1 SizeLat:1 for: ret void
+; CHECK-STD-LABEL: 'nxvi32'
+; CHECK-STD-NEXT: Cost Model: Found costs of 1 for: %e2 = mul <vscale x 2 x i32> undef, undef
+; CHECK-STD-NEXT: Cost Model: Found costs of 1 for: %e4 = mul <vscale x 4 x i32> undef, undef
+; CHECK-STD-NEXT: Cost Model: Found costs of RThru:2 CodeSize:1 Lat:1 SizeLat:1 for: %e8 = mul <vscale x 8 x i32> undef, undef
+; CHECK-STD-NEXT: Cost Model: Found costs of RThru:4 CodeSize:1 Lat:1 SizeLat:1 for: %e16 = mul <vscale x 16 x i32> undef, undef
+; CHECK-STD-NEXT: Cost Model: Found costs of RThru:0 CodeSize:1 Lat:1 SizeLat:1 for: ret void
+;
+; CHECK-HIGH-LABEL: 'nxvi32'
+; CHECK-HIGH-NEXT: Cost Model: Found costs of RThru:4 CodeSize:1 Lat:1 SizeLat:1 for: %e2 = mul <vscale x 2 x i32> undef, undef
+; CHECK-HIGH-NEXT: Cost Model: Found costs of 1 for: %e4 = mul <vscale x 4 x i32> undef, undef
+; CHECK-HIGH-NEXT: Cost Model: Found costs of RThru:2 CodeSize:1 Lat:1 SizeLat:1 for: %e8 = mul <vscale x 8 x i32> undef, undef
+; CHECK-HIGH-NEXT: Cost Model: Found costs of RThru:4 CodeSize:1 Lat:1 SizeLat:1 for: %e16 = mul <vscale x 16 x i32> undef, undef
+; CHECK-HIGH-NEXT: Cost Model: Found costs of RThru:0 CodeSize:1 Lat:1 SizeLat:1 for: ret void
;
%e2 = mul <vscale x 2 x i32> undef, undef
%e4 = mul <vscale x 4 x i32> undef, undef
@@ -156,12 +194,19 @@ define void @nxvi64() {
; CHECK-NOSVE-NEXT: Cost Model: Found costs of RThru:Invalid CodeSize:1 Lat:1 SizeLat:1 for: %e16 = mul <vscale x 16 x i64> undef, undef
; CHECK-NOSVE-NEXT: Cost Model: Found costs of RThru:0 CodeSize:1 Lat:1 SizeLat:1 for: ret void
;
-; CHECK-SVE-LABEL: 'nxvi64'
-; CHECK-SVE-NEXT: Cost Model: Found costs of 1 for: %e2 = mul <vscale x 2 x i64> undef, undef
-; CHECK-SVE-NEXT: Cost Model: Found costs of RThru:2 CodeSize:1 Lat:1 SizeLat:1 for: %e4 = mul <vscale x 4 x i64> undef, undef
-; CHECK-SVE-NEXT: Cost Model: Found costs of RThru:4 CodeSize:1 Lat:1 SizeLat:1 for: %e8 = mul <vscale x 8 x i64> undef, undef
-; CHECK-SVE-NEXT: Cost Model: Found costs of RThru:8 CodeSize:1 Lat:1 SizeLat:1 for: %e16 = mul <vscale x 16 x i64> undef, undef
-; CHECK-SVE-NEXT: Cost Model: Found costs of RThru:0 CodeSize:1 Lat:1 SizeLat:1 for: ret void
+; CHECK-STD-LABEL: 'nxvi64'
+; CHECK-STD-NEXT: Cost Model: Found costs of 1 for: %e2 = mul <vscale x 2 x i64> undef, undef
+; CHECK-STD-NEXT: Cost Model: Found costs of RThru:2 CodeSize:1 Lat:1 SizeLat:1 for: %e4 = mul <vscale x 4 x i64> undef, undef
+; CHECK-STD-NEXT: Cost Model: Found costs of RThru:4 CodeSize:1 Lat:1 SizeLat:1 for: %e8 = mul <vscale x 8 x i64> undef, undef
+; CHECK-STD-NEXT: Cost Model: Found costs of RThru:8 CodeSize:1 Lat:1 SizeLat:1 for: %e16 = mul <vscale x 16 x i64> undef, undef
+; CHECK-STD-NEXT: Cost Model: Found costs of RThru:0 CodeSize:1 Lat:1 SizeLat:1 for: ret void
+;
+; CHECK-HIGH-LABEL: 'nxvi64'
+; CHECK-HIGH-NEXT: Cost Model: Found costs of RThru:4 CodeSize:1 Lat:1 SizeLat:1 for: %e2 = mul <vscale x 2 x i64> undef, undef
+; CHECK-HIGH-NEXT: Cost Model: Found costs of RThru:8 CodeSize:1 Lat:1 SizeLat:1 for: %e4 = mul <vscale x 4 x i64> undef, undef
+; CHECK-HIGH-NEXT: Cost Model: Found costs of RThru:16 CodeSize:1 Lat:1 SizeLat:1 for: %e8 = mul <vscale x 8 x i64> undef, undef
+; CHECK-HIGH-NEXT: Cost Model: Found costs of RThru:32 CodeSize:1 Lat:1 SizeLat:1 for: %e16 = mul <vscale x 16 x i64> undef, undef
+; CHECK-HIGH-NEXT: Cost Model: Found costs of RThru:0 CodeSize:1 Lat:1 SizeLat:1 for: ret void
;
%e2 = mul <vscale x 2 x i64> undef, undef
%e4 = mul <vscale x 4 x i64> undef, undef
@@ -178,3 +223,5 @@ define void @nxvi128() {
%e2 = mul <vscale x 2 x i128> undef, undef
ret void
}
+;; NOTE: These prefixes are unused and the list is autogenerated. Do not add tests below this line:
+; CHECK-SVE: {{.*}}
More information about the llvm-commits
mailing list