[llvm] [AArch64] Reflect cost of integer sub-reductions. (PR #194594)
Sander de Smalen via llvm-commits
llvm-commits at lists.llvm.org
Tue May 5 07:21:00 PDT 2026
https://github.com/sdesmalen-arm updated https://github.com/llvm/llvm-project/pull/194594
>From 15eee3cb1282dd8732f5cad5d5262aeefcfe20e1 Mon Sep 17 00:00:00 2001
From: Sander de Smalen <sander.desmalen at arm.com>
Date: Thu, 23 Apr 2026 15:57:46 +0000
Subject: [PATCH 1/2] [AArch64] Reflect cost of integer sub-reductions.
The cost of sub-reductions is either the cost of *mlslb + *mlslt,
or the cost of a dot operation with 2 negations:
```
partial_reduce_umls acc, lhs, rhs
<=> -partial_reduce_umla -acc, lhs, rhs
```
(codegen for this was added by #186809)
The cost-model was previously a bit of a hack, since sub-reductions
were expanded and therefore expensive, although we made the expansion
cost artifically cheaper so that it would still be a candidate for
cdot instructions.
---
.../AArch64/AArch64TargetTransformInfo.cpp | 36 +++++++++----------
.../AArch64/partial-reduce-chained.ll | 14 ++++----
.../AArch64/partial-reduce-sub-sdot.ll | 4 +--
3 files changed, 26 insertions(+), 28 deletions(-)
diff --git a/llvm/lib/Target/AArch64/AArch64TargetTransformInfo.cpp b/llvm/lib/Target/AArch64/AArch64TargetTransformInfo.cpp
index aff89e00523c0..755321c65881c 100644
--- a/llvm/lib/Target/AArch64/AArch64TargetTransformInfo.cpp
+++ b/llvm/lib/Target/AArch64/AArch64TargetTransformInfo.cpp
@@ -6047,26 +6047,27 @@ InstructionCost AArch64TTIImpl::getPartialReductionCost(
bool IsSub = Opcode == Instruction::Sub;
InstructionCost Cost = InputLT.first * TTI::TCC_Basic;
+ InstructionCost INegCost = IsSub ? 2 * InputLT.first * TTI::TCC_Basic : 0;
if (AccumLT.second.getScalarType() == MVT::i32 &&
- InputLT.second.getScalarType() == MVT::i8 && !IsSub) {
+ InputLT.second.getScalarType() == MVT::i8) {
// i8 -> i32 is natively supported with udot/sdot for both NEON and SVE.
if (!IsUSDot && IsSupported(true, ST->hasDotProd()))
- return Cost;
+ return Cost + INegCost;
// i8 -> i32 usdot requires +i8mm
if (IsUSDot && IsSupported(ST->hasMatMulInt8(), ST->hasMatMulInt8()))
- return Cost;
+ return Cost + INegCost;
}
- if (ST->isSVEorStreamingSVEAvailable() && !IsUSDot && !IsSub) {
+ if (ST->isSVEorStreamingSVEAvailable() && !IsUSDot) {
// i16 -> i64 is natively supported for udot/sdot
if (AccumLT.second.getScalarType() == MVT::i64 &&
InputLT.second.getScalarType() == MVT::i16)
- return Cost;
+ return Cost + INegCost;
// i16 -> i32 is natively supported with SVE2p1
if (AccumLT.second.getScalarType() == MVT::i32 &&
InputLT.second.getScalarType() == MVT::i16 &&
- (ST->hasSVE2p1() || ST->hasSME2()))
+ (ST->hasSVE2p1() || ST->hasSME2()) && !IsSub)
return Cost;
// i8 -> i64 is supported with an extra level of extends
if (AccumLT.second.getScalarType() == MVT::i64 &&
@@ -6076,12 +6077,12 @@ InstructionCost AArch64TTIImpl::getPartialReductionCost(
// that now, a regular reduction would be cheaper because the costs of
// the extends in the IR are still counted. This can be fixed
// after https://github.com/llvm/llvm-project/pull/147302 has landed.
- return Cost;
+ return Cost + INegCost;
// i8 -> i16 is natively supported with SVE2p3
if (AccumLT.second.getScalarType() == MVT::i16 &&
InputLT.second.getScalarType() == MVT::i8 &&
- (ST->hasSVE2p3() || ST->hasSME2p3()))
- return Cost;
+ (ST->hasSVE2p3() || ST->hasSME2p3()) && !IsSub)
+ return Cost + INegCost;
}
// f16 -> f32 is natively supported for fdot using either
@@ -6092,11 +6093,11 @@ InstructionCost AArch64TTIImpl::getPartialReductionCost(
InputLT.second.getScalarType() == MVT::f16)
return Cost;
- // For a ratio of 2, we can use *mlal top/bottom instructions.
- if (Ratio == 2 && !IsSub) {
+ // For a ratio of 2, we can use *mlal and *mlsl top/bottom instructions.
+ if (Ratio == 2) {
MVT InVT = InputLT.second.getScalarType();
- // SVE2 [us]mlalb/t and NEON [us]mlal(2)
+ // SVE2 [us]ml[as]lb/t and NEON [us]ml[as]l(2)
if (IsSupported(ST->hasSVE2(), true) &&
llvm::is_contained({MVT::i8, MVT::i16, MVT::i32}, InVT.SimpleTy))
return Cost * 2;
@@ -6110,14 +6111,9 @@ InstructionCost AArch64TTIImpl::getPartialReductionCost(
return Cost * 2;
}
- InstructionCost ExpandCost = BaseT::getPartialReductionCost(
- Opcode, InputTypeA, InputTypeB, AccumType, VF, OpAExtend, OpBExtend,
- BinOp, CostKind, FMF);
-
- // Slightly lower the cost of a sub reduction so that it can be considered
- // as candidate for 'cdot' operations. This is a somewhat arbitrary number,
- // because we don't yet model these operations directly.
- return ExpandCost.isValid() && IsSub ? ((8 * ExpandCost) / 10) : ExpandCost;
+ return BaseT::getPartialReductionCost(Opcode, InputTypeA, InputTypeB,
+ AccumType, VF, OpAExtend, OpBExtend,
+ BinOp, CostKind, FMF);
}
InstructionCost
diff --git a/llvm/test/Transforms/LoopVectorize/AArch64/partial-reduce-chained.ll b/llvm/test/Transforms/LoopVectorize/AArch64/partial-reduce-chained.ll
index e54d1e9ddb1a3..b054d34fe597a 100644
--- a/llvm/test/Transforms/LoopVectorize/AArch64/partial-reduce-chained.ll
+++ b/llvm/test/Transforms/LoopVectorize/AArch64/partial-reduce-chained.ll
@@ -892,7 +892,7 @@ define i32 @chained_partial_reduce_sub_add_sub(ptr %a, ptr %b, ptr %c, i32 %N) #
; CHECK-SVE-MAXBW-NEXT: br label [[VECTOR_BODY:%.*]]
; CHECK-SVE-MAXBW: vector.body:
; CHECK-SVE-MAXBW-NEXT: [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ]
-; CHECK-SVE-MAXBW-NEXT: [[VEC_PHI:%.*]] = phi <vscale x 8 x i32> [ zeroinitializer, [[VECTOR_PH]] ], [ [[TMP15:%.*]], [[VECTOR_BODY]] ]
+; CHECK-SVE-MAXBW-NEXT: [[VEC_PHI:%.*]] = phi <vscale x 2 x i32> [ zeroinitializer, [[VECTOR_PH]] ], [ [[PARTIAL_REDUCE4:%.*]], [[VECTOR_BODY]] ]
; CHECK-SVE-MAXBW-NEXT: [[TMP7:%.*]] = getelementptr inbounds nuw i8, ptr [[A]], i64 [[INDEX]]
; CHECK-SVE-MAXBW-NEXT: [[TMP8:%.*]] = getelementptr inbounds nuw i8, ptr [[B]], i64 [[INDEX]]
; CHECK-SVE-MAXBW-NEXT: [[TMP9:%.*]] = getelementptr inbounds nuw i8, ptr [[C]], i64 [[INDEX]]
@@ -901,18 +901,20 @@ define i32 @chained_partial_reduce_sub_add_sub(ptr %a, ptr %b, ptr %c, i32 %N) #
; CHECK-SVE-MAXBW-NEXT: [[WIDE_LOAD2:%.*]] = load <vscale x 8 x i8>, ptr [[TMP9]], align 1
; CHECK-SVE-MAXBW-NEXT: [[TMP13:%.*]] = sext <vscale x 8 x i8> [[WIDE_LOAD]] to <vscale x 8 x i32>
; CHECK-SVE-MAXBW-NEXT: [[TMP14:%.*]] = sext <vscale x 8 x i8> [[WIDE_LOAD1]] to <vscale x 8 x i32>
-; CHECK-SVE-MAXBW-NEXT: [[TMP12:%.*]] = sext <vscale x 8 x i8> [[WIDE_LOAD2]] to <vscale x 8 x i32>
; CHECK-SVE-MAXBW-NEXT: [[TMP10:%.*]] = mul nsw <vscale x 8 x i32> [[TMP13]], [[TMP14]]
-; CHECK-SVE-MAXBW-NEXT: [[TMP11:%.*]] = sub <vscale x 8 x i32> [[VEC_PHI]], [[TMP10]]
+; CHECK-SVE-MAXBW-NEXT: [[TMP11:%.*]] = sub <vscale x 8 x i32> zeroinitializer, [[TMP10]]
+; CHECK-SVE-MAXBW-NEXT: [[PARTIAL_REDUCE:%.*]] = call <vscale x 2 x i32> @llvm.vector.partial.reduce.add.nxv2i32.nxv8i32(<vscale x 2 x i32> [[VEC_PHI]], <vscale x 8 x i32> [[TMP11]])
+; CHECK-SVE-MAXBW-NEXT: [[TMP12:%.*]] = sext <vscale x 8 x i8> [[WIDE_LOAD2]] to <vscale x 8 x i32>
; CHECK-SVE-MAXBW-NEXT: [[TMP18:%.*]] = mul nsw <vscale x 8 x i32> [[TMP13]], [[TMP12]]
-; CHECK-SVE-MAXBW-NEXT: [[TMP16:%.*]] = add <vscale x 8 x i32> [[TMP11]], [[TMP18]]
+; CHECK-SVE-MAXBW-NEXT: [[PARTIAL_REDUCE3:%.*]] = call <vscale x 2 x i32> @llvm.vector.partial.reduce.add.nxv2i32.nxv8i32(<vscale x 2 x i32> [[PARTIAL_REDUCE]], <vscale x 8 x i32> [[TMP18]])
; CHECK-SVE-MAXBW-NEXT: [[TMP19:%.*]] = mul nsw <vscale x 8 x i32> [[TMP14]], [[TMP12]]
-; CHECK-SVE-MAXBW-NEXT: [[TMP15]] = sub <vscale x 8 x i32> [[TMP16]], [[TMP19]]
+; CHECK-SVE-MAXBW-NEXT: [[TMP15:%.*]] = sub <vscale x 8 x i32> zeroinitializer, [[TMP19]]
+; CHECK-SVE-MAXBW-NEXT: [[PARTIAL_REDUCE4]] = call <vscale x 2 x i32> @llvm.vector.partial.reduce.add.nxv2i32.nxv8i32(<vscale x 2 x i32> [[PARTIAL_REDUCE3]], <vscale x 8 x i32> [[TMP15]])
; CHECK-SVE-MAXBW-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], [[TMP3]]
; CHECK-SVE-MAXBW-NEXT: [[TMP22:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
; CHECK-SVE-MAXBW-NEXT: br i1 [[TMP22]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP12:![0-9]+]]
; CHECK-SVE-MAXBW: middle.block:
-; CHECK-SVE-MAXBW-NEXT: [[TMP17:%.*]] = call i32 @llvm.vector.reduce.add.nxv8i32(<vscale x 8 x i32> [[TMP15]])
+; CHECK-SVE-MAXBW-NEXT: [[TMP16:%.*]] = call i32 @llvm.vector.reduce.add.nxv2i32(<vscale x 2 x i32> [[PARTIAL_REDUCE4]])
; CHECK-SVE-MAXBW-NEXT: [[CMP_N:%.*]] = icmp eq i64 [[WIDE_TRIP_COUNT]], [[N_VEC]]
; CHECK-SVE-MAXBW-NEXT: br i1 [[CMP_N]], label [[FOR_COND_CLEANUP:%.*]], label [[SCALAR_PH]]
; CHECK-SVE-MAXBW: scalar.ph:
diff --git a/llvm/test/Transforms/LoopVectorize/AArch64/partial-reduce-sub-sdot.ll b/llvm/test/Transforms/LoopVectorize/AArch64/partial-reduce-sub-sdot.ll
index ff3881f2c7fda..d40b984618d45 100644
--- a/llvm/test/Transforms/LoopVectorize/AArch64/partial-reduce-sub-sdot.ll
+++ b/llvm/test/Transforms/LoopVectorize/AArch64/partial-reduce-sub-sdot.ll
@@ -15,9 +15,9 @@
; COMMON: LV: Checking a loop in 'add_sub_chained_reduction'
; SVE: Cost of 1 for VF vscale x 16: EXPRESSION vp<{{.*}}> = ir<%acc> + partial.reduce.add (mul (ir<%load1> sext to i32), (ir<%load2> sext to i32))
-; SVE: Cost of 16 for VF vscale x 16: EXPRESSION vp<{{.*}}> = vp<%9> + partial.reduce.add (sub (0, mul (ir<%load2> sext to i32), (ir<%load3> sext to i32)))
+; SVE: Cost of 3 for VF vscale x 16: EXPRESSION vp<{{.*}}> = vp<%9> + partial.reduce.add (sub (0, mul (ir<%load2> sext to i32), (ir<%load3> sext to i32)))
; NEON: Cost of 1 for VF 16: EXPRESSION vp<{{.*}}> = ir<%acc> + partial.reduce.add (mul (ir<%load1> sext to i32), (ir<%load2> sext to i32))
-; NEON: Cost of 16 for VF 16: EXPRESSION vp<{{.*}}> = vp<%9> + partial.reduce.add (sub (0, mul (ir<%load2> sext to i32), (ir<%load3> sext to i32)))
+; NEON: Cost of 3 for VF 16: EXPRESSION vp<{{.*}}> = vp<%9> + partial.reduce.add (sub (0, mul (ir<%load2> sext to i32), (ir<%load3> sext to i32)))
target triple = "aarch64"
>From 26d2eaebe46947cf5cf8b2b1c4569aaf4b5e5ca6 Mon Sep 17 00:00:00 2001
From: Sander de Smalen <sander.desmalen at arm.com>
Date: Tue, 5 May 2026 14:17:19 +0000
Subject: [PATCH 2/2] Addressed comments (update some comments, add new tests)
---
.../AArch64/AArch64TargetTransformInfo.cpp | 14 +-
.../AArch64/partial-reduce-costs.ll | 654 ++++++++++++++++++
2 files changed, 664 insertions(+), 4 deletions(-)
create mode 100644 llvm/test/Transforms/LoopVectorize/AArch64/partial-reduce-costs.ll
diff --git a/llvm/lib/Target/AArch64/AArch64TargetTransformInfo.cpp b/llvm/lib/Target/AArch64/AArch64TargetTransformInfo.cpp
index 755321c65881c..3685177328f12 100644
--- a/llvm/lib/Target/AArch64/AArch64TargetTransformInfo.cpp
+++ b/llvm/lib/Target/AArch64/AArch64TargetTransformInfo.cpp
@@ -6047,6 +6047,10 @@ InstructionCost AArch64TTIImpl::getPartialReductionCost(
bool IsSub = Opcode == Instruction::Sub;
InstructionCost Cost = InputLT.first * TTI::TCC_Basic;
+ // Integer partial sub-reductions that don't map to a specific instruction,
+ // carry an extra cost for implementing a double negation:
+ // partial_reduce_umls acc, lhs, rhs
+ // <=> -partial_reduce_umla -acc, lhs, rhs
InstructionCost INegCost = IsSub ? 2 * InputLT.first * TTI::TCC_Basic : 0;
if (AccumLT.second.getScalarType() == MVT::i32 &&
@@ -6064,7 +6068,8 @@ InstructionCost AArch64TTIImpl::getPartialReductionCost(
if (AccumLT.second.getScalarType() == MVT::i64 &&
InputLT.second.getScalarType() == MVT::i16)
return Cost + INegCost;
- // i16 -> i32 is natively supported with SVE2p1
+ // i16 -> i32 is natively supported with SVE2p1 udot/sdot.
+ // For sub-reductions, we prefer using the *mlslb/t instructions.
if (AccumLT.second.getScalarType() == MVT::i32 &&
InputLT.second.getScalarType() == MVT::i16 &&
(ST->hasSVE2p1() || ST->hasSME2()) && !IsSub)
@@ -6078,11 +6083,12 @@ InstructionCost AArch64TTIImpl::getPartialReductionCost(
// the extends in the IR are still counted. This can be fixed
// after https://github.com/llvm/llvm-project/pull/147302 has landed.
return Cost + INegCost;
- // i8 -> i16 is natively supported with SVE2p3
+ // i8 -> i16 is natively supported with SVE2p3 udot/sdot
+ // For sub-reductions, we prefer using the *mlslb/t instructions.
if (AccumLT.second.getScalarType() == MVT::i16 &&
InputLT.second.getScalarType() == MVT::i8 &&
(ST->hasSVE2p3() || ST->hasSME2p3()) && !IsSub)
- return Cost + INegCost;
+ return Cost;
}
// f16 -> f32 is natively supported for fdot using either
@@ -6094,7 +6100,7 @@ InstructionCost AArch64TTIImpl::getPartialReductionCost(
return Cost;
// For a ratio of 2, we can use *mlal and *mlsl top/bottom instructions.
- if (Ratio == 2) {
+ if (Ratio == 2 && !IsUSDot) {
MVT InVT = InputLT.second.getScalarType();
// SVE2 [us]ml[as]lb/t and NEON [us]ml[as]l(2)
diff --git a/llvm/test/Transforms/LoopVectorize/AArch64/partial-reduce-costs.ll b/llvm/test/Transforms/LoopVectorize/AArch64/partial-reduce-costs.ll
new file mode 100644
index 0000000000000..2d83b9d7991b9
--- /dev/null
+++ b/llvm/test/Transforms/LoopVectorize/AArch64/partial-reduce-costs.ll
@@ -0,0 +1,654 @@
+; NOTE: Assertions have been autogenerated by utils/update_analyze_test_checks.py UTC_ARGS: --filter "Cost.of.*EXPRESSION" --filter "Cost.of.*EXPRE" --version 6
+; RUN: opt -passes=loop-vectorize \
+; RUN: -scalable-vectorization=off \
+; RUN: -enable-epilogue-vectorization=false -debug-only=loop-vectorize \
+; RUN: -disable-output < %s 2>&1 | FileCheck %s --check-prefix=NEON
+; RUN: opt -passes=loop-vectorize \
+; RUN: -scalable-vectorization=on -mattr=+sve \
+; RUN: -enable-epilogue-vectorization=false -debug-only=loop-vectorize \
+; RUN: -disable-output < %s 2>&1 | FileCheck %s --check-prefix=SVE
+; RUN: opt -passes=loop-vectorize \
+; RUN: -scalable-vectorization=on -mattr=+sve2 \
+; RUN: -enable-epilogue-vectorization=false -debug-only=loop-vectorize \
+; RUN: -disable-output < %s 2>&1 | FileCheck %s --check-prefix=SVE2
+; RUN: opt -passes=loop-vectorize \
+; RUN: -scalable-vectorization=on -mattr=+sve2p1 \
+; RUN: -enable-epilogue-vectorization=false -debug-only=loop-vectorize \
+; RUN: -disable-output < %s 2>&1 | FileCheck %s --check-prefix=SVE2p1
+; RUN: opt -passes=loop-vectorize \
+; RUN: -scalable-vectorization=on -mattr=+sve2p3 \
+; RUN: -enable-epilogue-vectorization=false -debug-only=loop-vectorize \
+; RUN: -disable-output < %s 2>&1 | FileCheck %s --check-prefix=SVE2p3
+; RUN: opt -passes=loop-vectorize \
+; RUN: -scalable-vectorization=on -mattr=+sve2,+i8mm \
+; RUN: -enable-epilogue-vectorization=false -debug-only=loop-vectorize \
+; RUN: -disable-output < %s 2>&1 | FileCheck %s --check-prefix=I8MM
+
+; REQUIRES: asserts
+target triple = "aarch64"
+
+; sub(i16, zext(i8)->i16 * zext(i8)->i16)
+define i16 @sub_reduction_i16_zext_i8_zext_i8(ptr %src1, ptr %src2, ptr %src3, i16 %init, i32 %n) {
+; NEON-LABEL: 'sub_reduction_i16_zext_i8_zext_i8'
+; NEON: Cost of 2 for VF 16: EXPRESSION vp<[[VP10:%[0-9]+]]> = ir<%acc> + partial.reduce.add (mul (ir<%load1> zext to i16), (ir<%load2> zext to i16))
+; NEON: Cost of 2 for VF 16: EXPRESSION vp<[[VP11:%[0-9]+]]> = vp<[[VP10]]> + partial.reduce.add (sub (0, mul (ir<%load2> zext to i16), (ir<%load3> zext to i16)))
+; NEON: Cost of 2 for VF 16: EXPRESSION vp<[[VP10]]> = ir<%acc> + partial.reduce.add (mul (ir<%load1> zext to i16), (ir<%load2> zext to i16))
+; NEON: Cost of 2 for VF 16: EXPRESSION vp<[[VP11]]> = vp<[[VP10]]> + partial.reduce.add (sub (0, mul (ir<%load2> zext to i16), (ir<%load3> zext to i16)))
+;
+; SVE-LABEL: 'sub_reduction_i16_zext_i8_zext_i8'
+; SVE2-LABEL: 'sub_reduction_i16_zext_i8_zext_i8'
+; SVE2: Cost of 2 for VF vscale x 16: EXPRESSION vp<[[VP10:%[0-9]+]]> = ir<%acc> + partial.reduce.add (mul (ir<%load1> zext to i16), (ir<%load2> zext to i16))
+; SVE2: Cost of 2 for VF vscale x 16: EXPRESSION vp<[[VP11:%[0-9]+]]> = vp<[[VP10]]> + partial.reduce.add (sub (0, mul (ir<%load2> zext to i16), (ir<%load3> zext to i16)))
+; SVE2: Cost of 2 for VF vscale x 16: EXPRESSION vp<[[VP10]]> = ir<%acc> + partial.reduce.add (mul (ir<%load1> zext to i16), (ir<%load2> zext to i16))
+; SVE2: Cost of 2 for VF vscale x 16: EXPRESSION vp<[[VP11]]> = vp<[[VP10]]> + partial.reduce.add (sub (0, mul (ir<%load2> zext to i16), (ir<%load3> zext to i16)))
+;
+; SVE2p1-LABEL: 'sub_reduction_i16_zext_i8_zext_i8'
+; SVE2p1: Cost of 2 for VF vscale x 16: EXPRESSION vp<[[VP10:%[0-9]+]]> = ir<%acc> + partial.reduce.add (mul (ir<%load1> zext to i16), (ir<%load2> zext to i16))
+; SVE2p1: Cost of 2 for VF vscale x 16: EXPRESSION vp<[[VP11:%[0-9]+]]> = vp<[[VP10]]> + partial.reduce.add (sub (0, mul (ir<%load2> zext to i16), (ir<%load3> zext to i16)))
+; SVE2p1: Cost of 2 for VF vscale x 16: EXPRESSION vp<[[VP10]]> = ir<%acc> + partial.reduce.add (mul (ir<%load1> zext to i16), (ir<%load2> zext to i16))
+; SVE2p1: Cost of 2 for VF vscale x 16: EXPRESSION vp<[[VP11]]> = vp<[[VP10]]> + partial.reduce.add (sub (0, mul (ir<%load2> zext to i16), (ir<%load3> zext to i16)))
+;
+; SVE2p3-LABEL: 'sub_reduction_i16_zext_i8_zext_i8'
+; SVE2p3: Cost of 1 for VF vscale x 16: EXPRESSION vp<[[VP10:%[0-9]+]]> = ir<%acc> + partial.reduce.add (mul (ir<%load1> zext to i16), (ir<%load2> zext to i16))
+; SVE2p3: Cost of 2 for VF vscale x 16: EXPRESSION vp<[[VP11:%[0-9]+]]> = vp<[[VP10]]> + partial.reduce.add (sub (0, mul (ir<%load2> zext to i16), (ir<%load3> zext to i16)))
+; SVE2p3: Cost of 1 for VF vscale x 16: EXPRESSION vp<[[VP10]]> = ir<%acc> + partial.reduce.add (mul (ir<%load1> zext to i16), (ir<%load2> zext to i16))
+; SVE2p3: Cost of 2 for VF vscale x 16: EXPRESSION vp<[[VP11]]> = vp<[[VP10]]> + partial.reduce.add (sub (0, mul (ir<%load2> zext to i16), (ir<%load3> zext to i16)))
+;
+; I8MM-LABEL: 'sub_reduction_i16_zext_i8_zext_i8'
+; I8MM: Cost of 2 for VF vscale x 16: EXPRESSION vp<[[VP10:%[0-9]+]]> = ir<%acc> + partial.reduce.add (mul (ir<%load1> zext to i16), (ir<%load2> zext to i16))
+; I8MM: Cost of 2 for VF vscale x 16: EXPRESSION vp<[[VP11:%[0-9]+]]> = vp<[[VP10]]> + partial.reduce.add (sub (0, mul (ir<%load2> zext to i16), (ir<%load3> zext to i16)))
+; I8MM: Cost of 2 for VF vscale x 16: EXPRESSION vp<[[VP10]]> = ir<%acc> + partial.reduce.add (mul (ir<%load1> zext to i16), (ir<%load2> zext to i16))
+; I8MM: Cost of 2 for VF vscale x 16: EXPRESSION vp<[[VP11]]> = vp<[[VP10]]> + partial.reduce.add (sub (0, mul (ir<%load2> zext to i16), (ir<%load3> zext to i16)))
+;
+entry:
+ br label %loop
+
+loop:
+ %iv = phi i32 [ 0, %entry ], [ %iv.next, %loop ]
+ %acc = phi i16 [ %init, %entry ], [ %sub2, %loop ]
+ %gep1 = getelementptr inbounds i8, ptr %src1, i32 %iv
+ %load1 = load i8, ptr %gep1
+ %zext1 = zext i8 %load1 to i16
+ %gep2 = getelementptr inbounds i8, ptr %src2, i32 %iv
+ %load2 = load i8, ptr %gep2
+ %zext2 = zext i8 %load2 to i16
+ %mul12 = mul i16 %zext1, %zext2
+ %gep3 = getelementptr inbounds i8, ptr %src3, i32 %iv
+ %load3 = load i8, ptr %gep3
+ %zext3 = zext i8 %load3 to i16
+ %mul13 = mul i16 %zext2, %zext3
+ %add1 = add i16 %acc, %mul12
+ %sub2 = sub i16 %add1, %mul13
+ %iv.next = add i32 %iv, 1
+ %cmp = icmp ult i32 %iv.next, %n
+ br i1 %cmp, label %loop, label %exit, !llvm.loop !0
+
+exit:
+ ret i16 %sub2
+}
+
+!0 = distinct !{!0, !1, !2}
+!1 = !{!"llvm.loop.interleave.count", i32 1}
+!2 = !{!"llvm.loop.vectorize.width", i32 16}
+
+; sub(i16, zext(i8)->i16 * sext(i8)->i16)
+define i16 @sub_reduction_i16_zext_i8_sext_i8(ptr %src1, ptr %src2, ptr %src3, i16 %init, i32 %n) {
+; NEON-LABEL: 'sub_reduction_i16_zext_i8_sext_i8'
+; SVE-LABEL: 'sub_reduction_i16_zext_i8_sext_i8'
+; SVE2-LABEL: 'sub_reduction_i16_zext_i8_sext_i8'
+; SVE2p1-LABEL: 'sub_reduction_i16_zext_i8_sext_i8'
+; SVE2p3-LABEL: 'sub_reduction_i16_zext_i8_sext_i8'
+; I8MM-LABEL: 'sub_reduction_i16_zext_i8_sext_i8'
+entry:
+ br label %loop
+
+loop:
+ %iv = phi i32 [ 0, %entry ], [ %iv.next, %loop ]
+ %acc = phi i16 [ %init, %entry ], [ %sub2, %loop ]
+ %gep1 = getelementptr inbounds i8, ptr %src1, i32 %iv
+ %load1 = load i8, ptr %gep1
+ %zext1 = zext i8 %load1 to i16
+ %gep2 = getelementptr inbounds i8, ptr %src2, i32 %iv
+ %load2 = load i8, ptr %gep2
+ %sext2 = sext i8 %load2 to i16
+ %mul12 = mul i16 %zext1, %sext2
+ %gep3 = getelementptr inbounds i8, ptr %src3, i32 %iv
+ %load3 = load i8, ptr %gep3
+ %sext3 = sext i8 %load3 to i16
+ %mul13 = mul i16 %zext1, %sext3
+ %add1 = add i16 %acc, %mul12
+ %sub2 = sub i16 %add1, %mul13
+ %iv.next = add i32 %iv, 1
+ %cmp = icmp ult i32 %iv.next, %n
+ br i1 %cmp, label %loop, label %exit, !llvm.loop !3
+
+exit:
+ ret i16 %sub2
+}
+
+!3 = distinct !{!3, !4, !5}
+!4 = !{!"llvm.loop.interleave.count", i32 1}
+!5 = !{!"llvm.loop.vectorize.width", i32 16}
+
+; sub(i32, zext(i8)->i32 * zext(i8)->i32)
+define i32 @sub_reduction_i32_zext_i8_zext_i8(ptr %src1, ptr %src2, ptr %src3, i32 %init, i32 %n) {
+;
+; NEON-LABEL: 'sub_reduction_i32_zext_i8_zext_i8'
+; SVE-LABEL: 'sub_reduction_i32_zext_i8_zext_i8'
+; SVE: Cost of 1 for VF vscale x 16: EXPRESSION vp<[[VP10:%[0-9]+]]> = ir<%acc> + partial.reduce.add (mul (ir<%load1> zext to i32), (ir<%load2> zext to i32))
+; SVE: Cost of 3 for VF vscale x 16: EXPRESSION vp<[[VP11:%[0-9]+]]> = vp<[[VP10]]> + partial.reduce.add (sub (0, mul (ir<%load2> zext to i32), (ir<%load3> zext to i32)))
+; SVE: Cost of 1 for VF vscale x 16: EXPRESSION vp<[[VP10]]> = ir<%acc> + partial.reduce.add (mul (ir<%load1> zext to i32), (ir<%load2> zext to i32))
+; SVE: Cost of 3 for VF vscale x 16: EXPRESSION vp<[[VP11]]> = vp<[[VP10]]> + partial.reduce.add (sub (0, mul (ir<%load2> zext to i32), (ir<%load3> zext to i32)))
+;
+; SVE2-LABEL: 'sub_reduction_i32_zext_i8_zext_i8'
+; SVE2: Cost of 1 for VF vscale x 16: EXPRESSION vp<[[VP10:%[0-9]+]]> = ir<%acc> + partial.reduce.add (mul (ir<%load1> zext to i32), (ir<%load2> zext to i32))
+; SVE2: Cost of 3 for VF vscale x 16: EXPRESSION vp<[[VP11:%[0-9]+]]> = vp<[[VP10]]> + partial.reduce.add (sub (0, mul (ir<%load2> zext to i32), (ir<%load3> zext to i32)))
+; SVE2: Cost of 1 for VF vscale x 16: EXPRESSION vp<[[VP10]]> = ir<%acc> + partial.reduce.add (mul (ir<%load1> zext to i32), (ir<%load2> zext to i32))
+; SVE2: Cost of 3 for VF vscale x 16: EXPRESSION vp<[[VP11]]> = vp<[[VP10]]> + partial.reduce.add (sub (0, mul (ir<%load2> zext to i32), (ir<%load3> zext to i32)))
+;
+; SVE2p1-LABEL: 'sub_reduction_i32_zext_i8_zext_i8'
+; SVE2p1: Cost of 1 for VF vscale x 16: EXPRESSION vp<[[VP10:%[0-9]+]]> = ir<%acc> + partial.reduce.add (mul (ir<%load1> zext to i32), (ir<%load2> zext to i32))
+; SVE2p1: Cost of 3 for VF vscale x 16: EXPRESSION vp<[[VP11:%[0-9]+]]> = vp<[[VP10]]> + partial.reduce.add (sub (0, mul (ir<%load2> zext to i32), (ir<%load3> zext to i32)))
+; SVE2p1: Cost of 1 for VF vscale x 16: EXPRESSION vp<[[VP10]]> = ir<%acc> + partial.reduce.add (mul (ir<%load1> zext to i32), (ir<%load2> zext to i32))
+; SVE2p1: Cost of 3 for VF vscale x 16: EXPRESSION vp<[[VP11]]> = vp<[[VP10]]> + partial.reduce.add (sub (0, mul (ir<%load2> zext to i32), (ir<%load3> zext to i32)))
+;
+; SVE2p3-LABEL: 'sub_reduction_i32_zext_i8_zext_i8'
+; SVE2p3: Cost of 1 for VF vscale x 16: EXPRESSION vp<[[VP10:%[0-9]+]]> = ir<%acc> + partial.reduce.add (mul (ir<%load1> zext to i32), (ir<%load2> zext to i32))
+; SVE2p3: Cost of 3 for VF vscale x 16: EXPRESSION vp<[[VP11:%[0-9]+]]> = vp<[[VP10]]> + partial.reduce.add (sub (0, mul (ir<%load2> zext to i32), (ir<%load3> zext to i32)))
+; SVE2p3: Cost of 1 for VF vscale x 16: EXPRESSION vp<[[VP10]]> = ir<%acc> + partial.reduce.add (mul (ir<%load1> zext to i32), (ir<%load2> zext to i32))
+; SVE2p3: Cost of 3 for VF vscale x 16: EXPRESSION vp<[[VP11]]> = vp<[[VP10]]> + partial.reduce.add (sub (0, mul (ir<%load2> zext to i32), (ir<%load3> zext to i32)))
+;
+; I8MM-LABEL: 'sub_reduction_i32_zext_i8_zext_i8'
+; I8MM: Cost of 1 for VF vscale x 16: EXPRESSION vp<[[VP10:%[0-9]+]]> = ir<%acc> + partial.reduce.add (mul (ir<%load1> zext to i32), (ir<%load2> zext to i32))
+; I8MM: Cost of 3 for VF vscale x 16: EXPRESSION vp<[[VP11:%[0-9]+]]> = vp<[[VP10]]> + partial.reduce.add (sub (0, mul (ir<%load2> zext to i32), (ir<%load3> zext to i32)))
+; I8MM: Cost of 1 for VF vscale x 16: EXPRESSION vp<[[VP10]]> = ir<%acc> + partial.reduce.add (mul (ir<%load1> zext to i32), (ir<%load2> zext to i32))
+; I8MM: Cost of 3 for VF vscale x 16: EXPRESSION vp<[[VP11]]> = vp<[[VP10]]> + partial.reduce.add (sub (0, mul (ir<%load2> zext to i32), (ir<%load3> zext to i32)))
+;
+entry:
+ br label %loop
+
+loop:
+ %iv = phi i32 [ 0, %entry ], [ %iv.next, %loop ]
+ %acc = phi i32 [ %init, %entry ], [ %sub2, %loop ]
+ %gep1 = getelementptr inbounds i8, ptr %src1, i32 %iv
+ %load1 = load i8, ptr %gep1
+ %zext1 = zext i8 %load1 to i32
+ %gep2 = getelementptr inbounds i8, ptr %src2, i32 %iv
+ %load2 = load i8, ptr %gep2
+ %zext2 = zext i8 %load2 to i32
+ %mul12 = mul i32 %zext1, %zext2
+ %gep3 = getelementptr inbounds i8, ptr %src3, i32 %iv
+ %load3 = load i8, ptr %gep3
+ %zext3 = zext i8 %load3 to i32
+ %mul13 = mul i32 %zext2, %zext3
+ %add1 = add i32 %acc, %mul12
+ %sub2 = sub i32 %add1, %mul13
+ %iv.next = add i32 %iv, 1
+ %cmp = icmp ult i32 %iv.next, %n
+ br i1 %cmp, label %loop, label %exit, !llvm.loop !6
+
+exit:
+ ret i32 %sub2
+}
+
+!6 = distinct !{!6, !7, !8}
+!7 = !{!"llvm.loop.interleave.count", i32 1}
+!8 = !{!"llvm.loop.vectorize.width", i32 16}
+
+; sub(i32, zext(i8)->i32 * sext(i8)->i32)
+define i32 @sub_reduction_i32_zext_i8_sext_i8(ptr %src1, ptr %src2, ptr %src3, i32 %init, i32 %n) {
+; NEON-LABEL: 'sub_reduction_i32_zext_i8_sext_i8'
+; SVE-LABEL: 'sub_reduction_i32_zext_i8_sext_i8'
+; SVE2-LABEL: 'sub_reduction_i32_zext_i8_sext_i8'
+; SVE2p1-LABEL: 'sub_reduction_i32_zext_i8_sext_i8'
+; SVE2p3-LABEL: 'sub_reduction_i32_zext_i8_sext_i8'
+; I8MM-LABEL: 'sub_reduction_i32_zext_i8_sext_i8'
+; I8MM: Cost of 1 for VF vscale x 16: EXPRESSION vp<[[VP10:%[0-9]+]]> = ir<%acc> + partial.reduce.add (mul (ir<%load1> zext to i32), (ir<%load2> sext to i32))
+; I8MM: Cost of 3 for VF vscale x 16: EXPRESSION vp<[[VP11:%[0-9]+]]> = vp<[[VP10]]> + partial.reduce.add (sub (0, mul (ir<%load1> zext to i32), (ir<%load3> sext to i32)))
+; I8MM: Cost of 1 for VF vscale x 16: EXPRESSION vp<[[VP10]]> = ir<%acc> + partial.reduce.add (mul (ir<%load1> zext to i32), (ir<%load2> sext to i32))
+; I8MM: Cost of 3 for VF vscale x 16: EXPRESSION vp<[[VP11]]> = vp<[[VP10]]> + partial.reduce.add (sub (0, mul (ir<%load1> zext to i32), (ir<%load3> sext to i32)))
+;
+entry:
+ br label %loop
+
+loop:
+ %iv = phi i32 [ 0, %entry ], [ %iv.next, %loop ]
+ %acc = phi i32 [ %init, %entry ], [ %sub2, %loop ]
+ %gep1 = getelementptr inbounds i8, ptr %src1, i32 %iv
+ %load1 = load i8, ptr %gep1
+ %zext1 = zext i8 %load1 to i32
+ %gep2 = getelementptr inbounds i8, ptr %src2, i32 %iv
+ %load2 = load i8, ptr %gep2
+ %sext2 = sext i8 %load2 to i32
+ %mul12 = mul i32 %zext1, %sext2
+ %gep3 = getelementptr inbounds i8, ptr %src3, i32 %iv
+ %load3 = load i8, ptr %gep3
+ %sext3 = sext i8 %load3 to i32
+ %mul13 = mul i32 %zext1, %sext3
+ %add1 = add i32 %acc, %mul12
+ %sub2 = sub i32 %add1, %mul13
+ %iv.next = add i32 %iv, 1
+ %cmp = icmp ult i32 %iv.next, %n
+ br i1 %cmp, label %loop, label %exit, !llvm.loop !9
+
+exit:
+ ret i32 %sub2
+}
+
+!9 = distinct !{!9, !10, !11}
+!10 = !{!"llvm.loop.interleave.count", i32 1}
+!11 = !{!"llvm.loop.vectorize.width", i32 16}
+
+; sub(i64, zext(i8)->i64 * zext(i8)->i64)
+define i64 @sub_reduction_i64_zext_i8_zext_i8(ptr %src1, ptr %src2, ptr %src3, i64 %init, i32 %n) {
+;
+; NEON-LABEL: 'sub_reduction_i64_zext_i8_zext_i8'
+; SVE-LABEL: 'sub_reduction_i64_zext_i8_zext_i8'
+; SVE: Cost of 1 for VF vscale x 16: EXPRESSION vp<[[VP10:%[0-9]+]]> = ir<%acc> + partial.reduce.add (mul (ir<%load1> zext to i64), (ir<%load2> zext to i64))
+; SVE: Cost of 3 for VF vscale x 16: EXPRESSION vp<[[VP11:%[0-9]+]]> = vp<[[VP10]]> + partial.reduce.add (sub (0, mul (ir<%load2> zext to i64), (ir<%load3> zext to i64)))
+; SVE: Cost of 1 for VF vscale x 16: EXPRESSION vp<[[VP10]]> = ir<%acc> + partial.reduce.add (mul (ir<%load1> zext to i64), (ir<%load2> zext to i64))
+; SVE: Cost of 3 for VF vscale x 16: EXPRESSION vp<[[VP11]]> = vp<[[VP10]]> + partial.reduce.add (sub (0, mul (ir<%load2> zext to i64), (ir<%load3> zext to i64)))
+;
+; SVE2-LABEL: 'sub_reduction_i64_zext_i8_zext_i8'
+; SVE2: Cost of 1 for VF vscale x 16: EXPRESSION vp<[[VP10:%[0-9]+]]> = ir<%acc> + partial.reduce.add (mul (ir<%load1> zext to i64), (ir<%load2> zext to i64))
+; SVE2: Cost of 3 for VF vscale x 16: EXPRESSION vp<[[VP11:%[0-9]+]]> = vp<[[VP10]]> + partial.reduce.add (sub (0, mul (ir<%load2> zext to i64), (ir<%load3> zext to i64)))
+; SVE2: Cost of 1 for VF vscale x 16: EXPRESSION vp<[[VP10]]> = ir<%acc> + partial.reduce.add (mul (ir<%load1> zext to i64), (ir<%load2> zext to i64))
+; SVE2: Cost of 3 for VF vscale x 16: EXPRESSION vp<[[VP11]]> = vp<[[VP10]]> + partial.reduce.add (sub (0, mul (ir<%load2> zext to i64), (ir<%load3> zext to i64)))
+;
+; SVE2p1-LABEL: 'sub_reduction_i64_zext_i8_zext_i8'
+; SVE2p1: Cost of 1 for VF vscale x 16: EXPRESSION vp<[[VP10:%[0-9]+]]> = ir<%acc> + partial.reduce.add (mul (ir<%load1> zext to i64), (ir<%load2> zext to i64))
+; SVE2p1: Cost of 3 for VF vscale x 16: EXPRESSION vp<[[VP11:%[0-9]+]]> = vp<[[VP10]]> + partial.reduce.add (sub (0, mul (ir<%load2> zext to i64), (ir<%load3> zext to i64)))
+; SVE2p1: Cost of 1 for VF vscale x 16: EXPRESSION vp<[[VP10]]> = ir<%acc> + partial.reduce.add (mul (ir<%load1> zext to i64), (ir<%load2> zext to i64))
+; SVE2p1: Cost of 3 for VF vscale x 16: EXPRESSION vp<[[VP11]]> = vp<[[VP10]]> + partial.reduce.add (sub (0, mul (ir<%load2> zext to i64), (ir<%load3> zext to i64)))
+;
+; SVE2p3-LABEL: 'sub_reduction_i64_zext_i8_zext_i8'
+; SVE2p3: Cost of 1 for VF vscale x 16: EXPRESSION vp<[[VP10:%[0-9]+]]> = ir<%acc> + partial.reduce.add (mul (ir<%load1> zext to i64), (ir<%load2> zext to i64))
+; SVE2p3: Cost of 3 for VF vscale x 16: EXPRESSION vp<[[VP11:%[0-9]+]]> = vp<[[VP10]]> + partial.reduce.add (sub (0, mul (ir<%load2> zext to i64), (ir<%load3> zext to i64)))
+; SVE2p3: Cost of 1 for VF vscale x 16: EXPRESSION vp<[[VP10]]> = ir<%acc> + partial.reduce.add (mul (ir<%load1> zext to i64), (ir<%load2> zext to i64))
+; SVE2p3: Cost of 3 for VF vscale x 16: EXPRESSION vp<[[VP11]]> = vp<[[VP10]]> + partial.reduce.add (sub (0, mul (ir<%load2> zext to i64), (ir<%load3> zext to i64)))
+;
+; I8MM-LABEL: 'sub_reduction_i64_zext_i8_zext_i8'
+; I8MM: Cost of 1 for VF vscale x 16: EXPRESSION vp<[[VP10:%[0-9]+]]> = ir<%acc> + partial.reduce.add (mul (ir<%load1> zext to i64), (ir<%load2> zext to i64))
+; I8MM: Cost of 3 for VF vscale x 16: EXPRESSION vp<[[VP11:%[0-9]+]]> = vp<[[VP10]]> + partial.reduce.add (sub (0, mul (ir<%load2> zext to i64), (ir<%load3> zext to i64)))
+; I8MM: Cost of 1 for VF vscale x 16: EXPRESSION vp<[[VP10]]> = ir<%acc> + partial.reduce.add (mul (ir<%load1> zext to i64), (ir<%load2> zext to i64))
+; I8MM: Cost of 3 for VF vscale x 16: EXPRESSION vp<[[VP11]]> = vp<[[VP10]]> + partial.reduce.add (sub (0, mul (ir<%load2> zext to i64), (ir<%load3> zext to i64)))
+;
+entry:
+ br label %loop
+
+loop:
+ %iv = phi i32 [ 0, %entry ], [ %iv.next, %loop ]
+ %acc = phi i64 [ %init, %entry ], [ %sub2, %loop ]
+ %gep1 = getelementptr inbounds i8, ptr %src1, i32 %iv
+ %load1 = load i8, ptr %gep1
+ %zext1 = zext i8 %load1 to i64
+ %gep2 = getelementptr inbounds i8, ptr %src2, i32 %iv
+ %load2 = load i8, ptr %gep2
+ %zext2 = zext i8 %load2 to i64
+ %mul12 = mul i64 %zext1, %zext2
+ %gep3 = getelementptr inbounds i8, ptr %src3, i32 %iv
+ %load3 = load i8, ptr %gep3
+ %zext3 = zext i8 %load3 to i64
+ %mul13 = mul i64 %zext2, %zext3
+ %add1 = add i64 %acc, %mul12
+ %sub2 = sub i64 %add1, %mul13
+ %iv.next = add i32 %iv, 1
+ %cmp = icmp ult i32 %iv.next, %n
+ br i1 %cmp, label %loop, label %exit, !llvm.loop !12
+
+exit:
+ ret i64 %sub2
+}
+
+!12 = distinct !{!12, !13, !14}
+!13 = !{!"llvm.loop.interleave.count", i32 1}
+!14 = !{!"llvm.loop.vectorize.width", i32 16}
+
+; sub(i64, zext(i8)->i64 * sext(i8)->i64)
+define i64 @sub_reduction_i64_zext_i8_sext_i8(ptr %src1, ptr %src2, ptr %src3, i64 %init, i32 %n) {
+; NEON-LABEL: 'sub_reduction_i64_zext_i8_sext_i8'
+; SVE-LABEL: 'sub_reduction_i64_zext_i8_sext_i8'
+; SVE2-LABEL: 'sub_reduction_i64_zext_i8_sext_i8'
+; SVE2p1-LABEL: 'sub_reduction_i64_zext_i8_sext_i8'
+; SVE2p3-LABEL: 'sub_reduction_i64_zext_i8_sext_i8'
+; I8MM-LABEL: 'sub_reduction_i64_zext_i8_sext_i8'
+entry:
+ br label %loop
+
+loop:
+ %iv = phi i32 [ 0, %entry ], [ %iv.next, %loop ]
+ %acc = phi i64 [ %init, %entry ], [ %sub2, %loop ]
+ %gep1 = getelementptr inbounds i8, ptr %src1, i32 %iv
+ %load1 = load i8, ptr %gep1
+ %zext1 = zext i8 %load1 to i64
+ %gep2 = getelementptr inbounds i8, ptr %src2, i32 %iv
+ %load2 = load i8, ptr %gep2
+ %sext2 = sext i8 %load2 to i64
+ %mul12 = mul i64 %zext1, %sext2
+ %gep3 = getelementptr inbounds i8, ptr %src3, i32 %iv
+ %load3 = load i8, ptr %gep3
+ %sext3 = sext i8 %load3 to i64
+ %mul13 = mul i64 %zext1, %sext3
+ %add1 = add i64 %acc, %mul12
+ %sub2 = sub i64 %add1, %mul13
+ %iv.next = add i32 %iv, 1
+ %cmp = icmp ult i32 %iv.next, %n
+ br i1 %cmp, label %loop, label %exit, !llvm.loop !15
+
+exit:
+ ret i64 %sub2
+}
+
+!15 = distinct !{!15, !16, !17}
+!16 = !{!"llvm.loop.interleave.count", i32 1}
+!17 = !{!"llvm.loop.vectorize.width", i32 16}
+
+; sub(i32, zext(i16)->i32 * zext(i16)->i32)
+define i32 @sub_reduction_i32_zext_i16_zext_i16(ptr %src1, ptr %src2, ptr %src3, i32 %init, i32 %n) {
+; NEON-LABEL: 'sub_reduction_i32_zext_i16_zext_i16'
+; NEON: Cost of 2 for VF 8: EXPRESSION vp<[[VP10:%[0-9]+]]> = ir<%acc> + partial.reduce.add (mul (ir<%load1> zext to i32), (ir<%load2> zext to i32))
+; NEON: Cost of 2 for VF 8: EXPRESSION vp<[[VP11:%[0-9]+]]> = vp<[[VP10]]> + partial.reduce.add (sub (0, mul (ir<%load2> zext to i32), (ir<%load3> zext to i32)))
+; NEON: Cost of 2 for VF 8: EXPRESSION vp<[[VP10]]> = ir<%acc> + partial.reduce.add (mul (ir<%load1> zext to i32), (ir<%load2> zext to i32))
+; NEON: Cost of 2 for VF 8: EXPRESSION vp<[[VP11]]> = vp<[[VP10]]> + partial.reduce.add (sub (0, mul (ir<%load2> zext to i32), (ir<%load3> zext to i32)))
+;
+; SVE-LABEL: 'sub_reduction_i32_zext_i16_zext_i16'
+; SVE2-LABEL: 'sub_reduction_i32_zext_i16_zext_i16'
+; SVE2: Cost of 2 for VF vscale x 8: EXPRESSION vp<[[VP10:%[0-9]+]]> = ir<%acc> + partial.reduce.add (mul (ir<%load1> zext to i32), (ir<%load2> zext to i32))
+; SVE2: Cost of 2 for VF vscale x 8: EXPRESSION vp<[[VP11:%[0-9]+]]> = vp<[[VP10]]> + partial.reduce.add (sub (0, mul (ir<%load2> zext to i32), (ir<%load3> zext to i32)))
+; SVE2: Cost of 2 for VF vscale x 8: EXPRESSION vp<[[VP10]]> = ir<%acc> + partial.reduce.add (mul (ir<%load1> zext to i32), (ir<%load2> zext to i32))
+; SVE2: Cost of 2 for VF vscale x 8: EXPRESSION vp<[[VP11]]> = vp<[[VP10]]> + partial.reduce.add (sub (0, mul (ir<%load2> zext to i32), (ir<%load3> zext to i32)))
+;
+; SVE2p1-LABEL: 'sub_reduction_i32_zext_i16_zext_i16'
+; SVE2p1: Cost of 1 for VF vscale x 8: EXPRESSION vp<[[VP10:%[0-9]+]]> = ir<%acc> + partial.reduce.add (mul (ir<%load1> zext to i32), (ir<%load2> zext to i32))
+; SVE2p1: Cost of 2 for VF vscale x 8: EXPRESSION vp<[[VP11:%[0-9]+]]> = vp<[[VP10]]> + partial.reduce.add (sub (0, mul (ir<%load2> zext to i32), (ir<%load3> zext to i32)))
+; SVE2p1: Cost of 1 for VF vscale x 8: EXPRESSION vp<[[VP10]]> = ir<%acc> + partial.reduce.add (mul (ir<%load1> zext to i32), (ir<%load2> zext to i32))
+; SVE2p1: Cost of 2 for VF vscale x 8: EXPRESSION vp<[[VP11]]> = vp<[[VP10]]> + partial.reduce.add (sub (0, mul (ir<%load2> zext to i32), (ir<%load3> zext to i32)))
+;
+; SVE2p3-LABEL: 'sub_reduction_i32_zext_i16_zext_i16'
+; SVE2p3: Cost of 1 for VF vscale x 8: EXPRESSION vp<[[VP10:%[0-9]+]]> = ir<%acc> + partial.reduce.add (mul (ir<%load1> zext to i32), (ir<%load2> zext to i32))
+; SVE2p3: Cost of 2 for VF vscale x 8: EXPRESSION vp<[[VP11:%[0-9]+]]> = vp<[[VP10]]> + partial.reduce.add (sub (0, mul (ir<%load2> zext to i32), (ir<%load3> zext to i32)))
+; SVE2p3: Cost of 1 for VF vscale x 8: EXPRESSION vp<[[VP10]]> = ir<%acc> + partial.reduce.add (mul (ir<%load1> zext to i32), (ir<%load2> zext to i32))
+; SVE2p3: Cost of 2 for VF vscale x 8: EXPRESSION vp<[[VP11]]> = vp<[[VP10]]> + partial.reduce.add (sub (0, mul (ir<%load2> zext to i32), (ir<%load3> zext to i32)))
+;
+; I8MM-LABEL: 'sub_reduction_i32_zext_i16_zext_i16'
+; I8MM: Cost of 2 for VF vscale x 8: EXPRESSION vp<[[VP10:%[0-9]+]]> = ir<%acc> + partial.reduce.add (mul (ir<%load1> zext to i32), (ir<%load2> zext to i32))
+; I8MM: Cost of 2 for VF vscale x 8: EXPRESSION vp<[[VP11:%[0-9]+]]> = vp<[[VP10]]> + partial.reduce.add (sub (0, mul (ir<%load2> zext to i32), (ir<%load3> zext to i32)))
+; I8MM: Cost of 2 for VF vscale x 8: EXPRESSION vp<[[VP10]]> = ir<%acc> + partial.reduce.add (mul (ir<%load1> zext to i32), (ir<%load2> zext to i32))
+; I8MM: Cost of 2 for VF vscale x 8: EXPRESSION vp<[[VP11]]> = vp<[[VP10]]> + partial.reduce.add (sub (0, mul (ir<%load2> zext to i32), (ir<%load3> zext to i32)))
+;
+entry:
+ br label %loop
+
+loop:
+ %iv = phi i32 [ 0, %entry ], [ %iv.next, %loop ]
+ %acc = phi i32 [ %init, %entry ], [ %sub2, %loop ]
+ %gep1 = getelementptr inbounds i16, ptr %src1, i32 %iv
+ %load1 = load i16, ptr %gep1
+ %zext1 = zext i16 %load1 to i32
+ %gep2 = getelementptr inbounds i16, ptr %src2, i32 %iv
+ %load2 = load i16, ptr %gep2
+ %zext2 = zext i16 %load2 to i32
+ %mul12 = mul i32 %zext1, %zext2
+ %gep3 = getelementptr inbounds i16, ptr %src3, i32 %iv
+ %load3 = load i16, ptr %gep3
+ %zext3 = zext i16 %load3 to i32
+ %mul13 = mul i32 %zext2, %zext3
+ %add1 = add i32 %acc, %mul12
+ %sub2 = sub i32 %add1, %mul13
+ %iv.next = add i32 %iv, 1
+ %cmp = icmp ult i32 %iv.next, %n
+ br i1 %cmp, label %loop, label %exit, !llvm.loop !18
+
+exit:
+ ret i32 %sub2
+}
+
+!18 = distinct !{!18, !19, !20}
+!19 = !{!"llvm.loop.interleave.count", i32 1}
+!20 = !{!"llvm.loop.vectorize.width", i32 8}
+
+; sub(i32, zext(i16)->i32 * sext(i16)->i32)
+define i32 @sub_reduction_i32_zext_i16_sext_i16(ptr %src1, ptr %src2, ptr %src3, i32 %init, i32 %n) {
+; NEON-LABEL: 'sub_reduction_i32_zext_i16_sext_i16'
+; SVE-LABEL: 'sub_reduction_i32_zext_i16_sext_i16'
+; SVE2-LABEL: 'sub_reduction_i32_zext_i16_sext_i16'
+; SVE2p1-LABEL: 'sub_reduction_i32_zext_i16_sext_i16'
+; SVE2p3-LABEL: 'sub_reduction_i32_zext_i16_sext_i16'
+; I8MM-LABEL: 'sub_reduction_i32_zext_i16_sext_i16'
+entry:
+ br label %loop
+
+loop:
+ %iv = phi i32 [ 0, %entry ], [ %iv.next, %loop ]
+ %acc = phi i32 [ %init, %entry ], [ %sub2, %loop ]
+ %gep1 = getelementptr inbounds i16, ptr %src1, i32 %iv
+ %load1 = load i16, ptr %gep1
+ %zext1 = zext i16 %load1 to i32
+ %gep2 = getelementptr inbounds i16, ptr %src2, i32 %iv
+ %load2 = load i16, ptr %gep2
+ %sext2 = sext i16 %load2 to i32
+ %mul12 = mul i32 %zext1, %sext2
+ %gep3 = getelementptr inbounds i16, ptr %src3, i32 %iv
+ %load3 = load i16, ptr %gep3
+ %sext3 = sext i16 %load3 to i32
+ %mul13 = mul i32 %zext1, %sext3
+ %add1 = add i32 %acc, %mul12
+ %sub2 = sub i32 %add1, %mul13
+ %iv.next = add i32 %iv, 1
+ %cmp = icmp ult i32 %iv.next, %n
+ br i1 %cmp, label %loop, label %exit, !llvm.loop !21
+
+exit:
+ ret i32 %sub2
+}
+
+!21 = distinct !{!21, !22, !23}
+!22 = !{!"llvm.loop.interleave.count", i32 1}
+!23 = !{!"llvm.loop.vectorize.width", i32 8}
+
+; sub(i64, zext(i16)->i64 * zext(i16)->i64)
+define i64 @sub_reduction_i64_zext_i16_zext_i16(ptr %src1, ptr %src2, ptr %src3, i64 %init, i32 %n) {
+;
+; NEON-LABEL: 'sub_reduction_i64_zext_i16_zext_i16'
+; SVE-LABEL: 'sub_reduction_i64_zext_i16_zext_i16'
+; SVE: Cost of 1 for VF vscale x 8: EXPRESSION vp<[[VP10:%[0-9]+]]> = ir<%acc> + partial.reduce.add (mul (ir<%load1> zext to i64), (ir<%load2> zext to i64))
+; SVE: Cost of 3 for VF vscale x 8: EXPRESSION vp<[[VP11:%[0-9]+]]> = vp<[[VP10]]> + partial.reduce.add (sub (0, mul (ir<%load2> zext to i64), (ir<%load3> zext to i64)))
+; SVE: Cost of 1 for VF vscale x 8: EXPRESSION vp<[[VP10]]> = ir<%acc> + partial.reduce.add (mul (ir<%load1> zext to i64), (ir<%load2> zext to i64))
+; SVE: Cost of 3 for VF vscale x 8: EXPRESSION vp<[[VP11]]> = vp<[[VP10]]> + partial.reduce.add (sub (0, mul (ir<%load2> zext to i64), (ir<%load3> zext to i64)))
+;
+; SVE2-LABEL: 'sub_reduction_i64_zext_i16_zext_i16'
+; SVE2: Cost of 1 for VF vscale x 8: EXPRESSION vp<[[VP10:%[0-9]+]]> = ir<%acc> + partial.reduce.add (mul (ir<%load1> zext to i64), (ir<%load2> zext to i64))
+; SVE2: Cost of 3 for VF vscale x 8: EXPRESSION vp<[[VP11:%[0-9]+]]> = vp<[[VP10]]> + partial.reduce.add (sub (0, mul (ir<%load2> zext to i64), (ir<%load3> zext to i64)))
+; SVE2: Cost of 1 for VF vscale x 8: EXPRESSION vp<[[VP10]]> = ir<%acc> + partial.reduce.add (mul (ir<%load1> zext to i64), (ir<%load2> zext to i64))
+; SVE2: Cost of 3 for VF vscale x 8: EXPRESSION vp<[[VP11]]> = vp<[[VP10]]> + partial.reduce.add (sub (0, mul (ir<%load2> zext to i64), (ir<%load3> zext to i64)))
+;
+; SVE2p1-LABEL: 'sub_reduction_i64_zext_i16_zext_i16'
+; SVE2p1: Cost of 1 for VF vscale x 8: EXPRESSION vp<[[VP10:%[0-9]+]]> = ir<%acc> + partial.reduce.add (mul (ir<%load1> zext to i64), (ir<%load2> zext to i64))
+; SVE2p1: Cost of 3 for VF vscale x 8: EXPRESSION vp<[[VP11:%[0-9]+]]> = vp<[[VP10]]> + partial.reduce.add (sub (0, mul (ir<%load2> zext to i64), (ir<%load3> zext to i64)))
+; SVE2p1: Cost of 1 for VF vscale x 8: EXPRESSION vp<[[VP10]]> = ir<%acc> + partial.reduce.add (mul (ir<%load1> zext to i64), (ir<%load2> zext to i64))
+; SVE2p1: Cost of 3 for VF vscale x 8: EXPRESSION vp<[[VP11]]> = vp<[[VP10]]> + partial.reduce.add (sub (0, mul (ir<%load2> zext to i64), (ir<%load3> zext to i64)))
+;
+; SVE2p3-LABEL: 'sub_reduction_i64_zext_i16_zext_i16'
+; SVE2p3: Cost of 1 for VF vscale x 8: EXPRESSION vp<[[VP10:%[0-9]+]]> = ir<%acc> + partial.reduce.add (mul (ir<%load1> zext to i64), (ir<%load2> zext to i64))
+; SVE2p3: Cost of 3 for VF vscale x 8: EXPRESSION vp<[[VP11:%[0-9]+]]> = vp<[[VP10]]> + partial.reduce.add (sub (0, mul (ir<%load2> zext to i64), (ir<%load3> zext to i64)))
+; SVE2p3: Cost of 1 for VF vscale x 8: EXPRESSION vp<[[VP10]]> = ir<%acc> + partial.reduce.add (mul (ir<%load1> zext to i64), (ir<%load2> zext to i64))
+; SVE2p3: Cost of 3 for VF vscale x 8: EXPRESSION vp<[[VP11]]> = vp<[[VP10]]> + partial.reduce.add (sub (0, mul (ir<%load2> zext to i64), (ir<%load3> zext to i64)))
+;
+; I8MM-LABEL: 'sub_reduction_i64_zext_i16_zext_i16'
+; I8MM: Cost of 1 for VF vscale x 8: EXPRESSION vp<[[VP10:%[0-9]+]]> = ir<%acc> + partial.reduce.add (mul (ir<%load1> zext to i64), (ir<%load2> zext to i64))
+; I8MM: Cost of 3 for VF vscale x 8: EXPRESSION vp<[[VP11:%[0-9]+]]> = vp<[[VP10]]> + partial.reduce.add (sub (0, mul (ir<%load2> zext to i64), (ir<%load3> zext to i64)))
+; I8MM: Cost of 1 for VF vscale x 8: EXPRESSION vp<[[VP10]]> = ir<%acc> + partial.reduce.add (mul (ir<%load1> zext to i64), (ir<%load2> zext to i64))
+; I8MM: Cost of 3 for VF vscale x 8: EXPRESSION vp<[[VP11]]> = vp<[[VP10]]> + partial.reduce.add (sub (0, mul (ir<%load2> zext to i64), (ir<%load3> zext to i64)))
+;
+entry:
+ br label %loop
+
+loop:
+ %iv = phi i32 [ 0, %entry ], [ %iv.next, %loop ]
+ %acc = phi i64 [ %init, %entry ], [ %sub2, %loop ]
+ %gep1 = getelementptr inbounds i16, ptr %src1, i32 %iv
+ %load1 = load i16, ptr %gep1
+ %zext1 = zext i16 %load1 to i64
+ %gep2 = getelementptr inbounds i16, ptr %src2, i32 %iv
+ %load2 = load i16, ptr %gep2
+ %zext2 = zext i16 %load2 to i64
+ %mul12 = mul i64 %zext1, %zext2
+ %gep3 = getelementptr inbounds i16, ptr %src3, i32 %iv
+ %load3 = load i16, ptr %gep3
+ %zext3 = zext i16 %load3 to i64
+ %mul13 = mul i64 %zext2, %zext3
+ %add1 = add i64 %acc, %mul12
+ %sub2 = sub i64 %add1, %mul13
+ %iv.next = add i32 %iv, 1
+ %cmp = icmp ult i32 %iv.next, %n
+ br i1 %cmp, label %loop, label %exit, !llvm.loop !24
+
+exit:
+ ret i64 %sub2
+}
+
+!24 = distinct !{!24, !25, !26}
+!25 = !{!"llvm.loop.interleave.count", i32 1}
+!26 = !{!"llvm.loop.vectorize.width", i32 8}
+
+; sub(i64, zext(i16)->i64 * sext(i16)->i64)
+define i64 @sub_reduction_i64_zext_i16_sext_i16(ptr %src1, ptr %src2, ptr %src3, i64 %init, i32 %n) {
+; NEON-LABEL: 'sub_reduction_i64_zext_i16_sext_i16'
+; SVE-LABEL: 'sub_reduction_i64_zext_i16_sext_i16'
+; SVE2-LABEL: 'sub_reduction_i64_zext_i16_sext_i16'
+; SVE2p1-LABEL: 'sub_reduction_i64_zext_i16_sext_i16'
+; SVE2p3-LABEL: 'sub_reduction_i64_zext_i16_sext_i16'
+; I8MM-LABEL: 'sub_reduction_i64_zext_i16_sext_i16'
+entry:
+ br label %loop
+
+loop:
+ %iv = phi i32 [ 0, %entry ], [ %iv.next, %loop ]
+ %acc = phi i64 [ %init, %entry ], [ %sub2, %loop ]
+ %gep1 = getelementptr inbounds i16, ptr %src1, i32 %iv
+ %load1 = load i16, ptr %gep1
+ %zext1 = zext i16 %load1 to i64
+ %gep2 = getelementptr inbounds i16, ptr %src2, i32 %iv
+ %load2 = load i16, ptr %gep2
+ %sext2 = sext i16 %load2 to i64
+ %mul12 = mul i64 %zext1, %sext2
+ %gep3 = getelementptr inbounds i16, ptr %src3, i32 %iv
+ %load3 = load i16, ptr %gep3
+ %sext3 = sext i16 %load3 to i64
+ %mul13 = mul i64 %zext1, %sext3
+ %add1 = add i64 %acc, %mul12
+ %sub2 = sub i64 %add1, %mul13
+ %iv.next = add i32 %iv, 1
+ %cmp = icmp ult i32 %iv.next, %n
+ br i1 %cmp, label %loop, label %exit, !llvm.loop !27
+
+exit:
+ ret i64 %sub2
+}
+
+!27 = distinct !{!27, !28, !29}
+!28 = !{!"llvm.loop.interleave.count", i32 1}
+!29 = !{!"llvm.loop.vectorize.width", i32 8}
+
+; sub(i64, zext(i32)->i64 * zext(i32)->i64)
+define i64 @sub_reduction_i64_zext_i32_zext_i32(ptr %src1, ptr %src2, ptr %src3, i64 %init, i32 %n) {
+; NEON-LABEL: 'sub_reduction_i64_zext_i32_zext_i32'
+; NEON: Cost of 2 for VF 4: EXPRESSION vp<[[VP10:%[0-9]+]]> = ir<%acc> + partial.reduce.add (mul (ir<%load1> zext to i64), (ir<%load2> zext to i64))
+; NEON: Cost of 2 for VF 4: EXPRESSION vp<[[VP11:%[0-9]+]]> = vp<[[VP10]]> + partial.reduce.add (sub (0, mul (ir<%load2> zext to i64), (ir<%load3> zext to i64)))
+; NEON: Cost of 2 for VF 4: EXPRESSION vp<[[VP10]]> = ir<%acc> + partial.reduce.add (mul (ir<%load1> zext to i64), (ir<%load2> zext to i64))
+; NEON: Cost of 2 for VF 4: EXPRESSION vp<[[VP11]]> = vp<[[VP10]]> + partial.reduce.add (sub (0, mul (ir<%load2> zext to i64), (ir<%load3> zext to i64)))
+;
+; SVE-LABEL: 'sub_reduction_i64_zext_i32_zext_i32'
+; SVE2-LABEL: 'sub_reduction_i64_zext_i32_zext_i32'
+; SVE2: Cost of 2 for VF vscale x 4: EXPRESSION vp<[[VP10:%[0-9]+]]> = ir<%acc> + partial.reduce.add (mul (ir<%load1> zext to i64), (ir<%load2> zext to i64))
+; SVE2: Cost of 2 for VF vscale x 4: EXPRESSION vp<[[VP11:%[0-9]+]]> = vp<[[VP10]]> + partial.reduce.add (sub (0, mul (ir<%load2> zext to i64), (ir<%load3> zext to i64)))
+; SVE2: Cost of 2 for VF vscale x 4: EXPRESSION vp<[[VP10]]> = ir<%acc> + partial.reduce.add (mul (ir<%load1> zext to i64), (ir<%load2> zext to i64))
+; SVE2: Cost of 2 for VF vscale x 4: EXPRESSION vp<[[VP11]]> = vp<[[VP10]]> + partial.reduce.add (sub (0, mul (ir<%load2> zext to i64), (ir<%load3> zext to i64)))
+;
+; SVE2p1-LABEL: 'sub_reduction_i64_zext_i32_zext_i32'
+; SVE2p1: Cost of 2 for VF vscale x 4: EXPRESSION vp<[[VP10:%[0-9]+]]> = ir<%acc> + partial.reduce.add (mul (ir<%load1> zext to i64), (ir<%load2> zext to i64))
+; SVE2p1: Cost of 2 for VF vscale x 4: EXPRESSION vp<[[VP11:%[0-9]+]]> = vp<[[VP10]]> + partial.reduce.add (sub (0, mul (ir<%load2> zext to i64), (ir<%load3> zext to i64)))
+; SVE2p1: Cost of 2 for VF vscale x 4: EXPRESSION vp<[[VP10]]> = ir<%acc> + partial.reduce.add (mul (ir<%load1> zext to i64), (ir<%load2> zext to i64))
+; SVE2p1: Cost of 2 for VF vscale x 4: EXPRESSION vp<[[VP11]]> = vp<[[VP10]]> + partial.reduce.add (sub (0, mul (ir<%load2> zext to i64), (ir<%load3> zext to i64)))
+;
+; SVE2p3-LABEL: 'sub_reduction_i64_zext_i32_zext_i32'
+; SVE2p3: Cost of 2 for VF vscale x 4: EXPRESSION vp<[[VP10:%[0-9]+]]> = ir<%acc> + partial.reduce.add (mul (ir<%load1> zext to i64), (ir<%load2> zext to i64))
+; SVE2p3: Cost of 2 for VF vscale x 4: EXPRESSION vp<[[VP11:%[0-9]+]]> = vp<[[VP10]]> + partial.reduce.add (sub (0, mul (ir<%load2> zext to i64), (ir<%load3> zext to i64)))
+; SVE2p3: Cost of 2 for VF vscale x 4: EXPRESSION vp<[[VP10]]> = ir<%acc> + partial.reduce.add (mul (ir<%load1> zext to i64), (ir<%load2> zext to i64))
+; SVE2p3: Cost of 2 for VF vscale x 4: EXPRESSION vp<[[VP11]]> = vp<[[VP10]]> + partial.reduce.add (sub (0, mul (ir<%load2> zext to i64), (ir<%load3> zext to i64)))
+;
+; I8MM-LABEL: 'sub_reduction_i64_zext_i32_zext_i32'
+; I8MM: Cost of 2 for VF vscale x 4: EXPRESSION vp<[[VP10:%[0-9]+]]> = ir<%acc> + partial.reduce.add (mul (ir<%load1> zext to i64), (ir<%load2> zext to i64))
+; I8MM: Cost of 2 for VF vscale x 4: EXPRESSION vp<[[VP11:%[0-9]+]]> = vp<[[VP10]]> + partial.reduce.add (sub (0, mul (ir<%load2> zext to i64), (ir<%load3> zext to i64)))
+; I8MM: Cost of 2 for VF vscale x 4: EXPRESSION vp<[[VP10]]> = ir<%acc> + partial.reduce.add (mul (ir<%load1> zext to i64), (ir<%load2> zext to i64))
+; I8MM: Cost of 2 for VF vscale x 4: EXPRESSION vp<[[VP11]]> = vp<[[VP10]]> + partial.reduce.add (sub (0, mul (ir<%load2> zext to i64), (ir<%load3> zext to i64)))
+;
+entry:
+ br label %loop
+
+loop:
+ %iv = phi i32 [ 0, %entry ], [ %iv.next, %loop ]
+ %acc = phi i64 [ %init, %entry ], [ %sub2, %loop ]
+ %gep1 = getelementptr inbounds i32, ptr %src1, i32 %iv
+ %load1 = load i32, ptr %gep1
+ %zext1 = zext i32 %load1 to i64
+ %gep2 = getelementptr inbounds i32, ptr %src2, i32 %iv
+ %load2 = load i32, ptr %gep2
+ %zext2 = zext i32 %load2 to i64
+ %mul12 = mul i64 %zext1, %zext2
+ %gep3 = getelementptr inbounds i32, ptr %src3, i32 %iv
+ %load3 = load i32, ptr %gep3
+ %zext3 = zext i32 %load3 to i64
+ %mul13 = mul i64 %zext2, %zext3
+ %add1 = add i64 %acc, %mul12
+ %sub2 = sub i64 %add1, %mul13
+ %iv.next = add i32 %iv, 1
+ %cmp = icmp ult i32 %iv.next, %n
+ br i1 %cmp, label %loop, label %exit, !llvm.loop !30
+
+exit:
+ ret i64 %sub2
+}
+
+!30 = distinct !{!30, !31, !32}
+!31 = !{!"llvm.loop.interleave.count", i32 1}
+!32 = !{!"llvm.loop.vectorize.width", i32 4}
+
+; sub(i64, zext(i32)->i64 * sext(i32)->i64)
+define i64 @sub_reduction_i64_zext_i32_sext_i32(ptr %src1, ptr %src2, ptr %src3, i64 %init, i32 %n) {
+; NEON-LABEL: 'sub_reduction_i64_zext_i32_sext_i32'
+; SVE-LABEL: 'sub_reduction_i64_zext_i32_sext_i32'
+; SVE2-LABEL: 'sub_reduction_i64_zext_i32_sext_i32'
+; SVE2p1-LABEL: 'sub_reduction_i64_zext_i32_sext_i32'
+; SVE2p3-LABEL: 'sub_reduction_i64_zext_i32_sext_i32'
+; I8MM-LABEL: 'sub_reduction_i64_zext_i32_sext_i32'
+entry:
+ br label %loop
+
+loop:
+ %iv = phi i32 [ 0, %entry ], [ %iv.next, %loop ]
+ %acc = phi i64 [ %init, %entry ], [ %sub2, %loop ]
+ %gep1 = getelementptr inbounds i32, ptr %src1, i32 %iv
+ %load1 = load i32, ptr %gep1
+ %zext1 = zext i32 %load1 to i64
+ %gep2 = getelementptr inbounds i32, ptr %src2, i32 %iv
+ %load2 = load i32, ptr %gep2
+ %sext2 = sext i32 %load2 to i64
+ %mul12 = mul i64 %zext1, %sext2
+ %gep3 = getelementptr inbounds i32, ptr %src3, i32 %iv
+ %load3 = load i32, ptr %gep3
+ %sext3 = sext i32 %load3 to i64
+ %mul13 = mul i64 %zext1, %sext3
+ %add1 = add i64 %acc, %mul12
+ %sub2 = sub i64 %add1, %mul13
+ %iv.next = add i32 %iv, 1
+ %cmp = icmp ult i32 %iv.next, %n
+ br i1 %cmp, label %loop, label %exit, !llvm.loop !33
+
+exit:
+ ret i64 %sub2
+}
+
+!33 = distinct !{!33, !34, !35}
+!34 = !{!"llvm.loop.interleave.count", i32 1}
+!35 = !{!"llvm.loop.vectorize.width", i32 4}
More information about the llvm-commits
mailing list