[llvm] bdec04f - [AArch64] Add new dot insts. to cost model (#189642)
via llvm-commits
llvm-commits at lists.llvm.org
Tue Apr 14 02:08:54 PDT 2026
Author: Lukacma
Date: 2026-04-14T10:08:49+01:00
New Revision: bdec04f74395d14010ddf9aa4e3017504f70cfdb
URL: https://github.com/llvm/llvm-project/commit/bdec04f74395d14010ddf9aa4e3017504f70cfdb
DIFF: https://github.com/llvm/llvm-project/commit/bdec04f74395d14010ddf9aa4e3017504f70cfdb.diff
LOG: [AArch64] Add new dot insts. to cost model (#189642)
This patch builds on #184659 and #184649 and adds cost modelling for new
dot instructions variants, codegened in those patches.
Added:
llvm/test/Transforms/LoopVectorize/AArch64/partial-reduce-add-sdot-i8-i16.ll
Modified:
llvm/lib/Target/AArch64/AArch64TargetTransformInfo.cpp
llvm/test/Transforms/LoopVectorize/AArch64/partial-reduce-add-sdot-i16-i32.ll
Removed:
################################################################################
diff --git a/llvm/lib/Target/AArch64/AArch64TargetTransformInfo.cpp b/llvm/lib/Target/AArch64/AArch64TargetTransformInfo.cpp
index 734339e5c7a05..c97121a83fa08 100644
--- a/llvm/lib/Target/AArch64/AArch64TargetTransformInfo.cpp
+++ b/llvm/lib/Target/AArch64/AArch64TargetTransformInfo.cpp
@@ -6029,15 +6029,21 @@ InstructionCost AArch64TTIImpl::getPartialReductionCost(
// the extends in the IR are still counted. This can be fixed
// after https://github.com/llvm/llvm-project/pull/147302 has landed.
return Cost;
- // f16 -> f32 is natively supported for fdot
- if (Opcode == Instruction::FAdd && (ST->hasSME2() || ST->hasSVE2p1()) &&
- AccumLT.second.getScalarType() == MVT::f32 &&
- InputLT.second.getScalarType() == MVT::f16 &&
- AccumLT.second.getVectorMinNumElements() == 4 &&
- InputLT.second.getVectorMinNumElements() == 8)
+ // i8 -> i16 is natively supported with SVE2p3
+ if (AccumLT.second.getScalarType() == MVT::i16 &&
+ InputLT.second.getScalarType() == MVT::i8 &&
+ (ST->hasSVE2p3() || ST->hasSME2p3()))
return Cost;
}
+ // f16 -> f32 is natively supported for fdot using either
+ // SVE or NEON instruction.
+ if (Opcode == Instruction::FAdd && !IsSub &&
+ IsSupported(ST->hasSME2() || ST->hasSVE2p1(), ST->hasF16F32DOT()) &&
+ AccumLT.second.getScalarType() == MVT::f32 &&
+ InputLT.second.getScalarType() == MVT::f16)
+ return Cost;
+
// For a ratio of 2, we can use *mlal top/bottom instructions.
if (Ratio == 2 && !IsSub) {
MVT InVT = InputLT.second.getScalarType();
diff --git a/llvm/test/Transforms/LoopVectorize/AArch64/partial-reduce-add-sdot-i16-i32.ll b/llvm/test/Transforms/LoopVectorize/AArch64/partial-reduce-add-sdot-i16-i32.ll
index c67fccc107599..cabb401fabc72 100644
--- a/llvm/test/Transforms/LoopVectorize/AArch64/partial-reduce-add-sdot-i16-i32.ll
+++ b/llvm/test/Transforms/LoopVectorize/AArch64/partial-reduce-add-sdot-i16-i32.ll
@@ -9,6 +9,10 @@
; RUN: -disable-output < %s 2>&1 | FileCheck %s --check-prefix=CHECK-FIXED
; RUN: opt -passes=loop-vectorize -force-vector-interleave=1 -force-vector-width=8 \
; RUN: -enable-epilogue-vectorization=false -debug-only=loop-vectorize \
+; RUN: -mattr=+f16f32dot -scalable-vectorization=off \
+; RUN: -disable-output < %s 2>&1 | FileCheck %s --check-prefix=CHECK-FIXED-F16F32DOT
+; RUN: opt -passes=loop-vectorize -force-vector-interleave=1 -force-vector-width=8 \
+; RUN: -enable-epilogue-vectorization=false -debug-only=loop-vectorize \
; RUN: -mattr=+sve2p1 -scalable-vectorization=on \
; RUN: -disable-output < %s 2>&1 | FileCheck %s --check-prefix=CHECK-SCALABLE
; RUN: opt -passes=loop-vectorize -force-vector-interleave=1 -force-vector-width=8 \
@@ -29,6 +33,7 @@
; LV: Checking a loop in 'fpext_reduction_half_to_float'
; CHECK-FIXED-BASE: Cost of 4 for VF 8: EXPRESSION vp<%8> = ir<%acc> + partial.reduce.fadd (ir<%load> reassoc contract fpext to float)
; CHECK-FIXED: Cost of 1 for VF 8: EXPRESSION vp<%8> = ir<%acc> + partial.reduce.fadd (ir<%load> reassoc contract fpext to float)
+; CHECK-FIXED-F16F32DOT: Cost of 1 for VF 8: EXPRESSION vp<%8> = ir<%acc> + partial.reduce.fadd (ir<%load> reassoc contract fpext to float)
; CHECK-SCALABLE: Cost of 1 for VF vscale x 8: EXPRESSION vp<%8> = ir<%acc> + partial.reduce.fadd (ir<%load> reassoc contract fpext to float)
target triple = "aarch64"
diff --git a/llvm/test/Transforms/LoopVectorize/AArch64/partial-reduce-add-sdot-i8-i16.ll b/llvm/test/Transforms/LoopVectorize/AArch64/partial-reduce-add-sdot-i8-i16.ll
new file mode 100644
index 0000000000000..ca595f0574500
--- /dev/null
+++ b/llvm/test/Transforms/LoopVectorize/AArch64/partial-reduce-add-sdot-i8-i16.ll
@@ -0,0 +1,61 @@
+; REQUIRES: asserts
+; RUN: opt -passes=loop-vectorize -force-vector-interleave=1 -force-vector-width=16 \
+; RUN: -enable-epilogue-vectorization=false -debug-only=loop-vectorize \
+; RUN: -mattr=+sve -scalable-vectorization=on \
+; RUN: -disable-output < %s 2>&1 | FileCheck %s --check-prefix=CHECK-SCALABLE-BASE
+; RUN: opt -passes=loop-vectorize -force-vector-interleave=1 -force-vector-width=16 \
+; RUN: -enable-epilogue-vectorization=false -debug-only=loop-vectorize \
+; RUN: -mattr=+sve2p3 -scalable-vectorization=on \
+; RUN: -disable-output < %s 2>&1 | FileCheck %s --check-prefix=CHECK-SCALABLE
+; RUN: opt -passes=loop-vectorize -force-vector-interleave=1 -force-vector-width=16 \
+; RUN: -enable-epilogue-vectorization=false -debug-only=loop-vectorize \
+; RUN: -mattr=+sve,+sme2p3 -scalable-vectorization=on \
+; RUN: -disable-output < %s 2>&1 | FileCheck %s --check-prefix=CHECK-SCALABLE
+
+; LV: Checking a loop in 'sext_reduction_i8_to_i16'
+; CHECK-SCALABLE-BASE: Cost of 4 for VF vscale x 16: EXPRESSION vp<%8> = ir<%acc> + partial.reduce.add (ir<%load> sext to i16)
+; CHECK-SCALABLE: Cost of 1 for VF vscale x 16: EXPRESSION vp<%8> = ir<%acc> + partial.reduce.add (ir<%load> sext to i16)
+
+; LV: Checking a loop in 'zext_reduction_i8_to_i16'
+; CHECK-SCALABLE-BASE: Cost of 4 for VF vscale x 16: EXPRESSION vp<%8> = ir<%acc> + partial.reduce.add (ir<%load> zext to i16)
+; CHECK-SCALABLE: Cost of 1 for VF vscale x 16: EXPRESSION vp<%8> = ir<%acc> + partial.reduce.add (ir<%load> zext to i16)
+
+target triple = "aarch64"
+
+define i16 @sext_reduction_i8_to_i16(ptr %arr, i32 %n) vscale_range(1,16) {
+entry:
+ br label %loop
+
+loop:
+ %iv = phi i32 [ 0, %entry ], [ %iv.next, %loop ]
+ %acc = phi i16 [ 0, %entry ], [ %add, %loop ]
+ %gep = getelementptr inbounds i8, ptr %arr, i32 %iv
+ %load = load i8, ptr %gep
+ %sext = sext i8 %load to i16
+ %add = add i16 %acc, %sext
+ %iv.next = add i32 %iv, 1
+ %cmp = icmp ult i32 %iv.next, %n
+ br i1 %cmp, label %loop, label %exit
+
+exit:
+ ret i16 %add
+}
+
+define i16 @zext_reduction_i8_to_i16(ptr %arr, i32 %n) vscale_range(1,16) {
+entry:
+ br label %loop
+
+loop:
+ %iv = phi i32 [ 0, %entry ], [ %iv.next, %loop ]
+ %acc = phi i16 [ 0, %entry ], [ %add, %loop ]
+ %gep = getelementptr inbounds i8, ptr %arr, i32 %iv
+ %load = load i8, ptr %gep
+ %zext = zext i8 %load to i16
+ %add = add i16 %acc, %zext
+ %iv.next = add i32 %iv, 1
+ %cmp = icmp ult i32 %iv.next, %n
+ br i1 %cmp, label %loop, label %exit
+
+exit:
+ ret i16 %add
+}
More information about the llvm-commits
mailing list