[llvm] [AArch64][CostModel] Lower wide fixed-length vector reductions cost (PR #211824)
Sander de Smalen via llvm-commits
llvm-commits at lists.llvm.org
Tue Jul 28 07:19:19 PDT 2026
=?utf-8?q?Gaëtan?= Bossu <gaetan.bossu at arm.com>,
=?utf-8?q?Gaëtan?= Bossu <gaetan.bossu at arm.com>,
=?utf-8?q?Gaëtan?= Bossu <gaetan.bossu at arm.com>,
=?utf-8?q?Gaëtan?= Bossu <gaetan.bossu at arm.com>,
=?utf-8?q?Gaëtan?= Bossu <gaetan.bossu at arm.com>
Message-ID:
In-Reply-To: <llvm.org/llvm/llvm-project/pull/211824 at github.com>
================
@@ -0,0 +1,119 @@
+; NOTE: Assertions have been autogenerated by utils/update_analyze_test_checks.py
+; RUN: opt < %s -mtriple=aarch64 -passes="print<cost-model>" 2>&1 -disable-output \
+; RUN: -cost-kind=all -aarch64-sve-vector-bits-min=128 \
+; RUN: | FileCheck %s --check-prefixes=VSCALE-ANY,VSCALE-1
+; RUN: opt < %s -mtriple=aarch64 -passes="print<cost-model>" 2>&1 -disable-output \
+; RUN: -cost-kind=all -aarch64-sve-vector-bits-min=128 -aarch64-sve-vector-bits-max=128 \
+; RUN: | FileCheck %s --check-prefixes=VSCALE-ANY,VSCALE-1
+; RUN: opt < %s -mtriple=aarch64 -passes="print<cost-model>" 2>&1 -disable-output \
+; RUN: -cost-kind=all -aarch64-sve-vector-bits-min=256 \
+; RUN: | FileCheck %s --check-prefixes=VSCALE-ANY,VSCALE-FROM-2,VSCALE-2
+; RUN: opt < %s -mtriple=aarch64 -passes="print<cost-model>" 2>&1 -disable-output \
+; RUN: -cost-kind=all -aarch64-sve-vector-bits-min=256 -aarch64-sve-vector-bits-max=256 \
+; RUN: | FileCheck %s --check-prefixes=VSCALE-ANY,VSCALE-FROM-2,VSCALE-2
+; RUN: opt < %s -mtriple=aarch64 -passes="print<cost-model>" 2>&1 -disable-output \
+; RUN: -cost-kind=all -aarch64-sve-vector-bits-min=512 \
+; RUN: | FileCheck %s --check-prefixes=VSCALE-ANY,VSCALE-FROM-2,VSCALE-FROM-4,VSCALE-4
+; RUN: opt < %s -mtriple=aarch64 -passes="print<cost-model>" 2>&1 -disable-output \
+; RUN: -cost-kind=all -aarch64-sve-vector-bits-min=512 -aarch64-sve-vector-bits-max=512 \
+; RUN: | FileCheck %s --check-prefixes=VSCALE-ANY,VSCALE-FROM-2,VSCALE-FROM-4,VSCALE-4
+
+; This tests the cost of fixed-length bfloat reductions for SVE, either for a
+; minimum vscale or a fixed vscale (aka VLS). Those two cases have the same costs
+; because the predicate setup cost is not taken into account.
+
+; Given that SVE has no bfaddv instruction, support for bf16 relies on promotion to fp32.
+; Native bf16 support through +sve-b16b16 can still help, especially for adding
+; lo/hi halves using bfadd. This feature is tested with the #1 attribute set.
+
+define void @reduce_fadd_128b_types() #0 {
+; VSCALE-1-LABEL: 'reduce_fadd_128b_types'
+; VSCALE-1-NEXT: Cost Model: Found costs of RThru:54 CodeSize:23 Lat:46 SizeLat:30 for: %fadd_v8bf16_strict = call bfloat @llvm.vector.reduce.fadd.v8bf16(bfloat 0.000000e+00, <8 x bfloat> poison)
+; VSCALE-1-NEXT: Cost Model: Found costs of RThru:48 CodeSize:27 Lat:33 SizeLat:27 for: %fadd_v8bf16_fast = call fast bfloat @llvm.vector.reduce.fadd.v8bf16(bfloat 0.000000e+00, <8 x bfloat> poison)
+; VSCALE-1-NEXT: Cost Model: Found costs of RThru:0 CodeSize:1 Lat:1 SizeLat:1 for: ret void
+;
+; VSCALE-FROM-2-LABEL: 'reduce_fadd_128b_types'
+; VSCALE-FROM-2-NEXT: Cost Model: Found costs of RThru:54 CodeSize:23 Lat:46 SizeLat:30 for: %fadd_v8bf16_strict = call bfloat @llvm.vector.reduce.fadd.v8bf16(bfloat 0.000000e+00, <8 x bfloat> poison)
+; VSCALE-FROM-2-NEXT: Cost Model: Found costs of RThru:39 CodeSize:27 Lat:33 SizeLat:27 for: %fadd_v8bf16_fast = call fast bfloat @llvm.vector.reduce.fadd.v8bf16(bfloat 0.000000e+00, <8 x bfloat> poison)
+; VSCALE-FROM-2-NEXT: Cost Model: Found costs of RThru:0 CodeSize:1 Lat:1 SizeLat:1 for: ret void
+;
+ %fadd_v8bf16_strict = call bfloat @llvm.vector.reduce.fadd.v8bf16(bfloat 0.0, <8 x bfloat> poison)
+ %fadd_v8bf16_fast = call fast bfloat @llvm.vector.reduce.fadd.v8bf16(bfloat 0.0, <8 x bfloat> poison)
+ ret void
+}
+
+define void @reduce_fadd_128b_types_b16b16() #1 {
+; VSCALE-ANY-LABEL: 'reduce_fadd_128b_types_b16b16'
+; VSCALE-ANY-NEXT: Cost Model: Found costs of RThru:30 CodeSize:23 Lat:46 SizeLat:30 for: %fadd_v8bf16_strict = call bfloat @llvm.vector.reduce.fadd.v8bf16(bfloat 0.000000e+00, <8 x bfloat> poison)
+; VSCALE-ANY-NEXT: Cost Model: Found costs of RThru:27 CodeSize:27 Lat:33 SizeLat:27 for: %fadd_v8bf16_fast = call fast bfloat @llvm.vector.reduce.fadd.v8bf16(bfloat 0.000000e+00, <8 x bfloat> poison)
+; VSCALE-ANY-NEXT: Cost Model: Found costs of RThru:0 CodeSize:1 Lat:1 SizeLat:1 for: ret void
+;
+ %fadd_v8bf16_strict = call bfloat @llvm.vector.reduce.fadd.v8bf16(bfloat 0.0, <8 x bfloat> poison)
+ %fadd_v8bf16_fast = call fast bfloat @llvm.vector.reduce.fadd.v8bf16(bfloat 0.0, <8 x bfloat> poison)
+ ret void
+}
+
+define void @reduce_fadd_256b_types() #0 {
+; VSCALE-1-LABEL: 'reduce_fadd_256b_types'
+; VSCALE-1-NEXT: Cost Model: Found costs of RThru:108 CodeSize:46 Lat:92 SizeLat:60 for: %fadd_v16bf16_strict = call bfloat @llvm.vector.reduce.fadd.v16bf16(bfloat 0.000000e+00, <16 x bfloat> poison)
+; VSCALE-1-NEXT: Cost Model: Found costs of RThru:72 CodeSize:44 Lat:52 SizeLat:44 for: %fadd_v16bf16_fast = call fast bfloat @llvm.vector.reduce.fadd.v16bf16(bfloat 0.000000e+00, <16 x bfloat> poison)
+; VSCALE-1-NEXT: Cost Model: Found costs of RThru:0 CodeSize:1 Lat:1 SizeLat:1 for: ret void
+;
+; VSCALE-FROM-2-LABEL: 'reduce_fadd_256b_types'
+; VSCALE-FROM-2-NEXT: Cost Model: Found costs of RThru:108 CodeSize:46 Lat:92 SizeLat:60 for: %fadd_v16bf16_strict = call bfloat @llvm.vector.reduce.fadd.v16bf16(bfloat 0.000000e+00, <16 x bfloat> poison)
+; VSCALE-FROM-2-NEXT: Cost Model: Found costs of RThru:60 CodeSize:44 Lat:52 SizeLat:44 for: %fadd_v16bf16_fast = call fast bfloat @llvm.vector.reduce.fadd.v16bf16(bfloat 0.000000e+00, <16 x bfloat> poison)
+; VSCALE-FROM-2-NEXT: Cost Model: Found costs of RThru:0 CodeSize:1 Lat:1 SizeLat:1 for: ret void
+;
+ %fadd_v16bf16_strict = call bfloat @llvm.vector.reduce.fadd.v16bf16(bfloat 0.0, <16 x bfloat> poison)
+ %fadd_v16bf16_fast = call fast bfloat @llvm.vector.reduce.fadd.v16bf16(bfloat 0.0, <16 x bfloat> poison)
+ ret void
+}
+
+define void @reduce_fadd_256b_types_b16b16() #1 {
+; VSCALE-ANY-LABEL: 'reduce_fadd_256b_types_b16b16'
+; VSCALE-ANY-NEXT: Cost Model: Found costs of RThru:60 CodeSize:46 Lat:92 SizeLat:60 for: %fadd_v16bf16_strict = call bfloat @llvm.vector.reduce.fadd.v16bf16(bfloat 0.000000e+00, <16 x bfloat> poison)
+; VSCALE-ANY-NEXT: Cost Model: Found costs of RThru:44 CodeSize:44 Lat:52 SizeLat:44 for: %fadd_v16bf16_fast = call fast bfloat @llvm.vector.reduce.fadd.v16bf16(bfloat 0.000000e+00, <16 x bfloat> poison)
+; VSCALE-ANY-NEXT: Cost Model: Found costs of RThru:0 CodeSize:1 Lat:1 SizeLat:1 for: ret void
+;
+ %fadd_v16bf16_strict = call bfloat @llvm.vector.reduce.fadd.v16bf16(bfloat 0.0, <16 x bfloat> poison)
+ %fadd_v16bf16_fast = call fast bfloat @llvm.vector.reduce.fadd.v16bf16(bfloat 0.0, <16 x bfloat> poison)
+ ret void
+}
+
+define void @reduce_fadd_512b_types() #0 {
+; VSCALE-1-LABEL: 'reduce_fadd_512b_types'
+; VSCALE-1-NEXT: Cost Model: Found costs of RThru:216 CodeSize:92 Lat:184 SizeLat:120 for: %fadd_v32bf16_strict = call bfloat @llvm.vector.reduce.fadd.v32bf16(bfloat 0.000000e+00, <32 x bfloat> poison)
+; VSCALE-1-NEXT: Cost Model: Found costs of RThru:120 CodeSize:77 Lat:87 SizeLat:77 for: %fadd_v32bf16_fast = call fast bfloat @llvm.vector.reduce.fadd.v32bf16(bfloat 0.000000e+00, <32 x bfloat> poison)
+; VSCALE-1-NEXT: Cost Model: Found costs of RThru:0 CodeSize:1 Lat:1 SizeLat:1 for: ret void
+;
+; VSCALE-2-LABEL: 'reduce_fadd_512b_types'
+; VSCALE-2-NEXT: Cost Model: Found costs of RThru:216 CodeSize:92 Lat:184 SizeLat:120 for: %fadd_v32bf16_strict = call bfloat @llvm.vector.reduce.fadd.v32bf16(bfloat 0.000000e+00, <32 x bfloat> poison)
+; VSCALE-2-NEXT: Cost Model: Found costs of RThru:100 CodeSize:77 Lat:87 SizeLat:77 for: %fadd_v32bf16_fast = call fast bfloat @llvm.vector.reduce.fadd.v32bf16(bfloat 0.000000e+00, <32 x bfloat> poison)
+; VSCALE-2-NEXT: Cost Model: Found costs of RThru:0 CodeSize:1 Lat:1 SizeLat:1 for: ret void
+;
+; VSCALE-FROM-4-LABEL: 'reduce_fadd_512b_types'
+; VSCALE-FROM-4-NEXT: Cost Model: Found costs of RThru:216 CodeSize:92 Lat:184 SizeLat:120 for: %fadd_v32bf16_strict = call bfloat @llvm.vector.reduce.fadd.v32bf16(bfloat 0.000000e+00, <32 x bfloat> poison)
+; VSCALE-FROM-4-NEXT: Cost Model: Found costs of RThru:96 CodeSize:77 Lat:87 SizeLat:77 for: %fadd_v32bf16_fast = call fast bfloat @llvm.vector.reduce.fadd.v32bf16(bfloat 0.000000e+00, <32 x bfloat> poison)
+; VSCALE-FROM-4-NEXT: Cost Model: Found costs of RThru:0 CodeSize:1 Lat:1 SizeLat:1 for: ret void
+;
+ %fadd_v32bf16_strict = call bfloat @llvm.vector.reduce.fadd.v32bf16(bfloat 0.0, <32 x bfloat> poison)
+ %fadd_v32bf16_fast = call fast bfloat @llvm.vector.reduce.fadd.v32bf16(bfloat 0.0, <32 x bfloat> poison)
+ ret void
+}
+
+define void @reduce_fadd_512b_types_b16b16() #1 {
+; VSCALE-ANY-LABEL: 'reduce_fadd_512b_types_b16b16'
+; VSCALE-ANY-NEXT: Cost Model: Found costs of RThru:120 CodeSize:92 Lat:184 SizeLat:120 for: %fadd_v32bf16_strict = call bfloat @llvm.vector.reduce.fadd.v32bf16(bfloat 0.000000e+00, <32 x bfloat> poison)
+; VSCALE-ANY-NEXT: Cost Model: Found costs of RThru:78 CodeSize:77 Lat:87 SizeLat:77 for: %fadd_v32bf16_fast = call fast bfloat @llvm.vector.reduce.fadd.v32bf16(bfloat 0.000000e+00, <32 x bfloat> poison)
+; VSCALE-ANY-NEXT: Cost Model: Found costs of RThru:0 CodeSize:1 Lat:1 SizeLat:1 for: ret void
+;
+ %fadd_v32bf16_strict = call bfloat @llvm.vector.reduce.fadd.v32bf16(bfloat 0.0, <32 x bfloat> poison)
+ %fadd_v32bf16_fast = call fast bfloat @llvm.vector.reduce.fadd.v32bf16(bfloat 0.0, <32 x bfloat> poison)
+ ret void
+}
+
+attributes #0 = { "target-features"="+sve,+bf16" }
+attributes #1 = { "target-features"="+sve,+bf16,+sve-b16b16" }
+
+;; NOTE: These prefixes are unused and the list is autogenerated. Do not add tests below this line:
+; VSCALE-4: {{.*}}
----------------
sdesmalen-arm wrote:
nit: you can remove VSCALE-4 from the RUN lines to avoid this message as it will never be used.
https://github.com/llvm/llvm-project/pull/211824
More information about the llvm-commits
mailing list