[llvm] [AArch64] Match indexed forms of fmul/fmla/fmls (PR #144892)
Paul Walker via llvm-commits
llvm-commits at lists.llvm.org
Wed Jun 25 06:18:27 PDT 2025
================
@@ -0,0 +1,363 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py UTC_ARGS: --version 5
+; RUN: llc -mtriple=aarch64-linux-gnu < %s | FileCheck %s
+
+define void @fmul_indexed_f16_256b(ptr %a, ptr %b, ptr %c) #0 {
+; CHECK-LABEL: fmul_indexed_f16_256b:
+; CHECK: // %bb.0:
+; CHECK-NEXT: ldr z0, [x0]
+; CHECK-NEXT: fmul z0.h, z0.h, z0.h[2]
+; CHECK-NEXT: str z0, [x2]
+; CHECK-NEXT: ret
+ %ld.a = load <16 x half>, ptr %a
+ %ld.b = load <16 x half>, ptr %b
+ %splat.lanes = shufflevector <16 x half> %ld.a, <16 x half> poison, <16 x i32> <i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2,
+ i32 10, i32 10, i32 10, i32 10, i32 10, i32 10, i32 10, i32 10>
+ %res = fmul <16 x half> %ld.a, %splat.lanes
+ store <16 x half> %res, ptr %c
+ ret void
+}
+
+define void @fmul_indexed_bf16_256b(ptr %a, ptr %b, ptr %c) #0 {
+; CHECK-LABEL: fmul_indexed_bf16_256b:
+; CHECK: // %bb.0:
+; CHECK-NEXT: ldp q0, q1, [x1]
+; CHECK-NEXT: ldp q2, q3, [x0]
+; CHECK-NEXT: dup v0.8h, v0.h[2]
+; CHECK-NEXT: dup v1.8h, v1.h[2]
+; CHECK-NEXT: shll v4.4s, v2.4h, #16
+; CHECK-NEXT: shll v6.4s, v3.4h, #16
+; CHECK-NEXT: shll2 v2.4s, v2.8h, #16
+; CHECK-NEXT: shll2 v3.4s, v3.8h, #16
+; CHECK-NEXT: shll v5.4s, v0.4h, #16
+; CHECK-NEXT: shll v7.4s, v1.4h, #16
+; CHECK-NEXT: shll2 v0.4s, v0.8h, #16
+; CHECK-NEXT: shll2 v1.4s, v1.8h, #16
+; CHECK-NEXT: fmul v4.4s, v4.4s, v5.4s
+; CHECK-NEXT: fmul v5.4s, v6.4s, v7.4s
+; CHECK-NEXT: fmul v0.4s, v2.4s, v0.4s
+; CHECK-NEXT: fmul v1.4s, v3.4s, v1.4s
+; CHECK-NEXT: bfcvtn v2.4h, v4.4s
+; CHECK-NEXT: bfcvtn v3.4h, v5.4s
+; CHECK-NEXT: bfcvtn2 v2.8h, v0.4s
+; CHECK-NEXT: bfcvtn2 v3.8h, v1.4s
+; CHECK-NEXT: stp q2, q3, [x2]
+; CHECK-NEXT: ret
+ %ld.a = load <16 x bfloat>, ptr %a
+ %ld.b = load <16 x bfloat>, ptr %b
+ %splat.lanes = shufflevector <16 x bfloat> %ld.b, <16 x bfloat> poison, <16 x i32> <i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2,
+ i32 10, i32 10, i32 10, i32 10, i32 10, i32 10, i32 10, i32 10>
+ %res = fmul <16 x bfloat> %ld.a, %splat.lanes
+ store <16 x bfloat> %res, ptr %c
+ ret void
+}
+
+define void @fmul_indexed_f32_256b(ptr %a, ptr %b, ptr %c) #0 {
+; CHECK-LABEL: fmul_indexed_f32_256b:
+; CHECK: // %bb.0:
+; CHECK-NEXT: ldr z0, [x0]
+; CHECK-NEXT: ldr z1, [x1]
+; CHECK-NEXT: fmul z0.s, z0.s, z1.s[3]
+; CHECK-NEXT: str z0, [x2]
+; CHECK-NEXT: ret
+ %ld.a = load <8 x float>, ptr %a
+ %ld.b = load <8 x float>, ptr %b
+ %splat.lanes = shufflevector <8 x float> %ld.b, <8 x float> poison, <8 x i32> <i32 3, i32 3, i32 3, i32 3,
+ i32 7, i32 7, i32 7, i32 7>
+ %res = fmul <8 x float> %splat.lanes, %ld.a
+ store <8 x float> %res, ptr %c
+ ret void
+}
+
+define void @fmul_indexed_f64_256b_trn1(ptr %a, ptr %b, ptr %c) #0 {
+; CHECK-LABEL: fmul_indexed_f64_256b_trn1:
+; CHECK: // %bb.0:
+; CHECK-NEXT: ldr z0, [x0]
+; CHECK-NEXT: ldr z1, [x1]
+; CHECK-NEXT: fmul z0.d, z0.d, z1.d[0]
+; CHECK-NEXT: str z0, [x2]
+; CHECK-NEXT: ret
+ %ld.a = load <4 x double>, ptr %a
+ %ld.b = load <4 x double>, ptr %b
+ %splat.lanes = shufflevector <4 x double> %ld.b, <4 x double> poison, <4 x i32> <i32 0, i32 0, i32 2, i32 2>
+ %res = fmul <4 x double> %splat.lanes, %ld.a
+ store <4 x double> %res, ptr %c
+ ret void
+}
+
+define void @fmul_indexed_f64_256b_trn2(ptr %a, ptr %b, ptr %c) #0 {
+; CHECK-LABEL: fmul_indexed_f64_256b_trn2:
+; CHECK: // %bb.0:
+; CHECK-NEXT: ldr z0, [x0]
+; CHECK-NEXT: ldr z1, [x1]
+; CHECK-NEXT: fmul z0.d, z0.d, z1.d[1]
+; CHECK-NEXT: str z0, [x2]
+; CHECK-NEXT: ret
+ %ld.a = load <4 x double>, ptr %a
+ %ld.b = load <4 x double>, ptr %b
+ %splat.lanes = shufflevector <4 x double> %ld.b, <4 x double> poison, <4 x i32> <i32 1, i32 1, i32 3, i32 3>
+ %res = fmul <4 x double> %ld.a, %splat.lanes
+ store <4 x double> %res, ptr %c
+ ret void
+}
+
+define void @fmla_indexed_f16_256b(ptr %a, ptr %b, ptr %c) #0 {
+; CHECK-LABEL: fmla_indexed_f16_256b:
+; CHECK: // %bb.0:
+; CHECK-NEXT: ldr z0, [x0]
+; CHECK-NEXT: ldr z1, [x2]
+; CHECK-NEXT: fmla z1.h, z0.h, z0.h[2]
+; CHECK-NEXT: str z1, [x2]
+; CHECK-NEXT: ret
+ %ld.a = load <16 x half>, ptr %a
+ %ld.b = load <16 x half>, ptr %b
+ %ld.c = load <16 x half>, ptr %c
+ %splat.lanes = shufflevector <16 x half> %ld.a, <16 x half> poison, <16 x i32> <i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2,
+ i32 10, i32 10, i32 10, i32 10, i32 10, i32 10, i32 10, i32 10>
+ %res = call <16 x half> @llvm.fmuladd.v16f16(<16 x half> %ld.a, <16 x half> %splat.lanes, <16 x half> %ld.c)
+ store <16 x half> %res, ptr %c
+ ret void
+}
+
+define void @fmla_indexed_bf16_256b(ptr %a, ptr %b, ptr %c) #0 {
+; CHECK-LABEL: fmla_indexed_bf16_256b:
+; CHECK: // %bb.0:
+; CHECK-NEXT: ldp q0, q1, [x1]
+; CHECK-NEXT: ldp q2, q3, [x0]
+; CHECK-NEXT: dup v0.8h, v0.h[2]
+; CHECK-NEXT: dup v1.8h, v1.h[2]
+; CHECK-NEXT: shll v4.4s, v2.4h, #16
+; CHECK-NEXT: shll v6.4s, v3.4h, #16
+; CHECK-NEXT: shll2 v2.4s, v2.8h, #16
+; CHECK-NEXT: shll2 v3.4s, v3.8h, #16
+; CHECK-NEXT: shll v5.4s, v0.4h, #16
+; CHECK-NEXT: shll v7.4s, v1.4h, #16
+; CHECK-NEXT: shll2 v0.4s, v0.8h, #16
+; CHECK-NEXT: shll2 v1.4s, v1.8h, #16
+; CHECK-NEXT: fmul v4.4s, v4.4s, v5.4s
+; CHECK-NEXT: fmul v5.4s, v6.4s, v7.4s
+; CHECK-NEXT: fmul v0.4s, v2.4s, v0.4s
+; CHECK-NEXT: fmul v1.4s, v3.4s, v1.4s
+; CHECK-NEXT: bfcvtn v2.4h, v4.4s
+; CHECK-NEXT: bfcvtn v3.4h, v5.4s
+; CHECK-NEXT: bfcvtn2 v2.8h, v0.4s
+; CHECK-NEXT: bfcvtn2 v3.8h, v1.4s
+; CHECK-NEXT: ldp q0, q1, [x2]
+; CHECK-NEXT: shll v4.4s, v0.4h, #16
+; CHECK-NEXT: shll v5.4s, v2.4h, #16
+; CHECK-NEXT: shll v6.4s, v1.4h, #16
+; CHECK-NEXT: shll v7.4s, v3.4h, #16
+; CHECK-NEXT: shll2 v0.4s, v0.8h, #16
+; CHECK-NEXT: shll2 v2.4s, v2.8h, #16
+; CHECK-NEXT: shll2 v1.4s, v1.8h, #16
+; CHECK-NEXT: shll2 v3.4s, v3.8h, #16
+; CHECK-NEXT: fadd v4.4s, v5.4s, v4.4s
+; CHECK-NEXT: fadd v5.4s, v7.4s, v6.4s
+; CHECK-NEXT: fadd v0.4s, v2.4s, v0.4s
+; CHECK-NEXT: fadd v1.4s, v3.4s, v1.4s
+; CHECK-NEXT: bfcvtn v2.4h, v4.4s
+; CHECK-NEXT: bfcvtn v3.4h, v5.4s
+; CHECK-NEXT: bfcvtn2 v2.8h, v0.4s
+; CHECK-NEXT: bfcvtn2 v3.8h, v1.4s
+; CHECK-NEXT: stp q2, q3, [x2]
+; CHECK-NEXT: ret
+ %ld.a = load <16 x bfloat>, ptr %a
+ %ld.b = load <16 x bfloat>, ptr %b
+ %ld.c = load <16 x bfloat>, ptr %c
+ %splat.lanes = shufflevector <16 x bfloat> %ld.b, <16 x bfloat> poison, <16 x i32> <i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2,
+ i32 10, i32 10, i32 10, i32 10, i32 10, i32 10, i32 10, i32 10>
+ %res = call <16 x bfloat> @llvm.fmuladd.v16bf16(<16 x bfloat> %ld.a, <16 x bfloat> %splat.lanes, <16 x bfloat> %ld.c)
+ store <16 x bfloat> %res, ptr %c
+ ret void
+}
+
+define void @fmla_indexed_f32_256b(ptr %a, ptr %b, ptr %c) #0 {
+; CHECK-LABEL: fmla_indexed_f32_256b:
+; CHECK: // %bb.0:
+; CHECK-NEXT: ldr z0, [x0]
+; CHECK-NEXT: ldr z1, [x1]
+; CHECK-NEXT: ldr z2, [x2]
+; CHECK-NEXT: fmla z2.s, z0.s, z1.s[3]
+; CHECK-NEXT: str z2, [x2]
+; CHECK-NEXT: ret
+ %ld.a = load <8 x float>, ptr %a
+ %ld.b = load <8 x float>, ptr %b
+ %ld.c = load <8 x float>, ptr %c
+ %splat.lanes = shufflevector <8 x float> %ld.b, <8 x float> poison, <8 x i32> <i32 3, i32 3, i32 3, i32 3,
+ i32 7, i32 7, i32 7, i32 7>
+ %res = call <8 x float> @llvm.fmuladd.v8f32(<8 x float> %splat.lanes, <8 x float> %ld.a, <8 x float> %ld.c)
+ store <8 x float> %res, ptr %c
+ ret void
+}
+
+define void @fmla_indexed_f64_256b_trn1(ptr %a, ptr %b, ptr %c) #0 {
+; CHECK-LABEL: fmla_indexed_f64_256b_trn1:
+; CHECK: // %bb.0:
+; CHECK-NEXT: ldr z0, [x0]
+; CHECK-NEXT: ldr z1, [x1]
+; CHECK-NEXT: ldr z2, [x2]
+; CHECK-NEXT: fmla z2.d, z0.d, z1.d[0]
+; CHECK-NEXT: str z2, [x2]
+; CHECK-NEXT: ret
+ %ld.a = load <4 x double>, ptr %a
+ %ld.b = load <4 x double>, ptr %b
+ %ld.c = load <4 x double>, ptr %c
+ %splat.lanes = shufflevector <4 x double> %ld.b, <4 x double> poison, <4 x i32> <i32 0, i32 0, i32 2, i32 2>
+ %res = call <4 x double> @llvm.fmuladd.v4f64(<4 x double> %splat.lanes, <4 x double> %ld.a, <4 x double> %ld.c)
+ store <4 x double> %res, ptr %c
+ ret void
+}
+
+define void @fmla_indexed_f64_256b_trn2(ptr %a, ptr %b, ptr %c) #0 {
+; CHECK-LABEL: fmla_indexed_f64_256b_trn2:
+; CHECK: // %bb.0:
+; CHECK-NEXT: ldr z0, [x0]
+; CHECK-NEXT: ldr z1, [x1]
+; CHECK-NEXT: ldr z2, [x2]
+; CHECK-NEXT: fmla z2.d, z0.d, z1.d[1]
+; CHECK-NEXT: str z2, [x2]
+; CHECK-NEXT: ret
+ %ld.a = load <4 x double>, ptr %a
+ %ld.b = load <4 x double>, ptr %b
+ %ld.c = load <4 x double>, ptr %c
+ %splat.lanes = shufflevector <4 x double> %ld.b, <4 x double> poison, <4 x i32> <i32 1, i32 1, i32 3, i32 3>
+ %res = call <4 x double> @llvm.fmuladd.v4f64(<4 x double> %ld.a, <4 x double> %splat.lanes, <4 x double> %ld.c)
+ store <4 x double> %res, ptr %c
+ ret void
+}
+
+define void @fmls_indexed_f16_256b(ptr %a, ptr %b, ptr %c) #0 {
+; CHECK-LABEL: fmls_indexed_f16_256b:
+; CHECK: // %bb.0:
+; CHECK-NEXT: ldr z0, [x0]
+; CHECK-NEXT: ldr z1, [x2]
+; CHECK-NEXT: fmls z1.h, z0.h, z0.h[2]
+; CHECK-NEXT: str z1, [x2]
+; CHECK-NEXT: ret
+ %ld.a = load <16 x half>, ptr %a
+ %ld.b = load <16 x half>, ptr %b
+ %ld.c = load <16 x half>, ptr %c
+ %splat.lanes = shufflevector <16 x half> %ld.a, <16 x half> poison, <16 x i32> <i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2,
+ i32 10, i32 10, i32 10, i32 10, i32 10, i32 10, i32 10, i32 10>
+ %neg.a = fneg <16 x half> %ld.a
+ %res = call <16 x half> @llvm.fmuladd.v16f16(<16 x half> %neg.a, <16 x half> %splat.lanes, <16 x half> %ld.c)
+ store <16 x half> %res, ptr %c
+ ret void
+}
+
+define void @fmls_indexed_bf16_256b(ptr %a, ptr %b, ptr %c) #0 {
+; CHECK-LABEL: fmls_indexed_bf16_256b:
+; CHECK: // %bb.0:
+; CHECK-NEXT: ldp q0, q1, [x1]
+; CHECK-NEXT: ldp q2, q3, [x0]
+; CHECK-NEXT: dup v0.8h, v0.h[2]
+; CHECK-NEXT: dup v1.8h, v1.h[2]
+; CHECK-NEXT: shll v4.4s, v2.4h, #16
+; CHECK-NEXT: shll v6.4s, v3.4h, #16
+; CHECK-NEXT: shll2 v2.4s, v2.8h, #16
+; CHECK-NEXT: shll2 v3.4s, v3.8h, #16
+; CHECK-NEXT: shll v5.4s, v0.4h, #16
+; CHECK-NEXT: shll v7.4s, v1.4h, #16
+; CHECK-NEXT: shll2 v0.4s, v0.8h, #16
+; CHECK-NEXT: shll2 v1.4s, v1.8h, #16
+; CHECK-NEXT: fmul v4.4s, v4.4s, v5.4s
+; CHECK-NEXT: fmul v5.4s, v6.4s, v7.4s
+; CHECK-NEXT: fmul v0.4s, v2.4s, v0.4s
+; CHECK-NEXT: fmul v1.4s, v3.4s, v1.4s
+; CHECK-NEXT: bfcvtn v2.4h, v4.4s
+; CHECK-NEXT: bfcvtn v3.4h, v5.4s
+; CHECK-NEXT: bfcvtn2 v2.8h, v0.4s
+; CHECK-NEXT: bfcvtn2 v3.8h, v1.4s
+; CHECK-NEXT: ldp q0, q1, [x2]
+; CHECK-NEXT: shll v4.4s, v0.4h, #16
+; CHECK-NEXT: shll v5.4s, v2.4h, #16
+; CHECK-NEXT: shll v6.4s, v1.4h, #16
+; CHECK-NEXT: shll v7.4s, v3.4h, #16
+; CHECK-NEXT: shll2 v0.4s, v0.8h, #16
+; CHECK-NEXT: shll2 v2.4s, v2.8h, #16
+; CHECK-NEXT: shll2 v1.4s, v1.8h, #16
+; CHECK-NEXT: shll2 v3.4s, v3.8h, #16
+; CHECK-NEXT: fsub v4.4s, v4.4s, v5.4s
+; CHECK-NEXT: fsub v5.4s, v6.4s, v7.4s
+; CHECK-NEXT: fsub v0.4s, v0.4s, v2.4s
+; CHECK-NEXT: fsub v1.4s, v1.4s, v3.4s
+; CHECK-NEXT: bfcvtn v2.4h, v4.4s
+; CHECK-NEXT: bfcvtn v3.4h, v5.4s
+; CHECK-NEXT: bfcvtn2 v2.8h, v0.4s
+; CHECK-NEXT: bfcvtn2 v3.8h, v1.4s
+; CHECK-NEXT: stp q2, q3, [x2]
+; CHECK-NEXT: ret
+ %ld.a = load <16 x bfloat>, ptr %a
+ %ld.b = load <16 x bfloat>, ptr %b
+ %ld.c = load <16 x bfloat>, ptr %c
+ %splat.lanes = shufflevector <16 x bfloat> %ld.b, <16 x bfloat> poison, <16 x i32> <i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2,
+ i32 10, i32 10, i32 10, i32 10, i32 10, i32 10, i32 10, i32 10>
+ %neg.a = fneg <16 x bfloat> %ld.a
+ %res = call <16 x bfloat> @llvm.fmuladd.v16bf16(<16 x bfloat> %neg.a, <16 x bfloat> %splat.lanes, <16 x bfloat> %ld.c)
+ store <16 x bfloat> %res, ptr %c
+ ret void
+}
+
+define void @fmls_indexed_f32_256b(ptr %a, ptr %b, ptr %c) #0 {
+; CHECK-LABEL: fmls_indexed_f32_256b:
+; CHECK: // %bb.0:
+; CHECK-NEXT: ldr z0, [x0]
+; CHECK-NEXT: ldr z1, [x1]
+; CHECK-NEXT: ldr z2, [x2]
+; CHECK-NEXT: fmls z2.s, z0.s, z1.s[3]
+; CHECK-NEXT: str z2, [x2]
+; CHECK-NEXT: ret
+ %ld.a = load <8 x float>, ptr %a
+ %ld.b = load <8 x float>, ptr %b
+ %ld.c = load <8 x float>, ptr %c
+ %splat.lanes = shufflevector <8 x float> %ld.b, <8 x float> poison, <8 x i32> <i32 3, i32 3, i32 3, i32 3,
+ i32 7, i32 7, i32 7, i32 7>
+ %neg.a = fneg <8 x float> %ld.a
+ %res = call <8 x float> @llvm.fmuladd.v8f32(<8 x float> %splat.lanes, <8 x float> %neg.a, <8 x float> %ld.c)
+ store <8 x float> %res, ptr %c
+ ret void
+}
+
+define void @fmls_indexed_f64_256b_trn1(ptr %a, ptr %b, ptr %c) #0 {
+; CHECK-LABEL: fmls_indexed_f64_256b_trn1:
+; CHECK: // %bb.0:
+; CHECK-NEXT: ldr z0, [x0]
+; CHECK-NEXT: ldr z1, [x1]
+; CHECK-NEXT: ldr z2, [x2]
+; CHECK-NEXT: fmls z2.d, z0.d, z1.d[0]
+; CHECK-NEXT: str z2, [x2]
+; CHECK-NEXT: ret
+ %ld.a = load <4 x double>, ptr %a
+ %ld.b = load <4 x double>, ptr %b
+ %ld.c = load <4 x double>, ptr %c
+ %splat.lanes = shufflevector <4 x double> %ld.b, <4 x double> poison, <4 x i32> <i32 0, i32 0, i32 2, i32 2>
+ %neg.a = fneg <4 x double> %ld.a
+ %res = call <4 x double> @llvm.fmuladd.v4f64(<4 x double> %splat.lanes, <4 x double> %neg.a, <4 x double> %ld.c)
+ store <4 x double> %res, ptr %c
+ ret void
+}
+
+define void @fmls_indexed_f64_256b_trn2(ptr %a, ptr %b, ptr %c) #0 {
+; CHECK-LABEL: fmls_indexed_f64_256b_trn2:
+; CHECK: // %bb.0:
+; CHECK-NEXT: ldr z0, [x0]
+; CHECK-NEXT: ldr z1, [x1]
+; CHECK-NEXT: ldr z2, [x2]
+; CHECK-NEXT: fmls z2.d, z0.d, z1.d[1]
+; CHECK-NEXT: str z2, [x2]
+; CHECK-NEXT: ret
+ %ld.a = load <4 x double>, ptr %a
+ %ld.b = load <4 x double>, ptr %b
+ %ld.c = load <4 x double>, ptr %c
+ %splat.lanes = shufflevector <4 x double> %ld.b, <4 x double> poison, <4 x i32> <i32 1, i32 1, i32 3, i32 3>
+ %neg.a = fneg <4 x double> %ld.a
+ %res = call <4 x double> @llvm.fmuladd.v4f64(<4 x double> %neg.a, <4 x double> %splat.lanes, <4 x double> %ld.c)
+ store <4 x double> %res, ptr %c
+ ret void
+}
+
+declare <16 x half> @llvm.fmuladd.v16f16(<16 x half>, <16 x half>, <16 x half>);
+declare <16 x bfloat> @llvm.fmuladd.v16bf16(<16 x bfloat>, <16 x bfloat>, <16 x bfloat>);
+declare <8 x float> @llvm.fmuladd.v8f32(<8 x float>, <8 x float>, <8 x float>);
+declare <4 x double> @llvm.fmuladd.v4f64(<4 x double>, <4 x double>, <4 x double>);
+
+attributes #0 = { noinline vscale_range(2,2) "target-features"="+sve2p1,+bf16" }
----------------
paulwalker-arm wrote:
I think it's worth adding "+sve-b16b16" to the target features to simplify the CHECK lines.
https://github.com/llvm/llvm-project/pull/144892
More information about the llvm-commits
mailing list