[llvm] [X86] Add bf16 support to isFMAFasterThanFMulAndFAdd for basic FMA optimizations (PR #172006)
via llvm-commits
llvm-commits at lists.llvm.org
Fri Dec 12 05:09:48 PST 2025
https://github.com/azwolski created https://github.com/llvm/llvm-project/pull/172006
This PR extends `isFMAFasterThanFMulAndFAdd` in `X86ISelLowering` to handle
bfloat types. This enables basic FMA optimizations for bf16
operations on AVX10.2 targets.
Includes tests for scalar and vector bf16 cases:
- Scalar bf16 FMA lowering (AVX10.2 do not support scalar bf16 operations)
- Vector bf16 FMA fusion for 128-bit, 256-bit, and 512-bit widths
>From ab4683df4cf9812dcade476d0b4aee0a4e87f74b Mon Sep 17 00:00:00 2001
From: Antoni Zwolski <antoni.zwolski at intel.com>
Date: Thu, 11 Dec 2025 21:49:27 +0100
Subject: [PATCH 1/2] [X86] Add support for bf16 in isFMAFasterThanFMulAndFAdd
---
llvm/lib/Target/X86/X86ISelLowering.cpp | 2 ++
1 file changed, 2 insertions(+)
diff --git a/llvm/lib/Target/X86/X86ISelLowering.cpp b/llvm/lib/Target/X86/X86ISelLowering.cpp
index 133406bd8e0d7..894ad3248ebd4 100644
--- a/llvm/lib/Target/X86/X86ISelLowering.cpp
+++ b/llvm/lib/Target/X86/X86ISelLowering.cpp
@@ -35542,6 +35542,8 @@ bool X86TargetLowering::isFMAFasterThanFMulAndFAdd(const MachineFunction &MF,
switch (VT.getSimpleVT().SimpleTy) {
case MVT::f16:
return Subtarget.hasFP16();
+ case MVT::bf16:
+ return Subtarget.hasAVX10_2();
case MVT::f32:
case MVT::f64:
return true;
>From ca1ccbff4f0c649ed9ace2c3d9106be17b34c157 Mon Sep 17 00:00:00 2001
From: Antoni Zwolski <antoni.zwolski at intel.com>
Date: Thu, 11 Dec 2025 21:50:23 +0100
Subject: [PATCH 2/2] [X86] Add bf16 fma tests
---
llvm/test/CodeGen/X86/avx10_2bf16-fma.ll | 241 +++++++++++++++++++++++
1 file changed, 241 insertions(+)
create mode 100644 llvm/test/CodeGen/X86/avx10_2bf16-fma.ll
diff --git a/llvm/test/CodeGen/X86/avx10_2bf16-fma.ll b/llvm/test/CodeGen/X86/avx10_2bf16-fma.ll
new file mode 100644
index 0000000000000..22281fded1295
--- /dev/null
+++ b/llvm/test/CodeGen/X86/avx10_2bf16-fma.ll
@@ -0,0 +1,241 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py UTC_ARGS: --version 6
+; RUN: llc < %s -verify-machineinstrs -mtriple=x86_64-unknown-unknown -mattr=+avx10.2 | FileCheck %s --check-prefixes=AVX10_2
+; RUN: llc < %s -verify-machineinstrs -mtriple=x86_64-unknown-unknown -mattr=+avx512bf16,+avx512vl | FileCheck %s --check-prefixes=AVX512BF16
+
+define bfloat @fuse_bf16(bfloat %a, bfloat %b, bfloat %c) nounwind {
+; AVX10_2-LABEL: fuse_bf16:
+; AVX10_2: # %bb.0: # %entry
+; AVX10_2-NEXT: vmovw %xmm1, %eax
+; AVX10_2-NEXT: vmovw %xmm0, %ecx
+; AVX10_2-NEXT: vmovw %xmm2, %edx
+; AVX10_2-NEXT: shll $16, %edx
+; AVX10_2-NEXT: vmovd %edx, %xmm0
+; AVX10_2-NEXT: shll $16, %ecx
+; AVX10_2-NEXT: vmovd %ecx, %xmm1
+; AVX10_2-NEXT: shll $16, %eax
+; AVX10_2-NEXT: vmovd %eax, %xmm2
+; AVX10_2-NEXT: vfmadd213ss {{.*#+}} xmm2 = (xmm1 * xmm2) + xmm0
+; AVX10_2-NEXT: vcvtneps2bf16 %xmm2, %xmm0
+; AVX10_2-NEXT: retq
+;
+; AVX512BF16-LABEL: fuse_bf16:
+; AVX512BF16: # %bb.0: # %entry
+; AVX512BF16-NEXT: vpextrw $0, %xmm2, %eax
+; AVX512BF16-NEXT: vpextrw $0, %xmm0, %ecx
+; AVX512BF16-NEXT: vpextrw $0, %xmm1, %edx
+; AVX512BF16-NEXT: shll $16, %edx
+; AVX512BF16-NEXT: vmovd %edx, %xmm0
+; AVX512BF16-NEXT: shll $16, %ecx
+; AVX512BF16-NEXT: vmovd %ecx, %xmm1
+; AVX512BF16-NEXT: vmulss %xmm0, %xmm1, %xmm0
+; AVX512BF16-NEXT: vcvtneps2bf16 %xmm0, %xmm0
+; AVX512BF16-NEXT: vmovd %xmm0, %ecx
+; AVX512BF16-NEXT: shll $16, %ecx
+; AVX512BF16-NEXT: vmovd %ecx, %xmm0
+; AVX512BF16-NEXT: shll $16, %eax
+; AVX512BF16-NEXT: vmovd %eax, %xmm1
+; AVX512BF16-NEXT: vaddss %xmm1, %xmm0, %xmm0
+; AVX512BF16-NEXT: vcvtneps2bf16 %xmm0, %xmm0
+; AVX512BF16-NEXT: retq
+entry:
+ %m = fmul contract bfloat %a, %b
+ %r = fadd contract bfloat %m, %c
+ ret bfloat %r
+}
+
+define <8 x bfloat> @fuse_v8bf16(<8 x bfloat> %x, <8 x bfloat> %y, <8 x bfloat> %z) nounwind {
+; AVX10_2-LABEL: fuse_v8bf16:
+; AVX10_2: # %bb.0: # %entry
+; AVX10_2-NEXT: vfmadd213bf16 %xmm2, %xmm1, %xmm0
+; AVX10_2-NEXT: retq
+;
+; AVX512BF16-LABEL: fuse_v8bf16:
+; AVX512BF16: # %bb.0: # %entry
+; AVX512BF16-NEXT: vpmovzxwd {{.*#+}} ymm1 = xmm1[0],zero,xmm1[1],zero,xmm1[2],zero,xmm1[3],zero,xmm1[4],zero,xmm1[5],zero,xmm1[6],zero,xmm1[7],zero
+; AVX512BF16-NEXT: vpslld $16, %ymm1, %ymm1
+; AVX512BF16-NEXT: vpmovzxwd {{.*#+}} ymm0 = xmm0[0],zero,xmm0[1],zero,xmm0[2],zero,xmm0[3],zero,xmm0[4],zero,xmm0[5],zero,xmm0[6],zero,xmm0[7],zero
+; AVX512BF16-NEXT: vpslld $16, %ymm0, %ymm0
+; AVX512BF16-NEXT: vmulps %ymm1, %ymm0, %ymm0
+; AVX512BF16-NEXT: vcvtneps2bf16 %ymm0, %xmm0
+; AVX512BF16-NEXT: vpmovzxwd {{.*#+}} ymm0 = xmm0[0],zero,xmm0[1],zero,xmm0[2],zero,xmm0[3],zero,xmm0[4],zero,xmm0[5],zero,xmm0[6],zero,xmm0[7],zero
+; AVX512BF16-NEXT: vpslld $16, %ymm0, %ymm0
+; AVX512BF16-NEXT: vpmovzxwd {{.*#+}} ymm1 = xmm2[0],zero,xmm2[1],zero,xmm2[2],zero,xmm2[3],zero,xmm2[4],zero,xmm2[5],zero,xmm2[6],zero,xmm2[7],zero
+; AVX512BF16-NEXT: vpslld $16, %ymm1, %ymm1
+; AVX512BF16-NEXT: vaddps %ymm1, %ymm0, %ymm0
+; AVX512BF16-NEXT: vcvtneps2bf16 %ymm0, %xmm0
+; AVX512BF16-NEXT: vzeroupper
+; AVX512BF16-NEXT: retq
+entry:
+ %m = fmul contract <8 x bfloat> %x, %y
+ %r = fadd contract <8 x bfloat> %m, %z
+ ret <8 x bfloat> %r
+}
+
+define <16 x bfloat> @fuse_v16bf16(<16 x bfloat> %x, <16 x bfloat> %y, <16 x bfloat> %z) nounwind {
+; AVX10_2-LABEL: fuse_v16bf16:
+; AVX10_2: # %bb.0: # %entry
+; AVX10_2-NEXT: vfmadd213bf16 %ymm2, %ymm1, %ymm0
+; AVX10_2-NEXT: retq
+;
+; AVX512BF16-LABEL: fuse_v16bf16:
+; AVX512BF16: # %bb.0: # %entry
+; AVX512BF16-NEXT: vpmovzxwd {{.*#+}} zmm1 = ymm1[0],zero,ymm1[1],zero,ymm1[2],zero,ymm1[3],zero,ymm1[4],zero,ymm1[5],zero,ymm1[6],zero,ymm1[7],zero,ymm1[8],zero,ymm1[9],zero,ymm1[10],zero,ymm1[11],zero,ymm1[12],zero,ymm1[13],zero,ymm1[14],zero,ymm1[15],zero
+; AVX512BF16-NEXT: vpslld $16, %zmm1, %zmm1
+; AVX512BF16-NEXT: vpmovzxwd {{.*#+}} zmm0 = ymm0[0],zero,ymm0[1],zero,ymm0[2],zero,ymm0[3],zero,ymm0[4],zero,ymm0[5],zero,ymm0[6],zero,ymm0[7],zero,ymm0[8],zero,ymm0[9],zero,ymm0[10],zero,ymm0[11],zero,ymm0[12],zero,ymm0[13],zero,ymm0[14],zero,ymm0[15],zero
+; AVX512BF16-NEXT: vpslld $16, %zmm0, %zmm0
+; AVX512BF16-NEXT: vmulps %zmm1, %zmm0, %zmm0
+; AVX512BF16-NEXT: vcvtneps2bf16 %zmm0, %ymm0
+; AVX512BF16-NEXT: vpmovzxwd {{.*#+}} zmm0 = ymm0[0],zero,ymm0[1],zero,ymm0[2],zero,ymm0[3],zero,ymm0[4],zero,ymm0[5],zero,ymm0[6],zero,ymm0[7],zero,ymm0[8],zero,ymm0[9],zero,ymm0[10],zero,ymm0[11],zero,ymm0[12],zero,ymm0[13],zero,ymm0[14],zero,ymm0[15],zero
+; AVX512BF16-NEXT: vpslld $16, %zmm0, %zmm0
+; AVX512BF16-NEXT: vpmovzxwd {{.*#+}} zmm1 = ymm2[0],zero,ymm2[1],zero,ymm2[2],zero,ymm2[3],zero,ymm2[4],zero,ymm2[5],zero,ymm2[6],zero,ymm2[7],zero,ymm2[8],zero,ymm2[9],zero,ymm2[10],zero,ymm2[11],zero,ymm2[12],zero,ymm2[13],zero,ymm2[14],zero,ymm2[15],zero
+; AVX512BF16-NEXT: vpslld $16, %zmm1, %zmm1
+; AVX512BF16-NEXT: vaddps %zmm1, %zmm0, %zmm0
+; AVX512BF16-NEXT: vcvtneps2bf16 %zmm0, %ymm0
+; AVX512BF16-NEXT: retq
+entry:
+ %m = fmul contract <16 x bfloat> %x, %y
+ %r = fadd contract <16 x bfloat> %m, %z
+ ret <16 x bfloat> %r
+}
+
+define <32 x bfloat> @fuse_v32bf16(<32 x bfloat> %x, <32 x bfloat> %y, <32 x bfloat> %z) nounwind {
+; AVX10_2-LABEL: fuse_v32bf16:
+; AVX10_2: # %bb.0: # %entry
+; AVX10_2-NEXT: vfmadd213bf16 %zmm2, %zmm1, %zmm0
+; AVX10_2-NEXT: retq
+;
+; AVX512BF16-LABEL: fuse_v32bf16:
+; AVX512BF16: # %bb.0: # %entry
+; AVX512BF16-NEXT: vextracti64x4 $1, %zmm1, %ymm3
+; AVX512BF16-NEXT: vpmovzxwd {{.*#+}} zmm3 = ymm3[0],zero,ymm3[1],zero,ymm3[2],zero,ymm3[3],zero,ymm3[4],zero,ymm3[5],zero,ymm3[6],zero,ymm3[7],zero,ymm3[8],zero,ymm3[9],zero,ymm3[10],zero,ymm3[11],zero,ymm3[12],zero,ymm3[13],zero,ymm3[14],zero,ymm3[15],zero
+; AVX512BF16-NEXT: vpslld $16, %zmm3, %zmm3
+; AVX512BF16-NEXT: vextracti64x4 $1, %zmm0, %ymm4
+; AVX512BF16-NEXT: vpmovzxwd {{.*#+}} zmm4 = ymm4[0],zero,ymm4[1],zero,ymm4[2],zero,ymm4[3],zero,ymm4[4],zero,ymm4[5],zero,ymm4[6],zero,ymm4[7],zero,ymm4[8],zero,ymm4[9],zero,ymm4[10],zero,ymm4[11],zero,ymm4[12],zero,ymm4[13],zero,ymm4[14],zero,ymm4[15],zero
+; AVX512BF16-NEXT: vpslld $16, %zmm4, %zmm4
+; AVX512BF16-NEXT: vmulps %zmm3, %zmm4, %zmm3
+; AVX512BF16-NEXT: vcvtneps2bf16 %zmm3, %ymm3
+; AVX512BF16-NEXT: vpmovzxwd {{.*#+}} zmm1 = ymm1[0],zero,ymm1[1],zero,ymm1[2],zero,ymm1[3],zero,ymm1[4],zero,ymm1[5],zero,ymm1[6],zero,ymm1[7],zero,ymm1[8],zero,ymm1[9],zero,ymm1[10],zero,ymm1[11],zero,ymm1[12],zero,ymm1[13],zero,ymm1[14],zero,ymm1[15],zero
+; AVX512BF16-NEXT: vpslld $16, %zmm1, %zmm1
+; AVX512BF16-NEXT: vpmovzxwd {{.*#+}} zmm0 = ymm0[0],zero,ymm0[1],zero,ymm0[2],zero,ymm0[3],zero,ymm0[4],zero,ymm0[5],zero,ymm0[6],zero,ymm0[7],zero,ymm0[8],zero,ymm0[9],zero,ymm0[10],zero,ymm0[11],zero,ymm0[12],zero,ymm0[13],zero,ymm0[14],zero,ymm0[15],zero
+; AVX512BF16-NEXT: vpslld $16, %zmm0, %zmm0
+; AVX512BF16-NEXT: vmulps %zmm1, %zmm0, %zmm0
+; AVX512BF16-NEXT: vcvtneps2bf16 %zmm0, %ymm0
+; AVX512BF16-NEXT: vpmovzxwd {{.*#+}} zmm0 = ymm0[0],zero,ymm0[1],zero,ymm0[2],zero,ymm0[3],zero,ymm0[4],zero,ymm0[5],zero,ymm0[6],zero,ymm0[7],zero,ymm0[8],zero,ymm0[9],zero,ymm0[10],zero,ymm0[11],zero,ymm0[12],zero,ymm0[13],zero,ymm0[14],zero,ymm0[15],zero
+; AVX512BF16-NEXT: vpslld $16, %zmm0, %zmm0
+; AVX512BF16-NEXT: vpmovzxwd {{.*#+}} zmm1 = ymm2[0],zero,ymm2[1],zero,ymm2[2],zero,ymm2[3],zero,ymm2[4],zero,ymm2[5],zero,ymm2[6],zero,ymm2[7],zero,ymm2[8],zero,ymm2[9],zero,ymm2[10],zero,ymm2[11],zero,ymm2[12],zero,ymm2[13],zero,ymm2[14],zero,ymm2[15],zero
+; AVX512BF16-NEXT: vpslld $16, %zmm1, %zmm1
+; AVX512BF16-NEXT: vaddps %zmm1, %zmm0, %zmm0
+; AVX512BF16-NEXT: vcvtneps2bf16 %zmm0, %ymm0
+; AVX512BF16-NEXT: vpmovzxwd {{.*#+}} zmm1 = ymm3[0],zero,ymm3[1],zero,ymm3[2],zero,ymm3[3],zero,ymm3[4],zero,ymm3[5],zero,ymm3[6],zero,ymm3[7],zero,ymm3[8],zero,ymm3[9],zero,ymm3[10],zero,ymm3[11],zero,ymm3[12],zero,ymm3[13],zero,ymm3[14],zero,ymm3[15],zero
+; AVX512BF16-NEXT: vpslld $16, %zmm1, %zmm1
+; AVX512BF16-NEXT: vextracti64x4 $1, %zmm2, %ymm2
+; AVX512BF16-NEXT: vpmovzxwd {{.*#+}} zmm2 = ymm2[0],zero,ymm2[1],zero,ymm2[2],zero,ymm2[3],zero,ymm2[4],zero,ymm2[5],zero,ymm2[6],zero,ymm2[7],zero,ymm2[8],zero,ymm2[9],zero,ymm2[10],zero,ymm2[11],zero,ymm2[12],zero,ymm2[13],zero,ymm2[14],zero,ymm2[15],zero
+; AVX512BF16-NEXT: vpslld $16, %zmm2, %zmm2
+; AVX512BF16-NEXT: vaddps %zmm2, %zmm1, %zmm1
+; AVX512BF16-NEXT: vcvtneps2bf16 %zmm1, %ymm1
+; AVX512BF16-NEXT: vinsertf64x4 $1, %ymm1, %zmm0, %zmm0
+; AVX512BF16-NEXT: retq
+entry:
+ %m = fmul contract <32 x bfloat> %x, %y
+ %r = fadd contract <32 x bfloat> %m, %z
+ ret <32 x bfloat> %r
+}
+
+define <5 x bfloat> @fuse_v5bf16(<5 x bfloat> %x, <5 x bfloat> %y, <5 x bfloat> %z) nounwind {
+; AVX10_2-LABEL: fuse_v5bf16:
+; AVX10_2: # %bb.0: # %entry
+; AVX10_2-NEXT: vfmadd213bf16 %xmm2, %xmm1, %xmm0
+; AVX10_2-NEXT: retq
+;
+; AVX512BF16-LABEL: fuse_v5bf16:
+; AVX512BF16: # %bb.0: # %entry
+; AVX512BF16-NEXT: vpmovzxwd {{.*#+}} ymm1 = xmm1[0],zero,xmm1[1],zero,xmm1[2],zero,xmm1[3],zero,xmm1[4],zero,xmm1[5],zero,xmm1[6],zero,xmm1[7],zero
+; AVX512BF16-NEXT: vpslld $16, %ymm1, %ymm1
+; AVX512BF16-NEXT: vpmovzxwd {{.*#+}} ymm0 = xmm0[0],zero,xmm0[1],zero,xmm0[2],zero,xmm0[3],zero,xmm0[4],zero,xmm0[5],zero,xmm0[6],zero,xmm0[7],zero
+; AVX512BF16-NEXT: vpslld $16, %ymm0, %ymm0
+; AVX512BF16-NEXT: vmulps %ymm1, %ymm0, %ymm0
+; AVX512BF16-NEXT: vcvtneps2bf16 %ymm0, %xmm0
+; AVX512BF16-NEXT: vpmovzxwd {{.*#+}} ymm0 = xmm0[0],zero,xmm0[1],zero,xmm0[2],zero,xmm0[3],zero,xmm0[4],zero,xmm0[5],zero,xmm0[6],zero,xmm0[7],zero
+; AVX512BF16-NEXT: vpslld $16, %ymm0, %ymm0
+; AVX512BF16-NEXT: vpmovzxwd {{.*#+}} ymm1 = xmm2[0],zero,xmm2[1],zero,xmm2[2],zero,xmm2[3],zero,xmm2[4],zero,xmm2[5],zero,xmm2[6],zero,xmm2[7],zero
+; AVX512BF16-NEXT: vpslld $16, %ymm1, %ymm1
+; AVX512BF16-NEXT: vaddps %ymm1, %ymm0, %ymm0
+; AVX512BF16-NEXT: vcvtneps2bf16 %ymm0, %xmm0
+; AVX512BF16-NEXT: vzeroupper
+; AVX512BF16-NEXT: retq
+entry:
+ %m = fmul contract <5 x bfloat> %x, %y
+ %r = fadd contract <5 x bfloat> %m, %z
+ ret <5 x bfloat> %r
+}
+
+define <9 x bfloat> @fnmadd_v9bf16(<9 x bfloat> %x, <9 x bfloat> %y, <9 x bfloat> %z) nounwind {
+; AVX10_2-LABEL: fnmadd_v9bf16:
+; AVX10_2: # %bb.0: # %entry
+; AVX10_2-NEXT: vfnmadd213bf16 %ymm2, %ymm1, %ymm0
+; AVX10_2-NEXT: retq
+;
+; AVX512BF16-LABEL: fnmadd_v9bf16:
+; AVX512BF16: # %bb.0: # %entry
+; AVX512BF16-NEXT: vpmovzxwd {{.*#+}} zmm1 = ymm1[0],zero,ymm1[1],zero,ymm1[2],zero,ymm1[3],zero,ymm1[4],zero,ymm1[5],zero,ymm1[6],zero,ymm1[7],zero,ymm1[8],zero,ymm1[9],zero,ymm1[10],zero,ymm1[11],zero,ymm1[12],zero,ymm1[13],zero,ymm1[14],zero,ymm1[15],zero
+; AVX512BF16-NEXT: vpslld $16, %zmm1, %zmm1
+; AVX512BF16-NEXT: vpmovzxwd {{.*#+}} zmm0 = ymm0[0],zero,ymm0[1],zero,ymm0[2],zero,ymm0[3],zero,ymm0[4],zero,ymm0[5],zero,ymm0[6],zero,ymm0[7],zero,ymm0[8],zero,ymm0[9],zero,ymm0[10],zero,ymm0[11],zero,ymm0[12],zero,ymm0[13],zero,ymm0[14],zero,ymm0[15],zero
+; AVX512BF16-NEXT: vpslld $16, %zmm0, %zmm0
+; AVX512BF16-NEXT: vmulps %zmm1, %zmm0, %zmm0
+; AVX512BF16-NEXT: vcvtneps2bf16 %zmm0, %ymm0
+; AVX512BF16-NEXT: vpmovzxwd {{.*#+}} zmm0 = ymm0[0],zero,ymm0[1],zero,ymm0[2],zero,ymm0[3],zero,ymm0[4],zero,ymm0[5],zero,ymm0[6],zero,ymm0[7],zero,ymm0[8],zero,ymm0[9],zero,ymm0[10],zero,ymm0[11],zero,ymm0[12],zero,ymm0[13],zero,ymm0[14],zero,ymm0[15],zero
+; AVX512BF16-NEXT: vpslld $16, %zmm0, %zmm0
+; AVX512BF16-NEXT: vpmovzxwd {{.*#+}} zmm1 = ymm2[0],zero,ymm2[1],zero,ymm2[2],zero,ymm2[3],zero,ymm2[4],zero,ymm2[5],zero,ymm2[6],zero,ymm2[7],zero,ymm2[8],zero,ymm2[9],zero,ymm2[10],zero,ymm2[11],zero,ymm2[12],zero,ymm2[13],zero,ymm2[14],zero,ymm2[15],zero
+; AVX512BF16-NEXT: vpslld $16, %zmm1, %zmm1
+; AVX512BF16-NEXT: vsubps %zmm0, %zmm1, %zmm0
+; AVX512BF16-NEXT: vcvtneps2bf16 %zmm0, %ymm0
+; AVX512BF16-NEXT: retq
+entry:
+ %m = fmul contract <9 x bfloat> %x, %y
+ %n = fneg <9 x bfloat> %m
+ %r = fadd contract <9 x bfloat> %n, %z
+ ret <9 x bfloat> %r
+}
+
+define <29 x bfloat> @fuse_v19bf16_load(<29 x bfloat> %x, <29 x bfloat> %y, ptr %p) nounwind {
+; AVX10_2-LABEL: fuse_v19bf16_load:
+; AVX10_2: # %bb.0: # %entry
+; AVX10_2-NEXT: vfmadd213bf16 (%rdi), %zmm1, %zmm0
+; AVX10_2-NEXT: retq
+;
+; AVX512BF16-LABEL: fuse_v19bf16_load:
+; AVX512BF16: # %bb.0: # %entry
+; AVX512BF16-NEXT: vextracti64x4 $1, %zmm1, %ymm2
+; AVX512BF16-NEXT: vpmovzxwd {{.*#+}} zmm2 = ymm2[0],zero,ymm2[1],zero,ymm2[2],zero,ymm2[3],zero,ymm2[4],zero,ymm2[5],zero,ymm2[6],zero,ymm2[7],zero,ymm2[8],zero,ymm2[9],zero,ymm2[10],zero,ymm2[11],zero,ymm2[12],zero,ymm2[13],zero,ymm2[14],zero,ymm2[15],zero
+; AVX512BF16-NEXT: vpslld $16, %zmm2, %zmm2
+; AVX512BF16-NEXT: vextracti64x4 $1, %zmm0, %ymm3
+; AVX512BF16-NEXT: vpmovzxwd {{.*#+}} zmm3 = ymm3[0],zero,ymm3[1],zero,ymm3[2],zero,ymm3[3],zero,ymm3[4],zero,ymm3[5],zero,ymm3[6],zero,ymm3[7],zero,ymm3[8],zero,ymm3[9],zero,ymm3[10],zero,ymm3[11],zero,ymm3[12],zero,ymm3[13],zero,ymm3[14],zero,ymm3[15],zero
+; AVX512BF16-NEXT: vpslld $16, %zmm3, %zmm3
+; AVX512BF16-NEXT: vmulps %zmm2, %zmm3, %zmm2
+; AVX512BF16-NEXT: vcvtneps2bf16 %zmm2, %ymm2
+; AVX512BF16-NEXT: vpmovzxwd {{.*#+}} zmm1 = ymm1[0],zero,ymm1[1],zero,ymm1[2],zero,ymm1[3],zero,ymm1[4],zero,ymm1[5],zero,ymm1[6],zero,ymm1[7],zero,ymm1[8],zero,ymm1[9],zero,ymm1[10],zero,ymm1[11],zero,ymm1[12],zero,ymm1[13],zero,ymm1[14],zero,ymm1[15],zero
+; AVX512BF16-NEXT: vpslld $16, %zmm1, %zmm1
+; AVX512BF16-NEXT: vpmovzxwd {{.*#+}} zmm0 = ymm0[0],zero,ymm0[1],zero,ymm0[2],zero,ymm0[3],zero,ymm0[4],zero,ymm0[5],zero,ymm0[6],zero,ymm0[7],zero,ymm0[8],zero,ymm0[9],zero,ymm0[10],zero,ymm0[11],zero,ymm0[12],zero,ymm0[13],zero,ymm0[14],zero,ymm0[15],zero
+; AVX512BF16-NEXT: vpslld $16, %zmm0, %zmm0
+; AVX512BF16-NEXT: vmulps %zmm1, %zmm0, %zmm0
+; AVX512BF16-NEXT: vcvtneps2bf16 %zmm0, %ymm0
+; AVX512BF16-NEXT: vpmovzxwd {{.*#+}} zmm0 = ymm0[0],zero,ymm0[1],zero,ymm0[2],zero,ymm0[3],zero,ymm0[4],zero,ymm0[5],zero,ymm0[6],zero,ymm0[7],zero,ymm0[8],zero,ymm0[9],zero,ymm0[10],zero,ymm0[11],zero,ymm0[12],zero,ymm0[13],zero,ymm0[14],zero,ymm0[15],zero
+; AVX512BF16-NEXT: vpslld $16, %zmm0, %zmm0
+; AVX512BF16-NEXT: vpmovzxwd {{.*#+}} zmm1 = mem[0],zero,mem[1],zero,mem[2],zero,mem[3],zero,mem[4],zero,mem[5],zero,mem[6],zero,mem[7],zero,mem[8],zero,mem[9],zero,mem[10],zero,mem[11],zero,mem[12],zero,mem[13],zero,mem[14],zero,mem[15],zero
+; AVX512BF16-NEXT: vpslld $16, %zmm1, %zmm1
+; AVX512BF16-NEXT: vaddps %zmm1, %zmm0, %zmm0
+; AVX512BF16-NEXT: vcvtneps2bf16 %zmm0, %ymm0
+; AVX512BF16-NEXT: vpmovzxwd {{.*#+}} zmm1 = ymm2[0],zero,ymm2[1],zero,ymm2[2],zero,ymm2[3],zero,ymm2[4],zero,ymm2[5],zero,ymm2[6],zero,ymm2[7],zero,ymm2[8],zero,ymm2[9],zero,ymm2[10],zero,ymm2[11],zero,ymm2[12],zero,ymm2[13],zero,ymm2[14],zero,ymm2[15],zero
+; AVX512BF16-NEXT: vpslld $16, %zmm1, %zmm1
+; AVX512BF16-NEXT: vpmovzxwd {{.*#+}} zmm2 = mem[0],zero,mem[1],zero,mem[2],zero,mem[3],zero,mem[4],zero,mem[5],zero,mem[6],zero,mem[7],zero,mem[8],zero,mem[9],zero,mem[10],zero,mem[11],zero,mem[12],zero,mem[13],zero,mem[14],zero,mem[15],zero
+; AVX512BF16-NEXT: vpslld $16, %zmm2, %zmm2
+; AVX512BF16-NEXT: vaddps %zmm2, %zmm1, %zmm1
+; AVX512BF16-NEXT: vcvtneps2bf16 %zmm1, %ymm1
+; AVX512BF16-NEXT: vinsertf64x4 $1, %ymm1, %zmm0, %zmm0
+; AVX512BF16-NEXT: retq
+entry:
+ %z = load <29 x bfloat>, ptr %p
+ %m = fmul contract <29 x bfloat> %x, %y
+ %r = fadd contract <29 x bfloat> %m, %z
+ ret <29 x bfloat> %r
+}
More information about the llvm-commits
mailing list