[llvm] [X86] Lower bf16 round-to-integer on AVX10.2 via vrndscalebf16 (PR #213276)
via llvm-commits
llvm-commits at lists.llvm.org
Fri Jul 31 06:23:22 PDT 2026
https://github.com/tfzee created https://github.com/llvm/llvm-project/pull/213276
Follow-up to #212245.
floor/ceil/trunc/rint/nearbyint/roundeven on bf16 are currently treated as illegal causing them to be
promoted and causing scalarization of the vector cases.
AVX10.2 has packed vrndscalebf16, so mark these operations Legal for v8bf16/v16bf16/v32bf16. The existing patterns already turns them into X86ISD::VRNDSCALE with the matching immediate, so no new patterns are needed.
Scalar bf16 reuses the path previously added in #212245.
llvm.round is left out since it is round-half-away-from-zero, which vrndscale cannot encode.
For the strict version I kept the the old legalization for now since the avx10.2 Instructions have hardcoded rounding and exception behavior.
(Further todo from my testing some of the strict versions need also some extra handling since they currently will fail to select and crash the compiler)
>From 92bd9b0d6f0a64b8509574e187754bcb62aa306e Mon Sep 17 00:00:00 2001
From: "Ziegler, Tim" <tim.ziegler at intel.com>
Date: Fri, 31 Jul 2026 14:57:44 +0200
Subject: [PATCH] Legalizing the common (non strict) rounding bf16 instructions
to use vrndscalebf16
---
llvm/lib/Target/X86/X86ISelLowering.cpp | 24 +-
llvm/test/CodeGen/X86/avx10_2bf16-rounding.ll | 1145 +++++++++++++++++
2 files changed, 1164 insertions(+), 5 deletions(-)
create mode 100644 llvm/test/CodeGen/X86/avx10_2bf16-rounding.ll
diff --git a/llvm/lib/Target/X86/X86ISelLowering.cpp b/llvm/lib/Target/X86/X86ISelLowering.cpp
index 712db68314e6e..a11152b7db66e 100644
--- a/llvm/lib/Target/X86/X86ISelLowering.cpp
+++ b/llvm/lib/Target/X86/X86ISelLowering.cpp
@@ -2610,6 +2610,10 @@ X86TargetLowering::X86TargetLowering(const X86TargetMachine &TM,
setOperationAction(ISD::FSQRT, MVT::bf16, Custom);
setOperationAction(ISD::FMA, MVT::bf16, Custom);
+ for (unsigned Opc : {ISD::FFLOOR, ISD::FCEIL, ISD::FTRUNC, ISD::FRINT,
+ ISD::FNEARBYINT, ISD::FROUNDEVEN})
+ setOperationAction(Opc, MVT::bf16, Custom);
+
setOperationAction(ISD::FADD, MVT::v32bf16, Legal);
setOperationAction(ISD::FSUB, MVT::v32bf16, Legal);
setOperationAction(ISD::FMUL, MVT::v32bf16, Legal);
@@ -2618,6 +2622,10 @@ X86TargetLowering::X86TargetLowering(const X86TargetMachine &TM,
setOperationAction(ISD::FMA, MVT::v32bf16, Legal);
setOperationAction(ISD::SETCC, MVT::v32bf16, Custom);
SetFPMinMaxAction(MVT::v32bf16);
+ for (auto VT : {MVT::v8bf16, MVT::v16bf16, MVT::v32bf16})
+ for (unsigned Opc : {ISD::FFLOOR, ISD::FCEIL, ISD::FTRUNC, ISD::FRINT,
+ ISD::FNEARBYINT, ISD::FROUNDEVEN})
+ setOperationAction(Opc, VT, Legal);
for (auto VT : {MVT::v8bf16, MVT::v16bf16}) {
setOperationAction(ISD::FADD, VT, Legal);
setOperationAction(ISD::FSUB, VT, Legal);
@@ -34549,12 +34557,18 @@ void X86TargetLowering::ReplaceNodeResults(SDNode *N,
case ISD::FMUL:
case ISD::FSQRT:
case ISD::FDIV:
- case ISD::FMA: {
+ case ISD::FMA:
+ case ISD::FFLOOR:
+ case ISD::FCEIL:
+ case ISD::FTRUNC:
+ case ISD::FRINT:
+ case ISD::FNEARBYINT:
+ case ISD::FROUNDEVEN: {
assert(N->getValueType(0) == MVT::bf16 && "Expected scalar bf16 result");
- // AVX10.2 has no scalar bf16 arithmetic instructions, and bf16 is a
- // soft-promoted-half type, so scalar ops would otherwise be promoted to
- // f32. Instead widen each operand to a v8bf16 vector, perform the legal
- // packed operation, and extract the low element afterwards.
+ // AVX10.2 has no scalar bf16 arithmetic or round-to-integer instructions,
+ // and bf16 is a soft-promoted-half type, so scalar ops would otherwise be
+ // promoted to f32. Instead widen each operand to a v8bf16 vector, perform
+ // the legal packed operation, and extract the low element afterwards.
SmallVector<SDValue, 3> VecOps;
for (const SDValue &Op : N->ops()) {
SDValue AsF16 = DAG.getBitcast(MVT::f16, Op);
diff --git a/llvm/test/CodeGen/X86/avx10_2bf16-rounding.ll b/llvm/test/CodeGen/X86/avx10_2bf16-rounding.ll
new file mode 100644
index 0000000000000..4b9b764a6f5ce
--- /dev/null
+++ b/llvm/test/CodeGen/X86/avx10_2bf16-rounding.ll
@@ -0,0 +1,1145 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
+; RUN: llc < %s -verify-machineinstrs -mtriple=x86_64-unknown-unknown -mattr=+avx10.2 | FileCheck %s --check-prefixes=AVX10_2
+; RUN: llc < %s -verify-machineinstrs -mtriple=x86_64-unknown-unknown -mattr=+avx512bf16,+avx512vl | FileCheck %s --check-prefixes=AVX512BF16
+
+; AVX10.2 has packed bf16 round-to-integer instructions. The integral result of
+; floor/ceil/trunc/rint/nearbyint/roundeven is always representable in bf16, so
+; vrndscalebf16 gives the same answer as the AVX512BF16 sequence below (promote
+; each element to f32, round, convert back), but without scalarizing.
+
+define <8 x bfloat> @floor_v8bf16(<8 x bfloat> %a) {
+; AVX10_2-LABEL: floor_v8bf16:
+; AVX10_2: # %bb.0:
+; AVX10_2-NEXT: vrndscalebf16 $9, %xmm0, %xmm0
+; AVX10_2-NEXT: retq
+;
+; AVX512BF16-LABEL: floor_v8bf16:
+; AVX512BF16: # %bb.0:
+; AVX512BF16-NEXT: vpextrw $7, %xmm0, %eax
+; AVX512BF16-NEXT: shll $16, %eax
+; AVX512BF16-NEXT: vmovd %eax, %xmm1
+; AVX512BF16-NEXT: vroundss $9, %xmm1, %xmm1, %xmm1
+; AVX512BF16-NEXT: vcvtneps2bf16 %xmm1, %xmm1
+; AVX512BF16-NEXT: vmovd %xmm1, %eax
+; AVX512BF16-NEXT: vpextrw $6, %xmm0, %ecx
+; AVX512BF16-NEXT: shll $16, %ecx
+; AVX512BF16-NEXT: vmovd %ecx, %xmm1
+; AVX512BF16-NEXT: vroundss $9, %xmm1, %xmm1, %xmm1
+; AVX512BF16-NEXT: vcvtneps2bf16 %xmm1, %xmm1
+; AVX512BF16-NEXT: vmovd %xmm1, %ecx
+; AVX512BF16-NEXT: vpextrw $5, %xmm0, %edx
+; AVX512BF16-NEXT: shll $16, %edx
+; AVX512BF16-NEXT: vmovd %edx, %xmm1
+; AVX512BF16-NEXT: vroundss $9, %xmm1, %xmm1, %xmm1
+; AVX512BF16-NEXT: vcvtneps2bf16 %xmm1, %xmm1
+; AVX512BF16-NEXT: vmovd %xmm1, %edx
+; AVX512BF16-NEXT: vpextrw $4, %xmm0, %esi
+; AVX512BF16-NEXT: shll $16, %esi
+; AVX512BF16-NEXT: vmovd %esi, %xmm1
+; AVX512BF16-NEXT: vroundss $9, %xmm1, %xmm1, %xmm1
+; AVX512BF16-NEXT: vcvtneps2bf16 %xmm1, %xmm1
+; AVX512BF16-NEXT: vmovd %xmm1, %esi
+; AVX512BF16-NEXT: vpextrw $3, %xmm0, %edi
+; AVX512BF16-NEXT: shll $16, %edi
+; AVX512BF16-NEXT: vmovd %edi, %xmm1
+; AVX512BF16-NEXT: vroundss $9, %xmm1, %xmm1, %xmm1
+; AVX512BF16-NEXT: vcvtneps2bf16 %xmm1, %xmm1
+; AVX512BF16-NEXT: vmovd %xmm1, %edi
+; AVX512BF16-NEXT: vpextrw $2, %xmm0, %r8d
+; AVX512BF16-NEXT: shll $16, %r8d
+; AVX512BF16-NEXT: vmovd %r8d, %xmm1
+; AVX512BF16-NEXT: vroundss $9, %xmm1, %xmm1, %xmm1
+; AVX512BF16-NEXT: vcvtneps2bf16 %xmm1, %xmm1
+; AVX512BF16-NEXT: vmovd %xmm1, %r8d
+; AVX512BF16-NEXT: vpextrw $1, %xmm0, %r9d
+; AVX512BF16-NEXT: shll $16, %r9d
+; AVX512BF16-NEXT: vmovd %r9d, %xmm1
+; AVX512BF16-NEXT: vroundss $9, %xmm1, %xmm1, %xmm1
+; AVX512BF16-NEXT: vcvtneps2bf16 %xmm1, %xmm1
+; AVX512BF16-NEXT: vmovd %xmm0, %r9d
+; AVX512BF16-NEXT: shll $16, %r9d
+; AVX512BF16-NEXT: vmovd %r9d, %xmm0
+; AVX512BF16-NEXT: vroundss $9, %xmm0, %xmm0, %xmm0
+; AVX512BF16-NEXT: vcvtneps2bf16 %xmm0, %xmm0
+; AVX512BF16-NEXT: vpunpcklwd {{.*#+}} xmm0 = xmm0[0],xmm1[0],xmm0[1],xmm1[1],xmm0[2],xmm1[2],xmm0[3],xmm1[3]
+; AVX512BF16-NEXT: vpinsrw $2, %r8d, %xmm0, %xmm0
+; AVX512BF16-NEXT: vpinsrw $3, %edi, %xmm0, %xmm0
+; AVX512BF16-NEXT: vpinsrw $4, %esi, %xmm0, %xmm0
+; AVX512BF16-NEXT: vpinsrw $5, %edx, %xmm0, %xmm0
+; AVX512BF16-NEXT: vpinsrw $6, %ecx, %xmm0, %xmm0
+; AVX512BF16-NEXT: vpinsrw $7, %eax, %xmm0, %xmm0
+; AVX512BF16-NEXT: retq
+ %r = call <8 x bfloat> @llvm.floor.v8bf16(<8 x bfloat> %a)
+ ret <8 x bfloat> %r
+}
+
+define bfloat @floor_bf16(bfloat %a) {
+; AVX10_2-LABEL: floor_bf16:
+; AVX10_2: # %bb.0:
+; AVX10_2-NEXT: vrndscalebf16 $9, %xmm0, %xmm0
+; AVX10_2-NEXT: retq
+;
+; AVX512BF16-LABEL: floor_bf16:
+; AVX512BF16: # %bb.0:
+; AVX512BF16-NEXT: vpextrw $0, %xmm0, %eax
+; AVX512BF16-NEXT: shll $16, %eax
+; AVX512BF16-NEXT: vmovd %eax, %xmm0
+; AVX512BF16-NEXT: vroundss $9, %xmm0, %xmm0, %xmm0
+; AVX512BF16-NEXT: vcvtneps2bf16 %xmm0, %xmm0
+; AVX512BF16-NEXT: retq
+ %r = call bfloat @llvm.floor.bf16(bfloat %a)
+ ret bfloat %r
+}
+
+define <8 x bfloat> @ceil_v8bf16(<8 x bfloat> %a) {
+; AVX10_2-LABEL: ceil_v8bf16:
+; AVX10_2: # %bb.0:
+; AVX10_2-NEXT: vrndscalebf16 $10, %xmm0, %xmm0
+; AVX10_2-NEXT: retq
+;
+; AVX512BF16-LABEL: ceil_v8bf16:
+; AVX512BF16: # %bb.0:
+; AVX512BF16-NEXT: vpextrw $7, %xmm0, %eax
+; AVX512BF16-NEXT: shll $16, %eax
+; AVX512BF16-NEXT: vmovd %eax, %xmm1
+; AVX512BF16-NEXT: vroundss $10, %xmm1, %xmm1, %xmm1
+; AVX512BF16-NEXT: vcvtneps2bf16 %xmm1, %xmm1
+; AVX512BF16-NEXT: vmovd %xmm1, %eax
+; AVX512BF16-NEXT: vpextrw $6, %xmm0, %ecx
+; AVX512BF16-NEXT: shll $16, %ecx
+; AVX512BF16-NEXT: vmovd %ecx, %xmm1
+; AVX512BF16-NEXT: vroundss $10, %xmm1, %xmm1, %xmm1
+; AVX512BF16-NEXT: vcvtneps2bf16 %xmm1, %xmm1
+; AVX512BF16-NEXT: vmovd %xmm1, %ecx
+; AVX512BF16-NEXT: vpextrw $5, %xmm0, %edx
+; AVX512BF16-NEXT: shll $16, %edx
+; AVX512BF16-NEXT: vmovd %edx, %xmm1
+; AVX512BF16-NEXT: vroundss $10, %xmm1, %xmm1, %xmm1
+; AVX512BF16-NEXT: vcvtneps2bf16 %xmm1, %xmm1
+; AVX512BF16-NEXT: vmovd %xmm1, %edx
+; AVX512BF16-NEXT: vpextrw $4, %xmm0, %esi
+; AVX512BF16-NEXT: shll $16, %esi
+; AVX512BF16-NEXT: vmovd %esi, %xmm1
+; AVX512BF16-NEXT: vroundss $10, %xmm1, %xmm1, %xmm1
+; AVX512BF16-NEXT: vcvtneps2bf16 %xmm1, %xmm1
+; AVX512BF16-NEXT: vmovd %xmm1, %esi
+; AVX512BF16-NEXT: vpextrw $3, %xmm0, %edi
+; AVX512BF16-NEXT: shll $16, %edi
+; AVX512BF16-NEXT: vmovd %edi, %xmm1
+; AVX512BF16-NEXT: vroundss $10, %xmm1, %xmm1, %xmm1
+; AVX512BF16-NEXT: vcvtneps2bf16 %xmm1, %xmm1
+; AVX512BF16-NEXT: vmovd %xmm1, %edi
+; AVX512BF16-NEXT: vpextrw $2, %xmm0, %r8d
+; AVX512BF16-NEXT: shll $16, %r8d
+; AVX512BF16-NEXT: vmovd %r8d, %xmm1
+; AVX512BF16-NEXT: vroundss $10, %xmm1, %xmm1, %xmm1
+; AVX512BF16-NEXT: vcvtneps2bf16 %xmm1, %xmm1
+; AVX512BF16-NEXT: vmovd %xmm1, %r8d
+; AVX512BF16-NEXT: vpextrw $1, %xmm0, %r9d
+; AVX512BF16-NEXT: shll $16, %r9d
+; AVX512BF16-NEXT: vmovd %r9d, %xmm1
+; AVX512BF16-NEXT: vroundss $10, %xmm1, %xmm1, %xmm1
+; AVX512BF16-NEXT: vcvtneps2bf16 %xmm1, %xmm1
+; AVX512BF16-NEXT: vmovd %xmm0, %r9d
+; AVX512BF16-NEXT: shll $16, %r9d
+; AVX512BF16-NEXT: vmovd %r9d, %xmm0
+; AVX512BF16-NEXT: vroundss $10, %xmm0, %xmm0, %xmm0
+; AVX512BF16-NEXT: vcvtneps2bf16 %xmm0, %xmm0
+; AVX512BF16-NEXT: vpunpcklwd {{.*#+}} xmm0 = xmm0[0],xmm1[0],xmm0[1],xmm1[1],xmm0[2],xmm1[2],xmm0[3],xmm1[3]
+; AVX512BF16-NEXT: vpinsrw $2, %r8d, %xmm0, %xmm0
+; AVX512BF16-NEXT: vpinsrw $3, %edi, %xmm0, %xmm0
+; AVX512BF16-NEXT: vpinsrw $4, %esi, %xmm0, %xmm0
+; AVX512BF16-NEXT: vpinsrw $5, %edx, %xmm0, %xmm0
+; AVX512BF16-NEXT: vpinsrw $6, %ecx, %xmm0, %xmm0
+; AVX512BF16-NEXT: vpinsrw $7, %eax, %xmm0, %xmm0
+; AVX512BF16-NEXT: retq
+ %r = call <8 x bfloat> @llvm.ceil.v8bf16(<8 x bfloat> %a)
+ ret <8 x bfloat> %r
+}
+
+define bfloat @ceil_bf16(bfloat %a) {
+; AVX10_2-LABEL: ceil_bf16:
+; AVX10_2: # %bb.0:
+; AVX10_2-NEXT: vrndscalebf16 $10, %xmm0, %xmm0
+; AVX10_2-NEXT: retq
+;
+; AVX512BF16-LABEL: ceil_bf16:
+; AVX512BF16: # %bb.0:
+; AVX512BF16-NEXT: vpextrw $0, %xmm0, %eax
+; AVX512BF16-NEXT: shll $16, %eax
+; AVX512BF16-NEXT: vmovd %eax, %xmm0
+; AVX512BF16-NEXT: vroundss $10, %xmm0, %xmm0, %xmm0
+; AVX512BF16-NEXT: vcvtneps2bf16 %xmm0, %xmm0
+; AVX512BF16-NEXT: retq
+ %r = call bfloat @llvm.ceil.bf16(bfloat %a)
+ ret bfloat %r
+}
+
+define <8 x bfloat> @trunc_v8bf16(<8 x bfloat> %a) {
+; AVX10_2-LABEL: trunc_v8bf16:
+; AVX10_2: # %bb.0:
+; AVX10_2-NEXT: vrndscalebf16 $11, %xmm0, %xmm0
+; AVX10_2-NEXT: retq
+;
+; AVX512BF16-LABEL: trunc_v8bf16:
+; AVX512BF16: # %bb.0:
+; AVX512BF16-NEXT: vpextrw $7, %xmm0, %eax
+; AVX512BF16-NEXT: shll $16, %eax
+; AVX512BF16-NEXT: vmovd %eax, %xmm1
+; AVX512BF16-NEXT: vroundss $11, %xmm1, %xmm1, %xmm1
+; AVX512BF16-NEXT: vcvtneps2bf16 %xmm1, %xmm1
+; AVX512BF16-NEXT: vmovd %xmm1, %eax
+; AVX512BF16-NEXT: vpextrw $6, %xmm0, %ecx
+; AVX512BF16-NEXT: shll $16, %ecx
+; AVX512BF16-NEXT: vmovd %ecx, %xmm1
+; AVX512BF16-NEXT: vroundss $11, %xmm1, %xmm1, %xmm1
+; AVX512BF16-NEXT: vcvtneps2bf16 %xmm1, %xmm1
+; AVX512BF16-NEXT: vmovd %xmm1, %ecx
+; AVX512BF16-NEXT: vpextrw $5, %xmm0, %edx
+; AVX512BF16-NEXT: shll $16, %edx
+; AVX512BF16-NEXT: vmovd %edx, %xmm1
+; AVX512BF16-NEXT: vroundss $11, %xmm1, %xmm1, %xmm1
+; AVX512BF16-NEXT: vcvtneps2bf16 %xmm1, %xmm1
+; AVX512BF16-NEXT: vmovd %xmm1, %edx
+; AVX512BF16-NEXT: vpextrw $4, %xmm0, %esi
+; AVX512BF16-NEXT: shll $16, %esi
+; AVX512BF16-NEXT: vmovd %esi, %xmm1
+; AVX512BF16-NEXT: vroundss $11, %xmm1, %xmm1, %xmm1
+; AVX512BF16-NEXT: vcvtneps2bf16 %xmm1, %xmm1
+; AVX512BF16-NEXT: vmovd %xmm1, %esi
+; AVX512BF16-NEXT: vpextrw $3, %xmm0, %edi
+; AVX512BF16-NEXT: shll $16, %edi
+; AVX512BF16-NEXT: vmovd %edi, %xmm1
+; AVX512BF16-NEXT: vroundss $11, %xmm1, %xmm1, %xmm1
+; AVX512BF16-NEXT: vcvtneps2bf16 %xmm1, %xmm1
+; AVX512BF16-NEXT: vmovd %xmm1, %edi
+; AVX512BF16-NEXT: vpextrw $2, %xmm0, %r8d
+; AVX512BF16-NEXT: shll $16, %r8d
+; AVX512BF16-NEXT: vmovd %r8d, %xmm1
+; AVX512BF16-NEXT: vroundss $11, %xmm1, %xmm1, %xmm1
+; AVX512BF16-NEXT: vcvtneps2bf16 %xmm1, %xmm1
+; AVX512BF16-NEXT: vmovd %xmm1, %r8d
+; AVX512BF16-NEXT: vpextrw $1, %xmm0, %r9d
+; AVX512BF16-NEXT: shll $16, %r9d
+; AVX512BF16-NEXT: vmovd %r9d, %xmm1
+; AVX512BF16-NEXT: vroundss $11, %xmm1, %xmm1, %xmm1
+; AVX512BF16-NEXT: vcvtneps2bf16 %xmm1, %xmm1
+; AVX512BF16-NEXT: vmovd %xmm0, %r9d
+; AVX512BF16-NEXT: shll $16, %r9d
+; AVX512BF16-NEXT: vmovd %r9d, %xmm0
+; AVX512BF16-NEXT: vroundss $11, %xmm0, %xmm0, %xmm0
+; AVX512BF16-NEXT: vcvtneps2bf16 %xmm0, %xmm0
+; AVX512BF16-NEXT: vpunpcklwd {{.*#+}} xmm0 = xmm0[0],xmm1[0],xmm0[1],xmm1[1],xmm0[2],xmm1[2],xmm0[3],xmm1[3]
+; AVX512BF16-NEXT: vpinsrw $2, %r8d, %xmm0, %xmm0
+; AVX512BF16-NEXT: vpinsrw $3, %edi, %xmm0, %xmm0
+; AVX512BF16-NEXT: vpinsrw $4, %esi, %xmm0, %xmm0
+; AVX512BF16-NEXT: vpinsrw $5, %edx, %xmm0, %xmm0
+; AVX512BF16-NEXT: vpinsrw $6, %ecx, %xmm0, %xmm0
+; AVX512BF16-NEXT: vpinsrw $7, %eax, %xmm0, %xmm0
+; AVX512BF16-NEXT: retq
+ %r = call <8 x bfloat> @llvm.trunc.v8bf16(<8 x bfloat> %a)
+ ret <8 x bfloat> %r
+}
+
+define bfloat @trunc_bf16(bfloat %a) {
+; AVX10_2-LABEL: trunc_bf16:
+; AVX10_2: # %bb.0:
+; AVX10_2-NEXT: vrndscalebf16 $11, %xmm0, %xmm0
+; AVX10_2-NEXT: retq
+;
+; AVX512BF16-LABEL: trunc_bf16:
+; AVX512BF16: # %bb.0:
+; AVX512BF16-NEXT: vpextrw $0, %xmm0, %eax
+; AVX512BF16-NEXT: shll $16, %eax
+; AVX512BF16-NEXT: vmovd %eax, %xmm0
+; AVX512BF16-NEXT: vroundss $11, %xmm0, %xmm0, %xmm0
+; AVX512BF16-NEXT: vcvtneps2bf16 %xmm0, %xmm0
+; AVX512BF16-NEXT: retq
+ %r = call bfloat @llvm.trunc.bf16(bfloat %a)
+ ret bfloat %r
+}
+
+define <8 x bfloat> @rint_v8bf16(<8 x bfloat> %a) {
+; AVX10_2-LABEL: rint_v8bf16:
+; AVX10_2: # %bb.0:
+; AVX10_2-NEXT: vrndscalebf16 $4, %xmm0, %xmm0
+; AVX10_2-NEXT: retq
+;
+; AVX512BF16-LABEL: rint_v8bf16:
+; AVX512BF16: # %bb.0:
+; AVX512BF16-NEXT: vpextrw $7, %xmm0, %eax
+; AVX512BF16-NEXT: shll $16, %eax
+; AVX512BF16-NEXT: vmovd %eax, %xmm1
+; AVX512BF16-NEXT: vroundss $4, %xmm1, %xmm1, %xmm1
+; AVX512BF16-NEXT: vcvtneps2bf16 %xmm1, %xmm1
+; AVX512BF16-NEXT: vmovd %xmm1, %eax
+; AVX512BF16-NEXT: vpextrw $6, %xmm0, %ecx
+; AVX512BF16-NEXT: shll $16, %ecx
+; AVX512BF16-NEXT: vmovd %ecx, %xmm1
+; AVX512BF16-NEXT: vroundss $4, %xmm1, %xmm1, %xmm1
+; AVX512BF16-NEXT: vcvtneps2bf16 %xmm1, %xmm1
+; AVX512BF16-NEXT: vmovd %xmm1, %ecx
+; AVX512BF16-NEXT: vpextrw $5, %xmm0, %edx
+; AVX512BF16-NEXT: shll $16, %edx
+; AVX512BF16-NEXT: vmovd %edx, %xmm1
+; AVX512BF16-NEXT: vroundss $4, %xmm1, %xmm1, %xmm1
+; AVX512BF16-NEXT: vcvtneps2bf16 %xmm1, %xmm1
+; AVX512BF16-NEXT: vmovd %xmm1, %edx
+; AVX512BF16-NEXT: vpextrw $4, %xmm0, %esi
+; AVX512BF16-NEXT: shll $16, %esi
+; AVX512BF16-NEXT: vmovd %esi, %xmm1
+; AVX512BF16-NEXT: vroundss $4, %xmm1, %xmm1, %xmm1
+; AVX512BF16-NEXT: vcvtneps2bf16 %xmm1, %xmm1
+; AVX512BF16-NEXT: vmovd %xmm1, %esi
+; AVX512BF16-NEXT: vpextrw $3, %xmm0, %edi
+; AVX512BF16-NEXT: shll $16, %edi
+; AVX512BF16-NEXT: vmovd %edi, %xmm1
+; AVX512BF16-NEXT: vroundss $4, %xmm1, %xmm1, %xmm1
+; AVX512BF16-NEXT: vcvtneps2bf16 %xmm1, %xmm1
+; AVX512BF16-NEXT: vmovd %xmm1, %edi
+; AVX512BF16-NEXT: vpextrw $2, %xmm0, %r8d
+; AVX512BF16-NEXT: shll $16, %r8d
+; AVX512BF16-NEXT: vmovd %r8d, %xmm1
+; AVX512BF16-NEXT: vroundss $4, %xmm1, %xmm1, %xmm1
+; AVX512BF16-NEXT: vcvtneps2bf16 %xmm1, %xmm1
+; AVX512BF16-NEXT: vmovd %xmm1, %r8d
+; AVX512BF16-NEXT: vpextrw $1, %xmm0, %r9d
+; AVX512BF16-NEXT: shll $16, %r9d
+; AVX512BF16-NEXT: vmovd %r9d, %xmm1
+; AVX512BF16-NEXT: vroundss $4, %xmm1, %xmm1, %xmm1
+; AVX512BF16-NEXT: vcvtneps2bf16 %xmm1, %xmm1
+; AVX512BF16-NEXT: vmovd %xmm0, %r9d
+; AVX512BF16-NEXT: shll $16, %r9d
+; AVX512BF16-NEXT: vmovd %r9d, %xmm0
+; AVX512BF16-NEXT: vroundss $4, %xmm0, %xmm0, %xmm0
+; AVX512BF16-NEXT: vcvtneps2bf16 %xmm0, %xmm0
+; AVX512BF16-NEXT: vpunpcklwd {{.*#+}} xmm0 = xmm0[0],xmm1[0],xmm0[1],xmm1[1],xmm0[2],xmm1[2],xmm0[3],xmm1[3]
+; AVX512BF16-NEXT: vpinsrw $2, %r8d, %xmm0, %xmm0
+; AVX512BF16-NEXT: vpinsrw $3, %edi, %xmm0, %xmm0
+; AVX512BF16-NEXT: vpinsrw $4, %esi, %xmm0, %xmm0
+; AVX512BF16-NEXT: vpinsrw $5, %edx, %xmm0, %xmm0
+; AVX512BF16-NEXT: vpinsrw $6, %ecx, %xmm0, %xmm0
+; AVX512BF16-NEXT: vpinsrw $7, %eax, %xmm0, %xmm0
+; AVX512BF16-NEXT: retq
+ %r = call <8 x bfloat> @llvm.rint.v8bf16(<8 x bfloat> %a)
+ ret <8 x bfloat> %r
+}
+
+define bfloat @rint_bf16(bfloat %a) {
+; AVX10_2-LABEL: rint_bf16:
+; AVX10_2: # %bb.0:
+; AVX10_2-NEXT: vrndscalebf16 $4, %xmm0, %xmm0
+; AVX10_2-NEXT: retq
+;
+; AVX512BF16-LABEL: rint_bf16:
+; AVX512BF16: # %bb.0:
+; AVX512BF16-NEXT: vpextrw $0, %xmm0, %eax
+; AVX512BF16-NEXT: shll $16, %eax
+; AVX512BF16-NEXT: vmovd %eax, %xmm0
+; AVX512BF16-NEXT: vroundss $4, %xmm0, %xmm0, %xmm0
+; AVX512BF16-NEXT: vcvtneps2bf16 %xmm0, %xmm0
+; AVX512BF16-NEXT: retq
+ %r = call bfloat @llvm.rint.bf16(bfloat %a)
+ ret bfloat %r
+}
+
+define <8 x bfloat> @nearbyint_v8bf16(<8 x bfloat> %a) {
+; AVX10_2-LABEL: nearbyint_v8bf16:
+; AVX10_2: # %bb.0:
+; AVX10_2-NEXT: vrndscalebf16 $12, %xmm0, %xmm0
+; AVX10_2-NEXT: retq
+;
+; AVX512BF16-LABEL: nearbyint_v8bf16:
+; AVX512BF16: # %bb.0:
+; AVX512BF16-NEXT: vpextrw $7, %xmm0, %eax
+; AVX512BF16-NEXT: shll $16, %eax
+; AVX512BF16-NEXT: vmovd %eax, %xmm1
+; AVX512BF16-NEXT: vroundss $12, %xmm1, %xmm1, %xmm1
+; AVX512BF16-NEXT: vcvtneps2bf16 %xmm1, %xmm1
+; AVX512BF16-NEXT: vmovd %xmm1, %eax
+; AVX512BF16-NEXT: vpextrw $6, %xmm0, %ecx
+; AVX512BF16-NEXT: shll $16, %ecx
+; AVX512BF16-NEXT: vmovd %ecx, %xmm1
+; AVX512BF16-NEXT: vroundss $12, %xmm1, %xmm1, %xmm1
+; AVX512BF16-NEXT: vcvtneps2bf16 %xmm1, %xmm1
+; AVX512BF16-NEXT: vmovd %xmm1, %ecx
+; AVX512BF16-NEXT: vpextrw $5, %xmm0, %edx
+; AVX512BF16-NEXT: shll $16, %edx
+; AVX512BF16-NEXT: vmovd %edx, %xmm1
+; AVX512BF16-NEXT: vroundss $12, %xmm1, %xmm1, %xmm1
+; AVX512BF16-NEXT: vcvtneps2bf16 %xmm1, %xmm1
+; AVX512BF16-NEXT: vmovd %xmm1, %edx
+; AVX512BF16-NEXT: vpextrw $4, %xmm0, %esi
+; AVX512BF16-NEXT: shll $16, %esi
+; AVX512BF16-NEXT: vmovd %esi, %xmm1
+; AVX512BF16-NEXT: vroundss $12, %xmm1, %xmm1, %xmm1
+; AVX512BF16-NEXT: vcvtneps2bf16 %xmm1, %xmm1
+; AVX512BF16-NEXT: vmovd %xmm1, %esi
+; AVX512BF16-NEXT: vpextrw $3, %xmm0, %edi
+; AVX512BF16-NEXT: shll $16, %edi
+; AVX512BF16-NEXT: vmovd %edi, %xmm1
+; AVX512BF16-NEXT: vroundss $12, %xmm1, %xmm1, %xmm1
+; AVX512BF16-NEXT: vcvtneps2bf16 %xmm1, %xmm1
+; AVX512BF16-NEXT: vmovd %xmm1, %edi
+; AVX512BF16-NEXT: vpextrw $2, %xmm0, %r8d
+; AVX512BF16-NEXT: shll $16, %r8d
+; AVX512BF16-NEXT: vmovd %r8d, %xmm1
+; AVX512BF16-NEXT: vroundss $12, %xmm1, %xmm1, %xmm1
+; AVX512BF16-NEXT: vcvtneps2bf16 %xmm1, %xmm1
+; AVX512BF16-NEXT: vmovd %xmm1, %r8d
+; AVX512BF16-NEXT: vpextrw $1, %xmm0, %r9d
+; AVX512BF16-NEXT: shll $16, %r9d
+; AVX512BF16-NEXT: vmovd %r9d, %xmm1
+; AVX512BF16-NEXT: vroundss $12, %xmm1, %xmm1, %xmm1
+; AVX512BF16-NEXT: vcvtneps2bf16 %xmm1, %xmm1
+; AVX512BF16-NEXT: vmovd %xmm0, %r9d
+; AVX512BF16-NEXT: shll $16, %r9d
+; AVX512BF16-NEXT: vmovd %r9d, %xmm0
+; AVX512BF16-NEXT: vroundss $12, %xmm0, %xmm0, %xmm0
+; AVX512BF16-NEXT: vcvtneps2bf16 %xmm0, %xmm0
+; AVX512BF16-NEXT: vpunpcklwd {{.*#+}} xmm0 = xmm0[0],xmm1[0],xmm0[1],xmm1[1],xmm0[2],xmm1[2],xmm0[3],xmm1[3]
+; AVX512BF16-NEXT: vpinsrw $2, %r8d, %xmm0, %xmm0
+; AVX512BF16-NEXT: vpinsrw $3, %edi, %xmm0, %xmm0
+; AVX512BF16-NEXT: vpinsrw $4, %esi, %xmm0, %xmm0
+; AVX512BF16-NEXT: vpinsrw $5, %edx, %xmm0, %xmm0
+; AVX512BF16-NEXT: vpinsrw $6, %ecx, %xmm0, %xmm0
+; AVX512BF16-NEXT: vpinsrw $7, %eax, %xmm0, %xmm0
+; AVX512BF16-NEXT: retq
+ %r = call <8 x bfloat> @llvm.nearbyint.v8bf16(<8 x bfloat> %a)
+ ret <8 x bfloat> %r
+}
+
+define bfloat @nearbyint_bf16(bfloat %a) {
+; AVX10_2-LABEL: nearbyint_bf16:
+; AVX10_2: # %bb.0:
+; AVX10_2-NEXT: vrndscalebf16 $12, %xmm0, %xmm0
+; AVX10_2-NEXT: retq
+;
+; AVX512BF16-LABEL: nearbyint_bf16:
+; AVX512BF16: # %bb.0:
+; AVX512BF16-NEXT: vpextrw $0, %xmm0, %eax
+; AVX512BF16-NEXT: shll $16, %eax
+; AVX512BF16-NEXT: vmovd %eax, %xmm0
+; AVX512BF16-NEXT: vroundss $12, %xmm0, %xmm0, %xmm0
+; AVX512BF16-NEXT: vcvtneps2bf16 %xmm0, %xmm0
+; AVX512BF16-NEXT: retq
+ %r = call bfloat @llvm.nearbyint.bf16(bfloat %a)
+ ret bfloat %r
+}
+
+define <8 x bfloat> @roundeven_v8bf16(<8 x bfloat> %a) {
+; AVX10_2-LABEL: roundeven_v8bf16:
+; AVX10_2: # %bb.0:
+; AVX10_2-NEXT: vrndscalebf16 $8, %xmm0, %xmm0
+; AVX10_2-NEXT: retq
+;
+; AVX512BF16-LABEL: roundeven_v8bf16:
+; AVX512BF16: # %bb.0:
+; AVX512BF16-NEXT: vpextrw $7, %xmm0, %eax
+; AVX512BF16-NEXT: shll $16, %eax
+; AVX512BF16-NEXT: vmovd %eax, %xmm1
+; AVX512BF16-NEXT: vroundss $8, %xmm1, %xmm1, %xmm1
+; AVX512BF16-NEXT: vcvtneps2bf16 %xmm1, %xmm1
+; AVX512BF16-NEXT: vmovd %xmm1, %eax
+; AVX512BF16-NEXT: vpextrw $6, %xmm0, %ecx
+; AVX512BF16-NEXT: shll $16, %ecx
+; AVX512BF16-NEXT: vmovd %ecx, %xmm1
+; AVX512BF16-NEXT: vroundss $8, %xmm1, %xmm1, %xmm1
+; AVX512BF16-NEXT: vcvtneps2bf16 %xmm1, %xmm1
+; AVX512BF16-NEXT: vmovd %xmm1, %ecx
+; AVX512BF16-NEXT: vpextrw $5, %xmm0, %edx
+; AVX512BF16-NEXT: shll $16, %edx
+; AVX512BF16-NEXT: vmovd %edx, %xmm1
+; AVX512BF16-NEXT: vroundss $8, %xmm1, %xmm1, %xmm1
+; AVX512BF16-NEXT: vcvtneps2bf16 %xmm1, %xmm1
+; AVX512BF16-NEXT: vmovd %xmm1, %edx
+; AVX512BF16-NEXT: vpextrw $4, %xmm0, %esi
+; AVX512BF16-NEXT: shll $16, %esi
+; AVX512BF16-NEXT: vmovd %esi, %xmm1
+; AVX512BF16-NEXT: vroundss $8, %xmm1, %xmm1, %xmm1
+; AVX512BF16-NEXT: vcvtneps2bf16 %xmm1, %xmm1
+; AVX512BF16-NEXT: vmovd %xmm1, %esi
+; AVX512BF16-NEXT: vpextrw $3, %xmm0, %edi
+; AVX512BF16-NEXT: shll $16, %edi
+; AVX512BF16-NEXT: vmovd %edi, %xmm1
+; AVX512BF16-NEXT: vroundss $8, %xmm1, %xmm1, %xmm1
+; AVX512BF16-NEXT: vcvtneps2bf16 %xmm1, %xmm1
+; AVX512BF16-NEXT: vmovd %xmm1, %edi
+; AVX512BF16-NEXT: vpextrw $2, %xmm0, %r8d
+; AVX512BF16-NEXT: shll $16, %r8d
+; AVX512BF16-NEXT: vmovd %r8d, %xmm1
+; AVX512BF16-NEXT: vroundss $8, %xmm1, %xmm1, %xmm1
+; AVX512BF16-NEXT: vcvtneps2bf16 %xmm1, %xmm1
+; AVX512BF16-NEXT: vmovd %xmm1, %r8d
+; AVX512BF16-NEXT: vpextrw $1, %xmm0, %r9d
+; AVX512BF16-NEXT: shll $16, %r9d
+; AVX512BF16-NEXT: vmovd %r9d, %xmm1
+; AVX512BF16-NEXT: vroundss $8, %xmm1, %xmm1, %xmm1
+; AVX512BF16-NEXT: vcvtneps2bf16 %xmm1, %xmm1
+; AVX512BF16-NEXT: vmovd %xmm0, %r9d
+; AVX512BF16-NEXT: shll $16, %r9d
+; AVX512BF16-NEXT: vmovd %r9d, %xmm0
+; AVX512BF16-NEXT: vroundss $8, %xmm0, %xmm0, %xmm0
+; AVX512BF16-NEXT: vcvtneps2bf16 %xmm0, %xmm0
+; AVX512BF16-NEXT: vpunpcklwd {{.*#+}} xmm0 = xmm0[0],xmm1[0],xmm0[1],xmm1[1],xmm0[2],xmm1[2],xmm0[3],xmm1[3]
+; AVX512BF16-NEXT: vpinsrw $2, %r8d, %xmm0, %xmm0
+; AVX512BF16-NEXT: vpinsrw $3, %edi, %xmm0, %xmm0
+; AVX512BF16-NEXT: vpinsrw $4, %esi, %xmm0, %xmm0
+; AVX512BF16-NEXT: vpinsrw $5, %edx, %xmm0, %xmm0
+; AVX512BF16-NEXT: vpinsrw $6, %ecx, %xmm0, %xmm0
+; AVX512BF16-NEXT: vpinsrw $7, %eax, %xmm0, %xmm0
+; AVX512BF16-NEXT: retq
+ %r = call <8 x bfloat> @llvm.roundeven.v8bf16(<8 x bfloat> %a)
+ ret <8 x bfloat> %r
+}
+
+define bfloat @roundeven_bf16(bfloat %a) {
+; AVX10_2-LABEL: roundeven_bf16:
+; AVX10_2: # %bb.0:
+; AVX10_2-NEXT: vrndscalebf16 $8, %xmm0, %xmm0
+; AVX10_2-NEXT: retq
+;
+; AVX512BF16-LABEL: roundeven_bf16:
+; AVX512BF16: # %bb.0:
+; AVX512BF16-NEXT: vpextrw $0, %xmm0, %eax
+; AVX512BF16-NEXT: shll $16, %eax
+; AVX512BF16-NEXT: vmovd %eax, %xmm0
+; AVX512BF16-NEXT: vroundss $8, %xmm0, %xmm0, %xmm0
+; AVX512BF16-NEXT: vcvtneps2bf16 %xmm0, %xmm0
+; AVX512BF16-NEXT: retq
+ %r = call bfloat @llvm.roundeven.bf16(bfloat %a)
+ ret bfloat %r
+}
+
+; ymm and zmm come from separate setOperationAction entries than the xmm ones.
+define <16 x bfloat> @floor_v16bf16(<16 x bfloat> %a) {
+; AVX10_2-LABEL: floor_v16bf16:
+; AVX10_2: # %bb.0:
+; AVX10_2-NEXT: vrndscalebf16 $9, %ymm0, %ymm0
+; AVX10_2-NEXT: retq
+;
+; AVX512BF16-LABEL: floor_v16bf16:
+; AVX512BF16: # %bb.0:
+; AVX512BF16-NEXT: pushq %rbp
+; AVX512BF16-NEXT: .cfi_def_cfa_offset 16
+; AVX512BF16-NEXT: pushq %r15
+; AVX512BF16-NEXT: .cfi_def_cfa_offset 24
+; AVX512BF16-NEXT: pushq %r14
+; AVX512BF16-NEXT: .cfi_def_cfa_offset 32
+; AVX512BF16-NEXT: pushq %r12
+; AVX512BF16-NEXT: .cfi_def_cfa_offset 40
+; AVX512BF16-NEXT: pushq %rbx
+; AVX512BF16-NEXT: .cfi_def_cfa_offset 48
+; AVX512BF16-NEXT: .cfi_offset %rbx, -48
+; AVX512BF16-NEXT: .cfi_offset %r12, -40
+; AVX512BF16-NEXT: .cfi_offset %r14, -32
+; AVX512BF16-NEXT: .cfi_offset %r15, -24
+; AVX512BF16-NEXT: .cfi_offset %rbp, -16
+; AVX512BF16-NEXT: vpextrw $7, %xmm0, %eax
+; AVX512BF16-NEXT: shll $16, %eax
+; AVX512BF16-NEXT: vmovd %eax, %xmm1
+; AVX512BF16-NEXT: vroundss $9, %xmm1, %xmm1, %xmm1
+; AVX512BF16-NEXT: vcvtneps2bf16 %xmm1, %xmm1
+; AVX512BF16-NEXT: vmovd %xmm1, %eax
+; AVX512BF16-NEXT: vpextrw $6, %xmm0, %ecx
+; AVX512BF16-NEXT: shll $16, %ecx
+; AVX512BF16-NEXT: vmovd %ecx, %xmm1
+; AVX512BF16-NEXT: vroundss $9, %xmm1, %xmm1, %xmm1
+; AVX512BF16-NEXT: vcvtneps2bf16 %xmm1, %xmm1
+; AVX512BF16-NEXT: vmovd %xmm1, %ecx
+; AVX512BF16-NEXT: vpextrw $5, %xmm0, %edx
+; AVX512BF16-NEXT: shll $16, %edx
+; AVX512BF16-NEXT: vmovd %edx, %xmm1
+; AVX512BF16-NEXT: vroundss $9, %xmm1, %xmm1, %xmm1
+; AVX512BF16-NEXT: vcvtneps2bf16 %xmm1, %xmm1
+; AVX512BF16-NEXT: vmovd %xmm1, %edx
+; AVX512BF16-NEXT: vpextrw $4, %xmm0, %esi
+; AVX512BF16-NEXT: shll $16, %esi
+; AVX512BF16-NEXT: vmovd %esi, %xmm1
+; AVX512BF16-NEXT: vroundss $9, %xmm1, %xmm1, %xmm1
+; AVX512BF16-NEXT: vcvtneps2bf16 %xmm1, %xmm1
+; AVX512BF16-NEXT: vmovd %xmm1, %esi
+; AVX512BF16-NEXT: vpextrw $3, %xmm0, %edi
+; AVX512BF16-NEXT: shll $16, %edi
+; AVX512BF16-NEXT: vmovd %edi, %xmm1
+; AVX512BF16-NEXT: vroundss $9, %xmm1, %xmm1, %xmm1
+; AVX512BF16-NEXT: vcvtneps2bf16 %xmm1, %xmm1
+; AVX512BF16-NEXT: vmovd %xmm1, %edi
+; AVX512BF16-NEXT: vpextrw $2, %xmm0, %r8d
+; AVX512BF16-NEXT: shll $16, %r8d
+; AVX512BF16-NEXT: vmovd %r8d, %xmm1
+; AVX512BF16-NEXT: vroundss $9, %xmm1, %xmm1, %xmm1
+; AVX512BF16-NEXT: vcvtneps2bf16 %xmm1, %xmm1
+; AVX512BF16-NEXT: vmovd %xmm1, %r8d
+; AVX512BF16-NEXT: vmovd %xmm0, %r9d
+; AVX512BF16-NEXT: shll $16, %r9d
+; AVX512BF16-NEXT: vmovd %r9d, %xmm1
+; AVX512BF16-NEXT: vroundss $9, %xmm1, %xmm1, %xmm1
+; AVX512BF16-NEXT: vcvtneps2bf16 %xmm1, %xmm1
+; AVX512BF16-NEXT: vpextrw $1, %xmm0, %r9d
+; AVX512BF16-NEXT: shll $16, %r9d
+; AVX512BF16-NEXT: vmovd %r9d, %xmm2
+; AVX512BF16-NEXT: vroundss $9, %xmm2, %xmm2, %xmm2
+; AVX512BF16-NEXT: vcvtneps2bf16 %xmm2, %xmm2
+; AVX512BF16-NEXT: vmovd %xmm2, %r9d
+; AVX512BF16-NEXT: vextracti128 $1, %ymm0, %xmm0
+; AVX512BF16-NEXT: vpextrw $7, %xmm0, %r10d
+; AVX512BF16-NEXT: shll $16, %r10d
+; AVX512BF16-NEXT: vmovd %r10d, %xmm2
+; AVX512BF16-NEXT: vroundss $9, %xmm2, %xmm2, %xmm2
+; AVX512BF16-NEXT: vcvtneps2bf16 %xmm2, %xmm2
+; AVX512BF16-NEXT: vmovd %xmm2, %r10d
+; AVX512BF16-NEXT: vpextrw $6, %xmm0, %r11d
+; AVX512BF16-NEXT: shll $16, %r11d
+; AVX512BF16-NEXT: vmovd %r11d, %xmm2
+; AVX512BF16-NEXT: vroundss $9, %xmm2, %xmm2, %xmm2
+; AVX512BF16-NEXT: vcvtneps2bf16 %xmm2, %xmm2
+; AVX512BF16-NEXT: vmovd %xmm2, %r11d
+; AVX512BF16-NEXT: vpextrw $5, %xmm0, %ebx
+; AVX512BF16-NEXT: shll $16, %ebx
+; AVX512BF16-NEXT: vmovd %ebx, %xmm2
+; AVX512BF16-NEXT: vroundss $9, %xmm2, %xmm2, %xmm2
+; AVX512BF16-NEXT: vcvtneps2bf16 %xmm2, %xmm2
+; AVX512BF16-NEXT: vmovd %xmm2, %ebx
+; AVX512BF16-NEXT: vpextrw $4, %xmm0, %ebp
+; AVX512BF16-NEXT: shll $16, %ebp
+; AVX512BF16-NEXT: vmovd %ebp, %xmm2
+; AVX512BF16-NEXT: vroundss $9, %xmm2, %xmm2, %xmm2
+; AVX512BF16-NEXT: vcvtneps2bf16 %xmm2, %xmm2
+; AVX512BF16-NEXT: vmovd %xmm2, %ebp
+; AVX512BF16-NEXT: vpextrw $3, %xmm0, %r14d
+; AVX512BF16-NEXT: shll $16, %r14d
+; AVX512BF16-NEXT: vmovd %r14d, %xmm2
+; AVX512BF16-NEXT: vroundss $9, %xmm2, %xmm2, %xmm2
+; AVX512BF16-NEXT: vcvtneps2bf16 %xmm2, %xmm2
+; AVX512BF16-NEXT: vmovd %xmm2, %r14d
+; AVX512BF16-NEXT: vpextrw $2, %xmm0, %r15d
+; AVX512BF16-NEXT: shll $16, %r15d
+; AVX512BF16-NEXT: vmovd %r15d, %xmm2
+; AVX512BF16-NEXT: vroundss $9, %xmm2, %xmm2, %xmm2
+; AVX512BF16-NEXT: vcvtneps2bf16 %xmm2, %xmm2
+; AVX512BF16-NEXT: vmovd %xmm2, %r15d
+; AVX512BF16-NEXT: vmovd %xmm0, %r12d
+; AVX512BF16-NEXT: shll $16, %r12d
+; AVX512BF16-NEXT: vmovd %r12d, %xmm2
+; AVX512BF16-NEXT: vroundss $9, %xmm2, %xmm2, %xmm2
+; AVX512BF16-NEXT: vcvtneps2bf16 %xmm2, %xmm2
+; AVX512BF16-NEXT: vpextrw $1, %xmm0, %r12d
+; AVX512BF16-NEXT: shll $16, %r12d
+; AVX512BF16-NEXT: vmovd %r12d, %xmm0
+; AVX512BF16-NEXT: vroundss $9, %xmm0, %xmm0, %xmm0
+; AVX512BF16-NEXT: vcvtneps2bf16 %xmm0, %xmm0
+; AVX512BF16-NEXT: vmovd %xmm0, %r12d
+; AVX512BF16-NEXT: vpinsrw $1, %r12d, %xmm2, %xmm0
+; AVX512BF16-NEXT: vpinsrw $2, %r15d, %xmm0, %xmm0
+; AVX512BF16-NEXT: vpinsrw $3, %r14d, %xmm0, %xmm0
+; AVX512BF16-NEXT: vpinsrw $4, %ebp, %xmm0, %xmm0
+; AVX512BF16-NEXT: vpinsrw $5, %ebx, %xmm0, %xmm0
+; AVX512BF16-NEXT: vpinsrw $6, %r11d, %xmm0, %xmm0
+; AVX512BF16-NEXT: vpinsrw $7, %r10d, %xmm0, %xmm0
+; AVX512BF16-NEXT: vpinsrw $1, %r9d, %xmm1, %xmm1
+; AVX512BF16-NEXT: vpinsrw $2, %r8d, %xmm1, %xmm1
+; AVX512BF16-NEXT: vpinsrw $3, %edi, %xmm1, %xmm1
+; AVX512BF16-NEXT: vpinsrw $4, %esi, %xmm1, %xmm1
+; AVX512BF16-NEXT: vpinsrw $5, %edx, %xmm1, %xmm1
+; AVX512BF16-NEXT: vpinsrw $6, %ecx, %xmm1, %xmm1
+; AVX512BF16-NEXT: vpinsrw $7, %eax, %xmm1, %xmm1
+; AVX512BF16-NEXT: vinserti128 $1, %xmm0, %ymm1, %ymm0
+; AVX512BF16-NEXT: popq %rbx
+; AVX512BF16-NEXT: .cfi_def_cfa_offset 40
+; AVX512BF16-NEXT: popq %r12
+; AVX512BF16-NEXT: .cfi_def_cfa_offset 32
+; AVX512BF16-NEXT: popq %r14
+; AVX512BF16-NEXT: .cfi_def_cfa_offset 24
+; AVX512BF16-NEXT: popq %r15
+; AVX512BF16-NEXT: .cfi_def_cfa_offset 16
+; AVX512BF16-NEXT: popq %rbp
+; AVX512BF16-NEXT: .cfi_def_cfa_offset 8
+; AVX512BF16-NEXT: retq
+ %r = call <16 x bfloat> @llvm.floor.v16bf16(<16 x bfloat> %a)
+ ret <16 x bfloat> %r
+}
+
+define <32 x bfloat> @floor_v32bf16(<32 x bfloat> %a) {
+; AVX10_2-LABEL: floor_v32bf16:
+; AVX10_2: # %bb.0:
+; AVX10_2-NEXT: vrndscalebf16 $9, %zmm0, %zmm0
+; AVX10_2-NEXT: retq
+;
+; AVX512BF16-LABEL: floor_v32bf16:
+; AVX512BF16: # %bb.0:
+; AVX512BF16-NEXT: vpextrw $7, %xmm0, %eax
+; AVX512BF16-NEXT: shll $16, %eax
+; AVX512BF16-NEXT: vmovd %eax, %xmm1
+; AVX512BF16-NEXT: vpextrw $6, %xmm0, %eax
+; AVX512BF16-NEXT: shll $16, %eax
+; AVX512BF16-NEXT: vmovd %eax, %xmm2
+; AVX512BF16-NEXT: vpextrw $5, %xmm0, %eax
+; AVX512BF16-NEXT: shll $16, %eax
+; AVX512BF16-NEXT: vmovd %eax, %xmm3
+; AVX512BF16-NEXT: vpextrw $4, %xmm0, %eax
+; AVX512BF16-NEXT: shll $16, %eax
+; AVX512BF16-NEXT: vmovd %eax, %xmm4
+; AVX512BF16-NEXT: vpextrw $3, %xmm0, %eax
+; AVX512BF16-NEXT: shll $16, %eax
+; AVX512BF16-NEXT: vmovd %eax, %xmm5
+; AVX512BF16-NEXT: vpextrw $2, %xmm0, %eax
+; AVX512BF16-NEXT: shll $16, %eax
+; AVX512BF16-NEXT: vmovd %eax, %xmm6
+; AVX512BF16-NEXT: vmovd %xmm0, %eax
+; AVX512BF16-NEXT: shll $16, %eax
+; AVX512BF16-NEXT: vmovd %eax, %xmm7
+; AVX512BF16-NEXT: vpextrw $1, %xmm0, %eax
+; AVX512BF16-NEXT: shll $16, %eax
+; AVX512BF16-NEXT: vmovd %eax, %xmm8
+; AVX512BF16-NEXT: vextracti32x4 $1, %ymm0, %xmm16
+; AVX512BF16-NEXT: vpextrw $7, %xmm16, %eax
+; AVX512BF16-NEXT: shll $16, %eax
+; AVX512BF16-NEXT: vmovd %eax, %xmm9
+; AVX512BF16-NEXT: vpextrw $6, %xmm16, %eax
+; AVX512BF16-NEXT: shll $16, %eax
+; AVX512BF16-NEXT: vmovd %eax, %xmm10
+; AVX512BF16-NEXT: vpextrw $5, %xmm16, %eax
+; AVX512BF16-NEXT: shll $16, %eax
+; AVX512BF16-NEXT: vmovd %eax, %xmm11
+; AVX512BF16-NEXT: vpextrw $4, %xmm16, %eax
+; AVX512BF16-NEXT: shll $16, %eax
+; AVX512BF16-NEXT: vmovd %eax, %xmm12
+; AVX512BF16-NEXT: vpextrw $3, %xmm16, %eax
+; AVX512BF16-NEXT: shll $16, %eax
+; AVX512BF16-NEXT: vmovd %eax, %xmm13
+; AVX512BF16-NEXT: vpextrw $2, %xmm16, %eax
+; AVX512BF16-NEXT: shll $16, %eax
+; AVX512BF16-NEXT: vmovd %eax, %xmm14
+; AVX512BF16-NEXT: vmovd %xmm16, %eax
+; AVX512BF16-NEXT: shll $16, %eax
+; AVX512BF16-NEXT: vmovd %eax, %xmm15
+; AVX512BF16-NEXT: vpextrw $1, %xmm16, %eax
+; AVX512BF16-NEXT: shll $16, %eax
+; AVX512BF16-NEXT: vmovd %eax, %xmm16
+; AVX512BF16-NEXT: vextracti32x4 $2, %zmm0, %xmm24
+; AVX512BF16-NEXT: vpextrw $7, %xmm24, %eax
+; AVX512BF16-NEXT: shll $16, %eax
+; AVX512BF16-NEXT: vmovd %eax, %xmm17
+; AVX512BF16-NEXT: vpextrw $6, %xmm24, %eax
+; AVX512BF16-NEXT: shll $16, %eax
+; AVX512BF16-NEXT: vmovd %eax, %xmm18
+; AVX512BF16-NEXT: vpextrw $5, %xmm24, %eax
+; AVX512BF16-NEXT: shll $16, %eax
+; AVX512BF16-NEXT: vmovd %eax, %xmm19
+; AVX512BF16-NEXT: vpextrw $4, %xmm24, %eax
+; AVX512BF16-NEXT: shll $16, %eax
+; AVX512BF16-NEXT: vmovd %eax, %xmm20
+; AVX512BF16-NEXT: vpextrw $3, %xmm24, %eax
+; AVX512BF16-NEXT: shll $16, %eax
+; AVX512BF16-NEXT: vmovd %eax, %xmm21
+; AVX512BF16-NEXT: vpextrw $2, %xmm24, %eax
+; AVX512BF16-NEXT: shll $16, %eax
+; AVX512BF16-NEXT: vmovd %eax, %xmm22
+; AVX512BF16-NEXT: vmovd %xmm24, %eax
+; AVX512BF16-NEXT: shll $16, %eax
+; AVX512BF16-NEXT: vmovd %eax, %xmm23
+; AVX512BF16-NEXT: vpextrw $1, %xmm24, %eax
+; AVX512BF16-NEXT: shll $16, %eax
+; AVX512BF16-NEXT: vmovd %eax, %xmm24
+; AVX512BF16-NEXT: vextracti32x4 $3, %zmm0, %xmm25
+; AVX512BF16-NEXT: vpextrw $7, %xmm25, %eax
+; AVX512BF16-NEXT: shll $16, %eax
+; AVX512BF16-NEXT: vmovd %eax, %xmm0
+; AVX512BF16-NEXT: vpextrw $6, %xmm25, %eax
+; AVX512BF16-NEXT: shll $16, %eax
+; AVX512BF16-NEXT: vmovd %eax, %xmm26
+; AVX512BF16-NEXT: vpextrw $5, %xmm25, %eax
+; AVX512BF16-NEXT: shll $16, %eax
+; AVX512BF16-NEXT: vmovd %eax, %xmm27
+; AVX512BF16-NEXT: vpextrw $4, %xmm25, %eax
+; AVX512BF16-NEXT: shll $16, %eax
+; AVX512BF16-NEXT: vmovd %eax, %xmm28
+; AVX512BF16-NEXT: vpextrw $3, %xmm25, %eax
+; AVX512BF16-NEXT: shll $16, %eax
+; AVX512BF16-NEXT: vmovd %eax, %xmm29
+; AVX512BF16-NEXT: vpextrw $2, %xmm25, %eax
+; AVX512BF16-NEXT: shll $16, %eax
+; AVX512BF16-NEXT: vmovd %eax, %xmm30
+; AVX512BF16-NEXT: vmovd %xmm25, %eax
+; AVX512BF16-NEXT: shll $16, %eax
+; AVX512BF16-NEXT: vmovd %eax, %xmm31
+; AVX512BF16-NEXT: vpextrw $1, %xmm25, %eax
+; AVX512BF16-NEXT: shll $16, %eax
+; AVX512BF16-NEXT: vmovd %eax, %xmm25
+; AVX512BF16-NEXT: vrndscaless $9, %xmm25, %xmm25, %xmm25
+; AVX512BF16-NEXT: vcvtneps2bf16 %xmm25, %xmm25
+; AVX512BF16-NEXT: vmovd %xmm25, %eax
+; AVX512BF16-NEXT: vrndscaless $9, %xmm30, %xmm30, %xmm25
+; AVX512BF16-NEXT: vcvtneps2bf16 %xmm25, %xmm25
+; AVX512BF16-NEXT: vrndscaless $9, %xmm31, %xmm31, %xmm30
+; AVX512BF16-NEXT: vcvtneps2bf16 %xmm30, %xmm30
+; AVX512BF16-NEXT: vpinsrw $1, %eax, %xmm30, %xmm30
+; AVX512BF16-NEXT: vmovd %xmm25, %eax
+; AVX512BF16-NEXT: vrndscaless $9, %xmm29, %xmm29, %xmm25
+; AVX512BF16-NEXT: vcvtneps2bf16 %xmm25, %xmm25
+; AVX512BF16-NEXT: vpinsrw $2, %eax, %xmm30, %xmm29
+; AVX512BF16-NEXT: vmovd %xmm25, %eax
+; AVX512BF16-NEXT: vrndscaless $9, %xmm28, %xmm28, %xmm25
+; AVX512BF16-NEXT: vcvtneps2bf16 %xmm25, %xmm25
+; AVX512BF16-NEXT: vpinsrw $3, %eax, %xmm29, %xmm28
+; AVX512BF16-NEXT: vmovd %xmm25, %eax
+; AVX512BF16-NEXT: vrndscaless $9, %xmm27, %xmm27, %xmm25
+; AVX512BF16-NEXT: vcvtneps2bf16 %xmm25, %xmm25
+; AVX512BF16-NEXT: vpinsrw $4, %eax, %xmm28, %xmm27
+; AVX512BF16-NEXT: vmovd %xmm25, %eax
+; AVX512BF16-NEXT: vrndscaless $9, %xmm26, %xmm26, %xmm25
+; AVX512BF16-NEXT: vcvtneps2bf16 %xmm25, %xmm25
+; AVX512BF16-NEXT: vpinsrw $5, %eax, %xmm27, %xmm26
+; AVX512BF16-NEXT: vmovd %xmm25, %eax
+; AVX512BF16-NEXT: vroundss $9, %xmm0, %xmm0, %xmm0
+; AVX512BF16-NEXT: vcvtneps2bf16 %xmm0, %xmm0
+; AVX512BF16-NEXT: vpinsrw $6, %eax, %xmm26, %xmm25
+; AVX512BF16-NEXT: vmovd %xmm0, %eax
+; AVX512BF16-NEXT: vrndscaless $9, %xmm24, %xmm24, %xmm0
+; AVX512BF16-NEXT: vcvtneps2bf16 %xmm0, %xmm24
+; AVX512BF16-NEXT: vpinsrw $7, %eax, %xmm25, %xmm0
+; AVX512BF16-NEXT: vmovd %xmm24, %eax
+; AVX512BF16-NEXT: vrndscaless $9, %xmm22, %xmm22, %xmm22
+; AVX512BF16-NEXT: vcvtneps2bf16 %xmm22, %xmm22
+; AVX512BF16-NEXT: vrndscaless $9, %xmm23, %xmm23, %xmm23
+; AVX512BF16-NEXT: vcvtneps2bf16 %xmm23, %xmm23
+; AVX512BF16-NEXT: vpinsrw $1, %eax, %xmm23, %xmm23
+; AVX512BF16-NEXT: vmovd %xmm22, %eax
+; AVX512BF16-NEXT: vrndscaless $9, %xmm21, %xmm21, %xmm21
+; AVX512BF16-NEXT: vcvtneps2bf16 %xmm21, %xmm21
+; AVX512BF16-NEXT: vpinsrw $2, %eax, %xmm23, %xmm22
+; AVX512BF16-NEXT: vmovd %xmm21, %eax
+; AVX512BF16-NEXT: vrndscaless $9, %xmm20, %xmm20, %xmm20
+; AVX512BF16-NEXT: vcvtneps2bf16 %xmm20, %xmm20
+; AVX512BF16-NEXT: vpinsrw $3, %eax, %xmm22, %xmm21
+; AVX512BF16-NEXT: vmovd %xmm20, %eax
+; AVX512BF16-NEXT: vrndscaless $9, %xmm19, %xmm19, %xmm19
+; AVX512BF16-NEXT: vcvtneps2bf16 %xmm19, %xmm19
+; AVX512BF16-NEXT: vpinsrw $4, %eax, %xmm21, %xmm20
+; AVX512BF16-NEXT: vmovd %xmm19, %eax
+; AVX512BF16-NEXT: vrndscaless $9, %xmm18, %xmm18, %xmm18
+; AVX512BF16-NEXT: vcvtneps2bf16 %xmm18, %xmm18
+; AVX512BF16-NEXT: vpinsrw $5, %eax, %xmm20, %xmm19
+; AVX512BF16-NEXT: vmovd %xmm18, %eax
+; AVX512BF16-NEXT: vrndscaless $9, %xmm17, %xmm17, %xmm17
+; AVX512BF16-NEXT: vcvtneps2bf16 %xmm17, %xmm17
+; AVX512BF16-NEXT: vpinsrw $6, %eax, %xmm19, %xmm18
+; AVX512BF16-NEXT: vmovd %xmm17, %eax
+; AVX512BF16-NEXT: vrndscaless $9, %xmm16, %xmm16, %xmm16
+; AVX512BF16-NEXT: vcvtneps2bf16 %xmm16, %xmm16
+; AVX512BF16-NEXT: vpinsrw $7, %eax, %xmm18, %xmm17
+; AVX512BF16-NEXT: vmovd %xmm16, %eax
+; AVX512BF16-NEXT: vroundss $9, %xmm14, %xmm14, %xmm14
+; AVX512BF16-NEXT: vcvtneps2bf16 %xmm14, %xmm14
+; AVX512BF16-NEXT: vroundss $9, %xmm15, %xmm15, %xmm15
+; AVX512BF16-NEXT: vcvtneps2bf16 %xmm15, %xmm15
+; AVX512BF16-NEXT: vpinsrw $1, %eax, %xmm15, %xmm15
+; AVX512BF16-NEXT: vmovd %xmm14, %eax
+; AVX512BF16-NEXT: vroundss $9, %xmm13, %xmm13, %xmm13
+; AVX512BF16-NEXT: vcvtneps2bf16 %xmm13, %xmm13
+; AVX512BF16-NEXT: vpinsrw $2, %eax, %xmm15, %xmm14
+; AVX512BF16-NEXT: vmovd %xmm13, %eax
+; AVX512BF16-NEXT: vroundss $9, %xmm12, %xmm12, %xmm12
+; AVX512BF16-NEXT: vcvtneps2bf16 %xmm12, %xmm12
+; AVX512BF16-NEXT: vpinsrw $3, %eax, %xmm14, %xmm13
+; AVX512BF16-NEXT: vmovd %xmm12, %eax
+; AVX512BF16-NEXT: vroundss $9, %xmm11, %xmm11, %xmm11
+; AVX512BF16-NEXT: vcvtneps2bf16 %xmm11, %xmm11
+; AVX512BF16-NEXT: vpinsrw $4, %eax, %xmm13, %xmm12
+; AVX512BF16-NEXT: vmovd %xmm11, %eax
+; AVX512BF16-NEXT: vroundss $9, %xmm10, %xmm10, %xmm10
+; AVX512BF16-NEXT: vcvtneps2bf16 %xmm10, %xmm10
+; AVX512BF16-NEXT: vpinsrw $5, %eax, %xmm12, %xmm11
+; AVX512BF16-NEXT: vmovd %xmm10, %eax
+; AVX512BF16-NEXT: vroundss $9, %xmm9, %xmm9, %xmm9
+; AVX512BF16-NEXT: vcvtneps2bf16 %xmm9, %xmm9
+; AVX512BF16-NEXT: vpinsrw $6, %eax, %xmm11, %xmm10
+; AVX512BF16-NEXT: vmovd %xmm9, %eax
+; AVX512BF16-NEXT: vroundss $9, %xmm8, %xmm8, %xmm8
+; AVX512BF16-NEXT: vcvtneps2bf16 %xmm8, %xmm8
+; AVX512BF16-NEXT: vpinsrw $7, %eax, %xmm10, %xmm9
+; AVX512BF16-NEXT: vmovd %xmm8, %eax
+; AVX512BF16-NEXT: vroundss $9, %xmm6, %xmm6, %xmm6
+; AVX512BF16-NEXT: vcvtneps2bf16 %xmm6, %xmm6
+; AVX512BF16-NEXT: vroundss $9, %xmm7, %xmm7, %xmm7
+; AVX512BF16-NEXT: vcvtneps2bf16 %xmm7, %xmm7
+; AVX512BF16-NEXT: vpinsrw $1, %eax, %xmm7, %xmm7
+; AVX512BF16-NEXT: vmovd %xmm6, %eax
+; AVX512BF16-NEXT: vroundss $9, %xmm5, %xmm5, %xmm5
+; AVX512BF16-NEXT: vcvtneps2bf16 %xmm5, %xmm5
+; AVX512BF16-NEXT: vpinsrw $2, %eax, %xmm7, %xmm6
+; AVX512BF16-NEXT: vmovd %xmm5, %eax
+; AVX512BF16-NEXT: vroundss $9, %xmm4, %xmm4, %xmm4
+; AVX512BF16-NEXT: vcvtneps2bf16 %xmm4, %xmm4
+; AVX512BF16-NEXT: vpinsrw $3, %eax, %xmm6, %xmm5
+; AVX512BF16-NEXT: vmovd %xmm4, %eax
+; AVX512BF16-NEXT: vroundss $9, %xmm3, %xmm3, %xmm3
+; AVX512BF16-NEXT: vcvtneps2bf16 %xmm3, %xmm3
+; AVX512BF16-NEXT: vpinsrw $4, %eax, %xmm5, %xmm4
+; AVX512BF16-NEXT: vmovd %xmm3, %eax
+; AVX512BF16-NEXT: vroundss $9, %xmm2, %xmm2, %xmm2
+; AVX512BF16-NEXT: vcvtneps2bf16 %xmm2, %xmm2
+; AVX512BF16-NEXT: vpinsrw $5, %eax, %xmm4, %xmm3
+; AVX512BF16-NEXT: vmovd %xmm2, %eax
+; AVX512BF16-NEXT: vroundss $9, %xmm1, %xmm1, %xmm1
+; AVX512BF16-NEXT: vcvtneps2bf16 %xmm1, %xmm1
+; AVX512BF16-NEXT: vpinsrw $6, %eax, %xmm3, %xmm2
+; AVX512BF16-NEXT: vmovd %xmm1, %eax
+; AVX512BF16-NEXT: vpinsrw $7, %eax, %xmm2, %xmm1
+; AVX512BF16-NEXT: vinserti32x4 $1, %xmm0, %ymm17, %ymm0
+; AVX512BF16-NEXT: vinserti128 $1, %xmm9, %ymm1, %ymm1
+; AVX512BF16-NEXT: vinserti64x4 $1, %ymm0, %zmm1, %zmm0
+; AVX512BF16-NEXT: retq
+ %r = call <32 x bfloat> @llvm.floor.v32bf16(<32 x bfloat> %a)
+ ret <32 x bfloat> %r
+}
+
+; Narrower than a full vector register: widened, not scalarized.
+define <2 x bfloat> @floor_v2bf16(<2 x bfloat> %a) {
+; AVX10_2-LABEL: floor_v2bf16:
+; AVX10_2: # %bb.0:
+; AVX10_2-NEXT: vrndscalebf16 $9, %xmm0, %xmm0
+; AVX10_2-NEXT: retq
+;
+; AVX512BF16-LABEL: floor_v2bf16:
+; AVX512BF16: # %bb.0:
+; AVX512BF16-NEXT: vpextrw $1, %xmm0, %eax
+; AVX512BF16-NEXT: shll $16, %eax
+; AVX512BF16-NEXT: vmovd %eax, %xmm1
+; AVX512BF16-NEXT: vroundss $9, %xmm1, %xmm1, %xmm1
+; AVX512BF16-NEXT: vcvtneps2bf16 %xmm1, %xmm1
+; AVX512BF16-NEXT: vmovd %xmm0, %eax
+; AVX512BF16-NEXT: shll $16, %eax
+; AVX512BF16-NEXT: vmovd %eax, %xmm0
+; AVX512BF16-NEXT: vroundss $9, %xmm0, %xmm0, %xmm0
+; AVX512BF16-NEXT: vcvtneps2bf16 %xmm0, %xmm0
+; AVX512BF16-NEXT: vpunpcklwd {{.*#+}} xmm0 = xmm0[0],xmm1[0],xmm0[1],xmm1[1],xmm0[2],xmm1[2],xmm0[3],xmm1[3]
+; AVX512BF16-NEXT: retq
+ %r = call <2 x bfloat> @llvm.floor.v2bf16(<2 x bfloat> %a)
+ ret <2 x bfloat> %r
+}
+
+; Folded load operand.
+define <8 x bfloat> @floor_v8bf16_load(ptr %p) {
+; AVX10_2-LABEL: floor_v8bf16_load:
+; AVX10_2: # %bb.0:
+; AVX10_2-NEXT: vrndscalebf16 $9, (%rdi), %xmm0
+; AVX10_2-NEXT: retq
+;
+; AVX512BF16-LABEL: floor_v8bf16_load:
+; AVX512BF16: # %bb.0:
+; AVX512BF16-NEXT: vmovdqa (%rdi), %xmm0
+; AVX512BF16-NEXT: vpextrw $7, %xmm0, %eax
+; AVX512BF16-NEXT: shll $16, %eax
+; AVX512BF16-NEXT: vmovd %eax, %xmm1
+; AVX512BF16-NEXT: vroundss $9, %xmm1, %xmm1, %xmm1
+; AVX512BF16-NEXT: vcvtneps2bf16 %xmm1, %xmm1
+; AVX512BF16-NEXT: vmovd %xmm1, %eax
+; AVX512BF16-NEXT: vpextrw $6, %xmm0, %ecx
+; AVX512BF16-NEXT: shll $16, %ecx
+; AVX512BF16-NEXT: vmovd %ecx, %xmm1
+; AVX512BF16-NEXT: vroundss $9, %xmm1, %xmm1, %xmm1
+; AVX512BF16-NEXT: vcvtneps2bf16 %xmm1, %xmm1
+; AVX512BF16-NEXT: vmovd %xmm1, %ecx
+; AVX512BF16-NEXT: vpextrw $5, %xmm0, %edx
+; AVX512BF16-NEXT: shll $16, %edx
+; AVX512BF16-NEXT: vmovd %edx, %xmm1
+; AVX512BF16-NEXT: vroundss $9, %xmm1, %xmm1, %xmm1
+; AVX512BF16-NEXT: vcvtneps2bf16 %xmm1, %xmm1
+; AVX512BF16-NEXT: vmovd %xmm1, %edx
+; AVX512BF16-NEXT: vpextrw $4, %xmm0, %esi
+; AVX512BF16-NEXT: shll $16, %esi
+; AVX512BF16-NEXT: vmovd %esi, %xmm1
+; AVX512BF16-NEXT: vroundss $9, %xmm1, %xmm1, %xmm1
+; AVX512BF16-NEXT: vcvtneps2bf16 %xmm1, %xmm1
+; AVX512BF16-NEXT: vmovd %xmm1, %esi
+; AVX512BF16-NEXT: vpextrw $3, %xmm0, %edi
+; AVX512BF16-NEXT: shll $16, %edi
+; AVX512BF16-NEXT: vmovd %edi, %xmm1
+; AVX512BF16-NEXT: vroundss $9, %xmm1, %xmm1, %xmm1
+; AVX512BF16-NEXT: vcvtneps2bf16 %xmm1, %xmm1
+; AVX512BF16-NEXT: vmovd %xmm1, %edi
+; AVX512BF16-NEXT: vpextrw $2, %xmm0, %r8d
+; AVX512BF16-NEXT: shll $16, %r8d
+; AVX512BF16-NEXT: vmovd %r8d, %xmm1
+; AVX512BF16-NEXT: vroundss $9, %xmm1, %xmm1, %xmm1
+; AVX512BF16-NEXT: vcvtneps2bf16 %xmm1, %xmm1
+; AVX512BF16-NEXT: vmovd %xmm1, %r8d
+; AVX512BF16-NEXT: vpextrw $1, %xmm0, %r9d
+; AVX512BF16-NEXT: shll $16, %r9d
+; AVX512BF16-NEXT: vmovd %r9d, %xmm1
+; AVX512BF16-NEXT: vroundss $9, %xmm1, %xmm1, %xmm1
+; AVX512BF16-NEXT: vcvtneps2bf16 %xmm1, %xmm1
+; AVX512BF16-NEXT: vmovd %xmm0, %r9d
+; AVX512BF16-NEXT: shll $16, %r9d
+; AVX512BF16-NEXT: vmovd %r9d, %xmm0
+; AVX512BF16-NEXT: vroundss $9, %xmm0, %xmm0, %xmm0
+; AVX512BF16-NEXT: vcvtneps2bf16 %xmm0, %xmm0
+; AVX512BF16-NEXT: vpunpcklwd {{.*#+}} xmm0 = xmm0[0],xmm1[0],xmm0[1],xmm1[1],xmm0[2],xmm1[2],xmm0[3],xmm1[3]
+; AVX512BF16-NEXT: vpinsrw $2, %r8d, %xmm0, %xmm0
+; AVX512BF16-NEXT: vpinsrw $3, %edi, %xmm0, %xmm0
+; AVX512BF16-NEXT: vpinsrw $4, %esi, %xmm0, %xmm0
+; AVX512BF16-NEXT: vpinsrw $5, %edx, %xmm0, %xmm0
+; AVX512BF16-NEXT: vpinsrw $6, %ecx, %xmm0, %xmm0
+; AVX512BF16-NEXT: vpinsrw $7, %eax, %xmm0, %xmm0
+; AVX512BF16-NEXT: retq
+ %a = load <8 x bfloat>, ptr %p
+ %r = call <8 x bfloat> @llvm.floor.v8bf16(<8 x bfloat> %a)
+ ret <8 x bfloat> %r
+}
+
+; Masked forms, exercising the AVX512_maskable patterns.
+define <8 x bfloat> @floor_v8bf16_mask(<8 x bfloat> %a, <8 x bfloat> %src, i8 %msk) {
+; AVX10_2-LABEL: floor_v8bf16_mask:
+; AVX10_2: # %bb.0:
+; AVX10_2-NEXT: kmovd %edi, %k1
+; AVX10_2-NEXT: vrndscalebf16 $9, %xmm0, %xmm1 {%k1}
+; AVX10_2-NEXT: vmovdqa %xmm1, %xmm0
+; AVX10_2-NEXT: retq
+;
+; AVX512BF16-LABEL: floor_v8bf16_mask:
+; AVX512BF16: # %bb.0:
+; AVX512BF16-NEXT: kmovd %edi, %k1
+; AVX512BF16-NEXT: vpextrw $7, %xmm0, %eax
+; AVX512BF16-NEXT: shll $16, %eax
+; AVX512BF16-NEXT: vmovd %eax, %xmm2
+; AVX512BF16-NEXT: vroundss $9, %xmm2, %xmm2, %xmm2
+; AVX512BF16-NEXT: vcvtneps2bf16 %xmm2, %xmm2
+; AVX512BF16-NEXT: vmovd %xmm2, %eax
+; AVX512BF16-NEXT: vpextrw $6, %xmm0, %ecx
+; AVX512BF16-NEXT: shll $16, %ecx
+; AVX512BF16-NEXT: vmovd %ecx, %xmm2
+; AVX512BF16-NEXT: vroundss $9, %xmm2, %xmm2, %xmm2
+; AVX512BF16-NEXT: vcvtneps2bf16 %xmm2, %xmm2
+; AVX512BF16-NEXT: vmovd %xmm2, %ecx
+; AVX512BF16-NEXT: vpextrw $5, %xmm0, %edx
+; AVX512BF16-NEXT: shll $16, %edx
+; AVX512BF16-NEXT: vmovd %edx, %xmm2
+; AVX512BF16-NEXT: vroundss $9, %xmm2, %xmm2, %xmm2
+; AVX512BF16-NEXT: vcvtneps2bf16 %xmm2, %xmm2
+; AVX512BF16-NEXT: vmovd %xmm2, %edx
+; AVX512BF16-NEXT: vpextrw $4, %xmm0, %esi
+; AVX512BF16-NEXT: shll $16, %esi
+; AVX512BF16-NEXT: vmovd %esi, %xmm2
+; AVX512BF16-NEXT: vroundss $9, %xmm2, %xmm2, %xmm2
+; AVX512BF16-NEXT: vcvtneps2bf16 %xmm2, %xmm2
+; AVX512BF16-NEXT: vmovd %xmm2, %esi
+; AVX512BF16-NEXT: vpextrw $3, %xmm0, %edi
+; AVX512BF16-NEXT: shll $16, %edi
+; AVX512BF16-NEXT: vmovd %edi, %xmm2
+; AVX512BF16-NEXT: vroundss $9, %xmm2, %xmm2, %xmm2
+; AVX512BF16-NEXT: vcvtneps2bf16 %xmm2, %xmm2
+; AVX512BF16-NEXT: vmovd %xmm2, %edi
+; AVX512BF16-NEXT: vpextrw $2, %xmm0, %r8d
+; AVX512BF16-NEXT: shll $16, %r8d
+; AVX512BF16-NEXT: vmovd %r8d, %xmm2
+; AVX512BF16-NEXT: vroundss $9, %xmm2, %xmm2, %xmm2
+; AVX512BF16-NEXT: vcvtneps2bf16 %xmm2, %xmm2
+; AVX512BF16-NEXT: vmovd %xmm2, %r8d
+; AVX512BF16-NEXT: vpextrw $1, %xmm0, %r9d
+; AVX512BF16-NEXT: shll $16, %r9d
+; AVX512BF16-NEXT: vmovd %r9d, %xmm2
+; AVX512BF16-NEXT: vroundss $9, %xmm2, %xmm2, %xmm2
+; AVX512BF16-NEXT: vcvtneps2bf16 %xmm2, %xmm2
+; AVX512BF16-NEXT: vmovd %xmm0, %r9d
+; AVX512BF16-NEXT: shll $16, %r9d
+; AVX512BF16-NEXT: vmovd %r9d, %xmm0
+; AVX512BF16-NEXT: vroundss $9, %xmm0, %xmm0, %xmm0
+; AVX512BF16-NEXT: vcvtneps2bf16 %xmm0, %xmm0
+; AVX512BF16-NEXT: vpunpcklwd {{.*#+}} xmm0 = xmm0[0],xmm2[0],xmm0[1],xmm2[1],xmm0[2],xmm2[2],xmm0[3],xmm2[3]
+; AVX512BF16-NEXT: vpinsrw $2, %r8d, %xmm0, %xmm0
+; AVX512BF16-NEXT: vpinsrw $3, %edi, %xmm0, %xmm0
+; AVX512BF16-NEXT: vpinsrw $4, %esi, %xmm0, %xmm0
+; AVX512BF16-NEXT: vpinsrw $5, %edx, %xmm0, %xmm0
+; AVX512BF16-NEXT: vpinsrw $6, %ecx, %xmm0, %xmm0
+; AVX512BF16-NEXT: vpinsrw $7, %eax, %xmm0, %xmm0
+; AVX512BF16-NEXT: vpblendmw %xmm0, %xmm1, %xmm0 {%k1}
+; AVX512BF16-NEXT: retq
+ %mask = bitcast i8 %msk to <8 x i1>
+ %r = call <8 x bfloat> @llvm.floor.v8bf16(<8 x bfloat> %a)
+ %res = select <8 x i1> %mask, <8 x bfloat> %r, <8 x bfloat> %src
+ ret <8 x bfloat> %res
+}
+
+define <8 x bfloat> @floor_v8bf16_maskz(<8 x bfloat> %a, i8 %msk) {
+; AVX10_2-LABEL: floor_v8bf16_maskz:
+; AVX10_2: # %bb.0:
+; AVX10_2-NEXT: kmovd %edi, %k1
+; AVX10_2-NEXT: vrndscalebf16 $9, %xmm0, %xmm0 {%k1} {z}
+; AVX10_2-NEXT: retq
+;
+; AVX512BF16-LABEL: floor_v8bf16_maskz:
+; AVX512BF16: # %bb.0:
+; AVX512BF16-NEXT: kmovd %edi, %k1
+; AVX512BF16-NEXT: vpextrw $7, %xmm0, %eax
+; AVX512BF16-NEXT: shll $16, %eax
+; AVX512BF16-NEXT: vmovd %eax, %xmm1
+; AVX512BF16-NEXT: vroundss $9, %xmm1, %xmm1, %xmm1
+; AVX512BF16-NEXT: vcvtneps2bf16 %xmm1, %xmm1
+; AVX512BF16-NEXT: vmovd %xmm1, %eax
+; AVX512BF16-NEXT: vpextrw $6, %xmm0, %ecx
+; AVX512BF16-NEXT: shll $16, %ecx
+; AVX512BF16-NEXT: vmovd %ecx, %xmm1
+; AVX512BF16-NEXT: vroundss $9, %xmm1, %xmm1, %xmm1
+; AVX512BF16-NEXT: vcvtneps2bf16 %xmm1, %xmm1
+; AVX512BF16-NEXT: vmovd %xmm1, %ecx
+; AVX512BF16-NEXT: vpextrw $5, %xmm0, %edx
+; AVX512BF16-NEXT: shll $16, %edx
+; AVX512BF16-NEXT: vmovd %edx, %xmm1
+; AVX512BF16-NEXT: vroundss $9, %xmm1, %xmm1, %xmm1
+; AVX512BF16-NEXT: vcvtneps2bf16 %xmm1, %xmm1
+; AVX512BF16-NEXT: vmovd %xmm1, %edx
+; AVX512BF16-NEXT: vpextrw $4, %xmm0, %esi
+; AVX512BF16-NEXT: shll $16, %esi
+; AVX512BF16-NEXT: vmovd %esi, %xmm1
+; AVX512BF16-NEXT: vroundss $9, %xmm1, %xmm1, %xmm1
+; AVX512BF16-NEXT: vcvtneps2bf16 %xmm1, %xmm1
+; AVX512BF16-NEXT: vmovd %xmm1, %esi
+; AVX512BF16-NEXT: vpextrw $3, %xmm0, %edi
+; AVX512BF16-NEXT: shll $16, %edi
+; AVX512BF16-NEXT: vmovd %edi, %xmm1
+; AVX512BF16-NEXT: vroundss $9, %xmm1, %xmm1, %xmm1
+; AVX512BF16-NEXT: vcvtneps2bf16 %xmm1, %xmm1
+; AVX512BF16-NEXT: vmovd %xmm1, %edi
+; AVX512BF16-NEXT: vpextrw $2, %xmm0, %r8d
+; AVX512BF16-NEXT: shll $16, %r8d
+; AVX512BF16-NEXT: vmovd %r8d, %xmm1
+; AVX512BF16-NEXT: vroundss $9, %xmm1, %xmm1, %xmm1
+; AVX512BF16-NEXT: vcvtneps2bf16 %xmm1, %xmm1
+; AVX512BF16-NEXT: vmovd %xmm1, %r8d
+; AVX512BF16-NEXT: vpextrw $1, %xmm0, %r9d
+; AVX512BF16-NEXT: shll $16, %r9d
+; AVX512BF16-NEXT: vmovd %r9d, %xmm1
+; AVX512BF16-NEXT: vroundss $9, %xmm1, %xmm1, %xmm1
+; AVX512BF16-NEXT: vcvtneps2bf16 %xmm1, %xmm1
+; AVX512BF16-NEXT: vmovd %xmm0, %r9d
+; AVX512BF16-NEXT: shll $16, %r9d
+; AVX512BF16-NEXT: vmovd %r9d, %xmm0
+; AVX512BF16-NEXT: vroundss $9, %xmm0, %xmm0, %xmm0
+; AVX512BF16-NEXT: vcvtneps2bf16 %xmm0, %xmm0
+; AVX512BF16-NEXT: vpunpcklwd {{.*#+}} xmm0 = xmm0[0],xmm1[0],xmm0[1],xmm1[1],xmm0[2],xmm1[2],xmm0[3],xmm1[3]
+; AVX512BF16-NEXT: vpinsrw $2, %r8d, %xmm0, %xmm0
+; AVX512BF16-NEXT: vpinsrw $3, %edi, %xmm0, %xmm0
+; AVX512BF16-NEXT: vpinsrw $4, %esi, %xmm0, %xmm0
+; AVX512BF16-NEXT: vpinsrw $5, %edx, %xmm0, %xmm0
+; AVX512BF16-NEXT: vpinsrw $6, %ecx, %xmm0, %xmm0
+; AVX512BF16-NEXT: vpinsrw $7, %eax, %xmm0, %xmm0
+; AVX512BF16-NEXT: vmovdqu16 %xmm0, %xmm0 {%k1} {z}
+; AVX512BF16-NEXT: retq
+ %mask = bitcast i8 %msk to <8 x i1>
+ %r = call <8 x bfloat> @llvm.floor.v8bf16(<8 x bfloat> %a)
+ %res = select <8 x i1> %mask, <8 x bfloat> %r, <8 x bfloat> zeroinitializer
+ ret <8 x bfloat> %res
+}
+
+; llvm.round is round-half-away-from-zero, which vrndscale cannot encode, so it
+; must stay scalarized rather than becoming a vrndscalebf16.
+define <8 x bfloat> @round_v8bf16(<8 x bfloat> %a) {
+; AVX10_2-LABEL: round_v8bf16:
+; AVX10_2-NOT: vrndscalebf16
+; AVX10_2: vroundss $11,
+; AVX10_2-NOT: vrndscalebf16
+ %r = call <8 x bfloat> @llvm.round.v8bf16(<8 x bfloat> %a)
+ ret <8 x bfloat> %r
+}
More information about the llvm-commits
mailing list