[llvm] [X86] Lower bf16 round-to-integer on AVX10.2 via vrndscalebf16 (PR #213276)
via llvm-commits
llvm-commits at lists.llvm.org
Thu Aug 27 06:49:20 PDT 2026
https://github.com/tfzee updated https://github.com/llvm/llvm-project/pull/213276
>From 92bd9b0d6f0a64b8509574e187754bcb62aa306e Mon Sep 17 00:00:00 2001
From: "Ziegler, Tim" <tim.ziegler at intel.com>
Date: Fri, 31 Jul 2026 14:57:44 +0200
Subject: [PATCH 1/2] Legalizing the common (non strict) rounding bf16
instructions to use vrndscalebf16
---
llvm/lib/Target/X86/X86ISelLowering.cpp | 24 +-
llvm/test/CodeGen/X86/avx10_2bf16-rounding.ll | 1145 +++++++++++++++++
2 files changed, 1164 insertions(+), 5 deletions(-)
create mode 100644 llvm/test/CodeGen/X86/avx10_2bf16-rounding.ll
diff --git a/llvm/lib/Target/X86/X86ISelLowering.cpp b/llvm/lib/Target/X86/X86ISelLowering.cpp
index 712db68314e6e..a11152b7db66e 100644
--- a/llvm/lib/Target/X86/X86ISelLowering.cpp
+++ b/llvm/lib/Target/X86/X86ISelLowering.cpp
@@ -2610,6 +2610,10 @@ X86TargetLowering::X86TargetLowering(const X86TargetMachine &TM,
setOperationAction(ISD::FSQRT, MVT::bf16, Custom);
setOperationAction(ISD::FMA, MVT::bf16, Custom);
+ for (unsigned Opc : {ISD::FFLOOR, ISD::FCEIL, ISD::FTRUNC, ISD::FRINT,
+ ISD::FNEARBYINT, ISD::FROUNDEVEN})
+ setOperationAction(Opc, MVT::bf16, Custom);
+
setOperationAction(ISD::FADD, MVT::v32bf16, Legal);
setOperationAction(ISD::FSUB, MVT::v32bf16, Legal);
setOperationAction(ISD::FMUL, MVT::v32bf16, Legal);
@@ -2618,6 +2622,10 @@ X86TargetLowering::X86TargetLowering(const X86TargetMachine &TM,
setOperationAction(ISD::FMA, MVT::v32bf16, Legal);
setOperationAction(ISD::SETCC, MVT::v32bf16, Custom);
SetFPMinMaxAction(MVT::v32bf16);
+ for (auto VT : {MVT::v8bf16, MVT::v16bf16, MVT::v32bf16})
+ for (unsigned Opc : {ISD::FFLOOR, ISD::FCEIL, ISD::FTRUNC, ISD::FRINT,
+ ISD::FNEARBYINT, ISD::FROUNDEVEN})
+ setOperationAction(Opc, VT, Legal);
for (auto VT : {MVT::v8bf16, MVT::v16bf16}) {
setOperationAction(ISD::FADD, VT, Legal);
setOperationAction(ISD::FSUB, VT, Legal);
@@ -34549,12 +34557,18 @@ void X86TargetLowering::ReplaceNodeResults(SDNode *N,
case ISD::FMUL:
case ISD::FSQRT:
case ISD::FDIV:
- case ISD::FMA: {
+ case ISD::FMA:
+ case ISD::FFLOOR:
+ case ISD::FCEIL:
+ case ISD::FTRUNC:
+ case ISD::FRINT:
+ case ISD::FNEARBYINT:
+ case ISD::FROUNDEVEN: {
assert(N->getValueType(0) == MVT::bf16 && "Expected scalar bf16 result");
- // AVX10.2 has no scalar bf16 arithmetic instructions, and bf16 is a
- // soft-promoted-half type, so scalar ops would otherwise be promoted to
- // f32. Instead widen each operand to a v8bf16 vector, perform the legal
- // packed operation, and extract the low element afterwards.
+ // AVX10.2 has no scalar bf16 arithmetic or round-to-integer instructions,
+ // and bf16 is a soft-promoted-half type, so scalar ops would otherwise be
+ // promoted to f32. Instead widen each operand to a v8bf16 vector, perform
+ // the legal packed operation, and extract the low element afterwards.
SmallVector<SDValue, 3> VecOps;
for (const SDValue &Op : N->ops()) {
SDValue AsF16 = DAG.getBitcast(MVT::f16, Op);
diff --git a/llvm/test/CodeGen/X86/avx10_2bf16-rounding.ll b/llvm/test/CodeGen/X86/avx10_2bf16-rounding.ll
new file mode 100644
index 0000000000000..4b9b764a6f5ce
--- /dev/null
+++ b/llvm/test/CodeGen/X86/avx10_2bf16-rounding.ll
@@ -0,0 +1,1145 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
+; RUN: llc < %s -verify-machineinstrs -mtriple=x86_64-unknown-unknown -mattr=+avx10.2 | FileCheck %s --check-prefixes=AVX10_2
+; RUN: llc < %s -verify-machineinstrs -mtriple=x86_64-unknown-unknown -mattr=+avx512bf16,+avx512vl | FileCheck %s --check-prefixes=AVX512BF16
+
+; AVX10.2 has packed bf16 round-to-integer instructions. The integral result of
+; floor/ceil/trunc/rint/nearbyint/roundeven is always representable in bf16, so
+; vrndscalebf16 gives the same answer as the AVX512BF16 sequence below (promote
+; each element to f32, round, convert back), but without scalarizing.
+
+define <8 x bfloat> @floor_v8bf16(<8 x bfloat> %a) {
+; AVX10_2-LABEL: floor_v8bf16:
+; AVX10_2: # %bb.0:
+; AVX10_2-NEXT: vrndscalebf16 $9, %xmm0, %xmm0
+; AVX10_2-NEXT: retq
+;
+; AVX512BF16-LABEL: floor_v8bf16:
+; AVX512BF16: # %bb.0:
+; AVX512BF16-NEXT: vpextrw $7, %xmm0, %eax
+; AVX512BF16-NEXT: shll $16, %eax
+; AVX512BF16-NEXT: vmovd %eax, %xmm1
+; AVX512BF16-NEXT: vroundss $9, %xmm1, %xmm1, %xmm1
+; AVX512BF16-NEXT: vcvtneps2bf16 %xmm1, %xmm1
+; AVX512BF16-NEXT: vmovd %xmm1, %eax
+; AVX512BF16-NEXT: vpextrw $6, %xmm0, %ecx
+; AVX512BF16-NEXT: shll $16, %ecx
+; AVX512BF16-NEXT: vmovd %ecx, %xmm1
+; AVX512BF16-NEXT: vroundss $9, %xmm1, %xmm1, %xmm1
+; AVX512BF16-NEXT: vcvtneps2bf16 %xmm1, %xmm1
+; AVX512BF16-NEXT: vmovd %xmm1, %ecx
+; AVX512BF16-NEXT: vpextrw $5, %xmm0, %edx
+; AVX512BF16-NEXT: shll $16, %edx
+; AVX512BF16-NEXT: vmovd %edx, %xmm1
+; AVX512BF16-NEXT: vroundss $9, %xmm1, %xmm1, %xmm1
+; AVX512BF16-NEXT: vcvtneps2bf16 %xmm1, %xmm1
+; AVX512BF16-NEXT: vmovd %xmm1, %edx
+; AVX512BF16-NEXT: vpextrw $4, %xmm0, %esi
+; AVX512BF16-NEXT: shll $16, %esi
+; AVX512BF16-NEXT: vmovd %esi, %xmm1
+; AVX512BF16-NEXT: vroundss $9, %xmm1, %xmm1, %xmm1
+; AVX512BF16-NEXT: vcvtneps2bf16 %xmm1, %xmm1
+; AVX512BF16-NEXT: vmovd %xmm1, %esi
+; AVX512BF16-NEXT: vpextrw $3, %xmm0, %edi
+; AVX512BF16-NEXT: shll $16, %edi
+; AVX512BF16-NEXT: vmovd %edi, %xmm1
+; AVX512BF16-NEXT: vroundss $9, %xmm1, %xmm1, %xmm1
+; AVX512BF16-NEXT: vcvtneps2bf16 %xmm1, %xmm1
+; AVX512BF16-NEXT: vmovd %xmm1, %edi
+; AVX512BF16-NEXT: vpextrw $2, %xmm0, %r8d
+; AVX512BF16-NEXT: shll $16, %r8d
+; AVX512BF16-NEXT: vmovd %r8d, %xmm1
+; AVX512BF16-NEXT: vroundss $9, %xmm1, %xmm1, %xmm1
+; AVX512BF16-NEXT: vcvtneps2bf16 %xmm1, %xmm1
+; AVX512BF16-NEXT: vmovd %xmm1, %r8d
+; AVX512BF16-NEXT: vpextrw $1, %xmm0, %r9d
+; AVX512BF16-NEXT: shll $16, %r9d
+; AVX512BF16-NEXT: vmovd %r9d, %xmm1
+; AVX512BF16-NEXT: vroundss $9, %xmm1, %xmm1, %xmm1
+; AVX512BF16-NEXT: vcvtneps2bf16 %xmm1, %xmm1
+; AVX512BF16-NEXT: vmovd %xmm0, %r9d
+; AVX512BF16-NEXT: shll $16, %r9d
+; AVX512BF16-NEXT: vmovd %r9d, %xmm0
+; AVX512BF16-NEXT: vroundss $9, %xmm0, %xmm0, %xmm0
+; AVX512BF16-NEXT: vcvtneps2bf16 %xmm0, %xmm0
+; AVX512BF16-NEXT: vpunpcklwd {{.*#+}} xmm0 = xmm0[0],xmm1[0],xmm0[1],xmm1[1],xmm0[2],xmm1[2],xmm0[3],xmm1[3]
+; AVX512BF16-NEXT: vpinsrw $2, %r8d, %xmm0, %xmm0
+; AVX512BF16-NEXT: vpinsrw $3, %edi, %xmm0, %xmm0
+; AVX512BF16-NEXT: vpinsrw $4, %esi, %xmm0, %xmm0
+; AVX512BF16-NEXT: vpinsrw $5, %edx, %xmm0, %xmm0
+; AVX512BF16-NEXT: vpinsrw $6, %ecx, %xmm0, %xmm0
+; AVX512BF16-NEXT: vpinsrw $7, %eax, %xmm0, %xmm0
+; AVX512BF16-NEXT: retq
+ %r = call <8 x bfloat> @llvm.floor.v8bf16(<8 x bfloat> %a)
+ ret <8 x bfloat> %r
+}
+
+define bfloat @floor_bf16(bfloat %a) {
+; AVX10_2-LABEL: floor_bf16:
+; AVX10_2: # %bb.0:
+; AVX10_2-NEXT: vrndscalebf16 $9, %xmm0, %xmm0
+; AVX10_2-NEXT: retq
+;
+; AVX512BF16-LABEL: floor_bf16:
+; AVX512BF16: # %bb.0:
+; AVX512BF16-NEXT: vpextrw $0, %xmm0, %eax
+; AVX512BF16-NEXT: shll $16, %eax
+; AVX512BF16-NEXT: vmovd %eax, %xmm0
+; AVX512BF16-NEXT: vroundss $9, %xmm0, %xmm0, %xmm0
+; AVX512BF16-NEXT: vcvtneps2bf16 %xmm0, %xmm0
+; AVX512BF16-NEXT: retq
+ %r = call bfloat @llvm.floor.bf16(bfloat %a)
+ ret bfloat %r
+}
+
+define <8 x bfloat> @ceil_v8bf16(<8 x bfloat> %a) {
+; AVX10_2-LABEL: ceil_v8bf16:
+; AVX10_2: # %bb.0:
+; AVX10_2-NEXT: vrndscalebf16 $10, %xmm0, %xmm0
+; AVX10_2-NEXT: retq
+;
+; AVX512BF16-LABEL: ceil_v8bf16:
+; AVX512BF16: # %bb.0:
+; AVX512BF16-NEXT: vpextrw $7, %xmm0, %eax
+; AVX512BF16-NEXT: shll $16, %eax
+; AVX512BF16-NEXT: vmovd %eax, %xmm1
+; AVX512BF16-NEXT: vroundss $10, %xmm1, %xmm1, %xmm1
+; AVX512BF16-NEXT: vcvtneps2bf16 %xmm1, %xmm1
+; AVX512BF16-NEXT: vmovd %xmm1, %eax
+; AVX512BF16-NEXT: vpextrw $6, %xmm0, %ecx
+; AVX512BF16-NEXT: shll $16, %ecx
+; AVX512BF16-NEXT: vmovd %ecx, %xmm1
+; AVX512BF16-NEXT: vroundss $10, %xmm1, %xmm1, %xmm1
+; AVX512BF16-NEXT: vcvtneps2bf16 %xmm1, %xmm1
+; AVX512BF16-NEXT: vmovd %xmm1, %ecx
+; AVX512BF16-NEXT: vpextrw $5, %xmm0, %edx
+; AVX512BF16-NEXT: shll $16, %edx
+; AVX512BF16-NEXT: vmovd %edx, %xmm1
+; AVX512BF16-NEXT: vroundss $10, %xmm1, %xmm1, %xmm1
+; AVX512BF16-NEXT: vcvtneps2bf16 %xmm1, %xmm1
+; AVX512BF16-NEXT: vmovd %xmm1, %edx
+; AVX512BF16-NEXT: vpextrw $4, %xmm0, %esi
+; AVX512BF16-NEXT: shll $16, %esi
+; AVX512BF16-NEXT: vmovd %esi, %xmm1
+; AVX512BF16-NEXT: vroundss $10, %xmm1, %xmm1, %xmm1
+; AVX512BF16-NEXT: vcvtneps2bf16 %xmm1, %xmm1
+; AVX512BF16-NEXT: vmovd %xmm1, %esi
+; AVX512BF16-NEXT: vpextrw $3, %xmm0, %edi
+; AVX512BF16-NEXT: shll $16, %edi
+; AVX512BF16-NEXT: vmovd %edi, %xmm1
+; AVX512BF16-NEXT: vroundss $10, %xmm1, %xmm1, %xmm1
+; AVX512BF16-NEXT: vcvtneps2bf16 %xmm1, %xmm1
+; AVX512BF16-NEXT: vmovd %xmm1, %edi
+; AVX512BF16-NEXT: vpextrw $2, %xmm0, %r8d
+; AVX512BF16-NEXT: shll $16, %r8d
+; AVX512BF16-NEXT: vmovd %r8d, %xmm1
+; AVX512BF16-NEXT: vroundss $10, %xmm1, %xmm1, %xmm1
+; AVX512BF16-NEXT: vcvtneps2bf16 %xmm1, %xmm1
+; AVX512BF16-NEXT: vmovd %xmm1, %r8d
+; AVX512BF16-NEXT: vpextrw $1, %xmm0, %r9d
+; AVX512BF16-NEXT: shll $16, %r9d
+; AVX512BF16-NEXT: vmovd %r9d, %xmm1
+; AVX512BF16-NEXT: vroundss $10, %xmm1, %xmm1, %xmm1
+; AVX512BF16-NEXT: vcvtneps2bf16 %xmm1, %xmm1
+; AVX512BF16-NEXT: vmovd %xmm0, %r9d
+; AVX512BF16-NEXT: shll $16, %r9d
+; AVX512BF16-NEXT: vmovd %r9d, %xmm0
+; AVX512BF16-NEXT: vroundss $10, %xmm0, %xmm0, %xmm0
+; AVX512BF16-NEXT: vcvtneps2bf16 %xmm0, %xmm0
+; AVX512BF16-NEXT: vpunpcklwd {{.*#+}} xmm0 = xmm0[0],xmm1[0],xmm0[1],xmm1[1],xmm0[2],xmm1[2],xmm0[3],xmm1[3]
+; AVX512BF16-NEXT: vpinsrw $2, %r8d, %xmm0, %xmm0
+; AVX512BF16-NEXT: vpinsrw $3, %edi, %xmm0, %xmm0
+; AVX512BF16-NEXT: vpinsrw $4, %esi, %xmm0, %xmm0
+; AVX512BF16-NEXT: vpinsrw $5, %edx, %xmm0, %xmm0
+; AVX512BF16-NEXT: vpinsrw $6, %ecx, %xmm0, %xmm0
+; AVX512BF16-NEXT: vpinsrw $7, %eax, %xmm0, %xmm0
+; AVX512BF16-NEXT: retq
+ %r = call <8 x bfloat> @llvm.ceil.v8bf16(<8 x bfloat> %a)
+ ret <8 x bfloat> %r
+}
+
+define bfloat @ceil_bf16(bfloat %a) {
+; AVX10_2-LABEL: ceil_bf16:
+; AVX10_2: # %bb.0:
+; AVX10_2-NEXT: vrndscalebf16 $10, %xmm0, %xmm0
+; AVX10_2-NEXT: retq
+;
+; AVX512BF16-LABEL: ceil_bf16:
+; AVX512BF16: # %bb.0:
+; AVX512BF16-NEXT: vpextrw $0, %xmm0, %eax
+; AVX512BF16-NEXT: shll $16, %eax
+; AVX512BF16-NEXT: vmovd %eax, %xmm0
+; AVX512BF16-NEXT: vroundss $10, %xmm0, %xmm0, %xmm0
+; AVX512BF16-NEXT: vcvtneps2bf16 %xmm0, %xmm0
+; AVX512BF16-NEXT: retq
+ %r = call bfloat @llvm.ceil.bf16(bfloat %a)
+ ret bfloat %r
+}
+
+define <8 x bfloat> @trunc_v8bf16(<8 x bfloat> %a) {
+; AVX10_2-LABEL: trunc_v8bf16:
+; AVX10_2: # %bb.0:
+; AVX10_2-NEXT: vrndscalebf16 $11, %xmm0, %xmm0
+; AVX10_2-NEXT: retq
+;
+; AVX512BF16-LABEL: trunc_v8bf16:
+; AVX512BF16: # %bb.0:
+; AVX512BF16-NEXT: vpextrw $7, %xmm0, %eax
+; AVX512BF16-NEXT: shll $16, %eax
+; AVX512BF16-NEXT: vmovd %eax, %xmm1
+; AVX512BF16-NEXT: vroundss $11, %xmm1, %xmm1, %xmm1
+; AVX512BF16-NEXT: vcvtneps2bf16 %xmm1, %xmm1
+; AVX512BF16-NEXT: vmovd %xmm1, %eax
+; AVX512BF16-NEXT: vpextrw $6, %xmm0, %ecx
+; AVX512BF16-NEXT: shll $16, %ecx
+; AVX512BF16-NEXT: vmovd %ecx, %xmm1
+; AVX512BF16-NEXT: vroundss $11, %xmm1, %xmm1, %xmm1
+; AVX512BF16-NEXT: vcvtneps2bf16 %xmm1, %xmm1
+; AVX512BF16-NEXT: vmovd %xmm1, %ecx
+; AVX512BF16-NEXT: vpextrw $5, %xmm0, %edx
+; AVX512BF16-NEXT: shll $16, %edx
+; AVX512BF16-NEXT: vmovd %edx, %xmm1
+; AVX512BF16-NEXT: vroundss $11, %xmm1, %xmm1, %xmm1
+; AVX512BF16-NEXT: vcvtneps2bf16 %xmm1, %xmm1
+; AVX512BF16-NEXT: vmovd %xmm1, %edx
+; AVX512BF16-NEXT: vpextrw $4, %xmm0, %esi
+; AVX512BF16-NEXT: shll $16, %esi
+; AVX512BF16-NEXT: vmovd %esi, %xmm1
+; AVX512BF16-NEXT: vroundss $11, %xmm1, %xmm1, %xmm1
+; AVX512BF16-NEXT: vcvtneps2bf16 %xmm1, %xmm1
+; AVX512BF16-NEXT: vmovd %xmm1, %esi
+; AVX512BF16-NEXT: vpextrw $3, %xmm0, %edi
+; AVX512BF16-NEXT: shll $16, %edi
+; AVX512BF16-NEXT: vmovd %edi, %xmm1
+; AVX512BF16-NEXT: vroundss $11, %xmm1, %xmm1, %xmm1
+; AVX512BF16-NEXT: vcvtneps2bf16 %xmm1, %xmm1
+; AVX512BF16-NEXT: vmovd %xmm1, %edi
+; AVX512BF16-NEXT: vpextrw $2, %xmm0, %r8d
+; AVX512BF16-NEXT: shll $16, %r8d
+; AVX512BF16-NEXT: vmovd %r8d, %xmm1
+; AVX512BF16-NEXT: vroundss $11, %xmm1, %xmm1, %xmm1
+; AVX512BF16-NEXT: vcvtneps2bf16 %xmm1, %xmm1
+; AVX512BF16-NEXT: vmovd %xmm1, %r8d
+; AVX512BF16-NEXT: vpextrw $1, %xmm0, %r9d
+; AVX512BF16-NEXT: shll $16, %r9d
+; AVX512BF16-NEXT: vmovd %r9d, %xmm1
+; AVX512BF16-NEXT: vroundss $11, %xmm1, %xmm1, %xmm1
+; AVX512BF16-NEXT: vcvtneps2bf16 %xmm1, %xmm1
+; AVX512BF16-NEXT: vmovd %xmm0, %r9d
+; AVX512BF16-NEXT: shll $16, %r9d
+; AVX512BF16-NEXT: vmovd %r9d, %xmm0
+; AVX512BF16-NEXT: vroundss $11, %xmm0, %xmm0, %xmm0
+; AVX512BF16-NEXT: vcvtneps2bf16 %xmm0, %xmm0
+; AVX512BF16-NEXT: vpunpcklwd {{.*#+}} xmm0 = xmm0[0],xmm1[0],xmm0[1],xmm1[1],xmm0[2],xmm1[2],xmm0[3],xmm1[3]
+; AVX512BF16-NEXT: vpinsrw $2, %r8d, %xmm0, %xmm0
+; AVX512BF16-NEXT: vpinsrw $3, %edi, %xmm0, %xmm0
+; AVX512BF16-NEXT: vpinsrw $4, %esi, %xmm0, %xmm0
+; AVX512BF16-NEXT: vpinsrw $5, %edx, %xmm0, %xmm0
+; AVX512BF16-NEXT: vpinsrw $6, %ecx, %xmm0, %xmm0
+; AVX512BF16-NEXT: vpinsrw $7, %eax, %xmm0, %xmm0
+; AVX512BF16-NEXT: retq
+ %r = call <8 x bfloat> @llvm.trunc.v8bf16(<8 x bfloat> %a)
+ ret <8 x bfloat> %r
+}
+
+define bfloat @trunc_bf16(bfloat %a) {
+; AVX10_2-LABEL: trunc_bf16:
+; AVX10_2: # %bb.0:
+; AVX10_2-NEXT: vrndscalebf16 $11, %xmm0, %xmm0
+; AVX10_2-NEXT: retq
+;
+; AVX512BF16-LABEL: trunc_bf16:
+; AVX512BF16: # %bb.0:
+; AVX512BF16-NEXT: vpextrw $0, %xmm0, %eax
+; AVX512BF16-NEXT: shll $16, %eax
+; AVX512BF16-NEXT: vmovd %eax, %xmm0
+; AVX512BF16-NEXT: vroundss $11, %xmm0, %xmm0, %xmm0
+; AVX512BF16-NEXT: vcvtneps2bf16 %xmm0, %xmm0
+; AVX512BF16-NEXT: retq
+ %r = call bfloat @llvm.trunc.bf16(bfloat %a)
+ ret bfloat %r
+}
+
+define <8 x bfloat> @rint_v8bf16(<8 x bfloat> %a) {
+; AVX10_2-LABEL: rint_v8bf16:
+; AVX10_2: # %bb.0:
+; AVX10_2-NEXT: vrndscalebf16 $4, %xmm0, %xmm0
+; AVX10_2-NEXT: retq
+;
+; AVX512BF16-LABEL: rint_v8bf16:
+; AVX512BF16: # %bb.0:
+; AVX512BF16-NEXT: vpextrw $7, %xmm0, %eax
+; AVX512BF16-NEXT: shll $16, %eax
+; AVX512BF16-NEXT: vmovd %eax, %xmm1
+; AVX512BF16-NEXT: vroundss $4, %xmm1, %xmm1, %xmm1
+; AVX512BF16-NEXT: vcvtneps2bf16 %xmm1, %xmm1
+; AVX512BF16-NEXT: vmovd %xmm1, %eax
+; AVX512BF16-NEXT: vpextrw $6, %xmm0, %ecx
+; AVX512BF16-NEXT: shll $16, %ecx
+; AVX512BF16-NEXT: vmovd %ecx, %xmm1
+; AVX512BF16-NEXT: vroundss $4, %xmm1, %xmm1, %xmm1
+; AVX512BF16-NEXT: vcvtneps2bf16 %xmm1, %xmm1
+; AVX512BF16-NEXT: vmovd %xmm1, %ecx
+; AVX512BF16-NEXT: vpextrw $5, %xmm0, %edx
+; AVX512BF16-NEXT: shll $16, %edx
+; AVX512BF16-NEXT: vmovd %edx, %xmm1
+; AVX512BF16-NEXT: vroundss $4, %xmm1, %xmm1, %xmm1
+; AVX512BF16-NEXT: vcvtneps2bf16 %xmm1, %xmm1
+; AVX512BF16-NEXT: vmovd %xmm1, %edx
+; AVX512BF16-NEXT: vpextrw $4, %xmm0, %esi
+; AVX512BF16-NEXT: shll $16, %esi
+; AVX512BF16-NEXT: vmovd %esi, %xmm1
+; AVX512BF16-NEXT: vroundss $4, %xmm1, %xmm1, %xmm1
+; AVX512BF16-NEXT: vcvtneps2bf16 %xmm1, %xmm1
+; AVX512BF16-NEXT: vmovd %xmm1, %esi
+; AVX512BF16-NEXT: vpextrw $3, %xmm0, %edi
+; AVX512BF16-NEXT: shll $16, %edi
+; AVX512BF16-NEXT: vmovd %edi, %xmm1
+; AVX512BF16-NEXT: vroundss $4, %xmm1, %xmm1, %xmm1
+; AVX512BF16-NEXT: vcvtneps2bf16 %xmm1, %xmm1
+; AVX512BF16-NEXT: vmovd %xmm1, %edi
+; AVX512BF16-NEXT: vpextrw $2, %xmm0, %r8d
+; AVX512BF16-NEXT: shll $16, %r8d
+; AVX512BF16-NEXT: vmovd %r8d, %xmm1
+; AVX512BF16-NEXT: vroundss $4, %xmm1, %xmm1, %xmm1
+; AVX512BF16-NEXT: vcvtneps2bf16 %xmm1, %xmm1
+; AVX512BF16-NEXT: vmovd %xmm1, %r8d
+; AVX512BF16-NEXT: vpextrw $1, %xmm0, %r9d
+; AVX512BF16-NEXT: shll $16, %r9d
+; AVX512BF16-NEXT: vmovd %r9d, %xmm1
+; AVX512BF16-NEXT: vroundss $4, %xmm1, %xmm1, %xmm1
+; AVX512BF16-NEXT: vcvtneps2bf16 %xmm1, %xmm1
+; AVX512BF16-NEXT: vmovd %xmm0, %r9d
+; AVX512BF16-NEXT: shll $16, %r9d
+; AVX512BF16-NEXT: vmovd %r9d, %xmm0
+; AVX512BF16-NEXT: vroundss $4, %xmm0, %xmm0, %xmm0
+; AVX512BF16-NEXT: vcvtneps2bf16 %xmm0, %xmm0
+; AVX512BF16-NEXT: vpunpcklwd {{.*#+}} xmm0 = xmm0[0],xmm1[0],xmm0[1],xmm1[1],xmm0[2],xmm1[2],xmm0[3],xmm1[3]
+; AVX512BF16-NEXT: vpinsrw $2, %r8d, %xmm0, %xmm0
+; AVX512BF16-NEXT: vpinsrw $3, %edi, %xmm0, %xmm0
+; AVX512BF16-NEXT: vpinsrw $4, %esi, %xmm0, %xmm0
+; AVX512BF16-NEXT: vpinsrw $5, %edx, %xmm0, %xmm0
+; AVX512BF16-NEXT: vpinsrw $6, %ecx, %xmm0, %xmm0
+; AVX512BF16-NEXT: vpinsrw $7, %eax, %xmm0, %xmm0
+; AVX512BF16-NEXT: retq
+ %r = call <8 x bfloat> @llvm.rint.v8bf16(<8 x bfloat> %a)
+ ret <8 x bfloat> %r
+}
+
+define bfloat @rint_bf16(bfloat %a) {
+; AVX10_2-LABEL: rint_bf16:
+; AVX10_2: # %bb.0:
+; AVX10_2-NEXT: vrndscalebf16 $4, %xmm0, %xmm0
+; AVX10_2-NEXT: retq
+;
+; AVX512BF16-LABEL: rint_bf16:
+; AVX512BF16: # %bb.0:
+; AVX512BF16-NEXT: vpextrw $0, %xmm0, %eax
+; AVX512BF16-NEXT: shll $16, %eax
+; AVX512BF16-NEXT: vmovd %eax, %xmm0
+; AVX512BF16-NEXT: vroundss $4, %xmm0, %xmm0, %xmm0
+; AVX512BF16-NEXT: vcvtneps2bf16 %xmm0, %xmm0
+; AVX512BF16-NEXT: retq
+ %r = call bfloat @llvm.rint.bf16(bfloat %a)
+ ret bfloat %r
+}
+
+define <8 x bfloat> @nearbyint_v8bf16(<8 x bfloat> %a) {
+; AVX10_2-LABEL: nearbyint_v8bf16:
+; AVX10_2: # %bb.0:
+; AVX10_2-NEXT: vrndscalebf16 $12, %xmm0, %xmm0
+; AVX10_2-NEXT: retq
+;
+; AVX512BF16-LABEL: nearbyint_v8bf16:
+; AVX512BF16: # %bb.0:
+; AVX512BF16-NEXT: vpextrw $7, %xmm0, %eax
+; AVX512BF16-NEXT: shll $16, %eax
+; AVX512BF16-NEXT: vmovd %eax, %xmm1
+; AVX512BF16-NEXT: vroundss $12, %xmm1, %xmm1, %xmm1
+; AVX512BF16-NEXT: vcvtneps2bf16 %xmm1, %xmm1
+; AVX512BF16-NEXT: vmovd %xmm1, %eax
+; AVX512BF16-NEXT: vpextrw $6, %xmm0, %ecx
+; AVX512BF16-NEXT: shll $16, %ecx
+; AVX512BF16-NEXT: vmovd %ecx, %xmm1
+; AVX512BF16-NEXT: vroundss $12, %xmm1, %xmm1, %xmm1
+; AVX512BF16-NEXT: vcvtneps2bf16 %xmm1, %xmm1
+; AVX512BF16-NEXT: vmovd %xmm1, %ecx
+; AVX512BF16-NEXT: vpextrw $5, %xmm0, %edx
+; AVX512BF16-NEXT: shll $16, %edx
+; AVX512BF16-NEXT: vmovd %edx, %xmm1
+; AVX512BF16-NEXT: vroundss $12, %xmm1, %xmm1, %xmm1
+; AVX512BF16-NEXT: vcvtneps2bf16 %xmm1, %xmm1
+; AVX512BF16-NEXT: vmovd %xmm1, %edx
+; AVX512BF16-NEXT: vpextrw $4, %xmm0, %esi
+; AVX512BF16-NEXT: shll $16, %esi
+; AVX512BF16-NEXT: vmovd %esi, %xmm1
+; AVX512BF16-NEXT: vroundss $12, %xmm1, %xmm1, %xmm1
+; AVX512BF16-NEXT: vcvtneps2bf16 %xmm1, %xmm1
+; AVX512BF16-NEXT: vmovd %xmm1, %esi
+; AVX512BF16-NEXT: vpextrw $3, %xmm0, %edi
+; AVX512BF16-NEXT: shll $16, %edi
+; AVX512BF16-NEXT: vmovd %edi, %xmm1
+; AVX512BF16-NEXT: vroundss $12, %xmm1, %xmm1, %xmm1
+; AVX512BF16-NEXT: vcvtneps2bf16 %xmm1, %xmm1
+; AVX512BF16-NEXT: vmovd %xmm1, %edi
+; AVX512BF16-NEXT: vpextrw $2, %xmm0, %r8d
+; AVX512BF16-NEXT: shll $16, %r8d
+; AVX512BF16-NEXT: vmovd %r8d, %xmm1
+; AVX512BF16-NEXT: vroundss $12, %xmm1, %xmm1, %xmm1
+; AVX512BF16-NEXT: vcvtneps2bf16 %xmm1, %xmm1
+; AVX512BF16-NEXT: vmovd %xmm1, %r8d
+; AVX512BF16-NEXT: vpextrw $1, %xmm0, %r9d
+; AVX512BF16-NEXT: shll $16, %r9d
+; AVX512BF16-NEXT: vmovd %r9d, %xmm1
+; AVX512BF16-NEXT: vroundss $12, %xmm1, %xmm1, %xmm1
+; AVX512BF16-NEXT: vcvtneps2bf16 %xmm1, %xmm1
+; AVX512BF16-NEXT: vmovd %xmm0, %r9d
+; AVX512BF16-NEXT: shll $16, %r9d
+; AVX512BF16-NEXT: vmovd %r9d, %xmm0
+; AVX512BF16-NEXT: vroundss $12, %xmm0, %xmm0, %xmm0
+; AVX512BF16-NEXT: vcvtneps2bf16 %xmm0, %xmm0
+; AVX512BF16-NEXT: vpunpcklwd {{.*#+}} xmm0 = xmm0[0],xmm1[0],xmm0[1],xmm1[1],xmm0[2],xmm1[2],xmm0[3],xmm1[3]
+; AVX512BF16-NEXT: vpinsrw $2, %r8d, %xmm0, %xmm0
+; AVX512BF16-NEXT: vpinsrw $3, %edi, %xmm0, %xmm0
+; AVX512BF16-NEXT: vpinsrw $4, %esi, %xmm0, %xmm0
+; AVX512BF16-NEXT: vpinsrw $5, %edx, %xmm0, %xmm0
+; AVX512BF16-NEXT: vpinsrw $6, %ecx, %xmm0, %xmm0
+; AVX512BF16-NEXT: vpinsrw $7, %eax, %xmm0, %xmm0
+; AVX512BF16-NEXT: retq
+ %r = call <8 x bfloat> @llvm.nearbyint.v8bf16(<8 x bfloat> %a)
+ ret <8 x bfloat> %r
+}
+
+define bfloat @nearbyint_bf16(bfloat %a) {
+; AVX10_2-LABEL: nearbyint_bf16:
+; AVX10_2: # %bb.0:
+; AVX10_2-NEXT: vrndscalebf16 $12, %xmm0, %xmm0
+; AVX10_2-NEXT: retq
+;
+; AVX512BF16-LABEL: nearbyint_bf16:
+; AVX512BF16: # %bb.0:
+; AVX512BF16-NEXT: vpextrw $0, %xmm0, %eax
+; AVX512BF16-NEXT: shll $16, %eax
+; AVX512BF16-NEXT: vmovd %eax, %xmm0
+; AVX512BF16-NEXT: vroundss $12, %xmm0, %xmm0, %xmm0
+; AVX512BF16-NEXT: vcvtneps2bf16 %xmm0, %xmm0
+; AVX512BF16-NEXT: retq
+ %r = call bfloat @llvm.nearbyint.bf16(bfloat %a)
+ ret bfloat %r
+}
+
+define <8 x bfloat> @roundeven_v8bf16(<8 x bfloat> %a) {
+; AVX10_2-LABEL: roundeven_v8bf16:
+; AVX10_2: # %bb.0:
+; AVX10_2-NEXT: vrndscalebf16 $8, %xmm0, %xmm0
+; AVX10_2-NEXT: retq
+;
+; AVX512BF16-LABEL: roundeven_v8bf16:
+; AVX512BF16: # %bb.0:
+; AVX512BF16-NEXT: vpextrw $7, %xmm0, %eax
+; AVX512BF16-NEXT: shll $16, %eax
+; AVX512BF16-NEXT: vmovd %eax, %xmm1
+; AVX512BF16-NEXT: vroundss $8, %xmm1, %xmm1, %xmm1
+; AVX512BF16-NEXT: vcvtneps2bf16 %xmm1, %xmm1
+; AVX512BF16-NEXT: vmovd %xmm1, %eax
+; AVX512BF16-NEXT: vpextrw $6, %xmm0, %ecx
+; AVX512BF16-NEXT: shll $16, %ecx
+; AVX512BF16-NEXT: vmovd %ecx, %xmm1
+; AVX512BF16-NEXT: vroundss $8, %xmm1, %xmm1, %xmm1
+; AVX512BF16-NEXT: vcvtneps2bf16 %xmm1, %xmm1
+; AVX512BF16-NEXT: vmovd %xmm1, %ecx
+; AVX512BF16-NEXT: vpextrw $5, %xmm0, %edx
+; AVX512BF16-NEXT: shll $16, %edx
+; AVX512BF16-NEXT: vmovd %edx, %xmm1
+; AVX512BF16-NEXT: vroundss $8, %xmm1, %xmm1, %xmm1
+; AVX512BF16-NEXT: vcvtneps2bf16 %xmm1, %xmm1
+; AVX512BF16-NEXT: vmovd %xmm1, %edx
+; AVX512BF16-NEXT: vpextrw $4, %xmm0, %esi
+; AVX512BF16-NEXT: shll $16, %esi
+; AVX512BF16-NEXT: vmovd %esi, %xmm1
+; AVX512BF16-NEXT: vroundss $8, %xmm1, %xmm1, %xmm1
+; AVX512BF16-NEXT: vcvtneps2bf16 %xmm1, %xmm1
+; AVX512BF16-NEXT: vmovd %xmm1, %esi
+; AVX512BF16-NEXT: vpextrw $3, %xmm0, %edi
+; AVX512BF16-NEXT: shll $16, %edi
+; AVX512BF16-NEXT: vmovd %edi, %xmm1
+; AVX512BF16-NEXT: vroundss $8, %xmm1, %xmm1, %xmm1
+; AVX512BF16-NEXT: vcvtneps2bf16 %xmm1, %xmm1
+; AVX512BF16-NEXT: vmovd %xmm1, %edi
+; AVX512BF16-NEXT: vpextrw $2, %xmm0, %r8d
+; AVX512BF16-NEXT: shll $16, %r8d
+; AVX512BF16-NEXT: vmovd %r8d, %xmm1
+; AVX512BF16-NEXT: vroundss $8, %xmm1, %xmm1, %xmm1
+; AVX512BF16-NEXT: vcvtneps2bf16 %xmm1, %xmm1
+; AVX512BF16-NEXT: vmovd %xmm1, %r8d
+; AVX512BF16-NEXT: vpextrw $1, %xmm0, %r9d
+; AVX512BF16-NEXT: shll $16, %r9d
+; AVX512BF16-NEXT: vmovd %r9d, %xmm1
+; AVX512BF16-NEXT: vroundss $8, %xmm1, %xmm1, %xmm1
+; AVX512BF16-NEXT: vcvtneps2bf16 %xmm1, %xmm1
+; AVX512BF16-NEXT: vmovd %xmm0, %r9d
+; AVX512BF16-NEXT: shll $16, %r9d
+; AVX512BF16-NEXT: vmovd %r9d, %xmm0
+; AVX512BF16-NEXT: vroundss $8, %xmm0, %xmm0, %xmm0
+; AVX512BF16-NEXT: vcvtneps2bf16 %xmm0, %xmm0
+; AVX512BF16-NEXT: vpunpcklwd {{.*#+}} xmm0 = xmm0[0],xmm1[0],xmm0[1],xmm1[1],xmm0[2],xmm1[2],xmm0[3],xmm1[3]
+; AVX512BF16-NEXT: vpinsrw $2, %r8d, %xmm0, %xmm0
+; AVX512BF16-NEXT: vpinsrw $3, %edi, %xmm0, %xmm0
+; AVX512BF16-NEXT: vpinsrw $4, %esi, %xmm0, %xmm0
+; AVX512BF16-NEXT: vpinsrw $5, %edx, %xmm0, %xmm0
+; AVX512BF16-NEXT: vpinsrw $6, %ecx, %xmm0, %xmm0
+; AVX512BF16-NEXT: vpinsrw $7, %eax, %xmm0, %xmm0
+; AVX512BF16-NEXT: retq
+ %r = call <8 x bfloat> @llvm.roundeven.v8bf16(<8 x bfloat> %a)
+ ret <8 x bfloat> %r
+}
+
+define bfloat @roundeven_bf16(bfloat %a) {
+; AVX10_2-LABEL: roundeven_bf16:
+; AVX10_2: # %bb.0:
+; AVX10_2-NEXT: vrndscalebf16 $8, %xmm0, %xmm0
+; AVX10_2-NEXT: retq
+;
+; AVX512BF16-LABEL: roundeven_bf16:
+; AVX512BF16: # %bb.0:
+; AVX512BF16-NEXT: vpextrw $0, %xmm0, %eax
+; AVX512BF16-NEXT: shll $16, %eax
+; AVX512BF16-NEXT: vmovd %eax, %xmm0
+; AVX512BF16-NEXT: vroundss $8, %xmm0, %xmm0, %xmm0
+; AVX512BF16-NEXT: vcvtneps2bf16 %xmm0, %xmm0
+; AVX512BF16-NEXT: retq
+ %r = call bfloat @llvm.roundeven.bf16(bfloat %a)
+ ret bfloat %r
+}
+
+; ymm and zmm come from separate setOperationAction entries than the xmm ones.
+define <16 x bfloat> @floor_v16bf16(<16 x bfloat> %a) {
+; AVX10_2-LABEL: floor_v16bf16:
+; AVX10_2: # %bb.0:
+; AVX10_2-NEXT: vrndscalebf16 $9, %ymm0, %ymm0
+; AVX10_2-NEXT: retq
+;
+; AVX512BF16-LABEL: floor_v16bf16:
+; AVX512BF16: # %bb.0:
+; AVX512BF16-NEXT: pushq %rbp
+; AVX512BF16-NEXT: .cfi_def_cfa_offset 16
+; AVX512BF16-NEXT: pushq %r15
+; AVX512BF16-NEXT: .cfi_def_cfa_offset 24
+; AVX512BF16-NEXT: pushq %r14
+; AVX512BF16-NEXT: .cfi_def_cfa_offset 32
+; AVX512BF16-NEXT: pushq %r12
+; AVX512BF16-NEXT: .cfi_def_cfa_offset 40
+; AVX512BF16-NEXT: pushq %rbx
+; AVX512BF16-NEXT: .cfi_def_cfa_offset 48
+; AVX512BF16-NEXT: .cfi_offset %rbx, -48
+; AVX512BF16-NEXT: .cfi_offset %r12, -40
+; AVX512BF16-NEXT: .cfi_offset %r14, -32
+; AVX512BF16-NEXT: .cfi_offset %r15, -24
+; AVX512BF16-NEXT: .cfi_offset %rbp, -16
+; AVX512BF16-NEXT: vpextrw $7, %xmm0, %eax
+; AVX512BF16-NEXT: shll $16, %eax
+; AVX512BF16-NEXT: vmovd %eax, %xmm1
+; AVX512BF16-NEXT: vroundss $9, %xmm1, %xmm1, %xmm1
+; AVX512BF16-NEXT: vcvtneps2bf16 %xmm1, %xmm1
+; AVX512BF16-NEXT: vmovd %xmm1, %eax
+; AVX512BF16-NEXT: vpextrw $6, %xmm0, %ecx
+; AVX512BF16-NEXT: shll $16, %ecx
+; AVX512BF16-NEXT: vmovd %ecx, %xmm1
+; AVX512BF16-NEXT: vroundss $9, %xmm1, %xmm1, %xmm1
+; AVX512BF16-NEXT: vcvtneps2bf16 %xmm1, %xmm1
+; AVX512BF16-NEXT: vmovd %xmm1, %ecx
+; AVX512BF16-NEXT: vpextrw $5, %xmm0, %edx
+; AVX512BF16-NEXT: shll $16, %edx
+; AVX512BF16-NEXT: vmovd %edx, %xmm1
+; AVX512BF16-NEXT: vroundss $9, %xmm1, %xmm1, %xmm1
+; AVX512BF16-NEXT: vcvtneps2bf16 %xmm1, %xmm1
+; AVX512BF16-NEXT: vmovd %xmm1, %edx
+; AVX512BF16-NEXT: vpextrw $4, %xmm0, %esi
+; AVX512BF16-NEXT: shll $16, %esi
+; AVX512BF16-NEXT: vmovd %esi, %xmm1
+; AVX512BF16-NEXT: vroundss $9, %xmm1, %xmm1, %xmm1
+; AVX512BF16-NEXT: vcvtneps2bf16 %xmm1, %xmm1
+; AVX512BF16-NEXT: vmovd %xmm1, %esi
+; AVX512BF16-NEXT: vpextrw $3, %xmm0, %edi
+; AVX512BF16-NEXT: shll $16, %edi
+; AVX512BF16-NEXT: vmovd %edi, %xmm1
+; AVX512BF16-NEXT: vroundss $9, %xmm1, %xmm1, %xmm1
+; AVX512BF16-NEXT: vcvtneps2bf16 %xmm1, %xmm1
+; AVX512BF16-NEXT: vmovd %xmm1, %edi
+; AVX512BF16-NEXT: vpextrw $2, %xmm0, %r8d
+; AVX512BF16-NEXT: shll $16, %r8d
+; AVX512BF16-NEXT: vmovd %r8d, %xmm1
+; AVX512BF16-NEXT: vroundss $9, %xmm1, %xmm1, %xmm1
+; AVX512BF16-NEXT: vcvtneps2bf16 %xmm1, %xmm1
+; AVX512BF16-NEXT: vmovd %xmm1, %r8d
+; AVX512BF16-NEXT: vmovd %xmm0, %r9d
+; AVX512BF16-NEXT: shll $16, %r9d
+; AVX512BF16-NEXT: vmovd %r9d, %xmm1
+; AVX512BF16-NEXT: vroundss $9, %xmm1, %xmm1, %xmm1
+; AVX512BF16-NEXT: vcvtneps2bf16 %xmm1, %xmm1
+; AVX512BF16-NEXT: vpextrw $1, %xmm0, %r9d
+; AVX512BF16-NEXT: shll $16, %r9d
+; AVX512BF16-NEXT: vmovd %r9d, %xmm2
+; AVX512BF16-NEXT: vroundss $9, %xmm2, %xmm2, %xmm2
+; AVX512BF16-NEXT: vcvtneps2bf16 %xmm2, %xmm2
+; AVX512BF16-NEXT: vmovd %xmm2, %r9d
+; AVX512BF16-NEXT: vextracti128 $1, %ymm0, %xmm0
+; AVX512BF16-NEXT: vpextrw $7, %xmm0, %r10d
+; AVX512BF16-NEXT: shll $16, %r10d
+; AVX512BF16-NEXT: vmovd %r10d, %xmm2
+; AVX512BF16-NEXT: vroundss $9, %xmm2, %xmm2, %xmm2
+; AVX512BF16-NEXT: vcvtneps2bf16 %xmm2, %xmm2
+; AVX512BF16-NEXT: vmovd %xmm2, %r10d
+; AVX512BF16-NEXT: vpextrw $6, %xmm0, %r11d
+; AVX512BF16-NEXT: shll $16, %r11d
+; AVX512BF16-NEXT: vmovd %r11d, %xmm2
+; AVX512BF16-NEXT: vroundss $9, %xmm2, %xmm2, %xmm2
+; AVX512BF16-NEXT: vcvtneps2bf16 %xmm2, %xmm2
+; AVX512BF16-NEXT: vmovd %xmm2, %r11d
+; AVX512BF16-NEXT: vpextrw $5, %xmm0, %ebx
+; AVX512BF16-NEXT: shll $16, %ebx
+; AVX512BF16-NEXT: vmovd %ebx, %xmm2
+; AVX512BF16-NEXT: vroundss $9, %xmm2, %xmm2, %xmm2
+; AVX512BF16-NEXT: vcvtneps2bf16 %xmm2, %xmm2
+; AVX512BF16-NEXT: vmovd %xmm2, %ebx
+; AVX512BF16-NEXT: vpextrw $4, %xmm0, %ebp
+; AVX512BF16-NEXT: shll $16, %ebp
+; AVX512BF16-NEXT: vmovd %ebp, %xmm2
+; AVX512BF16-NEXT: vroundss $9, %xmm2, %xmm2, %xmm2
+; AVX512BF16-NEXT: vcvtneps2bf16 %xmm2, %xmm2
+; AVX512BF16-NEXT: vmovd %xmm2, %ebp
+; AVX512BF16-NEXT: vpextrw $3, %xmm0, %r14d
+; AVX512BF16-NEXT: shll $16, %r14d
+; AVX512BF16-NEXT: vmovd %r14d, %xmm2
+; AVX512BF16-NEXT: vroundss $9, %xmm2, %xmm2, %xmm2
+; AVX512BF16-NEXT: vcvtneps2bf16 %xmm2, %xmm2
+; AVX512BF16-NEXT: vmovd %xmm2, %r14d
+; AVX512BF16-NEXT: vpextrw $2, %xmm0, %r15d
+; AVX512BF16-NEXT: shll $16, %r15d
+; AVX512BF16-NEXT: vmovd %r15d, %xmm2
+; AVX512BF16-NEXT: vroundss $9, %xmm2, %xmm2, %xmm2
+; AVX512BF16-NEXT: vcvtneps2bf16 %xmm2, %xmm2
+; AVX512BF16-NEXT: vmovd %xmm2, %r15d
+; AVX512BF16-NEXT: vmovd %xmm0, %r12d
+; AVX512BF16-NEXT: shll $16, %r12d
+; AVX512BF16-NEXT: vmovd %r12d, %xmm2
+; AVX512BF16-NEXT: vroundss $9, %xmm2, %xmm2, %xmm2
+; AVX512BF16-NEXT: vcvtneps2bf16 %xmm2, %xmm2
+; AVX512BF16-NEXT: vpextrw $1, %xmm0, %r12d
+; AVX512BF16-NEXT: shll $16, %r12d
+; AVX512BF16-NEXT: vmovd %r12d, %xmm0
+; AVX512BF16-NEXT: vroundss $9, %xmm0, %xmm0, %xmm0
+; AVX512BF16-NEXT: vcvtneps2bf16 %xmm0, %xmm0
+; AVX512BF16-NEXT: vmovd %xmm0, %r12d
+; AVX512BF16-NEXT: vpinsrw $1, %r12d, %xmm2, %xmm0
+; AVX512BF16-NEXT: vpinsrw $2, %r15d, %xmm0, %xmm0
+; AVX512BF16-NEXT: vpinsrw $3, %r14d, %xmm0, %xmm0
+; AVX512BF16-NEXT: vpinsrw $4, %ebp, %xmm0, %xmm0
+; AVX512BF16-NEXT: vpinsrw $5, %ebx, %xmm0, %xmm0
+; AVX512BF16-NEXT: vpinsrw $6, %r11d, %xmm0, %xmm0
+; AVX512BF16-NEXT: vpinsrw $7, %r10d, %xmm0, %xmm0
+; AVX512BF16-NEXT: vpinsrw $1, %r9d, %xmm1, %xmm1
+; AVX512BF16-NEXT: vpinsrw $2, %r8d, %xmm1, %xmm1
+; AVX512BF16-NEXT: vpinsrw $3, %edi, %xmm1, %xmm1
+; AVX512BF16-NEXT: vpinsrw $4, %esi, %xmm1, %xmm1
+; AVX512BF16-NEXT: vpinsrw $5, %edx, %xmm1, %xmm1
+; AVX512BF16-NEXT: vpinsrw $6, %ecx, %xmm1, %xmm1
+; AVX512BF16-NEXT: vpinsrw $7, %eax, %xmm1, %xmm1
+; AVX512BF16-NEXT: vinserti128 $1, %xmm0, %ymm1, %ymm0
+; AVX512BF16-NEXT: popq %rbx
+; AVX512BF16-NEXT: .cfi_def_cfa_offset 40
+; AVX512BF16-NEXT: popq %r12
+; AVX512BF16-NEXT: .cfi_def_cfa_offset 32
+; AVX512BF16-NEXT: popq %r14
+; AVX512BF16-NEXT: .cfi_def_cfa_offset 24
+; AVX512BF16-NEXT: popq %r15
+; AVX512BF16-NEXT: .cfi_def_cfa_offset 16
+; AVX512BF16-NEXT: popq %rbp
+; AVX512BF16-NEXT: .cfi_def_cfa_offset 8
+; AVX512BF16-NEXT: retq
+ %r = call <16 x bfloat> @llvm.floor.v16bf16(<16 x bfloat> %a)
+ ret <16 x bfloat> %r
+}
+
+define <32 x bfloat> @floor_v32bf16(<32 x bfloat> %a) {
+; AVX10_2-LABEL: floor_v32bf16:
+; AVX10_2: # %bb.0:
+; AVX10_2-NEXT: vrndscalebf16 $9, %zmm0, %zmm0
+; AVX10_2-NEXT: retq
+;
+; AVX512BF16-LABEL: floor_v32bf16:
+; AVX512BF16: # %bb.0:
+; AVX512BF16-NEXT: vpextrw $7, %xmm0, %eax
+; AVX512BF16-NEXT: shll $16, %eax
+; AVX512BF16-NEXT: vmovd %eax, %xmm1
+; AVX512BF16-NEXT: vpextrw $6, %xmm0, %eax
+; AVX512BF16-NEXT: shll $16, %eax
+; AVX512BF16-NEXT: vmovd %eax, %xmm2
+; AVX512BF16-NEXT: vpextrw $5, %xmm0, %eax
+; AVX512BF16-NEXT: shll $16, %eax
+; AVX512BF16-NEXT: vmovd %eax, %xmm3
+; AVX512BF16-NEXT: vpextrw $4, %xmm0, %eax
+; AVX512BF16-NEXT: shll $16, %eax
+; AVX512BF16-NEXT: vmovd %eax, %xmm4
+; AVX512BF16-NEXT: vpextrw $3, %xmm0, %eax
+; AVX512BF16-NEXT: shll $16, %eax
+; AVX512BF16-NEXT: vmovd %eax, %xmm5
+; AVX512BF16-NEXT: vpextrw $2, %xmm0, %eax
+; AVX512BF16-NEXT: shll $16, %eax
+; AVX512BF16-NEXT: vmovd %eax, %xmm6
+; AVX512BF16-NEXT: vmovd %xmm0, %eax
+; AVX512BF16-NEXT: shll $16, %eax
+; AVX512BF16-NEXT: vmovd %eax, %xmm7
+; AVX512BF16-NEXT: vpextrw $1, %xmm0, %eax
+; AVX512BF16-NEXT: shll $16, %eax
+; AVX512BF16-NEXT: vmovd %eax, %xmm8
+; AVX512BF16-NEXT: vextracti32x4 $1, %ymm0, %xmm16
+; AVX512BF16-NEXT: vpextrw $7, %xmm16, %eax
+; AVX512BF16-NEXT: shll $16, %eax
+; AVX512BF16-NEXT: vmovd %eax, %xmm9
+; AVX512BF16-NEXT: vpextrw $6, %xmm16, %eax
+; AVX512BF16-NEXT: shll $16, %eax
+; AVX512BF16-NEXT: vmovd %eax, %xmm10
+; AVX512BF16-NEXT: vpextrw $5, %xmm16, %eax
+; AVX512BF16-NEXT: shll $16, %eax
+; AVX512BF16-NEXT: vmovd %eax, %xmm11
+; AVX512BF16-NEXT: vpextrw $4, %xmm16, %eax
+; AVX512BF16-NEXT: shll $16, %eax
+; AVX512BF16-NEXT: vmovd %eax, %xmm12
+; AVX512BF16-NEXT: vpextrw $3, %xmm16, %eax
+; AVX512BF16-NEXT: shll $16, %eax
+; AVX512BF16-NEXT: vmovd %eax, %xmm13
+; AVX512BF16-NEXT: vpextrw $2, %xmm16, %eax
+; AVX512BF16-NEXT: shll $16, %eax
+; AVX512BF16-NEXT: vmovd %eax, %xmm14
+; AVX512BF16-NEXT: vmovd %xmm16, %eax
+; AVX512BF16-NEXT: shll $16, %eax
+; AVX512BF16-NEXT: vmovd %eax, %xmm15
+; AVX512BF16-NEXT: vpextrw $1, %xmm16, %eax
+; AVX512BF16-NEXT: shll $16, %eax
+; AVX512BF16-NEXT: vmovd %eax, %xmm16
+; AVX512BF16-NEXT: vextracti32x4 $2, %zmm0, %xmm24
+; AVX512BF16-NEXT: vpextrw $7, %xmm24, %eax
+; AVX512BF16-NEXT: shll $16, %eax
+; AVX512BF16-NEXT: vmovd %eax, %xmm17
+; AVX512BF16-NEXT: vpextrw $6, %xmm24, %eax
+; AVX512BF16-NEXT: shll $16, %eax
+; AVX512BF16-NEXT: vmovd %eax, %xmm18
+; AVX512BF16-NEXT: vpextrw $5, %xmm24, %eax
+; AVX512BF16-NEXT: shll $16, %eax
+; AVX512BF16-NEXT: vmovd %eax, %xmm19
+; AVX512BF16-NEXT: vpextrw $4, %xmm24, %eax
+; AVX512BF16-NEXT: shll $16, %eax
+; AVX512BF16-NEXT: vmovd %eax, %xmm20
+; AVX512BF16-NEXT: vpextrw $3, %xmm24, %eax
+; AVX512BF16-NEXT: shll $16, %eax
+; AVX512BF16-NEXT: vmovd %eax, %xmm21
+; AVX512BF16-NEXT: vpextrw $2, %xmm24, %eax
+; AVX512BF16-NEXT: shll $16, %eax
+; AVX512BF16-NEXT: vmovd %eax, %xmm22
+; AVX512BF16-NEXT: vmovd %xmm24, %eax
+; AVX512BF16-NEXT: shll $16, %eax
+; AVX512BF16-NEXT: vmovd %eax, %xmm23
+; AVX512BF16-NEXT: vpextrw $1, %xmm24, %eax
+; AVX512BF16-NEXT: shll $16, %eax
+; AVX512BF16-NEXT: vmovd %eax, %xmm24
+; AVX512BF16-NEXT: vextracti32x4 $3, %zmm0, %xmm25
+; AVX512BF16-NEXT: vpextrw $7, %xmm25, %eax
+; AVX512BF16-NEXT: shll $16, %eax
+; AVX512BF16-NEXT: vmovd %eax, %xmm0
+; AVX512BF16-NEXT: vpextrw $6, %xmm25, %eax
+; AVX512BF16-NEXT: shll $16, %eax
+; AVX512BF16-NEXT: vmovd %eax, %xmm26
+; AVX512BF16-NEXT: vpextrw $5, %xmm25, %eax
+; AVX512BF16-NEXT: shll $16, %eax
+; AVX512BF16-NEXT: vmovd %eax, %xmm27
+; AVX512BF16-NEXT: vpextrw $4, %xmm25, %eax
+; AVX512BF16-NEXT: shll $16, %eax
+; AVX512BF16-NEXT: vmovd %eax, %xmm28
+; AVX512BF16-NEXT: vpextrw $3, %xmm25, %eax
+; AVX512BF16-NEXT: shll $16, %eax
+; AVX512BF16-NEXT: vmovd %eax, %xmm29
+; AVX512BF16-NEXT: vpextrw $2, %xmm25, %eax
+; AVX512BF16-NEXT: shll $16, %eax
+; AVX512BF16-NEXT: vmovd %eax, %xmm30
+; AVX512BF16-NEXT: vmovd %xmm25, %eax
+; AVX512BF16-NEXT: shll $16, %eax
+; AVX512BF16-NEXT: vmovd %eax, %xmm31
+; AVX512BF16-NEXT: vpextrw $1, %xmm25, %eax
+; AVX512BF16-NEXT: shll $16, %eax
+; AVX512BF16-NEXT: vmovd %eax, %xmm25
+; AVX512BF16-NEXT: vrndscaless $9, %xmm25, %xmm25, %xmm25
+; AVX512BF16-NEXT: vcvtneps2bf16 %xmm25, %xmm25
+; AVX512BF16-NEXT: vmovd %xmm25, %eax
+; AVX512BF16-NEXT: vrndscaless $9, %xmm30, %xmm30, %xmm25
+; AVX512BF16-NEXT: vcvtneps2bf16 %xmm25, %xmm25
+; AVX512BF16-NEXT: vrndscaless $9, %xmm31, %xmm31, %xmm30
+; AVX512BF16-NEXT: vcvtneps2bf16 %xmm30, %xmm30
+; AVX512BF16-NEXT: vpinsrw $1, %eax, %xmm30, %xmm30
+; AVX512BF16-NEXT: vmovd %xmm25, %eax
+; AVX512BF16-NEXT: vrndscaless $9, %xmm29, %xmm29, %xmm25
+; AVX512BF16-NEXT: vcvtneps2bf16 %xmm25, %xmm25
+; AVX512BF16-NEXT: vpinsrw $2, %eax, %xmm30, %xmm29
+; AVX512BF16-NEXT: vmovd %xmm25, %eax
+; AVX512BF16-NEXT: vrndscaless $9, %xmm28, %xmm28, %xmm25
+; AVX512BF16-NEXT: vcvtneps2bf16 %xmm25, %xmm25
+; AVX512BF16-NEXT: vpinsrw $3, %eax, %xmm29, %xmm28
+; AVX512BF16-NEXT: vmovd %xmm25, %eax
+; AVX512BF16-NEXT: vrndscaless $9, %xmm27, %xmm27, %xmm25
+; AVX512BF16-NEXT: vcvtneps2bf16 %xmm25, %xmm25
+; AVX512BF16-NEXT: vpinsrw $4, %eax, %xmm28, %xmm27
+; AVX512BF16-NEXT: vmovd %xmm25, %eax
+; AVX512BF16-NEXT: vrndscaless $9, %xmm26, %xmm26, %xmm25
+; AVX512BF16-NEXT: vcvtneps2bf16 %xmm25, %xmm25
+; AVX512BF16-NEXT: vpinsrw $5, %eax, %xmm27, %xmm26
+; AVX512BF16-NEXT: vmovd %xmm25, %eax
+; AVX512BF16-NEXT: vroundss $9, %xmm0, %xmm0, %xmm0
+; AVX512BF16-NEXT: vcvtneps2bf16 %xmm0, %xmm0
+; AVX512BF16-NEXT: vpinsrw $6, %eax, %xmm26, %xmm25
+; AVX512BF16-NEXT: vmovd %xmm0, %eax
+; AVX512BF16-NEXT: vrndscaless $9, %xmm24, %xmm24, %xmm0
+; AVX512BF16-NEXT: vcvtneps2bf16 %xmm0, %xmm24
+; AVX512BF16-NEXT: vpinsrw $7, %eax, %xmm25, %xmm0
+; AVX512BF16-NEXT: vmovd %xmm24, %eax
+; AVX512BF16-NEXT: vrndscaless $9, %xmm22, %xmm22, %xmm22
+; AVX512BF16-NEXT: vcvtneps2bf16 %xmm22, %xmm22
+; AVX512BF16-NEXT: vrndscaless $9, %xmm23, %xmm23, %xmm23
+; AVX512BF16-NEXT: vcvtneps2bf16 %xmm23, %xmm23
+; AVX512BF16-NEXT: vpinsrw $1, %eax, %xmm23, %xmm23
+; AVX512BF16-NEXT: vmovd %xmm22, %eax
+; AVX512BF16-NEXT: vrndscaless $9, %xmm21, %xmm21, %xmm21
+; AVX512BF16-NEXT: vcvtneps2bf16 %xmm21, %xmm21
+; AVX512BF16-NEXT: vpinsrw $2, %eax, %xmm23, %xmm22
+; AVX512BF16-NEXT: vmovd %xmm21, %eax
+; AVX512BF16-NEXT: vrndscaless $9, %xmm20, %xmm20, %xmm20
+; AVX512BF16-NEXT: vcvtneps2bf16 %xmm20, %xmm20
+; AVX512BF16-NEXT: vpinsrw $3, %eax, %xmm22, %xmm21
+; AVX512BF16-NEXT: vmovd %xmm20, %eax
+; AVX512BF16-NEXT: vrndscaless $9, %xmm19, %xmm19, %xmm19
+; AVX512BF16-NEXT: vcvtneps2bf16 %xmm19, %xmm19
+; AVX512BF16-NEXT: vpinsrw $4, %eax, %xmm21, %xmm20
+; AVX512BF16-NEXT: vmovd %xmm19, %eax
+; AVX512BF16-NEXT: vrndscaless $9, %xmm18, %xmm18, %xmm18
+; AVX512BF16-NEXT: vcvtneps2bf16 %xmm18, %xmm18
+; AVX512BF16-NEXT: vpinsrw $5, %eax, %xmm20, %xmm19
+; AVX512BF16-NEXT: vmovd %xmm18, %eax
+; AVX512BF16-NEXT: vrndscaless $9, %xmm17, %xmm17, %xmm17
+; AVX512BF16-NEXT: vcvtneps2bf16 %xmm17, %xmm17
+; AVX512BF16-NEXT: vpinsrw $6, %eax, %xmm19, %xmm18
+; AVX512BF16-NEXT: vmovd %xmm17, %eax
+; AVX512BF16-NEXT: vrndscaless $9, %xmm16, %xmm16, %xmm16
+; AVX512BF16-NEXT: vcvtneps2bf16 %xmm16, %xmm16
+; AVX512BF16-NEXT: vpinsrw $7, %eax, %xmm18, %xmm17
+; AVX512BF16-NEXT: vmovd %xmm16, %eax
+; AVX512BF16-NEXT: vroundss $9, %xmm14, %xmm14, %xmm14
+; AVX512BF16-NEXT: vcvtneps2bf16 %xmm14, %xmm14
+; AVX512BF16-NEXT: vroundss $9, %xmm15, %xmm15, %xmm15
+; AVX512BF16-NEXT: vcvtneps2bf16 %xmm15, %xmm15
+; AVX512BF16-NEXT: vpinsrw $1, %eax, %xmm15, %xmm15
+; AVX512BF16-NEXT: vmovd %xmm14, %eax
+; AVX512BF16-NEXT: vroundss $9, %xmm13, %xmm13, %xmm13
+; AVX512BF16-NEXT: vcvtneps2bf16 %xmm13, %xmm13
+; AVX512BF16-NEXT: vpinsrw $2, %eax, %xmm15, %xmm14
+; AVX512BF16-NEXT: vmovd %xmm13, %eax
+; AVX512BF16-NEXT: vroundss $9, %xmm12, %xmm12, %xmm12
+; AVX512BF16-NEXT: vcvtneps2bf16 %xmm12, %xmm12
+; AVX512BF16-NEXT: vpinsrw $3, %eax, %xmm14, %xmm13
+; AVX512BF16-NEXT: vmovd %xmm12, %eax
+; AVX512BF16-NEXT: vroundss $9, %xmm11, %xmm11, %xmm11
+; AVX512BF16-NEXT: vcvtneps2bf16 %xmm11, %xmm11
+; AVX512BF16-NEXT: vpinsrw $4, %eax, %xmm13, %xmm12
+; AVX512BF16-NEXT: vmovd %xmm11, %eax
+; AVX512BF16-NEXT: vroundss $9, %xmm10, %xmm10, %xmm10
+; AVX512BF16-NEXT: vcvtneps2bf16 %xmm10, %xmm10
+; AVX512BF16-NEXT: vpinsrw $5, %eax, %xmm12, %xmm11
+; AVX512BF16-NEXT: vmovd %xmm10, %eax
+; AVX512BF16-NEXT: vroundss $9, %xmm9, %xmm9, %xmm9
+; AVX512BF16-NEXT: vcvtneps2bf16 %xmm9, %xmm9
+; AVX512BF16-NEXT: vpinsrw $6, %eax, %xmm11, %xmm10
+; AVX512BF16-NEXT: vmovd %xmm9, %eax
+; AVX512BF16-NEXT: vroundss $9, %xmm8, %xmm8, %xmm8
+; AVX512BF16-NEXT: vcvtneps2bf16 %xmm8, %xmm8
+; AVX512BF16-NEXT: vpinsrw $7, %eax, %xmm10, %xmm9
+; AVX512BF16-NEXT: vmovd %xmm8, %eax
+; AVX512BF16-NEXT: vroundss $9, %xmm6, %xmm6, %xmm6
+; AVX512BF16-NEXT: vcvtneps2bf16 %xmm6, %xmm6
+; AVX512BF16-NEXT: vroundss $9, %xmm7, %xmm7, %xmm7
+; AVX512BF16-NEXT: vcvtneps2bf16 %xmm7, %xmm7
+; AVX512BF16-NEXT: vpinsrw $1, %eax, %xmm7, %xmm7
+; AVX512BF16-NEXT: vmovd %xmm6, %eax
+; AVX512BF16-NEXT: vroundss $9, %xmm5, %xmm5, %xmm5
+; AVX512BF16-NEXT: vcvtneps2bf16 %xmm5, %xmm5
+; AVX512BF16-NEXT: vpinsrw $2, %eax, %xmm7, %xmm6
+; AVX512BF16-NEXT: vmovd %xmm5, %eax
+; AVX512BF16-NEXT: vroundss $9, %xmm4, %xmm4, %xmm4
+; AVX512BF16-NEXT: vcvtneps2bf16 %xmm4, %xmm4
+; AVX512BF16-NEXT: vpinsrw $3, %eax, %xmm6, %xmm5
+; AVX512BF16-NEXT: vmovd %xmm4, %eax
+; AVX512BF16-NEXT: vroundss $9, %xmm3, %xmm3, %xmm3
+; AVX512BF16-NEXT: vcvtneps2bf16 %xmm3, %xmm3
+; AVX512BF16-NEXT: vpinsrw $4, %eax, %xmm5, %xmm4
+; AVX512BF16-NEXT: vmovd %xmm3, %eax
+; AVX512BF16-NEXT: vroundss $9, %xmm2, %xmm2, %xmm2
+; AVX512BF16-NEXT: vcvtneps2bf16 %xmm2, %xmm2
+; AVX512BF16-NEXT: vpinsrw $5, %eax, %xmm4, %xmm3
+; AVX512BF16-NEXT: vmovd %xmm2, %eax
+; AVX512BF16-NEXT: vroundss $9, %xmm1, %xmm1, %xmm1
+; AVX512BF16-NEXT: vcvtneps2bf16 %xmm1, %xmm1
+; AVX512BF16-NEXT: vpinsrw $6, %eax, %xmm3, %xmm2
+; AVX512BF16-NEXT: vmovd %xmm1, %eax
+; AVX512BF16-NEXT: vpinsrw $7, %eax, %xmm2, %xmm1
+; AVX512BF16-NEXT: vinserti32x4 $1, %xmm0, %ymm17, %ymm0
+; AVX512BF16-NEXT: vinserti128 $1, %xmm9, %ymm1, %ymm1
+; AVX512BF16-NEXT: vinserti64x4 $1, %ymm0, %zmm1, %zmm0
+; AVX512BF16-NEXT: retq
+ %r = call <32 x bfloat> @llvm.floor.v32bf16(<32 x bfloat> %a)
+ ret <32 x bfloat> %r
+}
+
+; Narrower than a full vector register: widened, not scalarized.
+define <2 x bfloat> @floor_v2bf16(<2 x bfloat> %a) {
+; AVX10_2-LABEL: floor_v2bf16:
+; AVX10_2: # %bb.0:
+; AVX10_2-NEXT: vrndscalebf16 $9, %xmm0, %xmm0
+; AVX10_2-NEXT: retq
+;
+; AVX512BF16-LABEL: floor_v2bf16:
+; AVX512BF16: # %bb.0:
+; AVX512BF16-NEXT: vpextrw $1, %xmm0, %eax
+; AVX512BF16-NEXT: shll $16, %eax
+; AVX512BF16-NEXT: vmovd %eax, %xmm1
+; AVX512BF16-NEXT: vroundss $9, %xmm1, %xmm1, %xmm1
+; AVX512BF16-NEXT: vcvtneps2bf16 %xmm1, %xmm1
+; AVX512BF16-NEXT: vmovd %xmm0, %eax
+; AVX512BF16-NEXT: shll $16, %eax
+; AVX512BF16-NEXT: vmovd %eax, %xmm0
+; AVX512BF16-NEXT: vroundss $9, %xmm0, %xmm0, %xmm0
+; AVX512BF16-NEXT: vcvtneps2bf16 %xmm0, %xmm0
+; AVX512BF16-NEXT: vpunpcklwd {{.*#+}} xmm0 = xmm0[0],xmm1[0],xmm0[1],xmm1[1],xmm0[2],xmm1[2],xmm0[3],xmm1[3]
+; AVX512BF16-NEXT: retq
+ %r = call <2 x bfloat> @llvm.floor.v2bf16(<2 x bfloat> %a)
+ ret <2 x bfloat> %r
+}
+
+; Folded load operand.
+define <8 x bfloat> @floor_v8bf16_load(ptr %p) {
+; AVX10_2-LABEL: floor_v8bf16_load:
+; AVX10_2: # %bb.0:
+; AVX10_2-NEXT: vrndscalebf16 $9, (%rdi), %xmm0
+; AVX10_2-NEXT: retq
+;
+; AVX512BF16-LABEL: floor_v8bf16_load:
+; AVX512BF16: # %bb.0:
+; AVX512BF16-NEXT: vmovdqa (%rdi), %xmm0
+; AVX512BF16-NEXT: vpextrw $7, %xmm0, %eax
+; AVX512BF16-NEXT: shll $16, %eax
+; AVX512BF16-NEXT: vmovd %eax, %xmm1
+; AVX512BF16-NEXT: vroundss $9, %xmm1, %xmm1, %xmm1
+; AVX512BF16-NEXT: vcvtneps2bf16 %xmm1, %xmm1
+; AVX512BF16-NEXT: vmovd %xmm1, %eax
+; AVX512BF16-NEXT: vpextrw $6, %xmm0, %ecx
+; AVX512BF16-NEXT: shll $16, %ecx
+; AVX512BF16-NEXT: vmovd %ecx, %xmm1
+; AVX512BF16-NEXT: vroundss $9, %xmm1, %xmm1, %xmm1
+; AVX512BF16-NEXT: vcvtneps2bf16 %xmm1, %xmm1
+; AVX512BF16-NEXT: vmovd %xmm1, %ecx
+; AVX512BF16-NEXT: vpextrw $5, %xmm0, %edx
+; AVX512BF16-NEXT: shll $16, %edx
+; AVX512BF16-NEXT: vmovd %edx, %xmm1
+; AVX512BF16-NEXT: vroundss $9, %xmm1, %xmm1, %xmm1
+; AVX512BF16-NEXT: vcvtneps2bf16 %xmm1, %xmm1
+; AVX512BF16-NEXT: vmovd %xmm1, %edx
+; AVX512BF16-NEXT: vpextrw $4, %xmm0, %esi
+; AVX512BF16-NEXT: shll $16, %esi
+; AVX512BF16-NEXT: vmovd %esi, %xmm1
+; AVX512BF16-NEXT: vroundss $9, %xmm1, %xmm1, %xmm1
+; AVX512BF16-NEXT: vcvtneps2bf16 %xmm1, %xmm1
+; AVX512BF16-NEXT: vmovd %xmm1, %esi
+; AVX512BF16-NEXT: vpextrw $3, %xmm0, %edi
+; AVX512BF16-NEXT: shll $16, %edi
+; AVX512BF16-NEXT: vmovd %edi, %xmm1
+; AVX512BF16-NEXT: vroundss $9, %xmm1, %xmm1, %xmm1
+; AVX512BF16-NEXT: vcvtneps2bf16 %xmm1, %xmm1
+; AVX512BF16-NEXT: vmovd %xmm1, %edi
+; AVX512BF16-NEXT: vpextrw $2, %xmm0, %r8d
+; AVX512BF16-NEXT: shll $16, %r8d
+; AVX512BF16-NEXT: vmovd %r8d, %xmm1
+; AVX512BF16-NEXT: vroundss $9, %xmm1, %xmm1, %xmm1
+; AVX512BF16-NEXT: vcvtneps2bf16 %xmm1, %xmm1
+; AVX512BF16-NEXT: vmovd %xmm1, %r8d
+; AVX512BF16-NEXT: vpextrw $1, %xmm0, %r9d
+; AVX512BF16-NEXT: shll $16, %r9d
+; AVX512BF16-NEXT: vmovd %r9d, %xmm1
+; AVX512BF16-NEXT: vroundss $9, %xmm1, %xmm1, %xmm1
+; AVX512BF16-NEXT: vcvtneps2bf16 %xmm1, %xmm1
+; AVX512BF16-NEXT: vmovd %xmm0, %r9d
+; AVX512BF16-NEXT: shll $16, %r9d
+; AVX512BF16-NEXT: vmovd %r9d, %xmm0
+; AVX512BF16-NEXT: vroundss $9, %xmm0, %xmm0, %xmm0
+; AVX512BF16-NEXT: vcvtneps2bf16 %xmm0, %xmm0
+; AVX512BF16-NEXT: vpunpcklwd {{.*#+}} xmm0 = xmm0[0],xmm1[0],xmm0[1],xmm1[1],xmm0[2],xmm1[2],xmm0[3],xmm1[3]
+; AVX512BF16-NEXT: vpinsrw $2, %r8d, %xmm0, %xmm0
+; AVX512BF16-NEXT: vpinsrw $3, %edi, %xmm0, %xmm0
+; AVX512BF16-NEXT: vpinsrw $4, %esi, %xmm0, %xmm0
+; AVX512BF16-NEXT: vpinsrw $5, %edx, %xmm0, %xmm0
+; AVX512BF16-NEXT: vpinsrw $6, %ecx, %xmm0, %xmm0
+; AVX512BF16-NEXT: vpinsrw $7, %eax, %xmm0, %xmm0
+; AVX512BF16-NEXT: retq
+ %a = load <8 x bfloat>, ptr %p
+ %r = call <8 x bfloat> @llvm.floor.v8bf16(<8 x bfloat> %a)
+ ret <8 x bfloat> %r
+}
+
+; Masked forms, exercising the AVX512_maskable patterns.
+define <8 x bfloat> @floor_v8bf16_mask(<8 x bfloat> %a, <8 x bfloat> %src, i8 %msk) {
+; AVX10_2-LABEL: floor_v8bf16_mask:
+; AVX10_2: # %bb.0:
+; AVX10_2-NEXT: kmovd %edi, %k1
+; AVX10_2-NEXT: vrndscalebf16 $9, %xmm0, %xmm1 {%k1}
+; AVX10_2-NEXT: vmovdqa %xmm1, %xmm0
+; AVX10_2-NEXT: retq
+;
+; AVX512BF16-LABEL: floor_v8bf16_mask:
+; AVX512BF16: # %bb.0:
+; AVX512BF16-NEXT: kmovd %edi, %k1
+; AVX512BF16-NEXT: vpextrw $7, %xmm0, %eax
+; AVX512BF16-NEXT: shll $16, %eax
+; AVX512BF16-NEXT: vmovd %eax, %xmm2
+; AVX512BF16-NEXT: vroundss $9, %xmm2, %xmm2, %xmm2
+; AVX512BF16-NEXT: vcvtneps2bf16 %xmm2, %xmm2
+; AVX512BF16-NEXT: vmovd %xmm2, %eax
+; AVX512BF16-NEXT: vpextrw $6, %xmm0, %ecx
+; AVX512BF16-NEXT: shll $16, %ecx
+; AVX512BF16-NEXT: vmovd %ecx, %xmm2
+; AVX512BF16-NEXT: vroundss $9, %xmm2, %xmm2, %xmm2
+; AVX512BF16-NEXT: vcvtneps2bf16 %xmm2, %xmm2
+; AVX512BF16-NEXT: vmovd %xmm2, %ecx
+; AVX512BF16-NEXT: vpextrw $5, %xmm0, %edx
+; AVX512BF16-NEXT: shll $16, %edx
+; AVX512BF16-NEXT: vmovd %edx, %xmm2
+; AVX512BF16-NEXT: vroundss $9, %xmm2, %xmm2, %xmm2
+; AVX512BF16-NEXT: vcvtneps2bf16 %xmm2, %xmm2
+; AVX512BF16-NEXT: vmovd %xmm2, %edx
+; AVX512BF16-NEXT: vpextrw $4, %xmm0, %esi
+; AVX512BF16-NEXT: shll $16, %esi
+; AVX512BF16-NEXT: vmovd %esi, %xmm2
+; AVX512BF16-NEXT: vroundss $9, %xmm2, %xmm2, %xmm2
+; AVX512BF16-NEXT: vcvtneps2bf16 %xmm2, %xmm2
+; AVX512BF16-NEXT: vmovd %xmm2, %esi
+; AVX512BF16-NEXT: vpextrw $3, %xmm0, %edi
+; AVX512BF16-NEXT: shll $16, %edi
+; AVX512BF16-NEXT: vmovd %edi, %xmm2
+; AVX512BF16-NEXT: vroundss $9, %xmm2, %xmm2, %xmm2
+; AVX512BF16-NEXT: vcvtneps2bf16 %xmm2, %xmm2
+; AVX512BF16-NEXT: vmovd %xmm2, %edi
+; AVX512BF16-NEXT: vpextrw $2, %xmm0, %r8d
+; AVX512BF16-NEXT: shll $16, %r8d
+; AVX512BF16-NEXT: vmovd %r8d, %xmm2
+; AVX512BF16-NEXT: vroundss $9, %xmm2, %xmm2, %xmm2
+; AVX512BF16-NEXT: vcvtneps2bf16 %xmm2, %xmm2
+; AVX512BF16-NEXT: vmovd %xmm2, %r8d
+; AVX512BF16-NEXT: vpextrw $1, %xmm0, %r9d
+; AVX512BF16-NEXT: shll $16, %r9d
+; AVX512BF16-NEXT: vmovd %r9d, %xmm2
+; AVX512BF16-NEXT: vroundss $9, %xmm2, %xmm2, %xmm2
+; AVX512BF16-NEXT: vcvtneps2bf16 %xmm2, %xmm2
+; AVX512BF16-NEXT: vmovd %xmm0, %r9d
+; AVX512BF16-NEXT: shll $16, %r9d
+; AVX512BF16-NEXT: vmovd %r9d, %xmm0
+; AVX512BF16-NEXT: vroundss $9, %xmm0, %xmm0, %xmm0
+; AVX512BF16-NEXT: vcvtneps2bf16 %xmm0, %xmm0
+; AVX512BF16-NEXT: vpunpcklwd {{.*#+}} xmm0 = xmm0[0],xmm2[0],xmm0[1],xmm2[1],xmm0[2],xmm2[2],xmm0[3],xmm2[3]
+; AVX512BF16-NEXT: vpinsrw $2, %r8d, %xmm0, %xmm0
+; AVX512BF16-NEXT: vpinsrw $3, %edi, %xmm0, %xmm0
+; AVX512BF16-NEXT: vpinsrw $4, %esi, %xmm0, %xmm0
+; AVX512BF16-NEXT: vpinsrw $5, %edx, %xmm0, %xmm0
+; AVX512BF16-NEXT: vpinsrw $6, %ecx, %xmm0, %xmm0
+; AVX512BF16-NEXT: vpinsrw $7, %eax, %xmm0, %xmm0
+; AVX512BF16-NEXT: vpblendmw %xmm0, %xmm1, %xmm0 {%k1}
+; AVX512BF16-NEXT: retq
+ %mask = bitcast i8 %msk to <8 x i1>
+ %r = call <8 x bfloat> @llvm.floor.v8bf16(<8 x bfloat> %a)
+ %res = select <8 x i1> %mask, <8 x bfloat> %r, <8 x bfloat> %src
+ ret <8 x bfloat> %res
+}
+
+define <8 x bfloat> @floor_v8bf16_maskz(<8 x bfloat> %a, i8 %msk) {
+; AVX10_2-LABEL: floor_v8bf16_maskz:
+; AVX10_2: # %bb.0:
+; AVX10_2-NEXT: kmovd %edi, %k1
+; AVX10_2-NEXT: vrndscalebf16 $9, %xmm0, %xmm0 {%k1} {z}
+; AVX10_2-NEXT: retq
+;
+; AVX512BF16-LABEL: floor_v8bf16_maskz:
+; AVX512BF16: # %bb.0:
+; AVX512BF16-NEXT: kmovd %edi, %k1
+; AVX512BF16-NEXT: vpextrw $7, %xmm0, %eax
+; AVX512BF16-NEXT: shll $16, %eax
+; AVX512BF16-NEXT: vmovd %eax, %xmm1
+; AVX512BF16-NEXT: vroundss $9, %xmm1, %xmm1, %xmm1
+; AVX512BF16-NEXT: vcvtneps2bf16 %xmm1, %xmm1
+; AVX512BF16-NEXT: vmovd %xmm1, %eax
+; AVX512BF16-NEXT: vpextrw $6, %xmm0, %ecx
+; AVX512BF16-NEXT: shll $16, %ecx
+; AVX512BF16-NEXT: vmovd %ecx, %xmm1
+; AVX512BF16-NEXT: vroundss $9, %xmm1, %xmm1, %xmm1
+; AVX512BF16-NEXT: vcvtneps2bf16 %xmm1, %xmm1
+; AVX512BF16-NEXT: vmovd %xmm1, %ecx
+; AVX512BF16-NEXT: vpextrw $5, %xmm0, %edx
+; AVX512BF16-NEXT: shll $16, %edx
+; AVX512BF16-NEXT: vmovd %edx, %xmm1
+; AVX512BF16-NEXT: vroundss $9, %xmm1, %xmm1, %xmm1
+; AVX512BF16-NEXT: vcvtneps2bf16 %xmm1, %xmm1
+; AVX512BF16-NEXT: vmovd %xmm1, %edx
+; AVX512BF16-NEXT: vpextrw $4, %xmm0, %esi
+; AVX512BF16-NEXT: shll $16, %esi
+; AVX512BF16-NEXT: vmovd %esi, %xmm1
+; AVX512BF16-NEXT: vroundss $9, %xmm1, %xmm1, %xmm1
+; AVX512BF16-NEXT: vcvtneps2bf16 %xmm1, %xmm1
+; AVX512BF16-NEXT: vmovd %xmm1, %esi
+; AVX512BF16-NEXT: vpextrw $3, %xmm0, %edi
+; AVX512BF16-NEXT: shll $16, %edi
+; AVX512BF16-NEXT: vmovd %edi, %xmm1
+; AVX512BF16-NEXT: vroundss $9, %xmm1, %xmm1, %xmm1
+; AVX512BF16-NEXT: vcvtneps2bf16 %xmm1, %xmm1
+; AVX512BF16-NEXT: vmovd %xmm1, %edi
+; AVX512BF16-NEXT: vpextrw $2, %xmm0, %r8d
+; AVX512BF16-NEXT: shll $16, %r8d
+; AVX512BF16-NEXT: vmovd %r8d, %xmm1
+; AVX512BF16-NEXT: vroundss $9, %xmm1, %xmm1, %xmm1
+; AVX512BF16-NEXT: vcvtneps2bf16 %xmm1, %xmm1
+; AVX512BF16-NEXT: vmovd %xmm1, %r8d
+; AVX512BF16-NEXT: vpextrw $1, %xmm0, %r9d
+; AVX512BF16-NEXT: shll $16, %r9d
+; AVX512BF16-NEXT: vmovd %r9d, %xmm1
+; AVX512BF16-NEXT: vroundss $9, %xmm1, %xmm1, %xmm1
+; AVX512BF16-NEXT: vcvtneps2bf16 %xmm1, %xmm1
+; AVX512BF16-NEXT: vmovd %xmm0, %r9d
+; AVX512BF16-NEXT: shll $16, %r9d
+; AVX512BF16-NEXT: vmovd %r9d, %xmm0
+; AVX512BF16-NEXT: vroundss $9, %xmm0, %xmm0, %xmm0
+; AVX512BF16-NEXT: vcvtneps2bf16 %xmm0, %xmm0
+; AVX512BF16-NEXT: vpunpcklwd {{.*#+}} xmm0 = xmm0[0],xmm1[0],xmm0[1],xmm1[1],xmm0[2],xmm1[2],xmm0[3],xmm1[3]
+; AVX512BF16-NEXT: vpinsrw $2, %r8d, %xmm0, %xmm0
+; AVX512BF16-NEXT: vpinsrw $3, %edi, %xmm0, %xmm0
+; AVX512BF16-NEXT: vpinsrw $4, %esi, %xmm0, %xmm0
+; AVX512BF16-NEXT: vpinsrw $5, %edx, %xmm0, %xmm0
+; AVX512BF16-NEXT: vpinsrw $6, %ecx, %xmm0, %xmm0
+; AVX512BF16-NEXT: vpinsrw $7, %eax, %xmm0, %xmm0
+; AVX512BF16-NEXT: vmovdqu16 %xmm0, %xmm0 {%k1} {z}
+; AVX512BF16-NEXT: retq
+ %mask = bitcast i8 %msk to <8 x i1>
+ %r = call <8 x bfloat> @llvm.floor.v8bf16(<8 x bfloat> %a)
+ %res = select <8 x i1> %mask, <8 x bfloat> %r, <8 x bfloat> zeroinitializer
+ ret <8 x bfloat> %res
+}
+
+; llvm.round is round-half-away-from-zero, which vrndscale cannot encode, so it
+; must stay scalarized rather than becoming a vrndscalebf16.
+define <8 x bfloat> @round_v8bf16(<8 x bfloat> %a) {
+; AVX10_2-LABEL: round_v8bf16:
+; AVX10_2-NOT: vrndscalebf16
+; AVX10_2: vroundss $11,
+; AVX10_2-NOT: vrndscalebf16
+ %r = call <8 x bfloat> @llvm.round.v8bf16(<8 x bfloat> %a)
+ ret <8 x bfloat> %r
+}
>From deea630dd8448fcc3c71ee867218a3aa12aec898 Mon Sep 17 00:00:00 2001
From: "Ziegler, Tim" <tim.ziegler at intel.com>
Date: Thu, 27 Aug 2026 15:49:06 +0200
Subject: [PATCH 2/2] Updated tests
---
llvm/test/CodeGen/X86/avx10_2bf16-rounding.ll | 1065 +++++++++--------
llvm/test/CodeGen/X86/znver-pdep.ll | 382 ++++--
llvm/test/CodeGen/X86/znver-pext.ll | 285 +++--
3 files changed, 1038 insertions(+), 694 deletions(-)
diff --git a/llvm/test/CodeGen/X86/avx10_2bf16-rounding.ll b/llvm/test/CodeGen/X86/avx10_2bf16-rounding.ll
index 4b9b764a6f5ce..11506e1bcae8c 100644
--- a/llvm/test/CodeGen/X86/avx10_2bf16-rounding.ll
+++ b/llvm/test/CodeGen/X86/avx10_2bf16-rounding.ll
@@ -16,49 +16,44 @@ define <8 x bfloat> @floor_v8bf16(<8 x bfloat> %a) {
; AVX512BF16-LABEL: floor_v8bf16:
; AVX512BF16: # %bb.0:
; AVX512BF16-NEXT: vpextrw $7, %xmm0, %eax
-; AVX512BF16-NEXT: shll $16, %eax
; AVX512BF16-NEXT: vmovd %eax, %xmm1
+; AVX512BF16-NEXT: vpslld $16, %xmm1, %xmm1
; AVX512BF16-NEXT: vroundss $9, %xmm1, %xmm1, %xmm1
; AVX512BF16-NEXT: vcvtneps2bf16 %xmm1, %xmm1
; AVX512BF16-NEXT: vmovd %xmm1, %eax
; AVX512BF16-NEXT: vpextrw $6, %xmm0, %ecx
-; AVX512BF16-NEXT: shll $16, %ecx
; AVX512BF16-NEXT: vmovd %ecx, %xmm1
+; AVX512BF16-NEXT: vpslld $16, %xmm1, %xmm1
; AVX512BF16-NEXT: vroundss $9, %xmm1, %xmm1, %xmm1
; AVX512BF16-NEXT: vcvtneps2bf16 %xmm1, %xmm1
; AVX512BF16-NEXT: vmovd %xmm1, %ecx
; AVX512BF16-NEXT: vpextrw $5, %xmm0, %edx
-; AVX512BF16-NEXT: shll $16, %edx
; AVX512BF16-NEXT: vmovd %edx, %xmm1
+; AVX512BF16-NEXT: vpslld $16, %xmm1, %xmm1
; AVX512BF16-NEXT: vroundss $9, %xmm1, %xmm1, %xmm1
; AVX512BF16-NEXT: vcvtneps2bf16 %xmm1, %xmm1
; AVX512BF16-NEXT: vmovd %xmm1, %edx
-; AVX512BF16-NEXT: vpextrw $4, %xmm0, %esi
-; AVX512BF16-NEXT: shll $16, %esi
-; AVX512BF16-NEXT: vmovd %esi, %xmm1
-; AVX512BF16-NEXT: vroundss $9, %xmm1, %xmm1, %xmm1
-; AVX512BF16-NEXT: vcvtneps2bf16 %xmm1, %xmm1
-; AVX512BF16-NEXT: vmovd %xmm1, %esi
+; AVX512BF16-NEXT: vpxor %xmm1, %xmm1, %xmm1
+; AVX512BF16-NEXT: vpunpckhwd {{.*#+}} xmm2 = xmm1[4],xmm0[4],xmm1[5],xmm0[5],xmm1[6],xmm0[6],xmm1[7],xmm0[7]
+; AVX512BF16-NEXT: vroundss $9, %xmm2, %xmm2, %xmm2
+; AVX512BF16-NEXT: vcvtneps2bf16 %xmm2, %xmm2
+; AVX512BF16-NEXT: vmovd %xmm2, %esi
; AVX512BF16-NEXT: vpextrw $3, %xmm0, %edi
-; AVX512BF16-NEXT: shll $16, %edi
-; AVX512BF16-NEXT: vmovd %edi, %xmm1
-; AVX512BF16-NEXT: vroundss $9, %xmm1, %xmm1, %xmm1
-; AVX512BF16-NEXT: vcvtneps2bf16 %xmm1, %xmm1
-; AVX512BF16-NEXT: vmovd %xmm1, %edi
+; AVX512BF16-NEXT: vmovd %edi, %xmm2
+; AVX512BF16-NEXT: vpslld $16, %xmm2, %xmm2
+; AVX512BF16-NEXT: vroundss $9, %xmm2, %xmm2, %xmm2
+; AVX512BF16-NEXT: vcvtneps2bf16 %xmm2, %xmm2
+; AVX512BF16-NEXT: vmovd %xmm2, %edi
; AVX512BF16-NEXT: vpextrw $2, %xmm0, %r8d
-; AVX512BF16-NEXT: shll $16, %r8d
-; AVX512BF16-NEXT: vmovd %r8d, %xmm1
-; AVX512BF16-NEXT: vroundss $9, %xmm1, %xmm1, %xmm1
-; AVX512BF16-NEXT: vcvtneps2bf16 %xmm1, %xmm1
-; AVX512BF16-NEXT: vmovd %xmm1, %r8d
-; AVX512BF16-NEXT: vpextrw $1, %xmm0, %r9d
-; AVX512BF16-NEXT: shll $16, %r9d
-; AVX512BF16-NEXT: vmovd %r9d, %xmm1
+; AVX512BF16-NEXT: vmovd %r8d, %xmm2
+; AVX512BF16-NEXT: vpslld $16, %xmm2, %xmm2
+; AVX512BF16-NEXT: vroundss $9, %xmm2, %xmm2, %xmm2
+; AVX512BF16-NEXT: vcvtneps2bf16 %xmm2, %xmm2
+; AVX512BF16-NEXT: vmovd %xmm2, %r8d
+; AVX512BF16-NEXT: vpblendw {{.*#+}} xmm1 = xmm1[0],xmm0[1],xmm1[2],xmm0[3],xmm1[4],xmm0[5],xmm1[6],xmm0[7]
; AVX512BF16-NEXT: vroundss $9, %xmm1, %xmm1, %xmm1
; AVX512BF16-NEXT: vcvtneps2bf16 %xmm1, %xmm1
-; AVX512BF16-NEXT: vmovd %xmm0, %r9d
-; AVX512BF16-NEXT: shll $16, %r9d
-; AVX512BF16-NEXT: vmovd %r9d, %xmm0
+; AVX512BF16-NEXT: vpslld $16, %xmm0, %xmm0
; AVX512BF16-NEXT: vroundss $9, %xmm0, %xmm0, %xmm0
; AVX512BF16-NEXT: vcvtneps2bf16 %xmm0, %xmm0
; AVX512BF16-NEXT: vpunpcklwd {{.*#+}} xmm0 = xmm0[0],xmm1[0],xmm0[1],xmm1[1],xmm0[2],xmm1[2],xmm0[3],xmm1[3]
@@ -81,9 +76,7 @@ define bfloat @floor_bf16(bfloat %a) {
;
; AVX512BF16-LABEL: floor_bf16:
; AVX512BF16: # %bb.0:
-; AVX512BF16-NEXT: vpextrw $0, %xmm0, %eax
-; AVX512BF16-NEXT: shll $16, %eax
-; AVX512BF16-NEXT: vmovd %eax, %xmm0
+; AVX512BF16-NEXT: vpslld $16, %xmm0, %xmm0
; AVX512BF16-NEXT: vroundss $9, %xmm0, %xmm0, %xmm0
; AVX512BF16-NEXT: vcvtneps2bf16 %xmm0, %xmm0
; AVX512BF16-NEXT: retq
@@ -100,49 +93,44 @@ define <8 x bfloat> @ceil_v8bf16(<8 x bfloat> %a) {
; AVX512BF16-LABEL: ceil_v8bf16:
; AVX512BF16: # %bb.0:
; AVX512BF16-NEXT: vpextrw $7, %xmm0, %eax
-; AVX512BF16-NEXT: shll $16, %eax
; AVX512BF16-NEXT: vmovd %eax, %xmm1
+; AVX512BF16-NEXT: vpslld $16, %xmm1, %xmm1
; AVX512BF16-NEXT: vroundss $10, %xmm1, %xmm1, %xmm1
; AVX512BF16-NEXT: vcvtneps2bf16 %xmm1, %xmm1
; AVX512BF16-NEXT: vmovd %xmm1, %eax
; AVX512BF16-NEXT: vpextrw $6, %xmm0, %ecx
-; AVX512BF16-NEXT: shll $16, %ecx
; AVX512BF16-NEXT: vmovd %ecx, %xmm1
+; AVX512BF16-NEXT: vpslld $16, %xmm1, %xmm1
; AVX512BF16-NEXT: vroundss $10, %xmm1, %xmm1, %xmm1
; AVX512BF16-NEXT: vcvtneps2bf16 %xmm1, %xmm1
; AVX512BF16-NEXT: vmovd %xmm1, %ecx
; AVX512BF16-NEXT: vpextrw $5, %xmm0, %edx
-; AVX512BF16-NEXT: shll $16, %edx
; AVX512BF16-NEXT: vmovd %edx, %xmm1
+; AVX512BF16-NEXT: vpslld $16, %xmm1, %xmm1
; AVX512BF16-NEXT: vroundss $10, %xmm1, %xmm1, %xmm1
; AVX512BF16-NEXT: vcvtneps2bf16 %xmm1, %xmm1
; AVX512BF16-NEXT: vmovd %xmm1, %edx
-; AVX512BF16-NEXT: vpextrw $4, %xmm0, %esi
-; AVX512BF16-NEXT: shll $16, %esi
-; AVX512BF16-NEXT: vmovd %esi, %xmm1
-; AVX512BF16-NEXT: vroundss $10, %xmm1, %xmm1, %xmm1
-; AVX512BF16-NEXT: vcvtneps2bf16 %xmm1, %xmm1
-; AVX512BF16-NEXT: vmovd %xmm1, %esi
+; AVX512BF16-NEXT: vpxor %xmm1, %xmm1, %xmm1
+; AVX512BF16-NEXT: vpunpckhwd {{.*#+}} xmm2 = xmm1[4],xmm0[4],xmm1[5],xmm0[5],xmm1[6],xmm0[6],xmm1[7],xmm0[7]
+; AVX512BF16-NEXT: vroundss $10, %xmm2, %xmm2, %xmm2
+; AVX512BF16-NEXT: vcvtneps2bf16 %xmm2, %xmm2
+; AVX512BF16-NEXT: vmovd %xmm2, %esi
; AVX512BF16-NEXT: vpextrw $3, %xmm0, %edi
-; AVX512BF16-NEXT: shll $16, %edi
-; AVX512BF16-NEXT: vmovd %edi, %xmm1
-; AVX512BF16-NEXT: vroundss $10, %xmm1, %xmm1, %xmm1
-; AVX512BF16-NEXT: vcvtneps2bf16 %xmm1, %xmm1
-; AVX512BF16-NEXT: vmovd %xmm1, %edi
+; AVX512BF16-NEXT: vmovd %edi, %xmm2
+; AVX512BF16-NEXT: vpslld $16, %xmm2, %xmm2
+; AVX512BF16-NEXT: vroundss $10, %xmm2, %xmm2, %xmm2
+; AVX512BF16-NEXT: vcvtneps2bf16 %xmm2, %xmm2
+; AVX512BF16-NEXT: vmovd %xmm2, %edi
; AVX512BF16-NEXT: vpextrw $2, %xmm0, %r8d
-; AVX512BF16-NEXT: shll $16, %r8d
-; AVX512BF16-NEXT: vmovd %r8d, %xmm1
-; AVX512BF16-NEXT: vroundss $10, %xmm1, %xmm1, %xmm1
-; AVX512BF16-NEXT: vcvtneps2bf16 %xmm1, %xmm1
-; AVX512BF16-NEXT: vmovd %xmm1, %r8d
-; AVX512BF16-NEXT: vpextrw $1, %xmm0, %r9d
-; AVX512BF16-NEXT: shll $16, %r9d
-; AVX512BF16-NEXT: vmovd %r9d, %xmm1
+; AVX512BF16-NEXT: vmovd %r8d, %xmm2
+; AVX512BF16-NEXT: vpslld $16, %xmm2, %xmm2
+; AVX512BF16-NEXT: vroundss $10, %xmm2, %xmm2, %xmm2
+; AVX512BF16-NEXT: vcvtneps2bf16 %xmm2, %xmm2
+; AVX512BF16-NEXT: vmovd %xmm2, %r8d
+; AVX512BF16-NEXT: vpblendw {{.*#+}} xmm1 = xmm1[0],xmm0[1],xmm1[2],xmm0[3],xmm1[4],xmm0[5],xmm1[6],xmm0[7]
; AVX512BF16-NEXT: vroundss $10, %xmm1, %xmm1, %xmm1
; AVX512BF16-NEXT: vcvtneps2bf16 %xmm1, %xmm1
-; AVX512BF16-NEXT: vmovd %xmm0, %r9d
-; AVX512BF16-NEXT: shll $16, %r9d
-; AVX512BF16-NEXT: vmovd %r9d, %xmm0
+; AVX512BF16-NEXT: vpslld $16, %xmm0, %xmm0
; AVX512BF16-NEXT: vroundss $10, %xmm0, %xmm0, %xmm0
; AVX512BF16-NEXT: vcvtneps2bf16 %xmm0, %xmm0
; AVX512BF16-NEXT: vpunpcklwd {{.*#+}} xmm0 = xmm0[0],xmm1[0],xmm0[1],xmm1[1],xmm0[2],xmm1[2],xmm0[3],xmm1[3]
@@ -165,9 +153,7 @@ define bfloat @ceil_bf16(bfloat %a) {
;
; AVX512BF16-LABEL: ceil_bf16:
; AVX512BF16: # %bb.0:
-; AVX512BF16-NEXT: vpextrw $0, %xmm0, %eax
-; AVX512BF16-NEXT: shll $16, %eax
-; AVX512BF16-NEXT: vmovd %eax, %xmm0
+; AVX512BF16-NEXT: vpslld $16, %xmm0, %xmm0
; AVX512BF16-NEXT: vroundss $10, %xmm0, %xmm0, %xmm0
; AVX512BF16-NEXT: vcvtneps2bf16 %xmm0, %xmm0
; AVX512BF16-NEXT: retq
@@ -184,49 +170,44 @@ define <8 x bfloat> @trunc_v8bf16(<8 x bfloat> %a) {
; AVX512BF16-LABEL: trunc_v8bf16:
; AVX512BF16: # %bb.0:
; AVX512BF16-NEXT: vpextrw $7, %xmm0, %eax
-; AVX512BF16-NEXT: shll $16, %eax
; AVX512BF16-NEXT: vmovd %eax, %xmm1
+; AVX512BF16-NEXT: vpslld $16, %xmm1, %xmm1
; AVX512BF16-NEXT: vroundss $11, %xmm1, %xmm1, %xmm1
; AVX512BF16-NEXT: vcvtneps2bf16 %xmm1, %xmm1
; AVX512BF16-NEXT: vmovd %xmm1, %eax
; AVX512BF16-NEXT: vpextrw $6, %xmm0, %ecx
-; AVX512BF16-NEXT: shll $16, %ecx
; AVX512BF16-NEXT: vmovd %ecx, %xmm1
+; AVX512BF16-NEXT: vpslld $16, %xmm1, %xmm1
; AVX512BF16-NEXT: vroundss $11, %xmm1, %xmm1, %xmm1
; AVX512BF16-NEXT: vcvtneps2bf16 %xmm1, %xmm1
; AVX512BF16-NEXT: vmovd %xmm1, %ecx
; AVX512BF16-NEXT: vpextrw $5, %xmm0, %edx
-; AVX512BF16-NEXT: shll $16, %edx
; AVX512BF16-NEXT: vmovd %edx, %xmm1
+; AVX512BF16-NEXT: vpslld $16, %xmm1, %xmm1
; AVX512BF16-NEXT: vroundss $11, %xmm1, %xmm1, %xmm1
; AVX512BF16-NEXT: vcvtneps2bf16 %xmm1, %xmm1
; AVX512BF16-NEXT: vmovd %xmm1, %edx
-; AVX512BF16-NEXT: vpextrw $4, %xmm0, %esi
-; AVX512BF16-NEXT: shll $16, %esi
-; AVX512BF16-NEXT: vmovd %esi, %xmm1
-; AVX512BF16-NEXT: vroundss $11, %xmm1, %xmm1, %xmm1
-; AVX512BF16-NEXT: vcvtneps2bf16 %xmm1, %xmm1
-; AVX512BF16-NEXT: vmovd %xmm1, %esi
+; AVX512BF16-NEXT: vpxor %xmm1, %xmm1, %xmm1
+; AVX512BF16-NEXT: vpunpckhwd {{.*#+}} xmm2 = xmm1[4],xmm0[4],xmm1[5],xmm0[5],xmm1[6],xmm0[6],xmm1[7],xmm0[7]
+; AVX512BF16-NEXT: vroundss $11, %xmm2, %xmm2, %xmm2
+; AVX512BF16-NEXT: vcvtneps2bf16 %xmm2, %xmm2
+; AVX512BF16-NEXT: vmovd %xmm2, %esi
; AVX512BF16-NEXT: vpextrw $3, %xmm0, %edi
-; AVX512BF16-NEXT: shll $16, %edi
-; AVX512BF16-NEXT: vmovd %edi, %xmm1
-; AVX512BF16-NEXT: vroundss $11, %xmm1, %xmm1, %xmm1
-; AVX512BF16-NEXT: vcvtneps2bf16 %xmm1, %xmm1
-; AVX512BF16-NEXT: vmovd %xmm1, %edi
+; AVX512BF16-NEXT: vmovd %edi, %xmm2
+; AVX512BF16-NEXT: vpslld $16, %xmm2, %xmm2
+; AVX512BF16-NEXT: vroundss $11, %xmm2, %xmm2, %xmm2
+; AVX512BF16-NEXT: vcvtneps2bf16 %xmm2, %xmm2
+; AVX512BF16-NEXT: vmovd %xmm2, %edi
; AVX512BF16-NEXT: vpextrw $2, %xmm0, %r8d
-; AVX512BF16-NEXT: shll $16, %r8d
-; AVX512BF16-NEXT: vmovd %r8d, %xmm1
-; AVX512BF16-NEXT: vroundss $11, %xmm1, %xmm1, %xmm1
-; AVX512BF16-NEXT: vcvtneps2bf16 %xmm1, %xmm1
-; AVX512BF16-NEXT: vmovd %xmm1, %r8d
-; AVX512BF16-NEXT: vpextrw $1, %xmm0, %r9d
-; AVX512BF16-NEXT: shll $16, %r9d
-; AVX512BF16-NEXT: vmovd %r9d, %xmm1
+; AVX512BF16-NEXT: vmovd %r8d, %xmm2
+; AVX512BF16-NEXT: vpslld $16, %xmm2, %xmm2
+; AVX512BF16-NEXT: vroundss $11, %xmm2, %xmm2, %xmm2
+; AVX512BF16-NEXT: vcvtneps2bf16 %xmm2, %xmm2
+; AVX512BF16-NEXT: vmovd %xmm2, %r8d
+; AVX512BF16-NEXT: vpblendw {{.*#+}} xmm1 = xmm1[0],xmm0[1],xmm1[2],xmm0[3],xmm1[4],xmm0[5],xmm1[6],xmm0[7]
; AVX512BF16-NEXT: vroundss $11, %xmm1, %xmm1, %xmm1
; AVX512BF16-NEXT: vcvtneps2bf16 %xmm1, %xmm1
-; AVX512BF16-NEXT: vmovd %xmm0, %r9d
-; AVX512BF16-NEXT: shll $16, %r9d
-; AVX512BF16-NEXT: vmovd %r9d, %xmm0
+; AVX512BF16-NEXT: vpslld $16, %xmm0, %xmm0
; AVX512BF16-NEXT: vroundss $11, %xmm0, %xmm0, %xmm0
; AVX512BF16-NEXT: vcvtneps2bf16 %xmm0, %xmm0
; AVX512BF16-NEXT: vpunpcklwd {{.*#+}} xmm0 = xmm0[0],xmm1[0],xmm0[1],xmm1[1],xmm0[2],xmm1[2],xmm0[3],xmm1[3]
@@ -249,9 +230,7 @@ define bfloat @trunc_bf16(bfloat %a) {
;
; AVX512BF16-LABEL: trunc_bf16:
; AVX512BF16: # %bb.0:
-; AVX512BF16-NEXT: vpextrw $0, %xmm0, %eax
-; AVX512BF16-NEXT: shll $16, %eax
-; AVX512BF16-NEXT: vmovd %eax, %xmm0
+; AVX512BF16-NEXT: vpslld $16, %xmm0, %xmm0
; AVX512BF16-NEXT: vroundss $11, %xmm0, %xmm0, %xmm0
; AVX512BF16-NEXT: vcvtneps2bf16 %xmm0, %xmm0
; AVX512BF16-NEXT: retq
@@ -268,49 +247,44 @@ define <8 x bfloat> @rint_v8bf16(<8 x bfloat> %a) {
; AVX512BF16-LABEL: rint_v8bf16:
; AVX512BF16: # %bb.0:
; AVX512BF16-NEXT: vpextrw $7, %xmm0, %eax
-; AVX512BF16-NEXT: shll $16, %eax
; AVX512BF16-NEXT: vmovd %eax, %xmm1
+; AVX512BF16-NEXT: vpslld $16, %xmm1, %xmm1
; AVX512BF16-NEXT: vroundss $4, %xmm1, %xmm1, %xmm1
; AVX512BF16-NEXT: vcvtneps2bf16 %xmm1, %xmm1
; AVX512BF16-NEXT: vmovd %xmm1, %eax
; AVX512BF16-NEXT: vpextrw $6, %xmm0, %ecx
-; AVX512BF16-NEXT: shll $16, %ecx
; AVX512BF16-NEXT: vmovd %ecx, %xmm1
+; AVX512BF16-NEXT: vpslld $16, %xmm1, %xmm1
; AVX512BF16-NEXT: vroundss $4, %xmm1, %xmm1, %xmm1
; AVX512BF16-NEXT: vcvtneps2bf16 %xmm1, %xmm1
; AVX512BF16-NEXT: vmovd %xmm1, %ecx
; AVX512BF16-NEXT: vpextrw $5, %xmm0, %edx
-; AVX512BF16-NEXT: shll $16, %edx
; AVX512BF16-NEXT: vmovd %edx, %xmm1
+; AVX512BF16-NEXT: vpslld $16, %xmm1, %xmm1
; AVX512BF16-NEXT: vroundss $4, %xmm1, %xmm1, %xmm1
; AVX512BF16-NEXT: vcvtneps2bf16 %xmm1, %xmm1
; AVX512BF16-NEXT: vmovd %xmm1, %edx
-; AVX512BF16-NEXT: vpextrw $4, %xmm0, %esi
-; AVX512BF16-NEXT: shll $16, %esi
-; AVX512BF16-NEXT: vmovd %esi, %xmm1
-; AVX512BF16-NEXT: vroundss $4, %xmm1, %xmm1, %xmm1
-; AVX512BF16-NEXT: vcvtneps2bf16 %xmm1, %xmm1
-; AVX512BF16-NEXT: vmovd %xmm1, %esi
+; AVX512BF16-NEXT: vpxor %xmm1, %xmm1, %xmm1
+; AVX512BF16-NEXT: vpunpckhwd {{.*#+}} xmm2 = xmm1[4],xmm0[4],xmm1[5],xmm0[5],xmm1[6],xmm0[6],xmm1[7],xmm0[7]
+; AVX512BF16-NEXT: vroundss $4, %xmm2, %xmm2, %xmm2
+; AVX512BF16-NEXT: vcvtneps2bf16 %xmm2, %xmm2
+; AVX512BF16-NEXT: vmovd %xmm2, %esi
; AVX512BF16-NEXT: vpextrw $3, %xmm0, %edi
-; AVX512BF16-NEXT: shll $16, %edi
-; AVX512BF16-NEXT: vmovd %edi, %xmm1
-; AVX512BF16-NEXT: vroundss $4, %xmm1, %xmm1, %xmm1
-; AVX512BF16-NEXT: vcvtneps2bf16 %xmm1, %xmm1
-; AVX512BF16-NEXT: vmovd %xmm1, %edi
+; AVX512BF16-NEXT: vmovd %edi, %xmm2
+; AVX512BF16-NEXT: vpslld $16, %xmm2, %xmm2
+; AVX512BF16-NEXT: vroundss $4, %xmm2, %xmm2, %xmm2
+; AVX512BF16-NEXT: vcvtneps2bf16 %xmm2, %xmm2
+; AVX512BF16-NEXT: vmovd %xmm2, %edi
; AVX512BF16-NEXT: vpextrw $2, %xmm0, %r8d
-; AVX512BF16-NEXT: shll $16, %r8d
-; AVX512BF16-NEXT: vmovd %r8d, %xmm1
-; AVX512BF16-NEXT: vroundss $4, %xmm1, %xmm1, %xmm1
-; AVX512BF16-NEXT: vcvtneps2bf16 %xmm1, %xmm1
-; AVX512BF16-NEXT: vmovd %xmm1, %r8d
-; AVX512BF16-NEXT: vpextrw $1, %xmm0, %r9d
-; AVX512BF16-NEXT: shll $16, %r9d
-; AVX512BF16-NEXT: vmovd %r9d, %xmm1
+; AVX512BF16-NEXT: vmovd %r8d, %xmm2
+; AVX512BF16-NEXT: vpslld $16, %xmm2, %xmm2
+; AVX512BF16-NEXT: vroundss $4, %xmm2, %xmm2, %xmm2
+; AVX512BF16-NEXT: vcvtneps2bf16 %xmm2, %xmm2
+; AVX512BF16-NEXT: vmovd %xmm2, %r8d
+; AVX512BF16-NEXT: vpblendw {{.*#+}} xmm1 = xmm1[0],xmm0[1],xmm1[2],xmm0[3],xmm1[4],xmm0[5],xmm1[6],xmm0[7]
; AVX512BF16-NEXT: vroundss $4, %xmm1, %xmm1, %xmm1
; AVX512BF16-NEXT: vcvtneps2bf16 %xmm1, %xmm1
-; AVX512BF16-NEXT: vmovd %xmm0, %r9d
-; AVX512BF16-NEXT: shll $16, %r9d
-; AVX512BF16-NEXT: vmovd %r9d, %xmm0
+; AVX512BF16-NEXT: vpslld $16, %xmm0, %xmm0
; AVX512BF16-NEXT: vroundss $4, %xmm0, %xmm0, %xmm0
; AVX512BF16-NEXT: vcvtneps2bf16 %xmm0, %xmm0
; AVX512BF16-NEXT: vpunpcklwd {{.*#+}} xmm0 = xmm0[0],xmm1[0],xmm0[1],xmm1[1],xmm0[2],xmm1[2],xmm0[3],xmm1[3]
@@ -333,9 +307,7 @@ define bfloat @rint_bf16(bfloat %a) {
;
; AVX512BF16-LABEL: rint_bf16:
; AVX512BF16: # %bb.0:
-; AVX512BF16-NEXT: vpextrw $0, %xmm0, %eax
-; AVX512BF16-NEXT: shll $16, %eax
-; AVX512BF16-NEXT: vmovd %eax, %xmm0
+; AVX512BF16-NEXT: vpslld $16, %xmm0, %xmm0
; AVX512BF16-NEXT: vroundss $4, %xmm0, %xmm0, %xmm0
; AVX512BF16-NEXT: vcvtneps2bf16 %xmm0, %xmm0
; AVX512BF16-NEXT: retq
@@ -352,49 +324,44 @@ define <8 x bfloat> @nearbyint_v8bf16(<8 x bfloat> %a) {
; AVX512BF16-LABEL: nearbyint_v8bf16:
; AVX512BF16: # %bb.0:
; AVX512BF16-NEXT: vpextrw $7, %xmm0, %eax
-; AVX512BF16-NEXT: shll $16, %eax
; AVX512BF16-NEXT: vmovd %eax, %xmm1
+; AVX512BF16-NEXT: vpslld $16, %xmm1, %xmm1
; AVX512BF16-NEXT: vroundss $12, %xmm1, %xmm1, %xmm1
; AVX512BF16-NEXT: vcvtneps2bf16 %xmm1, %xmm1
; AVX512BF16-NEXT: vmovd %xmm1, %eax
; AVX512BF16-NEXT: vpextrw $6, %xmm0, %ecx
-; AVX512BF16-NEXT: shll $16, %ecx
; AVX512BF16-NEXT: vmovd %ecx, %xmm1
+; AVX512BF16-NEXT: vpslld $16, %xmm1, %xmm1
; AVX512BF16-NEXT: vroundss $12, %xmm1, %xmm1, %xmm1
; AVX512BF16-NEXT: vcvtneps2bf16 %xmm1, %xmm1
; AVX512BF16-NEXT: vmovd %xmm1, %ecx
; AVX512BF16-NEXT: vpextrw $5, %xmm0, %edx
-; AVX512BF16-NEXT: shll $16, %edx
; AVX512BF16-NEXT: vmovd %edx, %xmm1
+; AVX512BF16-NEXT: vpslld $16, %xmm1, %xmm1
; AVX512BF16-NEXT: vroundss $12, %xmm1, %xmm1, %xmm1
; AVX512BF16-NEXT: vcvtneps2bf16 %xmm1, %xmm1
; AVX512BF16-NEXT: vmovd %xmm1, %edx
-; AVX512BF16-NEXT: vpextrw $4, %xmm0, %esi
-; AVX512BF16-NEXT: shll $16, %esi
-; AVX512BF16-NEXT: vmovd %esi, %xmm1
-; AVX512BF16-NEXT: vroundss $12, %xmm1, %xmm1, %xmm1
-; AVX512BF16-NEXT: vcvtneps2bf16 %xmm1, %xmm1
-; AVX512BF16-NEXT: vmovd %xmm1, %esi
+; AVX512BF16-NEXT: vpxor %xmm1, %xmm1, %xmm1
+; AVX512BF16-NEXT: vpunpckhwd {{.*#+}} xmm2 = xmm1[4],xmm0[4],xmm1[5],xmm0[5],xmm1[6],xmm0[6],xmm1[7],xmm0[7]
+; AVX512BF16-NEXT: vroundss $12, %xmm2, %xmm2, %xmm2
+; AVX512BF16-NEXT: vcvtneps2bf16 %xmm2, %xmm2
+; AVX512BF16-NEXT: vmovd %xmm2, %esi
; AVX512BF16-NEXT: vpextrw $3, %xmm0, %edi
-; AVX512BF16-NEXT: shll $16, %edi
-; AVX512BF16-NEXT: vmovd %edi, %xmm1
-; AVX512BF16-NEXT: vroundss $12, %xmm1, %xmm1, %xmm1
-; AVX512BF16-NEXT: vcvtneps2bf16 %xmm1, %xmm1
-; AVX512BF16-NEXT: vmovd %xmm1, %edi
+; AVX512BF16-NEXT: vmovd %edi, %xmm2
+; AVX512BF16-NEXT: vpslld $16, %xmm2, %xmm2
+; AVX512BF16-NEXT: vroundss $12, %xmm2, %xmm2, %xmm2
+; AVX512BF16-NEXT: vcvtneps2bf16 %xmm2, %xmm2
+; AVX512BF16-NEXT: vmovd %xmm2, %edi
; AVX512BF16-NEXT: vpextrw $2, %xmm0, %r8d
-; AVX512BF16-NEXT: shll $16, %r8d
-; AVX512BF16-NEXT: vmovd %r8d, %xmm1
-; AVX512BF16-NEXT: vroundss $12, %xmm1, %xmm1, %xmm1
-; AVX512BF16-NEXT: vcvtneps2bf16 %xmm1, %xmm1
-; AVX512BF16-NEXT: vmovd %xmm1, %r8d
-; AVX512BF16-NEXT: vpextrw $1, %xmm0, %r9d
-; AVX512BF16-NEXT: shll $16, %r9d
-; AVX512BF16-NEXT: vmovd %r9d, %xmm1
+; AVX512BF16-NEXT: vmovd %r8d, %xmm2
+; AVX512BF16-NEXT: vpslld $16, %xmm2, %xmm2
+; AVX512BF16-NEXT: vroundss $12, %xmm2, %xmm2, %xmm2
+; AVX512BF16-NEXT: vcvtneps2bf16 %xmm2, %xmm2
+; AVX512BF16-NEXT: vmovd %xmm2, %r8d
+; AVX512BF16-NEXT: vpblendw {{.*#+}} xmm1 = xmm1[0],xmm0[1],xmm1[2],xmm0[3],xmm1[4],xmm0[5],xmm1[6],xmm0[7]
; AVX512BF16-NEXT: vroundss $12, %xmm1, %xmm1, %xmm1
; AVX512BF16-NEXT: vcvtneps2bf16 %xmm1, %xmm1
-; AVX512BF16-NEXT: vmovd %xmm0, %r9d
-; AVX512BF16-NEXT: shll $16, %r9d
-; AVX512BF16-NEXT: vmovd %r9d, %xmm0
+; AVX512BF16-NEXT: vpslld $16, %xmm0, %xmm0
; AVX512BF16-NEXT: vroundss $12, %xmm0, %xmm0, %xmm0
; AVX512BF16-NEXT: vcvtneps2bf16 %xmm0, %xmm0
; AVX512BF16-NEXT: vpunpcklwd {{.*#+}} xmm0 = xmm0[0],xmm1[0],xmm0[1],xmm1[1],xmm0[2],xmm1[2],xmm0[3],xmm1[3]
@@ -417,9 +384,7 @@ define bfloat @nearbyint_bf16(bfloat %a) {
;
; AVX512BF16-LABEL: nearbyint_bf16:
; AVX512BF16: # %bb.0:
-; AVX512BF16-NEXT: vpextrw $0, %xmm0, %eax
-; AVX512BF16-NEXT: shll $16, %eax
-; AVX512BF16-NEXT: vmovd %eax, %xmm0
+; AVX512BF16-NEXT: vpslld $16, %xmm0, %xmm0
; AVX512BF16-NEXT: vroundss $12, %xmm0, %xmm0, %xmm0
; AVX512BF16-NEXT: vcvtneps2bf16 %xmm0, %xmm0
; AVX512BF16-NEXT: retq
@@ -436,49 +401,44 @@ define <8 x bfloat> @roundeven_v8bf16(<8 x bfloat> %a) {
; AVX512BF16-LABEL: roundeven_v8bf16:
; AVX512BF16: # %bb.0:
; AVX512BF16-NEXT: vpextrw $7, %xmm0, %eax
-; AVX512BF16-NEXT: shll $16, %eax
; AVX512BF16-NEXT: vmovd %eax, %xmm1
+; AVX512BF16-NEXT: vpslld $16, %xmm1, %xmm1
; AVX512BF16-NEXT: vroundss $8, %xmm1, %xmm1, %xmm1
; AVX512BF16-NEXT: vcvtneps2bf16 %xmm1, %xmm1
; AVX512BF16-NEXT: vmovd %xmm1, %eax
; AVX512BF16-NEXT: vpextrw $6, %xmm0, %ecx
-; AVX512BF16-NEXT: shll $16, %ecx
; AVX512BF16-NEXT: vmovd %ecx, %xmm1
+; AVX512BF16-NEXT: vpslld $16, %xmm1, %xmm1
; AVX512BF16-NEXT: vroundss $8, %xmm1, %xmm1, %xmm1
; AVX512BF16-NEXT: vcvtneps2bf16 %xmm1, %xmm1
; AVX512BF16-NEXT: vmovd %xmm1, %ecx
; AVX512BF16-NEXT: vpextrw $5, %xmm0, %edx
-; AVX512BF16-NEXT: shll $16, %edx
; AVX512BF16-NEXT: vmovd %edx, %xmm1
+; AVX512BF16-NEXT: vpslld $16, %xmm1, %xmm1
; AVX512BF16-NEXT: vroundss $8, %xmm1, %xmm1, %xmm1
; AVX512BF16-NEXT: vcvtneps2bf16 %xmm1, %xmm1
; AVX512BF16-NEXT: vmovd %xmm1, %edx
-; AVX512BF16-NEXT: vpextrw $4, %xmm0, %esi
-; AVX512BF16-NEXT: shll $16, %esi
-; AVX512BF16-NEXT: vmovd %esi, %xmm1
-; AVX512BF16-NEXT: vroundss $8, %xmm1, %xmm1, %xmm1
-; AVX512BF16-NEXT: vcvtneps2bf16 %xmm1, %xmm1
-; AVX512BF16-NEXT: vmovd %xmm1, %esi
+; AVX512BF16-NEXT: vpxor %xmm1, %xmm1, %xmm1
+; AVX512BF16-NEXT: vpunpckhwd {{.*#+}} xmm2 = xmm1[4],xmm0[4],xmm1[5],xmm0[5],xmm1[6],xmm0[6],xmm1[7],xmm0[7]
+; AVX512BF16-NEXT: vroundss $8, %xmm2, %xmm2, %xmm2
+; AVX512BF16-NEXT: vcvtneps2bf16 %xmm2, %xmm2
+; AVX512BF16-NEXT: vmovd %xmm2, %esi
; AVX512BF16-NEXT: vpextrw $3, %xmm0, %edi
-; AVX512BF16-NEXT: shll $16, %edi
-; AVX512BF16-NEXT: vmovd %edi, %xmm1
-; AVX512BF16-NEXT: vroundss $8, %xmm1, %xmm1, %xmm1
-; AVX512BF16-NEXT: vcvtneps2bf16 %xmm1, %xmm1
-; AVX512BF16-NEXT: vmovd %xmm1, %edi
+; AVX512BF16-NEXT: vmovd %edi, %xmm2
+; AVX512BF16-NEXT: vpslld $16, %xmm2, %xmm2
+; AVX512BF16-NEXT: vroundss $8, %xmm2, %xmm2, %xmm2
+; AVX512BF16-NEXT: vcvtneps2bf16 %xmm2, %xmm2
+; AVX512BF16-NEXT: vmovd %xmm2, %edi
; AVX512BF16-NEXT: vpextrw $2, %xmm0, %r8d
-; AVX512BF16-NEXT: shll $16, %r8d
-; AVX512BF16-NEXT: vmovd %r8d, %xmm1
-; AVX512BF16-NEXT: vroundss $8, %xmm1, %xmm1, %xmm1
-; AVX512BF16-NEXT: vcvtneps2bf16 %xmm1, %xmm1
-; AVX512BF16-NEXT: vmovd %xmm1, %r8d
-; AVX512BF16-NEXT: vpextrw $1, %xmm0, %r9d
-; AVX512BF16-NEXT: shll $16, %r9d
-; AVX512BF16-NEXT: vmovd %r9d, %xmm1
+; AVX512BF16-NEXT: vmovd %r8d, %xmm2
+; AVX512BF16-NEXT: vpslld $16, %xmm2, %xmm2
+; AVX512BF16-NEXT: vroundss $8, %xmm2, %xmm2, %xmm2
+; AVX512BF16-NEXT: vcvtneps2bf16 %xmm2, %xmm2
+; AVX512BF16-NEXT: vmovd %xmm2, %r8d
+; AVX512BF16-NEXT: vpblendw {{.*#+}} xmm1 = xmm1[0],xmm0[1],xmm1[2],xmm0[3],xmm1[4],xmm0[5],xmm1[6],xmm0[7]
; AVX512BF16-NEXT: vroundss $8, %xmm1, %xmm1, %xmm1
; AVX512BF16-NEXT: vcvtneps2bf16 %xmm1, %xmm1
-; AVX512BF16-NEXT: vmovd %xmm0, %r9d
-; AVX512BF16-NEXT: shll $16, %r9d
-; AVX512BF16-NEXT: vmovd %r9d, %xmm0
+; AVX512BF16-NEXT: vpslld $16, %xmm0, %xmm0
; AVX512BF16-NEXT: vroundss $8, %xmm0, %xmm0, %xmm0
; AVX512BF16-NEXT: vcvtneps2bf16 %xmm0, %xmm0
; AVX512BF16-NEXT: vpunpcklwd {{.*#+}} xmm0 = xmm0[0],xmm1[0],xmm0[1],xmm1[1],xmm0[2],xmm1[2],xmm0[3],xmm1[3]
@@ -501,9 +461,7 @@ define bfloat @roundeven_bf16(bfloat %a) {
;
; AVX512BF16-LABEL: roundeven_bf16:
; AVX512BF16: # %bb.0:
-; AVX512BF16-NEXT: vpextrw $0, %xmm0, %eax
-; AVX512BF16-NEXT: shll $16, %eax
-; AVX512BF16-NEXT: vmovd %eax, %xmm0
+; AVX512BF16-NEXT: vpslld $16, %xmm0, %xmm0
; AVX512BF16-NEXT: vroundss $8, %xmm0, %xmm0, %xmm0
; AVX512BF16-NEXT: vcvtneps2bf16 %xmm0, %xmm0
; AVX512BF16-NEXT: retq
@@ -536,108 +494,97 @@ define <16 x bfloat> @floor_v16bf16(<16 x bfloat> %a) {
; AVX512BF16-NEXT: .cfi_offset %r15, -24
; AVX512BF16-NEXT: .cfi_offset %rbp, -16
; AVX512BF16-NEXT: vpextrw $7, %xmm0, %eax
-; AVX512BF16-NEXT: shll $16, %eax
; AVX512BF16-NEXT: vmovd %eax, %xmm1
+; AVX512BF16-NEXT: vpslld $16, %xmm1, %xmm1
; AVX512BF16-NEXT: vroundss $9, %xmm1, %xmm1, %xmm1
; AVX512BF16-NEXT: vcvtneps2bf16 %xmm1, %xmm1
; AVX512BF16-NEXT: vmovd %xmm1, %eax
; AVX512BF16-NEXT: vpextrw $6, %xmm0, %ecx
-; AVX512BF16-NEXT: shll $16, %ecx
; AVX512BF16-NEXT: vmovd %ecx, %xmm1
+; AVX512BF16-NEXT: vpslld $16, %xmm1, %xmm1
; AVX512BF16-NEXT: vroundss $9, %xmm1, %xmm1, %xmm1
; AVX512BF16-NEXT: vcvtneps2bf16 %xmm1, %xmm1
; AVX512BF16-NEXT: vmovd %xmm1, %ecx
; AVX512BF16-NEXT: vpextrw $5, %xmm0, %edx
-; AVX512BF16-NEXT: shll $16, %edx
; AVX512BF16-NEXT: vmovd %edx, %xmm1
+; AVX512BF16-NEXT: vpslld $16, %xmm1, %xmm1
; AVX512BF16-NEXT: vroundss $9, %xmm1, %xmm1, %xmm1
; AVX512BF16-NEXT: vcvtneps2bf16 %xmm1, %xmm1
; AVX512BF16-NEXT: vmovd %xmm1, %edx
-; AVX512BF16-NEXT: vpextrw $4, %xmm0, %esi
-; AVX512BF16-NEXT: shll $16, %esi
-; AVX512BF16-NEXT: vmovd %esi, %xmm1
-; AVX512BF16-NEXT: vroundss $9, %xmm1, %xmm1, %xmm1
-; AVX512BF16-NEXT: vcvtneps2bf16 %xmm1, %xmm1
-; AVX512BF16-NEXT: vmovd %xmm1, %esi
-; AVX512BF16-NEXT: vpextrw $3, %xmm0, %edi
-; AVX512BF16-NEXT: shll $16, %edi
-; AVX512BF16-NEXT: vmovd %edi, %xmm1
-; AVX512BF16-NEXT: vroundss $9, %xmm1, %xmm1, %xmm1
-; AVX512BF16-NEXT: vcvtneps2bf16 %xmm1, %xmm1
-; AVX512BF16-NEXT: vmovd %xmm1, %edi
-; AVX512BF16-NEXT: vpextrw $2, %xmm0, %r8d
-; AVX512BF16-NEXT: shll $16, %r8d
-; AVX512BF16-NEXT: vmovd %r8d, %xmm1
-; AVX512BF16-NEXT: vroundss $9, %xmm1, %xmm1, %xmm1
-; AVX512BF16-NEXT: vcvtneps2bf16 %xmm1, %xmm1
-; AVX512BF16-NEXT: vmovd %xmm1, %r8d
-; AVX512BF16-NEXT: vmovd %xmm0, %r9d
-; AVX512BF16-NEXT: shll $16, %r9d
-; AVX512BF16-NEXT: vmovd %r9d, %xmm1
-; AVX512BF16-NEXT: vroundss $9, %xmm1, %xmm1, %xmm1
-; AVX512BF16-NEXT: vcvtneps2bf16 %xmm1, %xmm1
-; AVX512BF16-NEXT: vpextrw $1, %xmm0, %r9d
-; AVX512BF16-NEXT: shll $16, %r9d
-; AVX512BF16-NEXT: vmovd %r9d, %xmm2
+; AVX512BF16-NEXT: vpxor %xmm1, %xmm1, %xmm1
+; AVX512BF16-NEXT: vpunpckhwd {{.*#+}} xmm2 = xmm1[4],xmm0[4],xmm1[5],xmm0[5],xmm1[6],xmm0[6],xmm1[7],xmm0[7]
; AVX512BF16-NEXT: vroundss $9, %xmm2, %xmm2, %xmm2
; AVX512BF16-NEXT: vcvtneps2bf16 %xmm2, %xmm2
-; AVX512BF16-NEXT: vmovd %xmm2, %r9d
-; AVX512BF16-NEXT: vextracti128 $1, %ymm0, %xmm0
-; AVX512BF16-NEXT: vpextrw $7, %xmm0, %r10d
-; AVX512BF16-NEXT: shll $16, %r10d
-; AVX512BF16-NEXT: vmovd %r10d, %xmm2
-; AVX512BF16-NEXT: vroundss $9, %xmm2, %xmm2, %xmm2
-; AVX512BF16-NEXT: vcvtneps2bf16 %xmm2, %xmm2
-; AVX512BF16-NEXT: vmovd %xmm2, %r10d
-; AVX512BF16-NEXT: vpextrw $6, %xmm0, %r11d
-; AVX512BF16-NEXT: shll $16, %r11d
-; AVX512BF16-NEXT: vmovd %r11d, %xmm2
+; AVX512BF16-NEXT: vmovd %xmm2, %esi
+; AVX512BF16-NEXT: vpextrw $3, %xmm0, %edi
+; AVX512BF16-NEXT: vmovd %edi, %xmm2
+; AVX512BF16-NEXT: vpslld $16, %xmm2, %xmm2
; AVX512BF16-NEXT: vroundss $9, %xmm2, %xmm2, %xmm2
; AVX512BF16-NEXT: vcvtneps2bf16 %xmm2, %xmm2
-; AVX512BF16-NEXT: vmovd %xmm2, %r11d
-; AVX512BF16-NEXT: vpextrw $5, %xmm0, %ebx
-; AVX512BF16-NEXT: shll $16, %ebx
-; AVX512BF16-NEXT: vmovd %ebx, %xmm2
+; AVX512BF16-NEXT: vmovd %xmm2, %edi
+; AVX512BF16-NEXT: vpextrw $2, %xmm0, %r8d
+; AVX512BF16-NEXT: vmovd %r8d, %xmm2
+; AVX512BF16-NEXT: vpslld $16, %xmm2, %xmm2
; AVX512BF16-NEXT: vroundss $9, %xmm2, %xmm2, %xmm2
; AVX512BF16-NEXT: vcvtneps2bf16 %xmm2, %xmm2
-; AVX512BF16-NEXT: vmovd %xmm2, %ebx
-; AVX512BF16-NEXT: vpextrw $4, %xmm0, %ebp
-; AVX512BF16-NEXT: shll $16, %ebp
-; AVX512BF16-NEXT: vmovd %ebp, %xmm2
+; AVX512BF16-NEXT: vmovd %xmm2, %r8d
+; AVX512BF16-NEXT: vpslld $16, %xmm0, %xmm2
; AVX512BF16-NEXT: vroundss $9, %xmm2, %xmm2, %xmm2
; AVX512BF16-NEXT: vcvtneps2bf16 %xmm2, %xmm2
-; AVX512BF16-NEXT: vmovd %xmm2, %ebp
+; AVX512BF16-NEXT: vpblendw {{.*#+}} xmm3 = xmm1[0],xmm0[1],xmm1[2],xmm0[3],xmm1[4],xmm0[5],xmm1[6],xmm0[7]
+; AVX512BF16-NEXT: vroundss $9, %xmm3, %xmm3, %xmm3
+; AVX512BF16-NEXT: vcvtneps2bf16 %xmm3, %xmm3
+; AVX512BF16-NEXT: vmovd %xmm3, %r9d
+; AVX512BF16-NEXT: vextracti128 $1, %ymm0, %xmm0
+; AVX512BF16-NEXT: vpextrw $7, %xmm0, %r10d
+; AVX512BF16-NEXT: vmovd %r10d, %xmm3
+; AVX512BF16-NEXT: vpslld $16, %xmm3, %xmm3
+; AVX512BF16-NEXT: vroundss $9, %xmm3, %xmm3, %xmm3
+; AVX512BF16-NEXT: vcvtneps2bf16 %xmm3, %xmm3
+; AVX512BF16-NEXT: vmovd %xmm3, %r10d
+; AVX512BF16-NEXT: vpextrw $6, %xmm0, %r11d
+; AVX512BF16-NEXT: vmovd %r11d, %xmm3
+; AVX512BF16-NEXT: vpslld $16, %xmm3, %xmm3
+; AVX512BF16-NEXT: vroundss $9, %xmm3, %xmm3, %xmm3
+; AVX512BF16-NEXT: vcvtneps2bf16 %xmm3, %xmm3
+; AVX512BF16-NEXT: vmovd %xmm3, %r11d
+; AVX512BF16-NEXT: vpextrw $5, %xmm0, %ebx
+; AVX512BF16-NEXT: vmovd %ebx, %xmm3
+; AVX512BF16-NEXT: vpslld $16, %xmm3, %xmm3
+; AVX512BF16-NEXT: vroundss $9, %xmm3, %xmm3, %xmm3
+; AVX512BF16-NEXT: vcvtneps2bf16 %xmm3, %xmm3
+; AVX512BF16-NEXT: vmovd %xmm3, %ebx
+; AVX512BF16-NEXT: vpunpckhwd {{.*#+}} xmm3 = xmm1[4],xmm0[4],xmm1[5],xmm0[5],xmm1[6],xmm0[6],xmm1[7],xmm0[7]
+; AVX512BF16-NEXT: vroundss $9, %xmm3, %xmm3, %xmm3
+; AVX512BF16-NEXT: vcvtneps2bf16 %xmm3, %xmm3
+; AVX512BF16-NEXT: vmovd %xmm3, %ebp
; AVX512BF16-NEXT: vpextrw $3, %xmm0, %r14d
-; AVX512BF16-NEXT: shll $16, %r14d
-; AVX512BF16-NEXT: vmovd %r14d, %xmm2
-; AVX512BF16-NEXT: vroundss $9, %xmm2, %xmm2, %xmm2
-; AVX512BF16-NEXT: vcvtneps2bf16 %xmm2, %xmm2
-; AVX512BF16-NEXT: vmovd %xmm2, %r14d
+; AVX512BF16-NEXT: vmovd %r14d, %xmm3
+; AVX512BF16-NEXT: vpslld $16, %xmm3, %xmm3
+; AVX512BF16-NEXT: vroundss $9, %xmm3, %xmm3, %xmm3
+; AVX512BF16-NEXT: vcvtneps2bf16 %xmm3, %xmm3
+; AVX512BF16-NEXT: vmovd %xmm3, %r14d
; AVX512BF16-NEXT: vpextrw $2, %xmm0, %r15d
-; AVX512BF16-NEXT: shll $16, %r15d
-; AVX512BF16-NEXT: vmovd %r15d, %xmm2
-; AVX512BF16-NEXT: vroundss $9, %xmm2, %xmm2, %xmm2
-; AVX512BF16-NEXT: vcvtneps2bf16 %xmm2, %xmm2
-; AVX512BF16-NEXT: vmovd %xmm2, %r15d
-; AVX512BF16-NEXT: vmovd %xmm0, %r12d
-; AVX512BF16-NEXT: shll $16, %r12d
-; AVX512BF16-NEXT: vmovd %r12d, %xmm2
-; AVX512BF16-NEXT: vroundss $9, %xmm2, %xmm2, %xmm2
-; AVX512BF16-NEXT: vcvtneps2bf16 %xmm2, %xmm2
-; AVX512BF16-NEXT: vpextrw $1, %xmm0, %r12d
-; AVX512BF16-NEXT: shll $16, %r12d
-; AVX512BF16-NEXT: vmovd %r12d, %xmm0
+; AVX512BF16-NEXT: vmovd %r15d, %xmm3
+; AVX512BF16-NEXT: vpslld $16, %xmm3, %xmm3
+; AVX512BF16-NEXT: vroundss $9, %xmm3, %xmm3, %xmm3
+; AVX512BF16-NEXT: vcvtneps2bf16 %xmm3, %xmm3
+; AVX512BF16-NEXT: vmovd %xmm3, %r15d
+; AVX512BF16-NEXT: vpslld $16, %xmm0, %xmm3
+; AVX512BF16-NEXT: vroundss $9, %xmm3, %xmm3, %xmm3
+; AVX512BF16-NEXT: vcvtneps2bf16 %xmm3, %xmm3
+; AVX512BF16-NEXT: vpblendw {{.*#+}} xmm0 = xmm1[0],xmm0[1],xmm1[2],xmm0[3],xmm1[4],xmm0[5],xmm1[6],xmm0[7]
; AVX512BF16-NEXT: vroundss $9, %xmm0, %xmm0, %xmm0
; AVX512BF16-NEXT: vcvtneps2bf16 %xmm0, %xmm0
; AVX512BF16-NEXT: vmovd %xmm0, %r12d
-; AVX512BF16-NEXT: vpinsrw $1, %r12d, %xmm2, %xmm0
+; AVX512BF16-NEXT: vpinsrw $1, %r12d, %xmm3, %xmm0
; AVX512BF16-NEXT: vpinsrw $2, %r15d, %xmm0, %xmm0
; AVX512BF16-NEXT: vpinsrw $3, %r14d, %xmm0, %xmm0
; AVX512BF16-NEXT: vpinsrw $4, %ebp, %xmm0, %xmm0
; AVX512BF16-NEXT: vpinsrw $5, %ebx, %xmm0, %xmm0
; AVX512BF16-NEXT: vpinsrw $6, %r11d, %xmm0, %xmm0
; AVX512BF16-NEXT: vpinsrw $7, %r10d, %xmm0, %xmm0
-; AVX512BF16-NEXT: vpinsrw $1, %r9d, %xmm1, %xmm1
+; AVX512BF16-NEXT: vpinsrw $1, %r9d, %xmm2, %xmm1
; AVX512BF16-NEXT: vpinsrw $2, %r8d, %xmm1, %xmm1
; AVX512BF16-NEXT: vpinsrw $3, %edi, %xmm1, %xmm1
; AVX512BF16-NEXT: vpinsrw $4, %esi, %xmm1, %xmm1
@@ -669,227 +616,204 @@ define <32 x bfloat> @floor_v32bf16(<32 x bfloat> %a) {
; AVX512BF16-LABEL: floor_v32bf16:
; AVX512BF16: # %bb.0:
; AVX512BF16-NEXT: vpextrw $7, %xmm0, %eax
-; AVX512BF16-NEXT: shll $16, %eax
-; AVX512BF16-NEXT: vmovd %eax, %xmm1
+; AVX512BF16-NEXT: vmovd %eax, %xmm23
; AVX512BF16-NEXT: vpextrw $6, %xmm0, %eax
-; AVX512BF16-NEXT: shll $16, %eax
; AVX512BF16-NEXT: vmovd %eax, %xmm2
; AVX512BF16-NEXT: vpextrw $5, %xmm0, %eax
-; AVX512BF16-NEXT: shll $16, %eax
; AVX512BF16-NEXT: vmovd %eax, %xmm3
-; AVX512BF16-NEXT: vpextrw $4, %xmm0, %eax
-; AVX512BF16-NEXT: shll $16, %eax
-; AVX512BF16-NEXT: vmovd %eax, %xmm4
+; AVX512BF16-NEXT: vpxor %xmm5, %xmm5, %xmm5
; AVX512BF16-NEXT: vpextrw $3, %xmm0, %eax
-; AVX512BF16-NEXT: shll $16, %eax
-; AVX512BF16-NEXT: vmovd %eax, %xmm5
+; AVX512BF16-NEXT: vmovd %eax, %xmm4
; AVX512BF16-NEXT: vpextrw $2, %xmm0, %eax
-; AVX512BF16-NEXT: shll $16, %eax
; AVX512BF16-NEXT: vmovd %eax, %xmm6
-; AVX512BF16-NEXT: vmovd %xmm0, %eax
-; AVX512BF16-NEXT: shll $16, %eax
+; AVX512BF16-NEXT: vextracti128 $1, %ymm0, %xmm9
+; AVX512BF16-NEXT: vpextrw $7, %xmm9, %eax
; AVX512BF16-NEXT: vmovd %eax, %xmm7
-; AVX512BF16-NEXT: vpextrw $1, %xmm0, %eax
-; AVX512BF16-NEXT: shll $16, %eax
+; AVX512BF16-NEXT: vpextrw $6, %xmm9, %eax
; AVX512BF16-NEXT: vmovd %eax, %xmm8
-; AVX512BF16-NEXT: vextracti32x4 $1, %ymm0, %xmm16
-; AVX512BF16-NEXT: vpextrw $7, %xmm16, %eax
-; AVX512BF16-NEXT: shll $16, %eax
-; AVX512BF16-NEXT: vmovd %eax, %xmm9
-; AVX512BF16-NEXT: vpextrw $6, %xmm16, %eax
-; AVX512BF16-NEXT: shll $16, %eax
+; AVX512BF16-NEXT: vpextrw $5, %xmm9, %eax
; AVX512BF16-NEXT: vmovd %eax, %xmm10
-; AVX512BF16-NEXT: vpextrw $5, %xmm16, %eax
-; AVX512BF16-NEXT: shll $16, %eax
+; AVX512BF16-NEXT: vpextrw $3, %xmm9, %eax
; AVX512BF16-NEXT: vmovd %eax, %xmm11
-; AVX512BF16-NEXT: vpextrw $4, %xmm16, %eax
-; AVX512BF16-NEXT: shll $16, %eax
+; AVX512BF16-NEXT: vpextrw $2, %xmm9, %eax
; AVX512BF16-NEXT: vmovd %eax, %xmm12
-; AVX512BF16-NEXT: vpextrw $3, %xmm16, %eax
-; AVX512BF16-NEXT: shll $16, %eax
-; AVX512BF16-NEXT: vmovd %eax, %xmm13
-; AVX512BF16-NEXT: vpextrw $2, %xmm16, %eax
-; AVX512BF16-NEXT: shll $16, %eax
+; AVX512BF16-NEXT: vpblendw {{.*#+}} xmm1 = xmm5[0],xmm9[1],xmm5[2],xmm9[3],xmm5[4],xmm9[5],xmm5[6],xmm9[7]
+; AVX512BF16-NEXT: vroundss $9, %xmm1, %xmm1, %xmm13
+; AVX512BF16-NEXT: vextracti32x4 $2, %zmm0, %xmm1
+; AVX512BF16-NEXT: vpextrw $7, %xmm1, %eax
; AVX512BF16-NEXT: vmovd %eax, %xmm14
-; AVX512BF16-NEXT: vmovd %xmm16, %eax
-; AVX512BF16-NEXT: shll $16, %eax
+; AVX512BF16-NEXT: vpextrw $6, %xmm1, %eax
; AVX512BF16-NEXT: vmovd %eax, %xmm15
-; AVX512BF16-NEXT: vpextrw $1, %xmm16, %eax
-; AVX512BF16-NEXT: shll $16, %eax
+; AVX512BF16-NEXT: vpextrw $5, %xmm1, %eax
; AVX512BF16-NEXT: vmovd %eax, %xmm16
-; AVX512BF16-NEXT: vextracti32x4 $2, %zmm0, %xmm24
-; AVX512BF16-NEXT: vpextrw $7, %xmm24, %eax
-; AVX512BF16-NEXT: shll $16, %eax
-; AVX512BF16-NEXT: vmovd %eax, %xmm17
-; AVX512BF16-NEXT: vpextrw $6, %xmm24, %eax
-; AVX512BF16-NEXT: shll $16, %eax
-; AVX512BF16-NEXT: vmovd %eax, %xmm18
-; AVX512BF16-NEXT: vpextrw $5, %xmm24, %eax
-; AVX512BF16-NEXT: shll $16, %eax
+; AVX512BF16-NEXT: vpunpckhwd {{.*#+}} xmm17 = xmm5[4],xmm1[4],xmm5[5],xmm1[5],xmm5[6],xmm1[6],xmm5[7],xmm1[7]
+; AVX512BF16-NEXT: vpextrw $3, %xmm1, %eax
; AVX512BF16-NEXT: vmovd %eax, %xmm19
-; AVX512BF16-NEXT: vpextrw $4, %xmm24, %eax
-; AVX512BF16-NEXT: shll $16, %eax
+; AVX512BF16-NEXT: vpextrw $2, %xmm1, %eax
; AVX512BF16-NEXT: vmovd %eax, %xmm20
-; AVX512BF16-NEXT: vpextrw $3, %xmm24, %eax
-; AVX512BF16-NEXT: shll $16, %eax
-; AVX512BF16-NEXT: vmovd %eax, %xmm21
-; AVX512BF16-NEXT: vpextrw $2, %xmm24, %eax
-; AVX512BF16-NEXT: shll $16, %eax
+; AVX512BF16-NEXT: vpslld $16, %xmm1, %xmm21
+; AVX512BF16-NEXT: vpblendw {{.*#+}} xmm1 = xmm5[0],xmm1[1],xmm5[2],xmm1[3],xmm5[4],xmm1[5],xmm5[6],xmm1[7]
+; AVX512BF16-NEXT: vrndscaless $9, %xmm1, %xmm1, %xmm18
+; AVX512BF16-NEXT: vextracti32x4 $3, %zmm0, %xmm1
+; AVX512BF16-NEXT: vpextrw $7, %xmm1, %eax
; AVX512BF16-NEXT: vmovd %eax, %xmm22
-; AVX512BF16-NEXT: vmovd %xmm24, %eax
-; AVX512BF16-NEXT: shll $16, %eax
-; AVX512BF16-NEXT: vmovd %eax, %xmm23
-; AVX512BF16-NEXT: vpextrw $1, %xmm24, %eax
-; AVX512BF16-NEXT: shll $16, %eax
+; AVX512BF16-NEXT: vpextrw $6, %xmm1, %eax
; AVX512BF16-NEXT: vmovd %eax, %xmm24
-; AVX512BF16-NEXT: vextracti32x4 $3, %zmm0, %xmm25
-; AVX512BF16-NEXT: vpextrw $7, %xmm25, %eax
-; AVX512BF16-NEXT: shll $16, %eax
-; AVX512BF16-NEXT: vmovd %eax, %xmm0
-; AVX512BF16-NEXT: vpextrw $6, %xmm25, %eax
-; AVX512BF16-NEXT: shll $16, %eax
-; AVX512BF16-NEXT: vmovd %eax, %xmm26
-; AVX512BF16-NEXT: vpextrw $5, %xmm25, %eax
-; AVX512BF16-NEXT: shll $16, %eax
+; AVX512BF16-NEXT: vpextrw $5, %xmm1, %eax
+; AVX512BF16-NEXT: vmovd %eax, %xmm25
+; AVX512BF16-NEXT: vpunpckhwd {{.*#+}} xmm26 = xmm5[4],xmm1[4],xmm5[5],xmm1[5],xmm5[6],xmm1[6],xmm5[7],xmm1[7]
+; AVX512BF16-NEXT: vpextrw $3, %xmm1, %eax
; AVX512BF16-NEXT: vmovd %eax, %xmm27
-; AVX512BF16-NEXT: vpextrw $4, %xmm25, %eax
-; AVX512BF16-NEXT: shll $16, %eax
-; AVX512BF16-NEXT: vmovd %eax, %xmm28
-; AVX512BF16-NEXT: vpextrw $3, %xmm25, %eax
-; AVX512BF16-NEXT: shll $16, %eax
+; AVX512BF16-NEXT: vpextrw $2, %xmm1, %eax
+; AVX512BF16-NEXT: vpslld $16, %xmm1, %xmm28
+; AVX512BF16-NEXT: vpblendw {{.*#+}} xmm1 = xmm5[0],xmm1[1],xmm5[2],xmm1[3],xmm5[4],xmm1[5],xmm5[6],xmm1[7]
+; AVX512BF16-NEXT: vroundss $9, %xmm1, %xmm1, %xmm1
; AVX512BF16-NEXT: vmovd %eax, %xmm29
-; AVX512BF16-NEXT: vpextrw $2, %xmm25, %eax
-; AVX512BF16-NEXT: shll $16, %eax
-; AVX512BF16-NEXT: vmovd %eax, %xmm30
-; AVX512BF16-NEXT: vmovd %xmm25, %eax
-; AVX512BF16-NEXT: shll $16, %eax
-; AVX512BF16-NEXT: vmovd %eax, %xmm31
-; AVX512BF16-NEXT: vpextrw $1, %xmm25, %eax
-; AVX512BF16-NEXT: shll $16, %eax
-; AVX512BF16-NEXT: vmovd %eax, %xmm25
-; AVX512BF16-NEXT: vrndscaless $9, %xmm25, %xmm25, %xmm25
-; AVX512BF16-NEXT: vcvtneps2bf16 %xmm25, %xmm25
-; AVX512BF16-NEXT: vmovd %xmm25, %eax
-; AVX512BF16-NEXT: vrndscaless $9, %xmm30, %xmm30, %xmm25
-; AVX512BF16-NEXT: vcvtneps2bf16 %xmm25, %xmm25
-; AVX512BF16-NEXT: vrndscaless $9, %xmm31, %xmm31, %xmm30
-; AVX512BF16-NEXT: vcvtneps2bf16 %xmm30, %xmm30
-; AVX512BF16-NEXT: vpinsrw $1, %eax, %xmm30, %xmm30
-; AVX512BF16-NEXT: vmovd %xmm25, %eax
-; AVX512BF16-NEXT: vrndscaless $9, %xmm29, %xmm29, %xmm25
-; AVX512BF16-NEXT: vcvtneps2bf16 %xmm25, %xmm25
-; AVX512BF16-NEXT: vpinsrw $2, %eax, %xmm30, %xmm29
-; AVX512BF16-NEXT: vmovd %xmm25, %eax
-; AVX512BF16-NEXT: vrndscaless $9, %xmm28, %xmm28, %xmm25
-; AVX512BF16-NEXT: vcvtneps2bf16 %xmm25, %xmm25
-; AVX512BF16-NEXT: vpinsrw $3, %eax, %xmm29, %xmm28
-; AVX512BF16-NEXT: vmovd %xmm25, %eax
-; AVX512BF16-NEXT: vrndscaless $9, %xmm27, %xmm27, %xmm25
-; AVX512BF16-NEXT: vcvtneps2bf16 %xmm25, %xmm25
-; AVX512BF16-NEXT: vpinsrw $4, %eax, %xmm28, %xmm27
-; AVX512BF16-NEXT: vmovd %xmm25, %eax
-; AVX512BF16-NEXT: vrndscaless $9, %xmm26, %xmm26, %xmm25
-; AVX512BF16-NEXT: vcvtneps2bf16 %xmm25, %xmm25
-; AVX512BF16-NEXT: vpinsrw $5, %eax, %xmm27, %xmm26
-; AVX512BF16-NEXT: vmovd %xmm25, %eax
-; AVX512BF16-NEXT: vroundss $9, %xmm0, %xmm0, %xmm0
-; AVX512BF16-NEXT: vcvtneps2bf16 %xmm0, %xmm0
-; AVX512BF16-NEXT: vpinsrw $6, %eax, %xmm26, %xmm25
-; AVX512BF16-NEXT: vmovd %xmm0, %eax
-; AVX512BF16-NEXT: vrndscaless $9, %xmm24, %xmm24, %xmm0
-; AVX512BF16-NEXT: vcvtneps2bf16 %xmm0, %xmm24
-; AVX512BF16-NEXT: vpinsrw $7, %eax, %xmm25, %xmm0
-; AVX512BF16-NEXT: vmovd %xmm24, %eax
-; AVX512BF16-NEXT: vrndscaless $9, %xmm22, %xmm22, %xmm22
-; AVX512BF16-NEXT: vcvtneps2bf16 %xmm22, %xmm22
-; AVX512BF16-NEXT: vrndscaless $9, %xmm23, %xmm23, %xmm23
-; AVX512BF16-NEXT: vcvtneps2bf16 %xmm23, %xmm23
-; AVX512BF16-NEXT: vpinsrw $1, %eax, %xmm23, %xmm23
-; AVX512BF16-NEXT: vmovd %xmm22, %eax
-; AVX512BF16-NEXT: vrndscaless $9, %xmm21, %xmm21, %xmm21
-; AVX512BF16-NEXT: vcvtneps2bf16 %xmm21, %xmm21
-; AVX512BF16-NEXT: vpinsrw $2, %eax, %xmm23, %xmm22
-; AVX512BF16-NEXT: vmovd %xmm21, %eax
-; AVX512BF16-NEXT: vrndscaless $9, %xmm20, %xmm20, %xmm20
+; AVX512BF16-NEXT: vcvtneps2bf16 %xmm1, %xmm1
+; AVX512BF16-NEXT: vmovd %xmm1, %eax
+; AVX512BF16-NEXT: vpslld $16, %xmm29, %xmm1
+; AVX512BF16-NEXT: vroundss $9, %xmm1, %xmm1, %xmm1
+; AVX512BF16-NEXT: vcvtneps2bf16 %xmm1, %xmm1
+; AVX512BF16-NEXT: vrndscaless $9, %xmm28, %xmm28, %xmm28
+; AVX512BF16-NEXT: vcvtneps2bf16 %xmm28, %xmm28
+; AVX512BF16-NEXT: vpinsrw $1, %eax, %xmm28, %xmm28
+; AVX512BF16-NEXT: vmovd %xmm1, %eax
+; AVX512BF16-NEXT: vpslld $16, %xmm27, %xmm1
+; AVX512BF16-NEXT: vroundss $9, %xmm1, %xmm1, %xmm1
+; AVX512BF16-NEXT: vcvtneps2bf16 %xmm1, %xmm1
+; AVX512BF16-NEXT: vpinsrw $2, %eax, %xmm28, %xmm27
+; AVX512BF16-NEXT: vmovd %xmm1, %eax
+; AVX512BF16-NEXT: vrndscaless $9, %xmm26, %xmm26, %xmm1
+; AVX512BF16-NEXT: vcvtneps2bf16 %xmm1, %xmm1
+; AVX512BF16-NEXT: vpinsrw $3, %eax, %xmm27, %xmm26
+; AVX512BF16-NEXT: vmovd %xmm1, %eax
+; AVX512BF16-NEXT: vpslld $16, %xmm25, %xmm1
+; AVX512BF16-NEXT: vroundss $9, %xmm1, %xmm1, %xmm1
+; AVX512BF16-NEXT: vcvtneps2bf16 %xmm1, %xmm1
+; AVX512BF16-NEXT: vpinsrw $4, %eax, %xmm26, %xmm25
+; AVX512BF16-NEXT: vmovd %xmm1, %eax
+; AVX512BF16-NEXT: vpslld $16, %xmm24, %xmm1
+; AVX512BF16-NEXT: vroundss $9, %xmm1, %xmm1, %xmm1
+; AVX512BF16-NEXT: vcvtneps2bf16 %xmm1, %xmm1
+; AVX512BF16-NEXT: vpinsrw $5, %eax, %xmm25, %xmm24
+; AVX512BF16-NEXT: vmovd %xmm1, %eax
+; AVX512BF16-NEXT: vpslld $16, %xmm22, %xmm1
+; AVX512BF16-NEXT: vroundss $9, %xmm1, %xmm1, %xmm1
+; AVX512BF16-NEXT: vcvtneps2bf16 %xmm1, %xmm1
+; AVX512BF16-NEXT: vpinsrw $6, %eax, %xmm24, %xmm22
+; AVX512BF16-NEXT: vmovd %xmm1, %eax
+; AVX512BF16-NEXT: vcvtneps2bf16 %xmm18, %xmm1
+; AVX512BF16-NEXT: vpinsrw $7, %eax, %xmm22, %xmm18
+; AVX512BF16-NEXT: vmovd %xmm1, %eax
+; AVX512BF16-NEXT: vpslld $16, %xmm20, %xmm1
+; AVX512BF16-NEXT: vroundss $9, %xmm1, %xmm1, %xmm1
+; AVX512BF16-NEXT: vcvtneps2bf16 %xmm1, %xmm1
+; AVX512BF16-NEXT: vrndscaless $9, %xmm21, %xmm21, %xmm20
; AVX512BF16-NEXT: vcvtneps2bf16 %xmm20, %xmm20
-; AVX512BF16-NEXT: vpinsrw $3, %eax, %xmm22, %xmm21
-; AVX512BF16-NEXT: vmovd %xmm20, %eax
-; AVX512BF16-NEXT: vrndscaless $9, %xmm19, %xmm19, %xmm19
-; AVX512BF16-NEXT: vcvtneps2bf16 %xmm19, %xmm19
-; AVX512BF16-NEXT: vpinsrw $4, %eax, %xmm21, %xmm20
-; AVX512BF16-NEXT: vmovd %xmm19, %eax
-; AVX512BF16-NEXT: vrndscaless $9, %xmm18, %xmm18, %xmm18
-; AVX512BF16-NEXT: vcvtneps2bf16 %xmm18, %xmm18
-; AVX512BF16-NEXT: vpinsrw $5, %eax, %xmm20, %xmm19
-; AVX512BF16-NEXT: vmovd %xmm18, %eax
-; AVX512BF16-NEXT: vrndscaless $9, %xmm17, %xmm17, %xmm17
-; AVX512BF16-NEXT: vcvtneps2bf16 %xmm17, %xmm17
-; AVX512BF16-NEXT: vpinsrw $6, %eax, %xmm19, %xmm18
-; AVX512BF16-NEXT: vmovd %xmm17, %eax
-; AVX512BF16-NEXT: vrndscaless $9, %xmm16, %xmm16, %xmm16
-; AVX512BF16-NEXT: vcvtneps2bf16 %xmm16, %xmm16
-; AVX512BF16-NEXT: vpinsrw $7, %eax, %xmm18, %xmm17
-; AVX512BF16-NEXT: vmovd %xmm16, %eax
-; AVX512BF16-NEXT: vroundss $9, %xmm14, %xmm14, %xmm14
-; AVX512BF16-NEXT: vcvtneps2bf16 %xmm14, %xmm14
-; AVX512BF16-NEXT: vroundss $9, %xmm15, %xmm15, %xmm15
-; AVX512BF16-NEXT: vcvtneps2bf16 %xmm15, %xmm15
-; AVX512BF16-NEXT: vpinsrw $1, %eax, %xmm15, %xmm15
-; AVX512BF16-NEXT: vmovd %xmm14, %eax
-; AVX512BF16-NEXT: vroundss $9, %xmm13, %xmm13, %xmm13
+; AVX512BF16-NEXT: vpinsrw $1, %eax, %xmm20, %xmm20
+; AVX512BF16-NEXT: vmovd %xmm1, %eax
+; AVX512BF16-NEXT: vpslld $16, %xmm19, %xmm1
+; AVX512BF16-NEXT: vroundss $9, %xmm1, %xmm1, %xmm1
+; AVX512BF16-NEXT: vcvtneps2bf16 %xmm1, %xmm1
+; AVX512BF16-NEXT: vpinsrw $2, %eax, %xmm20, %xmm19
+; AVX512BF16-NEXT: vmovd %xmm1, %eax
+; AVX512BF16-NEXT: vrndscaless $9, %xmm17, %xmm17, %xmm1
+; AVX512BF16-NEXT: vcvtneps2bf16 %xmm1, %xmm1
+; AVX512BF16-NEXT: vpinsrw $3, %eax, %xmm19, %xmm17
+; AVX512BF16-NEXT: vmovd %xmm1, %eax
+; AVX512BF16-NEXT: vpslld $16, %xmm16, %xmm1
+; AVX512BF16-NEXT: vroundss $9, %xmm1, %xmm1, %xmm1
+; AVX512BF16-NEXT: vcvtneps2bf16 %xmm1, %xmm1
+; AVX512BF16-NEXT: vpinsrw $4, %eax, %xmm17, %xmm16
+; AVX512BF16-NEXT: vmovd %xmm1, %eax
+; AVX512BF16-NEXT: vpslld $16, %xmm15, %xmm1
+; AVX512BF16-NEXT: vroundss $9, %xmm1, %xmm1, %xmm1
+; AVX512BF16-NEXT: vcvtneps2bf16 %xmm1, %xmm1
+; AVX512BF16-NEXT: vpinsrw $5, %eax, %xmm16, %xmm15
+; AVX512BF16-NEXT: vmovd %xmm1, %eax
+; AVX512BF16-NEXT: vpslld $16, %xmm14, %xmm1
+; AVX512BF16-NEXT: vroundss $9, %xmm1, %xmm1, %xmm1
+; AVX512BF16-NEXT: vcvtneps2bf16 %xmm1, %xmm1
+; AVX512BF16-NEXT: vpinsrw $6, %eax, %xmm15, %xmm14
+; AVX512BF16-NEXT: vmovd %xmm1, %eax
+; AVX512BF16-NEXT: vpslld $16, %xmm12, %xmm1
+; AVX512BF16-NEXT: vroundss $9, %xmm1, %xmm1, %xmm1
+; AVX512BF16-NEXT: vcvtneps2bf16 %xmm1, %xmm1
; AVX512BF16-NEXT: vcvtneps2bf16 %xmm13, %xmm13
-; AVX512BF16-NEXT: vpinsrw $2, %eax, %xmm15, %xmm14
+; AVX512BF16-NEXT: vpinsrw $7, %eax, %xmm14, %xmm12
; AVX512BF16-NEXT: vmovd %xmm13, %eax
-; AVX512BF16-NEXT: vroundss $9, %xmm12, %xmm12, %xmm12
-; AVX512BF16-NEXT: vcvtneps2bf16 %xmm12, %xmm12
-; AVX512BF16-NEXT: vpinsrw $3, %eax, %xmm14, %xmm13
-; AVX512BF16-NEXT: vmovd %xmm12, %eax
-; AVX512BF16-NEXT: vroundss $9, %xmm11, %xmm11, %xmm11
-; AVX512BF16-NEXT: vcvtneps2bf16 %xmm11, %xmm11
-; AVX512BF16-NEXT: vpinsrw $4, %eax, %xmm13, %xmm12
-; AVX512BF16-NEXT: vmovd %xmm11, %eax
-; AVX512BF16-NEXT: vroundss $9, %xmm10, %xmm10, %xmm10
-; AVX512BF16-NEXT: vcvtneps2bf16 %xmm10, %xmm10
-; AVX512BF16-NEXT: vpinsrw $5, %eax, %xmm12, %xmm11
-; AVX512BF16-NEXT: vmovd %xmm10, %eax
-; AVX512BF16-NEXT: vroundss $9, %xmm9, %xmm9, %xmm9
-; AVX512BF16-NEXT: vcvtneps2bf16 %xmm9, %xmm9
-; AVX512BF16-NEXT: vpinsrw $6, %eax, %xmm11, %xmm10
-; AVX512BF16-NEXT: vmovd %xmm9, %eax
-; AVX512BF16-NEXT: vroundss $9, %xmm8, %xmm8, %xmm8
-; AVX512BF16-NEXT: vcvtneps2bf16 %xmm8, %xmm8
-; AVX512BF16-NEXT: vpinsrw $7, %eax, %xmm10, %xmm9
-; AVX512BF16-NEXT: vmovd %xmm8, %eax
-; AVX512BF16-NEXT: vroundss $9, %xmm6, %xmm6, %xmm6
-; AVX512BF16-NEXT: vcvtneps2bf16 %xmm6, %xmm6
-; AVX512BF16-NEXT: vroundss $9, %xmm7, %xmm7, %xmm7
-; AVX512BF16-NEXT: vcvtneps2bf16 %xmm7, %xmm7
-; AVX512BF16-NEXT: vpinsrw $1, %eax, %xmm7, %xmm7
-; AVX512BF16-NEXT: vmovd %xmm6, %eax
+; AVX512BF16-NEXT: vpslld $16, %xmm9, %xmm13
+; AVX512BF16-NEXT: vroundss $9, %xmm13, %xmm13, %xmm13
+; AVX512BF16-NEXT: vcvtneps2bf16 %xmm13, %xmm13
+; AVX512BF16-NEXT: vpinsrw $1, %eax, %xmm13, %xmm13
+; AVX512BF16-NEXT: vmovd %xmm1, %eax
+; AVX512BF16-NEXT: vpslld $16, %xmm11, %xmm1
+; AVX512BF16-NEXT: vroundss $9, %xmm1, %xmm1, %xmm1
+; AVX512BF16-NEXT: vcvtneps2bf16 %xmm1, %xmm1
+; AVX512BF16-NEXT: vpinsrw $2, %eax, %xmm13, %xmm11
+; AVX512BF16-NEXT: vmovd %xmm1, %eax
+; AVX512BF16-NEXT: vpunpckhwd {{.*#+}} xmm1 = xmm5[4],xmm9[4],xmm5[5],xmm9[5],xmm5[6],xmm9[6],xmm5[7],xmm9[7]
+; AVX512BF16-NEXT: vroundss $9, %xmm1, %xmm1, %xmm1
+; AVX512BF16-NEXT: vcvtneps2bf16 %xmm1, %xmm1
+; AVX512BF16-NEXT: vpinsrw $3, %eax, %xmm11, %xmm9
+; AVX512BF16-NEXT: vmovd %xmm1, %eax
+; AVX512BF16-NEXT: vpslld $16, %xmm10, %xmm1
+; AVX512BF16-NEXT: vroundss $9, %xmm1, %xmm1, %xmm1
+; AVX512BF16-NEXT: vcvtneps2bf16 %xmm1, %xmm1
+; AVX512BF16-NEXT: vpinsrw $4, %eax, %xmm9, %xmm9
+; AVX512BF16-NEXT: vmovd %xmm1, %eax
+; AVX512BF16-NEXT: vpslld $16, %xmm8, %xmm1
+; AVX512BF16-NEXT: vroundss $9, %xmm1, %xmm1, %xmm1
+; AVX512BF16-NEXT: vcvtneps2bf16 %xmm1, %xmm1
+; AVX512BF16-NEXT: vpinsrw $5, %eax, %xmm9, %xmm8
+; AVX512BF16-NEXT: vmovd %xmm1, %eax
+; AVX512BF16-NEXT: vpslld $16, %xmm7, %xmm1
+; AVX512BF16-NEXT: vroundss $9, %xmm1, %xmm1, %xmm1
+; AVX512BF16-NEXT: vcvtneps2bf16 %xmm1, %xmm1
+; AVX512BF16-NEXT: vpinsrw $6, %eax, %xmm8, %xmm7
+; AVX512BF16-NEXT: vmovd %xmm1, %eax
+; AVX512BF16-NEXT: vpblendw {{.*#+}} xmm1 = xmm5[0],xmm0[1],xmm5[2],xmm0[3],xmm5[4],xmm0[5],xmm5[6],xmm0[7]
+; AVX512BF16-NEXT: vroundss $9, %xmm1, %xmm1, %xmm1
+; AVX512BF16-NEXT: vcvtneps2bf16 %xmm1, %xmm1
+; AVX512BF16-NEXT: vpinsrw $7, %eax, %xmm7, %xmm7
+; AVX512BF16-NEXT: vmovd %xmm1, %eax
+; AVX512BF16-NEXT: vpunpckhwd {{.*#+}} xmm1 = xmm5[4],xmm0[4],xmm5[5],xmm0[5],xmm5[6],xmm0[6],xmm5[7],xmm0[7]
+; AVX512BF16-NEXT: vpslld $16, %xmm6, %xmm5
; AVX512BF16-NEXT: vroundss $9, %xmm5, %xmm5, %xmm5
; AVX512BF16-NEXT: vcvtneps2bf16 %xmm5, %xmm5
-; AVX512BF16-NEXT: vpinsrw $2, %eax, %xmm7, %xmm6
+; AVX512BF16-NEXT: vpslld $16, %xmm0, %xmm0
+; AVX512BF16-NEXT: vroundss $9, %xmm0, %xmm0, %xmm0
+; AVX512BF16-NEXT: vcvtneps2bf16 %xmm0, %xmm0
+; AVX512BF16-NEXT: vpinsrw $1, %eax, %xmm0, %xmm0
; AVX512BF16-NEXT: vmovd %xmm5, %eax
+; AVX512BF16-NEXT: vpslld $16, %xmm4, %xmm4
; AVX512BF16-NEXT: vroundss $9, %xmm4, %xmm4, %xmm4
; AVX512BF16-NEXT: vcvtneps2bf16 %xmm4, %xmm4
-; AVX512BF16-NEXT: vpinsrw $3, %eax, %xmm6, %xmm5
+; AVX512BF16-NEXT: vpinsrw $2, %eax, %xmm0, %xmm0
; AVX512BF16-NEXT: vmovd %xmm4, %eax
-; AVX512BF16-NEXT: vroundss $9, %xmm3, %xmm3, %xmm3
-; AVX512BF16-NEXT: vcvtneps2bf16 %xmm3, %xmm3
-; AVX512BF16-NEXT: vpinsrw $4, %eax, %xmm5, %xmm4
-; AVX512BF16-NEXT: vmovd %xmm3, %eax
+; AVX512BF16-NEXT: vroundss $9, %xmm1, %xmm1, %xmm1
+; AVX512BF16-NEXT: vcvtneps2bf16 %xmm1, %xmm1
+; AVX512BF16-NEXT: vpinsrw $3, %eax, %xmm0, %xmm0
+; AVX512BF16-NEXT: vmovd %xmm1, %eax
+; AVX512BF16-NEXT: vpslld $16, %xmm3, %xmm1
+; AVX512BF16-NEXT: vroundss $9, %xmm1, %xmm1, %xmm1
+; AVX512BF16-NEXT: vcvtneps2bf16 %xmm1, %xmm1
+; AVX512BF16-NEXT: vpinsrw $4, %eax, %xmm0, %xmm0
+; AVX512BF16-NEXT: vmovd %xmm1, %eax
+; AVX512BF16-NEXT: vpslld $16, %xmm23, %xmm1
+; AVX512BF16-NEXT: vroundss $9, %xmm1, %xmm1, %xmm1
+; AVX512BF16-NEXT: vcvtneps2bf16 %xmm1, %xmm1
+; AVX512BF16-NEXT: vpslld $16, %xmm2, %xmm2
; AVX512BF16-NEXT: vroundss $9, %xmm2, %xmm2, %xmm2
; AVX512BF16-NEXT: vcvtneps2bf16 %xmm2, %xmm2
-; AVX512BF16-NEXT: vpinsrw $5, %eax, %xmm4, %xmm3
+; AVX512BF16-NEXT: vpinsrw $5, %eax, %xmm0, %xmm0
; AVX512BF16-NEXT: vmovd %xmm2, %eax
-; AVX512BF16-NEXT: vroundss $9, %xmm1, %xmm1, %xmm1
-; AVX512BF16-NEXT: vcvtneps2bf16 %xmm1, %xmm1
-; AVX512BF16-NEXT: vpinsrw $6, %eax, %xmm3, %xmm2
+; AVX512BF16-NEXT: vpinsrw $6, %eax, %xmm0, %xmm0
; AVX512BF16-NEXT: vmovd %xmm1, %eax
-; AVX512BF16-NEXT: vpinsrw $7, %eax, %xmm2, %xmm1
-; AVX512BF16-NEXT: vinserti32x4 $1, %xmm0, %ymm17, %ymm0
-; AVX512BF16-NEXT: vinserti128 $1, %xmm9, %ymm1, %ymm1
-; AVX512BF16-NEXT: vinserti64x4 $1, %ymm0, %zmm1, %zmm0
+; AVX512BF16-NEXT: vpinsrw $7, %eax, %xmm0, %xmm0
+; AVX512BF16-NEXT: vinserti32x4 $1, %xmm18, %ymm12, %ymm1
+; AVX512BF16-NEXT: vinserti128 $1, %xmm7, %ymm0, %ymm0
+; AVX512BF16-NEXT: vinserti64x4 $1, %ymm1, %zmm0, %zmm0
; AVX512BF16-NEXT: retq
%r = call <32 x bfloat> @llvm.floor.v32bf16(<32 x bfloat> %a)
ret <32 x bfloat> %r
@@ -904,14 +828,11 @@ define <2 x bfloat> @floor_v2bf16(<2 x bfloat> %a) {
;
; AVX512BF16-LABEL: floor_v2bf16:
; AVX512BF16: # %bb.0:
-; AVX512BF16-NEXT: vpextrw $1, %xmm0, %eax
-; AVX512BF16-NEXT: shll $16, %eax
-; AVX512BF16-NEXT: vmovd %eax, %xmm1
+; AVX512BF16-NEXT: vpxor %xmm1, %xmm1, %xmm1
+; AVX512BF16-NEXT: vpblendw {{.*#+}} xmm1 = xmm1[0],xmm0[1],xmm1[2],xmm0[3],xmm1[4],xmm0[5],xmm1[6],xmm0[7]
; AVX512BF16-NEXT: vroundss $9, %xmm1, %xmm1, %xmm1
; AVX512BF16-NEXT: vcvtneps2bf16 %xmm1, %xmm1
-; AVX512BF16-NEXT: vmovd %xmm0, %eax
-; AVX512BF16-NEXT: shll $16, %eax
-; AVX512BF16-NEXT: vmovd %eax, %xmm0
+; AVX512BF16-NEXT: vpslld $16, %xmm0, %xmm0
; AVX512BF16-NEXT: vroundss $9, %xmm0, %xmm0, %xmm0
; AVX512BF16-NEXT: vcvtneps2bf16 %xmm0, %xmm0
; AVX512BF16-NEXT: vpunpcklwd {{.*#+}} xmm0 = xmm0[0],xmm1[0],xmm0[1],xmm1[1],xmm0[2],xmm1[2],xmm0[3],xmm1[3]
@@ -931,49 +852,44 @@ define <8 x bfloat> @floor_v8bf16_load(ptr %p) {
; AVX512BF16: # %bb.0:
; AVX512BF16-NEXT: vmovdqa (%rdi), %xmm0
; AVX512BF16-NEXT: vpextrw $7, %xmm0, %eax
-; AVX512BF16-NEXT: shll $16, %eax
; AVX512BF16-NEXT: vmovd %eax, %xmm1
+; AVX512BF16-NEXT: vpslld $16, %xmm1, %xmm1
; AVX512BF16-NEXT: vroundss $9, %xmm1, %xmm1, %xmm1
; AVX512BF16-NEXT: vcvtneps2bf16 %xmm1, %xmm1
; AVX512BF16-NEXT: vmovd %xmm1, %eax
; AVX512BF16-NEXT: vpextrw $6, %xmm0, %ecx
-; AVX512BF16-NEXT: shll $16, %ecx
; AVX512BF16-NEXT: vmovd %ecx, %xmm1
+; AVX512BF16-NEXT: vpslld $16, %xmm1, %xmm1
; AVX512BF16-NEXT: vroundss $9, %xmm1, %xmm1, %xmm1
; AVX512BF16-NEXT: vcvtneps2bf16 %xmm1, %xmm1
; AVX512BF16-NEXT: vmovd %xmm1, %ecx
; AVX512BF16-NEXT: vpextrw $5, %xmm0, %edx
-; AVX512BF16-NEXT: shll $16, %edx
; AVX512BF16-NEXT: vmovd %edx, %xmm1
+; AVX512BF16-NEXT: vpslld $16, %xmm1, %xmm1
; AVX512BF16-NEXT: vroundss $9, %xmm1, %xmm1, %xmm1
; AVX512BF16-NEXT: vcvtneps2bf16 %xmm1, %xmm1
; AVX512BF16-NEXT: vmovd %xmm1, %edx
-; AVX512BF16-NEXT: vpextrw $4, %xmm0, %esi
-; AVX512BF16-NEXT: shll $16, %esi
-; AVX512BF16-NEXT: vmovd %esi, %xmm1
-; AVX512BF16-NEXT: vroundss $9, %xmm1, %xmm1, %xmm1
-; AVX512BF16-NEXT: vcvtneps2bf16 %xmm1, %xmm1
-; AVX512BF16-NEXT: vmovd %xmm1, %esi
+; AVX512BF16-NEXT: vpxor %xmm1, %xmm1, %xmm1
+; AVX512BF16-NEXT: vpunpckhwd {{.*#+}} xmm2 = xmm1[4],xmm0[4],xmm1[5],xmm0[5],xmm1[6],xmm0[6],xmm1[7],xmm0[7]
+; AVX512BF16-NEXT: vroundss $9, %xmm2, %xmm2, %xmm2
+; AVX512BF16-NEXT: vcvtneps2bf16 %xmm2, %xmm2
+; AVX512BF16-NEXT: vmovd %xmm2, %esi
; AVX512BF16-NEXT: vpextrw $3, %xmm0, %edi
-; AVX512BF16-NEXT: shll $16, %edi
-; AVX512BF16-NEXT: vmovd %edi, %xmm1
-; AVX512BF16-NEXT: vroundss $9, %xmm1, %xmm1, %xmm1
-; AVX512BF16-NEXT: vcvtneps2bf16 %xmm1, %xmm1
-; AVX512BF16-NEXT: vmovd %xmm1, %edi
+; AVX512BF16-NEXT: vmovd %edi, %xmm2
+; AVX512BF16-NEXT: vpslld $16, %xmm2, %xmm2
+; AVX512BF16-NEXT: vroundss $9, %xmm2, %xmm2, %xmm2
+; AVX512BF16-NEXT: vcvtneps2bf16 %xmm2, %xmm2
+; AVX512BF16-NEXT: vmovd %xmm2, %edi
; AVX512BF16-NEXT: vpextrw $2, %xmm0, %r8d
-; AVX512BF16-NEXT: shll $16, %r8d
-; AVX512BF16-NEXT: vmovd %r8d, %xmm1
-; AVX512BF16-NEXT: vroundss $9, %xmm1, %xmm1, %xmm1
-; AVX512BF16-NEXT: vcvtneps2bf16 %xmm1, %xmm1
-; AVX512BF16-NEXT: vmovd %xmm1, %r8d
-; AVX512BF16-NEXT: vpextrw $1, %xmm0, %r9d
-; AVX512BF16-NEXT: shll $16, %r9d
-; AVX512BF16-NEXT: vmovd %r9d, %xmm1
+; AVX512BF16-NEXT: vmovd %r8d, %xmm2
+; AVX512BF16-NEXT: vpslld $16, %xmm2, %xmm2
+; AVX512BF16-NEXT: vroundss $9, %xmm2, %xmm2, %xmm2
+; AVX512BF16-NEXT: vcvtneps2bf16 %xmm2, %xmm2
+; AVX512BF16-NEXT: vmovd %xmm2, %r8d
+; AVX512BF16-NEXT: vpblendw {{.*#+}} xmm1 = xmm1[0],xmm0[1],xmm1[2],xmm0[3],xmm1[4],xmm0[5],xmm1[6],xmm0[7]
; AVX512BF16-NEXT: vroundss $9, %xmm1, %xmm1, %xmm1
; AVX512BF16-NEXT: vcvtneps2bf16 %xmm1, %xmm1
-; AVX512BF16-NEXT: vmovd %xmm0, %r9d
-; AVX512BF16-NEXT: shll $16, %r9d
-; AVX512BF16-NEXT: vmovd %r9d, %xmm0
+; AVX512BF16-NEXT: vpslld $16, %xmm0, %xmm0
; AVX512BF16-NEXT: vroundss $9, %xmm0, %xmm0, %xmm0
; AVX512BF16-NEXT: vcvtneps2bf16 %xmm0, %xmm0
; AVX512BF16-NEXT: vpunpcklwd {{.*#+}} xmm0 = xmm0[0],xmm1[0],xmm0[1],xmm1[1],xmm0[2],xmm1[2],xmm0[3],xmm1[3]
@@ -1002,49 +918,44 @@ define <8 x bfloat> @floor_v8bf16_mask(<8 x bfloat> %a, <8 x bfloat> %src, i8 %m
; AVX512BF16: # %bb.0:
; AVX512BF16-NEXT: kmovd %edi, %k1
; AVX512BF16-NEXT: vpextrw $7, %xmm0, %eax
-; AVX512BF16-NEXT: shll $16, %eax
; AVX512BF16-NEXT: vmovd %eax, %xmm2
+; AVX512BF16-NEXT: vpslld $16, %xmm2, %xmm2
; AVX512BF16-NEXT: vroundss $9, %xmm2, %xmm2, %xmm2
; AVX512BF16-NEXT: vcvtneps2bf16 %xmm2, %xmm2
; AVX512BF16-NEXT: vmovd %xmm2, %eax
; AVX512BF16-NEXT: vpextrw $6, %xmm0, %ecx
-; AVX512BF16-NEXT: shll $16, %ecx
; AVX512BF16-NEXT: vmovd %ecx, %xmm2
+; AVX512BF16-NEXT: vpslld $16, %xmm2, %xmm2
; AVX512BF16-NEXT: vroundss $9, %xmm2, %xmm2, %xmm2
; AVX512BF16-NEXT: vcvtneps2bf16 %xmm2, %xmm2
; AVX512BF16-NEXT: vmovd %xmm2, %ecx
; AVX512BF16-NEXT: vpextrw $5, %xmm0, %edx
-; AVX512BF16-NEXT: shll $16, %edx
; AVX512BF16-NEXT: vmovd %edx, %xmm2
+; AVX512BF16-NEXT: vpslld $16, %xmm2, %xmm2
; AVX512BF16-NEXT: vroundss $9, %xmm2, %xmm2, %xmm2
; AVX512BF16-NEXT: vcvtneps2bf16 %xmm2, %xmm2
; AVX512BF16-NEXT: vmovd %xmm2, %edx
-; AVX512BF16-NEXT: vpextrw $4, %xmm0, %esi
-; AVX512BF16-NEXT: shll $16, %esi
-; AVX512BF16-NEXT: vmovd %esi, %xmm2
-; AVX512BF16-NEXT: vroundss $9, %xmm2, %xmm2, %xmm2
-; AVX512BF16-NEXT: vcvtneps2bf16 %xmm2, %xmm2
-; AVX512BF16-NEXT: vmovd %xmm2, %esi
+; AVX512BF16-NEXT: vpxor %xmm2, %xmm2, %xmm2
+; AVX512BF16-NEXT: vpunpckhwd {{.*#+}} xmm3 = xmm2[4],xmm0[4],xmm2[5],xmm0[5],xmm2[6],xmm0[6],xmm2[7],xmm0[7]
+; AVX512BF16-NEXT: vroundss $9, %xmm3, %xmm3, %xmm3
+; AVX512BF16-NEXT: vcvtneps2bf16 %xmm3, %xmm3
+; AVX512BF16-NEXT: vmovd %xmm3, %esi
; AVX512BF16-NEXT: vpextrw $3, %xmm0, %edi
-; AVX512BF16-NEXT: shll $16, %edi
-; AVX512BF16-NEXT: vmovd %edi, %xmm2
-; AVX512BF16-NEXT: vroundss $9, %xmm2, %xmm2, %xmm2
-; AVX512BF16-NEXT: vcvtneps2bf16 %xmm2, %xmm2
-; AVX512BF16-NEXT: vmovd %xmm2, %edi
+; AVX512BF16-NEXT: vmovd %edi, %xmm3
+; AVX512BF16-NEXT: vpslld $16, %xmm3, %xmm3
+; AVX512BF16-NEXT: vroundss $9, %xmm3, %xmm3, %xmm3
+; AVX512BF16-NEXT: vcvtneps2bf16 %xmm3, %xmm3
+; AVX512BF16-NEXT: vmovd %xmm3, %edi
; AVX512BF16-NEXT: vpextrw $2, %xmm0, %r8d
-; AVX512BF16-NEXT: shll $16, %r8d
-; AVX512BF16-NEXT: vmovd %r8d, %xmm2
-; AVX512BF16-NEXT: vroundss $9, %xmm2, %xmm2, %xmm2
-; AVX512BF16-NEXT: vcvtneps2bf16 %xmm2, %xmm2
-; AVX512BF16-NEXT: vmovd %xmm2, %r8d
-; AVX512BF16-NEXT: vpextrw $1, %xmm0, %r9d
-; AVX512BF16-NEXT: shll $16, %r9d
-; AVX512BF16-NEXT: vmovd %r9d, %xmm2
+; AVX512BF16-NEXT: vmovd %r8d, %xmm3
+; AVX512BF16-NEXT: vpslld $16, %xmm3, %xmm3
+; AVX512BF16-NEXT: vroundss $9, %xmm3, %xmm3, %xmm3
+; AVX512BF16-NEXT: vcvtneps2bf16 %xmm3, %xmm3
+; AVX512BF16-NEXT: vmovd %xmm3, %r8d
+; AVX512BF16-NEXT: vpblendw {{.*#+}} xmm2 = xmm2[0],xmm0[1],xmm2[2],xmm0[3],xmm2[4],xmm0[5],xmm2[6],xmm0[7]
; AVX512BF16-NEXT: vroundss $9, %xmm2, %xmm2, %xmm2
; AVX512BF16-NEXT: vcvtneps2bf16 %xmm2, %xmm2
-; AVX512BF16-NEXT: vmovd %xmm0, %r9d
-; AVX512BF16-NEXT: shll $16, %r9d
-; AVX512BF16-NEXT: vmovd %r9d, %xmm0
+; AVX512BF16-NEXT: vpslld $16, %xmm0, %xmm0
; AVX512BF16-NEXT: vroundss $9, %xmm0, %xmm0, %xmm0
; AVX512BF16-NEXT: vcvtneps2bf16 %xmm0, %xmm0
; AVX512BF16-NEXT: vpunpcklwd {{.*#+}} xmm0 = xmm0[0],xmm2[0],xmm0[1],xmm2[1],xmm0[2],xmm2[2],xmm0[3],xmm2[3]
@@ -1073,49 +984,44 @@ define <8 x bfloat> @floor_v8bf16_maskz(<8 x bfloat> %a, i8 %msk) {
; AVX512BF16: # %bb.0:
; AVX512BF16-NEXT: kmovd %edi, %k1
; AVX512BF16-NEXT: vpextrw $7, %xmm0, %eax
-; AVX512BF16-NEXT: shll $16, %eax
; AVX512BF16-NEXT: vmovd %eax, %xmm1
+; AVX512BF16-NEXT: vpslld $16, %xmm1, %xmm1
; AVX512BF16-NEXT: vroundss $9, %xmm1, %xmm1, %xmm1
; AVX512BF16-NEXT: vcvtneps2bf16 %xmm1, %xmm1
; AVX512BF16-NEXT: vmovd %xmm1, %eax
; AVX512BF16-NEXT: vpextrw $6, %xmm0, %ecx
-; AVX512BF16-NEXT: shll $16, %ecx
; AVX512BF16-NEXT: vmovd %ecx, %xmm1
+; AVX512BF16-NEXT: vpslld $16, %xmm1, %xmm1
; AVX512BF16-NEXT: vroundss $9, %xmm1, %xmm1, %xmm1
; AVX512BF16-NEXT: vcvtneps2bf16 %xmm1, %xmm1
; AVX512BF16-NEXT: vmovd %xmm1, %ecx
; AVX512BF16-NEXT: vpextrw $5, %xmm0, %edx
-; AVX512BF16-NEXT: shll $16, %edx
; AVX512BF16-NEXT: vmovd %edx, %xmm1
+; AVX512BF16-NEXT: vpslld $16, %xmm1, %xmm1
; AVX512BF16-NEXT: vroundss $9, %xmm1, %xmm1, %xmm1
; AVX512BF16-NEXT: vcvtneps2bf16 %xmm1, %xmm1
; AVX512BF16-NEXT: vmovd %xmm1, %edx
-; AVX512BF16-NEXT: vpextrw $4, %xmm0, %esi
-; AVX512BF16-NEXT: shll $16, %esi
-; AVX512BF16-NEXT: vmovd %esi, %xmm1
-; AVX512BF16-NEXT: vroundss $9, %xmm1, %xmm1, %xmm1
-; AVX512BF16-NEXT: vcvtneps2bf16 %xmm1, %xmm1
-; AVX512BF16-NEXT: vmovd %xmm1, %esi
+; AVX512BF16-NEXT: vpxor %xmm1, %xmm1, %xmm1
+; AVX512BF16-NEXT: vpunpckhwd {{.*#+}} xmm2 = xmm1[4],xmm0[4],xmm1[5],xmm0[5],xmm1[6],xmm0[6],xmm1[7],xmm0[7]
+; AVX512BF16-NEXT: vroundss $9, %xmm2, %xmm2, %xmm2
+; AVX512BF16-NEXT: vcvtneps2bf16 %xmm2, %xmm2
+; AVX512BF16-NEXT: vmovd %xmm2, %esi
; AVX512BF16-NEXT: vpextrw $3, %xmm0, %edi
-; AVX512BF16-NEXT: shll $16, %edi
-; AVX512BF16-NEXT: vmovd %edi, %xmm1
-; AVX512BF16-NEXT: vroundss $9, %xmm1, %xmm1, %xmm1
-; AVX512BF16-NEXT: vcvtneps2bf16 %xmm1, %xmm1
-; AVX512BF16-NEXT: vmovd %xmm1, %edi
+; AVX512BF16-NEXT: vmovd %edi, %xmm2
+; AVX512BF16-NEXT: vpslld $16, %xmm2, %xmm2
+; AVX512BF16-NEXT: vroundss $9, %xmm2, %xmm2, %xmm2
+; AVX512BF16-NEXT: vcvtneps2bf16 %xmm2, %xmm2
+; AVX512BF16-NEXT: vmovd %xmm2, %edi
; AVX512BF16-NEXT: vpextrw $2, %xmm0, %r8d
-; AVX512BF16-NEXT: shll $16, %r8d
-; AVX512BF16-NEXT: vmovd %r8d, %xmm1
-; AVX512BF16-NEXT: vroundss $9, %xmm1, %xmm1, %xmm1
-; AVX512BF16-NEXT: vcvtneps2bf16 %xmm1, %xmm1
-; AVX512BF16-NEXT: vmovd %xmm1, %r8d
-; AVX512BF16-NEXT: vpextrw $1, %xmm0, %r9d
-; AVX512BF16-NEXT: shll $16, %r9d
-; AVX512BF16-NEXT: vmovd %r9d, %xmm1
+; AVX512BF16-NEXT: vmovd %r8d, %xmm2
+; AVX512BF16-NEXT: vpslld $16, %xmm2, %xmm2
+; AVX512BF16-NEXT: vroundss $9, %xmm2, %xmm2, %xmm2
+; AVX512BF16-NEXT: vcvtneps2bf16 %xmm2, %xmm2
+; AVX512BF16-NEXT: vmovd %xmm2, %r8d
+; AVX512BF16-NEXT: vpblendw {{.*#+}} xmm1 = xmm1[0],xmm0[1],xmm1[2],xmm0[3],xmm1[4],xmm0[5],xmm1[6],xmm0[7]
; AVX512BF16-NEXT: vroundss $9, %xmm1, %xmm1, %xmm1
; AVX512BF16-NEXT: vcvtneps2bf16 %xmm1, %xmm1
-; AVX512BF16-NEXT: vmovd %xmm0, %r9d
-; AVX512BF16-NEXT: shll $16, %r9d
-; AVX512BF16-NEXT: vmovd %r9d, %xmm0
+; AVX512BF16-NEXT: vpslld $16, %xmm0, %xmm0
; AVX512BF16-NEXT: vroundss $9, %xmm0, %xmm0, %xmm0
; AVX512BF16-NEXT: vcvtneps2bf16 %xmm0, %xmm0
; AVX512BF16-NEXT: vpunpcklwd {{.*#+}} xmm0 = xmm0[0],xmm1[0],xmm0[1],xmm1[1],xmm0[2],xmm1[2],xmm0[3],xmm1[3]
@@ -1137,9 +1043,152 @@ define <8 x bfloat> @floor_v8bf16_maskz(<8 x bfloat> %a, i8 %msk) {
; must stay scalarized rather than becoming a vrndscalebf16.
define <8 x bfloat> @round_v8bf16(<8 x bfloat> %a) {
; AVX10_2-LABEL: round_v8bf16:
-; AVX10_2-NOT: vrndscalebf16
-; AVX10_2: vroundss $11,
-; AVX10_2-NOT: vrndscalebf16
+; AVX10_2: # %bb.0:
+; AVX10_2-NEXT: vpextrw $3, %xmm0, %eax
+; AVX10_2-NEXT: vmovw %eax, %xmm1
+; AVX10_2-NEXT: vpslld $16, %xmm1, %xmm1
+; AVX10_2-NEXT: vpbroadcastd {{.*#+}} xmm2 = [-0.0E+0,-0.0E+0,-0.0E+0,-0.0E+0]
+; AVX10_2-NEXT: vpbroadcastd {{.*#+}} xmm3 = [4.9999997E-1,4.9999997E-1,4.9999997E-1,4.9999997E-1]
+; AVX10_2-NEXT: vmovdqa %xmm3, %xmm4
+; AVX10_2-NEXT: vpternlogd {{.*#+}} xmm4 = xmm4 | (xmm1 & xmm2)
+; AVX10_2-NEXT: vaddss %xmm4, %xmm1, %xmm1
+; AVX10_2-NEXT: vroundss $11, %xmm1, %xmm1, %xmm1
+; AVX10_2-NEXT: vcvtneps2bf16 %xmm1, %xmm1
+; AVX10_2-NEXT: vpextrw $2, %xmm0, %eax
+; AVX10_2-NEXT: vmovw %eax, %xmm4
+; AVX10_2-NEXT: vpslld $16, %xmm4, %xmm4
+; AVX10_2-NEXT: vmovdqa %xmm3, %xmm5
+; AVX10_2-NEXT: vpternlogd {{.*#+}} xmm5 = xmm5 | (xmm4 & xmm2)
+; AVX10_2-NEXT: vaddss %xmm5, %xmm4, %xmm4
+; AVX10_2-NEXT: vroundss $11, %xmm4, %xmm4, %xmm4
+; AVX10_2-NEXT: vcvtneps2bf16 %xmm4, %xmm4
+; AVX10_2-NEXT: vpunpcklwd {{.*#+}} xmm1 = xmm4[0],xmm1[0],xmm4[1],xmm1[1],xmm4[2],xmm1[2],xmm4[3],xmm1[3]
+; AVX10_2-NEXT: vpxor %xmm4, %xmm4, %xmm4
+; AVX10_2-NEXT: vpblendw {{.*#+}} xmm5 = xmm4[0],xmm0[1],xmm4[2],xmm0[3],xmm4[4],xmm0[5],xmm4[6],xmm0[7]
+; AVX10_2-NEXT: vmovdqa %xmm3, %xmm6
+; AVX10_2-NEXT: vpternlogd {{.*#+}} xmm6 = xmm6 | (xmm5 & xmm2)
+; AVX10_2-NEXT: vaddss %xmm6, %xmm5, %xmm5
+; AVX10_2-NEXT: vroundss $11, %xmm5, %xmm5, %xmm5
+; AVX10_2-NEXT: vcvtneps2bf16 %xmm5, %xmm5
+; AVX10_2-NEXT: vpslld $16, %xmm0, %xmm6
+; AVX10_2-NEXT: vmovdqa %xmm3, %xmm7
+; AVX10_2-NEXT: vpternlogd {{.*#+}} xmm7 = xmm7 | (xmm6 & xmm2)
+; AVX10_2-NEXT: vaddss %xmm7, %xmm6, %xmm6
+; AVX10_2-NEXT: vroundss $11, %xmm6, %xmm6, %xmm6
+; AVX10_2-NEXT: vcvtneps2bf16 %xmm6, %xmm6
+; AVX10_2-NEXT: vpunpcklwd {{.*#+}} xmm5 = xmm6[0],xmm5[0],xmm6[1],xmm5[1],xmm6[2],xmm5[2],xmm6[3],xmm5[3]
+; AVX10_2-NEXT: vpunpckldq {{.*#+}} xmm1 = xmm5[0],xmm1[0],xmm5[1],xmm1[1]
+; AVX10_2-NEXT: vpunpckhwd {{.*#+}} xmm4 = xmm4[4],xmm0[4],xmm4[5],xmm0[5],xmm4[6],xmm0[6],xmm4[7],xmm0[7]
+; AVX10_2-NEXT: vmovdqa %xmm3, %xmm5
+; AVX10_2-NEXT: vpternlogd {{.*#+}} xmm5 = xmm5 | (xmm4 & xmm2)
+; AVX10_2-NEXT: vaddss %xmm5, %xmm4, %xmm4
+; AVX10_2-NEXT: vroundss $11, %xmm4, %xmm4, %xmm4
+; AVX10_2-NEXT: vcvtneps2bf16 %xmm4, %xmm4
+; AVX10_2-NEXT: vpextrw $5, %xmm0, %eax
+; AVX10_2-NEXT: vmovw %eax, %xmm5
+; AVX10_2-NEXT: vpslld $16, %xmm5, %xmm5
+; AVX10_2-NEXT: vmovdqa %xmm3, %xmm6
+; AVX10_2-NEXT: vpternlogd {{.*#+}} xmm6 = xmm6 | (xmm5 & xmm2)
+; AVX10_2-NEXT: vaddss %xmm6, %xmm5, %xmm5
+; AVX10_2-NEXT: vroundss $11, %xmm5, %xmm5, %xmm5
+; AVX10_2-NEXT: vcvtneps2bf16 %xmm5, %xmm5
+; AVX10_2-NEXT: vpunpcklwd {{.*#+}} xmm4 = xmm4[0],xmm5[0],xmm4[1],xmm5[1],xmm4[2],xmm5[2],xmm4[3],xmm5[3]
+; AVX10_2-NEXT: vpextrw $7, %xmm0, %eax
+; AVX10_2-NEXT: vmovw %eax, %xmm5
+; AVX10_2-NEXT: vpslld $16, %xmm5, %xmm5
+; AVX10_2-NEXT: vmovdqa %xmm3, %xmm6
+; AVX10_2-NEXT: vpternlogd {{.*#+}} xmm6 = xmm6 | (xmm5 & xmm2)
+; AVX10_2-NEXT: vaddss %xmm6, %xmm5, %xmm5
+; AVX10_2-NEXT: vroundss $11, %xmm5, %xmm5, %xmm5
+; AVX10_2-NEXT: vcvtneps2bf16 %xmm5, %xmm5
+; AVX10_2-NEXT: vpextrw $6, %xmm0, %eax
+; AVX10_2-NEXT: vmovw %eax, %xmm0
+; AVX10_2-NEXT: vpslld $16, %xmm0, %xmm0
+; AVX10_2-NEXT: vpternlogd {{.*#+}} xmm3 = xmm3 | (xmm0 & xmm2)
+; AVX10_2-NEXT: vaddss %xmm3, %xmm0, %xmm0
+; AVX10_2-NEXT: vroundss $11, %xmm0, %xmm0, %xmm0
+; AVX10_2-NEXT: vcvtneps2bf16 %xmm0, %xmm0
+; AVX10_2-NEXT: vpunpcklwd {{.*#+}} xmm0 = xmm0[0],xmm5[0],xmm0[1],xmm5[1],xmm0[2],xmm5[2],xmm0[3],xmm5[3]
+; AVX10_2-NEXT: vpunpckldq {{.*#+}} xmm0 = xmm4[0],xmm0[0],xmm4[1],xmm0[1]
+; AVX10_2-NEXT: vpunpcklqdq {{.*#+}} xmm0 = xmm1[0],xmm0[0]
+; AVX10_2-NEXT: retq
+;
+; AVX512BF16-LABEL: round_v8bf16:
+; AVX512BF16: # %bb.0:
+; AVX512BF16-NEXT: vpextrw $7, %xmm0, %eax
+; AVX512BF16-NEXT: vmovd %eax, %xmm1
+; AVX512BF16-NEXT: vpslld $16, %xmm1, %xmm1
+; AVX512BF16-NEXT: vpbroadcastd {{.*#+}} xmm2 = [-0.0E+0,-0.0E+0,-0.0E+0,-0.0E+0]
+; AVX512BF16-NEXT: vpbroadcastd {{.*#+}} xmm3 = [4.9999997E-1,4.9999997E-1,4.9999997E-1,4.9999997E-1]
+; AVX512BF16-NEXT: vmovdqa %xmm3, %xmm4
+; AVX512BF16-NEXT: vpternlogd {{.*#+}} xmm4 = xmm4 | (xmm1 & xmm2)
+; AVX512BF16-NEXT: vaddss %xmm4, %xmm1, %xmm1
+; AVX512BF16-NEXT: vroundss $11, %xmm1, %xmm1, %xmm1
+; AVX512BF16-NEXT: vcvtneps2bf16 %xmm1, %xmm1
+; AVX512BF16-NEXT: vmovd %xmm1, %eax
+; AVX512BF16-NEXT: vpextrw $6, %xmm0, %ecx
+; AVX512BF16-NEXT: vmovd %ecx, %xmm1
+; AVX512BF16-NEXT: vpslld $16, %xmm1, %xmm1
+; AVX512BF16-NEXT: vmovdqa %xmm3, %xmm4
+; AVX512BF16-NEXT: vpternlogd {{.*#+}} xmm4 = xmm4 | (xmm1 & xmm2)
+; AVX512BF16-NEXT: vaddss %xmm4, %xmm1, %xmm1
+; AVX512BF16-NEXT: vroundss $11, %xmm1, %xmm1, %xmm1
+; AVX512BF16-NEXT: vcvtneps2bf16 %xmm1, %xmm1
+; AVX512BF16-NEXT: vmovd %xmm1, %ecx
+; AVX512BF16-NEXT: vpextrw $5, %xmm0, %edx
+; AVX512BF16-NEXT: vmovd %edx, %xmm1
+; AVX512BF16-NEXT: vpslld $16, %xmm1, %xmm1
+; AVX512BF16-NEXT: vmovdqa %xmm3, %xmm4
+; AVX512BF16-NEXT: vpternlogd {{.*#+}} xmm4 = xmm4 | (xmm1 & xmm2)
+; AVX512BF16-NEXT: vaddss %xmm4, %xmm1, %xmm1
+; AVX512BF16-NEXT: vroundss $11, %xmm1, %xmm1, %xmm1
+; AVX512BF16-NEXT: vcvtneps2bf16 %xmm1, %xmm1
+; AVX512BF16-NEXT: vmovd %xmm1, %edx
+; AVX512BF16-NEXT: vpxor %xmm1, %xmm1, %xmm1
+; AVX512BF16-NEXT: vpunpckhwd {{.*#+}} xmm4 = xmm1[4],xmm0[4],xmm1[5],xmm0[5],xmm1[6],xmm0[6],xmm1[7],xmm0[7]
+; AVX512BF16-NEXT: vmovdqa %xmm3, %xmm5
+; AVX512BF16-NEXT: vpternlogd {{.*#+}} xmm5 = xmm5 | (xmm4 & xmm2)
+; AVX512BF16-NEXT: vaddss %xmm5, %xmm4, %xmm4
+; AVX512BF16-NEXT: vroundss $11, %xmm4, %xmm4, %xmm4
+; AVX512BF16-NEXT: vcvtneps2bf16 %xmm4, %xmm4
+; AVX512BF16-NEXT: vmovd %xmm4, %esi
+; AVX512BF16-NEXT: vpextrw $3, %xmm0, %edi
+; AVX512BF16-NEXT: vmovd %edi, %xmm4
+; AVX512BF16-NEXT: vpslld $16, %xmm4, %xmm4
+; AVX512BF16-NEXT: vmovdqa %xmm3, %xmm5
+; AVX512BF16-NEXT: vpternlogd {{.*#+}} xmm5 = xmm5 | (xmm4 & xmm2)
+; AVX512BF16-NEXT: vaddss %xmm5, %xmm4, %xmm4
+; AVX512BF16-NEXT: vroundss $11, %xmm4, %xmm4, %xmm4
+; AVX512BF16-NEXT: vcvtneps2bf16 %xmm4, %xmm4
+; AVX512BF16-NEXT: vmovd %xmm4, %edi
+; AVX512BF16-NEXT: vpextrw $2, %xmm0, %r8d
+; AVX512BF16-NEXT: vmovd %r8d, %xmm4
+; AVX512BF16-NEXT: vpslld $16, %xmm4, %xmm4
+; AVX512BF16-NEXT: vmovdqa %xmm3, %xmm5
+; AVX512BF16-NEXT: vpternlogd {{.*#+}} xmm5 = xmm5 | (xmm4 & xmm2)
+; AVX512BF16-NEXT: vaddss %xmm5, %xmm4, %xmm4
+; AVX512BF16-NEXT: vroundss $11, %xmm4, %xmm4, %xmm4
+; AVX512BF16-NEXT: vcvtneps2bf16 %xmm4, %xmm4
+; AVX512BF16-NEXT: vmovd %xmm4, %r8d
+; AVX512BF16-NEXT: vpblendw {{.*#+}} xmm1 = xmm1[0],xmm0[1],xmm1[2],xmm0[3],xmm1[4],xmm0[5],xmm1[6],xmm0[7]
+; AVX512BF16-NEXT: vmovdqa %xmm3, %xmm4
+; AVX512BF16-NEXT: vpternlogd {{.*#+}} xmm4 = xmm4 | (xmm1 & xmm2)
+; AVX512BF16-NEXT: vaddss %xmm4, %xmm1, %xmm1
+; AVX512BF16-NEXT: vroundss $11, %xmm1, %xmm1, %xmm1
+; AVX512BF16-NEXT: vcvtneps2bf16 %xmm1, %xmm1
+; AVX512BF16-NEXT: vpslld $16, %xmm0, %xmm0
+; AVX512BF16-NEXT: vpternlogd {{.*#+}} xmm3 = xmm3 | (xmm0 & xmm2)
+; AVX512BF16-NEXT: vaddss %xmm3, %xmm0, %xmm0
+; AVX512BF16-NEXT: vroundss $11, %xmm0, %xmm0, %xmm0
+; AVX512BF16-NEXT: vcvtneps2bf16 %xmm0, %xmm0
+; AVX512BF16-NEXT: vpunpcklwd {{.*#+}} xmm0 = xmm0[0],xmm1[0],xmm0[1],xmm1[1],xmm0[2],xmm1[2],xmm0[3],xmm1[3]
+; AVX512BF16-NEXT: vpinsrw $2, %r8d, %xmm0, %xmm0
+; AVX512BF16-NEXT: vpinsrw $3, %edi, %xmm0, %xmm0
+; AVX512BF16-NEXT: vpinsrw $4, %esi, %xmm0, %xmm0
+; AVX512BF16-NEXT: vpinsrw $5, %edx, %xmm0, %xmm0
+; AVX512BF16-NEXT: vpinsrw $6, %ecx, %xmm0, %xmm0
+; AVX512BF16-NEXT: vpinsrw $7, %eax, %xmm0, %xmm0
+; AVX512BF16-NEXT: retq
%r = call <8 x bfloat> @llvm.round.v8bf16(<8 x bfloat> %a)
ret <8 x bfloat> %r
}
diff --git a/llvm/test/CodeGen/X86/znver-pdep.ll b/llvm/test/CodeGen/X86/znver-pdep.ll
index 1acb6a49f44d0..afd97ebca3e0e 100644
--- a/llvm/test/CodeGen/X86/znver-pdep.ll
+++ b/llvm/test/CodeGen/X86/znver-pdep.ll
@@ -1005,114 +1005,310 @@ define <4 x i64> @pdep_v4i64(<4 x i64> %val, <4 x i64> %mask) nounwind {
;
define <4 x i32> @pdep_v4i32_minsize(<4 x i32> %val, <4 x i32> %mask) nounwind minsize {
-; CHECK-LABEL: pdep_v4i32_minsize:
-; CHECK: # %bb.0:
-; CHECK-NEXT: vpextrd $1, %xmm1, %eax
-; CHECK-NEXT: vpextrd $1, %xmm0, %ecx
-; CHECK-NEXT: vmovd %xmm0, %edx
-; CHECK-NEXT: pdepl %eax, %ecx, %eax
-; CHECK-NEXT: vmovd %xmm1, %ecx
-; CHECK-NEXT: pdepl %ecx, %edx, %ecx
-; CHECK-NEXT: vmovd %ecx, %xmm2
-; CHECK-NEXT: vpextrd $2, %xmm0, %ecx
-; CHECK-NEXT: vpinsrd $1, %eax, %xmm2, %xmm2
-; CHECK-NEXT: vpextrd $2, %xmm1, %eax
-; CHECK-NEXT: pdepl %eax, %ecx, %eax
-; CHECK-NEXT: vpextrd $3, %xmm0, %ecx
-; CHECK-NEXT: vpinsrd $2, %eax, %xmm2, %xmm2
-; CHECK-NEXT: vpextrd $3, %xmm1, %eax
-; CHECK-NEXT: pdepl %eax, %ecx, %eax
-; CHECK-NEXT: vpinsrd $3, %eax, %xmm2, %xmm0
-; CHECK-NEXT: retq
+; SLOW-ZNVER-LABEL: pdep_v4i32_minsize:
+; SLOW-ZNVER: # %bb.0:
+; SLOW-ZNVER-NEXT: vpextrd $1, %xmm1, %eax
+; SLOW-ZNVER-NEXT: vpextrd $1, %xmm0, %ecx
+; SLOW-ZNVER-NEXT: vmovd %xmm0, %edx
+; SLOW-ZNVER-NEXT: vpextrd $2, %xmm0, %esi
+; SLOW-ZNVER-NEXT: pdepl %eax, %ecx, %eax
+; SLOW-ZNVER-NEXT: vmovd %xmm1, %ecx
+; SLOW-ZNVER-NEXT: pdepl %ecx, %edx, %ecx
+; SLOW-ZNVER-NEXT: vpextrd $2, %xmm1, %edx
+; SLOW-ZNVER-NEXT: pdepl %edx, %esi, %edx
+; SLOW-ZNVER-NEXT: vpextrd $3, %xmm0, %esi
+; SLOW-ZNVER-NEXT: vmovd %ecx, %xmm2
+; SLOW-ZNVER-NEXT: vpextrd $3, %xmm1, %ecx
+; SLOW-ZNVER-NEXT: pdepl %ecx, %esi, %ecx
+; SLOW-ZNVER-NEXT: vpinsrd $1, %eax, %xmm2, %xmm0
+; SLOW-ZNVER-NEXT: vpinsrd $2, %edx, %xmm0, %xmm0
+; SLOW-ZNVER-NEXT: vpinsrd $3, %ecx, %xmm0, %xmm0
+; SLOW-ZNVER-NEXT: retq
+;
+; SLOW-BDVER4-LABEL: pdep_v4i32_minsize:
+; SLOW-BDVER4: # %bb.0:
+; SLOW-BDVER4-NEXT: vpextrd $1, %xmm1, %eax
+; SLOW-BDVER4-NEXT: vpextrd $1, %xmm0, %ecx
+; SLOW-BDVER4-NEXT: vmovd %xmm0, %edx
+; SLOW-BDVER4-NEXT: vpextrd $2, %xmm0, %esi
+; SLOW-BDVER4-NEXT: pdepl %eax, %ecx, %eax
+; SLOW-BDVER4-NEXT: vmovd %xmm1, %ecx
+; SLOW-BDVER4-NEXT: pdepl %ecx, %edx, %ecx
+; SLOW-BDVER4-NEXT: vpextrd $2, %xmm1, %edx
+; SLOW-BDVER4-NEXT: pdepl %edx, %esi, %edx
+; SLOW-BDVER4-NEXT: vpextrd $3, %xmm0, %esi
+; SLOW-BDVER4-NEXT: vmovd %ecx, %xmm2
+; SLOW-BDVER4-NEXT: vpextrd $3, %xmm1, %ecx
+; SLOW-BDVER4-NEXT: pdepl %ecx, %esi, %ecx
+; SLOW-BDVER4-NEXT: vpinsrd $1, %eax, %xmm2, %xmm0
+; SLOW-BDVER4-NEXT: vpinsrd $2, %edx, %xmm0, %xmm0
+; SLOW-BDVER4-NEXT: vpinsrd $3, %ecx, %xmm0, %xmm0
+; SLOW-BDVER4-NEXT: retq
+;
+; FAST-LABEL: pdep_v4i32_minsize:
+; FAST: # %bb.0:
+; FAST-NEXT: vpextrd $1, %xmm1, %eax
+; FAST-NEXT: vpextrd $1, %xmm0, %ecx
+; FAST-NEXT: vmovd %xmm0, %edx
+; FAST-NEXT: pdepl %eax, %ecx, %eax
+; FAST-NEXT: vmovd %xmm1, %ecx
+; FAST-NEXT: pdepl %ecx, %edx, %ecx
+; FAST-NEXT: vmovd %ecx, %xmm2
+; FAST-NEXT: vpextrd $2, %xmm0, %ecx
+; FAST-NEXT: vpinsrd $1, %eax, %xmm2, %xmm2
+; FAST-NEXT: vpextrd $2, %xmm1, %eax
+; FAST-NEXT: pdepl %eax, %ecx, %eax
+; FAST-NEXT: vpextrd $3, %xmm0, %ecx
+; FAST-NEXT: vpinsrd $2, %eax, %xmm2, %xmm2
+; FAST-NEXT: vpextrd $3, %xmm1, %eax
+; FAST-NEXT: pdepl %eax, %ecx, %eax
+; FAST-NEXT: vpinsrd $3, %eax, %xmm2, %xmm0
+; FAST-NEXT: retq
%res = call <4 x i32> @llvm.pdep.v4i32(<4 x i32> %val, <4 x i32> %mask)
ret <4 x i32> %res
}
define <8 x i32> @pdep_v8i32_minsize(<8 x i32> %val, <8 x i32> %mask) nounwind minsize {
-; CHECK-LABEL: pdep_v8i32_minsize:
-; CHECK: # %bb.0:
-; CHECK-NEXT: vextracti128 $1, %ymm0, %xmm3
-; CHECK-NEXT: vextracti128 $1, %ymm1, %xmm2
-; CHECK-NEXT: vpextrd $1, %xmm2, %eax
-; CHECK-NEXT: vpextrd $1, %xmm3, %ecx
-; CHECK-NEXT: vmovd %xmm3, %edx
-; CHECK-NEXT: pdepl %eax, %ecx, %eax
-; CHECK-NEXT: vmovd %xmm2, %ecx
-; CHECK-NEXT: pdepl %ecx, %edx, %ecx
-; CHECK-NEXT: vmovd %xmm0, %edx
-; CHECK-NEXT: vmovd %ecx, %xmm4
-; CHECK-NEXT: vpextrd $2, %xmm3, %ecx
-; CHECK-NEXT: vpinsrd $1, %eax, %xmm4, %xmm4
-; CHECK-NEXT: vpextrd $2, %xmm2, %eax
-; CHECK-NEXT: pdepl %eax, %ecx, %eax
-; CHECK-NEXT: vpextrd $3, %xmm3, %ecx
-; CHECK-NEXT: vpinsrd $2, %eax, %xmm4, %xmm4
-; CHECK-NEXT: vpextrd $3, %xmm2, %eax
-; CHECK-NEXT: pdepl %eax, %ecx, %eax
-; CHECK-NEXT: vpextrd $1, %xmm0, %ecx
-; CHECK-NEXT: vpinsrd $3, %eax, %xmm4, %xmm2
-; CHECK-NEXT: vpextrd $1, %xmm1, %eax
-; CHECK-NEXT: pdepl %eax, %ecx, %eax
-; CHECK-NEXT: vmovd %xmm1, %ecx
-; CHECK-NEXT: pdepl %ecx, %edx, %ecx
-; CHECK-NEXT: vmovd %ecx, %xmm3
-; CHECK-NEXT: vpextrd $2, %xmm0, %ecx
-; CHECK-NEXT: vpinsrd $1, %eax, %xmm3, %xmm3
-; CHECK-NEXT: vpextrd $2, %xmm1, %eax
-; CHECK-NEXT: pdepl %eax, %ecx, %eax
-; CHECK-NEXT: vpextrd $3, %xmm0, %ecx
-; CHECK-NEXT: vpinsrd $2, %eax, %xmm3, %xmm3
-; CHECK-NEXT: vpextrd $3, %xmm1, %eax
-; CHECK-NEXT: pdepl %eax, %ecx, %eax
-; CHECK-NEXT: vpinsrd $3, %eax, %xmm3, %xmm0
-; CHECK-NEXT: vinserti128 $1, %xmm2, %ymm0, %ymm0
-; CHECK-NEXT: retq
+; SLOW-ZNVER-LABEL: pdep_v8i32_minsize:
+; SLOW-ZNVER: # %bb.0:
+; SLOW-ZNVER-NEXT: vextracti128 $1, %ymm0, %xmm3
+; SLOW-ZNVER-NEXT: vextracti128 $1, %ymm1, %xmm2
+; SLOW-ZNVER-NEXT: vpextrd $1, %xmm0, %r8d
+; SLOW-ZNVER-NEXT: vmovd %xmm0, %r9d
+; SLOW-ZNVER-NEXT: vpextrd $1, %xmm2, %eax
+; SLOW-ZNVER-NEXT: vpextrd $1, %xmm3, %ecx
+; SLOW-ZNVER-NEXT: vmovd %xmm3, %edx
+; SLOW-ZNVER-NEXT: vpextrd $2, %xmm3, %esi
+; SLOW-ZNVER-NEXT: vpextrd $3, %xmm3, %edi
+; SLOW-ZNVER-NEXT: pdepl %eax, %ecx, %eax
+; SLOW-ZNVER-NEXT: vmovd %xmm2, %ecx
+; SLOW-ZNVER-NEXT: pdepl %ecx, %edx, %ecx
+; SLOW-ZNVER-NEXT: vpextrd $2, %xmm2, %edx
+; SLOW-ZNVER-NEXT: pdepl %edx, %esi, %edx
+; SLOW-ZNVER-NEXT: vpextrd $3, %xmm2, %esi
+; SLOW-ZNVER-NEXT: pdepl %esi, %edi, %esi
+; SLOW-ZNVER-NEXT: vpextrd $1, %xmm1, %edi
+; SLOW-ZNVER-NEXT: pdepl %edi, %r8d, %edi
+; SLOW-ZNVER-NEXT: vmovd %xmm1, %r8d
+; SLOW-ZNVER-NEXT: pdepl %r8d, %r9d, %r8d
+; SLOW-ZNVER-NEXT: vmovd %ecx, %xmm2
+; SLOW-ZNVER-NEXT: vpextrd $2, %xmm0, %ecx
+; SLOW-ZNVER-NEXT: vpinsrd $1, %eax, %xmm2, %xmm2
+; SLOW-ZNVER-NEXT: vpextrd $2, %xmm1, %eax
+; SLOW-ZNVER-NEXT: vpinsrd $2, %edx, %xmm2, %xmm2
+; SLOW-ZNVER-NEXT: vpextrd $3, %xmm0, %edx
+; SLOW-ZNVER-NEXT: pdepl %eax, %ecx, %eax
+; SLOW-ZNVER-NEXT: vpextrd $3, %xmm1, %ecx
+; SLOW-ZNVER-NEXT: vpinsrd $3, %esi, %xmm2, %xmm2
+; SLOW-ZNVER-NEXT: pdepl %ecx, %edx, %ecx
+; SLOW-ZNVER-NEXT: vmovd %r8d, %xmm3
+; SLOW-ZNVER-NEXT: vpinsrd $1, %edi, %xmm3, %xmm3
+; SLOW-ZNVER-NEXT: vpinsrd $2, %eax, %xmm3, %xmm0
+; SLOW-ZNVER-NEXT: vpinsrd $3, %ecx, %xmm0, %xmm0
+; SLOW-ZNVER-NEXT: vinserti128 $1, %xmm2, %ymm0, %ymm0
+; SLOW-ZNVER-NEXT: retq
+;
+; SLOW-BDVER4-LABEL: pdep_v8i32_minsize:
+; SLOW-BDVER4: # %bb.0:
+; SLOW-BDVER4-NEXT: vextracti128 $1, %ymm0, %xmm3
+; SLOW-BDVER4-NEXT: vextracti128 $1, %ymm1, %xmm2
+; SLOW-BDVER4-NEXT: vpextrd $1, %xmm0, %r8d
+; SLOW-BDVER4-NEXT: vmovd %xmm0, %r9d
+; SLOW-BDVER4-NEXT: vpextrd $1, %xmm2, %eax
+; SLOW-BDVER4-NEXT: vpextrd $1, %xmm3, %ecx
+; SLOW-BDVER4-NEXT: vmovd %xmm3, %edx
+; SLOW-BDVER4-NEXT: vpextrd $2, %xmm3, %esi
+; SLOW-BDVER4-NEXT: vpextrd $3, %xmm3, %edi
+; SLOW-BDVER4-NEXT: pdepl %eax, %ecx, %eax
+; SLOW-BDVER4-NEXT: vmovd %xmm2, %ecx
+; SLOW-BDVER4-NEXT: pdepl %ecx, %edx, %ecx
+; SLOW-BDVER4-NEXT: vpextrd $2, %xmm2, %edx
+; SLOW-BDVER4-NEXT: pdepl %edx, %esi, %edx
+; SLOW-BDVER4-NEXT: vpextrd $3, %xmm2, %esi
+; SLOW-BDVER4-NEXT: pdepl %esi, %edi, %esi
+; SLOW-BDVER4-NEXT: vpextrd $1, %xmm1, %edi
+; SLOW-BDVER4-NEXT: pdepl %edi, %r8d, %edi
+; SLOW-BDVER4-NEXT: vmovd %xmm1, %r8d
+; SLOW-BDVER4-NEXT: pdepl %r8d, %r9d, %r8d
+; SLOW-BDVER4-NEXT: vmovd %ecx, %xmm2
+; SLOW-BDVER4-NEXT: vpextrd $2, %xmm0, %ecx
+; SLOW-BDVER4-NEXT: vpinsrd $1, %eax, %xmm2, %xmm2
+; SLOW-BDVER4-NEXT: vpextrd $2, %xmm1, %eax
+; SLOW-BDVER4-NEXT: vpinsrd $2, %edx, %xmm2, %xmm2
+; SLOW-BDVER4-NEXT: vpextrd $3, %xmm0, %edx
+; SLOW-BDVER4-NEXT: pdepl %eax, %ecx, %eax
+; SLOW-BDVER4-NEXT: vpextrd $3, %xmm1, %ecx
+; SLOW-BDVER4-NEXT: vpinsrd $3, %esi, %xmm2, %xmm2
+; SLOW-BDVER4-NEXT: pdepl %ecx, %edx, %ecx
+; SLOW-BDVER4-NEXT: vmovd %r8d, %xmm3
+; SLOW-BDVER4-NEXT: vpinsrd $1, %edi, %xmm3, %xmm3
+; SLOW-BDVER4-NEXT: vpinsrd $2, %eax, %xmm3, %xmm0
+; SLOW-BDVER4-NEXT: vpinsrd $3, %ecx, %xmm0, %xmm0
+; SLOW-BDVER4-NEXT: vinserti128 $1, %xmm2, %ymm0, %ymm0
+; SLOW-BDVER4-NEXT: retq
+;
+; FAST-LABEL: pdep_v8i32_minsize:
+; FAST: # %bb.0:
+; FAST-NEXT: vextracti128 $1, %ymm0, %xmm3
+; FAST-NEXT: vextracti128 $1, %ymm1, %xmm2
+; FAST-NEXT: vpextrd $1, %xmm2, %eax
+; FAST-NEXT: vpextrd $1, %xmm3, %ecx
+; FAST-NEXT: vmovd %xmm3, %edx
+; FAST-NEXT: pdepl %eax, %ecx, %eax
+; FAST-NEXT: vmovd %xmm2, %ecx
+; FAST-NEXT: pdepl %ecx, %edx, %ecx
+; FAST-NEXT: vmovd %xmm0, %edx
+; FAST-NEXT: vmovd %ecx, %xmm4
+; FAST-NEXT: vpextrd $2, %xmm3, %ecx
+; FAST-NEXT: vpinsrd $1, %eax, %xmm4, %xmm4
+; FAST-NEXT: vpextrd $2, %xmm2, %eax
+; FAST-NEXT: pdepl %eax, %ecx, %eax
+; FAST-NEXT: vpextrd $3, %xmm3, %ecx
+; FAST-NEXT: vpinsrd $2, %eax, %xmm4, %xmm4
+; FAST-NEXT: vpextrd $3, %xmm2, %eax
+; FAST-NEXT: pdepl %eax, %ecx, %eax
+; FAST-NEXT: vpextrd $1, %xmm0, %ecx
+; FAST-NEXT: vpinsrd $3, %eax, %xmm4, %xmm2
+; FAST-NEXT: vpextrd $1, %xmm1, %eax
+; FAST-NEXT: pdepl %eax, %ecx, %eax
+; FAST-NEXT: vmovd %xmm1, %ecx
+; FAST-NEXT: pdepl %ecx, %edx, %ecx
+; FAST-NEXT: vmovd %ecx, %xmm3
+; FAST-NEXT: vpextrd $2, %xmm0, %ecx
+; FAST-NEXT: vpinsrd $1, %eax, %xmm3, %xmm3
+; FAST-NEXT: vpextrd $2, %xmm1, %eax
+; FAST-NEXT: pdepl %eax, %ecx, %eax
+; FAST-NEXT: vpextrd $3, %xmm0, %ecx
+; FAST-NEXT: vpinsrd $2, %eax, %xmm3, %xmm3
+; FAST-NEXT: vpextrd $3, %xmm1, %eax
+; FAST-NEXT: pdepl %eax, %ecx, %eax
+; FAST-NEXT: vpinsrd $3, %eax, %xmm3, %xmm0
+; FAST-NEXT: vinserti128 $1, %xmm2, %ymm0, %ymm0
+; FAST-NEXT: retq
%res = call <8 x i32> @llvm.pdep.v8i32(<8 x i32> %val, <8 x i32> %mask)
ret <8 x i32> %res
}
define <2 x i64> @pdep_v2i64_minsize(<2 x i64> %val, <2 x i64> %mask) nounwind minsize {
-; CHECK-LABEL: pdep_v2i64_minsize:
-; CHECK: # %bb.0:
-; CHECK-NEXT: vpextrq $1, %xmm1, %rax
-; CHECK-NEXT: vpextrq $1, %xmm0, %rcx
-; CHECK-NEXT: vmovq %xmm1, %rdx
-; CHECK-NEXT: pdepq %rax, %rcx, %rax
-; CHECK-NEXT: vmovq %xmm0, %rcx
-; CHECK-NEXT: vmovq %rax, %xmm2
-; CHECK-NEXT: pdepq %rdx, %rcx, %rax
-; CHECK-NEXT: vmovq %rax, %xmm0
-; CHECK-NEXT: vpunpcklqdq {{.*#+}} xmm0 = xmm0[0],xmm2[0]
-; CHECK-NEXT: retq
+; SLOW-ZNVER-LABEL: pdep_v2i64_minsize:
+; SLOW-ZNVER: # %bb.0:
+; SLOW-ZNVER-NEXT: vpextrq $1, %xmm1, %rax
+; SLOW-ZNVER-NEXT: vpextrq $1, %xmm0, %rcx
+; SLOW-ZNVER-NEXT: vmovq %xmm1, %rsi
+; SLOW-ZNVER-NEXT: vmovq %xmm0, %rdx
+; SLOW-ZNVER-NEXT: pdepq %rax, %rcx, %rax
+; SLOW-ZNVER-NEXT: pdepq %rsi, %rdx, %rcx
+; SLOW-ZNVER-NEXT: vmovq %rax, %xmm0
+; SLOW-ZNVER-NEXT: vmovq %rcx, %xmm1
+; SLOW-ZNVER-NEXT: vpunpcklqdq {{.*#+}} xmm0 = xmm1[0],xmm0[0]
+; SLOW-ZNVER-NEXT: retq
+;
+; SLOW-BDVER4-LABEL: pdep_v2i64_minsize:
+; SLOW-BDVER4: # %bb.0:
+; SLOW-BDVER4-NEXT: vpextrq $1, %xmm1, %rax
+; SLOW-BDVER4-NEXT: vpextrq $1, %xmm0, %rcx
+; SLOW-BDVER4-NEXT: vmovq %xmm1, %rsi
+; SLOW-BDVER4-NEXT: vmovq %xmm0, %rdx
+; SLOW-BDVER4-NEXT: pdepq %rax, %rcx, %rax
+; SLOW-BDVER4-NEXT: pdepq %rsi, %rdx, %rcx
+; SLOW-BDVER4-NEXT: vmovq %rax, %xmm0
+; SLOW-BDVER4-NEXT: vmovq %rcx, %xmm1
+; SLOW-BDVER4-NEXT: vpunpcklqdq {{.*#+}} xmm0 = xmm1[0],xmm0[0]
+; SLOW-BDVER4-NEXT: retq
+;
+; FAST-LABEL: pdep_v2i64_minsize:
+; FAST: # %bb.0:
+; FAST-NEXT: vpextrq $1, %xmm1, %rax
+; FAST-NEXT: vpextrq $1, %xmm0, %rcx
+; FAST-NEXT: vmovq %xmm1, %rdx
+; FAST-NEXT: pdepq %rax, %rcx, %rax
+; FAST-NEXT: vmovq %xmm0, %rcx
+; FAST-NEXT: vmovq %rax, %xmm2
+; FAST-NEXT: pdepq %rdx, %rcx, %rax
+; FAST-NEXT: vmovq %rax, %xmm0
+; FAST-NEXT: vpunpcklqdq {{.*#+}} xmm0 = xmm0[0],xmm2[0]
+; FAST-NEXT: retq
%res = call <2 x i64> @llvm.pdep.v2i64(<2 x i64> %val, <2 x i64> %mask)
ret <2 x i64> %res
}
define <4 x i64> @pdep_v4i64_minsize(<4 x i64> %val, <4 x i64> %mask) nounwind minsize {
-; CHECK-LABEL: pdep_v4i64_minsize:
-; CHECK: # %bb.0:
-; CHECK-NEXT: vextracti128 $1, %ymm1, %xmm2
-; CHECK-NEXT: vextracti128 $1, %ymm0, %xmm3
-; CHECK-NEXT: vpextrq $1, %xmm1, %rsi
-; CHECK-NEXT: vpextrq $1, %xmm2, %rax
-; CHECK-NEXT: vpextrq $1, %xmm3, %rcx
-; CHECK-NEXT: vmovq %xmm2, %rdx
-; CHECK-NEXT: pdepq %rax, %rcx, %rax
-; CHECK-NEXT: vmovq %xmm3, %rcx
-; CHECK-NEXT: vmovq %rax, %xmm4
-; CHECK-NEXT: pdepq %rdx, %rcx, %rax
-; CHECK-NEXT: vpextrq $1, %xmm0, %rcx
-; CHECK-NEXT: vmovq %xmm1, %rdx
-; CHECK-NEXT: vmovq %rax, %xmm2
-; CHECK-NEXT: pdepq %rsi, %rcx, %rax
-; CHECK-NEXT: vmovq %xmm0, %rcx
-; CHECK-NEXT: vpunpcklqdq {{.*#+}} xmm2 = xmm2[0],xmm4[0]
-; CHECK-NEXT: vmovq %rax, %xmm3
-; CHECK-NEXT: pdepq %rdx, %rcx, %rax
-; CHECK-NEXT: vmovq %rax, %xmm0
-; CHECK-NEXT: vpunpcklqdq {{.*#+}} xmm0 = xmm0[0],xmm3[0]
-; CHECK-NEXT: vinserti128 $1, %xmm2, %ymm0, %ymm0
-; CHECK-NEXT: retq
+; SLOW-ZNVER-LABEL: pdep_v4i64_minsize:
+; SLOW-ZNVER: # %bb.0:
+; SLOW-ZNVER-NEXT: vextracti128 $1, %ymm1, %xmm2
+; SLOW-ZNVER-NEXT: vextracti128 $1, %ymm0, %xmm3
+; SLOW-ZNVER-NEXT: vpextrq $1, %xmm2, %rax
+; SLOW-ZNVER-NEXT: vpextrq $1, %xmm3, %rcx
+; SLOW-ZNVER-NEXT: vmovq %xmm2, %rsi
+; SLOW-ZNVER-NEXT: vmovq %xmm3, %rdx
+; SLOW-ZNVER-NEXT: pdepq %rax, %rcx, %rax
+; SLOW-ZNVER-NEXT: pdepq %rsi, %rdx, %rcx
+; SLOW-ZNVER-NEXT: vpextrq $1, %xmm0, %rdx
+; SLOW-ZNVER-NEXT: vmovq %rax, %xmm2
+; SLOW-ZNVER-NEXT: vpextrq $1, %xmm1, %rax
+; SLOW-ZNVER-NEXT: vmovq %rcx, %xmm3
+; SLOW-ZNVER-NEXT: vmovq %xmm0, %rcx
+; SLOW-ZNVER-NEXT: pdepq %rax, %rdx, %rax
+; SLOW-ZNVER-NEXT: vmovq %xmm1, %rdx
+; SLOW-ZNVER-NEXT: vpunpcklqdq {{.*#+}} xmm2 = xmm3[0],xmm2[0]
+; SLOW-ZNVER-NEXT: vmovq %rax, %xmm3
+; SLOW-ZNVER-NEXT: pdepq %rdx, %rcx, %rax
+; SLOW-ZNVER-NEXT: vmovq %rax, %xmm0
+; SLOW-ZNVER-NEXT: vpunpcklqdq {{.*#+}} xmm0 = xmm0[0],xmm3[0]
+; SLOW-ZNVER-NEXT: vinserti128 $1, %xmm2, %ymm0, %ymm0
+; SLOW-ZNVER-NEXT: retq
+;
+; SLOW-BDVER4-LABEL: pdep_v4i64_minsize:
+; SLOW-BDVER4: # %bb.0:
+; SLOW-BDVER4-NEXT: vextracti128 $1, %ymm1, %xmm2
+; SLOW-BDVER4-NEXT: vextracti128 $1, %ymm0, %xmm3
+; SLOW-BDVER4-NEXT: vpextrq $1, %xmm2, %rax
+; SLOW-BDVER4-NEXT: vpextrq $1, %xmm3, %rcx
+; SLOW-BDVER4-NEXT: vmovq %xmm2, %rsi
+; SLOW-BDVER4-NEXT: vmovq %xmm3, %rdx
+; SLOW-BDVER4-NEXT: pdepq %rax, %rcx, %rax
+; SLOW-BDVER4-NEXT: pdepq %rsi, %rdx, %rcx
+; SLOW-BDVER4-NEXT: vpextrq $1, %xmm0, %rdx
+; SLOW-BDVER4-NEXT: vmovq %rax, %xmm2
+; SLOW-BDVER4-NEXT: vpextrq $1, %xmm1, %rax
+; SLOW-BDVER4-NEXT: vmovq %rcx, %xmm3
+; SLOW-BDVER4-NEXT: vmovq %xmm0, %rcx
+; SLOW-BDVER4-NEXT: pdepq %rax, %rdx, %rax
+; SLOW-BDVER4-NEXT: vmovq %xmm1, %rdx
+; SLOW-BDVER4-NEXT: vpunpcklqdq {{.*#+}} xmm2 = xmm3[0],xmm2[0]
+; SLOW-BDVER4-NEXT: vmovq %rax, %xmm3
+; SLOW-BDVER4-NEXT: pdepq %rdx, %rcx, %rax
+; SLOW-BDVER4-NEXT: vmovq %rax, %xmm0
+; SLOW-BDVER4-NEXT: vpunpcklqdq {{.*#+}} xmm0 = xmm0[0],xmm3[0]
+; SLOW-BDVER4-NEXT: vinserti128 $1, %xmm2, %ymm0, %ymm0
+; SLOW-BDVER4-NEXT: retq
+;
+; FAST-LABEL: pdep_v4i64_minsize:
+; FAST: # %bb.0:
+; FAST-NEXT: vextracti128 $1, %ymm1, %xmm2
+; FAST-NEXT: vextracti128 $1, %ymm0, %xmm3
+; FAST-NEXT: vpextrq $1, %xmm1, %rsi
+; FAST-NEXT: vpextrq $1, %xmm2, %rax
+; FAST-NEXT: vpextrq $1, %xmm3, %rcx
+; FAST-NEXT: vmovq %xmm2, %rdx
+; FAST-NEXT: pdepq %rax, %rcx, %rax
+; FAST-NEXT: vmovq %xmm3, %rcx
+; FAST-NEXT: vmovq %rax, %xmm4
+; FAST-NEXT: pdepq %rdx, %rcx, %rax
+; FAST-NEXT: vpextrq $1, %xmm0, %rcx
+; FAST-NEXT: vmovq %xmm1, %rdx
+; FAST-NEXT: vmovq %rax, %xmm2
+; FAST-NEXT: pdepq %rsi, %rcx, %rax
+; FAST-NEXT: vmovq %xmm0, %rcx
+; FAST-NEXT: vpunpcklqdq {{.*#+}} xmm2 = xmm2[0],xmm4[0]
+; FAST-NEXT: vmovq %rax, %xmm3
+; FAST-NEXT: pdepq %rdx, %rcx, %rax
+; FAST-NEXT: vmovq %rax, %xmm0
+; FAST-NEXT: vpunpcklqdq {{.*#+}} xmm0 = xmm0[0],xmm3[0]
+; FAST-NEXT: vinserti128 $1, %xmm2, %ymm0, %ymm0
+; FAST-NEXT: retq
%res = call <4 x i64> @llvm.pdep.v4i64(<4 x i64> %val, <4 x i64> %mask)
ret <4 x i64> %res
}
+;; NOTE: These prefixes are unused and the list is autogenerated. Do not add tests below this line:
+; CHECK: {{.*}}
diff --git a/llvm/test/CodeGen/X86/znver-pext.ll b/llvm/test/CodeGen/X86/znver-pext.ll
index 5fbc3c613f025..d714588d1aa05 100644
--- a/llvm/test/CodeGen/X86/znver-pext.ll
+++ b/llvm/test/CodeGen/X86/znver-pext.ll
@@ -586,114 +586,213 @@ define <4 x i64> @pext_v4i64(<4 x i64> %val, <4 x i64> %mask) nounwind {
;
define <4 x i32> @pext_v4i32_minsize(<4 x i32> %val, <4 x i32> %mask) nounwind minsize {
-; CHECK-LABEL: pext_v4i32_minsize:
-; CHECK: # %bb.0:
-; CHECK-NEXT: vpextrd $1, %xmm1, %eax
-; CHECK-NEXT: vpextrd $1, %xmm0, %ecx
-; CHECK-NEXT: vmovd %xmm0, %edx
-; CHECK-NEXT: pextl %eax, %ecx, %eax
-; CHECK-NEXT: vmovd %xmm1, %ecx
-; CHECK-NEXT: pextl %ecx, %edx, %ecx
-; CHECK-NEXT: vmovd %ecx, %xmm2
-; CHECK-NEXT: vpextrd $2, %xmm0, %ecx
-; CHECK-NEXT: vpinsrd $1, %eax, %xmm2, %xmm2
-; CHECK-NEXT: vpextrd $2, %xmm1, %eax
-; CHECK-NEXT: pextl %eax, %ecx, %eax
-; CHECK-NEXT: vpextrd $3, %xmm0, %ecx
-; CHECK-NEXT: vpinsrd $2, %eax, %xmm2, %xmm2
-; CHECK-NEXT: vpextrd $3, %xmm1, %eax
-; CHECK-NEXT: pextl %eax, %ecx, %eax
-; CHECK-NEXT: vpinsrd $3, %eax, %xmm2, %xmm0
-; CHECK-NEXT: retq
+; SLOW-LABEL: pext_v4i32_minsize:
+; SLOW: # %bb.0:
+; SLOW-NEXT: vpextrd $1, %xmm1, %eax
+; SLOW-NEXT: vpextrd $1, %xmm0, %ecx
+; SLOW-NEXT: vmovd %xmm0, %edx
+; SLOW-NEXT: vpextrd $2, %xmm0, %esi
+; SLOW-NEXT: pextl %eax, %ecx, %eax
+; SLOW-NEXT: vmovd %xmm1, %ecx
+; SLOW-NEXT: pextl %ecx, %edx, %ecx
+; SLOW-NEXT: vpextrd $2, %xmm1, %edx
+; SLOW-NEXT: pextl %edx, %esi, %edx
+; SLOW-NEXT: vpextrd $3, %xmm0, %esi
+; SLOW-NEXT: vmovd %ecx, %xmm2
+; SLOW-NEXT: vpextrd $3, %xmm1, %ecx
+; SLOW-NEXT: pextl %ecx, %esi, %ecx
+; SLOW-NEXT: vpinsrd $1, %eax, %xmm2, %xmm0
+; SLOW-NEXT: vpinsrd $2, %edx, %xmm0, %xmm0
+; SLOW-NEXT: vpinsrd $3, %ecx, %xmm0, %xmm0
+; SLOW-NEXT: retq
+;
+; FAST-LABEL: pext_v4i32_minsize:
+; FAST: # %bb.0:
+; FAST-NEXT: vpextrd $1, %xmm1, %eax
+; FAST-NEXT: vpextrd $1, %xmm0, %ecx
+; FAST-NEXT: vmovd %xmm0, %edx
+; FAST-NEXT: pextl %eax, %ecx, %eax
+; FAST-NEXT: vmovd %xmm1, %ecx
+; FAST-NEXT: pextl %ecx, %edx, %ecx
+; FAST-NEXT: vmovd %ecx, %xmm2
+; FAST-NEXT: vpextrd $2, %xmm0, %ecx
+; FAST-NEXT: vpinsrd $1, %eax, %xmm2, %xmm2
+; FAST-NEXT: vpextrd $2, %xmm1, %eax
+; FAST-NEXT: pextl %eax, %ecx, %eax
+; FAST-NEXT: vpextrd $3, %xmm0, %ecx
+; FAST-NEXT: vpinsrd $2, %eax, %xmm2, %xmm2
+; FAST-NEXT: vpextrd $3, %xmm1, %eax
+; FAST-NEXT: pextl %eax, %ecx, %eax
+; FAST-NEXT: vpinsrd $3, %eax, %xmm2, %xmm0
+; FAST-NEXT: retq
%res = call <4 x i32> @llvm.pext.v4i32(<4 x i32> %val, <4 x i32> %mask)
ret <4 x i32> %res
}
define <8 x i32> @pext_v8i32_minsize(<8 x i32> %val, <8 x i32> %mask) nounwind minsize {
-; CHECK-LABEL: pext_v8i32_minsize:
-; CHECK: # %bb.0:
-; CHECK-NEXT: vextracti128 $1, %ymm0, %xmm3
-; CHECK-NEXT: vextracti128 $1, %ymm1, %xmm2
-; CHECK-NEXT: vpextrd $1, %xmm2, %eax
-; CHECK-NEXT: vpextrd $1, %xmm3, %ecx
-; CHECK-NEXT: vmovd %xmm3, %edx
-; CHECK-NEXT: pextl %eax, %ecx, %eax
-; CHECK-NEXT: vmovd %xmm2, %ecx
-; CHECK-NEXT: pextl %ecx, %edx, %ecx
-; CHECK-NEXT: vmovd %xmm0, %edx
-; CHECK-NEXT: vmovd %ecx, %xmm4
-; CHECK-NEXT: vpextrd $2, %xmm3, %ecx
-; CHECK-NEXT: vpinsrd $1, %eax, %xmm4, %xmm4
-; CHECK-NEXT: vpextrd $2, %xmm2, %eax
-; CHECK-NEXT: pextl %eax, %ecx, %eax
-; CHECK-NEXT: vpextrd $3, %xmm3, %ecx
-; CHECK-NEXT: vpinsrd $2, %eax, %xmm4, %xmm4
-; CHECK-NEXT: vpextrd $3, %xmm2, %eax
-; CHECK-NEXT: pextl %eax, %ecx, %eax
-; CHECK-NEXT: vpextrd $1, %xmm0, %ecx
-; CHECK-NEXT: vpinsrd $3, %eax, %xmm4, %xmm2
-; CHECK-NEXT: vpextrd $1, %xmm1, %eax
-; CHECK-NEXT: pextl %eax, %ecx, %eax
-; CHECK-NEXT: vmovd %xmm1, %ecx
-; CHECK-NEXT: pextl %ecx, %edx, %ecx
-; CHECK-NEXT: vmovd %ecx, %xmm3
-; CHECK-NEXT: vpextrd $2, %xmm0, %ecx
-; CHECK-NEXT: vpinsrd $1, %eax, %xmm3, %xmm3
-; CHECK-NEXT: vpextrd $2, %xmm1, %eax
-; CHECK-NEXT: pextl %eax, %ecx, %eax
-; CHECK-NEXT: vpextrd $3, %xmm0, %ecx
-; CHECK-NEXT: vpinsrd $2, %eax, %xmm3, %xmm3
-; CHECK-NEXT: vpextrd $3, %xmm1, %eax
-; CHECK-NEXT: pextl %eax, %ecx, %eax
-; CHECK-NEXT: vpinsrd $3, %eax, %xmm3, %xmm0
-; CHECK-NEXT: vinserti128 $1, %xmm2, %ymm0, %ymm0
-; CHECK-NEXT: retq
+; SLOW-LABEL: pext_v8i32_minsize:
+; SLOW: # %bb.0:
+; SLOW-NEXT: vextracti128 $1, %ymm0, %xmm3
+; SLOW-NEXT: vextracti128 $1, %ymm1, %xmm2
+; SLOW-NEXT: vpextrd $1, %xmm0, %r8d
+; SLOW-NEXT: vmovd %xmm0, %r9d
+; SLOW-NEXT: vpextrd $1, %xmm2, %eax
+; SLOW-NEXT: vpextrd $1, %xmm3, %ecx
+; SLOW-NEXT: vmovd %xmm3, %edx
+; SLOW-NEXT: vpextrd $2, %xmm3, %esi
+; SLOW-NEXT: vpextrd $3, %xmm3, %edi
+; SLOW-NEXT: pextl %eax, %ecx, %eax
+; SLOW-NEXT: vmovd %xmm2, %ecx
+; SLOW-NEXT: pextl %ecx, %edx, %ecx
+; SLOW-NEXT: vpextrd $2, %xmm2, %edx
+; SLOW-NEXT: pextl %edx, %esi, %edx
+; SLOW-NEXT: vpextrd $3, %xmm2, %esi
+; SLOW-NEXT: pextl %esi, %edi, %esi
+; SLOW-NEXT: vpextrd $1, %xmm1, %edi
+; SLOW-NEXT: pextl %edi, %r8d, %edi
+; SLOW-NEXT: vmovd %xmm1, %r8d
+; SLOW-NEXT: pextl %r8d, %r9d, %r8d
+; SLOW-NEXT: vmovd %ecx, %xmm2
+; SLOW-NEXT: vpextrd $2, %xmm0, %ecx
+; SLOW-NEXT: vpinsrd $1, %eax, %xmm2, %xmm2
+; SLOW-NEXT: vpextrd $2, %xmm1, %eax
+; SLOW-NEXT: vpinsrd $2, %edx, %xmm2, %xmm2
+; SLOW-NEXT: vpextrd $3, %xmm0, %edx
+; SLOW-NEXT: pextl %eax, %ecx, %eax
+; SLOW-NEXT: vpextrd $3, %xmm1, %ecx
+; SLOW-NEXT: vpinsrd $3, %esi, %xmm2, %xmm2
+; SLOW-NEXT: pextl %ecx, %edx, %ecx
+; SLOW-NEXT: vmovd %r8d, %xmm3
+; SLOW-NEXT: vpinsrd $1, %edi, %xmm3, %xmm3
+; SLOW-NEXT: vpinsrd $2, %eax, %xmm3, %xmm0
+; SLOW-NEXT: vpinsrd $3, %ecx, %xmm0, %xmm0
+; SLOW-NEXT: vinserti128 $1, %xmm2, %ymm0, %ymm0
+; SLOW-NEXT: retq
+;
+; FAST-LABEL: pext_v8i32_minsize:
+; FAST: # %bb.0:
+; FAST-NEXT: vextracti128 $1, %ymm0, %xmm3
+; FAST-NEXT: vextracti128 $1, %ymm1, %xmm2
+; FAST-NEXT: vpextrd $1, %xmm2, %eax
+; FAST-NEXT: vpextrd $1, %xmm3, %ecx
+; FAST-NEXT: vmovd %xmm3, %edx
+; FAST-NEXT: pextl %eax, %ecx, %eax
+; FAST-NEXT: vmovd %xmm2, %ecx
+; FAST-NEXT: pextl %ecx, %edx, %ecx
+; FAST-NEXT: vmovd %xmm0, %edx
+; FAST-NEXT: vmovd %ecx, %xmm4
+; FAST-NEXT: vpextrd $2, %xmm3, %ecx
+; FAST-NEXT: vpinsrd $1, %eax, %xmm4, %xmm4
+; FAST-NEXT: vpextrd $2, %xmm2, %eax
+; FAST-NEXT: pextl %eax, %ecx, %eax
+; FAST-NEXT: vpextrd $3, %xmm3, %ecx
+; FAST-NEXT: vpinsrd $2, %eax, %xmm4, %xmm4
+; FAST-NEXT: vpextrd $3, %xmm2, %eax
+; FAST-NEXT: pextl %eax, %ecx, %eax
+; FAST-NEXT: vpextrd $1, %xmm0, %ecx
+; FAST-NEXT: vpinsrd $3, %eax, %xmm4, %xmm2
+; FAST-NEXT: vpextrd $1, %xmm1, %eax
+; FAST-NEXT: pextl %eax, %ecx, %eax
+; FAST-NEXT: vmovd %xmm1, %ecx
+; FAST-NEXT: pextl %ecx, %edx, %ecx
+; FAST-NEXT: vmovd %ecx, %xmm3
+; FAST-NEXT: vpextrd $2, %xmm0, %ecx
+; FAST-NEXT: vpinsrd $1, %eax, %xmm3, %xmm3
+; FAST-NEXT: vpextrd $2, %xmm1, %eax
+; FAST-NEXT: pextl %eax, %ecx, %eax
+; FAST-NEXT: vpextrd $3, %xmm0, %ecx
+; FAST-NEXT: vpinsrd $2, %eax, %xmm3, %xmm3
+; FAST-NEXT: vpextrd $3, %xmm1, %eax
+; FAST-NEXT: pextl %eax, %ecx, %eax
+; FAST-NEXT: vpinsrd $3, %eax, %xmm3, %xmm0
+; FAST-NEXT: vinserti128 $1, %xmm2, %ymm0, %ymm0
+; FAST-NEXT: retq
%res = call <8 x i32> @llvm.pext.v8i32(<8 x i32> %val, <8 x i32> %mask)
ret <8 x i32> %res
}
define <2 x i64> @pext_v2i64_minsize(<2 x i64> %val, <2 x i64> %mask) nounwind minsize {
-; CHECK-LABEL: pext_v2i64_minsize:
-; CHECK: # %bb.0:
-; CHECK-NEXT: vpextrq $1, %xmm1, %rax
-; CHECK-NEXT: vpextrq $1, %xmm0, %rcx
-; CHECK-NEXT: vmovq %xmm1, %rdx
-; CHECK-NEXT: pextq %rax, %rcx, %rax
-; CHECK-NEXT: vmovq %xmm0, %rcx
-; CHECK-NEXT: vmovq %rax, %xmm2
-; CHECK-NEXT: pextq %rdx, %rcx, %rax
-; CHECK-NEXT: vmovq %rax, %xmm0
-; CHECK-NEXT: vpunpcklqdq {{.*#+}} xmm0 = xmm0[0],xmm2[0]
-; CHECK-NEXT: retq
+; SLOW-LABEL: pext_v2i64_minsize:
+; SLOW: # %bb.0:
+; SLOW-NEXT: vpextrq $1, %xmm1, %rax
+; SLOW-NEXT: vpextrq $1, %xmm0, %rcx
+; SLOW-NEXT: vmovq %xmm1, %rsi
+; SLOW-NEXT: vmovq %xmm0, %rdx
+; SLOW-NEXT: pextq %rax, %rcx, %rax
+; SLOW-NEXT: pextq %rsi, %rdx, %rcx
+; SLOW-NEXT: vmovq %rax, %xmm0
+; SLOW-NEXT: vmovq %rcx, %xmm1
+; SLOW-NEXT: vpunpcklqdq {{.*#+}} xmm0 = xmm1[0],xmm0[0]
+; SLOW-NEXT: retq
+;
+; FAST-LABEL: pext_v2i64_minsize:
+; FAST: # %bb.0:
+; FAST-NEXT: vpextrq $1, %xmm1, %rax
+; FAST-NEXT: vpextrq $1, %xmm0, %rcx
+; FAST-NEXT: vmovq %xmm1, %rdx
+; FAST-NEXT: pextq %rax, %rcx, %rax
+; FAST-NEXT: vmovq %xmm0, %rcx
+; FAST-NEXT: vmovq %rax, %xmm2
+; FAST-NEXT: pextq %rdx, %rcx, %rax
+; FAST-NEXT: vmovq %rax, %xmm0
+; FAST-NEXT: vpunpcklqdq {{.*#+}} xmm0 = xmm0[0],xmm2[0]
+; FAST-NEXT: retq
%res = call <2 x i64> @llvm.pext.v2i64(<2 x i64> %val, <2 x i64> %mask)
ret <2 x i64> %res
}
define <4 x i64> @pext_v4i64_minsize(<4 x i64> %val, <4 x i64> %mask) nounwind minsize {
-; CHECK-LABEL: pext_v4i64_minsize:
-; CHECK: # %bb.0:
-; CHECK-NEXT: vextracti128 $1, %ymm1, %xmm2
-; CHECK-NEXT: vextracti128 $1, %ymm0, %xmm3
-; CHECK-NEXT: vpextrq $1, %xmm1, %rsi
-; CHECK-NEXT: vpextrq $1, %xmm2, %rax
-; CHECK-NEXT: vpextrq $1, %xmm3, %rcx
-; CHECK-NEXT: vmovq %xmm2, %rdx
-; CHECK-NEXT: pextq %rax, %rcx, %rax
-; CHECK-NEXT: vmovq %xmm3, %rcx
-; CHECK-NEXT: vmovq %rax, %xmm4
-; CHECK-NEXT: pextq %rdx, %rcx, %rax
-; CHECK-NEXT: vpextrq $1, %xmm0, %rcx
-; CHECK-NEXT: vmovq %xmm1, %rdx
-; CHECK-NEXT: vmovq %rax, %xmm2
-; CHECK-NEXT: pextq %rsi, %rcx, %rax
-; CHECK-NEXT: vmovq %xmm0, %rcx
-; CHECK-NEXT: vpunpcklqdq {{.*#+}} xmm2 = xmm2[0],xmm4[0]
-; CHECK-NEXT: vmovq %rax, %xmm3
-; CHECK-NEXT: pextq %rdx, %rcx, %rax
-; CHECK-NEXT: vmovq %rax, %xmm0
-; CHECK-NEXT: vpunpcklqdq {{.*#+}} xmm0 = xmm0[0],xmm3[0]
-; CHECK-NEXT: vinserti128 $1, %xmm2, %ymm0, %ymm0
-; CHECK-NEXT: retq
+; SLOW-LABEL: pext_v4i64_minsize:
+; SLOW: # %bb.0:
+; SLOW-NEXT: vextracti128 $1, %ymm1, %xmm2
+; SLOW-NEXT: vextracti128 $1, %ymm0, %xmm3
+; SLOW-NEXT: vpextrq $1, %xmm2, %rax
+; SLOW-NEXT: vpextrq $1, %xmm3, %rcx
+; SLOW-NEXT: vmovq %xmm2, %rsi
+; SLOW-NEXT: vmovq %xmm3, %rdx
+; SLOW-NEXT: pextq %rax, %rcx, %rax
+; SLOW-NEXT: pextq %rsi, %rdx, %rcx
+; SLOW-NEXT: vpextrq $1, %xmm0, %rdx
+; SLOW-NEXT: vmovq %rax, %xmm2
+; SLOW-NEXT: vpextrq $1, %xmm1, %rax
+; SLOW-NEXT: vmovq %rcx, %xmm3
+; SLOW-NEXT: vmovq %xmm0, %rcx
+; SLOW-NEXT: pextq %rax, %rdx, %rax
+; SLOW-NEXT: vmovq %xmm1, %rdx
+; SLOW-NEXT: vpunpcklqdq {{.*#+}} xmm2 = xmm3[0],xmm2[0]
+; SLOW-NEXT: vmovq %rax, %xmm3
+; SLOW-NEXT: pextq %rdx, %rcx, %rax
+; SLOW-NEXT: vmovq %rax, %xmm0
+; SLOW-NEXT: vpunpcklqdq {{.*#+}} xmm0 = xmm0[0],xmm3[0]
+; SLOW-NEXT: vinserti128 $1, %xmm2, %ymm0, %ymm0
+; SLOW-NEXT: retq
+;
+; FAST-LABEL: pext_v4i64_minsize:
+; FAST: # %bb.0:
+; FAST-NEXT: vextracti128 $1, %ymm1, %xmm2
+; FAST-NEXT: vextracti128 $1, %ymm0, %xmm3
+; FAST-NEXT: vpextrq $1, %xmm1, %rsi
+; FAST-NEXT: vpextrq $1, %xmm2, %rax
+; FAST-NEXT: vpextrq $1, %xmm3, %rcx
+; FAST-NEXT: vmovq %xmm2, %rdx
+; FAST-NEXT: pextq %rax, %rcx, %rax
+; FAST-NEXT: vmovq %xmm3, %rcx
+; FAST-NEXT: vmovq %rax, %xmm4
+; FAST-NEXT: pextq %rdx, %rcx, %rax
+; FAST-NEXT: vpextrq $1, %xmm0, %rcx
+; FAST-NEXT: vmovq %xmm1, %rdx
+; FAST-NEXT: vmovq %rax, %xmm2
+; FAST-NEXT: pextq %rsi, %rcx, %rax
+; FAST-NEXT: vmovq %xmm0, %rcx
+; FAST-NEXT: vpunpcklqdq {{.*#+}} xmm2 = xmm2[0],xmm4[0]
+; FAST-NEXT: vmovq %rax, %xmm3
+; FAST-NEXT: pextq %rdx, %rcx, %rax
+; FAST-NEXT: vmovq %rax, %xmm0
+; FAST-NEXT: vpunpcklqdq {{.*#+}} xmm0 = xmm0[0],xmm3[0]
+; FAST-NEXT: vinserti128 $1, %xmm2, %ymm0, %ymm0
+; FAST-NEXT: retq
%res = call <4 x i64> @llvm.pext.v4i64(<4 x i64> %val, <4 x i64> %mask)
ret <4 x i64> %res
}
+;; NOTE: These prefixes are unused and the list is autogenerated. Do not add tests below this line:
+; CHECK: {{.*}}
More information about the llvm-commits
mailing list