[llvm] [X86] Lower bf16 round-to-integer on AVX10.2 via vrndscalebf16 (PR #213276)
via llvm-commits
llvm-commits at lists.llvm.org
Fri Aug 28 07:50:44 PDT 2026
https://github.com/tfzee updated https://github.com/llvm/llvm-project/pull/213276
>From 92bd9b0d6f0a64b8509574e187754bcb62aa306e Mon Sep 17 00:00:00 2001
From: "Ziegler, Tim" <tim.ziegler at intel.com>
Date: Fri, 31 Jul 2026 14:57:44 +0200
Subject: [PATCH 1/6] Legalizing the common (non strict) rounding bf16
instructions to use vrndscalebf16
---
llvm/lib/Target/X86/X86ISelLowering.cpp | 24 +-
llvm/test/CodeGen/X86/avx10_2bf16-rounding.ll | 1145 +++++++++++++++++
2 files changed, 1164 insertions(+), 5 deletions(-)
create mode 100644 llvm/test/CodeGen/X86/avx10_2bf16-rounding.ll
diff --git a/llvm/lib/Target/X86/X86ISelLowering.cpp b/llvm/lib/Target/X86/X86ISelLowering.cpp
index 712db68314e6e..a11152b7db66e 100644
--- a/llvm/lib/Target/X86/X86ISelLowering.cpp
+++ b/llvm/lib/Target/X86/X86ISelLowering.cpp
@@ -2610,6 +2610,10 @@ X86TargetLowering::X86TargetLowering(const X86TargetMachine &TM,
setOperationAction(ISD::FSQRT, MVT::bf16, Custom);
setOperationAction(ISD::FMA, MVT::bf16, Custom);
+ for (unsigned Opc : {ISD::FFLOOR, ISD::FCEIL, ISD::FTRUNC, ISD::FRINT,
+ ISD::FNEARBYINT, ISD::FROUNDEVEN})
+ setOperationAction(Opc, MVT::bf16, Custom);
+
setOperationAction(ISD::FADD, MVT::v32bf16, Legal);
setOperationAction(ISD::FSUB, MVT::v32bf16, Legal);
setOperationAction(ISD::FMUL, MVT::v32bf16, Legal);
@@ -2618,6 +2622,10 @@ X86TargetLowering::X86TargetLowering(const X86TargetMachine &TM,
setOperationAction(ISD::FMA, MVT::v32bf16, Legal);
setOperationAction(ISD::SETCC, MVT::v32bf16, Custom);
SetFPMinMaxAction(MVT::v32bf16);
+ for (auto VT : {MVT::v8bf16, MVT::v16bf16, MVT::v32bf16})
+ for (unsigned Opc : {ISD::FFLOOR, ISD::FCEIL, ISD::FTRUNC, ISD::FRINT,
+ ISD::FNEARBYINT, ISD::FROUNDEVEN})
+ setOperationAction(Opc, VT, Legal);
for (auto VT : {MVT::v8bf16, MVT::v16bf16}) {
setOperationAction(ISD::FADD, VT, Legal);
setOperationAction(ISD::FSUB, VT, Legal);
@@ -34549,12 +34557,18 @@ void X86TargetLowering::ReplaceNodeResults(SDNode *N,
case ISD::FMUL:
case ISD::FSQRT:
case ISD::FDIV:
- case ISD::FMA: {
+ case ISD::FMA:
+ case ISD::FFLOOR:
+ case ISD::FCEIL:
+ case ISD::FTRUNC:
+ case ISD::FRINT:
+ case ISD::FNEARBYINT:
+ case ISD::FROUNDEVEN: {
assert(N->getValueType(0) == MVT::bf16 && "Expected scalar bf16 result");
- // AVX10.2 has no scalar bf16 arithmetic instructions, and bf16 is a
- // soft-promoted-half type, so scalar ops would otherwise be promoted to
- // f32. Instead widen each operand to a v8bf16 vector, perform the legal
- // packed operation, and extract the low element afterwards.
+ // AVX10.2 has no scalar bf16 arithmetic or round-to-integer instructions,
+ // and bf16 is a soft-promoted-half type, so scalar ops would otherwise be
+ // promoted to f32. Instead widen each operand to a v8bf16 vector, perform
+ // the legal packed operation, and extract the low element afterwards.
SmallVector<SDValue, 3> VecOps;
for (const SDValue &Op : N->ops()) {
SDValue AsF16 = DAG.getBitcast(MVT::f16, Op);
diff --git a/llvm/test/CodeGen/X86/avx10_2bf16-rounding.ll b/llvm/test/CodeGen/X86/avx10_2bf16-rounding.ll
new file mode 100644
index 0000000000000..4b9b764a6f5ce
--- /dev/null
+++ b/llvm/test/CodeGen/X86/avx10_2bf16-rounding.ll
@@ -0,0 +1,1145 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
+; RUN: llc < %s -verify-machineinstrs -mtriple=x86_64-unknown-unknown -mattr=+avx10.2 | FileCheck %s --check-prefixes=AVX10_2
+; RUN: llc < %s -verify-machineinstrs -mtriple=x86_64-unknown-unknown -mattr=+avx512bf16,+avx512vl | FileCheck %s --check-prefixes=AVX512BF16
+
+; AVX10.2 has packed bf16 round-to-integer instructions. The integral result of
+; floor/ceil/trunc/rint/nearbyint/roundeven is always representable in bf16, so
+; vrndscalebf16 gives the same answer as the AVX512BF16 sequence below (promote
+; each element to f32, round, convert back), but without scalarizing.
+
+define <8 x bfloat> @floor_v8bf16(<8 x bfloat> %a) {
+; AVX10_2-LABEL: floor_v8bf16:
+; AVX10_2: # %bb.0:
+; AVX10_2-NEXT: vrndscalebf16 $9, %xmm0, %xmm0
+; AVX10_2-NEXT: retq
+;
+; AVX512BF16-LABEL: floor_v8bf16:
+; AVX512BF16: # %bb.0:
+; AVX512BF16-NEXT: vpextrw $7, %xmm0, %eax
+; AVX512BF16-NEXT: shll $16, %eax
+; AVX512BF16-NEXT: vmovd %eax, %xmm1
+; AVX512BF16-NEXT: vroundss $9, %xmm1, %xmm1, %xmm1
+; AVX512BF16-NEXT: vcvtneps2bf16 %xmm1, %xmm1
+; AVX512BF16-NEXT: vmovd %xmm1, %eax
+; AVX512BF16-NEXT: vpextrw $6, %xmm0, %ecx
+; AVX512BF16-NEXT: shll $16, %ecx
+; AVX512BF16-NEXT: vmovd %ecx, %xmm1
+; AVX512BF16-NEXT: vroundss $9, %xmm1, %xmm1, %xmm1
+; AVX512BF16-NEXT: vcvtneps2bf16 %xmm1, %xmm1
+; AVX512BF16-NEXT: vmovd %xmm1, %ecx
+; AVX512BF16-NEXT: vpextrw $5, %xmm0, %edx
+; AVX512BF16-NEXT: shll $16, %edx
+; AVX512BF16-NEXT: vmovd %edx, %xmm1
+; AVX512BF16-NEXT: vroundss $9, %xmm1, %xmm1, %xmm1
+; AVX512BF16-NEXT: vcvtneps2bf16 %xmm1, %xmm1
+; AVX512BF16-NEXT: vmovd %xmm1, %edx
+; AVX512BF16-NEXT: vpextrw $4, %xmm0, %esi
+; AVX512BF16-NEXT: shll $16, %esi
+; AVX512BF16-NEXT: vmovd %esi, %xmm1
+; AVX512BF16-NEXT: vroundss $9, %xmm1, %xmm1, %xmm1
+; AVX512BF16-NEXT: vcvtneps2bf16 %xmm1, %xmm1
+; AVX512BF16-NEXT: vmovd %xmm1, %esi
+; AVX512BF16-NEXT: vpextrw $3, %xmm0, %edi
+; AVX512BF16-NEXT: shll $16, %edi
+; AVX512BF16-NEXT: vmovd %edi, %xmm1
+; AVX512BF16-NEXT: vroundss $9, %xmm1, %xmm1, %xmm1
+; AVX512BF16-NEXT: vcvtneps2bf16 %xmm1, %xmm1
+; AVX512BF16-NEXT: vmovd %xmm1, %edi
+; AVX512BF16-NEXT: vpextrw $2, %xmm0, %r8d
+; AVX512BF16-NEXT: shll $16, %r8d
+; AVX512BF16-NEXT: vmovd %r8d, %xmm1
+; AVX512BF16-NEXT: vroundss $9, %xmm1, %xmm1, %xmm1
+; AVX512BF16-NEXT: vcvtneps2bf16 %xmm1, %xmm1
+; AVX512BF16-NEXT: vmovd %xmm1, %r8d
+; AVX512BF16-NEXT: vpextrw $1, %xmm0, %r9d
+; AVX512BF16-NEXT: shll $16, %r9d
+; AVX512BF16-NEXT: vmovd %r9d, %xmm1
+; AVX512BF16-NEXT: vroundss $9, %xmm1, %xmm1, %xmm1
+; AVX512BF16-NEXT: vcvtneps2bf16 %xmm1, %xmm1
+; AVX512BF16-NEXT: vmovd %xmm0, %r9d
+; AVX512BF16-NEXT: shll $16, %r9d
+; AVX512BF16-NEXT: vmovd %r9d, %xmm0
+; AVX512BF16-NEXT: vroundss $9, %xmm0, %xmm0, %xmm0
+; AVX512BF16-NEXT: vcvtneps2bf16 %xmm0, %xmm0
+; AVX512BF16-NEXT: vpunpcklwd {{.*#+}} xmm0 = xmm0[0],xmm1[0],xmm0[1],xmm1[1],xmm0[2],xmm1[2],xmm0[3],xmm1[3]
+; AVX512BF16-NEXT: vpinsrw $2, %r8d, %xmm0, %xmm0
+; AVX512BF16-NEXT: vpinsrw $3, %edi, %xmm0, %xmm0
+; AVX512BF16-NEXT: vpinsrw $4, %esi, %xmm0, %xmm0
+; AVX512BF16-NEXT: vpinsrw $5, %edx, %xmm0, %xmm0
+; AVX512BF16-NEXT: vpinsrw $6, %ecx, %xmm0, %xmm0
+; AVX512BF16-NEXT: vpinsrw $7, %eax, %xmm0, %xmm0
+; AVX512BF16-NEXT: retq
+ %r = call <8 x bfloat> @llvm.floor.v8bf16(<8 x bfloat> %a)
+ ret <8 x bfloat> %r
+}
+
+define bfloat @floor_bf16(bfloat %a) {
+; AVX10_2-LABEL: floor_bf16:
+; AVX10_2: # %bb.0:
+; AVX10_2-NEXT: vrndscalebf16 $9, %xmm0, %xmm0
+; AVX10_2-NEXT: retq
+;
+; AVX512BF16-LABEL: floor_bf16:
+; AVX512BF16: # %bb.0:
+; AVX512BF16-NEXT: vpextrw $0, %xmm0, %eax
+; AVX512BF16-NEXT: shll $16, %eax
+; AVX512BF16-NEXT: vmovd %eax, %xmm0
+; AVX512BF16-NEXT: vroundss $9, %xmm0, %xmm0, %xmm0
+; AVX512BF16-NEXT: vcvtneps2bf16 %xmm0, %xmm0
+; AVX512BF16-NEXT: retq
+ %r = call bfloat @llvm.floor.bf16(bfloat %a)
+ ret bfloat %r
+}
+
+define <8 x bfloat> @ceil_v8bf16(<8 x bfloat> %a) {
+; AVX10_2-LABEL: ceil_v8bf16:
+; AVX10_2: # %bb.0:
+; AVX10_2-NEXT: vrndscalebf16 $10, %xmm0, %xmm0
+; AVX10_2-NEXT: retq
+;
+; AVX512BF16-LABEL: ceil_v8bf16:
+; AVX512BF16: # %bb.0:
+; AVX512BF16-NEXT: vpextrw $7, %xmm0, %eax
+; AVX512BF16-NEXT: shll $16, %eax
+; AVX512BF16-NEXT: vmovd %eax, %xmm1
+; AVX512BF16-NEXT: vroundss $10, %xmm1, %xmm1, %xmm1
+; AVX512BF16-NEXT: vcvtneps2bf16 %xmm1, %xmm1
+; AVX512BF16-NEXT: vmovd %xmm1, %eax
+; AVX512BF16-NEXT: vpextrw $6, %xmm0, %ecx
+; AVX512BF16-NEXT: shll $16, %ecx
+; AVX512BF16-NEXT: vmovd %ecx, %xmm1
+; AVX512BF16-NEXT: vroundss $10, %xmm1, %xmm1, %xmm1
+; AVX512BF16-NEXT: vcvtneps2bf16 %xmm1, %xmm1
+; AVX512BF16-NEXT: vmovd %xmm1, %ecx
+; AVX512BF16-NEXT: vpextrw $5, %xmm0, %edx
+; AVX512BF16-NEXT: shll $16, %edx
+; AVX512BF16-NEXT: vmovd %edx, %xmm1
+; AVX512BF16-NEXT: vroundss $10, %xmm1, %xmm1, %xmm1
+; AVX512BF16-NEXT: vcvtneps2bf16 %xmm1, %xmm1
+; AVX512BF16-NEXT: vmovd %xmm1, %edx
+; AVX512BF16-NEXT: vpextrw $4, %xmm0, %esi
+; AVX512BF16-NEXT: shll $16, %esi
+; AVX512BF16-NEXT: vmovd %esi, %xmm1
+; AVX512BF16-NEXT: vroundss $10, %xmm1, %xmm1, %xmm1
+; AVX512BF16-NEXT: vcvtneps2bf16 %xmm1, %xmm1
+; AVX512BF16-NEXT: vmovd %xmm1, %esi
+; AVX512BF16-NEXT: vpextrw $3, %xmm0, %edi
+; AVX512BF16-NEXT: shll $16, %edi
+; AVX512BF16-NEXT: vmovd %edi, %xmm1
+; AVX512BF16-NEXT: vroundss $10, %xmm1, %xmm1, %xmm1
+; AVX512BF16-NEXT: vcvtneps2bf16 %xmm1, %xmm1
+; AVX512BF16-NEXT: vmovd %xmm1, %edi
+; AVX512BF16-NEXT: vpextrw $2, %xmm0, %r8d
+; AVX512BF16-NEXT: shll $16, %r8d
+; AVX512BF16-NEXT: vmovd %r8d, %xmm1
+; AVX512BF16-NEXT: vroundss $10, %xmm1, %xmm1, %xmm1
+; AVX512BF16-NEXT: vcvtneps2bf16 %xmm1, %xmm1
+; AVX512BF16-NEXT: vmovd %xmm1, %r8d
+; AVX512BF16-NEXT: vpextrw $1, %xmm0, %r9d
+; AVX512BF16-NEXT: shll $16, %r9d
+; AVX512BF16-NEXT: vmovd %r9d, %xmm1
+; AVX512BF16-NEXT: vroundss $10, %xmm1, %xmm1, %xmm1
+; AVX512BF16-NEXT: vcvtneps2bf16 %xmm1, %xmm1
+; AVX512BF16-NEXT: vmovd %xmm0, %r9d
+; AVX512BF16-NEXT: shll $16, %r9d
+; AVX512BF16-NEXT: vmovd %r9d, %xmm0
+; AVX512BF16-NEXT: vroundss $10, %xmm0, %xmm0, %xmm0
+; AVX512BF16-NEXT: vcvtneps2bf16 %xmm0, %xmm0
+; AVX512BF16-NEXT: vpunpcklwd {{.*#+}} xmm0 = xmm0[0],xmm1[0],xmm0[1],xmm1[1],xmm0[2],xmm1[2],xmm0[3],xmm1[3]
+; AVX512BF16-NEXT: vpinsrw $2, %r8d, %xmm0, %xmm0
+; AVX512BF16-NEXT: vpinsrw $3, %edi, %xmm0, %xmm0
+; AVX512BF16-NEXT: vpinsrw $4, %esi, %xmm0, %xmm0
+; AVX512BF16-NEXT: vpinsrw $5, %edx, %xmm0, %xmm0
+; AVX512BF16-NEXT: vpinsrw $6, %ecx, %xmm0, %xmm0
+; AVX512BF16-NEXT: vpinsrw $7, %eax, %xmm0, %xmm0
+; AVX512BF16-NEXT: retq
+ %r = call <8 x bfloat> @llvm.ceil.v8bf16(<8 x bfloat> %a)
+ ret <8 x bfloat> %r
+}
+
+define bfloat @ceil_bf16(bfloat %a) {
+; AVX10_2-LABEL: ceil_bf16:
+; AVX10_2: # %bb.0:
+; AVX10_2-NEXT: vrndscalebf16 $10, %xmm0, %xmm0
+; AVX10_2-NEXT: retq
+;
+; AVX512BF16-LABEL: ceil_bf16:
+; AVX512BF16: # %bb.0:
+; AVX512BF16-NEXT: vpextrw $0, %xmm0, %eax
+; AVX512BF16-NEXT: shll $16, %eax
+; AVX512BF16-NEXT: vmovd %eax, %xmm0
+; AVX512BF16-NEXT: vroundss $10, %xmm0, %xmm0, %xmm0
+; AVX512BF16-NEXT: vcvtneps2bf16 %xmm0, %xmm0
+; AVX512BF16-NEXT: retq
+ %r = call bfloat @llvm.ceil.bf16(bfloat %a)
+ ret bfloat %r
+}
+
+define <8 x bfloat> @trunc_v8bf16(<8 x bfloat> %a) {
+; AVX10_2-LABEL: trunc_v8bf16:
+; AVX10_2: # %bb.0:
+; AVX10_2-NEXT: vrndscalebf16 $11, %xmm0, %xmm0
+; AVX10_2-NEXT: retq
+;
+; AVX512BF16-LABEL: trunc_v8bf16:
+; AVX512BF16: # %bb.0:
+; AVX512BF16-NEXT: vpextrw $7, %xmm0, %eax
+; AVX512BF16-NEXT: shll $16, %eax
+; AVX512BF16-NEXT: vmovd %eax, %xmm1
+; AVX512BF16-NEXT: vroundss $11, %xmm1, %xmm1, %xmm1
+; AVX512BF16-NEXT: vcvtneps2bf16 %xmm1, %xmm1
+; AVX512BF16-NEXT: vmovd %xmm1, %eax
+; AVX512BF16-NEXT: vpextrw $6, %xmm0, %ecx
+; AVX512BF16-NEXT: shll $16, %ecx
+; AVX512BF16-NEXT: vmovd %ecx, %xmm1
+; AVX512BF16-NEXT: vroundss $11, %xmm1, %xmm1, %xmm1
+; AVX512BF16-NEXT: vcvtneps2bf16 %xmm1, %xmm1
+; AVX512BF16-NEXT: vmovd %xmm1, %ecx
+; AVX512BF16-NEXT: vpextrw $5, %xmm0, %edx
+; AVX512BF16-NEXT: shll $16, %edx
+; AVX512BF16-NEXT: vmovd %edx, %xmm1
+; AVX512BF16-NEXT: vroundss $11, %xmm1, %xmm1, %xmm1
+; AVX512BF16-NEXT: vcvtneps2bf16 %xmm1, %xmm1
+; AVX512BF16-NEXT: vmovd %xmm1, %edx
+; AVX512BF16-NEXT: vpextrw $4, %xmm0, %esi
+; AVX512BF16-NEXT: shll $16, %esi
+; AVX512BF16-NEXT: vmovd %esi, %xmm1
+; AVX512BF16-NEXT: vroundss $11, %xmm1, %xmm1, %xmm1
+; AVX512BF16-NEXT: vcvtneps2bf16 %xmm1, %xmm1
+; AVX512BF16-NEXT: vmovd %xmm1, %esi
+; AVX512BF16-NEXT: vpextrw $3, %xmm0, %edi
+; AVX512BF16-NEXT: shll $16, %edi
+; AVX512BF16-NEXT: vmovd %edi, %xmm1
+; AVX512BF16-NEXT: vroundss $11, %xmm1, %xmm1, %xmm1
+; AVX512BF16-NEXT: vcvtneps2bf16 %xmm1, %xmm1
+; AVX512BF16-NEXT: vmovd %xmm1, %edi
+; AVX512BF16-NEXT: vpextrw $2, %xmm0, %r8d
+; AVX512BF16-NEXT: shll $16, %r8d
+; AVX512BF16-NEXT: vmovd %r8d, %xmm1
+; AVX512BF16-NEXT: vroundss $11, %xmm1, %xmm1, %xmm1
+; AVX512BF16-NEXT: vcvtneps2bf16 %xmm1, %xmm1
+; AVX512BF16-NEXT: vmovd %xmm1, %r8d
+; AVX512BF16-NEXT: vpextrw $1, %xmm0, %r9d
+; AVX512BF16-NEXT: shll $16, %r9d
+; AVX512BF16-NEXT: vmovd %r9d, %xmm1
+; AVX512BF16-NEXT: vroundss $11, %xmm1, %xmm1, %xmm1
+; AVX512BF16-NEXT: vcvtneps2bf16 %xmm1, %xmm1
+; AVX512BF16-NEXT: vmovd %xmm0, %r9d
+; AVX512BF16-NEXT: shll $16, %r9d
+; AVX512BF16-NEXT: vmovd %r9d, %xmm0
+; AVX512BF16-NEXT: vroundss $11, %xmm0, %xmm0, %xmm0
+; AVX512BF16-NEXT: vcvtneps2bf16 %xmm0, %xmm0
+; AVX512BF16-NEXT: vpunpcklwd {{.*#+}} xmm0 = xmm0[0],xmm1[0],xmm0[1],xmm1[1],xmm0[2],xmm1[2],xmm0[3],xmm1[3]
+; AVX512BF16-NEXT: vpinsrw $2, %r8d, %xmm0, %xmm0
+; AVX512BF16-NEXT: vpinsrw $3, %edi, %xmm0, %xmm0
+; AVX512BF16-NEXT: vpinsrw $4, %esi, %xmm0, %xmm0
+; AVX512BF16-NEXT: vpinsrw $5, %edx, %xmm0, %xmm0
+; AVX512BF16-NEXT: vpinsrw $6, %ecx, %xmm0, %xmm0
+; AVX512BF16-NEXT: vpinsrw $7, %eax, %xmm0, %xmm0
+; AVX512BF16-NEXT: retq
+ %r = call <8 x bfloat> @llvm.trunc.v8bf16(<8 x bfloat> %a)
+ ret <8 x bfloat> %r
+}
+
+define bfloat @trunc_bf16(bfloat %a) {
+; AVX10_2-LABEL: trunc_bf16:
+; AVX10_2: # %bb.0:
+; AVX10_2-NEXT: vrndscalebf16 $11, %xmm0, %xmm0
+; AVX10_2-NEXT: retq
+;
+; AVX512BF16-LABEL: trunc_bf16:
+; AVX512BF16: # %bb.0:
+; AVX512BF16-NEXT: vpextrw $0, %xmm0, %eax
+; AVX512BF16-NEXT: shll $16, %eax
+; AVX512BF16-NEXT: vmovd %eax, %xmm0
+; AVX512BF16-NEXT: vroundss $11, %xmm0, %xmm0, %xmm0
+; AVX512BF16-NEXT: vcvtneps2bf16 %xmm0, %xmm0
+; AVX512BF16-NEXT: retq
+ %r = call bfloat @llvm.trunc.bf16(bfloat %a)
+ ret bfloat %r
+}
+
+define <8 x bfloat> @rint_v8bf16(<8 x bfloat> %a) {
+; AVX10_2-LABEL: rint_v8bf16:
+; AVX10_2: # %bb.0:
+; AVX10_2-NEXT: vrndscalebf16 $4, %xmm0, %xmm0
+; AVX10_2-NEXT: retq
+;
+; AVX512BF16-LABEL: rint_v8bf16:
+; AVX512BF16: # %bb.0:
+; AVX512BF16-NEXT: vpextrw $7, %xmm0, %eax
+; AVX512BF16-NEXT: shll $16, %eax
+; AVX512BF16-NEXT: vmovd %eax, %xmm1
+; AVX512BF16-NEXT: vroundss $4, %xmm1, %xmm1, %xmm1
+; AVX512BF16-NEXT: vcvtneps2bf16 %xmm1, %xmm1
+; AVX512BF16-NEXT: vmovd %xmm1, %eax
+; AVX512BF16-NEXT: vpextrw $6, %xmm0, %ecx
+; AVX512BF16-NEXT: shll $16, %ecx
+; AVX512BF16-NEXT: vmovd %ecx, %xmm1
+; AVX512BF16-NEXT: vroundss $4, %xmm1, %xmm1, %xmm1
+; AVX512BF16-NEXT: vcvtneps2bf16 %xmm1, %xmm1
+; AVX512BF16-NEXT: vmovd %xmm1, %ecx
+; AVX512BF16-NEXT: vpextrw $5, %xmm0, %edx
+; AVX512BF16-NEXT: shll $16, %edx
+; AVX512BF16-NEXT: vmovd %edx, %xmm1
+; AVX512BF16-NEXT: vroundss $4, %xmm1, %xmm1, %xmm1
+; AVX512BF16-NEXT: vcvtneps2bf16 %xmm1, %xmm1
+; AVX512BF16-NEXT: vmovd %xmm1, %edx
+; AVX512BF16-NEXT: vpextrw $4, %xmm0, %esi
+; AVX512BF16-NEXT: shll $16, %esi
+; AVX512BF16-NEXT: vmovd %esi, %xmm1
+; AVX512BF16-NEXT: vroundss $4, %xmm1, %xmm1, %xmm1
+; AVX512BF16-NEXT: vcvtneps2bf16 %xmm1, %xmm1
+; AVX512BF16-NEXT: vmovd %xmm1, %esi
+; AVX512BF16-NEXT: vpextrw $3, %xmm0, %edi
+; AVX512BF16-NEXT: shll $16, %edi
+; AVX512BF16-NEXT: vmovd %edi, %xmm1
+; AVX512BF16-NEXT: vroundss $4, %xmm1, %xmm1, %xmm1
+; AVX512BF16-NEXT: vcvtneps2bf16 %xmm1, %xmm1
+; AVX512BF16-NEXT: vmovd %xmm1, %edi
+; AVX512BF16-NEXT: vpextrw $2, %xmm0, %r8d
+; AVX512BF16-NEXT: shll $16, %r8d
+; AVX512BF16-NEXT: vmovd %r8d, %xmm1
+; AVX512BF16-NEXT: vroundss $4, %xmm1, %xmm1, %xmm1
+; AVX512BF16-NEXT: vcvtneps2bf16 %xmm1, %xmm1
+; AVX512BF16-NEXT: vmovd %xmm1, %r8d
+; AVX512BF16-NEXT: vpextrw $1, %xmm0, %r9d
+; AVX512BF16-NEXT: shll $16, %r9d
+; AVX512BF16-NEXT: vmovd %r9d, %xmm1
+; AVX512BF16-NEXT: vroundss $4, %xmm1, %xmm1, %xmm1
+; AVX512BF16-NEXT: vcvtneps2bf16 %xmm1, %xmm1
+; AVX512BF16-NEXT: vmovd %xmm0, %r9d
+; AVX512BF16-NEXT: shll $16, %r9d
+; AVX512BF16-NEXT: vmovd %r9d, %xmm0
+; AVX512BF16-NEXT: vroundss $4, %xmm0, %xmm0, %xmm0
+; AVX512BF16-NEXT: vcvtneps2bf16 %xmm0, %xmm0
+; AVX512BF16-NEXT: vpunpcklwd {{.*#+}} xmm0 = xmm0[0],xmm1[0],xmm0[1],xmm1[1],xmm0[2],xmm1[2],xmm0[3],xmm1[3]
+; AVX512BF16-NEXT: vpinsrw $2, %r8d, %xmm0, %xmm0
+; AVX512BF16-NEXT: vpinsrw $3, %edi, %xmm0, %xmm0
+; AVX512BF16-NEXT: vpinsrw $4, %esi, %xmm0, %xmm0
+; AVX512BF16-NEXT: vpinsrw $5, %edx, %xmm0, %xmm0
+; AVX512BF16-NEXT: vpinsrw $6, %ecx, %xmm0, %xmm0
+; AVX512BF16-NEXT: vpinsrw $7, %eax, %xmm0, %xmm0
+; AVX512BF16-NEXT: retq
+ %r = call <8 x bfloat> @llvm.rint.v8bf16(<8 x bfloat> %a)
+ ret <8 x bfloat> %r
+}
+
+define bfloat @rint_bf16(bfloat %a) {
+; AVX10_2-LABEL: rint_bf16:
+; AVX10_2: # %bb.0:
+; AVX10_2-NEXT: vrndscalebf16 $4, %xmm0, %xmm0
+; AVX10_2-NEXT: retq
+;
+; AVX512BF16-LABEL: rint_bf16:
+; AVX512BF16: # %bb.0:
+; AVX512BF16-NEXT: vpextrw $0, %xmm0, %eax
+; AVX512BF16-NEXT: shll $16, %eax
+; AVX512BF16-NEXT: vmovd %eax, %xmm0
+; AVX512BF16-NEXT: vroundss $4, %xmm0, %xmm0, %xmm0
+; AVX512BF16-NEXT: vcvtneps2bf16 %xmm0, %xmm0
+; AVX512BF16-NEXT: retq
+ %r = call bfloat @llvm.rint.bf16(bfloat %a)
+ ret bfloat %r
+}
+
+define <8 x bfloat> @nearbyint_v8bf16(<8 x bfloat> %a) {
+; AVX10_2-LABEL: nearbyint_v8bf16:
+; AVX10_2: # %bb.0:
+; AVX10_2-NEXT: vrndscalebf16 $12, %xmm0, %xmm0
+; AVX10_2-NEXT: retq
+;
+; AVX512BF16-LABEL: nearbyint_v8bf16:
+; AVX512BF16: # %bb.0:
+; AVX512BF16-NEXT: vpextrw $7, %xmm0, %eax
+; AVX512BF16-NEXT: shll $16, %eax
+; AVX512BF16-NEXT: vmovd %eax, %xmm1
+; AVX512BF16-NEXT: vroundss $12, %xmm1, %xmm1, %xmm1
+; AVX512BF16-NEXT: vcvtneps2bf16 %xmm1, %xmm1
+; AVX512BF16-NEXT: vmovd %xmm1, %eax
+; AVX512BF16-NEXT: vpextrw $6, %xmm0, %ecx
+; AVX512BF16-NEXT: shll $16, %ecx
+; AVX512BF16-NEXT: vmovd %ecx, %xmm1
+; AVX512BF16-NEXT: vroundss $12, %xmm1, %xmm1, %xmm1
+; AVX512BF16-NEXT: vcvtneps2bf16 %xmm1, %xmm1
+; AVX512BF16-NEXT: vmovd %xmm1, %ecx
+; AVX512BF16-NEXT: vpextrw $5, %xmm0, %edx
+; AVX512BF16-NEXT: shll $16, %edx
+; AVX512BF16-NEXT: vmovd %edx, %xmm1
+; AVX512BF16-NEXT: vroundss $12, %xmm1, %xmm1, %xmm1
+; AVX512BF16-NEXT: vcvtneps2bf16 %xmm1, %xmm1
+; AVX512BF16-NEXT: vmovd %xmm1, %edx
+; AVX512BF16-NEXT: vpextrw $4, %xmm0, %esi
+; AVX512BF16-NEXT: shll $16, %esi
+; AVX512BF16-NEXT: vmovd %esi, %xmm1
+; AVX512BF16-NEXT: vroundss $12, %xmm1, %xmm1, %xmm1
+; AVX512BF16-NEXT: vcvtneps2bf16 %xmm1, %xmm1
+; AVX512BF16-NEXT: vmovd %xmm1, %esi
+; AVX512BF16-NEXT: vpextrw $3, %xmm0, %edi
+; AVX512BF16-NEXT: shll $16, %edi
+; AVX512BF16-NEXT: vmovd %edi, %xmm1
+; AVX512BF16-NEXT: vroundss $12, %xmm1, %xmm1, %xmm1
+; AVX512BF16-NEXT: vcvtneps2bf16 %xmm1, %xmm1
+; AVX512BF16-NEXT: vmovd %xmm1, %edi
+; AVX512BF16-NEXT: vpextrw $2, %xmm0, %r8d
+; AVX512BF16-NEXT: shll $16, %r8d
+; AVX512BF16-NEXT: vmovd %r8d, %xmm1
+; AVX512BF16-NEXT: vroundss $12, %xmm1, %xmm1, %xmm1
+; AVX512BF16-NEXT: vcvtneps2bf16 %xmm1, %xmm1
+; AVX512BF16-NEXT: vmovd %xmm1, %r8d
+; AVX512BF16-NEXT: vpextrw $1, %xmm0, %r9d
+; AVX512BF16-NEXT: shll $16, %r9d
+; AVX512BF16-NEXT: vmovd %r9d, %xmm1
+; AVX512BF16-NEXT: vroundss $12, %xmm1, %xmm1, %xmm1
+; AVX512BF16-NEXT: vcvtneps2bf16 %xmm1, %xmm1
+; AVX512BF16-NEXT: vmovd %xmm0, %r9d
+; AVX512BF16-NEXT: shll $16, %r9d
+; AVX512BF16-NEXT: vmovd %r9d, %xmm0
+; AVX512BF16-NEXT: vroundss $12, %xmm0, %xmm0, %xmm0
+; AVX512BF16-NEXT: vcvtneps2bf16 %xmm0, %xmm0
+; AVX512BF16-NEXT: vpunpcklwd {{.*#+}} xmm0 = xmm0[0],xmm1[0],xmm0[1],xmm1[1],xmm0[2],xmm1[2],xmm0[3],xmm1[3]
+; AVX512BF16-NEXT: vpinsrw $2, %r8d, %xmm0, %xmm0
+; AVX512BF16-NEXT: vpinsrw $3, %edi, %xmm0, %xmm0
+; AVX512BF16-NEXT: vpinsrw $4, %esi, %xmm0, %xmm0
+; AVX512BF16-NEXT: vpinsrw $5, %edx, %xmm0, %xmm0
+; AVX512BF16-NEXT: vpinsrw $6, %ecx, %xmm0, %xmm0
+; AVX512BF16-NEXT: vpinsrw $7, %eax, %xmm0, %xmm0
+; AVX512BF16-NEXT: retq
+ %r = call <8 x bfloat> @llvm.nearbyint.v8bf16(<8 x bfloat> %a)
+ ret <8 x bfloat> %r
+}
+
+define bfloat @nearbyint_bf16(bfloat %a) {
+; AVX10_2-LABEL: nearbyint_bf16:
+; AVX10_2: # %bb.0:
+; AVX10_2-NEXT: vrndscalebf16 $12, %xmm0, %xmm0
+; AVX10_2-NEXT: retq
+;
+; AVX512BF16-LABEL: nearbyint_bf16:
+; AVX512BF16: # %bb.0:
+; AVX512BF16-NEXT: vpextrw $0, %xmm0, %eax
+; AVX512BF16-NEXT: shll $16, %eax
+; AVX512BF16-NEXT: vmovd %eax, %xmm0
+; AVX512BF16-NEXT: vroundss $12, %xmm0, %xmm0, %xmm0
+; AVX512BF16-NEXT: vcvtneps2bf16 %xmm0, %xmm0
+; AVX512BF16-NEXT: retq
+ %r = call bfloat @llvm.nearbyint.bf16(bfloat %a)
+ ret bfloat %r
+}
+
+define <8 x bfloat> @roundeven_v8bf16(<8 x bfloat> %a) {
+; AVX10_2-LABEL: roundeven_v8bf16:
+; AVX10_2: # %bb.0:
+; AVX10_2-NEXT: vrndscalebf16 $8, %xmm0, %xmm0
+; AVX10_2-NEXT: retq
+;
+; AVX512BF16-LABEL: roundeven_v8bf16:
+; AVX512BF16: # %bb.0:
+; AVX512BF16-NEXT: vpextrw $7, %xmm0, %eax
+; AVX512BF16-NEXT: shll $16, %eax
+; AVX512BF16-NEXT: vmovd %eax, %xmm1
+; AVX512BF16-NEXT: vroundss $8, %xmm1, %xmm1, %xmm1
+; AVX512BF16-NEXT: vcvtneps2bf16 %xmm1, %xmm1
+; AVX512BF16-NEXT: vmovd %xmm1, %eax
+; AVX512BF16-NEXT: vpextrw $6, %xmm0, %ecx
+; AVX512BF16-NEXT: shll $16, %ecx
+; AVX512BF16-NEXT: vmovd %ecx, %xmm1
+; AVX512BF16-NEXT: vroundss $8, %xmm1, %xmm1, %xmm1
+; AVX512BF16-NEXT: vcvtneps2bf16 %xmm1, %xmm1
+; AVX512BF16-NEXT: vmovd %xmm1, %ecx
+; AVX512BF16-NEXT: vpextrw $5, %xmm0, %edx
+; AVX512BF16-NEXT: shll $16, %edx
+; AVX512BF16-NEXT: vmovd %edx, %xmm1
+; AVX512BF16-NEXT: vroundss $8, %xmm1, %xmm1, %xmm1
+; AVX512BF16-NEXT: vcvtneps2bf16 %xmm1, %xmm1
+; AVX512BF16-NEXT: vmovd %xmm1, %edx
+; AVX512BF16-NEXT: vpextrw $4, %xmm0, %esi
+; AVX512BF16-NEXT: shll $16, %esi
+; AVX512BF16-NEXT: vmovd %esi, %xmm1
+; AVX512BF16-NEXT: vroundss $8, %xmm1, %xmm1, %xmm1
+; AVX512BF16-NEXT: vcvtneps2bf16 %xmm1, %xmm1
+; AVX512BF16-NEXT: vmovd %xmm1, %esi
+; AVX512BF16-NEXT: vpextrw $3, %xmm0, %edi
+; AVX512BF16-NEXT: shll $16, %edi
+; AVX512BF16-NEXT: vmovd %edi, %xmm1
+; AVX512BF16-NEXT: vroundss $8, %xmm1, %xmm1, %xmm1
+; AVX512BF16-NEXT: vcvtneps2bf16 %xmm1, %xmm1
+; AVX512BF16-NEXT: vmovd %xmm1, %edi
+; AVX512BF16-NEXT: vpextrw $2, %xmm0, %r8d
+; AVX512BF16-NEXT: shll $16, %r8d
+; AVX512BF16-NEXT: vmovd %r8d, %xmm1
+; AVX512BF16-NEXT: vroundss $8, %xmm1, %xmm1, %xmm1
+; AVX512BF16-NEXT: vcvtneps2bf16 %xmm1, %xmm1
+; AVX512BF16-NEXT: vmovd %xmm1, %r8d
+; AVX512BF16-NEXT: vpextrw $1, %xmm0, %r9d
+; AVX512BF16-NEXT: shll $16, %r9d
+; AVX512BF16-NEXT: vmovd %r9d, %xmm1
+; AVX512BF16-NEXT: vroundss $8, %xmm1, %xmm1, %xmm1
+; AVX512BF16-NEXT: vcvtneps2bf16 %xmm1, %xmm1
+; AVX512BF16-NEXT: vmovd %xmm0, %r9d
+; AVX512BF16-NEXT: shll $16, %r9d
+; AVX512BF16-NEXT: vmovd %r9d, %xmm0
+; AVX512BF16-NEXT: vroundss $8, %xmm0, %xmm0, %xmm0
+; AVX512BF16-NEXT: vcvtneps2bf16 %xmm0, %xmm0
+; AVX512BF16-NEXT: vpunpcklwd {{.*#+}} xmm0 = xmm0[0],xmm1[0],xmm0[1],xmm1[1],xmm0[2],xmm1[2],xmm0[3],xmm1[3]
+; AVX512BF16-NEXT: vpinsrw $2, %r8d, %xmm0, %xmm0
+; AVX512BF16-NEXT: vpinsrw $3, %edi, %xmm0, %xmm0
+; AVX512BF16-NEXT: vpinsrw $4, %esi, %xmm0, %xmm0
+; AVX512BF16-NEXT: vpinsrw $5, %edx, %xmm0, %xmm0
+; AVX512BF16-NEXT: vpinsrw $6, %ecx, %xmm0, %xmm0
+; AVX512BF16-NEXT: vpinsrw $7, %eax, %xmm0, %xmm0
+; AVX512BF16-NEXT: retq
+ %r = call <8 x bfloat> @llvm.roundeven.v8bf16(<8 x bfloat> %a)
+ ret <8 x bfloat> %r
+}
+
+define bfloat @roundeven_bf16(bfloat %a) {
+; AVX10_2-LABEL: roundeven_bf16:
+; AVX10_2: # %bb.0:
+; AVX10_2-NEXT: vrndscalebf16 $8, %xmm0, %xmm0
+; AVX10_2-NEXT: retq
+;
+; AVX512BF16-LABEL: roundeven_bf16:
+; AVX512BF16: # %bb.0:
+; AVX512BF16-NEXT: vpextrw $0, %xmm0, %eax
+; AVX512BF16-NEXT: shll $16, %eax
+; AVX512BF16-NEXT: vmovd %eax, %xmm0
+; AVX512BF16-NEXT: vroundss $8, %xmm0, %xmm0, %xmm0
+; AVX512BF16-NEXT: vcvtneps2bf16 %xmm0, %xmm0
+; AVX512BF16-NEXT: retq
+ %r = call bfloat @llvm.roundeven.bf16(bfloat %a)
+ ret bfloat %r
+}
+
+; ymm and zmm come from separate setOperationAction entries than the xmm ones.
+define <16 x bfloat> @floor_v16bf16(<16 x bfloat> %a) {
+; AVX10_2-LABEL: floor_v16bf16:
+; AVX10_2: # %bb.0:
+; AVX10_2-NEXT: vrndscalebf16 $9, %ymm0, %ymm0
+; AVX10_2-NEXT: retq
+;
+; AVX512BF16-LABEL: floor_v16bf16:
+; AVX512BF16: # %bb.0:
+; AVX512BF16-NEXT: pushq %rbp
+; AVX512BF16-NEXT: .cfi_def_cfa_offset 16
+; AVX512BF16-NEXT: pushq %r15
+; AVX512BF16-NEXT: .cfi_def_cfa_offset 24
+; AVX512BF16-NEXT: pushq %r14
+; AVX512BF16-NEXT: .cfi_def_cfa_offset 32
+; AVX512BF16-NEXT: pushq %r12
+; AVX512BF16-NEXT: .cfi_def_cfa_offset 40
+; AVX512BF16-NEXT: pushq %rbx
+; AVX512BF16-NEXT: .cfi_def_cfa_offset 48
+; AVX512BF16-NEXT: .cfi_offset %rbx, -48
+; AVX512BF16-NEXT: .cfi_offset %r12, -40
+; AVX512BF16-NEXT: .cfi_offset %r14, -32
+; AVX512BF16-NEXT: .cfi_offset %r15, -24
+; AVX512BF16-NEXT: .cfi_offset %rbp, -16
+; AVX512BF16-NEXT: vpextrw $7, %xmm0, %eax
+; AVX512BF16-NEXT: shll $16, %eax
+; AVX512BF16-NEXT: vmovd %eax, %xmm1
+; AVX512BF16-NEXT: vroundss $9, %xmm1, %xmm1, %xmm1
+; AVX512BF16-NEXT: vcvtneps2bf16 %xmm1, %xmm1
+; AVX512BF16-NEXT: vmovd %xmm1, %eax
+; AVX512BF16-NEXT: vpextrw $6, %xmm0, %ecx
+; AVX512BF16-NEXT: shll $16, %ecx
+; AVX512BF16-NEXT: vmovd %ecx, %xmm1
+; AVX512BF16-NEXT: vroundss $9, %xmm1, %xmm1, %xmm1
+; AVX512BF16-NEXT: vcvtneps2bf16 %xmm1, %xmm1
+; AVX512BF16-NEXT: vmovd %xmm1, %ecx
+; AVX512BF16-NEXT: vpextrw $5, %xmm0, %edx
+; AVX512BF16-NEXT: shll $16, %edx
+; AVX512BF16-NEXT: vmovd %edx, %xmm1
+; AVX512BF16-NEXT: vroundss $9, %xmm1, %xmm1, %xmm1
+; AVX512BF16-NEXT: vcvtneps2bf16 %xmm1, %xmm1
+; AVX512BF16-NEXT: vmovd %xmm1, %edx
+; AVX512BF16-NEXT: vpextrw $4, %xmm0, %esi
+; AVX512BF16-NEXT: shll $16, %esi
+; AVX512BF16-NEXT: vmovd %esi, %xmm1
+; AVX512BF16-NEXT: vroundss $9, %xmm1, %xmm1, %xmm1
+; AVX512BF16-NEXT: vcvtneps2bf16 %xmm1, %xmm1
+; AVX512BF16-NEXT: vmovd %xmm1, %esi
+; AVX512BF16-NEXT: vpextrw $3, %xmm0, %edi
+; AVX512BF16-NEXT: shll $16, %edi
+; AVX512BF16-NEXT: vmovd %edi, %xmm1
+; AVX512BF16-NEXT: vroundss $9, %xmm1, %xmm1, %xmm1
+; AVX512BF16-NEXT: vcvtneps2bf16 %xmm1, %xmm1
+; AVX512BF16-NEXT: vmovd %xmm1, %edi
+; AVX512BF16-NEXT: vpextrw $2, %xmm0, %r8d
+; AVX512BF16-NEXT: shll $16, %r8d
+; AVX512BF16-NEXT: vmovd %r8d, %xmm1
+; AVX512BF16-NEXT: vroundss $9, %xmm1, %xmm1, %xmm1
+; AVX512BF16-NEXT: vcvtneps2bf16 %xmm1, %xmm1
+; AVX512BF16-NEXT: vmovd %xmm1, %r8d
+; AVX512BF16-NEXT: vmovd %xmm0, %r9d
+; AVX512BF16-NEXT: shll $16, %r9d
+; AVX512BF16-NEXT: vmovd %r9d, %xmm1
+; AVX512BF16-NEXT: vroundss $9, %xmm1, %xmm1, %xmm1
+; AVX512BF16-NEXT: vcvtneps2bf16 %xmm1, %xmm1
+; AVX512BF16-NEXT: vpextrw $1, %xmm0, %r9d
+; AVX512BF16-NEXT: shll $16, %r9d
+; AVX512BF16-NEXT: vmovd %r9d, %xmm2
+; AVX512BF16-NEXT: vroundss $9, %xmm2, %xmm2, %xmm2
+; AVX512BF16-NEXT: vcvtneps2bf16 %xmm2, %xmm2
+; AVX512BF16-NEXT: vmovd %xmm2, %r9d
+; AVX512BF16-NEXT: vextracti128 $1, %ymm0, %xmm0
+; AVX512BF16-NEXT: vpextrw $7, %xmm0, %r10d
+; AVX512BF16-NEXT: shll $16, %r10d
+; AVX512BF16-NEXT: vmovd %r10d, %xmm2
+; AVX512BF16-NEXT: vroundss $9, %xmm2, %xmm2, %xmm2
+; AVX512BF16-NEXT: vcvtneps2bf16 %xmm2, %xmm2
+; AVX512BF16-NEXT: vmovd %xmm2, %r10d
+; AVX512BF16-NEXT: vpextrw $6, %xmm0, %r11d
+; AVX512BF16-NEXT: shll $16, %r11d
+; AVX512BF16-NEXT: vmovd %r11d, %xmm2
+; AVX512BF16-NEXT: vroundss $9, %xmm2, %xmm2, %xmm2
+; AVX512BF16-NEXT: vcvtneps2bf16 %xmm2, %xmm2
+; AVX512BF16-NEXT: vmovd %xmm2, %r11d
+; AVX512BF16-NEXT: vpextrw $5, %xmm0, %ebx
+; AVX512BF16-NEXT: shll $16, %ebx
+; AVX512BF16-NEXT: vmovd %ebx, %xmm2
+; AVX512BF16-NEXT: vroundss $9, %xmm2, %xmm2, %xmm2
+; AVX512BF16-NEXT: vcvtneps2bf16 %xmm2, %xmm2
+; AVX512BF16-NEXT: vmovd %xmm2, %ebx
+; AVX512BF16-NEXT: vpextrw $4, %xmm0, %ebp
+; AVX512BF16-NEXT: shll $16, %ebp
+; AVX512BF16-NEXT: vmovd %ebp, %xmm2
+; AVX512BF16-NEXT: vroundss $9, %xmm2, %xmm2, %xmm2
+; AVX512BF16-NEXT: vcvtneps2bf16 %xmm2, %xmm2
+; AVX512BF16-NEXT: vmovd %xmm2, %ebp
+; AVX512BF16-NEXT: vpextrw $3, %xmm0, %r14d
+; AVX512BF16-NEXT: shll $16, %r14d
+; AVX512BF16-NEXT: vmovd %r14d, %xmm2
+; AVX512BF16-NEXT: vroundss $9, %xmm2, %xmm2, %xmm2
+; AVX512BF16-NEXT: vcvtneps2bf16 %xmm2, %xmm2
+; AVX512BF16-NEXT: vmovd %xmm2, %r14d
+; AVX512BF16-NEXT: vpextrw $2, %xmm0, %r15d
+; AVX512BF16-NEXT: shll $16, %r15d
+; AVX512BF16-NEXT: vmovd %r15d, %xmm2
+; AVX512BF16-NEXT: vroundss $9, %xmm2, %xmm2, %xmm2
+; AVX512BF16-NEXT: vcvtneps2bf16 %xmm2, %xmm2
+; AVX512BF16-NEXT: vmovd %xmm2, %r15d
+; AVX512BF16-NEXT: vmovd %xmm0, %r12d
+; AVX512BF16-NEXT: shll $16, %r12d
+; AVX512BF16-NEXT: vmovd %r12d, %xmm2
+; AVX512BF16-NEXT: vroundss $9, %xmm2, %xmm2, %xmm2
+; AVX512BF16-NEXT: vcvtneps2bf16 %xmm2, %xmm2
+; AVX512BF16-NEXT: vpextrw $1, %xmm0, %r12d
+; AVX512BF16-NEXT: shll $16, %r12d
+; AVX512BF16-NEXT: vmovd %r12d, %xmm0
+; AVX512BF16-NEXT: vroundss $9, %xmm0, %xmm0, %xmm0
+; AVX512BF16-NEXT: vcvtneps2bf16 %xmm0, %xmm0
+; AVX512BF16-NEXT: vmovd %xmm0, %r12d
+; AVX512BF16-NEXT: vpinsrw $1, %r12d, %xmm2, %xmm0
+; AVX512BF16-NEXT: vpinsrw $2, %r15d, %xmm0, %xmm0
+; AVX512BF16-NEXT: vpinsrw $3, %r14d, %xmm0, %xmm0
+; AVX512BF16-NEXT: vpinsrw $4, %ebp, %xmm0, %xmm0
+; AVX512BF16-NEXT: vpinsrw $5, %ebx, %xmm0, %xmm0
+; AVX512BF16-NEXT: vpinsrw $6, %r11d, %xmm0, %xmm0
+; AVX512BF16-NEXT: vpinsrw $7, %r10d, %xmm0, %xmm0
+; AVX512BF16-NEXT: vpinsrw $1, %r9d, %xmm1, %xmm1
+; AVX512BF16-NEXT: vpinsrw $2, %r8d, %xmm1, %xmm1
+; AVX512BF16-NEXT: vpinsrw $3, %edi, %xmm1, %xmm1
+; AVX512BF16-NEXT: vpinsrw $4, %esi, %xmm1, %xmm1
+; AVX512BF16-NEXT: vpinsrw $5, %edx, %xmm1, %xmm1
+; AVX512BF16-NEXT: vpinsrw $6, %ecx, %xmm1, %xmm1
+; AVX512BF16-NEXT: vpinsrw $7, %eax, %xmm1, %xmm1
+; AVX512BF16-NEXT: vinserti128 $1, %xmm0, %ymm1, %ymm0
+; AVX512BF16-NEXT: popq %rbx
+; AVX512BF16-NEXT: .cfi_def_cfa_offset 40
+; AVX512BF16-NEXT: popq %r12
+; AVX512BF16-NEXT: .cfi_def_cfa_offset 32
+; AVX512BF16-NEXT: popq %r14
+; AVX512BF16-NEXT: .cfi_def_cfa_offset 24
+; AVX512BF16-NEXT: popq %r15
+; AVX512BF16-NEXT: .cfi_def_cfa_offset 16
+; AVX512BF16-NEXT: popq %rbp
+; AVX512BF16-NEXT: .cfi_def_cfa_offset 8
+; AVX512BF16-NEXT: retq
+ %r = call <16 x bfloat> @llvm.floor.v16bf16(<16 x bfloat> %a)
+ ret <16 x bfloat> %r
+}
+
+define <32 x bfloat> @floor_v32bf16(<32 x bfloat> %a) {
+; AVX10_2-LABEL: floor_v32bf16:
+; AVX10_2: # %bb.0:
+; AVX10_2-NEXT: vrndscalebf16 $9, %zmm0, %zmm0
+; AVX10_2-NEXT: retq
+;
+; AVX512BF16-LABEL: floor_v32bf16:
+; AVX512BF16: # %bb.0:
+; AVX512BF16-NEXT: vpextrw $7, %xmm0, %eax
+; AVX512BF16-NEXT: shll $16, %eax
+; AVX512BF16-NEXT: vmovd %eax, %xmm1
+; AVX512BF16-NEXT: vpextrw $6, %xmm0, %eax
+; AVX512BF16-NEXT: shll $16, %eax
+; AVX512BF16-NEXT: vmovd %eax, %xmm2
+; AVX512BF16-NEXT: vpextrw $5, %xmm0, %eax
+; AVX512BF16-NEXT: shll $16, %eax
+; AVX512BF16-NEXT: vmovd %eax, %xmm3
+; AVX512BF16-NEXT: vpextrw $4, %xmm0, %eax
+; AVX512BF16-NEXT: shll $16, %eax
+; AVX512BF16-NEXT: vmovd %eax, %xmm4
+; AVX512BF16-NEXT: vpextrw $3, %xmm0, %eax
+; AVX512BF16-NEXT: shll $16, %eax
+; AVX512BF16-NEXT: vmovd %eax, %xmm5
+; AVX512BF16-NEXT: vpextrw $2, %xmm0, %eax
+; AVX512BF16-NEXT: shll $16, %eax
+; AVX512BF16-NEXT: vmovd %eax, %xmm6
+; AVX512BF16-NEXT: vmovd %xmm0, %eax
+; AVX512BF16-NEXT: shll $16, %eax
+; AVX512BF16-NEXT: vmovd %eax, %xmm7
+; AVX512BF16-NEXT: vpextrw $1, %xmm0, %eax
+; AVX512BF16-NEXT: shll $16, %eax
+; AVX512BF16-NEXT: vmovd %eax, %xmm8
+; AVX512BF16-NEXT: vextracti32x4 $1, %ymm0, %xmm16
+; AVX512BF16-NEXT: vpextrw $7, %xmm16, %eax
+; AVX512BF16-NEXT: shll $16, %eax
+; AVX512BF16-NEXT: vmovd %eax, %xmm9
+; AVX512BF16-NEXT: vpextrw $6, %xmm16, %eax
+; AVX512BF16-NEXT: shll $16, %eax
+; AVX512BF16-NEXT: vmovd %eax, %xmm10
+; AVX512BF16-NEXT: vpextrw $5, %xmm16, %eax
+; AVX512BF16-NEXT: shll $16, %eax
+; AVX512BF16-NEXT: vmovd %eax, %xmm11
+; AVX512BF16-NEXT: vpextrw $4, %xmm16, %eax
+; AVX512BF16-NEXT: shll $16, %eax
+; AVX512BF16-NEXT: vmovd %eax, %xmm12
+; AVX512BF16-NEXT: vpextrw $3, %xmm16, %eax
+; AVX512BF16-NEXT: shll $16, %eax
+; AVX512BF16-NEXT: vmovd %eax, %xmm13
+; AVX512BF16-NEXT: vpextrw $2, %xmm16, %eax
+; AVX512BF16-NEXT: shll $16, %eax
+; AVX512BF16-NEXT: vmovd %eax, %xmm14
+; AVX512BF16-NEXT: vmovd %xmm16, %eax
+; AVX512BF16-NEXT: shll $16, %eax
+; AVX512BF16-NEXT: vmovd %eax, %xmm15
+; AVX512BF16-NEXT: vpextrw $1, %xmm16, %eax
+; AVX512BF16-NEXT: shll $16, %eax
+; AVX512BF16-NEXT: vmovd %eax, %xmm16
+; AVX512BF16-NEXT: vextracti32x4 $2, %zmm0, %xmm24
+; AVX512BF16-NEXT: vpextrw $7, %xmm24, %eax
+; AVX512BF16-NEXT: shll $16, %eax
+; AVX512BF16-NEXT: vmovd %eax, %xmm17
+; AVX512BF16-NEXT: vpextrw $6, %xmm24, %eax
+; AVX512BF16-NEXT: shll $16, %eax
+; AVX512BF16-NEXT: vmovd %eax, %xmm18
+; AVX512BF16-NEXT: vpextrw $5, %xmm24, %eax
+; AVX512BF16-NEXT: shll $16, %eax
+; AVX512BF16-NEXT: vmovd %eax, %xmm19
+; AVX512BF16-NEXT: vpextrw $4, %xmm24, %eax
+; AVX512BF16-NEXT: shll $16, %eax
+; AVX512BF16-NEXT: vmovd %eax, %xmm20
+; AVX512BF16-NEXT: vpextrw $3, %xmm24, %eax
+; AVX512BF16-NEXT: shll $16, %eax
+; AVX512BF16-NEXT: vmovd %eax, %xmm21
+; AVX512BF16-NEXT: vpextrw $2, %xmm24, %eax
+; AVX512BF16-NEXT: shll $16, %eax
+; AVX512BF16-NEXT: vmovd %eax, %xmm22
+; AVX512BF16-NEXT: vmovd %xmm24, %eax
+; AVX512BF16-NEXT: shll $16, %eax
+; AVX512BF16-NEXT: vmovd %eax, %xmm23
+; AVX512BF16-NEXT: vpextrw $1, %xmm24, %eax
+; AVX512BF16-NEXT: shll $16, %eax
+; AVX512BF16-NEXT: vmovd %eax, %xmm24
+; AVX512BF16-NEXT: vextracti32x4 $3, %zmm0, %xmm25
+; AVX512BF16-NEXT: vpextrw $7, %xmm25, %eax
+; AVX512BF16-NEXT: shll $16, %eax
+; AVX512BF16-NEXT: vmovd %eax, %xmm0
+; AVX512BF16-NEXT: vpextrw $6, %xmm25, %eax
+; AVX512BF16-NEXT: shll $16, %eax
+; AVX512BF16-NEXT: vmovd %eax, %xmm26
+; AVX512BF16-NEXT: vpextrw $5, %xmm25, %eax
+; AVX512BF16-NEXT: shll $16, %eax
+; AVX512BF16-NEXT: vmovd %eax, %xmm27
+; AVX512BF16-NEXT: vpextrw $4, %xmm25, %eax
+; AVX512BF16-NEXT: shll $16, %eax
+; AVX512BF16-NEXT: vmovd %eax, %xmm28
+; AVX512BF16-NEXT: vpextrw $3, %xmm25, %eax
+; AVX512BF16-NEXT: shll $16, %eax
+; AVX512BF16-NEXT: vmovd %eax, %xmm29
+; AVX512BF16-NEXT: vpextrw $2, %xmm25, %eax
+; AVX512BF16-NEXT: shll $16, %eax
+; AVX512BF16-NEXT: vmovd %eax, %xmm30
+; AVX512BF16-NEXT: vmovd %xmm25, %eax
+; AVX512BF16-NEXT: shll $16, %eax
+; AVX512BF16-NEXT: vmovd %eax, %xmm31
+; AVX512BF16-NEXT: vpextrw $1, %xmm25, %eax
+; AVX512BF16-NEXT: shll $16, %eax
+; AVX512BF16-NEXT: vmovd %eax, %xmm25
+; AVX512BF16-NEXT: vrndscaless $9, %xmm25, %xmm25, %xmm25
+; AVX512BF16-NEXT: vcvtneps2bf16 %xmm25, %xmm25
+; AVX512BF16-NEXT: vmovd %xmm25, %eax
+; AVX512BF16-NEXT: vrndscaless $9, %xmm30, %xmm30, %xmm25
+; AVX512BF16-NEXT: vcvtneps2bf16 %xmm25, %xmm25
+; AVX512BF16-NEXT: vrndscaless $9, %xmm31, %xmm31, %xmm30
+; AVX512BF16-NEXT: vcvtneps2bf16 %xmm30, %xmm30
+; AVX512BF16-NEXT: vpinsrw $1, %eax, %xmm30, %xmm30
+; AVX512BF16-NEXT: vmovd %xmm25, %eax
+; AVX512BF16-NEXT: vrndscaless $9, %xmm29, %xmm29, %xmm25
+; AVX512BF16-NEXT: vcvtneps2bf16 %xmm25, %xmm25
+; AVX512BF16-NEXT: vpinsrw $2, %eax, %xmm30, %xmm29
+; AVX512BF16-NEXT: vmovd %xmm25, %eax
+; AVX512BF16-NEXT: vrndscaless $9, %xmm28, %xmm28, %xmm25
+; AVX512BF16-NEXT: vcvtneps2bf16 %xmm25, %xmm25
+; AVX512BF16-NEXT: vpinsrw $3, %eax, %xmm29, %xmm28
+; AVX512BF16-NEXT: vmovd %xmm25, %eax
+; AVX512BF16-NEXT: vrndscaless $9, %xmm27, %xmm27, %xmm25
+; AVX512BF16-NEXT: vcvtneps2bf16 %xmm25, %xmm25
+; AVX512BF16-NEXT: vpinsrw $4, %eax, %xmm28, %xmm27
+; AVX512BF16-NEXT: vmovd %xmm25, %eax
+; AVX512BF16-NEXT: vrndscaless $9, %xmm26, %xmm26, %xmm25
+; AVX512BF16-NEXT: vcvtneps2bf16 %xmm25, %xmm25
+; AVX512BF16-NEXT: vpinsrw $5, %eax, %xmm27, %xmm26
+; AVX512BF16-NEXT: vmovd %xmm25, %eax
+; AVX512BF16-NEXT: vroundss $9, %xmm0, %xmm0, %xmm0
+; AVX512BF16-NEXT: vcvtneps2bf16 %xmm0, %xmm0
+; AVX512BF16-NEXT: vpinsrw $6, %eax, %xmm26, %xmm25
+; AVX512BF16-NEXT: vmovd %xmm0, %eax
+; AVX512BF16-NEXT: vrndscaless $9, %xmm24, %xmm24, %xmm0
+; AVX512BF16-NEXT: vcvtneps2bf16 %xmm0, %xmm24
+; AVX512BF16-NEXT: vpinsrw $7, %eax, %xmm25, %xmm0
+; AVX512BF16-NEXT: vmovd %xmm24, %eax
+; AVX512BF16-NEXT: vrndscaless $9, %xmm22, %xmm22, %xmm22
+; AVX512BF16-NEXT: vcvtneps2bf16 %xmm22, %xmm22
+; AVX512BF16-NEXT: vrndscaless $9, %xmm23, %xmm23, %xmm23
+; AVX512BF16-NEXT: vcvtneps2bf16 %xmm23, %xmm23
+; AVX512BF16-NEXT: vpinsrw $1, %eax, %xmm23, %xmm23
+; AVX512BF16-NEXT: vmovd %xmm22, %eax
+; AVX512BF16-NEXT: vrndscaless $9, %xmm21, %xmm21, %xmm21
+; AVX512BF16-NEXT: vcvtneps2bf16 %xmm21, %xmm21
+; AVX512BF16-NEXT: vpinsrw $2, %eax, %xmm23, %xmm22
+; AVX512BF16-NEXT: vmovd %xmm21, %eax
+; AVX512BF16-NEXT: vrndscaless $9, %xmm20, %xmm20, %xmm20
+; AVX512BF16-NEXT: vcvtneps2bf16 %xmm20, %xmm20
+; AVX512BF16-NEXT: vpinsrw $3, %eax, %xmm22, %xmm21
+; AVX512BF16-NEXT: vmovd %xmm20, %eax
+; AVX512BF16-NEXT: vrndscaless $9, %xmm19, %xmm19, %xmm19
+; AVX512BF16-NEXT: vcvtneps2bf16 %xmm19, %xmm19
+; AVX512BF16-NEXT: vpinsrw $4, %eax, %xmm21, %xmm20
+; AVX512BF16-NEXT: vmovd %xmm19, %eax
+; AVX512BF16-NEXT: vrndscaless $9, %xmm18, %xmm18, %xmm18
+; AVX512BF16-NEXT: vcvtneps2bf16 %xmm18, %xmm18
+; AVX512BF16-NEXT: vpinsrw $5, %eax, %xmm20, %xmm19
+; AVX512BF16-NEXT: vmovd %xmm18, %eax
+; AVX512BF16-NEXT: vrndscaless $9, %xmm17, %xmm17, %xmm17
+; AVX512BF16-NEXT: vcvtneps2bf16 %xmm17, %xmm17
+; AVX512BF16-NEXT: vpinsrw $6, %eax, %xmm19, %xmm18
+; AVX512BF16-NEXT: vmovd %xmm17, %eax
+; AVX512BF16-NEXT: vrndscaless $9, %xmm16, %xmm16, %xmm16
+; AVX512BF16-NEXT: vcvtneps2bf16 %xmm16, %xmm16
+; AVX512BF16-NEXT: vpinsrw $7, %eax, %xmm18, %xmm17
+; AVX512BF16-NEXT: vmovd %xmm16, %eax
+; AVX512BF16-NEXT: vroundss $9, %xmm14, %xmm14, %xmm14
+; AVX512BF16-NEXT: vcvtneps2bf16 %xmm14, %xmm14
+; AVX512BF16-NEXT: vroundss $9, %xmm15, %xmm15, %xmm15
+; AVX512BF16-NEXT: vcvtneps2bf16 %xmm15, %xmm15
+; AVX512BF16-NEXT: vpinsrw $1, %eax, %xmm15, %xmm15
+; AVX512BF16-NEXT: vmovd %xmm14, %eax
+; AVX512BF16-NEXT: vroundss $9, %xmm13, %xmm13, %xmm13
+; AVX512BF16-NEXT: vcvtneps2bf16 %xmm13, %xmm13
+; AVX512BF16-NEXT: vpinsrw $2, %eax, %xmm15, %xmm14
+; AVX512BF16-NEXT: vmovd %xmm13, %eax
+; AVX512BF16-NEXT: vroundss $9, %xmm12, %xmm12, %xmm12
+; AVX512BF16-NEXT: vcvtneps2bf16 %xmm12, %xmm12
+; AVX512BF16-NEXT: vpinsrw $3, %eax, %xmm14, %xmm13
+; AVX512BF16-NEXT: vmovd %xmm12, %eax
+; AVX512BF16-NEXT: vroundss $9, %xmm11, %xmm11, %xmm11
+; AVX512BF16-NEXT: vcvtneps2bf16 %xmm11, %xmm11
+; AVX512BF16-NEXT: vpinsrw $4, %eax, %xmm13, %xmm12
+; AVX512BF16-NEXT: vmovd %xmm11, %eax
+; AVX512BF16-NEXT: vroundss $9, %xmm10, %xmm10, %xmm10
+; AVX512BF16-NEXT: vcvtneps2bf16 %xmm10, %xmm10
+; AVX512BF16-NEXT: vpinsrw $5, %eax, %xmm12, %xmm11
+; AVX512BF16-NEXT: vmovd %xmm10, %eax
+; AVX512BF16-NEXT: vroundss $9, %xmm9, %xmm9, %xmm9
+; AVX512BF16-NEXT: vcvtneps2bf16 %xmm9, %xmm9
+; AVX512BF16-NEXT: vpinsrw $6, %eax, %xmm11, %xmm10
+; AVX512BF16-NEXT: vmovd %xmm9, %eax
+; AVX512BF16-NEXT: vroundss $9, %xmm8, %xmm8, %xmm8
+; AVX512BF16-NEXT: vcvtneps2bf16 %xmm8, %xmm8
+; AVX512BF16-NEXT: vpinsrw $7, %eax, %xmm10, %xmm9
+; AVX512BF16-NEXT: vmovd %xmm8, %eax
+; AVX512BF16-NEXT: vroundss $9, %xmm6, %xmm6, %xmm6
+; AVX512BF16-NEXT: vcvtneps2bf16 %xmm6, %xmm6
+; AVX512BF16-NEXT: vroundss $9, %xmm7, %xmm7, %xmm7
+; AVX512BF16-NEXT: vcvtneps2bf16 %xmm7, %xmm7
+; AVX512BF16-NEXT: vpinsrw $1, %eax, %xmm7, %xmm7
+; AVX512BF16-NEXT: vmovd %xmm6, %eax
+; AVX512BF16-NEXT: vroundss $9, %xmm5, %xmm5, %xmm5
+; AVX512BF16-NEXT: vcvtneps2bf16 %xmm5, %xmm5
+; AVX512BF16-NEXT: vpinsrw $2, %eax, %xmm7, %xmm6
+; AVX512BF16-NEXT: vmovd %xmm5, %eax
+; AVX512BF16-NEXT: vroundss $9, %xmm4, %xmm4, %xmm4
+; AVX512BF16-NEXT: vcvtneps2bf16 %xmm4, %xmm4
+; AVX512BF16-NEXT: vpinsrw $3, %eax, %xmm6, %xmm5
+; AVX512BF16-NEXT: vmovd %xmm4, %eax
+; AVX512BF16-NEXT: vroundss $9, %xmm3, %xmm3, %xmm3
+; AVX512BF16-NEXT: vcvtneps2bf16 %xmm3, %xmm3
+; AVX512BF16-NEXT: vpinsrw $4, %eax, %xmm5, %xmm4
+; AVX512BF16-NEXT: vmovd %xmm3, %eax
+; AVX512BF16-NEXT: vroundss $9, %xmm2, %xmm2, %xmm2
+; AVX512BF16-NEXT: vcvtneps2bf16 %xmm2, %xmm2
+; AVX512BF16-NEXT: vpinsrw $5, %eax, %xmm4, %xmm3
+; AVX512BF16-NEXT: vmovd %xmm2, %eax
+; AVX512BF16-NEXT: vroundss $9, %xmm1, %xmm1, %xmm1
+; AVX512BF16-NEXT: vcvtneps2bf16 %xmm1, %xmm1
+; AVX512BF16-NEXT: vpinsrw $6, %eax, %xmm3, %xmm2
+; AVX512BF16-NEXT: vmovd %xmm1, %eax
+; AVX512BF16-NEXT: vpinsrw $7, %eax, %xmm2, %xmm1
+; AVX512BF16-NEXT: vinserti32x4 $1, %xmm0, %ymm17, %ymm0
+; AVX512BF16-NEXT: vinserti128 $1, %xmm9, %ymm1, %ymm1
+; AVX512BF16-NEXT: vinserti64x4 $1, %ymm0, %zmm1, %zmm0
+; AVX512BF16-NEXT: retq
+ %r = call <32 x bfloat> @llvm.floor.v32bf16(<32 x bfloat> %a)
+ ret <32 x bfloat> %r
+}
+
+; Narrower than a full vector register: widened, not scalarized.
+define <2 x bfloat> @floor_v2bf16(<2 x bfloat> %a) {
+; AVX10_2-LABEL: floor_v2bf16:
+; AVX10_2: # %bb.0:
+; AVX10_2-NEXT: vrndscalebf16 $9, %xmm0, %xmm0
+; AVX10_2-NEXT: retq
+;
+; AVX512BF16-LABEL: floor_v2bf16:
+; AVX512BF16: # %bb.0:
+; AVX512BF16-NEXT: vpextrw $1, %xmm0, %eax
+; AVX512BF16-NEXT: shll $16, %eax
+; AVX512BF16-NEXT: vmovd %eax, %xmm1
+; AVX512BF16-NEXT: vroundss $9, %xmm1, %xmm1, %xmm1
+; AVX512BF16-NEXT: vcvtneps2bf16 %xmm1, %xmm1
+; AVX512BF16-NEXT: vmovd %xmm0, %eax
+; AVX512BF16-NEXT: shll $16, %eax
+; AVX512BF16-NEXT: vmovd %eax, %xmm0
+; AVX512BF16-NEXT: vroundss $9, %xmm0, %xmm0, %xmm0
+; AVX512BF16-NEXT: vcvtneps2bf16 %xmm0, %xmm0
+; AVX512BF16-NEXT: vpunpcklwd {{.*#+}} xmm0 = xmm0[0],xmm1[0],xmm0[1],xmm1[1],xmm0[2],xmm1[2],xmm0[3],xmm1[3]
+; AVX512BF16-NEXT: retq
+ %r = call <2 x bfloat> @llvm.floor.v2bf16(<2 x bfloat> %a)
+ ret <2 x bfloat> %r
+}
+
+; Folded load operand.
+define <8 x bfloat> @floor_v8bf16_load(ptr %p) {
+; AVX10_2-LABEL: floor_v8bf16_load:
+; AVX10_2: # %bb.0:
+; AVX10_2-NEXT: vrndscalebf16 $9, (%rdi), %xmm0
+; AVX10_2-NEXT: retq
+;
+; AVX512BF16-LABEL: floor_v8bf16_load:
+; AVX512BF16: # %bb.0:
+; AVX512BF16-NEXT: vmovdqa (%rdi), %xmm0
+; AVX512BF16-NEXT: vpextrw $7, %xmm0, %eax
+; AVX512BF16-NEXT: shll $16, %eax
+; AVX512BF16-NEXT: vmovd %eax, %xmm1
+; AVX512BF16-NEXT: vroundss $9, %xmm1, %xmm1, %xmm1
+; AVX512BF16-NEXT: vcvtneps2bf16 %xmm1, %xmm1
+; AVX512BF16-NEXT: vmovd %xmm1, %eax
+; AVX512BF16-NEXT: vpextrw $6, %xmm0, %ecx
+; AVX512BF16-NEXT: shll $16, %ecx
+; AVX512BF16-NEXT: vmovd %ecx, %xmm1
+; AVX512BF16-NEXT: vroundss $9, %xmm1, %xmm1, %xmm1
+; AVX512BF16-NEXT: vcvtneps2bf16 %xmm1, %xmm1
+; AVX512BF16-NEXT: vmovd %xmm1, %ecx
+; AVX512BF16-NEXT: vpextrw $5, %xmm0, %edx
+; AVX512BF16-NEXT: shll $16, %edx
+; AVX512BF16-NEXT: vmovd %edx, %xmm1
+; AVX512BF16-NEXT: vroundss $9, %xmm1, %xmm1, %xmm1
+; AVX512BF16-NEXT: vcvtneps2bf16 %xmm1, %xmm1
+; AVX512BF16-NEXT: vmovd %xmm1, %edx
+; AVX512BF16-NEXT: vpextrw $4, %xmm0, %esi
+; AVX512BF16-NEXT: shll $16, %esi
+; AVX512BF16-NEXT: vmovd %esi, %xmm1
+; AVX512BF16-NEXT: vroundss $9, %xmm1, %xmm1, %xmm1
+; AVX512BF16-NEXT: vcvtneps2bf16 %xmm1, %xmm1
+; AVX512BF16-NEXT: vmovd %xmm1, %esi
+; AVX512BF16-NEXT: vpextrw $3, %xmm0, %edi
+; AVX512BF16-NEXT: shll $16, %edi
+; AVX512BF16-NEXT: vmovd %edi, %xmm1
+; AVX512BF16-NEXT: vroundss $9, %xmm1, %xmm1, %xmm1
+; AVX512BF16-NEXT: vcvtneps2bf16 %xmm1, %xmm1
+; AVX512BF16-NEXT: vmovd %xmm1, %edi
+; AVX512BF16-NEXT: vpextrw $2, %xmm0, %r8d
+; AVX512BF16-NEXT: shll $16, %r8d
+; AVX512BF16-NEXT: vmovd %r8d, %xmm1
+; AVX512BF16-NEXT: vroundss $9, %xmm1, %xmm1, %xmm1
+; AVX512BF16-NEXT: vcvtneps2bf16 %xmm1, %xmm1
+; AVX512BF16-NEXT: vmovd %xmm1, %r8d
+; AVX512BF16-NEXT: vpextrw $1, %xmm0, %r9d
+; AVX512BF16-NEXT: shll $16, %r9d
+; AVX512BF16-NEXT: vmovd %r9d, %xmm1
+; AVX512BF16-NEXT: vroundss $9, %xmm1, %xmm1, %xmm1
+; AVX512BF16-NEXT: vcvtneps2bf16 %xmm1, %xmm1
+; AVX512BF16-NEXT: vmovd %xmm0, %r9d
+; AVX512BF16-NEXT: shll $16, %r9d
+; AVX512BF16-NEXT: vmovd %r9d, %xmm0
+; AVX512BF16-NEXT: vroundss $9, %xmm0, %xmm0, %xmm0
+; AVX512BF16-NEXT: vcvtneps2bf16 %xmm0, %xmm0
+; AVX512BF16-NEXT: vpunpcklwd {{.*#+}} xmm0 = xmm0[0],xmm1[0],xmm0[1],xmm1[1],xmm0[2],xmm1[2],xmm0[3],xmm1[3]
+; AVX512BF16-NEXT: vpinsrw $2, %r8d, %xmm0, %xmm0
+; AVX512BF16-NEXT: vpinsrw $3, %edi, %xmm0, %xmm0
+; AVX512BF16-NEXT: vpinsrw $4, %esi, %xmm0, %xmm0
+; AVX512BF16-NEXT: vpinsrw $5, %edx, %xmm0, %xmm0
+; AVX512BF16-NEXT: vpinsrw $6, %ecx, %xmm0, %xmm0
+; AVX512BF16-NEXT: vpinsrw $7, %eax, %xmm0, %xmm0
+; AVX512BF16-NEXT: retq
+ %a = load <8 x bfloat>, ptr %p
+ %r = call <8 x bfloat> @llvm.floor.v8bf16(<8 x bfloat> %a)
+ ret <8 x bfloat> %r
+}
+
+; Masked forms, exercising the AVX512_maskable patterns.
+define <8 x bfloat> @floor_v8bf16_mask(<8 x bfloat> %a, <8 x bfloat> %src, i8 %msk) {
+; AVX10_2-LABEL: floor_v8bf16_mask:
+; AVX10_2: # %bb.0:
+; AVX10_2-NEXT: kmovd %edi, %k1
+; AVX10_2-NEXT: vrndscalebf16 $9, %xmm0, %xmm1 {%k1}
+; AVX10_2-NEXT: vmovdqa %xmm1, %xmm0
+; AVX10_2-NEXT: retq
+;
+; AVX512BF16-LABEL: floor_v8bf16_mask:
+; AVX512BF16: # %bb.0:
+; AVX512BF16-NEXT: kmovd %edi, %k1
+; AVX512BF16-NEXT: vpextrw $7, %xmm0, %eax
+; AVX512BF16-NEXT: shll $16, %eax
+; AVX512BF16-NEXT: vmovd %eax, %xmm2
+; AVX512BF16-NEXT: vroundss $9, %xmm2, %xmm2, %xmm2
+; AVX512BF16-NEXT: vcvtneps2bf16 %xmm2, %xmm2
+; AVX512BF16-NEXT: vmovd %xmm2, %eax
+; AVX512BF16-NEXT: vpextrw $6, %xmm0, %ecx
+; AVX512BF16-NEXT: shll $16, %ecx
+; AVX512BF16-NEXT: vmovd %ecx, %xmm2
+; AVX512BF16-NEXT: vroundss $9, %xmm2, %xmm2, %xmm2
+; AVX512BF16-NEXT: vcvtneps2bf16 %xmm2, %xmm2
+; AVX512BF16-NEXT: vmovd %xmm2, %ecx
+; AVX512BF16-NEXT: vpextrw $5, %xmm0, %edx
+; AVX512BF16-NEXT: shll $16, %edx
+; AVX512BF16-NEXT: vmovd %edx, %xmm2
+; AVX512BF16-NEXT: vroundss $9, %xmm2, %xmm2, %xmm2
+; AVX512BF16-NEXT: vcvtneps2bf16 %xmm2, %xmm2
+; AVX512BF16-NEXT: vmovd %xmm2, %edx
+; AVX512BF16-NEXT: vpextrw $4, %xmm0, %esi
+; AVX512BF16-NEXT: shll $16, %esi
+; AVX512BF16-NEXT: vmovd %esi, %xmm2
+; AVX512BF16-NEXT: vroundss $9, %xmm2, %xmm2, %xmm2
+; AVX512BF16-NEXT: vcvtneps2bf16 %xmm2, %xmm2
+; AVX512BF16-NEXT: vmovd %xmm2, %esi
+; AVX512BF16-NEXT: vpextrw $3, %xmm0, %edi
+; AVX512BF16-NEXT: shll $16, %edi
+; AVX512BF16-NEXT: vmovd %edi, %xmm2
+; AVX512BF16-NEXT: vroundss $9, %xmm2, %xmm2, %xmm2
+; AVX512BF16-NEXT: vcvtneps2bf16 %xmm2, %xmm2
+; AVX512BF16-NEXT: vmovd %xmm2, %edi
+; AVX512BF16-NEXT: vpextrw $2, %xmm0, %r8d
+; AVX512BF16-NEXT: shll $16, %r8d
+; AVX512BF16-NEXT: vmovd %r8d, %xmm2
+; AVX512BF16-NEXT: vroundss $9, %xmm2, %xmm2, %xmm2
+; AVX512BF16-NEXT: vcvtneps2bf16 %xmm2, %xmm2
+; AVX512BF16-NEXT: vmovd %xmm2, %r8d
+; AVX512BF16-NEXT: vpextrw $1, %xmm0, %r9d
+; AVX512BF16-NEXT: shll $16, %r9d
+; AVX512BF16-NEXT: vmovd %r9d, %xmm2
+; AVX512BF16-NEXT: vroundss $9, %xmm2, %xmm2, %xmm2
+; AVX512BF16-NEXT: vcvtneps2bf16 %xmm2, %xmm2
+; AVX512BF16-NEXT: vmovd %xmm0, %r9d
+; AVX512BF16-NEXT: shll $16, %r9d
+; AVX512BF16-NEXT: vmovd %r9d, %xmm0
+; AVX512BF16-NEXT: vroundss $9, %xmm0, %xmm0, %xmm0
+; AVX512BF16-NEXT: vcvtneps2bf16 %xmm0, %xmm0
+; AVX512BF16-NEXT: vpunpcklwd {{.*#+}} xmm0 = xmm0[0],xmm2[0],xmm0[1],xmm2[1],xmm0[2],xmm2[2],xmm0[3],xmm2[3]
+; AVX512BF16-NEXT: vpinsrw $2, %r8d, %xmm0, %xmm0
+; AVX512BF16-NEXT: vpinsrw $3, %edi, %xmm0, %xmm0
+; AVX512BF16-NEXT: vpinsrw $4, %esi, %xmm0, %xmm0
+; AVX512BF16-NEXT: vpinsrw $5, %edx, %xmm0, %xmm0
+; AVX512BF16-NEXT: vpinsrw $6, %ecx, %xmm0, %xmm0
+; AVX512BF16-NEXT: vpinsrw $7, %eax, %xmm0, %xmm0
+; AVX512BF16-NEXT: vpblendmw %xmm0, %xmm1, %xmm0 {%k1}
+; AVX512BF16-NEXT: retq
+ %mask = bitcast i8 %msk to <8 x i1>
+ %r = call <8 x bfloat> @llvm.floor.v8bf16(<8 x bfloat> %a)
+ %res = select <8 x i1> %mask, <8 x bfloat> %r, <8 x bfloat> %src
+ ret <8 x bfloat> %res
+}
+
+define <8 x bfloat> @floor_v8bf16_maskz(<8 x bfloat> %a, i8 %msk) {
+; AVX10_2-LABEL: floor_v8bf16_maskz:
+; AVX10_2: # %bb.0:
+; AVX10_2-NEXT: kmovd %edi, %k1
+; AVX10_2-NEXT: vrndscalebf16 $9, %xmm0, %xmm0 {%k1} {z}
+; AVX10_2-NEXT: retq
+;
+; AVX512BF16-LABEL: floor_v8bf16_maskz:
+; AVX512BF16: # %bb.0:
+; AVX512BF16-NEXT: kmovd %edi, %k1
+; AVX512BF16-NEXT: vpextrw $7, %xmm0, %eax
+; AVX512BF16-NEXT: shll $16, %eax
+; AVX512BF16-NEXT: vmovd %eax, %xmm1
+; AVX512BF16-NEXT: vroundss $9, %xmm1, %xmm1, %xmm1
+; AVX512BF16-NEXT: vcvtneps2bf16 %xmm1, %xmm1
+; AVX512BF16-NEXT: vmovd %xmm1, %eax
+; AVX512BF16-NEXT: vpextrw $6, %xmm0, %ecx
+; AVX512BF16-NEXT: shll $16, %ecx
+; AVX512BF16-NEXT: vmovd %ecx, %xmm1
+; AVX512BF16-NEXT: vroundss $9, %xmm1, %xmm1, %xmm1
+; AVX512BF16-NEXT: vcvtneps2bf16 %xmm1, %xmm1
+; AVX512BF16-NEXT: vmovd %xmm1, %ecx
+; AVX512BF16-NEXT: vpextrw $5, %xmm0, %edx
+; AVX512BF16-NEXT: shll $16, %edx
+; AVX512BF16-NEXT: vmovd %edx, %xmm1
+; AVX512BF16-NEXT: vroundss $9, %xmm1, %xmm1, %xmm1
+; AVX512BF16-NEXT: vcvtneps2bf16 %xmm1, %xmm1
+; AVX512BF16-NEXT: vmovd %xmm1, %edx
+; AVX512BF16-NEXT: vpextrw $4, %xmm0, %esi
+; AVX512BF16-NEXT: shll $16, %esi
+; AVX512BF16-NEXT: vmovd %esi, %xmm1
+; AVX512BF16-NEXT: vroundss $9, %xmm1, %xmm1, %xmm1
+; AVX512BF16-NEXT: vcvtneps2bf16 %xmm1, %xmm1
+; AVX512BF16-NEXT: vmovd %xmm1, %esi
+; AVX512BF16-NEXT: vpextrw $3, %xmm0, %edi
+; AVX512BF16-NEXT: shll $16, %edi
+; AVX512BF16-NEXT: vmovd %edi, %xmm1
+; AVX512BF16-NEXT: vroundss $9, %xmm1, %xmm1, %xmm1
+; AVX512BF16-NEXT: vcvtneps2bf16 %xmm1, %xmm1
+; AVX512BF16-NEXT: vmovd %xmm1, %edi
+; AVX512BF16-NEXT: vpextrw $2, %xmm0, %r8d
+; AVX512BF16-NEXT: shll $16, %r8d
+; AVX512BF16-NEXT: vmovd %r8d, %xmm1
+; AVX512BF16-NEXT: vroundss $9, %xmm1, %xmm1, %xmm1
+; AVX512BF16-NEXT: vcvtneps2bf16 %xmm1, %xmm1
+; AVX512BF16-NEXT: vmovd %xmm1, %r8d
+; AVX512BF16-NEXT: vpextrw $1, %xmm0, %r9d
+; AVX512BF16-NEXT: shll $16, %r9d
+; AVX512BF16-NEXT: vmovd %r9d, %xmm1
+; AVX512BF16-NEXT: vroundss $9, %xmm1, %xmm1, %xmm1
+; AVX512BF16-NEXT: vcvtneps2bf16 %xmm1, %xmm1
+; AVX512BF16-NEXT: vmovd %xmm0, %r9d
+; AVX512BF16-NEXT: shll $16, %r9d
+; AVX512BF16-NEXT: vmovd %r9d, %xmm0
+; AVX512BF16-NEXT: vroundss $9, %xmm0, %xmm0, %xmm0
+; AVX512BF16-NEXT: vcvtneps2bf16 %xmm0, %xmm0
+; AVX512BF16-NEXT: vpunpcklwd {{.*#+}} xmm0 = xmm0[0],xmm1[0],xmm0[1],xmm1[1],xmm0[2],xmm1[2],xmm0[3],xmm1[3]
+; AVX512BF16-NEXT: vpinsrw $2, %r8d, %xmm0, %xmm0
+; AVX512BF16-NEXT: vpinsrw $3, %edi, %xmm0, %xmm0
+; AVX512BF16-NEXT: vpinsrw $4, %esi, %xmm0, %xmm0
+; AVX512BF16-NEXT: vpinsrw $5, %edx, %xmm0, %xmm0
+; AVX512BF16-NEXT: vpinsrw $6, %ecx, %xmm0, %xmm0
+; AVX512BF16-NEXT: vpinsrw $7, %eax, %xmm0, %xmm0
+; AVX512BF16-NEXT: vmovdqu16 %xmm0, %xmm0 {%k1} {z}
+; AVX512BF16-NEXT: retq
+ %mask = bitcast i8 %msk to <8 x i1>
+ %r = call <8 x bfloat> @llvm.floor.v8bf16(<8 x bfloat> %a)
+ %res = select <8 x i1> %mask, <8 x bfloat> %r, <8 x bfloat> zeroinitializer
+ ret <8 x bfloat> %res
+}
+
+; llvm.round is round-half-away-from-zero, which vrndscale cannot encode, so it
+; must stay scalarized rather than becoming a vrndscalebf16.
+define <8 x bfloat> @round_v8bf16(<8 x bfloat> %a) {
+; AVX10_2-LABEL: round_v8bf16:
+; AVX10_2-NOT: vrndscalebf16
+; AVX10_2: vroundss $11,
+; AVX10_2-NOT: vrndscalebf16
+ %r = call <8 x bfloat> @llvm.round.v8bf16(<8 x bfloat> %a)
+ ret <8 x bfloat> %r
+}
>From deea630dd8448fcc3c71ee867218a3aa12aec898 Mon Sep 17 00:00:00 2001
From: "Ziegler, Tim" <tim.ziegler at intel.com>
Date: Thu, 27 Aug 2026 15:49:06 +0200
Subject: [PATCH 2/6] Updated tests
---
llvm/test/CodeGen/X86/avx10_2bf16-rounding.ll | 1065 +++++++++--------
llvm/test/CodeGen/X86/znver-pdep.ll | 382 ++++--
llvm/test/CodeGen/X86/znver-pext.ll | 285 +++--
3 files changed, 1038 insertions(+), 694 deletions(-)
diff --git a/llvm/test/CodeGen/X86/avx10_2bf16-rounding.ll b/llvm/test/CodeGen/X86/avx10_2bf16-rounding.ll
index 4b9b764a6f5ce..11506e1bcae8c 100644
--- a/llvm/test/CodeGen/X86/avx10_2bf16-rounding.ll
+++ b/llvm/test/CodeGen/X86/avx10_2bf16-rounding.ll
@@ -16,49 +16,44 @@ define <8 x bfloat> @floor_v8bf16(<8 x bfloat> %a) {
; AVX512BF16-LABEL: floor_v8bf16:
; AVX512BF16: # %bb.0:
; AVX512BF16-NEXT: vpextrw $7, %xmm0, %eax
-; AVX512BF16-NEXT: shll $16, %eax
; AVX512BF16-NEXT: vmovd %eax, %xmm1
+; AVX512BF16-NEXT: vpslld $16, %xmm1, %xmm1
; AVX512BF16-NEXT: vroundss $9, %xmm1, %xmm1, %xmm1
; AVX512BF16-NEXT: vcvtneps2bf16 %xmm1, %xmm1
; AVX512BF16-NEXT: vmovd %xmm1, %eax
; AVX512BF16-NEXT: vpextrw $6, %xmm0, %ecx
-; AVX512BF16-NEXT: shll $16, %ecx
; AVX512BF16-NEXT: vmovd %ecx, %xmm1
+; AVX512BF16-NEXT: vpslld $16, %xmm1, %xmm1
; AVX512BF16-NEXT: vroundss $9, %xmm1, %xmm1, %xmm1
; AVX512BF16-NEXT: vcvtneps2bf16 %xmm1, %xmm1
; AVX512BF16-NEXT: vmovd %xmm1, %ecx
; AVX512BF16-NEXT: vpextrw $5, %xmm0, %edx
-; AVX512BF16-NEXT: shll $16, %edx
; AVX512BF16-NEXT: vmovd %edx, %xmm1
+; AVX512BF16-NEXT: vpslld $16, %xmm1, %xmm1
; AVX512BF16-NEXT: vroundss $9, %xmm1, %xmm1, %xmm1
; AVX512BF16-NEXT: vcvtneps2bf16 %xmm1, %xmm1
; AVX512BF16-NEXT: vmovd %xmm1, %edx
-; AVX512BF16-NEXT: vpextrw $4, %xmm0, %esi
-; AVX512BF16-NEXT: shll $16, %esi
-; AVX512BF16-NEXT: vmovd %esi, %xmm1
-; AVX512BF16-NEXT: vroundss $9, %xmm1, %xmm1, %xmm1
-; AVX512BF16-NEXT: vcvtneps2bf16 %xmm1, %xmm1
-; AVX512BF16-NEXT: vmovd %xmm1, %esi
+; AVX512BF16-NEXT: vpxor %xmm1, %xmm1, %xmm1
+; AVX512BF16-NEXT: vpunpckhwd {{.*#+}} xmm2 = xmm1[4],xmm0[4],xmm1[5],xmm0[5],xmm1[6],xmm0[6],xmm1[7],xmm0[7]
+; AVX512BF16-NEXT: vroundss $9, %xmm2, %xmm2, %xmm2
+; AVX512BF16-NEXT: vcvtneps2bf16 %xmm2, %xmm2
+; AVX512BF16-NEXT: vmovd %xmm2, %esi
; AVX512BF16-NEXT: vpextrw $3, %xmm0, %edi
-; AVX512BF16-NEXT: shll $16, %edi
-; AVX512BF16-NEXT: vmovd %edi, %xmm1
-; AVX512BF16-NEXT: vroundss $9, %xmm1, %xmm1, %xmm1
-; AVX512BF16-NEXT: vcvtneps2bf16 %xmm1, %xmm1
-; AVX512BF16-NEXT: vmovd %xmm1, %edi
+; AVX512BF16-NEXT: vmovd %edi, %xmm2
+; AVX512BF16-NEXT: vpslld $16, %xmm2, %xmm2
+; AVX512BF16-NEXT: vroundss $9, %xmm2, %xmm2, %xmm2
+; AVX512BF16-NEXT: vcvtneps2bf16 %xmm2, %xmm2
+; AVX512BF16-NEXT: vmovd %xmm2, %edi
; AVX512BF16-NEXT: vpextrw $2, %xmm0, %r8d
-; AVX512BF16-NEXT: shll $16, %r8d
-; AVX512BF16-NEXT: vmovd %r8d, %xmm1
-; AVX512BF16-NEXT: vroundss $9, %xmm1, %xmm1, %xmm1
-; AVX512BF16-NEXT: vcvtneps2bf16 %xmm1, %xmm1
-; AVX512BF16-NEXT: vmovd %xmm1, %r8d
-; AVX512BF16-NEXT: vpextrw $1, %xmm0, %r9d
-; AVX512BF16-NEXT: shll $16, %r9d
-; AVX512BF16-NEXT: vmovd %r9d, %xmm1
+; AVX512BF16-NEXT: vmovd %r8d, %xmm2
+; AVX512BF16-NEXT: vpslld $16, %xmm2, %xmm2
+; AVX512BF16-NEXT: vroundss $9, %xmm2, %xmm2, %xmm2
+; AVX512BF16-NEXT: vcvtneps2bf16 %xmm2, %xmm2
+; AVX512BF16-NEXT: vmovd %xmm2, %r8d
+; AVX512BF16-NEXT: vpblendw {{.*#+}} xmm1 = xmm1[0],xmm0[1],xmm1[2],xmm0[3],xmm1[4],xmm0[5],xmm1[6],xmm0[7]
; AVX512BF16-NEXT: vroundss $9, %xmm1, %xmm1, %xmm1
; AVX512BF16-NEXT: vcvtneps2bf16 %xmm1, %xmm1
-; AVX512BF16-NEXT: vmovd %xmm0, %r9d
-; AVX512BF16-NEXT: shll $16, %r9d
-; AVX512BF16-NEXT: vmovd %r9d, %xmm0
+; AVX512BF16-NEXT: vpslld $16, %xmm0, %xmm0
; AVX512BF16-NEXT: vroundss $9, %xmm0, %xmm0, %xmm0
; AVX512BF16-NEXT: vcvtneps2bf16 %xmm0, %xmm0
; AVX512BF16-NEXT: vpunpcklwd {{.*#+}} xmm0 = xmm0[0],xmm1[0],xmm0[1],xmm1[1],xmm0[2],xmm1[2],xmm0[3],xmm1[3]
@@ -81,9 +76,7 @@ define bfloat @floor_bf16(bfloat %a) {
;
; AVX512BF16-LABEL: floor_bf16:
; AVX512BF16: # %bb.0:
-; AVX512BF16-NEXT: vpextrw $0, %xmm0, %eax
-; AVX512BF16-NEXT: shll $16, %eax
-; AVX512BF16-NEXT: vmovd %eax, %xmm0
+; AVX512BF16-NEXT: vpslld $16, %xmm0, %xmm0
; AVX512BF16-NEXT: vroundss $9, %xmm0, %xmm0, %xmm0
; AVX512BF16-NEXT: vcvtneps2bf16 %xmm0, %xmm0
; AVX512BF16-NEXT: retq
@@ -100,49 +93,44 @@ define <8 x bfloat> @ceil_v8bf16(<8 x bfloat> %a) {
; AVX512BF16-LABEL: ceil_v8bf16:
; AVX512BF16: # %bb.0:
; AVX512BF16-NEXT: vpextrw $7, %xmm0, %eax
-; AVX512BF16-NEXT: shll $16, %eax
; AVX512BF16-NEXT: vmovd %eax, %xmm1
+; AVX512BF16-NEXT: vpslld $16, %xmm1, %xmm1
; AVX512BF16-NEXT: vroundss $10, %xmm1, %xmm1, %xmm1
; AVX512BF16-NEXT: vcvtneps2bf16 %xmm1, %xmm1
; AVX512BF16-NEXT: vmovd %xmm1, %eax
; AVX512BF16-NEXT: vpextrw $6, %xmm0, %ecx
-; AVX512BF16-NEXT: shll $16, %ecx
; AVX512BF16-NEXT: vmovd %ecx, %xmm1
+; AVX512BF16-NEXT: vpslld $16, %xmm1, %xmm1
; AVX512BF16-NEXT: vroundss $10, %xmm1, %xmm1, %xmm1
; AVX512BF16-NEXT: vcvtneps2bf16 %xmm1, %xmm1
; AVX512BF16-NEXT: vmovd %xmm1, %ecx
; AVX512BF16-NEXT: vpextrw $5, %xmm0, %edx
-; AVX512BF16-NEXT: shll $16, %edx
; AVX512BF16-NEXT: vmovd %edx, %xmm1
+; AVX512BF16-NEXT: vpslld $16, %xmm1, %xmm1
; AVX512BF16-NEXT: vroundss $10, %xmm1, %xmm1, %xmm1
; AVX512BF16-NEXT: vcvtneps2bf16 %xmm1, %xmm1
; AVX512BF16-NEXT: vmovd %xmm1, %edx
-; AVX512BF16-NEXT: vpextrw $4, %xmm0, %esi
-; AVX512BF16-NEXT: shll $16, %esi
-; AVX512BF16-NEXT: vmovd %esi, %xmm1
-; AVX512BF16-NEXT: vroundss $10, %xmm1, %xmm1, %xmm1
-; AVX512BF16-NEXT: vcvtneps2bf16 %xmm1, %xmm1
-; AVX512BF16-NEXT: vmovd %xmm1, %esi
+; AVX512BF16-NEXT: vpxor %xmm1, %xmm1, %xmm1
+; AVX512BF16-NEXT: vpunpckhwd {{.*#+}} xmm2 = xmm1[4],xmm0[4],xmm1[5],xmm0[5],xmm1[6],xmm0[6],xmm1[7],xmm0[7]
+; AVX512BF16-NEXT: vroundss $10, %xmm2, %xmm2, %xmm2
+; AVX512BF16-NEXT: vcvtneps2bf16 %xmm2, %xmm2
+; AVX512BF16-NEXT: vmovd %xmm2, %esi
; AVX512BF16-NEXT: vpextrw $3, %xmm0, %edi
-; AVX512BF16-NEXT: shll $16, %edi
-; AVX512BF16-NEXT: vmovd %edi, %xmm1
-; AVX512BF16-NEXT: vroundss $10, %xmm1, %xmm1, %xmm1
-; AVX512BF16-NEXT: vcvtneps2bf16 %xmm1, %xmm1
-; AVX512BF16-NEXT: vmovd %xmm1, %edi
+; AVX512BF16-NEXT: vmovd %edi, %xmm2
+; AVX512BF16-NEXT: vpslld $16, %xmm2, %xmm2
+; AVX512BF16-NEXT: vroundss $10, %xmm2, %xmm2, %xmm2
+; AVX512BF16-NEXT: vcvtneps2bf16 %xmm2, %xmm2
+; AVX512BF16-NEXT: vmovd %xmm2, %edi
; AVX512BF16-NEXT: vpextrw $2, %xmm0, %r8d
-; AVX512BF16-NEXT: shll $16, %r8d
-; AVX512BF16-NEXT: vmovd %r8d, %xmm1
-; AVX512BF16-NEXT: vroundss $10, %xmm1, %xmm1, %xmm1
-; AVX512BF16-NEXT: vcvtneps2bf16 %xmm1, %xmm1
-; AVX512BF16-NEXT: vmovd %xmm1, %r8d
-; AVX512BF16-NEXT: vpextrw $1, %xmm0, %r9d
-; AVX512BF16-NEXT: shll $16, %r9d
-; AVX512BF16-NEXT: vmovd %r9d, %xmm1
+; AVX512BF16-NEXT: vmovd %r8d, %xmm2
+; AVX512BF16-NEXT: vpslld $16, %xmm2, %xmm2
+; AVX512BF16-NEXT: vroundss $10, %xmm2, %xmm2, %xmm2
+; AVX512BF16-NEXT: vcvtneps2bf16 %xmm2, %xmm2
+; AVX512BF16-NEXT: vmovd %xmm2, %r8d
+; AVX512BF16-NEXT: vpblendw {{.*#+}} xmm1 = xmm1[0],xmm0[1],xmm1[2],xmm0[3],xmm1[4],xmm0[5],xmm1[6],xmm0[7]
; AVX512BF16-NEXT: vroundss $10, %xmm1, %xmm1, %xmm1
; AVX512BF16-NEXT: vcvtneps2bf16 %xmm1, %xmm1
-; AVX512BF16-NEXT: vmovd %xmm0, %r9d
-; AVX512BF16-NEXT: shll $16, %r9d
-; AVX512BF16-NEXT: vmovd %r9d, %xmm0
+; AVX512BF16-NEXT: vpslld $16, %xmm0, %xmm0
; AVX512BF16-NEXT: vroundss $10, %xmm0, %xmm0, %xmm0
; AVX512BF16-NEXT: vcvtneps2bf16 %xmm0, %xmm0
; AVX512BF16-NEXT: vpunpcklwd {{.*#+}} xmm0 = xmm0[0],xmm1[0],xmm0[1],xmm1[1],xmm0[2],xmm1[2],xmm0[3],xmm1[3]
@@ -165,9 +153,7 @@ define bfloat @ceil_bf16(bfloat %a) {
;
; AVX512BF16-LABEL: ceil_bf16:
; AVX512BF16: # %bb.0:
-; AVX512BF16-NEXT: vpextrw $0, %xmm0, %eax
-; AVX512BF16-NEXT: shll $16, %eax
-; AVX512BF16-NEXT: vmovd %eax, %xmm0
+; AVX512BF16-NEXT: vpslld $16, %xmm0, %xmm0
; AVX512BF16-NEXT: vroundss $10, %xmm0, %xmm0, %xmm0
; AVX512BF16-NEXT: vcvtneps2bf16 %xmm0, %xmm0
; AVX512BF16-NEXT: retq
@@ -184,49 +170,44 @@ define <8 x bfloat> @trunc_v8bf16(<8 x bfloat> %a) {
; AVX512BF16-LABEL: trunc_v8bf16:
; AVX512BF16: # %bb.0:
; AVX512BF16-NEXT: vpextrw $7, %xmm0, %eax
-; AVX512BF16-NEXT: shll $16, %eax
; AVX512BF16-NEXT: vmovd %eax, %xmm1
+; AVX512BF16-NEXT: vpslld $16, %xmm1, %xmm1
; AVX512BF16-NEXT: vroundss $11, %xmm1, %xmm1, %xmm1
; AVX512BF16-NEXT: vcvtneps2bf16 %xmm1, %xmm1
; AVX512BF16-NEXT: vmovd %xmm1, %eax
; AVX512BF16-NEXT: vpextrw $6, %xmm0, %ecx
-; AVX512BF16-NEXT: shll $16, %ecx
; AVX512BF16-NEXT: vmovd %ecx, %xmm1
+; AVX512BF16-NEXT: vpslld $16, %xmm1, %xmm1
; AVX512BF16-NEXT: vroundss $11, %xmm1, %xmm1, %xmm1
; AVX512BF16-NEXT: vcvtneps2bf16 %xmm1, %xmm1
; AVX512BF16-NEXT: vmovd %xmm1, %ecx
; AVX512BF16-NEXT: vpextrw $5, %xmm0, %edx
-; AVX512BF16-NEXT: shll $16, %edx
; AVX512BF16-NEXT: vmovd %edx, %xmm1
+; AVX512BF16-NEXT: vpslld $16, %xmm1, %xmm1
; AVX512BF16-NEXT: vroundss $11, %xmm1, %xmm1, %xmm1
; AVX512BF16-NEXT: vcvtneps2bf16 %xmm1, %xmm1
; AVX512BF16-NEXT: vmovd %xmm1, %edx
-; AVX512BF16-NEXT: vpextrw $4, %xmm0, %esi
-; AVX512BF16-NEXT: shll $16, %esi
-; AVX512BF16-NEXT: vmovd %esi, %xmm1
-; AVX512BF16-NEXT: vroundss $11, %xmm1, %xmm1, %xmm1
-; AVX512BF16-NEXT: vcvtneps2bf16 %xmm1, %xmm1
-; AVX512BF16-NEXT: vmovd %xmm1, %esi
+; AVX512BF16-NEXT: vpxor %xmm1, %xmm1, %xmm1
+; AVX512BF16-NEXT: vpunpckhwd {{.*#+}} xmm2 = xmm1[4],xmm0[4],xmm1[5],xmm0[5],xmm1[6],xmm0[6],xmm1[7],xmm0[7]
+; AVX512BF16-NEXT: vroundss $11, %xmm2, %xmm2, %xmm2
+; AVX512BF16-NEXT: vcvtneps2bf16 %xmm2, %xmm2
+; AVX512BF16-NEXT: vmovd %xmm2, %esi
; AVX512BF16-NEXT: vpextrw $3, %xmm0, %edi
-; AVX512BF16-NEXT: shll $16, %edi
-; AVX512BF16-NEXT: vmovd %edi, %xmm1
-; AVX512BF16-NEXT: vroundss $11, %xmm1, %xmm1, %xmm1
-; AVX512BF16-NEXT: vcvtneps2bf16 %xmm1, %xmm1
-; AVX512BF16-NEXT: vmovd %xmm1, %edi
+; AVX512BF16-NEXT: vmovd %edi, %xmm2
+; AVX512BF16-NEXT: vpslld $16, %xmm2, %xmm2
+; AVX512BF16-NEXT: vroundss $11, %xmm2, %xmm2, %xmm2
+; AVX512BF16-NEXT: vcvtneps2bf16 %xmm2, %xmm2
+; AVX512BF16-NEXT: vmovd %xmm2, %edi
; AVX512BF16-NEXT: vpextrw $2, %xmm0, %r8d
-; AVX512BF16-NEXT: shll $16, %r8d
-; AVX512BF16-NEXT: vmovd %r8d, %xmm1
-; AVX512BF16-NEXT: vroundss $11, %xmm1, %xmm1, %xmm1
-; AVX512BF16-NEXT: vcvtneps2bf16 %xmm1, %xmm1
-; AVX512BF16-NEXT: vmovd %xmm1, %r8d
-; AVX512BF16-NEXT: vpextrw $1, %xmm0, %r9d
-; AVX512BF16-NEXT: shll $16, %r9d
-; AVX512BF16-NEXT: vmovd %r9d, %xmm1
+; AVX512BF16-NEXT: vmovd %r8d, %xmm2
+; AVX512BF16-NEXT: vpslld $16, %xmm2, %xmm2
+; AVX512BF16-NEXT: vroundss $11, %xmm2, %xmm2, %xmm2
+; AVX512BF16-NEXT: vcvtneps2bf16 %xmm2, %xmm2
+; AVX512BF16-NEXT: vmovd %xmm2, %r8d
+; AVX512BF16-NEXT: vpblendw {{.*#+}} xmm1 = xmm1[0],xmm0[1],xmm1[2],xmm0[3],xmm1[4],xmm0[5],xmm1[6],xmm0[7]
; AVX512BF16-NEXT: vroundss $11, %xmm1, %xmm1, %xmm1
; AVX512BF16-NEXT: vcvtneps2bf16 %xmm1, %xmm1
-; AVX512BF16-NEXT: vmovd %xmm0, %r9d
-; AVX512BF16-NEXT: shll $16, %r9d
-; AVX512BF16-NEXT: vmovd %r9d, %xmm0
+; AVX512BF16-NEXT: vpslld $16, %xmm0, %xmm0
; AVX512BF16-NEXT: vroundss $11, %xmm0, %xmm0, %xmm0
; AVX512BF16-NEXT: vcvtneps2bf16 %xmm0, %xmm0
; AVX512BF16-NEXT: vpunpcklwd {{.*#+}} xmm0 = xmm0[0],xmm1[0],xmm0[1],xmm1[1],xmm0[2],xmm1[2],xmm0[3],xmm1[3]
@@ -249,9 +230,7 @@ define bfloat @trunc_bf16(bfloat %a) {
;
; AVX512BF16-LABEL: trunc_bf16:
; AVX512BF16: # %bb.0:
-; AVX512BF16-NEXT: vpextrw $0, %xmm0, %eax
-; AVX512BF16-NEXT: shll $16, %eax
-; AVX512BF16-NEXT: vmovd %eax, %xmm0
+; AVX512BF16-NEXT: vpslld $16, %xmm0, %xmm0
; AVX512BF16-NEXT: vroundss $11, %xmm0, %xmm0, %xmm0
; AVX512BF16-NEXT: vcvtneps2bf16 %xmm0, %xmm0
; AVX512BF16-NEXT: retq
@@ -268,49 +247,44 @@ define <8 x bfloat> @rint_v8bf16(<8 x bfloat> %a) {
; AVX512BF16-LABEL: rint_v8bf16:
; AVX512BF16: # %bb.0:
; AVX512BF16-NEXT: vpextrw $7, %xmm0, %eax
-; AVX512BF16-NEXT: shll $16, %eax
; AVX512BF16-NEXT: vmovd %eax, %xmm1
+; AVX512BF16-NEXT: vpslld $16, %xmm1, %xmm1
; AVX512BF16-NEXT: vroundss $4, %xmm1, %xmm1, %xmm1
; AVX512BF16-NEXT: vcvtneps2bf16 %xmm1, %xmm1
; AVX512BF16-NEXT: vmovd %xmm1, %eax
; AVX512BF16-NEXT: vpextrw $6, %xmm0, %ecx
-; AVX512BF16-NEXT: shll $16, %ecx
; AVX512BF16-NEXT: vmovd %ecx, %xmm1
+; AVX512BF16-NEXT: vpslld $16, %xmm1, %xmm1
; AVX512BF16-NEXT: vroundss $4, %xmm1, %xmm1, %xmm1
; AVX512BF16-NEXT: vcvtneps2bf16 %xmm1, %xmm1
; AVX512BF16-NEXT: vmovd %xmm1, %ecx
; AVX512BF16-NEXT: vpextrw $5, %xmm0, %edx
-; AVX512BF16-NEXT: shll $16, %edx
; AVX512BF16-NEXT: vmovd %edx, %xmm1
+; AVX512BF16-NEXT: vpslld $16, %xmm1, %xmm1
; AVX512BF16-NEXT: vroundss $4, %xmm1, %xmm1, %xmm1
; AVX512BF16-NEXT: vcvtneps2bf16 %xmm1, %xmm1
; AVX512BF16-NEXT: vmovd %xmm1, %edx
-; AVX512BF16-NEXT: vpextrw $4, %xmm0, %esi
-; AVX512BF16-NEXT: shll $16, %esi
-; AVX512BF16-NEXT: vmovd %esi, %xmm1
-; AVX512BF16-NEXT: vroundss $4, %xmm1, %xmm1, %xmm1
-; AVX512BF16-NEXT: vcvtneps2bf16 %xmm1, %xmm1
-; AVX512BF16-NEXT: vmovd %xmm1, %esi
+; AVX512BF16-NEXT: vpxor %xmm1, %xmm1, %xmm1
+; AVX512BF16-NEXT: vpunpckhwd {{.*#+}} xmm2 = xmm1[4],xmm0[4],xmm1[5],xmm0[5],xmm1[6],xmm0[6],xmm1[7],xmm0[7]
+; AVX512BF16-NEXT: vroundss $4, %xmm2, %xmm2, %xmm2
+; AVX512BF16-NEXT: vcvtneps2bf16 %xmm2, %xmm2
+; AVX512BF16-NEXT: vmovd %xmm2, %esi
; AVX512BF16-NEXT: vpextrw $3, %xmm0, %edi
-; AVX512BF16-NEXT: shll $16, %edi
-; AVX512BF16-NEXT: vmovd %edi, %xmm1
-; AVX512BF16-NEXT: vroundss $4, %xmm1, %xmm1, %xmm1
-; AVX512BF16-NEXT: vcvtneps2bf16 %xmm1, %xmm1
-; AVX512BF16-NEXT: vmovd %xmm1, %edi
+; AVX512BF16-NEXT: vmovd %edi, %xmm2
+; AVX512BF16-NEXT: vpslld $16, %xmm2, %xmm2
+; AVX512BF16-NEXT: vroundss $4, %xmm2, %xmm2, %xmm2
+; AVX512BF16-NEXT: vcvtneps2bf16 %xmm2, %xmm2
+; AVX512BF16-NEXT: vmovd %xmm2, %edi
; AVX512BF16-NEXT: vpextrw $2, %xmm0, %r8d
-; AVX512BF16-NEXT: shll $16, %r8d
-; AVX512BF16-NEXT: vmovd %r8d, %xmm1
-; AVX512BF16-NEXT: vroundss $4, %xmm1, %xmm1, %xmm1
-; AVX512BF16-NEXT: vcvtneps2bf16 %xmm1, %xmm1
-; AVX512BF16-NEXT: vmovd %xmm1, %r8d
-; AVX512BF16-NEXT: vpextrw $1, %xmm0, %r9d
-; AVX512BF16-NEXT: shll $16, %r9d
-; AVX512BF16-NEXT: vmovd %r9d, %xmm1
+; AVX512BF16-NEXT: vmovd %r8d, %xmm2
+; AVX512BF16-NEXT: vpslld $16, %xmm2, %xmm2
+; AVX512BF16-NEXT: vroundss $4, %xmm2, %xmm2, %xmm2
+; AVX512BF16-NEXT: vcvtneps2bf16 %xmm2, %xmm2
+; AVX512BF16-NEXT: vmovd %xmm2, %r8d
+; AVX512BF16-NEXT: vpblendw {{.*#+}} xmm1 = xmm1[0],xmm0[1],xmm1[2],xmm0[3],xmm1[4],xmm0[5],xmm1[6],xmm0[7]
; AVX512BF16-NEXT: vroundss $4, %xmm1, %xmm1, %xmm1
; AVX512BF16-NEXT: vcvtneps2bf16 %xmm1, %xmm1
-; AVX512BF16-NEXT: vmovd %xmm0, %r9d
-; AVX512BF16-NEXT: shll $16, %r9d
-; AVX512BF16-NEXT: vmovd %r9d, %xmm0
+; AVX512BF16-NEXT: vpslld $16, %xmm0, %xmm0
; AVX512BF16-NEXT: vroundss $4, %xmm0, %xmm0, %xmm0
; AVX512BF16-NEXT: vcvtneps2bf16 %xmm0, %xmm0
; AVX512BF16-NEXT: vpunpcklwd {{.*#+}} xmm0 = xmm0[0],xmm1[0],xmm0[1],xmm1[1],xmm0[2],xmm1[2],xmm0[3],xmm1[3]
@@ -333,9 +307,7 @@ define bfloat @rint_bf16(bfloat %a) {
;
; AVX512BF16-LABEL: rint_bf16:
; AVX512BF16: # %bb.0:
-; AVX512BF16-NEXT: vpextrw $0, %xmm0, %eax
-; AVX512BF16-NEXT: shll $16, %eax
-; AVX512BF16-NEXT: vmovd %eax, %xmm0
+; AVX512BF16-NEXT: vpslld $16, %xmm0, %xmm0
; AVX512BF16-NEXT: vroundss $4, %xmm0, %xmm0, %xmm0
; AVX512BF16-NEXT: vcvtneps2bf16 %xmm0, %xmm0
; AVX512BF16-NEXT: retq
@@ -352,49 +324,44 @@ define <8 x bfloat> @nearbyint_v8bf16(<8 x bfloat> %a) {
; AVX512BF16-LABEL: nearbyint_v8bf16:
; AVX512BF16: # %bb.0:
; AVX512BF16-NEXT: vpextrw $7, %xmm0, %eax
-; AVX512BF16-NEXT: shll $16, %eax
; AVX512BF16-NEXT: vmovd %eax, %xmm1
+; AVX512BF16-NEXT: vpslld $16, %xmm1, %xmm1
; AVX512BF16-NEXT: vroundss $12, %xmm1, %xmm1, %xmm1
; AVX512BF16-NEXT: vcvtneps2bf16 %xmm1, %xmm1
; AVX512BF16-NEXT: vmovd %xmm1, %eax
; AVX512BF16-NEXT: vpextrw $6, %xmm0, %ecx
-; AVX512BF16-NEXT: shll $16, %ecx
; AVX512BF16-NEXT: vmovd %ecx, %xmm1
+; AVX512BF16-NEXT: vpslld $16, %xmm1, %xmm1
; AVX512BF16-NEXT: vroundss $12, %xmm1, %xmm1, %xmm1
; AVX512BF16-NEXT: vcvtneps2bf16 %xmm1, %xmm1
; AVX512BF16-NEXT: vmovd %xmm1, %ecx
; AVX512BF16-NEXT: vpextrw $5, %xmm0, %edx
-; AVX512BF16-NEXT: shll $16, %edx
; AVX512BF16-NEXT: vmovd %edx, %xmm1
+; AVX512BF16-NEXT: vpslld $16, %xmm1, %xmm1
; AVX512BF16-NEXT: vroundss $12, %xmm1, %xmm1, %xmm1
; AVX512BF16-NEXT: vcvtneps2bf16 %xmm1, %xmm1
; AVX512BF16-NEXT: vmovd %xmm1, %edx
-; AVX512BF16-NEXT: vpextrw $4, %xmm0, %esi
-; AVX512BF16-NEXT: shll $16, %esi
-; AVX512BF16-NEXT: vmovd %esi, %xmm1
-; AVX512BF16-NEXT: vroundss $12, %xmm1, %xmm1, %xmm1
-; AVX512BF16-NEXT: vcvtneps2bf16 %xmm1, %xmm1
-; AVX512BF16-NEXT: vmovd %xmm1, %esi
+; AVX512BF16-NEXT: vpxor %xmm1, %xmm1, %xmm1
+; AVX512BF16-NEXT: vpunpckhwd {{.*#+}} xmm2 = xmm1[4],xmm0[4],xmm1[5],xmm0[5],xmm1[6],xmm0[6],xmm1[7],xmm0[7]
+; AVX512BF16-NEXT: vroundss $12, %xmm2, %xmm2, %xmm2
+; AVX512BF16-NEXT: vcvtneps2bf16 %xmm2, %xmm2
+; AVX512BF16-NEXT: vmovd %xmm2, %esi
; AVX512BF16-NEXT: vpextrw $3, %xmm0, %edi
-; AVX512BF16-NEXT: shll $16, %edi
-; AVX512BF16-NEXT: vmovd %edi, %xmm1
-; AVX512BF16-NEXT: vroundss $12, %xmm1, %xmm1, %xmm1
-; AVX512BF16-NEXT: vcvtneps2bf16 %xmm1, %xmm1
-; AVX512BF16-NEXT: vmovd %xmm1, %edi
+; AVX512BF16-NEXT: vmovd %edi, %xmm2
+; AVX512BF16-NEXT: vpslld $16, %xmm2, %xmm2
+; AVX512BF16-NEXT: vroundss $12, %xmm2, %xmm2, %xmm2
+; AVX512BF16-NEXT: vcvtneps2bf16 %xmm2, %xmm2
+; AVX512BF16-NEXT: vmovd %xmm2, %edi
; AVX512BF16-NEXT: vpextrw $2, %xmm0, %r8d
-; AVX512BF16-NEXT: shll $16, %r8d
-; AVX512BF16-NEXT: vmovd %r8d, %xmm1
-; AVX512BF16-NEXT: vroundss $12, %xmm1, %xmm1, %xmm1
-; AVX512BF16-NEXT: vcvtneps2bf16 %xmm1, %xmm1
-; AVX512BF16-NEXT: vmovd %xmm1, %r8d
-; AVX512BF16-NEXT: vpextrw $1, %xmm0, %r9d
-; AVX512BF16-NEXT: shll $16, %r9d
-; AVX512BF16-NEXT: vmovd %r9d, %xmm1
+; AVX512BF16-NEXT: vmovd %r8d, %xmm2
+; AVX512BF16-NEXT: vpslld $16, %xmm2, %xmm2
+; AVX512BF16-NEXT: vroundss $12, %xmm2, %xmm2, %xmm2
+; AVX512BF16-NEXT: vcvtneps2bf16 %xmm2, %xmm2
+; AVX512BF16-NEXT: vmovd %xmm2, %r8d
+; AVX512BF16-NEXT: vpblendw {{.*#+}} xmm1 = xmm1[0],xmm0[1],xmm1[2],xmm0[3],xmm1[4],xmm0[5],xmm1[6],xmm0[7]
; AVX512BF16-NEXT: vroundss $12, %xmm1, %xmm1, %xmm1
; AVX512BF16-NEXT: vcvtneps2bf16 %xmm1, %xmm1
-; AVX512BF16-NEXT: vmovd %xmm0, %r9d
-; AVX512BF16-NEXT: shll $16, %r9d
-; AVX512BF16-NEXT: vmovd %r9d, %xmm0
+; AVX512BF16-NEXT: vpslld $16, %xmm0, %xmm0
; AVX512BF16-NEXT: vroundss $12, %xmm0, %xmm0, %xmm0
; AVX512BF16-NEXT: vcvtneps2bf16 %xmm0, %xmm0
; AVX512BF16-NEXT: vpunpcklwd {{.*#+}} xmm0 = xmm0[0],xmm1[0],xmm0[1],xmm1[1],xmm0[2],xmm1[2],xmm0[3],xmm1[3]
@@ -417,9 +384,7 @@ define bfloat @nearbyint_bf16(bfloat %a) {
;
; AVX512BF16-LABEL: nearbyint_bf16:
; AVX512BF16: # %bb.0:
-; AVX512BF16-NEXT: vpextrw $0, %xmm0, %eax
-; AVX512BF16-NEXT: shll $16, %eax
-; AVX512BF16-NEXT: vmovd %eax, %xmm0
+; AVX512BF16-NEXT: vpslld $16, %xmm0, %xmm0
; AVX512BF16-NEXT: vroundss $12, %xmm0, %xmm0, %xmm0
; AVX512BF16-NEXT: vcvtneps2bf16 %xmm0, %xmm0
; AVX512BF16-NEXT: retq
@@ -436,49 +401,44 @@ define <8 x bfloat> @roundeven_v8bf16(<8 x bfloat> %a) {
; AVX512BF16-LABEL: roundeven_v8bf16:
; AVX512BF16: # %bb.0:
; AVX512BF16-NEXT: vpextrw $7, %xmm0, %eax
-; AVX512BF16-NEXT: shll $16, %eax
; AVX512BF16-NEXT: vmovd %eax, %xmm1
+; AVX512BF16-NEXT: vpslld $16, %xmm1, %xmm1
; AVX512BF16-NEXT: vroundss $8, %xmm1, %xmm1, %xmm1
; AVX512BF16-NEXT: vcvtneps2bf16 %xmm1, %xmm1
; AVX512BF16-NEXT: vmovd %xmm1, %eax
; AVX512BF16-NEXT: vpextrw $6, %xmm0, %ecx
-; AVX512BF16-NEXT: shll $16, %ecx
; AVX512BF16-NEXT: vmovd %ecx, %xmm1
+; AVX512BF16-NEXT: vpslld $16, %xmm1, %xmm1
; AVX512BF16-NEXT: vroundss $8, %xmm1, %xmm1, %xmm1
; AVX512BF16-NEXT: vcvtneps2bf16 %xmm1, %xmm1
; AVX512BF16-NEXT: vmovd %xmm1, %ecx
; AVX512BF16-NEXT: vpextrw $5, %xmm0, %edx
-; AVX512BF16-NEXT: shll $16, %edx
; AVX512BF16-NEXT: vmovd %edx, %xmm1
+; AVX512BF16-NEXT: vpslld $16, %xmm1, %xmm1
; AVX512BF16-NEXT: vroundss $8, %xmm1, %xmm1, %xmm1
; AVX512BF16-NEXT: vcvtneps2bf16 %xmm1, %xmm1
; AVX512BF16-NEXT: vmovd %xmm1, %edx
-; AVX512BF16-NEXT: vpextrw $4, %xmm0, %esi
-; AVX512BF16-NEXT: shll $16, %esi
-; AVX512BF16-NEXT: vmovd %esi, %xmm1
-; AVX512BF16-NEXT: vroundss $8, %xmm1, %xmm1, %xmm1
-; AVX512BF16-NEXT: vcvtneps2bf16 %xmm1, %xmm1
-; AVX512BF16-NEXT: vmovd %xmm1, %esi
+; AVX512BF16-NEXT: vpxor %xmm1, %xmm1, %xmm1
+; AVX512BF16-NEXT: vpunpckhwd {{.*#+}} xmm2 = xmm1[4],xmm0[4],xmm1[5],xmm0[5],xmm1[6],xmm0[6],xmm1[7],xmm0[7]
+; AVX512BF16-NEXT: vroundss $8, %xmm2, %xmm2, %xmm2
+; AVX512BF16-NEXT: vcvtneps2bf16 %xmm2, %xmm2
+; AVX512BF16-NEXT: vmovd %xmm2, %esi
; AVX512BF16-NEXT: vpextrw $3, %xmm0, %edi
-; AVX512BF16-NEXT: shll $16, %edi
-; AVX512BF16-NEXT: vmovd %edi, %xmm1
-; AVX512BF16-NEXT: vroundss $8, %xmm1, %xmm1, %xmm1
-; AVX512BF16-NEXT: vcvtneps2bf16 %xmm1, %xmm1
-; AVX512BF16-NEXT: vmovd %xmm1, %edi
+; AVX512BF16-NEXT: vmovd %edi, %xmm2
+; AVX512BF16-NEXT: vpslld $16, %xmm2, %xmm2
+; AVX512BF16-NEXT: vroundss $8, %xmm2, %xmm2, %xmm2
+; AVX512BF16-NEXT: vcvtneps2bf16 %xmm2, %xmm2
+; AVX512BF16-NEXT: vmovd %xmm2, %edi
; AVX512BF16-NEXT: vpextrw $2, %xmm0, %r8d
-; AVX512BF16-NEXT: shll $16, %r8d
-; AVX512BF16-NEXT: vmovd %r8d, %xmm1
-; AVX512BF16-NEXT: vroundss $8, %xmm1, %xmm1, %xmm1
-; AVX512BF16-NEXT: vcvtneps2bf16 %xmm1, %xmm1
-; AVX512BF16-NEXT: vmovd %xmm1, %r8d
-; AVX512BF16-NEXT: vpextrw $1, %xmm0, %r9d
-; AVX512BF16-NEXT: shll $16, %r9d
-; AVX512BF16-NEXT: vmovd %r9d, %xmm1
+; AVX512BF16-NEXT: vmovd %r8d, %xmm2
+; AVX512BF16-NEXT: vpslld $16, %xmm2, %xmm2
+; AVX512BF16-NEXT: vroundss $8, %xmm2, %xmm2, %xmm2
+; AVX512BF16-NEXT: vcvtneps2bf16 %xmm2, %xmm2
+; AVX512BF16-NEXT: vmovd %xmm2, %r8d
+; AVX512BF16-NEXT: vpblendw {{.*#+}} xmm1 = xmm1[0],xmm0[1],xmm1[2],xmm0[3],xmm1[4],xmm0[5],xmm1[6],xmm0[7]
; AVX512BF16-NEXT: vroundss $8, %xmm1, %xmm1, %xmm1
; AVX512BF16-NEXT: vcvtneps2bf16 %xmm1, %xmm1
-; AVX512BF16-NEXT: vmovd %xmm0, %r9d
-; AVX512BF16-NEXT: shll $16, %r9d
-; AVX512BF16-NEXT: vmovd %r9d, %xmm0
+; AVX512BF16-NEXT: vpslld $16, %xmm0, %xmm0
; AVX512BF16-NEXT: vroundss $8, %xmm0, %xmm0, %xmm0
; AVX512BF16-NEXT: vcvtneps2bf16 %xmm0, %xmm0
; AVX512BF16-NEXT: vpunpcklwd {{.*#+}} xmm0 = xmm0[0],xmm1[0],xmm0[1],xmm1[1],xmm0[2],xmm1[2],xmm0[3],xmm1[3]
@@ -501,9 +461,7 @@ define bfloat @roundeven_bf16(bfloat %a) {
;
; AVX512BF16-LABEL: roundeven_bf16:
; AVX512BF16: # %bb.0:
-; AVX512BF16-NEXT: vpextrw $0, %xmm0, %eax
-; AVX512BF16-NEXT: shll $16, %eax
-; AVX512BF16-NEXT: vmovd %eax, %xmm0
+; AVX512BF16-NEXT: vpslld $16, %xmm0, %xmm0
; AVX512BF16-NEXT: vroundss $8, %xmm0, %xmm0, %xmm0
; AVX512BF16-NEXT: vcvtneps2bf16 %xmm0, %xmm0
; AVX512BF16-NEXT: retq
@@ -536,108 +494,97 @@ define <16 x bfloat> @floor_v16bf16(<16 x bfloat> %a) {
; AVX512BF16-NEXT: .cfi_offset %r15, -24
; AVX512BF16-NEXT: .cfi_offset %rbp, -16
; AVX512BF16-NEXT: vpextrw $7, %xmm0, %eax
-; AVX512BF16-NEXT: shll $16, %eax
; AVX512BF16-NEXT: vmovd %eax, %xmm1
+; AVX512BF16-NEXT: vpslld $16, %xmm1, %xmm1
; AVX512BF16-NEXT: vroundss $9, %xmm1, %xmm1, %xmm1
; AVX512BF16-NEXT: vcvtneps2bf16 %xmm1, %xmm1
; AVX512BF16-NEXT: vmovd %xmm1, %eax
; AVX512BF16-NEXT: vpextrw $6, %xmm0, %ecx
-; AVX512BF16-NEXT: shll $16, %ecx
; AVX512BF16-NEXT: vmovd %ecx, %xmm1
+; AVX512BF16-NEXT: vpslld $16, %xmm1, %xmm1
; AVX512BF16-NEXT: vroundss $9, %xmm1, %xmm1, %xmm1
; AVX512BF16-NEXT: vcvtneps2bf16 %xmm1, %xmm1
; AVX512BF16-NEXT: vmovd %xmm1, %ecx
; AVX512BF16-NEXT: vpextrw $5, %xmm0, %edx
-; AVX512BF16-NEXT: shll $16, %edx
; AVX512BF16-NEXT: vmovd %edx, %xmm1
+; AVX512BF16-NEXT: vpslld $16, %xmm1, %xmm1
; AVX512BF16-NEXT: vroundss $9, %xmm1, %xmm1, %xmm1
; AVX512BF16-NEXT: vcvtneps2bf16 %xmm1, %xmm1
; AVX512BF16-NEXT: vmovd %xmm1, %edx
-; AVX512BF16-NEXT: vpextrw $4, %xmm0, %esi
-; AVX512BF16-NEXT: shll $16, %esi
-; AVX512BF16-NEXT: vmovd %esi, %xmm1
-; AVX512BF16-NEXT: vroundss $9, %xmm1, %xmm1, %xmm1
-; AVX512BF16-NEXT: vcvtneps2bf16 %xmm1, %xmm1
-; AVX512BF16-NEXT: vmovd %xmm1, %esi
-; AVX512BF16-NEXT: vpextrw $3, %xmm0, %edi
-; AVX512BF16-NEXT: shll $16, %edi
-; AVX512BF16-NEXT: vmovd %edi, %xmm1
-; AVX512BF16-NEXT: vroundss $9, %xmm1, %xmm1, %xmm1
-; AVX512BF16-NEXT: vcvtneps2bf16 %xmm1, %xmm1
-; AVX512BF16-NEXT: vmovd %xmm1, %edi
-; AVX512BF16-NEXT: vpextrw $2, %xmm0, %r8d
-; AVX512BF16-NEXT: shll $16, %r8d
-; AVX512BF16-NEXT: vmovd %r8d, %xmm1
-; AVX512BF16-NEXT: vroundss $9, %xmm1, %xmm1, %xmm1
-; AVX512BF16-NEXT: vcvtneps2bf16 %xmm1, %xmm1
-; AVX512BF16-NEXT: vmovd %xmm1, %r8d
-; AVX512BF16-NEXT: vmovd %xmm0, %r9d
-; AVX512BF16-NEXT: shll $16, %r9d
-; AVX512BF16-NEXT: vmovd %r9d, %xmm1
-; AVX512BF16-NEXT: vroundss $9, %xmm1, %xmm1, %xmm1
-; AVX512BF16-NEXT: vcvtneps2bf16 %xmm1, %xmm1
-; AVX512BF16-NEXT: vpextrw $1, %xmm0, %r9d
-; AVX512BF16-NEXT: shll $16, %r9d
-; AVX512BF16-NEXT: vmovd %r9d, %xmm2
+; AVX512BF16-NEXT: vpxor %xmm1, %xmm1, %xmm1
+; AVX512BF16-NEXT: vpunpckhwd {{.*#+}} xmm2 = xmm1[4],xmm0[4],xmm1[5],xmm0[5],xmm1[6],xmm0[6],xmm1[7],xmm0[7]
; AVX512BF16-NEXT: vroundss $9, %xmm2, %xmm2, %xmm2
; AVX512BF16-NEXT: vcvtneps2bf16 %xmm2, %xmm2
-; AVX512BF16-NEXT: vmovd %xmm2, %r9d
-; AVX512BF16-NEXT: vextracti128 $1, %ymm0, %xmm0
-; AVX512BF16-NEXT: vpextrw $7, %xmm0, %r10d
-; AVX512BF16-NEXT: shll $16, %r10d
-; AVX512BF16-NEXT: vmovd %r10d, %xmm2
-; AVX512BF16-NEXT: vroundss $9, %xmm2, %xmm2, %xmm2
-; AVX512BF16-NEXT: vcvtneps2bf16 %xmm2, %xmm2
-; AVX512BF16-NEXT: vmovd %xmm2, %r10d
-; AVX512BF16-NEXT: vpextrw $6, %xmm0, %r11d
-; AVX512BF16-NEXT: shll $16, %r11d
-; AVX512BF16-NEXT: vmovd %r11d, %xmm2
+; AVX512BF16-NEXT: vmovd %xmm2, %esi
+; AVX512BF16-NEXT: vpextrw $3, %xmm0, %edi
+; AVX512BF16-NEXT: vmovd %edi, %xmm2
+; AVX512BF16-NEXT: vpslld $16, %xmm2, %xmm2
; AVX512BF16-NEXT: vroundss $9, %xmm2, %xmm2, %xmm2
; AVX512BF16-NEXT: vcvtneps2bf16 %xmm2, %xmm2
-; AVX512BF16-NEXT: vmovd %xmm2, %r11d
-; AVX512BF16-NEXT: vpextrw $5, %xmm0, %ebx
-; AVX512BF16-NEXT: shll $16, %ebx
-; AVX512BF16-NEXT: vmovd %ebx, %xmm2
+; AVX512BF16-NEXT: vmovd %xmm2, %edi
+; AVX512BF16-NEXT: vpextrw $2, %xmm0, %r8d
+; AVX512BF16-NEXT: vmovd %r8d, %xmm2
+; AVX512BF16-NEXT: vpslld $16, %xmm2, %xmm2
; AVX512BF16-NEXT: vroundss $9, %xmm2, %xmm2, %xmm2
; AVX512BF16-NEXT: vcvtneps2bf16 %xmm2, %xmm2
-; AVX512BF16-NEXT: vmovd %xmm2, %ebx
-; AVX512BF16-NEXT: vpextrw $4, %xmm0, %ebp
-; AVX512BF16-NEXT: shll $16, %ebp
-; AVX512BF16-NEXT: vmovd %ebp, %xmm2
+; AVX512BF16-NEXT: vmovd %xmm2, %r8d
+; AVX512BF16-NEXT: vpslld $16, %xmm0, %xmm2
; AVX512BF16-NEXT: vroundss $9, %xmm2, %xmm2, %xmm2
; AVX512BF16-NEXT: vcvtneps2bf16 %xmm2, %xmm2
-; AVX512BF16-NEXT: vmovd %xmm2, %ebp
+; AVX512BF16-NEXT: vpblendw {{.*#+}} xmm3 = xmm1[0],xmm0[1],xmm1[2],xmm0[3],xmm1[4],xmm0[5],xmm1[6],xmm0[7]
+; AVX512BF16-NEXT: vroundss $9, %xmm3, %xmm3, %xmm3
+; AVX512BF16-NEXT: vcvtneps2bf16 %xmm3, %xmm3
+; AVX512BF16-NEXT: vmovd %xmm3, %r9d
+; AVX512BF16-NEXT: vextracti128 $1, %ymm0, %xmm0
+; AVX512BF16-NEXT: vpextrw $7, %xmm0, %r10d
+; AVX512BF16-NEXT: vmovd %r10d, %xmm3
+; AVX512BF16-NEXT: vpslld $16, %xmm3, %xmm3
+; AVX512BF16-NEXT: vroundss $9, %xmm3, %xmm3, %xmm3
+; AVX512BF16-NEXT: vcvtneps2bf16 %xmm3, %xmm3
+; AVX512BF16-NEXT: vmovd %xmm3, %r10d
+; AVX512BF16-NEXT: vpextrw $6, %xmm0, %r11d
+; AVX512BF16-NEXT: vmovd %r11d, %xmm3
+; AVX512BF16-NEXT: vpslld $16, %xmm3, %xmm3
+; AVX512BF16-NEXT: vroundss $9, %xmm3, %xmm3, %xmm3
+; AVX512BF16-NEXT: vcvtneps2bf16 %xmm3, %xmm3
+; AVX512BF16-NEXT: vmovd %xmm3, %r11d
+; AVX512BF16-NEXT: vpextrw $5, %xmm0, %ebx
+; AVX512BF16-NEXT: vmovd %ebx, %xmm3
+; AVX512BF16-NEXT: vpslld $16, %xmm3, %xmm3
+; AVX512BF16-NEXT: vroundss $9, %xmm3, %xmm3, %xmm3
+; AVX512BF16-NEXT: vcvtneps2bf16 %xmm3, %xmm3
+; AVX512BF16-NEXT: vmovd %xmm3, %ebx
+; AVX512BF16-NEXT: vpunpckhwd {{.*#+}} xmm3 = xmm1[4],xmm0[4],xmm1[5],xmm0[5],xmm1[6],xmm0[6],xmm1[7],xmm0[7]
+; AVX512BF16-NEXT: vroundss $9, %xmm3, %xmm3, %xmm3
+; AVX512BF16-NEXT: vcvtneps2bf16 %xmm3, %xmm3
+; AVX512BF16-NEXT: vmovd %xmm3, %ebp
; AVX512BF16-NEXT: vpextrw $3, %xmm0, %r14d
-; AVX512BF16-NEXT: shll $16, %r14d
-; AVX512BF16-NEXT: vmovd %r14d, %xmm2
-; AVX512BF16-NEXT: vroundss $9, %xmm2, %xmm2, %xmm2
-; AVX512BF16-NEXT: vcvtneps2bf16 %xmm2, %xmm2
-; AVX512BF16-NEXT: vmovd %xmm2, %r14d
+; AVX512BF16-NEXT: vmovd %r14d, %xmm3
+; AVX512BF16-NEXT: vpslld $16, %xmm3, %xmm3
+; AVX512BF16-NEXT: vroundss $9, %xmm3, %xmm3, %xmm3
+; AVX512BF16-NEXT: vcvtneps2bf16 %xmm3, %xmm3
+; AVX512BF16-NEXT: vmovd %xmm3, %r14d
; AVX512BF16-NEXT: vpextrw $2, %xmm0, %r15d
-; AVX512BF16-NEXT: shll $16, %r15d
-; AVX512BF16-NEXT: vmovd %r15d, %xmm2
-; AVX512BF16-NEXT: vroundss $9, %xmm2, %xmm2, %xmm2
-; AVX512BF16-NEXT: vcvtneps2bf16 %xmm2, %xmm2
-; AVX512BF16-NEXT: vmovd %xmm2, %r15d
-; AVX512BF16-NEXT: vmovd %xmm0, %r12d
-; AVX512BF16-NEXT: shll $16, %r12d
-; AVX512BF16-NEXT: vmovd %r12d, %xmm2
-; AVX512BF16-NEXT: vroundss $9, %xmm2, %xmm2, %xmm2
-; AVX512BF16-NEXT: vcvtneps2bf16 %xmm2, %xmm2
-; AVX512BF16-NEXT: vpextrw $1, %xmm0, %r12d
-; AVX512BF16-NEXT: shll $16, %r12d
-; AVX512BF16-NEXT: vmovd %r12d, %xmm0
+; AVX512BF16-NEXT: vmovd %r15d, %xmm3
+; AVX512BF16-NEXT: vpslld $16, %xmm3, %xmm3
+; AVX512BF16-NEXT: vroundss $9, %xmm3, %xmm3, %xmm3
+; AVX512BF16-NEXT: vcvtneps2bf16 %xmm3, %xmm3
+; AVX512BF16-NEXT: vmovd %xmm3, %r15d
+; AVX512BF16-NEXT: vpslld $16, %xmm0, %xmm3
+; AVX512BF16-NEXT: vroundss $9, %xmm3, %xmm3, %xmm3
+; AVX512BF16-NEXT: vcvtneps2bf16 %xmm3, %xmm3
+; AVX512BF16-NEXT: vpblendw {{.*#+}} xmm0 = xmm1[0],xmm0[1],xmm1[2],xmm0[3],xmm1[4],xmm0[5],xmm1[6],xmm0[7]
; AVX512BF16-NEXT: vroundss $9, %xmm0, %xmm0, %xmm0
; AVX512BF16-NEXT: vcvtneps2bf16 %xmm0, %xmm0
; AVX512BF16-NEXT: vmovd %xmm0, %r12d
-; AVX512BF16-NEXT: vpinsrw $1, %r12d, %xmm2, %xmm0
+; AVX512BF16-NEXT: vpinsrw $1, %r12d, %xmm3, %xmm0
; AVX512BF16-NEXT: vpinsrw $2, %r15d, %xmm0, %xmm0
; AVX512BF16-NEXT: vpinsrw $3, %r14d, %xmm0, %xmm0
; AVX512BF16-NEXT: vpinsrw $4, %ebp, %xmm0, %xmm0
; AVX512BF16-NEXT: vpinsrw $5, %ebx, %xmm0, %xmm0
; AVX512BF16-NEXT: vpinsrw $6, %r11d, %xmm0, %xmm0
; AVX512BF16-NEXT: vpinsrw $7, %r10d, %xmm0, %xmm0
-; AVX512BF16-NEXT: vpinsrw $1, %r9d, %xmm1, %xmm1
+; AVX512BF16-NEXT: vpinsrw $1, %r9d, %xmm2, %xmm1
; AVX512BF16-NEXT: vpinsrw $2, %r8d, %xmm1, %xmm1
; AVX512BF16-NEXT: vpinsrw $3, %edi, %xmm1, %xmm1
; AVX512BF16-NEXT: vpinsrw $4, %esi, %xmm1, %xmm1
@@ -669,227 +616,204 @@ define <32 x bfloat> @floor_v32bf16(<32 x bfloat> %a) {
; AVX512BF16-LABEL: floor_v32bf16:
; AVX512BF16: # %bb.0:
; AVX512BF16-NEXT: vpextrw $7, %xmm0, %eax
-; AVX512BF16-NEXT: shll $16, %eax
-; AVX512BF16-NEXT: vmovd %eax, %xmm1
+; AVX512BF16-NEXT: vmovd %eax, %xmm23
; AVX512BF16-NEXT: vpextrw $6, %xmm0, %eax
-; AVX512BF16-NEXT: shll $16, %eax
; AVX512BF16-NEXT: vmovd %eax, %xmm2
; AVX512BF16-NEXT: vpextrw $5, %xmm0, %eax
-; AVX512BF16-NEXT: shll $16, %eax
; AVX512BF16-NEXT: vmovd %eax, %xmm3
-; AVX512BF16-NEXT: vpextrw $4, %xmm0, %eax
-; AVX512BF16-NEXT: shll $16, %eax
-; AVX512BF16-NEXT: vmovd %eax, %xmm4
+; AVX512BF16-NEXT: vpxor %xmm5, %xmm5, %xmm5
; AVX512BF16-NEXT: vpextrw $3, %xmm0, %eax
-; AVX512BF16-NEXT: shll $16, %eax
-; AVX512BF16-NEXT: vmovd %eax, %xmm5
+; AVX512BF16-NEXT: vmovd %eax, %xmm4
; AVX512BF16-NEXT: vpextrw $2, %xmm0, %eax
-; AVX512BF16-NEXT: shll $16, %eax
; AVX512BF16-NEXT: vmovd %eax, %xmm6
-; AVX512BF16-NEXT: vmovd %xmm0, %eax
-; AVX512BF16-NEXT: shll $16, %eax
+; AVX512BF16-NEXT: vextracti128 $1, %ymm0, %xmm9
+; AVX512BF16-NEXT: vpextrw $7, %xmm9, %eax
; AVX512BF16-NEXT: vmovd %eax, %xmm7
-; AVX512BF16-NEXT: vpextrw $1, %xmm0, %eax
-; AVX512BF16-NEXT: shll $16, %eax
+; AVX512BF16-NEXT: vpextrw $6, %xmm9, %eax
; AVX512BF16-NEXT: vmovd %eax, %xmm8
-; AVX512BF16-NEXT: vextracti32x4 $1, %ymm0, %xmm16
-; AVX512BF16-NEXT: vpextrw $7, %xmm16, %eax
-; AVX512BF16-NEXT: shll $16, %eax
-; AVX512BF16-NEXT: vmovd %eax, %xmm9
-; AVX512BF16-NEXT: vpextrw $6, %xmm16, %eax
-; AVX512BF16-NEXT: shll $16, %eax
+; AVX512BF16-NEXT: vpextrw $5, %xmm9, %eax
; AVX512BF16-NEXT: vmovd %eax, %xmm10
-; AVX512BF16-NEXT: vpextrw $5, %xmm16, %eax
-; AVX512BF16-NEXT: shll $16, %eax
+; AVX512BF16-NEXT: vpextrw $3, %xmm9, %eax
; AVX512BF16-NEXT: vmovd %eax, %xmm11
-; AVX512BF16-NEXT: vpextrw $4, %xmm16, %eax
-; AVX512BF16-NEXT: shll $16, %eax
+; AVX512BF16-NEXT: vpextrw $2, %xmm9, %eax
; AVX512BF16-NEXT: vmovd %eax, %xmm12
-; AVX512BF16-NEXT: vpextrw $3, %xmm16, %eax
-; AVX512BF16-NEXT: shll $16, %eax
-; AVX512BF16-NEXT: vmovd %eax, %xmm13
-; AVX512BF16-NEXT: vpextrw $2, %xmm16, %eax
-; AVX512BF16-NEXT: shll $16, %eax
+; AVX512BF16-NEXT: vpblendw {{.*#+}} xmm1 = xmm5[0],xmm9[1],xmm5[2],xmm9[3],xmm5[4],xmm9[5],xmm5[6],xmm9[7]
+; AVX512BF16-NEXT: vroundss $9, %xmm1, %xmm1, %xmm13
+; AVX512BF16-NEXT: vextracti32x4 $2, %zmm0, %xmm1
+; AVX512BF16-NEXT: vpextrw $7, %xmm1, %eax
; AVX512BF16-NEXT: vmovd %eax, %xmm14
-; AVX512BF16-NEXT: vmovd %xmm16, %eax
-; AVX512BF16-NEXT: shll $16, %eax
+; AVX512BF16-NEXT: vpextrw $6, %xmm1, %eax
; AVX512BF16-NEXT: vmovd %eax, %xmm15
-; AVX512BF16-NEXT: vpextrw $1, %xmm16, %eax
-; AVX512BF16-NEXT: shll $16, %eax
+; AVX512BF16-NEXT: vpextrw $5, %xmm1, %eax
; AVX512BF16-NEXT: vmovd %eax, %xmm16
-; AVX512BF16-NEXT: vextracti32x4 $2, %zmm0, %xmm24
-; AVX512BF16-NEXT: vpextrw $7, %xmm24, %eax
-; AVX512BF16-NEXT: shll $16, %eax
-; AVX512BF16-NEXT: vmovd %eax, %xmm17
-; AVX512BF16-NEXT: vpextrw $6, %xmm24, %eax
-; AVX512BF16-NEXT: shll $16, %eax
-; AVX512BF16-NEXT: vmovd %eax, %xmm18
-; AVX512BF16-NEXT: vpextrw $5, %xmm24, %eax
-; AVX512BF16-NEXT: shll $16, %eax
+; AVX512BF16-NEXT: vpunpckhwd {{.*#+}} xmm17 = xmm5[4],xmm1[4],xmm5[5],xmm1[5],xmm5[6],xmm1[6],xmm5[7],xmm1[7]
+; AVX512BF16-NEXT: vpextrw $3, %xmm1, %eax
; AVX512BF16-NEXT: vmovd %eax, %xmm19
-; AVX512BF16-NEXT: vpextrw $4, %xmm24, %eax
-; AVX512BF16-NEXT: shll $16, %eax
+; AVX512BF16-NEXT: vpextrw $2, %xmm1, %eax
; AVX512BF16-NEXT: vmovd %eax, %xmm20
-; AVX512BF16-NEXT: vpextrw $3, %xmm24, %eax
-; AVX512BF16-NEXT: shll $16, %eax
-; AVX512BF16-NEXT: vmovd %eax, %xmm21
-; AVX512BF16-NEXT: vpextrw $2, %xmm24, %eax
-; AVX512BF16-NEXT: shll $16, %eax
+; AVX512BF16-NEXT: vpslld $16, %xmm1, %xmm21
+; AVX512BF16-NEXT: vpblendw {{.*#+}} xmm1 = xmm5[0],xmm1[1],xmm5[2],xmm1[3],xmm5[4],xmm1[5],xmm5[6],xmm1[7]
+; AVX512BF16-NEXT: vrndscaless $9, %xmm1, %xmm1, %xmm18
+; AVX512BF16-NEXT: vextracti32x4 $3, %zmm0, %xmm1
+; AVX512BF16-NEXT: vpextrw $7, %xmm1, %eax
; AVX512BF16-NEXT: vmovd %eax, %xmm22
-; AVX512BF16-NEXT: vmovd %xmm24, %eax
-; AVX512BF16-NEXT: shll $16, %eax
-; AVX512BF16-NEXT: vmovd %eax, %xmm23
-; AVX512BF16-NEXT: vpextrw $1, %xmm24, %eax
-; AVX512BF16-NEXT: shll $16, %eax
+; AVX512BF16-NEXT: vpextrw $6, %xmm1, %eax
; AVX512BF16-NEXT: vmovd %eax, %xmm24
-; AVX512BF16-NEXT: vextracti32x4 $3, %zmm0, %xmm25
-; AVX512BF16-NEXT: vpextrw $7, %xmm25, %eax
-; AVX512BF16-NEXT: shll $16, %eax
-; AVX512BF16-NEXT: vmovd %eax, %xmm0
-; AVX512BF16-NEXT: vpextrw $6, %xmm25, %eax
-; AVX512BF16-NEXT: shll $16, %eax
-; AVX512BF16-NEXT: vmovd %eax, %xmm26
-; AVX512BF16-NEXT: vpextrw $5, %xmm25, %eax
-; AVX512BF16-NEXT: shll $16, %eax
+; AVX512BF16-NEXT: vpextrw $5, %xmm1, %eax
+; AVX512BF16-NEXT: vmovd %eax, %xmm25
+; AVX512BF16-NEXT: vpunpckhwd {{.*#+}} xmm26 = xmm5[4],xmm1[4],xmm5[5],xmm1[5],xmm5[6],xmm1[6],xmm5[7],xmm1[7]
+; AVX512BF16-NEXT: vpextrw $3, %xmm1, %eax
; AVX512BF16-NEXT: vmovd %eax, %xmm27
-; AVX512BF16-NEXT: vpextrw $4, %xmm25, %eax
-; AVX512BF16-NEXT: shll $16, %eax
-; AVX512BF16-NEXT: vmovd %eax, %xmm28
-; AVX512BF16-NEXT: vpextrw $3, %xmm25, %eax
-; AVX512BF16-NEXT: shll $16, %eax
+; AVX512BF16-NEXT: vpextrw $2, %xmm1, %eax
+; AVX512BF16-NEXT: vpslld $16, %xmm1, %xmm28
+; AVX512BF16-NEXT: vpblendw {{.*#+}} xmm1 = xmm5[0],xmm1[1],xmm5[2],xmm1[3],xmm5[4],xmm1[5],xmm5[6],xmm1[7]
+; AVX512BF16-NEXT: vroundss $9, %xmm1, %xmm1, %xmm1
; AVX512BF16-NEXT: vmovd %eax, %xmm29
-; AVX512BF16-NEXT: vpextrw $2, %xmm25, %eax
-; AVX512BF16-NEXT: shll $16, %eax
-; AVX512BF16-NEXT: vmovd %eax, %xmm30
-; AVX512BF16-NEXT: vmovd %xmm25, %eax
-; AVX512BF16-NEXT: shll $16, %eax
-; AVX512BF16-NEXT: vmovd %eax, %xmm31
-; AVX512BF16-NEXT: vpextrw $1, %xmm25, %eax
-; AVX512BF16-NEXT: shll $16, %eax
-; AVX512BF16-NEXT: vmovd %eax, %xmm25
-; AVX512BF16-NEXT: vrndscaless $9, %xmm25, %xmm25, %xmm25
-; AVX512BF16-NEXT: vcvtneps2bf16 %xmm25, %xmm25
-; AVX512BF16-NEXT: vmovd %xmm25, %eax
-; AVX512BF16-NEXT: vrndscaless $9, %xmm30, %xmm30, %xmm25
-; AVX512BF16-NEXT: vcvtneps2bf16 %xmm25, %xmm25
-; AVX512BF16-NEXT: vrndscaless $9, %xmm31, %xmm31, %xmm30
-; AVX512BF16-NEXT: vcvtneps2bf16 %xmm30, %xmm30
-; AVX512BF16-NEXT: vpinsrw $1, %eax, %xmm30, %xmm30
-; AVX512BF16-NEXT: vmovd %xmm25, %eax
-; AVX512BF16-NEXT: vrndscaless $9, %xmm29, %xmm29, %xmm25
-; AVX512BF16-NEXT: vcvtneps2bf16 %xmm25, %xmm25
-; AVX512BF16-NEXT: vpinsrw $2, %eax, %xmm30, %xmm29
-; AVX512BF16-NEXT: vmovd %xmm25, %eax
-; AVX512BF16-NEXT: vrndscaless $9, %xmm28, %xmm28, %xmm25
-; AVX512BF16-NEXT: vcvtneps2bf16 %xmm25, %xmm25
-; AVX512BF16-NEXT: vpinsrw $3, %eax, %xmm29, %xmm28
-; AVX512BF16-NEXT: vmovd %xmm25, %eax
-; AVX512BF16-NEXT: vrndscaless $9, %xmm27, %xmm27, %xmm25
-; AVX512BF16-NEXT: vcvtneps2bf16 %xmm25, %xmm25
-; AVX512BF16-NEXT: vpinsrw $4, %eax, %xmm28, %xmm27
-; AVX512BF16-NEXT: vmovd %xmm25, %eax
-; AVX512BF16-NEXT: vrndscaless $9, %xmm26, %xmm26, %xmm25
-; AVX512BF16-NEXT: vcvtneps2bf16 %xmm25, %xmm25
-; AVX512BF16-NEXT: vpinsrw $5, %eax, %xmm27, %xmm26
-; AVX512BF16-NEXT: vmovd %xmm25, %eax
-; AVX512BF16-NEXT: vroundss $9, %xmm0, %xmm0, %xmm0
-; AVX512BF16-NEXT: vcvtneps2bf16 %xmm0, %xmm0
-; AVX512BF16-NEXT: vpinsrw $6, %eax, %xmm26, %xmm25
-; AVX512BF16-NEXT: vmovd %xmm0, %eax
-; AVX512BF16-NEXT: vrndscaless $9, %xmm24, %xmm24, %xmm0
-; AVX512BF16-NEXT: vcvtneps2bf16 %xmm0, %xmm24
-; AVX512BF16-NEXT: vpinsrw $7, %eax, %xmm25, %xmm0
-; AVX512BF16-NEXT: vmovd %xmm24, %eax
-; AVX512BF16-NEXT: vrndscaless $9, %xmm22, %xmm22, %xmm22
-; AVX512BF16-NEXT: vcvtneps2bf16 %xmm22, %xmm22
-; AVX512BF16-NEXT: vrndscaless $9, %xmm23, %xmm23, %xmm23
-; AVX512BF16-NEXT: vcvtneps2bf16 %xmm23, %xmm23
-; AVX512BF16-NEXT: vpinsrw $1, %eax, %xmm23, %xmm23
-; AVX512BF16-NEXT: vmovd %xmm22, %eax
-; AVX512BF16-NEXT: vrndscaless $9, %xmm21, %xmm21, %xmm21
-; AVX512BF16-NEXT: vcvtneps2bf16 %xmm21, %xmm21
-; AVX512BF16-NEXT: vpinsrw $2, %eax, %xmm23, %xmm22
-; AVX512BF16-NEXT: vmovd %xmm21, %eax
-; AVX512BF16-NEXT: vrndscaless $9, %xmm20, %xmm20, %xmm20
+; AVX512BF16-NEXT: vcvtneps2bf16 %xmm1, %xmm1
+; AVX512BF16-NEXT: vmovd %xmm1, %eax
+; AVX512BF16-NEXT: vpslld $16, %xmm29, %xmm1
+; AVX512BF16-NEXT: vroundss $9, %xmm1, %xmm1, %xmm1
+; AVX512BF16-NEXT: vcvtneps2bf16 %xmm1, %xmm1
+; AVX512BF16-NEXT: vrndscaless $9, %xmm28, %xmm28, %xmm28
+; AVX512BF16-NEXT: vcvtneps2bf16 %xmm28, %xmm28
+; AVX512BF16-NEXT: vpinsrw $1, %eax, %xmm28, %xmm28
+; AVX512BF16-NEXT: vmovd %xmm1, %eax
+; AVX512BF16-NEXT: vpslld $16, %xmm27, %xmm1
+; AVX512BF16-NEXT: vroundss $9, %xmm1, %xmm1, %xmm1
+; AVX512BF16-NEXT: vcvtneps2bf16 %xmm1, %xmm1
+; AVX512BF16-NEXT: vpinsrw $2, %eax, %xmm28, %xmm27
+; AVX512BF16-NEXT: vmovd %xmm1, %eax
+; AVX512BF16-NEXT: vrndscaless $9, %xmm26, %xmm26, %xmm1
+; AVX512BF16-NEXT: vcvtneps2bf16 %xmm1, %xmm1
+; AVX512BF16-NEXT: vpinsrw $3, %eax, %xmm27, %xmm26
+; AVX512BF16-NEXT: vmovd %xmm1, %eax
+; AVX512BF16-NEXT: vpslld $16, %xmm25, %xmm1
+; AVX512BF16-NEXT: vroundss $9, %xmm1, %xmm1, %xmm1
+; AVX512BF16-NEXT: vcvtneps2bf16 %xmm1, %xmm1
+; AVX512BF16-NEXT: vpinsrw $4, %eax, %xmm26, %xmm25
+; AVX512BF16-NEXT: vmovd %xmm1, %eax
+; AVX512BF16-NEXT: vpslld $16, %xmm24, %xmm1
+; AVX512BF16-NEXT: vroundss $9, %xmm1, %xmm1, %xmm1
+; AVX512BF16-NEXT: vcvtneps2bf16 %xmm1, %xmm1
+; AVX512BF16-NEXT: vpinsrw $5, %eax, %xmm25, %xmm24
+; AVX512BF16-NEXT: vmovd %xmm1, %eax
+; AVX512BF16-NEXT: vpslld $16, %xmm22, %xmm1
+; AVX512BF16-NEXT: vroundss $9, %xmm1, %xmm1, %xmm1
+; AVX512BF16-NEXT: vcvtneps2bf16 %xmm1, %xmm1
+; AVX512BF16-NEXT: vpinsrw $6, %eax, %xmm24, %xmm22
+; AVX512BF16-NEXT: vmovd %xmm1, %eax
+; AVX512BF16-NEXT: vcvtneps2bf16 %xmm18, %xmm1
+; AVX512BF16-NEXT: vpinsrw $7, %eax, %xmm22, %xmm18
+; AVX512BF16-NEXT: vmovd %xmm1, %eax
+; AVX512BF16-NEXT: vpslld $16, %xmm20, %xmm1
+; AVX512BF16-NEXT: vroundss $9, %xmm1, %xmm1, %xmm1
+; AVX512BF16-NEXT: vcvtneps2bf16 %xmm1, %xmm1
+; AVX512BF16-NEXT: vrndscaless $9, %xmm21, %xmm21, %xmm20
; AVX512BF16-NEXT: vcvtneps2bf16 %xmm20, %xmm20
-; AVX512BF16-NEXT: vpinsrw $3, %eax, %xmm22, %xmm21
-; AVX512BF16-NEXT: vmovd %xmm20, %eax
-; AVX512BF16-NEXT: vrndscaless $9, %xmm19, %xmm19, %xmm19
-; AVX512BF16-NEXT: vcvtneps2bf16 %xmm19, %xmm19
-; AVX512BF16-NEXT: vpinsrw $4, %eax, %xmm21, %xmm20
-; AVX512BF16-NEXT: vmovd %xmm19, %eax
-; AVX512BF16-NEXT: vrndscaless $9, %xmm18, %xmm18, %xmm18
-; AVX512BF16-NEXT: vcvtneps2bf16 %xmm18, %xmm18
-; AVX512BF16-NEXT: vpinsrw $5, %eax, %xmm20, %xmm19
-; AVX512BF16-NEXT: vmovd %xmm18, %eax
-; AVX512BF16-NEXT: vrndscaless $9, %xmm17, %xmm17, %xmm17
-; AVX512BF16-NEXT: vcvtneps2bf16 %xmm17, %xmm17
-; AVX512BF16-NEXT: vpinsrw $6, %eax, %xmm19, %xmm18
-; AVX512BF16-NEXT: vmovd %xmm17, %eax
-; AVX512BF16-NEXT: vrndscaless $9, %xmm16, %xmm16, %xmm16
-; AVX512BF16-NEXT: vcvtneps2bf16 %xmm16, %xmm16
-; AVX512BF16-NEXT: vpinsrw $7, %eax, %xmm18, %xmm17
-; AVX512BF16-NEXT: vmovd %xmm16, %eax
-; AVX512BF16-NEXT: vroundss $9, %xmm14, %xmm14, %xmm14
-; AVX512BF16-NEXT: vcvtneps2bf16 %xmm14, %xmm14
-; AVX512BF16-NEXT: vroundss $9, %xmm15, %xmm15, %xmm15
-; AVX512BF16-NEXT: vcvtneps2bf16 %xmm15, %xmm15
-; AVX512BF16-NEXT: vpinsrw $1, %eax, %xmm15, %xmm15
-; AVX512BF16-NEXT: vmovd %xmm14, %eax
-; AVX512BF16-NEXT: vroundss $9, %xmm13, %xmm13, %xmm13
+; AVX512BF16-NEXT: vpinsrw $1, %eax, %xmm20, %xmm20
+; AVX512BF16-NEXT: vmovd %xmm1, %eax
+; AVX512BF16-NEXT: vpslld $16, %xmm19, %xmm1
+; AVX512BF16-NEXT: vroundss $9, %xmm1, %xmm1, %xmm1
+; AVX512BF16-NEXT: vcvtneps2bf16 %xmm1, %xmm1
+; AVX512BF16-NEXT: vpinsrw $2, %eax, %xmm20, %xmm19
+; AVX512BF16-NEXT: vmovd %xmm1, %eax
+; AVX512BF16-NEXT: vrndscaless $9, %xmm17, %xmm17, %xmm1
+; AVX512BF16-NEXT: vcvtneps2bf16 %xmm1, %xmm1
+; AVX512BF16-NEXT: vpinsrw $3, %eax, %xmm19, %xmm17
+; AVX512BF16-NEXT: vmovd %xmm1, %eax
+; AVX512BF16-NEXT: vpslld $16, %xmm16, %xmm1
+; AVX512BF16-NEXT: vroundss $9, %xmm1, %xmm1, %xmm1
+; AVX512BF16-NEXT: vcvtneps2bf16 %xmm1, %xmm1
+; AVX512BF16-NEXT: vpinsrw $4, %eax, %xmm17, %xmm16
+; AVX512BF16-NEXT: vmovd %xmm1, %eax
+; AVX512BF16-NEXT: vpslld $16, %xmm15, %xmm1
+; AVX512BF16-NEXT: vroundss $9, %xmm1, %xmm1, %xmm1
+; AVX512BF16-NEXT: vcvtneps2bf16 %xmm1, %xmm1
+; AVX512BF16-NEXT: vpinsrw $5, %eax, %xmm16, %xmm15
+; AVX512BF16-NEXT: vmovd %xmm1, %eax
+; AVX512BF16-NEXT: vpslld $16, %xmm14, %xmm1
+; AVX512BF16-NEXT: vroundss $9, %xmm1, %xmm1, %xmm1
+; AVX512BF16-NEXT: vcvtneps2bf16 %xmm1, %xmm1
+; AVX512BF16-NEXT: vpinsrw $6, %eax, %xmm15, %xmm14
+; AVX512BF16-NEXT: vmovd %xmm1, %eax
+; AVX512BF16-NEXT: vpslld $16, %xmm12, %xmm1
+; AVX512BF16-NEXT: vroundss $9, %xmm1, %xmm1, %xmm1
+; AVX512BF16-NEXT: vcvtneps2bf16 %xmm1, %xmm1
; AVX512BF16-NEXT: vcvtneps2bf16 %xmm13, %xmm13
-; AVX512BF16-NEXT: vpinsrw $2, %eax, %xmm15, %xmm14
+; AVX512BF16-NEXT: vpinsrw $7, %eax, %xmm14, %xmm12
; AVX512BF16-NEXT: vmovd %xmm13, %eax
-; AVX512BF16-NEXT: vroundss $9, %xmm12, %xmm12, %xmm12
-; AVX512BF16-NEXT: vcvtneps2bf16 %xmm12, %xmm12
-; AVX512BF16-NEXT: vpinsrw $3, %eax, %xmm14, %xmm13
-; AVX512BF16-NEXT: vmovd %xmm12, %eax
-; AVX512BF16-NEXT: vroundss $9, %xmm11, %xmm11, %xmm11
-; AVX512BF16-NEXT: vcvtneps2bf16 %xmm11, %xmm11
-; AVX512BF16-NEXT: vpinsrw $4, %eax, %xmm13, %xmm12
-; AVX512BF16-NEXT: vmovd %xmm11, %eax
-; AVX512BF16-NEXT: vroundss $9, %xmm10, %xmm10, %xmm10
-; AVX512BF16-NEXT: vcvtneps2bf16 %xmm10, %xmm10
-; AVX512BF16-NEXT: vpinsrw $5, %eax, %xmm12, %xmm11
-; AVX512BF16-NEXT: vmovd %xmm10, %eax
-; AVX512BF16-NEXT: vroundss $9, %xmm9, %xmm9, %xmm9
-; AVX512BF16-NEXT: vcvtneps2bf16 %xmm9, %xmm9
-; AVX512BF16-NEXT: vpinsrw $6, %eax, %xmm11, %xmm10
-; AVX512BF16-NEXT: vmovd %xmm9, %eax
-; AVX512BF16-NEXT: vroundss $9, %xmm8, %xmm8, %xmm8
-; AVX512BF16-NEXT: vcvtneps2bf16 %xmm8, %xmm8
-; AVX512BF16-NEXT: vpinsrw $7, %eax, %xmm10, %xmm9
-; AVX512BF16-NEXT: vmovd %xmm8, %eax
-; AVX512BF16-NEXT: vroundss $9, %xmm6, %xmm6, %xmm6
-; AVX512BF16-NEXT: vcvtneps2bf16 %xmm6, %xmm6
-; AVX512BF16-NEXT: vroundss $9, %xmm7, %xmm7, %xmm7
-; AVX512BF16-NEXT: vcvtneps2bf16 %xmm7, %xmm7
-; AVX512BF16-NEXT: vpinsrw $1, %eax, %xmm7, %xmm7
-; AVX512BF16-NEXT: vmovd %xmm6, %eax
+; AVX512BF16-NEXT: vpslld $16, %xmm9, %xmm13
+; AVX512BF16-NEXT: vroundss $9, %xmm13, %xmm13, %xmm13
+; AVX512BF16-NEXT: vcvtneps2bf16 %xmm13, %xmm13
+; AVX512BF16-NEXT: vpinsrw $1, %eax, %xmm13, %xmm13
+; AVX512BF16-NEXT: vmovd %xmm1, %eax
+; AVX512BF16-NEXT: vpslld $16, %xmm11, %xmm1
+; AVX512BF16-NEXT: vroundss $9, %xmm1, %xmm1, %xmm1
+; AVX512BF16-NEXT: vcvtneps2bf16 %xmm1, %xmm1
+; AVX512BF16-NEXT: vpinsrw $2, %eax, %xmm13, %xmm11
+; AVX512BF16-NEXT: vmovd %xmm1, %eax
+; AVX512BF16-NEXT: vpunpckhwd {{.*#+}} xmm1 = xmm5[4],xmm9[4],xmm5[5],xmm9[5],xmm5[6],xmm9[6],xmm5[7],xmm9[7]
+; AVX512BF16-NEXT: vroundss $9, %xmm1, %xmm1, %xmm1
+; AVX512BF16-NEXT: vcvtneps2bf16 %xmm1, %xmm1
+; AVX512BF16-NEXT: vpinsrw $3, %eax, %xmm11, %xmm9
+; AVX512BF16-NEXT: vmovd %xmm1, %eax
+; AVX512BF16-NEXT: vpslld $16, %xmm10, %xmm1
+; AVX512BF16-NEXT: vroundss $9, %xmm1, %xmm1, %xmm1
+; AVX512BF16-NEXT: vcvtneps2bf16 %xmm1, %xmm1
+; AVX512BF16-NEXT: vpinsrw $4, %eax, %xmm9, %xmm9
+; AVX512BF16-NEXT: vmovd %xmm1, %eax
+; AVX512BF16-NEXT: vpslld $16, %xmm8, %xmm1
+; AVX512BF16-NEXT: vroundss $9, %xmm1, %xmm1, %xmm1
+; AVX512BF16-NEXT: vcvtneps2bf16 %xmm1, %xmm1
+; AVX512BF16-NEXT: vpinsrw $5, %eax, %xmm9, %xmm8
+; AVX512BF16-NEXT: vmovd %xmm1, %eax
+; AVX512BF16-NEXT: vpslld $16, %xmm7, %xmm1
+; AVX512BF16-NEXT: vroundss $9, %xmm1, %xmm1, %xmm1
+; AVX512BF16-NEXT: vcvtneps2bf16 %xmm1, %xmm1
+; AVX512BF16-NEXT: vpinsrw $6, %eax, %xmm8, %xmm7
+; AVX512BF16-NEXT: vmovd %xmm1, %eax
+; AVX512BF16-NEXT: vpblendw {{.*#+}} xmm1 = xmm5[0],xmm0[1],xmm5[2],xmm0[3],xmm5[4],xmm0[5],xmm5[6],xmm0[7]
+; AVX512BF16-NEXT: vroundss $9, %xmm1, %xmm1, %xmm1
+; AVX512BF16-NEXT: vcvtneps2bf16 %xmm1, %xmm1
+; AVX512BF16-NEXT: vpinsrw $7, %eax, %xmm7, %xmm7
+; AVX512BF16-NEXT: vmovd %xmm1, %eax
+; AVX512BF16-NEXT: vpunpckhwd {{.*#+}} xmm1 = xmm5[4],xmm0[4],xmm5[5],xmm0[5],xmm5[6],xmm0[6],xmm5[7],xmm0[7]
+; AVX512BF16-NEXT: vpslld $16, %xmm6, %xmm5
; AVX512BF16-NEXT: vroundss $9, %xmm5, %xmm5, %xmm5
; AVX512BF16-NEXT: vcvtneps2bf16 %xmm5, %xmm5
-; AVX512BF16-NEXT: vpinsrw $2, %eax, %xmm7, %xmm6
+; AVX512BF16-NEXT: vpslld $16, %xmm0, %xmm0
+; AVX512BF16-NEXT: vroundss $9, %xmm0, %xmm0, %xmm0
+; AVX512BF16-NEXT: vcvtneps2bf16 %xmm0, %xmm0
+; AVX512BF16-NEXT: vpinsrw $1, %eax, %xmm0, %xmm0
; AVX512BF16-NEXT: vmovd %xmm5, %eax
+; AVX512BF16-NEXT: vpslld $16, %xmm4, %xmm4
; AVX512BF16-NEXT: vroundss $9, %xmm4, %xmm4, %xmm4
; AVX512BF16-NEXT: vcvtneps2bf16 %xmm4, %xmm4
-; AVX512BF16-NEXT: vpinsrw $3, %eax, %xmm6, %xmm5
+; AVX512BF16-NEXT: vpinsrw $2, %eax, %xmm0, %xmm0
; AVX512BF16-NEXT: vmovd %xmm4, %eax
-; AVX512BF16-NEXT: vroundss $9, %xmm3, %xmm3, %xmm3
-; AVX512BF16-NEXT: vcvtneps2bf16 %xmm3, %xmm3
-; AVX512BF16-NEXT: vpinsrw $4, %eax, %xmm5, %xmm4
-; AVX512BF16-NEXT: vmovd %xmm3, %eax
+; AVX512BF16-NEXT: vroundss $9, %xmm1, %xmm1, %xmm1
+; AVX512BF16-NEXT: vcvtneps2bf16 %xmm1, %xmm1
+; AVX512BF16-NEXT: vpinsrw $3, %eax, %xmm0, %xmm0
+; AVX512BF16-NEXT: vmovd %xmm1, %eax
+; AVX512BF16-NEXT: vpslld $16, %xmm3, %xmm1
+; AVX512BF16-NEXT: vroundss $9, %xmm1, %xmm1, %xmm1
+; AVX512BF16-NEXT: vcvtneps2bf16 %xmm1, %xmm1
+; AVX512BF16-NEXT: vpinsrw $4, %eax, %xmm0, %xmm0
+; AVX512BF16-NEXT: vmovd %xmm1, %eax
+; AVX512BF16-NEXT: vpslld $16, %xmm23, %xmm1
+; AVX512BF16-NEXT: vroundss $9, %xmm1, %xmm1, %xmm1
+; AVX512BF16-NEXT: vcvtneps2bf16 %xmm1, %xmm1
+; AVX512BF16-NEXT: vpslld $16, %xmm2, %xmm2
; AVX512BF16-NEXT: vroundss $9, %xmm2, %xmm2, %xmm2
; AVX512BF16-NEXT: vcvtneps2bf16 %xmm2, %xmm2
-; AVX512BF16-NEXT: vpinsrw $5, %eax, %xmm4, %xmm3
+; AVX512BF16-NEXT: vpinsrw $5, %eax, %xmm0, %xmm0
; AVX512BF16-NEXT: vmovd %xmm2, %eax
-; AVX512BF16-NEXT: vroundss $9, %xmm1, %xmm1, %xmm1
-; AVX512BF16-NEXT: vcvtneps2bf16 %xmm1, %xmm1
-; AVX512BF16-NEXT: vpinsrw $6, %eax, %xmm3, %xmm2
+; AVX512BF16-NEXT: vpinsrw $6, %eax, %xmm0, %xmm0
; AVX512BF16-NEXT: vmovd %xmm1, %eax
-; AVX512BF16-NEXT: vpinsrw $7, %eax, %xmm2, %xmm1
-; AVX512BF16-NEXT: vinserti32x4 $1, %xmm0, %ymm17, %ymm0
-; AVX512BF16-NEXT: vinserti128 $1, %xmm9, %ymm1, %ymm1
-; AVX512BF16-NEXT: vinserti64x4 $1, %ymm0, %zmm1, %zmm0
+; AVX512BF16-NEXT: vpinsrw $7, %eax, %xmm0, %xmm0
+; AVX512BF16-NEXT: vinserti32x4 $1, %xmm18, %ymm12, %ymm1
+; AVX512BF16-NEXT: vinserti128 $1, %xmm7, %ymm0, %ymm0
+; AVX512BF16-NEXT: vinserti64x4 $1, %ymm1, %zmm0, %zmm0
; AVX512BF16-NEXT: retq
%r = call <32 x bfloat> @llvm.floor.v32bf16(<32 x bfloat> %a)
ret <32 x bfloat> %r
@@ -904,14 +828,11 @@ define <2 x bfloat> @floor_v2bf16(<2 x bfloat> %a) {
;
; AVX512BF16-LABEL: floor_v2bf16:
; AVX512BF16: # %bb.0:
-; AVX512BF16-NEXT: vpextrw $1, %xmm0, %eax
-; AVX512BF16-NEXT: shll $16, %eax
-; AVX512BF16-NEXT: vmovd %eax, %xmm1
+; AVX512BF16-NEXT: vpxor %xmm1, %xmm1, %xmm1
+; AVX512BF16-NEXT: vpblendw {{.*#+}} xmm1 = xmm1[0],xmm0[1],xmm1[2],xmm0[3],xmm1[4],xmm0[5],xmm1[6],xmm0[7]
; AVX512BF16-NEXT: vroundss $9, %xmm1, %xmm1, %xmm1
; AVX512BF16-NEXT: vcvtneps2bf16 %xmm1, %xmm1
-; AVX512BF16-NEXT: vmovd %xmm0, %eax
-; AVX512BF16-NEXT: shll $16, %eax
-; AVX512BF16-NEXT: vmovd %eax, %xmm0
+; AVX512BF16-NEXT: vpslld $16, %xmm0, %xmm0
; AVX512BF16-NEXT: vroundss $9, %xmm0, %xmm0, %xmm0
; AVX512BF16-NEXT: vcvtneps2bf16 %xmm0, %xmm0
; AVX512BF16-NEXT: vpunpcklwd {{.*#+}} xmm0 = xmm0[0],xmm1[0],xmm0[1],xmm1[1],xmm0[2],xmm1[2],xmm0[3],xmm1[3]
@@ -931,49 +852,44 @@ define <8 x bfloat> @floor_v8bf16_load(ptr %p) {
; AVX512BF16: # %bb.0:
; AVX512BF16-NEXT: vmovdqa (%rdi), %xmm0
; AVX512BF16-NEXT: vpextrw $7, %xmm0, %eax
-; AVX512BF16-NEXT: shll $16, %eax
; AVX512BF16-NEXT: vmovd %eax, %xmm1
+; AVX512BF16-NEXT: vpslld $16, %xmm1, %xmm1
; AVX512BF16-NEXT: vroundss $9, %xmm1, %xmm1, %xmm1
; AVX512BF16-NEXT: vcvtneps2bf16 %xmm1, %xmm1
; AVX512BF16-NEXT: vmovd %xmm1, %eax
; AVX512BF16-NEXT: vpextrw $6, %xmm0, %ecx
-; AVX512BF16-NEXT: shll $16, %ecx
; AVX512BF16-NEXT: vmovd %ecx, %xmm1
+; AVX512BF16-NEXT: vpslld $16, %xmm1, %xmm1
; AVX512BF16-NEXT: vroundss $9, %xmm1, %xmm1, %xmm1
; AVX512BF16-NEXT: vcvtneps2bf16 %xmm1, %xmm1
; AVX512BF16-NEXT: vmovd %xmm1, %ecx
; AVX512BF16-NEXT: vpextrw $5, %xmm0, %edx
-; AVX512BF16-NEXT: shll $16, %edx
; AVX512BF16-NEXT: vmovd %edx, %xmm1
+; AVX512BF16-NEXT: vpslld $16, %xmm1, %xmm1
; AVX512BF16-NEXT: vroundss $9, %xmm1, %xmm1, %xmm1
; AVX512BF16-NEXT: vcvtneps2bf16 %xmm1, %xmm1
; AVX512BF16-NEXT: vmovd %xmm1, %edx
-; AVX512BF16-NEXT: vpextrw $4, %xmm0, %esi
-; AVX512BF16-NEXT: shll $16, %esi
-; AVX512BF16-NEXT: vmovd %esi, %xmm1
-; AVX512BF16-NEXT: vroundss $9, %xmm1, %xmm1, %xmm1
-; AVX512BF16-NEXT: vcvtneps2bf16 %xmm1, %xmm1
-; AVX512BF16-NEXT: vmovd %xmm1, %esi
+; AVX512BF16-NEXT: vpxor %xmm1, %xmm1, %xmm1
+; AVX512BF16-NEXT: vpunpckhwd {{.*#+}} xmm2 = xmm1[4],xmm0[4],xmm1[5],xmm0[5],xmm1[6],xmm0[6],xmm1[7],xmm0[7]
+; AVX512BF16-NEXT: vroundss $9, %xmm2, %xmm2, %xmm2
+; AVX512BF16-NEXT: vcvtneps2bf16 %xmm2, %xmm2
+; AVX512BF16-NEXT: vmovd %xmm2, %esi
; AVX512BF16-NEXT: vpextrw $3, %xmm0, %edi
-; AVX512BF16-NEXT: shll $16, %edi
-; AVX512BF16-NEXT: vmovd %edi, %xmm1
-; AVX512BF16-NEXT: vroundss $9, %xmm1, %xmm1, %xmm1
-; AVX512BF16-NEXT: vcvtneps2bf16 %xmm1, %xmm1
-; AVX512BF16-NEXT: vmovd %xmm1, %edi
+; AVX512BF16-NEXT: vmovd %edi, %xmm2
+; AVX512BF16-NEXT: vpslld $16, %xmm2, %xmm2
+; AVX512BF16-NEXT: vroundss $9, %xmm2, %xmm2, %xmm2
+; AVX512BF16-NEXT: vcvtneps2bf16 %xmm2, %xmm2
+; AVX512BF16-NEXT: vmovd %xmm2, %edi
; AVX512BF16-NEXT: vpextrw $2, %xmm0, %r8d
-; AVX512BF16-NEXT: shll $16, %r8d
-; AVX512BF16-NEXT: vmovd %r8d, %xmm1
-; AVX512BF16-NEXT: vroundss $9, %xmm1, %xmm1, %xmm1
-; AVX512BF16-NEXT: vcvtneps2bf16 %xmm1, %xmm1
-; AVX512BF16-NEXT: vmovd %xmm1, %r8d
-; AVX512BF16-NEXT: vpextrw $1, %xmm0, %r9d
-; AVX512BF16-NEXT: shll $16, %r9d
-; AVX512BF16-NEXT: vmovd %r9d, %xmm1
+; AVX512BF16-NEXT: vmovd %r8d, %xmm2
+; AVX512BF16-NEXT: vpslld $16, %xmm2, %xmm2
+; AVX512BF16-NEXT: vroundss $9, %xmm2, %xmm2, %xmm2
+; AVX512BF16-NEXT: vcvtneps2bf16 %xmm2, %xmm2
+; AVX512BF16-NEXT: vmovd %xmm2, %r8d
+; AVX512BF16-NEXT: vpblendw {{.*#+}} xmm1 = xmm1[0],xmm0[1],xmm1[2],xmm0[3],xmm1[4],xmm0[5],xmm1[6],xmm0[7]
; AVX512BF16-NEXT: vroundss $9, %xmm1, %xmm1, %xmm1
; AVX512BF16-NEXT: vcvtneps2bf16 %xmm1, %xmm1
-; AVX512BF16-NEXT: vmovd %xmm0, %r9d
-; AVX512BF16-NEXT: shll $16, %r9d
-; AVX512BF16-NEXT: vmovd %r9d, %xmm0
+; AVX512BF16-NEXT: vpslld $16, %xmm0, %xmm0
; AVX512BF16-NEXT: vroundss $9, %xmm0, %xmm0, %xmm0
; AVX512BF16-NEXT: vcvtneps2bf16 %xmm0, %xmm0
; AVX512BF16-NEXT: vpunpcklwd {{.*#+}} xmm0 = xmm0[0],xmm1[0],xmm0[1],xmm1[1],xmm0[2],xmm1[2],xmm0[3],xmm1[3]
@@ -1002,49 +918,44 @@ define <8 x bfloat> @floor_v8bf16_mask(<8 x bfloat> %a, <8 x bfloat> %src, i8 %m
; AVX512BF16: # %bb.0:
; AVX512BF16-NEXT: kmovd %edi, %k1
; AVX512BF16-NEXT: vpextrw $7, %xmm0, %eax
-; AVX512BF16-NEXT: shll $16, %eax
; AVX512BF16-NEXT: vmovd %eax, %xmm2
+; AVX512BF16-NEXT: vpslld $16, %xmm2, %xmm2
; AVX512BF16-NEXT: vroundss $9, %xmm2, %xmm2, %xmm2
; AVX512BF16-NEXT: vcvtneps2bf16 %xmm2, %xmm2
; AVX512BF16-NEXT: vmovd %xmm2, %eax
; AVX512BF16-NEXT: vpextrw $6, %xmm0, %ecx
-; AVX512BF16-NEXT: shll $16, %ecx
; AVX512BF16-NEXT: vmovd %ecx, %xmm2
+; AVX512BF16-NEXT: vpslld $16, %xmm2, %xmm2
; AVX512BF16-NEXT: vroundss $9, %xmm2, %xmm2, %xmm2
; AVX512BF16-NEXT: vcvtneps2bf16 %xmm2, %xmm2
; AVX512BF16-NEXT: vmovd %xmm2, %ecx
; AVX512BF16-NEXT: vpextrw $5, %xmm0, %edx
-; AVX512BF16-NEXT: shll $16, %edx
; AVX512BF16-NEXT: vmovd %edx, %xmm2
+; AVX512BF16-NEXT: vpslld $16, %xmm2, %xmm2
; AVX512BF16-NEXT: vroundss $9, %xmm2, %xmm2, %xmm2
; AVX512BF16-NEXT: vcvtneps2bf16 %xmm2, %xmm2
; AVX512BF16-NEXT: vmovd %xmm2, %edx
-; AVX512BF16-NEXT: vpextrw $4, %xmm0, %esi
-; AVX512BF16-NEXT: shll $16, %esi
-; AVX512BF16-NEXT: vmovd %esi, %xmm2
-; AVX512BF16-NEXT: vroundss $9, %xmm2, %xmm2, %xmm2
-; AVX512BF16-NEXT: vcvtneps2bf16 %xmm2, %xmm2
-; AVX512BF16-NEXT: vmovd %xmm2, %esi
+; AVX512BF16-NEXT: vpxor %xmm2, %xmm2, %xmm2
+; AVX512BF16-NEXT: vpunpckhwd {{.*#+}} xmm3 = xmm2[4],xmm0[4],xmm2[5],xmm0[5],xmm2[6],xmm0[6],xmm2[7],xmm0[7]
+; AVX512BF16-NEXT: vroundss $9, %xmm3, %xmm3, %xmm3
+; AVX512BF16-NEXT: vcvtneps2bf16 %xmm3, %xmm3
+; AVX512BF16-NEXT: vmovd %xmm3, %esi
; AVX512BF16-NEXT: vpextrw $3, %xmm0, %edi
-; AVX512BF16-NEXT: shll $16, %edi
-; AVX512BF16-NEXT: vmovd %edi, %xmm2
-; AVX512BF16-NEXT: vroundss $9, %xmm2, %xmm2, %xmm2
-; AVX512BF16-NEXT: vcvtneps2bf16 %xmm2, %xmm2
-; AVX512BF16-NEXT: vmovd %xmm2, %edi
+; AVX512BF16-NEXT: vmovd %edi, %xmm3
+; AVX512BF16-NEXT: vpslld $16, %xmm3, %xmm3
+; AVX512BF16-NEXT: vroundss $9, %xmm3, %xmm3, %xmm3
+; AVX512BF16-NEXT: vcvtneps2bf16 %xmm3, %xmm3
+; AVX512BF16-NEXT: vmovd %xmm3, %edi
; AVX512BF16-NEXT: vpextrw $2, %xmm0, %r8d
-; AVX512BF16-NEXT: shll $16, %r8d
-; AVX512BF16-NEXT: vmovd %r8d, %xmm2
-; AVX512BF16-NEXT: vroundss $9, %xmm2, %xmm2, %xmm2
-; AVX512BF16-NEXT: vcvtneps2bf16 %xmm2, %xmm2
-; AVX512BF16-NEXT: vmovd %xmm2, %r8d
-; AVX512BF16-NEXT: vpextrw $1, %xmm0, %r9d
-; AVX512BF16-NEXT: shll $16, %r9d
-; AVX512BF16-NEXT: vmovd %r9d, %xmm2
+; AVX512BF16-NEXT: vmovd %r8d, %xmm3
+; AVX512BF16-NEXT: vpslld $16, %xmm3, %xmm3
+; AVX512BF16-NEXT: vroundss $9, %xmm3, %xmm3, %xmm3
+; AVX512BF16-NEXT: vcvtneps2bf16 %xmm3, %xmm3
+; AVX512BF16-NEXT: vmovd %xmm3, %r8d
+; AVX512BF16-NEXT: vpblendw {{.*#+}} xmm2 = xmm2[0],xmm0[1],xmm2[2],xmm0[3],xmm2[4],xmm0[5],xmm2[6],xmm0[7]
; AVX512BF16-NEXT: vroundss $9, %xmm2, %xmm2, %xmm2
; AVX512BF16-NEXT: vcvtneps2bf16 %xmm2, %xmm2
-; AVX512BF16-NEXT: vmovd %xmm0, %r9d
-; AVX512BF16-NEXT: shll $16, %r9d
-; AVX512BF16-NEXT: vmovd %r9d, %xmm0
+; AVX512BF16-NEXT: vpslld $16, %xmm0, %xmm0
; AVX512BF16-NEXT: vroundss $9, %xmm0, %xmm0, %xmm0
; AVX512BF16-NEXT: vcvtneps2bf16 %xmm0, %xmm0
; AVX512BF16-NEXT: vpunpcklwd {{.*#+}} xmm0 = xmm0[0],xmm2[0],xmm0[1],xmm2[1],xmm0[2],xmm2[2],xmm0[3],xmm2[3]
@@ -1073,49 +984,44 @@ define <8 x bfloat> @floor_v8bf16_maskz(<8 x bfloat> %a, i8 %msk) {
; AVX512BF16: # %bb.0:
; AVX512BF16-NEXT: kmovd %edi, %k1
; AVX512BF16-NEXT: vpextrw $7, %xmm0, %eax
-; AVX512BF16-NEXT: shll $16, %eax
; AVX512BF16-NEXT: vmovd %eax, %xmm1
+; AVX512BF16-NEXT: vpslld $16, %xmm1, %xmm1
; AVX512BF16-NEXT: vroundss $9, %xmm1, %xmm1, %xmm1
; AVX512BF16-NEXT: vcvtneps2bf16 %xmm1, %xmm1
; AVX512BF16-NEXT: vmovd %xmm1, %eax
; AVX512BF16-NEXT: vpextrw $6, %xmm0, %ecx
-; AVX512BF16-NEXT: shll $16, %ecx
; AVX512BF16-NEXT: vmovd %ecx, %xmm1
+; AVX512BF16-NEXT: vpslld $16, %xmm1, %xmm1
; AVX512BF16-NEXT: vroundss $9, %xmm1, %xmm1, %xmm1
; AVX512BF16-NEXT: vcvtneps2bf16 %xmm1, %xmm1
; AVX512BF16-NEXT: vmovd %xmm1, %ecx
; AVX512BF16-NEXT: vpextrw $5, %xmm0, %edx
-; AVX512BF16-NEXT: shll $16, %edx
; AVX512BF16-NEXT: vmovd %edx, %xmm1
+; AVX512BF16-NEXT: vpslld $16, %xmm1, %xmm1
; AVX512BF16-NEXT: vroundss $9, %xmm1, %xmm1, %xmm1
; AVX512BF16-NEXT: vcvtneps2bf16 %xmm1, %xmm1
; AVX512BF16-NEXT: vmovd %xmm1, %edx
-; AVX512BF16-NEXT: vpextrw $4, %xmm0, %esi
-; AVX512BF16-NEXT: shll $16, %esi
-; AVX512BF16-NEXT: vmovd %esi, %xmm1
-; AVX512BF16-NEXT: vroundss $9, %xmm1, %xmm1, %xmm1
-; AVX512BF16-NEXT: vcvtneps2bf16 %xmm1, %xmm1
-; AVX512BF16-NEXT: vmovd %xmm1, %esi
+; AVX512BF16-NEXT: vpxor %xmm1, %xmm1, %xmm1
+; AVX512BF16-NEXT: vpunpckhwd {{.*#+}} xmm2 = xmm1[4],xmm0[4],xmm1[5],xmm0[5],xmm1[6],xmm0[6],xmm1[7],xmm0[7]
+; AVX512BF16-NEXT: vroundss $9, %xmm2, %xmm2, %xmm2
+; AVX512BF16-NEXT: vcvtneps2bf16 %xmm2, %xmm2
+; AVX512BF16-NEXT: vmovd %xmm2, %esi
; AVX512BF16-NEXT: vpextrw $3, %xmm0, %edi
-; AVX512BF16-NEXT: shll $16, %edi
-; AVX512BF16-NEXT: vmovd %edi, %xmm1
-; AVX512BF16-NEXT: vroundss $9, %xmm1, %xmm1, %xmm1
-; AVX512BF16-NEXT: vcvtneps2bf16 %xmm1, %xmm1
-; AVX512BF16-NEXT: vmovd %xmm1, %edi
+; AVX512BF16-NEXT: vmovd %edi, %xmm2
+; AVX512BF16-NEXT: vpslld $16, %xmm2, %xmm2
+; AVX512BF16-NEXT: vroundss $9, %xmm2, %xmm2, %xmm2
+; AVX512BF16-NEXT: vcvtneps2bf16 %xmm2, %xmm2
+; AVX512BF16-NEXT: vmovd %xmm2, %edi
; AVX512BF16-NEXT: vpextrw $2, %xmm0, %r8d
-; AVX512BF16-NEXT: shll $16, %r8d
-; AVX512BF16-NEXT: vmovd %r8d, %xmm1
-; AVX512BF16-NEXT: vroundss $9, %xmm1, %xmm1, %xmm1
-; AVX512BF16-NEXT: vcvtneps2bf16 %xmm1, %xmm1
-; AVX512BF16-NEXT: vmovd %xmm1, %r8d
-; AVX512BF16-NEXT: vpextrw $1, %xmm0, %r9d
-; AVX512BF16-NEXT: shll $16, %r9d
-; AVX512BF16-NEXT: vmovd %r9d, %xmm1
+; AVX512BF16-NEXT: vmovd %r8d, %xmm2
+; AVX512BF16-NEXT: vpslld $16, %xmm2, %xmm2
+; AVX512BF16-NEXT: vroundss $9, %xmm2, %xmm2, %xmm2
+; AVX512BF16-NEXT: vcvtneps2bf16 %xmm2, %xmm2
+; AVX512BF16-NEXT: vmovd %xmm2, %r8d
+; AVX512BF16-NEXT: vpblendw {{.*#+}} xmm1 = xmm1[0],xmm0[1],xmm1[2],xmm0[3],xmm1[4],xmm0[5],xmm1[6],xmm0[7]
; AVX512BF16-NEXT: vroundss $9, %xmm1, %xmm1, %xmm1
; AVX512BF16-NEXT: vcvtneps2bf16 %xmm1, %xmm1
-; AVX512BF16-NEXT: vmovd %xmm0, %r9d
-; AVX512BF16-NEXT: shll $16, %r9d
-; AVX512BF16-NEXT: vmovd %r9d, %xmm0
+; AVX512BF16-NEXT: vpslld $16, %xmm0, %xmm0
; AVX512BF16-NEXT: vroundss $9, %xmm0, %xmm0, %xmm0
; AVX512BF16-NEXT: vcvtneps2bf16 %xmm0, %xmm0
; AVX512BF16-NEXT: vpunpcklwd {{.*#+}} xmm0 = xmm0[0],xmm1[0],xmm0[1],xmm1[1],xmm0[2],xmm1[2],xmm0[3],xmm1[3]
@@ -1137,9 +1043,152 @@ define <8 x bfloat> @floor_v8bf16_maskz(<8 x bfloat> %a, i8 %msk) {
; must stay scalarized rather than becoming a vrndscalebf16.
define <8 x bfloat> @round_v8bf16(<8 x bfloat> %a) {
; AVX10_2-LABEL: round_v8bf16:
-; AVX10_2-NOT: vrndscalebf16
-; AVX10_2: vroundss $11,
-; AVX10_2-NOT: vrndscalebf16
+; AVX10_2: # %bb.0:
+; AVX10_2-NEXT: vpextrw $3, %xmm0, %eax
+; AVX10_2-NEXT: vmovw %eax, %xmm1
+; AVX10_2-NEXT: vpslld $16, %xmm1, %xmm1
+; AVX10_2-NEXT: vpbroadcastd {{.*#+}} xmm2 = [-0.0E+0,-0.0E+0,-0.0E+0,-0.0E+0]
+; AVX10_2-NEXT: vpbroadcastd {{.*#+}} xmm3 = [4.9999997E-1,4.9999997E-1,4.9999997E-1,4.9999997E-1]
+; AVX10_2-NEXT: vmovdqa %xmm3, %xmm4
+; AVX10_2-NEXT: vpternlogd {{.*#+}} xmm4 = xmm4 | (xmm1 & xmm2)
+; AVX10_2-NEXT: vaddss %xmm4, %xmm1, %xmm1
+; AVX10_2-NEXT: vroundss $11, %xmm1, %xmm1, %xmm1
+; AVX10_2-NEXT: vcvtneps2bf16 %xmm1, %xmm1
+; AVX10_2-NEXT: vpextrw $2, %xmm0, %eax
+; AVX10_2-NEXT: vmovw %eax, %xmm4
+; AVX10_2-NEXT: vpslld $16, %xmm4, %xmm4
+; AVX10_2-NEXT: vmovdqa %xmm3, %xmm5
+; AVX10_2-NEXT: vpternlogd {{.*#+}} xmm5 = xmm5 | (xmm4 & xmm2)
+; AVX10_2-NEXT: vaddss %xmm5, %xmm4, %xmm4
+; AVX10_2-NEXT: vroundss $11, %xmm4, %xmm4, %xmm4
+; AVX10_2-NEXT: vcvtneps2bf16 %xmm4, %xmm4
+; AVX10_2-NEXT: vpunpcklwd {{.*#+}} xmm1 = xmm4[0],xmm1[0],xmm4[1],xmm1[1],xmm4[2],xmm1[2],xmm4[3],xmm1[3]
+; AVX10_2-NEXT: vpxor %xmm4, %xmm4, %xmm4
+; AVX10_2-NEXT: vpblendw {{.*#+}} xmm5 = xmm4[0],xmm0[1],xmm4[2],xmm0[3],xmm4[4],xmm0[5],xmm4[6],xmm0[7]
+; AVX10_2-NEXT: vmovdqa %xmm3, %xmm6
+; AVX10_2-NEXT: vpternlogd {{.*#+}} xmm6 = xmm6 | (xmm5 & xmm2)
+; AVX10_2-NEXT: vaddss %xmm6, %xmm5, %xmm5
+; AVX10_2-NEXT: vroundss $11, %xmm5, %xmm5, %xmm5
+; AVX10_2-NEXT: vcvtneps2bf16 %xmm5, %xmm5
+; AVX10_2-NEXT: vpslld $16, %xmm0, %xmm6
+; AVX10_2-NEXT: vmovdqa %xmm3, %xmm7
+; AVX10_2-NEXT: vpternlogd {{.*#+}} xmm7 = xmm7 | (xmm6 & xmm2)
+; AVX10_2-NEXT: vaddss %xmm7, %xmm6, %xmm6
+; AVX10_2-NEXT: vroundss $11, %xmm6, %xmm6, %xmm6
+; AVX10_2-NEXT: vcvtneps2bf16 %xmm6, %xmm6
+; AVX10_2-NEXT: vpunpcklwd {{.*#+}} xmm5 = xmm6[0],xmm5[0],xmm6[1],xmm5[1],xmm6[2],xmm5[2],xmm6[3],xmm5[3]
+; AVX10_2-NEXT: vpunpckldq {{.*#+}} xmm1 = xmm5[0],xmm1[0],xmm5[1],xmm1[1]
+; AVX10_2-NEXT: vpunpckhwd {{.*#+}} xmm4 = xmm4[4],xmm0[4],xmm4[5],xmm0[5],xmm4[6],xmm0[6],xmm4[7],xmm0[7]
+; AVX10_2-NEXT: vmovdqa %xmm3, %xmm5
+; AVX10_2-NEXT: vpternlogd {{.*#+}} xmm5 = xmm5 | (xmm4 & xmm2)
+; AVX10_2-NEXT: vaddss %xmm5, %xmm4, %xmm4
+; AVX10_2-NEXT: vroundss $11, %xmm4, %xmm4, %xmm4
+; AVX10_2-NEXT: vcvtneps2bf16 %xmm4, %xmm4
+; AVX10_2-NEXT: vpextrw $5, %xmm0, %eax
+; AVX10_2-NEXT: vmovw %eax, %xmm5
+; AVX10_2-NEXT: vpslld $16, %xmm5, %xmm5
+; AVX10_2-NEXT: vmovdqa %xmm3, %xmm6
+; AVX10_2-NEXT: vpternlogd {{.*#+}} xmm6 = xmm6 | (xmm5 & xmm2)
+; AVX10_2-NEXT: vaddss %xmm6, %xmm5, %xmm5
+; AVX10_2-NEXT: vroundss $11, %xmm5, %xmm5, %xmm5
+; AVX10_2-NEXT: vcvtneps2bf16 %xmm5, %xmm5
+; AVX10_2-NEXT: vpunpcklwd {{.*#+}} xmm4 = xmm4[0],xmm5[0],xmm4[1],xmm5[1],xmm4[2],xmm5[2],xmm4[3],xmm5[3]
+; AVX10_2-NEXT: vpextrw $7, %xmm0, %eax
+; AVX10_2-NEXT: vmovw %eax, %xmm5
+; AVX10_2-NEXT: vpslld $16, %xmm5, %xmm5
+; AVX10_2-NEXT: vmovdqa %xmm3, %xmm6
+; AVX10_2-NEXT: vpternlogd {{.*#+}} xmm6 = xmm6 | (xmm5 & xmm2)
+; AVX10_2-NEXT: vaddss %xmm6, %xmm5, %xmm5
+; AVX10_2-NEXT: vroundss $11, %xmm5, %xmm5, %xmm5
+; AVX10_2-NEXT: vcvtneps2bf16 %xmm5, %xmm5
+; AVX10_2-NEXT: vpextrw $6, %xmm0, %eax
+; AVX10_2-NEXT: vmovw %eax, %xmm0
+; AVX10_2-NEXT: vpslld $16, %xmm0, %xmm0
+; AVX10_2-NEXT: vpternlogd {{.*#+}} xmm3 = xmm3 | (xmm0 & xmm2)
+; AVX10_2-NEXT: vaddss %xmm3, %xmm0, %xmm0
+; AVX10_2-NEXT: vroundss $11, %xmm0, %xmm0, %xmm0
+; AVX10_2-NEXT: vcvtneps2bf16 %xmm0, %xmm0
+; AVX10_2-NEXT: vpunpcklwd {{.*#+}} xmm0 = xmm0[0],xmm5[0],xmm0[1],xmm5[1],xmm0[2],xmm5[2],xmm0[3],xmm5[3]
+; AVX10_2-NEXT: vpunpckldq {{.*#+}} xmm0 = xmm4[0],xmm0[0],xmm4[1],xmm0[1]
+; AVX10_2-NEXT: vpunpcklqdq {{.*#+}} xmm0 = xmm1[0],xmm0[0]
+; AVX10_2-NEXT: retq
+;
+; AVX512BF16-LABEL: round_v8bf16:
+; AVX512BF16: # %bb.0:
+; AVX512BF16-NEXT: vpextrw $7, %xmm0, %eax
+; AVX512BF16-NEXT: vmovd %eax, %xmm1
+; AVX512BF16-NEXT: vpslld $16, %xmm1, %xmm1
+; AVX512BF16-NEXT: vpbroadcastd {{.*#+}} xmm2 = [-0.0E+0,-0.0E+0,-0.0E+0,-0.0E+0]
+; AVX512BF16-NEXT: vpbroadcastd {{.*#+}} xmm3 = [4.9999997E-1,4.9999997E-1,4.9999997E-1,4.9999997E-1]
+; AVX512BF16-NEXT: vmovdqa %xmm3, %xmm4
+; AVX512BF16-NEXT: vpternlogd {{.*#+}} xmm4 = xmm4 | (xmm1 & xmm2)
+; AVX512BF16-NEXT: vaddss %xmm4, %xmm1, %xmm1
+; AVX512BF16-NEXT: vroundss $11, %xmm1, %xmm1, %xmm1
+; AVX512BF16-NEXT: vcvtneps2bf16 %xmm1, %xmm1
+; AVX512BF16-NEXT: vmovd %xmm1, %eax
+; AVX512BF16-NEXT: vpextrw $6, %xmm0, %ecx
+; AVX512BF16-NEXT: vmovd %ecx, %xmm1
+; AVX512BF16-NEXT: vpslld $16, %xmm1, %xmm1
+; AVX512BF16-NEXT: vmovdqa %xmm3, %xmm4
+; AVX512BF16-NEXT: vpternlogd {{.*#+}} xmm4 = xmm4 | (xmm1 & xmm2)
+; AVX512BF16-NEXT: vaddss %xmm4, %xmm1, %xmm1
+; AVX512BF16-NEXT: vroundss $11, %xmm1, %xmm1, %xmm1
+; AVX512BF16-NEXT: vcvtneps2bf16 %xmm1, %xmm1
+; AVX512BF16-NEXT: vmovd %xmm1, %ecx
+; AVX512BF16-NEXT: vpextrw $5, %xmm0, %edx
+; AVX512BF16-NEXT: vmovd %edx, %xmm1
+; AVX512BF16-NEXT: vpslld $16, %xmm1, %xmm1
+; AVX512BF16-NEXT: vmovdqa %xmm3, %xmm4
+; AVX512BF16-NEXT: vpternlogd {{.*#+}} xmm4 = xmm4 | (xmm1 & xmm2)
+; AVX512BF16-NEXT: vaddss %xmm4, %xmm1, %xmm1
+; AVX512BF16-NEXT: vroundss $11, %xmm1, %xmm1, %xmm1
+; AVX512BF16-NEXT: vcvtneps2bf16 %xmm1, %xmm1
+; AVX512BF16-NEXT: vmovd %xmm1, %edx
+; AVX512BF16-NEXT: vpxor %xmm1, %xmm1, %xmm1
+; AVX512BF16-NEXT: vpunpckhwd {{.*#+}} xmm4 = xmm1[4],xmm0[4],xmm1[5],xmm0[5],xmm1[6],xmm0[6],xmm1[7],xmm0[7]
+; AVX512BF16-NEXT: vmovdqa %xmm3, %xmm5
+; AVX512BF16-NEXT: vpternlogd {{.*#+}} xmm5 = xmm5 | (xmm4 & xmm2)
+; AVX512BF16-NEXT: vaddss %xmm5, %xmm4, %xmm4
+; AVX512BF16-NEXT: vroundss $11, %xmm4, %xmm4, %xmm4
+; AVX512BF16-NEXT: vcvtneps2bf16 %xmm4, %xmm4
+; AVX512BF16-NEXT: vmovd %xmm4, %esi
+; AVX512BF16-NEXT: vpextrw $3, %xmm0, %edi
+; AVX512BF16-NEXT: vmovd %edi, %xmm4
+; AVX512BF16-NEXT: vpslld $16, %xmm4, %xmm4
+; AVX512BF16-NEXT: vmovdqa %xmm3, %xmm5
+; AVX512BF16-NEXT: vpternlogd {{.*#+}} xmm5 = xmm5 | (xmm4 & xmm2)
+; AVX512BF16-NEXT: vaddss %xmm5, %xmm4, %xmm4
+; AVX512BF16-NEXT: vroundss $11, %xmm4, %xmm4, %xmm4
+; AVX512BF16-NEXT: vcvtneps2bf16 %xmm4, %xmm4
+; AVX512BF16-NEXT: vmovd %xmm4, %edi
+; AVX512BF16-NEXT: vpextrw $2, %xmm0, %r8d
+; AVX512BF16-NEXT: vmovd %r8d, %xmm4
+; AVX512BF16-NEXT: vpslld $16, %xmm4, %xmm4
+; AVX512BF16-NEXT: vmovdqa %xmm3, %xmm5
+; AVX512BF16-NEXT: vpternlogd {{.*#+}} xmm5 = xmm5 | (xmm4 & xmm2)
+; AVX512BF16-NEXT: vaddss %xmm5, %xmm4, %xmm4
+; AVX512BF16-NEXT: vroundss $11, %xmm4, %xmm4, %xmm4
+; AVX512BF16-NEXT: vcvtneps2bf16 %xmm4, %xmm4
+; AVX512BF16-NEXT: vmovd %xmm4, %r8d
+; AVX512BF16-NEXT: vpblendw {{.*#+}} xmm1 = xmm1[0],xmm0[1],xmm1[2],xmm0[3],xmm1[4],xmm0[5],xmm1[6],xmm0[7]
+; AVX512BF16-NEXT: vmovdqa %xmm3, %xmm4
+; AVX512BF16-NEXT: vpternlogd {{.*#+}} xmm4 = xmm4 | (xmm1 & xmm2)
+; AVX512BF16-NEXT: vaddss %xmm4, %xmm1, %xmm1
+; AVX512BF16-NEXT: vroundss $11, %xmm1, %xmm1, %xmm1
+; AVX512BF16-NEXT: vcvtneps2bf16 %xmm1, %xmm1
+; AVX512BF16-NEXT: vpslld $16, %xmm0, %xmm0
+; AVX512BF16-NEXT: vpternlogd {{.*#+}} xmm3 = xmm3 | (xmm0 & xmm2)
+; AVX512BF16-NEXT: vaddss %xmm3, %xmm0, %xmm0
+; AVX512BF16-NEXT: vroundss $11, %xmm0, %xmm0, %xmm0
+; AVX512BF16-NEXT: vcvtneps2bf16 %xmm0, %xmm0
+; AVX512BF16-NEXT: vpunpcklwd {{.*#+}} xmm0 = xmm0[0],xmm1[0],xmm0[1],xmm1[1],xmm0[2],xmm1[2],xmm0[3],xmm1[3]
+; AVX512BF16-NEXT: vpinsrw $2, %r8d, %xmm0, %xmm0
+; AVX512BF16-NEXT: vpinsrw $3, %edi, %xmm0, %xmm0
+; AVX512BF16-NEXT: vpinsrw $4, %esi, %xmm0, %xmm0
+; AVX512BF16-NEXT: vpinsrw $5, %edx, %xmm0, %xmm0
+; AVX512BF16-NEXT: vpinsrw $6, %ecx, %xmm0, %xmm0
+; AVX512BF16-NEXT: vpinsrw $7, %eax, %xmm0, %xmm0
+; AVX512BF16-NEXT: retq
%r = call <8 x bfloat> @llvm.round.v8bf16(<8 x bfloat> %a)
ret <8 x bfloat> %r
}
diff --git a/llvm/test/CodeGen/X86/znver-pdep.ll b/llvm/test/CodeGen/X86/znver-pdep.ll
index 1acb6a49f44d0..afd97ebca3e0e 100644
--- a/llvm/test/CodeGen/X86/znver-pdep.ll
+++ b/llvm/test/CodeGen/X86/znver-pdep.ll
@@ -1005,114 +1005,310 @@ define <4 x i64> @pdep_v4i64(<4 x i64> %val, <4 x i64> %mask) nounwind {
;
define <4 x i32> @pdep_v4i32_minsize(<4 x i32> %val, <4 x i32> %mask) nounwind minsize {
-; CHECK-LABEL: pdep_v4i32_minsize:
-; CHECK: # %bb.0:
-; CHECK-NEXT: vpextrd $1, %xmm1, %eax
-; CHECK-NEXT: vpextrd $1, %xmm0, %ecx
-; CHECK-NEXT: vmovd %xmm0, %edx
-; CHECK-NEXT: pdepl %eax, %ecx, %eax
-; CHECK-NEXT: vmovd %xmm1, %ecx
-; CHECK-NEXT: pdepl %ecx, %edx, %ecx
-; CHECK-NEXT: vmovd %ecx, %xmm2
-; CHECK-NEXT: vpextrd $2, %xmm0, %ecx
-; CHECK-NEXT: vpinsrd $1, %eax, %xmm2, %xmm2
-; CHECK-NEXT: vpextrd $2, %xmm1, %eax
-; CHECK-NEXT: pdepl %eax, %ecx, %eax
-; CHECK-NEXT: vpextrd $3, %xmm0, %ecx
-; CHECK-NEXT: vpinsrd $2, %eax, %xmm2, %xmm2
-; CHECK-NEXT: vpextrd $3, %xmm1, %eax
-; CHECK-NEXT: pdepl %eax, %ecx, %eax
-; CHECK-NEXT: vpinsrd $3, %eax, %xmm2, %xmm0
-; CHECK-NEXT: retq
+; SLOW-ZNVER-LABEL: pdep_v4i32_minsize:
+; SLOW-ZNVER: # %bb.0:
+; SLOW-ZNVER-NEXT: vpextrd $1, %xmm1, %eax
+; SLOW-ZNVER-NEXT: vpextrd $1, %xmm0, %ecx
+; SLOW-ZNVER-NEXT: vmovd %xmm0, %edx
+; SLOW-ZNVER-NEXT: vpextrd $2, %xmm0, %esi
+; SLOW-ZNVER-NEXT: pdepl %eax, %ecx, %eax
+; SLOW-ZNVER-NEXT: vmovd %xmm1, %ecx
+; SLOW-ZNVER-NEXT: pdepl %ecx, %edx, %ecx
+; SLOW-ZNVER-NEXT: vpextrd $2, %xmm1, %edx
+; SLOW-ZNVER-NEXT: pdepl %edx, %esi, %edx
+; SLOW-ZNVER-NEXT: vpextrd $3, %xmm0, %esi
+; SLOW-ZNVER-NEXT: vmovd %ecx, %xmm2
+; SLOW-ZNVER-NEXT: vpextrd $3, %xmm1, %ecx
+; SLOW-ZNVER-NEXT: pdepl %ecx, %esi, %ecx
+; SLOW-ZNVER-NEXT: vpinsrd $1, %eax, %xmm2, %xmm0
+; SLOW-ZNVER-NEXT: vpinsrd $2, %edx, %xmm0, %xmm0
+; SLOW-ZNVER-NEXT: vpinsrd $3, %ecx, %xmm0, %xmm0
+; SLOW-ZNVER-NEXT: retq
+;
+; SLOW-BDVER4-LABEL: pdep_v4i32_minsize:
+; SLOW-BDVER4: # %bb.0:
+; SLOW-BDVER4-NEXT: vpextrd $1, %xmm1, %eax
+; SLOW-BDVER4-NEXT: vpextrd $1, %xmm0, %ecx
+; SLOW-BDVER4-NEXT: vmovd %xmm0, %edx
+; SLOW-BDVER4-NEXT: vpextrd $2, %xmm0, %esi
+; SLOW-BDVER4-NEXT: pdepl %eax, %ecx, %eax
+; SLOW-BDVER4-NEXT: vmovd %xmm1, %ecx
+; SLOW-BDVER4-NEXT: pdepl %ecx, %edx, %ecx
+; SLOW-BDVER4-NEXT: vpextrd $2, %xmm1, %edx
+; SLOW-BDVER4-NEXT: pdepl %edx, %esi, %edx
+; SLOW-BDVER4-NEXT: vpextrd $3, %xmm0, %esi
+; SLOW-BDVER4-NEXT: vmovd %ecx, %xmm2
+; SLOW-BDVER4-NEXT: vpextrd $3, %xmm1, %ecx
+; SLOW-BDVER4-NEXT: pdepl %ecx, %esi, %ecx
+; SLOW-BDVER4-NEXT: vpinsrd $1, %eax, %xmm2, %xmm0
+; SLOW-BDVER4-NEXT: vpinsrd $2, %edx, %xmm0, %xmm0
+; SLOW-BDVER4-NEXT: vpinsrd $3, %ecx, %xmm0, %xmm0
+; SLOW-BDVER4-NEXT: retq
+;
+; FAST-LABEL: pdep_v4i32_minsize:
+; FAST: # %bb.0:
+; FAST-NEXT: vpextrd $1, %xmm1, %eax
+; FAST-NEXT: vpextrd $1, %xmm0, %ecx
+; FAST-NEXT: vmovd %xmm0, %edx
+; FAST-NEXT: pdepl %eax, %ecx, %eax
+; FAST-NEXT: vmovd %xmm1, %ecx
+; FAST-NEXT: pdepl %ecx, %edx, %ecx
+; FAST-NEXT: vmovd %ecx, %xmm2
+; FAST-NEXT: vpextrd $2, %xmm0, %ecx
+; FAST-NEXT: vpinsrd $1, %eax, %xmm2, %xmm2
+; FAST-NEXT: vpextrd $2, %xmm1, %eax
+; FAST-NEXT: pdepl %eax, %ecx, %eax
+; FAST-NEXT: vpextrd $3, %xmm0, %ecx
+; FAST-NEXT: vpinsrd $2, %eax, %xmm2, %xmm2
+; FAST-NEXT: vpextrd $3, %xmm1, %eax
+; FAST-NEXT: pdepl %eax, %ecx, %eax
+; FAST-NEXT: vpinsrd $3, %eax, %xmm2, %xmm0
+; FAST-NEXT: retq
%res = call <4 x i32> @llvm.pdep.v4i32(<4 x i32> %val, <4 x i32> %mask)
ret <4 x i32> %res
}
define <8 x i32> @pdep_v8i32_minsize(<8 x i32> %val, <8 x i32> %mask) nounwind minsize {
-; CHECK-LABEL: pdep_v8i32_minsize:
-; CHECK: # %bb.0:
-; CHECK-NEXT: vextracti128 $1, %ymm0, %xmm3
-; CHECK-NEXT: vextracti128 $1, %ymm1, %xmm2
-; CHECK-NEXT: vpextrd $1, %xmm2, %eax
-; CHECK-NEXT: vpextrd $1, %xmm3, %ecx
-; CHECK-NEXT: vmovd %xmm3, %edx
-; CHECK-NEXT: pdepl %eax, %ecx, %eax
-; CHECK-NEXT: vmovd %xmm2, %ecx
-; CHECK-NEXT: pdepl %ecx, %edx, %ecx
-; CHECK-NEXT: vmovd %xmm0, %edx
-; CHECK-NEXT: vmovd %ecx, %xmm4
-; CHECK-NEXT: vpextrd $2, %xmm3, %ecx
-; CHECK-NEXT: vpinsrd $1, %eax, %xmm4, %xmm4
-; CHECK-NEXT: vpextrd $2, %xmm2, %eax
-; CHECK-NEXT: pdepl %eax, %ecx, %eax
-; CHECK-NEXT: vpextrd $3, %xmm3, %ecx
-; CHECK-NEXT: vpinsrd $2, %eax, %xmm4, %xmm4
-; CHECK-NEXT: vpextrd $3, %xmm2, %eax
-; CHECK-NEXT: pdepl %eax, %ecx, %eax
-; CHECK-NEXT: vpextrd $1, %xmm0, %ecx
-; CHECK-NEXT: vpinsrd $3, %eax, %xmm4, %xmm2
-; CHECK-NEXT: vpextrd $1, %xmm1, %eax
-; CHECK-NEXT: pdepl %eax, %ecx, %eax
-; CHECK-NEXT: vmovd %xmm1, %ecx
-; CHECK-NEXT: pdepl %ecx, %edx, %ecx
-; CHECK-NEXT: vmovd %ecx, %xmm3
-; CHECK-NEXT: vpextrd $2, %xmm0, %ecx
-; CHECK-NEXT: vpinsrd $1, %eax, %xmm3, %xmm3
-; CHECK-NEXT: vpextrd $2, %xmm1, %eax
-; CHECK-NEXT: pdepl %eax, %ecx, %eax
-; CHECK-NEXT: vpextrd $3, %xmm0, %ecx
-; CHECK-NEXT: vpinsrd $2, %eax, %xmm3, %xmm3
-; CHECK-NEXT: vpextrd $3, %xmm1, %eax
-; CHECK-NEXT: pdepl %eax, %ecx, %eax
-; CHECK-NEXT: vpinsrd $3, %eax, %xmm3, %xmm0
-; CHECK-NEXT: vinserti128 $1, %xmm2, %ymm0, %ymm0
-; CHECK-NEXT: retq
+; SLOW-ZNVER-LABEL: pdep_v8i32_minsize:
+; SLOW-ZNVER: # %bb.0:
+; SLOW-ZNVER-NEXT: vextracti128 $1, %ymm0, %xmm3
+; SLOW-ZNVER-NEXT: vextracti128 $1, %ymm1, %xmm2
+; SLOW-ZNVER-NEXT: vpextrd $1, %xmm0, %r8d
+; SLOW-ZNVER-NEXT: vmovd %xmm0, %r9d
+; SLOW-ZNVER-NEXT: vpextrd $1, %xmm2, %eax
+; SLOW-ZNVER-NEXT: vpextrd $1, %xmm3, %ecx
+; SLOW-ZNVER-NEXT: vmovd %xmm3, %edx
+; SLOW-ZNVER-NEXT: vpextrd $2, %xmm3, %esi
+; SLOW-ZNVER-NEXT: vpextrd $3, %xmm3, %edi
+; SLOW-ZNVER-NEXT: pdepl %eax, %ecx, %eax
+; SLOW-ZNVER-NEXT: vmovd %xmm2, %ecx
+; SLOW-ZNVER-NEXT: pdepl %ecx, %edx, %ecx
+; SLOW-ZNVER-NEXT: vpextrd $2, %xmm2, %edx
+; SLOW-ZNVER-NEXT: pdepl %edx, %esi, %edx
+; SLOW-ZNVER-NEXT: vpextrd $3, %xmm2, %esi
+; SLOW-ZNVER-NEXT: pdepl %esi, %edi, %esi
+; SLOW-ZNVER-NEXT: vpextrd $1, %xmm1, %edi
+; SLOW-ZNVER-NEXT: pdepl %edi, %r8d, %edi
+; SLOW-ZNVER-NEXT: vmovd %xmm1, %r8d
+; SLOW-ZNVER-NEXT: pdepl %r8d, %r9d, %r8d
+; SLOW-ZNVER-NEXT: vmovd %ecx, %xmm2
+; SLOW-ZNVER-NEXT: vpextrd $2, %xmm0, %ecx
+; SLOW-ZNVER-NEXT: vpinsrd $1, %eax, %xmm2, %xmm2
+; SLOW-ZNVER-NEXT: vpextrd $2, %xmm1, %eax
+; SLOW-ZNVER-NEXT: vpinsrd $2, %edx, %xmm2, %xmm2
+; SLOW-ZNVER-NEXT: vpextrd $3, %xmm0, %edx
+; SLOW-ZNVER-NEXT: pdepl %eax, %ecx, %eax
+; SLOW-ZNVER-NEXT: vpextrd $3, %xmm1, %ecx
+; SLOW-ZNVER-NEXT: vpinsrd $3, %esi, %xmm2, %xmm2
+; SLOW-ZNVER-NEXT: pdepl %ecx, %edx, %ecx
+; SLOW-ZNVER-NEXT: vmovd %r8d, %xmm3
+; SLOW-ZNVER-NEXT: vpinsrd $1, %edi, %xmm3, %xmm3
+; SLOW-ZNVER-NEXT: vpinsrd $2, %eax, %xmm3, %xmm0
+; SLOW-ZNVER-NEXT: vpinsrd $3, %ecx, %xmm0, %xmm0
+; SLOW-ZNVER-NEXT: vinserti128 $1, %xmm2, %ymm0, %ymm0
+; SLOW-ZNVER-NEXT: retq
+;
+; SLOW-BDVER4-LABEL: pdep_v8i32_minsize:
+; SLOW-BDVER4: # %bb.0:
+; SLOW-BDVER4-NEXT: vextracti128 $1, %ymm0, %xmm3
+; SLOW-BDVER4-NEXT: vextracti128 $1, %ymm1, %xmm2
+; SLOW-BDVER4-NEXT: vpextrd $1, %xmm0, %r8d
+; SLOW-BDVER4-NEXT: vmovd %xmm0, %r9d
+; SLOW-BDVER4-NEXT: vpextrd $1, %xmm2, %eax
+; SLOW-BDVER4-NEXT: vpextrd $1, %xmm3, %ecx
+; SLOW-BDVER4-NEXT: vmovd %xmm3, %edx
+; SLOW-BDVER4-NEXT: vpextrd $2, %xmm3, %esi
+; SLOW-BDVER4-NEXT: vpextrd $3, %xmm3, %edi
+; SLOW-BDVER4-NEXT: pdepl %eax, %ecx, %eax
+; SLOW-BDVER4-NEXT: vmovd %xmm2, %ecx
+; SLOW-BDVER4-NEXT: pdepl %ecx, %edx, %ecx
+; SLOW-BDVER4-NEXT: vpextrd $2, %xmm2, %edx
+; SLOW-BDVER4-NEXT: pdepl %edx, %esi, %edx
+; SLOW-BDVER4-NEXT: vpextrd $3, %xmm2, %esi
+; SLOW-BDVER4-NEXT: pdepl %esi, %edi, %esi
+; SLOW-BDVER4-NEXT: vpextrd $1, %xmm1, %edi
+; SLOW-BDVER4-NEXT: pdepl %edi, %r8d, %edi
+; SLOW-BDVER4-NEXT: vmovd %xmm1, %r8d
+; SLOW-BDVER4-NEXT: pdepl %r8d, %r9d, %r8d
+; SLOW-BDVER4-NEXT: vmovd %ecx, %xmm2
+; SLOW-BDVER4-NEXT: vpextrd $2, %xmm0, %ecx
+; SLOW-BDVER4-NEXT: vpinsrd $1, %eax, %xmm2, %xmm2
+; SLOW-BDVER4-NEXT: vpextrd $2, %xmm1, %eax
+; SLOW-BDVER4-NEXT: vpinsrd $2, %edx, %xmm2, %xmm2
+; SLOW-BDVER4-NEXT: vpextrd $3, %xmm0, %edx
+; SLOW-BDVER4-NEXT: pdepl %eax, %ecx, %eax
+; SLOW-BDVER4-NEXT: vpextrd $3, %xmm1, %ecx
+; SLOW-BDVER4-NEXT: vpinsrd $3, %esi, %xmm2, %xmm2
+; SLOW-BDVER4-NEXT: pdepl %ecx, %edx, %ecx
+; SLOW-BDVER4-NEXT: vmovd %r8d, %xmm3
+; SLOW-BDVER4-NEXT: vpinsrd $1, %edi, %xmm3, %xmm3
+; SLOW-BDVER4-NEXT: vpinsrd $2, %eax, %xmm3, %xmm0
+; SLOW-BDVER4-NEXT: vpinsrd $3, %ecx, %xmm0, %xmm0
+; SLOW-BDVER4-NEXT: vinserti128 $1, %xmm2, %ymm0, %ymm0
+; SLOW-BDVER4-NEXT: retq
+;
+; FAST-LABEL: pdep_v8i32_minsize:
+; FAST: # %bb.0:
+; FAST-NEXT: vextracti128 $1, %ymm0, %xmm3
+; FAST-NEXT: vextracti128 $1, %ymm1, %xmm2
+; FAST-NEXT: vpextrd $1, %xmm2, %eax
+; FAST-NEXT: vpextrd $1, %xmm3, %ecx
+; FAST-NEXT: vmovd %xmm3, %edx
+; FAST-NEXT: pdepl %eax, %ecx, %eax
+; FAST-NEXT: vmovd %xmm2, %ecx
+; FAST-NEXT: pdepl %ecx, %edx, %ecx
+; FAST-NEXT: vmovd %xmm0, %edx
+; FAST-NEXT: vmovd %ecx, %xmm4
+; FAST-NEXT: vpextrd $2, %xmm3, %ecx
+; FAST-NEXT: vpinsrd $1, %eax, %xmm4, %xmm4
+; FAST-NEXT: vpextrd $2, %xmm2, %eax
+; FAST-NEXT: pdepl %eax, %ecx, %eax
+; FAST-NEXT: vpextrd $3, %xmm3, %ecx
+; FAST-NEXT: vpinsrd $2, %eax, %xmm4, %xmm4
+; FAST-NEXT: vpextrd $3, %xmm2, %eax
+; FAST-NEXT: pdepl %eax, %ecx, %eax
+; FAST-NEXT: vpextrd $1, %xmm0, %ecx
+; FAST-NEXT: vpinsrd $3, %eax, %xmm4, %xmm2
+; FAST-NEXT: vpextrd $1, %xmm1, %eax
+; FAST-NEXT: pdepl %eax, %ecx, %eax
+; FAST-NEXT: vmovd %xmm1, %ecx
+; FAST-NEXT: pdepl %ecx, %edx, %ecx
+; FAST-NEXT: vmovd %ecx, %xmm3
+; FAST-NEXT: vpextrd $2, %xmm0, %ecx
+; FAST-NEXT: vpinsrd $1, %eax, %xmm3, %xmm3
+; FAST-NEXT: vpextrd $2, %xmm1, %eax
+; FAST-NEXT: pdepl %eax, %ecx, %eax
+; FAST-NEXT: vpextrd $3, %xmm0, %ecx
+; FAST-NEXT: vpinsrd $2, %eax, %xmm3, %xmm3
+; FAST-NEXT: vpextrd $3, %xmm1, %eax
+; FAST-NEXT: pdepl %eax, %ecx, %eax
+; FAST-NEXT: vpinsrd $3, %eax, %xmm3, %xmm0
+; FAST-NEXT: vinserti128 $1, %xmm2, %ymm0, %ymm0
+; FAST-NEXT: retq
%res = call <8 x i32> @llvm.pdep.v8i32(<8 x i32> %val, <8 x i32> %mask)
ret <8 x i32> %res
}
define <2 x i64> @pdep_v2i64_minsize(<2 x i64> %val, <2 x i64> %mask) nounwind minsize {
-; CHECK-LABEL: pdep_v2i64_minsize:
-; CHECK: # %bb.0:
-; CHECK-NEXT: vpextrq $1, %xmm1, %rax
-; CHECK-NEXT: vpextrq $1, %xmm0, %rcx
-; CHECK-NEXT: vmovq %xmm1, %rdx
-; CHECK-NEXT: pdepq %rax, %rcx, %rax
-; CHECK-NEXT: vmovq %xmm0, %rcx
-; CHECK-NEXT: vmovq %rax, %xmm2
-; CHECK-NEXT: pdepq %rdx, %rcx, %rax
-; CHECK-NEXT: vmovq %rax, %xmm0
-; CHECK-NEXT: vpunpcklqdq {{.*#+}} xmm0 = xmm0[0],xmm2[0]
-; CHECK-NEXT: retq
+; SLOW-ZNVER-LABEL: pdep_v2i64_minsize:
+; SLOW-ZNVER: # %bb.0:
+; SLOW-ZNVER-NEXT: vpextrq $1, %xmm1, %rax
+; SLOW-ZNVER-NEXT: vpextrq $1, %xmm0, %rcx
+; SLOW-ZNVER-NEXT: vmovq %xmm1, %rsi
+; SLOW-ZNVER-NEXT: vmovq %xmm0, %rdx
+; SLOW-ZNVER-NEXT: pdepq %rax, %rcx, %rax
+; SLOW-ZNVER-NEXT: pdepq %rsi, %rdx, %rcx
+; SLOW-ZNVER-NEXT: vmovq %rax, %xmm0
+; SLOW-ZNVER-NEXT: vmovq %rcx, %xmm1
+; SLOW-ZNVER-NEXT: vpunpcklqdq {{.*#+}} xmm0 = xmm1[0],xmm0[0]
+; SLOW-ZNVER-NEXT: retq
+;
+; SLOW-BDVER4-LABEL: pdep_v2i64_minsize:
+; SLOW-BDVER4: # %bb.0:
+; SLOW-BDVER4-NEXT: vpextrq $1, %xmm1, %rax
+; SLOW-BDVER4-NEXT: vpextrq $1, %xmm0, %rcx
+; SLOW-BDVER4-NEXT: vmovq %xmm1, %rsi
+; SLOW-BDVER4-NEXT: vmovq %xmm0, %rdx
+; SLOW-BDVER4-NEXT: pdepq %rax, %rcx, %rax
+; SLOW-BDVER4-NEXT: pdepq %rsi, %rdx, %rcx
+; SLOW-BDVER4-NEXT: vmovq %rax, %xmm0
+; SLOW-BDVER4-NEXT: vmovq %rcx, %xmm1
+; SLOW-BDVER4-NEXT: vpunpcklqdq {{.*#+}} xmm0 = xmm1[0],xmm0[0]
+; SLOW-BDVER4-NEXT: retq
+;
+; FAST-LABEL: pdep_v2i64_minsize:
+; FAST: # %bb.0:
+; FAST-NEXT: vpextrq $1, %xmm1, %rax
+; FAST-NEXT: vpextrq $1, %xmm0, %rcx
+; FAST-NEXT: vmovq %xmm1, %rdx
+; FAST-NEXT: pdepq %rax, %rcx, %rax
+; FAST-NEXT: vmovq %xmm0, %rcx
+; FAST-NEXT: vmovq %rax, %xmm2
+; FAST-NEXT: pdepq %rdx, %rcx, %rax
+; FAST-NEXT: vmovq %rax, %xmm0
+; FAST-NEXT: vpunpcklqdq {{.*#+}} xmm0 = xmm0[0],xmm2[0]
+; FAST-NEXT: retq
%res = call <2 x i64> @llvm.pdep.v2i64(<2 x i64> %val, <2 x i64> %mask)
ret <2 x i64> %res
}
define <4 x i64> @pdep_v4i64_minsize(<4 x i64> %val, <4 x i64> %mask) nounwind minsize {
-; CHECK-LABEL: pdep_v4i64_minsize:
-; CHECK: # %bb.0:
-; CHECK-NEXT: vextracti128 $1, %ymm1, %xmm2
-; CHECK-NEXT: vextracti128 $1, %ymm0, %xmm3
-; CHECK-NEXT: vpextrq $1, %xmm1, %rsi
-; CHECK-NEXT: vpextrq $1, %xmm2, %rax
-; CHECK-NEXT: vpextrq $1, %xmm3, %rcx
-; CHECK-NEXT: vmovq %xmm2, %rdx
-; CHECK-NEXT: pdepq %rax, %rcx, %rax
-; CHECK-NEXT: vmovq %xmm3, %rcx
-; CHECK-NEXT: vmovq %rax, %xmm4
-; CHECK-NEXT: pdepq %rdx, %rcx, %rax
-; CHECK-NEXT: vpextrq $1, %xmm0, %rcx
-; CHECK-NEXT: vmovq %xmm1, %rdx
-; CHECK-NEXT: vmovq %rax, %xmm2
-; CHECK-NEXT: pdepq %rsi, %rcx, %rax
-; CHECK-NEXT: vmovq %xmm0, %rcx
-; CHECK-NEXT: vpunpcklqdq {{.*#+}} xmm2 = xmm2[0],xmm4[0]
-; CHECK-NEXT: vmovq %rax, %xmm3
-; CHECK-NEXT: pdepq %rdx, %rcx, %rax
-; CHECK-NEXT: vmovq %rax, %xmm0
-; CHECK-NEXT: vpunpcklqdq {{.*#+}} xmm0 = xmm0[0],xmm3[0]
-; CHECK-NEXT: vinserti128 $1, %xmm2, %ymm0, %ymm0
-; CHECK-NEXT: retq
+; SLOW-ZNVER-LABEL: pdep_v4i64_minsize:
+; SLOW-ZNVER: # %bb.0:
+; SLOW-ZNVER-NEXT: vextracti128 $1, %ymm1, %xmm2
+; SLOW-ZNVER-NEXT: vextracti128 $1, %ymm0, %xmm3
+; SLOW-ZNVER-NEXT: vpextrq $1, %xmm2, %rax
+; SLOW-ZNVER-NEXT: vpextrq $1, %xmm3, %rcx
+; SLOW-ZNVER-NEXT: vmovq %xmm2, %rsi
+; SLOW-ZNVER-NEXT: vmovq %xmm3, %rdx
+; SLOW-ZNVER-NEXT: pdepq %rax, %rcx, %rax
+; SLOW-ZNVER-NEXT: pdepq %rsi, %rdx, %rcx
+; SLOW-ZNVER-NEXT: vpextrq $1, %xmm0, %rdx
+; SLOW-ZNVER-NEXT: vmovq %rax, %xmm2
+; SLOW-ZNVER-NEXT: vpextrq $1, %xmm1, %rax
+; SLOW-ZNVER-NEXT: vmovq %rcx, %xmm3
+; SLOW-ZNVER-NEXT: vmovq %xmm0, %rcx
+; SLOW-ZNVER-NEXT: pdepq %rax, %rdx, %rax
+; SLOW-ZNVER-NEXT: vmovq %xmm1, %rdx
+; SLOW-ZNVER-NEXT: vpunpcklqdq {{.*#+}} xmm2 = xmm3[0],xmm2[0]
+; SLOW-ZNVER-NEXT: vmovq %rax, %xmm3
+; SLOW-ZNVER-NEXT: pdepq %rdx, %rcx, %rax
+; SLOW-ZNVER-NEXT: vmovq %rax, %xmm0
+; SLOW-ZNVER-NEXT: vpunpcklqdq {{.*#+}} xmm0 = xmm0[0],xmm3[0]
+; SLOW-ZNVER-NEXT: vinserti128 $1, %xmm2, %ymm0, %ymm0
+; SLOW-ZNVER-NEXT: retq
+;
+; SLOW-BDVER4-LABEL: pdep_v4i64_minsize:
+; SLOW-BDVER4: # %bb.0:
+; SLOW-BDVER4-NEXT: vextracti128 $1, %ymm1, %xmm2
+; SLOW-BDVER4-NEXT: vextracti128 $1, %ymm0, %xmm3
+; SLOW-BDVER4-NEXT: vpextrq $1, %xmm2, %rax
+; SLOW-BDVER4-NEXT: vpextrq $1, %xmm3, %rcx
+; SLOW-BDVER4-NEXT: vmovq %xmm2, %rsi
+; SLOW-BDVER4-NEXT: vmovq %xmm3, %rdx
+; SLOW-BDVER4-NEXT: pdepq %rax, %rcx, %rax
+; SLOW-BDVER4-NEXT: pdepq %rsi, %rdx, %rcx
+; SLOW-BDVER4-NEXT: vpextrq $1, %xmm0, %rdx
+; SLOW-BDVER4-NEXT: vmovq %rax, %xmm2
+; SLOW-BDVER4-NEXT: vpextrq $1, %xmm1, %rax
+; SLOW-BDVER4-NEXT: vmovq %rcx, %xmm3
+; SLOW-BDVER4-NEXT: vmovq %xmm0, %rcx
+; SLOW-BDVER4-NEXT: pdepq %rax, %rdx, %rax
+; SLOW-BDVER4-NEXT: vmovq %xmm1, %rdx
+; SLOW-BDVER4-NEXT: vpunpcklqdq {{.*#+}} xmm2 = xmm3[0],xmm2[0]
+; SLOW-BDVER4-NEXT: vmovq %rax, %xmm3
+; SLOW-BDVER4-NEXT: pdepq %rdx, %rcx, %rax
+; SLOW-BDVER4-NEXT: vmovq %rax, %xmm0
+; SLOW-BDVER4-NEXT: vpunpcklqdq {{.*#+}} xmm0 = xmm0[0],xmm3[0]
+; SLOW-BDVER4-NEXT: vinserti128 $1, %xmm2, %ymm0, %ymm0
+; SLOW-BDVER4-NEXT: retq
+;
+; FAST-LABEL: pdep_v4i64_minsize:
+; FAST: # %bb.0:
+; FAST-NEXT: vextracti128 $1, %ymm1, %xmm2
+; FAST-NEXT: vextracti128 $1, %ymm0, %xmm3
+; FAST-NEXT: vpextrq $1, %xmm1, %rsi
+; FAST-NEXT: vpextrq $1, %xmm2, %rax
+; FAST-NEXT: vpextrq $1, %xmm3, %rcx
+; FAST-NEXT: vmovq %xmm2, %rdx
+; FAST-NEXT: pdepq %rax, %rcx, %rax
+; FAST-NEXT: vmovq %xmm3, %rcx
+; FAST-NEXT: vmovq %rax, %xmm4
+; FAST-NEXT: pdepq %rdx, %rcx, %rax
+; FAST-NEXT: vpextrq $1, %xmm0, %rcx
+; FAST-NEXT: vmovq %xmm1, %rdx
+; FAST-NEXT: vmovq %rax, %xmm2
+; FAST-NEXT: pdepq %rsi, %rcx, %rax
+; FAST-NEXT: vmovq %xmm0, %rcx
+; FAST-NEXT: vpunpcklqdq {{.*#+}} xmm2 = xmm2[0],xmm4[0]
+; FAST-NEXT: vmovq %rax, %xmm3
+; FAST-NEXT: pdepq %rdx, %rcx, %rax
+; FAST-NEXT: vmovq %rax, %xmm0
+; FAST-NEXT: vpunpcklqdq {{.*#+}} xmm0 = xmm0[0],xmm3[0]
+; FAST-NEXT: vinserti128 $1, %xmm2, %ymm0, %ymm0
+; FAST-NEXT: retq
%res = call <4 x i64> @llvm.pdep.v4i64(<4 x i64> %val, <4 x i64> %mask)
ret <4 x i64> %res
}
+;; NOTE: These prefixes are unused and the list is autogenerated. Do not add tests below this line:
+; CHECK: {{.*}}
diff --git a/llvm/test/CodeGen/X86/znver-pext.ll b/llvm/test/CodeGen/X86/znver-pext.ll
index 5fbc3c613f025..d714588d1aa05 100644
--- a/llvm/test/CodeGen/X86/znver-pext.ll
+++ b/llvm/test/CodeGen/X86/znver-pext.ll
@@ -586,114 +586,213 @@ define <4 x i64> @pext_v4i64(<4 x i64> %val, <4 x i64> %mask) nounwind {
;
define <4 x i32> @pext_v4i32_minsize(<4 x i32> %val, <4 x i32> %mask) nounwind minsize {
-; CHECK-LABEL: pext_v4i32_minsize:
-; CHECK: # %bb.0:
-; CHECK-NEXT: vpextrd $1, %xmm1, %eax
-; CHECK-NEXT: vpextrd $1, %xmm0, %ecx
-; CHECK-NEXT: vmovd %xmm0, %edx
-; CHECK-NEXT: pextl %eax, %ecx, %eax
-; CHECK-NEXT: vmovd %xmm1, %ecx
-; CHECK-NEXT: pextl %ecx, %edx, %ecx
-; CHECK-NEXT: vmovd %ecx, %xmm2
-; CHECK-NEXT: vpextrd $2, %xmm0, %ecx
-; CHECK-NEXT: vpinsrd $1, %eax, %xmm2, %xmm2
-; CHECK-NEXT: vpextrd $2, %xmm1, %eax
-; CHECK-NEXT: pextl %eax, %ecx, %eax
-; CHECK-NEXT: vpextrd $3, %xmm0, %ecx
-; CHECK-NEXT: vpinsrd $2, %eax, %xmm2, %xmm2
-; CHECK-NEXT: vpextrd $3, %xmm1, %eax
-; CHECK-NEXT: pextl %eax, %ecx, %eax
-; CHECK-NEXT: vpinsrd $3, %eax, %xmm2, %xmm0
-; CHECK-NEXT: retq
+; SLOW-LABEL: pext_v4i32_minsize:
+; SLOW: # %bb.0:
+; SLOW-NEXT: vpextrd $1, %xmm1, %eax
+; SLOW-NEXT: vpextrd $1, %xmm0, %ecx
+; SLOW-NEXT: vmovd %xmm0, %edx
+; SLOW-NEXT: vpextrd $2, %xmm0, %esi
+; SLOW-NEXT: pextl %eax, %ecx, %eax
+; SLOW-NEXT: vmovd %xmm1, %ecx
+; SLOW-NEXT: pextl %ecx, %edx, %ecx
+; SLOW-NEXT: vpextrd $2, %xmm1, %edx
+; SLOW-NEXT: pextl %edx, %esi, %edx
+; SLOW-NEXT: vpextrd $3, %xmm0, %esi
+; SLOW-NEXT: vmovd %ecx, %xmm2
+; SLOW-NEXT: vpextrd $3, %xmm1, %ecx
+; SLOW-NEXT: pextl %ecx, %esi, %ecx
+; SLOW-NEXT: vpinsrd $1, %eax, %xmm2, %xmm0
+; SLOW-NEXT: vpinsrd $2, %edx, %xmm0, %xmm0
+; SLOW-NEXT: vpinsrd $3, %ecx, %xmm0, %xmm0
+; SLOW-NEXT: retq
+;
+; FAST-LABEL: pext_v4i32_minsize:
+; FAST: # %bb.0:
+; FAST-NEXT: vpextrd $1, %xmm1, %eax
+; FAST-NEXT: vpextrd $1, %xmm0, %ecx
+; FAST-NEXT: vmovd %xmm0, %edx
+; FAST-NEXT: pextl %eax, %ecx, %eax
+; FAST-NEXT: vmovd %xmm1, %ecx
+; FAST-NEXT: pextl %ecx, %edx, %ecx
+; FAST-NEXT: vmovd %ecx, %xmm2
+; FAST-NEXT: vpextrd $2, %xmm0, %ecx
+; FAST-NEXT: vpinsrd $1, %eax, %xmm2, %xmm2
+; FAST-NEXT: vpextrd $2, %xmm1, %eax
+; FAST-NEXT: pextl %eax, %ecx, %eax
+; FAST-NEXT: vpextrd $3, %xmm0, %ecx
+; FAST-NEXT: vpinsrd $2, %eax, %xmm2, %xmm2
+; FAST-NEXT: vpextrd $3, %xmm1, %eax
+; FAST-NEXT: pextl %eax, %ecx, %eax
+; FAST-NEXT: vpinsrd $3, %eax, %xmm2, %xmm0
+; FAST-NEXT: retq
%res = call <4 x i32> @llvm.pext.v4i32(<4 x i32> %val, <4 x i32> %mask)
ret <4 x i32> %res
}
define <8 x i32> @pext_v8i32_minsize(<8 x i32> %val, <8 x i32> %mask) nounwind minsize {
-; CHECK-LABEL: pext_v8i32_minsize:
-; CHECK: # %bb.0:
-; CHECK-NEXT: vextracti128 $1, %ymm0, %xmm3
-; CHECK-NEXT: vextracti128 $1, %ymm1, %xmm2
-; CHECK-NEXT: vpextrd $1, %xmm2, %eax
-; CHECK-NEXT: vpextrd $1, %xmm3, %ecx
-; CHECK-NEXT: vmovd %xmm3, %edx
-; CHECK-NEXT: pextl %eax, %ecx, %eax
-; CHECK-NEXT: vmovd %xmm2, %ecx
-; CHECK-NEXT: pextl %ecx, %edx, %ecx
-; CHECK-NEXT: vmovd %xmm0, %edx
-; CHECK-NEXT: vmovd %ecx, %xmm4
-; CHECK-NEXT: vpextrd $2, %xmm3, %ecx
-; CHECK-NEXT: vpinsrd $1, %eax, %xmm4, %xmm4
-; CHECK-NEXT: vpextrd $2, %xmm2, %eax
-; CHECK-NEXT: pextl %eax, %ecx, %eax
-; CHECK-NEXT: vpextrd $3, %xmm3, %ecx
-; CHECK-NEXT: vpinsrd $2, %eax, %xmm4, %xmm4
-; CHECK-NEXT: vpextrd $3, %xmm2, %eax
-; CHECK-NEXT: pextl %eax, %ecx, %eax
-; CHECK-NEXT: vpextrd $1, %xmm0, %ecx
-; CHECK-NEXT: vpinsrd $3, %eax, %xmm4, %xmm2
-; CHECK-NEXT: vpextrd $1, %xmm1, %eax
-; CHECK-NEXT: pextl %eax, %ecx, %eax
-; CHECK-NEXT: vmovd %xmm1, %ecx
-; CHECK-NEXT: pextl %ecx, %edx, %ecx
-; CHECK-NEXT: vmovd %ecx, %xmm3
-; CHECK-NEXT: vpextrd $2, %xmm0, %ecx
-; CHECK-NEXT: vpinsrd $1, %eax, %xmm3, %xmm3
-; CHECK-NEXT: vpextrd $2, %xmm1, %eax
-; CHECK-NEXT: pextl %eax, %ecx, %eax
-; CHECK-NEXT: vpextrd $3, %xmm0, %ecx
-; CHECK-NEXT: vpinsrd $2, %eax, %xmm3, %xmm3
-; CHECK-NEXT: vpextrd $3, %xmm1, %eax
-; CHECK-NEXT: pextl %eax, %ecx, %eax
-; CHECK-NEXT: vpinsrd $3, %eax, %xmm3, %xmm0
-; CHECK-NEXT: vinserti128 $1, %xmm2, %ymm0, %ymm0
-; CHECK-NEXT: retq
+; SLOW-LABEL: pext_v8i32_minsize:
+; SLOW: # %bb.0:
+; SLOW-NEXT: vextracti128 $1, %ymm0, %xmm3
+; SLOW-NEXT: vextracti128 $1, %ymm1, %xmm2
+; SLOW-NEXT: vpextrd $1, %xmm0, %r8d
+; SLOW-NEXT: vmovd %xmm0, %r9d
+; SLOW-NEXT: vpextrd $1, %xmm2, %eax
+; SLOW-NEXT: vpextrd $1, %xmm3, %ecx
+; SLOW-NEXT: vmovd %xmm3, %edx
+; SLOW-NEXT: vpextrd $2, %xmm3, %esi
+; SLOW-NEXT: vpextrd $3, %xmm3, %edi
+; SLOW-NEXT: pextl %eax, %ecx, %eax
+; SLOW-NEXT: vmovd %xmm2, %ecx
+; SLOW-NEXT: pextl %ecx, %edx, %ecx
+; SLOW-NEXT: vpextrd $2, %xmm2, %edx
+; SLOW-NEXT: pextl %edx, %esi, %edx
+; SLOW-NEXT: vpextrd $3, %xmm2, %esi
+; SLOW-NEXT: pextl %esi, %edi, %esi
+; SLOW-NEXT: vpextrd $1, %xmm1, %edi
+; SLOW-NEXT: pextl %edi, %r8d, %edi
+; SLOW-NEXT: vmovd %xmm1, %r8d
+; SLOW-NEXT: pextl %r8d, %r9d, %r8d
+; SLOW-NEXT: vmovd %ecx, %xmm2
+; SLOW-NEXT: vpextrd $2, %xmm0, %ecx
+; SLOW-NEXT: vpinsrd $1, %eax, %xmm2, %xmm2
+; SLOW-NEXT: vpextrd $2, %xmm1, %eax
+; SLOW-NEXT: vpinsrd $2, %edx, %xmm2, %xmm2
+; SLOW-NEXT: vpextrd $3, %xmm0, %edx
+; SLOW-NEXT: pextl %eax, %ecx, %eax
+; SLOW-NEXT: vpextrd $3, %xmm1, %ecx
+; SLOW-NEXT: vpinsrd $3, %esi, %xmm2, %xmm2
+; SLOW-NEXT: pextl %ecx, %edx, %ecx
+; SLOW-NEXT: vmovd %r8d, %xmm3
+; SLOW-NEXT: vpinsrd $1, %edi, %xmm3, %xmm3
+; SLOW-NEXT: vpinsrd $2, %eax, %xmm3, %xmm0
+; SLOW-NEXT: vpinsrd $3, %ecx, %xmm0, %xmm0
+; SLOW-NEXT: vinserti128 $1, %xmm2, %ymm0, %ymm0
+; SLOW-NEXT: retq
+;
+; FAST-LABEL: pext_v8i32_minsize:
+; FAST: # %bb.0:
+; FAST-NEXT: vextracti128 $1, %ymm0, %xmm3
+; FAST-NEXT: vextracti128 $1, %ymm1, %xmm2
+; FAST-NEXT: vpextrd $1, %xmm2, %eax
+; FAST-NEXT: vpextrd $1, %xmm3, %ecx
+; FAST-NEXT: vmovd %xmm3, %edx
+; FAST-NEXT: pextl %eax, %ecx, %eax
+; FAST-NEXT: vmovd %xmm2, %ecx
+; FAST-NEXT: pextl %ecx, %edx, %ecx
+; FAST-NEXT: vmovd %xmm0, %edx
+; FAST-NEXT: vmovd %ecx, %xmm4
+; FAST-NEXT: vpextrd $2, %xmm3, %ecx
+; FAST-NEXT: vpinsrd $1, %eax, %xmm4, %xmm4
+; FAST-NEXT: vpextrd $2, %xmm2, %eax
+; FAST-NEXT: pextl %eax, %ecx, %eax
+; FAST-NEXT: vpextrd $3, %xmm3, %ecx
+; FAST-NEXT: vpinsrd $2, %eax, %xmm4, %xmm4
+; FAST-NEXT: vpextrd $3, %xmm2, %eax
+; FAST-NEXT: pextl %eax, %ecx, %eax
+; FAST-NEXT: vpextrd $1, %xmm0, %ecx
+; FAST-NEXT: vpinsrd $3, %eax, %xmm4, %xmm2
+; FAST-NEXT: vpextrd $1, %xmm1, %eax
+; FAST-NEXT: pextl %eax, %ecx, %eax
+; FAST-NEXT: vmovd %xmm1, %ecx
+; FAST-NEXT: pextl %ecx, %edx, %ecx
+; FAST-NEXT: vmovd %ecx, %xmm3
+; FAST-NEXT: vpextrd $2, %xmm0, %ecx
+; FAST-NEXT: vpinsrd $1, %eax, %xmm3, %xmm3
+; FAST-NEXT: vpextrd $2, %xmm1, %eax
+; FAST-NEXT: pextl %eax, %ecx, %eax
+; FAST-NEXT: vpextrd $3, %xmm0, %ecx
+; FAST-NEXT: vpinsrd $2, %eax, %xmm3, %xmm3
+; FAST-NEXT: vpextrd $3, %xmm1, %eax
+; FAST-NEXT: pextl %eax, %ecx, %eax
+; FAST-NEXT: vpinsrd $3, %eax, %xmm3, %xmm0
+; FAST-NEXT: vinserti128 $1, %xmm2, %ymm0, %ymm0
+; FAST-NEXT: retq
%res = call <8 x i32> @llvm.pext.v8i32(<8 x i32> %val, <8 x i32> %mask)
ret <8 x i32> %res
}
define <2 x i64> @pext_v2i64_minsize(<2 x i64> %val, <2 x i64> %mask) nounwind minsize {
-; CHECK-LABEL: pext_v2i64_minsize:
-; CHECK: # %bb.0:
-; CHECK-NEXT: vpextrq $1, %xmm1, %rax
-; CHECK-NEXT: vpextrq $1, %xmm0, %rcx
-; CHECK-NEXT: vmovq %xmm1, %rdx
-; CHECK-NEXT: pextq %rax, %rcx, %rax
-; CHECK-NEXT: vmovq %xmm0, %rcx
-; CHECK-NEXT: vmovq %rax, %xmm2
-; CHECK-NEXT: pextq %rdx, %rcx, %rax
-; CHECK-NEXT: vmovq %rax, %xmm0
-; CHECK-NEXT: vpunpcklqdq {{.*#+}} xmm0 = xmm0[0],xmm2[0]
-; CHECK-NEXT: retq
+; SLOW-LABEL: pext_v2i64_minsize:
+; SLOW: # %bb.0:
+; SLOW-NEXT: vpextrq $1, %xmm1, %rax
+; SLOW-NEXT: vpextrq $1, %xmm0, %rcx
+; SLOW-NEXT: vmovq %xmm1, %rsi
+; SLOW-NEXT: vmovq %xmm0, %rdx
+; SLOW-NEXT: pextq %rax, %rcx, %rax
+; SLOW-NEXT: pextq %rsi, %rdx, %rcx
+; SLOW-NEXT: vmovq %rax, %xmm0
+; SLOW-NEXT: vmovq %rcx, %xmm1
+; SLOW-NEXT: vpunpcklqdq {{.*#+}} xmm0 = xmm1[0],xmm0[0]
+; SLOW-NEXT: retq
+;
+; FAST-LABEL: pext_v2i64_minsize:
+; FAST: # %bb.0:
+; FAST-NEXT: vpextrq $1, %xmm1, %rax
+; FAST-NEXT: vpextrq $1, %xmm0, %rcx
+; FAST-NEXT: vmovq %xmm1, %rdx
+; FAST-NEXT: pextq %rax, %rcx, %rax
+; FAST-NEXT: vmovq %xmm0, %rcx
+; FAST-NEXT: vmovq %rax, %xmm2
+; FAST-NEXT: pextq %rdx, %rcx, %rax
+; FAST-NEXT: vmovq %rax, %xmm0
+; FAST-NEXT: vpunpcklqdq {{.*#+}} xmm0 = xmm0[0],xmm2[0]
+; FAST-NEXT: retq
%res = call <2 x i64> @llvm.pext.v2i64(<2 x i64> %val, <2 x i64> %mask)
ret <2 x i64> %res
}
define <4 x i64> @pext_v4i64_minsize(<4 x i64> %val, <4 x i64> %mask) nounwind minsize {
-; CHECK-LABEL: pext_v4i64_minsize:
-; CHECK: # %bb.0:
-; CHECK-NEXT: vextracti128 $1, %ymm1, %xmm2
-; CHECK-NEXT: vextracti128 $1, %ymm0, %xmm3
-; CHECK-NEXT: vpextrq $1, %xmm1, %rsi
-; CHECK-NEXT: vpextrq $1, %xmm2, %rax
-; CHECK-NEXT: vpextrq $1, %xmm3, %rcx
-; CHECK-NEXT: vmovq %xmm2, %rdx
-; CHECK-NEXT: pextq %rax, %rcx, %rax
-; CHECK-NEXT: vmovq %xmm3, %rcx
-; CHECK-NEXT: vmovq %rax, %xmm4
-; CHECK-NEXT: pextq %rdx, %rcx, %rax
-; CHECK-NEXT: vpextrq $1, %xmm0, %rcx
-; CHECK-NEXT: vmovq %xmm1, %rdx
-; CHECK-NEXT: vmovq %rax, %xmm2
-; CHECK-NEXT: pextq %rsi, %rcx, %rax
-; CHECK-NEXT: vmovq %xmm0, %rcx
-; CHECK-NEXT: vpunpcklqdq {{.*#+}} xmm2 = xmm2[0],xmm4[0]
-; CHECK-NEXT: vmovq %rax, %xmm3
-; CHECK-NEXT: pextq %rdx, %rcx, %rax
-; CHECK-NEXT: vmovq %rax, %xmm0
-; CHECK-NEXT: vpunpcklqdq {{.*#+}} xmm0 = xmm0[0],xmm3[0]
-; CHECK-NEXT: vinserti128 $1, %xmm2, %ymm0, %ymm0
-; CHECK-NEXT: retq
+; SLOW-LABEL: pext_v4i64_minsize:
+; SLOW: # %bb.0:
+; SLOW-NEXT: vextracti128 $1, %ymm1, %xmm2
+; SLOW-NEXT: vextracti128 $1, %ymm0, %xmm3
+; SLOW-NEXT: vpextrq $1, %xmm2, %rax
+; SLOW-NEXT: vpextrq $1, %xmm3, %rcx
+; SLOW-NEXT: vmovq %xmm2, %rsi
+; SLOW-NEXT: vmovq %xmm3, %rdx
+; SLOW-NEXT: pextq %rax, %rcx, %rax
+; SLOW-NEXT: pextq %rsi, %rdx, %rcx
+; SLOW-NEXT: vpextrq $1, %xmm0, %rdx
+; SLOW-NEXT: vmovq %rax, %xmm2
+; SLOW-NEXT: vpextrq $1, %xmm1, %rax
+; SLOW-NEXT: vmovq %rcx, %xmm3
+; SLOW-NEXT: vmovq %xmm0, %rcx
+; SLOW-NEXT: pextq %rax, %rdx, %rax
+; SLOW-NEXT: vmovq %xmm1, %rdx
+; SLOW-NEXT: vpunpcklqdq {{.*#+}} xmm2 = xmm3[0],xmm2[0]
+; SLOW-NEXT: vmovq %rax, %xmm3
+; SLOW-NEXT: pextq %rdx, %rcx, %rax
+; SLOW-NEXT: vmovq %rax, %xmm0
+; SLOW-NEXT: vpunpcklqdq {{.*#+}} xmm0 = xmm0[0],xmm3[0]
+; SLOW-NEXT: vinserti128 $1, %xmm2, %ymm0, %ymm0
+; SLOW-NEXT: retq
+;
+; FAST-LABEL: pext_v4i64_minsize:
+; FAST: # %bb.0:
+; FAST-NEXT: vextracti128 $1, %ymm1, %xmm2
+; FAST-NEXT: vextracti128 $1, %ymm0, %xmm3
+; FAST-NEXT: vpextrq $1, %xmm1, %rsi
+; FAST-NEXT: vpextrq $1, %xmm2, %rax
+; FAST-NEXT: vpextrq $1, %xmm3, %rcx
+; FAST-NEXT: vmovq %xmm2, %rdx
+; FAST-NEXT: pextq %rax, %rcx, %rax
+; FAST-NEXT: vmovq %xmm3, %rcx
+; FAST-NEXT: vmovq %rax, %xmm4
+; FAST-NEXT: pextq %rdx, %rcx, %rax
+; FAST-NEXT: vpextrq $1, %xmm0, %rcx
+; FAST-NEXT: vmovq %xmm1, %rdx
+; FAST-NEXT: vmovq %rax, %xmm2
+; FAST-NEXT: pextq %rsi, %rcx, %rax
+; FAST-NEXT: vmovq %xmm0, %rcx
+; FAST-NEXT: vpunpcklqdq {{.*#+}} xmm2 = xmm2[0],xmm4[0]
+; FAST-NEXT: vmovq %rax, %xmm3
+; FAST-NEXT: pextq %rdx, %rcx, %rax
+; FAST-NEXT: vmovq %rax, %xmm0
+; FAST-NEXT: vpunpcklqdq {{.*#+}} xmm0 = xmm0[0],xmm3[0]
+; FAST-NEXT: vinserti128 $1, %xmm2, %ymm0, %ymm0
+; FAST-NEXT: retq
%res = call <4 x i64> @llvm.pext.v4i64(<4 x i64> %val, <4 x i64> %mask)
ret <4 x i64> %res
}
+;; NOTE: These prefixes are unused and the list is autogenerated. Do not add tests below this line:
+; CHECK: {{.*}}
>From f86150a667e33aa0c2d41e7a15203a41865e8785 Mon Sep 17 00:00:00 2001
From: "Ziegler, Tim" <tim.ziegler at intel.com>
Date: Thu, 27 Aug 2026 20:41:16 +0200
Subject: [PATCH 3/6] reverted zen test changes
---
llvm/test/CodeGen/X86/znver-pdep.ll | 382 +++++++---------------------
llvm/test/CodeGen/X86/znver-pext.ll | 285 +++++++--------------
2 files changed, 186 insertions(+), 481 deletions(-)
diff --git a/llvm/test/CodeGen/X86/znver-pdep.ll b/llvm/test/CodeGen/X86/znver-pdep.ll
index afd97ebca3e0e..1acb6a49f44d0 100644
--- a/llvm/test/CodeGen/X86/znver-pdep.ll
+++ b/llvm/test/CodeGen/X86/znver-pdep.ll
@@ -1005,310 +1005,114 @@ define <4 x i64> @pdep_v4i64(<4 x i64> %val, <4 x i64> %mask) nounwind {
;
define <4 x i32> @pdep_v4i32_minsize(<4 x i32> %val, <4 x i32> %mask) nounwind minsize {
-; SLOW-ZNVER-LABEL: pdep_v4i32_minsize:
-; SLOW-ZNVER: # %bb.0:
-; SLOW-ZNVER-NEXT: vpextrd $1, %xmm1, %eax
-; SLOW-ZNVER-NEXT: vpextrd $1, %xmm0, %ecx
-; SLOW-ZNVER-NEXT: vmovd %xmm0, %edx
-; SLOW-ZNVER-NEXT: vpextrd $2, %xmm0, %esi
-; SLOW-ZNVER-NEXT: pdepl %eax, %ecx, %eax
-; SLOW-ZNVER-NEXT: vmovd %xmm1, %ecx
-; SLOW-ZNVER-NEXT: pdepl %ecx, %edx, %ecx
-; SLOW-ZNVER-NEXT: vpextrd $2, %xmm1, %edx
-; SLOW-ZNVER-NEXT: pdepl %edx, %esi, %edx
-; SLOW-ZNVER-NEXT: vpextrd $3, %xmm0, %esi
-; SLOW-ZNVER-NEXT: vmovd %ecx, %xmm2
-; SLOW-ZNVER-NEXT: vpextrd $3, %xmm1, %ecx
-; SLOW-ZNVER-NEXT: pdepl %ecx, %esi, %ecx
-; SLOW-ZNVER-NEXT: vpinsrd $1, %eax, %xmm2, %xmm0
-; SLOW-ZNVER-NEXT: vpinsrd $2, %edx, %xmm0, %xmm0
-; SLOW-ZNVER-NEXT: vpinsrd $3, %ecx, %xmm0, %xmm0
-; SLOW-ZNVER-NEXT: retq
-;
-; SLOW-BDVER4-LABEL: pdep_v4i32_minsize:
-; SLOW-BDVER4: # %bb.0:
-; SLOW-BDVER4-NEXT: vpextrd $1, %xmm1, %eax
-; SLOW-BDVER4-NEXT: vpextrd $1, %xmm0, %ecx
-; SLOW-BDVER4-NEXT: vmovd %xmm0, %edx
-; SLOW-BDVER4-NEXT: vpextrd $2, %xmm0, %esi
-; SLOW-BDVER4-NEXT: pdepl %eax, %ecx, %eax
-; SLOW-BDVER4-NEXT: vmovd %xmm1, %ecx
-; SLOW-BDVER4-NEXT: pdepl %ecx, %edx, %ecx
-; SLOW-BDVER4-NEXT: vpextrd $2, %xmm1, %edx
-; SLOW-BDVER4-NEXT: pdepl %edx, %esi, %edx
-; SLOW-BDVER4-NEXT: vpextrd $3, %xmm0, %esi
-; SLOW-BDVER4-NEXT: vmovd %ecx, %xmm2
-; SLOW-BDVER4-NEXT: vpextrd $3, %xmm1, %ecx
-; SLOW-BDVER4-NEXT: pdepl %ecx, %esi, %ecx
-; SLOW-BDVER4-NEXT: vpinsrd $1, %eax, %xmm2, %xmm0
-; SLOW-BDVER4-NEXT: vpinsrd $2, %edx, %xmm0, %xmm0
-; SLOW-BDVER4-NEXT: vpinsrd $3, %ecx, %xmm0, %xmm0
-; SLOW-BDVER4-NEXT: retq
-;
-; FAST-LABEL: pdep_v4i32_minsize:
-; FAST: # %bb.0:
-; FAST-NEXT: vpextrd $1, %xmm1, %eax
-; FAST-NEXT: vpextrd $1, %xmm0, %ecx
-; FAST-NEXT: vmovd %xmm0, %edx
-; FAST-NEXT: pdepl %eax, %ecx, %eax
-; FAST-NEXT: vmovd %xmm1, %ecx
-; FAST-NEXT: pdepl %ecx, %edx, %ecx
-; FAST-NEXT: vmovd %ecx, %xmm2
-; FAST-NEXT: vpextrd $2, %xmm0, %ecx
-; FAST-NEXT: vpinsrd $1, %eax, %xmm2, %xmm2
-; FAST-NEXT: vpextrd $2, %xmm1, %eax
-; FAST-NEXT: pdepl %eax, %ecx, %eax
-; FAST-NEXT: vpextrd $3, %xmm0, %ecx
-; FAST-NEXT: vpinsrd $2, %eax, %xmm2, %xmm2
-; FAST-NEXT: vpextrd $3, %xmm1, %eax
-; FAST-NEXT: pdepl %eax, %ecx, %eax
-; FAST-NEXT: vpinsrd $3, %eax, %xmm2, %xmm0
-; FAST-NEXT: retq
+; CHECK-LABEL: pdep_v4i32_minsize:
+; CHECK: # %bb.0:
+; CHECK-NEXT: vpextrd $1, %xmm1, %eax
+; CHECK-NEXT: vpextrd $1, %xmm0, %ecx
+; CHECK-NEXT: vmovd %xmm0, %edx
+; CHECK-NEXT: pdepl %eax, %ecx, %eax
+; CHECK-NEXT: vmovd %xmm1, %ecx
+; CHECK-NEXT: pdepl %ecx, %edx, %ecx
+; CHECK-NEXT: vmovd %ecx, %xmm2
+; CHECK-NEXT: vpextrd $2, %xmm0, %ecx
+; CHECK-NEXT: vpinsrd $1, %eax, %xmm2, %xmm2
+; CHECK-NEXT: vpextrd $2, %xmm1, %eax
+; CHECK-NEXT: pdepl %eax, %ecx, %eax
+; CHECK-NEXT: vpextrd $3, %xmm0, %ecx
+; CHECK-NEXT: vpinsrd $2, %eax, %xmm2, %xmm2
+; CHECK-NEXT: vpextrd $3, %xmm1, %eax
+; CHECK-NEXT: pdepl %eax, %ecx, %eax
+; CHECK-NEXT: vpinsrd $3, %eax, %xmm2, %xmm0
+; CHECK-NEXT: retq
%res = call <4 x i32> @llvm.pdep.v4i32(<4 x i32> %val, <4 x i32> %mask)
ret <4 x i32> %res
}
define <8 x i32> @pdep_v8i32_minsize(<8 x i32> %val, <8 x i32> %mask) nounwind minsize {
-; SLOW-ZNVER-LABEL: pdep_v8i32_minsize:
-; SLOW-ZNVER: # %bb.0:
-; SLOW-ZNVER-NEXT: vextracti128 $1, %ymm0, %xmm3
-; SLOW-ZNVER-NEXT: vextracti128 $1, %ymm1, %xmm2
-; SLOW-ZNVER-NEXT: vpextrd $1, %xmm0, %r8d
-; SLOW-ZNVER-NEXT: vmovd %xmm0, %r9d
-; SLOW-ZNVER-NEXT: vpextrd $1, %xmm2, %eax
-; SLOW-ZNVER-NEXT: vpextrd $1, %xmm3, %ecx
-; SLOW-ZNVER-NEXT: vmovd %xmm3, %edx
-; SLOW-ZNVER-NEXT: vpextrd $2, %xmm3, %esi
-; SLOW-ZNVER-NEXT: vpextrd $3, %xmm3, %edi
-; SLOW-ZNVER-NEXT: pdepl %eax, %ecx, %eax
-; SLOW-ZNVER-NEXT: vmovd %xmm2, %ecx
-; SLOW-ZNVER-NEXT: pdepl %ecx, %edx, %ecx
-; SLOW-ZNVER-NEXT: vpextrd $2, %xmm2, %edx
-; SLOW-ZNVER-NEXT: pdepl %edx, %esi, %edx
-; SLOW-ZNVER-NEXT: vpextrd $3, %xmm2, %esi
-; SLOW-ZNVER-NEXT: pdepl %esi, %edi, %esi
-; SLOW-ZNVER-NEXT: vpextrd $1, %xmm1, %edi
-; SLOW-ZNVER-NEXT: pdepl %edi, %r8d, %edi
-; SLOW-ZNVER-NEXT: vmovd %xmm1, %r8d
-; SLOW-ZNVER-NEXT: pdepl %r8d, %r9d, %r8d
-; SLOW-ZNVER-NEXT: vmovd %ecx, %xmm2
-; SLOW-ZNVER-NEXT: vpextrd $2, %xmm0, %ecx
-; SLOW-ZNVER-NEXT: vpinsrd $1, %eax, %xmm2, %xmm2
-; SLOW-ZNVER-NEXT: vpextrd $2, %xmm1, %eax
-; SLOW-ZNVER-NEXT: vpinsrd $2, %edx, %xmm2, %xmm2
-; SLOW-ZNVER-NEXT: vpextrd $3, %xmm0, %edx
-; SLOW-ZNVER-NEXT: pdepl %eax, %ecx, %eax
-; SLOW-ZNVER-NEXT: vpextrd $3, %xmm1, %ecx
-; SLOW-ZNVER-NEXT: vpinsrd $3, %esi, %xmm2, %xmm2
-; SLOW-ZNVER-NEXT: pdepl %ecx, %edx, %ecx
-; SLOW-ZNVER-NEXT: vmovd %r8d, %xmm3
-; SLOW-ZNVER-NEXT: vpinsrd $1, %edi, %xmm3, %xmm3
-; SLOW-ZNVER-NEXT: vpinsrd $2, %eax, %xmm3, %xmm0
-; SLOW-ZNVER-NEXT: vpinsrd $3, %ecx, %xmm0, %xmm0
-; SLOW-ZNVER-NEXT: vinserti128 $1, %xmm2, %ymm0, %ymm0
-; SLOW-ZNVER-NEXT: retq
-;
-; SLOW-BDVER4-LABEL: pdep_v8i32_minsize:
-; SLOW-BDVER4: # %bb.0:
-; SLOW-BDVER4-NEXT: vextracti128 $1, %ymm0, %xmm3
-; SLOW-BDVER4-NEXT: vextracti128 $1, %ymm1, %xmm2
-; SLOW-BDVER4-NEXT: vpextrd $1, %xmm0, %r8d
-; SLOW-BDVER4-NEXT: vmovd %xmm0, %r9d
-; SLOW-BDVER4-NEXT: vpextrd $1, %xmm2, %eax
-; SLOW-BDVER4-NEXT: vpextrd $1, %xmm3, %ecx
-; SLOW-BDVER4-NEXT: vmovd %xmm3, %edx
-; SLOW-BDVER4-NEXT: vpextrd $2, %xmm3, %esi
-; SLOW-BDVER4-NEXT: vpextrd $3, %xmm3, %edi
-; SLOW-BDVER4-NEXT: pdepl %eax, %ecx, %eax
-; SLOW-BDVER4-NEXT: vmovd %xmm2, %ecx
-; SLOW-BDVER4-NEXT: pdepl %ecx, %edx, %ecx
-; SLOW-BDVER4-NEXT: vpextrd $2, %xmm2, %edx
-; SLOW-BDVER4-NEXT: pdepl %edx, %esi, %edx
-; SLOW-BDVER4-NEXT: vpextrd $3, %xmm2, %esi
-; SLOW-BDVER4-NEXT: pdepl %esi, %edi, %esi
-; SLOW-BDVER4-NEXT: vpextrd $1, %xmm1, %edi
-; SLOW-BDVER4-NEXT: pdepl %edi, %r8d, %edi
-; SLOW-BDVER4-NEXT: vmovd %xmm1, %r8d
-; SLOW-BDVER4-NEXT: pdepl %r8d, %r9d, %r8d
-; SLOW-BDVER4-NEXT: vmovd %ecx, %xmm2
-; SLOW-BDVER4-NEXT: vpextrd $2, %xmm0, %ecx
-; SLOW-BDVER4-NEXT: vpinsrd $1, %eax, %xmm2, %xmm2
-; SLOW-BDVER4-NEXT: vpextrd $2, %xmm1, %eax
-; SLOW-BDVER4-NEXT: vpinsrd $2, %edx, %xmm2, %xmm2
-; SLOW-BDVER4-NEXT: vpextrd $3, %xmm0, %edx
-; SLOW-BDVER4-NEXT: pdepl %eax, %ecx, %eax
-; SLOW-BDVER4-NEXT: vpextrd $3, %xmm1, %ecx
-; SLOW-BDVER4-NEXT: vpinsrd $3, %esi, %xmm2, %xmm2
-; SLOW-BDVER4-NEXT: pdepl %ecx, %edx, %ecx
-; SLOW-BDVER4-NEXT: vmovd %r8d, %xmm3
-; SLOW-BDVER4-NEXT: vpinsrd $1, %edi, %xmm3, %xmm3
-; SLOW-BDVER4-NEXT: vpinsrd $2, %eax, %xmm3, %xmm0
-; SLOW-BDVER4-NEXT: vpinsrd $3, %ecx, %xmm0, %xmm0
-; SLOW-BDVER4-NEXT: vinserti128 $1, %xmm2, %ymm0, %ymm0
-; SLOW-BDVER4-NEXT: retq
-;
-; FAST-LABEL: pdep_v8i32_minsize:
-; FAST: # %bb.0:
-; FAST-NEXT: vextracti128 $1, %ymm0, %xmm3
-; FAST-NEXT: vextracti128 $1, %ymm1, %xmm2
-; FAST-NEXT: vpextrd $1, %xmm2, %eax
-; FAST-NEXT: vpextrd $1, %xmm3, %ecx
-; FAST-NEXT: vmovd %xmm3, %edx
-; FAST-NEXT: pdepl %eax, %ecx, %eax
-; FAST-NEXT: vmovd %xmm2, %ecx
-; FAST-NEXT: pdepl %ecx, %edx, %ecx
-; FAST-NEXT: vmovd %xmm0, %edx
-; FAST-NEXT: vmovd %ecx, %xmm4
-; FAST-NEXT: vpextrd $2, %xmm3, %ecx
-; FAST-NEXT: vpinsrd $1, %eax, %xmm4, %xmm4
-; FAST-NEXT: vpextrd $2, %xmm2, %eax
-; FAST-NEXT: pdepl %eax, %ecx, %eax
-; FAST-NEXT: vpextrd $3, %xmm3, %ecx
-; FAST-NEXT: vpinsrd $2, %eax, %xmm4, %xmm4
-; FAST-NEXT: vpextrd $3, %xmm2, %eax
-; FAST-NEXT: pdepl %eax, %ecx, %eax
-; FAST-NEXT: vpextrd $1, %xmm0, %ecx
-; FAST-NEXT: vpinsrd $3, %eax, %xmm4, %xmm2
-; FAST-NEXT: vpextrd $1, %xmm1, %eax
-; FAST-NEXT: pdepl %eax, %ecx, %eax
-; FAST-NEXT: vmovd %xmm1, %ecx
-; FAST-NEXT: pdepl %ecx, %edx, %ecx
-; FAST-NEXT: vmovd %ecx, %xmm3
-; FAST-NEXT: vpextrd $2, %xmm0, %ecx
-; FAST-NEXT: vpinsrd $1, %eax, %xmm3, %xmm3
-; FAST-NEXT: vpextrd $2, %xmm1, %eax
-; FAST-NEXT: pdepl %eax, %ecx, %eax
-; FAST-NEXT: vpextrd $3, %xmm0, %ecx
-; FAST-NEXT: vpinsrd $2, %eax, %xmm3, %xmm3
-; FAST-NEXT: vpextrd $3, %xmm1, %eax
-; FAST-NEXT: pdepl %eax, %ecx, %eax
-; FAST-NEXT: vpinsrd $3, %eax, %xmm3, %xmm0
-; FAST-NEXT: vinserti128 $1, %xmm2, %ymm0, %ymm0
-; FAST-NEXT: retq
+; CHECK-LABEL: pdep_v8i32_minsize:
+; CHECK: # %bb.0:
+; CHECK-NEXT: vextracti128 $1, %ymm0, %xmm3
+; CHECK-NEXT: vextracti128 $1, %ymm1, %xmm2
+; CHECK-NEXT: vpextrd $1, %xmm2, %eax
+; CHECK-NEXT: vpextrd $1, %xmm3, %ecx
+; CHECK-NEXT: vmovd %xmm3, %edx
+; CHECK-NEXT: pdepl %eax, %ecx, %eax
+; CHECK-NEXT: vmovd %xmm2, %ecx
+; CHECK-NEXT: pdepl %ecx, %edx, %ecx
+; CHECK-NEXT: vmovd %xmm0, %edx
+; CHECK-NEXT: vmovd %ecx, %xmm4
+; CHECK-NEXT: vpextrd $2, %xmm3, %ecx
+; CHECK-NEXT: vpinsrd $1, %eax, %xmm4, %xmm4
+; CHECK-NEXT: vpextrd $2, %xmm2, %eax
+; CHECK-NEXT: pdepl %eax, %ecx, %eax
+; CHECK-NEXT: vpextrd $3, %xmm3, %ecx
+; CHECK-NEXT: vpinsrd $2, %eax, %xmm4, %xmm4
+; CHECK-NEXT: vpextrd $3, %xmm2, %eax
+; CHECK-NEXT: pdepl %eax, %ecx, %eax
+; CHECK-NEXT: vpextrd $1, %xmm0, %ecx
+; CHECK-NEXT: vpinsrd $3, %eax, %xmm4, %xmm2
+; CHECK-NEXT: vpextrd $1, %xmm1, %eax
+; CHECK-NEXT: pdepl %eax, %ecx, %eax
+; CHECK-NEXT: vmovd %xmm1, %ecx
+; CHECK-NEXT: pdepl %ecx, %edx, %ecx
+; CHECK-NEXT: vmovd %ecx, %xmm3
+; CHECK-NEXT: vpextrd $2, %xmm0, %ecx
+; CHECK-NEXT: vpinsrd $1, %eax, %xmm3, %xmm3
+; CHECK-NEXT: vpextrd $2, %xmm1, %eax
+; CHECK-NEXT: pdepl %eax, %ecx, %eax
+; CHECK-NEXT: vpextrd $3, %xmm0, %ecx
+; CHECK-NEXT: vpinsrd $2, %eax, %xmm3, %xmm3
+; CHECK-NEXT: vpextrd $3, %xmm1, %eax
+; CHECK-NEXT: pdepl %eax, %ecx, %eax
+; CHECK-NEXT: vpinsrd $3, %eax, %xmm3, %xmm0
+; CHECK-NEXT: vinserti128 $1, %xmm2, %ymm0, %ymm0
+; CHECK-NEXT: retq
%res = call <8 x i32> @llvm.pdep.v8i32(<8 x i32> %val, <8 x i32> %mask)
ret <8 x i32> %res
}
define <2 x i64> @pdep_v2i64_minsize(<2 x i64> %val, <2 x i64> %mask) nounwind minsize {
-; SLOW-ZNVER-LABEL: pdep_v2i64_minsize:
-; SLOW-ZNVER: # %bb.0:
-; SLOW-ZNVER-NEXT: vpextrq $1, %xmm1, %rax
-; SLOW-ZNVER-NEXT: vpextrq $1, %xmm0, %rcx
-; SLOW-ZNVER-NEXT: vmovq %xmm1, %rsi
-; SLOW-ZNVER-NEXT: vmovq %xmm0, %rdx
-; SLOW-ZNVER-NEXT: pdepq %rax, %rcx, %rax
-; SLOW-ZNVER-NEXT: pdepq %rsi, %rdx, %rcx
-; SLOW-ZNVER-NEXT: vmovq %rax, %xmm0
-; SLOW-ZNVER-NEXT: vmovq %rcx, %xmm1
-; SLOW-ZNVER-NEXT: vpunpcklqdq {{.*#+}} xmm0 = xmm1[0],xmm0[0]
-; SLOW-ZNVER-NEXT: retq
-;
-; SLOW-BDVER4-LABEL: pdep_v2i64_minsize:
-; SLOW-BDVER4: # %bb.0:
-; SLOW-BDVER4-NEXT: vpextrq $1, %xmm1, %rax
-; SLOW-BDVER4-NEXT: vpextrq $1, %xmm0, %rcx
-; SLOW-BDVER4-NEXT: vmovq %xmm1, %rsi
-; SLOW-BDVER4-NEXT: vmovq %xmm0, %rdx
-; SLOW-BDVER4-NEXT: pdepq %rax, %rcx, %rax
-; SLOW-BDVER4-NEXT: pdepq %rsi, %rdx, %rcx
-; SLOW-BDVER4-NEXT: vmovq %rax, %xmm0
-; SLOW-BDVER4-NEXT: vmovq %rcx, %xmm1
-; SLOW-BDVER4-NEXT: vpunpcklqdq {{.*#+}} xmm0 = xmm1[0],xmm0[0]
-; SLOW-BDVER4-NEXT: retq
-;
-; FAST-LABEL: pdep_v2i64_minsize:
-; FAST: # %bb.0:
-; FAST-NEXT: vpextrq $1, %xmm1, %rax
-; FAST-NEXT: vpextrq $1, %xmm0, %rcx
-; FAST-NEXT: vmovq %xmm1, %rdx
-; FAST-NEXT: pdepq %rax, %rcx, %rax
-; FAST-NEXT: vmovq %xmm0, %rcx
-; FAST-NEXT: vmovq %rax, %xmm2
-; FAST-NEXT: pdepq %rdx, %rcx, %rax
-; FAST-NEXT: vmovq %rax, %xmm0
-; FAST-NEXT: vpunpcklqdq {{.*#+}} xmm0 = xmm0[0],xmm2[0]
-; FAST-NEXT: retq
+; CHECK-LABEL: pdep_v2i64_minsize:
+; CHECK: # %bb.0:
+; CHECK-NEXT: vpextrq $1, %xmm1, %rax
+; CHECK-NEXT: vpextrq $1, %xmm0, %rcx
+; CHECK-NEXT: vmovq %xmm1, %rdx
+; CHECK-NEXT: pdepq %rax, %rcx, %rax
+; CHECK-NEXT: vmovq %xmm0, %rcx
+; CHECK-NEXT: vmovq %rax, %xmm2
+; CHECK-NEXT: pdepq %rdx, %rcx, %rax
+; CHECK-NEXT: vmovq %rax, %xmm0
+; CHECK-NEXT: vpunpcklqdq {{.*#+}} xmm0 = xmm0[0],xmm2[0]
+; CHECK-NEXT: retq
%res = call <2 x i64> @llvm.pdep.v2i64(<2 x i64> %val, <2 x i64> %mask)
ret <2 x i64> %res
}
define <4 x i64> @pdep_v4i64_minsize(<4 x i64> %val, <4 x i64> %mask) nounwind minsize {
-; SLOW-ZNVER-LABEL: pdep_v4i64_minsize:
-; SLOW-ZNVER: # %bb.0:
-; SLOW-ZNVER-NEXT: vextracti128 $1, %ymm1, %xmm2
-; SLOW-ZNVER-NEXT: vextracti128 $1, %ymm0, %xmm3
-; SLOW-ZNVER-NEXT: vpextrq $1, %xmm2, %rax
-; SLOW-ZNVER-NEXT: vpextrq $1, %xmm3, %rcx
-; SLOW-ZNVER-NEXT: vmovq %xmm2, %rsi
-; SLOW-ZNVER-NEXT: vmovq %xmm3, %rdx
-; SLOW-ZNVER-NEXT: pdepq %rax, %rcx, %rax
-; SLOW-ZNVER-NEXT: pdepq %rsi, %rdx, %rcx
-; SLOW-ZNVER-NEXT: vpextrq $1, %xmm0, %rdx
-; SLOW-ZNVER-NEXT: vmovq %rax, %xmm2
-; SLOW-ZNVER-NEXT: vpextrq $1, %xmm1, %rax
-; SLOW-ZNVER-NEXT: vmovq %rcx, %xmm3
-; SLOW-ZNVER-NEXT: vmovq %xmm0, %rcx
-; SLOW-ZNVER-NEXT: pdepq %rax, %rdx, %rax
-; SLOW-ZNVER-NEXT: vmovq %xmm1, %rdx
-; SLOW-ZNVER-NEXT: vpunpcklqdq {{.*#+}} xmm2 = xmm3[0],xmm2[0]
-; SLOW-ZNVER-NEXT: vmovq %rax, %xmm3
-; SLOW-ZNVER-NEXT: pdepq %rdx, %rcx, %rax
-; SLOW-ZNVER-NEXT: vmovq %rax, %xmm0
-; SLOW-ZNVER-NEXT: vpunpcklqdq {{.*#+}} xmm0 = xmm0[0],xmm3[0]
-; SLOW-ZNVER-NEXT: vinserti128 $1, %xmm2, %ymm0, %ymm0
-; SLOW-ZNVER-NEXT: retq
-;
-; SLOW-BDVER4-LABEL: pdep_v4i64_minsize:
-; SLOW-BDVER4: # %bb.0:
-; SLOW-BDVER4-NEXT: vextracti128 $1, %ymm1, %xmm2
-; SLOW-BDVER4-NEXT: vextracti128 $1, %ymm0, %xmm3
-; SLOW-BDVER4-NEXT: vpextrq $1, %xmm2, %rax
-; SLOW-BDVER4-NEXT: vpextrq $1, %xmm3, %rcx
-; SLOW-BDVER4-NEXT: vmovq %xmm2, %rsi
-; SLOW-BDVER4-NEXT: vmovq %xmm3, %rdx
-; SLOW-BDVER4-NEXT: pdepq %rax, %rcx, %rax
-; SLOW-BDVER4-NEXT: pdepq %rsi, %rdx, %rcx
-; SLOW-BDVER4-NEXT: vpextrq $1, %xmm0, %rdx
-; SLOW-BDVER4-NEXT: vmovq %rax, %xmm2
-; SLOW-BDVER4-NEXT: vpextrq $1, %xmm1, %rax
-; SLOW-BDVER4-NEXT: vmovq %rcx, %xmm3
-; SLOW-BDVER4-NEXT: vmovq %xmm0, %rcx
-; SLOW-BDVER4-NEXT: pdepq %rax, %rdx, %rax
-; SLOW-BDVER4-NEXT: vmovq %xmm1, %rdx
-; SLOW-BDVER4-NEXT: vpunpcklqdq {{.*#+}} xmm2 = xmm3[0],xmm2[0]
-; SLOW-BDVER4-NEXT: vmovq %rax, %xmm3
-; SLOW-BDVER4-NEXT: pdepq %rdx, %rcx, %rax
-; SLOW-BDVER4-NEXT: vmovq %rax, %xmm0
-; SLOW-BDVER4-NEXT: vpunpcklqdq {{.*#+}} xmm0 = xmm0[0],xmm3[0]
-; SLOW-BDVER4-NEXT: vinserti128 $1, %xmm2, %ymm0, %ymm0
-; SLOW-BDVER4-NEXT: retq
-;
-; FAST-LABEL: pdep_v4i64_minsize:
-; FAST: # %bb.0:
-; FAST-NEXT: vextracti128 $1, %ymm1, %xmm2
-; FAST-NEXT: vextracti128 $1, %ymm0, %xmm3
-; FAST-NEXT: vpextrq $1, %xmm1, %rsi
-; FAST-NEXT: vpextrq $1, %xmm2, %rax
-; FAST-NEXT: vpextrq $1, %xmm3, %rcx
-; FAST-NEXT: vmovq %xmm2, %rdx
-; FAST-NEXT: pdepq %rax, %rcx, %rax
-; FAST-NEXT: vmovq %xmm3, %rcx
-; FAST-NEXT: vmovq %rax, %xmm4
-; FAST-NEXT: pdepq %rdx, %rcx, %rax
-; FAST-NEXT: vpextrq $1, %xmm0, %rcx
-; FAST-NEXT: vmovq %xmm1, %rdx
-; FAST-NEXT: vmovq %rax, %xmm2
-; FAST-NEXT: pdepq %rsi, %rcx, %rax
-; FAST-NEXT: vmovq %xmm0, %rcx
-; FAST-NEXT: vpunpcklqdq {{.*#+}} xmm2 = xmm2[0],xmm4[0]
-; FAST-NEXT: vmovq %rax, %xmm3
-; FAST-NEXT: pdepq %rdx, %rcx, %rax
-; FAST-NEXT: vmovq %rax, %xmm0
-; FAST-NEXT: vpunpcklqdq {{.*#+}} xmm0 = xmm0[0],xmm3[0]
-; FAST-NEXT: vinserti128 $1, %xmm2, %ymm0, %ymm0
-; FAST-NEXT: retq
+; CHECK-LABEL: pdep_v4i64_minsize:
+; CHECK: # %bb.0:
+; CHECK-NEXT: vextracti128 $1, %ymm1, %xmm2
+; CHECK-NEXT: vextracti128 $1, %ymm0, %xmm3
+; CHECK-NEXT: vpextrq $1, %xmm1, %rsi
+; CHECK-NEXT: vpextrq $1, %xmm2, %rax
+; CHECK-NEXT: vpextrq $1, %xmm3, %rcx
+; CHECK-NEXT: vmovq %xmm2, %rdx
+; CHECK-NEXT: pdepq %rax, %rcx, %rax
+; CHECK-NEXT: vmovq %xmm3, %rcx
+; CHECK-NEXT: vmovq %rax, %xmm4
+; CHECK-NEXT: pdepq %rdx, %rcx, %rax
+; CHECK-NEXT: vpextrq $1, %xmm0, %rcx
+; CHECK-NEXT: vmovq %xmm1, %rdx
+; CHECK-NEXT: vmovq %rax, %xmm2
+; CHECK-NEXT: pdepq %rsi, %rcx, %rax
+; CHECK-NEXT: vmovq %xmm0, %rcx
+; CHECK-NEXT: vpunpcklqdq {{.*#+}} xmm2 = xmm2[0],xmm4[0]
+; CHECK-NEXT: vmovq %rax, %xmm3
+; CHECK-NEXT: pdepq %rdx, %rcx, %rax
+; CHECK-NEXT: vmovq %rax, %xmm0
+; CHECK-NEXT: vpunpcklqdq {{.*#+}} xmm0 = xmm0[0],xmm3[0]
+; CHECK-NEXT: vinserti128 $1, %xmm2, %ymm0, %ymm0
+; CHECK-NEXT: retq
%res = call <4 x i64> @llvm.pdep.v4i64(<4 x i64> %val, <4 x i64> %mask)
ret <4 x i64> %res
}
-;; NOTE: These prefixes are unused and the list is autogenerated. Do not add tests below this line:
-; CHECK: {{.*}}
diff --git a/llvm/test/CodeGen/X86/znver-pext.ll b/llvm/test/CodeGen/X86/znver-pext.ll
index d714588d1aa05..5fbc3c613f025 100644
--- a/llvm/test/CodeGen/X86/znver-pext.ll
+++ b/llvm/test/CodeGen/X86/znver-pext.ll
@@ -586,213 +586,114 @@ define <4 x i64> @pext_v4i64(<4 x i64> %val, <4 x i64> %mask) nounwind {
;
define <4 x i32> @pext_v4i32_minsize(<4 x i32> %val, <4 x i32> %mask) nounwind minsize {
-; SLOW-LABEL: pext_v4i32_minsize:
-; SLOW: # %bb.0:
-; SLOW-NEXT: vpextrd $1, %xmm1, %eax
-; SLOW-NEXT: vpextrd $1, %xmm0, %ecx
-; SLOW-NEXT: vmovd %xmm0, %edx
-; SLOW-NEXT: vpextrd $2, %xmm0, %esi
-; SLOW-NEXT: pextl %eax, %ecx, %eax
-; SLOW-NEXT: vmovd %xmm1, %ecx
-; SLOW-NEXT: pextl %ecx, %edx, %ecx
-; SLOW-NEXT: vpextrd $2, %xmm1, %edx
-; SLOW-NEXT: pextl %edx, %esi, %edx
-; SLOW-NEXT: vpextrd $3, %xmm0, %esi
-; SLOW-NEXT: vmovd %ecx, %xmm2
-; SLOW-NEXT: vpextrd $3, %xmm1, %ecx
-; SLOW-NEXT: pextl %ecx, %esi, %ecx
-; SLOW-NEXT: vpinsrd $1, %eax, %xmm2, %xmm0
-; SLOW-NEXT: vpinsrd $2, %edx, %xmm0, %xmm0
-; SLOW-NEXT: vpinsrd $3, %ecx, %xmm0, %xmm0
-; SLOW-NEXT: retq
-;
-; FAST-LABEL: pext_v4i32_minsize:
-; FAST: # %bb.0:
-; FAST-NEXT: vpextrd $1, %xmm1, %eax
-; FAST-NEXT: vpextrd $1, %xmm0, %ecx
-; FAST-NEXT: vmovd %xmm0, %edx
-; FAST-NEXT: pextl %eax, %ecx, %eax
-; FAST-NEXT: vmovd %xmm1, %ecx
-; FAST-NEXT: pextl %ecx, %edx, %ecx
-; FAST-NEXT: vmovd %ecx, %xmm2
-; FAST-NEXT: vpextrd $2, %xmm0, %ecx
-; FAST-NEXT: vpinsrd $1, %eax, %xmm2, %xmm2
-; FAST-NEXT: vpextrd $2, %xmm1, %eax
-; FAST-NEXT: pextl %eax, %ecx, %eax
-; FAST-NEXT: vpextrd $3, %xmm0, %ecx
-; FAST-NEXT: vpinsrd $2, %eax, %xmm2, %xmm2
-; FAST-NEXT: vpextrd $3, %xmm1, %eax
-; FAST-NEXT: pextl %eax, %ecx, %eax
-; FAST-NEXT: vpinsrd $3, %eax, %xmm2, %xmm0
-; FAST-NEXT: retq
+; CHECK-LABEL: pext_v4i32_minsize:
+; CHECK: # %bb.0:
+; CHECK-NEXT: vpextrd $1, %xmm1, %eax
+; CHECK-NEXT: vpextrd $1, %xmm0, %ecx
+; CHECK-NEXT: vmovd %xmm0, %edx
+; CHECK-NEXT: pextl %eax, %ecx, %eax
+; CHECK-NEXT: vmovd %xmm1, %ecx
+; CHECK-NEXT: pextl %ecx, %edx, %ecx
+; CHECK-NEXT: vmovd %ecx, %xmm2
+; CHECK-NEXT: vpextrd $2, %xmm0, %ecx
+; CHECK-NEXT: vpinsrd $1, %eax, %xmm2, %xmm2
+; CHECK-NEXT: vpextrd $2, %xmm1, %eax
+; CHECK-NEXT: pextl %eax, %ecx, %eax
+; CHECK-NEXT: vpextrd $3, %xmm0, %ecx
+; CHECK-NEXT: vpinsrd $2, %eax, %xmm2, %xmm2
+; CHECK-NEXT: vpextrd $3, %xmm1, %eax
+; CHECK-NEXT: pextl %eax, %ecx, %eax
+; CHECK-NEXT: vpinsrd $3, %eax, %xmm2, %xmm0
+; CHECK-NEXT: retq
%res = call <4 x i32> @llvm.pext.v4i32(<4 x i32> %val, <4 x i32> %mask)
ret <4 x i32> %res
}
define <8 x i32> @pext_v8i32_minsize(<8 x i32> %val, <8 x i32> %mask) nounwind minsize {
-; SLOW-LABEL: pext_v8i32_minsize:
-; SLOW: # %bb.0:
-; SLOW-NEXT: vextracti128 $1, %ymm0, %xmm3
-; SLOW-NEXT: vextracti128 $1, %ymm1, %xmm2
-; SLOW-NEXT: vpextrd $1, %xmm0, %r8d
-; SLOW-NEXT: vmovd %xmm0, %r9d
-; SLOW-NEXT: vpextrd $1, %xmm2, %eax
-; SLOW-NEXT: vpextrd $1, %xmm3, %ecx
-; SLOW-NEXT: vmovd %xmm3, %edx
-; SLOW-NEXT: vpextrd $2, %xmm3, %esi
-; SLOW-NEXT: vpextrd $3, %xmm3, %edi
-; SLOW-NEXT: pextl %eax, %ecx, %eax
-; SLOW-NEXT: vmovd %xmm2, %ecx
-; SLOW-NEXT: pextl %ecx, %edx, %ecx
-; SLOW-NEXT: vpextrd $2, %xmm2, %edx
-; SLOW-NEXT: pextl %edx, %esi, %edx
-; SLOW-NEXT: vpextrd $3, %xmm2, %esi
-; SLOW-NEXT: pextl %esi, %edi, %esi
-; SLOW-NEXT: vpextrd $1, %xmm1, %edi
-; SLOW-NEXT: pextl %edi, %r8d, %edi
-; SLOW-NEXT: vmovd %xmm1, %r8d
-; SLOW-NEXT: pextl %r8d, %r9d, %r8d
-; SLOW-NEXT: vmovd %ecx, %xmm2
-; SLOW-NEXT: vpextrd $2, %xmm0, %ecx
-; SLOW-NEXT: vpinsrd $1, %eax, %xmm2, %xmm2
-; SLOW-NEXT: vpextrd $2, %xmm1, %eax
-; SLOW-NEXT: vpinsrd $2, %edx, %xmm2, %xmm2
-; SLOW-NEXT: vpextrd $3, %xmm0, %edx
-; SLOW-NEXT: pextl %eax, %ecx, %eax
-; SLOW-NEXT: vpextrd $3, %xmm1, %ecx
-; SLOW-NEXT: vpinsrd $3, %esi, %xmm2, %xmm2
-; SLOW-NEXT: pextl %ecx, %edx, %ecx
-; SLOW-NEXT: vmovd %r8d, %xmm3
-; SLOW-NEXT: vpinsrd $1, %edi, %xmm3, %xmm3
-; SLOW-NEXT: vpinsrd $2, %eax, %xmm3, %xmm0
-; SLOW-NEXT: vpinsrd $3, %ecx, %xmm0, %xmm0
-; SLOW-NEXT: vinserti128 $1, %xmm2, %ymm0, %ymm0
-; SLOW-NEXT: retq
-;
-; FAST-LABEL: pext_v8i32_minsize:
-; FAST: # %bb.0:
-; FAST-NEXT: vextracti128 $1, %ymm0, %xmm3
-; FAST-NEXT: vextracti128 $1, %ymm1, %xmm2
-; FAST-NEXT: vpextrd $1, %xmm2, %eax
-; FAST-NEXT: vpextrd $1, %xmm3, %ecx
-; FAST-NEXT: vmovd %xmm3, %edx
-; FAST-NEXT: pextl %eax, %ecx, %eax
-; FAST-NEXT: vmovd %xmm2, %ecx
-; FAST-NEXT: pextl %ecx, %edx, %ecx
-; FAST-NEXT: vmovd %xmm0, %edx
-; FAST-NEXT: vmovd %ecx, %xmm4
-; FAST-NEXT: vpextrd $2, %xmm3, %ecx
-; FAST-NEXT: vpinsrd $1, %eax, %xmm4, %xmm4
-; FAST-NEXT: vpextrd $2, %xmm2, %eax
-; FAST-NEXT: pextl %eax, %ecx, %eax
-; FAST-NEXT: vpextrd $3, %xmm3, %ecx
-; FAST-NEXT: vpinsrd $2, %eax, %xmm4, %xmm4
-; FAST-NEXT: vpextrd $3, %xmm2, %eax
-; FAST-NEXT: pextl %eax, %ecx, %eax
-; FAST-NEXT: vpextrd $1, %xmm0, %ecx
-; FAST-NEXT: vpinsrd $3, %eax, %xmm4, %xmm2
-; FAST-NEXT: vpextrd $1, %xmm1, %eax
-; FAST-NEXT: pextl %eax, %ecx, %eax
-; FAST-NEXT: vmovd %xmm1, %ecx
-; FAST-NEXT: pextl %ecx, %edx, %ecx
-; FAST-NEXT: vmovd %ecx, %xmm3
-; FAST-NEXT: vpextrd $2, %xmm0, %ecx
-; FAST-NEXT: vpinsrd $1, %eax, %xmm3, %xmm3
-; FAST-NEXT: vpextrd $2, %xmm1, %eax
-; FAST-NEXT: pextl %eax, %ecx, %eax
-; FAST-NEXT: vpextrd $3, %xmm0, %ecx
-; FAST-NEXT: vpinsrd $2, %eax, %xmm3, %xmm3
-; FAST-NEXT: vpextrd $3, %xmm1, %eax
-; FAST-NEXT: pextl %eax, %ecx, %eax
-; FAST-NEXT: vpinsrd $3, %eax, %xmm3, %xmm0
-; FAST-NEXT: vinserti128 $1, %xmm2, %ymm0, %ymm0
-; FAST-NEXT: retq
+; CHECK-LABEL: pext_v8i32_minsize:
+; CHECK: # %bb.0:
+; CHECK-NEXT: vextracti128 $1, %ymm0, %xmm3
+; CHECK-NEXT: vextracti128 $1, %ymm1, %xmm2
+; CHECK-NEXT: vpextrd $1, %xmm2, %eax
+; CHECK-NEXT: vpextrd $1, %xmm3, %ecx
+; CHECK-NEXT: vmovd %xmm3, %edx
+; CHECK-NEXT: pextl %eax, %ecx, %eax
+; CHECK-NEXT: vmovd %xmm2, %ecx
+; CHECK-NEXT: pextl %ecx, %edx, %ecx
+; CHECK-NEXT: vmovd %xmm0, %edx
+; CHECK-NEXT: vmovd %ecx, %xmm4
+; CHECK-NEXT: vpextrd $2, %xmm3, %ecx
+; CHECK-NEXT: vpinsrd $1, %eax, %xmm4, %xmm4
+; CHECK-NEXT: vpextrd $2, %xmm2, %eax
+; CHECK-NEXT: pextl %eax, %ecx, %eax
+; CHECK-NEXT: vpextrd $3, %xmm3, %ecx
+; CHECK-NEXT: vpinsrd $2, %eax, %xmm4, %xmm4
+; CHECK-NEXT: vpextrd $3, %xmm2, %eax
+; CHECK-NEXT: pextl %eax, %ecx, %eax
+; CHECK-NEXT: vpextrd $1, %xmm0, %ecx
+; CHECK-NEXT: vpinsrd $3, %eax, %xmm4, %xmm2
+; CHECK-NEXT: vpextrd $1, %xmm1, %eax
+; CHECK-NEXT: pextl %eax, %ecx, %eax
+; CHECK-NEXT: vmovd %xmm1, %ecx
+; CHECK-NEXT: pextl %ecx, %edx, %ecx
+; CHECK-NEXT: vmovd %ecx, %xmm3
+; CHECK-NEXT: vpextrd $2, %xmm0, %ecx
+; CHECK-NEXT: vpinsrd $1, %eax, %xmm3, %xmm3
+; CHECK-NEXT: vpextrd $2, %xmm1, %eax
+; CHECK-NEXT: pextl %eax, %ecx, %eax
+; CHECK-NEXT: vpextrd $3, %xmm0, %ecx
+; CHECK-NEXT: vpinsrd $2, %eax, %xmm3, %xmm3
+; CHECK-NEXT: vpextrd $3, %xmm1, %eax
+; CHECK-NEXT: pextl %eax, %ecx, %eax
+; CHECK-NEXT: vpinsrd $3, %eax, %xmm3, %xmm0
+; CHECK-NEXT: vinserti128 $1, %xmm2, %ymm0, %ymm0
+; CHECK-NEXT: retq
%res = call <8 x i32> @llvm.pext.v8i32(<8 x i32> %val, <8 x i32> %mask)
ret <8 x i32> %res
}
define <2 x i64> @pext_v2i64_minsize(<2 x i64> %val, <2 x i64> %mask) nounwind minsize {
-; SLOW-LABEL: pext_v2i64_minsize:
-; SLOW: # %bb.0:
-; SLOW-NEXT: vpextrq $1, %xmm1, %rax
-; SLOW-NEXT: vpextrq $1, %xmm0, %rcx
-; SLOW-NEXT: vmovq %xmm1, %rsi
-; SLOW-NEXT: vmovq %xmm0, %rdx
-; SLOW-NEXT: pextq %rax, %rcx, %rax
-; SLOW-NEXT: pextq %rsi, %rdx, %rcx
-; SLOW-NEXT: vmovq %rax, %xmm0
-; SLOW-NEXT: vmovq %rcx, %xmm1
-; SLOW-NEXT: vpunpcklqdq {{.*#+}} xmm0 = xmm1[0],xmm0[0]
-; SLOW-NEXT: retq
-;
-; FAST-LABEL: pext_v2i64_minsize:
-; FAST: # %bb.0:
-; FAST-NEXT: vpextrq $1, %xmm1, %rax
-; FAST-NEXT: vpextrq $1, %xmm0, %rcx
-; FAST-NEXT: vmovq %xmm1, %rdx
-; FAST-NEXT: pextq %rax, %rcx, %rax
-; FAST-NEXT: vmovq %xmm0, %rcx
-; FAST-NEXT: vmovq %rax, %xmm2
-; FAST-NEXT: pextq %rdx, %rcx, %rax
-; FAST-NEXT: vmovq %rax, %xmm0
-; FAST-NEXT: vpunpcklqdq {{.*#+}} xmm0 = xmm0[0],xmm2[0]
-; FAST-NEXT: retq
+; CHECK-LABEL: pext_v2i64_minsize:
+; CHECK: # %bb.0:
+; CHECK-NEXT: vpextrq $1, %xmm1, %rax
+; CHECK-NEXT: vpextrq $1, %xmm0, %rcx
+; CHECK-NEXT: vmovq %xmm1, %rdx
+; CHECK-NEXT: pextq %rax, %rcx, %rax
+; CHECK-NEXT: vmovq %xmm0, %rcx
+; CHECK-NEXT: vmovq %rax, %xmm2
+; CHECK-NEXT: pextq %rdx, %rcx, %rax
+; CHECK-NEXT: vmovq %rax, %xmm0
+; CHECK-NEXT: vpunpcklqdq {{.*#+}} xmm0 = xmm0[0],xmm2[0]
+; CHECK-NEXT: retq
%res = call <2 x i64> @llvm.pext.v2i64(<2 x i64> %val, <2 x i64> %mask)
ret <2 x i64> %res
}
define <4 x i64> @pext_v4i64_minsize(<4 x i64> %val, <4 x i64> %mask) nounwind minsize {
-; SLOW-LABEL: pext_v4i64_minsize:
-; SLOW: # %bb.0:
-; SLOW-NEXT: vextracti128 $1, %ymm1, %xmm2
-; SLOW-NEXT: vextracti128 $1, %ymm0, %xmm3
-; SLOW-NEXT: vpextrq $1, %xmm2, %rax
-; SLOW-NEXT: vpextrq $1, %xmm3, %rcx
-; SLOW-NEXT: vmovq %xmm2, %rsi
-; SLOW-NEXT: vmovq %xmm3, %rdx
-; SLOW-NEXT: pextq %rax, %rcx, %rax
-; SLOW-NEXT: pextq %rsi, %rdx, %rcx
-; SLOW-NEXT: vpextrq $1, %xmm0, %rdx
-; SLOW-NEXT: vmovq %rax, %xmm2
-; SLOW-NEXT: vpextrq $1, %xmm1, %rax
-; SLOW-NEXT: vmovq %rcx, %xmm3
-; SLOW-NEXT: vmovq %xmm0, %rcx
-; SLOW-NEXT: pextq %rax, %rdx, %rax
-; SLOW-NEXT: vmovq %xmm1, %rdx
-; SLOW-NEXT: vpunpcklqdq {{.*#+}} xmm2 = xmm3[0],xmm2[0]
-; SLOW-NEXT: vmovq %rax, %xmm3
-; SLOW-NEXT: pextq %rdx, %rcx, %rax
-; SLOW-NEXT: vmovq %rax, %xmm0
-; SLOW-NEXT: vpunpcklqdq {{.*#+}} xmm0 = xmm0[0],xmm3[0]
-; SLOW-NEXT: vinserti128 $1, %xmm2, %ymm0, %ymm0
-; SLOW-NEXT: retq
-;
-; FAST-LABEL: pext_v4i64_minsize:
-; FAST: # %bb.0:
-; FAST-NEXT: vextracti128 $1, %ymm1, %xmm2
-; FAST-NEXT: vextracti128 $1, %ymm0, %xmm3
-; FAST-NEXT: vpextrq $1, %xmm1, %rsi
-; FAST-NEXT: vpextrq $1, %xmm2, %rax
-; FAST-NEXT: vpextrq $1, %xmm3, %rcx
-; FAST-NEXT: vmovq %xmm2, %rdx
-; FAST-NEXT: pextq %rax, %rcx, %rax
-; FAST-NEXT: vmovq %xmm3, %rcx
-; FAST-NEXT: vmovq %rax, %xmm4
-; FAST-NEXT: pextq %rdx, %rcx, %rax
-; FAST-NEXT: vpextrq $1, %xmm0, %rcx
-; FAST-NEXT: vmovq %xmm1, %rdx
-; FAST-NEXT: vmovq %rax, %xmm2
-; FAST-NEXT: pextq %rsi, %rcx, %rax
-; FAST-NEXT: vmovq %xmm0, %rcx
-; FAST-NEXT: vpunpcklqdq {{.*#+}} xmm2 = xmm2[0],xmm4[0]
-; FAST-NEXT: vmovq %rax, %xmm3
-; FAST-NEXT: pextq %rdx, %rcx, %rax
-; FAST-NEXT: vmovq %rax, %xmm0
-; FAST-NEXT: vpunpcklqdq {{.*#+}} xmm0 = xmm0[0],xmm3[0]
-; FAST-NEXT: vinserti128 $1, %xmm2, %ymm0, %ymm0
-; FAST-NEXT: retq
+; CHECK-LABEL: pext_v4i64_minsize:
+; CHECK: # %bb.0:
+; CHECK-NEXT: vextracti128 $1, %ymm1, %xmm2
+; CHECK-NEXT: vextracti128 $1, %ymm0, %xmm3
+; CHECK-NEXT: vpextrq $1, %xmm1, %rsi
+; CHECK-NEXT: vpextrq $1, %xmm2, %rax
+; CHECK-NEXT: vpextrq $1, %xmm3, %rcx
+; CHECK-NEXT: vmovq %xmm2, %rdx
+; CHECK-NEXT: pextq %rax, %rcx, %rax
+; CHECK-NEXT: vmovq %xmm3, %rcx
+; CHECK-NEXT: vmovq %rax, %xmm4
+; CHECK-NEXT: pextq %rdx, %rcx, %rax
+; CHECK-NEXT: vpextrq $1, %xmm0, %rcx
+; CHECK-NEXT: vmovq %xmm1, %rdx
+; CHECK-NEXT: vmovq %rax, %xmm2
+; CHECK-NEXT: pextq %rsi, %rcx, %rax
+; CHECK-NEXT: vmovq %xmm0, %rcx
+; CHECK-NEXT: vpunpcklqdq {{.*#+}} xmm2 = xmm2[0],xmm4[0]
+; CHECK-NEXT: vmovq %rax, %xmm3
+; CHECK-NEXT: pextq %rdx, %rcx, %rax
+; CHECK-NEXT: vmovq %rax, %xmm0
+; CHECK-NEXT: vpunpcklqdq {{.*#+}} xmm0 = xmm0[0],xmm3[0]
+; CHECK-NEXT: vinserti128 $1, %xmm2, %ymm0, %ymm0
+; CHECK-NEXT: retq
%res = call <4 x i64> @llvm.pext.v4i64(<4 x i64> %val, <4 x i64> %mask)
ret <4 x i64> %res
}
-;; NOTE: These prefixes are unused and the list is autogenerated. Do not add tests below this line:
-; CHECK: {{.*}}
>From ec30f60f435680bae8c4140cbaf9fe5836a3fda3 Mon Sep 17 00:00:00 2001
From: "Ziegler, Tim" <tim.ziegler at intel.com>
Date: Fri, 28 Aug 2026 16:27:58 +0200
Subject: [PATCH 4/6] Unrolled, guarded, added missing legalizations
---
llvm/lib/Target/X86/X86ISelLowering.cpp | 60 +-
llvm/test/CodeGen/X86/avx10_2bf16-rounding.ll | 984 ++----------------
2 files changed, 123 insertions(+), 921 deletions(-)
diff --git a/llvm/lib/Target/X86/X86ISelLowering.cpp b/llvm/lib/Target/X86/X86ISelLowering.cpp
index a79928801d8bb..be9a474a17256 100644
--- a/llvm/lib/Target/X86/X86ISelLowering.cpp
+++ b/llvm/lib/Target/X86/X86ISelLowering.cpp
@@ -2593,7 +2593,9 @@ X86TargetLowering::X86TargetLowering(const X86TargetMachine &TM,
setOperationAction(ISD::INSERT_SUBVECTOR, VT, Legal);
setOperationAction(ISD::CONCAT_VECTORS, VT, Custom);
}
- for (unsigned Opc : {ISD::FADD, ISD::FSUB, ISD::FMUL, ISD::FDIV}) {
+ for (unsigned Opc : {ISD::FADD, ISD::FSUB, ISD::FMUL, ISD::FDIV,
+ ISD::FFLOOR, ISD::FCEIL, ISD::FTRUNC, ISD::FRINT,
+ ISD::FNEARBYINT, ISD::FROUNDEVEN, ISD::FROUND}) {
setOperationPromotedToType(Opc, MVT::v8bf16, MVT::v8f32);
setOperationPromotedToType(Opc, MVT::v16bf16, MVT::v16f32);
}
@@ -2607,7 +2609,9 @@ X86TargetLowering::X86TargetLowering(const X86TargetMachine &TM,
Subtarget.useAVX512Regs()) {
addRegisterClass(MVT::v32bf16, &X86::VR512RegClass);
setF16Action(MVT::v32bf16, Expand);
- for (unsigned Opc : {ISD::FADD, ISD::FSUB, ISD::FMUL, ISD::FDIV})
+ for (unsigned Opc : {ISD::FADD, ISD::FSUB, ISD::FMUL, ISD::FDIV,
+ ISD::FFLOOR, ISD::FCEIL, ISD::FTRUNC, ISD::FRINT,
+ ISD::FNEARBYINT, ISD::FROUNDEVEN, ISD::FROUND})
setOperationPromotedToType(Opc, MVT::v32bf16, MVT::v32f32);
setOperationAction(ISD::SETCC, MVT::v32bf16, Custom);
setOperationAction(ISD::BUILD_VECTOR, MVT::v32bf16, Custom);
@@ -2627,22 +2631,44 @@ X86TargetLowering::X86TargetLowering(const X86TargetMachine &TM,
setOperationAction(ISD::FSQRT, MVT::bf16, Custom);
setOperationAction(ISD::FMA, MVT::bf16, Custom);
- for (unsigned Opc : {ISD::FFLOOR, ISD::FCEIL, ISD::FTRUNC, ISD::FRINT,
- ISD::FNEARBYINT, ISD::FROUNDEVEN})
- setOperationAction(Opc, MVT::bf16, Custom);
+ setOperationAction(ISD::FFLOOR, MVT::bf16, Custom);
+ setOperationAction(ISD::FCEIL, MVT::bf16, Custom);
+ setOperationAction(ISD::FTRUNC, MVT::bf16, Custom);
+ setOperationAction(ISD::FRINT, MVT::bf16, Custom);
+ setOperationAction(ISD::FNEARBYINT, MVT::bf16, Custom);
+ setOperationAction(ISD::FROUNDEVEN, MVT::bf16, Custom);
+
+ setOperationAction(ISD::FFLOOR, MVT::v8bf16, Legal);
+ setOperationAction(ISD::FCEIL, MVT::v8bf16, Legal);
+ setOperationAction(ISD::FTRUNC, MVT::v8bf16, Legal);
+ setOperationAction(ISD::FRINT, MVT::v8bf16, Legal);
+ setOperationAction(ISD::FNEARBYINT, MVT::v8bf16, Legal);
+ setOperationAction(ISD::FROUNDEVEN, MVT::v8bf16, Legal);
+ setOperationAction(ISD::FFLOOR, MVT::v16bf16, Legal);
+ setOperationAction(ISD::FCEIL, MVT::v16bf16, Legal);
+ setOperationAction(ISD::FTRUNC, MVT::v16bf16, Legal);
+ setOperationAction(ISD::FRINT, MVT::v16bf16, Legal);
+ setOperationAction(ISD::FNEARBYINT, MVT::v16bf16, Legal);
+ setOperationAction(ISD::FROUNDEVEN, MVT::v16bf16, Legal);
+
+ if (Subtarget.useAVX512Regs()) {
+ setOperationAction(ISD::FADD, MVT::v32bf16, Legal);
+ setOperationAction(ISD::FSUB, MVT::v32bf16, Legal);
+ setOperationAction(ISD::FMUL, MVT::v32bf16, Legal);
+ setOperationAction(ISD::FDIV, MVT::v32bf16, Legal);
+ setOperationAction(ISD::FSQRT, MVT::v32bf16, Legal);
+ setOperationAction(ISD::FMA, MVT::v32bf16, Legal);
+ setOperationAction(ISD::SETCC, MVT::v32bf16, Custom);
+ SetFPMinMaxAction(MVT::v32bf16);
+
+ setOperationAction(ISD::FFLOOR, MVT::v32bf16, Legal);
+ setOperationAction(ISD::FCEIL, MVT::v32bf16, Legal);
+ setOperationAction(ISD::FTRUNC, MVT::v32bf16, Legal);
+ setOperationAction(ISD::FRINT, MVT::v32bf16, Legal);
+ setOperationAction(ISD::FNEARBYINT, MVT::v32bf16, Legal);
+ setOperationAction(ISD::FROUNDEVEN, MVT::v32bf16, Legal);
+ }
- setOperationAction(ISD::FADD, MVT::v32bf16, Legal);
- setOperationAction(ISD::FSUB, MVT::v32bf16, Legal);
- setOperationAction(ISD::FMUL, MVT::v32bf16, Legal);
- setOperationAction(ISD::FDIV, MVT::v32bf16, Legal);
- setOperationAction(ISD::FSQRT, MVT::v32bf16, Legal);
- setOperationAction(ISD::FMA, MVT::v32bf16, Legal);
- setOperationAction(ISD::SETCC, MVT::v32bf16, Custom);
- SetFPMinMaxAction(MVT::v32bf16);
- for (auto VT : {MVT::v8bf16, MVT::v16bf16, MVT::v32bf16})
- for (unsigned Opc : {ISD::FFLOOR, ISD::FCEIL, ISD::FTRUNC, ISD::FRINT,
- ISD::FNEARBYINT, ISD::FROUNDEVEN})
- setOperationAction(Opc, VT, Legal);
for (auto VT : {MVT::v8bf16, MVT::v16bf16}) {
setOperationAction(ISD::FADD, VT, Legal);
setOperationAction(ISD::FSUB, VT, Legal);
diff --git a/llvm/test/CodeGen/X86/avx10_2bf16-rounding.ll b/llvm/test/CodeGen/X86/avx10_2bf16-rounding.ll
index 11506e1bcae8c..0aa238c4c8ff9 100644
--- a/llvm/test/CodeGen/X86/avx10_2bf16-rounding.ll
+++ b/llvm/test/CodeGen/X86/avx10_2bf16-rounding.ll
@@ -15,54 +15,11 @@ define <8 x bfloat> @floor_v8bf16(<8 x bfloat> %a) {
;
; AVX512BF16-LABEL: floor_v8bf16:
; AVX512BF16: # %bb.0:
-; AVX512BF16-NEXT: vpextrw $7, %xmm0, %eax
-; AVX512BF16-NEXT: vmovd %eax, %xmm1
-; AVX512BF16-NEXT: vpslld $16, %xmm1, %xmm1
-; AVX512BF16-NEXT: vroundss $9, %xmm1, %xmm1, %xmm1
-; AVX512BF16-NEXT: vcvtneps2bf16 %xmm1, %xmm1
-; AVX512BF16-NEXT: vmovd %xmm1, %eax
-; AVX512BF16-NEXT: vpextrw $6, %xmm0, %ecx
-; AVX512BF16-NEXT: vmovd %ecx, %xmm1
-; AVX512BF16-NEXT: vpslld $16, %xmm1, %xmm1
-; AVX512BF16-NEXT: vroundss $9, %xmm1, %xmm1, %xmm1
-; AVX512BF16-NEXT: vcvtneps2bf16 %xmm1, %xmm1
-; AVX512BF16-NEXT: vmovd %xmm1, %ecx
-; AVX512BF16-NEXT: vpextrw $5, %xmm0, %edx
-; AVX512BF16-NEXT: vmovd %edx, %xmm1
-; AVX512BF16-NEXT: vpslld $16, %xmm1, %xmm1
-; AVX512BF16-NEXT: vroundss $9, %xmm1, %xmm1, %xmm1
-; AVX512BF16-NEXT: vcvtneps2bf16 %xmm1, %xmm1
-; AVX512BF16-NEXT: vmovd %xmm1, %edx
-; AVX512BF16-NEXT: vpxor %xmm1, %xmm1, %xmm1
-; AVX512BF16-NEXT: vpunpckhwd {{.*#+}} xmm2 = xmm1[4],xmm0[4],xmm1[5],xmm0[5],xmm1[6],xmm0[6],xmm1[7],xmm0[7]
-; AVX512BF16-NEXT: vroundss $9, %xmm2, %xmm2, %xmm2
-; AVX512BF16-NEXT: vcvtneps2bf16 %xmm2, %xmm2
-; AVX512BF16-NEXT: vmovd %xmm2, %esi
-; AVX512BF16-NEXT: vpextrw $3, %xmm0, %edi
-; AVX512BF16-NEXT: vmovd %edi, %xmm2
-; AVX512BF16-NEXT: vpslld $16, %xmm2, %xmm2
-; AVX512BF16-NEXT: vroundss $9, %xmm2, %xmm2, %xmm2
-; AVX512BF16-NEXT: vcvtneps2bf16 %xmm2, %xmm2
-; AVX512BF16-NEXT: vmovd %xmm2, %edi
-; AVX512BF16-NEXT: vpextrw $2, %xmm0, %r8d
-; AVX512BF16-NEXT: vmovd %r8d, %xmm2
-; AVX512BF16-NEXT: vpslld $16, %xmm2, %xmm2
-; AVX512BF16-NEXT: vroundss $9, %xmm2, %xmm2, %xmm2
-; AVX512BF16-NEXT: vcvtneps2bf16 %xmm2, %xmm2
-; AVX512BF16-NEXT: vmovd %xmm2, %r8d
-; AVX512BF16-NEXT: vpblendw {{.*#+}} xmm1 = xmm1[0],xmm0[1],xmm1[2],xmm0[3],xmm1[4],xmm0[5],xmm1[6],xmm0[7]
-; AVX512BF16-NEXT: vroundss $9, %xmm1, %xmm1, %xmm1
-; AVX512BF16-NEXT: vcvtneps2bf16 %xmm1, %xmm1
-; AVX512BF16-NEXT: vpslld $16, %xmm0, %xmm0
-; AVX512BF16-NEXT: vroundss $9, %xmm0, %xmm0, %xmm0
-; AVX512BF16-NEXT: vcvtneps2bf16 %xmm0, %xmm0
-; AVX512BF16-NEXT: vpunpcklwd {{.*#+}} xmm0 = xmm0[0],xmm1[0],xmm0[1],xmm1[1],xmm0[2],xmm1[2],xmm0[3],xmm1[3]
-; AVX512BF16-NEXT: vpinsrw $2, %r8d, %xmm0, %xmm0
-; AVX512BF16-NEXT: vpinsrw $3, %edi, %xmm0, %xmm0
-; AVX512BF16-NEXT: vpinsrw $4, %esi, %xmm0, %xmm0
-; AVX512BF16-NEXT: vpinsrw $5, %edx, %xmm0, %xmm0
-; AVX512BF16-NEXT: vpinsrw $6, %ecx, %xmm0, %xmm0
-; AVX512BF16-NEXT: vpinsrw $7, %eax, %xmm0, %xmm0
+; AVX512BF16-NEXT: vpmovzxwd {{.*#+}} ymm0 = xmm0[0],zero,xmm0[1],zero,xmm0[2],zero,xmm0[3],zero,xmm0[4],zero,xmm0[5],zero,xmm0[6],zero,xmm0[7],zero
+; AVX512BF16-NEXT: vpslld $16, %ymm0, %ymm0
+; AVX512BF16-NEXT: vroundps $9, %ymm0, %ymm0
+; AVX512BF16-NEXT: vcvtneps2bf16 %ymm0, %xmm0
+; AVX512BF16-NEXT: vzeroupper
; AVX512BF16-NEXT: retq
%r = call <8 x bfloat> @llvm.floor.v8bf16(<8 x bfloat> %a)
ret <8 x bfloat> %r
@@ -92,54 +49,11 @@ define <8 x bfloat> @ceil_v8bf16(<8 x bfloat> %a) {
;
; AVX512BF16-LABEL: ceil_v8bf16:
; AVX512BF16: # %bb.0:
-; AVX512BF16-NEXT: vpextrw $7, %xmm0, %eax
-; AVX512BF16-NEXT: vmovd %eax, %xmm1
-; AVX512BF16-NEXT: vpslld $16, %xmm1, %xmm1
-; AVX512BF16-NEXT: vroundss $10, %xmm1, %xmm1, %xmm1
-; AVX512BF16-NEXT: vcvtneps2bf16 %xmm1, %xmm1
-; AVX512BF16-NEXT: vmovd %xmm1, %eax
-; AVX512BF16-NEXT: vpextrw $6, %xmm0, %ecx
-; AVX512BF16-NEXT: vmovd %ecx, %xmm1
-; AVX512BF16-NEXT: vpslld $16, %xmm1, %xmm1
-; AVX512BF16-NEXT: vroundss $10, %xmm1, %xmm1, %xmm1
-; AVX512BF16-NEXT: vcvtneps2bf16 %xmm1, %xmm1
-; AVX512BF16-NEXT: vmovd %xmm1, %ecx
-; AVX512BF16-NEXT: vpextrw $5, %xmm0, %edx
-; AVX512BF16-NEXT: vmovd %edx, %xmm1
-; AVX512BF16-NEXT: vpslld $16, %xmm1, %xmm1
-; AVX512BF16-NEXT: vroundss $10, %xmm1, %xmm1, %xmm1
-; AVX512BF16-NEXT: vcvtneps2bf16 %xmm1, %xmm1
-; AVX512BF16-NEXT: vmovd %xmm1, %edx
-; AVX512BF16-NEXT: vpxor %xmm1, %xmm1, %xmm1
-; AVX512BF16-NEXT: vpunpckhwd {{.*#+}} xmm2 = xmm1[4],xmm0[4],xmm1[5],xmm0[5],xmm1[6],xmm0[6],xmm1[7],xmm0[7]
-; AVX512BF16-NEXT: vroundss $10, %xmm2, %xmm2, %xmm2
-; AVX512BF16-NEXT: vcvtneps2bf16 %xmm2, %xmm2
-; AVX512BF16-NEXT: vmovd %xmm2, %esi
-; AVX512BF16-NEXT: vpextrw $3, %xmm0, %edi
-; AVX512BF16-NEXT: vmovd %edi, %xmm2
-; AVX512BF16-NEXT: vpslld $16, %xmm2, %xmm2
-; AVX512BF16-NEXT: vroundss $10, %xmm2, %xmm2, %xmm2
-; AVX512BF16-NEXT: vcvtneps2bf16 %xmm2, %xmm2
-; AVX512BF16-NEXT: vmovd %xmm2, %edi
-; AVX512BF16-NEXT: vpextrw $2, %xmm0, %r8d
-; AVX512BF16-NEXT: vmovd %r8d, %xmm2
-; AVX512BF16-NEXT: vpslld $16, %xmm2, %xmm2
-; AVX512BF16-NEXT: vroundss $10, %xmm2, %xmm2, %xmm2
-; AVX512BF16-NEXT: vcvtneps2bf16 %xmm2, %xmm2
-; AVX512BF16-NEXT: vmovd %xmm2, %r8d
-; AVX512BF16-NEXT: vpblendw {{.*#+}} xmm1 = xmm1[0],xmm0[1],xmm1[2],xmm0[3],xmm1[4],xmm0[5],xmm1[6],xmm0[7]
-; AVX512BF16-NEXT: vroundss $10, %xmm1, %xmm1, %xmm1
-; AVX512BF16-NEXT: vcvtneps2bf16 %xmm1, %xmm1
-; AVX512BF16-NEXT: vpslld $16, %xmm0, %xmm0
-; AVX512BF16-NEXT: vroundss $10, %xmm0, %xmm0, %xmm0
-; AVX512BF16-NEXT: vcvtneps2bf16 %xmm0, %xmm0
-; AVX512BF16-NEXT: vpunpcklwd {{.*#+}} xmm0 = xmm0[0],xmm1[0],xmm0[1],xmm1[1],xmm0[2],xmm1[2],xmm0[3],xmm1[3]
-; AVX512BF16-NEXT: vpinsrw $2, %r8d, %xmm0, %xmm0
-; AVX512BF16-NEXT: vpinsrw $3, %edi, %xmm0, %xmm0
-; AVX512BF16-NEXT: vpinsrw $4, %esi, %xmm0, %xmm0
-; AVX512BF16-NEXT: vpinsrw $5, %edx, %xmm0, %xmm0
-; AVX512BF16-NEXT: vpinsrw $6, %ecx, %xmm0, %xmm0
-; AVX512BF16-NEXT: vpinsrw $7, %eax, %xmm0, %xmm0
+; AVX512BF16-NEXT: vpmovzxwd {{.*#+}} ymm0 = xmm0[0],zero,xmm0[1],zero,xmm0[2],zero,xmm0[3],zero,xmm0[4],zero,xmm0[5],zero,xmm0[6],zero,xmm0[7],zero
+; AVX512BF16-NEXT: vpslld $16, %ymm0, %ymm0
+; AVX512BF16-NEXT: vroundps $10, %ymm0, %ymm0
+; AVX512BF16-NEXT: vcvtneps2bf16 %ymm0, %xmm0
+; AVX512BF16-NEXT: vzeroupper
; AVX512BF16-NEXT: retq
%r = call <8 x bfloat> @llvm.ceil.v8bf16(<8 x bfloat> %a)
ret <8 x bfloat> %r
@@ -169,54 +83,11 @@ define <8 x bfloat> @trunc_v8bf16(<8 x bfloat> %a) {
;
; AVX512BF16-LABEL: trunc_v8bf16:
; AVX512BF16: # %bb.0:
-; AVX512BF16-NEXT: vpextrw $7, %xmm0, %eax
-; AVX512BF16-NEXT: vmovd %eax, %xmm1
-; AVX512BF16-NEXT: vpslld $16, %xmm1, %xmm1
-; AVX512BF16-NEXT: vroundss $11, %xmm1, %xmm1, %xmm1
-; AVX512BF16-NEXT: vcvtneps2bf16 %xmm1, %xmm1
-; AVX512BF16-NEXT: vmovd %xmm1, %eax
-; AVX512BF16-NEXT: vpextrw $6, %xmm0, %ecx
-; AVX512BF16-NEXT: vmovd %ecx, %xmm1
-; AVX512BF16-NEXT: vpslld $16, %xmm1, %xmm1
-; AVX512BF16-NEXT: vroundss $11, %xmm1, %xmm1, %xmm1
-; AVX512BF16-NEXT: vcvtneps2bf16 %xmm1, %xmm1
-; AVX512BF16-NEXT: vmovd %xmm1, %ecx
-; AVX512BF16-NEXT: vpextrw $5, %xmm0, %edx
-; AVX512BF16-NEXT: vmovd %edx, %xmm1
-; AVX512BF16-NEXT: vpslld $16, %xmm1, %xmm1
-; AVX512BF16-NEXT: vroundss $11, %xmm1, %xmm1, %xmm1
-; AVX512BF16-NEXT: vcvtneps2bf16 %xmm1, %xmm1
-; AVX512BF16-NEXT: vmovd %xmm1, %edx
-; AVX512BF16-NEXT: vpxor %xmm1, %xmm1, %xmm1
-; AVX512BF16-NEXT: vpunpckhwd {{.*#+}} xmm2 = xmm1[4],xmm0[4],xmm1[5],xmm0[5],xmm1[6],xmm0[6],xmm1[7],xmm0[7]
-; AVX512BF16-NEXT: vroundss $11, %xmm2, %xmm2, %xmm2
-; AVX512BF16-NEXT: vcvtneps2bf16 %xmm2, %xmm2
-; AVX512BF16-NEXT: vmovd %xmm2, %esi
-; AVX512BF16-NEXT: vpextrw $3, %xmm0, %edi
-; AVX512BF16-NEXT: vmovd %edi, %xmm2
-; AVX512BF16-NEXT: vpslld $16, %xmm2, %xmm2
-; AVX512BF16-NEXT: vroundss $11, %xmm2, %xmm2, %xmm2
-; AVX512BF16-NEXT: vcvtneps2bf16 %xmm2, %xmm2
-; AVX512BF16-NEXT: vmovd %xmm2, %edi
-; AVX512BF16-NEXT: vpextrw $2, %xmm0, %r8d
-; AVX512BF16-NEXT: vmovd %r8d, %xmm2
-; AVX512BF16-NEXT: vpslld $16, %xmm2, %xmm2
-; AVX512BF16-NEXT: vroundss $11, %xmm2, %xmm2, %xmm2
-; AVX512BF16-NEXT: vcvtneps2bf16 %xmm2, %xmm2
-; AVX512BF16-NEXT: vmovd %xmm2, %r8d
-; AVX512BF16-NEXT: vpblendw {{.*#+}} xmm1 = xmm1[0],xmm0[1],xmm1[2],xmm0[3],xmm1[4],xmm0[5],xmm1[6],xmm0[7]
-; AVX512BF16-NEXT: vroundss $11, %xmm1, %xmm1, %xmm1
-; AVX512BF16-NEXT: vcvtneps2bf16 %xmm1, %xmm1
-; AVX512BF16-NEXT: vpslld $16, %xmm0, %xmm0
-; AVX512BF16-NEXT: vroundss $11, %xmm0, %xmm0, %xmm0
-; AVX512BF16-NEXT: vcvtneps2bf16 %xmm0, %xmm0
-; AVX512BF16-NEXT: vpunpcklwd {{.*#+}} xmm0 = xmm0[0],xmm1[0],xmm0[1],xmm1[1],xmm0[2],xmm1[2],xmm0[3],xmm1[3]
-; AVX512BF16-NEXT: vpinsrw $2, %r8d, %xmm0, %xmm0
-; AVX512BF16-NEXT: vpinsrw $3, %edi, %xmm0, %xmm0
-; AVX512BF16-NEXT: vpinsrw $4, %esi, %xmm0, %xmm0
-; AVX512BF16-NEXT: vpinsrw $5, %edx, %xmm0, %xmm0
-; AVX512BF16-NEXT: vpinsrw $6, %ecx, %xmm0, %xmm0
-; AVX512BF16-NEXT: vpinsrw $7, %eax, %xmm0, %xmm0
+; AVX512BF16-NEXT: vpmovzxwd {{.*#+}} ymm0 = xmm0[0],zero,xmm0[1],zero,xmm0[2],zero,xmm0[3],zero,xmm0[4],zero,xmm0[5],zero,xmm0[6],zero,xmm0[7],zero
+; AVX512BF16-NEXT: vpslld $16, %ymm0, %ymm0
+; AVX512BF16-NEXT: vroundps $11, %ymm0, %ymm0
+; AVX512BF16-NEXT: vcvtneps2bf16 %ymm0, %xmm0
+; AVX512BF16-NEXT: vzeroupper
; AVX512BF16-NEXT: retq
%r = call <8 x bfloat> @llvm.trunc.v8bf16(<8 x bfloat> %a)
ret <8 x bfloat> %r
@@ -246,54 +117,11 @@ define <8 x bfloat> @rint_v8bf16(<8 x bfloat> %a) {
;
; AVX512BF16-LABEL: rint_v8bf16:
; AVX512BF16: # %bb.0:
-; AVX512BF16-NEXT: vpextrw $7, %xmm0, %eax
-; AVX512BF16-NEXT: vmovd %eax, %xmm1
-; AVX512BF16-NEXT: vpslld $16, %xmm1, %xmm1
-; AVX512BF16-NEXT: vroundss $4, %xmm1, %xmm1, %xmm1
-; AVX512BF16-NEXT: vcvtneps2bf16 %xmm1, %xmm1
-; AVX512BF16-NEXT: vmovd %xmm1, %eax
-; AVX512BF16-NEXT: vpextrw $6, %xmm0, %ecx
-; AVX512BF16-NEXT: vmovd %ecx, %xmm1
-; AVX512BF16-NEXT: vpslld $16, %xmm1, %xmm1
-; AVX512BF16-NEXT: vroundss $4, %xmm1, %xmm1, %xmm1
-; AVX512BF16-NEXT: vcvtneps2bf16 %xmm1, %xmm1
-; AVX512BF16-NEXT: vmovd %xmm1, %ecx
-; AVX512BF16-NEXT: vpextrw $5, %xmm0, %edx
-; AVX512BF16-NEXT: vmovd %edx, %xmm1
-; AVX512BF16-NEXT: vpslld $16, %xmm1, %xmm1
-; AVX512BF16-NEXT: vroundss $4, %xmm1, %xmm1, %xmm1
-; AVX512BF16-NEXT: vcvtneps2bf16 %xmm1, %xmm1
-; AVX512BF16-NEXT: vmovd %xmm1, %edx
-; AVX512BF16-NEXT: vpxor %xmm1, %xmm1, %xmm1
-; AVX512BF16-NEXT: vpunpckhwd {{.*#+}} xmm2 = xmm1[4],xmm0[4],xmm1[5],xmm0[5],xmm1[6],xmm0[6],xmm1[7],xmm0[7]
-; AVX512BF16-NEXT: vroundss $4, %xmm2, %xmm2, %xmm2
-; AVX512BF16-NEXT: vcvtneps2bf16 %xmm2, %xmm2
-; AVX512BF16-NEXT: vmovd %xmm2, %esi
-; AVX512BF16-NEXT: vpextrw $3, %xmm0, %edi
-; AVX512BF16-NEXT: vmovd %edi, %xmm2
-; AVX512BF16-NEXT: vpslld $16, %xmm2, %xmm2
-; AVX512BF16-NEXT: vroundss $4, %xmm2, %xmm2, %xmm2
-; AVX512BF16-NEXT: vcvtneps2bf16 %xmm2, %xmm2
-; AVX512BF16-NEXT: vmovd %xmm2, %edi
-; AVX512BF16-NEXT: vpextrw $2, %xmm0, %r8d
-; AVX512BF16-NEXT: vmovd %r8d, %xmm2
-; AVX512BF16-NEXT: vpslld $16, %xmm2, %xmm2
-; AVX512BF16-NEXT: vroundss $4, %xmm2, %xmm2, %xmm2
-; AVX512BF16-NEXT: vcvtneps2bf16 %xmm2, %xmm2
-; AVX512BF16-NEXT: vmovd %xmm2, %r8d
-; AVX512BF16-NEXT: vpblendw {{.*#+}} xmm1 = xmm1[0],xmm0[1],xmm1[2],xmm0[3],xmm1[4],xmm0[5],xmm1[6],xmm0[7]
-; AVX512BF16-NEXT: vroundss $4, %xmm1, %xmm1, %xmm1
-; AVX512BF16-NEXT: vcvtneps2bf16 %xmm1, %xmm1
-; AVX512BF16-NEXT: vpslld $16, %xmm0, %xmm0
-; AVX512BF16-NEXT: vroundss $4, %xmm0, %xmm0, %xmm0
-; AVX512BF16-NEXT: vcvtneps2bf16 %xmm0, %xmm0
-; AVX512BF16-NEXT: vpunpcklwd {{.*#+}} xmm0 = xmm0[0],xmm1[0],xmm0[1],xmm1[1],xmm0[2],xmm1[2],xmm0[3],xmm1[3]
-; AVX512BF16-NEXT: vpinsrw $2, %r8d, %xmm0, %xmm0
-; AVX512BF16-NEXT: vpinsrw $3, %edi, %xmm0, %xmm0
-; AVX512BF16-NEXT: vpinsrw $4, %esi, %xmm0, %xmm0
-; AVX512BF16-NEXT: vpinsrw $5, %edx, %xmm0, %xmm0
-; AVX512BF16-NEXT: vpinsrw $6, %ecx, %xmm0, %xmm0
-; AVX512BF16-NEXT: vpinsrw $7, %eax, %xmm0, %xmm0
+; AVX512BF16-NEXT: vpmovzxwd {{.*#+}} ymm0 = xmm0[0],zero,xmm0[1],zero,xmm0[2],zero,xmm0[3],zero,xmm0[4],zero,xmm0[5],zero,xmm0[6],zero,xmm0[7],zero
+; AVX512BF16-NEXT: vpslld $16, %ymm0, %ymm0
+; AVX512BF16-NEXT: vroundps $4, %ymm0, %ymm0
+; AVX512BF16-NEXT: vcvtneps2bf16 %ymm0, %xmm0
+; AVX512BF16-NEXT: vzeroupper
; AVX512BF16-NEXT: retq
%r = call <8 x bfloat> @llvm.rint.v8bf16(<8 x bfloat> %a)
ret <8 x bfloat> %r
@@ -323,54 +151,11 @@ define <8 x bfloat> @nearbyint_v8bf16(<8 x bfloat> %a) {
;
; AVX512BF16-LABEL: nearbyint_v8bf16:
; AVX512BF16: # %bb.0:
-; AVX512BF16-NEXT: vpextrw $7, %xmm0, %eax
-; AVX512BF16-NEXT: vmovd %eax, %xmm1
-; AVX512BF16-NEXT: vpslld $16, %xmm1, %xmm1
-; AVX512BF16-NEXT: vroundss $12, %xmm1, %xmm1, %xmm1
-; AVX512BF16-NEXT: vcvtneps2bf16 %xmm1, %xmm1
-; AVX512BF16-NEXT: vmovd %xmm1, %eax
-; AVX512BF16-NEXT: vpextrw $6, %xmm0, %ecx
-; AVX512BF16-NEXT: vmovd %ecx, %xmm1
-; AVX512BF16-NEXT: vpslld $16, %xmm1, %xmm1
-; AVX512BF16-NEXT: vroundss $12, %xmm1, %xmm1, %xmm1
-; AVX512BF16-NEXT: vcvtneps2bf16 %xmm1, %xmm1
-; AVX512BF16-NEXT: vmovd %xmm1, %ecx
-; AVX512BF16-NEXT: vpextrw $5, %xmm0, %edx
-; AVX512BF16-NEXT: vmovd %edx, %xmm1
-; AVX512BF16-NEXT: vpslld $16, %xmm1, %xmm1
-; AVX512BF16-NEXT: vroundss $12, %xmm1, %xmm1, %xmm1
-; AVX512BF16-NEXT: vcvtneps2bf16 %xmm1, %xmm1
-; AVX512BF16-NEXT: vmovd %xmm1, %edx
-; AVX512BF16-NEXT: vpxor %xmm1, %xmm1, %xmm1
-; AVX512BF16-NEXT: vpunpckhwd {{.*#+}} xmm2 = xmm1[4],xmm0[4],xmm1[5],xmm0[5],xmm1[6],xmm0[6],xmm1[7],xmm0[7]
-; AVX512BF16-NEXT: vroundss $12, %xmm2, %xmm2, %xmm2
-; AVX512BF16-NEXT: vcvtneps2bf16 %xmm2, %xmm2
-; AVX512BF16-NEXT: vmovd %xmm2, %esi
-; AVX512BF16-NEXT: vpextrw $3, %xmm0, %edi
-; AVX512BF16-NEXT: vmovd %edi, %xmm2
-; AVX512BF16-NEXT: vpslld $16, %xmm2, %xmm2
-; AVX512BF16-NEXT: vroundss $12, %xmm2, %xmm2, %xmm2
-; AVX512BF16-NEXT: vcvtneps2bf16 %xmm2, %xmm2
-; AVX512BF16-NEXT: vmovd %xmm2, %edi
-; AVX512BF16-NEXT: vpextrw $2, %xmm0, %r8d
-; AVX512BF16-NEXT: vmovd %r8d, %xmm2
-; AVX512BF16-NEXT: vpslld $16, %xmm2, %xmm2
-; AVX512BF16-NEXT: vroundss $12, %xmm2, %xmm2, %xmm2
-; AVX512BF16-NEXT: vcvtneps2bf16 %xmm2, %xmm2
-; AVX512BF16-NEXT: vmovd %xmm2, %r8d
-; AVX512BF16-NEXT: vpblendw {{.*#+}} xmm1 = xmm1[0],xmm0[1],xmm1[2],xmm0[3],xmm1[4],xmm0[5],xmm1[6],xmm0[7]
-; AVX512BF16-NEXT: vroundss $12, %xmm1, %xmm1, %xmm1
-; AVX512BF16-NEXT: vcvtneps2bf16 %xmm1, %xmm1
-; AVX512BF16-NEXT: vpslld $16, %xmm0, %xmm0
-; AVX512BF16-NEXT: vroundss $12, %xmm0, %xmm0, %xmm0
-; AVX512BF16-NEXT: vcvtneps2bf16 %xmm0, %xmm0
-; AVX512BF16-NEXT: vpunpcklwd {{.*#+}} xmm0 = xmm0[0],xmm1[0],xmm0[1],xmm1[1],xmm0[2],xmm1[2],xmm0[3],xmm1[3]
-; AVX512BF16-NEXT: vpinsrw $2, %r8d, %xmm0, %xmm0
-; AVX512BF16-NEXT: vpinsrw $3, %edi, %xmm0, %xmm0
-; AVX512BF16-NEXT: vpinsrw $4, %esi, %xmm0, %xmm0
-; AVX512BF16-NEXT: vpinsrw $5, %edx, %xmm0, %xmm0
-; AVX512BF16-NEXT: vpinsrw $6, %ecx, %xmm0, %xmm0
-; AVX512BF16-NEXT: vpinsrw $7, %eax, %xmm0, %xmm0
+; AVX512BF16-NEXT: vpmovzxwd {{.*#+}} ymm0 = xmm0[0],zero,xmm0[1],zero,xmm0[2],zero,xmm0[3],zero,xmm0[4],zero,xmm0[5],zero,xmm0[6],zero,xmm0[7],zero
+; AVX512BF16-NEXT: vpslld $16, %ymm0, %ymm0
+; AVX512BF16-NEXT: vroundps $12, %ymm0, %ymm0
+; AVX512BF16-NEXT: vcvtneps2bf16 %ymm0, %xmm0
+; AVX512BF16-NEXT: vzeroupper
; AVX512BF16-NEXT: retq
%r = call <8 x bfloat> @llvm.nearbyint.v8bf16(<8 x bfloat> %a)
ret <8 x bfloat> %r
@@ -400,54 +185,11 @@ define <8 x bfloat> @roundeven_v8bf16(<8 x bfloat> %a) {
;
; AVX512BF16-LABEL: roundeven_v8bf16:
; AVX512BF16: # %bb.0:
-; AVX512BF16-NEXT: vpextrw $7, %xmm0, %eax
-; AVX512BF16-NEXT: vmovd %eax, %xmm1
-; AVX512BF16-NEXT: vpslld $16, %xmm1, %xmm1
-; AVX512BF16-NEXT: vroundss $8, %xmm1, %xmm1, %xmm1
-; AVX512BF16-NEXT: vcvtneps2bf16 %xmm1, %xmm1
-; AVX512BF16-NEXT: vmovd %xmm1, %eax
-; AVX512BF16-NEXT: vpextrw $6, %xmm0, %ecx
-; AVX512BF16-NEXT: vmovd %ecx, %xmm1
-; AVX512BF16-NEXT: vpslld $16, %xmm1, %xmm1
-; AVX512BF16-NEXT: vroundss $8, %xmm1, %xmm1, %xmm1
-; AVX512BF16-NEXT: vcvtneps2bf16 %xmm1, %xmm1
-; AVX512BF16-NEXT: vmovd %xmm1, %ecx
-; AVX512BF16-NEXT: vpextrw $5, %xmm0, %edx
-; AVX512BF16-NEXT: vmovd %edx, %xmm1
-; AVX512BF16-NEXT: vpslld $16, %xmm1, %xmm1
-; AVX512BF16-NEXT: vroundss $8, %xmm1, %xmm1, %xmm1
-; AVX512BF16-NEXT: vcvtneps2bf16 %xmm1, %xmm1
-; AVX512BF16-NEXT: vmovd %xmm1, %edx
-; AVX512BF16-NEXT: vpxor %xmm1, %xmm1, %xmm1
-; AVX512BF16-NEXT: vpunpckhwd {{.*#+}} xmm2 = xmm1[4],xmm0[4],xmm1[5],xmm0[5],xmm1[6],xmm0[6],xmm1[7],xmm0[7]
-; AVX512BF16-NEXT: vroundss $8, %xmm2, %xmm2, %xmm2
-; AVX512BF16-NEXT: vcvtneps2bf16 %xmm2, %xmm2
-; AVX512BF16-NEXT: vmovd %xmm2, %esi
-; AVX512BF16-NEXT: vpextrw $3, %xmm0, %edi
-; AVX512BF16-NEXT: vmovd %edi, %xmm2
-; AVX512BF16-NEXT: vpslld $16, %xmm2, %xmm2
-; AVX512BF16-NEXT: vroundss $8, %xmm2, %xmm2, %xmm2
-; AVX512BF16-NEXT: vcvtneps2bf16 %xmm2, %xmm2
-; AVX512BF16-NEXT: vmovd %xmm2, %edi
-; AVX512BF16-NEXT: vpextrw $2, %xmm0, %r8d
-; AVX512BF16-NEXT: vmovd %r8d, %xmm2
-; AVX512BF16-NEXT: vpslld $16, %xmm2, %xmm2
-; AVX512BF16-NEXT: vroundss $8, %xmm2, %xmm2, %xmm2
-; AVX512BF16-NEXT: vcvtneps2bf16 %xmm2, %xmm2
-; AVX512BF16-NEXT: vmovd %xmm2, %r8d
-; AVX512BF16-NEXT: vpblendw {{.*#+}} xmm1 = xmm1[0],xmm0[1],xmm1[2],xmm0[3],xmm1[4],xmm0[5],xmm1[6],xmm0[7]
-; AVX512BF16-NEXT: vroundss $8, %xmm1, %xmm1, %xmm1
-; AVX512BF16-NEXT: vcvtneps2bf16 %xmm1, %xmm1
-; AVX512BF16-NEXT: vpslld $16, %xmm0, %xmm0
-; AVX512BF16-NEXT: vroundss $8, %xmm0, %xmm0, %xmm0
-; AVX512BF16-NEXT: vcvtneps2bf16 %xmm0, %xmm0
-; AVX512BF16-NEXT: vpunpcklwd {{.*#+}} xmm0 = xmm0[0],xmm1[0],xmm0[1],xmm1[1],xmm0[2],xmm1[2],xmm0[3],xmm1[3]
-; AVX512BF16-NEXT: vpinsrw $2, %r8d, %xmm0, %xmm0
-; AVX512BF16-NEXT: vpinsrw $3, %edi, %xmm0, %xmm0
-; AVX512BF16-NEXT: vpinsrw $4, %esi, %xmm0, %xmm0
-; AVX512BF16-NEXT: vpinsrw $5, %edx, %xmm0, %xmm0
-; AVX512BF16-NEXT: vpinsrw $6, %ecx, %xmm0, %xmm0
-; AVX512BF16-NEXT: vpinsrw $7, %eax, %xmm0, %xmm0
+; AVX512BF16-NEXT: vpmovzxwd {{.*#+}} ymm0 = xmm0[0],zero,xmm0[1],zero,xmm0[2],zero,xmm0[3],zero,xmm0[4],zero,xmm0[5],zero,xmm0[6],zero,xmm0[7],zero
+; AVX512BF16-NEXT: vpslld $16, %ymm0, %ymm0
+; AVX512BF16-NEXT: vroundps $8, %ymm0, %ymm0
+; AVX512BF16-NEXT: vcvtneps2bf16 %ymm0, %xmm0
+; AVX512BF16-NEXT: vzeroupper
; AVX512BF16-NEXT: retq
%r = call <8 x bfloat> @llvm.roundeven.v8bf16(<8 x bfloat> %a)
ret <8 x bfloat> %r
@@ -478,130 +220,10 @@ define <16 x bfloat> @floor_v16bf16(<16 x bfloat> %a) {
;
; AVX512BF16-LABEL: floor_v16bf16:
; AVX512BF16: # %bb.0:
-; AVX512BF16-NEXT: pushq %rbp
-; AVX512BF16-NEXT: .cfi_def_cfa_offset 16
-; AVX512BF16-NEXT: pushq %r15
-; AVX512BF16-NEXT: .cfi_def_cfa_offset 24
-; AVX512BF16-NEXT: pushq %r14
-; AVX512BF16-NEXT: .cfi_def_cfa_offset 32
-; AVX512BF16-NEXT: pushq %r12
-; AVX512BF16-NEXT: .cfi_def_cfa_offset 40
-; AVX512BF16-NEXT: pushq %rbx
-; AVX512BF16-NEXT: .cfi_def_cfa_offset 48
-; AVX512BF16-NEXT: .cfi_offset %rbx, -48
-; AVX512BF16-NEXT: .cfi_offset %r12, -40
-; AVX512BF16-NEXT: .cfi_offset %r14, -32
-; AVX512BF16-NEXT: .cfi_offset %r15, -24
-; AVX512BF16-NEXT: .cfi_offset %rbp, -16
-; AVX512BF16-NEXT: vpextrw $7, %xmm0, %eax
-; AVX512BF16-NEXT: vmovd %eax, %xmm1
-; AVX512BF16-NEXT: vpslld $16, %xmm1, %xmm1
-; AVX512BF16-NEXT: vroundss $9, %xmm1, %xmm1, %xmm1
-; AVX512BF16-NEXT: vcvtneps2bf16 %xmm1, %xmm1
-; AVX512BF16-NEXT: vmovd %xmm1, %eax
-; AVX512BF16-NEXT: vpextrw $6, %xmm0, %ecx
-; AVX512BF16-NEXT: vmovd %ecx, %xmm1
-; AVX512BF16-NEXT: vpslld $16, %xmm1, %xmm1
-; AVX512BF16-NEXT: vroundss $9, %xmm1, %xmm1, %xmm1
-; AVX512BF16-NEXT: vcvtneps2bf16 %xmm1, %xmm1
-; AVX512BF16-NEXT: vmovd %xmm1, %ecx
-; AVX512BF16-NEXT: vpextrw $5, %xmm0, %edx
-; AVX512BF16-NEXT: vmovd %edx, %xmm1
-; AVX512BF16-NEXT: vpslld $16, %xmm1, %xmm1
-; AVX512BF16-NEXT: vroundss $9, %xmm1, %xmm1, %xmm1
-; AVX512BF16-NEXT: vcvtneps2bf16 %xmm1, %xmm1
-; AVX512BF16-NEXT: vmovd %xmm1, %edx
-; AVX512BF16-NEXT: vpxor %xmm1, %xmm1, %xmm1
-; AVX512BF16-NEXT: vpunpckhwd {{.*#+}} xmm2 = xmm1[4],xmm0[4],xmm1[5],xmm0[5],xmm1[6],xmm0[6],xmm1[7],xmm0[7]
-; AVX512BF16-NEXT: vroundss $9, %xmm2, %xmm2, %xmm2
-; AVX512BF16-NEXT: vcvtneps2bf16 %xmm2, %xmm2
-; AVX512BF16-NEXT: vmovd %xmm2, %esi
-; AVX512BF16-NEXT: vpextrw $3, %xmm0, %edi
-; AVX512BF16-NEXT: vmovd %edi, %xmm2
-; AVX512BF16-NEXT: vpslld $16, %xmm2, %xmm2
-; AVX512BF16-NEXT: vroundss $9, %xmm2, %xmm2, %xmm2
-; AVX512BF16-NEXT: vcvtneps2bf16 %xmm2, %xmm2
-; AVX512BF16-NEXT: vmovd %xmm2, %edi
-; AVX512BF16-NEXT: vpextrw $2, %xmm0, %r8d
-; AVX512BF16-NEXT: vmovd %r8d, %xmm2
-; AVX512BF16-NEXT: vpslld $16, %xmm2, %xmm2
-; AVX512BF16-NEXT: vroundss $9, %xmm2, %xmm2, %xmm2
-; AVX512BF16-NEXT: vcvtneps2bf16 %xmm2, %xmm2
-; AVX512BF16-NEXT: vmovd %xmm2, %r8d
-; AVX512BF16-NEXT: vpslld $16, %xmm0, %xmm2
-; AVX512BF16-NEXT: vroundss $9, %xmm2, %xmm2, %xmm2
-; AVX512BF16-NEXT: vcvtneps2bf16 %xmm2, %xmm2
-; AVX512BF16-NEXT: vpblendw {{.*#+}} xmm3 = xmm1[0],xmm0[1],xmm1[2],xmm0[3],xmm1[4],xmm0[5],xmm1[6],xmm0[7]
-; AVX512BF16-NEXT: vroundss $9, %xmm3, %xmm3, %xmm3
-; AVX512BF16-NEXT: vcvtneps2bf16 %xmm3, %xmm3
-; AVX512BF16-NEXT: vmovd %xmm3, %r9d
-; AVX512BF16-NEXT: vextracti128 $1, %ymm0, %xmm0
-; AVX512BF16-NEXT: vpextrw $7, %xmm0, %r10d
-; AVX512BF16-NEXT: vmovd %r10d, %xmm3
-; AVX512BF16-NEXT: vpslld $16, %xmm3, %xmm3
-; AVX512BF16-NEXT: vroundss $9, %xmm3, %xmm3, %xmm3
-; AVX512BF16-NEXT: vcvtneps2bf16 %xmm3, %xmm3
-; AVX512BF16-NEXT: vmovd %xmm3, %r10d
-; AVX512BF16-NEXT: vpextrw $6, %xmm0, %r11d
-; AVX512BF16-NEXT: vmovd %r11d, %xmm3
-; AVX512BF16-NEXT: vpslld $16, %xmm3, %xmm3
-; AVX512BF16-NEXT: vroundss $9, %xmm3, %xmm3, %xmm3
-; AVX512BF16-NEXT: vcvtneps2bf16 %xmm3, %xmm3
-; AVX512BF16-NEXT: vmovd %xmm3, %r11d
-; AVX512BF16-NEXT: vpextrw $5, %xmm0, %ebx
-; AVX512BF16-NEXT: vmovd %ebx, %xmm3
-; AVX512BF16-NEXT: vpslld $16, %xmm3, %xmm3
-; AVX512BF16-NEXT: vroundss $9, %xmm3, %xmm3, %xmm3
-; AVX512BF16-NEXT: vcvtneps2bf16 %xmm3, %xmm3
-; AVX512BF16-NEXT: vmovd %xmm3, %ebx
-; AVX512BF16-NEXT: vpunpckhwd {{.*#+}} xmm3 = xmm1[4],xmm0[4],xmm1[5],xmm0[5],xmm1[6],xmm0[6],xmm1[7],xmm0[7]
-; AVX512BF16-NEXT: vroundss $9, %xmm3, %xmm3, %xmm3
-; AVX512BF16-NEXT: vcvtneps2bf16 %xmm3, %xmm3
-; AVX512BF16-NEXT: vmovd %xmm3, %ebp
-; AVX512BF16-NEXT: vpextrw $3, %xmm0, %r14d
-; AVX512BF16-NEXT: vmovd %r14d, %xmm3
-; AVX512BF16-NEXT: vpslld $16, %xmm3, %xmm3
-; AVX512BF16-NEXT: vroundss $9, %xmm3, %xmm3, %xmm3
-; AVX512BF16-NEXT: vcvtneps2bf16 %xmm3, %xmm3
-; AVX512BF16-NEXT: vmovd %xmm3, %r14d
-; AVX512BF16-NEXT: vpextrw $2, %xmm0, %r15d
-; AVX512BF16-NEXT: vmovd %r15d, %xmm3
-; AVX512BF16-NEXT: vpslld $16, %xmm3, %xmm3
-; AVX512BF16-NEXT: vroundss $9, %xmm3, %xmm3, %xmm3
-; AVX512BF16-NEXT: vcvtneps2bf16 %xmm3, %xmm3
-; AVX512BF16-NEXT: vmovd %xmm3, %r15d
-; AVX512BF16-NEXT: vpslld $16, %xmm0, %xmm3
-; AVX512BF16-NEXT: vroundss $9, %xmm3, %xmm3, %xmm3
-; AVX512BF16-NEXT: vcvtneps2bf16 %xmm3, %xmm3
-; AVX512BF16-NEXT: vpblendw {{.*#+}} xmm0 = xmm1[0],xmm0[1],xmm1[2],xmm0[3],xmm1[4],xmm0[5],xmm1[6],xmm0[7]
-; AVX512BF16-NEXT: vroundss $9, %xmm0, %xmm0, %xmm0
-; AVX512BF16-NEXT: vcvtneps2bf16 %xmm0, %xmm0
-; AVX512BF16-NEXT: vmovd %xmm0, %r12d
-; AVX512BF16-NEXT: vpinsrw $1, %r12d, %xmm3, %xmm0
-; AVX512BF16-NEXT: vpinsrw $2, %r15d, %xmm0, %xmm0
-; AVX512BF16-NEXT: vpinsrw $3, %r14d, %xmm0, %xmm0
-; AVX512BF16-NEXT: vpinsrw $4, %ebp, %xmm0, %xmm0
-; AVX512BF16-NEXT: vpinsrw $5, %ebx, %xmm0, %xmm0
-; AVX512BF16-NEXT: vpinsrw $6, %r11d, %xmm0, %xmm0
-; AVX512BF16-NEXT: vpinsrw $7, %r10d, %xmm0, %xmm0
-; AVX512BF16-NEXT: vpinsrw $1, %r9d, %xmm2, %xmm1
-; AVX512BF16-NEXT: vpinsrw $2, %r8d, %xmm1, %xmm1
-; AVX512BF16-NEXT: vpinsrw $3, %edi, %xmm1, %xmm1
-; AVX512BF16-NEXT: vpinsrw $4, %esi, %xmm1, %xmm1
-; AVX512BF16-NEXT: vpinsrw $5, %edx, %xmm1, %xmm1
-; AVX512BF16-NEXT: vpinsrw $6, %ecx, %xmm1, %xmm1
-; AVX512BF16-NEXT: vpinsrw $7, %eax, %xmm1, %xmm1
-; AVX512BF16-NEXT: vinserti128 $1, %xmm0, %ymm1, %ymm0
-; AVX512BF16-NEXT: popq %rbx
-; AVX512BF16-NEXT: .cfi_def_cfa_offset 40
-; AVX512BF16-NEXT: popq %r12
-; AVX512BF16-NEXT: .cfi_def_cfa_offset 32
-; AVX512BF16-NEXT: popq %r14
-; AVX512BF16-NEXT: .cfi_def_cfa_offset 24
-; AVX512BF16-NEXT: popq %r15
-; AVX512BF16-NEXT: .cfi_def_cfa_offset 16
-; AVX512BF16-NEXT: popq %rbp
-; AVX512BF16-NEXT: .cfi_def_cfa_offset 8
+; AVX512BF16-NEXT: vpmovzxwd {{.*#+}} zmm0 = ymm0[0],zero,ymm0[1],zero,ymm0[2],zero,ymm0[3],zero,ymm0[4],zero,ymm0[5],zero,ymm0[6],zero,ymm0[7],zero,ymm0[8],zero,ymm0[9],zero,ymm0[10],zero,ymm0[11],zero,ymm0[12],zero,ymm0[13],zero,ymm0[14],zero,ymm0[15],zero
+; AVX512BF16-NEXT: vpslld $16, %zmm0, %zmm0
+; AVX512BF16-NEXT: vrndscaleps $9, %zmm0, %zmm0
+; AVX512BF16-NEXT: vcvtneps2bf16 %zmm0, %ymm0
; AVX512BF16-NEXT: retq
%r = call <16 x bfloat> @llvm.floor.v16bf16(<16 x bfloat> %a)
ret <16 x bfloat> %r
@@ -615,205 +237,16 @@ define <32 x bfloat> @floor_v32bf16(<32 x bfloat> %a) {
;
; AVX512BF16-LABEL: floor_v32bf16:
; AVX512BF16: # %bb.0:
-; AVX512BF16-NEXT: vpextrw $7, %xmm0, %eax
-; AVX512BF16-NEXT: vmovd %eax, %xmm23
-; AVX512BF16-NEXT: vpextrw $6, %xmm0, %eax
-; AVX512BF16-NEXT: vmovd %eax, %xmm2
-; AVX512BF16-NEXT: vpextrw $5, %xmm0, %eax
-; AVX512BF16-NEXT: vmovd %eax, %xmm3
-; AVX512BF16-NEXT: vpxor %xmm5, %xmm5, %xmm5
-; AVX512BF16-NEXT: vpextrw $3, %xmm0, %eax
-; AVX512BF16-NEXT: vmovd %eax, %xmm4
-; AVX512BF16-NEXT: vpextrw $2, %xmm0, %eax
-; AVX512BF16-NEXT: vmovd %eax, %xmm6
-; AVX512BF16-NEXT: vextracti128 $1, %ymm0, %xmm9
-; AVX512BF16-NEXT: vpextrw $7, %xmm9, %eax
-; AVX512BF16-NEXT: vmovd %eax, %xmm7
-; AVX512BF16-NEXT: vpextrw $6, %xmm9, %eax
-; AVX512BF16-NEXT: vmovd %eax, %xmm8
-; AVX512BF16-NEXT: vpextrw $5, %xmm9, %eax
-; AVX512BF16-NEXT: vmovd %eax, %xmm10
-; AVX512BF16-NEXT: vpextrw $3, %xmm9, %eax
-; AVX512BF16-NEXT: vmovd %eax, %xmm11
-; AVX512BF16-NEXT: vpextrw $2, %xmm9, %eax
-; AVX512BF16-NEXT: vmovd %eax, %xmm12
-; AVX512BF16-NEXT: vpblendw {{.*#+}} xmm1 = xmm5[0],xmm9[1],xmm5[2],xmm9[3],xmm5[4],xmm9[5],xmm5[6],xmm9[7]
-; AVX512BF16-NEXT: vroundss $9, %xmm1, %xmm1, %xmm13
-; AVX512BF16-NEXT: vextracti32x4 $2, %zmm0, %xmm1
-; AVX512BF16-NEXT: vpextrw $7, %xmm1, %eax
-; AVX512BF16-NEXT: vmovd %eax, %xmm14
-; AVX512BF16-NEXT: vpextrw $6, %xmm1, %eax
-; AVX512BF16-NEXT: vmovd %eax, %xmm15
-; AVX512BF16-NEXT: vpextrw $5, %xmm1, %eax
-; AVX512BF16-NEXT: vmovd %eax, %xmm16
-; AVX512BF16-NEXT: vpunpckhwd {{.*#+}} xmm17 = xmm5[4],xmm1[4],xmm5[5],xmm1[5],xmm5[6],xmm1[6],xmm5[7],xmm1[7]
-; AVX512BF16-NEXT: vpextrw $3, %xmm1, %eax
-; AVX512BF16-NEXT: vmovd %eax, %xmm19
-; AVX512BF16-NEXT: vpextrw $2, %xmm1, %eax
-; AVX512BF16-NEXT: vmovd %eax, %xmm20
-; AVX512BF16-NEXT: vpslld $16, %xmm1, %xmm21
-; AVX512BF16-NEXT: vpblendw {{.*#+}} xmm1 = xmm5[0],xmm1[1],xmm5[2],xmm1[3],xmm5[4],xmm1[5],xmm5[6],xmm1[7]
-; AVX512BF16-NEXT: vrndscaless $9, %xmm1, %xmm1, %xmm18
-; AVX512BF16-NEXT: vextracti32x4 $3, %zmm0, %xmm1
-; AVX512BF16-NEXT: vpextrw $7, %xmm1, %eax
-; AVX512BF16-NEXT: vmovd %eax, %xmm22
-; AVX512BF16-NEXT: vpextrw $6, %xmm1, %eax
-; AVX512BF16-NEXT: vmovd %eax, %xmm24
-; AVX512BF16-NEXT: vpextrw $5, %xmm1, %eax
-; AVX512BF16-NEXT: vmovd %eax, %xmm25
-; AVX512BF16-NEXT: vpunpckhwd {{.*#+}} xmm26 = xmm5[4],xmm1[4],xmm5[5],xmm1[5],xmm5[6],xmm1[6],xmm5[7],xmm1[7]
-; AVX512BF16-NEXT: vpextrw $3, %xmm1, %eax
-; AVX512BF16-NEXT: vmovd %eax, %xmm27
-; AVX512BF16-NEXT: vpextrw $2, %xmm1, %eax
-; AVX512BF16-NEXT: vpslld $16, %xmm1, %xmm28
-; AVX512BF16-NEXT: vpblendw {{.*#+}} xmm1 = xmm5[0],xmm1[1],xmm5[2],xmm1[3],xmm5[4],xmm1[5],xmm5[6],xmm1[7]
-; AVX512BF16-NEXT: vroundss $9, %xmm1, %xmm1, %xmm1
-; AVX512BF16-NEXT: vmovd %eax, %xmm29
-; AVX512BF16-NEXT: vcvtneps2bf16 %xmm1, %xmm1
-; AVX512BF16-NEXT: vmovd %xmm1, %eax
-; AVX512BF16-NEXT: vpslld $16, %xmm29, %xmm1
-; AVX512BF16-NEXT: vroundss $9, %xmm1, %xmm1, %xmm1
-; AVX512BF16-NEXT: vcvtneps2bf16 %xmm1, %xmm1
-; AVX512BF16-NEXT: vrndscaless $9, %xmm28, %xmm28, %xmm28
-; AVX512BF16-NEXT: vcvtneps2bf16 %xmm28, %xmm28
-; AVX512BF16-NEXT: vpinsrw $1, %eax, %xmm28, %xmm28
-; AVX512BF16-NEXT: vmovd %xmm1, %eax
-; AVX512BF16-NEXT: vpslld $16, %xmm27, %xmm1
-; AVX512BF16-NEXT: vroundss $9, %xmm1, %xmm1, %xmm1
-; AVX512BF16-NEXT: vcvtneps2bf16 %xmm1, %xmm1
-; AVX512BF16-NEXT: vpinsrw $2, %eax, %xmm28, %xmm27
-; AVX512BF16-NEXT: vmovd %xmm1, %eax
-; AVX512BF16-NEXT: vrndscaless $9, %xmm26, %xmm26, %xmm1
-; AVX512BF16-NEXT: vcvtneps2bf16 %xmm1, %xmm1
-; AVX512BF16-NEXT: vpinsrw $3, %eax, %xmm27, %xmm26
-; AVX512BF16-NEXT: vmovd %xmm1, %eax
-; AVX512BF16-NEXT: vpslld $16, %xmm25, %xmm1
-; AVX512BF16-NEXT: vroundss $9, %xmm1, %xmm1, %xmm1
-; AVX512BF16-NEXT: vcvtneps2bf16 %xmm1, %xmm1
-; AVX512BF16-NEXT: vpinsrw $4, %eax, %xmm26, %xmm25
-; AVX512BF16-NEXT: vmovd %xmm1, %eax
-; AVX512BF16-NEXT: vpslld $16, %xmm24, %xmm1
-; AVX512BF16-NEXT: vroundss $9, %xmm1, %xmm1, %xmm1
-; AVX512BF16-NEXT: vcvtneps2bf16 %xmm1, %xmm1
-; AVX512BF16-NEXT: vpinsrw $5, %eax, %xmm25, %xmm24
-; AVX512BF16-NEXT: vmovd %xmm1, %eax
-; AVX512BF16-NEXT: vpslld $16, %xmm22, %xmm1
-; AVX512BF16-NEXT: vroundss $9, %xmm1, %xmm1, %xmm1
-; AVX512BF16-NEXT: vcvtneps2bf16 %xmm1, %xmm1
-; AVX512BF16-NEXT: vpinsrw $6, %eax, %xmm24, %xmm22
-; AVX512BF16-NEXT: vmovd %xmm1, %eax
-; AVX512BF16-NEXT: vcvtneps2bf16 %xmm18, %xmm1
-; AVX512BF16-NEXT: vpinsrw $7, %eax, %xmm22, %xmm18
-; AVX512BF16-NEXT: vmovd %xmm1, %eax
-; AVX512BF16-NEXT: vpslld $16, %xmm20, %xmm1
-; AVX512BF16-NEXT: vroundss $9, %xmm1, %xmm1, %xmm1
-; AVX512BF16-NEXT: vcvtneps2bf16 %xmm1, %xmm1
-; AVX512BF16-NEXT: vrndscaless $9, %xmm21, %xmm21, %xmm20
-; AVX512BF16-NEXT: vcvtneps2bf16 %xmm20, %xmm20
-; AVX512BF16-NEXT: vpinsrw $1, %eax, %xmm20, %xmm20
-; AVX512BF16-NEXT: vmovd %xmm1, %eax
-; AVX512BF16-NEXT: vpslld $16, %xmm19, %xmm1
-; AVX512BF16-NEXT: vroundss $9, %xmm1, %xmm1, %xmm1
-; AVX512BF16-NEXT: vcvtneps2bf16 %xmm1, %xmm1
-; AVX512BF16-NEXT: vpinsrw $2, %eax, %xmm20, %xmm19
-; AVX512BF16-NEXT: vmovd %xmm1, %eax
-; AVX512BF16-NEXT: vrndscaless $9, %xmm17, %xmm17, %xmm1
-; AVX512BF16-NEXT: vcvtneps2bf16 %xmm1, %xmm1
-; AVX512BF16-NEXT: vpinsrw $3, %eax, %xmm19, %xmm17
-; AVX512BF16-NEXT: vmovd %xmm1, %eax
-; AVX512BF16-NEXT: vpslld $16, %xmm16, %xmm1
-; AVX512BF16-NEXT: vroundss $9, %xmm1, %xmm1, %xmm1
-; AVX512BF16-NEXT: vcvtneps2bf16 %xmm1, %xmm1
-; AVX512BF16-NEXT: vpinsrw $4, %eax, %xmm17, %xmm16
-; AVX512BF16-NEXT: vmovd %xmm1, %eax
-; AVX512BF16-NEXT: vpslld $16, %xmm15, %xmm1
-; AVX512BF16-NEXT: vroundss $9, %xmm1, %xmm1, %xmm1
-; AVX512BF16-NEXT: vcvtneps2bf16 %xmm1, %xmm1
-; AVX512BF16-NEXT: vpinsrw $5, %eax, %xmm16, %xmm15
-; AVX512BF16-NEXT: vmovd %xmm1, %eax
-; AVX512BF16-NEXT: vpslld $16, %xmm14, %xmm1
-; AVX512BF16-NEXT: vroundss $9, %xmm1, %xmm1, %xmm1
-; AVX512BF16-NEXT: vcvtneps2bf16 %xmm1, %xmm1
-; AVX512BF16-NEXT: vpinsrw $6, %eax, %xmm15, %xmm14
-; AVX512BF16-NEXT: vmovd %xmm1, %eax
-; AVX512BF16-NEXT: vpslld $16, %xmm12, %xmm1
-; AVX512BF16-NEXT: vroundss $9, %xmm1, %xmm1, %xmm1
-; AVX512BF16-NEXT: vcvtneps2bf16 %xmm1, %xmm1
-; AVX512BF16-NEXT: vcvtneps2bf16 %xmm13, %xmm13
-; AVX512BF16-NEXT: vpinsrw $7, %eax, %xmm14, %xmm12
-; AVX512BF16-NEXT: vmovd %xmm13, %eax
-; AVX512BF16-NEXT: vpslld $16, %xmm9, %xmm13
-; AVX512BF16-NEXT: vroundss $9, %xmm13, %xmm13, %xmm13
-; AVX512BF16-NEXT: vcvtneps2bf16 %xmm13, %xmm13
-; AVX512BF16-NEXT: vpinsrw $1, %eax, %xmm13, %xmm13
-; AVX512BF16-NEXT: vmovd %xmm1, %eax
-; AVX512BF16-NEXT: vpslld $16, %xmm11, %xmm1
-; AVX512BF16-NEXT: vroundss $9, %xmm1, %xmm1, %xmm1
-; AVX512BF16-NEXT: vcvtneps2bf16 %xmm1, %xmm1
-; AVX512BF16-NEXT: vpinsrw $2, %eax, %xmm13, %xmm11
-; AVX512BF16-NEXT: vmovd %xmm1, %eax
-; AVX512BF16-NEXT: vpunpckhwd {{.*#+}} xmm1 = xmm5[4],xmm9[4],xmm5[5],xmm9[5],xmm5[6],xmm9[6],xmm5[7],xmm9[7]
-; AVX512BF16-NEXT: vroundss $9, %xmm1, %xmm1, %xmm1
-; AVX512BF16-NEXT: vcvtneps2bf16 %xmm1, %xmm1
-; AVX512BF16-NEXT: vpinsrw $3, %eax, %xmm11, %xmm9
-; AVX512BF16-NEXT: vmovd %xmm1, %eax
-; AVX512BF16-NEXT: vpslld $16, %xmm10, %xmm1
-; AVX512BF16-NEXT: vroundss $9, %xmm1, %xmm1, %xmm1
-; AVX512BF16-NEXT: vcvtneps2bf16 %xmm1, %xmm1
-; AVX512BF16-NEXT: vpinsrw $4, %eax, %xmm9, %xmm9
-; AVX512BF16-NEXT: vmovd %xmm1, %eax
-; AVX512BF16-NEXT: vpslld $16, %xmm8, %xmm1
-; AVX512BF16-NEXT: vroundss $9, %xmm1, %xmm1, %xmm1
-; AVX512BF16-NEXT: vcvtneps2bf16 %xmm1, %xmm1
-; AVX512BF16-NEXT: vpinsrw $5, %eax, %xmm9, %xmm8
-; AVX512BF16-NEXT: vmovd %xmm1, %eax
-; AVX512BF16-NEXT: vpslld $16, %xmm7, %xmm1
-; AVX512BF16-NEXT: vroundss $9, %xmm1, %xmm1, %xmm1
-; AVX512BF16-NEXT: vcvtneps2bf16 %xmm1, %xmm1
-; AVX512BF16-NEXT: vpinsrw $6, %eax, %xmm8, %xmm7
-; AVX512BF16-NEXT: vmovd %xmm1, %eax
-; AVX512BF16-NEXT: vpblendw {{.*#+}} xmm1 = xmm5[0],xmm0[1],xmm5[2],xmm0[3],xmm5[4],xmm0[5],xmm5[6],xmm0[7]
-; AVX512BF16-NEXT: vroundss $9, %xmm1, %xmm1, %xmm1
-; AVX512BF16-NEXT: vcvtneps2bf16 %xmm1, %xmm1
-; AVX512BF16-NEXT: vpinsrw $7, %eax, %xmm7, %xmm7
-; AVX512BF16-NEXT: vmovd %xmm1, %eax
-; AVX512BF16-NEXT: vpunpckhwd {{.*#+}} xmm1 = xmm5[4],xmm0[4],xmm5[5],xmm0[5],xmm5[6],xmm0[6],xmm5[7],xmm0[7]
-; AVX512BF16-NEXT: vpslld $16, %xmm6, %xmm5
-; AVX512BF16-NEXT: vroundss $9, %xmm5, %xmm5, %xmm5
-; AVX512BF16-NEXT: vcvtneps2bf16 %xmm5, %xmm5
-; AVX512BF16-NEXT: vpslld $16, %xmm0, %xmm0
-; AVX512BF16-NEXT: vroundss $9, %xmm0, %xmm0, %xmm0
-; AVX512BF16-NEXT: vcvtneps2bf16 %xmm0, %xmm0
-; AVX512BF16-NEXT: vpinsrw $1, %eax, %xmm0, %xmm0
-; AVX512BF16-NEXT: vmovd %xmm5, %eax
-; AVX512BF16-NEXT: vpslld $16, %xmm4, %xmm4
-; AVX512BF16-NEXT: vroundss $9, %xmm4, %xmm4, %xmm4
-; AVX512BF16-NEXT: vcvtneps2bf16 %xmm4, %xmm4
-; AVX512BF16-NEXT: vpinsrw $2, %eax, %xmm0, %xmm0
-; AVX512BF16-NEXT: vmovd %xmm4, %eax
-; AVX512BF16-NEXT: vroundss $9, %xmm1, %xmm1, %xmm1
-; AVX512BF16-NEXT: vcvtneps2bf16 %xmm1, %xmm1
-; AVX512BF16-NEXT: vpinsrw $3, %eax, %xmm0, %xmm0
-; AVX512BF16-NEXT: vmovd %xmm1, %eax
-; AVX512BF16-NEXT: vpslld $16, %xmm3, %xmm1
-; AVX512BF16-NEXT: vroundss $9, %xmm1, %xmm1, %xmm1
-; AVX512BF16-NEXT: vcvtneps2bf16 %xmm1, %xmm1
-; AVX512BF16-NEXT: vpinsrw $4, %eax, %xmm0, %xmm0
-; AVX512BF16-NEXT: vmovd %xmm1, %eax
-; AVX512BF16-NEXT: vpslld $16, %xmm23, %xmm1
-; AVX512BF16-NEXT: vroundss $9, %xmm1, %xmm1, %xmm1
-; AVX512BF16-NEXT: vcvtneps2bf16 %xmm1, %xmm1
-; AVX512BF16-NEXT: vpslld $16, %xmm2, %xmm2
-; AVX512BF16-NEXT: vroundss $9, %xmm2, %xmm2, %xmm2
-; AVX512BF16-NEXT: vcvtneps2bf16 %xmm2, %xmm2
-; AVX512BF16-NEXT: vpinsrw $5, %eax, %xmm0, %xmm0
-; AVX512BF16-NEXT: vmovd %xmm2, %eax
-; AVX512BF16-NEXT: vpinsrw $6, %eax, %xmm0, %xmm0
-; AVX512BF16-NEXT: vmovd %xmm1, %eax
-; AVX512BF16-NEXT: vpinsrw $7, %eax, %xmm0, %xmm0
-; AVX512BF16-NEXT: vinserti32x4 $1, %xmm18, %ymm12, %ymm1
-; AVX512BF16-NEXT: vinserti128 $1, %xmm7, %ymm0, %ymm0
-; AVX512BF16-NEXT: vinserti64x4 $1, %ymm1, %zmm0, %zmm0
+; AVX512BF16-NEXT: vpmovzxwd {{.*#+}} zmm1 = ymm0[0],zero,ymm0[1],zero,ymm0[2],zero,ymm0[3],zero,ymm0[4],zero,ymm0[5],zero,ymm0[6],zero,ymm0[7],zero,ymm0[8],zero,ymm0[9],zero,ymm0[10],zero,ymm0[11],zero,ymm0[12],zero,ymm0[13],zero,ymm0[14],zero,ymm0[15],zero
+; AVX512BF16-NEXT: vpslld $16, %zmm1, %zmm1
+; AVX512BF16-NEXT: vrndscaleps $9, %zmm1, %zmm1
+; AVX512BF16-NEXT: vcvtneps2bf16 %zmm1, %ymm1
+; AVX512BF16-NEXT: vextracti64x4 $1, %zmm0, %ymm0
+; AVX512BF16-NEXT: vpmovzxwd {{.*#+}} zmm0 = ymm0[0],zero,ymm0[1],zero,ymm0[2],zero,ymm0[3],zero,ymm0[4],zero,ymm0[5],zero,ymm0[6],zero,ymm0[7],zero,ymm0[8],zero,ymm0[9],zero,ymm0[10],zero,ymm0[11],zero,ymm0[12],zero,ymm0[13],zero,ymm0[14],zero,ymm0[15],zero
+; AVX512BF16-NEXT: vpslld $16, %zmm0, %zmm0
+; AVX512BF16-NEXT: vrndscaleps $9, %zmm0, %zmm0
+; AVX512BF16-NEXT: vcvtneps2bf16 %zmm0, %ymm0
+; AVX512BF16-NEXT: vinsertf64x4 $1, %ymm0, %zmm1, %zmm0
; AVX512BF16-NEXT: retq
%r = call <32 x bfloat> @llvm.floor.v32bf16(<32 x bfloat> %a)
ret <32 x bfloat> %r
@@ -828,14 +261,11 @@ define <2 x bfloat> @floor_v2bf16(<2 x bfloat> %a) {
;
; AVX512BF16-LABEL: floor_v2bf16:
; AVX512BF16: # %bb.0:
-; AVX512BF16-NEXT: vpxor %xmm1, %xmm1, %xmm1
-; AVX512BF16-NEXT: vpblendw {{.*#+}} xmm1 = xmm1[0],xmm0[1],xmm1[2],xmm0[3],xmm1[4],xmm0[5],xmm1[6],xmm0[7]
-; AVX512BF16-NEXT: vroundss $9, %xmm1, %xmm1, %xmm1
-; AVX512BF16-NEXT: vcvtneps2bf16 %xmm1, %xmm1
-; AVX512BF16-NEXT: vpslld $16, %xmm0, %xmm0
-; AVX512BF16-NEXT: vroundss $9, %xmm0, %xmm0, %xmm0
-; AVX512BF16-NEXT: vcvtneps2bf16 %xmm0, %xmm0
-; AVX512BF16-NEXT: vpunpcklwd {{.*#+}} xmm0 = xmm0[0],xmm1[0],xmm0[1],xmm1[1],xmm0[2],xmm1[2],xmm0[3],xmm1[3]
+; AVX512BF16-NEXT: vpmovzxwd {{.*#+}} ymm0 = xmm0[0],zero,xmm0[1],zero,xmm0[2],zero,xmm0[3],zero,xmm0[4],zero,xmm0[5],zero,xmm0[6],zero,xmm0[7],zero
+; AVX512BF16-NEXT: vpslld $16, %ymm0, %ymm0
+; AVX512BF16-NEXT: vroundps $9, %ymm0, %ymm0
+; AVX512BF16-NEXT: vcvtneps2bf16 %ymm0, %xmm0
+; AVX512BF16-NEXT: vzeroupper
; AVX512BF16-NEXT: retq
%r = call <2 x bfloat> @llvm.floor.v2bf16(<2 x bfloat> %a)
ret <2 x bfloat> %r
@@ -850,55 +280,11 @@ define <8 x bfloat> @floor_v8bf16_load(ptr %p) {
;
; AVX512BF16-LABEL: floor_v8bf16_load:
; AVX512BF16: # %bb.0:
-; AVX512BF16-NEXT: vmovdqa (%rdi), %xmm0
-; AVX512BF16-NEXT: vpextrw $7, %xmm0, %eax
-; AVX512BF16-NEXT: vmovd %eax, %xmm1
-; AVX512BF16-NEXT: vpslld $16, %xmm1, %xmm1
-; AVX512BF16-NEXT: vroundss $9, %xmm1, %xmm1, %xmm1
-; AVX512BF16-NEXT: vcvtneps2bf16 %xmm1, %xmm1
-; AVX512BF16-NEXT: vmovd %xmm1, %eax
-; AVX512BF16-NEXT: vpextrw $6, %xmm0, %ecx
-; AVX512BF16-NEXT: vmovd %ecx, %xmm1
-; AVX512BF16-NEXT: vpslld $16, %xmm1, %xmm1
-; AVX512BF16-NEXT: vroundss $9, %xmm1, %xmm1, %xmm1
-; AVX512BF16-NEXT: vcvtneps2bf16 %xmm1, %xmm1
-; AVX512BF16-NEXT: vmovd %xmm1, %ecx
-; AVX512BF16-NEXT: vpextrw $5, %xmm0, %edx
-; AVX512BF16-NEXT: vmovd %edx, %xmm1
-; AVX512BF16-NEXT: vpslld $16, %xmm1, %xmm1
-; AVX512BF16-NEXT: vroundss $9, %xmm1, %xmm1, %xmm1
-; AVX512BF16-NEXT: vcvtneps2bf16 %xmm1, %xmm1
-; AVX512BF16-NEXT: vmovd %xmm1, %edx
-; AVX512BF16-NEXT: vpxor %xmm1, %xmm1, %xmm1
-; AVX512BF16-NEXT: vpunpckhwd {{.*#+}} xmm2 = xmm1[4],xmm0[4],xmm1[5],xmm0[5],xmm1[6],xmm0[6],xmm1[7],xmm0[7]
-; AVX512BF16-NEXT: vroundss $9, %xmm2, %xmm2, %xmm2
-; AVX512BF16-NEXT: vcvtneps2bf16 %xmm2, %xmm2
-; AVX512BF16-NEXT: vmovd %xmm2, %esi
-; AVX512BF16-NEXT: vpextrw $3, %xmm0, %edi
-; AVX512BF16-NEXT: vmovd %edi, %xmm2
-; AVX512BF16-NEXT: vpslld $16, %xmm2, %xmm2
-; AVX512BF16-NEXT: vroundss $9, %xmm2, %xmm2, %xmm2
-; AVX512BF16-NEXT: vcvtneps2bf16 %xmm2, %xmm2
-; AVX512BF16-NEXT: vmovd %xmm2, %edi
-; AVX512BF16-NEXT: vpextrw $2, %xmm0, %r8d
-; AVX512BF16-NEXT: vmovd %r8d, %xmm2
-; AVX512BF16-NEXT: vpslld $16, %xmm2, %xmm2
-; AVX512BF16-NEXT: vroundss $9, %xmm2, %xmm2, %xmm2
-; AVX512BF16-NEXT: vcvtneps2bf16 %xmm2, %xmm2
-; AVX512BF16-NEXT: vmovd %xmm2, %r8d
-; AVX512BF16-NEXT: vpblendw {{.*#+}} xmm1 = xmm1[0],xmm0[1],xmm1[2],xmm0[3],xmm1[4],xmm0[5],xmm1[6],xmm0[7]
-; AVX512BF16-NEXT: vroundss $9, %xmm1, %xmm1, %xmm1
-; AVX512BF16-NEXT: vcvtneps2bf16 %xmm1, %xmm1
-; AVX512BF16-NEXT: vpslld $16, %xmm0, %xmm0
-; AVX512BF16-NEXT: vroundss $9, %xmm0, %xmm0, %xmm0
-; AVX512BF16-NEXT: vcvtneps2bf16 %xmm0, %xmm0
-; AVX512BF16-NEXT: vpunpcklwd {{.*#+}} xmm0 = xmm0[0],xmm1[0],xmm0[1],xmm1[1],xmm0[2],xmm1[2],xmm0[3],xmm1[3]
-; AVX512BF16-NEXT: vpinsrw $2, %r8d, %xmm0, %xmm0
-; AVX512BF16-NEXT: vpinsrw $3, %edi, %xmm0, %xmm0
-; AVX512BF16-NEXT: vpinsrw $4, %esi, %xmm0, %xmm0
-; AVX512BF16-NEXT: vpinsrw $5, %edx, %xmm0, %xmm0
-; AVX512BF16-NEXT: vpinsrw $6, %ecx, %xmm0, %xmm0
-; AVX512BF16-NEXT: vpinsrw $7, %eax, %xmm0, %xmm0
+; AVX512BF16-NEXT: vpmovzxwd {{.*#+}} ymm0 = mem[0],zero,mem[1],zero,mem[2],zero,mem[3],zero,mem[4],zero,mem[5],zero,mem[6],zero,mem[7],zero
+; AVX512BF16-NEXT: vpslld $16, %ymm0, %ymm0
+; AVX512BF16-NEXT: vroundps $9, %ymm0, %ymm0
+; AVX512BF16-NEXT: vcvtneps2bf16 %ymm0, %xmm0
+; AVX512BF16-NEXT: vzeroupper
; AVX512BF16-NEXT: retq
%a = load <8 x bfloat>, ptr %p
%r = call <8 x bfloat> @llvm.floor.v8bf16(<8 x bfloat> %a)
@@ -917,55 +303,12 @@ define <8 x bfloat> @floor_v8bf16_mask(<8 x bfloat> %a, <8 x bfloat> %src, i8 %m
; AVX512BF16-LABEL: floor_v8bf16_mask:
; AVX512BF16: # %bb.0:
; AVX512BF16-NEXT: kmovd %edi, %k1
-; AVX512BF16-NEXT: vpextrw $7, %xmm0, %eax
-; AVX512BF16-NEXT: vmovd %eax, %xmm2
-; AVX512BF16-NEXT: vpslld $16, %xmm2, %xmm2
-; AVX512BF16-NEXT: vroundss $9, %xmm2, %xmm2, %xmm2
-; AVX512BF16-NEXT: vcvtneps2bf16 %xmm2, %xmm2
-; AVX512BF16-NEXT: vmovd %xmm2, %eax
-; AVX512BF16-NEXT: vpextrw $6, %xmm0, %ecx
-; AVX512BF16-NEXT: vmovd %ecx, %xmm2
-; AVX512BF16-NEXT: vpslld $16, %xmm2, %xmm2
-; AVX512BF16-NEXT: vroundss $9, %xmm2, %xmm2, %xmm2
-; AVX512BF16-NEXT: vcvtneps2bf16 %xmm2, %xmm2
-; AVX512BF16-NEXT: vmovd %xmm2, %ecx
-; AVX512BF16-NEXT: vpextrw $5, %xmm0, %edx
-; AVX512BF16-NEXT: vmovd %edx, %xmm2
-; AVX512BF16-NEXT: vpslld $16, %xmm2, %xmm2
-; AVX512BF16-NEXT: vroundss $9, %xmm2, %xmm2, %xmm2
-; AVX512BF16-NEXT: vcvtneps2bf16 %xmm2, %xmm2
-; AVX512BF16-NEXT: vmovd %xmm2, %edx
-; AVX512BF16-NEXT: vpxor %xmm2, %xmm2, %xmm2
-; AVX512BF16-NEXT: vpunpckhwd {{.*#+}} xmm3 = xmm2[4],xmm0[4],xmm2[5],xmm0[5],xmm2[6],xmm0[6],xmm2[7],xmm0[7]
-; AVX512BF16-NEXT: vroundss $9, %xmm3, %xmm3, %xmm3
-; AVX512BF16-NEXT: vcvtneps2bf16 %xmm3, %xmm3
-; AVX512BF16-NEXT: vmovd %xmm3, %esi
-; AVX512BF16-NEXT: vpextrw $3, %xmm0, %edi
-; AVX512BF16-NEXT: vmovd %edi, %xmm3
-; AVX512BF16-NEXT: vpslld $16, %xmm3, %xmm3
-; AVX512BF16-NEXT: vroundss $9, %xmm3, %xmm3, %xmm3
-; AVX512BF16-NEXT: vcvtneps2bf16 %xmm3, %xmm3
-; AVX512BF16-NEXT: vmovd %xmm3, %edi
-; AVX512BF16-NEXT: vpextrw $2, %xmm0, %r8d
-; AVX512BF16-NEXT: vmovd %r8d, %xmm3
-; AVX512BF16-NEXT: vpslld $16, %xmm3, %xmm3
-; AVX512BF16-NEXT: vroundss $9, %xmm3, %xmm3, %xmm3
-; AVX512BF16-NEXT: vcvtneps2bf16 %xmm3, %xmm3
-; AVX512BF16-NEXT: vmovd %xmm3, %r8d
-; AVX512BF16-NEXT: vpblendw {{.*#+}} xmm2 = xmm2[0],xmm0[1],xmm2[2],xmm0[3],xmm2[4],xmm0[5],xmm2[6],xmm0[7]
-; AVX512BF16-NEXT: vroundss $9, %xmm2, %xmm2, %xmm2
-; AVX512BF16-NEXT: vcvtneps2bf16 %xmm2, %xmm2
-; AVX512BF16-NEXT: vpslld $16, %xmm0, %xmm0
-; AVX512BF16-NEXT: vroundss $9, %xmm0, %xmm0, %xmm0
-; AVX512BF16-NEXT: vcvtneps2bf16 %xmm0, %xmm0
-; AVX512BF16-NEXT: vpunpcklwd {{.*#+}} xmm0 = xmm0[0],xmm2[0],xmm0[1],xmm2[1],xmm0[2],xmm2[2],xmm0[3],xmm2[3]
-; AVX512BF16-NEXT: vpinsrw $2, %r8d, %xmm0, %xmm0
-; AVX512BF16-NEXT: vpinsrw $3, %edi, %xmm0, %xmm0
-; AVX512BF16-NEXT: vpinsrw $4, %esi, %xmm0, %xmm0
-; AVX512BF16-NEXT: vpinsrw $5, %edx, %xmm0, %xmm0
-; AVX512BF16-NEXT: vpinsrw $6, %ecx, %xmm0, %xmm0
-; AVX512BF16-NEXT: vpinsrw $7, %eax, %xmm0, %xmm0
+; AVX512BF16-NEXT: vpmovzxwd {{.*#+}} ymm0 = xmm0[0],zero,xmm0[1],zero,xmm0[2],zero,xmm0[3],zero,xmm0[4],zero,xmm0[5],zero,xmm0[6],zero,xmm0[7],zero
+; AVX512BF16-NEXT: vpslld $16, %ymm0, %ymm0
+; AVX512BF16-NEXT: vroundps $9, %ymm0, %ymm0
+; AVX512BF16-NEXT: vcvtneps2bf16 %ymm0, %xmm0
; AVX512BF16-NEXT: vpblendmw %xmm0, %xmm1, %xmm0 {%k1}
+; AVX512BF16-NEXT: vzeroupper
; AVX512BF16-NEXT: retq
%mask = bitcast i8 %msk to <8 x i1>
%r = call <8 x bfloat> @llvm.floor.v8bf16(<8 x bfloat> %a)
@@ -983,55 +326,12 @@ define <8 x bfloat> @floor_v8bf16_maskz(<8 x bfloat> %a, i8 %msk) {
; AVX512BF16-LABEL: floor_v8bf16_maskz:
; AVX512BF16: # %bb.0:
; AVX512BF16-NEXT: kmovd %edi, %k1
-; AVX512BF16-NEXT: vpextrw $7, %xmm0, %eax
-; AVX512BF16-NEXT: vmovd %eax, %xmm1
-; AVX512BF16-NEXT: vpslld $16, %xmm1, %xmm1
-; AVX512BF16-NEXT: vroundss $9, %xmm1, %xmm1, %xmm1
-; AVX512BF16-NEXT: vcvtneps2bf16 %xmm1, %xmm1
-; AVX512BF16-NEXT: vmovd %xmm1, %eax
-; AVX512BF16-NEXT: vpextrw $6, %xmm0, %ecx
-; AVX512BF16-NEXT: vmovd %ecx, %xmm1
-; AVX512BF16-NEXT: vpslld $16, %xmm1, %xmm1
-; AVX512BF16-NEXT: vroundss $9, %xmm1, %xmm1, %xmm1
-; AVX512BF16-NEXT: vcvtneps2bf16 %xmm1, %xmm1
-; AVX512BF16-NEXT: vmovd %xmm1, %ecx
-; AVX512BF16-NEXT: vpextrw $5, %xmm0, %edx
-; AVX512BF16-NEXT: vmovd %edx, %xmm1
-; AVX512BF16-NEXT: vpslld $16, %xmm1, %xmm1
-; AVX512BF16-NEXT: vroundss $9, %xmm1, %xmm1, %xmm1
-; AVX512BF16-NEXT: vcvtneps2bf16 %xmm1, %xmm1
-; AVX512BF16-NEXT: vmovd %xmm1, %edx
-; AVX512BF16-NEXT: vpxor %xmm1, %xmm1, %xmm1
-; AVX512BF16-NEXT: vpunpckhwd {{.*#+}} xmm2 = xmm1[4],xmm0[4],xmm1[5],xmm0[5],xmm1[6],xmm0[6],xmm1[7],xmm0[7]
-; AVX512BF16-NEXT: vroundss $9, %xmm2, %xmm2, %xmm2
-; AVX512BF16-NEXT: vcvtneps2bf16 %xmm2, %xmm2
-; AVX512BF16-NEXT: vmovd %xmm2, %esi
-; AVX512BF16-NEXT: vpextrw $3, %xmm0, %edi
-; AVX512BF16-NEXT: vmovd %edi, %xmm2
-; AVX512BF16-NEXT: vpslld $16, %xmm2, %xmm2
-; AVX512BF16-NEXT: vroundss $9, %xmm2, %xmm2, %xmm2
-; AVX512BF16-NEXT: vcvtneps2bf16 %xmm2, %xmm2
-; AVX512BF16-NEXT: vmovd %xmm2, %edi
-; AVX512BF16-NEXT: vpextrw $2, %xmm0, %r8d
-; AVX512BF16-NEXT: vmovd %r8d, %xmm2
-; AVX512BF16-NEXT: vpslld $16, %xmm2, %xmm2
-; AVX512BF16-NEXT: vroundss $9, %xmm2, %xmm2, %xmm2
-; AVX512BF16-NEXT: vcvtneps2bf16 %xmm2, %xmm2
-; AVX512BF16-NEXT: vmovd %xmm2, %r8d
-; AVX512BF16-NEXT: vpblendw {{.*#+}} xmm1 = xmm1[0],xmm0[1],xmm1[2],xmm0[3],xmm1[4],xmm0[5],xmm1[6],xmm0[7]
-; AVX512BF16-NEXT: vroundss $9, %xmm1, %xmm1, %xmm1
-; AVX512BF16-NEXT: vcvtneps2bf16 %xmm1, %xmm1
-; AVX512BF16-NEXT: vpslld $16, %xmm0, %xmm0
-; AVX512BF16-NEXT: vroundss $9, %xmm0, %xmm0, %xmm0
-; AVX512BF16-NEXT: vcvtneps2bf16 %xmm0, %xmm0
-; AVX512BF16-NEXT: vpunpcklwd {{.*#+}} xmm0 = xmm0[0],xmm1[0],xmm0[1],xmm1[1],xmm0[2],xmm1[2],xmm0[3],xmm1[3]
-; AVX512BF16-NEXT: vpinsrw $2, %r8d, %xmm0, %xmm0
-; AVX512BF16-NEXT: vpinsrw $3, %edi, %xmm0, %xmm0
-; AVX512BF16-NEXT: vpinsrw $4, %esi, %xmm0, %xmm0
-; AVX512BF16-NEXT: vpinsrw $5, %edx, %xmm0, %xmm0
-; AVX512BF16-NEXT: vpinsrw $6, %ecx, %xmm0, %xmm0
-; AVX512BF16-NEXT: vpinsrw $7, %eax, %xmm0, %xmm0
+; AVX512BF16-NEXT: vpmovzxwd {{.*#+}} ymm0 = xmm0[0],zero,xmm0[1],zero,xmm0[2],zero,xmm0[3],zero,xmm0[4],zero,xmm0[5],zero,xmm0[6],zero,xmm0[7],zero
+; AVX512BF16-NEXT: vpslld $16, %ymm0, %ymm0
+; AVX512BF16-NEXT: vroundps $9, %ymm0, %ymm0
+; AVX512BF16-NEXT: vcvtneps2bf16 %ymm0, %xmm0
; AVX512BF16-NEXT: vmovdqu16 %xmm0, %xmm0 {%k1} {z}
+; AVX512BF16-NEXT: vzeroupper
; AVX512BF16-NEXT: retq
%mask = bitcast i8 %msk to <8 x i1>
%r = call <8 x bfloat> @llvm.floor.v8bf16(<8 x bfloat> %a)
@@ -1044,150 +344,26 @@ define <8 x bfloat> @floor_v8bf16_maskz(<8 x bfloat> %a, i8 %msk) {
define <8 x bfloat> @round_v8bf16(<8 x bfloat> %a) {
; AVX10_2-LABEL: round_v8bf16:
; AVX10_2: # %bb.0:
-; AVX10_2-NEXT: vpextrw $3, %xmm0, %eax
-; AVX10_2-NEXT: vmovw %eax, %xmm1
-; AVX10_2-NEXT: vpslld $16, %xmm1, %xmm1
-; AVX10_2-NEXT: vpbroadcastd {{.*#+}} xmm2 = [-0.0E+0,-0.0E+0,-0.0E+0,-0.0E+0]
-; AVX10_2-NEXT: vpbroadcastd {{.*#+}} xmm3 = [4.9999997E-1,4.9999997E-1,4.9999997E-1,4.9999997E-1]
-; AVX10_2-NEXT: vmovdqa %xmm3, %xmm4
-; AVX10_2-NEXT: vpternlogd {{.*#+}} xmm4 = xmm4 | (xmm1 & xmm2)
-; AVX10_2-NEXT: vaddss %xmm4, %xmm1, %xmm1
-; AVX10_2-NEXT: vroundss $11, %xmm1, %xmm1, %xmm1
-; AVX10_2-NEXT: vcvtneps2bf16 %xmm1, %xmm1
-; AVX10_2-NEXT: vpextrw $2, %xmm0, %eax
-; AVX10_2-NEXT: vmovw %eax, %xmm4
-; AVX10_2-NEXT: vpslld $16, %xmm4, %xmm4
-; AVX10_2-NEXT: vmovdqa %xmm3, %xmm5
-; AVX10_2-NEXT: vpternlogd {{.*#+}} xmm5 = xmm5 | (xmm4 & xmm2)
-; AVX10_2-NEXT: vaddss %xmm5, %xmm4, %xmm4
-; AVX10_2-NEXT: vroundss $11, %xmm4, %xmm4, %xmm4
-; AVX10_2-NEXT: vcvtneps2bf16 %xmm4, %xmm4
-; AVX10_2-NEXT: vpunpcklwd {{.*#+}} xmm1 = xmm4[0],xmm1[0],xmm4[1],xmm1[1],xmm4[2],xmm1[2],xmm4[3],xmm1[3]
-; AVX10_2-NEXT: vpxor %xmm4, %xmm4, %xmm4
-; AVX10_2-NEXT: vpblendw {{.*#+}} xmm5 = xmm4[0],xmm0[1],xmm4[2],xmm0[3],xmm4[4],xmm0[5],xmm4[6],xmm0[7]
-; AVX10_2-NEXT: vmovdqa %xmm3, %xmm6
-; AVX10_2-NEXT: vpternlogd {{.*#+}} xmm6 = xmm6 | (xmm5 & xmm2)
-; AVX10_2-NEXT: vaddss %xmm6, %xmm5, %xmm5
-; AVX10_2-NEXT: vroundss $11, %xmm5, %xmm5, %xmm5
-; AVX10_2-NEXT: vcvtneps2bf16 %xmm5, %xmm5
-; AVX10_2-NEXT: vpslld $16, %xmm0, %xmm6
-; AVX10_2-NEXT: vmovdqa %xmm3, %xmm7
-; AVX10_2-NEXT: vpternlogd {{.*#+}} xmm7 = xmm7 | (xmm6 & xmm2)
-; AVX10_2-NEXT: vaddss %xmm7, %xmm6, %xmm6
-; AVX10_2-NEXT: vroundss $11, %xmm6, %xmm6, %xmm6
-; AVX10_2-NEXT: vcvtneps2bf16 %xmm6, %xmm6
-; AVX10_2-NEXT: vpunpcklwd {{.*#+}} xmm5 = xmm6[0],xmm5[0],xmm6[1],xmm5[1],xmm6[2],xmm5[2],xmm6[3],xmm5[3]
-; AVX10_2-NEXT: vpunpckldq {{.*#+}} xmm1 = xmm5[0],xmm1[0],xmm5[1],xmm1[1]
-; AVX10_2-NEXT: vpunpckhwd {{.*#+}} xmm4 = xmm4[4],xmm0[4],xmm4[5],xmm0[5],xmm4[6],xmm0[6],xmm4[7],xmm0[7]
-; AVX10_2-NEXT: vmovdqa %xmm3, %xmm5
-; AVX10_2-NEXT: vpternlogd {{.*#+}} xmm5 = xmm5 | (xmm4 & xmm2)
-; AVX10_2-NEXT: vaddss %xmm5, %xmm4, %xmm4
-; AVX10_2-NEXT: vroundss $11, %xmm4, %xmm4, %xmm4
-; AVX10_2-NEXT: vcvtneps2bf16 %xmm4, %xmm4
-; AVX10_2-NEXT: vpextrw $5, %xmm0, %eax
-; AVX10_2-NEXT: vmovw %eax, %xmm5
-; AVX10_2-NEXT: vpslld $16, %xmm5, %xmm5
-; AVX10_2-NEXT: vmovdqa %xmm3, %xmm6
-; AVX10_2-NEXT: vpternlogd {{.*#+}} xmm6 = xmm6 | (xmm5 & xmm2)
-; AVX10_2-NEXT: vaddss %xmm6, %xmm5, %xmm5
-; AVX10_2-NEXT: vroundss $11, %xmm5, %xmm5, %xmm5
-; AVX10_2-NEXT: vcvtneps2bf16 %xmm5, %xmm5
-; AVX10_2-NEXT: vpunpcklwd {{.*#+}} xmm4 = xmm4[0],xmm5[0],xmm4[1],xmm5[1],xmm4[2],xmm5[2],xmm4[3],xmm5[3]
-; AVX10_2-NEXT: vpextrw $7, %xmm0, %eax
-; AVX10_2-NEXT: vmovw %eax, %xmm5
-; AVX10_2-NEXT: vpslld $16, %xmm5, %xmm5
-; AVX10_2-NEXT: vmovdqa %xmm3, %xmm6
-; AVX10_2-NEXT: vpternlogd {{.*#+}} xmm6 = xmm6 | (xmm5 & xmm2)
-; AVX10_2-NEXT: vaddss %xmm6, %xmm5, %xmm5
-; AVX10_2-NEXT: vroundss $11, %xmm5, %xmm5, %xmm5
-; AVX10_2-NEXT: vcvtneps2bf16 %xmm5, %xmm5
-; AVX10_2-NEXT: vpextrw $6, %xmm0, %eax
-; AVX10_2-NEXT: vmovw %eax, %xmm0
-; AVX10_2-NEXT: vpslld $16, %xmm0, %xmm0
-; AVX10_2-NEXT: vpternlogd {{.*#+}} xmm3 = xmm3 | (xmm0 & xmm2)
-; AVX10_2-NEXT: vaddss %xmm3, %xmm0, %xmm0
-; AVX10_2-NEXT: vroundss $11, %xmm0, %xmm0, %xmm0
-; AVX10_2-NEXT: vcvtneps2bf16 %xmm0, %xmm0
-; AVX10_2-NEXT: vpunpcklwd {{.*#+}} xmm0 = xmm0[0],xmm5[0],xmm0[1],xmm5[1],xmm0[2],xmm5[2],xmm0[3],xmm5[3]
-; AVX10_2-NEXT: vpunpckldq {{.*#+}} xmm0 = xmm4[0],xmm0[0],xmm4[1],xmm0[1]
-; AVX10_2-NEXT: vpunpcklqdq {{.*#+}} xmm0 = xmm1[0],xmm0[0]
+; AVX10_2-NEXT: vpmovzxwd {{.*#+}} ymm0 = xmm0[0],zero,xmm0[1],zero,xmm0[2],zero,xmm0[3],zero,xmm0[4],zero,xmm0[5],zero,xmm0[6],zero,xmm0[7],zero
+; AVX10_2-NEXT: vpslld $16, %ymm0, %ymm0
+; AVX10_2-NEXT: vpbroadcastd {{.*#+}} ymm1 = [4.9999997E-1,4.9999997E-1,4.9999997E-1,4.9999997E-1,4.9999997E-1,4.9999997E-1,4.9999997E-1,4.9999997E-1]
+; AVX10_2-NEXT: vpternlogd {{.*#+}} ymm1 = ymm1 | (ymm0 & m32bcst)
+; AVX10_2-NEXT: vaddps %ymm1, %ymm0, %ymm0
+; AVX10_2-NEXT: vroundps $11, %ymm0, %ymm0
+; AVX10_2-NEXT: vcvtneps2bf16 %ymm0, %xmm0
+; AVX10_2-NEXT: vzeroupper
; AVX10_2-NEXT: retq
;
; AVX512BF16-LABEL: round_v8bf16:
; AVX512BF16: # %bb.0:
-; AVX512BF16-NEXT: vpextrw $7, %xmm0, %eax
-; AVX512BF16-NEXT: vmovd %eax, %xmm1
-; AVX512BF16-NEXT: vpslld $16, %xmm1, %xmm1
-; AVX512BF16-NEXT: vpbroadcastd {{.*#+}} xmm2 = [-0.0E+0,-0.0E+0,-0.0E+0,-0.0E+0]
-; AVX512BF16-NEXT: vpbroadcastd {{.*#+}} xmm3 = [4.9999997E-1,4.9999997E-1,4.9999997E-1,4.9999997E-1]
-; AVX512BF16-NEXT: vmovdqa %xmm3, %xmm4
-; AVX512BF16-NEXT: vpternlogd {{.*#+}} xmm4 = xmm4 | (xmm1 & xmm2)
-; AVX512BF16-NEXT: vaddss %xmm4, %xmm1, %xmm1
-; AVX512BF16-NEXT: vroundss $11, %xmm1, %xmm1, %xmm1
-; AVX512BF16-NEXT: vcvtneps2bf16 %xmm1, %xmm1
-; AVX512BF16-NEXT: vmovd %xmm1, %eax
-; AVX512BF16-NEXT: vpextrw $6, %xmm0, %ecx
-; AVX512BF16-NEXT: vmovd %ecx, %xmm1
-; AVX512BF16-NEXT: vpslld $16, %xmm1, %xmm1
-; AVX512BF16-NEXT: vmovdqa %xmm3, %xmm4
-; AVX512BF16-NEXT: vpternlogd {{.*#+}} xmm4 = xmm4 | (xmm1 & xmm2)
-; AVX512BF16-NEXT: vaddss %xmm4, %xmm1, %xmm1
-; AVX512BF16-NEXT: vroundss $11, %xmm1, %xmm1, %xmm1
-; AVX512BF16-NEXT: vcvtneps2bf16 %xmm1, %xmm1
-; AVX512BF16-NEXT: vmovd %xmm1, %ecx
-; AVX512BF16-NEXT: vpextrw $5, %xmm0, %edx
-; AVX512BF16-NEXT: vmovd %edx, %xmm1
-; AVX512BF16-NEXT: vpslld $16, %xmm1, %xmm1
-; AVX512BF16-NEXT: vmovdqa %xmm3, %xmm4
-; AVX512BF16-NEXT: vpternlogd {{.*#+}} xmm4 = xmm4 | (xmm1 & xmm2)
-; AVX512BF16-NEXT: vaddss %xmm4, %xmm1, %xmm1
-; AVX512BF16-NEXT: vroundss $11, %xmm1, %xmm1, %xmm1
-; AVX512BF16-NEXT: vcvtneps2bf16 %xmm1, %xmm1
-; AVX512BF16-NEXT: vmovd %xmm1, %edx
-; AVX512BF16-NEXT: vpxor %xmm1, %xmm1, %xmm1
-; AVX512BF16-NEXT: vpunpckhwd {{.*#+}} xmm4 = xmm1[4],xmm0[4],xmm1[5],xmm0[5],xmm1[6],xmm0[6],xmm1[7],xmm0[7]
-; AVX512BF16-NEXT: vmovdqa %xmm3, %xmm5
-; AVX512BF16-NEXT: vpternlogd {{.*#+}} xmm5 = xmm5 | (xmm4 & xmm2)
-; AVX512BF16-NEXT: vaddss %xmm5, %xmm4, %xmm4
-; AVX512BF16-NEXT: vroundss $11, %xmm4, %xmm4, %xmm4
-; AVX512BF16-NEXT: vcvtneps2bf16 %xmm4, %xmm4
-; AVX512BF16-NEXT: vmovd %xmm4, %esi
-; AVX512BF16-NEXT: vpextrw $3, %xmm0, %edi
-; AVX512BF16-NEXT: vmovd %edi, %xmm4
-; AVX512BF16-NEXT: vpslld $16, %xmm4, %xmm4
-; AVX512BF16-NEXT: vmovdqa %xmm3, %xmm5
-; AVX512BF16-NEXT: vpternlogd {{.*#+}} xmm5 = xmm5 | (xmm4 & xmm2)
-; AVX512BF16-NEXT: vaddss %xmm5, %xmm4, %xmm4
-; AVX512BF16-NEXT: vroundss $11, %xmm4, %xmm4, %xmm4
-; AVX512BF16-NEXT: vcvtneps2bf16 %xmm4, %xmm4
-; AVX512BF16-NEXT: vmovd %xmm4, %edi
-; AVX512BF16-NEXT: vpextrw $2, %xmm0, %r8d
-; AVX512BF16-NEXT: vmovd %r8d, %xmm4
-; AVX512BF16-NEXT: vpslld $16, %xmm4, %xmm4
-; AVX512BF16-NEXT: vmovdqa %xmm3, %xmm5
-; AVX512BF16-NEXT: vpternlogd {{.*#+}} xmm5 = xmm5 | (xmm4 & xmm2)
-; AVX512BF16-NEXT: vaddss %xmm5, %xmm4, %xmm4
-; AVX512BF16-NEXT: vroundss $11, %xmm4, %xmm4, %xmm4
-; AVX512BF16-NEXT: vcvtneps2bf16 %xmm4, %xmm4
-; AVX512BF16-NEXT: vmovd %xmm4, %r8d
-; AVX512BF16-NEXT: vpblendw {{.*#+}} xmm1 = xmm1[0],xmm0[1],xmm1[2],xmm0[3],xmm1[4],xmm0[5],xmm1[6],xmm0[7]
-; AVX512BF16-NEXT: vmovdqa %xmm3, %xmm4
-; AVX512BF16-NEXT: vpternlogd {{.*#+}} xmm4 = xmm4 | (xmm1 & xmm2)
-; AVX512BF16-NEXT: vaddss %xmm4, %xmm1, %xmm1
-; AVX512BF16-NEXT: vroundss $11, %xmm1, %xmm1, %xmm1
-; AVX512BF16-NEXT: vcvtneps2bf16 %xmm1, %xmm1
-; AVX512BF16-NEXT: vpslld $16, %xmm0, %xmm0
-; AVX512BF16-NEXT: vpternlogd {{.*#+}} xmm3 = xmm3 | (xmm0 & xmm2)
-; AVX512BF16-NEXT: vaddss %xmm3, %xmm0, %xmm0
-; AVX512BF16-NEXT: vroundss $11, %xmm0, %xmm0, %xmm0
-; AVX512BF16-NEXT: vcvtneps2bf16 %xmm0, %xmm0
-; AVX512BF16-NEXT: vpunpcklwd {{.*#+}} xmm0 = xmm0[0],xmm1[0],xmm0[1],xmm1[1],xmm0[2],xmm1[2],xmm0[3],xmm1[3]
-; AVX512BF16-NEXT: vpinsrw $2, %r8d, %xmm0, %xmm0
-; AVX512BF16-NEXT: vpinsrw $3, %edi, %xmm0, %xmm0
-; AVX512BF16-NEXT: vpinsrw $4, %esi, %xmm0, %xmm0
-; AVX512BF16-NEXT: vpinsrw $5, %edx, %xmm0, %xmm0
-; AVX512BF16-NEXT: vpinsrw $6, %ecx, %xmm0, %xmm0
-; AVX512BF16-NEXT: vpinsrw $7, %eax, %xmm0, %xmm0
+; AVX512BF16-NEXT: vpmovzxwd {{.*#+}} ymm0 = xmm0[0],zero,xmm0[1],zero,xmm0[2],zero,xmm0[3],zero,xmm0[4],zero,xmm0[5],zero,xmm0[6],zero,xmm0[7],zero
+; AVX512BF16-NEXT: vpslld $16, %ymm0, %ymm0
+; AVX512BF16-NEXT: vpbroadcastd {{.*#+}} ymm1 = [4.9999997E-1,4.9999997E-1,4.9999997E-1,4.9999997E-1,4.9999997E-1,4.9999997E-1,4.9999997E-1,4.9999997E-1]
+; AVX512BF16-NEXT: vpternlogd {{.*#+}} ymm1 = ymm1 | (ymm0 & m32bcst)
+; AVX512BF16-NEXT: vaddps %ymm1, %ymm0, %ymm0
+; AVX512BF16-NEXT: vroundps $11, %ymm0, %ymm0
+; AVX512BF16-NEXT: vcvtneps2bf16 %ymm0, %xmm0
+; AVX512BF16-NEXT: vzeroupper
; AVX512BF16-NEXT: retq
%r = call <8 x bfloat> @llvm.round.v8bf16(<8 x bfloat> %a)
ret <8 x bfloat> %r
>From f065ee820836f81257578ae253885469e43394c9 Mon Sep 17 00:00:00 2001
From: "Ziegler, Tim" <tim.ziegler at intel.com>
Date: Fri, 28 Aug 2026 16:46:06 +0200
Subject: [PATCH 5/6] Move to existing loop
---
llvm/lib/Target/X86/X86ISelLowering.cpp | 18 ++++++------------
1 file changed, 6 insertions(+), 12 deletions(-)
diff --git a/llvm/lib/Target/X86/X86ISelLowering.cpp b/llvm/lib/Target/X86/X86ISelLowering.cpp
index be9a474a17256..e95f4a542915b 100644
--- a/llvm/lib/Target/X86/X86ISelLowering.cpp
+++ b/llvm/lib/Target/X86/X86ISelLowering.cpp
@@ -2638,18 +2638,6 @@ X86TargetLowering::X86TargetLowering(const X86TargetMachine &TM,
setOperationAction(ISD::FNEARBYINT, MVT::bf16, Custom);
setOperationAction(ISD::FROUNDEVEN, MVT::bf16, Custom);
- setOperationAction(ISD::FFLOOR, MVT::v8bf16, Legal);
- setOperationAction(ISD::FCEIL, MVT::v8bf16, Legal);
- setOperationAction(ISD::FTRUNC, MVT::v8bf16, Legal);
- setOperationAction(ISD::FRINT, MVT::v8bf16, Legal);
- setOperationAction(ISD::FNEARBYINT, MVT::v8bf16, Legal);
- setOperationAction(ISD::FROUNDEVEN, MVT::v8bf16, Legal);
- setOperationAction(ISD::FFLOOR, MVT::v16bf16, Legal);
- setOperationAction(ISD::FCEIL, MVT::v16bf16, Legal);
- setOperationAction(ISD::FTRUNC, MVT::v16bf16, Legal);
- setOperationAction(ISD::FRINT, MVT::v16bf16, Legal);
- setOperationAction(ISD::FNEARBYINT, MVT::v16bf16, Legal);
- setOperationAction(ISD::FROUNDEVEN, MVT::v16bf16, Legal);
if (Subtarget.useAVX512Regs()) {
setOperationAction(ISD::FADD, MVT::v32bf16, Legal);
@@ -2677,6 +2665,12 @@ X86TargetLowering::X86TargetLowering(const X86TargetMachine &TM,
setOperationAction(ISD::FSQRT, VT, Legal);
setOperationAction(ISD::FMA, VT, Legal);
setOperationAction(ISD::SETCC, VT, Custom);
+ setOperationAction(ISD::FFLOOR, VT, Legal);
+ setOperationAction(ISD::FCEIL, VT, Legal);
+ setOperationAction(ISD::FTRUNC, VT, Legal);
+ setOperationAction(ISD::FRINT, VT, Legal);
+ setOperationAction(ISD::FNEARBYINT, VT, Legal);
+ setOperationAction(ISD::FROUNDEVEN, VT, Legal);
SetFPMinMaxAction(VT);
}
for (auto VT : {MVT::f16, MVT::f32, MVT::f64}) {
>From ceb8cb8494535a3b0f6c4dc0e299841618bab5b5 Mon Sep 17 00:00:00 2001
From: "Ziegler, Tim" <tim.ziegler at intel.com>
Date: Fri, 28 Aug 2026 16:50:28 +0200
Subject: [PATCH 6/6] Fixed formatting
---
llvm/lib/Target/X86/X86ISelLowering.cpp | 1 -
1 file changed, 1 deletion(-)
diff --git a/llvm/lib/Target/X86/X86ISelLowering.cpp b/llvm/lib/Target/X86/X86ISelLowering.cpp
index e95f4a542915b..d7a1800db8ab0 100644
--- a/llvm/lib/Target/X86/X86ISelLowering.cpp
+++ b/llvm/lib/Target/X86/X86ISelLowering.cpp
@@ -2638,7 +2638,6 @@ X86TargetLowering::X86TargetLowering(const X86TargetMachine &TM,
setOperationAction(ISD::FNEARBYINT, MVT::bf16, Custom);
setOperationAction(ISD::FROUNDEVEN, MVT::bf16, Custom);
-
if (Subtarget.useAVX512Regs()) {
setOperationAction(ISD::FADD, MVT::v32bf16, Legal);
setOperationAction(ISD::FSUB, MVT::v32bf16, Legal);
More information about the llvm-commits
mailing list