[llvm] af31bd5 - [X86] Lower bf16->f32/f64 fpext without a GPR round-trip (#218359)
via llvm-commits
llvm-commits at lists.llvm.org
Thu Aug 27 03:54:17 PDT 2026
Author: tfzee
Date: 2026-08-27T10:54:11Z
New Revision: af31bd52e300df040127e17c1823ee160609ed9c
URL: https://github.com/llvm/llvm-project/commit/af31bd52e300df040127e17c1823ee160609ed9c
DIFF: https://github.com/llvm/llvm-project/commit/af31bd52e300df040127e17c1823ee160609ed9c.diff
LOG: [X86] Lower bf16->f32/f64 fpext without a GPR round-trip (#218359)
`fpext bfloat to float/double` currently lowers `ISD::BF16_TO_FP` via the
generic target-independent expansion (extend to i32, shl 16, bitcast to
f32), which always forces the value out of its XMM register into a GPR
and back.
Fixes #155270
Add a custom `BF16_TO_FP` lowering (mirroring the existing `FP_TO_BF16`
function) for f32/f64 results that does
the shift in-register instead.
Only on targets with SSE2 or higher otherwise still falls back to the
existing generic expansion.
The updated lowering then for the example
```llvm
define float @src32(bfloat %a0) {
%res = fpext bfloat %a0 to float
ret float %res
}
define double @src64(bfloat %a0) {
%res = fpext bfloat %a0 to double
ret double %res
}
```
generates
```asm
src32:
pslld $16, %xmm0
retq
src64:
pslld $16, %xmm0
cvtss2sd %xmm0, %xmm0
retq
```
instead of the previous roundtrip
```asm
src32:
pextrw $0, %xmm0, %eax
shll $16, %eax
movd %eax, %xmm0
retq
src64:
pextrw $0, %xmm0, %eax
shll $16, %eax
movd %eax, %xmm0
cvtss2sd %xmm0, %xmm0
retq
```
AI used for generating and cleaning up comments and reviewing the code.
Added:
Modified:
llvm/lib/Target/X86/X86ISelLowering.cpp
llvm/lib/Target/X86/X86ISelLowering.h
llvm/test/CodeGen/X86/atomic-load-store.ll
llvm/test/CodeGen/X86/avx10_2bf16-arith-scalar.ll
llvm/test/CodeGen/X86/avx10_2bf16-fma.ll
llvm/test/CodeGen/X86/avx10_2bf16-select-minmax.ll
llvm/test/CodeGen/X86/bf16-fast-isel.ll
llvm/test/CodeGen/X86/bfloat-calling-conv.ll
llvm/test/CodeGen/X86/bfloat-int-cvt.ll
llvm/test/CodeGen/X86/bfloat.ll
llvm/test/CodeGen/X86/float-to-arbitrary-fp.ll
llvm/test/CodeGen/X86/fminimum-fmaximum.ll
llvm/test/CodeGen/X86/fminimumnum-fmaximumnum.ll
llvm/test/CodeGen/X86/llvm.frexp.ll
Removed:
################################################################################
diff --git a/llvm/lib/Target/X86/X86ISelLowering.cpp b/llvm/lib/Target/X86/X86ISelLowering.cpp
index 3a66310ff1b3c..82ae1621aa1ef 100644
--- a/llvm/lib/Target/X86/X86ISelLowering.cpp
+++ b/llvm/lib/Target/X86/X86ISelLowering.cpp
@@ -738,6 +738,9 @@ X86TargetLowering::X86TargetLowering(const X86TargetMachine &TM,
setOperationAction(ISD::STRICT_LRINT, MVT::f16, Promote);
setOperationAction(ISD::STRICT_LLRINT, MVT::f16, Promote);
+ setOperationAction(ISD::BF16_TO_FP, MVT::f32, Custom);
+ setOperationAction(ISD::BF16_TO_FP, MVT::f64, Custom);
+
// Lower this to MOVMSK plus an AND.
setOperationAction(ISD::FGETSIGN, MVT::i64, Custom);
setOperationAction(ISD::FGETSIGN, MVT::i32, Custom);
@@ -22974,6 +22977,38 @@ static SDValue LowerFP_TO_FP16(SDValue Op, SelectionDAG &DAG) {
return Res;
}
+SDValue X86TargetLowering::LowerBF16_TO_FP(SDValue Op,
+ SelectionDAG &DAG) const {
+ SDLoc DL(Op);
+ // Bitcast incase the Operand has not been legalized into a i16 already
+ SDValue Src = DAG.getBitcast(MVT::i16, Op.getOperand(0));
+
+ SDValue Res;
+ if (!Subtarget.hasFP16() && ISD::isNormalLoad(Src.getNode())) {
+ // Without AVX512FP16 we need a GPR to load Src anyway (there's no direct
+ // memory-to-XMM move for a 16-bit value), so do the shift in GPR
+ // instead of in the vector domain, since SHL has better throughput
+ // than a vector shift.
+ SDValue Wide = DAG.getZExtOrTrunc(Src, DL, MVT::i32);
+ Wide = DAG.getNode(ISD::SHL, DL, MVT::i32, Wide,
+ DAG.getShiftAmountConstant(16, MVT::i32, DL));
+ Res = DAG.getBitcast(MVT::f32, Wide);
+ } else {
+ // Shift the bf16 bits into the high half of the f32.
+ SDValue Vec = DAG.getNode(ISD::SCALAR_TO_VECTOR, DL, MVT::v8i16, Src);
+ Vec = DAG.getBitcast(MVT::v4i32, Vec);
+ Vec =
+ getTargetVShiftByConstNode(X86ISD::VSHLI, DL, MVT::v4i32, Vec, 16, DAG);
+ Vec = DAG.getBitcast(MVT::v4f32, Vec);
+ Res = DAG.getExtractVectorElt(DL, MVT::f32, Vec, 0);
+ }
+
+ EVT DstVT = Op.getValueType();
+ if (DstVT != MVT::f32)
+ Res = DAG.getNode(ISD::FP_EXTEND, DL, DstVT, Res);
+ return Res;
+}
+
SDValue X86TargetLowering::LowerFP_TO_BF16(SDValue Op,
SelectionDAG &DAG) const {
SDLoc DL(Op);
@@ -34549,6 +34584,7 @@ SDValue X86TargetLowering::LowerOperation(SDValue Op, SelectionDAG &DAG) const {
case ISD::STRICT_FP16_TO_FP: return LowerFP16_TO_FP(Op, DAG);
case ISD::FP_TO_FP16:
case ISD::STRICT_FP_TO_FP16: return LowerFP_TO_FP16(Op, DAG);
+ case ISD::BF16_TO_FP: return LowerBF16_TO_FP(Op, DAG);
case ISD::FP_TO_BF16: return LowerFP_TO_BF16(Op, DAG);
case ISD::LOAD: return LowerLoad(Op, Subtarget, DAG);
case ISD::STORE: return LowerStore(Op, Subtarget, DAG);
@@ -62482,6 +62518,15 @@ static SDValue combineSCALAR_TO_VECTOR(SDNode *N, SelectionDAG &DAG,
// Combine (v2i64 (scalar_to_vector (i64 (bitcast (mmx))))) to MOVQ2DQ.
if (VT == MVT::v2i64 && SrcOp.getValueType() == MVT::x86mmx)
return DAG.getNode(X86ISD::MOVQ2DQ, DL, VT, SrcOp);
+ // Combine (v8i16 (scalar_to_vector (i16 (bitcast (f16/bf16))))) to VMOVW.
+ if (VT == MVT::v8i16 && Subtarget.hasFP16()) {
+ if (SrcOp.getValueType() == MVT::f16)
+ return DAG.getBitcast(
+ VT, DAG.getNode(ISD::SCALAR_TO_VECTOR, DL, MVT::v8f16, SrcOp));
+ if (SrcOp.getValueType() == MVT::bf16)
+ return DAG.getBitcast(
+ VT, DAG.getNode(ISD::SCALAR_TO_VECTOR, DL, MVT::v8bf16, SrcOp));
+ }
}
if (VT == MVT::v4i32) {
diff --git a/llvm/lib/Target/X86/X86ISelLowering.h b/llvm/lib/Target/X86/X86ISelLowering.h
index 836997e1a45e8..b220c1bbe7bc7 100644
--- a/llvm/lib/Target/X86/X86ISelLowering.h
+++ b/llvm/lib/Target/X86/X86ISelLowering.h
@@ -851,6 +851,7 @@ namespace llvm {
SDValue lowerFaddFsub(SDValue Op, SelectionDAG &DAG) const;
SDValue LowerFP_EXTEND(SDValue Op, SelectionDAG &DAG) const;
SDValue LowerFP_ROUND(SDValue Op, SelectionDAG &DAG) const;
+ SDValue LowerBF16_TO_FP(SDValue Op, SelectionDAG &DAG) const;
SDValue LowerFP_TO_BF16(SDValue Op, SelectionDAG &DAG) const;
SDValue
diff --git a/llvm/test/CodeGen/X86/atomic-load-store.ll b/llvm/test/CodeGen/X86/atomic-load-store.ll
index 7cfe7af47748a..41131b28d9b23 100644
--- a/llvm/test/CodeGen/X86/atomic-load-store.ll
+++ b/llvm/test/CodeGen/X86/atomic-load-store.ll
@@ -239,12 +239,7 @@ define void @store_atomic_vec1_bfloat(ptr %x, <1 x bfloat> %v) nounwind {
; CHECK-SSE-O0: # %bb.0:
; CHECK-SSE-O0-NEXT: pushq %rax
; CHECK-SSE-O0-NEXT: movq %rdi, (%rsp) # 8-byte Spill
-; CHECK-SSE-O0-NEXT: pextrw $0, %xmm0, %eax
-; CHECK-SSE-O0-NEXT: movw %ax, %cx
-; CHECK-SSE-O0-NEXT: # implicit-def: $eax
-; CHECK-SSE-O0-NEXT: movw %cx, %ax
-; CHECK-SSE-O0-NEXT: shll $16, %eax
-; CHECK-SSE-O0-NEXT: movd %eax, %xmm0
+; CHECK-SSE-O0-NEXT: pslld $16, %xmm0
; CHECK-SSE-O0-NEXT: callq __truncsfbf2 at PLT
; CHECK-SSE-O0-NEXT: movq (%rsp), %rdi # 8-byte Reload
; CHECK-SSE-O0-NEXT: pextrw $0, %xmm0, %eax
@@ -257,12 +252,7 @@ define void @store_atomic_vec1_bfloat(ptr %x, <1 x bfloat> %v) nounwind {
; CHECK-AVX-O0: # %bb.0:
; CHECK-AVX-O0-NEXT: pushq %rax
; CHECK-AVX-O0-NEXT: movq %rdi, (%rsp) # 8-byte Spill
-; CHECK-AVX-O0-NEXT: vpextrw $0, %xmm0, %eax
-; CHECK-AVX-O0-NEXT: movw %ax, %cx
-; CHECK-AVX-O0-NEXT: # implicit-def: $eax
-; CHECK-AVX-O0-NEXT: movw %cx, %ax
-; CHECK-AVX-O0-NEXT: shll $16, %eax
-; CHECK-AVX-O0-NEXT: vmovd %eax, %xmm0
+; CHECK-AVX-O0-NEXT: vpslld $16, %xmm0, %xmm0
; CHECK-AVX-O0-NEXT: callq __truncsfbf2 at PLT
; CHECK-AVX-O0-NEXT: movq (%rsp), %rdi # 8-byte Reload
; CHECK-AVX-O0-NEXT: vpextrw $0, %xmm0, %eax
@@ -886,14 +876,14 @@ define void @store_atomic_vec2_bfloat(ptr %x, <2 x bfloat> %v) nounwind {
; CHECK-SSE2-O0-LABEL: store_atomic_vec2_bfloat:
; CHECK-SSE2-O0: # %bb.0:
; CHECK-SSE2-O0-NEXT: subq $24, %rsp
+; CHECK-SSE2-O0-NEXT: movaps %xmm0, %xmm1
; CHECK-SSE2-O0-NEXT: movq %rdi, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Spill
-; CHECK-SSE2-O0-NEXT: pextrw $1, %xmm0, %eax
-; CHECK-SSE2-O0-NEXT: shll $16, %eax
+; CHECK-SSE2-O0-NEXT: movaps %xmm1, %xmm0
+; CHECK-SSE2-O0-NEXT: pslld $16, %xmm0
+; CHECK-SSE2-O0-NEXT: pextrw $1, %xmm1, %eax
; CHECK-SSE2-O0-NEXT: movd %eax, %xmm1
+; CHECK-SSE2-O0-NEXT: pslld $16, %xmm1
; CHECK-SSE2-O0-NEXT: movss %xmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 4-byte Spill
-; CHECK-SSE2-O0-NEXT: movd %xmm0, %eax
-; CHECK-SSE2-O0-NEXT: shll $16, %eax
-; CHECK-SSE2-O0-NEXT: movd %eax, %xmm0
; CHECK-SSE2-O0-NEXT: callq __truncsfbf2 at PLT
; CHECK-SSE2-O0-NEXT: movaps %xmm0, %xmm1
; CHECK-SSE2-O0-NEXT: movss {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 4-byte Reload
@@ -915,13 +905,11 @@ define void @store_atomic_vec2_bfloat(ptr %x, <2 x bfloat> %v) nounwind {
; CHECK-SSE4-O0: # %bb.0:
; CHECK-SSE4-O0-NEXT: subq $24, %rsp
; CHECK-SSE4-O0-NEXT: movq %rdi, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Spill
-; CHECK-SSE4-O0-NEXT: pextrw $1, %xmm0, %eax
-; CHECK-SSE4-O0-NEXT: shll $16, %eax
-; CHECK-SSE4-O0-NEXT: movd %eax, %xmm1
+; CHECK-SSE4-O0-NEXT: xorps %xmm2, %xmm2
+; CHECK-SSE4-O0-NEXT: movaps %xmm0, %xmm1
+; CHECK-SSE4-O0-NEXT: pblendw {{.*#+}} xmm1 = xmm2[0],xmm1[1],xmm2[2],xmm1[3],xmm2[4],xmm1[5],xmm2[6],xmm1[7]
+; CHECK-SSE4-O0-NEXT: pslld $16, %xmm0
; CHECK-SSE4-O0-NEXT: movss %xmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 4-byte Spill
-; CHECK-SSE4-O0-NEXT: movd %xmm0, %eax
-; CHECK-SSE4-O0-NEXT: shll $16, %eax
-; CHECK-SSE4-O0-NEXT: movd %eax, %xmm0
; CHECK-SSE4-O0-NEXT: callq __truncsfbf2 at PLT
; CHECK-SSE4-O0-NEXT: movaps %xmm0, %xmm1
; CHECK-SSE4-O0-NEXT: movss {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 4-byte Reload
@@ -935,29 +923,47 @@ define void @store_atomic_vec2_bfloat(ptr %x, <2 x bfloat> %v) nounwind {
; CHECK-SSE4-O0-NEXT: addq $24, %rsp
; CHECK-SSE4-O0-NEXT: retq
;
-; CHECK-AVX-O0-LABEL: store_atomic_vec2_bfloat:
-; CHECK-AVX-O0: # %bb.0:
-; CHECK-AVX-O0-NEXT: subq $24, %rsp
-; CHECK-AVX-O0-NEXT: movq %rdi, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Spill
-; CHECK-AVX-O0-NEXT: vpextrw $1, %xmm0, %eax
-; CHECK-AVX-O0-NEXT: shll $16, %eax
-; CHECK-AVX-O0-NEXT: vmovd %eax, %xmm1
-; CHECK-AVX-O0-NEXT: vmovss %xmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 4-byte Spill
-; CHECK-AVX-O0-NEXT: vmovd %xmm0, %eax
-; CHECK-AVX-O0-NEXT: shll $16, %eax
-; CHECK-AVX-O0-NEXT: vmovd %eax, %xmm0
-; CHECK-AVX-O0-NEXT: callq __truncsfbf2 at PLT
-; CHECK-AVX-O0-NEXT: vmovaps %xmm0, %xmm1
-; CHECK-AVX-O0-NEXT: vmovss {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 4-byte Reload
-; CHECK-AVX-O0-NEXT: # xmm0 = mem[0],zero,zero,zero
-; CHECK-AVX-O0-NEXT: vpextrw $0, %xmm1, {{[0-9]+}}(%rsp)
-; CHECK-AVX-O0-NEXT: callq __truncsfbf2 at PLT
-; CHECK-AVX-O0-NEXT: movq {{[-0-9]+}}(%r{{[sb]}}p), %rdi # 8-byte Reload
-; CHECK-AVX-O0-NEXT: vpextrw $0, %xmm0, {{[0-9]+}}(%rsp)
-; CHECK-AVX-O0-NEXT: movl {{[0-9]+}}(%rsp), %eax
-; CHECK-AVX-O0-NEXT: movl %eax, (%rdi)
-; CHECK-AVX-O0-NEXT: addq $24, %rsp
-; CHECK-AVX-O0-NEXT: retq
+; CHECK-AVX2-O0-LABEL: store_atomic_vec2_bfloat:
+; CHECK-AVX2-O0: # %bb.0:
+; CHECK-AVX2-O0-NEXT: subq $24, %rsp
+; CHECK-AVX2-O0-NEXT: movq %rdi, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Spill
+; CHECK-AVX2-O0-NEXT: vxorps %xmm1, %xmm1, %xmm1
+; CHECK-AVX2-O0-NEXT: vpblendw {{.*#+}} xmm1 = xmm1[0],xmm0[1],xmm1[2],xmm0[3],xmm1[4],xmm0[5],xmm1[6],xmm0[7]
+; CHECK-AVX2-O0-NEXT: vpslld $16, %xmm0, %xmm0
+; CHECK-AVX2-O0-NEXT: vmovss %xmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 4-byte Spill
+; CHECK-AVX2-O0-NEXT: callq __truncsfbf2 at PLT
+; CHECK-AVX2-O0-NEXT: vmovaps %xmm0, %xmm1
+; CHECK-AVX2-O0-NEXT: vmovss {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 4-byte Reload
+; CHECK-AVX2-O0-NEXT: # xmm0 = mem[0],zero,zero,zero
+; CHECK-AVX2-O0-NEXT: vpextrw $0, %xmm1, {{[0-9]+}}(%rsp)
+; CHECK-AVX2-O0-NEXT: callq __truncsfbf2 at PLT
+; CHECK-AVX2-O0-NEXT: movq {{[-0-9]+}}(%r{{[sb]}}p), %rdi # 8-byte Reload
+; CHECK-AVX2-O0-NEXT: vpextrw $0, %xmm0, {{[0-9]+}}(%rsp)
+; CHECK-AVX2-O0-NEXT: movl {{[0-9]+}}(%rsp), %eax
+; CHECK-AVX2-O0-NEXT: movl %eax, (%rdi)
+; CHECK-AVX2-O0-NEXT: addq $24, %rsp
+; CHECK-AVX2-O0-NEXT: retq
+;
+; CHECK-AVX512-O0-LABEL: store_atomic_vec2_bfloat:
+; CHECK-AVX512-O0: # %bb.0:
+; CHECK-AVX512-O0-NEXT: subq $24, %rsp
+; CHECK-AVX512-O0-NEXT: movq %rdi, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Spill
+; CHECK-AVX512-O0-NEXT: vpxor %xmm1, %xmm1, %xmm1
+; CHECK-AVX512-O0-NEXT: vpblendw {{.*#+}} xmm1 = xmm1[0],xmm0[1],xmm1[2],xmm0[3],xmm1[4],xmm0[5],xmm1[6],xmm0[7]
+; CHECK-AVX512-O0-NEXT: vpslld $16, %xmm0, %xmm0
+; CHECK-AVX512-O0-NEXT: vmovss %xmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 4-byte Spill
+; CHECK-AVX512-O0-NEXT: callq __truncsfbf2 at PLT
+; CHECK-AVX512-O0-NEXT: vmovaps %xmm0, %xmm1
+; CHECK-AVX512-O0-NEXT: vmovss {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 4-byte Reload
+; CHECK-AVX512-O0-NEXT: # xmm0 = mem[0],zero,zero,zero
+; CHECK-AVX512-O0-NEXT: vpextrw $0, %xmm1, {{[0-9]+}}(%rsp)
+; CHECK-AVX512-O0-NEXT: callq __truncsfbf2 at PLT
+; CHECK-AVX512-O0-NEXT: movq {{[-0-9]+}}(%r{{[sb]}}p), %rdi # 8-byte Reload
+; CHECK-AVX512-O0-NEXT: vpextrw $0, %xmm0, {{[0-9]+}}(%rsp)
+; CHECK-AVX512-O0-NEXT: movl {{[0-9]+}}(%rsp), %eax
+; CHECK-AVX512-O0-NEXT: movl %eax, (%rdi)
+; CHECK-AVX512-O0-NEXT: addq $24, %rsp
+; CHECK-AVX512-O0-NEXT: retq
store atomic <2 x bfloat> %v, ptr %x release, align 4
ret void
}
@@ -1024,22 +1030,22 @@ define void @store_atomic_vec4_bfloat(ptr %x, <4 x bfloat> %v) nounwind {
; CHECK-SSE2-O0-LABEL: store_atomic_vec4_bfloat:
; CHECK-SSE2-O0: # %bb.0:
; CHECK-SSE2-O0-NEXT: subq $40, %rsp
+; CHECK-SSE2-O0-NEXT: movaps %xmm0, %xmm1
; CHECK-SSE2-O0-NEXT: movq %rdi, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Spill
-; CHECK-SSE2-O0-NEXT: pextrw $3, %xmm0, %eax
-; CHECK-SSE2-O0-NEXT: shll $16, %eax
-; CHECK-SSE2-O0-NEXT: movd %eax, %xmm1
-; CHECK-SSE2-O0-NEXT: movss %xmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 4-byte Spill
-; CHECK-SSE2-O0-NEXT: pextrw $2, %xmm0, %eax
-; CHECK-SSE2-O0-NEXT: shll $16, %eax
-; CHECK-SSE2-O0-NEXT: movd %eax, %xmm1
-; CHECK-SSE2-O0-NEXT: movss %xmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 4-byte Spill
-; CHECK-SSE2-O0-NEXT: pextrw $1, %xmm0, %eax
-; CHECK-SSE2-O0-NEXT: shll $16, %eax
+; CHECK-SSE2-O0-NEXT: movaps %xmm1, %xmm0
+; CHECK-SSE2-O0-NEXT: pslld $16, %xmm0
+; CHECK-SSE2-O0-NEXT: pextrw $3, %xmm1, %eax
+; CHECK-SSE2-O0-NEXT: movd %eax, %xmm3
+; CHECK-SSE2-O0-NEXT: pslld $16, %xmm3
+; CHECK-SSE2-O0-NEXT: pextrw $2, %xmm1, %eax
+; CHECK-SSE2-O0-NEXT: movd %eax, %xmm2
+; CHECK-SSE2-O0-NEXT: pslld $16, %xmm2
+; CHECK-SSE2-O0-NEXT: pextrw $1, %xmm1, %eax
; CHECK-SSE2-O0-NEXT: movd %eax, %xmm1
+; CHECK-SSE2-O0-NEXT: pslld $16, %xmm1
+; CHECK-SSE2-O0-NEXT: movss %xmm3, {{[-0-9]+}}(%r{{[sb]}}p) # 4-byte Spill
+; CHECK-SSE2-O0-NEXT: movss %xmm2, {{[-0-9]+}}(%r{{[sb]}}p) # 4-byte Spill
; CHECK-SSE2-O0-NEXT: movss %xmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 4-byte Spill
-; CHECK-SSE2-O0-NEXT: movd %xmm0, %eax
-; CHECK-SSE2-O0-NEXT: shll $16, %eax
-; CHECK-SSE2-O0-NEXT: movd %eax, %xmm0
; CHECK-SSE2-O0-NEXT: callq __truncsfbf2 at PLT
; CHECK-SSE2-O0-NEXT: movaps %xmm0, %xmm1
; CHECK-SSE2-O0-NEXT: movss {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 4-byte Reload
@@ -1074,22 +1080,22 @@ define void @store_atomic_vec4_bfloat(ptr %x, <4 x bfloat> %v) nounwind {
; CHECK-SSE4-O0-LABEL: store_atomic_vec4_bfloat:
; CHECK-SSE4-O0: # %bb.0:
; CHECK-SSE4-O0-NEXT: subq $40, %rsp
+; CHECK-SSE4-O0-NEXT: movaps %xmm0, %xmm1
; CHECK-SSE4-O0-NEXT: movq %rdi, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Spill
-; CHECK-SSE4-O0-NEXT: pextrw $3, %xmm0, %eax
-; CHECK-SSE4-O0-NEXT: shll $16, %eax
-; CHECK-SSE4-O0-NEXT: movd %eax, %xmm1
-; CHECK-SSE4-O0-NEXT: movss %xmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 4-byte Spill
-; CHECK-SSE4-O0-NEXT: pextrw $2, %xmm0, %eax
-; CHECK-SSE4-O0-NEXT: shll $16, %eax
-; CHECK-SSE4-O0-NEXT: movd %eax, %xmm1
-; CHECK-SSE4-O0-NEXT: movss %xmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 4-byte Spill
-; CHECK-SSE4-O0-NEXT: pextrw $1, %xmm0, %eax
-; CHECK-SSE4-O0-NEXT: shll $16, %eax
+; CHECK-SSE4-O0-NEXT: xorps %xmm0, %xmm0
+; CHECK-SSE4-O0-NEXT: movaps %xmm1, %xmm3
+; CHECK-SSE4-O0-NEXT: pblendw {{.*#+}} xmm3 = xmm0[0],xmm3[1],xmm0[2],xmm3[3],xmm0[4],xmm3[5],xmm0[6],xmm3[7]
+; CHECK-SSE4-O0-NEXT: movaps %xmm1, %xmm0
+; CHECK-SSE4-O0-NEXT: pslld $16, %xmm0
+; CHECK-SSE4-O0-NEXT: pextrw $3, %xmm1, %eax
+; CHECK-SSE4-O0-NEXT: movd %eax, %xmm2
+; CHECK-SSE4-O0-NEXT: pslld $16, %xmm2
+; CHECK-SSE4-O0-NEXT: pextrw $2, %xmm1, %eax
; CHECK-SSE4-O0-NEXT: movd %eax, %xmm1
+; CHECK-SSE4-O0-NEXT: pslld $16, %xmm1
+; CHECK-SSE4-O0-NEXT: movss %xmm3, {{[-0-9]+}}(%r{{[sb]}}p) # 4-byte Spill
+; CHECK-SSE4-O0-NEXT: movss %xmm2, {{[-0-9]+}}(%r{{[sb]}}p) # 4-byte Spill
; CHECK-SSE4-O0-NEXT: movss %xmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 4-byte Spill
-; CHECK-SSE4-O0-NEXT: movd %xmm0, %eax
-; CHECK-SSE4-O0-NEXT: shll $16, %eax
-; CHECK-SSE4-O0-NEXT: movd %eax, %xmm0
; CHECK-SSE4-O0-NEXT: callq __truncsfbf2 at PLT
; CHECK-SSE4-O0-NEXT: movaps %xmm0, %xmm1
; CHECK-SSE4-O0-NEXT: movss {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 4-byte Reload
@@ -1113,47 +1119,79 @@ define void @store_atomic_vec4_bfloat(ptr %x, <4 x bfloat> %v) nounwind {
; CHECK-SSE4-O0-NEXT: addq $40, %rsp
; CHECK-SSE4-O0-NEXT: retq
;
-; CHECK-AVX-O0-LABEL: store_atomic_vec4_bfloat:
-; CHECK-AVX-O0: # %bb.0:
-; CHECK-AVX-O0-NEXT: subq $40, %rsp
-; CHECK-AVX-O0-NEXT: movq %rdi, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Spill
-; CHECK-AVX-O0-NEXT: vpextrw $3, %xmm0, %eax
-; CHECK-AVX-O0-NEXT: shll $16, %eax
-; CHECK-AVX-O0-NEXT: vmovd %eax, %xmm1
-; CHECK-AVX-O0-NEXT: vmovss %xmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 4-byte Spill
-; CHECK-AVX-O0-NEXT: vpextrw $2, %xmm0, %eax
-; CHECK-AVX-O0-NEXT: shll $16, %eax
-; CHECK-AVX-O0-NEXT: vmovd %eax, %xmm1
-; CHECK-AVX-O0-NEXT: vmovss %xmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 4-byte Spill
-; CHECK-AVX-O0-NEXT: vpextrw $1, %xmm0, %eax
-; CHECK-AVX-O0-NEXT: shll $16, %eax
-; CHECK-AVX-O0-NEXT: vmovd %eax, %xmm1
-; CHECK-AVX-O0-NEXT: vmovss %xmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 4-byte Spill
-; CHECK-AVX-O0-NEXT: vmovd %xmm0, %eax
-; CHECK-AVX-O0-NEXT: shll $16, %eax
-; CHECK-AVX-O0-NEXT: vmovd %eax, %xmm0
-; CHECK-AVX-O0-NEXT: callq __truncsfbf2 at PLT
-; CHECK-AVX-O0-NEXT: vmovaps %xmm0, %xmm1
-; CHECK-AVX-O0-NEXT: vmovss {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 4-byte Reload
-; CHECK-AVX-O0-NEXT: # xmm0 = mem[0],zero,zero,zero
-; CHECK-AVX-O0-NEXT: vpextrw $0, %xmm1, {{[0-9]+}}(%rsp)
-; CHECK-AVX-O0-NEXT: callq __truncsfbf2 at PLT
-; CHECK-AVX-O0-NEXT: vmovaps %xmm0, %xmm1
-; CHECK-AVX-O0-NEXT: vmovss {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 4-byte Reload
-; CHECK-AVX-O0-NEXT: # xmm0 = mem[0],zero,zero,zero
-; CHECK-AVX-O0-NEXT: vpextrw $0, %xmm1, {{[0-9]+}}(%rsp)
-; CHECK-AVX-O0-NEXT: callq __truncsfbf2 at PLT
-; CHECK-AVX-O0-NEXT: vmovaps %xmm0, %xmm1
-; CHECK-AVX-O0-NEXT: vmovss {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 4-byte Reload
-; CHECK-AVX-O0-NEXT: # xmm0 = mem[0],zero,zero,zero
-; CHECK-AVX-O0-NEXT: vpextrw $0, %xmm1, {{[0-9]+}}(%rsp)
-; CHECK-AVX-O0-NEXT: callq __truncsfbf2 at PLT
-; CHECK-AVX-O0-NEXT: movq {{[-0-9]+}}(%r{{[sb]}}p), %rdi # 8-byte Reload
-; CHECK-AVX-O0-NEXT: vpextrw $0, %xmm0, {{[0-9]+}}(%rsp)
-; CHECK-AVX-O0-NEXT: movq {{[0-9]+}}(%rsp), %rax
-; CHECK-AVX-O0-NEXT: movq %rax, (%rdi)
-; CHECK-AVX-O0-NEXT: addq $40, %rsp
-; CHECK-AVX-O0-NEXT: retq
+; CHECK-AVX2-O0-LABEL: store_atomic_vec4_bfloat:
+; CHECK-AVX2-O0: # %bb.0:
+; CHECK-AVX2-O0-NEXT: subq $40, %rsp
+; CHECK-AVX2-O0-NEXT: movq %rdi, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Spill
+; CHECK-AVX2-O0-NEXT: vmovdqa {{.*#+}} xmm1 = [128,128,6,7,u,u,u,u,u,u,u,u,u,u,u,u]
+; CHECK-AVX2-O0-NEXT: vpshufb %xmm1, %xmm0, %xmm3
+; CHECK-AVX2-O0-NEXT: vmovdqa {{.*#+}} xmm1 = [128,128,4,5,u,u,u,u,u,u,u,u,u,u,u,u]
+; CHECK-AVX2-O0-NEXT: vpshufb %xmm1, %xmm0, %xmm2
+; CHECK-AVX2-O0-NEXT: vxorps %xmm1, %xmm1, %xmm1
+; CHECK-AVX2-O0-NEXT: vpblendw {{.*#+}} xmm1 = xmm1[0],xmm0[1],xmm1[2],xmm0[3],xmm1[4],xmm0[5],xmm1[6],xmm0[7]
+; CHECK-AVX2-O0-NEXT: vpslld $16, %xmm0, %xmm0
+; CHECK-AVX2-O0-NEXT: vmovss %xmm3, {{[-0-9]+}}(%r{{[sb]}}p) # 4-byte Spill
+; CHECK-AVX2-O0-NEXT: vmovss %xmm2, {{[-0-9]+}}(%r{{[sb]}}p) # 4-byte Spill
+; CHECK-AVX2-O0-NEXT: vmovss %xmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 4-byte Spill
+; CHECK-AVX2-O0-NEXT: callq __truncsfbf2 at PLT
+; CHECK-AVX2-O0-NEXT: vmovaps %xmm0, %xmm1
+; CHECK-AVX2-O0-NEXT: vmovss {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 4-byte Reload
+; CHECK-AVX2-O0-NEXT: # xmm0 = mem[0],zero,zero,zero
+; CHECK-AVX2-O0-NEXT: vpextrw $0, %xmm1, {{[0-9]+}}(%rsp)
+; CHECK-AVX2-O0-NEXT: callq __truncsfbf2 at PLT
+; CHECK-AVX2-O0-NEXT: vmovaps %xmm0, %xmm1
+; CHECK-AVX2-O0-NEXT: vmovss {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 4-byte Reload
+; CHECK-AVX2-O0-NEXT: # xmm0 = mem[0],zero,zero,zero
+; CHECK-AVX2-O0-NEXT: vpextrw $0, %xmm1, {{[0-9]+}}(%rsp)
+; CHECK-AVX2-O0-NEXT: callq __truncsfbf2 at PLT
+; CHECK-AVX2-O0-NEXT: vmovaps %xmm0, %xmm1
+; CHECK-AVX2-O0-NEXT: vmovss {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 4-byte Reload
+; CHECK-AVX2-O0-NEXT: # xmm0 = mem[0],zero,zero,zero
+; CHECK-AVX2-O0-NEXT: vpextrw $0, %xmm1, {{[0-9]+}}(%rsp)
+; CHECK-AVX2-O0-NEXT: callq __truncsfbf2 at PLT
+; CHECK-AVX2-O0-NEXT: movq {{[-0-9]+}}(%r{{[sb]}}p), %rdi # 8-byte Reload
+; CHECK-AVX2-O0-NEXT: vpextrw $0, %xmm0, {{[0-9]+}}(%rsp)
+; CHECK-AVX2-O0-NEXT: movq {{[0-9]+}}(%rsp), %rax
+; CHECK-AVX2-O0-NEXT: movq %rax, (%rdi)
+; CHECK-AVX2-O0-NEXT: addq $40, %rsp
+; CHECK-AVX2-O0-NEXT: retq
+;
+; CHECK-AVX512-O0-LABEL: store_atomic_vec4_bfloat:
+; CHECK-AVX512-O0: # %bb.0:
+; CHECK-AVX512-O0-NEXT: subq $40, %rsp
+; CHECK-AVX512-O0-NEXT: movq %rdi, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Spill
+; CHECK-AVX512-O0-NEXT: vmovdqa {{.*#+}} xmm1 = [128,128,6,7,u,u,u,u,u,u,u,u,u,u,u,u]
+; CHECK-AVX512-O0-NEXT: vpshufb %xmm1, %xmm0, %xmm3
+; CHECK-AVX512-O0-NEXT: vmovdqa {{.*#+}} xmm1 = [128,128,4,5,u,u,u,u,u,u,u,u,u,u,u,u]
+; CHECK-AVX512-O0-NEXT: vpshufb %xmm1, %xmm0, %xmm2
+; CHECK-AVX512-O0-NEXT: vpxor %xmm1, %xmm1, %xmm1
+; CHECK-AVX512-O0-NEXT: vpblendw {{.*#+}} xmm1 = xmm1[0],xmm0[1],xmm1[2],xmm0[3],xmm1[4],xmm0[5],xmm1[6],xmm0[7]
+; CHECK-AVX512-O0-NEXT: vpslld $16, %xmm0, %xmm0
+; CHECK-AVX512-O0-NEXT: vmovss %xmm3, {{[-0-9]+}}(%r{{[sb]}}p) # 4-byte Spill
+; CHECK-AVX512-O0-NEXT: vmovss %xmm2, {{[-0-9]+}}(%r{{[sb]}}p) # 4-byte Spill
+; CHECK-AVX512-O0-NEXT: vmovss %xmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 4-byte Spill
+; CHECK-AVX512-O0-NEXT: callq __truncsfbf2 at PLT
+; CHECK-AVX512-O0-NEXT: vmovaps %xmm0, %xmm1
+; CHECK-AVX512-O0-NEXT: vmovss {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 4-byte Reload
+; CHECK-AVX512-O0-NEXT: # xmm0 = mem[0],zero,zero,zero
+; CHECK-AVX512-O0-NEXT: vpextrw $0, %xmm1, {{[0-9]+}}(%rsp)
+; CHECK-AVX512-O0-NEXT: callq __truncsfbf2 at PLT
+; CHECK-AVX512-O0-NEXT: vmovaps %xmm0, %xmm1
+; CHECK-AVX512-O0-NEXT: vmovss {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 4-byte Reload
+; CHECK-AVX512-O0-NEXT: # xmm0 = mem[0],zero,zero,zero
+; CHECK-AVX512-O0-NEXT: vpextrw $0, %xmm1, {{[0-9]+}}(%rsp)
+; CHECK-AVX512-O0-NEXT: callq __truncsfbf2 at PLT
+; CHECK-AVX512-O0-NEXT: vmovaps %xmm0, %xmm1
+; CHECK-AVX512-O0-NEXT: vmovss {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 4-byte Reload
+; CHECK-AVX512-O0-NEXT: # xmm0 = mem[0],zero,zero,zero
+; CHECK-AVX512-O0-NEXT: vpextrw $0, %xmm1, {{[0-9]+}}(%rsp)
+; CHECK-AVX512-O0-NEXT: callq __truncsfbf2 at PLT
+; CHECK-AVX512-O0-NEXT: movq {{[-0-9]+}}(%r{{[sb]}}p), %rdi # 8-byte Reload
+; CHECK-AVX512-O0-NEXT: vpextrw $0, %xmm0, {{[0-9]+}}(%rsp)
+; CHECK-AVX512-O0-NEXT: movq {{[0-9]+}}(%rsp), %rax
+; CHECK-AVX512-O0-NEXT: movq %rax, (%rdi)
+; CHECK-AVX512-O0-NEXT: addq $40, %rsp
+; CHECK-AVX512-O0-NEXT: retq
store atomic <4 x bfloat> %v, ptr %x release, align 8
ret void
}
@@ -2658,70 +2696,68 @@ define void @store_atomic_vec16_bfloat_unaligned(ptr %x, <16 x bfloat> %v) nounw
; CHECK-SSE2-O0-LABEL: store_atomic_vec16_bfloat_unaligned:
; CHECK-SSE2-O0: # %bb.0:
; CHECK-SSE2-O0-NEXT: subq $120, %rsp
+; CHECK-SSE2-O0-NEXT: movaps %xmm1, %xmm2
+; CHECK-SSE2-O0-NEXT: movaps %xmm0, %xmm1
; CHECK-SSE2-O0-NEXT: movq %rdi, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Spill
+; CHECK-SSE2-O0-NEXT: xorps %xmm13, %xmm13
+; CHECK-SSE2-O0-NEXT: movaps %xmm13, %xmm15
+; CHECK-SSE2-O0-NEXT: punpckhwd {{.*#+}} xmm15 = xmm15[4],xmm2[4],xmm15[5],xmm2[5],xmm15[6],xmm2[6],xmm15[7],xmm2[7]
+; CHECK-SSE2-O0-NEXT: punpckhwd {{.*#+}} xmm13 = xmm13[4],xmm1[4],xmm13[5],xmm1[5],xmm13[6],xmm1[6],xmm13[7],xmm1[7]
+; CHECK-SSE2-O0-NEXT: movaps %xmm2, %xmm14
+; CHECK-SSE2-O0-NEXT: pslld $16, %xmm14
+; CHECK-SSE2-O0-NEXT: movaps %xmm1, %xmm0
+; CHECK-SSE2-O0-NEXT: pslld $16, %xmm0
+; CHECK-SSE2-O0-NEXT: pextrw $7, %xmm2, %eax
+; CHECK-SSE2-O0-NEXT: movd %eax, %xmm12
+; CHECK-SSE2-O0-NEXT: pslld $16, %xmm12
+; CHECK-SSE2-O0-NEXT: pextrw $6, %xmm2, %eax
+; CHECK-SSE2-O0-NEXT: movd %eax, %xmm11
+; CHECK-SSE2-O0-NEXT: pslld $16, %xmm11
+; CHECK-SSE2-O0-NEXT: pextrw $5, %xmm2, %eax
+; CHECK-SSE2-O0-NEXT: movd %eax, %xmm10
+; CHECK-SSE2-O0-NEXT: pslld $16, %xmm10
+; CHECK-SSE2-O0-NEXT: pextrw $3, %xmm2, %eax
+; CHECK-SSE2-O0-NEXT: movd %eax, %xmm9
+; CHECK-SSE2-O0-NEXT: pslld $16, %xmm9
+; CHECK-SSE2-O0-NEXT: pextrw $2, %xmm2, %eax
+; CHECK-SSE2-O0-NEXT: movd %eax, %xmm8
+; CHECK-SSE2-O0-NEXT: pslld $16, %xmm8
+; CHECK-SSE2-O0-NEXT: pextrw $1, %xmm2, %eax
+; CHECK-SSE2-O0-NEXT: movd %eax, %xmm7
+; CHECK-SSE2-O0-NEXT: pslld $16, %xmm7
; CHECK-SSE2-O0-NEXT: pextrw $7, %xmm1, %eax
-; CHECK-SSE2-O0-NEXT: shll $16, %eax
-; CHECK-SSE2-O0-NEXT: movd %eax, %xmm2
-; CHECK-SSE2-O0-NEXT: movss %xmm2, {{[-0-9]+}}(%r{{[sb]}}p) # 4-byte Spill
+; CHECK-SSE2-O0-NEXT: movd %eax, %xmm6
+; CHECK-SSE2-O0-NEXT: pslld $16, %xmm6
; CHECK-SSE2-O0-NEXT: pextrw $6, %xmm1, %eax
-; CHECK-SSE2-O0-NEXT: shll $16, %eax
-; CHECK-SSE2-O0-NEXT: movd %eax, %xmm2
-; CHECK-SSE2-O0-NEXT: movss %xmm2, {{[-0-9]+}}(%r{{[sb]}}p) # 4-byte Spill
+; CHECK-SSE2-O0-NEXT: movd %eax, %xmm5
+; CHECK-SSE2-O0-NEXT: pslld $16, %xmm5
; CHECK-SSE2-O0-NEXT: pextrw $5, %xmm1, %eax
-; CHECK-SSE2-O0-NEXT: shll $16, %eax
-; CHECK-SSE2-O0-NEXT: movd %eax, %xmm2
-; CHECK-SSE2-O0-NEXT: movss %xmm2, {{[-0-9]+}}(%r{{[sb]}}p) # 4-byte Spill
-; CHECK-SSE2-O0-NEXT: pextrw $4, %xmm1, %eax
-; CHECK-SSE2-O0-NEXT: shll $16, %eax
-; CHECK-SSE2-O0-NEXT: movd %eax, %xmm2
-; CHECK-SSE2-O0-NEXT: movss %xmm2, {{[-0-9]+}}(%r{{[sb]}}p) # 4-byte Spill
+; CHECK-SSE2-O0-NEXT: movd %eax, %xmm4
+; CHECK-SSE2-O0-NEXT: pslld $16, %xmm4
; CHECK-SSE2-O0-NEXT: pextrw $3, %xmm1, %eax
-; CHECK-SSE2-O0-NEXT: shll $16, %eax
-; CHECK-SSE2-O0-NEXT: movd %eax, %xmm2
-; CHECK-SSE2-O0-NEXT: movss %xmm2, {{[-0-9]+}}(%r{{[sb]}}p) # 4-byte Spill
+; CHECK-SSE2-O0-NEXT: movd %eax, %xmm3
+; CHECK-SSE2-O0-NEXT: pslld $16, %xmm3
; CHECK-SSE2-O0-NEXT: pextrw $2, %xmm1, %eax
-; CHECK-SSE2-O0-NEXT: shll $16, %eax
; CHECK-SSE2-O0-NEXT: movd %eax, %xmm2
-; CHECK-SSE2-O0-NEXT: movss %xmm2, {{[-0-9]+}}(%r{{[sb]}}p) # 4-byte Spill
+; CHECK-SSE2-O0-NEXT: pslld $16, %xmm2
; CHECK-SSE2-O0-NEXT: pextrw $1, %xmm1, %eax
-; CHECK-SSE2-O0-NEXT: shll $16, %eax
-; CHECK-SSE2-O0-NEXT: movd %eax, %xmm2
-; CHECK-SSE2-O0-NEXT: movss %xmm2, {{[-0-9]+}}(%r{{[sb]}}p) # 4-byte Spill
-; CHECK-SSE2-O0-NEXT: movd %xmm1, %eax
-; CHECK-SSE2-O0-NEXT: shll $16, %eax
-; CHECK-SSE2-O0-NEXT: movd %eax, %xmm1
-; CHECK-SSE2-O0-NEXT: movss %xmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 4-byte Spill
-; CHECK-SSE2-O0-NEXT: pextrw $7, %xmm0, %eax
-; CHECK-SSE2-O0-NEXT: shll $16, %eax
-; CHECK-SSE2-O0-NEXT: movd %eax, %xmm1
-; CHECK-SSE2-O0-NEXT: movss %xmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 4-byte Spill
-; CHECK-SSE2-O0-NEXT: pextrw $6, %xmm0, %eax
-; CHECK-SSE2-O0-NEXT: shll $16, %eax
-; CHECK-SSE2-O0-NEXT: movd %eax, %xmm1
-; CHECK-SSE2-O0-NEXT: movss %xmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 4-byte Spill
-; CHECK-SSE2-O0-NEXT: pextrw $5, %xmm0, %eax
-; CHECK-SSE2-O0-NEXT: shll $16, %eax
-; CHECK-SSE2-O0-NEXT: movd %eax, %xmm1
-; CHECK-SSE2-O0-NEXT: movss %xmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 4-byte Spill
-; CHECK-SSE2-O0-NEXT: pextrw $4, %xmm0, %eax
-; CHECK-SSE2-O0-NEXT: shll $16, %eax
-; CHECK-SSE2-O0-NEXT: movd %eax, %xmm1
-; CHECK-SSE2-O0-NEXT: movss %xmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 4-byte Spill
-; CHECK-SSE2-O0-NEXT: pextrw $3, %xmm0, %eax
-; CHECK-SSE2-O0-NEXT: shll $16, %eax
-; CHECK-SSE2-O0-NEXT: movd %eax, %xmm1
-; CHECK-SSE2-O0-NEXT: movss %xmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 4-byte Spill
-; CHECK-SSE2-O0-NEXT: pextrw $2, %xmm0, %eax
-; CHECK-SSE2-O0-NEXT: shll $16, %eax
-; CHECK-SSE2-O0-NEXT: movd %eax, %xmm1
-; CHECK-SSE2-O0-NEXT: movss %xmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 4-byte Spill
-; CHECK-SSE2-O0-NEXT: pextrw $1, %xmm0, %eax
-; CHECK-SSE2-O0-NEXT: shll $16, %eax
; CHECK-SSE2-O0-NEXT: movd %eax, %xmm1
+; CHECK-SSE2-O0-NEXT: pslld $16, %xmm1
+; CHECK-SSE2-O0-NEXT: movss %xmm15, {{[-0-9]+}}(%r{{[sb]}}p) # 4-byte Spill
+; CHECK-SSE2-O0-NEXT: movss %xmm14, {{[-0-9]+}}(%r{{[sb]}}p) # 4-byte Spill
+; CHECK-SSE2-O0-NEXT: movss %xmm13, {{[-0-9]+}}(%r{{[sb]}}p) # 4-byte Spill
+; CHECK-SSE2-O0-NEXT: movss %xmm12, {{[-0-9]+}}(%r{{[sb]}}p) # 4-byte Spill
+; CHECK-SSE2-O0-NEXT: movss %xmm11, {{[-0-9]+}}(%r{{[sb]}}p) # 4-byte Spill
+; CHECK-SSE2-O0-NEXT: movss %xmm10, {{[-0-9]+}}(%r{{[sb]}}p) # 4-byte Spill
+; CHECK-SSE2-O0-NEXT: movss %xmm9, {{[-0-9]+}}(%r{{[sb]}}p) # 4-byte Spill
+; CHECK-SSE2-O0-NEXT: movss %xmm8, {{[-0-9]+}}(%r{{[sb]}}p) # 4-byte Spill
+; CHECK-SSE2-O0-NEXT: movss %xmm7, {{[-0-9]+}}(%r{{[sb]}}p) # 4-byte Spill
+; CHECK-SSE2-O0-NEXT: movss %xmm6, {{[-0-9]+}}(%r{{[sb]}}p) # 4-byte Spill
+; CHECK-SSE2-O0-NEXT: movss %xmm5, {{[-0-9]+}}(%r{{[sb]}}p) # 4-byte Spill
+; CHECK-SSE2-O0-NEXT: movss %xmm4, {{[-0-9]+}}(%r{{[sb]}}p) # 4-byte Spill
+; CHECK-SSE2-O0-NEXT: movss %xmm3, {{[-0-9]+}}(%r{{[sb]}}p) # 4-byte Spill
+; CHECK-SSE2-O0-NEXT: movss %xmm2, {{[-0-9]+}}(%r{{[sb]}}p) # 4-byte Spill
; CHECK-SSE2-O0-NEXT: movss %xmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 4-byte Spill
-; CHECK-SSE2-O0-NEXT: movd %xmm0, %eax
-; CHECK-SSE2-O0-NEXT: shll $16, %eax
-; CHECK-SSE2-O0-NEXT: movd %eax, %xmm0
; CHECK-SSE2-O0-NEXT: callq __truncsfbf2 at PLT
; CHECK-SSE2-O0-NEXT: movaps %xmm0, %xmm1
; CHECK-SSE2-O0-NEXT: movss {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 4-byte Reload
@@ -2842,70 +2878,67 @@ define void @store_atomic_vec16_bfloat_unaligned(ptr %x, <16 x bfloat> %v) nounw
; CHECK-SSE4-O0-LABEL: store_atomic_vec16_bfloat_unaligned:
; CHECK-SSE4-O0: # %bb.0:
; CHECK-SSE4-O0-NEXT: subq $120, %rsp
+; CHECK-SSE4-O0-NEXT: movaps %xmm1, %xmm2
+; CHECK-SSE4-O0-NEXT: movaps %xmm0, %xmm1
; CHECK-SSE4-O0-NEXT: movq %rdi, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Spill
+; CHECK-SSE4-O0-NEXT: xorps %xmm0, %xmm0
+; CHECK-SSE4-O0-NEXT: movaps %xmm0, %xmm15
+; CHECK-SSE4-O0-NEXT: punpckhwd {{.*#+}} xmm15 = xmm15[4],xmm2[4],xmm15[5],xmm2[5],xmm15[6],xmm2[6],xmm15[7],xmm2[7]
+; CHECK-SSE4-O0-NEXT: movaps %xmm2, %xmm14
+; CHECK-SSE4-O0-NEXT: pblendw {{.*#+}} xmm14 = xmm0[0],xmm14[1],xmm0[2],xmm14[3],xmm0[4],xmm14[5],xmm0[6],xmm14[7]
+; CHECK-SSE4-O0-NEXT: movaps %xmm0, %xmm12
+; CHECK-SSE4-O0-NEXT: punpckhwd {{.*#+}} xmm12 = xmm12[4],xmm1[4],xmm12[5],xmm1[5],xmm12[6],xmm1[6],xmm12[7],xmm1[7]
+; CHECK-SSE4-O0-NEXT: movaps %xmm1, %xmm11
+; CHECK-SSE4-O0-NEXT: pblendw {{.*#+}} xmm11 = xmm0[0],xmm11[1],xmm0[2],xmm11[3],xmm0[4],xmm11[5],xmm0[6],xmm11[7]
+; CHECK-SSE4-O0-NEXT: movaps %xmm2, %xmm13
+; CHECK-SSE4-O0-NEXT: pslld $16, %xmm13
+; CHECK-SSE4-O0-NEXT: movaps %xmm1, %xmm0
+; CHECK-SSE4-O0-NEXT: pslld $16, %xmm0
+; CHECK-SSE4-O0-NEXT: pextrw $7, %xmm2, %eax
+; CHECK-SSE4-O0-NEXT: movd %eax, %xmm10
+; CHECK-SSE4-O0-NEXT: pslld $16, %xmm10
+; CHECK-SSE4-O0-NEXT: pextrw $6, %xmm2, %eax
+; CHECK-SSE4-O0-NEXT: movd %eax, %xmm9
+; CHECK-SSE4-O0-NEXT: pslld $16, %xmm9
+; CHECK-SSE4-O0-NEXT: pextrw $5, %xmm2, %eax
+; CHECK-SSE4-O0-NEXT: movd %eax, %xmm8
+; CHECK-SSE4-O0-NEXT: pslld $16, %xmm8
+; CHECK-SSE4-O0-NEXT: pextrw $3, %xmm2, %eax
+; CHECK-SSE4-O0-NEXT: movd %eax, %xmm7
+; CHECK-SSE4-O0-NEXT: pslld $16, %xmm7
+; CHECK-SSE4-O0-NEXT: pextrw $2, %xmm2, %eax
+; CHECK-SSE4-O0-NEXT: movd %eax, %xmm6
+; CHECK-SSE4-O0-NEXT: pslld $16, %xmm6
; CHECK-SSE4-O0-NEXT: pextrw $7, %xmm1, %eax
-; CHECK-SSE4-O0-NEXT: shll $16, %eax
-; CHECK-SSE4-O0-NEXT: movd %eax, %xmm2
-; CHECK-SSE4-O0-NEXT: movss %xmm2, {{[-0-9]+}}(%r{{[sb]}}p) # 4-byte Spill
+; CHECK-SSE4-O0-NEXT: movd %eax, %xmm5
+; CHECK-SSE4-O0-NEXT: pslld $16, %xmm5
; CHECK-SSE4-O0-NEXT: pextrw $6, %xmm1, %eax
-; CHECK-SSE4-O0-NEXT: shll $16, %eax
-; CHECK-SSE4-O0-NEXT: movd %eax, %xmm2
-; CHECK-SSE4-O0-NEXT: movss %xmm2, {{[-0-9]+}}(%r{{[sb]}}p) # 4-byte Spill
+; CHECK-SSE4-O0-NEXT: movd %eax, %xmm4
+; CHECK-SSE4-O0-NEXT: pslld $16, %xmm4
; CHECK-SSE4-O0-NEXT: pextrw $5, %xmm1, %eax
-; CHECK-SSE4-O0-NEXT: shll $16, %eax
-; CHECK-SSE4-O0-NEXT: movd %eax, %xmm2
-; CHECK-SSE4-O0-NEXT: movss %xmm2, {{[-0-9]+}}(%r{{[sb]}}p) # 4-byte Spill
-; CHECK-SSE4-O0-NEXT: pextrw $4, %xmm1, %eax
-; CHECK-SSE4-O0-NEXT: shll $16, %eax
-; CHECK-SSE4-O0-NEXT: movd %eax, %xmm2
-; CHECK-SSE4-O0-NEXT: movss %xmm2, {{[-0-9]+}}(%r{{[sb]}}p) # 4-byte Spill
+; CHECK-SSE4-O0-NEXT: movd %eax, %xmm3
+; CHECK-SSE4-O0-NEXT: pslld $16, %xmm3
; CHECK-SSE4-O0-NEXT: pextrw $3, %xmm1, %eax
-; CHECK-SSE4-O0-NEXT: shll $16, %eax
; CHECK-SSE4-O0-NEXT: movd %eax, %xmm2
-; CHECK-SSE4-O0-NEXT: movss %xmm2, {{[-0-9]+}}(%r{{[sb]}}p) # 4-byte Spill
+; CHECK-SSE4-O0-NEXT: pslld $16, %xmm2
; CHECK-SSE4-O0-NEXT: pextrw $2, %xmm1, %eax
-; CHECK-SSE4-O0-NEXT: shll $16, %eax
-; CHECK-SSE4-O0-NEXT: movd %eax, %xmm2
-; CHECK-SSE4-O0-NEXT: movss %xmm2, {{[-0-9]+}}(%r{{[sb]}}p) # 4-byte Spill
-; CHECK-SSE4-O0-NEXT: pextrw $1, %xmm1, %eax
-; CHECK-SSE4-O0-NEXT: shll $16, %eax
-; CHECK-SSE4-O0-NEXT: movd %eax, %xmm2
-; CHECK-SSE4-O0-NEXT: movss %xmm2, {{[-0-9]+}}(%r{{[sb]}}p) # 4-byte Spill
-; CHECK-SSE4-O0-NEXT: movd %xmm1, %eax
-; CHECK-SSE4-O0-NEXT: shll $16, %eax
-; CHECK-SSE4-O0-NEXT: movd %eax, %xmm1
-; CHECK-SSE4-O0-NEXT: movss %xmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 4-byte Spill
-; CHECK-SSE4-O0-NEXT: pextrw $7, %xmm0, %eax
-; CHECK-SSE4-O0-NEXT: shll $16, %eax
-; CHECK-SSE4-O0-NEXT: movd %eax, %xmm1
-; CHECK-SSE4-O0-NEXT: movss %xmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 4-byte Spill
-; CHECK-SSE4-O0-NEXT: pextrw $6, %xmm0, %eax
-; CHECK-SSE4-O0-NEXT: shll $16, %eax
-; CHECK-SSE4-O0-NEXT: movd %eax, %xmm1
-; CHECK-SSE4-O0-NEXT: movss %xmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 4-byte Spill
-; CHECK-SSE4-O0-NEXT: pextrw $5, %xmm0, %eax
-; CHECK-SSE4-O0-NEXT: shll $16, %eax
-; CHECK-SSE4-O0-NEXT: movd %eax, %xmm1
-; CHECK-SSE4-O0-NEXT: movss %xmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 4-byte Spill
-; CHECK-SSE4-O0-NEXT: pextrw $4, %xmm0, %eax
-; CHECK-SSE4-O0-NEXT: shll $16, %eax
-; CHECK-SSE4-O0-NEXT: movd %eax, %xmm1
-; CHECK-SSE4-O0-NEXT: movss %xmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 4-byte Spill
-; CHECK-SSE4-O0-NEXT: pextrw $3, %xmm0, %eax
-; CHECK-SSE4-O0-NEXT: shll $16, %eax
-; CHECK-SSE4-O0-NEXT: movd %eax, %xmm1
-; CHECK-SSE4-O0-NEXT: movss %xmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 4-byte Spill
-; CHECK-SSE4-O0-NEXT: pextrw $2, %xmm0, %eax
-; CHECK-SSE4-O0-NEXT: shll $16, %eax
-; CHECK-SSE4-O0-NEXT: movd %eax, %xmm1
-; CHECK-SSE4-O0-NEXT: movss %xmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 4-byte Spill
-; CHECK-SSE4-O0-NEXT: pextrw $1, %xmm0, %eax
-; CHECK-SSE4-O0-NEXT: shll $16, %eax
; CHECK-SSE4-O0-NEXT: movd %eax, %xmm1
+; CHECK-SSE4-O0-NEXT: pslld $16, %xmm1
+; CHECK-SSE4-O0-NEXT: movss %xmm15, {{[-0-9]+}}(%r{{[sb]}}p) # 4-byte Spill
+; CHECK-SSE4-O0-NEXT: movss %xmm14, {{[-0-9]+}}(%r{{[sb]}}p) # 4-byte Spill
+; CHECK-SSE4-O0-NEXT: movss %xmm13, {{[-0-9]+}}(%r{{[sb]}}p) # 4-byte Spill
+; CHECK-SSE4-O0-NEXT: movss %xmm12, {{[-0-9]+}}(%r{{[sb]}}p) # 4-byte Spill
+; CHECK-SSE4-O0-NEXT: movss %xmm11, {{[-0-9]+}}(%r{{[sb]}}p) # 4-byte Spill
+; CHECK-SSE4-O0-NEXT: movss %xmm10, {{[-0-9]+}}(%r{{[sb]}}p) # 4-byte Spill
+; CHECK-SSE4-O0-NEXT: movss %xmm9, {{[-0-9]+}}(%r{{[sb]}}p) # 4-byte Spill
+; CHECK-SSE4-O0-NEXT: movss %xmm8, {{[-0-9]+}}(%r{{[sb]}}p) # 4-byte Spill
+; CHECK-SSE4-O0-NEXT: movss %xmm7, {{[-0-9]+}}(%r{{[sb]}}p) # 4-byte Spill
+; CHECK-SSE4-O0-NEXT: movss %xmm6, {{[-0-9]+}}(%r{{[sb]}}p) # 4-byte Spill
+; CHECK-SSE4-O0-NEXT: movss %xmm5, {{[-0-9]+}}(%r{{[sb]}}p) # 4-byte Spill
+; CHECK-SSE4-O0-NEXT: movss %xmm4, {{[-0-9]+}}(%r{{[sb]}}p) # 4-byte Spill
+; CHECK-SSE4-O0-NEXT: movss %xmm3, {{[-0-9]+}}(%r{{[sb]}}p) # 4-byte Spill
+; CHECK-SSE4-O0-NEXT: movss %xmm2, {{[-0-9]+}}(%r{{[sb]}}p) # 4-byte Spill
; CHECK-SSE4-O0-NEXT: movss %xmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 4-byte Spill
-; CHECK-SSE4-O0-NEXT: movd %xmm0, %eax
-; CHECK-SSE4-O0-NEXT: shll $16, %eax
-; CHECK-SSE4-O0-NEXT: movd %eax, %xmm0
; CHECK-SSE4-O0-NEXT: callq __truncsfbf2 at PLT
; CHECK-SSE4-O0-NEXT: movaps %xmm0, %xmm1
; CHECK-SSE4-O0-NEXT: movss {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 4-byte Reload
@@ -2995,71 +3028,45 @@ define void @store_atomic_vec16_bfloat_unaligned(ptr %x, <16 x bfloat> %v) nounw
; CHECK-AVX2-O0: # %bb.0:
; CHECK-AVX2-O0-NEXT: subq $120, %rsp
; CHECK-AVX2-O0-NEXT: movq %rdi, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Spill
-; CHECK-AVX2-O0-NEXT: vextracti128 $1, %ymm0, %xmm1
-; CHECK-AVX2-O0-NEXT: vpextrw $7, %xmm1, %eax
-; CHECK-AVX2-O0-NEXT: shll $16, %eax
-; CHECK-AVX2-O0-NEXT: vmovd %eax, %xmm2
-; CHECK-AVX2-O0-NEXT: vmovss %xmm2, {{[-0-9]+}}(%r{{[sb]}}p) # 4-byte Spill
-; CHECK-AVX2-O0-NEXT: vpextrw $6, %xmm1, %eax
-; CHECK-AVX2-O0-NEXT: shll $16, %eax
-; CHECK-AVX2-O0-NEXT: vmovd %eax, %xmm2
-; CHECK-AVX2-O0-NEXT: vmovss %xmm2, {{[-0-9]+}}(%r{{[sb]}}p) # 4-byte Spill
-; CHECK-AVX2-O0-NEXT: vpextrw $5, %xmm1, %eax
-; CHECK-AVX2-O0-NEXT: shll $16, %eax
-; CHECK-AVX2-O0-NEXT: vmovd %eax, %xmm2
-; CHECK-AVX2-O0-NEXT: vmovss %xmm2, {{[-0-9]+}}(%r{{[sb]}}p) # 4-byte Spill
-; CHECK-AVX2-O0-NEXT: vpextrw $4, %xmm1, %eax
-; CHECK-AVX2-O0-NEXT: shll $16, %eax
-; CHECK-AVX2-O0-NEXT: vmovd %eax, %xmm2
-; CHECK-AVX2-O0-NEXT: vmovss %xmm2, {{[-0-9]+}}(%r{{[sb]}}p) # 4-byte Spill
-; CHECK-AVX2-O0-NEXT: vpextrw $3, %xmm1, %eax
-; CHECK-AVX2-O0-NEXT: shll $16, %eax
-; CHECK-AVX2-O0-NEXT: vmovd %eax, %xmm2
-; CHECK-AVX2-O0-NEXT: vmovss %xmm2, {{[-0-9]+}}(%r{{[sb]}}p) # 4-byte Spill
-; CHECK-AVX2-O0-NEXT: vpextrw $2, %xmm1, %eax
-; CHECK-AVX2-O0-NEXT: shll $16, %eax
-; CHECK-AVX2-O0-NEXT: vmovd %eax, %xmm2
-; CHECK-AVX2-O0-NEXT: vmovss %xmm2, {{[-0-9]+}}(%r{{[sb]}}p) # 4-byte Spill
-; CHECK-AVX2-O0-NEXT: vpextrw $1, %xmm1, %eax
-; CHECK-AVX2-O0-NEXT: shll $16, %eax
-; CHECK-AVX2-O0-NEXT: vmovd %eax, %xmm2
-; CHECK-AVX2-O0-NEXT: vmovss %xmm2, {{[-0-9]+}}(%r{{[sb]}}p) # 4-byte Spill
-; CHECK-AVX2-O0-NEXT: vmovd %xmm1, %eax
-; CHECK-AVX2-O0-NEXT: shll $16, %eax
-; CHECK-AVX2-O0-NEXT: vmovd %eax, %xmm1
-; CHECK-AVX2-O0-NEXT: vmovss %xmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 4-byte Spill
+; CHECK-AVX2-O0-NEXT: vextracti128 $1, %ymm0, %xmm8
+; CHECK-AVX2-O0-NEXT: vmovdqa {{.*#+}} xmm6 = [128,128,14,15,u,u,u,u,u,u,u,u,u,u,u,u]
+; CHECK-AVX2-O0-NEXT: vpshufb %xmm6, %xmm8, %xmm15
+; CHECK-AVX2-O0-NEXT: vmovdqa {{.*#+}} xmm5 = [128,128,12,13,u,u,u,u,u,u,u,u,u,u,u,u]
+; CHECK-AVX2-O0-NEXT: vpshufb %xmm5, %xmm8, %xmm14
+; CHECK-AVX2-O0-NEXT: vmovdqa {{.*#+}} xmm4 = [128,128,10,11,u,u,u,u,u,u,u,u,u,u,u,u]
+; CHECK-AVX2-O0-NEXT: vpshufb %xmm4, %xmm8, %xmm13
+; CHECK-AVX2-O0-NEXT: vxorps %xmm1, %xmm1, %xmm1
+; CHECK-AVX2-O0-NEXT: vpunpckhwd {{.*#+}} xmm12 = xmm1[4],xmm8[4],xmm1[5],xmm8[5],xmm1[6],xmm8[6],xmm1[7],xmm8[7]
+; CHECK-AVX2-O0-NEXT: vmovdqa {{.*#+}} xmm3 = [128,128,6,7,u,u,u,u,u,u,u,u,u,u,u,u]
+; CHECK-AVX2-O0-NEXT: vpshufb %xmm3, %xmm8, %xmm11
+; CHECK-AVX2-O0-NEXT: vmovdqa {{.*#+}} xmm2 = [128,128,4,5,u,u,u,u,u,u,u,u,u,u,u,u]
+; CHECK-AVX2-O0-NEXT: vpshufb %xmm2, %xmm8, %xmm10
+; CHECK-AVX2-O0-NEXT: vpblendw {{.*#+}} xmm9 = xmm1[0],xmm8[1],xmm1[2],xmm8[3],xmm1[4],xmm8[5],xmm1[6],xmm8[7]
; CHECK-AVX2-O0-NEXT: # kill: def $xmm0 killed $xmm0 killed $ymm0
-; CHECK-AVX2-O0-NEXT: vpextrw $7, %xmm0, %eax
-; CHECK-AVX2-O0-NEXT: shll $16, %eax
-; CHECK-AVX2-O0-NEXT: vmovd %eax, %xmm1
-; CHECK-AVX2-O0-NEXT: vmovss %xmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 4-byte Spill
-; CHECK-AVX2-O0-NEXT: vpextrw $6, %xmm0, %eax
-; CHECK-AVX2-O0-NEXT: shll $16, %eax
-; CHECK-AVX2-O0-NEXT: vmovd %eax, %xmm1
-; CHECK-AVX2-O0-NEXT: vmovss %xmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 4-byte Spill
-; CHECK-AVX2-O0-NEXT: vpextrw $5, %xmm0, %eax
-; CHECK-AVX2-O0-NEXT: shll $16, %eax
-; CHECK-AVX2-O0-NEXT: vmovd %eax, %xmm1
-; CHECK-AVX2-O0-NEXT: vmovss %xmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 4-byte Spill
-; CHECK-AVX2-O0-NEXT: vpextrw $4, %xmm0, %eax
-; CHECK-AVX2-O0-NEXT: shll $16, %eax
-; CHECK-AVX2-O0-NEXT: vmovd %eax, %xmm1
-; CHECK-AVX2-O0-NEXT: vmovss %xmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 4-byte Spill
-; CHECK-AVX2-O0-NEXT: vpextrw $3, %xmm0, %eax
-; CHECK-AVX2-O0-NEXT: shll $16, %eax
-; CHECK-AVX2-O0-NEXT: vmovd %eax, %xmm1
-; CHECK-AVX2-O0-NEXT: vmovss %xmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 4-byte Spill
-; CHECK-AVX2-O0-NEXT: vpextrw $2, %xmm0, %eax
-; CHECK-AVX2-O0-NEXT: shll $16, %eax
-; CHECK-AVX2-O0-NEXT: vmovd %eax, %xmm1
-; CHECK-AVX2-O0-NEXT: vmovss %xmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 4-byte Spill
-; CHECK-AVX2-O0-NEXT: vpextrw $1, %xmm0, %eax
-; CHECK-AVX2-O0-NEXT: shll $16, %eax
-; CHECK-AVX2-O0-NEXT: vmovd %eax, %xmm1
+; CHECK-AVX2-O0-NEXT: vpshufb %xmm6, %xmm0, %xmm7
+; CHECK-AVX2-O0-NEXT: vpshufb %xmm5, %xmm0, %xmm6
+; CHECK-AVX2-O0-NEXT: vpshufb %xmm4, %xmm0, %xmm5
+; CHECK-AVX2-O0-NEXT: vpunpckhwd {{.*#+}} xmm4 = xmm1[4],xmm0[4],xmm1[5],xmm0[5],xmm1[6],xmm0[6],xmm1[7],xmm0[7]
+; CHECK-AVX2-O0-NEXT: vpshufb %xmm3, %xmm0, %xmm3
+; CHECK-AVX2-O0-NEXT: vpshufb %xmm2, %xmm0, %xmm2
+; CHECK-AVX2-O0-NEXT: vpblendw {{.*#+}} xmm1 = xmm1[0],xmm0[1],xmm1[2],xmm0[3],xmm1[4],xmm0[5],xmm1[6],xmm0[7]
+; CHECK-AVX2-O0-NEXT: vpslld $16, %xmm8, %xmm8
+; CHECK-AVX2-O0-NEXT: vpslld $16, %xmm0, %xmm0
+; CHECK-AVX2-O0-NEXT: vmovss %xmm15, {{[-0-9]+}}(%r{{[sb]}}p) # 4-byte Spill
+; CHECK-AVX2-O0-NEXT: vmovss %xmm14, {{[-0-9]+}}(%r{{[sb]}}p) # 4-byte Spill
+; CHECK-AVX2-O0-NEXT: vmovss %xmm13, {{[-0-9]+}}(%r{{[sb]}}p) # 4-byte Spill
+; CHECK-AVX2-O0-NEXT: vmovss %xmm12, {{[-0-9]+}}(%r{{[sb]}}p) # 4-byte Spill
+; CHECK-AVX2-O0-NEXT: vmovss %xmm11, {{[-0-9]+}}(%r{{[sb]}}p) # 4-byte Spill
+; CHECK-AVX2-O0-NEXT: vmovss %xmm10, {{[-0-9]+}}(%r{{[sb]}}p) # 4-byte Spill
+; CHECK-AVX2-O0-NEXT: vmovss %xmm9, {{[-0-9]+}}(%r{{[sb]}}p) # 4-byte Spill
+; CHECK-AVX2-O0-NEXT: vmovss %xmm8, {{[-0-9]+}}(%r{{[sb]}}p) # 4-byte Spill
+; CHECK-AVX2-O0-NEXT: vmovss %xmm7, {{[-0-9]+}}(%r{{[sb]}}p) # 4-byte Spill
+; CHECK-AVX2-O0-NEXT: vmovss %xmm6, {{[-0-9]+}}(%r{{[sb]}}p) # 4-byte Spill
+; CHECK-AVX2-O0-NEXT: vmovss %xmm5, {{[-0-9]+}}(%r{{[sb]}}p) # 4-byte Spill
+; CHECK-AVX2-O0-NEXT: vmovss %xmm4, {{[-0-9]+}}(%r{{[sb]}}p) # 4-byte Spill
+; CHECK-AVX2-O0-NEXT: vmovss %xmm3, {{[-0-9]+}}(%r{{[sb]}}p) # 4-byte Spill
+; CHECK-AVX2-O0-NEXT: vmovss %xmm2, {{[-0-9]+}}(%r{{[sb]}}p) # 4-byte Spill
; CHECK-AVX2-O0-NEXT: vmovss %xmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 4-byte Spill
-; CHECK-AVX2-O0-NEXT: vmovd %xmm0, %eax
-; CHECK-AVX2-O0-NEXT: shll $16, %eax
-; CHECK-AVX2-O0-NEXT: vmovd %eax, %xmm0
; CHECK-AVX2-O0-NEXT: vzeroupper
; CHECK-AVX2-O0-NEXT: callq __truncsfbf2 at PLT
; CHECK-AVX2-O0-NEXT: vmovaps %xmm0, %xmm1
@@ -3150,71 +3157,45 @@ define void @store_atomic_vec16_bfloat_unaligned(ptr %x, <16 x bfloat> %v) nounw
; CHECK-AVX512-O0: # %bb.0:
; CHECK-AVX512-O0-NEXT: subq $120, %rsp
; CHECK-AVX512-O0-NEXT: movq %rdi, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Spill
-; CHECK-AVX512-O0-NEXT: vextractf128 $1, %ymm0, %xmm1
-; CHECK-AVX512-O0-NEXT: vpextrw $7, %xmm1, %eax
-; CHECK-AVX512-O0-NEXT: shll $16, %eax
-; CHECK-AVX512-O0-NEXT: vmovd %eax, %xmm2
-; CHECK-AVX512-O0-NEXT: vmovss %xmm2, {{[-0-9]+}}(%r{{[sb]}}p) # 4-byte Spill
-; CHECK-AVX512-O0-NEXT: vpextrw $6, %xmm1, %eax
-; CHECK-AVX512-O0-NEXT: shll $16, %eax
-; CHECK-AVX512-O0-NEXT: vmovd %eax, %xmm2
-; CHECK-AVX512-O0-NEXT: vmovss %xmm2, {{[-0-9]+}}(%r{{[sb]}}p) # 4-byte Spill
-; CHECK-AVX512-O0-NEXT: vpextrw $5, %xmm1, %eax
-; CHECK-AVX512-O0-NEXT: shll $16, %eax
-; CHECK-AVX512-O0-NEXT: vmovd %eax, %xmm2
-; CHECK-AVX512-O0-NEXT: vmovss %xmm2, {{[-0-9]+}}(%r{{[sb]}}p) # 4-byte Spill
-; CHECK-AVX512-O0-NEXT: vpextrw $4, %xmm1, %eax
-; CHECK-AVX512-O0-NEXT: shll $16, %eax
-; CHECK-AVX512-O0-NEXT: vmovd %eax, %xmm2
-; CHECK-AVX512-O0-NEXT: vmovss %xmm2, {{[-0-9]+}}(%r{{[sb]}}p) # 4-byte Spill
-; CHECK-AVX512-O0-NEXT: vpextrw $3, %xmm1, %eax
-; CHECK-AVX512-O0-NEXT: shll $16, %eax
-; CHECK-AVX512-O0-NEXT: vmovd %eax, %xmm2
-; CHECK-AVX512-O0-NEXT: vmovss %xmm2, {{[-0-9]+}}(%r{{[sb]}}p) # 4-byte Spill
-; CHECK-AVX512-O0-NEXT: vpextrw $2, %xmm1, %eax
-; CHECK-AVX512-O0-NEXT: shll $16, %eax
-; CHECK-AVX512-O0-NEXT: vmovd %eax, %xmm2
-; CHECK-AVX512-O0-NEXT: vmovss %xmm2, {{[-0-9]+}}(%r{{[sb]}}p) # 4-byte Spill
-; CHECK-AVX512-O0-NEXT: vpextrw $1, %xmm1, %eax
-; CHECK-AVX512-O0-NEXT: shll $16, %eax
-; CHECK-AVX512-O0-NEXT: vmovd %eax, %xmm2
-; CHECK-AVX512-O0-NEXT: vmovss %xmm2, {{[-0-9]+}}(%r{{[sb]}}p) # 4-byte Spill
-; CHECK-AVX512-O0-NEXT: vmovd %xmm1, %eax
-; CHECK-AVX512-O0-NEXT: shll $16, %eax
-; CHECK-AVX512-O0-NEXT: vmovd %eax, %xmm1
-; CHECK-AVX512-O0-NEXT: vmovss %xmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 4-byte Spill
+; CHECK-AVX512-O0-NEXT: vextractf128 $1, %ymm0, %xmm8
+; CHECK-AVX512-O0-NEXT: vmovdqa {{.*#+}} xmm6 = [128,128,14,15,u,u,u,u,u,u,u,u,u,u,u,u]
+; CHECK-AVX512-O0-NEXT: vpshufb %xmm6, %xmm8, %xmm15
+; CHECK-AVX512-O0-NEXT: vmovdqa {{.*#+}} xmm5 = [128,128,12,13,u,u,u,u,u,u,u,u,u,u,u,u]
+; CHECK-AVX512-O0-NEXT: vpshufb %xmm5, %xmm8, %xmm14
+; CHECK-AVX512-O0-NEXT: vmovdqa {{.*#+}} xmm4 = [128,128,10,11,u,u,u,u,u,u,u,u,u,u,u,u]
+; CHECK-AVX512-O0-NEXT: vpshufb %xmm4, %xmm8, %xmm13
+; CHECK-AVX512-O0-NEXT: vpxor %xmm1, %xmm1, %xmm1
+; CHECK-AVX512-O0-NEXT: vpunpckhwd {{.*#+}} xmm12 = xmm1[4],xmm8[4],xmm1[5],xmm8[5],xmm1[6],xmm8[6],xmm1[7],xmm8[7]
+; CHECK-AVX512-O0-NEXT: vmovdqa {{.*#+}} xmm3 = [128,128,6,7,u,u,u,u,u,u,u,u,u,u,u,u]
+; CHECK-AVX512-O0-NEXT: vpshufb %xmm3, %xmm8, %xmm11
+; CHECK-AVX512-O0-NEXT: vmovdqa {{.*#+}} xmm2 = [128,128,4,5,u,u,u,u,u,u,u,u,u,u,u,u]
+; CHECK-AVX512-O0-NEXT: vpshufb %xmm2, %xmm8, %xmm10
+; CHECK-AVX512-O0-NEXT: vpblendw {{.*#+}} xmm9 = xmm1[0],xmm8[1],xmm1[2],xmm8[3],xmm1[4],xmm8[5],xmm1[6],xmm8[7]
; CHECK-AVX512-O0-NEXT: # kill: def $xmm0 killed $xmm0 killed $ymm0
-; CHECK-AVX512-O0-NEXT: vpextrw $7, %xmm0, %eax
-; CHECK-AVX512-O0-NEXT: shll $16, %eax
-; CHECK-AVX512-O0-NEXT: vmovd %eax, %xmm1
-; CHECK-AVX512-O0-NEXT: vmovss %xmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 4-byte Spill
-; CHECK-AVX512-O0-NEXT: vpextrw $6, %xmm0, %eax
-; CHECK-AVX512-O0-NEXT: shll $16, %eax
-; CHECK-AVX512-O0-NEXT: vmovd %eax, %xmm1
-; CHECK-AVX512-O0-NEXT: vmovss %xmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 4-byte Spill
-; CHECK-AVX512-O0-NEXT: vpextrw $5, %xmm0, %eax
-; CHECK-AVX512-O0-NEXT: shll $16, %eax
-; CHECK-AVX512-O0-NEXT: vmovd %eax, %xmm1
-; CHECK-AVX512-O0-NEXT: vmovss %xmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 4-byte Spill
-; CHECK-AVX512-O0-NEXT: vpextrw $4, %xmm0, %eax
-; CHECK-AVX512-O0-NEXT: shll $16, %eax
-; CHECK-AVX512-O0-NEXT: vmovd %eax, %xmm1
-; CHECK-AVX512-O0-NEXT: vmovss %xmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 4-byte Spill
-; CHECK-AVX512-O0-NEXT: vpextrw $3, %xmm0, %eax
-; CHECK-AVX512-O0-NEXT: shll $16, %eax
-; CHECK-AVX512-O0-NEXT: vmovd %eax, %xmm1
-; CHECK-AVX512-O0-NEXT: vmovss %xmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 4-byte Spill
-; CHECK-AVX512-O0-NEXT: vpextrw $2, %xmm0, %eax
-; CHECK-AVX512-O0-NEXT: shll $16, %eax
-; CHECK-AVX512-O0-NEXT: vmovd %eax, %xmm1
-; CHECK-AVX512-O0-NEXT: vmovss %xmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 4-byte Spill
-; CHECK-AVX512-O0-NEXT: vpextrw $1, %xmm0, %eax
-; CHECK-AVX512-O0-NEXT: shll $16, %eax
-; CHECK-AVX512-O0-NEXT: vmovd %eax, %xmm1
+; CHECK-AVX512-O0-NEXT: vpshufb %xmm6, %xmm0, %xmm7
+; CHECK-AVX512-O0-NEXT: vpshufb %xmm5, %xmm0, %xmm6
+; CHECK-AVX512-O0-NEXT: vpshufb %xmm4, %xmm0, %xmm5
+; CHECK-AVX512-O0-NEXT: vpunpckhwd {{.*#+}} xmm4 = xmm1[4],xmm0[4],xmm1[5],xmm0[5],xmm1[6],xmm0[6],xmm1[7],xmm0[7]
+; CHECK-AVX512-O0-NEXT: vpshufb %xmm3, %xmm0, %xmm3
+; CHECK-AVX512-O0-NEXT: vpshufb %xmm2, %xmm0, %xmm2
+; CHECK-AVX512-O0-NEXT: vpblendw {{.*#+}} xmm1 = xmm1[0],xmm0[1],xmm1[2],xmm0[3],xmm1[4],xmm0[5],xmm1[6],xmm0[7]
+; CHECK-AVX512-O0-NEXT: vpslld $16, %xmm8, %xmm8
+; CHECK-AVX512-O0-NEXT: vpslld $16, %xmm0, %xmm0
+; CHECK-AVX512-O0-NEXT: vmovss %xmm15, {{[-0-9]+}}(%r{{[sb]}}p) # 4-byte Spill
+; CHECK-AVX512-O0-NEXT: vmovss %xmm14, {{[-0-9]+}}(%r{{[sb]}}p) # 4-byte Spill
+; CHECK-AVX512-O0-NEXT: vmovss %xmm13, {{[-0-9]+}}(%r{{[sb]}}p) # 4-byte Spill
+; CHECK-AVX512-O0-NEXT: vmovss %xmm12, {{[-0-9]+}}(%r{{[sb]}}p) # 4-byte Spill
+; CHECK-AVX512-O0-NEXT: vmovss %xmm11, {{[-0-9]+}}(%r{{[sb]}}p) # 4-byte Spill
+; CHECK-AVX512-O0-NEXT: vmovss %xmm10, {{[-0-9]+}}(%r{{[sb]}}p) # 4-byte Spill
+; CHECK-AVX512-O0-NEXT: vmovss %xmm9, {{[-0-9]+}}(%r{{[sb]}}p) # 4-byte Spill
+; CHECK-AVX512-O0-NEXT: vmovss %xmm8, {{[-0-9]+}}(%r{{[sb]}}p) # 4-byte Spill
+; CHECK-AVX512-O0-NEXT: vmovss %xmm7, {{[-0-9]+}}(%r{{[sb]}}p) # 4-byte Spill
+; CHECK-AVX512-O0-NEXT: vmovss %xmm6, {{[-0-9]+}}(%r{{[sb]}}p) # 4-byte Spill
+; CHECK-AVX512-O0-NEXT: vmovss %xmm5, {{[-0-9]+}}(%r{{[sb]}}p) # 4-byte Spill
+; CHECK-AVX512-O0-NEXT: vmovss %xmm4, {{[-0-9]+}}(%r{{[sb]}}p) # 4-byte Spill
+; CHECK-AVX512-O0-NEXT: vmovss %xmm3, {{[-0-9]+}}(%r{{[sb]}}p) # 4-byte Spill
+; CHECK-AVX512-O0-NEXT: vmovss %xmm2, {{[-0-9]+}}(%r{{[sb]}}p) # 4-byte Spill
; CHECK-AVX512-O0-NEXT: vmovss %xmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 4-byte Spill
-; CHECK-AVX512-O0-NEXT: vmovd %xmm0, %eax
-; CHECK-AVX512-O0-NEXT: shll $16, %eax
-; CHECK-AVX512-O0-NEXT: vmovd %eax, %xmm0
; CHECK-AVX512-O0-NEXT: vzeroupper
; CHECK-AVX512-O0-NEXT: callq __truncsfbf2 at PLT
; CHECK-AVX512-O0-NEXT: vmovaps %xmm0, %xmm1
@@ -3654,70 +3635,68 @@ define void @store_atomic_vec16_bfloat_align(ptr %x, <16 x bfloat> %v) nounwind
; CHECK-SSE2-O0-LABEL: store_atomic_vec16_bfloat_align:
; CHECK-SSE2-O0: # %bb.0:
; CHECK-SSE2-O0-NEXT: subq $120, %rsp
+; CHECK-SSE2-O0-NEXT: movaps %xmm1, %xmm2
+; CHECK-SSE2-O0-NEXT: movaps %xmm0, %xmm1
; CHECK-SSE2-O0-NEXT: movq %rdi, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Spill
+; CHECK-SSE2-O0-NEXT: xorps %xmm13, %xmm13
+; CHECK-SSE2-O0-NEXT: movaps %xmm13, %xmm15
+; CHECK-SSE2-O0-NEXT: punpckhwd {{.*#+}} xmm15 = xmm15[4],xmm2[4],xmm15[5],xmm2[5],xmm15[6],xmm2[6],xmm15[7],xmm2[7]
+; CHECK-SSE2-O0-NEXT: punpckhwd {{.*#+}} xmm13 = xmm13[4],xmm1[4],xmm13[5],xmm1[5],xmm13[6],xmm1[6],xmm13[7],xmm1[7]
+; CHECK-SSE2-O0-NEXT: movaps %xmm2, %xmm14
+; CHECK-SSE2-O0-NEXT: pslld $16, %xmm14
+; CHECK-SSE2-O0-NEXT: movaps %xmm1, %xmm0
+; CHECK-SSE2-O0-NEXT: pslld $16, %xmm0
+; CHECK-SSE2-O0-NEXT: pextrw $7, %xmm2, %eax
+; CHECK-SSE2-O0-NEXT: movd %eax, %xmm12
+; CHECK-SSE2-O0-NEXT: pslld $16, %xmm12
+; CHECK-SSE2-O0-NEXT: pextrw $6, %xmm2, %eax
+; CHECK-SSE2-O0-NEXT: movd %eax, %xmm11
+; CHECK-SSE2-O0-NEXT: pslld $16, %xmm11
+; CHECK-SSE2-O0-NEXT: pextrw $5, %xmm2, %eax
+; CHECK-SSE2-O0-NEXT: movd %eax, %xmm10
+; CHECK-SSE2-O0-NEXT: pslld $16, %xmm10
+; CHECK-SSE2-O0-NEXT: pextrw $3, %xmm2, %eax
+; CHECK-SSE2-O0-NEXT: movd %eax, %xmm9
+; CHECK-SSE2-O0-NEXT: pslld $16, %xmm9
+; CHECK-SSE2-O0-NEXT: pextrw $2, %xmm2, %eax
+; CHECK-SSE2-O0-NEXT: movd %eax, %xmm8
+; CHECK-SSE2-O0-NEXT: pslld $16, %xmm8
+; CHECK-SSE2-O0-NEXT: pextrw $1, %xmm2, %eax
+; CHECK-SSE2-O0-NEXT: movd %eax, %xmm7
+; CHECK-SSE2-O0-NEXT: pslld $16, %xmm7
; CHECK-SSE2-O0-NEXT: pextrw $7, %xmm1, %eax
-; CHECK-SSE2-O0-NEXT: shll $16, %eax
-; CHECK-SSE2-O0-NEXT: movd %eax, %xmm2
-; CHECK-SSE2-O0-NEXT: movss %xmm2, {{[-0-9]+}}(%r{{[sb]}}p) # 4-byte Spill
+; CHECK-SSE2-O0-NEXT: movd %eax, %xmm6
+; CHECK-SSE2-O0-NEXT: pslld $16, %xmm6
; CHECK-SSE2-O0-NEXT: pextrw $6, %xmm1, %eax
-; CHECK-SSE2-O0-NEXT: shll $16, %eax
-; CHECK-SSE2-O0-NEXT: movd %eax, %xmm2
-; CHECK-SSE2-O0-NEXT: movss %xmm2, {{[-0-9]+}}(%r{{[sb]}}p) # 4-byte Spill
+; CHECK-SSE2-O0-NEXT: movd %eax, %xmm5
+; CHECK-SSE2-O0-NEXT: pslld $16, %xmm5
; CHECK-SSE2-O0-NEXT: pextrw $5, %xmm1, %eax
-; CHECK-SSE2-O0-NEXT: shll $16, %eax
-; CHECK-SSE2-O0-NEXT: movd %eax, %xmm2
-; CHECK-SSE2-O0-NEXT: movss %xmm2, {{[-0-9]+}}(%r{{[sb]}}p) # 4-byte Spill
-; CHECK-SSE2-O0-NEXT: pextrw $4, %xmm1, %eax
-; CHECK-SSE2-O0-NEXT: shll $16, %eax
-; CHECK-SSE2-O0-NEXT: movd %eax, %xmm2
-; CHECK-SSE2-O0-NEXT: movss %xmm2, {{[-0-9]+}}(%r{{[sb]}}p) # 4-byte Spill
+; CHECK-SSE2-O0-NEXT: movd %eax, %xmm4
+; CHECK-SSE2-O0-NEXT: pslld $16, %xmm4
; CHECK-SSE2-O0-NEXT: pextrw $3, %xmm1, %eax
-; CHECK-SSE2-O0-NEXT: shll $16, %eax
-; CHECK-SSE2-O0-NEXT: movd %eax, %xmm2
-; CHECK-SSE2-O0-NEXT: movss %xmm2, {{[-0-9]+}}(%r{{[sb]}}p) # 4-byte Spill
+; CHECK-SSE2-O0-NEXT: movd %eax, %xmm3
+; CHECK-SSE2-O0-NEXT: pslld $16, %xmm3
; CHECK-SSE2-O0-NEXT: pextrw $2, %xmm1, %eax
-; CHECK-SSE2-O0-NEXT: shll $16, %eax
; CHECK-SSE2-O0-NEXT: movd %eax, %xmm2
-; CHECK-SSE2-O0-NEXT: movss %xmm2, {{[-0-9]+}}(%r{{[sb]}}p) # 4-byte Spill
+; CHECK-SSE2-O0-NEXT: pslld $16, %xmm2
; CHECK-SSE2-O0-NEXT: pextrw $1, %xmm1, %eax
-; CHECK-SSE2-O0-NEXT: shll $16, %eax
-; CHECK-SSE2-O0-NEXT: movd %eax, %xmm2
-; CHECK-SSE2-O0-NEXT: movss %xmm2, {{[-0-9]+}}(%r{{[sb]}}p) # 4-byte Spill
-; CHECK-SSE2-O0-NEXT: movd %xmm1, %eax
-; CHECK-SSE2-O0-NEXT: shll $16, %eax
-; CHECK-SSE2-O0-NEXT: movd %eax, %xmm1
-; CHECK-SSE2-O0-NEXT: movss %xmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 4-byte Spill
-; CHECK-SSE2-O0-NEXT: pextrw $7, %xmm0, %eax
-; CHECK-SSE2-O0-NEXT: shll $16, %eax
-; CHECK-SSE2-O0-NEXT: movd %eax, %xmm1
-; CHECK-SSE2-O0-NEXT: movss %xmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 4-byte Spill
-; CHECK-SSE2-O0-NEXT: pextrw $6, %xmm0, %eax
-; CHECK-SSE2-O0-NEXT: shll $16, %eax
-; CHECK-SSE2-O0-NEXT: movd %eax, %xmm1
-; CHECK-SSE2-O0-NEXT: movss %xmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 4-byte Spill
-; CHECK-SSE2-O0-NEXT: pextrw $5, %xmm0, %eax
-; CHECK-SSE2-O0-NEXT: shll $16, %eax
-; CHECK-SSE2-O0-NEXT: movd %eax, %xmm1
-; CHECK-SSE2-O0-NEXT: movss %xmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 4-byte Spill
-; CHECK-SSE2-O0-NEXT: pextrw $4, %xmm0, %eax
-; CHECK-SSE2-O0-NEXT: shll $16, %eax
-; CHECK-SSE2-O0-NEXT: movd %eax, %xmm1
-; CHECK-SSE2-O0-NEXT: movss %xmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 4-byte Spill
-; CHECK-SSE2-O0-NEXT: pextrw $3, %xmm0, %eax
-; CHECK-SSE2-O0-NEXT: shll $16, %eax
-; CHECK-SSE2-O0-NEXT: movd %eax, %xmm1
-; CHECK-SSE2-O0-NEXT: movss %xmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 4-byte Spill
-; CHECK-SSE2-O0-NEXT: pextrw $2, %xmm0, %eax
-; CHECK-SSE2-O0-NEXT: shll $16, %eax
-; CHECK-SSE2-O0-NEXT: movd %eax, %xmm1
-; CHECK-SSE2-O0-NEXT: movss %xmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 4-byte Spill
-; CHECK-SSE2-O0-NEXT: pextrw $1, %xmm0, %eax
-; CHECK-SSE2-O0-NEXT: shll $16, %eax
; CHECK-SSE2-O0-NEXT: movd %eax, %xmm1
+; CHECK-SSE2-O0-NEXT: pslld $16, %xmm1
+; CHECK-SSE2-O0-NEXT: movss %xmm15, {{[-0-9]+}}(%r{{[sb]}}p) # 4-byte Spill
+; CHECK-SSE2-O0-NEXT: movss %xmm14, {{[-0-9]+}}(%r{{[sb]}}p) # 4-byte Spill
+; CHECK-SSE2-O0-NEXT: movss %xmm13, {{[-0-9]+}}(%r{{[sb]}}p) # 4-byte Spill
+; CHECK-SSE2-O0-NEXT: movss %xmm12, {{[-0-9]+}}(%r{{[sb]}}p) # 4-byte Spill
+; CHECK-SSE2-O0-NEXT: movss %xmm11, {{[-0-9]+}}(%r{{[sb]}}p) # 4-byte Spill
+; CHECK-SSE2-O0-NEXT: movss %xmm10, {{[-0-9]+}}(%r{{[sb]}}p) # 4-byte Spill
+; CHECK-SSE2-O0-NEXT: movss %xmm9, {{[-0-9]+}}(%r{{[sb]}}p) # 4-byte Spill
+; CHECK-SSE2-O0-NEXT: movss %xmm8, {{[-0-9]+}}(%r{{[sb]}}p) # 4-byte Spill
+; CHECK-SSE2-O0-NEXT: movss %xmm7, {{[-0-9]+}}(%r{{[sb]}}p) # 4-byte Spill
+; CHECK-SSE2-O0-NEXT: movss %xmm6, {{[-0-9]+}}(%r{{[sb]}}p) # 4-byte Spill
+; CHECK-SSE2-O0-NEXT: movss %xmm5, {{[-0-9]+}}(%r{{[sb]}}p) # 4-byte Spill
+; CHECK-SSE2-O0-NEXT: movss %xmm4, {{[-0-9]+}}(%r{{[sb]}}p) # 4-byte Spill
+; CHECK-SSE2-O0-NEXT: movss %xmm3, {{[-0-9]+}}(%r{{[sb]}}p) # 4-byte Spill
+; CHECK-SSE2-O0-NEXT: movss %xmm2, {{[-0-9]+}}(%r{{[sb]}}p) # 4-byte Spill
; CHECK-SSE2-O0-NEXT: movss %xmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 4-byte Spill
-; CHECK-SSE2-O0-NEXT: movd %xmm0, %eax
-; CHECK-SSE2-O0-NEXT: shll $16, %eax
-; CHECK-SSE2-O0-NEXT: movd %eax, %xmm0
; CHECK-SSE2-O0-NEXT: callq __truncsfbf2 at PLT
; CHECK-SSE2-O0-NEXT: movaps %xmm0, %xmm1
; CHECK-SSE2-O0-NEXT: movss {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 4-byte Reload
@@ -3838,70 +3817,67 @@ define void @store_atomic_vec16_bfloat_align(ptr %x, <16 x bfloat> %v) nounwind
; CHECK-SSE4-O0-LABEL: store_atomic_vec16_bfloat_align:
; CHECK-SSE4-O0: # %bb.0:
; CHECK-SSE4-O0-NEXT: subq $120, %rsp
+; CHECK-SSE4-O0-NEXT: movaps %xmm1, %xmm2
+; CHECK-SSE4-O0-NEXT: movaps %xmm0, %xmm1
; CHECK-SSE4-O0-NEXT: movq %rdi, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Spill
+; CHECK-SSE4-O0-NEXT: xorps %xmm0, %xmm0
+; CHECK-SSE4-O0-NEXT: movaps %xmm0, %xmm15
+; CHECK-SSE4-O0-NEXT: punpckhwd {{.*#+}} xmm15 = xmm15[4],xmm2[4],xmm15[5],xmm2[5],xmm15[6],xmm2[6],xmm15[7],xmm2[7]
+; CHECK-SSE4-O0-NEXT: movaps %xmm2, %xmm14
+; CHECK-SSE4-O0-NEXT: pblendw {{.*#+}} xmm14 = xmm0[0],xmm14[1],xmm0[2],xmm14[3],xmm0[4],xmm14[5],xmm0[6],xmm14[7]
+; CHECK-SSE4-O0-NEXT: movaps %xmm0, %xmm12
+; CHECK-SSE4-O0-NEXT: punpckhwd {{.*#+}} xmm12 = xmm12[4],xmm1[4],xmm12[5],xmm1[5],xmm12[6],xmm1[6],xmm12[7],xmm1[7]
+; CHECK-SSE4-O0-NEXT: movaps %xmm1, %xmm11
+; CHECK-SSE4-O0-NEXT: pblendw {{.*#+}} xmm11 = xmm0[0],xmm11[1],xmm0[2],xmm11[3],xmm0[4],xmm11[5],xmm0[6],xmm11[7]
+; CHECK-SSE4-O0-NEXT: movaps %xmm2, %xmm13
+; CHECK-SSE4-O0-NEXT: pslld $16, %xmm13
+; CHECK-SSE4-O0-NEXT: movaps %xmm1, %xmm0
+; CHECK-SSE4-O0-NEXT: pslld $16, %xmm0
+; CHECK-SSE4-O0-NEXT: pextrw $7, %xmm2, %eax
+; CHECK-SSE4-O0-NEXT: movd %eax, %xmm10
+; CHECK-SSE4-O0-NEXT: pslld $16, %xmm10
+; CHECK-SSE4-O0-NEXT: pextrw $6, %xmm2, %eax
+; CHECK-SSE4-O0-NEXT: movd %eax, %xmm9
+; CHECK-SSE4-O0-NEXT: pslld $16, %xmm9
+; CHECK-SSE4-O0-NEXT: pextrw $5, %xmm2, %eax
+; CHECK-SSE4-O0-NEXT: movd %eax, %xmm8
+; CHECK-SSE4-O0-NEXT: pslld $16, %xmm8
+; CHECK-SSE4-O0-NEXT: pextrw $3, %xmm2, %eax
+; CHECK-SSE4-O0-NEXT: movd %eax, %xmm7
+; CHECK-SSE4-O0-NEXT: pslld $16, %xmm7
+; CHECK-SSE4-O0-NEXT: pextrw $2, %xmm2, %eax
+; CHECK-SSE4-O0-NEXT: movd %eax, %xmm6
+; CHECK-SSE4-O0-NEXT: pslld $16, %xmm6
; CHECK-SSE4-O0-NEXT: pextrw $7, %xmm1, %eax
-; CHECK-SSE4-O0-NEXT: shll $16, %eax
-; CHECK-SSE4-O0-NEXT: movd %eax, %xmm2
-; CHECK-SSE4-O0-NEXT: movss %xmm2, {{[-0-9]+}}(%r{{[sb]}}p) # 4-byte Spill
+; CHECK-SSE4-O0-NEXT: movd %eax, %xmm5
+; CHECK-SSE4-O0-NEXT: pslld $16, %xmm5
; CHECK-SSE4-O0-NEXT: pextrw $6, %xmm1, %eax
-; CHECK-SSE4-O0-NEXT: shll $16, %eax
-; CHECK-SSE4-O0-NEXT: movd %eax, %xmm2
-; CHECK-SSE4-O0-NEXT: movss %xmm2, {{[-0-9]+}}(%r{{[sb]}}p) # 4-byte Spill
+; CHECK-SSE4-O0-NEXT: movd %eax, %xmm4
+; CHECK-SSE4-O0-NEXT: pslld $16, %xmm4
; CHECK-SSE4-O0-NEXT: pextrw $5, %xmm1, %eax
-; CHECK-SSE4-O0-NEXT: shll $16, %eax
-; CHECK-SSE4-O0-NEXT: movd %eax, %xmm2
-; CHECK-SSE4-O0-NEXT: movss %xmm2, {{[-0-9]+}}(%r{{[sb]}}p) # 4-byte Spill
-; CHECK-SSE4-O0-NEXT: pextrw $4, %xmm1, %eax
-; CHECK-SSE4-O0-NEXT: shll $16, %eax
-; CHECK-SSE4-O0-NEXT: movd %eax, %xmm2
-; CHECK-SSE4-O0-NEXT: movss %xmm2, {{[-0-9]+}}(%r{{[sb]}}p) # 4-byte Spill
+; CHECK-SSE4-O0-NEXT: movd %eax, %xmm3
+; CHECK-SSE4-O0-NEXT: pslld $16, %xmm3
; CHECK-SSE4-O0-NEXT: pextrw $3, %xmm1, %eax
-; CHECK-SSE4-O0-NEXT: shll $16, %eax
; CHECK-SSE4-O0-NEXT: movd %eax, %xmm2
-; CHECK-SSE4-O0-NEXT: movss %xmm2, {{[-0-9]+}}(%r{{[sb]}}p) # 4-byte Spill
+; CHECK-SSE4-O0-NEXT: pslld $16, %xmm2
; CHECK-SSE4-O0-NEXT: pextrw $2, %xmm1, %eax
-; CHECK-SSE4-O0-NEXT: shll $16, %eax
-; CHECK-SSE4-O0-NEXT: movd %eax, %xmm2
-; CHECK-SSE4-O0-NEXT: movss %xmm2, {{[-0-9]+}}(%r{{[sb]}}p) # 4-byte Spill
-; CHECK-SSE4-O0-NEXT: pextrw $1, %xmm1, %eax
-; CHECK-SSE4-O0-NEXT: shll $16, %eax
-; CHECK-SSE4-O0-NEXT: movd %eax, %xmm2
-; CHECK-SSE4-O0-NEXT: movss %xmm2, {{[-0-9]+}}(%r{{[sb]}}p) # 4-byte Spill
-; CHECK-SSE4-O0-NEXT: movd %xmm1, %eax
-; CHECK-SSE4-O0-NEXT: shll $16, %eax
-; CHECK-SSE4-O0-NEXT: movd %eax, %xmm1
-; CHECK-SSE4-O0-NEXT: movss %xmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 4-byte Spill
-; CHECK-SSE4-O0-NEXT: pextrw $7, %xmm0, %eax
-; CHECK-SSE4-O0-NEXT: shll $16, %eax
-; CHECK-SSE4-O0-NEXT: movd %eax, %xmm1
-; CHECK-SSE4-O0-NEXT: movss %xmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 4-byte Spill
-; CHECK-SSE4-O0-NEXT: pextrw $6, %xmm0, %eax
-; CHECK-SSE4-O0-NEXT: shll $16, %eax
-; CHECK-SSE4-O0-NEXT: movd %eax, %xmm1
-; CHECK-SSE4-O0-NEXT: movss %xmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 4-byte Spill
-; CHECK-SSE4-O0-NEXT: pextrw $5, %xmm0, %eax
-; CHECK-SSE4-O0-NEXT: shll $16, %eax
-; CHECK-SSE4-O0-NEXT: movd %eax, %xmm1
-; CHECK-SSE4-O0-NEXT: movss %xmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 4-byte Spill
-; CHECK-SSE4-O0-NEXT: pextrw $4, %xmm0, %eax
-; CHECK-SSE4-O0-NEXT: shll $16, %eax
-; CHECK-SSE4-O0-NEXT: movd %eax, %xmm1
-; CHECK-SSE4-O0-NEXT: movss %xmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 4-byte Spill
-; CHECK-SSE4-O0-NEXT: pextrw $3, %xmm0, %eax
-; CHECK-SSE4-O0-NEXT: shll $16, %eax
-; CHECK-SSE4-O0-NEXT: movd %eax, %xmm1
-; CHECK-SSE4-O0-NEXT: movss %xmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 4-byte Spill
-; CHECK-SSE4-O0-NEXT: pextrw $2, %xmm0, %eax
-; CHECK-SSE4-O0-NEXT: shll $16, %eax
-; CHECK-SSE4-O0-NEXT: movd %eax, %xmm1
-; CHECK-SSE4-O0-NEXT: movss %xmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 4-byte Spill
-; CHECK-SSE4-O0-NEXT: pextrw $1, %xmm0, %eax
-; CHECK-SSE4-O0-NEXT: shll $16, %eax
; CHECK-SSE4-O0-NEXT: movd %eax, %xmm1
+; CHECK-SSE4-O0-NEXT: pslld $16, %xmm1
+; CHECK-SSE4-O0-NEXT: movss %xmm15, {{[-0-9]+}}(%r{{[sb]}}p) # 4-byte Spill
+; CHECK-SSE4-O0-NEXT: movss %xmm14, {{[-0-9]+}}(%r{{[sb]}}p) # 4-byte Spill
+; CHECK-SSE4-O0-NEXT: movss %xmm13, {{[-0-9]+}}(%r{{[sb]}}p) # 4-byte Spill
+; CHECK-SSE4-O0-NEXT: movss %xmm12, {{[-0-9]+}}(%r{{[sb]}}p) # 4-byte Spill
+; CHECK-SSE4-O0-NEXT: movss %xmm11, {{[-0-9]+}}(%r{{[sb]}}p) # 4-byte Spill
+; CHECK-SSE4-O0-NEXT: movss %xmm10, {{[-0-9]+}}(%r{{[sb]}}p) # 4-byte Spill
+; CHECK-SSE4-O0-NEXT: movss %xmm9, {{[-0-9]+}}(%r{{[sb]}}p) # 4-byte Spill
+; CHECK-SSE4-O0-NEXT: movss %xmm8, {{[-0-9]+}}(%r{{[sb]}}p) # 4-byte Spill
+; CHECK-SSE4-O0-NEXT: movss %xmm7, {{[-0-9]+}}(%r{{[sb]}}p) # 4-byte Spill
+; CHECK-SSE4-O0-NEXT: movss %xmm6, {{[-0-9]+}}(%r{{[sb]}}p) # 4-byte Spill
+; CHECK-SSE4-O0-NEXT: movss %xmm5, {{[-0-9]+}}(%r{{[sb]}}p) # 4-byte Spill
+; CHECK-SSE4-O0-NEXT: movss %xmm4, {{[-0-9]+}}(%r{{[sb]}}p) # 4-byte Spill
+; CHECK-SSE4-O0-NEXT: movss %xmm3, {{[-0-9]+}}(%r{{[sb]}}p) # 4-byte Spill
+; CHECK-SSE4-O0-NEXT: movss %xmm2, {{[-0-9]+}}(%r{{[sb]}}p) # 4-byte Spill
; CHECK-SSE4-O0-NEXT: movss %xmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 4-byte Spill
-; CHECK-SSE4-O0-NEXT: movd %xmm0, %eax
-; CHECK-SSE4-O0-NEXT: shll $16, %eax
-; CHECK-SSE4-O0-NEXT: movd %eax, %xmm0
; CHECK-SSE4-O0-NEXT: callq __truncsfbf2 at PLT
; CHECK-SSE4-O0-NEXT: movaps %xmm0, %xmm1
; CHECK-SSE4-O0-NEXT: movss {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 4-byte Reload
@@ -3991,71 +3967,45 @@ define void @store_atomic_vec16_bfloat_align(ptr %x, <16 x bfloat> %v) nounwind
; CHECK-AVX2-O0: # %bb.0:
; CHECK-AVX2-O0-NEXT: subq $120, %rsp
; CHECK-AVX2-O0-NEXT: movq %rdi, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Spill
-; CHECK-AVX2-O0-NEXT: vextracti128 $1, %ymm0, %xmm1
-; CHECK-AVX2-O0-NEXT: vpextrw $7, %xmm1, %eax
-; CHECK-AVX2-O0-NEXT: shll $16, %eax
-; CHECK-AVX2-O0-NEXT: vmovd %eax, %xmm2
-; CHECK-AVX2-O0-NEXT: vmovss %xmm2, {{[-0-9]+}}(%r{{[sb]}}p) # 4-byte Spill
-; CHECK-AVX2-O0-NEXT: vpextrw $6, %xmm1, %eax
-; CHECK-AVX2-O0-NEXT: shll $16, %eax
-; CHECK-AVX2-O0-NEXT: vmovd %eax, %xmm2
-; CHECK-AVX2-O0-NEXT: vmovss %xmm2, {{[-0-9]+}}(%r{{[sb]}}p) # 4-byte Spill
-; CHECK-AVX2-O0-NEXT: vpextrw $5, %xmm1, %eax
-; CHECK-AVX2-O0-NEXT: shll $16, %eax
-; CHECK-AVX2-O0-NEXT: vmovd %eax, %xmm2
-; CHECK-AVX2-O0-NEXT: vmovss %xmm2, {{[-0-9]+}}(%r{{[sb]}}p) # 4-byte Spill
-; CHECK-AVX2-O0-NEXT: vpextrw $4, %xmm1, %eax
-; CHECK-AVX2-O0-NEXT: shll $16, %eax
-; CHECK-AVX2-O0-NEXT: vmovd %eax, %xmm2
-; CHECK-AVX2-O0-NEXT: vmovss %xmm2, {{[-0-9]+}}(%r{{[sb]}}p) # 4-byte Spill
-; CHECK-AVX2-O0-NEXT: vpextrw $3, %xmm1, %eax
-; CHECK-AVX2-O0-NEXT: shll $16, %eax
-; CHECK-AVX2-O0-NEXT: vmovd %eax, %xmm2
-; CHECK-AVX2-O0-NEXT: vmovss %xmm2, {{[-0-9]+}}(%r{{[sb]}}p) # 4-byte Spill
-; CHECK-AVX2-O0-NEXT: vpextrw $2, %xmm1, %eax
-; CHECK-AVX2-O0-NEXT: shll $16, %eax
-; CHECK-AVX2-O0-NEXT: vmovd %eax, %xmm2
-; CHECK-AVX2-O0-NEXT: vmovss %xmm2, {{[-0-9]+}}(%r{{[sb]}}p) # 4-byte Spill
-; CHECK-AVX2-O0-NEXT: vpextrw $1, %xmm1, %eax
-; CHECK-AVX2-O0-NEXT: shll $16, %eax
-; CHECK-AVX2-O0-NEXT: vmovd %eax, %xmm2
-; CHECK-AVX2-O0-NEXT: vmovss %xmm2, {{[-0-9]+}}(%r{{[sb]}}p) # 4-byte Spill
-; CHECK-AVX2-O0-NEXT: vmovd %xmm1, %eax
-; CHECK-AVX2-O0-NEXT: shll $16, %eax
-; CHECK-AVX2-O0-NEXT: vmovd %eax, %xmm1
-; CHECK-AVX2-O0-NEXT: vmovss %xmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 4-byte Spill
+; CHECK-AVX2-O0-NEXT: vextracti128 $1, %ymm0, %xmm8
+; CHECK-AVX2-O0-NEXT: vmovdqa {{.*#+}} xmm6 = [128,128,14,15,u,u,u,u,u,u,u,u,u,u,u,u]
+; CHECK-AVX2-O0-NEXT: vpshufb %xmm6, %xmm8, %xmm15
+; CHECK-AVX2-O0-NEXT: vmovdqa {{.*#+}} xmm5 = [128,128,12,13,u,u,u,u,u,u,u,u,u,u,u,u]
+; CHECK-AVX2-O0-NEXT: vpshufb %xmm5, %xmm8, %xmm14
+; CHECK-AVX2-O0-NEXT: vmovdqa {{.*#+}} xmm4 = [128,128,10,11,u,u,u,u,u,u,u,u,u,u,u,u]
+; CHECK-AVX2-O0-NEXT: vpshufb %xmm4, %xmm8, %xmm13
+; CHECK-AVX2-O0-NEXT: vxorps %xmm1, %xmm1, %xmm1
+; CHECK-AVX2-O0-NEXT: vpunpckhwd {{.*#+}} xmm12 = xmm1[4],xmm8[4],xmm1[5],xmm8[5],xmm1[6],xmm8[6],xmm1[7],xmm8[7]
+; CHECK-AVX2-O0-NEXT: vmovdqa {{.*#+}} xmm3 = [128,128,6,7,u,u,u,u,u,u,u,u,u,u,u,u]
+; CHECK-AVX2-O0-NEXT: vpshufb %xmm3, %xmm8, %xmm11
+; CHECK-AVX2-O0-NEXT: vmovdqa {{.*#+}} xmm2 = [128,128,4,5,u,u,u,u,u,u,u,u,u,u,u,u]
+; CHECK-AVX2-O0-NEXT: vpshufb %xmm2, %xmm8, %xmm10
+; CHECK-AVX2-O0-NEXT: vpblendw {{.*#+}} xmm9 = xmm1[0],xmm8[1],xmm1[2],xmm8[3],xmm1[4],xmm8[5],xmm1[6],xmm8[7]
; CHECK-AVX2-O0-NEXT: # kill: def $xmm0 killed $xmm0 killed $ymm0
-; CHECK-AVX2-O0-NEXT: vpextrw $7, %xmm0, %eax
-; CHECK-AVX2-O0-NEXT: shll $16, %eax
-; CHECK-AVX2-O0-NEXT: vmovd %eax, %xmm1
-; CHECK-AVX2-O0-NEXT: vmovss %xmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 4-byte Spill
-; CHECK-AVX2-O0-NEXT: vpextrw $6, %xmm0, %eax
-; CHECK-AVX2-O0-NEXT: shll $16, %eax
-; CHECK-AVX2-O0-NEXT: vmovd %eax, %xmm1
-; CHECK-AVX2-O0-NEXT: vmovss %xmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 4-byte Spill
-; CHECK-AVX2-O0-NEXT: vpextrw $5, %xmm0, %eax
-; CHECK-AVX2-O0-NEXT: shll $16, %eax
-; CHECK-AVX2-O0-NEXT: vmovd %eax, %xmm1
-; CHECK-AVX2-O0-NEXT: vmovss %xmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 4-byte Spill
-; CHECK-AVX2-O0-NEXT: vpextrw $4, %xmm0, %eax
-; CHECK-AVX2-O0-NEXT: shll $16, %eax
-; CHECK-AVX2-O0-NEXT: vmovd %eax, %xmm1
-; CHECK-AVX2-O0-NEXT: vmovss %xmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 4-byte Spill
-; CHECK-AVX2-O0-NEXT: vpextrw $3, %xmm0, %eax
-; CHECK-AVX2-O0-NEXT: shll $16, %eax
-; CHECK-AVX2-O0-NEXT: vmovd %eax, %xmm1
-; CHECK-AVX2-O0-NEXT: vmovss %xmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 4-byte Spill
-; CHECK-AVX2-O0-NEXT: vpextrw $2, %xmm0, %eax
-; CHECK-AVX2-O0-NEXT: shll $16, %eax
-; CHECK-AVX2-O0-NEXT: vmovd %eax, %xmm1
-; CHECK-AVX2-O0-NEXT: vmovss %xmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 4-byte Spill
-; CHECK-AVX2-O0-NEXT: vpextrw $1, %xmm0, %eax
-; CHECK-AVX2-O0-NEXT: shll $16, %eax
-; CHECK-AVX2-O0-NEXT: vmovd %eax, %xmm1
+; CHECK-AVX2-O0-NEXT: vpshufb %xmm6, %xmm0, %xmm7
+; CHECK-AVX2-O0-NEXT: vpshufb %xmm5, %xmm0, %xmm6
+; CHECK-AVX2-O0-NEXT: vpshufb %xmm4, %xmm0, %xmm5
+; CHECK-AVX2-O0-NEXT: vpunpckhwd {{.*#+}} xmm4 = xmm1[4],xmm0[4],xmm1[5],xmm0[5],xmm1[6],xmm0[6],xmm1[7],xmm0[7]
+; CHECK-AVX2-O0-NEXT: vpshufb %xmm3, %xmm0, %xmm3
+; CHECK-AVX2-O0-NEXT: vpshufb %xmm2, %xmm0, %xmm2
+; CHECK-AVX2-O0-NEXT: vpblendw {{.*#+}} xmm1 = xmm1[0],xmm0[1],xmm1[2],xmm0[3],xmm1[4],xmm0[5],xmm1[6],xmm0[7]
+; CHECK-AVX2-O0-NEXT: vpslld $16, %xmm8, %xmm8
+; CHECK-AVX2-O0-NEXT: vpslld $16, %xmm0, %xmm0
+; CHECK-AVX2-O0-NEXT: vmovss %xmm15, {{[-0-9]+}}(%r{{[sb]}}p) # 4-byte Spill
+; CHECK-AVX2-O0-NEXT: vmovss %xmm14, {{[-0-9]+}}(%r{{[sb]}}p) # 4-byte Spill
+; CHECK-AVX2-O0-NEXT: vmovss %xmm13, {{[-0-9]+}}(%r{{[sb]}}p) # 4-byte Spill
+; CHECK-AVX2-O0-NEXT: vmovss %xmm12, {{[-0-9]+}}(%r{{[sb]}}p) # 4-byte Spill
+; CHECK-AVX2-O0-NEXT: vmovss %xmm11, {{[-0-9]+}}(%r{{[sb]}}p) # 4-byte Spill
+; CHECK-AVX2-O0-NEXT: vmovss %xmm10, {{[-0-9]+}}(%r{{[sb]}}p) # 4-byte Spill
+; CHECK-AVX2-O0-NEXT: vmovss %xmm9, {{[-0-9]+}}(%r{{[sb]}}p) # 4-byte Spill
+; CHECK-AVX2-O0-NEXT: vmovss %xmm8, {{[-0-9]+}}(%r{{[sb]}}p) # 4-byte Spill
+; CHECK-AVX2-O0-NEXT: vmovss %xmm7, {{[-0-9]+}}(%r{{[sb]}}p) # 4-byte Spill
+; CHECK-AVX2-O0-NEXT: vmovss %xmm6, {{[-0-9]+}}(%r{{[sb]}}p) # 4-byte Spill
+; CHECK-AVX2-O0-NEXT: vmovss %xmm5, {{[-0-9]+}}(%r{{[sb]}}p) # 4-byte Spill
+; CHECK-AVX2-O0-NEXT: vmovss %xmm4, {{[-0-9]+}}(%r{{[sb]}}p) # 4-byte Spill
+; CHECK-AVX2-O0-NEXT: vmovss %xmm3, {{[-0-9]+}}(%r{{[sb]}}p) # 4-byte Spill
+; CHECK-AVX2-O0-NEXT: vmovss %xmm2, {{[-0-9]+}}(%r{{[sb]}}p) # 4-byte Spill
; CHECK-AVX2-O0-NEXT: vmovss %xmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 4-byte Spill
-; CHECK-AVX2-O0-NEXT: vmovd %xmm0, %eax
-; CHECK-AVX2-O0-NEXT: shll $16, %eax
-; CHECK-AVX2-O0-NEXT: vmovd %eax, %xmm0
; CHECK-AVX2-O0-NEXT: vzeroupper
; CHECK-AVX2-O0-NEXT: callq __truncsfbf2 at PLT
; CHECK-AVX2-O0-NEXT: vmovaps %xmm0, %xmm1
@@ -4146,71 +4096,45 @@ define void @store_atomic_vec16_bfloat_align(ptr %x, <16 x bfloat> %v) nounwind
; CHECK-AVX512-O0: # %bb.0:
; CHECK-AVX512-O0-NEXT: subq $120, %rsp
; CHECK-AVX512-O0-NEXT: movq %rdi, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Spill
-; CHECK-AVX512-O0-NEXT: vextractf128 $1, %ymm0, %xmm1
-; CHECK-AVX512-O0-NEXT: vpextrw $7, %xmm1, %eax
-; CHECK-AVX512-O0-NEXT: shll $16, %eax
-; CHECK-AVX512-O0-NEXT: vmovd %eax, %xmm2
-; CHECK-AVX512-O0-NEXT: vmovss %xmm2, {{[-0-9]+}}(%r{{[sb]}}p) # 4-byte Spill
-; CHECK-AVX512-O0-NEXT: vpextrw $6, %xmm1, %eax
-; CHECK-AVX512-O0-NEXT: shll $16, %eax
-; CHECK-AVX512-O0-NEXT: vmovd %eax, %xmm2
-; CHECK-AVX512-O0-NEXT: vmovss %xmm2, {{[-0-9]+}}(%r{{[sb]}}p) # 4-byte Spill
-; CHECK-AVX512-O0-NEXT: vpextrw $5, %xmm1, %eax
-; CHECK-AVX512-O0-NEXT: shll $16, %eax
-; CHECK-AVX512-O0-NEXT: vmovd %eax, %xmm2
-; CHECK-AVX512-O0-NEXT: vmovss %xmm2, {{[-0-9]+}}(%r{{[sb]}}p) # 4-byte Spill
-; CHECK-AVX512-O0-NEXT: vpextrw $4, %xmm1, %eax
-; CHECK-AVX512-O0-NEXT: shll $16, %eax
-; CHECK-AVX512-O0-NEXT: vmovd %eax, %xmm2
-; CHECK-AVX512-O0-NEXT: vmovss %xmm2, {{[-0-9]+}}(%r{{[sb]}}p) # 4-byte Spill
-; CHECK-AVX512-O0-NEXT: vpextrw $3, %xmm1, %eax
-; CHECK-AVX512-O0-NEXT: shll $16, %eax
-; CHECK-AVX512-O0-NEXT: vmovd %eax, %xmm2
-; CHECK-AVX512-O0-NEXT: vmovss %xmm2, {{[-0-9]+}}(%r{{[sb]}}p) # 4-byte Spill
-; CHECK-AVX512-O0-NEXT: vpextrw $2, %xmm1, %eax
-; CHECK-AVX512-O0-NEXT: shll $16, %eax
-; CHECK-AVX512-O0-NEXT: vmovd %eax, %xmm2
-; CHECK-AVX512-O0-NEXT: vmovss %xmm2, {{[-0-9]+}}(%r{{[sb]}}p) # 4-byte Spill
-; CHECK-AVX512-O0-NEXT: vpextrw $1, %xmm1, %eax
-; CHECK-AVX512-O0-NEXT: shll $16, %eax
-; CHECK-AVX512-O0-NEXT: vmovd %eax, %xmm2
-; CHECK-AVX512-O0-NEXT: vmovss %xmm2, {{[-0-9]+}}(%r{{[sb]}}p) # 4-byte Spill
-; CHECK-AVX512-O0-NEXT: vmovd %xmm1, %eax
-; CHECK-AVX512-O0-NEXT: shll $16, %eax
-; CHECK-AVX512-O0-NEXT: vmovd %eax, %xmm1
-; CHECK-AVX512-O0-NEXT: vmovss %xmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 4-byte Spill
+; CHECK-AVX512-O0-NEXT: vextractf128 $1, %ymm0, %xmm8
+; CHECK-AVX512-O0-NEXT: vmovdqa {{.*#+}} xmm6 = [128,128,14,15,u,u,u,u,u,u,u,u,u,u,u,u]
+; CHECK-AVX512-O0-NEXT: vpshufb %xmm6, %xmm8, %xmm15
+; CHECK-AVX512-O0-NEXT: vmovdqa {{.*#+}} xmm5 = [128,128,12,13,u,u,u,u,u,u,u,u,u,u,u,u]
+; CHECK-AVX512-O0-NEXT: vpshufb %xmm5, %xmm8, %xmm14
+; CHECK-AVX512-O0-NEXT: vmovdqa {{.*#+}} xmm4 = [128,128,10,11,u,u,u,u,u,u,u,u,u,u,u,u]
+; CHECK-AVX512-O0-NEXT: vpshufb %xmm4, %xmm8, %xmm13
+; CHECK-AVX512-O0-NEXT: vpxor %xmm1, %xmm1, %xmm1
+; CHECK-AVX512-O0-NEXT: vpunpckhwd {{.*#+}} xmm12 = xmm1[4],xmm8[4],xmm1[5],xmm8[5],xmm1[6],xmm8[6],xmm1[7],xmm8[7]
+; CHECK-AVX512-O0-NEXT: vmovdqa {{.*#+}} xmm3 = [128,128,6,7,u,u,u,u,u,u,u,u,u,u,u,u]
+; CHECK-AVX512-O0-NEXT: vpshufb %xmm3, %xmm8, %xmm11
+; CHECK-AVX512-O0-NEXT: vmovdqa {{.*#+}} xmm2 = [128,128,4,5,u,u,u,u,u,u,u,u,u,u,u,u]
+; CHECK-AVX512-O0-NEXT: vpshufb %xmm2, %xmm8, %xmm10
+; CHECK-AVX512-O0-NEXT: vpblendw {{.*#+}} xmm9 = xmm1[0],xmm8[1],xmm1[2],xmm8[3],xmm1[4],xmm8[5],xmm1[6],xmm8[7]
; CHECK-AVX512-O0-NEXT: # kill: def $xmm0 killed $xmm0 killed $ymm0
-; CHECK-AVX512-O0-NEXT: vpextrw $7, %xmm0, %eax
-; CHECK-AVX512-O0-NEXT: shll $16, %eax
-; CHECK-AVX512-O0-NEXT: vmovd %eax, %xmm1
-; CHECK-AVX512-O0-NEXT: vmovss %xmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 4-byte Spill
-; CHECK-AVX512-O0-NEXT: vpextrw $6, %xmm0, %eax
-; CHECK-AVX512-O0-NEXT: shll $16, %eax
-; CHECK-AVX512-O0-NEXT: vmovd %eax, %xmm1
-; CHECK-AVX512-O0-NEXT: vmovss %xmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 4-byte Spill
-; CHECK-AVX512-O0-NEXT: vpextrw $5, %xmm0, %eax
-; CHECK-AVX512-O0-NEXT: shll $16, %eax
-; CHECK-AVX512-O0-NEXT: vmovd %eax, %xmm1
-; CHECK-AVX512-O0-NEXT: vmovss %xmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 4-byte Spill
-; CHECK-AVX512-O0-NEXT: vpextrw $4, %xmm0, %eax
-; CHECK-AVX512-O0-NEXT: shll $16, %eax
-; CHECK-AVX512-O0-NEXT: vmovd %eax, %xmm1
-; CHECK-AVX512-O0-NEXT: vmovss %xmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 4-byte Spill
-; CHECK-AVX512-O0-NEXT: vpextrw $3, %xmm0, %eax
-; CHECK-AVX512-O0-NEXT: shll $16, %eax
-; CHECK-AVX512-O0-NEXT: vmovd %eax, %xmm1
-; CHECK-AVX512-O0-NEXT: vmovss %xmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 4-byte Spill
-; CHECK-AVX512-O0-NEXT: vpextrw $2, %xmm0, %eax
-; CHECK-AVX512-O0-NEXT: shll $16, %eax
-; CHECK-AVX512-O0-NEXT: vmovd %eax, %xmm1
-; CHECK-AVX512-O0-NEXT: vmovss %xmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 4-byte Spill
-; CHECK-AVX512-O0-NEXT: vpextrw $1, %xmm0, %eax
-; CHECK-AVX512-O0-NEXT: shll $16, %eax
-; CHECK-AVX512-O0-NEXT: vmovd %eax, %xmm1
+; CHECK-AVX512-O0-NEXT: vpshufb %xmm6, %xmm0, %xmm7
+; CHECK-AVX512-O0-NEXT: vpshufb %xmm5, %xmm0, %xmm6
+; CHECK-AVX512-O0-NEXT: vpshufb %xmm4, %xmm0, %xmm5
+; CHECK-AVX512-O0-NEXT: vpunpckhwd {{.*#+}} xmm4 = xmm1[4],xmm0[4],xmm1[5],xmm0[5],xmm1[6],xmm0[6],xmm1[7],xmm0[7]
+; CHECK-AVX512-O0-NEXT: vpshufb %xmm3, %xmm0, %xmm3
+; CHECK-AVX512-O0-NEXT: vpshufb %xmm2, %xmm0, %xmm2
+; CHECK-AVX512-O0-NEXT: vpblendw {{.*#+}} xmm1 = xmm1[0],xmm0[1],xmm1[2],xmm0[3],xmm1[4],xmm0[5],xmm1[6],xmm0[7]
+; CHECK-AVX512-O0-NEXT: vpslld $16, %xmm8, %xmm8
+; CHECK-AVX512-O0-NEXT: vpslld $16, %xmm0, %xmm0
+; CHECK-AVX512-O0-NEXT: vmovss %xmm15, {{[-0-9]+}}(%r{{[sb]}}p) # 4-byte Spill
+; CHECK-AVX512-O0-NEXT: vmovss %xmm14, {{[-0-9]+}}(%r{{[sb]}}p) # 4-byte Spill
+; CHECK-AVX512-O0-NEXT: vmovss %xmm13, {{[-0-9]+}}(%r{{[sb]}}p) # 4-byte Spill
+; CHECK-AVX512-O0-NEXT: vmovss %xmm12, {{[-0-9]+}}(%r{{[sb]}}p) # 4-byte Spill
+; CHECK-AVX512-O0-NEXT: vmovss %xmm11, {{[-0-9]+}}(%r{{[sb]}}p) # 4-byte Spill
+; CHECK-AVX512-O0-NEXT: vmovss %xmm10, {{[-0-9]+}}(%r{{[sb]}}p) # 4-byte Spill
+; CHECK-AVX512-O0-NEXT: vmovss %xmm9, {{[-0-9]+}}(%r{{[sb]}}p) # 4-byte Spill
+; CHECK-AVX512-O0-NEXT: vmovss %xmm8, {{[-0-9]+}}(%r{{[sb]}}p) # 4-byte Spill
+; CHECK-AVX512-O0-NEXT: vmovss %xmm7, {{[-0-9]+}}(%r{{[sb]}}p) # 4-byte Spill
+; CHECK-AVX512-O0-NEXT: vmovss %xmm6, {{[-0-9]+}}(%r{{[sb]}}p) # 4-byte Spill
+; CHECK-AVX512-O0-NEXT: vmovss %xmm5, {{[-0-9]+}}(%r{{[sb]}}p) # 4-byte Spill
+; CHECK-AVX512-O0-NEXT: vmovss %xmm4, {{[-0-9]+}}(%r{{[sb]}}p) # 4-byte Spill
+; CHECK-AVX512-O0-NEXT: vmovss %xmm3, {{[-0-9]+}}(%r{{[sb]}}p) # 4-byte Spill
+; CHECK-AVX512-O0-NEXT: vmovss %xmm2, {{[-0-9]+}}(%r{{[sb]}}p) # 4-byte Spill
; CHECK-AVX512-O0-NEXT: vmovss %xmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 4-byte Spill
-; CHECK-AVX512-O0-NEXT: vmovd %xmm0, %eax
-; CHECK-AVX512-O0-NEXT: shll $16, %eax
-; CHECK-AVX512-O0-NEXT: vmovd %eax, %xmm0
; CHECK-AVX512-O0-NEXT: vzeroupper
; CHECK-AVX512-O0-NEXT: callq __truncsfbf2 at PLT
; CHECK-AVX512-O0-NEXT: vmovaps %xmm0, %xmm1
diff --git a/llvm/test/CodeGen/X86/avx10_2bf16-arith-scalar.ll b/llvm/test/CodeGen/X86/avx10_2bf16-arith-scalar.ll
index cc7f3ad0eb174..16f7f21f19cf0 100644
--- a/llvm/test/CodeGen/X86/avx10_2bf16-arith-scalar.ll
+++ b/llvm/test/CodeGen/X86/avx10_2bf16-arith-scalar.ll
@@ -11,25 +11,17 @@ define bfloat @fadd_bf16(bfloat %a, bfloat %b) nounwind {
;
; AVX512BF16-LABEL: fadd_bf16:
; AVX512BF16: # %bb.0: # %entry
-; AVX512BF16-NEXT: vpextrw $0, %xmm0, %eax
-; AVX512BF16-NEXT: vpextrw $0, %xmm1, %ecx
-; AVX512BF16-NEXT: shll $16, %ecx
-; AVX512BF16-NEXT: vmovd %ecx, %xmm0
-; AVX512BF16-NEXT: shll $16, %eax
-; AVX512BF16-NEXT: vmovd %eax, %xmm1
-; AVX512BF16-NEXT: vaddss %xmm0, %xmm1, %xmm0
+; AVX512BF16-NEXT: vpslld $16, %xmm1, %xmm1
+; AVX512BF16-NEXT: vpslld $16, %xmm0, %xmm0
+; AVX512BF16-NEXT: vaddss %xmm1, %xmm0, %xmm0
; AVX512BF16-NEXT: vcvtneps2bf16 %xmm0, %xmm0
; AVX512BF16-NEXT: retq
;
; AVXNECONVERT-LABEL: fadd_bf16:
; AVXNECONVERT: # %bb.0: # %entry
-; AVXNECONVERT-NEXT: vpextrw $0, %xmm0, %eax
-; AVXNECONVERT-NEXT: vpextrw $0, %xmm1, %ecx
-; AVXNECONVERT-NEXT: shll $16, %ecx
-; AVXNECONVERT-NEXT: vmovd %ecx, %xmm0
-; AVXNECONVERT-NEXT: shll $16, %eax
-; AVXNECONVERT-NEXT: vmovd %eax, %xmm1
-; AVXNECONVERT-NEXT: vaddss %xmm0, %xmm1, %xmm0
+; AVXNECONVERT-NEXT: vpslld $16, %xmm1, %xmm1
+; AVXNECONVERT-NEXT: vpslld $16, %xmm0, %xmm0
+; AVXNECONVERT-NEXT: vaddss %xmm1, %xmm0, %xmm0
; AVXNECONVERT-NEXT: {vex} vcvtneps2bf16 %xmm0, %xmm0
; AVXNECONVERT-NEXT: retq
entry:
@@ -47,40 +39,24 @@ define bfloat @dont_fuse_bf16(bfloat %a, bfloat %b, bfloat %c) nounwind {
;
; AVX512BF16-LABEL: dont_fuse_bf16:
; AVX512BF16: # %bb.0: # %entry
-; AVX512BF16-NEXT: vpextrw $0, %xmm2, %eax
-; AVX512BF16-NEXT: vpextrw $0, %xmm0, %ecx
-; AVX512BF16-NEXT: vpextrw $0, %xmm1, %edx
-; AVX512BF16-NEXT: shll $16, %edx
-; AVX512BF16-NEXT: vmovd %edx, %xmm0
-; AVX512BF16-NEXT: shll $16, %ecx
-; AVX512BF16-NEXT: vmovd %ecx, %xmm1
-; AVX512BF16-NEXT: vmulss %xmm0, %xmm1, %xmm0
+; AVX512BF16-NEXT: vpslld $16, %xmm1, %xmm1
+; AVX512BF16-NEXT: vpslld $16, %xmm0, %xmm0
+; AVX512BF16-NEXT: vmulss %xmm1, %xmm0, %xmm0
; AVX512BF16-NEXT: vcvtneps2bf16 %xmm0, %xmm0
-; AVX512BF16-NEXT: vmovd %xmm0, %ecx
-; AVX512BF16-NEXT: shll $16, %ecx
-; AVX512BF16-NEXT: vmovd %ecx, %xmm0
-; AVX512BF16-NEXT: shll $16, %eax
-; AVX512BF16-NEXT: vmovd %eax, %xmm1
+; AVX512BF16-NEXT: vpslld $16, %xmm0, %xmm0
+; AVX512BF16-NEXT: vpslld $16, %xmm2, %xmm1
; AVX512BF16-NEXT: vaddss %xmm1, %xmm0, %xmm0
; AVX512BF16-NEXT: vcvtneps2bf16 %xmm0, %xmm0
; AVX512BF16-NEXT: retq
;
; AVXNECONVERT-LABEL: dont_fuse_bf16:
; AVXNECONVERT: # %bb.0: # %entry
-; AVXNECONVERT-NEXT: vpextrw $0, %xmm2, %eax
-; AVXNECONVERT-NEXT: vpextrw $0, %xmm0, %ecx
-; AVXNECONVERT-NEXT: vpextrw $0, %xmm1, %edx
-; AVXNECONVERT-NEXT: shll $16, %edx
-; AVXNECONVERT-NEXT: vmovd %edx, %xmm0
-; AVXNECONVERT-NEXT: shll $16, %ecx
-; AVXNECONVERT-NEXT: vmovd %ecx, %xmm1
-; AVXNECONVERT-NEXT: vmulss %xmm0, %xmm1, %xmm0
+; AVXNECONVERT-NEXT: vpslld $16, %xmm1, %xmm1
+; AVXNECONVERT-NEXT: vpslld $16, %xmm0, %xmm0
+; AVXNECONVERT-NEXT: vmulss %xmm1, %xmm0, %xmm0
; AVXNECONVERT-NEXT: {vex} vcvtneps2bf16 %xmm0, %xmm0
-; AVXNECONVERT-NEXT: vmovd %xmm0, %ecx
-; AVXNECONVERT-NEXT: shll $16, %ecx
-; AVXNECONVERT-NEXT: vmovd %ecx, %xmm0
-; AVXNECONVERT-NEXT: shll $16, %eax
-; AVXNECONVERT-NEXT: vmovd %eax, %xmm1
+; AVXNECONVERT-NEXT: vpslld $16, %xmm0, %xmm0
+; AVXNECONVERT-NEXT: vpslld $16, %xmm2, %xmm1
; AVXNECONVERT-NEXT: vaddss %xmm1, %xmm0, %xmm0
; AVXNECONVERT-NEXT: {vex} vcvtneps2bf16 %xmm0, %xmm0
; AVXNECONVERT-NEXT: retq
@@ -98,25 +74,17 @@ define bfloat @fsub_bf16(bfloat %a, bfloat %b) nounwind {
;
; AVX512BF16-LABEL: fsub_bf16:
; AVX512BF16: # %bb.0: # %entry
-; AVX512BF16-NEXT: vpextrw $0, %xmm0, %eax
-; AVX512BF16-NEXT: vpextrw $0, %xmm1, %ecx
-; AVX512BF16-NEXT: shll $16, %ecx
-; AVX512BF16-NEXT: vmovd %ecx, %xmm0
-; AVX512BF16-NEXT: shll $16, %eax
-; AVX512BF16-NEXT: vmovd %eax, %xmm1
-; AVX512BF16-NEXT: vsubss %xmm0, %xmm1, %xmm0
+; AVX512BF16-NEXT: vpslld $16, %xmm1, %xmm1
+; AVX512BF16-NEXT: vpslld $16, %xmm0, %xmm0
+; AVX512BF16-NEXT: vsubss %xmm1, %xmm0, %xmm0
; AVX512BF16-NEXT: vcvtneps2bf16 %xmm0, %xmm0
; AVX512BF16-NEXT: retq
;
; AVXNECONVERT-LABEL: fsub_bf16:
; AVXNECONVERT: # %bb.0: # %entry
-; AVXNECONVERT-NEXT: vpextrw $0, %xmm0, %eax
-; AVXNECONVERT-NEXT: vpextrw $0, %xmm1, %ecx
-; AVXNECONVERT-NEXT: shll $16, %ecx
-; AVXNECONVERT-NEXT: vmovd %ecx, %xmm0
-; AVXNECONVERT-NEXT: shll $16, %eax
-; AVXNECONVERT-NEXT: vmovd %eax, %xmm1
-; AVXNECONVERT-NEXT: vsubss %xmm0, %xmm1, %xmm0
+; AVXNECONVERT-NEXT: vpslld $16, %xmm1, %xmm1
+; AVXNECONVERT-NEXT: vpslld $16, %xmm0, %xmm0
+; AVXNECONVERT-NEXT: vsubss %xmm1, %xmm0, %xmm0
; AVXNECONVERT-NEXT: {vex} vcvtneps2bf16 %xmm0, %xmm0
; AVXNECONVERT-NEXT: retq
entry:
@@ -132,25 +100,17 @@ define bfloat @fmul_bf16(bfloat %a, bfloat %b) nounwind {
;
; AVX512BF16-LABEL: fmul_bf16:
; AVX512BF16: # %bb.0: # %entry
-; AVX512BF16-NEXT: vpextrw $0, %xmm0, %eax
-; AVX512BF16-NEXT: vpextrw $0, %xmm1, %ecx
-; AVX512BF16-NEXT: shll $16, %ecx
-; AVX512BF16-NEXT: vmovd %ecx, %xmm0
-; AVX512BF16-NEXT: shll $16, %eax
-; AVX512BF16-NEXT: vmovd %eax, %xmm1
-; AVX512BF16-NEXT: vmulss %xmm0, %xmm1, %xmm0
+; AVX512BF16-NEXT: vpslld $16, %xmm1, %xmm1
+; AVX512BF16-NEXT: vpslld $16, %xmm0, %xmm0
+; AVX512BF16-NEXT: vmulss %xmm1, %xmm0, %xmm0
; AVX512BF16-NEXT: vcvtneps2bf16 %xmm0, %xmm0
; AVX512BF16-NEXT: retq
;
; AVXNECONVERT-LABEL: fmul_bf16:
; AVXNECONVERT: # %bb.0: # %entry
-; AVXNECONVERT-NEXT: vpextrw $0, %xmm0, %eax
-; AVXNECONVERT-NEXT: vpextrw $0, %xmm1, %ecx
-; AVXNECONVERT-NEXT: shll $16, %ecx
-; AVXNECONVERT-NEXT: vmovd %ecx, %xmm0
-; AVXNECONVERT-NEXT: shll $16, %eax
-; AVXNECONVERT-NEXT: vmovd %eax, %xmm1
-; AVXNECONVERT-NEXT: vmulss %xmm0, %xmm1, %xmm0
+; AVXNECONVERT-NEXT: vpslld $16, %xmm1, %xmm1
+; AVXNECONVERT-NEXT: vpslld $16, %xmm0, %xmm0
+; AVXNECONVERT-NEXT: vmulss %xmm1, %xmm0, %xmm0
; AVXNECONVERT-NEXT: {vex} vcvtneps2bf16 %xmm0, %xmm0
; AVXNECONVERT-NEXT: retq
entry:
@@ -166,25 +126,17 @@ define bfloat @fdiv_bf16(bfloat %a, bfloat %b) nounwind {
;
; AVX512BF16-LABEL: fdiv_bf16:
; AVX512BF16: # %bb.0: # %entry
-; AVX512BF16-NEXT: vpextrw $0, %xmm0, %eax
-; AVX512BF16-NEXT: vpextrw $0, %xmm1, %ecx
-; AVX512BF16-NEXT: shll $16, %ecx
-; AVX512BF16-NEXT: vmovd %ecx, %xmm0
-; AVX512BF16-NEXT: shll $16, %eax
-; AVX512BF16-NEXT: vmovd %eax, %xmm1
-; AVX512BF16-NEXT: vdivss %xmm0, %xmm1, %xmm0
+; AVX512BF16-NEXT: vpslld $16, %xmm1, %xmm1
+; AVX512BF16-NEXT: vpslld $16, %xmm0, %xmm0
+; AVX512BF16-NEXT: vdivss %xmm1, %xmm0, %xmm0
; AVX512BF16-NEXT: vcvtneps2bf16 %xmm0, %xmm0
; AVX512BF16-NEXT: retq
;
; AVXNECONVERT-LABEL: fdiv_bf16:
; AVXNECONVERT: # %bb.0: # %entry
-; AVXNECONVERT-NEXT: vpextrw $0, %xmm0, %eax
-; AVXNECONVERT-NEXT: vpextrw $0, %xmm1, %ecx
-; AVXNECONVERT-NEXT: shll $16, %ecx
-; AVXNECONVERT-NEXT: vmovd %ecx, %xmm0
-; AVXNECONVERT-NEXT: shll $16, %eax
-; AVXNECONVERT-NEXT: vmovd %eax, %xmm1
-; AVXNECONVERT-NEXT: vdivss %xmm0, %xmm1, %xmm0
+; AVXNECONVERT-NEXT: vpslld $16, %xmm1, %xmm1
+; AVXNECONVERT-NEXT: vpslld $16, %xmm0, %xmm0
+; AVXNECONVERT-NEXT: vdivss %xmm1, %xmm0, %xmm0
; AVXNECONVERT-NEXT: {vex} vcvtneps2bf16 %xmm0, %xmm0
; AVXNECONVERT-NEXT: retq
entry:
@@ -200,18 +152,14 @@ define bfloat @fsqrt_bf16(bfloat %a) nounwind {
;
; AVX512BF16-LABEL: fsqrt_bf16:
; AVX512BF16: # %bb.0: # %entry
-; AVX512BF16-NEXT: vpextrw $0, %xmm0, %eax
-; AVX512BF16-NEXT: shll $16, %eax
-; AVX512BF16-NEXT: vmovd %eax, %xmm0
+; AVX512BF16-NEXT: vpslld $16, %xmm0, %xmm0
; AVX512BF16-NEXT: vsqrtss %xmm0, %xmm0, %xmm0
; AVX512BF16-NEXT: vcvtneps2bf16 %xmm0, %xmm0
; AVX512BF16-NEXT: retq
;
; AVXNECONVERT-LABEL: fsqrt_bf16:
; AVXNECONVERT: # %bb.0: # %entry
-; AVXNECONVERT-NEXT: vpextrw $0, %xmm0, %eax
-; AVXNECONVERT-NEXT: shll $16, %eax
-; AVXNECONVERT-NEXT: vmovd %eax, %xmm0
+; AVXNECONVERT-NEXT: vpslld $16, %xmm0, %xmm0
; AVXNECONVERT-NEXT: vsqrtss %xmm0, %xmm0, %xmm0
; AVXNECONVERT-NEXT: {vex} vcvtneps2bf16 %xmm0, %xmm0
; AVXNECONVERT-NEXT: retq
diff --git a/llvm/test/CodeGen/X86/avx10_2bf16-fma.ll b/llvm/test/CodeGen/X86/avx10_2bf16-fma.ll
index 7ca17d81e59f1..d65c85e8c547c 100644
--- a/llvm/test/CodeGen/X86/avx10_2bf16-fma.ll
+++ b/llvm/test/CodeGen/X86/avx10_2bf16-fma.ll
@@ -11,35 +11,21 @@ define bfloat @fuse_bf16(bfloat %a, bfloat %b, bfloat %c) nounwind {
;
; AVX512BF16-LABEL: fuse_bf16:
; AVX512BF16: # %bb.0: # %entry
-; AVX512BF16-NEXT: vpextrw $0, %xmm1, %eax
-; AVX512BF16-NEXT: vpextrw $0, %xmm0, %ecx
-; AVX512BF16-NEXT: vpextrw $0, %xmm2, %edx
-; AVX512BF16-NEXT: shll $16, %edx
-; AVX512BF16-NEXT: vmovd %edx, %xmm0
-; AVX512BF16-NEXT: shll $16, %ecx
-; AVX512BF16-NEXT: vmovd %ecx, %xmm1
-; AVX512BF16-NEXT: shll $16, %eax
-; AVX512BF16-NEXT: vmovd %eax, %xmm2
-; AVX512BF16-NEXT: vfmadd213ss {{.*#+}} xmm2 = (xmm1 * xmm2) + xmm0
-; AVX512BF16-NEXT: vcvtneps2bf16 %xmm2, %xmm0
+; AVX512BF16-NEXT: vpslld $16, %xmm2, %xmm2
+; AVX512BF16-NEXT: vpslld $16, %xmm0, %xmm0
+; AVX512BF16-NEXT: vpslld $16, %xmm1, %xmm1
+; AVX512BF16-NEXT: vfmadd213ss {{.*#+}} xmm1 = (xmm0 * xmm1) + xmm2
+; AVX512BF16-NEXT: vcvtneps2bf16 %xmm1, %xmm0
; AVX512BF16-NEXT: retq
;
; AVXNECONVERT-LABEL: fuse_bf16:
; AVXNECONVERT: # %bb.0: # %entry
-; AVXNECONVERT-NEXT: vpextrw $0, %xmm2, %eax
-; AVXNECONVERT-NEXT: vpextrw $0, %xmm0, %ecx
-; AVXNECONVERT-NEXT: vpextrw $0, %xmm1, %edx
-; AVXNECONVERT-NEXT: shll $16, %edx
-; AVXNECONVERT-NEXT: vmovd %edx, %xmm0
-; AVXNECONVERT-NEXT: shll $16, %ecx
-; AVXNECONVERT-NEXT: vmovd %ecx, %xmm1
-; AVXNECONVERT-NEXT: vmulss %xmm0, %xmm1, %xmm0
+; AVXNECONVERT-NEXT: vpslld $16, %xmm1, %xmm1
+; AVXNECONVERT-NEXT: vpslld $16, %xmm0, %xmm0
+; AVXNECONVERT-NEXT: vmulss %xmm1, %xmm0, %xmm0
; AVXNECONVERT-NEXT: {vex} vcvtneps2bf16 %xmm0, %xmm0
-; AVXNECONVERT-NEXT: vmovd %xmm0, %ecx
-; AVXNECONVERT-NEXT: shll $16, %ecx
-; AVXNECONVERT-NEXT: vmovd %ecx, %xmm0
-; AVXNECONVERT-NEXT: shll $16, %eax
-; AVXNECONVERT-NEXT: vmovd %eax, %xmm1
+; AVXNECONVERT-NEXT: vpslld $16, %xmm0, %xmm0
+; AVXNECONVERT-NEXT: vpslld $16, %xmm2, %xmm1
; AVXNECONVERT-NEXT: vaddss %xmm1, %xmm0, %xmm0
; AVXNECONVERT-NEXT: {vex} vcvtneps2bf16 %xmm0, %xmm0
; AVXNECONVERT-NEXT: retq
@@ -74,132 +60,21 @@ define <8 x bfloat> @fuse_v8bf16(<8 x bfloat> %x, <8 x bfloat> %y, <8 x bfloat>
;
; AVXNECONVERT-LABEL: fuse_v8bf16:
; AVXNECONVERT: # %bb.0: # %entry
-; AVXNECONVERT-NEXT: vpextrw $5, %xmm1, %eax
-; AVXNECONVERT-NEXT: shll $16, %eax
-; AVXNECONVERT-NEXT: vmovd %eax, %xmm3
-; AVXNECONVERT-NEXT: vpextrw $4, %xmm1, %eax
-; AVXNECONVERT-NEXT: shll $16, %eax
-; AVXNECONVERT-NEXT: vmovd %eax, %xmm4
-; AVXNECONVERT-NEXT: vinsertps {{.*#+}} xmm3 = xmm4[0],xmm3[0],xmm4[2,3]
-; AVXNECONVERT-NEXT: vpextrw $6, %xmm1, %eax
-; AVXNECONVERT-NEXT: shll $16, %eax
-; AVXNECONVERT-NEXT: vmovd %eax, %xmm4
-; AVXNECONVERT-NEXT: vinsertps {{.*#+}} xmm3 = xmm3[0,1],xmm4[0],xmm3[3]
-; AVXNECONVERT-NEXT: vpextrw $7, %xmm1, %eax
-; AVXNECONVERT-NEXT: shll $16, %eax
-; AVXNECONVERT-NEXT: vmovd %eax, %xmm4
-; AVXNECONVERT-NEXT: vinsertps {{.*#+}} xmm3 = xmm3[0,1,2],xmm4[0]
-; AVXNECONVERT-NEXT: vpextrw $1, %xmm1, %eax
-; AVXNECONVERT-NEXT: shll $16, %eax
-; AVXNECONVERT-NEXT: vmovd %eax, %xmm4
-; AVXNECONVERT-NEXT: vmovd %xmm1, %eax
-; AVXNECONVERT-NEXT: shll $16, %eax
-; AVXNECONVERT-NEXT: vmovd %eax, %xmm5
-; AVXNECONVERT-NEXT: vinsertps {{.*#+}} xmm4 = xmm5[0],xmm4[0],xmm5[2,3]
-; AVXNECONVERT-NEXT: vpextrw $2, %xmm1, %eax
-; AVXNECONVERT-NEXT: shll $16, %eax
-; AVXNECONVERT-NEXT: vmovd %eax, %xmm5
-; AVXNECONVERT-NEXT: vinsertps {{.*#+}} xmm4 = xmm4[0,1],xmm5[0],xmm4[3]
-; AVXNECONVERT-NEXT: vpextrw $3, %xmm1, %eax
-; AVXNECONVERT-NEXT: shll $16, %eax
-; AVXNECONVERT-NEXT: vmovd %eax, %xmm1
-; AVXNECONVERT-NEXT: vinsertps {{.*#+}} xmm1 = xmm4[0,1,2],xmm1[0]
-; AVXNECONVERT-NEXT: vinsertf128 $1, %xmm3, %ymm1, %ymm1
-; AVXNECONVERT-NEXT: vpextrw $5, %xmm0, %eax
-; AVXNECONVERT-NEXT: shll $16, %eax
-; AVXNECONVERT-NEXT: vmovd %eax, %xmm3
-; AVXNECONVERT-NEXT: vpextrw $4, %xmm0, %eax
-; AVXNECONVERT-NEXT: shll $16, %eax
-; AVXNECONVERT-NEXT: vmovd %eax, %xmm4
-; AVXNECONVERT-NEXT: vinsertps {{.*#+}} xmm3 = xmm4[0],xmm3[0],xmm4[2,3]
-; AVXNECONVERT-NEXT: vpextrw $6, %xmm0, %eax
-; AVXNECONVERT-NEXT: shll $16, %eax
-; AVXNECONVERT-NEXT: vmovd %eax, %xmm4
-; AVXNECONVERT-NEXT: vinsertps {{.*#+}} xmm3 = xmm3[0,1],xmm4[0],xmm3[3]
-; AVXNECONVERT-NEXT: vpextrw $7, %xmm0, %eax
-; AVXNECONVERT-NEXT: shll $16, %eax
-; AVXNECONVERT-NEXT: vmovd %eax, %xmm4
-; AVXNECONVERT-NEXT: vinsertps {{.*#+}} xmm3 = xmm3[0,1,2],xmm4[0]
-; AVXNECONVERT-NEXT: vpextrw $1, %xmm0, %eax
-; AVXNECONVERT-NEXT: shll $16, %eax
-; AVXNECONVERT-NEXT: vmovd %eax, %xmm4
-; AVXNECONVERT-NEXT: vmovd %xmm0, %eax
-; AVXNECONVERT-NEXT: shll $16, %eax
-; AVXNECONVERT-NEXT: vmovd %eax, %xmm5
-; AVXNECONVERT-NEXT: vinsertps {{.*#+}} xmm4 = xmm5[0],xmm4[0],xmm5[2,3]
-; AVXNECONVERT-NEXT: vpextrw $2, %xmm0, %eax
-; AVXNECONVERT-NEXT: shll $16, %eax
-; AVXNECONVERT-NEXT: vmovd %eax, %xmm5
-; AVXNECONVERT-NEXT: vinsertps {{.*#+}} xmm4 = xmm4[0,1],xmm5[0],xmm4[3]
-; AVXNECONVERT-NEXT: vpextrw $3, %xmm0, %eax
-; AVXNECONVERT-NEXT: shll $16, %eax
-; AVXNECONVERT-NEXT: vmovd %eax, %xmm0
-; AVXNECONVERT-NEXT: vinsertps {{.*#+}} xmm0 = xmm4[0,1,2],xmm0[0]
-; AVXNECONVERT-NEXT: vinsertf128 $1, %xmm3, %ymm0, %ymm0
+; AVXNECONVERT-NEXT: vpxor %xmm3, %xmm3, %xmm3
+; AVXNECONVERT-NEXT: vpunpckhwd {{.*#+}} xmm4 = xmm3[4],xmm1[4],xmm3[5],xmm1[5],xmm3[6],xmm1[6],xmm3[7],xmm1[7]
+; AVXNECONVERT-NEXT: vpunpcklwd {{.*#+}} xmm1 = xmm3[0],xmm1[0],xmm3[1],xmm1[1],xmm3[2],xmm1[2],xmm3[3],xmm1[3]
+; AVXNECONVERT-NEXT: vinserti128 $1, %xmm4, %ymm1, %ymm1
+; AVXNECONVERT-NEXT: vpunpckhwd {{.*#+}} xmm4 = xmm3[4],xmm0[4],xmm3[5],xmm0[5],xmm3[6],xmm0[6],xmm3[7],xmm0[7]
+; AVXNECONVERT-NEXT: vpunpcklwd {{.*#+}} xmm0 = xmm3[0],xmm0[0],xmm3[1],xmm0[1],xmm3[2],xmm0[2],xmm3[3],xmm0[3]
+; AVXNECONVERT-NEXT: vinserti128 $1, %xmm4, %ymm0, %ymm0
; AVXNECONVERT-NEXT: vmulps %ymm1, %ymm0, %ymm0
; AVXNECONVERT-NEXT: {vex} vcvtneps2bf16 %ymm0, %xmm0
-; AVXNECONVERT-NEXT: vpextrw $5, %xmm0, %eax
-; AVXNECONVERT-NEXT: shll $16, %eax
-; AVXNECONVERT-NEXT: vmovd %eax, %xmm1
-; AVXNECONVERT-NEXT: vpextrw $4, %xmm0, %eax
-; AVXNECONVERT-NEXT: shll $16, %eax
-; AVXNECONVERT-NEXT: vmovd %eax, %xmm3
-; AVXNECONVERT-NEXT: vinsertps {{.*#+}} xmm1 = xmm3[0],xmm1[0],xmm3[2,3]
-; AVXNECONVERT-NEXT: vpextrw $6, %xmm0, %eax
-; AVXNECONVERT-NEXT: shll $16, %eax
-; AVXNECONVERT-NEXT: vmovd %eax, %xmm3
-; AVXNECONVERT-NEXT: vinsertps {{.*#+}} xmm1 = xmm1[0,1],xmm3[0],xmm1[3]
-; AVXNECONVERT-NEXT: vpextrw $7, %xmm0, %eax
-; AVXNECONVERT-NEXT: shll $16, %eax
-; AVXNECONVERT-NEXT: vmovd %eax, %xmm3
-; AVXNECONVERT-NEXT: vinsertps {{.*#+}} xmm1 = xmm1[0,1,2],xmm3[0]
-; AVXNECONVERT-NEXT: vpextrw $1, %xmm0, %eax
-; AVXNECONVERT-NEXT: shll $16, %eax
-; AVXNECONVERT-NEXT: vmovd %eax, %xmm3
-; AVXNECONVERT-NEXT: vmovd %xmm0, %eax
-; AVXNECONVERT-NEXT: shll $16, %eax
-; AVXNECONVERT-NEXT: vmovd %eax, %xmm4
-; AVXNECONVERT-NEXT: vinsertps {{.*#+}} xmm3 = xmm4[0],xmm3[0],xmm4[2,3]
-; AVXNECONVERT-NEXT: vpextrw $2, %xmm0, %eax
-; AVXNECONVERT-NEXT: shll $16, %eax
-; AVXNECONVERT-NEXT: vmovd %eax, %xmm4
-; AVXNECONVERT-NEXT: vinsertps {{.*#+}} xmm3 = xmm3[0,1],xmm4[0],xmm3[3]
-; AVXNECONVERT-NEXT: vpextrw $3, %xmm0, %eax
-; AVXNECONVERT-NEXT: shll $16, %eax
-; AVXNECONVERT-NEXT: vmovd %eax, %xmm0
-; AVXNECONVERT-NEXT: vinsertps {{.*#+}} xmm0 = xmm3[0,1,2],xmm0[0]
-; AVXNECONVERT-NEXT: vinsertf128 $1, %xmm1, %ymm0, %ymm0
-; AVXNECONVERT-NEXT: vpextrw $5, %xmm2, %eax
-; AVXNECONVERT-NEXT: shll $16, %eax
-; AVXNECONVERT-NEXT: vmovd %eax, %xmm1
-; AVXNECONVERT-NEXT: vpextrw $4, %xmm2, %eax
-; AVXNECONVERT-NEXT: shll $16, %eax
-; AVXNECONVERT-NEXT: vmovd %eax, %xmm3
-; AVXNECONVERT-NEXT: vinsertps {{.*#+}} xmm1 = xmm3[0],xmm1[0],xmm3[2,3]
-; AVXNECONVERT-NEXT: vpextrw $6, %xmm2, %eax
-; AVXNECONVERT-NEXT: shll $16, %eax
-; AVXNECONVERT-NEXT: vmovd %eax, %xmm3
-; AVXNECONVERT-NEXT: vinsertps {{.*#+}} xmm1 = xmm1[0,1],xmm3[0],xmm1[3]
-; AVXNECONVERT-NEXT: vpextrw $7, %xmm2, %eax
-; AVXNECONVERT-NEXT: shll $16, %eax
-; AVXNECONVERT-NEXT: vmovd %eax, %xmm3
-; AVXNECONVERT-NEXT: vinsertps {{.*#+}} xmm1 = xmm1[0,1,2],xmm3[0]
-; AVXNECONVERT-NEXT: vpextrw $1, %xmm2, %eax
-; AVXNECONVERT-NEXT: shll $16, %eax
-; AVXNECONVERT-NEXT: vmovd %eax, %xmm3
-; AVXNECONVERT-NEXT: vmovd %xmm2, %eax
-; AVXNECONVERT-NEXT: shll $16, %eax
-; AVXNECONVERT-NEXT: vmovd %eax, %xmm4
-; AVXNECONVERT-NEXT: vinsertps {{.*#+}} xmm3 = xmm4[0],xmm3[0],xmm4[2,3]
-; AVXNECONVERT-NEXT: vpextrw $2, %xmm2, %eax
-; AVXNECONVERT-NEXT: shll $16, %eax
-; AVXNECONVERT-NEXT: vmovd %eax, %xmm4
-; AVXNECONVERT-NEXT: vinsertps {{.*#+}} xmm3 = xmm3[0,1],xmm4[0],xmm3[3]
-; AVXNECONVERT-NEXT: vpextrw $3, %xmm2, %eax
-; AVXNECONVERT-NEXT: shll $16, %eax
-; AVXNECONVERT-NEXT: vmovd %eax, %xmm2
-; AVXNECONVERT-NEXT: vinsertps {{.*#+}} xmm2 = xmm3[0,1,2],xmm2[0]
-; AVXNECONVERT-NEXT: vinsertf128 $1, %xmm1, %ymm2, %ymm1
+; AVXNECONVERT-NEXT: vpunpckhwd {{.*#+}} xmm1 = xmm3[4],xmm0[4],xmm3[5],xmm0[5],xmm3[6],xmm0[6],xmm3[7],xmm0[7]
+; AVXNECONVERT-NEXT: vpunpcklwd {{.*#+}} xmm0 = xmm3[0],xmm0[0],xmm3[1],xmm0[1],xmm3[2],xmm0[2],xmm3[3],xmm0[3]
+; AVXNECONVERT-NEXT: vinserti128 $1, %xmm1, %ymm0, %ymm0
+; AVXNECONVERT-NEXT: vpunpckhwd {{.*#+}} xmm1 = xmm3[4],xmm2[4],xmm3[5],xmm2[5],xmm3[6],xmm2[6],xmm3[7],xmm2[7]
+; AVXNECONVERT-NEXT: vpunpcklwd {{.*#+}} xmm2 = xmm3[0],xmm2[0],xmm3[1],xmm2[1],xmm3[2],xmm2[2],xmm3[3],xmm2[3]
+; AVXNECONVERT-NEXT: vinserti128 $1, %xmm1, %ymm2, %ymm1
; AVXNECONVERT-NEXT: vaddps %ymm1, %ymm0, %ymm0
; AVXNECONVERT-NEXT: {vex} vcvtneps2bf16 %ymm0, %xmm0
; AVXNECONVERT-NEXT: vzeroupper
@@ -397,132 +272,21 @@ define <5 x bfloat> @fuse_v5bf16(<5 x bfloat> %x, <5 x bfloat> %y, <5 x bfloat>
;
; AVXNECONVERT-LABEL: fuse_v5bf16:
; AVXNECONVERT: # %bb.0: # %entry
-; AVXNECONVERT-NEXT: vpextrw $5, %xmm1, %eax
-; AVXNECONVERT-NEXT: shll $16, %eax
-; AVXNECONVERT-NEXT: vmovd %eax, %xmm3
-; AVXNECONVERT-NEXT: vpextrw $4, %xmm1, %eax
-; AVXNECONVERT-NEXT: shll $16, %eax
-; AVXNECONVERT-NEXT: vmovd %eax, %xmm4
-; AVXNECONVERT-NEXT: vinsertps {{.*#+}} xmm3 = xmm4[0],xmm3[0],xmm4[2,3]
-; AVXNECONVERT-NEXT: vpextrw $6, %xmm1, %eax
-; AVXNECONVERT-NEXT: shll $16, %eax
-; AVXNECONVERT-NEXT: vmovd %eax, %xmm4
-; AVXNECONVERT-NEXT: vinsertps {{.*#+}} xmm3 = xmm3[0,1],xmm4[0],xmm3[3]
-; AVXNECONVERT-NEXT: vpextrw $7, %xmm1, %eax
-; AVXNECONVERT-NEXT: shll $16, %eax
-; AVXNECONVERT-NEXT: vmovd %eax, %xmm4
-; AVXNECONVERT-NEXT: vinsertps {{.*#+}} xmm3 = xmm3[0,1,2],xmm4[0]
-; AVXNECONVERT-NEXT: vpextrw $1, %xmm1, %eax
-; AVXNECONVERT-NEXT: shll $16, %eax
-; AVXNECONVERT-NEXT: vmovd %eax, %xmm4
-; AVXNECONVERT-NEXT: vmovd %xmm1, %eax
-; AVXNECONVERT-NEXT: shll $16, %eax
-; AVXNECONVERT-NEXT: vmovd %eax, %xmm5
-; AVXNECONVERT-NEXT: vinsertps {{.*#+}} xmm4 = xmm5[0],xmm4[0],xmm5[2,3]
-; AVXNECONVERT-NEXT: vpextrw $2, %xmm1, %eax
-; AVXNECONVERT-NEXT: shll $16, %eax
-; AVXNECONVERT-NEXT: vmovd %eax, %xmm5
-; AVXNECONVERT-NEXT: vinsertps {{.*#+}} xmm4 = xmm4[0,1],xmm5[0],xmm4[3]
-; AVXNECONVERT-NEXT: vpextrw $3, %xmm1, %eax
-; AVXNECONVERT-NEXT: shll $16, %eax
-; AVXNECONVERT-NEXT: vmovd %eax, %xmm1
-; AVXNECONVERT-NEXT: vinsertps {{.*#+}} xmm1 = xmm4[0,1,2],xmm1[0]
-; AVXNECONVERT-NEXT: vinsertf128 $1, %xmm3, %ymm1, %ymm1
-; AVXNECONVERT-NEXT: vpextrw $5, %xmm0, %eax
-; AVXNECONVERT-NEXT: shll $16, %eax
-; AVXNECONVERT-NEXT: vmovd %eax, %xmm3
-; AVXNECONVERT-NEXT: vpextrw $4, %xmm0, %eax
-; AVXNECONVERT-NEXT: shll $16, %eax
-; AVXNECONVERT-NEXT: vmovd %eax, %xmm4
-; AVXNECONVERT-NEXT: vinsertps {{.*#+}} xmm3 = xmm4[0],xmm3[0],xmm4[2,3]
-; AVXNECONVERT-NEXT: vpextrw $6, %xmm0, %eax
-; AVXNECONVERT-NEXT: shll $16, %eax
-; AVXNECONVERT-NEXT: vmovd %eax, %xmm4
-; AVXNECONVERT-NEXT: vinsertps {{.*#+}} xmm3 = xmm3[0,1],xmm4[0],xmm3[3]
-; AVXNECONVERT-NEXT: vpextrw $7, %xmm0, %eax
-; AVXNECONVERT-NEXT: shll $16, %eax
-; AVXNECONVERT-NEXT: vmovd %eax, %xmm4
-; AVXNECONVERT-NEXT: vinsertps {{.*#+}} xmm3 = xmm3[0,1,2],xmm4[0]
-; AVXNECONVERT-NEXT: vpextrw $1, %xmm0, %eax
-; AVXNECONVERT-NEXT: shll $16, %eax
-; AVXNECONVERT-NEXT: vmovd %eax, %xmm4
-; AVXNECONVERT-NEXT: vmovd %xmm0, %eax
-; AVXNECONVERT-NEXT: shll $16, %eax
-; AVXNECONVERT-NEXT: vmovd %eax, %xmm5
-; AVXNECONVERT-NEXT: vinsertps {{.*#+}} xmm4 = xmm5[0],xmm4[0],xmm5[2,3]
-; AVXNECONVERT-NEXT: vpextrw $2, %xmm0, %eax
-; AVXNECONVERT-NEXT: shll $16, %eax
-; AVXNECONVERT-NEXT: vmovd %eax, %xmm5
-; AVXNECONVERT-NEXT: vinsertps {{.*#+}} xmm4 = xmm4[0,1],xmm5[0],xmm4[3]
-; AVXNECONVERT-NEXT: vpextrw $3, %xmm0, %eax
-; AVXNECONVERT-NEXT: shll $16, %eax
-; AVXNECONVERT-NEXT: vmovd %eax, %xmm0
-; AVXNECONVERT-NEXT: vinsertps {{.*#+}} xmm0 = xmm4[0,1,2],xmm0[0]
-; AVXNECONVERT-NEXT: vinsertf128 $1, %xmm3, %ymm0, %ymm0
+; AVXNECONVERT-NEXT: vpxor %xmm3, %xmm3, %xmm3
+; AVXNECONVERT-NEXT: vpunpckhwd {{.*#+}} xmm4 = xmm3[4],xmm1[4],xmm3[5],xmm1[5],xmm3[6],xmm1[6],xmm3[7],xmm1[7]
+; AVXNECONVERT-NEXT: vpunpcklwd {{.*#+}} xmm1 = xmm3[0],xmm1[0],xmm3[1],xmm1[1],xmm3[2],xmm1[2],xmm3[3],xmm1[3]
+; AVXNECONVERT-NEXT: vinserti128 $1, %xmm4, %ymm1, %ymm1
+; AVXNECONVERT-NEXT: vpunpckhwd {{.*#+}} xmm4 = xmm3[4],xmm0[4],xmm3[5],xmm0[5],xmm3[6],xmm0[6],xmm3[7],xmm0[7]
+; AVXNECONVERT-NEXT: vpunpcklwd {{.*#+}} xmm0 = xmm3[0],xmm0[0],xmm3[1],xmm0[1],xmm3[2],xmm0[2],xmm3[3],xmm0[3]
+; AVXNECONVERT-NEXT: vinserti128 $1, %xmm4, %ymm0, %ymm0
; AVXNECONVERT-NEXT: vmulps %ymm1, %ymm0, %ymm0
; AVXNECONVERT-NEXT: {vex} vcvtneps2bf16 %ymm0, %xmm0
-; AVXNECONVERT-NEXT: vpextrw $5, %xmm0, %eax
-; AVXNECONVERT-NEXT: shll $16, %eax
-; AVXNECONVERT-NEXT: vmovd %eax, %xmm1
-; AVXNECONVERT-NEXT: vpextrw $4, %xmm0, %eax
-; AVXNECONVERT-NEXT: shll $16, %eax
-; AVXNECONVERT-NEXT: vmovd %eax, %xmm3
-; AVXNECONVERT-NEXT: vinsertps {{.*#+}} xmm1 = xmm3[0],xmm1[0],xmm3[2,3]
-; AVXNECONVERT-NEXT: vpextrw $6, %xmm0, %eax
-; AVXNECONVERT-NEXT: shll $16, %eax
-; AVXNECONVERT-NEXT: vmovd %eax, %xmm3
-; AVXNECONVERT-NEXT: vinsertps {{.*#+}} xmm1 = xmm1[0,1],xmm3[0],xmm1[3]
-; AVXNECONVERT-NEXT: vpextrw $7, %xmm0, %eax
-; AVXNECONVERT-NEXT: shll $16, %eax
-; AVXNECONVERT-NEXT: vmovd %eax, %xmm3
-; AVXNECONVERT-NEXT: vinsertps {{.*#+}} xmm1 = xmm1[0,1,2],xmm3[0]
-; AVXNECONVERT-NEXT: vpextrw $1, %xmm0, %eax
-; AVXNECONVERT-NEXT: shll $16, %eax
-; AVXNECONVERT-NEXT: vmovd %eax, %xmm3
-; AVXNECONVERT-NEXT: vmovd %xmm0, %eax
-; AVXNECONVERT-NEXT: shll $16, %eax
-; AVXNECONVERT-NEXT: vmovd %eax, %xmm4
-; AVXNECONVERT-NEXT: vinsertps {{.*#+}} xmm3 = xmm4[0],xmm3[0],xmm4[2,3]
-; AVXNECONVERT-NEXT: vpextrw $2, %xmm0, %eax
-; AVXNECONVERT-NEXT: shll $16, %eax
-; AVXNECONVERT-NEXT: vmovd %eax, %xmm4
-; AVXNECONVERT-NEXT: vinsertps {{.*#+}} xmm3 = xmm3[0,1],xmm4[0],xmm3[3]
-; AVXNECONVERT-NEXT: vpextrw $3, %xmm0, %eax
-; AVXNECONVERT-NEXT: shll $16, %eax
-; AVXNECONVERT-NEXT: vmovd %eax, %xmm0
-; AVXNECONVERT-NEXT: vinsertps {{.*#+}} xmm0 = xmm3[0,1,2],xmm0[0]
-; AVXNECONVERT-NEXT: vinsertf128 $1, %xmm1, %ymm0, %ymm0
-; AVXNECONVERT-NEXT: vpextrw $5, %xmm2, %eax
-; AVXNECONVERT-NEXT: shll $16, %eax
-; AVXNECONVERT-NEXT: vmovd %eax, %xmm1
-; AVXNECONVERT-NEXT: vpextrw $4, %xmm2, %eax
-; AVXNECONVERT-NEXT: shll $16, %eax
-; AVXNECONVERT-NEXT: vmovd %eax, %xmm3
-; AVXNECONVERT-NEXT: vinsertps {{.*#+}} xmm1 = xmm3[0],xmm1[0],xmm3[2,3]
-; AVXNECONVERT-NEXT: vpextrw $6, %xmm2, %eax
-; AVXNECONVERT-NEXT: shll $16, %eax
-; AVXNECONVERT-NEXT: vmovd %eax, %xmm3
-; AVXNECONVERT-NEXT: vinsertps {{.*#+}} xmm1 = xmm1[0,1],xmm3[0],xmm1[3]
-; AVXNECONVERT-NEXT: vpextrw $7, %xmm2, %eax
-; AVXNECONVERT-NEXT: shll $16, %eax
-; AVXNECONVERT-NEXT: vmovd %eax, %xmm3
-; AVXNECONVERT-NEXT: vinsertps {{.*#+}} xmm1 = xmm1[0,1,2],xmm3[0]
-; AVXNECONVERT-NEXT: vpextrw $1, %xmm2, %eax
-; AVXNECONVERT-NEXT: shll $16, %eax
-; AVXNECONVERT-NEXT: vmovd %eax, %xmm3
-; AVXNECONVERT-NEXT: vmovd %xmm2, %eax
-; AVXNECONVERT-NEXT: shll $16, %eax
-; AVXNECONVERT-NEXT: vmovd %eax, %xmm4
-; AVXNECONVERT-NEXT: vinsertps {{.*#+}} xmm3 = xmm4[0],xmm3[0],xmm4[2,3]
-; AVXNECONVERT-NEXT: vpextrw $2, %xmm2, %eax
-; AVXNECONVERT-NEXT: shll $16, %eax
-; AVXNECONVERT-NEXT: vmovd %eax, %xmm4
-; AVXNECONVERT-NEXT: vinsertps {{.*#+}} xmm3 = xmm3[0,1],xmm4[0],xmm3[3]
-; AVXNECONVERT-NEXT: vpextrw $3, %xmm2, %eax
-; AVXNECONVERT-NEXT: shll $16, %eax
-; AVXNECONVERT-NEXT: vmovd %eax, %xmm2
-; AVXNECONVERT-NEXT: vinsertps {{.*#+}} xmm2 = xmm3[0,1,2],xmm2[0]
-; AVXNECONVERT-NEXT: vinsertf128 $1, %xmm1, %ymm2, %ymm1
+; AVXNECONVERT-NEXT: vpunpckhwd {{.*#+}} xmm1 = xmm3[4],xmm0[4],xmm3[5],xmm0[5],xmm3[6],xmm0[6],xmm3[7],xmm0[7]
+; AVXNECONVERT-NEXT: vpunpcklwd {{.*#+}} xmm0 = xmm3[0],xmm0[0],xmm3[1],xmm0[1],xmm3[2],xmm0[2],xmm3[3],xmm0[3]
+; AVXNECONVERT-NEXT: vinserti128 $1, %xmm1, %ymm0, %ymm0
+; AVXNECONVERT-NEXT: vpunpckhwd {{.*#+}} xmm1 = xmm3[4],xmm2[4],xmm3[5],xmm2[5],xmm3[6],xmm2[6],xmm3[7],xmm2[7]
+; AVXNECONVERT-NEXT: vpunpcklwd {{.*#+}} xmm2 = xmm3[0],xmm2[0],xmm3[1],xmm2[1],xmm3[2],xmm2[2],xmm3[3],xmm2[3]
+; AVXNECONVERT-NEXT: vinserti128 $1, %xmm1, %ymm2, %ymm1
; AVXNECONVERT-NEXT: vaddps %ymm1, %ymm0, %ymm0
; AVXNECONVERT-NEXT: {vex} vcvtneps2bf16 %ymm0, %xmm0
; AVXNECONVERT-NEXT: vzeroupper
diff --git a/llvm/test/CodeGen/X86/avx10_2bf16-select-minmax.ll b/llvm/test/CodeGen/X86/avx10_2bf16-select-minmax.ll
index 3cf28cce974dd..d9672fa8e7460 100644
--- a/llvm/test/CodeGen/X86/avx10_2bf16-select-minmax.ll
+++ b/llvm/test/CodeGen/X86/avx10_2bf16-select-minmax.ll
@@ -5,17 +5,13 @@
define bfloat @select_ogt_bf16(bfloat %a, bfloat %b) {
; CHECK-LABEL: select_ogt_bf16:
; CHECK: # %bb.0:
-; CHECK-NEXT: vmovw %xmm0, %eax
-; CHECK-NEXT: vmovw %xmm1, %ecx
-; CHECK-NEXT: movl %ecx, %edx
-; CHECK-NEXT: shll $16, %edx
-; CHECK-NEXT: vmovd %edx, %xmm0
-; CHECK-NEXT: movl %eax, %edx
-; CHECK-NEXT: shll $16, %edx
-; CHECK-NEXT: vmovd %edx, %xmm1
-; CHECK-NEXT: vucomiss %xmm0, %xmm1
-; CHECK-NEXT: cmoval %eax, %ecx
-; CHECK-NEXT: vmovw %ecx, %xmm0
+; CHECK-NEXT: vmovw %xmm1, %eax
+; CHECK-NEXT: vmovw %xmm0, %ecx
+; CHECK-NEXT: vpslld $16, %xmm1, %xmm1
+; CHECK-NEXT: vpslld $16, %xmm0, %xmm0
+; CHECK-NEXT: vucomiss %xmm1, %xmm0
+; CHECK-NEXT: cmoval %ecx, %eax
+; CHECK-NEXT: vmovw %eax, %xmm0
; CHECK-NEXT: retq
%cmp = fcmp ogt bfloat %a, %b
%sel = select i1 %cmp, bfloat %a, bfloat %b
diff --git a/llvm/test/CodeGen/X86/bf16-fast-isel.ll b/llvm/test/CodeGen/X86/bf16-fast-isel.ll
index 4259b811bbfe9..1f36d258a1a58 100644
--- a/llvm/test/CodeGen/X86/bf16-fast-isel.ll
+++ b/llvm/test/CodeGen/X86/bf16-fast-isel.ll
@@ -6,9 +6,7 @@ define i8 @test_direct_call(ptr %f) nounwind {
; CHECK: # %bb.0: # %entry
; CHECK-NEXT: pushq %rax
; CHECK-NEXT: callq foo at PLT
-; CHECK-NEXT: pextrw $0, %xmm0, %eax
-; CHECK-NEXT: shll $16, %eax
-; CHECK-NEXT: movd %eax, %xmm0
+; CHECK-NEXT: pslld $16, %xmm0
; CHECK-NEXT: callq __truncsfbf2 at PLT
; CHECK-NEXT: callq bar at PLT
; CHECK-NEXT: popq %rcx
@@ -24,9 +22,7 @@ define i8 @test_fast_direct_call(ptr %f) nounwind {
; CHECK: # %bb.0: # %entry
; CHECK-NEXT: pushq %rax
; CHECK-NEXT: callq foo_fast at PLT
-; CHECK-NEXT: pextrw $0, %xmm0, %eax
-; CHECK-NEXT: shll $16, %eax
-; CHECK-NEXT: movd %eax, %xmm0
+; CHECK-NEXT: pslld $16, %xmm0
; CHECK-NEXT: callq __truncsfbf2 at PLT
; CHECK-NEXT: callq bar at PLT
; CHECK-NEXT: popq %rcx
@@ -44,9 +40,7 @@ define i8 @test_indirect_all(ptr %fptr, ptr %f) nounwind {
; CHECK-NEXT: movq %rdi, %rbx
; CHECK-NEXT: movq %rsi, %rdi
; CHECK-NEXT: callq foo at PLT
-; CHECK-NEXT: pextrw $0, %xmm0, %eax
-; CHECK-NEXT: shll $16, %eax
-; CHECK-NEXT: movd %eax, %xmm0
+; CHECK-NEXT: pslld $16, %xmm0
; CHECK-NEXT: callq __truncsfbf2 at PLT
; CHECK-NEXT: callq *%rbx
; CHECK-NEXT: popq %rbx
@@ -70,9 +64,7 @@ define i8 @test_indirect_all2(ptr %fptr, ptr %f, i1 %cond) nounwind {
; CHECK-NEXT: testb $1, %bpl
; CHECK-NEXT: je .LBB3_2
; CHECK-NEXT: # %bb.1: # %exit
-; CHECK-NEXT: pextrw $0, %xmm0, %eax
-; CHECK-NEXT: shll $16, %eax
-; CHECK-NEXT: movd %eax, %xmm0
+; CHECK-NEXT: pslld $16, %xmm0
; CHECK-NEXT: callq __truncsfbf2 at PLT
; CHECK-NEXT: callq *%rbx
; CHECK-NEXT: jmp .LBB3_3
@@ -103,9 +95,7 @@ define i8 @test_fast_indirect_all(ptr %fptr, ptr %f) nounwind {
; CHECK-NEXT: movq %rdi, %rbx
; CHECK-NEXT: movq %rsi, %rdi
; CHECK-NEXT: callq foo at PLT
-; CHECK-NEXT: pextrw $0, %xmm0, %eax
-; CHECK-NEXT: shll $16, %eax
-; CHECK-NEXT: movd %eax, %xmm0
+; CHECK-NEXT: pslld $16, %xmm0
; CHECK-NEXT: callq __truncsfbf2 at PLT
; CHECK-NEXT: callq *%rbx
; CHECK-NEXT: popq %rbx
@@ -122,24 +112,20 @@ declare void @take_bfloats({ bfloat, bfloat })
define void @call_get_bfloats() nounwind {
; CHECK-LABEL: call_get_bfloats:
; CHECK: # %bb.0:
-; CHECK-NEXT: pushq %rax
+; CHECK-NEXT: subq $40, %rsp
; CHECK-NEXT: callq get_bfloats at PLT
-; CHECK-NEXT: pextrw $0, %xmm1, %eax
-; CHECK-NEXT: shll $16, %eax
-; CHECK-NEXT: movl %eax, (%rsp) # 4-byte Spill
-; CHECK-NEXT: pextrw $0, %xmm0, %eax
-; CHECK-NEXT: shll $16, %eax
-; CHECK-NEXT: movd %eax, %xmm0
+; CHECK-NEXT: pslld $16, %xmm1
+; CHECK-NEXT: movdqa %xmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
+; CHECK-NEXT: pslld $16, %xmm0
; CHECK-NEXT: callq __truncsfbf2 at PLT
; CHECK-NEXT: movd %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 4-byte Folded Spill
-; CHECK-NEXT: movss (%rsp), %xmm0 # 4-byte Reload
-; CHECK-NEXT: # xmm0 = mem[0],zero,zero,zero
+; CHECK-NEXT: movaps {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 16-byte Reload
; CHECK-NEXT: callq __truncsfbf2 at PLT
; CHECK-NEXT: movaps %xmm0, %xmm1
; CHECK-NEXT: movss {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 4-byte Reload
; CHECK-NEXT: # xmm0 = mem[0],zero,zero,zero
; CHECK-NEXT: callq take_bfloats at PLT
-; CHECK-NEXT: popq %rax
+; CHECK-NEXT: addq $40, %rsp
; CHECK-NEXT: retq
%res = call { bfloat, bfloat } @get_bfloats()
call void @take_bfloats({ bfloat, bfloat } %res)
diff --git a/llvm/test/CodeGen/X86/bfloat-calling-conv.ll b/llvm/test/CodeGen/X86/bfloat-calling-conv.ll
index d08749174f85c..25f4c9fcfee51 100644
--- a/llvm/test/CodeGen/X86/bfloat-calling-conv.ll
+++ b/llvm/test/CodeGen/X86/bfloat-calling-conv.ll
@@ -14,9 +14,7 @@ define bfloat @return_arg_bf16(bfloat %x) #0 {
; FAST_ISEL_SSE2-LABEL: return_arg_bf16:
; FAST_ISEL_SSE2: # %bb.0:
; FAST_ISEL_SSE2-NEXT: pushq %rax
-; FAST_ISEL_SSE2-NEXT: pextrw $0, %xmm0, %eax
-; FAST_ISEL_SSE2-NEXT: shll $16, %eax
-; FAST_ISEL_SSE2-NEXT: movd %eax, %xmm0
+; FAST_ISEL_SSE2-NEXT: pslld $16, %xmm0
; FAST_ISEL_SSE2-NEXT: callq __truncsfbf2 at PLT
; FAST_ISEL_SSE2-NEXT: popq %rax
; FAST_ISEL_SSE2-NEXT: retq
@@ -27,9 +25,7 @@ define bfloat @return_arg_bf16(bfloat %x) #0 {
;
; FAST_ISEL_AVX512BF16-LABEL: return_arg_bf16:
; FAST_ISEL_AVX512BF16: # %bb.0:
-; FAST_ISEL_AVX512BF16-NEXT: vpextrw $0, %xmm0, %eax
-; FAST_ISEL_AVX512BF16-NEXT: shll $16, %eax
-; FAST_ISEL_AVX512BF16-NEXT: vmovd %eax, %xmm0
+; FAST_ISEL_AVX512BF16-NEXT: vpslld $16, %xmm0, %xmm0
; FAST_ISEL_AVX512BF16-NEXT: vcvtneps2bf16 %xmm0, %xmm0
; FAST_ISEL_AVX512BF16-NEXT: retq
;
@@ -39,9 +35,7 @@ define bfloat @return_arg_bf16(bfloat %x) #0 {
;
; FAST_ISEL_AVXNECONVERT-LABEL: return_arg_bf16:
; FAST_ISEL_AVXNECONVERT: # %bb.0:
-; FAST_ISEL_AVXNECONVERT-NEXT: vpextrw $0, %xmm0, %eax
-; FAST_ISEL_AVXNECONVERT-NEXT: shll $16, %eax
-; FAST_ISEL_AVXNECONVERT-NEXT: vmovd %eax, %xmm0
+; FAST_ISEL_AVXNECONVERT-NEXT: vpslld $16, %xmm0, %xmm0
; FAST_ISEL_AVXNECONVERT-NEXT: {vex} vcvtneps2bf16 %xmm0, %xmm0
; FAST_ISEL_AVXNECONVERT-NEXT: retq
ret bfloat %x
@@ -56,17 +50,15 @@ define <2 x bfloat> @return_arg_v2bf16(<2 x bfloat> %x) #0 {
; FAST_ISEL_SSE2: # %bb.0:
; FAST_ISEL_SSE2-NEXT: subq $40, %rsp
; FAST_ISEL_SSE2-NEXT: pextrw $1, %xmm0, %eax
-; FAST_ISEL_SSE2-NEXT: shll $16, %eax
-; FAST_ISEL_SSE2-NEXT: movl %eax, {{[-0-9]+}}(%r{{[sb]}}p) # 4-byte Spill
-; FAST_ISEL_SSE2-NEXT: movd %xmm0, %eax
-; FAST_ISEL_SSE2-NEXT: shll $16, %eax
-; FAST_ISEL_SSE2-NEXT: movd %eax, %xmm0
+; FAST_ISEL_SSE2-NEXT: pslld $16, %xmm0
+; FAST_ISEL_SSE2-NEXT: movd %eax, %xmm1
+; FAST_ISEL_SSE2-NEXT: pslld $16, %xmm1
+; FAST_ISEL_SSE2-NEXT: movdqa %xmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
; FAST_ISEL_SSE2-NEXT: callq __truncsfbf2 at PLT
-; FAST_ISEL_SSE2-NEXT: movdqa %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
-; FAST_ISEL_SSE2-NEXT: movd {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 4-byte Folded Reload
-; FAST_ISEL_SSE2-NEXT: # xmm0 = mem[0],zero,zero,zero
+; FAST_ISEL_SSE2-NEXT: movdqa %xmm0, (%rsp) # 16-byte Spill
+; FAST_ISEL_SSE2-NEXT: movdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 16-byte Reload
; FAST_ISEL_SSE2-NEXT: callq __truncsfbf2 at PLT
-; FAST_ISEL_SSE2-NEXT: movdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm1 # 16-byte Reload
+; FAST_ISEL_SSE2-NEXT: movdqa (%rsp), %xmm1 # 16-byte Reload
; FAST_ISEL_SSE2-NEXT: punpcklwd {{.*#+}} xmm1 = xmm1[0],xmm0[0],xmm1[1],xmm0[1],xmm1[2],xmm0[2],xmm1[3],xmm0[3]
; FAST_ISEL_SSE2-NEXT: movdqa %xmm1, %xmm0
; FAST_ISEL_SSE2-NEXT: addq $40, %rsp
@@ -97,31 +89,29 @@ define <3 x bfloat> @return_arg_v3bf16(<3 x bfloat> %x) #0 {
;
; FAST_ISEL_SSE2-LABEL: return_arg_v3bf16:
; FAST_ISEL_SSE2: # %bb.0:
-; FAST_ISEL_SSE2-NEXT: subq $40, %rsp
+; FAST_ISEL_SSE2-NEXT: subq $56, %rsp
; FAST_ISEL_SSE2-NEXT: pextrw $2, %xmm0, %eax
-; FAST_ISEL_SSE2-NEXT: shll $16, %eax
-; FAST_ISEL_SSE2-NEXT: movl %eax, {{[-0-9]+}}(%r{{[sb]}}p) # 4-byte Spill
+; FAST_ISEL_SSE2-NEXT: movd %eax, %xmm1
; FAST_ISEL_SSE2-NEXT: pextrw $1, %xmm0, %eax
-; FAST_ISEL_SSE2-NEXT: shll $16, %eax
-; FAST_ISEL_SSE2-NEXT: movl %eax, {{[-0-9]+}}(%r{{[sb]}}p) # 4-byte Spill
-; FAST_ISEL_SSE2-NEXT: movd %xmm0, %eax
-; FAST_ISEL_SSE2-NEXT: shll $16, %eax
-; FAST_ISEL_SSE2-NEXT: movd %eax, %xmm0
+; FAST_ISEL_SSE2-NEXT: pslld $16, %xmm0
+; FAST_ISEL_SSE2-NEXT: pslld $16, %xmm1
+; FAST_ISEL_SSE2-NEXT: movdqa %xmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
+; FAST_ISEL_SSE2-NEXT: movd %eax, %xmm1
+; FAST_ISEL_SSE2-NEXT: pslld $16, %xmm1
+; FAST_ISEL_SSE2-NEXT: movdqa %xmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
; FAST_ISEL_SSE2-NEXT: callq __truncsfbf2 at PLT
-; FAST_ISEL_SSE2-NEXT: movdqa %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
-; FAST_ISEL_SSE2-NEXT: movd {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 4-byte Folded Reload
-; FAST_ISEL_SSE2-NEXT: # xmm0 = mem[0],zero,zero,zero
+; FAST_ISEL_SSE2-NEXT: movdqa %xmm0, (%rsp) # 16-byte Spill
+; FAST_ISEL_SSE2-NEXT: movdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 16-byte Reload
; FAST_ISEL_SSE2-NEXT: callq __truncsfbf2 at PLT
-; FAST_ISEL_SSE2-NEXT: movdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm1 # 16-byte Reload
+; FAST_ISEL_SSE2-NEXT: movdqa (%rsp), %xmm1 # 16-byte Reload
; FAST_ISEL_SSE2-NEXT: punpcklwd {{.*#+}} xmm1 = xmm1[0],xmm0[0],xmm1[1],xmm0[1],xmm1[2],xmm0[2],xmm1[3],xmm0[3]
-; FAST_ISEL_SSE2-NEXT: movdqa %xmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
-; FAST_ISEL_SSE2-NEXT: movss {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 4-byte Reload
-; FAST_ISEL_SSE2-NEXT: # xmm0 = mem[0],zero,zero,zero
+; FAST_ISEL_SSE2-NEXT: movdqa %xmm1, (%rsp) # 16-byte Spill
+; FAST_ISEL_SSE2-NEXT: movaps {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 16-byte Reload
; FAST_ISEL_SSE2-NEXT: callq __truncsfbf2 at PLT
-; FAST_ISEL_SSE2-NEXT: movaps {{[-0-9]+}}(%r{{[sb]}}p), %xmm1 # 16-byte Reload
+; FAST_ISEL_SSE2-NEXT: movaps (%rsp), %xmm1 # 16-byte Reload
; FAST_ISEL_SSE2-NEXT: unpcklps {{.*#+}} xmm1 = xmm1[0],xmm0[0],xmm1[1],xmm0[1]
; FAST_ISEL_SSE2-NEXT: movaps %xmm1, %xmm0
-; FAST_ISEL_SSE2-NEXT: addq $40, %rsp
+; FAST_ISEL_SSE2-NEXT: addq $56, %rsp
; FAST_ISEL_SSE2-NEXT: retq
;
; AVX512BF16-LABEL: return_arg_v3bf16:
@@ -130,19 +120,16 @@ define <3 x bfloat> @return_arg_v3bf16(<3 x bfloat> %x) #0 {
;
; FAST_ISEL_AVX512BF16-LABEL: return_arg_v3bf16:
; FAST_ISEL_AVX512BF16: # %bb.0:
+; FAST_ISEL_AVX512BF16-NEXT: vpxor %xmm1, %xmm1, %xmm1
+; FAST_ISEL_AVX512BF16-NEXT: vpblendw {{.*#+}} xmm1 = xmm1[0],xmm0[1],xmm1[2],xmm0[3],xmm1[4],xmm0[5],xmm1[6],xmm0[7]
+; FAST_ISEL_AVX512BF16-NEXT: vpslld $16, %xmm0, %xmm2
; FAST_ISEL_AVX512BF16-NEXT: vpextrw $2, %xmm0, %eax
-; FAST_ISEL_AVX512BF16-NEXT: shll $16, %eax
-; FAST_ISEL_AVX512BF16-NEXT: vmovd %eax, %xmm1
-; FAST_ISEL_AVX512BF16-NEXT: vpextrw $1, %xmm0, %eax
-; FAST_ISEL_AVX512BF16-NEXT: shll $16, %eax
-; FAST_ISEL_AVX512BF16-NEXT: vmovd %eax, %xmm2
-; FAST_ISEL_AVX512BF16-NEXT: vmovd %xmm0, %eax
-; FAST_ISEL_AVX512BF16-NEXT: shll $16, %eax
; FAST_ISEL_AVX512BF16-NEXT: vmovd %eax, %xmm0
-; FAST_ISEL_AVX512BF16-NEXT: vcvtneps2bf16 %xmm1, %xmm1
-; FAST_ISEL_AVX512BF16-NEXT: vmovd %xmm1, %eax
+; FAST_ISEL_AVX512BF16-NEXT: vpslld $16, %xmm0, %xmm0
; FAST_ISEL_AVX512BF16-NEXT: vcvtneps2bf16 %xmm0, %xmm0
-; FAST_ISEL_AVX512BF16-NEXT: vcvtneps2bf16 %xmm2, %xmm1
+; FAST_ISEL_AVX512BF16-NEXT: vmovd %xmm0, %eax
+; FAST_ISEL_AVX512BF16-NEXT: vcvtneps2bf16 %xmm2, %xmm0
+; FAST_ISEL_AVX512BF16-NEXT: vcvtneps2bf16 %xmm1, %xmm1
; FAST_ISEL_AVX512BF16-NEXT: vpunpcklwd {{.*#+}} xmm0 = xmm0[0],xmm1[0],xmm0[1],xmm1[1],xmm0[2],xmm1[2],xmm0[3],xmm1[3]
; FAST_ISEL_AVX512BF16-NEXT: vpinsrw $2, %eax, %xmm0, %xmm0
; FAST_ISEL_AVX512BF16-NEXT: retq
@@ -153,19 +140,16 @@ define <3 x bfloat> @return_arg_v3bf16(<3 x bfloat> %x) #0 {
;
; FAST_ISEL_AVXNECONVERT-LABEL: return_arg_v3bf16:
; FAST_ISEL_AVXNECONVERT: # %bb.0:
+; FAST_ISEL_AVXNECONVERT-NEXT: vpxor %xmm1, %xmm1, %xmm1
+; FAST_ISEL_AVXNECONVERT-NEXT: vpblendw {{.*#+}} xmm1 = xmm1[0],xmm0[1],xmm1[2],xmm0[3],xmm1[4],xmm0[5],xmm1[6],xmm0[7]
+; FAST_ISEL_AVXNECONVERT-NEXT: vpslld $16, %xmm0, %xmm2
; FAST_ISEL_AVXNECONVERT-NEXT: vpextrw $2, %xmm0, %eax
-; FAST_ISEL_AVXNECONVERT-NEXT: shll $16, %eax
-; FAST_ISEL_AVXNECONVERT-NEXT: vmovd %eax, %xmm1
-; FAST_ISEL_AVXNECONVERT-NEXT: vpextrw $1, %xmm0, %eax
-; FAST_ISEL_AVXNECONVERT-NEXT: shll $16, %eax
-; FAST_ISEL_AVXNECONVERT-NEXT: vmovd %eax, %xmm2
-; FAST_ISEL_AVXNECONVERT-NEXT: vmovd %xmm0, %eax
-; FAST_ISEL_AVXNECONVERT-NEXT: shll $16, %eax
; FAST_ISEL_AVXNECONVERT-NEXT: vmovd %eax, %xmm0
-; FAST_ISEL_AVXNECONVERT-NEXT: {vex} vcvtneps2bf16 %xmm1, %xmm1
-; FAST_ISEL_AVXNECONVERT-NEXT: vmovd %xmm1, %eax
+; FAST_ISEL_AVXNECONVERT-NEXT: vpslld $16, %xmm0, %xmm0
; FAST_ISEL_AVXNECONVERT-NEXT: {vex} vcvtneps2bf16 %xmm0, %xmm0
-; FAST_ISEL_AVXNECONVERT-NEXT: {vex} vcvtneps2bf16 %xmm2, %xmm1
+; FAST_ISEL_AVXNECONVERT-NEXT: vmovd %xmm0, %eax
+; FAST_ISEL_AVXNECONVERT-NEXT: {vex} vcvtneps2bf16 %xmm2, %xmm0
+; FAST_ISEL_AVXNECONVERT-NEXT: {vex} vcvtneps2bf16 %xmm1, %xmm1
; FAST_ISEL_AVXNECONVERT-NEXT: vpunpcklwd {{.*#+}} xmm1 = xmm0[0],xmm1[0],xmm0[1],xmm1[1],xmm0[2],xmm1[2],xmm0[3],xmm1[3]
; FAST_ISEL_AVXNECONVERT-NEXT: vpinsrw $2, %eax, %xmm1, %xmm1
; FAST_ISEL_AVXNECONVERT-NEXT: vmovq %xmm1, %rax
@@ -188,39 +172,37 @@ define <4 x bfloat> @return_arg_v4bf16(<4 x bfloat> %x) #0 {
;
; FAST_ISEL_SSE2-LABEL: return_arg_v4bf16:
; FAST_ISEL_SSE2: # %bb.0:
-; FAST_ISEL_SSE2-NEXT: subq $56, %rsp
+; FAST_ISEL_SSE2-NEXT: subq $72, %rsp
; FAST_ISEL_SSE2-NEXT: pextrw $3, %xmm0, %eax
-; FAST_ISEL_SSE2-NEXT: shll $16, %eax
-; FAST_ISEL_SSE2-NEXT: movl %eax, {{[-0-9]+}}(%r{{[sb]}}p) # 4-byte Spill
+; FAST_ISEL_SSE2-NEXT: movd %eax, %xmm1
; FAST_ISEL_SSE2-NEXT: pextrw $2, %xmm0, %eax
-; FAST_ISEL_SSE2-NEXT: shll $16, %eax
-; FAST_ISEL_SSE2-NEXT: movl %eax, {{[-0-9]+}}(%r{{[sb]}}p) # 4-byte Spill
+; FAST_ISEL_SSE2-NEXT: movd %eax, %xmm2
; FAST_ISEL_SSE2-NEXT: pextrw $1, %xmm0, %eax
-; FAST_ISEL_SSE2-NEXT: shll $16, %eax
-; FAST_ISEL_SSE2-NEXT: movl %eax, {{[-0-9]+}}(%r{{[sb]}}p) # 4-byte Spill
-; FAST_ISEL_SSE2-NEXT: movd %xmm0, %eax
-; FAST_ISEL_SSE2-NEXT: shll $16, %eax
-; FAST_ISEL_SSE2-NEXT: movd %eax, %xmm0
+; FAST_ISEL_SSE2-NEXT: pslld $16, %xmm0
+; FAST_ISEL_SSE2-NEXT: pslld $16, %xmm1
+; FAST_ISEL_SSE2-NEXT: movdqa %xmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
+; FAST_ISEL_SSE2-NEXT: pslld $16, %xmm2
+; FAST_ISEL_SSE2-NEXT: movdqa %xmm2, (%rsp) # 16-byte Spill
+; FAST_ISEL_SSE2-NEXT: movd %eax, %xmm1
+; FAST_ISEL_SSE2-NEXT: pslld $16, %xmm1
+; FAST_ISEL_SSE2-NEXT: movdqa %xmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
; FAST_ISEL_SSE2-NEXT: callq __truncsfbf2 at PLT
; FAST_ISEL_SSE2-NEXT: movdqa %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
-; FAST_ISEL_SSE2-NEXT: movd {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 4-byte Folded Reload
-; FAST_ISEL_SSE2-NEXT: # xmm0 = mem[0],zero,zero,zero
+; FAST_ISEL_SSE2-NEXT: movdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 16-byte Reload
; FAST_ISEL_SSE2-NEXT: callq __truncsfbf2 at PLT
; FAST_ISEL_SSE2-NEXT: movdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm1 # 16-byte Reload
; FAST_ISEL_SSE2-NEXT: punpcklwd {{.*#+}} xmm1 = xmm1[0],xmm0[0],xmm1[1],xmm0[1],xmm1[2],xmm0[2],xmm1[3],xmm0[3]
; FAST_ISEL_SSE2-NEXT: movdqa %xmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
-; FAST_ISEL_SSE2-NEXT: movss {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 4-byte Reload
-; FAST_ISEL_SSE2-NEXT: # xmm0 = mem[0],zero,zero,zero
+; FAST_ISEL_SSE2-NEXT: movaps (%rsp), %xmm0 # 16-byte Reload
; FAST_ISEL_SSE2-NEXT: callq __truncsfbf2 at PLT
-; FAST_ISEL_SSE2-NEXT: movaps %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
-; FAST_ISEL_SSE2-NEXT: movd {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 4-byte Folded Reload
-; FAST_ISEL_SSE2-NEXT: # xmm0 = mem[0],zero,zero,zero
+; FAST_ISEL_SSE2-NEXT: movaps %xmm0, (%rsp) # 16-byte Spill
+; FAST_ISEL_SSE2-NEXT: movdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 16-byte Reload
; FAST_ISEL_SSE2-NEXT: callq __truncsfbf2 at PLT
-; FAST_ISEL_SSE2-NEXT: movdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm1 # 16-byte Reload
+; FAST_ISEL_SSE2-NEXT: movdqa (%rsp), %xmm1 # 16-byte Reload
; FAST_ISEL_SSE2-NEXT: punpcklwd {{.*#+}} xmm1 = xmm1[0],xmm0[0],xmm1[1],xmm0[1],xmm1[2],xmm0[2],xmm1[3],xmm0[3]
; FAST_ISEL_SSE2-NEXT: movdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 16-byte Reload
; FAST_ISEL_SSE2-NEXT: punpckldq {{.*#+}} xmm0 = xmm0[0],xmm1[0],xmm0[1],xmm1[1]
-; FAST_ISEL_SSE2-NEXT: addq $56, %rsp
+; FAST_ISEL_SSE2-NEXT: addq $72, %rsp
; FAST_ISEL_SSE2-NEXT: retq
;
; AVX512BF16-LABEL: return_arg_v4bf16:
@@ -250,48 +232,48 @@ define <8 x bfloat> @return_arg_v8bf16(<8 x bfloat> %x) #0 {
; FAST_ISEL_SSE2: # %bb.0:
; FAST_ISEL_SSE2-NEXT: pushq %r14
; FAST_ISEL_SSE2-NEXT: pushq %rbx
-; FAST_ISEL_SSE2-NEXT: subq $56, %rsp
+; FAST_ISEL_SSE2-NEXT: subq $120, %rsp
+; FAST_ISEL_SSE2-NEXT: pxor %xmm1, %xmm1
+; FAST_ISEL_SSE2-NEXT: punpckhwd {{.*#+}} xmm1 = xmm1[4],xmm0[4],xmm1[5],xmm0[5],xmm1[6],xmm0[6],xmm1[7],xmm0[7]
+; FAST_ISEL_SSE2-NEXT: movdqa %xmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
; FAST_ISEL_SSE2-NEXT: pextrw $7, %xmm0, %eax
-; FAST_ISEL_SSE2-NEXT: shll $16, %eax
-; FAST_ISEL_SSE2-NEXT: movl %eax, {{[-0-9]+}}(%r{{[sb]}}p) # 4-byte Spill
+; FAST_ISEL_SSE2-NEXT: movd %eax, %xmm2
; FAST_ISEL_SSE2-NEXT: pextrw $6, %xmm0, %eax
-; FAST_ISEL_SSE2-NEXT: shll $16, %eax
-; FAST_ISEL_SSE2-NEXT: movl %eax, {{[-0-9]+}}(%r{{[sb]}}p) # 4-byte Spill
+; FAST_ISEL_SSE2-NEXT: movd %eax, %xmm1
; FAST_ISEL_SSE2-NEXT: pextrw $5, %xmm0, %eax
-; FAST_ISEL_SSE2-NEXT: shll $16, %eax
-; FAST_ISEL_SSE2-NEXT: movl %eax, {{[-0-9]+}}(%r{{[sb]}}p) # 4-byte Spill
-; FAST_ISEL_SSE2-NEXT: pextrw $4, %xmm0, %eax
-; FAST_ISEL_SSE2-NEXT: shll $16, %eax
-; FAST_ISEL_SSE2-NEXT: movl %eax, {{[-0-9]+}}(%r{{[sb]}}p) # 4-byte Spill
+; FAST_ISEL_SSE2-NEXT: movd %eax, %xmm3
; FAST_ISEL_SSE2-NEXT: pextrw $3, %xmm0, %eax
-; FAST_ISEL_SSE2-NEXT: shll $16, %eax
-; FAST_ISEL_SSE2-NEXT: movl %eax, {{[-0-9]+}}(%r{{[sb]}}p) # 4-byte Spill
+; FAST_ISEL_SSE2-NEXT: movd %eax, %xmm5
; FAST_ISEL_SSE2-NEXT: pextrw $2, %xmm0, %eax
-; FAST_ISEL_SSE2-NEXT: shll $16, %eax
-; FAST_ISEL_SSE2-NEXT: movl %eax, {{[-0-9]+}}(%r{{[sb]}}p) # 4-byte Spill
+; FAST_ISEL_SSE2-NEXT: movd %eax, %xmm4
; FAST_ISEL_SSE2-NEXT: pextrw $1, %xmm0, %eax
-; FAST_ISEL_SSE2-NEXT: shll $16, %eax
-; FAST_ISEL_SSE2-NEXT: movd %eax, %xmm1
-; FAST_ISEL_SSE2-NEXT: movd %xmm0, %eax
-; FAST_ISEL_SSE2-NEXT: shll $16, %eax
-; FAST_ISEL_SSE2-NEXT: movl %eax, {{[-0-9]+}}(%r{{[sb]}}p) # 4-byte Spill
-; FAST_ISEL_SSE2-NEXT: movdqa %xmm1, %xmm0
+; FAST_ISEL_SSE2-NEXT: pslld $16, %xmm0
+; FAST_ISEL_SSE2-NEXT: movdqa %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
+; FAST_ISEL_SSE2-NEXT: pslld $16, %xmm2
+; FAST_ISEL_SSE2-NEXT: movdqa %xmm2, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
+; FAST_ISEL_SSE2-NEXT: pslld $16, %xmm1
+; FAST_ISEL_SSE2-NEXT: movdqa %xmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
+; FAST_ISEL_SSE2-NEXT: pslld $16, %xmm3
+; FAST_ISEL_SSE2-NEXT: movdqa %xmm3, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
+; FAST_ISEL_SSE2-NEXT: pslld $16, %xmm5
+; FAST_ISEL_SSE2-NEXT: movdqa %xmm5, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
+; FAST_ISEL_SSE2-NEXT: pslld $16, %xmm4
+; FAST_ISEL_SSE2-NEXT: movdqa %xmm4, (%rsp) # 16-byte Spill
+; FAST_ISEL_SSE2-NEXT: movd %eax, %xmm0
+; FAST_ISEL_SSE2-NEXT: pslld $16, %xmm0
; FAST_ISEL_SSE2-NEXT: callq __truncsfbf2 at PLT
; FAST_ISEL_SSE2-NEXT: pextrw $0, %xmm0, %ebx
; FAST_ISEL_SSE2-NEXT: shll $16, %ebx
-; FAST_ISEL_SSE2-NEXT: movd {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 4-byte Folded Reload
-; FAST_ISEL_SSE2-NEXT: # xmm0 = mem[0],zero,zero,zero
+; FAST_ISEL_SSE2-NEXT: movdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 16-byte Reload
; FAST_ISEL_SSE2-NEXT: callq __truncsfbf2 at PLT
; FAST_ISEL_SSE2-NEXT: pextrw $0, %xmm0, %eax
; FAST_ISEL_SSE2-NEXT: movzwl %ax, %r14d
; FAST_ISEL_SSE2-NEXT: orl %ebx, %r14d
-; FAST_ISEL_SSE2-NEXT: movd {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 4-byte Folded Reload
-; FAST_ISEL_SSE2-NEXT: # xmm0 = mem[0],zero,zero,zero
+; FAST_ISEL_SSE2-NEXT: movdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 16-byte Reload
; FAST_ISEL_SSE2-NEXT: callq __truncsfbf2 at PLT
; FAST_ISEL_SSE2-NEXT: pextrw $0, %xmm0, %ebx
; FAST_ISEL_SSE2-NEXT: shll $16, %ebx
-; FAST_ISEL_SSE2-NEXT: movd {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 4-byte Folded Reload
-; FAST_ISEL_SSE2-NEXT: # xmm0 = mem[0],zero,zero,zero
+; FAST_ISEL_SSE2-NEXT: movdqa (%rsp), %xmm0 # 16-byte Reload
; FAST_ISEL_SSE2-NEXT: callq __truncsfbf2 at PLT
; FAST_ISEL_SSE2-NEXT: pextrw $0, %xmm0, %eax
; FAST_ISEL_SSE2-NEXT: movzwl %ax, %eax
@@ -299,25 +281,21 @@ define <8 x bfloat> @return_arg_v8bf16(<8 x bfloat> %x) #0 {
; FAST_ISEL_SSE2-NEXT: shlq $32, %rax
; FAST_ISEL_SSE2-NEXT: orq %r14, %rax
; FAST_ISEL_SSE2-NEXT: movq %rax, %xmm0
-; FAST_ISEL_SSE2-NEXT: movdqa %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
-; FAST_ISEL_SSE2-NEXT: movd {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 4-byte Folded Reload
-; FAST_ISEL_SSE2-NEXT: # xmm0 = mem[0],zero,zero,zero
+; FAST_ISEL_SSE2-NEXT: movdqa %xmm0, (%rsp) # 16-byte Spill
+; FAST_ISEL_SSE2-NEXT: movdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 16-byte Reload
; FAST_ISEL_SSE2-NEXT: callq __truncsfbf2 at PLT
; FAST_ISEL_SSE2-NEXT: pextrw $0, %xmm0, %ebx
; FAST_ISEL_SSE2-NEXT: shll $16, %ebx
-; FAST_ISEL_SSE2-NEXT: movd {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 4-byte Folded Reload
-; FAST_ISEL_SSE2-NEXT: # xmm0 = mem[0],zero,zero,zero
+; FAST_ISEL_SSE2-NEXT: movdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 16-byte Reload
; FAST_ISEL_SSE2-NEXT: callq __truncsfbf2 at PLT
; FAST_ISEL_SSE2-NEXT: pextrw $0, %xmm0, %eax
; FAST_ISEL_SSE2-NEXT: movzwl %ax, %r14d
; FAST_ISEL_SSE2-NEXT: orl %ebx, %r14d
-; FAST_ISEL_SSE2-NEXT: movd {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 4-byte Folded Reload
-; FAST_ISEL_SSE2-NEXT: # xmm0 = mem[0],zero,zero,zero
+; FAST_ISEL_SSE2-NEXT: movdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 16-byte Reload
; FAST_ISEL_SSE2-NEXT: callq __truncsfbf2 at PLT
; FAST_ISEL_SSE2-NEXT: pextrw $0, %xmm0, %ebx
; FAST_ISEL_SSE2-NEXT: shll $16, %ebx
-; FAST_ISEL_SSE2-NEXT: movd {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 4-byte Folded Reload
-; FAST_ISEL_SSE2-NEXT: # xmm0 = mem[0],zero,zero,zero
+; FAST_ISEL_SSE2-NEXT: movdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 16-byte Reload
; FAST_ISEL_SSE2-NEXT: callq __truncsfbf2 at PLT
; FAST_ISEL_SSE2-NEXT: pextrw $0, %xmm0, %eax
; FAST_ISEL_SSE2-NEXT: movzwl %ax, %eax
@@ -325,9 +303,9 @@ define <8 x bfloat> @return_arg_v8bf16(<8 x bfloat> %x) #0 {
; FAST_ISEL_SSE2-NEXT: shlq $32, %rax
; FAST_ISEL_SSE2-NEXT: orq %r14, %rax
; FAST_ISEL_SSE2-NEXT: movq %rax, %xmm1
-; FAST_ISEL_SSE2-NEXT: movdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 16-byte Reload
+; FAST_ISEL_SSE2-NEXT: movdqa (%rsp), %xmm0 # 16-byte Reload
; FAST_ISEL_SSE2-NEXT: punpcklqdq {{.*#+}} xmm0 = xmm0[0],xmm1[0]
-; FAST_ISEL_SSE2-NEXT: addq $56, %rsp
+; FAST_ISEL_SSE2-NEXT: addq $120, %rsp
; FAST_ISEL_SSE2-NEXT: popq %rbx
; FAST_ISEL_SSE2-NEXT: popq %r14
; FAST_ISEL_SSE2-NEXT: retq
@@ -360,72 +338,77 @@ define <16 x bfloat> @return_arg_v16bf16(<16 x bfloat> %x) #0 {
; FAST_ISEL_SSE2: # %bb.0:
; FAST_ISEL_SSE2-NEXT: pushq %r14
; FAST_ISEL_SSE2-NEXT: pushq %rbx
-; FAST_ISEL_SSE2-NEXT: subq $104, %rsp
+; FAST_ISEL_SSE2-NEXT: subq $248, %rsp
+; FAST_ISEL_SSE2-NEXT: pxor %xmm3, %xmm3
+; FAST_ISEL_SSE2-NEXT: pxor %xmm2, %xmm2
+; FAST_ISEL_SSE2-NEXT: punpckhwd {{.*#+}} xmm2 = xmm2[4],xmm1[4],xmm2[5],xmm1[5],xmm2[6],xmm1[6],xmm2[7],xmm1[7]
+; FAST_ISEL_SSE2-NEXT: movdqa %xmm2, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
+; FAST_ISEL_SSE2-NEXT: punpckhwd {{.*#+}} xmm3 = xmm3[4],xmm0[4],xmm3[5],xmm0[5],xmm3[6],xmm0[6],xmm3[7],xmm0[7]
+; FAST_ISEL_SSE2-NEXT: movdqa %xmm3, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
; FAST_ISEL_SSE2-NEXT: pextrw $7, %xmm1, %eax
-; FAST_ISEL_SSE2-NEXT: shll $16, %eax
-; FAST_ISEL_SSE2-NEXT: movl %eax, {{[-0-9]+}}(%r{{[sb]}}p) # 4-byte Spill
+; FAST_ISEL_SSE2-NEXT: movd %eax, %xmm3
; FAST_ISEL_SSE2-NEXT: pextrw $6, %xmm1, %eax
-; FAST_ISEL_SSE2-NEXT: shll $16, %eax
-; FAST_ISEL_SSE2-NEXT: movl %eax, {{[-0-9]+}}(%r{{[sb]}}p) # 4-byte Spill
+; FAST_ISEL_SSE2-NEXT: movd %eax, %xmm2
; FAST_ISEL_SSE2-NEXT: pextrw $5, %xmm1, %eax
-; FAST_ISEL_SSE2-NEXT: shll $16, %eax
-; FAST_ISEL_SSE2-NEXT: movl %eax, {{[-0-9]+}}(%r{{[sb]}}p) # 4-byte Spill
-; FAST_ISEL_SSE2-NEXT: pextrw $4, %xmm1, %eax
-; FAST_ISEL_SSE2-NEXT: shll $16, %eax
-; FAST_ISEL_SSE2-NEXT: movl %eax, {{[-0-9]+}}(%r{{[sb]}}p) # 4-byte Spill
+; FAST_ISEL_SSE2-NEXT: movd %eax, %xmm4
; FAST_ISEL_SSE2-NEXT: pextrw $3, %xmm1, %eax
-; FAST_ISEL_SSE2-NEXT: shll $16, %eax
-; FAST_ISEL_SSE2-NEXT: movl %eax, {{[-0-9]+}}(%r{{[sb]}}p) # 4-byte Spill
+; FAST_ISEL_SSE2-NEXT: movd %eax, %xmm6
; FAST_ISEL_SSE2-NEXT: pextrw $2, %xmm1, %eax
-; FAST_ISEL_SSE2-NEXT: shll $16, %eax
-; FAST_ISEL_SSE2-NEXT: movl %eax, {{[-0-9]+}}(%r{{[sb]}}p) # 4-byte Spill
+; FAST_ISEL_SSE2-NEXT: movd %eax, %xmm5
; FAST_ISEL_SSE2-NEXT: pextrw $1, %xmm1, %eax
-; FAST_ISEL_SSE2-NEXT: shll $16, %eax
-; FAST_ISEL_SSE2-NEXT: movl %eax, {{[-0-9]+}}(%r{{[sb]}}p) # 4-byte Spill
-; FAST_ISEL_SSE2-NEXT: movd %xmm1, %eax
-; FAST_ISEL_SSE2-NEXT: shll $16, %eax
-; FAST_ISEL_SSE2-NEXT: movl %eax, {{[-0-9]+}}(%r{{[sb]}}p) # 4-byte Spill
+; FAST_ISEL_SSE2-NEXT: pslld $16, %xmm1
+; FAST_ISEL_SSE2-NEXT: movdqa %xmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
+; FAST_ISEL_SSE2-NEXT: movd %eax, %xmm1
; FAST_ISEL_SSE2-NEXT: pextrw $7, %xmm0, %eax
-; FAST_ISEL_SSE2-NEXT: shll $16, %eax
-; FAST_ISEL_SSE2-NEXT: movl %eax, {{[-0-9]+}}(%r{{[sb]}}p) # 4-byte Spill
+; FAST_ISEL_SSE2-NEXT: movd %eax, %xmm8
; FAST_ISEL_SSE2-NEXT: pextrw $6, %xmm0, %eax
-; FAST_ISEL_SSE2-NEXT: shll $16, %eax
-; FAST_ISEL_SSE2-NEXT: movl %eax, {{[-0-9]+}}(%r{{[sb]}}p) # 4-byte Spill
+; FAST_ISEL_SSE2-NEXT: movd %eax, %xmm7
; FAST_ISEL_SSE2-NEXT: pextrw $5, %xmm0, %eax
-; FAST_ISEL_SSE2-NEXT: shll $16, %eax
-; FAST_ISEL_SSE2-NEXT: movl %eax, {{[-0-9]+}}(%r{{[sb]}}p) # 4-byte Spill
-; FAST_ISEL_SSE2-NEXT: pextrw $4, %xmm0, %eax
-; FAST_ISEL_SSE2-NEXT: shll $16, %eax
-; FAST_ISEL_SSE2-NEXT: movl %eax, {{[-0-9]+}}(%r{{[sb]}}p) # 4-byte Spill
+; FAST_ISEL_SSE2-NEXT: movd %eax, %xmm9
; FAST_ISEL_SSE2-NEXT: pextrw $3, %xmm0, %eax
-; FAST_ISEL_SSE2-NEXT: shll $16, %eax
-; FAST_ISEL_SSE2-NEXT: movl %eax, {{[-0-9]+}}(%r{{[sb]}}p) # 4-byte Spill
+; FAST_ISEL_SSE2-NEXT: movd %eax, %xmm11
; FAST_ISEL_SSE2-NEXT: pextrw $2, %xmm0, %eax
-; FAST_ISEL_SSE2-NEXT: shll $16, %eax
-; FAST_ISEL_SSE2-NEXT: movl %eax, {{[-0-9]+}}(%r{{[sb]}}p) # 4-byte Spill
+; FAST_ISEL_SSE2-NEXT: movd %eax, %xmm10
; FAST_ISEL_SSE2-NEXT: pextrw $1, %xmm0, %eax
-; FAST_ISEL_SSE2-NEXT: shll $16, %eax
-; FAST_ISEL_SSE2-NEXT: movd %eax, %xmm1
-; FAST_ISEL_SSE2-NEXT: movd %xmm0, %eax
-; FAST_ISEL_SSE2-NEXT: shll $16, %eax
-; FAST_ISEL_SSE2-NEXT: movl %eax, {{[-0-9]+}}(%r{{[sb]}}p) # 4-byte Spill
-; FAST_ISEL_SSE2-NEXT: movdqa %xmm1, %xmm0
+; FAST_ISEL_SSE2-NEXT: pslld $16, %xmm0
+; FAST_ISEL_SSE2-NEXT: movdqa %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
+; FAST_ISEL_SSE2-NEXT: pslld $16, %xmm3
+; FAST_ISEL_SSE2-NEXT: movdqa %xmm3, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
+; FAST_ISEL_SSE2-NEXT: pslld $16, %xmm2
+; FAST_ISEL_SSE2-NEXT: movdqa %xmm2, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
+; FAST_ISEL_SSE2-NEXT: pslld $16, %xmm4
+; FAST_ISEL_SSE2-NEXT: movdqa %xmm4, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
+; FAST_ISEL_SSE2-NEXT: pslld $16, %xmm6
+; FAST_ISEL_SSE2-NEXT: movdqa %xmm6, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
+; FAST_ISEL_SSE2-NEXT: pslld $16, %xmm5
+; FAST_ISEL_SSE2-NEXT: movdqa %xmm5, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
+; FAST_ISEL_SSE2-NEXT: pslld $16, %xmm1
+; FAST_ISEL_SSE2-NEXT: movdqa %xmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
+; FAST_ISEL_SSE2-NEXT: pslld $16, %xmm8
+; FAST_ISEL_SSE2-NEXT: movdqa %xmm8, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
+; FAST_ISEL_SSE2-NEXT: pslld $16, %xmm7
+; FAST_ISEL_SSE2-NEXT: movdqa %xmm7, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
+; FAST_ISEL_SSE2-NEXT: pslld $16, %xmm9
+; FAST_ISEL_SSE2-NEXT: movdqa %xmm9, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
+; FAST_ISEL_SSE2-NEXT: pslld $16, %xmm11
+; FAST_ISEL_SSE2-NEXT: movdqa %xmm11, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
+; FAST_ISEL_SSE2-NEXT: pslld $16, %xmm10
+; FAST_ISEL_SSE2-NEXT: movdqa %xmm10, (%rsp) # 16-byte Spill
+; FAST_ISEL_SSE2-NEXT: movd %eax, %xmm0
+; FAST_ISEL_SSE2-NEXT: pslld $16, %xmm0
; FAST_ISEL_SSE2-NEXT: callq __truncsfbf2 at PLT
; FAST_ISEL_SSE2-NEXT: pextrw $0, %xmm0, %ebx
; FAST_ISEL_SSE2-NEXT: shll $16, %ebx
-; FAST_ISEL_SSE2-NEXT: movd {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 4-byte Folded Reload
-; FAST_ISEL_SSE2-NEXT: # xmm0 = mem[0],zero,zero,zero
+; FAST_ISEL_SSE2-NEXT: movdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 16-byte Reload
; FAST_ISEL_SSE2-NEXT: callq __truncsfbf2 at PLT
; FAST_ISEL_SSE2-NEXT: pextrw $0, %xmm0, %eax
; FAST_ISEL_SSE2-NEXT: movzwl %ax, %r14d
; FAST_ISEL_SSE2-NEXT: orl %ebx, %r14d
-; FAST_ISEL_SSE2-NEXT: movd {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 4-byte Folded Reload
-; FAST_ISEL_SSE2-NEXT: # xmm0 = mem[0],zero,zero,zero
+; FAST_ISEL_SSE2-NEXT: movdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 16-byte Reload
; FAST_ISEL_SSE2-NEXT: callq __truncsfbf2 at PLT
; FAST_ISEL_SSE2-NEXT: pextrw $0, %xmm0, %ebx
; FAST_ISEL_SSE2-NEXT: shll $16, %ebx
-; FAST_ISEL_SSE2-NEXT: movd {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 4-byte Folded Reload
-; FAST_ISEL_SSE2-NEXT: # xmm0 = mem[0],zero,zero,zero
+; FAST_ISEL_SSE2-NEXT: movdqa (%rsp), %xmm0 # 16-byte Reload
; FAST_ISEL_SSE2-NEXT: callq __truncsfbf2 at PLT
; FAST_ISEL_SSE2-NEXT: pextrw $0, %xmm0, %eax
; FAST_ISEL_SSE2-NEXT: movzwl %ax, %eax
@@ -433,25 +416,21 @@ define <16 x bfloat> @return_arg_v16bf16(<16 x bfloat> %x) #0 {
; FAST_ISEL_SSE2-NEXT: shlq $32, %rax
; FAST_ISEL_SSE2-NEXT: orq %r14, %rax
; FAST_ISEL_SSE2-NEXT: movq %rax, %xmm0
-; FAST_ISEL_SSE2-NEXT: movdqa %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
-; FAST_ISEL_SSE2-NEXT: movd {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 4-byte Folded Reload
-; FAST_ISEL_SSE2-NEXT: # xmm0 = mem[0],zero,zero,zero
+; FAST_ISEL_SSE2-NEXT: movdqa %xmm0, (%rsp) # 16-byte Spill
+; FAST_ISEL_SSE2-NEXT: movdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 16-byte Reload
; FAST_ISEL_SSE2-NEXT: callq __truncsfbf2 at PLT
; FAST_ISEL_SSE2-NEXT: pextrw $0, %xmm0, %ebx
; FAST_ISEL_SSE2-NEXT: shll $16, %ebx
-; FAST_ISEL_SSE2-NEXT: movd {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 4-byte Folded Reload
-; FAST_ISEL_SSE2-NEXT: # xmm0 = mem[0],zero,zero,zero
+; FAST_ISEL_SSE2-NEXT: movdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 16-byte Reload
; FAST_ISEL_SSE2-NEXT: callq __truncsfbf2 at PLT
; FAST_ISEL_SSE2-NEXT: pextrw $0, %xmm0, %eax
; FAST_ISEL_SSE2-NEXT: movzwl %ax, %r14d
; FAST_ISEL_SSE2-NEXT: orl %ebx, %r14d
-; FAST_ISEL_SSE2-NEXT: movd {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 4-byte Folded Reload
-; FAST_ISEL_SSE2-NEXT: # xmm0 = mem[0],zero,zero,zero
+; FAST_ISEL_SSE2-NEXT: movdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 16-byte Reload
; FAST_ISEL_SSE2-NEXT: callq __truncsfbf2 at PLT
; FAST_ISEL_SSE2-NEXT: pextrw $0, %xmm0, %ebx
; FAST_ISEL_SSE2-NEXT: shll $16, %ebx
-; FAST_ISEL_SSE2-NEXT: movd {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 4-byte Folded Reload
-; FAST_ISEL_SSE2-NEXT: # xmm0 = mem[0],zero,zero,zero
+; FAST_ISEL_SSE2-NEXT: movdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 16-byte Reload
; FAST_ISEL_SSE2-NEXT: callq __truncsfbf2 at PLT
; FAST_ISEL_SSE2-NEXT: pextrw $0, %xmm0, %eax
; FAST_ISEL_SSE2-NEXT: movzwl %ax, %eax
@@ -459,27 +438,23 @@ define <16 x bfloat> @return_arg_v16bf16(<16 x bfloat> %x) #0 {
; FAST_ISEL_SSE2-NEXT: shlq $32, %rax
; FAST_ISEL_SSE2-NEXT: orq %r14, %rax
; FAST_ISEL_SSE2-NEXT: movq %rax, %xmm0
-; FAST_ISEL_SSE2-NEXT: movdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm1 # 16-byte Reload
+; FAST_ISEL_SSE2-NEXT: movdqa (%rsp), %xmm1 # 16-byte Reload
; FAST_ISEL_SSE2-NEXT: punpcklqdq {{.*#+}} xmm1 = xmm1[0],xmm0[0]
-; FAST_ISEL_SSE2-NEXT: movdqa %xmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
-; FAST_ISEL_SSE2-NEXT: movd {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 4-byte Folded Reload
-; FAST_ISEL_SSE2-NEXT: # xmm0 = mem[0],zero,zero,zero
+; FAST_ISEL_SSE2-NEXT: movdqa %xmm1, (%rsp) # 16-byte Spill
+; FAST_ISEL_SSE2-NEXT: movdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 16-byte Reload
; FAST_ISEL_SSE2-NEXT: callq __truncsfbf2 at PLT
; FAST_ISEL_SSE2-NEXT: pextrw $0, %xmm0, %ebx
; FAST_ISEL_SSE2-NEXT: shll $16, %ebx
-; FAST_ISEL_SSE2-NEXT: movd {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 4-byte Folded Reload
-; FAST_ISEL_SSE2-NEXT: # xmm0 = mem[0],zero,zero,zero
+; FAST_ISEL_SSE2-NEXT: movdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 16-byte Reload
; FAST_ISEL_SSE2-NEXT: callq __truncsfbf2 at PLT
; FAST_ISEL_SSE2-NEXT: pextrw $0, %xmm0, %eax
; FAST_ISEL_SSE2-NEXT: movzwl %ax, %r14d
; FAST_ISEL_SSE2-NEXT: orl %ebx, %r14d
-; FAST_ISEL_SSE2-NEXT: movd {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 4-byte Folded Reload
-; FAST_ISEL_SSE2-NEXT: # xmm0 = mem[0],zero,zero,zero
+; FAST_ISEL_SSE2-NEXT: movdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 16-byte Reload
; FAST_ISEL_SSE2-NEXT: callq __truncsfbf2 at PLT
; FAST_ISEL_SSE2-NEXT: pextrw $0, %xmm0, %ebx
; FAST_ISEL_SSE2-NEXT: shll $16, %ebx
-; FAST_ISEL_SSE2-NEXT: movd {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 4-byte Folded Reload
-; FAST_ISEL_SSE2-NEXT: # xmm0 = mem[0],zero,zero,zero
+; FAST_ISEL_SSE2-NEXT: movdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 16-byte Reload
; FAST_ISEL_SSE2-NEXT: callq __truncsfbf2 at PLT
; FAST_ISEL_SSE2-NEXT: pextrw $0, %xmm0, %eax
; FAST_ISEL_SSE2-NEXT: movzwl %ax, %eax
@@ -488,24 +463,20 @@ define <16 x bfloat> @return_arg_v16bf16(<16 x bfloat> %x) #0 {
; FAST_ISEL_SSE2-NEXT: orq %r14, %rax
; FAST_ISEL_SSE2-NEXT: movq %rax, %xmm0
; FAST_ISEL_SSE2-NEXT: movdqa %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
-; FAST_ISEL_SSE2-NEXT: movd {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 4-byte Folded Reload
-; FAST_ISEL_SSE2-NEXT: # xmm0 = mem[0],zero,zero,zero
+; FAST_ISEL_SSE2-NEXT: movdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 16-byte Reload
; FAST_ISEL_SSE2-NEXT: callq __truncsfbf2 at PLT
; FAST_ISEL_SSE2-NEXT: pextrw $0, %xmm0, %ebx
; FAST_ISEL_SSE2-NEXT: shll $16, %ebx
-; FAST_ISEL_SSE2-NEXT: movd {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 4-byte Folded Reload
-; FAST_ISEL_SSE2-NEXT: # xmm0 = mem[0],zero,zero,zero
+; FAST_ISEL_SSE2-NEXT: movdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 16-byte Reload
; FAST_ISEL_SSE2-NEXT: callq __truncsfbf2 at PLT
; FAST_ISEL_SSE2-NEXT: pextrw $0, %xmm0, %eax
; FAST_ISEL_SSE2-NEXT: movzwl %ax, %r14d
; FAST_ISEL_SSE2-NEXT: orl %ebx, %r14d
-; FAST_ISEL_SSE2-NEXT: movd {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 4-byte Folded Reload
-; FAST_ISEL_SSE2-NEXT: # xmm0 = mem[0],zero,zero,zero
+; FAST_ISEL_SSE2-NEXT: movdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 16-byte Reload
; FAST_ISEL_SSE2-NEXT: callq __truncsfbf2 at PLT
; FAST_ISEL_SSE2-NEXT: pextrw $0, %xmm0, %ebx
; FAST_ISEL_SSE2-NEXT: shll $16, %ebx
-; FAST_ISEL_SSE2-NEXT: movd {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 4-byte Folded Reload
-; FAST_ISEL_SSE2-NEXT: # xmm0 = mem[0],zero,zero,zero
+; FAST_ISEL_SSE2-NEXT: movdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 16-byte Reload
; FAST_ISEL_SSE2-NEXT: callq __truncsfbf2 at PLT
; FAST_ISEL_SSE2-NEXT: pextrw $0, %xmm0, %eax
; FAST_ISEL_SSE2-NEXT: movzwl %ax, %eax
@@ -515,8 +486,8 @@ define <16 x bfloat> @return_arg_v16bf16(<16 x bfloat> %x) #0 {
; FAST_ISEL_SSE2-NEXT: movq %rax, %xmm0
; FAST_ISEL_SSE2-NEXT: movdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm1 # 16-byte Reload
; FAST_ISEL_SSE2-NEXT: punpcklqdq {{.*#+}} xmm1 = xmm1[0],xmm0[0]
-; FAST_ISEL_SSE2-NEXT: movaps {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 16-byte Reload
-; FAST_ISEL_SSE2-NEXT: addq $104, %rsp
+; FAST_ISEL_SSE2-NEXT: movaps (%rsp), %xmm0 # 16-byte Reload
+; FAST_ISEL_SSE2-NEXT: addq $248, %rsp
; FAST_ISEL_SSE2-NEXT: popq %rbx
; FAST_ISEL_SSE2-NEXT: popq %r14
; FAST_ISEL_SSE2-NEXT: retq
@@ -607,20 +578,18 @@ define <2 x bfloat> @call_ret_v2bf16(ptr %ptr) #0 {
;
; FAST_ISEL_SSE2-LABEL: call_ret_v2bf16:
; FAST_ISEL_SSE2: # %bb.0:
-; FAST_ISEL_SSE2-NEXT: subq $40, %rsp
+; FAST_ISEL_SSE2-NEXT: subq $56, %rsp
; FAST_ISEL_SSE2-NEXT: movl (%rdi), %eax
; FAST_ISEL_SSE2-NEXT: movl %eax, (%rsp)
; FAST_ISEL_SSE2-NEXT: movdqa (%rsp), %xmm0
; FAST_ISEL_SSE2-NEXT: pextrw $1, %xmm0, %eax
-; FAST_ISEL_SSE2-NEXT: shll $16, %eax
-; FAST_ISEL_SSE2-NEXT: movl %eax, {{[-0-9]+}}(%r{{[sb]}}p) # 4-byte Spill
-; FAST_ISEL_SSE2-NEXT: movd %xmm0, %eax
-; FAST_ISEL_SSE2-NEXT: shll $16, %eax
-; FAST_ISEL_SSE2-NEXT: movd %eax, %xmm0
+; FAST_ISEL_SSE2-NEXT: pslld $16, %xmm0
+; FAST_ISEL_SSE2-NEXT: movd %eax, %xmm1
+; FAST_ISEL_SSE2-NEXT: pslld $16, %xmm1
+; FAST_ISEL_SSE2-NEXT: movdqa %xmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
; FAST_ISEL_SSE2-NEXT: callq __truncsfbf2 at PLT
; FAST_ISEL_SSE2-NEXT: movdqa %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
-; FAST_ISEL_SSE2-NEXT: movd {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 4-byte Folded Reload
-; FAST_ISEL_SSE2-NEXT: # xmm0 = mem[0],zero,zero,zero
+; FAST_ISEL_SSE2-NEXT: movdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 16-byte Reload
; FAST_ISEL_SSE2-NEXT: callq __truncsfbf2 at PLT
; FAST_ISEL_SSE2-NEXT: movdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm1 # 16-byte Reload
; FAST_ISEL_SSE2-NEXT: punpcklwd {{.*#+}} xmm1 = xmm1[0],xmm0[0],xmm1[1],xmm0[1],xmm1[2],xmm0[2],xmm1[3],xmm0[3]
@@ -667,29 +636,28 @@ define <3 x bfloat> @call_ret_v3bf16(ptr %ptr) #0 {
;
; FAST_ISEL_SSE2-LABEL: call_ret_v3bf16:
; FAST_ISEL_SSE2: # %bb.0:
-; FAST_ISEL_SSE2-NEXT: subq $40, %rsp
+; FAST_ISEL_SSE2-NEXT: subq $56, %rsp
; FAST_ISEL_SSE2-NEXT: movq (%rdi), %rax
-; FAST_ISEL_SSE2-NEXT: movl %eax, %ecx
-; FAST_ISEL_SSE2-NEXT: andl $-65536, %ecx # imm = 0xFFFF0000
-; FAST_ISEL_SSE2-NEXT: movl %ecx, {{[-0-9]+}}(%r{{[sb]}}p) # 4-byte Spill
-; FAST_ISEL_SSE2-NEXT: movl %eax, %ecx
-; FAST_ISEL_SSE2-NEXT: shll $16, %ecx
-; FAST_ISEL_SSE2-NEXT: movd %ecx, %xmm0
+; FAST_ISEL_SSE2-NEXT: movq %rax, %xmm0
+; FAST_ISEL_SSE2-NEXT: pslld $16, %xmm0
+; FAST_ISEL_SSE2-NEXT: movd %eax, %xmm2
; FAST_ISEL_SSE2-NEXT: shrq $32, %rax
-; FAST_ISEL_SSE2-NEXT: shll $16, %eax
-; FAST_ISEL_SSE2-NEXT: movl %eax, {{[-0-9]+}}(%r{{[sb]}}p) # 4-byte Spill
+; FAST_ISEL_SSE2-NEXT: movd %eax, %xmm1
+; FAST_ISEL_SSE2-NEXT: pslld $16, %xmm1
+; FAST_ISEL_SSE2-NEXT: movdqa %xmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
+; FAST_ISEL_SSE2-NEXT: psrld $16, %xmm2
+; FAST_ISEL_SSE2-NEXT: pslld $16, %xmm2
+; FAST_ISEL_SSE2-NEXT: movdqa %xmm2, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
; FAST_ISEL_SSE2-NEXT: callq __truncsfbf2 at PLT
-; FAST_ISEL_SSE2-NEXT: movdqa %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
-; FAST_ISEL_SSE2-NEXT: movd {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 4-byte Folded Reload
-; FAST_ISEL_SSE2-NEXT: # xmm0 = mem[0],zero,zero,zero
+; FAST_ISEL_SSE2-NEXT: movdqa %xmm0, (%rsp) # 16-byte Spill
+; FAST_ISEL_SSE2-NEXT: movdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 16-byte Reload
; FAST_ISEL_SSE2-NEXT: callq __truncsfbf2 at PLT
-; FAST_ISEL_SSE2-NEXT: movdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm1 # 16-byte Reload
+; FAST_ISEL_SSE2-NEXT: movdqa (%rsp), %xmm1 # 16-byte Reload
; FAST_ISEL_SSE2-NEXT: punpcklwd {{.*#+}} xmm1 = xmm1[0],xmm0[0],xmm1[1],xmm0[1],xmm1[2],xmm0[2],xmm1[3],xmm0[3]
-; FAST_ISEL_SSE2-NEXT: movdqa %xmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
-; FAST_ISEL_SSE2-NEXT: movss {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 4-byte Reload
-; FAST_ISEL_SSE2-NEXT: # xmm0 = mem[0],zero,zero,zero
+; FAST_ISEL_SSE2-NEXT: movdqa %xmm1, (%rsp) # 16-byte Spill
+; FAST_ISEL_SSE2-NEXT: movaps {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 16-byte Reload
; FAST_ISEL_SSE2-NEXT: callq __truncsfbf2 at PLT
-; FAST_ISEL_SSE2-NEXT: movaps {{[-0-9]+}}(%r{{[sb]}}p), %xmm1 # 16-byte Reload
+; FAST_ISEL_SSE2-NEXT: movaps (%rsp), %xmm1 # 16-byte Reload
; FAST_ISEL_SSE2-NEXT: unpcklps {{.*#+}} xmm1 = xmm1[0],xmm0[0],xmm1[1],xmm0[1]
; FAST_ISEL_SSE2-NEXT: movaps %xmm1, %xmm0
; FAST_ISEL_SSE2-NEXT: callq returns_v3bf16 at PLT
@@ -704,15 +672,14 @@ define <3 x bfloat> @call_ret_v3bf16(ptr %ptr) #0 {
; FAST_ISEL_AVX512BF16: # %bb.0:
; FAST_ISEL_AVX512BF16-NEXT: pushq %rax
; FAST_ISEL_AVX512BF16-NEXT: movq (%rdi), %rax
-; FAST_ISEL_AVX512BF16-NEXT: movl %eax, %ecx
-; FAST_ISEL_AVX512BF16-NEXT: andl $-65536, %ecx # imm = 0xFFFF0000
-; FAST_ISEL_AVX512BF16-NEXT: vmovd %ecx, %xmm0
-; FAST_ISEL_AVX512BF16-NEXT: movl %eax, %ecx
-; FAST_ISEL_AVX512BF16-NEXT: shll $16, %ecx
-; FAST_ISEL_AVX512BF16-NEXT: vmovd %ecx, %xmm1
+; FAST_ISEL_AVX512BF16-NEXT: vmovd %eax, %xmm0
+; FAST_ISEL_AVX512BF16-NEXT: vpxor %xmm1, %xmm1, %xmm1
+; FAST_ISEL_AVX512BF16-NEXT: vpblendw {{.*#+}} xmm0 = xmm1[0],xmm0[1],xmm1[2],xmm0[3],xmm1[4],xmm0[5],xmm1[6],xmm0[7]
+; FAST_ISEL_AVX512BF16-NEXT: vmovq %rax, %xmm1
+; FAST_ISEL_AVX512BF16-NEXT: vpslld $16, %xmm1, %xmm1
; FAST_ISEL_AVX512BF16-NEXT: shrq $32, %rax
-; FAST_ISEL_AVX512BF16-NEXT: shll $16, %eax
; FAST_ISEL_AVX512BF16-NEXT: vmovd %eax, %xmm2
+; FAST_ISEL_AVX512BF16-NEXT: vpslld $16, %xmm2, %xmm2
; FAST_ISEL_AVX512BF16-NEXT: vcvtneps2bf16 %xmm2, %xmm2
; FAST_ISEL_AVX512BF16-NEXT: vmovd %xmm2, %eax
; FAST_ISEL_AVX512BF16-NEXT: vcvtneps2bf16 %xmm1, %xmm1
@@ -733,15 +700,14 @@ define <3 x bfloat> @call_ret_v3bf16(ptr %ptr) #0 {
; FAST_ISEL_AVXNECONVERT: # %bb.0:
; FAST_ISEL_AVXNECONVERT-NEXT: pushq %rax
; FAST_ISEL_AVXNECONVERT-NEXT: movq (%rdi), %rax
-; FAST_ISEL_AVXNECONVERT-NEXT: movl %eax, %ecx
-; FAST_ISEL_AVXNECONVERT-NEXT: andl $-65536, %ecx # imm = 0xFFFF0000
-; FAST_ISEL_AVXNECONVERT-NEXT: vmovd %ecx, %xmm0
-; FAST_ISEL_AVXNECONVERT-NEXT: movl %eax, %ecx
-; FAST_ISEL_AVXNECONVERT-NEXT: shll $16, %ecx
-; FAST_ISEL_AVXNECONVERT-NEXT: vmovd %ecx, %xmm1
+; FAST_ISEL_AVXNECONVERT-NEXT: vmovd %eax, %xmm0
+; FAST_ISEL_AVXNECONVERT-NEXT: vpxor %xmm1, %xmm1, %xmm1
+; FAST_ISEL_AVXNECONVERT-NEXT: vpblendw {{.*#+}} xmm0 = xmm1[0],xmm0[1],xmm1[2],xmm0[3],xmm1[4],xmm0[5],xmm1[6],xmm0[7]
+; FAST_ISEL_AVXNECONVERT-NEXT: vmovq %rax, %xmm1
+; FAST_ISEL_AVXNECONVERT-NEXT: vpslld $16, %xmm1, %xmm1
; FAST_ISEL_AVXNECONVERT-NEXT: shrq $32, %rax
-; FAST_ISEL_AVXNECONVERT-NEXT: shll $16, %eax
; FAST_ISEL_AVXNECONVERT-NEXT: vmovd %eax, %xmm2
+; FAST_ISEL_AVXNECONVERT-NEXT: vpslld $16, %xmm2, %xmm2
; FAST_ISEL_AVXNECONVERT-NEXT: {vex} vcvtneps2bf16 %xmm2, %xmm2
; FAST_ISEL_AVXNECONVERT-NEXT: vmovd %xmm2, %eax
; FAST_ISEL_AVXNECONVERT-NEXT: {vex} vcvtneps2bf16 %xmm1, %xmm1
@@ -773,38 +739,36 @@ define <4 x bfloat> @call_ret_v4bf16(ptr %ptr) #0 {
;
; FAST_ISEL_SSE2-LABEL: call_ret_v4bf16:
; FAST_ISEL_SSE2: # %bb.0:
-; FAST_ISEL_SSE2-NEXT: subq $56, %rsp
+; FAST_ISEL_SSE2-NEXT: subq $88, %rsp
; FAST_ISEL_SSE2-NEXT: movq (%rdi), %rax
; FAST_ISEL_SSE2-NEXT: movq %rax, {{[0-9]+}}(%rsp)
; FAST_ISEL_SSE2-NEXT: movdqa {{[0-9]+}}(%rsp), %xmm0
; FAST_ISEL_SSE2-NEXT: pextrw $3, %xmm0, %eax
-; FAST_ISEL_SSE2-NEXT: shll $16, %eax
-; FAST_ISEL_SSE2-NEXT: movl %eax, {{[-0-9]+}}(%r{{[sb]}}p) # 4-byte Spill
+; FAST_ISEL_SSE2-NEXT: movd %eax, %xmm1
; FAST_ISEL_SSE2-NEXT: pextrw $2, %xmm0, %eax
-; FAST_ISEL_SSE2-NEXT: shll $16, %eax
-; FAST_ISEL_SSE2-NEXT: movl %eax, {{[-0-9]+}}(%r{{[sb]}}p) # 4-byte Spill
+; FAST_ISEL_SSE2-NEXT: movd %eax, %xmm2
; FAST_ISEL_SSE2-NEXT: pextrw $1, %xmm0, %eax
-; FAST_ISEL_SSE2-NEXT: shll $16, %eax
-; FAST_ISEL_SSE2-NEXT: movl %eax, {{[-0-9]+}}(%r{{[sb]}}p) # 4-byte Spill
-; FAST_ISEL_SSE2-NEXT: movd %xmm0, %eax
-; FAST_ISEL_SSE2-NEXT: shll $16, %eax
-; FAST_ISEL_SSE2-NEXT: movd %eax, %xmm0
+; FAST_ISEL_SSE2-NEXT: pslld $16, %xmm0
+; FAST_ISEL_SSE2-NEXT: pslld $16, %xmm1
+; FAST_ISEL_SSE2-NEXT: movdqa %xmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
+; FAST_ISEL_SSE2-NEXT: pslld $16, %xmm2
+; FAST_ISEL_SSE2-NEXT: movdqa %xmm2, (%rsp) # 16-byte Spill
+; FAST_ISEL_SSE2-NEXT: movd %eax, %xmm1
+; FAST_ISEL_SSE2-NEXT: pslld $16, %xmm1
+; FAST_ISEL_SSE2-NEXT: movdqa %xmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
; FAST_ISEL_SSE2-NEXT: callq __truncsfbf2 at PLT
; FAST_ISEL_SSE2-NEXT: movdqa %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
-; FAST_ISEL_SSE2-NEXT: movd {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 4-byte Folded Reload
-; FAST_ISEL_SSE2-NEXT: # xmm0 = mem[0],zero,zero,zero
+; FAST_ISEL_SSE2-NEXT: movdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 16-byte Reload
; FAST_ISEL_SSE2-NEXT: callq __truncsfbf2 at PLT
; FAST_ISEL_SSE2-NEXT: movdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm1 # 16-byte Reload
; FAST_ISEL_SSE2-NEXT: punpcklwd {{.*#+}} xmm1 = xmm1[0],xmm0[0],xmm1[1],xmm0[1],xmm1[2],xmm0[2],xmm1[3],xmm0[3]
; FAST_ISEL_SSE2-NEXT: movdqa %xmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
-; FAST_ISEL_SSE2-NEXT: movss {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 4-byte Reload
-; FAST_ISEL_SSE2-NEXT: # xmm0 = mem[0],zero,zero,zero
+; FAST_ISEL_SSE2-NEXT: movaps (%rsp), %xmm0 # 16-byte Reload
; FAST_ISEL_SSE2-NEXT: callq __truncsfbf2 at PLT
-; FAST_ISEL_SSE2-NEXT: movaps %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
-; FAST_ISEL_SSE2-NEXT: movd {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 4-byte Folded Reload
-; FAST_ISEL_SSE2-NEXT: # xmm0 = mem[0],zero,zero,zero
+; FAST_ISEL_SSE2-NEXT: movaps %xmm0, (%rsp) # 16-byte Spill
+; FAST_ISEL_SSE2-NEXT: movdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 16-byte Reload
; FAST_ISEL_SSE2-NEXT: callq __truncsfbf2 at PLT
-; FAST_ISEL_SSE2-NEXT: movdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm1 # 16-byte Reload
+; FAST_ISEL_SSE2-NEXT: movdqa (%rsp), %xmm1 # 16-byte Reload
; FAST_ISEL_SSE2-NEXT: punpcklwd {{.*#+}} xmm1 = xmm1[0],xmm0[0],xmm1[1],xmm0[1],xmm1[2],xmm0[2],xmm1[3],xmm0[3]
; FAST_ISEL_SSE2-NEXT: movdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 16-byte Reload
; FAST_ISEL_SSE2-NEXT: punpckldq {{.*#+}} xmm0 = xmm0[0],xmm1[0],xmm0[1],xmm1[1]
@@ -850,48 +814,49 @@ define <8 x bfloat> @call_ret_v8bf16(ptr %ptr) #0 {
; FAST_ISEL_SSE2: # %bb.0:
; FAST_ISEL_SSE2-NEXT: pushq %r14
; FAST_ISEL_SSE2-NEXT: pushq %rbx
-; FAST_ISEL_SSE2-NEXT: subq $56, %rsp
-; FAST_ISEL_SSE2-NEXT: movdqa (%rdi), %xmm1
-; FAST_ISEL_SSE2-NEXT: pextrw $7, %xmm1, %eax
-; FAST_ISEL_SSE2-NEXT: shll $16, %eax
-; FAST_ISEL_SSE2-NEXT: movl %eax, {{[-0-9]+}}(%r{{[sb]}}p) # 4-byte Spill
-; FAST_ISEL_SSE2-NEXT: pextrw $6, %xmm1, %eax
-; FAST_ISEL_SSE2-NEXT: shll $16, %eax
-; FAST_ISEL_SSE2-NEXT: movl %eax, {{[-0-9]+}}(%r{{[sb]}}p) # 4-byte Spill
-; FAST_ISEL_SSE2-NEXT: pextrw $5, %xmm1, %eax
-; FAST_ISEL_SSE2-NEXT: shll $16, %eax
-; FAST_ISEL_SSE2-NEXT: movl %eax, {{[-0-9]+}}(%r{{[sb]}}p) # 4-byte Spill
-; FAST_ISEL_SSE2-NEXT: pextrw $4, %xmm1, %eax
-; FAST_ISEL_SSE2-NEXT: shll $16, %eax
-; FAST_ISEL_SSE2-NEXT: movl %eax, {{[-0-9]+}}(%r{{[sb]}}p) # 4-byte Spill
-; FAST_ISEL_SSE2-NEXT: pextrw $3, %xmm1, %eax
-; FAST_ISEL_SSE2-NEXT: shll $16, %eax
-; FAST_ISEL_SSE2-NEXT: movl %eax, {{[-0-9]+}}(%r{{[sb]}}p) # 4-byte Spill
-; FAST_ISEL_SSE2-NEXT: pextrw $2, %xmm1, %eax
-; FAST_ISEL_SSE2-NEXT: shll $16, %eax
-; FAST_ISEL_SSE2-NEXT: movl %eax, {{[-0-9]+}}(%r{{[sb]}}p) # 4-byte Spill
-; FAST_ISEL_SSE2-NEXT: pextrw $1, %xmm1, %eax
-; FAST_ISEL_SSE2-NEXT: shll $16, %eax
+; FAST_ISEL_SSE2-NEXT: subq $120, %rsp
+; FAST_ISEL_SSE2-NEXT: movdqa (%rdi), %xmm0
+; FAST_ISEL_SSE2-NEXT: pxor %xmm1, %xmm1
+; FAST_ISEL_SSE2-NEXT: punpckhwd {{.*#+}} xmm1 = xmm1[4],xmm0[4],xmm1[5],xmm0[5],xmm1[6],xmm0[6],xmm1[7],xmm0[7]
+; FAST_ISEL_SSE2-NEXT: movdqa %xmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
+; FAST_ISEL_SSE2-NEXT: pextrw $7, %xmm0, %eax
+; FAST_ISEL_SSE2-NEXT: movd %eax, %xmm2
+; FAST_ISEL_SSE2-NEXT: pextrw $6, %xmm0, %eax
+; FAST_ISEL_SSE2-NEXT: movd %eax, %xmm1
+; FAST_ISEL_SSE2-NEXT: pextrw $5, %xmm0, %eax
+; FAST_ISEL_SSE2-NEXT: movd %eax, %xmm3
+; FAST_ISEL_SSE2-NEXT: pextrw $3, %xmm0, %eax
+; FAST_ISEL_SSE2-NEXT: movd %eax, %xmm5
+; FAST_ISEL_SSE2-NEXT: pextrw $2, %xmm0, %eax
+; FAST_ISEL_SSE2-NEXT: movd %eax, %xmm4
+; FAST_ISEL_SSE2-NEXT: pextrw $1, %xmm0, %eax
+; FAST_ISEL_SSE2-NEXT: pslld $16, %xmm0
+; FAST_ISEL_SSE2-NEXT: movdqa %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
+; FAST_ISEL_SSE2-NEXT: pslld $16, %xmm2
+; FAST_ISEL_SSE2-NEXT: movdqa %xmm2, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
+; FAST_ISEL_SSE2-NEXT: pslld $16, %xmm1
+; FAST_ISEL_SSE2-NEXT: movdqa %xmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
+; FAST_ISEL_SSE2-NEXT: pslld $16, %xmm3
+; FAST_ISEL_SSE2-NEXT: movdqa %xmm3, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
+; FAST_ISEL_SSE2-NEXT: pslld $16, %xmm5
+; FAST_ISEL_SSE2-NEXT: movdqa %xmm5, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
+; FAST_ISEL_SSE2-NEXT: pslld $16, %xmm4
+; FAST_ISEL_SSE2-NEXT: movdqa %xmm4, (%rsp) # 16-byte Spill
; FAST_ISEL_SSE2-NEXT: movd %eax, %xmm0
-; FAST_ISEL_SSE2-NEXT: movd %xmm1, %eax
-; FAST_ISEL_SSE2-NEXT: shll $16, %eax
-; FAST_ISEL_SSE2-NEXT: movl %eax, {{[-0-9]+}}(%r{{[sb]}}p) # 4-byte Spill
+; FAST_ISEL_SSE2-NEXT: pslld $16, %xmm0
; FAST_ISEL_SSE2-NEXT: callq __truncsfbf2 at PLT
; FAST_ISEL_SSE2-NEXT: pextrw $0, %xmm0, %ebx
; FAST_ISEL_SSE2-NEXT: shll $16, %ebx
-; FAST_ISEL_SSE2-NEXT: movd {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 4-byte Folded Reload
-; FAST_ISEL_SSE2-NEXT: # xmm0 = mem[0],zero,zero,zero
+; FAST_ISEL_SSE2-NEXT: movdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 16-byte Reload
; FAST_ISEL_SSE2-NEXT: callq __truncsfbf2 at PLT
; FAST_ISEL_SSE2-NEXT: pextrw $0, %xmm0, %eax
; FAST_ISEL_SSE2-NEXT: movzwl %ax, %r14d
; FAST_ISEL_SSE2-NEXT: orl %ebx, %r14d
-; FAST_ISEL_SSE2-NEXT: movd {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 4-byte Folded Reload
-; FAST_ISEL_SSE2-NEXT: # xmm0 = mem[0],zero,zero,zero
+; FAST_ISEL_SSE2-NEXT: movdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 16-byte Reload
; FAST_ISEL_SSE2-NEXT: callq __truncsfbf2 at PLT
; FAST_ISEL_SSE2-NEXT: pextrw $0, %xmm0, %ebx
; FAST_ISEL_SSE2-NEXT: shll $16, %ebx
-; FAST_ISEL_SSE2-NEXT: movd {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 4-byte Folded Reload
-; FAST_ISEL_SSE2-NEXT: # xmm0 = mem[0],zero,zero,zero
+; FAST_ISEL_SSE2-NEXT: movdqa (%rsp), %xmm0 # 16-byte Reload
; FAST_ISEL_SSE2-NEXT: callq __truncsfbf2 at PLT
; FAST_ISEL_SSE2-NEXT: pextrw $0, %xmm0, %eax
; FAST_ISEL_SSE2-NEXT: movzwl %ax, %eax
@@ -899,25 +864,21 @@ define <8 x bfloat> @call_ret_v8bf16(ptr %ptr) #0 {
; FAST_ISEL_SSE2-NEXT: shlq $32, %rax
; FAST_ISEL_SSE2-NEXT: orq %r14, %rax
; FAST_ISEL_SSE2-NEXT: movq %rax, %xmm0
-; FAST_ISEL_SSE2-NEXT: movdqa %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
-; FAST_ISEL_SSE2-NEXT: movd {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 4-byte Folded Reload
-; FAST_ISEL_SSE2-NEXT: # xmm0 = mem[0],zero,zero,zero
+; FAST_ISEL_SSE2-NEXT: movdqa %xmm0, (%rsp) # 16-byte Spill
+; FAST_ISEL_SSE2-NEXT: movdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 16-byte Reload
; FAST_ISEL_SSE2-NEXT: callq __truncsfbf2 at PLT
; FAST_ISEL_SSE2-NEXT: pextrw $0, %xmm0, %ebx
; FAST_ISEL_SSE2-NEXT: shll $16, %ebx
-; FAST_ISEL_SSE2-NEXT: movd {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 4-byte Folded Reload
-; FAST_ISEL_SSE2-NEXT: # xmm0 = mem[0],zero,zero,zero
+; FAST_ISEL_SSE2-NEXT: movdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 16-byte Reload
; FAST_ISEL_SSE2-NEXT: callq __truncsfbf2 at PLT
; FAST_ISEL_SSE2-NEXT: pextrw $0, %xmm0, %eax
; FAST_ISEL_SSE2-NEXT: movzwl %ax, %r14d
; FAST_ISEL_SSE2-NEXT: orl %ebx, %r14d
-; FAST_ISEL_SSE2-NEXT: movd {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 4-byte Folded Reload
-; FAST_ISEL_SSE2-NEXT: # xmm0 = mem[0],zero,zero,zero
+; FAST_ISEL_SSE2-NEXT: movdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 16-byte Reload
; FAST_ISEL_SSE2-NEXT: callq __truncsfbf2 at PLT
; FAST_ISEL_SSE2-NEXT: pextrw $0, %xmm0, %ebx
; FAST_ISEL_SSE2-NEXT: shll $16, %ebx
-; FAST_ISEL_SSE2-NEXT: movd {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 4-byte Folded Reload
-; FAST_ISEL_SSE2-NEXT: # xmm0 = mem[0],zero,zero,zero
+; FAST_ISEL_SSE2-NEXT: movdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 16-byte Reload
; FAST_ISEL_SSE2-NEXT: callq __truncsfbf2 at PLT
; FAST_ISEL_SSE2-NEXT: pextrw $0, %xmm0, %eax
; FAST_ISEL_SSE2-NEXT: movzwl %ax, %eax
@@ -925,7 +886,7 @@ define <8 x bfloat> @call_ret_v8bf16(ptr %ptr) #0 {
; FAST_ISEL_SSE2-NEXT: shlq $32, %rax
; FAST_ISEL_SSE2-NEXT: orq %r14, %rax
; FAST_ISEL_SSE2-NEXT: movq %rax, %xmm1
-; FAST_ISEL_SSE2-NEXT: movdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 16-byte Reload
+; FAST_ISEL_SSE2-NEXT: movdqa (%rsp), %xmm0 # 16-byte Reload
; FAST_ISEL_SSE2-NEXT: punpcklqdq {{.*#+}} xmm0 = xmm0[0],xmm1[0]
; FAST_ISEL_SSE2-NEXT: callq returns_v8bf16 at PLT
;
@@ -970,73 +931,79 @@ define <16 x bfloat> @call_ret_v16bf16(ptr %ptr) #0 {
; FAST_ISEL_SSE2: # %bb.0:
; FAST_ISEL_SSE2-NEXT: pushq %r14
; FAST_ISEL_SSE2-NEXT: pushq %rbx
-; FAST_ISEL_SSE2-NEXT: subq $104, %rsp
-; FAST_ISEL_SSE2-NEXT: movdqa (%rdi), %xmm1
-; FAST_ISEL_SSE2-NEXT: movdqa 16(%rdi), %xmm0
-; FAST_ISEL_SSE2-NEXT: pextrw $7, %xmm0, %eax
-; FAST_ISEL_SSE2-NEXT: shll $16, %eax
-; FAST_ISEL_SSE2-NEXT: movl %eax, {{[-0-9]+}}(%r{{[sb]}}p) # 4-byte Spill
-; FAST_ISEL_SSE2-NEXT: pextrw $6, %xmm0, %eax
-; FAST_ISEL_SSE2-NEXT: shll $16, %eax
-; FAST_ISEL_SSE2-NEXT: movl %eax, {{[-0-9]+}}(%r{{[sb]}}p) # 4-byte Spill
-; FAST_ISEL_SSE2-NEXT: pextrw $5, %xmm0, %eax
-; FAST_ISEL_SSE2-NEXT: shll $16, %eax
-; FAST_ISEL_SSE2-NEXT: movl %eax, {{[-0-9]+}}(%r{{[sb]}}p) # 4-byte Spill
-; FAST_ISEL_SSE2-NEXT: pextrw $4, %xmm0, %eax
-; FAST_ISEL_SSE2-NEXT: shll $16, %eax
-; FAST_ISEL_SSE2-NEXT: movl %eax, {{[-0-9]+}}(%r{{[sb]}}p) # 4-byte Spill
-; FAST_ISEL_SSE2-NEXT: pextrw $3, %xmm0, %eax
-; FAST_ISEL_SSE2-NEXT: shll $16, %eax
-; FAST_ISEL_SSE2-NEXT: movl %eax, {{[-0-9]+}}(%r{{[sb]}}p) # 4-byte Spill
-; FAST_ISEL_SSE2-NEXT: pextrw $2, %xmm0, %eax
-; FAST_ISEL_SSE2-NEXT: shll $16, %eax
-; FAST_ISEL_SSE2-NEXT: movl %eax, {{[-0-9]+}}(%r{{[sb]}}p) # 4-byte Spill
-; FAST_ISEL_SSE2-NEXT: pextrw $1, %xmm0, %eax
-; FAST_ISEL_SSE2-NEXT: shll $16, %eax
-; FAST_ISEL_SSE2-NEXT: movl %eax, {{[-0-9]+}}(%r{{[sb]}}p) # 4-byte Spill
-; FAST_ISEL_SSE2-NEXT: movd %xmm0, %eax
-; FAST_ISEL_SSE2-NEXT: shll $16, %eax
-; FAST_ISEL_SSE2-NEXT: movl %eax, {{[-0-9]+}}(%r{{[sb]}}p) # 4-byte Spill
+; FAST_ISEL_SSE2-NEXT: subq $248, %rsp
+; FAST_ISEL_SSE2-NEXT: movdqa (%rdi), %xmm0
+; FAST_ISEL_SSE2-NEXT: movdqa 16(%rdi), %xmm1
+; FAST_ISEL_SSE2-NEXT: pxor %xmm3, %xmm3
+; FAST_ISEL_SSE2-NEXT: pxor %xmm2, %xmm2
+; FAST_ISEL_SSE2-NEXT: punpckhwd {{.*#+}} xmm2 = xmm2[4],xmm1[4],xmm2[5],xmm1[5],xmm2[6],xmm1[6],xmm2[7],xmm1[7]
+; FAST_ISEL_SSE2-NEXT: movdqa %xmm2, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
+; FAST_ISEL_SSE2-NEXT: punpckhwd {{.*#+}} xmm3 = xmm3[4],xmm0[4],xmm3[5],xmm0[5],xmm3[6],xmm0[6],xmm3[7],xmm0[7]
+; FAST_ISEL_SSE2-NEXT: movdqa %xmm3, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
; FAST_ISEL_SSE2-NEXT: pextrw $7, %xmm1, %eax
-; FAST_ISEL_SSE2-NEXT: shll $16, %eax
-; FAST_ISEL_SSE2-NEXT: movl %eax, {{[-0-9]+}}(%r{{[sb]}}p) # 4-byte Spill
+; FAST_ISEL_SSE2-NEXT: movd %eax, %xmm3
; FAST_ISEL_SSE2-NEXT: pextrw $6, %xmm1, %eax
-; FAST_ISEL_SSE2-NEXT: shll $16, %eax
-; FAST_ISEL_SSE2-NEXT: movl %eax, {{[-0-9]+}}(%r{{[sb]}}p) # 4-byte Spill
+; FAST_ISEL_SSE2-NEXT: movd %eax, %xmm2
; FAST_ISEL_SSE2-NEXT: pextrw $5, %xmm1, %eax
-; FAST_ISEL_SSE2-NEXT: shll $16, %eax
-; FAST_ISEL_SSE2-NEXT: movl %eax, {{[-0-9]+}}(%r{{[sb]}}p) # 4-byte Spill
-; FAST_ISEL_SSE2-NEXT: pextrw $4, %xmm1, %eax
-; FAST_ISEL_SSE2-NEXT: shll $16, %eax
-; FAST_ISEL_SSE2-NEXT: movl %eax, {{[-0-9]+}}(%r{{[sb]}}p) # 4-byte Spill
+; FAST_ISEL_SSE2-NEXT: movd %eax, %xmm4
; FAST_ISEL_SSE2-NEXT: pextrw $3, %xmm1, %eax
-; FAST_ISEL_SSE2-NEXT: shll $16, %eax
-; FAST_ISEL_SSE2-NEXT: movl %eax, {{[-0-9]+}}(%r{{[sb]}}p) # 4-byte Spill
+; FAST_ISEL_SSE2-NEXT: movd %eax, %xmm6
; FAST_ISEL_SSE2-NEXT: pextrw $2, %xmm1, %eax
-; FAST_ISEL_SSE2-NEXT: shll $16, %eax
-; FAST_ISEL_SSE2-NEXT: movl %eax, {{[-0-9]+}}(%r{{[sb]}}p) # 4-byte Spill
+; FAST_ISEL_SSE2-NEXT: movd %eax, %xmm5
; FAST_ISEL_SSE2-NEXT: pextrw $1, %xmm1, %eax
-; FAST_ISEL_SSE2-NEXT: shll $16, %eax
+; FAST_ISEL_SSE2-NEXT: pslld $16, %xmm1
+; FAST_ISEL_SSE2-NEXT: movdqa %xmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
+; FAST_ISEL_SSE2-NEXT: movd %eax, %xmm1
+; FAST_ISEL_SSE2-NEXT: pextrw $7, %xmm0, %eax
+; FAST_ISEL_SSE2-NEXT: movd %eax, %xmm8
+; FAST_ISEL_SSE2-NEXT: pextrw $6, %xmm0, %eax
+; FAST_ISEL_SSE2-NEXT: movd %eax, %xmm7
+; FAST_ISEL_SSE2-NEXT: pextrw $5, %xmm0, %eax
+; FAST_ISEL_SSE2-NEXT: movd %eax, %xmm9
+; FAST_ISEL_SSE2-NEXT: pextrw $3, %xmm0, %eax
+; FAST_ISEL_SSE2-NEXT: movd %eax, %xmm11
+; FAST_ISEL_SSE2-NEXT: pextrw $2, %xmm0, %eax
+; FAST_ISEL_SSE2-NEXT: movd %eax, %xmm10
+; FAST_ISEL_SSE2-NEXT: pextrw $1, %xmm0, %eax
+; FAST_ISEL_SSE2-NEXT: pslld $16, %xmm0
+; FAST_ISEL_SSE2-NEXT: movdqa %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
+; FAST_ISEL_SSE2-NEXT: pslld $16, %xmm3
+; FAST_ISEL_SSE2-NEXT: movdqa %xmm3, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
+; FAST_ISEL_SSE2-NEXT: pslld $16, %xmm2
+; FAST_ISEL_SSE2-NEXT: movdqa %xmm2, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
+; FAST_ISEL_SSE2-NEXT: pslld $16, %xmm4
+; FAST_ISEL_SSE2-NEXT: movdqa %xmm4, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
+; FAST_ISEL_SSE2-NEXT: pslld $16, %xmm6
+; FAST_ISEL_SSE2-NEXT: movdqa %xmm6, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
+; FAST_ISEL_SSE2-NEXT: pslld $16, %xmm5
+; FAST_ISEL_SSE2-NEXT: movdqa %xmm5, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
+; FAST_ISEL_SSE2-NEXT: pslld $16, %xmm1
+; FAST_ISEL_SSE2-NEXT: movdqa %xmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
+; FAST_ISEL_SSE2-NEXT: pslld $16, %xmm8
+; FAST_ISEL_SSE2-NEXT: movdqa %xmm8, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
+; FAST_ISEL_SSE2-NEXT: pslld $16, %xmm7
+; FAST_ISEL_SSE2-NEXT: movdqa %xmm7, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
+; FAST_ISEL_SSE2-NEXT: pslld $16, %xmm9
+; FAST_ISEL_SSE2-NEXT: movdqa %xmm9, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
+; FAST_ISEL_SSE2-NEXT: pslld $16, %xmm11
+; FAST_ISEL_SSE2-NEXT: movdqa %xmm11, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
+; FAST_ISEL_SSE2-NEXT: pslld $16, %xmm10
+; FAST_ISEL_SSE2-NEXT: movdqa %xmm10, (%rsp) # 16-byte Spill
; FAST_ISEL_SSE2-NEXT: movd %eax, %xmm0
-; FAST_ISEL_SSE2-NEXT: movd %xmm1, %eax
-; FAST_ISEL_SSE2-NEXT: shll $16, %eax
-; FAST_ISEL_SSE2-NEXT: movl %eax, {{[-0-9]+}}(%r{{[sb]}}p) # 4-byte Spill
+; FAST_ISEL_SSE2-NEXT: pslld $16, %xmm0
; FAST_ISEL_SSE2-NEXT: callq __truncsfbf2 at PLT
; FAST_ISEL_SSE2-NEXT: pextrw $0, %xmm0, %ebx
; FAST_ISEL_SSE2-NEXT: shll $16, %ebx
-; FAST_ISEL_SSE2-NEXT: movd {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 4-byte Folded Reload
-; FAST_ISEL_SSE2-NEXT: # xmm0 = mem[0],zero,zero,zero
+; FAST_ISEL_SSE2-NEXT: movdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 16-byte Reload
; FAST_ISEL_SSE2-NEXT: callq __truncsfbf2 at PLT
; FAST_ISEL_SSE2-NEXT: pextrw $0, %xmm0, %eax
; FAST_ISEL_SSE2-NEXT: movzwl %ax, %r14d
; FAST_ISEL_SSE2-NEXT: orl %ebx, %r14d
-; FAST_ISEL_SSE2-NEXT: movd {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 4-byte Folded Reload
-; FAST_ISEL_SSE2-NEXT: # xmm0 = mem[0],zero,zero,zero
+; FAST_ISEL_SSE2-NEXT: movdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 16-byte Reload
; FAST_ISEL_SSE2-NEXT: callq __truncsfbf2 at PLT
; FAST_ISEL_SSE2-NEXT: pextrw $0, %xmm0, %ebx
; FAST_ISEL_SSE2-NEXT: shll $16, %ebx
-; FAST_ISEL_SSE2-NEXT: movd {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 4-byte Folded Reload
-; FAST_ISEL_SSE2-NEXT: # xmm0 = mem[0],zero,zero,zero
+; FAST_ISEL_SSE2-NEXT: movdqa (%rsp), %xmm0 # 16-byte Reload
; FAST_ISEL_SSE2-NEXT: callq __truncsfbf2 at PLT
; FAST_ISEL_SSE2-NEXT: pextrw $0, %xmm0, %eax
; FAST_ISEL_SSE2-NEXT: movzwl %ax, %eax
@@ -1044,25 +1011,21 @@ define <16 x bfloat> @call_ret_v16bf16(ptr %ptr) #0 {
; FAST_ISEL_SSE2-NEXT: shlq $32, %rax
; FAST_ISEL_SSE2-NEXT: orq %r14, %rax
; FAST_ISEL_SSE2-NEXT: movq %rax, %xmm0
-; FAST_ISEL_SSE2-NEXT: movdqa %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
-; FAST_ISEL_SSE2-NEXT: movd {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 4-byte Folded Reload
-; FAST_ISEL_SSE2-NEXT: # xmm0 = mem[0],zero,zero,zero
+; FAST_ISEL_SSE2-NEXT: movdqa %xmm0, (%rsp) # 16-byte Spill
+; FAST_ISEL_SSE2-NEXT: movdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 16-byte Reload
; FAST_ISEL_SSE2-NEXT: callq __truncsfbf2 at PLT
; FAST_ISEL_SSE2-NEXT: pextrw $0, %xmm0, %ebx
; FAST_ISEL_SSE2-NEXT: shll $16, %ebx
-; FAST_ISEL_SSE2-NEXT: movd {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 4-byte Folded Reload
-; FAST_ISEL_SSE2-NEXT: # xmm0 = mem[0],zero,zero,zero
+; FAST_ISEL_SSE2-NEXT: movdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 16-byte Reload
; FAST_ISEL_SSE2-NEXT: callq __truncsfbf2 at PLT
; FAST_ISEL_SSE2-NEXT: pextrw $0, %xmm0, %eax
; FAST_ISEL_SSE2-NEXT: movzwl %ax, %r14d
; FAST_ISEL_SSE2-NEXT: orl %ebx, %r14d
-; FAST_ISEL_SSE2-NEXT: movd {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 4-byte Folded Reload
-; FAST_ISEL_SSE2-NEXT: # xmm0 = mem[0],zero,zero,zero
+; FAST_ISEL_SSE2-NEXT: movdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 16-byte Reload
; FAST_ISEL_SSE2-NEXT: callq __truncsfbf2 at PLT
; FAST_ISEL_SSE2-NEXT: pextrw $0, %xmm0, %ebx
; FAST_ISEL_SSE2-NEXT: shll $16, %ebx
-; FAST_ISEL_SSE2-NEXT: movd {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 4-byte Folded Reload
-; FAST_ISEL_SSE2-NEXT: # xmm0 = mem[0],zero,zero,zero
+; FAST_ISEL_SSE2-NEXT: movdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 16-byte Reload
; FAST_ISEL_SSE2-NEXT: callq __truncsfbf2 at PLT
; FAST_ISEL_SSE2-NEXT: pextrw $0, %xmm0, %eax
; FAST_ISEL_SSE2-NEXT: movzwl %ax, %eax
@@ -1070,27 +1033,23 @@ define <16 x bfloat> @call_ret_v16bf16(ptr %ptr) #0 {
; FAST_ISEL_SSE2-NEXT: shlq $32, %rax
; FAST_ISEL_SSE2-NEXT: orq %r14, %rax
; FAST_ISEL_SSE2-NEXT: movq %rax, %xmm0
-; FAST_ISEL_SSE2-NEXT: movdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm1 # 16-byte Reload
+; FAST_ISEL_SSE2-NEXT: movdqa (%rsp), %xmm1 # 16-byte Reload
; FAST_ISEL_SSE2-NEXT: punpcklqdq {{.*#+}} xmm1 = xmm1[0],xmm0[0]
-; FAST_ISEL_SSE2-NEXT: movdqa %xmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
-; FAST_ISEL_SSE2-NEXT: movd {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 4-byte Folded Reload
-; FAST_ISEL_SSE2-NEXT: # xmm0 = mem[0],zero,zero,zero
+; FAST_ISEL_SSE2-NEXT: movdqa %xmm1, (%rsp) # 16-byte Spill
+; FAST_ISEL_SSE2-NEXT: movdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 16-byte Reload
; FAST_ISEL_SSE2-NEXT: callq __truncsfbf2 at PLT
; FAST_ISEL_SSE2-NEXT: pextrw $0, %xmm0, %ebx
; FAST_ISEL_SSE2-NEXT: shll $16, %ebx
-; FAST_ISEL_SSE2-NEXT: movd {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 4-byte Folded Reload
-; FAST_ISEL_SSE2-NEXT: # xmm0 = mem[0],zero,zero,zero
+; FAST_ISEL_SSE2-NEXT: movdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 16-byte Reload
; FAST_ISEL_SSE2-NEXT: callq __truncsfbf2 at PLT
; FAST_ISEL_SSE2-NEXT: pextrw $0, %xmm0, %eax
; FAST_ISEL_SSE2-NEXT: movzwl %ax, %r14d
; FAST_ISEL_SSE2-NEXT: orl %ebx, %r14d
-; FAST_ISEL_SSE2-NEXT: movd {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 4-byte Folded Reload
-; FAST_ISEL_SSE2-NEXT: # xmm0 = mem[0],zero,zero,zero
+; FAST_ISEL_SSE2-NEXT: movdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 16-byte Reload
; FAST_ISEL_SSE2-NEXT: callq __truncsfbf2 at PLT
; FAST_ISEL_SSE2-NEXT: pextrw $0, %xmm0, %ebx
; FAST_ISEL_SSE2-NEXT: shll $16, %ebx
-; FAST_ISEL_SSE2-NEXT: movd {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 4-byte Folded Reload
-; FAST_ISEL_SSE2-NEXT: # xmm0 = mem[0],zero,zero,zero
+; FAST_ISEL_SSE2-NEXT: movdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 16-byte Reload
; FAST_ISEL_SSE2-NEXT: callq __truncsfbf2 at PLT
; FAST_ISEL_SSE2-NEXT: pextrw $0, %xmm0, %eax
; FAST_ISEL_SSE2-NEXT: movzwl %ax, %eax
@@ -1099,24 +1058,20 @@ define <16 x bfloat> @call_ret_v16bf16(ptr %ptr) #0 {
; FAST_ISEL_SSE2-NEXT: orq %r14, %rax
; FAST_ISEL_SSE2-NEXT: movq %rax, %xmm0
; FAST_ISEL_SSE2-NEXT: movdqa %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
-; FAST_ISEL_SSE2-NEXT: movd {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 4-byte Folded Reload
-; FAST_ISEL_SSE2-NEXT: # xmm0 = mem[0],zero,zero,zero
+; FAST_ISEL_SSE2-NEXT: movdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 16-byte Reload
; FAST_ISEL_SSE2-NEXT: callq __truncsfbf2 at PLT
; FAST_ISEL_SSE2-NEXT: pextrw $0, %xmm0, %ebx
; FAST_ISEL_SSE2-NEXT: shll $16, %ebx
-; FAST_ISEL_SSE2-NEXT: movd {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 4-byte Folded Reload
-; FAST_ISEL_SSE2-NEXT: # xmm0 = mem[0],zero,zero,zero
+; FAST_ISEL_SSE2-NEXT: movdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 16-byte Reload
; FAST_ISEL_SSE2-NEXT: callq __truncsfbf2 at PLT
; FAST_ISEL_SSE2-NEXT: pextrw $0, %xmm0, %eax
; FAST_ISEL_SSE2-NEXT: movzwl %ax, %r14d
; FAST_ISEL_SSE2-NEXT: orl %ebx, %r14d
-; FAST_ISEL_SSE2-NEXT: movd {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 4-byte Folded Reload
-; FAST_ISEL_SSE2-NEXT: # xmm0 = mem[0],zero,zero,zero
+; FAST_ISEL_SSE2-NEXT: movdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 16-byte Reload
; FAST_ISEL_SSE2-NEXT: callq __truncsfbf2 at PLT
; FAST_ISEL_SSE2-NEXT: pextrw $0, %xmm0, %ebx
; FAST_ISEL_SSE2-NEXT: shll $16, %ebx
-; FAST_ISEL_SSE2-NEXT: movd {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 4-byte Folded Reload
-; FAST_ISEL_SSE2-NEXT: # xmm0 = mem[0],zero,zero,zero
+; FAST_ISEL_SSE2-NEXT: movdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 16-byte Reload
; FAST_ISEL_SSE2-NEXT: callq __truncsfbf2 at PLT
; FAST_ISEL_SSE2-NEXT: pextrw $0, %xmm0, %eax
; FAST_ISEL_SSE2-NEXT: movzwl %ax, %eax
@@ -1126,7 +1081,7 @@ define <16 x bfloat> @call_ret_v16bf16(ptr %ptr) #0 {
; FAST_ISEL_SSE2-NEXT: movq %rax, %xmm0
; FAST_ISEL_SSE2-NEXT: movdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm1 # 16-byte Reload
; FAST_ISEL_SSE2-NEXT: punpcklqdq {{.*#+}} xmm1 = xmm1[0],xmm0[0]
-; FAST_ISEL_SSE2-NEXT: movaps {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 16-byte Reload
+; FAST_ISEL_SSE2-NEXT: movaps (%rsp), %xmm0 # 16-byte Reload
; FAST_ISEL_SSE2-NEXT: callq returns_v16bf16 at PLT
;
; AVX512BF16-LABEL: call_ret_v16bf16:
diff --git a/llvm/test/CodeGen/X86/bfloat-int-cvt.ll b/llvm/test/CodeGen/X86/bfloat-int-cvt.ll
index 5dabb8cc633cc..208c6f6ad5c19 100644
--- a/llvm/test/CodeGen/X86/bfloat-int-cvt.ll
+++ b/llvm/test/CodeGen/X86/bfloat-int-cvt.ll
@@ -385,41 +385,21 @@ define <8 x bfloat> @uitofp_v8i64_to_v8bf16(<8 x i64> %a) {
;
define i32 @fptosi_bf16_to_i32(bfloat %a) {
-; BF16-LABEL: fptosi_bf16_to_i32:
-; BF16: # %bb.0:
-; BF16-NEXT: vpextrw $0, %xmm0, %eax
-; BF16-NEXT: shll $16, %eax
-; BF16-NEXT: vmovd %eax, %xmm0
-; BF16-NEXT: vcvttss2si %xmm0, %eax
-; BF16-NEXT: retq
-;
-; AVX10_2-LABEL: fptosi_bf16_to_i32:
-; AVX10_2: # %bb.0:
-; AVX10_2-NEXT: vmovw %xmm0, %eax
-; AVX10_2-NEXT: shll $16, %eax
-; AVX10_2-NEXT: vmovd %eax, %xmm0
-; AVX10_2-NEXT: vcvttss2si %xmm0, %eax
-; AVX10_2-NEXT: retq
+; CHECK-LABEL: fptosi_bf16_to_i32:
+; CHECK: # %bb.0:
+; CHECK-NEXT: vpslld $16, %xmm0, %xmm0
+; CHECK-NEXT: vcvttss2si %xmm0, %eax
+; CHECK-NEXT: retq
%cvt = fptosi bfloat %a to i32
ret i32 %cvt
}
define i64 @fptosi_bf16_to_i64(bfloat %a) {
-; BF16-LABEL: fptosi_bf16_to_i64:
-; BF16: # %bb.0:
-; BF16-NEXT: vpextrw $0, %xmm0, %eax
-; BF16-NEXT: shll $16, %eax
-; BF16-NEXT: vmovd %eax, %xmm0
-; BF16-NEXT: vcvttss2si %xmm0, %rax
-; BF16-NEXT: retq
-;
-; AVX10_2-LABEL: fptosi_bf16_to_i64:
-; AVX10_2: # %bb.0:
-; AVX10_2-NEXT: vmovw %xmm0, %eax
-; AVX10_2-NEXT: shll $16, %eax
-; AVX10_2-NEXT: vmovd %eax, %xmm0
-; AVX10_2-NEXT: vcvttss2si %xmm0, %rax
-; AVX10_2-NEXT: retq
+; CHECK-LABEL: fptosi_bf16_to_i64:
+; CHECK: # %bb.0:
+; CHECK-NEXT: vpslld $16, %xmm0, %xmm0
+; CHECK-NEXT: vcvttss2si %xmm0, %rax
+; CHECK-NEXT: retq
%cvt = fptosi bfloat %a to i64
ret i64 %cvt
}
@@ -447,29 +427,12 @@ define <4 x i32> @fptosi_v4bf16_to_v4i32(<4 x bfloat> %a) {
}
define <2 x i32> @fptosi_v2bf16_to_v2i32(<2 x bfloat> %a) {
-; BF16-LABEL: fptosi_v2bf16_to_v2i32:
-; BF16: # %bb.0:
-; BF16-NEXT: vpextrw $1, %xmm0, %eax
-; BF16-NEXT: shll $16, %eax
-; BF16-NEXT: vmovd %eax, %xmm1
-; BF16-NEXT: vmovd %xmm0, %eax
-; BF16-NEXT: shll $16, %eax
-; BF16-NEXT: vmovd %eax, %xmm0
-; BF16-NEXT: vinsertps {{.*#+}} xmm0 = xmm0[0],xmm1[0],xmm0[2,3]
-; BF16-NEXT: vcvttps2dq %xmm0, %xmm0
-; BF16-NEXT: retq
-;
-; AVX10_2-LABEL: fptosi_v2bf16_to_v2i32:
-; AVX10_2: # %bb.0:
-; AVX10_2-NEXT: vmovw %xmm0, %eax
-; AVX10_2-NEXT: shll $16, %eax
-; AVX10_2-NEXT: vmovd %eax, %xmm1
-; AVX10_2-NEXT: vpextrw $1, %xmm0, %eax
-; AVX10_2-NEXT: shll $16, %eax
-; AVX10_2-NEXT: vmovd %eax, %xmm0
-; AVX10_2-NEXT: vinsertps {{.*#+}} xmm0 = xmm1[0],xmm0[0],xmm1[2,3]
-; AVX10_2-NEXT: vcvttps2dq %xmm0, %xmm0
-; AVX10_2-NEXT: retq
+; CHECK-LABEL: fptosi_v2bf16_to_v2i32:
+; CHECK: # %bb.0:
+; CHECK-NEXT: vpxor %xmm1, %xmm1, %xmm1
+; CHECK-NEXT: vpunpcklwd {{.*#+}} xmm0 = xmm1[0],xmm0[0],xmm1[1],xmm0[1],xmm1[2],xmm0[2],xmm1[3],xmm0[3]
+; CHECK-NEXT: vcvttps2dq %xmm0, %xmm0
+; CHECK-NEXT: retq
%cvt = fptosi <2 x bfloat> %a to <2 x i32>
ret <2 x i32> %cvt
}
@@ -488,14 +451,11 @@ define <16 x i32> @fptosi_v16bf16_to_v16i32(<16 x bfloat> %a) {
define <2 x i64> @fptosi_v2bf16_to_v2i64(<2 x bfloat> %a) {
; BF16-LABEL: fptosi_v2bf16_to_v2i64:
; BF16: # %bb.0:
-; BF16-NEXT: vpextrw $1, %xmm0, %eax
-; BF16-NEXT: shll $16, %eax
-; BF16-NEXT: vmovd %eax, %xmm1
+; BF16-NEXT: vpxor %xmm1, %xmm1, %xmm1
+; BF16-NEXT: vpblendw {{.*#+}} xmm1 = xmm1[0],xmm0[1],xmm1[2],xmm0[3],xmm1[4],xmm0[5],xmm1[6],xmm0[7]
; BF16-NEXT: vcvttss2si %xmm1, %rax
; BF16-NEXT: vmovq %rax, %xmm1
-; BF16-NEXT: vmovd %xmm0, %eax
-; BF16-NEXT: shll $16, %eax
-; BF16-NEXT: vmovd %eax, %xmm0
+; BF16-NEXT: vpslld $16, %xmm0, %xmm0
; BF16-NEXT: vcvttss2si %xmm0, %rax
; BF16-NEXT: vmovq %rax, %xmm0
; BF16-NEXT: vpunpcklqdq {{.*#+}} xmm0 = xmm0[0],xmm1[0]
@@ -516,29 +476,26 @@ define <2 x i64> @fptosi_v2bf16_to_v2i64(<2 x bfloat> %a) {
define <4 x i64> @fptosi_v4bf16_to_v4i64(<4 x bfloat> %a) {
; BF16-LABEL: fptosi_v4bf16_to_v4i64:
; BF16: # %bb.0:
-; BF16-NEXT: vpextrw $3, %xmm0, %eax
-; BF16-NEXT: shll $16, %eax
-; BF16-NEXT: vmovd %eax, %xmm1
+; BF16-NEXT: vpxor %xmm1, %xmm1, %xmm1
+; BF16-NEXT: vpblendw {{.*#+}} xmm1 = xmm1[0],xmm0[1],xmm1[2],xmm0[3],xmm1[4],xmm0[5],xmm1[6],xmm0[7]
; BF16-NEXT: vcvttss2si %xmm1, %rax
; BF16-NEXT: vmovq %rax, %xmm1
-; BF16-NEXT: vpextrw $2, %xmm0, %eax
-; BF16-NEXT: shll $16, %eax
-; BF16-NEXT: vmovd %eax, %xmm2
+; BF16-NEXT: vpslld $16, %xmm0, %xmm2
; BF16-NEXT: vcvttss2si %xmm2, %rax
; BF16-NEXT: vmovq %rax, %xmm2
; BF16-NEXT: vpunpcklqdq {{.*#+}} xmm1 = xmm2[0],xmm1[0]
-; BF16-NEXT: vpextrw $1, %xmm0, %eax
-; BF16-NEXT: shll $16, %eax
+; BF16-NEXT: vpextrw $3, %xmm0, %eax
; BF16-NEXT: vmovd %eax, %xmm2
+; BF16-NEXT: vpslld $16, %xmm2, %xmm2
; BF16-NEXT: vcvttss2si %xmm2, %rax
; BF16-NEXT: vmovq %rax, %xmm2
-; BF16-NEXT: vmovd %xmm0, %eax
-; BF16-NEXT: shll $16, %eax
+; BF16-NEXT: vpextrw $2, %xmm0, %eax
; BF16-NEXT: vmovd %eax, %xmm0
+; BF16-NEXT: vpslld $16, %xmm0, %xmm0
; BF16-NEXT: vcvttss2si %xmm0, %rax
; BF16-NEXT: vmovq %rax, %xmm0
; BF16-NEXT: vpunpcklqdq {{.*#+}} xmm0 = xmm0[0],xmm2[0]
-; BF16-NEXT: vinserti128 $1, %xmm1, %ymm0, %ymm0
+; BF16-NEXT: vinserti128 $1, %xmm0, %ymm1, %ymm0
; BF16-NEXT: retq
;
; AVX10_2-LABEL: fptosi_v4bf16_to_v4i64:
@@ -556,51 +513,46 @@ define <8 x i64> @fptosi_v8bf16_to_v8i64(<8 x bfloat> %a) {
; BF16-LABEL: fptosi_v8bf16_to_v8i64:
; BF16: # %bb.0:
; BF16-NEXT: vpextrw $7, %xmm0, %eax
-; BF16-NEXT: shll $16, %eax
; BF16-NEXT: vmovd %eax, %xmm1
+; BF16-NEXT: vpslld $16, %xmm1, %xmm1
; BF16-NEXT: vcvttss2si %xmm1, %rax
; BF16-NEXT: vmovq %rax, %xmm1
; BF16-NEXT: vpextrw $6, %xmm0, %eax
-; BF16-NEXT: shll $16, %eax
; BF16-NEXT: vmovd %eax, %xmm2
+; BF16-NEXT: vpslld $16, %xmm2, %xmm2
; BF16-NEXT: vcvttss2si %xmm2, %rax
; BF16-NEXT: vmovq %rax, %xmm2
; BF16-NEXT: vpunpcklqdq {{.*#+}} xmm1 = xmm2[0],xmm1[0]
-; BF16-NEXT: vpextrw $5, %xmm0, %eax
-; BF16-NEXT: shll $16, %eax
-; BF16-NEXT: vmovd %eax, %xmm2
-; BF16-NEXT: vcvttss2si %xmm2, %rax
-; BF16-NEXT: vmovq %rax, %xmm2
-; BF16-NEXT: vpextrw $4, %xmm0, %eax
-; BF16-NEXT: shll $16, %eax
-; BF16-NEXT: vmovd %eax, %xmm3
+; BF16-NEXT: vpxor %xmm2, %xmm2, %xmm2
+; BF16-NEXT: vpunpckhwd {{.*#+}} xmm3 = xmm2[4],xmm0[4],xmm2[5],xmm0[5],xmm2[6],xmm0[6],xmm2[7],xmm0[7]
; BF16-NEXT: vcvttss2si %xmm3, %rax
; BF16-NEXT: vmovq %rax, %xmm3
-; BF16-NEXT: vpunpcklqdq {{.*#+}} xmm2 = xmm3[0],xmm2[0]
-; BF16-NEXT: vinserti128 $1, %xmm1, %ymm2, %ymm1
-; BF16-NEXT: vpextrw $3, %xmm0, %eax
-; BF16-NEXT: shll $16, %eax
-; BF16-NEXT: vmovd %eax, %xmm2
+; BF16-NEXT: vpextrw $5, %xmm0, %eax
+; BF16-NEXT: vmovd %eax, %xmm4
+; BF16-NEXT: vpslld $16, %xmm4, %xmm4
+; BF16-NEXT: vcvttss2si %xmm4, %rax
+; BF16-NEXT: vmovq %rax, %xmm4
+; BF16-NEXT: vpunpcklqdq {{.*#+}} xmm3 = xmm3[0],xmm4[0]
+; BF16-NEXT: vinserti128 $1, %xmm1, %ymm3, %ymm1
+; BF16-NEXT: vpblendw {{.*#+}} xmm2 = xmm2[0],xmm0[1],xmm2[2],xmm0[3],xmm2[4],xmm0[5],xmm2[6],xmm0[7]
; BF16-NEXT: vcvttss2si %xmm2, %rax
; BF16-NEXT: vmovq %rax, %xmm2
-; BF16-NEXT: vpextrw $2, %xmm0, %eax
-; BF16-NEXT: shll $16, %eax
-; BF16-NEXT: vmovd %eax, %xmm3
+; BF16-NEXT: vpslld $16, %xmm0, %xmm3
; BF16-NEXT: vcvttss2si %xmm3, %rax
; BF16-NEXT: vmovq %rax, %xmm3
; BF16-NEXT: vpunpcklqdq {{.*#+}} xmm2 = xmm3[0],xmm2[0]
-; BF16-NEXT: vpextrw $1, %xmm0, %eax
-; BF16-NEXT: shll $16, %eax
+; BF16-NEXT: vpextrw $3, %xmm0, %eax
; BF16-NEXT: vmovd %eax, %xmm3
+; BF16-NEXT: vpslld $16, %xmm3, %xmm3
; BF16-NEXT: vcvttss2si %xmm3, %rax
; BF16-NEXT: vmovq %rax, %xmm3
-; BF16-NEXT: vmovd %xmm0, %eax
-; BF16-NEXT: shll $16, %eax
+; BF16-NEXT: vpextrw $2, %xmm0, %eax
; BF16-NEXT: vmovd %eax, %xmm0
+; BF16-NEXT: vpslld $16, %xmm0, %xmm0
; BF16-NEXT: vcvttss2si %xmm0, %rax
; BF16-NEXT: vmovq %rax, %xmm0
; BF16-NEXT: vpunpcklqdq {{.*#+}} xmm0 = xmm0[0],xmm3[0]
-; BF16-NEXT: vinserti128 $1, %xmm2, %ymm0, %ymm0
+; BF16-NEXT: vinserti128 $1, %xmm0, %ymm2, %ymm0
; BF16-NEXT: vinserti64x4 $1, %ymm1, %zmm0, %zmm0
; BF16-NEXT: retq
;
@@ -619,41 +571,21 @@ define <8 x i64> @fptosi_v8bf16_to_v8i64(<8 x bfloat> %a) {
;
define i32 @fptoui_bf16_to_i32(bfloat %a) {
-; BF16-LABEL: fptoui_bf16_to_i32:
-; BF16: # %bb.0:
-; BF16-NEXT: vpextrw $0, %xmm0, %eax
-; BF16-NEXT: shll $16, %eax
-; BF16-NEXT: vmovd %eax, %xmm0
-; BF16-NEXT: vcvttss2usi %xmm0, %eax
-; BF16-NEXT: retq
-;
-; AVX10_2-LABEL: fptoui_bf16_to_i32:
-; AVX10_2: # %bb.0:
-; AVX10_2-NEXT: vmovw %xmm0, %eax
-; AVX10_2-NEXT: shll $16, %eax
-; AVX10_2-NEXT: vmovd %eax, %xmm0
-; AVX10_2-NEXT: vcvttss2usi %xmm0, %eax
-; AVX10_2-NEXT: retq
+; CHECK-LABEL: fptoui_bf16_to_i32:
+; CHECK: # %bb.0:
+; CHECK-NEXT: vpslld $16, %xmm0, %xmm0
+; CHECK-NEXT: vcvttss2usi %xmm0, %eax
+; CHECK-NEXT: retq
%cvt = fptoui bfloat %a to i32
ret i32 %cvt
}
define i64 @fptoui_bf16_to_i64(bfloat %a) {
-; BF16-LABEL: fptoui_bf16_to_i64:
-; BF16: # %bb.0:
-; BF16-NEXT: vpextrw $0, %xmm0, %eax
-; BF16-NEXT: shll $16, %eax
-; BF16-NEXT: vmovd %eax, %xmm0
-; BF16-NEXT: vcvttss2usi %xmm0, %rax
-; BF16-NEXT: retq
-;
-; AVX10_2-LABEL: fptoui_bf16_to_i64:
-; AVX10_2: # %bb.0:
-; AVX10_2-NEXT: vmovw %xmm0, %eax
-; AVX10_2-NEXT: shll $16, %eax
-; AVX10_2-NEXT: vmovd %eax, %xmm0
-; AVX10_2-NEXT: vcvttss2usi %xmm0, %rax
-; AVX10_2-NEXT: retq
+; CHECK-LABEL: fptoui_bf16_to_i64:
+; CHECK: # %bb.0:
+; CHECK-NEXT: vpslld $16, %xmm0, %xmm0
+; CHECK-NEXT: vcvttss2usi %xmm0, %rax
+; CHECK-NEXT: retq
%cvt = fptoui bfloat %a to i64
ret i64 %cvt
}
@@ -681,29 +613,12 @@ define <4 x i32> @fptoui_v4bf16_to_v4i32(<4 x bfloat> %a) {
}
define <2 x i32> @fptoui_v2bf16_to_v2i32(<2 x bfloat> %a) {
-; BF16-LABEL: fptoui_v2bf16_to_v2i32:
-; BF16: # %bb.0:
-; BF16-NEXT: vpextrw $1, %xmm0, %eax
-; BF16-NEXT: shll $16, %eax
-; BF16-NEXT: vmovd %eax, %xmm1
-; BF16-NEXT: vmovd %xmm0, %eax
-; BF16-NEXT: shll $16, %eax
-; BF16-NEXT: vmovd %eax, %xmm0
-; BF16-NEXT: vinsertps {{.*#+}} xmm0 = xmm0[0],xmm1[0],xmm0[2,3]
-; BF16-NEXT: vcvttps2udq %xmm0, %xmm0
-; BF16-NEXT: retq
-;
-; AVX10_2-LABEL: fptoui_v2bf16_to_v2i32:
-; AVX10_2: # %bb.0:
-; AVX10_2-NEXT: vmovw %xmm0, %eax
-; AVX10_2-NEXT: shll $16, %eax
-; AVX10_2-NEXT: vmovd %eax, %xmm1
-; AVX10_2-NEXT: vpextrw $1, %xmm0, %eax
-; AVX10_2-NEXT: shll $16, %eax
-; AVX10_2-NEXT: vmovd %eax, %xmm0
-; AVX10_2-NEXT: vinsertps {{.*#+}} xmm0 = xmm1[0],xmm0[0],xmm1[2,3]
-; AVX10_2-NEXT: vcvttps2udq %xmm0, %xmm0
-; AVX10_2-NEXT: retq
+; CHECK-LABEL: fptoui_v2bf16_to_v2i32:
+; CHECK: # %bb.0:
+; CHECK-NEXT: vpxor %xmm1, %xmm1, %xmm1
+; CHECK-NEXT: vpunpcklwd {{.*#+}} xmm0 = xmm1[0],xmm0[0],xmm1[1],xmm0[1],xmm1[2],xmm0[2],xmm1[3],xmm0[3]
+; CHECK-NEXT: vcvttps2udq %xmm0, %xmm0
+; CHECK-NEXT: retq
%cvt = fptoui <2 x bfloat> %a to <2 x i32>
ret <2 x i32> %cvt
}
@@ -722,14 +637,11 @@ define <16 x i32> @fptoui_v16bf16_to_v16i32(<16 x bfloat> %a) {
define <2 x i64> @fptoui_v2bf16_to_v2i64(<2 x bfloat> %a) {
; BF16-LABEL: fptoui_v2bf16_to_v2i64:
; BF16: # %bb.0:
-; BF16-NEXT: vpextrw $1, %xmm0, %eax
-; BF16-NEXT: shll $16, %eax
-; BF16-NEXT: vmovd %eax, %xmm1
+; BF16-NEXT: vpxor %xmm1, %xmm1, %xmm1
+; BF16-NEXT: vpblendw {{.*#+}} xmm1 = xmm1[0],xmm0[1],xmm1[2],xmm0[3],xmm1[4],xmm0[5],xmm1[6],xmm0[7]
; BF16-NEXT: vcvttss2usi %xmm1, %rax
; BF16-NEXT: vmovq %rax, %xmm1
-; BF16-NEXT: vmovd %xmm0, %eax
-; BF16-NEXT: shll $16, %eax
-; BF16-NEXT: vmovd %eax, %xmm0
+; BF16-NEXT: vpslld $16, %xmm0, %xmm0
; BF16-NEXT: vcvttss2usi %xmm0, %rax
; BF16-NEXT: vmovq %rax, %xmm0
; BF16-NEXT: vpunpcklqdq {{.*#+}} xmm0 = xmm0[0],xmm1[0]
@@ -750,29 +662,26 @@ define <2 x i64> @fptoui_v2bf16_to_v2i64(<2 x bfloat> %a) {
define <4 x i64> @fptoui_v4bf16_to_v4i64(<4 x bfloat> %a) {
; BF16-LABEL: fptoui_v4bf16_to_v4i64:
; BF16: # %bb.0:
-; BF16-NEXT: vpextrw $3, %xmm0, %eax
-; BF16-NEXT: shll $16, %eax
-; BF16-NEXT: vmovd %eax, %xmm1
+; BF16-NEXT: vpxor %xmm1, %xmm1, %xmm1
+; BF16-NEXT: vpblendw {{.*#+}} xmm1 = xmm1[0],xmm0[1],xmm1[2],xmm0[3],xmm1[4],xmm0[5],xmm1[6],xmm0[7]
; BF16-NEXT: vcvttss2usi %xmm1, %rax
; BF16-NEXT: vmovq %rax, %xmm1
-; BF16-NEXT: vpextrw $2, %xmm0, %eax
-; BF16-NEXT: shll $16, %eax
-; BF16-NEXT: vmovd %eax, %xmm2
+; BF16-NEXT: vpslld $16, %xmm0, %xmm2
; BF16-NEXT: vcvttss2usi %xmm2, %rax
; BF16-NEXT: vmovq %rax, %xmm2
; BF16-NEXT: vpunpcklqdq {{.*#+}} xmm1 = xmm2[0],xmm1[0]
-; BF16-NEXT: vpextrw $1, %xmm0, %eax
-; BF16-NEXT: shll $16, %eax
+; BF16-NEXT: vpextrw $3, %xmm0, %eax
; BF16-NEXT: vmovd %eax, %xmm2
+; BF16-NEXT: vpslld $16, %xmm2, %xmm2
; BF16-NEXT: vcvttss2usi %xmm2, %rax
; BF16-NEXT: vmovq %rax, %xmm2
-; BF16-NEXT: vmovd %xmm0, %eax
-; BF16-NEXT: shll $16, %eax
+; BF16-NEXT: vpextrw $2, %xmm0, %eax
; BF16-NEXT: vmovd %eax, %xmm0
+; BF16-NEXT: vpslld $16, %xmm0, %xmm0
; BF16-NEXT: vcvttss2usi %xmm0, %rax
; BF16-NEXT: vmovq %rax, %xmm0
; BF16-NEXT: vpunpcklqdq {{.*#+}} xmm0 = xmm0[0],xmm2[0]
-; BF16-NEXT: vinserti128 $1, %xmm1, %ymm0, %ymm0
+; BF16-NEXT: vinserti128 $1, %xmm0, %ymm1, %ymm0
; BF16-NEXT: retq
;
; AVX10_2-LABEL: fptoui_v4bf16_to_v4i64:
@@ -790,51 +699,46 @@ define <8 x i64> @fptoui_v8bf16_to_v8i64(<8 x bfloat> %a) {
; BF16-LABEL: fptoui_v8bf16_to_v8i64:
; BF16: # %bb.0:
; BF16-NEXT: vpextrw $7, %xmm0, %eax
-; BF16-NEXT: shll $16, %eax
; BF16-NEXT: vmovd %eax, %xmm1
+; BF16-NEXT: vpslld $16, %xmm1, %xmm1
; BF16-NEXT: vcvttss2usi %xmm1, %rax
; BF16-NEXT: vmovq %rax, %xmm1
; BF16-NEXT: vpextrw $6, %xmm0, %eax
-; BF16-NEXT: shll $16, %eax
; BF16-NEXT: vmovd %eax, %xmm2
+; BF16-NEXT: vpslld $16, %xmm2, %xmm2
; BF16-NEXT: vcvttss2usi %xmm2, %rax
; BF16-NEXT: vmovq %rax, %xmm2
; BF16-NEXT: vpunpcklqdq {{.*#+}} xmm1 = xmm2[0],xmm1[0]
-; BF16-NEXT: vpextrw $5, %xmm0, %eax
-; BF16-NEXT: shll $16, %eax
-; BF16-NEXT: vmovd %eax, %xmm2
-; BF16-NEXT: vcvttss2usi %xmm2, %rax
-; BF16-NEXT: vmovq %rax, %xmm2
-; BF16-NEXT: vpextrw $4, %xmm0, %eax
-; BF16-NEXT: shll $16, %eax
-; BF16-NEXT: vmovd %eax, %xmm3
+; BF16-NEXT: vpxor %xmm2, %xmm2, %xmm2
+; BF16-NEXT: vpunpckhwd {{.*#+}} xmm3 = xmm2[4],xmm0[4],xmm2[5],xmm0[5],xmm2[6],xmm0[6],xmm2[7],xmm0[7]
; BF16-NEXT: vcvttss2usi %xmm3, %rax
; BF16-NEXT: vmovq %rax, %xmm3
-; BF16-NEXT: vpunpcklqdq {{.*#+}} xmm2 = xmm3[0],xmm2[0]
-; BF16-NEXT: vinserti128 $1, %xmm1, %ymm2, %ymm1
-; BF16-NEXT: vpextrw $3, %xmm0, %eax
-; BF16-NEXT: shll $16, %eax
-; BF16-NEXT: vmovd %eax, %xmm2
+; BF16-NEXT: vpextrw $5, %xmm0, %eax
+; BF16-NEXT: vmovd %eax, %xmm4
+; BF16-NEXT: vpslld $16, %xmm4, %xmm4
+; BF16-NEXT: vcvttss2usi %xmm4, %rax
+; BF16-NEXT: vmovq %rax, %xmm4
+; BF16-NEXT: vpunpcklqdq {{.*#+}} xmm3 = xmm3[0],xmm4[0]
+; BF16-NEXT: vinserti128 $1, %xmm1, %ymm3, %ymm1
+; BF16-NEXT: vpblendw {{.*#+}} xmm2 = xmm2[0],xmm0[1],xmm2[2],xmm0[3],xmm2[4],xmm0[5],xmm2[6],xmm0[7]
; BF16-NEXT: vcvttss2usi %xmm2, %rax
; BF16-NEXT: vmovq %rax, %xmm2
-; BF16-NEXT: vpextrw $2, %xmm0, %eax
-; BF16-NEXT: shll $16, %eax
-; BF16-NEXT: vmovd %eax, %xmm3
+; BF16-NEXT: vpslld $16, %xmm0, %xmm3
; BF16-NEXT: vcvttss2usi %xmm3, %rax
; BF16-NEXT: vmovq %rax, %xmm3
; BF16-NEXT: vpunpcklqdq {{.*#+}} xmm2 = xmm3[0],xmm2[0]
-; BF16-NEXT: vpextrw $1, %xmm0, %eax
-; BF16-NEXT: shll $16, %eax
+; BF16-NEXT: vpextrw $3, %xmm0, %eax
; BF16-NEXT: vmovd %eax, %xmm3
+; BF16-NEXT: vpslld $16, %xmm3, %xmm3
; BF16-NEXT: vcvttss2usi %xmm3, %rax
; BF16-NEXT: vmovq %rax, %xmm3
-; BF16-NEXT: vmovd %xmm0, %eax
-; BF16-NEXT: shll $16, %eax
+; BF16-NEXT: vpextrw $2, %xmm0, %eax
; BF16-NEXT: vmovd %eax, %xmm0
+; BF16-NEXT: vpslld $16, %xmm0, %xmm0
; BF16-NEXT: vcvttss2usi %xmm0, %rax
; BF16-NEXT: vmovq %rax, %xmm0
; BF16-NEXT: vpunpcklqdq {{.*#+}} xmm0 = xmm0[0],xmm3[0]
-; BF16-NEXT: vinserti128 $1, %xmm2, %ymm0, %ymm0
+; BF16-NEXT: vinserti128 $1, %xmm0, %ymm2, %ymm0
; BF16-NEXT: vinserti64x4 $1, %ymm1, %zmm0, %zmm0
; BF16-NEXT: retq
;
diff --git a/llvm/test/CodeGen/X86/bfloat.ll b/llvm/test/CodeGen/X86/bfloat.ll
index 7bccd6ba088ac..be217d52fbb9f 100644
--- a/llvm/test/CodeGen/X86/bfloat.ll
+++ b/llvm/test/CodeGen/X86/bfloat.ll
@@ -11,12 +11,10 @@ define void @add(ptr %pa, ptr %pb, ptr %pc) nounwind {
; X86-NEXT: movl {{[0-9]+}}(%esp), %eax
; X86-NEXT: movl {{[0-9]+}}(%esp), %ecx
; X86-NEXT: movl {{[0-9]+}}(%esp), %edx
-; X86-NEXT: movzwl (%edx), %edx
-; X86-NEXT: shll $16, %edx
-; X86-NEXT: vmovd %edx, %xmm0
-; X86-NEXT: movzwl (%ecx), %ecx
-; X86-NEXT: shll $16, %ecx
-; X86-NEXT: vmovd %ecx, %xmm1
+; X86-NEXT: vmovw (%edx), %xmm0
+; X86-NEXT: vpslld $16, %xmm0, %xmm0
+; X86-NEXT: vmovw (%ecx), %xmm1
+; X86-NEXT: vpslld $16, %xmm1, %xmm1
; X86-NEXT: vaddss %xmm0, %xmm1, %xmm0
; X86-NEXT: vcvtneps2bf16 %xmm0, %xmm0
; X86-NEXT: vpextrw $0, %xmm0, (%eax)
@@ -39,18 +37,29 @@ define void @add(ptr %pa, ptr %pb, ptr %pc) nounwind {
; SSE2-NEXT: popq %rbx
; SSE2-NEXT: retq
;
-; AVX512-LABEL: add:
-; AVX512: # %bb.0:
-; AVX512-NEXT: movzwl (%rsi), %eax
-; AVX512-NEXT: shll $16, %eax
-; AVX512-NEXT: vmovd %eax, %xmm0
-; AVX512-NEXT: movzwl (%rdi), %eax
-; AVX512-NEXT: shll $16, %eax
-; AVX512-NEXT: vmovd %eax, %xmm1
-; AVX512-NEXT: vaddss %xmm0, %xmm1, %xmm0
-; AVX512-NEXT: vcvtneps2bf16 %xmm0, %xmm0
-; AVX512-NEXT: vpextrw $0, %xmm0, (%rdx)
-; AVX512-NEXT: retq
+; AVX512BF16-LABEL: add:
+; AVX512BF16: # %bb.0:
+; AVX512BF16-NEXT: movzwl (%rsi), %eax
+; AVX512BF16-NEXT: shll $16, %eax
+; AVX512BF16-NEXT: vmovd %eax, %xmm0
+; AVX512BF16-NEXT: movzwl (%rdi), %eax
+; AVX512BF16-NEXT: shll $16, %eax
+; AVX512BF16-NEXT: vmovd %eax, %xmm1
+; AVX512BF16-NEXT: vaddss %xmm0, %xmm1, %xmm0
+; AVX512BF16-NEXT: vcvtneps2bf16 %xmm0, %xmm0
+; AVX512BF16-NEXT: vpextrw $0, %xmm0, (%rdx)
+; AVX512BF16-NEXT: retq
+;
+; AVX512FP16-LABEL: add:
+; AVX512FP16: # %bb.0:
+; AVX512FP16-NEXT: vmovw (%rsi), %xmm0
+; AVX512FP16-NEXT: vpslld $16, %xmm0, %xmm0
+; AVX512FP16-NEXT: vmovw (%rdi), %xmm1
+; AVX512FP16-NEXT: vpslld $16, %xmm1, %xmm1
+; AVX512FP16-NEXT: vaddss %xmm0, %xmm1, %xmm0
+; AVX512FP16-NEXT: vcvtneps2bf16 %xmm0, %xmm0
+; AVX512FP16-NEXT: vpextrw $0, %xmm0, (%rdx)
+; AVX512FP16-NEXT: retq
;
; AVXNC-LABEL: add:
; AVXNC: # %bb.0:
@@ -74,12 +83,10 @@ define void @add(ptr %pa, ptr %pb, ptr %pc) nounwind {
define bfloat @add2(bfloat %a, bfloat %b) nounwind {
; X86-LABEL: add2:
; X86: # %bb.0:
-; X86-NEXT: movl {{[0-9]+}}(%esp), %eax
-; X86-NEXT: shll $16, %eax
-; X86-NEXT: vmovd %eax, %xmm0
-; X86-NEXT: movl {{[0-9]+}}(%esp), %eax
-; X86-NEXT: shll $16, %eax
-; X86-NEXT: vmovd %eax, %xmm1
+; X86-NEXT: vmovw {{[0-9]+}}(%esp), %xmm0
+; X86-NEXT: vpslld $16, %xmm0, %xmm0
+; X86-NEXT: vmovw {{[0-9]+}}(%esp), %xmm1
+; X86-NEXT: vpslld $16, %xmm1, %xmm1
; X86-NEXT: vaddss %xmm0, %xmm1, %xmm0
; X86-NEXT: vcvtneps2bf16 %xmm0, %xmm0
; X86-NEXT: retl
@@ -87,50 +94,26 @@ define bfloat @add2(bfloat %a, bfloat %b) nounwind {
; SSE2-LABEL: add2:
; SSE2: # %bb.0:
; SSE2-NEXT: pushq %rax
-; SSE2-NEXT: pextrw $0, %xmm0, %eax
-; SSE2-NEXT: pextrw $0, %xmm1, %ecx
-; SSE2-NEXT: shll $16, %ecx
-; SSE2-NEXT: movd %ecx, %xmm1
-; SSE2-NEXT: shll $16, %eax
-; SSE2-NEXT: movd %eax, %xmm0
+; SSE2-NEXT: pslld $16, %xmm1
+; SSE2-NEXT: pslld $16, %xmm0
; SSE2-NEXT: addss %xmm1, %xmm0
; SSE2-NEXT: callq __truncsfbf2 at PLT
; SSE2-NEXT: popq %rax
; SSE2-NEXT: retq
;
-; AVX512BF16-LABEL: add2:
-; AVX512BF16: # %bb.0:
-; AVX512BF16-NEXT: vpextrw $0, %xmm0, %eax
-; AVX512BF16-NEXT: vpextrw $0, %xmm1, %ecx
-; AVX512BF16-NEXT: shll $16, %ecx
-; AVX512BF16-NEXT: vmovd %ecx, %xmm0
-; AVX512BF16-NEXT: shll $16, %eax
-; AVX512BF16-NEXT: vmovd %eax, %xmm1
-; AVX512BF16-NEXT: vaddss %xmm0, %xmm1, %xmm0
-; AVX512BF16-NEXT: vcvtneps2bf16 %xmm0, %xmm0
-; AVX512BF16-NEXT: retq
-;
-; AVX512FP16-LABEL: add2:
-; AVX512FP16: # %bb.0:
-; AVX512FP16-NEXT: vmovw %xmm0, %eax
-; AVX512FP16-NEXT: vmovw %xmm1, %ecx
-; AVX512FP16-NEXT: shll $16, %ecx
-; AVX512FP16-NEXT: vmovd %ecx, %xmm0
-; AVX512FP16-NEXT: shll $16, %eax
-; AVX512FP16-NEXT: vmovd %eax, %xmm1
-; AVX512FP16-NEXT: vaddss %xmm0, %xmm1, %xmm0
-; AVX512FP16-NEXT: vcvtneps2bf16 %xmm0, %xmm0
-; AVX512FP16-NEXT: retq
+; AVX512-LABEL: add2:
+; AVX512: # %bb.0:
+; AVX512-NEXT: vpslld $16, %xmm1, %xmm1
+; AVX512-NEXT: vpslld $16, %xmm0, %xmm0
+; AVX512-NEXT: vaddss %xmm1, %xmm0, %xmm0
+; AVX512-NEXT: vcvtneps2bf16 %xmm0, %xmm0
+; AVX512-NEXT: retq
;
; AVXNC-LABEL: add2:
; AVXNC: # %bb.0:
-; AVXNC-NEXT: vpextrw $0, %xmm0, %eax
-; AVXNC-NEXT: vpextrw $0, %xmm1, %ecx
-; AVXNC-NEXT: shll $16, %ecx
-; AVXNC-NEXT: vmovd %ecx, %xmm0
-; AVXNC-NEXT: shll $16, %eax
-; AVXNC-NEXT: vmovd %eax, %xmm1
-; AVXNC-NEXT: vaddss %xmm0, %xmm1, %xmm0
+; AVXNC-NEXT: vpslld $16, %xmm1, %xmm1
+; AVXNC-NEXT: vpslld $16, %xmm0, %xmm0
+; AVXNC-NEXT: vaddss %xmm1, %xmm0, %xmm0
; AVXNC-NEXT: {vex} vcvtneps2bf16 %xmm0, %xmm0
; AVXNC-NEXT: retq
%add = fadd bfloat %a, %b
@@ -140,152 +123,103 @@ define bfloat @add2(bfloat %a, bfloat %b) nounwind {
define void @add_double(ptr %pa, ptr %pb, ptr %pc) nounwind {
; X86-LABEL: add_double:
; X86: # %bb.0:
-; X86-NEXT: pushl %ebx
; X86-NEXT: pushl %edi
; X86-NEXT: pushl %esi
-; X86-NEXT: subl $16, %esp
+; X86-NEXT: subl $36, %esp
; X86-NEXT: movl {{[0-9]+}}(%esp), %esi
-; X86-NEXT: movl {{[0-9]+}}(%esp), %ebx
+; X86-NEXT: movl {{[0-9]+}}(%esp), %edi
; X86-NEXT: movl {{[0-9]+}}(%esp), %eax
; X86-NEXT: vmovsd {{.*#+}} xmm0 = mem[0],zero
; X86-NEXT: vmovsd %xmm0, (%esp)
; X86-NEXT: calll __truncdfbf2
-; X86-NEXT: vmovw %xmm0, %edi
-; X86-NEXT: vmovsd {{.*#+}} xmm0 = mem[0],zero
-; X86-NEXT: vmovsd %xmm0, (%esp)
+; X86-NEXT: vmovaps %xmm0, {{[-0-9]+}}(%e{{[sb]}}p) # 16-byte Spill
+; X86-NEXT: vmovq {{.*#+}} xmm0 = mem[0],zero
+; X86-NEXT: vmovq %xmm0, (%esp)
; X86-NEXT: calll __truncdfbf2
-; X86-NEXT: vmovw %xmm0, %eax
-; X86-NEXT: shll $16, %eax
-; X86-NEXT: vmovd %eax, %xmm0
-; X86-NEXT: shll $16, %edi
-; X86-NEXT: vmovd %edi, %xmm1
+; X86-NEXT: vpslld $16, %xmm0, %xmm0
+; X86-NEXT: vpslld $16, {{[-0-9]+}}(%e{{[sb]}}p), %xmm1 # 16-byte Folded Reload
; X86-NEXT: vaddss %xmm0, %xmm1, %xmm0
; X86-NEXT: vcvtneps2bf16 %xmm0, %xmm0
-; X86-NEXT: vmovw %xmm0, %eax
-; X86-NEXT: shll $16, %eax
-; X86-NEXT: vmovd %eax, %xmm0
+; X86-NEXT: vpslld $16, %xmm0, %xmm0
; X86-NEXT: vcvtss2sd %xmm0, %xmm0, %xmm0
; X86-NEXT: vmovsd %xmm0, (%esi)
-; X86-NEXT: addl $16, %esp
+; X86-NEXT: addl $36, %esp
; X86-NEXT: popl %esi
; X86-NEXT: popl %edi
-; X86-NEXT: popl %ebx
; X86-NEXT: retl
;
; SSE2-LABEL: add_double:
; SSE2: # %bb.0:
-; SSE2-NEXT: pushq %rbp
; SSE2-NEXT: pushq %r14
; SSE2-NEXT: pushq %rbx
+; SSE2-NEXT: subq $24, %rsp
; SSE2-NEXT: movq %rdx, %rbx
; SSE2-NEXT: movq %rsi, %r14
-; SSE2-NEXT: movq {{.*#+}} xmm0 = mem[0],zero
+; SSE2-NEXT: movsd {{.*#+}} xmm0 = mem[0],zero
; SSE2-NEXT: callq __truncdfbf2 at PLT
-; SSE2-NEXT: pextrw $0, %xmm0, %ebp
+; SSE2-NEXT: movaps %xmm0, (%rsp) # 16-byte Spill
; SSE2-NEXT: movq {{.*#+}} xmm0 = mem[0],zero
; SSE2-NEXT: callq __truncdfbf2 at PLT
-; SSE2-NEXT: pextrw $0, %xmm0, %eax
-; SSE2-NEXT: shll $16, %eax
-; SSE2-NEXT: movd %eax, %xmm1
-; SSE2-NEXT: shll $16, %ebp
-; SSE2-NEXT: movd %ebp, %xmm0
+; SSE2-NEXT: pslld $16, %xmm0
+; SSE2-NEXT: movdqa (%rsp), %xmm1 # 16-byte Reload
+; SSE2-NEXT: pslld $16, %xmm1
; SSE2-NEXT: addss %xmm1, %xmm0
; SSE2-NEXT: callq __truncsfbf2 at PLT
-; SSE2-NEXT: pextrw $0, %xmm0, %eax
-; SSE2-NEXT: shll $16, %eax
-; SSE2-NEXT: movd %eax, %xmm0
+; SSE2-NEXT: pslld $16, %xmm0
; SSE2-NEXT: cvtss2sd %xmm0, %xmm0
; SSE2-NEXT: movsd %xmm0, (%rbx)
+; SSE2-NEXT: addq $24, %rsp
; SSE2-NEXT: popq %rbx
; SSE2-NEXT: popq %r14
-; SSE2-NEXT: popq %rbp
; SSE2-NEXT: retq
;
-; AVX512BF16-LABEL: add_double:
-; AVX512BF16: # %bb.0:
-; AVX512BF16-NEXT: pushq %rbp
-; AVX512BF16-NEXT: pushq %r14
-; AVX512BF16-NEXT: pushq %rbx
-; AVX512BF16-NEXT: movq %rdx, %rbx
-; AVX512BF16-NEXT: movq %rsi, %r14
-; AVX512BF16-NEXT: vmovq {{.*#+}} xmm0 = mem[0],zero
-; AVX512BF16-NEXT: callq __truncdfbf2 at PLT
-; AVX512BF16-NEXT: vpextrw $0, %xmm0, %ebp
-; AVX512BF16-NEXT: vmovq {{.*#+}} xmm0 = mem[0],zero
-; AVX512BF16-NEXT: callq __truncdfbf2 at PLT
-; AVX512BF16-NEXT: vpextrw $0, %xmm0, %eax
-; AVX512BF16-NEXT: shll $16, %eax
-; AVX512BF16-NEXT: vmovd %eax, %xmm0
-; AVX512BF16-NEXT: shll $16, %ebp
-; AVX512BF16-NEXT: vmovd %ebp, %xmm1
-; AVX512BF16-NEXT: vaddss %xmm0, %xmm1, %xmm0
-; AVX512BF16-NEXT: vcvtneps2bf16 %xmm0, %xmm0
-; AVX512BF16-NEXT: vmovd %xmm0, %eax
-; AVX512BF16-NEXT: shll $16, %eax
-; AVX512BF16-NEXT: vmovd %eax, %xmm0
-; AVX512BF16-NEXT: vcvtss2sd %xmm0, %xmm0, %xmm0
-; AVX512BF16-NEXT: vmovsd %xmm0, (%rbx)
-; AVX512BF16-NEXT: popq %rbx
-; AVX512BF16-NEXT: popq %r14
-; AVX512BF16-NEXT: popq %rbp
-; AVX512BF16-NEXT: retq
-;
-; AVX512FP16-LABEL: add_double:
-; AVX512FP16: # %bb.0:
-; AVX512FP16-NEXT: pushq %rbp
-; AVX512FP16-NEXT: pushq %r14
-; AVX512FP16-NEXT: pushq %rbx
-; AVX512FP16-NEXT: movq %rdx, %rbx
-; AVX512FP16-NEXT: movq %rsi, %r14
-; AVX512FP16-NEXT: vmovsd {{.*#+}} xmm0 = mem[0],zero
-; AVX512FP16-NEXT: callq __truncdfbf2 at PLT
-; AVX512FP16-NEXT: vmovw %xmm0, %ebp
-; AVX512FP16-NEXT: vmovsd {{.*#+}} xmm0 = mem[0],zero
-; AVX512FP16-NEXT: callq __truncdfbf2 at PLT
-; AVX512FP16-NEXT: vmovw %xmm0, %eax
-; AVX512FP16-NEXT: shll $16, %eax
-; AVX512FP16-NEXT: vmovd %eax, %xmm0
-; AVX512FP16-NEXT: shll $16, %ebp
-; AVX512FP16-NEXT: vmovd %ebp, %xmm1
-; AVX512FP16-NEXT: vaddss %xmm0, %xmm1, %xmm0
-; AVX512FP16-NEXT: vcvtneps2bf16 %xmm0, %xmm0
-; AVX512FP16-NEXT: vmovw %xmm0, %eax
-; AVX512FP16-NEXT: shll $16, %eax
-; AVX512FP16-NEXT: vmovd %eax, %xmm0
-; AVX512FP16-NEXT: vcvtss2sd %xmm0, %xmm0, %xmm0
-; AVX512FP16-NEXT: vmovsd %xmm0, (%rbx)
-; AVX512FP16-NEXT: popq %rbx
-; AVX512FP16-NEXT: popq %r14
-; AVX512FP16-NEXT: popq %rbp
-; AVX512FP16-NEXT: retq
+; AVX512-LABEL: add_double:
+; AVX512: # %bb.0:
+; AVX512-NEXT: pushq %r14
+; AVX512-NEXT: pushq %rbx
+; AVX512-NEXT: subq $24, %rsp
+; AVX512-NEXT: movq %rdx, %rbx
+; AVX512-NEXT: movq %rsi, %r14
+; AVX512-NEXT: vmovsd {{.*#+}} xmm0 = mem[0],zero
+; AVX512-NEXT: callq __truncdfbf2 at PLT
+; AVX512-NEXT: vmovaps %xmm0, (%rsp) # 16-byte Spill
+; AVX512-NEXT: vmovq {{.*#+}} xmm0 = mem[0],zero
+; AVX512-NEXT: callq __truncdfbf2 at PLT
+; AVX512-NEXT: vpslld $16, %xmm0, %xmm0
+; AVX512-NEXT: vpslld $16, (%rsp), %xmm1 # 16-byte Folded Reload
+; AVX512-NEXT: vaddss %xmm0, %xmm1, %xmm0
+; AVX512-NEXT: vcvtneps2bf16 %xmm0, %xmm0
+; AVX512-NEXT: vpslld $16, %xmm0, %xmm0
+; AVX512-NEXT: vcvtss2sd %xmm0, %xmm0, %xmm0
+; AVX512-NEXT: vmovsd %xmm0, (%rbx)
+; AVX512-NEXT: addq $24, %rsp
+; AVX512-NEXT: popq %rbx
+; AVX512-NEXT: popq %r14
+; AVX512-NEXT: retq
;
; AVXNC-LABEL: add_double:
; AVXNC: # %bb.0:
-; AVXNC-NEXT: pushq %rbp
; AVXNC-NEXT: pushq %r14
; AVXNC-NEXT: pushq %rbx
+; AVXNC-NEXT: subq $24, %rsp
; AVXNC-NEXT: movq %rdx, %rbx
; AVXNC-NEXT: movq %rsi, %r14
-; AVXNC-NEXT: vmovq {{.*#+}} xmm0 = mem[0],zero
+; AVXNC-NEXT: vmovsd {{.*#+}} xmm0 = mem[0],zero
; AVXNC-NEXT: callq __truncdfbf2 at PLT
-; AVXNC-NEXT: vpextrw $0, %xmm0, %ebp
+; AVXNC-NEXT: vmovaps %xmm0, (%rsp) # 16-byte Spill
; AVXNC-NEXT: vmovq {{.*#+}} xmm0 = mem[0],zero
; AVXNC-NEXT: callq __truncdfbf2 at PLT
-; AVXNC-NEXT: vpextrw $0, %xmm0, %eax
-; AVXNC-NEXT: shll $16, %eax
-; AVXNC-NEXT: vmovd %eax, %xmm0
-; AVXNC-NEXT: shll $16, %ebp
-; AVXNC-NEXT: vmovd %ebp, %xmm1
+; AVXNC-NEXT: vpslld $16, %xmm0, %xmm0
+; AVXNC-NEXT: vmovdqa (%rsp), %xmm1 # 16-byte Reload
+; AVXNC-NEXT: vpslld $16, %xmm1, %xmm1
; AVXNC-NEXT: vaddss %xmm0, %xmm1, %xmm0
; AVXNC-NEXT: {vex} vcvtneps2bf16 %xmm0, %xmm0
-; AVXNC-NEXT: vmovd %xmm0, %eax
-; AVXNC-NEXT: shll $16, %eax
-; AVXNC-NEXT: vmovd %eax, %xmm0
+; AVXNC-NEXT: vpslld $16, %xmm0, %xmm0
; AVXNC-NEXT: vcvtss2sd %xmm0, %xmm0, %xmm0
; AVXNC-NEXT: vmovsd %xmm0, (%rbx)
+; AVXNC-NEXT: addq $24, %rsp
; AVXNC-NEXT: popq %rbx
; AVXNC-NEXT: popq %r14
-; AVXNC-NEXT: popq %rbp
; AVXNC-NEXT: retq
%la = load double, ptr %pa
%a = fptrunc double %la to bfloat
@@ -300,130 +234,79 @@ define void @add_double(ptr %pa, ptr %pb, ptr %pc) nounwind {
define double @add_double2(double %da, double %db) nounwind {
; X86-LABEL: add_double2:
; X86: # %bb.0:
-; X86-NEXT: pushl %esi
-; X86-NEXT: subl $24, %esp
+; X86-NEXT: subl $44, %esp
; X86-NEXT: vmovsd {{.*#+}} xmm0 = mem[0],zero
; X86-NEXT: vmovsd %xmm0, (%esp)
; X86-NEXT: calll __truncdfbf2
-; X86-NEXT: vmovw %xmm0, %esi
-; X86-NEXT: vmovsd {{.*#+}} xmm0 = mem[0],zero
-; X86-NEXT: vmovsd %xmm0, (%esp)
+; X86-NEXT: vmovaps %xmm0, {{[-0-9]+}}(%e{{[sb]}}p) # 16-byte Spill
+; X86-NEXT: vmovq {{.*#+}} xmm0 = mem[0],zero
+; X86-NEXT: vmovq %xmm0, (%esp)
; X86-NEXT: calll __truncdfbf2
-; X86-NEXT: vmovw %xmm0, %eax
-; X86-NEXT: shll $16, %eax
-; X86-NEXT: vmovd %eax, %xmm0
-; X86-NEXT: shll $16, %esi
-; X86-NEXT: vmovd %esi, %xmm1
+; X86-NEXT: vpslld $16, %xmm0, %xmm0
+; X86-NEXT: vpslld $16, {{[-0-9]+}}(%e{{[sb]}}p), %xmm1 # 16-byte Folded Reload
; X86-NEXT: vaddss %xmm0, %xmm1, %xmm0
; X86-NEXT: vcvtneps2bf16 %xmm0, %xmm0
-; X86-NEXT: vmovw %xmm0, %eax
-; X86-NEXT: shll $16, %eax
-; X86-NEXT: vmovd %eax, %xmm0
+; X86-NEXT: vpslld $16, %xmm0, %xmm0
; X86-NEXT: vcvtss2sd %xmm0, %xmm0, %xmm0
; X86-NEXT: vmovsd %xmm0, {{[0-9]+}}(%esp)
; X86-NEXT: fldl {{[0-9]+}}(%esp)
-; X86-NEXT: addl $24, %esp
-; X86-NEXT: popl %esi
+; X86-NEXT: addl $44, %esp
; X86-NEXT: retl
;
; SSE2-LABEL: add_double2:
; SSE2: # %bb.0:
-; SSE2-NEXT: pushq %rbx
-; SSE2-NEXT: subq $16, %rsp
+; SSE2-NEXT: subq $40, %rsp
; SSE2-NEXT: movsd %xmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Spill
; SSE2-NEXT: callq __truncdfbf2 at PLT
-; SSE2-NEXT: pextrw $0, %xmm0, %ebx
+; SSE2-NEXT: movaps %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
; SSE2-NEXT: movq {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 8-byte Folded Reload
; SSE2-NEXT: # xmm0 = mem[0],zero
; SSE2-NEXT: callq __truncdfbf2 at PLT
-; SSE2-NEXT: pextrw $0, %xmm0, %eax
-; SSE2-NEXT: shll $16, %eax
-; SSE2-NEXT: movd %eax, %xmm1
-; SSE2-NEXT: shll $16, %ebx
-; SSE2-NEXT: movd %ebx, %xmm0
+; SSE2-NEXT: pslld $16, %xmm0
+; SSE2-NEXT: movdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm1 # 16-byte Reload
+; SSE2-NEXT: pslld $16, %xmm1
; SSE2-NEXT: addss %xmm1, %xmm0
; SSE2-NEXT: callq __truncsfbf2 at PLT
-; SSE2-NEXT: pextrw $0, %xmm0, %eax
-; SSE2-NEXT: shll $16, %eax
-; SSE2-NEXT: movd %eax, %xmm0
+; SSE2-NEXT: pslld $16, %xmm0
; SSE2-NEXT: cvtss2sd %xmm0, %xmm0
-; SSE2-NEXT: addq $16, %rsp
-; SSE2-NEXT: popq %rbx
+; SSE2-NEXT: addq $40, %rsp
; SSE2-NEXT: retq
;
-; AVX512BF16-LABEL: add_double2:
-; AVX512BF16: # %bb.0:
-; AVX512BF16-NEXT: pushq %rbx
-; AVX512BF16-NEXT: subq $16, %rsp
-; AVX512BF16-NEXT: vmovsd %xmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Spill
-; AVX512BF16-NEXT: callq __truncdfbf2 at PLT
-; AVX512BF16-NEXT: vpextrw $0, %xmm0, %ebx
-; AVX512BF16-NEXT: vmovq {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 8-byte Folded Reload
-; AVX512BF16-NEXT: # xmm0 = mem[0],zero
-; AVX512BF16-NEXT: callq __truncdfbf2 at PLT
-; AVX512BF16-NEXT: vpextrw $0, %xmm0, %eax
-; AVX512BF16-NEXT: shll $16, %eax
-; AVX512BF16-NEXT: vmovd %eax, %xmm0
-; AVX512BF16-NEXT: shll $16, %ebx
-; AVX512BF16-NEXT: vmovd %ebx, %xmm1
-; AVX512BF16-NEXT: vaddss %xmm0, %xmm1, %xmm0
-; AVX512BF16-NEXT: vcvtneps2bf16 %xmm0, %xmm0
-; AVX512BF16-NEXT: vmovd %xmm0, %eax
-; AVX512BF16-NEXT: shll $16, %eax
-; AVX512BF16-NEXT: vmovd %eax, %xmm0
-; AVX512BF16-NEXT: vcvtss2sd %xmm0, %xmm0, %xmm0
-; AVX512BF16-NEXT: addq $16, %rsp
-; AVX512BF16-NEXT: popq %rbx
-; AVX512BF16-NEXT: retq
-;
-; AVX512FP16-LABEL: add_double2:
-; AVX512FP16: # %bb.0:
-; AVX512FP16-NEXT: pushq %rbx
-; AVX512FP16-NEXT: subq $16, %rsp
-; AVX512FP16-NEXT: vmovsd %xmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Spill
-; AVX512FP16-NEXT: callq __truncdfbf2 at PLT
-; AVX512FP16-NEXT: vmovw %xmm0, %ebx
-; AVX512FP16-NEXT: vmovsd {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 8-byte Reload
-; AVX512FP16-NEXT: # xmm0 = mem[0],zero
-; AVX512FP16-NEXT: callq __truncdfbf2 at PLT
-; AVX512FP16-NEXT: vmovw %xmm0, %eax
-; AVX512FP16-NEXT: shll $16, %eax
-; AVX512FP16-NEXT: vmovd %eax, %xmm0
-; AVX512FP16-NEXT: shll $16, %ebx
-; AVX512FP16-NEXT: vmovd %ebx, %xmm1
-; AVX512FP16-NEXT: vaddss %xmm0, %xmm1, %xmm0
-; AVX512FP16-NEXT: vcvtneps2bf16 %xmm0, %xmm0
-; AVX512FP16-NEXT: vmovw %xmm0, %eax
-; AVX512FP16-NEXT: shll $16, %eax
-; AVX512FP16-NEXT: vmovd %eax, %xmm0
-; AVX512FP16-NEXT: vcvtss2sd %xmm0, %xmm0, %xmm0
-; AVX512FP16-NEXT: addq $16, %rsp
-; AVX512FP16-NEXT: popq %rbx
-; AVX512FP16-NEXT: retq
+; AVX512-LABEL: add_double2:
+; AVX512: # %bb.0:
+; AVX512-NEXT: subq $40, %rsp
+; AVX512-NEXT: vmovsd %xmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Spill
+; AVX512-NEXT: callq __truncdfbf2 at PLT
+; AVX512-NEXT: vmovaps %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
+; AVX512-NEXT: vmovq {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 8-byte Folded Reload
+; AVX512-NEXT: # xmm0 = mem[0],zero
+; AVX512-NEXT: callq __truncdfbf2 at PLT
+; AVX512-NEXT: vpslld $16, %xmm0, %xmm0
+; AVX512-NEXT: vpslld $16, {{[-0-9]+}}(%r{{[sb]}}p), %xmm1 # 16-byte Folded Reload
+; AVX512-NEXT: vaddss %xmm0, %xmm1, %xmm0
+; AVX512-NEXT: vcvtneps2bf16 %xmm0, %xmm0
+; AVX512-NEXT: vpslld $16, %xmm0, %xmm0
+; AVX512-NEXT: vcvtss2sd %xmm0, %xmm0, %xmm0
+; AVX512-NEXT: addq $40, %rsp
+; AVX512-NEXT: retq
;
; AVXNC-LABEL: add_double2:
; AVXNC: # %bb.0:
-; AVXNC-NEXT: pushq %rbx
-; AVXNC-NEXT: subq $16, %rsp
+; AVXNC-NEXT: subq $40, %rsp
; AVXNC-NEXT: vmovsd %xmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Spill
; AVXNC-NEXT: callq __truncdfbf2 at PLT
-; AVXNC-NEXT: vpextrw $0, %xmm0, %ebx
+; AVXNC-NEXT: vmovaps %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
; AVXNC-NEXT: vmovq {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 8-byte Folded Reload
; AVXNC-NEXT: # xmm0 = mem[0],zero
; AVXNC-NEXT: callq __truncdfbf2 at PLT
-; AVXNC-NEXT: vpextrw $0, %xmm0, %eax
-; AVXNC-NEXT: shll $16, %eax
-; AVXNC-NEXT: vmovd %eax, %xmm0
-; AVXNC-NEXT: shll $16, %ebx
-; AVXNC-NEXT: vmovd %ebx, %xmm1
+; AVXNC-NEXT: vpslld $16, %xmm0, %xmm0
+; AVXNC-NEXT: vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm1 # 16-byte Reload
+; AVXNC-NEXT: vpslld $16, %xmm1, %xmm1
; AVXNC-NEXT: vaddss %xmm0, %xmm1, %xmm0
; AVXNC-NEXT: {vex} vcvtneps2bf16 %xmm0, %xmm0
-; AVXNC-NEXT: vmovd %xmm0, %eax
-; AVXNC-NEXT: shll $16, %eax
-; AVXNC-NEXT: vmovd %eax, %xmm0
+; AVXNC-NEXT: vpslld $16, %xmm0, %xmm0
; AVXNC-NEXT: vcvtss2sd %xmm0, %xmm0, %xmm0
-; AVXNC-NEXT: addq $16, %rsp
-; AVXNC-NEXT: popq %rbx
+; AVXNC-NEXT: addq $40, %rsp
; AVXNC-NEXT: retq
%a = fptrunc double %da to bfloat
%b = fptrunc double %db to bfloat
@@ -437,9 +320,8 @@ define void @add_constant(ptr %pa, ptr %pc) nounwind {
; X86: # %bb.0:
; X86-NEXT: movl {{[0-9]+}}(%esp), %eax
; X86-NEXT: movl {{[0-9]+}}(%esp), %ecx
-; X86-NEXT: movzwl (%ecx), %ecx
-; X86-NEXT: shll $16, %ecx
-; X86-NEXT: vmovd %ecx, %xmm0
+; X86-NEXT: vmovw (%ecx), %xmm0
+; X86-NEXT: vpslld $16, %xmm0, %xmm0
; X86-NEXT: vaddss {{\.?LCPI[0-9]+_[0-9]+}}, %xmm0, %xmm0
; X86-NEXT: vcvtneps2bf16 %xmm0, %xmm0
; X86-NEXT: vpextrw $0, %xmm0, (%eax)
@@ -459,15 +341,24 @@ define void @add_constant(ptr %pa, ptr %pc) nounwind {
; SSE2-NEXT: popq %rbx
; SSE2-NEXT: retq
;
-; AVX512-LABEL: add_constant:
-; AVX512: # %bb.0:
-; AVX512-NEXT: movzwl (%rdi), %eax
-; AVX512-NEXT: shll $16, %eax
-; AVX512-NEXT: vmovd %eax, %xmm0
-; AVX512-NEXT: vaddss {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm0, %xmm0
-; AVX512-NEXT: vcvtneps2bf16 %xmm0, %xmm0
-; AVX512-NEXT: vpextrw $0, %xmm0, (%rsi)
-; AVX512-NEXT: retq
+; AVX512BF16-LABEL: add_constant:
+; AVX512BF16: # %bb.0:
+; AVX512BF16-NEXT: movzwl (%rdi), %eax
+; AVX512BF16-NEXT: shll $16, %eax
+; AVX512BF16-NEXT: vmovd %eax, %xmm0
+; AVX512BF16-NEXT: vaddss {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm0, %xmm0
+; AVX512BF16-NEXT: vcvtneps2bf16 %xmm0, %xmm0
+; AVX512BF16-NEXT: vpextrw $0, %xmm0, (%rsi)
+; AVX512BF16-NEXT: retq
+;
+; AVX512FP16-LABEL: add_constant:
+; AVX512FP16: # %bb.0:
+; AVX512FP16-NEXT: vmovw (%rdi), %xmm0
+; AVX512FP16-NEXT: vpslld $16, %xmm0, %xmm0
+; AVX512FP16-NEXT: vaddss {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm0, %xmm0
+; AVX512FP16-NEXT: vcvtneps2bf16 %xmm0, %xmm0
+; AVX512FP16-NEXT: vpextrw $0, %xmm0, (%rsi)
+; AVX512FP16-NEXT: retq
;
; AVXNC-LABEL: add_constant:
; AVXNC: # %bb.0:
@@ -487,9 +378,8 @@ define void @add_constant(ptr %pa, ptr %pc) nounwind {
define bfloat @add_constant2(bfloat %a) nounwind {
; X86-LABEL: add_constant2:
; X86: # %bb.0:
-; X86-NEXT: movl {{[0-9]+}}(%esp), %eax
-; X86-NEXT: shll $16, %eax
-; X86-NEXT: vmovd %eax, %xmm0
+; X86-NEXT: vmovw {{[0-9]+}}(%esp), %xmm0
+; X86-NEXT: vpslld $16, %xmm0, %xmm0
; X86-NEXT: vaddss {{\.?LCPI[0-9]+_[0-9]+}}, %xmm0, %xmm0
; X86-NEXT: vcvtneps2bf16 %xmm0, %xmm0
; X86-NEXT: retl
@@ -497,37 +387,22 @@ define bfloat @add_constant2(bfloat %a) nounwind {
; SSE2-LABEL: add_constant2:
; SSE2: # %bb.0:
; SSE2-NEXT: pushq %rax
-; SSE2-NEXT: pextrw $0, %xmm0, %eax
-; SSE2-NEXT: shll $16, %eax
-; SSE2-NEXT: movd %eax, %xmm0
+; SSE2-NEXT: pslld $16, %xmm0
; SSE2-NEXT: addss {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm0
; SSE2-NEXT: callq __truncsfbf2 at PLT
; SSE2-NEXT: popq %rax
; SSE2-NEXT: retq
;
-; AVX512BF16-LABEL: add_constant2:
-; AVX512BF16: # %bb.0:
-; AVX512BF16-NEXT: vpextrw $0, %xmm0, %eax
-; AVX512BF16-NEXT: shll $16, %eax
-; AVX512BF16-NEXT: vmovd %eax, %xmm0
-; AVX512BF16-NEXT: vaddss {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm0, %xmm0
-; AVX512BF16-NEXT: vcvtneps2bf16 %xmm0, %xmm0
-; AVX512BF16-NEXT: retq
-;
-; AVX512FP16-LABEL: add_constant2:
-; AVX512FP16: # %bb.0:
-; AVX512FP16-NEXT: vmovw %xmm0, %eax
-; AVX512FP16-NEXT: shll $16, %eax
-; AVX512FP16-NEXT: vmovd %eax, %xmm0
-; AVX512FP16-NEXT: vaddss {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm0, %xmm0
-; AVX512FP16-NEXT: vcvtneps2bf16 %xmm0, %xmm0
-; AVX512FP16-NEXT: retq
+; AVX512-LABEL: add_constant2:
+; AVX512: # %bb.0:
+; AVX512-NEXT: vpslld $16, %xmm0, %xmm0
+; AVX512-NEXT: vaddss {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm0, %xmm0
+; AVX512-NEXT: vcvtneps2bf16 %xmm0, %xmm0
+; AVX512-NEXT: retq
;
; AVXNC-LABEL: add_constant2:
; AVXNC: # %bb.0:
-; AVXNC-NEXT: vpextrw $0, %xmm0, %eax
-; AVXNC-NEXT: shll $16, %eax
-; AVXNC-NEXT: vmovd %eax, %xmm0
+; AVXNC-NEXT: vpslld $16, %xmm0, %xmm0
; AVXNC-NEXT: vaddss {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm0, %xmm0
; AVXNC-NEXT: {vex} vcvtneps2bf16 %xmm0, %xmm0
; AVXNC-NEXT: retq
@@ -626,43 +501,34 @@ define bfloat @fold_from_half(half %a) nounwind {
define half @fold_to_half(bfloat %a) nounwind {
; X86-LABEL: fold_to_half:
; X86: # %bb.0:
-; X86-NEXT: movl {{[0-9]+}}(%esp), %eax
-; X86-NEXT: shll $16, %eax
-; X86-NEXT: vmovd %eax, %xmm0
+; X86-NEXT: vmovw {{[0-9]+}}(%esp), %xmm0
+; X86-NEXT: vpslld $16, %xmm0, %xmm0
; X86-NEXT: vcvtss2sh %xmm0, %xmm0, %xmm0
; X86-NEXT: retl
;
; SSE2-LABEL: fold_to_half:
; SSE2: # %bb.0:
; SSE2-NEXT: pushq %rax
-; SSE2-NEXT: pextrw $0, %xmm0, %eax
-; SSE2-NEXT: shll $16, %eax
-; SSE2-NEXT: movd %eax, %xmm0
+; SSE2-NEXT: pslld $16, %xmm0
; SSE2-NEXT: callq __truncsfhf2 at PLT
; SSE2-NEXT: popq %rax
; SSE2-NEXT: retq
;
; AVX512BF16-LABEL: fold_to_half:
; AVX512BF16: # %bb.0:
-; AVX512BF16-NEXT: vpextrw $0, %xmm0, %eax
-; AVX512BF16-NEXT: shll $16, %eax
-; AVX512BF16-NEXT: vmovd %eax, %xmm0
+; AVX512BF16-NEXT: vpslld $16, %xmm0, %xmm0
; AVX512BF16-NEXT: vcvtps2ph $4, %xmm0, %xmm0
; AVX512BF16-NEXT: retq
;
; AVX512FP16-LABEL: fold_to_half:
; AVX512FP16: # %bb.0:
-; AVX512FP16-NEXT: vmovw %xmm0, %eax
-; AVX512FP16-NEXT: shll $16, %eax
-; AVX512FP16-NEXT: vmovd %eax, %xmm0
+; AVX512FP16-NEXT: vpslld $16, %xmm0, %xmm0
; AVX512FP16-NEXT: vcvtss2sh %xmm0, %xmm0, %xmm0
; AVX512FP16-NEXT: retq
;
; AVXNC-LABEL: fold_to_half:
; AVXNC: # %bb.0:
-; AVXNC-NEXT: vpextrw $0, %xmm0, %eax
-; AVXNC-NEXT: shll $16, %eax
-; AVXNC-NEXT: vmovd %eax, %xmm0
+; AVXNC-NEXT: vpslld $16, %xmm0, %xmm0
; AVXNC-NEXT: vcvtps2ph $4, %xmm0, %xmm0
; AVXNC-NEXT: retq
%ext = fpext bfloat %a to float
@@ -716,124 +582,117 @@ define <8 x bfloat> @addv(<8 x bfloat> %a, <8 x bfloat> %b) nounwind {
; SSE2-NEXT: pushq %r13
; SSE2-NEXT: pushq %r12
; SSE2-NEXT: pushq %rbx
-; SSE2-NEXT: subq $56, %rsp
+; SSE2-NEXT: subq $104, %rsp
; SSE2-NEXT: movq %xmm0, %rcx
-; SSE2-NEXT: movq %rcx, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Spill
+; SSE2-NEXT: movdqa %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
; SSE2-NEXT: movq %rcx, %rax
; SSE2-NEXT: shrq $48, %rax
; SSE2-NEXT: movq %rax, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Spill
; SSE2-NEXT: movq %xmm1, %rdx
-; SSE2-NEXT: movq %rdx, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Spill
+; SSE2-NEXT: movdqa %xmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
; SSE2-NEXT: movq %rdx, %rax
; SSE2-NEXT: shrq $48, %rax
; SSE2-NEXT: movq %rax, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Spill
-; SSE2-NEXT: movq %rcx, %rax
-; SSE2-NEXT: shrq $32, %rax
-; SSE2-NEXT: movq %rax, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Spill
-; SSE2-NEXT: movq %rdx, %rax
-; SSE2-NEXT: shrq $32, %rax
-; SSE2-NEXT: movq %rax, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Spill
+; SSE2-NEXT: shrq $32, %rcx
+; SSE2-NEXT: movq %rcx, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Spill
+; SSE2-NEXT: shrq $32, %rdx
+; SSE2-NEXT: movq %rdx, (%rsp) # 8-byte Spill
; SSE2-NEXT: punpckhqdq {{.*#+}} xmm0 = xmm0[1,1]
-; SSE2-NEXT: movq %xmm0, %r15
-; SSE2-NEXT: movq %r15, %rbx
+; SSE2-NEXT: movq %xmm0, %r13
+; SSE2-NEXT: movdqa %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
+; SSE2-NEXT: movq %r13, %rbx
; SSE2-NEXT: shrq $48, %rbx
; SSE2-NEXT: punpckhqdq {{.*#+}} xmm1 = xmm1[1,1]
-; SSE2-NEXT: movq %xmm1, %r14
-; SSE2-NEXT: movq %r14, %rbp
-; SSE2-NEXT: shrq $48, %rbp
-; SSE2-NEXT: movq %r15, %r12
-; SSE2-NEXT: shrq $32, %r12
-; SSE2-NEXT: movq %r14, %r13
+; SSE2-NEXT: movdqa %xmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
+; SSE2-NEXT: movq %xmm1, %rbp
+; SSE2-NEXT: movq %rbp, %r15
+; SSE2-NEXT: shrq $48, %r15
; SSE2-NEXT: shrq $32, %r13
-; SSE2-NEXT: movl %r14d, %eax
-; SSE2-NEXT: shll $16, %eax
-; SSE2-NEXT: movd %eax, %xmm1
-; SSE2-NEXT: movl %r15d, %eax
-; SSE2-NEXT: shll $16, %eax
-; SSE2-NEXT: movd %eax, %xmm0
+; SSE2-NEXT: shrq $32, %rbp
+; SSE2-NEXT: pslld $16, %xmm1
+; SSE2-NEXT: pslld $16, %xmm0
; SSE2-NEXT: addss %xmm1, %xmm0
; SSE2-NEXT: callq __truncsfbf2 at PLT
; SSE2-NEXT: pextrw $0, %xmm0, %eax
-; SSE2-NEXT: movzwl %ax, %eax
-; SSE2-NEXT: movl %eax, {{[-0-9]+}}(%r{{[sb]}}p) # 4-byte Spill
-; SSE2-NEXT: andl $-65536, %r14d # imm = 0xFFFF0000
-; SSE2-NEXT: movd %r14d, %xmm1
-; SSE2-NEXT: andl $-65536, %r15d # imm = 0xFFFF0000
-; SSE2-NEXT: movd %r15d, %xmm0
+; SSE2-NEXT: movzwl %ax, %r12d
+; SSE2-NEXT: movdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 16-byte Reload
+; SSE2-NEXT: psrld $16, %xmm0
+; SSE2-NEXT: pslld $16, %xmm0
+; SSE2-NEXT: movdqa %xmm0, %xmm1
+; SSE2-NEXT: movdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 16-byte Reload
+; SSE2-NEXT: psrld $16, %xmm0
+; SSE2-NEXT: pslld $16, %xmm0
; SSE2-NEXT: addss %xmm1, %xmm0
; SSE2-NEXT: callq __truncsfbf2 at PLT
-; SSE2-NEXT: pextrw $0, %xmm0, %r15d
-; SSE2-NEXT: shll $16, %r15d
-; SSE2-NEXT: addl {{[-0-9]+}}(%r{{[sb]}}p), %r15d # 4-byte Folded Reload
-; SSE2-NEXT: shll $16, %r13d
-; SSE2-NEXT: movd %r13d, %xmm1
-; SSE2-NEXT: shll $16, %r12d
-; SSE2-NEXT: movd %r12d, %xmm0
+; SSE2-NEXT: pextrw $0, %xmm0, %r14d
+; SSE2-NEXT: shll $16, %r14d
+; SSE2-NEXT: orl %r12d, %r14d
+; SSE2-NEXT: movd %ebp, %xmm1
+; SSE2-NEXT: pslld $16, %xmm1
+; SSE2-NEXT: movd %r13d, %xmm0
+; SSE2-NEXT: pslld $16, %xmm0
; SSE2-NEXT: addss %xmm1, %xmm0
; SSE2-NEXT: callq __truncsfbf2 at PLT
; SSE2-NEXT: pextrw $0, %xmm0, %eax
-; SSE2-NEXT: movzwl %ax, %r14d
-; SSE2-NEXT: shll $16, %ebp
-; SSE2-NEXT: movd %ebp, %xmm1
-; SSE2-NEXT: shll $16, %ebx
+; SSE2-NEXT: movzwl %ax, %ebp
+; SSE2-NEXT: movd %r15d, %xmm1
+; SSE2-NEXT: pslld $16, %xmm1
; SSE2-NEXT: movd %ebx, %xmm0
+; SSE2-NEXT: pslld $16, %xmm0
; SSE2-NEXT: addss %xmm1, %xmm0
; SSE2-NEXT: callq __truncsfbf2 at PLT
-; SSE2-NEXT: pextrw $0, %xmm0, %ebx
-; SSE2-NEXT: shll $16, %ebx
-; SSE2-NEXT: orl %r14d, %ebx
-; SSE2-NEXT: shlq $32, %rbx
-; SSE2-NEXT: orq %r15, %rbx
-; SSE2-NEXT: movq {{[-0-9]+}}(%r{{[sb]}}p), %r15 # 8-byte Reload
-; SSE2-NEXT: movl %r15d, %eax
-; SSE2-NEXT: shll $16, %eax
-; SSE2-NEXT: movd %eax, %xmm1
-; SSE2-NEXT: movq {{[-0-9]+}}(%r{{[sb]}}p), %r14 # 8-byte Reload
-; SSE2-NEXT: movl %r14d, %eax
-; SSE2-NEXT: shll $16, %eax
-; SSE2-NEXT: movd %eax, %xmm0
+; SSE2-NEXT: pextrw $0, %xmm0, %r13d
+; SSE2-NEXT: shll $16, %r13d
+; SSE2-NEXT: orl %ebp, %r13d
+; SSE2-NEXT: shlq $32, %r13
+; SSE2-NEXT: orq %r14, %r13
+; SSE2-NEXT: movdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm1 # 16-byte Reload
+; SSE2-NEXT: pslld $16, %xmm1
+; SSE2-NEXT: movdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 16-byte Reload
+; SSE2-NEXT: pslld $16, %xmm0
; SSE2-NEXT: addss %xmm1, %xmm0
; SSE2-NEXT: callq __truncsfbf2 at PLT
; SSE2-NEXT: pextrw $0, %xmm0, %eax
; SSE2-NEXT: movzwl %ax, %ebp
-; SSE2-NEXT: movq %r15, %rax
-; SSE2-NEXT: andl $-65536, %eax # imm = 0xFFFF0000
-; SSE2-NEXT: movd %eax, %xmm1
-; SSE2-NEXT: movq %r14, %rax
-; SSE2-NEXT: andl $-65536, %eax # imm = 0xFFFF0000
-; SSE2-NEXT: movd %eax, %xmm0
+; SSE2-NEXT: movdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 16-byte Reload
+; SSE2-NEXT: psrld $16, %xmm0
+; SSE2-NEXT: pslld $16, %xmm0
+; SSE2-NEXT: movdqa %xmm0, %xmm1
+; SSE2-NEXT: movdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 16-byte Reload
+; SSE2-NEXT: psrld $16, %xmm0
+; SSE2-NEXT: pslld $16, %xmm0
; SSE2-NEXT: addss %xmm1, %xmm0
; SSE2-NEXT: callq __truncsfbf2 at PLT
-; SSE2-NEXT: pextrw $0, %xmm0, %r14d
-; SSE2-NEXT: shll $16, %r14d
-; SSE2-NEXT: orl %ebp, %r14d
-; SSE2-NEXT: movq {{[-0-9]+}}(%r{{[sb]}}p), %rax # 8-byte Reload
-; SSE2-NEXT: shll $16, %eax
-; SSE2-NEXT: movd %eax, %xmm1
-; SSE2-NEXT: movq {{[-0-9]+}}(%r{{[sb]}}p), %rax # 8-byte Reload
-; SSE2-NEXT: shll $16, %eax
-; SSE2-NEXT: movd %eax, %xmm0
+; SSE2-NEXT: pextrw $0, %xmm0, %ebx
+; SSE2-NEXT: shll $16, %ebx
+; SSE2-NEXT: orl %ebp, %ebx
+; SSE2-NEXT: movd (%rsp), %xmm1 # 4-byte Folded Reload
+; SSE2-NEXT: # xmm1 = mem[0],zero,zero,zero
+; SSE2-NEXT: pslld $16, %xmm1
+; SSE2-NEXT: movd {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 4-byte Folded Reload
+; SSE2-NEXT: # xmm0 = mem[0],zero,zero,zero
+; SSE2-NEXT: pslld $16, %xmm0
; SSE2-NEXT: addss %xmm1, %xmm0
; SSE2-NEXT: callq __truncsfbf2 at PLT
; SSE2-NEXT: pextrw $0, %xmm0, %eax
; SSE2-NEXT: movzwl %ax, %ebp
-; SSE2-NEXT: movq {{[-0-9]+}}(%r{{[sb]}}p), %rax # 8-byte Reload
-; SSE2-NEXT: shll $16, %eax
-; SSE2-NEXT: movd %eax, %xmm1
-; SSE2-NEXT: movq {{[-0-9]+}}(%r{{[sb]}}p), %rax # 8-byte Reload
-; SSE2-NEXT: shll $16, %eax
-; SSE2-NEXT: movd %eax, %xmm0
+; SSE2-NEXT: movd {{[-0-9]+}}(%r{{[sb]}}p), %xmm1 # 4-byte Folded Reload
+; SSE2-NEXT: # xmm1 = mem[0],zero,zero,zero
+; SSE2-NEXT: pslld $16, %xmm1
+; SSE2-NEXT: movd {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 4-byte Folded Reload
+; SSE2-NEXT: # xmm0 = mem[0],zero,zero,zero
+; SSE2-NEXT: pslld $16, %xmm0
; SSE2-NEXT: addss %xmm1, %xmm0
; SSE2-NEXT: callq __truncsfbf2 at PLT
; SSE2-NEXT: pextrw $0, %xmm0, %eax
; SSE2-NEXT: shll $16, %eax
; SSE2-NEXT: orl %ebp, %eax
; SSE2-NEXT: shlq $32, %rax
-; SSE2-NEXT: orq %r14, %rax
+; SSE2-NEXT: orq %rbx, %rax
; SSE2-NEXT: movq %rax, %xmm0
-; SSE2-NEXT: movq %rbx, %xmm1
+; SSE2-NEXT: movq %r13, %xmm1
; SSE2-NEXT: punpcklqdq {{.*#+}} xmm0 = xmm0[0],xmm1[0]
-; SSE2-NEXT: addq $56, %rsp
+; SSE2-NEXT: addq $104, %rsp
; SSE2-NEXT: popq %rbx
; SSE2-NEXT: popq %r12
; SSE2-NEXT: popq %r13
@@ -941,8 +800,8 @@ define <32 x bfloat> @pr63017_2() nounwind {
; SSE2-NEXT: jne .LBB16_1
; SSE2-NEXT: # %bb.2: # %cond.load
; SSE2-NEXT: movzwl (%rax), %eax
-; SSE2-NEXT: shll $16, %eax
; SSE2-NEXT: movd %eax, %xmm0
+; SSE2-NEXT: pslld $16, %xmm0
; SSE2-NEXT: jmp .LBB16_3
; SSE2-NEXT: .LBB16_1:
; SSE2-NEXT: movd {{.*#+}} xmm0 = [-1.0E+0,0.0E+0,0.0E+0,0.0E+0]
@@ -950,23 +809,20 @@ define <32 x bfloat> @pr63017_2() nounwind {
; SSE2-NEXT: pushq %r14
; SSE2-NEXT: pushq %rbx
; SSE2-NEXT: subq $88, %rsp
-; SSE2-NEXT: movd %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 4-byte Folded Spill
+; SSE2-NEXT: movdqa %xmm0, (%rsp) # 16-byte Spill
; SSE2-NEXT: callq __truncsfbf2 at PLT
; SSE2-NEXT: pextrw $0, %xmm0, %ebx
; SSE2-NEXT: shll $16, %ebx
-; SSE2-NEXT: movd {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 4-byte Folded Reload
-; SSE2-NEXT: # xmm0 = mem[0],zero,zero,zero
+; SSE2-NEXT: movdqa (%rsp), %xmm0 # 16-byte Reload
; SSE2-NEXT: callq __truncsfbf2 at PLT
; SSE2-NEXT: pextrw $0, %xmm0, %eax
; SSE2-NEXT: movzwl %ax, %r14d
; SSE2-NEXT: orl %ebx, %r14d
-; SSE2-NEXT: movd {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 4-byte Folded Reload
-; SSE2-NEXT: # xmm0 = mem[0],zero,zero,zero
+; SSE2-NEXT: movdqa (%rsp), %xmm0 # 16-byte Reload
; SSE2-NEXT: callq __truncsfbf2 at PLT
; SSE2-NEXT: pextrw $0, %xmm0, %ebx
; SSE2-NEXT: shll $16, %ebx
-; SSE2-NEXT: movd {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 4-byte Folded Reload
-; SSE2-NEXT: # xmm0 = mem[0],zero,zero,zero
+; SSE2-NEXT: movdqa (%rsp), %xmm0 # 16-byte Reload
; SSE2-NEXT: callq __truncsfbf2 at PLT
; SSE2-NEXT: pextrw $0, %xmm0, %eax
; SSE2-NEXT: movzwl %ax, %eax
@@ -975,24 +831,20 @@ define <32 x bfloat> @pr63017_2() nounwind {
; SSE2-NEXT: orq %r14, %rax
; SSE2-NEXT: movq %rax, %xmm0
; SSE2-NEXT: movdqa %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
-; SSE2-NEXT: movd {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 4-byte Folded Reload
-; SSE2-NEXT: # xmm0 = mem[0],zero,zero,zero
+; SSE2-NEXT: movdqa (%rsp), %xmm0 # 16-byte Reload
; SSE2-NEXT: callq __truncsfbf2 at PLT
; SSE2-NEXT: pextrw $0, %xmm0, %ebx
; SSE2-NEXT: shll $16, %ebx
-; SSE2-NEXT: movd {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 4-byte Folded Reload
-; SSE2-NEXT: # xmm0 = mem[0],zero,zero,zero
+; SSE2-NEXT: movdqa (%rsp), %xmm0 # 16-byte Reload
; SSE2-NEXT: callq __truncsfbf2 at PLT
; SSE2-NEXT: pextrw $0, %xmm0, %eax
; SSE2-NEXT: movzwl %ax, %r14d
; SSE2-NEXT: orl %ebx, %r14d
-; SSE2-NEXT: movd {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 4-byte Folded Reload
-; SSE2-NEXT: # xmm0 = mem[0],zero,zero,zero
+; SSE2-NEXT: movdqa (%rsp), %xmm0 # 16-byte Reload
; SSE2-NEXT: callq __truncsfbf2 at PLT
; SSE2-NEXT: pextrw $0, %xmm0, %ebx
; SSE2-NEXT: shll $16, %ebx
-; SSE2-NEXT: movd {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 4-byte Folded Reload
-; SSE2-NEXT: # xmm0 = mem[0],zero,zero,zero
+; SSE2-NEXT: movdqa (%rsp), %xmm0 # 16-byte Reload
; SSE2-NEXT: callq __truncsfbf2 at PLT
; SSE2-NEXT: pextrw $0, %xmm0, %eax
; SSE2-NEXT: movzwl %ax, %eax
@@ -1003,24 +855,20 @@ define <32 x bfloat> @pr63017_2() nounwind {
; SSE2-NEXT: movdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm1 # 16-byte Reload
; SSE2-NEXT: punpcklqdq {{.*#+}} xmm1 = xmm1[0],xmm0[0]
; SSE2-NEXT: movdqa %xmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
-; SSE2-NEXT: movd {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 4-byte Folded Reload
-; SSE2-NEXT: # xmm0 = mem[0],zero,zero,zero
+; SSE2-NEXT: movdqa (%rsp), %xmm0 # 16-byte Reload
; SSE2-NEXT: callq __truncsfbf2 at PLT
; SSE2-NEXT: pextrw $0, %xmm0, %ebx
; SSE2-NEXT: shll $16, %ebx
-; SSE2-NEXT: movd {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 4-byte Folded Reload
-; SSE2-NEXT: # xmm0 = mem[0],zero,zero,zero
+; SSE2-NEXT: movdqa (%rsp), %xmm0 # 16-byte Reload
; SSE2-NEXT: callq __truncsfbf2 at PLT
; SSE2-NEXT: pextrw $0, %xmm0, %eax
; SSE2-NEXT: movzwl %ax, %r14d
; SSE2-NEXT: orl %ebx, %r14d
-; SSE2-NEXT: movd {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 4-byte Folded Reload
-; SSE2-NEXT: # xmm0 = mem[0],zero,zero,zero
+; SSE2-NEXT: movdqa (%rsp), %xmm0 # 16-byte Reload
; SSE2-NEXT: callq __truncsfbf2 at PLT
; SSE2-NEXT: pextrw $0, %xmm0, %ebx
; SSE2-NEXT: shll $16, %ebx
-; SSE2-NEXT: movd {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 4-byte Folded Reload
-; SSE2-NEXT: # xmm0 = mem[0],zero,zero,zero
+; SSE2-NEXT: movdqa (%rsp), %xmm0 # 16-byte Reload
; SSE2-NEXT: callq __truncsfbf2 at PLT
; SSE2-NEXT: pextrw $0, %xmm0, %eax
; SSE2-NEXT: movzwl %ax, %eax
@@ -1029,24 +877,20 @@ define <32 x bfloat> @pr63017_2() nounwind {
; SSE2-NEXT: orq %r14, %rax
; SSE2-NEXT: movq %rax, %xmm0
; SSE2-NEXT: movdqa %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
-; SSE2-NEXT: movd {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 4-byte Folded Reload
-; SSE2-NEXT: # xmm0 = mem[0],zero,zero,zero
+; SSE2-NEXT: movdqa (%rsp), %xmm0 # 16-byte Reload
; SSE2-NEXT: callq __truncsfbf2 at PLT
; SSE2-NEXT: pextrw $0, %xmm0, %ebx
; SSE2-NEXT: shll $16, %ebx
-; SSE2-NEXT: movd {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 4-byte Folded Reload
-; SSE2-NEXT: # xmm0 = mem[0],zero,zero,zero
+; SSE2-NEXT: movdqa (%rsp), %xmm0 # 16-byte Reload
; SSE2-NEXT: callq __truncsfbf2 at PLT
; SSE2-NEXT: pextrw $0, %xmm0, %eax
; SSE2-NEXT: movzwl %ax, %r14d
; SSE2-NEXT: orl %ebx, %r14d
-; SSE2-NEXT: movd {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 4-byte Folded Reload
-; SSE2-NEXT: # xmm0 = mem[0],zero,zero,zero
+; SSE2-NEXT: movdqa (%rsp), %xmm0 # 16-byte Reload
; SSE2-NEXT: callq __truncsfbf2 at PLT
; SSE2-NEXT: pextrw $0, %xmm0, %ebx
; SSE2-NEXT: shll $16, %ebx
-; SSE2-NEXT: movd {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 4-byte Folded Reload
-; SSE2-NEXT: # xmm0 = mem[0],zero,zero,zero
+; SSE2-NEXT: movdqa (%rsp), %xmm0 # 16-byte Reload
; SSE2-NEXT: callq __truncsfbf2 at PLT
; SSE2-NEXT: pextrw $0, %xmm0, %eax
; SSE2-NEXT: movzwl %ax, %eax
@@ -1057,24 +901,20 @@ define <32 x bfloat> @pr63017_2() nounwind {
; SSE2-NEXT: movdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm1 # 16-byte Reload
; SSE2-NEXT: punpcklqdq {{.*#+}} xmm1 = xmm1[0],xmm0[0]
; SSE2-NEXT: movdqa %xmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
-; SSE2-NEXT: movd {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 4-byte Folded Reload
-; SSE2-NEXT: # xmm0 = mem[0],zero,zero,zero
+; SSE2-NEXT: movdqa (%rsp), %xmm0 # 16-byte Reload
; SSE2-NEXT: callq __truncsfbf2 at PLT
; SSE2-NEXT: pextrw $0, %xmm0, %ebx
; SSE2-NEXT: shll $16, %ebx
-; SSE2-NEXT: movd {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 4-byte Folded Reload
-; SSE2-NEXT: # xmm0 = mem[0],zero,zero,zero
+; SSE2-NEXT: movdqa (%rsp), %xmm0 # 16-byte Reload
; SSE2-NEXT: callq __truncsfbf2 at PLT
; SSE2-NEXT: pextrw $0, %xmm0, %eax
; SSE2-NEXT: movzwl %ax, %r14d
; SSE2-NEXT: orl %ebx, %r14d
-; SSE2-NEXT: movd {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 4-byte Folded Reload
-; SSE2-NEXT: # xmm0 = mem[0],zero,zero,zero
+; SSE2-NEXT: movdqa (%rsp), %xmm0 # 16-byte Reload
; SSE2-NEXT: callq __truncsfbf2 at PLT
; SSE2-NEXT: pextrw $0, %xmm0, %ebx
; SSE2-NEXT: shll $16, %ebx
-; SSE2-NEXT: movd {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 4-byte Folded Reload
-; SSE2-NEXT: # xmm0 = mem[0],zero,zero,zero
+; SSE2-NEXT: movdqa (%rsp), %xmm0 # 16-byte Reload
; SSE2-NEXT: callq __truncsfbf2 at PLT
; SSE2-NEXT: pextrw $0, %xmm0, %eax
; SSE2-NEXT: movzwl %ax, %eax
@@ -1083,24 +923,20 @@ define <32 x bfloat> @pr63017_2() nounwind {
; SSE2-NEXT: orq %r14, %rax
; SSE2-NEXT: movq %rax, %xmm0
; SSE2-NEXT: movdqa %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
-; SSE2-NEXT: movd {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 4-byte Folded Reload
-; SSE2-NEXT: # xmm0 = mem[0],zero,zero,zero
+; SSE2-NEXT: movdqa (%rsp), %xmm0 # 16-byte Reload
; SSE2-NEXT: callq __truncsfbf2 at PLT
; SSE2-NEXT: pextrw $0, %xmm0, %ebx
; SSE2-NEXT: shll $16, %ebx
-; SSE2-NEXT: movd {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 4-byte Folded Reload
-; SSE2-NEXT: # xmm0 = mem[0],zero,zero,zero
+; SSE2-NEXT: movdqa (%rsp), %xmm0 # 16-byte Reload
; SSE2-NEXT: callq __truncsfbf2 at PLT
; SSE2-NEXT: pextrw $0, %xmm0, %eax
; SSE2-NEXT: movzwl %ax, %r14d
; SSE2-NEXT: orl %ebx, %r14d
-; SSE2-NEXT: movd {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 4-byte Folded Reload
-; SSE2-NEXT: # xmm0 = mem[0],zero,zero,zero
+; SSE2-NEXT: movdqa (%rsp), %xmm0 # 16-byte Reload
; SSE2-NEXT: callq __truncsfbf2 at PLT
; SSE2-NEXT: pextrw $0, %xmm0, %ebx
; SSE2-NEXT: shll $16, %ebx
-; SSE2-NEXT: movd {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 4-byte Folded Reload
-; SSE2-NEXT: # xmm0 = mem[0],zero,zero,zero
+; SSE2-NEXT: movdqa (%rsp), %xmm0 # 16-byte Reload
; SSE2-NEXT: callq __truncsfbf2 at PLT
; SSE2-NEXT: pextrw $0, %xmm0, %eax
; SSE2-NEXT: movzwl %ax, %eax
@@ -1111,24 +947,20 @@ define <32 x bfloat> @pr63017_2() nounwind {
; SSE2-NEXT: movdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm1 # 16-byte Reload
; SSE2-NEXT: punpcklqdq {{.*#+}} xmm1 = xmm1[0],xmm0[0]
; SSE2-NEXT: movdqa %xmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
-; SSE2-NEXT: movd {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 4-byte Folded Reload
-; SSE2-NEXT: # xmm0 = mem[0],zero,zero,zero
+; SSE2-NEXT: movdqa (%rsp), %xmm0 # 16-byte Reload
; SSE2-NEXT: callq __truncsfbf2 at PLT
; SSE2-NEXT: pextrw $0, %xmm0, %ebx
; SSE2-NEXT: shll $16, %ebx
-; SSE2-NEXT: movd {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 4-byte Folded Reload
-; SSE2-NEXT: # xmm0 = mem[0],zero,zero,zero
+; SSE2-NEXT: movdqa (%rsp), %xmm0 # 16-byte Reload
; SSE2-NEXT: callq __truncsfbf2 at PLT
; SSE2-NEXT: pextrw $0, %xmm0, %eax
; SSE2-NEXT: movzwl %ax, %r14d
; SSE2-NEXT: orl %ebx, %r14d
-; SSE2-NEXT: movd {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 4-byte Folded Reload
-; SSE2-NEXT: # xmm0 = mem[0],zero,zero,zero
+; SSE2-NEXT: movdqa (%rsp), %xmm0 # 16-byte Reload
; SSE2-NEXT: callq __truncsfbf2 at PLT
; SSE2-NEXT: pextrw $0, %xmm0, %ebx
; SSE2-NEXT: shll $16, %ebx
-; SSE2-NEXT: movd {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 4-byte Folded Reload
-; SSE2-NEXT: # xmm0 = mem[0],zero,zero,zero
+; SSE2-NEXT: movdqa (%rsp), %xmm0 # 16-byte Reload
; SSE2-NEXT: callq __truncsfbf2 at PLT
; SSE2-NEXT: pextrw $0, %xmm0, %eax
; SSE2-NEXT: movzwl %ax, %eax
@@ -1137,24 +969,20 @@ define <32 x bfloat> @pr63017_2() nounwind {
; SSE2-NEXT: orq %r14, %rax
; SSE2-NEXT: movq %rax, %xmm0
; SSE2-NEXT: movdqa %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
-; SSE2-NEXT: movd {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 4-byte Folded Reload
-; SSE2-NEXT: # xmm0 = mem[0],zero,zero,zero
+; SSE2-NEXT: movdqa (%rsp), %xmm0 # 16-byte Reload
; SSE2-NEXT: callq __truncsfbf2 at PLT
; SSE2-NEXT: pextrw $0, %xmm0, %ebx
; SSE2-NEXT: shll $16, %ebx
-; SSE2-NEXT: movd {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 4-byte Folded Reload
-; SSE2-NEXT: # xmm0 = mem[0],zero,zero,zero
+; SSE2-NEXT: movdqa (%rsp), %xmm0 # 16-byte Reload
; SSE2-NEXT: callq __truncsfbf2 at PLT
; SSE2-NEXT: pextrw $0, %xmm0, %eax
; SSE2-NEXT: movzwl %ax, %r14d
; SSE2-NEXT: orl %ebx, %r14d
-; SSE2-NEXT: movd {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 4-byte Folded Reload
-; SSE2-NEXT: # xmm0 = mem[0],zero,zero,zero
+; SSE2-NEXT: movdqa (%rsp), %xmm0 # 16-byte Reload
; SSE2-NEXT: callq __truncsfbf2 at PLT
; SSE2-NEXT: pextrw $0, %xmm0, %ebx
; SSE2-NEXT: shll $16, %ebx
-; SSE2-NEXT: movd {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 4-byte Folded Reload
-; SSE2-NEXT: # xmm0 = mem[0],zero,zero,zero
+; SSE2-NEXT: movdqa (%rsp), %xmm0 # 16-byte Reload
; SSE2-NEXT: callq __truncsfbf2 at PLT
; SSE2-NEXT: pextrw $0, %xmm0, %eax
; SSE2-NEXT: movzwl %ax, %eax
@@ -2124,9 +1952,7 @@ define float @trunc_ext(float %a) nounwind {
; X86-NEXT: pushl %eax
; X86-NEXT: vmovss {{.*#+}} xmm0 = mem[0],zero,zero,zero
; X86-NEXT: vcvtneps2bf16 %xmm0, %xmm0
-; X86-NEXT: vmovw %xmm0, %eax
-; X86-NEXT: shll $16, %eax
-; X86-NEXT: vmovd %eax, %xmm0
+; X86-NEXT: vpslld $16, %xmm0, %xmm0
; X86-NEXT: vmovd %xmm0, (%esp)
; X86-NEXT: flds (%esp)
; X86-NEXT: popl %eax
@@ -2136,34 +1962,20 @@ define float @trunc_ext(float %a) nounwind {
; SSE2: # %bb.0:
; SSE2-NEXT: pushq %rax
; SSE2-NEXT: callq __truncsfbf2 at PLT
-; SSE2-NEXT: pextrw $0, %xmm0, %eax
-; SSE2-NEXT: shll $16, %eax
-; SSE2-NEXT: movd %eax, %xmm0
+; SSE2-NEXT: pslld $16, %xmm0
; SSE2-NEXT: popq %rax
; SSE2-NEXT: retq
;
-; AVX512BF16-LABEL: trunc_ext:
-; AVX512BF16: # %bb.0:
-; AVX512BF16-NEXT: vcvtneps2bf16 %xmm0, %xmm0
-; AVX512BF16-NEXT: vmovd %xmm0, %eax
-; AVX512BF16-NEXT: shll $16, %eax
-; AVX512BF16-NEXT: vmovd %eax, %xmm0
-; AVX512BF16-NEXT: retq
-;
-; AVX512FP16-LABEL: trunc_ext:
-; AVX512FP16: # %bb.0:
-; AVX512FP16-NEXT: vcvtneps2bf16 %xmm0, %xmm0
-; AVX512FP16-NEXT: vmovw %xmm0, %eax
-; AVX512FP16-NEXT: shll $16, %eax
-; AVX512FP16-NEXT: vmovd %eax, %xmm0
-; AVX512FP16-NEXT: retq
+; AVX512-LABEL: trunc_ext:
+; AVX512: # %bb.0:
+; AVX512-NEXT: vcvtneps2bf16 %xmm0, %xmm0
+; AVX512-NEXT: vpslld $16, %xmm0, %xmm0
+; AVX512-NEXT: retq
;
; AVXNC-LABEL: trunc_ext:
; AVXNC: # %bb.0:
; AVXNC-NEXT: {vex} vcvtneps2bf16 %xmm0, %xmm0
-; AVXNC-NEXT: vmovd %xmm0, %eax
-; AVXNC-NEXT: shll $16, %eax
-; AVXNC-NEXT: vmovd %eax, %xmm0
+; AVXNC-NEXT: vpslld $16, %xmm0, %xmm0
; AVXNC-NEXT: retq
%b = fptrunc float %a to bfloat
%c = fpext bfloat %b to float
diff --git a/llvm/test/CodeGen/X86/float-to-arbitrary-fp.ll b/llvm/test/CodeGen/X86/float-to-arbitrary-fp.ll
index 7b01ed05706d3..b3f9e68fc3368 100644
--- a/llvm/test/CodeGen/X86/float-to-arbitrary-fp.ll
+++ b/llvm/test/CodeGen/X86/float-to-arbitrary-fp.ll
@@ -2186,15 +2186,9 @@ define i8 @to_f8e5m2_from_f16(half %x) {
define i8 @to_f8e5m2_from_bf16(bfloat %x) {
; CHECK-LABEL: to_f8e5m2_from_bf16:
; CHECK: # %bb.0:
-; CHECK-NEXT: pushq %rbx
-; CHECK-NEXT: .cfi_def_cfa_offset 16
-; CHECK-NEXT: subq $32, %rsp
+; CHECK-NEXT: subq $40, %rsp
; CHECK-NEXT: .cfi_def_cfa_offset 48
-; CHECK-NEXT: .cfi_offset %rbx, -16
-; CHECK-NEXT: pextrw $0, %xmm0, %ebx
-; CHECK-NEXT: movl %ebx, %eax
-; CHECK-NEXT: shll $16, %eax
-; CHECK-NEXT: movd %eax, %xmm0
+; CHECK-NEXT: pslld $16, %xmm0
; CHECK-NEXT: movdqa %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
; CHECK-NEXT: leaq {{[0-9]+}}(%rsp), %rdi
; CHECK-NEXT: callq frexpf at PLT
@@ -2230,23 +2224,25 @@ define i8 @to_f8e5m2_from_bf16(bfloat %x) {
; CHECK-NEXT: cmpl $1, %eax
; CHECK-NEXT: cmovbl %ecx, %r10d
; CHECK-NEXT: addl %r8d, %r10d
-; CHECK-NEXT: xorl %eax, %eax
+; CHECK-NEXT: xorl %r8d, %r8d
; CHECK-NEXT: cmpl $4, %r10d
; CHECK-NEXT: cmovgel %ecx, %r10d
-; CHECK-NEXT: setge %al
-; CHECK-NEXT: shrl $8, %ebx
-; CHECK-NEXT: andl $128, %ebx
-; CHECK-NEXT: leal (%rbx,%rax,4), %eax
-; CHECK-NEXT: orl %r10d, %eax
+; CHECK-NEXT: setge %r8b
+; CHECK-NEXT: movdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm2 # 16-byte Reload
+; CHECK-NEXT: movd %xmm2, %eax
+; CHECK-NEXT: shrl $24, %eax
+; CHECK-NEXT: andl $-128, %eax
+; CHECK-NEXT: leal (%rax,%r8,4), %r8d
+; CHECK-NEXT: orl %r10d, %r8d
; CHECK-NEXT: shrl $21, %edi
-; CHECK-NEXT: movl %edi, %r8d
-; CHECK-NEXT: andl $1, %r8d
-; CHECK-NEXT: xorl %r9d, %r9d
+; CHECK-NEXT: movl %edi, %r9d
+; CHECK-NEXT: andl $1, %r9d
+; CHECK-NEXT: xorl %r10d, %r10d
; CHECK-NEXT: testl $1048575, %esi # imm = 0xFFFFF
-; CHECK-NEXT: setne %r9b
-; CHECK-NEXT: orl %r8d, %r9d
+; CHECK-NEXT: setne %r10b
+; CHECK-NEXT: orl %r9d, %r10d
; CHECK-NEXT: shrl $20, %esi
-; CHECK-NEXT: andl %r9d, %esi
+; CHECK-NEXT: andl %r10d, %esi
; CHECK-NEXT: addl %edi, %esi
; CHECK-NEXT: xorl %edi, %edi
; CHECK-NEXT: cmpl $4, %esi
@@ -2254,40 +2250,37 @@ define i8 @to_f8e5m2_from_bf16(bfloat %x) {
; CHECK-NEXT: setge %dil
; CHECK-NEXT: leal (%rdx,%rdi), %ecx
; CHECK-NEXT: leal 56(,%rcx,4), %ecx
-; CHECK-NEXT: movl %ebx, %r8d
-; CHECK-NEXT: orl %esi, %r8d
-; CHECK-NEXT: orl %ecx, %r8d
+; CHECK-NEXT: movl %eax, %r9d
+; CHECK-NEXT: orl %esi, %r9d
+; CHECK-NEXT: orl %ecx, %r9d
; CHECK-NEXT: leal 14(%rdx,%rdi), %ecx
; CHECK-NEXT: testl %ecx, %ecx
-; CHECK-NEXT: cmovlel %eax, %r8d
+; CHECK-NEXT: cmovlel %r8d, %r9d
; CHECK-NEXT: cmpl $4, %esi
-; CHECK-NEXT: setge %al
+; CHECK-NEXT: setge %dl
; CHECK-NEXT: cmpl $30, %ecx
-; CHECK-NEXT: sete %dl
-; CHECK-NEXT: andb %al, %dl
+; CHECK-NEXT: sete %sil
+; CHECK-NEXT: andb %dl, %sil
; CHECK-NEXT: cmpl $31, %ecx
-; CHECK-NEXT: setge %al
-; CHECK-NEXT: orb %dl, %al
-; CHECK-NEXT: leal 124(%rbx), %ecx
-; CHECK-NEXT: testb %al, %al
-; CHECK-NEXT: cmovnel %ecx, %r8d
+; CHECK-NEXT: setge %cl
+; CHECK-NEXT: orb %sil, %cl
+; CHECK-NEXT: leal 124(%rax), %edx
+; CHECK-NEXT: testb %cl, %cl
+; CHECK-NEXT: cmovnel %edx, %r9d
; CHECK-NEXT: pxor %xmm0, %xmm0
-; CHECK-NEXT: movaps {{[-0-9]+}}(%r{{[sb]}}p), %xmm2 # 16-byte Reload
; CHECK-NEXT: ucomiss %xmm0, %xmm2
-; CHECK-NEXT: cmovnel %r8d, %ebx
-; CHECK-NEXT: cmovpl %r8d, %ebx
+; CHECK-NEXT: cmovnel %r9d, %eax
+; CHECK-NEXT: cmovpl %r9d, %eax
; CHECK-NEXT: movaps {{.*#+}} xmm1 = [NaN,NaN,NaN,NaN]
; CHECK-NEXT: andps %xmm2, %xmm1
; CHECK-NEXT: ucomiss {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm1
-; CHECK-NEXT: cmovnel %ebx, %ecx
-; CHECK-NEXT: cmovpl %ebx, %ecx
+; CHECK-NEXT: cmovnel %eax, %edx
+; CHECK-NEXT: cmovpl %eax, %edx
; CHECK-NEXT: ucomiss %xmm0, %xmm2
; CHECK-NEXT: movl $126, %eax
-; CHECK-NEXT: cmovnpl %ecx, %eax
+; CHECK-NEXT: cmovnpl %edx, %eax
; CHECK-NEXT: # kill: def $al killed $al killed $eax
-; CHECK-NEXT: addq $32, %rsp
-; CHECK-NEXT: .cfi_def_cfa_offset 16
-; CHECK-NEXT: popq %rbx
+; CHECK-NEXT: addq $40, %rsp
; CHECK-NEXT: .cfi_def_cfa_offset 8
; CHECK-NEXT: retq
%r = call i8 @llvm.convert.to.arbitrary.fp.i8.bf16(bfloat %x, metadata !"Float8E5M2", metadata !"round.tonearest", i1 false)
diff --git a/llvm/test/CodeGen/X86/fminimum-fmaximum.ll b/llvm/test/CodeGen/X86/fminimum-fmaximum.ll
index d96df9e30030e..953ada795465a 100644
--- a/llvm/test/CodeGen/X86/fminimum-fmaximum.ll
+++ b/llvm/test/CodeGen/X86/fminimum-fmaximum.ll
@@ -2212,22 +2212,19 @@ define bfloat @test_fmaximum_bf16(bfloat %x, bfloat %y) nounwind {
; SSE2-LABEL: test_fmaximum_bf16:
; SSE2: # %bb.0:
; SSE2-NEXT: pushq %rax
-; SSE2-NEXT: pextrw $0, %xmm1, %eax
-; SSE2-NEXT: pextrw $0, %xmm0, %ecx
-; SSE2-NEXT: shll $16, %ecx
-; SSE2-NEXT: movd %ecx, %xmm1
-; SSE2-NEXT: movdqa %xmm1, %xmm0
-; SSE2-NEXT: cmpunordss %xmm1, %xmm0
-; SSE2-NEXT: movaps %xmm0, %xmm2
-; SSE2-NEXT: andps %xmm1, %xmm2
-; SSE2-NEXT: shll $16, %eax
-; SSE2-NEXT: movd %eax, %xmm3
-; SSE2-NEXT: movaps %xmm1, %xmm4
-; SSE2-NEXT: maxss %xmm3, %xmm4
-; SSE2-NEXT: orps {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm1
-; SSE2-NEXT: andps %xmm4, %xmm1
-; SSE2-NEXT: andnps %xmm1, %xmm0
-; SSE2-NEXT: orps %xmm2, %xmm0
+; SSE2-NEXT: pslld $16, %xmm0
+; SSE2-NEXT: movdqa %xmm0, %xmm2
+; SSE2-NEXT: cmpunordss %xmm0, %xmm2
+; SSE2-NEXT: movaps %xmm2, %xmm3
+; SSE2-NEXT: andps %xmm0, %xmm3
+; SSE2-NEXT: pslld $16, %xmm1
+; SSE2-NEXT: movaps %xmm0, %xmm4
+; SSE2-NEXT: maxss %xmm1, %xmm4
+; SSE2-NEXT: orps {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm0
+; SSE2-NEXT: andps %xmm4, %xmm0
+; SSE2-NEXT: andnps %xmm0, %xmm2
+; SSE2-NEXT: orps %xmm3, %xmm2
+; SSE2-NEXT: movaps %xmm2, %xmm0
; SSE2-NEXT: callq __truncsfbf2 at PLT
; SSE2-NEXT: popq %rax
; SSE2-NEXT: retq
@@ -2235,17 +2232,13 @@ define bfloat @test_fmaximum_bf16(bfloat %x, bfloat %y) nounwind {
; AVX1-LABEL: test_fmaximum_bf16:
; AVX1: # %bb.0:
; AVX1-NEXT: pushq %rax
-; AVX1-NEXT: vpextrw $0, %xmm0, %eax
-; AVX1-NEXT: vpextrw $0, %xmm1, %ecx
-; AVX1-NEXT: shll $16, %ecx
-; AVX1-NEXT: vmovd %ecx, %xmm0
-; AVX1-NEXT: shll $16, %eax
-; AVX1-NEXT: vmovd %eax, %xmm1
-; AVX1-NEXT: vmaxss %xmm0, %xmm1, %xmm0
-; AVX1-NEXT: vorps {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm1, %xmm2
-; AVX1-NEXT: vandps %xmm0, %xmm2, %xmm0
-; AVX1-NEXT: vcmpunordss %xmm1, %xmm1, %xmm2
-; AVX1-NEXT: vblendvps %xmm2, %xmm1, %xmm0, %xmm0
+; AVX1-NEXT: vpslld $16, %xmm1, %xmm1
+; AVX1-NEXT: vpslld $16, %xmm0, %xmm0
+; AVX1-NEXT: vmaxss %xmm1, %xmm0, %xmm1
+; AVX1-NEXT: vorps {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm0, %xmm2
+; AVX1-NEXT: vandps %xmm1, %xmm2, %xmm1
+; AVX1-NEXT: vcmpunordss %xmm0, %xmm0, %xmm2
+; AVX1-NEXT: vblendvps %xmm2, %xmm0, %xmm1, %xmm0
; AVX1-NEXT: callq __truncsfbf2 at PLT
; AVX1-NEXT: popq %rax
; AVX1-NEXT: retq
@@ -2253,18 +2246,14 @@ define bfloat @test_fmaximum_bf16(bfloat %x, bfloat %y) nounwind {
; AVX512F-LABEL: test_fmaximum_bf16:
; AVX512F: # %bb.0:
; AVX512F-NEXT: pushq %rax
-; AVX512F-NEXT: vpextrw $0, %xmm0, %eax
-; AVX512F-NEXT: vpextrw $0, %xmm1, %ecx
-; AVX512F-NEXT: shll $16, %ecx
-; AVX512F-NEXT: vmovd %ecx, %xmm0
-; AVX512F-NEXT: shll $16, %eax
-; AVX512F-NEXT: vmovd %eax, %xmm1
-; AVX512F-NEXT: vmaxss %xmm0, %xmm1, %xmm0
-; AVX512F-NEXT: vbroadcastss {{.*#+}} xmm2 = [NaN,NaN,NaN,NaN]
-; AVX512F-NEXT: vorps %xmm2, %xmm1, %xmm2
-; AVX512F-NEXT: vandps %xmm0, %xmm2, %xmm0
-; AVX512F-NEXT: vcmpunordss %xmm1, %xmm1, %k1
-; AVX512F-NEXT: vmovss %xmm1, %xmm0, %xmm0 {%k1}
+; AVX512F-NEXT: vpslld $16, %xmm0, %xmm2
+; AVX512F-NEXT: vpbroadcastd {{.*#+}} xmm0 = [NaN,NaN,NaN,NaN]
+; AVX512F-NEXT: vpor %xmm0, %xmm2, %xmm0
+; AVX512F-NEXT: vpslld $16, %xmm1, %xmm1
+; AVX512F-NEXT: vmaxss %xmm1, %xmm2, %xmm1
+; AVX512F-NEXT: vpand %xmm1, %xmm0, %xmm0
+; AVX512F-NEXT: vcmpunordss %xmm2, %xmm2, %k1
+; AVX512F-NEXT: vmovss %xmm2, %xmm0, %xmm0 {%k1}
; AVX512F-NEXT: callq __truncsfbf2 at PLT
; AVX512F-NEXT: popq %rax
; AVX512F-NEXT: retq
@@ -2272,46 +2261,34 @@ define bfloat @test_fmaximum_bf16(bfloat %x, bfloat %y) nounwind {
; AVX512DQ-LABEL: test_fmaximum_bf16:
; AVX512DQ: # %bb.0:
; AVX512DQ-NEXT: pushq %rax
-; AVX512DQ-NEXT: vpextrw $0, %xmm0, %eax
-; AVX512DQ-NEXT: vpextrw $0, %xmm1, %ecx
-; AVX512DQ-NEXT: shll $16, %ecx
-; AVX512DQ-NEXT: vmovd %ecx, %xmm0
-; AVX512DQ-NEXT: shll $16, %eax
-; AVX512DQ-NEXT: vmovd %eax, %xmm1
-; AVX512DQ-NEXT: vmaxss %xmm0, %xmm1, %xmm0
-; AVX512DQ-NEXT: vbroadcastss {{.*#+}} xmm2 = [NaN,NaN,NaN,NaN]
-; AVX512DQ-NEXT: vorps %xmm2, %xmm1, %xmm2
-; AVX512DQ-NEXT: vandps %xmm0, %xmm2, %xmm0
-; AVX512DQ-NEXT: vcmpunordss %xmm1, %xmm1, %k1
-; AVX512DQ-NEXT: vmovss %xmm1, %xmm0, %xmm0 {%k1}
+; AVX512DQ-NEXT: vpslld $16, %xmm0, %xmm2
+; AVX512DQ-NEXT: vpbroadcastd {{.*#+}} xmm0 = [NaN,NaN,NaN,NaN]
+; AVX512DQ-NEXT: vpor %xmm0, %xmm2, %xmm0
+; AVX512DQ-NEXT: vpslld $16, %xmm1, %xmm1
+; AVX512DQ-NEXT: vmaxss %xmm1, %xmm2, %xmm1
+; AVX512DQ-NEXT: vpand %xmm1, %xmm0, %xmm0
+; AVX512DQ-NEXT: vcmpunordss %xmm2, %xmm2, %k1
+; AVX512DQ-NEXT: vmovss %xmm2, %xmm0, %xmm0 {%k1}
; AVX512DQ-NEXT: callq __truncsfbf2 at PLT
; AVX512DQ-NEXT: popq %rax
; AVX512DQ-NEXT: retq
;
; AVX512BF16-LABEL: test_fmaximum_bf16:
; AVX512BF16: # %bb.0:
-; AVX512BF16-NEXT: vpextrw $0, %xmm0, %eax
-; AVX512BF16-NEXT: vpextrw $0, %xmm1, %ecx
-; AVX512BF16-NEXT: shll $16, %ecx
-; AVX512BF16-NEXT: vmovd %ecx, %xmm0
-; AVX512BF16-NEXT: shll $16, %eax
-; AVX512BF16-NEXT: vmovd %eax, %xmm1
-; AVX512BF16-NEXT: vmaxss %xmm0, %xmm1, %xmm0
-; AVX512BF16-NEXT: vpternlogd {{.*#+}} xmm0 = xmm0 & (xmm1 | m32bcst)
-; AVX512BF16-NEXT: vcmpunordss %xmm1, %xmm1, %k1
-; AVX512BF16-NEXT: vmovss %xmm1, %xmm0, %xmm0 {%k1}
-; AVX512BF16-NEXT: vcvtneps2bf16 %xmm0, %xmm0
+; AVX512BF16-NEXT: vpslld $16, %xmm1, %xmm1
+; AVX512BF16-NEXT: vpslld $16, %xmm0, %xmm0
+; AVX512BF16-NEXT: vmaxss %xmm1, %xmm0, %xmm1
+; AVX512BF16-NEXT: vpternlogd {{.*#+}} xmm1 = xmm1 & (xmm0 | m32bcst)
+; AVX512BF16-NEXT: vcmpunordss %xmm0, %xmm0, %k1
+; AVX512BF16-NEXT: vmovss %xmm0, %xmm1, %xmm1 {%k1}
+; AVX512BF16-NEXT: vcvtneps2bf16 %xmm1, %xmm0
; AVX512BF16-NEXT: retq
;
; AVX10_2-LABEL: test_fmaximum_bf16:
; AVX10_2: # %bb.0:
-; AVX10_2-NEXT: vmovw %xmm0, %eax
-; AVX10_2-NEXT: vmovw %xmm1, %ecx
-; AVX10_2-NEXT: shll $16, %ecx
-; AVX10_2-NEXT: vmovd %ecx, %xmm0
-; AVX10_2-NEXT: shll $16, %eax
-; AVX10_2-NEXT: vmovd %eax, %xmm1
-; AVX10_2-NEXT: vminmaxss $5, %xmm0, %xmm1
+; AVX10_2-NEXT: vpslld $16, %xmm1, %xmm1
+; AVX10_2-NEXT: vpslld $16, %xmm0, %xmm0
+; AVX10_2-NEXT: vminmaxss $5, %xmm1, %xmm0
; AVX10_2-NEXT: vcvtneps2bf16 %xmm0, %xmm0
; AVX10_2-NEXT: retq
;
@@ -2340,306 +2317,231 @@ define bfloat @test_fmaximum_bf16(bfloat %x, bfloat %y) nounwind {
define <4 x bfloat> @test_fmaximum_v4bf16(<4 x bfloat> %x, <4 x bfloat> %y) {
; SSE2-LABEL: test_fmaximum_v4bf16:
; SSE2: # %bb.0:
-; SSE2-NEXT: pushq %rbp
-; SSE2-NEXT: .cfi_def_cfa_offset 16
-; SSE2-NEXT: pushq %r15
-; SSE2-NEXT: .cfi_def_cfa_offset 24
-; SSE2-NEXT: pushq %r14
-; SSE2-NEXT: .cfi_def_cfa_offset 32
-; SSE2-NEXT: pushq %r13
-; SSE2-NEXT: .cfi_def_cfa_offset 40
-; SSE2-NEXT: pushq %r12
-; SSE2-NEXT: .cfi_def_cfa_offset 48
-; SSE2-NEXT: pushq %rbx
-; SSE2-NEXT: .cfi_def_cfa_offset 56
-; SSE2-NEXT: subq $56, %rsp
-; SSE2-NEXT: .cfi_def_cfa_offset 112
-; SSE2-NEXT: .cfi_offset %rbx, -56
-; SSE2-NEXT: .cfi_offset %r12, -48
-; SSE2-NEXT: .cfi_offset %r13, -40
-; SSE2-NEXT: .cfi_offset %r14, -32
-; SSE2-NEXT: .cfi_offset %r15, -24
-; SSE2-NEXT: .cfi_offset %rbp, -16
+; SSE2-NEXT: subq $120, %rsp
+; SSE2-NEXT: .cfi_def_cfa_offset 128
+; SSE2-NEXT: movdqa %xmm1, (%rsp) # 16-byte Spill
+; SSE2-NEXT: movdqa %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
; SSE2-NEXT: movdqa %xmm1, %xmm2
; SSE2-NEXT: psrlq $48, %xmm2
-; SSE2-NEXT: pextrw $0, %xmm2, %ebx
+; SSE2-NEXT: movdqa %xmm2, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
; SSE2-NEXT: movdqa %xmm0, %xmm2
; SSE2-NEXT: psrlq $48, %xmm2
-; SSE2-NEXT: pextrw $0, %xmm2, %ebp
+; SSE2-NEXT: movdqa %xmm2, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
; SSE2-NEXT: movdqa %xmm1, %xmm2
; SSE2-NEXT: shufps {{.*#+}} xmm2 = xmm2[1,1],xmm1[1,1]
-; SSE2-NEXT: pextrw $0, %xmm2, %r14d
+; SSE2-NEXT: movaps %xmm2, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
; SSE2-NEXT: movdqa %xmm0, %xmm2
; SSE2-NEXT: shufps {{.*#+}} xmm2 = xmm2[1,1],xmm0[1,1]
-; SSE2-NEXT: pextrw $0, %xmm2, %r15d
-; SSE2-NEXT: pextrw $0, %xmm1, %r12d
-; SSE2-NEXT: pextrw $0, %xmm0, %r13d
+; SSE2-NEXT: movaps %xmm2, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
+; SSE2-NEXT: movdqa %xmm1, %xmm3
+; SSE2-NEXT: psrld $16, %xmm3
+; SSE2-NEXT: movdqa %xmm0, %xmm1
; SSE2-NEXT: psrld $16, %xmm1
-; SSE2-NEXT: pextrw $0, %xmm1, %eax
-; SSE2-NEXT: psrld $16, %xmm0
-; SSE2-NEXT: pextrw $0, %xmm0, %ecx
-; SSE2-NEXT: shll $16, %ecx
-; SSE2-NEXT: movd %ecx, %xmm1
+; SSE2-NEXT: pslld $16, %xmm1
+; SSE2-NEXT: pslld $16, %xmm3
+; SSE2-NEXT: movdqa %xmm1, %xmm2
+; SSE2-NEXT: maxss %xmm3, %xmm2
; SSE2-NEXT: movdqa %xmm1, %xmm0
; SSE2-NEXT: cmpunordss %xmm1, %xmm0
-; SSE2-NEXT: movaps %xmm0, %xmm2
-; SSE2-NEXT: andps %xmm1, %xmm2
-; SSE2-NEXT: shll $16, %eax
-; SSE2-NEXT: movd %eax, %xmm3
-; SSE2-NEXT: movaps %xmm1, %xmm4
-; SSE2-NEXT: maxss %xmm3, %xmm4
+; SSE2-NEXT: movaps %xmm0, %xmm3
+; SSE2-NEXT: andps %xmm1, %xmm0
; SSE2-NEXT: orps {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm1
-; SSE2-NEXT: andps %xmm4, %xmm1
-; SSE2-NEXT: andnps %xmm1, %xmm0
-; SSE2-NEXT: orps %xmm2, %xmm0
+; SSE2-NEXT: andps %xmm1, %xmm2
+; SSE2-NEXT: andnps %xmm2, %xmm3
+; SSE2-NEXT: orps %xmm3, %xmm0
; SSE2-NEXT: callq __truncsfbf2 at PLT
; SSE2-NEXT: movaps %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
-; SSE2-NEXT: shll $16, %r13d
-; SSE2-NEXT: movd %r13d, %xmm1
-; SSE2-NEXT: movdqa %xmm1, %xmm0
-; SSE2-NEXT: cmpunordss %xmm1, %xmm0
+; SSE2-NEXT: movdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm2 # 16-byte Reload
+; SSE2-NEXT: pslld $16, %xmm2
+; SSE2-NEXT: movdqa %xmm2, %xmm0
+; SSE2-NEXT: por {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm0
+; SSE2-NEXT: movdqa (%rsp), %xmm3 # 16-byte Reload
+; SSE2-NEXT: pslld $16, %xmm3
+; SSE2-NEXT: movdqa %xmm2, %xmm1
+; SSE2-NEXT: maxss %xmm3, %xmm1
+; SSE2-NEXT: andps %xmm0, %xmm1
+; SSE2-NEXT: movdqa %xmm2, %xmm0
+; SSE2-NEXT: movdqa %xmm2, %xmm3
+; SSE2-NEXT: cmpunordss %xmm2, %xmm0
; SSE2-NEXT: movaps %xmm0, %xmm2
-; SSE2-NEXT: andps %xmm1, %xmm2
-; SSE2-NEXT: shll $16, %r12d
-; SSE2-NEXT: movd %r12d, %xmm3
-; SSE2-NEXT: movaps %xmm1, %xmm4
-; SSE2-NEXT: maxss %xmm3, %xmm4
-; SSE2-NEXT: orps {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm1
-; SSE2-NEXT: andps %xmm4, %xmm1
-; SSE2-NEXT: andnps %xmm1, %xmm0
+; SSE2-NEXT: andnps %xmm1, %xmm2
+; SSE2-NEXT: andps %xmm3, %xmm0
; SSE2-NEXT: orps %xmm2, %xmm0
; SSE2-NEXT: callq __truncsfbf2 at PLT
-; SSE2-NEXT: movaps %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
-; SSE2-NEXT: shll $16, %r15d
-; SSE2-NEXT: movd %r15d, %xmm1
-; SSE2-NEXT: movdqa %xmm1, %xmm0
-; SSE2-NEXT: cmpunordss %xmm1, %xmm0
+; SSE2-NEXT: movaps %xmm0, (%rsp) # 16-byte Spill
+; SSE2-NEXT: movdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm2 # 16-byte Reload
+; SSE2-NEXT: pslld $16, %xmm2
+; SSE2-NEXT: movdqa %xmm2, %xmm0
+; SSE2-NEXT: por {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm0
+; SSE2-NEXT: movdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm3 # 16-byte Reload
+; SSE2-NEXT: pslld $16, %xmm3
+; SSE2-NEXT: movdqa %xmm2, %xmm1
+; SSE2-NEXT: maxss %xmm3, %xmm1
+; SSE2-NEXT: andps %xmm0, %xmm1
+; SSE2-NEXT: movdqa %xmm2, %xmm0
+; SSE2-NEXT: movdqa %xmm2, %xmm3
+; SSE2-NEXT: cmpunordss %xmm2, %xmm0
; SSE2-NEXT: movaps %xmm0, %xmm2
-; SSE2-NEXT: andps %xmm1, %xmm2
-; SSE2-NEXT: shll $16, %r14d
-; SSE2-NEXT: movd %r14d, %xmm3
-; SSE2-NEXT: movaps %xmm1, %xmm4
-; SSE2-NEXT: maxss %xmm3, %xmm4
-; SSE2-NEXT: orps {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm1
-; SSE2-NEXT: andps %xmm4, %xmm1
-; SSE2-NEXT: andnps %xmm1, %xmm0
+; SSE2-NEXT: andnps %xmm1, %xmm2
+; SSE2-NEXT: andps %xmm3, %xmm0
; SSE2-NEXT: orps %xmm2, %xmm0
; SSE2-NEXT: callq __truncsfbf2 at PLT
-; SSE2-NEXT: movaps %xmm0, (%rsp) # 16-byte Spill
-; SSE2-NEXT: shll $16, %ebp
-; SSE2-NEXT: movd %ebp, %xmm1
-; SSE2-NEXT: movdqa %xmm1, %xmm0
-; SSE2-NEXT: cmpunordss %xmm1, %xmm0
+; SSE2-NEXT: movaps %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
+; SSE2-NEXT: movdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm2 # 16-byte Reload
+; SSE2-NEXT: pslld $16, %xmm2
+; SSE2-NEXT: movdqa {{.*#+}} xmm3 = [NaN,NaN,NaN,NaN]
+; SSE2-NEXT: por %xmm2, %xmm3
+; SSE2-NEXT: movdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 16-byte Reload
+; SSE2-NEXT: pslld $16, %xmm0
+; SSE2-NEXT: movdqa %xmm2, %xmm1
+; SSE2-NEXT: maxss %xmm0, %xmm1
+; SSE2-NEXT: andps %xmm3, %xmm1
+; SSE2-NEXT: movdqa %xmm2, %xmm0
+; SSE2-NEXT: movdqa %xmm2, %xmm3
+; SSE2-NEXT: cmpunordss %xmm2, %xmm0
; SSE2-NEXT: movaps %xmm0, %xmm2
-; SSE2-NEXT: andps %xmm1, %xmm2
-; SSE2-NEXT: shll $16, %ebx
-; SSE2-NEXT: movd %ebx, %xmm3
-; SSE2-NEXT: movaps %xmm1, %xmm4
-; SSE2-NEXT: maxss %xmm3, %xmm4
-; SSE2-NEXT: orps {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm1
-; SSE2-NEXT: andps %xmm4, %xmm1
-; SSE2-NEXT: andnps %xmm1, %xmm0
+; SSE2-NEXT: andnps %xmm1, %xmm2
+; SSE2-NEXT: andps %xmm3, %xmm0
; SSE2-NEXT: orps %xmm2, %xmm0
; SSE2-NEXT: callq __truncsfbf2 at PLT
-; SSE2-NEXT: movdqa (%rsp), %xmm1 # 16-byte Reload
+; SSE2-NEXT: movdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm1 # 16-byte Reload
; SSE2-NEXT: punpcklwd {{.*#+}} xmm1 = xmm1[0],xmm0[0],xmm1[1],xmm0[1],xmm1[2],xmm0[2],xmm1[3],xmm0[3]
-; SSE2-NEXT: movdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 16-byte Reload
+; SSE2-NEXT: movdqa (%rsp), %xmm0 # 16-byte Reload
; SSE2-NEXT: punpcklwd {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 16-byte Folded Reload
; SSE2-NEXT: # xmm0 = xmm0[0],mem[0],xmm0[1],mem[1],xmm0[2],mem[2],xmm0[3],mem[3]
; SSE2-NEXT: punpckldq {{.*#+}} xmm0 = xmm0[0],xmm1[0],xmm0[1],xmm1[1]
-; SSE2-NEXT: addq $56, %rsp
-; SSE2-NEXT: .cfi_def_cfa_offset 56
-; SSE2-NEXT: popq %rbx
-; SSE2-NEXT: .cfi_def_cfa_offset 48
-; SSE2-NEXT: popq %r12
-; SSE2-NEXT: .cfi_def_cfa_offset 40
-; SSE2-NEXT: popq %r13
-; SSE2-NEXT: .cfi_def_cfa_offset 32
-; SSE2-NEXT: popq %r14
-; SSE2-NEXT: .cfi_def_cfa_offset 24
-; SSE2-NEXT: popq %r15
-; SSE2-NEXT: .cfi_def_cfa_offset 16
-; SSE2-NEXT: popq %rbp
+; SSE2-NEXT: addq $120, %rsp
; SSE2-NEXT: .cfi_def_cfa_offset 8
; SSE2-NEXT: retq
;
; AVX1-LABEL: test_fmaximum_v4bf16:
; AVX1: # %bb.0:
-; AVX1-NEXT: pushq %rbp
-; AVX1-NEXT: .cfi_def_cfa_offset 16
-; AVX1-NEXT: pushq %r15
-; AVX1-NEXT: .cfi_def_cfa_offset 24
-; AVX1-NEXT: pushq %r14
-; AVX1-NEXT: .cfi_def_cfa_offset 32
-; AVX1-NEXT: pushq %r13
-; AVX1-NEXT: .cfi_def_cfa_offset 40
-; AVX1-NEXT: pushq %r12
-; AVX1-NEXT: .cfi_def_cfa_offset 48
-; AVX1-NEXT: pushq %rbx
-; AVX1-NEXT: .cfi_def_cfa_offset 56
-; AVX1-NEXT: subq $56, %rsp
-; AVX1-NEXT: .cfi_def_cfa_offset 112
-; AVX1-NEXT: .cfi_offset %rbx, -56
-; AVX1-NEXT: .cfi_offset %r12, -48
-; AVX1-NEXT: .cfi_offset %r13, -40
-; AVX1-NEXT: .cfi_offset %r14, -32
-; AVX1-NEXT: .cfi_offset %r15, -24
-; AVX1-NEXT: .cfi_offset %rbp, -16
-; AVX1-NEXT: vpsrlq $48, %xmm0, %xmm2
-; AVX1-NEXT: vpextrw $0, %xmm2, %ebx
+; AVX1-NEXT: subq $88, %rsp
+; AVX1-NEXT: .cfi_def_cfa_offset 96
+; AVX1-NEXT: vmovdqa %xmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
+; AVX1-NEXT: vmovdqa %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
; AVX1-NEXT: vpsrlq $48, %xmm1, %xmm2
-; AVX1-NEXT: vpextrw $0, %xmm2, %ebp
-; AVX1-NEXT: vmovshdup {{.*#+}} xmm2 = xmm0[1,1,3,3]
-; AVX1-NEXT: vpextrw $0, %xmm2, %r14d
-; AVX1-NEXT: vmovshdup {{.*#+}} xmm2 = xmm1[1,1,3,3]
-; AVX1-NEXT: vpextrw $0, %xmm2, %r15d
-; AVX1-NEXT: vpextrw $0, %xmm0, %r12d
-; AVX1-NEXT: vpextrw $0, %xmm1, %r13d
-; AVX1-NEXT: vpsrld $16, %xmm0, %xmm0
-; AVX1-NEXT: vpextrw $0, %xmm0, %eax
-; AVX1-NEXT: vpsrld $16, %xmm1, %xmm0
-; AVX1-NEXT: vpextrw $0, %xmm0, %ecx
-; AVX1-NEXT: shll $16, %ecx
-; AVX1-NEXT: vmovd %ecx, %xmm0
-; AVX1-NEXT: shll $16, %eax
-; AVX1-NEXT: vmovd %eax, %xmm1
+; AVX1-NEXT: vmovdqa %xmm2, (%rsp) # 16-byte Spill
+; AVX1-NEXT: vmovshdup {{.*#+}} xmm3 = xmm1[1,1,3,3]
+; AVX1-NEXT: vpshufb {{.*#+}} xmm1 = zero,zero,xmm0[4,5],zero,zero,xmm0[u,u],zero,zero,xmm0[u,u],zero,zero,xmm0[u,u]
+; AVX1-NEXT: vpor {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm1, %xmm2
+; AVX1-NEXT: vpslld $16, %xmm3, %xmm0
; AVX1-NEXT: vmaxss %xmm0, %xmm1, %xmm0
-; AVX1-NEXT: vorps {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm1, %xmm2
-; AVX1-NEXT: vandps %xmm0, %xmm2, %xmm0
+; AVX1-NEXT: vpand %xmm0, %xmm2, %xmm0
; AVX1-NEXT: vcmpunordss %xmm1, %xmm1, %xmm2
; AVX1-NEXT: vblendvps %xmm2, %xmm1, %xmm0, %xmm0
; AVX1-NEXT: callq __truncsfbf2 at PLT
; AVX1-NEXT: vmovaps %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
-; AVX1-NEXT: shll $16, %r13d
-; AVX1-NEXT: vmovd %r13d, %xmm0
-; AVX1-NEXT: shll $16, %r12d
-; AVX1-NEXT: vmovd %r12d, %xmm1
-; AVX1-NEXT: vmaxss %xmm0, %xmm1, %xmm0
-; AVX1-NEXT: vorps {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm1, %xmm2
-; AVX1-NEXT: vandps %xmm0, %xmm2, %xmm0
-; AVX1-NEXT: vcmpunordss %xmm1, %xmm1, %xmm2
-; AVX1-NEXT: vblendvps %xmm2, %xmm1, %xmm0, %xmm0
-; AVX1-NEXT: callq __truncsfbf2 at PLT
-; AVX1-NEXT: vmovaps %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
-; AVX1-NEXT: shll $16, %r15d
-; AVX1-NEXT: vmovd %r15d, %xmm0
-; AVX1-NEXT: shll $16, %r14d
-; AVX1-NEXT: vmovd %r14d, %xmm1
-; AVX1-NEXT: vmaxss %xmm0, %xmm1, %xmm0
-; AVX1-NEXT: vorps {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm1, %xmm2
-; AVX1-NEXT: vandps %xmm0, %xmm2, %xmm0
-; AVX1-NEXT: vcmpunordss %xmm1, %xmm1, %xmm2
-; AVX1-NEXT: vblendvps %xmm2, %xmm1, %xmm0, %xmm0
+; AVX1-NEXT: vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 16-byte Reload
+; AVX1-NEXT: vpshufb {{.*#+}} xmm0 = zero,zero,xmm0[6,7],zero,zero,xmm0[u,u],zero,zero,xmm0[u,u],zero,zero,xmm0[u,u]
+; AVX1-NEXT: vpor {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm0, %xmm1
+; AVX1-NEXT: vmovdqa (%rsp), %xmm2 # 16-byte Reload
+; AVX1-NEXT: vpslld $16, %xmm2, %xmm2
+; AVX1-NEXT: vmaxss %xmm2, %xmm0, %xmm2
+; AVX1-NEXT: vpand %xmm2, %xmm1, %xmm1
+; AVX1-NEXT: vcmpunordss %xmm0, %xmm0, %xmm2
+; AVX1-NEXT: vblendvps %xmm2, %xmm0, %xmm1, %xmm0
; AVX1-NEXT: callq __truncsfbf2 at PLT
; AVX1-NEXT: vmovaps %xmm0, (%rsp) # 16-byte Spill
-; AVX1-NEXT: shll $16, %ebp
-; AVX1-NEXT: vmovd %ebp, %xmm0
-; AVX1-NEXT: shll $16, %ebx
-; AVX1-NEXT: vmovd %ebx, %xmm1
-; AVX1-NEXT: vmaxss %xmm0, %xmm1, %xmm0
-; AVX1-NEXT: vorps {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm1, %xmm2
-; AVX1-NEXT: vandps %xmm0, %xmm2, %xmm0
-; AVX1-NEXT: vcmpunordss %xmm1, %xmm1, %xmm2
-; AVX1-NEXT: vblendvps %xmm2, %xmm1, %xmm0, %xmm0
+; AVX1-NEXT: vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 16-byte Reload
+; AVX1-NEXT: vpslld $16, %xmm0, %xmm0
+; AVX1-NEXT: vpor {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm0, %xmm1
+; AVX1-NEXT: vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm2 # 16-byte Reload
+; AVX1-NEXT: vpslld $16, %xmm2, %xmm2
+; AVX1-NEXT: vmaxss %xmm2, %xmm0, %xmm2
+; AVX1-NEXT: vpand %xmm2, %xmm1, %xmm1
+; AVX1-NEXT: vcmpunordss %xmm0, %xmm0, %xmm2
+; AVX1-NEXT: vblendvps %xmm2, %xmm0, %xmm1, %xmm0
+; AVX1-NEXT: callq __truncsfbf2 at PLT
+; AVX1-NEXT: vmovaps %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
+; AVX1-NEXT: vxorps %xmm0, %xmm0, %xmm0
+; AVX1-NEXT: vpblendw $170, {{[-0-9]+}}(%r{{[sb]}}p), %xmm0, %xmm1 # 16-byte Folded Reload
+; AVX1-NEXT: # xmm1 = xmm0[0],mem[1],xmm0[2],mem[3],xmm0[4],mem[5],xmm0[6],mem[7]
+; AVX1-NEXT: vpblendw $170, {{[-0-9]+}}(%r{{[sb]}}p), %xmm0, %xmm0 # 16-byte Folded Reload
+; AVX1-NEXT: # xmm0 = xmm0[0],mem[1],xmm0[2],mem[3],xmm0[4],mem[5],xmm0[6],mem[7]
+; AVX1-NEXT: vmaxss %xmm1, %xmm0, %xmm1
+; AVX1-NEXT: vorps {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm0, %xmm2
+; AVX1-NEXT: vandps %xmm1, %xmm2, %xmm1
+; AVX1-NEXT: vcmpunordss %xmm0, %xmm0, %xmm2
+; AVX1-NEXT: vblendvps %xmm2, %xmm0, %xmm1, %xmm0
; AVX1-NEXT: callq __truncsfbf2 at PLT
-; AVX1-NEXT: vmovdqa (%rsp), %xmm1 # 16-byte Reload
+; AVX1-NEXT: vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm1 # 16-byte Reload
; AVX1-NEXT: vpunpcklwd {{.*#+}} xmm0 = xmm1[0],xmm0[0],xmm1[1],xmm0[1],xmm1[2],xmm0[2],xmm1[3],xmm0[3]
; AVX1-NEXT: vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm1 # 16-byte Reload
-; AVX1-NEXT: vpunpcklwd {{[-0-9]+}}(%r{{[sb]}}p), %xmm1, %xmm1 # 16-byte Folded Reload
+; AVX1-NEXT: vpunpcklwd (%rsp), %xmm1, %xmm1 # 16-byte Folded Reload
; AVX1-NEXT: # xmm1 = xmm1[0],mem[0],xmm1[1],mem[1],xmm1[2],mem[2],xmm1[3],mem[3]
-; AVX1-NEXT: vinsertps {{.*#+}} xmm0 = xmm1[0],xmm0[0],zero,zero
-; AVX1-NEXT: addq $56, %rsp
-; AVX1-NEXT: .cfi_def_cfa_offset 56
-; AVX1-NEXT: popq %rbx
-; AVX1-NEXT: .cfi_def_cfa_offset 48
-; AVX1-NEXT: popq %r12
-; AVX1-NEXT: .cfi_def_cfa_offset 40
-; AVX1-NEXT: popq %r13
-; AVX1-NEXT: .cfi_def_cfa_offset 32
-; AVX1-NEXT: popq %r14
-; AVX1-NEXT: .cfi_def_cfa_offset 24
-; AVX1-NEXT: popq %r15
-; AVX1-NEXT: .cfi_def_cfa_offset 16
-; AVX1-NEXT: popq %rbp
+; AVX1-NEXT: vinsertps {{.*#+}} xmm0 = xmm0[0],xmm1[0],zero,zero
+; AVX1-NEXT: addq $88, %rsp
; AVX1-NEXT: .cfi_def_cfa_offset 8
; AVX1-NEXT: retq
;
; AVX512F-LABEL: test_fmaximum_v4bf16:
; AVX512F: # %bb.0:
-; AVX512F-NEXT: pushq %rbp
-; AVX512F-NEXT: .cfi_def_cfa_offset 16
; AVX512F-NEXT: pushq %r15
-; AVX512F-NEXT: .cfi_def_cfa_offset 24
+; AVX512F-NEXT: .cfi_def_cfa_offset 16
; AVX512F-NEXT: pushq %r14
-; AVX512F-NEXT: .cfi_def_cfa_offset 32
-; AVX512F-NEXT: pushq %r13
-; AVX512F-NEXT: .cfi_def_cfa_offset 40
+; AVX512F-NEXT: .cfi_def_cfa_offset 24
; AVX512F-NEXT: pushq %r12
-; AVX512F-NEXT: .cfi_def_cfa_offset 48
+; AVX512F-NEXT: .cfi_def_cfa_offset 32
; AVX512F-NEXT: pushq %rbx
-; AVX512F-NEXT: .cfi_def_cfa_offset 56
-; AVX512F-NEXT: pushq %rax
-; AVX512F-NEXT: .cfi_def_cfa_offset 64
-; AVX512F-NEXT: .cfi_offset %rbx, -56
-; AVX512F-NEXT: .cfi_offset %r12, -48
-; AVX512F-NEXT: .cfi_offset %r13, -40
-; AVX512F-NEXT: .cfi_offset %r14, -32
-; AVX512F-NEXT: .cfi_offset %r15, -24
-; AVX512F-NEXT: .cfi_offset %rbp, -16
-; AVX512F-NEXT: vmovq %xmm1, %r13
-; AVX512F-NEXT: movq %r13, %rbx
+; AVX512F-NEXT: .cfi_def_cfa_offset 40
+; AVX512F-NEXT: subq $56, %rsp
+; AVX512F-NEXT: .cfi_def_cfa_offset 96
+; AVX512F-NEXT: .cfi_offset %rbx, -40
+; AVX512F-NEXT: .cfi_offset %r12, -32
+; AVX512F-NEXT: .cfi_offset %r14, -24
+; AVX512F-NEXT: .cfi_offset %r15, -16
+; AVX512F-NEXT: vmovdqa %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
+; AVX512F-NEXT: vmovq %xmm1, %r15
+; AVX512F-NEXT: vmovdqa %xmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
+; AVX512F-NEXT: movq %r15, %rbx
; AVX512F-NEXT: shrq $32, %rbx
-; AVX512F-NEXT: vmovq %xmm0, %rbp
-; AVX512F-NEXT: movq %rbp, %r14
+; AVX512F-NEXT: vmovq %xmm0, %r12
+; AVX512F-NEXT: movq %r12, %r14
; AVX512F-NEXT: shrq $32, %r14
-; AVX512F-NEXT: movq %r13, %r15
; AVX512F-NEXT: shrq $48, %r15
-; AVX512F-NEXT: movq %rbp, %r12
; AVX512F-NEXT: shrq $48, %r12
-; AVX512F-NEXT: movl %ebp, %eax
-; AVX512F-NEXT: andl $-65536, %eax # imm = 0xFFFF0000
-; AVX512F-NEXT: vmovd %eax, %xmm1
-; AVX512F-NEXT: vpbroadcastd {{.*#+}} xmm0 = [NaN,NaN,NaN,NaN]
-; AVX512F-NEXT: vpor %xmm0, %xmm1, %xmm0
-; AVX512F-NEXT: movl %r13d, %eax
-; AVX512F-NEXT: andl $-65536, %eax # imm = 0xFFFF0000
-; AVX512F-NEXT: vmovd %eax, %xmm2
-; AVX512F-NEXT: vmaxss %xmm2, %xmm1, %xmm2
-; AVX512F-NEXT: vpand %xmm2, %xmm0, %xmm0
-; AVX512F-NEXT: vcmpunordss %xmm1, %xmm1, %k1
-; AVX512F-NEXT: vmovss %xmm1, %xmm0, %xmm0 {%k1}
+; AVX512F-NEXT: vpxor %xmm3, %xmm3, %xmm3
+; AVX512F-NEXT: vpblendw {{.*#+}} xmm1 = xmm3[0],xmm1[1],xmm3[2],xmm1[3],xmm3[4],xmm1[5],xmm3[6],xmm1[7]
+; AVX512F-NEXT: vpblendw {{.*#+}} xmm2 = xmm3[0],xmm0[1],xmm3[2],xmm0[3],xmm3[4],xmm0[5],xmm3[6],xmm0[7]
+; AVX512F-NEXT: vmaxss %xmm1, %xmm2, %xmm0
+; AVX512F-NEXT: vbroadcastss {{.*#+}} xmm1 = [NaN,NaN,NaN,NaN]
+; AVX512F-NEXT: vorps %xmm1, %xmm2, %xmm1
+; AVX512F-NEXT: vandps %xmm0, %xmm1, %xmm0
+; AVX512F-NEXT: vcmpunordss %xmm2, %xmm2, %k1
+; AVX512F-NEXT: vmovss %xmm2, %xmm0, %xmm0 {%k1}
; AVX512F-NEXT: callq __truncsfbf2 at PLT
; AVX512F-NEXT: vpextrw $0, %xmm0, {{[0-9]+}}(%rsp)
-; AVX512F-NEXT: shll $16, %ebp
-; AVX512F-NEXT: vmovd %ebp, %xmm1
+; AVX512F-NEXT: vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 16-byte Reload
+; AVX512F-NEXT: vpslld $16, %xmm0, %xmm1
; AVX512F-NEXT: vpbroadcastd {{.*#+}} xmm0 = [NaN,NaN,NaN,NaN]
; AVX512F-NEXT: vpor %xmm0, %xmm1, %xmm0
-; AVX512F-NEXT: shll $16, %r13d
-; AVX512F-NEXT: vmovd %r13d, %xmm2
+; AVX512F-NEXT: vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm2 # 16-byte Reload
+; AVX512F-NEXT: vpslld $16, %xmm2, %xmm2
; AVX512F-NEXT: vmaxss %xmm2, %xmm1, %xmm2
; AVX512F-NEXT: vpand %xmm2, %xmm0, %xmm0
; AVX512F-NEXT: vcmpunordss %xmm1, %xmm1, %k1
; AVX512F-NEXT: vmovss %xmm1, %xmm0, %xmm0 {%k1}
; AVX512F-NEXT: callq __truncsfbf2 at PLT
; AVX512F-NEXT: vpextrw $0, %xmm0, (%rsp)
-; AVX512F-NEXT: shll $16, %r12d
-; AVX512F-NEXT: vmovd %r12d, %xmm1
+; AVX512F-NEXT: vmovd %r12d, %xmm0
+; AVX512F-NEXT: vpslld $16, %xmm0, %xmm1
; AVX512F-NEXT: vpbroadcastd {{.*#+}} xmm0 = [NaN,NaN,NaN,NaN]
; AVX512F-NEXT: vpor %xmm0, %xmm1, %xmm0
-; AVX512F-NEXT: shll $16, %r15d
; AVX512F-NEXT: vmovd %r15d, %xmm2
+; AVX512F-NEXT: vpslld $16, %xmm2, %xmm2
; AVX512F-NEXT: vmaxss %xmm2, %xmm1, %xmm2
; AVX512F-NEXT: vpand %xmm2, %xmm0, %xmm0
; AVX512F-NEXT: vcmpunordss %xmm1, %xmm1, %k1
; AVX512F-NEXT: vmovss %xmm1, %xmm0, %xmm0 {%k1}
; AVX512F-NEXT: callq __truncsfbf2 at PLT
; AVX512F-NEXT: vpextrw $0, %xmm0, {{[0-9]+}}(%rsp)
-; AVX512F-NEXT: shll $16, %r14d
-; AVX512F-NEXT: vmovd %r14d, %xmm1
+; AVX512F-NEXT: vmovd %r14d, %xmm0
+; AVX512F-NEXT: vpslld $16, %xmm0, %xmm1
; AVX512F-NEXT: vpbroadcastd {{.*#+}} xmm0 = [NaN,NaN,NaN,NaN]
; AVX512F-NEXT: vpor %xmm0, %xmm1, %xmm0
-; AVX512F-NEXT: shll $16, %ebx
; AVX512F-NEXT: vmovd %ebx, %xmm2
+; AVX512F-NEXT: vpslld $16, %xmm2, %xmm2
; AVX512F-NEXT: vmaxss %xmm2, %xmm1, %xmm2
; AVX512F-NEXT: vpand %xmm2, %xmm0, %xmm0
; AVX512F-NEXT: vcmpunordss %xmm1, %xmm1, %k1
@@ -2647,98 +2549,85 @@ define <4 x bfloat> @test_fmaximum_v4bf16(<4 x bfloat> %x, <4 x bfloat> %y) {
; AVX512F-NEXT: callq __truncsfbf2 at PLT
; AVX512F-NEXT: vpextrw $0, %xmm0, {{[0-9]+}}(%rsp)
; AVX512F-NEXT: vmovaps (%rsp), %xmm0
-; AVX512F-NEXT: addq $8, %rsp
-; AVX512F-NEXT: .cfi_def_cfa_offset 56
-; AVX512F-NEXT: popq %rbx
-; AVX512F-NEXT: .cfi_def_cfa_offset 48
-; AVX512F-NEXT: popq %r12
+; AVX512F-NEXT: addq $56, %rsp
; AVX512F-NEXT: .cfi_def_cfa_offset 40
-; AVX512F-NEXT: popq %r13
+; AVX512F-NEXT: popq %rbx
; AVX512F-NEXT: .cfi_def_cfa_offset 32
-; AVX512F-NEXT: popq %r14
+; AVX512F-NEXT: popq %r12
; AVX512F-NEXT: .cfi_def_cfa_offset 24
-; AVX512F-NEXT: popq %r15
+; AVX512F-NEXT: popq %r14
; AVX512F-NEXT: .cfi_def_cfa_offset 16
-; AVX512F-NEXT: popq %rbp
+; AVX512F-NEXT: popq %r15
; AVX512F-NEXT: .cfi_def_cfa_offset 8
; AVX512F-NEXT: retq
;
; AVX512DQ-LABEL: test_fmaximum_v4bf16:
; AVX512DQ: # %bb.0:
-; AVX512DQ-NEXT: pushq %rbp
-; AVX512DQ-NEXT: .cfi_def_cfa_offset 16
; AVX512DQ-NEXT: pushq %r15
-; AVX512DQ-NEXT: .cfi_def_cfa_offset 24
+; AVX512DQ-NEXT: .cfi_def_cfa_offset 16
; AVX512DQ-NEXT: pushq %r14
-; AVX512DQ-NEXT: .cfi_def_cfa_offset 32
-; AVX512DQ-NEXT: pushq %r13
-; AVX512DQ-NEXT: .cfi_def_cfa_offset 40
+; AVX512DQ-NEXT: .cfi_def_cfa_offset 24
; AVX512DQ-NEXT: pushq %r12
-; AVX512DQ-NEXT: .cfi_def_cfa_offset 48
+; AVX512DQ-NEXT: .cfi_def_cfa_offset 32
; AVX512DQ-NEXT: pushq %rbx
-; AVX512DQ-NEXT: .cfi_def_cfa_offset 56
-; AVX512DQ-NEXT: pushq %rax
-; AVX512DQ-NEXT: .cfi_def_cfa_offset 64
-; AVX512DQ-NEXT: .cfi_offset %rbx, -56
-; AVX512DQ-NEXT: .cfi_offset %r12, -48
-; AVX512DQ-NEXT: .cfi_offset %r13, -40
-; AVX512DQ-NEXT: .cfi_offset %r14, -32
-; AVX512DQ-NEXT: .cfi_offset %r15, -24
-; AVX512DQ-NEXT: .cfi_offset %rbp, -16
-; AVX512DQ-NEXT: vmovq %xmm1, %r13
-; AVX512DQ-NEXT: movq %r13, %rbx
+; AVX512DQ-NEXT: .cfi_def_cfa_offset 40
+; AVX512DQ-NEXT: subq $56, %rsp
+; AVX512DQ-NEXT: .cfi_def_cfa_offset 96
+; AVX512DQ-NEXT: .cfi_offset %rbx, -40
+; AVX512DQ-NEXT: .cfi_offset %r12, -32
+; AVX512DQ-NEXT: .cfi_offset %r14, -24
+; AVX512DQ-NEXT: .cfi_offset %r15, -16
+; AVX512DQ-NEXT: vmovdqa %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
+; AVX512DQ-NEXT: vmovq %xmm1, %r15
+; AVX512DQ-NEXT: vmovdqa %xmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
+; AVX512DQ-NEXT: movq %r15, %rbx
; AVX512DQ-NEXT: shrq $32, %rbx
-; AVX512DQ-NEXT: vmovq %xmm0, %rbp
-; AVX512DQ-NEXT: movq %rbp, %r14
+; AVX512DQ-NEXT: vmovq %xmm0, %r12
+; AVX512DQ-NEXT: movq %r12, %r14
; AVX512DQ-NEXT: shrq $32, %r14
-; AVX512DQ-NEXT: movq %r13, %r15
; AVX512DQ-NEXT: shrq $48, %r15
-; AVX512DQ-NEXT: movq %rbp, %r12
; AVX512DQ-NEXT: shrq $48, %r12
-; AVX512DQ-NEXT: movl %ebp, %eax
-; AVX512DQ-NEXT: andl $-65536, %eax # imm = 0xFFFF0000
-; AVX512DQ-NEXT: vmovd %eax, %xmm1
-; AVX512DQ-NEXT: vpbroadcastd {{.*#+}} xmm0 = [NaN,NaN,NaN,NaN]
-; AVX512DQ-NEXT: vpor %xmm0, %xmm1, %xmm0
-; AVX512DQ-NEXT: movl %r13d, %eax
-; AVX512DQ-NEXT: andl $-65536, %eax # imm = 0xFFFF0000
-; AVX512DQ-NEXT: vmovd %eax, %xmm2
-; AVX512DQ-NEXT: vmaxss %xmm2, %xmm1, %xmm2
-; AVX512DQ-NEXT: vpand %xmm2, %xmm0, %xmm0
-; AVX512DQ-NEXT: vcmpunordss %xmm1, %xmm1, %k1
-; AVX512DQ-NEXT: vmovss %xmm1, %xmm0, %xmm0 {%k1}
+; AVX512DQ-NEXT: vpxor %xmm3, %xmm3, %xmm3
+; AVX512DQ-NEXT: vpblendw {{.*#+}} xmm1 = xmm3[0],xmm1[1],xmm3[2],xmm1[3],xmm3[4],xmm1[5],xmm3[6],xmm1[7]
+; AVX512DQ-NEXT: vpblendw {{.*#+}} xmm2 = xmm3[0],xmm0[1],xmm3[2],xmm0[3],xmm3[4],xmm0[5],xmm3[6],xmm0[7]
+; AVX512DQ-NEXT: vmaxss %xmm1, %xmm2, %xmm0
+; AVX512DQ-NEXT: vbroadcastss {{.*#+}} xmm1 = [NaN,NaN,NaN,NaN]
+; AVX512DQ-NEXT: vorps %xmm1, %xmm2, %xmm1
+; AVX512DQ-NEXT: vandps %xmm0, %xmm1, %xmm0
+; AVX512DQ-NEXT: vcmpunordss %xmm2, %xmm2, %k1
+; AVX512DQ-NEXT: vmovss %xmm2, %xmm0, %xmm0 {%k1}
; AVX512DQ-NEXT: callq __truncsfbf2 at PLT
; AVX512DQ-NEXT: vpextrw $0, %xmm0, {{[0-9]+}}(%rsp)
-; AVX512DQ-NEXT: shll $16, %ebp
-; AVX512DQ-NEXT: vmovd %ebp, %xmm1
+; AVX512DQ-NEXT: vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 16-byte Reload
+; AVX512DQ-NEXT: vpslld $16, %xmm0, %xmm1
; AVX512DQ-NEXT: vpbroadcastd {{.*#+}} xmm0 = [NaN,NaN,NaN,NaN]
; AVX512DQ-NEXT: vpor %xmm0, %xmm1, %xmm0
-; AVX512DQ-NEXT: shll $16, %r13d
-; AVX512DQ-NEXT: vmovd %r13d, %xmm2
+; AVX512DQ-NEXT: vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm2 # 16-byte Reload
+; AVX512DQ-NEXT: vpslld $16, %xmm2, %xmm2
; AVX512DQ-NEXT: vmaxss %xmm2, %xmm1, %xmm2
; AVX512DQ-NEXT: vpand %xmm2, %xmm0, %xmm0
; AVX512DQ-NEXT: vcmpunordss %xmm1, %xmm1, %k1
; AVX512DQ-NEXT: vmovss %xmm1, %xmm0, %xmm0 {%k1}
; AVX512DQ-NEXT: callq __truncsfbf2 at PLT
; AVX512DQ-NEXT: vpextrw $0, %xmm0, (%rsp)
-; AVX512DQ-NEXT: shll $16, %r12d
-; AVX512DQ-NEXT: vmovd %r12d, %xmm1
+; AVX512DQ-NEXT: vmovd %r12d, %xmm0
+; AVX512DQ-NEXT: vpslld $16, %xmm0, %xmm1
; AVX512DQ-NEXT: vpbroadcastd {{.*#+}} xmm0 = [NaN,NaN,NaN,NaN]
; AVX512DQ-NEXT: vpor %xmm0, %xmm1, %xmm0
-; AVX512DQ-NEXT: shll $16, %r15d
; AVX512DQ-NEXT: vmovd %r15d, %xmm2
+; AVX512DQ-NEXT: vpslld $16, %xmm2, %xmm2
; AVX512DQ-NEXT: vmaxss %xmm2, %xmm1, %xmm2
; AVX512DQ-NEXT: vpand %xmm2, %xmm0, %xmm0
; AVX512DQ-NEXT: vcmpunordss %xmm1, %xmm1, %k1
; AVX512DQ-NEXT: vmovss %xmm1, %xmm0, %xmm0 {%k1}
; AVX512DQ-NEXT: callq __truncsfbf2 at PLT
; AVX512DQ-NEXT: vpextrw $0, %xmm0, {{[0-9]+}}(%rsp)
-; AVX512DQ-NEXT: shll $16, %r14d
-; AVX512DQ-NEXT: vmovd %r14d, %xmm1
+; AVX512DQ-NEXT: vmovd %r14d, %xmm0
+; AVX512DQ-NEXT: vpslld $16, %xmm0, %xmm1
; AVX512DQ-NEXT: vpbroadcastd {{.*#+}} xmm0 = [NaN,NaN,NaN,NaN]
; AVX512DQ-NEXT: vpor %xmm0, %xmm1, %xmm0
-; AVX512DQ-NEXT: shll $16, %ebx
; AVX512DQ-NEXT: vmovd %ebx, %xmm2
+; AVX512DQ-NEXT: vpslld $16, %xmm2, %xmm2
; AVX512DQ-NEXT: vmaxss %xmm2, %xmm1, %xmm2
; AVX512DQ-NEXT: vpand %xmm2, %xmm0, %xmm0
; AVX512DQ-NEXT: vcmpunordss %xmm1, %xmm1, %k1
@@ -2746,19 +2635,15 @@ define <4 x bfloat> @test_fmaximum_v4bf16(<4 x bfloat> %x, <4 x bfloat> %y) {
; AVX512DQ-NEXT: callq __truncsfbf2 at PLT
; AVX512DQ-NEXT: vpextrw $0, %xmm0, {{[0-9]+}}(%rsp)
; AVX512DQ-NEXT: vmovaps (%rsp), %xmm0
-; AVX512DQ-NEXT: addq $8, %rsp
-; AVX512DQ-NEXT: .cfi_def_cfa_offset 56
-; AVX512DQ-NEXT: popq %rbx
-; AVX512DQ-NEXT: .cfi_def_cfa_offset 48
-; AVX512DQ-NEXT: popq %r12
+; AVX512DQ-NEXT: addq $56, %rsp
; AVX512DQ-NEXT: .cfi_def_cfa_offset 40
-; AVX512DQ-NEXT: popq %r13
+; AVX512DQ-NEXT: popq %rbx
; AVX512DQ-NEXT: .cfi_def_cfa_offset 32
-; AVX512DQ-NEXT: popq %r14
+; AVX512DQ-NEXT: popq %r12
; AVX512DQ-NEXT: .cfi_def_cfa_offset 24
-; AVX512DQ-NEXT: popq %r15
+; AVX512DQ-NEXT: popq %r14
; AVX512DQ-NEXT: .cfi_def_cfa_offset 16
-; AVX512DQ-NEXT: popq %rbp
+; AVX512DQ-NEXT: popq %r15
; AVX512DQ-NEXT: .cfi_def_cfa_offset 8
; AVX512DQ-NEXT: retq
;
@@ -2786,52 +2671,27 @@ define <4 x bfloat> @test_fmaximum_v4bf16(<4 x bfloat> %x, <4 x bfloat> %y) {
;
; X86-LABEL: test_fmaximum_v4bf16:
; X86: # %bb.0:
-; X86-NEXT: pushl %ebp
-; X86-NEXT: .cfi_def_cfa_offset 8
-; X86-NEXT: pushl %ebx
-; X86-NEXT: .cfi_def_cfa_offset 12
-; X86-NEXT: pushl %edi
-; X86-NEXT: .cfi_def_cfa_offset 16
-; X86-NEXT: pushl %esi
-; X86-NEXT: .cfi_def_cfa_offset 20
-; X86-NEXT: subl $68, %esp
+; X86-NEXT: subl $84, %esp
; X86-NEXT: .cfi_def_cfa_offset 88
-; X86-NEXT: .cfi_offset %esi, -20
-; X86-NEXT: .cfi_offset %edi, -16
-; X86-NEXT: .cfi_offset %ebx, -12
-; X86-NEXT: .cfi_offset %ebp, -8
-; X86-NEXT: vpsrlq $48, %xmm0, %xmm2
-; X86-NEXT: vpextrw $0, %xmm2, %esi
-; X86-NEXT: vpsrlq $48, %xmm1, %xmm2
-; X86-NEXT: vpextrw $0, %xmm2, %edi
-; X86-NEXT: vmovshdup {{.*#+}} xmm2 = xmm0[1,1,3,3]
-; X86-NEXT: vpextrw $0, %xmm2, %ebx
-; X86-NEXT: vmovshdup {{.*#+}} xmm2 = xmm1[1,1,3,3]
-; X86-NEXT: vpextrw $0, %xmm2, %ebp
-; X86-NEXT: vpextrw $0, %xmm0, %eax
-; X86-NEXT: vpsrld $16, %xmm0, %xmm0
-; X86-NEXT: vpsrld $16, %xmm1, %xmm2
-; X86-NEXT: vpextrw $0, %xmm2, %ecx
-; X86-NEXT: shll $16, %ecx
-; X86-NEXT: vmovd %ecx, %xmm2
-; X86-NEXT: vpextrw $0, %xmm0, %ecx
-; X86-NEXT: shll $16, %ecx
-; X86-NEXT: vmovd %ecx, %xmm0
-; X86-NEXT: vpextrw $0, %xmm1, %ecx
-; X86-NEXT: vmaxss %xmm2, %xmm0, %xmm1
-; X86-NEXT: vbroadcastss {{.*#+}} xmm3 = [NaN,NaN,NaN,NaN]
-; X86-NEXT: vorps %xmm3, %xmm0, %xmm2
-; X86-NEXT: vandps %xmm1, %xmm2, %xmm1
-; X86-NEXT: vcmpunordss %xmm0, %xmm0, %xmm2
-; X86-NEXT: vblendvps %xmm2, %xmm0, %xmm1, %xmm0
-; X86-NEXT: vmovss %xmm0, (%esp)
-; X86-NEXT: shll $16, %ecx
-; X86-NEXT: vmovd %ecx, %xmm0
-; X86-NEXT: shll $16, %eax
-; X86-NEXT: vmovd %eax, %xmm1
+; X86-NEXT: vmovdqu %xmm1, {{[-0-9]+}}(%e{{[sb]}}p) # 16-byte Spill
+; X86-NEXT: vpsrlq $48, %xmm1, %xmm4
+; X86-NEXT: vmovshdup {{.*#+}} xmm5 = xmm1[1,1,3,3]
+; X86-NEXT: vpshufb {{.*#+}} xmm2 = zero,zero,xmm0[4,5],zero,zero,xmm0[u,u],zero,zero,xmm0[u,u],zero,zero,xmm0[u,u]
+; X86-NEXT: vmovdqa %xmm0, %xmm6
+; X86-NEXT: vmovdqu %xmm0, {{[-0-9]+}}(%e{{[sb]}}p) # 16-byte Spill
+; X86-NEXT: vbroadcastss {{.*#+}} xmm0 = [NaN,NaN,NaN,NaN]
+; X86-NEXT: vpor %xmm0, %xmm2, %xmm3
+; X86-NEXT: vpslld $16, %xmm5, %xmm1
+; X86-NEXT: vmaxss %xmm1, %xmm2, %xmm1
+; X86-NEXT: vpand %xmm1, %xmm3, %xmm1
+; X86-NEXT: vcmpunordss %xmm2, %xmm2, %xmm3
+; X86-NEXT: vblendvps %xmm3, %xmm2, %xmm1, %xmm1
+; X86-NEXT: vmovss %xmm1, (%esp)
+; X86-NEXT: vpshufb {{.*#+}} xmm1 = zero,zero,xmm6[6,7],zero,zero,xmm6[u,u],zero,zero,xmm6[u,u],zero,zero,xmm6[u,u]
+; X86-NEXT: vpor %xmm0, %xmm1, %xmm2
+; X86-NEXT: vpslld $16, %xmm4, %xmm0
; X86-NEXT: vmaxss %xmm0, %xmm1, %xmm0
-; X86-NEXT: vorps %xmm3, %xmm1, %xmm2
-; X86-NEXT: vandps %xmm0, %xmm2, %xmm0
+; X86-NEXT: vpand %xmm0, %xmm2, %xmm0
; X86-NEXT: vcmpunordss %xmm1, %xmm1, %xmm2
; X86-NEXT: vblendvps %xmm2, %xmm1, %xmm0, %xmm0
; X86-NEXT: vmovups %xmm0, {{[-0-9]+}}(%e{{[sb]}}p) # 16-byte Spill
@@ -2839,29 +2699,30 @@ define <4 x bfloat> @test_fmaximum_v4bf16(<4 x bfloat> %x, <4 x bfloat> %y) {
; X86-NEXT: vmovups %xmm0, {{[-0-9]+}}(%e{{[sb]}}p) # 16-byte Spill
; X86-NEXT: vmovups {{[-0-9]+}}(%e{{[sb]}}p), %xmm0 # 16-byte Reload
; X86-NEXT: vmovss %xmm0, (%esp)
-; X86-NEXT: shll $16, %ebp
-; X86-NEXT: vmovd %ebp, %xmm0
-; X86-NEXT: shll $16, %ebx
-; X86-NEXT: vmovd %ebx, %xmm1
-; X86-NEXT: vmaxss %xmm0, %xmm1, %xmm0
-; X86-NEXT: vorps {{\.?LCPI[0-9]+_[0-9]+}}, %xmm1, %xmm2
-; X86-NEXT: vandps %xmm0, %xmm2, %xmm0
-; X86-NEXT: vcmpunordss %xmm1, %xmm1, %xmm2
-; X86-NEXT: vblendvps %xmm2, %xmm1, %xmm0, %xmm0
+; X86-NEXT: vmovdqu {{[-0-9]+}}(%e{{[sb]}}p), %xmm0 # 16-byte Reload
+; X86-NEXT: vpslld $16, %xmm0, %xmm0
+; X86-NEXT: vpor {{\.?LCPI[0-9]+_[0-9]+}}, %xmm0, %xmm1
+; X86-NEXT: vmovdqu {{[-0-9]+}}(%e{{[sb]}}p), %xmm2 # 16-byte Reload
+; X86-NEXT: vpslld $16, %xmm2, %xmm2
+; X86-NEXT: vmaxss %xmm2, %xmm0, %xmm2
+; X86-NEXT: vpand %xmm2, %xmm1, %xmm1
+; X86-NEXT: vcmpunordss %xmm0, %xmm0, %xmm2
+; X86-NEXT: vblendvps %xmm2, %xmm0, %xmm1, %xmm0
; X86-NEXT: vmovups %xmm0, {{[-0-9]+}}(%e{{[sb]}}p) # 16-byte Spill
; X86-NEXT: calll __truncsfbf2
; X86-NEXT: vmovups %xmm0, {{[-0-9]+}}(%e{{[sb]}}p) # 16-byte Spill
-; X86-NEXT: vmovups {{[-0-9]+}}(%e{{[sb]}}p), %xmm0 # 16-byte Reload
-; X86-NEXT: vmovss %xmm0, (%esp)
-; X86-NEXT: shll $16, %edi
-; X86-NEXT: vmovd %edi, %xmm0
-; X86-NEXT: shll $16, %esi
-; X86-NEXT: vmovd %esi, %xmm1
-; X86-NEXT: vmaxss %xmm0, %xmm1, %xmm0
-; X86-NEXT: vorps {{\.?LCPI[0-9]+_[0-9]+}}, %xmm1, %xmm2
-; X86-NEXT: vandps %xmm0, %xmm2, %xmm0
-; X86-NEXT: vcmpunordss %xmm1, %xmm1, %xmm2
-; X86-NEXT: vblendvps %xmm2, %xmm1, %xmm0, %xmm0
+; X86-NEXT: vmovdqu {{[-0-9]+}}(%e{{[sb]}}p), %xmm0 # 16-byte Reload
+; X86-NEXT: vmovd %xmm0, (%esp)
+; X86-NEXT: vpxor %xmm0, %xmm0, %xmm0
+; X86-NEXT: vpblendw $170, {{[-0-9]+}}(%e{{[sb]}}p), %xmm0, %xmm1 # 16-byte Folded Reload
+; X86-NEXT: # xmm1 = xmm0[0],mem[1],xmm0[2],mem[3],xmm0[4],mem[5],xmm0[6],mem[7]
+; X86-NEXT: vpblendw $170, {{[-0-9]+}}(%e{{[sb]}}p), %xmm0, %xmm0 # 16-byte Folded Reload
+; X86-NEXT: # xmm0 = xmm0[0],mem[1],xmm0[2],mem[3],xmm0[4],mem[5],xmm0[6],mem[7]
+; X86-NEXT: vmaxss %xmm1, %xmm0, %xmm1
+; X86-NEXT: vorps {{\.?LCPI[0-9]+_[0-9]+}}, %xmm0, %xmm2
+; X86-NEXT: vandps %xmm1, %xmm2, %xmm1
+; X86-NEXT: vcmpunordss %xmm0, %xmm0, %xmm2
+; X86-NEXT: vblendvps %xmm2, %xmm0, %xmm1, %xmm0
; X86-NEXT: vmovups %xmm0, {{[-0-9]+}}(%e{{[sb]}}p) # 16-byte Spill
; X86-NEXT: calll __truncsfbf2
; X86-NEXT: vmovups %xmm0, {{[-0-9]+}}(%e{{[sb]}}p) # 16-byte Spill
@@ -2873,16 +2734,8 @@ define <4 x bfloat> @test_fmaximum_v4bf16(<4 x bfloat> %x, <4 x bfloat> %y) {
; X86-NEXT: vmovdqu {{[-0-9]+}}(%e{{[sb]}}p), %xmm1 # 16-byte Reload
; X86-NEXT: vpunpcklwd {{[-0-9]+}}(%e{{[sb]}}p), %xmm1, %xmm1 # 16-byte Folded Reload
; X86-NEXT: # xmm1 = xmm1[0],mem[0],xmm1[1],mem[1],xmm1[2],mem[2],xmm1[3],mem[3]
-; X86-NEXT: vinsertps {{.*#+}} xmm0 = xmm1[0],xmm0[0],zero,zero
-; X86-NEXT: addl $68, %esp
-; X86-NEXT: .cfi_def_cfa_offset 20
-; X86-NEXT: popl %esi
-; X86-NEXT: .cfi_def_cfa_offset 16
-; X86-NEXT: popl %edi
-; X86-NEXT: .cfi_def_cfa_offset 12
-; X86-NEXT: popl %ebx
-; X86-NEXT: .cfi_def_cfa_offset 8
-; X86-NEXT: popl %ebp
+; X86-NEXT: vinsertps {{.*#+}} xmm0 = xmm0[0],xmm1[0],zero,zero
+; X86-NEXT: addl $84, %esp
; X86-NEXT: .cfi_def_cfa_offset 4
; X86-NEXT: retl
%r = call <4 x bfloat> @llvm.maximum.v4bf16(<4 x bfloat> %x, <4 x bfloat> %y)
@@ -2893,22 +2746,19 @@ define bfloat @test_fminimum_bf16(bfloat %x, bfloat %y) nounwind {
; SSE2-LABEL: test_fminimum_bf16:
; SSE2: # %bb.0:
; SSE2-NEXT: pushq %rax
-; SSE2-NEXT: pextrw $0, %xmm1, %eax
-; SSE2-NEXT: pextrw $0, %xmm0, %ecx
-; SSE2-NEXT: shll $16, %ecx
-; SSE2-NEXT: movd %ecx, %xmm1
-; SSE2-NEXT: movdqa %xmm1, %xmm0
-; SSE2-NEXT: cmpunordss %xmm1, %xmm0
-; SSE2-NEXT: movaps %xmm0, %xmm2
-; SSE2-NEXT: andps %xmm1, %xmm2
-; SSE2-NEXT: shll $16, %eax
-; SSE2-NEXT: movd %eax, %xmm3
-; SSE2-NEXT: movaps %xmm1, %xmm4
-; SSE2-NEXT: minss %xmm3, %xmm4
-; SSE2-NEXT: andps {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm1
-; SSE2-NEXT: orps %xmm4, %xmm1
-; SSE2-NEXT: andnps %xmm1, %xmm0
-; SSE2-NEXT: orps %xmm2, %xmm0
+; SSE2-NEXT: pslld $16, %xmm0
+; SSE2-NEXT: movdqa %xmm0, %xmm2
+; SSE2-NEXT: cmpunordss %xmm0, %xmm2
+; SSE2-NEXT: movaps %xmm2, %xmm3
+; SSE2-NEXT: andps %xmm0, %xmm3
+; SSE2-NEXT: pslld $16, %xmm1
+; SSE2-NEXT: movaps %xmm0, %xmm4
+; SSE2-NEXT: minss %xmm1, %xmm4
+; SSE2-NEXT: andps {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm0
+; SSE2-NEXT: orps %xmm4, %xmm0
+; SSE2-NEXT: andnps %xmm0, %xmm2
+; SSE2-NEXT: orps %xmm3, %xmm2
+; SSE2-NEXT: movaps %xmm2, %xmm0
; SSE2-NEXT: callq __truncsfbf2 at PLT
; SSE2-NEXT: popq %rax
; SSE2-NEXT: retq
@@ -2916,17 +2766,13 @@ define bfloat @test_fminimum_bf16(bfloat %x, bfloat %y) nounwind {
; AVX1-LABEL: test_fminimum_bf16:
; AVX1: # %bb.0:
; AVX1-NEXT: pushq %rax
-; AVX1-NEXT: vpextrw $0, %xmm0, %eax
-; AVX1-NEXT: vpextrw $0, %xmm1, %ecx
-; AVX1-NEXT: shll $16, %ecx
-; AVX1-NEXT: vmovd %ecx, %xmm0
-; AVX1-NEXT: shll $16, %eax
-; AVX1-NEXT: vmovd %eax, %xmm1
-; AVX1-NEXT: vminss %xmm0, %xmm1, %xmm0
-; AVX1-NEXT: vandps {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm1, %xmm2
-; AVX1-NEXT: vorps %xmm0, %xmm2, %xmm0
-; AVX1-NEXT: vcmpunordss %xmm1, %xmm1, %xmm2
-; AVX1-NEXT: vblendvps %xmm2, %xmm1, %xmm0, %xmm0
+; AVX1-NEXT: vpslld $16, %xmm1, %xmm1
+; AVX1-NEXT: vpslld $16, %xmm0, %xmm0
+; AVX1-NEXT: vminss %xmm1, %xmm0, %xmm1
+; AVX1-NEXT: vandps {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm0, %xmm2
+; AVX1-NEXT: vorps %xmm1, %xmm2, %xmm1
+; AVX1-NEXT: vcmpunordss %xmm0, %xmm0, %xmm2
+; AVX1-NEXT: vblendvps %xmm2, %xmm0, %xmm1, %xmm0
; AVX1-NEXT: callq __truncsfbf2 at PLT
; AVX1-NEXT: popq %rax
; AVX1-NEXT: retq
@@ -2934,18 +2780,14 @@ define bfloat @test_fminimum_bf16(bfloat %x, bfloat %y) nounwind {
; AVX512F-LABEL: test_fminimum_bf16:
; AVX512F: # %bb.0:
; AVX512F-NEXT: pushq %rax
-; AVX512F-NEXT: vpextrw $0, %xmm0, %eax
-; AVX512F-NEXT: vpextrw $0, %xmm1, %ecx
-; AVX512F-NEXT: shll $16, %ecx
-; AVX512F-NEXT: vmovd %ecx, %xmm0
-; AVX512F-NEXT: shll $16, %eax
-; AVX512F-NEXT: vmovd %eax, %xmm1
-; AVX512F-NEXT: vminss %xmm0, %xmm1, %xmm0
-; AVX512F-NEXT: vbroadcastss {{.*#+}} xmm2 = [-0.0E+0,-0.0E+0,-0.0E+0,-0.0E+0]
-; AVX512F-NEXT: vandps %xmm2, %xmm1, %xmm2
-; AVX512F-NEXT: vorps %xmm0, %xmm2, %xmm0
-; AVX512F-NEXT: vcmpunordss %xmm1, %xmm1, %k1
-; AVX512F-NEXT: vmovss %xmm1, %xmm0, %xmm0 {%k1}
+; AVX512F-NEXT: vpslld $16, %xmm0, %xmm2
+; AVX512F-NEXT: vpbroadcastd {{.*#+}} xmm0 = [-0.0E+0,-0.0E+0,-0.0E+0,-0.0E+0]
+; AVX512F-NEXT: vpand %xmm0, %xmm2, %xmm0
+; AVX512F-NEXT: vpslld $16, %xmm1, %xmm1
+; AVX512F-NEXT: vminss %xmm1, %xmm2, %xmm1
+; AVX512F-NEXT: vpor %xmm1, %xmm0, %xmm0
+; AVX512F-NEXT: vcmpunordss %xmm2, %xmm2, %k1
+; AVX512F-NEXT: vmovss %xmm2, %xmm0, %xmm0 {%k1}
; AVX512F-NEXT: callq __truncsfbf2 at PLT
; AVX512F-NEXT: popq %rax
; AVX512F-NEXT: retq
@@ -2953,46 +2795,34 @@ define bfloat @test_fminimum_bf16(bfloat %x, bfloat %y) nounwind {
; AVX512DQ-LABEL: test_fminimum_bf16:
; AVX512DQ: # %bb.0:
; AVX512DQ-NEXT: pushq %rax
-; AVX512DQ-NEXT: vpextrw $0, %xmm0, %eax
-; AVX512DQ-NEXT: vpextrw $0, %xmm1, %ecx
-; AVX512DQ-NEXT: shll $16, %ecx
-; AVX512DQ-NEXT: vmovd %ecx, %xmm0
-; AVX512DQ-NEXT: shll $16, %eax
-; AVX512DQ-NEXT: vmovd %eax, %xmm1
-; AVX512DQ-NEXT: vminss %xmm0, %xmm1, %xmm0
-; AVX512DQ-NEXT: vbroadcastss {{.*#+}} xmm2 = [-0.0E+0,-0.0E+0,-0.0E+0,-0.0E+0]
-; AVX512DQ-NEXT: vandps %xmm2, %xmm1, %xmm2
-; AVX512DQ-NEXT: vorps %xmm0, %xmm2, %xmm0
-; AVX512DQ-NEXT: vcmpunordss %xmm1, %xmm1, %k1
-; AVX512DQ-NEXT: vmovss %xmm1, %xmm0, %xmm0 {%k1}
+; AVX512DQ-NEXT: vpslld $16, %xmm0, %xmm2
+; AVX512DQ-NEXT: vpbroadcastd {{.*#+}} xmm0 = [-0.0E+0,-0.0E+0,-0.0E+0,-0.0E+0]
+; AVX512DQ-NEXT: vpand %xmm0, %xmm2, %xmm0
+; AVX512DQ-NEXT: vpslld $16, %xmm1, %xmm1
+; AVX512DQ-NEXT: vminss %xmm1, %xmm2, %xmm1
+; AVX512DQ-NEXT: vpor %xmm1, %xmm0, %xmm0
+; AVX512DQ-NEXT: vcmpunordss %xmm2, %xmm2, %k1
+; AVX512DQ-NEXT: vmovss %xmm2, %xmm0, %xmm0 {%k1}
; AVX512DQ-NEXT: callq __truncsfbf2 at PLT
; AVX512DQ-NEXT: popq %rax
; AVX512DQ-NEXT: retq
;
; AVX512BF16-LABEL: test_fminimum_bf16:
; AVX512BF16: # %bb.0:
-; AVX512BF16-NEXT: vpextrw $0, %xmm0, %eax
-; AVX512BF16-NEXT: vpextrw $0, %xmm1, %ecx
-; AVX512BF16-NEXT: shll $16, %ecx
-; AVX512BF16-NEXT: vmovd %ecx, %xmm0
-; AVX512BF16-NEXT: shll $16, %eax
-; AVX512BF16-NEXT: vmovd %eax, %xmm1
-; AVX512BF16-NEXT: vminss %xmm0, %xmm1, %xmm0
-; AVX512BF16-NEXT: vpternlogd {{.*#+}} xmm0 = xmm0 | (xmm1 & m32bcst)
-; AVX512BF16-NEXT: vcmpunordss %xmm1, %xmm1, %k1
-; AVX512BF16-NEXT: vmovss %xmm1, %xmm0, %xmm0 {%k1}
-; AVX512BF16-NEXT: vcvtneps2bf16 %xmm0, %xmm0
+; AVX512BF16-NEXT: vpslld $16, %xmm1, %xmm1
+; AVX512BF16-NEXT: vpslld $16, %xmm0, %xmm0
+; AVX512BF16-NEXT: vminss %xmm1, %xmm0, %xmm1
+; AVX512BF16-NEXT: vpternlogd {{.*#+}} xmm1 = xmm1 | (xmm0 & m32bcst)
+; AVX512BF16-NEXT: vcmpunordss %xmm0, %xmm0, %k1
+; AVX512BF16-NEXT: vmovss %xmm0, %xmm1, %xmm1 {%k1}
+; AVX512BF16-NEXT: vcvtneps2bf16 %xmm1, %xmm0
; AVX512BF16-NEXT: retq
;
; AVX10_2-LABEL: test_fminimum_bf16:
; AVX10_2: # %bb.0:
-; AVX10_2-NEXT: vmovw %xmm0, %eax
-; AVX10_2-NEXT: vmovw %xmm1, %ecx
-; AVX10_2-NEXT: shll $16, %ecx
-; AVX10_2-NEXT: vmovd %ecx, %xmm0
-; AVX10_2-NEXT: shll $16, %eax
-; AVX10_2-NEXT: vmovd %eax, %xmm1
-; AVX10_2-NEXT: vminmaxss $4, %xmm0, %xmm1
+; AVX10_2-NEXT: vpslld $16, %xmm1, %xmm1
+; AVX10_2-NEXT: vpslld $16, %xmm0, %xmm0
+; AVX10_2-NEXT: vminmaxss $4, %xmm1, %xmm0
; AVX10_2-NEXT: vcvtneps2bf16 %xmm0, %xmm0
; AVX10_2-NEXT: retq
;
diff --git a/llvm/test/CodeGen/X86/fminimumnum-fmaximumnum.ll b/llvm/test/CodeGen/X86/fminimumnum-fmaximumnum.ll
index 8eed254a014c3..707d3b3425946 100644
--- a/llvm/test/CodeGen/X86/fminimumnum-fmaximumnum.ll
+++ b/llvm/test/CodeGen/X86/fminimumnum-fmaximumnum.ll
@@ -2651,28 +2651,23 @@ define bfloat @test_fmaximumnum_bf16(bfloat %x, bfloat %y) nounwind {
; SSE2-LABEL: test_fmaximumnum_bf16:
; SSE2: # %bb.0:
; SSE2-NEXT: pushq %rax
-; SSE2-NEXT: pextrw $0, %xmm0, %eax
-; SSE2-NEXT: pextrw $0, %xmm1, %ecx
-; SSE2-NEXT: shll $16, %ecx
-; SSE2-NEXT: movd %ecx, %xmm0
-; SSE2-NEXT: shll $16, %eax
-; SSE2-NEXT: movd %eax, %xmm1
-; SSE2-NEXT: movdqa %xmm1, %xmm2
-; SSE2-NEXT: cmpunordss %xmm1, %xmm2
+; SSE2-NEXT: pslld $16, %xmm1
+; SSE2-NEXT: pslld $16, %xmm0
+; SSE2-NEXT: movdqa %xmm0, %xmm2
+; SSE2-NEXT: cmpunordss %xmm0, %xmm2
; SSE2-NEXT: movaps %xmm2, %xmm3
-; SSE2-NEXT: andps %xmm0, %xmm3
-; SSE2-NEXT: movaps %xmm1, %xmm4
-; SSE2-NEXT: maxss %xmm0, %xmm4
+; SSE2-NEXT: andps %xmm1, %xmm3
+; SSE2-NEXT: movaps %xmm0, %xmm4
+; SSE2-NEXT: maxss %xmm1, %xmm4
; SSE2-NEXT: movaps {{.*#+}} xmm5 = [NaN,NaN,NaN,NaN]
-; SSE2-NEXT: orps %xmm1, %xmm5
+; SSE2-NEXT: orps %xmm0, %xmm5
; SSE2-NEXT: andps %xmm4, %xmm5
; SSE2-NEXT: andnps %xmm5, %xmm2
; SSE2-NEXT: orps %xmm3, %xmm2
-; SSE2-NEXT: cmpunordss %xmm0, %xmm0
-; SSE2-NEXT: movaps %xmm0, %xmm3
-; SSE2-NEXT: andnps %xmm2, %xmm3
+; SSE2-NEXT: cmpunordss %xmm1, %xmm1
; SSE2-NEXT: andps %xmm1, %xmm0
-; SSE2-NEXT: orps %xmm3, %xmm0
+; SSE2-NEXT: andnps %xmm2, %xmm1
+; SSE2-NEXT: orps %xmm1, %xmm0
; SSE2-NEXT: callq __truncsfbf2 at PLT
; SSE2-NEXT: popq %rax
; SSE2-NEXT: retq
@@ -2680,19 +2675,15 @@ define bfloat @test_fmaximumnum_bf16(bfloat %x, bfloat %y) nounwind {
; AVX1-LABEL: test_fmaximumnum_bf16:
; AVX1: # %bb.0:
; AVX1-NEXT: pushq %rax
-; AVX1-NEXT: vpextrw $0, %xmm0, %eax
-; AVX1-NEXT: vpextrw $0, %xmm1, %ecx
-; AVX1-NEXT: shll $16, %ecx
-; AVX1-NEXT: vmovd %ecx, %xmm0
-; AVX1-NEXT: shll $16, %eax
-; AVX1-NEXT: vmovd %eax, %xmm1
-; AVX1-NEXT: vmaxss %xmm0, %xmm1, %xmm2
-; AVX1-NEXT: vorps {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm1, %xmm3
+; AVX1-NEXT: vpslld $16, %xmm1, %xmm1
+; AVX1-NEXT: vpslld $16, %xmm0, %xmm0
+; AVX1-NEXT: vmaxss %xmm1, %xmm0, %xmm2
+; AVX1-NEXT: vorps {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm0, %xmm3
; AVX1-NEXT: vandps %xmm2, %xmm3, %xmm2
-; AVX1-NEXT: vcmpunordss %xmm1, %xmm1, %xmm3
-; AVX1-NEXT: vblendvps %xmm3, %xmm0, %xmm2, %xmm2
-; AVX1-NEXT: vcmpunordss %xmm0, %xmm0, %xmm0
-; AVX1-NEXT: vblendvps %xmm0, %xmm1, %xmm2, %xmm0
+; AVX1-NEXT: vcmpunordss %xmm0, %xmm0, %xmm3
+; AVX1-NEXT: vblendvps %xmm3, %xmm1, %xmm2, %xmm2
+; AVX1-NEXT: vcmpunordss %xmm1, %xmm1, %xmm1
+; AVX1-NEXT: vblendvps %xmm1, %xmm0, %xmm2, %xmm0
; AVX1-NEXT: callq __truncsfbf2 at PLT
; AVX1-NEXT: popq %rax
; AVX1-NEXT: retq
@@ -2700,16 +2691,12 @@ define bfloat @test_fmaximumnum_bf16(bfloat %x, bfloat %y) nounwind {
; AVX512F-LABEL: test_fmaximumnum_bf16:
; AVX512F: # %bb.0:
; AVX512F-NEXT: pushq %rax
-; AVX512F-NEXT: vpextrw $0, %xmm0, %eax
-; AVX512F-NEXT: vpextrw $0, %xmm1, %ecx
-; AVX512F-NEXT: shll $16, %ecx
-; AVX512F-NEXT: vmovd %ecx, %xmm1
-; AVX512F-NEXT: shll $16, %eax
-; AVX512F-NEXT: vmovd %eax, %xmm2
-; AVX512F-NEXT: vmaxss %xmm1, %xmm2, %xmm0
-; AVX512F-NEXT: vbroadcastss {{.*#+}} xmm3 = [NaN,NaN,NaN,NaN]
-; AVX512F-NEXT: vorps %xmm3, %xmm2, %xmm3
-; AVX512F-NEXT: vandps %xmm0, %xmm3, %xmm0
+; AVX512F-NEXT: vpslld $16, %xmm0, %xmm2
+; AVX512F-NEXT: vpbroadcastd {{.*#+}} xmm0 = [NaN,NaN,NaN,NaN]
+; AVX512F-NEXT: vpor %xmm0, %xmm2, %xmm0
+; AVX512F-NEXT: vpslld $16, %xmm1, %xmm1
+; AVX512F-NEXT: vmaxss %xmm1, %xmm2, %xmm3
+; AVX512F-NEXT: vpand %xmm3, %xmm0, %xmm0
; AVX512F-NEXT: vcmpunordss %xmm2, %xmm2, %k1
; AVX512F-NEXT: vmovss %xmm1, %xmm0, %xmm0 {%k1}
; AVX512F-NEXT: vcmpunordss %xmm1, %xmm1, %k1
@@ -2721,16 +2708,12 @@ define bfloat @test_fmaximumnum_bf16(bfloat %x, bfloat %y) nounwind {
; AVX512DQ-LABEL: test_fmaximumnum_bf16:
; AVX512DQ: # %bb.0:
; AVX512DQ-NEXT: pushq %rax
-; AVX512DQ-NEXT: vpextrw $0, %xmm0, %eax
-; AVX512DQ-NEXT: vpextrw $0, %xmm1, %ecx
-; AVX512DQ-NEXT: shll $16, %ecx
-; AVX512DQ-NEXT: vmovd %ecx, %xmm1
-; AVX512DQ-NEXT: shll $16, %eax
-; AVX512DQ-NEXT: vmovd %eax, %xmm2
-; AVX512DQ-NEXT: vmaxss %xmm1, %xmm2, %xmm0
-; AVX512DQ-NEXT: vbroadcastss {{.*#+}} xmm3 = [NaN,NaN,NaN,NaN]
-; AVX512DQ-NEXT: vorps %xmm3, %xmm2, %xmm3
-; AVX512DQ-NEXT: vandps %xmm0, %xmm3, %xmm0
+; AVX512DQ-NEXT: vpslld $16, %xmm0, %xmm2
+; AVX512DQ-NEXT: vpbroadcastd {{.*#+}} xmm0 = [NaN,NaN,NaN,NaN]
+; AVX512DQ-NEXT: vpor %xmm0, %xmm2, %xmm0
+; AVX512DQ-NEXT: vpslld $16, %xmm1, %xmm1
+; AVX512DQ-NEXT: vmaxss %xmm1, %xmm2, %xmm3
+; AVX512DQ-NEXT: vpand %xmm3, %xmm0, %xmm0
; AVX512DQ-NEXT: vcmpunordss %xmm2, %xmm2, %k1
; AVX512DQ-NEXT: vmovss %xmm1, %xmm0, %xmm0 {%k1}
; AVX512DQ-NEXT: vcmpunordss %xmm1, %xmm1, %k1
@@ -2741,30 +2724,22 @@ define bfloat @test_fmaximumnum_bf16(bfloat %x, bfloat %y) nounwind {
;
; AVX512BF16-LABEL: test_fmaximumnum_bf16:
; AVX512BF16: # %bb.0:
-; AVX512BF16-NEXT: vpextrw $0, %xmm0, %eax
-; AVX512BF16-NEXT: vpextrw $0, %xmm1, %ecx
-; AVX512BF16-NEXT: shll $16, %ecx
-; AVX512BF16-NEXT: vmovd %ecx, %xmm0
-; AVX512BF16-NEXT: shll $16, %eax
-; AVX512BF16-NEXT: vmovd %eax, %xmm1
-; AVX512BF16-NEXT: vmaxss %xmm0, %xmm1, %xmm2
-; AVX512BF16-NEXT: vpternlogd {{.*#+}} xmm2 = xmm2 & (xmm1 | m32bcst)
-; AVX512BF16-NEXT: vcmpunordss %xmm1, %xmm1, %k1
-; AVX512BF16-NEXT: vmovss %xmm0, %xmm2, %xmm2 {%k1}
+; AVX512BF16-NEXT: vpslld $16, %xmm1, %xmm1
+; AVX512BF16-NEXT: vpslld $16, %xmm0, %xmm0
+; AVX512BF16-NEXT: vmaxss %xmm1, %xmm0, %xmm2
+; AVX512BF16-NEXT: vpternlogd {{.*#+}} xmm2 = xmm2 & (xmm0 | m32bcst)
; AVX512BF16-NEXT: vcmpunordss %xmm0, %xmm0, %k1
; AVX512BF16-NEXT: vmovss %xmm1, %xmm2, %xmm2 {%k1}
+; AVX512BF16-NEXT: vcmpunordss %xmm1, %xmm1, %k1
+; AVX512BF16-NEXT: vmovss %xmm0, %xmm2, %xmm2 {%k1}
; AVX512BF16-NEXT: vcvtneps2bf16 %xmm2, %xmm0
; AVX512BF16-NEXT: retq
;
; AVX10_2-LABEL: test_fmaximumnum_bf16:
; AVX10_2: # %bb.0:
-; AVX10_2-NEXT: vmovw %xmm0, %eax
-; AVX10_2-NEXT: vmovw %xmm1, %ecx
-; AVX10_2-NEXT: shll $16, %ecx
-; AVX10_2-NEXT: vmovd %ecx, %xmm0
-; AVX10_2-NEXT: shll $16, %eax
-; AVX10_2-NEXT: vmovd %eax, %xmm1
-; AVX10_2-NEXT: vminmaxss $21, %xmm0, %xmm1
+; AVX10_2-NEXT: vpslld $16, %xmm1, %xmm1
+; AVX10_2-NEXT: vpslld $16, %xmm0, %xmm0
+; AVX10_2-NEXT: vminmaxss $21, %xmm1, %xmm0
; AVX10_2-NEXT: vcvtneps2bf16 %xmm0, %xmm0
; AVX10_2-NEXT: retq
;
@@ -2795,260 +2770,228 @@ define bfloat @test_fmaximumnum_bf16(bfloat %x, bfloat %y) nounwind {
define <4 x bfloat> @test_fmaximumnum_v4bf16(<4 x bfloat> %x, <4 x bfloat> %y) nounwind {
; SSE2-LABEL: test_fmaximumnum_v4bf16:
; SSE2: # %bb.0:
-; SSE2-NEXT: pushq %rbp
-; SSE2-NEXT: pushq %r15
-; SSE2-NEXT: pushq %r14
-; SSE2-NEXT: pushq %r13
-; SSE2-NEXT: pushq %r12
-; SSE2-NEXT: pushq %rbx
-; SSE2-NEXT: subq $56, %rsp
-; SSE2-NEXT: movdqa %xmm0, %xmm2
-; SSE2-NEXT: psrlq $48, %xmm2
-; SSE2-NEXT: pextrw $0, %xmm2, %ebx
-; SSE2-NEXT: movdqa %xmm1, %xmm2
-; SSE2-NEXT: psrlq $48, %xmm2
-; SSE2-NEXT: pextrw $0, %xmm2, %ebp
+; SSE2-NEXT: subq $120, %rsp
+; SSE2-NEXT: movdqa %xmm1, (%rsp) # 16-byte Spill
; SSE2-NEXT: movdqa %xmm0, %xmm2
-; SSE2-NEXT: shufps {{.*#+}} xmm2 = xmm2[1,1],xmm0[1,1]
-; SSE2-NEXT: pextrw $0, %xmm2, %r14d
-; SSE2-NEXT: movdqa %xmm1, %xmm2
-; SSE2-NEXT: shufps {{.*#+}} xmm2 = xmm2[1,1],xmm1[1,1]
-; SSE2-NEXT: pextrw $0, %xmm2, %r15d
-; SSE2-NEXT: pextrw $0, %xmm0, %r12d
-; SSE2-NEXT: pextrw $0, %xmm1, %r13d
+; SSE2-NEXT: movdqa %xmm1, %xmm0
+; SSE2-NEXT: psrlq $48, %xmm0
+; SSE2-NEXT: movdqa %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
+; SSE2-NEXT: movdqa %xmm2, %xmm0
+; SSE2-NEXT: psrlq $48, %xmm0
+; SSE2-NEXT: movdqa %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
+; SSE2-NEXT: movdqa %xmm1, %xmm0
+; SSE2-NEXT: shufps {{.*#+}} xmm0 = xmm0[1,1],xmm1[1,1]
+; SSE2-NEXT: movaps %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
+; SSE2-NEXT: movdqa %xmm1, %xmm0
; SSE2-NEXT: psrld $16, %xmm0
-; SSE2-NEXT: pextrw $0, %xmm0, %eax
+; SSE2-NEXT: movdqa %xmm2, %xmm1
+; SSE2-NEXT: movdqa %xmm2, %xmm6
+; SSE2-NEXT: movdqa %xmm2, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
; SSE2-NEXT: psrld $16, %xmm1
-; SSE2-NEXT: pextrw $0, %xmm1, %ecx
-; SSE2-NEXT: shll $16, %ecx
-; SSE2-NEXT: movd %ecx, %xmm0
-; SSE2-NEXT: shll $16, %eax
-; SSE2-NEXT: movd %eax, %xmm1
+; SSE2-NEXT: pslld $16, %xmm1
+; SSE2-NEXT: pslld $16, %xmm0
; SSE2-NEXT: movdqa %xmm1, %xmm2
-; SSE2-NEXT: cmpunordss %xmm1, %xmm2
-; SSE2-NEXT: movaps %xmm2, %xmm3
+; SSE2-NEXT: maxss %xmm0, %xmm2
+; SSE2-NEXT: movdqa %xmm1, %xmm3
+; SSE2-NEXT: cmpunordss %xmm1, %xmm3
+; SSE2-NEXT: movaps %xmm3, %xmm4
; SSE2-NEXT: andps %xmm0, %xmm3
-; SSE2-NEXT: movaps %xmm1, %xmm4
-; SSE2-NEXT: maxss %xmm0, %xmm4
; SSE2-NEXT: cmpunordss %xmm0, %xmm0
; SSE2-NEXT: movaps %xmm0, %xmm5
; SSE2-NEXT: andps %xmm1, %xmm0
; SSE2-NEXT: orps {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm1
-; SSE2-NEXT: andps %xmm4, %xmm1
-; SSE2-NEXT: andnps %xmm1, %xmm2
-; SSE2-NEXT: orps %xmm3, %xmm2
-; SSE2-NEXT: andnps %xmm2, %xmm5
+; SSE2-NEXT: andps %xmm1, %xmm2
+; SSE2-NEXT: movdqa %xmm6, %xmm1
+; SSE2-NEXT: shufps {{.*#+}} xmm1 = xmm1[1,1],xmm6[1,1]
+; SSE2-NEXT: movaps %xmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
+; SSE2-NEXT: andnps %xmm2, %xmm4
+; SSE2-NEXT: orps %xmm4, %xmm3
+; SSE2-NEXT: andnps %xmm3, %xmm5
; SSE2-NEXT: orps %xmm5, %xmm0
; SSE2-NEXT: callq __truncsfbf2 at PLT
; SSE2-NEXT: movaps %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
-; SSE2-NEXT: shll $16, %r13d
-; SSE2-NEXT: movd %r13d, %xmm0
-; SSE2-NEXT: shll $16, %r12d
-; SSE2-NEXT: movd %r12d, %xmm1
-; SSE2-NEXT: movdqa %xmm1, %xmm2
-; SSE2-NEXT: cmpunordss %xmm1, %xmm2
-; SSE2-NEXT: movaps %xmm2, %xmm3
-; SSE2-NEXT: andps %xmm0, %xmm3
-; SSE2-NEXT: movaps %xmm1, %xmm4
-; SSE2-NEXT: maxss %xmm0, %xmm4
-; SSE2-NEXT: cmpunordss %xmm0, %xmm0
-; SSE2-NEXT: movaps %xmm0, %xmm5
-; SSE2-NEXT: andps %xmm1, %xmm0
-; SSE2-NEXT: orps {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm1
-; SSE2-NEXT: andps %xmm4, %xmm1
+; SSE2-NEXT: movdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm2 # 16-byte Reload
+; SSE2-NEXT: pslld $16, %xmm2
+; SSE2-NEXT: movdqa %xmm2, %xmm0
+; SSE2-NEXT: por {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm0
+; SSE2-NEXT: movdqa (%rsp), %xmm3 # 16-byte Reload
+; SSE2-NEXT: pslld $16, %xmm3
+; SSE2-NEXT: movdqa %xmm2, %xmm1
+; SSE2-NEXT: maxss %xmm3, %xmm1
+; SSE2-NEXT: andps %xmm0, %xmm1
+; SSE2-NEXT: movdqa %xmm2, %xmm0
+; SSE2-NEXT: movdqa %xmm2, %xmm4
+; SSE2-NEXT: cmpunordss %xmm2, %xmm0
+; SSE2-NEXT: movaps %xmm0, %xmm2
; SSE2-NEXT: andnps %xmm1, %xmm2
-; SSE2-NEXT: orps %xmm3, %xmm2
-; SSE2-NEXT: andnps %xmm2, %xmm5
-; SSE2-NEXT: orps %xmm5, %xmm0
+; SSE2-NEXT: andps %xmm3, %xmm0
+; SSE2-NEXT: orps %xmm2, %xmm0
+; SSE2-NEXT: cmpunordss %xmm3, %xmm3
+; SSE2-NEXT: movaps %xmm3, %xmm1
+; SSE2-NEXT: andnps %xmm0, %xmm1
+; SSE2-NEXT: andps %xmm4, %xmm3
+; SSE2-NEXT: orps %xmm1, %xmm3
+; SSE2-NEXT: movaps %xmm3, %xmm0
; SSE2-NEXT: callq __truncsfbf2 at PLT
-; SSE2-NEXT: movaps %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
-; SSE2-NEXT: shll $16, %r15d
-; SSE2-NEXT: movd %r15d, %xmm0
-; SSE2-NEXT: shll $16, %r14d
-; SSE2-NEXT: movd %r14d, %xmm1
-; SSE2-NEXT: movdqa %xmm1, %xmm2
-; SSE2-NEXT: cmpunordss %xmm1, %xmm2
-; SSE2-NEXT: movaps %xmm2, %xmm3
-; SSE2-NEXT: andps %xmm0, %xmm3
-; SSE2-NEXT: movaps %xmm1, %xmm4
-; SSE2-NEXT: maxss %xmm0, %xmm4
-; SSE2-NEXT: cmpunordss %xmm0, %xmm0
-; SSE2-NEXT: movaps %xmm0, %xmm5
-; SSE2-NEXT: andps %xmm1, %xmm0
-; SSE2-NEXT: orps {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm1
-; SSE2-NEXT: andps %xmm4, %xmm1
+; SSE2-NEXT: movaps %xmm0, (%rsp) # 16-byte Spill
+; SSE2-NEXT: movdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm2 # 16-byte Reload
+; SSE2-NEXT: pslld $16, %xmm2
+; SSE2-NEXT: movdqa %xmm2, %xmm0
+; SSE2-NEXT: por {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm0
+; SSE2-NEXT: movdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm3 # 16-byte Reload
+; SSE2-NEXT: pslld $16, %xmm3
+; SSE2-NEXT: movdqa %xmm2, %xmm1
+; SSE2-NEXT: maxss %xmm3, %xmm1
+; SSE2-NEXT: andps %xmm0, %xmm1
+; SSE2-NEXT: movdqa %xmm2, %xmm0
+; SSE2-NEXT: movdqa %xmm2, %xmm4
+; SSE2-NEXT: cmpunordss %xmm2, %xmm0
+; SSE2-NEXT: movaps %xmm0, %xmm2
; SSE2-NEXT: andnps %xmm1, %xmm2
-; SSE2-NEXT: orps %xmm3, %xmm2
-; SSE2-NEXT: andnps %xmm2, %xmm5
-; SSE2-NEXT: orps %xmm5, %xmm0
+; SSE2-NEXT: andps %xmm3, %xmm0
+; SSE2-NEXT: orps %xmm2, %xmm0
+; SSE2-NEXT: cmpunordss %xmm3, %xmm3
+; SSE2-NEXT: movaps %xmm3, %xmm1
+; SSE2-NEXT: andnps %xmm0, %xmm1
+; SSE2-NEXT: andps %xmm4, %xmm3
+; SSE2-NEXT: orps %xmm1, %xmm3
+; SSE2-NEXT: movaps %xmm3, %xmm0
; SSE2-NEXT: callq __truncsfbf2 at PLT
-; SSE2-NEXT: movaps %xmm0, (%rsp) # 16-byte Spill
-; SSE2-NEXT: shll $16, %ebp
-; SSE2-NEXT: movd %ebp, %xmm0
-; SSE2-NEXT: shll $16, %ebx
-; SSE2-NEXT: movd %ebx, %xmm1
-; SSE2-NEXT: movdqa %xmm1, %xmm2
-; SSE2-NEXT: cmpunordss %xmm1, %xmm2
-; SSE2-NEXT: movaps %xmm2, %xmm3
-; SSE2-NEXT: andps %xmm0, %xmm3
-; SSE2-NEXT: movaps %xmm1, %xmm4
-; SSE2-NEXT: maxss %xmm0, %xmm4
-; SSE2-NEXT: movaps {{.*#+}} xmm5 = [NaN,NaN,NaN,NaN]
-; SSE2-NEXT: orps %xmm1, %xmm5
-; SSE2-NEXT: andps %xmm4, %xmm5
-; SSE2-NEXT: andnps %xmm5, %xmm2
-; SSE2-NEXT: orps %xmm3, %xmm2
-; SSE2-NEXT: cmpunordss %xmm0, %xmm0
-; SSE2-NEXT: movaps %xmm0, %xmm3
-; SSE2-NEXT: andnps %xmm2, %xmm3
+; SSE2-NEXT: movaps %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
+; SSE2-NEXT: movdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm2 # 16-byte Reload
+; SSE2-NEXT: pslld $16, %xmm2
+; SSE2-NEXT: movdqa {{.*#+}} xmm1 = [NaN,NaN,NaN,NaN]
+; SSE2-NEXT: por %xmm2, %xmm1
+; SSE2-NEXT: movdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm3 # 16-byte Reload
+; SSE2-NEXT: pslld $16, %xmm3
+; SSE2-NEXT: movdqa %xmm2, %xmm0
+; SSE2-NEXT: maxss %xmm3, %xmm0
; SSE2-NEXT: andps %xmm1, %xmm0
-; SSE2-NEXT: orps %xmm3, %xmm0
+; SSE2-NEXT: movdqa %xmm2, %xmm1
+; SSE2-NEXT: movdqa %xmm2, %xmm4
+; SSE2-NEXT: cmpunordss %xmm2, %xmm1
+; SSE2-NEXT: movaps %xmm1, %xmm2
+; SSE2-NEXT: andnps %xmm0, %xmm2
+; SSE2-NEXT: andps %xmm3, %xmm1
+; SSE2-NEXT: orps %xmm2, %xmm1
+; SSE2-NEXT: cmpunordss %xmm3, %xmm3
+; SSE2-NEXT: movaps %xmm3, %xmm0
+; SSE2-NEXT: andnps %xmm1, %xmm0
+; SSE2-NEXT: andps %xmm4, %xmm3
+; SSE2-NEXT: orps %xmm0, %xmm3
+; SSE2-NEXT: movaps %xmm3, %xmm0
; SSE2-NEXT: callq __truncsfbf2 at PLT
-; SSE2-NEXT: movdqa (%rsp), %xmm1 # 16-byte Reload
+; SSE2-NEXT: movdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm1 # 16-byte Reload
; SSE2-NEXT: punpcklwd {{.*#+}} xmm1 = xmm1[0],xmm0[0],xmm1[1],xmm0[1],xmm1[2],xmm0[2],xmm1[3],xmm0[3]
-; SSE2-NEXT: movdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 16-byte Reload
+; SSE2-NEXT: movdqa (%rsp), %xmm0 # 16-byte Reload
; SSE2-NEXT: punpcklwd {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 16-byte Folded Reload
; SSE2-NEXT: # xmm0 = xmm0[0],mem[0],xmm0[1],mem[1],xmm0[2],mem[2],xmm0[3],mem[3]
; SSE2-NEXT: punpckldq {{.*#+}} xmm0 = xmm0[0],xmm1[0],xmm0[1],xmm1[1]
-; SSE2-NEXT: addq $56, %rsp
-; SSE2-NEXT: popq %rbx
-; SSE2-NEXT: popq %r12
-; SSE2-NEXT: popq %r13
-; SSE2-NEXT: popq %r14
-; SSE2-NEXT: popq %r15
-; SSE2-NEXT: popq %rbp
+; SSE2-NEXT: addq $120, %rsp
; SSE2-NEXT: retq
;
; AVX1-LABEL: test_fmaximumnum_v4bf16:
; AVX1: # %bb.0:
-; AVX1-NEXT: pushq %rbp
-; AVX1-NEXT: pushq %r15
-; AVX1-NEXT: pushq %r14
-; AVX1-NEXT: pushq %r13
-; AVX1-NEXT: pushq %r12
-; AVX1-NEXT: pushq %rbx
-; AVX1-NEXT: subq $56, %rsp
-; AVX1-NEXT: vpsrlq $48, %xmm0, %xmm2
-; AVX1-NEXT: vpextrw $0, %xmm2, %ebx
-; AVX1-NEXT: vpsrlq $48, %xmm1, %xmm2
-; AVX1-NEXT: vpextrw $0, %xmm2, %ebp
-; AVX1-NEXT: vmovshdup {{.*#+}} xmm2 = xmm0[1,1,3,3]
-; AVX1-NEXT: vpextrw $0, %xmm2, %r14d
-; AVX1-NEXT: vmovshdup {{.*#+}} xmm2 = xmm1[1,1,3,3]
-; AVX1-NEXT: vpextrw $0, %xmm2, %r15d
-; AVX1-NEXT: vpextrw $0, %xmm0, %r12d
-; AVX1-NEXT: vpextrw $0, %xmm1, %r13d
-; AVX1-NEXT: vpsrld $16, %xmm0, %xmm0
-; AVX1-NEXT: vpextrw $0, %xmm0, %eax
-; AVX1-NEXT: vpsrld $16, %xmm1, %xmm0
-; AVX1-NEXT: vpextrw $0, %xmm0, %ecx
-; AVX1-NEXT: shll $16, %ecx
-; AVX1-NEXT: vmovd %ecx, %xmm0
-; AVX1-NEXT: shll $16, %eax
-; AVX1-NEXT: vmovd %eax, %xmm1
-; AVX1-NEXT: vmaxss %xmm0, %xmm1, %xmm2
-; AVX1-NEXT: vorps {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm1, %xmm3
-; AVX1-NEXT: vandps %xmm2, %xmm3, %xmm2
-; AVX1-NEXT: vcmpunordss %xmm1, %xmm1, %xmm3
-; AVX1-NEXT: vblendvps %xmm3, %xmm0, %xmm2, %xmm2
-; AVX1-NEXT: vcmpunordss %xmm0, %xmm0, %xmm0
-; AVX1-NEXT: vblendvps %xmm0, %xmm1, %xmm2, %xmm0
+; AVX1-NEXT: subq $88, %rsp
+; AVX1-NEXT: vmovdqa %xmm1, (%rsp) # 16-byte Spill
+; AVX1-NEXT: vmovdqa %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
+; AVX1-NEXT: vpslld $16, %xmm0, %xmm4
+; AVX1-NEXT: vpor {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm4, %xmm0
+; AVX1-NEXT: vpslld $16, %xmm1, %xmm2
+; AVX1-NEXT: vmaxss %xmm2, %xmm4, %xmm3
+; AVX1-NEXT: vpand %xmm3, %xmm0, %xmm1
+; AVX1-NEXT: vcmpunordss %xmm4, %xmm4, %xmm3
+; AVX1-NEXT: vblendvps %xmm3, %xmm2, %xmm1, %xmm1
+; AVX1-NEXT: vcmpunordss %xmm2, %xmm2, %xmm2
+; AVX1-NEXT: vblendvps %xmm2, %xmm4, %xmm1, %xmm0
; AVX1-NEXT: callq __truncsfbf2 at PLT
; AVX1-NEXT: vmovaps %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
-; AVX1-NEXT: shll $16, %r13d
-; AVX1-NEXT: vmovd %r13d, %xmm0
-; AVX1-NEXT: shll $16, %r12d
-; AVX1-NEXT: vmovd %r12d, %xmm1
-; AVX1-NEXT: vmaxss %xmm0, %xmm1, %xmm2
-; AVX1-NEXT: vorps {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm1, %xmm3
+; AVX1-NEXT: vxorps %xmm0, %xmm0, %xmm0
+; AVX1-NEXT: vpblendw $170, (%rsp), %xmm0, %xmm1 # 16-byte Folded Reload
+; AVX1-NEXT: # xmm1 = xmm0[0],mem[1],xmm0[2],mem[3],xmm0[4],mem[5],xmm0[6],mem[7]
+; AVX1-NEXT: vpblendw $170, {{[-0-9]+}}(%r{{[sb]}}p), %xmm0, %xmm0 # 16-byte Folded Reload
+; AVX1-NEXT: # xmm0 = xmm0[0],mem[1],xmm0[2],mem[3],xmm0[4],mem[5],xmm0[6],mem[7]
+; AVX1-NEXT: vmaxss %xmm1, %xmm0, %xmm2
+; AVX1-NEXT: vorps {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm0, %xmm3
; AVX1-NEXT: vandps %xmm2, %xmm3, %xmm2
-; AVX1-NEXT: vcmpunordss %xmm1, %xmm1, %xmm3
-; AVX1-NEXT: vblendvps %xmm3, %xmm0, %xmm2, %xmm2
-; AVX1-NEXT: vcmpunordss %xmm0, %xmm0, %xmm0
-; AVX1-NEXT: vblendvps %xmm0, %xmm1, %xmm2, %xmm0
+; AVX1-NEXT: vcmpunordss %xmm0, %xmm0, %xmm3
+; AVX1-NEXT: vblendvps %xmm3, %xmm1, %xmm2, %xmm2
+; AVX1-NEXT: vcmpunordss %xmm1, %xmm1, %xmm1
+; AVX1-NEXT: vblendvps %xmm1, %xmm0, %xmm2, %xmm0
; AVX1-NEXT: callq __truncsfbf2 at PLT
; AVX1-NEXT: vmovaps %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
-; AVX1-NEXT: shll $16, %r15d
-; AVX1-NEXT: vmovd %r15d, %xmm0
-; AVX1-NEXT: shll $16, %r14d
-; AVX1-NEXT: vmovd %r14d, %xmm1
-; AVX1-NEXT: vmaxss %xmm0, %xmm1, %xmm2
-; AVX1-NEXT: vorps {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm1, %xmm3
+; AVX1-NEXT: vbroadcastss {{.*#+}} xmm0 = [128,128,4,5,128,128,4,5,128,128,4,5,128,128,4,5]
+; AVX1-NEXT: vmovdqa (%rsp), %xmm1 # 16-byte Reload
+; AVX1-NEXT: vpshufb %xmm0, %xmm1, %xmm1
+; AVX1-NEXT: vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm2 # 16-byte Reload
+; AVX1-NEXT: vpshufb %xmm0, %xmm2, %xmm0
+; AVX1-NEXT: vmaxss %xmm1, %xmm0, %xmm2
+; AVX1-NEXT: vorps {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm0, %xmm3
; AVX1-NEXT: vandps %xmm2, %xmm3, %xmm2
-; AVX1-NEXT: vcmpunordss %xmm1, %xmm1, %xmm3
-; AVX1-NEXT: vblendvps %xmm3, %xmm0, %xmm2, %xmm2
-; AVX1-NEXT: vcmpunordss %xmm0, %xmm0, %xmm0
-; AVX1-NEXT: vblendvps %xmm0, %xmm1, %xmm2, %xmm0
+; AVX1-NEXT: vcmpunordss %xmm0, %xmm0, %xmm3
+; AVX1-NEXT: vblendvps %xmm3, %xmm1, %xmm2, %xmm2
+; AVX1-NEXT: vcmpunordss %xmm1, %xmm1, %xmm1
+; AVX1-NEXT: vblendvps %xmm1, %xmm0, %xmm2, %xmm0
; AVX1-NEXT: callq __truncsfbf2 at PLT
-; AVX1-NEXT: vmovaps %xmm0, (%rsp) # 16-byte Spill
-; AVX1-NEXT: shll $16, %ebp
-; AVX1-NEXT: vmovd %ebp, %xmm0
-; AVX1-NEXT: shll $16, %ebx
-; AVX1-NEXT: vmovd %ebx, %xmm1
-; AVX1-NEXT: vmaxss %xmm0, %xmm1, %xmm2
-; AVX1-NEXT: vorps {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm1, %xmm3
+; AVX1-NEXT: vmovaps %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
+; AVX1-NEXT: vbroadcastss {{.*#+}} xmm0 = [128,128,6,7,128,128,6,7,128,128,6,7,128,128,6,7]
+; AVX1-NEXT: vmovdqa (%rsp), %xmm1 # 16-byte Reload
+; AVX1-NEXT: vpshufb %xmm0, %xmm1, %xmm1
+; AVX1-NEXT: vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm2 # 16-byte Reload
+; AVX1-NEXT: vpshufb %xmm0, %xmm2, %xmm0
+; AVX1-NEXT: vmaxss %xmm1, %xmm0, %xmm2
+; AVX1-NEXT: vorps {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm0, %xmm3
; AVX1-NEXT: vandps %xmm2, %xmm3, %xmm2
-; AVX1-NEXT: vcmpunordss %xmm1, %xmm1, %xmm3
-; AVX1-NEXT: vblendvps %xmm3, %xmm0, %xmm2, %xmm2
-; AVX1-NEXT: vcmpunordss %xmm0, %xmm0, %xmm0
-; AVX1-NEXT: vblendvps %xmm0, %xmm1, %xmm2, %xmm0
+; AVX1-NEXT: vcmpunordss %xmm0, %xmm0, %xmm3
+; AVX1-NEXT: vblendvps %xmm3, %xmm1, %xmm2, %xmm2
+; AVX1-NEXT: vcmpunordss %xmm1, %xmm1, %xmm1
+; AVX1-NEXT: vblendvps %xmm1, %xmm0, %xmm2, %xmm0
; AVX1-NEXT: callq __truncsfbf2 at PLT
-; AVX1-NEXT: vmovdqa (%rsp), %xmm1 # 16-byte Reload
+; AVX1-NEXT: vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm1 # 16-byte Reload
; AVX1-NEXT: vpunpcklwd {{.*#+}} xmm0 = xmm1[0],xmm0[0],xmm1[1],xmm0[1],xmm1[2],xmm0[2],xmm1[3],xmm0[3]
; AVX1-NEXT: vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm1 # 16-byte Reload
; AVX1-NEXT: vpunpcklwd {{[-0-9]+}}(%r{{[sb]}}p), %xmm1, %xmm1 # 16-byte Folded Reload
; AVX1-NEXT: # xmm1 = xmm1[0],mem[0],xmm1[1],mem[1],xmm1[2],mem[2],xmm1[3],mem[3]
; AVX1-NEXT: vinsertps {{.*#+}} xmm0 = xmm1[0],xmm0[0],zero,zero
-; AVX1-NEXT: addq $56, %rsp
-; AVX1-NEXT: popq %rbx
-; AVX1-NEXT: popq %r12
-; AVX1-NEXT: popq %r13
-; AVX1-NEXT: popq %r14
-; AVX1-NEXT: popq %r15
-; AVX1-NEXT: popq %rbp
+; AVX1-NEXT: addq $88, %rsp
; AVX1-NEXT: retq
;
; AVX512F-LABEL: test_fmaximumnum_v4bf16:
; AVX512F: # %bb.0:
-; AVX512F-NEXT: pushq %rbp
; AVX512F-NEXT: pushq %r15
; AVX512F-NEXT: pushq %r14
-; AVX512F-NEXT: pushq %r13
; AVX512F-NEXT: pushq %r12
; AVX512F-NEXT: pushq %rbx
-; AVX512F-NEXT: pushq %rax
-; AVX512F-NEXT: vmovq %xmm1, %r13
-; AVX512F-NEXT: movq %r13, %rbx
+; AVX512F-NEXT: subq $56, %rsp
+; AVX512F-NEXT: vmovdqa %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
+; AVX512F-NEXT: vmovq %xmm1, %r15
+; AVX512F-NEXT: vmovdqa %xmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
+; AVX512F-NEXT: movq %r15, %rbx
; AVX512F-NEXT: shrq $32, %rbx
-; AVX512F-NEXT: vmovq %xmm0, %rbp
-; AVX512F-NEXT: movq %rbp, %r14
+; AVX512F-NEXT: vmovq %xmm0, %r12
+; AVX512F-NEXT: movq %r12, %r14
; AVX512F-NEXT: shrq $32, %r14
-; AVX512F-NEXT: movq %r13, %r15
; AVX512F-NEXT: shrq $48, %r15
-; AVX512F-NEXT: movq %rbp, %r12
; AVX512F-NEXT: shrq $48, %r12
-; AVX512F-NEXT: movl %ebp, %eax
-; AVX512F-NEXT: andl $-65536, %eax # imm = 0xFFFF0000
-; AVX512F-NEXT: vmovd %eax, %xmm1
-; AVX512F-NEXT: vpbroadcastd {{.*#+}} xmm0 = [NaN,NaN,NaN,NaN]
-; AVX512F-NEXT: vpor %xmm0, %xmm1, %xmm0
-; AVX512F-NEXT: movl %r13d, %eax
-; AVX512F-NEXT: andl $-65536, %eax # imm = 0xFFFF0000
-; AVX512F-NEXT: vmovd %eax, %xmm2
-; AVX512F-NEXT: vmaxss %xmm2, %xmm1, %xmm3
-; AVX512F-NEXT: vpand %xmm3, %xmm0, %xmm0
-; AVX512F-NEXT: vcmpunordss %xmm1, %xmm1, %k1
-; AVX512F-NEXT: vmovss %xmm2, %xmm0, %xmm0 {%k1}
+; AVX512F-NEXT: vpxor %xmm3, %xmm3, %xmm3
+; AVX512F-NEXT: vpblendw {{.*#+}} xmm1 = xmm3[0],xmm1[1],xmm3[2],xmm1[3],xmm3[4],xmm1[5],xmm3[6],xmm1[7]
+; AVX512F-NEXT: vpblendw {{.*#+}} xmm2 = xmm3[0],xmm0[1],xmm3[2],xmm0[3],xmm3[4],xmm0[5],xmm3[6],xmm0[7]
+; AVX512F-NEXT: vmaxss %xmm1, %xmm2, %xmm0
+; AVX512F-NEXT: vbroadcastss {{.*#+}} xmm3 = [NaN,NaN,NaN,NaN]
+; AVX512F-NEXT: vorps %xmm3, %xmm2, %xmm3
+; AVX512F-NEXT: vandps %xmm0, %xmm3, %xmm0
; AVX512F-NEXT: vcmpunordss %xmm2, %xmm2, %k1
; AVX512F-NEXT: vmovss %xmm1, %xmm0, %xmm0 {%k1}
+; AVX512F-NEXT: vcmpunordss %xmm1, %xmm1, %k1
+; AVX512F-NEXT: vmovss %xmm2, %xmm0, %xmm0 {%k1}
; AVX512F-NEXT: callq __truncsfbf2 at PLT
; AVX512F-NEXT: vpextrw $0, %xmm0, {{[0-9]+}}(%rsp)
-; AVX512F-NEXT: shll $16, %ebp
-; AVX512F-NEXT: vmovd %ebp, %xmm1
+; AVX512F-NEXT: vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 16-byte Reload
+; AVX512F-NEXT: vpslld $16, %xmm0, %xmm1
; AVX512F-NEXT: vpbroadcastd {{.*#+}} xmm0 = [NaN,NaN,NaN,NaN]
; AVX512F-NEXT: vpor %xmm0, %xmm1, %xmm0
-; AVX512F-NEXT: shll $16, %r13d
-; AVX512F-NEXT: vmovd %r13d, %xmm2
+; AVX512F-NEXT: vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm2 # 16-byte Reload
+; AVX512F-NEXT: vpslld $16, %xmm2, %xmm2
; AVX512F-NEXT: vmaxss %xmm2, %xmm1, %xmm3
; AVX512F-NEXT: vpand %xmm3, %xmm0, %xmm0
; AVX512F-NEXT: vcmpunordss %xmm1, %xmm1, %k1
@@ -3057,12 +3000,12 @@ define <4 x bfloat> @test_fmaximumnum_v4bf16(<4 x bfloat> %x, <4 x bfloat> %y) n
; AVX512F-NEXT: vmovss %xmm1, %xmm0, %xmm0 {%k1}
; AVX512F-NEXT: callq __truncsfbf2 at PLT
; AVX512F-NEXT: vpextrw $0, %xmm0, (%rsp)
-; AVX512F-NEXT: shll $16, %r12d
-; AVX512F-NEXT: vmovd %r12d, %xmm1
+; AVX512F-NEXT: vmovd %r12d, %xmm0
+; AVX512F-NEXT: vpslld $16, %xmm0, %xmm1
; AVX512F-NEXT: vpbroadcastd {{.*#+}} xmm0 = [NaN,NaN,NaN,NaN]
; AVX512F-NEXT: vpor %xmm0, %xmm1, %xmm0
-; AVX512F-NEXT: shll $16, %r15d
; AVX512F-NEXT: vmovd %r15d, %xmm2
+; AVX512F-NEXT: vpslld $16, %xmm2, %xmm2
; AVX512F-NEXT: vmaxss %xmm2, %xmm1, %xmm3
; AVX512F-NEXT: vpand %xmm3, %xmm0, %xmm0
; AVX512F-NEXT: vcmpunordss %xmm1, %xmm1, %k1
@@ -3071,12 +3014,12 @@ define <4 x bfloat> @test_fmaximumnum_v4bf16(<4 x bfloat> %x, <4 x bfloat> %y) n
; AVX512F-NEXT: vmovss %xmm1, %xmm0, %xmm0 {%k1}
; AVX512F-NEXT: callq __truncsfbf2 at PLT
; AVX512F-NEXT: vpextrw $0, %xmm0, {{[0-9]+}}(%rsp)
-; AVX512F-NEXT: shll $16, %r14d
-; AVX512F-NEXT: vmovd %r14d, %xmm1
+; AVX512F-NEXT: vmovd %r14d, %xmm0
+; AVX512F-NEXT: vpslld $16, %xmm0, %xmm1
; AVX512F-NEXT: vpbroadcastd {{.*#+}} xmm0 = [NaN,NaN,NaN,NaN]
; AVX512F-NEXT: vpor %xmm0, %xmm1, %xmm0
-; AVX512F-NEXT: shll $16, %ebx
; AVX512F-NEXT: vmovd %ebx, %xmm2
+; AVX512F-NEXT: vpslld $16, %xmm2, %xmm2
; AVX512F-NEXT: vmaxss %xmm2, %xmm1, %xmm3
; AVX512F-NEXT: vpand %xmm3, %xmm0, %xmm0
; AVX512F-NEXT: vcmpunordss %xmm1, %xmm1, %k1
@@ -3086,56 +3029,49 @@ define <4 x bfloat> @test_fmaximumnum_v4bf16(<4 x bfloat> %x, <4 x bfloat> %y) n
; AVX512F-NEXT: callq __truncsfbf2 at PLT
; AVX512F-NEXT: vpextrw $0, %xmm0, {{[0-9]+}}(%rsp)
; AVX512F-NEXT: vmovaps (%rsp), %xmm0
-; AVX512F-NEXT: addq $8, %rsp
+; AVX512F-NEXT: addq $56, %rsp
; AVX512F-NEXT: popq %rbx
; AVX512F-NEXT: popq %r12
-; AVX512F-NEXT: popq %r13
; AVX512F-NEXT: popq %r14
; AVX512F-NEXT: popq %r15
-; AVX512F-NEXT: popq %rbp
; AVX512F-NEXT: retq
;
; AVX512DQ-LABEL: test_fmaximumnum_v4bf16:
; AVX512DQ: # %bb.0:
-; AVX512DQ-NEXT: pushq %rbp
; AVX512DQ-NEXT: pushq %r15
; AVX512DQ-NEXT: pushq %r14
-; AVX512DQ-NEXT: pushq %r13
; AVX512DQ-NEXT: pushq %r12
; AVX512DQ-NEXT: pushq %rbx
-; AVX512DQ-NEXT: pushq %rax
-; AVX512DQ-NEXT: vmovq %xmm1, %r13
-; AVX512DQ-NEXT: movq %r13, %rbx
+; AVX512DQ-NEXT: subq $56, %rsp
+; AVX512DQ-NEXT: vmovdqa %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
+; AVX512DQ-NEXT: vmovq %xmm1, %r15
+; AVX512DQ-NEXT: vmovdqa %xmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
+; AVX512DQ-NEXT: movq %r15, %rbx
; AVX512DQ-NEXT: shrq $32, %rbx
-; AVX512DQ-NEXT: vmovq %xmm0, %rbp
-; AVX512DQ-NEXT: movq %rbp, %r14
+; AVX512DQ-NEXT: vmovq %xmm0, %r12
+; AVX512DQ-NEXT: movq %r12, %r14
; AVX512DQ-NEXT: shrq $32, %r14
-; AVX512DQ-NEXT: movq %r13, %r15
; AVX512DQ-NEXT: shrq $48, %r15
-; AVX512DQ-NEXT: movq %rbp, %r12
; AVX512DQ-NEXT: shrq $48, %r12
-; AVX512DQ-NEXT: movl %ebp, %eax
-; AVX512DQ-NEXT: andl $-65536, %eax # imm = 0xFFFF0000
-; AVX512DQ-NEXT: vmovd %eax, %xmm1
-; AVX512DQ-NEXT: vpbroadcastd {{.*#+}} xmm0 = [NaN,NaN,NaN,NaN]
-; AVX512DQ-NEXT: vpor %xmm0, %xmm1, %xmm0
-; AVX512DQ-NEXT: movl %r13d, %eax
-; AVX512DQ-NEXT: andl $-65536, %eax # imm = 0xFFFF0000
-; AVX512DQ-NEXT: vmovd %eax, %xmm2
-; AVX512DQ-NEXT: vmaxss %xmm2, %xmm1, %xmm3
-; AVX512DQ-NEXT: vpand %xmm3, %xmm0, %xmm0
-; AVX512DQ-NEXT: vcmpunordss %xmm1, %xmm1, %k1
-; AVX512DQ-NEXT: vmovss %xmm2, %xmm0, %xmm0 {%k1}
+; AVX512DQ-NEXT: vpxor %xmm3, %xmm3, %xmm3
+; AVX512DQ-NEXT: vpblendw {{.*#+}} xmm1 = xmm3[0],xmm1[1],xmm3[2],xmm1[3],xmm3[4],xmm1[5],xmm3[6],xmm1[7]
+; AVX512DQ-NEXT: vpblendw {{.*#+}} xmm2 = xmm3[0],xmm0[1],xmm3[2],xmm0[3],xmm3[4],xmm0[5],xmm3[6],xmm0[7]
+; AVX512DQ-NEXT: vmaxss %xmm1, %xmm2, %xmm0
+; AVX512DQ-NEXT: vbroadcastss {{.*#+}} xmm3 = [NaN,NaN,NaN,NaN]
+; AVX512DQ-NEXT: vorps %xmm3, %xmm2, %xmm3
+; AVX512DQ-NEXT: vandps %xmm0, %xmm3, %xmm0
; AVX512DQ-NEXT: vcmpunordss %xmm2, %xmm2, %k1
; AVX512DQ-NEXT: vmovss %xmm1, %xmm0, %xmm0 {%k1}
+; AVX512DQ-NEXT: vcmpunordss %xmm1, %xmm1, %k1
+; AVX512DQ-NEXT: vmovss %xmm2, %xmm0, %xmm0 {%k1}
; AVX512DQ-NEXT: callq __truncsfbf2 at PLT
; AVX512DQ-NEXT: vpextrw $0, %xmm0, {{[0-9]+}}(%rsp)
-; AVX512DQ-NEXT: shll $16, %ebp
-; AVX512DQ-NEXT: vmovd %ebp, %xmm1
+; AVX512DQ-NEXT: vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 16-byte Reload
+; AVX512DQ-NEXT: vpslld $16, %xmm0, %xmm1
; AVX512DQ-NEXT: vpbroadcastd {{.*#+}} xmm0 = [NaN,NaN,NaN,NaN]
; AVX512DQ-NEXT: vpor %xmm0, %xmm1, %xmm0
-; AVX512DQ-NEXT: shll $16, %r13d
-; AVX512DQ-NEXT: vmovd %r13d, %xmm2
+; AVX512DQ-NEXT: vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm2 # 16-byte Reload
+; AVX512DQ-NEXT: vpslld $16, %xmm2, %xmm2
; AVX512DQ-NEXT: vmaxss %xmm2, %xmm1, %xmm3
; AVX512DQ-NEXT: vpand %xmm3, %xmm0, %xmm0
; AVX512DQ-NEXT: vcmpunordss %xmm1, %xmm1, %k1
@@ -3144,12 +3080,12 @@ define <4 x bfloat> @test_fmaximumnum_v4bf16(<4 x bfloat> %x, <4 x bfloat> %y) n
; AVX512DQ-NEXT: vmovss %xmm1, %xmm0, %xmm0 {%k1}
; AVX512DQ-NEXT: callq __truncsfbf2 at PLT
; AVX512DQ-NEXT: vpextrw $0, %xmm0, (%rsp)
-; AVX512DQ-NEXT: shll $16, %r12d
-; AVX512DQ-NEXT: vmovd %r12d, %xmm1
+; AVX512DQ-NEXT: vmovd %r12d, %xmm0
+; AVX512DQ-NEXT: vpslld $16, %xmm0, %xmm1
; AVX512DQ-NEXT: vpbroadcastd {{.*#+}} xmm0 = [NaN,NaN,NaN,NaN]
; AVX512DQ-NEXT: vpor %xmm0, %xmm1, %xmm0
-; AVX512DQ-NEXT: shll $16, %r15d
; AVX512DQ-NEXT: vmovd %r15d, %xmm2
+; AVX512DQ-NEXT: vpslld $16, %xmm2, %xmm2
; AVX512DQ-NEXT: vmaxss %xmm2, %xmm1, %xmm3
; AVX512DQ-NEXT: vpand %xmm3, %xmm0, %xmm0
; AVX512DQ-NEXT: vcmpunordss %xmm1, %xmm1, %k1
@@ -3158,12 +3094,12 @@ define <4 x bfloat> @test_fmaximumnum_v4bf16(<4 x bfloat> %x, <4 x bfloat> %y) n
; AVX512DQ-NEXT: vmovss %xmm1, %xmm0, %xmm0 {%k1}
; AVX512DQ-NEXT: callq __truncsfbf2 at PLT
; AVX512DQ-NEXT: vpextrw $0, %xmm0, {{[0-9]+}}(%rsp)
-; AVX512DQ-NEXT: shll $16, %r14d
-; AVX512DQ-NEXT: vmovd %r14d, %xmm1
+; AVX512DQ-NEXT: vmovd %r14d, %xmm0
+; AVX512DQ-NEXT: vpslld $16, %xmm0, %xmm1
; AVX512DQ-NEXT: vpbroadcastd {{.*#+}} xmm0 = [NaN,NaN,NaN,NaN]
; AVX512DQ-NEXT: vpor %xmm0, %xmm1, %xmm0
-; AVX512DQ-NEXT: shll $16, %ebx
; AVX512DQ-NEXT: vmovd %ebx, %xmm2
+; AVX512DQ-NEXT: vpslld $16, %xmm2, %xmm2
; AVX512DQ-NEXT: vmaxss %xmm2, %xmm1, %xmm3
; AVX512DQ-NEXT: vpand %xmm3, %xmm0, %xmm0
; AVX512DQ-NEXT: vcmpunordss %xmm1, %xmm1, %k1
@@ -3173,13 +3109,11 @@ define <4 x bfloat> @test_fmaximumnum_v4bf16(<4 x bfloat> %x, <4 x bfloat> %y) n
; AVX512DQ-NEXT: callq __truncsfbf2 at PLT
; AVX512DQ-NEXT: vpextrw $0, %xmm0, {{[0-9]+}}(%rsp)
; AVX512DQ-NEXT: vmovaps (%rsp), %xmm0
-; AVX512DQ-NEXT: addq $8, %rsp
+; AVX512DQ-NEXT: addq $56, %rsp
; AVX512DQ-NEXT: popq %rbx
; AVX512DQ-NEXT: popq %r12
-; AVX512DQ-NEXT: popq %r13
; AVX512DQ-NEXT: popq %r14
; AVX512DQ-NEXT: popq %r15
-; AVX512DQ-NEXT: popq %rbp
; AVX512DQ-NEXT: retq
;
; AVX512BF16-LABEL: test_fmaximumnum_v4bf16:
@@ -3190,193 +3124,151 @@ define <4 x bfloat> @test_fmaximumnum_v4bf16(<4 x bfloat> %x, <4 x bfloat> %y) n
; AVX512BF16-NEXT: pushq %r13
; AVX512BF16-NEXT: pushq %r12
; AVX512BF16-NEXT: pushq %rbx
-; AVX512BF16-NEXT: vpextrw $7, %xmm1, %eax
-; AVX512BF16-NEXT: vpextrw $7, %xmm0, %ebx
-; AVX512BF16-NEXT: movl %ebx, %ecx
-; AVX512BF16-NEXT: shll $16, %ecx
-; AVX512BF16-NEXT: vmovd %ecx, %xmm2
+; AVX512BF16-NEXT: vpextrw $7, %xmm1, %r9d
+; AVX512BF16-NEXT: vpextrw $7, %xmm0, %r13d
+; AVX512BF16-NEXT: vmovd %r13d, %xmm2
+; AVX512BF16-NEXT: vpslld $16, %xmm2, %xmm2
; AVX512BF16-NEXT: vucomiss %xmm2, %xmm2
-; AVX512BF16-NEXT: cmovpl %eax, %ebx
-; AVX512BF16-NEXT: movl %eax, %ecx
-; AVX512BF16-NEXT: vpextrw $6, %xmm1, %eax
-; AVX512BF16-NEXT: movl %eax, {{[-0-9]+}}(%r{{[sb]}}p) # 4-byte Spill
-; AVX512BF16-NEXT: vpextrw $6, %xmm0, %r13d
-; AVX512BF16-NEXT: movl %r13d, %edx
-; AVX512BF16-NEXT: shll $16, %edx
-; AVX512BF16-NEXT: vmovd %edx, %xmm2
-; AVX512BF16-NEXT: vucomiss %xmm2, %xmm2
-; AVX512BF16-NEXT: cmovpl %eax, %r13d
-; AVX512BF16-NEXT: vpextrw $5, %xmm1, %eax
-; AVX512BF16-NEXT: movl %eax, {{[-0-9]+}}(%r{{[sb]}}p) # 4-byte Spill
-; AVX512BF16-NEXT: vpextrw $5, %xmm0, %r12d
-; AVX512BF16-NEXT: movl %r12d, %esi
-; AVX512BF16-NEXT: shll $16, %esi
-; AVX512BF16-NEXT: vmovd %esi, %xmm2
+; AVX512BF16-NEXT: cmovpl %r9d, %r13d
+; AVX512BF16-NEXT: vpextrw $6, %xmm1, %ecx
+; AVX512BF16-NEXT: vpextrw $6, %xmm0, %r12d
+; AVX512BF16-NEXT: vmovd %r12d, %xmm2
+; AVX512BF16-NEXT: vpslld $16, %xmm2, %xmm2
; AVX512BF16-NEXT: vucomiss %xmm2, %xmm2
-; AVX512BF16-NEXT: cmovpl %eax, %r12d
-; AVX512BF16-NEXT: vpextrw $4, %xmm1, %esi
-; AVX512BF16-NEXT: vpextrw $4, %xmm0, %r15d
-; AVX512BF16-NEXT: movl %r15d, %edi
-; AVX512BF16-NEXT: shll $16, %edi
-; AVX512BF16-NEXT: vmovd %edi, %xmm2
+; AVX512BF16-NEXT: cmovpl %ecx, %r12d
+; AVX512BF16-NEXT: vpextrw $5, %xmm1, %edx
+; AVX512BF16-NEXT: vpextrw $5, %xmm0, %r15d
+; AVX512BF16-NEXT: vmovd %r15d, %xmm2
+; AVX512BF16-NEXT: vpslld $16, %xmm2, %xmm2
; AVX512BF16-NEXT: vucomiss %xmm2, %xmm2
-; AVX512BF16-NEXT: cmovpl %esi, %r15d
-; AVX512BF16-NEXT: vpextrw $3, %xmm1, %edi
-; AVX512BF16-NEXT: vpextrw $3, %xmm0, %r14d
-; AVX512BF16-NEXT: movl %r14d, %r8d
-; AVX512BF16-NEXT: shll $16, %r8d
-; AVX512BF16-NEXT: vmovd %r8d, %xmm2
+; AVX512BF16-NEXT: cmovpl %edx, %r15d
+; AVX512BF16-NEXT: vpextrw $3, %xmm1, %esi
+; AVX512BF16-NEXT: vpextrw $3, %xmm0, %ebp
+; AVX512BF16-NEXT: vmovd %ebp, %xmm2
+; AVX512BF16-NEXT: vpslld $16, %xmm2, %xmm2
; AVX512BF16-NEXT: vucomiss %xmm2, %xmm2
-; AVX512BF16-NEXT: cmovpl %edi, %r14d
-; AVX512BF16-NEXT: vpextrw $2, %xmm1, %r8d
-; AVX512BF16-NEXT: vpextrw $2, %xmm0, %ebp
-; AVX512BF16-NEXT: movl %ebp, %r9d
-; AVX512BF16-NEXT: shll $16, %r9d
-; AVX512BF16-NEXT: vmovd %r9d, %xmm2
+; AVX512BF16-NEXT: cmovpl %esi, %ebp
+; AVX512BF16-NEXT: vpextrw $2, %xmm1, %edi
+; AVX512BF16-NEXT: vpextrw $2, %xmm0, %ebx
+; AVX512BF16-NEXT: vmovd %ebx, %xmm2
+; AVX512BF16-NEXT: vpslld $16, %xmm2, %xmm2
; AVX512BF16-NEXT: vucomiss %xmm2, %xmm2
-; AVX512BF16-NEXT: cmovpl %r8d, %ebp
-; AVX512BF16-NEXT: vpextrw $1, %xmm1, %r9d
-; AVX512BF16-NEXT: vpextrw $1, %xmm0, %edx
-; AVX512BF16-NEXT: movl %edx, %r10d
-; AVX512BF16-NEXT: shll $16, %r10d
-; AVX512BF16-NEXT: vmovd %r10d, %xmm2
+; AVX512BF16-NEXT: cmovpl %edi, %ebx
+; AVX512BF16-NEXT: vmovd %xmm1, %r8d
+; AVX512BF16-NEXT: vmovd %xmm0, %eax
+; AVX512BF16-NEXT: vpslld $16, %xmm0, %xmm2
; AVX512BF16-NEXT: vucomiss %xmm2, %xmm2
-; AVX512BF16-NEXT: cmovpl %r9d, %edx
-; AVX512BF16-NEXT: vmovd %xmm0, %r10d
-; AVX512BF16-NEXT: movl %r10d, %r11d
-; AVX512BF16-NEXT: shll $16, %r11d
-; AVX512BF16-NEXT: vmovd %r11d, %xmm0
+; AVX512BF16-NEXT: cmovpl %r8d, %eax
+; AVX512BF16-NEXT: movl %eax, {{[-0-9]+}}(%r{{[sb]}}p) # 4-byte Spill
+; AVX512BF16-NEXT: vmovd %eax, %xmm3
+; AVX512BF16-NEXT: vpxor %xmm2, %xmm2, %xmm2
+; AVX512BF16-NEXT: vpunpckhwd {{.*#+}} xmm4 = xmm2[4],xmm0[4],xmm2[5],xmm0[5],xmm2[6],xmm0[6],xmm2[7],xmm0[7]
+; AVX512BF16-NEXT: vpextrw $4, %xmm1, %r10d
+; AVX512BF16-NEXT: vpextrw $4, %xmm0, %eax
+; AVX512BF16-NEXT: vucomiss %xmm4, %xmm4
+; AVX512BF16-NEXT: cmovpl %r10d, %eax
+; AVX512BF16-NEXT: vpblendw {{.*#+}} xmm4 = xmm2[0],xmm0[1],xmm2[2],xmm0[3],xmm2[4],xmm0[5],xmm2[6],xmm0[7]
+; AVX512BF16-NEXT: vpextrw $1, %xmm1, %r11d
+; AVX512BF16-NEXT: vucomiss %xmm4, %xmm4
+; AVX512BF16-NEXT: vpextrw $1, %xmm0, %r14d
+; AVX512BF16-NEXT: cmovpl %r11d, %r14d
+; AVX512BF16-NEXT: vpinsrw $1, %r14d, %xmm3, %xmm0
+; AVX512BF16-NEXT: vpinsrw $2, %ebx, %xmm0, %xmm0
+; AVX512BF16-NEXT: vpinsrw $3, %ebp, %xmm0, %xmm0
+; AVX512BF16-NEXT: vpinsrw $4, %eax, %xmm0, %xmm0
+; AVX512BF16-NEXT: vpinsrw $5, %r15d, %xmm0, %xmm0
+; AVX512BF16-NEXT: vpinsrw $6, %r12d, %xmm0, %xmm0
+; AVX512BF16-NEXT: vpinsrw $7, %r13d, %xmm0, %xmm4
+; AVX512BF16-NEXT: vmovd %r13d, %xmm0
+; AVX512BF16-NEXT: vpslld $16, %xmm0, %xmm0
+; AVX512BF16-NEXT: vmovd %r9d, %xmm5
+; AVX512BF16-NEXT: vpslld $16, %xmm5, %xmm5
+; AVX512BF16-NEXT: vucomiss %xmm5, %xmm5
+; AVX512BF16-NEXT: cmovpl %r13d, %r9d
+; AVX512BF16-NEXT: vmovd %r9d, %xmm5
+; AVX512BF16-NEXT: vpslld $16, %xmm5, %xmm5
+; AVX512BF16-NEXT: vucomiss %xmm5, %xmm0
+; AVX512BF16-NEXT: cmoval %r13d, %r9d
+; AVX512BF16-NEXT: vmovd %r12d, %xmm0
+; AVX512BF16-NEXT: vpslld $16, %xmm0, %xmm0
+; AVX512BF16-NEXT: vmovd %ecx, %xmm5
+; AVX512BF16-NEXT: vpslld $16, %xmm5, %xmm5
+; AVX512BF16-NEXT: vucomiss %xmm5, %xmm5
+; AVX512BF16-NEXT: cmovpl %r12d, %ecx
+; AVX512BF16-NEXT: vmovd %ecx, %xmm5
+; AVX512BF16-NEXT: vpslld $16, %xmm5, %xmm5
+; AVX512BF16-NEXT: vucomiss %xmm5, %xmm0
+; AVX512BF16-NEXT: cmoval %r12d, %ecx
+; AVX512BF16-NEXT: vmovd %r15d, %xmm0
+; AVX512BF16-NEXT: vpslld $16, %xmm0, %xmm0
+; AVX512BF16-NEXT: vmovd %edx, %xmm5
+; AVX512BF16-NEXT: vpslld $16, %xmm5, %xmm5
+; AVX512BF16-NEXT: vucomiss %xmm5, %xmm5
+; AVX512BF16-NEXT: cmovpl %r15d, %edx
+; AVX512BF16-NEXT: vmovd %edx, %xmm5
+; AVX512BF16-NEXT: vpslld $16, %xmm5, %xmm5
+; AVX512BF16-NEXT: vucomiss %xmm5, %xmm0
+; AVX512BF16-NEXT: cmoval %r15d, %edx
+; AVX512BF16-NEXT: vpunpckhwd {{.*#+}} xmm0 = xmm2[4],xmm1[4],xmm2[5],xmm1[5],xmm2[6],xmm1[6],xmm2[7],xmm1[7]
; AVX512BF16-NEXT: vucomiss %xmm0, %xmm0
-; AVX512BF16-NEXT: vmovd %xmm1, %r11d
-; AVX512BF16-NEXT: cmovpl %r11d, %r10d
-; AVX512BF16-NEXT: movl %ebx, %eax
-; AVX512BF16-NEXT: shll $16, %eax
-; AVX512BF16-NEXT: vmovd %eax, %xmm0
-; AVX512BF16-NEXT: movl %ecx, %eax
-; AVX512BF16-NEXT: shll $16, %eax
-; AVX512BF16-NEXT: vmovd %eax, %xmm1
-; AVX512BF16-NEXT: vucomiss %xmm1, %xmm1
-; AVX512BF16-NEXT: cmovpl %ebx, %ecx
-; AVX512BF16-NEXT: movl %ecx, %eax
-; AVX512BF16-NEXT: shll $16, %eax
-; AVX512BF16-NEXT: vmovd %eax, %xmm2
-; AVX512BF16-NEXT: vmovd %r10d, %xmm1
-; AVX512BF16-NEXT: vpinsrw $1, %edx, %xmm1, %xmm1
-; AVX512BF16-NEXT: vpinsrw $2, %ebp, %xmm1, %xmm1
-; AVX512BF16-NEXT: vpinsrw $3, %r14d, %xmm1, %xmm1
-; AVX512BF16-NEXT: vpinsrw $4, %r15d, %xmm1, %xmm1
-; AVX512BF16-NEXT: vpinsrw $5, %r12d, %xmm1, %xmm1
-; AVX512BF16-NEXT: vpinsrw $6, %r13d, %xmm1, %xmm1
-; AVX512BF16-NEXT: vpinsrw $7, %ebx, %xmm1, %xmm1
-; AVX512BF16-NEXT: vucomiss %xmm2, %xmm0
-; AVX512BF16-NEXT: cmoval %ebx, %ecx
-; AVX512BF16-NEXT: movl %r13d, %eax
-; AVX512BF16-NEXT: shll $16, %eax
-; AVX512BF16-NEXT: vmovd %eax, %xmm0
-; AVX512BF16-NEXT: movl {{[-0-9]+}}(%r{{[sb]}}p), %ebx # 4-byte Reload
-; AVX512BF16-NEXT: movl %ebx, %eax
-; AVX512BF16-NEXT: shll $16, %eax
-; AVX512BF16-NEXT: vmovd %eax, %xmm2
-; AVX512BF16-NEXT: vucomiss %xmm2, %xmm2
-; AVX512BF16-NEXT: cmovpl %r13d, %ebx
-; AVX512BF16-NEXT: movl %ebx, %eax
-; AVX512BF16-NEXT: shll $16, %eax
-; AVX512BF16-NEXT: vmovd %eax, %xmm2
-; AVX512BF16-NEXT: vucomiss %xmm2, %xmm0
-; AVX512BF16-NEXT: cmoval %r13d, %ebx
-; AVX512BF16-NEXT: movl %r12d, %eax
-; AVX512BF16-NEXT: shll $16, %eax
-; AVX512BF16-NEXT: vmovd %eax, %xmm0
-; AVX512BF16-NEXT: movl {{[-0-9]+}}(%r{{[sb]}}p), %r13d # 4-byte Reload
-; AVX512BF16-NEXT: movl %r13d, %eax
-; AVX512BF16-NEXT: shll $16, %eax
-; AVX512BF16-NEXT: vmovd %eax, %xmm2
-; AVX512BF16-NEXT: vucomiss %xmm2, %xmm2
-; AVX512BF16-NEXT: cmovpl %r12d, %r13d
-; AVX512BF16-NEXT: movl %r13d, %eax
-; AVX512BF16-NEXT: shll $16, %eax
-; AVX512BF16-NEXT: vmovd %eax, %xmm2
-; AVX512BF16-NEXT: vucomiss %xmm2, %xmm0
-; AVX512BF16-NEXT: cmoval %r12d, %r13d
-; AVX512BF16-NEXT: movl %r15d, %eax
-; AVX512BF16-NEXT: shll $16, %eax
-; AVX512BF16-NEXT: vmovd %eax, %xmm0
-; AVX512BF16-NEXT: movl %esi, %eax
-; AVX512BF16-NEXT: shll $16, %eax
-; AVX512BF16-NEXT: vmovd %eax, %xmm2
-; AVX512BF16-NEXT: vucomiss %xmm2, %xmm2
-; AVX512BF16-NEXT: cmovpl %r15d, %esi
-; AVX512BF16-NEXT: movl %esi, %eax
-; AVX512BF16-NEXT: shll $16, %eax
-; AVX512BF16-NEXT: vmovd %eax, %xmm2
-; AVX512BF16-NEXT: vucomiss %xmm2, %xmm0
-; AVX512BF16-NEXT: cmoval %r15d, %esi
-; AVX512BF16-NEXT: movl %r14d, %eax
-; AVX512BF16-NEXT: shll $16, %eax
-; AVX512BF16-NEXT: vmovd %eax, %xmm0
-; AVX512BF16-NEXT: movl %edi, %eax
-; AVX512BF16-NEXT: shll $16, %eax
-; AVX512BF16-NEXT: vmovd %eax, %xmm2
-; AVX512BF16-NEXT: vucomiss %xmm2, %xmm2
-; AVX512BF16-NEXT: cmovpl %r14d, %edi
-; AVX512BF16-NEXT: movl %edi, %eax
-; AVX512BF16-NEXT: shll $16, %eax
-; AVX512BF16-NEXT: vmovd %eax, %xmm2
-; AVX512BF16-NEXT: vucomiss %xmm2, %xmm0
-; AVX512BF16-NEXT: cmoval %r14d, %edi
-; AVX512BF16-NEXT: movl %ebp, %eax
-; AVX512BF16-NEXT: shll $16, %eax
-; AVX512BF16-NEXT: vmovd %eax, %xmm0
-; AVX512BF16-NEXT: movl %r8d, %eax
-; AVX512BF16-NEXT: shll $16, %eax
-; AVX512BF16-NEXT: vmovd %eax, %xmm2
-; AVX512BF16-NEXT: vucomiss %xmm2, %xmm2
-; AVX512BF16-NEXT: cmovpl %ebp, %r8d
-; AVX512BF16-NEXT: movl %r8d, %eax
-; AVX512BF16-NEXT: shll $16, %eax
-; AVX512BF16-NEXT: vmovd %eax, %xmm2
-; AVX512BF16-NEXT: vucomiss %xmm2, %xmm0
-; AVX512BF16-NEXT: cmoval %ebp, %r8d
-; AVX512BF16-NEXT: movl %edx, %eax
-; AVX512BF16-NEXT: shll $16, %eax
-; AVX512BF16-NEXT: vmovd %eax, %xmm0
-; AVX512BF16-NEXT: movl %r9d, %eax
-; AVX512BF16-NEXT: shll $16, %eax
-; AVX512BF16-NEXT: vmovd %eax, %xmm2
-; AVX512BF16-NEXT: vucomiss %xmm2, %xmm2
-; AVX512BF16-NEXT: cmovpl %edx, %r9d
-; AVX512BF16-NEXT: movl %r9d, %eax
-; AVX512BF16-NEXT: shll $16, %eax
-; AVX512BF16-NEXT: vmovd %eax, %xmm2
-; AVX512BF16-NEXT: vucomiss %xmm2, %xmm0
-; AVX512BF16-NEXT: cmoval %edx, %r9d
-; AVX512BF16-NEXT: movl %r10d, %eax
-; AVX512BF16-NEXT: shll $16, %eax
-; AVX512BF16-NEXT: vmovd %eax, %xmm0
-; AVX512BF16-NEXT: movl %r11d, %eax
-; AVX512BF16-NEXT: shll $16, %eax
-; AVX512BF16-NEXT: vmovd %eax, %xmm2
-; AVX512BF16-NEXT: vucomiss %xmm2, %xmm2
-; AVX512BF16-NEXT: cmovpl %r10d, %r11d
-; AVX512BF16-NEXT: movl %r11d, %eax
-; AVX512BF16-NEXT: shll $16, %eax
-; AVX512BF16-NEXT: vmovd %eax, %xmm2
-; AVX512BF16-NEXT: vucomiss %xmm2, %xmm0
-; AVX512BF16-NEXT: cmoval %r10d, %r11d
+; AVX512BF16-NEXT: cmovpl %eax, %r10d
+; AVX512BF16-NEXT: vmovd %r10d, %xmm0
+; AVX512BF16-NEXT: vpslld $16, %xmm0, %xmm0
+; AVX512BF16-NEXT: vmovd %eax, %xmm5
+; AVX512BF16-NEXT: vpslld $16, %xmm5, %xmm5
+; AVX512BF16-NEXT: vucomiss %xmm0, %xmm5
+; AVX512BF16-NEXT: cmoval %eax, %r10d
+; AVX512BF16-NEXT: vmovd %ebp, %xmm0
+; AVX512BF16-NEXT: vpslld $16, %xmm0, %xmm0
+; AVX512BF16-NEXT: vmovd %esi, %xmm5
+; AVX512BF16-NEXT: vpslld $16, %xmm5, %xmm5
+; AVX512BF16-NEXT: vucomiss %xmm5, %xmm5
+; AVX512BF16-NEXT: cmovpl %ebp, %esi
+; AVX512BF16-NEXT: vmovd %esi, %xmm5
+; AVX512BF16-NEXT: vpslld $16, %xmm5, %xmm5
+; AVX512BF16-NEXT: vucomiss %xmm5, %xmm0
+; AVX512BF16-NEXT: cmoval %ebp, %esi
+; AVX512BF16-NEXT: vmovd %ebx, %xmm0
+; AVX512BF16-NEXT: vpslld $16, %xmm0, %xmm0
+; AVX512BF16-NEXT: vmovd %edi, %xmm5
+; AVX512BF16-NEXT: vpslld $16, %xmm5, %xmm5
+; AVX512BF16-NEXT: vucomiss %xmm5, %xmm5
+; AVX512BF16-NEXT: cmovpl %ebx, %edi
+; AVX512BF16-NEXT: vmovd %edi, %xmm5
+; AVX512BF16-NEXT: vpslld $16, %xmm5, %xmm5
+; AVX512BF16-NEXT: vucomiss %xmm5, %xmm0
+; AVX512BF16-NEXT: cmoval %ebx, %edi
+; AVX512BF16-NEXT: vpblendw {{.*#+}} xmm0 = xmm2[0],xmm1[1],xmm2[2],xmm1[3],xmm2[4],xmm1[5],xmm2[6],xmm1[7]
+; AVX512BF16-NEXT: vucomiss %xmm0, %xmm0
+; AVX512BF16-NEXT: cmovpl %r14d, %r11d
; AVX512BF16-NEXT: vmovd %r11d, %xmm0
-; AVX512BF16-NEXT: vpinsrw $1, %r9d, %xmm0, %xmm0
-; AVX512BF16-NEXT: vpinsrw $2, %r8d, %xmm0, %xmm0
-; AVX512BF16-NEXT: vpinsrw $3, %edi, %xmm0, %xmm0
-; AVX512BF16-NEXT: vpinsrw $4, %esi, %xmm0, %xmm0
-; AVX512BF16-NEXT: vpinsrw $5, %r13d, %xmm0, %xmm0
-; AVX512BF16-NEXT: vpinsrw $6, %ebx, %xmm0, %xmm0
-; AVX512BF16-NEXT: vpinsrw $7, %ecx, %xmm0, %xmm0
-; AVX512BF16-NEXT: vptestnmw %xmm1, %xmm1, %k1
-; AVX512BF16-NEXT: vpblendmw %xmm1, %xmm0, %xmm1 {%k1}
-; AVX512BF16-NEXT: vpmovzxwd {{.*#+}} ymm2 = xmm0[0],zero,xmm0[1],zero,xmm0[2],zero,xmm0[3],zero,xmm0[4],zero,xmm0[5],zero,xmm0[6],zero,xmm0[7],zero
-; AVX512BF16-NEXT: vpslld $16, %ymm2, %ymm2
-; AVX512BF16-NEXT: vxorps %xmm3, %xmm3, %xmm3
-; AVX512BF16-NEXT: vcmpeqps %ymm3, %ymm2, %k1
+; AVX512BF16-NEXT: vpslld $16, %xmm0, %xmm0
+; AVX512BF16-NEXT: vmovd %r14d, %xmm5
+; AVX512BF16-NEXT: vpslld $16, %xmm5, %xmm5
+; AVX512BF16-NEXT: vucomiss %xmm0, %xmm5
+; AVX512BF16-NEXT: cmoval %r14d, %r11d
+; AVX512BF16-NEXT: vpslld $16, %xmm3, %xmm0
+; AVX512BF16-NEXT: vpslld $16, %xmm1, %xmm1
+; AVX512BF16-NEXT: vucomiss %xmm1, %xmm1
+; AVX512BF16-NEXT: movl {{[-0-9]+}}(%r{{[sb]}}p), %eax # 4-byte Reload
+; AVX512BF16-NEXT: cmovpl %eax, %r8d
+; AVX512BF16-NEXT: vmovd %r8d, %xmm1
+; AVX512BF16-NEXT: vpslld $16, %xmm1, %xmm1
+; AVX512BF16-NEXT: vucomiss %xmm1, %xmm0
+; AVX512BF16-NEXT: cmoval %eax, %r8d
+; AVX512BF16-NEXT: vmovd %r8d, %xmm0
+; AVX512BF16-NEXT: vpinsrw $1, %r11d, %xmm0, %xmm0
+; AVX512BF16-NEXT: vpinsrw $2, %edi, %xmm0, %xmm0
+; AVX512BF16-NEXT: vpinsrw $3, %esi, %xmm0, %xmm0
+; AVX512BF16-NEXT: vpinsrw $4, %r10d, %xmm0, %xmm0
+; AVX512BF16-NEXT: vpinsrw $5, %edx, %xmm0, %xmm0
+; AVX512BF16-NEXT: vpinsrw $6, %ecx, %xmm0, %xmm0
+; AVX512BF16-NEXT: vpinsrw $7, %r9d, %xmm0, %xmm0
+; AVX512BF16-NEXT: vptestnmw %xmm4, %xmm4, %k1
+; AVX512BF16-NEXT: vpblendmw %xmm4, %xmm0, %xmm1 {%k1}
+; AVX512BF16-NEXT: vpmovzxwd {{.*#+}} ymm3 = xmm0[0],zero,xmm0[1],zero,xmm0[2],zero,xmm0[3],zero,xmm0[4],zero,xmm0[5],zero,xmm0[6],zero,xmm0[7],zero
+; AVX512BF16-NEXT: vpslld $16, %ymm3, %ymm3
+; AVX512BF16-NEXT: vcmpeqps %ymm2, %ymm3, %k1
; AVX512BF16-NEXT: vmovdqu16 %xmm1, %xmm0 {%k1}
; AVX512BF16-NEXT: popq %rbx
; AVX512BF16-NEXT: popq %r12
@@ -3394,81 +3286,66 @@ define <4 x bfloat> @test_fmaximumnum_v4bf16(<4 x bfloat> %x, <4 x bfloat> %y) n
;
; X86-LABEL: test_fmaximumnum_v4bf16:
; X86: # %bb.0:
-; X86-NEXT: pushl %ebp
-; X86-NEXT: pushl %ebx
-; X86-NEXT: pushl %edi
-; X86-NEXT: pushl %esi
-; X86-NEXT: subl $68, %esp
-; X86-NEXT: vpsrlq $48, %xmm0, %xmm2
-; X86-NEXT: vpextrw $0, %xmm2, %esi
-; X86-NEXT: vpsrlq $48, %xmm1, %xmm2
-; X86-NEXT: vpextrw $0, %xmm2, %edi
-; X86-NEXT: vmovshdup {{.*#+}} xmm2 = xmm0[1,1,3,3]
-; X86-NEXT: vpextrw $0, %xmm2, %ebx
-; X86-NEXT: vmovshdup {{.*#+}} xmm2 = xmm1[1,1,3,3]
-; X86-NEXT: vpextrw $0, %xmm2, %ebp
-; X86-NEXT: vpextrw $0, %xmm0, %eax
-; X86-NEXT: vpsrld $16, %xmm0, %xmm0
-; X86-NEXT: vpsrld $16, %xmm1, %xmm2
-; X86-NEXT: vpextrw $0, %xmm2, %ecx
-; X86-NEXT: shll $16, %ecx
-; X86-NEXT: vmovd %ecx, %xmm2
-; X86-NEXT: vpextrw $0, %xmm0, %ecx
-; X86-NEXT: shll $16, %ecx
-; X86-NEXT: vmovd %ecx, %xmm0
-; X86-NEXT: vpextrw $0, %xmm1, %ecx
-; X86-NEXT: vmaxss %xmm2, %xmm0, %xmm1
-; X86-NEXT: vbroadcastss {{.*#+}} xmm4 = [NaN,NaN,NaN,NaN]
-; X86-NEXT: vorps %xmm4, %xmm0, %xmm3
-; X86-NEXT: vandps %xmm1, %xmm3, %xmm1
-; X86-NEXT: vcmpunordss %xmm0, %xmm0, %xmm3
+; X86-NEXT: subl $84, %esp
+; X86-NEXT: vpslld $16, %xmm0, %xmm4
+; X86-NEXT: vmovdqa %xmm0, %xmm5
+; X86-NEXT: vmovdqu %xmm0, {{[-0-9]+}}(%e{{[sb]}}p) # 16-byte Spill
+; X86-NEXT: vbroadcastss {{.*#+}} xmm6 = [NaN,NaN,NaN,NaN]
+; X86-NEXT: vpor %xmm6, %xmm4, %xmm0
+; X86-NEXT: vpslld $16, %xmm1, %xmm2
+; X86-NEXT: vmovdqa %xmm1, %xmm7
+; X86-NEXT: vmovdqu %xmm1, {{[-0-9]+}}(%e{{[sb]}}p) # 16-byte Spill
+; X86-NEXT: vmaxss %xmm2, %xmm4, %xmm3
+; X86-NEXT: vpand %xmm3, %xmm0, %xmm1
+; X86-NEXT: vcmpunordss %xmm4, %xmm4, %xmm3
; X86-NEXT: vblendvps %xmm3, %xmm2, %xmm1, %xmm1
; X86-NEXT: vcmpunordss %xmm2, %xmm2, %xmm2
-; X86-NEXT: vblendvps %xmm2, %xmm0, %xmm1, %xmm0
+; X86-NEXT: vblendvps %xmm2, %xmm4, %xmm1, %xmm0
; X86-NEXT: vmovss %xmm0, (%esp)
-; X86-NEXT: shll $16, %ecx
-; X86-NEXT: vmovd %ecx, %xmm0
-; X86-NEXT: shll $16, %eax
-; X86-NEXT: vmovd %eax, %xmm1
-; X86-NEXT: vmaxss %xmm0, %xmm1, %xmm2
-; X86-NEXT: vorps %xmm4, %xmm1, %xmm3
-; X86-NEXT: vandps %xmm2, %xmm3, %xmm2
-; X86-NEXT: vcmpunordss %xmm1, %xmm1, %xmm3
-; X86-NEXT: vblendvps %xmm3, %xmm0, %xmm2, %xmm2
-; X86-NEXT: vcmpunordss %xmm0, %xmm0, %xmm0
-; X86-NEXT: vblendvps %xmm0, %xmm1, %xmm2, %xmm0
+; X86-NEXT: vxorps %xmm0, %xmm0, %xmm0
+; X86-NEXT: vpblendw {{.*#+}} xmm1 = xmm0[0],xmm7[1],xmm0[2],xmm7[3],xmm0[4],xmm7[5],xmm0[6],xmm7[7]
+; X86-NEXT: vpblendw {{.*#+}} xmm0 = xmm0[0],xmm5[1],xmm0[2],xmm5[3],xmm0[4],xmm5[5],xmm0[6],xmm5[7]
+; X86-NEXT: vmaxss %xmm1, %xmm0, %xmm2
+; X86-NEXT: vpor %xmm6, %xmm0, %xmm3
+; X86-NEXT: vpand %xmm2, %xmm3, %xmm2
+; X86-NEXT: vcmpunordss %xmm0, %xmm0, %xmm3
+; X86-NEXT: vblendvps %xmm3, %xmm1, %xmm2, %xmm2
+; X86-NEXT: vcmpunordss %xmm1, %xmm1, %xmm1
+; X86-NEXT: vblendvps %xmm1, %xmm0, %xmm2, %xmm0
; X86-NEXT: vmovups %xmm0, {{[-0-9]+}}(%e{{[sb]}}p) # 16-byte Spill
; X86-NEXT: calll __truncsfbf2
; X86-NEXT: vmovups %xmm0, {{[-0-9]+}}(%e{{[sb]}}p) # 16-byte Spill
; X86-NEXT: vmovups {{[-0-9]+}}(%e{{[sb]}}p), %xmm0 # 16-byte Reload
; X86-NEXT: vmovss %xmm0, (%esp)
-; X86-NEXT: shll $16, %ebp
-; X86-NEXT: vmovd %ebp, %xmm0
-; X86-NEXT: shll $16, %ebx
-; X86-NEXT: vmovd %ebx, %xmm1
-; X86-NEXT: vmaxss %xmm0, %xmm1, %xmm2
-; X86-NEXT: vorps {{\.?LCPI[0-9]+_[0-9]+}}, %xmm1, %xmm3
+; X86-NEXT: vbroadcastss {{.*#+}} xmm0 = [128,128,4,5,128,128,4,5,128,128,4,5,128,128,4,5]
+; X86-NEXT: vmovdqu {{[-0-9]+}}(%e{{[sb]}}p), %xmm1 # 16-byte Reload
+; X86-NEXT: vpshufb %xmm0, %xmm1, %xmm1
+; X86-NEXT: vmovdqu {{[-0-9]+}}(%e{{[sb]}}p), %xmm2 # 16-byte Reload
+; X86-NEXT: vpshufb %xmm0, %xmm2, %xmm0
+; X86-NEXT: vmaxss %xmm1, %xmm0, %xmm2
+; X86-NEXT: vorps {{\.?LCPI[0-9]+_[0-9]+}}, %xmm0, %xmm3
; X86-NEXT: vandps %xmm2, %xmm3, %xmm2
-; X86-NEXT: vcmpunordss %xmm1, %xmm1, %xmm3
-; X86-NEXT: vblendvps %xmm3, %xmm0, %xmm2, %xmm2
-; X86-NEXT: vcmpunordss %xmm0, %xmm0, %xmm0
-; X86-NEXT: vblendvps %xmm0, %xmm1, %xmm2, %xmm0
+; X86-NEXT: vcmpunordss %xmm0, %xmm0, %xmm3
+; X86-NEXT: vblendvps %xmm3, %xmm1, %xmm2, %xmm2
+; X86-NEXT: vcmpunordss %xmm1, %xmm1, %xmm1
+; X86-NEXT: vblendvps %xmm1, %xmm0, %xmm2, %xmm0
; X86-NEXT: vmovups %xmm0, {{[-0-9]+}}(%e{{[sb]}}p) # 16-byte Spill
; X86-NEXT: calll __truncsfbf2
; X86-NEXT: vmovups %xmm0, {{[-0-9]+}}(%e{{[sb]}}p) # 16-byte Spill
; X86-NEXT: vmovups {{[-0-9]+}}(%e{{[sb]}}p), %xmm0 # 16-byte Reload
; X86-NEXT: vmovss %xmm0, (%esp)
-; X86-NEXT: shll $16, %edi
-; X86-NEXT: vmovd %edi, %xmm0
-; X86-NEXT: shll $16, %esi
-; X86-NEXT: vmovd %esi, %xmm1
-; X86-NEXT: vmaxss %xmm0, %xmm1, %xmm2
-; X86-NEXT: vorps {{\.?LCPI[0-9]+_[0-9]+}}, %xmm1, %xmm3
+; X86-NEXT: vbroadcastss {{.*#+}} xmm0 = [128,128,6,7,128,128,6,7,128,128,6,7,128,128,6,7]
+; X86-NEXT: vmovdqu {{[-0-9]+}}(%e{{[sb]}}p), %xmm1 # 16-byte Reload
+; X86-NEXT: vpshufb %xmm0, %xmm1, %xmm1
+; X86-NEXT: vmovdqu {{[-0-9]+}}(%e{{[sb]}}p), %xmm2 # 16-byte Reload
+; X86-NEXT: vpshufb %xmm0, %xmm2, %xmm0
+; X86-NEXT: vmaxss %xmm1, %xmm0, %xmm2
+; X86-NEXT: vorps {{\.?LCPI[0-9]+_[0-9]+}}, %xmm0, %xmm3
; X86-NEXT: vandps %xmm2, %xmm3, %xmm2
-; X86-NEXT: vcmpunordss %xmm1, %xmm1, %xmm3
-; X86-NEXT: vblendvps %xmm3, %xmm0, %xmm2, %xmm2
-; X86-NEXT: vcmpunordss %xmm0, %xmm0, %xmm0
-; X86-NEXT: vblendvps %xmm0, %xmm1, %xmm2, %xmm0
+; X86-NEXT: vcmpunordss %xmm0, %xmm0, %xmm3
+; X86-NEXT: vblendvps %xmm3, %xmm1, %xmm2, %xmm2
+; X86-NEXT: vcmpunordss %xmm1, %xmm1, %xmm1
+; X86-NEXT: vblendvps %xmm1, %xmm0, %xmm2, %xmm0
; X86-NEXT: vmovups %xmm0, {{[-0-9]+}}(%e{{[sb]}}p) # 16-byte Spill
; X86-NEXT: calll __truncsfbf2
; X86-NEXT: vmovups %xmm0, {{[-0-9]+}}(%e{{[sb]}}p) # 16-byte Spill
@@ -3481,11 +3358,7 @@ define <4 x bfloat> @test_fmaximumnum_v4bf16(<4 x bfloat> %x, <4 x bfloat> %y) n
; X86-NEXT: vpunpcklwd {{[-0-9]+}}(%e{{[sb]}}p), %xmm1, %xmm1 # 16-byte Folded Reload
; X86-NEXT: # xmm1 = xmm1[0],mem[0],xmm1[1],mem[1],xmm1[2],mem[2],xmm1[3],mem[3]
; X86-NEXT: vinsertps {{.*#+}} xmm0 = xmm1[0],xmm0[0],zero,zero
-; X86-NEXT: addl $68, %esp
-; X86-NEXT: popl %esi
-; X86-NEXT: popl %edi
-; X86-NEXT: popl %ebx
-; X86-NEXT: popl %ebp
+; X86-NEXT: addl $84, %esp
; X86-NEXT: retl
%r = call <4 x bfloat> @llvm.maximumnum.v4bf16(<4 x bfloat> %x, <4 x bfloat> %y)
ret <4 x bfloat> %r
@@ -3495,28 +3368,23 @@ define bfloat @test_fminimumnum_bf16(bfloat %x, bfloat %y) nounwind {
; SSE2-LABEL: test_fminimumnum_bf16:
; SSE2: # %bb.0:
; SSE2-NEXT: pushq %rax
-; SSE2-NEXT: pextrw $0, %xmm0, %eax
-; SSE2-NEXT: pextrw $0, %xmm1, %ecx
-; SSE2-NEXT: shll $16, %ecx
-; SSE2-NEXT: movd %ecx, %xmm0
-; SSE2-NEXT: shll $16, %eax
-; SSE2-NEXT: movd %eax, %xmm1
-; SSE2-NEXT: movdqa %xmm1, %xmm2
-; SSE2-NEXT: cmpunordss %xmm1, %xmm2
+; SSE2-NEXT: pslld $16, %xmm1
+; SSE2-NEXT: pslld $16, %xmm0
+; SSE2-NEXT: movdqa %xmm0, %xmm2
+; SSE2-NEXT: cmpunordss %xmm0, %xmm2
; SSE2-NEXT: movaps %xmm2, %xmm3
-; SSE2-NEXT: andps %xmm0, %xmm3
-; SSE2-NEXT: movaps %xmm1, %xmm4
-; SSE2-NEXT: minss %xmm0, %xmm4
+; SSE2-NEXT: andps %xmm1, %xmm3
+; SSE2-NEXT: movaps %xmm0, %xmm4
+; SSE2-NEXT: minss %xmm1, %xmm4
; SSE2-NEXT: movaps {{.*#+}} xmm5 = [-0.0E+0,-0.0E+0,-0.0E+0,-0.0E+0]
-; SSE2-NEXT: andps %xmm1, %xmm5
+; SSE2-NEXT: andps %xmm0, %xmm5
; SSE2-NEXT: orps %xmm4, %xmm5
; SSE2-NEXT: andnps %xmm5, %xmm2
; SSE2-NEXT: orps %xmm3, %xmm2
-; SSE2-NEXT: cmpunordss %xmm0, %xmm0
-; SSE2-NEXT: movaps %xmm0, %xmm3
-; SSE2-NEXT: andnps %xmm2, %xmm3
+; SSE2-NEXT: cmpunordss %xmm1, %xmm1
; SSE2-NEXT: andps %xmm1, %xmm0
-; SSE2-NEXT: orps %xmm3, %xmm0
+; SSE2-NEXT: andnps %xmm2, %xmm1
+; SSE2-NEXT: orps %xmm1, %xmm0
; SSE2-NEXT: callq __truncsfbf2 at PLT
; SSE2-NEXT: popq %rax
; SSE2-NEXT: retq
@@ -3524,19 +3392,15 @@ define bfloat @test_fminimumnum_bf16(bfloat %x, bfloat %y) nounwind {
; AVX1-LABEL: test_fminimumnum_bf16:
; AVX1: # %bb.0:
; AVX1-NEXT: pushq %rax
-; AVX1-NEXT: vpextrw $0, %xmm0, %eax
-; AVX1-NEXT: vpextrw $0, %xmm1, %ecx
-; AVX1-NEXT: shll $16, %ecx
-; AVX1-NEXT: vmovd %ecx, %xmm0
-; AVX1-NEXT: shll $16, %eax
-; AVX1-NEXT: vmovd %eax, %xmm1
-; AVX1-NEXT: vminss %xmm0, %xmm1, %xmm2
-; AVX1-NEXT: vandps {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm1, %xmm3
+; AVX1-NEXT: vpslld $16, %xmm1, %xmm1
+; AVX1-NEXT: vpslld $16, %xmm0, %xmm0
+; AVX1-NEXT: vminss %xmm1, %xmm0, %xmm2
+; AVX1-NEXT: vandps {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm0, %xmm3
; AVX1-NEXT: vorps %xmm2, %xmm3, %xmm2
-; AVX1-NEXT: vcmpunordss %xmm1, %xmm1, %xmm3
-; AVX1-NEXT: vblendvps %xmm3, %xmm0, %xmm2, %xmm2
-; AVX1-NEXT: vcmpunordss %xmm0, %xmm0, %xmm0
-; AVX1-NEXT: vblendvps %xmm0, %xmm1, %xmm2, %xmm0
+; AVX1-NEXT: vcmpunordss %xmm0, %xmm0, %xmm3
+; AVX1-NEXT: vblendvps %xmm3, %xmm1, %xmm2, %xmm2
+; AVX1-NEXT: vcmpunordss %xmm1, %xmm1, %xmm1
+; AVX1-NEXT: vblendvps %xmm1, %xmm0, %xmm2, %xmm0
; AVX1-NEXT: callq __truncsfbf2 at PLT
; AVX1-NEXT: popq %rax
; AVX1-NEXT: retq
@@ -3544,16 +3408,12 @@ define bfloat @test_fminimumnum_bf16(bfloat %x, bfloat %y) nounwind {
; AVX512F-LABEL: test_fminimumnum_bf16:
; AVX512F: # %bb.0:
; AVX512F-NEXT: pushq %rax
-; AVX512F-NEXT: vpextrw $0, %xmm0, %eax
-; AVX512F-NEXT: vpextrw $0, %xmm1, %ecx
-; AVX512F-NEXT: shll $16, %ecx
-; AVX512F-NEXT: vmovd %ecx, %xmm1
-; AVX512F-NEXT: shll $16, %eax
-; AVX512F-NEXT: vmovd %eax, %xmm2
-; AVX512F-NEXT: vminss %xmm1, %xmm2, %xmm0
-; AVX512F-NEXT: vbroadcastss {{.*#+}} xmm3 = [-0.0E+0,-0.0E+0,-0.0E+0,-0.0E+0]
-; AVX512F-NEXT: vandps %xmm3, %xmm2, %xmm3
-; AVX512F-NEXT: vorps %xmm0, %xmm3, %xmm0
+; AVX512F-NEXT: vpslld $16, %xmm0, %xmm2
+; AVX512F-NEXT: vpbroadcastd {{.*#+}} xmm0 = [-0.0E+0,-0.0E+0,-0.0E+0,-0.0E+0]
+; AVX512F-NEXT: vpand %xmm0, %xmm2, %xmm0
+; AVX512F-NEXT: vpslld $16, %xmm1, %xmm1
+; AVX512F-NEXT: vminss %xmm1, %xmm2, %xmm3
+; AVX512F-NEXT: vpor %xmm3, %xmm0, %xmm0
; AVX512F-NEXT: vcmpunordss %xmm2, %xmm2, %k1
; AVX512F-NEXT: vmovss %xmm1, %xmm0, %xmm0 {%k1}
; AVX512F-NEXT: vcmpunordss %xmm1, %xmm1, %k1
@@ -3565,16 +3425,12 @@ define bfloat @test_fminimumnum_bf16(bfloat %x, bfloat %y) nounwind {
; AVX512DQ-LABEL: test_fminimumnum_bf16:
; AVX512DQ: # %bb.0:
; AVX512DQ-NEXT: pushq %rax
-; AVX512DQ-NEXT: vpextrw $0, %xmm0, %eax
-; AVX512DQ-NEXT: vpextrw $0, %xmm1, %ecx
-; AVX512DQ-NEXT: shll $16, %ecx
-; AVX512DQ-NEXT: vmovd %ecx, %xmm1
-; AVX512DQ-NEXT: shll $16, %eax
-; AVX512DQ-NEXT: vmovd %eax, %xmm2
-; AVX512DQ-NEXT: vminss %xmm1, %xmm2, %xmm0
-; AVX512DQ-NEXT: vbroadcastss {{.*#+}} xmm3 = [-0.0E+0,-0.0E+0,-0.0E+0,-0.0E+0]
-; AVX512DQ-NEXT: vandps %xmm3, %xmm2, %xmm3
-; AVX512DQ-NEXT: vorps %xmm0, %xmm3, %xmm0
+; AVX512DQ-NEXT: vpslld $16, %xmm0, %xmm2
+; AVX512DQ-NEXT: vpbroadcastd {{.*#+}} xmm0 = [-0.0E+0,-0.0E+0,-0.0E+0,-0.0E+0]
+; AVX512DQ-NEXT: vpand %xmm0, %xmm2, %xmm0
+; AVX512DQ-NEXT: vpslld $16, %xmm1, %xmm1
+; AVX512DQ-NEXT: vminss %xmm1, %xmm2, %xmm3
+; AVX512DQ-NEXT: vpor %xmm3, %xmm0, %xmm0
; AVX512DQ-NEXT: vcmpunordss %xmm2, %xmm2, %k1
; AVX512DQ-NEXT: vmovss %xmm1, %xmm0, %xmm0 {%k1}
; AVX512DQ-NEXT: vcmpunordss %xmm1, %xmm1, %k1
@@ -3585,30 +3441,22 @@ define bfloat @test_fminimumnum_bf16(bfloat %x, bfloat %y) nounwind {
;
; AVX512BF16-LABEL: test_fminimumnum_bf16:
; AVX512BF16: # %bb.0:
-; AVX512BF16-NEXT: vpextrw $0, %xmm0, %eax
-; AVX512BF16-NEXT: vpextrw $0, %xmm1, %ecx
-; AVX512BF16-NEXT: shll $16, %ecx
-; AVX512BF16-NEXT: vmovd %ecx, %xmm0
-; AVX512BF16-NEXT: shll $16, %eax
-; AVX512BF16-NEXT: vmovd %eax, %xmm1
-; AVX512BF16-NEXT: vminss %xmm0, %xmm1, %xmm2
-; AVX512BF16-NEXT: vpternlogd {{.*#+}} xmm2 = xmm2 | (xmm1 & m32bcst)
-; AVX512BF16-NEXT: vcmpunordss %xmm1, %xmm1, %k1
-; AVX512BF16-NEXT: vmovss %xmm0, %xmm2, %xmm2 {%k1}
+; AVX512BF16-NEXT: vpslld $16, %xmm1, %xmm1
+; AVX512BF16-NEXT: vpslld $16, %xmm0, %xmm0
+; AVX512BF16-NEXT: vminss %xmm1, %xmm0, %xmm2
+; AVX512BF16-NEXT: vpternlogd {{.*#+}} xmm2 = xmm2 | (xmm0 & m32bcst)
; AVX512BF16-NEXT: vcmpunordss %xmm0, %xmm0, %k1
; AVX512BF16-NEXT: vmovss %xmm1, %xmm2, %xmm2 {%k1}
+; AVX512BF16-NEXT: vcmpunordss %xmm1, %xmm1, %k1
+; AVX512BF16-NEXT: vmovss %xmm0, %xmm2, %xmm2 {%k1}
; AVX512BF16-NEXT: vcvtneps2bf16 %xmm2, %xmm0
; AVX512BF16-NEXT: retq
;
; AVX10_2-LABEL: test_fminimumnum_bf16:
; AVX10_2: # %bb.0:
-; AVX10_2-NEXT: vmovw %xmm0, %eax
-; AVX10_2-NEXT: vmovw %xmm1, %ecx
-; AVX10_2-NEXT: shll $16, %ecx
-; AVX10_2-NEXT: vmovd %ecx, %xmm0
-; AVX10_2-NEXT: shll $16, %eax
-; AVX10_2-NEXT: vmovd %eax, %xmm1
-; AVX10_2-NEXT: vminmaxss $20, %xmm0, %xmm1
+; AVX10_2-NEXT: vpslld $16, %xmm1, %xmm1
+; AVX10_2-NEXT: vpslld $16, %xmm0, %xmm0
+; AVX10_2-NEXT: vminmaxss $20, %xmm1, %xmm0
; AVX10_2-NEXT: vcvtneps2bf16 %xmm0, %xmm0
; AVX10_2-NEXT: retq
;
diff --git a/llvm/test/CodeGen/X86/llvm.frexp.ll b/llvm/test/CodeGen/X86/llvm.frexp.ll
index 3410fd5169c9a..fb778add339fb 100644
--- a/llvm/test/CodeGen/X86/llvm.frexp.ll
+++ b/llvm/test/CodeGen/X86/llvm.frexp.ll
@@ -122,9 +122,7 @@ define { bfloat, i32 } @test_frexp_bf16_i32(bfloat %a) nounwind {
; X64-LABEL: test_frexp_bf16_i32:
; X64: # %bb.0:
; X64-NEXT: pushq %rax
-; X64-NEXT: pextrw $0, %xmm0, %eax
-; X64-NEXT: shll $16, %eax
-; X64-NEXT: movd %eax, %xmm0
+; X64-NEXT: pslld $16, %xmm0
; X64-NEXT: leaq {{[0-9]+}}(%rsp), %rdi
; X64-NEXT: callq frexpf at PLT
; X64-NEXT: callq __truncsfbf2 at PLT
More information about the llvm-commits
mailing list